From 3d1b96670bf1bfb13e953e7111d486d958065ec0 Mon Sep 17 00:00:00 2001 From: wangwei990215 Date: Thu, 17 Apr 2025 10:29:23 +0800 Subject: [PATCH] =?UTF-8?q?=E6=9B=B4=E6=96=B0diffusers=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=A4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sd2.1/pytorch_lora_weights.safetensors | Bin 3357296 -> 0 bytes .../examples/text_to_image/run-4.sh | 10 +- .../examples/text_to_image/run.sh | 10 +- .../text_to_image/train_text_to_image_lora.py | 2 +- .../examples/textual_inversion/README.md | 150 + .../examples/textual_inversion/README_sdxl.md | 26 + .../textual_inversion/requirements.txt | 6 + .../textual_inversion/requirements_flax.txt | 8 + .../test_textual_inversion.py | 152 + .../test_textual_inversion_sdxl.py | 152 + .../textual_inversion/textual_inversion.py | 1012 +++++ .../textual_inversion_flax.py | 681 +++ .../textual_inversion_sdxl.py | 1070 +++++ .../unconditional_image_generation/README.md | 163 + .../requirements.txt | 3 + .../test_unconditional.py | 130 + .../train_unconditional.py | 704 ++++ .../wuerstchen/text_to_image/README.md | 93 + .../wuerstchen/text_to_image/__init__.py | 0 .../modeling_efficient_net_encoder.py | 23 + .../wuerstchen/text_to_image/requirements.txt | 7 + .../train_text_to_image_lora_prior.py | 945 +++++ .../train_text_to_image_prior.py | 932 ++++ diffusers-0.27.0/pyproject.toml | 27 + diffusers-0.27.0/scripts/__init__.py | 0 .../change_naming_configs_and_checkpoints.py | 113 + .../scripts/conversion_ldm_uncond.py | 56 + diffusers-0.27.0/scripts/convert_amused.py | 523 +++ ...rt_animatediff_motion_lora_to_diffusers.py | 51 + ..._animatediff_motion_module_to_diffusers.py | 54 + .../convert_asymmetric_vqgan_to_diffusers.py | 184 + .../convert_blipdiffusion_to_diffusers.py | 343 ++ .../scripts/convert_consistency_decoder.py | 1128 +++++ .../convert_consistency_to_diffusers.py | 315 ++ .../convert_dance_diffusion_to_diffusers.py | 345 ++ ...t_ddpm_original_checkpoint_to_diffusers.py | 431 ++ .../convert_diffusers_sdxl_lora_to_webui.py | 56 + .../convert_diffusers_to_original_sdxl.py | 350 ++ ..._diffusers_to_original_stable_diffusion.py | 353 ++ .../scripts/convert_dit_to_diffusers.py | 162 + .../scripts/convert_gligen_to_diffusers.py | 581 +++ .../scripts/convert_i2vgen_to_diffusers.py | 510 +++ diffusers-0.27.0/scripts/convert_if.py | 1250 ++++++ .../convert_k_upscaler_to_diffusers.py | 297 ++ ...convert_kakao_brain_unclip_to_diffusers.py | 1159 +++++ .../scripts/convert_kandinsky3_unet.py | 98 + .../scripts/convert_kandinsky_to_diffusers.py | 1411 +++++++ ...rt_ldm_original_checkpoint_to_diffusers.py | 359 ++ .../convert_lora_safetensor_to_diffusers.py | 128 + .../convert_models_diffuser_to_diffusers.py | 100 + .../convert_ms_text_to_video_to_diffusers.py | 428 ++ .../convert_music_spectrogram_to_diffusers.py | 213 + ...ncsnpp_original_checkpoint_to_diffusers.py | 185 + ...convert_original_audioldm2_to_diffusers.py | 1135 +++++ .../convert_original_audioldm_to_diffusers.py | 1042 +++++ ...onvert_original_controlnet_to_diffusers.py | 109 + .../convert_original_musicldm_to_diffusers.py | 1056 +++++ ..._original_stable_diffusion_to_diffusers.py | 188 + .../scripts/convert_original_t2i_adapter.py | 250 ++ .../convert_pixart_alpha_to_diffusers.py | 198 + .../scripts/convert_shap_e_to_diffusers.py | 1080 +++++ .../scripts/convert_stable_cascade.py | 218 + .../scripts/convert_stable_cascade_lite.py | 226 + ...ert_stable_diffusion_checkpoint_to_onnx.py | 265 ++ ...ert_stable_diffusion_controlnet_to_onnx.py | 505 +++ ...stable_diffusion_controlnet_to_tensorrt.py | 121 + .../scripts/convert_svd_to_diffusers.py | 730 ++++ .../convert_tiny_autoencoder_to_diffusers.py | 71 + ...nvert_unclip_txt2img_to_image_variation.py | 41 + .../convert_unidiffuser_to_diffusers.py | 786 ++++ .../scripts/convert_vae_diff_to_onnx.py | 122 + .../scripts/convert_vae_pt_to_diffusers.py | 159 + ...onvert_versatile_diffusion_to_diffusers.py | 791 ++++ .../convert_vq_diffusion_to_diffusers.py | 916 ++++ .../scripts/convert_wuerstchen.py | 115 + .../scripts/convert_zero123_to_diffusers.py | 806 ++++ diffusers-0.27.0/scripts/generate_logits.py | 127 + diffusers-0.27.0/scripts/log_reports.py | 139 + diffusers-0.27.0/src/diffusers/__init__.py | 787 ++++ .../src/diffusers/commands/__init__.py | 27 + .../src/diffusers/commands/diffusers_cli.py | 43 + .../src/diffusers/commands/env.py | 84 + .../diffusers/commands/fp16_safetensors.py | 132 + .../src/diffusers/configuration_utils.py | 703 ++++ .../diffusers/dependency_versions_check.py | 34 + .../diffusers/dependency_versions_table.py | 45 + .../src/diffusers/experimental/README.md | 5 + .../src/diffusers/experimental/__init__.py | 1 + .../src/diffusers/experimental/rl/__init__.py | 1 + .../experimental/rl/value_guided_sampling.py | 153 + .../src/diffusers/image_processor.py | 990 +++++ .../src/diffusers/loaders/__init__.py | 88 + .../src/diffusers/loaders/autoencoder.py | 146 + .../src/diffusers/loaders/controlnet.py | 136 + .../src/diffusers/loaders/ip_adapter.py | 281 ++ .../src/diffusers/loaders/lora.py | 1349 ++++++ .../loaders/lora_conversion_utils.py | 284 ++ .../src/diffusers/loaders/peft.py | 186 + .../src/diffusers/loaders/single_file.py | 318 ++ .../diffusers/loaders/single_file_utils.py | 1617 +++++++ .../diffusers/loaders/textual_inversion.py | 562 +++ .../src/diffusers/loaders/unet.py | 1003 +++++ .../src/diffusers/loaders/utils.py | 59 + .../src/diffusers/models/README.md | 3 + .../src/diffusers/models/__init__.py | 103 + .../src/diffusers/models/activations.py | 123 + .../src/diffusers/models/adapter.py | 584 +++ .../src/diffusers/models/attention.py | 665 +++ .../src/diffusers/models/attention_flax.py | 494 +++ .../diffusers/models/attention_processor.py | 2480 +++++++++++ .../diffusers/models/autoencoders/__init__.py | 5 + .../autoencoders/autoencoder_asym_kl.py | 186 + .../models/autoencoders/autoencoder_kl.py | 489 +++ .../autoencoder_kl_temporal_decoder.py | 399 ++ .../models/autoencoders/autoencoder_tiny.py | 347 ++ .../autoencoders/consistency_decoder_vae.py | 435 ++ .../src/diffusers/models/autoencoders/vae.py | 983 +++++ .../src/diffusers/models/controlnet.py | 868 ++++ .../src/diffusers/models/controlnet_flax.py | 395 ++ .../src/diffusers/models/downsampling.py | 334 ++ .../diffusers/models/dual_transformer_2d.py | 20 + .../src/diffusers/models/embeddings.py | 914 ++++ .../src/diffusers/models/embeddings_flax.py | 97 + diffusers-0.27.0/src/diffusers/models/lora.py | 457 ++ .../models/modeling_flax_pytorch_utils.py | 134 + .../diffusers/models/modeling_flax_utils.py | 566 +++ .../src/diffusers/models/modeling_outputs.py | 17 + .../models/modeling_pytorch_flax_utils.py | 161 + .../src/diffusers/models/modeling_utils.py | 1021 +++++ .../src/diffusers/models/normalization.py | 254 ++ .../src/diffusers/models/prior_transformer.py | 12 + .../src/diffusers/models/resnet.py | 802 ++++ .../src/diffusers/models/resnet_flax.py | 124 + .../diffusers/models/t5_film_transformer.py | 70 + .../src/diffusers/models/transformer_2d.py | 25 + .../diffusers/models/transformer_temporal.py | 34 + .../diffusers/models/transformers/__init__.py | 9 + .../transformers/dual_transformer_2d.py | 155 + .../models/transformers/prior_transformer.py | 380 ++ .../transformers/t5_film_transformer.py | 438 ++ .../models/transformers/transformer_2d.py | 456 ++ .../transformers/transformer_temporal.py | 379 ++ .../src/diffusers/models/unet_1d.py | 26 + .../src/diffusers/models/unet_1d_blocks.py | 203 + .../src/diffusers/models/unet_2d.py | 27 + .../src/diffusers/models/unet_2d_blocks.py | 375 ++ .../src/diffusers/models/unet_2d_condition.py | 25 + .../src/diffusers/models/unets/__init__.py | 18 + .../src/diffusers/models/unets/unet_1d.py | 255 ++ .../diffusers/models/unets/unet_1d_blocks.py | 702 ++++ .../src/diffusers/models/unets/unet_2d.py | 346 ++ .../diffusers/models/unets/unet_2d_blocks.py | 3731 +++++++++++++++++ .../models/unets/unet_2d_blocks_flax.py | 400 ++ .../models/unets/unet_2d_condition.py | 1315 ++++++ .../models/unets/unet_2d_condition_flax.py | 453 ++ .../diffusers/models/unets/unet_3d_blocks.py | 2405 +++++++++++ .../models/unets/unet_3d_condition.py | 753 ++++ .../diffusers/models/unets/unet_i2vgen_xl.py | 724 ++++ .../diffusers/models/unets/unet_kandinsky3.py | 535 +++ .../models/unets/unet_motion_model.py | 948 +++++ .../unets/unet_spatio_temporal_condition.py | 489 +++ .../models/unets/unet_stable_cascade.py | 610 +++ .../src/diffusers/models/unets/uvit_2d.py | 470 +++ .../src/diffusers/models/upsampling.py | 448 ++ .../src/diffusers/models/vae_flax.py | 876 ++++ .../src/diffusers/models/vq_model.py | 181 + .../src/diffusers/optimization.py | 361 ++ .../src/diffusers/pipelines/README.md | 171 + .../src/diffusers/pipelines/__init__.py | 581 +++ .../diffusers/pipelines/amused/__init__.py | 62 + .../pipelines/amused/pipeline_amused.py | 328 ++ .../amused/pipeline_amused_img2img.py | 347 ++ .../amused/pipeline_amused_inpaint.py | 378 ++ .../pipelines/animatediff/__init__.py | 49 + .../animatediff/pipeline_animatediff.py | 847 ++++ .../pipeline_animatediff_video2video.py | 997 +++++ .../pipelines/animatediff/pipeline_output.py | 23 + .../diffusers/pipelines/audioldm/__init__.py | 51 + .../pipelines/audioldm/pipeline_audioldm.py | 546 +++ .../diffusers/pipelines/audioldm2/__init__.py | 50 + .../pipelines/audioldm2/modeling_audioldm2.py | 1511 +++++++ .../pipelines/audioldm2/pipeline_audioldm2.py | 980 +++++ .../src/diffusers/pipelines/auto_pipeline.py | 987 +++++ .../pipelines/blip_diffusion/__init__.py | 20 + .../blip_diffusion/blip_image_processing.py | 318 ++ .../blip_diffusion/modeling_blip2.py | 642 +++ .../blip_diffusion/modeling_ctx_clip.py | 223 + .../blip_diffusion/pipeline_blip_diffusion.py | 348 ++ .../pipelines/consistency_models/__init__.py | 24 + .../pipeline_consistency_models.py | 275 ++ .../pipelines/controlnet/__init__.py | 80 + .../pipelines/controlnet/multicontrolnet.py | 187 + .../controlnet/pipeline_controlnet.py | 1318 ++++++ .../pipeline_controlnet_blip_diffusion.py | 413 ++ .../controlnet/pipeline_controlnet_img2img.py | 1310 ++++++ .../controlnet/pipeline_controlnet_inpaint.py | 1620 +++++++ .../pipeline_controlnet_inpaint_sd_xl.py | 1818 ++++++++ .../controlnet/pipeline_controlnet_sd_xl.py | 1499 +++++++ .../pipeline_controlnet_sd_xl_img2img.py | 1626 +++++++ .../controlnet/pipeline_flax_controlnet.py | 532 +++ .../pipelines/dance_diffusion/__init__.py | 18 + .../pipeline_dance_diffusion.py | 156 + .../src/diffusers/pipelines/ddim/__init__.py | 18 + .../diffusers/pipelines/ddim/pipeline_ddim.py | 154 + .../src/diffusers/pipelines/ddpm/__init__.py | 22 + .../diffusers/pipelines/ddpm/pipeline_ddpm.py | 127 + .../pipelines/deepfloyd_if/__init__.py | 85 + .../pipelines/deepfloyd_if/pipeline_if.py | 788 ++++ .../deepfloyd_if/pipeline_if_img2img.py | 910 ++++ .../pipeline_if_img2img_superresolution.py | 1029 +++++ .../deepfloyd_if/pipeline_if_inpainting.py | 1030 +++++ .../pipeline_if_inpainting_superresolution.py | 1137 +++++ .../pipeline_if_superresolution.py | 885 ++++ .../pipelines/deepfloyd_if/pipeline_output.py | 28 + .../pipelines/deepfloyd_if/safety_checker.py | 59 + .../pipelines/deepfloyd_if/timesteps.py | 579 +++ .../pipelines/deepfloyd_if/watermark.py | 46 + .../diffusers/pipelines/deprecated/README.md | 3 + .../pipelines/deprecated/__init__.py | 153 + .../deprecated/alt_diffusion/__init__.py | 53 + .../alt_diffusion/modeling_roberta_series.py | 124 + .../alt_diffusion/pipeline_alt_diffusion.py | 946 +++++ .../pipeline_alt_diffusion_img2img.py | 1018 +++++ .../alt_diffusion/pipeline_output.py | 28 + .../deprecated/audio_diffusion/__init__.py | 23 + .../deprecated/audio_diffusion/mel.py | 179 + .../pipeline_audio_diffusion.py | 329 ++ .../latent_diffusion_uncond/__init__.py | 18 + .../pipeline_latent_diffusion_uncond.py | 130 + .../pipelines/deprecated/pndm/__init__.py | 18 + .../deprecated/pndm/pipeline_pndm.py | 121 + .../pipelines/deprecated/repaint/__init__.py | 19 + .../deprecated/repaint/pipeline_repaint.py | 230 + .../deprecated/score_sde_ve/__init__.py | 19 + .../score_sde_ve/pipeline_score_sde_ve.py | 109 + .../spectrogram_diffusion/__init__.py | 75 + .../continuous_encoder.py | 92 + .../spectrogram_diffusion/midi_utils.py | 667 +++ .../spectrogram_diffusion/notes_encoder.py | 86 + .../pipeline_spectrogram_diffusion.py | 269 ++ .../stable_diffusion_variants/__init__.py | 55 + .../pipeline_cycle_diffusion.py | 948 +++++ ...ne_onnx_stable_diffusion_inpaint_legacy.py | 542 +++ ...ipeline_stable_diffusion_inpaint_legacy.py | 786 ++++ ...pipeline_stable_diffusion_model_editing.py | 824 ++++ .../pipeline_stable_diffusion_paradigms.py | 786 ++++ .../pipeline_stable_diffusion_pix2pix_zero.py | 1304 ++++++ .../stochastic_karras_ve/__init__.py | 19 + .../pipeline_stochastic_karras_ve.py | 128 + .../versatile_diffusion/__init__.py | 71 + .../versatile_diffusion/modeling_text_unet.py | 2508 +++++++++++ .../pipeline_versatile_diffusion.py | 421 ++ ...ipeline_versatile_diffusion_dual_guided.py | 556 +++ ...ine_versatile_diffusion_image_variation.py | 397 ++ ...eline_versatile_diffusion_text_to_image.py | 475 +++ .../deprecated/vq_diffusion/__init__.py | 57 + .../vq_diffusion/pipeline_vq_diffusion.py | 325 ++ .../src/diffusers/pipelines/dit/__init__.py | 19 + .../diffusers/pipelines/dit/pipeline_dit.py | 233 + .../diffusers/pipelines/free_init_utils.py | 184 + .../diffusers/pipelines/i2vgen_xl/__init__.py | 46 + .../pipelines/i2vgen_xl/pipeline_i2vgen_xl.py | 798 ++++ .../diffusers/pipelines/kandinsky/__init__.py | 66 + .../pipelines/kandinsky/pipeline_kandinsky.py | 407 ++ .../kandinsky/pipeline_kandinsky_combined.py | 814 ++++ .../kandinsky/pipeline_kandinsky_img2img.py | 500 +++ .../kandinsky/pipeline_kandinsky_inpaint.py | 635 +++ .../kandinsky/pipeline_kandinsky_prior.py | 547 +++ .../pipelines/kandinsky/text_encoder.py | 27 + .../pipelines/kandinsky2_2/__init__.py | 70 + .../kandinsky2_2/pipeline_kandinsky2_2.py | 320 ++ .../pipeline_kandinsky2_2_combined.py | 851 ++++ .../pipeline_kandinsky2_2_controlnet.py | 320 ++ ...ipeline_kandinsky2_2_controlnet_img2img.py | 381 ++ .../pipeline_kandinsky2_2_img2img.py | 399 ++ .../pipeline_kandinsky2_2_inpainting.py | 556 +++ .../pipeline_kandinsky2_2_prior.py | 549 +++ .../pipeline_kandinsky2_2_prior_emb2emb.py | 563 +++ .../pipelines/kandinsky3/__init__.py | 49 + .../kandinsky3/convert_kandinsky3_unet.py | 98 + .../kandinsky3/pipeline_kandinsky3.py | 589 +++ .../kandinsky3/pipeline_kandinsky3_img2img.py | 654 +++ .../latent_consistency_models/__init__.py | 50 + .../pipeline_latent_consistency_img2img.py | 956 +++++ .../pipeline_latent_consistency_text2img.py | 888 ++++ .../pipelines/latent_diffusion/__init__.py | 50 + .../pipeline_latent_diffusion.py | 746 ++++ ...peline_latent_diffusion_superresolution.py | 189 + .../diffusers/pipelines/ledits_pp/__init__.py | 55 + .../pipeline_leditspp_stable_diffusion.py | 1505 +++++++ .../pipeline_leditspp_stable_diffusion_xl.py | 1797 ++++++++ .../pipelines/ledits_pp/pipeline_output.py | 43 + .../diffusers/pipelines/musicldm/__init__.py | 49 + .../pipelines/musicldm/pipeline_musicldm.py | 635 +++ .../src/diffusers/pipelines/onnx_utils.py | 215 + .../pipelines/paint_by_example/__init__.py | 55 + .../paint_by_example/image_encoder.py | 67 + .../pipeline_paint_by_example.py | 621 +++ .../src/diffusers/pipelines/pia/__init__.py | 46 + .../diffusers/pipelines/pia/pipeline_pia.py | 1034 +++++ .../pipelines/pipeline_flax_utils.py | 616 +++ .../pipelines/pipeline_loading_utils.py | 508 +++ .../src/diffusers/pipelines/pipeline_utils.py | 1771 ++++++++ .../pipelines/pixart_alpha/__init__.py | 48 + .../pixart_alpha/pipeline_pixart_alpha.py | 979 +++++ .../semantic_stable_diffusion/__init__.py | 49 + .../pipeline_output.py | 25 + .../pipeline_semantic_stable_diffusion.py | 718 ++++ .../diffusers/pipelines/shap_e/__init__.py | 71 + .../src/diffusers/pipelines/shap_e/camera.py | 147 + .../pipelines/shap_e/pipeline_shap_e.py | 334 ++ .../shap_e/pipeline_shap_e_img2img.py | 321 ++ .../diffusers/pipelines/shap_e/renderer.py | 1050 +++++ .../pipelines/stable_cascade/__init__.py | 50 + .../stable_cascade/pipeline_stable_cascade.py | 482 +++ .../pipeline_stable_cascade_combined.py | 311 ++ .../pipeline_stable_cascade_prior.py | 638 +++ .../pipelines/stable_diffusion/README.md | 176 + .../pipelines/stable_diffusion/__init__.py | 203 + .../clip_image_project_model.py | 29 + .../stable_diffusion/convert_from_ckpt.py | 1860 ++++++++ .../pipeline_flax_stable_diffusion.py | 473 +++ .../pipeline_flax_stable_diffusion_img2img.py | 532 +++ .../pipeline_flax_stable_diffusion_inpaint.py | 589 +++ .../pipeline_onnx_stable_diffusion.py | 487 +++ .../pipeline_onnx_stable_diffusion_img2img.py | 549 +++ .../pipeline_onnx_stable_diffusion_inpaint.py | 563 +++ .../pipeline_onnx_stable_diffusion_upscale.py | 586 +++ .../stable_diffusion/pipeline_output.py | 45 + .../pipeline_stable_diffusion.py | 1032 +++++ .../pipeline_stable_diffusion_depth2img.py | 860 ++++ ...peline_stable_diffusion_image_variation.py | 420 ++ .../pipeline_stable_diffusion_img2img.py | 1113 +++++ .../pipeline_stable_diffusion_inpaint.py | 1430 +++++++ ...eline_stable_diffusion_instruct_pix2pix.py | 807 ++++ ...ipeline_stable_diffusion_latent_upscale.py | 495 +++ .../pipeline_stable_diffusion_upscale.py | 808 ++++ .../pipeline_stable_unclip.py | 932 ++++ .../pipeline_stable_unclip_img2img.py | 839 ++++ .../stable_diffusion/safety_checker.py | 125 + .../stable_diffusion/safety_checker_flax.py | 112 + .../stable_unclip_image_normalizer.py | 57 + .../__init__.py | 48 + ...line_stable_diffusion_attend_and_excite.py | 1088 +++++ .../stable_diffusion_diffedit/__init__.py | 48 + .../pipeline_stable_diffusion_diffedit.py | 1530 +++++++ .../stable_diffusion_gligen/__init__.py | 50 + .../pipeline_stable_diffusion_gligen.py | 845 ++++ ...line_stable_diffusion_gligen_text_image.py | 1017 +++++ .../stable_diffusion_k_diffusion/__init__.py | 62 + .../pipeline_stable_diffusion_k_diffusion.py | 664 +++ ...ipeline_stable_diffusion_xl_k_diffusion.py | 891 ++++ .../stable_diffusion_ldm3d/__init__.py | 48 + .../pipeline_stable_diffusion_ldm3d.py | 985 +++++ .../stable_diffusion_panorama/__init__.py | 48 + .../pipeline_stable_diffusion_panorama.py | 933 +++++ .../stable_diffusion_safe/__init__.py | 99 + .../stable_diffusion_safe/pipeline_output.py | 34 + .../pipeline_stable_diffusion_safe.py | 764 ++++ .../stable_diffusion_safe/safety_checker.py | 109 + .../stable_diffusion_sag/__init__.py | 48 + .../pipeline_stable_diffusion_sag.py | 886 ++++ .../pipelines/stable_diffusion_xl/__init__.py | 76 + .../pipeline_flax_stable_diffusion_xl.py | 308 ++ .../stable_diffusion_xl/pipeline_output.py | 37 + .../pipeline_stable_diffusion_xl.py | 1266 ++++++ .../pipeline_stable_diffusion_xl_img2img.py | 1442 +++++++ .../pipeline_stable_diffusion_xl_inpaint.py | 1812 ++++++++ ...ne_stable_diffusion_xl_instruct_pix2pix.py | 976 +++++ .../stable_diffusion_xl/watermark.py | 36 + .../stable_video_diffusion/__init__.py | 58 + .../pipeline_stable_video_diffusion.py | 673 +++ .../pipelines/t2i_adapter/__init__.py | 47 + .../pipeline_stable_diffusion_adapter.py | 912 ++++ .../pipeline_stable_diffusion_xl_adapter.py | 1258 ++++++ .../text_to_video_synthesis/__init__.py | 54 + .../pipeline_output.py | 25 + .../pipeline_text_to_video_synth.py | 663 +++ .../pipeline_text_to_video_synth_img2img.py | 760 ++++ .../pipeline_text_to_video_zero.py | 969 +++++ .../pipeline_text_to_video_zero_sdxl.py | 1315 ++++++ .../diffusers/pipelines/unclip/__init__.py | 52 + .../pipelines/unclip/pipeline_unclip.py | 493 +++ .../unclip/pipeline_unclip_image_variation.py | 420 ++ .../diffusers/pipelines/unclip/text_proj.py | 86 + .../pipelines/unidiffuser/__init__.py | 58 + .../unidiffuser/modeling_text_decoder.py | 296 ++ .../pipelines/unidiffuser/modeling_uvit.py | 1197 ++++++ .../unidiffuser/pipeline_unidiffuser.py | 1419 +++++++ .../pipelines/wuerstchen/__init__.py | 56 + .../wuerstchen/modeling_paella_vq_model.py | 172 + .../wuerstchen/modeling_wuerstchen_common.py | 81 + .../modeling_wuerstchen_diffnext.py | 254 ++ .../wuerstchen/modeling_wuerstchen_prior.py | 200 + .../wuerstchen/pipeline_wuerstchen.py | 438 ++ .../pipeline_wuerstchen_combined.py | 306 ++ .../wuerstchen/pipeline_wuerstchen_prior.py | 516 +++ diffusers-0.27.0/src/diffusers/py.typed | 0 .../src/diffusers/schedulers/README.md | 3 + .../src/diffusers/schedulers/__init__.py | 211 + .../schedulers/deprecated/__init__.py | 50 + .../deprecated/scheduling_karras_ve.py | 243 ++ .../deprecated/scheduling_sde_vp.py | 109 + .../diffusers/schedulers/scheduling_amused.py | 162 + .../scheduling_consistency_decoder.py | 180 + .../scheduling_consistency_models.py | 448 ++ .../diffusers/schedulers/scheduling_ddim.py | 520 +++ .../schedulers/scheduling_ddim_flax.py | 313 ++ .../schedulers/scheduling_ddim_inverse.py | 374 ++ .../schedulers/scheduling_ddim_parallel.py | 645 +++ .../diffusers/schedulers/scheduling_ddpm.py | 562 +++ .../schedulers/scheduling_ddpm_flax.py | 299 ++ .../schedulers/scheduling_ddpm_parallel.py | 653 +++ .../schedulers/scheduling_ddpm_wuerstchen.py | 230 + .../schedulers/scheduling_deis_multistep.py | 786 ++++ .../scheduling_dpmsolver_multistep.py | 1029 +++++ .../scheduling_dpmsolver_multistep_flax.py | 643 +++ .../scheduling_dpmsolver_multistep_inverse.py | 921 ++++ .../schedulers/scheduling_dpmsolver_sde.py | 557 +++ .../scheduling_dpmsolver_singlestep.py | 979 +++++ .../scheduling_edm_dpmsolver_multistep.py | 683 +++ .../schedulers/scheduling_edm_euler.py | 381 ++ .../scheduling_euler_ancestral_discrete.py | 481 +++ .../schedulers/scheduling_euler_discrete.py | 576 +++ .../scheduling_euler_discrete_flax.py | 265 ++ .../schedulers/scheduling_heun_discrete.py | 482 +++ .../diffusers/schedulers/scheduling_ipndm.py | 224 + .../scheduling_k_dpm_2_ancestral_discrete.py | 508 +++ .../schedulers/scheduling_k_dpm_2_discrete.py | 483 +++ .../schedulers/scheduling_karras_ve_flax.py | 238 ++ .../diffusers/schedulers/scheduling_lcm.py | 660 +++ .../schedulers/scheduling_lms_discrete.py | 475 +++ .../scheduling_lms_discrete_flax.py | 283 ++ .../diffusers/schedulers/scheduling_pndm.py | 476 +++ .../schedulers/scheduling_pndm_flax.py | 509 +++ .../schedulers/scheduling_repaint.py | 361 ++ .../schedulers/scheduling_sasolver.py | 1124 +++++ .../diffusers/schedulers/scheduling_sde_ve.py | 301 ++ .../schedulers/scheduling_sde_ve_flax.py | 280 ++ .../diffusers/schedulers/scheduling_tcd.py | 686 +++ .../diffusers/schedulers/scheduling_unclip.py | 352 ++ .../schedulers/scheduling_unipc_multistep.py | 880 ++++ .../diffusers/schedulers/scheduling_utils.py | 186 + .../schedulers/scheduling_utils_flax.py | 293 ++ .../schedulers/scheduling_vq_diffusion.py | 467 +++ .../src/diffusers/training_utils.py | 453 ++ .../src/diffusers/utils/__init__.py | 124 + .../src/diffusers/utils/accelerate_utils.py | 48 + .../src/diffusers/utils/constants.py | 55 + .../src/diffusers/utils/deprecation_utils.py | 49 + .../src/diffusers/utils/doc_utils.py | 38 + .../dummy_flax_and_transformers_objects.py | 77 + .../src/diffusers/utils/dummy_flax_objects.py | 212 + .../diffusers/utils/dummy_note_seq_objects.py | 17 + .../src/diffusers/utils/dummy_onnx_objects.py | 17 + .../src/diffusers/utils/dummy_pt_objects.py | 1170 ++++++ .../utils/dummy_torch_and_librosa_objects.py | 32 + .../utils/dummy_torch_and_scipy_objects.py | 17 + .../utils/dummy_torch_and_torchsde_objects.py | 17 + ...nd_transformers_and_k_diffusion_objects.py | 32 + ...torch_and_transformers_and_onnx_objects.py | 92 + .../dummy_torch_and_transformers_objects.py | 1607 +++++++ ...sformers_and_torch_and_note_seq_objects.py | 17 + .../diffusers/utils/dynamic_modules_utils.py | 452 ++ .../src/diffusers/utils/export_utils.py | 140 + .../src/diffusers/utils/hub_utils.py | 493 +++ .../src/diffusers/utils/import_utils.py | 726 ++++ .../src/diffusers/utils/loading_utils.py | 49 + .../src/diffusers/utils/logging.py | 339 ++ .../diffusers/utils/model_card_template.md | 24 + .../src/diffusers/utils/outputs.py | 137 + .../src/diffusers/utils/peft_utils.py | 268 ++ .../src/diffusers/utils/pil_utils.py | 67 + .../src/diffusers/utils/state_dict_utils.py | 324 ++ .../src/diffusers/utils/testing_utils.py | 967 +++++ .../src/diffusers/utils/torch_utils.py | 147 + .../src/diffusers/utils/versions.py | 117 + diffusers-0.27.0/tests/__init__.py | 0 diffusers-0.27.0/tests/conftest.py | 44 + .../fixtures/custom_pipeline/pipeline.py | 101 + .../fixtures/custom_pipeline/what_ever.py | 101 + .../tests/fixtures/elise_format0.mid | Bin 0 -> 14210 bytes .../tests/lora/test_lora_layers_peft.py | 2324 ++++++++++ diffusers-0.27.0/tests/models/__init__.py | 0 .../tests/models/autoencoders/__init__.py | 0 .../models/autoencoders/test_models_vae.py | 1118 +++++ .../autoencoders/test_models_vae_flax.py | 39 + .../models/autoencoders/test_models_vq.py | 99 + .../tests/models/test_activations.py | 48 + .../tests/models/test_attention_processor.py | 119 + .../tests/models/test_layers_utils.py | 528 +++ .../tests/models/test_modeling_common.py | 758 ++++ .../tests/models/test_modeling_common_flax.py | 66 + .../tests/models/transformers/__init__.py | 0 .../models/transformers/test_models_prior.py | 191 + .../tests/models/unets/__init__.py | 0 .../tests/models/unets/test_models_unet_1d.py | 270 ++ .../tests/models/unets/test_models_unet_2d.py | 331 ++ .../unets/test_models_unet_2d_condition.py | 1225 ++++++ .../models/unets/test_models_unet_2d_flax.py | 104 + .../unets/test_models_unet_3d_condition.py | 180 + .../models/unets/test_models_unet_motion.py | 306 ++ .../unets/test_models_unet_spatiotemporal.py | 289 ++ .../unets/test_models_unet_stable_cascade.py | 191 + .../tests/models/unets/test_unet_2d_blocks.py | 337 ++ .../models/unets/test_unet_blocks_common.py | 126 + .../tests/others/test_check_copies.py | 117 + .../tests/others/test_check_dummies.py | 122 + diffusers-0.27.0/tests/others/test_config.py | 288 ++ .../tests/others/test_dependencies.py | 50 + diffusers-0.27.0/tests/others/test_ema.py | 159 + .../tests/others/test_hub_utils.py | 29 + .../tests/others/test_image_processor.py | 310 ++ diffusers-0.27.0/tests/others/test_outputs.py | 93 + .../tests/others/test_training.py | 86 + diffusers-0.27.0/tests/others/test_utils.py | 213 + diffusers-0.27.0/tests/pipelines/__init__.py | 0 .../tests/pipelines/amused/__init__.py | 0 .../tests/pipelines/amused/test_amused.py | 181 + .../pipelines/amused/test_amused_img2img.py | 235 ++ .../pipelines/amused/test_amused_inpaint.py | 273 ++ .../tests/pipelines/animatediff/__init__.py | 0 .../pipelines/animatediff/test_animatediff.py | 358 ++ .../test_animatediff_video2video.py | 304 ++ .../tests/pipelines/audioldm/__init__.py | 0 .../tests/pipelines/audioldm/test_audioldm.py | 447 ++ .../tests/pipelines/audioldm2/__init__.py | 0 .../pipelines/audioldm2/test_audioldm2.py | 569 +++ .../tests/pipelines/blipdiffusion/__init__.py | 0 .../blipdiffusion/test_blipdiffusion.py | 196 + .../pipelines/consistency_models/__init__.py | 0 .../test_consistency_models.py | 294 ++ .../tests/pipelines/controlnet/__init__.py | 0 .../pipelines/controlnet/test_controlnet.py | 1151 +++++ .../test_controlnet_blip_diffusion.py | 216 + .../controlnet/test_controlnet_img2img.py | 479 +++ .../controlnet/test_controlnet_inpaint.py | 605 +++ .../test_controlnet_inpaint_sdxl.py | 304 ++ .../controlnet/test_controlnet_sdxl.py | 1173 ++++++ .../test_controlnet_sdxl_img2img.py | 351 ++ .../controlnet/test_flax_controlnet.py | 127 + .../pipelines/dance_diffusion/__init__.py | 0 .../dance_diffusion/test_dance_diffusion.py | 161 + .../tests/pipelines/ddim/__init__.py | 0 .../tests/pipelines/ddim/test_ddim.py | 143 + .../tests/pipelines/ddpm/__init__.py | 0 .../tests/pipelines/ddpm/test_ddpm.py | 111 + .../tests/pipelines/deepfloyd_if/__init__.py | 272 ++ .../tests/pipelines/deepfloyd_if/test_if.py | 120 + .../pipelines/deepfloyd_if/test_if_img2img.py | 131 + .../test_if_img2img_superresolution.py | 136 + .../deepfloyd_if/test_if_inpainting.py | 134 + .../test_if_inpainting_superresolution.py | 143 + .../deepfloyd_if/test_if_superresolution.py | 130 + .../tests/pipelines/dit/__init__.py | 0 .../tests/pipelines/dit/test_dit.py | 151 + .../tests/pipelines/i2vgen_xl/__init__.py | 0 .../pipelines/i2vgen_xl/test_i2vgenxl.py | 265 ++ .../test_ip_adapter_stable_diffusion.py | 539 +++ .../tests/pipelines/kandinsky/__init__.py | 0 .../pipelines/kandinsky/test_kandinsky.py | 323 ++ .../kandinsky/test_kandinsky_combined.py | 361 ++ .../kandinsky/test_kandinsky_img2img.py | 417 ++ .../kandinsky/test_kandinsky_inpaint.py | 356 ++ .../kandinsky/test_kandinsky_prior.py | 237 ++ .../tests/pipelines/kandinsky2_2/__init__.py | 0 .../pipelines/kandinsky2_2/test_kandinsky.py | 272 ++ .../kandinsky2_2/test_kandinsky_combined.py | 406 ++ .../kandinsky2_2/test_kandinsky_controlnet.py | 285 ++ .../test_kandinsky_controlnet_img2img.py | 303 ++ .../kandinsky2_2/test_kandinsky_img2img.py | 296 ++ .../kandinsky2_2/test_kandinsky_inpaint.py | 351 ++ .../kandinsky2_2/test_kandinsky_prior.py | 278 ++ .../test_kandinsky_prior_emb2emb.py | 247 ++ .../tests/pipelines/kandinsky3/__init__.py | 0 .../pipelines/kandinsky3/test_kandinsky3.py | 233 + .../kandinsky3/test_kandinsky3_img2img.py | 225 + .../latent_consistency_models/__init__.py | 0 .../test_latent_consistency_models.py | 259 ++ .../test_latent_consistency_models_img2img.py | 277 ++ .../pipelines/latent_diffusion/__init__.py | 0 .../latent_diffusion/test_latent_diffusion.py | 207 + .../test_latent_diffusion_superresolution.py | 138 + .../tests/pipelines/ledits_pp/__init__.py | 0 .../test_ledits_pp_stable_diffusion.py | 244 ++ .../test_ledits_pp_stable_diffusion_xl.py | 289 ++ .../tests/pipelines/musicldm/__init__.py | 0 .../tests/pipelines/musicldm/test_musicldm.py | 465 ++ .../pipelines/paint_by_example/__init__.py | 0 .../paint_by_example/test_paint_by_example.py | 220 + .../tests/pipelines/pia/__init__.py | 0 .../tests/pipelines/pia/test_pia.py | 311 ++ .../tests/pipelines/pipeline_params.py | 129 + .../tests/pipelines/pixart_alpha/__init__.py | 0 .../pipelines/pixart_alpha/test_pixart.py | 437 ++ .../tests/pipelines/pndm/__init__.py | 0 .../tests/pipelines/pndm/test_pndm.py | 87 + .../semantic_stable_diffusion/__init__.py | 0 .../test_semantic_diffusion.py | 606 +++ .../tests/pipelines/shap_e/__init__.py | 0 .../tests/pipelines/shap_e/test_shap_e.py | 255 ++ .../pipelines/shap_e/test_shap_e_img2img.py | 284 ++ .../pipelines/stable_cascade/__init__.py | 0 .../test_stable_cascade_combined.py | 279 ++ .../test_stable_cascade_decoder.py | 286 ++ .../test_stable_cascade_prior.py | 341 ++ .../pipelines/stable_diffusion/__init__.py | 0 .../test_onnx_stable_diffusion.py | 376 ++ .../test_onnx_stable_diffusion_img2img.py | 245 ++ .../test_onnx_stable_diffusion_inpaint.py | 141 + .../test_onnx_stable_diffusion_upscale.py | 227 + .../stable_diffusion/test_stable_diffusion.py | 1425 +++++++ .../test_stable_diffusion_img2img.py | 735 ++++ .../test_stable_diffusion_inpaint.py | 1668 ++++++++ ...st_stable_diffusion_instruction_pix2pix.py | 426 ++ .../pipelines/stable_diffusion_2/__init__.py | 0 .../test_stable_diffusion.py | 653 +++ ...test_stable_diffusion_attend_and_excite.py | 235 ++ .../test_stable_diffusion_depth.py | 603 +++ .../test_stable_diffusion_diffedit.py | 432 ++ .../test_stable_diffusion_flax.py | 108 + .../test_stable_diffusion_flax_inpaint.py | 82 + .../test_stable_diffusion_inpaint.py | 277 ++ .../test_stable_diffusion_latent_upscale.py | 306 ++ .../test_stable_diffusion_upscale.py | 552 +++ .../test_stable_diffusion_v_pred.py | 563 +++ .../stable_diffusion_adapter/__init__.py | 0 .../test_stable_diffusion_adapter.py | 950 +++++ .../stable_diffusion_gligen/__init__.py | 0 .../test_stable_diffusion_gligen.py | 162 + .../__init__.py | 0 ...test_stable_diffusion_gligen_text_image.py | 192 + .../__init__.py | 0 .../test_stable_diffusion_image_variation.py | 330 ++ .../stable_diffusion_k_diffusion/__init__.py | 0 .../test_stable_diffusion_k_diffusion.py | 135 + .../stable_diffusion_ldm3d/__init__.py | 0 .../test_stable_diffusion_ldm3d.py | 310 ++ .../stable_diffusion_panorama/__init__.py | 0 .../test_stable_diffusion_panorama.py | 412 ++ .../stable_diffusion_safe/__init__.py | 0 .../test_safe_diffusion.py | 435 ++ .../stable_diffusion_sag/__init__.py | 0 .../test_stable_diffusion_sag.py | 215 + .../pipelines/stable_diffusion_xl/__init__.py | 0 .../test_stable_diffusion_xl.py | 1102 +++++ .../test_stable_diffusion_xl_adapter.py | 711 ++++ .../test_stable_diffusion_xl_img2img.py | 852 ++++ .../test_stable_diffusion_xl_inpaint.py | 810 ++++ ...stable_diffusion_xl_instruction_pix2pix.py | 189 + .../test_stable_diffusion_xl_k_diffusion.py | 138 + .../tests/pipelines/stable_unclip/__init__.py | 0 .../stable_unclip/test_stable_unclip.py | 239 ++ .../test_stable_unclip_img2img.py | 300 ++ .../stable_video_diffusion/__init__.py | 0 .../test_stable_video_diffusion.py | 554 +++ .../tests/pipelines/test_pipeline_utils.py | 134 + .../tests/pipelines/test_pipelines.py | 1932 +++++++++ .../tests/pipelines/test_pipelines_auto.py | 353 ++ .../pipelines/test_pipelines_combined.py | 128 + .../tests/pipelines/test_pipelines_common.py | 1614 +++++++ .../tests/pipelines/test_pipelines_flax.py | 260 ++ .../pipelines/test_pipelines_onnx_common.py | 12 + .../text_to_video_synthesis/__init__.py | 0 .../test_text_to_video.py | 215 + .../test_text_to_video_zero.py | 42 + .../test_text_to_video_zero_sdxl.py | 405 ++ .../test_video_to_video.py | 224 + .../tests/pipelines/unclip/__init__.py | 0 .../tests/pipelines/unclip/test_unclip.py | 507 +++ .../unclip/test_unclip_image_variation.py | 531 +++ .../tests/pipelines/unidiffuser/__init__.py | 0 .../pipelines/unidiffuser/test_unidiffuser.py | 790 ++++ .../tests/pipelines/wuerstchen/__init__.py | 0 .../wuerstchen/test_wuerstchen_combined.py | 239 ++ .../wuerstchen/test_wuerstchen_decoder.py | 188 + .../wuerstchen/test_wuerstchen_prior.py | 296 ++ diffusers-0.27.0/tests/schedulers/__init__.py | 0 .../test_scheduler_consistency_model.py | 189 + .../tests/schedulers/test_scheduler_ddim.py | 176 + .../schedulers/test_scheduler_ddim_inverse.py | 135 + .../test_scheduler_ddim_parallel.py | 216 + .../tests/schedulers/test_scheduler_ddpm.py | 222 + .../test_scheduler_ddpm_parallel.py | 251 ++ .../tests/schedulers/test_scheduler_deis.py | 265 ++ .../schedulers/test_scheduler_dpm_multi.py | 318 ++ .../test_scheduler_dpm_multi_inverse.py | 267 ++ .../schedulers/test_scheduler_dpm_sde.py | 167 + .../schedulers/test_scheduler_dpm_single.py | 309 ++ .../test_scheduler_edm_dpmsolver_multistep.py | 262 ++ .../schedulers/test_scheduler_edm_euler.py | 206 + .../tests/schedulers/test_scheduler_euler.py | 191 + .../test_scheduler_euler_ancestral.py | 156 + .../tests/schedulers/test_scheduler_flax.py | 919 ++++ .../tests/schedulers/test_scheduler_heun.py | 191 + .../tests/schedulers/test_scheduler_ipndm.py | 163 + .../test_scheduler_kdpm2_ancestral.py | 158 + .../test_scheduler_kdpm2_discrete.py | 166 + .../tests/schedulers/test_scheduler_lcm.py | 300 ++ .../tests/schedulers/test_scheduler_lms.py | 170 + .../tests/schedulers/test_scheduler_pndm.py | 242 ++ .../schedulers/test_scheduler_sasolver.py | 202 + .../schedulers/test_scheduler_score_sde_ve.py | 189 + .../tests/schedulers/test_scheduler_tcd.py | 180 + .../tests/schedulers/test_scheduler_unclip.py | 137 + .../tests/schedulers/test_scheduler_unipc.py | 381 ++ .../schedulers/test_scheduler_vq_diffusion.py | 56 + .../tests/schedulers/test_schedulers.py | 868 ++++ .../utils/check_config_docstrings.py | 84 + diffusers-0.27.0/utils/check_copies.py | 222 + diffusers-0.27.0/utils/check_doc_toc.py | 158 + diffusers-0.27.0/utils/check_dummies.py | 175 + diffusers-0.27.0/utils/check_inits.py | 299 ++ diffusers-0.27.0/utils/check_repo.py | 755 ++++ diffusers-0.27.0/utils/check_table.py | 185 + diffusers-0.27.0/utils/custom_init_isort.py | 329 ++ .../utils/fetch_latest_release_branch.py | 68 + .../fetch_torch_cuda_pipeline_test_matrix.py | 102 + diffusers-0.27.0/utils/get_modified_files.py | 34 + .../utils/notify_slack_about_release.py | 80 + .../utils/overwrite_expected_slice.py | 90 + diffusers-0.27.0/utils/print_env.py | 48 + diffusers-0.27.0/utils/release.py | 162 + diffusers-0.27.0/utils/stale.py | 67 + diffusers-0.27.0/utils/tests_fetcher.py | 1128 +++++ 725 files changed, 279047 insertions(+), 11 deletions(-) delete mode 100644 diffusers-0.27.0/examples/text_to_image/output/sd2.1/pytorch_lora_weights.safetensors create mode 100755 diffusers-0.27.0/examples/textual_inversion/README.md create mode 100755 diffusers-0.27.0/examples/textual_inversion/README_sdxl.md create mode 100755 diffusers-0.27.0/examples/textual_inversion/requirements.txt create mode 100755 diffusers-0.27.0/examples/textual_inversion/requirements_flax.txt create mode 100755 diffusers-0.27.0/examples/textual_inversion/test_textual_inversion.py create mode 100755 diffusers-0.27.0/examples/textual_inversion/test_textual_inversion_sdxl.py create mode 100755 diffusers-0.27.0/examples/textual_inversion/textual_inversion.py create mode 100755 diffusers-0.27.0/examples/textual_inversion/textual_inversion_flax.py create mode 100755 diffusers-0.27.0/examples/textual_inversion/textual_inversion_sdxl.py create mode 100755 diffusers-0.27.0/examples/unconditional_image_generation/README.md create mode 100755 diffusers-0.27.0/examples/unconditional_image_generation/requirements.txt create mode 100755 diffusers-0.27.0/examples/unconditional_image_generation/test_unconditional.py create mode 100755 diffusers-0.27.0/examples/unconditional_image_generation/train_unconditional.py create mode 100755 diffusers-0.27.0/examples/wuerstchen/text_to_image/README.md create mode 100755 diffusers-0.27.0/examples/wuerstchen/text_to_image/__init__.py create mode 100755 diffusers-0.27.0/examples/wuerstchen/text_to_image/modeling_efficient_net_encoder.py create mode 100755 diffusers-0.27.0/examples/wuerstchen/text_to_image/requirements.txt create mode 100755 diffusers-0.27.0/examples/wuerstchen/text_to_image/train_text_to_image_lora_prior.py create mode 100755 diffusers-0.27.0/examples/wuerstchen/text_to_image/train_text_to_image_prior.py create mode 100755 diffusers-0.27.0/pyproject.toml create mode 100755 diffusers-0.27.0/scripts/__init__.py create mode 100755 diffusers-0.27.0/scripts/change_naming_configs_and_checkpoints.py create mode 100755 diffusers-0.27.0/scripts/conversion_ldm_uncond.py create mode 100755 diffusers-0.27.0/scripts/convert_amused.py create mode 100755 diffusers-0.27.0/scripts/convert_animatediff_motion_lora_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_animatediff_motion_module_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_asymmetric_vqgan_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_blipdiffusion_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_consistency_decoder.py create mode 100755 diffusers-0.27.0/scripts/convert_consistency_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_dance_diffusion_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_ddpm_original_checkpoint_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_diffusers_sdxl_lora_to_webui.py create mode 100755 diffusers-0.27.0/scripts/convert_diffusers_to_original_sdxl.py create mode 100755 diffusers-0.27.0/scripts/convert_diffusers_to_original_stable_diffusion.py create mode 100755 diffusers-0.27.0/scripts/convert_dit_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_gligen_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_i2vgen_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_if.py create mode 100755 diffusers-0.27.0/scripts/convert_k_upscaler_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_kakao_brain_unclip_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_kandinsky3_unet.py create mode 100755 diffusers-0.27.0/scripts/convert_kandinsky_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_ldm_original_checkpoint_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_lora_safetensor_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_models_diffuser_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_ms_text_to_video_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_music_spectrogram_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_ncsnpp_original_checkpoint_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_original_audioldm2_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_original_audioldm_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_original_controlnet_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_original_musicldm_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_original_stable_diffusion_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_original_t2i_adapter.py create mode 100755 diffusers-0.27.0/scripts/convert_pixart_alpha_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_shap_e_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_stable_cascade.py create mode 100755 diffusers-0.27.0/scripts/convert_stable_cascade_lite.py create mode 100755 diffusers-0.27.0/scripts/convert_stable_diffusion_checkpoint_to_onnx.py create mode 100755 diffusers-0.27.0/scripts/convert_stable_diffusion_controlnet_to_onnx.py create mode 100755 diffusers-0.27.0/scripts/convert_stable_diffusion_controlnet_to_tensorrt.py create mode 100755 diffusers-0.27.0/scripts/convert_svd_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_tiny_autoencoder_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_unclip_txt2img_to_image_variation.py create mode 100755 diffusers-0.27.0/scripts/convert_unidiffuser_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_vae_diff_to_onnx.py create mode 100755 diffusers-0.27.0/scripts/convert_vae_pt_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_versatile_diffusion_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_vq_diffusion_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/convert_wuerstchen.py create mode 100755 diffusers-0.27.0/scripts/convert_zero123_to_diffusers.py create mode 100755 diffusers-0.27.0/scripts/generate_logits.py create mode 100755 diffusers-0.27.0/scripts/log_reports.py create mode 100755 diffusers-0.27.0/src/diffusers/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/commands/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/commands/diffusers_cli.py create mode 100755 diffusers-0.27.0/src/diffusers/commands/env.py create mode 100755 diffusers-0.27.0/src/diffusers/commands/fp16_safetensors.py create mode 100755 diffusers-0.27.0/src/diffusers/configuration_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/dependency_versions_check.py create mode 100755 diffusers-0.27.0/src/diffusers/dependency_versions_table.py create mode 100755 diffusers-0.27.0/src/diffusers/experimental/README.md create mode 100755 diffusers-0.27.0/src/diffusers/experimental/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/experimental/rl/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/experimental/rl/value_guided_sampling.py create mode 100755 diffusers-0.27.0/src/diffusers/image_processor.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/autoencoder.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/controlnet.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/ip_adapter.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/lora.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/lora_conversion_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/peft.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/single_file.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/single_file_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/textual_inversion.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/unet.py create mode 100755 diffusers-0.27.0/src/diffusers/loaders/utils.py create mode 100755 diffusers-0.27.0/src/diffusers/models/README.md create mode 100755 diffusers-0.27.0/src/diffusers/models/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/models/activations.py create mode 100755 diffusers-0.27.0/src/diffusers/models/adapter.py create mode 100755 diffusers-0.27.0/src/diffusers/models/attention.py create mode 100755 diffusers-0.27.0/src/diffusers/models/attention_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/models/attention_processor.py create mode 100755 diffusers-0.27.0/src/diffusers/models/autoencoders/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_asym_kl.py create mode 100755 diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_kl.py create mode 100755 diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py create mode 100755 diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_tiny.py create mode 100755 diffusers-0.27.0/src/diffusers/models/autoencoders/consistency_decoder_vae.py create mode 100755 diffusers-0.27.0/src/diffusers/models/autoencoders/vae.py create mode 100755 diffusers-0.27.0/src/diffusers/models/controlnet.py create mode 100755 diffusers-0.27.0/src/diffusers/models/controlnet_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/models/downsampling.py create mode 100755 diffusers-0.27.0/src/diffusers/models/dual_transformer_2d.py create mode 100755 diffusers-0.27.0/src/diffusers/models/embeddings.py create mode 100755 diffusers-0.27.0/src/diffusers/models/embeddings_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/models/lora.py create mode 100755 diffusers-0.27.0/src/diffusers/models/modeling_flax_pytorch_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/models/modeling_flax_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/models/modeling_outputs.py create mode 100755 diffusers-0.27.0/src/diffusers/models/modeling_pytorch_flax_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/models/modeling_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/models/normalization.py create mode 100755 diffusers-0.27.0/src/diffusers/models/prior_transformer.py create mode 100755 diffusers-0.27.0/src/diffusers/models/resnet.py create mode 100755 diffusers-0.27.0/src/diffusers/models/resnet_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/models/t5_film_transformer.py create mode 100755 diffusers-0.27.0/src/diffusers/models/transformer_2d.py create mode 100755 diffusers-0.27.0/src/diffusers/models/transformer_temporal.py create mode 100755 diffusers-0.27.0/src/diffusers/models/transformers/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/models/transformers/dual_transformer_2d.py create mode 100755 diffusers-0.27.0/src/diffusers/models/transformers/prior_transformer.py create mode 100755 diffusers-0.27.0/src/diffusers/models/transformers/t5_film_transformer.py create mode 100755 diffusers-0.27.0/src/diffusers/models/transformers/transformer_2d.py create mode 100755 diffusers-0.27.0/src/diffusers/models/transformers/transformer_temporal.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unet_1d.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unet_1d_blocks.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unet_2d.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unet_2d_blocks.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unet_2d_condition.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_1d.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_1d_blocks.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_2d.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_2d_blocks.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_2d_blocks_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_2d_condition.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_2d_condition_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_3d_blocks.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_3d_condition.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_i2vgen_xl.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_kandinsky3.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_motion_model.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_spatio_temporal_condition.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/unet_stable_cascade.py create mode 100755 diffusers-0.27.0/src/diffusers/models/unets/uvit_2d.py create mode 100755 diffusers-0.27.0/src/diffusers/models/upsampling.py create mode 100755 diffusers-0.27.0/src/diffusers/models/vae_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/models/vq_model.py create mode 100755 diffusers-0.27.0/src/diffusers/optimization.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/README.md create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/amused/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused_inpaint.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/animatediff/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_animatediff.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_output.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/audioldm/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/audioldm/pipeline_audioldm.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/audioldm2/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/auto_pipeline.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/blip_image_processing.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/consistency_models/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/consistency_models/pipeline_consistency_models.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/controlnet/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/controlnet/multicontrolnet.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_flax_controlnet.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/dance_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/ddim/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/ddim/pipeline_ddim.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/ddpm/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/ddpm/pipeline_ddpm.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_output.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/safety_checker.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/timesteps.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/watermark.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/README.md create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/modeling_roberta_series.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_output.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/mel.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/pipeline_audio_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/pndm/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/pndm/pipeline_pndm.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/repaint/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/repaint/pipeline_repaint.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/score_sde_ve/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/score_sde_ve/pipeline_score_sde_ve.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/continuous_encoder.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/midi_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/notes_encoder.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_onnx_stable_diffusion_inpaint_legacy.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/stochastic_karras_ve/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/stochastic_karras_ve/pipeline_stochastic_karras_ve.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/vq_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/dit/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/dit/pipeline_dit.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/free_init_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/i2vgen_xl/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky/text_encoder.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/convert_kandinsky3_unet.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_output.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/musicldm/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/musicldm/pipeline_musicldm.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/onnx_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/image_encoder.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/pia/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/pia/pipeline_pia.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/pipeline_flax_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/pipeline_loading_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/pipeline_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/pixart_alpha/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_output.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/shap_e/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/shap_e/camera.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/shap_e/pipeline_shap_e.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/shap_e/renderer.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/README.md create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/clip_image_project_model.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_inpaint.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_output.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/safety_checker.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/safety_checker_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_attend_and_excite/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_diffedit/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_ldm3d/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_panorama/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/pipeline_output.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/safety_checker.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_sag/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_output.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/watermark.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_video_diffusion/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_output.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/unclip/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/unclip/pipeline_unclip.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/unclip/text_proj.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/modeling_text_decoder.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/modeling_uvit.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_diffnext.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py create mode 100755 diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py create mode 100755 diffusers-0.27.0/src/diffusers/py.typed create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/README.md create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/deprecated/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/deprecated/scheduling_karras_ve.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/deprecated/scheduling_sde_vp.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_amused.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_consistency_decoder.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_consistency_models.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_inverse.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_parallel.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_parallel.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_wuerstchen.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_deis_multistep.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_sde.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_edm_euler.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_discrete.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_discrete_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_heun_discrete.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_ipndm.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_karras_ve_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_lcm.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_lms_discrete.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_lms_discrete_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_pndm.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_pndm_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_repaint.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_sasolver.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_sde_ve.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_sde_ve_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_tcd.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_unclip.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_unipc_multistep.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_utils_flax.py create mode 100755 diffusers-0.27.0/src/diffusers/schedulers/scheduling_vq_diffusion.py create mode 100755 diffusers-0.27.0/src/diffusers/training_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/__init__.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/accelerate_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/constants.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/deprecation_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/doc_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_flax_and_transformers_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_flax_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_note_seq_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_onnx_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_pt_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_librosa_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_scipy_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_torchsde_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_and_k_diffusion_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_and_onnx_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dummy_transformers_and_torch_and_note_seq_objects.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/dynamic_modules_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/export_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/hub_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/import_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/loading_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/logging.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/model_card_template.md create mode 100755 diffusers-0.27.0/src/diffusers/utils/outputs.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/peft_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/pil_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/state_dict_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/testing_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/torch_utils.py create mode 100755 diffusers-0.27.0/src/diffusers/utils/versions.py create mode 100755 diffusers-0.27.0/tests/__init__.py create mode 100755 diffusers-0.27.0/tests/conftest.py create mode 100755 diffusers-0.27.0/tests/fixtures/custom_pipeline/pipeline.py create mode 100755 diffusers-0.27.0/tests/fixtures/custom_pipeline/what_ever.py create mode 100755 diffusers-0.27.0/tests/fixtures/elise_format0.mid create mode 100755 diffusers-0.27.0/tests/lora/test_lora_layers_peft.py create mode 100755 diffusers-0.27.0/tests/models/__init__.py create mode 100755 diffusers-0.27.0/tests/models/autoencoders/__init__.py create mode 100755 diffusers-0.27.0/tests/models/autoencoders/test_models_vae.py create mode 100755 diffusers-0.27.0/tests/models/autoencoders/test_models_vae_flax.py create mode 100755 diffusers-0.27.0/tests/models/autoencoders/test_models_vq.py create mode 100755 diffusers-0.27.0/tests/models/test_activations.py create mode 100755 diffusers-0.27.0/tests/models/test_attention_processor.py create mode 100755 diffusers-0.27.0/tests/models/test_layers_utils.py create mode 100755 diffusers-0.27.0/tests/models/test_modeling_common.py create mode 100755 diffusers-0.27.0/tests/models/test_modeling_common_flax.py create mode 100755 diffusers-0.27.0/tests/models/transformers/__init__.py create mode 100755 diffusers-0.27.0/tests/models/transformers/test_models_prior.py create mode 100755 diffusers-0.27.0/tests/models/unets/__init__.py create mode 100755 diffusers-0.27.0/tests/models/unets/test_models_unet_1d.py create mode 100755 diffusers-0.27.0/tests/models/unets/test_models_unet_2d.py create mode 100755 diffusers-0.27.0/tests/models/unets/test_models_unet_2d_condition.py create mode 100755 diffusers-0.27.0/tests/models/unets/test_models_unet_2d_flax.py create mode 100755 diffusers-0.27.0/tests/models/unets/test_models_unet_3d_condition.py create mode 100755 diffusers-0.27.0/tests/models/unets/test_models_unet_motion.py create mode 100755 diffusers-0.27.0/tests/models/unets/test_models_unet_spatiotemporal.py create mode 100755 diffusers-0.27.0/tests/models/unets/test_models_unet_stable_cascade.py create mode 100755 diffusers-0.27.0/tests/models/unets/test_unet_2d_blocks.py create mode 100755 diffusers-0.27.0/tests/models/unets/test_unet_blocks_common.py create mode 100755 diffusers-0.27.0/tests/others/test_check_copies.py create mode 100755 diffusers-0.27.0/tests/others/test_check_dummies.py create mode 100755 diffusers-0.27.0/tests/others/test_config.py create mode 100755 diffusers-0.27.0/tests/others/test_dependencies.py create mode 100755 diffusers-0.27.0/tests/others/test_ema.py create mode 100755 diffusers-0.27.0/tests/others/test_hub_utils.py create mode 100755 diffusers-0.27.0/tests/others/test_image_processor.py create mode 100755 diffusers-0.27.0/tests/others/test_outputs.py create mode 100755 diffusers-0.27.0/tests/others/test_training.py create mode 100755 diffusers-0.27.0/tests/others/test_utils.py create mode 100755 diffusers-0.27.0/tests/pipelines/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/amused/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/amused/test_amused.py create mode 100755 diffusers-0.27.0/tests/pipelines/amused/test_amused_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/amused/test_amused_inpaint.py create mode 100755 diffusers-0.27.0/tests/pipelines/animatediff/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/animatediff/test_animatediff.py create mode 100755 diffusers-0.27.0/tests/pipelines/animatediff/test_animatediff_video2video.py create mode 100755 diffusers-0.27.0/tests/pipelines/audioldm/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/audioldm/test_audioldm.py create mode 100755 diffusers-0.27.0/tests/pipelines/audioldm2/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/audioldm2/test_audioldm2.py create mode 100755 diffusers-0.27.0/tests/pipelines/blipdiffusion/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/blipdiffusion/test_blipdiffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/consistency_models/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/consistency_models/test_consistency_models.py create mode 100755 diffusers-0.27.0/tests/pipelines/controlnet/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet.py create mode 100755 diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_inpaint.py create mode 100755 diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_inpaint_sdxl.py create mode 100755 diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_sdxl.py create mode 100755 diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_sdxl_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/controlnet/test_flax_controlnet.py create mode 100755 diffusers-0.27.0/tests/pipelines/dance_diffusion/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/dance_diffusion/test_dance_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/ddim/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/ddim/test_ddim.py create mode 100755 diffusers-0.27.0/tests/pipelines/ddpm/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/ddpm/test_ddpm.py create mode 100755 diffusers-0.27.0/tests/pipelines/deepfloyd_if/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if.py create mode 100755 diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_img2img_superresolution.py create mode 100755 diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_inpainting.py create mode 100755 diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_inpainting_superresolution.py create mode 100755 diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_superresolution.py create mode 100755 diffusers-0.27.0/tests/pipelines/dit/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/dit/test_dit.py create mode 100755 diffusers-0.27.0/tests/pipelines/i2vgen_xl/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/i2vgen_xl/test_i2vgenxl.py create mode 100755 diffusers-0.27.0/tests/pipelines/ip_adapters/test_ip_adapter_stable_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_combined.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_inpaint.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_prior.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky2_2/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_combined.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_controlnet.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_controlnet_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_inpaint.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_prior.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_prior_emb2emb.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky3/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky3/test_kandinsky3.py create mode 100755 diffusers-0.27.0/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/latent_consistency_models/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/latent_consistency_models/test_latent_consistency_models.py create mode 100755 diffusers-0.27.0/tests/pipelines/latent_consistency_models/test_latent_consistency_models_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/latent_diffusion/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/latent_diffusion/test_latent_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py create mode 100755 diffusers-0.27.0/tests/pipelines/ledits_pp/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/ledits_pp/test_ledits_pp_stable_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/ledits_pp/test_ledits_pp_stable_diffusion_xl.py create mode 100755 diffusers-0.27.0/tests/pipelines/musicldm/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/musicldm/test_musicldm.py create mode 100755 diffusers-0.27.0/tests/pipelines/paint_by_example/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/paint_by_example/test_paint_by_example.py create mode 100755 diffusers-0.27.0/tests/pipelines/pia/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/pia/test_pia.py create mode 100755 diffusers-0.27.0/tests/pipelines/pipeline_params.py create mode 100755 diffusers-0.27.0/tests/pipelines/pixart_alpha/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/pixart_alpha/test_pixart.py create mode 100755 diffusers-0.27.0/tests/pipelines/pndm/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/pndm/test_pndm.py create mode 100755 diffusers-0.27.0/tests/pipelines/semantic_stable_diffusion/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/semantic_stable_diffusion/test_semantic_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/shap_e/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/shap_e/test_shap_e.py create mode 100755 diffusers-0.27.0/tests/pipelines/shap_e/test_shap_e_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_cascade/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_combined.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_decoder.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_prior.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_upscale.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_instruction_pix2pix.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_2/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_attend_and_excite.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_diffedit.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_flax.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_flax_inpaint.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_latent_upscale.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_v_pred.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_adapter/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_adapter/test_stable_diffusion_adapter.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen/test_stable_diffusion_gligen.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen_text_image/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen_text_image/test_stable_diffusion_gligen_text_image.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_image_variation/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_image_variation/test_stable_diffusion_image_variation.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_k_diffusion/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_k_diffusion/test_stable_diffusion_k_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_ldm3d/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_ldm3d/test_stable_diffusion_ldm3d.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_panorama/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_panorama/test_stable_diffusion_panorama.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_safe/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_safe/test_safe_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_sag/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_sag/test_stable_diffusion_sag.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_instruction_pix2pix.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_k_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_unclip/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_unclip/test_stable_unclip.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_unclip/test_stable_unclip_img2img.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_video_diffusion/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/stable_video_diffusion/test_stable_video_diffusion.py create mode 100755 diffusers-0.27.0/tests/pipelines/test_pipeline_utils.py create mode 100755 diffusers-0.27.0/tests/pipelines/test_pipelines.py create mode 100755 diffusers-0.27.0/tests/pipelines/test_pipelines_auto.py create mode 100755 diffusers-0.27.0/tests/pipelines/test_pipelines_combined.py create mode 100755 diffusers-0.27.0/tests/pipelines/test_pipelines_common.py create mode 100755 diffusers-0.27.0/tests/pipelines/test_pipelines_flax.py create mode 100755 diffusers-0.27.0/tests/pipelines/test_pipelines_onnx_common.py create mode 100755 diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video.py create mode 100755 diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video_zero.py create mode 100755 diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video_zero_sdxl.py create mode 100755 diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_video_to_video.py create mode 100755 diffusers-0.27.0/tests/pipelines/unclip/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/unclip/test_unclip.py create mode 100755 diffusers-0.27.0/tests/pipelines/unclip/test_unclip_image_variation.py create mode 100755 diffusers-0.27.0/tests/pipelines/unidiffuser/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/unidiffuser/test_unidiffuser.py create mode 100755 diffusers-0.27.0/tests/pipelines/wuerstchen/__init__.py create mode 100755 diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_combined.py create mode 100755 diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_decoder.py create mode 100755 diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_prior.py create mode 100755 diffusers-0.27.0/tests/schedulers/__init__.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_consistency_model.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_ddim.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_ddim_inverse.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_ddim_parallel.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_ddpm.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_ddpm_parallel.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_deis.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_multi.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_multi_inverse.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_sde.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_single.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_edm_dpmsolver_multistep.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_edm_euler.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_euler.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_euler_ancestral.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_flax.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_heun.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_ipndm.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_kdpm2_ancestral.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_kdpm2_discrete.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_lcm.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_lms.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_pndm.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_sasolver.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_score_sde_ve.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_tcd.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_unclip.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_unipc.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_scheduler_vq_diffusion.py create mode 100755 diffusers-0.27.0/tests/schedulers/test_schedulers.py create mode 100755 diffusers-0.27.0/utils/check_config_docstrings.py create mode 100755 diffusers-0.27.0/utils/check_copies.py create mode 100755 diffusers-0.27.0/utils/check_doc_toc.py create mode 100755 diffusers-0.27.0/utils/check_dummies.py create mode 100755 diffusers-0.27.0/utils/check_inits.py create mode 100755 diffusers-0.27.0/utils/check_repo.py create mode 100755 diffusers-0.27.0/utils/check_table.py create mode 100755 diffusers-0.27.0/utils/custom_init_isort.py create mode 100755 diffusers-0.27.0/utils/fetch_latest_release_branch.py create mode 100755 diffusers-0.27.0/utils/fetch_torch_cuda_pipeline_test_matrix.py create mode 100755 diffusers-0.27.0/utils/get_modified_files.py create mode 100755 diffusers-0.27.0/utils/notify_slack_about_release.py create mode 100755 diffusers-0.27.0/utils/overwrite_expected_slice.py create mode 100755 diffusers-0.27.0/utils/print_env.py create mode 100755 diffusers-0.27.0/utils/release.py create mode 100755 diffusers-0.27.0/utils/stale.py create mode 100755 diffusers-0.27.0/utils/tests_fetcher.py diff --git a/diffusers-0.27.0/examples/text_to_image/output/sd2.1/pytorch_lora_weights.safetensors b/diffusers-0.27.0/examples/text_to_image/output/sd2.1/pytorch_lora_weights.safetensors deleted file mode 100644 index 9929e734f3976ba2b9359816b46235afbb918d58..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3357296 zcmcfKc|29y-#Bn78HxrHl1zy@WH>lyud~-lN=oxg35}Wz(Ws=N6p1vUStvAWaP~TD zohH#l1JOJv4N4=;`fca*JYT&Yp5mP6cfWt#d%ySI_v>>$>$BJX?9X1?f1{H0k1$go zpSjaQ{igbb`uX^nQemdk1Lw{43pJ%ogF;Q0aZKk2ObfM{8n`gPXY!oDDYHXtY;F8P zL#G9V&I}Bw`;*XlegPqM|ID;`fBZiJ>})~#T@`ULb#Tq!NyUBTo@I{oB+0=tvzLt}S2AY_@ND<*8dp|8vDlDI7;< z2QF8x`cog&{P0K}94C9OlL9mfDnL>V$H|$^Ao(Ly&(!=VNi7^_duLk(h!jwOq!Ny^ zGmlL$d9Gp79Qy5PesrV`4$q#ih>UJp0cv=ONhusY-@%?Ie|Y4sKvD;LjpuT?|G;5) zIRDq19~p%e@g1EMCu)BF$RHd$JFXK?0iJp*zHW>8*PU4-TSOU$Rq=VYidD%GDJh0y zXV2yTOKSfumlRNmWE75_gDtx|`}<=1N0A&6l0sOIqn#phyJ^*^!CMU(g=5FHwO2%T z%UzCS5Z2?!l{2sF??C=>r8QiyWfqQ|lbti)UalmnPm&{vq!8BQ#J5v`M^Oby%Hi0t z`+Nt*s^o~26vLXF?HmVr#fknQiC9fTT_PETV{gxQ zXyg}bQiPgbkQ$DXl*6%i;5*qVAWHRAazsjs;jkMdo}&UxO)E$ZM@dTI*t5+dMP&EC z->o&4OHv4jy_a=RL~b{&AT=BzDTQOtvvpKNcK>@lYAixh2*;jp%ab#&>u*Qo+9Am- zvL;`{J;#470QE`DUuM!CpDoA1&Y9atm#;~ldQ1Nw!_;6R$vAAs-`S~=PH}Tpazsjs z;W#+3TU!PEatbI&G786m%VxLYL~=w(3XwI872s)FL27u1ByAeA>FvyQZlnjmk{(x32%Qi6pad9NBGlBmLUukp3Y=b)1H}NHPh>$=2Skk+yb|Kus@64aZ65 zVP_E79ZAC-+vc+5h?PvkapKn9={C4?YZ9tyMX8ZENi7^FCkMwydhN|2$rB@~gyX~x z`Y0m7n^=|_TyruDhutNygLU%%Z2!JdYHZEPB<$!2moH~>*WZ@>;})Rd(k-)aoVj+5 zHIx4}kLr`;$RnAAQHa zG7CHZ<6z%NgS$B-IbtM}aCqECpKmn@)Wo{f;LxAU!r?jDHr8`%4oRLENhKT}&$jM9 zp`nh_AE3W=0h-oPYWVslvvByf?CwNyBspRvlW_R<>~E$3kfQ36%){Yx*{76>WyupO zsfNRMs+&fZKc5s)mt+)@z}3~yp#YH-8!8zXxKK@nLl z|CXt4*J5j1_f|yR#H4)5RDXve*QV(=hDABq@*3$1{%ggokCG#uep6Ty&#tjy@+OfK zRiJ)@*wIN_b_}QCzIStB^2AHVVOjR}Y)O!-NJ)P}MHH#un7ZLt`^KI%H7Qo|m#NSzj_n-2}cZuU@0Fsp%}`)LZiZ8l#4DNQPli_I3{S zT9xvmwr!gRmgB8(*&rFAqqr_(&NnhiC6BZ(2984mG?dl0jGy z|BpH4X2nMVbx6iw-yFC5V<32b(0{$d|HmUi#lGd=a7*bog(b1etzL6dK9W2+B!jRZ zr$2^Wn+Z}_HIi{y7SGPM-rThOEV*JO#jvm8*w!5Y@ZXn_W}*~Qkz^E>#O`hD_21TgK?_3Gbn z<(5%c5}zHy`%jCj=`pHLk}Hp-kl!zt$qiCiU6Q_D#@G|btjt{r+O@TVkN_{ zC@1Mv=f57FG!vzex+J5pBp$1h`{z@@J(~=|zV>Fz&UDHRYGPe#c+JTuEXj_?o-iTD z-8d=3945B z{k z3vyzQQjmMiHL)%=oJ%qaOJbju$e-d0s7o>k`#!GiA4j@0J-O>|PX2YhHCVfUQ&^JS zABVcgN0MW${f5-NL;c6`Ak73RsxJNJuq<|{vR-FWewJLZl44jC&z5~LPmb%HG@lev zmwr>&7rfagH}zWX&5V=$b?G;R1=+LbJIEcQCf22f*PM*Pk{o#T=Y^XYBl+vnZwL!= zWOoJsX)8AZVv|LI2Prk&U;0g9NzVNGbHehGrV+c6Qr=ZB;&9L7_gPR zUJG7+mOQbNVOSLVnqIvYd^1rBsY@~nOLDTUKQAmFNv;@4AuNbz$F0|bZziaTb*aJq z(r*fT>;$_RsMm&;NycGW&iwjk;WRT&^;mMnN{V66Fkv^~fA3fREmjJtOEL;evbS~qdn^7Q zN%F)<24O*5cAN6op6=g3O{_}|@0Vm0mc*W7^7mflKa%8%krcv$oY|K>8|uORF)*dr z=Wz|befnF%ig@gUctwf+Aw;aEp%>zQJ6KQMX+d&(>Mj3&yxVOgN>UAbzBD^E)nFnu z>36HAk|RtwG6^fPXV1cv`?ve|Te!w@N#_vN<9-A|!LL9(Gzu{@iX_L26{V$tWy|`^ThWGeh*7Bu|K>4%XvX_xe%;oyk9b zvcEksY1&J04cBa$gcUjcaa?#K5&B1wJQ0#QSda4`hjTa6qo{%;)37Su9|vK}uaYNH zQVVP1|8ewmGffI9NK#4NNsoUV^(>!AjtI#dtcRU`Yq06wq}A56g4FP8lSx<+d%o<-#m&c zNHPtp;ZtQLv+o2p&_nvy)h4~DHLa)JaLN8nVM**mwg%hm%^}IN%%qMw53l}r6r28&NkgQt zf+W?jD)#-d2HT@eic&q5Jdu)G>P-Cl-&1U6q!dz+q!L!d`{QtK`9$(WNb0Eb@aun1 zv6-Hx6{NSq(qXxBFP-AhyBoNgKf4yJpcGi>mTpg8m`$gNu9{4 z;c?{Vh~!8jnS=GPCv!L0)NYce-qM#ZN>WXoitV*G)M{_8N}fnbEv$+C-`{ilZ@Hw1 zf+UmFiJTgKK+>cr$-hIAIam+(k8|f6DM){dP}2%hBdbj+sS~l^QI|W96i|<(4%Xwy zew9-mkK%SnG73v_;x*jX`qw0?Pm*VuNgb@mnf+Q=LygELc@$QVq?$Sv`>b7&DtRI$ zwXh~0`^;8ulR^rTR8l8mzw@dnkvtKSI#>_e0&K8l+oW7-T0v@Lha{u09}2Ad`wg|( znnRK!L^4O6hdt=#uO4=dEAS~>!!=tbVMTRE5-Lh0PlTk7I#1p7=-=*Y{}v%d6(pI4 zRY|{xsX&!Hk&;^KOlXYP2BB`U!!+sz{ zZjZtWl2pU0c#Rz0@W+MiZ&9kJk|$D9OP#6iV-s=|N$D@Akb)$Yup&OYNm7(Zo(M@D zb)LE}1jrkqrWK^dmYbBqezussX=`vYvPp>~e?^ix>O8XZHkKK)e}rhwhKoL=!?QNp z5~Jl@Jfmx8GGfjNXcjLZUwf@3_g*Iu>+Q9~!z=+>wHYjK5v)f(-c=(HbqCRbW3!>Q z@k_W)WhV*lH5d=7v>@j{$Iz+IzmZRCDadTO0N*$ph)3w`g|V+^5rc(U^nn3qaJ;P> zoO^2%6c`p`UnYQ9-|tGwZCy!At;6uprw{z_03F)nfeNi@+m_sGtxEq$^TbQ?)QQ!h zJ6XH1vFoFZb3UU8|nJs19!gI=Ldu$)SW}@08-T&&R{V z28m=>NLTR_tw*HDdJ2i#uSSmCSOoP8YRN(01hRIF8STC)7PibP7L0N;C$p!glksyO z;U1c~&_5&r@9p4&VNkmGW#CwHr*kI>Sl2^hxHl2Mjo(DiRa-<42@%sy$I|J(0|!H9 zw;Qo;J%xUT#pHW|5A*gS`|qrwb2waB8&F6`SI#6> zL$||;H@lFClfDC zcRXVH%|dJFS+#=fXxCn{F0eIBP%g*&BfK#P zn~GO@CBs(+8!(-rPSno4!tE*&1Q_bW_9j7uQ@aDcQYyn&4@@E4>;>3$S0`d(@D?6_ z^_hGha*$SO`JQeOb&I(8Zo=b6?ZOVpD#H0ST*2sD3D9HONl1|}a>_ZA-Z_N!ctQuUoH<4T~(<6O1nTan?g=Al1px85f1w8!5o1_@t zK?`#u;P?l<@t_+Wq4ukjv}y4f;;8cqUwOpEN<01VM9)~1IOGW3Qz?)1xqcoF*DVl# zTz8xF$sY-q^erO0EN9{!JA06VMgByBBgoqid-0vZUG$q3DzJRoSmMMVO7{4s3c9B} zrVZbZMw6Vq$Zd~v#A;a??Kk~Ad4EfjbTJp;F|l*uR5LwDn{B`q{cTCc(v{*fpU%-9 z>qSJc=pI&1UQASa#uF?52ZYye5AEDRkEE(EfSNzdaaipn=u7>iTYMcxicQMMfbe*d z9&iR;Pt_xWh3~}g?y5+%uc%5EdJMsqBP>MdT@VR4VvlEZ6p{``>PR`po($BVjbGLp zkzH+b@c84aVCvjkB=Ood@>;l?1p7M)-7ejOUu?TeA`{GrVbCyfzm=IV{~-^?b2G_; z4TgBm$u~Hgp9;m@iUcEW>ESI0wP-ZrHGccl54JtF5T04rig--9O2?eikc4mgjFX(F zz_B_*Xp_#hc&SHAlHs?BbpB{2?4j09vLSI8{h54n?h}wk4&Qi64l23h5w?~RsGlHE zEq5og3!?D>Q#EXS;4uB@b~^%QRMRC}SCH9r-oeA_JIEE|*#gG;BBVbA(LNUHw5xd# znWf$jGCz)jg;6yyz(EauxNsc`qj%z=6T|7!puHrq)mT`x&JgF^w;*Aj`-!Kv23&jg zws_yCa!kEBOMmL5gkN2HOgi5iN{0<-M|kF*%#{TaIPgv%?8cn{mw?sy$isce>v$qg zIp2$DUfu>iCUVR~rtp9q3-uvvcc+U4D^x3vtdP>zWXu)1jT7;d$Gp{`)VO8!>*}I*v z_Ecv$W3d+Q=WdM0@9Zooi5f!BKDPuWzx9H{yd<#L=`+1zS0s#2I|*M$N7AmZmyzW< zCurI+4BGY1BoCc(NV{LYaHHLE%$$s;e^^?Ri#9LejlOZDcx#m)>V^bXsJM|Sg_odq za0fc^)m9>!(q6bAIS+anXu$l}UXT`_Bg<;?Vfbi2XyyBa94T&;sTcwzut%x<+p&R)Nf*b?n+Ww;{$_C*O0d5p9Qy-_rkd`N)m16J)NPa3!^y| z_VJo0Yo6VPu71IeJ9Xw@81$1)LTwY zM0KUlzjG%krh{nrQ8{qD!6bRKrPhnsZ4x6Siu1YvcO>HV-Q zB(uU!!-_8!knwE+uN$?66NX)ZcY3SA{uLS!uXzDitvE`L`{IpMHFaU!@*eP*g){wq z+gHJWm+b}NvxA|@o$gS(#b&HklSa3V$fdKrd}-f|9r%O!7<%YEp&(>$f1IRaNCNge zhf7v|5!j65LyyPSByh`1Y^;?F!}Bf38V5`A@CJe6VTq^Fq z2q<-%SA6X{cI%^kGl5KKBth6Ivz7&v=twP|5w9Z6%`exew z&2dtj;RiQQe}U5%4I+1!OoewVV@X26O~C_kF`a%ymHzC|hO`vE5L{axCAQS}hI2Rz zNcR&O!h(DesjfXExHMOTL^*CGZHimd9VZW>Rgo*%ZmmJ=7f*wF&eNeaPfaq{KaHNP z*_s^4uqV%+PJsP`63CQZv2e>Rdl=Sxx*+$GJv~25LVt(b@Dx}9=Vz+3`@=N2=k)>T zHYo@@4%$P0xxW%G-I-2yEtbHoyLE_?Ub@(1xHC~nxX*~x)P?-M)5y4O58?PF@o=Uu z!ljp%Ls3;KtUYfI-(;+Z{99TO9vuL;U(m$g+b@8tsS`wXmaee(msPMO%7(k%y%1P_ zjeweKJcwO?Ba)qICfOeNn4B7{E)ilQ$=Lka^zdgl$&|}2=ozP%iTmq~rqS}X^o@C@ z@Xp@FxKKq`k}`QDX)iuS_e_bQXAT-ne;BWf^Mfuxk;Y?OwO|XKkYbH3Y9bcfd@VRItc@Wha$EvXg1cH%=I;=5xh95;NgU|4jpK*`Spc3yUuE~Tr;UAxs}UX2wTUYkwN z6V9X`jC>#-xMn2IimbwVb63KjB@wvm*%-XWHG&K|AcPB^nZg{aDd^#n`*834xp2dz zHMHKK3Gn)=G;%ul7W8E86o}N78LYNBC|KgWWEEg+EIZ*lUC% zTx^j7$)QR*5&IFB30`8YvL!^P^n$qc=`}R3w+8$eG9Ct>t07-|+QN6qQM5`ChHplm z#kLj~>5!p6@RVV@pq1eQQa-8+yxQ#^y<+JplG!c0B!~dQg>UZb_%7@vlN#t3(2z7VIL7CSNC*2o2R*Nv@aO!@LKZ z$klBrq$Xhrd;c>OKFM5w=RVz!Ee&$%WY38(!zK+rwD$lSjxFi62`9AjsqdG0RgJ@FAf9{U#OoxX~Tdd$Hi3>=9?M^`-Ju@M}re3G1KVNTi) zj3L9Gwi0Hle?i;w60sY2L6@CdO=fJ2f+-%41^3gd1tH4oNs;<)!C~W25*Ct$xla?} zU3CXC+^`Rc0Bh-`6`!EN^zHP5Bv-M`stqK`!vw46O7KVCN_zII_kzh?4GHf<4csy7 zIoy>3FzkOO11>fkK<*+FSnzcvDS8qO zT}B#{ytl96o>%#F(91Vn5W+jTbrT1{C%@>^awv+I) z@fQ4{`*d>i;$0jmMsRjV4A*z>24^MeND`Mj!R1jlwAkVc7~JX#oo|thQUyLc$QOTV?GKQkG%wwy%g>W_eG#{vCj<_$b_=0w7j#Y5AyQ)DP(BAoMW zI?-!q1t01$F!zKRoM=%l)+>!7ALq28Ee?2Ko40Gpj#10u-97_h{=0Lqz+*I2O^AY? zMVWNQ(iL!i;sMO_Jw|I;cY@B|ttIDuQpo*|TVR0Z0($C^JJ4z)hX~Fsgg=ef(*65* zz%g7qOqA0}xzLadv?&xkD)5JG7F{5oADrR!UlA}c>n-j!)f|4-N+9uN@8|(tKEgv! zMv<+tny{*q0Q;Z6hj-uq1>-;cgq?%8kf)Qc(^EokL%aO8!s2J23GDY38+gp5XO>*S zeXbOW=V{d7;ZI)B!>hc+uZ2;x&mIeM^xR##%U2OSaY}39M!(jiWdB2&*`E(1)ds*k z;ad{fD~V2h=t=8s@F0nf1h(z=nXX7z$6a?9(T-K!QRRtS^sBL(i5Wi%y7G)AH`J7g ztwn<1SLsVS`raM<{*{vC*RLb|%n2!^YSwhPFWH)y7VZ?p9aWNu-VMUVqxf)o?-=pi z8`1RZHJ54q!n<^b$)+T6)GF9%^-y@nWjlFv-8T9`|cLY~y^B9PDNeeS6=dtrwY-5k*zvO)rPT%f=Ssv<)NSeO?^wabr4p8=)o% z+E7g|?chQ-bkc-@IY&rO2NgJcLpppQJSR?n`-&9WRN#{bx4}`1PSb~D_3)M1HpF<} zGIA~8o;VEqFHL@DF zxLJeBtVYAlp7yjL<+I@XrpLJBuCuUfMH^ysQW=-qPQqlk7R*|;hK9*n#N6^Gb~XD> zM)@Fmao`ET^ca(u=$3E8^+LEcMio4B^J>9`)=B1W(+yLTnDax zq$^g^hi3C$RB; z9^aLud>4?C8(rXppFPFHIvk@*A1;ES@0OA!1v+HMuwp!HY$bjd8z(q@DxTz>+fPrx zHe@b-D>$@HU-HhNr6eS12w7vaA3kukkaW4QTkvC7KAbqmmBe(jqObef3D=lfz@U@~ zR6Ffi;Ahw($|=zZbi9?q_a4&@4U{+l_vxl!%tuuae)1>78~oI%PyBakyuni@$@ep5 z)vYtSdT9_t4^T8S~caW^A2Tm9GpvQXsz}h**)VY?QsndhIQ=eZ|Gu_t|Q*MzJ z%*lPTKqoH^w30DL zGjk7HOhNNq^7u)in&?~TY_wriw5W7m5B@X{EfA(;2&zG6G&po3SiQ-b39R1YY;-)6 z;<)IeUuqf7ard~Oc-B#o#*0Lzr3L`j_gzt+uD?VR<_!g)(<*Ak*m6JYUH4FRC#Nq)cU%4qElV?>!up|S(6G5mluYWKpRq`H38KAwkgcrUrM5wR<5Y!^;G74 z-_49eCvDJmlm$Ay2Liw6$<(E5Tp;%M1;?jqpr?gP!H$4zDq@{Jf@N7$1@|)5apGg9 ze?)8WXrV4J($Yb9zV2o&-@e4p20^0yOP-))pDt*7$c9=T_KX_75i=K89ih&=G6xw2 zUqpL{bVlEE+Muw)myGF_Ya(Yhf6Bz~nRIkVbV>CSnHQPqdB!dgbSkgTybuBcpO-@aysz3GzVPlu7M8p6!Sxi z`T*U%9_aCqIk}b#YynJYjY=Ompsk{=3k@OyLWEc23@dutdos&keoXo4y@Y1~qJ9Dz#4rXop;lRR0l`5`0Pkp`rjwx3%0ah>F(YFqpsN&lf zM8|IxQoAgCz-*IFK$2SF>{Hzye9L>zeAw`e7hiIWdU>!N7+dKM+^=U+9Y&vLHio8A zr+3_CK6!A^o!Vmll0)XywO)MzS=HxmyH7Xsv*?% z?`N4}4Qmv#VIc555QNrt?#fKs2&q?h_fy;W2bgb}yZLR5PEs$rk7o|JXoK@1=D_rs z9oUyUgMX8+Ez&!sjI>-BDsIDQaPGY(#d#n==K?aBPKMgx+|xFwETjjTf1*2jo^plC ze({E(LR(Xt`)Gk9v-E%xoI$O=td3kg1~NNDrYRElReCX!KhAd>Q)F^V zRC;X#Kbbo~G}!wYbI*MVlibq>y^C)LW>=i&jmqqWDi3X-F82D#UzL|k^|E=zv>HbE zhpxY3X0G%?O8a#IZMBJazl9p&+;l@Zxq}gNV-iRJeUV==ME4$UVw7^bqx&=Gq1Ll3 z!2pX#q6Pl;z$GG*abG!Wa!#3{uM6zaz{Sg$o*8XX+0}j2?J-3ZpR@pb zdIo_0h3bIU=^QgCNtM#ww}~=ac!FuCzEZT$ya%W`6iuBKTY{otmMHv$0m^#X9i7{~ zj4Dz#0BSh^9j_KJ*#p)#9)njVQ zxZv9Rk&MZt01>b30OcNanu+J@qK)b5luels+-=(v_`~+d_MJ1Bt6Ie4y?jnhp0kDW z?hwyh==*>TAfhLjN?qw$$To%$rV*B&EgGi?Tj>^%m*`#U30-Z zHT1>3AIhC;jg|~^pw@eh2a!P#?YuDtXz%c168Bo6PRnwr(5?IUvwtZw)-^k+GYlyqV&)-t-B08Ycc_V>~YELv=EdU7v9guFya8x`$ zjR_f@L#?vHl;M00Pd&R#oB1*>)XsrXD8Hm++k|(UR#j9O9#v>;xW7_*64wi5-49j z1e|R9iW%y4nK?E&kJ-2^h{}jo1JQE|sI{wmqA2VBlxV*(3JCf}eZH-Q;OLEveehT2 zS9|B&VfnqmW0O4QxmP*$b$1R^yzEGBvZx0b*>x~wc1;7FuQmWHzO+PlDt$$T)?Coj zN(UJ}4;8f<`jKfv%1`BH+kvXSMhF;hp!_&$xhwD?DjA)ooK`NRB=gI8j`z1PA;)ac z`YEcY^qwAAwjF{*@u1vQnb)XRzlQKDRxcJ^QWK)cG3BCQmHX6YMjxntG6yrea=;TK z8{}TKp3hW2W{$TV$GpBch#!m&GEre7YJFxLQ{C$aPp4xv@1x%!zU973YAc;eWiM-i zf-KKcUhgGTv2HEDjgCI$-YS=}9dVI*zt9%lSv!c?7RcxCXs5v}cDO}}Y_~JVs7R(F zQ-){0EW6!9g3Ldvf+mHKixg&E-JhJwcLVnD_g%F2Ba<2+uE z8Ew%Py;$;ABrtNNhWSjPmIdaB=Ic8%(>cAV(N4Gdd1-p!_O3q6iBIZ)+g3km42fQXf-8+xjMcr_y|@beN_BbPd)qy=*nm z>);}$bxeQo;JXzPwmRoLe47KxwpSOuQnKQQlw6}Um%n5J?rx{1I0rL7%ezqv&Gpgi zDiYsI=XC`I<)Kk>`lf;9++j z5FH7j2E@CdpKH5;`xQH=8LvB_Vx=)iyvP}Z;cwKIXj|%YrW-mIXb4)(>q5=zsKV$J z3z!v}rA(ynWU#yx^Gl*{^ABz7Dat#oha}ZqsEz@znXkPdGsx>J6IO0YwZrku6q_mF zT0aQ7UHi%ee_qXGlssdKwv6QEtExG5>vdnSB-sL~ zcJ^Y%j9N(D81{-9x2_jeW3Y)*n$r^XZ_fue{Gyq_%4jCGW-D_(s8aO&ixtp&=Zz+2 z$1!Jolz`Awg>iah2VzZBL{+2Iz?qJ-knw{t{E)<+;QG{|qK{WLQ40@OF{Uj%(M4F2 zdm^$kQaYss?3^Bm-n|-%I^PHfJ#9;wirdP_@Jo^?`?U?Q7ki+7F1qMUm>aXUjQ~{c zaz*1?DWe4mDokc8E|^iC$kco_WcO;}qDqGYjPBTY>YMs7YSONUBKGYtFz`-0)Y?m0_^1U;ef5#cnrP>CaZe~iPEW+MD4|^3`+(xZB~(R`8+a42 z7;S&94SL^J7GdF6#%`x7|9~_juwF6N8p$75}sZy2~*N8qGQ$Hwn|El|UxDlmH+sbap^K4tdnJdcmgx{Fb^(p*Zb!QNMTEwX59bop%>;;S}Xta#?iJGYF zLVb-Xr<5mUQntHPz}fj#QiC4yPG9D+GFMDuF+Ub+QYtylpwntI6h1>4yzI~$%wIN>+M@CxH`}r+7@+@CH1cvQ zbZaqSCYN>vBlK(;<*0a(*TpAH&8is6{9YQ9x_1nvlkEqFZK|R!dFmmd=6WhW?w9EJ zZV9u+{1CNZ;|nToUm+EiYL7e$-;1mcQ=sE%4wHGr3+1JH0*#wi=u(s#5SZMjW(Mu1 zg7@`dUTN3@mEtTWxa9%H++ht<{L5G5F6xXlXJs=dmYRYe?ERGMAQ5w|&lkQA8=_g_ zm;9w8KQZkmwx{|(TZ}Af13}v2@nGb)Kr}%s94r|(6vgeBi*miDfXcnzAoX&=sV4^E zsAukM5Fb;*{2ch2nPoX2wXL|oPp%vQE*%a=*^B3((bW;aDt#71K zq&tJ!ogIq;|m_HI~FGIT$ zxZvmQr2wQyBD=^?6h3PfNK1_bmCH+*#KgH^_s>OW;;sy;!cB!5OwC2H7gvJ#=bM-f zlp1ol=nuXP<)X!x{ZW@%Z{T`o3Fv}WqsLPN(a(=INO$ULC$0=nHpwb>C+_3;TBg26G5ozSUJXX$*&6uCZVz?v)EtyHZ3O69HVS-{7k;#4oFyve~qIQe~MyN09{5%fLdmIWZmtD;bw^mQn_ ze>p$kLEnaZu^j_}VZ=1>CUPSBiKZjX$J0Tq!!l4bYqMy6=^PZG6%Ce7 z3j#-8PeFOH<51dMZ&04O9IXD9%GAzYi_DWgQ?Cpsfr?czjN_zG-3yG@ zt{GtZu)!d&TLc=AJ_z`K38Acg9l@UaLqM$>4d#trfQ~&H2UJ5n(A^=0d}Ni?-c^;_M?ezQ4Ssw9|gD3n9d&62~MAv3o}l4BkLdNK|`Yi z7^HqraOaZ>d_un=6lgXV8DVBfqq zv`bI{9yIPd9yR_WQ4b_!&Czq%^XF??^JoHo?NUR`K9u6NA@;E3ngfp9(oQUnzaiKh zw~Ab{>qic~ZAF$>n8UNyHi8iKUNAs)4e4;`3T&fW4Cg$$D5x&aq1|4ZLT>L(*w8+N zTr*3jAB6NHjLHPKec^ZLaWIP1AY&LEyaayOtOaexKE#6MVtT@iw&IxvTX9E^k9h37 zQ7|aYK;qqNC@Ju@61PK{DE3<>!5=@885LS^Y@e;9=E*#KA=QPhWuHsf&Dcf`UH201 zT$e#KXUF5^)3(uVU9$yeK6Zk^ZAG-kmzQvW<3(rgRp@}b?7jc7b7u$+B zufHqEC_744PC5(|Jfq>+150ShbbYd1D;??zH<3O)D(Dy5BcYM-7(HyjQurQq6@GkZ zgVD47Y!2AKbw^i{T(K6MzN$4*a!(@u!CguA;a6m2I1gg!ByPTUNLctH>GxYp0e{jxBk^|mdMOy5T)Tregg+ktrD ziOaCJi9Xq}|2u|TpVR4XJ78hgN;vgNIH^tQ2v55&CIxSiF_ou!J!9 z=J|ADqR`-J@ZxRQK7>4r^1YjK)hHT~GJ4;)mmflQ5CkL`D)TN19wwmrYgdvZP zwV>N{T0y#)hLWfvj_~wkGhytPXY_>G+ac4^0S>Vclk?F<;tl!1Fe?uUetkPG?pVkn z_u1#?->>8&lg=5~sS6gi%uc~&DXQGwFJlGHTt-1~$=!s$GhLgang8`nG z`ix#y)muQnOQr``>5_ZkCoMR<4KGd2q~0;RM4KPDX{ltC-LS9>tNrE)gqZ$07F(psIOm z!C1p{xXa!H_}HcQ(7sq(LaUU)5B+Rl>XXR=ogG1fPfmsS>abjTGFOD-M$e*K>L(GK zXe=)BeGNx?o)n*paKHyIslazty@+2IK$7N6guLrhA$5HdiSj=V_r|x!L$BEo^Y0h2 z3P%I_FFA>A)Y_6`?X2ODBpxZ+`3(B-49QA%SUhF@Anf^IKIA+o!;XuRNJY|QvP^hX zkYN)?E>HBN%g9|^b-{-8T(ORf)9x%h-}(sb_h6NvTk=g(#s7kbA8Lh%@3R%&?V(Fv zjL?S%cp;>9RyEPeN+Uno^dze37wO#v8^mUtL~xo}8p)Zzm#hyzOpoNuCtZ)2k#Xn} zxyV@uTUVIC>TWS`;}1PD*hv7p2j-JIQ@_CFp90~qg?Zv-xBW?e>J0cO;VX3MZ6I0W z6^6A}k0)_A)da08LP>zTHx7K$k1o1)hnS>xg!5JpA$w{h*zXt*zJVFyhleAH+l#4W za_`eH<3cTVe3C(yWNi?i-`*K6zOM(h1{jg6Yc|8ei50=Lf9*@SR3xD8*PHEWNR0Mlh zrPEqDg*d#|ar#JkNAkL5ieQOD1lg~nLZ^1PM6Bjzu`_e~;I9);$YQVY(DoD`>MUH3 zA1*Y4nuj!Ci)CNn)`EOE%X1Fg-Y*?q?O{oHD-Pqz9*OYl;5HDNU4oyIEQM9IJV#2nOqCoj^}?)Aoho&Vb|#WWYQrEIHYI--OtBRa@wyQx_)vy$r^^> z=~jA@DP|csDf9&0oxhBb56-kb6GW6<9AIF0f3i1uI&pjNf@z5|nfqx!S>!ib_{{MY zj7Xb^Pb;}$m2vDlDAhLv4-TxvS!-0t()7*r)P=do`LZ{8JE|KVTGO7y7d+sP?YD}& zdzualRzHO1XI8?iH!i^aNfnT3V-KQ^J>?IYpCa&CUPRlxUrj#LM2f@TWP>(wUBm|+ zT1qNix`{^)H6oUmM6|18OX42o1O?+p;eP55;8qrjS$#uS!RRAbfgZlv<2I_zSTO17S#O{^&P-?#4xexH{k=<`&Qcn-#7 zUB3ggSvzGoNpuX(D0vFr>+1^hqVn;No7;#%`Y{};epbA5-ABBRzYI>Q)F7J6jL6)j zuEZuPnM}}GK?gs%L>suXGp?0=;7!jC>}F+l9D~cE$GZ&%+59 zL-2~R0WkMiHLtgE2Ozq387r&aBcVzR8JaW~emk!v{Gs+5FMqa*7M!u=C>H ztI3cH2MJGnO;FxjnXGtsNuc~PjMO@vgT6mK$c@_W6;Jm)s5r9wrX`4`{fDAQf^Jcyk6nlmc7{Ps45-q9)l;c@6~Mf z+)b3aCgUh?ABo$7t+Y$$EV4r@kA4;yLk1Wu7Vnz_NJh)|;-x#Ippv^ey(l=Bc8q@s z2lf04@xhz424^2_SKOIYS>LAv`Y*uCUKmR_`V}O+Vo(PqU9;GU9|c*K=wbnlWVtAU{(l^uc|4Tg z`-iO|J7uk`l~9Rf<{ZzNB1uI>WDQY-s7TRfClX3Uls1%#EMd->=cpvKD-uFUl=Nv| z6!o3o@Bewt>)Gbq_nh~2U(cIcSLqS%S#-!W3Wfb$NZku$nfKx4sQUOJLtL~`vU;jEmjBr!|ylrz~P4kScoZT!i#3a;VF3T}C;)fgO+RWv(Zf(Ge$erYp1_ zS$X@RCsV#N6Zzw4@-angiyy0~Nn9toJbR7syvnImH9&J}?CFIPRmQfoLQs3?By(V- z7L}gdKtAT~rn9ULqUUF7(1imJnYZj@RASx53^iJ!QhA=BzV)df>vb-&*ZIvTR^LIq zQ&$B2nPqh5qob(TE!8@oY(S1ulFXN;<+Qr>h@hCpqy6Ka?BqG6^!~^ZRxbJ(W9A%! zG|q(5?kyoyW%oFp)Gbc^6XfZJo-kC`dklROX+zE}87MJPoIVz`Bjey!)_=H$alQA8 zVSJX+GZ!|ad=Zvj`D};6>!nd{MH0Gyae^v6)nTTGBr!YI9<|;n$FQ%zJYZ*83#f|0 zEmZ29&d5cEqmxez*l8PUXdqV_9sgL)cFUXAM@G4l$%V_JpqiC5-5zVcm(eR`X()vs zi!>j-VctjHpbn<1*;(~#S<}u5_S)XlNaNc9765m2XV^iIt~}{c#JP`Cku1$mmQmvG9kn5!dYpjc$2`8)CMz)Mwjpel%)+!$`^?A`XbH1Y@2P40H5=!FyAB=P z7v{!74z3q}RW|lJHH1S*%Gl*r3Yc z|8}hbn@_ZWZ*h;o%(PU#%x4*3lAX^D51vXkM^rqA)R!OOWFp!o1@u=wp+AbRsY z_!fT*9QRsAjtuzV&-`&*ypF}*((c56sx)MtdV;w_ZQO4aE%>&*E*#QK0E2c?@Q}#| zyw2t%|1u|?|1v`r{2G&mKl(E8xAiZ8{(g)4yDExY1??sH*~?_y7)3evrS*u|%Xx6J z+bnqNpgz=^kN0=Z09!!; zSSvlu$-f~^UWLvebuB|!KFN`oJvD-1KjdInXBR+oL$OTIZ>#r;LO$|%9u~QA4Sar@ zkDFHR!#@|OlLGN0*w5}EHZk18zi^9(#S5ii@@-LaezOKiSDQ@yqXGPsbeu08T920) z8{kgIIi$<~5O`EL06xrr4h-sDu!4F%|EJ1-TuYa~{ByCgm?|{$zbkye)(>Wo!g=$^ zF$qOj_bm_1&-Vgy+iUSez8dU~)Px&s&EYYNbGY$99*EB>27>1I;K<$}%zFOg*4s;g z&Ii|kbz>Lk8xw=}6Mw&F?D`TPON_((gn8x}P`vyN(Aqfw{&ov+ zw{$Y-znX|gIq6vY>}$|>v4pSnyMwQz?1SI8ZG&A;mVxst0Jy)e1Sl@-4r!-iCiIsZ52#kYWySL!2 z>8o*%tq0D!F9z~&CqZoS{} z>Y`r!UETq#==Q`-GZw+?HOjE6!V2o!-o-sW&q42U9_&{91H?y0FjUG1pEvRONps%u z8zvb7&6qlT+a&`_x8>q9BKLsPN@-HQPL=qX=i+yg_wWtz-#9#071wFy;HuVn@cyRD zc;ZYt_K&%Ve>}N_GcU@+3APeAcgBL+`>#O2GkfsHzY>(Z%>WsxY5Ye6CppSk$PN3Y z^Y3)#aGfRQ!BNlSV2wpBFkLG}{NvVv(}7yRaL!rolC}0^bFB*SzO)`cZ_dG$8$aX4 z93!aS<_e`%IZ)PrAyisBn*{IugL6EO;(v!mz}frf@Z0f3ygxM(qnbpJs+0)E*G7Yc zEft{mj*yRD`xC!Xr9%F-3cNo3rQW#jGybr^lB{!FkKH<~NX@Dzp!!A^xT2p1K3%XT z9kByIY2Qg4z0!@Nz5PFY;*A=K|9%Fmh<4+}x`vQ8o(0C7_h7;B6zI|L0b5oL13@2x zN*}gD?foq7eRT=jcqHO&+uiXQM|X0pJr1}Jy#u8o`C$LFQSc$m7-z+v#K!Ai^Y^`* zPju!T2HRsPxBk>+?j>0xICK(`6}hH3t%pmx*KC3ZK0M~1U;PB!sVn9-FO8{hl$3?q z`ZM73CkC)J@+XM&lO(1}N9s2wOmG*M%^=kv4*y${jCa2}fHV5aK=PD9{*gdge6DGN zJCA1ow#d&Wmm7?Uwazj!6!{x)kJ^FaxdWj0z-{3F-x!|CXu$#R9uSu|4(#e?kl!|O zAPQ!J(USz&IoLyNECVj4=7HZ|mVn})QZVAWK90G13LiD*;37{c*nJ^{|Lk6GT~A2> zF1j5BuSjT<1qHuwSfB<;tN4wJ50rz$Tf+F^ldOS7#|^xoS^^frcF-*GSyLq8rAZFlOYM2UPxmwUoV^|EZ+!zajeEg=-OgaQ+JPOmrS9lL^B^V8JuJ`gu5upLt<3v~ZDxQZCKFQ%VWGzJ49Y zy>f(Yz6;rsuiH}8C<`-`M~^M5~y(?|`%p52^=?liOWaaMd$8P`j8% zr2YL?sAXA54nqZNyeW%Gq07Cxxa4!F5I(Dz~pXsr2)--WBf6(^>_{Xb;jn1Qf{^l`DX z<5w&>wHjZ(5(wXA?Z>t+=D?S3qrmdURqlUID#XP>jZB}RNKUKFhq;w$c&lzV-thJZ zUf>;txAz3_*F5URW%awTiz^?`OA*e7EWCwlDr@lcpGMGeiyWE!tQu<=NfRUKnK1L} z8{8r6AJnI55II>15_)wu3>eXX9v)+0PvS~==YcF4byOi!?oJ_hCM==Jtxbds`jCkl zFM?zTxi42sVpU@!t9Mis_gU-!+hxVzAB{vH(VPhSwZg%=jx%6egs_&Sz5vn5V|cUF zIM+y(kJU!~Fzap#hu|TgzF`ooRB8b8Vy#KvrH9(*PCn7>FJJWp!V?8rb&dfB=6Tu>YhsPL*B>!!Ef&G3m81=PM6*FCPSkMPInD z)b|6!r9*(@-GINj?!!GAU+~{XCAiLU1WZ5X0}jtC1&jDvu=LJPJaqFr*40rUQOR!n zPtEW6-Lu%*Jtb#waF`BOlKI0eoTUQ-D*C|wF?)W&PzR@f{2gCa;}o{sp#!^5dy%-! z{^Za0&BWxM1+>Tssn@$?%)R^L2DngY4gL!}1C}hdgfE2b@{?vwpd0-Wr$_w2uinVv zgO&2+;>styD$xF^9ogzSn^>qDkoot*fedWN zlkd#Jj}_AD<&x*X7T$AQ8U3C=#XuTA-eyfStd!uz@1MZ6t<&+(Y)5iqTRZriTo3a0 z9tMxD*pjkv5qP(9FF$Ip0hZilPGH?~82#1~a$07=i_6U5sklB)tC=#Cb}@oeo2L=l z@(9n(e}oUI=#gESvxvExutTW40`{c@0&(^MIC}9k9$ub>=S)h*f+c=@)D*!GH~#FH23OWQ*dqNWY}8v4&;0GW4uZYysi7kmnn${E033Q ze&oqO*I@-Hw^ABbY`cU*T>ashDrw+n-v-+I7vOHr86Z%`Aim%ia5L9{BW4k}H$4~M zm3oDXRO*4<{tNi`hj={OZa41!19AL`3h-WWIq+%N!Hw*!1ZQIkxaR6C;Flf(9-xCC z`M{nyl}ixGDpMj}#UZ2Q;^EJdSWa8DyXU?mE* z)q21Or~CN#@LSLqEg@t+Pk}FLA~^g`30D>d07fzlBz_VFzkfMHXqg180vnoo%4;PhXsrU(FX;ei-v=Ho~ooGH^+`4GxIJ*qxP!uD@6En{ujvhmt8lK0tWWO2u0;XKHFkUioDJ}2qI z4FgpmtamSIE&0XGV4|_wQ+K@geJ*w%JQ4tJ;D2%3ac^S!R|p_oEyx=vbknp?S==SMzoo`D0B!s*xb*p(cH*a{G$VI zy<36f%SE{>%$4{Jr_SP{cs;Vc_Af3>?Zi_y?B%oWm-y8yZLy8(RPxG8!7BFCJC3=$ zIliv(gCASsj<1`Pft!yrK%=myc$yZ#HEO8<{|Q!u$eaKo@V^4&*C}G}+{;*BsR`Iv zJCpl1W`II;laeWzown$aP!$WRI@Smxp_&xL(Z zavSz+H3ZVr4&Xx05T12Gjl6l50z6xt`6u>dW2359*z@imd_q;8dtO-uAAXdD)hep6 z-?29Sh*=dLuUEaU6Hn{C4JX4>-L2q`%15we{aWJns|y?mn+~^qbp@(_R*+x03n6U( z2)4C#fjPGIoZM&W^>0-rK;M>i+{rUn@jW`!$%h--L`qSS+<12ZXu2IBLrok~*BeaK zoTc!MsYXOeIIHOPVjkg~e~8H(88W2HC5N3X$>$?Ae1nod;NDwx;bfgU+}`|JC8i}}Vo zEBJfkC&3q<;v^@l36Gn-$Hw1}gW2i>cxRa+Nn8F3`(6}A3 z!3yL>MlsG_ra)Xi+591NoDwjh^AH?dRl&DuT);2g*1)ByqQqV17fv`{i*M=Vac*vy3~oJW z<&Gu46Fz@w;P}}cc;s0O=kD`C{8-58MJE`*+eY8O=+EaMf^`NOvK}C5`B~teKLvC> zqZ2I$o`Ij-h^>KO>;QxJJo^sR8>@l+q7iV^ zZ!s2GGr||0DF@1Z--5{OQO<2i5fbC^5{rm^zzVzOkY9Q?K#9sA7%G_tD??()o{Rb5 z<+ESh!>)c{;Sn2wk5@bj{hmWVH4UR0Tuo-@xzKYt!N^hQS6RPG3Y{<%p+>jL1k;~y zMuA*Knk1fu_RYA=`rey|o}cRygh?D?HfpIM~%DR0v)CZWFzb=zp7mcq&GvL#PRjXFTzBxC8{ z)e;S#;{Kw}oW*F=EE93E&C%xin=tvO5%Ljlr?>h=&@^R)7U=||QImh{^zt7}gd=5- zI~TLnLRLiVeLlNcA23V*it^4lIZ+4mXRPu_4l`$@9;NGMG6&~(v4YeWOfx7#QLn0z zRZs~#Pw1pLd|L#S#*D#>;2=A9p;E)TZ#-hW?>hBqf@t$C!pL2o$z-YK(_u$9y7TJ^ zrumc+HEmf(zkFJV1ah-z@6Pi~$K`*F*9Iv#HuEl3h!{mVQ_fTFCJVO6Adgx4U4dQF zeT_XgYlxhkFl5Dc9%E8XjnRp&g=n_HQ>vB_N6+di^12m7QTp;rLe}^j3iW%3hPcJ- z?3_2&pZH=l_E8X>{q7<%anYtP!j%})FC|o>Bbfd7@IC#R;wxn7+!;rE#RfO^A?C=A za`u3G8nU|fmwjz{k^b2%#UlOP?97ZKXi}X&n)?SM&36OLjhntm=KLXeOXLC%-ckm@8W^jB>j4SCbba*C5t0>=t< zjDAJ|tsbWzNt~ZV%WRQqF|wC(@9T zwSue8z3f}JPQly4zs$+?76|xuqunn~A@lMxDC1fg@=dHj$;%7bhz&E5V7er8Vp`ABB7x3ttGzqxBEjh^2s4#eO*C%KSRE1m&LzV_(|tr;q$UpcC6}p|@!z=;UWpG}f2I3|Eel z>x~cDQcVZRp0CyLuy8v1DQZvqjb&IveF^H=`J36ZZxye4 z*o>;AYcS(C^3iWu4|H>P7K-<_K_)~MjT$DR+Yg^INVbWM%~5Ee2VWx1<{~=iDD=PG z9zyG~jxhh6?OBWPB6QO0CYas4gK<3h7j2t%4GlUcp{sZeJ}u^e_I^6e{)s(@d?(pa z^^0qeJG{kCwRfgD9Dnp~k0T0_or88;TQl3PbRn7BHt6<94$C{r5xPMK(W28Fx_JFM z8Zmoj!)xW+Z1MXt*nY(w#T2O1e!EO+`_7t~J(7X!Zm(y=n8giW*0ZQ||9V#H*fjRt z@igXg@@{I{_s3dsn;YH2-^wPvmfS8hsKniqY+;>p`mj*vZ@IO85E#ExxJ^6(IRuS6eZH@5q}u(`G@E^xqSAg>tlB778P1=z6RAc z^rO3s5}LJI8>y?_MpJl|tZCyN`qtqi)1g){+ysq5+finvnsvY- zH>x}&M`ya03DhL>5d0Y~Xj~LZ7d2j_8&@r-7j<2jaj)a-aFG=4Z({{l?dHBzWu3=OT6MQ07hZWosq9-*mU_UDUrL1~okMM+^3e^U5up zX!EYg=#NJXdqMj@I!UAqIc!XzJ0-SLHy2M-exic5w@I+1b|G@8I>>sK<|ApYn)l`=%{cR%E=f3ps*EzJc8ahD`P$Kyo_gqSzbvor;2lA}sU!WeE(^Uo zvV~?>iJ;f_RFGgv9%{PSN{2iDv9+HWI-~Il4T-H}Ir1Eo)bN5f^qm*HS6Ggdymldp z`~m7L)X#bxNFY)F4pRH&rfBX;XIi*p1>M+v2&R5hq9FyZSCJ6(j0ApQ0es zOmusxHEJ*#Br%8fvYW3Yv$8#VtnHbb?3QEOnQz}BX-@DnTH~{oa`R5nkj?4z)3pY6 zz4m{CSHOe~2r@xGryoP(-=dJ`0#l}a^?CvKS{lt+Xi10H{9+#16`))r1)6MM%4XWP zqITXb1PgT=Hd${#LQ$19d|)NGX2~+U6YtZzpEz{+yaLMGpKsm1?;v_q>_!7NjqzVc zUu1*kyP<*SN101MQYgNEod&3EXL&NFRA${|8o$F#P*)H@ufD87zY_CMv33DoFsMMA zoh5i?x_m~pYk*ZWTtKCZ#i+rjGiaB`ES}+@4K3QyK#eT8%<3u)diAR$3b~s}`B%3f z$4Qm=?$0Gq>#+eE=lE0YQhl^8WjV^&*ur-IRigeHR?L3RTc*nIsI~m5ji~zX8M>)n z2kE|ki}+LA8!E+IsQQ7I%mu3`N<$@S^;|c0B7B&JMvI|pwK00h(gwvuEN5(bpU~JW zALhU{Kf#)|_tf)u472|CCn7)j6B1O|veO(YnHlQ>=-&?Yh5>mWMj?L@FXvYgv*fb} zTEG4-NJ7>FP3CRU(BVQ$y%|&vU5JED4F&>hs19 zRj`5f^1SlL+C00WnW*P(EiDV*fXV{5(%%bbGsX1}=s!0PdhXO(l#nmgaJf$C`CC24 zFxjUW6c~qQgojfvW}JR(x3~VaHk&ctr^VXu*-j@NzQoELw?ZS!F9@XbkE6CI5#DFM zIU6fv9u;3^8S8@b57^LuF^_5M z#X&*T^+~*SkyFryb$*Cib>2)V2*wYqHz&%6lAv0`x4_| zOXD|oqP+@D)UKtk?bb0So4#24ib|rzZbnG#gE1@jAqx$tP8I%UA)U2Pht6gS(8RDK zv;X@_R;y-&8J~R$U0R@tg2cO-ez6*QE4Yd6@R-G#I31y`c?{|57^Ck4OwqXUT(ss8 zgLZ60l!VP>?eDmwK&>HUe`^sn7Td`@jyOqIXvSKfY?#f=){$>meA5Vx-A-fPS*Nj1 z+sc_&w>{auQ%bzlpbR9xte4G;*~RWm2xrr`Hn6e$C(JL8m1wQ@d1QF(BDxwV%KS(T zM@MxJQNctSy``d$?zP7WujyxNuJ|jaOVWxlYWq*H26v-nl4p?rcSq!=xD9#8T}0Ze zMR{4)e6%zF6YC}GDu_IPn7;c{j21ujwXRUlVm3akKx-n;3HBE~Vmho9S^twV^k8qD z^(CwMH0oPDGiBKWBr#2SYZn~{q!H5ys(s7UKC}n_GhB#Ei>tjQ!7z=!x!Yb zy@XZxSIb5!*U=pTcC7jIjg+j_p%3p@(Pv{D=wda>z8IFGGd&9ETH`U=P=B5cKRus$ zCs%^{w-hmE!$U|8=_09$azRj-bi>ZwXVC*UbBeO>GG?_;!Tj>$EIr5~?#MT`KkX+S zSTQ8{KII_#T(}KIKVQbn-q%P&`6>-fV~XgE&MF~K?|`n!9s>uHezUc|qNsJL7=2|x zty9y!qs6^HX#R&LD%)?t>KJ*^rkZ=yD#wieT53#Rf7yT@ySLLcso&^fmNq*o#;_-Z z9?Tctyy%%}7RfJ++_c5|nPNXFB4Z(Bq}EP&z83J}Xb4N6KT&2vMM` z4rd^@r5d~$Ygf?aG3o4PiE}j2s|H?t-b}ah4N>|o(T1Au*XX==Ur?)q32&`RJW_f8 zo$h?Tf=U!$Vx73%jPg+>G?B51YV@Sgc}u@j(cel``o#qb=bEz1549rYzth*Q@xTzR;^Tp=@z=4l~4U+ZmE>8YU{h10JB^a;ZR5otK!(K z4dP75mO5*Tqhgfg%|uaq{i%bEzrgFSD5F?;59JN4qA$K(t%2oTkR#62xiTq`GP% z^!BFt)O@8s^VLd=H5*k!CnR5?rNie?>@Iy2YIBusYmG-emVX3R3#FL^31D4+@foWi zp94Bos@Th4Tv_X%MdHwHm&N+$~BFm$#Rm7tut0$BNjRoH*Jp zt;L3NHnF_v259|;pX}RPlI(vGiPX1|Wq1y2(1RBu=+K^YTDxop9j$H?w0@o}Q2&|4 z*x6*VE_ew2$+kci3G3+r=V2sz>L#=Af+`{{+gYx9A*-db93{UXglA=dOix{YjUVc#8;-pchvey>Q}ZiTb)s|u|w6vub6zzN%V`&ab^R~K-boa z@B%J7pqrCDsq`IBB)@wly5S>5&)Yp0_+K|gl`iwB#Ffvq=l&8@7$CzlTAPeoYi#K) zcS&B?iF|tGi35G7rNa38Z>1lnDx<~f-Sio+kggbc#CpJ3D$t0gG0c8;#-e!IE*?z} zI_t18WnY>2A1CSE9T_Nksxp1E^fu`DDbXNv@R-0q@By3ihf6o+ZbFRX% zX*7S6BHOkS({mNB$f?hQTC9#l5v&iAKHSVSQn`lRUANfb;(1h{($9E0n)0swO+yA- z4H1t0gN}>LU;6;0o6$f_FrVtvm{ zr-$%Wy4>Ut?KDY1i7h@zDSNBn#Kl2WdQ1x?|J7^A{rip0Xr05$JoymK+o6q`d!12A z+<$_;*XP&`y+_d2?77J7q$S-rI}~x&WzkIvr>OlyH8kU&3MzL~q_)?2D4wr~KB@*F zKaVUy?~}Rcxt%xRZ7HC~|8~(qRaW?&+mS}kandMU*XMpsVna0~c)n*&vagfvtOI&? zQTIosXifiCwr1TZeL391@}FE~RyQ@H_?B`s`_p;aJm3QjpL5`hw}WK(m~qr)J=6F4qL$C1M`TV z!FPOBvK0@$l^_iszxl>h(&Sl=9Xy#j8|yoT6NR8%aCXXOc>0+I{P!;#-!m13!I`0? zOG}>I@LvWK+efiScN?%OvW9rh2OPa?4J=Om5AR=*i|ZP;!l&o0h(|IHCTl6c-Wv@# zHrE%v;S^wr5*|n>h{v@r7JywmU7Rd;3#?b00A_Bs@K@CZ&SDd5 zxGm!X=ZT#?r&(DF=4x$$#f3$HXe=YAFWljJ|5yU!t3*g;Hy>|FvnT6=2iE$gP(kq^hAf|7zxNQR)b$|v^`-jY(53JW6=ovhHZZVTjTF8#K zl%!^{)h9(wa{lxV=ppAwB#N}*^A`vl*J}b#mLDKG=ZkUMD_`7lVLGvzbP1IHoC?qQ zyx<eWRVvQc{pS$O;pXtt4s%iC9B ztFJ0#)2?aoo=Gic=f3AOf0_qgm8>C3XT?d^$P}XYUl6&xZ8h=F{tf)**pRFD{ov*H zsYI4u#=qihK*1M8?kr`6dE9$E^qWgckM1WgTyo^xrf!ocu z=?Z{_b;4|6+!od=PK7rI^>L2D1dxat=I$;PA+6g?$bDxao4PNNG`VE+yZ*#MjnzW8 zMCle7yL|xsR9a2cV}syE8xvBTC4oi6EnxAmCJF70fC+nz;XwBQF0yil&#u$~J@tBA zRv^svZXCw0eim@v;oID+D<2A3fLmB$zb#DKLF>cin?`5L5F$Z-Eg*^sb(=Wy6JL-12pnZ%rJ2kuE%_@dQjpkQbg z3B1+-EW!_h7Cjz$qtXM0_Ksnb6nAp()CsPB)NE{-qff{R2l7Ip2rTr^;8r|L$6uA2 z@#ItU;Def%;8d0g`Pml;6rUZ&-u)GrCXZNc%}n6$y9TiCqKCNOr3xq6Xy666C9rm! zIl04gA>QvV;>Hu4u&_T6s>lk_D>zkC@m+;y$qp`Ibf+BU+q5?4*XF44ZmEzkjz5!@T!&!Jo~W*+3`UXgsr`Zr9$fP z!ngq3mTE$5y^P@f<5?X0u{-=Tt4g_AvpXM$mYIwsl z++f-5IWVVDo%F3ShpIbdU>Q>lq6e#SXp;e~oDu|6-&n%@PD>IvDFrBITf&(eo4IXj zO7)f_R`B+JO?cM7m)Iyu1lXD>!feMNytA(W6g^*yvyWMV`F2x)?c^fPOQWy+=&?BL zJv0S-+c3oQY&o!vdJd-l(1Ik>k~B#!zAyZPLewIlH#%N zm=~=L#Reex*5XH=guK9F_q^~|p>s_6W;ux0Kgun3iN{ZF_V8$T91_E4W zlKVXk;PIvdIM^{A9Ev>wR7CayyRi&>AvF^$wwM9)880$oDNDk>zu^b|X2@8=7?@&` z1Wd*w$i{P>V3w~bd@gwwI1H(Qy?w*nX?zN%9!LV0XANNaiDIxoY${o({uQSL%?FvU z-AJLuYA9PofJQWgx^qt8Gf|KnjOoJd=eU3aC~D?5*Q7x!J1Lp z{Cx_ZpgH(2_CpRJ@MAmYaL8vIGqr(xd0-_`T9}KcpFa+ImKETS0&S@BDh)X5tC9Xc zsNVmM5jM~o#wtg1v8tmZnKHZ_=B%y*cHIxb-mmBJhW95hH!K5qFH?rvpSkeaj#=bq zqYiEnXyOevYl!>SkJymQA(z$NL6g2LQ9vDF(@|UKaQh*!(-tQ$Zu{V=Dw$wW(phl( zK^oUj*FY$SJ5YD^X$Qa58NqvtPjqz z5<1Fo%66gGs;U!^I9E=okilAZcX7RMxmSIq<1l`rD|9iH9pEHBUkoGV)u5=P3hq)a z=GHr%z`1k>=@hrb+QTBSe^)YzPwN9Rd?T)2s0XW>uKljjq2jGqQY~nm=8C*E-P1?(+kejQ3kdZr%CFW&-ll+B5XSxds z+O`CKQ{BejZZC!>707cBv{Ya-e>HM)$`710^c}Q6G$G=TXF=;{W^iqvIJ6AE1QvVW z!D(NTuq1O6%v?Q-XdHe3?#xdG^=mf4B|eTsLoosFjQtD@Zu@aFwFKZuNH@^yQs8C; z_`+}3*O9M|W>EPm1AAxMl0LavP#c?*a~yM0H*-31U1S1(*1iBELApe)DxQC*K%5Nk zdkq9x24F{{6FmRv3*IEHM83VQ*A<9-SPi?ML*gkXWF(GGg}Q5&6HTQ> zaOSck(kXErn9o}R1+`kF`sY$SQnVT-sf_Ur_B4Re_CR-Y#&tP2i9D=QfGP(j zk^UN?XE42r`{MI=;NL6`qKOo7zoP@in||RQ=k0Krsvlm^=E(I^O2ztz-{2h%+W2#H zDiCfe!pol_M3JgblTJ&;A5ylEg-)wru+S&wrV|a_8!z(jgx2A??;+F)O9!c+ z{(?r6b9jfsC?0GI<3^}2hBHoYBp3X|NqcD}$WUEK&LruR{FP_8R=3W9w!sKec*7B@ zEbxJK_4Wk3bcExP!9=-Vku;X_pwmu2a{2WPEO@^kZb^TS7oPKk{T3&|f$Uc7)oH+u zocMvKZt*0U(?nou_%&?lxdRSxOTnwF9iUZqCcN}v37nDjk)MA~hi{QS1HL+NlPk9O z4foiMsie|s92AD0#f^$8#Qy4UzSj~-EdQpIQ+#DQxn}JNr(`T4EBCL1dxh(Ms-6ly zKX{dUtHgtB0-u0f5`?wK+VGKMr@^d0nbq=)s@&u>(4dB?iU%>W#GdH;I4k+2-M_kTDgH!Wm;M}j$ z@WjLxXlfo!jLuBP5u+yXWcCtleMI{IJ;)Poec;&sDfq}s25!r)0tM<}q%KqyUV`=5 zsB;ln^C+qQo9zPl)I6U*{sOe`$W-lgKpKAj2$E|_sW{@yl3Hb1uEVSI?NA4Lb za@yxC!rPj&z`SDy@Fvd>*LGFms`OxTa^Fdi^=c0U#lP_bOW}Dp!hJS{h&%~{7k2)^pJvOGZGIB4AgBjl9g!s(+daUmyV9`#_n^?(c^0Jp zb|twp9^terPH^w=RzjjYh{F7(#BHh^tVSVZrSA?nY4>V4^gf+<9}zM;V*zB$s1=S z5~Gv@=cJAUIp39J=Gm9PuEGG!-Y<&N(if4#AFMeI5%cf?%p>>01f0ip)=;vj2IxKD+Hp&g)EEvM^GEAHR)vt@yU}>hyBV--vLZZtXCv(V zyo0YY97P&J9)SawWyzuxYqCtO59CY8gK3ZV5G{khc;-MntX&gGe%z0NS43RkWPRm$`8MiJhcH`6pjCJ{C&V%Y%`425S7b4*pYUuJ6kfgQ-=a zFm15~Txu7>9e659;$LdOMRT{1)pd&?nHU9cs~=(C{XsAw+>xX%REP3Tzp>TJBSb-b z7gpVR*Qz&ZH{oUFki#;j zk+C4gp{ejnQ~;FqNdZwafg;_ON~k10TIg&nE9y|q5lCJA;p8N>F$ZDf2@0p{FOBhP9o0bd~t|B~GR`~T#E z;6<{)swteL877d5HFvP2nlYqx_FxG^G5rz%m)tkwU%ZnJ^-k!*O(GG{`AjG-`=bfR zU8CXQ4nrs!??isIctY*FZJgVFE zuD@}CT^y#J3-G^DWtd%!I5l;`eF^Emcu__p|BCA*fO_L$`L8JQ#wv-N@m&E8H>VP> zxtpQ(v`DzkB!%d$ItUe4t%Cmtd~TVfN2W!@lk-k)uv#M>ysJ$h2Gf?2R@E31`Ol6# z(2ap}Q_kbFyF8#|@(f%t8U@x|jVIfs6FJxK+{Y%rTET+-spO*2clh)lBr}vaaNaKA zf80+zqP=`K(CJWzO{?QchHw_;N7GDD?`aROo9-nGr@X+UZOQo7_9||9?jk7LxtI7~ zOaOfUDU^I9BK)oE?ZJZ^96$*d0ognYH9b+A>(TCXT4q-o|r{q{y*#i-^&; zG?>=E2JdfRpuE%taO8FcmJ*u>IBV5Oy;cA5hGI~M>-uxE{Q+$>avx^a=YsMycvmqLKNp2*uM^?eWSI+FY zNo%NNb~-|?;mnzJzD!N@AKHyPB8uuM%*gjyRHu848QUh-P&`|J(#=w-MZi;(^K6KjeeobYeeE;5$!CtWm30C; zEBrCa4M|6ZI?tfLq$s=5U@wZhlt-rtYu?9n@rKXiUg$|jjKD&-5bfzur04bf8Ka0= z+P7`7kbRnpmOZBEDd#K`=CP6f%sz~a)=uTcteDX-(ko4!{v@EZ&T{6E@&(#iFVPSf zqCi)V7_rO8MX2@b&GbTM3Y|1l64jSZLj7_*@S345UG9>^#I?LY%cHUw)@VN4_v{;+ zWn_RH72;S^1p~Up{sQaaBFlJt3Exe*?2~oAr3zYCKZSj`aR>FCBadP{tOT#`i%^a5 z|50@2aWQ^h951DX_9CH8ix!mEf$`;rtPODO3#zyIeouX*M*bI(2Ze$M-?g)dVJVM2KYv6$nAIe~SQL=GX^9;aa4 zITeW9>W4)|0(^6H4;^o=g?k5SNfU;uLm?g@f3yAx{$wttYXTlqYpq*sRdgfRzHx#a zXGELeD2(b{C(H@B0v02NiB4|4Ko2b%EOJ)Q#0#4P>GXMnL`yq=)5*J^(H{2@$h~3) zCad*fzup;a$TZ>x_qwx}r`qFx(ZQiEdreIbngcC}#&T&P z*$}l7M_L_$cS}!W*xnQv9(IzhFq(`CPn<-T7sZh2deczkQUrO8n;}qkUp4XlU#3Uz z2u9>J2v?knlAIKtfHAts;JDTs1JB-*%qZ_9oi-nY4oPG1X4oCHxbTm!Q}cqg*N#D~ zDnlLkZRC7LAoeQ?==lm&cwVVXLr%XZ^H@1aTizA8UL6eoX{@GyCS1X7V})?Pm7+lV zis1Om>&Q4I;LqAmWG?+m7Oea#S)r6ist5RywIK*TwnypQi>txs(ll@TW(G20tVp5H?kbZ!FF;nErzV0 zs0BKOm-ueseaVNmp->)t8L|xb(xemADatP`Vk#R)Qak$M0*pNOA18#kz{_%R!YxQS%)xW!- z;E^Myrca^PEm1gaMIR}?IE@VP)|FZ~ctg4d2gjaRVN{wijbyxFVe(E~bW>k?vFIXp zYtBMr9U0^5&?@4h8zi)q?7~;G3}N`CXfpfpVKS_%lPZ3mPT%&Ff|t#HtXlOGr|PMp z@kJr+zT1TjXZtbu;5@qg`by&Gwi*M*sX=M@X8dn`7Y=WKM%u6!0&`->R;LOuzI6@* zUKhhxgI@e4K8AFSy7XDGjBj~75biX{LHh7&GWf($=_l2p@bYIO*4@y=sgvZ1y1N!+ ztSuGJtQ?G|ZVkp+S*b9X*n)|=l$70iE#nAjfpL(ZC~MGfEJ-UOnHNsL(hfHumv3Xx z3^g2id5>iAh~r?CkxMjJD8Y@p+B7Hg1l>9A39@Rx$$2Gv67_Q&)-E?9A4K@Fq|@77==x8Gn8shFwBktr!HBHW5V2r3@pV}VqsujA+|o|^rR_VN zJ7X|CR`-GI5}68L=Ix<>#&4l(4oG3ILoa$v@D@7c4Fj8PJh@WyjwVhEqYLUEN|dh{ zk+lOl=-}H!B%PH5;OxF2G`QM|Nqd(-mDzDVR7)AR%XdK4+;gOHbs8n}x=8KCDp=HT z5btZfp|u%R__`|&UrWr1URDHkS?x@$^KG!w$^#yLa{{q#4ek}4m6Tmp5akT*Azw$v zQpcQoII3tXp)?-La!o~IS%&H1EfGxV&=(CY){*Ak+fOdkg+j!qP)HjaBAgSFOds`~ zpvvnRxR|+|ZcyGuk9J*w4d+(S=Z9?IYpCqr`cE3vSUx0gGPQBlFIl!dz*)M^@-Vt} z$Oa6?e4>v8TF@a5ARpt#kYy8WNII`A9U-`c?IDY4Y-uSB|1}V7ocG{t({y6l`3xsb zPlMNEM~QMal+#suR*>>|A~D@~ojfEK=qvt6UezRn_qC^Z-n^DdeFd;9h$r_{(u5WN zRnjuMHZp!=4O+?kpQmppQI{3X#DDT~QX5tWk}s#w2yM{geKtHV4~3VH7UIvdxwvJc z8|?KzL2}c|(f#IMnzTKNhB>peb>3|lzOx;Brfw6uJP!uT79HdlUBpM1QotKGvnu>! zVQup)JT-D3?3_LTs+Ld3@%ah#c0nj~x_-d^>2FZ;gNpRd*{9(CwVq5rbsR=)+eXX$ z)I_DnL&$QMr?7nFa*RxC08+e%Y{{Na#vP0#$*Oh`6#SW-*EqpX{p5-IRl&42{V!Fm zC?+lYri0d$Q&jZ*F2q&Kaxx01gwfuaP%8UQ{u{gwC6}sU?c+*3kU0}SX-mldY;&pp zobhn<=vQvcLrrS!EAx5YoDbK3CP9#ghe+wj8RD|Ilmv#P5|8maP`_xhuyLZMWQZ^n z^gK+cvGfd4cX7ij`tMM2g#zv2rr`LUEhJTMpCsPM7KRUN!b>NsNalD?80-{^+Xhaj z#w7)CWPlc=y}pfOG&;~Xx&Vi83RwNA5lz{ZbY)-+%x=|}x#+!N@%}GVT7Ll+UtLYh z`LU9NIf>ADF_)#_pUb_KlZ=fEzvsRCCMIrKc&Ol-V3Vd&BzSUiox z0j=d&#%e-j#&wK&vk9)NRzPB5nnZc-0NnL#F8N%XMpBkPfEOo{=>$BF7tf!i6HcdL z!oDeR;ie53m>wodWCfAfUZ7Vi8%Wt?JDM8hMGp^qfi><%(%~a>;oh5hP&Ctu$XNqM zdvC!rD|ICw(tp#PyUybB0Y~x7pE#&^u!^RoCF2d{A9Tf&t5|Ye52mfvrv+Jdm?!@n zSBH+4yLx7jH^9GTMB8dp6K5@3yky?rONr&NO!_@ z$=0>wp|4DgxBhmMPbXp|)q1+@`vBo2FpW)55SEy!{Cmy@~hGe@wCsUrQi$;yG zq1A&P!tXaXP;E~JHoiN7JKq|Ru4*Mw$N6@mAqVK0C4t394oYt9z96fI9_XF7m_B%y z3cc|kB{hL3@J&iSnHre~v29DB?T9CQwkU_Aa)#2yX3MDOZe*t>dBb!QPgdMgi&fA2 zVaC{E;kA4_sI6-UWqo(aU}JsJkK`lxasOj{n)DGv;v z@L%wL_$#_4^Evkm52Xx|xl3iNS;c<(Y{zkE9+v^~-}7+Ifif~RKO07>6-qW%)RL!b zUeasvj`;nAs*nWD#-qg#V40&aX541beRd-JIZ{Wac?`g<9vksP=|WvxyIeiIR~akZpJ*n%zGoJQAI*gMx-Wz)XD-1*NAJ@2 z*>z<8+rc>ZyclFxWy(8|9vjU(I=;poWxi>+i{xJziCL%JXfdJ z*EkDh?IE#S+(o0__@I)m8P{ZYi2V^hINVWpO~x%82Oq+xW81nda4wf8_pJ}%^NZ6lPv-3PKa);Y9ehGNuiikx zm65 zZY^yrY^R>%s_9Gd60(E*rG+iW;r7)PLdUjFoZ~?WovH|5GrDQx%F$S$ZcmP6X-XgM zYXgI%AiOdE|1$_YcCY?SBXh$*ebPmt%LO&^AZ`%dGVUdg8Zi_`Ur-h0tvLsO=XjD6 zz7k|YuHgMww=g$+J#1ZZnEuR&rk%fQalt?v?Ap{x>>tI!L2ePOzp`1_;xUB$Gq=Xm zR-I^XI);BcC<`)j&k3KcPNMgtI_dM>Ht^(S3f*lNOG{-nB527NOsUXEqfld!$%3t@ zukjq3l?H%Z^FYW-YA2uekH&{-;dt%8D{x#^gHi)tVehrU@L=8z&>P%CO;Z_R*wS+0 zg)LXXp;i_5cP)W^MaKBUC6Rn=(-fVT{l)ZN9+ec#%7-ZLcSK1=jKMpnVtnyh&|M&f z(Arkv>8e^`U+sA~oQGtt$zY*%^ef_)c$a=Q{6Ty748eJ%8vAi6^M|-GFUY=_3@Y9(3xoc*mkv6ySLpmG0QD5d!HqLR;m)n| zvaEFzDP53GN8T)<0juW1;nY6TGIgOyX9c19%pTa@^HNf!>WL@i8pxD>XIzd;XzYws zzFCTxUz7#Q+vLa+m60enA`gOhXOiBCA=20VyRg;j4Gy>~B&P>l$6xnzUMTI^Qtm+YVJLyxvk#Of{gppRRG7s@}vjKqTo z@~6>i@(p^rd%8%`=Nl=jNQ5&H^Qp7%P+D>2CQX&~>%}Lm==0tYqGVrJ+;L6j1b?sv zHxK(K)Ya&c{P77T^Vb}}M#&>m73xeHZoZNf))s@urKNam!3C5CRuhCm{0JDZ<;dR$GcKzH5bl0$c z`gGqqVyiqGg-zGM?C~7QfHQ|A0@zFs-!mnXLx1xI=2~ENw*wzpslay~ZxBzqi@Fa2 zp<;-QbnGGrQno!?Hcw~;>FTL4_C^h{zEesz%l9Xgl9Tp? zJHZZF#&W^%A&~v=4686!j#}N+CRV|=a88Ddf0f*gq5DDNmY#}HgOX6s$qU@uHwXua zJ-~a$aP*uzlA4wIL1bGaS)l$w(s?Zc|K8pWAI5G1jl~k;yW=zrX-$K{9v@-*X9^dd zJ{5-OSigjk7{!g$m_SUcx1Z?x?4zLZty{xSuD#cULGV0X(U*G zyIIo6z2%SOTn49EEPN~-A|l6i*nP(G>^e(1M*PT~tHyz>aYqifXWbg+&x1IH0k$d8LgcA+1Gu+dEK|U9bFwF|Rgf&lzQM zfyd{w&wd6o?J_=}ldC1)K3R|VbuD2;efs>nnzP*QRTntsjv@bY_HnK~U5B^NH|COq zPcTtQM!cgqNU(UkDSPym9vf1cB&d1sC3DxxGdmCZ^82$UvrjUG?5-tEg3t3qIaz6B zw$+w!jk-;OQN`Bm&UF^-rbnjynJ4#|SjW>$_YiMxi^5(m_j0O?L3o5w8SlY_H*8|) z>mkf1r^8HA?r%ZE!XaG59!LJtt!A<3-cIht+g*ZH?Lq7&<%zs)Z3sK~p&YxQ@&>ci zUV~|t-IrQCwB)^x>#-*2#0iEpFcTIU^D}fkS*wI@asHk*?(%n2-bj8Vr)N^i^wxE7 zCx&feCU<*t1)DVZ2YUaw{pnws;`_HbR)vZ$?jOw!pWiEtkpYL@%p2~`MnG5`48dhS3-q`gTX_Fmm+0(*IpXb3EPYYsm9(BsRltS_4T75n>Ybd`fZI4*- zi2^rb(PQpJry5@`d%sE6kuJg4yE)O*LHyU-6WKL~9oWTFUNf&2edEG@j$-YvFX02X z@8agojOApQ|J9hET)W0r_lw93oe-__ioo)m6vKnqtDHJTkbLxnk9B*zy|*JT&4 z6T7g-y9M>{c8NU-+%7TUKVOtF8A47ohfa1f8@?&oSX!VH<@o?rWo}3rp zmRsCP!+XqsdkNpPbrNe^d5H;{Zpq(vJta=su!s+0l$h0fwfLJEf4Jg%zHHcm3P#Ud zoi&%;GiOzd;+vlf1y}WDYqOsMFW6Ti7{A$(UuAcmxj*u~_{jWa?2vQOf=55@bJwEv zSbvck-(8=~rS)WUsYQiMie4V)|6Pl5{XCkl-{H+#eTd>kpUl~qe{TGVJBsYG{Hg5m zkaaG`zwdDUohoeF;t{;dzq3q5@ow(<8XbPr@%8+h0mXvQ2}8Ks3HJOxP+<2+WK60* zvV7c^C}!ApUT`O9rFhcP*^K+T%Ur?IS-j=7j{^V2Ma;HoUd${PV>WSi7Bl2j7+)ac zlD<0ghg;|}hY20KfbZ$8W2(LVd82Zf3wo{*>oPfrvr^1r=70hFG3>AaHl{L)r!R9w zr)4*is$=;MEnm*dxP`OS`X}}noWW&I{mHF~i)Ut8?h>p~bma}V-Q+Gx+A4kWI4*I9 zGh3Tu%lBpmum{tgGM9oU^8S@Cm^DN1aU0##*td&nn2{Cw?A^0p1)qlNaw+rbxpF#? z&nZ!6C08c1m+zSI-IY;{-J8FHZ=?ITOEuG(TqSQNTtk(Y>r-d9ynn`cnvUatr1`Ra z+LQQk->-8SG3iWc;1#Y{L&VtVJ?7*C7{+qKM?vVccijFH>U{EAf5zA`mAf5qSJ3^O zVXyef+@+N=ujD2z=IL67eQ>>p`=0kh;4=C>qwvLm-{+Odbte~bTO#t9q{tE6+Rr6S zdiOC-&g`RQ{;aXBYbn=RHJ&_+yPHIQfYp zHs{Sq#+L6FpBiPswfNRB)v`5wy^K|rnQ@Eh$}nMrCK(8l->dUKUnMfR=Z)Cp8;`hr z?*q)|Hbt?jZ<*M`V+PxCE?)2=q=vDM%IDNZsj&$%K487|Bwp=oDRXaiyLi`_75Dyob{i>?5)XR@~*9Bdgs})>94HV7hDV%qIA8oW}`j3&TI}F<0WAKm>RJ0 zChpwrcdOZ7yMI=O9-Sjd?B6M_a=Krcy3&z7WNyPZZ6CsqHJ!;WWd`%}-dVAlP32sA z)*@bYtvTBgDdM}{FJ)evjOW6wHu1T`x3L|Qhp`=&>-f*L>zVMbFn+~218(MfnVV9@ zh@UOnuG62ZQVSSSDCb%t+(9HIc;0QY%o@4x0!l#es8SVje*~| zd|eqMHq@Mtk8NX~KL5!S?C)Y)UrPipt%F(BP=EHzm#xg$`MT_f?RwmzAEWsDp_5sS z2eWvk!plrmj%S5=%3`8DF%?RI>`To+cpyMbx7K&CGtR=n+E4deXd6*p^)Hm_e4#7rJh&s0Rp^T}BY z*?Vy@%+{SOw>9VrH^cas;7P}OPBLI76D^yA__4~8U-QA8?Y~{-;xV~f zS9>&?U2d(++waI^K99b@ecTtvZcfwY+ujUh-INoUC&XGj@Uz|y}A$@kU?JO=&TamR$j^=*L+p-4Nb#{y@5KKCNh zjs10L2EQ=-74uD`&Udf)EAY}S66Xx;=1xDF%DNmWgo}Z{`!oJwrB{;KQ zmQ5|F;~Zj_@ayMTvKC6?IL5__FUY;d83j#Z`yL`=a6L>sTf?0#er&{MdCp*ioo2A7 zrz`QFe)Wpw4^eLOlR_qc!$SU$%M{l0ZXy@sIf9=v-Al&iPUGJ1QRLiSYw@P%IYG~p zsodI`QDR?j6TzOm<$UWa4L;sz0`JBbGuuK;nS$^OOnc-bF66r@D_yA0esvtqC1uED zUk-#HHnc&|zt@Dde6KAo@=4)*Hw||Q8?MW5Nc+Ip_V9v@ojuIOzL9*|q@P^x)Tf*j zPjV(RmH4ZctzxbD3T*7rVE&=Nk@=UP!v9;in;Cz25^wgrLo9b-jiA`EkNMc4$~z}4 z=MJ2G$Q5e#FoquYn2EFAO>8(7#3%+S%X*)9<~z9Zo5wF;lI||$Ov?8$fiJ|2V6r~z z_u891liJ5*T{z1P&)UE**^|uN>rvq~raSQBU*p+AzMETFZ^=BGw3(UTqs{(kHsBp3 zYXzIv&*DpF3}PmI-Nr9gT*XfNPmh0g@HF>hpF4A;H<-Vcuf)V88**#Ao499ghK$ZX z3pS>wNc=Ao1wlej)<}`z(Z_}Vqk2)iEWwxGr=!3+T)D(VZ(Ai`ua078x)0>jW_NO@ zCY|G^JP73XMZaVM19!3uJm>MFL)NkVJ?FVe^A-7~KsA>IgN=FP-6D3C|15UzW^HEf z1WUF+>mGNtJYO8%^__9{*}$u;{v8bXA7SoV_;M?b zR>{1-{o=4e0zq`?G44s3A|D$*wvtc%$ixa3i?9D2%Rh?Y#cN*c@G<9POyf&yIS;`) z?sKR!vl6A8**srnmx>{~<+40?BcV)i_jn1Hvp$|V>ph=)y>lf?rVipCJz;cVV!4X(h;ft~MJ%q`TkX1#K+h<_}bz<#N`&Y2z#Vnm0H_+@5JZ0E<-Ovns} z%7#`~W=p|gZrJxJywl(ou3zB^w^iSm<+Kkn21$FkiY9Y*fV>4i-o2EI@DcK~rcb@GW^+%^tQAc4zQ}cVjpc-Q)!A!Aop)|=V!i!+_?K5%PULutbKbU3u%~^q%gh)0 zynm}HKP2flcVYPt?v|=G|L~HK&6u``zqB}(`#0zW7c6JS?kxVxt>2d|_W!5IYF@My zA381bq^ryP-2bJ>nB8B+t85CmzHT$t^+94~!>A8T|Cgcc@)b&az@Fs{U+&Fzyeejf z`1T35y!pu;_^HZ143+0&HTMV%e`E`;N1YOgij-LexoUx{XBVTSrNOVl+b(W*<=KR9 z|G4_dc&^6Hp-f_nec3rzA-@*#mNJ}q%@<~0`X9vqK(px1K>#XHA z6uGj+!-n#U2CK494~*g`CH67zdtLb_)6Dq3Esnfe^gC|N7!Te+s>d7HS8zl6E-}pB zla-q-1*~U-$CbA?$MRps?-l5I4&kTA%whe833Yu`@9NZzb{_I|61k8ro5cRoQv`0mg+SyUmBk>tB%Q7 z(Y~5&UH_BHId!+WP!E0nrT8PqN*Xw`BkFAD!2)Ldr7e|f(xVU&dUp%i}(O8pL1o?-e{6Ez3#< zF6B2CEM@gFBbhz#$M69=uQO#z!EEminaf)|kM-HKgjpI?z%=I#WQE_F#G_jR*xyrt zpLI{f2X2=6n0ar0XM_VEJF?9sYn?NDcO%Om-l5Caj()+s)zlZK$nxN~<@34Ylf1d% ztDV`C33GVk1Wi6}fd!w>jN?sGwAh~}4opw!aqe;c4{pcgQEYT@9OL|Rmbhu2FFR`5 z2!5Q-A!fm{C|1Mm0aHfC@H0$IU{>6A z59lSq9JgB97Bvt>zciqsB?4}gc;OlaSAJetEO}S96&4K*Cmv^iN`woC;5NU}?A~+N zB?qK)VYAgOa^&a$ylPWPEY9}RX2DuA_**?mJ-wfLMlTSaxk_k9kutHhD}nlZi>cCz zT%plz1*u$@Db7@IfcJx^^I_(TQCTn#{?4l+^Ou#;gezA`cmeKp4Hp3agvtMH3B0_~48u zFioWx)_xK$R~VxB*%jz%4~HA6$%K=!_T+SJpk$pVaoCy&`W>_AofW@vV3-1l*qgmB`iBbPb+%|*qmwhB;S91YQDjo(csbxf` zQ}%4t9k3zlB^}Xq9~^_%V(OtAgjbPeJ7s6?PhUD=qhTPvmu1py4Q+*{SN7n4?vQZ^fr+DI$F}PiQ4E`YOT+BNThmg_**M>3MF*B9aW zPfoNfPUi1x^Q0}l7ii-B`M4qB2JBP+fJqVmh;K+VEZ^S;EpnT1cIY4KuImLBTUP+@ zpF{`7E8?2%4E-(e#B86*wD!GF;ys{3xMa90mdi3y72j{s{p&vhuiFSu2TqVy*^d-y zyxIWar~eS?)GhZJ+@w^@52EB63?3!Xqm z>1W`ghC3)9?2g-_Wjw11@{&sz@4?%z^U)>-Jv6ux}0!>Fx-3#x}|LKErUiVGzb~GIw|37K!|_RM=x?j{|PW(|Zd| ziIMX+IP=PzhOF90S3BjvHqjGMT9g3cbr(s6!%b56W)zhK9VLU~T4=IkF5rz&bPp@T z1tDjN<%o-Tpw$THH&nwH`L}emMl{^E)C1kA&*;3@+R#5{Kdv*s3ndvZvE18^+I;rH z?PrF;Q3n}gaLO9|`1>){Z;k_dg*aRoxd%1lf76yJvJGX}TKwU9k!nT6;P+LQP*A9a zj!NbvXz_EZ_R9hGC0An3@LhQ1;XKUvb_ORnXo)N=H=$=_0SxIq4t{E7lGJ!(vZzG_ zf1Q)D?*Wfv9%RA**CN!j)RTrS@WJOssS@`c60mC-N_K@Ufz_5e5b7FB{*5yf{UcJ&wA2bVDA6lTD~ZZyIgCgj zC{5V48C4SXF-iBgM0v+&s$D%y*!H;vzUg1Z-mWtIx@;km(br-5dK+9e#shZP9s$SS zTChLk78w6LDs#qvB0p>5;Hkzsn7im4XqcTN1+l@5c0~dv{wYO!{{(n`Yyk1m8VfTH z$xFRUJgRrr{bmS8mJ{WPZLi%()8C)$xgqca5=6GBEHC8=iWmz#??yFM#(^ij_8Jv1{pK=(LqX^CJ&w(o|!x`Ew4oOqmDv+U7EzlFVsV zm`0O+TqO5LAgYJ&#{1j{s_J_S1h2jERb>UvsP-oTCG*MXws3mlT(a=j!}E9~(v$wR zcu$*+9Pm|PrZ86X1<4&e0LFewrU6!!5*aT8^A(TN2=&X<$Zt6O-g}U~9+^olxFn&z z;3DoE-~``0oB_O#KoVa>^Wz!O>LK%BXTB~h93BAG!CT-)R1g$t{@~kx-Xb;|if~oa ze(Dpmj94Z13SEchqlIY@dY)y0^lL-FFl{Uw?FKQX%`{#3fJQyl#1ln9AayAf-Z`)s z+$NadpSjOuuDEB|e!&Bk>JO9Cavt>IkK6EHii6ZA_Yp`J)k~K29D~#a&#>}KD|Ppm zIbo~bL0QTm@}ut+8ToWAF590=Kl>gfI!k!!@OwML_02F_<147ESD^gUr;>|H%F$qr zwluPN2;R7~OQPI&j7BXe6BV; z<-$PN5bY~$N|TfL$0x&D$rs5c&tssY^O60SBy$eGvw#IkrLb_Y83u`d25G7l$A>$+Wk@VhTytv^y*-_jC3(WGMa!(k%c0`N?UsTSK`AM(W zV2l4)>7g@e_$1_#cq6j*0pW0eT@se{Zar?sxgu`hk7w=$T6BUqn|kBakRNt z4OC^k*OS}g$@as4AzS$g+*gUg<@5A0RMiJkgL1Xs#fShc72cM%hdgMJJrbw$>>8eoP-i zU(O@LKB?iMUnOj_njFrZSqL|TWANS^Cpl?>YNR#1iqoG^^L&on-5+8-xAn1Hwo`4&Vf^lZKRhs9)&HW30h{&gTu!>@!`Ti z;=MG3yt+`wI_RyzX_u8Ge|s*|QmH463XaBrL`_`ibN~+>%ob)^Nn!rDt61A#EUP!q zP+UHcCY4{H%IUu3t-iU`ydn|?9@s`!6%GfhcizG$&nIAGZvYB%WxT|H!PrxM26PTLbnr?`Vw@x>56aey?XKn~k(-vXPS7@5a?FPt2z zN9sxwgr|a^k)=zoLHTieoUE=$cibBUUc>5OTE=Zs^TZX#B}4sTJbjy zA`VS}oqywL)r)C(b-xH+7;2%3@E+Ma>m&c%t{CNirK9U9HFW#|vue(UeoEW}*x&wT(mY1$Bo5?jk*p5Y7lR>@E5xa-Kkr-yGfT_Hz z%mw@ttSYXeP1ywUSMQ3-nA3|d8!>dRx@5u5r$juu2t;wV z__RTc)9V^>zkMKnnj`b**6K)Y6y&8PhtFJ+i(ahr( zs6F$PJP8&N$wN0X-6@28IT%MGs?2d~WQL3lZ}oBu zjd+=Yg*j>H_a|5Q)ItTD#`~d%{YkR z1@+BhFia-&rnf+9^ky<>XM}=@a-;A|!z&>!e+!dOWs?W*lSzTzawzc016R8U62CBp z&S4(Hu@(F2!1j9KtvRQm_>s&-?RS<$M~I}mJ^f@Zg7=cizmJjg!&=F#FY755w!@cA zKggSoG%!CW!6NJZc)9-!^?E7Bt*?bpW}r{rZyZQBy_1l&MulXZYB!x$^BIXNjJ@gWf_-IpfL!Q|D57~RxXpAlC1;3 z8vURh%z6_+(?|+x<+qDCSnh%wpv8~2Whrf`Nf(vxsGBHLCKS0V3YvP3u zMMQgd2CcoP2FvZ8;K#^Q!helE{Egx$iBqEs7)&{am#68GyQlV$lReidd9Hvfl1pHb zg)T0bb{v8prGwYwFW4nx6gF{w&^x9W?N3$F+26zQ_MEZsBVrEDi5@8pE*MS5m0pza z#?yt{APxqGD&RK%Mw+4>1gEOisqaHWxWvyPC%*qAUj*lIpTrS7>t~@w>q}wXyz6KT z#YFyEJ-z47k=CHYz<8NM>)d4+tD8*g0uQ7A0VOPruf(3MXW(i11L3%XDx$JZKln}k zalxWUcw@IkGB2c#>?uAXbjn)EkGt1H-!|q^rMuU`e!u~IU~dF#>Qk`Y#Rn7T3=;W$ z*M^e&1?2hJX1aBZ2=14dLqmfyO#Adxc))W3xj)DN_HPIB)nykv7+QzR6qI0LLOeBJ z=ml9*=EJbM7vQb20oQ+xV{4p?a9ftF4*PGS@v1$1ik%WZT#^CpZfjAIJB)#Q<|4GmM9r)ez;N+#hh+$H>k zVwl)iNmT3}Q{iAa+!ov_^9W3U`otQj`8gE-s1JuzOWUD%r4`A%mL$=P2@wsNF9%BF zgb-eP3|5}JME}&T2g6NnaQbi$J$~pU?F?Ovk;mRhN>WY>J6^ZZE+c0g{l^DNR#>9X z?c)#^7!0ok2k7Ouo3LLi9f$68M8DS`p>V=MT;cp5v7BBEv76Sw?g!T3xW1kIZU0T0 z7M`bM=U`Dy>IUrGEWvME{vP9k7mhoKv%{@N+!N#MOBIAl5rdBh1o%x zl8bSV$;Ez6m>Qq}N+*ki9ugJIJ@F9UuWBXxrW)a$wFgkmZZJu&vc}NHBbeLqLfE0{ z0WLuUL?+)fAS~Sh>e{zln?f9)fSGFro)}`2vmEqoHT^6 z^q;~t$sH9-P?u$%BkkYOA-<6q{-PDS!X69#^ULUiG52Lo`UtfDrHQGv0@%MOR?;s! z4-qa_fv$}vSoFLKqYc$?^chXaCw?&R&sChV`2%dvx(LG8sS;tb3yECyo1Dq?#*#Cy zQRlx#q7!P0mFG;baKuev;r-L(YT+@wlWGV$ldfQE#$uEot0WDYCr>;4wvgy}Em6n- zEh@FXPfji^gvm>f!i=p((v7OGC70Y5lc#F=aH-E)RKIB&F*Tiy=k9I9j$}FP4NSmo z{|13y%?42KUnu$0@*mlHS&LR59S9Gk2dG?%7s0MRa)`9Ek>(C?ZAmJ0a4iyp)N|l@ z##^{7zKi-iTqYZExr-+rUW0$Dim_r`1={^dhg>aZ5FCggY}*n^cZDa`&oQTqQw2ch z=aa;b!)g2Uw>0ajAw(T?gs9Gw!rZ21cZMbQNUa zL)rPKfA|($YIO&{sf6>&1>tbaGnMqFnSjc-2{`uU9&&fxGs&@L6=?qRmxPsmr;ltG z(lGNwD2-!LJ#GO+K2Q-Qblwn7ZgGU;=AOhLtQngw?W4bLe8=^Vj*&*S`83gTCAw{O zz^dRML?dcHD$75`kWXJpYWX%g>(K~k)3Ctu3CE$|ID~pMX^Ou8Oee3J6k(!aJPkY> z3vUcNp>ydWy#34(75*I{t@Ua2VaG@Mv`}`g{KbI8KKw%ye)Ldd*}TE9h5qPJVFZFV zq4?Eqq%`|W1$o>29kMBEv(Wtkg3$F$jOWY$(X_kBqQTiN%x62A~(%A%wcYGts+;j*Ay9^L{PY8qY z7WXBa8;ww5fjT)^HV415Pw2|oUhp{NEF5mKfsKPr@#=9k;=KAXZuzYWMR&}o!52m8 zxA#jiVeuA7w3rBnn{6c5I6KLK)uHHo^)d;}>p`=tTKKrIj~4kIBgMrZAgM}6xZWiS zpRl1YDpLppR|mlG&8nDiAs(h38zIfD@2BCG3bJ8?!Pr1|kdDPQ*au2zUm<|5FD<~h zO@kGS66jLhbkbikm%dQSlsw-O4?VK;i|lfFX@jIeGDY__c#(2&oN$6(8C`atFUc#M0F;ar4y$=gRrjdD#7nc$^P&YtYWAY8^Le5w>#uY& zWP851{ZQd`mP|?W#%e=VsP8d@qd1ZVobLuzi?b*gZ;4^S5Af41Ein3aU$~`IRb(7l zN)$|&l6UV4>5g-Spufchtn@8KPoE@1S+18*?)iQ^lX{LdTQOOrE88Ecl+MR*n(Csf zR-I%I{Q?dHjEIqg6?Wb-ucO4@ zKNQw!#X!*G4cJi9L_1GrQStH^^m6+|gG@!ZEA}w8CdXkTH;`Pwczo0(h25ub%DyG< zr#Ji8LGg1TM>5H>dUJ>c(OGi_gY7-4-#DGbH!%w__U{l_S)v0k8pOCuUy(ffF$V09 zE1|r>U~G3@g@8^6|eq5@4t5 z8w%f!fv@r{xcBlZnM?gJSN=KXYBHp~>l}rsv-sZOIh4I0jlqtitJUY9Ck>^ufQ0^} zt3GMr^+k@dE!N`dWhospb!8U4-4dara-XcPJ_-Cv?&7IL3zP}`hjWqMv}_;;k+-Ik z*+B>w+w*0Q?Ds%?ViD!{Dpq@#?tuF5XQ-URKs>tLoy=lqL+zj&G;IDW`ov@N-1tk3JYH6@Bmq69^63<4Ml~4`}ELA(c<(g5J&>!k(~&B#&sOu<;aF=&Ru3 z0Xk?(_2}+(Gx55|ZYsbK;HK&}?0UNmFMe_*Z*8=yUs%+U-aciC!0nJ2HxGa(v1#Z` zqrhQoF}bL^OtLD>4L+xPIk*+<#FAW~L9VzWGy+cUb8GzmKNV zF)2J8Ir|57TSvomwd2?ssEPjMi#Vm!1FiN=gLy)Zb6@0RT6(PuIy486$Iq^l$ExZQ zOSR|XO3O@|_-jART!rMp1tXY#sTs-OEKFNxOLfFsXc5g}KiwG*a_i5d((QZV4}3m| zERW+ppNS~%+D20rw84nH2<(2c5~tHgBzR#59VIsq7Oiy$!!Kv>ufL4it?!VnwHXg4 zV-jIScZ%%!IwA?5s|)eVhC#hj8rIfo!^yb*xbUl1^);hPxawI$yjoLn=8L2B`F3AC zvS>X1`8W;Vm;y9uorBV*uk_FP1n_t;0+#fs;0EylS<#>O;?*iQaPrVjS&CdSI!rr3 zEk<0zvnhvYL*f@msryJS?HVS_es2aA>(jvKPCcE*AAl!b#rS*aYZARHhu-~~PTbd8 z!W`B6;@*W%=!Hl}@Cz`8>mRP;B%AMOY?+AxjeT^<(yMTFd=uVWVJ3;x9FF6B$_YR6 zHC5#QMCvAX(+LOMV5IpGSbX1_Zaw>%&X!9=ov=LdPCI#Q@XtUE<4LHp z&m31>%mA~qS0Hky8TNlU4&L{k0@DWqM_p$;m=3C*&9i{(MxvjWvsmxk1!iN-9JqY`4!LIN2p#4d@c9l;a$C6A(zA&|jon&`=*JY=9SU{-E>6(_)(lJ+i-N7TPpy zCjNgkG4f6ePJGRxfADtj{X19KS+$Z(fhD1-E$njfFexyXhvNm6EScbpul6uFW_c)1 zEL8^U@@*(K8Y}+$&JR~+mx*_}y~mZ4TV+477>`wii=_`X(a#&y2{WKecBSVbeK@NT zPWg<*$`Pq}L^GB??^H+2!@~TmlS9v3)uP*Sb>MvdamW?2o6g(%OR6s)#Z6a2sD;HL z__X~z{Jiyn460v@dUc0sTR;bF_RPVtDkV_Tb%R_NGRFokE5$P-`{CBPLT`Y#k>p@R zAzj-Ti7~02w9(%cxBFGmeZOVIC~FTI8s!SP_&a3ihqLrk;KVZjq6D4z-)3|BZVB? z*G&tK6;t18e?0d4KEBJ)#C5A3p;zb}^qx~dy?mol=^;;e-SaqHrHG#E?}wvO=EAbD z3bJaVunXKVMaWBUmHAB{Cb=EDlLRU*rn8r*W9#jAv^u%~<_5k&@f%f|cfSP2wEV=+ zu^Bk8RmL>iFTm5br-=Q98nPp2H+ai*z&86o^6c*=d_N@47~Z_)6amR;~Pxd@p!N`hH~pt^Mv{7_GX zUhh1rv!#gNIoFJiso4fK@n^`+#W9d{vK+ten1kjH;BA&*2jl|SLmvzp~9YX zI8G^+!PV^bV4Q~$+jea z=#;M~!vmA(rJhflS{-wn@*tJxK%i(lY>ojv(c*}63s6BCh_sZ;LYP&h>ewz zV3)IaAw3qf+`_=}KoKoYT>_bZRZ*OK0h48F#PNMDte9d3567pGyKTA9ZrcGrslcu2 zn@%k9TA^BFD4aPWfuASu(~*mV!7@n+@|~Z;#ne5ri<9F(XLLK(|IwhY5I|E`DAoI|7q1Ti}+xdYuF#CuOiP zWCO{cRS#~%j$z>d5!8%3j|N8X;Qkvuysx_)z9-c}#cDa|oIVK;2t96JJ$l5k`Xj*W zM?UWu73v|k_2Swe&+|a6}j?rB)MrPZJ^utdl+8rG{!r z-K5q!8LWTJ$HzC;$qJ@-5R>uN0%L0_J|2@uv(G-ja9vN-sQgWmvPVJZ*ym)O9D-8o ze}wTH0`ZY;7``|Gr&;CVfKLIWVRIpdzT5;?w={}-J5SLF_nGj~aSk&6@x-6sO*?Yx zpfMqjoqxqw$S0p737?y&)0~%d^6N4hoW6z3+M*ki_}Ub1dRBs^%hhb3D$$Auwe;y`1x;Bs(>n;gat%O?FzKC{c8 z3QVD$IWR2zCqKc!o>rbI#vjv?z&q?chJQRmF5S{0_EYq*?avrYd1(X_3aKpB=m;1J z3@*Ee3&{AbH|f3XcNq031_#X@M~_A7NO-*ps5zGmN8Vo|bL20ByX#$&yUqbJ)0R=W z$ziBssX~J)a$sWrXT-yJ4QPFu3x5|Kr1edibp5l3vU!W7WNE`P(p``V^}n`( zk32yQg#GD)QIP-Y**(R1bc)zb*gL7U0U(qr{;+6Y`#^6SLZ_^sP7w=Kt7D2TER&dqT(7# zMcdD!`=tXUa{`5wwG#R=sTFT#rxU4{2aH$MhC5d3Fqq2`YrUIJUjIJ8{_qFc=iX#^ zxb6i(uMJ@9?IzkfI~`XkIl$k$hv9_vW`Z;N!4{if>ec#+TATHd`IGiS#Aa2<9qcQs zH1{B*S`N~cdOPs8*p8gB&x7#oqpSZKjS+sFr?mUoK=fI$1g`WpK=p@f#KGq*HS%%+ zE9$`K54|tjx?l`derbhXOR{i8;4u1q*(fmkp%05y2gv@!?FIwJ3>TF5LiN~qbZVVQ zR?=nU(5PMDB9f~f{WcM1Zr+2rPEYARt9n_-z<;!D!C{=RxE4J_-LUIZ9gT0OhXpAb z)xk>jm|*FSK_#xRL4Fyoyn7pid&Uav4K4UXQXuPB27b3JBQzugZob_o`{pgoH@9E1 zeq%2Q^WRLN7flPBL|()#sGFRtS%dGE0Nkjq0N<&X;A7ZC5`4lGE=0cq+i$0EL!&(^ ztSTpyM~%Zh7eB$M{WE~}r=;}38R&ZWnL(2PSonUpOm5OYp#$uwc+ZkTAbwpS%fAeD zV^%X$gf1^fBRSwF*23eTwzS{&AH;KGKBOd_q8ZoRiT=4>a-#VGtW7JYI*LkozqnN# ztYu54iwC1aS35mE%0jmC&~x#b36t5P3={IF<~%+A%c6SSi3}yZodI_mrxiSGM5UUzTFdY!BwUv+(Tw zBXP&j=Uij9lspOuLl=Xya9+MbysmFOdInyH5t4gYv@{E1mqcRXy_a-*-Z_EeVkg;p zYZo|achRa>1yFPC1-&@`7;T*SnjG#4B#*03!i>Yy1z&JB%I{i7#UIMynRO+WCFGIK z|FrSc!)ev!myGeid1bm`Y7-q5!%D7bh*7b*Pbi0_Skn?p)W>ay+yy=4nZi!l@Xt3% zu(kwzKM}HYfGO&s8if5&T@|U{wBguiuu&GcCmwZFqL3aI)7G`-_+jQf zacXxrnb(+&>A8a??l*(b_Gl)Q%{7!95Jb%tvH{Tc;uox0+f5lg9Z4P-g46y5)O6@T znEW&yhTr%?bkzbdMaE$13|DyRQvkExc%p}?IsNnc1pK)806NDE!L(n6@a=tztYC%# zy|h0HK0Vxn6Lk;6#rhrydY*zW`X9!uo4Vo@kK1&7nK3TCdXVZKy@(%NMv$ug#-t?Y zE$LQ?h1{$KI6AQv?P79Z`i%7$qpyg8CgaJ#a|z_mw)5n}$aCUNgIkGB+IO5ceF&(g zRHLc&I?xw9Un7K$z}zW=;O5Z&*aA^kK5NleDu;D0+;q3P<860bY*@Z{tfa&qoF+*EcISK6gQ1n9zn2jX>t_S$`8DbCZiz~^;EaQ=4|ZrFbv zXVXwHSt*a_E60*iK@F_LJe4Hv`AuSC)#yq216aE48!;`KQY{gS@!T~#i9^~4SZ^d( z-FVuCmQVdcPpcHu#HSZl9i#tp6YE0oLYDd zUJBa6pEiA>ynh33-`PuS3c}&jiX+r9o?>2VKKhItj)4OX;K;3uKv_ST%goybhgRp1 zMTMR;zkVTz8cXSb3kPL4w>Q9!IeXDK*addTUu1nWhtvBi$;8;)3KipT(j=QAq$+*n zslAHOrMv=^HqOVqRj*}MDf_{-hWOwoec@eku9`I5X(Cp)^GL^sRkXC(g~XmxAma`n z$K6%!0K%O&+R2`3Bza*|$_)6m$%wqw2_rXjPC)vQ4d`=z7IC|L0NB%3FuY>2I4NKb zrVfZgGsQ?^HuWDyE1W(pl=a*U-ys*o|~s#eRP@;PT1&8jWHAYky&uD#2c*glJLR1IJ95p z1KtUz(H9nBXm=-re)e+herC&Ab4^TfA*R=|*8j>HeG z#U^h@SntzB|K3xE0BgZ-)LqCYUi}4APfmf$6Q7E8VuGOYziW{8!VVn+lVNt~MAWq` zr4c<3Wz`eY$T+Kwa4YRA{5I9Z&ey`-c2P1;Tw_f3xjd73w5}!>l9$q3ItoCBMB~Os z>10QvHGIjPSpB;^7F;fM;_2JYs9zsXHNSL1l`5)IU<%n;`EA!4A!eY zE^2^5zRLLBUB0^3=^>a@zavF=9+RKXggY*)1tr$UVVaA;A`<4od(SFK19^@H^XiEQ zF9ti+C~E7~O^!6{co z2~8Y~3eGy9TZ_0(l7SoKuFF0Op2Oj#_UK)Dg~aTQq(1jDA^DLBSv2V^dF}j0yy)z1 ztWcduY9l68EB^dOzr@`|kspwiAxfJAAPKIIg2NV4nTgWrFar}jq>GZR3wtuUy zq4LKZVNT!?4C!})Mn6qO*Dbe*qS`dncoRp-wOLr_q={$EhG1Mn1*V*-z&~Xj)j#Fyc)(*7hUL=R(^j4Z>!4&hjod=0|)Tgj#fmc2iG6y}6xVR3{TjGt&$ z{qj?QO!c!0^wX1K%|i_|NXx+D4JT-9)Kc+u7JiH+QF8e*OI z;}JbO+GXX6Zx@@P!<_AywsF2pW#bVNcD|4v`MwO+&Iy4;qk-sWa+}V%`wL$hNyrSn zV(J=BQBeg&TYvX~UC<32Kh8qJP7;A-EQOv9eHgQ$9M>>sfN zhQ2odS`~xW{}b{*%VUYfcMHrUlW?TqMO$eJaG>2lqL-;Ix&5XTt#xAYTxmIs4c`R) z4;muJ9%C2i5*&JU2J)gRn!H-4T0j3UO-$34tXS27D@@kn_+A&nj0k{euQAX#vxYqI zK1NgT#e(I+ZsJnE7NZmn(~oNfUF^{vTI`~UQ+UDCy4SqA_lg1a)|!mVU>A7F*HcH| zDx6Xohy`ug^eIurjVG(%^V@SU@A^3Waz&2(-H?T^TI{NaZhi)}$BziCc0*hsZ9)zC z!IH(dN^#%}6Ex8YgV737s<1v5O8W9qy}K5&N0o{tcjki4koyF3La=dQ2V4x>hMBR8 zvEy0>-dw#;Jau0TIK%|W(n=oSf-w>_wP>df!ktXA(pCH?>Neq1fhI`=P@SF76M2J^M50>pryh>)B z4aH1*JA5s2MI)bVyfg9yK6_>Z3SrBr9V3O1>kpv#O(@N4^~6&Gmv1k7frg(*hrt~> zk}nM##Sx{kvi0o?U{CydQ0lXW@jAnB=xvk@&e(_=myQaWk{y4;+Zpmr%JI|E8d}}@ zj!fEll*-1m((BLW;h{czIF)@2`>Jc;)}elQ@8w4vH|HYhe2(IbVx)VX<$H;LbS$Y z!L%!I;o{;UY+)RtzBM+thCtV*%Y3p+K8Rm_K);}s=$u~19|==~X0lLh;=K_~t@35- zFH&5$`KxTSr88!!sZq5ZwaD8Ksy>)cpjqdu?4f0ic(&{+88CA=wmDWn=;Hzi(On0w zyDK0s?jLdJzZ{&j^00RGY&du7JO;<*f!p#%8hdL4dCqA`=A>y=Ydr3xio6CMeX$Bo zO*ulgemp7DR@*HbzJDrpGS3D*VHaiiryRmvtZ8-bD14;hjf&o}NG5xbxkUtgJ@$Z3 zS*zf?91SaG_LA5chA?{DVmLf#JltKs3@^f>wdYfxt1r;A%Ya0h0}Qx#9QMyWjT>(az{4$1NcL%CyejEK2g6y^ z%J~YEof!yM&br{z+H%>1;TPe=i0u%$Tmg!+T3~B`fAMuW1{$&us+`8ar0LVelh%5o z<6d<%NY5gbFAsu^P8OZwlMIuGnPTt)VeU*{gkjTziH1rh4S4G&`1Lx(QQim+zJS(_ z8ANjT5Q+rtYuCSI)MqZjwi!`S5^)S#x65Ps>ulUNB^ZX^>;tp!@?fB6h7-E>6P=^& z_<M#9ZaBD6NZEwUs%Z#eiy5gXUizAl$^Ju}|9#X+pl5rEXtGB;BMASYN z(T+|BFxmSTudZ80L*#$b`!1tlN6lkW{hjdw;3{@*YoOb9#6qxx z9etZRhK4SY)9b~awEWHya zeD?Anl?7a+2@U5kZN*bk*%k->Hbul@cL#1BKLo$*i-nW(zmtx4QP}J%_*)zcv3^uF zovoaSw^NPq^8yaOc8rJep1~yVpAJqAd4Ss4|3Sk~bp-oFvcsty2b?-bPi758M*|0B z1`H7Eb@`&WAqnF0PU5D5ez>A37KX4L@c71JQ1G~dHM8dk`(}UW+h#I^mtH|pfZ(2vL&^o>**oj{N5c}mOO`Ht(h=X^#cr_{+?C3 zZ4QU-oWO`fC1hjgG#qA>N44_|#eLlxxQcxaeyJ;QYxqB`EjEO%p)!&@&zstvk*|KN zz+=Nm5&V^VL5gm~!H|_wa{2Kk9M>)e`y2xH-5s);%Pr*6tf3^|)r$Pc^1<9k3K;Ep zi#}{Q2-DB9xMKVza$l-lEzdd95jjz?$2N+%t^N*24^74HS^2W@dV5hx6ow-rmFV?> z2VnKBJ!m!J3d-cRiRJU#sB!f{eDrz&eqJLZh6dXBHU1#+x!MBD>Xad_Gllf6oQb+R z>9FNSGnJnVB&bS++|eW&P^wrxDAXK1Ox$U$Tr@;zYE+*Xu^jH*i>3MD1`wT944%r9 zu}@FnmP(u8PV67j|Mz4ZCOpq<5{S#cn&HwpM@e|UPzaxz53FeuZun4ugKJ;X>t7z> zP31i>)^-FaAAUtdM_)n70T)bpehTu0^FNJKtM1{;Xxff;=(jc#FDSNyU7Q&PZQlwV z?ME?nTnl`*&cc$(_gUxWA(8}Du}u3#v#hSmP7*j{KQH&uLN+%+mEIaQs5&@xIzAO! z(&-Kvbj>}G`F>qXN=E7lyTAWfmWjr*^5)PtaxSR|cZFL*9_3R)3b5}zplI1Y7@Mt1 zZaBUrrSm6>Z3p_me?<$h^ja$HHqyq^p`T^DvOUSF7!|zRISacse*=?c({M~bWjZ4x z6m4r3!?d8wAgaj&TaB@(_M%JHk)I1+7PirVyd2Ww>n|M`9Z~2ans@;JFVf zz^z_@OWh^#>vx$|Z+4sR~jCodPFQBdQ+OBfDu8iSE+?&Xg;{q7R?R zW=R`PjIIFJk1163U>?|-{~-X^>`6g`u%~fJ@HC*V`wMMJ#!xL(q(8rnIN;@)Ic*^ zW0#E4ie~OPX&5?~9P6*~_ZSZFwrfIVK=Lub|l`k1@ zHjuxPeS`Z^@5tY5>CaSdcjPN#b}))Um)V67V9M%(MKk-QF>m#Cxoy7Ie5lR@)?>|C zX7^-u_OVAAir$%*g?nIl9`d%vfA{xk_ns9_0yxfRwTM;jr5x=N(0{`xBKY^*SoD13D z#VYseu~g$GQ=N8(Q_k+s*EN?j3cbI$1cM!1>|JlR%jBVG&nK3Br8$iEOIyyaiA{Ho zjv2?^&F{zCELp(*w6kUAZL?u#PIz9m!&A&_6f1BmRR{Au-|jH`hf3J-_JZsEng^R} zQoyLZyU$&Vv*Noh#xvnH6B&1{No>lRTU_rE2exzFLN5Q{7pY1GFjjjjxmi!Tn3>8m z_`lQsiXLAc#Vf|?@f%0_vlXR#Cgf@hU8Mt;ik@9?VJ!|BGH1dCKGdKRCUV6eW=ZiQ z&d0YuyX(t)ZkoYJ{;X+=bnIO%{??SOjHIxF8|8G6%P2Z4J*;KTT)j#7Hm7uM@&q&f z_6&V4M%`2Bl8X}=3>m>5G*n=pE$m_TZXC)DpEZ;nGGh(%I&z-0_?tP~>-w5wzgV!_ zXTIltoQRXYAM&WmHB6bm@uHAXYtrM&NhCK($%*US*u&g#c*z+uSD9aPH?XHnHt`>J zU*~#l$MXG;U*ZnFnalRNKNMA1N!ZHw$3<$ngZSTZ2f4R@75G8&BfjrV0B7QF%pX{_lvABPl%I1(o_}D}#HmiBocA+*-osOoJ2+mGzw>-F zAN@tq*>-+0WAA@esu37i_2rlce?v2Y%cf&^%>@Jb?6D)*KfQL$p!eOPAAQxL(i8(e zta1yp?VlOXiT_FG|Cr8BSe(Z^-e=7kde3CMj2zjmztW|0j=F5^oEpIqx}J^Gif4vQ zpTQQ^?%@K|+L^IVf2xkntrw}z5W3cDr}BHV_HqXrEqPy`(d_#VBlxH$75-nKJ~uYP zfW50?CE9stFgGu^fLV6ihFiM9i9fI0#N9VF;CqOpNFib&zv5Yy)TMqjUu;ZynellPe% zA1i*V&`a$$K*S$AK9}+7WqGODRMzeDSl%lcx!;AKImO9KxoulUv-`phOLIQFVwC+B zh&21u_~4hF+|jI?0-IKsJ-NA^oBlV2yY=}4*R&t_M+tsx@q9=2eqcWH@%t=xn&Kut zYU^Zv$pUx2;*ByNbMFW9bFT$!{UM!END%sxgMV@}BgDL4uRoWMe%!K?Zp`N!{n$ST z2Zcahtw^^2+4b3nTHcA0dm z<$G?Bdla*F-T;2?pKXjr!flTIIf0M=HkJ*F(P3A`t1#13o4B|lIaa2rmz!ZG^t$tXQV8B9}=%Ii25fXf=DvV<~Gb_ek_1zlqxzHjufbTgRN9k}vWv`N#e7 zdC7dueGhs9CpXAn`a%Rm&888F4eP{lZy<}GTY~Wn-?s95g6BwQTlyT{d<5o&% zaW_0NxufGlxRRCvrtzDx=w|sD{_!|;Co_DS0 zUQYeYGz@RyR?Jr6*Znk<&U*HZ8*b_^6ERCxO-s;rgsaCU9YU(u@Un5x|S=Ijp7 zMCOwBT4pDhu<6@ob3MjooYU_v(Vfdn*gx~u@(Uj><=YIk`N&RV)-K72|L6OW8+r1Q z^hCo{zIDGE>tWf%40C+Ta6TIR*Jon3DqWA6pmdW-T|Y{iCv;*iDM@C&6-J4eZ!`H_ z=JVN0bU2&-Q_8sPQ{@#iFN+q3E3k&qS424{o-i-EC0td@BW~`;v&@a#BU#C56}}?T zl)tA^Cfz;Rxn10kft$F)Z5qt2t@k;1(@V_ndkVaJ;!%-8=m&0i zP9@{$v!4@7G?|#W8Qiwh9Ha8jlj#tzXS%$GaVIBlX4L;?adxMx7(<0B=F8I^-08-B zqIjjJqTh!4OkewMF1+!Ws3@zG`+lQ{8yD-t-O}GCT6i*>v%S09c~-j`AM*&RbPk{A z%G{edGl9|9vXkS!hc9GezX{Cam;tPGC-5zP)uPYZ$=rD5Z0_R$8|Q@S23$~r4nLS$ zNjGXt5q-RVRjRx+gAssy+=Q)a?Cs7`{Ec60rL)S^`K5A8xw_NpOxEB}T*}ZgrZXau zsqbFQ-8=Dun-ni$)P55#tLXvv)Tf&}vn)b-w(UHpI-;2w=iA3b&-Z3}8hfSnT319V z5B^F&emu?8w2oyL8K2;C`!+KPPuDu1x)s3q&ELXk4jIeadoV)xY9Y7xvo3dK&?kXc z*us_gXE4J}7jS)pM(|&=A2Zt?n{lESBe+b*K+eajAG7+pk~Fve6KAoejJq#z_@2K| zWK+ZR`S>x&%<&d@-ft4PB;w5~UjM)Z23W8;tN)2^?f4~{u@xDuUG2`9{actj9;ckU z#E#5}Lvd1xz%G7x`!eIUvyS;^qr*g~Z)ZyMTo~DlG5pU819p&bhs(`aDsmqEmTT$% zovB`Mn^_ZV%=hd4EOoWjHPksSYK)*PmJ=P)L8$VJgY*v-rf zT)_E#sNkk9SKv#6GPsl78QfLB0qjqm{(OT-k$rMt1i$Nn65DH+AaxBhXI-wH=FX=T zaSdJrS*x@4qV3J90w=Y)s@>7QDsQe4U!MG$3Dl6|uZ@z5Zie2fdT$QWo~C$iLi>Kk z?c4-TeHIb1~FUPr_V9x7%bKWnmi_~`P=ElES%j6#a z%A5&Q=BMw-VYDO$d~D2mZe-Fr=H9PYoQaph=kZIcrCf3SnkRR~}=W%;mck1gkK z8qeTU`Z=%}F!q*0{t#(d*>Q^| zlrdBHiJ69PI$WXYb&?f~hYX9~U}=b^mS2n;&I)r@Mx1aVuqX`UdekZ`kry7e(yY1v{C)vi(xE zr$*eJ*Ja#8W+=njU6%$uvg87RFi1!dL=l}eiz@}|=;TKJv&q@zY6nf6b@D@M6a+P`~n2lx{ z%%t?I%+B&eZYW>Ig!?UG-rU*F=~vk?D0ir`0BaLF8>ye zx0u52U%QcSjNQOLTI&c+7OvtMCUWsPN08 za+ww1-ZDBf$1#mfucRwOsu(?sJtZLJl*<4W9 zC2oq{IM(fqA$ux)K0hW`#9Mq5v*BBWXFocC-}oztKlC=7J+{J)trJ*$?;YpypTZgT z_Y&b9$xh@4)v>((Ei?A?bys$<&?S4!aW+5szX|NF{#Tii-8Z@O2mf-mAMY_rIV1V| zJ;L``>%;0QdGjM;-!emj7W3aKr|>^VPh?+ou3}C6*YWw=SMveZ>)De3=I{f=e^2Y|+E1{D<#NT=PI>wvoHUT=cxo z-QTpDPk0@`ir;&%uEKnnX!VkjchBWYpWWleWqpxO8aR>FuJ`1p4|C>QZGJKX7ENOZ ztli8%OW470vo~k;QfKm&2R5^Vj&5eJsVwJjvtd&<@@j**YNGM^QPJkgnVK` zhAQ(3zm)mfJ8g{fu3l-ae67^3$z2+xo-6#WP0To_eNvfHw)A;^J@>IUh3R`~&L3ZT zms`8_GBauF2L54FG#j@$gjLoK1*K-9H}!gq#VQwO z@aoB&&D~jy`UQ^J(l3VT8@rCpq@(#Fk4e1z5-Attx11mC>%fjucV&ZhPcX4>X0T@Q zL-}9CvY=ZC4xXOkYSV7nIRGfX|lOsE{oEmWV# z@s-icy@N4)?1^#gCk-)MyDXW}I#$Zunr^_KO%-v@n}CCUq zAIGjOv1RLgYMBcT3)p+-wfP-eX7cK`H@Rf<_e>N$%;{b&u6z*^#2uO_&%P?@@xz3N^ZOmQRqmG`eyT@M9NX>li^Tb2U>OUULk%?DC zW1dDZ+D)2FPS+&G2dOkCP%Dd(HYb=GKAnuaKI zvK%M&@uOx=w^)IXS@e=~6WF-Jc3bc{E1mg&bt2|vTbxwCAV8SctC=xBleo5+^{le3 zU^UC0$M;Qm&AqVk?jX+0U5a zLu*7Gy`0qaSS9DLnZ{_e-{Hp8-)CClx){yf`=pgS>bPvLXr}fb$9Nu!bO5l zjQLKMJY#vA9h>=`gSt6Yb1(kK0t5E>R2z1u!WBkF$$23wNBGqd-`vQhr7{Fp_dd@#x9mi`Ilvy~j#U*6-` z(?h>96>`A7c|DRJP?E_-y{zUYB%1SOBbP8o=@+IXdM>m7_+rk*=s2_a@KGkcU>MVp z7R`^>Vc4&tU{>$&A*S}}Rc>X2Ft4u5VH`(168+5ZWETg>@$#jXd|loP=~TVhj9O|8 zH>snJOMJaWlu{MI>W{AFrvF>O?@;L$JsLiaH_Sf6)F10$wl3Fa+`OB~h4+b2uHlP| zH)+E2Tm7(beF&wO6X>WEZT$1tkm^lS!tz>UIC*g&o$neXaEG?RpptEvUpE-8t_eW0 zu8aORX1K)i@jGx@7zQ7@jo@l!3`QOtCbLO0m;8LX4*ab1u_}5g8htqkjUJt#e&7nV zy7&N(@90MkKYl3Aa2_L(UvQONhwC%-0zDqmf8U73IB9Y=nq;|$&*kPzj zFaP*P4rvzu&@W_iO;A&^)s8o+s91rC^0zJNezP5{_z5q;`Lv!pU_>@G$QT z1T+g74X+o})iNAC4E5=`Ny)HU;IXV)7)1h)%fwD!7eewT4?OC*4C>pzkS_UJx=Qao z8Fq0l+AB#Rb4D~OcrAl}jmDr|e21QXa0KL^u7{=rzVz?eO>iit4J)qpN5#P@{E-Ox z>JtNM=$~S3uvW;Iy(v(vW<76^gczB4rlkX3CiZB#HH&0)+K_X{w$STD$e?Hm86c16 zQo3coY{s@~*|Wnt>7}1fWve5nLdT~@`s3efHk*wn(#2U=ziKgRnnuGwxlXF;>Wy+= zF9B<(R2?2a99B)$#J)|#!02u<5uZ9wY|plVbEqXQIlma)a1uOs*g+neR|s4vL(m3A zF!bF|kNi0b<>p`L-1vI&{409ZRRIzF^s#Gk$(LqgJ=p-qt_VPTA@|qXa-DwvuNJgY z+Cgu0Bvm_8O17J6!kA6HgxP8=b2s?{?frA{pq(GtAaDii#0QDyqXejn?I8~TZAQQ8 z_o4H=GO_04Ky8u=dJWDK%hgxW>|F(vV+Cbr?2gds^)1nO=EG zrtS?|Eo~VXuafOAsls{Q<~Xxr81$1-648-O&m6h~ z6Y~^c#hpTY^0gezQ57BfWxx&2(3g(l_gxxuz}($m3XJC5g*)}i(!L5(5|u3 zR5Rom>R)ar1=X)`rF9nV>UR_hqP*ZylRVsXcOvHZ{n0Vg2|9KOOs|EC*f9DxFBej7DBt;$n5`awj|@zq@`xLD-+KX@PUz#YPhB+SeJM#BWeirUE<^gx zfi#wJz@tt&vZSBkboJ#p{QCPdiLq*?rxzUp_nE%9_n#*Y%BJwEbvXPNsto-DvT=iC z2|46{lg8-|rfZki&~-2N$ugb>!DRp2WWTEt1m|iB8h8*YBweR#RtDg?b!++ChK=M# zKni4z$`i|)i^$J{0kT<%t>Wx)vt_Gv9f|x64qnur!2`;(v2COQq`uq_xo=#+;msD< zJ7g#XZL)zmCmI=@D4^Qm5%{HeGP<<~Y1kch9k5zE>Kq^d>)5b8PJz%Be387o;ady-Y{M*ls4!_n$%e42R=;9ta&OVc{ zrbFRkfD`qNPXiZp1HT$0@GI^I;kS+O!*MO_vzvx(I0d`;LNS+Ff?6+%sAaDSd9!gC z90FJ1dknCyh*CM98CaY5jC@Tk!|`vUU}M}S(2b2olU2Jg-Ih&<4Y4E_i+l0&ZThjW)P`pxCL@Y1rG&hfj&)PARqRdYSbG*^4l{c zWK%qjDE>mH&E5@e#wwCc(XV0Db_FKz{1G_C|0cuK5FUD|;M0B=K<{V@m;`o;WfAGH zUUY`qEpHPTVR@vjq!vG3o`E6n1P+nr1`?jax*& zXU>IZSC+vQYa@)f5l(W$S5viVDiE|{2Ktz0iM4umlRsySq3vGG{|udZI8{v?fMws3 zB`G9JvSgR#&fGJWvXnw7A)-R0w5QctD!Wi9(kdmTEO+Lfp_0&|P5C60N{LFTl$P)O z?Rj|abKN;Jci!Loq8zG=j;D6<&zCeXS&nmQ>5NRqzHL3dkk8GFOIjk;quZFZC$i~u zr55%UR;8c5{bsCcm(m^1>uA!st*AYxh-&XBW7P)Bgr};@>7qPE#$b*a+G#!$nLOtF zAuI3EO)@_8h3z1FaL=@c#36a~&^r*d-1J21pc~oUYe%*wNyxGr(xxe4+-L0{T|RdJ z*&I@1j8~VV?{Qz51*^7k-R=%1Ve3^^>t8p!{Pz#xi)y#})y`^daG7icA`LbV(OzE@t8A!H#kWL*6qk%HdnBY4q)NImD zR@rX_%_!YNcUaA3S1rBI?mc~qUX7PS^8#L>)x3r1ZoW5?y(#7(5O+g!H7QE|Bh&CT zbrbsgsvSK%T*FSu;hMaY_S1Ze8_W;Mooq+Z7yekAIgLFq&bUR6G5QA^sZWW5!yWm* zY^~EUYnix}R`cuFofFTnw}j)&i_51`PT3@MKR^@ZPs?GImh`Z(pL5U?yN^uiXeT$% z>Wl{7SfV!JT7FjfBBVCN_1T|U3%jo>^2IcKsjreIJIm=b``hgsTYe@8O)_tQBBdO2 zbM`jluXUI?qov9mZH%Rx%EM8YNhEdBEvGLR%hTSpqs)&ZT&ub58S`+VI(jwd8v3aI zojobJ9L@WsAuk2XJF4jG8GwU_A94qrg(A*IfD!c3;t^RnE z*>qqdJ12+o|GV*uF6ya4Zpu>ZsSV>yz`MEhR7NBFOjJU!eSc&H#;b|B zJ^Yh2Jv+_DOLYlj+E3H?d~<3%$&kta*g$Q4-jYHGMd7=TCs12M7 z;*!q~lP;x8iai{R&dx%6OV6Wcb)T3B&wD5&ED2psGo{uwACbseBV-bFOL)9v84Wl8 zM^k(^(KvNUwDrF#_6ieC2X`*vlgYO567z^22xwrBnG{k7cT?*Bej&R!T!2P*aWf2q zThQzcat>ocK9k+o#LVceLGnJ?i0u=jW@od|(9RcZt!^ahbl%JPBMPXWi>UB)x*0w5 z?Eu}%vG1~+8(O5%1?{QNVl2Zyp`*hc!e^?b?C;275+YMWX=f)ikX=kA)ua$3+yhV!UttE?C&wOdetkYLh2?J zg`Zfn_8dCp;wq%j^@1@uA;YFf*wH6?2bqi2DO5)$5qUh=$K1R-8AazGr@?1?Y1dT3 zfITy*)tD%~FxZQHr*1;sKf9=OnHKFF_n_x`V`<>eWO}0MBVvw~3hSht>6@i7?B>&o zOr^s!eusTJ+tLxm?+Tto2P;jf$&LMNN0cR<`0@bSJ=ukt-FZ!?B>bk97tNWx9SNwd z)*gZ0G>(7W#m333r{mJLG``-LIpx$xqj$Sfm#ZU;@R|m5_%ruEXxPXqGH+;OY9eY( zo`ObXUPAw;f7!_=D&gKkhfwy`Y0NdBNEH1qA1$1u<}j?LLDS6wQP|B7bVKhtbcXw# zTZ2s5`qEA`QzVVv(G&26c7&O@O%ZMWxSLik`^d_89Y&+)b?DE>i%^eT5XyP&N;Pl& zWX#syVMl_;s8dQl6S7HxdB1Q!n^4g!blfINlY`%|xAv;AkNfsfBN9m)S7O#=Lp;q< z%A_;jo@O?RMxs4CKB5~UE72=uGkSe!0rNSkkQRlNGajP3Og6oMN~CO%j7Kgy%Ypcp zJN8h^r@PopeM#zBn@+Fz^O2$9DAj1vLQ~jA^ecTonpINIIBu+9vXPEM^5J%5voHhI zbI-qrRugI8`*8M(Y9{jZZAF$$HoskVCN(_Jk2IZT$2E+^ zryHo@@JR;fv@mBH%b3fY524M{py6epA6@=s5;upc!}u*&f}R#8(vl!R-^v|eQ=gdg zla><{^zIndcZ;J3*7UPuGTJBzFJljTmIqGjra5z?K`jN)=x30FesdYU9~iK_5_FNsXFYWVN2%z=OdJ^JtUk* z`snS((tI&ENk!g(G%rI5S^!$(=c_|}Jt0qZ1g!T*Rs^BQJC0Gb1y)i|GN^4P! z;268&KpBdTz6YP3zD{p;)eD7RCFo6mS){YPiawnp=g^{ci}g+0O5cw;(hr*jbmzfv zWc=BN}k>QWAvA;57O3*N3IqvXitk|!>=FD8P$ogtV^s9TJg;meRyPm3<^T2&S4?z zmqTgWyjJ=-Jb=RYWpvp`TXwF@Z)VschnAmR&t!bf;EO1vp$|@!E-QCNwiEoRgY6+| zq)@?3e&b2;@O5_0gjAX?-$F0yR?>XiUbauK2OZfc%fG+LwgEggV3r&`$xqt44qdmi zq9J>Ku$_AnQEErE@Q*v!6wj7q_P-cqR_nLYUaOgClDaKY@Cu*9F8iCYD-`kcsKbKZm2 zj9oybH`G!4`71~%bSkZ!e_a^#TmlvNd9t-8_59T*vJpJ`6(?R=%BFf__959t+aEV` zb6qObUFsidueU(ocjwS2#@Y0h>vsC1!x>G!6@e^No0-6qImm7!pYjb98iH%L(%?m( z(eAY}wAsp$p7uO|Bwt3-ZMO%|Rna!Iy!0%xH%nkY)~`Vjb}}=U=dfX@0j0PU@%LVC zps5?~qBx*Q!}nIRcW5tb94q3Gkgkr5{)jgmchEpfIwnyzEQiTU;vADrlN|g%xzNn| zGGw=N3%jA}FUr5RpQa2PWWVyF=+fbx?22D1C~8^_?VQau1L{xlpDnVW?s3c5XZn@M zz2zOV^j-8YLy9!;mN+@5KIr8Y9CJtGnx~!Q1kCbWN8-0+j%;IzMHFn23w|4?*ua%ckeWF=TkLXvE)88 zcqtZ1mVFU^ky7UNN$=3ARr%D`Y!&11;}Y|WX0Y>RRkH?Fg=9BNjM8i;o%b?_Oq)p(VhCRrC?M4rA;BpzrtFh67>@O{$^ zOzNVbx34LTj*lkC+#_*XkRrA`t_^+0*Av~}gSg5#j{GrR2wioH0lVZKUibAV=(^{M zZ&eK0=b0sdos+){CaxAI2QPjE^ZEus*v)VFM6x<5XI+5GOn)rmw~T1x%YrSZl0dne zUj3_r?IdO8Ynd&euhO6R!@8h#0~U8)DKsEzncDif3=Mz``(9Vy#vtP*OVt6bc!5K=m9P zZZILc#ZM3;bt(KheF`c65(I4vPl9Ht41E5UreNBVo8X|n7_k=r4&DXmL7k&9aOURO zL~Q9h{Qa5^E)LAamrFn3wXfTO@%dr!X>S57Pjx0qXJ?0ps$vg8L`CLCSPb5?An)H}1BYY*`{JI9D?py0~@X*9(o|3g$j; z75R;K{Pl$6-!|eCs}JG%j$3$ZqZ+`mdET&|>sqatkcZzKSHPL#X*j+v7yMiwYPXr? z7=p#9cfob*p4EMJUPJ1) zzUM8Q&)rA0y}&j4B|Mnc4<_zQ5Hx!a^L7sB3np90;=4cY@m5sHks$|3;=1~{-Gh<0 z;K$8)uS$i*Cd( zH;QOqX|CS@*1*4QdhpU%Gk688$o{rnKs+i4o^4BoVa6+=r(g%M4gHTNaVi0lmA~=# z&@A##+<Wg1*{eLit2 zb`ad}rMz=omvz)Sl58^>sz3E|BTNK3q&nW6^z_Uot((_EQTH57MWzwcSbehI)yw{S z!4<6duZheOB4#K|?5kyvMCHXyPA2wVZK+es#goTzods?`tF5`|E z%r7g#L3t`f|4uZxFD8f6x9H<^S$Sf;Ik^7W`V7+Yehhqgno^&xA%olU3_$GDeK_nh z;k}>hfb~ZG$O@N>`23koKz`{SaKkN$_qvp8VEZ|fPq(x1!k5qRzuGDA@wH^UrltW8 zSuH27s&_bsBL+A^W#}LLl6TN?FMPh#mPm&L<2T)Xg5v+Ski}o7!@L7_L@7dzuqPr& zcg+UqVPU{qSnPuTWR!D!l`|}+uj={k@}zXg3Qixoh+Q-8;P8-fkUAJZMgp&azj?Cc zRe_g455yAtcNQ?n&y~!dmO`ALN)g|Co1j;gCYT%_N=!R*v73Yk`1HsMH?KbkOJ``p zndyn7HD;KHE2qLWk$3USyH((LXfRmUEDaqu>=nd-YS5`!UaWHc zIB(0ZGhj)#G)bLnN`AO*A-><&KnItdgiR_T9V+fbWp_F3mJ%WDv4!M)(Nw{JRsj^r zdyl{LW4rE}KyXM{4&L4Ksozi-L!34gV$UrfL5Wv5$&tw>kz_kqKC~O!S6Y%UeP0BD ztG?mjpHBr2v0s6>|9;3sjp2~mSaM*K7CyJT2@J?t5u4uM_}XDHn67ah#P(<4?SIoi z_yHI2Fi#IonVT)}4Vg@?F4{@_;|d7xP7o~?(%PaX^DSvE2c;QBq)*8f2bKiDU5{ho(~=0Xwk8wLK9WJM|7T0Kf`ibz@F48#Kp`7j(xffw3Ls6Nv}2u>`qLF z6Bd5MdT*wY#hzuje3Jp07BCZ-d5gm0=w;BnvI$pe0%CjWsNhQXMIP=c#K)W!fy@zE zQrou|d&*`D=6%tF5z6EEhwEY93Ueb68IcMej`V@>gd;%iw>a7P-Gn#2T@qxsOT&&g z4}nOsCCPTo1HQau{JC6~#6^4nYdq!PH--XDF2N1fIRyikKQ27B zXOFPZL|8XE52`)9&TDae08WZlfVGJ}WP8nG-a8)!NG-g`^m0X_ zzi_;s|K6UE<}bjlhXR#1Q(%_o9Js@x2aK$WB*v$k@IAa4x4m5itC0^$gKGGyL`qCRs&J&;YM4Ji$E2#`!?5i+IHX-l3R+H3le{!;UGxSa?#$6!{EIz)S_dvsdWY3g^d6gI& zo-`#Z^80yPYF3c$=q=>P13rE|&b?+kF*46~9kIT*f*9^ehEE*b|^vCZGIxC{9i}N0NJ=IEJBo4sTY{96Ld5 z3IsQ#iT2&KB%RwE{`fkPIBtH(oAq20Mo<&ttq?^9V(j7Ko;X;hqY9c;&exk}Gy-FI zo=0!af+Et6unRKqQ`&QZpP4SU+4%q~csfD9Inunn=?}sAec3>I!Y$CeIElm(w`X0Tb#i@TWaXan$Nc;8U<)@ZyXcF$+=w zWygy-M)fbSxoSg7XFkCAX=}YRqX(~gPUji_IE|M_OayO?J@GTOiO_MoBurW8NrH3@ z@LTeg<7T_Co7EEdv!fj-3{T@ay>jH~3mGWk6M!X?pWyE^CCOd8ELfQ~fgBa90$Q8Y zVQ1~b`mw}HaBCxm^T3lQwDlJGN_OJ5KdmS|Kv&BF@W`>L%jIeckNSe&L=m+ zuj9j7^PH6i|eK6uKABP`gh0)L`hZ2fly zTzn-RFL(V3Zc2A!r^+ncR&^buv!8k9D|bV+v379OC6qW=yW+&&jrdu1F)!+#Gc?Kw z0+;xj#OSm%Y}Yk_5o;33wCQi~5A_7t==EICBDokgtlI>QCrOd*T80E_Y=dPs4zT-- zBpLW)PF_~VgUNZZaNin9!t)IvWz{~UNBRmd-SwgV@2*zCUlBjvVfpQ3MDU#_{Y?a3 zt`vYu+b@DYw^xt{a<(8)aE+VKgHZ3102rK{M?iHL?>(@FXIC}Uzbk);#UvWRFSQ|# zF)ZYTuUJnM5`uwht0-yac*FFDJ;2AK1zZ!^iR}hl$t3&9(7130d;-?PZ5~g-^36tM zPmCzo?(un~lwl{D5wwE*yOoOt$iZ@$cR+(6^=! zFG@O&UG`|f#lb`1!NB4BkzF!7c+!5)vF%+aQsQp`!xW?7y4J05p0*pwj^m!y-&^pG zHuL(irJiJJwg!BrAquT>_P~uhEJ;#5Ltak3g5^CNK}4tq4xA!Kk_^1yc3)$d{c1DR z<;~;dr-LBB=or2;b+vu`A2sOn^bjw?g?rz%cjHf~8{kWhrCxLIlR)~)5|VkJ51Kb9 zz^__~P_pR|G@BqpX7lCA>d7x~c0V8beK{-0-q{Xfa%SQ=+y4lDdKg0$9djZPeIHM$ z|ABdPvp{mzDg1cZFP?6pC7f|-!2ZIREVvdHKwj!y!inx;FbnA7pUNH}<3=IQP58lk z+{S`U4dJ{;Bc{ObKS^LTBu*NU2^iUal=pGx6)ch63~b_FfGj^bk}NR;SD2QAx!yDJ zO0Mx;boU;IfJz z6ra2fh<}_+ipFLG7pc2ojte<1Z$MNn(Hl#Ac5++U)1&bpaapv(Hfy}FZ9P~Y1a3LWZItfpL z9b)^4aYeYmawX$?>mPMgJIAUYF>nw*;QarK zrn3?uwe;v)9~Ab&iGObK3-+9t2W=f?P(*wld#maflN9AeCwuNkUUvh8@2~Hn%bjYF z-sR^)nLQs^5uYpUz1efo`8mmqWV{ixZkr7aIWI<6SsbJWO_?-q-o%C%%{6pl)>9gx2knLH$S$_5L!Vw$F#rE5#+O)a4v@e~&`L zzs-la(9<{y4L^@MPq6$97g<#IHk`F+D->Qz_M%5WbN!|JOKF{EIa;rH6~(U!FlvmxOpksoqt8~}MtQE*!Us#Z*5A6{ z)LdnpSzT<*nz4o`Oll+bS#O36YbqE);09FNF%8X3osZn-7_+y-Z_rhT?x2}_&#{d* z`%$Oqc6R-h2D&T$0ebbq5zRBON1pQM(cRYzglpucI0X70rTvymP^qayL$2X5=68N2 zGTmd&nB03sn4qupKOceuV!ddrv<*9rEoIXigJ8aNF!SSW3bK}4j-!xyaiY?ldR6Q;-X7$&g(_A(CL9X1ZP|GX5wdFO&`TD{ru z77zMPQ<7FZu4Y!xI*y8x|6`pi9FcRKJCcL`bX`I%5-GPrp-#=hrDlF~`yn0jVxAJ3 zXcUIl;&bTjcQ2H#aE$MvqtA>A)Ec&=D^c?w#k5YV0nN#^pe44>$YQet?W7!|3TM;Z zBQKa^YeMj3FR&4jWN?N@wjVZf06$NP~u;3Zz zs=xP|8MvHDjieO0Hj{co*g0e2$kjffH`j<2Gr~gK_`A%m!$p)I;)JwbeBv((xQKiO zsmy|~Ams2tvmtid0o0=qLoXN$gv-C1F(wOq=wP@kop$If=SuOX9hR4vKUWLb8xLpF ztM9zw->Jc9_^}j<|D8vzFZ!S;!&#^{B%anK6rwLq<%ssjqFs6r?b%}5@O!QonmT+C zb*$`VXMB&ND+LVtwab7VA3RIFe>9@O<6;gv>8A8}r5w%tPo1q1meFaFd8lN)60+|8 zj~b00r9!6(4tqsbAz!0X;f}#cjK+sSWG%&`lJcEM`A`=s^;=2hXT4@(A3tN_;v`Vz zgc0UZ+#nP9M30?q+{oRRUZ9DgB`7j`F6+BWimso%ghjdE7^`)c(5{)oNMctMG8wH$ z>*_1{kw&+uLyRU9R)33@9@C*}u#wq&*#b?qwMY67S@zY3k!HjMS~z0|+U8#&><*G@PzIuy> z>>A~-F+Pj-y-gCvoeD-GqVnur|C?y`luhi}moq5n*@dXubTlB#3adYiFq+Mfy<(M5 z6Y(Qf`m`JkyP1hBOz-kf9Gnj4ov&a!TV+|3-qXw>tvEV%cmea(Vku4SyMQvu2)n*2 z2jzcrquVqp(GfFGDx%s!*KhUXN4xg%i#9)Dx6Wp0rf(ee^XNu{)lKx`jB(+H_cxiD zv7T&Rc?JW7PtoYZH*6GhhiXC2zqxiNx2}F>QpJ7f%~KL|k*F;bKa&WT;)#sp(cl7TI^uX#9EFS`&bB z)2`DbK?f=p7PFFSyJ=|mc2+mWo!=a!hKk0M*=MB&)cF%eYeY_<_HBRI<6)aAZ^jn- zda)6zT>po;Vbsmkdmmx0`{yB^q$MpFsG(&itKlxA3{*NQfpYDeSo-@B$5iY>*WO)Y z?Z2!X_Vf>LANqVWm^tN; z!75&rYp6Up#4d0?eCCwE}thd85_mULf05vyd{J#(Hskf>OO^WV1?7m>OJ4&4aF@nGv_CF|!2; z!n7P5HtSewDwiPlG;f|>H>1$e!s*pz)OJi=GW)A-HL58^+s4t_FzRi6{O`}dyySIAG z+nwk5R$u&S_zpX?%A}VaP+yP4WwtUWdnMULV=GxD*Hm;r|Az35b{~?|3MH@CP+A%J z5*|{x%6iSY$y_Rn+KJEgjT(dLO-HmqiW!#M1eTG^s+lh=X>xKg!&Hga3)+=Dj-9sNuG8HU z4uJE7p5;@jnLnFm9`-_uTB;Fqc`e2DX(-8*OS#%kWDf^Trwf&;SV@x)$RTqW1-}1B zuQ$lD4{m;AH@%+21TLO~&ffb>&o}5dJY2Q_P5QMNeOSb`ojHd1@e}a|-6!wpy~xcp zx9u>y^Vb{n@xod<_;-NJ-Q2~H!T{8tql@5kd73mRg2Yy=M~9+4Xt1Xe3351!A|2P$ zc4tW%_3H@uF_wZZ8Qi6>!>Xv`?=JdBpv&~b0`}A5`>anS%dBxR<=+q9hoarS@~g_r z=pJpZ!}nAJeL6KAEqi;7ag}Ugfc#E2W_mVSN`$OuYcBm1um%}sH40~5oJ+5p$kMK7 z-{@%0bN1xADE5V+G5zQo!2CTqz&<{u+u*xInR&(TqrMiunO{ayG{M=1UA{h*{g||n z-C@#A{cHZF~&g_FFacT{}Cnhk?*o7MIyUHktEJhRR zG2g4KpC6LG9-Z!f$$D^W`uzt#S#vE5roVcCO<#S9-TG)ULJDq3d`1@|uEO>89trqy zM?Z1i>7PuD2IuX)*@m8jA}TR!3KPKjfji)BVI6ZCNnMRa&UfqBk5~GbhB~gp=F*9( z^?2w?&wpsC;t2nctQxB7(?$;>r5sjDbh8=7L3F{z05+`U6t&dRWM^FBI2`4#tR~kf zh)T9aD)!DO=bjx((#;d@fBnn>?}KaVl%4Bile$g6y)=eNRYuDbJUM6kbt<|>1vPpn zp%2zVrtnKVqj=Vho@kiMTKO8Xw)%S+nKPZ_btX3(6dl2Sj;wopx|{II%>iL;odA7J*iCmAd}AAZH2I2YPlPsViF62SAdM~2wCRc-^1fF^ zt+&5svU!u4jxtmFVOkN=KA6s+7+2cvl!LV1&0_8@|0}Gjn1dSs6m#A%MC-0kq4Mfn zQ`xGGwj`OOMLlQGpeMuD&e=!ZpLMW-(~dC1cO>Yv`Oa)=KG$$Ih@eTkZ0Jhe73>p@ zdCYW|N!058Av!88i(DVbA(d0wOzVpj`k-YyVSj8zehX(Z;6^Jgey`v#yY?V!n(&O~ zca8HGx8|}xws7Csxa~%rDjJA)WIy%Xm(EOaU(JU08q;@y>PYJO5mev!fbD;Fj>cq{ z(dVvfg%9f1b9=)_tSCNC|0bE!Zx8063{`P(YUI{s2QxZr_Ix_)19wKL z*FX)|#~G`E*PKWE6|*@clYIg3*&me|bB9=8FSN{|#d_?0p`+RhD7V)3;2- z{be-8?yj(JPdBrHJEtpV{-6gM9QmO`lNyvYOdB@61IS4873;cw3v1Jsj6`D%8)6ii zsIH+j3F27iqgi|T=KGGIKG$^AE>S{HzPNy9?X_jfT%wV2L^_)DyAqZ0x!lcy8fMe9 zkL(eV9OjeVblQJ;9hG&vN@GkWpzi9oXp`R+`mt#(a%p(Mj%t^n@~jCCJ^>DN?$$eu zQ~q??Ib6+D09JTpDGw#iyUohhouf~q^pMV*<8bcLXw+9;K{sk>(dy%hC^^2Ix>#h; zxJTxwBk2Uw#g;SQLif=Rnb{~~)jU+>j;YW6{2;&RXXDZ9T5CrNcv zYjH``8>+#&-n8V&maP=3&e~9XOzvhw zdWPph*Ka67x7Zt0xXzUBdN0XWTE2|R6-B_B&(i3w*D7XJxE*?zc$Vr-HA1yOKn|^Y{R^|KgEZ;YP$LE1|v(t-}GzDmeKf3*@%D!iSxA1o0Ik_@@gZS00Lx z+J-XV;~fcfIfq54r~@fAi324IyrBMw23))UHTcc_tasG`Uf4ZDGCpJgEq|FqCAnxa zJe}jgcy~YrcVFk#lYn>BRAa>sOPF+`4a*xjkVzTWvCqM8K&ROnTs(3LP(w2!SUSY( zwl^nj!%DC@cO%bD7C@;#((rKnTJrns9I`S(j>PX1;4}M{K(i$*=rLOe+^t5ig{~>w z`k)ll#>|7h6U4ycD?f12ir4r-gbM`ICc_zL1IQA8Yw~%lAD_E_6yLq?4SnKA!1Zf- zgrrwj3cNO|}oVGaz{jVAt%U$8=)6-+7;haNu{ zksCLs5GS8S&?2w|o2XpH0|k;KXX`XDWnE~!#GpIbUwThq;&B~!{wV;fk847U(H3BT zN|tl6S%6esPkh5+Cj96pPu8Zq!mw!*tlO;&8!|26C5Z`e>zD>{S~ZJ^2Q^{QIt}<+ zd^Hay39&!?iN8xN!|v}c0LKj)FqJ9S_DW zWAn)R&(n#XAAoxXWmk*_^TxZCxHjDi}8NyOcLJRxBJqm22aeB zg)NQ@xo@aUsw{Ni`ncV^F$rsYZt_&JDpUX*I?}Om(Gk3CS~DJdZw}=jJqBOY{==>I zQ%Os#Jz*}e;QAeHFzIX&u<)7>Z%T>6imVtSI%5{u->w9085OW(>@lx%sS0`Ncpbcc zei0mTlNA&nsKF9Hg*Jw36=AK`Qj%~Y4`ZiJpe{dwl*CPki=(W_mJBUoJhlqXGy_Cw z83woR6oBi0Lh=2gvp}j-h3FMILmqG=XWa*Yk+oEPjWWS~C!AsSp;_c#|8zK2w}EH% z$e(Bp=t5CDS@O}_i#JWT26Mx`z@Mj1LU%;r%fo~CQQSUItnPqo4?6IiCaQqnJR>~y zLO!V4?uqx?-vFX!FR^r{A>4WI0l2u(fE?N^3SEC2k#F4HrVq~&UZ~L`t3Ip3Td~$a zL`s@GQ?~&B+Gb&1?lF92ojfd<6pmwe+u^PU^GVXt1@Mv<$1N{y<5^eS!F90!@cgTA z`~i31vpxlw>i0VlwGEs9pRS z7^=-96LaQ6j~R|+v!WOL_I4F1e_;X3@8se8OQ#ZYQ3(cYYQ!YsHwe@@32=oXX_`?C zE^sc)|5|uV3jAPGBe)$RLO#ps!Efu6@dIsj zvgeOG1PT2ZmwW~y>Qay|Z%y9FE5h@)_u$igFK}OWApG#tl035}kNkV6A*v@U?sbToKiXv&$R+db|>w+^zvi?u~d; z<9d9hdkqXb=1oivszaxLeIPc#iKqsN!fh3=aox;hoV8&FT(a>7*6rm%kz>icwKa)g zV2Td-@8>A4Tf_%nY&j=KksMsK&W5z4{RKPjacja4S@_1N5~n|sB5F^A;gssRU@jE=lfMbM!Ak)nSlE{TUE$4{Wb8Q!4jfLra+IAI>d10ec)s$OAbt3 zj8~N9gLwnb@S4QyyjPC@;jc|UK-z{y#96`(UUEQ0^OH9I=XM^LOk9AE?ElW2k|{;H zL&eC$l^M7fCj;+Pakx4|o7hgC3|ZY1I7)ZB{dK8VSkT)q@ILz*sJL>jkf%TKCcCph zsa%yLh`qo$HooAKV3TmsuXo$(0geJBjrH?ccU3Z^pO}euDOKs-p$74U)7=Nf?%?~jN`o6Nmw=XcD=$J#5-9j z2Rc`b@fPXW;-0;}_;FVU_FObsaIM&(-d4n$SS4@5GNn1ZO2floDK4?~LOwjQn&b2| zyYWSXw|0foKH<*)rUFN|7_juwG2ZiK90zZkWbd8T1g2KD1Ha=FVD~MG2kj#TZ6OHX zGPsVDv`zz+T{fV72FH&5+Q~a@aFCbz-!yQu&6QUfO9lRciTH!6H|Wcr3g^GL2i!x? z;0uA`WWL;DL6*}*(kWB`-;SEYu<8onxp68?npuMFo?qnAZ5cp$QWdzWT7=tooCD{g zTL4c@0zOC=#bfPGu+6+0`v=;=p^kD~?|BWRi3iueE13jM=H+64!*L#qq)E70CGLw9 zf_}w4LmKczR1@B}TM8D(ajr^-67b4EnuH{Ykn93y zob{38$N!dtulCcxmz?8x!D&YYXIS0Z9t-;-}7~K4`3x5k4 z6I8a|5R9u{wNJnNgcmlX0ro4j+I+<=oCEo-AlOwL)>quEe}lw$(bh^pY_Se7Yna5F z)7b=`YpK9i*nulv^a`At6~SJ~CeSMPn0M{|e%vqe2J;>B@!})$aFN0S!55pyyiv}p zyeq9w@F9@a`lg1QPUe0;(F{>5cD#d;!6SaKY^-DQKt#53`?1Zy1Qq6Q6{ zlLcGmPJqZOjaL_8fZfgv0{+}qe3^4;Y5$b0_1`QBCVbEqXcm;?EUy!I-R*I_{AN4w zRx~Af$A>`Lyo0>2^HadqTP9%nrFHnY@Js!+5C7_Sb!&r58zO;Th_b!i_#*s(myPA) z{}UwI&KCr%X~ss+#spgOUxBCncERjl4&dILM?CMqSpt*MJvg7INxmCg#8Yxbpyfn4 zUZTwe!N}ziK|zu#E-{_T+k1!te%u0ZOG$tqs!oAtYHL7pk0JE!xQhe7DC4fOE}VQ` zk~|6A2`VG~ct^H$;X79q$du>4V8vJ@_#~SPLLDXG$KVRAoY2ARGWX}j>c0aCMi}1| zT>#GSn!-!0+$CrjQX?|o^*{yZ(bSwR0e#(Mi0MH!SlKjSzpx?^G}p+GU%Wwqq_~aX z1ts`l>leEh#W!&9=NUw6*&%FnRsggGTX>;XWguJlPOzPG3MVbh;4Ts{Cc%?OS4G<8Ki-7$SyUZ1M%$&i}+|RVFZ}Arsf;aRUKqxjw82YBanInF08F3A6*MCM&~fHKE5NZm6EMgt#%|0y~T zN2JqMlfZe zo*>fNn0w8#yJE$%5&nXKTb$}!=6uiQEPhW$qNR4=Moxy6zTji90>|uiJO^$s=1%-H zMQ~EFm9yh;Hz%&if;+xJTfo24#)-3e$|=d?3YwzLxhprS3mCmI0$X)o%SFsAL1tMi z=TcLerKy27_xLym!TcG9oN2Q!apovoCPx*%u-B{l!pRc&} zJl|V&4)^_u5Bwp=2b@D43H&onCpnu`i#Z1_jxJQ2JzbD)x|%C>ae-i2qbWDI(1pKs z*jnJ!XvhuQ*}->YS@%is*z$^nn4mA^2uD17I(Kbx7T>>6iYw=1RpCA~ThQ{`U0~Tb zksI3ofZrBU!`GRfWuYGz!e;nQNf^lmGLaw4m^#vEZF}9;fQ6 zkiT$-0e9gA7s11!aa_kd8GiPK_==H)Df}&<$BjR6mT%}jUf|?oC^)tH5l1RYN^s4g zh0~sPoD;s}jpg=+ZjR;J5B!Sg&cjaEKbYD9fIYFhTOXGPb}9gui@7oV=4O1lKl7QZ1~qlxt9LrI)WeR z8eFOL4IB?%96#F8MBrD<M;Hz612`bK;a$_CXtjfxJoLM(oIs5`P6@i@&0@tQ_+-+;;S$ga+v53!h=1vKn zAsDy7mE*fTm@lZm$65Y$j$pj$rwVRwCdc{AIKha4F*nJ6AzyvZWsdLCFPzvPCW7@7 zE^%0|9)G7(DgV#kDvo<%4L?k@fRo)mQ84Zq;VKuD@RzJ@<0wu0#tG3-;J4K!@$C*Q z`~oJPUMjQ`OoA?1+z4YQ>XEj-)Vb} z(_uMT@MF>#?$Q_o0UC=qbvX+Jc-NLI_~*iZJ#i`LAK#O+>B1<#*%|qUyY6zFF_15r z`me&|uN42s5)*!p>O;;(J88jRnPkqk(h~kRonF4CK^Nyoy`>NtYW^98RA&+yX*^7t+SE?1`OE1&$i#&_1&9N;iW(s;|p9d{znL--8qru|9t_L8dX%S<^t?Eau9>=j-#^+BjMzfsVHT; zmo88`3zt}ik5XHxu&eA8sU4ex?t5&&=V~DQJaHWZRU`4(wkbT%#SerJTz10IL@RJA zu;M-4`jPer$x~OK?KIaEX^n|D{iVAR(|q3GK*(Xi8*&?Ox#W=R=R)ydE)QJ26}Wf8 zM#$9icBuVqg5_`SkkbPf!6ea9bbGFvX!FxC@FFmRJl|$a?^WNRza!IOZqF(#6F)?W@q}&lZG*%^S7|i=pkyVWRvs3Y1q0rm#EzxqA{Y=FgJf59c?nh8A=mG@x`mr zc5o)E$=@RH>&wyoX1w6iy}j6PuZ#=2qQTFh+`^1bbQJ^OSJzDaF_t6T42c)(47n|dHUs7`f46?ktq z6~e26Nf76*20AdG*L^SwR(^>hwQjxiUsw#DS?&Y-=Ie=8eAl9L40k~FixND$dw{N* zcaWGnexln}o1sKrCCMvQf!g51^ss{zTg9eN%2q2NxAm{^>$cydVS*&?%kQHW_cx%r zQ!GicpCD?lHKEUF6pEh~(ZvswF|AF+JoxyFd-b@aNSAYjcGrZ{%{PwFwdyZv%8P1< z>24N22>nf;nqP!%pXU*yKwJDf#tDtOXEHMkbiw`E7toHrL?g}}1)=$2Y(AYMm>YMG z*<1CK2ACcJbu|lUbWMP$>%n-pv>%!-Ig+a@Q^>Q+x2c(8CAV#MIW<3U7UciyC#Se2 z`0e$4%uAd?KQC2)jGr&ae9jwT81z9@D!jhhF*MSJnR=9dF7INHJ)q!s5639l?<;8W2_NRE}D%PKt~e2)oLIQAbjG5?JWa9r5dQclG0Z)RA@>TH>zi&v`{HGg{iqL?rNm*@-5@eJGl#U_ zIs@y~W3jw5lc=TN5gtnKAxD;;q5g-K62~)>;P-qZ*t+8q9K5R|a$TfHHZB@N5-WR{ zwR9;7y1Wzv@;1THn*w@fj{-f`7mRlIHxbjIqv-j#hSAWk!LV~abg}vqLZZfrw0>lx zZ}BlY zmtPc<1DRuBN!~dcq3ulcSm|GS;~dCTI7PX|^)%Jw6}_`z0a|^kAVIg{(d^zJIlnUp zH-3CVdBKmta^3(AjN3r=_AErXJQes8G9C_h>!7>!3NqI(8nY5Mpl_HrFSSuXuZbq( z(bYjPvAUnACL}`q@j!vRv?@GSXrs<TZIW$C+XKpW4Jf=EF?@SB5B8MK*dH8eIyA4iOiGBsTJktWh%2vV?4|AYLUC=tFo^B)J5Vs7`WTu8#>5mRAG&`I1l zT><>2tpw}bY%+PLGlstTi{3IjU~R}nc(g7bW{o8NV`Jd_BR zvtPjJP-$LxfCm3$eGENjLbfU$_pmTocC~u~DR`UI+6e@<8cD zKgl&3@Wy{8eY(XBk{_HTu9xeCkA-)^bbkl6yr@q)=RT)9 z<7VJCB32 zvQwci_%FB_exN>jFX{7XiNc;gy1amouV7K=Gvltud$Kf*mtGnjTh3vcaR1VqIG6Py&7Cr2dF z^IIomIqjtP*S{q4br!<^E==Rio&11l?f*(UWV@-^k}+iR@u}dj(;AlSX<@XqBVftp z|ERNsJE>X{MPc`4wD$N$O8P^vH!l%%eK*2S;WSb|Si+38`AKHJz65r6DU|3h!Hk22 zxbg!RxLK!Z+VeNUWXs!dV01dux2A|Bo2Nk0lNxB*-AVfmjad(xDJUF23E{Dkf>n## z@aX7nGIeYQ*wwydeeQXndEhdxReeWz27mBi^h0{VC=1>8s-Zlyj_cu|#%o$3jwipg zL$6>zEf}fCd)Gb-l}1ab*wj);Y*B}S7sp`Xv=n;3C7u-A$iV$O6G6}6HC#Nh29#y8 zvFp%ZYH+lNKH*H^R%f?kRf8?eI{AiFnUs;{aW42~qAHGk5+QuNEgSipdhq=Ij|@+? z2c6XPAigUT&X{D8OHz(_t41DnL_ejuat(OYpY>1pzoK>$56OgE*FfEMCVs8Aq2ZNL zXnZo7JLOtE)W~*8&TE8vcDkY2P8Pub7Kv6CpX&w-C0 z-Yh;TnS!_S+Y{mF{7MKpw*+)`JfX{8fRoxE39Q{3F<<;XHHnLdh<`QQpC9en zHE@;$mVOq-`mz0ylXhUzc2`EBDiz;m=djPz4@~jMMQq;_fy>zcVq%FDT=8Fq3P(=S zfy0$xf5?D*keds9y*l(QnIgKb<_f%!UAQ>ho|J6Mgz@UXY17dZ`r%U+#>6>6=yz-3 zm6s|QQs+x8-zS6k_hhDNTnX}8?BHf?1~6(>G&6r3oT&MYxx@0Jl-cvp|C=r99bPQl zE&T!N1D}x4*K#7QjBBuH>r1lywiXPWaiCv{GjPkh8Z=mZ0e_ns({1PO6E)*N;`K%Y zj_y{2?>BzKhRjZQ^3R9{oRNW@HzdGAR-J4)b_%C%ht8Zt zsRM}RzVq>TQ7aiXS%CEeTHro!GS2rq4!&3BV@%*wIGO5(Y)XQh>Yl~Cp0N$1wl?94 z=%4s}f(Pwt*+v!^#F0P!_rdVx2=@P#Vfn^KXqA}`Otlgxv)5Qtv%F^dWGCT;74JfS z_xX6)Y$hD|G@I$Kn}ZM5*`dXgv$*~BMHFMt3K?t%{F7lv5LSD#KDQ!xK9&8wIzGU; zX0~T{k{%AsBslL^E>#Kc5ZbjmW6Z7$G!<0P&jp{E#PKb{>|!rEPPK`2F}rZHUJ9iA zv%_bqT5O_x713R~mFNfPF^4M7B2gU6yY9aOjD8La{kH_bOq<(em6i;7phEE>sluHH zwgC6}K@>Y7g1Mi+ptJe~*qx9G=N7NT3aMmZod1K4s{+B6O)1Yq>p(5_3z)( zc$*1i>g8RmSFs2CHz{GGJr`e2HRkbHHukG2*C2LIEfdtJLHdPp&~`E#-|2Xf^h>L$ z+9-Rh_N>qGHKRKuot_D1|*q!pV3Dm zTZUORRd_<9n&o>|(qBV0!W(tF!JLujol;1^XYsZ4ePa?FwK_#kyAS$irJQzN!Obd6Ofq#Q4LP@<;@O24h>*rt7;?V%;pH4|s{81dyjYPYlrMRl* z7^pHw;c{&OeYz=~{G<=@&QnvEB(N5a^sa`o(XF)R|D13ucPw9cm$u5g61nZC$O)f2 zcx45TmaW=_;Zw^ygPA24Ht&ggu>4ouqR!7H1A zJ0{^cx+f6N+R0+Vdo7W1%R4-uIu4Zd#Rw-0iADour-o59@yRXNRcH)}{t@U^@KNaP zsR;A8T=7P@Kg))2GAvNb=?j_nhJ}K6>=&l%#ABCuK72|^ zA)XzMz$}--$G+Wk+26n9(oKE5d}5R?4toY!Zf5vs{TJe|a0!Q4&Z4SSC@7s}?^Ur; zFh?z$XfIe!^1`g?+jSjuADeCKU2g@m9;t%i4>Ndt#12!}`@x5Sr7)K!ktsf9=x-f@ z6Q0Gg&(S*YJ*_KJZks^*To4|<8Amj(bz*PkG5Y%x%dcPH&Ac86BFX21p+Gs39B7df zjTJt|trN^dH2XSenwQhx@^>NAV-_!?c_(yQ%tWTm2i%O0qQyxG(W+T2Pgq<|ls|Hh zK9?^-rtBnoZzu;#mqBn~2qWVVN1H3=l9C4u84M^!@5N&=-fa;T6J2J!FFwI4wGVVq z;w0YWTmvbs*US+68;KVy!hpLn@5YS~sJ9r476-PmS+*Q6&OcPYIA7>` z&YL9qZAVx27AW31miPKw4z-fb0^8?H=;Y}Ffz&@G&|WnY4{SFE^>^cVBh~#7e|0mQ zw%|akW;^-!-vB-o3!pXg9+BZ65$v6(j_41s<<-v)LJeuQ)gVfi1{;otE%IxzeaRi_ z@63mP`?Jwsl|kR`@fdVJ9A13GA@rJ@6|U}(;?ANOLXD7JIK9_6-eI5fWsFD*)ysgv{sH_g078l-_n;O*cyf(7Cd}K4(>H31>MnmI)~2h4w~g9~o>)KK zu9<|tL_N?eI|~xldSYzn8YW%ZjCXx(Je0Vd!l$FdV7@gRA}yYg#*koGoIgUh7kT6E zC*P>n*Ap;OqAWDZje|987SMO44xPFp57(F_Fq4;kWVklE@GrLy=h$vQ{Z*=HYdT95 zKbOseOdl0+f@TUoT740o_e#YXsz+d|-4$?*0PuWs1xgiukUMNo^CCktTqGfc{Y}xZ z=KTQU)MX}ODYn8@v$8S%@gBOAf0K!O8;M`8He<-@K{&8vg2>VRE3@^9JF-$Ei8wey zj@iwInBf9eE9CGV+FvHkM=lAD-*|~_evkS4`eK+d%dJ6Qsf?Vv&2~(0@kEd3PFRwn zkC_|Ja?SpfW9|54DBo30F6+z|Z7T18hy40`TB2NdOKgMn-bIXA_bPLEtcO~zM( z;`vxSI(|9Md|QhJtEG4j--L8IyB6jhRwBReg~BSP&eF|!H&m_CMJ6N$2RFXO?bi`1)TX;@wdw`D`Ox z(%y$_OfPdboD70{$JJocy;`tyP9d+Y9BH0L6~MNu;1sC{)AwePxDGwof&pOQ@j7Z^7?9kA_MM+Se3p}Tx?}lz#mxG zHIvzKi_Lz0YK2SLA@GTK5;v~^xMO{uwoBH)sLOenyU!GqE7pN(q7J669S3_JMhFLX zn!r(y4a5`&NME-khHtx04lu?LbMQZCc~k^<<0WB6kR)DqOeE?yTQE(FV6>mO2otZu z6<={YWFvwjt-hpi?_`lr*B@qn5{G_WlFsCh)q;X?deA1fMAY#-6s(-~Vf_nt_?i~2{?k)}fqeZ*lPv@1ZHo~;~33Qs` zW;PExh!ZDOk+j4Hj-19-M862!nr(zLB;PT8?XUS7)1m;re?!&0K+^W}B2g_mgi8z_ zp>lE#Jo{$?6W_chd7B>s+j$L3zFb9hwNkPy>oJ*Pq=I5J3o}^t+O3Yz9_dax#lH^X zOP*6_*JE`5TXpW=kRkfs@et_P9iy5os{53dAKDmdz%=&ksWR^wS>JjZ$3@J-)U&K+ zo$!G^do~jzW}FtD+&%@GR(&K*>|QLa)J4fZy;SFC2i0s2M|@R6Bll$BD@AK8{IehB zUTnlZ@r&?savJ$)sEQ9S>|+*s=;FD+?PPubd!pc<1{WnWz-q>KYSVrcytG!~oG?By zoQl*fc{UiQ0rnZDQQd!*~Hj0;Y zF5t!9vDmNPfoBHPusJH6-mf1bF4bd1>Eh4HfzLvSAFnRlV{-|dTes7mX|5>s=K}dr zZA3Qi9Sauc#^4>^W@>t{8O$OgXn>dmuV;r0K0GImc2!5o;b1A;lkNo@jEqsZgtF`7 z2z(h$dJkU!)7ny`odP-#Wrp{EMu4+;ECfzH1zXd);6?5ZY`eD( zq7Np*>a<%pq2xC`!fS%Un8|Q?dmYvt@_^K1{%HF;0P~t$(W1x~5*kZ!%dvD==0mA~ z6A!~tX2M4ggK$eW7iR8?!rr%2MNuOc&~xTo9R5#-Y84dGF%K=^x5X~-kxs{{^$Uo6 z#uU(-KZbnZ+-Ji0dzc@qVsUZZN2(WZhRv(Q(CA+mwT+SiU;A{%iSL9?#Ywc#BN`Nn z9EInqWZ*x8GxT@DBJ{F+BJ^|@t@VPVl-r6u7*W8Ih^ME0$fApNtUpouS zGG?;($6Iut#T6`s_u#|#r{{j&rp|RyxY?_KylXpxiH0RG?3s?Atv7?n*%Nl@PlKts z$8kv)9~K6@M#Fv{9T+o*EXa8RD*R|n{B)c4TP2YlIbqbgk4HD$c}rq9eGL))P@8Y!-@*VHpCeJYf1FJ9^+n1!(P%0f_6v z4Uh}(r_C3ZlplvJpLEf~{v&(m)1n^jx$xD)mN@#$!j;lE=-b#pJr|Es%R}pFT|x$_ z|0cxy*Xo$kH3@pZR}C}%g# z*>Im^9K6KczApje)Es~Y1;QWY%~WEWCSHB-hs&xGfI2%1D#erOhSb-z?Yj;3-}}sn z6eL93ZAUQO?-M}kE#k<|U(Egqg(rxkM zVlAAwU8Eq7g;AJ%fdg$o;Sy#Y+x*`JlqVA&eekEWHb0B z8;EXU(cC+7w}{rh7sTm@HZ@+TDoVTQ1TZ3ri*Jn+8EXFk$gP3sW!v!H<*(Fddk+-W zTG4h(Jv^~rp6uEyPuDKo4wjE@;j!UZ2)>?#4V*?sO}ie?d|rd&nt#wkDhq@SNt&c^ zCcx>2C1fWzkM8a_q+holf>@J>)IaPYZalIAht`CFYmhv2zw01rujHuzPkC4|W;v9# zG(((>E+#!$LB2^@!0T5Z$gEi@P}|!>i?cUEzGgT*SY`qdC&fh*!>i%)7By%eVehz7 zhZ&vWaPodT+uN20P&7%JgwI=q5n|8DoI6=UPFpnTUfsZ$`L@#B4PS_L`4uXYVofW4 z$YV7v7rM>ghhOL4?+V_KDP}E#pQM2{=Y!_r&Anyt0l=# zTMgdUCEGz|rvTnr+GAl-Ke_m0C)k~MOrOuc36|T>;GZuR+<7Tx7*iA3eerGt{ z@M<-L`^^BOm8ZzfY31ba`kV0jMmoz0I0|RqB;%{CB0RdiT6iK`6?F_|V!8i0SSjoj zuA_2{O};X8$lIdy#wV0czX3ja`-QBPmCERx!s`LP;9a#H{Nn%630|Y{<3KR=xRQYH zQs(0n5<`sr?IG~z0nD{j5eBcQg(crDd3{sAlE(Bx2w0s?gUs`BhHC?Re&(a3*EuM5 zvLz1t&yjcSU!ZQ+W%|N;1P>;JGapu7$JBM~9h=QG2adIb(sgcdf8!(2iQiAs0&kO# zI~U>lEM<@j%>?7aW@wP8CW`WX&gye=MCZVG>ONr}tbO$zSNh!#gdPa7T<+};_n^^*(Ne_TqH4z@~DaUaq zTj`dq%dy`<44>Lp({T&4aQ%W*+S@Zss?X1WxlUJb(}`77ZM2mU$cybmGIisWFDbUGA&?-c^?3Pkumqvu!s zf)7_NBNJ?j?#qK{`KAOc?%W5Wx0~^-SrE@EM;~~ts$@xVxG*A~&G~(Q3GO=fxb^V? zoZKcvG1p^|E+H#=e7%v#7`+ib{jnMg$^)rX)l0HYQy*s^IYs0z$YR@%`_wl~B#hQx zO}#E<0y8`t6UQWgewd7C=h;|LaWlqxz2#8vYJsZX3!rAI0(`%(k51FSQ16>fRPuZr znuke18>B+$<9r<0J5E$1`;Vmlx1e`9VVX`iArWBbmN@Qv>~FK3JM?7N&oF5ZCx%z?0y!sRtT_DrxZ)}`~n{(DX^Pc zh}#-Vxlcbfa`rpOimb*ypjKmDA?!4z3qJcm#~XQ;O>v2|n3mIjb``jiZh{;|q+jA& zN%@O#+^%4QW3I_j&q+bBuF@3G`)PJ{@W4!0@Ldr1s}k*pXRAVnPu`s}w|!?|Ff}+!p$QyFs`%!G$DcObzQsQ+EsL;qTW85_KjHdR;^So|4kQZB|Az5Y#b

4 zUT&A@RHL6T@Om8Y@7`flo)81_Cu-=Weea3UPkV@ba}&0Wk>+*xo`M_RY1|&w9c0_H z1w7jf39L@8fNuR`&?bB-RJoi(b=j^Jd-HQNw|Nb0-8Kc*gb3hB{sQ>3?<28KQie-@ z>NsccH2JM!N=DwUrInLDptDCJoY}SwrW-}$jkjNDS4b>usyhxJQ`O;Wg(D3}nM#Ys zUB~4HA$0jrw!qQAega2c zD~fD9=kPirgRrGqg1qYsAbS-wU}c;>e(s7vhnIipA;lEfyzCt|aY2efyb+ z{6>1Xrh~SQj)#ECrx;!>3w0`+FskFIkfwa4>W`XmRbnhU_85ToZ$-HMc`tn5romLt zOF%2s0a+Vy@Uz#)YZfI?ojpWCgAG}~fdyXbP(U|_2hdZfNjj$Rv43P6`IVQ6xBuzE z0ecrv-1&e}{1XO#8SMY3a|sV|Zc+Oerr^sw;f^0oB1QX%z-QceG?$gfB7Zw(%Zpk7 zjdPfEXg1|9$`|PDenAFKhwx8ne8tX;4$%0thc2(bgX3*k4#_)9y1wx$$hSrdCDV!+ zueGg2pMB3gm~O-FGcle^xEuB-#Gu-Oeda3uxR8|My{EPz< z=DD;s!yA_rq#@_hSe&e=gXX^9v0kkowyieDgSW0>T5C7)N&J9+y)|IY+96Cj;)A|^ z#WcO;8j++2!8zcvFtO!7I#J^ny*uj=*&kOXPYR;wVj0x^6-oU4 zbQmk8Wted`iKZ*&LXqozS{{Cj+34a+W~`qeFf@#%Ejp4SzC$ylG%uiPf6Rd&mQB(} zHbPYCV_MhpiY8T_BWN58)zb`U`PMze<#7v4U-=qh6lOuhu_^S!sa&q;hCNERR^l!r zPjnm(#;5c*ksB5^90$MJREU|; zBS|YgQTZHpVGjYoAYUa?G zOQcd;PBhwgm@?6RklH4}yP>fae9qLsM497|8R0~WHp~N)aldKlS3_JZX^K_Vj~L6E z59EW?F)&&eL3bInkTfYnhzo5%51+^Qskj2{W4fU7L<=rFkVC%KC4lm;wP3!ZkJMjD zC-0^bp`-d+kX_C4@pkBorp)#tyB;n8U6(NOt)PL1j>_U{hvQI@(TmR4G;rF)B`8u~ zkC~5-vpkRK!V`zh1ox1g|1|Y-Pu(|1_XqQFaLoi>wyqtcvhzH6B~HPYi?)MBurJJ? z8Ox;P|0DOOYl_AgyeDth@gUTrQkeSu6?yy38IBa@a{c^Qa9cM;<51@^`e4gPQd4}7 zB!8!VTY#4t2_$D@&P6yD!k1m9;Ek;lvAL@S?mpv4cC zNvP6`j=N-tO;;qbp~;x?E}vXANQC=;0`OjO72SBGiTG9C1G(rGWa!u!P)XOsTRyun zL-r==tr0MO;w;O;a3UC(%F&DgCr zUJ@{VH?wYfGgd1^kpmGf!cSg0yw%T?$+Y`tVdxfR?oHCbCJkS5%I^e*YfQii_h;Z6 zYk9m^n})oGm*64p4mz0vkQ9_+_x5;fJrTq%_+T)VDJ4BG@1Tc=vv4;_C!cgq;E~WM z8g%~^v9d|V?^{mbE{&`7T}>a!Xt{~|&NRWk@*A9e$3ti{Cjo!$4MtslBpx2QO49A~ zA@44M@MGIhOI{r2jXz1(2O5A}!8YPDT?~4pYJ^3%bw$g!pJF=4UMJ6A-T>owGT0sT zj4N|R49w!q@%HH$5Sy<=S@trh7!b@`+ekinWAh4D;N;?{D$E;mP5K`Hg?sV#pBXBs5#LJDkp#B9P*bG z2K0Ggq~AXD{1Ok1^dWiuFojs2n8154;Yj}FQE0rh3~rhn6E<8s4fzfFAmP;nE1I&& z9rs(X%)6efZFof=9@vh>=PjYpB?0X($%)d3=HbBLHR11!dJH>Q4A(xoFfQzTtFEhy z{!>|s=8iV#*fftGJv;+N91pbpZ79sX+DR{rI|n3Z1V8*dNmeiEVX!Ei(Z8#V^5OaD z|F;o?)vfXB%xd_U`wq-!@F3ul9?M&3T|;v>C2+Zo$q@6A1s_gj#D-@lNG@dOPARPD;tg6Af9yyN@b~ zVt|?`*7Fc4=Ny5@-zUQTgCWG!IR&lOD3Zt+c~I4vB=XjNDg3pFVW_eZEG27*+Abxq zyV!!#Z?=GooeK?`{hK-jYO>zOb~=2hfqvsk@G2k2A+M7Qx4Z;&;+#lqI;nu6Pjp%D zUk-}a=aO4ye`tf$Y}_AO2YY4bKHtzJ{BE=kxf23$ ze^D4y_f?xKGdK~QcU00fSvY@*g_Z3x=#ThqYhpGTNSpSxoj4j-s<^u|23t^AgF0_#p zGThWoFmw!u6t7Y|S2ROZu=gZ=W^|J#yKR8LJx;hS>JBlC=!Fkz!?fb!H`?cVndnL{ z1laT283GTV6h3j_b%k9Bxr%}dy*ADWR?djaIa~W7}E+7V-+3+ICg}R&{ z6z)%p!1#7$ylCWtC*|ewTYodj8GD*M{o(`1k|kj)tC{@wp1~jd=~&aP0!b;8nP)$_ zc-ERn6f_n2zW0s^B!qor#_kRHTqO(&f^NW-hBq{JYZlh%Tf&=lVxkM-+p#QV7cyD0 zqCuYz^ig##dZxFb-5)->Z8AlcSWa>t&0)sN#nMOqo{$_N0kxY4fUQ^~GOtp}J%7Nj zvogW*&}F8ay`zjwYlnyVH_&fvE>6_wWP6|YqWgcNG_%GL4~DNIV_aW@u6Yb@jxHmC z8jaZdb+xd-JBAtkAx#$VcYyD6rwgyFp3M7nhF!~W9zz^-xr;Q#=`@l6f;k)6d1(O# z3n!2bY;V2wz)4iqO#}Y%Il-`nZ9_I5C+S|C~qdm$M9?-s z>jb2hT>^CrR(EyT1a_?s9>0#t%grUba$LxU%APKr|f=~*|#_+=AC?bfQySjpqGAgvkBr6;j|&_~elJZw2e)RM|_?7|hWqx1|hDbgkF1(tZtH4eNCYWcEPt4K2_!;IWR zOtQ2Wew|x{)3)g0PxH5AwT&s}TvdVtlIjBGk?YjIiRF4(4b$EY5-?|RC>HaclJ3s$ zI7@Faex4?U+4r4Lb88FLlqHz8)&dtfI)aCKI_;{G#5lX1G<(c?P#WdI^s{_;=y?`w z_q9Rw?#I;MZ=~IKE`jf?!|;rJp`}qXabM^KJQII_7Ra;nBb_)JDBVbcqvy~f zxAly<_AMq`xCw=E$%32y@94{e2{i8a5zH;K;O%6D@b*w1j*mBk&DRQWZLb16^=0+( zKmIWEpBEG#O%g_%vm7kNF{0IzwWzE>oagt$M|g91H#}rg+BFM2PSvnE3UP8~^*xB+{y}*8jwkHDvY2jv;EtMC z55dTxJhFYNE#?Vz;DGBG{>A5VWd6MwIOUo-@AZyk5b^m!5?cegk+~Qpz93G$OL$|t2){E1 zzgN%2n2{3l`A92C<;{YlYYcGuuNj0la*94{|3^2;&!7&w8*!6#0cj}BFDU9kKHBp9U7brE=(9UE#2z>d5$aT%3bw^Y{s`(;WzH>1)U(SZb zYcG)2fNb(7?J4y-G(gt;n2*aA$g*>tBP{dW1*?iLBdTQ4Z2dOcq#XtE{x4x^aTOUo z+D#oEYVn*M=Q8hu!m%`M9KCNW4?*s?={lPP@I9~wOtA<*dlHQh6C>~@Kz~~Z%7~^%jVKUR(Zlz!O5gg?Wr(c z>dA}={8M$f- zd(QL_cb0Fp?qE8Kzxl;{TO%iYu-+FWfAT?^zZ>2yzlKs_VRY4t3#91jB7t*O0~smb zMo%rh3F@cD3m5#G1Pfy_fj40-M#j#9zv>OFe=-!$o9-l+{AQrH*gYEKo5|`Yk~p~T zCM>!30`>;42Kh=O+;~e`l=_fE*0w0Yl`&88@4F0|SQ!cBjmgmFm55O%pVFrtdh}!J zZAP_jI&Y=T47`={5o!~JG^oabD!$MM?~F32h(%al{sgdkGHqNGi!~!6QjoZn%v)Ya z&+om1-X;kox$i3+@|?kp57*=F4avj{uUv!~o(J$tax>ZIR6wpOeWD(>L%40L}(+YhGloYE+z4U^B}6~34VHcmvLCR8fEwGMH|-F%5~t;H`I-se_0CWTVA8Z_3KQ& zLp60>EDpCc!l2)!oY+RDK%>oEwDUOtLR*%D6S1Atme!&Kiyhkhb2g|a-6eYq`ti-L z3fg?XkZxI5+K_lUfiz?^&|x)q_AVfSFSRPb`^;n7c%&Hx+m1lp zYfo5s%^p{MlSKbddZ0h^0{#&wiOwI}1k=@<>B2UGb&DrMPI?p04C#ZWvST!GjS$xw zb-=aeBD#Z{O$t4Zf}7G3Y$|_Aw<_$UuDT6GmI;I$?O0qpq>q8uW+3VDLc`VzWcJ1P zM5|a)G-+unLTo;iM>&CXq%679Hy0xJ9jBjnmk6Jl?t$8kyGdWuAG)~W4B?~)Bj@Bw z`01~~+eJfhBz&A`_mWS<I^__3%AR4)v@XHEIx{?;HwxX37?p3Def)(Twf2pZtKOYfvBk&QvuZ`uf#ZMq+X-*>`sY#Y%Z1 z3B!}@5RtUkbZle3e3G=Um8f9%Y~(T-iaIE_YadN(Z)Ij*wqX*U)gqTm+!ZGNzS1J+YNuQF<5 zIM}jA)t*%{{>z6W*Xh+pP4rBphkv%CoK=3`$XG5|&b%`zVna%k*&mCZ!(E#5(C4ym z=!(7&UB5gQ1?ODg-@g!vwk(&Sue!8opvN>;*nDYArC163I>rTVd-IAd+wFs-C1%hY z?njvim$xzPPqL8miw?AJrz3q{pNV!#Y@+j)=-a>f=ZH>3?4|cF>YxS7H=wA&CT54H zEYkM*hW7nPrA6`SC^z6P8)hies?qS7E*R9L-tXkmypj5r3##E<9-@frA<44Ys}{p0 z3tLgdk}FI@ssakxoxv(i#{8H!Ev(4Reij6ew|I*gu*%!p=y;(8<+BOQPq~L|-tK+W z=$t0qd;AQQ47^O~kFWHM=N10h+4gjG@fqkan1(j5Nkp>2VpQT;6nfOQnbzB4nj1f< z_1UO9=k#dj)~|93xt!jtqp`HNQrP+J}0K3DM;}V)n&F zLiDF@98#D6#&r#AsOM-5%5+Mj)`yb$txnS^2nj^hvcj!)qXyJ+I2SR-^>oX+LbQ#t=3$GP}vv<`kAO@LNy(Qwo*nJf-k!aKEG6Rx_; z`I4s1A-YrDIbPQq*!pfG?7tRHa!rl{=j-$K!}3NxB*FIfl!yXs!-W8aTI-u(&& zQ=*}%W){5iIt^N-A0biq7vQ$aUTpOGG^Qy}f#HxCkv1^|GB?e^O>$JwSF{ih{{APR zk+TK;>m7NWHxgi}aw2rjh=zmFX~Z&mDM#PP5h$(I5WKik&+`|KB(plgNa&_8@*`vy zEP9j2d;Ic|Aa#ua_FHZNP^1Pa=6q0&7iW{u6gN1>z+T{l(@9)W3b{9s29*;{$;B%f zWV1#p$Jfk&J1nJ1Ql&ECy_!a5m0H1q@0s}Lym5T6;yYf|ECo#+%!usfxg=wb6`8SQ z4s8CK5AWSehbH2wFs2}ztUIL%Vy({dr0fszlFytMXk;hAti?I-!K!qqmUo1>M#}L# zf+fi8tSRKbEM@rQtpf=#R3^61G{~VHvtaQF5Abw9=XZR2*c`VYcg()*!J>`u7E;Sfo;lp-prT{x@D5=+eU1?iJl zK#zuvuq~YPZ137c6jE4_CeMRK$N12)cOB_bawdOgJ8^!td1R`@a%d$x2i_@O3-d4g zLIbHyq_sH-_;e}3oJ}gQSa%Fhh_!-bt9bZi)jrs{cpp*l)5J?IxDxlZuEhWGJUE*O zu*AZz0{O$@`1S!Ius$oDyqU-)DeW0#U40%@<#-&Ytm8=B`~##7CBssyBj8?1Cdt{I zO=gee!S;7mb61Bh2eY4egW3reFYA5*%4e&>#$Gk}B#iTM6~E$*J&FZ-OL<`VXfAH9 zJq5N-+6aX$Hb6P=Q1T?u3GB@KjODzAh}5L_ptfltjI&+@`BxpG`?&E@XO0=EW2$Txhl1iY=~S3 z73QylY0?F-+B*m8-a1C)`9UPct%L-{94Ee@6#ia^d2hxoc&EnQc)8iKJjwc#;OT+G z(CAwt%&1N#17&-`)8lhNiBt->9-M=F#%16NtPMX4slY|Us$^xtU7Y?(h=lKYh~qni zpc0r#OgMOo{Y~u-j=3Ptf*#9)+qkPSite74o!klKVspj9Z68- zTOr9te#G#}OAkrFFs65Q_s9Zpm=Ye@N!iAP=}sAw&`uG}b4y}X>5sV^ns2SVYohN<97 z#AF=k^Srq)BNEH)T}bqD_mTE@k%W4N!7Mv}h(78^Nk{`HKR)QJ>9O5R z;=rRjUL++E636$mNipljInaL!y0zDUxZ9y%SLq`xb5I4_g({O3|H+Y>^D|)EjE8{r z6eS}^)QH5_1#ro68MtVP4Ln-N~GXknGCzf}$fO_|pa@_%=We7Ji#d{^P}wdnXGB+?hrE4jhAa_doLX zcSOM8)mz{}-Zrx3;~?0OI3f5`9)o>;^n+jpBRKi^LU^-~<0#))N#-O^B_0ZV!d6(3 zgA3=uoxb8EF?9q~a>VxX3U5%Kg%B=(zQ$%2)cu=C$6vb19zS+{lrnfKcY-oBB* z6J}TAiKU5H{QV1%B1CcChuI{xScdFyUI5R1h{QSG#@HcJ4PPdnII&X8Zt7-ds4VXU zYu~z)AAilU%a%sFn(8#3>g)`GNP9az`YRP$WW>YTf9a%gRU1AV{s%N~s0D?lEZ%!_ zuV5^DH9>=Bq_aE#&U-HleT9#c!dqoz;{8c@(o6ztm*f-NkWUUSD}XKgQ-Pz&UL5#W z0au?X;eA`|1p*hu3kLG8@@zl12IDOa!;afc~0u7&!TY z|J=;M{FMa)Ui@sLc_0?f*uD|&%2`XWLO!_c+Q=LFQek`1JxJhI)XwW1DTk)9rEnm! zoD`pC;DD41tg%;xn^XX~JNqnt+?8sV*ChiMuGYl|#vI9*X*t|-sT6*-uOv$T+T>f~ zO*~}$7As~eLLJXF!17v&;LpKaY}8J`!^4uz^4{yo*mZ8~)^CKV&XeJ)oF5=3vl!s4 zZ`f_wNwDqkPrN{K1u2=vamduo$PG>_doBDWczCTHFJ7bwKTk^}pC8AOH=k3WVrM?i z$2*C8n-BMUuYf8@1P&A0X_ zKQjjmIL#rFs_F16n+iY9&n1_`Qn7~cDjd1p4vVrbApYJ|cuzPJ_toU#ra=L)cD5wl zYN>D{C=M1KO(tusY)HH8IEeeR7)l=VB7X2P&fii%%HQOX6NcH)yUPq-KG7nmH|rDZ zcKic+!n^Rez?VFhb0lFRUeIAf0=a6PN3!LziOj)lSZ1xun}2N|5z7iE>#{B2nsw%6 zUK%$}-fbbgZ~!gql!4M5A-LK{1Zw8ZB0AQo*!E={(X8D^K4qoDJ3UHZ{B8=ou`Cg0 z(gdQu@iLIAPsF@XE85$kr}oeokiTm;z`-Hy(DT#0rnT! zTQAP%?!ybXE^{_Z|V`W|_b``(d+e{eJ5OibcPu$H6~^Ru;gR6ixbRvR{_7c$xz@OBRyGPoKf!j%~y` zXb0IJl?7+4Wd!rQT%lZ<2b>zRk+f$n8M`|5=hH zlSSY&shhT9hen!BLW=|j=B~IQuMb@B=mF-^OPX=jolKy2ON7XuoJcP4}J@O`cKY3ZOoy^IJ zgKgW^!p0p(;S-G`@W85YGM4ZFyqFvhkCpC)Q7<-=YI!Yk_6v8PpqoxYtFxfl#n-^% z-zLaRY=+@;you>qV_>;DmIOye6REmPsIXRuh`kLWsu`<^|Cybzp+}my7-R^htgo>< zZ(t)RYpkV1syS4aQ2B(%s*8Dx73!x zW8&qcRQ47)_%$6iMyJDFx~ZgYSCb&**a@=gVmaCS`813T5+&m&q{*L&NyK&$=H}kt zg0I|qvF}%TJgrwClv8AtvO%h)DA6>RRHYa1h=yi^oGb@?=BQ zWMX$;9lF2K;=Q$83NI?`Rlfl}5s} z4w5MjB_!}B_iVPFfLFgX0{Im!;Ozr-Fkx7TV^>#W?U4j>_9?eN>db>)p|!lqurD~9 zo9BFIe{9w~A4z!0M@YcY93sRXg%y*+vFitGsP1bF_uK1|McF@XMHe^no+&!nc|KW$ zH#DVTzuRB%9XA)ec*8$%!nFghGeYpnQg;|S-I`qQH6!xc@uYw5ZgLZ*!D$+gvG0D_iWt@e z!bU#@auDq%TRRvsEj#Xce)0CjtcrGht{z0rZ(sOyXghH3*2_4kvmgfky|x?H$;FdMH)Uu7 zBH_n?XgE3J1eq;$-L4FE34+e(bjgm4Da6VH!qiEIr0QoeeqDv&()^jQC$tI6 z|5Jk#B8%Yo;xEmSBp+yTpFzaQ5c&;ilF^$oM9SHeC{$?@=L$6le*P06S9`MLrXAVB z{{eo*j$&sEOEPYtN|wgVhnHr%!1agapi}Q@yJTBeA_rCzkqQsuVe3mi{*8bi4|p^m zuCB+!R}d-V`9WWPwcsDG8pps%WcRyQAeR>n^@ob#Rc;R4msv`deX9^`it7Z5T>N^$ z+EKjX>{CE9nsF_-ia)Vw!0*Ww@ZozbDbtB1z3TpOig-J|y*#*iqv0N2M$LRIrzA&4 zY$lWE_j3fUJHK#VnsgAo+m1+WT1g(&yTW0|8L)3d3RIew3DwWUk>vQ{kiJq3dnNvsg83%`!;B&GSKBr!jYj5?o$Y0Jf7 zozh`we`_N&wB1Q=)hr_orAfs2`cmS$G!f3+ya#7SN5fFzNN9H|lPKPtOuk4O;`?u8 z@qWo)VBS$xV0F&`PT#Ns{@%BeSar!@^wk`Aii5OYZ6hp7fsbec2#UTI9T=#2w zJ~~1xxopc+&M#|9@3+UYfQ@0Kti(x8cn^JhBpg++0n}q6mnJE$KvRy#GqXzkX~};r zsN%*N* zzn3N^b)e*uakk<00R5tv%)VUmj0XCu(GTlB(b?uFTKbRIGt$GeXyWh;#>b+B35<_H zUrMCW+Oug${?uGnvE(gV^7uM?QEeERsvlxCF&ER&F}G+ot4@^h+CGy$ zT2#%iZ}Fk~2cju{_mWk=ezv9ALlX^~Mlr+bh5Tzjld1EC!;Ev&Q>Mh_I?8iC_lb+aRo6&q^7HB4r(8I{(QX zI`xi9%ho;Gs9ha5ztUY-_;Y_{z{A#zD5WpQy!@$Y_SX`d}Sy5AZ_-Q5!LxR%DJ{bl=x)oD)AJ~aL^g@#<5O80zA zXa2sgp>T_j{hq#Z{!)vTG}q`QGn$)&_IVv+@J&B-$1;o>+)-l_P8~v(k{6ln&sgL? z+ZmPp7m3;~C(tMtTlNa)sD8WMnT}K)qo)$nk*o~I=h17YN`GI{z2_zD#ZD>Ht7FC!5PN<)|EMbGYCYJ4mbclmTWsqQ*j#|@-k74?%2?_w-cKj@PMvs4?Crn#JX_GvnbP9C6_MD(eZ-Uz+E;5IidhEPDtb++yEE@r7Q z%N9Fip<{dOkoLY&*7}e=eVucTEfmwWf8zd&PE0Cemv6~w3A@wJEFT$UvPF%M#aee7 z+tG+LH>c3jHxHP~c{|w1Yr^*4njOsF-i36bv?RS8p~&h0X*!-%NDK|!smhZW)c6(B z2P=*<-d`TjrYgeD@~=d9)eln(4>zPLo`F<$^|$O+l%m^Ps!>9pTC2Oa1l=Q2$=Lp9 z#I~Om(BsW%>?8T}w52|a=~rKeWGkC!!K{bK*5oQbRyrBEXvD&s&FOTKvL0XLc_NZp zP>zDDR9Z!4HzT+`h3@*K$$$HFGxd?RVwUeY!M|yAg}$Uu*v&^|XzID6C|hPTwc3)2 zPI>NUL>(V7z-Jy>d1hVfvC}gt+0(-hHWXnlzDhzy#2Ec*b!J0)p2OP1x0qb>HhR~+ zg}?pn3HEQ2fG&2DZQTmY&`Ia_bVhjso3Qmb|Iu_5|d7nT# zjD_qi25XVYp~tKXUxqpasxX^|)tE8A8T4^cHOBxRVpL=(yWAuQH4V$QHjhfv+ow2w zUhhRNqi4=Mnpw}h+4!2>%|UF)Y;G^{{sLRx)=OftUb4Z5SbA}>A#yk#fsQmJAeots zZ1$`eY_(5uD&vn@wtkKF0W7-(;hplnzVrSo9NH-~D(v^vY=*_a} zNMwl}`cHA7W!lP_t;+YLX!5Tg$VTNZlPD2KKkUz8L*K@t=$d5K?e;y|8nhAFn)cBe zg?ZF5oTWJjiuisH{ORAs7Uod%4BEY-j>)s|qp!;*(Jps0R43euls#gZFQS>KitoqR zn8hNSQxWXZ=6JMoz=BGS<)Nj$*-Yzv(N^s_k?85^K_;B*#^%QMKV<}tck zc9!lK6tSV%LepvDT-OY09s0bjZXL8IDIY-)oy_ zf8I(aW3Ib>+$?Dny|j)^TltQTmniUU8cXSioO5)w`fl_(HIDtZ577PMNgM~NgE`$l z86B+4CcY2+`OC6rw4Sk;MI&k+wj|waZ#ges4rG3H@?~9?(L_-`^Zs56Tjmi;N3DC2 zEZoX2{HMa+-&%!^z4(eoXF1b9A9u1vl1lL791hVi#!!FFRVZrZ7~lWMMrLYU5t`!n z1RWXOOgC@fq58SP=&U>!s;e(1{I19P&?((B7`i1lMX5<*r+O693YH=O& zVqOKa!nB(F?6|~~MI5Jho?Fp7H@lJO)Y~oK;xpFjxo+#1i(Kz}Fc+fC zE;I605momywDYt#8m*S46IRc-oV%U9?U7{q`)mVxDR{|z-%y2=6qC_%RT*^KXOjKk zEj=m?v{3?^!5TU}K+30Pwcy+pOpe)P+Ue4Xj8fjSmD6L0Hm8v~Z%~W6fNL zzftgT*qM8FRM4if2x_}*4s(3gTB_%MI>V)h zy*Yge?F?&Yt(WGqUf;GdC8?a7o10@!txRLLp0}Wh`<>Y`WeNCk^*J>2%{|6Z-vm|F zG|=>|X>7`H1VfTtF z&`Wt&sBwv)<%f$3TJe_i#t!~q3yYPQ6qhIgrP+zW&Y_)Gwq#s+H(!Tnj3E zq(d{bCZmN9HBirk5$4e>J(f+-W}g4hr%SeC)X4QUM{o=K>h}fyj{R@Zq4K{eDT*D0huUZ<~cmtwWK1MJ(dkALVy@k)CC(qAFDH8-X4QVacAOYHjhsN` zi5m6;cK&puygBWea-ACeVCaTR`RMZBe(J9mgi3B`vp*m9u>-P;(e|+%_J;B_`-Y5c z+Pu|WO3O~_O_IIo0?Pv=-@%|o=J&Ys%sJ%Y|z&18k2AEvjy|A+om z&!flk)$PNIchIU77R|pSMtM@A+*l&0Woi?(cjzQ1dc%<18%g^T*#Wdm$(gz3%dseD z+@-g(Hd4*!m&n7mWEgg_8dViNX8TATGFGfY4$e=Q{*>d?$J`ywNxt23TJBAY_1iA$ zUR2Be;Jgd!na#w;S)A)1J*T|AskGC#oc+Pergr>vRH5TWe=Bq{rCSEr(H(!O{8|Y# zZ0mr!xSnCG`V8h~s~pl@T*(YeO{4OEZ=<}YuGDR{3ev~k%=LRbWO{;IWB0vf%Ewjd zN%2x<_7gEAT@*&&u@CuCN4=3(gbiwOos9li%C#`9j~Ka&Ke*ht8OI6xNM$1qFir?nRTAwX#w3Cb1X# zh3G?(Dx~5r$1m=@jdC6bqjp9XE&V2h@&nu1PxEC_hJqXQ={QPN6QlXNsyykM?w?HV z*V}B3!yL-mt)SlBgZ%9-q09oI(UxM#)2MAAftn|#vQqgKXims4^Z;_1YZYbMy-1FU zO}Dmx%A4SS@hzZTqUkudvX^e&@|B4Q8D}jlHK|$cF(l39k!$96Gjdilkk*MH^mR3) zUq!FcFctZh8!(jqyc|pQ2IbMIoj=%*f>abNbsXIr8ACDB`;h8WVIvy%Uxbt|jgQHFX>G@y5bi>cH^ zD-DpF!gPIAq~>xDQB<5JGK)Gwg;^fQ0^CAfhilL-x`2w^v!Ljg8GZUXn05rlB4x!w zG;Jal@zVr|*Z+XUC0fv;#xUU8?Br8~uFqi-?)v$)LzJezZ<00VOzjw`iKT zq9{i$FI9AjY!7Kb_U~4sSJ^z~eZZtvhY3BD=Cl?i)y?9M_q5SKksPMFZ5nmky^7A5 zZ$n?j*wCTrlUrRg+USiq6J&hVo|zG`f$niAWfi<1GJiH!Fgo=oX}U4MyT3~m8 z?z@^xJ-6SdRx@tXU6J8*=W!=A&YdY!B9Ee3?klOms%bQ7!AZ_Dog&n;qvR>jhQ3Om~zwG_#SiFV$;8wfCPOy@%$5pf;NKT^cb?>$&rbD+*G-$%n@aC@UvQuimvIp(qTXc&`e2;2n(Xy%}s{SAMded$GO-XrHG~1e<7YX3A?-p_A z6=PIz_$2KPl4v!#k%Ij8Z>J0FmbAn^P;2QinuP8}FQh*8tNE8^twtX+PPPQ!D{hGl zB-DO;K7A1+&R#iFNspc#Lxsbl_LiH>&~KrWsP*R(j=i`L1+03?##OTnu`q|Xoxg*c zIahJqbTQ~~>Zc&&xfd2ISH;y=jlkN=cA)(C0(j-!TTn1Zn=m_#$a%(xREg;ldSDhT z=dTrv-}43Uuap4?(E(lykcW?VzX6^u^N8MmBcSEqaV(?Ui`7Tv3Enu4Tg~ReLa8#G zlGGwdQWYW&S2y7eHL`HM@)&3c6Cvu2&+YcF*C*FP9Pln_dAKcFh-B-S!rBTw@JIUw zSbia0@bJV59Dkqzn8ygPUb8O#CAErNF{s2^o9}_EU!!rfwF=n1=Ayv*5$Dyro+pS^ zyo|q(`+ym9CE%~#Ex0A56{t2Xg*Ee^3iSGwVYQMh?AJTnTvBF0guk2tzj7s^P-`9U zu(KZwWj6>y^grP($%g>dw~!ZqXYbGGI4R63b3CXOj_Tc11AK1W}b@Orio{ivQQLJFa$IbXxf*Gti@VEJQTLHc%(#CsQ zb`ab9xWPC7G~rHFQQ~vEnfIgUDK?$B0&W$X5Uq-L_-)%a92&2M8_dLE^0^w^H)KQN z=uv@2moZdX@e#aqSPY+)cM68M7UB`L0A9gPd;Gb~O>k|tC1kT~xI0}tQo48siv@C z0q;TVWMWps=PfhR$1k4n?KY*2gIq6$c*iLS*vqkk=?-zcg}h1dz${_-*?TjVGW%$! zlrRdWa@o?0 z{O`a_^7xlG?m8h4W!@BFp@EA)j2y?kc?2Zg*05XQYfkp>av=G0d%q)rU6Ea<&@1Y~j<5h2!;%v09FVz&&iW$-N0|LGF=Ql>>FC9lRe`~g{byBWwH z{f^zZ_x_r@(-U>kC%2!z1K-|?6G5*IUTQKEXzI$6NHu$6R;^DgbV~8~Vn4Fe?Kse# zJp<}JeFJ#=6M3N;so?SVG=Z7@ZTu~^9~>Ibz$O821;OeUdB>F;aNqd^?7#&Ys10gN zY^cZY(MMi4*=IXgO0k>#6lkwe58QX20IABYO&5b-g9*td!TxFkm|T3Fx5scP2$A&w zXZKykA}RW$Vq686UVOw``OAtFz2cESj;2H;+mdu1`i2)huL2_3Vq{KlAW!Bc#`1>U zy!hLb2wIC0s?VJ z_zT>wJDEV4fA~#d2A);>5{t>T^K$Rp!+1oCc$EJqxG-xQcK>f7P>k0glA(D5Tfqb{ z+9)lUBd!59s||o#L5}eF!*@V%PYSvOT>@;^Mo=X^7qqSZ*DNi(3>uuVf-;`R!9JGJ}p_1qZ2 zgj@$$t*8%s9ra+tMPZOQQlYSh$1{ z{q;tq^}fDskj}8(pj{VEzWN={Y7D^WX&nfzdjcw+tN^+(u6XS_ad@Ry1iFg$3U-vH zfYUeKVVQd+{#hIaHVo_m7Yg3-wwJyaC}vJ016mtEb)uV~PVNeJan&JF{7TGQuncQz z>yzXe>I54s!ESAnNaNOI!SGurQhHsPq`d3^T5r|K^rchD)xO90^}g2v4a3R!yMqw% zk$H^w6xkE+JRzd7-hdn&zlBpkCw>z&m#m#juz)C&ImU@#Y*8O*;r!}*uPQ+QL;568 z%Y!`EwjiTNrjqcaP>|}a3e6k)?QYd*z;-P)!Fufvc=&#Q(|X%saQo&=Vz2)bUlNxm zJtNZO>|GDCu3{fpcUXpu+~@vrl~2SGmhm_uaT2_=s{@Z5Tp7glP~rIWO|zKTRSzRS`eDB}(cydqXjs zY8-ds8E;P0BZ1=N%L0*(S3sC^3YJa?lbOd%IT!C-@YDSWaNB@LMayR1AN@zzf5kjd z@Q8uuv+si_X*F_9{5v-AKZy&J)C5^uVU58VPg zFU`gk@=w7*DOuQRtRX18REt$^D8Q6$s_<&%9sD!o8?fUCVErwdc;Um+L}%R1uE)TT zjGU{-XHE|ZOwQcr48VtNIbC391JHfz&Ujt z*h%LbXkBWIr{+w7=hvr$y(JZ3ck>s#u-_Nj4ogCd06rF4;e`VyJ_5PrpMcENqu4R} zEjX|)7nr|Nf~7yKzy*_L{LKF~9$G#ds4Z#$TlYu_UiAL}?ZZL%Io*xBS56_5*Q_R? zDtWw9-7D|{BYh|gD9pj0B*qGR#wWs|vGvP~*vsJ>erO26_jDf5s8(07o0*UKI(M*g+%fP;w*&alnQ+bA zzdVmmm$Bph41vj`Zy;`4K0sF%fLF#H0!32?AZpo#Ww#Gvy9z^ISHK9iNyj`JE)$q@ zLjtC3=engAbikE*Y54G#2Z&hz2gqieu;9{da6DNRdhoKqJ;5bxt}%s- zD1YaPdhGy(E2?po)eW%j?ix^}bsZ!Pc-WqKC~wQOtPz+@gyDu`h1m7{e4u>G9hB^- z1}n|fi1B4jUSZNfe9T)5YF3oMn-tsQi23NvvLJd{ux-+OI&a! zrIR=HNGtE0tqoDj62nH40pQO+SDwm|2f*C43I9IDgBv8cJZ7~iZoD>`9E|hAchmHs zd$}x*daD388?DA7+h+)R=W4^jr(Wc8fi0BpIK_)-dyG$9Iff653}I>YA-q=83636W z0z$7MfOC02eo$4*`Yrq9J;NYN0OJIfP5p&p#l@f?`A^kL0X2wpD7 z;EV)A%4l%RIc`x{TQXfx)Ghy+t7<60r z5GeJW2G@oU+wFDxhh4e-`mLm&b~Wisp?JG7+;jRPi1tu~A-?fI_{wKIRYMF`g;BsZ zM`P8|Q7q!Vk=NjN7Ju9z0d5-3hj|DC?Eq7_piT$YB)#E1F8vNJkGo*mHY+%0tN?|k zujMIB(}ct_9W=Div?d1&+#NKUns)WsEq^e9ea%ncM6DQ5_&k)S z*ZCb>Y*U7AGf!jJGEHdXG7Pr+^@D@@w?WAq1-QUyCfu~`15YGTnpjuOg{QT+z5F>* zaPx{LR49++-Hq}GGhTQBBr%UHbT9zeD}!g6tOFItOyLC>&>Xi+7EZP~gD*s?5yLY# zz)q|6xH`mDusrk$KDm7uBLRdr-kHFlcPikHm=*D|Q-rUsOvgIh+^gbe31@CPgyHpY zkk_7u7YDe&!}r(Y-Ch?!tdSaAaFT#2_dWwj6-&tMv;_76l-HT2MLweLXv%svbJ>PzqgkyF@QTMy2cmVjPHYQ!L_1!r8HVfSd> zFs~;r7d(k!!MCM4M9Z!QT=850rNqq1BjXg@KU!iKX(LCZoztXEJk*cJH;pFaNze5^Hvdww0p=J84RcLNKG zt{P(9d42fUvM#*!jwkt}^$AG5_zoO6FZLyA01t>7z*SBQ;0z~ySZsU&T)fhQr;jZG zD{egnMD`+x3^#*jF6-dogLZJ0Z7(p5I*LcviIU-yVuZ2MfY04U;Ga58SS%_66|-&$ zTx^Y?inBMCTXRKVa8?Q$b~-|*Q%>*$(S@#FT2L=jhCKJxAycO56K8vSVhm@JNn6dx z&IRgZEL9n&t5gYU#^)0QRbk>W@q#yXnE-gLO5iP+77T?}ctYq@hkwbJVZ#}hLDvHd zxUVw}R6Cdgo}n04j1?jeYJ0&YCl1b;cnZsfn1fe0&g1(k3&_y|m8P}wsyI`r4llj2 z2+ldBPp;nK&2VWcmV^-^%;^uKK>e@lP zxjYXS-*E#w;(y{~W)+!naS#^+S@=ZvD|WKB;Hh*jhl?H*@@jLh%jb0%H*`a1hjcK6~30#Ctv4Z1&!xRdEGi+nnWvR5x4q1fZgv0i=TG@xq);$ znHYIqyakW^vrz|T9Rv&&h&=W9uvIwkEt;%=!a1$kv7r zif4k|XU1@$!DOP^kchQ{)L=rS0WfrOhi^g`2$Y|F!@J%I@J*5VMA^xP9A>TxZa&k8 z{pGWu!tJwQ+Uq@Fhocoa;a`VkAH2g$6h8AQ?cjnGqg^DOi zNKpyd%aX0)cfP;>?;q#fv&@<2c|EW9h;X(N&!z*fs;E+bFtWXUpLUF4G~ss}N*EH% zzHUwC_9fb}!5w~_ZbTQYnY@Fg{!5{Y?)6c@p_vFZi_juDHM*r$7s>KYixl@Sob~WG zsO_H)@*LdERFd+MfAVvbu4>7CEjQwwk-Q^{ciVe+=u@}OYb;>(ZuVry7y7aR@Gi4H z+CQ(0YbvypUaGx=`aT?`N!i`Bze-hfbMO$m`1Ts_8Yq$+*}P78?qDne6@_&BJ9QK~ zVlDHJ+{V?e|IAiyeJu%;-avyA3G(lcVY40>qgCHO(HZ$i(2ZB>?B1wV=&Oq}eXYGg z9C$sMZku|ac9rO%=&*TQ*3$_{s%MRChc0KzqYH&v?(?~(Kru28pTVSD67KFXuCo&uL~`el;Mi(I!Yab0Ll9 z_R(&$cv{k*$jLW&qkpdcutishibJI6c;R>ac4P+~+9QMRq@*MBCF7A`r5CLnet<5} zSuXjd=)<|CUqoK=MV!ae0($e-4D?nZnAyy=WkqZB*wv~R^sHzrT2nie#r#|+l;&EY zx_2Gyjg2C5Fj5t%2G%m?VNX!c;VAapUWYTcR$`0w5;$$68*KM~dZ^5NAG7;z!wfYv zMLxSUIpgyOxR34E>9X!@Nwob$$>Wo+g)<(gunfm?c6zs%v%mI{6^xuhXN8{S_S(y! zpUM$TQj?0_G*4iKM^Dn)jV4qr{{s5jEo94_y3iJ!MIXy=L<_V(3p>rebAg)W!UIbB zG^u%nC~<=M)$ zQk8;Q#%j>#I~$m+lfA?vqX*41`;D%SoWZUvgske$F}7kuAk)m)APFdB?-K-^T8X}%K8%XLR^ryko_-Oj{s79-7) z92RC+!g-a=LLKKn@m`TwHf{e9cD+Cb?J{d*bc2j^{}n#_$VN*Vx7J)}#`{chmIrbR z-zf|qQb|ijnMxYl)oI;q%*xU&xZTu5vRaZ_z?MGft9jgOmzW(1Nc~>|=f#*EjHqezG~k=3iUGzGvTOJI*gg0|hPU zc=`#>`hhtYxSz6NdQI%_3r+Tr%je=6-=c(YZ}$3JG<#U`m`fhgE;0SF1jYJ1Kx3z$ z;Iuw2Ky%d$C6hD?P~pwnEG^lYYAu(MKJwkdeAd*WrAl*9M_Lj>%l;x2VIf!WS5_)| zsmkVeE)X_V@8CA9%|N9ao7qVv$aYN5WDDMJWc}MXVbZ%4RI7DCQq>=a=Gr``i-y{x zmP?<7b*g+m`d%p-eSQ_(mv9tzomjy8sw`0-EhqU zX^Dp2X#9U3&T9J=w(`EY@U4v%pA&b88rqzuMFySpWy^GS(=|;vdT2W{la6C^J7=O9 zvXlAeN}K(ze8HKx-=(b;a%iH?ME3o1HA*V8WBIp_qti?Cxv?XwY5E5#w>#KR(s_Z; zZ~E|z#?GrkA6%7%K0h+pnMWUK#?FH*Y4I1t@IeR`l1WgzEiH;*9*$xQbQ*+I8*-UGZ!KdU`FAYuHqTN~|ko1#kGL?U@s-@hzeO~vV4DXIcrInM}z{Rn2+4}qR=agj5?fXksb>le=tv$?+9CMM> zeLlgnQ7u$iu7<9(kCs@wzTzqm3ekv^d~U>@G&JRQGu>>N2LU&)t+FZ)cDR%&q*Q_dsHALWwr|8}vd_xDk? zVX5@lt1sfOqYKz$Uj=EWinjFOmxpYNoDp61YaFr_N!Ty%QfeIOhYsy5W=mo_;Yfg) zO1p(f)2f?0+r5U>`}8pl-fQZu^pdj<`@$mYe@b-k1fc~xICHP;D4z1-CNo){x7=U9*JV>{ZVt~M5-2hl*x`vB8OHU zWapPfGQVNT?DnUV=uPxxYCn~CH|~&^j?pz@hgb04{ka`%s?~gUO}>K7o88L^w{1th zPscKo*9%d&fg!4%*n$j}>}8$zHAGV~^Vs(l0kr#UImpn1EMnU6@3i`u6n^*YkdepQ-r&JSalzTq&`F+!Cl1x;eG zoX?+|Rm#SGnMMQ8E)q)TY-UUAOSyS-j9J#1K~|oZ18v*7P+@l|d(2iKy~?H3bbTE< zT=jvDh^R&}^W>zDy|1tzd={JZRwH^e=bSKSmKU;P#hhUAI$E6EWU#wRV&)8tAtyP%3`&rnClw{GV~1^-})I&w%?JOMcvc%gW=5O&+(C;Rx$ zm(?za6aLCO$(1H3qT28@raLE&Q~dWFiTbBo4*Rj(&9u&+m5vJej=bdhQS<%7SYBDb_D+dkvIooGeT|LTqkv|DORRE?C8y7O^Sf3|V81(LkU{=L?xxjf zbnMVvW^3dtnhN?+YVlKH^42EK%UeUb`t>t3Vp1-$kG(*HmRw+yoC44gJ~!%++Z2?j zJV@XEcN`TpEI@Yzh7wJ81r*$*hc-O>!v`x^&aT8MOKCQ8Xi}lD2n#=G4|i)6;$tblan=-0Dzav>0Hep1bTG%TlzKkp%_u9yz--p3A{-z`~P?uMjKRYY5J z*GsgbQqh!q-jWe^+Vo%L47M)kDq3gJCA=W8MjKIP3AJkq&=eQ?mTo)TwUl z{WJ%(e0m={_VO+QW=Gl3p5NT(agR{eJ3n-mokfK?1zc*Qtte*aaYSXiQE0+NKFIWe7RxKxfG&A#Mh2%2p&xQ{xR|IL+`4Kx^gH4J+rNAyx*vX3cx?J} z^l9?~lsmJEg$A1;^-g#0Sp5ZdrFs&py7-e?7JH&+wb`tL=lg%{&_OeJ{?w!VFdM6U ziX#CrtbX4P^wy~dDa@M!^^(&0nXyYO3VJOTth}c%AS-s}@pJvu2)2Z_$N@ zN7Q9i9vWBq2Fjd>V*N%R5E#|SZYcF~-*(kVW|fNAI)kf{H4BUAgwLv?E@=~c{r3qs zo3CXjTkdC(s}FGLor#=IeG1z6sfJAmEnt1;6wrvrXHg6AMajI=d4!A`TOZ;jlEG$d zT-pI9(&s(UTRS-Iap%yOJ^d(t{BQKmRYbit53(oWqeQtE$1uE8i8D3GLmXD-=N9Ua zr$Ce1j@d)KR+upvCl&6g>QA)H_9Ghj>PQbI$0OD@6{+skkY2Q&j8^SkfCguGQuO9B zx*KwoNzHZG)Ax7Lna)&HvuPO;7d3KeM~tbJ!W-n?&wFkKi|E@i*<9Vc1+3`hVeWQ& zEZZ<)1D{cwioU*JG{QUvMd}wZJE+TiDz>9Hfu;pEYk`ca9!H4^@s*tLDd?&E&7FD)}}u zdo__OTKa?*898x1b{_0|&KK^#o!^Bx(U<$(AQfg^`o{bXZc&SC4$S696n#Eyv}BWf z4l^{DWh1JeN%Cwf(eF`ONXyoSQ|#KsX!}0ernsJ+DbHjpdQ++FWCNsm(2@o8u4d~O zDWZ!){`>Mtv&7HkB70yF&ocCc^k>!urlYZm33gwj%2*v)ho6(2?l56iwO6hHMBCumzyRq5ltL8fUbO(r$cMi*v!1+Y*ths>M2EN_z5GlS7SW- z@-vQ&fpgh`@`J*CF^;IDU!9rN4P`6FM51l)f{>T4gfm(88tuz?%(g74Vk++zvoPNb zH2qm69bp~FP6p|Va!;LOilg_UcQsyYAb1y>F(L-dKIcrjI^(EP>Q%OGsD>yrdN&I@ z7cCsqX^8&R8gp866-BfB&ap~iDf@e)ou0d#&U|eAxr))IP^I=Ew|ps~o%*9VWqN`w zHjLz4AHSx@HF*CopQF*`V88^aJGpm->lk`qiH?tq=Nc4cx#Ym1!VNl>NZHGfB_H!a zf2!5E%V*kz2NWrLl3u|(pwomVXg~LhcUH&Od$au?-_YYHkFlg=8Fa%v2zhiD3cGAW zNdIAF>3+=`E=%8=ySHf#Gnh7lJKNSF^wJ4nC5~;}y{sJ6Zfq*L+TVxC|EAVGMUU6*Cg?dzV`z^gMAI05Q2p&=$Oss9_H7w%}0f%35ra4mLKEm;)p)V zqGl;+%y+#K{a6$$x&8*Donv;RZ>#c{zFId`|2&nQKgZX~JacKHQ#mc`ID>LePDK40 ztI@kBQQRBVKw2^5G_C50L}%-C(eB@u*dla>YiPDWd0&T6JHtj{$kPmomc=bp>u~^` zPt9h!!#e23%}cl&wi5KHlS$sa$YZBtj!Ra|cu$Rb6+{=lsY^vJy0mfRQr40(hlMTL z#m|v_V0Y@KGn3>o@XPrX&}g|WZ0i09IvrL)S%(Xl-1;x=%zEd{SNHObWi zL@G`Cu-|rD+_udScE!B_R*Jsls>f(xvdx%OkAICdCa4mHT28#@zjEw@MB0Z zF3SFmv%=0}=XZy}rS@E)pkF0e(m4ozE%*kmX9hsAp((Juoh3-F87nS6m~{?AVVB<0qm$9gV_F!h72zkNX=Glr@OVL;})J-AUmos12q z;KS&fcwITc|C}w!k4k@-xOF7dmRkqI_+EbRCSB-k{|fYl1b~rm4B_EvuW@7) za$)TTG$h*DQFY#VXC+HoAuxnfAFM>-N2$9#~ha6~-Eg`ddSYeF;~9C5w-E%4Yg z08TNz2Ab5Dl9j2}5H_C{KmIogt|>RcaX0dCx^F1V9cvGV8aP6c(Kv_$-QdLyQ(#}x zRJfYjSCiG)_2^B!_QN}DY9=68^1frgp2_f9 zkSkekX9CB3O$MJ;qO{t0D+6v`U%<`MprO;!frk6ygbDbEdVd)y9!dIkWx zDhtQ`rwCQ7g229w4WRXp9=Tz22hX_fMy{G)2al(jRvd_I5WJPHC7q5_FqrN{+#il1 ztM+w?y?^F_tMEO33-cp$C2j4ulvV1c#+ z4E>XX*A6FC*A{_L=xi?+Bn& zU5_1KxRLjtwV?4{dAKq|os@U`RwTTTgAd}8K~N^ov?+}x1;r|m3kd`h8dnIydpwAX ziUK^i|2*(DX$3LV!Ru%j5ta7Qp7kN49k=XR+eVjkc7CIX_LnSX)vf@z~;Nl?yL?| zlZ~PN_ZqMYIpH-~y14(zLZW%;H>lp+AOIfcfVb>;-2KgfoIDT=@3%O^gg-npzv~#* znD0+e1R|-tPiW4=1kl%N59{l;fPJ<`@UYWGaPDsf2s8BtU!_)1wagyAdUGGIX#R)y zOqbyA+hws~n7ugsMhdp5HiuoZtsq+?5hv`A#y@3c36qxtYI|;j>DYH)VL2+(hofX%f-K?42^hC1_Z zm`N&dTb~dPEX>B~7f)iTt)W08zZaOF9nSwpXYrZSOT~pF6TxN2CNOlH8uVae1WDrU zIO6LRGIfooxcPY-es*pgbYB(9ul>BUcF0^1cS0A8pNim^AX`v>#2)0y{sWS>gIKNQ zqIjLC2kc*|35*P`fON~rWO3{?ILdJc{u~oXj)&aCVS6lL{G$!ftI7xV{u@tp>f!{+ zDyiW4f!|nji4zRlR)W_TAHtqVBS=uO1$NqW2FON81wr?g!ISPTg4fq{VcGR!Ko$X# zQUl0Rq)TE#8u3k4L#Qz`3~N{)#G&4hR36K~wK~i3$r-ZbM#OP}%J+NNVDJ#I=^h3u zyYJ&Z_hC3Za6f1|!TW4~+`&e+G8I_;#FBT;W#EXy5#(2hEQvK)Kyvf;k@5D*BusZph8SsCSl9MWE?Z%Gsv?E#@Sz$$Szk2z@bj~Wxo-uBSKhx#sEHhl~r*n z*A_ai{*3cJdc&Am8pI+|4SFB^jb*|u;hvH+g7+p_uKIuF;BC{xJhQF>n$QGd6>_$s zy0;d0-3bvqQjmf1=Jv#3n+&wlOaKL?=3@B=8gLXT0!wC$0q2J(!pD`ta7CCad6v2k z6s)(0;k;+>eyKa0?Hq<%pKrq%U0I-Fg*9pTt_lq*yvWG>G2}tn5ZwN&T^v{%2a3G2 zDzpt3y_W11v2X0R!S+C-7{B}$7aFRTndwn4BWLdYjlXrx`f@Hy58 zG8FEAhCvGN&^1=90j)l@xX99l{P*C{b0XWZ42de)h*gpLgsM|Ci9(_FS*@Gj$F^j)%dt7qG&Q3X4?i;}6w+nIckY-?L^jeS|X%6kzG+}8>Hb{4E!QqGG@Nz$0 z?0wM_3~c&@Rqh$X(iADUYq$+0NCQY@z%`(wv=lB4G>5w^>VZ+FklgXNBN+)jz?pwX z9`flBc#J+EmUSKn_w0$l_J>u8`+@(&W$U$puEH>~cf3@;w|eZ? zFAw|*!Fwd+F6ZO) z!EEu9H&bx>#5K6AQ4cPTx-Pcwza=hjjvy}wMC4I8!|kS@@Q)uG#FuV=!jC_XB^!>o z1MgxpczCe^+4tKQmPtCnOr?=Tm|g+&Mgg*3&y~oj4g*)$rQymu#w7puInZ@yJwB-! z4szc2fPa_g0V*)S>*O5ZTKA1OeT5CNM*i=fMXdoVUZ6RyG^!Ie>NaJ}4Gm*UylWS!?oBB6aiaQl$pt6Vld zGhdCwj#&-Yr;UO_D+xGo(VJYos7KDPJz1f9s~#70>*FWdHpC=Hyw(s%4VctXWudZIK5Z(o*XKIl?uVFa)%yUpUZ2&yoRStIU+C)C$20_YUKsJ7t zV1=F>LE^|cz++Spn0N|7_vkZt!Bls6A|xN@Z%V}LI#=Nx#$KS}?sC$<@d9|Z>N4J! z(Sg;@-xLIS=3r^aQ!rxhLxIv}Q(S2g1hyEi#iHRu$%l7|K+8mi4BUH-EkonMjlMsE z2QLe8g^Ma_cO6O^{FGr&p&N{PJBnmIcmoRFl;Tg_TJW9wB|s;;!O*P=?W@RO^me@E zwvwPht{Ftd->sPC_6gVQ8%<9C*ovyECSN z__lS}W!zOzzv(>qIJgsTxZ(m+M(dDV-lZD!umgm7`okp$ok_}yVPt;rLXZ_a5?I8$ ziQB(RfNSzLsL*l(+%eh;LFi4~q+W>U_8t=4+Fy<%Pw+Eo*REj2;5raK%Yf`UxCW3J^(?pkq3Wr0rHCedhs4iHkVn;lxR|-zvjlm7$)xaW8JMyP2N&M>4F|qNpRCMX& zd3toTEizQ~MMG8PM2pQ&QvE?KHtKsEm0#wJJg%m4oa%1+<-jy%KEIA$t_fwmezn}v zJ#pxpJ)a8&yqM`k2ei-`qQ{SAMUJZ`(SJP`sfpDYmTRfS%rD%a0;g~2ud59T&hemD zp}ezb_atWNwt>;?8$4(69Bzo1g;q#cqsl$!*_O$z^hV(x;gcOnthaa^Ds?-I_Qs~5 z&&`LpeXr8!pUOyTJ0YC~7Ur>@N>8SJAd?;&ILh?bcS`WPbF8+=0QmGqvYTWcvWZVd zH=0*-FIy)v{W1RBd%JwP@&u-Ao~-E1_DpuyE|Jp=NI?FxYLN|k$JVcIq$M-1qin5H zDAU@Ob}2t4eQxor?A0e$pTlSF7KSsU2alQDt!Qqsd@QQhn#?qNPq6^6muMc|#p#@~ zLSDYZSxum^=qSxc757}}Y=i6U+aSlZF1$u%$P)P^5q9{l6WSN@m#uF&!=3IO$4a|$ z=!(rPtoF35v~E#8pw?6`8>CU&$;gPw<#YY!g<9Pb3v!vn47L9v&daTyVuQU8_s@{_c%66FOkj-i${Ojb~Bx}3*4JQf#kU3CUkMy4s^OVp9?i4G(4h}Z77Z>>LHo% zthSt#vph*Jz5Gho7=L45!}95X_eC~r+JETJo=R4dQHlH-hOq`eV`hcMh)y;3Gw10$ zsr%h4=;m?8mD}7wz_b9FO73t|maj%(U9(yC??e`CXO4z9wlS(;heD4oAae}ga6!|J zk!Y9!Q#zN59$YxbLfhV;$#eAB*r(Hx!NsSjv8#r8I$2A9<{5E6NISJHIKd3;-*auA z8A$GT7)!L%MsvS@r`m7xQ9|}LlE&u-FWE2?-LKrs{u@07fe#5t-ts1=EZvQ=KIn^n zwH{*uBYT;pYFMZ%`Zc4H z~Ll&*SLJ_5CbUOOyBKoq(P@_MwJYjJmoTkdEIdcJaN8XibGI^ARjX zJBudM4}Kr$pCj|B%OgXPr=JYx#A%9Vsp=u0-ygZY&bOT6>M&|{+JFlV%4TLGEzzf( zPpI^w0lJGn3U54$Ws$05rJn+4vy}ye%E?Bs2$er{>AXu^)5GH|`t%OIH(i4A%;Gtn z&%AFf;04!M{hfxmUtm2K<&OEmYq8hZ0+F}r;;o*g@+L-&79V9{3)DxNWx zo1GGa9v$*UB83;+_8V(C_s?0}&tv=0x07duNpY#@t)3}+y7w`sJ%S(anZAx%SK6@H zx(&i{!QS*`Mv_D;A(7dI5KiG=I5YiONINu|B%Koj} zT=LQO_8}Y4K$9z5BrE3& zB+iy6$z+EnNxYvPx+Pu27UdjcLu7BV-Nw#rx4ARw-Wo@1D@)mrfkUWcvmtZo9WL7G zT}_L>i|CVOZAfTwl;!;%~m{zbQ{c+!ec^%w9%kxjMzBQN8s)_)b zR&2rC9Z$1_7Dd+nJ(}D5@QZL>vJcw5U<9gr{+ipZzlA-{I?sI)pPo$HKcm*+GJjSP0M=~FR!v~wPEfk#nIx<7kS zt1tZ2Udf;TFPN%+5e0DDSlU-xq3g`$=)sgF?1}U_r9b&x=~L4r`5P+Op1aSuiWzxG zaBmw@TAxH?ll9Q%15;Vaxmzq@)m*rkrhw?VhMfc=+-uhHJ8u)yV z4b0lXe(b8}Zk^WSOhePqgt&Wb9 zZAp(Q@A}wqO8DV+B-<8fB6_Qd|3C!V80drcQEM52E9oIefsBnDN zO7!d4SvqQKGwB^3s1brr5Kqax%+$Q z&Hjx{&aQ!z+#Jb}XA3veawOAia^(BsbCHwqB-&uo$35F(&D?Gku$gh@C^Foc4cos9 zH7Fcsl%lZV+8gdmvo$BX(T8;kVmO^? zd)c)U&Z7LACm5)2#!~_Wbnvx+>2@};R}mwaj%p$IeD_3_9oWK(X2l>YUu$$Qtds7! zQcDar$s)fsE682`f==KRxs`8tU+LOj+CBaeyEf|}8uc;{-BeU!=Lcn_p8GPnvhV~n zehgoH+qwwZt4f7G*M8?7B*t@Q4V&2Z?QLjda24t=kkXS!;!*9X%UreJRH5FxVEQIx zGHRdvkjj65O%)nd*xJ=sxn(1CIj=w3Qv0(8@M``7q^uJne03t14Ku#N@@5V}88y>U z^WjQ~l6NL8e?1=Q6(zGDKZnzG%VuzAKW9)=?I!v@H5%)g$~~N! zE986WNOJWJ7iLj|uKP#Q_8+mx)nE={!GPR^{T)vRqV9$5M54 zsX>?ZcN(Fr$tm2$DMwHe@7V4ATg)9G&B*p)A`5l)m3*pr&P~{KlO9^Dj+UkyF?&H2 zby6A6#Z+ao&{a8Xmij?%o^k~0zkMGa$?jr~Y5W|G@DCcd$CzeB4Wf#h*(}jJN+Osa z#3?s)!p{@8v--UeXu~Kyq#WOYt{XZ_7koZL!^Rw^-|zgS7n=0ZD&M*6y<7~vrR>Tq zrj&B?tFN)I4U^Hn-#pWDHUYVNZss1lE7RWrQAkFynzqjOVOC!#iiQ@*FWejLJ#>-& zGZ{t=y!)UMb77NZbXgg-g9ryw|dFcgucQKGLgO}e=c z?cF+;T|V;%75?0ZZaoQ>6lVR0r2Y}i?ezh)c;W=KK(UOCxNX2ax}_~_OY}q^8Xahn zD2f|${~H1zmz#J)UYhPxL3ioZBb|%K=sOQvbbMhqimh47)D$z2%;}@->8w~bV#OVa z)|v>~@Zc~qy~}e%|6M|9;pHea`2oE|>5zVlX zc2!SAZ_`bYXUs#Yta?rO=$|}oC^cu{eiCL8-@yE*chHW-d(fQmCy>f7Li0AvM@#$DEh!97W~hJ6@5R*9p949#3EIsTwEsX-IhfD4(JPSTW#hhj6q*2bgcrB_(Uli}>j*$YV1Q%%%VoXqy`NT6yBx7a94Yc^)V zVPrRY4724~m({V=lJ}Ok*&*e1>>Pie=xGt=XRgX{wh@EZ|5Z7r)?$wAEao%ov`M0#pJ!08&px&| zv<1~AMX(RYTDcL;l#3bG#)S{|a#QMzh5zghG5xW}nAr3Wx+&w0?qusD{U}{T>SMXg z%Jb;&_Hyp9!5*}1tqfaUHJ1x8IDzcE71^7#1+0MI<7^3NMlq3uZripG?fh&noH{;< zV#7F^y~v4m?@*F9$t6$&+Xj-Ud=Z_iPhg4;Uujq1V)pNC1ZS_M&t#J>p;14bgr6s0 zVm9w;xp9{$2h|3Jfr2F?*eR5Utso&Qy7ab>-GQsK+{sMIui&xY8W$ zeS496o+Hnyx*xNrHFj*t_=_y_8>G?vx6zT3BWT>C4#~T9(}fnhU!wyrccT590@;=` z2kDuOyw~%Z2kh9C!&S)aLvdbX*zG@%DesJ;GlQ>kHMcHu+07H_!f<=$DQ!S+EH6sT zUN+OEAHPz1ZXs7uf0BKkYfqo^KDXl0L1@d57A)pHn*&-qQR^ocmRyoTEz{VXf}JItcd)kC9r~Ik?523htRAq2rXH7n-$n5v98y{L}!E7u)|yO zS-`>oN&4(yIzfFj%Iz&hhXY(V``ws^4s}BlCmT@D@afFvRveOh)rU^~`p%8Jaha*j zuYqU36r#h*KT%;)0{Zk>fzB&A2G=bY(i(n-aL$xs6dRjD_XW7q=plRQ=zW(_+JwDy z?A50<3?65@GS)IDeIH7mx}nk){U=A$4eJ5SA>O=7PvnX+F2r%}qh zSZ?Hhr&++o5*jhlLNc-CisYRAKVuK?P_`=xqE3S z)3uq-48NUY9=Bppq3052a>*I}8E3|3CQU-~Hf`jzy&t1hwl5_wi#xfchxKTIx2n+D z`Y4)uxQ{)5ew>{db%{1<3~*B`pKz;3sj!-$9`0)V9M+suM5h(=L#lkH!!kt;rW?Tf zR%-z&667-9>$bp7XbbDS5Pbb%2r0{r0P8J`;IBKfAkud+c^zO3{rg73F(==Hp!t`< zekX0RCEAi02$jgsqiaEKm5bQv+z{yVp8|NeTouaJ9s&QBeG??SUWuofb^wzAEzos$ zqZsZh$E1^iZArJlv!@(8?Z6v5fG>vLvGX!oOJIP_ISCNn8&Unb3g3^m5!=F zrRogU8>vIy&g;Ft_xyXDb+-pg^El{TefBDl>bG9xX8cgxQJsRfrfI{3lU6um(@F7z(BW`?^D{7C&kZ|w48?yOPKm1z zUBPa_8Q5%LE1vhU4iBx51C=w+g4K^T$#r9I*s$^)SQ{4&%KogyTN{RuFWM2}B}*@g zeP%=mTwiFw62AY`e&ilds=p87-thuG=PTH})sgfH9)N^~C7=W>!AJKQLis=<{u~g6 zxv?MdsCZrYP@xbM-HyT9!%qRtTbhEIQE!3%TRFIWP#;EJEEZ&Dc45B`zN!2t475zD z!X<@i6*Y!)fW~QkQjq-)FAV4TCp8VSaH=Y-AaB69|Go;QHyDD2p+E6^+KHdooB-Jl z*2Hv_fRycd4qm=d0O!-cV7uQ#2$yM1(v|A4Q!oaedWz%@kN!l(UBIii^@-am-@2Ne z-VB6MaUh^4{Q8hr!^uvqF~nt#BN;jv0&I(w;FaDc&}h{rxV^j$zGF; z`+6BGM!3Nqk1=q?lU97|lQsD?(Hts?9N|}>00$jXT(VqW0NX)DSZI<0f;OH88tkbc z*gXbZn$!WBro06~jjk1rA)O$wUkKw^8NMI<9X}12N|r>skOh5t_-j8l&=8DT(%awuo*+51BLfUo-l(}~1AM2D~ncU-w( z1bNUNiT{mQjl(|5lBccN__2MOcuB_p_#yAAXk7hOyCf@a#-2mfU+KxE33O7flStKlWZ0*!xX`_Sc8u6$Sf1ubLcuUpE%o z%DWJM-X|zuk^oi}Wa9hrW-!oJKIX(+K7>Ghk-k=TRe(nHnd~qxE9w%M@-}=S{};m*eR$_ z_=nZ_>%VBTEnK(yKd|8G2y(cn3tuqk#yJKac<`4s90RZ8=qD5m8p^v=1TCnDj}t+U zQFc(nxeC8feh7kBy1*@89HG~{Qv#qJ!aL;j;3VTh>>E%9awD~g+U_WT3-Uorn9ToC zbl&k)et!VZ-g_3=TSi9QbMAAlN|Lk?4bf6b8kB|x5)ld6Ni-=%N$xrKIcTC%X-QwS z6P1vLsNeJZ-yhex&wZZrdOq*-e#4bhG%vo2m+-ktHK=4VgWNx+N}5;k z$<%eTaHO3%SZL9Qg+-0Htw9`yX^rCM1{ret;d0=mHlFO;Ho^;EJ{5{PTR>y~W?{`k zTk@`@8XvyZDU?|f0Tyjm$6*g<2!)ruiEj8AaIIk!xU98-gX!T=>4dnD|Lu%>5GOEk zIz56HOaB1>PQL|pM~&f1&+FjZR7vt!c^M87XpkE5ddwDAa-7E~HfYu5X40J?Pi7*F zaF|Z+|&JHw$!f=~V^#|8GEaB}|fx?aC%-rj{jaY)c-Ui$Q7z$(OwEX_9{ ztd|gY+wQ}H>n3D<_)Pp!;}$TnPX!~XP1vH#jX3O{M%1(IKpTCE?b#CEr;$lumlweu zffK-kFTVgQT#RkG=k;g%&18~62-)RPfz=K=k?PHFvC)7(vAd`Ows9_)Ngn1UNZwi(qTwb@ES}qAHSZA& z17+)ioj6~VaVM|sRX^4=oGSD=^9)>%bq2M|SKt=QRID-S2>!Is6SEg&@U(50c>Nxi zz{}z!oH6t?GtknGD{_+P#zkGHW`j!+U-0_>dseu6Phcg-T*wJvnNeJwTnqsP zedqA!O%ve63s%spSRICNdfvIIJYv@9B77O(3VhAJ0>>?Cpe0rhP9H4*rOG^Le!>bi z9+*L_c1|Ik{crH69TQ>6a|OaTHWpwIh(NV{RO}(<|4M6xD%wM>5!lP&p}P^4G_`r4R}`M z5+Haut=6l|x^bQ=pF9yh+*M`@o$-oEUSHYt@b8+6kJaFc^A_?W#aJ9}}uw~6W z@Ssl*Piv%jLtMW5%tU>VI8_I3n=S@Rz1{=u3>EU*rVVRsyyqUEXwLB$e?jB2qu3*D zP#8Kv8?JvPjh71M2yb4h;%)d9%9|ic!)5vaqR=P8_J8BxNW=w@UMCN2c5uv~n1xW` zF6VVEt;1=CiTH4>9#-Ri=Q&CWz%)P)F1@4%O^$EGV=E8wW-b7ATYY+{#DQuU5p=ki@>HHCHTVk5uV$^`CR3V z$nn~@xM-XSzT@^5-&$G3yZqgQxLK~l?^m6|zujEmIy=s*_PYQld%A!x3wuGu-=0v#+z{$Wv*6?uDX7S~8JWGNP^@>kP~rV+?EB^^m~&@3Xj2HpX9{Dn^zRCw z|LiaD)+q$XP#Z3Nt3&)khw%6DdZcf0G1!#04To4ez~Pw5pzz;TFlkO3@6LUDSlaan z*LOX~eo3prW;YE|L{h-xC2iO%*ONEUJ%=nvEO869Fo9=ZOvON34P44d$EU7*#pMeu zz?HRWyq<$%#7njX4;>A>~7?}He5 zN$_z%UU*>Eef+0i2j-UjytcwU4<{VW0&BHPd7IyT0wS?l_(6Ln*pO{Qf;#UAE%f*> z8O(B1hlP!heBG@? z)~P7LylWdkjrj$TnA8cbHv*sL8sWM6yMznb-`5r%dIx-l`tYsx z7B}BOGhFbq1>7a^xcEjKuh&uwPXEa9eNu>ce)xftf6W3`OSH+}ZQA4q$BK2xo5O(l z+GPDWYq-79A73#ZkFPeI=jlqxKurfRGS*X%Pb};Ma<}FY0es3^lqOquxV-_WpOb>u z7D&P-A1CwE!Y

7+2&DEgJmb>g4-8 zuV6jWzIqLLJkynY%U+MQ#K!Q={xXcEl?fG|$HiBk0sA5u`1+U!v}#YsPeQL@?lDSE z&T}FYRx9CqDU*q$!a8zKEt2e>r3#NMnGA~#uY|R+%i+qGYS1_n3ttqd;k4@iaKn5Z z@-})Bd~72R5A;Yw*Ipfbxo`;M70RSj-I?s$VFx}J08&3?76Ii8$$#9rHG#xuJl)8X zta$50!e=Z3QG4TsuI_;(^rithIdwOFd)N=ud{ls+N3z^k2#r98nG8KaNEfzd3WcL`SvSGrrmt9k~{<2H*83< z#Tp`f*CQMd?FXxmPasd1=#%?<)yba0tGKaz5ePe#0$Lyc0!2pc*kq|Blv$yx&#QkZ)4N73-Mr^5nR$T8$W%jM=Evv$;0M_ z#Ln`eFf4mF&_Q}IyvG>!jF^EsrE#F;yEwUc-jKXFG9Fsw{S|g@w zI^K55m|O2LykVOyv_N9SImErLW0E(oE#=Ox0NDO1QxMLBq6z!W--&j*tL=RqCjLFKA&D06=` zoZp@U&PIF(-Ty*i=wBzeb3_YPeKI1awr?T5TH-|EV-gMbG*_RSz0MW=;!9X&YV`)sm)(3NN^T*ES_ zR|&`WZUcUvLab1@$Q?(kz*1!?C>{skxvpXEexZfeSm}`GKFe`ni6)%ec2TJKlG_8> zc)?3NDRS$E32DflLRLDPlD($NU~ebKJ1_TwbmwHaV8Tzp?;aHHT6hY753RrtmJR?P zm2|9qK$&YH>yyg$NARBs&Q$!(c4oOpF7q|}C*3}kpuIP0*wwS9p>t8g?1HD-^#{LE zy_@Ij=+-{n+&y?K>yVI3%W{yucT5uJs6N8}COA zCG_U#|idta?ztrzy-yIbboPG0?vZf=9t8zD+ z$4)}8!~5AA^DeXLEnU>U{RUl^UCC&jnN45+Fk<_2GpOL1Hgiz8Ou9G*&47?fO zU)ps{)ZkYrY9^DJ(8ysr$GR51yD%SRlr5%j=geR~C>65mO9GJc#aIL{T^FgghO%ca zC(?S(_mNcohb?cHMUMWpY`(!KYLhU99ulM@*9$U&NRPve`kid%g6w(Zv?UZJw&$`> z%M4Lh=1nGZS}-!Vzlg?uUuCP;Z=l;a*Gd1ZJVr4%oJOd;L^dwjbf3-^I`Nk-KkU~7 zcK4ocQSdP*6l>tY_q3N~)2KXKn(xPqth$L(uZtmhZqD6T8%D?0++@Yih9R|p$DF)* zDs^30ib_*AiZDCG*e~s;dYbo9m0TSC_sNgm_|J)^epaU|>$zE7OeU+xo=5Q$53_;! z@2Gx%B(2SJLus?ZsK*IE>Nut$`1iD&Kj-dcX7zMs^ub;UU0pt&E-Cm!MZ^2iV8IY8 z%XN}Z_~)Y=22Y94RyS7hcrbnVx}SY><1hPaU@djd(?Tw)7HIF{Y@~jfb5N?~p{guJ z+H)qJ|K+$8lQ=(+Hv8w%_Rizv5yw_dSTvWhk+h=jHW;O(8_=z5v32K^ z`E?`P*q4VUBCGSCnf~jqL^Id9P;d9YjPw=l`YvY`QJ%Uyy8Y4^ZcyM;+xeBW>9Hk$ z+3+)3@v5f&6>q$p_QhV=(~P4{q?~R{fRk8uSty2`yZr`Vc1mGXdnZr z?)GDU+&hOtHkOKZ1)gU1`$nT&r{nC1X+52~R!7hnvPU%LJ%>34m)HJ<_x!t8JSlyUJUk}RA^Y6pZ@pK+a zE@tTt>mgRXKAY~G`+;A*>I8b@|46i8&rfnI)|!TXSxY@${Y9-i7{QzCB{b3h3EJx& z%X~+Yba>5KbRj;B{Q~P)rSY0n{I`T?oXIrw<*qr>4*bS$zBUOB`X!??f+n^<&I?&( ztD%j$w)C>(YZPTwz?!Db70tTQ#&%t~OY=7O(@*)m{OE;Atc_0r(rePJFEcpLkL<6Y zPNm<`nGhaz)0!lTa&4u}M*-6iM$qAO{rVdfNpv_?5p`yUqQzsC=<@4H%+>%sI{ocJ zMrP?_lz8$AC8~F5zR@^ly!r~{ugGy=AFY|FN!^SfOp?}L5J$;B)zMy&ioiiLkJXn! zsIA9~o?DxbB=xkJ+$;gJHSRkTr3k4?t1n8sXiUew5i#u>w;;*7wGP*=Cw4N(p>G=+Z}d+THI&IeI86qvb9G0}p7-hNoMKw~?FId8 zy$>1hC}(Sm@~H5J7+qdsi7@p?>kn>cM?&_av#a!Imm-6z)~nFVr=PHz>%%E~%8q^) z>!jZcUsJmlU)I$lKop^D$G7ahiblS!WIYdVqpqDM$S_?Oec6D~l`AGl`KMA#bPU3jzk(2`T z{9i27+g{E}DHqUub0;cXDvS1b3{xhx9`Q#5Y3P4>{IUh7=+d`G(ajlgw0PTeDtwrX z;*KGDM*9;hsNY8wNExH5%(2G>`b@a^CuYtzRl&lFb0{Lqfd4=vTojy{gHERk=J0`tXeCat@|~kZL-p z`W8t~5=XjlJ~OU-5eoD^Ot*%}EiS&Gd)XqkTQdP^x0zC^69(I_E}<#gX0aR6M$qM@A|~c*5;f9YLuWs)LQ09v zD9f&g`JN($CRc0_oN%wF{5@;Y{%2imYtk=PG{mv(1E*okl{$9j=S`wxcQ&DZqYCu& zt^@sbH6L+b5@T(06m7%K%&y7Zte{5$EjQAt-*s^fYCBacv|ezMxqh9yXDu_M9q|Vc zh~uNjHy-mBN@Vl1ryQgub*t&z951B)HjUr<)00}tUS;09g;P131kRhb25*&1VmF57 zGP#^U*G(*o>Cr#Mv<#g`4Q^gY*LV&ZanMDTvHhab;56hvqFbMQH-&MY^MmGI-Hh-P zOyw6SGN#gL%=frCqD_(KY53DO)cv^&J$CIm`!nDsV=^<9`QdnlTGijCySIPj8`dvE zcl0^7DEu%p*Ylc)H_WH`+${E@E1$++-N$^&7O#KkKEQ8WyP6(-ze<$RQb{K0#iEX| zd35SWE9#MGgzi3Ffn+Q5m?(`8Z0F|)R`a4H^WxiG<|1E#W=?YyFt0m^U-~_?Mmm5R zS{y_7b>C9=L^FmwkwayhkFnI2q8E9mY1UtDfyE&q(|4+!fAo4W7oTlp_Zsw*gPO=^tecm^yi@A_3mszxC8y~xjC}#&!o$?l%YozLZ-t=P9S-^ z8#O&TOm}i^xpy|FMXM|dnQ;-Lbfwom8j_kr_x7q&(^>k(U-2&IqVNVriW|kCi3XVIW?&G!FVKdTb&-%okh=m-jDoy zjS<#QV&68Em>gMvD8V|D3ug0Uu zSu8tYfj>KId@I`-l7pNsJP5(8iHUPC3Bt`>>L^mn?}|R;-|YQrh&v#}S5qIhqC*)i6$` z>PXG)6w4l~XE&=IWXZyX=>5wUl* z;6&r4i6@?JyW@mZe# zsZ$Kalx{%V2Qz8t2S0Q+!Aju!;|714M+6#Mum@QsA>@;Dj;yK7W8aRvp#8sX>Bk-W zm>KUkvf^n1CcDy$WVwgYFQF;SmY?qE0=LUn8$8D*l-E(6f;M)4LMnasSy}M9`y;n$ zk*BdieCobDk?Bhr6m70~N|AO2`)HgoQ~+i#Y_k{d5jVzw1pcJ(V8=CGVDU%Lmkc^)Fo#vhPbNj&9iwV|aJ z6;$z|xxnMjC1g#vixdv)^3^m7Sq-mO^mcF}%}xc(+W*`*510rEzn@~n?p>mGe{_-9 zQictSPeHuU?KE1}mu;zTW}!nLqjPE>y&Ka@A0-{9pZM<>9ga_w#um)fts6v*Ixm>^ zw_Et{@1!uFUhZd_a;n+e>++b`4=&xP6 zXwi=-%KN26H|}ge?JulRMQkFisTR`7i8IlrIrovC-%*AMn1eQiKS3oq7-^dBN0-cB zGRn5!7`yTf><;Z=#`MZY{&vBL$a&Rq_LF%H`)gShwXk(y26lg9j_r-0bsb}(-+6QC z_I;caxw3$5ndw!3Vpk@AN%u|0R`Wj8s*R!QKhlso=jwi%dz6YX@$`DxOlE538XE3< zo&J^H!ya6)1)aRnN#!dEot7X+|4p`Km*s`hB++>^@!SDwjKyg{S1~h^f0wDLE?|sR z%#p*WBhwLfi*3Gii}vikEAsiZ45?=h(rpi>F}0cV7=d>Pl3SF8{#(8ZO=`|#$A%2( z_&a@~>~a=WONj|if2*Z+yPmKE*S4|OQYNFNpAbrGb{2>|dX7%}8Pj>jvMArCg|Sy! z&g}fQfv(m&$E@1zLzgdBVD#?pXWssCqFbt8Fgq$4QTy%jC}VgBU8b9YypC{9;e%t0 zPIxFY?NBg(ZIBBBHT&6>%ioF&V#`qY*BktbnW6N&>=zWBua1b=T6+F~4imXOfo-T# z6L{YepirtsEs|d`lglU3SLi*P_&0(ktd6G{Qu7&2<#xVI{71CBW)BV6Jp*;C=m@%& zeid!Jpd!dpx`|x0W}{P~c_>-=8XB2$99{2EW;(Br6QmaRGjSokC~n$gs`2(NJI(F@ zwJ9p2So;j;#T;S!Vt!KPA`j-rWp}#!LLREWV}YXm66riSCsbm2n2rRcv!aqZx(V;0 zeeJW^N4=PShOw-l3d?AI)9_7u^Fs~N-lqVu#o@y zhdC>{m4WPYPqH8L{26J^jT)sDhtff>=)vSx%2`_I@`!KzR$xZAM%FMtwcfF>9?e1h zCVKSo;&k*Qp@0$ZculuYzQcTrx<%ifVVU)3Oa+Z6ud)#mdB{TcI(nWFN?k_}vF#5% zk=X25y7yA4$i1VRZC1I2a*x-cQRX$BBV$KHlux6SS?B1c+4;=Uc?=CXY9P9qH^6Ru ztWh7nUlxh4=wW)EWYUexqu3PrQugA`g$VZ6GD>pV^!8E}G{w#gCGsHqedG&jF+D+D z;|?-H{(F4^Hc9-Et*7lMt{j61hcJ+7&D)r)#bXYn(`8ZZI zwxt>wXO0t?D^8}q8SB~hx-zy~0BC*7AY1u2h9A;#g)SM8L*uTeF(>U+(b5UstWMQJ zet%sKn-|ZczXEn3@ymuZvsEM#sq*TZggNN(1AS(FX{Jbxe}l?MYtgKo9FVv}`~ZBMPW~$Y&Mw0>pk;EKrR5l9;9qiBI?$19l&Cr9M zoAZGFqji)C0%idlRI|W9ST%rys-z zUV0P$pTGFwHV0^7D~;!v%_ci0*9+&09AOVx0z!Zj*>!N4P!u{1|0`|~2B_8CtWzp5{e+fJjwIKX^f*Iy7FUDqkeORnsCR|h&Dm-V;`I@#t_>Str zv}|p%#SP)q4g2wZoeG@mQ3pn6Kf{V{Rp8&gTcB*dDQW3*aQ|o)1Jf&Xp{vSt;uGaT zu9yYGJC!d$`N(hgbO#|Qig1UZh@o>EBgK6EuXTvyD_W)1hZaFC3;|a9= zGJ#}+C5(M+0WsZD>Vz@Z`$5U|9azK5j9gG&>K=FF6o`6%o@)f#llYV-oKV7b*nZs<9#ZNB zsJ|At$W>!?X9%V}YzFrx?g4`};w0q@cTV5@JUG*{2^g=}AokOPgaaP(Fk_1wxREPI z9`_rNomMA+w^}}^G(A?=%B>^&u%Par_f#U!jw5j2YJ90M8jF_M@}S6>Jn8g+3guev z0RwW-%sEeZrYRPD8xY}OuWV4aP#1o;oeA}C+LN+J4{*|ytssv3yH1IpSi{Yl6r8hg zU%EsaTA67;1J(W5$x(?MU;Z77|GX;nTWy8EYMK)xvwjf0|2Q}sP%6~-X~HpCF2rTF z1F5|+p2!cL#ug{yg&~U5;p{yO)I79bcxe;Y7~5WrZBsSLg_}OaLPwQ|A6P_M$2;L} z$sm#;-vCyudMC8zX7J*IlXywG5*#vZ!~PS_fU(|Ax1f`$+}!K{Z$+Inxowi@_S9q{ zvE&+T3%2!vwDgbQd|(qeRXYvpT}lSYe;!~z6YjcZlCa`o8=Px6hUYi9gT)+^bb@We zc3o$&*THdQ{QM+bRlFYard$Wb+g5>dWdJ8H`GS2q4&b&}IpIec9oR{Pz_C~otW`_o zKC2e=dT1+D_E6$xRVH@%O^DrTngVkhnc%d|XToRK7qyX6FD{y>>ApNhkdm0_luICHpUB=@y~*8T@A7- zU@lC`m4v*RZQ$wLTJS<%9{)7ACGI^lK#n{GN*2w4&b0*^=YD~8mtO$gbPWQIW?+LS z&DeH@E*xtM0iSX&f0)9DTz`*3&T1>f#AS^E9AwcRLPQ{R^umzv20* zgg-OHSsQlXM9Q{ByxMe4qJ>zvkbrY~SrPnA4+pTxV5RG^tyw);fRN$Dic^%nbGh_R6|$-kIL zjt%^DOWdss5+41;n$EsZVOAIDo~}-$4Q612ANh5PkJ5#+XS-;yJ8(E}PqsYqCDyy#$mQ%l;mQDx(I{VnPt`eqpPw{H=qDWY!hj%FOa?lD&UWr_Fv=m*Q%t^8%x?tn zUej4@w$~GTv?-EZ`I^G8=NG_{UN>B{stO#-zk@YnbHL@hCqN^=3S4}B9GHYy<0iW< z-fHKAK=o4+b{sPWTOF2TuWMSMWY%qb=kjk5Gq4{#H&-RCPyB@wjD2C&eIpngQj1M3 z_ktU5O<_=$F5I=@CZ4wcGT6UbhqTQ(g8ROx!0$1Q;EC*Jq4Jn48F<@(w`mxWB@&;( z27ODQk%j}#mB>riWh7;02l!ezoha+8DZf22revps2@ z3rWd2Z67eU( zvX$0E;rtqMVr>@&^|fHKfi~$ix+8opIT2cA5T2avJ>H@$f0DX-0w}oNjTZ&YB%7bh zlckp@L4kamyR2p^E_Qf^ZztIhzk?6)g$E_T|M(JYIduqFUcCmCqWM zF}$(0^MEerob9klz`cK>uxzIxcu}z)2Ps7hjrQ!}`37-4^NupSV0tQUlN!Ua@!9xi zj3hCJwy>OIsACtq5lMI+99{Jj|1H!fgP#IPR9gus-kyMuc7)-g+#xXax(|G}NDW>M z`U<{2OTec$kKp?*X84BTWD+JU2e(oWf(p}v;JDi;$Kz-~HSVm*A=`Fv;*J9RWFrpU zubabFZ23!b-lr2L)k)? z3OAwqxCtbChd7vMJdx}5@_GA|Y{?e2M!Yz`83Y9u@!ahnVBf0wWcDl%_Yf+?lj?L~ z@C-Y0&%>LDpAI8`RGkPZB>1moG-O?Ccy>n(!N#Naf#F3xDAPNM7uqYpamU_Z<0;<4 zxaW1e(|wbPXObE*&HW`56%~Vu*wy5TKSRc^Ef5}87U4M_8%SM>Kd~C*eqXBNNXgbG zK=`%PU8jCM7}~ss&_R2=Ht9EZ*|(Own3D>ocWX#1$OZ43`Hy#TneDZYkWcS zE|w3opam6bzOYjNo9}wVIr_Y1e6${DmMH5n}AxYl29{@|tcjD(qFX8HbRhZGL zhqH^{;|bFDaJPdJ-0|=ihMC?lCrkk*qyKncQwwp(JwLKcs7~ZNGM#%xkOkcd`j2<@w{fsK=OCC2?@8F0hNXq;HESulF-c~gJoVE+kT?%%*+pX z!-?lO|Dyy98Cs2fbGY8TsyTMV0fcu`mfPch79KPh!Jb>6;(!8Y68)nVJ1@5(lecWe zpAKtb*MviE9&^v**pS&Me2`|li6NxO8 z9Ld-^1@P^A z16I2)M=~OE@FLYHvVN!&kLgT;9z1WjL~S zdA#~Pcn&Wett!Bc6zJhL-?~y~nXpwIxhxX~BODI)t5Ovf$U}g>c^wb7xQLBgS-`4k@_FlcwGUi?mNgU-oFb!3sQo?&kKRp%tts|@;bg(Zb2^3-;I~M2ji$o z`mlF0B)_dx2zRCiu2`i-j+glf7v%$Bc6%+p@bwUQ?;-)sUgqKNzH+2D<{W78wt&-I zf?->F9Eh5+4(9!{g|FXLfK9JPcuT!Zh=LjCkl*qTfAr#d8R>T9uCNa)H8bSeR7u#Z za}YSYMUq?I>ZC1J2k$r}g9oJi;q6~$u>Y76bRMw*d1`y0(QO{wozRJ|uE$v6c|8d2 zABR;}?|zf@jL=@HB;E;G=^jG&lGQTC%%<-g7^=N_Q08 zxHuQe&nyQ8L6V@;b{1Bi70-Q_SAoBT5ImQcB9GLfK=Cb4_$F#Tbe?R-J>yq{lVdTM zRjCB$U7q8j<*$V?J3VRHlAFvz??k5RS2n#{Dxf995b1JrZ3o*1{**RjW`$TiUtLax z`Wv65iDqx8)%HRZ-&w|Zl&xnE`(&_wt&do0^MTKs`G#L}W{gdIeIB|__{&d^U%~jy zSVuq3kEZi)K4EwVt}}(ElIU0G38uzTq29=)jJkJyq?6Cfp{N)Oba~+#6nOLm?FrvW z7dIe*om2t4x?7j7zab;=Y09CSqd0%s^TSN@1zqYk@|^ZkhCLkdl^j|y!YDSo)?evP zL3f_T({!g>&^IfIJ@7o1e_)#%+NfB@C`}koeXNeLc?-q({uU?EH?FO+{(CT*v}Y!I ztyoC|CEd~3#Cf#Re3)pJEM`ihBB)y@*LZE2KnuDIP{;UP$nx|9mY%F)UAI=Ei0d*) zbJj|#;rE=0h&e({&9BpG%OdEO#vt-xWeLmr$*`fVQ<-z=A4ChjGicNrp`%U(VBv0A zl;n95ooKK@w}KMrCRffy_FulJx=zT>FG!-@w-2IeuBo(RwI$>HppBlbI>My7`?C8a zdJ+7T%N}|iiGBlurp4uuTr(}&5zXC3^^Izrd)Zm0Up0d`GkJ`EYZ zg&rAhLCm;R`e}=A_J=mnlY1{xBfd`k!2Oq~ z5F+6(@D0X@slV)_4adLE^%e65Zc>E%>-y)eAtXGX)u&;ko7wod1%0(R$?8unLD_i+X~ORe>a4jOg;*V;j?WK@K+`b< zT6WX>pqM^*rcysLo8yxHCev*?4(Lt32UW1CrH@noi3&&^X==PfKL%Z-3l1lsIGIzl z(q}&^cg{w{B9wkWW6a}b9^JmahH0rCMRRXTu!nlLQdX%D={ZdlFawIrlBweaOR7Ck z)xBa?v`YeMoy--z)_%%lxfG*KXZIuVjERDPXeIj5bQAjU+m!CQsLVdT)5Iiw5)-hR zj-re=lWA#WBHE(WL+<;&K#xA$MlKx|lpQ|8M6A0_Maw?X$k0P{cI|mqH&YfZIVXcI znw(&D7yO`;K@s}QE2jyh6E*#*WQ)V^vzFTJD532l`|{m&rupU-{twqAw10S%UG|}x z>2J-`sB0`9`Iow)VW|Puu%0^{dv~qqO2>G%xBfNXCG`qgjB}~}PBB(O zWQ^{%7tyaR)6v{7iTv~|9((&k27RYcM}G~Zz~Ff$=%+2FIk%FK#+)+7xXz93i)>(? zT5dtNKVM=K+xD>@nYN5vW+|08@taM2e+`YUD?#UTw@|ka1+>W|2n{?^uAks7NzIyW zbN#IV_G{}rbcXZcbnQ7Qx^Xp&%I@1sG3N)Pzvq$VhvLxOB40GI=O2>rTSt|azNPXh z9?0!XIkVj}53S}6vA)ulEO~PP9oc`4KKe9H;I;K7|FQA~_UE-^dOfEBS){F@x_Vcc zn(!TTf0YUy`GC*>w?1?VpQbXpw2n1kWIyQF7lh`Zhc%AqNQ5SR)}(_*nIuRtp!i|CfwLPn>?n*Q3d5AC|>!6^k6(UX&un8RL2X?L;$HQu5{!R{-} z<<9ls@5ouIplij9%2IdU)su zz1`r-&Q%1}L$)00-XVufP`KbsT%SFQz8kzcaJ*d0` zQ0W>Cx`XXzm}Nq?)Ule)YF9!=fvuv29lPe!9^DQM}{ z`OE^TWpvJ~C`L>2Bx8Gy(1KYy^jgO{I<@yXOW%8<@@OE~t;_L1q0OijZDgyg=c3{5 zn?-#VEZTHLz5e>UyNrGL9%>iujnX3P>H4X`=w_4^)nDaUFY_z`ZLe}cix;I)xj#e9 zliu^_#}7$CnBr3Euv&`(swb^ z2a}PZRuD??+dvl`RG>CVf0=;g9Ffa)Sn;Q58tiO_Y$DZI-%e@T^q-pGY|MV-GOq^h z*>sW?-alH^-AMbqh@_LIGTEYQjd zFDjjX75O;n(C95*f_$fQJ{2F3d@xr+Yu^YQyMIr;|rS2+_6^{4Z%XZA9AkDt@- z_e+q69_MB2*B3Ou)TezioNq9sgM?jr#*9yLM~hF{(@9|&^igmlGv$Rl)jYbLHoqfu zgRU&36# zxTA$pN|U9V76h{oucpw5p(;_<8x_{hJ&mr)NurNF9%K)#&0zOPl_OQlQ?ApIJ(6^cZlSuX4>P{L zZ;8kWr*OQ6&uQ+g z!DKr9=VY{YhC5xus3DthGrDQ#Wxo5B1mx%6PIRW`GI9el$Z#|r-BozT++M*k>of?} z$qg`G-1Ew4*J--hI)^!4T}!K!_acX?ZW ziYA>tz}}qyfgOGFpUBsQ6&3C6rFSCh*})6r(Jy-)da&UO6L9k}v%fqCec@Lk&5>^Y z+7IdilMPeRQ7dI~@>?^*DHR3pR!Xtar&{;{eQW5py${ie@dxRy6FJBPbLD2$TWI#w z<8(Q9R-{70j7m24h-QbJVi$a)RLsbQHW_|lPUPvZhmIbh<9>Le_uu}aJf}|fS=4hh zn)`%qdZs1DT+wZGWX7VCfNxgO4+hz3l&j9QrZ`3(OwcIT4GNV2{SZnxGT3 zZhwfh`CI~(Zd(jXqt{}H&JP&FKX(VEIxNLq^!la@7-_W^8y?-q=_?09#ho@fKJ_S$ z-}HkX*lUUIJr1Pj?+4)(oD|GO@^edej_Kt?XsF$n{9w_}ir%Wo_e3agsp&|J`HWZI8iGiT? ze&EA1H|GSbgg-J>d_L5SP8(e*8T&N?S4+~VVxEu|-!>qL+QVd-BV52^@l@2VTTegz zXeX2Qoflp{Z$}f_7}Lg}`7)On&JjL>sMJ>^|UN&?XB`c3KYrB%}PJ;yM} zXe9nVZUE+Y&XK>aS(qCu58+j(QP1lWIqn<>ef^Ul-Ow83HowH~WMA^E?-FE&+Q2fM zaC~pR3R8D1qq|LqgZajP#QumDd>Q5;sDdw+%1Cj=a5(&E3V9{| zLc;frg#~rh)aP|9c~#jClNT1i`M1B}RF)%|vpG*VX5myE^yO??dp& zX)1jtJc%nphtRI;o}hngC(13k2s!FO@au30j#U_dhZE0`&C%iDyt;^v=*XmUu3yRS z3!%hs#!4)@H4Q6D8+hMrm1J&-3DnD{K+oGWJlJ{w&)VsUrhi+HN4jD$^FoI7s=7Kh zF4_p!&03|l+B@N!VWs4XFiUDPuU2^TY!kTk<>HHL&Y)9Zgi@R+v)^k4VYdg-SKqJU z`Rh73@LMCcJy7B~^CF0RW)7d^Ct{`1M94GyNw;=7Q3d{dXkl&(*8+}UvfTp6iGM)T z>`Zx%;CwRE;sLm}sDi7<1@v9`0d-~ZIJiX@w!1EZBirUe5#*Gd-yDyPG&Fu zs&^%o4OvoKvu&{KL^SC+`WI8?mmzsA0u^o-v3p%hZg`9nE&2Ey!aqkrZ%+VC*1rwR z`|;9_i`%K(tWmHq-5q?NoB{VT7fdns#vk+2sos@tl;wy>)5=gt?>d6(?~3tpz!LO- zDG%2+m%-5LUo>>tShzm?7O3Ut5m&baXlNFYZ*PimMX!R)uU(8ktNzkxIZb@txSCp> z$b*pwJn%BHz|!`5)R^^^bc!y~IG*X>Kju5}pK+YLiYVgQx7lENssmb2*gLb zlJpd9S@CWQ_JQ1gG_Gu}$UZiK%!FR@Ek*}Uz4=5mDz@W{mKfoxlVhc|-S@Hf>PB=M zGC|tV{hSVL9s)im6roLS4xSqA06L2lz~Q(p=wH~5@)JMMrACExiOO`kp+^XViq1-( zD{N<7MqQ^nZPW=UCy|ry9)MY!AMkQ|$jjd%mBuvSm9M43x7jmjTzCjRc5A?WlWWQ1 zgv0pJeH*b2@&KpTpP+tb3s#<)jeVxE`1zAJCale-a_Nnv+_M_?&Gd%>XR}F)Nje^{ zpNGFpT(FlE!_J~);+|-NK^I>WzrAfz%chsmt7nB?&-OvvZV43VdQ!oYD4JQehp7LV zjLx+iNQBK!RIylv|2;^ehS$ny&u;_RvMdMeUqq1mV<+OlkB=a5;Bu+Q_qkGGMj^IM zEQ94Cwmj!|6I`mBLG$|8qS~3QuxoK8nW6BIj7;L_gny&)T!TJTcJ8O^Jk+6`&oGzq z99f&R0q`nHlV3A_2v^or(Z^w5$Rph_l(TE0WiBgVYg3Ex)14+#-Ccz;r95)uawuF{ zej4{h-K2hXA-Lf1d2;5zKe$PKHhNuYpg;fH0|#5wWCwFoK&+&Scvp!INna*>?JTBQ ztD=NzSw(EFxfQx@E5emkXNiPK!oaZyV3@}zV%nmGTON1dsPra!d5wT~IHcm@!)D+q zafjvzW!V4snJ~sU8N!wLM0bugJZa$>dlQ54QD!RbkAZ zc(PEXiI!QbLU{gL^ip00&mRtub$mRIavIa5d~6>a>IcY@D-EFOxe;16tzykP6;br- zAP!jMfZsa~fyPERP*}sCyEg2lk!MO-n>CBzsArDk{jXebnL0_RntK6r8q+~b?lUAW zd`uS{cnHT17*LN__lbYkZcOvJOAqq2U}wDzWXhHhdgiGRgXdqyJ<=rM%3m8$ZD=B$ zQlA0bYkywK(oKIy>;SNGmbx2@pz0u{j&}X-<%?_@xBl{#AHpSrR(F8H@%0 zT%e<9J>(xufYD#Z%9`IRixMZS#!06J$sE-#61PnWG;yytjV#TEvMO;WXIa+q%t4v$Xc;d-1)T)(fN^Pg-5lLZ-&@-dmbF3zAA?|mZk=3St#3}pEH zsU4<2Nhbd;=zvFzIqvYb1l=RAVAV)PnYyDZe!K2WSFK8gWfxE4-@WO0`)54lyje{Y z-##Q+Uk2d*0dd#<0555 zy*xtNdMBM2OU^)Bm;mH_wvy@-NsPtKHDLJP1DdOS5N>SbInJu9(NQ{BHhFNrWTi(A zomgT-eIK2K>~UJ~zJ=ocIjQJkVv3Cx8Muc0qo#JfxMQ?XmN?5t`ZwVY$!%*SHTHAB zYSt)96$QfRKX2((bysM7T zuX`=IKW02JSIm@VAO9{r$b?gJZ$4zomJp9GyGi-4B`|7O1W-jy(QUgUaA=n)iMKfb zCa2axNS%~6o=kw-pQc0j^ejB%tU>i%ZlHFKfLeHD;oJ1vR8GB*Mkt(s0y$s2cuGYw z;CmH5ZW@3w;fFCK{5{(EJ)&_f`{AMaAavT$j-%C5g>x;E;8|4xtqb`seG*@XHNG2! zud3J4K$}EtnC=NXE)<~1eU!{te-MsrjUrli9+O0y540_2v2;$-GHPLak&b*2Ng9-8 zWcmd?JlQ=9PMy4i+xz*St7}4Ai*b1Lp$|#~{Lwk#64=hag?5rR^iQ4_h~FEb_Tn_@ z52a*?GL(`e{baf}J{2CPvQ(u=9Umy2k(M7Yq=nCI&_*^3?+T9c?~WD9Tk7CJd1rJu zB~R2wd6R&*mq?P5JhSZCHt3&aLGF1cgU*R<7++XUu7ITZRw!hxkm8h z^&8=uIzzflkN_u1vb11o6L?QAm)a`aXEm0U(XX`z(%&wH;K{~Al6bq2o=k?R(MqU$ zU^aZ(vkUxR=!*K|rr_1TYw`4}71ShXICKmghS@=iFugqn{T;@@t4vAM&%pHbJ5*(6HVyKB4F}g70_}c{i+sm{XPzcx^0{-T zn7?#(Qv`VNGctdgh7sfT7w9Lv2D#oxXw|ham>6IP*A7czQujD)T4Mxq&-EbK>jXV; zgyONyE+oES9?86XNa$l%j9xZ2qEOJ5*^D_W9MZUi?lz7C-^dbbea8vS9Ty{B_rRlH z9-@)mNqV}l1REzAKv2D+XosN=GVAj&TYII?PN@sBUbbb#nx^L-7W=R&$G z`3hpN6Fm4h2K{UQqj%I94CH^G*y-|6Xj@98CHsgDUHm-JCT zWfL4T?*|FEGncmNmE!ZdkvM1PU|FZ}b)nY?H#)Z9E^RG)kL_FLW3X=xTweSFpM<=Y z?kyQFjZspC&kvml_k1lbo_h*S?VsSGX-V+Fzs#zR@w@Y~Uc9lq)9Ie+r>`^VuIhw}42WBKlow@|43l)mcmrfPc$sudeUiSsq|idB-$aVeuxV~&nCTZ)a}m9W-V z8H>!WQLhntGPST0dbpq)&88~J*z|PJJz<2ue&2^b!AnWZ*umt+(g?8eN`XWlHBri0 z4f0gG0irx2p!o!vy(jrD)^ijJBJ-_)v^?@OH@WD(> zer%0qE+g@5&_Gf&UK`H&?!{rP6=+)CO1-s&sJDM6bh(#G_~}u;UUdrX)vx2b7bT#F z+liigB{+Iz)5OYYKpVaA@je;5wQK^sFjSK1qzpr{(@OMY^AOrPd?9>XEydLz4M2Tu z9o2W!p)ES!DHoGR=I>M>ZGIMDaq1Q!k!HASs~cWkv;#cv_s|V%^Q1fNkJDYRQpift zTxc6`ifuE z0?hFYMP-wva47u(IXGk(z8j@Q^$bR0;qeKWZjeQuOji}YZ2T?VK2IJ0O-Q15l(RAD zu^sfi*$2BCv&nTaKimKIJCcq3kn)!a zg%GFrg2u=0pg!Uz?Cx;I8bJc#&L!e0ow39%o3GI;1%R0oC9mr~K-S+dxVv=>T)FB+ zt)>;?(`X-}%g?O@?T*4V&J$$OZx&OBRfae#Gg`W)N)IfySc7;)3zw~M6*89n5H5S$ z3D3RUA!=PXom%sfShBNW&hapaU*jU3e|#cN?z=#B47Ei5?l;M^%y& zafs_MfxXQ>_+rOx`p%H0sW)cg{WDMUOR3tnJJ)J{#o;8%7uV)QuPCynsPV&6n#sWC@z4=o{{bHg+rFh3`mV zJ5z&`897*OFbO?tgMoWjFYGw6w#D{vs^j5rt!} zG_vVul)<2H7(P9mito>N)9wdx(7*jB-Qwntk5i1{-hFrCyrB_p&Qc-s?SR&XuEc}n z16qWCAchUo(ZovvYoz=PU>B0|FMZOcN%v?{?Jg{I42S8Hi+G-U1TLOlio$C;mbzim-)JI&+9len#NLNshFI|J%aWj&3M%_19v%;L6f=%%rlw|imB1$ zs_i-6Ymy{AUlfKOCfQ_@$1?nMLm4-+wLEEZAF=+=m5O-}m{Ch8uIBHak5)EH+xDc< z_{byl_rUw)ZQW^9nz;!Unrk56=EP4M#t7Ff@gz3L`|j>-rb>5yN}~qXkTH!mBvm*b zy=Jya!{#>AF3h4Ho65<9>o(FKfo?F&(HpLQuccER0`cVR4PY9i0`-SDblzW#JXcxT z;KD%Hac#WI{eh}|CxM-KLONrOHf@I&kll2N%#--R6?an{)7MD1yuON=pHy(+iFmGdTRN00eEF+;&TlZQJl8DsCR%N z(KPM?XKMr4bG{6V_3uEd;Z>=ycOz82Pp5xz3yGR_n^cc?#jgg-h)!?-C<_9q|J#1j z*%B@c?g`fen|v|G@YZ%6|YGRg_a_TV+$#0wx%u{uEEjRhoEhn1ql;;=$D3JqNmpuBYVCa zuU^tblSEzc{PCJ*eY%0YcnY7@IpWkqL*c529jR#^id%VS$X8aCmY*`lfUg^IW^*eV z@{7ipL;m>vR5a9A3?a9V%YjAydR+9np9ZK|$|Aq#(09`lz-rqmD0lRSm6PL1d65pj zo^e_tY;L5o7el~wYdp5?+0Or;$3Xe$IGAzi96fPEQ+DOh3Jh+)NlIRvCIe;mG{}0V zQ1etfNvGkZ6tTUO3>OVht{4y zPg?=L%Xr$^u{R?WQ{>*8wNriVHOI(SF1snD*l`DYn{-HKBXq z1@8j;@4AGtXP02dEIsh_t)}nqwJbrT*hD=hEB?qYqs{>BMutZ~2j7_5A z%~@DBnZV}l&fqXy4cZ&@@KT5o4lA06+Xo**4Ye5j(t3%#nqP`}CCRvR-3GK|osqpe z1lh2QcydZO99_PTEOe7eRY(tJX%*6yQ+~snL!rd}el9t*KLYM$#lq|}doZM-442=S zM;9NjgOi~MzEVsJ?W5`|9TWtZ!C!B8GWzZEe$eh;qhH^Y@$uMRfeh&CqefRCZ8F z9^JEjK=642$!>cB_1j`VBTk(TRXBiIMk56Pt*$TXNQ$ZBL@|?Yrk@&+z9(Sb8 zAQ{;@xQnl?jr2Z@{+sx0h-N7M=AD<1PXCmyekX*p^+lM`+$#N)Hv=s*`{{i}Q@A&~ z58AZ@G2Oe7=1-e}skJ5`+{>?r4ufTLH9|3Sl`E!SJj%W?RmU`ic4|xK2T&(Kk{yhVM0<<-JesC#GO~`)H_q(2X0WHbQabRCL_e zfR1Nh(~BSSr0%cuWdZvN;C68xbw0uKh0-3-d;>m5@K_OBLRXP575gyiLJ%sQ90md7 z)>9+p3f!x-3>Dsd!+5i+!uxx2Az5n?*(l1#f7`dg<2{Ld?JEtWHm6`f!EYK>*GreQ zu&~mzPdev1z(Y%Sm@{=BxwJ$JQc5<1?$!i4UYJ@dz&Q# z7F6TlPb_fDweV}MFLdZP5Yy9LF!@Y7mb;Ze-#H5cUyRTwb(eJQhMzQedoZ;&E}&Iz z|4BYx(*=>}H*{ty&`TZ1q=h+J&~sV>Ykzmb$mgx_p~W3G6^)cG){Ox9j&f3HcMo5E zI03t?w@SraEqagU&zw^-p_cE7-jFSVxa}bLqBrA5Oz_y{mus7&2X=Lux*RRF&Rnra_ocAAbEY2gVte?>7#iL~B zcl8n%y99F7){Y+WD-}+czLXw!I{~+jXTsm<)i`s7H7u=<651EZK}_IM?N8* z<2&&0X%Q$)uz~pYvrwW_0!|;QA?Dx{hlb1t*cAU!+A(A-^qqD>6PdOw%4rJLO?XOU zk86qUtd4}r;JMiK<0?_{ZIpT?92Rc3*T!huR(g8SUy}2{7%YrVgWZRHFl1II*p{Wh zlb%qrqN#+&Y0M(0wG}am->Zv;xS&!{0l8G#Bo$V);21AAvOb-q?BRRloD0FPm;R8? z>c_$J&|=X2rT~`5(y*%e1kU5@8XsKu;h&grx`SAVw)SWexybP#XQP7^75`D&paHUL z35}%ZKq{oY7${5pYXuL(W{^sE0q8h1L+7pO)Nr4d@YjeRq-^#`IBT9rPh{y~=@*9l zs<*{23zou@oJ$~TRu;|aafQo#p0DzV5C>i}Bl-&Cah3g0(R+6%G<1zeW_%qvE_Z-u zYx(1a$R@C{y9rC$EoJ9St>N#zMEZGL0j=J(fp@Pxg=d`#s8(tVI$;;d*#HzqUP`3G zv)!O#B!TNy&q(@$1E@IYFc@A@C6a9+WV_7@Y|n|ssPqF=Q!b2*8)*a2{$<0mU%Sb% z@F!HqXaYWKR>gNkd_HNcH zz)suq#O-YXxEzav&MtkSfpRk5)wCva-d-jy&#k2TvCx4A;Lk zF!)Uk9o%q*u6Y@X4PM1G?T?XYNy%a9+Hx;)d+0B6HLqA2f6yN9i&l~Ac74K0d)m-y z_-eX!eK%Z??t`-?<+Q8k4_&=?7~SJ1gSx6nx>s2OMdx*ABrY!+zlAWg4O{ zbBt(1odewT>Lmvfs<8E7IdpOQP#k@q>>KqI9ri4M4YA*a_lE2PrR@oLX-6AA*&atN zGyc-K4`0HKY7f+Yqb&1iupqLwi}aa`1@CR_AzS4$Xu@?{JeX=oSJn*1+gph7CU3m#TGtZLK6=I0JRn2>dHAE^yLeRSQV^ z8pKm>+-qrn^*PLpm@8Z<+K+qQzoRjO^6=WL1OVH3e0}>EiZh#u`_?gdELoNB@r{P> z>{Yt^$v5IRDjltYBQQ;UkZ8KB2K$%g@SNLmuto7XW>{?jz4Roy{`v~x?f!Z^Bv6(e zGLogu=?2h_$rB*u=4&|}1}A7cx!!PEmw-TO)4nBg?xSg7>e z@B(0`b(4y%RxnqP2QwsR@jx}7t((EKd$LYqU)Cv!*E=grh@6f#3yW~9!x*%-*$Fmn zfx_4{Ta0&pDlC>JqN;Y2bkDOm)c#{DbxVxJCPgcJ9&wfInwl(K{4I^PY)-}dn<#v- zE+n1_>tNs=H&9a@ihCD7!O|6@(M0rB_+i@=O1Auy>b=_mF8y(s4>2fY^GW!E{baqb z6R9q1BBw2v@_zbk^7rmrc&`(W8H%^b>PO*Z+vX9ZwzC!Tyh^c`XEwcZGsS>EVF)cv zw0Cn3#x`xCDqDLg{~@`5+y>0Q>LJ(^)2HN@u+-!(d$4IE{T=m;4C>!U%)jtC`?nRs z{&5pY^yaHDA@eD^R^&ig%Sd7O)NJ^tQviFCUlae4IT{$g4iWOu`2sY%sfq$?m(j(y>sXn!HJGXec|)=BQUY^CTvmHgm)f`K`;2B)b3$1j4xEd)nlsRWJjv> z#P>jW^5;9X6bt&FBbxIoQvP&5FNssJUn*m#L ztWcP{8s;2cB%QD;8*>-#1l{|Y@Sl-3p7T<`kro*kemM_Enhb#T4##Qq*+weg_L8Pr zY=Qe-I;2`DOE|=3253(-!I4U{pg{3IX^ifCbdSuCcF8ZI2D-b+Fx7pqJy}88yWR=H zd|wF{LlL@#Wk{!-~Sdq_27^Bd;;!|=pR=-v zSw}Ivu^NJ|NpFZz(>-?T>S?q;#7J6M0ob!Bbx%HFAN0GBEAVQTkcNIt&= zj#WRxODASx%eOSlJ(NNe9TGs!DjxD4$-ye49Nbi!inek&nEoRdon$+}!*Y=Hgl25q3iB7*5rcD;MDJ}nadFM2 zz87v0&Sbc#cj|F$2px<8w~TOF;3MgTmVtEgx&|1y(3~h-k0HP8ZVIPPwIdrly2;(q z`gCg7VA)KqQS7J@$I-`n1bjRAPf|LS;5I=6p67F(sULo!%H{ur<}2qDdGP~c>;JZSwEXrE`9(%?~Z|BgEYK)ayl*?t_8;@YtWwNXc%?%J9(Fu ziEGEa;_}?rf#0nNG9b}bYF@pDe9C)3_T0XUt>b26o$MoF=4X-D)-!Nhml+(?_>KCi ze0IO9hsuw=k5PKB$wG^@uxlU+CpX@vmi12X&A)}ptq-FHC5mt_%$ar^+#ssBUL`5Z z4Mpdljxgt(GOka_A)frXd%6!LyppGnkeKB3SjYa7tehrTQ zho^7vqd#6fhq3`BIMLS^Eld9hTP{waZ*z0Ou<9DTX=7nzUoA$go(qDaAY6K1RrKXu zDH(KD5uMdFz$3U1Hx!?d>S;&8^skiU<<7$OQ!44VZVA1p@{YDYdk2fsM?scv7L0dq zp}qPCanJIV`2K+xQ0q?A5cBg%1I`PZ`l>K4?S{k5A#3Q5c02eoEdZx~zDB0KUWG;v z?4)AU>=KUh7tqaJifHhdHGYzMXk3`>8XQJ&^4mq15=#NQ*q{~zW z$h>DilPV~=qt&FF81eZh?WnzoR*Un=i-XZ{C;uKfFKeO73aU8KDMad=Y!1(+yGbiu zlStHb73lw1N8gWm2;3f55}Q^+TO|vDU2+mqf8HVoicCoAzk4d!~lzc5))Unhtn%(GvTQBmgsK4HO8*kPR`# z=>N_Xx57o>O0;mM>1iT}8AK-@`~l%+ZTPNp9@L-S02#sQ!m0){*+o^3N;%%SY`dM^ z>@R`!LnELqCjdKysr1KIT|`wMI{ftjlCj+g@(#q}$(OFsw>FS`Yj{Uk?>6+_ECtmK z0enW?fxeErfez(|@v&ex+rMNI{*l(h?0RRi)Z`{LsSG2sTLD6k-9tsU^jIRdR7kQ8 z#o@5|rhKML2Ztre;fClW7(P~)&YiA`;4~D?{YGNh#!Q+&=nMXM6D3)qBEzo=pUJlJ z!FYOsHXKQerJ`teQdo1GIL+7$7taocmwralwwbeG#1A7ZSoxXmyE_xD#xc^%@l#=l zmNPD8PQb~6Sk&Czi7ziq!bDLU&+pDBC2)$meH=oh10?vyra*W)EFC3p{csb19$qfC zhfz)+X!sT%k~(e)T&i}!MFEDg6?rUirvpKc)kJvIflpG_z&_(E(u&h-iTwB!IGd;| zD(ExD!=CFP`|c$gsy7SYSntH)*ODa9e8xd2&op)2ryyGN=ot1i%@zLMWCLl>#$xH9 z>$t_f6zz1kvrF$xgx5A*!iJu$!jya`*zc2ua>x9HIS1>>M+oxu(|DT`=XzRyw+Q1B1wWN2vXoYS7p^ z3-qlf5ryS9VU6Q_Y8SH_&GttF%^FV{e&@rZ7s|3-zOQi7oN`h%$Qx7|3c%GYflM1W z9wtZ1W1zAsxuy~#H2UO3Kl$l`w~w4i7&MH!DNDh$Bn#(WspaeYVZz9MEgYvC36PZr z)1X0Ozqp+c<0Lq(Qbf=DjlttTjqwq&r*kY+(D?jKGSG7;%$^&C4TFMVXyjAakujET z8J|qY@%gih7gyoC@2hEX-hL|gwFPEn8sg@&`=vWBYtt{2GjOQg7D(6=3f{ewfE!c* zU*FinoofzALqE{-Ym?~1-lMe12w+yoXuAG0&nZ225+9fNfY+Kh>C5Fh7_&W_oSE+n zw*rsQtJOlN@QsuHy?KM(!>?cU8%lA}uJxd0Wr}lD^5A_>IQ9(51RMKB^s4Lt6fPPn z3SFv$Z15DMqI)>u%Uz-MvwWz?SAYw%H`Cv%!m(Iw6*@N_Mh(Yfs5CGaO0Psw`;{lL z@*Ce1t(^t0eqSf|R1@j>?jn3wdmi@)_^gBfO+sSrVNT#B^l3iA*BDo_$4qo&{bQPV zp7(c5S&&9X?L=YHdId=4ncvQF8X|KS0Y>U3pxdYWd_*XN$}acN8RWQhN5D`~)0U-R zbzvd4?aL-(uk-Kt?RLUF(Z-8zFUXT6?(lT766D7|!ZS0cl2hSdVb$ds#BR+~a2+PZ zQ-|UtqubMIz}9FYDPMtd!6DL7Q@ZIPcn0^{qG|s7Q0(y?PyhOV!D!YI-ru)}2`B35 zXSV|B==2Wo+0!Sns9#R}WRLOfFFVou+B(=T4#UBigWva$klpYJ#{7gj)S2UtpWkbs zL#I31jh;dp*$dL=oT^CcUo2dRy+^7H!*F(p3Fh4>giHC0g=|R=8u9gk$k&tbS)n1$ z|C&JqmlvSfGDQ*?aSjKr9tuPEjzos%`?ot>g6E60;E&Bo^gEpcrpmswH6ma7LdQjT zaPe}yGo^_L?QCGy{9nwLbtRnsJxfM@LJMblv!9zVQ;XaBt(toqexK=5H{oLauhtCo z9K=jJe@VP&do+`}-JY|#r_R{rMmpGc4d>!S`vsGC)o?+dE11Z46k61RoTztOhF!OVMZcU$!wqVJ|6!CJMH)8T@D0{f;3(uX9ii5ZJic{5(F@~DO z93CoRzTUJD`xi9}dV+onuG^`w4Y&HZn_7d}hA$sE`P4)vZL=>or8kvX^yZ2{zNnh9 z*So^`MW>2=vyjU>dV#YPe`C_T+XR92X3VV8YmA(E0Cy-dpILrtE%zy1zb1O{Np5)I zI_C3f5w|kSk4ZXj#$l!$Ya`dfy<2ip>|E;4IS+2&HjRm`SttA~?rL{n{u$?q6E@xE z?DB%Rj>-XSy`m*IU;94q;k?Fe4L-uWiAd(uN_`pWS~;=tlaZX3>3`e}s}yGP#y0LA z&%^Mvp2|t0H!-_<7jnygP828@E@CQAr85Kds~G3P*8UUT>|>x{Ul zd^Kkwn8u{@oDSnx3pm9*S1wB1Sde&ECN>>a!5q4Ih)amkWyZig&X8x7^c>S-ue-K$ z-+bOO9?CVG{@LN&h@tnn#wnvEOEP(PuXiDn9~#7!2(B@UF_yc$?KpSuz#qokwn6MN zzGN;Jnn^TMI=H0yevEI; zFQz8tAa}2(i)noJi&GZKv4N9fImedIj8Vc@u6~9lD{nenB6m4YkZWTnm@!d7(lelt znf3Gmci=qFsx<#B_;u%(U}ttAv-aM02Vsjh)6*R0u*ijX`K65K{_>8_#J+fkD^y?d z-o1};Pc3GejJ>$q#|vw0tFLieZV%uNWJC%Mqy&jSrYlR#*H>`~`(l_&qmFWwElU`S z6Bn4A*fQ?s<#Z`8dosm>J24df;CvP>-PlX zx(+bUc6<{&eR75Q{jibgX}G{`9r}%P{TRWlNm|YYp4DQG`HztdzNNrw9FymndAZD* zr32aWDVh?^MUetUcLhdlX2UZN6Bw~U8aL#M26N$>P_nmgB%56s#QV(eG06?vIUB!S zj92V1Cc)ZY&~{XjHEzAcnH{>!9C`4P+q<%qk)Jk1^4UpMvZa=HViuZnFE$)w&T!`$ zg;ftZub!jKKFsCl;dU;DXN!0)TE&!mhj53*&0PAV1LB=WKQIN~R3us%zqv6+!`Z)f znT+a{VO(1HNX|RgP4He{k!?LKC;1Q{FY)Brmy_HEOU_*x#$Gt8#?IxLwUVKSxtRr% znH!zS)t%p;h`(C5a4VDtu(#j*$DA9ZAt^aORZyYg!UavZ$62lz#2nq+EslQ|zdQ+geq>P z|3hY0#on4?#Vbs^my_h5;Y{|(N;P&!R9HIK?t`nE`~evaRTFX@gOs1nHFc& z9>>v+Sf(V-MI2!ZH9MO{oV;xrbD96H`<4SGVK2u^sw2-a;t*{%@Z1;fU0NQuGct#1 zTl<4qTVX7z^f#7dYpbxfn8HmOTEL`+trb+JbcuVE?=ke&OD@Xu9p`(bo9Qj>=G1)# zvbUStm@v1uob6;OccA_Wb3^|Ob3|KSGU$vV>zFV=(tLad7dMS#3eLAV{8Thz%+@Ax z4!?o%avZ=(>e2+k%I@5~=1yjEkiJB$Jh^5@>wd1owm^`WYry91m>?N9f2`z%nkF0i z^9=X(&kx34m;YN=gmUN4zF^>of+R(|ClE=)|9Y@8V7in?v5v#Z-x~KTEZ`=$oMzhoJ0-Y!W)PPWIYDy&^<;Kad=ryX!8&a5 zs1hvY_nC<|b~1~bE+m z_HV5j+maMUzjuJds!&z3j#Xhl?~Y({R=0Db(+5aC&QNCm#oDsgM?{jZ$t?oE$WPop zwSnvmFD=P0-=~~@FwX-{9U^&J+s2rOl`&d#a=31llg#0&^PD%&TocOMbDJiLnOeg~ zT){E}N#5__>_4{}CgRKu?rmZj7oeKW=vn61G?dO}No*I`3J`QT3yZ^ zaZr)mx9wtbrX1lGgvT?xr;O(=|GLYVzE+p~dTGukYQ+j3etF2Gctxbjw5?J$%efu=;eMdPhb@H7jVvh$1#zHmE4+Z#e#xVzZz3{U6vnzlq}sF z#@w!~;zoT6soLayjuTDP5KmY+lw13DJ?Gf}iP5;|Ct&LiaxrnSVuj%X?%k~%Zgq=1 zyEZAeru*6!CjQrOiOkK4EvoEbd_OvpteG%*FoklI7$ury=v= zR%RA4sjg;{303a|-p<>(!A7IGDFH&pe)wdrID_XU_WE&an1RfvpGw@Y`Zgx8Ly6VS z{lpDfdRaVU)kX0D*?xyMEEV5+$4E>A4A^tUbpo@h7vgtca=G~*k1{Er+PGOGt~2!> z?M!8#jO#4-X0k5*<2*^Rpw7vfyEa*iyR|%yf$_THYu;l7SL!b@!KoalGJ7=FR3l%P6wkTTExs^!??r8o-!F_!z2^qa+sIL3?(lzmU5BXbHpwOUoyYC zbtPlxzh`168%TU_m@pFz!US`xUo(f?he)QSrU@QsR5{oN1#!2X{TbzXq0A;_MaklT zd_iZpH}^O;fvFSrS9fa-SYs9N-C?oO8qWJq3^U;58}1zy2rejW zW$2fZK@3SvXHs!dt2-^-9&PGwVNRR!c58CrbX>r+wE(|l25#-tAd0`CRS8PD)z9uKeN1T+4Hl7}alJWuxp^^PR7l z@BcYVT=?gEbHi%3@yiI-OkbHhty;{O2(6h0_f*D6a@k?S?fU;II`eRm6xqNC*Re3HYLp#Ka|Ls*xx)wl zB5>P{t+1_q1ynKo2-td8tUJdO9t}2w7XH;dKgkv_IX{3^7?^{<4!;1=$2Y=zTeo7> zgL82XV}jWtQ~W)7HI&zogr8^4hs=F5>{+!K#{KsVq&q^WuQ|-l4jl_EJhp?bd9Oh0 zB@H-hfd-B)GR6|A4dBIiGyFDXJ~SGYYAU#F2TVF-;1yd3ysSYPZ@#z~w|MS?W4vSW zj3NQnz1Pp)*dzl^${ezn+o}oQRL+OZ+vGuH=YVjT-4#|QXcoS(VkP|Xb|v06c|7*m z`$hP0Z6G_DzT_>M^OZd-s9}4v-|9>)E;GMtJ{b2L9As1oI3gz;0P{ zC_jG={#GW%_E-C}Oa88bejOngN}6G}mgU%er3>^vJr^T!C+q#(62H?8z}>l9;KNsz zaG~@x_V`g>_%w7A1|5^}$3HGuEldJy7wAFFs*}8c!Ws}J@`qDbgyOOG6L9^1Q?PfK z72c*f2WBn#$Sd5t25kBi4j;VOfkWmYtk3IbXD2VgeNM~arezDcZ|?;=^4~om&-p#R z@-6Y_d_!Jvs2dFNScgv?or*W<3OJ{LES6AJfV~z3NbO$%&d-*B2Y0LEcYj4q7el1* zZq}SPVtb73I`W9ExF&(+yj0lMS*pP9!4Ecn%M$p*cP6$mnhFPZ#)tHaZy#<)4Yo1JDn z2lMv-VKa|ThmpQw-h7P~5H6C30}JNjwe7BKz?Gxy!rdzHR<{A3c%+s$ZRjdn*m?!* zTOx~7H0AM-NCTTLNoCE?QebnUo_B|1o*Gt7fx0Q?SX)B_2=YbjuP_}j@^e7=+Q69& zmb=5Y&eg$Lv&X@f>9W97cOL9hy~6ISP=|X{CqwTVMZn|>;Hh6O`1G-#Ebn9sD7n9c zefPl{JdC!)&+ANK{h2JVz;GTb<1ho<9GS)18WymVq~kzS`73s-S_61czo5x!k`=4O z*T*4YMo=@hf}525d8ZE~0@H7I*i8+ALd^w-d5ZP-fS|dCt$grCcv@Zx?!NaAq>==n z9je9lJ+lR>vTdyO0!`?hpa6F@=wqiJ+AnGa;pTAcuYC4z-y`61TnI*zbf8IIDf^*L0uFZ(1A2z`@ZBsCGx4_`@#imQ{+6#m^Yzct zha(#3LX|pd-f;uDdDc?{i8M5nsfOFVeu+H0#-O&Wv-Ic}J}ut45nUJ0Lo1u*nHOW$ zpfABTv_RtxO{$Bh>ne1`E3POZt-u1b&K2OOyJ4<9I zWkrAP%tca%lb9oNQ_!{v9FJ7phz?LgdgW&vjX1{T+*Z9n1`Wp;xvS3fal=^@a_+q7 zd#xt@ST{X(DUAtw{D#Ro6V9w#q>6T_t5ciIlk{)fGNimdmbqp3 zj#*~ff)p2Tr)JXY(6y~^>EDU=RLV6>&?*4LgMX82*CbZC1(3tKlk7Jhe zTZrbg9R^KDFNHg4ARbI6+6y zny*jq&d{WJ!ZegP%SJTK(4O=MTp%(WM#yVf0x_F(fp}g^W`4B=(BAkSWY!`fxUW3O zOsdhPPAWOvo%I7oO=CY=xooT;v?ZGV_{b3^-fIuduGSJP53)xt(mm+USB{&RQ72k= zbv*TG9wY4RT0kPgkK14FVz$}J6TNgC!y+Sie( z8D@06fgx4Vo6TTX8T)GuZ`}1?O#MFn$ zHXKjyU(2AWDbtZFn@BSC?=!bY(zqN7M-1!l7FBm2rgM`c(1kH4*n76+XyLYJWR1HK z4Suu(M!7NEyDKL+qI7_6PP@V!4b~_8bUno1HbBE(jS)Ax)eyJfMRfAs>*%wmIbFdX zWO#BnQ0T04%wxuiY4}-7w4(z#ZrUpPTx}uWd)hC=?+&4|5|agPSOrx{aCzw5VWRhb z4v82R(AW|=di_-eo$f9n82I2ob3)r`#>hr8$ni)bg`Mb2b}`why`ENV&tVj!cMuQV z%QSL<9a2gjk4`$RU~X++Km$A9@`uA~7+1pP@o1S=Q@LQbE%S0}mYqu1da6Lr@lu5&o{p^0E zp1%);9myE+QY_p+NuKe`Ij(59ZgWa*oS! znI>_Ztv%7TqNIi)#%0D-#2B2RyY?zEtLF@okM9-56<@zF+21Xh{t0LK>F*AqnY|v! zspKH-{d|(TkNrYrH&zf|aDu93DN=*`nIz&drbbdH(9^3D0`dG(ve)i5ijtYaoVz@S zP709~oc)l?r&qF2{q8!N`7;9Ly4>Wy@Xet5-mX-8oH1kP7D?6*W>7EPR3^7309^_T zKt^|FlGp{F%#)YObj++gv|v4=IWAUo^CD-oYxZWobFV6zZuEdDcw0}G=&?-O>;Tas z9bG|>Q#A?sAj_Bz?_=&St|Yq04w0t=J1dU`kIKb=8Wa9jxwVLiPuWW_|gZl;pA>omGvb2`*k$E?k4@O?H9j!1J_XvEE84a zI#JU`BjnxZi~7n7k-%5TywZ(CN=x17+(uVAB)JeNYnjlj+ZWK1rs-%wg@i!mR3sWY zbdJecvV`s~Yhr?m%*6vc3i*qjw=(d4H>67!pgnC3sBL2#U+TUt-&J}Kah$)6-V+re zsjympPj)VPH(f!HATtO?By3U9LLE9a#)x`6I6&O)EkUbh6!H&RzCfd~<7wx$OzxTJ zq7VK#Og@Hm0F~1M%rSmtBGBlq4&>ahi#UJurjvL6g6!^8+R#|YD8GJ!0$=}!e1i@m z*MM&NR3?&E*Tjh|E7wxHG6SS5H6ogzqJdIWZsLNG2GXzfh{&&(qkDg5(}DM1;c3)fLdcdSkEm3(KIz#VkA&g`#dE#Hy4k&eT zGr3ESFvVAU=%Krfbh36IDwcT5zl1tS-soutIXxvC&Ml_bFX@U~9c!6K=Vis;yp-tq zc@vTCuB(jkl?oKP<1%_I*G=ynG3U{>nUBYy^K~E4{>-B^r$T}1M311e^Rtjmg0^_<(oFc`>??9A<}NbZ z-ozK_pC=c~3g~*S`@ivu6Dm9=Lo+t&(j~T>YqDC8PX5Jf8nV^O99`n+iCXSjQHiox`sC|gw3cK1e74(( z#@MVv<6Z4V773I2hQboEtHPcs78TRVE0DB4kYEC2CJDy(pXIV^U(xqeX&UnCBs~yz z6jl5DCMRod@w)>esYKX6zLt6xJ$|=~mPY6!?~zej-OXi89$%uJ-5FdSXg+`IgiC1N zg_+cUYCc-$V@-{`45=c=0_@nU&H2*eNN8EA>z5gD8GGU{lU&6KW{L&`LloFM_+|8jwhe>EF@!r5oy@pO>$=iBdW5N?wPb09ZI^+yl)oK^9%K9${&`QW3?9j=)TJQ zni_~UR=y^-;H=0;=1kagyO=*ET< zC~fRgVrn^$YJF*@wG&P=>Nf?XcwPX~-}_jUQ=Eg=RNrETsU3U%Xny9PK6sc?zRSz$wch-;a?`P+d zD$a+y#rO(3pM8a2z9WOynC7B;UPg59#tj^&*^sVCu0+cVv{3c(FLa*qC+2pv5&B>? zg;w%1m_k>Eem^RKIz}*zI_XJM<10~d;3r6=yGcV>J4Gid>5+^;y3<0PK5za?BtQRv z3i<&wH|m5aX270g?|4pz^J zGbLdqijG(zs7L11IP4@kF5MHh8H|rejmwF>`}7~zO>UzXr;Rf1m%_Ncjx_ojy9`x&NKrpoLqYil z1+3mBhrSmUFv$_W$dc)r)M@i20vm4;7s*}BEu&3XeqIi8vo@!-GY6REXU9^DzjYks zEgvmvTtx%Uq|g~VJQ#|AA(@I_zrp3u(at5N6GmDI^~8#;7!5j=E04=s|O zh*E4M1WEIwnN7QNk^UEf$lA}JuHNE=&c&{vr=Jd?kH?QA@cJlI+*MAuO_vlqLay_= zew0M_h)|bvA=TU{i`G~d@=q)rPdnmHGMTHJ$nn}#`tJTtvM}f`l|5ZcQfh9HkChpW zBigFiKCm`KH5u^lQmcW{j1+=&`gX$93z%4j%2~zq>u;K|euL zX75Gwzc`A{l)8|!1C>bU(LS{PM>4(lqmr0@flP4S4~|tEfp|Y;(dPD3WV`q$)BBw3 z16@o)U&^=Az2iEm`so5vX}S~ro^*|wGO`7Qq$tvGa)=PJk-@x%ejV6&@Zecx5iZSEu`Gg>JJ%ocpws4cEqCPt{3_8%ajG@U$!z8 zYdPMh$_ZN8I2+|C-z7T=-!dr$&uEChl;C|d*VTQbB(^i&LgwX8Beq5MWTTfBKQ8+N zv*C*glBwcm(Ps{HY(oH2y=_Z1t-A- zAnO55OtR)Tc?8nu=kD_Z-)0dl2|Zf;=Q5u6Xa{qnJBH(FABEwys;K&53hMm>s7X1W zs+hb)hbMg|?e`Z8dhXQG%4tos**yhm>-I7N(-zUB&j-+LRgM?;b{4|>4M<3EBWb$2 zh<{ct2F>O;JAK8%+^K2tO3wYCyw$K(WrRVC;dqf|O+n*iCoc|;m4 z*Px5(b_iYF&t&?jq2Sxj)Z}If^Tt}6On#}ymyH}xCAenheb*^S&-^J9HQpaxs_9xV+ zfzd?q3Xvk}UBLbBND&=x`W_voSo9|$l9q5e<#R#VG;d@t6MfVnco{~+&q&2;~$8&x}gmOssMl&p3wVRdGYXS(D?%>A`JWJ(_( z`FI5r0)7$p)=lD^SxP#y^U;FVA$0SY4kB?WwB6HK@Zz^K17f(WC6Xg#cMx&xG(iWa z_mEltOi9GdQ6?yQKV$n;6Rl(4Fn_LU3M%ieMSE_k(=qCcky2Er3EPq-%4tfa>HYzzr6rF^wHQGa%DJe1;SiG<^bOVcb9kn?3H=@Qn1(lgLopSv znC){P0GCKVp6rU>?63VpB}scO>cc+r2k_0ftij=_RI%SH{7sJ z#B^Lxw*jVG?_opNonwE9%GsWfE9?=|+4%eCG+HL8FHx zcIZ@r^OiTVL54oK|9&`>3vdUmW9RWu*;C=ovLT=xR||5w9C$xNe7N(|1MDZOU+g$* zefT{;4}`nfW9yTb+0*N^f%V&k!aHKl^^}v#%Iq=0Ze7~A=!-MV&RK{%J}YAnw}0#r zDWTA4oC(-A7>r+MXyZ!p4tPPU6R16QhBrCKTH-7_yy*vbS3j@>Y8lRg9lNvH+m8ub zEbQQ+19f0DaT~aD_dVDup#T@uNCKaXd04a95_dlI!B$15n!c8q7qbl^ zl2Cax4o5GI#2X5?z|WOp)~?ZomugW2yuSPalM;-eO!_q3*0G*-4gUmI)h~fj`3-DO z6<_G(@~i3Nf>W%YgCSdVy@6G>GRBO$0B22R;1e(n^4oUfH76TE_L3CP<315|J=x2y zcX$V`yh!Dpdsxa^#6JY`*>SAg?)uL1EbuAv8k!y<&RqEnrf1Kct9hJhG za$B~tAdL-|*Wp=8*}~-+3t;NpP+0jm1OM5)oVCcf%#PX!@VwA$UfL`#KuYYe=3pON z>iXV3;5T8H#!q15=Duftmq)_y69GsGn=MS+EQ8lg(T1DeHQV={ZL)V=wNH4UFP69I zmp3bA5X83gDezU2gfE|r1x8a7;egjZaK<|gm$p0NKF7V#VwWK2$JNW@3!+%)7oP~Un^wRN_)PvCcNj*4XkOhZN z#zPBH8TJ{Y$$AM^f>-|swP#x>0wy+z)PW;Lx4d)q0Lgj~&xKwmx`5VqB{q9@V-v?-<30Xl0IVb@;jo%@*fnY{ zoVcb5eDaaBPhU|XoTF337VLV=jVe+caPVUG{35ddGewfMEI0H5Lz|d^{dZ7lYfBPgb`1T^0dFLXCwfQ78zYN)r*S+Dj zHV+u|&k-MVnh0k_X2PIT!LY9?3olFLW0Q)pc(U<-?DVSX(Eq>$TpKzbJ|ew9w%ZjS zWXJMeX>5dQ-siwF*EFoOQN*?yZH2cs*uz_s6EVgmysfExsO~uluC=zuyAS^X-}S_- zy67}}QtlB5=zhSS8a~0^JmkS9T|5S|w*_J0^}|5D#{l#^Ph}?ruY;^!727Z_i_KUS z15fH0!}%dc+2o8Btc>Rs0OXok|7;U@nQski;vDc-S_RHoy5TJG9PDNg2lX#Ru@S>F z>?hky0$xwJ+-I6Meh|08{t-Mbe4(j}cZJQt7Ad~?%@!$~yZ#(FC!USvp6S52V-8?( zSQt*-p^v`=uV;HRMuhQ3jY7W}TrSadHuRK#4$5A};QK2bvF_bOY^k{&{|!!tes^xN z22N>0g-?&!;MgI8^&JIikPuN3%> z;ekIr8hDqE3V!l42~W(KhY#AN!I8a7AV}E^UNnw_F&>;X*~by@s@uXQJ=g*k|2GZV z923C6V~KE_WD<;@yA3Z`x0#n5c7x5mdyXBI*MV<3)p7XQv#iDQzxLT_rkFRVhWkH_ z!H;&TVfBD%(5!noh+8lgNJV9UmFmmcyjQ+>Q`vb|-v1@rxnT#mq`!n+cjG79I@y)2 zNsR-guM&U+-yFLvFv9CsWeKn7Ex@K>b!;}%z%tTp;O($H*8aSeUDH1fYX>R7o9ry? zDZP_j?X8M$Ys7<)MGJ6_UNEjb%*WTBCqN~c{X+G;7xvvQk9ot@y{v2OG#t{q3r^=e ztzSYjF~3oX{TAvCS1w9{<9?rK4Oaz$MfOU<{bNt_b%SSGTc)ws3hxH?q) z=K=%!^LVBA|AHINg-`}c7mM-U;Fx+U+@muc4sCG+PqY-^lPz1ZP1zOpaaK0e0oHIuSXxv5^Gje) zuYqvY^jToY;u;t}GzI?DcEEEr`0S?ar+_mY0GWPO+_7W^EZbcLwrCWx>0|uxpVzDK z#V>w1&~_6(`EEU=Z=~?EZ@w^7iu0ECFT!zKCc?t9>GnIdHnP`6GFYd?2M?Vp5WZUP z!%8ICfha}^EB6-i8k}x5$^Ba|)VG+34<8(l$?FbQZDj&%WtPB1t8eU5(!!3J^Bdf| zIv_mu=@73ptWPK`xC=Jg)&YYQKK!zJF-*F@9@i~=52}ys!Z9=cu{~en;Ai1!P;g)Z zoOyg2?6Q-@8d=?7p9H`I>bu!{1AAbz;&S-a1cMioBw&B59KO^#1}4K?UWc*)II{Kv z@J`W$-)@=W_o8F$qif!9@O3LtxuT4Ze(>iF23`lI0~^_l)5}41p5LXzPIuT^NoU;j z&49gi8M6mx0q|G23jY3U2?VdwdHv%6eragU8buliqiz|1=c#kKb*ULNoEpno0za^I zeKD(**TH1~Ex|kAflGeFm{;$Z4je0H!oBk^1NOC;U3RPmjQ$>B7yfX-1=mX1QR4~t z)2|y%rM1n%zRC*THg9>Xx^gm}dr2R+|5pwl$F76_qSDyJynHrlZ@;}sN*lZ5)?>EJ zuaBJ;BI5NG7TT}WGv_VtIRcb9W23UJHptzx2&C`)3f8EN0Wq&81GaD#{=6g%W3v(9 z^zRe9#=nexV$TZAX6S+eC0RV-z9(LJ&L3N?3Wkk=RqTC_<=E}VJUp;I5sEURKwfMU z>#rw;Zx!`{|9&Te;zRm)%6pbww|W&cdGd)hFL1>H&5Q6mWe=G5>^htBr>DYv57{GM8fi)bDdXXWjzG6kCPvOTV$=Bw2P9 zI7?p$_akawm1oA8x$G3)Bm+_b3G8a|wACo~D)04|}!yr=Ia@b@E@ATM3W zb}y}9qfRX2O^f&@v`;Hy{bxAZKX|+vo@zG7A3D--&k75mo!7-KUiX>}E|O1Y&i2FX-mE09?&$122{^;6XqKsNolbeUn(=_i|WxF20E|7^?05%S{gK}34tlvz=WhvK$^5i?ucKDL;t1=4) zM4HgH{sS%?ptmszoHkBlYevhOE(&^Ck14YFH=YlxYgOT{GtumoQ!%_^ zh1+0!>mFg-9&O(5)qrP{b_LXCOoH*i0p}<(*lzwZn2{`pYhxzkgzeJoQ*|rYd*LS7 z&Mt-iZ;hc#|4yuNWFv5Q9}DZg$itzR_t;m>iQvx+JNUSLPt#HrJN!HJBT#=}&C7|9 z#+~nO2tC(+WS=f}#y2%gaLE=oSW~HiGaN+1feR~u^XL>9U|RS5$40%BfeUkw0JD!X z@x*`=oCiD)>`YGwE9;E0#U2WZd|JTw<@QjrV++qMPL6E^4}pDJ6)5=A$!7FA!wC`A ztc0y1{#RGTW>4&7W2I+e=jCIu#8Nx3UT-Vgs5ui9y2=2Ly(Vx*$`ZJ?T@$P0lPvF& z4d-du4m7KzV7+&yQ1^{5`|8gX@G#61jMg6#=4%+?zs5;Tho7ARGUHl>XQT9RWWNEN z(sGjJScFZhoP2@)2^sw4fDP<+mxuW|{=#buidD zV7X`_T+y-wZawdeE9iP~#8Ax2cIe?3TYvHB`lswb;OHEQ>r=s-g=1I+PzyqSD?!VB z6gYoBz?Lg}LamRoIK;~UEOwqFY)ahDYAH$MyH9f2QU7KT$oBz-mRvq?ks+3!Z3nmM zs$;je{=i`Pp73{=A}beq0%T901Wpwi^Ey@me0H@v{%O4#J2_e4K#5}5Gc^{b{+tO` zc(<_4<79A*bP|5kJ_bHEoDWovdUJge1=xROFWYHrjqTzb;gqK@!QiZ~?C!w|-i^mM zzzR(i$fEC-jBGL+1`QX>1(=UnA4OWZq|X z*RnX4r?DAC<)+z-bd=Z?^SrU11%eBecEC^>E}s`x4$f-GvO!DR*!ACJ+3J!@JZs&D z?Cd2ou=L{fz%qY|aP+ws-gQ+T>opaEPizvm*I$jLO95Vkec|yl+W5kWwZc21Tw%KS z7C4@w#NJg@7*At?^H2-ouY4Tn?(s>Y0`DnnZs(%-D<3wTn*E;rVOqtLrRGN3jPZ`hEWrY&+ z7lB_V5Aa5lRNAMYGWL#M#5$TIXoJ22zuIj0C_52Oot}@Im)5hHbHmwo zWv;)JAOZ~!4uN--k*w!NA6C+H2^_4_z~6?Gz^Soy!fB_rv(|%NK=slS_GL^r?`@JU zWc!Zt_Lp1$HSbq~;7duu6E`$Kf%ZxvSQX9wcpVPkI8W7;cY8KzlMOVu z5e2VWJ3#S{RD4<41^j)Y%$~ez!aDPxg5O&ff{5c=gxB(H*|Y;Z5aqcVes9cUXJ$pR z#@s$dspmP67?};ij->GB3D4%5$iY6L&J5p|%V8b+EU}ToJ2u}=6--U<2L;FM z!G>{iaMhKU!2RMLe5-FeyYNalOmmBX8t-E99q#X;%2&bm{vP|th|4TzPY2UIg+S`% zM92k{@mIH*yyQ*yLFukL+?kOH{uw993-@RO*EeSYw;wh33$|#%m_OQ$TCeNu7wv84 z$z;ud@!na&)`&U62>cCv;!gwX5+;egbvjT(#a;q9-gWWV&GhMl)96iY86C*INaCMc z(Z7jq=uGWd8e(z_UDVow%*H6t&=+~s(S9Mi?i9Wv6XvJ3%^K9TFV=8w71-!ja&$(YE)+aQYa*;#vZM-))Yzakjt1h8sf%e`=`vaqJeJ0f#nkWUH)iYHG-T4{1uZAZp;v$V=<~%@ zjEd(f%6sWUu+~pDJ>&PTc|*)%B-o!6kVqIjt083?hM)!n?*boHjp3i4)gq8 zB!5hRl)!u95O+WKf~wt3W2!T2iROx>d=kO2U4QnG@N4cQtbH!p%CQZPNr;f$88xw- z%6l?<&TTUOpfr_^Os46jX5xZ}wPf9|_sGa&0~&boi2ew;hQk_F5J^}?Pt2AVOYcyp z7n~q{t#Xm6dGemVahk$BT4{-1f6M2)NEnb=zIDt#uc>t2^mO#}!B4cLbBw_L!CSJK z-N*5??u%}!Ea01e8^<#nUW2CWSVn%>bTda1{_(SfF0@JUIoMm%&X}YwL%Gh(a_?K#7FF+S4mhSLPDvu>aq z^(Cx+w+Zfz8)Od0nxmt#&NQOhn)#++hE|uYq6g!iqa4mt|IntD6Vh#APMy=CgRfI* zd%6u8EB;BO#@mshl`cpoL>{#ly&ye{2vaB3L*IBzMsK@<$+LnqWEX74$Vci0Fx1?BVUnNtaV}}xaFW}>4i;?w5F*U^m}jg1jhz6woeL z2B~eSW(=Y!(_;ONe{8jipz?R5ATsd?`Bs*}Xx==IzMm)bu7#Rt_VY&~t%6oMckW6q zgLIiftyGjfX**5wDWb>J?@+(DZ~2DdCG?gs9i41hMdIrZGy8)J>C*Ml1h$UQ>tk;c ze_;cvZ!AJ=#L1Ldvbd!OT z*zdqi#OyqQLLan{!+Wxswr6LlFPBlATw}u+>hv=E?4DBo+snjY^KnLT{a0e)ahw@m zbDeSVlM}yayGob4Mv@I?ZZsE(fmG= zU7d|0Z3I-X576itv+4MeyU2Iue){b95$2p&n?CEP>!L+ex;htcqTqw2Bx%I8FR)4$(a!zlm7>DP49rmw$e%A~JdHjBZrA z5|uSGi5`~&kXmOZ*xH_pS_^ig+LBT_p*Dk-tO=k`Z=@sj6?brvN&)gT+sr5j?nR~! znba=i6X!R1NmiMe(v3<6M3h!ZZGRp_$EM|?3+-mq+CG8ajP6A!YAW??OFb&A`Wt(x z?RZOCCZ$JbT=qs2JoD*{x#O6da?&)gAsvme&k#+o+sUY9MbooGr-ZlHuAxirU%}aj zoY0@Z7IZA}1bST-j}jx)sk`wm`uR-_eerK4k?l~YNAF)nU;L`bng>E==#33pY88ZT z+s-C+HRsT>nIEXetYYTi_a^>G@O9*%IENe#7-4QXohIpN8%dFeH|2-hql_MHBvZbh z$umwQyEQj5cTNs4OP0LhOTSE__ieKn?Wr8+FC&JLyA+6)s*7m!@$JmEsS#*o3deC3 zz2-Z%6{7XKAERj;hx^P;E5an3BAeZNspyxCXrIz)l6zK996UXo=E$_7Xzp%VcE}gi zTu>M9C@P}-^OuO~kq{~_(xu{ii_r0t`%pB;&a*z9Mbiu0iKDWBd2a^Qy5b-a8&5}vzfGcQe1as_rK9EJZ==%lA@rbL5z5sHr@LmKfXT(z8Q;t@l;Ott zJGY)jx79So!7dZ&;LI+%D7}PqjtNC;YHa8s-^)}=zMZL&QX@OW{V#YT-ABHtxpXyZxYfw)eK!b)Zl9t#RqoWP z)|S@&@D}}!)uS?jC#Z*KG5y@$Ofxs1K?OM`$fw~P*`In0#Y)bnEw^=O=)?2qR%kcB zMoLJu9Q|mtQZRzdX*78KB#pbT&0IBhB~kH-%q;u!%q?#_aq)y<=H{ZeX!-NA)Fq&Y z%O}64wR&HP>xU_1o9O~Habg=|C|ge6yn0K*_XwzFojGIjca-UM-z@5J&_rYBnNru_ z%jnsT>uB5BtLP$AfVONYLc3SIhlcKQXvXe2++Dtb;G^F*RP;}dT1~k}9iJ{ozRP{l z{GQ8n)yh~J+|q>7zv$8p-Hph+&H_DbkD`AKkC5Qi6Y0BKS*WVtiJ4!Y$rQ>wCt82c zqA0l!L=^FWz8u(LC&ecp1ga!`4Z3gJKc&RVhJVHrzl< zr@ZGYdsflhs-tKdiD2ly5cI=;4d?VsW7_S!(5759WP}EZ#^dLrg!GL_B8V^sM3>Hd zA}#PR)fc=`HWtT9Nr_MBo6+Yp)R>Lj{OW4)i_UsB83p~*CgvO;c>0F}*aV2Xmy(uddNNJjHSn^ zLFEH-hM!2k^<={YvY&aJ@s07GlgdPtM!Zf2DkrN>z!`?iaQOf+Elo2+R}(Q9V&H!tFxUWYmr^O;9& zUi5pqoY*?yD~VUmqm64M@hyiF^vujfD2B_po=kP2pA5Jx{mWytqc)HJktn80heTwp zu@OB#iStmKoM61ZZA7nSZ}8VyjUuV4E@H8NCvt3j&HQLK5J#L!WM;&Up?7%wj7hsZ zVpld%`(=FEyJ;GIq`n-P_)Zl?`<2pXLo$q|M-}=w^%C8>(i7d%+KxiTiRjMh{C?qQbd@)c8lM=;^!}$T!uHDQ#Fz&1!9Ex3v>_N0dm;?j5!G$7q6$MIax;BA6JaPtG$g|Lh8N}Ne=qdN zypa&J`g=D~`4Wg0`OAnqrX`_#&o-p&ahd#k&ZoXBFQQk;VrFL6X4+A`A(u1NHQ@J7txm8F32{0vf#F$ zU(}Nxfr#xa|I~B^`ao+BF9*4 zq9@`HA@ylELKAq2G(;OG1HDwVqvXFa5ob&IQ&H)N$b`pMU8L|DZ zm~6?~NViYc7g!u{MO$Kw#6`2Y?@CIQx!_}p`bQ9>Sf)#Ps!Qozku8c>XwcZNKlXD7;) zHbSPKw8V4kMu>&rC$onmxrWqBipTiB70tBKf{@Osl)O|s@F_dnOYDMv0N12D$#3(}3n=qfgGIDpNnOSa0X!UW< z5k6rYx*Wv0sL!dP^TQWF;*MZSCp@S16EjJF)_&@l`;e48%|e^Uc9Ml#FVMa(zSJ(X zg}fR%LZ1B|VYDZtqWU0hX1Rp|{b4#=aEEhK-zsgP<1L*T<-K3X$_sL6J58V$r{t2x z8P0+}`5P$LjdQ7o%Fu#28sy&|HMG((o>njTOB6y!NH8~xjVjimq**^j->u?M=Rs+d zc$G(En>SIB%|x{S+;tMQIvnNMOQD7x%eHr`(BMqiFp7Y&u>EOR;*&)bPez)y^-PEwkeEZTsBjjahJ4@8zz!7Ka&4y z5*ThUMY>f$knD}=nfMq~=B{p?ltcFihRJlvx%k+Df^hFLP>F0YYEZ6Dn+E< zZbdJH@{m$MBs!RTgRW}(%fGAooMdGVB2g;GT{ae>#=;_+wMIoSvh5)Ag&#uO_t>D) zPnq=YziISJk-YfbZZ+|g!y?=-y&t`QGFfyc*PL3(hEnd;WZs825^1S!(rWL=c)CKW zG^dxmZd*;eGrova7YS&6KrStLHN<~!RYt8QQ07x0MA!B9(sd2vX{`EfnxdpYTY$pv zycG@0D^iE~7IeqTZf1IU8@WC519?eyB5`Rgqj)zT4TtDbn;qdaktc#@jK`DF{2L;} z(n6{fIYMfqjhV@UYkc`#^5Wl;-xxdT4d~#V21=*BLgT#jP~5u?Mk~sk{@`-|Gjq-% z*5@FK*Z4rS6&0x6nqKD6xh^vC#%Z)|RRDc_tdHzla+rzUYK6+#8Pw;r3e~vnLvMdo z6#Z^lg8ut^mfqUsiyr7zq1X3A37)v0rseRNBL{j(;CeCnxWya2ubPG|0u$(9d^vfr z{soCmJWJ+Y8jHUj&=9RI>O)s%)R5t`Sx8AVLJutXM{`PE5xC)sNGW<8edZd+r0m>| zG#vS8>QYNoUVRVM$9M8S-yKWxq{87V&NrXSxX})+ZbtjC0}`#ADN2^?B^jO0#Ho>? ztvhwy>A%TiuNLHl#rw>Evh+Z-e(eNQK78KQe-LLtWhDf5K<&swJD?$r8MWv z`wWQ|dt^yPwvsGalKnToKVW9AbItXh_gU`ybIOm)@bD zt9wv>;wg-*Fb4B*by_nzO_JQ;g=N_B+iw~6oV1ox4z<0KpO zxrYv7ZhnM57qh`YaD3`*PVW74@rESC;QA2840OU7F0-py%Solkd4^K|B|cUBYcUH;?md?%^U-_XuXz7;q<3pE}xg{$;X+ zfn48<%d*|&KS9*1<%`boBRSo-ut_o4EhrBI8$)Cq8iTwb*Nw0y}e4 zAveKFox5S<$t1X+kTDHW@N>}+anr1DW{|%rSNt@W!88AH`{mO){THR2O}QU4RYl0% zA8?s*y&>kVj;>`q@62W{q@;1X-js=jgLZS`)4;tKItu2n=6qC%Opnm{jBDKF&D`rT z~vf7@hF5fNis%YUp zs%0@+1`7O}W|s3g_DgmKbQzZ{jW;})jPPPg96 zgzq>m=*zv&S%#lsG`o7LDHmrFSXJ{<)nKh-S2M zx|#CpCNeatDt~!fllZ)+2dl5bb6-ylId&7-XD-n-M=@##$CcLSP`M&SF9ve~jl`E*b#qIg>g!AdUCTM9aWR4FU z%}ZSr*tX5eEa&9SM3oA8?XDbVQMNkU_p4K^=j+2OT=~pwPoKzYw~ul(R!wtU|5kzj zbuga`z4MltwD&*G_d^|b&P{>!nZ8e?3~1S^}F#kgK7mAE&7;o zdL@kQtJ_?RSp}mIE1RfUj$wZPo6RpgCuA?oBP^%)k||y@n-~5az)n;Wv0dgIXYUlq z8{VU1s>_Hi*?5<8Gx*AAHQeW1KKS$3M;WnB@w1r$0hhV1&(m1FLnHWN zFBiUX=rV4@Y$HB*$Vm2n&Ob)u?Krk@ImeX$xzF_a7BQTH9IrYjfKPk;hY@=}VG^pU znMrDzyw`g@HgDK|CQoG+WBGRh|3309Geg~(HAz+x*oHZ>uXl9`Zk{C*C5_=pOB z*w}%eI^z7a0p_NgLMU5OA=CGB~v^XJdFmqwOg?*gS z1zW!JSW$Iqu#nw;T1~vBSBw8VE{3^VC(m2!4rHuCGnsI6b@oiuB__)+iA$3Q&u+P~7hD}iBpD+2&sJ@bNgD-3FKS~dBrBN-i-rI;T9P&ZjEqm9_`ogkf z8(O(afipkvTo$*v@^;laeO=bsTal|j-^-=`(_*L8%JEr2M!Zk49}~4em4EYUgxHtS zVwFu5*prMZJEXTrJU6z6yR=1>pZCOwdA&n+K5I{L4~O5A0i%Y9hdq*K^LENJ1@*m5 z)k0f#%KJ&2Q=S52bmJx$^uv<9bw!DP^iQ7;J~NajiUw@U&|&h}c`(!U9Y%$v?x*{6-=tp>1fuV3fJze{6IPUpBCle4(nvxakP zfM_CfqXQZ4>`MOmP6NQ>2;r^Bb2|6rE-=T~>>bTcN`BDuJQjf~v-JGHX9``s~htF(nV-_37FhMr-r57uF?uAaxGW!N(LayiVifOd|r z4;ju?Yk)EbXBcbd&M-4mY-pi0$Z3TpMX{I4CgM% z-(WtE`Nw!^mNCO;KH=I^BuxM23dVUNt395pxa4q>2;^~V{FdHKNF&Fnw=QeH55IFSj zcg&LMc~s*{IoBMX*~7VTt?s4V+ubeV);vT0+Max|vQ7h6kf6uh{(7x?{;b7}!sIK} z?la1n+AoHJeKlXW#KnQ!nwh6Lm**F`HgP^T@#_G7yNv{@@Bb1m29 zmDW#Y-n2Auhdoo6y>pf^3TH&jwCOsmpgxIt9y5xyIhTHElke>YQE%N z8=Q3zdCIdRoBJ4xvLjXBk4gl6r!Bb0ITyvfH&=0IWp|_K*b>IN?h~`q@ta^`R1*_7 z$c}$AdX2#JX%}~>{4D1)FGsvq{TSDXUj%JUi@6_llh}c=np}TnIWrjmm9}Du5xFdK{ivBGN1kQ`aRb%FPI$^JdG_J{R1Te1FhYE;d}5-81JnvodKAyMK})`|qI*d&{ANdG~$^YjKNZZ-`ylWNRDVsV$5Z zm5bQ#U;Np~Pyz3lwT;cooyF!3pUw__ACa9ng1B~hwJ#&Z(R8%=b7vsudB?m>t3uO zDzaX`7I1f**Rq!fPT|))pUAq_3}hV)eAuXDVE2ZNU?24OvOabL_?SIG?6e=gY=qx< zKH!2QV|IN$FTcc*Uw2_4KkRxbW8|}ee=64Ct(@*MNwb&nMZ}!d_AulYv<_k&WV-ux zzTs?<)(%0ha25L`U?uMtvx!x*N@ot;cjh})q{c9*sT`{%|yX2{wI)@Yb3`+CGozIpdj{(7T7-#*WYkDAHy zhgz1g=C&UE&Ah3Ma=;D7dh;eWXx3Ne<4+kA@w|vHcAL%)2=?P!`f@oOGL_eve_O!e zT)t*9!_Tdf@ol%t+<%%Y*kh}Ntln^$uF+~8-=emfy=SwB)9qnd`=)`s@bVH?Cu#~? zy32;ISsuvePbR$QEGvHE50;;4ww7JAA%HKOaGY72GM*h37Rg?gc_6}n1FJR7i`AR8 zf{&QDg$=*6m$Upeh`;W=o%aYn#aVo_;&(hSXHT79!E-i2?4ss0ZpYsV{64=}adG!* zewQNQUG9!$3sN_-&Sf&azlt;Kq_A8Nd&`S=ydlr&?$+dM(&z94PA+7XJ}&3aNep<8 z=u&26&KBO~x&eFKOrKwCKa)5A>Ba3E7s@{v>B`sos<6M_MX`(J?O65Z32b|)E6YbB z^M1`5RwZmYn^xe%rc^KD$2o6iop$YJ-bMs)`_~TS`&Dl+kB`{#Uzr80Oi{(x7VctM z8N+>Jh9A3!3E-cRP_}tri-573&knskgO^suv0t~&m2vF8awDgPu-h6(^Sc))aX)3g zsz>ZnUaGc+&&rk&*(z-K714fTcBU`yZ>h!qc|VQ)TKS7v(zKDa3Yx}#IIhXs{8+); z1uka;B3ij;Eo<31qXK#J#0d6!gFL(1>5RDW0?#Na58(syuX9|{4koixhjq06CLTi0 zF`A=;9cwo|7bnfu;*&QO3+(+$WI-iX?5Msuynli=YbsYIUb}xhFI2ZV&D?-H%} z@t(t2`D+IJ*{j-o@tPrG8`WX_-abVZTbVs_d1GYpPPQ*E<|N>=@XWSLkg~P27@{T zeflHVkTbxiI970zH@^~l*0)p-cF^Kiw)itm(PAzmYXoz`L%=yD`*WIdW!(73V_3;O z6W%BP88F}!I7H`n$h*WanfF1T#Si_MLB+UdfHvxl=s1tVk{;a_4r zcUc^1ufa-LZ{ElFrr_hhXIw<@D=u~FPVPawF`qCxnsXEh*zFT_`3s|aIjaMOT2N4vF5<+oeR+k>uLN~b1&_?0p9M;0UX=6MXac$5i0;hsMGc(^Wm zyTy$8?zxY9K9n+F107htLXKY%(#!b0Rpz@pKXalz>%^fo*6jFP!e2ew$&Csc%F6p0 z@_#a9pFe5JF7wW0{N~N&ChyT_3l z=gV5M&#Y(i(JpP=wfq=?!!ixNXqc=|+jEMsF;-wa<2Nxg5@ML{u)*vr+fq(ENsrAK zpvQk0euaB|#+dne%8;-2AH$BX&u5N#O=Gm12eV}%WB8IwHsUFshWyI)2L;c{2J*4- za{LIr--3rWI~ZNBchxN-9kxuVgb6wVW-#(+;w7jg|7SY`n~k~{IokO}#Bl1VhB+zo>X?59c7_+j_U zxPeiwT>e&L-h9s}c5Rq4v#cX%N}KNwro1YbyXK!QHl1;Y5x$+m7xa#2r!APmHm;t; zzEO?lvWqRMJ=EVa;*)O$0ZPj3kBe%2pK&tx-gPP3I`mPYnj5h_JC&?EzY1>s@FDGq zD$?A+$0aF;(x5`7wa{qYB`Gk^#La8Zz*4D-$XzyH&3N&RT$MAR0lxKgSY-u2F~}c# zXHKO$5g~Z;)gA~cQbe6o!yvjzP4wAWh<%NT_@%U+Ngfvp{vmJh?m;(+``6LJQ@aGT zTWKGTEFJ_Cce!B8+E$WwLlM`#tta)eT*p#=JE?dtFa5N{882`9CETbw9>d2_nrH1O z`dec~nhxwmgH5$?d~k=vJ!C(!T6KV0RruH9F5Emgo_>rTLq{L+ByrKFggNYKcqf)= zWw>VkzKJdAKT!aiSFXTy?eRj*&~fD7tY^wkVh-$;pq{Z079TjINE`|)BB2UD(f3-<`S*&w!@oqc=>R!))S z40>qbdl^T{%3=;pKC_BgzkUjV1Ac?=ktSH_p+uIo7Xh!8MHf3Oi!`Id@vm>LY@hgt ze%^2a{fEi|1Qun$rBWr_+jm6x_DwA~9StN+cXDakb_%~^sAOwW23~i6FPZI6eB5@flJ=kZ48fH!W4=)RNJyly2DV_p%kIaFEKW2eZe>NZ`&&Sw7+3 z7TTGXhGBOeg3*Cxcb)8Qy{~<-XakEw8(N`hN4aFdlmz@WaVYp!Xo@x;Pp7V9 zzM!w=EcU_5K>RqSoD{Ww5o$EXkbAXPh|41zPO@|td{@l`_icOOefL$`J+BlqITL!e z#)w#seg*ZPN`&qU*5SqznRv5bW~cIX#-`Bm7*YTI5NPhb(TxL0!4_x+w zT}SmgaGiLbg}i#8pj z@toZT)X+VH=R&9A&FZ6=ZYR^D8Kl$f$@*mN$5-T)Y7wS4ETn_&DE&?H;BZYO+;Lh) zv^>_ssV9HTZJ zn{hJgadxEZnKv$IPs7R8%iz@My|}PE8>iXp;+~I_@#8yH_&U~^KC37N@9S6KUejSo ze@7$)D4fI#>}~R?;xJri{i*lTucTeyLh|RVnur~}QMhJKIF1;cjKxjblG+u?h{J2oF-lXNr#tFFR42 zlLB~ng`)iLMbJCSQ96I`P&i>R8#)Vmu%p=kmd&dcs)pFXLc3g{$Ja67x#v3h@@72F zFdYwmmToBcp(VXAdl`{au*H=fZumgy69%lhi+WpYK+eep^Uw8D?d;K_&<7y&GU*4w z*6Z+K)(-5dGKXRInHaKW6Y-vLj&^&@z^vP|QPrjzY^0i2q82en@)qGfIp6l^^%gF!2bHfFEEpYCz&PR|4Qc5Vcw+M9vJ zm^`94?u?{piV1!{JOqE;_<*zQb>NBG6>5;=PPC7YfHP)~QKFQC40jWR!VxrZ)+w@f zyDgpJ`JY6~(Hm?2%)+ig-VnW%l6`5zh{5*tp!R+y3XU%&fo}kobt*~c$8SgZp-E8a zyb6X~@r9{_pGl^*P)VsnG^DS2N;aZ?_ZAJr!@r3v*`fl8zbB(Wx)J*?&xNop7a-;K4(Kg7 z3X5*=AmbXKt5G!GKCX@r)ix2^TQ^|P)fs3Ia)e%bc^S?6(rMzObU3;)gM`1;!Hu1l zsqUtqH z|Aq@D$QT7S9w_zuOebn7L4{TmnOidhySxq&L5_~7dE_}5v}_0(b4%e@vLQaGN`&0q z4<&oAa`^O=k|;GsKuU&<#5%X_D804~t-34FN$U^MJk&{Vt`w8n2wUOF%xM@t>Nmev zpbql8+n|?|F?BDslIw?-;q^pCn75@s_WgFGZEX@n2UG*srU{vcj^X#7!DwUc1M_GM zEP14h@vFu7Q@S0WJxM2F1=k^~W-a)aKEnr%zF4wwFzord06ZKv(uhq3!hlUY{e%vX zJnp8@CSRskZNCE;87H*r8%`(Aa=-*Rf1Dqyj9=c@&_c_7G)*)go-B(4J>MFfuX-F7 z5^WsiP>gZEj?vK%EI@18QyCwzSu%LVP05vSs%WX>*N*`>RPd>j|5!CXo6s696BN2bUW=A$&~(eCkUkj(;lgTe_IqM0$|=*d!U#Gg`*u zEha~FyD?LsLFX=ULibA}q^V6)K+`OeJdO;2DLV^kd~`EJuSmhlj(8ZFoJy14YSP%a zla!UcySV;92x~ima-AGKl6fEAmhU2E^H$(@I~nK8y$p+n+@zUW@8Q>$}w_gi;7tMfW8;t3crq{xa_WE?q?+Y09 zMn`mWXALo}YNs)tS@2{=85$_b?wu}vTS$zL=QYmTT$b&MX;(|=W|Zfh~A*j5NvrtPD7rRp+Av^Ir0|kCNH0J(8IN;!$I1 z3YZU1gfEG;!f2Ij-23Sgb_i2I(eM_%KPt%F^r>Ixx zJID?=NSkL2lTKZ2Pp$}!uzHjOc4yj3A0Imb&p&(+n*51@!#YQB?ecVRyM2UAyEm4) zY3qs>IB7y$L<8I#orhALCag=&C-X-B6Q*Xr#`*?T)O7Bl?Gduv^Ix*xV*98;&2v1x z<^WvoZG-vCzSAwXvfR_im-uPaI#la81;cL}L(iC2^3aKaC)an$j`cV;8-hUOM zWPIw9x@@>{eKJ&4X#@NDDowN7N2l~I!WjowkY^5Vu-?8A$2Tv)FwGm(uHp{5)vQF- zvGYh}k1|Gx6R|q!C!94ufMKiJ$%K>1(0oB#RB!!RXno}{)mD5?y80gBwc+Z_rICkF zS?3Mi*7lC=0mV&^1n>6B~RpkQvK#N?g@m`RmH`{u4k zqpEXMcrpwZeKN(=79MuIxdriC?m~#!5NeI*AZfM@e3|}&rUcIckvRj7lLA@s6Cbo2 zewmg&FQQv->PUOX8`A^(mVhRc%Y_aTNHC)m$_WcN) znmrlxOb%1uR2B@^-GGQ?bzn67KC=5|qmY*uNMQW~;eh%VE&qa=3qS0%-j4M{oWT zewbh_+P_d6j9)a6D>F(!`@b28rv^d#$0%w#-487)lO>~Xo+sK~XFya`B-G% z%!c`so`R~`GvTJnYmyINS@^T#2pzst3HyI6UP$C5w7RHaF>k|UOrch<;r5R@jW&+Hw@XRbNT;WwruYUso&~zw#KH(aYh;T6y+rR1dYz`$3qd7M$r^f(7a?@b$Q2 z=vmt3)w zY03{MkY$&IVFB4{q2rUKi*241t0D194A-7Ht}L zf(BO`avDR7VOd`%u`wJ2L6KKUNgxG(_eA>3bUyv4y$*hK*kSzH$0%($Bvdv^Aj6_R z3c~|m2<>lG-dCkM>D%dc_CwYJi7iu1POw-_VEl&M0W~Q%9c_lOVz@m4>c11(T0OWW%E|Xd=i) z9e=`i{rO41PpZK7v;RR)ej?3MK7l_fRPfIr5wTi57dz$(K>W!W*9Om&WnJ$k-M?HQ z&1Q{kM=}y3)U`0>(jFW`f6zBNYH%YpN#YNj@K!}5IkojFKAn98wyyg}i;F90iHrw7 zzN3P~TjfH_L`CW4<`LlLs*4)!vU+EcDjw=MM)#$(kVYK& zER`_s$A`gluWfY10SjDl^a7dgIzaU7WG`7|9fq|XC-JUx4L#VG3D=T4P+m0=Ll<;F z$ML3Ha-1mVkr1ME4Xel7RiE(u6MVSXV(-K$dC8PP}qXZq4Bw9x<(U2W$anqPudZgS8&-pLFjotPn zW2Gw$*}oK=3hPN(d%rNu`Wc+il5J+!%KXvCK`P&S7b?euQCpV+7`$Q@T3H@IGs84^ zbjbz}4i;d)cnK;smlGqYE2zyXg7)EyaF<{WKHhYhjI%GL-RrA`D+1LqxwIa&i%c-^ z{%*X#+7@GnNAP>Xw!oBG2VrC0S4psJE>+Wd6Z+rs;3TqyzHK9gl~ej;n!D4ugFhoo z+d2x?-&CbA)tP##{38DEwV-&Vh;BM>iP|4r@kxdX{MTYdF0MKU-{0M#k_($*!L5tb zZ%{v7nsb5*=Qo1;mQMh-0WRIZ;K0Gh@ags#^6;xG=^S_z$JYjk zZqFNslip{78utWMlhQ%s(to(%z-$@k_8h)HX+&F9r=u@F6|?)y*mGsU<>e`2N<-~8e|57WBJb9Hgw(6s=+9X^v=PNPT4|Kq23o@i6P69_2rA$4d zW^{%m)anp5kRJ^-;vu4CZh^S}Q7gGn6;ADpHll0L2V9qX0FwT8Gl#X(pjSgym)uE! z_1BkzW`rqPs>zA+eLwLM{tHP??;$(F%6Q?%SeW-~9UT2WorJbqVeQax%$_NQl%WPV zKIsq<%W8FnwF=V6@%iw#wiHY=zGLa!SX#8CPFU&PLbiHT5uMprF}Oz$8xKu`@!`4n z^+N(I)%z+DE{wpR7wXA&O&3(tyG=Z7vf=j0QS|sziaIMiq;s@Q$)E3z!m8}Ubjn{b zEYF_?uLbWU`87fu_Tvvmc%@*&zYO?WGDg}maj+<(dJ*AO;^FRU7gF#j2j)8;fu~J+ zxO!uaBdfejn+VvBINPhCxX7bTq$nnx1vOjAyE3nGY8a z0kwTfPc89->%lw(xu9g;n=3H?)fJq*@t@?b)jq14{Fk_?6v0uMmtyi+Sba%o$Y-uaX6l{7>~@cxe%Azwii>W6#p(bSFN#;0p!o z&H)#eg6oIBpt=fD(xGvhtaF?v;liEq+p9KGJg|_Yv?!qZ_&6kaXR#zPoouN~z`)xv zu-mquI==-xv3)AN@4ANc2y{`q+k>{Lox|ZHJE7?0U`(7V5H*pRbexZi#QMinh&rEt>jay~}@ z@xPO(RKEdrt4e8Y`#X@=WpRLQ2^qO&5hSR#fYWh*;W)S9a6?0dc*Rp$JDB;3m>&byL3t(Bjp)}j? z8cKG`W}+e1blVFH`k(CX&f6)W;~r0DI}O*+);SrVPAAg#pN|O>FhJ^-T22!d#p1Ko zeKc%lrtE(LsncFfsr)-d2)}<18fpzeKcW`O8z!RJ7$xa2hkUYpR~R(5xk2Eq2D&ie zB~~o@1P`B&g-MV0;-p0>s3!A>k4$yKFFn`6`hg929`?hgx85i>B$;*Zw*Z$QSGtZ2 z#+`Xb2z7c->S85$dD$n)hXWH~@~aZI)+Q14?hP09bZo)QrT@{WH{&ohY8PZi<_ICg znP}^V!q;YXUPtEX%nvq3gZ_IY=};>f{(CFUG;~MvoawM_=`Ql4uYv>*y-n8ajKoh@ z`^l!4l^EAOn##LZLb~HcY_NYM<3+SdeDA5Fb=&}~=szpV4%i7_)~v)o4lgBVx*p4( zZ$F0m%8CA{-4kY?RF>Xc@sAwuj(`L~GrhFb4jYFpmGr|P*tlsjc#f49ttB=@qbmn~ zJV~T-YZahcHybV9^`W26BT}h8h4>CQN{5?FqT(G*aIdI^q{)0tRzHg9=*}M4)!rdPm4wR7m19Dh3c?S#;I7y$c@x&M%z~1|<{8MbRD_ z^(`J&IoILz25X#I=m2SE@u;(Ns&LlMLR^p+PG+TkCQH`qL8V(Dg!ZR_nji)pe7wok zoyYL%J2`1~uO}(bJB3l^S8>$lNSxm)ggWg#xOY<)3DK72dK@so^(!^WI$`vKu(gT6H#GBz4V zw%7yx?>-zcl$}Anr|{=ODefF(M$QeIEaOINi9(_VNcb5}kgZI>GbID&2V3IT#j<&M zU@2NSs6zB$ClA7>*M-s^6BM4ahv&jXl4ai}o7ev#*921~t_^+k zamHF`Dr?2cB`-jF`wx-SHj%gQcy{f{`&hl<8(r}HF&%vE8%@7-0iz!l(rq`F;>4-x zbfV8{bUyTf=qNvcEF)D~S+|Gkmo6s$)D?!w*xXvoc?>B^B(Ez0LXADK#W@$ZCVYnU zQ(0itE02emT3lo6hji6mTpn(Qa&Gs~_YvYI-j!~cwT!Nyam%UEt74*j z2DZ)MwH%zRhk zU-5*hxlE#rj~qPK=>hxj7??NY5P7jj1%42jztm%dK4lM)BQt%5CCYXtU0dHzcn{jz~RFZqreKZc;Gco&Yp6Q$RyVA+(h3WL~)`K+gxtyrLU>h9s(G!}-oq&NScY)DpQ(SeY zoXmLt4~N-4fh~4bIKbf*>pOTqH1H2GNcR*J8y_PPU;*7)&%ors4B996gBtR>dOk7nun_tEPNfbni*WpTb5S@>p~pSPppoqP<#y|kXJ;D_t`?HYdnsU- zwg>`RO+n3f6E3bD1kX>d#$fwX7`?|G#50f34`WM&r6d}@%O^;pn!>RwqZ-??Du_z* z8R8aL3@?7DCU;L zLTINWxxL#Cyo=wWR$ee{3glpp^aA;OEC$*On)#Akdm*ErF*i#@0^NMVRI#}-{~O~*3Y9(etkmAO=(aQXhgoNP@J~-Gr0W9!LsIX zl$&rLitl8Tr{Bu)QIv#acU_cx*(&2X*~CHXL=l*HEEVp)vjA%U38H-vhB zC09S{OT}G(F;tejhLSXNiF`tTPB-PJUp`1{ePq7U-ECxN?@oLr9VqfG_(|AkpelKfhoT7KJuMfpSE+UEzfwlNh)2X6bYc2W#29eWw1S$DWA z`vo=KihF{*cqBIX)3L?*aEx11z>4`HP!p>fE8j}$gp1|S>d_@V{>mn z|H&RmGCwDg&;7&uMcu^Q*CVM?UkYgUCgIT14sw5Q3EfsOSt##RMNaHu;NQp{5SDL< zr(z3H^kom(f1!v8xu!4q9L!3q<<>y;$OP#2^oG$9QpqYab4dGFCE0r44x*|~k}fS{ z+<&MD7c@2F&R^3d=JRr?Lfd#c!^Hzn$HahT@Gv@Z-BToe*C05h3Sy6B5uL zSy=a*gv2%AF-axFm4}j=7mMMkEX!``jZt_m_K)yY?P*xCXguz|ww%N)T7`S(>Z3LL znpm7If`k|gT=g>=rLs)TO_P#Iz-?{feasa4oDQSI!(_OVVT8-I5=fF=6B&I+wi~a} zgB5S0X!Y5JaC4&p)W)0vtrHSD&q~${pOu+e_Q^824jSQ@iv;uHMq{Uoo+SPII53&g zjH4{p!ngBbxMlnXqS!DM7sf9nx-<5|(*_erdgY95A&1apA&X_c=V|`8yQDbJjPAIW zF8n0xa}r)3rWgMWflrT>aEo3#ZHiJ9jkpmFO$NUBVyF{6&AVa`?t*rMEPAR;9iRQ~ zki@eEcz5$?)c&b0dQ>|`boA6I`eF}9Q@WCwtXQzaKz9fanP1-;Xs91o%y*EA{qbmyF z+(GwtDcbrR#|<{_;8Br*UPfc!RhJ{an;eW~-~7>iW)2j16k}zb6K-L~NcQPx61|AY zV0o?)Q%~xmgIF zar#8OkkSg#^BROR2E8EV2gl*U7zXbq?IZ`6mV$Ww0Ghj|T3Fc2(*D=iB~M(R(sSRo zqu;P5+T5-vdR-q%UIbqu8`ONj=Tr?1U-6SpIg*ABq7>+LDg;J(KkZJuFNqSJAkO7C zg`2wcN#~4{{nces-j%{l^zcZ!}N-m7`)d& zG{HO=beH}kwSF66*4|mTtT2{D{gbEj)4E`T3t*JnVGMb0N9T;|qpC)#L|nfd438Xw zs+J&#OjL%P;7IJBe-JOf7$th%{h4&d?!l>S2nidx7rdK*+%#2?E?uNdgG)A|qh302 z<5{}>wi#(vUx1ww3O`;muu}airl|NqYh*fpZCeKm5+)%v`$+vu)o6L{73%8q0K)&| z5#duA7ijNbSUdkcj&JD1#nodVOv43KG9qcqOoXY9b?~I15Ho_LXfJ<|Owia) zO4luesSD3iAtNtcKg1fwk6@wK?k8Ff3Wk-}vWfPDTGH260*_Bvf-1&fj3N(*>@Lux zKm~a5tQB{h-wzw7kHQI4`ygh?aT2d(3gr(ILGF%*@Iqx3dFwp`??23j&ZK;L;hq*L z-?0)hFKmR4ZL?t7)oWyPn-%(*)xeRi54d*KB3R>UZ_UAi^z2j~eDcg7XyxRgFm6OP&oCP@T`!V4{BUvtZy(tts$@DrZv!Fj% zQ<^9?$KJ7<=qZyH7^pJte+->@Sj=A(#w%1pCCb(!B3dMonmhLlk|j%vh>AiYC5lQC z?fYJ|qok0sFU_5MCM{ASOSUZ8m+V_VLVok_%=3JodFJ`vd+xdCeLt^M{H=`7S*!}; z(tmW6+!emLa~qvha|Xt`d=*-pROR^!i8%am9LPN$Li$|orDNTq@tyq_INr>0hJpt? zuusD?Z^p~|(c8TE;1FzfohDj4|1l5ivzfOpy8zLqudqB=0u}`WL^~ra;Y-hV;Z%zy zkOD*SroaMIPnE(?gKszM*HKF5Q`ijH~jv}dYIaX*qzg`YVv%G7)as*z$_S7W|*f9jNNx4@|CAV$trc z^m&;Lbr<-9R>OO`2-E3KjWZzc@R`PbTa*y&= zu=C7fIQ**umUXFu|4c2>`B*t{icSNQH9ELb#`&CJHlO}@&G6Ia&pclKI*+`|V79yw z-4P#3>o1z&V=D#8ujl>v7{f@ojn`Ga&~cZo zsMD@!sI0!j6C#@F=u*V-auKk_JBZgVe@=~cEu@LQ<0L+3@_CC^3x4Wc3C|yE!MA~Z zap27wdfvPb3ja!J?~MdHG;bK0mRJQxonyIIQ49p07y;kxl2G8A&5t*oqQ%Cy`7fOy z{&i6Rp4J*B4bbQ-Ro+2G>*G&Lrfw9`?7Qo5e?bp?xY!SeYK@i1a&Gyjh1QsvU&p!Z zerCv{!*F`o9pMtW@8J5hKQD6g1@YG~{#TYE>3QZLuPYXC)lr7v7+fOET|Ep$E>odM zqnytDuZlabdCj-`=0N-FY(!UIu#9%arzttuq_Q1nOuPogkAh%|lR5OH&x7wPi$KTY zBoBdC@b#A+jQIMM&N^BIXTuxd=cNo_lhVF> z!8fUqk2HIXmUmKMVEaR1$h{fZWf&V3Fg6mBMd3Ct(c#hAoq{ zq=qBM(<8TAC6#M0;osSHST46o#y0DMp=BL7;;SEKogV>_hF{=Jx+=(*dP3d@J$xlw zk7@4n;I;Z@YWnaD&JEfFK^Oe^W}!7~(;F@d%8JA4zr(@k*hZPQd@aa5-wlJWPDZ(@ z_py&lH;oZqz^LU1`0D#gqS+h%;|63rI_-6W&y5N2dP69-ZqgB|HFZm#zDtF>wN*G| z<1rkyHW}U4sqo+-?y$6xf3Ug=y87CGwF)n8uP`e-0F#HeG_u%oDT z`X^n+t#HLy4jbA6;B|KrjCZ_-*Rs0d#ikiT9k)z!i!{#(|Fc-d`}an0o4`!W z+2c&6pb|`5I|cOC-Q~g79M&jah3N7KzT;0GwOTxu9-H-wF6cB9J-q1y(JF^!Q;|oK zjo;VEYRzf7{=jb<=9-JU?N*50wHBjJNgZ6W>N*U3OrCLsL~dzpeIKxO(#&>J(52`XL&i zJbffxMV<(=*ZIKMHOFA)&NtvW%@{Qo=)shz5XkGb#wxWxJg!^>&s06&SV4?rg~lm- zb~+MH4Yb3h%KOl~WDm4$D4@*~OSnczBZRN`0akYT{A%)6ZtW5bm;YG9?j;(~wJ8D$ zPa9*6t3HeOvxFh)Hc*o0gcCY4ahtP^FzfjkQCoRGsl|#a8K>|IxQsdsm$ky-jcyu- z-&_y6FKqF}&Nv7bo#2@xCh~5Vp%RC>@z5qQ6CD{9k7F84q*$~b@6p#FwXKFt5gX`p zna)7{`e8oTu@!dSxsGcys`v!?3DW-ea>%1G)99TmI&d#Pp2pW6z|syqy7J*X9w3~A zX@z$9w^>d!>&jC|m{1QZ&4gW~(pTMQ~IM^DE zMvu;N2s|V63$ez0B&_Np!Gd`LRTpfk|R$9xfT%Uw0&vpr)bZfxg!42@>aepaE$)Xz@FX5vZ_pwAlo4ek~ zgdKy`Y3IE<-YR>>!i+hn64r$EUAovjV+{EC=z`yeEEsY1FF!Q-B)-y*0<%*Cc)Qbf znE(Ad{mDX4a77Ed40f;+zK$Gn%-uuSH)Q5t>#M~=0|J6lwQ zidzGDOHc&F+7!dZk_uE9IUGao4MORQ36M7I3UhL}EqQt7Ke~Ufj87&uh0V7(k7EaM zRL~d*KF}8)pNPXVSL%fCWIbgfq(Xv+B`g^Co=OFKNngJyczUKYOmhmsj>0B7e$`Xq zwWRy7;8!=lTi1jsD^%&W3vwb)`5PdN_Lk_Mx8a!&vuT}~GF%CbhqQWKc&3*NN=44J zbhicGIH@dpc;^&7v^kMGUzBNQ@8p1V)-pceo2`r${0#2jwd6NDf?)mODClb^fsIaA z@PYesXuEul{<|Z)Qwht4P|JPz!)i5t@tFxjjvm7y+ZFkurr~(n^bZ~zstuy&4H)pY z6h61w^CF`Hy6bW%MBbT&G)R#ybsj~RY}E&cS7&k8;t*PX!5lPJB99r|Lc;P{}bvr1F+%rT>mclR8R&SKz6eqv75&na;q}K=gLz5y|kKZ*i5zSjgSI2VJ}r z!PLkI`W+9DBp(_J_J1_UYqfe>5I~`|Y$b++l)kw03)|I^U8hM7}jl5MZw#VS~BN27;-$U~Jz4*tj5&ruf1&xyxMbW}c zG%%_ahWV}H0cy7(;S~{GN?-P*qG3@WPLflFn`Kfu;cGJ* z^;bd9R}=Y@;AN=ab&X%J(Wb-KO7WlET0H9b9;Vjaww_S>5l+ro0#0j`!2RGP;iOwS z@OyG9j$5}CUhkfO`j>iWLz9$x{{9LP_ngq>imEif{5>6%w?yWJ^1=k0Ph3B=ivGzD zg5jbw!e9$^NmN1#=8e+jR%KCecgbH2^gl>{2OmR?qmFpvQ#$UBHo%-$p1l6(a5(=X z4R)&<;AV?CP_w*@Kh6uqVKM!9!FGAb_HCm5j(roZPM6i}50U7u>*A#Xbg~cZgE6~y5oG`!)(|Yp-hvO%g&*X z&WoAxC^~+iwM;`_$Mous(0R6h==%f8(z@BLsI)_-qcNESbf6YCeXPc-DVMR+flU966sQO67&m#m=A?`5lx? zG)24bO+Z^oKD_a@!~230@F~8JG~(TGnjO`M+P`yonNAHoSAGEg4qQdGUzK3IpAU7} zb%-lun{jp2mo8p*6vz1WNXjP!($&*4gh9fcV5eWq6aQYv`=3MU(hhZO2~iT>ia3Z% z*VsbOusQH_#9IU-Z{Y*UM^5~&2$No3#dJ-8Cnh>@?oKQ`-~3XzK1>7VhL^+iBQjrB zt~I0tsYrcEB^@IC!$WfwzS{ZYNw3gK4@-BH>mDW9%{ z$5b!kl3t)n`h(z2+d2H0Uxud+SI|h={qDyM2UwtR4!%7(&v(muoF|WkSnx6e_uoDM zq2w;I@+35NYLTG3Hbm&ngV1ts7}>sqI%Vk52?4TP?5HhZwLcJtbh*Jdtx;GuAR9;j z-3e#4+j!`N0Ni#;15f99P>(|&cv6HC_zudU!L`BgsJRtS{+Q3-J}HCd*L~*0=yb{Y%vh){s73Gd-*8diBNCEY#qABoOT9ld%QC=!vdL@r z2(Rb_W9E+S12k zhvSQ(`CJgN4+oxnk8{ReW&fTG!uiFQgj@F>2dCKKsBpWPR^8LX+^+|O7Asc5B%iBv zeZ>_Bo)n5I717XiIZNWcs}G->;UyVgOlQps<|B?_pZ+D1-}7RKVoNlUTAGuuK2zAV1%|9l*O1L= z$Y%Z1vx)YoEb>(^lh8T)*?CbZxwn5GskAR91~byc8)6m;0!A$n436C(s1#U`1M_CF z$UoEBtH;xrdt?TCOHxTyWeRz+C7CE@8Z)bTGX7K-C65E9l9i5b}?b~JuI3HchVHBy&YDw$U;%vt)wh&uFfP!K4-AWTIsB^IF)%%S;fkx z+LACnhiscVheY>FWybEw#NulrX`P-(>YP&uyPC;rgc+=#PddA}TSykw8#1LYrtI+{ z6BaorkA+1>6G=}DnNS@=>>g&Z3n>{SEF_)y?Mx$1YeGoB`w1+5XDsXYI*Q#8y%5)g zo)es3*C^YwS_IqrZ)C->Zp2}_2Z^opAmbFWh~}_#Hbps&)ikECBEe?XUF}UKbb65s zu3kj7|2Ae2wvhw{`jK6KHb6afo=bB*h+WP_%-5Y}c?q-SYCh7`WpUn_N#7!5}S3j|~ePJMQ zOOaTcygOulYLTsAg}74O+m#@$xRW5hsbfMMMoeeXb(5HFff?IdJ)A9>YD9J%8$%iv zjwahrL<{cAIBI=wwuz@IH;HE+v?jD&gAqj|)|zI_UMOS|zBrvt*GyyQyHePDlPJ>a zyp!Fu3}b_jhBD0!Y2>qeGP_BVSlbU-26)dFvOH=lE1d1etlWHMpO0gW!$*_7v&R#g zVfy6L(=|-1%9V&L+=+pkJCR>zz&8A`Bv;NDk%`(S$x&07 zWZ6Wr%261RjJ8a%V+5ma|EAF6`xgLsD{L1Usm$#Y&8{+1g!$7#NQuD{g3! zJ<~@JYi)V!6H7D231hUx&JTx*_0>EWbKOc7zu!o%-SZ{NnSn(0g9j^Gu%10>@n+La zyTqfu{uRUqDUjPb3S_cD0vUZSmaPyavYHdIY+(9crc#zk?&zhF2k+8|&+pUXmeKi!2(~M{GK=RXjw|R(x~(X0iX3 zg~YyaGP|MV#J080W;g35v#JA5Buo`Z|2?+kndgJWow_k(-tugw z{W6I~{FlPm^)wQ{B%kp?87yUJChODBfv7y!VNZO2iWfU;vS=GWvi|x`_IslryC@B0 z_|u%6@l|09MRIJeXFq1A*(Q*f>9G-$2eD_B8f@|WT-L2qNbXN>CZ;^%Q^mQza(7hHR^%Eh%3OL`QiJx!b;)h-w{KK#VOj z)Lg{Av=<6g8g$wBI7K!$Vlev_uR}bGUD$YY2R3Pq6Kh_4O5mtHgv}fNP<-sa-{Kp= z)q-Hx$!zPppJKGsU=GKf*vW=y;`-c=WbN2V68dJ6=C8#pO_;}eALX-?re_7;9_We> zxHnsm7?x2pZ{}oi@aFB-#(%a7issD`98xl8S2RP(f|sFWx_=;XKfjbnO+(qMR$1QY z_N`2Q$78Ya6$6sqcP>foZ%#^ub4a7y7Pj@E2ivy9haJ&eLf%OtSZA+4D+>r^e-%Z< zwsbw4x7VFn8@jXeddY0j!M$X{i)=#eGszPZiMU(*Rj|zGvY@-=tDxoHd9i=cIl-dj z)qG#<;X)Siy`RZCg0e`jN-DV(m_h~~_7_P1=?ex~ z%@inW83|T?^=1i+{!C_}vE>9w}mV&IPP? zpYC}T>+|AMhFRi0O*!JYMpg1{hBoVH)@3dqC$g+)1GasY3n`DAMRs1EN4l92yY*ct z(0xBu;CaMCkTfEZJ+00r$`&akd3P!)trQ6I%_dqum*@+83jD3zvOL(Q6KhFag9q8W z(UtV}D6oc#se<>pLcs_Mf|EfT*n=f2h+cpjQG4l1M&3$im-5nyMRyW0XhYtE zQ)Y-~7OfUH4M`Di8XQf;)7LY*j&GG} znC5~ccFSNdds~rD9u@8(pJ5M?H;o~a2L#D#c?etF6~qj`Ul0_X{v@9KvrGJ`tV=v) zk{vnvOP@{rpu=uWFk$93ZfuL@dZO95j@;?*MgHyZBb^J@vB{w;8O(HHa)t8+`x@iL zq~?^kYuj=0hn7vO|0FN+GtrBDs$NTuKe#K{7cb*d4Sg&wn)q4##-p_6#cO>*b@hB} zg{=M77dA(;t{ve-tiO}E_=J-9Y3yNBok!>^QFpRgA?f(ndIhsI3|6*h; zvMA=iD2B~h=*)({9!a*pA48g->XQ{_%h|RAL1gCjZ6xJI0C_XbmC!M&%*j!oZS2%! zWzADbx#eWGBEpb){hY}5i5!^e$E9S`S{Ksy&l2+CViB7+s({Q~olBPN%_R@^BoV=e zFm`5R5L5fUovr>dlX$6nv2|s>Y(>;sHsVZ`V7^O|wG!*7x%PQaP3FH$B7ZT1iB;2> z`qnfSoe)jFOpRvt+ap-4D2zePBx13!Qfw)`A$G8pXI*N^>~Kc}X}=dpteXSKrblip z)oK>e9YV>^0U~m`EuL+WMw3~~B8k!DNYZ4nh)uUKBC}?UCGWSH5(C-yD~qGZsh9}z zBzQMTU!+Ce9vI6ujT^yCzw5C)t60`73?XDf012}UB(JNcv;G}M#QnZGnLXWzgdNLa z;jWov(4kba@M9`j)G(Ip-LA@xW)5aQ=4r6c%hnUAK@c0V%$x0+?ZcL230SSU4%szv z8mSN)5_Ney);we*(Ou$8{u!?$yJGyw_1qYy_h1*}g*(~%QJdMr_GqGZK8z^64ks6f zAF^Jgl_U0wKQG>NFIDWUJe_z}3z_?S%El^Mv1N)$WQuYW3;w>FEpv%rhjd-Zu0BF` zv(<)`EVE_4^VNxpix!iV$g$o44fgI$96NDoH~FdS(*i8R_dvi_P5%*@fA$(>OnJus5ZIHAGzw8-*tW%a}1 z)>LMdX~$kauw_GX{C;Y~9&X&*j86&yjpm zvM1huBgwdxfvo7wPSz3{&Ma%fnNGS7@eB_ld*%g@*Uy)bF~#mI>xL^Us9wnC(o_>W$5b}r>Ljv1$(&SmnvgAdF>LUM-Q>~BXrj0|QpUklCa!-5G4+LN?Acir_9h{Q z=$eJIDY86crTl2-(dJ7A1q886;Lp4Sf$Tl36qHZ4vp%UUSJScMjdhpfHLIyPm#kO& zS6;aF?Y7n1TS^F;#b{ATMFx0D5we?Jv3H>?!wPWmSJIQgj{SA7l9G_+xl9=Nhk zf$r?g+X$8!l1ywf63C)Kv1IW-BeK^*#;v?$!oHs{Wb4Q0ipw1ai=Qg)tWnn6Uvs(k zmbfbCo#0LC7eQD{vtZhynS$B&3&n?~Ws4OB2gS?Godpl%-iz~Mmx<$S=87}!T9C4- zI;>#Sc$RHu&hqB?k)>}4>zU=n>}po9ZHeKe%P(`I8L zRGGsYBk_~(F#_)aMFJ^C2qZSY#T}_tg2Zv}1%LlE2%dQvvJ{XCCM`82#r8@>_0@J( z&0LA%=glP6!jEh^cT=o9pbxQsBu55x{uX>5wTvCOJB_f)B}Av*iahJPkqn)`fQ232 z#HOrX!``%Ax2hYrsD|3Q)wsQ<)~9adkc##U7Q8Q$`G%yjZ{}t!a+50Q`0GZ(HaHMH z-5cV-+9-k5@(w|x_?SS^s!_20xLAB{_dc;_$bIpv0faQ3U&EA4=CK7w9a*);JT|2} zj%eO?B_;irktkIQcEo)ex%f;(>Ut-VEjM0?O-I}i#43CcZ2$Q}pfJW!ut}vr{B%-< zSj|6Iyk2Q!NyVw8H&Fk5| zhGa6MDV9CHAJ6qLmPCBml&mo|WY0Uz*ihApEdEj?n{B*}NcIPklGH#lWWB6b{7@yM zO;?eWvUOy2w<2R7|tq%w!SQB7r`Dn7*fS2&sVayN-fbQk0pzO%|sGZ6gs z6kiy3v{2AF|H#vhg?>TXM?PhS^3NamR_I7rr;{s&L9BRWBgBe$R4>2nbJo;iY_VitkD zw3^$@&yXl9JD}%A9kd&Bkk9J;!^c0DlRkJF0LNw6(Bo@Xf~Mkcn` zS$q=j8z-Y$br?>WU=G@uHF#S6Aw43?E_VFf3qqZ#xO9p$*qLO*JSS~j%kSZj4F@4p zr4l1+$I&F; ziz=dgtJ}0%Gl!<%HO5l~${2WT5ZJzv<+0|Nqwg|`;&*$n&!y+W!u?(R%DNQcgkmkQ zPMd-@UIH9>vzyA*p5)=~YV@TOdDk))zN5!f%iUO=p%bcf33GX z;CFvmcKjV(y)ji{8q>x#d#ym}=4pUHZLYCL2#EtK$a1JPy0>k?4;gXLcWn?R=MKWaxXHLXbT!Id3!o=12GF}p^Z53Ieb8h{ zDa`cKkl?v5d{gT{+SFMEyZhJDBSV8>xPumS#O>mrCVr6d4Rg7T|033|jhf!TsI-s3%-swX~#(m5;9WM2axzibs=y)X!BZIXEHBuB^}S%OMsLuEO} zm*M`40JQIWp4#*^;Zm;hnWR)R)ZLZ;eUs^%qgVV6Ah2Y~;t&&g8QFuX>V1uav z-p+q3?4Lm;y*250>HQ&m_g^CnlCi0>R!4Hr)>QoK#U(QYwzzebn&?<<3Exw1iJ!J> z^2`Cn`W_F^Wi{BJEfxLV_`6FOiYHJ68edo5_}O{6|{{&ZFkfzr($vOKf9 z{LS|))HY0qn(SCOEXW3(n|9o(%77kGw}2U4%^0({44S6b3op!`B>G}M9&T=z$3;tz zqqml=aOv%%=ozApjX|a0@iP&3PtpZFvoqN9eFIfFYD|@mXON$fcD&Lo3#VCnas3~r zc>UNFnlm|(!=S&yaUYE_cY!*!9ea>FYntLawSPE#a~vPHu$y1KqR8BSMZvq%&mraK z46Z*wjB5(|qat^Ad&qS|Tw0atUU6)(hFWGHR}q zAyLmb4V9V0A^Vb)zcJ2*>pnj8&m0164?suc7>icABS^Cc*(RX zWoW)8gOLZxEHC*M-_hl4A&iO&(d=+qD&_>8? zw&7Og)%06cJPf#NkGd;cX!Ry{7&kc%Gghp{sJ3V{dh>`^+yWTt9ZfTXwnO);Z}d*X zRetJ59BNiX1?boffdPc$!Hu@4KERX5U+;z-}#^tzn4BynS}9in_>FT z`+WK#WznUE6F71FDF`dsOeco##v5b1uvsieGXpncLh*4N)Ha;H&yW+D?2E;VE3ySW zGv-L*&E{g~m?Cceeh;1a{RX;UD?*JeZSdtv6wbd}0sl!4;`=!%m{1>$=9wSi?!)&u zqNXn#PMiX}bw;4!g*Eu_qq@*(?@jo%bTxm>U*loNLagetwoQ3-Y5gxW0%5)_Hc*@nTofvYheC~IHqQ)ifeB4!D%=A;Oejp%qdK!g1g5s>}Ufo z|9t~BBR}v-;}1ggPgg8Cyc*|!7r~zk74X4mA`T0--}LwLyL?;9Het{(HQ@z#Dj9DR4|OVju=GG9ANcAx zkKD?IM(;go=ESo?pR`ykaLM4tW7PQLaT~xVTpMPEjDxP>@>re!LDFD96D%L}5nb&t zf!Xi+icbd*MHByc$kly^#dD=x^H(ZeXs-wB_ns9t?YKab>1Grznuki6bD>fp6&B2% z0)j^Z@y>Rk%nxRTmt-^h+{16tE9N&}ka`B~^`7#X&s$IsC%#X^fb8+kKMKD+1hS8C$JqKDH-FTvJ?nevWNWMj4w9 z8ypI`zjGdlrN#7#O|DQQa5~QXF9$0BOoo3U_jyB|6coFx;mgNlUNUh7KWy@ZPT2eo zx|0oIR_0Nzs?VX`Duz$2%;gbwSK++YXow9r!RagWK=+IabO(k(q(UCNyOF}pI&O14 zmtpYn;b+OTLJCjjDsxXIdpy6Uo(H~)lg-s-eBnhGAmn~COpTMV#6>5eTk{5g9%4dW zw0!BqA2OzIau{w6JIP;eFNFo?t_e-eW@3(L4J=-|3fYkna7%6?ciSq5J4^4v6GKBh z7^wZ)c>FAaJLqYsV6?`8#f`I`!Ccs~j3y$oPb_ey;I zRu-`pUy1xJz~{OS>bL$aEfAOSvN7f&YOMy_=EURO`gEL?TUg&roEUil)t_JR;5-6KbYGzw@qj?3Q%+`)azcEAc*=2KDI?WKq4n2u>$CW@iX$?&3--nN_PsM9KxA_LY0bqZ2 zf+!#-8s8=!2Fv$*;NF7~z?7o!iH0_QuMWV7(02HEtQZtlZ2}{sGoUZdrbd|xaOI9X zuIqmd?{v$wZ;p`?GPn{}Sj3~J{%zb5q=e7T>Pg@KHyl$<HE zXjWN_-PQTr%Dw{oZW{n9JElQVa{*g+))gOb`3?5z^4OL$0~BQ%&~B*4y@5mFNz5Di z_rNO2sOQV!g6&1#*&K`C{hz{cw{dvp%P;EJe4TH*stb`VfzYQAae`*Kj5GO8l0LMT z#^||$PHj78Pq0P}<-Ta;s|sx+A7J%_48F-|26`9C^3>;y!#s-z^!wWgT5X?-XaCH@ z+AV*v@9h^j%B~D@KE{K0Xc;V=)d>!J8+c{!LFl?9Lg%8FU_HMap6z#`ij$=<`|KIG zzjYB_)678j>Lo-5&xeKOIl|42YVc|3Fg9eBE-o7Q9c*7W!+h_V;HSO~*XK0A(V2vM zyxj13BPQ$irV{3qANZpfvNT~{5XK)*G(^kolzrs-qL7pFTa9XY;fV7 zY%e`6mw@tbwYf>48y_^@7g9%mDGP5kMdd{o#@iBds{Fu{8@-%(F_bp-DB0VVSEaUwOr^5JLKb~>r z3k`65NnM}qz;p{!9K0ru{+E20Z~1r#j$fMzZ`bB<{l%p?W>p1@S^61lcYTp$|2f2$ z{5Hkprvd0)Q30oY^k{fv1+1H2Akzy==+|#+aBS5snMX99uGF5vM;!mc|LJ_@%X5nG z&74wx`bz;F7N7&mmuBF-Pdl-5bUbI8t^Dpa3rsdqMzet&>yx6;XZ$h9T-68ee|*Yb zu212+-G6|4;4&OPq91ClS&CB~n`lOhDNbl?gst_Th1)K<2#MuT(WX-q!Dib-P_MH^ zTMrd1Elz};?u#-1*CbTETrZq{DSuVU+!M&YQF`?*q*7ig?Xz#AP!64%6Yc*R_q z;|&e2xv*KH@$xOpoU(&I4U6Y{ZVshIGgH7N`yvF^zlJ4pNf5S6MQS&H3o1NKmuy^k z7|+$Ng2pqQJSi{*$4Q?G%f$=%kJSZ|{l$rR)AAG+ykCf0$IgNaNzIb{hNqJ7KB_oW z%DvR~(m$(@fx)?6_%iD*I&ANUJ5S5u zj>%eRyEYb2O{l{j=OJ{_05uUSXvDQ+o=H}CyJ7w91N4VtCfz1C1ax9Gr5z{Fp~KTk z)L-6HHHXe8l|og0OOc|a9j8vpfcfY8h;lbCMCr6P zq1w!T(*8bEG4!UQG*UxXxcA^fR0@8DnldlmqA~%bbu5AAHATXY+#6>}U&7$JVc5u$ zu|u(o${E%3kX>OgalmApxkmtVh5JDNdm@G;Ef;poEP~gs9q@dd>^Gh+!L!5npoHb( z96Ljv?%hhJ-oDL08s<{@QK$KZ3z;zE$4U5Ey$i10-AXn0d4NXo0vK=7%exNN(b(3V zSlCaWx8JOyRu^`IF6yA;_~{s~&_vq?mvd+chx#y2{88FQU$G)AoOBj0J--an<=?2y zMiC}Q&ZZihlX0BVA^g%$99{Vrj}{n66ilLP3zADL8lX(xt`Y=X4s-Dodkk$2hN9?EFVo=9+8KC4G86xkoRz$bDCD~DEAV=n zCw3ei4Ue)9;);F;_;bxN;iZmf8np2WCP!ycy~Ji{Emxtg{k7mlj}x6@`h|~kjiu9@ z9bn_bNhsjOIJzQ&C+j@p4U@Lv%#+i2?Tan2{O}W$e>D=@p0|PA5hwh&l~9SFJ*<#6 z;1JDR-fE%=>m?}=GpdpLuPvlE`ZNpMQ;kXV>`L^p+J`%iQX0Hb5v?!fp$G1RLo$v> zW8^zHAnV(Oj#RTA-ft*3C0=GJ8?!`2&w5j_-)cgSSU-gIoWnbg$7dSbL?Am#fXk zyN-+Lrixma{(J|j^xDgEP_!VSE=FjEN}_jRvflcxfX+{eLsm2ywu}x1iveQl*0TY< z>je_|v9IB)n*hFfr{LY0^)T|AH}(#_&oloXhbRACqu1>WXv+c_1AY5RNV+>fn(rnj zNnh^|pU-B1nvENe{523WtTmwJk|qyG^?-Q`(_#1wEvb$6Lo^!Q%11?B!TaGdFW=?| zv^xJCoi}ftWN@M>MNtB^?YcsJEWgkKokFN+^v97;G9c&d41RyRF4PQr#OrPT@sHEw zB${6r(AHnu(7N{>{`X0rUwfnpCwJv?L60q@4l}3SFRF3O!7a4tMFX9Ec`Oh691fSC ze-r-dJu0-<83k?4(Qs*IHukAKOMlrK^RvxkB(c5i{O_SzSh8mm{2Hl^At_GOVPy(6 zIGzcfJKA8`kdd;Qz8tHajBsq-A#QxD5Bzk{LMP!4`cGLI(pM>g*<62E-1{8YFS!iz zo603Rcbf5xNjwg&iN+bLy<|*Mv8)~xkUKFpnD9#pm(5Gy9 zONU9FV`lM=XREO8cMIGft%=iLE295)E40mv$5lV(Nkrkg`0wc)zP2*~2KpZ2SGwEr z#D8PJ;JgkLemKLghpS@DsiFMzw+k}w>3IHOY$c|Y-{f^$bEpEH&R=Jk3Fox$)OFq3~rxHa|An6z>mv zOAX_VWt<6wirh3B`(7K?{yu`|+!8T&;dCB){RRFI9-yU~^N=|2q3r>c@UKF~;fvYK z+e`bw>ebbflF3*AYhce*b*R^!>kvQ0WQJLHRtc1$B88}nU13YJ& z)Bj2j^BZHec=+-fVRc&}Jd4S~TOrHPYHGYJCutDG*j4d<7ZSi~Vhc#W+?2$RRFgEx z_=6h3>)`ZofI~S4AXR%IHji$`DG#hrGWjfq^yG5G4|39^5A#Lsqhs;L`%+%is3T)T z#ezpFmuWP+Ny8gAIJ(UN(yW>!lE4H>TGBK0$iK~cT|4ON(l1o0Ax)z6Z4a#Qyw0CI zQ^n8Izth)ah9J7{L{Ik$9I6q^Z?ft5?AaJ@@UaE2Mrok-mjL{FK?j|^Txq|P%90b~ zv}N;%O1v-A@pm~EaT8-%cFEyCe7Kf9cBCd_%bH5b;5ttXTHHhzIgDgBhquA*+e2Yb zMju@48X_1ys8GgZc?aWq6D5)@vWzvADm;FA54O$M6dB~V%h+Txw&J*c((UIzqQd7Y z$s2u2OLJ85S@JTRIb$I{4qq$j7*vev!*76vsT~Xs@xTc)EqIgWMA%g>I}=K_1J~UP zEnfdKbms9?HDMgLQ?~4~FDc5BtY_w)v2Q6_6s1s6iAtOHQld~Ids>i0D#?0hoS{@m zN-2_%_RU+Nw21fqe?RxS9P>QS%zVGUJZ{XUgV*%=EIMR&l%1jefmWwA)>%r;ue+A? z5}8ZiVP|a|NsrLu4Z@9PnoEX@;^Ri7z-ey)|U{ z$Wauq?KB&fvK`6FJ>#xD_n{}dr5xt&D;BW*a#TU#E+=;|k6UvEaf=Q*BBN9{bmq=$ z!OT`moBv2~i-K>mXBJ5y%j?5+IaNtCG9i`2{!h?A#uK#K;}|MFuoyjHW8jS?f@gkd z40ZRpfE23z1wC=fR{qM>nbh6D0_J?B+S}P+{KfX-{o!?c16dDHW z^wZi==5_;z|7xs|#-lL-JN_5#V$6}mcn8~7^N3o!l67cHDPS{SrE@l`gSa2hucI$1 zN@)Hj4cgr$P6s?>k#WOyw8z-UVQ%a{HtJ~k`g{xT-m({V?pHUfWoLpFL@ph~yf->jREz;##74}tw_UC{Ii2W0r@7MeDi z%66E%WrNUT>#L~5))IaE?-W{8zJo?uWz(e}Wl(%t zJ2Z%{pt~m~(Kew|?85B3H1=)+)wUFN*gUp@y?Q^HItVi7+;bd}_k(xTC&QJ#em94j zMbBaVWYy5TaXBRNbdp1&-Bv18HUo7&59AyYJh@M@mE6Ic#=50T5~#cV8oKDtR4VE9 zgw1~v!3{3p)0X{{IjO5m{et<&sO{Mh`rXY8{R-P9=;JatJ&y!x*Pc!HlE3Wt&R$yN zILu|bv~sVy>Z!{JhjfqVp}Xt0A>3EbRtxrZGr0!pI2_G2YiiJ?*Gkb!nW;!(n;M1P zImqdo5>37mhomjnqU2@g*lOJ?Xe@mXy5qPP$@)=KL{7tNH0%-lzilmo<&{RoNhI>$~WD z=?zZ5=rBqZn_M5c^*ArDWHGlYU9`SR^d#4G?3p zY#L|P&W^yH?5o=m^ng+&63O>MyDnC7Uxou|NWc-Ec=iV>KgF3|mx!R|j}N0CPV2ez z(duaST}?DoM8Ho*5_bKr0Hi9qg!{7m8GHCc9bJW-xT>eQY|Xz~?p0n}UFhe2E>tiF z=o_Td=bmn;`o?p1U(<35as__U!_{2UmISo&&PTd8K@VBYsNf#zmC#(6g7&`*rVS_t zth(GcFW22&DyW%; zhuN(6_bBV?iVR!NqfZ+7toR^DTb=eI#gM;jj{RBWIQ0UZ6gEWFPAMQiCsp)OUWd-T zmCY?^&qEa*=TT&AIFfu3OZDBRp@XaXxe3L4w50C~7nI2&hA7fskHgUCpnT5BYc{(} z;CkOqt{{#4)$GxE0-yb#%XKjlbyP*ej>gZVXfygr?^kc4i#7@zYcBt|T+eyLu*8+k zTQjp>XVeMZw0b~lZ!Dt84d2+M%UtNgRsnxkQifLCa^g+37u5TMHfUGfGHRP%!TJk& zCF@uzhwxEn>Q-C8{t!u{^W;u)UytX|;!D1C$p$mpRw3Ys_xhvgyCaN9(hxefIR+Ij zx`R?XezFgEv*?|?G^+eKl{M+uOb>qD!-?!EMn8uK>9JYeba+&~-g>zxEq8ywMs6vj zTeerxQ-;sDN~aEVyn29*Ja>ur^wuKkwKs>$h}e$)q(z|eBQH>wY8e{o;OL;%8E$d( zaWq}x9BmNaif*{5qR0FB-0s#fb|-&Qy;RF?)M%VSo2_!uhv>s-T-yg_AKgW}vw3Lz zRC=A5sT?cre!K2=Yd8IIG?lw=4UoU`T(navnD%NlvJ?CRG*jX#8!I1!s%*PiRqhrv z_+`wMro2Uwhj5+9CPR9p(E*fr1k>;B+=xaqR4Sbr8+6Nn`hGrvO zzM}{wzEI^=L|)*gUZ_F^W<$Jgp-*(sT#+aL&WEc`t7Wxqk0P(BZWK3eq`?k)NayV! z+_oizesIm^oePn0_^oV6MQ0pEB06U|nKNqCEB7Eh8B$c28YqUEKU>m^37Z9XFc^V% zr=Mud4Mu*_>$sh!vh{67OR2)hcpcxsoT})}W{Y0$Vef>^a0qs>734#_K*>6Xc+O^L zz&dj&2fta9XrsI!$7Smn+q1ThwgiW9GpBYVWod@1F8K%_oxG0THJ#xKPsg*OQNOrB zt(mAUTOXB-LL_{*6fIel#&T}AkWzUnyFz?F^788C#DEB0k&%vch2qe?kB?B&k4Ids zwJ~jysAUh0Wpj37vGjRV04;bTg>G$pLVa4K&`898ynRZBVtR8&D#!%XT*M(t1~?Bed4X`%Z!O;PIaTWENDGL?EGf=Vyh;1G_{s{7$?iFUDC{L8jq)*-Y4n0 zb^-Hirc4Jf#-l39Cc5r%39{P#i~BUMkQ$eqMtLtZ(Rb}6uJ%FJv_^d%>aj2-q+skLXg%)3sM5u9z^o-&Dw<%iL6 z{FGWQF5(W@y`qzEJfl|}Bk1=n!<^4dK9_sqEb`GXs?S>0MgKfCrRfj1(}_WIbZxGw z!}4XPspCdBx?iP$&U4bE5-a3T@eMt?-#3P}t?%Y$(o!0+S(tq~xq~xRv_@afD>*RV zmvMjE?;s^LXR34VGFt8|jo9)!bl%*CwyhZ9j_a=E@^n53Joekslx|b*yKzOGf6htH zEB6CccDPZuFJ(XS$*e@5z5j3zj+AmSFHcbYFBSB9a{+P=^WZ&5dBjUA%RxgOZ;|1~ zxh#tQMm3mWHYfZrRVde@Z@4(L-#?T#+pN@v|O0Js&PkP zkt=U@Ste?=dr4*f-bY9NUg1iP$f9`3CN9Ttl0(bdPWE@`Cr;(&S>!aOmhdkoBw{zEZT?i9>roiL_z^4-|k#JcV~NMMIMB6;)cuh41GnTFD7DAQmj_u~5$ zs=OtY{ks1rO5VAivqUNM=hYOt@^UBoSyatcc0u$YU5Fl8eU5u(QAHCE0Cb^f2f8rd zo2#2<#d_OEp(9!`9B2E3z85*fWvr4y7aYEGkB)e;Ceto+F3VS=1Z_oB)SpSS_H7jG zqu${>r|?kJ50AQig7s_TzE;|(IR~|-vgp{KJyh@NJ{14Qnx0Bl;H8G7vT~|lI3S~j znsO8A>51cL-MGMO9t~*Q*v~oz`Kzq6)CH6yV}?pk?m)t(ggc$P7)4Da3w%7ItX5VG z9gB%TW}nVbHNm-W_4hos@Aok5l8d7Jjx2V~A3IuOYQq7e(_G4i6V%E?;PpIFO&?0X zL~#*!xt&4A_37KK*&qATIEmfAdDkW&O87O|q2EE2ie7A`Pd}TWzjYtc%EkJ1v$TB? zQ(r*;y{bl^Ew|S(vqV^#K?x*ea)FBI9Y+5R4YQMfXgiEmts`4rbn`@~KLRe>&!e5e z#{^k}#a#Q_W2m;Hy)Lb&f`fOXZ7@b(5}arwblzj9k=!&q&LoGC41MG5*K#l z{9!uTGKVNE{z@%QC!@bRKXW&Bs8FAJzn$X6C;J!z@zB}9um_gw{uT|<)U&>foQ-=M^RYkF&q4P zmIC_kErBcAe&d7sPr(z9Sgh6X8%s@I49++10aEA8@rc6-yyL44cr)z+USjzU`>I{X zr-?6#KP?0M?M8v(Z!4mA=n+^t`6!M(rU7eSrZeVl&q4AW2!alOWAa*OkS5zU{PAN4 z?q%d)S-duG{uqdF{oGmmvL*n;rKrFMp^QdTM@ok?MBkAj$nO>4e;KX>Chu$ z2+#AAgr%_>MC^_(2@Ki|GG3a(2QTvR-$Vc5@`X~2#r{{AP5KOebP6+b92#+^fjM@K znoK6;7X$xE#^i8xE`HoF1s*7=!0X-!5yeBx@lQ+pUw5d($ZD8qp-3~ZdGM?USp3wja@LFC3eKvgvj2k)=NaXpvqey==<&AiNs zyv_*z6I+0bMpN*-G#yy0rv#*PWAK$rk&r@h!umz?|0reg-+j*k9=7+eR)s8?V^G4M zQ)GkBX06~audskis$1|LPe&-RdOf2jWJ0_;3UT~8c`&i^J(zlKJ1(-f2eSe+7>`fN za~sy!X8;@Li(%XK z<@m1c6mYjj8Y=vpL}2I;SmdWk{EkfU9X{pQ+n2Qgo$;ygN!V|2_yWP#=5$~|M7ZF7 zw+`Hzs{~&3OOPAl;jpe&9PUgR#Iv=Zsu{fCR;GaEnb z>|l*g%SgcNS0N9ZmKTOM zyfz})b5@eR-5MlnyA?^08wY)jKfvVV|G@9y0Y)^60xfrQ@>ymoIr8!x?(KMr6VES# zXNykYnZjCdPG$jqzQhp@Mg{X<%{PZ>`<`Ggk1BBgy8*Gc25`?rI*_|Q0G3~If(_J< z^krSc-mD?muyzpa*Vlx#QHt=cuov<5{lwq=Mj+*Ey$o9O6`@^{7(DU#8R*(^0Xui^ z24cUT@*~z?1md;EpnvEpmj7x^+A-i<(0 zPw@ZecLYDom<3juBC@YU8~<4B0_@I&;%jT2YWui&T;|*f64xEXUYV1?9&(58uxK32 z-6TwWBIQZdDa<$t80&)WyEsBl7kft^XYK~^puJ@pfA=dE9Jb;ScvlFD1-szH+hd?r z>pp(_tq>o`UxufIM*(7Y8>~$Dj$a)(3l241;&-eaVeV8a!v!Cwg9|}_@im`i_=VG1 zX4yE$KPlM=fs^DbZVO;e- z9#^jZ0|LDwncSR7Sh8z2X4=OXtspfRSm{VM1_%+UMJYI^I|CO^2OzvI7f%-o0rGEu zfc<8F@YWbZ;+?wyH>g&EAHiy{D=Zk?X+Hrf(|W+`5Ch;p&<(1~wV<+ZJw8gVVXyf| z@d_U!a>OVRue&S{-;AHFmbFqOh9MRFY_%)+SE)I$I(iOwc&0JcD|JZZnh9pP(0M$% za54THYsDYx<%3Y5iGNl+t34>y#h5iXgR8ERH41Tq;Dgb0A`_R7S47T({yJ{_4UUzJ z!k#(-SJ_(Y@W2o(Y1D?RzvnOk4@}|Q$?t*46gzCV{xx1HEktU*FhHvQJYOb59}BB? zg4!`DSZM_9_oP0+_e7QPl&{zD_XYJ>OOM4;$KQa82@~QrAj~9W+QA!D25^~#I@5nJ z&+gM9H8O48E%1c*is{R)qI!}jt- z3xCC;qEq2UeNBG-3b)#QCKhD&FJXA2*A^L8T#B-ChIqh?x|Sh}8;wawwm%G+c#Ka-S;LP;rC7RRC72;(L3-cR0fz$yWX}ax z$opXmry5?t71D9c?^$9%FFXOP9cTi^8i&CzmFd{{ARjwQ4&f&gGvGv9BWSlbAtr_T zkUNwOe#-bm%^Gc3T6c^8zOxy8-)Bo&6h7gV`)r8lgfjfJ-Vz?I>9;?uy#Onyy2706 zrsSjKb-pl<;&;!=z`~@>BtCd2C`{y$<+I)4hdo|o+n5YMs*6aqupje%d?}HNe$C%+ zQpbPtS%*~gTEWMrs_^TLGLT|0!p9w(@s)aNe>zf@`Fu+oeE)SE#8({$Gn6jjsmxzy zd~FwS80}zmR}15--$Ot~xE#Ka9)$xA?!&kK)Zkfx4}s*XP`ujs9)Gi51vozR)$Z30 zRcISAhArz7nB0CRn4%#^ZiZaMp9`Gelz{?pX7DAyV#YmCo)rhS@r=kMs}Qp1m^!RdSgQ z1FNlKWbc6hxF!QY;YLXa>kIMeuvC0!%NM@*&H!xtPny_W6(#qEYH-iVFLnX3zT|;X zjGbv|3qRFPoxf%CAbx+n2U~BlBD>m>K*%avetvr~eqOs3OB#O#cf&0}dO-^~^ztik z?bwNRl#*~~A`7MXmqF)b!S6!gyI7*(3g0%bhK9u&aP`kXprz#o-<3ZD4LbzASeUw4w#^8h>Bq~j=)R^V5W56;zC6RlnYyy8DdeD&2xUDYz6m<^z}`!oop zroacpij>{chlW-DFfdV?7^bcPpF8D9j6o~DcuAL3)+{7HMFYt1r3^HGrA}tup2^(( zVazZ2qD)TZEd+}4xA183BoZzElpoWU&cr4?1(f~*lN>Ff?80N9sp=E1a%;v{?<-=p z=n*FHTOM|J3y5@glD$e$B9?A7CqsAtV&7$oWUrwLJlUs8EIM=W!$-Av%)`RWGst*W2zY%q z8Hkq@0nPi;P)v0xE>OJ$e!U3*e{4?sKtsUb?NzL$fY0#4l_A`_@s zAK*vcoWkU9S0rshI(YdWWgs=_Hprb}$r$C1;P|u4?aTQR%=HKhkg`RdzqC#dJpH1> zeD#X8ZF>-q=eb0at*jD|FE|!F~^Z{ znfUHk2k3l*V&ImA*L}T$gJVN*X7F>sPYlIp^p#)PvC?e52AwBz)3@C zIBww@97v>L~->D9xHB^Apm?6P07Gv~QzzXuU!NJ86KsfpZ z&QF{S-&L)}SyCbR)K6u&q4zHq(Kw4=tjq-E3u5@2{ewscYJ--JuM8pHd z^5Z~Q@gJzMzs+R%9RZ=kao}^(8IZia5tj=WgP;K=@`U^Z7tVM?>C)fWx#uq4_Ra-L z9GT5TZ+eF#P4$7x(=hOw??BYQsWICgN?>b?+u-cA7Vx28hq$WSfVmr=fuOU;!0T8i z;(VwA=w6uvBQgN_))fFa^%%_TGJ<(aOTp4v)p(+-1c(i}l8HO+I6A-<8e7Yd)-P9? z^H&qVd9@Dyv#^=OWPvO^74#D5>$&5xArI1Os|lB$xP%|Nh~im!jZR{QD+i*aymI|ph3dk3lsIlH^EPnW3@$tUL-D{6AYAzll;C3pcDK7XXg!r z;2dMJf7Np)D`N(KuV*IyXt4sno#y~8qb13-i!NY&h!R|v@D~hkm9=aCdKerVnns=- zzR5Jznv;#5s_^VoHw7Lw3oL|Ny ziUx#l76NOk0(Ut7!uL0*5YM&WY!j5lpwFkxxM;Qz#<%R5qPrXzy{tprPaFad77W%h z-b$n}ww!4;T}CAS-3KR}m5JNl8gTZ3G`2pK2R^*&7R4ElE9 z8y_*A(wWV_8u$;)-HCyU_aZRyW*nF7&coQ)gs;j4m>G^OF(;#`gK3Eza zk-31|j+gVV4$i`U!jdpI=_KACy8s*dHsT=jsrIsKRG5m-)%e=+v!JT>5Ewe+4{E6t zU*;ad*(w7Jzv2>@|5+azWWHlwSEk@C`I*?ZuNP0N4+9}hlWL8o=Q97A49VJ*5b)oE zUVe<646%BsMB1gai7Ow0qaNm9@k0~F%nre|l7jWFQWk#hR3%2eJcsxavyc+BWdA;6n_NLWD?{PE#rnd|ciyg&f z&mJ<*LJ$e<+luSmtZ;brO>Fk90O#lIg1#rjV9G;B{8!Zs>Qt#>&#W}OQcnqHnM{Je z-$)SmeerzV^k9BUs1nRRF`3CZs0$=Q3bBI`54ss{!n?IE1Lx8x46n_@dvss$kEA<7 zrK<;+!VN3Q$ZHXzCN_vA4k04;=_KA|wg>plP$ezF)5*-6zkqW;1D_dQ0`0SE_@6w? zp>?|{`P1_bD=dce-Z~ zU8_y7{-*^ADeuMi`)c^Uw^uO=$quSpNf%iEY7+kwf@=nl>3W)nMicoK%@tRbI0&YFi(*#U)blm- zS*RL60@Q}~0*zT~$zrSNz(iyNGk9?l+?yk)eNhj9Yi}7ov9Ar^lTad$-JGEAykUD@ z&^dguXFXoAyq;rNVoFswOs7xzz=g7^Q)kf>E= zDw9#c5j{m@9+8x1$!cG?XQ?^4KI{WL@{_Tt zwmrmct4WVlHa6Ypz}E{zjP7)6=(79~PPuXx>zTxXlyPr7cg-lU{94A3d#nvk^k~E0 zU0zUU=20vqa8$xI)`WJAF^VGyrsabIH^L()V zM-~>I^?;ugRf210v(Reg5^_?*00#7%@^AR`G8}%zVnM zkevljEM5rh6s*V`9X;aNtN_O(j`Jm-29SKG#l$`FCDRuxOO8nPFnOnCVdCc1Fnys9 z$;?qBp4+c8XBwvyr>ld&IZ)t|S=fRdKMGu%L$it4@j-lZVINa^&YiFZiqPJ`4yyj} zCT@a$eIV4DeA>FgzH3CVPgm5zf8Kwm=Do2N33XTlN|xLKpFQT2(;Ha4C-g9Q^>igr zNIV7JNhg4_(Wy-PmH9Ai)&pE!Jc?8KLZoQ9C2T2Z1}b~x$eqab_>syZa3}N=$V2nU z;M$#JzM$?5H|fDe7gR}msupaE60Bcwr+~pv;o90C0R)X?fd2eIqE#<OA66`clf| zSIHz;IC~KuvUmW#2>cEQQ?kKhy{AA*p8?fQgZSuy4WPK>H8X57jRdq$ht1vUaH{Tg zrgU{3eq*agjEr7@cQ;yV-`Q*c?*{IJzh~!?xiQ)Jm+JJB+PnZ6ads|A29;1-vY`jlcIf19vBU zz`r{V)Y{LV!0mP&wI}bbfWK5m7-63n{Or;h@MQE9*mhYFw%#bmpS)#BU8XX;sG41y zT1mjmih1Ph*6C30ydj*w{TrB_YDK2>FD1x79}6xQLJuKCzI(XCly7}trRQV8_nrmA zFM2|&S2;jb#t8((DnhM^Wd!ZiW9H0G;;%8XB>N9J!rYZZ_{6Lxey7KN{NuF@L_4HF z)0yA=_?i%WOCc5<_;dtBwswNa3(f&O7ZsvPgs|MsDOmTp6I}c17S2%cfOA_7$Z8!6 z_*xLVc)ms-{`w@zS4*BtYI7a~3Bw!Uw&4h{av8u1KE=R!-Yn?4b{H%V7kD(KbV*}E zIELRof+vBOL5uZoe#>4bkon>sHlMSMEE)TWFGS@q_DM^jvb_eBAO8n79QVKtqn+6E zmkgBCXW{KtGhy7F7QDRZB$!s^MOOLwkevJ5;nPEd_{p0FFmF$e{h8*Q%xJ13Y@fQ8 z)D9kDnkUZy8Li^vmVmCZ9A65JnFMe$D+v#Zy})G_w$R-`Rp2-@B9lXc;Hk|E`NiC7 zeEL=}S+vs(MtSK#uu&Z6uI~kV?G>T;!&GLmzavreGlI6kf#hRIC01AX08Uqb!|PQo z$hgi@xYV{6Q=LyheXT!k6Z6I0yM##A@-%SZ@&w-0>_KL{&jkHrQ=z_-Cn>n6fz$n$ z!QodQL57bg>Gm;)(khRc6hS@V{)LhCnUk#jiJx_ zxn#*D2RL_L8TheYm5A7Az(2m_{0#Xd+;&777K_d$9`%)YpI}|KFXfSp!@J<&={6+g zk`wW}vW#rG>j%eLeqiI=sWAGO7Iu)ij5W85lP{yQ@EYGPzR0IIY$E=WAN+nhT(FV? z{jX}!_t|S)pgV_HcR9f${vUx-mLIvV#|O$y_d&atO0La`rsdYv0s@y`MFas`ZR;9Am~Iv0L<+|QqXERO$IxCKc>&>gg(5SEC^qn=_W&F(Tu1iGe`HK%mh)9_h`@F4OyHW`Js@3eHF)CfP4*9L;9r?u zgXaZKg7sW!Hk3kYpIkS$;95;o@j0|XPN&u(x=E3fbNAc__ zTZnp%FkCy<29B?^Brcyzu(4A#blS6+Y^*lLOgxVi#r~B;0>mhMe*=A>^zq zF*>&r|ISJSKH5UCYLYAXqGOEn8;b#((Sql>=);6__GI(sU3kNL1vqeuV-}qX1jn0J zkSUM)K)H?%Oy&@*)5!xCvvi5it8!1;7Kf3S@m!3x2MZihB=8z{PF3 z_{S0tGW@R?*n7(Zxszr@J8d#RcMZt%UvXfU`+a=#pclzLm(On&k%o!qNo< zaMSnIM3mRdc+6Wtn#{zpqQ5dI9XBOCou2%;$Eo?ZgOdNoL7!AkhETO3|Z z)F7%|Zh#Ai%&TTuFf@%3tk-98==vovy1@vRQz8mV-_)8CPUh$=|i=wW_(Us z42qd&fwxo>hU`_t?1(Wu_(T;JE)&BS-4`+~31-A8{Sj`yWB_|En&KDQKY`Nz4KR0! z5P6w@46~BAK-iORtaZ5`Sl+w~hCuE%;iyZME4A>^*AGV9sje0~2tVCUy}tT^aKxSyWz(NlMr#LfhF!sBZV zA3wC$bT%LtirwJGLKnE~_Yjte-wl{B4YJKf1a`@cfDF6;7(EGNV)tE)RFwO}{d)f3 zjg}o5sMja1QzfCVz&UqcwI3VW{{x~k>ha2LmqE_!UVPQv6SOLy#9G@IF=gkK$>>%M z*f7}#mnPQ%Mg9zuI;zYs(3=X|Y%_uVZ9mAS$dbQjvVkaB3H8!c@x#r|@cwyII8>V@aSSehX$vi`>Jqsy1Gq?Y2N4gpgf{7WNN&C)c`G)KE9ET6g!6p(dD(3o zcdM2W3P`KvJ3R&kZ)(6B<8^RN>0wyUO(L=b|3Kj23}O%xEqJb8;9bT_z;v}YTvg%# zBi-hZoxX4IPpK8;zsAGlt6(-QmXv`59e{ii)cnWiM4))OEuQAzk4?Yl1H0>gfz+;M zaQX=5XKnO_i(0?o8wGq2V6_8Y+hqoq7f*w?^j)C+!A($1R}AM1_Q|Y;)ZzM>5m2Ut zfwvyX!@G%_;8<`VoN_k~`URR2x6jtltv`UQ-?b4wK4(j^+T>uUjRy%o5JW~jgrK}M zBIAd++RgWN!j$rLL^_OOpRgl1bL${Z1wCMJtsQyb7XwXPjmd0e5B9!ygsZEwK;pin znyTMZ$)|5gV84?waj{i^p;nr3H@5`i`lCeTl?9Yt9z-k_nG%z^st}egAo|6^i zkrdS?n^sMM*?w;1eSZ}WFns{ZR>u*w$n{V@QVU+Li-5_4+r&oK+$mz`1PWs z{qr`<@ZSL_Y3c~%E**iwbSZ4vb^*V+0N}4OK~6#z51QCaC#Qsr;0a+n9OW4Y zgR=(tTz)OMSt3JLzCQ!@yzwOcCyYs6<6PqXSa25pqZ)fB4B)Nd@t|j`5m_7a8rKW% zD$IJc8%j!KQ%Lk#xCT2!fvRV zIEA>oPJvP3bL>y-P=b}g%k6}z3egCfL~;k$!w+N2!I=$CFmdimtQ=uP4#)3;*PmU) zXO;+)e*)L@cx5_2@^U9!F0+Ps<}ZU*oFm*9W<>bs42XBE18KO-;o4*V zAn&8VX^iHOwSWEL_w9?xj+R(bvdJ74Ej1=pa>Asl$rj!ls02bQs+hN1bZMxA4}0y_ zEcOkqL9d1uqKkNxi;Kio-9J6zsfo1&~EzbBw_E(@TT$i4x%r~ zhp4t(3?c1e1PI+OA^46WAc^54?t+^sp(D12>=tFt~x6Lw;yUjn#CI6G=pSWm;R$6?Z zW7rCPlGChF&~OLG&)xE#8}Mi|B;Mmy4m-A4fT#%uckodJ2Vt zY|Rzx6p(4!X?nCxk;XOuN86;1bDsOsNRq8QT6n61^So7EH&!23H)H8c+Tpcd;8c=# z*xB(IJrwD{2Sy&T2N^@!Vo<}KwJ$){@6$L~@QaI$j;xc~sE!ULzah)h%V@=59F=Oc zLwY?))K_gE>dE*=r}s$H=aEi4#~6Y>>`S3GePI;&>Cv?so6!%sAXd@6hiAGKp!wly zbX!mbvYYKmfou{wG1OGIubW3*^wjC=l_$7GyTZ{m(>t6;kr4G~e$yurrR+KJ7pUT2 z0ky8aNbhEeJ6xXo4LbJq@)U$*(3LNeh^ID;RCJE9+D%Dl___*`6-!_ZSp}NvwVM}t zAdpS0O5%7I%h9bb5wxv72Zc^m;yvsb<8Jh=3tpvfyJUZf#CCITBMM5WaX++{|#8PSc z&`E&~6b-ThSe8EBe+HeYMZBLMA($JXheoF$w%19C?soYNb;h*l@!BM znW@qYi&Lmg<`rvn?ksI;m86?y?V=j35r|p(nayPC(V7(X`jW4MT=W+a&UrM6etGLf ztCAY1Z`48BGsm8_%Q}w4t38p5%S<${U5aDO`?*S^KGgAVE<1JiK5DpP2L1Ih2FZ8d zM|M7j$gwk%E{Z%tgO=5xON;cm{)GY$(wtvhjM6PG-)D>)+G~Ykg>E1ka2DALa<%7A z?WCSp%#d^55wtw6i`}#B65CL90F_$r;{xz*wkSZrpM+hb$ zHV8ts|8m$LrUDnA89;B{-mq6Zr_$oJedxDiE2}g92>GrwM=5_5(d3t-bWU~;D_`(19f< z>D2@GP;crxZj!JaP3nKiE*l8tU74Fjw`9Jjk6f%#q-G&nwab|%^CGB_donFg6-Iq^ zWvG8`3(qXCr>;XWiuIT>%I*E}dsMIfKL1k}<@v3K;% zlnw0e4-!=RYdseiw-G(k6sFy|p0s8|JaRc{Op7`vxDYRU6d_VpH=vYEGeRT5n2Hcx zC?C&0n7x~>mK8(q|CUm5eRcF=;4oX#tW8H(XK>&Byt#v#R&;zTKh z_?pxCfYFNCvFyj!EZyXC3jLNHWd|giP^bPO`mW86-u@(7?;AE7oz1;P6+i7{8<3iGa?k$RQN!pgs+Lj4sjpSxww>J1YZK(Y zr)cET5|tWi(_2UmrEQ_7#*VN?W(oAKojvRF!vhU_U8aT8qPexsZ%~8MAa&V0rM|^5 z1NjQc)~D*0(0zwhkl?Bua*ocSi)y?$>)aEl%_5hI>{p_z_Ftm4H#3O7fgYLI8jOVh zn$zcruzs7tE6~yqz}qp?5hY(a&S~=`sZMqo+gx%Doqp9oZ9=b8DN`ZtY3X6sW6mR1 z^rKMyEVH+DPZqC7PGhOO<=2;?Lzz+ziGkg$v-U;gcYXu!YUFG*PsWbhb)uW9?CD0U z1waGUBe~NzwHS~VnCQnI;lVd>3lSu%%-VY0(YOnc`71R&#S+tU+)mio2dbUsbLY+&O z5sjY}iB1olMvp?jvX#Akyw-F+^{87zhcoi2-<^}_(6zIi%4Gu@*La%BBy!X%u$cW~ zbs06L$I=^dnk#w zPuib`-|(ge8Yj`%w1eDR=XknoaXpu2T0|`z9N4dJx2U`7MxI!R5;xeg480JzEla1r zXYVOkAb-9aTC!{>=eME`WhPoWe2Y%!Zbscky2Ew6!ZZ48l*VTENi;??WiNBX0wyCY zWi`FMZ6*2_D#4w%PDDOV&D__Bb?9G6E!B^$K;nzL>Xw%oq6vljyp+uX#&_QCx`3-OiW-v}N1zy4{QiftCZ@=gkLbb#o~#wP(55Z4bE8*%#0wrH}N!f<)Affb|@r36WvxXW))}JAR9$tWV`1vw_|-ZtC49y&lCml-nt{SMO@h7cj98Y z5>k47^B7OVMh0DO-HFa{>#2gZltY8!`?|0EY4nrLIrQdZCMi>!UGK_Da#!V!(`lJe zX!cTd^ggMVHvPCuE95%amb|@OMy;S$t%+wRa{sX#u5CgR`}|Rr=X91)yof9|eL~{i z$GBvF<$ANK3&{Ij85^(IO)ua>?CZ9vs4)2gyYy%#_cU>e1I?I$ZuaF-jo?bwWNRq9 zW=AF}pPJ2P?AlJ1T9PPfSj4(UHz5(LSb=*ZOW-vfK~J{m)x#iPdgWX`8@%rd6@UAM zv)o>Qb|pJfefM@6pnMr62!`SJch{q|S37u{4QyzW$Tv>sR3cgu^o!f{c^&N)@PlS^ z66nt@)2WWY-+UmwjOQ_TAu4-z0Bzm%uNF41LS673y6dP_f6YZ6X*G%>vB%%Y zQGr{!KPZlCdb1AYs77)73klkGf&Mpj?N96js~-ukiMiJ z?_`D+3QG!L6SECiIWM!iz^ESfd7&@6s4ala7vz7NXB|O(lO|Ec6dpJEbSCPQ$LQTE zGkUx!1^qjS&|51H^yTUSdd%n$jmbJfg%8)DrRhc7(z;podp=D;-=zj2^Hhn3SH`mE z)qH5=>2l=N1E}rcD%Sq`Bl@aOKvVxu(S0~#_5E=Gw@2AUR<`UtpL3pjq$p%%R7x5Y ziH0)LA}N_6%IaG*R9a*_=REgNkp^i9Q5xFXJC%O-_b)v6KIh!?`JB(|Ew+rJAi^(o zilGOdNwqo^@8wV3SPIpd1iH)gG;`0wkUrfp0eyZjmHw`iYYjIqV`1+u_Rqu+wq($k zzhy9o6+N?>QMquIJ@+++s+Q)V75lc*eU6aMe)|qhmS@oP%RkWZ(#2?N_+zv=yHPkj zAP3E(yV3C7GqnAOG}~~-7{yjguy3mqk=DhFw5?poQAEQZC`)GDxlmTlA>LyrZh1vi|YOK zLunqr5%0b~;~{Zd7(S;8wVxD2%R)BNcWd1Erw!bxcee=IVcSec*)p0JHJ^=Yl4iC& zIfW+O4X0-O0pW@G3>v#qmNC@eOgXh&^l>Lb=6{Z&C2_;(c)1JRCqIXYZD~b*&ok+2 z<`@${D;|ZN({#IbM4cYpS3&n=-w|GqK8B7I4fAKk8_;s+6y~VPclOAv3M8s0(fZ{Q z!ycd5&CicYVIP*{)6l>SHtE@9RNa(APy47bJ#07nYH^#{WR-zF27Y22{>xyC#cYL# z4G2jjnH5!&bQ zomu6+k$Qjcp&$Qh(9jckXk4ZVCsa$e3ZvBNnOJk-tOE^*RV!hVkH;XnmQMCWg&b8e zNk>ngmoW}~hnZJ4W%Sc>hVFT<)cPqh45f%%WCv%~piGlyYU5zk+O%JRo_julO4Uws zGZ`Ib|B+Jk=4&Hu$dz#GcjsnNUtH&F1uLLTV?!#-*?HYVb*S!#0vno^DbyGK;};yd zfixdE(^}Jy%p93;wA`egO?z3%EN|0jja;>qJ=HwOsA>$MD-|mAOGFtvL!*wB+Es~? zcXMlnk}UcmcZg~Jr-(kE5(jdYQ|1?>o*%g_=K@)2B0;p{a)a zf})-5VqbR@I5h$FF>#FP#hnygG8TS6V~(_pdf96pmuUWuP;}f}6qS2jraJ3y(6;C( zq-<0;XoHh~>!ze4dPRE=f23p|>-#d7{vP2`YS}G35OIVaOmq|8F72X!7fCZ0WFJtI zLnd_A>ItZK+G^Bv<2b#>JHm(*>}ID9AE!g-j-px7+RXgXa%#(+wcQ(T36s5p86}B- z^m>^O-JAFtq4$naKz>Txe=}4jL%m&DPEQi0(%!^N(tGF(&;TZ2gTjbd7g5bL$5;$9iffYcn9r zojGUG_T*q@#?${$TkH?UtK~W~b>A&!Q}8FIsPr-Qc*_~eQT=pp<0Z!B>0i`R_m6+) zy$0HDor4}M!OUAL9rPe`h^ERkpm^=4T-N#q%7hiO@3-X8JoN+=_p%CI_sMY6!C+`EKx(kCHQ!)CG%sWLTjRX6~(*c(evR5djHZA;rNd${7nfvnFW8-&?-_y z-)}uYf5m#ZO&_hKQC0W(>aTTawc}~}ZTda_jOK8DjlmXr_QxQ*Z&Ztt?L+Kk)s;;0 zj!*2Ik{~2)aE&?i+=b45X2jZr++%J#h;e&?6n|&_X6h;`#lUGbj9>NvdMLmGg&QOy z&0_^DD_KqtZLVS;^>w4O{a@J5C=)c$(SoX_F><-El)ig8m+cA@b(>e0hvx4sXRf

n*1P1LV)mJUucp)odT)KXidRlb;uC_QUV%YJX6d&`a^iJ3nbr^+H$ z=A{VQGbNK+E#a(eCsF!SLI;hkzlY{Eormt??px_X~<;Agf)xa&RBzFM*3zny%~RscE%0R#C%2c!uUN~tZIo`nmbWu#A)`x zI(u~Xz5;uM$MQA&L{MP$a(3F-8z#6@nVCH}!7c5yL96=<1-I9-lUUwiStiXc6}|Gf zLbD&8L@z^=n97a1%trYi!hnk{Y|i3lWTcXh3YROP*!HEu;BUXFjD!iw|0JLjAJ3(s zl_9k0%r;hdQTA86V8I5M54>WV`W4+=|)td#cgc^Af7_s)Fg6!|l@ESx9+kgjp9e zpSJSXF>9wzCwj9nU-g(JHTodWb_FJ(HJy7FvZRI-ktn*r7+uIbi~fym=37i& zf-1wTsog7gwm^3Wzh6I=dbudLNnJ`p!J(;0YA{K-VBK+QlNTb4zhc)qqr{eZ5_y%; zj-QD9!B*;O;Y-)<)nHSHJJ=O=)6ko1rD%ZiaQ?b(t zRBy&LYT=W^D9$-Rqb1Iu2-iwx&Ej#PZksDDsd1$7``u9;9%V8f96{pwcJ#^iPBi43 ziSACXr47*{sCI5IcScQSsy(jL;oA%wuf$n?HEWooe@-(KWRoaSt`c^7O+rOQrfxoe ztkE&i^E6o1SGele2k2N@$<7Y&re{`mF&(IipEAHlogW{fgPpxpVcd*OdZbO0Bz~cR z3lj7(-iDTa*u$TYa#MKjMg-?+E@S?_>J?@bon*FO-oox&Jc;)GSH}EK(rC@Su#_oz zY(ax0#_6{OZ|Q^P_pB?UgJkRuG3%G;()ZHEY_DQCd*jb3Q?uEmj#>1X(JRy}3()K9ubWN} zn_NS|kNoJb`-`cG)M_N&8->;^FlQ6XZyK#{vHyw7qq8N_tpmS@*tGB|t%>KfP{+)d%=e^R>bh!>4%>$_UJD{PQaz8|w5W}% zxf3*Nbu@pfiZrb-Z)PN_B+$bx{ml4!F2g}Dl?^^I0VQmcXtk&)Lh&N9><7s{mWp#V z2az94pW)iO$C+QNw{cB&BET zR`+EsZ$2`^@njot-y_ z%N+T}9$n#$sJ=T*ty#qu9&KPZ{z*aE!|PBIUj;pCA7a2`X%wuJhN|Xgpg%GCs3(D? z@bW7*;`%VXXSJ60_m^@1ip+l_N#_$2u9cHbfUD+0ZtL!_4_NMznL>gfUNSVALW$u?IZISdS}P zXq%Zmzk=%@;iQcSf&C~ob|Ra*fXDWCw$n{2%5EDj-Q=%q>_lT`NodQ3MReU>KRR-G zLhEPl-u)@nbm1w%2=n}I3;WDDnyX35pa=T~(FCJ*w$8AJ*=#hMKHJpF%rag}7hDlC zYR;e8Kl-+;#gjbL^~jT!FB+t3^YgG8zQC@vOQr2Qdzj_hJDJNW3>#LxjJ95oaZ~)f zn@#$ph34ef(;Yb#$U~}w$|ycS54)V$>4&>1e~&RUJ7X#n6KPAQ!B{$2(1`BPx2R0@ z07AJ%D2{)U{VXeu9)yUxUDdK@N~Tt#bDxITy6;0w!ZT|+<7hlJc`55=?RHw|XS$tzDPWsJtC{}k;b@)8DtgCqIl5r_0&OsMMfw_xQFCk? z&E@i9MruQlRBHx*_A50m13-svUcMX2y-`6XX@sc>7DqMqicHdIvYW)z<#g}wEOhX+ zE%R~9QRMsP8WV794)Zo_GaZaCLv4lI>EE}M>fZ`Pbv@VUq(CL+`_G4HO>QeQ@p=*8 z?^qH4^0E8K8GEw7lg~4n`lV=Y_)}(+V+H;9!JHZxIiSnwZS;^r7kWr*=&JO2sB+~! zHYoTOtaE?CX6DPf>16n^Q?zqo?YfSkjb! zNj%MSAv>b)L`N-d@cSB{B8NY_(5=rJZZgV-)bG`4x@y}bI`PP7dfump(OXfBF8g^< zH&D$_ZRlfyJN~hICzK*f%?{ev{+B8k>!GYT<<`7gYbe>H%bw7bMUpZ(LS^H1ticC+ zbke?={HP9@kVue(PJ_*>*Q>vD`7Wt z@}?&Gw-g8uA2C9^d5e+dlx8Y(BbJw3_>f)n-&{t^&xH+YhqS7r3q5P*^5fpr(#XmA zbcN$(;T!ons`)RA2IT${j!uc9$^%pA=J*2$MmZv%mp z#9a9Ej!`adMRJ)URDMe(dof}**FRf=st(*`V!XFe>OKXIoT2Q%6^v4}r_giY7@wVS zg55aF32*v$o-J7APuEl>p}yoQ`rvViurRs>y=<6`eznCTC1WZ2AI>KE$x_tOZ#|tR z8%l%jce1Tl?jWV#JT&+{mmMD?^dyL4e-#|0jj)HFtG>Y3H|2V-m78eX&OoLPzh`dT z_U1B^+G$u;ID05E8W(y03x%=WQbR>klSqjI($Ew8)A()mO5->+j#@*G>> z&wWRQ{uO0buPj7TA<3+@wi=po)`cn1ucFFaJ*(+~Xlvh%RFwDiDZBQeY-{=I8?>5T zPK$Cg(Pg7U?8^3XYBb9Ae3V!7D_5KmEWINhzu^z?d*E6P9x3UQVfoSRLyX*?B7#7Y| zMD|pH2{3bOop~jhDzQ_LpYkLWtXWET)D1Cz%)T(+I`%V_FYgFPtTO4P#u&7(s$UqI zRD)!~dRT|JM9whRqDsE1Xy?*Fgf1kbReuVRFk?T`nH|r}|13w-Ja!=7f};#y?hNf( ze~o#z>M@Jl1&m@vK6Q=T%b!x!f!1~$Vcz_yM}`j97^50Rntv&Zn$Pcs3Bp2l>Q@hD zmZVEpxYMrzcBtt+%N(DeOiS|iqto5eY;>$W z{ZYJ&JoSn}A2rV)&7WV$iqav<>bCLc*$pv0LHpPnV$G<}UWCoit)=o^=TUHJ4|8RP zD0jAyr8k~P3d!jRVe3FH(;;_+uK2wdeYiiy_D%Rp`y#E-hV})_&b24#)5Vo2RjrYo zDj~;Z&m2Nyp-J?$rXzjWna+exX<@8a+(QSNRhh#_Hd6b#0k-eSD*7{NfSE8bi!Jb+ zE8KVN5|V!{?N-Qej^BtSyUTV4%}g!eTxbUxnc~Xaf3uD9Vw90;vp10o!0KGV-L#lQAUFZ67Mpk1>73pXXZkC-iM)! zANc4(nIy9Kk-=O)Vuf1QW}|P_chHob5FMvs=glI(L2x!np?Ds5+dj>SkD0&8tU4`Bz*8d(eFC=Ck(ymIY~ph>{SU(i`YurKl;$pE3>KD+w*+p zMLw1D-puwnA3@_w*04txa{uE4*ST}{W16b&Kv$@$q2Eipnfm7!gq;R&=(>41^xE@& znxx)`*39TdCk4s0=hX%pw&EYEJva$HKy#2Am!LfI<*o3xxD&Jb_FX1Lehp2u6f(cn zo>AZTcNi`^1NlElM7RBo(Ca;qS*P1uSiObwSqHQf?R6n&C)e+^b?#v|n`O}i9EOdyipXWF{jE1hb2A7Eyv zl+k4>imgZNH?u9%t0EG6DA%GRy=A zwD!j_<}_2y9A!GF&(1O=sxzO?cs&QL{MU^3^e(0(q8;7(bCrfE$Y6_^Cz<=U5v=ZI zAv!&D4OW)=qlGSgG;wkby4c32-T}(3G}(!c4RF1ZRa>dW;U?sLyaSQ0pQwD3A74uT z1{3BMz_w;)FezR&>?Z49!Y~7FjcmQfXzo46bT8RV?T$@i)^bkT0GCtXsWgZZH7ZzJ z>v}d@!w9|ikfAD{E6_9kVY+ZHhS_OeY||Z4w7~Bj6|S#_a7!(-tS^?D?mS3gnF&4g zv4c9CPotf=kiJQ=pkIpwRC?keG-RDcrB1I$HIG|}+iV{yBRz&z2RhR5q@y&WQ5nVi z&gAkL4Cv6SC#=Cyitg0p@>5(>Sy|NtMr{P4hK>}J*NK@EB55eqO^)`HbR_b-8l4E^ z?s9}~rg}eK^21l1L`JQrQ0ucS>Xu=Hv^zrh*F$A#XxAmy=*mA@o!LkyKfBDH|D4LN z)h?y=C7o>O_KB@;yJiXR8Ren#Q+a4l@H_TXua}$3E(PX@dxJ8iDLgo=VQ*@a*)b4%$q-Q`+ATQ+FWLr#VCu$nfi zDwTkRdiPlASCx!8X<>e^HlhCw_p`8GgVq0NMb8}`Md(-s^;{9h&eh}o6{!N08Mas0 z=X9N!tGOEtANYoB8l32m;W6rWjPlp@NV)l~oJ7AjZ)Ep+#L$J`H2Kaq6{t&+8-LN> z4zv!RU@}X&6w%HiDk<;6tkLhLw!xXGa3lwPXmqCO%_a1837_2@kC~$}<9y*Qdn9qQ zfXW}(geK+=G1tls=y^PoA?zCJ6`~v1&oLk9Skus)x*_ zW6zj{WtmDbzmp{J@1RMX-!qVIn)3tjyAVT;ex zr)xK{C0ZUR;@Amc-Rf_wlWrr8tS(1u>;h3l)-$H|=LL4bhL=LR@Dw9&@t)CsD4>#4 zW+F8ub>@WYX%w)pmrW?%j{duB%mi8fW4}iyvxi4Y*~)hUrU*u$b(a`^0WC)DODpL^ z#csjipeQsiea8Vt((t_R4&1b6fOj_hc+0|u8vNHL7`{q$f=1U|NakTv!uOvI_e>1L zE63!Ba-kV{=gA{WW}Lwz?*Mt*<4v}g@WJ-C;ds7^8RS)j5XI;z@D3N>esbw+UU-X8 zV5jU%Mz_cjqtSZs{my0(uzfBLTkZnKPOZQp@*(g|hY?)S^BWAn}`ug4A-SlYU@b?Wa)6oHIf2+a4%VmP1zKbB(_yw4f z;R{vMRft>2F6`vH1b!E@A<_f-@JU!0nXAs-A$8~mJUkt4H8_YL*&Gq%=*YvvJI>-P z*S}cemjaw!Z4PJDgp%CwQ{eTs+~Dx+T8Ke zKR0;4KO6&+c`t!mj40R|mWOA%2E!_|i(tK|Bk`Vr@gYl9Xt5EHC#R*Flb$?kDIH6N zsd0A&C8MjsxATzrD6fPfSN6gnv<1`qlLU>SzwlvI9+4>gj+I80$oZ(Zyrr={_;Q&C zj4dn#llQxmqbJvqwQFo)?6@PIH!B5S;+~86vkq*!?tYG|E{kV-z=<_v5h~Y2NqthhXCVWE{Epl;DNyPpq!k zgtuysf*GA<;CqH0~tRa^79gCA$Ec=vCnHqoUwT$9q?MSvey1Mv_dQ!e!JS zo(c~&OeP*HVu;%J2l#cr9GnyHNP_ecS@zl+y08f(wQwi6yzUA3_j4-f%LpWegHvGr z$X|ilq;(+5!HWbBniKC^`n<7jIdJfzGr9i8h%mYg{5L(PdFiV;#BHhz{JMdgd9jLz zum1fO6wKj#gwk#>>7fCw#(HS}vsfLJ&N#q6N&})rlScdXtx0dJvVS z@tnMhvCf7P-ln-tcxv%U42F!!^}%>3ziB7P&o0Jw4|WKc={n@*^oclgQ!$PmMsSBf z2`2L8VF0#(S9t9L`+2WG>xrFBZT8Ed&_W#M&YX@{Jj@0wTQ7o?f=j$D)+g`{oPrPe zJHSgvns5}H1j|(&p!TLaE|Fy+AoaB$X)2in{R$mGVx<;|d+Z8AZ%=?*Mc0A-Iu!h@ z5QEdx?ZM~U4tR3rHelxDNhYid<}ID;3dR(F081f+lH}3um7vR+hj+{k1>V<61rG1s@U7|_H~?_w%PBMPZb1QW`rKx`BYT@*-#%lq zzNZ)*y`~SXtmUv;kp%gAuo&N1-~bC0Bfv85nO40N;z^I2!Ob>j@>klE*aj=W0kAxAkUNPutCjw6|ISlT+zk(%=r-JO-rk4GB?_3A(pTMvEO-cR9bZ9=8oBuU> zF7S?S#-#o`4qTl`rqN*#U+Imb70=@C$NC^EK@Ki_qyx7dJO~tJC&9*3f3W>W6*4)^ z40LOD37SJ6;_sG`uvpQT{CJl^?7MD(ZLb4~TZ$L7zBf%EG=VL8#h z+<>m_ehr(s=q`1t{M2T99RZ3e>++Bt~mP32nUzh9WY6SG+Y0x}!@B|9FD8 z>F2SXXC0o}4qznTklfKRhV`5edT`}3@_LynbZGnoS`N;EtHT6>v5#FCpOpsThZ=zD ztti~@FGU_53WD9X^2Ge`e4@UNN5qCa;p#zgct2O0-0}~Hc9J!C2WL}Vz8b}I-ZG5i z?+ySz*FPA~eg;a$3vij)`kwDfp{p0IXZG*tLCN18I+!B6eSmTb>ptz#o4azyo#*h#7f} zRdOIquQG%$)+>>U-JkF!je79VzaKByJOQ4&bqmNpKZ*?;J#qWTUL{H9&FrvNqSIaf<%)S|X<}ZWCt##pljm^0D z;9Fp_&l4T4c37D382B)|s;U%6k$*YDs?v}Y6Jo3{Cw)@@VY`@jKJo9dFYoQ;J`j`e!9M{5B z?iLQ$@*$@TjLE(`*TI;-H(8V;iW};Gf|$ff;6m6eQdm`sH<^0Eptt?Fc$*B_%M$Us7Kin&=2z3&FA5!bQkM}_o&#}8O?NS(h3{;j``Z*TB~8n)tO|Ef=5aylTIB`vtf zAQ$AQxx5(_AR!}7AcnVyy#{j-8zrtJcb0*QYTLeW`ioovraHwbY349qV2Wh*z zT;H|3kRwJf@$GMs&|WJGYwhoKM}ao^v z{MzCL%k*rBQtNl@<`x7SCFG!Giv$ecI~hV>E4*!zClo5Z0@F$iN#2=8fZo^<)ua&O zkfBGKm!*QB=n!7?gN1?}Zx3R>Ws;=rV71^P<4*E8Kd$kj5`5OjBaJ-nHx8DDKh_zM z^)CMeB7veZ%Fqd?$(UlKa;CJ0rk#WD8xv2N62cq)|p=GZzBX7*dds`<-Ei$_ z@_-ijeqR+zxbMXh2Te#|e1PEPR1KVa+yY9Xn?S-7L6-wR1=dZ!fcdI-*iu%E=xO9) zo2FyH;D#l%{m=^*PU3^B^M3KJyG`T;JraSfUhBwtDN(HYWEQAC!vn*!6=3rlbt0N6 z3gte_LJ8k8Eb=G;AJ35oKRVWch3aPT@k)Cba^wr>mYFHg5CxGX9zbZ zs*`JA7}Ts-N>;9PgLiRh3y6_~mhC!3NasU8?>4MC`T}I!`Gm**L=d3x4ODY$s>M?ueKP$E0rfe%kNdLe_IjRzNpQ0`bY(sy-6MZtG)%EY|(}J zrilWy={UABn}$Q*81qWn9iZJoRrqh;xXa8X7l2aJESTby3|55O!^&@+z=b!9OgXm? zYg?w0M7MpeZ~T{%pqsI?l1@P*s;AA{zX-0W;id5f&V z4p4MgN$^Kmfh^zCk5i8`T~G<6a)wg%`odiQaG)XR54q2?BCs zKIF1h5-xvdfJ4^IhAkD}akJiD;5)dSyi3b%7PpsyS0C)ax4dgX+%Z4gbpqpGGu(-0 zf)s3szXNh_Igz`2_3>jCz)LRIz^vz6uxhm!^hq-!A9mUk{RJBEs+b|6dsK;^%Kt68 zjqusFQ9L#rkB|BXz@jEU(#)0%PAf_XE_u&@9zR|+hW25g^6|dwp|<0AVa-D5abh;; zy&?(UUhKttelI7tioLZ_hN88&B35t8_p(rE=P*hN#RC45+Ry}R~6sF5xx_kW&JF{ zr(+(tRZO159OGt0KbS%b=tgj76tR)EfMI(qLBVbn;xiZlHzutRz&AFqZ<_ls1w@fPf@#@$7gG9q&hwBjw56u7+T5X`ST*Rrm|1#~&K;J3H6NVMlu-s4C~(DdBg zbwoo2?o_`DWGuGh2kSM+EA@C7F7X1`W+f7%%q;w%%nG+`oeDb50EBP%fmz!C{2jl5 zgpDqRLz^SW_n?>9n7xeGc+SCREb72MK_rmkEX46Kj3*Abf!8nPTc(dJ74-SlbKmMU zc=lpDn2~*-CvVCpM^?w-0ny*CS<~~t$H7S?HsS+l<;Roxf+;YoW?VqWpM!~Ot#BfD z##~Af!RM9O4jNRxYs;M48^cx8zj^iJvrFTZQyCy(2(L-=QWU~??bd~_76^fZI+ zcL_vrXeC^5d#ymcRE8KFn?^bws*rQ8IlyIs3@Ltn5NCa8#yYzqx#o~5ueLx8+Q*#Z z*-Bc#SAXY1-A&V>*6S&_*>5t`ZJ9*gq{Nb0BBPi@iomNMopIkPbNCBx!|fr9!9+O- zExMfHgO_4Zz45T1;^ZB?yY41lx7QR-=@EgOUuNN1lF|5F|6^DEgMC;_W(AS)vn0y4 zJY2K%FJABV2^`v40XF?h$IsU1g8c@b@Wb99?6>AW@Opg_a8q|CjwV;|U}q?C_b$Ro zA0%PEpcj0D3=9eRfg9#R{9gKM3#;6P?O%lv5w&DERkB2IG;162;wzJ}8-4<>>Kyzb zWExzxXaq}cC=kdzZot8J6iM$cIimhxCFJ?4fMH2<^0zpWG$(q&fHYTL%kTo&dvF~- zjP!Z>5m}(XL4i!Z_e3z^w*d%PpbpPP-N0YC`ERShSl+|adNB02Bl&N@7KWxbV&fC* zq1_uuZXWoK3n%U3Z8T8CFP9JVyb6Z}BN~f{lfNU8lG22lGS2w#u>kV0Ctt9_WE9-b z9T({39>9CXXORxGf|hHXXXP0iORh{Tmofp?a*x9GC;kEF zm|56m(L#7%S_1B#8c9SC#1O;1I`HpDN$4y76PH!$5R(c^^7-*+5P8yr7qYtwd>aVn zX8Fa5)guYg^=T!^yd^78t(-_K8z;c=>yt=OTs~+kY23306O!)G?k zham4U5Jp}Q)Jm--#~t^yIKH4bUVZ_7nlA$9S!RKYhdFaG-hnu5S%d%mJnNdhTosOY zGyzk4X@M+l1d;RfVUylz;3cmLw%UwiGq*!PG`FE8K3@WSvVIC?7OV2kN~iKpY|92B z9XztZZW4T=#3Kn^489l;N2Yyp_jkU<_l~L(RY%0ltWU<>-Wnw1^-MTpX$P47#So_D zPbbFfXOdX!86nn8+N7nKxb_~_~wsFOK+bG zskmuD)I}9x6mK;c(pXGd_3z>f>0)pKcV^GN^Fg4N?TPQdSHRJZ2C!w-L4i+PC@(J0 zf_%*{2bMNJz==POFzlQW3D7zMbn*nixl)7t`7fFbdj{iBWpmP;KLg&GqD8hva~@=o z7>u0U0(^Kou=x4QpcX79YHPKi+@)$&v(LK61~vHtO3GNtY@c%*+FKk9mlk`g7s~RdH{;FO((+ zuk8b>bu7NJYPR6(q$%W^_jGWs;66`%G#WR(-9dVGe8bUkKLp>+9iYkKgFN4Lck$Cu zYdrJ2H8feP30E%U^VWJ{5E8c&Jj%8bY}tJYPiayir)}b(`&3tu>w6LS+k0Wbw_YI6 z)mSCAEP>G#Dj42SCKvP6i8c2f-cvdv7!@-Go|d0@nE}d>Iqd;urp_b!`?bj_=f6A+ zc@=WZXb)~V6o~I;#KJDIN|)^CF@m3_HsrB^64;Z!l;`JkUT{Tv7_ZzkeiR?muKTu?r(5yN*s9U zzKY1{Mvs&yb^)p8_I!qR_ zxUWqab~et!ZNdnUq-=y;*&>kHrw1c@GqGaYJm}2XZE;zLc^O-#Y9~Wcj_GJdNyZZ_koVNlE;~L;ioi)7a zwH73HZ{ZAvYq(HenfyJ&hMKb2SyPa#(rXSn;(TyniF2vkpP#G1EP;OR2Gz}L$F zrmN?G=qhz`U)2R)?|y<;5An%K$6Oo{0f}!-GK@KIKz`g@3XVr2lIUM0P@Oe{ESgI3 z9W^-;QJv1yzNSP>SNF6S2N}S-|9Y{+b$9qyoy)K=wE<1$f5Fsxd6*@;lsHsNknWka z!0VJA98L@;-*55o%%Br^$RZZ*{(Vu98tMf{Z!y?h*%0m&Ga)$pK5ks<18onr2=ad& z0*cDPc+kQf#{I@PR&NludZ&Rw0|%JB+YnmI)bmQtYm?7rvCur?HRiFZa89%-^f|H; z+a8)qmPej)Rs8lEZ}H%h=uKKg&3z(lRonq`wnxBjMKSPsI0Y+5Ccvh=xup2PWAM=X zBko%6im$5J!CAR)z=6RivS4&Fk#1f9qh3jpnrLm<^LPT(DYAqXr6PD8Zy2AwU5^#T z#&GBGAMnTG3ig+=g=_De#h(fTfL5;x?lOIbYcG2f(_{;htQP{#^hyIG-gfwv-sL@9 zEe?O*ljiQm7?E77E#TkxEM8sTWpGt>5s8X^3eJp7#Gl&T1+zlGfgj7}b7^Ta$c*?h zkTK$boqtNh@Wso>?S30_+ua>b7-7JT-FvX*gedG{Ig#87cEFL33GbcpQGt86mmmSE zk@suDVSV&N-l9#Rpt52)Jos%Jxh-#kpFVhw)tx<{Y1~G#vAr3~SQ?TAjRxTP1#Os^ zit#_^RiH)AiNuvjxo+rR2gEW9@#>2*Br;42yofg@%MPuAb91urvbcV{nH>SCUDrWF zOc8J?xsPY8o=(orn@$Gh9pTgypRpj;k0cD4l9;&>pw`I_I=>Bu6UW@Z1pgCYnzIS{ z7U@Nv@8j~9b}GUOxxprH6cXOP?RJ^(!A9FvU`h;8C5C_m*7ez37d zup*=k`-%Sn^FDCTQ^FQ*>~C($$yFjY-hL$bGq-pC^T1}@`LO%&B#{2i9uIa*#u2HB z;L|)=QsCnR--%0-QSn4@_(YCi+vc-iafdb3KE8t(>{22cL&{)0=nJ+F$N(2U)bsvG zq>#!zeL(eVI=t%|0+S1#;o(b~BrGrn-p@P-%uStu^*cl8wyX}{UljrltXd2o#unoI z1;bcuy(@m*Cl1dzZv`eFr;^m3U3h`|e0cHc8$s-gHax@dGcV!r!Ipqilfm3HV>ozh z0-W=f^KAG$uyn%#;K+Q%718!&k|`gr%h!c!LQOco=NEqdPmMcM8IgA)4{^tt6kMpY z0Qa~Az)0RFymg)(Ia#9v9c%Rkee5vElG+AV;Vt0FG84>~JPYFbrxE?I{a}1bE_V87 zN3=H{2SLXd!<;of#DiP64n{I~p@TR?=k@VbUInkn$&i>%jKzO!=EAZ3@fGm7 z8xOtCZWTPes{qe`NF`F0gW%B7+n9N$36~9A2XzTrFmaYDcBwvx4_mW>co{=7d-@XS z9yeL=IS=wcGD5C~4oIpKH}1a*E*$MP>PbDX#aIj|>NAS%vYEd9EId|37soA`g{ zy_B(oY3LmA?mqxNy?hTc68WTWgsaJ!HREj#_He164*vH>nUr1r4wkUr@v8f?v2*e< z@Th_ZbNx2KtNkuuu+EVbnP=hqb56Ou@%V-1R%n2keq&%wM;Wf#r2)@m@=2zaH}qa! z>DuJ`Ug+VP$LGv za`Dt+06s360sQAa2kkn`poY;*c=m$=snRzEq4#R>!l4+rWfJ#$U1!eOMp3 zU7jmQ-dF<8UJE9!ZlQScop7uYlMSpk#Phz}wFoLMOoO%WRCtNMUBG^&Z}_?GV(6-4 z0;;<_2`ZF=i}inDl%TO#OPZ)K#$VEl6xMo z;AA?PHtS5wqXkQFx%39of6EsRb}GZ}fh6Gc>m=CpR*CElS0(Re-UHW225;=8BH1f- z-L?3p2h0}}Cu_$ya`{F&WJKP8{Nnm6f2ar6iBAA6n)h*0i4s0GW&kj8`JSdd#?Vg9Zbr_Cc!Tu z#;#iTl}SXb2{c)jg0H0>2TRMG$h18UTn^t-TyRYtm|2|0y*^1;YAhHgZfyeY#?Nr| z&L|@Ejf2mY{{k^bZ9t6OR4nA&t~=KKw;4g*;7 zV;LBltWMs|jF34y>5FfJm%gPO7`waC$-nJbR`e z*yWFb?U9r>DiuN&sbs;SO}0d#${%*z9mWe839{~xCn;4EfVck6_zSVe2KPUJy~2g? zfMW`fs?j1#xg5l%d!azJ)*d!ZlqRB8Rp8;MF&RIkNW5xpgXfXvuB=&ohMwG*V`gu-vjKrOrhOl$TRk{Ak+IFf;;`D#Gz&>Hn!Uc zKJjM5y45oXe(M51{C*^Om#qTVEp)>tRo7!LhxdX|MNhKAeKTC7oQl%~h9qiSnOJ#m z!nrwXTsr_RfsE}>uU5fa@)edE06jw{^-Cczzn(0CP?HbUv>?zQ@6aY($-{H^p z-B_h)3JEGZ4}9EHVP?A-`5o*FI;yI$_K2?_-|YqGB-Vj(@d93O00TWzck=QUWP*47 zIf5r*H?f$eHd(gy4X?K01m2SM9xuQ93H;Z*2+HWJAqH1Gp!ohkJahOS{+4kZJSVO& zk8>vPkNWXBjzkDKle?Triin+OXEur?Mn zoo8n@I&@H?q$ri-NF|k0s^9*#fA4Ew&(6&Ena}&foVY7mfA}B4Xz_h~DXBh4Hyl_2qr_kiPP}dl`-5g5vy!7X3N$>aKgx!eRP#;*Zt}Ome{S0dZTn=OO=?XB|@DQhaC4kenhr#}C zRi-O#uI;_8qTbH-LC&SAF3i%`%bCpgF>I`441V)#K3gy#z?(g+IMLR{V3U&$bL;H} z=HcSmjIN3i8@FsZ+g_Ri>WiB2-*@6n@8k}=a*jK*(OHANZnB!0I(-T|7H7>V|Ix+y zG*f{&>cE0yFLc>?WoE2@oI5C}vk)20GVBI(Eq;4~8xwF!nKKw-2_C6f;d<+-j9L$m zaoejXIvc&mKW5(G9~YhbZ_RfHrgIg*jZMK)Xg_L5+U&B?_lj0n5-^=s+T_Enj+5yD z%@t(Vbxjm5rxS#>w9->L5gy%z+Q{#^;`sL?>baTWhv!$O@OXSdF z8k^~bz%s#%=kpbhY#c;<+n{ zTKC;Sb+%E+&rA<32v0;IS%GNTmICDYpAPJ3e=K^<-#|CI*^!$@M^S*vC-h|aI^Eb7 zO!Ybz!kxRsdAT~N^ukyf(ya*<-D%gM4JrBLpj8@q`e!vQpA(7xZeI@@`I}+Qg#wcL zF_kzMMxvS(6M0TP6sn843ubv=fZ8iYxLMT;>F?)(G)=w=mgK#G*K1qoXk;n*=CujA zF+bpZ)`NU#=W@4huO%Lb^k~)PG~zhdQBd4@p0-ExX{+>YVlvi7?!7&N2CO8}+D|&@ zz}EY8O3!*!@+y`JujmVB?)^iIKNzF>jxs13<-_p9^WevawN!nH0auAJfFbt3;0RBZ z9=zIs!qF(x*t7=ivXP~>Cf87XP!^TwZ2{ed7Eo&C0{F3D8dV&BMQoL=(Ebz&6np6$ z87>*3okbJSO0UUC)PEt^6R1rc=9i!q9c8F(yA`FkiqP?y#q?RpQ&fCco_FcYCxS2M z(pjZRw4t<=>X}}F8FOaRp_OJx&nE+_)UTm|dvpm$@*QgAE<@tGa#6|9byVP!411>R zMlGK;(BxG|X+pq0c2vxS{#Y)FtU@J)_Dk>61C?ns>60?u<`F<#HF+>(+D+1#SwU-I z1X3TGMcZa{Qa95cdSc-Ox_az5d2ZH($eeh=E&QA0yPQHV3NnyoQ=344z#dxU_oIoE zQ=rkW!{BZSSh5Q^mOO`U**8^HGv6cesum;Z+z}j!B$}iZJD+Bf0SI z6_L3mCD@$Ogf=?q(2C5BwDENj)tCK9)-DnV8uppfw{KGDyt`fWc>`GKW{W(oNrz^B6J5O(a`GyiU$*3 zL<5KC33uD+2v%^-csXq!C>pxOEl#~j=4zc36b5Lap*?Zrx34taYc7j2`Z#E*=@(*| zRfF!@E`Tp2OX%h@QFhcg53Nd=heQ`;(KW9X=!3y)xa+YuTs-kVWO_T7tA95I4cysA zZ*F=YZeS+e{y!{)#YKNI!<%Pq^ftMd8q=-?0d;O<{?IcyS&uzMqjYF$Vd zjLa3;7=AQh@G+9z*^V}P@8R~_ry%7ERm7xlE=sG(BKN~2gbK6P)4KZ;h3A8Fsaj|| zl1`k4hF0ySSvv0oXmlFg^;;cTb$mmYclOi&Oz#PN_yfZqn3WE(`9za*`_8tcSTSyU;>MIr`qLnEVVsg5<=Ekc+w=-2XP0u6djYrE2rY z0d+);dij)ZbOV+Yzd%i27E`6PG$^KWhK~4lqIA`R)ZghaR6T!%1VqKsyf=PuTXZ?i zFIr0+?dzbZV3OK}XK;2fkKw}Zv&8M?Bl>P`3KfM;sq>sy&~R4<+#9UP+Z}onx-7Dy zm#-cm$CigtWI7r8F(-(OyFGV6(Fg6^|B-0*%|rnbO4O!Hf@;1RMjM`9CKZe#4HGkm z!uMxs>*6F786Yiupu86qta#2CG-%MKf~QPWrvsPcx=FB8cNs06cplCF9fOoanXh?y zG03~-2HB;5jo9N}ZtGtmj9ih4;$CvlsjS_|(@>9>s+@s>`wU=CsWMvD<&2&!nTdwx zDNsj_G##mYM>P_zP!->Wv_WtdCPI-#Wu8nYTC1V&d#lK%ZGQ0DaXzh5(iCO~N7JQV zdr{LWH+1IKE;4MeAANejfxG%T;kJ@Ukx6-)$VPb4p34uACFm2hR_&slO|Q|&uLX3w zW;e~O2x2FTNm7N;TyDnk2E=}pp-~QJ1S#pa>8$B28JOJ54%`VPe|qM_%>_STg1i-N z@xhe+xquGHSI`Ikne_Omk`SKPLOz??Xe>R>)qW5`7VM2g8PD&*YHJtn;&bnzRB)tZrjHd*&-l&D5=j=p}{%~pM zPc`13Wsl*tm($P{S!udWFO1B4wiIPQ*+g@_bLq#U@yJ-W4z7GHL+1=NQ>a|aIIcFN zUki=tvBh@i{YM+vwtp8jJ$nyzTFLXS{d)xGc}+zYM}<&rLO7)+0`$&?LwG%vB;vwF zrnkfib!|RE8!o;@gNl;8cJXcW{G|{gnQ%mK@99x`OTUe5Q+f_7M&gO?2O~OKx|8_M z-a+5Ej*}IO`p{G>YqapW5hpIO5#C3Kkn|BP(M6@p=);5abS$|Yt}N?<6<%RZbs1+RluZ|kK4X|I2f2L^1YGg7p3a36zAX5{R zcqJEZpunygXj+*KCtS9oo95~_5lgu(SL)VO|ZXu-5@d}+MK19zns=%_4 zA;IB`G4Os2MN)IGLc4rLp_=0;?EmlpF15fkGed)2@4Exe=r5&n=?rAB$eaGMI*LE% zsnPhE1ES8g4D>Sm9O-zLhqUu1Qwj5Ff~-~*G+*D5e&}uxJd@o17Ps`hJ4Ye3BjsdU8|X?-9>+3UNh)0WHJ_Y(ecWwo>t9lZB}|bLcG(Yt+`!%UB(s zPqkt+(5QG7yge3z(*7iJ3#a`iXexGS`_iQ>L z(TAGYJ%er6ZRq|9m>6z(M(^Dyf>R!(LfKyjkeKRy((C(!WZs-F_;OlYsCI7$4LmTL z&id((45vCG?frhVJ7*t#`GJF?tu5ff#e=f1mdZjye z&h=pQcSAaCuHDU6^zWe|s**hGd`U7KO5ygasidvHnhXt^(i>XsFf(<86q>}-g<`fy z%EgMzoZCm+I@NepU-pq@&!50|-j`59dmGj4dj}V-lMz;RnbLfR8mgscKwtkzhB*ZW z+;A#EGj*Ftw#a>|waG(9q2HnFUps`JOcHh%%%EA_sbtPyMR-b=54*N0aUFpqGEez| zd}ci-^)QQ$wdA5RK6Omp?qC89Eol0BSF~TUl8W`5pfkJt&}SB-HA_tBuAUq^@=2G< zan0yo*{S5?%?@h$Yc>fu9!qD9q|r1*Gw#|o573l*->94Y8M=7v3~j!niWDdH3bgKW zQ6*=PSSa45Ve_tYTiUEpU1>i}{PsprGD){l5Sa>wOOR>O z8+1kCHYyn3jMk4Fq)WDr;*yC0$Yb{luIqmjd0TbQ)7G#!dgV|&Obf`PVohb_T<%Ts zExuf|=iW%Y3xdeW7slvM@*?8tbA$eynMDorwh9t6hmeL=KN{P@rN>L=3w75$Mk<~~ zP|UN0jBB4DX6?`Ee->9rdD1w^myV~qlaslAe`?WZ{T%pxYZ;wVu^J`(>Zcm^w#0vG z8trdPrI8O)xxYga=pmVpbj=C{`tXh#mE8PV(^h<@DovVc8 z?V^y;|pZx=Fphz-zfgfZX_|W2yW8OLn^nM zk^5o^q5g$z5|#T}Fj|m>_Wcz(+I^BjkGN0tXZUwgapWxy#Cfy@YpkksEJ3@z$dp@&c8W4srzZj z)*_DPj~yhZ4yK`3Pbp1%PoBHqd=&9M)J$ zp;f1)dG}V?Q($$Pep;E1P97SAq9$u((G8LC-$A;|H3$hMTTtbB1+;r+1G=lb6{$~p z#nfJphKF8<3QpRUqNgKe^m1)7%54!s!FUhVwJ$>}%e7$=I89lnDZF2mImB_hJiPES zRA7>53Xf09rBUt!NbRy6{iUskdN%k8LuzBG{{C-ND*7OGEK;Ejq=}x15h9--Ib^Uu zk^I_|1%W5M8bV1(_LmxTv1IkS{}zE>V}qNu(61aaroS2>)O!z&jqO4 zCj|kSV)$Dk6Ky==3vcJSBS}9^w1U`Dhvb9k{22{8caD{yUhx4})BYV?H|~U%JbFqJ zD}-cJ+lJPt+oCRY3AiM*25wyPgw7zFY5#pavNC-egHr_PjDa??&yYhpi<6I?Nq2&nM7wCAS7UL1$}eJiu{o% zL9z$lP*<<}Nc-w!;ltyK!p_|{Xvj1j)Y=wADg@30%j#C<9)cZfp=ZnmOYg-x_kSU}!Pm51Xm&l8ROM6P}LKDuBm11hBEqSL+Y zBuq1iMl8LC27;E5R-OiVJMS0`GHHfN$~LGV=qxHCRWz|Di9XJn3Exk6O19teN9FhK zqk~`8p;p0UdaBU^<@uNiGv;=(sULJuQr&x+y7(yhYm^M9s%0U2c|#Z#*+m}7?xg2} zas`j$MFw0c;B9`@L#IzWPW25M;N`_eXt%FF8lEfip>a9<*JMuvo4&wn9`gl{Ki{CG zBKIzpwvwYAs{~hZ0IKwqK;Dlixjj8zgpzZU;0eS zCW}+279(oE=^oXM%0NA77Cr-zyOuY zT$8(@w0Y_h!Kc&{v^eh(bQ+bRSvQMlUB?VM?5|5dYG)A#a*pu2M4vXV;e}jX+E+DA zV70YgFlJmulI7IV$9rnj^M($zD>0@0fd?qrF^V+SuR#0t{Rq9-CAtHz#M?AB2Q@fE zBgM};-2JvXB%8BdWFcvxh%ZlIfLAy9_D@0h`Kq#Dx>7Oypk73a`{j{sRzHbc&C-%5 z>cXN*b!xTc7kybCMcrj4qT>Bw==bn*^p4|?ZcI}pA2k!n{fT$cC0P%G_S}U@3ChA3 zesYDnj-~0|bUXU^p$xYn&EK~J<|S-loM5XiouO=$$%xy)VPh>gW8~S zUJ`xfc?N|a*@QBF$nvgBCDQ<-h44~)rl2+bFQQ)@xy{iIaG>%dycT4J9w+-k^<64Z zkd#aAUE$CqyFT)LOClZI`kmdmFj7zQ!9cvuPcO_W>!!oZYS5>+aJ9>h|n)D9aLdbfXpS0gbqE^P{v7h z!7J~bbjiB{r0?;Sx?DR49fJ!5SN;+jD$UZx7mw5JALN8;qo>(p4a!tc$3{4R(H8jQ z%^k8w&_jM11)vn;Ksu_m59z6xBd1xhv{vmid{$A;2o}cDuk{+}j?zo+A{b3eQ40KX z6ToMSDv9;7S~5B}4E1hjP{o2Xw5dLt^r(C!FY|TK?j@4E>Y;CffWMolmeeYNlaw3k z%q)kq{7+Fsp-}X@{^f4sy3+p9Qn>n_fPT@R&Z@{r@*WvIBeFW?$VdJ^qHstV@lNH! zZYg~xt_q=_m)9Zl(VghhhFH3Ht}^wVcbzv^fb+HpxG<>+{g38?p3lhB^Iv?kCC8j)to`TH%=o zvuNO&BJ%Ok1ff+xB60J-fC_Clk+e(z-)Ti7bqPLHIAwyYd!%Wej*yg>Sqk5pR#7`P zO`vqSnhn|Jj;^srq8Z{+()0N&WM;qR`tQDlyF`1RH-nGpSm$~wx8*o0o$wf9k-HSA zq)Tt?NEdJyjuGRvS772@O`_V+L?2hmp<_Plh>4vzuSm6?9({5H-prT*br%VsJ>5$j z(>}nLTa}S*JeO>UIzvW{Uy-C%WjNi*4{|{)YCZf8#$1%4tJXWy^qgs2;rA9|@Jome zwN{E|44>(zigOfh_{kk^m8F__e(3DnBEJVcqXc+S1(&Uw3RL0cp2kUcRxY&w!%FPFJ*3&IDg9b5{bHyKf#d zzPlcYPj{fz*DliglFo2t*b%C=`2%-wV++ansUlF!jS*a8hoG2qIE~7&AziDk!?;=@ zbWOd8ipPX-)yP8{?UI1HQz3dnFT-6qi{X*~qG(y&OwzL=hxYvi@WxR!LWU3woL|D8 z(T@?mtGbJ9c1NRdmpZhfJ)Pt(R6~CA%D9aW&rwIwcMm#qitaA+LSmPP1?GFE(K{B& zL@%#`^|O2@+6T#!YHu@UT&JDXp4$qOvX{{9VZB@ncpRPj^Ngl1ekpP;W08Yi1-i{h zQ|}L_(N9GwwC%t=!OqA+D*ZyA7wpa_O*3P;XSVqW!s#`UE0l_w9^8dJt6NBV6A#1LJ3LUzpJ0+VF_3#`k`7fYo==UqLC93BgYLT)fr2x=1ZuaZA~j_L z%0vut@uL(}o&K2|^uJ4|9$HHZ)_);z%NOe3vz|F-eMc~um`PXv=@s=|>Cks}TWPjk z8ro1h7aiI&1~W^i38#x$AldVFq}0TpxF(DX>csMCbYVUHy=f&aZc9R1WIut?SCCXc zrV)8{NKQUM@X5UrJ#v!by)$lut!ATKiy#cm*Fd396hu6q<<#R#(=-9NB5)<7*Cj7?}@u-0NaJir2#E!5(VvyNdMddkLmg zN$}qD-ot|A+34E)dem@Rv_EZ?0@y5vj)MJg#FYnZYnJ}vX35b zcBBt7x1ddr_KLb+`e6OwIbongjz1GKv-np!Bvq0ICG^k*cHz72MVXQX!%Rl7^%fYxR* zHAoZjBEwPdvm?~OX#-mJll1r-h2;Hz1cu%o*V?F-vS4r^PW^S{p1 zxgR2F++G)$80kmjo9EH@O)cD-sFP%aT92TvR)JUAn@26c3{+bvkId@L(eRAfD7)7b z3QZGHWvh=s`)q({j(LKc+gAruS}$`y!YILtN@x07tql$>)kb(r1X}RL92NA{pqgtC zjqTrzibb>HC@RC79x2){?Ky$$FD9a6Qy#*Nn?4EZA8&&vUM%LV<@-V}!y+i>EkPqK zTWDx+jmYyGBi{!flk2sr-06n8^l$1tLLV5R*}uc7_l#Sl{muh;jX6Mf8!e{8 zf*)k(FLho;?I4=|$pcldJdRdQ(ia*?A4M%Pr8K?%9h_WV4!1WgLfnUg+(S1fqDR-) z(rw$qk%f+mFzmNJ0XYV!@%(Di{Nfdz`e7cbvYbJC#$%Dv^c=yaygETf10OoJ`lF?@ z{}JupIJEY{ZSMP2mf{zSc)M67G;O{T3Z6GWMVC#qzjHbTe(w*(XTvp! zWoqfW{$471It;BYSxY6jWhf{%1ZI!RqEAyxS-z_aF-d+b2y+^PBgQFkT!x{lm66=s z(@&w7)CA$b1I6f{)m8MA%s{0-J_vkz4QWox4tVuI2IX@ysYB{t=H_4mb-TQcx(So1 z&Wkv-X@sSAWo<~@zme>_;egu48;Scgb?UJtl&;iXMZaHLPPP`e)6!T0TJ5zBwf`wV z*K411H8l%SnywoCn0+1%)p+qT{UgvVONLHmBj`HyRkZ2T06n2&0L`ZOQ@62IXdyO$ z`x-8wvbPF!*OyzgIbc1sd7X&@oVU=R{O8DE_yK&Meul10Tfn=weG540=!ZUruS4ez z-w0~NCUJjrW+CDta;?`NBtagXaQ*yA=!j7{YS>~-qi@`!d)^6P;D$0<*j#};{2#$k zW6HheCAw47zK)8LPw4NS@pP}XfV!RI*8RRBtgznLO@aU~xdSXKc-7-r+O|61y^U}vedc{=a z9Fis&e72Up=nID0QJ!d7ix0Y7KR`Ptyy7Zq#3Ml?;4XZAfO}4F0ezdG%60W(Nw#k$ z(k=)^tD!oYx2Bp6ZJtc;u82S+aS|^=<`|9k4@DMp(vhk09QsL01(o0LV*S^YAgO65 z=^}?;@WRDkG{#~Eeb!e47p8I1YfX9TwOv)1s=gc@7kZ&{3k|5ov3mq&598iXb@WPs z=nUnXgnS%iNmYL_taYs#`Lk50XmcF&J%w^ zpw?S&USzE;9ozp51;6kV%(d_2x`!B0njt#34VA!F&(-v@WjX5Ptwz^1RM47x$4HN- zIju+-fn)P^=!d{adP>O~1umY0+8Q(IzbhAs?-fT_B%0rt#V)2xJ=~CU@jY&Zvk{tE z^o(Zd{)TfTKhrDvGf?(IP2^G(Mx(Y*r1l%{3ts&He;d(T&u>stWiK6A z|A;P1+W@7Vxu|x$jtrYjpw)uq=zw0LV10oQ-qpH5*LX$Kz65L3v2GHT72S_mpLCEe z=N}*@9V-M9^90CYs|G!JV2rMNpwIo$Urwze9zx6MqHfT>2k;bJM?Y!3CAK-%s6O~T zJv8ARZC8jux=q6*aMe2EEw&IDScQnXCi7wEj(GHRslM>@6CGin?|W|Kjt`_z{u~tE z%z;Cuh{jq@ojt~ubLVX z>!!_{J=+r*Vj)b(4n%HIQm|U8geDgLMWIuT(ZM~A=tyBXu^eB3a@3zeHx(loye@bd51r5!;!X+&>8)G z6rLBO=RHm!fg+-^9;c{}-*M#hfIn|)AAe;Wht2!33I(D;}Yvs;$4S_{?Jg{VxbhS-^P5w&TIHx0rJ= z2$z_w#hw~vVDRG-X7AyNOt<}feBZa4Z~c5a8<^t45wns9W83Gl5-$}%x6E=jF5?II zFYGF|e|)&s_qs10ah}f9MKZwt9u5p1wPeg|-T>d( z#kk|*cP!zm$E-bmmS0&qf`=~|VE2Ju&a>E0_~=d!JElAYJ_JL%@}UWw>$8IJ&QWFd z(5?z_Sq|W(5B&f*yB&BOX#@gEBQTz1&+KC)Sot1Bd@*_lXG3rynA_^e{!Nr+2bC}L zcMMVBZ}JYz$#7$m5XQ&MCos*Qit(iU576K}Cn4aQHf4jQ51aUF=NEw`>r|P5piMaHO)$vnr~&mWwqxX^j{jY$ z1=F_l;%N29KWt-Z@WUwQ~G{%JIdFsd*ug`T30#uRMx(N-Sk1CR;IkK_S*i zk!3@Rhw*u>N$lNUGGIJ+GDvKk!eo~I#0%GTg9cMMHgE)B?g3Mfy4sbED*t@_Ed`7oWnc2^x5m-4(wUWzn~^; z4REP#=Bx|$VP6bq;|CwC*tiLE8TA`$7}bzUut7En-`P}(y$mm5Er->0f3N)pd-rX@ zH_txB7lNy>qgw~qzw`voS>_2ML|x+5f_6}OPl>H2-!NU40Kz|2fVSU*c*oN@V9`Gp z@bjw#^QyE7$2|=Pt=5z9q?j^X)m{fIW&1h7<$7SPPY14ci^M0BFY*&QrP!i8hq|Cu zp5Wl|db}X?54PJc0mk1>!tsZ4aDnASka>C*TaY&le(syagmH`T#g>`uwxd$aU`8^& zd9n@%4UYoR)SG2{@8j7EFN1Y5dW_4v`+PteE9Dld`c(Al{%kzW9CioW5KT1(g|KGT7z zoIcJm`VRgpUy8BHKfKvpi*<6+=Lq-a@s0P%GhZ{5@RL7#!0hlKET(i3%O$4*llXZc zdb=KT)omx(?`+JJDN3ycc9JicJ2nJp0jeBb$4|yzbq%gSfovBFALwe!jHy zaqx259L88MnRSkMf%o1r=bN^7a9n@%0Pt-JKk$?szBN;nFY~5bZUWNh11|FcMdapY$`AA+S>=t#Y;4S?x@NWCBp!Kc=(|>#>Q(%U$dfXM@ zwr&xgrj&y7hwkB!Z2h`%y>d?3S$~}S^c0RvP-SJ9Y1k&jgQNBGJ|{7ED!b~p9k~8h zyVkF36|MpR)xgZ*MYKd{>{zf1`FJ{W{EH7YHLn+p{y&5?B7%;}33E*tHH%IHY1Us{NGg!VR z3aI?HV7j^Q@szq!?6Cj8YkRU`vRxo%qa-e0-OSHCa23yT z+6&fh*1+#hDly_c75Fe50J}x`xuW+v?Akl=_-sDLYaTc;Z#3nY=X56?3GxM1oA2UH z`~KrUSBwDRb2`A2s{7b3EQv3rn}!qLe8YA!S2@~R;vi`CeSTS!1iK=u5LA3MWRk4r zv3-RiZ$|q)&9Yg3EF4cGZX*YGe2^m8lY=NtTg8`Xy3FF!?p>m){=AA z$23~NnLUoow)1U#;mL)}=&f4*m5tZzR8v#&y&LnHnf3G7FWbdA@)@`B&<0ty%2ff6 zT!{mjat)ZDat`d(`G&VvW`i8B`OMSTrP%g#C|H!23vNY@fTShb%)ZW8e7sqKb+af3 zKZkkj$fQ+xa>yqjX7m>rJeOzRD&^v7&HC&@lck`2%VUuV8iuE38{pv|zRVh~B%62B znJ@WfC)Vy%=5KvG8R&SQ2d`b`Gq%n1YK@;C#QdeU%$lJ2%=NfoFh3!Hk5s>ci+whH zC;m8RpjLB7Wq& zjk(v)Vxs;4dwzd~walI`8(pxp=tg^glNDvRywN>!`S9N@g;q9QI_$uCF zngG_ie&?(_Aj^!~I`QMu#Mxt)9)ddFAV{j1#GJI|u=mdX<*Z0(09EdvfqCs+{@1Pf zxK%9+FS}#Ks$#+#YPF4*CXxjUSh4AtZKfLth!8-+~F_4nB9Xfja!9*gBIELh7MA-MhL zPf&Pm4#;uq$G6>z?M`mb1sYkgbwzWV!G`Cq_+iN(AiixJ_k=022`y{cJ<~1N%}Z{8 zDD{5KKd_EbbP{40ywX;fQ3zaHL%@2AN1Qm(ntoQ%2(PaAjbE)(25r+^*fU2bGuxgT z;&1&z5Dxs9V6VxXwhmJkJ+{L;TQ)K+Gu4^Bmb00sC&IyVNiOr=!5wGbwP!zOXJdtt z+aOKu9ndyaVME`Gv97Pv@sktIV8J2>)@9Q}{5`w}Y|S0F8<5z;Ou06ZSt7>;K4R7Q z#|sIzdx;wk`)LE@EmJw9%bBqXzXGiP$+7vD{rHQwrQ#U@?))kL%KP z5f8nfAXxAgk6OJ2*D7y;O4V2RXYNFH!edpYzB7d{-`K~Gd+yDjfhV$aH%5a(|1Nx? zav^iQM3R4{=o!xKRbXe&i^p?`CvI`)u?kW#Hn=PXmWGtr zRo7@?rvPWL*D)R(ERN=XcT2{4xkhYLW13yOs|jHG7h_eGBkbIMD;)Nsf+H7l3`@NC z;;j0n$y!bv1N>RP0baBL$fWo3gW4~1q<(e+g~>LIRdhQzHcx|*%IE|4jMlOl@CFc$ z8iS>0O7M`}HoP`gm)$;m0Ea!*XTS+P=9tAl@J{3`X5^*f2O5jmlH*3q+FiB`9I3)V ztr_^f(pIdr@fSXDZ7beqPWVffy##dR3<%`p;-xwwkEg8#97&D>vZufD^AGG`e~9<+ zqiT8h``I0Qse#=r&)^d<_^riRI^BWgY@5#B`uLr1Hd}~gx2ymQlfv;-p)#uqU-A??1p^G6qANE$dFc%>fnx7GT5s7ko>r-*^;wGh@<&oV=o~_{J%Q{g!3K za7)TKg>}DxlY%50JeL*ymuv$gi8TQ7W`U5-M11t+b)fuIlC|_c%YUe~fVr?Qg0Fi( zl2!BR=0vU6#(#s2*cZ)O%C(`z5xWrzxDA~>R-frFPpIF zQ7^Fi`4rrGaTae62;eh;4>`V}5O3$b0~cqlWhYro6xrCO9PhWP%!LFY@LzroOyl@+ zuqYcX{Z)e{UfbC}t&LdA^(Jt#Q(=xw;jz`%XW-O)74~CRCVqUf6aQDM#5!wAfC_FT zSW+qr8edNmd44hApS(BcMOY%R2)AaAIQ#?}Pwd%_>A{#6IS!-^pMn!1L%>#O&i0<4 z&fG!&F-yMtW9iAu+3M$)z`1b_0~7)Os`y6stNsKgC(evXe!d640m_X0z%tIy8=|#( z$c1f~(uD1bazIj%Xss?Z!OCw{S&td+K%4)?PG1?ar}sUvKtjR zzVMvw=5yuXrK~q+Ks5+|(T@k++p1^SF=1|9Z{obS z^JDLSIEW96BaX=%1!naEJ-q&^D=v1s2+T9ygYc;v*ag1r{0usXy9|aoPNMVX{hBba zK4&GCyvhgjVkPjmj+y+scMkJY_KD~W`BadVp~rVoh{X@DjA27L3s&{pH}E9d6_nc_ z#Md6rVE)!L;+t7_frOJGJ0kiocoQ8BzA5)%R^N_4H?R`u^;)tzC#~5LYiCfuQ-@|^C7o#Q3SVlYtpN|BAl4p0Z-bV@M&uJBwwaCNyr;dZx z5;ZnvRV@f=HesG6h61nP3LJDP2j|Htuud6f%;ZZSz{dv`Y-1?FO8<-)i-tt-&_*AG z4WGgn;w1sxw+{I4PQxBIwVA^2QjDF0ELf^{hrj&nN*w1#`7a8T*_n43+{CnoI>FWx>)20mU`E~{H*DnD4zZmG(>a#j0l7aBvKkKS4 zMesgOmR)j4igog|VjsTMU_3L<*1f%-1a1e60Sm2I98z!xI9poO`Q>QfiSjZa_uwv2 zuh0t8l6{!$2vhdo&(%0JZVhf(c^fRU* ztjqL?p8cQy4A=`Yz2HOhcl;wPp0A%i1(=zg2W}Iz89TlnR;)V-Zmw8>_sHJ^hl7GZ zCvMY!Q zp9N0!e79SBRgua2ZO0^}U#@Emui`%rWxv+a_C3flpF_u3uAA9_I1a`^$;=~n7%mR@mXn+mb=q?qOryd2WZoI~M z)}L^t*gSUm#2?_#^>83*)`eXTYqB#M_JO?@qwr0wCct}?g%ziBKv}2**rYLxW!r|Z z`tvg2>GuRo+h>Rmhxp(ZezN%4z$A9t7B0{oISX<;6*-C=8SqD58@~%V0u}^k;-_W~ z?5iL4Ob77f9}09}-q_3H2e1g+^*h>qKHrWvMK56M_P+=H-ywg^UwaUG%AeU~;>^0) z*|K*RKz8ZxwX8?-BXBGKHfP%G`|~JnycB6AN(JZb`PFlaGHKZsoknYT>x)bYl076M(q;GBCVU3gF9s zz+V3^Kut7r+Zn3Fta$K>Q;^#XRv+2~_PR_5na$_%C#m;X&EKA_^4bn~1q$pW-c^28 z-(7yCPa_Tpdw^Hf6x*)9IE_7CAaXgsyaW7$_KbOY7Zxwx#SWNV^R!8G8KIe{@u8%tyhpSd#*CHk6r=BE; zpRpKgK261o??|%GZcGPF+-~sa_7eV&@^KuXCB|%+(G9q!U3gHth%+r60?h^5{K2Qw zz^b@pQPZ^$+&Ah4+`w?0%jp10Wr)d7v0-B0-{Lg1_VY)Y#F_X~OU99(jtAt-8Af9q z|Ig8V2h{le4*+k`9w-`SMyXU9bkDix+y*HPyHZgoME1uf(jH1liISv3Dedlg?m5>e zijb_xibNSHM1=42`~BIU-TU10yvKRJ-dEN>1O{_PbK4_qp*|f$p0D7&u`+MilH>#z zI_Ps9hA+r<%Sy62FF{2nN=-FOupiR}iI~WEbM~ z7hHLK1vu|16q}T~a%Ly2xe19g;dmPvc+*r79)HSvGdIqOI}09zn_e+wggOJ^qq0D` z!$kN|J_G#7I4!PdyGTCmkmJf!E`bYAjEVAp5cJ$P5w}g)2k3e+d8nyD?f_kGS>9{% zvPu>@_Z|Wsrz|;}GT!@&p3H4}URgxr3rR@dfZgChW$v|{KDYYub#PvN3Ot*7 zoSY6;=89aGlYak4pk=}%aaPNJAbCzHvFwT^8@4HNzT-cLm)Fb#vvT#}ESnx;bZ)5N z&dD(H_&+`uP%WwLem<0YxK53)eMN)E(a(vB!?T%bd@bzR2;Ry(W=y67H?Ck*uK_+^8j70nA^oYlL;WKy%DrZMex*A6)xHw!sp`~1?t)U zcF)B5;))f=!S93R+z^e0@T9pGh$xohieqjPkIdVkKN!Q>XiM(r zQZw#YNtQT;P?B2Eux#+vdO%mkk7U>5(A(=tNF!mO7mKM-!x98O>e$@R97*9>-NZ zIwA%!=YVIP9RIvqL+6AQ;QE^t#CWqRXKi_gjB}TPCA>@bP060yv|a_?k{W) zTONX!{Q`@GG2CP4G@x{RKAg~h1k_$ZaHhKq*Oj!8i}#0c{JQb5<;F;E)$*0x>ik#a zt4FJNb3 zNXC442^I%k6em1Th5>f(!J_Qx+|~`coZkTNKbg2d-&EecpH~Z>?>j~m3oz(;UP#=!(S zthP4-z1lL|!n9EMBU~RIy3E(4b_H{HRvUogJr~6bojXa3PdaIvJr$acQW3wleno1J zULiQRl%%o%D`&II?J*(z;s2P0XW6PbkmkoJ~f4)mJ-B!SQ5uY#Jy-l6nZ zP0P5x4K0FCnUi5S_W}$rc>^*g+QB%_HQdP=r@*^?>Ee()dG677FL?i`EKF#<0Sw1a z=0+^}4{DA(MH0M%$jiz&@OG({U6zdo44p8ZbM8Qq^ zEVHwFHvu|dwy@i$K9Njza~AAeF$Pu^eFBRYa0Hyxf-y!RVDFAY;zixE#J?!PZuj|( zaMlQAI4OTLys>vY{QT(<(OT%n4ZW~{OT8At&1*(5I7P(GoT~{nnh%18ompZ=#dH+mx#G2_p5>5LDMoQeyRT-)GM>&#+|(hz8X~nafk_L zul|}?F?BQ&uIWURn!m6_dV4Sub zxA>tUS<)2`4rIB&&O@WQ#x1wV$*yMdL0yK+l?w$$r(XbzgBQq<)(PaMR}(m-J{I2E z(+BYEn*e48fP}63@K<%J*ygMr7v-n}Z6hq8aJsoT_)&tuEbKkmWvRg}zpD;jHFpzq z+D@S9?*z|ml;O6_7{;xkhVW(O7-;?UjNs8ezFzW82WI(v0=Zo!#37>}^k?#}U#kXp z@Oc|pUL8i-6C4Hgi(it8{Y79*b~E{Vdal6qwQg-=h6QKVtIOF3dV(EaG+<$uDGYOX z2==`Zaens!H}vy(&f|P4nEp!#RLj!Z6_dJwSGftCwBtSTG{)TC#U@;;f&s{$zO2@+ zdpb0|_L4M}%R)>2!*&5>lexZnW!M|rAs{>XoO*33d34>8ta9%H_kz^mkT=2N9?ubC z^Ql@Kz4Qo-N*xKytoWUCn*!lp|9-OCR*REXt3gS61L(Ga0v~;SF8)eKZB6kKPG{;O z*rGiH?)>wX6wFK`?{+(Z67Oho;?X$HMkj%|IvK!2rj~H&f1NemAb~7s_XYhk%fans zE5+Wc#K7p;6k>f$Uz~3+5w7IlH$n}-Rd}x8GH+RNA6jQ{(;APu+0Wuzc>MIOk2bEp3{Kc-W@>ezfN){(~;OtGvItXT*daFl^pd70{7NcgVa%# zBtqy$^t_ef5qAMs`&o~(6r8S6Uf4)_!iSR$j_-lrV|6ah{583ILl+izRoRKJ{{v&s z`~r4TU(nNYhj8i*=is+MRm@26ucGf7ar_&W;_#Dql?QN#jBAV+=3c z+6V6P|F?^&1`IsoE54g^l@yv^A>Zxv#b-C1CAoEvL3CKFm_!(JxpC8B@OWqF;bs7R zm95~8(;A@A3_lCa^f;3ZDb`aC&Btd(Et1*S4i(?r<#r z91$x}3$rKd6z_oGFG}3b+wVYDuMG^^F^$VR?!=KpX(WD;Dx54TB?DkQ_qXkbxN^f2 z5TEp$xahqTuPj;5+r`CXVzC6QIZ`dQT(`a2Ovevq$8RP#O4Gr=JIlc|e(%WyH-6vN z>=M}oeNjo{HY1dZ}N1WBj0x$qcG=q{TLW(Ava=1I<+i(DmtzBP)2y&cF6 zpC<6bYZx49oklwIMiFm;Ele_WZJ--0Iwew-^i5%eN)k3^gZ;H>`D#0hFw}|)BKY|`m zDzLGXhcP30>-7Cyr=deE@7ujRsBcY~X5sABL}k z4d*pw1en+M806hq3gzt7xaGa-KsSB^_^8=NN>0eY6mJcf`{o`QuDFZSMb|*$^mE`> zY&mEwvxa?--8sw76GZ2^U$yI3Kmv_Q!RL5AxK?i&SYloUHtrk4*<7s!F5DNpsJP1{ znBTh+H%yt^Dor3vXB4;Qdp*d1Ue>}DDWrtRqg@4L<6v$yiFwSOvjz4*8IbX+W1>zG4M zJC_O!w+!d|RdP_I^ozJ3kO58dBe}*@A^iDlP*8X6Pc5fClPkY!4Cm(-l6^b%VZTf$ z%+ap|wDn5uFJj1PXsUCgil&h3UzUyY3iyG&$O? zqQ@2P9eSMT1snp0(n`SA-xDD=A>vbY6G&58EJ^vr?+Xe~5l=fVCePn*5m=pnMkZMv zCd>X7*4kB#hT<$IuAlb?rS4nBx$Ze)%o)PC<|HD?ZWo8#nFT`HUXpjR)^JEpEl~UB zI8*c(g7^||kY!;D#@+u2B9jeZMpZ@a3s*-@x5yUC#mwYB?D#Fvb9w;IMlTjyek&jb zdb!}jg^}>SmKlsYX~jJnqX~^1zLT33*T@{Fa*$vAkbF?ACGUP|!wXanu9v+?R&x(Y z^QSw!?faDEtS>is_U7KRHqx#F$<+^`$#jak~>IGNDJ^T`5g2uClYsx2dU&pA#;7; z%It(~_~_zPe95mIPc*4Ta~t?wHPJmNWoS7*(poH(-_tAXh6h>DB?ElO`m<1(AHx1J zun%XeKz!-lR=VJ5Hr=SOfob2dlq_Q=Qmn8J|JW9e9UcUsy`D+zv3Vs*YAq6uZ?a*Z z3+^HPr7PKBV;b8z;W8E)Y-9Vk7P5II5|ldZF5A8JD)V~#k$QN4#~=6R;LNkdOwn1) zKDzuz9Tw?Idm~iP^v)W|+vnR+^j|kz@pd!qBNA+ybd$A;RPmOe7~s5VGcuH9&`p!e zXvDr)cJ2BXnsR$7D}Qap)*V_+>j!h$yt$Wfw7VIr|L+jXYsz7}%~ms)!%4VQxfj_N z#j+zK7BJsOmAL(WBJK~|&n%R}QBG$ptM!azCW;o=d&^0xSg4LQIZYHcx19CO%EVF9 zLi%H?GP|cMPu-iwpu^4E@S@LmaPZm;w)=z?qkk18hI*06@0bp5ugPa|I~}mL=Q3vg z=>yFPTO)K1n2m*9rA#+G1z#IlAXIqE-@pETRC3zy2(_N$foE1aGPgomHfBf^di8Y} zs_;@_jng!w5f!>B0*Z5}Uwh zjHXRxqwk%;-L0P_!+hjK**|O8?4UdBXNocYtD8^T!g;5q5Yr_)BH2ofox%}TUU(8X zjV#~3LM`@Xc%I%bI`)h%ijBE0Y%-RUay8z}_xLe7*5)~WxnC344q1jeE9LOD^JB4O zLNfBIzsG!&D615mXFYp@8B$FYF53No`|w_qY0Vx&GqeRXMzjQFys^avAvUy1P|dbG zd*IRL-I98%h1kLOGYWq?$Z|WEvcGlyIOItot$D7EWFqvK-mn~WQSeXrsW+NU9J-fw z9DmL(-93leW<1BUM|{U&&wXiRZ6X`BU4XvdJ&QMc>*IrCCb19GqVRn03OefS6v@gk z6VzVz4A!sK6s5hNjgw3-Vs1+q{W)KaiI^0f{<{v37Dwalo0`#}Rw;Vey#WUsrQyzZ z5@b~8&79AUlB&0-(qHyE(!-m@EVpb2-fCrr#>ts8lZn>SKPQx>W{WT3>O|g6JiVON zj42a#@iP@mk2El@C?h1UvDTm#z{6Xbs&t{e;GcZ-t!6jC?Om^9EI&NJH796aT zxV8O;IksoepSlL5|GfrUwWZ?o_$+(rx{~SB4K&206ZI@h!aw00^xIHFq_XxoyIhch zmycW_X%GI5{cj|*T5loVHNKICzfwgl-yPX*t$Gqam@OQ;H3xf~F+eggru^?%NbkAG z<0Ie4h+eK|bcE7VY~C|YI{NYeUU2k0UUkPFyNpQ?%KlBp^9JPToduoDef4&vzUvTO zkt&PhH4AW{RX09Y5k+g!6k2>+g-bfMfsJTiA^B6-PbD`a>D@FnHs3`F4|g1ekL48M zU8(PxX-=2q0q<#?2)r#U9N5k~3-5&=Pt;4Qt6kXC8EUvlbG&Fy)&QEB)y?R{3M_gg zkM3$cz*8cg;`!YYv{3t~r2R051NDwGn*-H&>}dX8_7x|*cHvdLxky*kV)vQ#c9o;j zOlRbE-V48O(82ePe5K9*Y(-@YZ=tZQm#{(QY)q~t;#vR9r60tOxJ*dV#>5Y3M8a_V z*e8kl$G@Ou{5f`MRwACKE6?l#o$#X(HaP#xD0=aHBvVp8Nk1R4W2+C@GB2wu^t{pm z{I1SQxNVgs?s9#GL@*0m?=xq$cW2`Cxxp+dWgM2vU5-Q>nrKpfA)PS&1a^C#$TXku zp2oiQtjWlU#f}!CK$m08t0f#?e@pOdqZf4doqsr8@Cu#(c7i=yQGu3i&lB2IoI>Tp zb*ZAVMCfK&$;MADV$VE^gi&$Y$Y7KyE|pDWi7Fb>oQgzj7$#tQ8}jL3cN{fn3d7D3 z>0IEI4QyhN6N{WAW?7qT==+UrES>dZeCZM!Kll~jG`xcoI~387ufZ&2egdj1nL^={ zXnZMt6|SA=P76Pd!go@#nc3F`Ea0>cOYiJNF2>_<(XG=EoLNoHOO$c3a|WH)F^grj zJf-{Ev$&9!W%TIFQ2bAKD69uM(!^)R==~FI=@!Lqc5dVYdct@a zEdRGO!D0l(e6+R!IobsNow@qU2MxPY+^G3r<?6fi~pJRbM_Ac}up zPouph*uAqDTLDvcqVyw*50k?Nr?xQ#xp%^XeNuMt{9UG-7mw{CchjKZV{wX0Hr=k< z%xV|+N@TXWvU6qC!fEY}H29wZt3H#C^)`NBe$@%s<=k*wx9a}?zt1-*LzX_qh%E@1 z$nwXuv*k-d&@WUekv^Wpe5O{hy}1EcXNe7K4EW03g{tU`$px(7zkx1FQbx1gt?=Rv zRj|Nf6q=^nj;6Aqth}y<=_U=uS3At;HKxV+Hzl&wRy*nM@fOVZ%0V15tQRf!(qeN* zcq1GYgXZ#|sa~=syCS%akB*GNpYDC2b$6qX^vnSMb1smMTV==+o8PjtK|}H0Hbc>< zvzxJ*PboVid5Qmi4Z*1uGuU7AXqtb{K z>Zq(}m!kr-~1#u(<+-Y%CET{-=&nU@f*eUMe&W z{3#q%nZ_cA^>AYqKXED>6+|ApG+1%$JE6tcBGztk9;t5e#@9WHap(6UI_qWuyZv@I zUej4X8{g-$knbV5;mRxQHtZ>XevsT+YLD5Rd(^XSI2wBT9Q$#Q zk6}(LGi{UmEYxBX4csuD#lBZTI|CNrwK7_w%iX@{#^iMTy`~OL$ukiRt(%O)9bIs9 zH$b)>64v0@kHaqCqag_{EXZ;(OCBYM++V)HEIx>B?CEAsSxNZp9b?+0J4T>rcNV9; zR~Ko`QAfWk-GwVee$02a8~a{<0&N`F$L8pP<1D-&k+$5E}V= zIeJl~hW(mV*k-N8$V2@QH63+^wq%;IYv~rEu8UfhH(kcKcM^nPbH)y-h@CgH>omgCkSb zh{DSriSX@-OR@Cya=f+rqEIWapKC07NMF@?Q={zhyw_ZYYCUwNkBkp9+Xu<)EUv^p zo4feuGa74xA<{iFm(U#ne(c7AWY!x{h-V!7j1)F*qrc_8u`4APsLt3O!lEhdxXv@4 z9Z~2PT0W6MgEg)!7P_-fIlKpaF_Zl>xnUdP~q%W z+NM64-kxzC&p3+N%?Zy@)xjjT;^Q)w@|Q#TZK~3z(S7*$h-ujW+%Be6pvZ#m4nw2+ zud(ST+v%69fo#m1@ffyhN$vhF!d<@}v#@Uyv0_#yyLYq#`}bbN3j#$N-uV8`7; z@j$qc$*$Cq+}3@%{>`!j{} zLWbhm@mr9?x-vF(=?u1HpcPH&65`2OCuz%M{w#6j&DN98X#;O$$tK$4yNO9`-u)Jq z_Q!)cW}U%mnL_Her<~=dl;VS%pW@WKaX4JLh|OCBaJ7D$B!9-B#I{Nw-9EtQT=AFa zuKx&?bMIv%cw6X`FpiD?Zv>i=bO7ye(Gul!E@$^__Or0}P3+@XEmSC?IQyw34U}7r zx1Ap@5lpIN`p!v`+To*)p0LJ`$zB5#XUVjck$$W>G)W8H2o!pIa#ehKJa(DL|?@rWk+m z?8f&mtKxf{3f{9=K`L*iFNt;<&HfbTuUs%{-CC4jhP*@ zOfaQ^8F#6?XaTaU3c#F0xiBk+!&`PHu)&-tv}mCzj{SNY*Jg>acepzG``(52l}u#| z>~rz{EvlkR`;SW6s=RTus|w}h+h|_BGq%(EO0Vsl#M>I#>`r0|s;lv3QMP;VQpaKV z%hfhEr+vMopx>LBue&B$y(bGDO6;Cf;S%y{SH(t#*ZCQ4KV~;t29iC(jV{0#bsQ4grk!KcjrA#Z!x-9@*Te8Bhknda+p z9>R~aj6^pT4RB$rIs1D1C4N5j0y7-(k!>1t2gkj%!)zgB{W%=o|Fww4%>GOBSC6A( zqK6_4bb|FYr?Vc@vACD`qf?(o)2@i~_`IG6>%Csf#x11OGBz2wCh6xUf$eA-EKA@WqlnA*3D)voe6B8+*#~3`Vd~0CB-*3S+n1t?_-l9 zRpEC-S2kpyE){S7M;my5Nq9p@Pp8=8o^6k@`H{bp;o6#LGH(*COQ=L@h0j^wvR28L z=D$csSyejk^-KEUy8-^18$tE6V{y_;1r`|jQ1~PM7<;?xI6Yi;fi-#iOa59G@t+j$ zLVfDOlcG+u*!40Zufy?JcWWm4H}fxj4x?#vjw8(49M9HI8;+NqPR52q>V;>N#mwzy zB2#s_&ng%9G473)NU2pu^wYDJ>S%{DnVg&0`@nzPb}*URA4+2<2Tj>t1z)r_BnI#P z*@xuc|3E?wJ9>R!3ma?p1gp7*qA4L7xWzw#K64CaIzdZu-N^Asy8R=)^JX;Oo_$`j z`QtEbbX`xRqHKa+vmiW0dQ1Xk_Ol!NS7XFmXfmHZ;+KspaLKBJSY^long#9j5S{l}1iwCyvZQvXk2A zqWvpy{E8O*ZIMfszO*-B3k$7OVlqCeq8ARsaF@?F{AzEWX{s!<=ITfWbZ*& z_AEkOVH4SL-iQdwK7uxB)Y7-vyI?rdJ$whJY3VWB;#3sXoQU3tRM4u2?yPy-7*XUVaj+1-FV3Rhe|@ImxoNa>=390q zXd=@(nn^8OqNv|h58NFESx&k&ju7#`XGIe>bJa0=x%LUZ{x=Gxt*Bt&#(uoBsTT?B^o&+4R#o(bN?+##U(qY(6gSBreE)bg?mh> zv4I%>xU-5TKlz3{=6;sgx{abGZH0Kwf(OD#kjdLP582t5vUsQYTDE1&f4I%x059P! zO-m;wEW2p|d)0UXFRjoNg&jy`t%b5kNg)7#FIvxLY{2Y%oj#p@z!HbJoujvJT%g5% z>U5qZ#@dNt^gCZ~$vSE#UGF;#-E6O;R%?fg22UM8*T2bNxkDOkz@h@#Kf8gi=d`k8 zuAkYfv18eajtk5uIsiSho*-S4u%3Ajk->(G{g~f1QyQf9ZQhbZYYtj4?40ZnNA#!RGUsf~`G9m1xcj$j5u^0Ciy1?IAKK0DSr7aIN7|2TYpSB{ z`=M#fNn+=(Ka1}cKW+hHr zJyq&p?n%c-^L1C7S4aR$QR8k+WMh{@>nr}l3vL=>cilSLy;{l^SqCvYJ!>}n*#kVF z{6m<3T@h^+I-^rQBbe$>F$KQzG4I#`S z%7DFJt;bB?X0Rn|)9|I^`K%%E9-SMRMCq=3EYA57ic8#&M4M(~^O)J9NsERvk!28; z(~!X{{9mJw_I7M@h6=tJ|3DHyGy{41H{z!5=(X``W2o; z*M_cQL9-Ir!psMUx|OlwjHsytPT0!EpNR(O|6QtUS1 zJRxMK#rAagxJEqlV-CG-HibL7ellAhAz=k-DX1)X8cV-$Pq^ex82xeXvSgLdb-KCi z1pZ+nrI+9TM$5k6KmnH%7}~o7T`D<%e+{e0`)8@4_`kh)KjT<(NrEuE#fKhRIgU4s z_u>Pl>e8y{?aU-O5NT=opnQ+jc+x-~`>gvKs5z*C)i*{K-@ z(sQ-Rc$NGq$<|^2@S>P<7FxvDBsU(W@*m#P_-RQjjkl*WoYHB;nPPlm5TI?P3hWtQ zZyN`D>B)uqXhXUZs{~>exvf}o=XEK)wLO(33dHnNKs{c5*c^*V8JlNzjPaB7bfvK` zou&F#lEl9gC;u(LAL|z}t3LocpBjx5mKWet*MDG-^1AT@eh?W7UBI}QTWx4 z9Hw{v2fGs3h)fnB?DUksq#YB>OxGRbbG#hxK=m%V_nQvBJN-HyeGResKYj|$!Y|@O z_x7`?bL>zYD^yM1$XvH7F|C7%%yHNV>5+eC?AeIJEPL5VyuGrA z$&Kr#_AlGeye(>|_O6W7Ph~Q`;uj>;wHU`vc-B$7hPiA~y9dT&O>y+6i>&&66nj*k z%q-6)<2#xDwDqhro?7`CO>amf*=zRWPZn|L2EVr|^3FO8NhXS_-Ny{4+DqihR-=xg zU#UiFBwGJRo(|KK$1Md1@qX(JykyrJ^e1o{oo=5);|={pwq~bra%2d*mAMFA9XScF z`JPL6s;M!J$Yi?2PfzqIy93KNk7I|ACZXR@PjYQQQS?A~i9KENO>*f@I-5Kgf{yEt z!kO6|b$qjp&!LrBIo|_)Q)A7fN-3;k%@LuDLo-qeeoW6Ao@H_0{vm^+B3b* zQq~@wM~~mlphLRV@TFo`{B27WeJD}KF7U6zR@9sAh#3I0qsZm2Zu zpf}MN-%q+NkG&c%GOzKwnf2Cv89a(xAUi}RkUMnwYI}lH&Dg@GBZpF0Zi3WX;u!YvSoX1B-`{MFNyV;JE z<#dPsGdg6I1wME;8xNm*4PQ~=4Ze>Pq~UUYSeN!vAJ1e~Obt;3F~@r>`&s7CJQmJ! z(Yb#c@u7oyEaat<$RX`26CduwIx1zjTCIWI;P+fLz1zu{iY{HU>=4^CU4^;J+AwYZ z8Qg~vzR2amLfXG56%Q@7mqfOIMvm1xQFIMua!DFo@3mQ|KYSl=FkfH|U2g2;WPNs# zG_jpKSJ9o>bA&Ap!zHE9B3X-@JuBY43-65F%YLtw#Vc>MvG40#B;H5)`uJ5f(cBP- zr<2{+!YDg7IHeY8AJmi%Jm#2g#0+fK6@VJwbfMcJvUpfR5DTk5E-~MD4!fCJ5AXIi)b!MniN@aSD2Ip&$32iIUg_-J$_~ z*Esjd)!0`-S!8=yhKc8D;Sx_Bd|}ocHg?7xto^1M<=jzYC+kwsph-6qEZvI}K6~OY zCH^=6a}w_N=|CCVt}yx0TZH%DK4ce;jmJZJ_OfFaA`08vj5kd9iC4AF z#vN_7IMmpZUHBJ=M{c@-UC%v4Bg|v4Rn<-8{b4uVy9-O-_dlomFRW(Onyc_+t(!Et z(o#Bwuk$W_zYf=p)?y8peAt--O6>mB0rn~}l9*^qL=%okJ>xH#(L=|5a_wPeDq<;*zkZzaNcJxC@&vN`d z46>KrVUoDr+01pQxzzJy6t(Y0(l5&Dq8U|2EXc`~DUbDK*VFFs-riNF`MZ^>ZLgIC zZ7rwEmx-}z>|}Z|GZnvXE~44~b*OaF6)G})Cwv@Cv1UgfKJo89zIF5$(<@bDcl?$~ zD*sEwA&D{ETFvc}j)Tvz$E!}ddW08xo4cCbP6|f2p%1@l7=jg<4jwz$gbYN{I7DX@ zzG8m{Rje>WzXMY6iT<%{qb(of$F5<=s{YY}^rgbm;gNK0T0V6?!tW3($)qoO)=2s` zJ>ku{3!Kt})wtHF0DZKI#*1Q)`}-7+ut`=7<1Z$1*&~#SD)9jarfWRo(FrV zkCPqE2wsIPTHfJt%dgTU6^F6u_U-s0mxgdm4GTB4MV$*ZP>)g`ikr8EdB$~8GUhrm zcv#HLWwThf{%*X$dNVExctVdX%w*f+N3sX`?eu*|9^1zy3MWn zTkk2UY->R?oS9Bw%8UpwGucy0KHc%^_A9!NM1G;RlJu7*1 z3jG-PhXxi4MP0iSn0@X?mOO7Aj_t`{24Y2K5?hRw%f6xAsFM!L+~sQ@`|trHV|;2L zg{}GZ5XrR|hz=e*$LGvz;DY#*6*c~GDQ<%$)Y-(aY_zL_#U^d{}NQ6E2g4#vn65Xb7|ATGQ6)Ml3fhVML)~_ z;?1XHvF4&_)VVPR>xcN@YYp=m{E;ji$H)3!CZ~`~{UA%Yd2gte7q2ebunsm=!*#uSsODZK5ZW6M2`ThINs&lhgrQAy;U zJR&9^j@KItdROHN$=)#z@9^8nyc94}YojmEyBws_i z!O-zjxLN;hk$VQCVZ*CGVA}bwK)_oWPVT$GmmecQhW0GZ?%F#+d_=tX(!_e=sa!s@ z@?0zUwr?U#@wMc(FJ29g8+dT?D-V;1GgG)H6%pX{Jvs8yz#PVO6p7WR6oI4J5YCay z5u6>b2({K)!PLno!Q}l8pu}S`l)1JVY(887KKpLrs#RJ*XO5?!e?Ec|BWogGpbxfu z_JV`omAJKC63~7157{-IH%L}~1@-R+$+#_=WZS+(v9-Pov}-(jF;j z-jhnsTXqnq8wbhWB_>>7nH-E=a#9RePKUY^f`I(EJ78VxcDU}|2vR>zhP&sn5&V2x z0LYss?baGwnSMA`E9!B;@1VB*for!#urgN;6qb*M1%X;zqRvX_ zKj{40?js|S1EjucPxx(IA-tT?4LW8l*k z4ah`KKo-~^K`ay+rml@9; zaj&=A-O)|1Z?NN*SM4U%1;aSY)NoGw{VZ@lZZtgQ?g-)3C*txmpTxyRBDlWU1wPIm z%2^a!b3@H}1O-G3ZXD<%XKdBsq%nPhvIEPx-OvEm1imHq3&uewx6v?kVzoGP{CrLY zECJO$M%>AFS-@-9MDCMK3z<^i3NCu_^QIBeB;}?(H{E9dZ2B~u8}Nw$+j!DBuYV4i zk@Er!8McN*ZVe^_TP?}I35LWZ#{_sEoDC1wd&21&-^tE}BcT6(`DBu`k_b(e!JDeb z;PL#S(CFD@s5E&R**5GikukjtX5H0<)7tleu8W4m|5-g5mn_4*8yF79ME8m%g{8zm zs0S+@9k`DZYQeA-HDpo36LPV03Q3al;CArtlwr|Wf#!Y@H~jJ>E`7NSJmN zI**00-Oz#4t~cXi|MFjq-xo;&tSbV6@dupXz-ivBwnt_j42%dAtTxPnl1+j)_D;I~x>>%lMgw2VlfOXE=B6O%m|Q z7OI9T1BEp`#IPq342omIn78kotP-_h; z#ajLrRV5P<=?rDiL`Or5Nm}tnu@q@oXaKs(* zdY3a-x7-JO_Hc)8dV}K2_?ca!;wc~~sUnwm?*-fX+rS2o6kq`K!5oVN;>%4bz{R0g zJly^_*m%f-JDkXyHcNtl(&Gu-pUIwrAKtA*OLYvl5Uv!|Z#0AFR*i;5{yOmVzehl6 zML&V=n%w-~V)C*-Q}Ac41AL|1MT&Ra7I?Am;Ns+sAb->*VsF0!CjU+$`>lG&zcYrM zzuyq17m;DQRKK=z4JwoBGkB`8RT{CJeokoa_)C1sD z;LbVjL-1q%adF68V-nw_4#!&oXu04aDAQd8=|?v>1^xw{p0XeqUIJSx3}AJ`OnA7z zpDa@7CJHgUp_DU?yLER6nRfLT2@1SM%qRShqC4@2`uXDk?mi;9%MlVHR|vZ^`D*KH^nUgiarQ1)`6`=l61NW=E_(%( z=cj_C#ot|Z>fInId5ZPrTw%quMw~Hp6x?;8e5QECxWCYX z{2JZ>F8wuvBF^a;$IaAhUte@_T9qGn-R1`_n4bYM3J&3+z2@+N_%)chf?G%WTd~X0 zf1phZg1xT}fPnZ}@W|=Y*jPgvtM^VJxAbPiDf`ajqI6?e zp5G4wJ}f8a&KBbnBTw+)@fKzGSDV0ZtLglE2dXIlsgLv^&TW zb)1KdC%HqnO+2#B#t(MCRw725-tgn)2vBwOf_RZMoRd5UYAnBs2lN~HqU3O5S+O4a za5Lu)ZvolX$+xH24(gJz zKF7iRHWRXaz5)5;2FT50-$8PlJyE);OP+@CNW66&=rTBq|1B>9`_34`uW7%q@T3q| zvy({9=}7QtZVWMyW8griGPJc!#trhvM6zD<;ox;4+;b9;sxVDrrjZGXG8clA8`}7l z?=Iu7gZjjL-(kFaX9w8(%BYD^G6jUmil6C7x6T7(^ zMaRwiMf3Z!@scogFymMl)>xMYW?D8jB`)j1g$356a0rm5g5TiI=V|(@vJZJVFotQ~!dO$o>GSg-)>h`X*2m5J9YK zJjpF(Ex4Pn0`;B@VZ(w<(ZUf8aIJJXsULvgxLg+QOZ|iOxf;THVHY3V)&cX~e}cH; zK+vA(3X^N>peC&4`*B`Pk|Y|AmO21V>%76*u^+I9uQb$*vJy#qIg>T+EM{Z+_+Cz} zz$|1MmTnkN4!%9lt@=jfBRBM9`@xlz428n4ngLKvJA}OYbd(=JY>4>sVf$`vX9 zcN%ZuPb7Es1$aWED{NTHz_ucWT+jZ1i+_&7!A&Qz^F1+`Z&QX*`s_PT|2JnZ4>U7LA_Ch~C?H=D`J zoJZK7z9iSi0+tljW2M~RIO(2{tchp>HOjZZ{#8$a_#PkpARbM>mMw?xC%JL8{y1_Y zMh|LMJOyr+kN7R4bct6rg0JGw;N46wzI@J{lt<`_+?!0ui%HM<^R`dH2@e*7C2x=6 zx77;dW8xQls)Iyt803 zx#`&j>dSP=)do|z!Fe2Sn!G#tn0x`u-Jk>)m7BwsNt4L7lL*%>Uj-(d^W&8iOTdkC zH9navgmNdR!cAw5iTT&B;DNd#sn1OUQcnHAWQrBMIzth7rVoMqXZ!G(v!ltl7MBCgg!P75Ez{=1IKGNvGu2wa?(=j@5 zb-*gJfZKnx^B(h0=uaU*Bnxzy4}ue~N5jtHqj>VfX^^jV8Ee;T!c$gJVEE-6^5*$3 z{))FfBIdw-{K0dn!tI?y}07m8~nWJFqm9=k3W04Ey?|7hTE(sVDtcI zK20^?{j7=rrl0q6vOi~qpIijr9gKyW6(CXE^aW_nH7B`Vz2IEGF8pQZ%=vRCMWek> zfK}$tu#(?Rp!QFLEdQ?;9NhL7^p9@_Vkdw6vGOS{IlL3cEgb|W#X~^5#TROS83e0l zY{8Nu8S>6WowJ@0G^@VPYwdL)Tx*teTXlg+8IVjjQU;sKrHT1&&X&E>Eb0+9;Pg4A zcwu7~*q2d=w;Wr6@9ECMF#}t%!uBuxCPl>aJ^vX*{K(@g-53ow+_5D7Id$0XqCL#n z%aB;koA=xB0DJ5o!L)QHUS(4acB@T+q3@pKFP}QFpwk`qKF%RM$GdoHUan+bv@!4M zps(n8{8em3PJ^VsBH)pC96P`C1TRc2fh5BL-q^~6Giwcrq(;GWrCPqbHKAT3Iqr#*vJ=e+@s6noX`QKPC#jFC=$~EL1vp6p!5;f+g=3V6*!Q z&;wOs-v0A^XNydc7wbq8L@sdAw}sg58VfS#`2dHT&%g^cSGeApft|IJ$w9L$yhdpz zyu*DrZg;eZ{3kh*Vf#u{uveO7C*Q#Hn|^}Ui#! zREKxt=O6X}^-~T|_0e=9JpN5od~ZBwn5g2Jz538B*9#YgOoW!Ro`5=fPjvL~L6O?O z0KVIOf03x=1(-Qw4RKv5fN^H_I877)lNNK&z<@E-xK@O@ab^&{AOVkCtxMFs^odis zJyacK2vd z(M0WW(6vXNyq#~(|9oI79M*6lXLqSWmvBAs?T7>$&+h=E!^UIjd#@W9`LQHzem$_+ zavPTyH-L)m!{BK4eZ0Vn+m}Z?!*jJS14o?-oUfn?8kPv5b7vdqdb*083rNnd$;3H%EdR%n_rT}!6zK7OG(7A3S!BO!5&UYx z8Lca;!1iCyK-@krB7E=($kYN{&OXSEHo#d6OL6o~J(%caCwe<> zHPBi0706qt!3_uhflncQ_>Y+bIJ>8mZw?~x-ugy7X2CChf{qFxyLORC z6Q$t4KYxi<_-Mc^W-r*Jl}HK~rr|d)o`GY|6XApK_o8M&AZ*mB0Zist$Sd0l3QGZ* z^S~IGj=769CWet#s)w6~)`P+hdq{bfWKYK^eywRCKRj_0o-rpEAIW&e|8A!a52XcT z)stUAO=}uI=;wZ(KyNOvFWCsnq^CkDJ3H8U#|W<9G!A+ighIt_#lTyZTRT3>!yh6M zo|>^9CsbII$`k!y|65h~w@rqq$=iTOoPF@y{0NY!%p&hcxXWRGJMqEL8+hRu6==B7 z7v5~KC7zemV62KiKK{rM-ir7Rv_6&qo7b*H)uNc6xTy|LwLAhUj}76rfk4+kX41rk zU&WVO7KB&BE}W+|4}8kniOa*I;L5mVuyV=-B5vM;N0~%GbL&nJeXJg@?w-;#v1knW zeV0c*XQlBaMb6N4-&kO~V^LKb&7V#tgfjg76$fr(3oU8wU zvp$o^(7F`(^ekt~>8~R@c(q_47AQ)0bzgdh?FnQfE{lTk$q}}Z<#NEZ9BR|dmlpBJ;4t2 zXlN3hE(P)>AsRaNK)65Dfx8nui`>1I2AzTjz{3sO$=3Z>6I(-H_P`-^EN^~SfFC(G7T^ux(3nnimRwRNmAFq}gko5^>q|_t{ zEF9xYOnt)1&NDV}e&k#54=*Ni*VYg(3p<#6>I?8{zkTdW|34y2(|vVfIrz&BYs{9Y66(el#9y{Tk!{tnhiK&p?;I2H#ar z0A8yG!OARm>}atKM;d&@d5unBb#)isTXGjPZGFrScAbXDorXmC_%S%~Q-WZ%3Aepl1?yXm4P?yRf3 z-vJ6fQ0!y!7dO1K0eVL!lW=DPys29o%UG1+NRI^Iu<@O!XWuW7K0N`-B?ZD?;uH97 zzZsmurh%$ocYyt{HJs9P5WF;5g_}VG$iEZ-hU6jq~u1RSNbN2R+QMm5v4?;@@^KX?hb$zyB}knwDE9&h-tu!QKDWsl@}qSjdwLo9R=bKQU5JGlFAj>VMjG(d z;#~ZA+Au$9{~S>9%b0K5qDwU3N4`P9I5Ml}Be?Z%6dV^T!E(N~Bwt?*dYsos{AQ4$_6Fp~7XjnOK2TFp*(4PgL9*1>L!i1GTWM?rUp^1wrnAMItNfa; zf zXY4^NpZY^nBG4Qb9PxwcazUhI^#ky%*%BzYs*=xhU5Mi2Ii#=m6JF-d^*)bpAYS3-kR5Ku zeH+7w9|{4}bf1VOyyEc%iN*MX-!f1=ehI8{TMfTT{7A#QeK=6RNVFDQ;UD>XR}_2m zIq+GgPVOz;jgM=4k?7tRSU#x<$IgzzZb@$e&YcH!`nrJ6Lhf1eo<^i@nZclg!SM9i zVQ`i2MJgRF`2%lFh<39)sXBZQU;1GRmk(;gOi zxwQc!1wZ1sY7)sxl!Z%vnn7=^0_?uYhRg}E;G4TIfu)01q_lQ6kzc8bvxa7oZ~uyc zsh=+VsH#hhdOTpE(-Dj|hvR$s_P{G48()2%3)D|?8C>&Z$hTF-Q0HDEiJ89;JEj!k zc?wZjW5<24r{y0oG95+SgO-ufW4D2Jpdy@6Z*COrj7$NGWDRlmZw4AfIADu&E5M@_Yk}}R zB+`{t;HgP72w4yVWxh@!J9~cN_$Bk;wx_4SpWc-O`8S+sP4Ojde|o_sJQ+$G zih+w24+JFZ!Te@_VsWFPsWM?L`2OP&9{u=!$8$-EV*fZ8QiVVB4fi`fJ;~coX-Na#bZ8pU zw=*Tyvv|bul{Jj{`yD@SeF#F`E5X8khUmRt*R)zsKw{nm6NSbQ;CE^~ocKWlL~nN^ zzWx!g@Ol_Hf2IY8$;Oi+FEw~m=Oo|AUk3gN^9D^t3qY~o3$WMY0v5m4h6x#oB!V#@ z>rR`1vTr&~U(+}6nNP zVE4(y$02JWeToZV5uNjy$&WQAJpChHf7Ctwb3EEFn z5S172tqa~gUdrgJ&pRdEK%?-)GO9b9Ls>8ly40!Y?6?TqZ z0&~=5iBH^RFjiUv<_VVpsX+kuJ2m2c9Y=swT^`8VFaT`rkK)p$M}Swl328EwhkLfA zN{>VIxAY>+XOgM;E{iE7l63uR#rtd0A zZN!CDjksv-A>idT8D{-y1@B#a;J%f%@ON!Gel_|qKKhx<**tUw?9-e|u>LN*FoVHA zzJKN|(c^M1Yufq5bR7R(getlDT?xMVTm=-Py1`9f9-PrK6+Yn1b<_TV=0q(>2?~xs1{()BcM}it zS~L>yO|2Dh%ErY+Ge!nRIefwM*3Kb&c+*MnmV@|~jyGxW=RUjFGX5*o4WhQ}Ad%Wa z$EM?LCqYtLEskt31!+>i=z%3?2n#Gf-Vr}u$k&u+X^ z?L77!cnu=`3P6SyOg za;HoAlN@(|wJuVmX)f1WyVZiHK9?o~A7e;NW~XRtU|rM6n&tdn;W$va{|}ZqU_>VO z@W>0{9gKPvphkooPKv(*&h*K{11);w!!=`3laGJ|Py^g$!JWspDC3X&WZ-g>iy{q9S_ET)+Gu`>-q3VkLbvsH(-<Ox8U1N5Xf~E_mFtAFKT)SopzUgQJdLS2{ zPjA5${ck{M;s*YTh1$H4r5V7~{4DQL`ciT^SP>3#zWd6ebfTcR6`uI|70+LN5|m`+ zLz(H>I7q<+uGka~PHjJkm&Ue(*PnXu&T~9iKHCjsJhz7D6DN^X{2RcY%LzNAJeN59 z_ZOJ<8j&XsPUQ68zgWIS7Q}o^!-_)=3m3G><{Cp7y^KB4c^RJ3lE(%1XC8SfW5kzaB`J7Y+!83wQp*$sTxAB_ph+;MJ_jS z)@-u#WItYKzZsv1dnHP(ti^AV&hjq!t|z{HPaHVMjNi2)o0w<@!+2&9>Cm49?Vgu| zw~PEl>Vxxe{GAoR`n@}L7p4=h8R|H}*pIv@+Cc7qTZ8lWO(VwcA=ozcDSlWvjUM7> zQH6kE=3x_``MC58d-`<^12@*QYcpOjZ;a3*A%f zLs^E=Xms{@>b+nOJ^gG7iuVjbU8;}itV(Zz;tg%&f3Xlvo~TIgF1UbfoLQk8ssI~QFXeS~(*KuGC+4a!@Yj?Pb7hQgvQvQM_eA@RPm)T4JhJ3DkY zbN#gxofAXY@jvbhRPU(^?Vr2T{G)Ga*}Ow2!N7<{Z>$wtrgSiJU#GB1s<-Gdi_P>w zb0%_=OJP!4wxiph{m?d7C&~RU6X}9DEyl?62#eJo(>J_eIypZRnXes<4n*rQ3T~I# zNt1GEm31j|_TOu9z$zu-z$R&=KRFHk$WNxVWfmx;rC0E;e=A+lF2mh{pUiA8T0v_( zmQnY)cTwm#ieiK#Xnch+W!HvM!vOOf*Jmw;IrM+|v z^=C=YCHA|Ol_cD24cjS_rgl9$=)d$FCQsuaJMBab?GHFASiW9|ePdI`j<%9Qp-XdU z;GsYipRGa_ZpEP)pV!j6&p$9T$~y6NF8jS)0Nsg zUZguJ?CBYuKIr{@GPU)XO3URnC0@=kZ2Bx^cK+S5=&76;+VW2hRTvyVXQro%*KEuY z`v*`u_~alRy01pVx+-Y&w3~tn>!Q%_?VLf=*MnSZ)6vh6vFwjIRWxm#w6H~7$~BBvq#u8RA^FS7$oVzj(KMKXm$U8)sM#c70-)kYZHF-P2y9ClE}MeWyAnBT`oqu*Z5Y?e+Fz5C-Cb*?mG z%pUg+z2r12_f?06j45D`|4~Af@6WLO+H)vz;egn6l>jBlOc#_+tz?IH_t49E zcW7w*AXD+|1Ra^&%dYuPOHw#nOK9}OnZ4H+#9F+3OuvP%q^GCGFfvY2XkniZy$Y+? z$eu#>^M`geU;ml-d@7>z^*2%9qL1wUlnxrt42yT)e=O*#j0WxVp3wHhd1zJiQygEk zkor76%J3fC121D=u~x66n3c0GvQ-_L>{iYriq+4eCr1G~WAt%m>zf{A;&ql8`Q1nL zq(WKc7iz4xwI&^Jzk)GR&P1Id#uDYWDJ+cINp-~yC@Cu$p(T^3OhPI(6?-AGlu6X@ zTM^aT|Bvy!%XtO;}Z3r`7Aj^RoV$-95|O=k?Cbu+cr?Ys4sMWxFMeE0XHAcmbD_|2`&T$?nW1FH_`jHvO?R9#mtKB zOK7K-CQXdK&H6kz#-xwkOcx%Km-GqFvxVE_#oDtQY2BS1w8YYAISoJ?Ps9#_s{@jRS1E%zUbs<4TW; zoluY3PAavo97(_5#@NS)iti3&((1wg>G}9wR9HZ($`GMoaA9mjo@3NOdg(vpX zS8qqe2Z9AOVcR-p?*h)PP0nNOVwISUIveSwJ$G2=%tAKSVGk3Nev1~(-OfymOc2=W zY)7wb528x*`>4-Zh0Xrpg%*$%Xzt6Yw5XoX$jEG=8doH!`O+J9=z1kvX2?UwbGD#U z!(Z9$U!0f@O)KWNj2+sWoQiH&J!Uhul_ImNKgD`qhr|PCofz?lHT2Tz|ERj^5aWlV zY0jKDl&-N11;k%v*XCWLcN+T8KyxnL=Iy{1=~tm<(_^UmzjdfQGnxIqI-mJ!^P8-? z+s1r|OsBCQJ6Pu22Ijq5IlA;BjpkOS(alaZoP#AVF}GAihE=VMNB&+U>{dXNee%%k z*~jVh{r@pm$$n^2Vk#>8l!!k2w6gEGIhz8n@pR?$mrTPfj6NOgq=E(I@N6SQle%9q zI+q$)BfENJpkX4scIG&3FPWMi=(HWCwx>+w3U4Jl@3e8j4u&et~LDvs~&b$Wk z(t|muTWc|U>eLSS`+gdV!b(B}6og7!!cmvsN2D!PO3}|}blc!dCckhYEk3)8K8tZh zJC6Qi^3>L{4`<6ucArM*XnPrB)V-G0MY$olv4PC|xaV}?gaX0h&M9nvBW3GqD%kz1 zLG;3daa2Ej6YG$*m%XQYi22_3lKFCD8oeFA8XeHw$-MqKiXGLrpDtDIW_4E`XB&^0 z(G}sb;;M)oIwLrf`iyE~<}bd<6io|YW>g9djDkZHQ_f zI3vRsaqPH;sYn@arnxDZG}23!N_~_^k8i6;re8dc`bq-P!3BF5=YZ3wuq>5YtwVg59x#YaV-)t5>@cx!N@$mu(LI!>ar$M*gpo*u2#@nHs(I z=<MAyz`u1^nRp;n6 zi?QgjQZ@RksZK2n+n8BEm3DBmc^;QHvs0BNNW=RL`zGRzfZDuc7n#=3qxz;Os5$}_ z{ilFb_DLg4?zgI4QH-u+Mx(V(mB@(LQZ$f!3{k4eQIddhw`8}Jij$DoILObC?&+~NeObJteK8j6n+(~!TJ!1~uAn5HMSM7QFaA4H|%OE`5|i~GE+`RsY>$n=;fQt4D+4zzshISaF&3cg2{rOLo&i6 zj@(@O-mA=>8wQMZ*)1kyV;8;A(ZjeX7NHQeF1F&lKU;nG4;!7>Lj?!g*{@1|j4fV7 zw~Cd7YggN$d7dk%L9qi^eX*W>R|x5ni$#poqWx@EsXJPm^NI3ftGEI48npdkFE^c#CEoHE3j_MwTSlqq`SBB?rhfE~qCWmer^&EDBofcEb0rA9%cP}v+iwr*P-XZ3l~ zp-tnF?57SCr0GO7OSmB>)fO{}hH ziT;%GY3fyioRe?U68E!A|xIIu>bao`ShZYPHh@bw_19m zGykO_>ArJljhP|pNK+Hrc9gRGr^8H=zMjC(A`xAAHy1r|Fcj`GKg|?HXQ2~y7LtcC zJ5Wtow|MK}mrQY^Gwn>iD;PEN6%+C3A3Dy>cSrfHWSSjbFmnvI(AfEpS!F9FI%B&I zYAA6*`;_J~KhD2Hyt{koBY_fq7gs>r|J#gM!wEF~5n)}=wV}NeBG?Ig3en1YTd6}p z8O<9$&nQUJ;HZm3?7`Y-)Y(!ZuC)3l_}x^-#&X7nqCwV;89MIL9>x_&V=)WY9YgGZFtagF5OhI72-Wcj+Eyk z^KpxjvEd=4>pg`&G3ErutMzp4xfa2?rJt#{>Q`2EJ4@HLOQYJNS(K5Pj^g&5M_Zme zqW>=57tFFa$ed5qrIUkdS#sqPnVku#=WJON--u9jeky(35l%DFJoaotn?QDQDYZG^ zOGh?y*$RHS#6L)t^*W%(#JH5OLFFZgddiVbX%QO+j-%7jQ&6SsXwIzLBzRi;j_I5o zL5X||)e1781^NeQbLJL0&vZ~+W1~s4d@1v6%|G#4##eA++$kn@Qw1$Nx&wWfv{o>< z%91K=>}FgCZISMf3mdr3nDX-v(T~ZV)N;&!wEfms_QIR_^n3O?YGo5n1M~*y?vkx2 z>tzt~+CGLoJxQBR2ung)3ue(>mx#ZFKW@G9%U4w?krc&usMQCPMA*(6QWn1SS5XZMVqwTvcGS4DEA^vP> zs&)*~um}sZd3A=EcP*ZkF?3_U4s_5>H}}x^l$+$(p9{2VtrAM_8B66}WKbvBW9*HR zYI9%*>%%bZ<~34dveDxtmmx!TY;tvO^J_$$3&9^_?gQl{1?6 zJy7`EdV1;Bh~RJXF4Q<{1n!8=7X%hNi`V@whVr{EqsfYmoFm=B-Wd0e>g3I4H{IY< zJgWdTw~wXg^%y!Bk;9-DD`-Pi5bfZ-hvkn>Pc=+pVM z&|XTIGsTLVr7uJ~HlIb}g9Pmg3PCT8@1mplzIb`DG-E#`VV)L`LiV0_8O?RG1Q9#b z5FNW1naTJvGeUcrl37zMFSLn2Mz`E~j6Sb;iVoCR zp$YS?={U(B#J{qeo*iP@VER7Wvu~Fffs7PCzT1Ru| zMGyB<(6k<49!i<1wl;T1e|s6`3@@B4+ra9TVQl zdFSAERZ94##g;GD0mjhhk3 zm`G=FdzHa(73zNTBzr2dOQ0Zjq^>pn;{Iv_)YvaWwYpZJil@pFv+gdjcBHd#3TG3g zZVVHgJGBfc43^N!Q?=-kidfvg(vmJdFvLbqIYl*k`e^Onlayb)8(Fyru>N~?Qp2lK zl2=7X=&X=HMta~F4X%n|3oA!ZvABhO;V3VRZIGbY8a`__tqe)*qEJBHY=Jc~mdxtZ zltACBw2!luw+$B}^*Py$wv&jJn*EcJHcAj=Z0@C@1y@-QW(Ey2TE?1f&1brA?P9^f zA*ScjO*Y-@6SMbEIZf7S5ZvsOr{M5BbXe{rUHmPcnLkyFrtb7(9y%5?FI11Su_@iu zqkb<<{q0L@+#H0PJu4}`Tm!G3-G@$Ina=RXOQU0ToZDirK|`X8#rK@9B2VEOx=CJ9 zID5-jv}fTAr1MdhX85kD(tO7XDyjilk}JXcv05 zY%MiyxymXf%%J66%aHQ(WWvo@u!HV*7)Ke-rnqlMkIP#lh3XO3uw6>>VESeHsQErJ z=_WKVZx!n?zl_GZ8=}_I0v7Y83X8l>v8(O&(CA_oRZTQY4RFG3sMItK|i***WOm{U+wt(^fHwv14b%wo=g&eaW6Q zCpu^}iD4XX(C{0fsMNrYI)2Uv+g;AFu9SU5vWp#Q5DgG~TnuSG@<3-l z{1$v+2k2SeY!sU`ixvGlO!*J2QI}pT+N64o8q4isGF;xUpJKT4r9;u^L#;E?Zn}(U z3TNvpgt8i|0;wjq?-|n`zO4hy-G%FgRbty;CcQqB^RVIVEnP3dQ z@YRIw|JE=M+jG!Xe+3j3WQYC%&K!66jkcR|Jr=+5)bgP^HMA%tYqYh5r$QZR{ImkL zukjn*D}2OEPhW}1=Tb%zf>GA73FvmImT=Oq=S*+QE41*)RrZzi6L#>m1yda+W)4qH zM$656nZ5tX)7C|8Xi6d1Z{m5P+nXf|`bk9Xs#=B5XXdPin?S3S&2gZIBl!V8#?~7NWi5qvZuUDj@cd1L*?D_+2 z`e}Kg@}5EF*^Vq`IIV^?y)eLz``LqDs}9rDJ6q91i zQ3A(V2k1eq^=#t)N@QyL8((SI!X-z=px;#^7d;fz02zYnBFIF+H}vrzI^h-9_CFo7;xnJ4OM;FoUwu&UX zO`V-_GJt)1%9O78q{b|}Hz3gaYlZHPd&epqJVrko$e;zLYv}6s<4Ea|7R~V6PoMA1 zrju8_X4JIi(Sr%uNG)U;+GLtbH_uH%&LcOOZ-KVVk$dt~W*lPjincSR-_D?+my}Mo zy+Ff8YFHay9KE0iSyJiE{C$(m9&|j7U=oXJf^*n+%kMBdCr*a>Ne8&|!1MIa1Op@` zj6x%y3}{vBMM}i)QNOx{Fvrzf9C|1O>9kd%ZHzn~~I~C5U&Um3jGE9t}1&(}%83XyZ7Rs!y9p^_Md2f-lpA>niut11b4r z$BQ7eyDgGNURD-@&kgA2heeeC&x6Kqct)Qt8$e%+dXaw)kM&!Y!bZF*6$gD%WUjSU z(%>+AW=Bvn?ccD0icUBR)CEV;d(B*Q;LmkteZUIF)V2ooPuD`ui`$ug$z(KE$%f9T zy$GrPF0n~n5Sxq>6o$Mp=!>RQexhGA>w|hV;+< zV2;+xyj=yRJ&dj8WPnr$a1(VT9H-YK1Aqip}6?sae2 zfy}9>{|}d6mDz^I`n{r=);?(Hvyd_Ye~{EWP1L7d%tA*CrZ%V*f$Oel?E4nB<4qEC zZ%{>;wO>THn;d0Ja!1oQHde?M972y4Ekw(gU1I*n(3!{8)P!-oMSEJQw4hZfN!sqr zJyS%R5UG&7%C0OWvV`_Ud(lFYC>2T3a%S$C6lG5$$xgPEEkr2q`Tu_Iz31M~IWzM- zzvuh4o?eNR?EM_8&i;qyS(>9?U(V50Zk;^G-HKV%x{4lr#*+Nf=Zxb}s7 zM@Nur)I!mk;x$v3K(;w|hJo}{MxvqT+F66g=58K@+8kP|EXCj3{EkBZ)oYnnGR zjf_jlqAt1@xVuw-aTWFxg*d}cnp%Wl{;wp|R-eikH76^svaS>GYnl$e(NJ5Lu-f-K) zM!D)JTYB|MBKqoeh??Lcr0;N+amr)VAZr=VK%J`*+X@YD>aqa@I4X5L54d%ms(V)p@!WFXu|ua zOhrsAlcb=~#Ce4f^lYj!vQ<{;MExnQ@OWzph(~c8|Q16EUc1mD1lXjpvai&a3uOk|}-i#AjZDY79BXoY(BcbgU2l{^cDMl&q zH{;>2gw~nFGW$<-ad97Wk*HJMaipgM?OwSU{kK^QS^PcE4K3B6YdTW77mvR%w!R*8 z&9f7zHzJR5no>j6&JEB3^{dRR0}77rx98DH;YJ#Mpo5-^$`NXWMRN<**3t1i12ehd z1Cu9R!Bx(YK$pBb>8-#8Jjc5Yc_krw_F*y|YnO#gd4{s{+IVUsZNdy|I--e7duhO( z5~O}%9vVuG=KFu6oSn-lZffo}>a@s`zc)`Y`+BO7$`%V6^uib|$veR@`BIGPnmSQL zrVcvWbDEjopFy{WF5<4K=X1AD3#itO5=Lp#LvBLqI9it;N5`yV>82UKgzJkKdQ)hP z5-ML)`xDtH0aT)d6K&jvp8K@FhxaI+R6rfi`nk!`2k3#M^=R?Z3p8fnR{_vkB*Fa3i~IoWp`|sV1~Yr zM-5{{=*hNE+>N9}rh|7`s;x?9#Z%Wob%se>~Ocl#@ z(W=AUH0PKqN*j5>tnh7R_Wewy1)Ht;Oa*3MN4C=0yDO3V1;P#NTZ69GFF>MHIWBO$ zbkhf0A7SNlKIh`4#3?42Fo72BojDjjo&~F2UXz5Eu$LG6`(BdEw)qTWrMOH;T=kt;1 z{J3b8!7fAYmd(^vp_L&?L#RnDnJ!kAq3SeMIQGLC=G5>~#6y&u6u=R5w4jB~e{fXf z`t(04k@bP9WCStKOiYlYkv3zh+R3v^N6^tt**xd=D6=z1)$v+~gs9}x$nvwj^#);T}8)HU~+JIPY$^ZM6RVaX+~JM=YoXz(%T=J$kZ zf*>T5X-2*7HVU;TiK7GRlemp0KZG^!#xT{BE-;Q+@${jz0)@wB(v=lFuUF3kMciJ3 z7EDV;%eLI-o>UZYTYmqhh0HKf> zZp|WkuKt@aX2y5aQdA?_e%M*)Jw=xGc-V_hne7qTU({_nV{c8%0+%z3_8&u!tX&wF z)nl0@yt{5i+zcc&%%5$(8c}8a4d~$)qo$izuFywEb?H)XzSCp*d`sfzM_*oI^>hGW@=W8%pXYJ@d#i>Z^ETWrhS&?BA@7lVQ z#uc@;Fo#tFxW6u!(bXf6c1wGr;*|$!9>1MC_VFV3+3y2$xpfSa?Cis5E7mZ@X6DSa zEg!h6UMo<#*i|%fg&MQZOqE{RQclN(H88IH;H7wYFEWB(L}jbn(Sg=fns)e|2u;wW zNq!{^$n-($Trj5@eIGeR^rPsRvW_DsrG?DzN1We_MQHGoCMl7W=W_V|?8~X@%yCyk zVbRk`C_~sr1sdyUyW3>ceqt$&wzNbRG4e>X`T*U%F@xrvN)ws{EN5;u@$b>1)8w>D zsmUcY3dOv;f=+dOV}6qonr*ZNH7yN7i3PTF^~;4c?!RAXkKr$_T{DHc)T^VJ2jWGu zrawXIXN*z95gY39=KysRFQrZ1TWR)kp2eP3NY8ao;@PHW7{AgBoYkq5Xx~`=JZW~2 zalf8}_HC_0jJGPcx62Eq$t5u}#EqD&6wDcGX(Q8{m(hXrFHFf{J<9!g#;g=BMR~!8 z=s!NwRMO)^PuyO_=~RD5J+fyw>;7PBW_g4@lTT$f>s;psV|Sy6*2B!l?eSc{#2c=G zuZ!u)@auzrq44}jD9!QoqE=zHD6>xk&Cn8~=lS~O^yTN!X}3d+xYQ!LN#u@pgpXU!G!dmWInt?S^-SgASY)>r&`;g*s0weUww`|6%RM2qyI>_! zeo>BIR+tDKz0@3&g3IZ^(_DI}{U)_nK8%Cke8;yPrLD5?+(HKD?v(UGmQP1G(gA2;I`unlm zl-_-4T$?)$6dNn_F$(G?PmLex69<2j@ z=)G?dde(B2>jF!VQo1e;f0W6ktv3e!}PVb;#;Y z5SKGm2IbhUKw9!c%(!KoUiPd#Ri69%s_#qWxd<-uxHcf$-l&+ez+6BAI@@z<>dgC zGf8G{U*H+fKQQC-a5mcGna2d%=Q4@@LfYJrCPc4qq2mP>TzatyHI%h*G&)_!>9)+F zEB3GF)Q^|YQ`Z1GckU<(eK424#4+fPic!<$RA0Kjq>2XbIL`kj+O!S0(2^)uE=FCQ zn`My96bTM|Taej$ z#PuCLj{eOV;EuhxfTT2TP?wowdCy%BC$TsHnf47i;BDrjcKs7A0aL$H*>!C&O*)pEc*R@CUQ!)K&odaB3XYSN?LOsRnBsz>JFwt>r;x3Htt&V zf#xnM>9mh=EyzS>Z#0poTPxT2Lyf8wIx(enwW#XaHZGgbGW=BBimdyOa8qVD&`o=; z(#1up!Ya#BI{PMb?kP!?Xe*j%l!Cn1MEQ&!CsnYg49gh z(4q1R^tJAORPMTqi@MDBqJj>gi&vFtyUHu3wZ;hby#2#H6W>e!K8UB%|8$x-(!!jy z_ou51qN(H$FQ!#1g8O@5E_bgZ7d`8AU^X0?&4o)`qdm5(X>Y@7?r4NLcfo4BNRrRv z3?%7O(^spg^4Gh(!&}KQ^8O zJx4c=S<>L9NwlM?0nrB^(Vl%4Lfua_BKHOlMjGYOrxkaQ`;)o!tg*rWfvrZPGkvS`0?Gp(FCn>!Xc1Klz&=KlV&L<81dbimL8C6~3M`@Ts`#sQ(o zN#_b3W7;k$-Z2ar?wuxaQC8{m}+G8IYSsU zl1JCg-GuV}`9A5V3c6dl1kL?&0c~1bNy}DM3w7sQA%;R3RF<%Yxn^sP_QGPu{^9w-^|^~WIQlLZdENv+@uFB?$@Sv2a3th^!s#Z)+bcj z7R5}O<%}Ay8KSShjOe^>ZANaDUegC%FSMz!h0c?!i!StgNe5H$S-t7Sj~hsDb|R|M%3DKm3ax~%d zS{ifq7FVF_Nx_X}s7~$x(;%}7SuK}Ci9%WQOq0-!{63iL=s~m3J{5jBqSLhM!)s=# z)JF8PVhVaN^pJ)}?57*=H=sOhhH@Tl<*rOiWab^$Vj8Yz5kIN#=zGx_+9o#{-Rz4* zCQmmpXIwB{tap%}_bx@>i`AR@YTh%8wbJQOe>rkn`i17r{YXRipJ(>ArgKkTPN9cF zMRby21noAl;)YJlpzg1dQ0K5IbzNV_wbdV?w#}cxb*K4^*4!spW7b$&WZ{F1MQx%N z_S1w@O8D$^#35SvdJB@g1nD2m|4`fG*OYI1qq`4oGFtsln6rzg!v2~*dRICFwHXA{ zV)0{~^uY?6IVj8ReVNS(bKOyn`Z%62a27d9h0~W;ccUAHxzuOD0nUm)`$-;ZrK8&$ zX)51GJ^yh!eH|u5YTgovR@gIJdlb=0({E^7>`5-lB8iE$$z;CX8bjG3V^p~N63)~P zu1esv4yDr_n(2Z&c@Qa-C^SMYuEMq|4 zM6RcD0iAly3-!!t|n zPUQ6d9n;0{2lGV#L=j^d>cY=2Xr^|eHOPSQt{UFi`e^`@X7N4cgyntYbZhB+}w-!bx3 z0~coIFWRGfixU^qrDL<}&>YueD4|}KcEzfq?D4y)ChzuWn7DzNJUf9-bya4r@L8(c zsd1=tS|ZI3>!Ys>9-xiul+n&94#+W>KZ{qpBj|Zpv^(?&+FLxGGOlfCG&+^5k;_N+ z8&YXkib~VUZ98bt*LnuJ7&5aZ)*-r~o!-clZ#vRmg|3NPqkXDV>3@&9L_0FiA@=ij~6^&U&?L-fS%@gOLS=0~>Dtr>zi!Y;(z7L2J%TJ&?I!mcfGk<<;QKe@3ymxQC z3UhIlKC14@L8^5%v{U^Z6CQ1O#9(H|{0IuwGA?XnF?g{ppTv{(nklM9zgL&jj7~2t)@|>Pt-xVj=o;|nfp;) zfs9+#xtKq;D9gndCHwpkscreowcIhMa^Drv^o=PfEN?vLV<$yd%$IfiwOW%}+T0QS z_yQ~2yrIMXC~nAw^> zmg*$;an9pTQk|Mi8q#r%&Tm6>bpBmo#<^lD;~US}N}Dywj3gt0Vi$*nN0D^;duGMC zR^hbar<~r=Gn5U>K(Z6`9mCc086*9rXt?|u4O2hOC?*u5bHsrbUf)Ug$!(&ENBz*T zj$d?4k~Y~ zY!!;VStE*Ssh*%qGrouTC^{(J(+^tmf zFl7=wzlqY-(skTb$t1dZj2dP7#2t}w7t+*MZ!(wvg!!lj!G#AMN}ma*K?SP@g4 zbDjB~cocXgN-!=nA9J}!+?gjXPeeXHRe48pEM1tXz{zjXK`R5LX%R2{Q7kZ|`etJg zQ#l4r@@qn`wD_U)F+ns`OO9W+hB+t0QZ!epn;Uf4K@H;|vRqimCA^%0T*uU*AdNw$ zawH>WLkcOJ>q|gs9AE2qTsu(;w ziHWv9Oy@+*X9nC)p?Uw=FdwEIM|kgXw0qS$G}a`EiNjg+^qXGBCC7lin!5+RUL|7I z$@OqzTb0lir)l)SPd#KJ{y}?anBKrt?m|7GXlBWYg0x z7wO=uc4{3T!|XQ|(7{K8$o}y;bl`_I9eXZ|{#w199-X?@F=@&ZMvs^B8GJUSe|~L) z-{qsxArnpdL~1AfFo2n}AD+>LlU|Ar_fdX-yo9@%d`ncMKSVgOb@XZ9D4O(hBjfqJ zjMla6;D#fLxM{g@RHxwzSJBtXeXqLBHR5>g&-BO4=Bim#ocD{&eZQ1`R2XDV>pD_d z-VI~(>pERfzLjDZ55};>0d*-|d-WHL9lQ+454bXvUiK3CZ@0e5dA=JL|7>(#WL#I{ub27&ZgeOmx z!s)UX7}zwCj;DPAcHH+! zW5>W)0Wxq{i*obYn3#no)Mhqh^zW$9W{*QuqT`3KJ@zL+_QyEAe`jf7ZY$lDXGb6H zoGNq$-t_Cei!_W;LyeoA$PJ+yk$bw7++lJ}@z(7uqlJPL_4t#)X~SgOX%q zAC-#UVoDbpt1>kkzEjiUlk`$)4X3g0GPiku0kSb~7H;}I6YU=yq2cXQ>BD+X7~1d8 zNUmt4ZsptPETacpyX#uoxLdo)bVChS9YD&3#~^E#rmJ10T6puU*hH#V7Q9 zwksOP&mT?pc*^Zq^n#mHS0);%|G`;pZe$ETzC+^ail|^)E2CuXioPU=a}KkX(r=;4 z2v5AhJ-%~;nr?GNAH5^!=gu|Ejw>PPW9k>WzG?$!7-_`aKA6Y6m0!tcr$2E;;eH$r zXhaLGZ=e+Ci_EmciFDH~-r1`Cnwo9(L&{?6)Z=&`4Jyxsb^T+}?%m&L+cFy@a4};J znjT|<>%7rFzUI<)sSCMpzDI-G8b#07*wUXKtLPSfCeFvEO_Xs`ojGju4sE!v+Vtbs zZGO%8kE|Q}i^}HDLgM#+b9PJ(s)j!d=q-PIGp+Ou!YmM?B?#JPeJd7KP841# z?WkpYioTjX#Kkwyp#EK*+`tWenlaW48H|tQ8tS{4?|!DtFDrG&f={RE^d+T8`|(-y zJw}Q;SkFW6HLq|sH7hwl>Y4h{#ay?=Jm&DRU(E4k%EJ9E4SarfkkZOzW<6b@ev%zI+rlG|xuU7gf@$g9Wr^&lZGk*~9ZEhrsbeM)3UOx45g*8V@%= z5E%G2u|LYj!8osBu=(vu;J@K7xISGM+iXxI`l_jTKkpy-p+5rtJs1YM+8@C0kGUXe z_z#%tDFpu4X5rpmL-6cY8%}y&j3{hWxGpEA#^P$w#bx){BW#%pU5L7>qLyfAbE+%aNLI(XjVrgsy`Y<4^;U13aE z$6|cHcNjP9al_#}?~sg?IJmiaf)qV@tYG0vA|5Owwdtzl?aAfjx}qU;oTv%6X&b=t zn^Qre`F#Md4uFRe$w1uoJXoD(PPB`aNW(rk@?5^XVT$)+aJBjbmcCFUm|#AW7}{9D zS+k7bD6xdI#BMeIS#CzmG{571J9WYyIt=`?Z?j+1PTR}x|JZncg8@)+lO}-$7qPA7 zXZ&7N0k##iVbfy@tZd+M96K%@%>MfquQmc~*UtiM{jL!h$zgDPI11nF@&LKHvSeL) zKj^xw;xMWm19ltM;(z00VcMy+pnKpn2+HZic8`|`@)Y;r?xn*G+kc=P zPY-ra{*HlLhu|Z#9dDJN3!mK^OZ1%Du-2b(B(%VSr5B&GHIpUDtK1?i+xG-3R#t=4 zgU8vc9x-5axh0Xhu1kFCHxa$PeVDzpnuKb~!QPVV;A`1+plFl>=F3}>8#gAB<9fV1 zz!+xY+tKhw3>TCp7rh1cVj@)o>YCMJ+HrF5cc;z;IqZXjOb zy5#362jbk;j&plzu^8t>e%HDX(-ULJj5Cu6TqH(J*5uaLGkVs{7;x=81C>X1fwfpMuBKUvTXW)=_P#1^%S0)hHjtcT$E^K@&rtdK8D8OFUY-9hw z9fAi=svvfWKdbI&0|O4Lz~}K#!T$2cpe3ys_6LBFXL;QXw~aBhwrC|xlQmUR!{ zd)z;4clZvzV*DCA?9s;RU=Zt!n&C@jiUb|=CAr0k*_fEyKs?t1 zYX25OO{ITe&M5=fyk#}XP~J>_ml%?>4;_hSWCu=;{Dm8&Gw=`*CpWeEYt>y!>{vx| z=7=uI@k+#@4IlBfG=%TCg|Z8I&gzLLM?kD(D$vvj2A^sS$q|#eq~*E-k=L*w%z-oD zjz}EdnO6y<{`r6wsV&6RD496^6_EI-HN^Jkaj@s&Bap;1GmTalK-bxG$quLa%pMErX^L`m|GhasTU))Tt9f%^z zQ!Pl?VK1_M>Pkqg9H3nED-bou3|78UhKHPlQ0F1fQd?~W%SL2LpKl=H)Fp|_s5seB zItQ#t(go#_F5u>$WFX&l6~w;AV3qDvkR|^~pswjd%*Vu%Kc6R(!`=cCmBbM3h7c0( zqeCov9m$j@ibOEDm@?MSv@#rYDj+f9buX&%eIjpG?W7I6+h z#7DLF9tYJfCalu5c2M(d5t#(S$!MejIX&NsY;c(-fZ;q)z zTS=E_L@Qx+TFq_?%D^!JUf}1`+jzC{OwhcI=SPNo!LN1O@!9$!!5cY6vOata40_+i zevxPdb*h?zQRneEcToa1wB8PK;veBDe|g_ja3eb|L>VNXyNuhue`M$DjKh}$^I5$i z8DhO(7w)VGXOHG|0PFju_Qo@wy8@G3bS9Pm91R&Uo8 z{B?~1$$9ly=UF8%QgtA*YkC@5KKo<6z8-;2bQQSJAB8(PS-iAU7yo!IO(t&Kig#K} zAme|p1rNRSK)us90d-sqrkl2bPZ9Z`O2rh+&^yhqH(?-C+7|fQ%>%hpo561RV9>6T z4_uihR^>-MC>rQu$4Xd0x5g9rw&*v zfaetnC<@yy*xh4^H5RVLm$|7R{cRh*W1|GQ_18hk{ga@6@I27_mjlM>wE_$4H9+pu z8<2II1HU#{u!ed`pwC~9O>>UG`?9iG#l_i;SG6}dOqI$4=gXAwUUxWFbk9m`OtY zmXfmvG@wn_M7VdQ8gySR3H@EF@phBFn9hsGo^O5%#*a6KD-SC|d4+Glc)}p>!Etan zI`$B#wyXp*d)1-A!JxR%hA9y%Rm4s&*^El$&!8tfMJ>c@Vo~Fc&M)hKfNeRfYuT?tiKSx;+=7Gj<1F`hQ|dFEfwH#*D@>-aZey| zN12RwaVKUwc*k6O6mhLJf=Pi>VDJVxxG#4h+}S@5nk5c`xbM0UnwZ1)2Iq00!vnlB z_ck6GnMu~X94E-sb_1Ko6yee@cUiaVz1T@fiOe2Zge9AN@wJhM>_52+jvioDivq#r<@x~X8bY}$FJiCe zK|WTPl7^UQB2jP|PZqn5&z7CX!B=|m`8+9@vrP+jxQfF?8v5|JOCwfOd5#ay-hiiu z3CNi@GoV$}YPe_ne0Y5@6xPq1gcV9n@YdA=LGI6qV9%c4SZctDI4C-jxt=RYrxyU1 zE1CsdK{;?SECdc#`|xd_UA!kHA8(UzB$ele!QxKdeYDRAjyDQ`4gbl(4%;EnbtM}l z{S1N|ysPY;3xlxOwPWDrl|k11?>I>6Jm8|)`tWc29$54J7k;rcB>!Dn zg1fKp3bwp4WM6j3fn^de1V(xN;8t=Z@K)7^%ItKwLcs}sQL=#Tww_QF=K?iX>%g2! z3nvB5Vo!D5u_N2ku4wy*o}T;!zAv3FGoJ%Jkxz_+^&<@ zTdM+xT-k{4-@C@TrpyxDef5bo`V-48aft_IhN+-(Ycc@&kpeL#j2%n2;EJ`M8dpX} zVYRdwpkq@4sLIm@p)yj?f8drN^kE`fBAg*8bnaodx23XYm!4$_GQjPqI6S#J8`QcqgV>on1sA|Ua4jv1 zmFS)bPW;y=m@+=M@k)9E_FKcB#SPzJ&4?iU*Yy*2jhsw$<{6ViniEJ^iGX!`SR{b+ z^>E;Lb-}qRQ|#R62L=yEW3h{G1#ag)3rx+|vUu%Y&~mDftxPQh3uASlj)WHMxnuzM z^KQ|*;XlAb{`s1?FbNbMm`v8TKE&BKDK2V#ft6L$K-o|o2<;38ExHsG?3cz4JwWh& zUplrKdx~}Ue1d;^e!#c9@-aI01^14g#;e_4V84hiyw<)9mz=mGcp`rgAq&4N_q;Go$6r?1Gj^nU)%6{W>k>b;DUDz?{Ub!9U&0A^@larGX+|S z+F-|H>&Bw$eGZfRS{oC(+`y`P>l&JO{}y~q*@$P#Ky3Z!ufr<7UG&%M2D|9=b#^JU zNRVEq$X>db=%DpE13WS@YY5~S!Ao8R;wht}?9BAqhLL@@*}j`)K$uo-zx2^@pta0W zV6`tC>oXokZZDm-BN1?a+2lgaRT({4;2MXA?9cUj&YSGl*gBl};clbRj9vKI9%DSJjB#P?cUIHi2D{=-`0n|S7>a+!kvj^oCMN-& zw^f16!vh0_jlz(f`1!&K zSU;@@d+i-&yQh|8m94dSG^-u&4$8!~8Ygi3{nH@sS~*azSuW_jA^^+eprz#5b>6iq{t2{ZyMk(;39xXEKK!&*9!6G;f@B4G7;_*L zJpFPWD2RpOH+C;Tf0H_#9`*zL*}#I{|HeRpybT%s&kvtTR3gR>xZc{+7GF2}Bha(y z!17l4;PzL2h}!db5uyp)tMQfX+-5;k6PvM<+;cql63437Uk7)ew1eDz#h|sCcUU&B z;A_Hcqq1ZZ*qprwlMHDRVQWDiOjygVJTr|9ZaIdRP07RCU!(|HdS%GYndd>A_ZNZZ ziL)S1Ul#uevBnqTT|w=(m-vYLC-CHkG%ev=V1~h2 zF!S|xJa|zFJ~WO69On$q>{DzU$FBijofpINAs$e_!4�SOL#>yuj}3pW#H4XooBL zhD7|P6p1xACwt{p$)0z|vE>bWD0|TkzMb+2REO9?y?-~^hh<`oPabySGu1I*SLtto zdY23iix7cncU4IIjWbv=HX0XIsbZs47K`84fK@g{!1_-B*wj%gc=I#|OfDM(cdvZb zIH|}K8Z%?zg@gt$*Y7b{x4jO0ed5Ub*loy(cjly2+l`d1k;Lv|LHPd6L_zAhEbRP7 z2r3hN8`gWAZ`{Xw-|r7Lu>S%d3cCL(3EVojgUwT?5V=AbGN58e9Jc(!L7r)itVe_3 zedH~yYb*kP+GDZ4krdf^LK1JDg~%FZdFZ*NN-$5y42FE}!~va?fL&-m@J~*}D~wFw z>+cgl$OS!6Izj|0<79wIoSnc!x*A{qN5H3D288taV_%mtoX{4H-&-2s>aOu*QqoO4 z|MLmFsNf!6q!&V(3PaQTV>A_5WUD$K*4?t>0z;}fdG(Yb~4zF+^G*p!oN>3qf;t{xhff~8Ka60gp zu_sp)#K?h{2Bh3E6%e3-HS6BVf{YF=(Xt6KMFSfhYPD?8EiI_v?3XO;!wcSM30S zt4{LI`9!eNxC4YgSVG=LdJ|GRo4j6F3I0vi!L9dV@JBQktH#{JufvXj48sf*yqr3!F1NF*?h3nf<5_Ykowq2#Z8FPK)z0Y%;+lK7z)+zdX7Tjq-s ztvTITY|#~duhzj5H3hOoT%F9CB2T{hcj0@JB(Q~`4C|=~@cyzY!Q2Kn^2lpB(LOw% zJo%6T%!(c0t6meB4a{J3P&qp?G?S#Y*%GT0wq)sIWN)^w?FvP_`R4N8Z6Lfi<{i_!wU3K7s5in?RQD zl_ArTn}C~(EE#JpLH-Hb@zR_5c*!vcx2hS#pVtjx{D}?V%)~03uT_J?ItuU-`=j{# zg+pMo_h)ddG7(%T8Uw@?^hrReHn}mKchhUtvkSBJNI&m%+9st-4)Okw#?gt;wnZ5- zmPU}?Hv-*rB%#tzS*R2C4>;=7fkz)3+3G-3{Jl^Yx7o*lJarzS^u>X6+CehAu^3$Q zcY(p0HgIUxTo@5O2@kiofn9-bfpdBXut`zHTf1jMC8Q6J=$JyJ_aA;&ig=vxI!iEh z{2G{5?hZB7R>AB%b3AwYD7Gk9C&us>eqZT^$AMdThocISa_PmQN=XuJwSWlH7ZI;+ zU((>M1@~u4!Q&bdut(=N(9fFyEnZG)4*18c~IiB|w-|XeRY2EnToDOVmyN8_~5Ge?W&1ft- zIw)A&5D6mq=kgZsYWBk3C+qNnfcx=A~!0k1Y;iJ@UkTS^#D&MunkhK(CoT3PH8&bfkYxX2d){4Aomm;Y~Q;Flj z)ubTAouKVYh)MSX^1^324B_h$B~F!K<1}TsKfa`K*QT?Gj|oUkh>|(wLOW&LmT_eq%0Gos_TCAs25vb}0JU$SQ}bfm1ba zSS*QP5Nid?ubRSJMl<32+5zy&S`XX~5Q6!U69t7{8c=vs3YxZl1C7_s;L4WqWJDU{ zcB!Kn9Q}?D9x;Jc6~DpOa#%D z@PGhjzc+xFMY6Ci-T{WqUj)CJ%z~O1ZQ*~>KCr4j1K(oCl4E}*h_%dhoIKv0cvj3H zq7G>?mcOS$Z6%>A83GYLXF+kPI4tCyDvs4fIQ0Djyr8xSyZ10;jkP&>-pcz9M$L%t zqEPrmX%YO!J0TLn7Q^18<&CVx9#&_z7XIp-)aaLC5A&?dp%3W>bx#~%ZxuhM7BmGm zK9+=+b5-D>ceBa816D+D^$b!oXF4(dIG*TQ*W;|O9NxM7D85Jh;Os>J-rr^ikDqjg z$t!>3zSZ6Mj!`#OTKXMxYWW~r-GP=~1 zsP%rt8-u2hab5b*Q(FOstv>+HmX3iRHgy2k=vRW?s=J{1_CxlE>qgwQN)cOYzGTmI zC1Ll1g>ZcS9Qdnn2K0XE1Z%g?guez1;EfeS;4klTai8Q$9xibtx*?h*K*EkJwVX+c z^VLX!O)vgBs6)hlIFPsmdy=C*fmC}klO_B z>uQwlolG3-CXxz%&!=6=*J(az!ehL3*ppu$ z6L5N?IEnFBAu{(ec-Oo%@lBdexb+iA!k9s9-m6arzgvD{>Y@6;49Zz|~o`KEeyhUoqtXa>gG`}lAj8E@GKs-JV+&7H9fffj})1r zs!qJ@sTsxFOa`@g(&@*ddt>#NE|FVTM zpPEDKb$wuMvoW0b`x}-z9S`qaOMr$d2Vs+53Yn)p2dfVUV#zHs?E5(lV1eE;=)Yw? zRM&Qb25!1!!wXN+_rj69%9&3dCpkgs<~Z`nFp(^r7)L_i=fDD`aI*QuZt~H2ClSrt z2fuxfhR!cyVR~IWESR1~)>}jqh2#5)flD;+ieCwpT%*XAk1=GeVH}avN`|jy#KJ`; z@$k>`1UTS!kQ}m)g7#md;S|$YXl@im1pY-}@!JRBe!yeEO^_Fi&W|KM+oQ?wsaP`R zc>=Vk4j>_4R+6ExWn@9sQur!63Ld!{3%8cUL&?+>qG+^_h#ie4iN9ir^pzMmK4d?f zHWUSqy2irViR;Kp&-;SUJwt*a9}j_xz9Qai7fpsN6UeR!Nu>N;9DM#Tj+}e2i%hHD zPRhI&z+ZatFq0L|N^kpHZ^@x-wpk-V=>guf?7 zlPv}@q*gPIypKwR+Y?rh*JEwS!68WI9-j$!s3qVdEd@BvSplChvmxJ%qRCe-j?6#J z_h=s^L*>XQ`1(UE4CDV}*MkYfgM`3sf>2m_eh19Fe}LdT-d%Ykk*|Fw!qY+VBt9dC zFS>`pQke+&s3eN?2YqXFdz{MV99qhznYrU<*6X26=PKwhe=W3KAVJJeZ-c*Pg~B5{ zw!@JnF(m)N1`>N}11VUtjZBIQf}?)?|M@HizNwFcxH_D;E--@kT;!p1k_5cn5rZA_ z;{RvpOyj9~;y9co*(GF2cA|)XvfMLw=30`yBq>6Y2!$frvt*a0EJ+DPXrqmL=AOBg zb}g2QN~t7;7VRYc&%68Tp7WXWnK{4TocTTv_%)UR`0`ocpi~}qt4PL2EmQI5$NMlY zPXWy7Kv1a~3O*cK4U%<)v|QpEa4pCSJovr}=w-QLVd)-rZlvP1#VOb^H3g7}nRt`W zG(7#a3a0v&QDL1a)VWY0RVG|Z{d$#5Prv<#vv|r4{&kmbn~9g@wt+u2sTH{&_n1Yv8|^FfZghx#aN+@5OW9K&_w-Wv!JjA*-$6>d;Wq~~B5weik0^k(R;IxHz*JCm zWFa*$?gxhN*n!`X?%=alAkOj)qAT=Xx5&R2@()EErRGGv;fvo2rX%Zo`G*TiDAx-e zwAl$UZ05O#UT{{FcDFO;^m#0&hsPwTH8O4VTKt}}+t)z(J`)9F=JNP=Wi92D!l!a( zU7e%fEG)8#J_RhmaY#n#P1({qOa~%XMONl)cpV#&^}=gn%4LL?O8+g zO5H8^`oKm!GPoHlW#6Tiwtj2jH@>2~G#vRiV;5t!ErGP^DaGb%nT8@jXBSV6a(9oQ$`5{}T6|SOPN62y_WDR|`gV(6 zW@843vp%>{JQL(N$>FC{dN}WIHd3|yAcy)Q4N8_f(Qd{D z^t8XO^ySi~77*V_+ZNREE&5rn;%C;e&i>!+n+kMG!!N$=wqp9z%5~JhzLV6fbWh4$ zCzsaVrvfIYhyvZJpVYbmB|0W{f<}9;(UR7f#<7Z^*zdUQi_E!nq^Au3QKPrDc$+eI znxT(hotDI@50hGc?(L*jy5>^0Q`S&km61 z{b$hMo@1mj)0Sw@eal$KdJ<^|KBIYm9l0%;hcuR6Mx_%QQS(k|-m}G3RYP)nA zwIA>$)Qm%~L1N5e7=5deY^{kx zsB4UbWz-2OR$k&}h-5=vW<8qK;D;nHV!@}UdC04-9<3Sqf~(ji@ClL2v0LXxvzWFlWjbv#%!!u!=`fOJ6c&_d}4nc@a|}-AZCt zH<2~5%b3e)rv&}c?MT>1xQf+Bh~c*Za-|%(X@>@QPzsQq%Fy=`4>=FKpJxF zGA8yK*O`JWTj=w27rYkXDA@eu6zRKpkvw^#gMQNM(8|YRJTIwg0-Y=cTPC)WT`KFy z(Ot&8m9_cEb(RtN8&oRD(9I%Rw|AjH*}2f+lL|Su`k6}A`LMLvhzNe>pFo)8zPaP&9wd00zUP=SKA$5Xiq18%7E zm<*abJ|t+EKZi_wltGFHvxMi@&*!d_W%D1$MVZEiqex2G1l>4yn}o>ap})tX(2c1Q z)$967gz`FH6il!~9u>v-}Xch8k#a&px^CVHfVG+oIhZJzmp+?UzRCfOt;b8$Xe z^h}Ik_gnBqP&O2uWdzUEDDcvnzYF{%4M{?eE)j=&(U@^A)4fImtsSmKJ1ASSa>pt( zIR6N-V|hWnH`B=xhX^v$nBhK`CS=};*>JuC4@G^b7bLD;h+bVgEEsV;0-fMD!ZTr6 z#v=MCFEWwa_O%(kI#NS+=y##k1=i%i3N(I-g?QnaOBuZYNj-E^y z;xoywQQN1>WVhxuM!)O`Iq2hw&bj0=ywxgXYrGfv`KlX6H5?{e65heN11HJP1b?LH z-@)<$nqk&QTjU--#QgI;LL!GH$f_1C#^~7%qEw>IvSH?HgP4pXMk;?5#7?<>r zDcx#`OzzG>YtzDsjAtg34!*;>Z)!X#mc8KbeFV224+6BL8mU&dL^0tWY`3zX>ts5{ zydGlEnTcxH9VJ7q7KV{iMHm@Wui`%IxWI(Eb})~<#^Jd#Qrc_tjz4MV8`rg%*KDKps3f!jEM8{Wl1 z8S56qkvkvJn!n|!{?riqyka>~{^N^gNbyL3({v_TiH7rJ9z)5FF6N?m8fpwW!d$=T zK~^iBLPKLy1uLGcM}h(FwdeK(s}S!42?H6S~k(Libv z@@SLNIj+T^Iip`~MEnBY5zFrf(Z~S<5)+sK*SR$bVD?2ux#&EySg%LgL`Ruv7sr`z zb<#IWXahk1+c77pBxijK=8{s z#&tZ2TuPlnMx|&{%Jee@R>4e-(sm}UzJ*Emyu&D7dWi=l-jFt}t!T%nGZ{LYLmshg zieC?&z?hxQ#A~S%(nF4dspl^;brQMg*WRT_P$SM};KM*h@kiM6sEYe@r5Lg`Z9oU- z&0@^0gGrT?1&Mxk3p(1&LIGmW1iuEg2q&%%_S-m+$5J9>bBP9T$woC&cqxypZ2l(r zJllt?V6&dDc70A%0#m1 zZ#=61mcfh%tMH=D6cBIwY~f->0rRbr2VZt9LgQwQ@cG3el2OPZgk`h z1X+@w+a1WZk~G}SnJ)Ak-bJW<)=l-a4Bb#&faF!j7?Gna*G{~hEDdx#@fw?PDE_ zX2!)RD_j*2byp_1{Wtg1=s~io=LvIiQ8zPH*d%B?Ys_T3=OUj+74XKCJo0&V9duSx zAmQ9JNhLi4YWP%cYl;F%uskg|yk3fD^RAh^XWeWwMk-N< z^GRZQPG0E8UCH$~kB0kZ_z}0A)uepV2W`(3;i<$eAu_x7p$VdeZatGG$DV&B_ov)~ zlgHmOeu0RYBlrZrS*!9^M*6@?m#64R;v&{9`2+fPr;>d`KIp{cW9H|~t?1;K2RiQY z1U_`nLb<{XmN^@j(Y@1g^e6R>kp5J_Ncq0Tsm2`#KZxf`-jn`|A4C_m3+|B6A&?FlpY z_7!reyOz8Nb|wpguEB#jCMfZaF?sxEA9}>rsxJ>qqK(CWNb7_ux|5&5Z65VUKGRB> z4Tpcj2h)CnuMtz3rBoNGJbja#@?>+7i=Uv8bB~cP-OseoKF1`v1e2pHy#<{oOi-BY z3K;KZfX2GxNdFvNv?t&`>S$%%6x-tAVsU-a`KA!1?v*5Wqg2uF`64_(MRL39yJ2EN z6uGuE6{&8y!}u|BJonYj#NmV=82{l%)Qk_J4%`M$*gb$wXXS|HRVATJ&P!&!g()fn z7f_#_P++pQANsaYsAN|cGw{enblf>B>q$<_XW#hIwo)>e*Z2)%1|iyB)5qK8-GWI-DiPvO%55g ze1tUi#Gulue3Hl3)|~q)@R)6lAZF=RXkJnQ7wpnQe49CF$oDk78Y)JHRO`9XS8_<> z(Ou|F$VIGY9E<)nye2O4!DQ!AAJTqN1;zc^BFJMis@}m9%)%-eGAH>d`I$Wi5z7zy z5HBzE`Blz*=gKm^y&62Xz+Y%>0Ecyn#luPFD@-`8iu#@_p?`ORxQnZ!kkgbxnELfK zIqn_K-5}nB@a30qefbK;!0&)y1Ir@GQaVnsbS5$C8bjyrn+jVr6@+0U*ND=8Aw=}< zN~klOgJz04kik1?3`c(!)7$xm=}3Ldtbbw-wPcl8CW{5ybk~T<7Y#t-!A2x>?hMq` z)y~A*guN$UDkb9=3z<} z+?>wqlKq6vSR5jE4j*KWDD)zy8+q70K#MEVmcnMfX%v+lhjjKR3pIKi$ddJk$!uA9 zwAcIu8lRtxzD)VT?0Zp#_a176H=-AihZk?5=XZI+9n#svXFd--a*-ytWJGz@Mc2`! z2cML0FBA+bLHMtxf-J6Bgi8AYP=a~`vi~lP7VNS|#U7$UqP!THHt3Nl16@!M=8J?q z-;hl|%TRZI#U!Wp5biZoqR(4PrdVwvHJRsO%CTMOYVCS7UHA}2x8G-s@EtOorvR^3 ztKsVEUh-s$EHBf@laaf<3|)DDk6E(+6_Wk%8*6bRxkcV3jHzcU-WFg*+JZkb9x)c^ zPAE;P)e4B(<|KwSKcLerr#Y#lfuU2TkQa{(p}~(AaMIF<=ce!orq8n{-+3&Xe$EWx zqJPIp*?A7xc_)CI(40ri*IW_ov5sYQi%y~?UvDt3-9cP0?+1dZqQA(r2Vq3b!4&;! z$YT^oz0ukqr=XeF5qM#9KKjf)U$xX?!QlH4ay&_rTz)CWz)hn}ceW367mqN@&WsXj zXaF)v7@cXp#XRv@!m|9f5*P0l&}P;z&^2^H-W|>;=g(f4+FZwUR8K>veGU=k!9()< zz!t`#tq+c8=OaC1eUf;ALq4v{BG0}qBq=vVgmL|M1TH6A;E<}EF#OE$s-L056nirAQ$52?q`kNe=?IG{)9YFD&6q&cG6~;Yj zU|o|r&|IepP5HDQi8jj<#mG^ne~}THbh1I#gRGx(!(1|>xQa{{JH_N*)kAA1q=*rm z!t@CNIhVbY^vn!EyZpJxc@gW*48MVnA5A1(B!kqPbYNyI)Iw6GZFBqC#FK8MqEyOUej7m=ZdQ|*EK}N~OI2VUs7XrH^N?Po9x_kT zAWJu8A<vY+sfQU4}#<>DDa8U9A|FF7$v zJQLC?B8t2(rlRF+y?3jZiuX2%cs zv-lx=!tR4!nl+omT#{p^-;PIxi+CiXje^>{&Lj6Th0JX?Y0|42MgEqpMBPVAkyv3f z`c$NahP*|1ha=CSnIe;nm;;5JE#DEdp6_hkI+y1wR*WWXo8iRQV8+pD2BY(*2JP)K z5pK@=%;;4e;LabqO&-3d5a=pKuhXrNiOGM&JiQ#XU7Sr$+>C+Zg^_Tl^9Qcwn>$R4 zTs?F>cLaXQxIzB3i<8+NvT#SYgCMf_IytEKh}k(KgM=z>M@fzyXrqIG^r`@G*OJ|1 z{Qg#OQ~C{~+u)4u7L>voK`r#TdxV5$2;pYajd1%8b)H7#Q=l(VN6?m6+(x$xTx*s! zh5QGYFV6D3rW>tj&DvDbw|ft<)?7gD3`d~pzh#k*CdFIqqyX1^jV7w}19EYBI$E;r zCdxWjA#iS*!#T04ob2AV6ZS$y=Hsl-1>?Dg z5wlT}B&9fz#L_9STS)`y?0W)7RLV%caTT&s3uU}J2hhPG_TXYS5xhK=EjW!uP{1}x z;mHGBIhWvJtWU*{AC1M4^b(@)>d|M(@H;6u5mt>;P#*!`5UqI3IXPCewX(Zdc zn{>`CBxYMM$v&6_bJZUZEs1*qvxSRM_>6mGqrx%feTWZZ-#f}2a~6daSM_0Ot_s@a zaFg6leZh!P(!7mI$_RFEL)5Sw5uP7}AP5KaS4nMvHcF6e1GcuO87Z4q!K`CdL_+31_diiNwCrLpDhr#B zx@QC;(V<+zgAJ(1-<-sMNhXH1hsngqTe#cYiM-v!5#}@O{-DcgviS9RX7aWsnghp~ zs|D8R@}9FOUj820Cb7(%34EVPwww0CPdF1E9_e7cxT{IR^l3=(Y!$IK&_{Ap(ggQCe1n5U^GLU37dHyRgcURkbTZ*IH(ET+o(fy zyLWMau3jL>)HP+YRP|XV`*k#KZi+PRR9QaNFu6GQvp@%^5%;r=cxA|avVG@L_{Z@) ziOQ@b$2=_16vI>K1w9GLRa<0pp^03HTu(Y>l#oP-6=|n0k@>DcsKQo_nOOe{#ZEm# z^n*U47 z__+$@28fc5J*i}-ogPZ5Oou%py-?~`A{kmW4@H{4N08MTDHs-UJC!(0)p$8sbmugo zhp(Z-HnnJ~!a|aqS0RXF?xB_X8_|!Q%4ooE5{7R|XH1Kx!Nhf&7`^gUWa}zVTwW4H9GaA_)2t?LOE@Z0NJFY`U zBdKG@@X3EBNJqyC9q9LEti<%;C2A^H-fJtWeRTjoIYa2~(bjy_7KQ zQyHn}Ek*CoN|6luFJrRk26uLcBoYlRKuI7Ft!OJoTCwTOyxeW%g5+B?uRQ>Q>0QWv z(^>R;ksMN6lFSr$+i@R&dfXg~VK7eU8zks7hP} z-BVgHF{xaAZV(IAa8Ot zNPLD3iU~eUo}IDctvq{#IeB^+dean-mY8~wl|cdAvEwsIgYN^P^Zg=ouU(3Woy}w{ z)%A#DW-yuqSaz{hJzRY@9G+YvPO9Z}1!>3y6_%!;woSRD@~|68eu7Xe?>h4Sv7ImL|ZD;V+nN zf9EkT&iq6B3$%E2Rs$NGqeu4r7GV@$J%W;szeq%kH;P_U2nz%Cgblw_k;jKlGNGVM zCW$gn-){~v-y4apD{nwgi54`v_#QR7US~p{^`Uz6Z;X58Ae=BagZ!uzIFhmps$&ac zw|&3BO zf==Y`gu&t=@WGZf@b8kN|x?gDdD{9JC&-PsrbB*Q*4Qj6@6bm&qdu78s`dtH||G<7v>Q-Q?g$pP{)-b@%2c8naq zzJ@$pb`dfrv&gi|2Z@dPZgNBU7ShP?MS0JpgwITOActMK%r>3tf{q1JuqGv!)EUkY zw#mw(o{Vr9ef}Pi8L5Z<-`>F=cNDqZE$hf7qf%}I>&@O1k%r7B;)&GGnJl}>gQ%HV zl6i_*=-u@vP*(dD;mI9`^-XWc-^K5keIhMrU+oqWp>$YqRF%hOq44#lU%Y;M@P?hB2#^FUQ7>-YMvdY>ONg&{E9W1 z?fl!^*|(BOQk()J5iVrQf=p)o+;Z|$jzZ-z!?4NqA=4Fph4~UC#~Yux3|B39EC{s- zVq^x#h*EAV9Nxcf3Y%q%m^$`4$8+wJ@t-kBDWL`_ZtNqSw>*)St{c+t zNhiSRDZ2DM9WDHqL^kdv%oCBjR8H+tvb*63I`tKx_{-*G|Lj<#o;iZfo_0gji7O0A z-Gw;nP2@(q2szH3Pi8+(L!NT(D0Q%e$Vb_br3RlM!`8U9CxTgj*^r=jMmu>wewtDE z_yIk0$E2S&;0;+G6_mBFgn7+-&>79H2iWcYm*+(`3aGIOC1*Sorscnry+#LNP+$a@~M%uSwmBk3E& z$%i4dYi&qkTx>|OxHRFH z+=o}eA{4lLDOsQIgS@96BKBK*pu@o$?&=>EY`5#XrizQ@u0D*z04AQePEZ`$34NsW8l{LnLav9zY5s<<6!wvqKeoZ= zpY53i7yO9l7cF#%tHK+;Vu9x4qpat)lw=q=kmpr55wFb-eP7-|rsX=I_YGf2@RGk+ zr&5(HZ+t|W{EdXWe`ug-o4kmAt39IBLXksXB{_V>mNYF=5Vl)r3f^@=n4|I;-c8y7 zHw&+zzEc?_bHjWxJ^Lt(n&z#*q+ zM-b{?7O@zYMRkic$@2}DkVq=K$LV|uZ8;tW67~pTNUIid{WM7)ZI(ubM(t2;^F87p z)j;~qv&fprAoAwJB%{^p$f&ockkGd~8RK<@@Nvfkdd8DtJY4(=>mV9HYGD0&yfQ;9~tw!ZWy#b8D3v^hAb0} zW-{j-hg+B=G;8gD1P0}i=6h3wC&G1v+g|0Mh2_VIYLMs8ZoMT##I1TA7#D8{4>8{TAHEEjUg+g>p;z0U^)xEx{{;3Z-^ z*9xt9e-;Jrizo7K?~t9X3fU2GRFL_v5<0ysLw>(91w%*u$W;|BpC{FxCMSO!HOjx#elB+w~+NkU3m$>p(YbRo|eW!c*y+ea_p4v|cv zkD-Q}@{Fl$f&hbG~l`TxJaVg|=W(YP|uR=ja66E0-X{cuS6n@Ii<2r3r zLFwUZQ1+F>TvNsQ=47qJx%k;-D zAYz-o2`-sVLu%Y8@@42N(

fE{O%AZGQG}y~j7^*b7V=UR-D4_SrYytcSUcC zHQ|kZW!6_-N%|*y$u2d3g>ll)eckA~m>Y+tiSF zN(Y=xeIPAwXOqjTCE?}s*)$tQY92t3xIX00qzv(#OyMG;FSL4H49}GB5}ZmHMf>tA z;fL*GP^^UQt$vT<(JR|YTezdZGV=%5Y|@%{|D8K}v^bP|YmX=|b!iDPU2jB|%S=Z# z>vGArgAGJXJQXF_yn#1mMj5{17bb`M2}(t~l9((1p;F#e)E;n-G4`JVzsSzy1;y5o zjdPWFo0Eg!hstX7F|-gF;!>ohXbMf8mXPww7lL1HS+Mhn9JYR`hPJSC$-{ja%XjpK z(yz@0Rn$kA9qkOmJ;GqJ&Rycpf6rJ|HQF+@s+jxcKfqT}2D83r($*uhscO_rt^DbM zOEpjOXU$s-p23+o>HINjYos>rd6~kIezS?nT^oRvH?Oz#AQqscMGEtKSS-UISNJ%9PIqnmG|B=0Q4ggR;;g}nkV@;jTt_e{Tm#aMNMjS< zBoI=X0NT6{g2D&sc!@?KFl^*=j+8~wsPO>RF6-Trt-G5|Smgux6$tNW$OO48tL5hT z0_s0WNj$v&KBe5g3j9mi3S>8I#FhK)!HLx^)a#f6N?Gq7Cm2VA{QhWgJ7z107}<#b zrn&%KKY!3$tpf}-e6jo4d0?7`F?(jdj8fM6La$k|pZdg4!__yF@iyBOVDFTSzl>&K z^JljBRzx;aj_u^jIw|6;4`RSx=PBp!CwcH+Y%NWe@^EYZ2<;rJht1}S zfGH}QIZ}7?Iqy5FsS90swiUHgK%Od>?mICZ@0*+hR0dx1gG^=dc)Sx{w@3+uD}JWs zGxp(cG8y=qVI)ZUunt%K?W3o>*n~fP4FXr5Wa1Y`7h$5jnR8B60R+u5rR0Cl08fWE zapVPW=&r=SRLt8-%E70a{%&GPPwnmIAU5~%wb2?ZZZX42zpiuc1YDw1ceT^omV{DX z?;H4;Wpa2zQwB7Q-lsg5)KC=_8^Cy?GC0C{PUl}Z2xhML;0Gen z=pH;%V<%7y@WiJ>rSYM^u3(?tO6+OD@~WE2a?6qZvMw-(S5`k+fz3=~FM;V%1l>b`+0kh3zO zGsMr)2X>j#lT(txHne#t_HE=uW zfR_!1;<>>=_+aZ=u(u}$UygUc4|MJDNh?2)bj}m+{ql*rC;yvTRj7>Hiu$OtqY0qI zp^xJ?x)Z0z#eux;>9}@QAa+?P3J$z-!4{g@)N+XvbX2VtogCHAzk7cfsM)Ut);~4H ziSkn5Y2tOdEO!Y#jVlZ0@3EmglBR-x=34mP>E;$P_cGNI(@L%RCZryetKqj*3#b*| zJ8`OKI-c<=1!#ZTgFk)|qsu}k`Ojy_Qv&^JN+H9PT2PuzA54tkZ;n&sV6VOwXbEqpqqDs6nTMgY z=?*6>YH@9w4~|MWOkBeJxR|Cm!;_l*ua=!6$<5Cdo4uA_`MIbmG| zDLU--In|8W(sKw4-36C|U0MR*HL0nlfE)`7%mrpJP=jHaboW1Lh z%@3>6RCF~ZRggxN*UqOm8o!|SsTE;&i!>ZqcN8Qq&BtjTgw|IUqbo=AD3xa#I1K56 zE^U2qYHa~Md>(;v+2vruZ3R7p&G{(~YJi@j?s(yn8C2>tE|8lG@lAtLaJ5DYEMLC? z$Nz<#P65jV(P}!SxY13~r`Nu~#f~(?9l;~7NP7Xha-nn{BOVo%JeMDXk z+ix3W^90Jcbif@X?Al3{i*(V(Xdd|Bs)Ab%XHjo^B!EYUC=FJ|Vlpod55EWmIw5QE zcwH3k$b#6TYb8jN(xdFFmVq!f`?g4NHf~}+4miQ3SXwrmZ!6^rtTo2yO#K5bUsZ}I zL+?)NKwVYK-MDmY?lv1wTek+FXPMabdLH#(>^|zdiXxU>;sz>fiotQEd>~?5g6F2} z10SX)(F4U(@zbOaRP5hF^dE}>8oD}Q{ZGcgSydk&`k;h^TiobQ-dU=+Cj!SX1~~S5 z0FF|CU`e?&eI@V>N1O#Su46NwuibbFLp=IKv9hb_)j=zJy}+1)B7F z$`5pBSEyds2_2*Z~8la#fi~kES0N(4SP^*Hpad3JDwaUjBELW5SO^24_ zb@658GmQLKghq_ zNy+OLQLB_7-m|)#PP&-^rj2a}xxU-+53%*&b%YzXn7I+-O9`NB`dWNGT7h13?LB1~ zn%t5)e+OP+ZijE}o`+3c3<2-fC@r}%n3CC`PX|&L=!p?~AV0N((!FDhaeWG>Sh12m zRJj27f1o*Glrffegh2m%9^LoQidMK-$3N`;nX_!21}6L8QW>`_se4W?_?T=wnD6|F zDtcs&cTSB58=Ny~ZhaN?#q&K?wVQos>$u@n7n)i;>f*pq$!7eeNEGB{WmDrRnpmak zD;;C}f=-HG0+f6Dsn)fx>5Vb5*xA|-3%h*5>kUCT?deP$C$_$M`S5Y78=GL;#X5LN zO*sD0tqN|Yufk7Ks_6Ag+;L=JCzZ9PiryWOjF+tm#obPuz&?H`-g)OAece$5FJG8N z883R-d~f+HI#y1Uw)Zlk(Dw&?vl%`t$Ik`yr<>ynOJ|^+ev--@&!+|+*7MijZ~@w* z&9tPnG_ZK~kn`j54!X7sf$eV{fLHZWEJ$57VIu}q*o0+roZ%i)7n!uHV-(i0uL>|(_gFIDEpKcQ2y5+JpHx}H<7L2 zv%4o9$v41yGrOsj%P0BM4rOC?HAft|WH*@qZ80`FRYZs1^27QmlAvVL6u$&6*!N-* zX5^B=zX(-YZk{B4)o(VCDpI8{=o#QEHBaeF^4)adtryg*<8Sy{KeB=Gp*^6_KNpW^ zh5(C-*YuvI`xJH*rRG&|Y0V319FesZw8c|*TH}og-Kmj@?^@rbQ{QC)G4}V|j%(Js z0j>1rOI-f`!(2R1N(yKl|3aM+dr$vSRHxDTd@S*B9p=wV1V7np{19jPV)oTkNb+*r z|DP`4^vnRkbJfA@z*s!Ayo{f`eSq?C5&>&w3TcIbsUTl&AGm&L26(vFALk#)1T)Ue z!Gf#F__GI}5(FjTjQX8?*!zLs8j(z$Q2b1td=`&grEGERmH_Z2C~t#pxFwE`(ZZaBRIq(t zD)zm>rCOE;0jFVcy526G<)`h&6E7pNr(henQy+;(=o-4H;5Y49<_vnL-lcTjW`Vlm z%^>r62HqDQ0j}1ea%Kujtb`~RE@iqA!~jD~9C>S1ND zVp$hGXW$N1ebj|p#k5yUKHb-OhqLNdEcGs84ZYam!ZL8}W>dB)pys69`2i$p2l|GKdckAx+2L^4a-DA$w z;bU>seP?g%5i(4>yk8BDl>1_NDG&T;`E9D8+Z?Q=ZScDtb8(%!3Env>0&+TL<1Mud z!147;!0nF@9*Zod1 zz{e;MUy?Av8X+nes3d?9AdbZYEwFN=4ffn;30@LA4D7q;TZ{DROFJg0$GjXqU*Qiu z99PSKwMr3${W(FaIPa%dpLeCd->l?kPya#B&rZW1j>Y5SBpx^@q~hG<1^9ORG%T_{ z3QXMFiHEl+02hY=%HUoX{r8Cu=35lfuTDtQWo6COB9kDzNM;AU`7 zkr<~NuTKZOei1PDa33vyB@Me=@W)@irGPmL_TvvZY~C4|VnJ~hrK@p;?#~eg`foLX z$tpYipV>z6MkN|#KUfN!E0*Ig@@s+7$RH)bT;y~gHpbC8MeM1z+w{C^Z5-JWJ&jpPo*sfr>E6ZE@Lj#tbA#_Cwo2Txb-vI*31zns&c`yw^BGDcq+AX zt0Xpkmr6Zqm8Tx9O9NjAGQi;0So~Zn0z?n525J5Cfh|i6I72u~7wJNsEKd7^ZrWwRIoi*!h~iw?h*#CT;I}z>f)Sr^+*COW zfHN*2zwiuQrx(p~AR*o|)d1&8+kotLie;-*viz7S;LrJgoWKH$7G;)cVc~C$9ffR; ze19pYLV*7{Q=q8hfMcw}KtIE> zE#Lakf&`Xx^p4FP2Rnetjnly+z2*4+ga&oUI2xxpQn+6|2LwBMVLVw#o%%XKg@%;# zA5Bo;j9xkCoO>3v^PdR*Zg7OkJGP%X|8WJ~GpzbZWuJDEMT zt*z1mQtvQbv&#cmFNNSwk}o#UZ{a(Zd2?DK~)$yoQ= zF3i`D0_#4+VqVh=TSe=GbYs6O{d%q`F8{la?%PJ;8a++WezJ|eykZCF|96cl^h>}8 zwwkdumLK)IDxdb&6T=ZeKy`ey1X>LW)S4Rry21t5u#C?L>o8mq z8jOvUL%~(WB%Jun2gFJoqEh-d;GBxpK;l#~2%M1x^pDu%h{0eG`)V)0TU`S<>N?Q& z7tc`ZLT$0KodI8hG6(-0{IS?$FIw-O7d~RuOwG%CXX}1q4wz$cl)i7Gj(10} zydX;x+UoX9;Lm1h6--sAWmRVUeaF609eb2$`NiQ_;+z#u$o~(_Y)HkSpK@rEAQifx z$R2Drnn9f^gxGy{Aph7>E9eTw-m5cfFv&ty5Tk4elyb^xm;Wp~nqwO}^{UT^rYuOg9-}_pkic-L5 zMNMEKy%pCt?goiBa%i6raUAXcg?dvpoh~g%qdyOn(F&uxIO*qI@uu=6;L^|K;6mLZ zoHgJDT>1;>Rsbo_jh0kH%XfN`zXxbfJkj(C$Bj zs0j6&)abk(+wk-8)SZiIoX{0~x_M~_9lh;6m1lX6n$T*b^+MQvjLpC4=yD6JIgYCsu_+JLgvdBI__p@sY{&b1@GQ zq#MpI_6G-G3d@8F1i#F-02_N>oI4o}?4-p(T9zxHQjo#EZ@zG}GZ%tFeM|7;1cmLm z9^iMdAsyK~NWXWG2EePAazE#Q*KZcbku#Tq#>3ut#ODybL{%AZIC31Bd3IRD&k=mt zc$GRl97ewiNdqMhZ7>qoz~L=cU~v9y{QA2hb#iJC_2&Cjy5w3NU-v`;-ufs8TiP53 z?h?}YmP-V_7c~b@cl%B~b4$R%I!c%bb+Bp49F}$Rm0tGs0zLGgm9v`rlt0$+gU)X< z#XAiC!(vg6Ak0J!H>_6&y64w{5nDC<@r)}7v%5w+OHRRNuL7vDt$Fl{4^yx{%X$$% z1VM#r9DeH_jcb$QaQuG(phF@MAH8*!68Y)_WW}oKL&>|T9gQB~Rnbgbd`kw`-r@ku zXLijC=EZ^@>2i8F^$o-~2j6)^ z^sCjTcv^rMcpXwh$8cgP+>mSgF#I>QJ7W%hr(k8f-BuU)FDj>JtpUKlVlm);o{JR> z{Q>WZCTQ@wN5?&r=g*Jpp#*=F!SZv?;9y88y}_HIqyb0BWG#`gXwVb^nY#izq5?4V7#aKLCL*4e*-ZU18fsFn+G z{HX&%8}woLz2$g+@d2D`>;Q)uPaLkd9N+$7izjL{ zoh6Pg(4!+A7utof%DUyO`m_Od!K_7C^NBv*kj!z+vU?8iy!|s*!Tl~vfJ}`G z*3wA_&i8wG>M=d+*Ly;MJ<`~uO(|G+P70po9Sh|Q6S*@!8YWGh1)DSiaq&}kC>9rglus#IReBH2vU@?rjCcwpM2RL5c8fZH)04FVrSf| zU;zVDgIH}PA?vtl7EC`E#P0uX3mR5#7y6X12aOKNFf~^a%ImGdziU%rlV+e$c=#wQ zzxce+Sa}_<(K7@FYVL(5o3`Oz(;ZMom5+}Vh2U3QzbeRfHCEAE1wKpNWG$^~d0*BB zgLk@f2(%-?^nX5pgrG#Bm9mi(OHxd50zX;w?$AEdc0lW$NM?_gJa?AVvp;;fKqNHC^fJI zk-bZShk*~4zOWqU9o@wBeHY-R1cwNRl|6W6cFf<`M-pk#I?++4F4->{8<%@Wt`4yImZ&0hGkg^g!~0Y!mq@;6Cd z<`z|0^Lf9}^TaLTmF#`^-?ls8iE;(IZqf^O-V!xDX5=H-&hgjmK3hS1#ZPq$bY=0k zCPP@hu@_XXBkbnq5D3vh5MeW2sX#jsGc9^Zc|2eqg4vpZvLv4gQ49EPuipC2}X6W{Oh&W}j3 zYi9HTk^d*Q^~fNIcqI#FT8y&l>%IX;BQ5Ca%(*bUd)QG2H9X1j23Y^36I_rt#swKi zf#hQwq1%Lzx=R(KV7K-bI45Q@ymCJUk4uV%_I9gqN~DNgw$c&4^P7Sj>1oz|{VqJZ z+XJ4doz8|#SdK$kjvtk=5w2)~xLMg3c5E0AE0_D=G~ZvK-TfnQ9<&DIzhU;{vkWk2 z_FN$E%)s8Dd92|gjxUpNj*Xa&;PsH_AdHtGOtZcSvbU}UDn-)l&>cg#siYP>ix9EZ zlM2|afhvIaiw~C78}eet9$+VjC4))JeOQ;Oc;4(CA|MKi;U!4Uz~09eU_Fj+c390D zXEnKj*~T40jTkl7D9Hd?%0C8MhTM5&y=81tg%xZ*rhzY4CbF-x$HSO+<`}%mhAX-P zp^0lcHZjkD%j#XA(9s@9oZo>pqV__svUD6HOu>h4Cc%LVDR?@$kiHG<*_j z2ao;QhZVZFzz(T4uw3geyX;Q|H%q)LJl6w3fYKy5vQESr?o@^9BZm0k(-(D8S#N<{ z({>(yWQpTXEoR?GYr(K(T%R*t7XKUflC6%ngOA6q!-5q7_=o9v_U!B!c$(*n7hH&d z`Pl)`uP7MH8mxi;J#1%bL11~w(a`EWVjlsp0s)%ZZqZ~?v$bsO~O=ds$gQ8>)xzTLcO z**N0wG~UvPcy{e!3AQ>mpKVRqUiVUiVN0i{^Hj59fm&oWID?krz{=l3=Q%@c!Ie08 zW!D+u;P&^4qoU8PoDRLb|>Ya$v=Pm z`koO~ea67kQo2y<;5F!Oi)8!Y4yuJ$Wj9mr$<74rmYf*S94(U5;w0)+o1;8e)GCxa@x4Q z&>D_A%7=EB`Ecc5b-bs<34Z8(224wug(XD{e#zHo`<;2XykZY#7nwu7!8zD$TM#_( zWh!)(*^L*(Pvc&$HE`mUH(=-2Slk}%1T9uLK>8yJT<-P7Pq-fb>BH%;K4uwA+L?)S zh7+JnSF3PC(L|i%qy{xVrh^r>v25DZC9JJex7~AA4~M^l5WF{owUTyt@02`Vg^~|W zn#1`of&m=R4ZyZ8XFylmCE%Gcjon*d1iuGQ!0Ogr?6(3%nD%KB{!kys`j>A7!r058 zTDS=|D%bEH|N8`P23_Wb$)$m$CC|X%R!JBrtB)5&PJo)1eDR0FZQ%2~M}YmSgJ*kt z!>oh5;rZWB*wgC|z;M-g_J@BTJ5JFX>^oTpCeBpG1~I8vYq1qX`f)fpBN5vl*^j$E zL_wGQ7`#LF9y`kfgKq0iu(`{u?$5LxaN_3@-d@Eqxb7YVQNc$+nd}_6vVyYHA|}J< ze=>NzKgNQdwU_H|OO=3YhV$^3H}iSRGECs_fi<`{eF+qZ`#`OUC%*0*2XFj23Z4w5 zu;b2H;TfM3g_@RbSpUaF+8%bzk*>nl)r8WWvry*QsJ_MZRUIJ^KT3De% z0`y&$jh~tv#sMLD@LFvWzQ_4RHa;|iarH7V(AosQtZxu1dbEI4tuJ5$HH6P#9*;aT zz~n^~Fc&lnzZSd(XP+Ns6Ep zsT0D@61vd3KvI~bI}vwz41*|#PeOk4Yu577TX6S{G#kN-=cVdr3*UA=5?YRG!wwH~ zY?I)QBV+xbzF9C{@FNK7o|p;a=K0}E;Vv-c@p55Qv~9A)MGvI$sPEsK?sb{&%#!%99qYfU1B&KfS*51|Fs-o4&VMi*=bjA# zUq{_wKxZ4+9BGY%J5Pcn%Z<>mB4KXsvqktZoC>c`Im5Q(FSWarat{PPun>A!m_zp; z$APYrJkC~dfFQgG6mQbRGo^Fb8Utk<{Ky}7R@q^_EeSAOD;5XNeaKpCerNfs$H4R3 z=i{cZ1Sr**4xNMl!`lX<;K`q+cy`w>@F}#JyE8puHEqOf%7>*mg0}%q7(UOcRu%xW z{wwUWxDCKF>>|iuCu5p?7$2{hGjQ^XM0_PL z5&oOrF0X~MDY8*obSIGlN81#CI$fsZA~!|6I7 z*uc#N>_5-xP;KxIxWRbArT1oIAkXc!O_auE9WU5uJx$17SHWeSPWaclIXHfnHe6@r zkMGpv2^;-?v!*fQ@ZWPfxFBc`UgzMC8#A}Tg8_^16Z}8@ zZfuGmTZl(?LrHE{vabHI@QuMbutM7hxFuE9k=S=2|6wV+#63qSx7we5wq^w`TPXm% zyWW`Iy2$gmPyqC1UJ?Gux(1$}Oa&3+l32|bDPZ|OH~8(D97H#F@)i_H;fN6i+Tn0^ zvkqX_Z<2stwbIyc)${PV5C7pF?{#o@+zKq1Jn{ky=9?v*%4oqxQfa@b{ z@qSY~I4z?ay*Q|aH^=ufgIV91o1?|Fl$lP?Z0kW=PjBQ;Z&YFQZ>aLudNm@2XK!fv z=~>i%Ae+&0IY4tBI3Pupc_{SWsHm${8Li^>Jku7OrpA#M`Cp%0Cv7{2NOSWAdIZaG zd((}?;vPdg2NmijYbDa3+^)2s<0HvG^O4`YPaW-%E+baThv-6|dC0)3hI-{*Ks)#) zC_v!|^1hnKEDdxZ>J_7qTu>x!-^7f!ts-jTb(4&5%V_q&LVC&Mk?3&wMO2sKP5vPfQcC}XcKiLR!^(LTFEn6%VW{Eo{8P1h+y|6NTNy$;!lHu*nCB^_IkMt3zMm1srN z<}F~xIk(ZfU;F8N~0SdYIYIsq-6AjM^DSY$SGo-W>emh`yhqBqCG$e^kto!a1s zy3;lwerOc~-EGj2#yO@qL5JFQ<)Vp&Ug)t@DTA~2AuDr&`d>ewRho9rj)I;s5U+oy6IlQx>RLb`d0BR zvNYaHoJuDPCU+hp>nDfP1-q}3)uu-n&*k~Zr%;p5z7a_8cJ`s#!D^b2oQbZhT9ND* z63pwJNsPvV7JkN`M6{c9le_!U=_L0Uy0N~8KAig$Ejpz`PuHKNaQHTVnM)oui25Y* zncmB2tiMf%6ym9Rp9^!u*MQ>;nIZlEM5sn@2|Yh>nUb@Y(ZKJAMBlcPk#bc<;$a;| zj&}_yx@VK224niz;~?`_)gGC?E~498?C29e1*+_y$ov&PVRok;6CEmuL|v1^&@{{E zBGKqcVk^Il&NyC4-!A%&R>gNSVrPy4aA7IhD$pkNK4a+b?{~@YH%GKw*8t7lnTR@P z%|qX(j`GK~tB9`3cZiw>tr){=325L-lSq@_Km>6?$mGp1eXX8^!p|RO?mL#F!PYdS z>U9W>jPwDoQEl|oKN~G_c_}KWl0>s5Wa*~5LZ)|OFP~|D%Vhqpqj2&?Viqb%AH5!6 zKIqs``=z_^0*gDeWC1|^Dkqt&-xns^n^D9x=@2IZppjlO&@{oh1$WCqH5YlH6pGNa2Msxd=`K8(}FZf4{-px*lZ zsQZl{W1Kn>J@Ao2wMolqCb=Nm_w)gop^zam2(zRwSA@|uN|toOI15z#{2ntIa1g6L zFXVJAAI%dhB6qc^q^4;M+Iz>0p7d%aY8FRm-3~ppJd%eF4`d=qi(vX^$2oGLtcHjy*986+Ju4r0Qfc znzp56MAH+!($k~QtS8WAN@LMFMgx_5twGu)M$`emWoG9$61esMzwsk{*8VYFc)b+W zjhvxDYUR{p<#cB2KsSL7!}P853UoF4HJ5K{V>(wpVqRA{BiCgU_)~+DsN9%+sC&DW z=*H)E6nx+*zw$pslg6({Z~K3u?4X$_&QBFhaNmZAzp7x9_7!?cvxUDlTb{{J+lao1 z%gAGneL6F7F;XZ!NoS^V3=@NLLN+|4j=hHDy4QU4T_P1Zb)67R{Wu4mx}qiiqY;nR zcNkNwvVrjaIx1IIncVbQExb#%>ZC;BR8n<>nhdO?+JYeDc+6u+yon98 zkvU24G+(6uIkM!HQ4-o`6->{D%A!5(R@Bu~L3D7I5vqN~u`z9Dpz4Qe0{XKAp8Iu; z^!@fl$&m+9=$yqUblp=j`lXnta?gO)7ZuukCyn}*ZlRvjWKm%A6f{RCg^pi8i{2j| zBW_%qObhPK&jkZ~YRC+FvJzDh&iu2~o)GVFHp^JC!PR94Ciru9IK%J~RGT zEa~HT8NWUo67!5o8gSEwM!&yK`(C9X7XxXUJF6MFED+OOw+qm}bq0de$H~OLKaRw^ zxX}H+di3N1Rq;WGVtV?&$LP%Q0@Np;i3SFn@YR~9#PUZXQu#fD*_vuiOn2H+Wt*wA z_0Vg2M6rYz?m0s@7+VtCy_Lx3Z8kl$t${9!c!+`v_mO}H5~%C40u69@#Yk3fM`(T< z|JZMY@;?CPnWUyzYfTB-9QK?_w|yeca!n}uSUsAxV=-M=d`YzU?|AgiQ<{lkikRA} zQ_OtofO=kfp;4VfDD#yGoe{QyjvJHBSm&>$O(~M%d_yjiqQ44u2kfE$J+PpaMhl6k zE0rk{meA-Ld-Up6GDW4?=(P7jrZV{@ift;P2evk%%w_Fl@+UdeV{4DHT(6_eryr4< zrwbW1&6{+`#7832)oT3Ei@J0~NtOof8H<9SgrY;4`betqEVJAC0sS`HiQJf|Chp|z zMy*0A>N&ce8oZrNA7^}|J-d34VX-1=bK=hLaz~DJyp6G(brWrSvWWV8yF(A1JVk#z zk|3|vMWH40r;|lBuhFm1di2(^oubSB90O*W9rAwqnt4(|$izd($+tE)V$Pe6_U!tH zyk}$)iOxZ4@!tWIvA`4sU-qO`=4%+KG#>JHmK1~D5@O@}fOsXB&|FtJ@}*`1zbv{Q zEe;#v_n01|rSs3w@-)a~h;ESfza)|N0GC*hD$v|;oUg-MH9C|!n3kB#{ zASUW3bMeDtRClPIFDs)(|0B^%@}%XWrnh2p&00hj&h({+clFXty<(KLeJScFl%=~2 zQbSM=LoVX8j~D zoPDMY%|0Z{r1|^OOX7pb)mv`l9OQPszmy9 zj4F*=GN0@_8cxmbU!W_@JW*px5%a0?7l}}O$e@PxqU_Q?bk1fg=JUl(Orfa_vR-wQ z9((_Q$jt$C$GIXhUVAU~vN%jP;3iRUXcF__gMj&a#(~xzsaDii#O>3O^Y|<6elKF2 z|M@cEOg8gx&tdeYOb<1fui{wH9RKBxC$nhXb0$?vT6j(`2OS=YBtHWVqAitgm|J~@ zwB@J|dfCtFd1F_m^De(=eZ2Pm-a>dA($({1Ysft41Y5w;=g4^~7BC!`We@;c) zi=_HNZlq!1k&hB$`+A=B|+gVEGiZC2k1~_O=x9 z+wy76kOexGmdrf)oI=9;AJGrMoOrL9jtcGWF~3)@MKh27NA23vMJ^xLpkwM5G$39V zdin%19VT&T-IPSys(X%n{7;!0JdUF#GcSsy7ke>!lcUj}o*-H@00J~6sLRWC=01p6P+fHPLi^*LwIlLj3+yseelINpQY zT^~=KM=WXgz#ST|tSwNtswPdErSy$r3Qef+;J8#`x?1Kl+Tkie*F4?FbiRwF={ZNK zS-pfH!}=3yO?l5`T3kTtrSY_etEZzbPGf4kucDZ~6r6HGhR(l}L|<}w;H7V?(5ZY& zwAe3?_P?}7)5a3?@Lmq7jtwAx-Z&%0tc%QmM>*+#w*wu$B}bRZR-mW{52@>~J+zL$ zfY70HWacVWfdlUXy|eZVDnIvvN=>ELmfaOjXqek$Brm6jPi`ZUyTKtnGZ!G*f9rPFFGh zl4-Pc@IO&Z#W<#Xc_dnB-NfvWC}F&1-O!4be`wmPJQNk0LOnM80~MBa=yrc1Ena(o zDsfy@%T+8fc*&z3qC+HJ?hSn#G*;a7=m8T|{+rHNWJT|mPC{OpVKn!xAE~{cfPUW7 zr#0IfsOz*$>a!<>D!>1Ql%3w9WufwdhRHugdJk>M%dBaFbt5*k*C3DH^r|PCM&8Kf zR}ec<&`A!Djlp-WIicNJ2hm;a^8~(2=a+VJMiCzguKs9esvpdwLv6Wq`@&lyGu5$F zJHZSc#UJ<;)gH_qlU*p|gAy{2m!Y=>QK(}_3co(1hA|A8NgkB#LO*23Q@yb3czei4 zk%^?bxIOSHI(xnf?eJQPa;43v>#Izr4f-&`;|&D(&ZpVCrS7H|DB9jZu8_@i-0&yz!$E=U%HfzSJ!6=I8^6*Iw_R}9sZhR> zgE7&lGGWZu*oh~8JBbRP%n_IaKJ9;~iE32mQz@5DzVn`2RP7z0+RUb9yzn-QeUcg>Lh01kAO>!)8$}FJ$&>z^w;E}$`(kUXKz3U^AUS%!w$ma8=^MvM~vog17^l63-o!3 zr1+$k40=DwlkOdS%NUvdAZ@#Q!G}e^8H2S&{QZaP(Z$b7)a-MYXpG}`D)6vmk`m;} zo2j#CYtJH%Rdk6hGC3>ysCk*TnV2)Ti`JsJiBWny8RvBz&ZeRNyRf0 zKg~tgrPRe$CIyW5d3bP8rtUf!j8qFrFl5|3N@Y_FNZ`0i z)bMNp72lhM?reL@SaG#jW{@O`l0Po`op6$tP4Yr(l&;aI*HzeGtDNarKV$LyV1G2{ z&mYD*C|wlp5XJ1x^7lhoOHiVu zF0)uRh0fk0CGs$NAllpU2?eh^3R?$d=^yxzhGu=De=Vx%3!Xl8wcd>0{%$AOF(0WN zT!1Fcn@&Ehj6!)T7|}ga=we+hIkrZJIW<{XJjXkSBK9afFPX+f#ylhCGTG$)ge*qJ zvVkV-p=hk@akS!soWSPYS+wBQTYmkAW3>5n8C`oVhO*TI%rB0AzpLj4bqX#-{IX&? zFg}W2KLm)Rx+4-TY$DHpr6Fto@o3DntIV>2Y`*Z^DrA#1ijGe|%QVjAGdJglQR~df zG<}IHMO&tdFpWXEElEf;V2_sTSWvh`i#lW;LD|adXej6Q*f}gkMN{4~N_w2D#m5Kz zlIEicy#OsSHb<68ZA7+R4VCEaM(v=3iR_MNrth=?6{6R)W^Wl%{2dLV^rO)dX)&MN z$w61woJ7ZZ1JQ{ma>#s9KeQWCqc^#}MtDombnucL9p#zOESWgc z=-7oEH;b7h&$+0lO&9tw_zd zQGl$#$vJ^~?tViqt4>92&G+d1_A!*Cx})zL>u-@@KM8$wlm2`^m2Vd4M|&Rlp?fZz z1Eh6;d^Eg6N98}$?~C5iT`w=7Or0O_;E7{Uu74ioJ^jZh`IXZ-(dU`0p=KoYz?Rx& z>mUxVkJi2#qSGE9V=kq0b-w<7e)xM$u_M$W;jQxM^6d@SxxJn4&$bg4%ipEuy4%US z1B!HfB*S#Sy+YHDKO#%LM^I0*HZtnj$k;30CoYj|=v}#ZdUM4LcHyB!60H(O<)sPV zrLPjDU(-cqA9Lu-b&tqti%Le;=m;4s&!F2cD4?G*#VCG67NtOavb^96`lUF7J`TPD zCvE*oeE+Ep`;u6CB7(_E+0+4(%WEP8;kH_c`Wer6*p_ekXauaxmj86*#0gz|4c zOQ&R#H(mc`Hov#_3-flyH55Emnd!EFjvl%ekz?y}Xx&IRnNT4|=eFEunp2EuuVx2# zRz%Xs0|`X_VF+nt*Q4JJpGB8e8H-XoN{Gu1lqn}CN(X$LozcZh|HEE{-vrW z^2NOzy&dkOZ{nv56061{^UsTzqx+hWv(I}}8&iyizCJ`ViVevZMvHFs|4QmJzccY4 zyvdF5O#1VjhahlW2eo?pjY@?$iUaL`GAA~0UZ(dfqq6H5{jGFe)H{8xXi0@P?HU*) zX-bb#qkAva@rgzW5klsv%x!+Yxe2mu{Y+N<9E&y|E@l$*=c62*J~XJbf|TMkWO=rW zA3QA9~TFo)z(lf1lsgm^;diZ8DqmcHNsM%*B>+l$|S62s}bGy&^kPeXN=T@N0 zs^|EpIsTFU`@3|oW`IT*N6{$xHk8bh79~25q05+D%6G0nk>fT~q?S$YPT5KYbBdX3 zoZo6^L?ZfzW076|Fjd@klGd(^MG9fuz4756+9%I)41I{oIY!QtmVIU75_X z_Z)LKqzE-^&SiFtjiZaB1WahkFw=5Nihdvb!}Kh$f`K;`M9jQQM#U-=IZLL}6Lr&> zwHB51?|}>yo1@3StDng~?$1TC3UxV7*i?~I=wljb=SKT$IfsJlE3|#14f$AgjEp}T zhW_^aL(LtNQEKKsYBcW=xgd&F<8dUWlUaB$>r^+xhR1wjX8s!H*^z= z8}X#zwHEmu9K!z<@m&tFP=*BI@nSxgn8 zQW%-$dZN{u#N6CjLvGq0ru@bSbZzS`M*Yu0(cas0$vG`RYuCpxZr9#3Yi4YrzwkP` zc!~iDduD_37WavUtrjw_${lFaZNz*knJRdryM%uHP$$y5^oLw~;Y@xRzl1SkhRJN5 z7A8-8UbI~7Br>fP5s6usQA6^1n)E1%Nf4b!av75;DVl~Vq74M99(8nqdN{hfGKBi+ zhthuyQ>nbr8*M6)q(;M)=+C%oNM^e?YBAcuw|pH;j4L8&-Ry3H{$`+$h1U?5A!odM zn$e7ZUul)YAM)bAQpB6GkEyqvL&j7DBHfX(Bxb=N^X>0-+Fm_EmL2P-N$O<`lf^No z@+Qz6t4^e-x|F`Uol2hwB#Hd@$Na}_@$`E|Jx!_LJXALn1hZeUs4cHg^f!`QSrN>l zN)PD9s`cSPk$@JakG&;fVAkn^*Ko1JPkPRV6 zkyFk`rc9-rn!3c1&t~z++Qp21F^~{w<}RWG&ubBwJC#YlcYwBdr2d?He zr@TT)v(DC#gv(OY-=LZv(=?;zOI1kXPelsrMf{{>JG-uv+5-v!>$@3h3~phvj2LuMb_Ni(Qpy zY08MmF=7&;7fXm<%2^72`jOFdEWOS3&z{sQLS4}dQ1c`Q^mi-a-&|UZOr=Z7yvX?^ z4lPC()m6~-gGF?=@fzCXuZG-iHKO|4GSt6OR@`&d0>KonkLG%hlqs)gCf4XPy$j#- z6Ze!7BLy+~YHCe~?_NX?^AD0qZ3EG4QS(paBN{ZsEFdp8yx@QOf*$@(bX#D+}D zZX&}Q*Yg|qM5EUR)y$UV)&v@4@>z+EbZ6%sQDmo(cJDSu10GNLO~)FTjGIeof|Ut% z+UkyGS`VJ)}UsiY$AT%#9$1=wcU)UUoYE4At+R;A?SqUiznMNitU$M~v7pcAxT28w8}wvvFcQmEqwSL0sm#h^ z`g=?+9Uk3)26c}Ty^eWU--BC5cho~U?_HQRX_M%X-&?TT{8J2sGB*gR1%_qOWFN$or-hQhlC@HmE4m^HRmg zHhL}{`MHEvT0CL)aDK0U+vCAAJc-_UDox#YN{b#^q|j|f*+_DtV z(qMr?N3$qUu)Krw>KWZ{Ldcn!Q72()Vc6;AHycn=CO?n1GT$A3-M!&bUlV+a37D4#YB zU!^M-my7Htwva(B6S^yPEe*cZg*M$$V*VLv)1VKkv>Gz>>@-Moq}xfe$1Hl{= zh&-ykN-a?n6ZA%hSREM@wKqr# zesKgSnkd_lxytgOe%^dsj<#J)eVgzBG^p^+~k)*+Qh#{EGhMoFgVz z%xLYIeMIHNaYUy#iT=8IGkZtApx>U~$+!=%==iJ>BK6OnnV|ig)c4Ooo+u9OPD&@A zt~$_v#mCSB)k4(LFpu61xk)6%Px;5x641T&Na|lZ!my?-OvdC2^54x0q^1#$KJFCq zgL~9aIw0u1;}i0t{ym9)nS)KYa(&2${`8Q(4RaiX(y-bw=-JcLe6t3b;8n)om0zDI6Z%5<;I7(wr?M0CZoj;ihB zoQUIBivllaQit>V=sK<+(;B{z&mKxdHm@b=;2d*0C{BhO6we^Py*B*Zi?gV3t`jm{ z9LH=JP!v~z0%TNJNXMEMi_&%~hy!Ob$nLN%8vh}dIr;YkbFH8P$^BbK^L#F#oUym5 z=9&A{Auo*y$Hx zl&HUy2JH%=Ej_hFsX-A<)EuQcMU~98Q7vL!Y0K@?givRhqiE7zFFwiXrxE8;Ny+C^ zWRt{6zG+Z3(|bggrmqg7?l(8nx=c$ZD5V08dR%2vB7}5_l{yo?a0=3&l!~%tL`?Vw zY5wG`e@UPIClqhLlyM%Hh|)Z^q3%iz!IrZX%#2DeqFiu-)*qY4TolzX+a?`>Z7Z{g z>&dgIWlcPqkaYv8cD+Y~3#0{kfmUd_*?7Sc89$~Df5J!3=%JVC{m5%WFYgIN<^L-zwjC&HDV>_OnmMB7Y(({qk zh@4>6Y#VfnyJx9MrO@7>!z6!S5fiFZK)x<%LDg>_qSN!gkPeFrjD7YkBwSiTpR|Q@ zoJt8YzwH_dTPcTT_pPQs1|KuNUk{V-veHOSN{t!`1L&G?5&`%K$m?4il zD9)*mmzwsJIC^BQR_OhM)-gUZ!K(T%2>XuYe5F)ou4`78|t zWkbhV70D4{_pyz5|9f5b2Pgx~ruATNe+&395+YQK?gws*<9R>k9s<%M7ldaY_X%ge zG2^9ZjE5&}7qPbw6mI0qbFoMCT2XrLy(_n} z=dax6-7@^j)`(Mht~QF0WqN^URT<#PH?V&_SF=-p3E6||W5CRVLZ0@QBHoePajbMr zIk;%{jkok%1>E~D{k12caBZh$N?KZ ztOoN(XTxLBL9ELIX?Eb(PxiuoTbwm(F$ho002i}cf$o0*`qx&lKROqIJ!(!MEBiw2 zs`4kohzgcn@`ua3td_<8DTjodWJvh@_gY@Oh|9a(wgHD!5cpK34p*xR!L@=);cqJq zo_6jB;T4CEtlMKr7@as9L_HJ=W9MDBo9GcqWS@1x(*zTJ!@xmA(+3GL!yqycSxpnbE##_OR z)%rpW; z-pjOnR_# zW5@bIY>gh!UUCpj+EdNj*Y-&G;hQ0N+-(dcH{GuF+@~OH7OoI-QAZeWPrwA8qScj)p3wqq!oowj`$ zp1%HvFi|#zjhdo~&z=axS<3gQK*-tL9JaeTzeErA{bzHaBLi=biz^JQeoa za}2I`cMHu_z6n!y6o846c6RR`Ed+@`79ZYaCrq<)$2W#8c-2IU?cH*N&B{t)Z@+L8 zt{MIczHMB|o9r_aK3`Z14uq-UYon&@t)JnTn%VHe?wYZ|vi z3s|QqUO-(YqAoFG3cPVENF-&SCKYOQb}+!{W-X$_~Zm#ZoK zeU9f7J{OEPY+)TYUT1p_j)#&7)odw8{{OV>nb5TB4j9by=OlIRFnehtFQ)!N-K%v9 zudw#|p;j5F%u<+s=E-w=U8^)br7l&rBem|sH{p`Q2`NbljxHynC#0%MOfi)g> z?FNVP_p(+21HkEwBkMo96I=~=&#vS0%mZ8=?e3Vlte^e^aP)&axPMFl_jShz53~E& zqkkq~dnrHMv(6Bt`;3R@Gn)XfHUyp?Yr%%?838lAw}Vl<24>ziVOJaJuzQVlalZ9t z_Uu8(F5_PZF8efr`Pxme$AHIvy(0sQE$)KptS8jf=mnBXf7E^1It@NBIs@+ClxI(} zw?O6N-Moov=CI;?kg#~tEB1Y;9G>`fFE-)6fiH4<*=fQq5XHO2D#0u4yhjhfd;6oT z95apM2y6zpLdYgK8G;2&4tuuJ4cyB-#Y%sD!1iC%$L%(I*v31G@XE90pn~{8=^4c= z`0dYnh7YoBYL+;)t4et9e-zz$JXKF02XMQ{PPRgXEG0yonR`Z-R;5%*5)qY3Dn+HA z$W|d0l0Dh7BqF&p_l%OPDJ{~z7p;~`rR2HKfBfU+yv#k$e9wH|pVMu){$)LWQK~~8 z`+Nj>O@4T~$vJQ_LB3&4c_-df)dvi#JMi!|1-SdDFDOmjji+B0Fj1bOAn1rJ!C3=9 zr9hc1Ui})_ox6fZ96Z6Ji_&B`8sXapNqp5)Qsmym;Cqh}Ez0zbip#I3!A>zb;@> zy&OsN1X&Vl(20+Sm@r(#4854lN-&}I1@)m#La||Bz7~yIaF?hdXA^w<~$n<=i19sQ{X&_47V7inp zN%fP$qvG3$9n;Ocq5t^OTgULh|Jw`~Rroqj#h_15DHchXhfS6gF&~qE;Zq4K`0f_V z$h@F^fO&9~kuzG0lOLw=tFqtJnYcn&lQI=(c_~8ex03j)&quqfpLT;|m)vky>TG_^ z77mzCNX4V)+;G6LRea7{5mz1TZg6=S4gM`n1`j^m#M9jq>>V|(@LiXGY?$k!M6SeC z;6!T$GPTqM{LT~hUhf;nmPt#=#SODS$WA4AUhO^z+#M@)KPKVrfgSkJa~4cH$T5=> zMX=${A>28t%|uqYkwu5S!I*XqmTA6&mqdBuiW5iiF?l_3-W33`B^7vT!ixsqBUMau zf-VRg-U4RyDdUUBI+;q|4aTXw9gK=M31b;8_|3mKur6&eXy2#}&czhjFRAYY8#6_r z`{WfM)2j~j`R>D8k1#%i*^kaBWV!XAX{D1fEHd!a`0 zHgK z`-H7Geq{C+EC#PmBGB{W0a$)Z5_>!x1P1qez)GWdCg@87Gp*SGetFQtP_cT@arzqW zikkp#)$GDqKYQ@vT^07L>gC`~KLQ@wSU~ltI$U(?KX7ZXg>QT#0^9;0fRRZX7G4}! z{}GVIf9Ele^n>vhAPsM%I5I~pLh-$24mjld3fwEq?TydN1v*cPL2$A&lelv`PYh|KLXDB_P7km@t<^>~t&a?AJyI z;Jzj6iLJLVORML`cRX?)_rLDuUvOFnHhr(cMMDHQ?`+1_ZngGB!8>sK?x#3PFM@3D z*a8;rN^7{Wj)m<9SHOqKw)lZj5ipB+ir?GJgqD}%@kVzs(sHL1|Mgi+kWLBqITXQU zT*wA$;T}-)+d`1D?LA%^Eeiimj|Bl*HsIOMR2=h81)F+Wkrft|Kr>MsduBSrl_p~F zS6>PqJu3|tyqQHr%)9aYOAbIE#bQ|R*wP?y zr~om#6G-T#H+Vy}H6MQ~20f7z`9GeHf;}1rSUpt>js(f$laJo;*PYx7x^CK%qbZX> zQ1}{>{&bj;R`g|phvFK}E;1ljnfpTb`2+Bnwt&P+yyN%2Z|83ti(*QDijkXBrx5LF z&q2oDS2zdzGnajNWOO{1?^J!48Jqr_37jIdqJ~BBF0F7}y)7H3E-@u~t6nu6?LUf7 z<~HG#i+zC`UoJCY0%}|dj9pTYGCEiLjJ!aN+cl001Uk?1BahXgW0BA z;B;FnY!~(dtJw5`?p_O`R$R@WaBHx^FI$l?!u?J8!VbF!JFUo@S~Ylm?6KXfohLxq z->Z!7Mp0O5>j&?hn#m{_3B4)%6dXDxCiEtU@n={Gb``o1cWZfKESHT9lXT$C-gN$n z9ququ=+lh!D_vnsXTyjOJK$SIXY7?xjAQ01klD9A2%~ixjJs8W(c}-fvP=Y@5AXq6 zM_yx(BP-!-2~(&Qs*2xtx?#`Vv3T8_cye-$KOP933@sUbuqR?FJS1Pi{3bD=$f1@o zzR5yZ`Mg17NfkCvG6Zk%0#Z{Q3w*^>nX}b4(9YQynr&&q5-$tErrB$WPv{1c`?J1b z#@GSep`1~44!#_ zV$*v8Wchgl6f70M+IxmDV|+Qx+WQw*Hc!F_HrwF2KgRhJmnabLNge!3-HrH*&`i;k zO2m!>n{k7pBcoSw-hSHrN8mNR%YT&rhtUf&;TMZN#$m$G!`Js!SVyrEuhFOnk2fZO zfTIPVFWyY}jMj-$@`f5@y&l$^!fHMWx&l_HbTYwV)nL-(1R!$YDNgCw%_KFP$ENcC z_}vGS85x^$M)JsBpzle+oo!FBhWSl?V7osk2({zanV$hSEiqvJs{ubs9DuTZ3@%Za z({O+c^CwyE!HS8QcoElXS2%GJ;~S<3aua`Iaj3vgKcIy5ADzOQ5zBFsg(Tm?;3W1Y zFY#2vT&94R%EWg|FxgF8z*3!BJeVAWRidNu8P<_+s4qd*pBn=Tx^MAgdH?!lhb7^& zB1!&*ybHh{djSoVHVgYCn3W&Am|SzzP-F#BVRR?ms`*on2=5&-q9#l9_uAaVW*AY$u? zGbSYhr}R<2_QPf1uHpi`la>Qfzb7DZEC!?)Ujh5x74v1{&9PqOZ=5EcjCVR5z(#!v z_}8N*P$o)1Y*-t3EP4#IWxun}tE*_}k17QzZfbDHj|rgtW)pK*ask7Pwejz+%mn*> zMgy;@P0W%ME5`g?5;&(A0{*?SV9X8PGp{ed1M8>AV1tQAz^S4vrb|xft8hp02Yu@8MfHI6QJL~gg5)Qnl0{Wv4gBL&K7>V8a%#WJ` zjJCB1d0k}<9IxNv|IS(nvfV22*7_(A8(xH+=Trl&Fjd@FcM2@%{0p2TJ^-~7bHUD2 zGr^vNDZtO<6X=~)0InKmfV|xvSC%FT&C|gGu#cV0{}Z$i^R&xx*=~m4OICwmqh1i7 z5)ah3_wyqn3PBQb;p-@$0=Fv`;evmqKzn{VIB#Ue9Jv&S+cRe3nL0{f`xHrdX6!wG z?~`vJfIj1kFTV_~TG=(&6nTMn-X3_7?g&1UoQ5S%FhZB)%9TvZKASpY&T}fBXtArKx6iAd~&@5c$^giQw{#$)Mj;|rSl$qmJ7hv*L#>1kw3A~ z8zubqLoZX)m%$HaD(ocGHh`JwMmXOp)80-#26LYWftmedjD&vqKBp>tThPyx6e^QF zH%|eVZEx`2kMqEu!nYU*pa0r-A?E9rcleREE}VQ(iM0IEf#E;Zfliw}v{E+0h2cXW zp)eLK+Tj2v&l$s8kKF;y703CXlkLbc11osvT_e6~tO-Ii%)pciYpfk41%jGmfqsi2 zbQ2n_jonQ^x?l@dOtBQ2lfQAWQ7tG8vjxY#xxxMI7&m>^hFe!3!hDyL_|%8*jO8mn zKHgvt_NL}k3do&#@N{_>U81>k09G++GOZSW#w79&wF zPwIRK0L!mn`7l#*RwWugThfHLo)!gio|LaDsZ3TKy=w2e>l5bhF9qb;E9_SF514G( zgxmc;0S(>j*e*nmpSZCH_g^UIbL;0oiI<(&ujCr|q}vFNL@a?_%QxWz8wP<9^kq(U zOF*7t0)Ii+Loipo4*XuGjwRgN825ZXzC~LLSe)++&H2ah0ij>LIVYG=FMozD;!oiF z7Ef@dj2!bSwu(O>4w=m^`uO0gPoQQ<75Y`DGS|0iz&|ew_%^wD_|#r0{(~JU*jKI? z!09WXN!d@VXD10;8#Vx^%sJ4r%mAB6bb;%q?=#o`y#dV$680Xo`M?B-fuvRdWBV%c zhuKr$3F{dk)p8wdcjq&1m!`lQqP&p5fB#``srv3Z0E;a=#%YwiGh7F-2~-)@JXH4WPYVz9~68uUll!xs@! zWXSU}&|CI~-`B^0rrM*-Ry|>02XXkU?nl~n*SvHuv*aXYkD3cxi$}lp$ z8n2VmBWZhONMPR#SoR?X&k9#1k9vRM-&<^8ROwCp=b$nPzw{bgt(AxC-emJl%7?JL zk1=>4v`+1(NMIgr2CJYdNqTJyWj_1khx?SsoVtsmEj=SJ}Q$2_hI^aio z+wjQ2U%;kck$=~~gnYO!0@IUb5!H3>VC@G5=I=c@KJ2c|kUB)md)(z;*JIvH}sgUp<2)`yaW4+00q~qNi5R=-B8}x%fbC?#i zTBZ(*DNfF-NH3-xV9LPm0RsP() z;;`O$hOm3VP-q5{2FKw)Kw-87v8|mA2SaD$4jB#7utu9qnPm<}F6DQ9;%!c(2mUwCJWD*^D9Y2jUfDf;q#B#qh$f=}Z z+_2dQPVmhKYaWf^d;Mu3Wm6LVUT}mF(a?mJ84BdHxHYUetcm~qP$uTf&Ee1*N77km z2^x_iw0b!TE?Aord$*U&DS<2$BeL+3&}*N&o-q7sL%6O-9M&naM9RG!^feg6O>ZZ{ z9oA+fckv^}d9xavBbW#;pOYogo;En|ml*8UP~=x^ehGqZx?=qX1tOPl3hSC501=A) z_|hYJ^78fs;=TaFoio?sHwTnR$lu>!;EEn`a+tvUm^2AKaJ>d5ZFvZwHQku?)|@BTr`5NAnHZ zUV!C)*I>a4RWQfTl`%c_9q3OF0CnE2;Jw~Yyu^6{+`KIpllM;W#-wS$N%lPm)tn6$ zoxTai-#@@tZXxLVG!-ALH-m+i^}spAkZ3NO-cYejn^-;Y1af~2$rtx4Oz|EOl5i;= z?-fjecg+SF`!Oe&q4*Lj&i(<^tt+sC3Lj`FzQ7j?P2ju|Lt>p$4@_FOG75=p!1iJ+ z|E>Bb@IgD3=@OqpNXZ9a`|l_|C446`S9dlHtbNz;KY17uub;WmBXbnu8=kf#uU8u2=<5=2i$@(lZv_Qw zpZ4I4BPHOn-!VMp>kANnyAH^&o=LJMRsk!osU%2SgulL6AMREv<9`%?3$~qg0(v(x z!GU67K-vE;_;M>%*mq+H=c`rP3R@M9JjwG83#hkTyeg> z5qLOy5JUiLU?lbjC+*vX?|aN*BBO@D;c+t@E-w%Ag!AFkg&To-ga+!9Nzsz;`!k?@ajh-|L1Yw zETuzY(nR5lnQA0K^)+L%Z3F%iFaj>x)bo4S-N$8SX7E<=C2ZQI11-N7fH!|85gTC- z&dAtAGU?S_tX8uavg_V6{bx7A3G-FqlF23{wYM5PcKMC>S7l)dWpNm+q(X*!nn3;Y zTK+;t3tHMxEP8hsT-y8;?`RT-XPojdbnOHa^Xu?JUs;m1GoN`r)XU%aW+5rdJV+|Q zLUO<geJ*L7f$}X_m?mCccTnC?rc7ey{2Vt|k6=@CLL3G4Lh~|hD`Lulr3{=<* zb&f8EA76UIV<-Z*dScKw*^gf0;t`=^*iGG$ic?ck%7NpYWn(9eBb)58^Mqhul8sM;eBs ziP81>Ft*YMa$A={m(L#1#@rQ(JlYF?yzqoS9O9rLLmi6V*$#89?t`XXJK#@oXL8?s zKiOonh~a!dCUxA1G;!Z zR4Fd?_k_%$eK1a8BP@Cy1!ey_LKnCbcE>ux>jlB^<+7z@diX|?l_#7vhU_CB^_IhB z=XOBd&l}*Gg|RR_X90Y@V-I{K8winoBs{j-9WIIrgLy)0ZEWB_81rv7Yz+LBu?4ClOf% z$m!Ap#4=Zh5S@+Wk%T#%=pr-?)s*4V`_^zk(u?S>*+&Yt>>~C!j;QaNNrryB1G@$x ze!5W_PTjefFiqRZ75i-@cwacV5{Jkm_fTT}njz`mHxekk9)0^j_}5SYHv31x5^x0P zsQHo;D>`ugwkTpQVomlJcoAR}Ox*s9A{yy7WCJgpOg1wl-ejCnG0I2yV6ozS=!DGrx;j(*kp!dIRaAck*Y);<+^LOgO ztBJwDsjV11dpr?_^=rV$<(9B0(2vRXH-c|VO`!UECAj$I2c}(mIvmT;B7&v9%#;~7 zakNg)6{0c@KsOwA0tM=o9c-$W49+vTHb=W{ZnCNfjU&NTml1j=-@-b9?2gg zQbg-i31fDCBQDuxY0o7Xk{K(7-rR8${Jdo{@r$(tC1RhMoFoqSgv`gbH}~S4Cz7yR zPn4`Z=L}mv$AK-L_7Jd($o$Y2@Zz&M-0@Eg|Fp>hVR8@n?uv>~a^-hW^CgbIxkcEy z_;CW+d{vnQt@sGS7fm3hcjmyZr0GDuQwkcrlZSt*K7ce22e`Rr3fUAR1y4=TB>H}a z#7^@b=%{#)(+4NQ>`F;!GhaBX5S$0ne?bLCrHm zc%5`ogM-5d+_FTB=qyqoUdx|j_YXYWI@O4HeK#PpgEO$E@P4kIuT8Gk8AEc_9T-Jb z1E-N*ut!~nsF&E2pJJ-uj<+oN`K<`p3*TX;sMdm~9(7E6hVYEQrVm)KTO9cJ+`zg; zI&iz0F3eU+0e6I7&yVZIJCmyMg=L@lt8I)(q|h4i^|m1_37=#W8Q=GD_}Hiy_Kwpa%U-wR^EzMo%+ASV zW3Un#&2%8s&X@2H&oLt_jm~j@b(SGljc!5Pa4v1}DxoghjgiK33wZh6Z@N0~6jfC< zLFZd{BJ+`Ryk+s<*`xli*vLh3wDqJ(VM_j467`9bjlQTciIr=Ulq&~wF{)dj&ry}%H8bYLl{kJPDYQP)Nx0) zYx00Z4SE$kh8&ZgqF+k`Sf#!DsB1VuJmxe#JT`&O^;d0du24q&=m9j+W5Ax3%%UPf z3$1=vmtbO%B=zR~nleNr3vEB+>0xlDI`@r?LCyy<%^-$0B2uG+ycKqjXi~Nu+Q{hps+f#F;+pW}9pj z8&fo=QrY-2>T}>XlKQ@alTi|IsV^1i@gHUE44dEV8^6!&@t!AaXKN7G^>6_7)hRfv zn0gAi41Y!8FSpU+r!P3=)EIOwClzIhZ9{wQbJ6M_8YoZp0=>K^j-}h1xj8e0ee74I z=&Z^P)V);=1+IQ9%$5doR=-+UtG;$rx~T^hw8x-}u6H;q^K^7&2ITo%b!K&4t|IO3 zOfIXnm9x?^qyld~I=|~O&AYIVHbk66&fI72)9P=uAmf#g1-?ub9R8r5xqZk+zMpF; z=drG@A3!z!U)DcH)xj_zfNnZwMRj|2q5JGT?(YqIG|%?~DR*iH&=1K_&l0*D2}s^dr7-RqtQu)>)h5rNbTPvZo=dZwD?p5 zyDQ%hHJI_}-eZv4>2m=czkHqzixH)IKGrBUe+H5{=Ygcx2D4fcYaG%pzTv){!6;B^ z3-{0PI=k?+4h?DRpu2oz8&BwjbGLOG(7V0c>4l1BB-VbI2kC~xz!4GsY z2cx##E9s_gGy1PJo+|Y*^w-|SEU8VWQ7*+#n zP%ecR6*`qIpTt99LZfg;R}zZxtYf>;8KkoGIvSaEotrEzNq6sBj@BDkpq1ted9Qpg z@}AY_qL0T7XaQS~9`_vK?x^HYtH^6?X7W5v^vE~FyZ4*wN14zWQ_|_$Q?c~6vH{Cf z>NJ|iO=Z(3&F1uLj-c+~X|!j0JL<_DKsg=FXhp>zLH6NFHgv5CdcM|<%6DlwlyVO3V9rQ*(Cwp!CEM!NsCU} zxe2-XpF(f#MbZ7vuiUGQ7wGhXBh+?n8a*^|h|T=j%tkpY(V`vKC?gU@&sJ0R-3E@V z`cX{9l}hPm(HOMiK{%)MQHskPFh)+Gm@}I7h<%AuP-S!|=UcpwyEtt*eH*3Iw^;{LEqSwjkVmJltkM6rW<^4EfJbm z_t9pb05+j+fV*#!&z^m%g+8r6L@Wvdk-j?SR6Ola=&2xjBkT|7X(!XDyf~g-T5g7R zrO!ckygcYiXG2s|@{V@LR-xleEOp5(r809h(2Iy+YH9xpiSJ%Z8;>03A=1UV^yzXp zvd*!4`xbDmPKiPjsDtxSbwg9PG*sZ>4K>#5l6UnY+F@ zgZC`xD{GM0CXij_O@E78(ed6fB<6OU>RAqPlY_fiQIyMOeQM+s;CobKaFdOCype0T zb4W1dd?`I6HCNEgtA*k75$F5Bo!c9rhPsxIae5L#!tXiG=HyNGTeh0Osvw#T+Negi zY*uxUWm0&@o70dVu3``N&!+MtgPaVBVw=9|^PVj~%BernKo+@LZ26H($o8x~DxG_X zzEsesXN9gpIh8u+@tW zFUq1aQnBb8Zwa@1td`mv-QYBqJm%aUzv5PmKNjT1QgpYihC8Qrh8qdt3$zrf1@9&w zrWeLm)4$K}^ERoiq6;<0X}g6og6&B>%RxQ%aO*=9vM-vObT1cqR9~n4!xJ1%`;`ll z{I78h`x4Q`byie1Et`7yjldi4N>~HIYjiNv5Gkz5;nLbhXsdb{8jhIHeoIRu?fYla z_mv1OK9bDEzmh;fO1&souawT-nuPYwKgp(l+J*XJ%&Ef9voxmbD~*)>z>T@E)G|mE ziGO~}PJ5<@=5?E+nvgUkGL}rycb1J_?ZnziG_$o|HzLgy26X8@ee{W0Mu#P5B6fov zeQHLNSUJ zNF<}?7kQ}eej~DY(9FuYLU#2YyT+}bA5*8(8@PlusdN{sfI0)O3*I)wu@=5*>>F)W zq%x4pCQj6YDI8!G`FYg8;~ML`@h~T$vmFgsb1bRr75upAhicRdQLo=`cIcEftx;Wp ztOHf3Sd=_c#CVk_(aI0XeKdY}w7bs9fSga$e%QRX|J zo;a0*!kn+6EmKypPqts83$C)LUHTXMV8t8O>Cyz$?$tzx67o=ZlzJm?)*`O+x<1W& z(8Tjt^^aAoyTZC%E}$!wJ?O4p;adBA82vrA6U}L?p?&RUbZY*26hHi%w#vU}_w5y< zyBn@@tsFarmhCKkzx*ws_RWi?d10n-KG zAv@7C2HhLvb1Dy)39*h)UL0GD&p;Ee9iZ|L>x3qY9(&+TGK!m?#fgU|aox8X>C~Lrykkunf`|Tk zX!o7xoZ(%6)cE`fufSc*VgIqSoVwu&dNY1EYTi8;729>7bcw}u=urcAd44x4`f>wW zK_zZ^Q$A}wl#l#2R|!NudLh}5xoCR#TQ2cJG%GKZC3bAbCES&qgh zy+nT{y6B82z3iTFdv1A{(5G#=MFSNLP+EVAAn(Bdy?!Erq&zR7KdznQ8uEP6(#KQi zG4}`D-StxF?1dK2?(A!>?3xL^{CW>{8`or`N=^yfgifE#A!np9TtSOfnrL#KF=`9V zWXle#vDuqM>DRm3NZcY_Q0(xKz1Clc%-7elBi~k1xqU5E()uzr4oIdOqg`3;P4TGw zx;blqDFO|L9;R2{$J6a5@3^)4vgpJ+W4iBC0UNH{$(_m(n!4pCY=h<-dV2aoo}A5p zH2O{vd)7XN(=k&P=v@urM2z0DE9cxm59W!WsQe8!dZ0i(1BsJB%KvWzDqr(l@W&=@z&3Y;ROD5`FB4R%J`kn7^vXS(v%? zIKB~0S6+iQ-TuIq8ouSCzH^-TlK{H$tCWN4N@wKwS%xk5pXdC|4LU8_96NIhqq z`aj&oy(>D(1#7LQ8M;nrR*DyO%9+@B==cm?$K($7Y~?9r(Amhdt_wnI#IJBY_L7aK zZ2od;u_oL@FAY?$ltWLXC{guPCpOz=9lfC+P6r($xWl!Dbp8H3uIkT4>Sef;Zpv|| zecTc5S==Mu?GtWX^DAxoX^%j#L2ohLG50>bxn~F0o_UM2u=~eJ`s>iQ$#^ z#!Fo3F_41hofl?OwWp%8CDIP_ZbTu|y{nMGEdj+xzu{`)+UV&Ee*`?~4Ky}%5*wo@ z^j`jpv z$Qa{(IbKAK74Nyza?TXqfP%d~R`m6oAG|BVqiK47GiiUHDcWXfMl~v@Q@E*){jp*` zy9GC))4F{4OPG^a4U|Q`Gf$9%BcfFI_A0crz7yGWo3`^CVH~I4q(MW=AHnyR zve5N=OSouW6ZhM^njTu*$^PMAqs}5KjWOe4XvU=~IwIc1TOBJ!4{cX=$oMEurLIro z3e!AUy#>0BPn!ccMb?pODG+wchOg}OCP*jeh0qsg9|$JANu=`EhB(_-mDKT8COSAn zw6RU`IyG9bjGfo=n8gY6kzS-4r+(@mqE9^7WXB9{wa_Sx-V}wh?XRNgW<%WUzH{i> zPEAxM=0?S1@6vtZmh9zpF^7LOb7{M*mc!SMEIL&q5Cx~p@K$o(>^<*Mu6a{9J*|6& z`>qv^)Jg+rztVFyJxLt(*p&0qn{qgr9kY42K0ae(Y70@_-6WP@dII_T8PghPDTkMu zWt{AY0dj~c;-yBE(v=<;X@`w8^|Cz4P0{6ZAM@X{)$Z1utj;LPIiQ4QiyTHpzKxt` zpg3DDf0`~@D33mKebiO1UGQN@4fTJ2O}A<(a*bQwQCaV`^ygeZ!A93XHp?QGcBmC2 zg<2h2T=7ld9b3$qX?mjHFT6=X4WO@n%g{;ZwH(3@*`iHdHSFBf1hi_QBf2VG%_`0C zrv8h4IQH>fZi3eU7d@kn%@xsTT=yv)?U=s_ZNJ{jxeO&xlmEuJFbi80r&EpMSJ`q; z62%-eK2sENlwq$bN264G_ zG^Y4uuyfX)=hz#k5%<%cYJWCsoal~u?`C$f^R0ycA8WUw#%YmMyFZ0S7FyBLy@7PK z(?vAouSzBMC{XFNOteN_lyQk1WQ~SgxMQ+5^mx-Sa=iADbH6soJ5ZL*-6**)P|)tB z0aH8BnHdI+Cg>cueg9KdRl1y-pRwgUmsGN=W(;tTZhoR;65rT&&o0mzcM=7Yg?@@> z$Pg#8Cl$@{cSTF4>j~Tky#!M~D>lBLE}Yxwz8AWi^{k6+5E{(MqM{p2=mWt`?rE3> z{jPL@@-I}N-{s2G;vAovXy&uCoPY85Prk%H{_hIhRI~Dmv7DVx7W*aRH#dE3G5}97s(fC~sW>u7 zwg7XL>o2nTX(H4{A()!?R&is=0-9~1kEA~G&~vm1rG0f|^&X$0sYXHvG~yFmdFduP zarF|r_4RIU``UNZ#4M4`_U@yg&5bUzO+^h-2L*O;GP-v^4jB%+v29K|G)%6X{3+rF#@n zEAL|Z*tib8doJA=))<492vz2~-wSbk$GeyrOY#ObEhQk5RQ7Sv*ArJgG z#%*nKMJ5psQJ^*;H-~?*Y3)=$Hgcig16VuhLs>Ck4vkpV%UKJuc(LRk~`o zKFiq~vz6NVNbBw+L7z?-a(wVw@LN3ux!S8uyhX$HtaQ{NP(SPaAv@u-RSD~1}Y3C$zQ}^vb+^R|F%lX-KQ)MAq)cl(skjp|F zo(u0a7i`hZYjT|HjS;Ths2yD~PG*Pi*fg4U1<;16b<|O3E7CYxPB%Z^OlK;a2z}U< zG+5&|x+-+|@xfyPwR>mKj^?$3_kP!@Ti_>Ny`3gn8^mYVnU!&&s-tXTQYxK)a5b{Z zmKJsuong&fs<}EhLu7GUtTEf}EOL~Wpf?U2<*G&dsZysAl~Jmu7s{H^i!;k08J0ks zroU&a=3hYL7sb)KZ{F+g)XE|5#II{eSLOzt6nBKRExRp{ zKibH}<@B&qW+zd1_5JAMjwtp)qzb!W$Q9G?c4E%Tc2)@W+)fX3s>);8BLILbrQOdvA+chqdQU4^AdJ!_yM~p zTf-qkqK$jqX-^*pFGRdMr93s|`Ro~G9$j|K9gR*o!zvEspw1tI>>1nDT;A?Z`fXYl z%h*UGSvHL2O}s9+yZkw~sYVJ_vAgNfzgLj7>s#K!OO*6Tm2%|aOmyF>hqqhO8a+}i zqz_BCqg)jU_HRrdN*yiZKCQe!B}+^kjJO72@0AYMqOQ_-A@DD|pf?RI9ym+`?KSD~ zJMpRZ~U#`mOn`1RqMCV<^xs2 zLak5S9n-69mB%Em^XyIblUfw2IC7V5Jp7jaN_j`0??1rxTbMQ;A6&B^*ET*k@8 zXxi6Wu6gcaba|+agHETgxJN8I#p?t;7T`y-N-9zB_Gze~cRPBky^QVh5~m60tJ%;C zqO?Hhu9cn6qzCfdQ2b0WhdXAbXwx}i+uMMEJ>uezUbQQ65093hCH{N4A=6}PBRS3v zst<5Kb1aeaQ;zbqhus<^62AH42Dh9A1vI(C9X&57^a_^Nnu6W+8{giViLV3h0WzJSyA#hYI=!IMt;R^oXb(TmCefQw>k$ z5(8RT$@Qz~NW8Q|+OA&IvR;mT$v)unW{kA;Mz~-zzOHf>HKx^Xzm4H^iyOaQqT~i{u`!ojoVA9*R~$E-BgZd z#dAnb|2nGmO+oS%AyjmpGyULsl=t0Lg5G*-%2}IP(HRPD^rdVw&D-XI-YLAM0k_>b ztv3V0Ip==#z`c+ST3^mB|5HpG^Q6#o`?D-RI}tq;I(dap-V6GlG(#G9fExOQqOpq2 z^sUTXWHWw@4Zq|@3%}hHlzw>xP3PI7y2>c}=EWg;dSxzZKRBIrJYmG^@_x%bTRMrLs#l9K9egd>Y}o8 zj#PAK50bwZPercEpnPT>J^I0evaV5Ri?KMm*K32&hj5743R)JP$4hlk`5#5+;aAi9 z2k@4L(lSbENGX~Wy63s)xhX}ZJ!lUkk|I%r_TJiC(x72P-Sgb@B)cezgviK>^vxhdT~qiKoEBwR)uSdSa4^hDYpIUf|aBj;Y0BxY~vD* z$9yJf|H6G(&w^oNCU(-3`*m^frd%q^s|dbK6~h76XJFX*QgUQ>5~NNpB9~}S!SMTW z@V){+8lKI9dXhF!Z08WEq}>VIGaf-X^B1`PGZPQemr3zO%Cyyc1?*k;6?!}_gW6_` zIZ{Hk5n#|Mx+B9C^L7n2c4EhC(%^_(!-g9Qob>J-5H3 z+m2kJZyl|~rX+J zpy8@tyg2no^x(2+TszQ03TuZ$Og2lIH$vkF0#6|lDzP>9Qo10~lmfwp=vTT;O(n^xE zH$el(|L(&TQ?F=)d9kqWTsWS8U)Z;3j?`81gk5px zXkSAaD1USiN+-vW?@A@GddV?NY%GP7k4s4BL$7g$=mU0SpBX8!bvJ!~9~+)77VSuz zxq)HW1zhgXfGuvaI2UItV1B6wmXK6{erIgpP?tUZo%z1vKZQCXUTLrwU23DFLvPSW zepkX>j}_tk$V~iPQVWhG_h7c~5tuWzkuB)I1xp8XW65({$V0z2k&k^K4m$B3=IOqL z#|y=A>zE!W?2);sAR3!w3g}I;M^? zPM)$Bg;&#i>8%e`@wD~;UUcyZeIzIyUwIdUe;+NR)#nM(wZ|)Ihq_YCeRY~H?|co6 z`idd{yT3e6_HVpx*cA#!+QN|MI{3=RLVW+&0DadmhW$pOioABX0WOmr#4j_3uzaN~ zmJ&0E?-w(9XuK00SlR)@7KG8nx(zsd_9rc$p2)bbc-Y`>C|$?5hI~KSOscWuNj|S) ze6f84y=!=!TnKODm;2A**qm7Qv0EGQg1(D5r~SwsyW8N#0UoyhqD)St^w5Fgraa0ayL9xx`nHNXh!|hK?*o4hFa^LGGIv;C*BT*vr8@64>ByU_K&ZyzilU5aU=nb z589A1nagnTnM&*vNs$w+1~Bdb3m5W=pq2ZYj^LLi@J(td_VznRp4CJc3$!o>q=@K{3|KDR9v-v5t)sWmbki(coEHOG6{9o<*qARSL~JeJVo zSH!W9<6&}XMgdH^@{&AuZY#cBGLDTd?#K0T0}N2`!T&AFBmdt1Lrz`Jr9BSDhClI<(m!91p%(P#r&X})o;!X1(I_dN=7?wV8nF3>KA65}Q^&S2S=X3x6QJ9+WAbaIX^F)=3#*6?*? zop^<%K9#`h%l?ul?=(O!VHMtp{Yu)ukt9Qu7oBi338v<} z!xeE`vH8sq$oFv>oVH}VF}bRow+$2I;QrUVNl`m!yx){LPn$4VbeY_Hpowg32qhnX zQRdWC@1mb5L_*IQf7rs>M@!s%NM7NoLyg=&JP00>{TV*c{_<~j`|c!m)hY`-Cv%p} zlRJxz4{I>|<(<&y^cel=To^Vfn1Gx;l$?|); z?BKu=+~&Fu);vmv%m4mhTiuX_zbJXUx5SkMx<}xuo9 z^&WY|+!cRzyF=OwyOMVCqx5K`0$g9n^ya@jCpmU=_&=5JqLxG!7SCX>&?flevJAY{@*6E# zpaJ*n&%oQy#*!x+e!%DQckw=>rFg376dTzdU_Q4Mjpd`%X4iQP`? zxzEE9nZ;PS`ZaBo6b`#S$#G0m>q-5R#q>d)8*pZ(kvu%9N(yE<;{VPYLcQR7cqi!P z>Eny!GrMN!cPpLzvHvcdXA+Ey&KkoifqLv*=?n815BgKRbFc`dzzZMx;7g$<-Ubx| zSdy|Emsq?aKhGrNr-zD34&S_v5s6wdr1lJ?0grbo>Ir#bA`d!jBQK`O%ClRc*`0Dcb>%YbE0ydRnfyR5@~FkW`}sQd zyZhp>34|L?r@`WwCvZ?C9}geb!J=d`Pygb)4w0!=T)prjl)tkAF1+MOUSBDTtG6Gg zt=-P!-Ojn#Pc937eV$Fm2%W^LZHn-L_Dx7Vmg@L#^*UVlEsS2NAqpq_Sa4P86J-mUcqUcKoG)5v08@A_ugNC2V0INM(NIE|3!ZK;*kdMES7Wp8Bg~R;dUHvY$PEjxUZ6g;8 zgdW3!dhTqNNHd=Qa3Az3F~Bp1CiqKv48G@{g$v_MVe9ZD9eUyoe)o7QB)8B_t= zmp&v#qSnHgZTDeNS}^|EoWtvRR)Gr^h{21Groi7_DfD5ZUpU^c4VSRA@Nl~i>8_he zXC=?@?29kKESD^3tzJsrtO~-L7D0#GRCiTKCi!!S{O22U>zfnHq^q`BOAES0EAOOJZcXGQg3 z%7%BOc-BX}zAc3pGt5tZE$_k0z4URtwi&_$p1e4=e z{=?rloaRZ7#bfDZd&riV6#P8Qn|%HtnzWw!1*fWHIv&mD<0^|_SgEbdiEV19_iOs$ zQH=s@o$pP)x^a%&-D(eq^$)`YJ}cOrdX>DKol9@aYG-Txs)l!&834UmZ+N(d=>zIJ z(K>zw%uLj3?7L!;t}SxGdv{%hM(K^@v1h&HX}S%UT4>^#1xeWY0w3q#Z&x_^r3GFw zm4l`#+n}}q3#+Aw(1&ji;TbJUe0Zpn_GwDymDZ{lyWgp&6Q;kwo-LN}ol+b~{ zQtXT)emG!5FTb)Jw)P|fxoTUww3cznu3Zmz$Ij5z8_(eW?l#);%_TZ;t1PC)1v~0T zx8aoKQoI`<7In0AsDtALr^z-}75%=i96G;`CjYu*&^;TKaqERixIi$M)>gO${uWo! z)+`P9--}w9{-+hLaehXdzh*o%N;!Dm)GWQ-+6tFRE#O=UwZ(I>Zy2Y7>5zy^fTROd;Hk6saXjN} zFpjHc!%E=U{$Fl z924{Vq^jXL_3rvej_-N-93a%fWXN3P_K# zO0u)V0{{9XOSipw40ZgBNo7w>oOwC}df)HGA}i|YlCEvoKvA0WcA}G^@xx?b|S^6B)1NfV=i5)$j2c>JZA=h3Pb3R+c zFY{O6HWokUhC?@8c(M+D6?%&Y*EM1+zmD@>B^qaHzJ~s4!X5t^%E5mR*8(-Mjd<|r z68Nf6AE$iJ#nrdspnl^G__45n9KXchAr$l%3-CuHs(bI<*iaPRRe<3to z-a_9()p+3(4!QHkC3?X+8tx3pgpCK8JNL6Htuz)5#bbtG*oy{u{P9ij_M^J@l58rQr)1|q*DAAqJ%l0V#l|KkY!#S|s@C2#6VF*fRdSg4cNbDfh zj2mWa$bokUq3bsZPVMz0Jo>{J-GBHs)LB^or@%Dsl%FJz>t$oibHkEz*Ku%@J>1PW zLD)YfId?Xd;zvIl;o*ll_-k4ajhOSgQ~ffOGb@Jdi2bm>Oco1&S_QRn3u!NKja|3) z2Hi7z8}2g8^Pzh*Xz{t2HU>EFuyMR%t3+U67rT#B@^g}=daN+ zH%obLLdLioxsvxy;>k6F!X38%0`O7`TkI?%L&hJwP4B#X35xD$#%V@;oIe{x;Iv{d zz2{RPU7fkIW3gc;gp2xM6;n?=BKHBS$S%YEd-|YLd^D7H&10N=RnVAw35cw&W|i({ zayh1#l_5pK^@9TVc~B%&de=rey;r7}6lJ4si7Vs`{SdxTI7^EgyTg9@UflFOmwYD_ zK`(xtgmaW@U_?(Id(B4+YI)NO+EDrhdz;r1{B&>xuHMQABl|yKn_f+Le0&YQU%D6C zzMx^ph!5Emn#+5fn7#y5_0|%7aadcHMz0_K+Lp1m| z{YfbX@2qu#?YNVEBfK5<^YZW-S0-kJRT>H z&rQ6bANxIkuUE^HRh5FUq%e{G+*D7W?E21l;TF-?LT+QoUq@G}$l~0HCnOjm@uS%d z>^J3wcuk`-Y_spDXZg*b=WPwR=U_id<=8y5`g!zQEFx? zg|@xp4aI|gGF_cosL{I#QXj18(z)k&WW5!feYY2WIsXZNii#y)Z&pD%zbv5bP#bnW zHUL@4H+h|99Q--^0$rMJ4A+*4(XBi!Z2svl9gOzTKM%w~`j`WJ*K!R1$vXvQzv*>+ ze3S;AR&-+@JsUWfa|uR$*p4q*DdW}B<8bu87P-b;2R}27g6%uFcw1#U-Js=+!iJR4%sYOO#XTy3HP5^#S_lj2^Ixx#Mnmx!)OzpT%#pCf4zf@y<*OLWiJH> zQWQ8pYBaH;e;z{>UJZGDDZCH+cHxS)-@Jk*J@VETEl%z78PKZa3>|i#CZ9*2BWwLu zb~s)i$FDwg&<-yTWAD^h^5L#ray_3h&i%cX*5#_;|HPshH{uHX@7e>B+z^6$WLs&e zpuJdWlQ+rNe}nwup~kF&o8wTYv$SmeFJ>0O8z1>yP6|J$hM5jMors~|gcKEAZnmwwoWVcqk? zWb))kT;CUr^DDgIdu1)WKBf-RE)iJqz6NJqpeR=Fev9qe3dlE38k`fCztD%bHIOx+ z1J+d3VP!WB8Xl-*H}8>vg&Nu&tB+J+yQS+mxk;bsBh89%Nl7REhqvIa*K6RaIt^?Y zA_M(BRN&jSj`-l>ZnAM_EG`~$$HzDCB)^x`Kr7dy?9cb}@q*w1*dNfs-pfg0>t!y& zZ^IMe)g?b^ca?Hlrz#(s%@*Ox=&ST{=Rqve_LP>}+e4n04ekgt)5ZZJAK6O2%VBj! z44jfE!#13DTK8W$c3bp@H|!ISPsE*o$w%*!bB-}^ZX;7;J2U`odVk^LnU`Shlou3~ z$s^CK`b-zNb;3G*38ugnh_lVJ@Rnb;IOi3nm3OU&8GP+z%aTrLFD8lu-JP%N~v zmw`*0JaFJMLl}4E9{H`RfD{4BSoeoB zXD@MRy#V%NP2l~r>iALmMWz-k3YVE>!VLpS^doi*+j+69K>8P*&!9@zq zyWNX;{{rmj-zFni>9-xs&@Un?0{rmXuf=5Ee2I>R-^_imbb|54>tQ}~aeVAoK2Gx` z;joo9Tkf?DZf*?678+tW+xadwT5b*9yHAm-%LSl*i4LYwG%`1C_GxpVFko%%2vuX}bL%WUlG-=We9Mh zH8$hQycE(~%bj*2446Ii1)LpcbFj2y9Ng|$2_-9%>7RE>=pm|z@%lbz=2jBPS5mE5 zd5Z^p64yyC(jOY!WC5b@}_*XY5Zw8@Fnxb)Xz7pF2sPuf2d>orm$C2|yoE>Vn;cjihTyAar;p zNlzmOpdVC4e|B^Qy6y=`e}NB5U*L~|Z##fHMLUtayg5Q{4k*mP1{g`SQrY%fk%+D* zVu|z+{-U=D*&P~$(!3|sIj?x6F=h+o2U0=wj#w~$-x|rvej(0Whrno?1+ct*mAW;u z08AUs5)W)jh%+w&fYPyp$gm?A)yswb`Ne&9 z=p?0fL4{I3)XZ&)s$^w-`_5_%9iaBMKOvUfc}W!9?jgDYbJ3_(1t>_X0zv)7K*kzV zJ`ev89O@MjRoy{Yd9Fm$rp90>Z5)EKsxu!I!&JIhD?ezgctcP|hFbp^(fPyB(;LpS7EejMc;+KC?S zH3H$+l+e=|Hi~>jqU8;x+{DCo?n#mUcENXdS$p3;XQ?=3Q;CN56rYt272&;*aA`Y9 zHS{VQf2!O<4QzQs;Bqr$9PySqyyO$*tCI(MUR5%`e-&C3UxE&$=@{QRe1OvKlv#ht z``HGUd(y~B$q}S6OtC4oKZJ3jJ~4O0lWOVkr&^;sx$kO4QH_TIP+^#EM?EA#@ZQ@* zt%L${YkNc8bi6{Xl#WIr4awk?Y8qJHo&stjg{isAO{|&~Zmf|@4J@t)K>kWuV6}ND zcoSs?47TN?A=xT$IrTK?Tv7ro!W2MGS|j(xPyKbivOEjWmtTV*53zEot_ znTJ|(YJi<_4G>#e2H^dz;Kvs})c#r!jZZKvtFB-q#B%|nW04>)>^Sg*+ll2Vm#FJg zc3k$0)zl^@L1Ni(O}jl;l~O;lp1P&+iBPz~BTO1LQjRaisDpLppsYs`6|}L?c(EPQ zySNZl_1FWeGcLf|NgV`!E(XO#CCIk_3|dxDjzWWsfWZ7{lzL|mvI^UXHVwF;yLtZL z1-}cJYO(}N$052Dy$ehidVmCE5AJQ|r@Fke2(6w`sPto2lq=tNZn=dd z(H*uHmGvJ2=h9+8M&%m7kUfFqVOL~xFAdTERifK!G1S!WdBAFB0Y90VmnrV_}L1? z9jHJnjB3!MI2pvZVFN&0)`N^{evr;#p6|t(3}>Vq9iNwkQui<{t;=yBePI%qZH@t1 zD+p@mw)tTBs%B!3+%PNRU=kud^TAu;Q=pnTE1_x`!1!rCy12Lq?WxK^CUah>dA=1m ze`Xs1vjO1UNEZ0yRf;MvRUwPOLZt5=1)`2*BCkyuXz#obWaC}b{^N@W_f1?jcXhMCIbhtBjDSZ8{)ri z2*&1_fK!}ZK$Vn1H}xJ81!xVB_mBhWwK3rQgJ={Sy#rZqj73l74}BN=A-6_O5d7S7Og`oK5&SRF+uA7qbSPJ z%$K_BBm*jSC6V~Lg=l_~DC%vsq{4nQ5nY$13D0C7V&cp(bU!{HI9d{XjA zne&~xcu*dAm97C#u6`n_|Jbk&MslvQev|EUGpAS{dv#c~s(OTG;y0G3S>1Zch(o9) zY&C$TVPKC}1V}&IPMml$M19(hDaj2b)b|Jp;_HPA#DQ(?1W~^bR0ryU3ZL%;)w7Lz z&hG}R;%pZ2QZbC;zurTUX#-S+9{?(Aw@?pUr>JYD3y@2`6=3ylN0*jZp=7gVsPL9O z@NxUa@~PoRpHAqYwb7@jAML_~@%B5!RV8DPLYGpDOgpF-A0JVN*L|b-M3xdhtXZye z#dlUk3m=ehzu&ISa4q?V%cwlxE$yclE~0LjYO;iS1*wJi01N){1+#C&xjqwLsOvuV z)cl+@)Lv%?zTdC}Lf4lAQrrry{=Exy)|r7d2S$nJ*N2I;OBwABroHVNz3$WuUp~Q} zv828=ePJCMl;f7=G55(Q85EEIQhUzaqf{3JpuL}-bA1@*mhFTD@hp#EqPhXf=?s@z z)=p8@^NYdeX$y44!yiRR>LZ=1JYwS=c{FijoGSHIV0e&EsG3(=Aaa&rG%+5LVUJ*H zsBay3qp}$2ge(S`=_k?V*WSQy&j%u_)c}aQR8V?rp0J8L%?Mty1fYwO!6TnUYF&*M z<$64n`|_?RN)5V6oz3p0n$D8ww0|4%FK#t*aF#&o@AwhBIv;pN{$d- ztrDqaC)%mE9lME1w;jZZhyWzKVF#!Ru>uCg{Y2(%9dMw>73oIarzVv=QNUJ960eY_84NbCW_){bE7ZW~Z@Xd4oHy8*mf%?IiO z1AyoBeCmsMI@R+RQS%Idm&VvB1vx142wZtDLf^veA}(0CAty0!s5T{6m5NtGa0w7lmY z=C=R`Ex&SK#*b6`=YOUWYOZiU{X5t0_9%vxnY562Uoj62*LtuNjX2zVB`Gw*8lsHd zjKQM*Wa53?1`ypIOGIZYQZE)J6Qk`7gpu`DYV%+}(Xcmw*t=$sI0z*We_;)!-Rwbh zH{=mwoAxqnmes&tW`kQXl>MdvX0U=Bhf}MPN?n4 z25N@=RI%hg?)9+(Vqcj)^-xe5tx7_4`xQZ}n8-hWlD{?VC3W+w_Op zwVIDgJRl9^k_Cym#RA}Pz!9YMH4o?{rGO7MO~kLt4b*~%Ux?u$f(nci0Pdv;Xdamc z^aQg3Tjvxvl2`z4t8YZ{?+ehwr~yJuVISE2H3Nw`W}pcxTeRoH3uP(<1xcRw=P~UcflDm|GY;T?bCx90S8FB+!saw$7Kc~QrvIO8GV8B|Fp99<; z=Og|%`Dlm58LEr51I*7dMXA1~$dh>vV~yvC-eV!$clRF>dJeUe6g0l)$vsGL~on3Fd>FL}P&zdSNAiewB@~T6gym#?R`gVrvq? zpxr>+{s0JMd9%7$c2uvd75HRm2p*@pAt5M+6z{BJSbD~yLBjKM-K5W=o(jb!y@9cs|iuFG=*5Z z{sDD1A)x(w!YkGS>nq$d>2Hav{^Qhqn^oXotTM3Xy=S>sDgt0-igGqdAmP9TD9PRz z8ChC`&|+tBhWm@veCh;f>54%cube`=)0EM5o3}({s}LI7I7NBedU5Z+v;^w87RVGP|FOe6u56smvj87wVbRK)474w!N%L8AC@!rS8O&24uuOJ4@il(69 zz1b*cu##B6EDeN9W+0itK&0xwg6e*Gj4*F{NiBEeqe>Z8^<2{l@FO|~?RmKyO)hY2 z6R-$GSM-v=Zr3F6(Zm_NJXnBYZs!BPQ)j@v#nxc9CKaiLMFX8HrQlWYQSfVI7YH(s zMa&F8`l{iCrs_DLM^FgW445Mr0g-3TI&@k@7<2+PV0qO7EbDKkp8LEbx+WF?0epc_ z&N~hqsy&h6cnrD}xf|`@e~Fc_zl6B_rh|BX?KjJG!BQ09WChCNSwJ*x1!&gKK%LI{ z;7D>B7>4OUs>7Ce-@2T7t}BHkKJTIEwACnBV2&7MV$>=AwLlL!5_KjTV9IYkVECG7 z#l=6yf1GVmZOv`!TDKT=>VzlyCl(E^d^`c}X}E*8Z9$;#&U>z|)@D>CybIMvHWBlJ z+=*+QTZxVTN~oPHRFF{LZdPI49wMbs7Hs+wj>7yiL6=DmSY4I@ydB?BYeb{L?ZJ3p zac&9d>{Uc=Uo3%|oiW&cT@&o@R0V287?RcUMSEY$A_Ge^FxI{Zg~}Qtg$O&;dxUXB zs-*(iw@1K|u|&}DvziJHI|{NH_hjc8A)=qj0fGk_h(&4x#Aew3HXg$9VrG@yQ zi$+OETOa`RkA5p^6;q}vI~#~~ z0saJEyJ~!5q*A8^IV$t`wfPwiJ-tFCUCG z<8X{jdqV6KRCZ@#29 zf0iP?1b?S=o+VI2?i;DcdEr!Zf;9MXKn7G@yh`j!F+rzIARycJ0h?z`J!MNDB{QsT zy!DcuaZx4>T?tdp>ykt9=9}&*|*_wpld%={VoRb0@LQFQ@??*qfL3|5(RKsUdK|#y_6A_0<|#3@X9MtLzh>D+ zE;s%x?}<#c+zD1(H*rY1lL%c}NUhx(LiJh_)Z(grM5Jjw;qqP?g{1EX^_k2&vAl)Q z-Ts@J?>$O9Zk7fQ?6)@VH43h&cxZH7F6!h67Hkt$wbGMUMlst z9@69g$z4@r2)edN0d~znB=l%As;pg1tcY$R&Uv^~%QP-<=RDY4?lvapx6B1yv*|!# zC>bRDlV_Y9b^!QUflh`kE#8s|@{G#S^K->$<(^nnZuf2f3jW=wthbvO5W#U2zH zoeECYB!F7MWuQY)j+=jd8BuFq&NUm+ry>e6fxB}K+IX=Pl{l85*E(fPtjE-fQfkl% z%~Eu={V%sCx`;bh!Mer}B-88&K9)uRUy~3pZI+Vm z!i-~Kh6>9O2C*Y41ds6qHvJN!?2^8*b^q6^%uS%p(z%bycI%EtWwd+>UQGu zg^L77UXt4oDUOu#vRT_p_o2#X>!`~Xs)+t~2)Nceq1MaHf2<#el)ILq&u5+xY3XY~ zhp7ME{$_0FX(}E;N*`fh=bt!TP}%ptxEUthL<%3WYP!5{p)B&uw9+#Lo>fO3I4I=AyMckwj>EUXji&3LG;ft z3jH?pLZ7M$WR$iVjKGcH3-fIK$+t&Nz0CVfAq+gU_6J&*9l&13mnJ%IJu3blgkJb) zf(0@zXh)hP8oXO<$s@A8S#_1{Ua_ z-Uc4LkO!{oOVQlc$3$j-4KSXG1t|%kgxS^C#Piq1?M**6ZlIXhp^fpE%uJnPSle5W zrq}^0(=Laod+>6K9^h=O3esF-i~5G`(6;{=r?8|Y6+JBqwlowm&J;m( zp+t{xC~6^eOoxb9>Ym_X*KXt~6oS^eF+N&nGa%Em3TgdGLhhSZBE#5O7IB(Io!=!* z0FzA0THq=5@P-`l4jv?G{xa@z!2#mXEd%t$>@#KFh$%cBMyE8PTcft3PbtIp-@xL*0F%Vbls8r>v=k?{BaK1S#S>ZwdNg;7xI7rHI9P zA;{A!7Sw1)06TOL%$&MP49>Hrj?9))St1EsUk?|s^TT$;+u?$KJ==zC&-sHL5eLwQ zzCg6(y+4|D%_CMTouSyAl}Nkw6eZ`o6OFBO2P)P+K*5oj)kqaVD@vV#+I&e68NuY( zdb&iNhbBd>_Mq0B_|3Yx!3a1+9;KeiY$T4ANurD z3i$Lhc`-YgxUevfh*B~-bpm-r?igf4rkp!U=gln>+ZSXJqR?w;6(OcFehmktY9 z`C21B!VWcyn4oZj)nLW>UDTbDw_H)rY>M)?M7At%;J?Ei^h+p!mCey8StSWnW@ZBM z=b1pj*Z`%zod;$VB>?DPe9((!L45o%Ds1cKnW^bp!5#kBGfgVk=nyxaCGN>AS~?)1cg@+g@5l7JuBA{ z+56O}PxcnTW!GD7|H69eb-E?GQZ>K*w2vLDdhdPXh1dCr@^VL1X%0bI4Udq%KS!(_ z(jf+|%$T^MIdK_KiB#n5u+^^t?fS z#~L8G)rSa*(FD?IE0I*DAu5}Rr(&Yhz>TCBRHKxDj{XZmQ9efc5rVV%T_!(4IQq z=AZ9MjTsJ7T+^3C{TpGVEjddi+r6VKrpIeci|HV1HHJPO=+$2iRz^;k-| zkzlW^F$j1(%@xvdN5}RvJx)b`kWP64)zKVcZEUk~(1$uIar6`A5*vbCuABf-=|{lW zU^Hmxl|tP0Erj!eYQm>k7@XE^Vd*~cAUflnjTJ<{a64MExS>it?K*eGfli(XwWXT? z%4Hjg@-6=8=dVIyJY5g%+x?2#r}l;DeV4;kixETb9E5?Hz*11`LIQ&XVX#3u6{P1eYo6b98Ooq?cF&Po9OfNOP_A2)elWXsyHQRn~bU_kQT7=hm zrk*@;ia4?6E4Lc#q5{p2Q+|X0acdeg0n0pri9oI4G=#A1%O}RI`CfCg{WXjA-VaTjKkR%#kTH3hjs)r z{yG3Y1~9d_?`9~K(j=PItwDx#DB&sUOIeXNpm*dO;rrPR6i{~UKlMaVysjU`71F1) z^Oe9%m@A+o8>v-u#{sg=MSI@YQLXxiz~Hx9u8yV#2pu{_M4wEeN^+A~Zc7fL2eFn+ ztg8%KTK1q%Z7V^!i8nZND2|AA@CP@`7!T_-QzHl!1~-^EPdcNEP^vURl8-l|X1E*O z-l~IaH=g6pZ0I8%3Ca@5j4MLd>>%OEcY=D^QA5a-2ZQiscHq6%A@pgq;C~dId0b52 z8^_!Cv}>cZFC9CVa-1F+r_ju>NEE%?bu@64V?E4sFPc4Aq7g;SX?HU?e-d@-0tgRXRIqSQ+RhIx$M;|A8@& zPXLER4L0*k1)enh5%`aD`TY(40?M{0fnT-5cz;y^_Zys5|4S;$^R=@NzmNvE7TsAar7faEkgr;pIAR zcPaTQPEz(`3u>Y=^8+Q}sHw*1so|Oq|X9lvQV> zil;Hw%RHH@-d%Xvq!`v$k6`o48DM^RJtiTI!nkSU+1e9dx%v7#JlVVooZfr}mthUi zyK4fgyz-auNtQCZ`(z~ZIDa1VVYec?|F#bEN@X{*@#;n4u3d-NmlM~pdcT9&&`a)Y zzR^tPOOGa-llBtVw9B(vKVv~)LKaScmFi;ePh37!#PBxP%;a5A3;_X6SHL2HXu6$d zk#oF82ov;#d*(^cXH}==f*Y!PnE1W_F@_rc?A&ibOy*cSHdsc3QHUJFJXN2?RAy$e z9qYG(49mUDZx0MI^ir9KnycV-b~gKB>Q3H>6T_yOnKI%mf2Q!Z4YPG|3+}I&j5B9? zfjyO7myH|KSu5ExEbkD*#7DgVAhTu(e#h_6?q__)I5JN+9%sj| zzl?Q%MYHWr7lpp+OPQQ(1?Fn)7ToxV$Bgk?;bO&G#*`?;G8=jm!J3cRY}nT{p}Eqp z221Vh*xMzZSM*z!9T%oQU3K8LOTy;2yrvg#aIDEemm-_lyxa4vzztnEv)zxq=>Xb1O( zD>s1a)|;Sq_9^gru-s*>TrV(lZ@}x1=&~KF((!ByJ$4@F4nAHj$F%-h&d!=Pfw}Wg zmYGm>x8dkg2jQpd+Q28wf)$6{#f+@roN$ zxr*x_FVbfI9@@`7D6nHU#S}36E8pVuW(USx{Sv<5Ifqqf&|zEMZ{jsiB?KP(i%^YC z0~tH+9G!LhD4iIojE-G0rF4&!z|e39`7t|<#4Vmh+`jgpzB41#yu1|cy)Pyjy5(pY zFBFyONgzedcj!rH1~Ljg2yLQ*sq^=AD&PASMSMx8lSjEM%WrQwbpIIA9Bx7Cfe&DG z%4hzxm&J4~Spv1r_R^~4U1)Uf6hvTxXx#yGy8WIsT$^c%hGbLFYxxXn-6SC{GWErTRKnl)ThZ*=8EC`tH*mY+C{@U7hH_lCu@gGdaeqQl zNT?khdaor;JyC{4W6aUwAZ7Y++X&niyOgi@b{sOg`xJ(GeCAV5?Qv@!pzWKaxb>#Z z^kDc-v}*nxIQUba480i&&2{gbVhZ>!~KyAz8DYILbCKa8e4xdfXwNtM2j?DyG5toK?JyNKvN&zvxRY{g| z`~;ELPWW+xG=Gy!iYQ~wWpZxuHgu~?3hDn~=?-Zzof>+AJSm-s(pQ`%!@ZK!%s3Xw z$GH=g*&Z}|{8JQoph1AQ_pPQA@HPu@fDI@sn71{(cV=q2`5TE+X{y{Cs-%{tDV+DkWIC zDi%I`I~Cnpzlb(gn~2%}!NIn!s+i&nwNQne7x_L)P&Lu}5+!RQiebj2u=j_sa^6ysR>Y`s0- zXe!6X{H=_dYIeixy(`GV?*u(puB9J;bim-}7fG@3DUpAxA#R>vL8Xn3!F1OmwDNlp zvK>_BvTVj;ndE(R!u9~PcIXAF_=SkY=VWTYuO*|`>O~iHL}c8GCMcm2z~#sb(8l;x zNN9Y5sMqVEV7FBCVEQ8%c-I02E&B!r^P=IWZT*Pv*-eVO=289OM7lZ2kJQwe)0Z~I z{HhRV8X8F%+UiC1?FrOM9Y)Ya8+U;Gfw3g{sV1OV+xjzcT|9VRl*(x}1`FSPH6l;G!$ zGf26w2F-69pzR-0(1$70p-){i5qHbe;_kjSy=1e)8EM5Qwi(%g6Ds6t{3?7px^)NERU zM&EeSgRNeu`G^sH?=Jzjna|)SE^>tLr7lxT*Iz`Z|0#N>A3?8HUxXJnM41Z&bwBW$W6~&*O?P#O zM#BrXE;I$rF?T_eE{M=MIYr_Wyn(xpg!);vp)i$Bdg!y5Y>k-AxsVQ_2Qv5J$Nd7* z-L!&SxxIo;e{YS3+)g6Tz$GZ%Wg&mo&HF_2T^O?cy%(XE(P&$|7Ro8GLei=K$dJWe z^fx($Z}cOD&X(Vg9G`xIdv!n45IUK@vuKBdeFL;}lREAHI?V6BxsOW6qzjHpS^{^i zU2yaD7OI!kAUb+#32CX-6ZC@Hl(%dOiN3u-6utyd$FyoD;OlonYQB-ak#;&EZH&0i zwGKsc_sfU!qhze|`Dl4vAn>#6sxxaV-?|6w-vyoBZ^%yU~>TCaPq$h)kB35R`9}BCThSqwj_s zdvnHA6dv0PCv!QlEn_0+e*F^K=axyU)}2R_v(hL~Go;#Y`1~^IyCk3@lL%(ZQ18hz zk$)YYw=srr!8sd?d9Fnq}Uq9ZmlNs!*!5; z`~>l}+HvnuylJd8p|o=S6X_Lz@*ZQrm4lXxF4) z(4}AoJ=?zuxy^N_Ia4m;8S7i&Ev?NYF#HoTkZq>bk)d>ad>h%Hl8)Zl6p&vYM$|vF zPxNnc9J^xDa^{a#4o%Ga04l2cxr^+K9Ut6-&`v3 zGY~~?OQ8Bf7bK}}MmLyUqMcSQG_3Uxyy?E3?pGFxQf(S(Uv(ymOYSB)ydY>+lLft0 zqmbU-A+q8}CkkKn4=x&`At*C+g8y;(p(_=yNk5uPq)k3iUuj8E(N#ON$~l@oQmQ9O zTYHhj%oTK2UJ2S-vIFw!b73;~-OzePabsW)-6rpfqE%W&+0V!J3(9Ox+%K^H$fLq=9Fq{)ukkaI;B z+|0;}`c@W_AroI%qN+u=G2vvyZa&?odWubIGolsOL#gA;73j}DKX~ZrL(z;uH#k>* z9fGZm^r!whs(xS@QeTpU#2eI-8Dg7De0p}%0I!cIuH_-kocNSqKdA}MEC4jt zaU#4IVN2CMrBToIzsT`}IdJ|Z0opZLK`=5&lJ5Mr4cXn9jt*s=q%yJ>X5^wki)L2PB@LM=z|XC~XSdVNwNM=Uqg5R)mYz zFb@zK&_c(qno}n~b@9Yed6A7xJ&NDBg1ilhrV?-0!*`+5#Kr17xnJN-FD%_cN0-;b zo08tNUp0g}9V%z@{MXPbwv%4dFs4Ht8sfMh&b#s7UbOb17yUlD0j-?l38nN5k?H&; zh~YHMwVKhVRYLwkXm%7@;j0Wp??D4{Co_uP%cCX=Lzi)?iJk$t%QdqEyc2Hi&5008j-ti zIPA%bLhoNBkmVeIbaGn>wODk3+86F5>I<&Gb6i?I+2#-5v}qC47!;$mK4Kc-rbz2Q zwnN{~GvJ2AC3L-#JlUfDhDuINI*0KJdK3 zpYUrG2FzGZ%}&+Rub(oIv%3dcs4XLi&)f-@*c74OK{;{qfEOL0N@(0#6(qlR3Jd^V z{Op)_FxqW4opQ+p25S9-N3_*xv+%v>&9|xO$NOPo82t#E?-7v8X-4$!s(M#KNV0F}Ckw@hj*uq59&?#KL z`1S|Ud5cL@Q{p$YFWXPTRjZNpM>VAI)Qv7#EKehC=AciX?!Zp@Y+Af(2${a*GS8WF zsL)Odx!x9_`EoBg?*3nLNV9?Jx8~5*0(+`CUV+Qv>Y>-14^xq2ynQp@g<@}gA+uhO z707fIz*PG%8nMWkD)cMU#kWR8&It%@slEUw$VQ{FUk}1}Urz9CBnHuwaBpT4$5R6|2rsReC|(*Iz#rFm8I85Bpf2A(&{I4c!Iv*z=_;#6 z7%^cE?KfIW4{=}-g269B&0pt10`d^ZjM}wFuRcCakq8o)Js)>v4 zIwOs>0mwFi%dtO~<~#59K^kM8BP}a)IClMZYPc;L!K3eJmf3DnVYCnKzL!s*c8^&RupKH9Sv4 z6U^k{`aLVC@Zb^D+m(#McwZs&;}LI4)OwonGLTkDo71LC+H^6;S(&AOf=-*6MQ=aZ zgp6G+pruwEP4s^PKUd|#6!in>F_aPy-ke7}`h98f%`O=5>?M5FR?J_wv=m*Lr$>Ez zWCi@_UNrSvx~N26nTWXT?%r?8bZO@cXm$sp&PV=q&*K&7m)kL7|CgYE=BHFz!48!t z%tNQQJ*4x4Hq*P~T*clm^-+NBO6V{;hipIV3@_fYqB;fAFlo$9wCv3gstnR%&%Bis zM=|3?*R0e8%@5s?xc?COa8LpUbu2=a^;hV#h8pJIi7j+@)G=z{v=e#UmlN2#>XB2A z%IVS}If0bs4p?>OxX5>8E427m4g*79(9FKARH9RfP7S#U4LK&vt>Jw7+U_*$>68#0 z9hXVB$5-%Y+#F9$W^biuh4)aa(`?cHoSXb+4{!L)QHT0^O9*BKnen&2X{Be}w9s2i zTY6RP1`1QX4hs~sXxv}`9V?y5^}yQD)vo#Qx#Kc=TzQ0^xpo#k`BY0DZkP&#Wv;>3 z6Q+t6c$8E1y&@X>c{j?uo&u@!Pr4+=kLugWB8kv6^mDqbIOF3lX4aHEr1DXN$|+^T zW2YMV`7U*^?2s?|EXtyxOZ=#={3yx1unvvS0`#hS8C|#Dlb(}*fQ+RBNty*qV$Pj~ zIyTj$bK!UNBQ29QZB9T3Vn*l%w@2{DwsT0m={J#G@{kt!-h^eVHzC6;0ZnVnhHoOI zP|4*hw00;TMeT*OZ*~RBdAkj1wgw=FjK`uW`$t4*cQJVpev%AP3&DPkyGZYt6_LF3 zfX4a-)6WX;;RW+vB;?M}2VP8~&huu_8U9(se7Fo}t=K2}6Q)Hi3Xg~`Mm>VA6Kmne zQ3Dd;N8mUq&JVF9l;xBHVb;^Z zA1}~U^>`%hZAJgP=Ro?>ufyph$DrhpCqJy{HC(py8F}E8NFy7Z*~ME2NP*Q{nwCN6 zzG#p-tOr;OZe+xN+tb)T~%RY8oCAP}oG;w_hX=UOpD(IGc!F zt4z@N6g$%VR2%(K+(H7qx{>uKIa-{cN`Jr4ra)l>6+DrIzu()_Esm!2+NU;p|MOj)T;WLT%ENZwcDGC3d`it%;(c7EKQ@|&T|4lOGuf!hvAyUQ#A7F z2XZszDx8|jG40JSQ)!J&Xe^CHD`wVEIqO_xobZ_LGuEbYTpxbUOmkHHzyR5=*+cqQ zU5EVGVv#5-6OFCQL0N|BNPE-+$y`g!JWF?FoS`NT`KpzALApp;8) zXwRYDJFZZ>+!W+uGan}0dxVne?;?k-+T#CCJ*2z)%TfA81r)YqCAy&Xge>=sq%WR^ z(aoxpXw*JFI-i}6mPPJFhHi6E$G&{H^(sbos}GVJeQk($Sz7#(SAZJk2EktKBWStg zIQmQGCdq%)B5L=)hi*^31y2MwqA71?(c?C9M49IdqcV%g>AQMp8^>+6O-q8Yll17| z@oM_|r73#6vxBCOXoz2(y^CHt7z>j3*wANNs>nN=r(~3Bpr2uTsao+NXatSun?WV2 zv<=bTZ6>IBBm?bO#e-iTWszyKmQq~}OB(qW!dat_VbS^HRCeuYI{f<-@qKiRjy2p( zGq#_BSIg|_nQaH@zM)VmVfC9N#wCll@v}e~rzN{<9u9_Q1A8Qv;^Mz-~KiC2DAJ0M_zul4f*9;WfxDa(6 z)Tg3FbLpKNYgD*ig&HqZMyKDFqA@-roD(!#%Fc=MT<7 zB{J2>t#uMjzqJ5vX`hNTU#r38=|{=j1ykvoO%(QapGG=GX;kR3UbN^uiU0-IzFkG;Z(Jj)47rb#TXd;uA;&O|2uHU{F7sa(y@v~09jWB|Wa_UR zj6MZzqNlgqg9oo^poeogr21ho`t79zH#F!AETZku*5?Ljtz!kf8o+sq7q_r?H~%Cv zVn5TXWQhTaHHpxE)%q|2M*7X@CX0ZbpX zobJWsX}l~{h6gy`TVa&P!4n)PuES=K)h#hz_~ zZbvQ)S8pDV9Dcgcv)-mOdx0%+?#-uB&>h7uX{MIVE9i6AB=|E(j*d&Jqsonk`O&X) z$=e-cE{9B76tpZTQ+@)xjNCo}awt{A>YtoXoBbo)cHA^27m|Trsf7}s9J!Y{dbEzotS{8HLv9pfgbAW2&GaB3yAt$Q(BeV zjAAE-q3Caq=rfKRd#$mOej7Q=Pu?g=3#S8Eq-civb(~S$l|r&`{!)7WMKV3w(9b{O z-$!L%rIHmg1@vuhGHvjvL~%8B#0{&WJ^63oS^S77$XN*-%%75Rt|oBy`yFUEKb+oj zh^Et)I3i=~9Yoe~J>7Lk0^M(%Ol8a4$hD_-^r-LvlCJL}ihJ#m3Ezv%X`PNNN)xHy zj~)E_MM-G(oPK^}RT*`cNu;;y&(pV-l~7T-mSpKJK^D{Zi`=pbN%YLP+dG^+i`0!3C_qDNGnkd^dX zv^qP5=D1Zu<>_AZV<~}(1qtY=*#h`uZy{>+QxhMUGnXpwYec$h4UpWUW|W-1j_TAb zpxJd>(4x1m;Em!GI``Xhc;U7j+3s&bWe%K21+uT;0=;;CidYRUi`V14#_vdjvtNxuAEy+@&jYH3;Jcd&VGv{yBQr z=PUH9Sxs}#J`i>2FM&U5w$gx0$I&IX`Fs|HptpiT+8^VOq&{6mb-DUTQO{JoY_O0< z4qk(%Wh;=45Ov~rCboqPHs z?6wO=R7QlyoD>gEZ0dn8a?;y%K38-$zf-ALJw*)X$%kDf{z5&LI5 z;DxzANzvJI3JW1f-*+k0ta)&v!U8cq{5%w-}wSNa9C-e+b3r55dtR)$~oM zE;?A`NLh}hT;jBzN^xxXXK%fcOv*mKv)o~ls*p-szs1lcQ+Lp$&+cf>;)_r%D;91T zzZN}nP(h!1pHYX_i^%t3EB#Ay>B76OX+PndC1rLrzEu-dpHf4*TJK@@p`-Mcmp5ed z%>{0HGpW1YDO%s#22ak4p$`;SpqHD3kQ|ReXJ6{kzNR%Oc#kH3lT9MEj;VkehN@!K zXP0R+$J`G7v4<%ivZT956$Rv;4V{-%h>RU(($1$A;s`HFhfivu>A5HAoEIhN+w5b= z^JPCuy53B+&rYEmKe?fRm;$7+^DgRnszXZ^G|||VqhvW(Q?JT7gB;Xzh*tSDbZ^xS zBvGs(KB@0S&7=O3w^hispr}$~M8=izCtf>BEUfRqi>H;) z-=$9Uw5pCE$8RaE`>iHWznMpN_7)@OXb*Uon^~g`r|6r0FZAw1JbLy=Qrx)wo+xv= z9&}x{mzEd?&>gx3u)g&=e|YXW8zCx~Mdi9>(Kl_nn zEKH&27TiJh)Dtb@Jm*tR7Ngc24Egp-bRhRFS~N+GvYJ+OZ&o8hT(4mJ_HwlU$Pj}Pq#euhrQ^e?<3UXTts+{0Z8UT9!>UpgpC?7nxJM&lG+~7i^k!!?{y5E z_U|ek&|ZuZOZv$RYb#Xubr@b*J`0tMTTc?!slxgc9b_MoN0&`brsuuRqrcq$TAk)m zR5|GH{1kNWFx_BksA)_vKk(3gDS5Og{=BbfI zLDA^Sn<40Eaf$IcmyS~Fq-j=F6}p`2iE5;FQb*6Pl-V{3-fzB5Ci?F{A&wJ~+i)S; z@Nv7ycN*b8OqwowZDfrcycRIIMRRF>_AxSJav{AbvyTp4=eXE!u0!vpBQ)^ec)`6TZ(!H{YGgQ2ORODdqk+7o$m4}9-TOva ztfnx`J(~(d!?(u?#*Saj%)gM2&fe^xHq%#8f2GrKntBw<+u21A9c)BDU9*wvUe1N_ zd@J1Zp8(khUx96!OKI5la(MfqG;$vzjlpC$&;Gw!G9ngLl@w&5niUu#7Eda3lbK?Myi z(Gct$uom>57(I;;uQKJP6h>JYSN(h zVq|SH1HF4@Nfpf=!dgWG!E`zR2X3w4_`Ty`VSNGh+L291zp|CiwyA?##g_Ct z=MYN{ccMp1RFIhHp`8=+kYdzI?l-C@9{>C@3Tu@|-Q)~yQ2 zDirSvAl}C$#G_VIXmLb6td$*P!95=ov2i9{Hyfb36}MpI#K*Aw-7Rv#(}tdHY^51- z^_2Je0vrsgqML$SP>0nmlyI&Csi|l3JzG=A(A8{MHdsv(x9+9ZDa+x}3HEf?-ZE6$ zTZNJvil}_yTmn9EbJ*AVsMt9l-Cx)N4_)6+_UxR`=8di-RdL9Jy`E?l(VyU(EXgyiT0pX~u~0vFnRi zDc(wEue3Hkc5*TMsf1%itaf4pLm%^Y-~9tV8BgL#)U<*RPsXxGw@a~0!{-3AsS?af zNkw+;9cR2~c@!)` zU#`gk2OH}frfj|nriCLsy0}KTBIiA5t=48Mj{5PQs%J5NL8G`NVg)l_PnT^N^>pzK zhy>y4x4>|pJN|IMiM5gG09B6}=B~fwrGa3ix+Ib1OLs! z$>-JB(=tnhD^LFdzYHRn$jswlP}K!1eD%ahE4@Ko1&hC$&tvx%_W@_sYoMsV4ZGU* zG?YC_1JOe!j9n*W6Wj)Xu4tz4^hhd5o^C2!)G60+HEb=Od4gjUrKDg>J4NPVT|G9^ zxy!RUzZ?91tJr6z609K&s*1l+Bbo4 z`1~C9$te$D@X3*RV)X-`C^mIDmG%R~oHAnT_gJy(_SJyL-;J3a9)MjFq{OCO z8^aVQ0(NP(EYopGijm?n`wnY@f%LQ=V2->rD`z{EUD~U_{9O2%7j7C0&W1bz*Yhmc zlxW0g58TAj(%!t%lx!fc=)?r_`mle11~WOsh&BBB2l&X1V|O_;2o-+IvpYW);NByA zc5VS+6goM7s{a69wdp6%{FXjb|Emcv9dXC!?>6BnUdqg=u=_ZAqB}b#OqLD)cn5&$ z8{k!@D;saG#O~!7Nc%ND0eyXS#v`f}^felQ-;bxTBz`6{SgH#qJXp!@tkz^--E?K- z-&X@>oDn6eX`T=YjFUyPYsuOxiF9f^VzJfJvi+PE^ZP?{29a)n;8?4~1!)zU& zg{@z$#E~O@%w)U>xOblgs_W-6?qBYMI^R|>CgBlQlmClvRL)|ODkm_z%-z_cQ72Y0 z*oJ*_(uUXA{1mI0`mp5%13ZJSLeMMvgWvs-WFMHff!xxNhC6A_^+64L@cg+8*_-ma zalor@4VvfU@ymB!tlNfFVA2!_Zo%I`$J3f!_3Hv??)-xjHRrRJJT?jAA3xxA#rv{) zns;!-?khMbyNy>n?K{|7Iu{>vJtMT=@d_W}7>SFcSrDG8!rqg3ie(h0G9zDd@V(1w z>06&PpLf)rg7_RQDM zSn%u!E-Y1Hx@F>7yX_O$N26}+ykb7Kh~AGcFFMXUUH2GYdY^(lt2W|`))MT3Ps&VI z;#$0G{8e0V`!_J{Z^fZ6v>U#iv}5|Fc`$U|Z_MzGz?ciUSkbJL=efgzT~)5etgq_E znoKpXsj&d(O)ILe1$vBxn-zFo*$-5khe7*hc~+NqnioAhi#a$r!(Da+xq} z$oOn|0TRgr;51^Ev3K=i;c14uLkgNg9E##cMQ;X zxdQ^%f59icJ=ki8!s#P+-FUfJ+T})x1nXen4B{U8;5ofU%u}7o>{M$*A^lGvGbhiRJ#PuzQd#TPCm1 z8eH;YjvjFWm8>l;dC>>F>pa=%`yumq3S^^ukK?ct$AF)y4=+3>ocWS5yW#uAJGjI5 z84g-*$tr%e;WU(*K;I!5Z1K&+VJF{Vvkrge+>gmj@xyo^;u{9izs# zJ^)>NCkT)Klwgj_&tyzooY;8R%q@vp~$E*n&XfwG?yC>dDCt244^`*Td$#T;ky{tZ=Tyn{$+71ab*OO}F3 zKmUN{+yvnRE@NO+*9{`>>a&i2rr>#_l>lAHz^|)o!2L=k776cz$thaQD>FrAkzok1 zmHY>?tLNhnc~(rf4aLcijM>8rtU-TZD_&pTTt9E4G?Vr0AGR%#WwKXK0>SYIu+>mM z_MMuC3nu39*5p6n3A+j~nBP9#i4hAoYQ3&6OK9YsXrw@Hmw_-fZ3O3u$K$;k5n$z7 z8QgZmTKK2#A?{NA0?zfwFx?CO19JWr?2OtVp!Q`e2n%swI~TIx>h)K`B{t`9x1B!o zY{PF*epQ_T8alv)0B|u|gGrV64yM{!g4wIg*!5F8owY)hv9-2?FmRo|Fz>K_Lua@- z^XaSzd|JS;9k)urs__o2(e!J0mDf6u*KW;NhB>g4-k-%Gy;4j;%@wfjKSySaqBmaA zrO9lTDr`uauFqW0(qzkocY(Tt7h}18E-(G|5>Vm(0Q{FJ$9AT4gFE?Cf$Ea!KH5nY0yaA+NC}KwwcgFSRIs8^V9uEe%G7*1#+0M@~xXeHq zw-`)df;6Pr4bjKI`-0ibX0;4lxwaInT&yD;UpETk6W8HF-QBpS-icZDuN3=C5aGB5 ze5{VTn=G!wQpm1AJZ;N^B2>=~z6UWsiG_I@D%Ijj6R7O1yS<=k$NoL+*J)7D}A zpPz7u-ZM-bwV8*n{lM9usUT&8BDj=d&as?L@T&jDF{^U=z}xh3%+TVC;EKpWsCcMI zXdJG`%8&oqa6d(k52VB)NGGXPg9Dio7)S%+|^*zzlPwYj`^U|{Rm#B(GFVvo56IQ z-T_V}jRPvnO2B_1JoeTFT@aFnScy_Gp7u@^-%M+CSvm6#IOBN?uU~XcxJS!^^|W_n zFD5((pTEm9x96vT33KjZV)dj!rg9I^STTZq_*=k{U%zlh%M_;1@;x4y(9c`ho-efV zkz!B8LA9G}KFfj#}r_RH3jU5@EN?mqH$sCNV-G?7L-Qnfp~_cZXI`_fy)WvJng>Eo3vODchm81s6W% zGjG^?_;BMVpmwUz`ed9EYE4gnItn}u8Ow@_v3H8JnRrJ&u~2i zF!PBFD~CtHh**!gs8@pfPFw>|LXLBctY5&#cD%6P{uS?<2V0QZ&Q_@>B)nR?llecm6!m}m^)EfE+T3HZgkD;VX4?0AkhU;6G6DgPaK zPe#n3^jloiU5(!u#%9r&k(F}rKv7XEbLBj_Vzz=VaV_|mkoOpr_)FFffZ zNJwh})|Yazh4(lRv5sNGb5DY}Kd-^ns$}8ns8XI~wG(^Ddm0XP9^`GbOT(xd@3O-ee(JFciyCp*=WwR) zrX)Lebqv#VPmR57bPgn%9maRtIzh`^NoJ{{51U!3$0+aD!r;(F7j`0#anzlT%@2RX zA74#ov-VD84qh>2_RVY%N^tjyNxm7YV=)P5e^O)fV~T}khUI|w#hU>Sxby4A9Gp^p zANb!1#-9|@K>pTyxa7}ueErHlF#1{<3}oFE+B}(nt!n<^nB;h%vhyb%I=>OeF4JHn zm7imis@Fije<_=EPL6qVQG(sizG$c|ywAHBBgYtMV{mHBWgPj*8&6st!qj|J;=%e% zygac8?CALgzQy(dA2tDWHrMjJewnawDQe8O%os2+PKRmNRA6@iE2eJcLiXx}eY`~V z1d!Vrzyw~AV$x6gu~H|tV|D%nw(~szRxMmzYdDU59?E0ZzE8*DdDECDhwFGh$4N77 zahRt#{0Z#NbYS1g&SFN=e1W&$kZ}G1S$5sxLwIdo3z!nOl()ub6TZ>y&#sNtckWlW zU{_pRj-77Tg2(Ak!M$Im?7XF0z;DGUHdIT6-JF%gbf|JU=Oz`VeM1jeDy0mPC)u$% zg%YfXs7N@NxRJSlxU<9Qg^XTe3efA?2^KYUxOl8)vBlj?;b+4VtUKfj<{cZuSS?OF|3;U+d#08ohJM6HSx2R+Ue=T%ovKy9xXQ5xP_4fndnw=3lz|&+ZjvmH+;nh5w zd7lBfF^=_I8_cA?L#*Yp5YRtl#UA^k&IB}>v8PU1*j0jJfgNA425^Qva2D>DFBR+CY z3t#?>aDJ#9>nuMkeAIszyx6}APkm$nh8o*=SxsReO4gMz^O9k=oV|ga4$WfUM9su+ zFPY(%3vyVwqyprQJMduw?^Y_Q`l$I714Glt5 z8+Fg;o^w?w$&75tNH&=vrKzEzrHrpqnnn`c^SS4JwTLn@6Q!~;vWv{${rz?SyN`Q6 zkMsF_&inN`ujh-F%1LQxoe!6KxC{mMyho4J>QPSdenEWGU%K*j54~%x%e8;)M#7uk z{PdNkT)bsJYWynTpQRZIR$o%)uc!9V*z6(b@oX1?o2nEU&0fwAQ(nxO-Mk}X){Ws* zrXQoL1Kv_CffHBCKSj-_;;6%-x2V?sH=P-N4|PNilqX&|{T9$ic=A zwPa}^i9sIn)>$do>FO?ay%}0HV@QS|yO40(x)PAi-39#P<9XurW2 zn2C2sf1!8pT}S3Rrz%VKhX|q)d^mY!O+G@#0`^f+<+o1TE!Z(?8khgyM!_q^aKXvQ z8`LuWvN%zFAQ$+zllJ5d;sgIEaKrn0X;$_{BqqPnut^=XHPoM8w@a^5_b*4il`6bu z%~|Tb!wnr3*>e9KDW_}tJ@{n%cACDxhPoYS7M!!Ye>uAdan4ouA{z(Gr7|tJ+x^`H67WmA&%});z$1+!s+as!s&Fypp1+>l;9FY{cGK2 z*BWTY3U}^~+Yy>)?naH)?x(v8Ot}jeosgS{0k>`CFsd7piVoNjs`lfXV0_F2s>3#; zhu_y<-Ynxf?xjbl<4z-f@nR2d%Fvgn?R^J5@Ff%Nnbb;^9DfRqAJOI#%O0Xl`}d%6 zQ#K=e9|KNcA62!$|1`A{rqc^Bn|7z27fi1e3sl?h2tpE#`DbGZ-JSKCLezL}4*>YE_{Pa#)=8djS95x|x2fGT{CH+Hh+fPKqC=D01`1YH)sc zjrfq(v&1+j?7W;i41b zC&GQ4LE3x5erug^hGpRJ>UC;!m}&s90O`|s)a zi>?BnYco;Q5yZEZC!_wm^EhMgY5d@j0bKRbcT_TV2Dcz6md^Ml`<@>+=%abI^y&ix zK6j4-HyggAN9xP@t{@98E%*Xm%ggvlH)e7ULC0zBx(Ku-@df?hyObCHu;g}oZxz2X zmuanW>-mjtCfxYS)iikPb(!85#{ISNLb}HjXw|Y6w9sHG@0N81=`WkW#c8gf>X%ix zT-7J^^z;sTRpFADX`Au7-tv?mSeyFH{= zB_~`<=E$Kr!zQ5XYZzS=wSb29nQ)q~C(_#NrF4~+HJ@?f0?m7whgNRhg2peJLq8=Z z(CX=1I4^~7^n{!NZ}qB?!iQDp%jh71dhr5&WseC>zQoXkJ@x2KtpX~b0|l;W$Q(4~jV`xt?i_x_sPWvSpCkB;W%gXgDmP9! zaR)l8`x~9BHbFkiu8URlCsM1=t78IUkVDVs{Y0WC=G1wQ1^01*Io+xG0FB9?E!Hi*haQLj z6}awPOsjYP6=+nC;qLhr32b+MqLt@ssK!DOH#p)FeLmNWPadaCm6yyAjNbQ~hFQAs z0om8+np}xsTFwhf%Xho>x4R(e9>UoM*3+lUUy8q7!AO4aF#i507aBVEq+s)wB?8@9 zYpK>o5nWp|ggy@`LGu36@MP{T;NIO{!{^e`T& zT#TmeQT~X$ADayo0Fo~&c*6g{{1JpGVrjZ}#u zwdh;0Cv*#mS^j(=- zl4Zzm@X+Fn7a4K4k6fd-7fS{HY*>}u$w+~MvlT6um*?|zJ;gC8p~(NsP`-purh%ts zaK(nlP~GWZ`gm^`3Y9-32)5SYvsVc?>(`ZNRvC|88;R(Vc*KPTb)&tv3(*?sQ?c*) z7*t{P7d?#A;Fd0qMnC!rY3bJ!Xx;LEXu#qQ@w6dJ(9h)>sAAa|#B^oFrZGn9G~dQs>9W_eG@ONW@Lo%DOK|aaIJXxjdTRwdxO&#;8&5?oeJg zua|yPC=@8=4dqPQ<++jEPLw`zWz|~wk=$hWm*RlkFU5CQ1Ts<@NS&G`i0(a#OqLBn zHGMv4>74>Ps6YqJYSTe62Z*@ieGB?F>p1;&_MP}~Y(0`sTTdg>5m&rcixYG) z*B;IQt$v}#ul~Fa!4*5+GG#q&z7xo0O5y~gV$8)hWqW1o>k^94(MLBo81Vy+Tyx!9 zx&fV^e2VTH@fZb9&lfk8W3)d^gMa@dmj2avF8FYG1@AQNx~qy_9kpz*rwyCO^6hm+ z^tM$An!IV6;Leq4f{WuPp_X-nxtMu_&@LI@uI$?)>SrvQnP2m0e(59hcZP*{h*L9K zI8#8kEMJ25tPVjl{uyw$@|AgwDWA~sIfhlSPnoA*mZ4zVBypg^w(2m z)c;wY`+1v3u?x@B_@EI2*Mm#>5#JQ}-RbK%yZTJD@B3V?+J7yYQftd?8xu`ireMC& z&8O<@k<)Z$;P}g*+|>Dm78QYx?Gq|$w&3DH}(>dF|hv<<{ zBUL;0Ry^~)KK-;!gEy}8;PQ031ioKKa@$9SQJ=R$Zc%Fv?XwwYX1CCZe@(bASJcr>!z6lFVLsgvGm6vH`7StOX~EAbROjveSEBEeUC=V8Fw{ua z(2VnY0FsV zDf8UXiEUck6uT>^@uU$yq5q)Z>3o0QZp%p;TC2-HT$|D5fi3+n1ocZ04# zZQcO>)0vMns$>SQeu+`z8IJtTs8QUaLlW9l=gMta@QKQG3&k^!59eN4nsBb0vQh85 z8#J?KE((%&M+0*ex#^n~`7{ODx^|Xv+BV9#gdNNH82t~ZJ>Q7`oOjiw>DeScd#5?Q zkg}ZrIo1)moh=e~^;@9@;|Fj?YU8+d^J7tOzbEJX@-q5QRg<^ZtDeBt+MF$AAO-%}iU!Qta5 zeb`YHm_Cf7>4f`u&X$WbIVE1_Rz(LH+VL;kbojL!oCM{Mh3IB!G}T{h%&+e^qmMS# zqg!^qT&|lb-Z3UGgEhZbuanNc z_g-*m{~&JSAARxBr0Zfm(O^!v&4`P)z96uCw~-sBR)UW9pQI=A2lKb4cBAl_RcO@> zN8Z7pL>#mshGK&mdbRo=E#EtuQ<|>L$H;hf-$IYko0I4B_Zyzl)`^sAJMd^t zgco-;M~Um!XDGC8G@lUa%-NU>=GNLL3G@q})6Jh|i))K4`7ukQ1V`#xt0enY@?$Hc zF1dRcs_n7j3e5Th6?W=ebbJjexcUNpIQEB@{d1rtNm{7aE*IUNYQ^oG{aCOyZz_Ff zQ%C1m?w}1jE6~)OL45Gt7wF~mIsA^c0sL!STe@WP2Q=Z13U}&_7C(Pwsd(;lTlBN^ zy?D~1UQ}oy)6Q?bMmJN0bkn&xTwA~=v~-*e_x!0E|EV)f@a^*?agDw_R~2xIuG=t~ zYd)PJNYhkuwaPI;9tyqm$;DurmGFoP%#HaQ-Q9F(-w*NgmC-2b(N#3@o`8GeB8Og_ zf9pDLX(gSdslXpze4TO|66i(6f2ix-49&Zqays9MvQCph%eToChN5dYi3o@yqm za1Z4r^nREx5(t-5$ATQ1usD!TYpy}F1`g&F^q1|%$qyZ`|m9@;GhaOJ5?kY<7>hn{p`lOp2(t4 zHt(Z#ZtX~r@CA)1ROQN@K3p#BMtpFfhuH7eU79RTq5l@Fr$LGjXa+x&)4zR(ehZXG zhk^oW{{f0ZT6Fn0`$uw0k?X~GWel~Hky+?lpeJwh!yXwtv*b6O9zqXi774H@1$k#B z2zvY5(UTX>D9xw>jUTi@OnbFB)-aOKKT=O!&!4DDp1M{%H0(ZNvhdX-Q3%pdn#VWp z^F$MFtMgNjX`=bpL};z|JK8iEB1Nve!p2FQztJvq#5M`_>ODbQ&o8GR_fJ9=;%cgNqz}z2YN0J22hckG|LDdL zPrhc~Y|d47)*!3jUm&<+!D(yG;H%DmLT|Sl(y0eMIIEL#-0wSvywh8IJ|tR;%U7Gu zAJr^(bsTBPH2R}}Q_vzkO)Az&L2vGK(i!KhxT78Y zf{B`KXh4-C&mS1bd0Cf83mK{KQS6sJFr$o!|LXyjCq<5Pn*~XQWP$o!^QQ9KY3#+Jk4( z{9kVLP4ZIu4pG zdxO1s>g!I}VV_GirE(aghb(!Mw*G2i{7%uB# z8Cq200DLKfJ*5Zchf!wFTs_z9GEY zQOMR*2*`%9SBSIxI@US%1xBVD;j$)%<)zwWz^9w6GEFSmX)u~}6l9a#mrQsh26s#rfa9MYVlAh|jtYOX1Ht8NNn$MV7x=<5YdPFF zG>2uSCXyNF|05UUma?Tedc^QdES7{G74ADWPL|udD@khh7U~aL3m=9v(6I7?L$#h{ z&6XtMy?mkg?fBPZ&W=$c^NsG^B^Bmg391YFK{i2B!C_h}E3*qKnR71qJmH zF!jhV_$`NF#M@Uebzlc(y@cC+VS{L_Koyj?Eo9$y-jMeyk>t#VOmN<4h|k>)!byvM zg2%;RlH#@=EVQ-ABaeKd)Mz4nCRCHfYGR3rEF1jxlqaO942P3(xnyxnH+f{02uV!> zY2Uw8vNF3)cy!`qm=onHI$@B;rYt&0&cCn0?vBGrS?^BCzt6GY-!d0JuKWs153R*5 zq3SSo>t~7XsoRjZE1kKX_y$d$32^V0ACL+A*kG}tXs&21#0~$$ln+eqNR2zi`zb;^Qz!2Z?IWBCjA0idp?!;-^k4iE$=EB0~fvmp6m$h3yA#1ItkPi+f zqLUAX;D$~ou>bL#ZJZp=TV!fT>ndD`R`d*R+5`{cQM5B5@oVNxB3KD=Sl6WDt!6{2nsR=sm6k$-WVe8|%xm+e%E*zz^*thW(f zI*}kLeHjF;50B$N5reSlw9}IFc~4mM{wy-U$`3Rne}b;fUySn~vaYr~s0ca*FHQEPKJp)&yI%t<{qwO`eEKpblVrpdO_SI2V5mi|i6 zSiKtV6;B{)ucLACTp#v0V>r<;{sR`07T_$lcR}Z`0;)K zIXLnvx$yQS*4_{W`rqU5JGnz-hbspmf0IbVvtq0jSU{p1TG{99EUuw80&m)t2bG~W z$zRD^8Cx@%q+L;cdKl_&`a=mHT9!sXGbNZ)Ue%n2_%3 z3^4v@1;{i3O5SV+n>YKQt#TN?I{F4otDH;fmVT1VoO%nI)iuE=IT^Yx&&6`Pl;QHx zDnMU@iFeCXGEZbC@sos;ci%se3a^iNVDSOm9L9*#+Z%lC>`BC6wX*0_=Mbs&r(WWv z`-;VO|7Eo=s$q&S7ig>nXy^_krT=}whkJjsj8VV9x+7mQ8U2HuYvjo4fMQl+A4HVr znZd`9EO=GdB)P7%1b69-#n#Y;E%Sb`mRo*=xBZWK9g84}FQUltIda6nX$@>__8>ne z=L*&jZH5$uVqz$N5WczFiRKO;1}6W4NpR3UVb#lA)_GbT+V&@bPw^ZGW09aFHySeK zf&U|S0cuV}f#g*!E}fUbrmayCy_^1yjk{dLO5Uv}gSlnU5FZAEC5lp)a}_N5U<-Dr zb7o7<#*i%4baL0Z9CAXHq#^HlqKC!83jvo1S+Ntscsb&lah7a9bPtbKl;Gc+2En9L z^9b@WAv2aPCo`YKf!nk=y#DP6{3twunYsC3ja99%^0NoovW4O0=ZBJf*Y$97#10Z@ zdWc*Ys0D)sM?g1iHtEnlh`p01;LrOCLD}y-iI~>{gFm#hG8s?in~e1obfXqOytobT zoqCzPm{*B$-90e;JRTJMtl;mwr7*^F7%aRY%c!@j5rfNf;A)EkIddl(-Zbndx8jVj z*mSL=eHD-EYSQ50##7|}K^yY0=s1{v-^G?i6~M6Md9Y9KrSNEB4!H&63IAJJRAp3x z$B*jaTx3~-1?~%(&TJiUDfDE;If*1P-ka5oIzi$SnnY;CzOV{*Sj{duuYB`k@X_rL8Q^^EJ8F znMF3Jj*!~_Q3i8w4X`!bPi{x%K=7|*c)sg3{^7WS^gl8oMOo+Ild3WnNkie&{Z(WD zZw#u9t64K!OBN5K!kW(G(0L%0izuyvk;&7^A{kp&Xcq|6>pGz+IhQ?}{T9!AmIn=0 z*Kxt~SUiXuD=GNToD}HmlaHmc@`$V+q1t*anE!DzYj7J$o{zC6kvFHqk!uFvQ1A-wYV?=8&+~I)Q}g{YMPr?8u+8qwMa6F>Fb(j9Yl>0A4SO zBwq}*!TxMIalhm&iq*~&e&4o|b$D2lxm6Y-F-Z z8%)~zkgdj3$;mULU}stxiJPPWYVVk( zp{JMTK1rR0eYVD4G5~ml+#y(BuD? zQ)w6Rn*Y`ixfSW)?${_XzMIL0MRSrM|2|$V=m1uwtB8uTvgF6uYFO+)Or*FW0UuVc zhM;qLB8St-pl@m*R9gQ}va@L|bxcZPns<~qg~7R`?%*}1JmN5{Km{b|A;7b-w$N#k z4V(Qmf!lk42HjWDuBcdqftLci>`f2>iJFDoEWt^K>AjC*5IqGe zeJO*x50oK$P_4v!K4Ic9_IUr&Y?6*r;lQL^F5a|^9Qb7bk!wG(AqpPwDC8fkwL3(V zg7nxWPDK=0@qqpOWi7hZx|(@xErU$dCY;=pjq8dRu!RrG*_O?t$&0e7aK~C7Yv=!g zq3dqq1F?A|d{PRz9DS68I-FvCCugvgvK->WY)AI>WHNp=H-V&%h{h!geo69Wd*v#V z6tHhtCrOP_BH`Nyk-4L~V7XB^%$pd7cdN=Wck_~vdTG&j4{?1Eu%58u=^I7b@~9S zd6$B3a?P-%>?*rmRY{%%3W@rGVfe_4QWEvoPwW$@%ZC1)4kPMz;Z)6NFh699r(PK) z+=y+!(tAA{|J4ht$rv=byUqy}hs`39+oGY(r;%+RcojF?{($xFgptU72T6rhA;{0W z$!2TbVjHGgz;o9;vgnX3vwid&nKLBs#&l7|&> zion&+CA+%RMcvJ5pq+FWevcHeEq3`ZC_fl3-7O%azrJJf2lL3Iv{YeLin8=x#3-mb z^8kmARU&^=hT{T{Q!q(oB1zOg0q!Tp%AV;$;qGOG*+q9V-lLZe(&-f*?btvr{J9Cn z!rL%2J%jjM^dtJ|9_;kukHYDx>F}FhMec_b!tn1s_=eSfSZ`E~<3hgTwh4P-s^}wI zt)~wMFBamLswz_R`2f7zB+_;BJ4>5?SMrp;Wh%Z2tY^Oyq6>)`?T+z^Zx4ljb+i}$gm zE!r$){~$81$pH@gT#|TbTZtNUWLe0I4KVWaE*v?v0j}(qliqO56_x~?2hA1zMBl8R zO`LfG2b$^Q{qH(q?9#Plp}Q%0*%t$Mv`-2DYws1F>psLn^Cx1@ygH_5^$91syo4!L z4J;w$K9kISi+_ETurYT|vrVZ1AnHwmDdSGCb}uuy8=MLJ+Qb}0U zJG^(ue4q#1py*5#X+Eh&N*5hrB|4w6VN%OU8s|XK}%Yhb$}DRsysqE4j%>bvKxMyCXydz*7!k&5uAvOgB3qkfG^D>?}H-9 z0TqCkCF&$r^E>-~ISyY>_Jl|FhR}PhTrzB{7fieOk;Q)zv*)+8B-{Q3Gdr)LQtxr9 zqHnqP@p`iqc4NN@F;vYVLw7#okA(qQJzf`jJ z(#RLT1#o%PBcXKb1k(3!8XSH5h#eOj3D*&+u<+qH2x*ds=%2rsQ@9Gd;B$jS{@6h# zUAqX5&FhHy!lQV~Os*$(Revzmb&- z`>^`_(=hdCInmp@1lAo;6E!x*L+76)a_`L&$wajsM0=dFbkNZvX!@W94mYyNInIr= zKaC_+?aRoOSBb!Re_?I&P7qnwAOtMm4m(K*Dd>&D0T-8DZOmC18JQD(jTpD}=%#+Wy3Iy|2x4b;Wmb-ggQ_coaQ$R+7UQ`p8%N21yHhAfry zl+1nBMZ|f@WK?GrjB&dm*|Nc$_`Pssdu)$`=9Bd>c8wnQyh}|qy3da}ZkT877Mmgzk^)TEWZ-=V^jl6UfsEHP&kSR1Y-zAGNFuM!Ed+$@E- zJGo#tX&t#(G?;vMJwq-X4JFUTVFYV)64SSZOfSm;j+eF)QIVF60hbS(wq~3?6+)DL1Y~8p5%<1}BtLrULZ_e|-mh&nXhI zdod21u^#G+Y#{4Y6Wbo11_Pr@@jlO?qIDWd@FPXW3oaZ2D@92#XoV$N{cI#G9C^rP`GarhYcEFc&kuv7papHJ*s!+=@2n2WX@UpiDSbqW% z1{mmr>y)3QV{t0<8s8P>zB9&s#{==~HZ9V&>8a%C;cDzVW(1VV8Ie4}bFwPfoLDx? z7<3IXj^!Q$rc|LTB=fsT=gN@~x4IomzRqXz)du`8zaH43=KzV)R(NJpgrBXTY{F;( z_#V?C0VtZxYMFx{SvU%BUwMZUXbRXSHiP!lBH?8n5t;hT3eu)4OMabx$^3L>45Q}5 z#QDoin6qRSc$g{@kM^N#oAp4pC#sxmbeu+VA8N5d0h^dYnk$tYM?0d`a19xM?!<@F=n%(23lnN@P<`> zq+-ee=(#mex>>0muUj_(PVH=GDNl{qC$Gy8xHA=8DmRdXuy6=FZ!bC`=R!KpQ=;jY z!Xj-)!0o;fL|LN(_xMZ6imVEdTYr%}y>|+F1I^g4_!`#Qz%jLkZFpDURY|hVF|to% z75VwAjhPk9BD+I_@D;KG@4TN5_m%~b$Dtq?Pw<0SyFbH=N-U&q0nebR zaslKjIYU6u1QtNfz`CZ)$yy2Z?5?UK^72=S(~em7^h%7R>RT25xu}DfW`D%f&(DPGS3kh|t}$RX zHXfRGWwU7`_rtQ@+mfaC4WvJ`cfzboEy4@tlR)i)8{>D>!4%?}h--+YNzEIfz5?wJp( z-aG~QxovE{+!~nuR8iU*8woN0DdN`(k&rT68D^a^gfK3hJS!eWhP0ZH)eR-scdv}2 za~fcAVGmw4uB!ARc*wIlo>(O7_;VLv`k& zk}@&wu`ndo9+TNyc`<(J@sZ5mV*~ZOWgQ=hv-w%|jpV`n1~92ykFWZt;=D{9w#ob_ z8yBTdnkcZkL_Z;77Bt^H~$V(|(Ht-S&YdG#hqpdJGdc`|!K&50GB2 zze9+m4%}7^5*_8vfn3@E^2PQVE35n`bUSTKP=El3TrYwmpDASAfn!2yG#N*vX^X~v zGm$EuItgpe>%m*OaFA^5V|ArZA;)llD0gNB$;~qt{r=e`<38^vcVB*D&I6L+aQ#_) zW`8_Z%T5yZHRqC0_8F?ZPYG8|8VO^rZNu@qmJpL~#pKH~1rj!`P3FP)LaL0d&d<#n8j>WOObh$aJZ5TgC`mS02TizWyak#}yKvWsBLby{=$qHkA)3sTKye<^II3rTv!6OqA%>% znZYFC+GT9;-V^kx3mJ9hC39YHDa}l~0>LN8LPn1|(E{Z}%3`!lxKU=O7Bipi+dJT{`lnLG}8Dr{VpEkv!mVbUHOypeYyOB)+n(PH_q6g^&(BC=yn2vmca1-PEOG?Sa*Yt)Xmf|ky0OCC zi87q8jA5n~-9Wx`Mp(CAht0I`hKc)vWSV^}uK1BcGG)Bc1-sfLb4J!PtL`9p_sf`g zm{}9Y+GQ+xau!*OE7%RKu|#~>h`3juhvL$EaBQjY{rQYO&nhlmS3nTjyqlra=E$L5d#T#Dgv8i50T=Q@ppfL_O{$C9nwLYDMl}^Od z{rs_f{b^QrHyej(UnGt#^{~977-tQ|5TKe1o8>n$<^G$@S(=2a&Gg}HYC7w(9!|3N zoFjo2;ZR?=i+IcbAg&kx3b(tz~MNbD(mgQ8%8YW7~a%T@2m-#bUVcSSP_MMYF z3M?0P)r^!3k2r^Guhp_Oa^pzoBs*}(+r;LyzmT{!$dPwmiXxk&GVC=pmpPlNKm<9; zd(_H-q|TEB7VH!{#A}20l3lRmzvb||DTsu4PR3?q9zo!(>qO_h2Ye1XP10}K2zljC zLQmxYCNVxOxt3=H5xd2#;fyZGPq_rg_sC}DR~N~;+v8xB4JDpMp^PV&VAdNY?DEZ* zy`P%}n`i#Q{WDL>__ulxbbSGA8##cq>RLknkwl^@-3R4Mw}Ngwh5gfF!T9k^W~^!f zR~JTurz{&5<9Y{npN_y`xxrAa8;Ay?J7j9Xe)9FQ9Gj<;z*fJ`g>x_SK&y8WJkI(= zO4QH8Zogj;rFWmvS5RPu;rIOiBvKZ={#%2(vAwrn^z1M zsFjdIb-!@8=`*2!lol?GpNRJ^Z-%U87r@@dp5#21GPgl1afl$6%t+dXV^2&WMaxz2 z{+19Xw4ESI=vS21%os?%K9zM_ME)bYe_w)W(@RNZrwQqWRN>S0VPrv_9~*Kg9(EN* zNpiIo!sa4f7%%G>S`5xaca5`fNv{AqPx5BR8d~tQIA5~UYy;K~v?UvBdUZfG=n$AO=m~7UxShpbh$OuV)l9r= zHU!MPN*YNd9C>6!_}AkhTQ*HQ6?tNtQN5Tyo<{Q2m0@TkB}Q5Y$sxVj5c#=)uhlb_ zu1P5e1Fb{QTS>^tcnh*=`bMF4>?@XOvl&{GLqJ|4^NBAek)e4i5V~?OiTt21Ej$}1 zw14}ad2Q}t9dB2`seLLCc(a6LF3yH?4X?n(pb9*U*FxV+U1FGODLR_Ii(E|`3LSFZ z#Lq{<0%jayxBj{i7J3|tuSby~@*|`sUy{ke39I03(kHy=c)X0SnvX@iH~Hj$0rD#E zz^m|Bf{O!KVckvM9<3msU;Sh0%@JVr?lA02QipuGb`VgCt&0k)vpQ?d0v`@!^@EJm~}8 zxkC&rBFVPNb9Jny%x7l>YF z$(4(RM)G;yafT|xXbg;$YuFF8%$D0JoI=hCB{*M@+Oq6wd zNlEVxJ?Wbs5!q=HK@PS}CEn6_vOVDl+1ac`P9%9UyJ6GGp}|9;ruqRZH!>#QEx)qE zFS^OsN^|l#WE45mC8Ok)JQZ5>&w-4W*~}?#0E;{O0{-0niZv!EvJ$)VI7lyz4c3ez z`HyeoTQWU=e#l^0`^%fu%Dl)W#>IF<)he>bZvpvY?#L!)j)IJJC*W?>RI=jYP||ov z!pu4+;r})cA?IGrAgA9=2fxEpNrLh!w%yE2(#P)O6%VUqp7B4}@k|@!ci)yMj+H03 zW*Crb0dFKrvaS;w+VJQE02{)JA zWa0s~vTi61cn~oOvfGp-yN4ID-WjgM)^-EjX_MfXQ3v2t)o^oV&cfrSG_`NlPuYPc08>l!Mf%oC`e88eVZI}n666xY_?{<`diq% z{;f>aGLEb&ii9?C6PZ0EN%Hg7Rp{TABso=>&p%G-hc=^a(B~n0evFd!GD`EoSN;mq zasA4wWo!It_BUMJKabq1%D`9bRq?+ki=@p7w{Yv86d2&N0&V2g*=|!=-{ZqYkTLWG zpovi+|8*YOjw2x1a1M)jvXuNDnMH0lrNcS5PptBnJan}jl|LPU|Z`L(>_y$|$Rjn37|#D`1$^+TaX~ z8vh$YCuftGJ!P<0`y<|BvJx&F+zP3#?R>h7T^#$nkNBsVk(kMiaPVg_cE271>VBDE zdi6J?;%HD7{D1FhBI^(ofr7IqXgmIg`=ueWP zwgbHV-eC*(auN?OCC6$W2pg9~f~N2z%qh4GMIBSw{P$BJesU41xZn=590}V-7n66& zE~L=uH2L(?MApM2CmLOHk{}a%nJ;r6d)3sk8=sS4aoP#&x&94K8#fQXTqsW>%?OOH zk0$T7E`;h|;ox^~Gf{t|MpnIc2Fn}gn1Nk6t3SN~#)SW5e`dvy!#Vdz&Xea*y4{a> zTP$W~dTJ1*_#7$+?*tPnB}Zt zq)Gcbo49`uS@y~g?u=UvYg-S&=Yvfosp|(e6BzqgVO%l&kla6#*pe5gg9lw!A z)+_x-R%yn8VSF(YEi=ITzbBFLt!KnD)(TMTMt6Rl{7zIM%QYeQ#r%007jB79GOe4K zDfs?pINy46JQw@QS6nb%g&((S6yH4WBwZNumX=J2;-BS>=e@6fL{q#r^P7!!@CI?= zbmx$-sGv>6PhI*3-O01yeKcYOe4P`2Y@sTuzhX_T-h4wjAMR4sn4@&2)Sm!VFwOd>KZDlaOqhc|iqg*X$-8O~Kytsg?T4%*+mj#Q5Z6C~y zL9Tp)S}bj=GUWBA@1mMt`%&)mdAw^`pdiBep`b{(mEY()i~qRPg;yMp_^ygo{1BDJ z{C_>g)I%$fm-+l$vxSH|*E57Wytx_$@));KzQ*=B*E(QNM^c@Xw_M zpJlq{S`V(a2%+|RZEn*Q)2am33EZcb0h&r>ywayifE^y9e)6y^p^!ez<_y1fbHV z!}yGL!iCxea7O>41&yy&xFZ9q1+^J9V#S@?sc+vg@d(o%w8`NveYo`f_0b03zIG1S>E9)><<#w#MK{v+nNGEszmuEYITe94p zdo<6C6DZ3GtTyU!wz`Sb!C#-BDSttb96XvkKi!s2te557Qs?mr+ZXUd=MwIyUKIW3 zEYGzs0;E$okSnQsi)Q>9%?TZZbZ{4Qso=G^o|T&X%S)}uL2`)t)PAm7ukstE#7(3n zeZR#zw`Fx=Xh_d?j9EUrkY{7vC;C zNy{w51xF(NtL}a>uiuK`A6CugRVOL)pT8ZY>i7Of(VfRr_4IK7x0QX%UP1^ZWC?fXp1IjlvZReD zr6jGA_9C(svK5s|v?5Yuxij~SB|@pRNgKsa8>Nz@QqOt*@VA$jd(X^#=X~Cu8Sq)@ zQK5?5KTuWS$MZ0VgHPUC!>=a;K1tM7sE!M;pf)}X|jvN8qR?F7W;i7!!d zI0F6)*-wI37|TPi3LMzzDCNaR@wZ zHG>1QJ9!yx-8|FB=FopKXa9`1f)9-xq5Rrz5N~TovIcF*agAhOK7QjuHubx{QoDIS}`tBOtldn;gB!B}bHQ^B%b$0G-d< z9X?N)569MRh6euT&|*s~-e_BbPla2PvZCqOVq^<>bX@`F+ zHCwstM-E`4=kR3r?uHxDYIwUk3-Hb;!SbkiB!VgswfE9Qz?pEDr#0YR`zC?`t+~X1 zXCPcM%K26)Ysm56!SK0qD*4JRCnvgs$(<>N`23jyoMBUdn~x|$v!}bjjPK*2{0}v# zpf?Q;)h`96#Tfi7Tu020ZY7h?zQFf$4+AgLsU+XGlGm&ufH^zGiE)?)x$;L@Xs4?{ z4$4}QiVPPrZ;3hZNqK@Ve}klbeh^mtahrEyo(w6y)(V7MmqM}frSO&88(xjRfOFqu zU~x2nM}1|<+qF?(%Zhk%E5(cqhxLMs3q6Qx=WqNW=R8p5vaP#2XOIiJjd<$!M6fZ| zkl6ndV9Vaku#)r3;@26%y|0C{imZ=TXPc_yLesPO#MIg3Q{8v0ko6ZIn`Vo*?~{bh z*}HMZ+&&`iK#fueRFjY;?v{GO zpPOe8#zC7*`<#p4>rNrzCQiUPj1R#zC4BPSa%eNfj@(UkC2D_bfzyO~ocgz$H~;f( zUUP;Gte5ozXc|dwE$; zO0gpy13cN0njO7s@#^WX@$Nb6YZMFwr1_KzS@(Q1xz=8a?aK-|Z(=I8Q;>#K*aw`} zr{XSo4f5YdTkKG?NQmTTkl!B<k0z;8wb1|#Np zao0x2o4wjKZBchQLop9XM>c}r2d-nMD;Ds1?=0Yc##$JK7l8-?e-bfQ^@Fg zX}BnF5_vmMg;?G7#&wC)NOO<}`L}R7VYE22x$Oz|2%bmMesQ6h;1IZ6L5=*b7bj9V z2T0S{BC^OOkh4%u2q%3r1^a`Xh+ujg=uSOIbX|9mVDo5@@HB>O@9~CP2jcO)OTW&#<&+Y$Aoo*2GI#&94Ms zP*H&P&zym~`z4^eMS-~0SK}5tF*uheK{9tH0-jtG;CV?y>>&^BmMw%2LZ`y!kJd1W zGbeoS#=wajLS@Nihu8}{c-y&kU;$?m4gdeUu2zX{tX7f#UiR=D)|ik^{cNnaBZu>{ z6Jcnl5)c1g0B2sR#h;B1f{D)UU`l&7Sa?tc{&fL3=$4$&>I?;fl2zoDdj~J;>p0@Q zCk0=A8Hk%$&hE|eB}WT{z#`=_DARcaY}PuHl#K?CIzv80x2+XCeP>5D*X0Q{aCFVL z!^iN}tBADDd(X4c3KZtu`~Zqty`W3cE#Oj@1^vC2!;*Do*nc7eZ9Z-SW|O%*?bd4S zKd-Jv{Q#F!RCx?cm#)F>l96CccO9Nz^IW(qY&Kr!KMq3nA4n3{gS@m}FmlWc@;~e# zLT5W-aYk6j&65{|I>2F$W7IYlL3>exB2MR~R8VRO0qr%l`ewCw#i`#hVj_M1Mc@XqsieVVfGfx>y1lm&}GOvQwdsy`Hc=O&gA$ zv%n66-Jp`oG2fl`3KTrGgd9Q$GW=>CmDDNNV7!*F`H6UZ>SFT4=_ik`txLS7E+DqO zocp(SfOAgQ;H1kdNJjq)oXS~13mvS;n1wl!OF1PR3Nj`N335+KuDmn9US(&k1GnNo#MExrv@r^^xL-BYo{ zf3G+PXdIlfV+t&u{e~yn?g~6CrV!JC!?>nQ0s{pJ9Fg!0=*SYlw|rXta-adYKhMV- zdd~CeNF8WDeHPrYx&<-^>`A66AZ_PHh2?c!Rpv8PVSu*QVs>BPC}5G zJp;ZyYXla|PsSgYKXVklxehum6ao3Po50*!4R)zpLR&=z=n=gD?%o&;M|mp1GGr@k znPLqi03UvucgnF&))3wnO(#FO4Bg6hPjXp>+t;W($A1pZ1wGPAU{2^(aNu?>zPtP@ zF3Aic?twF*u3sqDzdwf093KH4%_+QipF!X~R|&p#iGXIw>tW{7e!Nt5GHF*5BXgI% z!54Kid2fjx)(Bi8ygg2m91Gq7|2l4lO%WYAkz3cA zXOPE7!m7jBYhZ2HO1#G|2A^DbnaeR+3rqh6;GTm&f%~SLShwvo2us{Z^48muRWp^y zw(=0tld&6CZoL9#zSD(|*!h3xqq?w+fYHB+0WrhNl))gjY&L z;rY81Nwvi_Y<9Pe_x@cE4rtECayJA#yV9q^F25}Nc101MP+?9|b{0Bjdxhi4Q@FiD z{SJJp#Ef)IOb3w*H6Z=i2R6z{5aebJw=$`qAqN1lz$qkNqzwUQIBEk6C^x$Xe0P2g z-Y&KU_0^l;iNg!XjbRgPwN?@u%02=a`y;^8ayuC1*aG~Pw+O2@N5Z?G9^lBl8eZj& zW%$yPcVNwKH7FeStLEgQ5?mT!39q&-fp7KJ!H=s=;pB?Na1EDJZcOgRs^d@NeC0Q| z#)^S=vwwkupZ4Khc}5VQO@LM>A|XHZF)x005cI#94k}LW!^w4@aBNw!$_(<=fyd3!-rA^$c8;#(j3ulE_HWZgMnTpJ#YdlROMe#Yd`3amQqBJThxN z7V*k(N0UGF-5LYU%(=by6IXIi@)fQO6DQJ-25{)b6T*f6jo>?VTClOGykQ3*#bc&?vSfufQLURxrfsj|om$FSOWy0?jyGpPRIVOe6w3J~ zTdauG;znSyREumWSO#BOS^zcgQ82UL8aA$QCVMKC$rx0DC*r4&=QFGzQ+A7|Gj2Lw z@nSyM66X(EhTFlM+?#kra}Bgse+Jkd9r*n2F)(t+7CNh4#GBf*z;~rN!u?74;KIDw zM2j=}lN~PMO7kq<7L~17-OmDw|CmI2nmMoF_c~bMwI2HQE8t%@DQFxB0_SJk#x8RG z;BonK5QtU?DEVBvfs;AdP>=nS@VKsnnwij!5_wfXBp5zy| zH+%f|J zxhJWcp9P}O58)Nft01QEFAzn(0hQd}Hf6;!cy+}Oo?LhWH0G7y&tH@9vIpKUX`BU- z{@x@U(hG(UGG>E=l|Jy?3lW~Z%ZIFaYe1^H#=+VNtDwJm7`!>D8#E6(LycH<7&V;- zpJkQc2m4pz^x{9@H`nEG(zUhbVeuDX73Y}lwaCKSh4N%|Xe&6!mSBFD25BCf0e+Y& z2!UOWu*R<#cfZLJ`gM2{BCv(;WB=o<<}SR}U8UyP*(y+e@{#a&m=Sr?zX5ExEJF%D z$&=;wQ)}d7gJ64wBJ@t_2C}**z}KIlLi?vZICqQ!>`way4xH43D^p#0r(-*W zYIly~_fJIJ=k0@o$d5bs%zWzTm|KSbTW?BkIyiw!qZHx917f5n&4(;WZoty((s`e) zo?}g=ma*;a3|Ml50iNq5K~N zt>8DHbaoz3h8+hF^R9vntpcEYa1re3_l4hkJm5+7DbUYB9%dhRBOCblg>Dz)ap`7T zLidT0Z7zMn-rO;4`}S7N4eRg1tMapn3sr!3Zn<#=PBwn^omf3jRi*{NvQ{VWPDSXKv4ZTShzQWx|5W*x$1+;C8w<>XmVcFqaBJKu=R zFYzQp(L3>!sK;1Lj;@>ssJ#st2P-YI)~hcHj%Et%&>mhk(b;VJF>IggZIs_uc|--uU=6 z*#2=f{JE<|_)}UMCNKGeO|LfM2Av(`{te9Icp89wzN9dZS%+R$|0o0#0$p? zC5ZSL0nwj*6u4wgBP~j4;Kul1Ecfjs5FZ=EIZd2RaG(anl`BKj*A0$yL`ood_^a?w zmJv<~KF)h{BN9K`C`*3*7!QkfI*@~=PcTz6ojmB&g^?2>nb4vHf5mITz8gb$BiA>e zcxMKAlx0auW7@#=Ef~x{X-)oaHX$aHtw_7sGhC**8{BQ*3Rd?D$lE!mc=cahVBBPT zxU%w-W3Qwn9+{m0dA2@qRtwWFBb3Ssx66V^cP$?`p)RY&ZC_S{^>? z3gBr!e*-qnIK{J?yoA)n^y6?tUE+?mk@lt9>Bq`FAbQ<_|@W_Ad+?j#95wg!)L3X|Nff|wq@MFXSEa|WmrfN(l z3opeIV~a}Ax!sj;TN85T$ZL@C)f>*aGa1%}g}_eTyS(P+`EY)d5FDPWiC>xJ0P}qT zaN|@j=oZM3sAfG2DngumZ$>91O$zUvXk3%WjV{O-5-%os6aPBHzXx4m?qmi^?7I!TzAuCux98ynJ(IY73O9efd<2MP^WfFsa^RG$ z18q&(@J(HLI4!ab%b#n+al1{(lrBAT{cZ#pZbhMk6|!ZJNG!Yt>yREEu9#XjzbbIlE+Jn`l%S)zaK5)Pl93U|wu;5$D#KWPJJ z0|~-0e(^yFlFVU{vjTjo-N#!T>kVILnQ}8GV;=XaGHK{;#+Kp?UcTuy=BRcY#)!eS zJIZmo$veF8ybIaiCla>YTMk85-f)_w4(FFkz#ZeHi2vylZ23$T3aY||r+?1}M(3*_F@ zITDjN z4eyV54BSlLX}0vhq1ky*U&8q!cOx{A0O$-tUX z2%p^B2u|W2uj5I{<| ztR@ED3rS;-0kPV}f-_ngqWMAU*=tTLA0Xb8_qx2quRDoh z?F_OB9tBda^GM^Kd8D0@BFo#<@uwxygkOFQZ+sW)nAb2DRIj)MX3tfJi{B;+FZ^5& zvgZ54_WfMvS45xVb%+W# z^^@Uar%9M{eK%=h3&`e8-)i(qw-c|cjxekvAGFM^!7V|qFfle8PyNN+iF$noU&iJM zvl;R@^i=a#zQp=td^zdY#7)ruKqr&Y1u36+-XloV#~*4ivLBAt5pz z#HYxG?BdSx>({NpGw-Ou?$_Pe5$1r4Q?aMU z0%5{-Bk=9PVpy)X7`~KX;HK7Qd?ijCZWfIvW?w^a-gFHjzrCDC7R15_pOvx4i(q(s zVz*`;3 zGk+yb9*(WW-I6gN}=VIxBRlNzCHY6D(9YZF;BrbqlVm57a!7a5xM1WWuohqvvtBhEo4Q2Bbl z(0tSzYo>4=I28_fpUfxVG!_64>ZN1f?=H}xYBCHOm;#v>7eIxjA=E2YBW_+(NvusW zIPs|h8ztWWi$-6AdFd8##?mUDNY9M7^Oq7y7plTAKP`~6(~TSoc?aTOw_(Afxr8+o zKo_e%aNl2DxcQMM2+i){4Ho)jv3Xd3ffaewvMmNlW^9=9v@-HlgB?l+Mh!Anm z_SKBsXnV>#qPVH1*v*D`mPf$pE8Ss^Wtz~+ARdlwSVgooZvkDiaKL4X2!t zf|;w6!Q6EnytFa_5i^q`856|FM9p<@*4p)?==5~1ufQ5CeC|nZ$IgIL@63lb!_#5i z(*mITs0SpLT0*ryS$N6js2bPu9VF3vDw%4e2<=AA$Vj_5JT0~mOsIWokJb{sO-vt_jaR)PmatlCb+f17fb00!-_V<1gE6 z;c5+KLR}r;icvclpD9Vq?AJi;S-~LR`Z}0DPFr|vj}|`s+YBb0QGqgJhe7CGUwGo+ zIC7>lvgTi(5;=a{nCuSU5B4pZh-W_Ik(KKq)YmbB)=RwLHrGB5+)*HH%o7|??1oFk z{D3FpLtZqjgI#k&;Slhyv9|94cH%>h*3#$kogQz;q)h?juriOlo@h;m$8iST^e`M@ z(vBC&2MQ&H@jTc5C-{cRTCind48CRm8UJ&zg{mGFXBJ3IPJbw%`I;BS(Fz8 zzwgy2=NGMjjiUi%_l226we37!qn}poTN_1A#hmBdwqYR0otMyv++h;109?eU@OZfy(YQYc|dzB3oFMC6C{TVPf@Fu9qv+DywzyY8g#?J;QPK!)AbQTytss@=~58)G-#oG2&yj@(!ntu0paAVv&&KlPv zreAGwPlXsvwd%kRcQy!%`vb_jxhrb?|4I<6ms7~pJyzssyfn!zdW-XVmGEcw4|e{g zO!g`G;jS%rapC?GxJ0-G2Xb@jQH2yTGr^3U$d@4})f36k>{y)G-wO1CR+DPG<-)$1 z*;TS`-)q_?NRbuon&ir13z#4N56Joc2g#`CmhzcCI{3b@Rjpv+#E!TG~TGhvEH9?yvI~{KPd}O zOJ7C&3_{2)cQH6;71yQp-38wrnTogFdIO%l@&gT7VI0oUjbn29vD5MdV0b(Rhe+>r zSnJ9seK{3ab?r6bleRcvzJj(;;l*V57uN&D13aFeL>0LA{1TYil8pnVWz?vhUqptv z^-uewBOLkX3%|5XhCEMOI5&4bv3&YNcyN;pQMtt@t(1T{+Q)I@-qLER56bYJN0y&9IHM z0>L&}c)|sPV(vVmgfD}IpALd~d*%?E6)t2=s|~p)yoH}}^YyT-Ab8`Z7SHhFTG-eS zfj4O8@E)I1CU^b?;T@B7;H2Hk@YZ~35?B=kHwcdcyZ8lASs;LSwElwj`-{ne?meU? zPJ#UR?L}7qxem0~X>z@pZgAA(8>r^y0&5=yL*Oq7_gyf8U&Skdr1v*$`S!PPvfv7@ zc)$@}o+}F{XUfCEHe=|--KqOd7{WSSmtMG~E=dYqL$1CU2XES3#ENm+WLt;>9QrZ< zN*(5Mmb)dKzMFuUx0B%Cw3-?#uS?)f(qy>Fw#rd7!5Us#r4F6B4xrOvlJMk2ZRl6= z5fArH0%1`^c#6*hRsBs^_{N#2rEqulT|#`P`Wg6eL6?Rsj;9B`l~Hps=c~-CquILs zj9{G%z1E1SkC$q#+jUtQ8~g?ZoBw8w13b~$T4&Zv=OrVV(Tz^ozDGGH^6B1Z!6Lo8 zR~dtce@K1aIXb6u2l{aTCfn7SM2YNQQI5AMTD8NLHsv7%Zuc=;MvG{a^aW~cvWQA1 zH$fe3AL^WSoKF7l9QO|CU;dwkvylF+2Kr&WB$blJ^rr3tlpLZ%y{^4uKBgy7^Qyn7 zK5j9)e&!&v#_0x=9B#!fGhvZv&uljHVh-DXDh$?UMNICbG#Z~`M@ z(8r!aCjOo_4cN)hrAgK3%%xENz_0O0@pKXUF6aTuWsLYTvraSKWzuMI%z7GLFUKZ( z#M9|V%h5@>T$-?BVSs{fk(dtGT zOd@;K$Er49Z#1J5avJqVRx^CnI2!0~iV|lUij+Oa7%!!Lw6JC^vXBN;#+l17X=gDl zc@ap_u^e&ED2l_8NM7KN4s)w+O^c?WVV9QRgRTlgFLAbx%X4%vu!dgBGof2QeWcSi zPecDs=%b01$7!7?n*I`1B8`EoXyt`xtZF4gJ)bMnKgs$6? z{@cP1{%oP=ig*3k}4qeLYL?4gMM$hu+3hoz+ z(aFI}Xv(XHD1UuA?Vs;}zJ#@~x37(>WgK^NG5I0U;tzkB@&aXc`bRsaA@?3ha!p4n z2jkheaaY+Mqe?cVHkRf+y37Bftw&cLIm=cwnxF|bZ74X~jrvK2QmJ%%`uDveD}F41 ze}2J1ev)YnT03Zh+$4Wet;NpB=1&kh>s~?!rl_Mp(;jB{w}SwC%s~y27f|!D-=c^{ z%6e>zLU$&er%$Z)QE!na9j>#ZS;Dn!Yu8dDM+%vdD<9~gvcot|@eiYC@f6;Tx1-_5 zuW}xvm|)a)Is41~JnQ1$j&@Hyfo{v6V~m#q>QZzPr9@hb9&eLmvL%$MNx}(!UG;kY zx$mFo)+-vU(;ofWO#3XPe1=fH6gN~Y@rmUhIED&lUZo2v-yz=xGnm|!htZdyeRO2zDkX4^QrQB_1b}b zf7vc`M`|tA&i*j7ruye*v!7=Q=<)W6tjz~5Z#7K=UH|+EbkpE<{TXtSbnWhO{fG6(IiF`{!C^O3WBJI$Z@Mikw^b-gQ|ru`XCv_QcT zb@_DjssB|*G2kY1dH*3Mk#~;r^zvx4{5XN5N}TBBWCh0Xwkd5({>Q}KZACX9wc`>U z0o&e@#lFz5WR7$mN6T}?X!)H~6y-cDs#tuN76&Sz0mbw5cCjZqru3a!yfH^YuM2G6 z%uvzr__NF{DbDJ6*2(T4JxD*C|BCkR)kCshTu^Mo8)oaZD5!Nwjn3{4KpRJl(Z;WL z_?MMK=m#A`*7TBetvLII`uC=x;`m-#Vv3RJ7FSWGm?heyznJNSzG$%DohGm6`o6^O z(n($0`19`w&<+U=!wLat@2V?I(Ap$sQN=hmPYR;Sf_LbJSSHd8k7j}&rJ|wu^R(b> z8J$9pvNA)4w4%g?^}QxQ8_dLN&(?N|Cdj(dUpy`9co8B)&l7B7zO3kbNHFaPOkoy( zzmKvmHnKC@Woj2iOs@TNb}sv5)qc9v$eJ$v{Edy50BF#|m`;~BWZxb=L9KslBaJJ4 zoZ%r~yY^cGEr8!yjeU}o*>sq;W%}|*f?C70T8Ah_dS<=$AGVYW*%9ZMgH3#=bpBx2ODM8)b|b<9Tmc zHmOiF<;WdW)ZK}S$tHGLg^J*qY8Glg%B>!vJoXEDNnQSy@|BX`u+{7JQRp%SLF4Re zC?+a_*4OC>I(~0Q8L#f~uf4wj9Sct*qjAR>*3g)yxo%`q_l%+DJ!Mqs+I`4htBuCV zJEBhJC2OqRj)X<3bY+AD(lxb1#Uu8DomNfE9NSaOlf56`6uK~)?el=edzy2z!-=t4hs@Kq^Bem=fZ9basV1Ocm%aLK>Y&7h0j#Zg;n<);x zhuRJqq8+&o=$O@MIy?6Tn!9U?Akx~OmPY9#C+*9O$Beg3c#b#y@c0WqO3{WIhrdDc z5dz3IA7K0kYtS!nipe%Lr$d{onTRD?wev6CW}62K+12zH?bWRmy*p)2e|nqOetdEo zog2OiEpCU?)!nwN%rAfnj##5z)#7y493N4l*Hs#_Mw?b@TG2nZt*AojWOm!;jVQxQ z3B|PtkaGMPM#aFe)^lPY3LZR3UgI9--TQmQyF>=;ZhJ405|V+UJbaGv_8MX{b^fsA-w2eZrMJe_5wB*2{g=E5DY?AvjPwVCFRmfy6g z)oxoxBVV<#k#Zp@$6AEeyg7=@^(NDQmM4gPkv|=t6)KugGet18r-efMNK|370_7Hd zMA4{_h*#W4U!1H&Wsgevw@iXj*QQ96WGbZIFO-?wbJ6HZ-*#qUfEv{rt0&tBbeW5) zUDW7MChgp8jr#7t=2tsBVON>)C|Py@&F(+NDhlWD8wOt?zXQ)1OM@2XLwf;|tUk(` zY*eZ>KGaO@;|=Kt%^vhCW+7EKRHS30DQI1}IHOT#OH-3GP_3^7js9?p?uz%JCllhR z@|=3cchw61isP&4T*p{cvQmKw-0O;(rlgAOW^tLso=Up#LlUj`ibURnDM-vz88x*A zvcsB2f)I_hqKd+LKL0=-eevx!HF*A7^y~C0bkfidWeuKYZ#=}jcfUTOt$D8)&ztp3 zcLzcjn^vNhx4%U>_ao4{SKPiztcyR`=Y~S0|6?o}V>Z+who79dg^mn*qIBI?^ zOWSXm(=S`c=+*=G(5K(O5xlpkR#*EN9r&V1r&v6sL+}uNv$g`czu$=@9e?5Y%mlXS zRS_);MBzAXB4^^m<5?rp9quO6|1+(n+_%o+f zunSh77kTRBp!*ZP@|9j_2o};Pk*(i8%BqU!V7avDp{q7J-};+%FIr0DLrPiu0&(;> zU^-KC)`-0|aEgh_e#w}Oj*5cg?y~!rB#Ca%N?}LRj-y#OZ_?{a-m@`lo+0j4Gc@6M z9AEO$9D03|9&_&8e<&mKJv#ZIlV-+i)c){gC@*51AirxSI$K@L<}8?0TP*BFBlsXv zjf-bB;uS>l_zVj1?W7VI2!2iFa-6*k)AoETl6bNH9+Dcvi7+MeR7& zP`iMt%;IwXJ_>?e!7+T5L3LEISB$;8ZZ@l){+<<|sD|9nM)Ti);Y^Uvjg(l{vAdhK z*lm^?bW*J)QuqJBQsH-IN5my6rK~{BcKWq(s?F?Wt%B(s{Ovd?c;$MwBg%R^eA*CZP<92 ziB7CXS|2#qqcDTI=(Y^|7iJM&*k5(+lHjTY(T zi-<%&GW)%hx&(#O^IvX?TBAnTh8PDldT=(sTzr^0WY_1HprUzi4ou8ugr5!`3fm>38)^(e`r8Sl--@J{)<*{mwqLhMQk>8ACex z@k#32bd{2g2kA10KWyP<8G+K8lSoT70qq#oqQ|~pXVWTN>7-m&x;v(Xp1FTdBy%Vo z>5N(;``gzTxiU*Mf1W3kX?>WAPuRsy<}CWQ5CL8Ec?a^4{mo=~UPPWADF`2SP`@WlFdkxm%T=Mr!PnT?Dwe45(qVSEUI+bLz62dJNnl`3AL~$} zj&>gYi@dK~qmlm7=-=)0Ov>YU_J~#)`<3s^CjW7xqB}DLoF+}bpH!tK63tYzs1d0- zzd+6|#?-C$5+gHS9wiK4W`-9OGFOWG*u9zWn78ljYa16gvxT8o=%*iP4BRTieo~qX zoBTAy zFPb4a+o?#r^AT&ZMZUH_;THQ_ub0gYSi&B(J%*~sDbmx%n;h=xGbOEX0B7W{%Q* zdQHsM?N8X(W8QRMQ9JWQ;THX6?8K%2}yi4y+W)35o4w8O#T{fs_b2&0^ zeo0&CS}LD%jIl3h;S6GZ%6-#VzPmGh$FD}m>*SEuTRWP2(uB5*b7l)~T%>F7`q0q( zbJ!j4f?4PB$>{XgDzPwP!(^nOctY z4GPe=3@?=JX~a}WoTBR%tfXcN&P?Wmcg$F>B`e+PhdhiXQne628eiwfY#XpaGyN8! z;fEe{tx#T&>GP14OFBd^uNz`JIzpM_J_o52Eaf-6JIy$dO~_SY0b0H1C^wIqO#eR5 zr9Dd~3wB?Y72Mmu0T~a+u#(?e*e2~h(Q%onG+cQ;n*K1I@)xJEU4JAH@r$Bc-;L1+ z*KU&NH|j`g)glTX@@W13J8V?J54I^M8D0C--P7Df%fyc|7p;zo+?o=o znw}!s(r}Ue(x zmBzPud}Hoi(-lt%JI&w+H8zt#2i80it=ad6_Be7K0mCKy?DnrBx+j)CG(SQWF0ou3 zxt%{LQIl>>)k9k%c8mVG)T63c0j*MLLcQBu*>Cq2(7Vf?vZmi6(eJ@RQRKN47H2qwEd`-=yb~mVo>{?NeJ~48D5x-9*%dV7u#getF7Bm;hs3= z%1J%6L#7P1W;;`#>;fijhAFf7`uJJ}on>an z-p}q>>_fxXk7plSb57htMcRM;1l=iYqe&CqvG3kG(8(2dk?ri8R6#x$)s{ztwaL$^ z#-k!gt{r8&i6-rN^N2Y&p#k0M9%MG&szHx@d1(0dKI&;`KpK}B(nI`K_T9gUg2+~X z(V6{^nTwMDu@e)A*#1WvwB>vyQaPZDer(J}^kTWl?Xx9vcYa61S7p<=71!t|hok5~ zk|kZU%@YyNZRoq?D!LsMQ+v+#t&Ae=6uB=R*G*9r8HvD0u zQ|_{Jid7gLDMfaQd)|+S)+KX> zjfaR`srZfEb<&>RyZ#Q@rfN|=ZAYXPXHYwR{7z)6y@pPoU5lRAU#4=4G|~LJ4an(-?7Vr`K;2L66EA8#W&ubNpHqYuC?I}F~Q1pB9-c8 zY+u_r!Jn>p6eyQQ9m-xI{%%udtEn+E_$x=#vlH2fnRd+X(hp3Z0Y%bE+t8+Els&Uf zUyu^i$~{BMk?K=_=2YnKR_7+e+Tq=p#mj(N&=m@WZF+ssk31O zd)RX)v+A%L^GHP?(*CSa8xePw&oIes$OdnEywx1_z4Ah`dB;V$$G+0msaH_c*$b$o zC5nDNUPpxnLi%XnBn^@)W9`!Sp`7n?sRqrXU6w)g;&@lCPwxhuSDlF>HD0kH@y?Wt z_8^ZFA&RW#qB{7){~2X}0Yk)D7fNZk4x9BjHvuos!UbSJbXS*;P zHw7c%n)4!!n0Yk#-C_D;$0znlq6BN_J+*e`KY3IcEl-1n1dNM>9NYdej=gTz$Q;!# zVYBZ@(?B?is{3|QsmeU0E4hhw_)GJ9_YnTJPn~R_{ziapLzuxTFM4m&AUooBgt^Q{ z@qa9PC~9{!5G1{*rX#itD&WqcwFTqogb!&Ho(rI_9>@!dJKnOUG?s2WWrA8KHgWEF zH~lZY2py?PXQo|w!Ll}-_c~sMAx>q~pvId0yuyvm0L91TzY&eR+Y zq}r`V(86`bg1+c8Ox6lYbRIaNGa_}`QBoo*byi3Jz1~fonP}G0I2~2`rZSZ?v?%jJ zm%Xem$uNR!T38k@5*|)Q-zUc*soVml_3{Sx$B!9Ymx(gn6m^era$8BSCAT73=U8N1 zi_ywO`n9D$8jz(*KibLbLv9tfkzL|8I=U&AKU(mWSu`&OEgU*QCvG-jhwBa^i+>!t z9~+CL&aa}W=F#lLc{3^cqfLYF3K2Tjgbt}+MNQMK(RfMDlLJ^@=no%$^!I_EBZaa{kLFoB3g?1FWWQ7C%k;H~ad80k!1JFOxwPG`6aW zmPDVXWg&Y-#y4&GAGoQd^)0TcG&vO=)E#8%nV*cwlyg+?@m7lF6j#vxd5`!MrFC@WhC~|o z@FM-ur;Ic09?{uW#i%KBAs=FNNLao-1AVw`jxJ8>Va+$}N8*-n!kS6g*d1M? z_|G3li5-Nw^rU?}`toouomzYVeK|hF#}1mim49-e_SrQw8GaFN?mi&AqwYfNdk6W= zw;a%VUlDz?Fc=+PDeIRz+i4%-AknTYlU? zzG#K~IbP+Yv3Tmf0(zm$neJ8N(doHMpxK&M{_BHUS~_wcDp=OTySuuf62V@!ZH2osVF+dCyl2`G3?;$#caCd+;cnX2%AwNLnG5YS$X9{)L=>vtKp(S zy-yqmmmV(Xm-pXf7bz_ujW+|i_vsU8x9)tlexWaIJ*Z2OP=P)XZbL&bfb~szNRIjD zqD_;5xGG@{6>O9^DRTet%kNZDuWjn=r7fB0Mf_xTu&If+=q*F1*2}vMeB$|vLt++f zS3?1!b0|DO4tC?=3bnbH|RXa+Av%PP#5~?q|soRRTT@j5)(Rmc0 z;K4^O8jae{KSFIYRMC3Zv*bT!H@dt7((5&GXiG{IzpASe?O5?v_&pKQ9q==_*MEX_ zK6Zn4ZAn5ymuK*-`x{}F(XC>0B<)%m+rs-Du_Z+=h@c{8+3pbbb zUi|Ifm1xEmWw-kAH~6sr3vArSNc!yB2->vq2HP>Kgr3Pc#b>V7Kv(5%AiH)2zTfjC z>zmod%KQnXre7jxP+~R(OP?d%oPDgeKS0xL7NNTaU)c?r9RHJ?;&I`A>Uvun3EX^X zUG6eeSwDw2O1RF7oKMngX_nM*aTe`a9xPU=RiJP3sV? ztUs;d!R`%o?le`_d)i#lo^7gbNd*b?ZwoI9Oglpz6i=g<)sw01#YE91(#~F$^iP5! zchdPnXFfK5ytpCl82Y5O3GMivL|619YOlG9HixSOOE`7B& zSA56p&fmtOjCBfbfy=iGW2P;pojD2Y^GyYO!*(~;psWB{_1N*YMy1H>nvl+OP(ywD zB{K-0rL0>fVFzaVqpSmd=#2OSS}yVS6jzL*X(zu5eS6K&Y`Yk~-6@{m)7!>ZWLimo-WPcfY-NkIdg$hc{lae_Oi-=WDio%24z-@#$kV8D^lZ^*)c@`lQd_x>-a2>; zJr$SJKIjxMma?K-^wdd6ExjhuaGzx1 zJNa&Ygk;a7;lg2hB<~3C`b~(wB<-et_fD{{2KTeBPQ!GQ<#OSGcM-xH@3FjtvRl-U z6|y@KOHb8yqZ{w93%8r!=bLxd3(fd;8k2sU9{FL#{}NDYW%&(B>pQzG)}O~JRF|V* zcZpX_Iv>79m+)XAic z|J7(hKPU$BZ)fhOH8o@T1>eS^l#?Uf)^_{TBdfMjCszwP^^q5SxbHB^4Lgt0zL~Kz zzNVw`&;8JM?RM5?se*WBxHGlS)RXwnH2IG3apH~T2WZ_3EqeUtEwo*w6kRzymPV!> z;I|k~A^RVWW;@(3Q>hA7zR@LMuBGECwiR^uDJ!(Gw8?{)7c@5WE_CJplq5k9R^p%4>+j@&(UA}UtsHKo@9vX*! z=zZX?e~jQ&9g<R?GW3X3}wCfx^)6SR~&%j@>yUi&fgJLLa}elk8mOp!O5a zNZgY{w|C|X=YQ;B%Z*MW6x+n_eG*G`<8{S4ITz^DsQc*X@NQpxK@JY`=}s zWJuP~x!NB^0T-sxGvNzRb-WIG{xX#M$MMwn{&c!C-5zzaEv(_N4!saCT3kJ#;N}yv z2|fQYi49ZVOxycL@<&pw(DchnZZ^js(VAtsD5!cn)#-d93J?6m1|HhY%5K-A%{PN- z?hH$0lmDFhUkYKX%2N0;{Z#aL{yX-r!7?)4BMQx&ewnUiezH5muc0Z{39R6xG}@o1 z$;wPU&!%1I5pCZdLDdtJ*&PG_2~n)FP@sMSaV5E;p)hNSuTEC%^S7LS8mb_nMq(6H z`I<_NH7D3rnVM~|r@w+7s7Ol8`?daqHuBN@0)Z4Y-=iiDI3c;8cHBgnHW}M#pvU6U0PZtnJvp+623T=h0OGBq63e<3k!6QBAIwg@G^1~SkqRB z*RSuWXPa(_W1w3zKI+uN^`Or{oJDGCok@*BQoeONGm=Y7dyXa1uOqzyq$CZ;VUx zFM<1gOYq6POYydHEqrUcn3-7P0v|pB;M9)aRqdZtNq9snZgkv_#ex^0!E6@k2+0=o zwQA$MjW>V<`z*+Qx)1QrW{~Yy&w)JwQ^}u)3&`cA(cn?gKTt8Eg_-ay1Di(~!Tnh` zap^(<5G z*j&+`+^IW`7w#Saj*`52Uf>u4quy4x-nAgdPeGXedI2=G@vm0i5(f50TfnYaeo!qo zAAI6OJm<$hpkvd>4NWkBP5RRyV=^17aW*h`-5WgXniJXk z(+(eboDBYKw}(eOc7jL$CIG&N0}i=&!0s2DLH27skRf15gY*>mMH#`i-b5fZV>0o> zCUEC2511DpKtiHxfRl+2@vAi<`@d%3@@3sP`gNvY5RHZ(XWRkzN>`GbLsMbFpd}m_ z`5efP=fOkkBJi9PSI@gr4H|0yU`XzQw;d23$p6In-Wh|N!y&n_dmEWL$%tHZ(Sx(w zy~tjz2r@s7B}?8t0g1Iv#A0QDB%`+%bI)91^ezGC`%;mtUvV4{{Me79)=sPL=#zpM z#i7+N!-K0C9U1WLxC1w}N3!5}-<-TIvBG~(D?zD%2u4nO6{*}CAy~9M9cb1!Gkp(c zk;Wnf6dV){5Ma*rO6(GyW!_lf zMjT^xcNUgx%wvIf0n^)Z4i8tD!ufKN9MSF7crg4A$dj3j7k%0ZEHq@Hfh6N{;hznh zroIqqgJ}VKfW*BU7UJ4g9@5jF)cVm-L#dz%nITE#59?p8=NY{vMQjticob_SfTjRij(MBv?%J>30?&g4Y1 zGn5xEfivQF;l3SxU`d%Z*{|hGo`W9zY*-9jnxsg*x+SSkxdJZxeFZ)F`M~SMX*?$< z0$Y47=T<5iaJIM*2;gCSI$9OJ5BZBZbxTHYZ2(NtoJwA|X+yP-{gRyWHST-(1ma`j z3YK|VLRn@rSd4VRaB~*^yQT|UckaWB$7qr5L${fZ3uSnRS2tdM&=AaB76dwFf8(^Y zsW{woBkr2n1<Iz}GSh;p)tn*x6FfIZ%a6; zf(Md}TfG9Y-faY?Uo-`IrMhI#9~T(B%9pHhvm$TdE0A{W7-uRk3x=vsgYHT(SY-PH zCr>Q~QM*`Jo%9Z`NqLA*7W=~Fem^+Y#R)2eDuT!Q$3V@?d(36MXW*oP4ZOdpQ^F^i z!IYJyfeP;)ocGZX%Iyz@W;08{cnQDd`(zpChj`1 z)b-n#UCfAxQ82?JmGgP7P1Nq0!hYThIz0J_bF&GEdDhN7&y#^`U-dK7I!3@9KVRU* zlI*U|fja!>%pVSVFU2y}j^yApbE2|g3^WyZF|v0%acshJpcZiwt9?{pKFo20AGv$L zd1W&eYb=1rN@v0Hui4<(iawAx@EaVI9I!umHoCI;nxr~SiYTpKNSk4 zM;-9$8~_j>v3n10Ed};s$Zj4nvss@;R`SWVQMZ4ODwt<>T!9cw9K@1_)KH05R%i z*!rs$NuQ74ua6o~qvr}pG20JLv~7f+&@h@AAj~oY+A8Omr#HsKt81m9X`w3oa{VxN)6E1Gz9(?wj-S}1Cl0Lmx(3hwcoMKO z10dV&J1A0Q@b)wFu}H8A3!R1JLAw%kReB0mi4ks+=>%HSElAC?2e{t24eyjWi}UCA zaZ_VHGbJ+L@sN%Rp6}WUriV;}(z*)JM3T$8c;*|vE4f!k7f`HTYzVttH;~ZXbI6LO zM(hy!4hz0=lL-fwP)~J^q46-dlWn< zosH?PpR4HP|0#c%qSA>sHguD7#TH>ycgWN8Lfxlw3u=N^k?x#})jviZtEjF#e zZ98Uw(W58AwO6c&R7)I~n9#;t&6tCq)ZcM+yzK}`Zygwadj`~=vIJWm%^A8u`Y_6qyG+XjZlEIHAA6{Vlh@t zRs*GP-!QfB<^Zb^=kVT9AFG|B8S>%yQW7avBKH`7Jh+Tt;{!4z;Fl4!X4C}=>(*iK ziE~JzOb?TIDTx{LeIis$_J;?(%DBI~=5T`CbneHKndDngEq>{pffu`A{5)MBNAMP; z_PjQXmN{J=C-{RCiZhsBy&ka1$AbJ=P=eD(%mDvhe8uz5oCU+vTwtkVIM#?Zht9kq zaatjfq${iA`yYmY>0&F=I`K1Z=&&Zy9Vz&;p%YvxlGOM&*MJ=(R)9w;(qzWIiDc}O z*+A}&3t_%#!ySr6K=Dx)j&1J8pC3DN`US;c&y&fdC)NSJ8(+;emBeum>+FG6Vk7vW z?@i{q?;^DSHhBI#67L@^iG1o%aLr;3clH)>EmZxDSFdKk)hEHs$?%szCQE~?(KRPG z4sFC?1^Q6C!HMiRDkSN@_kt0_3NU$I9VlGh0d5y3fXPxCq<{KKyhEr!_Q}_CFY_G0 zUWIz*t?4=Dj7tY-+ZDyUe04!^U9}b498l!EjZffBWeH%|J0BS5E(J%KEx>%28{xj1 zli)}dxMj5$aO?HM;=&F*XU8WD1@aG z9pJ$V~pl68d&f+X0R zo(Y6|E5OGLUA(r>6~DLDWV8jT+&%r#kk8P9_WIJ*wLaQ##&lz{Kxw>$0UE>2Uw@q& zPRe55?6M$d=j)O#eE=L@3}F1^4^EkD#4He45ywFrxOcUL%ky?FJgbXf(ygTQ88)Xu*bgX5`+W0XgKN3isdzz&Kw8Mg(PW{lNjm5qN`6&x^P?NdY$Qn1>^O z{9wYjPQz86)=bBDj(c(5rpj%d7*}0V0Aj%*5V^5e(7x7!w2c0YeVVP|;yXVvu#O)dPc zXzob$u?t{+%mY?1;L*kGs%aa-+@&|1Tk?5BSI<-`^yrT zbd&cW`*9Mm`=JRh{F5SDhibrI?ko77ArEh#9}Qy?_v71zr|^IG>#?=x8s^mx3HKqf z3XhwfhwHR+fc*m(BBihul)H?DCbgbWc))->RW&4L-zMXhVh6ZXQm2gndKPb4Di7b; z+LJw_ro+EolH9CqVfCpMWgu-qx}Bi>v-?6fKHy)gShs$!G3@_gBh5tD= z;Xtb=paaY!JNU_@_dp(J9B~~eT~NY{zp9eg6-V*80)LV-!WK%a=|GWmD`yq50E9mE z!qpwuvGfIH5*=&}=N9LKmTxD4(Z2cQ#MVmiVP`p3Y`cQj`Ahyj)DHf6wv3!=O&}+& z%;A@eStNV8A#^^n5bt--0n0{@hMN^GfO zCBG8JeZ?`(+WHB|T$ayePBns7PaxEJI*lAWPng0Fro>Ty6)8|QB0@eH?CVRx6NNK@ zed`V!)bJUsofU?+-@1a|u2g|EOcwT^Hy}TKvv9+`ES%CBz-ZpsCc?_`mr$dI(J8U&+E>Sv> z3syz?kW<$dkoPwZltxw!PdT7lk(SIinG7Px(G$7dw{z>KPq zP|4U0mN&ZKhm94uLLn17SKEV$mww<&jt#&?^Ds^;)`Pdc2tZzQH>14BifQKOkvvI_ z^?LkV7<^o__E}hirR)mHMS1Vk3PZpv?Tz=-U;xB zToX4X(gYsgeHQNvy@fraUH~8YeZX{60%Ldh9$q7w1TV=4!sREdiSO4%WW*#1J7=*6 zyy`ZMtd!RzcO(#i#BTkhvE} z;UD)a!Q)kT1aD^l0s<)y@Q8g5@;jwTs^N0tQKJqULd>z)!4sir8sbbC$A^JL+^Y%%-<6@k zy;z_VkdKG{##SHSt4xCad0~U~W!&-miD`+12D)#4<8`XcD<_5(JcLu7Cy}uUMOe+k4yKoR;qxC{pi__@JbS>1 zpc2WobRFTd5-IY-OB(91K4f~@QnJ#Z7Ed}P1>Yr>V)K%JAo<-?m}QB%-;(REA2kK{ z58IL-ZMEQHxf8x?a1YdktHG0z&Lp>kfs>-7;5vJW!(nnYj&5{?@yjQ}q2N>4emIn5 z8M(nzO4jh=+b^JUs1++_`oj}5LttdF3}Iu&!kbE`f#~)iczU`Im%RrtWThq<9g~7b zlYX$sI*@33O@~WXI>LpkzTupWu~<7k8SBq%#Alb6REtEj$>CgI=sa-_G;rTnZGQLz zUhprOTOF?n(3@<=^1m_gTk-?UEwLl7?}~6ssvVBnn1|J@*MQvd-As*(CR|oH5&J#5 zj%Cj!R!{1XA&lhvS$}RKyrpCY)7Ot7>CP75f%8j{v-}2V4$6_ZD{gVI7tBevy&U;z z>`7MEy$4p~et^1fh8RZLkT31MKz^D6JpWu1j&nD|dzMdviDn3H;F6dZ-XDNuSr_aH zDy!1wN%;+HQzV4au6+_CM?xR2erOjBtgSi8OvH^tw>2K#PfUl((twr3Dn z-9WH)q#=CUdzy)uVNYCUFCgYS2$yv)0QfJD+GQMMsX|b18#d z+W4~)W^5;Teexos<1b+d<kO_1R!XKRbm>uPP`0ZlT}i5BsYWGUeK`tGt9{9>7&R4atN2vi z+xr3(4Qr5+{GUKLLqpPEJeIMvh_yNbm) zsS|^NnyE0sHJ4kkLxIeGqz7WwOXJP5gZRmMEm-^MfS?A*l4^ou$})LpE&sz zcPv&X!PiT1#d3F!4v&N<`A=ZmcV%Kc^$n=0vVi%L8nxP^9^1cb!3mOgRPv?~B%xUb z2$2tVx-o~OUwng$A~Nv0+-u-JCsP>mC>E?&EMuk|?gST?n8SN|Wx#2H7D+U@104QS zhXp^x_)?ED=`yn=Gw$sIzjj+N>f#n`oKpvE)ePV^jYP}>S5hw*j(vByfG_pUAYi^F z>A92+?q;Wes$Cg^q#G;1-YptuY$zY*PDbLP3BIg+LM-w!(hsm4VXV!!`x4c!`U0g!b9G(;Z-IN zFE=zJTAIf|h_?xpGi?A4jf22t<6r!8)^yTQe;3HDHN#!o%yCwXgfsR?QiF{#Bd;Ya z{eBvI~v?14EBXYe>fBuO;l#0S{(6R@5lH<)^of$dl=+R z&c~-`mE*0FKA}a^F2VMlJ)G*q^&mCo9^RSRR&_$*5_44KPELF*0+TO4W5&!Oj9SIW`f%nRN)yG&+x&n zK~Q4_!HLahz?hC^Y?MBQ1S?vDe)I^t)mQ_~234q{pANkH3&6|njrdB)c5ckeR;Ghh zgc2tb5pxX5jX1y=?5xJc9s?jJUkl-ovl6M7YCg z5-E$|KxL*aTA)4HDL|I|9+HEKrM9rkcQd%?Xaq*8-UL(s zdcd$_U6Q))48AF;anjd&K<<|;c9O9ATz17^odz@FcWM;aYs-Jm6;tx?vM0Hvl!4DI8Ub8Fw2Ac}A#uKST5z?@hA^f1K*S!ywoVU$>t{3a$YBz^ z++heq)MLSeqtMMDY9V|soFy8(S7c%A*_*h3kzvTQ|;1dE& z<*#znc4q_OHV=4E^aYH1#eqla6YwVaIvlT90lr7w!4t<;GIQcDgEK29fE~?!Sl#zG zwu=~1U6Yyx{yJ&G%U3T65=K46MQz16d)7!0s;NlqemimsI>p?~ac#I|=smurJ|Ew2 zk;XAI)4+jN8FKxKJo&U*AIx|>9=b@pV7j{k$!TS4cxmHqCQPdcn+8AO{B|D#;9n2E zbpIF#I${IIYTO1*T4wOoYZla%SKtt%dd_IpSkRUI9G5uEC(ou;gD=_R;fC~^VD(99 zAhRU^-1=q&FW!Wpr{pa5KKBBAo|pqFFN}qAYtult7Jb$r-RBW5Vg+E~*!zMccTc$F@)Ci6 zurju5M4(MtAMmRb!Cx;GU~JvN_rZ>&|KM%_($C|tg?FllhmD!W zRf;4aZa0^@ZxrB{RxuNjmoj$-;y}@t_kw|c2XMu)3wVLL4lMa54|7+0gVu8d_ndA7 zcUMdWGWH|LxKlcC`lJD1u-OpKTVBaY{ZWP=FB{;;64%G$Hw$s_0E_}!K%c=Mrj?2%_fq(@qkea;d;hIJx0 z(@6zBd;0+RURwuNovy%N)=B0Rozm@;GX?FOlvF zAkkhgah<;u3{_19ITs3C9#-oyHPU;TxwTvIwY=vzv6uk$4S$)>4?lpB(ub>$r1D_x zs@%%?Z-YT?;RVJw_cUW~F`9f8%_N&E{5YS|!+36fW_87tn;`0QA;UdhgbNJh;EzUa zczs?Wc+g@2&t2b!ccJr4&DTNZe0D9^uv-y_%Z32i@B<*_oHK5Dp%0lzN3h5J6;rCE zO(LKso(sykf;;12yTo;ofnEat90j0s_%@dT--5W5RJ?1dEx4u_1EjZbXW#EEPB zv5dt;{Au2!>KEgG;}c4bAl9gsS)kR+6wTWraZhZ-Y97Vhjrevm)^6uwd1T|MPbD>tC}yxOL|8VjNl zLF0@aO>F4+d0T zJNp9b#M*#_$M3+?hm*j?y8~dA^EK|uB~`l2;qM=r%}wnf?B4_!pVV*(DiB!#^_&$%}nzB-VH=Suy~x40;?WqkiTppj-GrR8#l?2%$8MGa?(S*A ze=pRzQyX+ba15N?IU(#?(;7(9dVFJ`{U#W69_>a5t{Tq-f+Qzwk z+=p*l>u@uzMv{#dp*U=-1}QDpB7yTlKyCdKaQXYQ>J!eEc%7U%SMlsCNKKIAQeK#m zkMYgGPgx4qlzW5mvzwS(U$#|0sr$^8ACAG@bLHSvk9uxf%mTRKyc(Vz6N%m19H4L3 zZSeGLIXIX)1o}oj!S#Zxcw69Muv)4LH$`iKm!r0Ve8&~wXT&C;YHA{JP6=`D7$tb; zB@1n?-p0eP453U?GOp@40_y6!@w(5C@Z;IjL80vfPQ~8}u6#WN8V^r~LKeW$5|;m4 zV5CApsNe^l)L~B z=4CMsn+^f^l5nX$bwH}Y3Yz>JNe1ppk>ugCoVNsQWNcc6)5>k|qR;>EXq7VTKcxm& zdYO>Yq6x5S?nu7`WVw z+yTgWfcvIVlEZzLUp6cZ3p_r55XyRVCF z%Ht?SdyqU1y@F%zZE49PNJ(c%qh%rqs;A4 zG-!T_J-G?FxfM?0HAyL(7`Yz#?}=i6WXxjUB~7RDH^Ql78bp(dmLTs73wgm68G8Le zHoYUap8W%wCH%V6Xoj_`+k}*D^xf)XtPD03*GF~>%e@~9(>@ndSBq%+UGp%CJ>i1d z-T(0|H;&NKEvfXV?Q+zsWJf2zdcw|P&!LTu``NQo4vKC_xzoL(V`z#v7X3}DsFozso7&rq5cycpG*8zKJ{ z88p{V%v=22j2w!Skjm0#HdVccUA3T!j*r_<)h=G)FK&5DpRE}~^B;H8QrHE}a@|;Y zLn&%|vyIxv=~L&S40g@n5Fe==!H(*8MOwdJin?^wSi|Zqbn`EsUC6&dM;kNw$9p6! ztNlVYu6G_<^z{MXa{n4NxO%VPf8(hAh1+0VO0degPzMgWh47SufY3{4p{r+@d^BjZ01 z!PH8&ZWmz%r%&-=>s}#;u$$!3iZ|@|MLObsg_rb~?_BYr6NC@xJ&R%-)lg;i8oJ*! zfSwZ;(tve|beqd(k^1H1y#A&cXodTJn#F{(;+b01F7hv0XauR2>qL4)1EXno3hB%} zx9Owth9rLECw}HYqfmAJ8J^YsP1i+!q&A+<$l(=7uZi5Llv17O zQTl+$zbc5%7Hvnzf^(2tLm63oRLGA^yU35+<%33xR9Mf)Cy~CsArea}tIP_jvKbT`xnP0TMxAwv(5rDr%jZQaIx$f;n{FFi-5^I}lleRq09O-sCIp)u7D&O-C6 z^;qqHGijyX5z)?4i6&~?F#7AeQ7HR85sk17q&a&&u%%_&sr|+RYW=>9e<5M6TnVk9 z$1V((hFIZC=};s z36(r=(tywr{QOnR(3pm?XvqvozB|B`oz*>^zGK%>X{9dqQq^2qzH>& z{c&Kmek?@MyOrIfEsn9?2MXxP`%-N5JbCu(;qR>bqD-VFxGk)``hdN1?I+*)ON)x6|>4OZmKT6;W!(81cAyE%Ze&;fHoSU|Zg;LId|i{Cqu(7Wt+jvr0R5$q^Nl z)xDP{Ii^bZdy>p&Yb0{7kwNQkedLcPO+mA?#)F+xcbn`za z6f~TKT;BN6c{fJUvQ-b!P?9;$zOaQ_Uz~-aueH&b_FVK?;}xB6mBY7*Ke7WJ&)J~B zrIf1(;^oYr&y zQWejhEUiH1cD_hM)so)b*+`AWKgoaBrlBpGOR18t1zp;Ij$S-^oZeVk%-1c~ML(CE zq}5LUp$+5B@hSCo31bDJSBEO;@0&kaui1e8;-!f!l|~`W40-z6b3Oa(WtK3lpn}ew zw1O@^r-Eq5ef0bL8$NVPJzo&`lvVPXjQlq$(#-XL`P5ymbdfk8ZQ6I2PQ7NyY9~Nm zTdxisObcX>MDAs`7fcZUTJH}p_gtn44+vUPxtG>tdZObc2FUV~B)ffRE!rzL!Y!d` z1hW2dlviqs^u1QvjdNpAK}|C6ZPki&Qp;%F zQaSYTl?uB?$guk=&h0O+9Lux|SN%tU@#PhtWj^fF`AxqK8I7 z?Ds8{jhvB5Q`8mc_I4E{IBg@|e``Ct!d8W<&JeLFX0rT?lu}m4E1CVZ{u%pl&2)ZK z_bi&Mdl*$7)S`8YlO+5(LeI{cLamE2Dmj}c{3^Yg`Ykk~CwlXQhd+*`W-kx3J7ylB zF{8C8Y&%OgK5b?L>(9|?E7#MYm1kJ7UxzRbgbG*pEk>v`huRP9ML8G8pk`&(Z8WkjnoN7nrqhvQifEglm>!Ty zM6Xtzq}vXyq?yNy_=F>u*y}r8#2OKQd0n9BR#uUMzA$U~CWi`?X>ypViC-geuY{#& z&hs6fax~EYD0-G}#s1bk!F!yILCV7C{GA)m;Upt-zG$Yrn;uz-ek}__B`TIQPqmD1 zjZ0v6uRP0}7|uh5*;BCB;W^YY?E;z>=Sgq<9*rV0<#@vr`{?lS8`gA}B3-=rKe~0@ zFY)jRt0-v+Ax zNCs^i$VcxYCAk(ybJY20Ctb3AJ%24Y3@tWjrf2Tj^0jSm=#JUHSl^s?qQR&<8t0e5 z(^HLTf5JqupGP|DdvuWHA1;J1m6sxyErB-NtwrxOzVKs<4x_@A zZtUCvOV-)7h;FlxLCxD&A!or}@HrOKX90<9-l6BzNT5e0`>E85pG(jGI>iU98|jur z+G*Zx4bsqZil1Q;n1AamjcJ@f(>y|Xu+oJF z`xXh`+aE@qpR!ShH=*XoJK1YK)%*i@D++Hqqt2I~*zV0f^p#2uy4w~-e}7)bvWekr zzO4h_#NFjDMRcKOkLB3G0V_0aLn>M>KZk~#oleID+tF{je_1aHi)-Ac@xl?kVQhf* zTR!E(8+KP<4?4f+BCUn}D9isFFEwC9m49DH#&f%n!mGpV$<^)X)y_Y>>qdF@BJ&0L zoM@)!T;;_eF^K(r#2JNIY-ICN2UwTG-SpIGJ^s+_L0)d4UAQu?mL~6)1Ul!b(dbuY zsLW@O{apPIKB^8zeQq_Pko({H@!`Aa)9bNl-S^Ext1;$uWN#-O_E=2EE%QSmk<<88 z4>8|uF(9cw@=@TQ-|W}}c67v!Ab6_1m_Oa7jHaZ^q2C41soQJ|)Tcd%SJ#_IvvSnw zC*2<5+mVTMw#+GX`htL0ICcwVOj?Tc?mS@Q_KXvIjZ>pTUgc~ukw=r)tD!l5vgq1o ze;OotKfIo`fo|;EhAf`yi4UgfpnLZo)Afe6=tYARTflBWHB3IA3c8W8a4PkAWQo>y zkD&j$_p#F()}t%$XVAK17m#7oa-Mx?=hjp@k6)a*p0_QEMbF;dM~WBsqVER{X^8O& zn)K}_yM6dG@7rMHrX6|`#YLW@TwxZq7v4ozGc8ex(MguMszlEuJZE*{o(nyT?o-dX zH$_Dw`h=b@C7c4kKcXmocYd42G`4%qG3q<9$mPRn1r+S3PD>1msPW=7R8^KttB%+r z{m;f|@9Zdaa_(q!qS}BDaaw}Gt8a6Uhz^Ys3i4M4KMu!G!QS{nVEY>-WHa0BbZ;w%M%koiYe~jtlA4=ZEb9zT2 zeP2~tY?>x)*|UO9zz5Ky@t*81jc!Taa3U&d|HvjhQFJ>1HN=ZuCWvEyJr(}inTYbs z7g3>pG9CWPvmO)A@H?+oAg_Z*(Ux@^QTDw?3CHB0@bnc4tL2O=FAPkf0g0zk+KFoP zBeVlGkCY&|D=<2%YljkoO~g(SCy=&=i}1_E6O`4i=WUPvpb9qw5H=}A+pQYWrv7R+ z$L9q~y|I2_RZe0ztQ(@+HG-&7os@Vz%s}X$ElNJ}|L55Y1+CpeJ$*m(%RZeF z=_$-Wfd$#Z#(H@=cgY7~UaXee?Av3}onl!Ejb+&NN%lx>9M3n%zLD@|IQDkr9llb= zgALr2MC)|s@?IlHP`>{qzr1FE59q%rVTiONyGjK3@HeW|su7j+DxhK6S zVWnF>okd^n$fvJKGb-GpPv?ra(tA-xXwn^9nrnTAJv!?M@=(qYVe6;#$?8vhZI}v` z8B?^#odQZ1kGm#Q{Ug>RYYg8x7M?X1)Epi>s=h?>ve-x|wm-ZVu$w^GrW>}jk^ zwL0(iFA0tJ{J|EyzDz5!HY5Fq!F1m0TkQWRy7PFfo-P33b|MtABq6(m?04q5gHV(&-uG{RJs)#t&Y3gkd#;v+ zah1w>yw8$76-c{`MP^Z&b5Ocuh64@N8*swb;_D%9Y_5}ueslH1KfUfeUv}Z zo=X`xiyAD|Ys|Dhuu}Q@js_y>bm;y;cJ62Z4IQt=**=L7e4a0Y;P)h&6V=Pkdsa@R zv=*R+3&#uU&+Ac(e}}m(^VcHZ4~-OWu%h1ASUTgA8nu?IpigeT;#xj+aOLi;=#twQ zsvn9$TPLKVuIDa-t%q_^L*-ifY9y6|fME5tyQnuw3$-NAqEEXISKm|oNDb|z*^72BkX&mX=N&kgLbF*kq=vA!Yyqn_ zbw8K1rk!Z#rL&I=$8o3iWgM^5K`NRGxY+BM>)T>QpDYSNQx_zoc{MSJ4-@GM<<)58 zl}yetFiRlFG-B0G_i|>(Kk|MKpkMz@;Hv!0&|c5+=;f!kf}#yssJ|^5`NnQRW@4fI z1D-r8kzIwWkKcOJr(3>__ z-|>A@B7J6-$|YJGQ*?a*PLp7`2*=&1;=ycME304AX~Lop551^Vmj|#t6&{h57GX&R|T@}49ZL&LR!LX)R}hxO)lwXZ{^g|zY!tmfoNg% z6^RMx$(uU5YiK5N)_aU(h%)WZI>6iXJ?up%XYTH+Ue14RIa0T?WeaobzXI*6xI+!b zB522eIXli)*0IcFF`Zr{;%Ju{M5iXzaaa2%2?}4Bp$COq(AtmXsB}XFHQ#gs88*D& zwA%l23y&AFbK^p*Z{G|>K0OiinPwbKwy<>!xPBO|2^V+VpCaLS{6s!=JM)lTb43+Z zyX7PAnRck*P6YkzYeen;h_b8HUBGmAKX&N$N~E$<59#T@;$}M>r@_@PIUC-}4jL># z_V0Gk6=E;Y-EW0#Qd>Oz@JpL(^E6}~r+d;|fi;>U>_y4KI#gtqBv|134GE%+sa$O( z9d3KVPFeIr(D%_DZTU~U=GOI@G-K6&=;r$`f{NTEy27jo{hOXj1LADxCAJQs=PQuZ zkZjGd+XqnXr9PUmLaWBe<2DjmaFQB+E=F~qMNrt46ZFUyeXh218gIz8q-jy$5- z$@VC$$qH5ODMnWH)6waMm)w$+*GROzn;RC@Mn=c$xz5|mxXq8gu<^%qS);g66hovP zD=p7+3VdEaZ+9G<9I=n?|7nVTK3PXMEp)BURZC`1PFzE+znat0xfxt(3cpTbmGGU^ z3Y0dGLXU@EqT&S>RG-tWx$;&PrI(5#ELX-o@VdzsH*QC(&PpJ7s)Akdu8>B|RANgK z+Ue-U5}LWu9ofxILyv<;*vPJaN;?YC;i<+bJJt{_=srY8l@gKNjUJ{rJB^&td`t&# z)S!t=Owq0AJ6!PwqndfVHMg#L3F~_watmtTBAKQ0&_LgAbjP(C^~wI?d~(LPTalY- z>UCxIMt&$A>1L{5o)e&D_b>B%m6srY=nA)PaShUUN<)rbd2Cr^JUXI%m!^)k&>XE} ztatEJ5_evPJ(f7Zv1ryR)O7MOGSlrxV%#gP-tj26FZ?VrzMe#hXC_)caUzXpHwkR- zWzpj+_wf7BQZ$m~=*~({WHqKq<^RZXd(}ednsWIX$&Iaay8jd^ zDRG#cayf@wSXC#O_Umx$>28D?9w$?#GJtFCrkvs6QgpQ}5sAoo)51%8 zY0kGv+|)gn1ow7$2po$uXsIqnZC)(<{u`$C&$76jutH8_%T_x6)C=zJaZB3q>L4;d zw;Sz0p@*i8rgM7YpR04bmQthL8m#X9{roykKz1ROT-r2eWZ7L!eQK0z*4PY@RoBYt z^p92apt&<^>+Mh1{Y&F)?&_j#xixgwiA{ora1pvF&9Wxia~FNyGQur7umr`~yV9>K z6VUeJQR=>B916em3B7JGrhXRoG<$a*ikb0%3*BVRndaO_?o#^n#H%%Ee*IAteNVI| z#rZKm_hyn~K#B_rR6R+r8a}3k?q@|qu2XJnFZa%K7kAFupB~&osnpJPoaD9)O>;WJ zj@;ct<8-c|OW&1i-dYqf>Z7;VyiRwjt%%r1wLj_D^e(h|9Yz0AtB`}2gW%ncOnUxg zE-Esys97pDS{+c4h<3kp;i0q&%3QvQcKbc$*6lx!yaolZ{Le9TC^kiqq@_pKYuuwx zcif>{#g@PE=_6*}(Q$KGzsrW1cmLG@X)Nz6pV)#qQ}@7d0><&!;- z<$^=V^ky#o>t@dNd^P5Ftxje4UMZu-iv`?*EgWm{Z67ydwuWP^Rw66^IF3eyUFD*a zzH%a8&hqBnZhE1A3UXOCMWDPe9;KNi(XQ!ntPZNFUZPmbN}e!uJo3bjuD#dCUO#T& z*!})2_t)zH5_$EQ%A4k(1ls{F$LV>sdh-YNctsq2Gu50+_etiSP3T74&l&Vf+y_=P zrWAS0hM-3;ngk%Efz?quMi0%Dp$|-cvd~SLUUR8nH(M^Fb^5vN z);C9y!t(>@)x-hxGg;R0daWC(&rU!|o2GNg^=edd)h$Ni4YX629MaMxQ@Y66W@ z(1L?|1yOS8Xnf!SR{zp{HZ`UQNz9U7c{fVbdXxoO zzO?l7SGuTPi|#T@p>N-=q`|!sj!%-FvR?wm(Z_5K`db)N&Cq>x=}JACAmWG;YmU&_ zcW==Ndk)ih@sR4oTfe#C%R$spdK%mIaUFe?@RX}tJi^9X?c=!N>ugHxQGusw8nW$A zsV-FH=dsM3g$||w8goa7t1bFTrA`{v+)^^9g&*|DMU`nalUH7+-#VYLGmE9!7==dk zJo+F_D^H>yQ%$MP!sB$=iEHem5081PqmhmKa-QA`ji*cTTe{Lf*75!|aq1wY$1Pth zi2}+*>AK=^)K4yumRjFKHg@0H$y0REGv_vTmwysFRo4kMoV$tkrzWAV2ksy*_v`c{ z{7eI~CZp4V@^tFLgJj3FY~+^pocr+^AfKP#s)v79)8g~ZG^ukP`i?NGeUEbbE)VFX zU)D%ra4IRR?xo9+PPJB4AZIyi9O_o-7p%M*$ql}kg)W-KpgqgqBfkekptnsF?VB~l zQTj+Lnq2f$;5&T*oosU(i8fZUTLN~mBHMImdyWm=xyy)+uo9rZzec#r`Wk3*z6^W9 zrv*LOmB=~P%GLyK)*~&+!V7FFvb*o!5_rRxx zI?le0yg#ctY8C~enN~h*dA1`5|IS3$mLyTh+do;!=XYoTKdZ?05#)L#a*=gzF(+rZ zoxNZ)$VFW?q;etu$q zhjb7BHA}q7dh3Iqq0>kyj!D2YgY05)~YhD=dLK7Wb%WH$O-4xq%fdZR%TrE)@CEtFi4gcd)nm z-CF}{sZc-*=jw136K*5BAJs3&>N*=v=YAxkQZnEEZgp|Mee`=$C{kaPMQ^*8aA{?q1SJi&tVZ$}yL?XqYI@eh zT3@t8uD83;3CYFiS?~|~!`Oq(2Ho;9P{g{7!ZHvsM$8BdXN1=qghFxs{z8>QY> zMGfW|v|D;F=e=S-WxGTjV}s+7?Z830h)F|petJ^<5;rb$;s@@^y?fQCm`|*yohGt> zkVGzYW?qtJXPFSXO|A~LBx=^UdYQ0sGQ7KNq3Ia4 zs2LSq&SooC?PUvq1|5lLL;1_HcnDFPj;)tNF5l%@qpnGUq+u~M6gi`6nxJ zHyhQ?l%t(P`ZP^h1O2+$${*RqY~V;Q`|$1KY7gERY&_7yi8O7a`=Z>?e=e~pFAkvWjZ^8`u|I6*^h;F!Uk)uf?L)I#5obC>3z^1xcCX1b_98vH=_O(QrdH71XMsq}zqu`r{?2wQe3Y^WtY5yiDYNy)mqrNsiF2 zBkK`fl_LnX+l=0~r&4*pcG~TBo$WCbqxQOI*sidA+E^-r+Cw$@?{XS_MbjNqKALqvX|G&I>x@dPV3Xek-*5r zk?}uB&vPZ{+>#u&rbzRD?? zr-lP4B{vl@yUw9~dl8!vFpF>1m`E3dWTBhO-V4^`e`ZJd{I1Gm4SMDGflCf^K-wNh zXl}4LTPh<)OCCd{u}G4d{Ype1!9(tWYax3*$(9CgGDJQB?^(Byvg)^w-qYFF<~S}- zDX2c!Z7PV_vy*+jJ_`M{uSH6c^U=?`VstORll9Mt;%cM#Sz^8)sdLm^WD}7< zdSCGuxX?I2n_50_CM9<0&pHvuTGwan^!6hZXl|!P`VRDW9p=QUdwENH9xC77K^N9) zqZiM&qvIasoQdl*E;^H?6Vp>Tr#s3NX#6KQ`6gYEx7`|U*VIKH3*+hDwNYH?t`FQ* z=Q1?mFGRB!gwWx8?@*st7J9NvzGnHdG;Y(orF1amA^WTG0lWS^pA!^CBIzqT(e2CQ zQOoYtth|K;ePFjAwdNbsRXXPEP=pvV>AS_X>FlBDg*%b_salcCY1{<#g#^GZECrrgINFRUJQX7~mgO%%s;^O3?0*hU&$$-=Mc5Iy7XclP$O; z3aesnu{MQmeC>TMiaJHP)AAOG1T3aOI+JVOW@WKz!y1mi!V1`JD-@{ezMyK4&O4~` z(MwdQWybZ7*|1YX57X19x3FOw%BaZ2WRz?2o(p|d&l$KovY{P;g7#$+j)TGfxceSc zQFLuBHC>&=jdY69uhF7tNzoG8bY?qMo}I{t$U64Up{?0XXWfIuAx>fY{;!;}h z$(*ff))PF4S4O{X1tR}zn>nJ9%YKfCMQ8k0(c)R=2qZ72C@>v;Wjg46Eqj>5yNzx? zFVPvZqiMr?7kV_~05<_ophve+>KAE7_pPZ$OS^Z|$gPkYIsOWT1swIb-N?mG+x{bDT%Vrf35@k$1=Km3}F5291(LsdYs0&3V zG`*5uI$?*V*`{z=nR1*+!+lzyG8c^&97c~%x^T}@6RFqN_pIMh6+Opi5>afjol zp_0Z)$n}~d4f-3#&3YxE#hYwV6L*27D^Ig()ALdG!-@1!&|I|mRW(;R<2`zv6iX8> zbg*i>PoUcB(dr}rakOy$N?Lq}pRuZQgLW^~V+)pkW>X$qrNJ}8(Gze4{f)SRjD^#= zzkwp0!gxPAI#1GZMuikBy6p#7ZZ!^#KY4(i_e$1L?DjFb`%DmZ;T%X z;?EA|uB9z<52&oQIchp*$3`0qxyN@7qmb*F=nj9^R^M31PRdf?Lt;bP(3OoI40l(n z@3ld@=9ba<6J@#D5M|a)Bagb@52Mq_3mOxbiBwnIt^V)T1kN^DgFn;wx=3s%J12N0 z+T(s6k&<-uGW!@SbEc2C`|r@=Ybl)4R!q~oKT_GS74**z7sTXGfd}^-M-vh%nA8dv|Y>UAR@hWI>b0JaxxdL8{cY~_+Ke6;7JyOs(nRq9saA|mhP_WAnD(CaP zk}LFtGK8w+JMNo@Tc(A>e{#*JR&RgD%Lb^48^5>=tQl{k#M-voSe zmC29$(zvj6A)FL61V-ECNqg8d9BeNE&pdsPr(2y8l0yYx$? zq=^z0-wNCn&A-R?{VP=b;|izj)`gFXzu+b1o5^FS3SYI)C9vQoR$Dy*%I{DoQUyLx zvR?(JGm7x%>C>29>x*MX#Gx+b-;f)hVyv(4Wp)Y{!k96OUYR$QC8Qu%x9#CgcVvcKHT|OrK)$!u?E}cL~;5y|-#Z z%Q2v{YY6MV*M;hh`o#RmQu5Gb5yXMBNx7mnJQ5WRKRx{m9?2wu?UBV`sp0x6nG*@{ zTu>zW@k1ACD9FQ-ZY@%D`4PDMI-b-Hm=dmjIlQypo_tNU1NlDY_F{ju$WM=FK>Uz6 zxWVkeru#nPr>!>_S+$?oVVD!zC9K4T{heS*{HX9PcL_W-+60SYl1bXbeR#n@I^5>E zm;8CY1zKfLoMo6mN_JisBc80RN= z;^;JWvh08h`JCSe#y4*zCfmZuctM`Sp(IJ-SlTOe=~ZB^x$eT+4@3`Q)MN=(wS+@iYODl19(@9Whw_kfO$0?(SCfTqYvEdf zsBq_#2vR1KF8m=S2iI1u)@52=n9mZ1`f~0+_tO zlT^tuI8*dI)=KFIo1=C*n2l@3BS0NG{m8)9OCAc9rI(PExe~K{1xWxn& zTo%S?&4)+wSK&qcp4E`32g6Ux!8t$&8n^Io@pC&s`huInq}}I)XJ<*0sWmsS>c?2p zWJmG5J{wZGSU}#?Jpz({CXy}VlzC6V7#@$dAb;l=!Tc9pz_NNi3BB?cPqkH0*VgKQt^ApLbzmGtMm^AS>{DtoZ?G^r+YEO!}=Rlq+lb$zQ$dr3w&_8Ms zu@##KUCQP_XqiOzt*gMX*IEEvz?*I6_4sn9B$PfVMYfr+_?40)Oh0Ew=Ictq`SUE{ z-8P2&JjXNQ5&R3fGFSLj#RN)~szGLr8rVNK7(PFY;2EwH=tb)ZAK6WSmb_1rPBnn$ z*>TMA-obw0Rv_H;noO!GU^wId&|a=kd0k< zRpcM6kns`k)1MAk1X+^aH=pqD-_G!slz^C>d|x$f;z@kzkAcIUEvA(v4Lb143uEw~ zloLLhFe*$-n@@@gVuY2Qwq)B65%|r!7hD)@WKzn5u(VF)Ch9 zysfH$#wIB&XK)Z0@9P4AMk;XscNsu}@`cg^7;ihMLcagE3aouC5Wcl?Ba4d?cx!4N zSp^1gxT_q|Iyi?+I_U%?Y_q^X(q>{wg86fD71-W&0SqZE!e$q*F$4ZLL59^YoOCG$ zCkSsce=n8d$#W!t+}-0qMU-OMYXQuOGJ7aKZ#vfhstW_=uw<(F6!GdgNOi z*mwuLbZE!*kGJ4gIpfJ8g{REHuobY*J_v_Ki{p$U3+Ay*O4Xl$&!9Uo1OL(g0`zkm zFzjK3=C@9QsX4{q4{vWgh;ZQx{2Ymy`70=*<| z;*ziVkBE`cXJW3DRl~&<-QW>~ZN*`J+e1=E&U&18c z7#i+TA%;Fp_$%L=y{KU+5m`4A_}5qvn>0mI{JtGG1O~$_Hxan)qbiXqHXw?W{hKrd-(0&|Sv7tcseQf@o_Xg?GG-jXOO)baYw1L?IU*Eau+6{~r%J&U z=U-SSLK4%kZ6Hu%AD&*I3_FeM@cB{^_^_oN3@R9qC(})!P)v;cH}xs@c%%q@4}Qnd zc?v{PTM{2KV2IVtKBnnn2B?XDic4>$K+6x|#K1xV1}SF4m_$?f1Qg@lFN_)0>hbW% zdMP|7HxE>cUIDVaGy7tF5B66Igfb`JgU=@8iK)&^qHxgz&did4zp}K+p~s6z<=Y6T z801RMxVsU!M+PP&nBy%@+2HAF6F9{q9Q@pO7Zd|Mu>9N<-gUfz+rCEP=`;Ufg>PEq zDQ{lC4OfJzw|vPWum6boBu%nSZ5sS3W+%KLbR#2kf`z@;UW1+&q1gGfy0CK2Y*N3R zg-UuCnY-T?5sxYc{N>l>@^){q;-wOJ%KHT^*O5@MDhuD(C`V$AwPCTd2vB*yh+JP0 z%CvSF!(G?^3b%6^!Z$gS;qz%7jL*&>pnO>$-1qzmnp*kuL+KC{FAyQSdz46y`x|^* zQJm!Wio%L%X5h$Me^`~3&-|V|72CrT!ucN)nBRfcMDIh9@aG?65bmUc1s6@>gyjo} zUS9<1Gg(UB>5qdqMSkNIZ4LNm>@obkVNCcS`6ixqz#kvX5ry_zF63*&2Vf(p0xk_K zykh1@qARrt*{luKJoALX@)1DdwIpG*vT#A;KH{HQ23#lU!xzgXK-kqW{QUb7&}Q_^ ze)3%dQdT4m(W5~evQ2|*?C}!rata_ZA6CMRHTQu0)iRttV=+8i;Xzt;dxVnn_rULG zm4W|;CT4QvZy@s0OX#9|4w%lrgfo*yfla9dbev%bN2-Q}9l7(MFW-CkEiVk7t}e#` z|JlP`TnrpP#R}egTm+1+kAlnLh4{l~Rgznp0MXYm;U{$oBKmYANY`3Hq@3HaXX|9x zQpz~Y*dGkaM;{5RS47)*XW20qkI9iKpJv0pGzHSb13@c2N1THI6MBfl3P@kj@cwkngUyjfbaW)(1$ae=>9P9e*} z2%dV%4$fHO_ddAT*8V;1?;#)OKf!gxPa1&YyuPVfm6>;NW)U@R= zTZAG1Ds`cj_X_9`ZAyllFM)osE^t^y6!zQKRelX|hGC0e;@^`iap46C5Toe}#@d$S z)rOA&R@nghsgy8P))$Kwn8V5mk3jLlwM10K6(*kdg$@pH@H>n3*m~d-BY5x80XCEy>0GaarI*tvi{u+J~H3lF59g zme_1tA>T7D1i?E^K~|^)sJw7T_;yZ^(6ap|KAdI(S8RA7gyvJh&fa4{*7GgsxUC82 z1hWv#-vC^ivcS!8`-SI~css9t1@5vmfsEQ&usEm+|87r$cTA(nru_LZi*bOb_;`K6 zA9whJN|4cct4TorZ+yYX8GhxYN%N=yA&V?QR)`f@dF(To{c5Ocs*WdA{_iyw$hgC@ zjY47FTXms}-+SP-=r~i;GnXhjSQ6=np3u9sRXG1~KOWRSj|akP@zGamc(?ump?}gf z{9I=~d3j8p$bAb0?N4K1a^*IX++zv~4nF{U;-BzNl|t2yh{HJ8N)G?Ff}}WS41Dbw z!G^)>Nr=-su-R@YX_O0qR|WmdqR0Q3looArRsOOtKO|2WQep&WFREZ9g0h%v7vo`U zN~fP`yWH0 zt!QRd+skdlyh)x6EN%hUu1+91J%dcy^&oO$?mQUfaRoGQSV<0laE95<3{hAmjdet3 z!+n`7F*^AM6il!dUTgQqedFJN^O5EFt&OQa&n%UAP-w9| zi3oc#g=bTJV92i`-W}NhWOI|ie)mA4;5(nhA6WyJe+Y!NX8J@l_My-)=C_btUm$EL z6ek0C7_ZX%4Ybc)1}%zR*t`Rh$dVqQrUuEh(edQ>w|~IGbPXB&a{!A*UJ@E?v%rx8 z1gWS9*uXoImj7-z#P@kJBAqty_%a_b^X>w&Hd>U#@7YF%DIlly<`7Mv4B!-MjXOj- z@%b;7M0Vf`&UkCRNO2w56fO%iy(Yk|-|V1+izbK*dj)q`zFTAt=Dfb(@NKRfG%X)bCW+03ZM_LV!5|8L zGYBGEUIq$tHU+_u75hnL+(sD1TM(sloJqs)3~ae)Jd7(7A&2eln7AvCaN`F@IBXe0 zsLC{Qx?~M(mo+032hK3+9V&2P*9I8(a3dd6CW7C+OW@qa66E}lQS zS5g!KKKUAK|F;)^JB0DV>|AVM7zNe3Qn94cM`6($-WIz58dr&{37M`|eCW6%43gN2 zkF6VreYZ4$6~TF6+8-;druG(N$9dS+a|mQ^7bU$l@?^bz6rAxW5Py;LXRf?D0y=)i zg1lu}Rc0gyY?!M=J~bc0^1qKV?MKw0$eRdeXLKGYu$>7b`J6|QKi@2~Vqsk&L-M|# z#n$QuBs3+8%r=}49!vD#j$O6*%sbvMc;^Paysb!Ua2W1b#pijQh@9KU+q@B#AmW%f z*`Bu-)Q8VvlK=2|pS%Q-y}_StnlHip*m7pX(2j)uO@+&%l1R!O4_IrH2vcO{!6rLr zqNCb^J8f*?iMmCw=Jz0Oh?636 zO`5k=0JWHj%-in~j4FE`z#Z~rY<#Kk>S7PFew_-1-JQo}FXv|F;pEzgNfd!Be2) zT0MxRd$H#0`ND<&MZ(kt(nQks7_%uqLHH>}mGnN-g3341kKs~KaoQAG*gnP@Vs4~-LnO4_ zIGxY$zhimcW{e1?I9b~dil-@pZ;I-$roM$)zkLq$S&HC`pC7>u-earZw;iY%q>0oH!J?|_~_vN$jr+AW?!6dmgw6fl4?QdUqVN`q+C`~V7*q~Y)U#;OC)4`Ivo z6<||2!EBfb>2oC@FI&X{-&9_0aMG$fN zxe$ihs*uSq2Ei$#MCfu;0>1e-f&k z$Tffkt4xXT)(jjXuMT@Ih!X#EuO05b-j3hxl*E#^{sILVEus)8Mjra!0x=#w*sS9K ze)p!3`Og23^A^m4bv+)W?=rvVmTZP6cpI~}X%o3U-xFVx4uxH&cfj6nQe?Ph8U8bP zjL{fyCL3Clgg#!2fXZxR($M7ykGwG@N?CVs<-Pgv^;vUZW;!Tzn5qW_Cuf4qrzR6; zl^Jle;2CeV8k4^9^I$bUGo-^nhDdFYgV#H(f%}0aLS=V3;GpytT-FaH2>~qZdceRR z-V4b4YFYTPW@eQjZ7yt_It6anEP)SK2jH?OO}NaNcUN4M@mNPRP)hbCo%fos{NJ@? z`He6L#woyP%e6#l%$zuXjfCrBZiA{)UvheqFNtgtfp&?19iAB2g8gnpB#uy0&h0+b+8@{;-$A3 z_TN}f9Bx>WF9V6Nwaf=xl}W${k6JTc2M&Y!P8V=nqzuPq>Jpn(-!?JS79Z?>827)yYET zCn9j4WT}HY-T`gyZ6t>7lkrN~6qqG9n{+mKVvW23d`aAoFf+2i#fCDxT1bk3m;dzl1q<+V8D4StO?b_iLZ08X-0R!D(iYF(i0wlVBkBuS=A32|X` z0+e?*gyOLWz|*^hKt;z^kqYZQl-b-qM1S`-Gh z`or+P0_JYYJ;uit;Ee-s!Tp<_;QP{V0Gn&UmeDhK?)X-mp5g;P7+eR2fAbj+{Uzjb zb|mchI1$eJEFg6^m0)zhIC5~LP`Jj^6zX2yfDf9jW{U4u0)--3QZ{T1Ry?z3&i!5r zPtQvg#uT%7^!F6d->3&XG{S^=yQE-$=1Y9~s1DYuXW@gIM(m^iO!)M^8+LE8u42A7 z+3#6@32Qx{z_fix5tduL#*+o@KyLIWSoKZ;xPFlX%~!t*wa(oFI{VK8bdH6)_KA?f z))5SP-QnHBW1wEuKahg^Rui&uF|gQbrq5%PUJ zWGvf2KYw5OaI?0sUQdR^M#YhA#|)s8DF?UA`GmKP+yL8C-J$uRg>bXaJmOHGOAM+5 zVbts>IN3~^48HLq!|uD`_`=yl?fhh@to{eE4qW2m1@rA9h}#7v5V-OQo>HX*RrkLFYYyB6QEa6{%&0%W zBQJnf!4qNFJURHXq#I{0{Enp-1d_y>G-kFTfb6dAq^o--$ccyp)6-;$a#=i>K6eS3 zrPmLRu6oK`@U|g7HBzu+c@Fro&YRRlyF#~$al~WibC9|F6PEdGLJUU!;LRl_P;%fQ zRy=hK^i=!9l+~t~s<+~6v2tLhDPJ@C!S9(~=kbb8C%7ZvIv{H+@XeJI`I@{GTzB&& zlUxvp?*upFobVbP`(rIhy&4Xkqpjdqn|)-1&SDbMxe798_5t0jxGu1<;t z4?_armFrW8du1;$ua_Z`e*NH=oDHn36+ow&QrxPmgLe)D!Ayf9AP|*jAeY|nWZWBz_l!CBKo@nzuP7Th1)HOb4|NY z@NqGG^F0(VT(wSk&3z`+mriu>Zd!^5<14VGt068D+XM2O)yc?1J$QhsLZ^BUGP&V6 znE7QUyqmR=hzZ=u5`*b**>V8FH_RtS#}5*JW(zzYD^5x*cM`WE2l(UYPh2pT&48KO za1xV+*PjuEE`AU2NS`h?ShN<78;BuFg*L?V$wnyM;s<@~IovB&2x=cLhbLR}gszeA z!TQJ7am9jS=KO`(L}VC2<>ya9NR5E}natp}oLykVYXk@XBm5}(ga4nvLiCz8F=--&3GOA4)hyr2A_|< z!^I0Npzx$VyxlCx*CMBoy&B47=F-J*ZTuimYDojP{SJadC4t!5BnUgl?FGW@2E28u zHoRtg3#^zE28~lf$^QM8P(x}3?8x$l$yWqqfS)-zxOWdU;Ad>)UHOkZSQ?4%onFi5 z2zJoxn+3_ZI0lAqi^C0TH9^mxHK1&y1P824!pA1~+o9yj2y+%GfAq zG2?^qfYt;eUl~F+a__;t8wy$tuAbc?o!ai#FV}sGP|@*v*97=#%H) z=8`JDKIF12#o1n&%1hQ{%{GHp^?lY)u)g_DhT0J39`p{W=Tg6nK;OM|R@Br<%aZ4?}p4+9cTg{uRE=*R>vO z@&Z@$-V1Bb+rg>N`4Q^muJD$F3mDtuMr2*ZVUIC{{4hT}<5V)-{^uMnJ!TEP-39PU zqAZEIXa~!83^CnxpYZ~@|M>RN9?Zv^mQ+^<(=g$$s4%TOGDUEGKF8`ad5a`Cpi?uzhSeF zgv~qbiRnc}XmxcjaA|ZUZpP=qXKPEW;H*Z5q-Vhe+R3DpWm zR7lUplZoCti2L;=SWWr|Q>iXW7K&^nmmgH%*V^~-6`yhBj)ENVy4QvOery9DR1(OX zs#e~xP6hYPrZRn_flzR`3aH1c!jzX=v2OlEQsi(CG`)R=TmNK&N8UcfUwc?68?=g4 zGP|JnspaI1xdqvgq&valh|A`9W6s#-9GH(Qlbj{_FKVLhBZH@?}>) zo|&f!t75I-ZR_VkB^y;dKBSy^p|=aXe{c*Wt{s4ie@=5F_g`@uf4`xLwwtL#JBJ3k zeo@tbfLrK)gnM%Y2DZgwA9oUnTd>Ydc=Y5Ihn`KOz6Us&UAC-eQw-mxszNp#An zKAP`PhMZ5?p|$0OTzirt-_K=9clQ=?S8Llj@9fj;TFdk3Lq|C8ZursSi0zK+BEAZo zWn3Kpd&WBjIls7O^Gfv0e<`aLa*eAr-il-z>>aO-Gef@fa#`1McRCvQqq@j68xIzB zA%E^EtL3|wuDCOvW{Wtn!rdb1f#X?PsAPd6{>RXH2h{k5aXe{ArIH3MEwmF+_ulhd zEs2OCicD#RH1+}T)&%P_-1h#nAE%B zyu@TEsp=%w@#AE7Hw*T@KMl_Qvcj8u&t)%V4? zb!sQDvD8`+eK|*vl&1opEw>0H?+=4kk|yWJ@-Dhpmd`QT8x7aa-K3(=O6ZsG0+bi4 zg`fLUXkkwt85(>|47X=t`TJ(rJJx^}H-_jt<9DRMh>w9rD&=P zlX}lhAnJaAb=%+5yXFak2vZ$h#~fUqry zz~3=f&GR`4M({9DHE@a9l%;;9ZKJ)LE2@e5AtjyIaYQR?RhNXQ>}w*N(KnGs00)Z zr_q}5kJO8`iq4J8Bhy>=l44&4m@Q!pv!@KwU(F}UgVU#obb_N`wRAjQ37Lm;86DO` zhQ%b=sz~IdZeX!v6NhEBfIj=C!)=|C4W`}77#8q|Hv61F_ht+Dv`c{VC+ks_01L8s zcspHSbQJA&J*NwL>qtyX9LvA!BOEFhpv_@ZIF%ob$0rWqKe$d{(hq7FD+vqh`bczq zHkr<@A$h_i5M8kjg^wEqJB#di3-@H8`lk1Su^e^m*=>(qz0$(4UBUG9W^-KVGK}ly zG?BQgJ1}<89(Vh*$>#kQa9paB$c<5Y_P+}taU`D@d?+C53iCJ@GCFA2V||kFhUx7z zbI^F;fxuaM0^VAwz@MY*1>X;3QF>4U#tw6#?}{wVgZ0o6FabZf{eV+XB~VRLlbvZ^ z4Nrfn@N-Hv&{9c+&J$^aLvf3U>XSuu$qsAb?awvrJK-hx-)}8!{G*Mxf_4CpTSXQ# zjLDX#PI&OuDM4s<9jSbNfxP#!$K>TJKvn4inAm>7g?dS79iA-s_xKcCovKB9OMD>Z zXBe!`7EqnTE>IC#0Ru;eiMdz?iaF+z9S>44ap4paN)q@V+?QjI+BsM`5{Ll@mtoGH zY+M@|3(j0Ku1jhQIeXoN+`N1Wj-)t%xFW%PtP{Yr^%gk##*@t{vm)E9LP`Erc>r4tYOx@n zPMw}A*lHjYuv9BxXjKcY+>s5t%r&t(JB667Nx-EaePQFOMey=>H)K1%g?h1Au$ehR zW!9WUGmUYwba)O9Wde8G?il<&?KISnr;?0`ws34lJUUk`;b&}(MXh7Z-|OvWkeHyy zeN^;^1TfFWc;r64WWEH-7i5v=p>J@SS0VZIlEBG57YJ+EkxPT3u}eY*=MCAwbJ;gw zBzJ{$F4q>`n-v2!k9R>(WdR1g$Og&Iz4){8C5ZhKAqVC>fG{UZko#6onv>YVw}mYj zRx|?>ElOd{ab=SCZ4z;As3G=ca$KR>L%M@4MV6|n2zD}k_Q7^1{ARWZ|6(M}8H*t; z$^#H{{18duRS-pHZgAw2E#5ZEXZWf&>2enroKyJ$)=!iM#@mXaA$^Mdze z?1m;LylJ<-gUq;gl5;sZgYaja#ORK9V66IAU{~KmJWhJk^V@Ujvqy*EblDbAcaX#4 zu?x_tT?}>ZIhgz75~P?h)>y-PRAW{#;7l!H@2zvNXZvNrieFdwPp^j}cV9R?C|eJ= zhK>s)S?WUH;bhX+TMpdFBmsZNDx9fy-F%1w#XaP9%~B2o3rSv^}h+< zwwe|RJ)!K>Z9Lf@0i#>xai>-c3bo7Gj;9=;c4Q0GbkByS@i*|$zzw|G(;?!vEIH&n zNyydQj0Z%dXz_%48Xw#R5|_`>p9`%}c~c7P+Vg{|8=Qv@EmsKa=)$rW+p$5Ahf(_Z zc6K3pk7J_w_hv`hjBck-61-R(CybZ z@*goh-(c1y5RJ0s1Oyy|UTFoo{}Z9bQbw?8&qQtt-xiCn9YT#kY4A6dq>i`O!4;Ea z`hG<(E{~QYEuR-a(41tBqlZ3;I=>x99|WWJvS)BmYYK>L=pruHBSDO*GCi5{h?vbj z2eu5)buy`-|CF_%srCnnjC{p0S|kcDi;mI9Z_8+&o{)Xy?J9VnGLtF|_@Kh%TH;u$ z!SvKPU{>l*BBq@IbB6CXahCx!!4()}=c4J-2z1DFqF$pH1X00!T)xTyq;&gWV)c9L zr@vBB#c4A7Ob^CCJ)vmxp&Lva-8ucOTzqkK1C5M}Arg0|LXCt9oCyvST+RJRHvNbt z`yU>p>Hi(Yr47Xx{=1yxK6xTcZ~PAqtsCShD<#AId2Fzw7BI5his?{xQbp-KxUz8` z$G=HWXnH+>v;099z5788I5%IT&K!LhX;+4puA9)6HH?o%tf8M_Z8>c$!)^ECY3{pN zGk`uDB`daOhtxq#p7O> z87hi8@1BvU0x8s+R8AVB7~zwJD0VtOp>;Rfz=iFJ93(k<^0*A*|LjPXMJ5*g zh#-<)(>e2pyYaPh7bjCwADin!1tROTLA5~){&?J^L8+DSGpr4yt)6hSPiUf*tRfV= z=hNkQp6E@o;HpUr;Q0D6GEwpkv?q_@p%cfkX&_B7=U6fdh9+}GBLAc9oATi7%{ubE zSqgOzFMx^O8TgxFzTW*_OYiOC;v@P`>y7>~7R_9`S$ah*Kz7@W;B!I_@ zhuD?>n5dkT=eFMb&T5!3m$P&58cy0DiFc`!pkvsJVSS{+4qXf3i1B?`m$L;zbM#T2 z+26E7)S<#45$r9NqpZ9VcjTfU@>iV0FEh&ENqrKE%shmnD-uz;>nuLpc?j$SqVadz zbDXNaoUHeCq;F#!(eJ7f1gr^$!YAqEUOXG$F1-h13iUW4bT>WQ8;RSVEP$jF%WzxT zXA+vIiZ?RHNqs~*bs%F@>!vNk>q-YBp9RpdH3_c|7STeG!-IMXoNX)(;ji3Yd^0%{ z{<$y4$q%wYx<{1$vW~#{zwgn#&2ypRsuDV7H(;Ge1Q;|wA>Pi>*c)yp)c<}5HeSsY zlt0dcd|3vV;!d%+^drfAwgb)TW};+FKkkZS!RwDR$(0S8uzkfJ)S4y5ynmfweo2b^ zY(N5y#y`+;W={?AT7u|qhqnzxU};J+o|$|AQx;XjSe^mbJ0z7;9vDVgU&O)oUj?p} zafKJ3j}q0QDstgy0mBDehY^pX$w7e^xtr2QC7)b`u;YcGDSC(V#p)#_7wtx=(~(3` ztO8bt-3*kkn=vU?qC#28ArZy*#9T_KMTiuu*n>|jbSEhx39BIp68|ZX?jbiD} z^$y&3WZ4N0C_R}E@AGfc(eEXo&{0N`>x{`UhqaV!-$r&S2jZhIhiT=@B2@D@Nz8fi zU{qI#QN_tHr)@r6GF=f*Kd6KFYqQb+;sKJqtC2iB=Sf~FoW`ojtzgh0Af-$<-dSNi zp5YIpi%U2$=xE@S`Z8*3+)h_14Czmvk$qJx*U!4{_ z-t!()MwH?Amu9^4X+0|LXR&VH3#6CJezWD~UFWpP3<$)RM382U*~C(PKV3KBC784Z zVT$BKd^62}`+Ax#Ote~x-3=2VY35RL!}~oxnIesupDvP-J~lmI6$*9LrSz-+c`W;o ziF%Dan3Cj#$L`61uWtjH`Z@%D{#XtZb2iYP?m%*SRTTd1_rT+4?n3St8{8n`i!Khm zG$}d=XYg8R>uO2Z;}nd!x2;k3;{))^D5AM4szQOg3n;GELi#F`DtQ4`mCM2EIc}KF zGlaOVRA}Fy1gcL;@ZqZe;M|7-7_j^b`69h+<g%J*Nh?OyVr5r_o`yU#wK#LZ>3-< zZwYxcA&pv^t-z!?3FOw@FsdI=2~X)-h%xg;i#HF*a(M~3%rG3M)XPJ+ zaWwVxr^ace(BS!ktnOe@vNn-?Db7LLzw7AM%dgO{*c75f{LtV)Imj+*paq{V;e0b| zn6aP_=FfG2E5dq6E7*mPQ^x7ayW7E>%Yq5By9JBPWw<|Pd?cJ|hVc{~Nh(t}ST*_` z=N!M12+9kCX=Q*twzRP;#y!%Hx>o{x(jrokwB!?m1NFw2gnLCgQ5M%odm9#P=L)Pzx`9$ONc zvL3Qq4PfkO5zE#ZF@16_&gsZN@JNQ-=6qmhO9>Bo)Zx-tH5iip3a?8VA!^Y|Fkd!E zKa|~tGcB6v-ZY!Dl91#0{LWRXD8_V1 z6~3;9oaeVe^3p?+qrVpCT+4y?(NWH(}6+v;;?tQ7|6bjrWub@ zAk9P^VrIYQtGe>o@hSnZLZpj|J#dA!d(&X;;c0aGS$#BknT2%=tvKUr?Xa!*ByEfG z0=?a}tV5P-wycVKU$GtGM*O+-HJBMC`o zq5kb0c*`(r4}4ChW|O~>ubdiU<@=o$OnU<}*&KRx>K#byt!8f+|4RyOClM>5JQmFk zU|%J1f{w~0%=^Mr3pJVljWC~@&M*?nuepV*L!IgR9x z2_*O5B5F_PfS7hA(LUu)e}2;kyHn3;(XHu1#g&DSwb~w>%i3Y}f2zXJM=x;6j3an^ z{t^sY9D-Y(j!@pyjr8ed<{UK8F3_lXLAkZFv1CvOltV7T=jwEPqjCwSSe+qBTwOXX ziGddw7z)WwdDQlyw67qZM!tv=3|hHTt0a9m5&jrez0-*B#3<>~5+&qc2F{sQi0AI_ zgo-~PJek+VsX2d|MusY(M1%lT^qh(2QA2olSb;wJA1=b!G`Cpv}cPK8g#7z-7R9+(IUO~fxWx4A7wV1bYA5_Gd z<4iFgemWrqJ+lQm#UH1o2P@`vseSaC!&2O+dY&H1aibl2 z9mMM1V!Ze@37rftp!WB2x-e-C9JWuy(aaD$)x3rX-Q{5JQ9m9x&jmi~o`Gk9ZM4VL zpX5fL!?)=io`{wBP#>bd0R!TsSgBbfR}dPjn8wZK#F69*T199P=^nVFG5YQxg8xPR2z$ zbII%4J~Yhkr@<1%j0fZs*80WK1#L1Uf4)0y73+kDJtb)OYz7)+nxT})OAM)B3l=N} z^{qMsE9L%D*?|}AqDSJeUEv(gSM7nrf2E-R-y^hl%g2VbDj>#*K#84C!6RxFz+E{S zJ!UPC{Wlw^Umm%4b&Sdhop5Pn4%_wgK{&6JLk7BCNZQ~N)(cAf=LTatX`L3XZ3*=K^>z3!Bu|Jsj-0h`v zqbs50s0B4$R!T>-w8^rQtMR>^IsNA5g{e)o@U@@;JC$tEhu;J%o~dxBywIji{dees z+kNzK{{&EI4xsFUJX&ILf@;dohtbG>K3^{dBHnbv^x1Q8)`a6Y{l_};XQ-QQDh&rM zuX_+XO9`LTQ)p4N65|@0;7FVl*N9=|o=9j#C+E+w!7UGm9iLEhVKUBV&PmDQ)6h1x zk-Q~Rn06xvlNG1ps^9|F>Aa~RayFhf-7FGBpWou|xv&pb4o-nr*R?t2n<@m$9vi}f z5k=R>B#`^K>_Z;=6~y+;feHWh=# zR2fjep+Ga^ccH1M1=XLh8MSU6fjb|9=woZ1MbP_0h z(Z+?dw&O1EX_yuDhZt>L4)^lHP>z+$tZ~3s9c$42c_YL8aVN42m)$h87ILqaqi=*K zMBi0}8`@IbsmzQ-`A#^lvdj|nD}ROMmL;6gUE-80^8hlPt+3G51$I#<+&wRvOxQPo z7gN^@Ibo6LH#e8mC^pk`W|E+_p$M+#RkY4X!FM;UU43b z6b2febq%yF{b2iWC@#K~gQ>F^ zuCdUbW2RTd@fVuW##<3Y9c1gOoXZX(jcg}lq~UgBpxqv@rLPE2z)n9;Bfyh z=dO-1ghMp82}+5d*Jq4cst<9;(m6jrPsD{28u)X*yW-CWKRLs*&XceI8mZm8K=*A9sPsO9^b=9QJ$?(^xhJr7i6&-g&xLUR zYI=K*2buob13gq3u5F4u*|>fH56Jx^k|~mCGvgFkrI_%i`ecwhI>Ass@giKlug(SL zy$kk#4s&KV)2n|DLc!c78u0cFsO(-qme2Giti>sCWu`Me7_ETT&&<7f)Rc@jXHvcu zhhJ4WMnw#&;Mw-Gc=Xc);(0U!{32`6_VO*LRP?}G&kM2aau0ObbK#l20`j%Yg;|e} z!Blt&IsQ|i`oIZ%9qLJz8l1zY4l>(NdM9Ei%DfD0<9TK!1ag*^3 z0#nbE`p#IazI7Nv{hm?jy>|54i`^I{w}@+a`aMT#e*(Xie0>N*7M=oLo+7t@@?wEh)5d6rp zgzcGWu&&o1R@z#^eWft`cRmEBi%OumcNfXCIz(1naK``Em*Ks#pCs?uS#nlY0fMWq zL&v{I3~I%Kcv%Uxu5l4;lSt>>f8BvLr~Al<&^~hW%s8E=z8e>8 zAL6L&GZT*De4HP9lHT0Oyr&~EP}b8=O}$Dj@$fm~S;VkDKQ~jY@)&45 zF_j+vcb@I=s1_rN&XRAb`Vcv1HOfAd1^bM<eJVfP2k=;JxciVf!j2@b>Gcf2yqEdlchkh>yUI*QH=S z%@lKOzCre@r8pm^b4RKS@j5&v>v}Y4verg&v{eqT+)u-*8_VIDg+1A2lmp?6TOs7o zFS@-~$XPD%$7w0MX-J6(&ids^>-EGKUOpG%Bx10r)EVNPqESZ5g%eWn4VE7-g8Ob` z=$7>hYRvxa!0=Q)6yz8nC}n{CccMp(f4Vv`Hv;1>aafTTT=vv z?f^fTZ;de)t8i#Qk#jQp2Hw3ZDa>EC45!{SCING=LC+Ic zl?@2a9P7cui)CP#ZV0bk%8B4u5zcS9LVVxmVCJ=}w68LsOubPd_@cCi7*(IZ zvH6PVtzm=pN#{U5=@>oU{|TDUN)YMrd*sDfA`y7n!~Nz&hKpTJbp3rnPCgqQf5*Xl znE~9$nJhF}QX@Dw90Fid2SzUSp!>NVvoB7^L*>`8I%x`iS|kC*)!(S@NHhLYh{H7r zhiOCL3DmOF#LGt$_?BOa=`}-f!3%3GIvzI-dEMs)U1c{h=)fPMYqJ$1KP6#~$|xp! z?-Q(AHyH}HkDydp9<4akM+}psx$G5|)O>Oy84)kT^FAkGeP|)xc=8Gb6(8BP6+>_~ zsSf^rHvr?IDvY<@L}gT8vdyj

tqe@Hki!#frYeaIht(Pnc_5=ii>7Grt9X&YwbJ-y1>M)N1BTzZO_AUpXsp zZ-u4LQkidx|A^h^9ng3spN_s?0|};XB(+u(jPjyc2nWDdHTIKqn+ z&uG=tDX_iXg}f=Ke~r`g zBZp+_N#nZPY@FPDL2xRcamXo+3pUMrOXB9ff)djVvL~T}C||dw=AsYrPTF?LbJHWm z>Y7k}hf*VK0$~9Q_K%d22|Y33r;rX!w{M|Oy%hJOuo(CY?t*8x7LAEi;<~<9;@CS# z;WX)8xVDC1#WPvVEn7g{tR(Sn&N_P8VGdk6^&g(Uw-+n)TyX8Ln;1IwioTB0#UR~_ zFeUCP#=%dN-T8(5(KCVYrU?4Z$r%1LJ{1c+`u`$I2G(ZoQ0n=u4BRe??mEY8t3|z zNU~+)QheuPhOB>c8E)w=FnDi*teg>!b+Iz{-(P)(F*z5%eijP$-7n$GNi@+!-<4E$ z^BbaghmGT3RD`Q;-rSG*-IKj7_Q=@*PQ)f;^68pgs(T>&~BL;yl5SQ zBl-?_r+bL<+IaBLi}6|NKZSSat8r&S0!`bn4KBVtK}1t4z++)OJ$+Y5^tXn>TaQ#+ za_b;uS;pe#-gOuTb%Bz(hxhR4tqAlBe2e>(xZh7^A=j3O3a+g>HdisaVk3IY@QVl_Ah- z4L-GCe3YqY(b_Qs^RGFJPV_xZ)C99c`J0R664A;zM_wwuJn#&KFn12By*A| z9aRV;x6o)&UpHY-;)JiU3699T=a-_g^M98 zAXzILCDK&kjwbLeWW&g-#XtuuQec^Mo*>(3DPEa<8}mdt(0ZbPPVW9f?F*1|G-4`@ zktn)2{1onN%ZJ^ct|2e@1&)q3lcm2bIW^Nb^j>Q>F+6+%_AZn1A^EJ8O19NMgBcscX)5D2ePf@7WmzGx=8Kf@>1$?AJg$z`?W<{c zLmdrc95nNUnS}o+52(x^I`SqE(%FyDomrRJR!7NPrSn9zSR5`oT_KGD!==|1mJ^3ly0FaL;w zb3qr8J@8r(nAJ+f_ZQm%sw-YUA;cnAV_ z1;CCBS!lkZh)u4K$b)lXB;eUHIJtFHV6cl#;M5=bpl}?%Mb3tYd{rzQdP8hJ=}FbJfEw)(gDg-@U%gFH(xgt-H~@WK6+Aa(B>5@s#O?K~9?rW+q(e1Q>3 zXAbF=KVngh%|j2%edOe84g_xC(b~e7Wb|VWNU+{f3)@&4>tqA}N`GMchfbOsD~d1w zLyT3y{ECI;;WV+g_s_!*%9&iJ{~2DdhK>Q)E@Vg0S|NEBJnk z!2E&JWUtJ5XkC4SL{v@S8edID`7`3eD-L@FV}X-^Jlp|CCtQWIc3jHq)&dNeNbY=n zL~?fN;^#&kY@-Ohi>z>?k2aORKOOuV8*$z|DR7i{N)zrd_r~5Hx=y?dHcjb<>p_*+ z-{eASgL+|ciyQ39JV&nRwBv9~0+njt2ja6=)5iIOq`Xa;tGVtl_b%OFbrD@F!-?`SEgY&oiG}GoRPS^ld^0G-*S0(Gc493-8xOL-r;)e^ z7Sp#$Jha$37xLQYqy4{g#9h*zKCsmkDqb@pl>?#p(vO)@4;j-pfs=64coKPY=mlA) zS<1N(Fj*+0K8_8Se5v}wICS*r!pVA_xWA0T;d)A*D6t^5Jd&>*$uPeS>d5fRQxLCt zff1*w5zR4EfxmJV@;Cf}zf~M&hPfD)eCs1i)V>PX852=VT}l|M9)pb%l{89O7NvZ@ z!qR0)vyJVKS5Pb)Uptt}vh4*Q9gg)lb6IlWnM`c@d4ur19fSCZ8aYC7@j_ zYACaT-2Kbst-jr-O=Ia`Zf^-!s)Or!|6G{F`83TTA9`+e%x*&yb_ln>j`Lm3U;V zhyHml3anjQVMA~sJu~)??D*QlA9*eVCXtCKu5Ak~6DbJS=i#A$h4jW~It^&Z#obIM zEuP6MyZcXq&rP$)zDOT1o4y%EOvT}^O)YhJeTK$7dPuUmq^U{S1+0pg57BG1=!iuW z?D-f&_|kW=w(J{;xY0wq?!SaNYHRS0TPD8RJe?lxn}+Jv7Vx2OHT-Lf27bl_Zccj$ z++i}V$KLORPI8_E5G6Y84-amdO(*+a{({Pv<(wsMN71#^hhO*iED#NHLy568)cLCLf&dgG9YOp zFmZc`AFszlv!EBgyFKREUGgEHbN0f!i$-KcbOCOt?Z5?s_gLdpPCw81LHiG$KyBdz z7^yi3A5cLkHG*u!9LQLFj*h0SLU)cH$F24}Ily?tOy!a|Ij3%c!gf7;($2+dlfUTs z{3LYET7zwGh4}b!67@FP0$yo~NZgC*kC&Nnbk|DZQ04+MyV{#Jgl&Qk(r2KmA_}Xb zZ_$!TF|hRS63na?gQ)XfX#KU5?znuG7!^cgeZ~xY7!k&HeeMrGpEbaQ>`1m+?tZ%5 zv4m*wC*qCo4fMyaWYkua<*>)>U@qg=d$c@zg%fMWir*W5B$v@?tR(mb@@6%8ENoN;k?GnOE z$0O{u12d@8ltj>KV%!3nCJ)#5PXt@-j%~a`E@$Rk*q15bnO$N{^n3gWSKfgp--P zJ@NYukDu4j%ZwjZtz$JF2)KvRPfn6@xW(l3pK}C#reGcz&*6^T6li^Sq0bgaz~Z9$ z(BOTJKHqK+3SKAalx#opRkw=W|6CC@lrG~b_iC66)`aDF3Pv(7L)4~P7%9ELN$eJ4 zP{dBuJ?V_iMQ1=NA&*X)ctOxuQ;o&ljo>*t4|RU_LT~#-5?85*-{Q97$N@JPX;j6> z7aC~rFHQJ#+X z;UrzH0GD2v%FX=q15H)GLp>pM$FmeVi|vN{+wCDnqZs{jW^hM0O@Ni|RaANY3J93J z1T0O$!KOt9{WcszM-35LxS^5CoONO!dvcgY?@Yr_!-XmcO`~SoS#n0$1K4(r3of!p2W50U7QGs3VQS02k-%Z%KNyKZn@J8 zb1$CYzxnZzFS&6xSj0^fdUezbW*%J)YGGeANNrEw)13Cl{2`kIYcVI z+2N<&)p$ZZkFM7@MC+U|6109Fn(gsoz9UDFzN1XOX=k{g?wu!|F7?H%9gom?`vpko zRKkGm0@!E5#%BxEh2_WHm|UtEd~^|_*0(*_w$Kt4WTPQuO%(k#-~s+Mleko(iw3(! zfpM-awwQgzuM6H0Cs#Gtw%HL+Ce~5epi|fo_ec=i+=i{S2Cz@o68%HQ`S;3x@b^22 zK(LetzO{Z2fA6NjF4q_s@VY^ZzB>wRYsN{cdm51q?*-$yiQJW+bD`8_F`64{QhjAM zcHUaYuer+PuE+>IZBop6op+Y-thd6KtncKiNEnUnxPuMLYUp+C?Qo+>m$M?j5Wo7W)2peT^(M#^vE--}~rizZ#sbZ9{dnPB_vQ2WAGT^j%UCJ<{p{ zht(GV-~Tt??RP7FJfqHJVrAINi^n+I3*MnZw+7_8W|EiZI>_oSCzosGK5pl59746mgxN?22TW~0oUpP?zBBjZJi7S ze$YynC|ZNT;BC0wlMc1aC;1G6c)|DFtDJ3SEZVT)63M;TNx#j$54lnQLCcFm&JGX8 zUm9SCOSDQzZsB8$8|^1IbMBLs%pN?$p$wHKUtxPHTq7=q=Rx;<7iI7E2I6)X7Rkla zoaeE)U#=8BCSQRVldtr>(lD9f;EAJEQ}C|SCuot{gAK9bI4r70TX{L;-JM$EYox|q zRq+6B{T&fVsz{MvOJZQ&;C)VsMHxOFSxqXriqOA2ls<7+rp4ud$)MV6wrjy5oV_EL zYQC|ANxj-2-)jT$3+~X2UPbi#cn2>(pA5NGr)jg~PkM3U1J1#PMf6T16~{ z)q^V_L6pY{;KtRJI#0yd+7c8l&LX!P$xBwllLKKK{`M){TWjR-yP_?zwY|*$cc=z@cMpO==1w?YZ~_ci4?!T)4*k+S zxVU8kT8?SLfLJb>H~j-g?t22QT&9T<(oc!e*ftD*lTD?}GC5B*R>1RjUL-Kh5p*s8 zgWAz%!NIMgP`$?!zb$a4I#uGpGChT2hH?;pPzfq@Cvz8S6wL_#d~{bmPaAD-R;9& zp{DH8%xYZuw;kiAF}Y3A24o%dfo=9z=${5FG}@a7M}I1Fef*iMOcn8Roes@^xdun?TqBo!KZ5T+1=Mp);Vi=_ zuuqM~16!}anQ6-~&OH{^8tVx!dG@lbr!sedKNBGSmeXHSRyi>Ki+=Uj3nKnhMIKIbUT*+4^5@6nYdRyfI~95gl>K=F7Q z8cuL0*%`mdyA9Hi@p2YyIWvx-LtTU~kwtz-Ny?^isa38r<&^N5|k+v{!W`zpfz}sGmhXTq%b?M{{tK$V)-z*9cm(uK@nby9fqn zE?~N55WeoeKVZrQn({uIh_TMAYaZ`~2f`g{bH53R!{n`T_1Ivr;@W(t81ADM;qzkJAP{x3mR%yS5y*@$M9 zy$~{A$m#U#CBN&-aIdX`u;}I)8e7T%ftLZd`b;KKoUM*Bxi_HPOB>?~_LA^3CU|;K zjB%3MVXCMlUgak;qtKcXoP;ZPCcw6;Vz8~2Ca263pybX7abKB7)NZ6eoL-h7f0Yr(+H(pw zJD>zxi@eL;V=?X9Tx#PC5R{XO$9lnJ7fU@rn>|~cg|6g~!yhjTHHtIm= z8H#_Wejo?ZwZZ858!R%Pi|2p2vQ7PV)9E+8>0VP&?v$iXVtLLBZ{05<%P$4u@AtP* zG5G}v2`Rzc$76zZA#Nn|tN|`{sRN&xeB358NJV#OERrug4*9_Dn zk+4{06FFYYcoR2_2>9mSsNAapjU{=c^VTd_FPV)YyUIBE`R)+#sh`eCse;HRO7;mP zP&j)jO+EY=<@=0bmQOYf|80*MM@!LWi#l^pnGH_gXX9kA7BV6dg}xkF>OM7t`nW8h zsXFacY1L}}30KBTtFD00R~Vqvona~>+z5eomGJ9(1LHOJf&c^N&i?P5;EkCRaSfUy zw9#l`oTg7Hd#D9A-}ZqYGRH~iK1~uU#zzsAD583O3V58JfS%DQ)G_2Dx)@!f4=T^0 z?V)e*t)m%U%?%`jjI&Gobs&EHI7-xf^|>LA1y9<5c|Fw-J=Br9L8`< z%@WWUx=j=-OJUikQI1XSEIQS$16w(tsjml{G6_k#BsP#Ke1zqtAPHG8Fud(>U;njOp&ij&&-;T<;{9Sue(K7pjMbc^qZ6i+dyXC zHM%NQ1HAV~!_GlNoT_w-TxvQa;H)@LOlq7VF2)w_f7}9rpLT-Ht4qY{dq2tgZvjXL zjT3f^8AiA-BlP+!vQ+dMQEDp?m`-rTdnQarS;}9qN2L&Q?M_06lp^K?d;p6rD%_F2 zJm6exC*dBa$;G1|$jF2WoU|d9ya^D)r_vobOX&}&^lSr@h!0fgor^mi8E9*}IQj+Y z!r{+Ph(Yua<8bcB+W1V^F-t_aRi+QZ8O7wi%nGp7k0+ZLAKK|QHQ~}(#n|^#gG*L; zVsM2b+Dk0NTQA>%W5;dUGAP6=sZ(I{{(A&3{bRbl!5E?SOQ1Z@2#Y%YLqnqyvWW4C zo{aB?uq%(Kb6o{G{>p+2?>o?c!4r%*EKOcKZ2)NtPyUnPDl+leIr2t~Lc|V52#xzk z${(Jh$>HMIwlN6qiA7`1^-_>@%Ag{Pt*U?j)@CEGX%Opw8+ zl`6P=^r%2L`5;RCy2)8FQ~?2p*TPo8TJj-q7BSji3^T{Skkuv~u0Ny|14mmc*+T)8M~?MAk_26RY$2Mb^unx3j3)YSv)O zJYLMUFFd+kg@5S3c9uku9#0=E+3X!Ltm9>`d9V6oSPAm4tnRJ1udlIGv9>E-X+7&= z0xNlo2HPjzn73qu4zGTr@51f&&O8rKeb(jS;|q(+SFrA%NnjbLUgOEQXgt3b9?RdtF+hFJ6KfDL;vRQj`_VK28XRGHeGM3+i&@r*`(jy%5v%G&B%@ijmqzMm4*%v^ynn$nZ0=@l`TM)R zE=!74+ug!CR#(Of?U&>CX8f*~zff;&w?vkA)cS>Wo68BF(M?%?d5#^cf0u=I3&UpD zZGOh8>NVkc?{#CX1!vyN--*0Djc4`ydtcU{oN|rVxq4H5QeL%nk-HwNL+u{xg0Cdo zE=8U%9(CRN^<{=J8{f=&>*ByOURA;Jmj&LQdkH)fu?p*>eJ;FH5_LQs_{tM}xmPdh zuExt%e8c-O>o0F=$2aSj*F#vtD@}M`ate43hoY>%>`$wk5%0}=obiO<+j>boL-(Bw3r)-ws-@ST_^|{!H6(_IA^8HVV*Sg|4%YaVcxf=^v zLsdLhuhv9fKEsa|iskXXHmdRblS^40*$%Ao8Oy92f1Kj|PtlpjQ}w)YJo_%PFCqKB zggA5V8R=UQDx^eYjnW>ay)0QHDW&XtD23vlIrof;qC!dvl@je)T4>kr{{FjvpV!RX zXP#$1@AuHPIB+U(4WC8U3s&+SOqY}*&ev#~I`kZAb`h$e{z6C*Vlg1!UlCfLgy$^b#dZ+uwM_Wz81@ z>z8Is{ICimeS}B&{JKEc?|Oo6{;Eb_-mC`KR^6fLuYz>!z?tUlvtqeFe?}rbl#P`3 ze5CW1sR8wgHQ}+55|MHJbSR-o7O$L5Y76t%a^q{FedsG<$F3rcnwzM# zY#DgdX3FgK)M2`8@1U4Nj~%nq+|l`2kHE~uSA>z(Je0Qk37F2XCtD9zfPLv2NbjO0 zDHN?qvdO=|2hPD8_T6Ai3|X zAXEH%K65BF7j@{KBxL@6L7@(}(Q(=D+_H&XsCG>{h*^FMX!ZXlW`0c24tlS_+FPPz z_I5tbfA|bKlXwH&A2eka?P@?#{5}1~uQD0ys$^518-5t z+%fQebT*o+^ouA<&}H^L`V78CSrdu_liZB>dFb@a3Eyx)`&1Q1?-hlfBufWHc zBJk=#AUeERk8D4U>Qy`X0BkBZwMGArO};5kBF_;_aHwz{x_kg&?wCX<tCin2y zWalC>^mZYd^NvQ>fh6jA@C~hr@&W_)Q%E3~?+A&}V-~#61s8LQ5e~Y9ic%`*21N%{ zR(1mwH{9lPV>^-Zie&Il`X$(!QVOzx66v>%|NRt`(ZajdL>=EDXWrI;ekEAbZ85Dx z>)YpG?T%hF@|{9e%^C@26cWKHXHg%o6e(3Vp$3(0AT`vO;VRoPww*o@iWX~jGl6C}fq3Xne*i;}};Z9o)MqJ^0@jM{8f=C3NjfV~#r z_1YZztyD5NY~D^!zhb}@tsBUBNjOq!)K+MvAVU3hKetdeWp9=E%hNqx&M)AnC3n$x(E8tIU=V$ z_GtH5JeZP{V&t`ok=&=b%%d~2nLTGs2-yW?w0WNn5qPi=y|)qrvyW^}j1aWpx52U+}(VAjPnq6Mc9qkFG~ znK&P5=6%KubpGcvwC7wUdh}}}IyG{eKB^sthNkSmpH;@7zC;6QMGb-*CNvT9T@I9V zUPWJ%GQm3OJaB;(B%2cA!DAI=w0regAammo$T+*5W`y?Y!#%HW*F4hhF`A3yv-nCG$gN z$d3kffIO-XwiXzXQp+^RubaPvm6}JuvppE3X1@a4t-FZ2v;H8)I0|g?7XqfA5Wu~7|a+-aa6@7WBV2Oj~)?(L&% zPcQ^GCW`i!dY^1;{p^Z;D4l>cZz|Fl8jwhKql-d!5 z=J|-Ahe3j1-fVv`Tj>J${3jknpbjLiSBg?<64678Mq=4eEZxU{9pHO?De!lZK`VY2 zp(Sm$ogo9K=M zD#(Ax4g?3LqDzDivY%N7?oT!YJ!wgFSxyEN9yen8rtF!Ok_hPOSD{2tJ=%6|8@>9m zK8oJ`moSVtk1QmfqkDXZCus8vOB{5^JBbVXucPOm3&10ZEufyy zG(UgmKG#Vh6&$>K8VE-`0&e1C4s)RZaQY_VYWUh^tgP@nOsuq`HsLV-+f7?CXy}c)b#>5HuIO z8umx$p3Fz<{&|8B#iQuj*dyBgr!gwvjuQ7>KhiORJ@nmZc{;y;H^`k|gu-Xfb99S* zz=b|7D90oSD6|xy(-nir|N2*;e{2@>-`}^OdG2-KsC=59S7Zo|XQ+U;d!)e!-WA&2 zo*~W{t8iEUCrao$J_iqOmZBHFwuIa%X?nU!2!-r%1$VVW(6g0x;M6-c;>Zv{HCZ{1 z*H70Gd;XiIKb0wglI|{WBK1CCyY-3Z{_lu~X?xJmwby8-yq;ce*gyvf?Q;CsD}+qc zwi5Yum5$}DhQty933AR&hFc_kk&p_wO`P-K5+7bzfq%~=P*HFL8u)aBUq_hJt_EJ{ z<7ZRQd?^Jq`F zq|n~C4Ult1C9y(lGg`ZT8R$!j1L0G?V6t~HdR`g`*pL^f{?=8rL9mtXwF{x0LX61b zK?_pfwiZ3}>;kr4wJ6N23@I#k1e;eHp-WZb=+N)&$Tmj?$ZlPOW+q-Cosv#;%Rjcc zD)uB&Tp0{@*mnco)$c_9(rRK9n$V3hKZ$U$gY?Lq4npdW9@?;uZHknhLku}TrK8(y zne`?L%xQ^qpf_*<$xA;Ws<}zP%ZzobiePCM$pfBzOi;rywVd-X|Wlsp| zxY&zALQ~Mw0Wb9E#|`3Ii6pjSJQVP`>Uqdo{r%g@9c{G{1jE*}Z2|DV-X~WFVbVpzleQcH`irk|> z!%qOXS#p8(EJflD`^VARE1xE3Rns|A*HB0EWmFKXNECb8GEb7!8A;1ZhZ)OaT02LU zarwl*^XJUtbCxHGYjU}u!@Cm%sc!-yhnAvFLJ=&LQUL44&Vl;uc+@%}$uxQ{XI^<< zrH@pQpmUA}(0I6x82ovO=wUA)%K93zELqOX`hmGx1xdt`koHKDRPC;zoo=208X9+qyQu#I4tI$*=V)Yu#u2&;JI{yGSFPoc?&Y+~+ZFU&$zD*s{Vuq|o1%?ZN;qVOH=?97jtt!v5AH3UjrtnrkcpZHXo)T- z)OhDIIQLV4@h;gwSD5TZ=HL>z)O{RvUffFmJ!4H*FomEyVJ+CaEfv(4tALTxpIps1 z_lbm15*#a0KyxpD2La@H^dH~%>b(6MEzYmSzuBJ!Yr!KhF?ksXt`gjyy8VwE_UzqASc;=7Xeq=bSs;$`66$El?NX3oq=|m zNytlkHe>cC8?E-Z~Dxq{Aa=#P&B%1sOc1rch<6Qp)WCi5%fNJO9}4PwkGi&Hq9h+F z#=uC7wB&nQ36UA%yZ$k-ZK*u+7c50Ev5MX?d_kF0Zl*IAg8Z&W!#Q-r|AFX{(fP%WNX3s^f zL=OMHY~XeuM1`x9r{HmL;8q)wOy{2^?;_Evn^w&272@PngbP|AvmD(?c!j*4UIqJt zW;4|zpHaz^(`aH>57;*N6aC!!5glqSq_gf?0@=_eqB3u?+2O-C0Qvow%A;M5r%TR( z{Kyfol>UaaSE!NyIN!jmP14}qqPxUK;Yy?+UWBSc-f~@zP7vRd&k;ePTZzLTo+CEv z9#Gb@2BK%vfy{FwZeM{2^WgVAl%|)9jCUN2bA_jTxv`T%Jtl zJB;78j)3k{n#}GdWfIB$2I(=`;AX`naz3Ah%9wTN% zevvMEc@xbRe}iUXS0FDH334jx8A{sv1bz9gM|Td~BBXYo0%^}A!1%Nt;d`PDz0f+% ztu5UI{5KvWtSx)dOY^%xuVp!y$&29oDJdk_um*&#_5-=l6F7cqMfZM&0xjpc#0fz^ z;@67LfI4vrJnY>7I4canmd+PsrMDFPgU7k7UG*2pcDRTIuC8@J%2P`PYA?XR6Wmtb&7KYciL6r9{^ z2y{5J(L$YgXo7LRjW`+4hMP0*P)Xi4H%g%-oWwE8YFN`n0Z#5hvax!Nchhb$o=^T)TACJ^gcLo z6XSVAs@(`t|6K_^POqTFeKrF3)a$gKxe*=Jy%gQq(L%g2Sp^Kw96(9JzQFF-F0^>w zJD}@(8SE>v25mo^i4~hRGQuXx49(-8NzJ*$rGWrG-`$!i2K8tX90!JdkjxenB~LXz z29jzuh$|XKE(re(7I#aNIg7TV_WH}nC{g}c`rkW3!GMzFEfwiu2Cj`OxprovCSxqD^JEwXpwKN zkAsXmT1;_l2)Rt=6wujGK)fC*cijKom84UZ$PM>P!J~jL^qcHmq;SIuGE2f86xW%9 ztm+8n++Q2!YLpMk)Js5}Lwe-P6W@uJkS-$WoI22$nFLZP^T0BU281TaIa6dKKBMIcmhX zDOnJP<|W8%^cTHV-iAC9^cP4f=#i`a4bh~F0@HGWi&UM}(RhD6H!5riP_C9Fe|T>t z$lU{Ai?0qjqH_Uk`R4{g^3s6dt{^7ac>+0|S&se-nsA6*DMsckw;&C!!~nnOcr@#_ zD`Ts~b3C7_iw<1lFfyiIq~E1)z*)Ki*y_yzfng))>Pi>%>UlHrGspzVx}Sh$CWr9( zIYA8G2?waJ8a%821m0O>5g+(7D=+FMXuQ@1e%5A!)+?6~=Y=9uMSVwddU1r*ha6P$ zz=8R<;SY+sSB$oRPma4)kJ5ET<8I;KqF zjgu(ij}p1C&54On`3Zbf^vTl$wP=}zEoq?i0th+R0%OO0=<0NHOU38>R)0Cvc|=@6&aBC$8EnO|+up z2HLVmpD}S!VRrgTF=b0Wq2nn^jF|Tv=J0}Rw5eVdy6Pp%l*G zbk>0i@cnB&eWPj>qrP%Ene}Q4T%lWm{M7(t)<}_sSE`5}jjzB&Wfoa}G#IV^Ap-bLrisNV!$5P77IU!5oDmcX1!p(hI4$R)fgW%Hdz%CvD)1CrQ(5y;^ ziBTl|A{T|L34&c2isS+NOF+PHG0=UOjt>7DMyvJ-Gxn=Y!KbM@hmN3l;F_I8INrZT zI9Wto4pc^KfwOLWxBM|1GURg)>fWPACv83hIsyhk*NKy;fYe9- zK9!(FPUn!eZ9o0f>;X~y<36(M$sy`prit@M^cl~!f6(viZ^4rcYra#Pe-9CwL_U=j z=)s^B&HKETD7fnmViP0KZNCfjp}G3tbVxe#FYcy)7EBP`ZX40Bykz=ZP$H`SGD9cC zd4uaR>SU9GA?fk>A3Zf~MaYNrfxiy`c|^Dqea}4#W}G!Z&0#yTOQapW?aT%4T@@hQ zL>V1v_zYSE3Xv3d7-=k#U@GSo5@jF4n%Sx>kp80sENK}>!%KGoiQ1(gB$Gu>x8s4Q zv>dr2+k<>xX2aa_`Um2>M95*Aeo(wuiOCqTCOsls(5)s>M%^`p96BPyybt?C2MXT= z?QadilBHQFQ7@cHFz{qPIBAfFW+^czRBm$Xu0Em(+Cb}FangM21Tg&f9_ib?1YiFpf}mD0vWoN3F)l}iG-20}i?(|(-|oeMjid$| ztdwG|8&w18p$=L~VjNZeHetNOOiBOld+5uLPVhn0fmDmg;95?}qwqWHNir>zy#GFo z?}1)R=GMD1k7`#iokv4Rs>zWF@JvN_TA!ff))&BfgWt4J<7WDT{|&VE&V9#5{(m1@ z5zK@i3}eWUK+?IznTbE<&Io@BW@di+lZGO0jQ7Y{bhCURQ>0QuYc!Vv?>m6Wt>r(s zC^RM))y!p>pC9N8?s8z?$~v?x=R9Z*`c7XtR!6KHSpzyu_n}}~jajH|$x!@Rp2qb= zdT;jt`kkW9%#s@b5}Y%{Q2r#C;Cto_he&4V5#P}=CJHXye}qmd?+5L-Ye7o+A`(4P z;yZO3key6F@n7&&x(I9!Zs;Dg8dU@Mr@mwQ>!!B}Wvr=A|H=@9AhokPdBb zt;lE{wk2P?`;ea>I5TyVRbU+-bH`+x zogmnz9QcQljBJ(w=^>a&m~|fqQnL=AHnN;Z_6{T8d;5{gv!WQKOk1*VhXr%MKbVPp zZ%pd`T!O?TI??7rzDtxp^Zk9h8_hf?(cW%p=I}OI()1aZK4!Fpv{Q2@(>AVPhFk1N z#Y-h%WdxV_IX#No`V@519a+krDMSQV$>QQMsApKVvgyoB@1#r zNV}`v41M)Gh#0nHL>1i_v6p+n4OLT`I%vXB>m(V&CRwt*IUm{GlqJ7BGa}Q_Fq%Fi z1k$S#(b{bx$nHWKUAH&|Ssncbj_VBpx_mA$G*V{5cM35|KkS%o{27Cc*b5Z=)sOk% z8_2Lm*Fad;D>Sgwky(7snz8*Fj1F8}P3*0}=)12Hvp>^=c~|xZBwFug?gT|KQw?Ec zgHJdkca)3jRg4(=d?C`ZJ_CN8okQ*(n@?`|Y{lFw7ABJppW(Y6F$&#yo={cY0DLcM zk|*Drl5Th%lfd6&H`cyGH|~$4NPeH9N?DT$O*11m7m1LxnmNn8Cwz*@k0zKYv4rLxZ0kTsgm_s;(8E9Wkj>&P+VqXq<>w^LLW0e&nVD3O{ zPg7*5`u`a3|13zq$q_Umx0(Dm?nNr_os$_)2(nmx2@{=2FuUA*Nxkz+8MBRtQJ>UJ z`cSVoYMCDi%xkOBb1;OYOy-lKh91nt%hTX5-yt@4LYq-^W9cpHJV@af4!O%?1>^j4 zE=k?fC$l5XNLP_HjL9WCa$%Y}Sg~KgVJPl9F^hjEob-tYPYbqyOMzNwX-E$8E1aU0 zPG3gf3ll-{>fLnD+Et9;Rx{?VwksK(X2>)M07YYNm1cs6uX$AE#@;~M}|=7fIN9IN0fOxM}t|FHiz-h)+SwDq?ptM zDbnw*4%wCF%GmswONNZulfRCul3hI}%sKbNU_?KSm_C+AA13V4>x)x>Su8*(2x5s1#bwXAFqaibW__b8IhE0c0!I<5-9C-lKJ)`qv4yC| zTc3RBEkb4l*)VoroXHIq)};JmYjU$C7u~yR0RqSPKF6{E#`gIRa@GX{W{KWHa#Mjf zxwvcxv-__XxnNL;snVN48Lk%OzaBT{`RX+Er!WuMjR}w|FAkzTy7S4q3oXcD7cVAd zlLxubR)&e%I))xqSrA|g3%<(CCEsr`ATQ-fFhKSuctqa`S1^ih|5?nu z|BqzyUj&iX1523vs}wo^peflz|3&o+)KRq$VA3UY88S+j4ERxpuB%R-Jmk-Cd$_M)%2#~4C-*& zjIvC_!G;yL>87_SAfhb~j168u(VjZQ;oX*G*^3!azFr&skkSWp`L*cX@M*N+s}{L# zX&TzMH2^;vItTAwQ>E0KET9yp1wJ|~i{+1WGmbZ;;S1$z+?4khdw!F`PujDgK-@j} zZu3Q$>4jk3zE*5sv;ZITW1+x~hwOuRS&m$jh?9cVQ#QI!8e0Pd|Mb9LVQWa2v(f+;-n7|ltQ>L-u)yUYpi>R zb<^G0KkplPProW*>!RP(-@S|Q{Vy_5(RofwQ^q;`sd6D!RhrWxp~0p8DeLmuycFRM z^}leMN@edRDByc%VyPvM0RE{{4)4sZ#OLEFIMICxsst$VH2crN-Ved>aiAVY)nE#? zA6yON_Vq&`JIbdjt5R161mR`}L|MHr<~g*@W23tZDf1E`SR75_ckA!*-|E-k!{IBb z(8*dXr;!iG7xwaIiwtq3zuscxi`DSZ;S;zxG!NJQ%7mI^Bc58l7$*tOh0?WQc($k` z^irP)?Eq)V{L%n=@fZX9wA4{}i^dkh6_x!`X&q&|*#% z8&LX;`j%G=hr+9HCuvVDJm$*&EMABsLiF+8!Aq26v1)#>kRZ0XCti@sZ+)mc(qn$c$g03yyIydN+l)8&Uu5@t?}vZF{i&o~CeYq@6DqR(T> z!=TMS$reW;U8kq|V))UsY3TQ#AN+JQh2jNn!^dJPsjFYo;nQ$OxNN5YtB@y(llVNv z+((z$UFWw_9fc36v#Pyt&W2Sm@{tQZ5~kq9`~4dBM{2-yJ-(wSKMuYq_=7u(<2Z}y zwb0y64l52s!B5K#;jq$seD?i5Y_y-Ggl1o1Q=S~-JfFkGiXuYLpY_A(3FU0>`$+7f z+{#|wa~3|~(QvKvPo8;4FV@3MTmkhtU-Tdr{yT8$FrW|*?c~uvEvT7R__m$ zli!6uavT>ojNrg)m2fiS6)UmQ1gk8vg38X-?CD2t zY{}WH@Br-Q_43Zb_#%B)P&JdX?TmyMD*a(k)OPBmQ81n=&W8RDG^MRXusyjwRO|5+ zc&afBM(^up`~++hXrp3-X}nZQ2v>c4 z$UC}76RwS!-?Grz4DQmc;k>$E4vx1Q!}Bg{@cp_@Hia8SXB!jf^wg@T8 zUd)|W!=z!4k)`b9FB5nxxB~kss=*}@Z>R^COL6L7U-+!@AWq89r{3nu!Hd$*IHj$f z6gOrbjFXXZn&HRVho*#JO)pCoek`Eo?azht>qAMgCWV#*B9_oit`t^eia8ChyaA8& zEXIL-VQ}MdA#C${Ewrr7~eCE&&-tWGp@J^8+ho@%A*4ZAXv=%qB!~J$}Y=p%A zW(AZRX7ScVIn*_wk8Ec0b~v=>CaW%S6Ye-C2$f=2;edv-@UevyKDqxXZVRh|rAN-g zNOw6*_-jy!Q)jSx{86}W<1rX?gQ8AL4Zt^Z!&ujz2A*!w!l&oHu8yByyRsJPd;N!iypu&!R_o$Iq{anT7=K-S%lBz-GX8K zV9ok14e(N!iW9Xy4$2}&*fi%gb$TDeN$lR=GJEO}e(R!(&;Mm8$FXcm)~%7c-G2=h zB;ACcOV#;}@uhIXcoqB6^B^0rUmVvx&cYk-3}fHXC-_?Xd6@09kMHN;vqPURa2hzm z&mAyL6!)bvFVnV_XPF^Ig+^~(jR={4ATsTGu+UrkwlTnG0B53r4z zSKw&$1T|=}l~t&IgS)yH;C?r~min}0j53DcX?1xj|L7GclI{z2!)K`cR^b-MbX}^L zrwSt{%&^)ierCP@KfImY1_j*DP-lV)v5&(PR_~X@-HB38qPg81lN-O-4fjt$IjwV4 zVS|Fx{`YRM|6LO$=zWSBdHxWODWzeb93R+yqZEtpt;5cqZg}WFW%$;A8AZmgqfDA) z*%yZnv4j8V;`V?7T=--&Y~7@Tr-YPoj^k^1U2zzH=lI~vq9nM=mP>kt^-!W34b+7* z%kk{5P?!sziU-paCh?0*yEl+#s5xbLDBbW=0MtAD=a zrCRM~FV9xxX%sDiFU$T>`mt&7mT)XCUDSlREpgOJJB60&Upt^r(Mq^kQ>jHVBY-v9 z)iS5F56ypiIiDk@2V_s~W8Zzo2fadW8c{f8C5ltKQ+9Cjw68 zp1ydm8UOd#Wi~2UyhW?t4VxKfW1j`@;H2_gzFT7xpY?AHN4`orJ@*R3li!Q*HtS

jKy7=s}<5=>) zOKkK+9zI_=8=4C9*Z}!uEp2;cv8CZ3wsSZYH^DdHUEE9f^(f2EWXZO~9Z=#&yZO$u zm~*)ElM34d_$jN8$T5(;IY)ji|M8&+9 z#nU52&>^Rg^?TR`KVLWJd#v1GeM%O!?7Dzc%fSTdpx;+a{IkKpwwW4fl4;S~q1|$K zTM6e^eGFEKx<%z1O2XEyb6R4&?_rL_2{x-nA8Q@hg~th*7MmB1>=rvrE$hs~BeIas zcQa!D{_@1rNC8p`Vz4jv5|w|(8;<^7gKgIxgUQDKV9`Mrs(o}04)3wYt~I&XPWmo9 z@aHLH_}sdWI<@SpogJ+5{_UKW<0siw3uK%uf)nuikrJ4YI}RJarNP^N?L2>m1{_tL)IfQufjg2{zri2oKx!;n#_I(9vFyat)b=XWp3LIivAxc#Z}XSgOviTW?cz zQ$72QpOH^gd5E6|TJoMOzsyOWqX6&sEr$XLMl!Ye=rgt`M~R))o(p$WHu70p9jx2LbvELt zDLm|%jg#KT!NAwGcwWN*d;UNV)siQ{I!=XAnf8mRdJi=`P$CVBp5@_LPx!fb+d#N= zN)6}SPGa8##==-jJL1mdlCGcw&d_wEqKev zm5R_2<0uJa!;`Z7s9!Aj0 zausi1tp;nQ*Ux75onvVk9jKCa1A-P!{9j54E~rSt$;UETZN?989e4qk*2m*O$!C;N z)f%|r%4z6c){Qr(&1-owm!FHXpMWnh#d%m(jt3u!ws;&VAl1XF*d=HobKhL;b_7+_9C5!sz9YEPV>SI$rTHx-3as2h!2TQ;h79`X0KL<@*E;pMk zm=48{ho#xCD=xzghmt6R+Kh#7J!Y|e4V<3QaXK+)6|3C( z|NQ(ioU~^aKJ8@=1u1Fl@M;*&M0a!EjEdrl>u0Gtn=t&_`YkoUFIlLD6Y!$!MJ(aF z8>=4qj-A!+QaNYu-~%eY_=e*wETf!@j|oTPamNMtz|kFW_mv{f&bNbjac(ud9@tFf zUPx!-pX zq?LzYzg!WU9lnL_mHA7pU1@>0Rq!*7?tDW#?-2|EV!{8-WU&o);$$?tWpxoDXZec*E-mPYoB6KPc^Ln zyN-FxCSlgqA5RObw1`@DaoXYw*-Fb3@PPjgxcXEbE9?UC$C4%R?JGO%uCkZ%PqBgl z^AxelhUZYy=pcJ*=W%LsaRavW`^C6E8)Dnc5jnA;9giC8QSnRy<#{xl<-=y+^0T6p z1Md{Ow#y&7-Y=!rEBvKeB6*l38ln9_8}-{j4Suhy$K!QUPGKi!Ii;R5ha3A&;iD$^ z*^YQ$POIQiywEc6X~q6@0-{OgGMO7z7xL{=d)JYX}op&J*?SM3>4PI;#%tv_%nEn&&6NK+VsqWu9D-d zk8Lsx&^?L89$7fe&y>Qoai7_$1PbPL#87>lv+#a#5PSEuDNfGLgzvWJK-KXFY}k@R zc+l2{GW9CLl|zkGA^n1S=tZ+u*OFPSl-aoRS2Xp&HU(ssVA$$j4bXi6H=N`d6l;`4@ zThH11N|4Q=+S$pNr7&n92EJ|6Xvr76!fbQ9#`(523kL6QWyzj!IC!xETdA#w9?F6( zagWP6`7g_O6XYkXxon0b#b?&%zMG-c|8m$GWgF<8HbBWJnL_2uV;0GhHGJmAwSOEH+2>d} z))WfWe}ij%W;sQqoa8f^zjLG|ZsMV%=5XN_Kh|%K1w78-bHT29Q1DGKZc})PdxAru zZtZ3Eh?GRjYPs#y*6YtWGDUTGd^Cot`heMAs}bJbu^wL78ib!%`@s37N_?k{170L@ z1TXO1%}dYj#mvk5ck^34*jvj{^>+~T%cEm~_$b!w6 zWGS^1qu8z~1COpb4)-2s@!qB-P*lqXzne^9y0;voEa*af|F;jm@t*}8K9P%SpyTJpbE-wioy}Xy6{D8EHo(gzyX{CFmu5Zc>3g3d?oHSPF`L? z?X)St-6jlsEiewI9B_u5Bzej@=^Z;KJ`t`IUk!89XXA>s@7N4ah(8^e$LD3c;jLRN zTCVM~TeWS3wcOA~bcNg5{oHO*r=??8|nuQ~@Ot*nY&}GU>;T62C`TknvK`;;vJp21*ap)BLCV;FZtAoCG0r_Mzi> zd|G<}+k8?UCPbg&-8?4O;-IC?{&wAm^R`sO;P81^Tq+*EOn6CU#OK3zCuZ2eMM5pt z&$h8IQr)4xX&R^5Hx7p9m{1!8197{>Qp^=d#)q|b;|)Dic*wJ#Wo_lCU~yemva^q! zKC}{E@sEObRwnqs?WLDO$FZjg(fa8TN?!zT1Dz`-$z(bO z8%!p#SKd{_KcdZ)j>{Ej7FG-e!VIu_Q4}0g=%-vNb>XMU60Gga;j>_0z?KXjI5;SV z+fyWAj1j_Twj*|p*lav+n>gj}YLC6Ph~c#R4Y=suNj6Hfh+Vhk2Topb75-Yyh2{GT zDgRU}EFLk+KK$=8rN4uw4EFN5$UPSL*VoT*+_efS+;|Q5NoKPaKgC)qa<5P`8XGAm z@+~y@xB;r^U*~%$@+pZLf;}VZ!ExCB4reV9gk>k{sr4r^@QbH=aK?crI92W&XOmw) z6&e}^ANT-F4t4SO%~E#O(S3NTdmWo8e}bBf`-H74f^gG-5;U#n^JlIt$96HJRA5j! z)=<5{oBv4wj_QR#!Pg65u4xW@)+!Fyk1Dfk-HqW}!m-6*?|EF$ceI>d-i3!F){!wb zXJKeiCGXm!n=ty^GJJDBht>UP2=DsV!orw*+#dazqxRE)TA`?bpD7wq8OM(C%!E9c z{JS#HsOUW_{}02K<|J&*^MsyN&#Bl~W_ZyxidyV8hvkcN@#}a+Sae&LGQODz)Ak&P zP7Mp#BfMx&oNc zYM)Bp_2=2tvN~P3rJl#Htq)QCYhu`gAKu~&x{}kWqW}Y!W2|*E6&uEQ!jkkp*d@IV zmJjyfWo@(IT!Uy_sjW#R@A1Rf)d`DTuwxr`t7C261z4@E2PxlJDCYVTpZwj;en{I4 zC+BNpk)xZ~^-or^;fv+rbD?-PWOe~-cSZ-Bz4%HEX>NtLpUzMo{2FV~ni|-0E*PG* z8HFR+qvWj4bjs^a5MHVoiSHZPv(Ekf6pBv9t6H=0P9sxz^W;4|P-Ou*{9Z(Vv@q7` zm1@yT7J$n)U862PEoQxp4Y8N4C${4=&W?N}al+Qktjk~_^{4tQJKw&FXX%s6rdW(q zi_a>uKVUgkgauoy>1^Cts|+t_Uc!C18>w8Ei&X5m2qh~Q0T1d_vR3yc@cNJ6c^d;% zThb1Fp^%>kc1>Q28$XzGQj2(4nBRvux2p-Cmw3z$IZ42Ll@izo*1(Gg)tzpL8R6nP zr`Uwber$gE0qm7>Vb=&k!(&mXhz_eHaz-zr#vZeiH1P!Er*Ym$dL0guCBA#d|pSor5C29V-1&Ryl_@2JI@eN3TYQ`hNdDeNiD=bbY_@u%kRSo{<-}>hVH|W$}fxq zII;;TE0vK^A(Zf*bKj#yrR<`thNLngrBbpAS=l2qltM&`d(M52($XXFrEI)kLMk)okth1%cUD{=_0RPakQ+w4xN<_qn{ejaZHvH<^4&aOP@H>>fYnD zQR5POq9>WVnq^A4D0!PH9e22ogI8#~k{rGMHHlr*bB&6-tJuV}#js=bTacvSJgwyA z@{)VzppdT(NWtMbEz3NPT-2*jn4KI2^>eXUL=K*Y&;Kk& zZ`XXL_iLxOG5a5|t>!C`wx%0Zy)lgqm~jCme1h~zWu%Xg*gms=}JjoN1#9 zd$`L5KAg0(C0+abHr1P)$qqbA<$M+Ex#4tsbhlvwx;e3c&eeQNHX5d)ZKluI8kHjY zS+j%tow@{pcPz4)=`76Oy2Rb;c!%b@e5TWHm$F3;<-F9e^GL#bQ=6E?WuB!<0eyQm z5v_V8%*ArIvGLRq)`S;-;T}GjfpnJeX#C6uT6=mPci?*m*HC?pIyA~~X(KC;eo8g1 zaZEv##u_%L?l&uKlENvAy0BL?tJ!C{EZSiDmsaPv(>ZP?lt1r0b-W|Y`SBg(X5W;y zSskQIJ7*-L_vH)F$-U|5m9U=ngwYoa&N*p-B|7wWrn--(g-)-!&-Ueh* zr;k?aG;vl@htSXdQuNl~3+FwgK>On5>Esi_dQI*L6qiT^F83nQ_x>B4a*;9e8a*pe zpCU@(g%l*SCS919wu#PKvkV>E_=+29osMKl61zd+7=2ha&fN-GgI0CN)8!c@GLhe{hNdu|Nrzq+W{CV(Yf3FsKeH#FZlJg9)akU?7*wQs zly+VoW#1PaLyw{fs`0GlMDBj1K0Bsx>drE>_n0udBi50XcoTsBdmzm07)<4IfzT;G z)NuiyN4phywjYTUgP~MyGRAU}TqxCkhc3PPP{aRvxn7%ztc}onN}SHtc%D23lGoA46+d{-6wcB+ z_dL1CTWw1?ld(KuCct-wUmM77XEvjMLjS#PObQP@oq}>C4WP#8`2@XS^EVsr)7ZJXMr zX=s7+bK}wL)7Mb++(TS>>`k_NteE%6Gl`b$$Y!%9^>MiZN%Ue#JJs~bpa)j?(ELx! z*xIR@=yz@!UASK8E-Dvuhv$jdlx?a&!H;6O`_p33p1L5E;`M>ML}t*y+8#Ptc!ngZ zDg`e57)$5>$z+e|PvV+`me4)d%emHr`_NcJ31_lPK+U*f+DRO#!FxS=b6l7QVPa7)hgf`ZbYMs^}pE9ic-|J{TdCW)^s5H6qj}pqO&XSqMY6>^!pzZ6u$U5 zCz5=T4S#IT)k$@57TZ^#C1+343GVyQP0Dbq7TMD4tAx3LhLZG)O91s7o{Wz4*rEe| zzO+l2Yd1cO;UV?ebl!ot+-~81r(lt2o4w2gRW*+%M*x$m!vZ4hHgqK8T>j|IMI_OXYa;o`nfIC$8YX$hKuw znx3+R6>v8_hiOM)4QFyk76rL3fbtdB=!tb_SrZ9qo3&w@DDZ(l6XCzoXh^uP+3ob%%>={B{Ej6egqN2V9C@tIrIi2!g>t?3WV#y=aQhO?`|CfmHzI3`{unEmhDWuXq2Wd){ z2A#%!L9>0zP`buyw3pRHIXk|vQ*J5RNazG3^>aq_p3fm#f9e2jIpd8c2+sqxL~TX^ zX{Wg0B+NeYh-c^TZR8rx=y85EQ_)q28pO0tqX|92zp#tXvR8CmSwqJRbhfNdV2Wp; zJnW0kToOeS6O^dt-6=?6x&q3sHQ{8F<*1Hm4EJg{jItf6G&)9)Dw!@vrKXOw^uQJB zb)XNeyP8D@29{H;gUu9KskOx(6_)7vCDhX{hu!?ck`B$sbk~n3NT#Koy5bEqbXq*C z^SgmdY+X*@6&w|2U$i*?{r%MYoCy+K|0ndX^=T?prs}pHT=T`ntS5Js zmIPm?dqwVXBJcj9X-`VHZhjnH7~@Eb0$$SBVtdh+ByTkH%q{xbR~Kz|cr5sQ!5!UP z*+2yYvgnp~6xzADiS6Ot7fdZ;x$==~v}5@z_RRVZXwAm~w(#2rPW5>(orHz^vpW;f z^*2{gjJ`cLWl|d3DXg6jB&|fNl-44ReSKP*>ggdeGbj(yaqQfROIqjD5F5x6LjU$25Q}+*;dkLgM5qPk!_qg zI`X25+C(HFRNBS5#&>fa+QBr{RM^ug2tmO^Z`jZ|`N;i}7-uT4+g2zUfTUv7kliIi zwz+Q`I%y$FTcW#J6~!X@xVew2Kb%0jZZ>g4BIfMf$KAZIV;C7~6;pN30q(8A6I85g zPFga*0{LgA}mccWQ&4KgpmGi~#>w*m~r|`ShYyA58Czvc!37WlA@$~lC)(7s=q_Y1DNE&nD zA6ohv{L2`@;g?PVpMWx8W~50*EH`7R10|sJ$5FoIpNo8D6-ko)xD5DN$^)573y^YU z7Uq#yEGBoE@tt`e907`ON1!gQHGPKdYhwxF7L&O%{H)!Qg>#{i4&=|G7uZkYZ);Av zG&EDphPL%8aC2cY_#vFRet35Z!FdDtzh@fIB{CNz9m&A@wyIE9`y`{$Z)&~rXf(;R z8p6I{$U0Qc531JX!-S_vaMrHvU~6tL{Pkfs%<>6_cDodT)+%H2d%_|-`OE@R_p}zj zi$#p(S3_8*^^qTZu)D>Pw}M1bRh%_`oqeoOW;kUo`U~qnl$1 zWqinDA7g8|=Spy4`~+zAI|9CYCQd?3&)|EJd}i=I1Vv)Iu+wretnOZfrOMU7^W!bx z$LEVceU&pTT#^7juiXfqi^SslldZwR(iG6YL79XO>yqCQK4hqP8F`T^()w(m5Nl?O z5U0T~a%poo_KaQwB{Zzys)K&;WL+Y#XfP*-&P$S|d)5=Y=90D9iGActc|ShUo=6~D zj1|7V2cBlNU_MR+o?VCWk1MKJqUSF59v#Kq{|d41MmgA_o(7vIC&TMsoM1v>B-ERf z3C|BE!!S8R=4F!|&WLj+HSQKbFTA8xTB8j&oxO;^de0_-p=a?)tztg^bspbN#_oc- zd?cQ~R0=FzOVjzkPHGW>LJyv6(9OiJ!!6QbQs9KwN3beq4!FPZ75LBf8W;_cg^P9<0XNU9 zU|dZLUeG$rKePWhZu;{U{|=px?e@En^4GcesN;H2)26{F7}$cnQ)HoU(rj{g$A9?9 zxH%bV^~FWm&g78wYBG}RN|H`q!S;5t(EXkt-g9dRoO!qx-%NAFf2W?sg6RY|hD?Sd z(~`mYmr>Arg%?zA7GSZ;^+QC|zQ z*4aV9ao^DlTY}&{s=aYeT&nN4q?mNHH=iT9*K)kBz1?CVbN@5GSiRZ-Og*t zKHXDHQAh?D9*_lrIL$g!lO^KN^673Sd~k^IJ?#b78_4484{m{H`!^GTMj!s+8$;ZV zxC4>)Y0zSg7OZi@z&BV8=fCS=&S)3ltoaJ$V&exq@%SVX^|=>!U7JCUipY>}!KNf~ ze=0dt8%gkrB7ovu;aHA1Y_dHB9WxAxySh1!zvX}xrs;y9-SZ(`Aqow;@}P6>4PczX zXD%At1}AhDz^%6d;d+wFzYw=*!cxlomM3V$wFUbhc51m zRD*J|51GG%?+^~Z%=!*a=FNs~ z1u@vWUxNSc93Q`^2(b35v;i8cA6GG;Zn5QR!$)goJlEkA?tM^-cP~{S*BAcelLi%jtI2erd0;Ev zx&Iu_7_{Jb1&9Kf3;OgRv89@dDd?S)?>{8L+B~lixrnu0He6&&|+vEH2Aj} z2JljVU2i!!KK%h68EgdWzD^=k!xmn zXNeVF_(chIBj=`WKkaWZwFR{h&*{Lj@;#uY z(H@3MD#O*q9YAVkELmNoOv+=DiK@`fNNK3Rxw;I|t2&EcJ-P>8I0|Rqjv9f`<5Qtc zfdSdKB^GWzA`Z9RNq}iu|6xnlIZWtR1iJSGgOyc1Es@e5gq08@i_ba|wT0f8&YuU9 zNHP?8y&gV@)P(1^RROKQbg*;DZBUc(6~FXh$Woz=Blq2k9M5h7`zrO|zPMy~N*BO> zIvM(`bA)eJX25O7li;u6jjcU)G9*g>G>$NPkMsBP`MXy3Gs&}N!6ifQL7E{2d1Z@X zLuU@?P+;KbMhn2NNPy$VE8*X|a5yCG!tb~*j|D|Xu*TOo9M8Xx4Qrn;#ZO|uR8)o) z&b(`p!fS}~%sHh0+j_F#y$}A=CkM`78Ur7KC1K?oH85+J7w|e~O*(|;G>Wcg`pt&0 zl7B89-*S{Z{KNps@w<5Tf^vLO=(=3fO9EEOGGIxKHCC-G2c~+9KzB|Ne`4uuEMaa) zK5q!O_VxECBl!o&(igg2$a+2!7`8zk834BJH{1nOHqJ_R;_@$ z17l&v;sTKL@vG1qiHDwL(BbSTR|sK^EMzxqua^Vi?(F^TnvAp z)Q`EgQ7O_OM)sGw*D`j3O0H9{F z2mBpc3}g3?oG2IbtT<9FETnuC&NthRQSbYDU7Z(1nO5!iIcA% zDP?yOy(R}RxXcVnFJWL~L<#uh_=t~ZY#}B_R>W|fKJnA-#PtV7fr05|tg)K#@#z74 z)BhWWO+G~a%tTT?lMmp6A$&{jGQMdON(|(l@jtCk$D1C!1a32=@O$4897uPN=YGM& zc1)U_@E>cnP0R#e`wzjNXIH|@I+37w%0W1MEe!-bIt1@zv*6#ibnrGa5TtZy!f*Ss z!P0l()@0^%Xt0NZ;aCrB{^SJMTDFe37taSyTYH$s^M+u?M+GPmcgNc7=R+o0>jeLN zOFEe3+{f3|(7;IniH0@oVDZH_~raKYJTMrR7V5todc zaA|AC+^^P(@)`K5_CNmfg!?Tu>rUciDQ7(MZ7lOmUg#{HdI!co&%!Qmo)W`pXP_ zV#qbgB}APIC7&$6;@T(1(AKDiKT#wYj<-ocHKF-gTH6WQ4Q9fat&aFU?;L|#<8i?2 zI^dl)oB81F4NeK46>#UVto=aYca{qI!vZJA0fH()JUuF z^QNdLkl`su$q#ZKh>h;t1%Ty%{m1PpT^(=Doa{h0~LVJ zA`_-{?l|Lpx)yZ2-wx)^(udzNe=}K6^kDIcWKdaj3>Xi~GshJs;#b{5&%Z-}OMXs+ zy06rjqN#te%II|dl8pma6RMYzNp(6zaC0U33Fk3}izL9%g&|uQdIoc)qmX0bY4 za{dVOTy`FeIZ+1V56y-yVOiFF&n2M!T;ZDY_Zb-XzXopKHYf5TnmFOwHyk~A255{` zgoDGop+-Rryf!NtM+_e(PtI@P|NbRQ+}8OsBGP6+!o3S0dAk$%HaX!ITQfchcnZ`j zH;_d_({QkA3($1E%}7)Sf#%+JtUWso%<2DzeIKp|MH`oa?c6%(FEnYyPin#TS`~Ob z%NQG^7K3x|2nh2U$G+!ONdhmIT-1vrttQcU&CFu#p?{R=Z+pn}E^WiI+Q!62cPXJS z2eG)hG`OK?ge}4tn0Y9Jxq)kNb>5VgV^X37U;Kb?Y;+U8Uv}W~KuvOFzb5%NdIuNI zVwsMAu{id|D|{!YxAmn;0=RkUhIOq`3+S|7kMFwg<3CRk#bN7>`8r$7A|- z$dTtZ|KT_lKU`*V0EZU%^HpBnWL$f`fnv`e{JzR0u<^Ta?HZpz*3QjilnyoH-65Rxc{i39%Vzus4}*6*=3;k72LzFPW^1M!F-X`*Byax2 zL~xz)v5*4SgO0!;p*z^V?-$k`RD&bB`FM}$h?T~}49pf96TPl%qE~j51V^`nCz_`j z+t11*zPJcK|J@5N_~d{h@uy&&Ob!_9S_q6?4?sQV**HwN@AaA{2`VRk!Cmr`z^+@}ghN`YEOuFE#|PFUSzBvtx{gFz|C8?>UwbokLi|NjO8S z1Gp})1a<{haP1?ZooVMz<{ye6JF63knX@r}jqwD$`p;HeoT`N@B#p7BVJ~ix)g@gC zE3t-iPV4hBbvPfKW87RFA-7-ycJhk_>UW>w;qExT@dlx>a7iAnc)Sww578yZJKa^>{*=BAJnw$zP$= z3z~%)2z!Q{@lls&_^{GzpmNq0D!<8vk2WU4GiCyOgJ+AECjc_3(OM;tf8{C{G2 z5O>#rp1csKd1)3rXuOi&enSn1-`6`|+aHDZ#vKOv=M7hMoUrb*5w z;R|E%g(o!jnWhNr{NNE>6QU; z8QwRyJ#K9or%Vhq{h6(ubIBtw8&c}hz@KNTNRDagkRi{h zaF2%2Z|zouvqkoSt3OsT3Gyn;L#;e49%Vwlc%H}mUKx{uz3aeT!D4dr=})jy*98_{ z`vyLCeFOY$N?`E87GN`42xQ?m&?g14__lMP-e=?j^3w;E{rVtxXAdZh$%PA(`as_w zRp|dN2Y-t^k5#m;VjbH)(D+u2Y-wK1Z#XATj>Mh^C$GM=>Q4?~;{HCxpQ8SNoV<^G z3k@G=aP2o!n;i}=Z#e?)ud;`gSe0*kP?9`3k`3yQt+%W*7uw@XV!>#8AeOl>4(wm9 z0xzDm^XncjfluS#gNOU~L8oKNtywa+z|4qpW_x#~(7mVz-^TU>^-JMcEkcsWjOmh3 zyHd!o{|TbvR7$3+H2~$~%Frm{UTbsBayYHf60Vqj8a#bz&oqc=@h_Dm;~PVFaOk$( zWd1UB61M*|zXQb)yY~UaIVq8xd|rmdS#3sRuomx1JIhy)QpYxSCzw;NIbf8R&VQy` z4}vD9!i&e9;nXK9;fqLln0QGacKiv3^LLl#Z%N5;3VAlHrDT86VB_>%&_wlZ^kCvYyaUvP~d+pY*VdKU2)NZx4; zly-(*kp|EzVh&Vst_3=$V_-*K7W{BF8%pq2;qFyyft>0yFh#@zrq7D!=Q;%u?-^x8 zdtnB-EZPa&)9&G*WwOLEbQL*~+R1++oxyjDT?2X+r2!}F6vkXJk>u|yAg`jfk?De1 zew)1=Ow~$+B3BaN>YpF+y#+H{T&KLWetfZ_RqwYm^glC=#C%=_jC2&pK${zBZtB9a zgW{xp!3QiKmIqRX%*eM0KhmSQnUqet4K9c5!+*-kar^{HvXZ&WWO8+kXM+RAz4Gwm9Wm=S5l`?y zm*# ztg`AgU*h}%_`!1fZdA861(+(mzm8oPxO)7ctN_d@_ zzNBPV0P$8nN}dmUW2I9rP)pJax9P=f-!5?_>vNf>$Ze44e zuN|C~PjJQZv)C&sm6&Pr$hY^C!SzuU_)^9P%vNM!Wl18AueE_>d@8hfX#?+X_JlVM zJOmx(esJj_eHdOj7amc$!3=~cLB#_hmhRjI&)fLGJZVwr5#I?meUvA= zUTZ=7Iw?~8ZxK0O^9yU1+mZGq`s5?+#|tv9-b{~&6-8UX zBB7~fGozSk5PDLR9VK9LoeZ2engUP51Xw=f1*Y!LaM=xGlJh$mSJb=(<8u}Y`>ETY zZJiRdkJ@DYWNi&t-=s@)lrq7`kU{?FmQkGU@DC5|{*2*bIhK#vMvFtMw^dRjSxPA^#Svs?H)uQeqKiPv%eHeb@m6C?FETbWt!epyv)-3l%# z=7UKSzk-sF%lK{!+JK+eFB~{660W>^6+ANE1HA_oiTzx6tp2i(`Mv%!h~jeb+kOKO z67>v3ue%5gW_N?P4-8@ff`Q_#AjOJLB*J$*rp49B_`$#ljxlnqe44m+BH}+WN1|}CZ;^ySY*5h-3 z;z@04q;sMxS+evJkpH#}94Ouhy`onDuM-0xuvi3;n0Wrm!>M4cY7CARCZ)>7943kC zk>qZuHV8k`gddHrA%}emF|?4!;|?Bpq`(ElmX`z1kJ2z+{t#jFCK3f7bMnPU0@t6u z1j;PrAhucqc{)|V=CK=4>}tl}zfS;hWDk6Suwqa;?LQARQ@q z@vH>M=}HD`uIu9G7Xir3w}KNNy26L)hvA5CGv+0i7)S6*qJkdaM8133=d8sgQH8J)>?U38pZ)Hm)lq_^&RLM z(xCtyplj^}c-ZYbDBC7RUgnBgZ+sQQ^u?V5 z&)4h#HUU4dNN*BQ`N`qnCEeK3WdaEqvB8h;2H<$lJ2-v#8-GQiKiob3*gARQKdk8I zLB400ka_bEVQ)U-PdPW6zhsXG8Sa{g$#5>HSRIC^|CiY6S#$;)RwQ8OZ|y*7gA{z? ztO%P`9Pz)0)t~`B2L%SRVc{uZ5Bp01wr$@^?EcInGW(X0AO4HUQa^JdSG0n}Hp)Tw z9-+x)76NaY_X6z{u>Nt|`_Nc{OYvMS&We}1_%aBZju6Es9yuRnN=TOZ%U$;Ith z#(4}M^Fy%Y*$Qa$-#++Z?-BTT#cQzmqB{&wih*h#D`8KzHUGxkokaiFLel*(nH+!F z14MRh7DoG1iDkQ)vQh}W-cpnF#~ z-#^nH`Wfj#SuHVqzhTJgw&7)LpGLvNA4#CYU;=Za---mQ?ZHzwJOek!XF`Y0E%2eS z09;WEV1g8y_)}bdJ(Dm! z5~RUD=7N$$Gf4Y?<4;m%vWV3R2i8Xx4r2H9$6fp<|$3oQgIoYmmOn^}Bu@pX8^ zwh2(}Tr^CH@Q0Z{?=pwG^hxbKRWkMXV~l5=1jlU_ljq|Wr1j2j@_o8CD0}i8q)8aT zKXY#Zt3S z@c<+(lkT;SA6-gzv?el=FZ9V3!v$dL)CU+>*%A+l3OwL?2VdN1f$8@yFlH7Ar0_hl zAoLnGoGpfjuk`cxgf1m28nPrEy;YEmHH|k;u0i!h?0sfQRm4@cOeNgwwwP z`{ZfxIMje6pTnT7V-Qg7eFL@}`fFX{+r<1(HiXx@J~N*SPYG);(qKn&J^tFJ3sO!j zfG-}6@P{JALE-3c&=x-j*Eop7Clv?yi$b)ZX72+K=j#L;%GNPa!H0nq@?gyQpTROK zNZh~N2TOkl>z(_2K|~}ERxQ~JL$g!hk0TKf$4m`?mmGf4EGQc{~0jioKR!6nmZ80R7nQyyb}lc=y(*@NJ;Uo}{T_e!|OS_8EV zGNDq*As83zi34pHf}ToOaOU|1JX&niT5NxX*gDNr^R8&6K{~9CP_Z66X4LIyST`>088u7Cp%Ya zl7`)SB*|(ax!Ko;dyO($n~dg=jVCSuv1haSa~c=mX>zh8MVG;sEh*lg5=O*693~10 z1;pp?F;HV!&X-EY$u62ID;{IeeASR~pXAJkuj8%i>n*Ng%gdA6Dq?7M{<#lB$R zD!@mC=UC3pX~mz+?gOiwo$%VlY`9_bA!yb<6aO=aB4lGOasIZI{4Kr-ZtZx&*e+fL z53bk_anb@j)G~#st6EBYqCG&oj4&fCZ4LPvtxbN0gb>>c*TCJI!oG~D78JS8z?cDF zyliqD344)@KL#8jVwni1|JESea{|aNu@$7#3KA z#wcwwd6V49udGetGx-}Jyln+_JY-0-Wd^p19R$&j?%>N7^?W-IRaj_#AAbn{4aO%@ zoDgJBoEoi2ypt6Qo__u}wJ75G+DK0d;eA-=T* zjAZImB6>av-+s*D%@0g~)U15GVN?f(&-uhPe~@DPUtXkNZ%UztH7PV$b`h<+t-(pk zC9!*U-eEKGthSg|F&j@=X<93s+jden;bjLs;pQ%zj(+5J(pPRtJn3^DC~jakw+D#Q z#w{xB#it2WTbPOy5ow8h&b&lz0io#L+$NsdVo9XE{t7Dflc7(3ZRh4b)3VtvlSRj} z^bs2*Cup-wC;l@|AW2)o2JG)+_m6mUcZ=V$2R3J;M}vc~XY(Zbd$^Jws4qart8#_2 z5GB0q7weGuv5n{w)T2UjnU$)UX7kA$(hV0CxzVX=)OgVnw7hUF-JP4w_Vr7#qV3kJ7l(V{mQ=z91i)|2gKdtK&puRhE~ zG52k_oH1RN7ySb1OmIgFH{GTIM-C&$MTyj>gP|R&0(5)E6>>ceq4vQcbhGL;{l49X z6}|PGZt$Ihel}0EDb;?;6@?_wjzy2@GLLS-JC6g@JXwOL)^-#v<2xd9*^^slG{6pt zCR0JQmOy8DHr=)$jh3S-?rMJ=o1o`O3s`xZgPM=gMvF4R>OISloX2wXN^=z5Zo7|i zXWyiqKaFUc{c9Rycn{g+rEu;a3lkJJ3|^j+3H!jhL&>9YAz7e_y73|DUm+7+&$!O%(INkeGhDJ=2M*Yo{ z)67*8=rs=T1~PSQ7Nv4X^MC<|<#N!%bV;h8co99ie3N}6=%N$jTj@VpZNzDT4mA}dcBXnPJ_sO<`uRmwSs+JE;I_} zG@(nj2=$|jTvuN;>uKb`Z4Wa?Os}NPf`etWF=UwieJ+V}D;BfST&YO4w)T>HuJAr?WVHC;4K!kwOhkXl7?9@6iM8wkGR;=wn?xI-QxsS$uwinD<|~Aw51%2OL4D)e5zZE1+i8 z51i@EI(ooGAJtu#Lw5IhDCnD=ASxgqMc=y4H7hrBOD-6(o4V7Hi^mCQ^K=vIIja&C z8(Z>XI$Y?BzAjcXzmnSRI>B9Bl#3kSGzi^3OSIfmj;5Xspax^d(U;v;bOI-1qaB)! z4qVD-)qVz`s1JHbTcnuQheo46m(Fsj!^>HILO0@)s_Cp{6{v+jn?koX_Wbtg^ylPf zv|~Eqmb~|1vz+RY;Z&A=wc-MIMJAKmji()(6$GmcvNMj zxDwkKRN?19NB>xJY*q(4vGyZtAR$HH-NBrBXrge&Ycad}&U72DAcppd3Fy9&DC!$jpyLhIbZYQ=HeVZ1<@?owUx$0yU7lw-$yvGVrkn~iE!hyA{kRgDR6pW=oLGUp zt}R4!{#c>uPny`fuQcdeWhXT8{1?IOz9gQ_;RN=Pt{10qNDcM-ykH|L_`E;sZzFj^ zsG~cdjp>M{*Lt!zH*$xozkZHg9cGB~Ms~0Y@AIg={dHdBdwG<+18_gDoIvgl@6eZY z1#Wigem3lhHXHPGj$o?RMQXWWC1tl=M|o8r;r^SZ>|oVQ?rX^=8XT*PutqMuwZld5 zZ(b@rw8WSeIybWX{>kjh|DN-DPVPbXM|8RSb#v+G=Iu0a^)9Nqu!mcV>e2gyot#*J zG3Ti?oxR@m1eL75iXI}(HZM0-dPO3WnoUnZSG&%!_IJZ+{g(A;bkben@57sOxE8~G zPJB!Eix;u&3rdl*kP7GaZ>EIv_3pWk~;gB|v8dv2c;L_YXKTgJrO z-q!4+5B*cwv>;VBKWROk7yg%hZFhogDgMpd+~a~g?;NG1R7D_t=>q%X_ha^|eg(R3 zVIz1I>B>4RdqLN7$7r*DGfhf8Oxx#gVJ8khr5eJonvrlVYK-MIP0A`<@nB4ztQ7C0vjppJwRYMiOt&u=5_Qq3yz%dXKy!)Znhl_GQ%2 zQZEM-wc1Iz1~{Oj%f7J7@>|g5z|UN;Oc{OIx|C*!6i~fYnVjTs1DAXyh7$7++-rL? z^rmwKY74)^Ml_#CyKAI5t=I4AHd}daMSU@Sd@BxpvOG*fOrlY~S_qAwGLin%vqG1* znxMYj)@}SsK_2xyv~2MZ@9v8^Z4ps%tfIp- zI{9fVEw0(Ya&7k1a(EGvd-{;QtnCetFGxXBSA0-!G31^esp3d}C9==FCiJb1Q1V6{ zw4-we>i7Rk&%HQAtEmH(>U)V|^7nKkbKzDyl1To&k9 ze}~HAo6#x7n>4GmSg@(=6NkG_qDh_6XyuU+&i9-r8kq109k>(BT^aktN-QSm(%V|} zW7|FU&Ok5H{5y~5{ACv{n=btC^{x!Ql60h9KRdZQBT6WA%n}9qFQB}AMQHG16y2(( zPGfH?(7Qu_sYY2T8gA>P4d)l3L!K%&2gzfhkN89|bA1^4_DVQ6UewGb|GmY_{%XLj zj<#oaM6TnKcvo20;5_nW+Kar=TM%Q3({AXMn*@&>C^5Fyh#SbTxhlz z%~KSkhh!YtDc_Um(#AHP-`0&(s{aOiZtY7hF+2|qMRan@E$_25><>5VtPyhfCv;|y z{zH0UYV^+L1c8j^2kuk(Oq6;+9PQ4rphl8Q>GJd8^z`=;?%V6{{BsvL-c!-rXktn@ zJ@88lRa_n<8>@D(|Gpm+NRLHvN^kl(`Ts5nsBKd6KlhcRLYtGEyC8a6jp zXClQiUp6=EE_D2N29?S#L#5ddxM?MeI8(-k-cnXXMH8$!*-M?&G$@{(>}g8n<>t_r zK4WyRl{j+Q_>L7ew~4vhdKwN2xH-}uXjj$%$DdQnUNnBl27@>*@9z{UJtr0I7tutq z>VDje{C!BWvz)%1(vFrty3ZE>ZAOyD6*TqlVWg0u($>4AhV>biZfm?-NS{WCuzNzD zBUty9%~mSnlDFluos2C_z2!~5#MuiZzPWNw_g?0to^3_0-w)B8RsRHkYa*z#6Xuqt zC9%cdBe;7)r*@}YJRRLRi9Vl71k)<~IGy=JbbFL8{dnjieLL<;Q?uR)`dg0E+O`3z zq4x@fstRq>|0%i?f2y804&c`8YqDjRB}=x%nRCxrB1CDSXi;g=uB4rktt3lIl8Pi1 zQb_KZxo573M1|67E51r-lU6Oi`}+g#%j>+{JM)}p=6Qcyf}s3c1>r)E6>#9>RHA!d z5(YJSGaAlLZfb94acTP#A!igU*rx}1XIAvX|x_ATHV#Jgbdz;i)yeK9wU^M|v0G)d=*EEq8T6?47vw&11q5+eKl1bMwXi_B8& zC!M~_gwGA+-6Ab~-8`>6B0tsQ;6CON(X`)5qD6M^y4q<_6@=3wMF!?p_>m)3F3=`B z72dXYVD>+ff?E}ya3>04$qKP-ZdUd=LU!}XEe%EPVox!-n5qc%rdp6_H9fL@wmkV| zc8Tk=DkLFCRx%qx_mOvJ7I5{`b>V#d(?n-%4|B}3O=QPL!M65ZGMfq{)2CFBrpt$j zW<(?Nc{PU8hHX$$`!u8BC`ZhaesZs3{*s$rwdCc59Hv?5!`TeQ!Z3qcvV(8W7^gG| zEVEY#70kGi2u7b4GB}L1cS2ip*iD z-`t?aaqdK253}$}92q^RFLK>9Na_MI)9w$?c&Fy|2S z(q;}r+iQ_<8AxQbhH>r1eeiH%85b`R4{f&T!ZMp8n5nh{_VvY*W%G4tX?Y#Up6Os# zJ`E>lo5i80?@K12(}h#5d%@)&D=NU-YytB8B0p!%Hcxw-Sn^9eq=0) z;wf{FxJchOjO^(a?#ZZ+TmQpjg4OC9$%yGkX6$nu$uT6vzkD89H1Q_4FR_wXq9-KR z+ynMIO%STQv~`;>?GQ$PQsBlRb+-z>C*$~2gKX(d5cIZQhxyTGnVGBgnT^Ni5Z$fm z+=@+@Gvc&_S;e*R;-g}CbmBE)m$QZ$X?NvyZtG{@4_5z?ax46%B{bqGkWte=@U~S3$;){_tZsx7&%L^&%{YUZ@F1E@ zAN<5Q<~IqXb~}*sd54*e*Df-WqFLe&HzoMKp@vCn=fP{y;mmo(zg%mi53F$h%kvPK zUI}v|VDh@#P&vty(DX85QSHGiUDyYO1^yxru${SQpvsAr*^rrOL(n;@2znpRg`OL& z$o#=!*yJ^h?06$h5(Ill{EemDkr%zpMPFm?eEK-|>E91-*T==Y8-3RW&yGKX?$R@v z+FxPN%smAz%J>L-9yBu>Ll(p2306GVAkW?LU%?ENFx&a8=qvUYVUlQ6e8&1hd z=Bzd}2->nVnSb4f$fA4s4Cc9$6EO>zqyNdemDOzIrLOtQE$WqkabBVs_{p|pXPyH%g9}+^(1PBsE5($&%JBcb~~21ocwsGAo46M zNOvEGjh(#$&VK_rUzH0}H;X)6J|?_tX=HuzEbfQ%d*(lp$-fWQ!_T{u$=2QSMAz64 zZt$K9rT0__%%x@$E4M&6@qI5xK29O=2gKa&Dapg`+cYPaqwZ#{zZZIVgcG+>LdI;* zalNloIAnaBIapUiqQggstH*cFd(%3i@JJ0Vm+NPa&z-}Zvi$^y>>G*s&UMg;#KD*o z=H%N439clklk**w6aFr1Bn@ws;ovq8M!wRJJRdY5V{ep%*ZxV8U)zQSb3E=aM?Myl zz6E!gVdvw5IhlzNPyH%*pqI|MB$)|q@N7m})Za66N@UKKY-diL`vS?Fd}#fB7diB8 zH(9R}!}ya`dgdg8$(zqwAPSTBVwi5N%yGm)+)Rp@^ zd4lj*j3KeEnIwEKdx6wl3g>mM4<}BQyP<0!5Uef>C35THnIEH}M8#QNsB_`3K>AN6 zJnVCwlbo8zRlWU0{4FA(rvDX^Y;&3Er$Y!&*`McN@4>9S>ByZIvtw>7pT-24AAvsi ziv;(}c0f1j=_F0_7&)^>hJ5$A&aCr#0M|H5G5oz}xY}P=89%pHrfZG`qi7Jyt(l?g z*12^V=^Pdp`W!V79uoh~Ec|C6bd6g;@|MLBJLL?)ib)ahz$pomZ+VM(9qs_18T)Yj z&PFIRhhpA)=)q9MBql1Vo3pPS;pKZL@#N;-WaiI12FIREa$6NVl?&Ir3VXhaoHL&S zXj&9W_T>EIw%(3r>i0UpUztX5r?))$vsdIky%`gf=4^#_0aeUQuLhX2byT2gF6(xx zK?VBxlrTSxTmEkdlBRoBa7;xDiV5HF4m-DUHA!Mj(M3B!SeY*T6!MrBU%iT}9MLBC zr#r*Hc{-dD9S-MTKSnBF?1P2BdtiNM613QNnFROu!R){mu0Q-Vv-_Sr_d{_58NTYu zY@2ZwUPj^Y^elOjH}eQLemH=%{#g!pKTN>>5=fF9Km*>xX@EEjVYUW5N0%w5ZNjLqjV(=4(8qw$kd%DW^_E!FjHgBdECW* ztTwsYoq>O6BhJLeh*T!!!eyuO$P&AJIFKnzYCgRtd(yWvnT=&!+onR2*0UZuI5rWU zp9D-kbeaoZ58>TOL1Yo_LY_^jBl%Cv$wJ3@%-kD(Fs$yVpo;Zl@(w#NVM}Tm*Op}F z^NK2NYQL)BuXYKy5Z&aAB_&9TN(nKDJir|}xeW3*Z6JCJA>2J<8MoiO02-XQ0HqbA zge~Es-mlg*_-2+=Q(=vp?_p4_xsm3&RA)=AaVqA6Jle@f}%xmb_$RAeei(b^1}c)crRVl z=e1;#LmD~!WDf6-RV`Ee?E|#e_2&-Q>bXr?Q;2)!{b7dUa=GEmqXd5R0r8sN#MakK zFuPb>_|vO`toSJ=d@>(%jiXK6n+uWL?ENZmuhm?5!6b+2Q<#MP%qKvTh<}_zRuH5P z{pOrA3yF2uVR%M$y3i`Fk@#1A;XOrVq~l#4tZg;p%^B8YRtm3>m~=^@`?KSWO6h8* zexnu?HvEE7GtFVFw2;@l_@Tf~elv4QV=eRIj~mZM^C#!Aq6K~_JNGBDp@95&9KogPwUPZ1ws5V;HVv&!BH|rq;rF8_$+6{%ZYt5y%omLa!T$cU z&}Y?R^67R7bGYA@dGW)G$X2`+jKwU2k@Frhw+kA{_RehP`QS$4v|djzH!%ht8Cyx} zp2jhv3;N`RbTQ1x&4fEwrt$sIFCmjTF4n~T3k`(U(gJQyMh3SQ>?H-Vqa)}#KozW_p(2mtk2yJpU=QV-AW585AK1+_IXTF z)_;uBDo!wS%t$cuY`Z8k>H;k979&2D+jt>5o#em#%jBYkKKyijJFoqtfLztbjORRC zQZ7m}zIgmKxzOkW!$RUn;DHdh;&m~+wk8R_kniLUbKAJbr{|Cvc~ZjMv2>`Srv#}b z6XEdg#c)$!9x0w|M|P_%gqL>c6U}{-VYQ+qPvV;aG?;ZA+K)AIo6jDFpAyUAUc8>z zMhr1d-FM*q3(4@&f(R1h6hz)jgfUkfQw2+mQ{dxek#O3BX0kG!Wh@RoB_phwurM_b zF1jx*jFOFqnO13V!DerfA@!ZO&7VY+%6Vj6)P5$QJdnK5<^=isZopJqdHA*EGbd?l zz;s{R&BgzE#ch?;5X{ReXKYlibF2JvNwdowuAm@{*!-;K*2iw*?voqjxw|XZwMhNzYaX4{L*l%Ouh^ct!ss0mjA)kp5pM|c^c|J4Hm4`5apxvKIV#F z#K77Aq=j>L^I);Bn%mzkGH&SYB;jw%6!@{y7T%eV%@pXA!}mQp+y-$YE>TeoHrP!g zZ63vN*_a=3?iaxIGNy!>oZxkIE#Vd)xWWYb&~TZkFWJBF1Tif*NbFxLLAm)IFz1)( z|H*SD{^7&SO^+#3f;~BfwA`mD+>*ny+%CwfxNU0+B*Bi+Bw){S*!n7y(b}_>^d5T)=cre3w@0I3 zg1S8n;5^}#)O*azjChhh+Zaz zk_$QImG^LMhA-*c+el=+zA%$#`4gqFbMV^L3_;eWBA#r6B~!+W6?vK&OzC_V4mb4h zRJLVscgOw_{-s0Qodbhps(Li(!P~jo=l{sOwTfg;vYzl&YX%v=wg=j0>5DA?iO}Si z4R_Vr2gc@W!r_LSu=33%!Y_A#8Eelo>mL@81LHf$liN}-v-1aI{dG5-*twSkCuA^F zQzk*svj84`za6TP{}@Z9N#ukWqT_Q0D36Om?zyXI&;8lXy7*X48LHZY!a5MO$Or1h8oF_a9y;^>9 z2TPlH;O`m17EyTpvkzuaKk6NMx#$wJq39-harZKuaWa^AUGZT?edUBxv`d-G6@Qu7 z6B%U0PK9e9N+wpz^SRXFW$?Jy=%BXm}7+U%rgzTfMO!V(pjNbJD=&%Mb+8%~P^dO_}rGE_qL3bhfI8T*~ zRE^Pp)?R^yC0(Ms2*;pm`#z>{4Ieh{j)d|JCW31XyO;&Cio)hUFX5f-R~ZY(3}#U0 z8sT-A@QUWB3%B@`lP`-WLVcY~V)*eYbGGg;7wj+UN1lJk{a(Knj^8>7H(U&XU9ksv zyo3>+o~Xa6ku(Pi<~1;jbdGZ6FO*5h4>RuZHC=9Ln>Ep#^@}sOEeE9?x_Emfym*U4 zvS8TSSgzILEjbn-4+|Ds38m)+VJ01uT|KJ8%0mjm4e1@^RM{l*{H++1dRHHQiHaaS zZSwGWiXyZ3$^z!JTnJpncr$}0O2pP&mwX6+POlxGB(iQry-SxQCfs#8sW>JH8(z4< zWq)1CVbM8r;4cYI-b+R3X&(iNhJkRx*ES-*Qi}{*941%xr9jX2EZB`&xo>50+)XRt>&xqytvA}C;=v?hX(GuODuhCxL+y;|O>fbCm7Orb zU6X{f32;^CN%Cn0Bo0-ZxCL|(%!<6qTPX6@oi8tk{!?DU{lPM1-q*P#C^MT2aMyL~ zat|S~Ql(H^)MJzw_kvSi6bYR5PQf$>(Vd9M)4X5OlENncxdOF>F6OvVJe+3wo;NK; z19r^XNEFwLGm~4Rh^?$HTzg>_vH3Ml^0KGU&#bMW&&gH1n6z|K=)Hyb@4@7vhcj8@ z*vPru-3otRN`@sRi1Bt<0z1Bnax;+<7vZuJu1i%XpAPb2ZuKL4?9Fc|U#dWw3Z#T? z>qZzCcu{1Ac9A&!+r(n2tuWpvlpHF(NSdDV;H7N?yrQea%w8ElM#@F!$R^jB6Y+VZ zdz%lu;3Y2H$lfLWq1kYSz9Csxx`NqboInZ-kCTv#%8W^$790){2&R3MAVIeiU};@A ze4@4hdZ)?>jUHK%9#IC&d~rAUbEF*7AN1h&omRwY0Szg|oy0wQH@Tk_3|HAp3DZUlaM{1s^AMOP&@lL_}2Ai}o398;s?(BWS*@|VuucBPH@$Y{* z<1@G5tkhC&W9>co^3@DjvOk@7U0a8o->Jn3n&r4>_f_B*yEVk4U?D91)J(1{xWq(1 z4<^-04bbUPJZa4ko$my1B5P{yk-Y;fanF<%zAB95A*tUy%P)$;+-)EdUP&~)0)U+iEV``$|a<&48nD$x5r@8R3uE{eK&gT1yhLY1<6sJL=HVyld>my$R^D+Mw&hfqu|^e*`oQs?%As zrE&DWRRFvh1uFajP(A-6DqY#mj?jsyQDTJs`o|17miM5eS7OnjgLlxjxGf;SpHSbT z75K`^+u4}?9aO0~=c>||#or(|QWF^V&(%?(9_X)X1Y@F^BDJ;$=$&Z-IGlW)+F^VS z)W~-tDaB_ZMuyz44jF z4#{0Z^?mEmocc8IOKUl@DZ7Ej9gPu52n73Po1xiz&mh~wN0IsP3lR3;B@ie`fuuk= zdTjeI5l5u~ra#J{M%Gz@z_6((&@KXe%s&f`n!W(D0!z@y%sBA)Z7F;EZ4q^I?lmed zGz3km(g3N`D$yd(17LmG1n~U7K-O-32T=VKiJr5^s3=uOB)dY5S}{cvEnV*q?4L@} zk9~&0j&mc(r136j@G_^R4m|;1mRF_~Qad)Vsh0gImcr&=iDjGKoCQ|~ z_M)v}+rV4Y0h+u<0eX7{9sOL#_Ae`CO-D@7tvzvoJiAXV$lM5~AF={1HJ90N36VvU zei8YdUyk&8;@FxK06aNd46alN!Lq+)NNQ6(yP8o(b8E7x#``ZQ$L)z|W_=Ngbmyai zc^^b?csXd>wR~!mX6U$Ig6r1(h1C5U`Rt~UI24w4gQ6DP zXPx2?vC`Wjk#4*lDlU77R?bcYAraHS2a7M@?gb zS_KYg>QjGS%h5;fl%t7hr@T+uIE1FR~s3C#Jk1$D@up$b>a0NZvk z5OeGyc)s-@xE*%~xF(-RIOsS?Dq6&@NvowYcYJXT=J6=?@<0$1BMI!>-4Q3tQD|ft zdTytV9?V+`_PRV5bq5;}{;>j$k9C4~hC|?I^ew=Dtw`U`DMzJTB5HWlh3f1G=*oO_?QuJ+65+wh%?w9BXD5RLvp0eI)~DcWQZz_A z(1iApUBLKTH)xx~fp?<2((xPDqEq2@C@;PhC0|(PI#l%;{25^Ao2%vMVonTiyeCUr zS31$BE*j7cr2}Y7%{^qdl#eV0&yn!DHQ4^anr4Tm(I=gp>D)kBoa3&J_UIU+mG75Q z-6gBor1CfvR40eg*mY!3dzWADF4c=?QBa7jHI zRR6sRzCN)6UnOTFpRo$$^ePGse8r%A+D;fCuwOvAx$Q$u0oT!$4=Y$Jh4TQpKLrZ2W$2k*&p}Jy9<=Cv z3cI?mf#11X9m%hqB%1AgjSk*Y!HLssu}W`YZv}t)>wxn3T(Iw~F*RdvnQNK&1IlKqBud>(D3i0ADF5a| zhdiF(hVP7%TWoc` zGG+6@0t|nq(d5;`sNs@6UcJf$tJIW(yM{(|g}5B8eB&NSMXe||%>Y+jcf?xF`uI(h z0v+A)9_+jK7(Cwj7dVVPLr&axQ1{22mX=bdivqnhq~EEE00jqGrUpc zd3W}P3ZSFwX3^1x-XbHx1e-gkp(eYts3a1*@<*e9rr~37dWjfaQZOL7Ir^j64wUbG2EEdGgzhaUg`P6l{qS4h6JbkNPhCXsY@JP~m|Np5?~_2ouqGYhX#j%!@1b=U=Yx|wAqbRg z2a>v?HBw2EE;*@1dl?OYJDcN>>CQsRZt#gKnyCrmEBB*J?_T6F=MviYR`eY=+6IO_ z#Bl5`i0$r5;Tdn7>0_G&)XWk&5S9{;h&e;%`_R2se4bfKYJtOtAX~n zWU#typ-b?MBWPa1Z1i-YkdoFjL&NHYpyF^PkWS16!Ev`)uCk~4OwQX|A(2zTTjk~G ze2^IMd|F1ypHf9)2?;=F>l#*B`4Po<+Ogv2&QPbmKVvt?%t9)4njrYfQZ%Eco)V){ zz{d-lsOH2cY_e?}=xfafI=i-lZ~25(XpIHv%tU&wjs>l5D~kiaOa;}dX<%NIYZ?|iqT-m4WPl?~`uRCLG z$Ki9X<*Reh7rF|arjMg~kU%XDX$7Cf)akuH0VrOO!;!|7GVV ziqKWS;@SdmN>L6yRSicz(y2&#iyV^lUcf%M_lc@~KLzM%FJ!M@Q3N~G?@|+9U!dm5 zsez$m578H+GhpuCQR=@s6H2n-DJ3oTnNpRprWS>@vQ;YU!Ou1bH`bhrK~weRyrR59@onz5Hb1t0FY zE-B(8VU({+SI%Fw^`IGjI{6@w`NvZH#dAPy*lPYlyI~M5+XuGeLttXa0;HB7%Ki)u zVs(EgQl3xB&`{wA*QPHJ1=KDbfPs|Ghm}Y~1OZygwN1-9AS&ZyF2yXADtR zty19Lwk>E!)CKe>aypuJcso_?u?18HC4iy}+F(<}eOLLV+U%DxL)WagreJ5@QMA#= zxMmL7Os!maiQUtd1}6L|qKt-5fDMfg!HGpO*f_2VMXgLm>#V1t7MI_A{CNeXwW5{X zf3=jacp#cxt8o+wB`%(ZOSLQI^_8 zc5MH9WEj(fo^IcPE^XWe;*6)EPyAn0=O^o${BX|YK#T{9tB63Itv>7w!$McZN2gGB ze*oGw#U4dmvf=kFnG1Ncr&8+QZm@CNGO4fA|D(#*3{nwe8(0l`Ay|}s7>#^AjF7k? zdVlgaTjaR|WFB?|=~spvTK3TXf^2SO-Bo&HGuQ&ccA|1 zb5PQD3hkBH$-cbYjlMi1sCcdk_%rJat8Q``EuWBtjCPy_bM3$IPjo+^R;?LfWtZ(^ z4a${ojo<2`_A9*u)QecuK`A0}(K=U=t&bOEjRWiS->&D@W*|M|yQm4uihlczF4^k6 zEWK<=joA(0x_SyiFFpy7pYAbKo;M2I4LEkhnq$k`X7Xbf=zu2ei6Br-oc7GNpwH@0 z#7$ybkzZ3A!b^&f?nN>BLY1ZKq^}%E3y`E!oQ|UHYgqojFaGG{@HOOC*-!PJH3x^@ zaG)lMh(%dwe%}ey_-Z-Y72m;L zC0XF;>j*GOY`?3|v<>)(Ffu6Lg(GLT>k=nNRVO1{n3^t8>Am?f({)y4fv){ z+4?hA*@zQ~Y^3r_sA_2s5cCh^J#;|V-uWW! zQ9BT{uL*?eiy@mT(I|_~eN;NP9lUj0<=PeLK=sL=r{cmFBGRlvNM0(Q+8)gZ2F^xXZE>GPGE80pf;QY8W1P~L&ARy+gsg7<9Co>^$=_yk0g zSNx^h?Ol!>mILxp*C>VHS!iHW9{9w60_>|i*rLjIbYE;ea64g7zgFYX-3l%u%XWwB zJK_us+gm|E{AzY*t{0_u?JryYMIGJquV(9px+uNNcW`+#8Qml~;hO|biCHkh3n zPwgy9L13yGFx89&=Au10x~3cL-!A9+o+o7O{hE zomyF#PwiE31~bD1=%~qUR#W7A-EUFCJ5>w8(`~G)+IMU8ep4-*y5ldK)NmFQFP{KX z1+V#smx$i8;}TijDos!^V?JvowvSJBuA-_68iD8Ce;^`78Iyr&?AIA3AeUyrx_)c) z=*3RzX}bj|l{yYIf=AFTX9LQwUO`k>3Q^FBTB>k72E53Az%TVzN4I)^fZPYW_?vIb zpcUph)Z+^kDCTTFQdOA>te=}v4{ts|K7A@kh2enVz9``C^b^hF&j;r;MaIWrVmkf(WFlcZ`}~brD25-xj$L9$-^-C#qbViB3*d#E(+?_=n~zBa_`#$Q!I= z|728wW%~xutk3hnQ!XA%c;AoyT+2n57M?{1=bH0>7iXe3JUgJj#uPg&j6>?Heu8Bm zN72ydX0&I15?VXwcTJDWbYSL_J3Un)24SZzVAh`v^rUp@flY5IWt}IeUu_V4R=&kA z{dJnn{`8KrQSU`wErB>Wr2@rMTC|w9CEDIR0K(tz;~yNaMGxxMqH~Qp^lQF5IJLG5 z(K_bH@8~2vf0xKM@casfZs)M>k4BImN<_6g{{uU8o`WA9IzXdj1(lvs07ACgql?FF zD3y!qpjYIvmo5JQq<%L8l^`3iW8?r8H|rES@fKp|v%_f1GDUjk-vVme=_*$C_yrVp zYYbGqSb{84Z=jQpoT;F)Uu@l$Q8s(b7ppz4K@&yB@O(`R|SSq zc*#&8?fc!emp=r8S4yzQ%hOO`$P@J8lnjo`8$*AJ;y{z-TTuGSnVuR|0=n8{@l5_> z6!O;qe|%hkPB@(dRV{Ck?}#bBXL1&0$ZrB$mj9w!!uNrVsK=EK6sKmrE&&l&7+QO_ z5Gdbzfxxj0F#Eh7JNIS7U!K8 z&6xM{1LnKnd7Lip+c6GG9m~*&FWmRpyWnK1ImVMM|>ty@(nxB->SUp~M4(7mNbi8>avjQ-|`!wP-w44fYRX5MmX? zS_FJX^Ez1s76a7L-NM?|yaV0Hgbq;mvGy_KAlmt8xN;a<&9(E`1v% zA3cDMw6uw4fjXPa@DM0b!y2Q-=w}@t#H|fPp6?bRa7h7GCx4~l zO%LgFtU4UME%tPtB37S$+3q7S=t!3(=`>P_<$(ENNdt-NRmOmaAb?u`t(EYqk1bWj`+N;k6u=0oUL z_&0yIp4{-e739z>7HHhDrir8rgtt&FcgG-$7 zP^&TB?{9_4WqS7Vrk&4Oyn=(BMIA7PN zBA+_|KK&T%UKWX(=iNp3Urxi_HSXB``9#{fu9(^<5U?BnT}4mz;=sRY6Y-lIU%bp5 z;*kHW>A}ldr~nz^dnFU_U`Yiyb2FLRbGjC6+*?J}IA39ZG+Lm*967x5q#+U-jZg~C zhrznu0D$Lw1m>;}P;a?2h%FHSeU3+aU01|+jMah0JWXWvA`wMT+YWZ0h)2qnbJ>A~ zYf+=SDZ9uonSVi{7eubC2GTR0p(p3|q6tT*;>vHb*rcN!$SYYQ?@xp1PJ#~3y^;+0 z%X83=OCONGA7SfnDx~QRK?CS@mp5KDs)uD`v}yC2>+Flg z6TpkF=cvusj<9E*BvHl*U#VA5zk%SAaFn6gNkgxGe?bo0Un7ZuN^nW^On)wzVgEPExPS6%uyLz9 zkX@okH@j=mQNukVt6hns{E z&SyO6Bah}|eX$m_%T}D#RZ2qzgPY6=>Jc$+VNWifBm82;Ut#jo#=v zVZ{)ZO&T`?i6u1QNPS*b1^7;K1Q8s>qHG>V(4p+ByCpnl=94) ziCX5b2L`hp=q`~Fb6`q6y6pN86v$4czl|u-lXpHuqo-JwOE#xfh&q2n)fRCpo|xbs^rXb6!9caz_O zc?Fa3!99IcjI|n&@;U{;)d#n3ewrk*-o@z9_Bs%>TncAFD`2r`4LH9-2o(RELC?pDd-tFIvz#H)mlz>jJ*R2OSV3 zw;MPbRJoRDrGkHjuh6FUhpvxAHlM)^S6c1(VmiOkiZ)aC$2$WZfp0@J`ezVNc@@b} z+eWl-!4)39)AI*;Z4~+UUpkOe*-|jESpsKf8G<1L3HsaRIdqA&A-%gp6))~8M^AtW zI9Rm?*~J@!WumMBzhh14oveUgT0G0;`8-LKR9%Sno|{gI(kFq`Jw9lX)}{62)WM^> zk;q16HN|bt12?ACi~Lu4d@y%9tmA2+q2>n!x~u(l~1k~xXq@41Xsj}qk|_3=@S<4r$W-r8;w9#0m70bna1(l24=qEPkK>A2CnlF6s zIy7I6-nCYp-aPLGI&S06MvDAK@%)>pTBDSD_HmruS5l2C+UwDoKLsE#TNztRO~Vg$ z4Dit>a`X?wXUI^)8QZL|!i%}zVC@`htoPg&NB`2rcN?bCd3t-$6o-j)i*qa}?$ky3 zYi()oJwEijNoKUJy9ZV+$0*q{21J%6q7?ZIuw`Z@^=90f7AnZo7waTN-c14Gdu!nP zk+%3yk{qohvkpwXeT;ph)r+e9B0)}#GhHAK=o`Z!w4s3up6zQ+KYb-l2W_89rwo|m zk{HoGd-W~|t}p@#4JqhhfT%zuNN61Jn@i{99KP>%L{ZHK!|=iulI0r+sV zFa04&4y$Z2#^d6e_*|4W-Pu1G_+KkSzo{=k<{L!NVj7O!V2Xn#O~+FA9BBW^r&00N zl9~gBJp9k-B?zC}07fkIXum~1^uK*x*aOMqn7$}f({7C;@nl+7HX8^Xia^)FXMmD_ zhCr(c?fb%?y4Yf~Jl&>hMAt6)3tTdd@Y}IT^a3SK`re1-bj@39Y&72-^iMBhLneIU zKb8t~J!qu-%?ofQQnxJgD7ri}Tip}dJ>8j%l zOIcasH9b-|SY&yI-PEC-D@OrYXGa_9+T&dJt7z5LkLd2o8>rK85Y%x-^wbJNx@+Dl zI?a`b|KedVI@gQtoyO4Xs4=usYBK$?&zLUMx&$iTXk)eea`fzl)9AFFM)Z)45mt%* zf_mTEVB2B|+!yf&Ts|yL3taT+Fu4n0N8cbik~<&0=?zBXC+ex8lQY4M6}GhQ15LU{ zMv4AauZiWn>yet%7Pjx#U3M(<5$NA3joq^vkoQ+xY}GS~b~wEpoUAAYddK=jc_bg% zoxhHtvUAcvzE)Jz-hV)a6g)sL}PMHf12kuuuBN-mcGR_QSy`TJfpGQ0xJ{JYvUcZwbG^_-8a%tqMv z?X&qKrDLGLtpGgtFax~X3FyDHsq`yzbGlzcgD#$6h~2{d@EM6eXpYiiJl%aU{ba<0 zO?>&A{jetuP1Ugm24$MqNt8E7IZwoGhm7cNegaUhEy^W2{uS7lyh9sX(_BYlyio4> zAn?~T1$B;2#{)NGvFlb7+^rNPGIPs7gLfk^FIoc{jEB)pVIjJ|`4sYYe1w}y=9*9u~E!XkNkl7*-q z(Q~9`mDi%v>B&@xS1$E>2V`BX+>s_)f@F*>!6v&*>PyOL>O|xN_FH`{3%7gX3>9mP zSE%8BD|e1dYU+~_#>rEJO@723pS4#mDd zhU`x(VxE!?o_T%(4*#r1e^8YIq3b1S^9W2u9q&P%AJnnu>k0TpX&K71*Pthg2)AVq zCgCZIwUOe#7NBjEhPGUx@!|0E>>7(o;H&wGUvobYohZCXO@9!L*6zPZo$R~?YULM$ z3!~cjNKz?vYspfu@bMlhsn;J(_Lc`mK6L<0vp_--k5=uu7TpzkV1?WRXiK#w{oU#c z+Oj|%mlt)QD|QB=+&K$iHEkDIUwQ(}EiOe#+G?Otg$2Aj=16dFggvXOEjnvEhU$3l zso=q%)Z=eDI5oi?-)j1ej@Vk#H@bU}W{w9Ies8WB6g{WyyFDO%qdI7sVD3U+I)mO1MhlQfi|MI})(KG1iW=oL1IT>wsU4eVL zW+Lh1j&xF0G3tna12kg~xqRTCL~eE*I;=m2!qq&HL_iy;3i=O}ud1c)8Z?7_J?i+h zxPoYA%M71N)5I2Wru3IaQTAB(dEjl)30ilUV$}?Dj14sL$~19&PtlIfO-$$O96#c^ zWTXwmM(0wD*f)0I`yJ#Hl*SG;l~OB3wZcsc1gc)E(KD|8Kwg^JVCl7R`uUq5BJ^HV z-=rU>b~vBgW{@FE@H| zY7SUer$=we(!$*q4M_ZZKG^^BIMSYTfv+y~2KuXAK}v5v=*xPJZd`Vww=({8v%Lzv zeaZ|h7q*Bt&R;}Nuos<+`Y*$Q68^MKff=pkt48Y@dSc;$95nrfDC%-vA_{AZ0h9gS zqp@e@q8lehX!58(_>k^PuRQ2T16LFJsFWpETHg)E4oqi%?NJAgBg1TElr(++Rx7Bq zN*Arum(l**C&+2Z9Mo}72diy6320<6k)$ELQ*i)1nj8-*Z)cz#hIUxL(-Rp#QN)*ai_vi=Ds)cIkSNRaJNRcN ziK`b|(*q8cwC)T8TIsDZ=GQKv*PH3m{%;p-m>Z8;tshg1+>f)XUf-g=+W2D` zBTLbmZizdiXV5k_ld=DbZ|qO4aH^@k6_hwn#ngE_>~>BHt6tEi?M^w;V`q8vxQ7L8 zGRGW$TkU~Udkt~MWCz?`BncHsAzM86e4bE5X^^6%l984)wSV{bA9${F?>Xo5`Mh5* zZomA+F_+`z*9(q%ED}VI5JC6^J)Ymv>e@lSklKflV+6c8Ev7!}7V?UJ{xl6cJ%$(c zU0&eX%kb3o^?1qO=kTt6ohk5Z<_pX@2dnn>&$ShqKTOw^%kcF7m78*_gtzRdsDP=K z;4LWlY0AG;X)3zmYHjm@d!}w{BzS`X1B+L2zOkfK;JwtC$s4HEe(?z}|QcFH>Tu){}>|n}6M^DM=xw+9%EuOU8kb+~@*hGXmk`#vDAhati6C>rkqo zg}x6y1fd4jd9&g{Ie23S}UBjxY9%1v=lMp*C0DN+Kk-h_mFihR&aP+JRUdN1a0x3nT8xI za`x*R9GtQUvpzgyEtij@GsHvCb!!DwZ{G!n4mv}u*E%vzpO32UpPAI%r%`af1;6l5 zK=ansVE$?v+J0S$>K&1!(MTF+gd}6B3nL6DH7AqkEUeQ~M!vEwkopVQ^{s;W*6az@ zdd4WYgI`nO*R3XcZ<9EQ)=P!c zvwyQPMzM6>)N`00KAS%N)e2X1E?|peGdawC50APN;uf!g`sp(Kg^OZPRVIa$=Bi;= z+j{ibAxG`vFCnhaf+dY2dDxL)Ks|v+R!>9OLjO zM{L4jXJ8*BcAX*$Def?OYyhV(enBOJ8esQ@N1zv*ijL*-{Dp_oVBn4#EqUt=xXT^G zTYq6~lNi7K_%t{dF9$!Lh@-cj45cx1A<0*oY%Ghw#;v!A>*#I5A=}W2IgYt)@g&tw zl3yFu$!_X<#B8c6g?f%Rv1hY5-@DEpVuT@B)=+?F^i1I1x-+=SNCaVABfOs>jdhvL z?9X|LH1p6WdjDOTV1342e7{*9r@PL9%qm6Up;P0~d~G)Rm-yjs-#nb&b_qV^Q&@k@ z31zSBgd!I^yx$p#Cq%@Bzbxko2W#Jehom-{()a@QBqvfYo$n|;K}ckmX5+p)0l4g# zE_o<58xtlM;}J!`>g%VWRqGoZaP6TpmIh(|GA#i=uM`X2N^o+PC}cJl;P)q|=>oB{ zpgrdl_$DRdHJSUU^~)2Vh`eT_Qf@G+%Dcg&JPH>3#nWwj$Kb6o@2USxOE~!N7S3Lz z$RCLhWq#h;jL#?46Q_$^xUBLp4gMF;qW&wG`13uD504?+ODHXRqCyh48j*mmVDfvT z4lQ`Rie{)wuqsRSgmX^DlGRby7zf*nrd6-QG1_V~so9!=V)p5@y?Fx-Uq*2L{stJe z-A**GOTqAG8P;mgJt~^>g+BOv1}(SwvExr1r85)Pu&=h2v7c_|x6GM`C>JFP9t&Peb<)E`|H=B~X6~4{!f#xy)=*9Wxi0Mipk^bO{({>hv z;mY&yc7+6pKUD=k*;u&XpGNFrx51zOcHSSCbF^Ex0*4=2<3qzM%*PEs=-+oNnIvZh z2OULV+^z(8XQBwAcMZ^_VSx6oS&jGX5iEA*6PMmtay9D)kyVv}wYo3xRlGY{y(0_D zACE`Ay|^$|xCd@|$l#4tC&-H*OYkMrL!*NmG4ZfJy6K-}eYyPUBJl%Ekl`qndxnEy zZynA{lIO4Ut;fx~b%>YpTJZ2XMgDE%!=R#!uw_*;)6}{Y=DsRNGphnj6sto02~N;_ zss%oDq+^>*EVK#Yz=4Egip?*!`=A?cxFd_N56RL`rcJf&lceI7pIBI6g#{~L$Sl=4a)Ro;DtliXjKFcpJr%s)4VfQk4*%uFC0~csu zR2I%qyGO0=OoI8rCAcHq9QOWH!}ASI#8hP;oSs~YXWI23qBb5*JXEA}noLn7MhaUD z(`cAsRly2K8`x~S3tVleRMrJ1`Cg* z<1m-IQPO)!&-@f2dfT7SYYh|d+4vKJ*CCA%d+a6kun55Q-TCx9$C2(Z5C!|}bn*rh z=)Eo9=$sTy>^|YbguI*t7nVnYa;}mv;w4tm)71evof#jSg<*%bi zG9v=%>SJJ*-DzMYrys3!Og+jnX+M;B}0 zZLc{nxF!i7+UrxMQbRcMED3`Swc(Pd!ve>P&d?{4jdzD@FwN7OX%kg}p8?h0c^)?gO6&H?1 z4Gmk1n5o&#w0eppj*Tz@murd8Grt~VTrLoE#|k=p*O1=T z`Hz215Q&YeCHY7GNWwnf$Fw2(BW(MTL=O*|K~LrmDq+x5o3nH=b==VnMw24(&Dsjs zkTRZoRw2wnjtx5Y&UJ7&HbFS7o62r*?!%Xr{dV6NQcERKjT{9k@8`j`A8e@M-hk?{ABIZ zGvSwBIpnW4#jh|-ZtR^ej2c{uaqs1DUE~2c)g1!3D4e=x=i!Vu*`&lr1v0bN!u}`W z=+dwo)RJZJn`j8+%W?T!JxL+^;RX|a3wCnTIx)Y%o=&pZZ$bg3Cf>#_--RdgDSW8cKd7 zDf25Ck6RP?+18n)QpJt#IO__Nwpu|?OF!=Y`HZ@TCXzATC-Ate9mtuP6V_fpFLGJG z$dEf!H^>a-@^|Ch#!ImFfHTfH?2UV*f52>xZ6^P8Eq>c}o@{gp7Kn7DQU0`9wC>$3 z)D(S5>koEO|Ess?te5??e6k0IpVvUgWNT_2z6s{&W--Z2Gs(E72PABP89w&%CvTjJ z$VR7L;%hB}yCPG_`RTf_?o~S#DZ0ll)>4E+%Y$hZ?;6Q}UP1fw<_S&OzTwffT-rmW z*ov5$u=C7jN=XDN8eCzn4{E~pJsjU_lMQ&)Nb~odTtkwyC54q~p#U$_aMF4ya(C5a z=xvta=dB6=#ZXI7>c0=!`!Yzg(_Ab#dXcUCorG0uPT-VfOYuU#w6Jq#Brdey4KFU0 zv5C_cz_TkI?CbdbICd}{e>?sp3SCB2lnH>n?IU==eaB zBjyJ+>95T`#3|=E*)YC~xV#$1+=$=AbYm_0U77)>;%4J|^}DES)lCk6{?6De3*cC2 ziuhkwDLlKAO<BYgrhp8vcrw&upi4I?k9JnFnsN_wllK4*pS>WKh)7Bb=MH9LkZ%YkhA8rL-Ghm&h>(?ANl9H_EUdO2A(xD*=&8F>_{^dJ zPmhF>AJq?GM!phdqP*yhnpJ4$(L}fG&u1QPl_%q0aqrioIsAuNbvWNCm!u}|MW;gr zWKNw7rk^?vpPxMjtE9z1m&`%?j~lVWGzKU3gyNTv#;781jlR34K!x53YPp~eiq#t+ zLB+E@ zoacDPXT)*I=}CNtpUz}VRT!vTbEJz7=dw2n@?p*2M6}5dz(p?ah+EbyZ26k1d4M^h~^Xo)1TmqT!C{vvK4*htjsYlw7*930ua5N|4crQ&PPkV$MI+K8wK zKgaXw?RPzp<=MezDA+-W-y^!TNE#p5=8(&J5qS7lGESYX5AQmfX+@#~5mC?tnVK4y zXjO()msrZr3MC<<1!!9Emz;I6#GYFvP%tf&Ry`1qplyyI6`GGeKdMk_q!G2oOF(&j zBD-J)g-Lg7=_v^%Jfd|1zr~s`EfGT2?)(T_W?ux$H@(J5Fdg1n+QC9i?*Dsv1%~IG zrjNr{kPekVT&y$;Q|o1g*CRwoj`l;i_IxhBF%sbie4h`+2SxbnT9U}0!0-4(=?C*u z`U^>ZXGm-(7c%uRe0*1B!9I4`OML$*f>wqA@?82z>N#ngm`J97yN`-Pw$d~ zC5pKG^G=+zv7W7qwFuvx#_^t({O*DB3wUYL|qltaglWq)i-m3vBGGG zdKFJL2vXqvis$5{*$+A-YrrV?KLs7T^SIsKpQ_9lB&$YM@zL)DyxLjD>=QXgCH>-I z^NwP=Y25_=gTNSktRzZ9Usyr1Ul*EgAVj4;0wT#@huLiE=9Tcwsh_!{JPi)k21D(OchJ5vnratiKxfepoa@KA70ztL zzmn}#SSG_9X)A#nA;)S@>~lmhH-2#Zn05BqLm=iJu)L!B| zE0-Oc6$jqNpXr*LGwDk6d}42H4`r@f*}U0T;oM|*D#Ur)?xl;3z*=kNoIVm zB!&M?LY?nv5b{U^ibsv$;_C_ItYta+Z<|aT-NxZAR~B0~Rg&9zs;q?lSd5A2Ao>w0 zIPdX9cY;=QuFKL_9%)A2T9C!}xPSYHTm9jmpM`S+e}u zm)|oI!9cqeMF_rn2g+Nm$(RQZu`{#<**0ZtzhV#ZNB2?OW3z@z44kdT2>3bV*s7N zYU2#_RStvg`c2TfS`&O&Lvnb^eEze}P>9twhkEr$=GeeC3^1vMmUUM6X#F*)TC|!U zeXpFj&vnMQpB%&ex*ukGxY5(oYUqyED7Ij+GyDwO%~WjfrWrj7n{*_80kHgI2 z<(nV|WYuzL|p9rc}6VuSh~P6|p3KEk5-MB=1)nL+$L{)bp1$3b(mppoAxghL_+! z-@kN0|9$pw?rdnjHiIb6&p_TlIdfrxGuBrq^8?9S6pn4diN_uAaA2?L5$+7=w@V&n zefg%wY9=%(RSkET+f#ksG4d*!!QH8s!7?yKaOh$#>ZA#9fs+QdJyFjwK6KLw8<@mWP80)wHM{Ase;O?XWqR?Ong?rXw(A<|em>mmRy;{)G zD@u>txsHA%(KI9E8Qpc+4xa@`Vx-Ae`buIt-e{l17nrFNC09o<&)*BW9IyLt%uYN_ zp2H$?6{XVGz`C41!YQpeM?yBj+c=y(&;cj+>%yD;3V6I*mD(OOgxMpv$r4Kw>=?3S zRffZe|Lo<=aN;${b&W#npxI1cGce6e*Ik0AOF)QuaLi`kyvEaWSwB^bN@^|zC zo)pof!57B~7i@FD_PbXZU$=8OG-o{6svm#{-7)m>;Tfb)K?2_FJ3xY^m%^TeV2Gb= z&ffdC6ZGxVN#g3C922D&)gGkKm$N<4@2Mj+7VoEfwfjg?m=uot<&P!>9VEQq{bK3O zm1uTv26?3<&P)!8Bwu~~A(SbluWnC8+k^8dM8=|IyA^3G=gufnrC2<<3NrOhfubg# z*r*Fg>ed=qu=6AtRokGr7~l3Q@Gl^D{Xps6sM)|v3}xT zsPBJ=OWy@Tu4WR6*)&S*ExI7>i~=c9$$%o&co^6!4gbk^LB3%t*?f2=UK;sF7MzHJ z@d{x~`?oLHQzxL&OO~OSUp}UPKSh>}i@;?IMe*>>Q;@!=0aR6Q;HwM2pt<}pyH@!z zJ^0`2*B+`K%fVXFd7pZ?jI$W zX3Qn$lTyd9=}xLsNq^Jg#&Qy_O5f@1-2;sB{U8UH_b} zDG4I4w0B_Vo>{O{;{ZHu=7Vp+blk9^gbrKJ2dtaL)7alGXyYxw#=JcE`Mi&ujMl}2 zgKDt;_=ar_+ln*j;j$juzaAs$lkDNc z0!JKqT@2F4``8y=10>IHJ~V&ysI~Z!NCU@Cp)O9z^w!}>_}&`I9Cf`6ThAFm^^dnG zrql_y7Vn|@WnYQ)k-x-wZ8F^5l}5LAP8RyDb7b$%*bJA-(+R6khu3o-lWdNGpX9oi zES)HeOJb~W`_rdTc03D~zOjHoTOT;%tU|N|JJsH+$LK5sa z*+E}UE`(VE9ieLC64)3B)a}4jdk5Ww_cx8lsqvAl(L+;MFBeLbLiq5Z>pB<+^JuD=4jF;} zgjemB!gi4_r2K{ua|9*egqL85ClnN(5=FjfK5LYc1v+g~9Gaw(C{DZwiC+}p;E!rL z^`#1NdKXHKP3B|um#6S%`7m-z*Yw$Czqd+(sGgPRRXp6^ywfk^#q~9Ok(B{bL-#Ehm(id57Ty-(d(} z5CJ;9^&B@*6O~qTyszu!%&=B3c`)KCFv>U$Q?E?N>lNC-l^3z>%zajFWIc%GNs+UH zUHIba0AX93=*4Fz=^KrwuxC{e#=3REo;fFRa=s}X%bAHXTt7wh>22blT}dh5fp&@h;CxDXaMy&!T=pOL_( zK}shrDivAWs)YHom$DJ0(zdbi- z@tkJx2>1^*t)@a$U=(8mqFAQB2Wpl*fVtOQp}|=m{f!ATYf2PKI&=3zhl7y)pB3Q0 zI8b?TnVvVPyY|4Qihgi-504Ax;q-GBc%*0=?n{bB$<8$(I-JAybKh{~CGKG2Wx(9% zo^6l0j>Y#Vz#b2sxK-h*IcZaTQ(_zn7^F2p#2D$sT0Mr=?eUM zO;t2qX9H7i-y_pLWnhTeR?ITC06!mn@b>kl&D+D!7v~B0aqmpPn3Kr=776yJ3SeZd zG-34b!PH!q*lwK%H}jfEOW|s|{jCU*Ry++?9?F6Kwj3C9%MU+%ih{!&+i>kJ2Pi$v zG0zIc$Qc6>SUvMPz1{Mat``pj<3FxMM_ zKa!RosmuyX2YBbR3)JMd!@V3Ko5^;9w#a?bDE)y<=U8ze;#J7R?LmFlt?*+*Cw-%R zO|W9#BBB1VpG5BdIR3bW1{n0_4$n~EfsS5%j}uMb(lTdRG@ZsL=}Ggjcbgu4S`>;7 zZi;wS#t2qe6{FX!@i2aeme8~~7N$&@M?~L6Ldzb?-t#UX8rN=t!VgRAda#J-R`@ZD zP8VC8|A`r$3b6XG1jbjWfp?7`?se|L<#XIYW}+QDcw|WzZhgT>rIynd?RU^1RfIzS zReI&laV%EYj*r8{V1}F+_U+q?=Om{=gTI(?-$zYcs^JXB({0ft=Pp+_bcZP-y8M#W zWzcNW0*5o+g4s<^xMQpZjbeka8(OGr8f^^+0Ob>U5 z?PcBQrjiY0*^Z0AXq&-gEqfGg$|R4}l~5#jDmp37BhHPFQDdeGzu+qme%R+h)U9%K ztuJ8iCAs6Ck%Kt5JeMua`A1;NL>#k6nvGwmj^$UqAu>%;xLwN!>Buxlcj949t_Cu* zBEU*Q7dEsP;flryL``r7%^!Qy2S2%FpW$r&Ifq@s&O!dOz_x?g$K-_VQmneo)<~Swx_^Xd#>jhe2Z0A z@1$)v#9_Z@7~YUe!hXLDSb1$NIXmVZR(Xv>$7oT$iN7R&+vpNu`erGfy-^U%s}qo= zV#5NnG)=@I~ z_%B@Nl#KL&23dZ83AOl75xNh>k|p0_LEh&XZ4&dOGfcKpWsO9daHJ4I=AK4n&3DW% z(e>!^FbTf9E3xyre&jCGJo@d;Bb4(AWN%dzQ7MT6`Z;10f@1TbY2Pk-RWuVllH0&^ z=P8`*XAK_L6JYpS1w@qGr1gGA5R)n)%wJ@N3L+xJwOxVq{lB-TJwlG0z674OQBV_q zjieUIfSW-zs(9)$AK!?e$xRDz#vpPu;w+Xhe3UKvYbx}aL7xqOWyHHea7I-MUgEgz zhd37M1j@r!k5Kw0cNaBP5aD0R*#Sq=qQKYv61<Ld`AAJpQR>@!b zWRpGgznMW?&wM3)qmHy8lflmg522^VmyGK^N>+yKgoFdd+?{jfIRCOOMv0cfsTMOFd(9Eb zpYFieO?DXG8IH9TW>n_^#mQb4plfvrG<7C|-(1ob!(w+q0{xNC zIevZ-UF&;zHl6T|~UTo=Bd1iEG95vF=4Cc zn^^2)seMotPsiSqb~1Tbx_yvVCzT6ojp9k{^+)viOjoRSDWI&HC(KAZj?pT{bbPiS zKJ(N>Mqt8(^A5p@PfFzU>a*mrfCam~jdaJ}PxS7GomjMT2wFq3aYl0zt_=)F zoJTaVyO7AKn-QJJP!j49&&YOa;D3+gQSn0}w&|HOTf9_-&Gx6rloC0i+Wg75BwwC_ zgM#q4LIQXnsbPdeJ8{=~I~)$mp%O(_YoFM{Gx79^6J6zvSD`e$XT#Xl;iq^>BPTZo3`u@ z!n-@xLvP|n&=GMU#gEU>xYg5fijfhhT(P3hE1pryN6J{~R0NyH#lzjJVYvA^=YiZ{ zfRQprL~1HG?lWcrX)eDE0)7UKn;n7QE)}7p`aVc~^ad+RPN1HvEPbxHo&;UxJVM$h zkd1#%ir?mfQo>&-35B_A|Jc|Gsnj(C3x4xYdFTEHvWBk6m*?@ zFv-e?)EB+N(G?viF+UDW#@wZCQGEH7B}QY^;o=fAoHGA+ZQ=M= z@N%X-KATesC(5+ps#`v&@`~u?{W_#*aFmySkMkdW&xh=f2Qkwn2#s#$K+*2qkRPIj zmfjzTqJae6wXhwm!xZsgpahnd?L*dTI=$bON$L-c!@SJbm@_F6kIY#~KH25LX#;V- z^T|Sze8G$SH#mfyN57LRts1bfAqgU!?~#X+uW7)W7WP)%A~;dFgGSF2AlWHK)nBQi zM@k!PvuY+mUo0Sa7w1M@n1FNltz&xA?&GC%a>7r24!~}afsn0VXw9=Aa^X%TqnPJR zqK60Q7Tbv!)1w6r!x3<{bQ#o5nZ}EBD8M5DS#ax43CT=YPT+jM8Jpy(;e&q46K3Kj~!M?;Q*i>!Jd``0@GD`yB+OM08 zrGGXYJrGPAHvXfl_xsZ0$1=%~uP!5Yh}%!jXuv}+4Tz|a5Juhhq{g}1;Qr!Ly5szM zkQ0fb5oHrl^79wQa%?UtFZ9P3!?W<^y)78?yc&;i^E=P@Z=p@n4sUIG%!Sd(8rCg75SpXB$TD=fdEh?)jI7~LX4{u)#Q z|D!ReKUNlgeilskdo{4tGN;(akLOHvkBk$(@3cpA(H+?GBol1Jh7FeBA=^yoNQovB`Y0~3zeqk* zFM}QHd+4ptEiiN7CVY{-f|kclF^uXv-0}D@$YVF=X7|$tGneC{kEsx;sDu0Sr@=m* zb#Qsj1Ntdx1~@v_gX9KxB75@}`aQ_T&Y08W-;gG_#ySYI51fE?Tz%=xBqI_LnvGjN zIl+>Fg~Gk7Pmo6`fP=rnY47FtMCfxA?jN6w`z)hD_m?Nc9Y}%WGjEW1+x^Ut?{-=Z zaZQ)X-{$1vlKMmD#_2knGb$(HN?rq0vPru z{U&#qUHB%489A?o|J^X5p^{0wV(B4n&gwIoRkq;Y^Jb`Rw+w^|I<@BBld;TbgudTg z4mY=KXJkCj!JeI2FxI_^$~(t_eA!Q4#?SAxUMvZnONPiL-5Q*@VHZ&jT#koVA0xXk z1Y6&v(K9?f%F-dA9?+W3xz?^?9K0HevGN{Wzn(AGEqsNE0sty-z#hDwhLjac424 z-n#&$4J%;h$7Jdirh@k+=R?Xz#G6J&7=L*YNcj(A>uCk_>e&w!>dQe+?kQ<_RD#lQ z2~5m+>_baE`eoTHZf^c6e9pfQI_u}d21P*m`2~w2$O1_(JlRouSXy*AZu< zF&LZc#@^Etkg(ZH(C>Q>$uP2@rAyzy$hc~B+4>aP<;M$ety~Sv>|sSk5{+<#(%e?p=0}4_^_}CUz-`=w!AU?z$*P>(!1~!H@~M!{}a>Y_Ycm; ziQnqj-Q~L2HZ>ZK?^{Nxt%gwHW*v?z*iXHBRp8E0IdN3Vgn;01P(S>QDz%i7v zIxB-|eP;&;2hQT|N-Ol0`%Et6d&9upE?oEB1+GR!k(>=D*z2#3W8QyiR5@Qt*wo{L z0S@7KBfgL7YB|6<)dlz<^#$`$Ydq-H-@w8-yI@LF6prKSn7wE0ann#R86W5X%CCC` zDq8}`%(yU6jf|j7hBFrJt>paSc0ekV;9%DZ_4<=tA`{>hAnF`H@3zu!!oVwKW%LPhbl5_nKpfLc|4rVslq0ar{u$P&gJS- zkNamH167+W(zE&?T%NLkkf%QIs$&((Hzv`_zGfZ~O3t^-LiBGlvQnIzuM;09 zw6@v=1-r*#;DbD8H0kkI8C3ybE~hyR>ex3)*r{1`hF(aOV*HM z`sYaXGX=hsegS;+;9IDCNrerrXk}p7DIlHW>lG zwoYbp*E-m=GZ=n9=_KN_lyT@r0@3r5fX~7Z&gC0|8M_X_ensy6P;!EU>kdF_!+qS` z;z2!J!s(;bV7Oj+8g9?K0=Kg_^JQhVp@}=27bT`p=^|r+r07|qlx{%F{V3>FAA)8) z%h-A|_#`loP%&|^op=iLV-)yhLPO|v@`PK(?)ZJ@Hk4BIBKqCpq*iMp$(EmkuG8L; zMFBiIyWaxIrpU`>;|Lr!63<{>BMK&bi=mknM0|KH+3)a=cB+?9&EH4BJ^UfE`gz1>zZitnPDEL$V)FUW zO$=|#!_F%PWcf2oQteoYFHA+jy*`4gsSePUQEsbXe(yJ6mQW4b_nk}&4k zb4a$H4nMdYlyajQ*t*!#!UM^$U&0?t{mtNY^&DKYZx8#?*9I+>eYv&pihe#-N2eYf z24zQk^7>-}?A6~)0>kX^_gp2EKO&&B)jH9*)Ea)@dI#5bNa8#n3I3Ofttj(a0n8o^ zQID@xkd)s`R1S`(wX*}6V?NbjF+QI}+U#Z_ZV{fZufu_yBXG?k0?*~0C*NZ#;GfDO zkgz!m$LEIPys#jwlI+3&@gSP)rNFm~(uJRm@%X$fo^iivgoi59!O7Gb#*2#(Q^k{5 zY9}ddKH!UwvKL}lV>wy{+^111r0A;oX&_-Y9xU5m)N~$8U{y_9$ms{17iK)4(?~>f z&W8PHTwaeM;k)6^4;!2~`!vojh=R?ouc&z5U+Qh}4#Z9GK-I4dTrhR6srNUeg>MV+ z<`Z$g+uG0MrSuHYEw#pf^;xjlaTCh?TTcg@mElz6Zusruh>#~noAo@Pe%oJqtt12H zACtkaD&deDdlEj58Q@)v4ME>{&hPQB1*R=q1ZHkr7IRw?b`5#M=7c;dzsQmGUMBsc_Hn&Kg?z?I)N)}kK^FzW)!WKf}kC*uKra$@MHU}4EWMJ;kD;W2K z%ND9GCgH0%4?O<@YMuy2-<5T6-u53oI({uJ;^wnAyjc%ko3GNxi!}K?_7c$d;wMa= z8IN~l5{R^31DUWt4&&Kbutaks%)ApvYBjx4Pgx87mq)S}MnypWQ#;vwc`PdLiNUy) z#^h{+B~jq+A=3+2L+EXDE=$K{pK>zcl!gZAOs)Yf^&-4e6NFdxdn1G`g61ZMT)A)@ zT~t)~=LR%^KeI?se0U8wn7a{~?e~a|+X|T7u>(}J{$icmIN=J#6fDu0%lGGUZDJBy zd`)dB{<41om|*;vRUe}PW{E(@Pq@#W>7wD;%uls_9T$-?ye2sL>H}^Z48}8G3(2y_ zsgRl=22brAuuP*HmE8pJSo9r{3yDRSW7d$*%>XH>WRsd5z3BevsUTJQ3S7%*gyPm| zuztb?VxaYmbVTtWw$&N<8Z|WCF$q2XYlK}@m%!#h5(awSCR_RnasP)F>c2=DhTd!g z%S-0?V{sYo{}>PTE4MM$1qmp3$ro4Tug>F53>bs=FDc=7t-TO8sTqg<97DM$+0=T#0*#~t5KfuG-F-b|=bbXV zz4acv%iIQIp63w#=kXw169FG)4A3{x0wO(T6IND@g=P0^=(JBgOnG(yZa({*VM~nn& zHEF^A#9VM;0ysxqB?&P9L5V#73YLh>;F&3J zYZE;ii^NcJ0?4LLhnE_E$Vs&V=F_+_NUIXiBQ77`oLV^qu|sm0Vrd5|Jz|h1G6ikEbB^(u zweZ)6%TKN=q59zqV9oRtTF_Sj^g;+PdDVNEba_7>JQPg^6sFJ(T8^f>R!)o7Z( z?i#H4SC0SkC-8NK4&i^td+~CI0(IZlPp5J3mfKoCGBtwBQnhzuV$>VK(^WdutMVe2 z2QPt?uL3x*Wf>MN(8cNH({bVM)o{|$pSief4O!W3023x}B^fgu;bhTUo?~tSy)Jbh zhbH_WpFU@RR@Gt{TKWaFmzb0KYYR~5#6VARHzs}H=HaBnu%yy}j{TBLi0mlXY5w48 zMohrY@pB+BdOUuuc*FAlLktf7MXx`Uqc?=^r1aENke|H}M4sJ*RXI`Q`M^sQt;@v= zTKTj`sg}!K_A{!F-$M3lwc5)(18lnSot3q);0KNK#a5FvtTxi1mKRJxI5q)}#EHm^AE~J1Ec9`D2U%}U;L#Z)bbSA0?A}*`y!HBEaPU6+J4FfH{KtdC z^0Qo@_A6`f6CpJ8G&!eGi4XKLXb0z`7#qG)pm%$@z{7T9*UEwFk38YmMgT z*3wFCfzPhlJO|E6)@rTK^PE4GH>9m4__SD4F!kAgg0J3IJbAY*g6Pt%fa0#z&WQCyg=u&~f!!?4RhAViE#%lx# zYnBQ^kMIO@TlVn&u5%M~<+urq`+Nlx|HsgI2U7WkaXga93T2dJq@+|N-1oWfp(3Lq z(xRax6)G()GPCz4A#F`5?s@Kelu~IS4fQLc9qqk;_pkf^d*An*=RD8(em*+uI1{&} z)FfmX?fY>F9hO}Pvu~B)^%X^UW_KBuT=B<%K7nvFVj29%41&&YZaD3dCmiZ^lQAl0 z!4Po{L^S5%%lk=M+p`i^aCeU zgy$z!AwF6asmcH>u6#?i3m$PZ-#w?{hWDwl`2(6NjsmwYF{to03g@`Sp`rFBEZe*t zhWy|8A)~-R<`8<8G7i*iaSulw;WP>k zbHxto_`!N0OuVlSqC#C547GHO)oE@FE4Cgjv%F!XZ`gHM}cM`v+$yhNl z9d;L{fZUoa7@WNg(sHBl%#H21XJ{|4Cfiy-X0hLpeVdU2Yc>OOG znnYS$tVETTKGUW)eV$8P&wQty?ccfA<=?rP_Y`6Hm=uWCNkg@TsW^3K2AUc1u%Xon zcb_IWyUhu0R$19kTJ@*yN%w-fq>-6*r3)j$LKK6skrDVrBMSd`?SY6lWq2lY4}Nej z!$-dg@YdB5kgP3)Q@3`*9?iG3Dc}p|=<%N0QT2nHerhfCNR8!Yw1jbfC0X3B#60fB z$yBQV{mgNNAGx10H<#U*PzZ8dk0b0i;FQ#8>SG{MD5!$2v?7>dXw z$a9XzQm0*T#5W!WPE7{0hk7{gz7{Omt_eS~jUj*NQg9`~xVLRNt{$`kJ2c+WoNe+j zYLy&3S$2`*bMMf;ZCzY_@qJFc=LOf(F$uoB9gfGFjPY#eJakm(pix(Ta&7^yxGOHL zoP3}MH{IZYDUQu-#Xk_PCVpnfKu+!Av+%+&6s zHE+&Qmvx@7FK!mDec+8x{uAS&xIwsQt}eLg4+Z%qRoMSw3f%SJartfsEd1+^Um{K5 zU8pOzJ(_|B=f-1&?eCw!?K}q z;D0y~B*~%Je`+{<_uB~L4WhuIPoBto!GyZXhk24d-IqmvQ+C2>mVn1!#bVR?XsoEP z!MFtzK`mw?NH>oGHMb&&zfppE6N|AZHV5aVE`eia{`g&C8UA|ihoa`6)N7m?q?itb zv=a}x@kx!`&%K9f$BhOWk(o_>JPNUJdI=bJ7r>92QkbkqP+{Iw&@pm=@s(5IQbI4C z7!=F-C6;i$dsN}}_+U)hu?ZG_T?@6}eIdhgM_qg4-P+ppJo}LvsUki7iS~T)57FZ( zTHLheO*Em+1F9?BFgkw*hEAD|uX}*}79}JrUmw-Zj2-vW8 z>gJ~ga$P^UN&RcN-h(kXmd3;WyjVDJd>gEs{<jRlz zzjM*6Ccr)2O!`7|4VUq5E;qvII@f)>l511FNEh@}(~x5`sQYMTG%~P+jDW$Aqtypi z{!QjKC`Hjbp>b6GZ5q|L$rimf+h!jY++CaT`jceXaK>FKET>IbRaEKob=scy)~@{i zOwrGCIrdSPPuZtDpNaY3=YZh|FR&Op4a{6pp(Qp0Eq112O>YX``#Bc^DtvI5f)8%g z@I&QIhOmB*9$uNJi-{_xXgH`v(pD(WtzS9IrDFeQV|M=3F&gV&#St zPPxL4%dXIUWI7ls9l|rchvA8=o;2Q20lnLlq4)1Usxh@6W;GAQ6^Y(@kP`WuosNqjVSW^g8#RBeD+ zzw{w%hz`8lIRe&L91>l6`bskI=sSsuP9n{h?B*htODMRUq(1A*=;Avm@P1r6UL2W% zUR}v(V6p|1w#UG0!|m{GX%wiFO3uP|Kkd}4rst|>$@E=wX-SzU_kEiy*HxL%@q!Yr zou~nkMhXz^rw2YC&eQ%aZCs+kIZn~Ei^J=&*kF_d{-SuW4U2~hTUsPxY&>U~GMbzG z!&a2BIDmS858-h8T<(v4Jhx@&0?gmF2+9-}K=Qz)aIqyD^kntI+JIcFPRzx)P5SU` z{UAJ4s)r@zZM3Lv7}uxDlkU>6peYkh&3mqp*6*GOE-sAO+Jwd8a`JZiKj zL+|zkc$OIpBQ0#Q9e{0T+*#>wC}#|vW`rs9Qs2egS9fQpgoAb(XI+?HvA zntlXm#6{ur=Mfka5sl9F!zmn|z>RbDy+I&T|)~CppidZd}ea5q&n= zkqbO!!O`?F+`GkT(9oHR^N*(BkS%F=WY!9p8@3vcTwRXp3L%)gw~JdG@PRrGdqh)v z6)<6lEoL7Vfqy8#p$2$|YPE$K&wQ z$zXN|pgqP08qD5O<;@-3k2%k{{WJb>ZA+IzUiC_R5gdS_+CgafTaT;zHKIYz!H#p`?b;&}Z5kg-<*D1W2UaJSuZhE-5M)v*m z8;T3+Zi>3|&q$JHUXZAasFa+@i^5mM(U352Cmb`0hntg>!GF05>iZ19q$j^=#*CA+ zv*{EkUv!FF?Tp<1yu0?hZ;h&JIb>2dd&*qN$r~Hs^OlXcFmeN~QHezT1Gnr{)~>co zsU2RoDJakG;&Ur}ebX9#{$~ZlS6IUpoo#4h5(Ns@+abmv229+ux$wp7>EHzk)IKen z&h9Uw)!z-dFL7hIYj;L-slQD)-%t~}(|H_?y!BBswdp%oS^bsXOMFk|hNxg%kUnQR zeI)IO7)YJwxKjUO18&YTYr5#K0mTtA+OVlP&d{-d$&1Zlnf4eMU-XP_e)FEI9?-*8 z&40~JJK>D(6P@5+?i4uG;0bO2O2D$V6i<%Xjji2f*kfjk-+HZJ^%-jzdY1$G@y-3Kd29z@@p`|zCl zVbmV76?`s2{vayOrY6oD)mLM?L9RLXltD)gb0Xm&5f@R|iLDxJN zg8wOS;j^kFLn^x^hhIx1o2$mqJM&DrmgVC(D>pf=>rf~6@Py12=lq5a+S*CKT`)w8 zV=|8JfZ?EY!U7BqjD^>6Rd(dbv?x}OSK$|Y9gQ!W6`&FC!VQ`MVoiuxwnTD(B$k-dL~O1Gn`J* zah@l*p3jH4R->!j<+g$FV68IBrz>N&=@2Z{vV+DEW6|*8c$7Dpj4}=}7S!$rOK~xj zFD(OM#x4w%)$LV60yG~;f&(@+Twjedl)qU^{YO;M=<=m-tTq5g2#avj`q8+vV-94R zt;av7*I;YpV``{xN>}fBQn&ePlElWbKfJPfX1_L#o~_w?r0$X5MNz@I(J*Y=B(zF1 zN8JKXOh{ReXGd>=(&RAcV_^+ttBN6fW;)iVrQ!O+IXJO5fR=wh%qhENa$nz$;Y4!) zriECcsf?eP^TrgFZ@jA|a=P4tyBQ+Q<4f%mJ}BaX*>&8GPskNDALok7qj5*ZX1IBJ zHH3(Y7)Ve*D-2%5KZx{`rM)wsr{)HC~1|)e>Eog}{Iunj|SI8gW$Th6*O15LxZskZd7x| z<`iYje14AeJfM!1-SRji;IW?WK zpgT2^xW|SP&Tf_^9iZ62ZQjsKH-+A!J+a?w|I>cV-O*N+akDir=k+ZSw|Q?}^ne_W zJW=6}50klLwHwb4x!XwheOyPywL>L;T#DeDN&)`5SAZER`Dh3+js60=R#K7RcgMH`Bp!0{T+kvpi zsV|=P?}riAil}RqiFLAl*IdYkFJq=d#84$rk*vb5iHk7#f(q_BoQ21>WW#Wiba+&1 z07X~d(;3DSAVAv?)-)-@jbeLFm93F@tUoB3_ANzXd#e!c_b zeW7`M4BEqbDDjGdn1Xl^Jxhc9le^GJH3^3fkHjlI+PE)QmFo$s=Gwm?*OoaK=Nl}6 z=CS_pYl9f1Yer-5aT}PctP4gr=E1$0CZI4-2MhfM;o_TSI11EJ`t3I7=+@0WZcv7w z(^H^kSPEVqnv8u0ZbBc~diqUv_s4{Pz5TL$coh{cs= z(YWt^5~kN%;rO6Q;PquRjPKXQl^edIN`^k%qyaJ9zbqS=y1$AW?lc|G?;3-_%SKVH zw_y?tPB9jZ8O{%X$nu*j|1h?1EE$U52G0!|t`gHxnxP--#|f2(NW=39>F(kF%^zgEz%otx?SS_OJy)@(FAF9P=~ zPB5%w4itBHbKcr-=u%Gwbh-DQ8tNL@gOMDyH7b|r#F*B-ytlOO*!uhSLw;?pO|L1f zwOg?j2aMkV_3fL$*)#^4mzqGVcr417kHnxH6BPNZgBahf*nM{$y63FLU28qSPsT-F z)9r%>A4cJQ^H`KihzA3?SlD|t85Ujl#I;X+;j!UtS$J+17_BMA_QPfHs{gu^;o*%f=a=JkC>kT?K`#km7 z`Gj&7`JBBZo4V^4Qj_0-bWX@ze53CRKb#goMEFXWn5u-iTQxxcBhIevvJsu=K04R^9Zm~65Rlft&Z z#nl@id(B4pC{@BIR1UQLesew}|8V6!$=sj;J7rEp83X?cqkSAV(S3Fxkz+b;ADo7|YN0UQZWH=F3d1*fo3ZAbGd6aufTr!fF!<|q@KTx$=MTH! z7cCbIbX$QlV{UWh3!A88Q!`EWeMkFRBw$c_3OwJH4z>T1Vd>&qvPpR>{UqN?Gko^Z zMHbm;JvtBW>E%FTbpg2XX^^9ofo&5q&|_LAR_{%v-&aYvfA7;c_nZ!{aMV2eTT1z& zJGN+_V>3fGGRwqc4!JPMBnuXVhkY4EGp4P7&)q3cyAtevqAE1!kK(;H#X9I_RDjdGF%oiVl7=FW)Lsdb5_ z9$t$UaqB_OcrBbD;qb(#1Oh9{P)%Bjn;sRSg~K*cY*D_XC;WbT__xvmSf%v|!RV8(e>WA|7f)sxWF7c1}(Lo!5zw(Y_UqL@mXrv;JT| zcL}_kzX6uW%ga0zJ#@|c_w=9GgW~#l^m>;8)9%DWPVFw}SrP%IMsb*WD;nv}ZTQ!u zKWxiU$9s#FQRm8F#D~^U9WxOp+RAFQ>sILdK7*R-Z@2SV{!f&Z(aOaXuSeyct>Ex0 z9CQ~ghp&bYID`Efc=1MG9ILs9j=nS=7FkTi4=Vv(pD1HTjf}eD&s(7cPddy?dx(csBP)X&+~N!h^Fp{FBahPz1+mig4EB6Q|$D8?GOk zjh{8V(78Gom)sqTtL9llz@gFbN!J}pos%)NA{Dksli>KH7+8Hc8CBM$z>R&$Fl=%h zEM%kMI8H?^C374)YCWdC>7)rMec;NGzuezV1`r(IN;kj1$8GC-i!;9Uf-`k7hm1@s zY)~769%MWgJsXZI?+%CmLWe>8z{gyJtS8wvF&U3$?ZU2yk+^f`0(_z}7h3gagL9)L zq}GqkLzb$4)~C+Gv2Ti_G!!ToV}sfWxcb-C^&v$xwB|4XOi-G3Mw9 zc)r#gwEgwLIxdc`lesP*%EfTK{W~}VnTMv^btGCJ9D`TkDy`J^f=DM1wDX#YIdV&J z=jVm6K+zxfRxd!cyoLD7ZXr&aw*kKW34x%^XSwE&H@K`K*{R!#(VVPkIx6Eimuq>L zzMbDnH~P<}&&GV=Fvf#(?Owd7QS3O11pX!!v`S=_Ob5QC9_W^)E#b`K#elY zxWk=8Xw}xK^zPh1$-~TdlFr+fqRsxcqUU!@xRC=|MK^zDNmgHNmW<1b=R6&}FvMpT z`2U#&BZ}SNYVJht*jQzH#LSr9`ut8bd65qeytD*XZ3qIN1<3s3F5j!A^G-p z@TplqY2gBHMi_7%cHUfO)l>*E_C!+Xiih33@KexXI}h6f_Va=t+8LgED$*)Qg$36# zP-l83u0NQ9Zf9l4fu>-Qr&54O^V~y8(4Aac5A9QuD@XgBQJPjo2PG9kdwMZ?(E>dG zuN32y#ppUrLv|J$4#DBtFm$2+9F7bT(% z%X;jYk+meXLXp~8<)fBs2|RQygqQPjVXE6kh&7GEsj{9~c~UsqY+ZzYXM

!&2x{ zTLrrR7J{3*KL)*Cj30_uV*cnn8vV13Yuvksb2O>oYIW49ZlE@IeULi$d5H(NSiBPd zTN?~szk}f3za5aRc$i-E*h`-eJxSF*uK{bjaJ;Fz2CGMI$1YP9jy<=g2DCrD zQhkg1xaL4(S1yiA%)(JM1!yEZ=SH=w;U4wx^xqK$%<7(rd8fj_=vN@fSp>qWOhCHQPMFzJg750~!1zJC;pV_H_}w0jRvw8k{!AP!j^71Wo-U?cZ^hh`e+xN} zM;2U;&oJmv8jKQSEo_-#g_W|obePK=T(ZRj_ZEq;`nGI#_q`a19IgXnmcYII8y4#{JwwH#1n3?~CG`c}UZK zP#fpJoO}B{8pYkEQ^JR1yr~|3%~FGzjRS!*HiYO>YcBq@HQf_Go=#CeE&2A?2+i+} zfLnIM;Y5C4=sUKTuJBfX|2E3O#S!J)xL>VwgP9u4iTlEBcW&W=78xU^sYBy2Jy7tr zg9ht4pj^8c<(y|?`AdSg(lPP7ps~8Y<;aLF?0AsCPFVTivptds;f!_@_au zt{iQcDdO4=8FO1wgSiE#0^#iJVAS|;4aOJ-V*Q#>{E#0Cx~(gr-F_9QakC(1svBB% zc;P##3;r-I!b{27urw|k9{QBR3-2z@QuP(}>;H(3i@#2vHEn?DiBZ@dwizdG+JU>Z z=40Rg0-)G(8JL^Tg9G<(+7IeK+kP7vCE0M%Pa}J~awfCexsGe+x!?PeM31J{ z)tMbIlN1kJEb8?SM!&uD;a}WBxFT5wD`sfW+mAnpmbdrgq`M|_oxP5rG{_nMUY&_W z<9Q4n*}yfFRa0ss(9G>~sn33E>_6HTMEh-Fle-;E2x{Oo)hlU%Rvk@pzejKR+F?(I z7(#d1!%f~3CY49vti`dg&m|I;=)^*O)-CGR`G|Y){sGq~se=2Yv7i3z-OD*Q9pDbr zRL*3II_&RK!}tOX+;~U}`*nV#8vWmJCU<0>l%2g?+Qt-!=$D2=ZX{t*P7-byn~9T` zf$xTtVUqJ+w7+$d6U$t@uMU>cwl(|d;J-%P@*75U&Yl;NJI$9R6`|4K zl^BmzYBAWAmxxJMbFlyTd^j>N4}yXU!B2B6mMa;+aX$l?JkA)()EFJr=R9}8OW=h4 z7r6jF^3MvG zD_dKKIrN8zov*o1*)b5Ju}iiWN1@yIU8o`+1W%UgqURtjv^_rvo93>p4U(U3`#h+@ zzG{e?NKxzzx`SNsQG*jk*3ZK50yT76r39MxGQW%XC1-o%tt5PpI#=61f-8@z6;--k z;D;!PwY@`*BT?CaKBM{Pm5ZMZ$JE;c6nc zU}`k?u6jTBBP$0;Q2};!=AeRR0p8Qs1()^ec-c+?AKlkLzYCq5{JuNXxb-ppx%Lrl z%XWpT*ZI?wsG;_}aIeeTEzqF3v?0EP@SQ z`B1608)g@zi*8ORkkDPnByNLGNSf*wOa5#!74?=*7TtKQQ}?rcHh9hPMkV{1sQI64 zcH5f--gT)sF)0ai=BCT`@pGcHqx#bKq3Tp@wTiwfj05NNc-f3J7F$N{!W9)8aP+Wk zaOb~Gutc&E$j)F4X$*x$ZY!WDVl#-ttf|duInGfZINFuUT~<%wRF)=F--)TTw*N^w zwK5yXO)rI? zuS)>m?15$D67j;sWawF*0FE^&FtT7MG)}ibx0B}h`Xb`dI> z2I`stJLDbE(s~W5%=Lh&N#3|{xEGH2;48bo;wkBx{nY+N#A$o|&+0V)m5OjLa2xwG zD~Mb(O=pP?TZAJTKS=3)D;C&wf*Id1Vf$pP;VB;J;#)8J2$eI7Sjga|WUJGCA@Ers zZ~3{CFT8i2m%sO0C<*kF);)J-TBbwAG4k)&fc>k4`_=~J+>B+iJJ$2bu4gOAqopcj zfrznH(kSuW=qh$NDTsHs%n?pJcVg)e+{uT4anc#D8konedHlrN?b0<59}2Btz`U;< z;eGqplc*!H!q=Nx3)jP`QWAaTu=0h?{*zzqd8boM;5%g;o{HYUfALF`EOcldig>wp$no$`?%)6u8?@9H$Z`bCg%@P!|{0I^JZkC0mm z8ss3I%ucOr6Pk{+* zN0yVx4|(Dp_)KW*zAcU#SV1lXWHJ9-cP7~~oY>ts0bAbPX0vrB2>uSUnXjcIU+~3< zjX9jcG={Fg*&YW;uvsR%^judwc(IJ@Jn$eH+c-ex4JqJ14ya>`WQ?PKzGL{=n}f-v z%YI~RjWY>LjAX0VB?u};S-gYZ&E%VBTfA)@cJg#`NgvjN8E znE2uYq8ye@vM%=JuQuKz8L^tY`$lW#RxQUuv*M=eMcvXc&y+S~^5puE#$QdAL} zE7`!#th**1_~9iRrg@ERaW5t2V@4CDM|#rpKF$1Et!$!x{SRxasbcf0zLV#7)yegb zD@mwoi;OwrEd*8D5#JU|hmkH@$zv4cAY=}-)CJ`u_#!^6JJ9nxf(jOT;`bb zH3>h2G`oBFu-dQ~m@`n8(N*|}O;(aJdmpB1F$&MSnq-hCZ1$}5C| z(9?o}jQLgUZN=K#zX(O&q)eFY!$wTl!%tE2blAD#sS?0HE;^go_!sx7gv2SlG zf6aR<`Io&{*rj=b>TH*~9q$lj=z43f)q&JIO8c*6E}dNKDY-jVP*W=9w`QE=KiVWisJk|cI(wce-;E|Qe{6_j(|+<{d=hKV zc_SpH0#ZkvE-P3?9lh&;2Gu9Bvav9Ec-d z3y+B%$6FGg76*r;LvG9R6rT0*&-bzghrWuH^Ig~gSyr{hxC`zWl(O0Ch6K#YWxVgL z})lJ}HGsJ+Jex*W4mMULGJJb$(>V$VXV4 zKAN06l1rTXTp>@320FA}9$i1_Q;qmwd9U<~!hW)G^J!LW#mnSk6Ufm;k63xuPFAG) zO8D>kao#96QjF(63ES0g35Ityi0DzIl;$h25kKtN^@0SUBG;6NHIb~{UqPPlkz>wP zGRC*f0se!lCV478%|4Y{k$LT(d6l(egc+@BQpmW0W@=*44vj zX>K5oe|89dk6PLL^IbweemjASN)9$VoP-yiuSng3U*y+J1BW-l92VeHLz0bZgw{VQ zLE(w7_pc||ZX73GsIr>PQcPeWg%hN|k}2yJZ6vGrtzt*CD~a&Wj)k>N zbhx$5ifG04GV_LR=|PDvInh|kj=c&7x27EF(({Xj-s~db__P|9);5j*vU(v|@W7B? z{a*$-yRDS05Z`4cjw@K>C7H9k@f>M#@nHsPPlSEh)$Dp!e{%8a12THLmV+SU23FlY z!>0sa6Ml(HWQfFf!m!ro5^wK5%s}qGu-I1K;nMVV{HU>gNqgjMS@t=bm(N+w64NQM znfsQv>#XBP-CNCS^ds5454+fi884WcN;U6&>7v-FHBi_w^#{L8=KI^xlgPidxgwq5 zrQ{I4`vi*}v5O37uwsWolh_!o>E!B;b7Do?LHtaMD&ACXAv@-OmZ|E-vLAKkBxB}L zsga$G{j4eHP`P;^QE7H1KHEEmbg8*uJ}-vU$Xpxkm$d7b4|+?S-3AF4E%ljEf)3yM zppn-;bDtmF=pj6PxLbIo^_AVdmCRZO`U{_!t=4k4Swx#az&7Qwr96`%iURQ=qZ zv$FSOEU(emne5s1OZrz$#y=bReZkm!qpSeGT*yR_PR|RDc`YJiNA73P};{#cg zT>;BC&k!f;sWH#&5xigJKvq9xA_??7EUb8XnO*j`6ei`Q3nx`&@9j24v69R)Gj!fD zarn^+;Y9j56%=wYMc|Sw5bUCI+0B}8ce+JqzT_fl@Y_HGeXG4 zL2Ri+sea~MGh(toiOsy=%`RQP$(lZfFzr+yX6@B2=nTF`RHv=r_mA%iemU9C zA!71v#^)B2Vda+6l074t5g8$4(xi}-FCFC9vSu8VIn%P5S?cc~rytxShth&%_crnciFE+U9)7|fB&OB-R(ybOVA|>N;+NOz$fonQg4eG9 z*cE4n?mawBo>U?4{+rRIac!_%sD@dqe>uao~g{!Xcu{>uDt$YW0G6 z9g=bGR~Cz5<$jVAU&h{QtYs!A{MnD#(QNOkt^5>QQ(~MjQLs|?6Q&H1XQ~hMcr~@% z%y#eq;?y~S?G210i#E#GX+!k*U>Rr3#@UyBYRl)pJetiPOKV|dL$|O&#@8Oae~B!| z`GmxWydppA`_%Whb|jCw)ye(WDa`s*r%S?=!- zyz1_4taOS88}#G3@JrW?2t@;!%hFipKD#K1mxT7N zW%@66v-q>UMAO-V%<{`7!AmJqSBxVU_13Vm%F#q4*$|b!4rPaHCG2`pG&x(Dz&m{Kp45@x7JPLY5{FkMWqZ$t6 z?6*1O?hljtU*}`l(2h<%?R>FRxcs`TyG?E zsnMqgnIou>@a|0Z-?807rp7^%aYo01l`E0C=f)Cyk4wV7hCY0dQn>JI-$C+ZzdSK` zX2cG?vnNyji)Al1qwo$Q`TI*l3Hhon)}OeSxK8^`l$Sjt9~K!i{M?Tf?K>~k938=q zdo_y#qxT9vBagA*d3S_uCAEA{=~#B&AduYhpGiKD$KvlFL&(Ch&salcu6U=zDOOrB zn7k;=;%y#WB+5354u1Kn4)Dl~w0Wipe<#+HS+SuqzwK_4)1X4GJbEFV7J&+`T4%d1#a_HCB^J%ZgHGLD?Rbdv1O96}PJjx$|Zp1xeah79~5<6Ff$5%WBgoV`C@Wa~(pwS~K~m#=iBdGnt@!K3i0EEcs<$?JWg#ngFI$)Q zn|JD*z>bak!8S+z<9jY1Byu_5$*>vQSy;&{=|wkfexZdYJIfpd^|5K>@zyEQ@=hs> zejLCmmi{0;syq2RXU>tWT0Aic_9Es3rn3j>a|F$wtNHzJ63AbV{v_br6@Jg&wd}$K zb78Q~d3JAB0{?171@T#tOY~d+5RDz?Hhh6+cLj1P`52ieSUTLWp2i*>-$0BWB=N7PI`cjxUmw;4Y=UPX zze@iIo20AFKJ~vUTz$J+#t#3=Itt9$ycm6wv9CmY?oz%uxc0u_}<@YhJBQ}s{WPkZ+poSZfcTmCbspbd@9+F;xpufeyU(!R!O$LpFrk4_h$dX#YA^p z6dO6|8~Oa2;}vb{gnsT<#3ur?$=k_m1O?#@vA(HBwj5bQQu;h)&d&s5w9cKs99$@D zF)Je(?UhV*n%E(I z$gHqxcCO?!9x$T9;WbgD*y1!lO?MKzY_^|V)Bh=R4qcJ?=QM?QBX81S7e;zq68J3t zX{5i~DRyM_de$p0Vxwk`Bx$;X$fDE!^(7=zfY*8K=Ekk0IO;8L9otG$cV^1?+_j`# z+nBf4?&TXL{|PPKLB#NKJ6qx!%b$r}$0}#+5Y~$=$X&XUts8DZhR1yt9{fpQ!CCg~ z*3W}NTJ=nFAxOU7b(NmO^`oUMT*%sq#*He6Q8lQ60X#2V5dlu zuws=v%f2AxTRz_4$1XfgggL5w$IR1wt3w^%GB%VgmGu>hKWaMkq4P-L#b0EXe--PV znj!df7Lv2VHgV*aD}vPkcd^x70|vp!C&?}&Tvpk`R)5}Z5FUI0!NcQ};2iy0Y<)$K*%WSLhl^yMXOWC&q>?QZ@~xz$PbvBG{1oZ;G>Wtp0pZN&`v&hi(j(hfj&wNJ*FSD$3HjLh}=u`mGxg`>_OM>Lc8=b>Gp6Xax;)AZr?_l z1veQFZInZKVH+P2xJ^(KUlWdtp742-)Y!j)980_QYb26Do(}Z--ignZl#jL-`iJlnX(xfbhQ~ck-)x@^S zls!~SB?s9P;b6#L!Cgg{v9Tu1TKL2?)t2!WRg(EVyKLEwvOD}8AD-N}T`8DXS4%r? zw_(jvj^A`hss7AbRnp@zRA}s144`UfR_Oe{}MPbU8cA0Os zFC&X`$O9i6*5WfoFnx7SyhxHxhArAeQ+}^vzm z9XI&?!j4YTnoMWzOT$m$noytfj`N&ViItPb)7*&##K1Znp1k9rsP-e3&ej5<_zSgGl9~H7M7&7aXUL zhU@xjWI%^JM%n1{vp+wWy~P*9=Wio928p<_l&|d{qCmVP87#{quzA2WoI5n17%L7I z?_HM(RogV_{U2|opW+;-+94tK>HMa{c3gm9;Z3RLk6gOZaF}@4y&!_CdvM*-Q&{oz zJ56)d##4Ehh^+f8k*H%FjyeD$ja0M6f9JdqmCh*yrkIDQYW5IFI z1`<0*1)GKrf}n@fIH!~2Fy*Jb_|-E*F)OyrJOS#M%I?8zN%eM?Moki z8BLvR-RT)A14f6vFgxHhv?*tR@djIHEmB041Bt|T{}`CKdK->i;6Vp5bBOrNPna*8 zDe~_tLvt%-xV`!Y(Uw)j??5&@A3YG`TmwK?ODdvSee}4M4RuK_;&Tz_ zlDBo!$*&J3x=6jeK-aw;f4&>`(hpImU=YvIEE8xTB9jXrfS!W){ASdgQO;u$N@!SOsT z?9qgMiWA9QIeD?holO;t41j%RBM3x*5JxB?spzbQ{+4m5X7mIEdrsnspj*Vy?!L&eZVsOC z)sY<@`h+Ww8in^a382o!1@+DslWB7Ne1DQFx};giv;#hIqq}|KLh&j5TxA8h!of1P zjee-4x&#ia(Wip6hq(FCEc`EVq%5;E8ve?XK{av-jQ3On!<=N$dcOo$Cmff4_!J9s z=ii`fzjsn*@@!Z+_BI?o`wU;^2GHAO`SADPTU2v6DLPg^5?<$BL*J?4bXiU#DDEtR z5#{$Fb--h4wCugm>I=ny!?9r3uu62W_ahzR-c7f8@ot#5_te#T)TL-yQ>BaQu zia*@?8H0)Nnx|+}ShgstpojitO~if=dWq?k=};k;fIbm6qI=>qux->Mnp+W#LDMbB z&neTWjo=p-o~nQ$cG*;%qfEa9hr=yq5e#<}hy{J}xZ;;C+!xtFnf_hrQT8R)xk4MJ(z-h~;@OR7&o@p&7Tej>G*1C9$ z8op)I++WY|jm0Q@v*{+??fef$cx}Z8Z$t6qEh(;<{F)q44#%e>3*pN6J$Q9<5O=sG z5#r>Guw~pDki%V`m2~#jFruks2c;&l zL>j?oediRCJF{s_Awvc-`)aqJnG4HuTm0KTF>X*%J8V(G&*GEe6q1gSCq5u zC-&VN1WG-{twQ%aP>>;t;Rs^nQG8)jLPWLuf za>LiXC+pQ-h{)FOWJ%0^GHl)l@VqoooNfMvTe$TGI4!f`uJ#(xw0U0WQ=$l0Ctbkv zPac2+&jxy~e+9-XW>V(IH0TX^3;_~`4mmm!D?hEqsFy6*Y|)k7w%r2Z4S*3fS@2Un z7jt#~z{#D@NO{yi*~fBkG@I^B!}gAcUnBvBEIENaQ_@kb)<)!J(Qk8`1)a-c~2-~lL)S#5KYhMz6)Vo6z?+D4#y4Up0&?KJyrU%pS z7NcpK2e_*KCW7Nms5K-Lr|AD9FUE^N{iZqD_)m*+J7$S{r-xyEPN^t3>?#>p9*3=; z>M+q{7pZ)cK;`9)#gp5!p&ELrLs|;%{!vB(qkPHuLDrL0)U}`o8zcF|5JG&a+T`$1KrfH(T#xD5lXg?X?*a%O}Q()t}6uhDtONsX#F!&Wt zRMyX;;e(gqu@3`Kah(sB@m~n8o^TZH{BF~UPvxYm%+m4Wod0m}>u->l*^IjuC6W40 zKD+K@BK%u?6g}G7xe&cY7+&Q8;lu5oxf?aEoa5nWa#$E{vPUR%1M$ zi17)haP!(K zxSgB>Pkw5MbA1xXwWY<}B-2RfwYiAKrU9ZWi$~GpC1F&?=U5su5162*u5fDU3H+gz zh6t@b%ykpc?BA7Urm#a*1&>Mz0jR(APYM$BvaKN5}Sv` z-0r*zOc+;$oA&RdtsQOTH2>BV`~EupyX*&!wnaEL!w6r^kHtOOO?X=}n5xZ4hr9hZ zNnn8xgF>8fV1O$oNutPt&cWiy!W6ur#?hyq!8Eet8yE8^24^4g;&aDOh=$7tNo^iz zlkv{6WT{CM#Kc?Sn-%=qK)V5D3>J4|_6h@l!U$i{*+ zx_;C&S=FoKoOSsv^5s@19eeUEw~6Nj-O|#*ckX+TSl@<}Yw>8U6(t%u|E+XeWdyyp zPE+h~u>k(w(||_?A)MUoNEi|P3Wumk2@NddVlF@BO8?p6&A=N(+p-RW_4Tmv;y?(@ z$|mw>1HrgONBn4I75R0@iZs?5ihq1EfxuA@aQnJA*wFBoWK?Xzo(Xfn?_(mIP0@QNCOH|kg8 z7qh{@dOw9;s|f1!MhilwIMcW{zoidTjLDpHkMZ6PUD+B&5Bw}t!0O6baB5nImtMYv z!iVJ|BRZxCrN5yOY-1fu2tjm9c=!a2u8P}Noz zYntC7@6I*D>8?3s;4yRDvFI<=8CwGlm7zG{#B11dW-jXctAW6ABqWFBK#B4y3~m{Y zX=(BxrWbMhvv*Q+D=RKC+ZgVz+lI^Z7otbI0QWzdN2hq$VR!U6@OXYwnqz7Y?N|z9 z=e{CqwxrU7ODJA(OoiuW9?+HDM*;&RU_N3XIlaXIuk!rr;g`d4!KdLkYOtp4*H9zs zVXDRJtjclgTYDzOFc}1!>|nW>6Z};$MwQnwpd|4`QDPL_*0KggyEvE@zCm<+9M8Hs zI#Cul>N2ja}hiS_Yij4<}XBd5@@AQP?fBDD3`^zUPtI{GB z4@aWCYy=MaqX$cC5{aABc9_}xjFa^ag@^Z)QKfGiu6L{57SBpMhD>;bTvd+6!MGx#~FanN&z32PI;^{6H9~u}=MOc+jc;QGD!*hBW?6Mbj66z4`~xkn&QvBQQg3G->?8puzI0PoHFaN3G{ zbVO1(Bwu?&oZO6vTe^gE8J0ogdmDur`xG!zUP0DQSn6JSleAqvg^jVpz|v43n}Z_o zoJJnkIPN|*x6g(z5o3t4xh9Pn)5oR#(!;~s-=Ozs1z0O+6Mb4-fO!-5;@dMvsQGA5 z$a}khEI!5e;YTu1erpvyf2El`8h#!g`m4!;O^c!buR8|V1>uPce$@P25Tq+*bL+EX zxifW-uk=@PhLvL1LGIsl7_oj_}~<7+NFfgbAs`MK`mW#_c*@R ziXay2RzZzh9evt&f^0C@3-aSCx%4`IZ?-Mtb!TT#-@6z)+>QZ`0J3nyWZ2H<-k#AL zNqkOw!=$#!ve{1#L;pwzQhY2MJ_h%2cQ@qW79Sm&d^r%GwoXUW!#}C(LJne9A%?xN zfvb-+5t0IlV{kEaST6$A%dK?WLT#p5U07e58||7~8E^aaWgz(j{N- z5q7r;+7!-0n{$ADbJS>(;TZ5z%n*GuA4Qa7KhdOS_5HETNgF&PQ@fy6@2W~0G}2)E_2%rdTB)-XiqP~ne(p* z=WGqbC7Ok#HMv&gpOr{p*jJo+b}by^f`q3Ruf&dxDQcYuM#Os54)FhbjyQH00}Y7BA5YHX9~C!v zc%lVYo)e=(ZagUvzu|wU0dRcGa$>nIhL}JIYTmqsgI5Q_)7h&)r9(|NKXjL@B$i(Fwuk0Fs)=iZbzg>j|#|Of&!r3y-E7_df>k@A2 z;`5-taFJ+g@MB62W<&l)EAR{h?0G#FV?9=4Ub`yPcRAq*ix0FgtAs8XI}tnI*Mrrz zR(fE{FK+qCSaQmD7VRH<8~rZ7!eyH#fKXXY_HJt*Y*;Z8TGToCaq<hLOVog_!7z0P`Y;kdw8L(fA;4I?`N?xB(DeD-SYvX{u4U6!eTq&UsGT|^) zq09Ftl3I(8u>Zt8;g7l-(AS9EmIYRDu>C&dl$Ue*V?IJl(I6=GcqOt|4w2d~-$Pb< z@%i%CSF)K;jX0Hv8}!8+BXUY`1fRa@B{f1d99>#W{+oXVG8c}8P4_x!>9aFX{XCP* zYkQ90s+-BYPeE{KP9FU{O@zx%8{jRAX1JO00*cROap&ev#0P_x;itVPxtpt-U_<3i zoH!sKKHeKk*dNI#v(uy*HwAcBF%CMbobc|7G!nV~6#QwCVyC>k7+vmj{x?IY=aiXf zq#g_BHcy1JlSSatUrl>_-_z9M2)aS557&05p-+c4K2aY7v}6lb@_gDU+v-G3*6QG5 zm4)uF^!Uu=%iN*<37BaVNWKRg#RqW?L~oG|o!LG-={boS|b0kPGbv>m^rkuw0 ziVF1FGjrLT*nHT#vy1p}>5u@&aL3MAGHl{)Qp==r%R-03wzXEo{NW3lMJGVJHNSQ@ z__wiGbGW;@i%b7+IVgG+ih5#1!j2dNvQ&2$di6NNjPheN-rfd>^-M?ehac$shdH#Y zaygWXLcrvZ9^Bt^iq|pkL)*?S;kUQbVQfY+bhFRFcT6`~nLm*>{CUrf2-uF}Eqsaf zt@on;l6eN;gzq5tW(zkfFqY&sOyD-^C)4-}Gm+Bp#ZdidEbiF-2-Ql*Le)YYP%~VJ z1&(jg@lh=BQbNqGEXTC5<0x}p0i`#_q34Ug)Yoh#(aWr*OKRj~=4JC?bMIKtdQgjA z+T)@0N)g^2rT_;v9VKaJ+~LE6K~OL<26~;;aLl>aY?1CP(0I6?=LB5GyUCd_`tv)I zGwC~+ISGk+S~nfPY7(4L{DaHo_k#ASSY9jP3QbqzfS8OCPc93Dyv}Fj%6DTZ@@}NW zC2*VKW~8V9K+;>PWm*2pt&iBAW4WC0WujAF_FtYWA%soW9Qp z{Oe$OQO6e8yv-XtjH_YSrR&u1PXR<+1~Oh=Nmdd5fedp^$4{vn$*J;qxIgGF z_qBW;o$9-rOcSO-McX!5c8$WUd}Zhl2}7mP=kbm1NU;2;Ci|ln3j3#oV(!*mP@jGu zTy!fKm-5Y+_xsO88E368Wq? z5KZS!#*U6YCe5^pXeDhUce(}w>~AB3#aVoR@RiIga|X}Dx9LZ|e@kk8L!QaB@NPN_ zqk0AKxWW)yv`o-Q76VxiZ9v{+Bc=cN{=C|XD0l|Zep`8TQ6C5mH!5hQ=}FRASqV4J zjS{^mxlZ?#bmD^3c64BmKJguKnLN07f~5YuKz6LCrk{+HK(ICyzOa$-GdK|D_6M+Bepb?M)ic1L?Tr=?5sE9*G{mH6T812i9R*^Aa>m)TJ}Ym z_jotw?v_qfNhoZxpJIQcOC+yc-4>Pvr@bzmZ zj?($fCG9E|}thfc-FN)L^Wy^nlShi6~z_5bp>}v0&yy===k4qg4T> zSZpW5G<_g_qm)s#jiH0q+rg{Uf%t+OiUZwq(YMSSljJ^$y26xDKFR?dc+GKA-9(5x zlPuEG9Vqj)V=y>b5n)&#*;#yvT0eIX1_a&Uj?Jtk|17IT$M0_hhe?0YXE+P%2D?dB z=Nm!pmt}C_^&gS3i7lRKJWl2I89@E`eW-r#J^8tKA=YcB5~XS9k*x`!#+rBGu(Le2 z&pv>@YfC|Q(-!D+Dx!6E5h!SnrbjkO=)zO_IQ5nfJ?3f*a4s-T(jMx<$ck4$o}r?(aWQkm@(%# zRpZOJZ&Ub>)42vEf6icB{UcryKfg&Hf$Y|2X!(xfy5843FVzv$ zzds~rRE@;_sZvpQL@4exRFM6&Nu)LPV~DQ9Zgkx6iWqY*VEf$V=wy`xH=Bxq)jm&J z0*>K&b`econI)t*2ZBMD3eHWMisIg9)ai8wJo!Jfbq`0jj9ZGrdrExoI2l|godUZx zjWEf-l(v3)OM`Y6gRxdDe&V_Fhkf@6|J`=v`C(IdmfQid!rULnY&{9H-VTF%Ro76? zubq5bHG#G~_oRwTg1EBt?s$ClEI8VA3#-nSfroz@&)w0IE%kJ#BP2R7oPY5Dxb{2# z8mA8>0RoI#p2+Fz4~NN?M@9Z~Su8aTg~0f!cw6II=eoj*7cQ*{WC2a@JXxzg|POEl_}p<3Ewm;p(`? zb`Q+?_XJC-tl6IHhvC@*jyQ>yz}M7HrnHed|ETmLqgv|Fy z*mm|l_3Mzpu=pi3eqtfrcd3E)%ZG@Bw?c5~#W+~Lwg$q^-sCo~a6*PrK}nsOY;5NU zGHS6u=UkBhWv-`*=FeZEL2GlM-!cspKiEG^Y`~)eSo50P9Cj2EPn1AlouSM)ioYjn1k(H#b&-dU3)W7a z0GqOGVTHc$aX|!=4G?b#N2Z?VU+*-gRzSPM2iSf<%bf*a|vjCeSX(fLh+a zJ802w+8dIB# zzd8P`H;qcJ`H~?u{P!);5H}k{iaa`Q^Y!mPbf9IV=Q4j!xcr{fjmf}c?)hZyks_|m zcm$U7^GDyg1t6>Uqg}eI@xh~3P%etXpSMc!`}83o*RvB@lmBR7KrKv=nc{SlRC;1l zApZ0?M>oqAlG{}s7&9h`_i0wr+kuBsVBf|y-`k6Y9|}N0!v-cSih>z8YEX9+?+^Cg zhh?9{vdstNz-;3)taUU2*GN6Mk~D`py&njjuan8d9Z~q_b}ZVKu7cV9q2%gxZCLjw zm@XM7B_$5Gsq)xncp110N|(*Sk3FrVBS(ejqpqOl&tH=V1D4X%n2yce%%hDlYW!>RX1Svk!r}U&cH9V<8kU+2ZXod@T5f?WlSEB))INH zwV;pc%+wHWoLoX`uf$==l43|n{YW=>#DSk4q4iR}7o47rhTV#?&e=EUvT?)V{LMmg zW|xY%T>T#1IHMfifASHR3J@eSHp8i5joc=mrPRcz1hr}|;!ur9Fx#C((wA#P!HJWY zQKZA&qAEDsCyHp_dcmdb$))mJ)5)4!8zJJ)5E_yHhQ3Z$q9)yX)bp!0R=(T@aQ_0# z+I5w*u52dT?q5cetE#lGeF5Fa_uW^2>d4-;yK_%E$MatBY}m*|;g8~ASh0imt$Yo^ zhD?+?EI$F+)7@zLAQ3e;-An~}x3T|^BfT+GpW?}p@Hl56wt5-jPWuABDs3l?8fnlU zW6$mRIFaPNlER%yUR?jvB=RgU9;L7nrwyusV_$h6*5_+*@JuN--^wD7H$T9w?!WM; zem?&G+Xm-sGN3?wo~TrKU}^);pwk+J{yMYN53nB@`PNRl|Uei^e%oR`CrqITeWB2F<`fPt!?_NI~|^`W)GF+#1a4&(SAP zMGVUl=-icFcquIlDwEx5Rl;}@+wF;2eYW)F(O7Nso0Y3XKqK5xA)c9`+N@Atty7J-(;VRe@CPcr5lgRSbC!qiP99Vd+ zLD=FNjLSc+gzSiN=COGlu#qB#oUfv zGo;5~CF8oImUwV~FtwJy1vhW}p(n2$rG881!Cmb#y0q>ajb>74YyDX&|6vgfQ0?a~ zb$QaIo5Wb$kU)Jb?BUclH)t6hLxPE0fm*ww>&(Nz-pHguaX#>3>`hQg zbtBIw?Za$w8+mehrEI^E9-Kcj74)<$$&PQXP52jRUvDc(jV{BF_&8^48XHm^|w{XvOY8r-8w+17DHqTLqw& ztdApp8qiSbsy8iE{>=noB$jC$-$e{ z)$F{wJg}Vqm3+I|hkaI`$#L5r?oR3@JkWiHlnhfs3(W}dia!H4TE>gp&Rn9eTIX{M zf{VahxtZS7&_L6xE68cJ>C~T_0!AO!L1Wv|=xt5!TZrL4d_l^AO5 zBE;+u*6?7pIZ0eDk1=_>cs>6>+#1jiTUSmHj|g~2m|s=oz6HWJUY9<6QX&`gBM3ic z4v<;7nqj*F|9)hmCOfOt&oh!+$T8#paGY=iRVN5Op6Nya&U^zn|dbB~8*h z@)sMzdwK(g{>Ghfnp-+85Gv*$h3C&NQ}xLQ$*kyou=?_68gg$G_Fef4YL7OE8VBe= z<3K+g*>2)lz*Qd9tOTkgTSXI31pdHxLS`W*mutY1h0a~e{nse&{c@4D?(}B z%2RxeR)NUe``B>F5~AlQ$c`ip$7;I_=rGSfr}bChes~x>-erlS8?@oFQ#GBpR06Ra z@1={(CN*E0$lAnQJiWAny!=y78xny=n-zoZLJ?IpVZi=f6ZUGC;Izc!kQk1fyjq+n zqrw+0?{$*DrS0^}D|z}K&zi1sc0i@_hh$HcGJfDWq+UJMU}CaT`fc5zxy zqQNfkFRp?7w#k_6r4PPWhhs|132>`7$3qMIsVF;$l$)et)XsZ2VvPb;_w*BOz0G)4 z=84|kyTRwB5)JQ&fvYPrFfyeGoLi${QJb20a?%uhGXFQ(c`gKZbOzGRmrU{Wy0f_V zQ3h(o*y6!iOJKp9sW6>y>*{F|H~4%y9vc6i%-DREIOVRAtra_<;@Hy|9dMK8r zdUJLcz4E7*=;q$1O#LDfS~pQ#p;ZmWX+kWw3C4%BCyHC&^Ly&dKWY`KAX{=Q9#`&F zCf+`B;%)riWJ6V9(l%#YvAYFAuJ#Kh+6{E+$f>ZRDVrB;X}e*MA;wJvYNM&1YeJg8`ECM6}kp z1c?HBaDROeZgyY9Te)uFH+vO58tx1~X4PZobUyRun}uxD=_XnhaSVR%NvF>gFTiit zMjDYjnFbhm)6$MJB(b4`s~*`7`=m-#>Yqn0C4~?R`4431Rd-SQ6kfysd7#YnSP!k& z%I7}1XdpY3!>$en3e*&kbVOjs*QsJ+^PXPu*5YTjEZH(@5#ZTHFoSR69@u}R&Odiz z)x>oeo4OeMcyUtd**-$}Gu5cQlk7asie@uWuS^$+zGW*f^$v)GR&ErLH+l z^D@`twX;3kfcXww?)5;}yXqdgomLc|ALT`Fj9Erfm?7fh2d|Ox-{;V33n%D*zZ-!K-I?088%gMX4~gI%D@VhL=x z-cQlsGac(0OBp{(6$O()dlG*(Wvzpy%YTEMbv~z*`UK0DO(efwMMA83CoPKMJ++o| z!FZB8*{rq(ZMM6Cc1wlm;u3dKKg|Z#tV+iZT8Y>(CX{&pH&h&~H47tTTGD9}h8z|t zi_LE@1FI1hVhyDZHX9hTkHKE*IfEzBaer7=poimU!lYCCV%D zJsyI@?{(4Rj2Y-XSxzQ&y+LP%8* z0`H;J#vzGs!Co2;Tfc3mRf`0i?UK9TzV|)pFMLlTJ4=PUJP<5LR?wtLp*Zy09N6>a zE~=E*V6Rja?>9<7WqKtj1xSQ-(>KFs{}9pU&5v-?uRq*Pqno@2=ssR3^oQT$*1$X? z4Pl{Sr0udm+5)aR)mN4Ta{@p>Xo; zEPT^6T(sBp6I^)rQK}J>NL5a@)81F#=)JrQ(3dK~=#Ra8HcKWvNLz>>T(!YziUSOy z!H_$AI#-x}2sX?tMDu?|WVgd5ypwE1oV%s?FQ$bqk5WUQ?thriP9le%ErD+yDa3gu zue}K4InKk4NHOsO=Sv6R&v`lVdslbz?9ph_=+HpcpOFabl`J7|S1_OF-~?T|@nqzKBjl37 z9SF2I1C^(&={~P>@Tg6fM)Yyg<+DXt`X`#+hTCZPI|6^LcutnyDWZEdJ#fyV1ZXIZ z#T_H7fmIHmSy_ka!gZEpx-=I}CYPdE|1o^?IYAsBE5pj=MX+W39?|JzRiGYX0e|B~ zASXAC9JwWrS=*DS_s_+2>N^|ZhF{MJ^4|4Fi{&YIZG^1mfgot&PkX3e!Bx-R)iN?^HkZ{UCG%OEQ zMBApN^!?N?_`-TIG=%{uWh9aH-W=z9&jf$}xI-psDbxQvK9Yk~m!a;YK4=ul;j8N^ zr0(2kSkyF}1oA%k)Iuw$@jM8hj^$C$88?YuNC9db%)!NF#ZWu$3`y8-gctSa;)3A! z)Na`+&PX!>J?@vFdc|dIv^d&yo55-P9GUa6-`s?2Z3UPrpfo0lBe| z+S85^8ghef)Vo(;GTX zIQ7ea%-~Tr%=Zh&SjWm*cB07}*5ti`H2>`#iM3L{U5QNgs2ZxvTY<*&VurjU232 zR;rWBej0Ah5U+an+atmzm=l4)-lamhc_~cHu#bYmjsKYK-!4nGRy<==QZ0qm>K3fh zqbp3*mRsxw4^8RtwQ|f(H(Oyy={t#^-aNKnQA;@eZ8-B{kO{jd@I%=MUyeD{`j3gK zmy-&Lw4@(59bhDV0^yWpVa&mrFhNSC64Sl2m$h~XW?B?HW)z0KXAji!oQ(BhfOpE^> zfveYZ=4{OiwzK9I8&KB5wmv??$Qiz7b_Ge9+^`i)*5xZ~$;6M6hN0t_54|=_)0KFp zqhOfyzwbZUVL4x!#Ntsx7w`S7`hk2ll`&;gjufy-9_>;A2EoXc2 z3CkI$2rl~UKe@cnRO-?n61gA~~mjVe}dj0rRFrif|Z zT+OyM4V2#KP-AcF7zoGB&tMd6!`Xb3V!{5MhC+kCMU01P9m9Q97dGj?oqqVtEcTOm zsswj^WhZGCv+u<%ta>`fjvp4o#GFtU&RSZ)JUDD1)crk&J?yL|Ws;*>ldh4WU2G8LQZ1om-OOgQ$kHAAm37OEc^>qsTm;G(HCe0~TkIpM{=PEZzl z&MaVT-W4*f>y9v&m^YG}#nEN`1I|x33)7KUyT%G0%~>aXb^w@4p7&hp`$7_MeXyjY zXBV^cpg%LVB#IgD8NoBxq|DLuU(D0l@7YN{8>D{89fFkQRjhi&dj9{sVFT+YO4Yxp zvKbAb%&8_Vp~trCOkR|l&`hU}F&XV3{dpyu^(6-3UCo5Xrp?G6e4W|9SX=n*KMm%!hKk_Lhh}Ek#!9B$&Q#dE zhGFkqeZ~|8KVpaQYc}pwHZx>-jwGchm#ycyn^UhoEE_ennA!ccmhrwnQ`&ecm@(8p z!+LyjWKY@7Vwt7hdeQEe-kg}pzS zr*MarSk)!8X!%s(8k!~hCwYiDe`rjEgRusdO_x@(Bck8jLf)LiPON+TTFTKq1 z(B879sMQkXlMk7hF-?r_ao~YWx^~>e9m6?RuU$i*A#p?c%K>BY$8lDFJ_%o z3mK{2OTpWJf0#e-{Fpe)XhB-}Tz0fXQ<#wA%>K$UVBcx=vhAwHf=!Rsvu%NcnBIwH z?EAKV%zpXX?3BuKM&fgp>3$V1NxPLQ(XYN)=JB7qK;dr;yVv!cK<}uQR8>t^;D1$7 zGMUSitk}%6Q??#qEz&+S%YMCOLwh4xy}QR*{i$=M$0lDEs5nHkJBOcSUnTt$cuGqc z?c-yFb6nh*xH;OwqJ_UCC$p|gxXd+z`p7B51A|l9R<~(P^N$7WfqozM=i%37U7BT* z5x7uN8dkvU&y8UH@)FB7Hs2~s?bZ{YIic=WB!&!{&*}?3c@vV|C6Px)p zwvxV?62@p)cd}jyN$j9y8p0msolLuDH1j>WhtciVlwLc+2rsKAuzSw>us`B0r4KCc zGB;+nNXFPq5Ki}KoAL2zDQmBMSKu%FCV02@C_8C&ilECNo3Tv4#Y9|jVxt#zN%TJl zv8N8Vu=$GeLameWf|Bzt%*Bon#`#YtJKQ^5uyWl(R!2o%+H^hgdVJ$d!Nh${WuNV4 zF#2C#vognMCNSX$J5M@BGFW-2v^{(sv;WXG_Uqde#&qf|!7!fTFv8^(`|XjAECSWa0xNJouz-`hlt~tm9f&$%Ta3?DMNi(xd)mOpKcmyVO36Z5(aJ1b4eI z<)Z?bIt*bx2F+%NydNx8p0GqasX;pUaxr z>jQ9WM4vPi}cOFh=biYhx zPySYsdd;0GRo<1)9DFfdXfy8!d$N}2PnO(g+2#_au&d=7KaXac)?FRMxN9jhLpxmA8){z!^S1UdUq(z9*4=&0=nWbyeA}WWoZEkqnX~gH zJLI>n)T!V%Ydf`1V4?`pNj}!nr~BQdXF`IRtBoNnkTN!T_*|*6wKwy*j>#19#7z@p)Xt!cb6Tc^nta+W`WYH;G>8CfJ*(m=7JoE5Hna ztlI_3H8mf3op)p81EMV)SJUf&%y;5uPb22W!uZ$Duj0#t?8d-*-%GgNs0 zP>JNio;cR)=?vkLJztq8qxzWi*WG2}_Fs~WKs)Jki$r#Z+mNzFzLy!TknhY9TMOaD z`K^Kge+8zn={R$FPs)rl{1^|fy0S@U<=LIK z)vW!@FO0IenY8w*oX}YD9%G-D$jI$?mn4a9F!2l4u>D%v(y&YC*~|&L(ri4fCDtGaZo^Co7~tQt0hyT;^3 zC$g$XdL=$uM5r2VCS2#L%zVGGdBz2!cTA#I7vnMTe-vGJIF;`oHbS<9%rYx`bKcKG zC>2@lp*^IcsX@MFZopzP_UahBt#v2Z zoi?X`H$|bB!%A$KbpbOTEyNeyynR{WKYS@FNVLOc$VsVq_}DxZJBsXYXhBe&5hQ4jbzp@`@>u>kR(zIGjQ|pVw}H?L21DiG{)%x{LOPG zqmkpxzbnr4hV>fy_(>4NZ2rW0A3BQachqSAf?3pXRywPFiGeI3eR8l9h$t}h>g5i! z3RR`TZ-dazY(AaW&g-lF(E`1j3!vq*GEKgDmAU=<9CkgdXSEKwb6Z7qXtIzFbzEhR z=aOe)%9~SQ?2w7U2c_7~=fXtHTblk8=g@UEi>XJo4Icet&gPx_iJ~%+G*&6shYxQD(xPIA?%{IXLO7_#!y&Q~XzQ*+%rEH?vw~(A-6~Er#y_xD zM#{M7Z64ICbBJS<7f}sb3{M2x(YRHgj@TxF>^m0S#_I7&w?1tqcbV?~Wn|8T64@oG zL{^y^63xa-FjY&C7#`FlTS+)*E_DWfiy;{9dBmt(`UmQ~&UyEr@1Z?Jn&XgH2U~)T zXmp-AUB4_0FYXm2HyySRqUcM4Rc=>~-C2%r+S5?$+%Jrl4~7q}T=;#6Li0p8Jl=5} zUoV`%d3Im1#+S0!LoZ-cpd6NZUgWIhA91#17`BUApq6{>3np;ynG1Pa;G~lZ$)2;A1pgAE zM_58j7F}RJ277~wK|XvqHjQ?+t)Y313RHSwEjE}K(tgePRFs`X11_17$8tJEV!(zR zk}ZdiOP=5XmLF!jL|B)7m=ET$nuPy-08rvX`iu9F*UU^3Tx~-p>$j2fM4z;MVyH)x z6&V*^L!u4TiFBwsvKANES!=ft`9eXm_p1+(*^+dmGmF(yYJuezfo(3f3b2B)`%pl8n7-z?~9n@x(FfQ`322@Vp0J$*MS zNmU`|^L!b)<#r4n64#&wpZoB5pdBXgdQV?%k|qT&MQHKdHZbtjWy1dUva^)?F{5N1 zvEaW3Dk8j0@r%Xa_sWKISfGT7$;`x;+6-e7_MTbCHGos05`>dBm$lZ*W;=2(;5pBe z=)<=awohcht`EYX7I_gDi9d#<@cC4~BoVbOZ0WqPQPAe~+Wks7h|;@{VnFy)L?d~s z;^~Q_Z#U3uOJOE@jm;eNdy|Eqck8p>(rvrLB!Ty%)@3mKm+a z&oNg)>OwO*2?^6)?+w%;>lZww&P3C=8j^TjkFRFi;LaWMi8FJ}`u^N+Ol`+KX3x6k zI6N~PR&VG6{?!Her+N{UZI8mH?F_YhDZpszJE6K~E2A{-3b##LII6A^)UaNWwcjoj&@Fc9MH6oj9-a*qveG-?Cmdt7!SjjktWbo9)E0UZ zlY}pjAS^%V~UD9CW=>At^gb z7U{~-0to$?9&0`Qu`4m}iRIp(SI&u&l%uD*CCThhBizpqeaYfD zdurFk(?EzRmxsZy?j6}ZP3uwZmpg*sVab9KuXr+pgl;VBx zXtO8vt>DM&YlLY4-##Yr$|~v?z`-Yv4&(YGGhl4$PCE100@Si~qQ#%@LcUBmJuE3ylF*#I&gawx8g2ugzOU z-^=Qe!$AhHW?ncn*QYX~=M?DEKE&>d_fX)u1y3f9V^GHxcKJBsM}-J9emIMI4>h20 ze-p}o4#DZqx=?87HKt~( z!hGyg4MYPE0XkSHN)II35YGq~Hhs?&GRBp~kU8^6%56EKuwgdwh%IMUoiibVd(C+r zt(qkM6CZKd8iA@?Ut?`u6dEmD&FxEmiM>{e)IIkcR%-9VKL2P4Z9WI(%1E)AO(eA(i z-fLv&qjwJYfjVJ~a}%^WW#Vb$G48t93$Srn8Z=vK)2o>|7;T|}u7!r^+xr3cEHA~m zw{D?IbUZSWmNc_hm^z9H(!$1Rv|ei=oozUs9$7Sp?(QDL(bcsO*!Gn%_ctMRXO2Rq zmm@8BYenOpHDJD}Kf30;h2KUWKta}*-PvA+xx>6Yar8VK(hkKZ_Y&9zwsVNLoCW!z zB~5yHm_my?i+ZwuQLWgFwtBeJSbqny?FAvh^El+TbReW%&>{QF*8vk=jvuC6W=p1< z5Gk*A_|fwLem$NCYqnXC&?j@qlaL$Wr?nT3X=xK$aS>KXl;hkqIZ&+;#WPoDvPybU z@Lc*K6L;Pc9~g&2b7x;#v+(aH!C`T}MyB~nAdLBf~8i(a2QuuY1DiMw;gmkVUO~{%}C&!ei z|5x5o`nlS4%Yk-$mFa=Hy5WGOA|&z3VaRx1#pH~{qTk6MxIJ2r?$zCiZUqY15;2dS zDqKm^VG`FZJA)g=IK-NMG>?X#&TAiyocR= zqacvQNB*K6$(`MW4Bmm(8#iErbs@IBt_;64HT;6lxArrO*=GFG^#>m*U%>gM z_ds&P21w<-vd?|TCA$rw`$rD@!*d)}Lu$D7d!HcJQx9$bTZN&Gi-{xKgZ7^`VUOHu zb_WkH>SjzOB1*HsbVplL1aM=fZz#kvOBVo{9M>PRzsrAT}7Yu$PL!A8NNw_I@D;13S>A7Vn(V%X(S0VbYjxkJC)*|*&$j0*p5 zh?k!ZB)J`yt?NTsy*pUisY?&ErUN5C12?wkf%H6GDiSq?RDTI&c475wLQ@Z|aX3g}{6~mzE6LC-*^( z1s87|m_q;Un@TOWX26T#WVm>F1}^f{qYZ=a0rS`4GWQp_=8F}j&+9R+bPbjr?#D+N z3y73U6!H|d z0JWs`sAAC|dcN*}vRx}tqAtI2^z(`_169scAoBJ zl&6nk0G}>uwDrQqu8)kBt}VM?M4bAMwt+~+dWOH|I&$qN@HKxf-ZOZNQ~SEmf9+Y= zc`lR-e&sOl&MA--oKD6jCCI+~{b07{I*9+bl9tYqWDgxuXJ)LJ4Qn4=1pcv8%+Al@ z*#9h?IWcpX-O=)n4e-l^BQ7eWV?>AkPI%7lJbweDRI|Yi*3$iMH<^`(XVQcab?UJ< z9)7_L?vp+-dUcuxees_s^cV|q?yR<=-$cde60vaz`qNMuvOWN<-pS(2Y#+!pZpQ3W zN#L1u69*PBSUp3DQ|Gb>KVA6+y3L~)A^eBgD*71Q4oNe&(&wY#5d&tyYB@5*ijy$I z8kAlX0rTG;gvLRBx_=q3D|~1vbE)(fYuf{aXo}O?P3Li&u`cXf(8*qXeh}XLut0ad z1;E{>2N^}P=+)y27;z$rBV{wp&H0!B2Nqu8B=G$MPN*c&ojZm0tn_9gJ0wVJrYazotC) zb`k@6!;t1bhQ7pV_W9|Dz^2V2={nDF;rT(v#khxKYLkO17njpVbc;`FP>r6*a?{chZsEYBi%S*A1BLjfJ4))x;391Y}~PP;}n_u3I8Vx4!-h zISbB$N!tqSG*kd*g|oP8r6IVGS8#rJIe=O%w$Ipun}cF;lsk)a=FM(AL_?px5A zwsy2Idjvp>pWSG6g;kEzq|LiIgzIzww(_o*yYe3%NSH>9*5)%o`DZZW*;z*M>msZe zkY=N99Kx^qb7-(Nhy4Eb5qbg6}bTR3}h%S!ibF5-2GDrAbk%% zk)EYQo#tGH#*Ah*`|%{|to{jh@d?bsE@wuN{{bZH^U)1^tx@pybfP$AIvKt%K=&@w zr6~#u7$O?Z8Fk$RqeG6&p0d;McI*QzbNhumbNrxs%|*1YI1P(zRA`frEUDUbo5Qc1 z0E*C*GwLwH1O5vZj`(hq7&hr&}Dj^bn1j^vUzsIf3(mpaRw&8f!z8WXaKf?G-2{bE}2NOYl>&=`0!RFsL zVfVU=*b+90ILHvWwp-wEV-r(WFH7%Ac)-0RS3G=w3UNw%h*hE|;P0ml4BuZ56TULU zT%gKI?Hc78wC=>ij)y?ya0oorZh&vOpD;LW9G$N{W>03}rJQ@Gxs92RC}Gg#tMndhLx z{$9NnXSRrt;LcX|(XcqG>Cb|fE9Y^~d`d$Hqao~D^owTo;r*$w8kJczx_c@t9!O-b?{j1M77wx;99%)P4$+?X)~lwB zig!D~@rX2u$?$~g7+WGDIunK3*Rh&Y!nm+cmeD;T4@IH{I6tS1t9UYjO*s_+7djWw zDK82kw%r80Lfl}Dy*&NcF9M+r>R78c40`9}Nw~*m)+ApIJ{_mHJ5LedP9@wuUeBtuPz^vddQ_!_FmtFs5#RNfs0#`ZmMN;O?oM9NrA}UhzBUQkN-C#35W& z5@4rg)`HWTG7LBo%4Qo*1JmaUaLm4pS$?UPlYVO@kMHKAJ32SvBOg(!W#7VV>zt2$ z7jxmp8dWx>sgo7fnF=9)yKudwAqwjWu`7di!HueWOx;@pn5upb9PRf(OW!u=@;MC3 z!YLr+@`!m@mc(_z5_YFSDnlr1yO$|6 zx(b$qYQ*cyGkDnxDF!^O0ra9y z!0UpIxIlUZ{4KlyU%n@>HHM?OqBRViTUIl>7M@_fbg42q)h4KEq=Q3paokG}-ol%J zD4ynI&kX%{i(UCNm9xDriaT5660*ECLC&xQ{wo5IcKO2$@A(7%xv8wx;z{<-%q2|a zy4TFn-d8Z1-p*Q@PN5s03XpRjuR|kegZ0nn+u*Ef8_fFq8^IIkaIF^A(>#sC--qD1 z<5h?sn1tQmZ^MR=0tk=g`3??$XD1uQN#1}f985LgIL&wuFZ~CZpBwW~qs#?IE-k^Z z_0zdFIdhofn_bvzD!W+C+XnA$36PV<&EVW;$5vi9WBA{u!@Z7HUf=r#OyOaukj&S3P+|&kvv|y$4u8r% zo;nJ(TeRW$WgBLnxF6h*k3)9?hjVxNR7S#1KAk7E;Pw5 z8>__{pBS89HjYczb)(o&A(|ECqE>zF!WWPl!V{7GoI2{^`NdnBE-L55u*p+Vw&1J zSQ^3e4sA|_pgVTt?V@Nrcq|W!oQGJ;gRii=%m?#aCagu%;#kKe{IDXT9%jy;g2FvX zDCkgvVv8s)`%wY*zii;cw!3h3{2W-H=R@UwE9JEU*do*l@@C!bwP26*C&)^YBT|iTFwx)^b2Zt7IbE8D@vj=O zD>?vPJTd`JsRLSX_h8dbdq8E3CH}M%q&^cJnB3B5J+CDiLe`5@bCWVA!nYcR#Fc1H z!%Eyj55bS$1Ncp_is`AF$(WT1Q={D3Oj7hWruD^PCWraV^H;oKtSoKtdFnB6-CO~$ zQZ&(-$-;Bnw8%ISCl1?}V(mm7Zaw-KBdYRH*`$-{jS+**^JkLoYJQ@6z=Roe*G7vs zub?QTh%Nd(2uiUf7%_60O^B5y%^L)1N5UWW!C4O0Oh%!Ioi1}D<{?CLr@+i#*Ktv9 zGxT=9#LA+t4BUAFvsQaUeY7dHemR}Qr`f`0sjGNn=oDkHgfH zn#4!tEKIh>;7RW>c17(MoaOly)ADTLsF5yt{izuTY)nzdK!B0nJsqY7zGma(B+0|& zYV_yAYgnl123zzaFu}#2Gi%>oMj=)hr$_RW7uq+OzBdbLc(M`MrD#H0(>awhhPA2j zWI6i0kY)aLoZxwm+-UZl%c!?+7&nfp;PS^WLFYpqBpDT)c__-_v3LrA^H(y$xdl;VzSW)Z=X;kX07h5b%G(?Jb4iWeUGq7dA)2955ImZ zRbZ*B5%sTLhKl>YfI@DE)v=^0Aoczc$mk2uE~)F~-tI#vWSj;Xhgf#RH3jV@wTS42 z-<6;GgHhT+hNk87=HbjXR{GKfG`h8w=T{8Hl#PGjdjJnF4A)?W+I_sdFBDE|eFBO5 zj~Ho}b{yZaA2^kYzpR^EKL@QZ;Ygmn{Jd8E9AP($nxbX3@D(Q?W zNA9*>Ez>*Js^FVk9&A&0^EB(1v>5Y>5cXCNx`<+?DG%%Ou<%BRJF|l}PG6BP;nNe=ZIfzUM;|yg~Vf5_>+@z5_2;2Gw z{+u(XInS?iZ*c_3+K>rWIkp0Kb^2Hz{=F411$|+w-tm#ej^>yqRE`b%-I*c&W>}ob zh5N#3sGcf}^Qjx@Gns|S%ManU`z1{E(R{q*Pzete;=r88q4MW5)KzL62e-ST;Iarh z{@+d}!cT@Y4%vc5a|nsdai_)V57;z0Hw?Wx|C)~+_v-a@rYBFBF|AVP`AwieK3zPeoHR#mTY)s1trVGO6ld_(B@O)$(_7(b& z_#Y!!AS40bYqvvgq$ru&x{yqq6Q)--FJ>DI4p6n_n@Q%w`#{^|x&P3StmS!s-_q0Iw}1FGHQuN6%pA%x7@2&V$H*m#2KW?^qR+Z`gP;8&w|j6Pa%#=oz+?)ynmv z@2U;RuA*L$)h)pIgNKOD-#naGU`HdBa#`Dr{H0n1_;%MqY`LRO z=kr^FkIxVMExCs?e!`5ZbQqCxr8cJDQkq%e(+;Pf%_BFrdx4*{1ygWf3%U1ah?Nlk zgf}-%Ky|Ax?e_8mxk5Uv(V_Y z5p@-2S&x-&w4~UAJoC^e&w2O%{op#H!{2cdc$DIKGxBu03YA|R#o6=6 zm7e76Cd$HXnE%*|c_!#dkC$v9e_y5H-!Vy=@G_J(|DFy1c)ECFhaOD3yoM}T>q+f& z7l7ANQIaTZO46>Dvi&;?nJnozI381r$)9TR8hrp4cNa1$27aW)XFhehuS#qyG->Dc z7^0U9=-Uwt6 zcVAN2`V#if3xXNf0*Os}89OMt73#14!wm~lVd04*RMEW_UQ}l@1vw6+@f*-rIZn88 z;ZeqWs~xPpqE0f-S!2)QL|55@Yo6o_xNzLPaE_c z?LZUFON?V$IE&yvS(sqGN>d>=mllEw1~6+xYwG#uHM$(qZj;J+1rnU`g; z@L6g(%zKl9{UtLvlK<+sM<)tFu=5%oO4P(DtE1TRWjop{uu@MZG0K(>^#AK`E1DSFjmGpQwp%Q`Xc_3$b~;Y8t`k} zSMGnc8$fukGCIGR0)=-svU9x+U^EcmL!vrIh%+wso3#oSW0B48dSvNAtK;MtudSUQ`*ZFBFinK9ikveyZ=Jz0jeb{QzIqXIIs zrZ5?+>zUWb3n4td6b}%6Ue4hJIHd5C`f1TvEa3}UySKBA>KkCQXcd@V{*76$f3ssJ zZ=m3)DHE~P3QB+F;2ISlh?UA>U2RKw4Oa7$81G9KY@m9q3k9f0o)GlyImn2#%rlFycFbNZh#3iPIkm6d8-xoKq zTqg$KctO-Ze?{SaGjDwG z(g9QFm0&bapF7bb#OghJ&;CuXV*UkpF|)QMvwHkDIF4&0+1VvMkm{5O$BV117o3h_ z{Ac9C=Zb#Zz<(1aILFzIXFTA@jRQFQEFE2?l^wd`$v!_ek8T+Yg*RP7 zRL?pWQuGyQgGCJ!qHz(`rR3;(xfz^%tEYG{D-AS_*5Lt>F8KcSA6}JHrZN1Iq|@;k zW^eHX?fG%Iqh&D_T_a8fXC_0(w*o8=Gw1%^XbFrlA9_Iurp4KLL!7^JzGT~AP(-nl^%F=FFzqgHb|J%T} zItXzFE-Zp;H|J2zO|N-;(g;4-U=BkkdZ6t@Idop1iAOfhWC&M_u$N8Q+=}_^p}gf} zdP^j_-v7fivO08E??tqUQsS=B`B%x8uLm`WUZC{SnA+&`laIsS(emGP*zjjHNVph* z+j=`H_U8zA{9vIYcL~lhG*rqTl}+5oX4oFrs_f3Wad*zDw(K8;twa`*N`RT@L*xMPPEKDzS_4yL@o8)aVb zU-u*oJPPF8kef-oH{3yk-RY351MG(>qHr=ogceGPuvPOGfzeu1y1u$ANqKZH~vJKIw~G2!X%;!rWU@jPzB$yjfSdm(d|BjSQJ6-z4T-za-5%aSk7L z9D`5;Ykc7&Oa0c%(&9>9mQ)5u4mrFc_Nl%~smN8#JP& z+hjxqQ(M`k!Sm@s-}!XE@*8IE?OwR^a|y}xl_o0^QjkZ0LTNT38;#`1>~CCHI_AeH z^87y!u8UBeJ--;CTVm9IN*q@0T1H1Cj&P*?lt|4bW8(U62l+7d3RLWy0#>!1IPvBl zW-8AnWjYU0WlWwXE!tvjmfFKe%}i!}JyIIO7w2Lx3cvC(5DJuHxob%{pQrAdyyF}TgVcyj^2 zy}F9mpNP>#(?#*_DSosczR!HPsYW>d{LK3EJgu-oiGB-QOwBFkf<{+8cmCa<>>is! z^fWtz9wTp9S9LkekRte`mcX;M9v~gt3$v&UvCK2z45hpQz2cQbL(P_K4{O5!Yb&a;oS!*&;*p7}Wt@tss2qP4pg0t;1V!!_t+F0$v5wBQ0rBcZWtJ;e9+?$x> zqs!UzA69^LvJScD*^eze44Q)C}TW+UqHiCD|VZnKIT~%5$_UFs_h;K;@ZWK;P3{NiiWU0rjNVg#8Eut&PUgdlwsQ^ zRj}WD3!a82G9&XMKn}k!i7PL_VPo~mwhe-ixVx3%tIWlc8+PzT+Kkq|ECw~6XYfGR z6xR5C7r;hqWaCm9t5v$>nNzWOu!M-r(k-h7DUQ~=!xz}SbusL`}Z}W zXLB$VwU*$43kFz_Bt{+^?g1M69v$ECaEYcgnLD)_q%IzVF%1(m-62iZOQl1L2atie zD)8h5%X}1^2Zv<^$m*xtalm*86qn6~uw!koqp2FLvX$w`zu%10$~!RomLpj7r-N#L zGyAZ9f++j?ldE8ZKL@Su8-385xaL`+3Ml1{`nd3T6SRy1y14PH!=wBj4QQ;^ro46V> z7I>j;>oicwse!G}?XmCOD0)xG;LIEPR8{&kURrXSAycc_YZ5;(PSzi{*jF*H&2*{8 z`ziEPMhR|kJ;!`2=!ImvS&)393iJIoLYYhoY?3)yN&m!h9jj(yee*P0RLWZ?Ru3>? z@B7f0=glackfdRH471~s2lQ4J!`V6JjL?!V;3@A9E9XvN&iOu;On0@e=6%n%lpaVw zbQ%Z$+XD0RbVz253^M0?S>whJ&~_>e<*gqfxl#`6w9lY(Muy|n?h7E)v z_kk`)@_;JU_DMt4yvy)%iXWt<27}16sl*^U4Q$4z5v42XkQj3v=Lh71l7cVup-PiJ zX7P9g03RCT{)Gj)78rOn`x5Hw<5%$^DAbp zYQl7jD`-0}joYS~4p%1TfN|($-c0fy#S%DSj z$DnV(7siGUf^w?}`E@=WPkptfKeaW;=)FU9uJa?44{!vtRw>p~~`vOn% z_!hKn8X)50i3>sj#fJ4cu^Iryi6MNB8|xxo5hrODk%M}6OX!6N_e{KE6jWF zgY|Ef#|vIZX=vdw((_M=su(S$U-+j|;pMV)!8|@NJ${CHJLM!TJrzj?7RR#Nb6zrD z*F)go?jztcrw2{$h|!RSAi8heF47P@o3#5hK>M+0ylnL*BG>;Jb+bAck_|*tdpmVW zm`;}ET!;M0L)7p=1o-p-9A^IkoLr6HOS$DX8t*Zw0iF(b_0n~ONDt^OSsh7M zMP9-1qGya|*G&4QWibhL1iX>-km>!okIc;Vr%{!`pljF!*P?uBPy1f78yHkP^ zSEI?a^^x?fk|8@7wF50jj#7I`KN9Ef0ozIiXhzKm)ID;9dGhlF+;$d(z?r?Qw1pap z`zwpO(dW_mWiYwe5<+)NQVhCaiybLp)H1XmrhA9MY$JZ6?!Sq0e;pzQ;3X#FU6?Dy`P*S8egB1Op0Z*Rtbc3c%B;3UXYdyvpC8uz5W_cVf zs7M6EuYxpDJ>=o<0<{C)Zni=$dXVa@gZQpT*Zp)Npi>Fiox1rYk3gnS`mDWN5U9TQ`3fk#yCfxfTW_D)b4c zzVT$0npG&D?QMLqDGF47xY7G#t?=+~Fa8rq!9GiGTC8BeeB!@>T`?4^l&c^$qlW!^ zIf85O@gp`(wIJSSavAIl$!S*^cI+mzNgxtl+Z>jC1!y^<0 zhK)#W%}lb&Xfer>5}eXs?MqJ8MXor&P~DGXkw{DS0v)u5|yNqr^s=*D}4SiOHQ z9ZQyCZ~7xTCd<;1@p%0E(2sF-`GOH`iZr|0g=XLK0Xz9IXyxF#?U}I z=X?XRfTuyK*x$e;4j%#@}2 zw|UxfVgQ{{u^QG3=)mOPF1AC2=LPpL#@HTL>ORnj zZ}XYjxKS8UnB=YzT}Qk#!qDDDmZvZAd=b)?ME&h!JnHcq`<@6<)q56n&bSKEsLTVA zm%}LA=S6S(atNVAnBNSE_$=I*2qnRhaoThy5edh1fz<}r7& zW!7^HK2(6$leC#Db(Evg=!~hnoKAFf81~igqE&)kC|EX=etzkN3(oJMCzPvUdiz7< zQ>cQ9&?xx%B@42)Wa4Va0@QO_NlWft$EC4_*nf8xbG2eWEi2;U4*>(5Ik*LW^L~>u zbC!dktT&cMNkH$Q0X1ALhE17W(7HbePcE5Dm?#{%g#1oNM<%`Jlu^c>f3PY*8Nn` zqKW(dqZ+xY6bw&RZ6tU{l#cu=Vrtck(KpG0a65TfKfY!lrkRkOypi|}dSlQxA*z_f zV(2Xos=s86-C*zxUa!|CdpvZ(>FQ}t`dTSkeCi!He`zqjJ9CA}KJo!lZ*}9_pGJ(| zt2BTc4?r?Wh}=CpjAf5hi2J(PXx%P_zUuw#v+1wU{OE7?^MDvlked!_C%SM4@B z(u+}_F2MF+WjfyVis274VH!raV$i*QuodkC4fpG4`u88x(;I?&gLVQZwjQ1(J;5jb zVW5-Mjbm>o(eig46pL%oV5L`F4tWaP_;GfP`B4}?_!IPP&O%(4D~O$6#73Sw3`r6W zz~BD@^Nce&zW>i9x9HK!WiR3EMl(9&!4k&%wFsS;<;=MCbl|4>A8~uuPqtPk4ut-2 zX!}h8nkw>;)sSSlM&qjVOvxX%M64fHEWgHCmMjA6Y^JeOb|v6gK_j>Gi4DX!I>3;i z1bwB~!fbGDMBkI~aOvJpb~_nm?=mhVcdi}w##>SKlWRd_gCPwtOlB0huI&H+hM}(Fa8Ji>wBPJT_NYSn;h$R z`V>5$B}gBIZ$y-y3*i@s&~mLeJf52X2mNg@*Mec=(+;qC+wY-m`Uhy-#!tiE-eP@n zO3}5k2Yv^L(5zKu5aYB8bT#>D`mUcazMi1LiE3OTmc-1fv_>E8m$>`SBNW!JhObBE zXlA({B!-LPXTJgF_=+-2U;Y)I9lXSJJeg#3)^24je~Gf+o)5^n>mn(7jOdi~SXMiU=c^*WwN~YX95*}pDpt_!WVe(lNbmcH`cGqo4 z+h&IIr4R-NtI&VvJW|ig0T!4hgJlu#jG8}kM5#jw?o{!-bagfOw^)MMu1qd}37DZ05obEml&y5TBi_;q5gvqKh1d#Cz?ru5cRTom~w z$RmErU2+RVr-y;n>b=%NW(B}W7Nrm4B=Ov`lPots9F!|hvGz%A@J2@hlW@ zJCK7L%G6NQz7~oj1?k;fd)gV#%UaL4j2gqnWLvK@d(HR-cnlJDMcf1IaZvyh%Ty3k z*M^n6mg<~~?QnVGAXD%|3N{&ZV%dF97?o@X%P&t@r^E>E$`{Lk`+6qRbmb|E?7RV9 zqET3LY9R!vJcFb5;&kw!8ttmy$BZ>xa+b4|(GJ zSuB`Dsn9h7%Cy_Pj8V$aVIo@NxJpHO)aMBor`BI&f+y_(1ApKf=VUzW`UjZvLNr$^ zk!}4SMQ7qp)%S&QLP~~Y4vC0RniTh*y{GyhZ2<#Wrz%kOxM}# zhDwnJg;G*V#J54wKMV3)iYsM$^kFVP1kMw62r^ptE4RW1Ec2E7ej0~ z>E9<;^W7CV>fW8^)rxg|umApVE*h8d>n{WfGeWj2b$OhB@jZ z3H;-)&T}SOT5$_+jPgBdtvTKcMEL78ZgKLrx0u7l!+FkdX*1{H z0crmBpK|6{*VOb0@#s(vW*1p@m=8>0Q0q=56)zP+<*rNw~L~ ze!Kei+sXWG_xsG3mq>B zSKpa`m-FFQnT-O&bEayqo`XzYgbe;4hByr>Wei*-3m;M;)&0`}th!6ZiOxLu0kvqK)R+lVbSJ zit9PP9}}t!ax1DE1Oc2oQ#&}~?_10dev9B2SbpVvowR{-YHPGz4XKRnizT-NS@7gIvX=%ddygD1P0qtlYfqGtzS@1G~krTL?vd4P{17j}X1 zEPXJ)R}358Qz*CVC)>*=(OEMhsk%feI{!TmJFMNvaru*=@ODbgSJ5+AvDO2ZO#g;k z@9!ZGIQcNWcobBw=Yv!KD777tBT?$|u!cDwvsf!OC zCD^8piMaQMF`-ZFv9V-2iY~3eJ*bUO0&<|JCyCmtJt68%POyI9JOns&QsbkC;pmBR zOzGZ02pLtbQE#2b)Vk))7**Og%f$FBFAxa1~usLu_x4hrU(6T(@0=t zq=@`ohLHIC7<}?;Ao~^+;+1X@WL)3T9d8uj`7G_48zWN$R{LvfCY~9O7Zv}Jn6H~j znzy9j%E5lDXL?E7FKr4d*3#{Z&ym0o6^uQ_g#x>LEUF2GH}eezGudyr!*ma9iHoKU zg$hh}+Z7c5t%k~XTk+)cb>z6sG-$n@2cu`IX=l4F8##21d~9)q^^FmD#4DDrbQq;$ z3Ky8eKQurX{D=zNFpRiu2>SNva5ch)o-Y2yesTAvPAUzw>E9f9pke|07bei=E#px5ydv_p5&o+V#tYxp^$Go=;_&h18rKlxmX zcM3qh8bOlm9D&EJXqdmJA4g|bfaQQGTE5!_nadj)8Lv#rePx0tkEbK(szF7F@J!xc zMK$F<_KM4Ky3;<3ZQ0O_D}RV`CT6u^&!984_!W$oBOO>dhpY5P;x6)WsSWbNv~a60 z3xA8J;Kx)M6m|H3Qm9Vfo^!#4$1kGd1#3opXdk)dQ%ZcTlVEc98g`vQG#>ge9^?+D zgQUqQ`a%GV1>dK);*4;Yi;&Lq=`4|KS_6(|y4Vmr52t5Ku)*8J!9%+UtzVBo_|8b! zuRmU3t+ouC9PO#s3O9UvEv07St`a=en~Mq8En%Zp5!?|nv^V^&z^_LpGfi_I(OK_h zQSx{uZ0%GNTv?ET8K0_PM|T`P-=qQ4yF;O)%oFX$?ZVs=S4`iO0;Zzwh^mxi{+=Y@ zEhW5)KZ6i|zm-Y&IZCQ0g%Q1z->A(yIb2j61|9~(#5FU7iWmFC!kn=hW|9%?)KUk_ zCoieESRLHn0p*$HR#`8bmYKk4pTy5A`8MP_eB5{i1x43Cw^SLLTh#M>|;e|E>X3UPxRtg*@l> zX|P+)RUrGX6~eS-;KiEXARCmx^?zSU7M=S=Uuj#jvL`ko6Y&n8h*|Pn`jzm+=hx(1 z_gb(OvYrZd)}z55q_IYtAd{+ze{vd-b2y#Mx{|=T9=wI#>pk^|ysKPerO= z%!inv^-y>AD>)&NOL{N_t|VS!6xU1uyT=z`K&zK*Q)y$7WPy5h9d>s%Val!{S{xwC zbHQ$iS-u$aH$*d?E3BExu1e&_H!;CQkyCiYz7rjXUXfz;LC&;??O=Cs3t1Mv3et_W zkX4MPClU@2<(<#ybk~&_IxU+$G_{$yWXTHTv=P(Y55b%>TJS*nGgC8DpPig_wPwcl zN-EQ;3PEv)$^Nkf;*(Uz{4Q!jHP?+Ot5-(+KPeG@N;tT;i9?g(8Zc5RqH?Rk@P71U zxD#dyw+w@zSo#P0e-I}FXJuKX&Xsh@mxpxSu^6mf@)b?x3aRXwPSRfUlu2=ypgVf% zfu~`OAL=9Uw~&)CJ<*grIdv9JcD+^hr=Pxth93?_1z12{}B{cAm!ZYC#b*A?x2>^fi5fdqXyH1B6a~F%LsxwOtG^q*yZXb`d1E zcqU$SzDM5je?ZUXvrsbr9Il)E1n=AK17$Hya=Ur~xCSo6fM_XL%{_^Enj3J>zsu;? z`x=9tHwm-it=T&Q9eU(J7oMK*mzJw;fK3A{1!6ThI3(SJv*n)BU;5i1>_;HlA5H*2 z2T^bntA+DV&0%D355{&)zypp(gf4EO_Cu=V;;y;$g;N$L{`^5E)Gj7cdFkZ%yik~V zz!sE0WRM^KF40v15wMh*MvL~EVtDIDOziN+CB`8%^|BK_>sg4Bz-Fp1HQ1bQ;|O2RVIuWU(zKFWUrHE)L++`}5&fY!h_LCc*B= zK&o^vg84tI#xA**?D5)-L4!y?NG+q%doEz_mKW@<&7q{xREKy7ncU;PKcY+fjWFcK zUHa+HBrK5(0Tp>KR@pcn4<9;6Die#aM`DEGDJ%e|`j@ctp9GpU9)_&(0kFq*J$_L- z2jSoSLBv9;MkZq!9FVldV0Tx1^Ew}%%N+ot{B9DKUImf$!mRMy*O-mM%zNIHW$-gP zh(0%#qx4_~FnU*@J76XTZ=Mg?w165qCNib1BJlBGHCd$-jy0=-5mWi-=yw%I3nbXP z{kyT}KR&!u`Ae+|QcrCS#1~Qstp`;ZmuCu|7N!fUlcO6~V z=;NJeCz5fZ6@2>2sr=7hG^Z*NHHNmM&Cc<-uj&=)){zwm5({wu_y~M-BMJLmW(vG+ zb8%z9Fgao{g+|Knz??!wSo!oP(FtgwM#`J0&p-{lvxp{@SBl_ue+))t6frAprEp>R z2fExL%q_45b^$lZ1Q7Sug~5BU;JXtS;7UiKxJ#MH1udbVyNf zEzLBEgy%*Rsr-z3x<=g>Lnc0>r>QLLCBk87c_KbWQ5c`R24w@SAf#skj&9E&fiFwx z&1c=jZ&xlEo9BvAn%U&!A{+8Nd4QR+G@5)}fN!YjW4@5xhR<3+4;flEeFou_DEoS2C~+`+vuRw{9n! zZhZrKZuz71@LXs$hzHvqeJJ?hhx`5#&lZLe_L_1r9@wwKFe(GDHLBaq}M#ZDyf9CY;QGeKJ zQ2~Ye9?-Yi8v?szh{)4)Pz#fT3)%C*_n$Rot~L|n-RW#q<5N^`KZSZCZR||iP9rl4 z(Oay8Ui9!5Zs)XvXl*|sx@XyN?Re%$sl32*y*>O~F3uaMD+0;<_hA3;A|!k-#NQK+ zp~&q=Bq7KMtP=C-+W2K8U-~Pv?T;LEww?jE)qMC=x(#!&46US3(=M5b)IiIC9lB-( z=IwzvFY^vDnD7fe<=3InlnbEG;tp2lEvFK1p0<4C=9!!Efr=M z^YAWt9}taQY%+=81VZ@d_|>++Mjule6o}{$lB5! zK4-w5!_wvno#sWm2^@SLL0*G4u6+H2Je?SVmLFQdA>=c6i=+k|_Y8v5)6~#y9fIBW z?{s)HoH^K(NhR~`VD*1ec-yg_jbvS+=tU^>`3IwK>SQjbt`^(penKbK2qOQcWApou z+&ej|aqIINm>E(Brz_-eU`-j;j^9AR(-(T4M#A*YN-}P86@=Pt7T!(2LAl{K`7ts8 zTF-y*pXOaMwzV+kem;AqPgU;{!Q7YZ56qc!u2lCB@5blf#0&U3B)Deej_E zCI05eV*d9oI$h}%6C%v-*L@j+$<5=T*7^!AE`CLO-WXz+0bC&_nnmYk*s??}p4(Ib>>YFKvsm;ZD+UCLWEk z+*pk|x;pSb$ljv|Pn><=WW^(tFA~Gv?V^IWYaQvT?;psx63a&h2;In|LggXPRUGBsiXZ&=9kvs~0h!_^v@y75l5CUYYyh^?VN zP9)=dqbzn*tO+CEPvVW6mLg;*+#^0dV)V$1`^>Eu8QkINp+w(kjJ({NNK7Ob0y}=3 zAk8dtsG0=-!7fpIxInPR7wqopaDT=LfrfS{61P5QVSNd1T{>+!`KP2TKaHaJT;}(J8qBg8K7E4b0L~>X0Ofb5$oQe%a*U$QhU}_Ylf@UNQgi<*{8s z1!o1c!%pH$ZdzFK=#Keh{M9)y?(rvhQyK_grrCi>${VUXtB5JjRDkvq3Ls&2kG(Yf zfY#4_32963Gi~LmFsWJ$OX7Dxtni-}5lkho`kWy}do3~}66n#Fgqt2GK(&!KVP?o+ zNN6-OZgwj<+VKH@-zkH+nd?x;D+T>Do?_R3_o?#xK)9Au#cEC%PnRtT$9YDUXj<-o z%xg_vx0N^st+Ye4-!Gu<%xa8pc+9W4L?lK<>OW7)!4PuAw<@ zx%8EBf3pHAzo(LYrW^%(I+I+&J+D3;#ms-Q<&Gx32{p?>hj4Qqyt5 z;v)ESDhu6r*3pZHqM-YVkRdHap{t5O&x&ke?xF&AeX1Zoe<#C|L}VhJ2JluzE=px( zvnl>wXkie;+7AMKe%ucKrWcT&2p?)aGKo8&Z%uCAI*mD(+To_HAI=-M2*-{tpk*VQ znCIzkpuX@AIsN1(8+Rlal<&z3ysm3P+S<32|J4>ZNJkUSKR@O`%Qs>(eJhdAQo~O- zG|_tDN_yCD7I4p<$K4r1CW~$+xxZl!^5W8IX7OXPrA3`FRFlKk1NZ62du0$JF`GV` zvJFfwJRvRtBV@UJB3v%(#5<2~L!j6S%r_EpV?|wXiS;OWi!A)yy5{}?Hw_U{3n-y?T+a_>G zdP}Z-T8=9oe8+_^Semp>2QL~G;@2WIbWXWN`o0_DmlG|_w!mQ4zPf=C@2?~(TmHi- zy$M7LUeZ+SBZ~HB7O)19y-;;? zDeOIvNayA^lEK>ftaeB#={%!E*$)lqq4|_;O<5@z^;STSO@>%uoem-*_GB$b30U_) zl>Q@${k8_+uzw43rRQP*{G{9BZ;(w*5@>LTVQ*}7AeUq9$q`pRoV+&<<|}xD_Umut z&>s_l-q$dQ2ox7cmv}&)`EwA@4}|00CE(|Ah2%#gIh7Pj587rhdzK}^!1dsoV;_2% zjh$2Jfj#Z?$VFQmGB$@&sXQiqrymTbET@|1X2IRHih>vG{?(W*a-;d>WthEiDd{k7 z!by*0aVS*_AN~=h;c3)DiJvYOF7G5`MxSx$^m`H`VGM8LOd#;*c4%Q5*p@T?+=nC% z&ILWfbwQbMa@HvFwnxzG-@+hZfjG8)gThm9$xhzw5+Crf9c_wPnr zP?b_JL^Ub&Hk!l#B+;7UBJgGf`u}7?aP4)KKMHV7Pt= ze7z?NCN9}b4%C?ldGe3h?AvlU<$gDw(7H$4n)c(V4mF%vw+?=gE@IbAU{UIH=w4sK z>d39ZyqdFgx@I1%+ZBo;T|Y_l952SXVkt;U>SOc3Hmq3xgjUVz!MLfZHM2UdL6PNL zJgK@GeV(_nzF!X0byr$ow03w=@(FmyZy1Rbx?lb)%d@4XXtO-mzfRh3xFKS~0B zF9U~_B2@j9EtbpZW7%X$(ET_Mul~0SA82fbw(0p~-V#xKdOiS79FfAfeU-$_@D0fi zkAf>UE1`RlDztisVLks6eGv12jcuAn15%U)?bfScnwAawFQ^IUuDe0!z7u|XK&6oH z>xdS$cTj%jT4ZM#!5zbVB5+Ivmot69yez2+3LLF%3xgP&!rMk<49Z>Lx(6oV4v|a} zb@UR58J9xN?j$nbz>z48+Trax;ZPQ(Pxl^{gi|x}7?t;C=oZj{5prU@;0k~AJ$nb< zLN~GUu|wBLH~eq=Q#i-##VJL;!rA8}+0l>tP7bXh+2f3Cy=5 z7DhX3*jbIb*mdtRoO|BIsye4rs~%aLJpT?x7A4|my%?OS(nBBGzrvozz3`vVOrgF| z3l#oF(i4$M_;pqX=?K*Y?}13P*%FF(@^@g?pIL(S)iUV&se+~kxB@tsAwT9h=JSyr z9yJB~;T$Hm(3gUyIfn7O$VUGuplA9KDrHvi?#5ifCG8T}PQ$?pjj`g`9VjVT#XC>V zQNQEP;Bav-B)myOFXJ4tZr2{%w&NH5+ruG;wu|GAh`IEJiHM;3TL~kXWq?>Z-W}RM!T6Rbsu=?-9}zEClORVhK}{ZS|nge)53rf3X$K+_mYz$WxSc$|u8VG59KNJ@JVspev7A z!RTUV_!vDIcb*6*#}3ZLu8wh#I$a9pM6V^CRFOt*&jROXC&~S->G0&#LI^sN20tHE z(fcXpXnF55rNb39R~Jgq-!2D9@q{}x?e;y|qGO4(BVW@txqF~4qAmDaqKYY9x%kQT zD=`(mh1#h~5d2lh4u4vS=+Z#`NS;AKxt);hyc5r4SJK6Ay2<1>*MN7bo-K)_XrCw4 zdS*_5k!N}|?eG*{uGxDsUV9B(yU|QuPlyDY^fvOVWiuRju$0L3RulU88f49y4-$o! zXyVcg2z&Q~+5JV1clYT#*c+@4LA~kp5pxv{XNlnw=Q{GXsuqUHt(tFOjc<<^Gt0al z(v*<1bgHaBO4&D?54(p|Fl7vZL zsIHzN{#DQs*493FyU_sE3LDs+7xG!#onv(MV_EvQJ`xt&XrR$5J;9$~6;e^Ajiru? zg0d_I($7!8ga!Uo$3zZitjNKJqm`(UAWMvcReAmLF;FQVg~6AC==)Du#OdlPvOIen z4&ND}CgzK=*(-pS3#Q<8pC$A{k=B$Kh~1^fN>Is9EM4QW@CFn;zQ@?qLSoVnWpIqwzm=85Ob1GD8Ye|<5L zEd2x<)AzE47arlu3t{-~-vUA8)+=!G!7_+lz8sZoBB5w!7;ao`11^p~nC$i6FyhcR z>KY!3c>!DC#(EBZST&Qk`sWjBu_tqR8< z?!@us;y5_r4w^j?_7l-vWUT)zyhIOp%otf z`P&U7#pE`1l@;D=yE@3AQcTT%Yk62`d>*%*d2!q=M?U7A1{9!jaIKPMW{vnE4fz{CW0!jNzNl@U7;;8*u+$&-YuWM|uJvI^&erWhvd(}lH66azKxcOKe2{n_0q&3;#vfdUy*g28S zH+re%nlHFIici>kQJiJ40A7}M;YIZjqVHEi_6cjixwj;^%4;Iw?!}XI-5F85 zT5QVodu{=@WU|Ooac%sz=>r-k8N;MI66jQTgiesVO@6KXPV=ipN%`P(Aye0$Znld? z=Rk4#@bqHl!l5b>_c#tz>!f+MMkOe-%>i$o3L!hd9ErwFFx>Ehx|GG^PqQ*wuP}=k zi&*2e)8d$uR7!uJ4+fc8s~NM*PP5IC1}vxYuh8>w8MpXy$+Qgw->>WkRi&-CaOwbQ zyC?^Dlhk0IW*zzO9RmZ^jf}4SK`55*hWm||2`{A-iaviLr@;)oa)s>WNS=_@{+Djl zSOUAoEu!HojbM++E}YjZ$+L=56G+JFz?F=CYU+O;)?Jq&;e|mkDKrO@OeW&~ybKJz z$Aw+xnIzBqHr+920O^lv;w`KhexEd2&riBovtTPz^pMH<#FxJu6~ zawhu}irA3y;$Lw=(dV5IbMyw$ zH1@{yGcruQa65>zwu`M;r9dal^a0BgsdVYQI;yiu2Li2?;J5TMm~|i+1osz%evW|7 z2tNx_ONwb?Q7(k;x=+zc8I^*Q@xS@Pc~P~8FkNc_{UCZ9*WNY3?rXvHKw&Hf`OadoN;q{6P#0+CnFGU&pDspHQUU6KA8n`Sk5JdD>5}SapJ@kknLXncPL76;!r@=DpuzJR zInojV>;1NJEwi`6%>8x5Z%QC&ha^IE^I9Bu>qM*nG&8z0$Ahd=GS#{i#d6k6$1062 z_-C#z^;TI#|7(fG{dVv0g{=nH;_WfGZoL_zPRYWZ-XhRir6L&Ec%PhJJCByUkm4Pv zm@jD7|3yn1>$q7T2t7ZP+YipNa=t;bSicEb!Qd!vbC0WTOJAf*@jP^$z#-++|K)^8?`bAYdi?^sNe zP;P7lja}XFRl<{E;Y{M={Sub;?*yl+LXxND$^0ie9XDN0WDE6EsqsQ}@X(dRiWisY zkRXHx6!PhUW;K}99!1o*oF#Wsmk0`k{N6yV3B2nim2mD=GK?fg5M|NpaL`zew`0Ot zxODdzR36)iI_;J?cD;hxa%?5uUs=FCP?aQmVoII!#ui9aSFGBez1()tl%z3VU0l+%Nx`SXzZ_5d_lU9i(o>`3ch3|R%*Z%{ zp^x;zRiu(0H_<}Lxg8`kpcRL1a`AiU5E(oEj*Qs-h7ofit5@#|%)Un0LhWfFG4E}+}nK*pu_>w5OO%(2D9>yPU#|he{#khNO zcA*3TR6KAHjnbwFZv9IJ?S4h zYj?KL+X>U*qj3q=TGr9C+@na069op(g6T6q4RBcBOwCTb!)=lM6;y^v&GiNxG!Qhm=(FOu)Zi?!YP$oIB8Kqt%$d~3Dv(uQt|U-N<#2De zfMB6!rm%kWAiGo^urC)nLPzWww)XG~co{4MN1b_4wsao4MlZ*7k06|Fn+*dgXD~~p z3Ik_GU}WBM>{BIB7%&K}(<`C9emz~EmCT-AGZo`aD@bd=Yp`6WLht2Fz?LUMoxC>( zD(8>U|6=Olb;eZQ??tacwWo(Jb!sJVL+3zuW(B-YpA1Gx_sO&QdaQYrHpq(l!SYF5 z!H(r7sLm9!-Zz4X!P6n)rMec~4d&E>WZSYc8K{)r>$&Q2}`Ti&j=M-zfA|}gq5iu!qcGEFw989{!G6na@ ztJUXVjz~QdwRAN!-VPLsAFslcjs#S1tBpvG##@HVQ1=QVSA!KJ-fQ^I(yiv>qKkf~>Ap9bV9kRpG zcM)jvJr-MMOYt7Q`v+D%`sAjAXAQCMXI5n&gWmX1XcFq%f6}74$q(WP=?_G4?|Pu` z$7siqLE2p82)%m`!`zG|57`D?mivZk^au{DPsU$Zg z7UIIs+9Y~?1hfZ4qut9(#Mmwz8dhJ$_qttV(hN07{2NKE8y_L-SxgglxI%Wtd)&R~ zCa!)A)X``%erB_1O5;_g@Y7{_NooR`2+yfwpCefuYlMY=Z-FbR0G^XAKK{56r`Bsj z0Ih)iPmW-W+gWtmRgR~-t`qehPn16On7sTahbF#*a8KzVY5FPxKC{g6h`BEFeDN$C zJ^h<0m?-6m z)R->{q&i~T*uKpk#IvTFt`)TdyXE_!eo6=RHtwgV=VaolA8uscSuX0v*V5mm%YdXv z2#%j!fn6bz^yndR7(WyPo`$=ypESa3dr$Chn+qEXD`3OM2>QX}J1e5_0ovax^V+?S zkjR%(P#QW7{bM&kkjP!c#_7Pha|GY3O7S#w-D=d;Pos}$8`!&Xk~J!PFpj6G=SZfQN@OYs704Aun$) z=m_VRU)Scb1@`ag)nGSVT5=GU$knk$rsDXtc{|lsSH@kVD?kY^K-I5C8az3h4jXx5 z+0y`2)Ru>!PzPLawTC^^eTV##UPFD(ZGyywXTfD@8da?26EBBG+@UhaRB0c;JI7X| z`-B)6E=+(I1FG=fi8E+=EdxswmjlEt<47GlQX`r&1;?1b&>?i1YLyE8xEa&Ha%MdF zDeewy)EB}0F~E@_XX1D-pNgDJu{+8doW}QovYjK=o)5r-zBl+rftms_ zsTnMPdmzLl4AP}JGUz8!g_W;FdGbw(%(i-8$ZNEP3+0W(sn~_QUwl>Avm7GF{!OLJ zTX<~Xo;u=Ioe$zAyYTH6H7NU&2EX>3Q@cT-AE_aU8r_^EaNHR~dUZpY4RZ>GedSWx zL^HuQ*aVhMosPd&3;hN`QmC!*9ma;P!XLQ+sCW8JHXkp==PIuF?8_rM^V|_?x;O%= zj~6hsKt^D2|2?^M>lQwH!4(*4vEVCL$Vp($%-tMmeABRk+d@-Mt?kWXc z&QHoUh{eHj4{TP;g2h~rqD)k^j;62#+~ z1&W2(fLE#r72Exf$#PrCuAP{LS`)sqk&Ldu$;%$cSC!yA#Xj=IZ4)Ri`b<9uE~mN) z+C1>&gW<1a@O(W94HHvgox2sCzi2A?Z)7|ydyr3JCT+zB7w3|%Klwy)mo!hy=?K-8 z_Cs&djVe)piC9x4)4Q*P**7H%T8tIZ`^#VM(l|K+px)#c>}7LDvS)rD#+Ofo24xAnSe45*ovDPMFAjmzKXKj_v3lmPYbvSAYh+sF z`1tzIT3XGxQ_XUB6rG$zdk&JDR+_D!OO>)#VPybsD>yr@yUlz`hF zp9kj^(RgL(22Nh_m^k#mf#mmn+`p@Ih4bs3Xxn=n(gUZ!i*H{^Mcfb?XN1FT9^pZjmXM@3`a(emwRB&=6rl2zfMh$kw0{s$|aOrh|EEq&eWOEyiu zf)eS)G;*9Js&_hIfA#I3z<-%LKAxs2`U^XZxtYgXmo0-Tyx zf>)}_sIA#lfuho0a2WO_kEVvfHUAMZZQ~-yvYrpcXE%dPPZ>Vy90&1fX>dK<2j6FJ zgr``A^=b0)Dt*mBHmjT3sq(&;T7e+ZfriU-8X z(5PlMs^kF(l`LE=>VtmUuF`k;O1Ns&4Qg+h2@;-;koWicsi%UCP^a4fR%nge<9uLn zK|Jj!u7f#!-E>aN8GN%*gm?S92{>JfLhVuwR7}ewWy@^v5gCCS@G9| zbA0I{0JhhPJWxu+9{U+IcicECPKqHU`79>QAA(tH^}y{-C^gfs!!whkadK}6F|t~Q zNygGNzW*#W*`o;y)=kIe$+oytsg27yCG77URB`U4sj&UDGEdaKoz_)YfqnEclnr(! zb1a?cvojvZ+*`ubPSKzyHqme(7nub84CtBWNFEswlzFFwmyX9kNbpnKt#lkTTY~6g zu?UboattFj$>R=J7a|d31*%0ObdlJ8G>-X5PTZXZgKD=ZSNbj)oqm-bCZZUk_Y`zu z8c1vGKjJk@9vZLQg?l#LY=S(?y8H;m$j0}0vwkfO9(o2P>poNMoLY9PUjv?+8%*|g zr@)x~HsYYN1pBmwx=rw3V%B^b>xJFyMJ-d*lGBEBqT)4jI(NWnO)K!-CL&F2MDb`N z8vpYaN+>6Rc1m{5zSQxMu}usN_b$N)MQ%*5iy_m#<})+-x;##ZUWNW~YCMs&1R<6w zf*4oMf|2Gk)bxHJnfqLx>+jWqbugElJf?wW=%6u zbD2bRa|y)PrZXU*bP<$kEW__td31ID3^~tysJ$=z`VH`=Ps$B`7KUxvGf{KG$9`OEkgY&E)dRT=i}PbC-Kgb zXngRf0{c4hNJaBF-uIO;SSlg`r&2eee4+~F_f#>{_8zCsmv^Ccs}tsHKS7@~J}S@H z3{vOi@jtycWWlN=rh4&RyjA^)JnUai`7eHh%BnB$`I{BkMKqF?U6=8n@eG)07>xNj zRxr4`n!dRC4abc9>51Vuuy}cen5;PsO^*qLZ@W!mJ{t->w-R{37|FEsRP=$rwEJuc z8zR#JYbSgGX_ZOz?+GRPbh9elZr6hN*-?1fxfJ&|N`l*`cQwxjIR(vMvqQO+BF9@Eg{A47` zN9f)N;ah8VmG0V^ME9Hbqw-n_!N9>S)N_9=vC5c=RdwlPtZ^>wJ^KT6(i@3IZU)4z zd`69Ch7f~f1;_|=<#h}8uqLvp$o^5GZ#Rq)jdSsMq1y)URetc;W)U=e~PNR405-&@(?yWq>DHA zkiCNk;JH8=-X?X?`~AL{z>9|IfEcx)o5+fOT7o6zx8Te7L?}p$1g(?{%8z?`=Nz5^ehdB*XX7l$O`EZ$rFMGf~(Yt`N44=b+VMSDPG9~(D zD!gFZE5z)OKkCIS#QnOp>=jN7{U#p)J)RBV_gx%o+w#fS!ijbZj^cfge)(9S-RUk*<9*E!8h2h`l*sq}i za%Q( zof0*xhEK!s9!wpA5rZDqBsLJnetSanslTWt z{}@l%8_^VNbqd0G)xK>Wuq#+VgAcEVZM_TNv2z3&J@$y(eBcDW^02_2-xkBD;TBkJ z&cTxw#rW~*TKI3mN78v@F+7jzWa@?gSD%$|&bRbBl}eum;e1Ir@>>u4N3G%MUp8KV!!K<}Lj)H!G3c3&wx+xZi{riu$B z3S-ID&~hl7tio36sL+#b)!^nBMY}H)P_xgTu!cv7MAbuTIW-xU>dCRnb;}@g(|J1g zy9~L!#tv=|s|$=eZ<2%mW9Yo&se0o$Ze|nNtB~w1?s*>PT$_|pi6|qLLPNg>Ei$4M zl_*q_QK6wy?s=Yj4ymMp_CRGsWi_ZM^1FZD*Zt$Z?!DjVeAnmsyzxsj=T`)DkOcuBb{o`MdTpd(}XXau8_bRL&fy(u|I? z4K8>zQ@y;uqECU(0Y-mCHKUAY` z91yPeiGRYDwy$D3W9`{{{7=k0fj;u4jvtxroas~$pFsM2yv2`ldrIj-IqKSyBC5N5 zC9AHZMEW`x;=8}!F!l?@St(^hC#ShO)QuP|a`1^Pdy6?r_1~>x2PBiJh*$HenG>n3 zgY0jn+4(Nzedh|vE1XXy%VJ9NrW$F|qFHbK98+>VR!pZm#lGfxQ%~A7rG8eC6OANoW3IZMU>B>+BHIUR!u>#iPLmE|}@fMtj(PtayC<@+h#zF$k zoN|>ZRiTzni4c<9JIR2u^z&il)yJqf`<2urn@fqi(N3RhqnJMqeyl}R20Qjqgt|U| z0lR-oFTYAgg_*som0f=^liXI?%6NR-K}I}qV6I1-u=_7>U<3}bY&~nos{Xf_EpaWz zLvIYJ@q?Ny)54NocSXo`9W~T;t}AQ&)xc@>+hM24FShmK&3CAOKS=Uw<2sgWX2K8r zJA<65a-G!}bD;`)MA?w?V@%4S_tb4`JBrvUOZ|7-gPmupO?4a+W23g8qYRcUqat@+ zp{_n%PpOG)cG{|g*cUrhs9PC%)cYL*EsuRaBUen3RR%Fg-sIfLZl}z{-d-BDv z^VI2;g7e~L#99{2rs@f!`d-03rooT|RyF1j73%Ct?(36e4u8U=mDzEDH*y(i@m{!o zqEmqyOdcXFt#`Alqw>j^>=RVvp)&sOdl?jyQ^_Fze)h?|6lS3OF}p|Iq@Gx3%1zNs zr(DJ4xB)+LI1k=uT2tRL8Br!|uxcOWl0J|1Q2oFt zRg_VeJ{}-r(Ia-CzJd%`w3A(sT1u5I;IbDV`%^K|+LY5>W&V{jBJ7nt7F6&jiuH1l z23A6`RNgTsYQLm9WoWjARq5+z#?^YL>8Q8tnN^KUT&uvo;b=c)@=1)VwO)$*r1~}6 zdEf-=RXM~=eGz51rr0qT-yEf~&X-fe{~inGk-wSPTEVVSNu9#a&r-Bc4htH2l>C)o zN^0W|>gm5A#rzsp85? zakAOHSBj}$H>a30)z{d4@^=}}^CuW(vqAQJNh*6>jK>Tvwq=v0qp+$0K@IDavCl@f zv(-45wVx-=96^zc(T5RcbC?fPIfKK-g|{(QMJ3enpz|z`>|uI;Ev(<%lFp7u1+cF+ zO*@S)QDDPg-C#FtYhtIzy{JXRH>dqG-jN3*+88&f3bvfTkKrXMac`aOBlk@BQd7H= zsMNsql;*-fs{UjYzvqY&8sN90Tq}$Rew&UY{mY)BJx%1*DDHUr*ZCq=?2F4h%ZgbaBaIOUN z{^2-T!ep{}0wvt6HwQ`oD0SxO?n0*ZUI29#Wl$647g!nBjW|QwlB+D+Kwbaa!A#9- z=U<6RVy@|?ku6_CDILc-)WBsI_PHaMu{Q8y&z(I_`p(qje-Zdcd{@w5lZW*vGZjP1 zxzLx|o|MjVZ&K8Xuj{A@uj{OJ&>l+8Ba+=-@4;*{^kla#{6=kPI7j{w-b3> z(AvW;+#E`!{XWRlt*c;)2>JRct90r{MGQOR(?VvU@*`??u_~qWFpXWPFH70J`pE{= zmXl<~CCX!rWY^ESNLe@QvTw8B@;ls$nK=)Ckrz4!82$UPl!cNFBk}PO zIlo4dNu8ljMc(XV2R(h+xnqi?`}qTu&)?(h)v#@3Wx@-lKc8imW{fboOH)W(d!HRz zU+)w&tcY*q7E{x%C#VyAYwBLB;Mu=pEi)*s$^JTf4L(}gM`pLHvA?&AvHMPmu zRQ^_Z?wNmQ_yZHYTo@N)Mh*@~`Ev z<`M)uv}2kgkJ3!;&3EMZ&7G{Hwq5-x?+;`~?sjI^!zbj!*Nx1sKzXhOYtNl*X`$Y2 z)?)8hd}m_~{g{B;9jsqeAu~1oi80hnC)a)MW)mCMQ#)S&VO#}w4&PU;XP1kwB~LH% zWPfoYS&6Z9>czDL(v=g<)HmHB4Rt6=B}>)mmTs}&dkIsQ3p~jtlciK^k|SFmE>pj1 z4Q7DQZpsVgQmO5;*bw3ut5Q)!#@$vTPnzdYOIGGGS4&S);(eX$;zwM{`@}^1c$MHVZ)|X)2!X6IZV^KF7mqQIfhSpv!Xg#REehwS#w37T6#vA+9%z| ze9n<%cYYftwJvOBC+`WK8waXcru}+}>>3harTF8?0 zJoS3@e^D&?xquvcW6sdJJE@`c2I^6+Ir-N9DwS`1pL+2ljfynQWJ9=BZ1|O7M$WjG zD(bsM^_W8{;DjbCzQUf3Qn^AMiIHXAPF!HWy&ojE2MK1*@7vjHj;Zhsfzg2hz!RfLt#1g`8OF#GG>2#B%);S+)EYa(ekG%3#MkQf;W5T=4fb6JM3Y=g+*( zw*Ks7ZWq_Fvf(TvR-nmF&V9icUwF+dQNG8%P}ZT`6(5jSGiOrSDtT1(mtv~phrE*# z*iT7cC}dA5U1ZXZG?Q}MGx%$DGpT*&6}bu-7L>`GNVflgFFT7l!i?3fBNumhQ4v0t zlu4T{^+hoIX#4rqscHUMR&=EU>o(rPiYbe8IgvHg{9l{cVbcieFI1%xiD0(xxeoWf zVD=`bytZ*tnpI^FZ%!h`SRrS8&mDOkFYZpeo^B&N7+{MHS7m=72EL2kqzx#N+kyfb`OHuDApNLqsOl> z8N5f#UnMy-evpWTO<|uWwBZ`WyxkKLcImUkZB0;Um zaAa0X^^nhQ)>4_(Zv4nkj^y&!zWmEhV^oRT2w8MfNjgYN@O?%(^AZFCA%mpz9bk=n+z z^uM4MpE}RJ4zpoCJJ00uUg}Yj*1lA9;&%4n*L0Ru^Q3AYMKdvbtf>CPd^XsxnHhLJ zgQDZ2oV@J?v-!vB{u!yDv;V}XCP-G*+)Jkm-qSXLofrkdtp>nTd<`RJ~&nJ0RLX zEm^aYJr}ovEUq@-matZA{J#?RWMwxaWgkz{0^c=WNDaJwSDSBe=_9pZZ58Em!Ghuz zi%`0j%9QrjY)a8qia8;6m{cwjW`$@eYVgSn>Yr92ewb_TWD-+AnwZX~Of^X6z03;s zc6~9)ktP^2UxWR7Aer=7{8r$#rAqDSe#Abh{>CP2R5M*qY z->{V|eSC{5kp4*(OMRv^LpBM#5%`SP>)B2>E`6Y6boR04q&RzK!kfA0@{?V7p@uc- zbfZRH#u-b2=f;sMsbqrK9;&)AF7L)U%I!O`ygblrB zNLpI%VSamjpxTr@s3lRor0`!Ec3ILJCPn8KB}Z|Y51*45?$SJFVw~V>o+)I%duFlB z^$zxyn$$^mYCX#<;1s$i|p zRgSL$i_jBeGu#tp4;Ng0h6E5obm++%q@OMf9vwIWMnr{iU2-P!QJ3Swo-iO;`k0o5Fc)}`D4ADr5Osa-fucw7tu z9*E(H5)yCU@RE*RTSK4t=82o`&By6_t>~bm2S?{Apf@i&543V8(3m&|ZC^cLX4nE) zG;bB;evl)kp7`L4g$DTaWDSthC@Yz3>ZJ zt7&8X1}|EKvl(an+l;$R3~`pOJB*O4L3fSxQ0I(|An_7~G8-QNk<20BbyNkNE>OXa zdynxXHkTly@7vMEh|}O%mp%+{@rIIIHO?)U8t}c;6yAV~1s?uq2;ux%uxHp6?^R!i zAK!k3WG--EQOr2~-Xt1%Pl@833_tk6WCnD9Bn|i5%L8f4(;(@w6x=tl1LD_tp!`EE z=WAILNbZ$@N_Cw8kb zH@hmqo}zdp|5Fk_9X7+8YZ-KTV;<7iya$f(ngDNP7tt=0PU{(QadN!}-fQg$Uv2lq zA`t)!^=`#0{g>fYVQD~O^Jz}2TO*=!R>D#nfFJ($#C;Ls*i1`&0#zR|$bFj#FB}YpoZ}i`w%R*1 zYr7ggBYG6&dxR15UlpLDU&}aG*VZDjZhhddXv%Tucg8cS9R=$ZeYj2274!B6VB5wx ztiJpJ*dUn)wx@Go{~I;v-DFA!3`EhdHp#)V@p&*X-kOH)QTS`}8o|!f7`qk->34v4Y#PnTb;IJ-;aJ!00vMmM7JNKw1T(MC z1ToX9IP#J--29>e9DVNq|6Df4Rt{a@?j038{?!sI{g%eIK9ay`N)75(3a)MwtARwk zfY)2P0`Ho%!#6*r3HSo(C??7iKk`b!-)aObxN&V zs>cY`NJkix76$L`H-s@6Qb4~~9mXAXhp|`VP+G)yKm^#pMaI4`c+NA#-+LPMEswx2 zH}Amx{(4x((+qx0c|$DTDTfX`Np#c*Y6c}Dh*;;|3-(1#(<;%$V0=IuOGREoqi-*u zTtj(aeZUTf@905!vlJlm8$df(TcbJO>w%cT6fxQn44PIb;ZVnP^jvq8_DGX}hf3t} zDLr#^?14L0UlRt7Y>xZOM58XJXDE9(hR}OMZ$Q#N-wI9H~~J)5cv3d>%fG**I@d*6+Bw|8>J^| z;y5QhZKiDn{`o1OGv^wR(upJBj#M!?z0({kOZ`Ji48XllqtJ=TIZ!9_C%7&X1dlJi z#!1aI#R&&I@YG5%%*&U-v(_C&iobe@fh}q{(oPtSzOsdNE}qc$X)`E`cc%Z`H~}~| z^Rbwza?2@ev1zBfm7DJ>N3HHzz;1)pIA5QFMT=mtFARp9(mQe2Gl2 zECqjb3Q(BUQ~KoV25_cI2#g$C4pUY|L$~IQP@=#9t7ZKICJj}n=x_ykHTxtw*y;f6 zZ+`?)rLl1DsuLjm{a0i!>4(2oCEypYTW(dmEv8><-a^eu=TT#h8nGhtG0`pY6O0ul z!IbVj0P;oXjs7z9;yxKHvFipc{dELPHq;RNvK9ijG&`VRYJyh~^MLf`Bfxj7D^Tep4;(@l8hGeIWGRx#ct~vbaG|>@6zLVh zV@Q|(({70JR}J}PfUXFc@gN~gbPv*)+ADv_=Cs&PH<4J z3T@l61b)9aAI^WmL&MD$@JI;(t^4C)$5Bp6>=Pe|_EW+3inj2DRr*cykGpw>(dzL6D!6%1Bk`DOE=Z~qN+{P2HB zkwHY>4h^uO=QN#FYARqf|D&}PMd9;l2aJ`M;Z_H89J%8W*#4joCAHe)1P2FvT;n(> zZG264-bZL_su#}wG(|6E`+)O72fS)7jRN*~p~!$HaBx}&Tm7&@S6ggxN{E2@7(=0_ z4TC`b;~W@iIgXb09K!pL9mLgkvrxRD9F{pH4yVpfg5r)L;1To}{k9_T#7_tO*!D7U zSKyUDvGWc3dBp;IY5T%E;v9U9t^t-(l>+}{Ke}<=R$A}P9n|CNhxy_$XlZaNkk~H^ z4*WL^-Ua<2iWYp}JT(GX;j|0}$&r9|HG+bRC-B-AFIWOrqm$q4KuP0cr1zM_myGT3 z)hatUW5XG`;A|TmT1DWuzgL3dm(tLjnE)Zbx!_x8Ec$#z4yHcVMn1Prq8+^iN<0;b zE4|j^q}z9pwwfMbnj!x4*AJ(O4ue~&OF4rB=V|@c39!N;k(acyi=ekuf)+`Er~QH^ zUP81s#}QaS_p|PBA~YAAyxfKow}hbe5ff->g&nM`mI4#AKY@)OLSayO8G5@Sj#wn3 z1B?|1^kRJq3d|n7a{Tk?HYlHVZ^MVqMQqb1O2&R4C zMpx*PMAfl_Xl(QqA+gXNp4o5*C7n?OTl%JW7iN$+Bg28%-XV@-<6W@j4MmuK+Xh{) z>IcRTKLX2X5&U|a4ZLR_i?nJ!fLDHRK|(X zXiY5JCycXI6yb4Cf7Cxe41Fpq0zz}YqS;x8L6~15n0H+csb$&WBK7%LZJQLX)D?$K z3A5p6r!|n(a|dd-(umx04o;aRK*8TVi9GX_q4G6L7~+!#lbmlM+b~lFp6_rcsB9D zr5jyRS_?8&$B6}Y_kmf=7=6s-E~uL84!cz#{MM%ozclG${-(WjUcyP#a$j(!jGIty zrx+Ao^8#%RNChVjq=T0SrD22BDmdJ4gQhezq2z}U;`RG>klMHtwM1S-nVT=7>ziCr z_e&e}-Np~eoEjif1Y9Ru#YRG7SQocLQLOZl#D9~|p}a$%>Hk7walyqX{LE@Ea-N+9 zMD@K0wZg}AuFoqVVpM=G&lbb#cVD8gnWxdpK3!Ng_6%@8?}gr1o9LvZKZx^^3$nIe z2ge(Yv5M(wz+~HEJB1@?q^^YsQpGS_#CJ~$6oS5Yu$(Fd)mVI<4sxK zv1P8X^{G2-8x4bUeXqdhg9kvH-g21elnS3#A4O%K#?ZP#A>!IyZ=AnY0nD^Fz#5K2 z==NYS{lruV@5p*i^v>7^ZvPfSkGJH~G}3_vqJi+iDi;{oe+4`{C=dO0+tBZJQT%EB zJ|O1CfT^#w=u9IrbMz1RVGU_#9gWGPUZF zyoUq)Dr5}Tr5M4Grd;IfX@i&_n>j}DyFpYU7mjRb29;;kpxHKU*eGD^)R6 zs%wuCca}}lSwiMes^1?NB~~M~1udZ9{1>p_NE+VAG{f$5ZvnFZ6WFqG3`mOy;Pmeu0?8i(qf7DpnC60ZUu8XdjI{6mVz-t#v&Y{TdRm zKFQp`i>dd?EEMaZr0M zJQ}+kO7(GIS?zw1sS`@uBU8LZD-wIWOao0$(m*}10r@X|hM-G6iX{Jm<}KbZR4fgK zUbn$=VVZFNhVy74o&)#^ogh@k2xmXvhKPI>Jh;#WzT+$5o(^Hy@}iV@P`?;Z&vfty zz3uoy|4tnAsR;aZdw?X9=3_^dN#t|Z9{;$n2|ZJEQFn6<;zX;!!=atJmJkY6)+?y?{}r>;Q{?t)b&Ik|A^}XW}B+O zSUd;0@4W(!r#wY@BSLkZBbx}beKp9%el=Qwyy|WPPc)e!#d#R04Zq76;{_5sv5Ac( zc5Jib%~TRW`g9?1x0;1bjh*0iV__V8{U}}dE(_>p-{pl$2or7>q@mh_F7Q~~2foL1 zp;?0gZgtZ|j=qDS#oYmS0ZrI%ZUmh#9sp(QatUFV4ls4!5iYG;i4NK3A=|S8?)L34 zd`)d9>QvkSlArHJV_G%z!tA4n7pwqxE1pFSH!{#4XD1lv{)E`-6%B&FanSPHZ$a)0 z4{R0Fi(I`e@fwY}aAkNQD1CSktoS(xHhXi>s*qkVQq2eZA{Agy6F_-|OVIBC3BU{9 zgZ%z(1L6WcQ&v(v@)TNvR@nuhCd+dmq5c6%e;I`8o8v)w43 zTDQdnoiCXIJKjWt*zuLnedT;~&=(TDUyWc)tOy9;zeDHil)zHWN!}eShQGEs2s)+J z_>Ej6S|}u76b_|>tm3`&smm+r0lbgn+wmNv{SwBRCRd3N1v!uq-Aw$HD?%o+Ut2+VA0MPC1XMh8?E;c<;{yd!xd3a^+0A9A@!eX;`(ht=Q` zt1D<->>+SuO&mBImksWGtR`HqodzBk#vOMZ9OvD5_#ZF}OGT>EKL~xn;QzVvFfq2? z5k6Oc#mgW^93`zb5W2JGLc4|@FuH0l(iE`t()4J+zqkx^eA@-B)AnGksW)`xhaS`x zg26dtp<|gR=2VK0kyc6}O z+(8eYegIcoZvk882mpLL*xqP{?S1;uqKtA>#WlpEZkb3u_ze2iW{-QvXTeu%Gf-o0 zE|43e1s9*EP|T)cz^&DS8ReT`g5e>Qd`J`yd^f;X#h;+ptT=pO-izoDra1nV7gqei zfM2bbiNKQ;oV+$n%nMS$zWTwq{*5h`=}Ey?S3+F>ToCV#7lGkCOEmjX7|65{Fw@UY zpz&M2*xO$THr_jeL@U}jO;Y3F-GkZq(Oy#+>VE)s++79F6jEr1rZ~*I(GRwA+iBd` zOsi4ZJh4J$=v^ZY4@!BXaSa~YT2@Jm9G;C2Tgsw3@iibZMIGC`cu&ihs=>P@J!q7d zf=jmS#6-~{8&h2wfa-41@iCQ=q_J6^ak5L6?v&`sbR@sH!Xl)g)8+ z{f0Bdm*0DFI41$$7BHH=c5vW?)LZ)7q!!FKeghhNtO;qG7T~k_2H|Rc06Dx;hws{? z;fh7`V2j6H6i_#wgGz(1( z2*WRS-76~32*OXLEfsA~|E36V$7`grKw zy7gcvLKD4ddX4-}dxF%h3AidX5kFnc5NlFIVfklMv_no3d+BdRu_Aq7!?zyd@%%@? zzVjyHt_a4C&+M?Iq7y#;d=?yA*)FJel5o(CARMt)8|@kUj{xSgv9d-BlKA0?KPsjX z+qxHEU`67a%O&u$2S#voo-hjCYKXu7G{X}^N0Cd(F0eb_1m2ps4@x7yAmR0*s4rFs zuRG^W5QHD$8+;Bqjp*W&x{@&E#+2hDrR|`odkI+H)B)N8bg^l2C^pvLj3?WU3Ao*X z#IYS4pcI<|xqpwKy&C}R7mk72K^Abm<{%+SnLUMpg|2zyKZwhI#vmWHB3D zcH#t>8#DB}5&L#q)biDsjf zg0r{5wjYFKjiUFj=fYb7yWlY<9Trdhq~+H8!oN*1@OF1P?5X;N`~@@Qj;UWrKIbdi zQg4M*r{941zOA@$tpmQZwHP?xe2J<|Z_?j`6|h35E`0kf1Z@npfMP|maQ4z>bU5$< zIJtQ)N{*={@)Ijjaq2!|ReBICzn}yQ{8{>ZMix*Jae;Qg3x*vlBFx+W5Z9U$F;Bo) z?K-4@m80C?C&9Vf@?#X#vbpHFgb&$_T(>@QBHcMv%Woih|Gv8J zFQZ6JGKQAAF&j1PIgduS(40iQVvy}r0Xpt@fzQhp5Q@Q>;L)W}RNwRt#6LerX!5=R z&m}Q1KtY)9&l>?eWi#jr2>3+11x1>*qFvkobcTBjZQ>E2d~G%Ij^V1CR3c-V}K`Q9!_l=Bm`)EPo_R2_epn+x*3Ttabs z7vk$2VLJGD1a#injn5!H(AWAqE!3>f%u#0Y4RfTMj_mn#ZLVF^5_{~sRJ&A`ipjuJagJVf_Pf^htoBUn;>7U9zH1}Htgi`phlans^q z;?H7X{OTmcauZ?LWKlX&{r(!I11tPCZ42_hkid&O`i1A7Zh`lYF2diNN!%ay9{7Cw zj-t{WVQ%Xo;C5#O%sUW{GhHOnmwi8YW7l0kX0j^$*sy{#VI~Bn1v8qV%8I%jLZc{H z<~HJ89!AQ&Mc`VRHEIa6LFD>e^f}fCo-;9liv_Wv+ty20=x||i=N#y_$_jmba)w^( z`Wg5&ia`0@gTP=zCD5>31h<5*fS!Z`CjDj-GX4yDTc#8Jnp+p0luhcL&!^c|3Ssf}OSmk`Oj zEU|pC9sa)15MTLbK(Gc6QA5UAWNU8#Cfd!RU3e<2tk?p1Yg34YTiQ|AWpD5^zn5+< zY4@Q;gWAe5VG#nO#0q4Y#?7mSjWn>Ja9WCH; zvzvtEF}_2$$uw{X)rJbH&(Q4iFSV6{ zf%!>f^;-omI4q4NG)924qAz?^KModJ%*I6R0GQ!<6*L^>!XG!{fmT2_S}V3295MtT z>Af@b`RjttpZ){9+rNQxyE{Oy)MuW!g)Xd)+yz6UI?>x%L6CNGhr5K+;Q{p_T1-&a zmkZ{$1|6a>sNf|1pYchsS3DTIeCt8mZ(gaJqjMAWAJae=ZiM3P!+}s&I37E0oQ3md zrC=zMkKU9}z>mSHG$A0&K%_RZ4rwsz@YBG3 zbf;a5FwXZz=LEIJilmhwr)vSs-YX3+S_grsj60m7)8imFBog{>_QZdONX5|Pul!NH*b5EmH=?|4}wUyB*I`F0$>Exs8)TK^x)7W^J6XU*}Aj^D`m-%Em= zp$308eh06T^`K+iUqb$=6;`V*LjgV(Sd*Dc3~2Tfa>F*TRmKBeh*iUL4MUM^jv~Ce z$)HX%{35XaYYJr}yr8LE2;|?tPlq_`;Y8R5&K*=j^Agq~pR@taWNH`6w-PYvU)@1k zw>D!RQW$><+>b;Bbfp}nH$eL1GH4;X5AHP0pe1J;pus)gIHLCtqSIn$LHg-jr0Bzi z%2I+ogq#7k`=to3_HF{&ne$+T?-DTo$bH`6TYIoR{~NC^`72ml@)HaNxx@6-Fc=lW zf+dDwaPMDV(70F_<`~7$7Ts&%oAYXTnKMm?B=pe#4yA*Tu=fc5m;?`;)$qmGMKHqb zEm)iC0Z*P&wm2=h}(A$4AeWqE}eBS_7RKpvSPs+n}49-LHWvgkV0ck1v{9!Nit2)ObPxnbar|bEgKtE~tQ?zgUfu#bt5ECULAH-9@iB zc@D(cEd+OM<>Bb9H|TP>9^NLQk9U5!Ld@S{gB25=p+93J=4nd7@@oZ1q$~<61TV!i z-<%f2$R_Z7|0G!Fe+rCeDZ<6l<;du-2;O$U1}`AIX!b=j=s4HlC^B0N&Rwex2eO`_ zj018oo>qc2HRXts2t;!Qz0W*rQJjAK9%#xFoM2WM_DXSJTY!py3m%W^4F96b3+=G+ zy1DrC<$2g+UmUh+6x{SK--CP(C!vvlfq3B+8ywXk%bUZQ0YUS27#6MoC61l|lL4lf z`1*!(RP+|Hj>iYP|9+q^m?zVgAzC0ubwB8)N$_4n0Y+|=ha=YR(0s{ZwB+FlFrGkz zImc);dbtMpTna&&7mYYUANLb?|C~YledgmFm3{c@VIJ!6l!IsO&ES7?-QYXn+h9f6 z2&cq$IlL0!4T+#m&=y*OLWHvsp`AeQbk9cT^{%3Ki<*cC*GY6PcLNsQr}Ag?rQ;L`X_CNJ-{xbVuB9ODMJ_1a{(^AWk^_tvgyCIjR`_)##H;3b zfpqsxfS?vY&Wpl!4S(EQUDde zg;@t}VUn;9@pRrbpe1)5@m6rKoQ)dPC~QEx)otLE*?Q=*CJu=${|3}!GZ6Q|Y_v#U z4qn-Q9tDvuI4;W>+x@3VC-oYl%D!-zGOwGEGaW%`xE}3TRffdF4e|B32;#O<6$(Cn z8bmG8g+uD=iTvsu(C?xLjb_Y(&ZAqwXb%(%?54`ZH_OHPHs5@S)nuyIf2l0Jb zRlI0W3|E#(V-ea9hvp08XM@8)fVV;$8$I!gmqVaZdwk;g&@FtY72a`X>JcDIG$%xhYZJIV*qpW1%`H##W# z52fV zNE~fR;LUv>>5Fe?5Vwd8*zjyfUESro^p4?U=tZ|HRJ%xEGT#x89H;18>g$mA2N^sR zq=ut=D}a-L!Ce!hOMKX!jb6p=Ml)O$(9_<#D53cW=MA`s?5}Eoh4F^?yw@`D=bb;E zH7f?Kl9`EXRUrVA*+|iu2Hhv!&``+2x`QE=V06kHB%Y5059v+lW8zjYmShSC6{`U2 zdl@~;umGV?%}^+K2b{B-5l=%Bhd){j8dOe$^b&6r_d^zT#$DmrxST|zCL^FEaW{x8 z5rM>~r(jOFDgDprI?&qt0!b=`NB#_b)_=aJ&%CGenX(z z@eY*re?Vxn5Z*iQ4tl!C9`9{8hL#mZ@b61cNM3S;kx!bD^h!;vbixz6ef`Rt|C0t1 zAC}|3`!ug|?onh}{GH}=?7=>pa#Zv3Cc5EV3>Lf3!V8WIkmC{VOs%D27(Fd7%P?uG?eYjhcSy66^&5zPTJY+BKXk|F#$cRhVsUIk0I zHlZZln>Tk_=mYL$1H5yspca0a&pX=VfVcl0BXa8I!_WXf_;cD(N08@qK`d@q_7GjbLm*+^7$S7HgI_r+*lI8i-Lu^cHko-6 zJK-FtFX&?pTQg9lBSt>5Yw>is0k)`K0YehbA^EMkFjCO~hmK@`l&`sfEf)NDzrUc1 zvAaNTY93g;B#x+Fd;=u-0_1vT1Xa2nM{-v)h@{vbM0Q*YijoM&zE;Axe6J$B^R^r5 zKYNKb`91=*X9fVH?+tgaI|PjjGlB3qF8E}$l^E9sD9TR(Bt`#2ycbO4!wq4P|=W@r^*8UJau?7{K3cNBFc0&eTOeTqd~^D2(MG^#|(E zZ^|6DJupDd;bp*FbSCVpcZG7DZrE5_2RbS$K>0ALf*hI^aPyz#NF+uTU%beG6~bHa_|tSe z`X?7x#8wim|JEY?ihW2qTMZu%vxb*eYe4ymrO?wa8D>3~gK7JYft}JK_)WM34qq>a zf6Nc8`@T#bUo8#CyB-7ZVOcUd{#c#P85`tD6~@rtEEM6&<#{x!J4_$0-2@i>6U?ZU zP0^J^Gw{LK9xPot8*dQs-1~!qaLz(L_|o!9;34%KtQir3DGUp4xI3cJ!FDwE9^jcK z&YZ#|Z}cadf?uvUgXEwXaPsgL;?k~dgx*ylG$|{Dhk1Vqj$Owh2v#>(1HoCAz0sc5Is3){`!rOa{;Mp8gl$CN6c~;EClPenOQr8`*q-z$=mfDBo zg_ZHyZc(Uuc0CMK1F7Neo3m^k(!3fvDh!Rf!X z@#LvY(5_YjB!2G)abwGYPyZnlktKzXy_$n-utE*c_A8g0?Li2OEs{(35=% z=xa$Je)mfU&)K_#c3otGwWj-lwZd$H=2MGsc-Vlp%hUoD!Lz$TltpDKgWyP{8kD_c z1AEHMkSCeV5ef^0a?AF?8TluX&?*)1+^iBMrZyng2~T>x^>!dcT?K|0IQ!O#{_ocq=YN1oEhoey3) z9&7KS`RFqGLb=n!qrK=;*ItmGpN*cqrBIQe2MDeEgedX{I3zJaEB0Ncsj0`DzdLMT zx}Gb7YNv=Gp+fXYxR72Ea}Ugxt|Bx>x(Ko0bU4})Ouv!8PW*RT9c2CsgTH&5LDm5= zw7i=kMjC#jwW&h5JIf2tiI##2#ww`5@H)DLB%$l^BZQy6E-alG2l)|C(c8n4C@D{n zyJHiOqoxcvdDj+7KX?Epi-Vx^@gR8DRusRN-T}`}?T7F8UI!Uwn)qYvI^0;i6Bk(W zh}k+FXx6$k6m2^jtLpebhg=7&LGHpUJ&s`2wQWT5Gh5_TDTXD^N}x(pLCsQ^kIJ{z zfsDvZI&gRfEc7fQb~{SKvxn1x+Nc%E9km6KSB`;%bPssAryK1kzY5Osp3uGTpCfhg zC%{!f8_(@F!C&U9Zr^5vkY4F|MQqaEOCo0c<2lfuj@>U^fRFZxU*w`pQ?+OEWl4?fbwtle1 zRU1}?E{6*&)}XVGPXmzqk+Zw-5J6O%fM`<-aK=Rr8Yrx#9r9PfYhAJ6y#Vuqz6hoYMe7VdemvyF&BYb&&pekJfBj=Q!(7G4Pf!gF=J$uykxI$9Iwow%cq+ zA>;nURr9+@%fwNzt^1#&GjWIV``@_iA^T3YL@4_<=gf0P2!*txB($eZE837Og(QVg z2_;cTm~-YimeQWGwo8kIl=_IY`Ofe656oQGbIm#Pe(v{u-!BvL?$kL@=QfJJf6#;6 zyY5haez@~|wK(7&dxmfP$p|bqfW+0M3fNV6kOSuph}Gm~PVpQ_mPaYV={@V2rXGfaAI5PmSUUsXmizer_HaDYS&<~}xd0w^`jH(^ zHj=7&X=L$<>)`v&`8dWkmL%$|BSW1EoWPA*psji{OwK6*F=mAzp;rf78rubUXB6P8 zxpp`tX)VcG-;9kmPb1UQAT%qMB2ijyd<~Bk*uq5{Mp{0=R>O1gYBA(|_+c=ZETRWL z*^P6&?+Ex0bsq75KFGvsrsweL!3eUAa|R4=*C0brhd3*;9)X992flUHlGx1NOlB@z zKzfo)iOI=*IQVb@XH#n)m?S?Aln%sTj{^e0oA(|V#95Lr#TwwH$9D2|$#%lTnv&(4 zq~MZ+Kk?5dJ&DEgN#>-jdgAQC{*$U(jJpc__IbfzyJVcwmH^N!tiU+*kS;BAhuj8w_sloML(nM!=BM8l1;@sJ~ znrJ`VOAI0iHbRnV#Yw$iBF2>ywWGGMQLo|L^>lrVs$-A1u3cCV8_}jPrY>8ow5l;LUkWICcCzfZUwAONk5MuDVjr#QkYRd`SiW z?Kf!{-{}KBA32Gg_U3@kUh|;0*ecj~t`y&0vl1*-*$7uA%!RCi3k>+#!kOJ}4PCFx z!mz13fcWXD#NW6XAGyvWPpdufg*n$j>x;?o#xlpZ`r`O=xAl z4{mvB46ikGg5Mid;i2&?e(NSl5c_Bt%wIW#n+){Hr6}Uq8gBx3-uMf)Yp;bf6|dpQ z^_KXX@i)Acdx(r&cn?NmF7r24Ed*C42KYO?Tsg7PEdJ7S2MnH^3?B33$zjWjU{=in z`2L_H%$h$0ey!}s*R5^I_L(tcZK@2;9Q*(@=L87nxDMPqF7zsN+F~K`hSczb$&>R- zV6)a)&c@Cy@Yt_qFyfv!s4Ync2Yijdx6aQ(xb!!n$Nw=WWO4#H)|CeCS|0=3s`uA@ z+k6rnov8}LZsh>I3(EMljy&m@=){*ccjKb{OMt{rNSq!nAX*(S`4=Q?vBw8P(7#(8 zZ$9@EH@8NE*#BG!|3Vlpn0U+?tbF0z__-g4!=reiR!sGtId?cvg@6ck2~PU*Hz2Cs zk`p1zgAbFpz(r4!_-DV)Am`7lC#qHUgj2%6{h}IJ{gpder}P&0MU-;#o?PTKO`QO; zy9HqVzw_8Y(}S26PbC+=OLBr;zwlQ`nZVG=vCw>05{z75jNf+*ftwDJ&PL&bAfodt zR;(`r83`}IX@fx=VIU1R#7L1_4S8VUMpe9c=nmg7N(!2;48dJ9^}t>!b;zB)n!k9V z3oM%J3jcD-LAeXU4HeU1xaNG2s1gE>t{K5udm8YS@v;NABeSlk*02dX;2rzdZLtI8U9_Q(b}?XnAY&Cw;-!gR=ee^a8Wc9V0=h;2RDQ8tNIV%PqA6U#!Ut|OxdN6onLpHws{xuH#B|+Hp zCovwe0gY5RyFZE)?qnx%?)mP3>pX*?(!3q;EvKT+e=P$yeTf6FQuLs*U?cyVRV$d6 zb`L0Z4T8RRGfB491lW91+Ib))2~1sM3V)uqAYW&=*9lU`_)EjANK*Mva8HGDYE=B7 zS$`eaa&ibaMtSj98+K!v?06g{??JNCrO9Nj6dAXtoZGKU_|q1Yf$vMDVNGu?|Nc@1 z7_m?dnze_)#iw(fBu_sCb{(GN=&CmS&t01!cRSpG>dE((*Dbi`)fZhE6*|nyC`^VN`TG@T-^jut?}cvAmsKFMaVm^nz79V85sdp% zOyHBrT)3x6686+X$lkFhc&HZar74_r^MC02h+ zmUzbT4WR&QOe`kTw*qk4GIWimeM0u*_Qna`b``F?KhEou*A#x@Qc0 z3=hZdhdP1n*&E>DDm$p=xfh0V{$l;a)!<^1EXnzG3fp~HM1Hw11`}<^!FkIMAh_KD z{C;01!r9b4+M!ybU4KcW2HjVr)8rL!Q%O?cm! zg~HRhqxjK>Z+OK;HCSg93csg@!s?}I;OCw`>`^j}1P-nvwaLp#x~d;cD%=dZ1W{m$ zK^foZ+ZY~vJe#PCMiay6RG`0e3RGOH4C#+cpkc2b+|r+d*NSc?*H^Z`Kq> zI7ta-y;ykErW4SM1)#iVI&^yP4(|<%K=tjr$l+XHviw{C-fmLM(JwUuEqSMb--kp{ z_U9*-(r&`5Ca&RY@p*hh5j|WmiQ}aIYZPP^p2Jh(Aj~_kiLBkSjocKECyB+q;6jiv zhsdgs=35aY@8wf`-8~m{UtSKk2Sh-r<|v?iA%hdnEyS+EoXRzW^Ehq)SzNwtt@DYn z3aqyFCsz0yj9W%5>KOHpnE$j7aOw=4^wJDSchV?WC~a4G)eA_~>aXE`5HQztj`s-1VQScs3#pAU7k7Q>597xDJjtZ=`m3nNtzLMNX% z5L8UShK}FPubhgVL^q3*gHI4Ko4kpvZipez<|26fgd-9A`GQ|UHsQWS5T?{jV1=2} z@W;9Gq~gUB{P4>b?4ELzBbJ(h7iaVEhlX$1H?I&+UB0t+zKAzbu@ug;MKye%UKt=B zeWT9viYLBis7-9w{D`kcNw=RxSs&tP9>G+3~qi{q821p5lAz_E>bQ2*LA z5@H+#LXIv3oTY1V$R$r4WK)A>*D8^Xa3(aVi{al~ahSASIz-_9s=D?=5AkIS9b#|q zO^O#;Kxg~U*f_lxmzHk?djeyzgToPUxB41Je_mtl^&&9M?gi$mdL#^ARk0S>gfKGx|V-N@YQF3epPiIKKsxNHx}q% znH%N!*JdTY>0xU)4(?!2C<7Jw7lE%im;5-UhI>YI$*zfcFzVM_q9`#9Y;r#e*1da& zbr+U!cF*M!-M~7|5~cz-?K=gQ98ku6EC2HCEG41C5QkJw{leG$G@HB_Tg`DA5yf-G zyh)tPTv8L)jU$c6@s%nA$Wy5Y=l=O&%e-z7to0NmUuXv957dd#76NwdevRvbmxBYX z6tp>)a$Y49&avt^rvQzGWb^$Pn2KGYeP}u~nq^Ab9y|u=XH7`rcX1eNst2zsNLB z3nPl8wRS1IY{Wo0?raFtE`TuF1fSP?VM%8wNXfs4X@;%5MF$K}X zk3mg(E2y<|1+Up!EdI|PK79+|rq_Ykhd2^8FpkWsNFwumxx`zw9vs$lCBM?{;yLkR z;IotzwEk{QPW61j=jWRdX_eVTdwU`&C`u)6E3G-IHmanuVi9>&5=dH0G4^RH1#>^? z!CoV6C{rLw4xO-uH}6}+m22OCzqJ@&$v<2d*`NnE%2>f8CE7UXvjV(n{e)v3GYa&E z<;c`4j&PHV6R~%-#ao@?@T8Lug&tu8=zT~DYVoZgSLp-Tb3}x!Ue?Dqon;Qz?UyD^ z--`KXyR3-B_luy+dN1DWX-N+1GNk0~SDc#PhLvClIq7_is2?%MyT&ESa=B1!b%J6c zNCfK|2zPAjYB_wB7|8i!1rl#;=PNdtLj0V?z1k9$7k&c5kjuWJi#n6=U*g~=bNBLR7 zpbc_Z_G==jygLrA_{HI-n|@GqZ4^8{al1|><0~*k-oosCAhb8wDXdUA#E*y-o;USe zY@(q69yCYb8T&(luTWe0c&`RLw5ex}RRiXB$4J~W%ZT3rtQ8(j)>+q8)50cki~Vhs-Mss~;28pOVO zDLA`%6yFn@P4+177A_RoTv zvvxwcPJ&;Y*5~uD=tA4{+o0ok~+ z0mODZ95&}|h39P};oaL`v4bZBTV;NOcg~?OH^ZKcjs4^JdF29M$8@kND;B(*FUcQX z>O;OiUro3NQ}OMxH=JERG$9#%4x9q5fO>cmzT4`LXD#+7@)-peL60K#Fulr`QJsss$E3lB zHe>Ac!h`7j(Xg`d}`5Ve)cI2yO$WmAu0 zZSfCy2HMI$)LzMP@EHb$cZYC8WhwUkcnFkiv?AT#wiE8V8T<{rci@Fkn-v}AgLc^& zFye*?ndKqJQMWxxrq;W|k%HS;|9v%Qyw(RdqH?fbZz<6z-bPl!c)}HW@gC|LV~6%A z9QUdf|H?W6s+co?J?(~bIvqd?Oa^Jar|^0!dy<31NFGsyORLNIPaj?fn_q--;-ueW zg`OMy&`1mNXu_PBx46Jplb+zcd9wiK>5%gerx7DZV|Yg;1=t4u=5%i>!%IFa$0Z>R z!n~C$nHeqvjebsnnfWtd!&M9T^^rH{UJC|)c#mtp;(dJ2wH3I%Tpr%t`x6&UnZU=a zwXt`1hcn;24;Tb=fJH0*@`ZhJ{2Rbq=*6FpAKnxP>yKRGJpJVf2W}*S>*wY<8@kMZ z{I1Wy^}>8;(Ycy@vo?8byW%o9aP%#>iiN)1PZ@Z9M=>7!`wkz` zF@yOn+R$O(C0Km@C$O8JjJHTxlFr1hU`@&gJYDw`*w)nwdLLQCPakaY@T)&yWy2^x z^t~s1wfZ|hBK*fjYSS)4(*!5Wj zT=xEm!@Osb>Fb3ZC-!=z?C3@yGAW&JV08H;Xg&Im6Z}V?G~AxbpRDIXGP{)V z*v4esU#1S$IZ43ZgIe%owiVetG=n(S#}h&59BdUM3IA3a^Y^Q2z_lZb`N`*m`>M;{ zL^o&!39LAS!wh-gUrQ<^?sOhGYNhPEyE7=R!~Y)0V@28fYL;` zZwSr<65)2xYOf;n^^^xI;@sg%x*d9sAA=S-3_l=N={}f1VY|>2VPu;HVjyy2h8B{Uf|)-EF|FV~dIMtp9L^Y9!dZTXk+ywF=k6%~^pkAKk1-?iyhR+syLccgQ_)$BQllHh4ho}jAM9wGRL(de6 z71Sh0Z8XT6K}+&`-XZ{PW)QoZFF7K5N5HexHvICxvv{#E3ln{%lRu(93F7?@usKc# zzR`Kc&w2hBtGzWP9WkL~`;tT)Unoiz*Usd8GrJAmD@TDuM6#_;LD|?%r*f>-yNv2SaSwTTE^n5dzRp#6(u;U`2lDBh8LU? zqq{;MeHX_%+u3G6Ho=8#>s5{eQCZk`X%?fcG| z6?_X@TAsotrOLwGgAx2tavyvEe9|<0V&QrGX8RuS@b^0YWbGDCaJt6->ow>@6sZ{Jkxh>R;h(4X@scWWvf}q` ze#7b%5ND2YwDBoyFmR9WSDgzougegZn122-V^z59XA^+i70GmO4YI674vJ0T5GvwN zD$IgNZ^bQ68`2;W?|ktQ5j8R;Vm97*?lGrlnIbV>$-xC`4&YTD!t1WTty_B{4I6pS zsg<+L;cV?Ug;g=L;rbbyaL)oID3!Vdjwa~Ap|lCUdhsDp_)~>Xeh9E2EESv28Q|YN zZ3q2ww}Iyi^udqsv7F9O$PxUl2aZ>#U_Y5JSj)qJgH$uY=VP1ktDhoJ@umkf@fUW; zUJ!>@c`-mbG8jl#HVE@%sdasI{osr+qp+z*liXKp0Ks1e!N%=cxG?oAKmO-KzLL<( zDu8}OZB8sPPdf@8IjX}ykFBBUkulJ-wartgrAR#{9Wt z!vL-yzZ;4;8iUa)| zap&eQlZha#NSJH?T)SBLEvPGU<7E0_y?e%qcKkC(d+<@LsOvpqB~euVk)#-riqhyC;qY>W#CbNJP0pSBnLW`Np!vrF%Ay`(uH2wbZ8Q>@!SgAC3^AR@g&Z& zgm7@uAp>}QScuOhIfB2RMM>2H31Ju49k6VY2q~>i1iLT!GOsr6WgpH@q|=u9p%C2| zcDMgSKZh z3ZwjQX;fkDT`qI$B<)w5f(nxd=vH)|ev|Yg*H)XLs1<5dGHd~wDsRjSvjjv zP~WO5_RWF0^ksiM>WnSs8oOk&8GnAWM!{!b-9QUl{l*0Oq`R?Zi}{RiVg&lNuaM1p zBFU>6T`p)R8C^0?7esNnHZ zWRH4>Mu3DSIa47F!)=-a2eQMM7K zyjy8#J~_nw^=SkBTP4cuoPLvA_~JfVtuY;ycXy-RH;fU>SFI0r7h%HcQt8xVj_C6R zJLXyBYi6l4MJY*QXhiA+Rlcc<8s-Ve?vKID#%q{a%b&zzO+}h%_lYk1wVgg0Z$Ou> zwWA^iD1UmO8n{ZI9qZ24dd&0op~?zP0*nAgq8cyEs#EVjs_f+K=%Am z8dC00+K;DF2N72kQ@5M$YtErQRUgm_uiL02e;VDiS%&K}>jYEycP(N|bn7M7Jwf{{ z`dOU|iRf?nAwg4?BYRQoHF`GXtiUU5z3^>Sl-C(*i z(WR!|^vI|f^?sg5+bX@Nj-@M_*RNlH=GY;@m2K;3%yL#B@y(MN*qz3Dbp2(MCiIb= zk$|3kbAb_!OQA8Q_UQ7X<+N7F1{R6!7TmifJX7=H1(pGd^^ujo*c5x+)6aH^L zT6*;=APW>yJzo!`YBZz|%dygW;l{@T#=fEqaQIg{n4Bigj)HR6P?WV62v zq9;;j6h*F}YmEEo-okiB?9we{Ym$RbDkP%pjB2jK;X7zc>_Yk{D4eDkyhAd673hq6 z20FT1w*J(X%jj^{4*H*mEpH-0g5BUW!Pw~8BJ`?_JszHkZpPi@#;kXsx2zlBhy7dW zpI@u!tbIS3ix$JxRjql$_&G+p=ai)rtlLF9=%Oy;tzA{>MzLAc*)lkvDmeet}oxahK zM+KEU+BTq6KTRcpu5{`Xj1D)js}EihTxz*O*NRmm<9J0B?0J=Wr8X5wyDCxU;|Mc) zTLs;|){GviSK;0l1?XT{y5NwT5~>q+&hNf;nqIZp&&+B1$UfVfMW6ltB)Ai(Ppf6U z=-Bxy$i6m}`o?@@>`Pw@xg9sqr>}`LEa*6@Zw;j9f5db5wod2WjEzUZIXBU*TkXtW zBu9G%ap=I@bivcPGAM(yhbFylWI34&yW z++~*5ih4 zR(YV6)Ag8U^F! z>BYOcNL%=`Pt|)O9hc3mJ7&PUdtVxbL^z=5vYJ%q+X^O0T?!>D7YJg4otf0n zhOBpcIc=POggw__fvinDSgQP)`kl^X@2y8l+tpE*PB4mSzE5>5BG;l9XzEfmMoHcqeUlUAO)q&#-H{k?C0$BS zeTCF(%79?QP$MnA62rR4nb)stn^v!%zJaPNzlw6_%ArI1{zIW&3+aT|GuH28C0!$u zPVJ+W(d%{RncqjwAcc_wY;#l?T6*U#8~x`b`J5IV4)I?5|H9 zg_+#1EoYHOjx^G6WoVpBGd-D-%YM^*hq5w-Ou3gA(W1ssniQ(Q%aB^h*tZ;} zog1E@wxy2+v+E4eyk|Gq$gvEXx?~;0KPBuj-h7N!FR^596r|v~cAyI>kAS&YLLZe$|=Jyn3F_{%cO6 zm-NL@!7?jmy}Ti^u@VcycTKY`>im<6)NkxyZ&j{Djs8h!O^*~|m*p^7A2(CET_;(~2Ql=Vkok6La~QJM z&qp~Le42P>9j!VakBZGLP~h8QBsN_ZHF#RpH+W=HAvTlikle_8e|whBg{zRfj0x}M z)B~*IaZ5V$`5=>a_2+NjLhnQ-^BatwHyH^s|?pbkXk>JXCo& zAN7tsWhSnxvT7l{0*TX_=vmx0czte);9*fBv!_y>cO`Q%+Ufq4O?{_AxhsD#r_O$4 z=HyK0ZIn2H3Oi;ZjdNP`?*coDC+(#2u9egwT9U2stY!|st6(--2Tl`HP$xaD)bqBLv3bcmJ9tD)I@!oEotn?E#~WRHa)N)m-WV<7^StxnBx^EC@{~Xelk-8 z*{)-3d)9yK$Z`+*wDbt--%3$%MjJXJ?j-Q(c_-LO-h%qE0q(`UjwqJX#B|-5hGwqJ zW~+1>7(u@^YP(#H_8m55qocG?OW-gQ@EfBuHNyF=LDf=xoM zfxj0j@4tu&#Wx84vj=RZT`4^o-^5JqG^N@j8_?b?G1PeWC~eLiWsLRq(nm(g?A9D5 z!PsIkWF!=AUb_>NeNlm0ESBH}MYJ;`F*8tr{YC1#%?eq`Y(che;wag4h&550pk5OB zG$x8N&6+n@qvpr#ul{sF-X25)j$9C^1qCAQ7oPN~opI2W+twlM=egfP!0QrD<0iLpWI%; ztl|c;IdUcFSE2>I=^oBaTOCXjMb9DgMt!s>wuFYC&u2$A@fl8W3VpxvGrIls3LQ{{ zh_B;>{=590yS81KzDa+8rfm$Oz1LG2RxJzJ8c#>u4?X2^d z!5+cEzQwd}(+qZ^pqLhy1~Wf1mZHGK)yz8S?X+dhb=D#08agvA7V+YOQNaDn$p8CF zRxHZGUlWoZ=`n1W}?Bd z6uLu9l3HkeV;qO_(cN})^fXGy>U<|d8>GZJJMPpAM5^Pshg1gXsYjL6Y{xc1iD(sS zST4hyy5IueC@LZ!>7#6j&Ix9|$!oe={3!j@U(cGa4x;b#t1zV&eOHG7ao5>>gXvm3Y?6x~q^j(dR@hLM2C2c*1(lwQqmaxWf{%~K*rC!qZncvq ziXEJVG zwt@!!K5nJEH=4L{4qX|&LH#$#(hmCsnx#9&xOmN_o&K4$TKq0;Elx!Ok#^cseu*n; ztc61I(*=4J<&0}p0!^P3NpF5}W&Xt7Li4|L)4`S>Y{9XwG6nd^Ohu}E1_Y@S!m~mICRGyx^s`m@o zABC?3YAec7jHMKPte%fzVKMvF-hdW2HL)iWO}Na@Y?^PbM^isNqZ(-m^mgJI+UnAR z7QEe0f0J@1@UM{|XN4i+oZP{N_AEfUFOJX!LiY7-r3NXlj(1q0z?ROaYQ#=ms|-Lb=*HN+g|<69m&b~KMM4mG6_ zQ;ySe$-``N_YrjGdj^{1AH$AKTT9hB8<^a&HfBgJj160Zxd*1+;$B|IXO$k=@O+v1 zsDgKv{ut}#3bwnTh%cIKk1R&HLawUl&Pi1I@jrHgi9ljA_At8+r7&R)lX>w~?QHj` z1UkCCkcqh*z<7NTM=c(Fbh=58ZdHEH-catP;dc|zE`#OlU8Q8SdVB_RESJyaD6K@> zQ}5B}n)L{*j-mfHEJO-H)@VR+AroDsgA{IBv*3aVTEbAq-&>Vh#xG&=%T6Gxj!a|~ zo`gQh9YzOYZJ2;R67-_%47TFbSJq_HdX(MRNW&_pQYKXs*$U^NPa=k><^Fosip{1~ zw;r-b$w8VZtf1d9Pl2m_JC~d5bdtvO3x5pPBpAU96BIo148?E%$(%_8l*8tucOJ#) z5|u%{@y6)m)*!Y%Ko4ap0p4_h2lM&2CfjB{iE%n0MFTy>nax7Rq|x{(v{6EmCbR^i zJa2@aHhyMxUJcWW%N{Tre9th6Z?4nExV?AxPCEpdtf-GXPIxcM9xR*6!qN`(e4zqSFSoA$ zXRk>ok3V6oJC~r97xVGzJp{Lx3PN1h7y`Gv9Z@ly3`Qo7iaS0s4X7XDzLxUop( z{W%mV>B!D`JEK15$VFzq+%jsNGzm4n;-f&TG&V)SfE~I^=*rx{ol_3Z&R>ZcoeS-6X)%baF5JRPKRMX9V}w+yQAl&r5_lEkX5-^ZHQ z{A0^HjOqAm7v$pQNIOCmP*&J;_CGX;PkuM2Qq^aX{y-xQH@i&Lcb!I$lCM+i<(aIi zvnHM6@|89HuNWC0^`Pf+U$U&G2bC)AL4)0=sJKiDD(`C zXX+lQ^A@|zr)93g+>a(_(J45Om9`HxLy7J9Mq2s?CnJvH9pib`(ph1OehN#3$^$n?oACXe*M9mJ5`pr=qjqSi^z z|Gq#!M8%?4N2=%^QF;14tAd^7wTpXGYzew_@C<7)(+L@ORkG8Wt8`UE9eeJJDEe_q z_#glJG|KD>rDqPEMtD^(y7VxX_FVbIjV!Q7^KIQ(Wi?eQBe#v-;C~T(k{;rYx2mH} zev9b03rFcu>9a^WZG^?a8c5cr8QH%Rt#{}2u@^Jy=$^0D^!HZ|D&Me6_>5xcyFHRr z=r?4CGl#f0#D^JO&nxK9^#Qi@ohRbSTQKpdmM9IkB2|GSeGsfeXBYJ{cV=v3`?XKf zQ<9p3ij*&G+=_Vg_C*}~?*4SwS~sOzy3MGH~D$v||!ub2B|$xb9Ykb`_S3)!z1 zOX)rfFY0!y39*Nx(6c9HjMBOcbiy))?o#5?$M;3pfB}1YO{9h$e)xbYO%FjvX4%L< zC5d^hY=p$GrEw>&c9C|sSh~#kDjLnIME9?CGD*Lu)aN`lpbFDPX!$B}YGK&PXq3O= zYR=)ZYmaKyOMNk{e^F~$@43PSX?0EF4b?Z%jOAA7`K`m~(~t$q=rTkHU1aJ*=dGbX zBodjwi;8IOvq(m9ZYNzbEfM*5XrNaS-`Gy+8N|zVGs-?-gQ7-%F#(B{^q1jF!AH3} z%;>`Zm^%yFg&vyqLLK)I<610>0>hKhd?*B zDy#99Hu^EK&rI3si99ytS~zn5s6|DNd(*fDK<1+?vU}P}?%Pb;e5ch*^QSBa*zR?V}i|e93acyMC>tPo!i>H#q z!<6Z|$4r`$PXo1*sgl(ydTsI^^jkWe8|AWqDeLW~|IPCivYY3zeCx-;NR1L?nn_Ffynw|MNlc`HeKQb5%bQ-~62Y_Fn_q`{#|o^1w{`*IyjYH>il6xIKy4u5y>H{Nlz7-q(o=(IcvL%7X^&Z)A2oIf0HZ z5~J}>9`qwm$jv<2h_bBw3B zbf)PQTcKD=E8Cjbg|j75f5bd$qFG7XHDXbiZYb0E;Q>vxyGLz#BaGHe4>V_n75cqV znwh=A4AJ5jH2jwaH(4T$>b&N%il(7ZZ(}UFKRFpqecg;C-g&c)t6!qz7ANGq$cR37 z_F&Xfr!iY4<{(~W73C^8(~IM^tX+H}TQ+J;CAZnqz`{6c{9b~co@9nRZ*D}oZjJQ9 z%_5r6Hi>u7bWD&cP(qY-DSG z@25E{XV7_>cbIi+1{ke^tCXYHP8VnHV@v#&vLe$I(4tfQ%!<61?7{W9w5gzm;eFYI zoPyWVoA0daSEnCG4jr-V+Lzndgn0%`N47sLpM9Q*wL8JwTf2^)8=FfNzZ+2fvh_4d z;R@Q|cwSH|DUT92ZbT}&7g>&~KXq+hi{_?zu`VxW2&Qxy(t#$&nRCDO=nr!C5Tao%@e`^@KV_Q*kJr@9fXc6n6F{%AcQ)!gfQnC~F&)zG;N&Pn|>^Zxh&$ zX~)r-S$53n??;gp_GMI6yal@A=b7U+*67gaYsN=C1_fs2Ac@OL?3(}fp>>;W@q@sN z=!=sfCG75zklBgS#wHf>2-jr`V1jC0t_VwO_N%mUF3{QW6p$Fk;GH zpBH4Gl4Mqz@zJ80v*^6WK0$#6iQHEeWS7q_D8IlKDyY^1XMIpdFFsgaBpvZ`w1j`y!) zGKOZMu-{*}+QQPP$b?IjuPFkwT zRoGq%XM7uFF29LmBlZ8H;qVgFI>@5Z(*x|8gWHgg?oJe$JW2;j{|R;r|DUCO@>mefJ3Bogl_~rCA2(uTH+o;MB`7H`K=K(^&=oTe6s~ia4vU?jR)scnur!qQ zC`~~9v<{^{7(gPUX-HlxmpS8go3;6Snf_b+gBdTr$i``$7Dz^Qu-xXwh)AjSelzWNOrFT68IU?8@h!kiR+CP zBum%dY__A@I;T+cE%(`f&L7$2^8xgBfmHo>hf#KRw}p_2oHC?DM%u+Ta}bZH*QEaiN43e4Sa};xY|+q%5c0kThD;RwdZMsbW0h1JTOv z?`(UpE*f?b-Veo;Ud!%5^ItAuBm-6ncZyf2Q^YLR^=dz|OU!0uAM2x#H@&Do?;C6S zM;(bayoWx!XQP5*Un;V21*;+Yn5yRwBBQKI8oDBi-SFQ{n)&%DRUI2d*AvsI#_UdR z%yBLBX;uzZ`)R_vC@zVH2g>R3lRE6q6ggB|?MkO^4nSpD6@uPhM+JFq3#s_jY$`JO zGYYB};n`gJA4BKiPt_mBaa$2GG9oi0DQV!IbHC3eA*r-bs6;8Lw9`;X2o*9yLxn=6 zl5x+u-{&f6i?#;sp-9^FQ@{HU+}C}b^ZlM@eV)&|ki7joP_*yvW#J{a2K@WT1}?wd zE%C^k4?D~(ptW!nh$@cIff*lQ+th7XnEIF~M*Jfe)~$i}Up+{ak`TtaBD?;E;ZaW+kSI2in>npCCM5OZSt zg@4BO3bXJy`8?%3wb;Lp!mqHf}Fh;g;rXt ziSaQ78IN`WWEhNr3g0ad^yezhN%fXJzfF+Sbqj`);W)aX1S}$xVVk}-rgiQB`7_h0 z&vh5VOkF6k30gw@zV^|cHx0CN^FOjD{Q&iBZs1?XZ3o4qdpLcS19=`b16Sy7!f`82 zq+!aNA?BYBj5-~O_vZB=J@ORW7wo58#!iUI9*!$&65;EL6!^9V_=Pu`;nb!y;%lQy zPu9j_tU#Y^tl3YZ2PJ}!>r2wNLnK|{|dtt%=P15XQ1;bT!rP8G-Aa1Ay zi_mO@Tcwz++XUmce!v5(+u)Rg61!m)nA#qv(dRI)w#5x8}9C!`uKKQ{3M{St6k0YPBT7I(b z2DIzmD${)|pbDjrp;jCLKeTqi^sX=T-`os*{qZw7zvKb@*DxIO^P6S-)lBTY z7ld0n)yGG;RooK-|y*>#^;C^2EoXQfazndltiaH+G#fT zl62GrkOrkQIGGmUX`@`pv0tIoJ8TMW9#n&@@hV|ok&ZNcLKoS*%aVH3jFwiP{fGyS zbij+0VDeeU)ZOH_gyt*i!_>dF(uHRq(qZ8Pr1jx7sF_oY`{E1m-298Q>z*5as+^5G zCp-q3fjuM@dH{3t7uI+-!XSG+NL}N~I|ZMFVfT~CRc9|OleQ5nmoP{qgJ8t0LNr`G z8(s#FmKa{ifb$s{{MKE(}YI9+=uIX~(mKJL9v23Oib@cS(4W7!7E^?`J&=@{s7 z5!0lZhD6my30>wy6P2P6JQes4vp&y;)Gz&QnM!#;-8;D=G?1B>2yUH^7}n`k#Yycf_1R^`M#;E{ohWz{LF_wP@s&^&oKiGkPG>%iwyB}79em(R@nF1Q zvlog#C&9DxyNK)ZTwMP9GuD31L;YWO;m?am+~zr4xVhzw@W3(&kf90Ce4zmOo$jbI zGz&|Y9L6)##*#UD7x83eIyMb%0oWyMskma&pD8fNL>GUTtpe$p zT5MAdq0!^DMXlKYobD35o2iR>1*at&WX}hm=ED0ey2SX(e9U~t(UfQJ$sH9n(bcO~ zc(7hXwE`;W+^?>%-A(`j{4{Jac}f(|by1ynR+6RGy4cozMv}j9COu)kQL-m}JDN?< z5CwbsN-Za7W1jX+0vCKp#tAF%xi28xfhKZ2Ydyq&Sc{Kce$m~DU)av&6rKC`z~eP~ zLZxOaMu2!EN0AU_4GKKM3wJfc$m(i9DHo zjF_4&#}^u|7&WU@*qMBchNfP?;IBR8obg&HewvDY5l*o0dN=e&jl&ojTX@_R9XhpE zrt%WB!rOm~@Q>mLI_}^pNGtu0k%uDi+&Mk`_Ixz>^(DfI@e@HX{Rjm4-l1`C-@$5$ zHY}rWXxZ>n66H`0QS`=L&_8@9^2XJqp-5A7T(Ar!GcHr@^jLhc>KTzaaoE!L&~)Y+EP6SNN}KE?{e!k6 z`*}AC)qdghz6gwbFatN{16uw}p$p!8rd68v$gX=sP5X;UWH#;?aoU#jq1 zvy^TcJ&w*SIZYcM#$flR0P<_%Exh@CJ({Tw1DidG_&UCg`j~tq1`2=JWA>G@8P+4= zzT{Z&o$(DSzU(1d;3;kFIZB4Ar;%rKSZFx<95yJ9BzGp)Vp)kAtjN%UAHy_pXm~Pw zd^H-}ehh&TCtuMUo+)^Hua+=(-9Y>=%@8gwU5+=Sr@-tRAE}_v0>)l`PN&fY`Cd+&6TB53eA`3nG;{FMC?AkKi-h0FH_`Fk2&z}f&>H=NkkBd-dU%(>oCiLVto%qw zvT*pH408lwi!E1oCluAH8wRN)pij zsgh{S&9BtjZw8F-UWQwf6{HUWLilOftH`daG3+)g6DV$0hgsL}3rmN)fJ^#OGWqcb zWRi2pYV`&@CKAIH=jX(&cLh1xT}=$QUh4aK0Z|t=V&)7*T)F)YUF$R!`@jIlXyUX&ajG6Xn96_I0XHU4>obb@SFt1v4blVtaDbLUs`YPJAORl{KI zg7bKBmm19YI|$Tn^^v_Y3r$Bi9}*p>2~YN1!9*q!_6SwUzpXK7uI_^Efl^vu(2k;z zROtQGgd@`v@Gf(N)|SSgjr0b#%`gM|ciAjEtNHtkcpaKaS|bvUxLb%QQ;_ zUXEzr%R!S*4BS59L?TZ)(7jKRaM?{KVuF|XdwvJW*cHiGwN)Lq`b~lPUCBaa*TLX; zv4~2le9&@05p)hdMP@XZ!|H=cB=k%MJUYig#XbSrT+9GfS%1IR`3p79yg;4}DuE#( zz2ww{E%0PXB0wTb3i*@%WoCeX`q ztDw(M8SSPH#D%__Fi)JzU*9sC7Caq>Gg6PB)%JbprI?5#NHUH%}OaIb05pY7t?ZhV(^`uF5L$oS^;}xx!T3B zbhx_or(~~lE>Rtpj?IfQs8-2t$h~%fB&;vS1DXFMb4*Xc_rr4Fr1+9J{*I$_+|wcF zP&bUZ+YEapHN4KjR`4D$PttU6EH0UtOBK3Ipe5ECx$A+#jFw(7JFpK8c65;~@;R{D zOb_!4k4Uy##o?UX4mjC$2B6rNI@}A#A+ZrqIawK3_MgFjJB(qA#t$%x=7qk)HK23K z4qsO857_GRQ)X{6(RGGO*GDt#S7TlAJft@a}Fx+VecHn&m74vNy}FK~O{ zbyRE}Ni9yw^gLO^WHC>kSnR$>mshL-?*$eZlj{PKm&?JS;TX(hzQV=(*NNT5ccjmD zH@Ub#Rdl0ZJevGb6&<_bhx^|>fp5dUP^M=C88V;%e%#9?WTA$zW}P|}Ez>~Z>k_;f zvjd!-HB$4PDY(C=kGu@>Mq__jjV50JDc`K&Q|n7%R?>TvC$iZerCE%faygMl&) zqV2_YC{2AKx#czrE~Mz-a@9=or6~biqW2>Ah!HC6DMW?Wf5C22A~Gx5;i{86RF2<< zT2mLnglSXJev&heRrZ2E1+MhXV+DF_Lj`_2d1YXKE;HEukfMEYn$c+6!+4{f2 zw#~nZbW$wjo5*8ZtAPBtU4(Dj9BJ{)Txe(a;|J=B57s3^(~~&3Wbc6onpcrmj@y9t ztDu&V2$sB=2s7~{9Mn|CxBrPD?nW$L>9(Z>lZxRdZ-N^ppCdODQ^582df4~20Y4{{ zq3(-0u=&Sl+Pt|1)|E7a=U6rDkLM(j+1v1JTnzfUuKuE)Cxti2#dp{X+KkNLY7g4;gjz3e~-tz{f5v5_YliFz8Ma zSe){OHygq+alj?`Cc8I{$_gh}`hqMSqtY!tvT2Dk0;;P!!)O=7D?Oi()wN%%ldFMLN(|n9U zA@#W0?l65e?;rnc%saA4e-+if6h&2)6h(m#Jg~7z4i={!BAw%L9K>w|+(ZK*XqJWPt5rX|kejo{FM%f!iR6!_W>l8PhaF?{1HSpKXSFFx-9 zg_?!9wC5miV;csK0c&`2iQS)xZeSfdP=`UsQLA)C4Gv~v$ z9!p&0nIgPBLkWJ4mctKMOmP1B2JnoVfo|W8$x{6$oL(|WwDQPGT$%opJS?71uP0^V z7he&XDbwnI`1hLxd&H18$0^eI7+icL8B_C&$gf9Y7+vaxwlN#=&a`xlT6-DR(*~m7 z*G5#*C)2npZP7F38~8DB7U}VOOAB?9B=yJj@y^yfqP~9)d8a%SudZ7Jj$v)Ye*O&@ znyDtt+?5N?x-+2Xo(pVaPO%*~Pr>s6(I`J*4=AUPmd1?_gtI%+Vcoz|s>IEt=F@dS zuCa+s`C}!zR>WiSd?&ah{YVGjDHfhE@`a7V%HX?G5Ur7(!aYA6qz{%Q3)eSpz*9q9 zgr6>IfyB$03a|bn`RT7Q>O?(Nk|RDFUWXx;Ea=ZVLl+w~gRzjZH&7fWTkBt{}tjaV8AYA~%=2xm04=uFFE z;{Qce+OGN<{H9gVr86&3wl*9!#*N1)<>hqnRR>~zIfmXzNR&)0){r=#tS2j@Qt;Dm zV^LtElpc->!85zc@X*#p@U%J|dw1C5`88K*-85BTmfoeet5l$5VFGw{{E%2yKS$|^ z8mg`Bh=*SL!j9S(AhMl6t`@zK*lbTHC$c`#8^KW!suxGqs+w_T>=-g)Ycfn;P>J*Q zPiDj798pLDz)f#6psb%9Z(gchI0Ckp({tWOLFA^9QBsXct`SJx?7outhQ0tT4Q>6I>F`_B_v>UD*tuO z0BGNI2R^t?#eT;=@<9>d+zkyP+3gIW7Axrm7avTimubu{Ex~f`5q)c53a@4Ql;Qnj zg*$a0&?-wR{OMbXW!mFWPk+8_j-m{Qf)3;7UDc9?#4#f8Q?sz@sxtidVlB3>SWk0q zJz$+Tr9gG3W}M){v3`l`-UuB#t~vic`zen0tHQw>V<>~Z__-z3#h2uow-VQFa_ zJgdBckGJN)>AEBM{_JsK<9Goc4d|CFc|8p7u4_X5Qz>A4xfLy^i|E6k0-2t|2rT~_ z4qJtWqVc;1fv=B}G*Yk^6^|v+(*l6r?t{X)_YMmG&aZ_r>&B43v-;>%9Z$^R&&o8x z>xF*9I_cZx5&ZG-14ZZSJK0?OmEbcjh8pcHL$lnmSe9f2Ei!HUURfRVckLKhW$_

xniPY-Z3eoUS zJtv!`k2nIJq32}hVK|0Crc48xMjaCt;lL^LusT+Ox)r(6!x}m8^t2cJUQ`V>?-Pir z<6I2jNU4>qmver0J$%r#!M;oL>AdM{X~YOO98xz5 z{>(1LE3eMe?2TWbaitta3dVs?IY-%}e~HmwMUs>%C#$2ch@(6mC+=outKHBI?dHmeg1ZGn|OGF2h zaHmxv_6j!Ok9$keJ*G{WIk2Z{dCmn6z!j5POg8_``Bjxlo;SYHI=cS=63T19o@!}v$_ zw}hT{4N$h_2EG4G_AT@FgR}$zXl}WIR~8Hdqiv^Y-^+ONmQxph$$pWaoa}1MH z8CWXQzw}@Dh~KqllCvJQ@ZAT==g38{`l>6P9gBFkqq` ztd!|27&k?%uB$?iFb|YltcPbcL*U}TpZMRH2SmQD2YeI`;@ajieCPU(SoU#{ZZ-kL zQ`_mi2czj$91F3=ic}0$M^^5&t;C1c#V}&VOvovBqM?`Gk<6hpiTqI;lII~OSrwa2lnxp| zPs@93Cj+Gu_(5c2U@r##7YYbo2(Ha^9ajXYL$AW8cirUA3YJ>EdI4v0_QBnvTawkY zM$i$t9ysmS6XC-ZMv$!cUSb!z6Nbz6(=~S!(DUUvVk4MLcc&2e^2ZP2p9qMSXg{p5 zh$8D&G~>mtebjuYfG)FH52J6cM{lT?qS=tHBj)Pk%(T@!c*mCASZVU)o0FzQw9&n{^N4e`;tF2 z(oqQ;`!u0hwzC@a<^Z&J4}%WJpD=5uI%J%=O+r81rKuWDw-X{anDNG z@5#}y?%+;5vScqUvkZs1A)~0Sc?EdwHiUQE{z`nTwQ+2hsmQ!yC!{x9qRzYo5|eTm z2WB2Yj{;Lj*!qcXoppwGD`nAPe~O`?u>-wJQs9nQ6Q_toG1d9HF{#jf8_hs3iPGZV0e`=qIen_Ig~)xt9NYRsE3f2 zC&x}Y`xwcu5*$*Z1TU@AvC8SPOh@bq1%622i2PA>uk{?}basd0ItBIJ?$CHy}^Vj2x$pv>I z`Cd>Zw0t}V*H?Ftd0t7Rywel3CD!Qfqa}$nD1=wrf?qF3oGfoJJ4Q?8y1b3BQ3QDH7l3TNOf!^kY z61#(MB^8;uq{ja$zxTjeYQXq`;;v~hWKj_cCZ^I+gFoPV&rm3Cvyy6l=q4X;2J#)z z^58V$3ijAOAS=4&lb^R0$!Sa3nJ&GGLxL^Qvmz0z=Bq%SsVaDn`AtV}vcgLd?!+Lv z8TZ^=mlrc}XHHwL+I|L&$aW`(%LgEclWUj3qNB(mA51^lXKz)KORl4_95q zh5ZJ2yfXoIZk>!vLk6RX-3om7B$dj^yI^(Uck=LVaTl)%~4HMi$&9TFPVT zwP6tUM)Z>(#Mzu-)WWSdhE-tr;#Q+e3q<+ z3m?9wPqt-(zw06#m3ALqW!|MnrdZI_VUzgVPkn_O%m2~b&oO9qQU%kkz7o&KJ$TXm zF`S4q6R9;N(1GEZFl?_p-Dl`b%b!@&>O)$n;U$7ccL(5S`V?vwt-+o#jl{n;8vL5S zk@87B^ygy}xD@l3`WW(LUQ{=nkQCw%hcal_|3&YbDvAo0$cZ@h4LErkkLExBNpf3O zVca2^hF-UZ&fB+_{BiSv_UU(t=FN-5ZR#xeCA&9G-6J5TaUF(B#^P|z`_$Um68?1V zAY1E&py2-u{))Csj+|{K2WF|#JxQvd_^5+UDOJRG&f_JGafTxJnu{Z1nw63*4d=M4$4y(hpvaqJ60!agDJ9tpId63c-sOjj zUL3@^0zcBD;S7;KXW*{ImuQya73@68(U*Hm=pd0H_+E9y`m>L)l2#4Ssn|oY=^*V?QP(&>I~g5kprLg32@rd5{*n=;fy&+pc6L= z{Fl^IgI9N&b)of)A!-YeX21H=0WH+dh)(8!ytB*gNECp#@&f zSAyC}viE*j5!42&gMW59J-yTuKR%F?dhgH!Nm&>yZfJ#AcTcS9oQ~~Rme41&@8N{z zeYByfTas1aj;SRQP?H-ZwY7@EN<{^1N-`q(VTM%sgg2Sy>5Z)#wJ>F=I&=3oxhCD&v%=GfCvesw%LY`GwLk~R>}?N~;kdoP_?H&wXecr(qqT}d{H_CeR|CO9o> zhR0dwKw->M;d9SiSn>^s&GdGf80}9=4SR%fQ>-z<;2JKp_r{**N+QeBaI~^miVyZ0 z!y0mj4!HUsQI_7Mxj*8uKTBB@dgdHys+P^V8>fNq$#&8-@&v9Q|CX-%xgRbPd(n!0 zEHP0V4o)@uBzmP~lCtfKVWoO9E`4=CQn5x&n&421FGD|~{>mU&d1e%rc-^HZZSG2( zyYlc$hrZ~Z-fQ8RC$b($$Y@c9HVJsr=1qXxTDYs()i7n64cvsyO9G*B?uvPwR(>9yeN| z?&)>#VOAi9Ke;CS?r#U1|0{%$h7-ht+l>awv9j}>4P zo8|^f=iPlzZR6FX@r&!}F{9liH8cZczUI;<{TI0OS2KB^ngB!I=L!XC&KPjGpH9Ek zN4q1>leWkz7%OsPeGF$16nw|)03ODF8>lS+Jg%|mKWE5(DFPCZmw4g7|Rs@^dVR>vaP7nGi zDV@ramD83AkEkbr=b8`5(Mrke@FeoRj05d?zv#%k5pbzyCTzdqfgX!3!E0ul(C*R! z%s6_8ReA8ACmhKG(Yf#ED6ej7l{gFyZdV*>VGR7eV`ImHV%<`{wbAwwUx~z8`J58rtkER zEbXl{drfQ~XOWm25}c2h_-SqVG_CSD*%}o?K0iE*0hR^iPQN`&&+QUsTi-&zuE(V2 zYbj_vEg~K+gW+jkAUSm2dmC`SAoE&*p@fm`rwLDobBw%1i4ne1j34 zBhf>3D9JvcEDTJu!WYeJF#N|BNZg?XZ?kto<->ZyuHJ`drK6CJ87Qp{+y%<#Lg>KMvFz(V?`g258hHwwd?=3E6-{%3X2N{(9G=_!L}^n6rkWgu5y_eWj5-RA>oMp~7SjH%j2PBv^}J$j%!?aokp1FK>$fMcsg%G7ZzrcjHKZ z9!LH$nb=B7z+vA@QZ(Q@4L)s%A79s!bJ>@KSHBFxURlrQ=+Ag^^!__y$L@ip`HO`n z4I}WD)D@@XRM2gocEG?Bne?Hd4qW&vCvw=o9$w9;K;;ECFn^`7bfM`==w0;GGjcMez(ec3~;qTT+R4M&AX9X@=pWkK^IO zFEBrN1D36EhNA1g&^P$DuuLJ1&aM-~yAe^)Fy*|^Y~@2?kk?^+Vc`hp_B-)yiM%ZT zQ$wq%$>_)Jlsx>D1UKVeF^RMEz$Cs5lEH@#I60kY+56#6+AEx$nSts_ z9pq@kE68}WRPsg^%O{RqE<7;G0(DobLCLyr61nyd0ACECzb6=?)<3*^F+o`r+fcr%-TklCZw=FiE(hLna<>rC0vzChHukNq=o09oTLIX}3>; zbNh4Z;F5xm(q$SgwM#^_WFpScE5OkI-pOYEXRtG^627&b#@4hjkob1-o$KrIVrD8N zh6m9~zlpee!euho!HT|gn1U)bPDI%~0l6PO5Zl2~`7|YZe|#G}%2S~wFYgk^B@d`i z1VQ)Ue{@+v7tC9&giiBELe=(jkfCovX2hPvBul`*bKdbokKH6kb(a#A8!pgQHU)PC z96=w&T|hkypvWu@4L(hR3;FdZb0QQKoQ;L~P3jWc{_|KHVvk3j1z~Pk1MK$R271M# zvFhJCSZJq*|A7q#KG{ixHfyMP`zD+}*%U`zyDUlmRt*J@iqXm6L$;6IfWiJqGN);X zCI@8Et-I&IwIT=bHSr*07ENWU3W{C(eD`uUMAb~;0BhcS1aK=|b~6ep%Hj+9gV`jd z4|NBIZ{ki$_j7HES>llC!?}_M7jy*^(L2VQT7#w4Rx-I zHd5e&#)UD`tBu^Ma4Y_8;c&k8YAW~nOA>QzNk7x_=N5Op_FCPeU#3j!Wn)gR;y-Sn zvkhY#dPlIXUxDA)ty8NE>$zW#I|NhxwAio!mai>OWpwP*m) z_;X9f@Fv;r>;Z+{%wRKnKD@$)zg=m}+Uv-eDaXeN{>02@>_24-?2p#g-M1ajy=&Ft z%{P{dNB3-I0%*2N^!p4Z?YuTyr(Mq+iz{P{9NsWTpXtavbSHCRapl~;qJFNd=0EQ5 z#{Jx=1Dlzb3PP?W#g>^o>z!CG$Ad|gmxv#0+VaMeRk)JCElmEmenu}=k6r$=fxB>1 zneRAfz?%ffI8%onaZ6I)35u7z7WCZKVuwB{6Y~xWncy3P*tFLJ_|-vKoKxf-&imD0 z89Tg;nR3q@S zn`$ljb6MjA5d-dWm!X!^GSTEyj%1532RO5cUt!3%T~MsywC*<>jMOm~Ux*oDZ?# z7yNc(MlTt|p8pWRED6!!9vKg2CKXt6Pd2<3SBID~XU|OGZkLYbtLn}%sahVaDwnL@Q_R6bcbN7WitNXGGGcink?IFhhQH3w$(+n3r8VW27;ao4YZU5yrpf4$Ti0407-ktXS>$*SPfpgpL#tmL0W_m8{7nEnr77I=1aKZJ2**<-_SjZ7J|JN*W$F~Xm#@B1O z?dux^KYAw!RA4l-X?+gUnrY0A7^yB^KFXXonPbPhUHijqcl^L@U1`EEIT0KB?8!H#e9d>3H{9}@i#Wsy_I{Yd@U|vmgVH{M zjX@+grG7OtB zM!|%%GV!w8@43}srtC9qKN%}Wo!|9-Kev1Qa={xdRrc==b=Fs-r6y!Z0mD3X;9B7~ zcREDEG3p-~X5l39yC=uF?-f%y@k9&$^q|?y^+&zT*q#dlVVw^9=J{eSZT(Yj)zCEV z?vv%5*LDwPWKs$@@ST zNM-igA}zMl{R^XW=oMo$>wMi~K^A8*T8>{m&`Wo?K#erc-=%_Snfr;^{bVvfZ+o@a{Ln`3?BqCC&$$D+`_<*#+i6$11>U3C z0g}7iJcVO|Go4+`*7khn_up`4|AIP35U9yoF3#l!Mld}6bdR$s*5MbV+A(J1*08Kv zi}=5N7F=h%fLA;7rRxBUN`ETWz z;J?Gz64~#(K4i_?)Ohh-MH+n7meJYEqA6Ubt0q4~#+gX}7RI&wn7~Y) z`CAqz&kOv0JovQc5rQ25>FmY!-OP@^N0=$k!r0nIGv0L5Fy8FLa?Uhal^=hmS};0T zi_MRI$!#C7fsa3OQY`%F&faNp36+4)WVYmGL!OYn_gk58ILeTT3M_lwagL8ed ziHnK4z&SC-+$GUgL7CA&M$2H7Sa+NO`&E7%D_MPs>80?EFe;owMB--pXVyvoN-Z+a($!Rtp*^xF3C=QL%r{Oltp+)3lA}PJb4=E_z_c z=GM+)#%7M-Z_D1(Xmu%b?D9In$hp2u_KHu8*A^eap!FGCe7z>WJ++4^a5LeXpEZe< zwNEm;W{u%@n+f@c-=nyhv4z~7L$2a4Z*K^084hP}|5jql+7h|qf_u!bXM@>m>!&h{ z?#*SFnJ00rs>AqGzOy(_xg|VJ>#1cA1hc^oX6(gOiFkOsCD$Pw#b!^Q!1!dzxJc)H zx!65<+;RJ5wRTtM@V(b`c!B!>CTh+{?s#t`>;6i>t?ih_hkfl}H0}fMw(~C+Jl~Hw z-z(#nE>ma27iiQarMGg2UVRp;?$Y6N9wl?30YjMXhiS}ShigoFYydl9;aPFDS}KaX+ch%yDZYmbPExO4Jw9)6+ znoehfeWo#H8FTno9$j3r)j(EG*O-0Vv7Jk5(c}l7{Nd`htY7Rsh7|-}(`F|wdd^I+ z$`sV!<+&q)8tkfq;d}!4a^17PFzc)zFhMP2S)+Pnwx52)ZQC2ktvs9~u3q+oyE^#} zlPTB2?DotMY|!at;!MUc4yg9G4O|h<))P-u2E44OaS5#*7Nvz=hsm*fy2n zfLRn7sZyz&p913oY&*rpFw2S|QkKsYZg88S`%mja#z#1y~ zv)$20nPOuxmtv&F#@u_$d5PDE?>UWU*7#3gX3$qmO;;$h%~7wmD1qa?Je|k9mc$Ht3LlO`5D)2jEwuvbKEcSb!JnOFQYf&rnoOpDqHzg;q}%kyH;(S!HSy} zGplD@Vbo4V)IP|eg0?5E+%fRrS~~nV^Eg9R#s3mxcE6D6zG1<`G}Fs-&dZ@tBiY$aYsAXcd3i1f2+zJ@d;s^ zR~fP?cTD-0)<%5hMj`Vpu94g9;>gCo?G`_5U&!8Hm?GGDzn5Ft_NneoLA=>S0dP~qrqB)MKI*qME-(|527DBivKWtI9qqdlCxWXi~Bpbp4*Ze!o2-5h5h4j zo|*Q*p4+iZn_=D`s`dIeoR55`#&kynF-}Tx;;o6+{ApP(@<^|oTQu_!6RlLnEEEO{ z(py?MyGUToFE}w*2F>A*FAHE+zlvZDE_X6h^IZ9l!43@Rcqmxs^pMjVJCq+7<;EBJ zZe&YdYqNt}K5+XM2wnbho0+)D@`AB;&l%&n72Lah)48c^5`$ zWFQk{IG2f^af1=`tMM4{l1u-3S};mjAh`T6fpb4Mg#W92ic4Zxc0#_oT`?LMQy%jkXF-4pM)(dH(s*1z4%7OsRVca-N- zH7b}p4hxvo<(C*K%NI1{0p<){qGlnf4!nrDHGM^WY;|}ZVaoxI)nO}G1 z*&OE)Oxq)QSDk<}j7ad4DSMbwH@#ZIU0a*Qyc*)k6}^fRHyh+I`&N7xzg~TU(O99! zTpCr*SavE2bRE9SEZk3V^Je68PP==#-b_bs5iHk)6>yYkU;4s2o1WAU{3 zlbp^-Res0dHZD!Z3;sIBm)TmJSGPmq4A*UNlu32%U=q7Gxpvnd;V#O_vG=SRnDUEJ zuFXFKa1oQcqxC@I28nbO7~xS28229~z9+njp@h7#~FHlJ@8!@SeRB zrUZR}kz3x7cWTV~c2)xBeYK{?XRd`$@*E<5aRG+yvZWeL9MUM-56QPwNO2wz)q}~b zP?9XY6ETRL6NTx-2^vJbq!m&Vte81*pII@rLukV}gTKc=GcTpYVdkcD$J3HozR&5mEF@U$Xn7r&ir+^MdP$u3|aLRVs4G1P_!7`dhR7&DdOQH zm0Udfv=YBZMq=nxNxIrL3%0wPki3$9$g6iCM+z6>3cYyBIjc-Nh6CWs(FBb8J&m+_ zQn2d*5~0dxf4}sghiYfgs5pI)N<7Yd5=mt?Mwo&3+=w-kQ#Yfz#`Gjq-Qll+AsXZz-#}({+t3v8Lor2_AD5Fk-^+;7bEQn?V!ty!pm3o z%$c7+^G8MKtTr9`W1R>5Yx5=6|8a@huK7?Q$b({>dovwh=}Keo8In2c9dXSnXFA{YBd*zP1B!bjLGh_G4YZV_ zKb2hRu%9cud$E)RHP|pP(2pCpB(cd+!6e*DpBNmQO12dq6U?kOG;6IACTYHgr&`DH z!lVxvQ{RiXx`U|n%N?NVzm*)lzaC6ZhSI{_u^?D}M+6cpBLDdfCixVy9-g|Sag_== zxFH1eu}f-N|Fd%(aZ2 zC;b50$96;i@_p>T7#=H^beb1&(h65^y^EP&#qn95BMExZ4^20(!0nsEY;>?Um3t;b zuRb{drOctrKYW6zLGx7lDJ_rrdwenPvi?H)sM?;EYVC(h`XfxFcNuPrc4a4c^sy!b zgzPKrl83K67Qjcx$waw1 z86MUtklkCm*}LZ`91Qve{ST#R@oWPqyLT7n`sILvkp!F%ILeyTETXCp6sRhbj~kAx z1b(9zT>)xnA-$ZP8Ot_&8d`_Zm5rz=Z%^(?9>L#M0W@)W1SZSQrc1A?q4#VBB5^Ab zoy*Nw3#}6<{5^o|Pf3Ooyl~DX*Q-qH=M^|NEFY)dC1iWA9=`j$i{`JJf&svTe1#tz z3)Pz#UGy4?w-H9V@fmx1@fRGIDL^V&j2}*AGv}Pzz;Vou5Q~_vYZO zJKxam#(oa}lMyo_e~4G_zX>mN?T1%Vf=H0s+t`!6iDbT>jF!)1Xo25sYN0&_Yu(e( zb;?YdMuPF=6^6V{D!{t8p|o#l6MpgC0wsdJ|6$7!c3?s>^O{wLE1Q+!_x3wrzrF@T zlNZsT+;Lvht4qwk8)@J&ejUP(zGS+lH==$B4;g1MQg*JH?YlMvf6c3)XYUL$KVvs) zx#odhZXTPzWIFYjEY34usz#Jo)e7>1#fa-KW4vyrM)Th`n2 z0qK5lDm%l3OJw52CW_jhyh-z!`HaV%yG)*jDzox=7NWTdJiinMmT5m=)f7kS_iQq} zTd#|m<9C^aUU&MM&msAL`CzDV34)9D$!gz8R4w`t%myWJF~5no96i9XQHJ{b>&3b6 z)ak7KyKr;YPFA?J8jhN&5x;^ue3p?3x{DK`CfS{aFKuQe%Q@`sUkBj%!?$Q_)yfZJB)!|s(z#8t(bYTcTN)rY0Yr(Yj{cSnaTdUb>O^T?a`ODLXk zdM89awuP~;cHaSq6*qPcbcY|g7<%zI_20ks)rMBV|U~8EVa6k0J zPUn+2eA5n2>m;!CK@xQF3TKiN_XGXr%Fx2A-;hXWAjiWE&tzYNdE>1hIWYy^Ik}+j zH%ZF+_P~zeAXxrppMclc06*lA^=z(%wR7v(!`FS_wB|NmWL6lhH*kZvc>=69Se`u| zBTJ7=m0)j0y?{v>VnkR@jZ`+SgQ`qtP`x&wBZ3xfB! z7U-!O5BjlCpXkPMFgz_0I?VK#lf$CKUTOr+2bN$^XecHel_s9CJJ}5pci^b|9j5wH zCu}f($L>i!hZ3ip(4t+Hgp4*p^=UV_i5GDCBqy@w-!J@8yq^h6sX~)-J96y2fTM9c zg82u8m_7?_(*6Dj4F7i+J~0zX|J*U;g^LsQTqP!4Clfaf)kDvX16bMj0WD=_&`nWm zSw|&t*8ZUkQTU`uHoDwK$)P$%c<&CTHe4F(f1HQ)B}bu0Nr=XV5vt#&L$ai%;`?xM z`i-MOr7lmPW3E0N>4tZVsl;oHv}4$fx{~brVI6X;RbbIg^2Kn&z3kVw)6sUhAd77E z7W5MxWwK+RGydrp;9!|CdsuM+%&ktspPx0TY=sQjm}Ug+eV_66<4b5bdlTbdWXe{p zt%lU|24rcg25}Gffged?v|pqW2W2`zRs4HH;w)cWG13Cg>qR*_121`Nx+Cz~)ikWW zKc8{S*o8ZvzJ;2969x12H>N#X9z!n!44N2`fJM<@G%Q0J66=uQx6TgY$dFMaSw*3H9lvg5=JcxX|e4J~U2Es$h zaJp*=c;tOSNrh?!ZsXPk@AHy1*D`aTpJ z_=4uojG+>il84)+=!R?apkt{QJNYyp&qOSsei?z-kQ4$R7RVCgIvt{zvjO6jpRqdM zbJ%>&QP?P@WWC_*9L^lSXnfbh#j1W?d>SJGjS`|nJza~+Z0Ty)8fpUbw%>*yF`7j2 zi#Y6;Tg0h&vl4eNlVK0!k8|{Pq_O9Yo`;w8BreZN!j}%NV0v{fx%OL$ek{tis@!D{ zWi6_3KGXwL7S%&dejjcY?!oIDpQGlAyJ!(x3Fhk^i0Hu?bo-%rlu4WfGP{bIM7#f( zTBT9z&2Q#FP+}plTf}M1)Dui{^(@HzKT` zS6%_Kc!X6yeSulq;fWTz(;@6r9Xq8~Fb69972Kf&T5mn)Yz;rjyJ-*%yXL%S;4g^?0MB%T!_-(TDYW%K=5d*l{Y zb<0pOLx1*ex;d>@&SF~x`MJ-$A40#Z863?wrNa|qcpvqB(WWh%*_U~g_cGp}E!3@n z!99YU+XHp@OQsFaSUBSOb*Ai*xACBBp94XM<=GpR+fjdd1=Bj_g|_4t^Ec@QZ(H02 zMqJbY$JF?zVCvjqMihBElr*bo+ z=#~h|vgTOEBy#>8d=K9@Yha_XBd`3~6mm;45e5$hvOgZB;rAylSa^{#@=GSO?73v? z4Mt9!b;;iu-|if?rsV~bB(b2Ob|i|4Rs6!y?wo@9s)~3>I17sHo8hddK&KF<$?O?X zq_>YqLSgr0lHQvLhKtL26+5NL%Qv#*#^Mp2c4Le=(b&o!JLHYIl_$X9f;ae<9zrF9 zm$?5<3ZDE{i*BKlVW*`Sbp5JeXK&oc?$HdviQ=r-8 zfE|Zz@lW`EwqWl!Sd;Dt-d`?&&&PAf@wSD1$Ia;Fj_XjPV?|05-AQN6aYz}x4q7X8 zag$IcO05jCK3A)RTw^U_5?zLCxu>!CMgo*?j)zH;tI>CAKK@!a#5rKoF2ETjh;8^w z)DQd&lP?+)k9}7d+X!1QC^Vt>lT=Yl>MtJqeFVIN8kur4ch17;S5dWS3QgP_1TjK8 z;P2iWtifl&46a)N4Mus$4r=4o*@a+kcL8(GX=1xdAv8~{#M{5pkQI3df<=IkS2Tq@ zbJ~Ko{u)%IrVKZk@}W`pA*MNT=;7a4yh?|I?6oISUH&NW=#nVh7FowS z^si;(WX_|qDvvqhISoP-Z{Wk``AoUNDc;M(Cv5PWo9OoTB5wBS!xv@$!4~Z^kiNs1 z@JnvuK4CR%T)7g>bp+YpajmePX@rJree4V8_J#o)e`E{Cm96{ay7i+fb5QXj^U`nlsPRkXh=$0uFI((0P zA#shZb9pdlP17o@uh)btH68dPa1G4+@eu<98$haG7V;+)VY%sCD35Fbsg_)pY<0od z*Vg3c-fhh9{St8N^+kx?y9gG%S7Yzc7M|au5~z46@H=c%gPj^`b9?OAf-f=biH&A-w`xBW zZMcebZym1G3V`||ZL&OHuw zEH2rSuS4B)Ov$6SN~G%JOgLgxg@r;nC>_zpW|&`s?-v80G*f}>xUT|EU9$9qlp47r zElic(dQm@h2MpVP1{^Nj(?+l`4VmF}UzvF|(qw0dFjctFjd^2@aQE3> zh`uob-wzElDN6sa;HwZFGIylgZ{32GfpdvMtQ5Ka-4$5NGx+>8i@U}oNlk?f9xO`( zbypQ4KXwS$1Xh7obqu;tQ@W661?ww*Fjt)`ae?nPbQ{|O=~br2WZY0B|*kQbc3EX-EBUOGAKgN1vo*S%st@r9bpsdG{G=A9a^aa zY*Jb;;Es$)Q?(8wN^kbWKmCav(?!F5eZC6x1{Hz)j>#*H9mV$CbP<{2NV+Ou=dU%TDx|0G%O7v37+QW7?3t>J2I_YR4Gi zF{_i4bKv4d4VwPS9UI@XoF7*T*~^9Lz~4{d-Xd*MVJnFjSRv3entHvZn)g;4TaqeF3TC>QEFH2VoOEAk}pyEo=&9Rx3SY4u6b?2cKPd zs#=Lm^v z+TstAZz>?|m=kUOTo2RCcd|Lx96@ohDb?J}gSS0fn6Lw@!Kp3>Q^cl_Q_j^K`}^fq zMCv=sY)fEA&klpK&waRU`HvIlk%6PP>tO6k9X!aFr+mI92^PM=IoQ4q-1~&ttGm?T z@WMBkQ5^@Xf3|~4ksfvN{DMt87jQnCSkbg|=kWDoD^gz&0W}VLNawXq_FdBp9Ly*} z{Z47}%i$N+_8&m`b=A1Z$pqC?c4M`a8xB_1GChK``AlA(oiyDFmPkK^=()OZf*C?D z+dnvAWih7bl(W}FEXb+&DkQe^@Na$x8`$)abzof~VMv8}gmPq)=42XN(G0PZ_QECQ z&+PTP;#Bg%1Y*BxGfw8D!}nHcY9Xvncbkc!qRV31;-^R|?^}>-W6E$UU=8HieC7nK z{)!qmny^nd5q;~P!2jo}*{M3%yy_fk;UKhpT?2l1G_n!iyO=w|noP!sF14Dokol13 zPomBA$UC|DFz0(8)`V))t=8sDh50WGA_H*Zp9+yOaf9*9&){O2j@MT_$M&1+L9_A# z*7>W@J}FV6{G}g^1(*@rra}sCjWg*D$mI1!kvse#+Bve6j@5P`yRj0d@ZBM5_gicT zY!%c#8`zj|gKN#&tn&df!FT1~2NX+VvBSb|w& zHdGR(inn9tq?qt($gZPQ%x_N6PC7gP$0EuTPD)Cu4HxrZ;GXu+1LeQf5_ zOuYT6jyc*K0<#uJ3}`j-~T2iPFsard+ON(GmB8? z_(qieJDsk2C{7LtzP-4iIkQLuqNVp9WTo!d{oX=_unJwOkB7NWhQY4qX8KVVz4 zjeK~BP`S{VF8TNpygmW3l03t7ydP!q>_0z)Kgwpg8{m)>|rmNr{TTX;&kSZ1Y(VRwD=iJ zU3U1Q+rw34ZZ!{Y@wU;|?X@`dtShaWFHJ1twTS=Ka(J6%Lu}C%I7bVy%Kr{)aw>vW z=qCv91_xr0c@Ms3c~gnsU)k)6>mah=DrStWC%YCb$F0f&+<9&^PF}c%n97$RgnLns zg-Z09^>$)YQH1GYdj*`jK;xD32sUb{L)gn}prYc#7VS$Q5+QeSNvbQ=6dlD2Wj`?7 zeu(++g$s#DD~9Z1f=9lkL9t3WG1OHjecxBm)YKiYNg|lYSM7v~rvXG6&!S_BAC*;| zfFExL;?c(`n3ZOVQ)ND~?_X7c21k!6NOh6S`i(lxoy9{bTmQ0&D6Xf0sHaNXF z$UKNU$H~s=NAYn&z7D>^m3k|{WZx+;yu8b5hVB;TqJ^=*l6L;|LpH9KEl95b; zvNs#5E<-O#i?B1awXlDI7sMMVl4bAj!lpqtrZp@HzUO+8%}Po*P#H+it-gb2IIGA? z!R$IFqD$x7#v_wEmD(&8X5T$gBmUjKV4|Br-2JR+jm`vp9eXeYiih(r-@aeAe44asoWjx)u3m^qnh@F27e9d3z}j&WJ^@QVkh3kGPX zYXaT8$z*B&S4Qh*7;(|nqZd<+@MDqS?)s%q_E;<6!Ib$_tab*he40dSX#Bh|}|Y z37Xw|7W#L-;XJOZ!`E}nA^u`8>g;-k^CkVT zAHK5v%5(N-n0XURnW}PmB3C+&1Eu{KamJ6DCzdh;R=1f(W*NH(w~m72%{Oq|^aST$|14h1J{j^d?g%q8qLR6B^e$E| z_`=@L<-lOsF%DxRM|Ve0hUj&r%u2;0IIijnbBlY>^J3v+t%sd#@4pnw>@SAII5_ zn|{o4yvZz6N?=qaS2JUxO8Dit38{^`#s1r0g3sQ(fk{^mgUgZoyg<)2`1#cjym!Tb zhRAKk6N1^>`s*5eBK(3)`;^JN-}N5VXF1Zh6DN~;t!mcsy9pB@zf&;IjN$^PJoruL z5bM*&Sc}MIgi}(BbEavcMT`Vfp8O5R6(7Q^c|YJ~Ll1lnY-Md~qjAHMl`wngA@j)N zIovz!4x3_yK>Xha_RoTu{217z zwSr0HP9?$qZZt^jAAqGEk&%-m+kWO_{)=UBG*pJ($@s-Kc9?-D_auCn_X!xr91Zr# zlhQZi>^38Js+&HC9^4s^i5AL&`~je7A={4MK+B?3*xwkx1IHH5@Xb;So||9GKs(o|2N zr5w4V&l-R9#oGl6f;u`I#m*MNpRCjD)sO(-d`IYZ9&cv${P0%QiV zS=~fQwmIb@>zGo;nlARHZ*P=xR5~BCrK=A>M20JwwJ#A(hMjRZ&>bfiO(ov~+i{1> z%!ah(2e8071?E^zVgq0d|Ls48S$W|^?1>@G{^vMXG{dpWk2ndl8=$iaIEHeWyt=(gboz*2l>c(Iu%Sk zc$nii&Yt_@L;T$=sc%#_Ce}r==7*nxcAhRs9~P#$?f2Pfc6#LE*;Jgd^#Oa){tWk^e&TJfTu^B5VwdT9!^>IGjWb?r zCy+;F;W+7-1F2v2o_E)0I=%dA5-qd70^U1*0Joq5TfeNLyL)WNg{}c?;s#+;#WK40 zKO4A|m%{AW6op4N7ok_9KGC_P3bW@;p{9KU=su7}b3ONg*pGOK37Ag zRAO{loc527;de0+JR+w^L#qavsy;8at#=HB_&i=;bRs)DX9T2vy2E6{Gl)_O@cyeN z*?(gWqZ{H!_ZSqwu&@!)7?}oF%f2yM>qHr2pL5{fccGyr-j0~`DM3Mz6pc9SN83v& zDr=Xq9bSFty-?8On5-un?-*Pmw-~JM%h1Os{m7~_O_*}{8`kt3$87;Snd7%MvYulJ za9Jf2Lk0cb(?U;3ONzh($6;pPA8mTY+8WmU^e1Qjt$|DDTtU2DnC{ThrQ2V>0yB#& z#%~-L`QxeByMG>=fV0V_Uz2I){8`k5m=NLS3E0^z!NThia0&~8lv8f7|Hd|su=F7K z#$@5OgX664Gy-}P?g+d!Ev)&hAvibX6ferxnBfhVp=OILv2h&0%Ha7N^DPuCd`3__ zFBq6hA6d2AWeuO4mO^NiBWifAM>$rTtSawgJ~0~Tb3}s%%rNf3G_X zbI*=4KaU^gsckw7axb34GnW>;iMM$}tx9Os+|Eqh(S>GNdf3{ph(D*lgIa+H?e5}4 z%#wHxomy|;ssJ--J0$`Way1Ck)dPMnF5p_}G-xXnkCgZNZ<67(NbCZ3lAFwyNSmbx6rFRKDMmGyz} z-0>p(^s7ZiZvv?t8^9B+3)R}-j%&Llsjr0#I0~@bdG4p!uF*t+erzA;ISX*(pOa|O zH5Zm+z8*u8ZZmeVoy^AkIaK>WI`35BSMU|wze8MeI;s6H^xCc^>y{~y>LDvOdGG++ z%eN)hUigxSp6#s7Gzv3i^-0XCcSw91ko`0W+nXIpOROSl>@g-H#@W_7>&J0xfEvWe zm16R43z+@an4}$=!19*J5>w4GHhNH!es(GXmD%CAYOw>zmN10UT+aNde%N)f9llnp z;MQ$JxaJIpo;E%XAp-s_H{JuZN2jnSuDoQ_yw^kAj8jbPuRn0h;y?DdyD^Q{IgCjr zU)W1GZt-ScIfQ|W#A(qUDO}KT7ebFI6XpE5xIq6Y_AJllr1wd{IJt@Ebp`$eKW~`6 zMFWS0PT{?hR5(}@ikD`jAj`Hv%AA|v<8>SD>|OEV-5gl#>WAvS%3zgiL^g){;qs3v zobr$^2EItp6)OgCX8B7{@;W8(%xy(Yy&|^FH3N*dxS-mVe*zE8ceqxt4ZCN32g^eT z*gKg&*y$=Nv?8Pj&d*mTr`FtKj?VJOANI+x>|FwLlCusU)oUi&yU|y#`*GslWQ;DB$LAa7^E}_ZM>Qq) zpqt?lRB{|(rUVDz>$)PGBbJ2Q*TzG_CM&XaasrO5(IF>Fg{kU8b+|9P1-A?8gVGlBD4Dr z(<|V&SacxD-`}HW{Y89kW=MxOOOT5;F0k&Z66PN(xfK3Vlj*B`0J*(c5MB8bmlhwz z!IpGfb-Ee2H?uucnKF2YgOH|3`JAo+ilhBz>f-}2oDmDE$3f&d5}s7qptW0vev@qh=G7DQU8GGv*>8oJ@>-l} zllGxU&ph(SxR==&EFL>`qe_H z8fJ)LJ6<4vS%Cnb8h|cidF)IzgYdTha7lL@RAxN_+AqKteTvvk?XmbV_W?>bCE%@T zWvIU;67C2W;jwFrfsSm1C5q}aP&@*RQ^ZNqH(BENnW0Aqf8gS}B(&&=;2iQ5Vpi9@ z0*!x-=)s#4V-^8~+i(I56;fCy<|91xSxy?iD^PtV8D%CPM(LqgJUQOV zu{b0NM%kKFz0Dp+SrNL;R)b`YKZC(5JV+XwO&6Z@!5czTKwtC~tdRPG#c%s@OOpZ( z_`HrdZ+31d)p4cm8|TwCseOWHHX%DjgGi43bNDS{O-C&xNbb96k~d*GsS%GSYD-Ni zM|THJy0V7)zsbZ&v$SYNY#AEqo0E%i?zC!C7kE93B;+*OwG`OkUZzF#X8`EQ>t7%@lFHL#4jfg3VQoqP(>M+rf zo>GXF2? zPxYYPTJx#H`C%M3PK86GawKnE0_1S^kSUIW`lGdpTGY&^?`Fr+Ru@BJ@Gh2|I37lN z4#v~4cgEy*!VdCUaxFQTHj!-G)5mLU6Q`LDl2r4HJGp7IkVd=hrq=@&(2qGWR9kHT zwoTng8V;=>@rp6DQb&o*{xWsVUMEpRUsj{>i zbwqty(bj?cE6oKS?XMu(+lIKg14m>8euJ%wpf0(cn(gBfeb_+GlTg7k?51MwQ%FUA z0{P(`NPa9ihqv!ugl8viW8ZU0M*2zwIkS$(Gi%sLZ?*-~{nIkA@V5+AduoSQUfUDc z(*PT1)Io33TPV;T!=V&2+Axv@2T}sbjx|Aoo`g$QWQfyKg8%od*L?c3d_Acx5usgw z-07H^2W=J1b?2h3NfB-*QU8UK?flK;`ypxCXSsvwMnuwOWt%7+m`CiF#gJmYCrPoD zA%FT-;>{@wuwbAZn*?W0cuP7~k6AX{{4xT6{;*(tsUC-R3(+^ui|Auc6y3028%|Y; zlRTUaS!V;uPfiVTFPKx2xeI9i;q7EowGd6+6iOF83Z~V5zu;(xHMS&QLm&2xwd`Ol zt@z8pp%((y>BS84c>5gc<@pzOFZ>I^y2e;C^9<&1oC#XdQ5d+dz}oY&9ckj9Mn97w z>^7J}{T3^rnqmd}Sn3cbplu6WGT2J}?JUUj%faMeZ4e2HHX&{Q?ZZRiIVkz04eE}q zr@32al526l;LPsN5b}E&*&cr#jn{5s6gf3u^m7;GKUN~)M?8pgvpo^nm;s`*j9_+Y zAO3U~^hP>c;q-_YBRXvd=2f4^`!_d}B&ApQd5tLDmwOSLg53qRU4Rkg$3f_UVrJ2@ zB)ZXa5;%oF0XP9AAKwUz$UdJU64buo0a;V*%yWzQFDA3aIjyGK+7`fa1__ zn)3G~*wR3kqG#7lCH_fEiFKqbd7kM= zJ~b8Ll=)JW|3{q~$4$TmCu|_1nu0@}EO~W%DOIbNAQ^!`ZV8vb^;zp^!u(puljacX zDGFr%q7YiVje#Pk8Dx5*68W7NL+xjCm|ZpViB7RK**P?U-Y(|D>epdV`K}oh&&5)? zBNNF0&Le-T)yaV?d+4+Z2~y*|ij>?BCY*3pGWEMS?aH#jciFF)vPB8>(6~5R-sew} zdpyYCWIp=em*myX<-wiwL8!W}KP@dO^;h$+A~msjHyI~B6h{2+5VeJlI4gKc~v`b`-VR3{OyH9$Fi|BLx?r4X~uUSzi<|9I)vH3 zRlzc=0Zu2EGYXFk$XQky6HVpm>8HIcbQux-!CdCA$}x0&^9etQKEW#{J0Ldr1Cpop0>Fg@u|2;a0^Y^5W8G z1R)1J`?L<<#A{$~hzxoz$;QycX^&8qv#R^-djsLvNUjkY@M7AH*z zeB{E+X}*oey~^=>d={&dbOyAWjsmId#DdjD?2}Vn@Ho_rvoJ`8xNi@`n?7l{YE2uu z2(YU+T6?jYp9n$$f%r`JEp~OhWaR{VF$!{0nUNs^$M3GfEZM_n^)g%q({}88FGxI@PB#GsFRA5VOA2#ys5j7 zjiuro^UeoovD6t&?LPK+*aT8oG@_|tdO<@yOi$KK%J#?6^g8TnWq!Hhh*rCT4ffTEQrC>G;*Dc1H&1z^ogs$Bka(^{HL-6drNb1OpOa?XD(-g za%I6a(;j`(Us*ftoE>?<8o5uMo13Ct}$V2n}nzoiEyUGH zV-+-lRp@DvJ7+XKW6pIM2MrJBJPWvg`Edipv~q#a5~8a z5>4OZxhY=k%eHC=Ry1V?C)~xnw_}3cN-Fg3mIUVV+~)${a1L{(wTovoYR3%j8ED9S zEJ{n3Ymm!79^@5&aG70?g4T|+AP>#AZIbHmbU72FU_7#ca?&u3_mi#rGq;ngy zgPWP1s~#|B{%@K5wMy`Nq8|JFlrWKdoP=pnRrq|#L+IY4Nj_-zu&ZY5<)vR+&xjQ2 zQjN2=OuO=alv@xC%M@Q=?2c*VT%7^gC1r{S|I?+b<~?P9mCh&L>Kv-(a~x-#k7tk9 zJ;d}N4(C?9I61174Ia|t;1;UJ8F^-i7c$y7mhYP}%di9t?G;JEHB<8RQVGL2Xfq0q z@^I2&DX|OAW6jkbVD`>V*n8Hz;jCXCP89#f47^aK)?^S3d|TOd0rKn+S%#*qT%@=p;_fJ6$y!f{kUtLH8~w0PZ3&b)t;05fZzFBODxSiWn@lv%l(^pw z0}X{U;H$F$ANH2RivmgNrI^a3Hrs*d-m7r_$1&`Wmtt*=g)lek6HoTKCwlmvfRy|j zIM-5O>8Hu=AXC`|$S%*CSqCmeJ^+1lsKHT@j0p9U-$*#rk zt-oHoitk4cTBpADW}9BdqJv}!!`(NTC?1>49B30Ig?Ry-+`4jTyE~4P6f$}Hg1eay zZYFqriXz#tqZS-LO~FrsdNxJnE0ddW5%g73nPE$nhBE8BJc?&*yx0RaTI04{nK@^h z4sL&ClUO9zd~w%wew4f^S+VR&(~_Vn{?EwsHWA+3!NE}*eJbn7k`D^=cXUYzxW&W{F%3Hc^CaCnkC2{8nH>- zA>aJ3=XqnQBZnKcdmC4O-~oR$#NzwU;YWna;I8EyTa&Q`0mQN%rH zY}mBaznvSB)zdgK{L1EXQ!PL4;@2jTt6BWgM~k`H3mW)WmIm<4eh1rl)4kla3WoeM z1L-!sQ$}pMC+gYwDp>N3zwP7?Pi$!%SZdDwDaf52tCSM#)w#(x|31K7{xX|8yzlV5 zOZK-LC)}^Fsc0kI2{#M4CxqL$e?Hmro37{ZwOKW8q?Q<0x+0wGX_dg`-t}ne_sitF zb(ZkwyX4M0a=Wsrm6Ocp+Arshu(pkU{nrH9(vsZ3WOKf+%M`Ad^iWf5R0@CNE~(}d zA7r_;hg0~WofBuVl+Ai)cH+7psLywxA zR0KIP)hhgL67QRI^}jS}guLSZsrLSvU2=l&XPw4X6W`2L=MC5dOFXf; zGVqMMUBZQbKemN$_s^Jbv-dj(IdtMmwZ@jkR z9xQEY((kly%9ohPFFzx~cU-5(H~Fl|4NZDAZ(TzsH}}m`o8d1rxH|Wg`K}uG`THGz zHC6vAwh7T`rMJYE@-JQ2w#f;%;ud92w7uJ1&Yz)kp~+h#y9t%5nvO?j*c4iS;_5a= zH6BeZ;D+xIww>}1nsgNvZMNb;`eEuBZdXd)yu%L3TwYx;-(~Ah{!^DIzTV}HO}Fk= z*yL$MazEWovKi^o;agN#^R-s{aqoFLa?cAgs6U1r=N1h9Yc!vm#W#?D#BV>C$(1N% z`6o)Ak{vG1)a;BF-$~t*yZ*E*|L|8cuJ(N++v}Rs_@X(}xIsqcT>M_qbZpJErYTKC?uRuW;%ZH?{qJQ~&-CR zsBA(cDI<~);wz!Oq)AglB@GQ4+MjbjEkYzwB1%@sN=8QcJ-`2ZJ^l4L=iJwI-Pe1$ z5n3LuxQ(8PGr4}ueP(R`AZcBXaI|~{{5WijdwxG89~Wjdgl|3q8OJw+rH3bEZ;55y z6fR=#MSX75mkK!k*8ta7&%{%o9jvXK&qLap+1N88hFQOqxKVEPt%wYZxfTSL$6 z%ebi59OV-4kdgc#Sh`^|%$zX8^>2#_-~7u^oZf-Ue;h%s%1*LpR|1xw`vMyU49MRP zH%U$RQZUZphm8^Hoa_0|u%$K}0!k0T)mIi&_gxIDSoj{q!i>=CZYip^Er;&hccem3 z1Bd*&Xa$?tcrb1yc{HH_9an5|K94s@r$}b|vbU9cKvQrA&e@ZR5Tk`B zczym268xxqqLq9dTMb&iwVA7q$ z*u-%CPAGD74Q|7$%xo$>%a9C8>d}S40vwC{jpSCd40(7fn4Ih4@v;=kK$km}(|>(B zJuX&@eY&Q2`d1jHswI*iJcYH~X&+sh5Q8WE6j9y2g+9FD0DWy1cpz^+{Uyj@3O9_s4%USwgxF@;bb(zY&YJ#!xFna3o1F~vL4z<5?0~2T>w0{!dWENK8;jh9V zN&Vrn4P{Hr=5cbIS3qM*0!&!Vtpen_g=bH^FPq*X1tq3UebK>0#$6#pM zK~Sf6*eJKT@T<0!S^P*9pKR@ief#8biqiT{Vsr)zi%^c%0HNJn_OUhzY*6t zTx&e(2J4>WgVORW*uSEeJyEh7e@>T2m+d-GOjA%aC=3$cCZoyHF6>I*g8Bv`uqmHGT8AQD<4a`{@jrZ_*$O>TGdXOOR`zHjgemApm$#8T?H-IQ(RS?({c;_#8c0 zyy^Wco1cW(3{-FrJMEi?ZRA9@9} z?mJ`VjZBD(yF@BVE}+Hb5UbXf+hG304klkJP`Mgc9xt|n6a@%!Rz7wlO>cNEzJwoacppYhg-k*GurDd>6oVxODYRoa z1UBndSQoUnQ|X{R@H|5oN3$=HF{v43*K!{k89EK~>)zA#Urg{>;3WH(#~C!6%Ow>O z3+TY6d~(~b24BfE(yBeJbW|-E6b!mp)l->d=$IgPwoWCcr`eE%ZxJ{+65`QZ+Y<~>+cznU*XPb!JoqBfSdnAxZ9(!!y4t8F6$V}T%PqJS8CMV_% zTYq2a%@o&a;6|~pa4p@4bE(t=_gT)yj+3#3jlRyrY`cIz1OBkNpY+gQBLYV&ML71o zgVZgPpX22&%Q1VHOcn3DxQCG3|Zq2 z`14mBOp6!i8up9e+EPbmZj&}Qcc%*^dAPwVS9#*|b|zZ8JY_O;L?Bu4Cm42m(83M6 zTyP6wB)j7&pH2tMuN9BC`0r4%{4%wiGY=Y4JgKip8NSYq2hl~}sMMhyU}|v)*Df1s zY`CL`DwB_KPh$q2ev^w@SE@j`+Mg<#2C-oSQ$RP{8#N}i@zIK@^q!G2xp!tJbZi%a zqHPpzeHemnnQCg`b%bo!Mo{ziA;T~PZ)DZs!Gg)??M}qGSgIiPFp|v>?v}XLotzmU&zvdOuU-q8b>)oS+3U(lB zqRP#!`-aPYPGHrtdE}$#G9>OHG>RjRrgH+YT{eu|PX58};_(MQk8NTXs>xy8l`~kz zW1?wH>j!3{gg(v?;66B7&X{PQ1K*QUaNp}S@!5Z!O!Mu;<2;_($M*}_Zlyf>bD=ww zGpV@b?kebcYsrt{ zWc?Y`JyeW*CMs|<-xN=IJq58F7iiIkL2LgfCZOrQ2OXm?gU8kxq*Qe~S}x2X(iQ8W zXsbN@QTjm=Zdl+(xqCe3y9#!0pMrunyU^Su2)!@)u}*6oaGr+)>3baxSMJ&p*$s)- z=6e1(dD@10FY00spJ~O0XHJk+9|#Q+N%;JZ3hX|v2_ybuRN&wTdU>@Yn1rpPno)ct zTjmIyEICTl=7>lL;lC75Yi+9y)k7_zT^$m z8DD0LG>ka;3o9D0JYGY~6K_%+7^YYF!s&8Sjl+FuczeG&*<8~@jg4pGc#sUYZm1W7 z8n`%f@h@D)Pe@y3HjUFn`ZDJ~^7?2e)#`u7m;@iE(UbKs;{A-S{o6@rzdQ&{kvpK- z`VB1{;^#Qnsd6HG9AQ}RHX2V8<`#9%hG)Aaz#-@)jO7*+*BkHY?;{dixyw^nmW(F27{Ni8he^OU6i0QW=YTDr z6LZnDi#a@gkDT+!K@(7g_5ar4jKWO%?Lq;bAOFq1wHhUPZ@cJjn+o9j7YOfl;^Egz z30(R|8mJ(PE1d;70opehn_^ph?d(P+ss>17Q7 zuv7_Z?2N%)Ap{oJ=fY%jA?~?f0nWb@*@uzFB%#k6BX8QnBE}1zsDET_!x$QpCZn)pF2_%MPn6fwZjoPI}FzwM8$>ed;@9!Un3}!0lk#;ovz1dHT@9M#=_#&|P zJHwo7x28v|81i#_INZ7`kER+6>6=+E;bcw*S)t-A`f#j{tM{8syh^l1TwGZurnR(aQS|^Y*n{*A<_J-rUvX zwQ?jJwRI(nEuQ0-Nl|Y2qhI6+I}b`s&(UXp4w9m{=@{+ofQQc~5cPsL)Np4KrqdFt zI(>-Y#sq`JKL?0>O>xEMMKq@51s)h6B>hJ+v74DeA2eFv-JPCr@v{JKxM={~o@nrx zTTZVyP2u!=a^R_#DvVuuft$z7A(!KaHf|2|j>|VH>z#o|;|wwVdo$`qyksN3FGCr{ zII?A@8a@{qhT2V5=nQ#hurN-Ah2 zGuF~Vww~PwV;>%2!MCrZI#UZW=4}A&_F|&Zv;YdU=P^sW3}KJA26c(QY&E>Z5!3|B zsOAAR=KIx>MkStZ_Uq4FJh3^PUFxn0<Y^J+T z#t`)q0}$(Q;{4Waq4y%K@WFva@Os@0tofdex3d?)Ue_%2efN|oRMcXvOcY8u{Q;-W z6wG)PfwAId^CGTL~OEnC7CJ4uqgt70dCcLzHPWu`)c=L!b2L+d* zY`rc@81nRhxHxc;*-8}eI*=4&E#j!F0S|nVz!6Ir&TU7aRu7=qMh5qLs=2FSFdlfKq7Ock9H7i&Z?RFYoFh-qUDsfw7 zE)bK53p{T14c0K;6YkA^Ll?|kjlRFbV8^G`tjt#vj>y{>-h1gCU3D}aFN>yQ#r^>N zim_nrVvB3b8&QiKfdg&NQ4FQfDRUa|yV~MIr!x4&V~K0^x8bj*$53-38ft2f5_=y% zW~QqFs*Xir5Rcg%>hl;EYo-v*S&n!iJsYQLRTF?XC5alV zi^4}KUqys~hV91bT}PPBkS z!*kwS=NXaSR0rlh2Z^|mE+hW(2{|O^4%7D@CbvybW1CzK9z3*wYw*zq$N!EKo6>Cj zqOHvNw#^(q4kgpZ^`CLvyq4-qrIHm7r@*~q$>6g23%&Q#lN-mI4aIu1sLz%qz=@NF zE&bf-g;ozQ+05f|9~nO9R$C(Z)p>! z8}G*dMV*xIv=?u(&l?q>{}0RB8_PB9+#>*%WBpXkB#7=3t;V?APCE036%=)taEkA* zq4#@r8Ov{9Fyi`LythFd-j-yNN9DJ1X!klOYL@~*D|wKk&UD$+P-d~%57@u0jymqv zgUME1uJC6A_#;>Yx--hx4X^i^(c|`hl&6#si;H>i`Irk6sS=A(HLK}h zUlx`B+)NcEtJwwJLE!0s4x&ywW8%a$9GBAN*3|#Tvn%8{<_;V@{Y(yPcB^4tS`1FT z)Ij@+wqb*}J^WM10Tr$z5nDYAa#cgH>6s`ehlHX1Mspb7T?9j7MbPiaB~33klh-Ty z$!{AON|hT?gzpyVlADQVN|gy+RRS5_i}3l2v*hr$+q5R<10&e86a(UXVUOwtDtteJ zaOT~?^m8wnj=gR$%|HcnA7o>3b{=?~-3$++2k@*R!3e)@^2;SV!a#2o_O6b7he8KJ?v zc=^*R?l{4=3vu|sYmzK)FsIdpm+;V-EM~7*Mh;8F5jE8w_#$hGGaL4y)@)ZSjaC58 zHxH(y`zmVMttZv1*1^0PjVM*FLw+{rGp}yk2g!su*6R?ZxjWpc_Oi|J%7?)e#c0z1 zm8ZXs*ugrVYcQAhJ@;SN$JCc+=v^ysC|#|BGc)tqP5(aPrD!MU)ei#K;a+%WB#Gw7 zHAtN9JTh-%2(C@L1|M@Mh^Hn}13?{Hb9XBJy=e&3KJ~)!#g6zi50Yr;9|T0JjgC2lZ6cMxowB;X$R;Y-rG}Yo&cCVhyr8PLm=by9FGT!VNT^- z5L^%mE?g}#=5!U*T83$qLMrq%CD3Y*M4Epp3Dbh_!_f;qpgv=qOm41W9ELcgbbdB% zpD}E`cCdnu1??lTKT_dXNIX`r@IbZw=h$UE4ElP6z`E3a{C+bHJS=$l3DGN*<1~%i zzRH4ZP-gMi5+@?Nm&XVUen-fHdNy_Ezs7L`6)Nmb7`K4A_#a;t*h(&gW0KM6yibX9 zjPnN0+KR)ju6TA`eF3CbnP6t=CS0s&2fG0U|!`Hur9h0o*6 z1C2Z;?j7*W+Jx()cz?6Sib`Gp(rGk6O}nHTj1Jt#=G1B;WOg4MhYQG!1+E~vI0>JY z@p$sWhd}tU9HxhTq+gS@P)sWeUV4PWP376ta{B;PSYHm3yCQjhh7dRE*ssP^;~~(z zF2LP>=^R)Md;oq`XNdN<#f%jWczHz#tL^myzPdi39iNf?w0u4;kCMd%nHo4=oWVp- zWAKTEJ$|1#L~bW7go^MKcoM;&yH6B|>i-~7;ma5YrMVFO$PDgo`iaMS1Yp=u9%jtH z5A((2(ei!^nHBsRAYcej*6u`6pMUgg&o0{jIGwo{mW}g0bBXtGob{{E@kH;7FVw8R z#Crao4HDmG(U^BfKz_#+?EWK(wV}%)vYZb(WpCjQr(jqXL?KZvAALLdvCL1B!#&kT zwj7XxYrzKaUqL^#_1UqWuTiqfS`o5B5D0Psy|Y1sL57kgHW_suE^!C6m=8A>v^ zd2<0sy4gW;Od)ebBNp0M4bnrS39#!!4?V!sPa|ti6O{-%=7Qs82T1yH^pgb=n$KDt}2fb??N!_TS{XIhWK{`Q!ccoQ7|35877D zfL-06QNU#h77fXhti(a&X#FK7b8j&EXRg4x@ob0?&VofP zVGbscV7-}&?vuiTQ$n2GF%dZFa*JFDtfx05zoS-wB-}CYf@38a)-R=HAm3v#Zkn?R zqGr#6^z>2en56(h+YaJktqPJXB~NE%6@xCHEIRir2Cq2|C~Y7BF`YkYObo!Gvt>}% z+ehZFcg7a$LfWyi5;jEiLxQIXBlcyq(P5Vy%$9pbPhc%QcV!P2NW9{`{m;;^z3J#b z@rK?>^|bz_cZZ%f^}{>58{zJa?bz!;s9lE{4p%Qmx!PH@;h6BbL^}JPmdqM=*VsO-Cq(;p2(&qwiT$m;Udxl>*&wN zY1GBNo5aqKzz?g6(R<@pcGpQ2PT1mTEF2plt4<1X+dYo4%TM{kml{2|_oE%HmHpXx zaV1zJ)(OX!W)q*>VGQv*jg~!^upuxDZt}_DQDr|;<2Xh_1YL0|TMXg5>+z_B2O5aH zfZWM&RNu6buuT!v%ij-ojUdiET|vL4g)w{G^ttMb2`=1C!Tif>`2KG@Y>=)c*2ktZ zzf)_eo*>ftgc!0AvE+}D(IIg(^Q{m%NpA^tYLmQzUr`q(tRmtAyo+5 zQg6^;k38(uTmd&?Q^2Xcgd_+nQrD18bpB`y#*g|uncA@r{Tp!aSr@? zO7_?uBNdz%G}>E~d-ZreUbZ)(6_xyWru`z%%i`&{it+S(+si4)CYglwp z1UiKIIsc;0Vr0^Ej<|jpyzFZrPC`8Vae)Fle*K6WtqM@sEtKXSDx$5D&X9gQ25a9n zlCdufu{N_5p5IuDGs;t_O1BeQXlA0?pB`vX$tC;ux05)tU8wlb9erP9qkT#%%`pgs z>n9e2=E1G3Nsj~0_YeafvjMl13lOb$^I_I)Kg`Uq!+}NrfyvVXEKV%oG3!ranX4)D z+fNByUnjwTvp>*TpZAivt=c#xd={|gf0*^d7x0_&0&cSMC#GfX9(pjU*m{e%CzxGq zLr?Wca5sJhfpY2eis?Cg=C}cmxPD?h9W~MWjy}>YhjGX*3?^q^u)eTJ2L+%Ji<&JCzr4arrHj2s%Z7 z-i)LoJU_!Exr^v+(}FRvcbGXiNcg>_0n^gqcxO93Z_U%%`7_DC;}zsu`bG#XRAt%? z3rYFnsn&h>9N@`NIu_sM(qE5`An3bk=EtA-Si>c5SCm6{m9*ER@j2JQs#dRB%h;QD)nS4&8nC3oTm0!89J5 zcANTbsN7_$TU!SDhh$Od;9s)wPzPPND}>t>$%VBd9mFxh9;T0fCuOM{ z8ue#};I+F8aMxfsoC}=A)mY{Lt0%AG>!)>ST%EkgP=hy(bjkuR2z_yme7xm_FPh(@h^Zx%vxiB@ga~fU5#yRq48W0FtB5C$Uo6wP znJG`qWKf5am}`c}N^5bd*G=JU>}dk6^$j#@Ln5{Ge*kAA%rK+sIbQxJ3nTko={9*O zvNGf}@$agJ0{dv3ss4wk^qj`I^fY~x8N)c}S&$vAZqUC$3*)XmBF!6E+O{_ZWhAc9 z;apj+;3aeJ-p4EON`*9MR=F?{eO6)3A9oNsIwzT(w<{ppArbm{Z1=s>FOuEzQFKs; zg4Z@((r6_HpLf(_#iudcH+-CAs})e4e<#T2&fi!#FB|GsUxzJoiy9LpH}t}(m0DnVI0j75uf<_mH)_uxOh-kh;Iu(W z_-knji;~SD;b|-`?yZN#S@%ewXCXUvRT|bNUS}iSrKxOaCE4D00sqtU#mMg~aY>;f zt#&j5nUC@9*5(#wbit{5H<&u` zfF9b@LRLO~4@J*HiS_L=l#pJ-j;#6w1=4fD_RtmRRz6HP`!z__-bzrcSqRI!`iWoz z1D}6e(mTeX82!}_)YE3+?<56IRb?=~YCAydK?fIou7qiUOOZLgl9ybOgk3a>>Uht> zdGg!wSYjYun0uXg^%bC0*mg2iT#I(^OJQ0xH8^gi8dzxCUVqH^0L%&*1GlngWP<0f z*avBoR244R<=g^QN{C8rEov$WI{Jz8x zSEX8dGQt3+J`$m$!*9q9gKIFXPy}~3=g<}JF2mOPy|Dh&98P=Id7`H%i+9c6Ga`of zN%f4C_?~}1{0$spxAvcAa(L;FEhXyQg?POF-t{N+-RBLM*<%eO!@RpX{5^?}nhw(} zBdJ895a)gQFrB{FpVTJj!S)}~BvPjvH%HXq{+>#D{M88E6?nMqN>^qD6jo)bQPAT}f^&9=Q zVm{vBUPg)05uAF<8Khecplx?7Xne^+3*|gukMMkm9gE=YsV3r~{fOjx3ZcT*{f!0& zl;vybAOosZ#9FqMJioIK0vZ)zvVa4dG~8+7`$DABCU96~3E_)z1fk7qVDy;{4u@mETbidw9qKcjD88eYJG1_ znzfkpA13zx0Bk$<8K)c-f*HT>QIpmPXuW)e1j*UKceO`UvFjJnYF&@E^B$2^k2B!! z_#O|3#+C(&|zp4}*Sr|xTW}SwUpK8EH zaRVNg4~5pwFXT_}X?!j$4yNmhNPxaA{wumgN=&D5D&I^p?~9Z0#mfvF$sD0ElM8X) z)<)p>sYHdiCZ?lyE$LMnBNDQqa9z>T%BOM$n4QRF_q^cMa?js@J1rG)ezPDWw9^e; zH-(b6wD~lC;TSc|N(0kL8yckF4>d_otQ+^#;OgrQ0A5=m?#ekhIr}%0{Y>+TTVZR- zA`m*6f+|;9ND)Vy2 z07hBzabx`MV%4^7C|4SRQ{Lvly^$p}^_)F^R>@_23I?dea#0X)1J zKxca>7>52JeE;>RxtNQC`h!2lE;6} zqebj)bUeQar?JYsGt>iq)@0G4vNrbNb}RbFLy2q5+fceISV|prRG1FKWf&wOh1I-V z!a?)KLcHb)>G$YbMebfGoa>ANbZ$P_~+jYYnBXv zv{VotYK*35X#}p(T!u8m+*cDUaC=l3nmmvX&eP z=VhVn>QTNd8Ilep!Aav$P_LSf@gxdPh(=?~r5IW_VF-)8Sy-`s4hr4bfcjmlc>P@2 zOrr&bRYxoE<^BEW(mxfa?zBg_b>c9Pp~ED_ej#UhT2Za@FbJ2A~QLoc!|7Rvzsx>DS+78a8%pT4oSB~xE)F&9Gl)!vU=+o60l8*qY@hl zZ@>P4TYZ=5iGQ=fr9_xnyjuuQhd-+`>;FrP?kAAy8U=Kl@)o|oU4@5j595;R_pHgS z5!&;2k`+2Vm0r?4joU(b^R?V`P6BfQ4R>CJMV3h*_Uk&)uFNDwQ*B}7i!_^4+=$Z~ z=F#rV_o%!ug$`)zA^nvA^p8GxfHq!!5Xx5c-67lWdEqUcR1!Kq1vWn!hS+&|%%Za| z8Rw2!ps45t9)}e;@_JwCwsW_@b>0ADkeYx$Zq3HHC?wzXPiE zuQJCSl2E*C0M5BIqG5L(4Dfaaxckd+&FSgT{v?Gx;C+oMuXSYnWJ9o4HjyTl#X{-x zyL5|CI+gvAMf<$N=^yX2){(`xz)eV$G}SJ`PFYQqT`-r7dy8PhvJSRrmkdNV9K>t= z_sBc}A+BiUMmT0^fmf|`NvgUI{@CjS8fN@Npk;SMr&tzQb6FooLt^lcs2yJa7=gY$ zns9WvCx*+$P!AUZhtyN)z59EaI!%5eaaf&jIj5lO#eAqZ;0ev5s+c#mgoaM%phf_1 zZhVo74L5_}1^;EXqpb#F^A=*qqF|hmDj>5D<yJj=|kaapd^4<#=kI9W%wz_jZ~T4 zv<}i3W(S@=0@H-qoXlr0;Avt4Bcb+}w7ajw`$A$|z6mYdCL_R!a_ymt%@2t9`Be0g z>xMTD%jxfSQQTq~L^e2nX9rjAC-;=5qFK{XZ0ym8(2!^3V&fI2PhpH|i(esh<2{(k z(;Yg$hu~=18N6`+Iy3WU167|=Ld8e~l)ltMn^hy={Vfmo{Nn9wj4p%XBoo-IwhP60 zSw%V5bEuK6k9J@FP-7}eL=-J?>&zASGw@C0*1LKL%8$uN_H(M?A&NJ*w8F9%kLeHN z%^+#lNClPN(S&#AwC=IHbx-Xk{I2E#ifMb`65&AT)Eu@>y9!DMrs9EK0~pq7W@9Dz zVdSedYLa8*XN4bxq&%{|i#*?&NN`HmKf}{!dx%f1De=ImFuv$EGjk;cLoGAZykd;^ z^V48o^A`Ac*n<1%>`vIWP@ePfTL!LKd6neEv||!4r*+UZiag=nH_er@Fn6B^qY=l; zB-I3??l*p1zy3IN_Y{X;K1-Q;n=9<_hz!KNRmE!#&mg-}3zTB7Q-7yLXp?Xbzh@Vb zk8MI&t;xr}^3mqj$JxT;u zd7Cy)4SiQI?2Z3`TQ~

f)$(>@4h(Ifi!LV`SmA{czmWo;J;q!nk!($Rpo?hOsXj zBRs)qYA&OvL+;?a zeB}phlWj!rC>=Nx*M)a)@21~m<4CM^E=oK9pbG{zQMs~A(EK?-;EofE zP*3MJ*`1RILDrdYG$9Es0@Ue|c46+76IZ~qJswA*S#nU|DoXy|2K&ezDj}^*&h5HR zQ+398TE%QCwMZZ1CTfYrG!E$PtfEEB8)*9_UR#~>Bhq(nJ*=zIg##T&NMzX)GQn2> zlgu6Tx^4pgIbPW9*95`wk(gLnNbpn|+ZXkNFo z&=%fWEW?8HZs4-*2RS94L?%W>;m(_A{ILEJS@w(zL4RDJzPy?0Zof)O)HLaQ|5M;J zcRdweV1IK@z~dNS6d|MX5S1c>lxz1O#t@&fh0wOs;^g1RK&f=e^V zrHv=aD(VaGUVkMzckF@fY{%FAyLp~GU4LPHg5UA_U2wsY%2}*$5TXi`S8a^cM zKyTxjDKcEwDh^s|U7;iW!Dyp33uKdoVB_Dl5RtW(zWV$EZf(z?T8EaPm`58t+3^UK z<6~Kis5BOYBXigu1OOk z+4RMk6@a#zFl#`Z+vzkNT_(Q6se?uIb$N*OQ^m`0>P8O66!3Fit`vg_&DB`BI0yVv zeZlpY5{!5?psY|e(oqNe)?tHjGs0o@gFxi1k5WCASZemxA01RF+%?W*tc_Y>`ig3@ z>c=W3Fe4n}7yJj$lv1(&uqEp4Nub@I6(FN%(CWc|VIb5p4{F5{V3v^%SXqf+rMobB zqg+8W*Iyxf%a`CZFHv;)lt7mL;=}B}YLKzO7Og#+NJ^m{S)2Bcz8V)s1IZGo+UbiAaBQ}Z`D?utGNo53r@kV z1YP*G_X<{CD+B|>uZ`Z(9V9@q3w?$cqmX$98|z}lk)IVuMsG@>)9f!a`<*)I^zXs4 zh*_N5;t{ZE=L}f>xEZHt+$Jtz(_ziGT~ymt6)Kjd;VtDY;F6q;Iq#j}hcsjLvhOHP z{2hmt2^uumuC>v9NEbhvjx$CZO+l=8fE-j(g>3?v?1k~upsR5mheTIHs&5j`;+-?T zv$G(r^d!E&SWl}XJ~Pdt(p*#H94PQ`#hudgn5az})*t^11Yc$rp{G7)Q``MF;y{x4c(5WY9kV4C7_@!gSUX`6B zY!skyj+q`%U<=P^wRZkhIhx&D!=1muGIZ5h_ zv+0bzf2~s|j*vT*EdCLSgl$pk*j}B1X4Ye@@%V2@J$nvh{yR)|Wn?jKzxNaOg#sKS zODF5A7Sk~9zwgvjB!p%z|4z&kKf=<5YhWJb=WKkhh#CepbmP=gY^w^uwh=S>pF;s^ zmiIQgI)=bTr7d{eEgbnx!=d!@4)AE=!m4m3c%nBIbnX69cjr}P(W_9fF&IaI=R({k z+dh-FcmrytBZ-|WiqMAFM&`Nt4Adow!`C|k@RlRV6^qzQMP?tue)-irZMBS*TCoj! zd3pFSUN6+Ve>rfAmv78EXNZ@~qi`gslI)5MW&Re(;V5tR74Db~M{;xFmLjs^tzQ}@ z4vmlnB!Ji|R$=u$-tQIXpyJG>jfP!^=;HJ-=0yAvjQQ{xA0?&|PCz2MMD7ElNI9;6 zg$}uPBM(abL)kek+9W={21LOfIy~nxuV>`p`*eQJ=8X*~5TH$ldU|Ns$UJN_DF?5) zk4eHXm(lMF!Lw_nFw?lO@$V{kbkrNCZC@UdiMb8ruJcL=sryYh!Ji=g-war|fZ{!` zI9j~)0PK3G4>N-taQoP6DwQ$~O2~e^9jiqcB@OsJxD~>mzMna<#tVmb$Y*J0ms)^4En{lGNdDb znKpVC-4B*gi3q?m4QF_HsR8VJn+qSDV^HJmIn-XR2LE0>r_(N6JDi7wK0NE~J4wXt)s9M4^?M6K0lFgZ7x)G|@DH2(p* zTUX#CP6+er`A+;PSWB3)bb4Uzda5|I8r<*M!-tL_GQ*pntG@dfP8AjBHm{rx@@WrI z$;g~4Ha;c?^y4A1s~HPA!{`s??{v?> ztuV!~oX#1ew0iP?tv*5mERz@s0_%U5CpbRubEX?}^~vS$ILx ziK8=OLt7Lk$-(oMFmv^FrZe*yaSorsjD{_R7c+SJh>IpFAD@Z&%eC-*Ln$%|*D-DX zeNd7qLNTLMu)D^~ux#P6-3I0X z7oh5W2~JjsfLi!__+_IHYs9T6-=AHqQ4DX#Is6v26~EFLg>Ugz?>q#{RJwMCDiPdK z#nyEtQdY(fB7UwQWtr93XMK|dFU5BL;K!xabm8T-c{*Pu zZoP&VF`^jdA&>5pVQ}9TV3Bwxa-#GxsOuX9W{#7Duq?3M?1Y<@r*NNpa`3*+dJLaB zYHhT)lFqrmndn%jQRAj1_+77$$WD=h@+plZ=erj!OL=LnpccZ@K!kAnt9;-prXs3v zF>uonQjqVA=F2WHuUm@IX=qF1tHj4(Q8P+29i*vIO#@UZkI{=ksdUO2T})dSh`+4s zabT%A_x$NRSn;$GcOQ*}S=MSo%2u1wTx-;A4i``_ndevterm*Kqn4`%0_A*Q{=nH=)n%F``t zaVYZ{^iTUz-crA|-XC-nf`3yaLI18*LRiHIf4_|(|3j9y5(CYuV z&?J(BUw6pEMU_-ck}V|4Md^^U(h;3kK4sgcg@Tw_J$=4=66*it((N&}WTLQ>urAB+ znpP3Y3{9uWsx>4rVp2n2tD{=M8))62GFviBkP$WTLX{qUHiefX3z@qO zd~`>lsHqx?jh$dtiOo$$xJ^5bA?To3yjcpA?~Iaqzq5mvbDA|_7LAYuQb7>L08LT^0x_#g_`%5e15PT^}G z3AB3n8a0I5QO0ctq5rAS{VOujdpH`NUx-BRHy1c`X9~AxM?ORhGK^Z!8M5z>JSynL zlAN}=9Q$u4FjYhqw=P)-D}F7Zy`$Cm)xMmyKYNOduDwR4?azesk=bzTcsxGhpU#o` zn?ciD{zuVy_~rP1aa_ApS{m9Z(h}|aIz4x^P?~5WmB@-zR%mG{QVFH(A|sh~U#I6z zB%{pCln}lmBqOW*p5H(4Jg?Wi&UMb`b3Si%HowZ3x1CibST?*(u+3mUFU_?zWGt%{ z40omplSZ9qyW~=ZAra36)lMP83CF6~=L+N5$giFJ(Z$t*6Y)iY1tz=s*Omv^ZIQG2 zgtJL3H>}SV{rSTViFVs12l50udXw4u$`)2zZwsIIW1(QT=FMMZ&igBE*Ae48M1^=$rX#$kT9i6wusu9>}a`YNkw8qLcZJrH)j8^`}u2@%p? zlU*)$GzsoDCbC;4df7jvpZQYLvwYpn%?;ly4hW7+(_k-d6$sVKj`EK`I`K=qj2m}X zF5nkgUl3$+O}|Kvw_1x>-UpcNxj4T5+xV@Y>6ZP@J}QE zwN_pbnkMGboWl#&Kdfb|_G!B$e80qwXKxCZ_}&!;2Zyi+TDFy*I{#{>ICOMDf1NziiLi$*{sA#1>tXzPnywK!>{+C!v5x~!k?#}v2h*8 z8|Ix9d7EBu7hJ!5Nib>0VWH>FMeL<}Pel556?-eQn6JG4TwwU$6oLKVKf&OWKz93~ zGTwLFS3z&$?}mV;k^G;|t9+x?SAoXH7p!_%3LnC!^Px$Fd|gEef8@+wzQ;q_Ww%`_ zYaX|b-7RG%@T#4`rk5QOc84DkbYAuqyw-oh%4WY5l*pO0W?IMC5h?*}ZH|YK)=Xm4 z2mFLe0Um-MVTlbBN6lwlRCf!{Dz9UY4{TvHNX?{~c9&gw2h;RR1BM_UT#15Cr2#Vyz`J3CX z3H&vX$Y@Lg*%Yui%JK8^p6b-QZGzH%4MTy7W$=XVt}kkNI* z=L)>=(lvFLD-V{kDs|QDfLtJ3`%Rg5jMEYA;Z0%nk6QDWY<{wC%S?qD^1Tgif$74+ z6W{oII`!+!uBiS=rdAfg*kQ89$?Y86SU?`8^253a_A;yZdv7Q6Var^E zwZXY;)u(0biXJz9>#y1TpE?`6dyjNQgFKPB=0(X55G!tzu;`kG2svMSl-o7MKltu z;dQs3VeghEt%L*Q`O!iu~Bew}(wq*+)-Q^oiXTDh`{&hq z;jXW9`I3bt0wY&t-shc~uxnnXv&^Zp@cJ=H?!R=pv*}Px9?913TLA?Cc zhJ(R@{Iz!ly0&rpLf0Xl~Y#}ds zrIKyfTg2XclqZaK*vb|SiahC0Dut0g!>}foXRp6J!B;ew^3xgQ+zAnCc`XuU zj08Hn`BktLza9#5h7LMUx;7}>HE1IoC-zX_(cHsUecZs_8YjbUo>Iz}kzoGfA5C_5 z{9-<#|ATPu!>#N&&QqZDtlovK{KBCk z!Lj@v-Z=FG9%_ovlN~SHksa`@nU@fP0_C0Yw2p|iMP|(u+&V!Dp3ZW1Bg{fDdsF?JK2?QfiDNODsV22Y;M4qQ7YIHm_|>5?4&th^P! z@8D{|5&!xI<`ES#$(Alsu0H&Twj&KrpMrVGVHXx9OjzZh9lXMvj}7aj{dntvPWHcg zIRW`&!0L_j5U8DfCp>uGoDV08tZ;HR|H3Me%{{KqZeFV=ICrzeIVf&EyMeX{Ztwjj z^5Ofk(dQinCZmp^;m*CzRT81Xh*esG@0%kU8de_=tTA85DwbujQTt{2zxVHop6#3LWVgG5ttLt?uRNx+(fZc~rmQUQ z^U;spc)`+z+v~?q3Gx*x`q{8tS)ou(Z=B1Y(0zPf;8s2~MwdTj<-#ukP2sX1M*OyN zd6&BpT7pPR1#o-zmTtHkPD3wR!Sj9AiwBx|QQ>q@%# z@e^+M+Ew&sVjeXKl%>`3k4QC1!W)@$Fls^?L~XrB{6$QafRa6o*Y&;RT$dQU7O`Ph zrkG*Jqg4>;tb|RivvAVjVmzFu0Sh;Y^8NqL#dW0%P<&MyWZc(-v5&oR(1FGEBYi>t zc?OwNQAi$48U^FMbTIYpK4RZxBVzJtz`kJ-yZyTs4fM?deo8v{cNL-wR#LKIEOB4i z#PrO*OpmHrLau8x{&o()-O>55AXyrYUGw2EVHWYwIzU3m`3u#jQt_v78fLeof#=Rr zy3BG7l$;NO_bDr|V38^H&>xHIEAG=*?JV%iUGeJeKC()bZ5{0)fi!17Z3z8Y_v=6h zC08`K_oHUfu~QiuAM%6N%Kxp`Jl;aK+bR&rt5u9ba}{kqtiyaxXrroCeYD#yoT$EE zOCZc09kzR5d`CKH`c4Dmv-V&wDFb`AaI~au8a%mwhLAe0AiLZ&W z^GkAJxg|ZQ6Tqz>yiV+#&A=@_m53i3hbnf*>Dnt3$+ySRq|tu_o}F)nKMfNx`6Pqs zSAw8_a0kt)&4QlOG4NkPCY~)#r8s^D7Wsal9rNyz>ezG|o7+ks`Q=d~4O!rxz2Qn; zX`?tpsg=Skpns&%XL}qf^oOI?&uGwE@`;ot0*;#Xg8nHC0tL@D`XwV8_imq!O>N$= zo3*B#_hM8$=#Ledo8asuJE9@Vb#2YQWNIH;oohgYe<^@=R+-k4tcBX=u#vrg!AZI2wxczBO=r-*WG z-9iu7WD`cItixlK>G1kfGO1EOOy8Kj<^puh;iSxZy4+0w z;>o(;NvC1t5^Fj=Vkvf_I-1M-fYfnMqU3X&uH2wR1O1c9vvbGEL_sF}GFt}pVJaqW zmcUC5;rOOvHl9$6g)bVB@Y5flOCP@nibDT~jydx(% z>xpiTDwbK_B#~D=>47cQBsWfl@yoL0f>OL-|7t=ObWFn?H6jkn8yR5keyTVA)kX~? z|06$$Ijr$lfVIKT=(e^nDlmzn;Vw0l?D3*AXMC(b*c(ARqb0C6`8_GxWe1CfR>6mf zj$rn15%#`PK}iieuDU<%g3``f?#bGFWS`$U(Da%CbLHZ3QJp-5>dt{g8D*GZ9*6zu zvmmVbI+G;xi~NnA!wmg*jCkhuFn+xT#8a_{ZZ$N*OJh7RF+q7BfI9R!JHE-xtQBSH2m8z*DlsaR@b?~n*1qn);$#X6^6LK zJ)5*#ZzfffWiBq*eVle4Nr0_8mcjH%Ven(cTD)at22mf7JDt)fIz(+{L*yB>CV?dA54m_k(~RhY+BmznzK+3=#z9>Tw8qPEpU?9HEpJH!3ZuZV?q z<8@r0rW=e{SWa5ch2itqTAF=tD*BDr!Ype1Fm-I|>Q0 z(-QR`k^2N`^*{C-(6P(P;1@lHX}Z;IT^^MHXutk$EKWRuo7M(G2Ipe0XjTX&4K$XQavAJd`4tQih|GRKbI&B`* zdMtsCJL}LgYAW`fnhd4C&yl2@*?4W61rAh%pu6{eTDWQr^qlD?*PhLWIhPq2^D_dA z!HBxw-Alx^E|Y%e+jNtqGME&Nh7)eniFNxZ{5nOP?3KCC$oOBOp^Kzwc25vy9@&9m zGiIal&jPSFddrO)VFkAY2g#4Saky!|K9;wAC9d;|spQ)~^g|uN;^lo*Uv9Ym?wi?k z-A)f&`rst7U%Y^6Eq1~^jfY6%ATP=eZe^BO-KQGw&`H8s33~T`AYaYj)3nbon7a#{ zomz&?n3ugLsY-+=b5Fsa+M7twb}t)hWu*tp{cmt8uV)Y`Mgi`TNlwaXjrA?-0*S++ zK%!kbf*2a*GB>0$xW(>`jJ|&kZN)&m^2HT<2iL-@B@>uqd4tTynWu^Jt^Y*1>}fDy zycjke67lkuDdLF$O*;MdOGdV26qu;l<96e>^vQg8Jl-=vtB?F8FM5>W)0D07(8~sv z|H?({f)n(r`U|I-ds8XfHJS{6oC>ZVRcJ?m8djH%0zY4O{J3v4=GAK>^KmQmzgh*d z<4s_XRtB-{KR`WNovHq`EO_KI10J!(XcjjWW+umhP$!${_2uIDUygXF^$9t7Ya;p5 zF_F&I8HXKjH!{`d7tradI@l*&!RQvaqvQHCDn0)N34UKicW5G+>@@~j1zI#JwTGOU zU`8h@9H3K__Yr4TBXaIo3mG%gmO52d5Pj}7k*-t)Uc~p#ue?cpba&8ZQ&af-posL< z*kb!ebxh5ec5R{c5c zFSIMs6_Yaj!9OApV+&avyG@Zcni;~6p#pOMkRN_KWsgo3PLTWIGBbKzHTbj6(^u5IC`z)AR(NNd4^c@MSYTZu4o+bT!qiD$TdLE&ITkqczZdjtObk^z z^^mOC5(B6tMid`fkfc0Cl$6Y&DbFp@Ajux@U0e!iF%RnpwqTW+H##Y70+|hE+|b4c zH0f6%xAL7HJWTB*ZJr*$-WU(VJ7Tb1CYRiORtS#Em&3^RV!UV9MiMH@$>zuFm@Qor zwCkE82Ckcq{0J6Df0znAC#A^8{gZL{$#-f$LUbM6u5eGgra`#889?)LG*q&O$iO%l zaZVh%jMm`TKufw;!xEk?8U}G#De(iMB>h`!z-DKaJ*ImbYHLszggjs zc*O(m?-+;MGb`w>Q#R0jcr>J_5cJVeruQyhpsyl5F+JoQi9LCOxm!!=QX2)_C4ZdT z;;sne0;2HqzS$T%eG$CRo(xALGsyf)v9z!MuSJ+HVYP^7gnO(1@(J6fxdh z%1HJMEim{HOmoJ#V%9%BJXNa$vb%dp3fD(Vmlsf3Z2+=b1-9FFGwiBMPBj;vk)CBk z^vSWY_~cY3NnD*ze3QeFim!vGaj(gN<2zB(?JAKj*aBa>Q{lshwfOAa4jSUQ0pwgG z;mf{cyjLj6nJ>9a4}B8S;cu_W+1Y?SC!El6W)Syqu`L{2mg6+SXbr4dqy-+${&-dC zE*%JAp}AVbjcv~h;$i4yKk(e zoWx_2WEKU#g(qp&MrBSrd@|%7wgY!pS(G_oWmZxhN8M|8#W9}IaIdDGS|5*uXaB8) zzjLFEFplS=(RPU>^-qh!fvG*z*MAQ1=bbU} z$Y)YE?*sil-4@-`zfw&-q<&0R-NYmjucPTdQtL}Fc)|!Y`RM_A9;!g4TbOzA)8W;A zd#7jq4w!Y*PQ)_1PumZK;4F&~Bq8Tsoi^2ns!#T0-StR}wcAc7{Z+)1#$uRuPy;I7 zGI-r|0V#Hw2czytzYEpXx%o<_bclh`JJevEo;@n3Kc=gvu7^#sqhRXY zh4?9m(*CibpqpbvXT*l#v`$azSD_8wHMwNrQ&Zd-ca``oNyQg=QS_+M78rfo90u%G zVCc0A)ZkJg@Q3}tU7&&TmWyG!&O&%|$^(s5OUb@+eO%x+5f9$~N9OduqK2Nk9RH0I z)k)njbV$yF)yyO;nxBqwLDtYHJ`xwa)q_S6V=VKGH#&akbsG7fE!IEL!jX4a*k9=d zU+hC*P3wGeX6g%)w4}?a=kLS%5j#&22c<%iq;Q1Rl|Q47atF!JsEOd0vxhePV_{XE zB6M#O)!ywa_gln1X)K=M7}ByB{`;~NR~|XYnBEXMkfWvHNq#Pw@x}lymO8-M{1bG8 zXR8#Ljlt#e%$~7 z?lMukbA;NBHGwf%MO3vy3~72hXY6;9%wIbZR!+8{TblRKio3tL%j3ixX6>dt{MX-&kRs*d@{-V<^WGC?8!iM zIaU8_K&M(YXCGmPHg_r*2k{ATxbUzO1jdkWr)*)8EajfxIY2(-%|Wegj@;bT zX|Us&DgMh&r)9x?nvkzsYZ2fPsRD7@BQv9c{J6^!EeuO$!lnH zH29N6n)dnB=XJ_ru*(hF)j1BA?Ek?v9X5vRa(RxDPNM$VzK+Oh&%zg6aa~xL8|sd# zq;IG7aQ_^BIdQAi$@a1P=zn{Rm}R2cUpLbRK8fDjaUb@Rd}|{((e#4^u9$>78f>xm zYAIcB(?f>@{-mbe1aFTJLoQ_!6Sd12tZg-E z1Y+k-hqX()sb~5cs_LN)Cbrd7u~8d_{;a3>6Pl=6X)ZTz!FGDwb|WoZvW#4QG#O?D zSfaW3KkmzGZ@Av^ltxo!s0w;TZi~MAfh)Y~SBHPCC%z1|jbklGEGX&>yFaNoV0fPHuEM=^=$=;~EosI@ka&TIrzj zaV3~NE0pXa-dxBgKkB=tkI4O%$K=?pct@}SXK3WW?IcqepO{L2tQ2E%5-t)_JO@>M z#qe%eE7x%J=*8>95+pfT7dFk4A<;iS(syhn-7Uj{rk$+%p`8`f2|6Bn}}Y@M4)_id=4cRu{#7;Y-nzaxV) zmahlPne!n0W(r=6^uPhx0N`vj!0P=>Jf(M&zPmrf7=^YF+s~Kjo0KDTyvAyf2pOvZPx|=S4BOp2Jf=ST~IS6~;4D&1FXzo39Dm_{ZKhKJ0f`<91IFv&Z z6IX!mLT9kOy99+Y!?aE;pS*vR%*f4`!en<}>M>^%PVbFF|9Bsm`QJ#4bA3+CQ!kLe zHfo)%ia^p!EEXO~R&N7~`7 zpc`C7(g;i$B@R2Q_2>tk9Ed!+90nUj=lHNk+_am45V36n?3p@58;;nat*0a&mR*jy z_LFJ1n;|AGPljn~VL_!QFS4b~XF%O1MERFjFF z%jnA$SGlp@@=1513r0JwgPi;6WO+(D7Ha0v=+&>O&!^u`>W8EtW^F(5+dl#qKa{}4 zX=7m9zoks7!(2$!nhGhRGsKEfqF#K0Cs=2PK**$}SSphFCT%OGl@Z0{rCTVYQ~sLj zxLRRpw>>S^W!XLc9CiMlvg98-)wY7^;psfYD8yI}uuq9O0qkc-(q= z1ysIyPX-H@;P>E_#ChRNdUS3lIV4p`0-n8~xd|`0&Q?=;eXTs|JTu29DP-l@F*xN|6{FR+j%$h5B23K@ZboP-^JnYd#$H0GVunna{G&Y`PfONqiwwG$Uc- z(Nu>G7rqgLj9{|xas|=8a+Pyy-a5!xMd}Gkk4Z#y$Km5R;i7G*yrYkCdJ3 z!blW4XPu(ww}p!ClL>glJ|*k^jD#`&zS94u4H7xS&tyu;Me?W67~7`YA;s3e$*LJy z!c@DA%KcLG?)(w0AO={#*qX z%YRb!?nKObG8fYz67G7QBDeFl(VmYS?UO!C0w1Zs>9Uogo+lJ`DP`iu%gW%wguxp6 zmFSuq;W@j#)Lh0F3tY_6bD7Flrnd@G=&SOvuylSubK-^*NejMCqu+d`W{KHk(w_&UDxD#hh5oenWH58l88B*v z1$|@?3|GH&(e`R@E_XyJH;}iFyVj|QYaZ#*XNk3x9dne+(RoPdE=MrDeTlq?tF3<~ z?tos3`6xIPgiSVa@ZLfN4A=f611(P&&BQFsimN4Q1zynP@`C7eEXI#nkLcq+2kDX> zHjKrXHWI%=4djYeK-qp*IBUEa$CpT9z>BlA&O4pzM^@611S1$>t_%IJo2$JUfPXqI zFfA+s-PNsdb&Lr}L^e}(?>$uGYy+u1yoG3tQpcgzVY=5<8}&P`63;)gxcCcQ^q1XM zrZCkXw(Lr!ew#0n@}P(0`G1;NbYLy_{Q`lu^|8d^;c7giWQU(h{c-xiOLUBK6qLIy z=f;a)rT?ryIbAd9CP_6s74*yC2giQ$Huniv{zigj6Fl%{L=^bWN+@pn=WRW?O})S zuQJH~oq9OGM#N9F{zuN|Ceup$N~hg+0y6zdKa+G<7D}E5!tR#2)NRdrlyaOxBrIA< zM!O_TT;5BkKUD#1=@RlzYAltU(n}VK&h5(Y7UT2DJGi11Dcs*v%1&)^Q|Ncmd3#~% zd2-8C9CM|0(97m8U0v2mSg#0}UY-G_D_o%N+gdz*W(0OFn~w*|ZaAIKPJ_iG8krV} zsdQLb3 z4xU;NgYu8E@YnSKIuMo(A64~8cqsvgusGW8>xHdrd_eogGO$_W4AD7r(Yf?FLsvP& z>_a!n;*J?uDNrS|bzSMpmlGMKq?Oe8)kO3*PQeq8b@9q0H5i-r&B-FEo~%)ohPXFl zFkyH*cjKNNmAkQ<-qFdYPn!fV|ARY(UC?4$?IzQ0zcSFYHx*rZ8(cAW2V@Q!)8`w^ z(C~pIUH^mKO-O*l9}=ZJXI zIkz04A#x({;*#)`&cPE!TB12>tkVzYJ=EpaOLF8)3|AsQ2PbHl;NS)k>vpjkE|*Bb zE3JmOu5u0JKURaM`?TS=?q~=QpMZY<3hDg|{bcEn733(%T(1Ln5Tg)0-f$G1Xk`2LIpuS=h2)F>S z?z6?0iZQ61QO14S^MIC~34&XOKgjTNR~+oygfet2NwJ*+>M>*S2R8%5`izNr9Sbi_ zw^OTkQh4pcF>0}CH&f`w(CScaa<%>+G3yg)1#$B5dW;hUj1xhP4Xw~^ml=1+X%;Tu zWdMmWQSjry9kTn=TPha(h7;4cN7g+)OD1cs1+AZEaN&0<_WnCc=jbj(zV`sxx-%P& z3euT5%rioyt4WQ?SS;H8g8L%kw;PTAM;2P@g4JLWsZpB^mRS=}Pa+Ekzwc$f4!hEs zwpO^YPk_$zc5<5H#ZEshw3%H4vE;CAF!?t4e+|clmX`X0+{|oFdRYJo-;zyBd+u;? zoob-p7z^vwV(MT24aFPJD`|!KPkPqrHh1I9H0m+Wi@ZqIrrRIyq1U3daAxfl(wg*% z4j2kRQ0@eqJ#6vUPG1Z$noj-v*1)_D74o6Cg6zyIB#lKoiK?KNyef$W?bR7%%(4*t zTQ&|FPZ`0#Z?Cv>PHI@Qr;amHoC9RaY+NWCjvBtX_4mFPlhCb{i%sdG`!DCwm);vO zIlPa!ha^Fo#4mcTD6<5Uhs}|vp*WZbI=vbI8wwqbnZGatF<+NO+11IHX zQdg%WP0+14ODIQKZ!>{kWfZg<9J$#}TEy_%+sor*&>R!;C)7NY-%#>^2DaO>Iq zL}imRcI>F84?OaTLNx(rKLdzb-bC()DC4*mN0?RO#=I+j%#5?H=Nyd7$lbg{bfSeV z_~zUtB@rTBW5g3KGVKhfJ3$31dKE#}L5Gxin_|-Rk97J2H}2-DnQ-V?0a+O=hZP%o|Je>#Px^BHm@$y)qQ~pd(%&tL0f1zuZ-8Sec|o(Y-ZZHO&BU8PUk!f2k!v| z9CvFO?%Jj-(&oc(-`^F)%fJb)*pI@fo7bEy-*?lS+te}8;R88d)=!F6%8Ab;Su%ca z8K+!60w>JtBF$;VA~XA2;`}0!Xq-|6o%&bAzCaP*2LGhGW-7Q^^&%%Dv5nkXycp>~ z7q`V~Dac8P;pq%pP>?LAI@5yCecT4puopx%M+OfnUm_cm$8mn9vx(uKi*ywELce8Z zF(n&4(6&~ew5R!kjr%BQG76;L^Zt_HE^AoP@P&?Q6V2hfN|=M4X;g7`sAxX(o-|oJ zBvPRjbiDp7X8Mi>I$w7qX}f-$r2qI##h1lXCh)N+ilB>@={J+78x~{up7)I5`7|i< zT*_QHtA$@1Q#ji;IhZ!il04eJ6+H?niPfw;@W@z79S*f~7jOEJ5vtF=FyIGgG|dZ>LCEm!p>&lgXjx?^|KvRY}LpN_e}nEhT1=N9fHZfeQstDE4I6|)AU2O{cb zjlM)PP3!3P)#IqDs7{~xo{F_vvv9jq4xB3<3AK7L^y(E?`p88C()WmCz1b3O;phQ6 z#_A+ZUCPpu+3s-DXCD(_=8HjP19YBbG~O9!OsYFWfPT?K)7SM(Nh?o>{oL`sqCWl% z{Z2>qO+ojzN|F*M1t(%tX}$9cGU}K#R7SXo|GEIMDS+ z#4%PZh^+LU28WvTMZ+*x?B6$62O*tniVZc$qs+e2a zbBvB2)n2zM(FUfjeMUyavY4{$C9zg5H3?to6am_E}(_=!`Wd zOtEc|CS9^8&*_QrNZ6CD2<>%Ss8{b{;uh-;-z2{g)-{!kyf(g$W)0IF-(5J>>f1D@ zq?b$xHiU&zYnVLGVkT|qGgo|X7I*W~EqZ3Ws4rUHP3}(FN`KWHu9thfo@w~|j*0%9 zPTKs>Q8|mYS35r)iv4;U21Z)d(i(T9BEd=j#0zeX_FdEV=dH zE?SZ8j*HjoqvWGW5N@gJSR^Hb%NI0It0%MIlTsfiIBg7r^A$uIlRQ3Ku?XgE&4H5@ zv9RmE%~*6~1Mm@Z;fhuhQOH-u#j~8qmQGC^Q@oab?Y4!Ox4vY@g8zX+l3LK-NA$fH^GtvDqojmm#7uofWyIppY_L!^F zwa@yg{taymx{^Vff9|6-xemm!Cx_lPkcJN3RPN=D+ti)WC3&N-)OY!%(M3y76Yr8$ z5LKrMR|A94irgk6=17y2wG1S^kHxtr7AWY*!u*Ko=+Zb3Uf-Dx%8dz7$4muDuPA)R z8_;*vhv@~2c+&Txk=74vrn)2kGK)V2lA7=8)XQihZF&-k4*UYzwaSYO?7B+MeygS5 zmndLllQXz|0Wyb3psrjAH9YAGWnQ-2JBhj2-986n)7OHz?FuaZJ_j>%uF|#rjl^tR zE7=^dlWe~@x`rEK&I$vFyYzqXZw>oO<_ZvQ9JXHK5B=9qc#!;Ew??k7X7oeJ5}tOSuk^~9q^nHg^{3C1EVD|v1Kl`anhi`zfQy2A$bPgEa~z$F)m z4RwT5H^lwzfasgbvS3zXPZU)T>1jOA|;^Q}&^sJaTs6180l=Ut+d&_aB7@2R3 zr0Z3xbZZf_+wLLbByK}9T)6tc58BZ53yF)!Cppzj7V7`FFp_IrxB>|z1>TQH@2=C1 zDT>$0z_5Jde>V#aGPxdXaZ=w~)wYnPXD;5Rp8P#9cNrf%aKy zut`xCPrI9o=51p5((VW8zF9!W&e6fuW>R$3W>@$!ON_Q%`$e{^++*}2Bhf$OE6s3p zK{bg~#&FI|SgFuR7Y3duGjjT9^SaHrw5q*+$=4|uYc_*=dC!M@*;kZ#pF_X@+)AhG z){*v#L9)2vHjTO6LqmIR6UB)Sn9Z^4;lD=-@NVXM{ORC=H%Cojw%1FerJ^gh{D}o7 z6x^pJwgK2R>_ECMWH79}GK9`KO2ls8pk4#UwCdsH)-(xnI&Yh37) z119);gey_qA_ZG)9uUboAG#~&FPHRM1-CpnP1N5skOx9H&Q7s|F3n%Z%$%tKmCI}C zMaf}$^Jp7AHo^(e+aKo@i)ul;5AINyNrkF^$dR{}c=DPiM0yAy{MASDTsj@j$Ipdo zk*BY9t|&CcEfr43dk_cH6x==T5E)K9Lhn@_Ah(165~nA+SU)WTecpy+n`nWhb!ml@ z{grLl7U6}vG_%3$Ll$k{kO^gC&amiN0_M+GCvSDFz)$3Bo3Steb^A|}>0CFlPq%>c z|H_!I5?3lbCV@IG5rpY!Aj9JdsnQ!ccq7uBX7noI(v4#1u~Zc!w*YE4e*9x5;#3*=khu=C z+xL+^KY4238joF9C*bBP8+c=1K)yx9;aS%^MAByo_*5mPnjc;A7VA8xg0zF4aQPfrPv89fATRIoY=rC7n$&+K=UUiYTx|k5GI(g)7dks& zHa9^+n%V_Qqg>NC*rmFQnxB1EU&D`qr}`2&K5_!&^Owkwya~ov-z3QzG4QHo63mK^ zMa%iVSfllW5&IX6@lN-se$51`*Hlb4N=kzEv|cVu%^OC~&L)OW{*j4&n&kdzGfrk; z0=*|u#|?}bPu3fbWe$C6Avv3W5$*i_)WE2S3Y5yZC(J=+mCbdIS*d}ists`6>8CXL zQW;5+w4r5^>R64BxwDfuP^c>>5sTI{r+h^^t>jFi5g5#r1cZR*B7HD$nu4b+%czal zID8aSOONT@CzI9xkUsydIPiNio_U)NR`(sr59>Gdy-zK9A$F0g3XR1T^c`t0y+At; zjKLS;YEUc}j`!Ak;{*Q%@N2Xloi}?@vNuz5XX?=;4{KgZH6X=8YGf1l{x zD5YMP9#JAu!-y#*aH&n&Xm@NC)4AgSv3QWjU49<~ZLNc3w#!_MaDUFNGu6fR7(;w0 zn$N1Fg^*VUCZhSqT~dFnp6+(JNq+UmFbXLn9f$U=A+BSN)9qDi_?U`38xcpzIFHwK z^J_c0CGP<(I9Nx#6NZR<(K@V-F~;g83qeSWsn%9=lB)TD?CAJT$NA2I@n25SWeW<4 z{mpHlcV{crQ~gLQ=kDarhA5FUI%k+DX@3}X%?5npcq&^K%3PG+!;ESENzx+TP}!h7 zs@XOc{#-soCK;&Wq-(j%iNPWwJ+qP|&fiUSYt8YJX$jnM-U3ITmSBYP1-h!TiWFD{ zF(($+(V~AgV5~kBCwGh>>ghp{SDZobg(iZ=Pdk`*J_+CM{Xi=d9q|P}84tb>1l`R+ zmBX{5x)&L+2Cw;e z;Zegi@Ofv8s5dHxnx8q4v1&JdZVjio-i0t(VGCSlcHok?3XF5U6zu#f1`-bk>D!H8 zxNirdVarN0NIvX}-RXasfD`_ZxYi%EZKq((mVELx#EEPA(!p$6Bms|6jhuWLLEgr< zlJ?m_)VPDt-1hs-o@2d?PK+;2B=3olu7^l4t)+(7*MMPp8NCGd5T9TOs!KG{cGVBg zaIYKL9cIp)Pxhe3`qt!k`y$-Feh#*$#lY?o4`%3e6zE@sBjfDG-9pesbB{yy2;CC(-wtk54N{b6fP@60Mn+$p}ebZu{s+ zus^vFm&|XWTII3e``Dw9ld(u^U z!!r*`H^;!9q-;EGB_?f4%>XBtk`*M{*jMIl2ZC6pAG%G+2h?Xw#z%nb+gZWeNSvm&sp>2~aP z{}7O#)Xug)k6{h0H?WB%4_JrmYT!^x3+F<|IMd-)&RqW<$S&@j#Za+6rpVrkH9jN- zW^Nf}l`KTa+x#_v>}_Mq^=p`j=1H6Y{w?;U%u#mwXaF#|yc*mSY5rt;!r6i|dU%q& zIhG8O0HHJW@r)ELAbDsCGprPfRSa4<{Z(&jpSfzW&5M+nVWAv8Dya%)t#V+LK2618 zYn%CSk|9XC(#fP;JI0S%{+gY?&ybaTI>1Vc&ta+0AZxKOl$rEYlUcp@F`H9o!dhJG zU{i-uIY}(b%=RH{ry7qjYre$vhuPw$vL)E;KW|Xyrwi7l1>@PW^{j+wue{Ou7?W!_ z8#`ULV0qe4+1AP+cBNh)<5U#JoNbc{;Qk|LF+g^?kW7YZd;iY$qF{u^#;F_FJSF_beVRkDL6{)DEQO~ zLF&CQY}$8=DH#xF`6v7M(>$kv*lIN(9(;^>5|qnW2S8TM{SuR}n!)M2AJ5P4(qK3p zJxt5f4fu)FJiKjo3|PDTGpFKPH@nR2D)VYn92;>}2S2aPIkSyh;t#@g=Q@B zU|JFj7^&WJ{-J5-*_;PZvI$(DU&T&$9Z|Ik~8>Vv~K!rHKw}r3&%m$2(+#70m5KY zp#N?e&U~*6=2m}Z*6kl=T3_c|FZCbhJAgoI{ozE8LPt8kzOsXL;tJWnYHv8EPbUGd zxuL9^nK+Zark(TeKpZQxT?xPbc8!(sdcZzOmSMt5vRLEWvbZ|5kab?`03woeS4`PHc#OPtA*XTYX}+cWFo&n%3{s!FGD{UEfSv{oVAWT|ICSvvnDH*W^;RfYHR8*^ za#0=!-0^4MFE?OrqhFkT>I}H$U;ql28M8|kykiw(&DfIV#_XkLW5&DQ0)H8jz_#NX z_|COc@yU?^&dRz(aHU2PXcQm9zXQtHUy9~zWbXww=t4JZq0q@M)3U>3zCPBe_w_&$ z$Y$*xO=V^tN$2!VX=Tbi#TdHK44B{jkBxi8!`D_Nv-=vtftg4vk{)#kAE(HjsH+m$|C9h2Pg(#Hs%#jz^O%L^}4b%!~{p&@QgQ zn4kF0=VULyepmjn`~PZ!&8qTvpIn=f}-B2~LYeeOMrnC-OjUZU~H%4ranHlI+E>ca{#OA%<-jZMSSMrYmV)p7j6MM z{DfRtk$>?!zv+h(aJ6{Ea3)04oDp)z(YInU^*I4z1}7VuYl z@;Oc|J&aL+NCT~Og^~WA%}EYig(nZq#4Xnx!0|x{vu7M zxq>TLU*-d->viVX$56c3cP4f^aG&k8k7id`h4Tk)37LqKnhZDo80P3G{p8Fp|-%yFaF?1XS-yth1s znYG{vlNV#m__ZowUib{qwk?&-FzV#!G)6NqjrSSfr=tGruQnJcmc~b(t_ILaq@Vm4 z0KPnp!@qM+aN4mXeqz+l*cf?(H6wdK!zz)E_y+)Wp_W+g?g=)B=riBzW?N4Tox$9R zR^!{wIAp$@={|fbD+h;I`FULFvm?oXa4e zdHXP$5nj5={(K&cYqNszu=_kLWv~>i6uf38YnQR1y}6v9xy~H>d*f{UGi99i;s#qK zX9M)(6*0M^z{u`I%=~x_JU-^cv5)x1oCXXS5x!;De8S99`4V=+JAdYV;!SI#N>`@6 z?=W-OY!!&-uE%@B`z{|=}X7RmxR>?#j>mD5BX!M5HaUBeCdGcv?d&y34;nf6u|N1$m zt7-yB>AK5K9hPU7ZZl-(eLKxwTX4{Ne}+CV)jZ69zG5bJ`y&Z{g<1m*KX(xLDxP7} z7h}Usf7pTuQ}8=L1@yezgx_`D;AC!=2jBXm7Y#4|55MzQ#(Jd_@DE2mvo-h_tLOfT zT{Yu44n6CNKldL3b#XVDkR%5X|56Fu97`Kn+iu@hF>uA9%Rnw}A_=SrP|a zKk$HEJAiP-)DwKuuNhzxR#vOyhL57XaD)M{tEp#PeJ5hA1uIy|fMa!9j;(CR z<4ks9;UF`iJQ}ZU5}lV~76Xfvc1Aw7imB@YjPhCrhi@8Y0}Q6%q@8N`=H+N6Wh|S~ zahl3*(l503KH0}Ca&2RIGu|;{*Tb0~EQh^hA}8`tZo~GsBbb?;{{frq70mOl-S~&i z4u0^3AmC7E0B&i{W9{fLs{`zrXk7-}mPo@o)oM)8?h~N*#yeKwnMljRykpaYYB;if z<{-UPSi8iuof$JW0KfWHvc$~{E6mfyYW6C?zD|Sv!L}3bI^YE5H3%o>T4k)`fB$@};vupQE!@g(Ea|%bBnNk{$m&Ews!N5qcKfRG(^Cg)5 zWitcMTb0i|Z+7Dx7}x>|BRiSOO*XjG%@Dg~djg(K!=f6~C>-fE%-$~b0Ktw&*{UE< z>`=E(N z$Jnj&2TzOiaql)Y;JtklmfCKJwckAA{2l+tSs13lhCCK9*;5ojr_&T>de8!9y@3^b zGRA<>GXBPjEfd4fm)~H<)!W%4&-Q`(01NA~W081BCY@QsuVP^DMs{OLJmazF8N1H$ zAlNe^0Zwf^fQ1(n!0}p7{N#BzbFg_McxT7}`#HPuJil6o?1}`}86__XPhNTka!71ZejQV&2V|&e%Ic|KCsSj@D2VRI}+h%O8i&vb> zjyapK(~4%XrOA&u&exonRMF|$DrCR)#m|mFK3)nJ4gFw!zPRCmm0ooM-%-Ab(;DD? z%!g6E^MRd{DF+Jfr!r}>^MPhk2s7DH4m@oOz(>bq@G+78tvxP}@oUau)iZ0E@5haR zP+}fX=#0Y-AyVun90$HC7=o}IH~didDVw_QfXD+dMRdm2#8ngDGMjbggC0jM{B{3& z=9;ZC?pUCU$A`7>tr_v4YW4*Nu8+npQ!g{xn{+v|cUH5z2NF0H$1k&C5gIsrnF)4s z)dJSvLzzDZJ2;O@jet?=9Y%X&8UMj0JLXu!UF)egwfPfnH?a{tt2yyYX5r5EXm-Mx zD-1DG$J$XM-_inIP-&=vRhJ(DvZ8lgDeoBWOnSud^XK5x4eFSuEb{5pVm5l>YOLXF zir-x_1Y+}-0P`=;ncjc3%*;E8Rr|h7)Wf8*O6xJ-tFw=Jv*iZAt6Y_7^*_O=xBX$c zkxFdH1|7zqbBZx8;j*>;p3J4{m+Xzr$Jx!v3Y_alH(0CxPR3#vgYl35lE9A%ehgwE zb7aSb+AF(W@HdW#G-gItAgbIDJXl}M?#qqmoQ++8!{05(&Yj!9z1tU=8?E|ytJPJO zzs3p-{f!he+{gNAECTXD-BPh2N?6zFBV<4;s{#9L=< z$AY@;pg}#IdA-IH*9h$KeuvHAqsBwFDrFa_GD-liCnjO3hf{!r#Bp{d?=h>y$^k6% zn)$HG3X4XkI3{x?z-8arYX?3v`Vn$ODqIeb46OvhGwSCtetfU zi)Wo&#lf@kl^}7J1+#F%3})3~O;EfyjM+Xc(*4wV<4x6jfVy%Yn{!ki80XIfn?+iS zPSJj2sNa-T43yzFUy9&lzEr@|cUAIB9{V!=CePXWh4TChW2x*jJ9m~7-N&52Oqur& zt=XQ{;f(6NCv2f7!V^E3fMa*3fjQ;t@#67z=7VG*qhwiBr>LI7Jaw3aH=`ux{UcD< zk}1!Qw@$-GyAFT{D|Z9KoJf3-DDgvlHi3Ej5!jgJjD=tSv588Oq8g|JI*gX$bxoR> zwoU5g9yzGmNim1D{ZzA@edd2FxFE_UB}017UdfixiM=U&A!g-7G@_svIf zhNxd{y;a73j}m!RSBx=n53DdR@E?2h$_4AVB*fm@m&OK~SO9~~`>>JYQrubO5AJ5k zTUF)EVq+_IGe1N#3D1j*Im`0man8JOtlXUdf}eh474Ju~>e7GNGfjhxNdtFznfb?UNK>Vkm0fF)<(4Hf0=Hg#V%FX%V>O=6Irbxa(WJvIT+^+teq z!40==n+Tepyk+%d{*Zjx4Yx%@U)ce71m>SAozcdPO5NXI>`o{xBx2eF|e=WL7$ZTZ8@%M!Jc$QTVSmCjiVYY8#uOH~+^m*&Dj#V3(svB;&$$6*9 z4`(X!9Hufe)wy_Dp*A-Ac@UUQ75SB)xMC02<+$PPaj;MR!kI&?2X>FJ z#R@4AV7f~TdwIcTu=N%LkahO&6xqb#QNY!H}qcRH}-ZpG?G`x(FSIwzMhBg^wyJ9LRnSTY4q`jH6gFMERN?n(GuwIeH6 zydI33oMt!?%kb@ki?DYi0MfNapo9~H!@if*WvR{K+bU3Y@}McWxw3^hIp`tMCKy}g zsThDq#T+K$>eEF#r(9uLd4LjJ3b3=TKqK$m5l?OP44FGpC{{xeJ?lIHav~iz= z57_u13}`k;fg2w}aB^ZOw+vCGt8v3S-_BpXQfW*NM&zF6`wam+B;^ ze`n9{HN*MZjqD>gbL_!QWS`p(Ft4QNi{?}Bn6&Sqtk>%K;6}~{Tw*xH#uj*htX1Wl zhXGv-ul~5m7o>}KyeMPm{GE(lip+7;3UP2;M+IljV!q{Hcaaj)f_nZDY^6 zT8lB)@B^mwS_}IvR31m1w+H7s)WL=ShM6^KKUu340a&QK4YwH?;n!mk;JE)v@V;RO zn3Z>%@e1?9;#1^sN&gl$_pLNG_vSLwb1GQVV+E{4;2d_`=Tn{KNCUfO(f}*hdx~vr z7v*)l?qp}Z^u=GU&Becjhd@q6JELM7!|acDV9FMmVH042e;xSAdamO$X$?!j-#Rn= zLFpm?M#5EgMb<^8%X1t5P3cX3;}pduM>s_dcBP`~#fXy%T^AogP`=dsvk>%pBJ{5#7Tj@^|k{+n~(MEY_uJatYj|In_F zthQf2dsj3I?^&z_GEZrOddrz$AW{OfMA+lTe|EsZ*b2=4Bnu|**pA`WHT?Wt5G+gI z!>G?9{5faJ*=Xy#%-d5*jMO)X)yA*0erA5S{HZeD9_I-pa%VF?XKAsYU$iry9FH+7 z+deVylErq|Hkq6Rd^TD(E0gT_i@CK) z!2kO0=%NK1mNAn_Cj0ikw`|%#B0E>4=Ury8i(R-jo3##=ApUeT1>Qd5yh&R0Drd zt!L_{D}%lFb}=OJwDoC`cOW1m7)NY1W48pZ0e^o_x~|}4hFuCDvtUS657$fJ#Rs3V zUd19k*F#$-yHyAO3v$Ms99cYSv<~FGmjwzgZyD{sCah!dGtS1yWa|^}zb@>P9=6^I zud`D5YJ9=A_C-sVOET&6lK3D>1Ahtk!i%=*;fCiMfy|Rx=&)B7bukg;PueCS?I#c^ z#9bA1zxznaPc9&hvunwrKUbjN3=vcP#6~nnZx+4WTrGGMCBxgzz9m2O<>}`cMk4lD zInpkii+ZEPg-Tmppwl%)Uf8Y^=qQ|u&fMKgUqtR;{JkiB{>uazXI0XW$400RcS6(J zRJg;cmVA;mK^IqeP>qiXw07@(C~1=nH+v2s`KxKz_3bCo8kj5y(f{t_&$d=K*f}LdK zku06#a}L~SUySBl`#}D+ULxSDlCXX9MtZUM4jl16K$lA2CAIAf=yk&sWUo62&(%YL z^nocT&FK@YJ5`Mebrp!^*lOhXzz%h${Nr}2yr7}}N$ADGAv7qtopzO75rnsvQvKJ_ zXlS7p`m6a8s@L8|_h#gh{_#F~L6m3z`cESCsCi92LzPiEKLp+$yo8#gE2)1=2uYoF zgE(vy=Y{lDpkS$dbhmT|ymGn;{a7X^Y>Sab@tEa$wy4p2G9QVF-_>hw}jiVv5%?=d6QdydhuKjDxsr@|Ba_#Cd#N+Z%f9NHe(3D4jx zMEoRp!CHkVOkjf|%0EKauV-OTi8eR4+8>23gf!K_l6KEfLtf+GNx}MV6dSn^y(}Fd zj(1Ioy@7^L^qNxs3YkzDn~19z4J8o6)G)RBFBa5c2DEqV1!v zsJ2HOY#LLbwss5YhdB)>N#Y?=xpxA|+clHd;vQ)ImrdaPI!4Nbnz~XEM%^8<9q+{eysr zKX<0TdiVl)?^E>T&?p*ejzbsZ#0B2A^C`zikJ_(4iW)_{fgvRg^!ctW9aNY`^A2dx zro>wMZqHiyAf}1CuSP(BhbW-5g;(GvZD#~hQ>gWRB_tOnMd8ULnh<1&R2=>3dXIec zu%ZmM)%_8)Bo2}L3RLj3NduW${UQSc9!NU12h~i9MZIfhz&A3^+;Y_ii_c8$5p=_j%wdxJGnud5;32=A&X*s3}Iv z)NgYOu6I)A)G#!5n@rEdh0;e$y2yFWzi38EB=K0lMaL!WNz?>ywCLAF;i7U^l$CH5 z-nq`u3m1%`tJpv2lAr}&i&Ba1g(bpuqTJa6qYSQJrUuyg9!7`pjxE9o`ZFIc$l7G&2Xaa1uqV|w^`p~;d$_sWRuYuSz-_bA=o~Q{DqOV+ z?QDF=mCfEq2WSL^|K8Ku=cx!qiL(DsOo4m11VOGmkJprXk%Z~J6xfd(fP>EE=+oaR zG(&g}NjPkvo{As1aLpO2II@f0=ar$hzpLpX-ZJ{fA_Go|c!c;F0>O@(YAAD)E^7DJ zhnq{gQIP+7sxexD{@Y(pS09-}QN;^#z2h*|{&oQ`ILHB;iq_N4xHxptSHz)y+lG4o z-4hsO>7%55zBuUfZ6y9UoeKSG;bu-MJX|(WxW*y@sb2LV`)_w~?Mo)1hmOW5>#(S< zovuUH&a04`L@fGoWFB4As)DxLw{r8;9bt3Nb3v5Pd{`BK8_mD}m>heaM0HwK=-rMK z;_F$8R`th{)_6=GuC*Y8mR5Ad_FVeraWF|yJ%RpCR^qOm^oJPKrO_mA2FmgjG29ln zphK5?NR7pDVs?8vEh?Xd3bj+=_Knj~xl;@hd=i6=TSv&mBr&>`r;Oe|9YE{c&!FFy zny}e)AyxIPrfAPuxEEZL%zHW9onC-5?ilxeU;IyDey; zObT7pwhQT)ix_L6UuobwL-hB}K6qcmO>@r>QMDf_A!7@1)OAo*xFWwqaAI%^iBX%5 zjHdF?*WDMMbK|xtcG;t)7ZoaVs8E@PH_fdINEc&&`e}nWs1Lz*;!3~@`0-rsCI3!=3 zMwn+I%sWXp>UzUb-IMg?<}&y#p_FEs3?o{gC_FH+gxg$yhTHh(Cg*lwBt559!7YBU zk zqu+O$p$7N|7B}4|{;NFE_WTaA@2dh@zF&fLEm=xie*MR7OfTY}`3PV_iVE7@n~F}t zEwtnxrgf23D7Nq|NtfP2`VHnH75@y{(8uH6UwMKS=Ip0AO-s@HZ45j+GFeCzSHea~ zTfs-&I}}OAn!4U#sDl6>)U_2@w}IPQWhE z9E2y;7E+y%RQNQ;3R(At!xJ_0g*z?egc%XiBwO(SbbVZb?pKc>ZI3U+=-hN+EQcb? z1TCt!cnK74`Mt zswb(i2}OT@ZbR(-L7F*!2?+|_k)zm1&ducx)r3OvE18oc6jHmuviqj7UJc?Peu2uCSeuse6UAU^ao z+TP{^X-qQQ_JT{V*59B$PiAs2iWQ*!)Bn&MaOPY2=7KVtMQJxZqe`k-Wzj>tAA z^dwY^X5PI47dFm^N1n>_mQ|UP>#mWhN^AzYI&VTeT%B9Or zE7DHL(Czls@W1&|DBfoh3O)4$Zb{A{NoGkXo0w4p7*C@dv(eFoXXzbFKXP+HA(HU; zL9Sm*;iezEL{51-pdF7KVaH;EtWVj&n?*wS>aZBe^c{nu;1}+6i~rDA!9S?$9ScvW z1)#-mYH3P$IoPfy;z^v>M4N7k*uev<5ZY?PZFzE>TCT1_+v`q}+&%xHpjls`YZ@OW z6s&=>z9$FCf3|Q{hlSzdBwOOY2|Ea zJ2Fd%(;mb0#g@p+tQdF)j0-kMm7=2^cS(a6qvxO*lhx=LSHuIc9ft}54#GtptA$7AWpK}F*9!Fi?V-g} zFOk1Ni|H-pW~5zTCor2?1{=4{qD)jVozVA+98b3p)GvsK_1lu*wh&vCkv@tnUrH-ZjmN#YU)7{fU_H>e024T-vcdAFf5>yy2XwP_e}oO-?XC6LW4;v+|wvN{|U? zv-wObR_>xEszvz8vQN}UR+cwy1$+tCRWoi)o@8UCo&5$HDx&MzF<{L+XFYiWiKO>Rr`=@m7UwfLWe-a&> zlSGa;y@f#u<8aqdJUUc&l+KCyOI75PX=-g6`raEyW5cEj6{*Y+NzMFC9 zFaAk45A)D2&UErRJQ(F_iKAPM{&a4S0yS4zfQ~Nhp_QWV&K%q@Bb(2KsirsE_rQ(*L z$M7n=q4^JS&K99p(Wl8g{S-RfF(1xWZKn3;AHbJ4-=ggwUz1z*IVjbuhr5GRaPOW= zpu$ZjxCy@LTzP8-t}`kUD6W-63e|=*K+%xg3ur=;*V1Wb(^|@P{Kvf;{DXX=eX#t& zEPBjgg7EB~Oyb;DL?k9!p_vVu^bm}qEwKxE4u0))P;N-X%dSIneoX}O=4CLv?)wHQX zU)K^;v)~z;aF~xSI>o@rT1)5~UmiW;r;7%&kTKXN6CQ z*dXHmbdsS@ba&H{9|u~>-oH#AZp;uSg+_9ByiPDu62YqWXWMR+8Ob~ zt#dt(j--1b+e^)Kh$%p=B|h})?^DpJ?jT*JrUwT&Bugv7I}4(a zyY4*d?(S0P=R{fh=G-7>&e4`f8?@=!JK7SL(U$kMTJ%85I1Ou z%&>Jr8+@W@-MzK+!HpX7@ozcG)G{U0fV@yT@(Q{7MM&OlIZpz?T(rC+5gD%9i^|>3 zQnSN9(2V7X%Kl*}r%s=4e>#sE_Q(i-JWPb=;uKLailkEEr_eNIJvy|p0D7`+)ZKPF zH7#9@0?lvYg|p9~v@>f_%JcQ`TF4)GsX-o>HJqXU%H!xRpouc?JwuvHCJH?h-H>Z` zFs*%b1%-UHphJiLqm%YpQjM3oT$LwsyxUI@oos!d`1yy>M-RJEBwUHSca$RY|DOxK zm(Z41lwOH$BdFGiRG5E+Z@R}|Xw43Rk|l+zG>%#xk42TABI%c~C)DTIRBpyjF=W8a zrdor5x7t$zos6uZ#|vJf%zX{y>sAA5v;7g2Q36OyRR8zKEaVlmCZXYgbHu^g0@*6w zfy?I}2Y3EHg>`~nlCfG*c;>nqaciesS=B`H-jmZM zuaXPGOZ1DC3{sO(;W zz1603va_JJ4H3P@8FI{UtzC#0E*bv1cmxZ=t1Fpx_JLilr*t}8fawD z%f}*U%$Nmwl(hrx_%#uoUj7Dc$ef7M-^n3)pK(|^Wi8#O5=X4J>(c#B58%Y}B93=c zDwpT7pT_D8p-D5#1>1~kX|RPC`o%mY|1O)-z~N#dzJrG~otMyE`*i5l-l=R6xQZR8 zIg{mwRAC;^M39i+M)N{kxXPuPbQXTYt*L&;y?Ag3+}iq&@C-VL)RqO6<%=i@e`Z z192m&?h=D^WZeYeZWY8N<1l)i&_r7D%Avrc8VxlATC7abIGp@k%)gFjVQd$NAD!-V6ws?`b`|r zsQVExbC-yVxH5h+?0!6Q1q z$d{I1|l!iZG469?MkEH?${%}!$*+Ut!nOWM=@IU zsh-^7)xfV#8;Ea%G;epm3^C@iXhWkCKGOlv)Xr9VbEH5p-Z+a)G*lGEs-L5=E6>yG z8`jbO89@k`P!&TvRJBT;-cuC7LrvYp zB6}ICJ9>+2ZnKs|{yq(J{^(Fig$7#hH658*okTl3@@TJqKYcn+jVl)VobC>skG2{S zqOfxmZJxXt@$4-`(ZM~)#N`}1F?S8^e$;?6e=eituhPgJ?H>5_N3US@ZZ9+fI*E?* zRW!hy7o7zg$%}Q$$k9QPCw1g8(Y!mGHhigu>Y6XmyKFCN@n$1B6z~*S4=VF2OT1Cf z9v$Ab728q2hz)x+)d6mD=|=1PrXUePi03@Gm8{qQP0lz~qZPq%fc++q1k%;i?vM@A zO#Q(v=$VPGUDiYMEo=n&!KO$j?IWCdFo+1(aA~&WN;vgZ2}u95gc3_PYQ_`5OkZU> z+TTQVmpnlFDzz{zfkn>oES=mxKI+sq)%)na{qjr#(RC>n&gKe*rZP`s`$_+ z5nEBsF)Bhl{x5DKn7QvfaeYO&)1} zKaL*N&7gHZe+kCpYhmKG8ECQ59&%)wGGt$agl)YehdjCjOl6b4;XTYqrs# zE$xEe^AeD^#%0*KI-Q<3Jc;UCKM_BN9QdEXA~N^E8}2&Q*#blRJ?L2MCe(R3gS=mP zgYFRIA;*m|(9SZC{!$Jltv?8wxo8w!UF$?94wrIo7>}aj*~xTuWh&RqaxqHSxLk1a z#CPP+|3oY%@~L4MB4M9}wA+3MdE}TxL#n=_e>+=YOGq>7x5=h)VULN8TO-<2IS6O! z$e@bU0Q%DAA2ctBM@p~ck?bRDr~tfs;} zO4M&afs8|GsxBi>&z0w*@sW7g7TyNiOpl}2wOiq^xjo>&)S&Y#TR8d^p-gQw9O$)Uz4X|5WqRSwNwo6XeyBxr(9w&R(XN##WcD_1 z%3XXKs(*EbPd=K_rpf^_GAoweY(51QEPv6)_Yt&tW*X_3QI2XJ<}!5$E~4j(w@~8K zDx&^uCG|TxK$cz-lCvQaDBFEK-T69-#?9uS@BQvXvNRgW$y*7+@A4?GN+{av?4~P> z55cA95>eXkN+^*nppVt2p{&lQL}j)eJ)hu5H+tSf&P&Bf!nYCXvr2lNPe> z&@5{FyNy(pKSb6$lti3!2|C5cfuQziR3H}#yUsD_xNW)M_~KMrSY$*kH$J3`d*x}@ zt|BZ}w3wbwNk>kv57UwE4w6>i1k2mKsBhkHsORH_%9=CiW<60)^zIXaC(crj6>E_C ziIZseqFso$sGL+)*zh76cc3+{lDvX18pOBfH;k@$jnXR|(UX;scQ4Wu)$VQ~Yafd7 z47Bdk;f0S$W-eFw!}t|9_s$RUXYeB#?(c+R+QlemuPcg8SwXKEmLUH~IqEw}K>Ytn zLi6xRP??{O4wx$m;@TQ%-bo40b+3ao*Fj9!Q1YC%@5&Om1=bM9;7oY=&nwbvnnMrO zrlO-xKjDUTf9c~dl0u1f+o^@xGP?0uB_#{P=yi_?yvtYJ;HnIF@^gMI)!a54J~l{4 z33;|?M`bWd>h&Ox_vF$4%rG@pxHD8p_{_VG$^Zyd!r2us;RHJf@WdsFu36}mh4Yekg zkewQqFzV)XWHKTNXXciQ4($rW!!v|V_;;6%*$HA{aE-c@#Z%jg zFVJwyPC?jk0rF4U!R1RQAfCZO_@8<(nydXDo!Pt-PP5xaZry1|reQZlb7Wa~!gCtB zH87P39_*koQ+(m3I%V>@Y(L#QW(4(LokX3^(kSc1b0`~-3pZ!a5{y2pB89wu)D&8a z_CQ;Dx$3+i^vEHkb*vsKcO6H+^rl0_R$KUfCqu(lM$!1*NVc&46iqv^gtl}T(%k(y z@TBd2II1`iTFYglwxBe!M=6w4&atLb7YJZbt10>DzY-}uU2gFt#XzOmTEw#e#dh_CIpWH5yeu;-bMvcJPkYR@9f^ zhU)B2(P^`K;GWq@Fvw~h3QoL0x4#4EzELXt*da#O1=hgvmbGZ^oLu;4f+7^#w*&4d z4n+!^c5(R+>#_Iq`8@baRNtdF!M1rlXeG@gho4h&eZCBxO)Al~#zykY+79hbYY_b1 z7y(0cSwtT%LYsfJ<*b;3hJeMFu0~b#T!Xq8%%fxr|S{aw_TJsWO@eA}2 zUkuvR7SeNpN9mjE%~0i_E%KdYj+~z7QmT3j_P?wW@N}+2XQmHMc-TZT0@_iHpb$!> zw~(P6fBN|EDw0dK7^!}6y$a|cGKuHw3;%P+XxwZ84+6YRwTp)#C z&cUm>OL6Jr67+X}COysSq7bugQnqXgeNp9rdYd|7>~%m-`)Hw6V{zzgemni~If`5| zaYRyQ!)cd63`#roLm(OdlRo|^i#|`DLiVQKfJQD)ut%vCb$QT6EdEU7-S@Sj(ia+0 z+GHa-AMmI`{tMBpeKwV-tUw1=izbQ^<7BB(1NY~VEc&4xBeg$H=#D}RwV5-Mf?QjNVp|SO8!jdYCW5ZUZ1F><=Yl|Xi{zYdf9zPZN z$;b;o=%vxJL}y|@Hh_c=0@2=QU*QtX70_^B6%t=|kn|lni@3Z(erFgKj@1K zZx<_y5NI*z=bsqp)|E=R8>_+XdKq~CXBHZLdz+rx@SapBdC6QHkw5nW82-Nmb@y=M7bUhxqPQHt3w%j5-@jCcHbnWAKabe8XaO%<6 z4d1-{g0}g*fIlancDmmYVAOeB8T zumzr6I)T?!Jd;XqGNwzD6H$);G4w{wfbLAe$n1uQTkNDp=U!fjLVYfX_9GgCn&q=d zO@2S>S6@fJia6I_-v-b%@`-|dT-tgG_*Jh?QM!dOJ+}?N0lbW=;redWc$1tJs)o;avq;Wb7@aOgL)RK3R{L6#a0Vq_lBVD#%XBh(HFF8 z5Yk@@4?)&)7OIW+fcmnE!WH#tDDb{HQL}%DzV8b}&+hr7*I(_pmp2~~IQ?y-awT(x z$|*^(t4)`8@$U_;{#s`$BV&UeYpKJ(|4wkbhK>`1?l;u9rUOb{*#eWbc0+ZW*YKjy zo?dBGL~VRuxbaLPED;XUe_d*5Nvi>rx?2lJ?s%Ym!gi>=D-Nwxw4maDYsuE?lc=lY z5b907FG$pErCq1;(d+mOvc1ZP&;T=9`Zx)X-!A1=g$B{+rH|m)lfSg8A{c!Nt)*iV ztpv8oir|XQJ+j$b7UjjrAe(&^sQ9New|{RD*FC0~42`St^m66kWq)nn?PDFH=Qj%N zoEb~yw?85&yAIL?Yv!Qm%`0f%h2zL8vK8K5nMuYf2pp1dAHbVD-t6{XDC==u zaLwD1Ub^Ltnx7kz!YTXU_)9)i8`Yw}FNh&YHibCIEJ6`me^QL1sgHUo^sMkjMGxN* zSH}n3g?KORxtsx~i!yNx7Y9+fk3Hz(>~`Ysd>`co#G_+kF*KxL9NGw5six8o=&$=u z@abj)aqCwnFBX16_mAqJ6Sfuf#{Mx{o;*qx_*g(BaSGKub3v-Zg;YWAF^rj6Lcc63 zBr*%vp^}?URMsmAHTD&vx%|l@A>SPI*Q=7|UY`u_B{&u}C&5JPXr#iB2d@}Nm` z4t=*X8<|9n5@V|#YOXAaeE+0V39Tk-HFSYU<)^{MCq+o;HdScfc%9lG$V9boH;MY& zY?^xF0hhfknst1OfPG35sBfT%x}1ulGsotlTz6;clC4GzEVQWDxCz=g?m)eiM0;If zIL!^PqubN3*m@*$j67Nn6eXHUewS192tpPnK;tzCI%-0lgdE%}+PJ$SM^6CyGzsgKV1Ch`tu-pmy#a>mgak zdfmuG^S*{7_fi#C=}Wgz`rsD!9lHP-C60)^*NC0jAz?^qPB@yedoPMIdr1@ArqY+k z)>4ay{&bA;McVBW%tkO5*%yQVw-j&pT|5}ul`;FEGEEnp^*q}?RYiVU|Cu?6W zRX;x8inY-!rU6^lP$Q)pq!{UlaCZbk?dz8dR6WyRGYd9J(oUA{Uo;2#@U7Z8j&mgSe;J` zYaIDc!=va7f00P3`z>1&9fGnfWT@QvMk`PQB`a&ED=pY^UYYg((o;By>>5Z_}Pk*+QX^S$3kBJtD)2Y!09yVoeAN^3765)W@ThfoW9d zKQ-j~Ugx#_(iu&KOg_t&=FtkD1mUHaD^Td?!H3`2iP$;rRB%5V9t}Ir zMr`n*Ps&y6Eq7J$sy8dd?@ERGgD-;FLp%Sl15Lqf*WE_M&zDBq?nl7;{^QWoFG4zZ ziWn_wrii}wU!}brGw7X)iL8g`E)=|HEiIh&2>C5oPLEBNuRm(5&G#D5W2X(9&`)d? zZ^vdM>oxa`bqm>y$UIgM zl#QfVMMS5!Sb<`SMh5T0j;yd|l+HhM6?W{P; z{|@*pJomGR>UG|uF5>x5#PfLGomjJVryq&(KPQorRWZ~KDI)Xe4nAsYB)>oFFU_4) zNpJsCuYY$-f=a7e(fvcC(8&^3DB;^k`#O>-9OsVu4zcZK@jiQpTK z=b*WV&ar{bMp$`AAmzpRGfll0=*Nb;^w`}5*1_@~e=K$ynsfOsy7g67JU^d>6n4h& zE?)0YtN9A*7Zi>T&6lPX{&L8pa}t7Ij`L}PQFM5h4+=i;o+Z6Y`PQ4&bae9p{`c$4 zRP$L9onh)gb>B{4jfzIq-%Wi%^WTJ`wKFy8EW21fPy4%ACF&Opqtu;Sf!p2n%S7k zmVD>V#Q)Or;S-&D-L>D){?Ke5h2^^GuD+!GnM>L9JVZ~-|N7x2-)yioj| ze13GmXOz58ib`LStN+(4u2&TP@mEwO(7RKkP)pk@(V}g8StI{K_Q}3Zo*4>f-$w7} zX;BEO$m?S_Z`GmuN0qP=ubpV!o3ph2@I`9B?Hro^=oXS{W~pEoM5iZri8gPjLUTuA z`K3RnQ>i#N*5`2-_1$q7wO5KB#R++vC@uw@8Lg$lONRYJJxzE{P3rDL|Lz0oQYR-F$MEB$_k-I%|m>_;I=IH^k0BX98!oR?CK zcZY?Y-D8pDiBwT_+F3MyMmg$y^`2d^%>j8So)rc?%tqe|ucMIJ+xZO51IWq3M>w~n z7L}itb-nRjo7b*#K!UV)tatNDR9(A@Z!0;AQhZ9;ueOEsw8|d(Q2iRc`f&nXy?qM0 zbY5P#Y|})2XCbu_BL=)<9@CZfuh7{4PQj zc)0N;pC&nq_Gn+EoAy_U&WadTb!P^$b@{>dlSZNP=_n~k z8C~n#h8hYqMeF~Bv!&BU)4vP8($%Bp=t&}Ry zcq&p6XKu%+{~>0tCZRRmQ>gCODd>kEMPuSdqiqA$;NrAHBA`U^f#m0`x!W7z+bc3u z_r?*rVTuZxl(L&vD9jSho6(F)v-i=7st?$CPkH{V^?r8BiB>+KD2hhmUe+G{$6AQ< zt1&d44*Clzj= z!NwdrhepmzAv3#H{Iorki3&uTNS(UQwV+Ci6zO8Y8+M-ZU*YgemHM?w zBkYH)!)(sf7ijBuIafosHY6X@!5dGGLg$h)K*06J}z9*z2!$ktk$()D9I;lw#Q z!sK~A$m*j5N(v9)xBLC!^(_rp=d*qYo^(ZZZ?4ggKZ;nraW%BTa}k=JT)_Lt-9QgY zU-4g#6!BrEf05YZ+SvS<56?2;#Y>HMxVBwAp{uND9<+;d7MVa0m@;HAhi9+|kv)FKC(70IJBIA}hwdo&eS6{xU#!X!R8s+nkE;=C^e1NYx zw~IGxJwuy*tYi->?WU@|Cn)#on^19FJ{$Ek0ZA-Mr*&)e=th+U>YLI*+dn4ICwsk7 zM7Rb$7Gp{;&%VXKl$S)~@1COj?n~CM&}=~9Y!o%oZK64Yt5}%{B{cKQJ!JgQmfH6y zP>|h9e;rBX)Bl|33%4vr9=pm!9w|X=wM{si+gT~xcSVa$)>fy_HN?N=?p$hOu30~Q zN4xktW>AlKRXQc0j6QsvNP}%;TxropzA;9VI%+>-H-9pxTT{}Jv9l*T>x~S0rtQpH zxy)eORRWOZC3`+(UpWeoe~j#$wb^U+Cg@RcFdKX^8Tl$mQbo-VY}%es>KKzkl~Ee) z73YB$7RK>b7e2F@w&hgisSavsu4T10>_barKcmXFV3DxwJ$w3wX8oqe5u%fi#Qg$s zu5w+n7ggxr&vwmAL;rfq(EOWA=-#85!YOja$m^XwJzH1CK1(vA%c7U?k=d1O>{dh6 zdHg7S@_H`%*^*6{IHsZZpG&AqKpG#Be4Ou|KEi(8qKCH1GIUqC4SJg-&c8G%p$+V5 z^v8V%imgne9r^9NV1qcz>Ml=nXPZ&GE@zaj{FF`KmBG59AMB%ljY6ma=+!3+Y2rFv z_CWCfHO*5(LpHK}w89Q_Ph}@)nCnW%oR@M1&I*l<>U%{i(AcTxQD^v7I=Gf$2OdhfCg!-J z`3;Gzfm1PEGNYKah*GBV%{tU8$Ay<#KA-ikpN7nZH^ zi^_l4S3ZjLnzuL?a(pjsdLT!)X2-Lin*Z`STSn28h1Y24tY})9K8?SAJczN%Ds1><&Skx^&LBY*^69sZbC6_jhN1oniBe4MF}mb z({tU?$g^c4Q^dn&QITGsP>^hloT(eTC3`k<4N#<}3)Z686T!lhjdtj3#uK`Iu#ZpQ zbrT&C|0kUWwd`W?TJ-E_Wrb_5q9qmCG}iJlJL6kFEiS8L{cFllY?3T$D$SysJcA8Fe4}K2Q$&mK=zUk=^v6#`E#mCl%gAsV;MdJ7IL1=}i%{KHB5sF0 zqb#|Nf}5heO;%Z>na{*@+$WcaM;csUA;NUv!G^uBoAf?Ix(^{x|mMtU{rkp*;F4 z^rv>`=Aw6%d7@v(QV|xUp*IW5>7~qBR5CA+YGhrY+!zD;dqNqV;&zq)5%A9?ap7PV;q*b}^$1 zY>R2E(k(Q6($KZ>;AOs^0qFj6OV_CXVrbr=Au@k#L1B0!{qt=f0;qdl+_JyFQJw-$(Det0IT&>vSxHf-{ZC z?cNC#WPTXkT$GAlo`CF@_w6WQsE|6U+~yD0`=c)X9y)2}BVmK;SAKIh(3rwc$TfAZK%x=BX|-B?gZOM;%z?9Kx;Gpd6Q z`%Yq4+x4O0vn6~=(s^ES@pa*dIOE%LvVl*WR)b33E~b~RU#2pPi_jJSOxDkdK`jZ% zqO8RcC_sZjFUOxi(#!PNU+4GGJIUGXA%{L;gZU*ocr20@h0j5$%kJUSl+qLt> zeSVpOt?QY6c4(ljiGGaz#!la9#1}ce11W_m^zmo|w0XdV?lDkywSTk=)s1&VoiV#m z;a6oeVS$Nj>!lNPj$sS!8O_n^AGLI}>piw)T)%L@KZPdE?PK_-#&pl{OjNS$E?xOI z6|E4ip>n@N*{k@?QO-Wcs=loJ(u@*oJ)^4LGUA34+i&d$`{=fX{HYe)8 zd;~o-(qlDaKk_f?X42!lO8tz>VZz?=g@`HLL!SB$rad zU;7e{UV5dl@;z@vYutlr)e~>_qo)crkLf_AYGYB3;RM&>0V_J&N{QC^VqW0sCv2=e z3@_wdLjhF`-E1_UeQIun(&szU$Wj3hXbU~OXCM1tlPtQ2Kl7pSwRBo(FS{*p9=j?_ zkv*&Zhdxt1FC68v54mnVf|@N4(NTvCX!5OIe#@vkG~&j6Ds#mZ^{nut4I*QDuJ$i~ zh~)7OtujbfKOZ%tOyu`en?}1Ou`gVAiz?$s(Kiz$U6VFCpsMxN?2F<7Hu~-eyDUr@ z^eM_)=VqUyop;`#cJa)xyHN>UywrnckDE@f9Z^Ml6f#)z>C@?v zxS#wXg?X%$jRiYZ?*O|oR31rJu0+$a4x_$9b5UH+CwAWE3SLzIh_07CEt+N(2HKOp z^G7WQsgH6C|4pTyO_kb(&LnIT*A6fEv28PH!p~&h-k_a#`yjy=e-_eev)%mOxCcn$ zoFY0RdrCNA_CfaP!OLjok+H6OUiqRUTKD-WZXVRZ?w|0raW0ZD+l4erqeLI{+k|z? zm3Wx+kgfY6O^-a2byb~wOmyUmrKtRn4Xr;^1I9$@)0Zc%Q@4&JcGNeBez{zsW+%-= z<3o<24J-9n^}z)39Vh{v)cQ^IX8jsEF#8lr`92O!Y!>l(FK?hI@tovyaxyFYb(o)d zsg~!OxAM`oiuC55Sejt`jy?0kiJEVm;wqHcBC?)bLnG}R(XXV(bhMo{T6-r1?L;S$ zd(Z%Dh_veMeXZ+1B|Tx6rWv7ebLG%1w<$DJF`YNwby1qY6 z|1>{#?e`G+Yi=zyIOxwF{c{FoMEKKzv9aubzmr5QucfJ)@iM;8=?M;*Fbiw4V%BE5 z4EfR(3L{`7^#3}8D9=m;Ndpt$xAUfO#v>zAwXY6H9NR)}?AuDFA6o?zY)9dps?nrC zeHrQRjf25$ZGs))DRB0;RQP^rBKc;n3qutw#5|AS#wu)G9NgO$xoJ5YOCBVTKVs@nEBJe7~m{CJxaF=0votH)o zeBZPc?$D1Rg6U1b+R}zxk#r(bc~)?)&H*ulZ8fx2SqI1G$B{8##LvxE zRxczvK986MYqvnVqImdq%Qh04dj(j_EF#L8(WKpa4qQhQu|B&To=#p5Zw4+S)*GLJ zlsgET`#V8{A8zFE_@^i7RD^^8INrT!(Rykc);3H!Xoh2g69_n>s;I^fu@h9uGh5PaxZ`{RTO!)XBD^ za>TyjFGz7)C1w}b;>C08@s)Fbz|RdJeOyS&7p#MkWA(A!!ARylQRN1<6>_Yjl#5%14Y^FFkU7x;SQKN6$FO?v zK)D_Ka>k83boQpu@%cfmL%RU)ibAo&^Xz~a3G$PIdcb3T2-B~$l-K0F%4_eA1xTk>(M!a63& zOu*gsxCrbLLb-wGT%DYTCw7uIhI?aVp=P-PNz0juPkTl&C7}VpA>kj>njnGA|2-7w zZj=zL!~NVnxTnT%ML+jn+HRNivK%LKI$7{vXEj*ap8)=|jKi}A?=d+ea^z0tD6;AD zbFlQ>X->~A4CJWn1*4RYW8F7(%$&rZIKDdzhxbasi!rakPpzw9VeA!9f?B!7MLt}6 zcp`pt_Z^m!{||VmBn!HZN|G#_Zv3|3D_9rZ1m4X)4;q?#K>L~$JZgEI*yZ8`yd$48 z5+nu46fDG3Ob^xV)_l#iJlza#EiA=L`@Qk}l^3{$*Q7yV?re~hzaI1@Jp@e+L7X_m zisllfj3K+9+f25&@w$yX6DHz?zlJ}wiSU;YO#``L=QQ|q~g4Fk;ee?oBo zekk{|pixkzTE}#n%8=XR2C>m4HCPsLgz>vYn7fTLKpbDlg+>(O^63(!=-5TE9p-p15}Xwl2XxGW z@e?Hlu(3^+$@?-~cQ!;G`wc08IYw*o&gSpj!IHE1QAaV}9VP|#?>!8xYn#A~vR1IR z)dN4;s?J5~R)Cs+r-9>$J$U;)OOW_7N6gr*!tLcpL5aLMsJ^}noL%}5=nUP&rtN?6 zlAKRCO!fulR1?@+AGtiha$%a_`44xnzROn7<=Vu|lTg8@FIs_7wNrsX(M90o8 zypV?5Q`$kQ_W+m!j$_rW+IZyuti*yWToDero6Bs$o2s*5{KscN)7Fse?yuz>-#cTI z1&Z9f#WTRkJ|K9YABK+X@62%lw$ zA5+S>SnsY2&G7(MDs|j*2}ZDtTg+@2c_eT+_7;cP|HIjTd_bd)H8Az?fYx@wFl6gw zVtR8NyltyRzBF3kXEo=+YZo1`t4EcrFPlze7pcPmvBNxF>;-LLClaN?FTlh7HI^_s zi$gvrlI;rRpt)O_To^SGzBQQ&!Ch-2>fQv-j5rciM@OkRWaB10YlzrOoY|>B{*!9 z8%$s2373qvCx1fi#LPZzXe;3k*G+UF^MAhpyDcXXPrV=#e#8;_cVEJq9g~Rvo@u0T zh9i_d#gbdY6G+fOL(95nNs-2O1N4xZN9%;rsXHiC@4)u&AIN zXQ~U-t0=ut<_<0 znIZY+yM$>QEEi*9E-;ykF5*`aEL_*R0B$dwN^aKp!JZuxFykyiMlCx47HDQ*2l$cG zaSvl;3WM=AnU9=$ybDm@7KBxY_F}n^dDyQ14%pwR57&!7SJhRZG`A1ue|LfBugr$-%O;Tw4K1kn!H)E;o=q&|Y+-YlGtu&~0H5E>z{UPE z@g#{@Fy_w*VC2SwH3#+Z_g)T24O{}7D;j|~7KL}+c+Q-T8e5mlti#O*kAW?|QaI#& z8Cbc?3DibC!j}t+!7c|qZg!RcYo52jxwHu1(PN;q*-YrTWg0o`8VEhV%8=+q6UZ{p zVPKhi3D=;vpnZolG)!y6sw<74(>ZryKXw{X;Q(x`c#WrB_zm_pL&y{uk&+);(5uo~ zkeYk|jKsTQWydP8-fs*H(({Fr&O#!?OoVGfk1`(Li?G&V73^;kz|@`0b)8U1kh`vxR`i@ab5ILl2MQLa;aww?17bh~7A`M^ZWWiW3;Ac-rQ3OoPy z0ByDd%n$j4B_|&OSJGr5(=7v!cZ%7fOOAp0166nw?Z)qmzJg^LXK>uc4@pD8OLbWCX$05@>Jq(k`QT@u zI&qTHB#wdlFlea=M_g6|&YDX2P3v{A^nMQBH^B(3ZIK0w4<_NehGW4-F@y46kO<#v zREMQX*?8$@D^m7Mm2`^F5`xhQ*l^hkU{KKyR>Y;_9hH%sj)@k5Kb1*;tOcBTW)^ln zY7f^vw}8DT?8(!^-oW&4JMNg+&pB7#2WPT_@cPpw*xmdU-hBQqFckCEf3U?^>+#DP zd#82IE0)dy4q@Y<_f{#G8ze=}j@)Hh#$T=TuC-v=+Kq9Z<0;@g#Bt>v^T3#%L|iy@ z8OX=Afi|fF;FM7{j)`8+WWv|rcbB<2Bc0$>%c<@%C5paMKY7{G>+(PF5ca<0Y(!cFk?@rYR4b%cbA~tI=S0 z=`F^6ym&v(G$aS2^dU{_z|+ZFFeknbm?t&jA^$BPH${$wDd`b66(hJYP+a?TeZ-p_ zhH(rt43@~Y;yZ3Hv8m}ZyvMr$j2Nq6$t5E0-iD*xvfzHk_(Ngc;Kcjj+oI1PtmY9u z@0bLZ-ung?PI(PpS<8^dZ(G56``;ilLjwjLv?7D^Zh>F#zk{>oDq;?(3XuqV3C=Vt z5PMk#BB^K$1LtJpFAL=1VjDBDC&7{kE0v(mBUzFue2wpH`2lu~If(<_-vSH0_k*$Z zg*fB04oDS0pAJf)%(I5JI*qum+)bSd#>4U$FpIp7kHmimeT51n^xZJlRZ@X}?yta) zV)Ws8^C@sjmkU{wGXxrx^VxmXFjx_%S;3&m{Aq95Rs1Hjw9`Vvmd zO1!Rx@X4!wJhW*H^f_b;;bCz12I6Rh6l{%i z2Xj2G;8*<$qe9{T-kWeRG=Lwi_*9XF3Yp7{Cos3&J zz&*dV6-WJu#tsv20n@E5_|M{H;MDqfoc}`z{O|dZb?%nLc+Gb#d0q$Rn`En#fgvQd0@? zA|#2qMeh9guvd=mfrohq0ucJ5{Q$P!akYp5t0jz4poaCK@6=zS54@kKc@ zx8X3Or&n7WSv(Qml#n8YPZr>VqBN#JW;FEXb_hz86v;SSIf(Ab6UP~i*x-vcEYW*E$9+k=G;$zZ;fFVirS z1#6o0nXU6|p`DUFED!M_OwC!G_O3usb6p=SHTqb0a)tmHd=&c#+7DtUm3DA+1Irm( zD8j(e9UvwDDz2Y(uCBpd3555Q#Hm$k$G57_-^|bR@sS8Kr8><>h-hncw5# z>e9(D*;NmIjFce>b7hI|IHS7eZR#Xj?**8)F@S8U6chAPjx%qso8#G+*5N!)2`KJF z09}WRSaDF1>!n>_eaCCy-z!NrgoSh7t4=W+!*rQ56_j~vnT?f|L^x(B44;U-3FbzO zVUm9ifi;`#Fmpr`WV`?bM_B+oWm{eRY_%E#*=n5TElmc}e1Sm>c7vg#GeG380sM{0VD_K%#dpek@Wa2UAWTLH z-?*?F-yp+SbL)N5RSAsb97(nwBU+zM#JQR|@b;cGioSU(bNZ-o>L*dOh zBke3m|L9I`+^~U;r}lukfo=Gfi8isEq(+<%MPSdiG|)2N9G(ug2bZtB0xz$9#8Y)B zP&d&aN6wbu<7bM1YGWz5q%H;jMt=mBcB;f^>@mjmeIR%;Ism@!wm?Clh=?AZ@| zf0U}2l`nRx-UNacsX-j-HI`H6|GC z_x9ebyZHGHjtV-7r?|D(e(F35;Hz%DY3dJbJ6{=|>%D+eZXUzd4_|XKFE#Po%N1bt zat}OOIfUy-Oyw%4Wq@9nqX6970&G6jFjguAPuMdS{@ZE@+vUZ)N^ z4;gJ6IJrrQF!IsByh{qYD44_5KcvX4L`C2_ssJ2yj{_Ix^>I#)5?IpV9=@NOhWG2A z0B`K|xV~YAyT9K7kGrmbrx%#k&G2dhQ@R#rvl5)zyaL1-Xv2X~j_^aY9yunVB<`PF75IPGsOvfYUhrvkN!{pbb5J|; z8nbv?CbMUZB=g~$18it;gErD8#Hil}tm;u97WLL-zwH-LeC7nUk)HwXo7@C#8$5B* za5U~UP=p10mBa=0H%u4Ca1kNhV1chPoYnXU2WK5;b}X=i6^Xv^qKP3=@eL;8-zHDL8El-tgF7;7Jh?F5ne1`UgO+<$@sybc(89+BUe!}2 zeXA}oj;9Oojm8E{daFUsm2&JU^MHvK!xjx!m0?!w(tKSl$xM2@316Qb4$cV!>+Xrq z%i5J=iQ@5pVAbTuoTAw8r0!%)cIo{D<#(QN(LNb~__u(@%o-eY@BovTa~m7nypN}= zUIky4er38MPT|G_=ke80u}91~MlgEqPmuASxF$%vg8%dl)r}5bzy()sq_q#?uvO_eQ279U zD4~q2_4eW0y%oUob~LkOvKpA=I2S94UH1_M6_|BU$7UNA;A3@9xXFAjSX-V6zKGiK zuhtl5I`GGO2feZK_OBqKnlh%wOM%*}S4J6*gNNe z*v)s{SvThyxBpotej_ac!(KkX=VK0cwSF|Cuy_Fudp?JoUC;tfE-bCv>7xt{wxxkK zmDBiidKR9f5slZInPZR3tANx&BOLH}xNhOgV*Db-93Pyjhohdz!Qm^l%$yI`89}W( zQ{y&_%Qy1eFW=3e>gR26%32C)-_-+Y_4{zfZyhXgzYKi1L-9W72RWrYh8G>eSa&gy z>b` zJaC2qJ?C&^$7x`5aUPR(jm7IVk6@e7F(l-@CRShiqpo?_9E`5K4QzKqPXC;mAog9o zKz`Fx?sT9Z*PWyaH2-SjbW4CQ2+Xj|=H1|5#$>R5g$#`Sz6+FRUB@0X_u%!HW#Qvb z7_7c<3Q&H++T<5T-;SXkUDb%*qd_?>`mDMHjQYIK&yRt zk&ghTWK@E~<9hM)AGuimLjtZGcEHm!Rsor;tDq<6Dj3+Z9@Qj=-mgw9 z*^W1km}1*NW#Bm2h^rSZ$3$^2h&+4&{Eq($%rewr`sYz3_pTi%Wqt59gInA;_cY)v zc4g0ct_>)NzSo_+n~EojohsT(WI0cVliZ=Z`@ofxO(5yv0}vb4hgCdeK-}#v z{LftMSxsyL`*)Wzm*Y-@Q1%I!>hK7cAN(oUr*j$@S9gMWOKR`}iC}OtqZ|ZfK(2jiX7J2wjB#2Td=u)G1%?f zJaNhaM5%u?Jdk=BgIlhUy*e2ZB?GdInGZ~+U&n%%W%!A02ncA8#2=F%F`FbV z3-VVTXRdCK#Ip4-K;D1SaP*%)*sb6P6Wn1!g68Ou=evwx>FwJ!;LS-;9dizhew~Qh zQ-c`~)p2mGnH}s%F(>jx?p*3I3-WfMB~c&NhDrObyR6o+C6P9kBa*pr1njo}L;6R^lal5}o-h>OCngF2JnT+l@+n7p|U{4kd# z9)=u6;+6(1q%!vzJb^^wLrOJ0v5~WxC`+G zAY;N8CL!uLxBueRDu6i zeE_vc8~R;VA{)*dg3}Vt#G=KTywAseG>OjmQBH<48GvMeSfjD znj|#M_vZ#b?Z9E{j$lWxR*)w?{~E{1lZePMBxy;OCw46oK=FdGvrA8 zH9e-bsDVonJ6EH#T)~egmN+3W6vX&W203~a_+2Rn{;atIF0G%*T|01$n>B5ZAgUIF zc)#T!Ft-hNzr6s4tBs-OEnE1&z=$-*O2Td#3vxb9k6d4B4U;!i;)R;>B&xmxR~;pw zW@0NB@A(usEk|Pi)5(;{UorM~M{5T0+kTO|t#tMkXlwD9&0RSZ5Y9U0kPn!_RjS^nT|} zW^*sV=W$|}!u297rS%9z+0|q5vz)=HxJ{L>n7lMv*#`fH*0a}A(k9kVMN$6JLqy~5*`zp z4ARcq1DlUCu?*G#ABh%u_$w7l{Otv{qxWEsR&%)ViX8O0B3{=)FTwrBV8R5A;FMJ} zVfK@wV)v38oGv*A-s^KG-}8O|^Qp_2`0#XY2On0KQIUp~p2{$lnv0nmZp-VWqt0;d ziEr_f01^Ikel!fNIEt6`Pl4Yj3;<0jCnDG_1@Apkh6hB-u-|I{*M%nnqigzjh4(h_ z_>vW8FsYrB%KQ#0e?0}g>as*>q8zTVxQH)gF2wKQ7I4N1<0~IrVf=Vi=)PQ=2#flF zue29bUGoeSHVKK$qcm{ivIm*z{tJt`^r6y@>-cVl6DjNXgx4N1f@QaxaN=tfPW`tX zQ2%WW&c#D6nI;3ZhZ{l9sf9TAl{#*|Eg)fc$B~_V9`F(?LuPn4;61nZ;h>F5Fxcf3 z?y0!Nc}*^5Wt zHewsp8t}V$qw~~>K6s7b6u8zvnD49e!I58M@xPudj1F~zDzg&&$0ZOJ&r^Xtg96eu zHv#kp{lU9v0hT+v50ss`gRkc#;2A@joLT5P#^f~sAAMc}2eB0(*XAj{JwJ%6Sa=N> z5A6kGe|6%cfmcE8hjQQ~QUVkDqwznF(ZE4B6OWob0e?lUAlrm;*1I=j4Lbo|lTr&l z?MuX01iQhIav#{ioW^ITj4%PdENl!O1OJ;inka1OW!Se9VVe60NZB%x;P(Z9pecF(m@yqaje&-xubtpGkI4E*0#2vy|u;`I2c;Auu~{8hC!! zhujxG4<}Mg;L#KMV8!L*;NC+WCb@H$*roWK3v8PJs}2=|!R4>;@|hCk_ybL7n5YMD z|Lek@(-$&I>-FJvI-UK(^1 z`$tK@v@f#w#`XDRft5Gu_%;(7Z4UtYCSzg8TLt(*?HgV`kz=es20^o-sZjpQJYrwD z20Of1M*NP4l6CgWV5MJ{cpv=7Y0R+Xx?b9XoXTn3fY|*Ooa6v?9rVdv$vwF0X*d3o z)xvF8SPXP_E)(}+Y~V>F6WF)No?J?iX4L;y;60{Vpn1~){H<#UoDX5iV6zmd_@WHg zrD4!8e4SBya1!jB9fMz2crj-jUEw-wIT*!gl46rw{0BNh@P5ZTu(Ed$%-ht2 z-M425h9+~Mde^nO@I{XWaHt*Igm}W*6;`mg*Pevz8WeoVcP6vl_u)fsl^|?E1{mah zpprU(FE6>1Ge)VLr}K2IH0udgo_-sYT1yb^xiiVZI6yq(Al$gY2dv{}Kr^8b=EVp| zwvh-|+J-^m=?b@~*pZs;t(e*_AzD3tc2U+qPl#QI5p1+jxj_e7n$wI&=!(r};5AG~ToI{vJ| zkj9}QY0z&07jiy|RG3Gn-8wsD{GQX-7et^C&QXw>SgxLWgTBI)<5* zDY!N<52xCOgOQ5SKx%d>1NAR5{M~rwfzk`+lU0*I%bQ~kYu;d%hfZN0tN6mZEB*ps zXJ_*4ExnXS&hILCL|xS*vMSo~KC_(n;BuOtaR zpA+IA$C7Z!L z8U#O`J!4nk9=a6Xd>I5M8!RXPRwdxm3HD;AxOlEqqeor^@E* zorybDUmJ$ak|A?t$ds9)h`rX?3k`Sy`P>cSPO1@|Zyz0@&P#k|3BajS zpNg24XUs>-Oyg&W2K zi|M6!ZY#sB@N_3H3e`z$j2qOVElk>Fkx!cahx3})adNX}+)wN?l%X101 z+IoO7`Vyo?l_9GM2T>$T(3x5N11pc*U; z)POA?PUDP{zudlj(f!ofUL17m83-_&3dF{31CIihgJTQEk+_$qz&g=gc-ZzH@KdD{ zS8sUBwXM-78o3%leDo`hCKQy0S219JEU@C1T6nvF?!Qeo9XCw`H3_WAQA^ZSMeL%g=)ayMN<}^5gN0HM_VYsZmT? zSt#>iK?k>XwIUdH+`_H9Xoc_8lz_ur2_q|~!2SLy@@oe&%jb}HYwcj&j#`dArwwi0mEp}(8sz7w z5iWmr0vj8@#oJPgL5znBw%NQG_E$N;Y-tzr@_Z_HsWccCqPg(lN_Vo$PY zZsgNtH@H(-0u;>|L!OOD63NSK+Nb0zf*CBJ@n~hI7pV<}o{)ZDdUMm<}jZgwh4tatbr`ws&w(@vXbjNNn z;zrI#PbLtEGHZ!Qu*F=&bM^!f@_i2U&)CAGC9fdP)eFg!ar5DvPd}Na#|iscBNq8) zjyiF&@O-lb)J(XEeGa?@TDug%nz9sTW9NG={&fvbI5QpQ8tTAfK9=O;0b5M>+QOEn zF7Tkmc+#^hjJs1UPu7FCxVHBmsC`ny?8=!0#qVjuEU_PW$w^62r!yPoOi_ia{xc{0 zHurP3;{L=qY!*o$?*xzT$pU8`=fLWJ+Hl8GGt#kD8hGvDfHrXeTPtI6&&<8J@5Xlg z%2OLZpJLW%zW#4 zD_lv*$7#gxuP#iPun%{5&Vjx*P7vNQCc``Xn9qq1!1dSzjPaZXE`C6iG0OVjyt}{f z*t-vbac3qDlUR=HJp6I7zX$jX-r>ajCSbAT2ta{XvGX_sraI&d2>z=8SpBnjaKs$Q z{r3oXy&D04fB(SUgI~cqvYU&(osXxPJOtOx9EkI~Z5M?(&zbUgOU$umWY+0H&bh-E zJ11pv!(#b(YL5YIS$`08#x>%f`)wJ?!sl2vCIeeoWq}mwJcsdhC&6z+4QQdRM%IKH zLd&_5P(nc$Ce)~tZ{p@4>dkwscS@9<>hyvq&K)57=x=b|PZb7|iDc-i7<6~sjK9Q) zI90##F!sGBDYCx|ey>!9>6^64GC5sH-%ke5mzMzB*c14QX%~M z7eF(s9~b0&1`@W_c<04(Y?%KTn174~p(RaVQ?vs}@3zNb!d%=c^8{Q!UCMZLZelY3 znS$*XOSnrbYBB$~3!ERTOfLS?z^eC7;tZWHpg&2ONVRm}$F5_D^F1Z#XeLRnBsc?+oG?sSw1U#>rz^|T@ zNP9*jUfVksUWqV-VH(op=Ds2@HSZzzuU90QRhls6g$CJY9p5-)lZS(UTnC2g7DTm2 zf&^`PfFIUn0_D$R;e@$wa8~kd+;I6NFkW4Q$87ds%+4*xLu@g?dD{4H>keEz{s-PC z;ssi2^ zz4DlU^%x(xItt3}st~=UZD8;IVG#K8AHJ>b0t&yU;y-`Bgq6c&MDHMb>wjLfyqXvDMlF=I8Tj$Ef$# zKrBZS-hHkGjb)6<6q^U&!-gkd)+8xVFPV#{!RQGizkAy!Iv1fOT zp;Ols@?Lf_kk7lrG;PiU_bw-58_jm0Hjv4cYgAzGzq%qQx0%ycoC;Dxb3tI35bu8T z3T!+jM&d@~iA9hh%x{t+`bB5CZ^L&QJ;Xb>eKPuxv}(Yd(c{c(I>J@_Sc%tJ9Rp%X zuRw)~GSL^gg>U0u;O^>f5IeCIC!br!?U=q3%WXS{J3sye+50CDtrQI+W&HyOrwsv_ z8SgL7UmgLQljgyRtCx_OMX$I~@s%)!^Ml4+G32YO5EHS9WP;*tthM47NJ%OI$?+IO zKD_F9xvmsX(aXTxXAui~6_T^Zd|{_10naM|jM*y>Rb>sxn;2z$xW$*)MO%@opAc@h zI0&LdXLYxCnndBZ7j!cm0)002F!)CV_#UA`#NJ4dh;s&HSMLPkvyOp9^A<7I9m~i< z4|g);5CA6?uK|LW3&=TWLUh%9;hADFXyeue8crfO@?sDR&4air-@|~qfdYK?vH@3h zTxaf1nMp#G z)g*Q?N>Dx^3%DK+f+O*Eur_rW89S+rbISLE5>k5b+_`C_sND!pbhRQ%Pug%|%w!lb zqX6ieyO6|SMdGl>5ZclO?tb{JFBSG0@I02oC8wl98nc!K!PK!R~mu%fYYjiLfKV}NaytWj&^ih!TEEaCg@`lysk>uKji@4J) z59B>E1a67pT**0OJXDnorGG4iZQK^Jx4RJE*8UGh*D1n_I|QV-W(EGUel6*>up{eV z#z1*v3{0H2K*jB$@J8tt;$<|3ybqEhmK!c%bW_AFS`TmoJ|^((Ej2jvj1}?Oc$tg% z90k`U8HzZ#xdb^j;-FGXqUSQ1yzHJ0S51R>_Nyr{WARiNx88~v-&4ZoYT1njMX6x@ ztOIz=yclpO`4}))_zlea&)`{~<#2rKHLPb8&)oPc-B@qe3%*`AhF>0pa$BaXB@ zwCK4xVb`dqeiIuk7M&kc#(?#r{zli4P0W@l9ZYtb1oLg{LeL35GSkN?K^f(-U^G$> zXJ|{1S=Q|!%)y?CZqXBQ&>qqEVz`wOSO=p@%;X2~tKECIq--oRNk8(V}ckj%jm z5Hxa(>zy=>VZ!d?j`$2Pot|Py{YGqD+QEg^UjwH$KVVG#t-&|J7)%Nk$ci*gW;o_C zSO`YIngiV+vqBLo8n}Te+NZFZL>6~sM+p;}x``7x8$tH_0cMM=0rrk50vX@u;;7+# zZk6*4ZlI`(yZC(~Hhho;JWooKR&gQF>Ry1q-BRWJ^p7&8ot>a8cLF$5?*ao3mp2M$ zZ(uI3X~Yjhy~(@Pi}05RY0OOZ_u%`WD}Gn)j<-(z$#`!?j;DMpfty|*E}c{YX1OiE zsbQ&(T{HR}rTmk*t;2cL94$lW}}{ zwBOMo@&{uuivjPXyMV&3?~aUW22lMLiGR=72s(AgfXha!vCjlGd@BAuh`nZpw_nf$ z)iXIP``|1#Uu^}XW5)q`r}tn}e-HSO5CHl_e0I}Gf84FF1VvhJF1qCk*Q8O3%S##H zm)VBx_GSVzU5+`vsT|A-RCJsqp@g5>1za?*e}`Q+EC=$-syX!c8j!o<4J7Th;Xs8d z$Mv7m85_;}z;tX0mR@sQ<}R~rx9GQ<`-N>ddEi+#5sOxwfM?$GKxJkr_nhqkD^~pCJVGzwjpp@S z$lDC=l3fY-?5PUum7g-xR+WJSrQP`cNe0^TTe&Bz`W&R>Z{Xi0KSVvGR>z7qJuX|d z8|*Gmz@foLIQCK+u=G5|Vd!ECgi7d`4&E67Zu;4##9mVWS;m@U`;~xadYh zTx0F)Xx@7sdG4lAvwO+72SQpS^vA8?4 zvoY+QA9wBQb8NB0hN#7`utiIT%#c3;?AIs&!`J5;rIr`r%8{+iObZ=onQu+Xx5>fh z@23%y+-YQTqb8h^P=}MlSmJs8H~uc-j(l(hmS3$%mJjKZ56|tO*0V%>#$OXep1N}J zZOuL~BGraFmrjA3meqj#3**VBg?>0;)D^ySUkPWEHDqDBB{ce2$0?sCc<^c$*m+HY zWO!uYwW^35T+2Wci7}*KN)`&r)S%1+(Ve5+9$

i7b)0j(gnI;g_ACmzI`Z^g6cI|TfVBsF zN$Sl!5Z2!Y-ndDU|JGgueXe@2PN)NSxJi>(UXz=o5)4cft$-w#-MD{xCSH>@bfKq} zV{XX0fm=ga_}8I&Q1NCG_$=zxB{U1ce3Mq(*ds@JH;BWRR>yFlmH{+Y*aS6iN0J1h z3Y%730;k_tz@F7J$c8R!=>AHA+}4*NqAerffL$ioExOm&(3Bv(pV5OA0Jv$yk&MzeGcl;&u&3{uX^cQZIq-tatd)(dl?{x-N-2@(+wP zkR}S5)@1+JVA7Me5(;cANRk%IeKG6;k2aKGI-i2bU%?lCmtBY`X)W&o~NJPS%B<*Cvx_S7mrE@H9XRZQ-l)j-*7| z2&Tq}igp>Lz4}V5T!r$8!iHolVH^(6i1Rs6@BIoVH0qN%)|K0w8&nvvcI*EtCLTie@ z8s5hSZqrE0N`JV&Ly@Gk$-=PyDe#D%p(qE}g>JVJ$U`QMXzvPx#*1c?`JtsC>XIj? zp}}#AE@O!H%0Lp8JeTPIWT3{P-(2T90ZhL79za`Df`rasTKh%Em=p#0!e|DG3lL@I z9ZqCj{vvYjvokEIN5pB-af(aLolfNyuY;mu^tqJ`q$`9K>_4l;LBy{K1T4pE!8-JMa~C$;7*riIU+U zGg{pN4rum+q0}#!MDK0vj(&jE0!4jZk1=pn=1xwd>kPN_P!2YFr4IJZ-^SJNxd|*K zNaDM>gh?H(BFO2w}IJm~r~1lHBakaap@wQke>>!-;CzOYwzJ@InQw(?f@^=TZ1&I1>j?}5bTO*!tH-&W7(yZ z%pax=M1Cma#)R47AG7>$y5<}_Za5gc*S?4Ug&)PWs;1;|kvPo1<%S~?`+)e-kHDbl z13va)42bOw2fHdZ0pxlRPt%=*z5gr(F*X=nQVzi34}C!5j2mFNx*R-7q={GfIp*}# zgSetbf(-lp0njj$`H|oT)+_U1?5d0SQFT0*)P5O%T9Jw~Z=ME+$2{fYKQw~^)hFO$ zodiiYAL6deKZ~m-Uci+%UV>ozvyR`EmEoAd>o_>O8EDDoa4Peg@mmv&H8a~l+J-SW z+w?1to8Jef**(X%TYdmD>5V|V$Ow!nt-|4jDtPH0FZ?FR2*0n|4V<)3;wcWbn2No? zVangZ7^h->n+Z4@&7gsAT>Pb-IIlaeQO430>}w0gd`}QiO4_cm5nw9!%~8|D2#nqc z0sCc3@Qw7J+>Gh*V8P~Hc*ZSpys~ovZr4=h8X7c9%) zhT)P@{PTknbgPQMPb8IKqbSR%5ldo5JZ^9a?XHaxIkn))?$=Bfe~SCk*0HNmpIvm6g9Uj*WlP2d-wg}Al2A9Tlx?lg20L3`pIY<;~Js600YC2dFWn&ZjL zb@_h$zI# zoc7dYeDNy8Mdt>X6EW4y)x0w1#*)_#ZGQ{!)5U+8Hy2~U-L`Z*QeXr2KFn{FscB|% zzkUIy#awVprdQld#VRISG0hKWr4mY;b2v@=-u!Vac3dJ$gnVF~^| z?LJfYcPFz{`xO%~S`WH*eFD;IZyU>}F9GA%%Rmcd8(98+H75EEiwn>RPq)xzcI>K_(Z()AN4(^gt5RN=j0W!x_ z0H>SQF#c~EkkC^Eb@^|>nQ7xhzlZf8M3iy%PWpSX$&Ll8|01#GD=DzgJr(G0Qi4wF zj^X@-E-)$~33Dqt!1SQm*fgsYaKB~Xm8>K9h|xx_XZc$PwT(rhyg&?=NG0I~0nb4H z#2K)=bd>q~Vjj2}CPN;2eBy@Y&mx;xKSoU35dX8%0B?0Ipuv_L$0BeR$ow4-M~%91 zq3C>3aHo`O*=~vds9!+mQ-yu<+Kp0kUaJh&Fn-I!S!hDeI<77(s%&ADwnjBEB7 zFul04vGnRbyfpe17TSnr!G!7HO~^OLSamUS(P0=1L-*jFHo)8N22Gan+yi4TJg>451nGVP3L{~}!^)dX%=3lk1BBS{ z$(6=S6}v!Uei?H|S%T{a4&YK%COH1~BWHW!3zIZAfUKFnfb2|UV7SX4$3|IuICd+9 zZ(Ja$^E7r8Gv5ikOBaAgR_U1CSI+#o>_ybY=aM8v09)$=vD~R%@b~&Jpuejed$gbU zf8PMkd+Y|y+g(VwZU$5FuL&enWrMbYY;5#X9dvs+lH55i?OgQXE5d^Epfua4O#Ao$Fa;z>0x;_@el5i~u%5?*gRtIpU?lmSR zdsmp4AV7ByKTT{$~qV+8<8L&o3Y^ zJm$fJkJRy2Lj=R$nZi>)G)c;XdPiO_nrt&zN`&#tp=v4 zg5qG@$RfBXCxHCgr3uFDg>Z0;BV0euoZ!(^FkAU zvm#0PXbPC?R0<4xuK_9A<2c<%3>fR-1|kk`6oCeeY#5_J$K9t^bJUcweY=SG4H-Y7{=Z zC>uX(*amXE2AExmpRkmr7%{C-fD*Skyxrb~cmy-V@z*pceQ+iB^H2nAPz->JL&Hf{ z^BM5;fGF3HR3S2p?|@zHevJ8>5ZH9t7j8P_PgtoJ%vsG9WX0MgWY@}2_^mF=88wsV(hrw-+ zmyy^^Z}44NlQjKjO>{+koMF{(W_@J@@%kG`G>$HY5l=+>tY!wn+tPthy5658^fAn{ z{>4yN(hKS)x|5*$U0}tKACZjkCJ$cth@OEpxINL1_zyc0A1@Zh$*;mYMf3e1_X3c$ zGZ{w*YXYHMH#VN%iIZy2g7|1Z-0~n8Ryg>=vzGou<>3iXJw*;$Nl%9Rj_DDr&_0mh zsztQ*$C7nd|9~HPe;S`9{|8m)v(PNri&!zMz@?o6Tre7qJ6|}1%rjdVrS@eaPTUXH z*L#o_{U^-YdLNEh8H1p@!`%88FTq{~89e83D%U9d!5u$z3IEk{hGUO!1dkf1sN1*; z@39!gxyJ2{0pCx9;Uns>BGe5^&WQp}o11X)JRKa0>hY_t2}JGLALd`*dT>9z7r)hA zfiuexcya3#HpqDZ%ntkqo*9;6_wq`7^z~%Yd!Yb#-rozP%|0^K%`#+{U^6$3USOUG zXEGA1pKyuyaHEyj21ZIf3MaIz22yq<%(cOB;36sI&{}^y*S8q^_x|AIQlvPOIww** zFoqnsG!c?1lR%w)E;GJD4UA4ag?m)T;(@#g>#2&$djb30YQv<3+Ol(cSEGBQHo)aZ0SpL^aQ3qoV)7rF$TV!$tto47+ z6+9F5KZ92S{ae|1<#HKPGHWBJwEHk9^;*K{*-yYNO%6t079&P?UU6l)YZ5EaH@YkvtWBJ-dVPfn>#ceR9;#m z;$5a(fcy#0L)s0n?`MOWj9_ljw{buapbx+7Xa;B49XR~WJN(z#6t2(811G17x-=Qr z@ZL59PF-s;=ePEo;(i*7P9yVD&eYbI5Q1fq8L|C zYjSVqaU2;^2rg8(gZyU=+`#TL_~M(pOoP@(5wG$Gj29MzgL-#y^s*v+>QoJAY?c7# zdh+<_wKv>+8&TgvuYl3qA%@+0&fo*77Vz!rSSGwO1y4Pk%{2_kHsA&+=ZN+A@d&j4rUdf)-GL z(kBa zG&@>|c00=O#5b0G&FrU{r|$_bF%Ri*Pb1xCt%LH5>)6qT2s-ZQ9o{U;5xvjTrhOY_ z=%Mx$WFccl^N;ik#iwh~q1Hjc_Lk4=81Gd2%;_T?Yg5Z|ez#~|g*vLvET^qraj0Os z3%ftDlD$2t1>KfaL4p8#-dR40Zv#W@Pw5iGu8l)Y$2{3T%bf6j=LG(s%mjMX+l$Uf z3gdSt7bA;>%1$~#S6P|Xus$O{y^Vm zKK%PXzERVYYX7-OZ|Hf_9sC9UBtC{V4<1Lt!yW9GxUu}L<0|w`P%4TwDHp1&??)^4 z5WdY{ieB=~U>#R(7EF_oWz z4)fFOB-6M%c^Vesz{YEf_Hgf$qvH?FLb4tyNOn~nmB9hXPw61rd^eJR8B$B_ z@-pa8^FidGu?KC)dJVh+HnSP_0L_ds5cZnY(-`kAG^8*c9h4P$VXt+vf7$(f$LYs( zvWW)0kbj1?oSuc4&MH(r??;iUE_3tjDH`Y|M1LG{HI^?L7bTulgzr?#-K1q9LOk+1Dd=wm&cqQ1K z)GWx(W!dXhsVMo`Eb0-rSMWjAhJH{fM=!0SkPUZ46^>P-b;|c?^Ep`>rfq@( z4~}!8NBC|5~3xv-f)A$u$ua?1*6h1%}aC`^1p^hm)+#*G!bJ9L=uu zD5PiH=Ayb~>*&PWiS+y0Oxm$5m7Oy8KYsa|WVu+D|L^e_!K}pivJ!6YVXY*I>i?r95T@ zUt8!JjXJbgQ;sfwdztrLe+aE^n#ty5tz&o3S8bYAf0os6HAdGr@$A62P4r4b3mfx# zy1*pl0l)0)IliPuin`0xQzMZZ_nE_C`YyaSG*1Kc| zw~rR+Pe)6a=L$A-x}b;(WmFxf(#0;iB;a~g8_szW6U*YLg?XXxGq;b_g@Wc2J{Fl%$p z3_UJ6hwcx@qB4nd{1%5+HfzAJY3ZQ`6liMMBqt`MD^0edDX&yfxP3lp%Y4c9wKY*E zwhB$x$UvDfi+L&avnbADi^%mF$G$7;WG$!UfQ-AvXymg8>yUkkCRsS34^_K)#aT~i zQ%j{#UbG8Z?NmLhd+jo9{Ah`6r`=+oRc;V2H7w(2$S-6YM0?XEJ*Y6wMAj+!>K^{? zdx@sNWg>6Si8}Vl1psAhTlll)d+3a)o2>CPrKT4?)+lj{1PXjKpDt|c<^Af*XyA?U z^uyNy^iORQ8VZ_@{8^=@;``^xosldgUGarCES6<&jo1pyrrNMGS7_K51$x~=7$|)zbZeazb*{0@9+1Z2afe@-M@S2tY0kM*&%X@Wd7i9Tc73k z<(xq3jl22j^})1g`8X$wgz5a6?ZN1p_%sB&W}!EM<4|rQMSsoHcRC&Iz|NlY zQBdCOjUN3?q-$or6KWJ!v0A1EPHy*8QM|<*x@P@El)Ox)DFa3ed^blsT{!4~!v1cj zm(CBN@OjTDpVEV-^jg#Z%P+L8U7Xj{9GOa4okQqk6{a6Qlp~eM47%RdjP5L5NvqDgqY0J&(7*c! z>4a`S)YvD3d=E#^keWmE-H8x3Zb1?1;cf_{!EWKyaZNN&aXd2nH%xt`rXs`5F}#X% z6tXk*K%Ox#>14Ij=+A6Q1p)N}CT11C=29}9q*R5@+~0s^El8n}En}&=>}US^guncq zwp2Rs$PqOzt3fyB$MUXC;`DQS1`=gl>{^;j?_zByjqy(Wv@w8fjaW#V@7waxA2nG+ zo4u%P?+!u4hmXi}ZJ=Q3vU>LU?Q>{l+5_4?ya!!)dzF_n%0-XTlxWr5aK5cd0d;QC zr864~(SekQ{Qj1FzW?8R^f`OGV9dnNf{TmO`P`ewSu2Y}NLl;<`*-99@*2L3CTvZj zl>=G=enKIgeff|89qSd&dEY?g&7)b5C>g%UOCrGH`5udfHtuK1;N>Ev>1u<|nf@Z+%HsoW>g_v$lU zSg(gl4$08bnYPHLNW+Qq+lSs~Li)1U9vxB9r@r5mkw@JEWZ$gJUuZ93L)Pe`chN$2 zZ{;{T`I!Zpb!Z*>cwgEHkI$f$OJ&$EgM_NsxgbeK330`T*`Ze|Z0TwxbXDi7&}9D> zdT*sM|NQY`T6WB(X-TPg(}(B3c{iIT1g~nq88UGbnP4)CxyZSR#{8*oQE}1EwB)+-r0=aaV9kGQZ!no^ANo-sY5D(l1&ZY9jVNU z(|ldzeZH_W0Oj9KpvxN~P{;)%-nFU&*`o}0ZdfUQpR;1eofzTge|be$9Y~;|bSv9> zy^rU7)}zj+6X?O=2ke0RCSlv!Y_zTE1>d&%6q^5B-sy7MMEd7JKmTai04!}kL@(!T zKtE>er8A`p=(Gtu@5=|#?&!TpgqsVes4q`AvmiE(?*t7$^S0 z#6rF=Bo#^Ck`=jtkFp7ainQkTHS|VvAF{T-KpX!!Aw9XxZ0@&PJZbJ_L!Qaf_C*}8 zX7Nrq_uhP-Rk@B-O#^AzgG%%@QjK3BQAQ6Q&SK9_`$bE~J)xgpP`|S{T`J4y`dxX3b@e)6yYJwsm41?U?tAj^TDwgD!WZb4b|ZBjyvHU5oIsj8%lX2|vQF-m2rb(E zgMZ~GgYJhkv##$oY1LYjrg#4~qgrMUinuMtzfGQrrtLh7t`=&sOFwwheZkuhb9O$f zw9yNlJO7XWy1zna$EyJ`%^%c83+ZtUNN@q(Y z9?HP;L7jVcT~P{?GkJLVp=Kkv=Pl*vVCW?D)cQSbl{cefHMdb0#Ul2%g%lc{JIwbK`SFhji}(jOHVTa16w`iT zI+_?VD9CTm<1M{!A(VX{Egt8E){m=2W=lr-Lyf!G{PVYIy6r>u3iFHZ{w+^0!)L74 z#hHA})K4UO%xaXYI+G91p2E*hsY0Kh&fo=lUr^GsKHm27EdF2Iek%6g4ZfFoMR(6p z=O=ye<{eT5NME9wa&z0!xumbOdhie%bT5h?*>#Uj75Nj6MLIj za35{Dq)xOpzGcI=*&>VhB4pZKE%@B=ov%zeP3u1`Wg|Ynq{WX`BLAu=H0IJo-ppV# zN<~Jf`?|bS_LwNz)NhQAN$sTt_Er=Zj?(*~Nho^BEHv9)f-X60z?(mcqt$gf=$L1} zpv-R^byuiDk6J6Kt^QwrN%m*{cVs*7HsL;7dFD9k{jr3v?9N1aItSU@=?_uVRBygJ z-50H(_xMTsMgF3LdnsMyi6ly{v(G0D!ow$X=)~*NXz;}>+PCJTaMhfNO+~T=yt0ZW zyY|9NLH5kI^pb@Zdu4PAmA!^p^_GR`=H2Bmx0k0~@2;ZoyLb85>MQxQY2)b0NDuz? zdn06}_nO_Io5|}L&xTzEZE!Tbkv_fEkAlW0vCEA`bH_z#!F;nj>_=T`K}_KYFYT!3 zbob}fCX4fPXJ#V#J+UZs zI7_%dOqtd8)uWr%hO_SqMCVI;d$hQI6n!BMsPpqYU8%120WfvX`XU%Lh)Y>V4VSCZ8o%m{TY+Ew?kOMC za*i79-;Fd(0Xw}S1huU{LH$afu}1@RsrZX3x_+Z1Z&Q;b@-YL*+ z>e#-6rmtwC&8s%^0@3GFqx^e*I=XN`9LFcROu(%6R0$Z&&^(`P?( z^5C42;G-VPE;ZXgb)B23o3M~7f15`ayG*9RmQUb4H&cFQxTBL{=oEfch9s-HT8(ZB zIf>@HY+{`@J!j8=#lZranc0R`;NckXno^;lZ_ z`>Nod+zvY7$9y#WUCSxmsT}DS6r+-W)pSOE8!dF@*b#XajV^eNcAs-`!eTdgzhY11 zS2dd!=Wx`@f>MdfL^`kP2p_epg?;kQh#fJmp#Qa2A{DWBe3r-??pM8-N`I*q9<)-V zX=wy$URD)s2r=Vl7@t6Q@}3J--}%v|Uu|riXCA-Qp;KVSYp?}^cl;`ubX4=_JJoBD z>Sk;l>D zW0z^-@MD2XzAv2|dW7G(SIAm-DLP%4@(FF2Ih)R%P%B)1dA4xcMm@UCd?G73?*z&T z^x=QP61sHwB;1>OnJ!5=%|0++h*ITtirn4leCUyDY@7Zu%EhZuGItX#pUU(8t=mvt z;0?Zl_oHswke*g~fEJ}@vhADeX!@UTe29z;de(&ax9x-;e>@iL?6*K>H?jbFvTE1ES+#wqc>j%AtkRZY!K6KH^p#)%-}Bl5ojX)Xzl%Jh z9pk^EOO?Ff(8wBkOn(BpWg(i;2li9n;%&U->~xg-Oh6}#oV1^2SR#!Djx0!4r|geK zY~{jGHqYrh89QqZud}F-EnAjG<89B-G=o%>F~JKh^S;D~JhMZyo#WYl_3J2zcNMx^ zTZ!f@vZI6fyJ)_1A3e1;kzU-fmH+(JOAslW$wrNjp*LMq5OmEFKFU);xi*8WP4-o^ z^^`Z7JE4yc9(c^_G%BN3ziQF_Z!`Nv!dM`7UeoFAY&DuBnoU%OpVA-Y@hE3qq@ZDj z0MBg2TJfGpLv$`%@y;0Sw+f`;2lgVL&QrAYLuga^ z8b?;{{){HBFOEJQiJ@}0G|{1eks^?{KsVds$x2R?P9v$-V_~pv{rcZ!E0peXn?kjX+(9t`Si=^1Oc@_PS4Mu zL@iGXsG{pJRQeXepGkH6scnY5>(u>1@w#+Y)qFKtrL><`UE74bf1hXHemuc`Tcaj& zSZkn5%IDy0e_5(*W=kKs))6!DS?swd;k13>UH->yYn1#_231D2({1N>p;L?#UA?Yf z=qb07My$Jp{&PJ<>uy{?|DscokK}PSDZZPn%BbX9b9GQ^^KJfkS_vO``*0ckIB5E2!wlUiOq{3Nln&=yb%b zTTpV!zp1C|u+aSEIJ8veA39#SAEk6R(U%4jn*8=x(u}6>LZMy}?=@!$oLh4n?b&Wh zYom7YpWoW^hp`^NiE(BR_5~v?$0C$^|1MH#azHcsC)0hU|D)(i9BTTyXr(l0Qc@%( zRHjB$_r9~Av5ZkkNM#C@424KYb0N(XDWxcsjCJoj`_ZIm5D`+LUx|>68PfN@|KJX1 z*n6$D*J1^eMC;&7MI1fkX^1Hsm4v?LqcGy}Fc@5DB0jfbD9U_2fx%}A$b^NHAx;bE zfV-^loC<{{@6Vy8Ko!{m7PCJ3^M4k;grl#o5!oGq#%bF^3Bee)8Z1VscopW?tKpzZW^opwBK2!@Hf;;T3>7yC3aQ5qJ z>}P^X&mWZcyFC1^zhu!N$D5sc?)4MO@q7qecx2_u9V|kA% zogGDo2gSoNM|XJR5)bTG9dx_85uFU~K*Y`~NWaQq%&v5}nr;ATIrku?eHez07KogR zRR#Yk4S`5yfk4ShjdTuVp`xw}^)2`(IJhesxQq$OjE{wbZZ}IdmdS`6 zcJC%JM#j((W+YT~%OF;n(!wW$%W=i0nUb91M;IkH7Gl@d(C<0>(b%hy-f6MJDhGQq z`ilV+bR?tma!E$G&RfXl{UB?o0?v8=1nu5v&_x!GSQZpd`o^uqpUxl1xeQ5m9xR4q zYTHrQ`IK!an+{vy0ChY6l5TgtkBR$Zap=J;I;C|RoNw!eUxyR1FN1+N&}#Lol44mw7?M9~$Fmqh$h%bTOIo zFdK|cCBg0y!$B-tjD=G+NqD!p*gobN>|CFL(*jh%*YYM@xcdW**H6PMTUd~Oat{?6 zO3^vs6`f|{4#GpFxUb)Y^k=Q7VI_)i#&?+b_xY)?Rg&}Fq|#64eJBM7OFLX4GYgI1 z48_rJ_6ellHsfu{nkYyq6`3E|5AG!ju<7@4Do%e%TK)uKpN6s#2K3qf` z^2N~2r%;EK0c^gUNo#wqQ$FMbIh0z5@)p}fFH`hUhzV3C`6TxZL=iX ziq66QKgA@+)n7P*NHQo(Z{XK@OWHN1g;veIfPWolu$b%&a+{~o;VDaS;p&xm$ZtJ0 zR*V;=wS|GS=3U%=(;6)*8ff3Yvv9b}fVxFprCN!1L<(=c@vYw-@;PfHjM;hyLe~Dk z8*h8b!?PWt#FNRQX-p7+<|GX7eaXva6@qNZWJsEmiXT-ph}5T%wBf*@NNe9g@&5W% zq_b0-jFaFVp56wqxYJ*PXk4J z@1>}tJPqb2{2?&l4DS18i=>xyqM1}GZm;RYqN$^Z$+8c0Xj3-sGgTD7HIap1!35Oe zQ%JGuQ80B#B&Wg-(ZTC|MDLmgnR(V-bhhjTU7Yt7hEJLbGaYU4+uh+Hn>$84UVRg$ z#5)P~Dssqy1*wvZxm;Mr^wIGZ9dvKj2&^mdfNe1n{@Ea!$axK;Q6uEUlVq+DMU_2Z znDdMJZ;KQ-Ep3D%^CS>v{=&!W)y3N0X{804TVeTWMQoAv616Vn>2JebqRoq@;2e(- z+W2K8?upKYcZWP7W~eb4;k=fpdWP~^lG=Oby+{n2Ex?YjR5bUxD%h5h09rFE(PHLp zunQWAb8F_oCx;dZKVB)p0$1XM)%nC;7a>b=2Qq)_X=-N)S-5ZxIucVz>Dh=QO-Dd$ z!CHJ&vIH%?CV`_^0eeS=g0ZRq-dVgQBPab8=$N0NEp;WJt$Y~YwaE&TTu$Js%*$A= zzXWw&OvjI36@dE}?&50qP84&Uc!;g517@yfERa9lTsny6;O>S;SL~$u5l)dL z+;R90T;3@oo-}RW-s4AF4wVaB;-yqFlFK8tp?O=!YQIV?F>wja3u#l zbU?3ZANtLbMxTIZ#3_F=30X0T{>yiUb(@c3m2DdP>Y6dCX`h70$B)9jNjn5*jtAo7 zTrJ#`>jvI4FLa^{e(Ivj%B7DG&70fz7H5iPSZ zC2BG=aPtx$*sY{R>#Ln$zy2Fu-Rdf;GYE`)JPkalEA*yLxTr-L=&!L@@o6vVcNvS{ zhQ;wMQs?O>B`tBl##%bfYA^M=Sq<)@CvYTBqO#k060ge1fKhxZ3?2W3W+WsMCkbv4 zyZ-_7X^KQCd$)p3?@7pCl0fqFYOpIh1Q*w((KmNAVOPjEx@n_3E`RS$Ke)~$r&`Xz zC{tOopGgvxzPtc_9nZ*l1AA~S3ZYrwL&5a^Mog^|(#7+t=~}bNv|*U4IDVWDS^483 z*&~~WyZt8<+3Z!I9+XJ-nB|D}NY>@ITQA7q9G>pc2DQfL>APw$%JsRloI{5c%fKFpQqN$Ny-y7e@p;SepMQ83nP2+Bz~Cxt*+ zp;qE0Vm(K~V|_Leu5~;_gxjU*U_&mwU*bazew@Sd!bDtLY9x^UlZP%ha!_IU7&KX% zC+PgCgiX!W5Vl}5{#(2i2l|GBl_XWiKz9c6`>bJ^jG5r?j39jWMMqrZ>_A*XW9Y|Q z3HZ~c30)`cgj?&+QBGQma0P>+pt9uTV2_3|#+L3RaaTKe$Rd%_G4gH{%}qw|+8C(`km6Qmxc>pPlIY z2od?{GYOw`=Fr?oTU=3<&qsM>A{;#=Sn_W+M*o!+{v6Dp$5(x&g(CrLkl=x1 zJ3f)7k~MI6_*TKMvqc1C#$v-@FCOnc1{YHE>BxsWan$s8^!b=FvbIDqWC~5wc^BC4k?Ae)PR0O|8__#nH8yG&JiH&M(jxSQX_8)ShKxaNtG! zI;bU1l;CrhNiwEX?u9;s8K^W&AAYc7P#2a{`|&Gq=HPuiME9cDR}8Dps)z# zA^KEgiLR1*Y?yzg=*jM0)SLI2X6o$#Pa{Qg=9(GEMS9asxm@SqOyG^u!LN862`8HGFT2+B$FcE(~$(QI{CP}m?<3)}UyXl42N!age z3o_5=!pi5zKs$T_#u{ybnZ0uG?D~Bon`8l_UwtAn)~oS=OcOqcZxWr5Jxjj548?uZ z6@}5iMCjr$R%|{hR8Ss#o-9?EB=l@P1EYhpME^Vvp`-;9r9HbzF8M}c+4u3(_mMU} z+_M)$3lqTOdLKAvCZc2cOt`740M;GOSoV21eN?W31}y~J%ZFk0#cnA5r-8Gi{E2Eu zqiBf1Ty*sl(^dEWkRt&CY+-{)o9kIxutZh-DD0O=wl0$F=R-wfl6t9ox)R)Mhy{&Z zmKfccLee+iM6JHt;A0_$=jJ`ee|k5FPi;9g$Oh7$6ib-)a6DZ;`ysU+bSIL zcR+#U`Ml0G3v`mRsLt~77#q_<_o)2<&*eS5!{9t9>|$cg!RF##r zMO5rqPo5`D#p5f#!s5@qDED#)+#Qe-A1hJD*yQKrK;2E$m#=`nwSLH{D#O6~Kx#i{ z8+<&d4&V2jC+%-^@WWM8vcg`GT8$YLRB-xOB()G_*UpAZ-D$|xFC*ic_Mn=_A5n!& z7p$@jK{q!OoE26{THe0E2GcGuDpkfC)Saru#la(4SxA`OM}O%q!$ZF3Va~^sc>H=A zrfHmpvJikK<#cjMW~^vM;V&Ge5RN^qX}Ct}FcQ0=5R)^31Wq)BVY!PSamPB`A~i_V zJSKu+MlF?Xlp?{?GRPa}bljQvn>?)<583HakbfZqEZ$0Rr+iOH@fnBqCraRn)Ns=N z=@i}_xP?#rhXB{Rj`H(bsFH)eM33VL*vON=UT475=Nz8=RY&FoJ))w7V0OpX`EXd|_U9h?@NS53-+{3>LuW1Yp7aEPdNc?Joai6*FOsucg-)CkM)*u! z7{BT*-0*!0FAV-MyJy}7=m+R~9SbL_?Ld3K0$LdypfX)E@fB|(b{q1Ltliv)tphC( z`|C66c%3ARTF$`MM;_E`7ms&92pdL*lKr|f!6y0%5s#UG^}_Qob2pDlXaA9Oo%gUK zsRR8!$dczOgk}fCFk6Wn(TLj(9nUpH-B06i$hiOLiQo5#+GBH=^ZhvH#UH^{sa^Q# z<}Z3G`;4G!$10kesf6b`nad3v%W9aQ9X#bIyr@#yply1xCisAxzLc<)L@sZuM5nR|xX zPl@Jp_W2XpN#%4+&Kp#+X{OH_qsW^m6(~AA3&}`Jn*Q`8dA&3PoJM)U?0HqBa??%H zw_OK3Qr07?eWbG{SU^z3IQX2YNUo@yC$}HiOLE(v3XIMf2)CQf#+;oFT$j`gL7n97EaGJbz96Ri7yC~=xPoErEBEC|# zf-;-`(AIO)p{2u{Zj#YNvkYDAi0`E_k2BD~>INiCa=@YH65U9ahCsA=CTN{ILc04x z@ZYD=;>@!{VYz)F3ja31ba26^)@Sgt{{*o>@SL^LUXLsPtp-m%Q}pQGS-cp$Mba-D z!?L1qiI>btXgG3Tw9;Nv0&Sgumn2KnIrAI2a_lm-uwDU|hBVNLKT|O7#df&<_BK%y zpkPt^81$O-m#q8RfMHut(M9J@p~2ztBDO6OHD=cmL0A>xu4TYYe;er7wOqv2-orJ~ zLX=yyn$$-I((SD`ar&rf^n%`TJo9e?Mwx9T@3XY{&m@hjlwc`YyP9F6R=lVuE*!0* zm!iynxgx8C1{(hJKWeSI4Bqbbf*ZlH5IEr!X8qYrcQh%&`tlnj;sy`8t~`nFOhEHj zJF&WXE*@3vN3)^?#!x8^r`FBD|E@d)h5plEvml0=b*Br;;x3Z1^=7okK8KnQjYNa* z8_BU~p1$^yV124;;<^BRVeo*qIQi#4vRCMg$(r)$EaMOI?$LBxXE8OeSP5IJdvM_}?oL3^%?>yaJ-&;LXjLbJm4ydmn|F^Zt?w=?s$Fwt$*u zUqJ1kZO|>z4Sk*Hg;q5=Fg+p(wAWn1-@z~7Z~riS?Ii=vhsF|ZE#3FJuxO{4!6 zzM_sMS}@kXpI+3AWoO;^%sVDuBMTkh2_~qhz>~~T;5H{)w9GgOwer-35%q_pn?eKadl<9*|tJ* zZu)}iLuVY_@C{`B-ZFC>mEeJ#5%FqAa5%Pyy6z6aL46Jc)hej=Hx+Vj+=khkX2Of? z3Gl@t8EzER5yg5nvi5{7NdKM*rRB>-dm{z}>ZfzarXhRC-9RJk(f!2B%OygitCaBG zgK+}QfayYyMIXuHoJyP)+)cC0>&U32@APC76?`2dE4B!8A(e(+xMfHKcKld_wU`rl5<*sV{!XGp2R+VNfqJ#q2CYoO)Rk!@54xY2sdTLDC~SCoYCRFKlti{zF9S z>P@s{5-~qn4~7_J!n7+^IObXu?Nyh;izgkix@jn86dIEW@8U$UlKZ=M?F!=a!4WeL zyrk1R$HK8MifHqofyyV(7G5)2hO>5L6181V={f~vI31%+H&-^(1)C*3Q9>B{8%=_5 z1KLnJI+*5vu!ZEiwZtXzE$jhyfOLxGN&?J%X@cUS)4KemBjn%8Ia-|DZOHgg_ulDR(oU5YDXN<*A<)Ic}GJ! z?-CtPJJ_0R%Et9ZVE*D^!UfWM={GH9;gQvPxTJ*vk8oM>%)PS0FRZDs&7%%N?@Q_{ z_6f?YxhlB0VF7&+Xhd?9UPF}9OT6E&11qMlgZ$rPapt}(dfHhASAQ)8lbd@)8nT7Z zH0BObjx&er7g9j~+6LTbA-PXh)ZmiW_drSJ2++)0Jl5-iGF!iquLo`k_P2$@J%|C@3QVbuSDl9WW{Y6cBrn9hWAqJVan%EkmJ;hF?&ns z-K=iBs*_HJ7RJ)kI~=LZgH^CQ<($CA)`u?NA*q4l`bBE1;~>g=sQ?at!HEkhm?8W~ z7(6f)4>cqGvAs;R;#58MX&xv_bhPUg-72_aiRhpG75fO7j{}n_-8H^=m<9PXoq38v)O5zM`Ib?KHe#y9BEe zg08?;oZT0Sw6RGfTij1yblswACBfJtpGC(kPKSm&ZycDze01Ve@C5Jyxq37+7Qm3=Z$Sk&k z1$8DcInNYwqplILl?d}c8~_|Q2B#QWK#_VpI=jh1{Es}*3z?z9+GB+{*>OD1uy295 znk3P4rHg`j?`}Y8#s}(rJ)a!xs3o;emeS}g1u)<{1=DsoV2!t>P*dTwAjMk)EpL^h z!t*VBV2TIaJN+MhJTE~IVLuJ7%$9{iHSrLj+l%$S<|3_ER|Q?SGB7Jyj-B^7O`u<- zDo)i`7s@3`!zT|1{L4)h?=!L`7yAQ&cxj2Z9`B{Tk$Ke3iox^3X5y(qAhtsdS~tzd zE6dLlZ@+JJm$;K|x$qEcinL*of2CktYa6MkDI^|mN8sZr;du1BBeu``2utHflXos0 z{ykBHP2-=F2ZO`OxOz9>3=3#V))`nOFvPCRI`CV25%;Bf;=CuLfmh%pJ@Yqo4X%WH z`Xk8P(gN7}bewp`lm@}1KPk|-vk0VPUBJ!aIw~*phbbF{K4oajHxq0fx};8MW`uu~CXPj@VF zoRWZ2e5Xh!VT5Rx;w7m6F&go6EXfEo5Zacdl8YsXrdnonN9r0p74}Fp(O(%Ko$*1H z#}iSc-z|!CIE)t$A@$?scn%S6UD86!zl-2m zu$)lw&tcdzMnkyZ!4+bfTulsXSi0`rOi|&>IqF zSl>olWG(sRyVq%`=@P-cTg%W{(UcfAM}xY13iOVgh&EnD0-t0(@s6NEEU1)1sZnN- zUiSk18gD{GK^T|0RfTkEsAApELlE|&nNK%OgXDYK(5;^cm%fzKmIom+1EP;5^agkd|EKZkT3iEO< z)7+|aKuswS>kEQ*FoVMt3h2fECXE}yuziy}mH%pi#U2H?|IZ4TzPg9l%r}MbPsee> z(Fa&II|8m+E`ec(bAg$mEPT6Zp{T3O4g8tcaJX8c^VQo0?{DkTQ*letC~PV5zbk+e z`IDqnc_k#I&LktP{=sdhE`CLC7pYph7{6!-NqFxv`u({Y4mWavT!#=`GF1k;)(jCU z`zL{tv;-@=tS`Pi^eeg1u0i^xmBham$%&2jUW0*%hp<#r8Y)7U;JB|cLbIj5C^aNj zG+7}ILvtcfEBP!g>rY0HWP2z-phuVlb2z{A1$|K=g&7V*g->eaVcp61=rG|4Zn?jN z)R?D$%i^a3Enzk8`LF}8&P^sq4oI{D!&NZcDh6-+T*X5Zj$^cYnCQt4d)lU{1@E`l z(axtau%P1@4*W0%&4?o6nDZF#bQ!_$auy19+~#v^UCEKp{&2>;mXye_X!B5;NZrnd zdqU2owA{JsrQ4=3wiTkJQRt2*t)J zbWuqRId&@tJ;JrIcj6AZZ~kv$+@T;0e`1AcD|b=<>xHDr;1xaYH3O#{RRELN+w|`d zUG#7rLw+ncigful(i;9l!i_8AtN9YVW#4n6v}!JF-c-*%4Aq0<4@&Wk%X2dOKUt8C zjTUVitfDKntUxz~i!|o`5OAF3j7=Ff#9dNrx(_iVcQy9H_LZ#=J~9y!HNw!u;4zhE zIjY&^PFLIgBUQ@1qGX)_7@Jo>6YpAq+ls$bzR^qYPp}CK#U60Vd<^Cd?GtPG~3iD-C+`(rD|%8>BYVL~QBe3=62*z=z+n$K_}V?_)3(sp~i zb^0dM59UDa!;K)?A*sDj8)C(x(P*=o!SeORI8`7mKJuxB=JpPdh#gA!N${CoPdH1O zca6b&9alu}R-J&6`qyyq;AG78nu?E}90cXsL9$cRi_DG>!5>{y@q1G<#LsgFdp&!a z+1rF?k4DoKH$3s5<}+dtl7nvRPLuf;DLpmJ45F8u5KUn0sCCtVs92&W7Z&D2`&n81 z_*)N}?vI06Yn3o;=xDsG`jD(U@&ONbOVNm2B0MA>3Ew&u+1ew+#laI)FlNm^ayn$V zSnXLXZ8<9sKKH(hRApy^d$$?hOEjt^T5;s7JobxbN%x|2^{r~DSZ@Y(>TQ$Lbux9K?Tek!rJ zT}`6h{pk|>r=lvKkLWcaO;mOMB-l`v?*eG;T4&}ey zn?swUC)0D=hGVJRUbJ=!fL}k(;2P^Lj4$e>$Bli-thnveOE4RqUIn6S&^*zS!bDIy zDGQsTIXd}E6*T0ZhjGC?p8ho##yyniQPe+?@HhwLH*3I*`)$-^rxCpF_8@i>b>Op$ zyl`QB5?#1)43@kd3saqi@L+Eee!tQR%66;K$o&weG^9e=h8WtW%8H!*5=q9y4=_&O z8Y~LhV1rW;^)?itLbMd?GQUjZ0A4co^N&7D*HXGwEWrp~4GZ zzS!_A8Wu(rik=D7(BtG7sOg;xa-rABB})sKiod8^Werg)KS4I@F@WP2gXPF967~8o zZE26A<5wI+Dbv$s@v6{GFzUHB~J0B&6jDE~cEpf~a@uJ&vP zsry&ZQ%6&H$?Op}Z;wIFdjLwWdNQ+JTj5K%v=Fnmixm6=$abj#dT@Is$Yn?YcOe06 zW^cl2m$%`DMn&PS%f+;7a|34ID}%E`jzIQ{gLEJ`h;Daxfdfyk;nvHGafnz}e0Phq zSpUdvY$_aw6PvY#M&Do2gg$Mw`McrMy2z9D_wCfxPpFWvUl6W5yD zLJyV45)E7(PD*$~&6^KWhwZ=Vf{)fvZGQ(^E#<@|Ljr+`Pa+G>DGDzy7Ez}@8BE*V zjE?4`$;kC%pwjOo{FZRUZbC<#JL@W%?b?Bhk2#{Ak{-GIdNJ9gE01yN6EP#*LvUOB zDtbhj@562lL^5`dX2*R^UV7h-g?Jm{=D^J7ARnp|EMB9A!tz>rkiwz3HE|OVV z*~olYFBmnx0(v)HfiD7kNhyC$Vhfm0H%vRj?mzJdzxR%UWZ!s_e|tVr`<0Bvo|B`lIFb!PVAZj6Bp)V(YuxO$f8#Q-Q}`)xwIC(1(%@TXK#9D zst(<nKs3I*5Y{hO;5pL^O3_hKT%fq;jN)?A|n+hF|C)h6?ipZ)!_OPGB8d zvap-H_KGIQ_sR(m%~^mt)B5SiQ$}QLjO<(>5}c9D@(;I3!0IttAT>=y zP8aM16R#$)==%@KSBAlo_2bw>8#lp8*YN;43eao5o*WH(h!aMJ;yJ-hP|%PAo3OEX z>{cV)?kWvYQNeI-jw#IJvZ=$*&k%MlT5xo`Ip(RF!ts;W>5tVJ@M-;XT%bZ0#5a<=cpMrQFNc11Ibow;^NXTu zs7ko3AV{+SEfpqV<+jE6R7wmxTu+c$lLyJT)>)$H?X~bqrc*Rw#8AP~{8(6a^)XuB ziKJ2Yj?)USc=kY5DjqhNA`IQX5Cv}OL~r0C4vA61RS#NdzBmLmbv}sxnk5te+&`4> zh=z;BwNM>ViL*wlf>-PbQj(kviyDM@X>kaY>Lt-Nr9RO3^dhZN?4@Z3D?#XHNKf_5 z!WD<-V+BL-dh`L@oR@>!Y9El-ej2b`WsvN5afQJzw@GtTB4kN;OV4kIXk%&)$UPZB z7gX&64GB)zIVTdT)2__ys ziI;cEgYo-1dP?6P@;?-V+lO)F z*1Hg@H1;rlh>{^IOtQMH` zEv4@@Loob#0VJr@QQkce7JXB~-SX;~Y?TNu%MOE#R)j!$%MZct>!~m#(<)|3(0XL-&5sz8Ik?Uzjl37wsRR4X3R@lhX^_Gp~(chzZ&1p4x`Y@0hv@O7d z^`U}aSHp0{!y0h?u>p>hJBg=ivnW1MO%k6+(dDPzA9!mxc(cn4)*xMK3u`1j=X3Mt#JsqR;Hu+rKqedtFkNK*QUi{#2B_)TPW10z1^>gA zRQ891a8co3y7_AXJ?fH1{|+7&h0c7AZ3ot%joW4LpoMg==+!)ng;yoI!}RNLq(*`89!fZUqKRPTu5dDW#8z0eT_5&V4nybM$HY_e zy-!%0Az3mT%m(Z5(ccjIO!@RqKG`h6Eb4@gj*|WQX_XtS8n#CEWM$anwH13zb_Eu-izR zo{TJjp5nKpQ}QmJ(1FiFUXp)#XK~(qMakT-BNjAn!PGz}h|NAg>JoO7o38KSOneh0 z+@35D4%9<-%PE|>O$g6Z3Sk_l1xLr*qQ2QVblnn-S4XAOrOq4q@{eKQJI4l&ymX`V zsS=b$-6j`TJwUbcDKPo6L~maGA1N?A0^S?)VM6UhqS%)!;cJxXm(!=P#;k|r&E1CA zb`8TbMJmG6JpdzJCH!vcMVd6NgTzR9twA~t2i1@-Kb_Q(_a!uDd7`Wd*<^p@Do zi9zL)qo`mfi&pdQ(_hC5J28u8lY zI%sMr#*AwxQ6v95?bYVV-c%{!MNLJatV|U?Kl7bVEGd-S?*S+;S(7)rW}(6JKEnLH z0d~!L2Fa`zGD60b1dRVBvfPwPO<*`ww`$?dxK=1QnL(FzwZXrtNtSy2rM_3B8vh{iv}tCH@WW*O$y9&9KI=pA+%WmU{B?+c8*ruogW& zn^EO4RRZ7aY;yN^D!!U%A`Y(Ef}Qpe;N@6{H9-@F{EtO|lULx*>OgESIEelqY-wt{ zyP$HRA?>^s4&3v2{JXx4T$0RH-nHcEn@d)3HK_)3@=sCq$)~`rZmB4Hehd_hzfH9o zt`pxFIdJ~2i!Yaa!(qPIq;d~WU zeuzVuAZE%iGOuhhdWu_dgZTy=^6?WnF{lN#D`Uv7(IN2Q@G*=utA&sLne=m#04{&< zLFKqwYPxnKG>&YcXRI}FvPB|oGaE0~Yke%bqE`*EcUNIqVk}meNDPZlPZcCf)^2HKVI!SC1S;A3|X?u&I08}tto>U@_LrnJ`(jle3_mlS~3-X=nQ#BfO6 zi^TdEifzn)q8+zaGG7&gY)=JF`ud)J2!D!C#@5ltBP27Vh2EsTzk)V5OX|F&78-q{ zpB`474SPqv69hf82IJEGM8QMi9}*V?vS*aB^|c$)y@}*OySn(83X1O5Zp1SIVHMPonAW@AZO=Q3SOfCW(w6 znc%W&$xKE~0^QKwNvEehCfEF2U~&C-I9a@e%rWD zc?hik`4UBoeegK&HJj1lgp6o8DEdu=7ZVD|?=kNLcaPVg`5=;A{@U=|b}n63WelOZ zDG(^}!_a-U6L+^sgL$hzIvT6NUgu5F!&lQq*5%aUh8aClA;AgWT?IoqD`8{i5nMO# zIaX);;=-LPi1+g<$$oG!RK>Mh z(!pr_8{F}5s$gyAEV3nc7o?3+6%MMmh*lM@LAQCC@O+p=d)amr9eb|g>&7Yg-r*K` z|KTPbKX8k5O`^ChS4NclQ&ku)rG*(cl*P|f!!YmJM%q7gI*Rnh5_3}l?pWFbV<#lw z?OJuLlJNL<VJR1LnWf$F8}z!T)$6d=Mp)Ulxt@-Ny@P z{Qd#VbS7voeUqHeJr7%DrGPOwEa7MLVSpTi=*}@@b7LgX2eaVgg=yqB#tEvwdE>Ap zH}DTlg0BW$SS0a%x!nC;bj|E098IwU(7ukaQ(kD5wuy^gQb``Hy$);t+d`_#eiBiW z3Vd6?jD#ic!A0iglJm44rW$HMzQFSTpP`gwq4x=j#jlsu;eMjJr!S39|!I!PFqa37x>Ux;yC|7dW4 z9c;My6I1zpqAk8bqIRbw{5jSK=Z;$d)#f+g_yR`~*f)dT6~CZEiNupas!e3={|6il zUelLJFCg3Sx5V@2g}{Bb2Wi^IQ(u!|^v+u$cCjyMrTj#kQq)W~ScYT!?+wr<3IQ{R zjqIVP_00P*dVHG2c}@^g%-99Z;ATiNw^nOy=9?F5va|Wy-20kv)}uL^?KDo|t+&i& zi}cUf4t$oqt&x_*JmJ1_@^=*Y?D9FhwXQDf^V*)nS3<^eZjSB5#<#YK+)VDoTub(1 z$2c~m@+@bbid^KP| z^V-SRm{%IP9Cxvb6G|DfMo)e)`KFru*z$D$fVjEvR zV6%Rm4;SoW%Z@HE=jVPs#|`0!vK=ctZC}Z!atkzuacL$7Z0G`sUw5S}Z+_t?*JG%{ zynGgXT_HexP+Y~ym)s_Y{ZhJD=n3oAJBzBbOr zUY=)?YB=X%>zNVD_VU5#z~@x#War4bGIw69uz?*`%!>V5w>Rof(oKsMIOTd7{z~h7o-dSVXQk9| zi>q^N|9;+Mdw+0vshlM9Ve^G(e%6cw-0blS*m|$yjN|U%+?B#NocQn#=GpB}rA^1q zGETY@fAdXaIMo!D_`8B?{bj)LUwzl&l_cGd(wK2D0 zUr*5ElO$Qy(+=w~o=;9O8}EGQa+~zoYRx~Kp^*`vy~}`o-gDZzyTO?m+m^xD#VK*C z?o4CTW=Hc9BUoM#wwB$x=N8l8Tvpn8C5S7szs=a$hjB&)eq5*bbgub;Ik&4-;`wZB z&VIhRgN2I8yu4VSjZav}XN73#BVtp-F_Q|C5k==gDzHs6{ zdmLhe2VXIzKTY|#-=p|FnKM|w!w0yIyC(dDEhG7A*D0*)jN#n3e-@liL`13L6BTZi z`Fn2V?#ZlY>o9h!=K|irLxz1cKc8!{*q1KZD^TXMZMwJvPd#nrt$mnxIxWma7e#*BtQtnEQphJ0X}${cx$9#L*aoRUiv$38I83f0_)!Moh5RcdVUPA$G^ zaS*$6;&QgL(}z!)kZT*2};Zup8wMqhR`d!z9Yhv`QcCx3J9=)x6Dw45iyQ!{?2%qDhW%YJsw_M?2g zl0GYNIKhWaSxQ_#MV{Kf`DkWiT&WuGwyzy_4%H)a6A%``J-< zHn9bNr}M?(43qD-h+nI&$D3%KVCcsqT+!o%(tw+F-2R`hnLq0!9^_k0_`px2c!$O- zwkwLCa+{Oqvz>XX*_H3Y`Bi&1l$vk5!wCxY`McH@Y<-j#`){c-H+g0?ll#qF8v>MHqobl)1 zW>00Wjcd1&8CJ&S=Ml!Da2=!jZk%n_j}?5$DkJ_;#4RRHlHJrbzKPShmTr4eFoQW{ zGmP~f%koE6SU$@tiv3a5!AWQNvNwn9s}48^Y!t>*Rt@jc1$do!Fr_UHR%|E4jaWb=kmhO;%8t&6U(HV*8_3 z^Ha^sxQ8JD%zPDF*74iS0&N^)_|!I9<`Z{H<_xDW+>j58iDHbDYHTzA z{^nBOdY9TbvbJVtUNC;!hqHGs2-!1R|8a3IM{ZNTYIr6sn= zD9UN`v(9Z`h4sIg{i6AN`N>6$UDHSArT7zbZ`<9{p6{xqN@>@awK-Qy70>UtwT-D^ z`i=H+F#}6&je0LJZc$yFoPsTzTDFpr@m|k?<_)HKR5urrufw-ppTjE4tMGC6m$Hvi zyqV=0HvHIS@0nj4*OxA?USK;{Wg1tQ(#8Co5X%QG4&f&+^JA|cvtf!doJvksRWi#% z#&8b#((Hq~NBLEEr}I~@Ze)WpR9Mp$*BQ2Nv+dRLZ0jH6ZZe0O2RKWODz36rzf`;5 zk=u4=BHQF3$$79d;j#l{n4G3jjB%3|=V!c-8Ecqay1M)tx3gD|?|v}A=q`8TvlltA z@!xV7=~OM=yXH}8bHH@w#ziNNjYTF*ZwEKp&xK`n)iWDn`nXq@bophmYuSkdRm|1m z25xQIX50T!blq_^|6jOC+8P?tPSFyL`#FcotdPWJ^LhCRq`J=mooAWy#EZVT{jV|_=pCV~dTqKE^B$Df}t#Pd5C+gq+OPO^eL*F7;#&dC(OzP^An2j_vJczxA_Jgci9cE6JB`TD`*Vn|U>w5Z<-AEqSHZ^Pl@wU4Q!<7dp~ zeviQKVD*A#21=bHe$ zGFuC)_mq-!(IL9I;Vel54u}_Tg$PNlez4+J6IG>eJe}zvhvUw)39Z_@;mi7`w7upw zeAemUiLo_osj-JVSJrYQ@4z>-xC{NcF^)wR#S zu0w}m^nW9mUKG%$5F?mB%oDF2)n_|hP3mH+ft6Qh!o>7?8tJae`vN!Obh`p{ouA3u zbV!U?FdOq`-i2Y#3nciwyv z4u8G`&qvL`(0}2WA2EszmRR#o%^9SAe>P0eN*Bf5=HzTO3V!tIieW29an&;&UV5Wc z%sD$#*cD{R^B0YvGrvtSK>G%~-1Lgp+T@AP0=)6Y&8P6^{2Q7en*)~5R)T)_57Y3n zLKtrQO=ut6ESP8=7te&fr`~}LbaQAN%{#T9j8><@qlkF1meTRdj%*C-=+4O}X9|)D zK4Q@hR~l!iPLpod2x&|7aIS73l-cCc_Z_PU@5y>qdIm?Tw$qVyO5|YH3oQo*;*3l? zjyu6{>A*{B>UWoBBt3z)RyQ>G=g&ra-qMj912Ah_8&rHq3Tx<WIzw&hNOxr4?-WrY$2<;wEsK1}3Rt0~<1sTH!7wYbtV4HJHQ;z^AFp8B9xu=u0J z3pcs*TFVl6;HZI`#>G&zcomFzeolO<=|GD&xZ?C-@fg%OlYQ^j!t729zHLz|d|9WC z&wH!WfvtaNlHPc>sXGjXgS%2fe|y$D6DV{W^j?(S`6!LH9Vm=jt_<=QohWC^IkDb1 z81LawY)p*cZ3Y*Gk=y&wW?CzTC8R3f_nmI%h5`Pv|MjXLrKb zi5jp#P6|7mZwPm^N7BESOK8QzJfT<4eIe$WJs5wuES2t>0*WtOjtxLLJ=nwJRvkk4Ck-qp)j*8}G8aC<&D1KljW?1MdPQ zsWtT!oX*PfST1XD%BPdyrYbMl?K+62$|uqzMO!K!z7#H$B46-~;j)0n(o834v=L}`{Iti&NajL=oS0 zcg4vYkI+zuZhTd)McAc#jt2WVgIkn7uageP&Of$zz#)<1+f?b}_hH-_`;+|Ir=fd; z5p~@?7pImc;lIxlxKC^@8E);1KV>$V?j3LG?DSaR)qQyErvK>JngJ-E)=2@X?wnoJ zlZ(FUpp8wFH178~wP2cn(YH|%#lzoxq@(iJ# zziQxPj2fFc7Q(kYZGLvQj$%&_Cg(w`;b+?#`lIR1|9M#90mBoR#it|YsLS_*m=L({XZF5}>j$)GTCm$LSsr?$A-Z%q zEglQL4t`(XNLL$QlrGV?h8ZpWKoZ&rGn|M9H{}TPPAuh-W6gPPNDXKgq!}>`*wmun-A%c~-qalpbYr3*e^9y*lXao+Ceh1TMz42_OEkB<1 zT-f0HM7Z8oEMDeZIvVIgFO3h{V%c5{x}R}C=(ui&V8@3yx`5HaUOeheCj&tOG=TuNvt{-+?J%L;Q9HWb2 zi{V1I9LWxs`@-<*;pAQBfK#hXam=`3oV!IE9~^S!jhCairq5_>c(|6vDdphWf!TO? ztt+d)9*i4em+&doP)?9;hQaeD!s+uJLiv|p(7G_0?1$LV*T=Fxdr@z3Ti|o5tUf9( zJ^uoV&%Y3&t}cc%VP_!lj1FH}|CQFxR;B^7{*i*L4kae5;nUwubaz;HuClIx-z{U% zuTF;xr=1#{qcwlc3X{sm1% zec2iFi9KtFK^Uz{b034{* zWxYTxZqV(H(TaQN#PN%jHvZ94rE(uSxLuRZZdotb%ZH%Wk`JEuIL(!dlb=urvfscyV9is0qQZIXO5ISe~j zO78!LQ}UVs>>+!vC*F3W(7747@5wqgdQ-&zj0|w&>C<$@^fXzkcaSI^goU$uaZAfh zP}tWg4E^KEt9H+Ye45Fbg?Zd`JZ*hPq0 zYxHrt%_#8S(O0m#&m~Cr1#S0Hk3A1m`g%8KBkk;$X;_mO+-1kl@JBGUB zq%Fo+cl$p&ulpZqR+fl$4=g$3)lG3}u?TLyZeo4%aBQ5aOD)1lXo_1!&+oqh`TQ8J zjY(sR{4~7xdY!;G7P3oy1X~RB!tPHjpgm?7D{i!5{gis}-y+LqB~7+5ROIrL-O+gx z!ubV}IKqB3UiaA+c-bfPyG{Qd+P^`lwdw^h~v$nwWc$IyYvDR41< z|7oA3L$p8o4aC7R9{B1#9mzwyR3=D^-B;s$mps(XEoSes5452@9ZP=n#7M^cX=hd4JLZ07JXtZ?Ti)CJX z<>LS}oIVrrsD{ zl(iUN%(g?DH#4|2^qrvm@(7vVb)t2HE5UpIB{-p#!x#UWvzlovnhzO3qZ(CN@8mT) zxzrT9XiN(C#MQGA35-i?%Mod{%G{gUJtP{F5^z`7TISm`0>pG>R#Os8?G%xwe!yW@%KL% zr0CDm0+D`)kHqf{F*L9t8E#*E0Z-Okr3$-^bY!0j-e~lNNlIoCzhD1^m2n}w!0-(D zEtSWICh0WxL%V3^8_wSkb{D_YmV!~tEBNosBTz`K6jFa!i?)^*p!B#l2K)Y`&q~g) zuumj-y|lnvp4zOJxr5d!&BWG_u~_S%$?NWEvaFFnwMJSKNHmo_aUQLuXbXH9ndr%aB_sOz4cI0F$m{eIx+) zc3s2qip9LdEf?dn?~B7mcEIaH3b4=S4ZX`y#*x3$_;Q;atE~&h3l00g;{A5coVk@x zR>mNyzY*t}&tmD2Nd9_xJiflHiD_0D>}2$Vjt1!9we){r9Av>EO~2?|_p_k?EC(tI z-8pmcC-MnbMm1UOS`@d2`VPr~!D)MGx8E(21f=q+HW>$M7LLnr=~qn2?~b`g_d$eX zZ@Rdro(dNmaZYMI^=eQAm3&*M3l7AJjsQ%^x8Q1(T8eNk#L|_c(O5l+`+jeyi{Ik0 z{l#3cH?ZU1N@r<|x-p)&{0ib&HSYTTKcdu`vN+@c_^qisXXR{$-b-~jt#2N9w5}4D zjFN|f(f%O+rXT)O`y<_QO27-r#zMiHJ)m{sfiTN>A9Po166JXX-CkD=D+iZQOKLal z3L@tta%s2U-W#~YYdE%O)y#`~_y;naRs*3o$!ksZ1B0#PjUS zA!bH8*<7m?`o>m}{g$41a$R3^Oi{)!uB&N_H(=Nuf8OP!#wO+Og*Dj|!CXI%vmuH@ z@-Z{4^U};@J!;UpCX|7dOR2 zw)_`?lP-GTkHgw*<~xmE zZNCXdt#8Ge>-q3K`Gl}{?-bDVxIorX&eD7bBR={d7l+J>Mx$3NX->mH9@2X)&bmGY zBl;}kdEYbXwUQD_%DW2V#YS@e{syXNc{S9* z@E|1`IImeOja~t^LCFx;U7btK?$FPue^%=3KygR2ztWopLYO2s~18r`HFwHjMUHVe^@7@?Z(h`IXD>jhS?6WxD zzCi2@=>gv!{iW*C$uzA%AM;oSzN>;){L%PGvLfJfKt;yqm z4g$yhSjm0Llp==Y}O=>=Q z687HFOV-1h@vzSdd@Nxme*M%JRc73#-KSl_z(WuH%2vYOJ4)RD=NVXWb~<0Qeow>G z%*gbPH~Of%vAgwZiEm9ftX*l#2i4^8sKf-)F3;lWwO55(msB}^pd-htI7qGxmg9wX zA^3g!cEP~uH0fW-5~Jm!Dd))y*bIqOWO(d!*1QW4^lSmF=}0Blex3AwSs~nsc`4l# z{E%jL$Z~ssDPZ~LjbK(WOhvB;2~*9h$-2f*NOQ8{ks|}}F1p}?7#Ytw!<>sQ15R5Mi%-kE zIi`OAomke7zr@J&yql@e+<6_I_iUvGwl9 zG=AhSiBHR*Lua-Mf8EFN=&1pG_wsRgcDWL^p0nb$XM3_rtR+65xeHpREMPMn&tr{a z(C)hxC0MSM>{4WK9&k;%b7Q&m*{t`Xx@9O&jj_O>B0IdcuM1AuW(}>8x?n$XmuS#7 z0-t}@kS=4?SLRn3*{vQjtDeH$u3cHFWHijkR{**1 zjnucI5NfyeLg&Xp_+A{#^S|$h74LJn*e8b7>~Dp*x}J3C#cr=C@GoA!m1Cw7i@ExhWM-HBpjGG@RzFaCh>pebz`{N!w`}Gx??;4IB z^q10SU-1|bRzk+l*3g<*U%Y9M!bwpfyiG0`%m3^US3#w?tWlY)K9-0Hk-6j@br}wn zYvP{kPspI$oJRFDU^quDnYx4s*qyy6qaR=6^E3R(3OG++;=Y- zA1;gM#V=Atwe}%c6r_dLrJ6h~x>DNf!5n@O?8)iH<8j-@Nl52PSjo7Alk}(HIgdkd z`Clcy@baV1n`7bUp{<}j=Oe9diiTNR=L`4KdSd-59ZXpuhnp-@7MlVp`H7uU7|FI{wuq-+&HE#YgwmUEKIE6{6M0Y?k#VAkh&5x)$g`lrsq^)r7cjnSbNuj1gG=<27KygmEaC-euGV&H7QEryZ=kZt4V4XY?>TiypZ=QyBjokCW#(WUtL%=gac^k(q&P6e zEcm$bv>0(gNqWjFmsY24ggY|s?^fnsays!roRGbQ9;KE@w0C8KRcn8$%N&Bo-u?&c zV+;s-iJ+-CguT2=DP8jfXgnT5%i9qBLTjMDEJk`|P&u3)*DAC{E93gJ!)WsU9CAN@ z4>oq0NqJsb@Kf^u^lUyu+Z5Vp;`17qa4VOLEN9bzJ74Mdy=-CQMR#7AH4yD(`nq{v z15Q+|7Z&;`u-#}g{-_v;(OJJ>bPo|KcPZn*L62$6!|O0_kS=|iS3`}zQ=ss_CDi3! z1yrXs0&aav7H%aTSL_3D$E5(=-7$uDulA(?!$fSzTqvt4(VSP%MrYP1q4m=7*qEfp zt><3C-Y%-5%6C7$@U{U&**jmnc{a}+VQ9XulV05j#UIViILK7y->_8Y_(ump z-#H8Rof|C6?OrQ1jl2udJAc9V_Ky^?w@6}sZzQhY^Z>j=T{x(J7p#3Tn9mE(>5|$p zF!45mY>5_+E$a?mvIG0Ae+o!{+=1r$9MMa`9~^Q&ijI#@!^U-&1s(Nr>bvcX7(PmN zMw--FFSQQVpMOt_#*SvQ-hFt!e423i^i6U0^LBVYrylaf2qjb`WvB7XAXIW__J~2f0TCoIcP^cCEM4j zI4h?YW_@dt=@)r4v->0QrOyRA(e4HJHEl!#&sQ)sq8zpiJw{6XT-bZIF@~=11eskx z9C~aug$-zif9nE7vzwwgzeyb>3qH`Wm3G|e)dZp1C&gP@i?Q-lDCRwpu)(gQba;p_ zev{RFvr$jT{dGT@^>z$8{Tqt6x(;M_AD`EEY3cN@q0q~QHyZr~rU z!2zjO{Pn;`SkT9pqwH7V?5e3)nc&NZrW$yZ6r1q=SCi@MBn{lsH4&RL(z)XO65cS` z1zr84=hS-PUzO>l6!;e%?(+&z;2fz%4Rdz9NK9tAQh%oM3ckDHUwI2mj@| zv3_rPp0478Y3I~nMGKk1^nf zi+b^?s#`EY_O86zw@Zw4oG(=V9*C`KX85^4O1bGdklb)s#;0j;aLs5O5MOy`jEONG2%RdCB)9-{A`gN4UTF}uv1Ef4&rrLO&O;dFgot2GsG1=wS=y$X+a zHxu?mou}8cT=U!brR5wvBr+`*;I|(Vt zZaAXPR!)2w!Eu5Yj_B2&7at43QJ+lUbi)fufA1(=-F1hD@#XGPKV>~Q)PEl>6#F3Q znIj1O*jK|C4eIN~=}&9LwUeDmUr*+5xw8O1-x9@b>wPF#a#nm)nSukh#p5->}eGTmK8Ds)rW5Kv{`m+HK+Be+E2j+;dnN^d8Ke^!adGKVH9k4NlXYhF7;w<9q!- z!L^4X?K3`Y7>b(c0M)l_9Z+hZwn>3!=!=5dRzCd7MF8n^?hBAd5 z?{;L;hkP&5u(|*bl&r!H;WN4aObh(_VH^Ljo6oxI(lLCfEf!2m;R)HT)XO3RPsE%6 z-Dj)#_{B(mGU5>wUt3S=yC%XH&%e-nr9CgsDd7H#eDL}CCh%w^XHv`0_^qgD&lYy5owhQrMf0sNVwHE@!FDcL6M3XaVcyJMtd& zD8Xyu9QY#K1EWplU|7fw+|9Jo;C=z)538!9s}ht6|R_CPbcJl!B|RzC-yls*mVmX z9vz3H9(iCN!-Ve#=&(c`l^2Y;nyV&sZ3A%Xi%{DN&@kxlNdqF34 zw&BspbvQQJm+^xNhSOx8*liGZn61S3MiYhKT@O)a;bjWldO$M%+$>C3Tr8^ncL@ln|BK8LzGzA^(w$fS&vMW zP;f^dpyh+ikEbLAY%HU)8OtcCw-crW|AMcF$FZ`35wAa$h$+bfX-vEyPVDZ3T@Mc8 z#7Rr3PH82s(wdB6pBC`1UtPdpMl?8G^@d}{pQ&$E4k`axjACw}$m}*~b!>tU4ialH>@-x7Wb8bz!`I{too1UW0sMJ)6x)khYFg7apILhvCP{ zNptl_>K?TWQf3|^>%Yo8?;UXS-8gi;JrDKjeR+W503otZA7Rh0jj;KZ%)=4jCJadJ zkADVUhOa2Ov;xA}8w1;y~5;h@ETc+oT@2B5O4)b*rKq^BZXN0ef7q_Zn#FNOY{j{He)9{kNKg$_RhOjywqR}agS{`qQw!O!Qh z%D)-hUb!55t=7T)1I9o%+gtSh_at7$dU6a(fTCr=qA-u3^grv%1 zQFy;wdLei#^@-I5yJ}sEhkO0|b`;)5Tm#|o2k4A?;4xHbp|I}oFPht8!%ew}mz^73PwKfy zAMC9XCy#go+viw->CgKR?(huaa#mA(Xs*ziIFl7i68J{mOmIlQB^f`gFL!%;ko;7# z;fBv^@#BXb;JYgf9f#kcC4B~Pvfm8qm0Ch>jXTBL=YaeJ?+V|Wy3?O|a-{din`$%6 zF{4!ujdEwuN{en>{&EiP_%azauFJT|p02QPjUC>8Alnnz)=qmAC-T7E>iGIxFPze~ z2VKj&OFHiZ@tE~w88<(cRdhel1*H~vT$(GwqXz2os6dD>u;HAzD-gv4MFRXs;F6 z(dRi8wVR>EDHDEIpbjUZXVZ$|%SivDKGju?#d)VS&|&@q(#vXt;d@;%kk|# zE!@BOFgO|E$(Z)xE0m9Q5CX(YWRh-y`S(`9>u77C<#q$5nR{YI*&q1jx13(}zYU3V zmT}jhc#hNxM^$@I-jI7+`XKL?=sa#V&078fnl`QAgj)-__Lw|w+$S%+JYgFAxpao+ z2K0uv$u(fQCjhOydf_h@SC(CVMa3!3_~@WHj{f4yk-M^BS4tjtjr8WBca^cFvXVwN zRtUYr`{3X>RSG|;gJ*0KxU=6NwnGP8@zIrsBqd>3`Z!sr(<>^o-%I0vcZHP7kq|Yy zipJc^q=vqF1d{`~+rMQv-dB<3jRxUr>sIKOY6gE^PbAet2r>8sM5vzW1Krx&L z7RPaw(GaxszD^ONzk>H{MGOg#A;(|;;P0vb=wQM#@wksZ+#Hm~MV~gyX1BGt<;H}_Ru$11M?<(cR~2KV2dL?-GJD88nIF==3l#-+7~SwhxM$-^iWQ>xa)Kov zPuAcm$2Bm!@g7z8OF*yV;W*xV5~q%sMHer4W6~opI5Scmy|OIuzde(=Vvjw4ewmD0 zj=S=&ymZvs>xN}E1=OKx$yfhmV^VYqKKZR%@}2Z!o(~QmK9D`U1=>614p=Pl1S7Kpw9{n|UGei0DQUOxytN-D_0ENw z6AJ8}+Ltel8igIbhT*K~JvgVbzp!A~EWYey$GwJW;%Kg}yMoHiC&JBLhhTtuClt>p zf_i5^=sw}55RtcoY8@A2xHJj_#8{qw$DJ;}jKbnS$++vUOsE?2QaHL{xHx9SS}|ny z2Vr&7FzQI?!Y7`Y^5Wy3kh-;8n&j9l8L(E1Mu}~p;9m+`TJ5m5RVp+Wj&q!)+xx|BApXw!!ShT{z@*3O;j~iawVn z@QWfP2%a(>Yk$RIT5&Y%y>z8_${LV8&;HQBa;Y4d!R zPYmMS&OLBqJ~8(`>d(`M}HTv(|^_UH%Akb+e=`B%xCq( za38Fhufg~3Dsi6HA5eaBOFR+P3H`JM`ttg!pd=Rv8rfSQ{MUU*(=Hanx9V|y&yi@| zdjiHr0IQYpH|%xIvQH2&+}C<)oc@JIQ8K@ zRgL6T-2+=rZJ<=0893f`Hu~Qh$Oa3}gX;NEe$iEphm5@sStG6sF<(ZaeAWcKI7Ewg zdYq#}|CVEVx-TC5ZOrF;7JHnjzYT{Morl!rU8sF^w#4+3nGpIg7Y0rJMlUw}r9T%_ zG5x&EyDq56L;>C!835k&Jc@gTjmt!pp69>D)PaZ0a?O zKdK_9n?(yHr>kgo-F~<;vPs%5R6}9xHCX$!gmt$MV~69Zc+})Q)!nedr^ifi-BnA@ zOb}^d%Qcw1uj6#+O<=7qH!F8c8^U$}%-E(u6W>g|0Y;^3A%3+s^&ZomX2^C~-|0O;HVNB=43(xj0-_CZa@67S7(P8=C~bJbKoF5F2Aq}o!48VqjWjXEO6%beYG$% zUEqMlve`!qapdJVZqlicWYu-!6Qg_ZiOJ_+#imp;xnD)K%5oei+v8MD3xR%Fi=Z)9 z4c-_wQ9t8Oy0UvF^^)~^QwAxbX?!vGe|F&U+E-``Y2enhQFQ-bG1|(!>yr-+;>`;x zU`tUKu6q5H+``_2U$-{#Y0yP-uKy+kU7jmMc)o>*!VAKYllC~5h&vwNPwez{Zf+t^1jmJ>)3}K+PE7?3>Cmfa*!?lIE z;#;{QivE0<ocyRl+;9UT|%sHYYbY z@a(!0=yAnQRJi1Xd1|}i{QI{gE(?H74_9zVTq3`AO+aJoMDgyjp&ZtwyG-Ac$A|+x z@$QWpx}-f8HaMSwiiNFW*17A@ql{pMSB0cCZWTm)R%E#4! z*jNE4xk`-gn+6LKSAfx{8uIA-10Jdl;yRt?$|`j zg4a=TRX3hxwiNcA)D+^hWLT79!iQJx5!=(1*(z3@yn6khihDbu)6SU9wjYJb4RPS0 zR7aCc-bw~{=|{Qey0H7nRnV-w7{m#{N@pfeWI&#{7o0YBEjr_DFg<`Z%aJ ze3LlU4FE&mX5n%80BVUb;*i=~^zGPiA??D4`7KiG7{jqaTG!$R|X zHon-GD=sa^7Yf7Cpgogkn)&gcrpxf-Vi8pD_T(3tGS5|(Atu}#ji+pP^YxqS`F%|~ zZaeFZA7)QyqwdqVa#H|?ojEMyaf2{??jabaV#!8BOgOH!52|PepmT~USB_Pt$4&#W zpj-_n9LuN27Xo?W$M0b0`byRxE3zyGNF1f($8Q^Vkjtj!aDIM2Q8mg16Ww+3+WCZb`fGWr#Ck$fpofQ1`fgk!#^z;t^r-h6ErIkmgu@U}`YI+x2g z3**_!FAGmjk-$=i#o!q%1*70la*FyxBOkp5zoaBV;yjA3O$p`tT0g8;^Ff#6!zq68 zd^&MP3jf^ugTb`lLJRl9=nz{tutXOYPWvT&Qud29uKk6F2V)`W^L091x((iI8}X{G zzhUBeTUcpOA`#p4jI6w zO#NYT^Ec@;&D+pYmIqIQMx$ch8@jev4u|bNLbG0Oq;QSlGOV$nwo~iGcT3#Gsy%z@ zPyQy(Ho3-P@_4%Q?7ui-k2eABwd}8T5IAU?Vg;s+<{&lT?px8B84Seyrq%qt6|H={;=1oh`ipk zL+yi|;JZCU9ONGb*6#;UZhj_7Y>o+MRZQ9Ep)m)1I7sIXs^GGV{cz0h4ZQtmI=p>xb0QiV45KJ|u;Yi^059Egl2981jmpzoCz#KF&6|NMC2J2anUMAUk6wZ1n#_ zpX)rxSUd~wpO(SP8HU30aAn>yIE^a&^!fU=Z?ICgA7tu#3c~d*VAy(}_Tw?~*=!F{ zUvgl{m`v~(vXu@N>Y~|!o8)-dO149l2=9kZ#c1!>K({Xv*k2)|jzHYK*9D)gabu@6 z177rY5th{X;f2A8JnGdZ`1rX13kuiZ!JJIqu}%l?gsJk%$Ge34_1V-z@2Jdwc0x2y z>;gu;H;JCLM_^v#X#PCSfjzHS;||&HnQ{G;*km)Es^3RRLe{!Ktj` zvxC5(rUh2K?uOnWa;%wdgi*&d@W(C_eAwI}PP5lRAJutWU2VhbntS7kb2DWa;KS7b zTr;{Iyb2x(T^CD4mAPx_zz%P`YH|iPdA$|)MEnu{R%>Ha`FJdRy^c=|7{%`b2cV7l z9qE3JLBjmK)xwnXNuvCr=QKk_r1nc~G&fl0EObyAHm7azSWS5U@z` zK79XJKr{ZGhB0u7I!9h5|Ex#wXKWVKY?&<4yxA9ob9doj#0N>vmLzy_`6+nF{6B*l z3n8>sHpA~V6s!-Pg_UXjx!(*c45=)KSu6U|^=0jJuwNnA9^Xj|S_~*`><@9&6X#l zyF(^VZk;yJ`WFK^vU(Q1io7hhP76|B$9dKDdeIs&4-OyJVGPVy=|Ax%rUO8;W2$n#Vv`zMu4Bp#2& zs~&%aXQ@(XOlzU$SSJFH+wdSjMeOQi&hCQ@`G^b)Ui|Un4J(v+Y-W|XW_mHK`LjjT zw^!vvFW{mjmiT|%8#U1#8ji{EW!WcDt@kbp-5!RvTW`QEhgqWX1~b%_c@_gE8lz{` zPTDy^OQwaSBL2uh$D0OR0jUtBxdqObZl_N-t_yKK6Zqz)HQ4*~YK%TQjfY+TN>?Xk zVgC=a(7t0L|J8p)dp8xrybL1>dvC~7jPFB(=3rhD(MFdeb+NU^8$1mXcu#XQ`@C_- zq>#g~a`QH_8N5n({G%r=m}^gCFbJ#s`{Ezn4Yd089YLII#UbPN(=1(mJmRK=q0J02 zr=$@6H3*(v>_bKeBhhQ$E>K-uLr+dE0cS^d-qkjYT@lg6peGKljo}-sbXoVY8A_)g zr6H&RFT-EJj6n-&`wbJc)F~$o3u7*@*TW4Xx5AvWR&4dTE7RaH__zKP`Fkm%(WdS2 z@JVm6qU|lbDN4rBtzoEEy^>$%OX1<5=41@!KbGk)&$%Cvhvp#uMoaWdwncf;WwGiY~&Chkj8IQ?!{1#MW|6&>z8hp)mEHnKZMqnxXt zapG!W&e-L2CzyquANGh#;@inp^C#Wyl}BY?B~&_kGmIaU&!Dh^ds9Bjty~Wu?KNSg z!cTB6ohd%M^;nz>Ju&uL8zgBSqPWMZka*z-WF`F*j$Z8s7Z!gNN18IYZ+axnc58$F zHqLluV?LjFp2VAOt-_%rH$wNPb6G8TBrmfZhlX>kxOZDWobzFXaM@=mwae8?T>j+H zPvx!jKK~fJb=BiR@Rl8L#ANeh z<)s|6JPU`7cgB&@JRbXI6@oB3%{g0zF4~P1D`?!54 zL`hkq5=D}-%;($_EmDM}Qj`)&JL%hM4T~f~?)P%e>umy13be#~=Xefg=*X*)Kf-Z&`7 zeEi_|`qzRhS#V>>d|iwPr)B7%`I5Q(?vrAl`QRMBhToM?ua>-pbtyiB{B6Bohey3jr?C0^qggdFk|pNFsdvr4hFjE&A3Dw|ZgS+`EPjIXbi$71_U=4q zCqSmv;p?+Z+|^Yxx~Rk6$U~hnWAwYR(r}!mgYoE zeKS7?h0?9?yKutS@Ik|Z#HMS$yC<1Q(^V1 z?(O4%{u+*z>@wEkhG28&QE`^KogAyREViD-yH`K-E!}+bpR2iU#IU(v58t2J>1W=X zakuWwJw5YREqran<(W0kD>uJ=F0B4`X1lrU>$B!=L;Fofr7qXq9(SoLJP>FmJvFVq z?3d+@b(+$gCAQ01rdloLn{FSg_w)U2e(b_Nj>vHn&cjP)EUuwX{XpO8`ev8xdhNKW z^~ZYce9j5lXWf5$>uX9PnvUntzabv`S8z|uH_Vp z4V&%FSK#=tmvM>|zMJPYy)<{3a@1V#W{~;fLlNcy?dF^k<)thWl^f>EI}*%B`De5U zzZbDSs?+S~)t%;R%ei$THldu01!p)$%ud#&`2qED4<6K2^>XTe>sr?PvwCZ9M$F?_ z+)?J3>6x(h-5oU3b6m;!5~s=8NpIAh>aeaqbtS)^>@umJJ3MJFzVnjV{3tC>(UXz- z{+=B3w*~4P)`pAqf6G+NgSY-QbK8@GY@=3u+NTMmE(C)+8OYjn7V?EGp?%72;$d+M zuUyZD_D}BgMd1wgynzrf98Jc=0}mka*E47hJA>Ajcj9jUU??hbL))uj5WL|rIc1VU znDhEPo7wx(Lu&^p2iQXH^{bfd=>hXs^H6zeFS!)I6$>`Sz~1@$A^4#cxj*3wI%yJY z^A8uP{yi5u_^|**_AB#Ltz2;37crhvcM(nKoC#-U2cclcH!A#PjM{CR1_Jzh=Hp~a zafw?9&KQ*9efm8Ebv7=ia)LK7dP*!~vt%a9zgkJ8Q&e&3=Rt6b4#SAb$7Ek*IVy}x z^OQs8f~U$gYCLTMbadTmO7(fzneU7H-8dY@W=k;MQ^9x%&*Ux>GvYaQizCPPH)NlY zAg_CeLD($_SEwI^eziCFkekHRYgiE{mmD%Q_Z4^hjuu=$kb>EZY-oJbOM0&FEMz1a zp~Uz>viik$xUFW157~osnd$`h>dIB9AyET0Gzn}w(=aME7<4o5Q2oNc_;&jx&Vz3c z0GD-R^6VMx@XB>)boe*yzI`3CcB){MdJALIkW8{*5jwZVaAgIOsA%ne2%nx!a*mFX zM-F`EdZQH060Aneoj~NCM3Bd70=!C@bEvleBo=8k(jM2F#H(JOTc0;dG_s=bs^2i< zpd3cCzptY|_xym+mS$Qqb{6}%X;d;_sMXqqPJfkoux$`Gc&>+m ziv`r#RE%vU@)?4kd4fQIAulo921>;5qp;ip6!f_ZXV{S>ah4E)OhI0>!4T*_J_d^) zyWw+tB_8(WfXqw&_c4y-Kdt#_sQ8zU;e1Jc&x}CrhzN3k%Z5{zjG;AwO9Ug-F?+I< zWH#4hkgh!5&daCr2`lIhlVDUmyAa$j{exY$hL|C<9kf~<@yi+kuRv{?(MQ%;c)v-eP8tisdji9vzwKZtqVUF7|}P2XQghRI4Da6Fz)Lnb2WT-$TF zajgPYhu;Njl@#v%6M|$~g#-6no&Zm?P>bZ_>?5~zdSR~iBM?8Q4XM_zVaNU{ zL|owd!1FVa^VY_o-an&7M0u+tKSg0J9)TrXC?K&BMl1`HE^ZPX_V?aIG&Y8`fn}3Nh?RFHeG|0 z!}jQVJObuavEWMJ3+Ae&3ORdwf}DTl13JPBQS-hj@`H9fGUYkGb4kV*%Q|tcA|>Mc zQ}L!&98~rHhVVC$5bw4N8jg$;Wj>3oWVjk6b>kVi>;ibv;sdU|`t-*2ZSY^!Mm#AR zM)Pfoz)Mz%o%DVmOs(u1x#dt0=ng0Ls5OO>7Ou ziNs}NGS{x0-YdR>SEAkU_}5D`O>hj1^EcuBNh9nyH;ifp8>w`HG2HvO8yr0*vHLfN zF6iim_t{9oI}Z>gZ5{Z$OAQ~hlt3VQ4U|OBMV0H*G2G}WESdTRwJx;b%BfShbv6Hc zb~ETT<36bL3!+2kNeJPHql5lCQd3ZXj|TF%=YPtPee-=$*hrUZbX1Xn{w#d;pB=OM z{c|Q=^D#^sf5#_Am+^7=dN}S}2)8ujaCPQXkoy&lj={!eD*r9y^Jqs%wW$(FO*Mcy zK}#@q>3^hh+zrIycY=NRJg7{(i>>B?c(c--8Qh9^cEfj2+6rh!^cBE$)NO7a`SmKH(lZ(xqNp6&TRWjtUt$?t4+CvPW#?) zJ*Ej_ve7yW%$|))MPA}xXCbsGsK=oN5vbfbOqXnu1Lb>TgJOdOnnA^ zSltIbfzJ4ntw}Ha3xRIwL8gzB41$iLcv@o@cg^2#bfoktwe!lsds|A-xHS)!R2t*! z-Qqa2+yxA0*fCcvys=hC5y}cnxJR1KGEIjAxz3r3!QTBE`fEC4mV^YSbcVh$fgrSG4qQSUH0K5ITGdA=eF=|8#jbuRS}{10@~{TQbYHR#(pmDcIG;bDnj zBAs4GPmjp5-ie)p2eP7=Bm5quU2t(j8%+ z&r323W@F&))E zVs`fC;L-LzNcYXhrWZ!gDDZ*aI3SG61^&TZn>X|+pADgYz6tiFN75y;WO**LFF?BG z9XJ4%D6~tEoNrmkmJ?k7iO&^jjj;kTiJA{rrul=;rgpMrY&rJbok1G5<#G#oO7Lm# z0@m^Dza*jNCJY_)gPS&*Y^6YZ=B{NfeyNP10_!6|r{@{&vz-rppH|>itz5d*QwOz_ z{NRrLIEj^hiP&LCz|9$Dyq|GT3|=5R^7LKHgq{0f~rv1DLY2l#Egj{pA5CsOgA zv`(Ovic4OnoUf|v6RW22rjl~{_dp=^kM^aLwfRJ&Sd`lx_Z<`0Oog+3131tn2}_2} zz~d<)@8Zj8(6TbJXFEZ%Vg?E)tS0(yMu-oTajI+uB&n2)_0A9_nY@L$AUMxNChZz3IbjV|L~hAgZGyu(v8g)@LtuG&lk%fhs;%o zp8Mes;!8nh{eP+6Y?IDWE&XxTg`TgtoJU(`yR1GH0YJN%bFO7y&=x7?0bej zNm?K$_?EdMriE`b7K4~~0hRRDgpd?{ESauEo{(9XHu#c%cPrtYKh(zI9Yhga(81S96akw8XgqGnh-hkEUScySbg@^ zo5HZj-YQ(G?D0E} zn-W3oYFp8BOAe&^Y;0!Nnh#SxGhYmP0nT8l&NSj4a9uXM&8vckHYc;rXp~ zM(*w1Fn&h@B}1Iy;tU?d*L)%B_b7?+ivo?GXF;nW6byt|Fb7(hZTb7jC*~wKT&f0_ zn(AZDl0y*uJP@Nc93%slq0H#U8tyy(TAxz3nTTIg$BnV&6x~GFIm-=TwOAuO_V>m0 zL!#`#dJza~6r^iD0vu_%N|r4Q$IOx)#AbCpypk8gJE8h0cjp$0o1TKlQ^GJoIh@2F zH6+3z74*)R0ye54-(@;Jt-BFTr6Ixg^Y=iRNdJgc)aXRo7yO@3( zgZ?s({l(BYZv-Ch*-3XL)x)v4NBB`t5f3WLqQ}r24DFf)2Q(J4J74jLpzjOjXg-0u z1Otq@lm!N!E8xYgN%&}6Knf)4aHeE8sY+N&g&&-R`jH*zc}$sX34Q}gmQ~1Jn!-(d z6+vses>$%*HzZ(Z3C__t3HgdIa5D;E^X#cOSfhvnm2z-s+9Z*#E99@MUZQl(g5J9Q zk&d`8hH2wxq4!cH^?fiM4$r*+3*2MDB}0$>Us)NE{~FDCv+EAES=+&|+`Vww?^sw9 zR!!JkpdVL#A)%`o+83+MJ~7ILpRWq>N`W8VTOfx8tv|?%7X&ZG9H;gVv{BK4e_v{s zHZB`1XO8cX!_yUkB+Gr6J1JNRKkDWA>?=ZJI)cEV>KfMP))VKU6>$ISW5O$C(B11Z z%EyP3ky%HvJB0;zere&}1IuCH`Vy2F=_W=#rg)7%4+vhZAbxyb;dE6n6L3(B)U<8| z9{T}vXMGHjdpeE2>&`?`&x=?p+Dh$R*x2&!DBowe1J#eJiO8&HWV`)pQX(bEJ|Aq2 zagKdxrLfSvxkHJ!r+Y5GJ?##S8P;I`)q~sVt$-18e3@NW1MvC7sd#OP9bV?Y;$xK| zK;<#n{LK~ypU0Es3vN->1mrvN<=U<0DW2yqRA>nwwjtggr1`)$oC3p z;SxNZScnc5*(A{BFIm+pfZ~7d@yksF*0d#YnM0Bwz9$%Wob!YRDWstaFY!n>MXjiG zRE|GMJljq}#c5%7LhvQ7`H(6b#}c7-*)q7RD#Xj@G~>XCA^JSO1cS4G!dFEhc374V z=h4kLT*XP_Di_~@h(S@d=JmrQYpfB2jr{P{VPV*n_7>x!OCUzK0{?zbq$i7c5Tg}C z;urj)Lko>DGGM5V7PqUwcp@P6qZGOkz%BF}`t zWy31`&MP6GM^|9#DK9#!{SV#edxm!DB%^l!We`0W2oi-tII!9mR|zf!fw*9@^s+RQ zSh5iYw3>-{wGyy{GH`oM3CZmi;!QkA!P;FtnB*YJ+xiHReEA9UEXzPWuNL*W(ZpgC zk6^%S6yke@VF}@=l=+342F4}d9=RuWuA(Q9hL#wsbuz6M|31L#{anWjWb;@5%OwUEb2jg(A_%jSlUrd?X z_K;FDh)xRLP&PA#JNLhF655#o+a%+0(DWW>jx9J(ObdFTdATCArzW7HRVk)T`HI=2LFhBYXN$|E5#go?+FLpgTSW#* z_KT;`Jj&-C^>{P=t5smsk4KM;Oi%)*%UM|L&%O?ZU0615*ZL4z`aNwN#b z%&2qlX$=A6LVf&R8;eK#X7e7t=I@0E^l-}4FnVe54E=3eh(q5S@o2g>Y1|@*gBo{1 zrvEE>ajyz?{t|*-Isq>h4{$w(gPG06-%;RuD10n5!Fe4qESvHu=sq}wt-qEbPs>%| z<7p>mkoiL6m0UrrYMA=;oaY|edWcxsPQ?Y)Tq0>D#ZzVX)9xNA%qU)nn_Vtqo{b!R zG#rSF9)5zC4@S7!LWU^uUpLj<`HsH5Rzc3U+0y4)igcmYI}orC0>9c)xk99R&Ha^H`m$R7>-TxE^LvKwjqK1ooQe#OKY zcTiuU5^V3EpjV{j*lU~P;qN~w{1Nj8;%8`}XP7^>pH$-gGoDW;MI51h=66)x5Czq} zE7=!Mj52z*<@k8VrjVW0%6R9Q9Rs(S^V|ne0U#TlD`1%zx1i8|9xho zRVPU-szzhu3?}_}E*Ka4psV^P(y6+T?u|-;k*+wbJ~ab&tZT)bA0L4V-Jp(YyJ67% z9QBUua<+neZKU0&UQ@F5j>O&KZ1TQeVoU zhld*3eR7hiKBPdW8bY>T40z!hNb4u+TeTi+OLOO}Qn`%$iy z>?iOuZJ-Ug*Ae;b$JXXaY}gbG9}*L(w9GQ1q$k6~Y@WfpFaHp%T+Ok%&UkC=(dAmXuY z8fI=tftZnd;F3CrD(EVL>>NYv)QmxOM@>4hF$#qLJO`tXZ_Nu`2TtMkz_c?6m^SL11Swe$KQ*oWKD0{DaDI|YC2hK6Op?LN|!d#D~vnyt? z!zAB=gQNhRyW}^^@3SzwD{>IRW=?|9hdo&NT7WFOQ9zu&><8O=OWH200>%1!@lB^F zec3yeew?a=-wYHXTc@27$*7?rC445`+*{PAB#D-Cty!trm7q!rbHbq2cLp{+^7KHODGjXa{J{cd#re?hoSpQ=?EV%Q6aC}C% zLSxry^K}o{|4|#CYlOiW^=f#K96^n(@<8I91m?`U1CKdzP_ZH$`;v;mE#W+_xA;!B zI|iX+OfLSaSq@tbcx1zZGH~#i$@S`t1=pY;+7rDK7T55jb$bW+cc_t=vgYAC<-c@> zTM}(Z+&3cO!VH%WAmfBM(R`Wdeov5KYObp9W3 z6Psw51}hGBvCi(Db0>9M2kzmNa=6|{qZjjTeHs6oG898=E_0C(kx6aSk9=6 z|3*`zNSqaUn35Z@xOZ;|oqT8vMp=Xz^vZ`Td zKnHS$p*4@e_?SDiYJ4s@>^lHkl|s=yO^BDJQ;&g9obX(c5v*2yNJRZ_!aFZtnq*wg z6^Y%A6`?YC`ppncI`tFmyZM>&(;}K?D~}Sx>(DQ*6gjg?xw9|aBwCx3F+1}c6qfPV z-}f*$viS=b%rC@^N^#f#tD${G1zl*Z%^PUcf_tkLaW80=0jqy0-R@aQ6}yDN{+SGL zO6pkmcbYx`Ukp(8KIW zFeqNY>o>K)R2mDt#YfP8wKhbuI;q7kYgkg10~^Lipi5f?d$&A4eojlBo3qd4_$Y_ktbzY8VRn!gGJJ``9eM^6x_nRnYx^;zHtQK?a%ed`AOFk$ z#9VpS4DFAff%=Yf@T2?eFvv_RZN(7Bkvk6 zbCqU~Xgo$;eLLd4bRo5#6$}_MLeGAg%9Fex${y)|Kr%Od?Nm^=3y1liz zjYB1*vP43%rIJZYFV^SzYpBX!XOxr)gxXJ0tY(P~^y`}jI&f+Z{aB`hcO~OU z%idPjNd5(qD4K%P>dwJv<3k$uFpz%cPUV?iQHF@ltvJ|z5?uq&(%23O6iWKS%#Qd- zU#|^e?s1b~jchU;+CBq3&!xeJy-D2F$-)@zev0az(q_M$x&l)4>S=A$9!S)zgyqRm zbD;17CVFPE87f#YR6$_eI6*-#X(A=JuV;pz$_=d_#&%- z{_aa;MWw}JUt|i6&6$b9uS(4aW1ez0ez3vrnL*gd=_Xwk{A@mY2Ho{`HgDC->u}vL zo7qzPkN*6#3RUf*=(htl7@}Yb@xQObY!NqdXLAnKjCX?Mz!pr*KgOT+4pC#7CCqZ8 zJi0A6hk9xs0k2>$6#u=DhUv)jnr=QM_I$tOvv5Bcbo?T(l4pY&*BV(yduiJ5c}SMl z;TJs-)RpfiFLq9ar%LfOv*0TCh0HW)`j~=GnQGeTX#gMPInd>^0Ag0F!O*n1bgP>j zds+MvT9hThTNzo0ZVq!Wr|v$J^lS>MJ<)>ynib*reRmtKOe;hWLQIt4xU%8_kB-b8r)Za$|rlw_W~L>`s7 z!aBQfdSi(z3~q_Q`VG*ABNr4J}`ZU1-`2nBinyo zAP=>_^Y@z?qUYa;pHoZFE58>L!}idEhad57raF!$SK>#}QR?RviBd1;bG3>VF!eci zFfTiSTVNgx^?d#H^{NW{1Qf!E&1L43aw#?*Q(zkjnZnn?Em%>s3jDi9I29i)_}!eJ zT#-#Hd1)Py%%a#RnCWMRzwE-{)osLn*(mfdT}(D-%VWdJ2;3+vNDS*jXpjC3n%q1U zGOM!ba5Ul@GV*%S&tME`x1gvEm)NDwWV*L*ndtf{Q@C_9J$DPaySn10-^>IB?G`fE6t! z7^?Udo4*Z!^!H>a)$SsCJ^#VPxG$|e>W-$Hd%y(b@^D75j?Nx?hQpSJA%2MkKK(#p zNIZ!+?rkRD>`b5`&w|w6ki(=2Z@zCfAJ=wxgY3o`@MF(DqPr^wLbtjRZn^+HFO)RM(~*7M0}k`SR6FqHh6-oZbUP zG#!p~t)!xv8RVw(T;l$2Hl@mS%+0#9xJTnYF}W&$wI-G{@Rv0E`|3^jy-JoHp01Df zyA^niT2II-m8o3ece9AP{z|kixCr+acar&EZllo9Thh@ON4O>eaJ`+)9I&1Z5wnkz zavveyf4?p1=7|KZPs$T`_U0PKF0O@LyQATd`c`-1j->sONFGm88)sQ8*(PaJ!L!RLNA2@j;2^Zym z#(0(i8btEJnAgQQ!&jQ?$ECI4^WQ)QJT$?oE6M!OwsCNqei5wJ)^iWVOTmjKF7#}x zB@!DG@yo|4=(=tePe)D!=DKf(MuR*&&l7@g zf%80!sJLP&=hUtQSlyG4^ES++Z7T_+qy~U}NjIztnuLR)<7gOnoyO5BRCYNHt2aau z4bMV2d_4=#R@<>kQcpmt)Gchhm50(Fy3jQACi8+9gtG+K!S6k(+;ZL*(AX%(Zr)Lj zn~c}d4SQ-)#8w{emG6V|gCe|`=M={GM&Q_`WUhtkZIV|WL5-Gq!06~Nn*LJ<46PoL z>@Aj5Zb3Z+S?-5KjcU3$^b+lPT}>}`b+erGLa{q66QZ2=!jUV|Si3KmYV!S&9g%C% zs;U{)G7agaz4n+oCdMAkdc=`WP^HgLR1n7+4oV8Kk;NM44BvDC)lzf?v2@)x|{bLdS zcoBMEi(e9WClH%#s7`9-O=()BEXOu$n3mJ^YeRf(C!*=PYB{& zJ-+@q(}C+hio*{7tE5C>kY;Z^iW;_iaA5B}D%ke|Oh1L9>$oiYNPZ%yJ?W&kRQU5L zTL5p8HYnXPOxp#z>1n-d{ETIUB)UGw-Q*(Ox8f>M`4IyLYyzOM=?28LKgNEcB(Tos z(0Fz$Wy05>MBo#8>?aq-54MttfLkQ%{Y?6C_cJ`QI0iCa>afDAcF|1((NrhZ6PADI zhKFsD@LcyI9;rA7ekropsF=!mK3U1lS&@d4y#*xVWeG@Z%fZ2IPE<-~1zxuB!Dx$L z^#YHFX}Z1$@2OA$bZJhZF+P0XE36FfTh!C(Caq*uqYOOMAA+4`i^($6Z^Vk9_p z9a*(^A!d6T8fy#^?Ymp?+w#R!itim1Nr{8KbuMgrIh8$YJQ%YpLvdVJ5N=!UK^;dE z>Uy_=Mvc@m-QgRs?(#+|Z>J4Hk1kZ*z<7w|)7&y=OE-d}= zMSvv~Y6@b-?s|M8oJ_rT-A2vewS@a|fL^c^r$^{3e7iIqXUq@;kIZ}g*)5m*VA(e+ z(5=AxHQ<6%{--781wfo=CQKZgLcwAQ7|1t~T{p$4Qs4;5`z4AOJlCN4!)TCi*GB)U z@2H|y2-_4Y;gygYuYT+$v2hS#dri4SijGWyM=AX;9jC@#&ONwGKe9JxZ z_Np+E`S^ormJ)-qhi=>CTsCY;mUliNov!~*0orng= znl=e@yNYqiTX*wKf>KyI$YIp1)Yx99gGiq8aY!i_LX$aexbglb=?!+63K_z(9HgkGLwg-JW-!Y5qQyaRu zx!-uWW9KXo|KWyKI#a2h$Z?bm7Q@};m&yHmt4R0gb!Jh=8i+U-gvSg2^2W3SEi> zFr(%hHr}OGUd`O`2Qfr`-c_jDx*U$C%|a8aO0519Oka&l!TT@+61%P+i@vfTH)9ow zRvS^be**9;>l`OTdJ8t6yH2`Km&2;ye|%3{9|>4v+G@(Jk#T#0Z@OnG=~*GV_1`R~d42Q7)tIQ_kr9 z_C>Z`6=8pA0NFRE=y**zecmDldjIQtg$SX6$U2Zbcn}LueZhCkO1f1b70!pqffzdn zepP1zC+i;xUmS_ApN~@VSf4wfn}u@Qv_M`<2~Q7aqJ?B7I{dvySEdN#7TG&k@0-R9 zwFkgwyKgMz+tIktMj9SFS@D|HdYSbpK7@9vP@@YkI4UWW4E~+NGfe$O_SMMn{NI$J z-^e5_Y_tarJAR&eMjifjzNa7N{3d}t>xu8^dFt%FnLBaI1=4?L@Pf1-)B9_*fN@L! z=0zcxo>AoOw*ACxlPTxsRGbG_1r5mTzKJ(F-cgO&O@zdi5$}?VFvR-|Z@l8++sXp? zJ5hy8uUSH(m@k>;6-VQ4tN_EAr7&-$HhhToAa0+YljV8Mw9qRO{=3pdE!G=jphqd% z*6c?~doNUY_8RhQ48SF*5_Glse)Qe|x^O%OrZg82`>fS0X7?xZ^k@|5oeTz*vKk!m zFvY*O`FZvm4|-DU2fw+$0DYB?;?q+|&&vBF|vWlMu2|2_T@$jv7zVLjA{EVP=&J&L!&b zd@vZRRD{6OwhLeR@R;Xa(?Q;NG4`AlU{CiFfSSGH^>6u|$m4v!Tu9B34$M!2hHdVk zmRkgi<~-t~yPCOM)5D3Z;Q)FqX(2<8_&xR3N946uG0dy;g5?WUp;h`AJsb2F%Qjtv z37^}bd~PR7luyUeAVK!2UBS%lt_+aaUkFoNzS3jIv@ut3E9~5KkBW?oqY#}-)Sj=0 zFJp?}KbFiDO;rWD(uNo-EQO#319r$cExdMmgkE0V$>kJu5SDx^HcaLB`jqBD*y3+| zu8JLFicOHQE(8=8l;hSr7U(WJ3C<^tK=a-w6z^AI*Ub#ZcS9C*@r50<>99YNkUPY! zZa2|PNsz`34zDjKip<$6?XD zcq06{fs%oAdjF*&+ozoi)^79R`iU&as;pylYLX!{B#jL2dk@T&Cmhp|oAh5~15DfC z4;q`gMG%bm4uW}(E6RjigBeLeh#j+#^|=N&o65lcdIKEa>B-l8FX^E_ zs+j-Qk%r?@(6wK|9n8+7acUt<&}RSc1BE#YS!&vDa-1T>pyBj-*x!tbmW zmeIce4BA>xmhM$zdq;KAb>T%M$QHo7eKR-c;t}M2NCStGe!8Y98D>*t=5ANPJ0nH7 zz@Qha#3pcz-#zOz(15UM)5$c0$Eeu8jw@iC2d$aEh@{dQdTpSR>#in)*$E}wXRaw^ z?J_@9u6Y47jd$_$uQRwb!;Tf7V*!E<=V--WH8?WE6aO3(WQxb9Vf08PD4mqzEnSmM zQ;UBva%J({S)Xsx2=5`X4kGZ+qa-+CvJ&(ExWI?z8Msu|6T4k+<4)hb)JxYIn$QiM zUWyazYyY?-YN>c}RRGuj9MHZgLXcSfg}8WBVRMua_(!B;mSZyP_lm)(`rdHxc?uo( zWU%Wbp;bZopsY2D}rAk0=PuJ5<32C;+{*4gssQ_k_QUQc(-5o zKuB{Cn8ZtfSHmsheL9WIEgQw0)g(CqnM(C6W2ltMG37 zc9O5eKbLbl58g9Y+|L6oe9b3_C1V4)O;QLxDe2{KyYp8C>OD9eYjtUl`yYbwkeH{$Ti~HiQ-Yep{q$CazCyAzOxnVY0vj_hKC? ziHTsdjxNSs?j*us-)yrDlW2p51pX_7`QS7kDExaz4)3;`tLn(Uw9Q4oYlw6KfaQW zCbLQF))nwzbPArhC`GzSGOSOLq7S}S!+HloX7BsOY`8cF^P>lFUT6oP{UCF^=^-5( zI*QYVo6y+L4k8|RayGZhQ?-y3h|2whv(FdN@P$zrd}SLvo97C8nv(SCpEzbBCj$?Q zi<2V52e>~hi%g{r^ymDw^iuy3Fug9v+vq{*_wZQeReB<+NJt_Q2G41;i#+d)R}j9Z z^Dv|>3fJC_g(sIlEqyI@3RiuMIldw-}mP0>YoKA zR4(BiFP{ODhc3m>$?=c|t|o`C%_Tf=DE0^}4Yz`y zeWE;>Z(gXi=^{3o7L!1aEoA?zDRhp^4?63uCZU^^X(GRq`=F5DE%m>Ob6)9Vz=Jj#CQtv3dN0?%OQP?SCGH?P49KUm01fDW^YIq^Nk86Ypm#EU_0Yu}xis)qS_!U-L3H_2E;csSf=x^X|18f_Y*W4oC)=-~ zSXB#kKYNiWE?2-M+7@u%u^X!w6f&|6i(yQ_8{-EeFnsSxTD*TbZq%O+6PY$BFSZ4y z$kmaor@zm zJD8y@n=rXd5FaFNq0c){L0;r<#_hE;{7!8q8JTs|eG|dp3=#58Bbj=hSqqV`jWJ$M zkSr8%g=Kr3=t8+Bde}LSm=T-iHu*gXTBZPzmSPf_6YJjT4zXtq62lc z?BZ6)@pW`cC0Raf#?{N|CnF+va5V5F{X8Oz`)B_k+}Ues8BW1pt+S!w(J%6^+nnAi ze2<# zAqD;~Q|VYxGHpJ%59~zwdC)C>-%aWiF*Lr!@j4XDY~$^P3!+omj=Q_@g>wuEOkRp* z0W+{ndp;a(-w$iJuITrL1y4`3L7CkMT|2x9p7&(HiTVnnv`v^7eMp0*EEy%4u^qVK z$u;b94+isjM$o1Bi$3f-0LH-?sGoj-1dUbG$E($Os__!&zoq~}w=^)<%NOF44?uRd zO@mtHPBf7-#^tsh{C)co-F8wCjJuVn-nW|Rp8c8d2M3mc8 zFcMaTw_)%mBVxQN5Sr5-K^XIrOzeM3qWQl3uS_X0aeYk2_1EL;6U8j+u`94VtOjhV z4dK6+<@nL)6kQ>45r5iL;$3wkoH!vyXYGj~AB*k+YpEs9TDFACvx|lUdam&FtRb3j zspiHfCWEm`4z>I54CuX(B9TkaV(PIiOzEB<^mWWXj?4iq7;<+b>B^nNj^E{MXlrNI z&5HxeZ&h5Yret$`Rgdld4~UyvIokd)g1%kCc*ej8AH0vlVZ#L6V!07zoP?;vun{s> zs$qP~Rb1b<6)vq*0Pl@;jGV&{a;J1J1RPO=k{`abDS0i*28S!~+^5w^ZrWCXnrI5+9{C?El4pTYxil&%2+}3D&cek24OA_UC#84HfmNJ})kot| z_m~xR&rf2!+&S33u9okW-yug2Or_tRve0FZBNaUJh(0#dM#uG`G*-Ke&KsYBuPphd z#KbZ5X!#6DXXZgj&jdD~c}g4e)aZ`tU|gamLX*6v!FkpdTCq7FO1rG#vWg!%3FXr2 zz8UzhxD1XR5`lLgW59yV<>uMnrP}ZLU5p!w>=P|fn46{n9qKjsBzP&l$lpk`>dSH0 zvlG}|c^?CKN%%E&i0J|qv%W=sd&FIPWB~| zrDThQ5ZbtN#*7H56j~{3D=pd=rLtvB_LP)pvm_L`bHKUi3PT^RQzf-2~E7v+KBg%?p&=nT3{JQ$(+_?>Qyl$HhYwugg{dY2kkNP*Z zTH0edt*EC$1?>#yI~NymkN*;Osk^=Tp_D3m`L?n2E9ZNho1X*S%z5+UZ!7Sh*OXRG zKkZtzxMw;4ZqZ!6SaTfrBQ>3h5P6v@rdd$pso9)svmW0dUPNd8%B26yOty95cT?|e z6{zgI&9wQRqx7cAaK6Fh9AA|tZTIr5x!wP>H6Ox9sIPYdXr1Ku^yn{fy1+wo7oHuRdqHvG^6gqeh?Wl-Wv;^*zK}-g2kM znjE6(3X%V~wVtZfd&{X?N>gVw2Iz}zBUSl@$GM(6vHZW==QuOh_qL7&MrRB^LXtTSc4v5YmzovD^!vF?R6_>*?>(D&({u z`}u@`kK8racWm|~HIXL2kLz>J;-gc-Xrq_MdE+UoXsPM9=x)1*bZF%P`q9`_YO&+F zD*ZL#)V^|_pBZkzOI}&Q>x?PjU+5Bi!iQqYa-SA|OdXf9KvFY9YIaA2b=u`=2?@Uz+)is&08!bt$rperYPj3!W{8ZOjlIc4Uxq z@;=6k6?^h#HP&{SqglL3_;tENC{t~96iWUcH&6Yf42v@0<&KaOg2xOFEzTyB0#V zPgJBGXZ)eU)+O;VJ7?3uZkhD3Mjd_VbTvKgN}(f((7C0BPU`HTwx zLwy2$W9vp*MM9RIbNIRFTieC^Cg#!&ej-1(#B<7Niw@syI-Nf5QB9du81d`7P4Z;pxL4GVVCE3fUO8{I@%h&gez zD7Q%W#Ai}dWjFHyw~q444=?k}A}{grtqnA>(3E;NCSSw@5%~@l#_-bz30`NRXI16P z7%tf~f`0BDC(7F$qoylc(7GR5>66_rY&Xm?qXXoNsmokZ6?K}YpT1j7CtG*Y_UGj| z?pi!`LF9+{*P=>yS`GYjcld3v1e=OaX}rnp+|LSg#)+f3%xVx;qyUsZt7`UhsBw+ZRisEQG7j> zpZ%1&kfzE{_-(@<{!+yI@)^8R#aa5mv2tEsrHk5-lR|47wsU`{9plQS-`n2KFJR@K zJfbQ+KhQh31@jL)Gx$3{CsjW^@tP~R8&4;DI??u@=F?HR+Wh9SO77o93^X(>sh<`n||sr{Zymzc1HLzq=O2JIhv43toKZ z9^IV68y|P!{@D0Z?pHU{KU|(wO--z=x;J>2zJo0NZfXjZx>EHtusTqdSp`~ZSE;g)q9@jj4sAdvQLe9tP<6ib8pgTom;A=cY1OQ9nN#U!As~_ z=@WQ~JvscgpU3%AM{0Q6EgDqAom6_e_L6GH-KqR5QN~3jZyc{GUQ1D&3BUH!7plPF zDZklekgCWkrlsyG^0Im7c#Lr2t#$Qzs$~`RF!K-JYFwSY?Y?}lX9l)>bLU6A&uPNV;frLg#jo=2*MAGve8EXT0f+B?0u% zy+dqcK2(`yh|ZZO$=veelk9E=msf>4o@6`S!|CvtpH%Iu6MTO5Av!~QH4X2aY`Z3F z@%E3K_=V|9XcBMm|2k)K^PZ2h8%}88g05|$XSjuPB9H`k^1oeFpZ`nl`8g4&!{7`Z zW`CR}*$pE9m6(V-t;z*kh_W=5#eB!z+f-WrQ&HcskUE;(!==sr16#*x@}d49IgcLVEdO7F3%}9wux;dCE2{Ni zIeo=Yv)WeF_fRLLMq@1MMhH~3x7XGiDo zkq!rF$1_VPE&Y1B=$#L}>AwovMoWV7p5Md|{c`0!{+mf}P|BpG&ds;=Y>cwKaAh@r zc5*3SuqWO&_^~Th{(KXyFS={xo&{bn|BbERSiR~r?PL7B%ryR?$gk{rA&d*G-pqTJ z=Fvmw#C99_@_#dy(otJx@NJ^`9ER%Uq8hA-`zWhW{Y9Y8Px4dY?H-Kb_b+U)y?IrN z-kaLV)dVl2-Y8$=>-}`9mvfuxbkS^>xLwF4SSRsv^Op0KUUzuaWiROFm)Ue#c|3K} zS&E+7l||oscald zEw5pEhnn2Bh#znLjF&Sc=^JX&bdR?Ub))narStYQUzDXt5u@|@?bcd$ru)|L!ml;9 z=FO)l|0p%8|AkDoFe-xYDv_}B9N9!i{*0nmebcV~v|LK$LDQucg?yxTB`si&`R3B` zuafw+FLn7-!Oyrm5@P(~L!ucO7UMw=B1!v9Ri+>^J^#QU-VRmMXb4 zU@LvDxQJ3w63|iR{oJg<&zv}XLY{TFlh?bPj36C~+f+osdIFE68Cep2MWZ8^n7`$yAv6n4^W2N_=Wg_)i9 zzbIO)<1cm9>xr!cRAZ7QZR)c@naEi8-{l(Lt4)a??zjJe3 zDwmO-#1H$*&{z7`Q_9j4s1?&<=*6;?d_mJrN^VmSePOh;N^46!pY`NFO8?VR>PO9N zySJf7>1ov^wE9(+p4EMaUXp0cpI_-obq$QM-R_Q@iTeQU_uYr@&8(oe3ElYQu;pA} z*eR|s;y;=^oot)btin}qkfxg(IetyByj?iGr^>xUth#M~DkrG_XKOn495>BzG54`A zgwwRj<)7Y~&99P~VOzdmPUOdI;Kl7f+PeAAr6b4c(O%o!sWmf?^8r`Z($%k>I5#hG zzCXE@BaTj_UtDKN<-@g{R+toh_53g|8#}%_CMArYq?~74C2qnxFStPIPH5$pP7_i# zHA(#TRoc}t?qc+=Yg%;Xiac&G`4KO>ROGezc#Yb7K7rHiJ;$4ey7Kpn8u-h;GSw3| z{ibxSG^;NWkLck~AE{6MJ@m4RTWNRguT|%~g6Yde&UC`cf~sEaT=wF8IojRjepS`z zI{NuvJG(G=yLirGCUw7Eq%S`5v~|T;zTjLwSLoJ7zjSS+i^jjD)*KS` zma_Y(Z1$%n~iz=2B%JXxkeam7B$3=)%eT}7?@EnCi>HNqWU?5P)$*< zVI#H2)Q>v1I)fV9&`n=Geu6LAeU!c%^@KWac-P@46m@V3N5aTv)lM|A3ZK9oqn*B;K!L*@It>gRBX&^TlGl-J}07% zn{zjpUN&^PD*tE(yYZ|atv2=mw{Lq3cQV?LU$JtL?bp#a)RA3noTLv+mu?ut$Gmo> z55(J5NvK)a2}8R1M)~b@PMI%vZzPLvjoHd8%==I^GpvfY6c6XuXg9H`on*61towWFpXwmk=o2xQDWlg>yj52b zKd)dJe@?g1#yx+9h~uqAN}~^RJ*eo-|1OY5`D#6q_0)R^Nvq;@)9euxT_&m z)Yu2IRS&PZ&>KE6w#_{e+{cP@T&TG#|7CY8FF$;ipLKUTJw5+0w@*5mOq8ggIpZ$Q zN~Fb?I2u)b*&Jqf@wqa!PV+Xk>+C*S;nh^W^1(K`{*MPAq7^|GOX|>Z|5U55ds5UE zPd_f_oKkg^D1R+zJj~7BDrL7>5Xtj%&d}M9&AGU>Mx4@^H2$!O81-?|8SY#Db=x0C z$@CGi-Skn9Ep+iFo-5iK&#j(1%02b}X?wdbiFW;yMITy2QJxQE`KXiLwC751N_BTQ zWh3e_?VqsO_OgTnojkpWy0t~$cJ(+nKFPg;uJymq@AmmZJv1`rT~EBF%7Z&?lXsT! zb9dKqn;l!&gXuZ+$NDJxwe~sQ_GJxi-c8aA2f}$xt!mEQB8Seml4RXijv?1x^dR3n zYk^T2iga3d1-q*DvEnVJtV3=ylpnrHFitEM495$bRwv>i?BgAvbVnmj(7TT#fyo5()07l-yurwP7Sl1~IukcB0?sRIG79hO zK!aL|E){=K7~I4_iW@0^Pmv8;6v5tGXviKIawMa5blJ(@cd@j!8Vjr3**ouR;Yv^- zxniX~DSOMCjXbDK#`cGktq(29_ir^(X@@nuuw93WzNvy#tp()o6cHO^-$8b`fnXob zmtY_6|4CriV)DlM?eI=CWAuQeCi^??F#FWYh^_XvAT#YxV!fps`&H))zWBbF9atty z=2;vikBeBVaW6&8rsWez;l2oRdG-`CQu_($cxkYgKTKf_y$xAu(|Kgi3v<@HXb*e$ zr)ZY?yf;ktzJ~%nYaP5Sod;89`H}53&Q$KnO6m~**a4-@~cz`Is9oh>1^XmKL4r1+8sZ}rdnCC ziS%6dW7!4p+Zie1A1jl6mwNHf;hm5>*^G1@VWIVBGFbb{A@%qQF4$&|bt|>V*n)TX z@tuI}dO3!j_0@p=bJ37Y%i2vk9yB7)eE0-a55mDtVkX-m5i2-R>dZb49z(W2+()k7 zG6IoffZA3aNQ&W|2guG@i!wdKI-0t*R&7m3X)N%n^Zg4O7L(zaSaW*EL^ zjQSrzUe9?*K2`+szU#=^>vnAP#Avoa!-n0OtiiHH8=&F!3No&(8TNQMlWvDt%vajY z4jkt3`U*F)CG;aEXhg93V8ga(T7sd=0O2wIDt>C%4c}P;lgrFegapFb!U$phGA}%{ z&YHME#aCA6OTpp&TZm^3m%y+h5)4iR5$-zvMER;JqG|REymvhjd#B%L9yWbtd|qUu zKgojBn|h+~#eJf0q6Fl+I^dNXU4o6a={7f}A7gB`t^?|6Bs}`ZGtN3wVSVocsPdoA zEVk$+x)oNVHb-Mv4~1t-vvEpsB%Z|Qf>RTpF!5GlQ1Ejn-fAqs>!-|w>%+pZiRi?E zp;$~;GJz-VCCt_fVm28y9`Mq_3yqUkVf&g#%(H(-z~S9&CcsS$Kb%j)HAN-Nf|3q+ zw<&>WNPomUn12b)jl`I~5LJx1Ue0XLd`#?knaG$SFFaS93&H8X39%}Brgio=Lb{+5 zqc+OnwVgwZFfk7UjzwX%h^j!%I3Z9dosN&zq!XhtVt7;19BiGDkZF2A^rvnnUXNK> z={QN3T_jbBZQFC1A|hWncsdNyB^1DU{!?s@tr5I0-$^vp>?g*aosQqMPC!Lowaxt_ zCD>c|h#05nfG12XV6jUqxU?yd3W4W|1eshBOQD{*8F&m<6FV5wzSTs=L3u*8Zk#Z{ zei6RzDij{LVh@|52AF9IGB|@=i#Js6p!SW0;PHARGxzU5;jF;F5MUL36o2!>bT!cAMT;f_1p*f<3y*Q;Z3Pc@*t55wda3rZ%_a8P34KZ;R5cb^_H?%GV zDX+`KoXg?FI+J+-iY~-PGyMQ%o@ZX(MrHKlR!WmD30(RpH-XD04z z_r~&;hp-l#aL$=-xKQ?!IPv*Eyj?MeX&sutF4^ybrxhK^3DxIdX8(mnYbqwRuXTfue+qh!$e-Ql0AO8DRjS6CQOmfHtq;{8~l$IhH zs4rqd+7zL^ixw&AY(}=5)(}BHw!#?6a=cX8DHx~ciwlk>wiPn(Yes|$sU*Om4_br8v;sJf;H?A@w$3=FmkDd_5RV!lEDV@AU* zzgL}BWbopaQQWAlN=_d5NwhZt+bUW!wnN%T8pgomIdw$S zoO5_%n>dMPPbv%c%w#R@E5YisEf9Jxj>+9;#g3VfgC`IDgOrPspt*GpTl_2q&j0dd zPCZV<%JBne8`y(7-wUzv_C<*PH;>%WrBB)&muHQu3z?su*5uyCGB8!UWmEt7Duk#m z!?+!r@W#WdkiM#0s3$IZ7tSM0H1!4Ds*e(0Yn8~`L;>7d)D7jek%SUxuq8Jp;r_qF za4+v2Rxk2qpYN!}dCMNawIV;*7iz+e*#E{b;xY0e3(@q6;j)@K~_O^A6m>0r;|y z6I!gj&&)5(#}YRsc68lhl*+HcfR49NWLO9%&+9Tfb=BD7mf4W|@C+6=e}yk0=ddBu zj@`BYFYXtAh+dZK(QDp2=zE03Lx)@ld>6$0J}?JUW7dFr`Jk}&_XgZ}F^-9NxfQn) z4Fnai5)x1S$2fl%C!MDrg2q=A&KUC&t)`met%L8NJWLjHmR90$ysWi&jukntJBR68 zIiFRLdWo@Xiojaw338@W*^!%lnAA9q8Fm*63I^IBB2|s7l0HGiKCh^laC8u)u5d6l zGYu})Dxnsqf?-(|mF5c$W4{lJY6G#Dw{`}8YPH52J(2h{<`wdJ*$5-PDA}0=nzz1# zi|Z`#cf7ZjW*N~s$i&J3Gu6~4Am2iSv)_8T`EuD z#vy;qyndWG5WIjWyC;UmtNfUIhhlL-#&f~s=os)4KEb5B^*DdYH%1C|VZqFM%(T&b z{I%jJ26GE=>V{0X;_(2+Di*+cg&>e!a|-XdhU1glYcS_yHN-nCu{qm9q4YdE&~*{< zhMhUwb+7@Chvfw5!)>D1@UH`%AC`M!zBZ&QaS?fJ}L-WyD~J)2xC;#^O~bMPZ$ z4i-ksk}+A2V2Z0Jc<5b5)8vy-=xjnvicQ50?GNG9qcptmRgq=Q&Z3GLhb7Lv;MnmH z4oMQsy1miDu62bFmm@<;K8l2R*%!=PKOQEgdcx7_MnY@pRZ+ab0Y?h^(7hrZ4IRdk zX3y6Wdg@6xO;@deHHKZ7mGK7z4kbiMeuP>XF`@Ypxfg8kX0ih(f139 z!aZq_EaM8>G_>*A2~GA-j}*I3^$HWXSc}oeyI_946jn@WhLVz03=HbUt8+@2xUG3i zyoVxdwrVn)oL+)G#^van^#g*GRmkwr1@QH$JbbaJ#H17XL`jB-<=&ozi%#8v&s)Zk zDf%1)T}{T5t9>AS3<(n=eTlk*CzxL<58(QUJ!pR201g$ZnC-AvSkNJh6*n3fqOXNG za_u#|{1676O(}4s$rmD{o}*E@4Gz85MMM8e?Ce<&ck*kP^Rg+p>-AG+`sqJtoW$e) zsmGaG>O1yrh$pzDeXwzXA9H<764sxShu+^wV0mVa@J)y&nD^+gdvCwP4Lhgf=@2)l zte6Awfy1~%b~ffE5%BT&cx>ABoOq=$0aR|sVvSlg7zJyRX1&*8&&^_((W?#TteoNI zA_eHg9e6tNKHR-+LB7B97aq)z7I7Q(*{P>5pxe)B4A5n;Z)F7TuFJuA-i~Pg90NfC zvxrLqNobmJ1HSE*V&6_isPD2OHD-Q?>c|u{3XF#X53hjz5z!Q@eO<(smj=W^;u^ri zVTibCiRv}A%;nSpV(7_CCSIvmFnOL2_C9_G+!7nWpofGtd5VasJ%S1T>k!(b@aEN}&`NSWb_OI8 z@2bwg%2~z4-{djmvfUAoYpjR8TUO$H{SGu6y8#oX7UALT9-#7A#Ot0SjqXO>M4HWF z@To9nKfGzfSKkeA(oSa_KXU}HUW{f=3hNoIDcLZ4yaDOjvj-eM8vr|PH%>Kv%IJS6 z6#SZI1+xE0RPoRy3)p*LDm4w5d)n~Iunc6CeBk21J=jY>CtPK36PA)YV3ql0Vu|Rx zaFn=AwCPTR1f6|0!CqS-O*|h;)|5g>js*?|-(zYL9ogN7>hTuJFyxjEV5a{GZ>e9y zZQ zIvFeBs>myN(9Ke~d7&+~7U?p_I!(YupoE9xig3%A>%>y2e&S)67Iubck#eGG*!R~P z2u{8UV2r=s0`89|sGrhf)+^nB?$(7w!cJKtaGo6dN@hLtX#s-?=cclo8?wOk-7$3C z8OZJ*RAT2%I4NkNtC+X8^`JMr8$x%Pz~SB7iCxW&@Oe%i=+oKIGkR4R>S9Dr8BGMA zPoo&!VL?0lmENofq_KT?DOX@8I~w8s>u4JmP@WWk!4JKf(UHwL;J0DNI*S zHgkPMlTG_I4b^@&Fdx*f!TWua2w{2_DhTo!LDxLIDm9AxFN-7Vp+tI@k6^9eWVZOS z8m_XH1GA#%*i4zQ-o47Wu0Reqnk%9=gBayI-kLfa%UJnn+Ju{r@+Qd zo;{YK%1)Z6fTH*vex~ifX8jQKoIVXd#d5($N{)4PQfK?OS7OinX6Vmp#DC*vuo7+k zsP+8`V-{Np@)>Qyi1zKwi&;FzS!!^(SEDgc^H#oEod}Hp#}#JRJY3mcrEeiD2as zOZ-vZg<7vh;1T|S^_vZeUY{&nFSm!WIeHadBur$M8{CA@uO=AK=fQ3WO+?YiBsjDw z8L3b!)^)Q3MurtLR~+YKLa+}ksuMAOy>=kIVGKzhXaOs;To_rLfx`AN)39&D(;p&kEtEb}jTe?85mf9@zM72vl|}k=le0voNU;bVW6dWUd(7@thE- z&b>?|hE2u_r5ku;))FwRlY#v0yKqf^F=5?x-Da=3E8&0YA}*hO42W%yi24B$Z!`EF z-Z4Ie%#O`W;H~4tA65_7FU%GE8`j6=zA3nVjE}IlOo}{h)k5efxMJlF3DMb^2bxco z!=0%Iz&2Epbut*wUfp9PQ0?c1pA>blP$B`a%YccWZzcSBvR0t}B?!JuwGlpcaKfFU zccXn@g3RyHB@&9&AXDKI)BfIoZJ4v4kpykXb$P<%1i9iVjU^EH^gA3X7uD^{3HFoE z6I|gR58(sFSh!#s%Y0M9k4@_d9rwG$m;U#VZ@<(=fQK zf}qDMvDasG(c{&`H`HM-IR0R&=PIK7 zSq}Cu3WaO)4ubKI{qTO_dtuI82Q;b9B*;cP5L3PmYmeq&EioNk=knMhr$Lq`JCMm& zkHPqrQuw(b0(C~xnHA3;!N6}HrXWO|tt-hxQ(1YUVAz2u3j7Lv$5w!8Fhl(JMiO_) zpFnxJ6zCibMs0o%^m-@2*#jGKu;b7x}LH!YlG`+{&e^bw3#jwk1ZuEhg~dzeqQBurKC zf-_~_!t7_CaLFGp47jajfMB0*vt$v2UkN)Xp>9;yQcFv_QD%%C^!W_PQJm5q>o_y z(s)oYs$~*%8)3toAA;PfLJYQX7g{dy6DrS$gO~4@6CqQLSv_|7|kyoh-f8?MS0 z7{o!%sd6IhjtoToR3e5qijglv1_`}_eHdBA;J!o^Hh0<-++^5E)ThLNN=vse;G|CF znWcBJ-~2Y-8Tf*Nrb+NNIRlqkO0Zc&dzn7*zr^Ogi!FV3Z1k!kVbv0OGDx|t_$D&T6wsi#2wV>=5dPZmlLrzvMdBK!Ji3{`VE&>GLSug zpWtocA7+vB3!Bp(v*6qQ*VwC;#$?SJ3t@v{*lwQ+Hjl@!agw#DJdlV=AB05nDS5ax z3UJHt9-c|cf)!53q1Qec1Kuc*3pS(+L#k!5DP9p?MZCegqTg{f)PU3zPr^Tm6UnGZ zWgL9{joI>^0HX~zG3rRL@KH!V$`P9QFCrE1*o=i&u6oS9?phnEwl?UQTPJ*_c|v%$ zjUyV(SHUxO7aFE(VhyxF^7hL(8s`KrVuR2{q!YPau7?OKJG9E|X1W{hL39R3NY_bV za`ARv2Q6eGW#EKWzAxy(^!L**(Q$3o5u@{zZ3}#>YstWo)~nm$p!EC2{0ou zS{SAm$9V3(E;`pc2<4qgFd7<;d6A!)Y^sik65Tg9c)TJ0`q`kwvQgq{a|zhgnd0)B zPl)1IU9f1V3Dk4Th`r{?!b}mTGJpF|^f~=ka06yAT5?H5;8%C}6VAi>;wr*%E+K4f zQDHZx4cmnOsjt*6v%@_qc{n;Z8K-QwhRF;I4KeZfL@`bX**IdzIUnov2M9xP zS!S-5mryTAmTWChfU?yM!X2((iDA2MFp{-Iuy0lzVQx_g3(o&0-h)3RUP&S1wxked zM>N=@cHUUi>dySW{9UkUs~mY{uO_2BtC<;h<{?oNFGu=hO@ibtRsuq&h`8+Djmb~5 z(6d9GSrR!1w>N2$fB#&D8rcc3VBs#v_S=j5(~m>MQV;0Rv;fO;W6=gDhw{}@P&)Po zGsfvPZq;u?%^)p&n=wMXk&!1S)Hy-$@^Het+#TF(5@DRPKIH9~f_O8S2wbj3o`3O! z=dYw!`DYvVYwU9i<{J(^3&u$K3t8Rr6jBF{IM*?sW| zEQ)wR%ECz&KQ)h+k!*Qr6~Tm8sXY~V!x6x95jhTv)(c6 z?$NRM=jUJI1i=zU?k~Xc%W9lAR`gB2vWBBw0Yq)41^7u!#&2@IOm~?sjIFvv)M#E{ z2B(!0RVtFCbrBE}zfu^ZT3-}rjG4oiJYf2j380m5h_S47hR}@?_-mUF+->p!nVkmg z-&_Oy{ccR!Ua=thsOctQOXXRwFbk? z_35H|CBLb1_~|Mb41W)as%wQV>EV$3^%mnRI|=7aRVP~??IxZEK4PR-N~6gMIWog^ zKC!$nn>l#NLi8*X$Sq-~L1;D^gJUOQs^$#b@XnAqKT*U1w^P`1;VL}e^b&q%8?Y_CwM4B=vGBX-4*YZG38G9U7N*E8f;(5wF>e=` zvkwv-(6?kVtXb+U@D-S|GfK*s%-?sIgOjSUFfa$i>OCQLxDqCARUosIblC8Hzi+-t zlOe6=SVHq&H&(DI7H2#fkHf~B;r2`ua{8$r&;jpP9&i9RB7JWF+=oB^M>y9e$nN)xd!acAfX*|(;XbS4A zyvhhRQ4sam2>9(3qrUpMz)61_Iy$Bjr9PVE9`O#rT-){V`ur%tMj9fi-_KC-1dg-2 zf@>y>Bl9|>1m2EO7$9SS%Dg%_*NYREf9SFnI@_>xbTe7;s+BqR)yF-H=HTg-r_ki#6{0&w8)FSRVeFz6aJgwJYi9b6 zIG8(+sdvtYq47=VEcg$M9xLOJOB+F7OW+#7@;$ zaPI0uNV}v51{Ip@lW={eZr?U$l(1w8Js!-Lq+)WA3OF@?X1>@slO4k^jY9|1yis zk5$JR|6N4WYF`X<9Yd>J_ z&c4M=CXHc?b27Ahj^F|JTR1Xd38{Z`8IdWp7QX*^2_87zvKAhchVL3<$s4{;VMp0{ zTrzl&8D};NtU`APF8%$?ys3YNYxjtgbwB0Usdh`*+m535_#}#1v#XBq9?XDI_8JbC zzk{5Xc2xN>n;l9r!k3kjWNB;^CeK;GPR-FI8Ve?{J<5||=P^gJZ2T7Cz`{7_%VSuRYbtEbokb4n_7jIDeZpziAHnehw&YgBK)AM{7c(`tVAI0q!imBP zW}B=kJUd&6j)r1vJMAqbunZNg9&1uBbGE@Qvc-xF-N>%e*ap9Onoe1=Y&Ik3Q6n;aM~6twzqI99KR7&Q8g zwR*YmrqF==Fn*k%^kW#PMQN}z!|d7TUj)8V3X z1m>qGGNJ#nVf}FhSpV=TS}eAKx_7E%Yu<7+I~oq54{s9JPnLqZ+H+#K{2TsuXoGXd z?aAiVRm9r6E19ju)3ElW82ePt7B0~;&!r z^_W$mX#IDmaH-)dR4-R!FRYekv`6ciKgFqN?feCO-!}^9+%+K^ zq7E@qqx$UDRt`OEzrotElOSB}8+w29LDi=eyIXk)D$C9weH=a>*u)*d_&sv0)dWM3%P}IQ58i?P03)_%?JRs%H;(+a z{4<0L-PmVw510cDmZYZ2U68kwCSTUPB65#j!m_)Ia7&&8MBG2gED`)<)=IhJa=IPx zXgjf|L7pwK-HB7=bjkPRLU?LyikVmZFyZJAl-lLU%z7$D&Rkjt`gFdlvvgzxioAo+taS$aGYbmU**lG%~CV_hAl zlgW(X-f-|Y3x_f(N4E3LZNWs+j6BwTAIh${kqbU-g?%3uvTD-Oth8ndq-~Ud@mI$Y z>jVPg!LJZyxH^jU}xeyd~iy1hnM>hQfkoW6Eg)R<Uie6XMq1KnB~$vI9$&^t3&7}6r>G`K8Cc9KKc zVt;1oA{|_~WC*|Lo1m`BOnmn{gD|p@gMTx(W3AN?0u9)btjXs8`iO5jDHuAHOH5IYVg}xMLG5Nsmb|7xocZQK zsy>n<-^#lXw_Pv8E0M-9G$Ml&Dz#W2nJ%GN zg}0(s;2DC40g zsyA2OM@QcW`1wuoP9(;Y8S>YDR*d{x{^?My~at+0vX*0vvlLPR(X+2n4|3_3tI0`ev z4`AWk1gJVZpFAe{f!XY|fZgw1i?K^*qo>k9WzTh8c=B2C=8QlU(pxNtux?dn=UtO$ z-*!%CdmfYvHr>1nZ#}2Nuhn4?5s`_nB(zxFNsYMjY_wn>w)vWdvOyNdPGbU}?{W1%5M7R-5vfinAhbvUX|8&3w1aoE?ojFl+v$9V?Z!K2v{ z7f)zn#v1ey58`iNjF&8)k5(m1?w7(zb3-yzMTl|J0$J(BukoJ30J_Di!_P(uvgvgr z9N8g(Wo8MErQQp&ZM|lBTSz%TUftt zH8U_2jh6Lkgx1$1n4(||_2n-iB4#S6?Z^bD>NHfFFT4xq7W1 z^tnp2OLx8j3Fn>Uf)O$D&2|oF$-fem{GB7*7L+agkvNgva!`(pi&;e~jFTYk4|Ji) z$M^8V(wV&Qr50ArsRUK4RoJ-_*u?3)NOQb~$0Nnip|cVCMn!$8^`BAda6kM=T}z^^ zIyrQBDct?qkJ~=Dux7J4d^2pt_9d#3*D@kVrQa%~#kUDy@c1*~WYG$#_me^KoD@2S zmNE0EPayiI?14+k44kT~K>7Q+Y)o(!PEh-S&97Qu&Q1q%hRkDl-lm1G-b;|%`X-Zm zour_}l|@DPP3#HNBCnpSBSuXv@YCcN+&yb6Q9XSe5vNjtE6S8HB<`Bv`Pm3&+~)z9 zEIdyRO#G?%-Z+kA z7b<(Uijpn6IdkV&LMo+Av}hNVBvR3a>_W;;ktL#%gga;MoQpz*7L`aT+qWVi(yD&< z_ZQr`uh*P8pXYf#&*xoJB$|S%hAI)^oDP0`+fFM~D={i!Q>e9Y8xeQ?o^|qtFs_VK z8C|i+p4<|t&PchRV?>XuGB5Ex;Qv9H`8B*5Cd@21*3ZxaR^POS&Hp$%A=T%w;7l4Ne33c5V!8nPW-MqV3|eW)yV{lRN6SXm#Dc?M>v&#LQdvcU}ZN4{i;-A7V6wYKi~Bso0nISO}iDj{uduT zUZ#u6m2-g1j4n`f*O)BmT?aHo5U)prGff%A{(| z07J8nk-ID=xJ7}F!OvN#NcHw7l(2USDTNlHgy|-r6|;zZ*pdqN3WA`|RHcE$X9y&) zt%UdMHqaHpqqFoW#_MMxP|X!4YiFi{plFIYFepwQE`36CBd?=R2Et6idKrdVd!75U zz82&iyhHa@&m#8)U{GBJ!Is=Lz~l97>z-5o^s%%m6r?Z7yxMJqR9`P4e7G-Ae?%(% z#7C96Yx)Q*58`kIbQNaD!+Sui)f?G-9t4JE(O}{0eS}fVAN2O}2OCirVKVbi1#!US zF2VV>0KAnEXI8{2kpEWu(di=Xbm@v;;F_BzlO0+Jrhoba8{NO?NV^1itX+iOc_V^A z=JUw4TYsTGvoO?jtJ&sats}YpM-*XnJqty=mL$piU*Ji%18EWS2G##SKz^qqsp_IY zK6$azMyah8SgYIxyRCkKRFq3h>$h@=3(tu3n1g_GEd|AU%(uDS90@C5k-c#eir?zQEb`MN+S>EzD90*p zSg0ypu|S4aQS&F(S*)qfIYkoPUt+;}ZC5aM-<&!4;3vr5sY{d=Mj)H)e<Xkk_Agpz(^=wUVrCIt2={DH`yEvyHC(# zm(i-eJ4meUKKN(|tbYpU)0E`{K%g~5fJg$Edyd0&FpiAQhz9U>^a5XXi)r64rv%=u zL+E$>2Ktq-JXvv$3uY_qq7NTmNW4kwru!!rf~d$4G;QJoex3u{Lq3!EC+-J~KrAX5 z^0$gqTnN6@_Hj!~jv`Z+-C*09xy<$p8<_Yf{|KWrFTkmnLJoEc^s~a1;FOma(UzNu z=IYF6J_Y6wqer?y@0%p_wq-dJcYB46oo59aZcruCpeXsFlt3AtMre*#4X_btq{Yv+ zqMsw;v|sx@;8q}nCbvCCuJ!6*Rn{O%RFGf{*F*z9&R<%m{2EYJ{RfICGC^YPY|sQ| zGOvY}G5+PDpnNzCz^&KmR8eC_V%;iG@iGi#wqS%GZ%-!5*>RB@l37hZsY@p+NilFNi$#_j zgF#KH0jZ1iYQ#M2kwbJCCtKaF!3=CR*i|D`15JK~dfw1gLkiOc1={e)dWPD8k3J$A@ z;9n)k(~^tsCn%zs#!mqeL^8@1_Ke%r>)`YQ1JqG$f$B^Sg0I*10hzXFM5T(8ZkyMV zU}75=mVQD9Y+eJg$X=l4aRa&gi-FJI%aO>&UNqJ4%4Q+Hfl3{pfIHXq$kc*O#2TOd zV7*T{GOAKyf(LkrCz*yW$C-d>eJxV6#1gse%0^0o*5s^dXHw?;1C+H8f_mjW=#Sl2 zX0llxNDK1!e;C=4a~3Wm^JBizZsB_XjQ$%%;J0jC^z^h<^TpHos~B zga)R;ao=h*t00;fizk7%`~$E%)s((|zKYw@lVk1N-9SvOSWU~tEMk1Mqk(1xhn(FP z0#dF%KpUnVL6i4Vup}sordGWIYflJ)?5ESHo=QY7pX!0%w}nyEbtB|3$Ra0h0?_YT zgxsFhBbonlQF)~v(Ul#BQmqyeRzo!?y*(V6-!QUKT-#jZFg;4lJ8ukx!joyoIz_=w znVZdDpF?O1zl7GvS&t+FU!t}>?|_q4EC_LsVYm`oxN`3vp;d2!kYm|>5aV$UE!TYj z;+RP;*glWBKyE}`GYzcG+!ldl=5xXEfpoB`G#U^p_UOo`CA58;1Ja@%0OqVLZRqj~ z9qK8fNBWv;JJ%FjC)MsJYED-odovZAJ3U3nO)?t!epW)~TX%qh<303|@BfhD>p1lG z$y54gGmk!h^a6++_(r!^Vr1!{hz8bQ0AVk0qP22S;NgLrT(!W*=)|Em;`2Zupp!o$ zdn7_E&&?rPbeoAQsT#C_cr4dBDh*|MoB+=sB%-{uIP{c<&_LN0V1DHa-EpYH=Kf`M zv`eJ`7`yX{nwAeJh?QoxPn05IqkG^@=3k=kPalDwcZ1lYjyAN$Qxuk+)_kEetoxDjTT$Z9Zc);xnGf1KKBIZo4&c@eRZ`L{ z6+BW?LvOZ}fP1oExWb&ZVAwyFmd0FSG)o7S^mh;=;YZPb`PYG$x4^rmkPl7@OQ7-~ zhz=g>qS2{Lr0X7nR?A)mdmetGjYZ^boI38&wdM!u^WGFvJ9G>9)h5wjj>dpB`=n6W zZdvdEU$Bz?T?+hlLQsXV0qFZzLi;I4**s}6L3>ZCfde0!!DOKsm~6R8KYCV(3}sb8 z_^U9mtAT_1E}>SMgFJ>?K+jGipmIuu zxzlunR@J)7T~{+3C4D>!LMBy-YVB|kUU!-}RM(5{{|iA$i`F1JAse7`Yl2V^cx~f~ z;_3IN7Fn8(jy)(F1! z{tBWXVMhjP+&oBo7M(@;PEoYat<&gYxi3mxDudF#P9WVklE5*_gq&~}%z^x!1ZVF& zvKW*hmj!M~ zzM>c1n-9htH*@o~|JXDnJp!TM2f?3H-U#!nz~L!H@@q~y7|58#Y}_6UwEknke=;gS ztTrB59AQA1kVEZc)*NE_iu2%>C_}F-QbyOVUjqM*jvboMWB|CQjFq!Kh$ux0o|?R5r+zmP{6}>o4V@!q=l22P4q;KVIO*EIIUQ(GVES7YFl< z70Bij-%#+U{b20y8?Z|undWXdi(=-#BEBs2W0EGged#DLU-Q(aF6J^=FW6C5bs>>dH3Gd|c|gtAnwC8<#FeWbrRxNJ z6aPFH;_%NqVAn*PwS>4blO5JiS2q*@qmaEsQKL5qyIcT14%?y~M$<^yG8i~Uwj+wF zLK%j4K)>WEaQI#{{cNBUSnn4lR(8*4UL~Z0GZym9*q0o7?2i`t_ERandWNpG%%e%- zNTw#a@}?+xZsHQUVws0l+*Box-BbaOt?q(P^NdmAyclp^KNr|7kOps)ucDDrPn5~O zOvLw`r_noIbhILZd-&H!ba9G{=v%HtwEZsBf+U!v&oXF*dNNomtVn+A(+7oA3@t10 z(wUu#xA~dePLKMUq6DH6JW%=%Jl>EGewk$hPT+6u1TU7JeXoZoTD%`MNmLSME)wL5 z(L_Kp0qC?r6L(581e9rrgJmzzfF&Xo=>CCxRGU0bdq!viy=haBrneT=objSt)}IBM z>c+MF*VVwo%Y{(!i>JMJ^#kL8C{S?YI5=1D3c8&xA*L}MEj+&m^;Qb}vd>na4)P-PZ5u)ohbeSU(1+Zj zo=7;Yd<@*b!~n|=mBji`Z?I zU5r6T%q&nQW5RVD9RdNCA;6_i19dElrxgpP!0};OVB_6NH|53B9QA`hVsRK9x~v#n zZE!^!w)7#fzBQ;MZ2E1Jaii>G)O3Cptx2S=u^EV)-<0{I|^5u%DekCFddg;y_ z!F;N@9=%yHj2<*w0dHj?@{#vxuwa%j5Kg;Gf4?q*!e!>63DrzQnpoJ3S*nt~+mpcD z-S+fQ;a*_#c@C(O(*#u;Caf<_xdPq7hv?;Z74%u+8nNy#M5XU3wCL|i^@?Q3`wQ=a^dp7b-;>#>Ekzft zbS$CY_F2+ix7t|2`(QS1C`fGz^hvs=%2R_kcwEujc6z%?iiJW$EyXq+yB~V zWmX0}T~bTmRLB8eF3Xco+8zR7y)=4+K1D1jPzR+J;vht80l}*=L`w0;=-R`f=&PYP zX!KVD-#--r?f})~GE-U9Z2R6gCkBIRt>!W#*{I zLX`2nU4W+V+oCDUW)Kum4m_5>14yR}Nga7W)CVmfXWy~_7Ur3VtE~jOFUbk~@FhfQ z{yHM4ZP@0)nPGHRphp~jRifivC+XYnT41048k@9*>P-8}aG>uHN+wlEllPnY39{i8 zcqg<8slPgpNi;e?fN^W^i{l zRdRRN(5QH(FsXdP0_2{{CuWP>0*lH%2(;){M)R`?BOEV;u4w2IehWp&-;P0`*5w|W zGbbL{owFfpM_a(GrOCi^wlHxtQJKl9YC%tzMi2*&^`ptNM-ch(JhB>!2l~Za`jT!3 zD$R*T61rDtU6)9Z%UuWREWZ#-(3BLW; z-+?rL4caHS3P>B3(tpwtQB~Up`h-+FH(sxe8`AZPR`b%~atxM(4HwkGVd^HTz5NBr z1&V+nlcCzC3l)g3y^WTRQU#~mo}iG)z06q9Cv=h)Mp<&c;3A;|yhN>Q-v{O)rDe~F zybXIvd-r~@(ajmCsxISR1D{b!>~`W!=LYn9*-aqEr$P6J0`UFXTylfp9+~DVGV3Hf z(DDP_V0NYulK!#-CC%A}va}*-C4pzK@W5L%Tl6YARM&5FPx?2~UGB&vYM=m0}L4~FkS#fy=sZq6uo>o{vKd~wYU(W9U zS-v7nnywnyazKF;kuL;0FUWu-`(UJCq0OkiG-u>TCTd-kq4pe(41KHwj#HNpOz>#Sq?l%V@or?8fy>lEor=r44Cf1^A^JA#6J%a9}y@8vu zBKgy5G187}0P6>>$&~1~;Ke$GJlUndZl)C3ONH6IN=`xTKhDqtzUxThohTVvVa0$> zX*&4h5whrB9T*#bOKhrB0;}8;tusT_iM(gw=y~;7y7A3L5cbwZkN`r0yftEsO8r}b z=kg13l4?ZFlTsu{e>b^B^{QZoABpb0`Uuu)Jpo6qKLY;kvWzI!B2Qo5L@M;mAoZ#h zNV_o^rsL}_0ZU;D{kLcVFv?K{rmr)B-N_6Tcy^fUw8;-BKMw*+majk+1A@AF={Rz6 zv}feDi7??ss$kl^nZBwSgOp=jz#*kwWex zz#dDZK_M<`ynY4s9TOtBSwE4_UsW>hzB2j#YCD*(v>z-E{)Gx87c&R`N;1~w(hOPx z$!|J)$Szqaa=Q5_vQ+*IuK6pF%SJ6}<dpmCro?#$e&I>_j?zDHUL!sNCa zfKf}^%e4O$VfaO&f@}p{CjPfOBT<)uhO0YKW5qHuS92G+ddUcY19t`aHwsK{g9_6n zK7m9ARY~+>A+tXoFgrCrqLz|rWF_boo(Nt-@&uZ@R`&^aKuv@=JCa1-lZhwXJNAPQ z<=H62%LAz$eTO!l&EWcUiIP7JKLMwJIPl)73%&kmMmF3HChH|d$aq1Y?e>8+jDyQ= zru&LuF0678tnU|Rmd7hI-|O_ynU_mYuSE+Ay{pLhU3r0u)XPAjs~dB9(`Kf0O)b)F ze*{{-j-aF7Kag2X3zE0`iXwL#knbN_lN!3ef$9Eb;2HT76*<{3O~rD89*`y2U-+J0 zmNA={Q7J`Bs&{~A4s72{fP#PRDD6fI@ceB|x-^{t zZ5=b2x#640=*20*B?rAs>I&CZ8b*GIgwkcH?N0jiSD!^K33C9GT19AJipPu4<61 z(K{e~;s;7Ur9*CtaUeHX$&u14vO(#*BlPL^-E?o>4G?*mt*u&=00xB;fPrleP_>(& zD?A_3gJ)Qzu;LovaV?1LEjw&Hj_yLmmve~eS2I9)IhQUw#nQjtJ^}q9o!qInY4mPE zw#Q-z4d(rN7R=$t(ub@Y=zW7Z0L|dkb}l(fOKf_Ceu^;Y>Sht}PcQ=#SuVuz?#v_p z?07?joYSVo43?miy4k3BZyEC0_rO{#_B}W>(T-9=n&_F4ZS>f`L^Lh%J+Az_kPykx z0U|31)U=?U`_`e7){aa8ckV4A&q#zIyZ%vbOwUsRTRIGVAPcN3YLC)S`hAJruX_m3 zGa+%$3b)LH`KXLpy#6MiM^lC(0SYEg6`=E z`sul1B#Ui*yi zsQWTT04*WZ_(vN!)aVf&b5fA!4qwopvJO4o-i^LYRMNl828k%z8L1zWK-MdI0q4P1 zkY<%YKTDfJC!|_X^q&JKXXfh5Lzc^Sf?cBDxWgwefcy-Oja1h=V*b}E z;BRS(Pu{E)6I^8!^57jsi=Wl@%0Aqp)vL3O4cAof^&P2Q#1z~KE3`#bGtoBojw>OL>6uw| zKwo_p6Q2X?jL*zUS%sF>NqXnCuY)t4flx);lgOJK@Q#SE(al(43NyONDvk3 z3Z{xZk+||KdZ~FA+8H1UYJ^!@S7HrF7vDlxttddL#e?X>;z>~X){(d#-DOjgwU-_k zs3Y7T90hh`zP0NaE-=jG(x^!rNNLvs&2kY`X!##9yAoErYW_-Mj*Jhoh*SazCriO% z!9HX6fh6Q}S=GAmi3FH4fC+VG`RuGy`{uXiMPbxh%~hP&8FHauT*557uX$UukaqdZ<-y^asBSY(PlqtL--NwU}|^goVJ zbvc|btpsB;o#D=>j_~al2sg~$$1Cey1sC6NgM1$f?;5j!N`=!nO5GOfQf1IUsf{%F z9*T|LIZ^4pzoA&{9h|*U8^+6g738Duh59Qpa9E)_$`(Ts8j5UxRSP2bOceIRUx{X5)s?2KZ;_6CBv9L5buJ?6r}4ywysL`qMef_SpBI@O4)*bv7s!KQIfUjvzrE{`ZaW zPUsh$PoASohZH^Q)%S0DZ!WCUk=CPF{IF=`@pJ>?x1gY%0nvsXe)gGQk42l-h{ieBOs^8 z0y@{ELGGtOT$oma?X=-8BwN6jMe?tQsziK-vF2(6>$*erxApE|NBXUCw!h8`#g>Et=L z#KM%1w|K(FEVVB_70>(?!se|eacadvEYER=Blq;!JA!I^XND?&nb;X#`k8s`y2TqQ z_vktB=m$3{CnXJ&sD^VPEFTZ48Q|*^0!CGUHD!8L)b`!T0F3=92@_jv;Sa|uHs0eA z{;)q6|C-YSIjh!Vq9GrO8(`>j;tHjFOBp_Q%ZHL@tMI+rCAja8nC*`9s<8jR3D|M6 z1I}D$JxPpl=I$tGb38cki@O=m@RUAPx7mvd ze_9IpkxMa`eoKY%TFAJ$iu_yiNnQut$I3}LU?W#&te230w;#9-sn;Cr=D80)2tSKm z(POfmP=PX2nw*|xj_}ylK&(y|!d-UHsV(dF!Tsh9(6i_-D`R_>I+2(}ku$vT9Kr7p z?p)}-!G+DL_=}I2EQS3ZEXA)#rixF9!{9Gb(06iy2zd7A6JE*+JGI-~>lx^+GC~WD(z-yzwITIg}NP`GjoNu;-wRcT} zX*V=s>n})sKN`;7l(UCQ|AerSSr|?WD55rmENC^GF$X-QR_u9hT&eU0H}_GD4wdUjci<>KSAHQ_?mxoV^9n7F@vI_By440tm~4ft z_g$WJm>G;d(}0h^iQ}w!v>kRlEQDhPa(GkuGJNmGS!&A-1z72z4R^hZ!P0@!|W!}#^W~j3~DNo!Tt53m^mDFhw1Sef(;l*rSUZ&DA zTplva>GwE*W3yE8m~{&d{$zvGY}6>Oi8MR)EQRga`4JvaIEH)MjQPSfn$YEe1N_33 zfgg2Ca1^bN=eEw|ZvmTl(w}GY_w1d4%l>9z;~U4Jf%bDa%gKexH>rXbdj7DzbLZhD zh2gx=4b^bjEORWc-bH5ZX~J*D-%uRJ3{DyAu%mv*eNM*921rpfbI}CaTQ*iB~M0`hj9W=5_#xHj#!55=V)V#C;d?s}{oBv=J zH1dq0ZgU}fLe-JoudpA+Z(7EF4E)ZyXEn@*JfDFBZr@@3Rfch_MGDT(l!IS;9#A7< zS~#M<4fcFGfQ^|5c8hGbfWLU272oBL6PbDZ8$f;c{wzNCZrnddn&X?!j-C3{gH?C)jIVO-yvO8l2mFhcb)o?Mw*h0Gn4{5n#LknG!r32wL_p9*o*AiZKq8_|?pa8erf598)iP_e6a`EfM9x&cj zRiGP$;JPbUvAlT;rLVi4icOw@qZMcJzhnt`k{{LxVrMr|fj&V{@0BJbk7C#sS%dd3 zK0+;?YXuMYTjHHvC7APZ9B$Sd$JBj{3tvRxZu>Oc9G}8I(7K4{``Y5hQ*rpwreLVX zte}RUVc67P$vM1dnDMP^eo z9-G(5$}XPGAKkeOk6m+sN5!vVcf;$rJ8J^l4=CVCrv!NIKMUwpn1w@IXx7qf6Mnn@ zE2Z^c2^nti8=tRDq8ttx;o~o~;0ftmw(D&owbw%q+C={4sNTCs9T@CpXKa|{O>jO@ zc&jNT`bi(^KW&1Z$0}Hb)+a0}#^NV@19sE)B~Z`nE=-nr1h3dgV#|aVtkgMK+Q*y-mfA6y# z?+cXJhSQuRi7PPnmOY%W@&o71vgPlUi-V$NlF(l$l1|wCn_Z`CjBS#SP_v(eWS6d%YH(`xS?C{wVTu-o-Dn3e)AN5xb3kf*4okq8L6Tcqg>7+*XKi>;HtKx;WuE$GY zI6cn`2I{F` z<=t@@`3A!q$Hmydp);^v|2%#==FjQ*v)pz`e+Yv<$g@*+?yz6Ll*+Wpr{*6_!?(WK zut@Y8=a%OUwlOjs+jVrqj1_roc-whw{Nq0C&q#o>Avzpa+6RgR46_<;YEaf_DKFu8 z4?H*}V8)iMg9Xn6vGeEERA}%g_<}5lZ?-y6DJ!nwH?f;}Mtc)E)k-U|TwD;An6(h* z?Oq69Z5m~}I~L%}F)t`#2jb8r~I+n~8N{5!5vmJ_Z6!xCeC+Hv4%&5 z-@`&Z#O|Ps@Rg4#?9yN2*h;;d?Ftv+XRR*f6+c37_VX2Z{Z>P`Ho}jmtQ-aj>u8)m zsK^oCpU8?Qonc=Zy%R*cpNCqpOYzq_OSs2sF-$diL2bWz9CrkzLfzHYaJZ<1@@|i3 zT4mDU=OGW=mef?7XxiaRQaC8dw9AN2eEtee^kR~L%!PNd1$$M zK3v70!P<__#4?xaDa{)KM%MHJSbQ!E4+uC{eLpnlo63*bC*Nn{A0B#eZ`xY8M8H2u zj_eZf6JueY={dpOV}doV6q7@l8I*F8A}lBta4hHq99-SaDtCY3w0tmtW6|gFs_P%A zV-W$k$wY>;I<}tm{UAzNtE=-jceOyxQ9+%nvWv4Q>m_B}8^^{=RN!y}QK%78j$8f} z;28Twc1B%$JN@1K+GzLVZh$!M}g) zg1JE$%dHJ%KUZ;K{)SL^cJeZnYvIMy4aj1DcU^^UhNv$AHCORE z;uu!U6}4qV_F=K|Mrv7pC0yDx8}s8#VMe(uo=423G>o0#tXrF4b+{c|u<9E#IdBdd z>3m_=Z)(O+^BsGlK9%wpQn!_iu!m~J|ESeHV^qw4F;L5;1TS?qfb&X^;Aa^LSn~7} zidp_1voR?!eMdO9dFhFR4=Tev{!Nah=qUBPaTct9(g(*!_u~Zu?HUn#3R*2Y#go03 z4sWGb^6ZLh+4N68sL#9#-1Trf4o4p-C}m2OO24PB?9Jssm3^>Jj*D|SNz_10Ia{vM zhi{x(gWZQ`z?C12v6;0QHX@h6_g-Vv62=H`b)5~vL(i~nQ<;$B{GhhUT!w+u3RKEg zBYd&(K3;D1AI{7+!VyE6)ZU&XY+>tx*V;cHM-koNPFH zoq!X5)3~&!nQ|W>;X{2f>P}?{uGp8rn({Q_qAsw?R$)i@vp2zxFytl`I`OuY7Wb)|E5%iqB#w9snALLG)}G4gyYLE!yh$$ z_)tm$o*Mf=%{d5e!_kQX|?rm6pvyqid^1>S5 ztEln>Yq%=qC?sZ*)cSIaFA6lj+ZkzSpsR)l!&u5jPn0^pd zTOnIduULGEIsv`Tk5gCt7vY|NXYr|UhL|tl4NDGufMq_CaO$5v747SYDXVxW)FTBo z{deMd{=V?eBX6Eu-e>IW>P=NYt;EdY<51^r5*#Vd!bjuFps28@t<1MbsI_R6-8?G_ zzpS4P8!W8we}B(mI+%gERxxnxb{bz6=uA&a3QCkuVb|>6?2^2Aj?_*koFaA&duT0z zZ86W_Y{yu*1?^&e)61z--?y>>7cH=vE}%q8#yOUk`Zz6zPr>f8DBg`cJG|6Wks5c? z#;Ns1Smwob?6~F4Zw1;cHHnW>Q zcfh7Sg8I>46Ru2t!Y=HzgXn!H_S~Ebm%C%i_4GzI+3X2bxTy)xn&4x+=RMCNF$d2& zmCZ7nhS>9dBr7p)P5GfVd}xOnfAxhD-Z`fj_*gfRQ+n0FR`>EdJb$v7%CETsE7}<- z`#YApc62M=qAOsAUL3;R!BNou>sD&x;r+1NxtIO;NEl8lmE-(2F}6o}KHj!d0_%j9 zvpc&yAU|Lu?7a6F8ZHyIUH`@ip4dGH-yxz|eeWDd%sB_&@PcuVcLH85xeS_m{b1+% zOW;SL#?)*peaun0#on%1iC=!2;)MN8f$Xe0fZDmaupVFn-j$6_`|4jxR&ruL64#oJCsv8JII>MvRhh1&Zde}N*5+$+hih)l;P zV_&fER=M$RZqsDV!^Gi&xDo18zZl+hQ;j-a6@!oWi%~}oZ^C8%g_M`B32b?=7N5U~ zV6WCq${Y2YPIAQ?#^RY3ReE44Rv4jhe@7nm_pRW3y!eC4SJ=okJ zFRJco0Q3~-e6`v=tlP$BYT0pFtTH1R4+t4zR)T^}7i!ph#k2VKn<%Pq2T9pBZG+$6 zS3xpH8GEa!QpffWbNfm=mFLG)X0y_!v#E?djjVA; zG_`pNhOMg<@wE$)?5he5Hr-AG+W(ZtKS#r{8F)D98-55#y5XcC;EkW9J|w5X@7-V69qQSz+w~WF{T{$k zt9^J*`FSw=@+6gzu?OePEQAiZws>ajA6$P=%=YxRGi>k37PhY9F}y!&#|hLqjORp% z;_sS;@ML#5)tDy=t#V4Bj7>6oEoLo#B40}eDyd^y?*?hH)C_Vp>)_YAo7iBcEuMGa z0<2Pdg>Tym=%(_q>_uBqTbG7%-j)~#EU_V-_f?3G&0Pa9t{mZ=JZ^`F^-ZbXyilAh zqK{YIc7u`5ascP`^ogP$=J-G(`5{*ay(`Rs{5-#8A5#(3r_|vcTnGRU!#ZVp7_(yTQ+5&m|DKx6b{_m3OD4qz*mWBaMfW^ z{NR9KK;e)Nck7R_XC>ESHsJtc6_m$;e?|EzwsQEsP7ReniLlp0-r-#_^WniC?YxX# z`OtP+k^Rpom)g5Zi2pS-l4ENUh^vq8#mOeS*a?;McIu-;mTM(oYmxuBl-1kK$yVo zr|+|k>k{DNv2?a(<~w$zJ{7*}kjEb`)KPObJYu^=)Yy-QH^WT>tMJm@;;^%ryrqgc=~W*_zd{BtrN$2rNckh`EXH9FV3ZUc$&6#EG&_O zix#Yh<#(N^2gjYU>*;)s{u>(Z(bM8=ms!N7(=zx}++}P!s4qw^T?Yf6Cs0N25q6AL z#kXI?!PbcbSn5L#v|PJ{GRo-3UhE_<2ryVQzMR@^J{xvA=0apPfi0B6DV5^@mk!9_ z&F-O;oKq3CBFhw?%|CfI*#MD zYq0EshTvIkpn7BgF46GA8{G^!H#MuE{9Fb0pllg6y;GAK-zBKSw%4-Gx35sGD%;^) z!*$pQN3sEi8Swm`c-3MEClL(kYica-P!{ez zV`E$SUn8~u(G;tCkfCH0oe4n6Ac$)KfWQ^Bb^2Y&y$?D&a%-`*ceP>ca;5L8hf_To-NzI0q&FC^*@TvG%m*P4dbMu zO=wY((nhptpL6CprKCkv5-r-KO;SnL(4I;Q6(#ybl!TC$IcJ`elolZ(*@{w@MAi`L zKmT|0W?syEp68tVzOU=PzG&OUhxGYw>4sqa5>)8-6y15rXS0`@Fr5o8Fi+@VWKpS$ zo;4}Z8InP8)Vqx3_}xQS)+#G(%7CQ-BSaw`zK0mKQ3#Emi-~J-nGvf!k+iZwRW{RS1PhX(Ec^lDD(K9T5 zp@J&h%muv8G<5&D7;6!n#V9PuVvIgdr^!Q#^kR>o_L&wY+8lZy z@igky(ZpCUI_tw?HZ9{7 zx)*XD6_zHU$+0+UC{c?XyLFhYx`Rx~iwxR#-~#KKT~0LwbKuXP{^LqE4=~ft4ltRg ztV|1;zjjbryhBnLSF_RS+k(Kx{YQ8eaV!^KxDlFNA0wPzTIck!qe(^%?+8QV3 z$i5zaR^Fo zlmR9Qx4i>B;xr>;UNv)%8&4PgSi&A0%b{wy3bcQr9`zM`<{Ou@8a)K@w6605uf@}O z8*b9PAt@@7{+anOcQse>fEoHZdII64$I&<2NcJzDfwK2UF>K-=CN6MJgWN%%?<92sM(n;cNenJ;t;gMVgLi$I*?#LFl;rPFB%W z1!*Zy<%hNj?`rr3*|z;Sf^RiKgdMcOfMlC-9AQk{d}S zC^E$xeTzCk@4ooO7Ofs+ks@UuO%E9MC!w(C3Ve<79jb!hP4 zdfi5k4#uIvk!iG5axHp!r2yS~vYmbrFqU_=Z$;r;{U_SrvKqTFuc4V9EwnHqh4z%qL_c^x;3~~j zs_<2aYIISg+7rXS8YgaH@n?`-*Cm6hG_SDFmhNEPxP7$pd?dAy8|D58QfA5pnd%pM zf1`j#6YizGam>zYHDuP>jJnnGxzBGEA|qjWdUE(D8(|YipBxNCF5|g0y>t~_nvjEv zPv#8?8m!qp{T;%`470s^HN7E`|P(%~Ul#XdK-&R+# zRO=ppmT)0m&#q)PICpXvS>B+zdP?YaC!n`4)zK$}`jMl@`%=7$O^ddf;?tJ?Lg$!q)6IYDb!{;PX`Im9->3B`%XygFDNBlO=hfHnjYTBV! z!5oc0PWKK-H~d(Bn{PeCgdaVy10BeD$qetyLf_MrsOyb~=tzGU>*B^mCzP$(hl>a3 zLEcivWv+lTJ?}8rbDAZMiu=H>-Cj?NI_i;Dtpald+o4+j7`D^BnD*J$v3Gq>p{G{^ z=+~gz3@R-~&1FZC?(Cmj<;AgRVzL)~TO&lTO!}ioIUP*O_bGIXwgoLYxQRYrvWm8S zEu_NJ=b%+??#$b&w``kw9IH7YX<^Tb(WZSj1^&4)S|S^RHVmC$!@G0Po%bRQ1{0#( z?PiD$x?0hz0WaW-V*@Pr_e_L4giv=xIkPIz86^zopkI7J4RSq^4rFg(#d_B<@ z4kLGyWelDrGQO{m($7sdkxh3I`j>Kyy&pe@42ON0t0(K|*!c^{aq(Xizo&rxxvzt{ z^n4alQq5rg4h_-uuJ^e1|LtPGSo~(if0@%vg?03e*Df?4JYnQLyy?D7XO#4$iWMp! zr;BX!Shwglc2~|i*5A#U={Vm`6#TTC#>vQ1$FL>z+zKUH*t{HNdp<=f zan9(pYJhJz#{a-y$0NdJ#Hcm(0rBM$#qEB++2e9WGR7P^&;M*Bn|-w<)yJz2Tc^w0a9u zA6CK!fE%b)HipV3c~jraNDARjenx-`o!kEaF;m)6$A#1Eh3ofOTaOr~%6ls9+5VDI z-%`vNtjuKwxVuop#XHF5rU?BnUkjZmzQPZ8|HQ7oC(ArI;>3(@I*&ZZ?y#ntgOJec zWc0eJ2*vmBW*q-aMP)bS8DF`Rd`I6Bru2t2U;T0r8ac2VUFIJ{630i_&y6+onvW1V zIXfGz&J9CeLziht1E1#f9ptu!&!>&E-=mU%M+iSjpyqQFsJzND+SJ>GVo%hf6&omf z{Y)Trb@N0go=Bm;wr9|V&U4iD^umU>k88Qkok7eQm6yzlkGeD_f0*WKU1hi0RWZ^j zA@qV92gTOhXStJ{_AgLS_~qn$l57LbaNwB{TapU!tfOb%Tw~hxN|665%I zqkEiTdMj6+#;lpyu(z$3t)2goy)agJ2RG%}8uK~SL!p~dG}b~|TawYud^5VK zAc}2WP|f!@38zO?Qfc_aawI^2F{}J8)7L5DNM&9;?SH%st$FncO@2F!ruQpbh$2sP z#O5kW)oEqw(h%JwU&}5({)*;aI*k+!vY6}TzQ~seqb3iWsl;R`N|=&GFNf4K=gQ8o zyv?6bcU&xOdMQnVxO-`1vM$njj+wLmQ~6KTlu**UY_w;~DjHgsK)ZX(=s)k%)M`Yo zp?|XwI+qYmr>;AMWM`$bl0wNS=7lHPcHa+u)1Sil(<&;nR|!Q;d&PXeSc>YeW-w{} zy=YKOhaN0yMk~$k&_hB=s9rl6%{8@0t{>9T^N=3ot8x#Wez%(@dk-Tk?VYS$a25UL z#AkLk8Iz$W=V;DZeKbeX8YO&N$9naZq0dC3K~CUtmDzZO8^P&hjat4j2D;76$fYY( zTQP*4zVH|_5&g*qP4hwO1rpR?J%_ISHp61~dl#lb-iW5~^ilPs9U2-CrZWs%naxsu zOf=;&)fw4n2Csl_VAN^3*$(7wCEifezk+Q)qDbf8FQBQaIb4q`5hy{}lX3f3L=#u? z*v^J2bl|QE8ceRETduu;C8qlHsM{>Mth$XI)|O}g4H_|ueO*k#Eg^QrSUxwhw~0UK zz8m#^&1PntvTk^M-h~Q<%hAMDOVK^iRO+ayN|#>Lp*8y7sibZ&(wu#Q?mZHN(wghp zA;xa2lxNNLlE1Kr}A2ZKW4NY8}gJO@>(i8n9?1FP!7#*uB6qmD< zD!DtciB*!c=VYM8@R%=pTAV?l&lL2vB!kI1-Aa|6>dBwTSiahe1oSB|3dL_Aw4m?+ z)w~^r-dNW#H%4aC6`r4Hg`gJS|2u$AXKvGw>myA3szI8$Z4^mQt4E%#si--;1g$%< zwITf#VA~fyWoOI?K#`iim@JuJtiMYvm1%87*+0~f$5AmQ$&N0nrYmPpaC5^3(Q?Z$X33BV z6)KlOwH*>D_3J(Z)J^G(-6edT3NA&=tnfx^yvSYkx46`Zxzw=uSr#k{c*z*-cPBt&067z8gv2 z4rPy4Kc)9y?PdaPBrG!jND}z0no<6HlU@D6Pv8mTQN1^3X=Hym%B;CcP1mW@LT4p( zXiFCT!&j!c*8BNSXI!Dq$FHz@pqyQLOb2aBHl$vkf1u`5ZAkHZ5>wP5&lvBR(y+AL zga(VBqN7FP%uf4<^!H2$8Xdr4_Z{dW;=Y1&DBB;=PYulVfpzSo;2MOoB57O9TvU*f z!9TZFjcjw%Kzk*=p(OdS&?2OapoJl*3bbBl-_3x9F@_=>TD{z zw}NY|D@%(B zNxD|DjNKSpOC=W_V|O|Sq0ZmGP}A;M$~Sw-7>oq7?|pNraLg%okiV8%tNB9jYoe@T zvknumHkEDXucHdHOIa>Yh7CKhoL>}{NgF>srQ$o%m~lqFp~$}*wR##c!bWP0#*ay= z-6wBx{^AIuys(9>-Q|cP4?bq}WSY2Zf-h2SzYv7P@b`O~EZc<$B_eHcuhT9O`w4fpP;CZ@BFkAeOXfaXjO!|A?5Z~%@ zFZ%H?g$)`MqsG+&9qOf^hAW?mJo_Qzrov<2PEwj@sfu!2=TLb?F^gN#0?lXCm2!#{ zk@lZcti7`*Jt#Q;lC~rI-fjUh+!}+b9rfrfYjytB${Hs8{!wO{;fCnHigf0+%gC~42R$)SOy35dr>Q=*=-omiX00V3 ztt&~QJwdD3sq!V%=QN+m`+Oed@wF_9_YI;~PM4@K*M%+lxV(X+CJ=3PqQ15?^ zd2_>#zr2(|l?pf6&79xtuJ0MN_Inw6SD1yKGPh`$(O!`!C z9!bBRi^k0bI?L{SWc@lD-MZ{b)~w&oKkm-w=UG&t;ybI*C3b|Zt;nFczL!|5hiOzJ za1eDpX=l?HB+$sAeJD3>l2I=cvAFQAjw;yS;{BcUniiH0jO{8d1Ak0r4ONHMCF z9%g-drZerY%+X4|5A$v28`kteG96tsNUv1tu&cDPnS?!}Xw{cC>iJ{_?VpoNJ4&Q! zZkh`X>{(4e7AKd*gWG)N77> zyETg&;tDdTYmz3jJpDBC2)WC)IbPtd0M(3T(F6W9XK57iT&-QODKqYM%%!2zlsGE4B&(fIcDV;z@<(rZIhx7Dthc_J!+fDhC zdUU)#o4&X@&*F-90rDQOq-%7f=-oFNXiBUb)xBxOU7xz1zO+b0KWulRs@FNRrqdZo zd&Qv(wI2k5w%_UXvQjqVk`42FLX}-uHp%9Vsv*nn!%W1n9=2Rsl+hlrMHM|y7|~uy z+IIaLX!Ma9Z*P`z(cOu&eR;= zPW#`)QSGrXG?)Jmohn(#7+(&ciXPfj->ns$a>_$bKN@p~SRYjR)rIYTpTbToD4<65 zipYHZH8k(QMS6LW4=SGWAG>?r8+5^x$Ic9S!lI+AX?bV~IlUjE*+(Myb9+B>*B<}C z3LM6$JxvPbe3?c+Thyaz9#bu1^M_E>_z(;9f>AIxgQ;4Ri}q;T=6Bd1WL7uy((@Hd z=p&IuG}L)Noz`SRHyFxOp2{3lc+`?sL?=;%PLXc-W=b#0rJ-^!Rr;{N7mcXT zVS4u-VlS!pa#M9yqky_2%m$u$!yNw%s=cHHh3t)^BBv`^-?Sj4Zgrm)*)2!>d!^K> z$$}16*a|$2wNey6Zhsc%V`!yPNFSr z_ET##8uf+m@_Y+>vT-(>dgU>lP`Ja-XwRg=sdaS5*(!ARKXGdPUWDEgR-(}YPf_vR zlic`?nsnD4fk!zf4@nONBF_XDbo^#7-6pw+`i7;U(!=K%F`qP6PRp1)*WmJ-8+iQs z+euvg-EX<+YigP3pC8ctryHn^j2cDyFZkJIx$K|ZQu<6#+q6C}qc^vlrW?$L8TsxE zS{*HqUf~$}MDquGZABQhJ|Nbh(Gf^fn=jC{vda8&^<;WR{So^#{|R+nuSJcIJEPva z+tHov{~-s#yK?k-5a-E)LZ2-yX6)UQX?CA8`rw&DEr;z;PJj%0I;W65 zTd9h~?cAwHi6q_iNsKNGdVzEw?51;mnX$v;k@RxF09)-IK+EG-p}_^~`H%LfQV~Hn ztEwx|<9C%&J(XzY>3aS;o##Q6==AAe&T z{>9O-haHUT*c~LPbP}Bx?6h9fctXd7%IIh?VU<2kXFm40vA-0Rs7UQH+N+{Wk4L^? zCrYLw)p^Oxj;*s%otw7BhLmf_r*tmr%yC2}O$OBU+7L5%CXG#U7N-@ur;*SbKAJit z@YPwKML!OD(I}-3{^qjB=o*uNz}hervTO+1`tM~c<_KaXu12s-U>-kcTZ&-zEoEW& zegWF>(1D)b{E*?Tv!S~OD1Dnbz%_|qiaHnmK%R4NqxVi4XwRx_YLu^ygsmdb{c9X* z{m%p)eW!yAwwa)Z?^n?vjc%&no7vt;Q|JC`+eOGk#52I#^qs)aY+&1l_EndcS21uui5&$~%fWx$S_c5%}6L#L% zID5N$3T4h_qpf{q6u2A#V{8#B+T)C3JEfQ}g$ye0hN$wtzbMOUg7uDkfS&G;VK-_f zQ?}($eBzy20gZ88&Amc{2t`DbSY60}hA@K$Y?sK}ovkI+E{6J6boApd%$&C1MwTImPV5fl?aPca+wMwV|5evwZhY|JWPHBpNaWzw@r7DZ1d`NE5{-*~xxI zT2$$Y#OpsXIIvmZy?su3_dZc^1Hqkhst`$(i#1etkxL=x_;(d z)VDYRg*G=pX=MkRZ1{?O*XD*Cf9^&>Nxsb8;1YUrLWqjKoYA218zAF>E!6t0JA$Lz zk)Z8F+THE!isc{p&&s7y%CjM%t{xA{)lTM9>o&0y^}#hSm5xT`(^&q?U`a z=+t}Ks4--i3gfHn$D8Zn)CLV?vLPAiycb5lk5sb-qH5^Ae;cFD+QR{n43@hmhsp5N zrEv?-&}_f=)WKSteYmL+I#%!D_u0N@7m6y=TRZ;=Jl!vug|9ML)fbY~_{420E}q6z z3*MPB&qM6pz?JMnv5Tl^Um40ZHbjs83>(svRVgRGnA*$>MAuzbvyby%v(ro#@jofQ zW7iKKK#RYhp@SjYka<7?P2`?I$t#7>k#+rSkE#xloyJAp|K+i7MxyBFhd0<>Pg$h3 zt^yf{ToUy8N72vs+tJj-SeAVAXClSz*mj49?AL`4n9oa=(A485=!~Jbh4IH)7;^U` z3i&EZbGXBZ<2=I7s!v3--9GC_l&cU~a@hik?sS2_w%U@<5$(Y7*lX-i_ZCOg-UX6&hQRfEo%zbk z1t30PALlBm01K~HU}512{Ce4C05vqoBa>#}>-Zi_-6KU_D$9c81TS9jT`{b;%mdu{ zxF3ELNg`*&;s{?M0m{7&B4K5FU~u0)nAIOe1}C*(mZl83wS7AAAc{~$_8b1=BnR7i zn?Mx*F7|8-B+h(q7-HuKwe$l>oc%C(wPP+Z@odIRe;)zD*TUdk9zz}wYcd`iD)_z~ z+)%cH++Sxw-fHW@(M7qy$F(1O{yxN6@z5EFT;;$GmX1VbM4LqFy2B0UmXmcJ+VJ-9 zJZSaCko?^%g!@!g!2D7r5GT`7pT8m$R-N5RjL9BSDi#Y@JWqt3V(G-%Jd;f4CBeEz zd${_rA1Sa5Afay@p=op^vl~D5#7$Fh}g(kL;P#3;w`ZGVEkhL1zE|=nH<`06VW)`2)e}xY$nFP~4 z=YcElm*Xo{_v%~Dg@PwbI`NUl7T~v=fa>uV_)@e4`P@4lcE-rSsY>eP@m?LG7pDa; z?$CrkM>I)lUldmFlHBOymRq2z<2Tb4#FAZY59zX}BjgWjtn&>8-#e*j`%e8o-CnRx#DR$%;UFH}_6OAhbcPV_JDhVv&2fv%w;R`NS5;J4lf z`Oe*7eOV<|+SZ9*4pjgfq(gR9SirL{jiBCqN1}Jfg19D4g6(sxVEYb7BEj|eKRMiK6X#9JzXd`0oMrIlm?N2Hk^us}V|X1~SYGPTF5rmf5Tj3mO!0OI^*8Gg z3GfQIofjjrG4n~K6R>ye?@7SAb(JYH$K{;}2IId2lb%p7)4RVS6ba-=hH z9^CdMmfTi~fXeoXuubg%xv5kG9wZjyQ2`gm`q&8&G}#CwPu|6jiq~+C)^$+RqDDTL zXhHdG73eUaMIMLylm4@Q@ZE<^u-+h$T)m@7jtCfHdi!+Xgse8%oEHV}MQok)0zOGt0+bZGisk;C0^`T7frC{T4{V7bU!DcSP5xn!_(YIjg}UI5 zvlxD#Ur=un<;OX6>K8aDEKGiWoJN?V(_yt*3c28(1l7z^APh|h>i3 zKyPSPydM5-Ur#`i6WLVl0)01bBFhat;HZi>JPEu>`0Q|~lC_n{YKDMXy1wujR@e=#@;3s;3o9^DhgRT8wrbJo@K;hO#j@uiJ7-o|Y+5*k=RlHfTY`Om#B8bP-8=tPXY0E`(+k^N5eU zG--335A{O~iMWzJG*UBy`MSnLCdZd-3fKspD&65X;!U(JdqUL!cXIB?dNNq=3W@V0 zj#{1&7WpWRx#^!d|1DD`mp&-M!BH7FBzUK`9$E;?Ow~zknJUQ|)__eF_JpA2u-V!c zK3Hx?-k4f}lu{G?%SIhnj;Vkd24=AGEh3*oxa8Gb>&jqovui!(jfY^NZO6f9(+mOAq6&2M)qn`4W;}cM zYNBSc9EO^$f+HA)$JYt1D!yXKI$HG=!gCIVg`-AW!g zMiQIS2)H=03ZE4@3try64!o{WoG79~A~*`LZl(n{Tez7Io^wp1iz32%` zruxJ6PyNYq#{!UjHwlYY>S5OQ5V%r36{?j@Co5)%l4l2}K&e?4JSERXKpCF~-v#`a zsr+y#+T=v;T{=jbPlUn>Pa&9Ow+kD~Ir2C`*?`FNuXy#N+OyQb zBL9gjhaX5Du8)!= zn$9w$tL7K@JG2gJ3LYPCyQ>_RAsb%xyC$BgUKVFYa~cP#rSWo03vlj+Od#9VWnQR1 zX}lwh=ID zj*gBCShAxySYZZPXeOACKaS~+5yzJ&o`4WW{+%NNZRRGMFS_7T_tRlXqh;-X< zq4eqzUfLIH5Slp){Clm2pXhLih`u2l@GavUv7AHXz6O%#TOy#@@11aWWhnWpE#Mdx zT9bCa#pFsD1K+RONS=#sgpREn;i9V^J3|r!0dLUpGRj zJ|`lyP6hrrtwM~55nk2y4;cPiK_ZtL!?tI3;AQAig1sEzfu#$H!fziEGs_Kzq=di? zLlH!yGl94`?uDv{mO+c9>xk!af6{u>2maW3Mo-VQh zQF2_|)9b_2uSft>B8Nftjr*ofu3g~4%sp7)eF?t4?K(K{uFhP4ezN)3t`K0Bz6OWr zc#@CfR`Bxf#ZY9U9Z~$23{grVp(Bap>bw*%>DZv!*nJG*>xa8$mMg+|C$f`4}8EY zZ$uKEpcuGbAr#J$OC(7vwTb&_T{tA>0J9#Llde5qaLn6opqqLZXkU4YTOHQGccp8{ z>EMIpTU0oV*}95^TweKqVP8WD6S%(#Z)9{1%D)7n56z*ph z5%hEsvDY+)2W_^%!*mCkCCKRSnZ zb$}(4o9g4=DVo{;P{Y>?TR6?7R^-SJ6KJO43)iF=l6)5)J~dJYg1b*~+O(3e;S@jk zP&tIGi`+zBfvs?X-xx?*r$n^pixd52-@t~1J-j>mr?3u`AqUiRfzYw{wc$cq zca;78%~)l9UPntDH%tiYAtGKZVuw zHo&EtJ|vIlOY+aJhfj;zf$xV(;8jWLGcV~w;VL_lmbC#E*+sw^-?x%PpA}%)`&w}6 zP&udm*l!SW))?@|)38b2b@1_`2#|Q20ONy$h{L|^B=mX$6#Bgs6eXL3=uv;(DMe%R zbQcjg?PD`ubI1h`9a#!e1n0rZ7&D%-u_BQaWI8^cz8ar-%0K~u8}iMc;jgy_IJREG z9Ow7D%o^eoIm17!2zQ4+Y%JD;bIz-hcfqYVB6vE{THRz`c2$C`)$$}&wU+QfM=-o4 zV99DWN|E5d5DFg;fm?@X5v!m6@OPs);qF~W_P^N%ol1;hC;vbE*H@j4K85h%LNz$G z;2*AOvL^!tTzJ{f7JSVb!l%N&;oCnBgK8m9qJH>2r#5pHY`POpy02&vW&OEeyU-@i zB_&N#<8cMEOZA{n5rB`(?qZ?cHsrP|LmGg^#SEe})pQp6NqoPoKad zE9*GJ(b>G^Czg=DU9(`MYAgP+U7KvQF@V|+tjNx};?VWGD@?H3MskWIIh_eD_*-u- zo>M8wIY~;uRoh?KZc>Ak8Fc`M^?c4&m)p3bx(>hJc7kVT?ZFe1G6C(Chs~N@(|A8# zYzG6{F4&@Q3Fd@2fEL*>U~xwQr-X)Laocb(Zg3XgzM2OPJ0*kF0zTN<*30DY}^DaK~H~*4)$Gmr}hGW0yD6w!n0rwuuhpTFi zk=rj$;m`iRz`g}DVN6C7J{#*!LhC)ClHktSa>A3;ZVm;J8q@Kz*LA#xoo>KT!1CJk z*avD2Y=dQjbK`Z_KhEvSH8^_yGoBB5#~D4r2WLNh#K-sl#4cuSz(i#MKEAP&S2G-o zo9y4!fAif(eu>4vb2uE@RqQ1tmVg|s=fXyf6)<&|B{`@l4^M28Bd;gLiQnn@aQy>4 z(s02L${U+O(dQ=QLV;U-Ilqc?ZA}Kx=6p^4t|=+Nc25l+sT3w@e{Olk!9hqW@q6JhOHsT(Zf6oEOwVmlxTO4@sRh->?(kad7k4Ao!Rxl`I95nEW>#T3s&3b`3e8d3+vNPE_;QO+C1BY4-L?O4Ot6L{|#0^%WqI6U+g4p+PdCe^pVu5;T+@5LQt zmEit153?W-V;o_Y+$wlwWF<*nJb`sAtH7jy$#>K3CSG=FEeKw%gZIad@l4;EgWMZN zq`FucwhQ=bIf=`O%s)S}L&^?5Dct}+?G-RdH(J4g~zBZN^{m6OQ;tu=g`;vCS zIa8711>b!PCfe&mA#1iBrhMH*oYy)--RRZCFL^ETPIHEW?hMLH0AlKHO>Cd)!WR>3 ziPRx4I4Zb*YIdz9euKjBO!H67HG76Xv`qrN%kpG+TnZ+nOTw}KnZ&P3QqWgO!Jh12 zK*3l~9}wwPF-kRfvaRH4^oFevyh37_m-f{Wko0RH|~qy|{S zkj>7p(8`X~hZ)1*Qa$qTsUCS;VF>4c`C6B?vatS+>;hiEsuc5Y_g0gMQU`bq*u!Pd z9f`L>Ch1(C4!d4vz>%TDB(ilAT=T+@ob>S{@6Y(b?T_nux3cm$5@L(Nf{J^*$L5#7 z-B~wrwe?FZ|D_c~%$vd6{9rRD$n?x1diEZ|E{}$G){(I7P!#FD*~2?0x)jXoJP3r=8{>Ua&g8~cd#G~35sGK6 zCepH7h{x3+xS%KyDxMA|cb{g#u+9v!3T2U#V_EQKpc$OoVNN!>tR!;VAUsRIgE2D+ zlC*Cw`5Y_?|6Py9=jWz^`kqsuv1U7dnIHoDy+nxh#-CWsV>;YCD+{hS$s|f&vdNmh zEI3DI1?0qAk;vc6iLTR1Sn^;2PRN)E)?RDm#Ffv*2E_>4#Ue7{q(Tf-j3Ix-jEo)T zz`Rj2xQqoPWX%=)dB;`o-||Y(o>7e(-1ETq$ZEXh)@^*bs|;8KsKS6$4YKb7B&Du` znb*`vaB3_Q&zo<7bC;b1m0Q06ql=$#@M}e4ETRUt$~hC^Rf4Q~js-kC%bg4i5rFdz(T1BO^RIo=m1ZNrcy; z55gne38ea#6mGpg4aD0AaBTLz=iTgU1TUr#tfW|uFJIz=x=0D~R8mk|*Q&w-!Tca! z&V$o>@}wY+Ko=j&oe9J@31&T2#_&1e!XMQp#CwJf)g00wrT_!Gw4wNpc_ao7f@hD1apCu`I8b*4q_!)N_RZsfT`C2yuNNmW zQ8rL~whdte0qJpF0jniWf+Jmt_-K9{j_61RYvSWUa_S{4z9Af2sOEu+qf^L?&Z)5I z*iX>7Wg2OV-Uy$M3eGD}7eX(rgM)3&yd8S)%yi!^;P6y_)=NAr;1!=4FomYI=Cw6n z%#-xj;@mYe!R>BeVBx+Jcb+cetdz9kNzXLp4X0ah&N!R_J+cpQNl-oR{=f%5P1@v^ znKEp+s}4mJ7m@t-Eil{Hmw=hU~a+_Pzy$ZRqzN9OZ$Kavv-j_dE21j_ZT>z z7e!p64iHcGSm?Jn4hm%+AgizW65hS7P-0#fbov`ZZiL14LU(bR|4koa( z#)OQYvmhVDI52322M!YQ02v>)gUqQ;f(K(8)@jZGy*6H;s5cXb)bjA?!$vUuc`;b- z(}pi;HP_p&FET&T`q*43dm%6Tjv<+UzyQ)GMi3Yq5YfhdoaNXISoMDJB>5Ru(0v4? z1oxO&<5hh2ARh!}#*wV_SeSM30Ca9VNId3<;h&SP!0*&a&V%sPcq}yo6i+nbc)wOG zCRz?sEFr zM0538xaS+i;jb`gxq2JCTyYDp7Pf{SAC1T=V?=s20epTvmV7$38@evp3vGq>k{*4^ zdzG&R90i%<-$U>6Jf5Ba(ktTdvGNG4H=Y50@K%#OMs5NYv=dak?m{ZLx%kVW6X3Ob z2DlZ$!xz5S8XxY|^ogU(GSz42~*SE3M1hU_BIX`%2!(-xRAa~t_7m?zjQw1$m+YvC&y zdveEL58w2%26Cl};11NopGy6}l~>u=bB-OZ;JJXn3RiHuFAJ~H^vCnfcLCo+E6B)G z4S3fVLa8Z`6yB48FBIkz?FKor;z$`<_I_YM%p_krs+ z9^TT>g$?Z9g8=!DAVg5JZcG}*k6(|1!!A?cNW^D6?CAm3yX=Wcs4po!;0Z_d`+;NB z5Z?Oq56;m32}I|x_>lW$uz1l?An~IT4{t~SMmG!a&x#{hI6MV-7^&l|2P;AS8eI_m z)&_q{{D~{$$3Wkw7vPk`5Z>Pt4DFl)$TD6KA?ZO-dv+Gaa@in3_Xc=YR)U*dVu6}% zEcRNNh%1!Rfok?lICZro*>Gz%DJqnP?T7wx7X1*#H(Qqr&T)Y{@68AOwe`4#e*tfr z(hM3`M-g4uaA+8?2cEedO{NcWhzT@=5_?_Xv3Luz{ndJsm}m<>nR~!s5jV2G&lGN( zYCt%-V&vt%MQ{>(kd2=AE?2}YAv|OZy|ZsFo$>2A)8aDH>=*g%8chq=EIyD%7h7> zLo5VL#Fl;g;menMNl;@vsoowBe_Zm0|3){E9p3ASN0S%q`sqzx`MAUC5-;dk=S>1` zhx2^)m~xsP3ISu`I-Xs1Dn664AKZ1;2cr{le4QGf!8l*{3LG&o-_yr?dn0$s3d?q zi`-09>;#OW$n`KJ%b5&cUrYYPyTa)^#Nhr9lH~Z!c|_hp8tPuMA-A4c!P+`!*qQD~ z-m0z=+(&kB@h3+pn`ckFZ=C~g#}jaMR0sZLTmed4J>WZLJ^8)YS@2v4YIqq7a6f(_ z4*no#I&@43%(-e09m)i{;13&tmSGEBcM8F&z9K|LQj9bi3OG1d?_%N3yI@UW3()^_ z4=->o12un-;TW}M{N?vWkZ@}^F{_M%J(|0qyi*hz&6q;;*8KtV-tfVld#`bzp)s*v zVgO$i>chG*Ba**RFp%yRXw;SQ@IP)Oxwf23jzw@`=FoC@qmWC~9ADzqnF5WJcN3gb z>cqR`lgOUFB-p$-89F2ZlX zw=;p|Au|yBbtXP8k;MT+8THb7gJvI%BzXTM-C_Nt6Dd3AN>*)L3-h-Zg9j6axL~Fv zNfXKfk9y;YhfO5>xGffj6&)bQB5T2U-%2cDav5ZPk_3^W2gs0oG~Aka6g~<*M8bWJ zn@=h)jar_TG0M1kckZX_En{Cq2l61Lbxg9`Q=pj*H(=z&=G>pW}5KGE_3qwkutm|&mrRu{-pMb6EqDM zFcHrRb58s#1Al9C@uiU%7`A;62}%tm?Uj4srcb{(jaxeEwD&dCAJ@3ek-Q|}ZdZE2 z$ba@wp!bs!yHv7$XEv;&hvC)YOd|4J6=be01w@w0r|E&VTUkmU|?jduRyfdIndNv@XXR*ouC_3+Wtll?{XJwNW z8D&LA5sBx%oTDwXK}8fHt0YMjl}#d}B+=5+rc|E$a&Fq+_AZsQcWG#9_?_Q>yk7qD zob%k*eSOCJsvuc@`}5Hdi^sHwKSe@GALz*IrAxlc!m4c#sNB0MnsW3zS#d?A(DZC3 z*ZP%{TS3q0D04lO_zi_EzsJG725TJN^pOOl{-uG(e$l`!*U8Imb`TLe44;>E)1}1& z;Bk;E40|*TXIQzSdV~we-giKn-WRTiE5o{6U3@dcA03B#!f9`j*PZEwbuspIXwFW_ zmquNGF6;bD7L*?$`XYC(c1RX{TD=U9?p}s}99Dt~ts!0+7in~*4hA(hk>P(WP|f=z zIqPTy(@V7QV0u4LmRgJKI!l~Ys1MrCoz!x}KeB=h0WAjwOu6EWR@3}ItK1(p+>OPb ztCDG2pQS|lvYD*%zd$Dq2m-STe>~b}GG5;h3eB8&_Lz)BAM>$zscjbgr{s+ne=LSw z*|~7UE*7m@tEKIBd1TC+Yc%0MBZ+qN9I{Y6d-b!HkeuaxB#QbXNBB=S`Lj}FqqXZ{ zSz0@dir*+%5p#yNH5f`iW;z1a>Y~G%A=nl_6o#L#lPW7*lgK*{XrB5+Rnqaw55o?6 z!LkjLpwVgy2Ap;V%XTMJ?Qz9>7oFjB>Qb_xy?|OyD4?#5(ZsCn49yAt6-w3ID=YzwZc?;Q+O+H3mFY|sHwOHO%;k^ zc>Eer*inL2nMn}7B_7?h6VT%Ee9);`1NFCxQ3XmdeNG8XAF>cHWX^;A7K!jCClN`r zIIFB44{d@cShY_?wS;`MSdjxaXXQh~w0taEULqC3H%hoWhQv(vnH1t>Ve_Id)Z0ZJ zvvakeR51)Bk7wd*{V=q?849}YXXwg>*U1A(JE>N^O<&zp#r?jj(EM2kKHF*F*E`ek z(!u~((l`sWGUni#|K`E{yjc8OJs(dUn+HBmwZLw=CcZDzLZ7o*;CKJ8J||R2 zbYAR{_V`^TUy5$iY;*-0l8{+SNcurOl4D@$(+h!sfK5;6qn;2u;R#)#`i3T*e?iv#iARf1F)&pj z5msHEhhL`h(iB5q$%g6Gl3(rEY>==)(lh6lwC36^sb+Ymq*U1jvK@xwUnf^g=@|*W z2RIUFY)bdP4xsY}EF>ZCPtp*Xvt*ga4HBz#g)Y0(;r9KopQPWwCh3C#@**quCgnp< z6UCl8q{8zG&2@T2L#DQnaV~#}Z}vwzYN$Cje=>oG7Y0Fei#e*E$f1%sc_bJXkR>~2 z(?b^?(JdX_Bs}#zIe6?2eZS;9wG_P(#_tc3vdN+j+-DJ-N{UCv4GVGQ@`Z3-H5{JK z4Mu5g1ooL71&Zr!aZRo@G=|zk?Il|@PTho2yEZ~>;U-YLQHxL4x?$jfVGy{QgCb63 z+zpl^zY}Ds(O*MqU8+nTyd6z^m^)2eZ%1pwEy&@Giuh`{Di~HO!Rjdr=(PKQ1nfmRJ}rx&OH8nN znkl?bF^0ccA}dV%-7*Bb>cY33Lts;s8{WE9PJ+x$Xi{*oq_gRnV5ydbjykbu?iddd7sle?QbS0X z;UM1E-LQA&0jW%0mULeGIO(!EBA0GR7S8{)5(>Rnf#i8EhW?GldiljLX+Ep3(O1$#c`YYu?J}J# ze13`~zS>OZ&Oc3U`^kX9HYd1U&Ee(KsrYVXKFpkv58i9Ud95lI&m9YZ%{ft!-5rAu z3&V)fkG*uR)hzni)rAz@~7P`}+QvIln*raaaI4hr&@g zItBGq<6&g#TiSI(6}W3k@SS@?z22DPx&K@s@$O@y_Ubadt6xY9Zo80CS|Y!E?{&)g zD}k!F4mNHNzz?^rq3S{uU}icFFJFOf7E7ROcLa`jGZ#MnTnH+n-{9yo8N3yx4f{kb z@nCXaTnW-8Fx-lg|x{xXq4xF)F`Zm{F_`y zO4KLNyT7tUepdm>59*I8mS2flHi2o^24J{pJZ;6w3GCJ~LzUf*Fw@fubesJ#dGcZs zJz5#HU-ZGH`K82YQ#N+fEV#Qq8^Ralqwe@Zc(=3=!!io-Zo_KuI2TJlJ9UsB;+^XK ztgBS{(PV7q$G}`s`;U{eMelkJhIb4@t0DjB6+2Tn^g#4l7P#P6ab_~KbOr840Iv2) zg2873q5X3VM#N1({WTLnt-e3V)S6(B^(4G&HwQvb&4oUi{@7}rL%Y6@BO$d{rH^H- z==goPsGF1qKh3iscHatI7kf`SSY3mBq_-sHo2Yc}@UK*<)(raJ9}RmZ8zGl&-aK=p zlT@?ogY>3}vSjAQ{dDS|^`vOaDY9i!2UU-og}2>if>lQl9G)MF2}j4_oPFXBdaE5c zA9qLPG!;B?<}Vpk{ft~<8tAU~fE*0JPrn!6rizO9iRzJ5`1mLV)A}dl9NT0FFG-}E zpH`D^2QHFbS@UQ=@!31PU@n~gxDdECF_@AgOUD}0b7n_-*+^jeaVi zWz;=VGcbqT+gL@b7Pd$Y!h(q9;8r@Th!Mka4djj28M^M!Okm0W7{4P3>kdS~nzwEA z!MOj3?eR`hu~DGYi%-)(3c=*Z?*QWAm_;~yhf z#B1U9Ya`GkMFLq}W8tfUD;E9Dg86kf1?|-KahLQ za`4snD~-P5LbAlaQ(nzpH??S5-hk=Q~l;|3iB( z6u~{KV*H{|ieulbfw>=!(9_?m$okLc3Dw_0M;Jb(@nK)dzgT^kuuK*Q*VvIe4L7Ct zfB$uhJ2g>~JwOh2jZnhd=lh|sR~g!(S4!)(TO{$r1!=PNf6Zjec&VGT;CP>hQ>625 z*fbAiW3gqaGkiTL&aM5O5p_38x9wgoxwQF!Tc+MYsmifq^yjpL#PsYHvCG?4Sr5DbKldM%H2fp8-Vqqb~eSKA0(L#!gE!bpx!qJ{U+z)mi^iAe9;eb zHQ_DYvR4PW33Bl3$9%YcJpo^wUWjAIC&9ONFIaHK8ii1E%nx)2pBw3f56PnOh8fbN zOiQwVd>idkyAa?vPTB1cU&c=x^t-M z?CIoI{zkG>;zzIa|3>ptK9jqi>Tq)RZyIr8ptNU$zI4RK%Wn3$w#^Tws*tWQLPHPE zq62LSaao;?Ij@&Nz}!sOH7)~1H=$%(Ww*5PNVN2*X0^nocPcvHoe7>sGeIk68hZWk zL+gFMp!muUUbp#UK*0)-HC~2|u9>J?n-0&HuOQPK@~QJN8+yN?NPN#Gfy;ss2p>7D zOYnv|kzJbP(jWcmtnpftF?^O$!nd|YAQ@``h2kt+yf+0KcPs*WU@4e>%)rG5Bhe>1 z7CzX_hl-Rq6v3e|F~tQvcM;qg;0Ok)%dzG|8YCtzhkr@S@blLGIF+}C4J~$HzS;x_ zzBI@Fl?E{Bu{kKe7=%*;v|)p;HXhiai53d#@a}Rhejl0(K5Dt}EHM{pwJ*l3@`MZX zd_i)@3qyo5fYs%AJh=i5XIDUqodRvt{w}$_L5|FRDo5vSzDny;Zj+_`-jc5_m+6By zi}2^;MBvUWhJ4S3Xf^e&$Xsow>qdQ~s|)TB{gs0O6ispHY-bG6GKU$*%5v-$aael+)$h z3@|(vh^BcV;+`r1>Rs$<*u;FYSYZGu^|GYL)d#}vaawp)&k(nbG6C(_Y}h>}2Xi`e z@vUh#TzMs_KvqE?8)y0B zK|@dQ@AiWsxBT!}wH1^tvqJNkws`RR5YTAzmYmq%B)vH{Lt0KYOKgG-asE9MV5Y`U z_*k4fIvnxf2TRcI8Va{<9kKay3B!Mu{*S&)Zs zPi2BhMGh34$wQ^j_V~!4KcKEQ_^z`?$E`8&#UT!tmBirC?J=<0XgrwS0kokWSUi3d z$j&Y!5!07ZFOMv`wRI)AKiC=0JRE^doExf$y4iq$MoGlaR_V8!hoxQhb&`vDiFkhN zJn&|Tko;>N{@$VvyF+zQ?wTTQ?4t$MFZzOMiaa{bl1GK9a!~Q(C7tr}H92kflxRh~ zr`l}^ST4+i;BN~c!z=+$1j*7DoxPHsZ;XjSVW)I*3kTN?hhp%Bp}5P`4b1E1q2bzZ zdd5xxZ{c4uAtVdWvgIKEE(?zD$;5FhiX?Giucbc?+NBHD#7UBueV444wUSP;tCiYU z-Il~{-6tJC^n*mVTaB!Id`2pBUl(#3^sp{i2Z^c%1j%iX2>o753uc+pCPy`rS-1@S zVp3sI-cnF4PQwXB7wB}eiv;4%l4lNe^zwy#*t9AS?~30yCqwh0@!UGHlUqX#w&u~u zb!*7(%VtD=?;tui*MmMTwIg3^%cWCV-nx-{%U#MH zORkqC1J&{*OK<*^-g~o|W;j$6z1(6FF{PGjk2Z#7W+qs*(ExRp8^Ma3ws^(B4%}bc zfe>qh&U39`%LhxW8Dfk57Fj`*PZW$j5`+A;c{p@vES!2d76%BU;YXJT7!~oT;y0Dp z?ewK5=K9lN^FzoO?EuNON({v4;(K>L!N#3W`H*{ZRher=WDgiwIDF+4bh5tRS)L-Hh}OM3U#8)?9*6O!rf z9mL4#9<9ICMyFVxCC=_Xv~|Q}QW|GVqWas@tpk(kowO99b})&&a*n2_MhD@7!$IJB zCmf_8|pGZ{&l<(PAv$UIdqnOW^wIVw|Eo z8WyQaaGfoP-eMkZ>?p^gq2I-2ZUU;RiFDkWr!~O?TA+j<6Zz_dhH5UYZ`pQ7Yi8s`vtCQ{@^@}Lp@xkR^ zyr5;2AKYB*hd+xq6GgikYPu+wP6*8tv8eL;E5$@H6{hTkMDDx z6xB!C(X-|F@w4lWuYToCs-s5JvDU8iMu8z&a>EId9=TxPemBhD=L&&KZ_x97&ynxi zXUMp#=jl*?L+bA$LqecMa$clp?ll`ockx3A_pLwKDGZ^JQySdHOfiv;>9v!RIU$mw zZjl3@?TKfj{W1HLH!Nrz4b8XQ@s60S+MCHk<)<3yipWw))2tTBtKqk$y&iDMxYkta5@5zbq&bwW|DQS$Id916?Z% z;jl&_4(u$%oZAI(T=e19lz=@D? zb~Nt$%HfpDuJCW7nEffX$HdBEF!X^1j?lG7zl&B7-*!U!+VdUcF1R68n6beHOFrbg<%e5N_#eXyLNZoRnc0`;{ zsGA^GeDNAU1`lD_=a><_Gg{4y4l`nv>Lcc{1{Z8SU^(fXyc( zQMxJ)C-=s|j<^I&coGH*GV$=@#5^pH4WNN>L&zS>NTTv;5e>T~`b4AFP^rvnT0C_Q zS#;P46ctVI;T{uoGB5#azbxAIw}1pY_>)xw#?y_Vg>Ww^A7_LWV!K)aw8Kc6al(uw zt{z5OREAQko){dmDhyti&IO5FG!E(1L)`&7kZEBAQ|=jJ$wR>4%`UKKmjq;PNU-MK zR?;55mYRuBoMY3s6Nef-s&USo#EYJSPUm6NHbVxj#oX%`MJ0G7tAIn}7Q+X{1?Zcc zhOt|dfdcM%6n` z;B(9gA50vEzb-mL$YK;*!ema7j8sJFIDEZR#MUFJT#Fg}8j^yWuvDEI)d`L=+ z$AH#YoFA41wKo0ex;kI_-&0wXxoSZvOcqvuEH z{Pg!^klzoetYn@D41=NlzC@UIX_3@c-

    J4ApSJ$NM$l znfF~u<(CJ|CqMx{j0nb)MOLsl{)V(t%)6b)KKlQB6diZ^rL@U5hji-bV*0&%q*iGK zOb?%cPl~V67aP8lPn~HnF4h4LZ&sr{`M=557n(5AVjk7HC6KfK^^=}mKZGpFv~9lT zWsjeV>&fWJu{8113Hq=@jks%S({nNhB{wb}AQ4d`h{~Pgq^S7;EhyE5Gt7(ZiO3)|x+yQE5KamDb(ZpT9t)X_ZC*Bw;k0si3z;B-&ycBhh2e^ss#Fa#y zg`>yh>3~mO(6RdtlAtXn_}`fRL}yhAy{%zSM;#D%rHQt9-%=l2HVh*n#1MO~6-#C1 z?$Pw^N2Ont`vA$kK$kAqM%R>9&}G`)#C^mk`t9Wa@YpsIAO9T+Gk(@dmo-jA=N&$< zNizo$y46uj-whmwX<*XJGAtg5b5uKRWkjCf480ge}Ha66ZJS zI4mQcFqd!&M;4Kutp8}FtQ&MKy-RmLT#7ry`Oj;PJ2=SAzzdoy;gj`Y$>E4dT%}SD zpG$3|vGPaA7|RhjMAR|HZ5XVoNp(mfBRT`_=wFLsjM{ao?!O;0%FW=^Uf z7rH5b?@tE~4kdqd|I+!P4s=&T1f&&;*?BEXI{vCQ{oZR0gVV;)n9Gle@o^yaSCUZj zY#iJv8VQ?k^p~)TIFug}10Oy|;kssb^62j}nD(F$Jr15B9orw#F)BtlC({6KG-u&B z!-ZgFk^wn2OR-O1GkDfC1nDsy?E63$Ca#bHZRH*sp?sZc`ilO;knNHKCb2Xuau0Q@ z&?dp1rnq;pi}s_{=y-p;$HMSbYy|XGG{9-$ z#z5aGf}@K8#`RF@e7-*+ouA3o-6nYawWuFozd|fN7=XFQOiViF0BP0Zu<4W)Ub43Y z6?qMe`k@Xh7VCpayaBphDx_b$HAOi3Ez9phmj2u%zEquHKRs=U6%xJ)~_U$RZD12WgBg@I6y4nAAK?O3hAzx zBFT|mPS>R`Mx2rfzfYvVuz?HlW71quDqDylYKzcE#s}n2reeX|MKIKUF-)~uh}Vwl zpye1nI72yT-Z2{ex4b6mn=jEzb|!e`j|CWfnhCFFOvg5hnHaid4!m!&hPb3*ILy}; z_sR~0gpUA|7ue#9^%`gzWe&^xO~KxTSupNgGWhoM!=r!y(e^{~@XJCCcI}YC;g)G+ z#h?vT@#;tElFi0sVEPCYV*%jta188gAA_UpE>OAmo5&w=Z*j@t3te_O64IIyDB8{=Dgz5i+<_80b;42NwzP@9eYlr4UVcEz zc1^(x>eHd+VGxXvpMf6|_e$ayW|GAF;*@!zj`WYwg5^sGu24f#_=WKZ9vwl4R`Acu9tOLr@s>ud$|ycw3ITcGFt zQJ^myi>AlJ0osG%@yalq^SCe0cGQ74QZe85@-3C@DFrj-V$2FG#+-e{aA@2(#4{t{ zbH532>7g(3yA^Sd@n7<+Le#q)%y9eVPo!m%s7tq9q)Q*WK$}hwmHe4ZJ|Fw(rr145 zx^VLZ!2dkZ{N)%d+~x@i>*k|JZ4?~MUkqlYnOLTNgg#z=le}#{L_T$|qu-TPF=^&E z@>59{N{qW{;9^GRkd1V>aGWMtee@FsHh%>~o$#imZEE#tFp(OXha$1q+OBa=A zlbBm-#PMJoscF3{t?#-aJ$_2eZS;=Er`|@m;=C%vhlk;XJL~@ zBpg|o20HG`Q2Cc9l>H6C=uh5gG<__nZj%8Y?ks)prH*BG+i>b^oI2JJZEXfbZ<$c);Q$P z41!sy&QRt|F#7!rm~~?+S{MeQsi_x8owV>+fhP3KH-r2=YN+;nIr{%fhMutH5Z)_# zL$6L2_eb8C5aEkMDyG7rVPl~yQp{Q}l;FWO6ZDMQpR?*r>h)xcL>9S6;tM*dZ;kob8NN$;x+ zCnq==7;)be-5caF;p7PTv&j%29gu+o9coY-*AK0>MxaK)Oz^Jog=OES;wG*?>}WE< zT`>k2n>h$Zjl1erkYOT$#*LEcpLa@Q?iY}{+1sR{Bjagv>`Zc?GzmR6EP>Gtb3w)_ z7EcWc#%Eq}koDOUF1{Isa zFGxbEg+w;=nzT#YQ4jp)g%IM5pS*2)!J7O87;cqix}l{sh=#9F27GL<|a< z4I2L$;q+sA@J#dujxipA^IPK3b9ofJd>;Y2A7_a>oP7NAGY>rLvj8;<(ZA@mm<|8Ldapt{#kC))r7uZVsO2c6jHOzPK}gNMCd-<8?8M zuITF39C@{q*z8J_j2v+ExK5Ee4xhG*glDOPsnZ3z?VclCOqzk`DqV330Brog!NkoH zToyb5clt~O^J$yOfS$v&x~7&cFsLQCvmZufs6u_{0QgWZkE!8iRH=74Y4smWelL-w zv+S!$pTJhSXZ&KiFfxZENA9Otu?NW^!*k^N=@<0Bq&}d!w2pF4gYgY2K+ndFG-dBb z$t~4%qSxU-?YjP>u4{jjfga7|QN}a6R-=K`NIR*1%?29w>Iu==JebVPil!}~$LLe@`$6y^ zdOGU)ioThZL2#iflJ+tW;{0?j1k=lLi+S%StJlR(98F*La@ z!KH>u_(nw)-1@4)zAY*^bbvLSO&*HbJDt!}XBfoI846Fk?6BR&9^tDUOnK->mv)IA z4sYg=`x;Z|$baR;OF5C&hl!rz^g3dmdY{(qxI+|%TqQ+)J85gqar%C9Jz;H4@NsHWBskY-ZSAElmZc=xNME>D}W|u?$(7DPFzqn7rA6xt(=AAuy z_6&vZ@h(ui+X;2X>4C$A0l4(2HOh4ig#L?7uyltdv^x(25~G2)t^q1$yTP{xfNG9l z!t@%Nxn~Q}`&dg7b+*x2^0jClTn!sORm0Jz8&PG)cByBYhGg3zRY_ITh~|S`c}S<` zg5~NwI3|~erw*rsjDH%A+_Vh0Oj`zqS4Yv1Gpb}x&2z~;Z!_xU|AG1(Ya=_APLL+M zW}39cPuwT@;_v>RI3{pB$S04)aWyXRx6T#XnnvQ26CY`xAAP`U;V*LX%4@2;+Xe4e zdw{0=DEK5Dh1yFGN@_N1QO$-$^l#M|;(tva?~gTr-cN?GNSqiCImLoghh;) z4O##(1LLr-O#(Len-4P%eN_<;Do9kxz7;`2_)_o;fA_(%sVQuN`@Pk?nMvxe&Lsg^%Cz-_AM984!y)shqCWM3 zl0W&9zR$#aj^cIcMyHvQhVlVmoYq4t$Mwfv9R+AmdFkd{JY0Hy?AhiStF}sB|BOVF zV_^{YFcQ+mj)}W3D!pys~aQ~y^|re-WIibWw9z!6UtKM;E0bhYOYVCg^w1IpYIN%+uh(ue<$qTJR9pbdxMk9beJVRg9G^~@FXJ=xBQO5XJV%D{Pg#d zx`XY~8Lx_@P3tbXMahYKvoTYlP25M`I$?oNl68q?p(Zu4ilp1!jfvN=pH!o73)!-= zhYX+9O;h4;6P2#}^jgw8TAuQT-24}YYqTaqu3a!(+cq7Cwlzvxb6jZdhcc;8(=AEa z+zI%(MD(Js9S^f^kHOf6yOQkl6Dd8JPIn|G5U)@DVY9C`&iJj3dAmhFzJD$pdzp(P z)N^sPO%Ci=io&yMvGBQFobh%=VxjL;wBJ1n9_*Y9e{l*XJ$5J8^*f}!&9YP`UP5eC zb`i^Zp3Z3rrH|{1$dOrV$y(=1nrT}{x7q9VLf5qftd!+Yfu7irQH~q5;l5tp-N-bfH4bYvewy zB6HVN(0!MyXuzy9q;l9WJiEmPEPd6%chC?d=}gQH-lNG5Tj*o6J7h&f67I^524U?2 z_}VW4zYI$y(=zT#b6%BExwMreXk!7!4lRU{%>|HgBp=JC<$%xOl_;}62fsS!!27Bp z&>o?R!*?0tJySE-qVIwg6GlSna`E$zaK{1j(&^SbLeiEJvMP86y*QwRs46znGqW~R zzYkU9$#fHl6+1J`O5L#9VF1WzYvCQ@1j(xvf&VucO-GFt=dH2W`+XeF&hvzAv;WaA zGOtMBg26CcSwSqnL0X$4_A%VKO)|vpft=E4+*1|=t4t$c{?&1K!#(N51%@ zqDMgD4%~7riSHNP#BnxhY;kQmd+bolbk`>_U)II4B`f)3wT--b+e>zG(*QoKW*z@h zECTwoXb%7CK#%*onfBZ=^JV-YlU3~5uRg61SHsm>f9B^VHRlKQ`z*f8#?R!RGud?xg$9mItT5aD5C*cwfCHA#%-k!Q$sT_H4-& z_cD{)f~Q{$ll%CLyFcEW_qe)?_f>r+7=J#-C#Tg3vzM5$`i9eZ{p>65na5kcEUihn zzIqAoe6x|gU3`PZJ0csWRL9wD`YP0Vtro%^wfT={FLFA6EBOz1k*$3)kewwpeD};% zBFAnC|2O;!OMLu?ZGTzEZ5(i;CE&|A{=>hYLgmE$Y|s5C!h?q%+~AQ$d`04THe7X^ zFu6hO%BS?NkR=m zg{ax?EW_q|OY?$grtl6~JNOJ zlG8F}_ZMMsW7e##Dpf_y6hkfk%85cfhQZ`rNJDOivUcv7V&15^*G_s5P z?YW(D-}vDJKM5x@cJVgKLHu#voUamN*~Zg5-7Q1H`S&^z?1tXb)_K`c+-Hq7%;SY8 zTXbnLTNPEnPZj%*+b+6u332NDz^D6o3oE0Rkz0fKog08v{g}sQ&+BI2j<&gvuiw?8 z`+b_QwrT-OS!>AewFzXI(NEo{m)i*)E(6)ei@#f%N7o3uwU)Nj>rP@>ftqZxMIl?H zlgBa*hV%I=ZG@x~!})bDRti&3?QMBBco=(VuilDwKiKv9>-?~L*}THZ7UABU-<;0M z3H+N4`mF)JuUPNe75r4qc)|6wBJ01}pTFVNm(9Ox<}rC_qcBo)y|8)sPN6h*JMUAQ z-eP>}F5i3WlW^x)yfCWRyk&WuC$D1@z@Il+C|GZ;<6_Rrc_gc9awESuv#N)=tYh^8 z;qk(y!q-KiLeqBxo_f9zlt&e^r{l9)ZZ;<~!+%w5ibpdGebx`w?taQX`WA9(3LE*K zr#<)!CZF98evoOks?6tGeO5E>G83LHUdKNfuIu3fk?hfwRAzL{i|_hZ=RV+*EsG!a ziYu^R%YGKhvZE0z*~*693`7>i>sV+0=B^QJYekyyR?eSuNdF;rvB&ZeKc4VM;~iQ< zN1o!B>O?Zb=0f35+**GAh71;0aEg2UaUi$e?*-ocSIAm6{u2Jylry^fg>cyWJ6m>M zffvfp@$wCUtXpRwn=wzq4{&=byu03~RqNFuLG@<^XFa2s_fhEMq3(WKSS2~hyf+MM z>4f_ zW&yjur%5oFx`xkfIUzJUT;cj}lW@Y|EqvOS5PszS3U2x7QEXMN39A>ozs=I!-M`3e z=QGu1_|_W+tueAw_yPlcb~n$EwXIQO#VW1bSBw3u`D8a|xOWeeTNcf)=-BR_UD4Mg zGW-`;c=e$m5>|Q7B*I;)Oy(2&JMi;sW6><(o8W!#KW<=Wt-E#X6!$%iLGI)I?FB1i zbKc;?Las6TDto6}Ap~3hkiMR@kvG0NL?|;k&ioeU^V`nn^Q&_<@P57Z>~HWGCTH50 z|2$EKpJBY3t8}j9gUdo#_o|VsYoj|G(GtzubMv^bwPU%As808rlSD?b<`#~PJ|n!# zEfq3?#OG-L1i_?EU@M2t;@b@Fut1dy{K9!MTzW=^(7%rX%Qslhhu0kx25%k12E%<} zOk4tU{&h!SI^#VelFGQ>CL{T{djUe6N*sPEhYz$je(Bu!7=|Y~n6M9$r?lm`N7+=RO?bZ^kO*jx09d z824IIBs@8;!>7xg;LhH<&z&_v=08Q1O^VRqqEiEz@~HxL;ia_@HPwr+FqGq8rP(v) zTf-96&GCsxC6n0Z@Y~y_329@BZY6ml11Xz*3@gVQn{c9nq1 zt2sT(GUoNRor_&&%s%oKOk+)=kUwVw@7{X_QzpCzyNDX5q-e!l6scQD3ue1MY4fUPA?#6^IsZD|oB0ku&aaY030kVBxT77f z_)jtGgot}L*|||AtZIhX(_g9Edf=plf0ts!N1a?RJbG@!U4IwBAFPt{rIVL)W5xH| zHhLTTv;8gK5Ojf=P3MGfiBX(W-3D&?mI%hB%e2<&=CypT{wS13>~Zh9`-X4-?Z|dq z-NGC7HEBKPwvE4^yFpMJzDm$vtlE+|CYzQe$1!(z4>t6e2frySOelORZ7G|wPT2SC zf^d5Kkk&VKd)TGr?(EXu0{0&!3Bsq%^H|oi*KC+)Bb&755dX+>jL2&(XQek=g+W_) zbH@^jglDD=y#9(7mTb9OxElDGZ~75}kx3%cLZ*gimi<}XCoMjH5960FPvl42j^WMi z)wn^CVQf&HAgIm{;iq}%v86{RvB7;xTIMg5^RS-eAsF^4<`-_TXBV{$*hm=-k3z+I ze$lr5!qg*YIP*D|**kfQ7SvnMiyb2D_tdpq@#1~_^Ci{XwH6gt-(e}t`E---8`!{E zv|Zv)-h06un??!ug%f=8lX~8~U6HRIx0GLFzKc!0G>G4N{25z#{TNp_xr7ZYoxvtH zU1`~_U@E*n=+B+e?BF_P?qtDUGR&vBPP4vxaqR zm)3Z;Ub|S>dC{IkoyAzG-uf41c?D;9q0-4LvQ&1Id>l-cR} z1Kh&SJf`r;%fn)%GW)yoAal8t+@dyj3u~8gVzWM-=BJp?5?NIZe1?Zri*BJc-!nCi zzi*r?Na~Tx_V?o=Qjnbr-^%><+Xzwoc%f>OKg)U<#D2Q(=U11+^C@F>h4>?K%xj1i ze_5+Zu$e5wK6T#aOS*UR%lqvTercKW%XW(ASE8Ko_z@Gl<~p|6Yst4ZhO`T+OTyU% zjc;7x;~4Ia`!2So{h$zW#EmaktKo~DdkJ4Z9B}^=X2a&Zw_`7=EIbzaY++zBU3|W0 zG3l?$mi3>4*~_ubOsS$!2<+3zj!kNJUv{^ZFI_J(t@Ts+7t><7?-R}sq1;9@ebz@u(?jgd;>#lo_`xm=H40RO$TQV3ID%@3N=#;TXPu=Kk&d`e_8Gv4}#3+uGwZ{HZpoyDKR zikN#$A;XnTj?;8sIO!a>KH(P60yBi2-=#vyPZKu6Xf-c)@EBvCE7^|qce%@fcZAJL zXSHNbC~qm)@5D9#lV_V1_wm<@tyqG!HedVuAXAUf;a7V1XWv&e3a*pa@gsLW6E1Z| zvc2l>*+^X#ULo@YlY{Hr`i|8++vLpGM||LqO|s?t+J^FDjcU2nbR+ik$QAhOM33nIP$Bi|?YwwEWuS=l(qH3dYIElk~n+tDSaxw+U;j#mKUzsA-kzDSsGR%^nBYToh55K{U&<3_?!Depef-}M& z(?=}|r!3h3hnc7rzG zt2)9%X1{N1U`&|sh2Cd|>a&;|mhs7DdYqi5q5BjA6_2c=>inJoyV+kuO%Lakb-Z=A z0$-eQkQ?aZ$KUKU77kjzWJfc~SVobDaB;}}mMJf5Sy{aSAAd|k2)}r%rOBg|KOZ#M zV|=Yu>w4#J+y|=3#`bm!UX`c)GoN zck&0W?=&M$R%;SFyJwfMxZaqhYVLBs^e>0~Qy9SS>30gft`1_qeb#Z4EL53i#40|- z=?*{axMu6<37P!hn+d%0@oFwoC)It9)mm<0T>{ISr_Fa=QsZCyr?PQNika!B5`KU| zrhBF4RzBiaU+(fH@f|GI^jNG=#uu(H=U4nX(Bf*X?s2L28=nza!?hnd$_==A(fvu& zJpPxuc>jKz##d{Haf)}c`PY>@`I&7_eAg|#)-4AgORO^L3um0;B%JSCDHLwF z&du1T>CqiGpId*|j7fTC@ScadS{kym`Do{-?mvf(Zuy?)$5uUd<*jOUT4MKoV43y7 zyvCK$eBSI{u6O+lcHxVGhm%mm#vM^*>p$u9(MLN31DPl8V-4bkVehkeb*l^f0;{F` zKli8Z@h@)i%bnxcuG9Va&nt4+4!?tJu%r)vwBb$5_+|yxKVb*8Qboi1V>-b*o1QUGo+4T?G*_QU2WazUI%k;u6GxV4cAvkrH`&&vVZ6Iq%Q;V7r0<#PJevcML3UBMvzu)NREX&Qg(UXgR|bdwi{kjkd5BZC?b# z@3LTBO#oO3uEOo!baJIb7K`#@S%cE2?2VjKs1TnGK`pYd^yoxhV%tMjgrCj$O@2-{ z1uw4Es8xmPy?1Eq;&0@&Z4PL8_@FVXgsA%v`(1)T?dxLRaF7Bf)%?Pg>PZl;+6qqg z(@^5o5S72qz_Y2dF(@sOWHrn1a?|thi^gkqwYdp#AtiLYuma9Wp-m~7Yc64 z!=o8Ns4jdz-krQ2D$ccGM`<71wXhx%$63L`$M*^E{8ZZEAw){&BJv`ng%k|( zF=FX;NPl*Rl#TS$z^O;@w5KpDeEcmmJM9Er2Ln(gJRGuT1YnY80_kLoY3S@K^5EKc zC?3^?Plrxo`y_Sp;^8?wJSZzz=6nI4=NqDgOdhGXAkoYAu3VWtG#PfI#c`}+zLjIG2UrB*PfA>x-kr~JHczBlfK1+n#+vef=xOB+e zD-F%=rnvf)KX#mdM2{X>h$lCH2GgI;7&nwdEw0ZN=HDm^7MrcYt9P!`CpnMk%?P2+ zdMy|7jnA+rzM0^MHhEnB=I$#=jdcxLz@XOuZHh zJ2~N)YG#T1oknoJmIvce+`u+mwF9Z`sl=^&8`Rbm(N>3R_ce$WL{AYGBJ{gP-53&2At>KH-ahzJvOp^0&($eI+xWupu?!8Nb zCyTDqsG~#hEMKVUs2Rcf$Er}jwh59BI>Kn>8dTSo5Z)tOXom7b61F0UC`;F4%2F}< zdGQGxEp33N3G&d~C5fLUF5%MGW7P1Aq##Mx5g)i|fUWf}>LHU-`);{4r17SM>Ci^B z*ma)k<9w8il-FQOsxcfr_JO`Ok0j$~-)G!>M(Gl5U1-)Cf$HaP(cO9hW~l|@*tWZL z2m2DYI4p&w=1=gkyr^KT!j9| zs}iTO+B$fnAe}6}yACEi|AK#aZK5B(+QF2A5@hCZ0KI=s5{e-Vm3(?}{rqq|v|kG( z_nybL@w1_!?F!jNHNfFqBMy%(!j8p?yp#o1c*dK6lTjd=pKm75g_*fB4-|P8$xlGJ zI-H!glcjcd+-d*RJ>=KV48|yTD==QVbRaX4@*kSAQ;4g-5>E3^r#*pRXx^7X=!tv|8i#{P{E=d!lsE_OhZ|wiP!U@vG6O}v-$Jj< zVVYr~gtMMMg^u7(?r8S}@Gq=D&jSN=+P{2exY7vX4GaV~efBaDGpcCZi@$U*HW8+d zek0}Mr3KBs2Z^rCKc+yK&wT0`izz{w)b>_A|MQYdsL5BQy{{LH%OdTV( zKN_j|k2z#u{8zTcm%!%_DY&zIKAf7}OLyNaq^4(T$YL8^yl;7tdKj(6+J(YA>v`3< zZ_aTTHxyo5ep8gEB%e)2bC=P@i3zYRc9edUQ6tt-fUB-1L;Cs#m^D>IP%xZC zex8`jtJ|U{XbSg-u!R}WF4SK%EWS~T%4=w1`ICk(GAEJYd_3#07?Pd`($<_v=-eMm z-p|N^PPwHptt1KpCyENDTe0{>%neVfHRCC7dtq*39Z0m=&_|LX*tdKFGq>qB=u1`7 z3eO05^&$i0BA-+5~ z@#pAl%|0~koI>6WmoXw?PLQbq?9>r2x>C?gw1fPxt@sU|5K~5()BuwEP(V8NHDKKt z6W+}$BKTy%7aHuY%6oQlKi*uDOqZ^=W?CP$WACALgk%4YhR&J=jcZ<0hiCsm%GCw9 zTw*7^zW)ci_gNmDg1$a99(VjZ z##p4bQxDz&oG+=zoVxG`hUNC48sCJfluN@3yHFgs`jOo+c8l0_O##oug}msurF5V> zjO0Y0Cg{PvAY_oU6?Z$KIN{xlMe)3ax6)wQdJ0kIlq8k`q6-UQ{Fye4d z3%*5%&|hB6is_gzgR()p z4lhp(#F6o{AR@8=lB8u>-F4;U#4QgRw$dEVc^^mF(yzqqmLGlXwGmS{`w_9$TjYx{ z+wr!B7L78VO~1%b6*vaPz$Q&^tgs3pL#sN-x5p*aV*D>^-gXQYbRpQ+oW-uwRrqID zKe7f7ptE~9%qvO8yQh!h9{n^F3y;M+6CFYQ#a}XVArw1HO+cqd3~c`jHyCj~n4(;#lGEDAPglCwD0*m;G#JXOOgsuDn^ollQ zJrSo~X>p+96@rhY$}!Ag9#(RzP|I15SwCbVxT$*`j@OLAsyA|io~tdyR^by7Zyp7^ zx#?&>E*2Hb-w_$126n}Hh`L;Bpvv~rRhYV789aTP$);V$!DzQNwe-e#iVDh2v^Rx5X}sy*6-b)lcfOKOx? z0;kLDAnBQmVCEHVy2te;X|nUeIGt9EcMHet-<43q?ZB?%GE`t~P2a~S720R0eeK4t?XW(N6p&S(x@p6?GcZI9L+>*SRf|YU6Brt zEpLIIr@u$PN{+}>o zhBr1EnBfn?uRs9G}*hRj5GXPr;r7V?wn5&lNPzxVK};~dWPolP!%crX0c z5dKDAV69_k&_gf&k=tu$z>seM=2)|I!^4a8Qfdsu=G%d8{|Ow4j>1t#5y1*!!l(aV z0rl6+gp2bei1~vMa1EBht;d8>C$<^bvho7Cd3Y_3To6T#uXFLopC-`QnZWM0Sqlvr z^KnB%BAp;6BAEQNl9ZQb<6Oo7Ufyv>+hwO92Ahb?2IEe8oU`e*+Btg75nkZd}-c=2fJV(-~@5lk&pk3){yKYTVZ2vIbLB# zsJE3q8K_;2g*(nep@b`D-X35xUIfug8nUz?@hVt`Kc-tG)ff@kS4{oKcl26Y8Ts;f z0Y3cC3LYnz(4CwWC@s2@F&0?>c3q7$vFKla4$j8-hq6?0EQ{(MyM{ME50Nil4e0Y>QCvT~1rG9f!WmpTE>>BChrhMa z^l5KE&gL?at4|@y67PVi{z}7S3EbS(j)7~6*dB%Jn5?ahIprtmqZkpi6ZT{x+|xvB z{t_B_AQ`?I$Do$ja`;>*PQRqZq3`}>c(`y0gr5@;g#4Tfn?mX#(1!s(jYHI__bYko zy$dIy#*>NkSHK!HmrudM zwq#UWoC}ldC1BIp8oK*!K9ja)FN{S;L!;3IX6d#6z;WsVveBRnFY2FTHA+~LTpSLw zm))w}zV{R?4o)M-cu%>Zg#?t;9>Ux&!_0}l^KePw6hW~Fpu;)}mUpicz0z2=K-GxJ zOpyj-$*ZJHESHG4Cc+Z;?{K#Y5MYl6}*ULujA~@7b0D`eNP{_?4cG4M&m8)~hrA zx9vC>Jd}sx%mc)tV2FG^ngxsGff^agF)fe91oE?1(z9ts+DYl0`^Zm^L`*>LRhE^^?lELbTvQ>`d@Y;slS z?d)8KS+iyFn_?%sXJ`dB9XkPR_C_>L$;Zx}#bi=NDtvUGN0e{4pkv-j@aMdwzYn~{ zg6|)xfp#41KbA)wg#3t$K?2y7UWDhb|FX{H7W>>F3zuGfMP8}}qpr_+5arItGG7Or zJFW)&mB&HfyR+c=!=G;4Jx_4%oF7J|hk${Fg5bo8nea*EED_nzOEn_hV9Aryv_r{= zHYa|glcq)!mzCm>mzqTP&=pv!b_Q~n+n~SVS<)0E%Im2W!`@@Ybo04bp+|VniN0hN`})OO^3^MqT0P9c%B*(OJ@k;Ia{oh{ z?YV5eP7oZ;*TBk1q+6FRg|8QGlYzlisD7!E|L-3G%gn{RMITRMx|tUi{VSj!g&G+@ z=>^%YcAjz*Y|yCoA3pY250Ax)*qZ+kobpsqvBHNm{bFcvb~x$IeZ>`Ugy+rk7q`;N z1SN&{v1iXrjQ!7=F#5|t`+Eyc)Y2!%BiGpTA{>`|gzNJM+dZ=~$We8QT zB#*Zrgzb+W!*#b@vUY@kh+QQ;HM0W`F4>0d5JwLhEX8ZfTIs2LU9@_xg@4={sSST3 z#AdnEfcdt-o;yM|Yf9iBTb4K*rcuRB9sZp(chl! zROMP2xiW4J{_6XWKC2ML3WYIppjVBK&VB?5@w+hTYYGnJt)AD%6lU;BL3dR+8s2r4tY@rlE3>OYQ;&*rBRXLl)>ta1~67>2^CSqDH& zLJ{_%0;-zG5Y5mMxVF9&RW5hZ@X8JlpFE1zyI(=ul~yvC>PNz^>cG6daM)(P3EZ`X z_0?6FeVzFMP2K&ey>~QPARj|F9antzF=}bU&VQ z6y??J--!H(eeh~QJTbX3T~N5wj>LSnqo)Er;OJ4ro|0prJ57&wjE0cb-Xe&zu54)SCl_75*IG@!LWu4Aih6>p42nJrEh=!Tv@pRuf zd6;u#0_3$%$EfYfV6vqMjt7O{VmT2Uoa(}a%o~r~4G!?J+zy_OA0c&{u3_Nwt5{*! zNmU>NRVPdWu-c0Aj;r$KoZvvn-2iBhipQc03(@iKZ*GvEJNNHD3uu%qrpZQ1c-0^k z^{1!NB9mO^kad-buk~ekxm$*J*;dH8jXOajQo^w7;~U^)W#E_L56}`Tg|^GT;i*%H z@ny|>%1kuGtCOQ~*Mpn5ex@l`FL)QW-FQu`{yOqgAMJpD+-IntFcUi}LUDUQH|^x> z@Y@lW#DFCo+>@e!gdcu;_99B%ok|tRB zQMWo#o_c>Y-h5@mWFJ&TR^bvgPB5dB6^R+(=z@@>Q{!waI=&eL60}?8Y_t zE6I|cu*ks0$NBK+z6|C^Ud6jxYPr1WLYp2pkS5P=qcLkV&?_{Iyx*Qn=H9tTTCP>n z*ahX79jc5QcSVt+NN-&C_XhQxr$g8A)2RCa6H?}J0aA*C(7as%7aWzvzL)7RPu!HK zEb#}`Nyp&$*X1}W{SbW_3rP8U#<(wiFB*xhV`V2~fK2LHNNe9ir)Q>sMwPgrM3pP# zPx5H+%Kg|T6G;<2RZ!~BbXwXid@q_ipk7l9XkV4Yng4AQsD2T{u8GxDJ3}01?*Bvh zH?HBKfGE6{I7&ZEO2b+DU)fOoVydAc2M-^VA%QccifY_AZ5woqYDuKSeaZFaczG{cyl}9#QkXgN?KF$li@>>EMU~#@O)j@sjO>!<6hj8JO5A8a3{)}IzTZ76zBV0q8QN;fj!D1zPcZ(c8 z(^rMc2`nsWcZCJrk~k$&crRNe;ODSX0i&p_n}?Pgi*CL&3hKG=MWqy=(r_ z$mlbqSoar^^s%9Lw~NxJ>kfdmc@wyfD`&ifGa0A+XrfvjfPW_fe7`iEtf>vc%?_){ z2}d14wB8sFM?NNtBO<}AXdG`%-z&I#@j3nHCx#OeS3uB3H_-9Q!k0T{V5*l1d=qkR zCnEJ=d--9AlNkqz`e|H)B_GjayBa$$bUgUV7(m^-6jWNao7F2WgwKx~@IqcZw50aY z%*TP~wt548{o{p{wd_0ThM4%8pGLvHn?A#?W??9DM@ zrcL=nzFi5yhC+GNE!Kc#P7=bqDS{3ur!$V{=3uk7IDXsGf=Ydpd9U;pQRQG36t-W% zlVN3S&#`V;^z3}YZ<8Ne13=H!lfE9k%C*Va#fThSNM-!e zNK<1ZZWO#E7ZTM#`{gcj+B^%Q7QUe!*E>kiys41-@G>P;Ux0U66S0}U?XC=wvdMZOvmA}!dmYWGJ-9h^I`GE*HBV=5;rX`q(>|U z;C_7?+#ZO4`Rg}uzXg8>%hzdOwQM77?8qefs0HztP7x;&RXA#H0X^$ybR5Y#?J$%m4&q#*H&V>s~bpcz6?K}ABVWI5b!SeM;fgr z@&0@|3D5Kq%OuOeVUi?osd*ggJ_=y_mi5=#`c5Nvo-aj_kl!S$r-Quyss!1q^2zF~ zOy=}{73dwVfPhvmgeQ8zJRt`p*Vs>+euSe`RRl^Lxn^Rimx#F!BUv9|jn}txV2^JI zLf_~-!rkUVtowr4@L6fZ(M+2f9<3$*t4+cEiYJZ@rNL*VOIW5e6_m~2K<)jTs5@8% zBER<%(KD}WE2;op_yy!euR2Qh6v4TTnmDuhCB75(BqP2}#H!C1IPQ9QOJ5TG1Hgz;ba~n=Dt+4uqpjqLrF9>bkWV5~P{AP&KE31Ta-dYFz{o1DXE0)0J z-BqyujS4B6*@Y8zONgnAklQs`3ZENZkrGu6^cht~d9_+#$PM@@n+L<39voh`foyd? zMAe1$@8;SIxO>)<+8Uv@)iK^2GA9_ae=@VndpTr&Hn5#< zqQUy82gp)uvZbJa9n5k;P2X_Zr9T&xSJ$A@f*z)&je+T=D=;W54Yi(~VnaTc*rCV{G69!p~_ku z+~&{7Ehr_Y4wXQIWiX@#ykI=f1w%`A3wz7V3djHKhM-9nl;_A2M_VbLpf`}NUZRD= zQ6V_6&=XcA3bmVYg%CNg1mo4_!evQja#G-W3vBHeR)5bKJ78qTE;?b_&s?0 zc!MD5wj(jDosZ9T<}r*`8NIpv1`&N91`iJg!a|$Bi+UIH8`6P0;bP9;7OJZ);8Q~e-1?DE>Y4~+r4CL;^xR+DFlwq0A zt;+B+xRu5qm*$mcx}(&m3ihpYJ=|ONgti)cz)T_2BXzS1?qtV;#w|mPcCSP=Z6&fu z+KaXXMZoRKRJ`Bwon#*Rhtp&t2vu#xW#Vq&BK#X%^JD4KA3@ZwA(-rukR#SN3-D_5 zVRBcvZT8mXi`4K_C$1f*iHSoFw0ygw;K%Q3=C_;^@PG9A>K1b3n*+G)oQ6-& zRj@5PpTf?D1jbwN8mBCNZgTLW1%ymJORM9iV)P<&A^n+59=SbbqD;%kgdbk8wou4v zHMda>)jL$T-VIvAr7(!OMGSr_fyvLuFp^USU~?H-sy@(cKOSy;V?ZY?N`s9#*XdlP zFkIkigJYJdpeyty)V|yTpHzFu))!55ie4g&e%D9i3;iVI#4;+EoD6+uY{7l>5_jgT zMs(d&4Z5%D&^0*=T|bV%(YI0Lu$%^-YC?SS#tOHFNaK@Di}2)BS5z-Cr?)<)VB(?x zC`-0Lqt>@{SlGAa=|-_DTs7c>K?O>R|04#W{+M&{Hje)))RxXl<7e3$T+|ZB2}wNI ztkDUF6yo_^{+@7OcP6tTYbQqUn~C=G_mh>K_rQ91Cv`j>LtV`00398sJ2yC^`XoCj zs(yzYlWdIhb%uwhgJ8dn1uilddco$e#5u1H(zqkBIPL9gBC0-HsAyV|GL?(e_nDGF zU|fzLdM}{t%lWWr;3$g{k1)SS2PQ@-_RAl-8gxV1M}~bHn@z}5|nT8#}xlt zG^p_eM6NZ$TBk7KcH30AonMBzO=pN!i8+WXDng);UB6;B1&oTPf|*Mdya{&$d+q&1 zv10&ZF88u??uX){M>16GUJD*}L8@cs0pI(Bajw4vuX}Yb&SW3pilF)EyI}%M*`-N6 z+TNq7P(zzPB?1M`EcNA|$E626fLn5mZWihn&(+1C=|mfAK3)wDKT2SKjoimcc|x}R z!#TXUN1kc>s|btsc;lXDPAGZj2D+!pqv6>kBF+4v(>C>x8PZ9@xk;YT8+Z>lh9*G9 z?$0#EFcpf7a#3|BOFGJ4lfnm6V75#n4ZYqcoGIqwe|<{u%cTSrRrTqFX_Ra@T?|t{ zY@x|3X2I@q4W3Vy1sFTe)Cg`qfBQiH6Ag}Z;{R7%qA!?3?n+n0mjtfokdbsbMu%7iEMkd0VT+dxD_@~PX z`RXCQTuL!a(0)k^^s=GFz5~>3O>okp8T71?G6=dP;6wHisEXSFhK3$3m*mp3}F2L)^QZahtX%Pmos*>lbW*v=#Nh`_&6-qn7xWT%nWJw6We^ zcgPf%E-G!@MsJN@Pa_gq*tn?qV6SUMJl8f0^>!Z|ck4ITNBSv^>5v3nj~vhs8m5m5 zrop7Ia=;1Ez)vMku*^G#S;?P7wH%H@Slk6#5Hvy$%Xwq-UqUVsLU{ksl#@i7rnML52TQubC69d_Z>G-#H3m)m)Po=UH$kc%rVk}@G zcw7%VsY;LMyTJ`3epJvL-x9R@Bo1#>Euiel8N92%5qH>L7xwnw&`G=y3w4je+-r(- z^{*I6TjGmvZq7nz`wCwa_;lCSJ~F}k7rF9XieJ+D0&eVHf%M2566QZd^rUl`s)Tt^ z_%j|O2alt@*mg{vC`P`yOku-D!tnd~T*Q7;xCkG~)cvb)%-}FtWEldoi8}DnOsGdp z6uvdxlySUt8dwa=@uclvQ(2Y|jVm26F(DtPMKkCfBf<4^wkI+zKA2i5E=YQxM>jEQ z`17|D9C-2ovLj@1$u?!8KQ0rm2E8U-!=dz^(#${Y#!@|95!T9i35KgojpecM| zO+4X*;#Ots&s1R=+AkIi#G`?`K8g4=YOtT)3B8`b3n0&N2UXgC7Gok_QZDL+jw$fE3`-Br`IP^W<7Ac~OSPHxOmS`QFP3K&@!50*W(F1RDz$`um zUwpd&3NK#ZzgJ4I*Q*J(INhU#TQ?A?KqRKwDm>FOsqBqoj5l1vbkSK`b2f$T<9ORbr)?;Td^NLM@%7S0JV{1 z)&d(B^nw1ktK?G*g+~?V$$|w}Nanq4da*GaoPx)p&yNl8PAZviDP%ZYY_jMKS8*sa zNrFV-_n2$&p8Q*M5!d-g!nS$uNrdzRq8E}u7prK|CH-@uK3N7rY!~D9iz~75(n4NN z+C+Gucp2D_O0f9IFPhwPl`0!>c;esfXu!ogaAeywD7&`W!)+rB-blu>ck_9}=Mstb?dfFZ z^L<1{YZ97HY(W*%LUh_Soz88%fQLnSG_3FxIGd-zq~IlFd-Q6U``rzGGoOg`rZ)0y zkrH?vn26uh9NB4~mNL`UPv;2A&;v$djY6 z#WsVei#Cy)dOT2S6~zNOx6t?SL-?fZfl5Cf;+o27xIpbQ$ZND?nr4fEo0?H0sq)5~DFco=Rd902t}A^V(}MXs)`LPghj9Fvj(BD;~a8>V5z=IP)bFo*2T zsDtZc5!imVo+_Cx;Q635%&WY{?Edr^7(=i!IFCM9w~#ma(@Hp#KO2T!fLsdN z1Vd4>LSx7l94GX06m8s0x+CXP%eN-P=9U33b z2OWyzkow)ably~1qUDkWLsz?*?}|w@rKFG(-LU{GJ7i%}TsNb4#SOPaRnq_X{_yXc zD7L5SF^rqB6;H5x#tnWzA zJi9{L)Be`(aF2oOYeJwQJA%q7Z)Y;*2-!;KVv=2Jg}3)vP>;IFxL|fIeiLZH{YRNX z|ITe_c#@Ang@C@-)S+&rE7UAmf+jBafw6QXw>H+13!eM1v)u$*t4)}L)6$@!oFllT z*n`qNGhx!}J(R0`43f&d5l)_GT28;FC+bv44m6V(%Qz@sAtulm`@$UZlooi*+zOFA zKRhy)0L!W!$WMa@G)>Np>|H)d@Ur zV5U<6T()^mojHL_SxFGAxSj^b{e}GFloB*rWsi^EHQ=*3k#KcVHZ%6(4d=Jzc-F$( z2RD{ptW|9PN6OOXqtiGMv?v{i$G)ASQD6eGI2Dfn;zP!!H_ZAYvT!#33%Kt0=8#bV zHK&0I)40o8;)ZL(S<+sAuh`VIpA16$`(+rM@zHmS1A1rIDh0%smxUp9ib0RO0 zyk>l|JafTP&|G_|kF4ymT8!3~I2JM0&deUwU3iFWgXHO~~sux9)8lW;K}ExX7j*O@@2_!4G%_G2jY>>+K3 zAAvppD%stYiA#^f;N|V#*?niS!8zm@V#N(S8efA3&W5xx&ISX5*1$-a8r|umjtPru zz%y33QBOvc_qBrsjj+AwpDiYkInh8ox5a?U*$_$#)J&u*7tPm;xsTI^I&G~N1@M*@OB?v4lNp2;6QX7nRmJY zlx=6Q+ZUA4_^t_{BEo~cQ3>$J`x20 z+FXHg{}lZFy_TsPUQMcx+fZ|NEs$TaAKU%%u*IW`F*su8m^g!`qYLNe!9#ah?E2aqNUArXkp#3En z>Dh&I&+H{(JCm75StXDpnZhKAwcwxsl;KZ)4y~x0h#Gss$=U}~QS@REqm_IR)^z`b z&&^WMHLIWdZ-{Od;=H4oNFc!^`7S1-mpmL9KO&n9sgU z7e`HmfXi8!vEx2;D$S*pXZB6Cj|=yP+~jBUpnFm{0` zE3@$)OxQ2%O-?y84}YaI39k}pdEqUXyJ!wvytIM!OqzrK{Xbc?FWExonZ*@yGho-# zUR2bxfNyiOa7TwI@7A|Icp#Mkrn@8Q-Yfw#5?V+3-@?fPPd!NN_rj+WM@dE1Lo(rF zCu8p7Np!Oopha6I=o_1oLy3HLN#JvMqL7Xswq2`Dc$*L72ZQlswHsblx=&Wg7Gcr; zZten09oU!BLMlh~;kb?uQFK2GtDJ1{%!f2Mc=QVCQb~oe@kMmr4L3-(eGEiVA5U9_ z(#R%p*u2(=-fh#N9};8*Z&be!FYkEayKb`Ju$vh~`GwHYNii_*$ferwpS_TBMGHdJ z4q*blL+ev6Lq=aRb&Jdgk5{+BWo|z9kDOq+t#hI5xo{WFo?;AZogf&>GRNv#3ntig z2I!wngl$63x?1K8%{;Hq?Qb!Mn%Pd+slNwrkDVef#l;1l5vRDiLhqP=>OG8nGsxJ$ z{8~-^Jg781h<&?{L*K_YXwO>(6>tEzW}o3s{z3_V)j7PfOcd^%%|MQeHQqal23yGPT1gv4wPDQ;dDyvZ885b`5vzrBJQ0m%`qpy< z?X9N(yE6i37+w+P<9(omX9IA(R4`1^JBMx_S(quc8S|bS(J#6Jrq?VOyu0G*rv^!A zSiO`UO1zAb#^Qp|g%rNuO2+T05~SJr05aPSLbIAZY0}n&f);65Z!ibKU@v6e%Y>EE zk@S<(4&1&w1bo|fU_O6`(98FOEEM|Q!rqsFalQ)bZ1+LF*JPgD&8tjDOCO#%-3ZpJ z1=OrJ7YwKOvaH($aJ$@&728jdtGm8&b!S+S{?~>W^!h%{XqW-bm$U^X^ORUSsZZ3` z%KE*Id1 zKSiV^_AA+bSxK;AT|N}|rPHqMtHF5DQ5*^I$8M22A>%KcYi9`H-Ftc3y!0eFDsLO~3S&5C@Z6_oIuFibM(KUkc@(#g_#tYyYsRZrR404}1 z;Oq=b`a3!u1IE|k@^hjBlLy_x-2+$H)kk^Ehw-P;d?*?QE0ZBLVk*5;*-Zc1&OrBd z{`f^D7Wa1uH8-~&(oiv%Y~lrjiI)O4jD8@Of6Sqlkx_WsK!bNbsf4k)tImU&LHO(Z z6-f|r-UrIy>kVQoBjolo3rS9qzYGb2zBs_0l08pI=$EA0SBvxXx$Qm zu^%F9v;QmNJ_{qH?`#4Fed@>h$Yr?My`LJNA14Um&t*mS>yfU+P>2o?$9qHzlGljv za*yA_i}^EoMUBhZjGbX*_{}_0{bN2k>=cPNdRoxIMp6))G!tTu%_cJ64ncdxZn}3g zotR88!>xZq(YtUmh6~SIRfo_!Y^}s>`m2TA@d7aaS_G%MFAzzg2A9(j4}wqP#P+Hy zKJJwfd}(MPVOqK5*KKLgWQDs9=H;R6`e!CFOGn^?T{!;x`4XR%ZZ9dM$B7#2mr4XOxunEymA>N*PJ7 z6c82qaz%ej!>GnHY;1{w_;vD7^D`TK1O36zO9Y}Kqy;B#T!)vp3B1l6#~Zh_h(zzZ z!rnA*U`m~5Q}?K|RA119vgLsgbhMFj*LO4elILN~-XNM$+#sB?G=Za(8a)=W94;9? zp?`#o;rt+Dsyi#0IC+-CSK*RuwyXrsKaa=W)3tQrKTC)g{SMx{5@2LPEQy;COhPke z2s#PUVXsLL6u%E+FNuIs#u*HW7lk_I2vF?QhS9xu;ppBE5Oey4(ARm1M#L@WmXsIqPpwD&s@YJhzx<@-4&7Ugr zH$1wEjZ6X%6b0;dADDO@%xnzBeA3sdEDU#tF@9=Q`h_ZCeVQIzMkN2)~egy6{ z+0j&PJo4xaC+E;D`L&pB{aocnak557lf0>A(L8WeyvmEIa@A)5|YS!%kt;n_d_l>V$vXqnFP-zlqW5!Ka zSX7(&;;TukW+KPFYg%p6lAZh_qeq-oC)N3ygZpZ4Pk+Vfow9=8rgf=C$uHBy{`oj= z_rY*}(T{Tei*Zl*@9QK@bYdG#Y+R1>Ns(O*&(4!09aF-&z1g)!yvC^ZX~{FrSc-`8 z{k%P#vB?J9eCNIz*Ey3pW~Jk~1?Qe{Hf;*#^gF&X@tWqtXEc8DH+|m5fx`lRdGQ5) zxUDVslnd0h-mT}$=Pu%GiaN}(=6vJS{MpQD*0ih%RJ7!acqee?*y`}JuLyUT>pIn@ z-j3%SRew<K6mKpXB{Gb!{OgId``6_weuWCFq-HurM2ELUanD1YJXGu3MlsT7Py*9D9m&M6g<;0b3i{=-42l5*i zgc)xd*v0*|;ev7X_*VXbkx;(l6BBNg#yw6uF*f$AU1hTKwlkOgpw1}|60SI|f5jKO zx3yN}+BweS;GH!RHN`dQ!kpHO@u7VFi;qSU7v)U?>|D8_21Weye=AJBDmm9&HIw9K zu~Gc8+fVsJgLWouGy6F$>c{wf^$|5@hNXNp19NU}-v$2Z?SA~)9X!swy=ORw>>u*E z(|(vVyzMfX*|@ZJca21C+mG^^?a!9;M|WT4bodX~wEM5EvAc1c^CsXoUq{cTc7gq7 zPScO~ob}fZb39i`asFHuHL-|LSQc3Gm-8w1yfL3s#A%-p!1sA;S3B@Rsy1=uKhBJ2 zinT$9HTaFAoA}1#f(@+q#q$UBXL4}1F<*1UrN&qEK41CgE)$=H-X_TwHYToWKdLSq zyT&&%U0q{(`i#j>&AGLX*B^6gzDRSkcIoktC@JvgUG?Ft@fze<33KW~mM-Ck{+Y-< zu;nA?VM_?#L~@j+NC(o?GIu6ls6OjGZgvNVuM*V zxiu#_l1y-o*lHb)&TnB}*K1PaPZ)+%@SM`OA< z=A(Z&A-am(x;10`|2aDEc&fiYj@x_7-b#^-5cj-~GMYj|OH*hNX_AIAQ$!L?ZAl>| zanJiCiK0|wG&EJxki@qs`Q69w@B6s-KJMppKj)nH>-BoSpKF3@Nwkjv{nGM+yt(;X z&|xzhj>jGa@vF@+@v1A&_1p>B$8N*@HR*V}J_#hoCgIfTMC`vIgZ=HcbOLO_q0`oZ z)SGg)^3(O&4{myhG=$X92|NZjT3IFp~^>R;qmPr5V3DL79?DPR^w*O z(4B)*XaNM;X>gzUuF%8GsQ*X}{&t!)ofsAZHy8Aw!<%t5L8$;YECxnDDpTc~MOdY4 zLzlYMkvdsLkm)%MYI|S8WT^+NqW2|C4e7_Sm9{kDXCw4~euZyijOm9FUVN^r4Y%5_ zM`o?kLi^S4!0^5*)~!s!`HSqS=9@a)e^gn-(z*)IEWG)m%dK$Qe>EAAJC&v>)#(ep!J;Gy}$x!H<8khG+d1s+aXXVS6lCs$az!2C?K}oguBz zy#;~IdF*~xjG)#hPZ%*r5$hgaz^3|npeGvy)swR!XulrUxm1K*%UtQp)vENFPB_~> za1_#94vmKgkO ziG-5@d(eBkJ!CGJ$`sR1h&VShF)hHEHG6-7e(c1%CGjNj-aD)f9;z0sI%O z%RkHXu}9*+LFZTm`4Qy-p=)QutUW5Y?yor=CfX~uh+Fez8}H*z;VaA^)Z!NJL_B9u zrj?ps@!=uzJqg&@kMM)HX|&}mfW!8;x-mAbN?i2UYQE#vSzTkMknD*LKnL_Y61T{Jc_QiR_9@Oyk+Yn!mkG^mSiBp8> z`Mz{{;XE4uvK^G1DKGLHiFLC_(~#fU5UnZ8$Ek0lvs<<4qjOm}{7esf0rcr3@%cX8Z5RKz8G=Y^Ud9O!+| zbJ(?bHqAZmLO&ZA@vH3~oG4DFEu#0m^GHS8q$0dRU8(rbF?6(}G*^+Tz?~*hRI+Od z&9r=hjkoQ2&i$RZO=32@a4CQh1*!P)r8}ST>JxUomI5vRD7gG5oPPc@oX%OQ!P|^w zc%thv8qn`U{jyB>@?Irst~!S%`qjYdb%xx3@&L;nxs19*NKo6ypHRatCFO%h6VFSt7h@L+tT6exH!CX{tymy1moF->5#*n zxMrC#pJP;kC*SxIhxms$K2DSWO|s;}@A<&Nd%fgpjy?CP)#ED*`w2@4p(D2~#LZ=@ zJn5AIn6C1p^VOE)h^Y!ZPyHhv=!&Pg3+1_yvm;-8FC2YmH?g=6BF>N~I%tMXP_@92 zDrlcT!vQ(IapW?5W-^u!zFa{M$xfnH`p@ytiOtmhUMN2~aU?I*>x8T06}YaE3?G+q z5(Pc>u**w^Pc)9AD`)%DqWCe~`AjnPTW7+LmCWQ>k*OqNyns%$GbGbw^|4@kKMsnn z;W0HS-t=EH#8q7*lh4kAeaXAv^U6ZlureK*w|diAW4EFjAhw>{3@JOmuz<*Q)bW86 z{i6K_Edw&i$WbGChSfOk4+n^Yqd(WZb_FiluYzA5Gtp;r7aSQso~wxVfR*cXsBO*w z_Wl?G?t5EV(1)!UZD`K&*?W*_8V+4H|Ja(RvgB0MLi+Kz4ISfV!n;MB^?gdaVTd4} z=&g9iRIe@I5i2K%=Exh+V_KzXowts^&-{#MY+KN3Ae&UoQKz2{In)0_!=|gM;gED=d+nC4~g@o|8Z6DY<7fWjR_c3Ov%krKP-x%1vM%gLT@I!VXIByj3 z0QSU^5lM1-_&Km#kx)Df%bTz?2B1^c%dp!mac|5BmTnUiC>_~ z_9#e2?-IN`R0`wer09ayQ8X25@PJMlysQ6-_R?2a=-~&%O45n$8!>`vxm(kT%7eI5 z!5il0jic{&w}aHJJgl82ViQgn1z&eLuw3zQ;T+X(*nDs{Ou2dvHvM{!VP$h^#_OrH z*LEH~no)-Pb~^EsmZp5iIb*I9r7k)zTk|6aE%*T^U4A>$hOQJbIO-y!=y_+LnWEQz z+--*1*O{m}wv(JM*@gf9rGe|uuQ2iZT$BhL${U{_gAK%+`W^fWQDG8Xex~TNl2>qB zLv=n_U`G#LHo^}j)>MAG3C(o&!7JK?CjJPZNg@VDZLmBKvo+-UMK4GgH^QP7rv>-^ z0X>}SKsTA|(L2w^@-a!CFvD^ojr3gyX;w-c7g@oiu@i9jwC$+#xtiT@b9HbyieJi zhom{uv)@mbSxvYC{WBgCEwE*h5z)q%cUtWzil*a_41>ePsz~emm~SlpA|4?Nidyds6uZN^_m8IG8^ZMoZE%XG;IQAVaQ)clU6_a`T^OMl1E=`xL7ob4g zN4VppA}&9_kWR5Or0w5ysC-Z*dRi<)muXX|>X+BBDg7`W4prg7Tm5*!a3wCa^#+WQ z_oP|1cGUc=CM{PQ#vjET#H53P-29pqPi}qz)>&e#;*$zZiJpK@SM0%Iste(%ndr9K zy~pLxLSXCC4Sc%qQhxQIG+*<@iCXQ;M~TdxxaopA|5a@+*gStK?aWz3=T}?N^%XC1 z;f_?YaGfqyI#I}+(`&HzrXNl39!j${P3cdCS6I?Jgl7D9L$ULZLCazE82csFIA@|5 zo#1#8?+M0Xc9{YbTXeAeKabN`|8ES@zUIW!^=fd=z9)FBBL_N;&8EaFo}Simrt{04 z_*uz*uz1%%VonZ1;?Irjq^TOsHkYMdem3;og*;U4IR<+N-v~TcA0SH#M0M4)D8XJ= zbAjCIUm_jY|CUHYQ*SUWp(y1REPUsJI(a|Tw;euhQr9KW5B=SIj9Vd;X%JN`Ro{N*qPfb z7`N7h7tG4XgdZPC>A!w9wxbe)dlX>xiB6Oefu+5N)Z7V^i zK`-wKnfFA3#Ct`s@vXyQ`YAK&vEc%4-hCT$j%1j@4eUSm(8&(L%7LO7<%{>chUOB{L z&H=Pt_6z*96Y#g{cGSIj6h3#0&fDjhaBt0CQmuOzF5OlKSut_6IpoF7>KIn8YrskO zR$>17B#`-7hKWDs2>)?49-*$n`vQ(K`)ob3NLn9)hse@wgI=_Y6!ERJM69&Kx^&tI zPclQxj_VlTMkiJ&>{(iZ3;&3&jrJ}w+3F8jI{i0P9To6VUz;)9N}YL^3Ts-UYF7+MuJx(Twv^T-Z7OGm}1qX!7_i^j&{OEI)sSE*#JCO*T0`y5-2MT270GtdzA|8=75G;`!J>= zjHq8Zhy&IeaQzc6Ui)w#Zg$&K-XHl1s_XLwn(aw=Ufzu7I$y**k6D7h4RSoO<^sGt zwHbt04wg}zDO(6L38+Jau5;l&P zrEkW5$LINW?9I2aAb&-MhUeMfu*6m@u3N_c{PW@UZ)E5`i$eT#PM?|t=fcZrWARYs z9Cl{6BHui(1)cx@8n+I-HS@ji&=u^XXS(3I5!YieeM2v4EUbVcNj0rQN zVY0)i%?lNpd3q5hY&2s9Ya;k@AK@GG{5I1&fh#J zXbhfBcf_ux62)fx;lZJN*3{W_-Ee*SPB?4r7Z)LZ5}UJOseOi4@J6Jvnmj@uB~jdp0#a0T5xPM5FUb(0v( z2nF*c>9Ew`GW4a0*4gp0xFBDhYo%hp5r8RqzT?DMqARQvTAv??$G{cjyO!P$eWkCvq&dbLpb(wBYtJcM2Jx`b_Y zdolC4Gtaai!tbvMgrtL`;AHX$SWz>KI<$TjYE&umQQt#&gVS&xT&+oK96acTFC*#9 zf8wYzFhKq(=Lrk_ls&p+AIQ>*-V zv;Ih`HPn>4o~|TMk1BEL_SIbf{YpO7?E&u8b*1f(4ueTzDrg?*WFH>vgFJ_5Ze$$C z>uzfC&|8yelm2X~->{l)b(}?S);VzTuSysw(%%*>HsF8e%h7}Db!k|&8yzSafN^g} z@$iMk>}}q2f#I17eAi~|axY;bEq|3nPohVe{^j%e`jzPZ)zb7=EibK&z=0#P9JVGcP_nkX*<1I zkVrLW&k{xFLbzpfAuL@w6wGoE9A11A`rhA8JxrtM?68q^;oR9gqDzqnSEPs-uZIMF ztK_)d)z#ElcL81ZPl`5H%JC60?sQ+N3LUxLm2T_Rxv5xYjF1-U-THg zof(&)AdWH@Xx>9_^yd_`CmmTEc&*2|`O9gvh|yVBtIu=RZ|89`$$Z-B@1g`5p<_h%Z+~taw|j2OGjSTV zTPQ=fdYaInTMdwy5Xf)-3Z;T)YiNDxCTO*D=Mo9K!Q!a}b=;yv$9>k}cit@o?YB)} zzdC^Yy}H|ZXrLN@mSV#_4Fhqh;W=1&dn%8-w~B{dk>VqLN747%iNAGvi<59_Uv$!)Yo$bt9K525tv5(NA;~1(Ym*Ao%cE<{pJqlmf496)BV$^W`E_8dE2$vR*V|j;)1RknQdJ-$pvt>ZA z^G7X6SJ=|Dz9kG>bm_1YhLF_f;{3T(hWGX7z&8CWAk`SabQ-jI$!!IGYfQY$O9=(| z_TC7FJ3JKW6O+(C-Vna8dWRntE73cj(s1`iGmIKiMUHH=;zQcoKvodWWJWhZ!Nccx zqhE?xc&H+adIsm$$ijWmdGgBEr>K#f49>Ri@QOq#_<#9^J)h2!T2&-3y(IAa*%F){ z(GCj#uE2otO?)@~IP{HnDr?y132g}gCqn;#;jd)k=cGUb^3@1EnPE!~&)w+h*U0D=32iEK-J?eSbbL(xQCUKOQX)B`=3+T;vLQWs0^D+0ih_4rR9S8kR zKER3iB1b@kMwSB)_#{u_;}kYP<=1{U*}7{BOvvBlh0E%HC!yfc?RHP-4L& zJU636@aVy4e$^#NxY<1qlBRLoJ!v#(-`a*-H^w0KHo`~`d(tMp0nZ4G%l{1thGTUD zu+Op)oE(@{`ZeM^S+W%DiD$SN227{!ys*N{VVeI*KQ& zr&6`Bjo6=&fpf3^gL}C(tci{%i#CemtS2onBUX=ulW|1bz6N6l7UK2gTAaoGV}Dxq zV6gjBmbAr?PQR|ss|R%GDQOw-oJqkU@FN;6zKM^sa@h>MA}lF?$c*#^EPmM(yj&qk z^|1_ZuCRc%`E#LpsWe%5*Bj4jkAn8=Eu^Dp4gCD^2CsB;QbF(H_Oxt`td5qVQiXUS;!r#QN8Ka40gh0SirzOB40=uxr2x$08^=IkXigGBY}gDH?@ zkRf7yJ}0O1gmBmG7%D~WCn2}}gpZ_g!VsMFvNr)T0N$rlWJ%88ol61Mk& zBL+KbF#EtQu)1rCV8P~WoVcTdYz*2Xe4uqkk@t~QA$^1(Nl>SU{vHPwJ`kg%XPbwX+k(i=un0E9o7^VIN-OHC)hG907YdlDzn!Cw=A7*28Wh@lT zG{v~h?HKPq2IS(L%dH}VV3R~0?4Pj^^JZ=$0V;~bu1Az3&?$ii#RSM~UCw?yMQj=9 zcex(eOP01?Br=7m5>+R;L@$Ws8=h&$~Tj`yXv+F%$LJ#<#)- zQDduAvLxzMRN#`(`{>?%A3WZx^6p4I)NqT#^3T8FUStBk{~8F3H}1l7b+r)CcMIQ@ zOa`Tg*0`m5DDE1fz+Ij!B7a9zye^ezWfeZBw(^87$z&nK4duS%aR zTmn`na^cgBFtGWzm0g@I@=-q$zGiwY7#g_<8)6M;%(MnJ>$N1&*i^$RG6{yJJ|UXx zJt0`^6dt~|%jN77Yu@y{3MNV>k%huetaD!k%aU)C{thh|+@yiO-%f_6_htC)lgOi< zeE`;L>fp!iiR6Go&Ax4aY7Nk>9cx$BF;^9`K#Z!Oa(wnbqlUf*k4nEYtoxn*2;4 zGcNm~t#JgX3$w9i-5eNuB9oj7*$1gnvgkELoCiL4Moo83V4B64Wh3%eWm=iswl19g z_&$D$*aN91rexusM}DS+v35py{#nx*}YL#G9M;Qq$Ltfy@im|GOXy6H#YsQ(%ac)0=W9vy^Y*;UY8 zD}>h~#=QC0MOfA4gqI%=rNNHt@JS8gVDlVy%R`PwyKzt&(&23R_9%Kr*T9nGbe!|) z6I#s7L`Au8!kIt5vsM*HayI$~KAzwW-;enVCA+p`QGpB{UY|#vmTiR@)`tamvP+>y zbrRjFn+d0a{-8mK5KS7}1mB!>n1M*M^iu0?Ezyq&5=sd6+9V*8HiW|~O?*gdp zxyF>GI#{i*79PJYP24PRf~nRnR{v-XdOM%NMIF1DDY=CnTF!Kz&JbKy6304qcH!WK zX>5zCA`N||%wI*x(T{N!RIEpl=102WZ_&EBd7Tt@$Q%?+!IY+@sYmcxYd3s1wZj!P z(s&)}r}7;r@jclfrkr|$!>V{8?CS!qwNsA<4mKTUeG)EMIYT8PET zNEA;ngz^IuVOab)*m1`m`PJ7X=+=I6c7+tzuG69Qx$bcJi$9`u96BVA0_*%{gpGlC zC|eip3diEvrdQ~s)(0mFq}kQSyD`}G6E08;#mya0Vcfj&;OefyuJnCDlhY@JImMe{ zQNly`x-**%)MvB(FO8{`D*)J^fs#Zkn)!4Xzh-66Yvw9|L*XWD3QU3enjE}x=_wO4 zt3~!}AugQq3U4^-Fr~4YU@&JYZ4vqL7iQF;S9A*QN~l3&M=^e&X&$jN--ch}6Y;NL zGsMY={2T{+vZf#rLxs~|u9_~aaLvb2+E1}n`8@plkxX*>K9z)c`1Gw$x3(!lEqZZf64%4v*xp8!WZ>_Ocu6Yg1Ck;+jQn5ExG^15+1zWQ0i&ZPHY zp5{DK6>3&&ZLy6p8p z@hYB3vc%`nFJSxW-K<>mACsQ&1|^=IgNdiZxx_1dWXZX0JguEt8QkjT!7&++Jwiu!pp5 zoB@B!T!iP2xWOT#3_Nqa8dNuR3Y53qC27LP;I}LnwGA6lZd8Zhr^#LLk(&u4x(#vN zzM=47%q=p#^k(@Tkb>@6aV+Uw0nY!h6s{bcf@7nC(WEpCJ1v|=er~>S=8K0oO@A-$ z;pfpiA{I1anlUGH57?QB;l7?Jq_=V=94UGNf$D)+^R$Uq!$>q$bKmyzgcI-C-#sIk`^8KWao-ri z6oc`v{#3qwd^Grp4@0jyJrq{HM#my8Y*AhTd$;Vw+?XfuX^R6K}#9+hKBYjB7tXSe_ZTK8bmt_pTH_YmyUx`TV%PC)wu2_}*-ur|ma z|1&&=t%pr9-%^zgS1<-mOvB4pkF)T5GucD~1AH7}k9N})=?fnjkQW~Yg~7e>VAQ{| zOw$+eOXZ(%-o6J|M7rUf`FUszE)@K$=mI^Zm#pj9_tFMuTa;g2fRpw+!be{#T>s1+ zZ2E4%L~RQkFo6l{=eArAC=ZQ+v!V?753=d>XTFJB#b{eBjZw zbc}qT3=${3$i8+lj6Z&jgf{zPl7$uOG4# z`~OoBHt)EC?*}QaaR@+(kxNNet$ba znjTLE_p2hv!zq7QGtUw%r9DVtln?B;_Qk}1qT1+20>&>F6a8PZ$mX0f_Eu6!Fg-;L z0u8s|E0uK6w|yk)ye|O3e0j3+P+WcxkHmXI z{n2Kq8$1D#{$i*sdzqxHJp{$%5ov$B4m|k>Ru7rra6XaQ>iOW)H>2P%DIzLchrmdu z4fwMwk^K;P&>_lEc*A7~aZQO6rr6$vvoxOgJ7i$xrU*B_N)q_~t76c|@2XLX-U5r!cV&CO16T>^{o&pX9SPtl(!$NU#)913qY!`fEKHlPLvuyildH>Hlq}hgO&?1H zzD=v4V97ZaI?Y3r2NmNwe)n+U{Jn5Yr%Mnn%3wFzl>*!UA1nG;BFexWBZU(Az$}_d?UpEw1oIXTgY*GLjF3P1cy(#!gq$| z!hIcO?D3G>EH~Snn&fPNg@0bd4~bATH8?I1Ea-bzT~ zlRVk%0GGv8Xz|p;utRAtE+`Fx59Z5VvX|B|Z#f&xNF9#qQsvI~Rc^CyH%&0sPo2-y z&A_#pAq+D7V02tMJ~rhzJy1P1)3tNI`(x zSp4&0pK$FJUC8-x71BG$foOIZ)OaS~;7uoEeb@d|n4l_8m%VQg_N%=?yMtNK@3;cj%oKpH;53u&cfs4U;)$1N zZ(8(s33#c6kX4?kXmNHw8L5*F>DgLnQ_u|ytOlUFGF#+*w4nKSRccUtn=Riyh%0+~ z*B_@!9E*8Dt&KdrRjo$>)V(iZ|7#Y-?CvWv7ji!#|J zs&Gb|V6Ng2RB;)G%e{0^sr-a+)cY|oWw4m|$fkm{w=|g)^8+@GxQ6)^#)6?gGFXni zDz`W%3F~71BOhxy#%1I{Zs`U5PsdiU=y)JYeh~qt#obI-ScMTP3J_zbi+rjUI{&*s z%q9tOcug`XP#zHcebP$Om5;KgnF?r=6O6%x<9W4QSmS62I=d1uBW)}UH;Y4JrcCs` z$FS$N8)5zfCm0$dLti+KW-H^BxUG0P-1D|)&8HGUWrPh0IeH!4#Jgb2^hfLglg6#@ zZb8gC9g;RvWoRc^bg5V%#+nB=d*<;~9%SFP>qY<#*$QPR%im^(n z5LN^}glofO`9Q@dye+mG9DBYBhraZN?T@Cz#-ZvMmiGo8IVGcl#VAloND=-Pn}FF* zo(h$(g@ah#hw@IHWu$pTAQ;B&6h4@B7j*rrAWQoZ{V~G`%jQ7e>Y)8r!6v18&H;X$Nx&ZFw)2ge@Bgi(SO!pLT(X85)}~! zDIDhZD)X_gBxv5Z=a{RU$)?aFEaO@?8orjsZk-6IsN0P_H{0O-?xEnM8G~V;E$R0k z<1wJ(J#1Qj9JIr#h}6A3aCzKkNZOIC#+|&J`m#Ybvm2YBCYz9TA-$8k;9KJoW9~!Pnu_sF&f}^~YaPP?x;58%^ zrk&pn-)zgVWmFQbd}Brbe2{?&sndXY?iF;+b>@q2NYeD0Y1k2~K=n`^ZfHw_rLO`H zecdl87Ru2{#-cou)=1o|yBDq>)ut1+JVhJBLqsy~CBFWR@ZysPwhk+W$twEr#o>rB zzbR3q+cjY5#&*)-VMV31!ojLe4~l&kv;8wCW9TbSwD{2vv0_S?cw{txw#q#kb5j6`_t(802%(6~A`FthpDHsJqCVs&}k#`^Y zAO(N8C%{cnrgnGo7_P3bP6G<>K!cJHGPlHIN%$)q9B}~FJ@vziF^2r4=2MmyG7}AZ zC1Ge$H)#FT=c~foU_hsXtsNZAhb=Ln1sZiMuyzX^SN=w_(|QHVPwWz$P}s{V6$im% z@I9K2EM~U5maCFq&!wn|`i=858VDWzQ78 z4qmfi+Hs(_s~U_pO~P5GW1#1Tol<-AqB;23+4QuS;&^F*P ztUkXBni|`QyJa-I4Zh31Ih=;7k~?_uyeoLJdGyonldR)e9$pO8#C7JMVZ3HIRLn0I zI)8i*oYz6x^ca{S>KXB$ds^_l)sgN9DrC*yuVCK7Ur-ny1-BPgg0JE?e4&30R{U## z5$T(W>dJF0YwAq=<1R&GJ2mI!dvE`uy-rMfTZ_DWNk|p!7vQeJjd|L|Poh|VGc{4e4Qk+NJ>w{#oH{ka8AylWS zW9Gg+Sa|t0{+$%UCWicC|FyMX&%7tFdW#P6eLEdTSUFR>o-X3iA}*XWJ^|)BDPqmX z7jWUtIr#dp0ZYA7F#f3T)y27ciRZ#N94__~R{CVX4)@;>VRDAZ&$Ne2#vx?hy-@gS zb{JQh*ogXQ4iNKIUhJ^hMU2lMPc~cch2CvPVYVo*4r&b2D@z2kT{2ke&EL>2KNq(> z+=gQD(d6F|b6yy}5{%zZ73qgBpnRZaoG5 zn)RTip8?s^f1%-D4JZjR#l|rYQ882s7o3)-52kkz@rKc0tn5Hb-yvk(ISbo6;xKrB zCs+(i2aB_@WLcR8?%QodO|7?;m%5I{waOE<2LohKd0n_IWA@ZMmVb-B1Fe&I6-kq(22X7TabyXz>wWPthdqL2(ekBofH9~Bf z4*x0jP1vKM0-=Fvm=f|)ptUlU%o*4SOP{of^r+$B z6T!#z7pbimCmtz|;JZhgxYz84z}4bh{HRBH+`t;>c>fKQ#RhTvDSxKBIgIok(4sNB z_Twg{KbRXQ#&dP|;OIDY6#LW*SKqGg?DvAf!Nka9F zi_l{628O%u#6PBv*p#M?rvjQ;ma88YY%ODQ_Rk^sZxF=2tHt_zYvE7dSZ0@cD_Y}t(= z_)f_b3l2U=H0i^^wUc=bLonsbrd*Jy;n2rYgpwS(16G6d5f_u;CD zPJSePkd%NXxV`ufmYJO;Q;t0bJ+liW_EZ!3?L*j;+*2^%CB>Q~8?k(SC*<^eVip0e zIMwVlD*jDoN{!DUfof9Q@5S&XIR%oUwYj3B1iBYo$I~4bV1CaM?q55EH?2_NQ3s1K zJz^Q1GggAe_ZNf9he2pw_#Cqwtg&9Dk&P3Lh(2!c5|b3Ac&(2C@1eWUIyD^}CEH-E zC_gsy>l8k|xD7i?wK3al90dII=UmK(+llmnC8O=}bgT{kaIXU$N?u@V$zm)x`-3cO zbP?q~HMsNLQoPcC3ZEr;(QCg1bm&JL(Z0EiHOhPi<>7jOJrg15xgpWFyi}5RmSp0!^*b>jTawpo z978u4X;SQMhm*O1V3R)%TU%`*|JhT>*mji^c17U)=GUaBySIGqpg!MWT!S^QCX?y4 zWuWAD0rxoc5V>G;Jm;&y54B2?10kwh|4Sqee7ZVa-jtlW9aF9C-LH6 z89sLZbzBx8$)&3l>4U$2Awx=!+PN!H>7_-mxWt{xY>$QMb`$6*2T>-++>&R=%W}Wn zC-H@mEH@Tw77U&H5$Z0Pfm_XPa{3KI%%M_jNW6;+v>YgVQ3b;bI$`z8Y1q&sNymO= zSam)hA6!nu^XCotr+*^-qF06V^#zd839Tf^P>~M#o*+8&TG6h!Iq)auIoQ=~1J~uI zFo;rgytt@$VPzsV?Jb0-rH*|4BWtdbKrIMaAt}UzRCbQWz_Rv_mL2SRMCqa{Mo@S1wEBcvbSP!e$n?z@B7yvDW zUI^blA8X>*5cMC8==%O3%(E)OB|*cuPM0#y^4W<=IU{hnlKR>;1j*TCVqx1sxi9(d4f{4k{()K-0k=_ScT zC&-RZ7JmTyW7@FgWhjv`Qlc^WDxep6ndxkDq;44^55;R2o3Zx_9vyxd9(TQhAv=%3 z#f@!H72`o~Hq7M56Fx&1(clV0K7-%RA7s+7!(_fF>#}y#N8)z5im8Oi(BCFwVMF|X zuxMZ?u8BTM-YD5~gW_+{+pkO0@81zxtBcV~58c_uZ?f?00pjPKBd5)h9tU71Lg}a+!QS*MNoA>}GwEO~@Eux;f&}3Yok^*iIs_^>C1-Ppu z0w4Yf25WT` zdCa`An9;fv-wy4Dg09c7Zf_@+HeH4GvKe^Z>Jt37E6aJej}iS`zgjptbQ0ZI5(4H6 zjj8^(L8v|NKxfo-;*szSyml;}I7biQrmz$|9QK7I?CTX)8N0!;HKW;|&M7?Ss|opW z&Wm5W)=C2J>e99cQdITZdw3>s17odcLVKO4HdWAsQ@5(f2IqtDF-;CqlSFx~jymUQEz|21P` zu@5i*uFq;u$?)g9jtfGDwxN^#UR?TWA6xbL6O6f_hfx+t)LtDzFZ{{OvK(-@%L(}T zdY;QC)1COj(2-u9XUxN6y9EvziR@6qeQ1^Y2Pay-Le}akF#DxIJw!cS?}lu~8EZhK zNsWW{NKF=dw1ns_i-XnIzQdcJlW_3d02y1vQ1iMqEHzUH`xqlw-8z&w&EAdvyW>D= zz?mCMk0O&C$MKD4vVjInpwH|@n#p%l7=Q3B+#m49exC(>$0r#%*R zx9=u_PmiGbyDXfx<2$$ zx$|kq?F<+tJwRUEN{2bQ_lUdrC2}d`0E}G_3++XwykNZ|mgmm^Z-)U!C1YX!90?4V z?Jkg3>SmR3KjFKcEVx^hFq=Cm_%)~+TUOl=7-sLr*vVpW=vy`#(F&;kIt8AuIgFP= z9})kZ6Pf>}U9eZ<2+7;!!IMR~hDMQZ-O}8SPs{Z9x=C$lVRjIIX6InX#UjR!N|LgO z(L|@Y0F`%(@)WLRFk4&(%MK%YZtNM5wzx_TDaGRTi;FS*F2VK_94`0Q zF;97KoFB7Y;A5}>R)%WxT49wiJF_0E(kdW*!Xx~sVv9-E+B|;lCU_jshlW3$aqo#@ zR(?es9u#3eb_Jf_l$hvZKBN z_E@E2@S%gars5J&POV4ZM?%zzT#4J~D8Z9Qf1oSSfLhISfYQ+8_(VdBtM567-^ZG8 zZHXvcZdglZ??{31`^^ONH($s7*)EW(Jd}oKHi4d%BKP`hC3=qJ$(*GUyz@voJ_xnp z7s`HP*~cr`QN?$rNLWh>tpBH6JcCyU(zaiy^8VwziBCKmF0RP-Uh8`*K zT{Q9ez_4=ky5uA{5i|so?C-HkmktzM{QN(L&ij$-H;Us~5t*5l5lN(}!M*o6*KSIb zG&D%GC#7#g$=*UXS(UO9aXU;ly`@`pR@AKSq-sgSZFUITS z4CZ>F0%z#XOu-e2#f;4G7~!(vN~`i&CpljY1)PH$URrHwOUUN|#4h1umlQCK0KyKjhU7pUI+!^k|J zA)Mj-n|UZPx_W7yu;ydc5#cgfCm}o=u!`>ojzYm##_z3)@TK;6VSx2jroYWf$SJta zbnUn(c%@{(w5c!RN_PI@?2Mnx@sd>JN=5A8UYb3HThuzjybMPs-KB@qeAkO>Gh-f? zF&)pndAFORJN_o~YTR$ehWn1Wa*}0^*7h(SY#L`n@_gO+e$=Y;{b5O2? zvvA#rGbXpjBPd=&d5^kD<{ z<}M4awM&~|C~2*5ZNvuQ3!~YL)N-*r>JcexU(Ob z;E(s1Q76oWT~l3!*Q6$JYs_T?4r^KFcJKoZ;~K~LyYveuE$@TCbBUU;Phc*TUouu$ zF>bz)*CQu9U^HC){-+Mt*5)cluSj3$TDwj7B&kg>&3c0H$2^v^A!jW2l2j10QhgcM zd)x{x|LsKXO#K`tM%hkilxQgA1aGRbe9RN7mMjxKJ?YBzB73=O>F0uI z`7@k>-F?gluld4hpvKr|=y2z%G%|l;4+{G^9HDp}kJAw`kCXDuhO1TP&7FPy0Z0C- zDQDy8$-??b7vWQ_yxfiQ{lbyu;vcCOp2&Q@BMZgL5<4 zlA9Ga$kBbua)xis7FxSDV{8iH^ODXwN++Sspl{@19u+4(<^~S8o>19T~;w+_vECI+!LnqM6J@TNw%4 z<>a{rB{`g_4?ZwHF%yK#YSg(FC*8SyV%_#!>U^Q)4lSnk&1&w|Bf4DY@s*t4>=w?c zOLMuBuKk=91`HE&WQC<~x}tCkqaiFjcbBPXyv?~Kr^+=eY~qBu++kjq7cyam?!v&@ z00vgcaX*E;7i+z~T>r7<96OUi=92bfCOP|yIMXJcu@6|tNe^GhSz4jWbyu+G#+W}} z5dCJW)$iAnx%aAMxtBf;bMlVO5cpoV7dF4LW$_ePc%$Ynvouj)+ zRoIX=O89WWT;@kesnvb)th-{1itv?p87H%2G*{OQglW|-HS4>tGCDIdn1yW_jGw%$ z)%0g$goRBC%xsP$XJyoA?vnGVR!?-4gaX$jL7{0sQ)zd$X2QG#fph4bnvEx~F#qs6;(U-2>j16i1!Jmib41tmggqyJTPbB<;(X%e zSrzXLtBDP_W^PY&4TS5{YOUT!wp+Ri#SFmi#cuIV;LvG1ZI5dXl775+Uj)fB*t`xx^TIyz{)0Xiq+uK z9)=UT$m;B!G|q|cTFw}@k(uKA-b%>#<2-GdA~;a<(=yvPj^Vjp5PVNF;|OQeSG#O$ zt%)5NvRaw_g0sq9lM$y53BJspBe`S5Gm`_l7!-TdH#drRWWU!i+u9#mnY?r6 z6ff;(&W9hd+AiJ7Dfs@%Dsr}pu(&l%pjI_UfZskbpFan3PMYa(M(64XDh-fX@Jzrt z_u@RWKH8G=+p2~0uKhb>*Xk~yWyd*fw;nJqRzEmYay&O}$0kl({xy!$qR? zh9o#!M$|16zbbPY9d8JxPE2KzqO+_t_D|xNs!ZTYGl#AGu6}2B#inxxry4U^sWr@+ zEZ>^y>$5p8c7C&+x{B?$4Pt##b zyHK?1Qm(XmTP40{l1_8p1ottuaV<4lX8hw6$m?=vZ%`247pKKvz9h#P&}y%l-bZVO zk3Y8Jg}r6$O^ms(^nz<1gtlA#{IZdgB|V$b7nXDEH#IRcdNVklXT|$=v%fO6M}Ji( z%)7+V{Sd*i`SGhJ68^Ii{CvvUtn=W5)QhBJ7P>Nkm*;(B)0k(spU;0%1%pMm_WG$`F}gs+t1VDQ#C zREZD5lN)5|#3|*ZaI!c{GTM^eZpWi%gVjLLs|7v@nj%|?58|xfSmO13HwHddfghJP zgIBRJI?b+x$3gpG>ElJHp?ipgPLG9km3MfHBU|x(zc*ZYC_$hEb6WoN{kr~Oj|j=zPue5t~V8J zE-{2pGVgf*iocWp{4_;h9)E%v9mk<*iW9^~B=h(OKN5%U&h+lO17!cWXtt#38r~S2 z1##bC*zAl%btgis}r=5qzyTlp9*)jBe&v;bOdIY62LfHowr;ynWW5}<=E8(Yu4$N+~ z!r0Q|gy(wzG(*#f+9MO1Yk!(X+n<7?UZdEEPm{S%U}WWr8oRJwGNH>=4HZu<2zKQ>6=TU`QocDvxG4R$0N+3Un3J@4Uo=lz>Lyy( zRT2l)B#@RBz;Ayey0FEL%vTKRgUSuU>~MHYQ?OLTz9zT&9Hb&7sIlVmfZ?mAk z*BwnS3FxqU3@2=XqnOKQLp41cX;zR8`0S5_Vdyq+|X-n8g6S*r4dgz!<74n#XP-WGPC^+^-V9vp8w?NGqWhno$?S~o=8KpoKujz zs{z#VjM*Q>Lb^V%6SjL;kQfPfsBV9QGD2OEpM@)qz1&SUtGlw?d9g6=N*ql$iGV#x z6ENa;C6wf*ih1nm`24CY7V%ktf&fxF;T_xli$U2wDgK6-KUCxDMJgIvht2IQ;?c3J z|KMk$=ftHd4}K8mhhK!|N1}1RVjz{0TZ5|~wBYL#`%zUXg%qur&UUCq({;3C<}Kc^c{V=RY+-X7Bw@=ZA-Jd~utU5v z;Jq>tc2>v2oJ(R3vO_LLU(SG?KJQ@s#q%&26ifQg=3|A$DOee{1|Cm{17(XgVzd4$ z)$N)^d%mji-5#f5msk5a+x84$XJkBKi&Ai{@pLR-O{n7eUdoGnPlVqXJl%K^6s7X0wrqTDYMf`~7w4<~N?$)bSj)Pd`EoM{`O4gq5W0aWXIF+!+#o z`XJ4f>VcE}$U5KNOvY@ufI7O~c;!za+1!x>RWz1v4bG)z#c~k&@*C0JbOuMg3!-Ir zy66V396WMQL?g>q+-9OmEAc#z+4&y)q;Ami_D5n~TQWT#zXO`yh0tX#x8QKCB6c42 z$3|Y?mOrvog$qBGW7XFIsRI^ z8=x|2BSha7-${4P$daDfc&Bm`7>hYpJd#d3XQjZ#hb-)TdKukn%3%28FRVxr^J!na z;~lTr02UMKNxzs4dt-G89_)w~v)3=u*B8G7`#J-s7in;hUHwaps|skRdp>$hFor$l zOGue)1~Ir?i;CaI;is%z(*5=bZt}i_GkuIx-Q+J7Em6n)y7*t!L44wv?!0PdXg)kmn~moW-)V z!oS_J(6aa=O;I?4XPn~k{u?*EaH*JXIP{2^Xc^Gf*2%DHARA2>4VZB^2d2p6EBBw$gvn8K8feahK?mz5@ZaM2LAE1RpY4g^{;e#*<9Xyo(V3i3xw6b zQXo^z+OuvRAQi!-#9QMw)lfPKsqVIDs$B)grfZ?8TQ2NbIS7I3C#lt8WtwJb!<&2T zKe!(*f>47eT^|a@X8H3{@xzAIA0s2!rluDraHi6cTZebdY#zJzlQNj zx+n(gW!_j;F zuuSrP>9pDjdT`a!kMRYEB zfpOzql9wC~@{)S|Q}f&K{;C$bGv_iUIUS&jzdnWHU3c-VRWxc={w00fz0e+k8ii+M4?($LeSA;b_ zKQsy^=1N0&RXgeSy+Ln(3d8f~PQq{Rx9opOQdH){btsuPhm@QZ=dN>Rg5dWqG)lD= zxy9V!t?N#qMctP%Jp3j}@RY&N1rp$w>Cbzx;XZY`^PMhVc^A57Pr*&M#A(SM%6u;8 z7kM$q990eKc?LfA;t(`*bhzCCa+!C*xOz79{8A@tTh2gS-+of3T8&Ld%cxzFB94UA zlTl9N@NGj65gzscy;YBB=%WaFgL@I?PV&PC`^BG84QF>qzZ3E&UM8z!Vklo=2J!W4 z*k_FkAZ1=Sb+{)(t#y6i?L-?o>&rRZD|iiSdnmhPuQBzUup5^2y`(0C#+d4yK?_0( zA>~y(y?yrz>IV+;%E$MT{mUYuLN}OHI80@)=OAq_%YyH_#_|tY?w}W6q|)%2r_m|= zHf_2!lSss75w*HS@YZ`Sc)hs~7v@Tf>9I;w-=hw6GqNG`@FhI(zJ_`DGMi30kaY3n#9> z_h5I00m|7$?hdH`dJ}&%bYXLM z5|OP)gp{3)bjPPrh%b-Q|Mmz#duABdIw&0voXp3twmbN8ktrI~-6XARRmB|Ktx%&E zN=Nd#=&94Q=}FOUlx%&9CcAQZNk`72Mf5|^vW|v~FcYxSdr6`{?}VMdGjVs@6HI@o zf`2j_=*yjdcuvf1@Y-c2I<$@^NvqlKrsu7UXvJ9vC* z6jViWfHydn?TxsAsv8S!Vz~lgrU4%INrVG2l;bUux&vG z8J^~Y2R#zN;&>;RW?aRc`33m%^+GsmdmQ`~%Bb>$)hHEK0sk5Qh8X33a;<1FD`S>L z=J-0})}036Bk_!+tebYW7q zrR#v>HeZKJ&#z+3o>)>yA|F)MM z=lGGnI9Y%y%i4Gnwgj9~tC3{R^Ltge8sCGwE_8{bMy`gbnAmZ zcT%a*s4FxlLR<8-MZi{EbM%V z_wWXXUpW;%49P>H81FbUwik;YoPr!W5i;hl!o)x0`N|h3!Q#Lo-mk!wp^ zQ(Pr(&+3JOdjis_nn*8o{Ghek%_OPpFO=N40!O*BP;hV|9<#K<oC$K(JkL? zzh8|KFRw!p?8TebK~VPTCq_<^7uES@!oIx~BrEP4j>-+i-_EN1SjY9~Tcym;-O2+? zyLdACggUCOxPT}mxb)j-OpRCJOK)-HHQsfEz2*PGlGz)`kJVnV_i-xsHICz+2Z5m&~=|oxDB-Gw1b^Wmk>iBdS2|$}#xX z?FSv`+K$h&omp<$yuR_)!3@A{TQ$d2XM+&L`13uCCMpEO<^ znM3`C_K7%6SHqGOqZz#zTz`%mb}ew7-++f(Ivw3l1>=7c^o>$ zpQT&AX`!KX3R(5n05{aF&J4=uZ=+@U*|6kyHPIRq1Uk!qW8#WaY6S-l5eO$e)6B(7C9BgnTW*F(K2i zTf&=nQg~ZBWCg8;p{?JkO;a}AS$LXV zyK51A_3#9K!AG(#S_Dg_ztFEGrZnNhQz+bxyp&fm;JDliwjOWd*)-=tfSm*29ZCLG z=@{q_i-Ava_c60mU$pm#CTi>mBG%YOP`Qo97^dK!&?ufweJk%FOoJn%t_c0+uvAmK zK+N}hz-BICh|73u;_MwQl)R+{YGSV4Y4;<9H}I0QM!ctwKH1^FMUQx8={E>FbP#iv zd+=Dl@%)3!|KcL6B5_EK3$3e4q-O31aM!sacuP95rY?w9`z6te5)W!EqYS5xH1PT= z^67%XBHU?S3?rG0z@`|{#hSs?r<2%SAICz#;u-eHjB^#bVWH2U_wk1*dLVi)E>^sGUj| zuDf4~$#TAU)iMLswk{+|XX?m{&{4Re+d{0}sS}C&lA?PGwj|0vA9EIqV7Pe&+_3Y) zZK72qUEqU?i#Nbh4LywHIY8g>b~>kTC$z4rBP-Ti=N;m3DchGqLgt;qC4WjF|8*99 zBNB7Oil1X+eFjybei(4oR?HB5M_aC$qxw=2JAdOwUWJ!5pQo2XBF8)-<{eF>cQ6$# z{=TKkJPlFWkJGqrhYHHk-EZC}ok!tsdVtpVkdZG%|uOvhoM;kHf zfh$g$ycQ3ZP6XGR&SD;59F?lS#<3Z$MgyEnF6-;kx3wo|MO_vK9{x%WiaDQVwORCU z_c_eEZ$P8NZ$itgTv$Hbgq~v(;HafNEaf3lo30Bw&eU1p2f* zLT_^ui2C!Jm3|Wp|Gr-&wxg%fmc#oX#jSv6bj%M|oplwnc5SGSaWpLO6|uS(bKvQ` zJh-a&nzDzd5z}{DsLSvFK)J3I{k&e&mRMUD)AJLH=BB{>)3Kmf8%1u8X(D|W`TRT6 zCqdDNCNj3;99Lr4ZI$TyKY& zZG(Hh%c#MVAoP~Az?y3vkfN%Jdrukz@2Dm5XmX@yW!v$i-AS}ty9}(k%Gi3{4Bd*V zP;}WHo7eMklwTHUJ&@r-pRl@(Eiueop{|?Y)G4 z^%WpmejgO<7oy9nIe59Ii!Mw3K;vB-aEk6*NZw;WmpZH=M>J|k=GG9nm->unm|KdP zZ;Y{GW(+Og^MLLb#DZU37W!=r25&uk@SGh2hA-vtzF-%|xf495DUS-fj0mf~k*D$E zH}$(Gpz>icsO<#*@^h4pRMZ61+_^ zeWqa6DKw7Lf%ZQs#Ql5> zHs3V|{VF9GdfiVYDh#2nHxSaq8D7ZnVbWkMO_a_^h<1c0V{SkmPviVeTHPZK z=p9o;KDgD;GnZL>sBTUp52}c64oBk5t~7k;w@5VV#38VGy%y(1-=@odY{l=-^9dP@P-BwC|jxNa^GQq1wY(YCm67G*)mA4oX5(fc1I;Kz`F4qf^ujPu<~8efECZ)-Y+xL?5FxI(xTt&V3*MtD~b#Smxf z!}#I&L`-q>q6u?LX#2#Im@@ka&U&_({AcKiWuK-Yw_ys7H_XH2!U3}LxR5)rYAad! zcZgbjyG)W+w2{4SnLKHQ_23wJl{~Nw#fEXq;d4O?iB?yJF2)eAuNNlDuD0PU$IAv7eexwv9R~#Ro?tSNBkDx0H;i^6Y}{DG&NN!a7r-hN#N{)~$hxA6U6YHUKA7Bt3Pz3LF}K|%n1B0m9b%{P$~{f~&JQ4Lzw`9XQZdgv-kBg}=P z*siz%5>|TR*VqfN)W;gO+*0A2Rc!~*K0_VePr|cW?P!yufcftLE=*m7isO2PQ;#I$ z#p#MLotV+k{ac|T}<>oMT@l26M{Ze|#w*=CMx8hdy9rWm%z2x{=We{ArLSD)#v$C}l;dNIDENVRp zpGvhvLFHXET6Z6>P7THa1PIFS0X(7G>iPXs?5T;Q>ZLH>+*Lfnd|Wv z%}Z|3#CkUn-WZ2W5f3kE#8Bt-VAi%pNXFUvQp4wcILgZ!9%BYxST>HIbN)6g54wgW zZ3B2C_Z-b~+lkMc)?;4RLCR;<_&3!9*@vtE)SWNk$$bOxa}aq>o-@R}a!c{E{v!=u zgETkc9@)bS#@j7-(O@C|m4Olpd1hpS>@jhLUILclSTs*H86mpxYTJ@mvdkWpb&+W+BuL zromWa2K`T+<*vN>75>3EhGB%dIA7BIz#>t6pOfcrFTCD;5c~1CEHVet}sr0~L22r|o zj;z^p3msfI;{6hB_DstMyzO<8H+p?A$f?MoUTQnGWu3>zMxR*i>|@-Wy^k^N?{g}* zUXwWNPQ@SDvY?#dNv~}@0?PW_(3O1DwO0Ar2>9 zaQrTy_cw0G?4!0k_kf#l-$s(JFgXDo1J1GKO1~lFSPiwi9#3C38bH#*1tR&t0D`G! zaX7n;ByM*CuG$#-q1;M5UzX-SF8jiMiByH~RDYUdEQjjXA5ebg5JY+`p;05tA@%Ti zE@$d!{+N>qsM1=E7K66n^Dd$r?QFF%~wvYE^-TZg+`_Mz3DH#Fd#2T!8-8a-dT zAM=|xgQn^ycqUd{=32}E`BP8CJmMDOtR{X=&q8*1Q0#g5F6OH%Xo#G0%IR;bbGYYk zEqc|Iz;1R2xORM?p-F#;(dHHKGwCj`^I-~AxzLSYLYBa!9}!|^cqY|NmH`}Az_tEb zIF3m`lfOTaC{EzpTE}DgolV##@8p;Nh}+<4T;foO z`rpjJC#VyicGqFYZdd$M8i7_8iSYNIw8-|01V4Mvb)KD+Db78=46|mQ!QwH;G5mug zj&6yi15I|gp)DGA4>?fVdM*?lUPlg^#6#&#V`1T`?N}ibg)b%4aJ;$|%1>0L{kyVo z&JO{pdeMra_Ap#g_nl}SOC~HvP~+BIQuuEN$hWlco|rU|d(+=gxs^LHSw4VH4qkx? z2T!7K_b!~+@QVhNo+J365HqU2@xoj!z;|jqj&)s%%3~_f=kFjd@sly;>l@(ky7Bbq z!m03d=Wdu%nMdBP492MFSl+8(A9m%%>oE3>HJUj!vKRll!?URi=?5u&x|)B5H!1uV z%imuI;|zA;IDbPNy?gD z%1#kcJ-Hwt<7UA)g>A%dzy@AoFlelshmKjt;COflERuhYegUmq*{7bE%>Pbr4<%#E z3`qzZ{|Zis=jvt#jYM)k4?I_|V0~hHdD|9kgGnG>~Nz@RhzjF}WDZ;~X@Fu|0-pVcJ0ty2s+YdL90`0TbZJyhp=I zXEE>i5X#Ka14eo`oiHjJ9A3JR(K))j4&hJKoutb*zdc0U-izP&!+z3sHVJNPi~DHF zI_f_Z2!(T`X?}$aEX#_d<;7wjlywn=@4Q5FWzW-XJ13*_>fglEB zWA^>Q4q-UV6!#KuZ|lRfh;699b0I$eph*=I`gnOK$DmF}KRtUTiCxU^X2KNBIA_Y-G0n1YzuipgO$FxQNW3;N{Yr^RmY-7kl3Z&uQP75Bk+a~C~Uok!%NWWci8gm2Jnag+4hf|j2SE5|2DsMQENR`6IOk)(rdyN_!>vBRVN`!_Fz#>^(pg&RM`=4-b$(KMc)U z(e(1QbExgw0P|x9c+X6i;-*Xm+>_3Qv_A-%(`WK*lVa(pjqljYDYASyr%;sIGL}tP zkjK-vp3IMUtB7{HeCXyYkFkA5pqNjc%*x%Df(LSc;AMj%F3>!Vwu^hwqtO-(x^L6- z>C$9wa~ES%tpxT9D0<5rq7&LKk-iH`FcJ}rJ7fZ&sW}X^O47wWNf|Voq+?XmMvy;# z4)1I%!~CFIWPZ>^x-6WFjT>)~{&N%f&f?j~coIV#8>F#QUY3bOhM?{UJSz+%Vjx zTX^GI3d?X3NUw7@nZ7O+d%9+`zwEt;wTCLIEIf_NhjnqsBQBXNQ zbMjB|Dx2vWh|OO4@G2^uyha@`7mgFP9f?E76_IdyqzD^5)98Yb$#m7;2}rw|NpbZH z*4H%^Nb`72tNQjzrfJ~Q2gLbu6I$XOF%4Re{kHt!St|kYSQRDED z^IVa8co%wxD)N&G{iyY!jcokB5A2+O8_~Wm9ZPy5=}~3|4cOz1Hu=f8_>C6zKCi$( z-8Tyt=bB*8fnbs_u@gQ|xd=~RMZ?;n7@ljW4u5}hGF`f>m^ivw;lvYHz&6{6G;7o?`HUgnpXu!r()`58pRh5n2_)Utprm9kk$a;I>&|&W zUuy~!axPKd+zAlTIfrK6Zl|TwT;Y~S8i+*iNd8!Vuox~7HxdOTZ=DU4=JtTaB@@2- zQaRBCNeS5g5Mgx3BGP($CV#E>9`>X23tYEYKo138q(p8Z?`WGO-SGPm-EhDH^J}>H zcugj{*?Yt53~QXPtVI>RKNYqY1<@0Gg4i7|reIH83-Qvi5PRxA(jWO6eAvgO4|20% zj71G;nrw$#W|a_K2^V_Y;x_&|x&^;V-6LNvO2N3+$#iJ`VcztzPdvqe{kV=QPGd$XUcpI?&$+-RS0X)7dl&;fJ zK<`;vkg(SU4t*9-{eMTOM3O&ll-vlqGdI$0y9L<%VGGavnKi%Y(GXdGM+dE2OwniI z0Oqgw#&iCC9J6x9@Vl-W;ef|O@_E({=zgD1cE?^Iw;FVCFzYd1AMcM2W=%w?aVt6+ zIl|$rWLVPqgq#!CebPRSs5qmLDBSI%UJ_eC(X0*kDJkJndvpF*b$60*@iBRu zT*_X1I|64`uEy}I&xMxztf;!|T~y;q!=TG_JiBBv%wvo&u%eUR+xL=`JTW3u-yg<1 zUsyO6agjus`s1q7pY+kO`{eb+aWpAf?6V1#2jedTw0T`S&+6p?x-aY)7&mPNPw75% ztt-Ns(6Kmmzmn*{A!qP^tBvv7Oxa~~#5LTaNf7RLje1U##H(gnX-p)zicJ{orosCk601?QIkX`uLQ%pW@nJ*IQ=J;{UHA$ZOp6Q*g;#koBSU@zN8 z%B@5&*{u}DhuEQ_>Pi}`w2nOdRg801KcxGg*pY8f!tiR2Hf+&S! zF{?O4yOzp=<}ztI_Te$ybu}M$J@QA5-BILnWC)&EGX-7ty#VE5d3M(6LApwk2lq^V zlh8bAdU0DYWI3#YlpuSmdgLpaYVnl5`>_)O96z&ZZ7JxlZh`w>+=jkpZ@TGTI=(rk zDoPJc$KWeRG3nlUaDJT4dPf{)Kd%a=Hk(?oU1^G_gUJ9h-6&`j?V(FvUWP6&3gbTn zVZM|(94eXsb!LCbWi4;?S1f?EjSn#P*drRI+fV#7M}c2SFC_Z)lRvdqu%IK4uA6rb zOA_U9ecDXO*V6!*ap}aVx(SP~hHyU&v*=e4gPX)E#mU@#=Aa&D~U%fq`QEX>Y><5S4VIc)Ls#SCh*tG>pN$=VS4UWe@MhNdx>l zP7dbO_#h10V$QFvWc@5-JhEIuv_5NuZ8?0C7O!%E%~1(xCGUl8$5+GNI9*~tp_|n_ zHCt4$r=Q;W`GM^DQ^CqDJp;0O!BB1+Ln7*wc=MgVvbmj=kXtJcGMv4n?)4<$z6IaO z8mATHz{|HZuEK~4yfygG_Sj&0#%}m8W+uLQXaFx;?BM*bRm5hQ1m9;*45@yi0@~7w z{K=Cl;DYvk+O{_own_ESoQ^tffA1ZTxOD-aUtEe#Dep)>r-grH-w&X%_L)Q6P@h-iwv0*fYT9GYH0NjhhK~$ zroAC_fl2|5kkq3$Uf%)xYE5>sN;r1qXQA8sV$9jBOT<)Y_Iz13r049#&irb;9(0E- z@VOxNLj_^Bkt3TqoCW*p9+E)4GgR;1eDZqEbQqj86CO@`LN5I5CtV)@D6N;kdZLf7 zd-BA(+#*_kI|}CC74O-6j)uMGzLAsJ%VEa0*CaorlDuvR#$YFN5O1~7GrHSIer6ge zGY;q7u_Sn+@DHee*#bS61#CfPK6NOb%vU}#hIeau0M4DH1d{Dkxcb}@@jOcs{=Bk> z`lMZC)(d@pM#ydqbaf`yv6d0uRG?;VE9Zi6lQQupO8^9i(-knuw>n7(bqqVKYL< zLCkR_+=!Cwbn!j`XILF!+d_<14gwS6BKm#s8R$vX!`^90)Wu*9Yrmq6UWNcoYfKTtu>Uqp<3)G9;||LF-2!z*BEC@#pi?Fh*>uaj)FY`aTSV1zAe` zan}=>qnR^cQj#17XY7Q?H#)Fp&K0t{vKUlHyMpCoJN!CtE&hdlpSP5^ zwRR^Q^Gp~r1)0sl1Og#{~NzfCVBSVWNB;{S~2mRu~F_8iN9$w0rt89MKx z9XMzj(RByKTH*FaBBN(QR%*6FeX&2h|CC2lzsA!K8b5JC*#uhfF`UYFycGWy=1_@g znHc|$Ll(b(Bve$O#O7H7tnlq1KBoh1t%^ z7~v64?Cw7!Q~W=|oNfs`IwlH@yUxH?&P;m$cQi~(nMF$7CWA?dj>w^T0nEzO1TZ{L z0{NpvJA(tjb8;-*=#xpRF22Cfw+oTEy9^G!&ZMr!akN1YB(95e(Z0U}tY2NADa)V1 zgXBnz(~yA9`B!*{eqR97t}d#YejH|Pzm5TsL2zMqB(AF)iyr<7F!6l^Ni&s3qjPhh z`=bF}wdOf>o57IyQE$or44wHuRb3Ruk$FmHAu~nh3}IW+GU=k+PsmGBROs8z zVXud~qs%)doG3lZbw;LgZ(m1as(wKTY+fmwpE2fMkCf(7)$srQ_uLq6*`H-jox??gtohXfZQjT?VR zU#3UAYlsI=if8@&88?M(=dvaQ(5q)=H10WjntuAHjP3;P;w~Sm6*P&;H@^L<&Mt@? zWi9R$(x0n$(#7kKQeKM&{i}DHJ~x!0dK+M4^Wg}(eakRAP(tbOxG8EJzJjD%3aDq& zSvvXJYwq`dAGze#n=YG&N|7L=j1`#_M%TSCrWF@ouv0%oa3+QeT;4D7<8}S^qOEBu zEM8`f_S}$aJnglQo5u)uvgM8_D0&GeIJ$@3ZSh;l$ip(7%=1 zC~4DF_A!Iu;dked=FlTHE29`m=V_o^AB>cRJC$KMSsGw*n7zU0p(r558`A2~#65A?JI9S3NbbO(_kxH&h+YF5gy{z_0 zz3i5{IP~mND*AQm1{!WwLvxzNc)#!433bb(oM^2neUWjM+MDu_r}$!&_uvRhhh@m^ zzaBdG@J+OD>uh#RL>c{cn#r1EEMbM~M7FiPfIZ>w!*vhz(~~C`vj%H_(H|aCtU$_$ z{<#v2+`Z?cciSD&`VW=RwPrFsX~}YbZ2!58h08R~43kFS);YG~>OJ&UrJD5&x{U5l zKZ@>;3<<_o6rqpv)Y;p{+O$?Ag9{#VMfz<|T+)1VIeq^+&L$z6Q;dwKqi6TJJlU`j zb=xcPx`oUt)7?>YRIna331^Ne)8}&F#|^amUl^-m`PD^oek>PO9fRCDvT59iDr)1^ z;uk&#UC8ljde(UbWO(T`M}9tiJF)|9Fa3vF1(9@Qt{BVxIE8kW-Qkk2J0Z7-^ z3N;qdW!K*n_6q%%(E=cf;V@J3-&~gc*U4?a)5`wctl0SGz&)CpoXR@N@1?pPwe;BLKdA4{TiP0w#A$ud z;12Iir}o~(sK;56*Ecqay4fhBnz2sW`9K~Ge?7%6`H(J%O1XgcD;uywYqi*@>JRAI zpeU+6v49T8=ec~Bl}Gb(>`{YkJ{m5#hAM?~*QSr+bmQ49bhd0Yz2F?q$!FzKquxj) z{%xF%`fbF%(pblaPEAD%?VhqdBO<)^aaFX`Di0}%r=igtVeaE;DyQDr$d#3DWt}H) zq3tUVv7_VD(8;UOsA6X=a$H|aUkMNRt-WA^R`p(@-^Ev=Qvt7C4kiY0;dc(R*N&!e z6Qd4smNO>sY%(7q@gK*iSegU!cyJvVU&tkzYmXx3_gA>#v$v6p$5py_Y>boIHicRZ zPp5Y-6O?wp+a)&tGOF3RjItU9g8U^1P#qUcc>@MWq&W+Xf4eFWH=jZ>Hc#YDI4MrW zbaU9fkM9VM9nM6PeqE!#6e`et^D9)aZZEsaFNaz@-%9V?SWotezeTKQGk5NrJ~f|= zIj(0Lt**=B7W}HFX>ILvLhxZ^`*9lk=v^Q2tvT;9_WeCKTVfm1`Y@I96c328mdJwG}&1IW)%4t|-7fbrZkxGUp zozgzQK6~(++Z0wMn7ijQf*Yrxx39)r!qpDbYl|Z3XuLJOn)QL}o%flQTX0BVduoE9 z*j`Ssz9)p6cqo(gb9HAs==;kH+^?W4^s{moJy#Kno=$(q=^0N%1%Xr0W|w%hVCP;g&)@`*c_-cY zq2L&GOUPmOo2#&3rXI2`n#>LCs^%IC6dQfyL{XKtC;K8<2Zc2IvX)o)E~hmN(LIoY zUjCM*wdIL);bkeb`bD*1^yC3rD|H4{8hocQGKXl7i!+;UPww~ z*kyvI5zk?BGs<_@p}$UAQdNZl^nJ@z3T|yc0^bARM}#u%R0Bxmk}|?pS81b7Db<^N zkUjf%nC1@^ptseP^nGS)r$C(=46qxSTXJds~?6-2FRQ1VZBr-KHolx!j=|s@6&J5@}def5UEJBg}&9qd);VF-#PZblS`3*!E8!~)=~?l ziD-Z8O8R~LH~VHFl-qS(2~CgEMO#b9xT_l**v^`4T3Ka)F6%#~xog+assbCDl4gh= z-WBJ*Nj>NOc|AlP*?>C6&tq)@6H)9oBYHU{9m!+}^ZP!wXvrT#$``RfKQHH_lGZ=0 z!ObMLI!lDNO{12pcFm;T!I+jTNG-E+yS)-?kck6wl$ZCqLjkrvGJKeALi|yxpkgrz0nM z_5^MAwqhF_54mjnb(k`uUul4%IyxWx6$Q?E!$n)#qC;Z0g?#{@%^kO)2Qx}(iR%@b z*;K=coR1|Q+TNVBd>*hw^XB?{^GKuTr8Tv6~5wBYM; zy6uK7ZN63@%t6Pa(al1g^RGCHi`&HBnS2zDgd9VUTY!XbzGlSrQZ>fJ;W~3)uau)68gGioU42z!F%g}l=F%Cf$YSJ z1cp3wTKsT;4LI&W4S!BT=gl4I*gzqD=%&c#&F>^6;T-DxvWNEN->28N%Dc=@cWN}R zs9`sOF~pmEp8nZ9fRs|wXi<2M;NHE-+(r|nMytkf^!KYeN_TnW5?&_kVG~2y)GflE zW3vJqWtC397G`smgVr?cHAc3=Tu{1wfuJv0nb%DNxL|#MH1UKOFQjJ$H3`y1@kWP{ zCgaXF6nUdh%HO!VnwF?&aF8`AngeM!rovX8n%m6g3jb!JZKv+Cw+|GfpMn6Pzjqco znk?->rNoOm_C8V<0I&BV-&TycL{mVnMzl7m2wYz^%|T1-sD=d;yF7tHKYed z$o3Tn+4XV<(UdOAhTjvX_x(%J8*o>!VY3#xT*6Y-Lq4=}%~C4*frm;`*0^wsKhyk{ z_2`sh0oR{?jcca!(3SNKg7@!l!**r?UDYPk{UD#VSWiXs9^YjZ_6hagoQLe6!>aVu zJAb6%R?n_DWx-n(l}2}IcG2m7{&3C`5`y|rH559DM_qlNaIZV3Q~&RVjjMfRkZsNk zYU_HQ4$cwd=^T;db^jNHVhvL1)Oa5n>ZC!(rqrU7Kg!wVu1)NTWJTTpca@vG)t>(O zWY4*-tYqiodhYJ$%iQM0@@VYGbCfS%Ml)ZQpetv0po803biMs5+BCkD&PI~RKj<`f z+q40FxLbz&#VyzyA(n>Uhc=nMqmeiE zAP>#sG~wO_c6LoI`dV|2Za?sa^XP6xXQKXaTzx9ad4CWcn3vC8)w4otCAQJ7#$QzI zvo#$Lo6Fsbm*O5DolJGxoT#e*Hm*lK21)KoX0^WTLc$ddjZ%1pELG33{okw+3Xc^W z*1L_i9FC_`HQv*Nq1ovD*{fXr%Pj)^C4E#vawpQCSx4tubHraaOVGSE7-hsgp$ES? zHLfxH!?#XD zY9+4)RB0x=J4Y4yR=#nz^gD(|GEZ^+Z48<>a-B=aQln=qmLccxDMC%}52`4eL!&M( z;;Fit(LsR}oo&^_3Jx4#ha|OF@%t;$hj#^nKUageq@7)KVy5sOgo^?j@O3FF>CC5d zw0@yeNSG(Ca7EVh?76si0jOsB9JZjXl66uVb~%;O%%xn0qK9Glj-R%S)8?x46kw0 zTfx!LB6KHa2NH4CLQ0u3?0vx{^j5Hh_QzZlBXb zMcKT?D7xpo%bX{CwBn`=4SpL#_vX)}i_#_vJvJ$bhL|*3Mal5A0w$r^OFFq>K>?D~ zwm_d|De{=v=g{ZlHncf(7ah4!N*=b&qn3-(TqHg{pws81yRdHs>^F;i?x*P>bzi=L zO_#kwMP}ckRf*E*M%D>Lz+>*$l?3jO-FEin!9hV{b~3eSBV2mrXD;=V6Acb6r~ii1 zk+Z}DPCP#Y#Z)Ns&Xji2cMIxWMB@LlzSd=2wAnRk7P=c9E6Jc>br!Y#8cvVAT8oHI zBKl~09LWfMG7T|L*e@3IS;??sTKfKwzVEE3%1|*v;;FB83{CK17cU*VDZgN7>{?1D?q&OX`&; zLLb>gp(h2x{o#Bwwj#WZBb&rotwjUu@yo7gaBCKK|L!QKdMuMpb6Jf%_C=sP=T0}V1NK&kI6A_aq9b~}f@>Ok={chT&fdn2PM)=hhR;^zMvjX& z8h!PpGuJ`Xwxd%J_NAI;O`x3qyD5!I56h84=~?dG;Hf4f~*-v#d*h%TajEh?y z@=tf-7OzX@B20qmCIvpKTf|UhA2aUL_b#+PtB-r~gQ9(+*6>?cHFvzurEz%Hb%DXY z7MJ_$FFGGI9%gqe^g(f(p0S2PjrmG;sLh$l6`3 zF?-ez?zK@6>Pbgs?dqI_Mn@?LJh4f0-a8XppnAe*Qb^(Y+!>q$C&?zraD#7 zpl_OJ^;~Tf(w8dKi`?lv_X5`7gBR@|bfnA2uAyd|8hYJ$H|=i@fbmRtF zxKFV$>fu@TYMcyj#rQG$N~8%5-sB4wY}TgF!e61Ic@A8%cvFK-bA$wl+EZsIIndS(6cM{d|$jnfmyQE~wEwB~dI+&@=W?7bTOG+s{D;JNJ z7e&#p*L~^y>vPcxe9J;{oQBJ^F~4{ec5qn(%iXh>fI>vMmAZ8|Ev)v@a& zeVTEaNFJANjIg|fM0VJrl_uL@Wuasvw@8KVF4{|%-JgqyU{vt!L={)V{m0%_v7#Rh zt0})|G5bfskK$!AH0N6os;(2x5(%C(%IW}ju6Q?k;wR3#_ujPeyYS}Ns){DAE)!Cb z=1a&VsE+d;-ouKCrc?Ybl&)4^jt=va&_Hn>la?An{~erzBJNou=cld6B+(Jg8|TsN z-p7L5&ceOxvk9nG@|nxzCrZ5IlciCBG32hn<(ymiVO04u0-d$CMG8Vsc*r|LdamR& z8)0wB8b2=LBL7-*o&8f-&4DzkURcNGZs2i&OOlY$79wz{m_)y;B~k6ules?Cndrer ze@^9IY@n-$+ewdNDXEOa^-Kzz)q(Hbff=E7yINT=z+vZ zmx3Ti^n$qVfQqRDhH z?i6Zqze^A0pG9LwV^MHq5;v4ctj})ystVGMk66w!}bI|r*CbVW{9jyl0T#zsqQD3M-)h~2Y<7bD^&eyK2 z*`P8FvXw&apZB0+uT{{Xm?o~?x|`}4Xi#aKhhom=b2Bpsx&FNZBs>jA-Qrl(x%nR3 z*R+p@u8u{j;)uN&IECfK>eC{dR=nxMBWf0FiE`c48kYq3qo%N@$V=aedIxT#XCJ(w zMZY_^(I8{CIA$IyoqCGZd)vds3O!CcD~q`CsLSZVoicWnM?1G$`a1iO+kqZ7oH2wqLgtXuKECj?r3bO!>Oj_o@?gTf)wox;lgVX!@r$ZtX#Ds- zX38f)&AL#yd($|cH76Y$yXA}D^cX;sgiQ^-ysKcvy{)8`-36_DL&@hOrf|XEIAZWn z4l33c;df6@@G}=4#nMp=h~so+a?<1%h}HRwcN#y&^?PTK^>ftWPpm}V8FgZ_LRWaa zeg>J?TgN|8VoffD8<4mkdhpH{d$MUb5{i|`;!4G(c+%Su{%{nizup{7&K%ngM<#lb z@Ud_xdsLF#oi@tcUA&RFYHr8FXUf2Z5{iG_yb6Zy2SMjoXRz>I5BO}PK(?K)0Pme< z<1FL^MH<(U+Ox(m>5L``JT(t@e{U0rudpIb(^~PR*WP&Un|FLEhkec^VO}J-R2G&vj$j+DMX)O72w1r3 zFQe<~Pa=l*;ioS$N%X~euyjwVX5ACoKhzy$U9t_g297UzZV9KN)6R zb|E$9OUU|P9#G}MEj%Xe35#}yL9^@9xW~2?jQ$fPj#EtFj5G3Z@Ph<78*(1J^1BZ7 zB_CrM<34cjl_OcZD1#q95QnV~8^PnE7Njt~pT9*m5`JB*M;1-dgPC#3`2OHHv*(l! zklX5tb!IVS`$P$Nd|D-FL@UAGy%JEZ6V_i_Sd3>3p948HJfh})4L_)<0XCz;g{hh} zj{PM@hGu4hS|u6co;DR8QcYzJoRlNxRRcg(JPm%ynolzPuYx~`(ZstCk+-i@$Y%T# z6umTn(f4kEC}%4&b~*$9u{3f%3-pPa|9wzc(*ph#a9Gsd8&{od#|@@ugshUy`0;f) zc-r9%F!XN32L`+GyTw-+Z!bUL^es!87Ul}s=;uKi4umULJCT#T|KK_27~mtJ17ph= zq7r8Xr_NYShBW5jmIyU)f5IwIePuh)ip#-vS31GtCMn`?um>jY9c*|zQ;f*;YB8G} zVxVlxM6#)856IXhPtLUq86~P;@O9tSATDtbM`%is*w){m`s`(_?I65V|6BwX>DiNi zgZA*SuL+6&21!}V8T@a>ASelZhgA=XkVE&(@SM$JgqbA^#ZJ#APv-vt-EzzEr$r>pskqvsH@9v1H8F9cgU#NaNS5umXs4)?xugN1g|(;_rrLV8y%`VM63AFbuOM@8@bjIrp`~kYEc~b#4#%!aIWJY+pq5 zEq;MVLdMZ*xgM~3$x^60`69pTMk&@^Bg#}tUd00@o8c7Ut!{nU#ZcqKKJa^r7kNpf zpvgu{GP>atP)a_BJGNBtCBL-dzj8moTi!${n)n}{&z}Kh?i9`H7^r_zK83 z`0Si_;s?{d*%E6xGx$fBEDUZDC*FkC zzLm+m)P9W*Uz!J<7E9wSaUVSQKO<6pY!6XzG=$N%wj_4pVp!Ud1s$HGlA@7UpuR4h z6p!)XyS&?A#VskK(0#-y?f61+NZ%V)wZxE)3+^!feKHd(tczJEoA8~=bWq@Fh}UHu z1_k58;6cwPzJ2yoa^c)^Y`d}qBnxX;D9bUkMw;=FJb#$10-PTWtS0iG?BKn5)5)6M z!O;4n4B6}1#UyT_U|xP6Uo^xKDsa2-caO8!>UJZJ-dMuM z=YZR>hk)Bw4fxRs3VBZQ@I$&1Df!a?CWg)?VlMss#1-mb#%L_Ou`mZde5Ob~Zs36q zpL00txH&xL_zCCe)PuGVW1?4aACKiv2g|&J$*N7tQ0HhMEnup9qTOAnH-Hl^DPKDOzGni-V(t*SwZ}_Ikifnv31D?O`M{~N0=?gsTTHY6e24X&Jg3Tt~0 z<2#eaz!h_Ca&osH)3b6RF^`Xig9qjj6Vn{%=L(_P={&q&{V>?$WlnA?R|Dl~V@y=x zcObK>mEYF39D1CMC-)L#AnsXB@URXUb?;>|rx<{{m0nPNd6u(g!B_lJMFVTn9sH#s zYf0RB5m^1S9naq92fdE2Cg;p}#Isw#-+r`+e?c~eMCr(pxo{5rsFTGs-?+&nP2v$( zTMoQ=D16rp-GNtP23~X96K1Mdl8FtjU@mtS@BEhGv^QIWAKzbzGleW*$LbB(*#9$d zI=um83VDLNtOH;h<}s=`}G=aYfgTVT@EA*{b+5>(KRC)7^@eoO7a zIhj82yn`(1y*ma{&UAvpN*-g|oP_JV{()a#3h|oAQBD|`2AJ~;Smp|j9 zC3N)v1G;idNlWSw@aC-X;@BN>t?NHNJ$WyxD1>gFske50?8nRZO2e$_!@s_E# zuzXh^^BXE-dxKs0{faV>ak2t`-(f;hFV``4$EBh9-ZreMVo$PE9^$Tg2UsXKnbhBo zChyzMV#TyLAoe+>;g;PbXy%y<{++7Ek+a2N?Xd(>TkxE(v3@#?t2YK*pzyA(d>>e| z-wa+qXG40u>hOti&eCV>)@Anb=y`LNnicGRdKf=r z?MZmHkcH!%j3acGLXSu}GHH4tIC3io7tg-~Mr}>_FR>%Ms`s9;)PO*ioCPB8v&gi` zfzbc7H|e*Dg|;SXOjL?A*_qEatTryg)5ENpwZFy5xk3%+_0=-OsHv1cYWEK>4oU)= zwn=Q&3IquRPcTNI;?QG5bSN!1C{D= zV1H)^ex2*-nH;=36KXzOLy8|;kcLQAye8(C({Qg9Bs+yHjcb1JgN-5S{JRjY&prgY zdre5iQ4gr}emU8HX&rg<&k0^$G6f&~hoERp7>V069qtJ>;9Exf!ym|x4Am&tMH zcG4XeN&m$rM>JsZ@HOl=bqK7QIgLzHd;nll3LYtX02Y3Hii>9M1JbPuWWmZX$WIL; z0XsAy^XE1``fwRM4<7^-RvP4qj~gye%LXAO1>i(PEIbu9GEF|6<_I_N8R}z6)IcXg~h8Q;CR`io$`9S|nnP33)PU zv_a~3Jy>)sf?3K*!m@wD-f6!a_^~Pok>71eC-7+)82ve){8=w zorA!$@)GlUaYRE*r#w7(>I$B~?E`U54&>-GM2x2!f!{Ia{N-~Yl&bW_M(tN|oy=@F z;s;2xhcD@wqz$t20vn`_W3vllUczQy}kA z014at6x`nF%UCK7AhAV_l!u9<2HmOssNwo3BCi^>}KMN^bXTpJ6cJMUxJAL8)Y zBqb6w^&~F5KMiEgi)X^@i-5^kC_M5cg1k?S1Fu`3;OQC~WLv^2s2%#)GvB?||3mU-47?>+mCT z1YA0x1Y4e~6Q1TU^EEcL;f(ZpxU5JETn_OEvF%>?$xdnb_~k<9hKD8iS^dfS(dKC+ zGQtNsNBWY3#*uJU$Tb`?%Lletbm8-OKj^Tz0RHUo!&#zTxL;BkC&e!!)BJVdgy3rD zg6o#hYvD>bP&^3;+{$oDaw>mbyf%E9kcJ~Ri;@QOYQC!?f=X5hr77ZyZ{kc~IJ z;M}ZPq+BhQyz_O0i(E~Jp0)_I>63#SHD{7n9qpiDohg~@y$i3I?uf6|TQR1G3O$6}Z3UDaLb-gAY5m;4drWnFNbTq`0UYgwMIbS6KK3ul~Hx`GIsA9!imd zxzVvC%5E8y@41Nk@)$UtmIjqi$bx;M@%Y4orR3DJon+Bd9isbg92gm#0@B{{FkkHw zm@?x6$g+HeedOhcZ01&I{o0J^xkQ8RB1`CdDFc@+c7k7RjL3fT#jtxq5#Qy0r?c_0 zCwR@RT=43q3ICD{f^GK*z7cmFPwG{GmsUQ)<3ir{G-1DM|Emq`x*y2Q-7p)DcdUie zW$)shTUz-Fhx;(fa3Pb!llhjK2C(bGG}6zsfoXqgah};!xN5E_FT1{_ivbeyL{Q(k^lGa#s-(5px+o_BsXT8vMnh$5+CDdDBR)^L(N!|AdkA zHN+FGb3yit6!=H@UJblF%0xI>5V1Z_SP*4_%`yU*mNRCgv%?>#SKq|DKNsNL6(@k+ za0Td&HzrGz#EI3(4DkJH9P!*03LFpV!ntzK!NoUYVBkOnHd*})=dDFdea$xF^g0Kc zR-VUYhk{{Ai67a)nZnNbu0(N@9SnLB4n(At$w!yBpncUf{MPR{_$Sqdf3WR%IH!$S zIMV}9DU~7RH=UsRwE&@B6hbaZyZ|4wbikR-*}y&KKmJ}XS*W~Rlzcy64|OF~VEz?L zVs%Xvwks|nufA!)its=Z)S(2A9b?IslZLSBu>-j`{uHPjn?(+1rIYKmEx`Fh1SpvW zz%@5FsQI)LM|t#v569*a(@#dgN?|3?t*T|LZ@%I?Ox}TMrz%;$zKq%bZOr*s^9r(g ziwQJZ*@WvlBj8`Z)7Uu29dfBwZk{JfJK5y*AO9_!-~V*N3jhrASKLGH7S95rm}KkiR9R zKy6Aet}$cqn%+EYWpTe@eQXJs@>v=h_I>0>nCOwH2|LKtAqBXnNd)Vm74UlfN|<`B z0x!3I2TXacq+!-P*vKi9Lr)Iz3rgZi^3=EZ(ojE8@>Afe#>p||!wl59RL++Qgy2<= z3f6wVj_gY@gC#HKfj|3ZK-Fs#$eU6hIEy!*6xvRPV`t-_$`={(rM3Z-zLO_+Bt+qL zFAs8MeLkc0yASuB8fV(46Kwa-AM*VW`E0lnDyr`$H)6e^Sj!qR{mNqaRyu`TaaV^y z^P`FG1w#`0UdX@RC=H#G%AKWSg9z6q2SXOEz%`ma&{j`}yfZ$9U5kGKWj!NueCsUm zP<|Hdw7t)dDANUpkq;c&FG9p}r!vkv*8-WlI)uv*#Y15-phVM{7#C@i)_ray@wGTm z5-I23o25zqJ8ciqkE!Ip=YeqhD}7k5=trETeuB84b4g$13Z}gL4`^U}@UqJD%*&3) zxOdue+}`92g4eF$XS{GEmu0-+$7k|*ea9TQ;& z2Wirp7exNt-UyN^jp6sxnz;DTM0ovyE_~(8BOhYIz@$1CvS|ApX#Zj+F>gSy#N#5M z5*PUD&C#H#MxDG_>`4A8CBhlKN3eX{BzR*<0+f0Gg$Z%Lj=K`~F`sQG6JXp9F04ub zax?OpdB`2wo2{Rtx8^h1Rv{MdbyW$)ue?}QB&ldsD zGvxVCM1$}_#cRwz-Yv$-_5@>R`3Qdx&?ATHPJ>dn2>kQ=2XMrAGeKAO<7UAd(B{*G z*FF~qpLGZEPq*zrMcbaAR0YTy&l@0I`5*t>3w2oIFbv+rI1st(q0nG#3h6j>6DaY1 zGMh)1GQURT`6}B_VOwF1&9f4N9pA#p_%;_ZxaeJjcikw+)0qqmohFjuduRE+QHy|t zt}NUpyO%$~I|W!v{sm;W4yY8-B@ z7+LaGDjU?{{kZRW3Vg|PfcG97k;h*}@Zg+iGAGgwE-l{8=$ZvVQHx#p50HUs2G*oc z(-kJHXa}hw0J2BckrkI+;as&aXOfRUT=RqmPNBr) zg(586bpltY`@$J&e?i-yKp=Ij1sCeq;p>bXxjL&8l#jo{-Ojc!Bh7;B$?ON#EBf(D z6D62fHH;JY%E7IHw!}?c3to0w$2>2y2YQ==`TjC>_`=6N<{__{|66q!^me|&1tm+M zR$C}p+&>l4eL^-X0-aP?`BT|0u&Q)Fo})Pn`cq$bfZrAzOQ&00F3E&e-ab1k;Z|$6uXv9yVvJ zFNQmNt(Finjv=#Y)n|P9MX@8`1@<>h84gH8)@zi1lJ?@;lX%ge7p=QmKKos`YiZfE0ri$TSI5`5>IPVhY#n9kp&fE@Q1}NIB+111fHG`?PNAX zR2~X`E|CKJ7r2w%=PgLeWCJ*F$3}Ah?HagnSw2&x^9Ea+6=R>PCh(K}OnB zsrLYXbD0IN9a{;Ci(UeryBSnF6iLdgb>ZUQy2R$85)swhfVZCc4UTLV&O&E-B-z~w zoH;0m&x;s9*+up+rg9S5H%lFRocMyZ_87sZy4oaet|=7Ejw6TIZQyiJEsiu2g^60H z`LFk_f$0IdM1CZI`TK7PsZ)<8O;4=gJNXy*)V#Uy(GrTA?`$GFTLNI_>X*23eFvy= zn+<=Y*Wjj8AApR)e|R5B0Jpb1=1(}cupzbe8^7t}GJNdbZswv~4!-=1M{2L^2Gd4o zgNMJ07-`SlaOTr3;CNyUR_s|%lHJz9y1=F6jpEm(#zq32g}LFOOxQtvyKGGhmktrp2yFq0L=M#6vR4|gDq}dX(L8?!_slcjiRCRz?%Bvo&ShJtmM<6Z1h; ztOBW8w++5KtwjzPzhScX*%HlScewIn2for_3cc=Hk?nb|B=glWDC2nncVth5PLZYj zq{GHAed8o@CVCWXp0*6C_9fxmAXT_GT9=&9%mFg*XOJgKap3YhMKbS#KTz5J87RvY z;75}tLSsb}n50>Y>qU0q&D&ovIU{1=W7|VUY26FvQoaG!uL~sONj13Npp>6omkf0D zEy)CN1-NcB4a;oZ4c8@{2D3+(5}OC1(5_pamkc?bToKdQL!WisIJ@P>=@SCUBu^WcB27Q|h36r8;!0qz_S-nu!G-B4ujl9OP$7?)FZ@Z$2N`h(2yK3q0>7#* z{I-Sz0YMTNry0USI6A> z`dcQDTx@Z2BpFaJa0nYFTm(~Nor$s^1HaBvBdS~$R&0EZhhDnDRpBCV-ghl>!zZfY z-s42*{wIRq{nMamdI)K9M6eVRym9YD@*-{@{{`B@C~pyiNAjd_jruX*;yxQsa}9*S zLcY1~P#w57pCPlktKgy6K1NSm39dDt2$xH20@H`;@J%)fhqQ~qC4Fab<)!=l#s7{l z{qxJfEqW7|Ittn5PuiHsqFAt?!j=qPyvDqibS5v;cH!baHF)02i1^e*L)Af5SR(v> zAAJb`DiQ(s%{oaaK5+_k1yKSYtEJoNWT%$A*(Ba;Lz@!XM0b-vD5<3^3HG9GpI(Ml9Pt^XE>z z2JWAW#bL~TCMkXzRP^`_E@*iX$=A8y(s>(L@1_EO^-Lx9CUe0?&XFjt+|R6h|B9*K zFv^^+Yr^MWOorQoo`T32Rgh|X8>cRCfu<^uOiWt?Me1Ck_q805^U;>1-1!S0=sS=e z{SbIz?PTKplZ87xmXI1LO?ZCt9p=d;b4biY`DUs=_-6!7_(qv5w3pdUzLjRee7KW@ z@Uvi(z6t5F2nFV^^`V1Am-FAPKbUVzZQ$xpGf8!fG~5?#3Oh1C;9TkX%!7B6K4K>(kvRQPf>R$a zBlq><;r57N(*9Ij=tF4XC!N{}?j(mp_iT6aZFLA(&T5lMdD|I{56QT1Z#us1>B#TZ zQ-JAP;;>+W6lgfK2k+Wii7i@tfal*ic;S*UaHeq)@u;zYuWVZIOEmzCX0728BQZ?3nvXl%COF+NiSny?{1suFIQ>fj3!3)240HZDqqPt-ltj#yYVXzjoOjji< zMPA`2+1?m+ea15DCxGQW;h-b$7qDJ13S_Oj8|DsFFp=g;WPAQ`{&!b3`1adGJlyaS zn{q9DLHQ0as;EWAtHhlWmQ94ilJ>aSW*L*2WkyhpBdlLOgzZEVq4g$jxYgc`6yS~)8UX;t4XdtcOdn8QSjE>D5AW{7iQe@BavoHpvAT?{9Ee{z|(i)Q3%#lYZ@3NSKR&|@=JnxEpHE;x`av)+{REhH<0> z#9;DER}U7UWRjXt3Fb^)ES%SPFn7*naJsY?x9#v`;Hh-jq8mw$xMuUiu7?ueP3oX* zW&n9HtOqSFsFCPSG3bBP9KQK;3{>we2G6To!Jf7C{PqXi@Md{ga-;A$c)4yCG~N+R zqI`9r;;a=w(!mE`_0WXg!uqv=FHh>XXyJLI6W}?~RUmG)8t$mv%-`-M2_?E-fXoLS z;Q4kVD0xVO@DHB{%T83|-;X6pdDKl1Fbc_~NOj;5w1w$7uS61clCe*}8+p^J)-XNL zko+wYf$@Pu;BEFRtn8A3&n>zEo)tP0f%XBAc#DrqJkJ4x4Kv`S!boua`##{kP>mcd z*a-g$=eL5kMldGtMs60G;wQ&Foc_e!=C2)Cis$s}LBFCEBur{5blB5`C2mgw55rx_ z+ZPP<$k!*!g}$J~LP@y4&7PFk*g%1&AAFE3K^_{O!9zk0?91B4a3lK;$9_DGA1I{( z)emCu%(p1gtZ&Qs@;LnP#zkShdXAk&X7jg&Kj%+aCIW+yA{_M5B?=L(;1x3g-_P~| z0}^L(^ri{WID0N!;_gL~&b2ZMbTipJY6G)RTH`5muHggc#o&$Hg>c&%e{%5MOj!GF z6t6h9g3L;J3b5T-rgWJRT=gRz>UmznPVe=JiurH+xXX?d$h2XZP8+Dzxf|q;$C2qn z!Td>L)j%aY16*CV02VHg<`3x#&+=_Pird1~Nal-XT=Vrc7~E}y|252nKHtpAB+U^=c;XF-(@*w@bXrWEPU6$*G|Q%N)&G#G9x#4egb1l*OJTuH5e9i2`t)r8VrX! zlZzfjz#&T(uF6csL$5UXySu%JmBvY+yIX@ucrp9~&ELTM?Lz*w>krU6l7j>ORbXQ& zS@_iM8Sedhy5YjqZtQE@1a$wlV871^uyRKRS-5-)47s(93^|X2mcl6@VL=6+a4H`k z+G-5@#mm9?r>S@W&BW+~E!pMv8mw6+j=NsSz~j#sfon}`neFKlVB5eIFtezSzoB^% z=%^~e250|*DN*~u<2BkOZJ8(BpHYW5$J)V7Yu;hSsR6k4ZZJR@@^EgC4b-TxCVJjA zV0AJ9N-N!n({v4?Ch0};+U?=Cut9wP!!!O!dzQv0Uia-$wE3)lWE>PyR z;F1a*u!cQguupKJhHiWGjZ3lhG}wgWTmt_WLF!)vlD_yee_Jiubl|(o^QieB#@Z(C&1Nh z6v#;rgW#DOz$aM^!b62P@ZTzOf<$5@dkkpSn`8FTRQQ9pj3g8r!h@b8xb%9ub70pH z$hY)&j^&y2Cpt_dN+K@!P^2oj(vr&DH5dm~kK~|om=4+hB^Ss#8-U!WdHl+q_wcLD zo52T_i@0=HjO^9Wgh$@|z&%!}An)Z1W^T|pW{+zAPtke!bM^gk+{h}U%oH-qE)qWH zeC|;hWkf`wD5R7U6^h6nrDP@%4Jnxoe9rmYqi8P;m4r%L14X6v{oUVR@OeBw_ug~f zOET0JWce}vVduEc1jbjkq24#% zE2jp*&Tki5U*;AAC53T;{bWhJaX_26HX*pLDu_H5>j(M`GVE3eWH_|L5RrjL2DUH7Iky2<|bg z;RWkf**^L%O%&GZL3!U=>{-$Xo~!ti5i31`XZ929DQ|nNF}c0_7|>d(PfV+K+T1amiI@I29oFV{gGw%3XUEZL9(3+@2 zoN7V^x?-#TD!=-pMh}h@#$pyatLqq$&&;_dE4N;C8V@wHCTEEfHUnC z_~o{<*exlIM?HcGo}L1%|6UeYYC96wpiR6JJQ=~AUzYHRRX?75rn;5c`T&n#CbayN-wg+g}#sQrdm6be{p&C8T_iS#zItSb%GT;n{Xxk{!$=D-yH7R=OVHG@0D-tuBOo}@EO6r{bHKz{g_g8S|Iurwoy zY+bJcFK%5x-X2PaOJto%vCI;9_NNXxVl)?q3iGhK*eCEX@hE7%rhv^K>4W^@W#mEj zP2l_OE506^gcoeM$dhi^4X5V+2R3X}5Zu3$WE^Jb(5-(rNK1CXPox;g z)I7l*RTH6nswcVoyBC{|J{Ax|KAfDQ2AvAO;2!R5+CH=ao9@4d!in~to8VvBR6=|H!*08C za9GqW{M-H?NWQTLrd&hBB7Gltu_cS#jyEJ9mMf9AcZ$&H#6g^F8bDU|Il*)}O>$<2 zJCqF!Cm?+~6mDq7Iy+3Do1zKKQE?_KIJQRjs|r!m?Eu00C9RiUO$R@2EXR^}cfc7t zwvvA7=`h7`7a0!xfxlb}0%MQ!@L1z}fx_C?V8hc}_~!dJ_*-l?7|=I|#{YJ;diyB= zv~VSkT=El$%y|h~*AHUrTW4AiTFOG-j#8W!o(A>~{KEf<<&gBDWSDug1P46}g4cJf z#T`Za1r1%6U~ro<&reN{{3$Vo_T}E!b~ zQIJ~`OWgl9;YkY%@Yl(M0$Gd(%}KmZ}l9(oH%E!x`bnd zTVlU!2s*l>2`{i|Ye{v$rx-(YFv}s{R#B48S1XN&wpCcz}wpXMk0#IM%ssf(;ZE z$)|5dxGO46;PB>K>%04HAcG{q*NtYRzkCjarcq?=#80gYEJDF6(^I(MMJBjXl18?b zTSDRbsrbr8b9k`Z7yecABVIR^Jl^GaWYJ*#^gW-#F&S4VUpQWFb)f z+L@?$r@?K)nWUy$7yn*ng^i^8!Inj)#3lJoYuCUul44hhM=wc%!H669r-uNSIvPP0 zD2wHDE@ErFC7?dppLmu&2FaJAfcE)U;N7ljEV<|z-gmK4AT`S$XYq~!`-F@5k}tuY zLr+>mdd+dsgbi?R^#+b%aE9_z6^N}c4n9gSCJrm5;0eK0Vt@1o_~foYoDKD1!B=P7 zoNO^t?41TaFP=!|?MQ;r9Z}@W=t3Bu;ZLG_cjLAGdO$nJlIVza;|{%5p!Aw85vWXp zFTcJAah;2x;wlxQL>5@%z60l`&m#gA&a!EiiFpV7}6577S_7E zLfa+#1R+yTU?ty+;A8h=aB`hB$ZR}}zc-&2%oR~04$Ysz0ux0dx?2M7EZGlQ9kuXx zr*uFBRyerv8MYqn1LENqfPR%GOw4ou+Bf{5_GVXdA#*<5;C~)C4_SZ&-(o=WE&?;{ z!-A=GzU0OW2YATHgy4mtkS(4{#xiq3%v&+??x`gaZ$Fh9gD@$fbNR$8aop^ZqI;V_KsfF7OlZ~7jZa1X!uAQ4 zf`CdD@H%Tg*g1X;7v7sn4s<4hGtm-c$)<@gdTszQxElob`gxKZ(IlSNFHvZA%8)Es zu?W_sZL)p5R10>vox>`x<#|o^B5+GeHZ1J+Av*Q`(4wG8aJaOvRW$aeZ9CT^sADg< z2WJ7V^=EKk$}_=>$wxuJoDkbNy@gogLLbhT$pX1cI(WHWW_aDfW?aErK)ejp;O)tY zV39ynaG*7qC>5vTlH%#0$wr*qj5377$&N5~W*T0bX^-PFbl{h93f_mj<}Lbou=Vnq z0p8uUlgOv3AHZmn4|rie1}s*~5;PCN-*5lJ*R6fXGUNiElxO1gn;-B@(l>+4yS`!D z{#jrs%!+s}kc0EI#d%XzRI!oqWN=J)4CjAN1=CI$<4)~ppnOgg2j#j8l>eK1U4 zm=3Olea`(@)ldZf-r`5j6j!zEn{NQ|u||A$ni9-I^3gxBLq?sUw8iq&N}_z9nk_bw%cQcK~Ur{!3n=mWd={1puFTyQ{f z1c?5#iJ%Yj;a(F5;+7;0M>DcPs;E2J(&G!ntCDcd(WT_<{6MI|ak6(aBjB{@Pl44$ zOIz?v847^hM)+uVDE=I=0DKie zWQVH}&gvcnMHYTwa83@c-PwZcc6Z^K`xIK2CzaxY9}CDA32C^;zX%`8mw?4jx`0(N z#XsKf!tQ_f;)G95Fy>$m2~f+1Q5?_tDET(7lC^>}J0_52J+|;plnIPzlqKg|PxET` zi4aQ#Px3}L1)iDUMjFHIpsChMa$x#d@Lz2dm=L4`YmF?Rv&3h7WXBTNCTCB;s3)0f z;RQ=(@X5YaLC{?7ICiK{we25xj1%rpg$37aVfg+!-o2~=JkKHmxCqr?{jCCCST)Bz z7s^9b&85&t%$8iqn+A7x%_ipee6iefF(_s1i9`4UAcFhG3{o7(i@;>C^otugQZ)~X zb4;oDz4@^9S2ecf=1*gvZOHy_?r^ZTA3Mix;mzS4#Krfb@Z_Kpd}VeTEY%Q$Yj57c z@(PHo-Khs1^JkEFi&7A!vYtGD9YER(GC*J*Y&pEC0$3bNfXH|*nLlAZ^vs<{R6TBj z#Zk#%*|?Aw9&G}z{+1!p#qA*a+69|Kk2d0!Z@z-NvJM2u`QjVAKr(pQ6-q7%BRuL4 zn1!LF!rG9?HvPprCUVSx`3syh;TYE9ZnPwh+uWHa2W!8sC5NA010M!%VE?jR@L(Ar zsy}kEnUy7$*Oi3j;YmEj)@g8OfFto+Zvjmf_h8NV1TaOS4E&zB9gH6v1?P>@@uYZt z9Pxbt_6dM+S*{Csl`x+>eT>QX`VKEIQ-Xhcxw+xbiEudYDXCd9V(i5>`)K>IeHm1gea4vTWsJf zO=DOCJqeL_fQ6qNNQ1Efk@n)j;s;ZRZmtI0nkf!F^SzEo+7d6gZ%LX{I|2W& z94`N>2{yJRVSVfUz{&73*n3YJJ`KA7ZvPAiOZts*u=fLjOJY-tr_BUd5HJmkmfpqc z9@->q>^m>iVI63YO~OW3G6}PE1{|+V1-rGBpnT$1QaR&7YpBI8f!o$gV5CwMTJ%i9 z9{2shXWe03yX+;9E}ViBdNRK`8lWY__~8wA<~?j+@C1ek+Kahs0_uzk(IX6{T0|l zwhv1O{|AakRp|#vk131<~=Tm>=m4WqI1f^4SGE@?;IqKdOf3 zmDh*8|E%PZeoyF_mxW(lHvlvTVoJ7j>7sB)ni%Iaz6=aiI04$0%CN7V8@OFU=$=t0#p1;?} zo2`^d#fv}~BfoKW|aq66yT=BPr zhn`>zH*O|vzYO5(zN`3NsUzH&t4|#J8nNs(Ie2ol1d&ts14}~AToasD;_rVx=Y>AKHu%oN(ihHX`>;+Xg|F|k~d{YfzYm?ye zhaxN4WtJlj`L9-xvYKQ6i=6PTy4$Tc$1REY?Tw^~gah~fErRMcL)br#=-+YCZ_f?#scC0umL5r5nt z1>&xL!oS_-0GqbE`2C&)e8X!1Y0lk;(6yPwuI(i_C#^vq%=r&LF_t99)}IE; zc1i)U(szP_@^IV#l2-{fOgjg5=qD1hIor8_h!y1MN^d9^RFC}wG-2ne9^TgSMn?zzSO@sgC zJ;pb0szLqxH+icIT5!`)GxlCj@XTMwv2<$%i0)GarS4+9g1?Ex`ff5W@q!&N)DVSI zx1RGZw8>)`a|QBBzYjFN=JbApGvKdL9(i~_3AD$a0GV~ON!kA2;7!JS;ITI-IzYnOBK)~gkHolUbMQwBDUynZo5sz^ zj%Ul^Bu^tkVwK^>|9Wxff+a-p*flVH!XRFfAP2pqXOR+hE5U)fg}BhB7EHIDEEw`U z2_k+*;3dT+AfsmzEO=DLV|MPuZ?202rL9x(_l-J4cZoKXs+>V=bwbI*$Kg=u&CQ%0 z*TMTEDNr;;TyTGd1XuJ z!iha+5m#z8YCf2+D-m`wTHiddJ(%I1TDl;vE;Kw7!$R!uOLbDH=bSm9PqQE+h|WGCW+DO}3Xxkez+W@NU#) zfoattf$q!zxF9%!to^fISsj4OvnJ^E$EPaprt)0O41|yQmt^gr1 zvSh%t0UuS}4*uvo#5v*X;omLhWcRc&u(#EQq*qAdTK^`@Uy=@fc1^_|=a!NwUpGMu zB^B~a+#0@}XvHJS2l3^}dEkiIBZ2paulR^VCD^O~gg4^;2egR~gKHZ#;j1c1Vqg12 zaKvmch`SaB9+Y(B%%q=!Wtv!UuEiU_`zHg>@Kj0LM+3+XO@ePjx8O&?4A^`vn}p8% z3Cv#Jz?HKzp@L8wfWLXbZR9*J*-sl^`*0WZ3)P`Dv?CK-u%-Z4LdJ`+Yu+QozxcSpkhq|_Kotk@!#OrB?`dp zMhocQ)r$4g8h{m<1J`l$9-nz%F%BEUd%j77OS$GydS_1SwuVBmIxY<=%O#U1qLaZ( zIcJj8yB6>KPZPG@9~C_66LOqPGGV#1MM~^8&%(tS%G%qJ_`=Iz%lth&{dfK_>`n%h z_wXXkQV94ycOf0G=fdraCc^M#8f1ac5*$O;=nhHr!3Z9x6kGl#Pf~;>{1|DBoBzU41 z&C{-5M$U7bTROSIV*;m;Eaml}sq8L}X>Z^i^2u%8ZSYU48_jrN&Lodl3q5F7s|JIug1MSo1Gex5w)qs>4yVophBvHm z$xkfPhd;1 z8F=|O3FyCryx+}taC6oHyz{3h^p#U4D?W1X@PHg?P?`bj^v(+eMngc#H;rg!7(#=a z;pDe&ESWO#1aMg44pvWG0F&hRrZRrnP^? zldMue|5phdzWNL35mh5cKK~~$aEOIE9czg1adE*xgWp&+KMD>{jwN5VTf>cqvdAJC zZ8*gznV95oeU{8zV%%j$3{6LxXK{DH*ggqp<~18ncGn}*{@nw;YwN*Wr72`e93V5x zANy72yC75M4s8mRE^m%uSljEMeKA}6jml3o3mK-RORpyIhA z9KRlo<%d_m?DO{I_udsSzjZG@6EP0b|CE58CshQUdlE>neH32Hab!8)AKJ_tjpL1F znUS4>E4Ia+_j%L%X2E=uiA4WNKDKZ2CuOKc(B2aOeFjv>=2MSBeUve5KWAg>*S`mZ znp%;~z3*_bv^^2+(tsWPTYa_B@OgC~{+_%XuD;-m zEv2e3aod5@|2qpjq8#99*=wNomIDdgGYtM3h-1}NanRhyoajkyf|D%!vF?wh#C6tX zP}A#=Pwt;Xip?5eEw>N6nEr_Y1ghWU8dYBkxRB^!qX>vb7?K59SCvA5=3x>W~eyi)`R9>>G{v-af3 z3pMy!Hk?Gse6iu3UX3;6C2UiEOeIy#UT|xO7m+C50Bv3fOBV`!!r!!$cg>R@X#D}m}2}z z@Os=Flyy!bJxAQhyt?VoT~?WBnk|Agq86N|gfjjVF@RsUJ7J@@JIUb)9!&H;FSvDR z0>r8A&?D;sxP873KN+gTSJ(POtWni^C2s-=e6XVhuaFd69-c_H?^*)Wn>z4Ck9qiJ z<^rOodVYeOQ&01#; zEJ73TVc2KdZIbG zU2x1hoEVB`0foRiuu^?8NlN_1cQ1R7W^@u3r41`heT>2aeDI#hy=@!YBefY{iY5)4LFV4iv!_z-!-IM z-V)}dbGzhM`*BgFIWMcNMX)qW4@)W~!?O{tL_aVdUL9XX3SZEG7Y)bk@Z%!XQ@F8e6(92bWMLFVA=J^_B!sEJ3vtrYYX)`ElI=92X3R&ZKB z*K6?Q$<=HI#AbMr)6s2ULkh=ex3c)Tr4GA$E|jCd(;Ovo+AZY-U! z6sk#>5QkSKz^pZdTu`0@52if=j+zCn)>FE9ry9z@+mjBw7c*v)kLR<=&125kQJmYC zB^=_7$XXGLTaa9yt_MFn<@Dz_i-1zD3XE-6BPK_J9)~wR0KQuvH+Ppg^GfEL!*Q%+-H-z0dHVz9+DMAH?xMa-g7* z^VxW73;&65nj{Ymu!_4A9AAcmD&a|t*IL8feKxq~VvS&`k`$@yoC1=<5HY6^PPSh} zwnWZ>&g(7 zC4mhef$+{W(&PIK&rUZdoqbIp;gK@2b)QQ%a@zL1IX=Ypx)z*Z;sZ@*2jcO>dhjfD z35h9CBptat;G>a=rxocy36A-R)7^`YAJN8(mGVJz%4E2vRTG5WRl=$u_A96`bn04xGGP2#RL4;wf*8@PY6>cy~=GSe?Uhk>|I=-0$h+-IsJY z1;vna?l!<}t{OR$*9e-ltg*wWG0}_iBWbBRa9h(ZUgO&;AxdK6_z@)j<$qce3lT6&N!(heHZDlbFBC#FXz2AIB1$ zxyu#i&h;cBzL`)*jPuGes>Z+M`hYnI!|`xC^bg!l9;hV1THh4%aouIG;x&)Q>kza) zRagP~#xH?;kz7H&ZKbWlrupQFG$Q|!=fa6g7LoP)ZUWS64Lzq9V3WhP0GCLT)rmjB z$ouJV#RDUf+o=q{YXu4pdisNx7gzGGpFr^9@@P1o5<{Nc?gqNM=Mw25Me_T)03Z7? z2WEXs=S|}+gYW*Ol93Oq;RS;vvbaPaN}BB^l-&hItk#pOuE%(f6e(}h**g4VP>7SZ zt|WWLl)#^n7%Xyi7OYt41Y-r9<^u)*6S)QC;*RrR+~EX}>^h9qty8hRxdNP=9YzM# zzXH*rXdD>pOa5Dy2TM8j)HH_cQPlguNxk*ZP1lOVW{5%QP5-cS<#aM(%R}Iy>Po8J zzT4L0If4&YWN=yFb;0v|PcT-q2~^Jb%B$jfb$7KHAWpOo+rGI86b%SIv%Lnp%s2{8 z49vp6>MOzbS~nuOb{yzw%fp_fnS!n?KRCr^5#Dj}y&x%nB3aDTfjQxoyjJzg;E%Zk z8MfFAm9i9x?UDufq~NsRPf{cBj*tfbYOKK8^I>4$83pJVxtny4J_SQZA7d>^Q(V~? z486D*pG#Z#P&Mj3K3}ejPg#BvxT&OoQwz>wpLwagZ4zd@DJGBc;n!BgK>Zld`Q?w* z&YlJ;JH88|m%amw(bz-u=A$k2hW5k+=P|f)V73~8j*+stx{ft_ z^Yti{X4Q?(j~Fnb8@w3ryN&FBoCo;pKZ*3dU?$qHU4YJfu%$*3YtX;hji~%u1n29n z&Pu$%thZ-1GePS(n!oI|aPUGg6IeEd#>acm9@9B&%L#ktlx_*V6zhbR>6+3htNDzw z${wWrK1*1gz84j&&qngYkJyWSr`TIQ%5;Z}4ArZCN7+aDtj(4IX5#mEjH-_aqVt!#<|0o=wJBys?J&`z6 zf2oOG-yTCt#|GKGdlKlmvIqQv-^b7+<%7bQG?u^Y$y27~(?PW9(@Zpaa3M41x*uKd zjzPQp7tl_b1?Xy{9@Dbf6TP(CgCdPP=-k2zYSrC|d=4(6e-E8uA6+h|x7yFJzn@MP z>R6vgx+R(D*OmX!!&4#riElp90}sXRs>MFAy~*#8x+3KAb^bvArH@(Xx3}n#wWrXY zfpyd*$QX&cAEVVnwlv4Ii;f)dL}ORd*fUEH@?U-VORvm&iuQ{s^LxHg#_K{hBR-MK zn0zq8o}Ajl{;RX14~y#=iA9}Ecl;H`>v00J?muZJ(aVmR)MZZt`{ZcknL*@#{4aB5 z&p}e`X-vJ4Gvh)M`8mJj+bY;gG}f$~NxwP5oZV@{p=iB)gQ!*18q_hHzK7F=Tn4wX zL^dqod^%V9Nw*~wgwQ>gEbW>V4x!AqpI9fi3VP|yl(tAoh8ksm36s_^GxiPf*O{NyVU%FK1anbq=#lKTb7f zW>QQjnq#Qd z;C;08c!O}o_LV3pYXx!-tl>M+Or&yin9P{>ifJ$8vT`rWp~ly8=vZwV8nV8MhIjP| zl`Ynw8QT+RRmC~_vdfH-OfP2+))%p-V=vOpTb3Y2+QokBplm8n3Kc1d+vR?#U~jKE zD2zOs0)5MIO)TVk8tvh%JEpmIt9=Y*LAZRVJcK3%$4dJNmIvrXBhw0^M#u0Jkj+xibzBIDw}7|N95R6dN<30 zj?ceG{>^7m`|TXEM(~YpKITl@4iVvrr~~Xoc^7nNWSoUoi|BcM6QNtvsY_w(iav(Ur%e@vzMDt7(Dk4##W1nc=^0cyLl0wsxmU_S?>A*mDY z?A@h2_V;tn*EZJ+Rfrv;vL4FRNUjcT346`{X)2&)Uk*|u!l&hjofxs_^HAW|y>x2s zGv@e|MbtUSim!7tj9EOT6>T3#N4oA8`OCIWqRn@*skve_oxaG45mEA{U++(8Tg~*L z3CES$;Mg9vBd?da)7^#cyh~v|?Mp&BI>A&X_Zt1|dKIm>*F-P7Uu7r9MpOG%JDNMI z1If4Nq9`R9c6*cz8kn7l0-KNW{q0?u6-$>elJ!N*(}N|{MLh_WSv#Q}#$xoF!%X@< z?>=fsC`U0j8qto+O0+eSuqRVgkU8^%otz@sw&tK3b%@@N#QeDDKI02Ji5aI-&$MWa zm6;X{wF1K>oYv$z73_ibB&d%@LLfZFq3kpu%$b2}OMgJ4cVaM*| z({N8Ywwz;H6tiS-Y96jHwGhCRKr5|D&ENm-?b;&rUcv?97gpPp4ahIsd4K z;>cX!k9hn9w(Va&b=qBnq8|4!Ay*{Yq&f?jiuDdiX+jgqyqkiqez)N_V0ZNSoC_j{ zEE(^T209*k7oA_@K*_ax?3?WUXm{r^G~yYDyfcsCt$likzpezmznjl~SaF2SeV)d+ zgkGj=?;WP9Uc>CMIWtf}*DYq-+70~IoS*bq)*R;1#vX=5JVQNerclcxIrMbkE_O## zHY$8^kxu`3mhTfOOWl7&^3^(w>GJZ8H2ks$d-aqxoq0El{&_BhGxuZ0w$PO>Q@zd3 zn|h7@JJ-+O5xJKtyiH?H|9ZgA&JSQi;`Y(<;u^l#Q(r0(HyPd9kVX}#A6-~}l)iTs zWy{}c&{gOP`zF(XZY?gN5qdl-IjGNuys|;Ma^AYC7rVM)Tg}mLR!Zg~lrI%f|Ws{)ZxD1`E)aJbNkJF4yeSVld zroRo{*~f2hqECG#%)fbE!p$Bz)MVd8zQ@K`RC4web0zRSonZQittnTbHtJJRcSianmN|+ItTgsbbxx{pTV3(|S^ynX^b+$a5j-;&EJ zHAnIfMTLgLGH9?&48=L7(wRb0{>!m;bjQUNS#O%a8eWzZe*9&?Y?0Dp9@Z_T@jfcd zG~?B5m(dY)BE^kLuF|2C+w1w~!ZMhMH796g(q=z%IkvO!~dek&a|b)53SA%+ZrxsQIOeoz~wbWFGL54XNA1SH5dTjdi=}pAgaE+9=QU-Y3~gZ=p+lU?9j$&Yi$Wph;>(TbrsHdv;NHgQ>x&`!wT{7fI& z+lJEvvn$Z5?3u_h@&PlkZcvzkjMz0LzkfoTMT7i8vn;xHM>0+I z^kqsL)!KBN?=sT^T-ZXrDQK)iihf!qiiVm?(3u@_b{!GxP=R6|W9F%ePM_ve!<-&o z?X588bKEdpwX+dbk`y$UH%|SwToR6N$)zu}9FX1#$T;%vFwbkG+b)&;rZKmA*$GuQ z*#5>BY~*$gr1xu>6?tsHw4__0jJul5glm_ne3ltXIJX_`-7QHQ;tBHFcay3|7cg>Zcg?Blg1NY8dQEo;&h?lujl zzqeK}9l?vy!%i)#dtBPCdhu!06Ig;AySQHDu@U-dWhPwDdehUkskGqA3zU9Bx~*6) zhc$mbiXIHEqn^3_Oz5gfZ8PLbsFtP-3Lj{tp`!cfy#5XJq3;dG%srmE>zLYopQTCD zvlZzc4N5m&sb|$w73i(tzw~)cBNN%AO~pGa*}KatsBEk=^Q;6>@m!d z70=7jd@{h&_=jvnbQ+Bd8G$2H+}Y!{%aDG&iJi7=1}pxxkG4FsV$BY{N0W}*BAX35 z^lGG}-Ipd~)+2Ei>hE_&Q@2H-SD}cIu*uBn+G9wp=`HM&Z;DC0lNTs7nADMd| z7loz$62gyPMj7+?I#fPo4q7sFg$dy5{!0}yb~zoz)I>d%byK`c?tYw4$v!Sy=FdyE zzX>rO=U&p>z4c5LaOC=46E-p-ly6-yMR@XNAG&r=kFAc9zT_;Ys`F$ie@+z5yLMEh(x_g1Hwg@pRz4m@=-ru zo^iW#knLnzsELdtyW+GJdTD=(fBe%dzK!xQOYJ>a!!|A_@uCRLcHV>POi!V3!x$!Y zjLV_l?}iMMt=X`5TxL?)2lTSO7X7jROQ%hE%&rO;rHdl-*^}3N7}KQd=*E*ATKi3w z%Dk~*L^{{edFMURv#kltqvw>)Qwk*~@oaW>pET7NNk)m4&Fnh=J8YVkKC(PshFn&b zBBfE?HXn~ZdaCLrJEdkae^160`u5jaD*ZzhJ-VkzZ=K6!(~nA`62~t7$&=S9{V5}~ z_`a4Ls8|mpjzZMpMEQ?PO3=-Gxi<41Qg-JnF0i*By=2xXY^9Z50rcB*E$VvW1`}no zpUS2!MLRC;L-V)pV`9!NK>KFhpr!*c=x-#KxlsF#YNgEM4U3-OSIQnhKgQOehWYvI z(&_K$@1xeFqa>G&8ota=xFyQYY;&d8Gb$LpnkrORdli{44q>uDI7&!g#a=b8XJdTQ z=y|1kRPxd%VQKgfy<%cwH!v1M8#HTh{M&5$T4W*Jy0(Vd6qCSQatcTLKhHoaYTwvO zm2>oCPB`0}w30n`dA@K~z&qMr+rUScuM3^dnvg3xKN)7$I%MNI7lqp+nm4qcZnIWp zwnwDXIou54-qLO=;j7)6S&>Y@co=9XIGe*>XIZlk}8+^p`-J@4RLrZ%d+jF5_lAf5TSbckrD|Rpv?e2@G_ouyO60S}_ z_xjt=aK|wk9;}7J#DCJ+T9S5d4vEO`>o)XA2_dcL+Ys+iD!R;fqp7F&A;o<$=!=a! zigtO%7_~Q}_j_fi-0ypAEfdT{G|WM%NtivftdnhQ)IshG=F!OPbartVmy>Gogf;d# z$NDsc(MDb|(%1e>AI@0CdWvQ+WjG3%h~7c#F7;5OOJ`7NO*wzcJryQus&<>*zE`ZR z$zjGTvzR#$rtX51z4dIG3?tDDWpDv_F)i$wLCTP%n$&Dy;lLBeoaDdBbDn+Az_94fJG0ttPS_n@@me5q?^|V2*2f3`MM|EypXn56P=Bh|4ElGXHnyh<3 zkNP(=4?6T%k*>G=-{3CWzNDU}x)LUk%Zy|{A7JgPdug%fYxY3eeYW`bE!xH@oRW1h zUC}5@O=31tWxgpYEX`o%KR?D|phGwM?V`1lOBvIR5$wz|Gdu0RdGx#D2zsFCL07pd zGp8$a_##<0RQ_QW)#%hiFV&V(pU*m!nVUuj=whTCZBMh+y=dcl9adIl3#!>%fP!R( z+3+lalqVg4rSe?vc=0dB{h2sA@oJDiYVN@b&iM#G$?m`c_Z{e=^*-A4s}aszwV#=~ zhwF{yrg51u2kHGTbvEjN1@m^`JpW#PJ~C3gh2C7gf?hBA%P2kkMlHGgslA~$P(>wS z`kJTFw+npfs|y|MzSa4tZebbPs@llBF3)4fLgvv4@{QE~ygF4&Y-3DCkJE~ElkNUJ zDPXM~CnBg_NDpKrvq48Q7>iYFQK?)$8*wWhDZkWb_9)1-O_~=^qvr|GjEtw$=lmXe zvnLx_UMfWEVsn`m|CiKcu8LjVOhcxDdBraIc#1wTkDzVWIax)^KUQDvI8xEL%1k^W zM2&KfnN!~-sA2RHW+cTB<Jg#kk!#3Lj75@twT!T+k@c*C$mYXrV*er+kvn#1WV@;`Ve%cS*k;6@ z3MxPm`9`#Vrjp(J>H*f=^)FhXauW6YnZVXB6;OI9ieI7^PG`{#)K}>!tsrHr!pLv- zx2G-IWN6LTE!@kE&v#c0Nid6df^qmO|Kdd1B> zjN?3zy3M4vri~l;lb&tka)dm1RYk`b&)ZYlhJ-6n6|yHrsgxU;pPB#Pr`m*HxS8Q!&N1nO;YzlmlT4 z6=t9pC7S3(VlZ8Eu%9#Z)uxrR61fa@gq%I?X++u%I#IoW&KcGcHfrrbgZuWO3k#K4 zz3gnXBsh%;-E#}|PwQqzq<8Vng{tg1&jREq9);eJnJ8S*nu7Zk?CO8T{M28?$VTcA z%DT57UA6zhY+5nP?tuM#ew=R|a@*LBKBs+PQ`=@!pHmd|*-$F?=qb`a*TNj=(m~H9 z43YO&F6+~1m{~4+klAIPflSiuP_BkF;~QRuoGuj7>5I=YUR#c{Zpnpgfm0g7e!9X2 z;cYbj{1F-$DPmK@DAjnd4?Q}bfPRFJur3~1w8}sc^*DsGrb7zC@)Om}-FSv|)49yF zhdQC=%}q>@tr|ju9_W;TAD5$SO%LtXLFsd^2=9PNDCJcqjaGe5jYeLQ>gy)to`U>I)S=fcR+(Za~XdPb7b-B0N-QcDq+FsB<7OsLVB|!pA9EL)T8r1v~yV$ zo%%imZQL}-56g+8H~!clG=!Nx(I#3HHHB8=5Jv8dydD0h-6rPmfI_|nGi>w?WSO;! z9vUj8ZXf^9gC3>GWaA#@$nsqJ?4JpG)uBW`ly0CuO*|QwKOfmFBRg6cox){0a0Cll?&{gm78bY`OYXL8u@-Ktb0t(smO_Mpplz7-zgccYFs66{5f znW%fOF_PtK5qF6-jKPh!)Oq(BZZC0`e;*Cg;{8=9aQgsVdpeKSykpS?{d**IRx$fS za|sHFOkzLe;X z7z1BjFNh}Iz5&PU?t%WLJkZ!Y8Ei`2;c3$X+}99+S`CzVDb~X+i?NWr`~l1?ro7PH zK>T{mAMFdriKhy$K$iO#UgdxR*hOiO94P_atyn=E4EBJY)l-B=k!UG*IU z_w_*evSfI6Y&75G%Xv5$cZSRwJq>2aAEnvbPNK{MKKN%UiB*4|LNB@yjRf8F=1>uS z=wzu|S~hLVp20u)XbR43`UFc3&ZYbFL@*}E2()~>vG{o~PAs2;ts_Nf;+O}%sw=71 zQ7L}Yc@t=%dHABU9ZY)nlel{=^mZ8onl`DzJJUMp%7HQrS}=zywWf=BaTn;et)|rT zPd{eFCSu&TD5~S2fL-2}IKJWu6b+lgiQ+#Zck_I>JH3)rl)NAj_s5~Bg)P_^4wCKB zx6t#ZEQGE1=RIUB_#w@Ed9l&WIDe%X4e7TK4_@3$UI)pDjmv6zRh=T5@ArTd5J}JP zHA8yf+XNcnoJNSU9XdIkMMo%-tiLgo=&vL?yEW)r`?u)%>Md2T*a5Hp{uXT-R{-NW z-|)2b7v5rV|qfN$c^fHn*2=u_hb z;CzpRc{vxTS=w=Y8gDFWu0Kz&o%07zKVQ5hpGx~Swt%c@2vF|?(s?HYmQSak<6#UB zeqMonuLxETeu2`hBkMa+gy29t~bao69Er#Ycxs53nVkH;*f{v*$V$HXc; zQTccG-Lk+(`?X7X(?HUCs2eYWl@mY}aa|eAn zz650-94GyPAG{5x_t1|~zp31DB~(~>9&{cx(yhNW`Sly3c+2cQl7iUL;>sdG6W4mM zv>6X`&mP3b-VR`vZH~<+Ges>{P9$$pELVS^8bkg>q4N5p&>ErucJw54)CAFo_QAq= zcP3Nr{v=pA{RcVnDFbGnbOD9=6UYze0L=XqNvChGfU9yD^xQ}*7*aZk>IZ!xEbltc zk@pLv+sEQxEe=ePB}ljC&r7$&0A=oY=nX0 z&8%TvFW)u-dj&>qZi;onL052Cr6zA zNy8SeNZ>~n;%D0mh+AeWUb#9J60E{OE%P!M8%9D-i8XO;+l>o8sq&SdPsKmCpNNi+ zc@0~OYKYF_RIIi>OsawxV(7k8V1CO(bh|x*99+^*{cm65ZH_*L2R`@m66Z+qpPNq< zdFicy^QCbl;G!Yl?fet^FW-jCS8angMI-oo%9Tjg%U!tV`4mW~GQ+c`N9kRgh4@n? znuNaJg7;<5L!yqmpfF|@X0Mt-E`5;9$&+6LA=x2VrJ0QVJWV{4#$XYr2&FGXL*>OH zY$(bRRt}vfQ){f?-0WC5ZSt2y^_##swI0&+dZairs|*z8y5e8|IPUlUg*4Q=4DsL; z9MK$14m)&E&2iglbw?uUHr`9~y4-P5Mgy!%9?#LJJb_wO?ijuGAvfxf3T`pjO%Kk| z15x8hln ztqH)-?F*sb{RdsRHyYnbJK%{jN>lEsfKC}pZu3mh@FCKMO%?FDqKW(Y?l~Iv?j#AQ zFC&V(7l6D=FjQU$f#Y3MMYG25p>t2F(~B?0@}r=eEUelI8^(LV(BD~5KK?!F@@ysd zRVC+19ZRooY6rFFna~#8iN#%A)W1FuEOURNo%c0JD^&yW(TgbE`h)jSQ3Llr-4Fe` zW4L4YsKW0)PjUM%L(on)6Wx=&DB<#^5T&>am@MuVg`PD<-}GFZthO8Umn6v-$UbesNf{Mhd8|I3w-8;V|I5MJbJjw(MmgwbUb)RvXnG{GjszDj&8MFgU%cRwDLrVx&bo)~CE1EFSS4^EHrqu!4PUxOK zgX-#+k+l-uJzL!sdxFAnaJn-_*q(rQKL{M{wuZn8Bedq#VDQdeWbGv>es+2X4LUR# zoRYtYq+e+Bk5BgEyKdA)g|ETN8V-5~?m$uD5WOLN zm?rd`pc*IM!9R;)G+VG1UM8NCXcKMZ#N_$(aHKIPIhWu9>&fKzay9XC<0#ZMOorYM zgS2bjN{BcwgN~0oMd^->G}3h(4jb&o#bx_==2tJ`n9a%B~p9VSO z?2jwx_fMBluP_Pk^|Qn|YysA}cEQxNT)L^_1pWxPi%r#O=sCKJYFB9T4QI}PaSHJm zv(p!SXX?WP|3uv9e;%~Q{3hYa>*<((Z6c>Rx9N_OWYm6LPL3HA-~ppEuvGdePRXo+ ze>42BGFBOq_@VSU9 zn{w7w<3$4-p0$Sp=I|cC`e<#M^&^8mjg3LI+o3Qj3wc>18pwM~0adMefeyE$vG46P zY_4xWGtN5fZttL3uOC9t9e?NuwIhLL-q829Ml|W{VtBCZD!Q+~Kw^aFNIv%}(N@vH z2iq(#F~}VM*zSSd6&xDQyGA`a67k8n75s=Qi4NQ_ieKJq19koOIH6b@-|QSlx56Gc zeYF6Lla@ijc1jM4#kl^|PoB^8FObuI8&VaL$kpvVSTbBsE-$UXMMk4hB$@@gRmQ=w zWk<;3-YKx>)^y-}&uifa@qSb@oGmh%776_> zLR4@J74ciO`8$fIz~X(<;`PP;WO|SZo%^#CZeLQsKhjlnx`qt3J0(WHCk`0uIFheC zRTUjt#sRA)#jms;MXf&B;ka=RcujU;G$XtcT37xQyxe^qWzBl&giswwz*1+VzX@5??_~oR;5~E~d?%5M$J3lT z6Rp-Emcp8975K`R5{CNBv7WF5w$4|^g1-%H>f@fH3ae+;h1y(EjQ1#r*b zfhhDWq`T5Gc~ zG7Zy;4uE&MJbz8nNmM?x5Gyn9^Il1JlFE5Q;CpEwhP{rUiMr;*Od%TQta66wGzJ1L zyuo2Ff9%-u53cr^V?neQ-zQ=-#IKAMrT*B=-XB>{>>ghvYtoas3knFF{uV^!Ng7%j zrDI7-1*BXCj1}Ie?OJ9M4l@H3@jB5IJtn@JvJs8OLGbxZj6F9~^!VI1`g-I;a#Z#( zZmCY=7T(MvS61p{()uhi?PC-j_%oY+E!E?X)KG^*o&B`mehoe~h(?|INvPbI2GeIu zKxW(n%nxfPArB~O>Fy+`@rCBJ^x*|}o5jtBe0daW7~Gj1X6X6z^=F2U?U6{O<(H=>djMe=W{h4Px1_8-A_VxzUECm z?toG4_rd&NIo_Bw0Q=g^;e+K*{P8>*f6TLmqMM_|#|~RVu1^(?tFs^%dvWM81xOu@vndAq4SMak=>)y$lelZU}WUQYwG8M^T9}rwk@IK-gSw3 z^rUgJ&qeS{)8)q>^@r_sp;W0ZnkJCcP-@}>@`PHb*2&KnO!fD);`R&GUv(!MX`T6t@s5T@LS?-)!2_ zCv9AplnE$P%|!Aas;r6%6D`3v3H z`I!kx`3eu>#AKFICO8MhLPd8Xwe*dkvwd?QeN++pHEqGgKQwtkcuiC?FG09se>o|B zdxmV#8HK}*XJJ6Y3w6~4ku|r6U(5eP68;ip^2X9sQ7P}?U@=ZMZid}b6F}`$A(-bB zgL#q|e24eJYMUY|owkN6x_^RN?5L(UTvqc24wk}Uzsa!sjTCh2_wkmB&vJJaGT7=B z4haq)h|1V-u$wr7f2txIZ^lP}=F-d9xKx9_epQB5HRf2H(?Ok$yT~5|d2uXa12-%3sNkmRaQ{!Bs5!Ap3Dv{3yw5?{x@9NgPuLwEeGz%y43 z@rAcK=`~nE%bKqevAGwqI9n-P(sz`$b)>_TJ3o8%(bTE8LotuH`>UgMxnRT4^d{_4H~GCO?Qpp$XzrLot2ldkmjkJ;hU)^BRj1 zs-R=tX`C&Lhy44IG*s^?De1U{7X!`%GvEQo=Fi6w`}|-}bv0NI7vt$yS9q(>or6d_ zHDcP_%9FdFg!DrTF=aEMe9L%{E6+gvSEI!ZhV|@aC3Ae4vKLo8(kATzOQmd;vxZu}yC_9t~0=Gi=Jz@>UORFLJKH|Bwokc^pU?jy>5^u_F z!Wl7H+_2`V=sxEQPINs&x5O;QCEH3wWnz$tG?{U?u;l>=~l#x|msErho4Yf#D^MZQjb zi+^@j!jTzvP+A-Wq*g?(E}jQBY-(}8;1VvJw2@{zm!gxQH=ObLNZy}}Kp!PHu!-y- z;?5%^wcr`qpg9umXO4$?UZ=PlbhaaZn?7GW^8>0}^TYn9V#u6!6Q-AsKqI3K=%*z@ zr-Rj4yX^_@oNPXLW~AVmhO^N6;VSNkSuYyDN#cRqDk0+^i*Uq|O2XQzBJB7~&)Wxq zho2K(k}E{F>;Opxg#!QP@_rhz$%4LAaKz)MJzs$57N_#1IyUJRWbqb^>S zK;Ue_9IE}fS+v#v3bhW==D)EBhhz2}RA@rFjTer2+ozz^(X& z55$rH;)!L(bgXO*SsWB8%D4I>y1PFXH$`OQn`>dz?oc?)ex^&^&Q6D)tJ3I5nH`|= ztq>J75}W_S74ue-1^?{_xi}jT9Eru1Tnk88?Tk}5=u-Zu z?PSf;Meym!TI_jL2NUA2piR6huB>S#n=(xyVeB3>cv(xsCrI(#n`?0D=+W5Ns?V?M z7$gg0Y+(5=$;7=a@i@v&qDemQ!>UyY5cZ>jZ0B#lQ^U*g-oym*ZMPL2Q8I>}Fx@6{ zYF3iG>*c~g=}=s8N)vBfwxxe7PUFwKG}KlV;+!ro7~KB`c7ZaSS2)c(oV$f?9`ls= zEH~#Qv9;L!#Po46PXfkWq}y11n%=`6jfu zd;;#zy^fimda2RiIjU{@nuOif60dg>;GqM05SMuYhedCxg3EJszoSprnLBeoNbYk< zWqHKm&>PAe%)nSlsa#p~6H&L_RA{eShlXR<;7zM36q`kc-O;c(U#Z>QhnnF{&s7k z3LRQ_ar`!n+gSlmBzskar>S`F0()rRkpv+I&QMgtQYp<&QmYpT{DWLn{yRimuCKuA zzst~bxeV@KBJnPCE<{r=AC$j+or<^Q!=q8ocwjgl2c0UR)9IwpU3n#4qu~ZGce)X2 zszcu`ze9qni$zWS{_t;I63*N)M)DnB1H0p6V2|f!T*Nz#v%J<2TX{=rev3=J0~1iF zwHV*qABVAqK^WU=1^u2iXm#=vibmbXX?=$<>qZV7yfz)1V#A5*rg}1W!#>EdoD1`# z9+5EpODLP(NSeP|aG`0+OZ<{4wHi8}nY<-s4tbbK%_6Q&GlLG`pT zbkyr##MOQ~zK9n>{mT%DOE!RW)D_fcp9Ie+UGWa_1XvVjAiUeRUgC?E7B@CHLSFVE z@QIXYTN_fi;YarpxA|pIe5#2S#XRMOY@{Gs*$nZp+hyZdp&K^1T`)JzT z%TTMmoO;{;ApxyrcyOvZ{-*uB=*!>1$8RQ@%=e%VuNL7e{S){uMi;s|+)=r&05Mvc zKe9Rk_pGplx11;BWut_*d-e~qNByGC0ZtNqYyh9njiLe2h>>UNsQdH+TK1>}Kj){y z3irFzd%7|1zwd@~wT|M`F*f9Jrw1NovuOOQcDnV?A`)X2Npx?&7b%pp*mr0?42-Ar zKjB-HQ!#<$$RJG5^}+KWcHl&b=OBQ5zzu81fyc;dvZBinnkMB#rj8Gs5Sv1_u?;_{H%U&X?(qu} z^5qcr>{o-rE@C5ie zcMSiGFtF>lE+aE|JQO*zkC{-H!TY%0TV zd@m2Ns~$stO)LI=Cm<1P)cNw4R?x}1o@8oRGhtP?Q11sT>4*O`VbswiFyCQ_-J1;I zv;@=7>fpj_gPHKPLk}f^ai}>NOVt-Qb5DgEzStj*oYs@zP$5s1lHcQmX)dD0Bhx{4dYLF? ziX42|+(f)bgfVTaGr^}Wmj;|Dg+{3`WJ?dy|F)`<-WY)EyMDubzXaM*9tYpv92Kg| zhlt*PmB!}6JK$*-N4h@t)6@%PI2?F~CM+`HfBSugs^2KX1>B9K=lfp>l;HV;M{{Y+ zyD}L6?IN!3OCSe5u8?Fa4e0W*0mOCOaeT}?d@{Y>?TukEARY?32bE12=RHR)x7Wd8wgLOUocqP?S`78T7c!nJt zg=y`s*!@wmuI?R&!`=Dh?!sm=;)0n-;BXmNYmP+iWra8_PeI#F5A&o-$e+AcRPHzr zyXL)?oN3)eEiDqJztaSVZGeF%%7~${TEnrW;gEtFcjZqHS*=Pa5Hj}7OyP77_ee`|6B-ol|1DUroCXt-{sY>@czO7YvTI!GB9s_%9oLX}b4m zcrw!u1|O$``ndt>+A{)*i>}h(MUKS0$_NjPnIOu{>f}X?`vGsPrN!0n`^buPF82Q5 zVN>KyQGCHN82)Qa-b`2k8`4!_h2wO-K^TMCI_k)pS%B~dWUM@#pxFGsa^%D`LC$<948va=)$3!;ZT0hn=Uj+rZ#%n^sHL|=yuxE z^m18jF+PlGzcS(LM;>N*79rAq^l0EC(NMMtdgr;~>n$=kbJ<4fM4L&<&MO!ZB&0T4 zzC7hwk@RH$G0g4>;<+8jB8D6Dam>uMxYSb_T{FLuu~vM@2(N|ez6{Yc8w(J~4HMHj zAGlgsojA~&M>le2W4B%s?LVN4ygk3^rN9RIQ7ps%Tx)`t&i|oSe&b+Toe6H*R!)f1 z22#3vB}R&JaY~33{&P{|O<0!+Y;Y;=9+w4B?1M z*lkHM7$>Cg7P+QkNB9Eja6cJGnWiIWOfRK=9PAk7A_~h?z)fEr$fD|U>N+TerOXKN z&)~g;HG4)f>}4UFV^2M5+Nn>u81ec8uByjYGW}dG-jccuS%MYNVXy-5{7GE7=OQVt z>;mbx8VHO8(|BS2Q1{ zMCy_$dryn_{zSq>A4l^^M@ZgyGgOk``;Oz0xUR4gE^XX_`zP2*zRg5ZGHp5C#P=gU zL+YsKI+wH?Oai0e*Sy{vX1GY=Crs{6L9(_8qg{JxK-4hNC@#gvHxIy#IlqXW`Vstm zRTj4@&!=-_(vc01=c$i!!e@`od2Rs`KJlX$Y~CD$J?&LwyH5;GvV9ERRy`zjbSqrA zx&u;n@8CH-T8rP^S+pr{kmP-faGdoin6zgtToOORphJ-;s_+CqQ8CPRiAB!EPh{Y} zzPK*x5nbND4Zj6^qOwQ1X!qYAUTB~X$X@Lt%Z+dI9?SAY?=P&UT2r3#a&Gz{xw8zm zj6P5QOg;zMIn@xo&zj7=@(=F${-T|b0sr-O(N3M)G*HbPd$M=JqxHX`V)jANtvz?i zr$3gE;GRVnPO-!S_5UQ8GZj^4*Wf6-5+Z)D!k6(FhaR_A;oBQ(pzv)qtXR4cE-Xj_ zqd_Zlus==1-x`2kLO7`jsuD@9TuG~5BMkjf5MOx~Mb##%P$=*qjvPfyc6^@oU}h3FLehxax28ELXP2qRzLhbU8dSlh${sVqhQ@Yilp zq4rMvR?;lVd5prKVMdXxS|7BcBVu0pG*{Iv{C+K127LM zf>^I-SSaybC5b+=Y_b7NVXa&-rme9n(RM=e= z39r)T;Gm%#{|)H$qG4hn3?~ooT7s)WGc3Ae0tUPM=%aSnlB_qAd11RaCyWlwl9H|D~`doQt*>LB~(4O)bx!V%Ri5b@a`r_Pupx|Svf zYFbB7spTk!hGwAhH4W-?;*ZEha~X_tsDrNGs*vDk3`<1WM9zB-9p9x z3s>+Zf<{9XpWeK|T<-l#uIZWJ{3j!w&v|B9)8?PuV(NhvnG7KehHN)W1 z_K4&%v8d;?h_~tT7d+Kn$qT-J8(R0SfSA%qRDAq{uE8!~o7vg3%jVU_t!>)E*&#*EjTG)ZR8&{&_!qFI$7HW&hC=hu`5k?~~+!Ht$b zEa2-Z$%(#ffd125gM-(yY1rFzT(lvU{8m+_%e*{z{OqMTd%#MpKUmET0#E+oKeIt^ zyO798tK#OFUhuqjBhcy4QtRQrc4!-*3!=}KW5eXzWU*63%DprRbEcta=1 zojHQzpG)vU&Nzv09l%V3V*7LxvdDN8p5zC?*MJ!i+!YGu{OzbxD9xXz^$mygo{%-M zW$@QJ3L0l0B-6V<0C$#!~6*y}@ zleyb!Xz$}r(A{eW1#ew(Pp}aP_SQ+TcszQAo})OhM%Ecihn1ii`0z@ z=;}Z3F$QY&>(dTdbH001KtMoF$3B%R47+VTP&XwBc~X`gr!k~Q6(r=NyXd8ba7}QCA|kqd76{ma5{}c)-(`SdnDi@ zzh-Q=G!(^?JSZ7x6Pc+jAc8q|Fi+1Mx-G_{^PI;hVk-%=XbZ5PD(Ged0NtQFG`plv z7_T8i7A?I8k&@i8oODx{LuQxBDaD(#_Hoj z@LYL=rcZOmvU|1YW4aB;K03q0koNaX6n;)`{$B)2vh zOy)R4@`oe1^Np_9X`Lzz)lVf*a1f6M4uQ-2yLe~kOyIqWz*pjzcyT62q`oi@9M8*( zE#wXXKQCqvGiB>mz{TwD|h(LN{0Ot%tD z=sJQ9=bMN{P@iZ`#x;CrsfaggSHmO6T%t7)hX+)qLw(>g_-0-Y#;uZ~KG1Sk0e*hGKxdguzJugj*OeMFnKEPJ`0uFsP8vhAEd`qO`mbO1W+n9-3N5-xq0-1T|S&w;-p zI3Is}jV0$2nn`X^IzE$Nepv1R>%8XBttpC_*dIhpx9Nia4nIb3(BvOEQXH=`wc)n>Ya z$fCxTT!=St0n;6;QTMnK|4_d-IGnwJeHOQH`MF;b?f46?Xo0*~{=-pvs%0eE8`1;o z8ct&2(kDb!)rY!E?r%0#eI)6JB09dkgExH2CE1=^*qHH^{+wusZtTTncZ~Q=?g@1Iky@&L z;5#`!Cm9U4W`IH7WLP4*h44Pkga7UcVaDbW{2#(2_;Gv`@;)phwMX7T;)-mj`)Chqe?P`}{s>H1p9kUl zy6KcKLvenO9Dn#u3w857ircnE;_t#w@Pa0g5mD(F*eu~v)mG3#DS!I!b{K3ps{o@W zd!tLcD(^F+kD^Og@w@pZm{d9i?`4$XzpQ6ie`h^kMgIvNO!8yhuUQ3pi1}%=xNx>oi)!JbeS)3q3aPgac)A3;%acuP5?&B zi#$wSPW`Pc;L_Y|_`B^0zNBvia z=f8^C1K}HYfCQk@J#G~BpMEBf95c8A!{2zs@(MLRy$P7-^GQ==ARhhxMzq)A5?!RD z#s5{Bftl$_v{P9DVg_`e#BwQ`Xz!wWw^tEMN%rAY&PdW(d>7Ujw~+Y9**L~`3@R)r zhJVu}^Ac<{@Ks$}jmZ z2J?3Fz<78b0N#-Ve&q1K{D-*dTP`)oqeR&<3bt3=BFhgei&|gT3fH~P!l!@#kf*aI z5?e!E2;>#xYLz@{a$_z&e0T#ws`A0|h!Wmx91ncS-Il*~v?LF|2;;w>Ax_GNMLT}k zNV2sXVSmwhlr05(;j$ds6K3(|%LS2=FG4cNF`-lYE5W(?F^LM;g2xB*CG+Aw3%TVo ze3Sh9A` zorF)9Rx!1O@7RwGDIE9kWWj^A-Ry=#)0mZ_qikvQMOKfW&R%%;9}_ckwD6_;AY0n^ zh%wP`WBGe+7|->N!Y0K#Oo5!7P@h-B*<3e@o8hC$u`$jPbl&M=9(7oA?acPGarJy= zVke*dX8(v;pORM{XPe5n@7ya$opqi0m{7?C{aVaUT5^|p+7`v|Rk*^n)ibyUMJ7Vs zT?;tJI8p3az2vIx;c2YHu5f0pYrDWYSC_G0_=q!QSVwR*<3FZ)!hdYSdqwWi@3d;} zn|U0oUvZA#cB~c9(jcbZ`Ze2U|4wl0*ENCcvwT5hn=@zsrvUas-X})5>4!iEvYA7v zDni+rLyUZKIva4j!?AwfcR}Oe8pqE8^4tO4%k00(cZ{&6PT=?|f^*uugt?nCg%LDA zXNqSlGTL#c*zHjb%us=maM_qlX3_;|?w&QO!ii&Lxwf0D*xnO4%z2G8_SumbW@@?t zXTPMw`$$#|^Jvy*_GI2y!J)C21Pdosvi89)Y?IVl=6fx&-K!!v`{VAgh0BuI^Z9Y? zhGaEn)=w*T!;dCGifuWQF6n*TVwJ@x%l%|_uNcq%T>6MPGx&|odB5OKaEl1VMt;#9YnpwHOgypp92@C5As}gWGn=h@%b-_;d z@T#xOl#s7%OZ|4%cF36t$O&Ni54SPFB{JMW*AML8U7MNI>t_TVTQ4zh@DAGUL_v zOYEQ08yr_#Cw4)(CVSE^n|bv3Gh3S*$Jn$iV}+)tn43O)&Vc5ACRk@L1D0!<)h(}> zv&F%z-?=Nyk;EU2z4RSHYK1)Ka*!0W$HJ<+6I=@4Bf~Hh~^8nO-N&IZ~w&+)J$bFjVCj)!TSY! z?D_;3CeCAg;%~94Z~tSa6xOrm37eUdrySWd8%^f#QDb(>Rtt7eg%MOg8ZE5qjdmPA zr&Un!*r__9M8H-&-OLy(Ph+QYi;HCt1IR8iGp&6C|CQ z*Vr3*u}trhB+e-#YhmKVJf^2mpZoBB7^~VC&c2d6%CMm-%)<~ zT~U+RNoHSJ)%p>fmU1_?ZO#jUg`keP7XFvz|9!~`2^6zpM`xk!S2gbB)=!*Q`N=aT zsZAEjg*I{i+kB3lXfEj{UhtN2Pwl8)P&~wWpqRy4K8<1SANa@W@M9RK$8k)=RvYe@ zWzFm=TNlS;UvDx&FJlCvNvz}Db}2R@ES|w(Fi3d)1#2RzB9N{=2t`!(1L_VU-5EmG(1Aw!kqTD`xgL z*S``_wiKDQj7zV+n(<~|S{if?6{lKj{N_70OZj?0eozh14h z&6stqlIH$C`G>tHn8$Tieb0QkJDF=Ldxh!07R^o%yUM(gs5+)YH#i;V>X_C@CE>mt zj__!_I|~OHrei@F=kcX??C~#`1S;2bm`G_p6R-Vj$o zVNxxJ8!@<#neaJ=Gvo9)#-Lj)*ii77wQCAzrYdCgg>#NeX-T$o*9C9yOr2(6>VY9 z%@Q`!(SSmANDK#fpf^bn;8uJ1=IZvN` ztlH^X$n1Ibg5%t}O|U)pDQC>K(Ly=<_nfaQpR?%yN$|3ZWvg4e*n?HQf?)OrN6og1 z9d|mOu`cdsK1W0e8vk{(ON&1{w&ibScV-oES|m)HICC_kV5`V2&*+(vC^ixn8YywR z%D*!EPI?L7_-R#d{V$WTj(@|}sT>t(&YdLbgjvRp&zj6#{>hUu&78?y@J~ya)0iQc z)-d2WvF8{Yu)CQt7sd;gwXGJsTPelb+ZM6=A3L(MX_jD`@;9cjK#u$9yPBkT_8Yrp zN|6N^mVp0Wjbrocoi3a>v4?q=s=?JN=W*8UJJ0%labd>~{b5hU z=FYgc=K%Ao)0w@sq**e}I)$<-&uLc?43T%9a4Zr`;<+-Fa#nB3j|%=k86VcT~j?!o{M zp_%k%?(A8V*|p{q)9j|hE#VJycE;@!TH81XN4HEAR{LZKzP7I7YD}Ia^tGGBeNpEt z+?rOy(1V8=5RBv6DbC}}jGw|ix(kHmwgz17I~#;sq9pxWYD?{;oq%x&O zAguC{<7PzbIesSo!iv?-%+i)ALd`xqZtTV=#;WKtGj~lByXTM{w?Z+L^**+Ot1uRX zU!Iq-^3ODca|LRVVZj$_NY7Ee5wUTzgl`v{|Ju81$PB;)?&b6|PV>RbWaUa?m3vUKhunRZ8W5o)` zS(D?#O#2cdy!|7M9kG78ur{nnGCVCzm}{#c^pW%p9a?pZz3VcMd&}S}yFGR(S7Ttd z@I{XV?=3cv{688(tz9l$-^otGYtOa0#ugUBOHP})BaX>%=U*veEiOG}tQ9JOVI+Gz)QUyq7&sdmSznDAjgXH%ZD+^PjCvg=QX$cDp2-h&! zf@#+?;kH>^VWz!{W6D1)#Wz-z!a~Jhxvz?0{F(YS5X80>fasMj$3LPBAb2ZJ3 zgm2sBxR$ln!u8x*rg$GRv*h%->pL4+Wvv?ax1<|{mlId*Rm>Oagov2CZ&kS8wTcF(|p4AUo%|Ua?mF?7 zw$r$5xF^%wZi?$nXVE9NTky$iQ+j{w1{$y55C83%Pd9tX@rfapqI?!LJ}6<&mlhs{ zn};r=cHuWT6O#^#p|Kd|5hZL_DC58p_3e(VDiF-Nr!IDYUiuE;c8f#MZRYn2@v>W!1IGyyvR?-n+RtPeYYH z3ZBB^?+akKm@@xoLNyy+!yfR)G&cxdl9xg+ycX&jp46I zwIE>cS%HIW2{XO2pX|{)fTPY@Q@z%FeDe4t%r8>{_0n+=cTbmXsTIA3$zHss{D+AD z@WA~>N>q7v7EzyJN!xUQzx?A!pE}+^wMBDiUQZuBcAiVMc5fmIXZxVmE1w;&9)}K( zGnx9rR@Spqi5rZ`!AnCWLHXebesHJ?4OzK>&wKotd4;WmMR{U$w6+YNuYH(B>CWa4 z_Pf%8CA)ZUsTwx=JS1wk2lfnEg5$0`f@|O@5-x9p32Zq1qL70HybLeI7}0SL&ACD( zVAVQvl#>cVx6m)RXYEdWbVZJO*J$8$+Xv7gGlS8Pmt<7vS=>F(37ohAUG+Q#9Xjk# zUGXFszh4Vg&FyT{?QE1j5hRHD_=bIwbEedO5!ZjYj27NnPZsB>b9s3g^m3XHgGrar zX#8=^zH3Tr>l}HWS$4(L)jipurJt>2%W5<6mGmHaVc@|J%$KKb z(g$&G>rB!!tPyrC$Vc}Vl>TTl!G*=X6@>p24%5-5lAW@cw$_Gv%-e@%eGBOM*#Fo| z16S}cwdcvH$(Y;d$KJ$v!|t9v6%pGf(V6+1$;rDj!E9fR_l}@8sSv2TC z0c@{*gG(BcA+p8;KV5NyqgGF0vCBA~)oo9s7Jou7%`e2Q&d7Oe@jgtv9E2yrN22}R zH4vSjLf(tb7t}S@!PzgP`SDh98j@$gdw;*h2(htrR97}`-m6V{UmFImG^B}EOPTXY z1v)aU9l z_v7^63*E@oAT<_Rrqd1=X7P+ zaJ7N?czwfrTy7I1xZ&!~$EE1P#Hk#US1Z!`rH|1z;4PN;#VUL z$?cx$+&cRNWTh0c+UviGaq=qWD$0smbioQePnpwDWieiLNS0=7xd!`X?_j#+Lp)PB ziYLt%;~N%j2ercc?1B9oviVMF#pY$l&^_TFi;WQDXCj=iFY;n}Rdh3xt}%kT%wDG6 z<4Jeii^0i1D1CP~6{8l|(8RHl{6@E^W^K3(%0s28#~x#Tw)GW!8>k}pi3&d0J{fhU zPsR94aggA28QYc#(RTc3oY?ymx}Mna2jj~~d-`(T@IW7XQ^i1O{zvdD(4o#Uci5Xy z4?cSHdFa;m;+pTf@NM2`I(4lkJdYSn69VMnz!iH)K~vT__!`{*+JW7GJjZ8Kh>QCp zgeQ`GioPs$J2wvV;~7K^4@I~1dU(^l8P9KZrHd5~!=z=#MEuKYn6NXDnJrZXLm$fL zcWcrO8^yVxCWa|VcBAsr26&Pr#xIWiF3=GCX<0kNnmyVOI zn@4hGPkk^OXTwbs<+1BQ3I;1k@M9Kfu>Ru|K6bnk-PRre_Csyy;W}wrDR%>V{+=Rl zZ;10>e!)rv%PS^mxltBhw3NgBvw_gx`~}W-Uxl>_ zrL6Zd!1~HUIDyY$@<>|o#yg4(>Dz*hYr;W)Y&!hOTFIAw9nEgLtl{ZN_B2v)HQG@Z z`n}OnxbOC7m~%P?3uV;d@T`1ju_4rB);Z!-AV&t*Xi_CPO)5$|rT2WbY3v&}8f^Iu zuV;S19>GT3*D#46&Nrkpn-u8mHTJZA*BJVA%sUc0#a(#yr7}%+?!=^g6`mI+Ni-|f zxy`MQ7?9BqGxv_CD*~(FppK}=C^W~Pd(NS=uPi@5+KeZykfTQ`<$2G+6JWdGBPuVN zipxarT^V><*tQ{%MmM+PJu`p&6D|qWrR&+dWJg}dr1|%vI1KyY&DnVuJXt+K(0P0q zSN7>7i!$PwhHp1M&vzs{n;xS|>;#;kZb!d)r(?a$WcqZ-RHm9Tfu4&Vf^zB0X~eXr z#7SxZ`XwAG9O(w{+IH;Wc(`xRxEf$_C zt02{D0e90TJaiN2!j-c8eY!QZzb!{YY;Qx?@M}1KxdC^GmZtk|Skq(H6Y0~8@pM(p z96Dye9_M;@W6cX${(QzPUXfpdJ#CWQ$^)yg=9B;ZuktlygljsPv+!>j_5B~>Q;EaGI4PUCofv?i%uLwx zk9L5WRq^X^{Hh-CK#;@%Xaj2{gXGZGN?oN3gGw>Nyd|qH- z$~1fyGJ-cstYoSXfCCvb+1Ml-x^h*D=xm6x@AO;o-4#1-ez^!%ZW_Z=_e;Upp`&Q) zMsq%2*OYdxUCIy6noDQs58;|~x?%U<{S~=qQ%RWpC_d$g2C=e@#`C9?$@`}Y^sa+D z?phTH-**f}waRd8ydxkF{fi;AnedAen)I^56;vp1frLIEWWEs?w=)=g!esfn{rPax z@jTp%5QCTLzXhgd&G<6v8LWF@%iV)^(1KPIdcvp_gWDd!9l21{tto)rR+rh-h90)g za57H(9RPSJgy7FYfoJm^cH5|!{r7quG-rzO!SRP!S@vk2d}$D-KDmqe1y5l?mFOwP;C^ASH5l~qrXtDm3kZezDRK0>t7%u!;!Za za!{Bh&o{Zt(UiU_7_O^>Q02`|=dZyhZ=LA8appXsRGgYE1rn(5_tAzfH6r}T0xNnjO!QlVG>@s)p!ZG|pn=0_P(8E*{)^mz zLym4IYW{k{UenbyhD_j@3T=?l?9L@0rHL}vC27apeaw}+!v0`=>gqTI_YM<-fW|Bm zIK+istkvMhuBy=y=UOe>X|#orkZxC?L6*+M@YG3V03Sa;e4+NtovMo zC)S-ub$KniVvRnYnEnS_BBgm%q6rPIScw;N$7A)YLcIQ1iEeq-gs#<=R9e}bjtD)- zqDO`ksX{6Gw%V8uX|81}wDlk${1WP1D*(Mex}2Xn4_2Z+N$%Yg{!c-lX0@2{mUWtR z&E_nWmDi@Yu>+3=YSXuU{W$cSsHb`3O3t|Hq3hOm_PyjXX?+`uAwApR&UZ1sChi?3 zEvX|JF~xAlUk;1Q=aHW#hG_cc3cj44;%rwYs-XeyxHjb=376=BQ8PE-%f1vClW-hk zH@$#WYs)Yt=o_S`q%g5Homf4pk6dzVhW&~R+ozQXJZ=QSp&j99Fzg$$f@VTGtw5|h zi@nHIbG{yF&d82Rk{i-YzMr?{L6P!{)=UHP>V9hGk}n`bKgP41hD9whT;*N@ z+6Fq&{3oNab5Rd8E6dXn3mP!ZFBLQ%t>(R+?I1q!q@ZA$HQt|CLiXrfb5?!&2wVq6 zeAyR8Hg~Tx%Q*C{LcL8+l)rut$ym|W%GCA37h|6gNx?%Ll9d6|U;h!vMN*RUUWacS zC>Pd45PN+@Q5Y%8{X2W4oD5kslA2uJf`ubzvS$zcadx~%d9BttT+t#44l{2<k-_0dbgBD}Pk8)GG^jL3;-mHdNbtoPR)1BTp0HG* z$3Kjw3iphW46lIpu|@1pe=OR|*unGXL%`nS3dq^L!^$9Kyx*wGT)eO0(i1~zS49Rc zZMTGVgGqSU-V3bWe8RLh<*;b}L8i(t!hc6nalQIH9@04r^$uTzzODoCBgYyij!qyt z=PPje0xwuD;uRnI31MQhG7nf7K!1(jhT96};orpuT!)`wW7t0Ybjps@P56$QBfG(T z!!a=1oDGlDX2Lsx9={ZI6ygK=$f6&YVVY?nyL4yP-6xm>udefr|=Lxkak0Kx6zYf%np@ zIQN-7$v*j3n6qI7lU<_E#ltTHa~aCt*(bnL)a06E)7dC>T*iBz5#iIi@F=zc@o*}`Dr+>Lm?@ix}{ zSIG9S`-?{=Rf9lXxfRp{WP)FsvCqeEP)-52IaL0IBI_w z{Ff`zWIqC~DHmT9LIA7GrI}7vgH{hyNJ6xT566vB)7=0=c&XgB`@o^*Q z+}%r5+U+sQA`GvFB#3zD4`q_yp+;P1v$;1c`|R|#J-uZa1gZT)qiQg#y_?VSwEmuA6{ z&6Vutu@QWkjXbDzJVce24pw9_fE$MpJhCDimrWmn`Vo5gS?xDgCGK^m22@@mllSl?XH-nu!~jin@ddYX+zECbx<J!1Cu9jU$X z1`qcyVHyV#aMGTguw!iylNqvr{m`ny%+*E0p+BI)wOTv9!*I%w?T5qMI8^_8)~rla7kQ=tHpN+XP&ariLr0XJh+51q@8R1X-q> z-Eo#@vU7)EZk#2blq%{X&(4G1%>kHXa~4nUmEjLUy&zet58s3&vHOeS*k13KSo8NH z_^58i$=(X+@tHP=&Aes&OR|G*aAI z*l$UGVZ$plJ@^ElE51OPe=AUY#9wT?a0XgPkPAxOPB9*fmp8-Z+8c0p z$51x@Z2&$jHG`qHs(f#=4`f|4 zE#vs#J#xI>WDL#AYa`Bg+Tgz=PZ&FX7I${}1-+{|+}A6{davmaUueXA#;eg7uNUmZ zB6}XTMx@cNn7|i@F;rdMg9&+WAnCpjsbZ~PSt?-$L7GhuOqlY zXbVg~6~=bDh^+6^av^qv40R}c3Y*8dL+qsOs35`*=9RZ_rB4mqzC4ya{H4IJX3S-t z$L1rJ4(CI&_u~7B7cpXXEYhEkfqXKh_nKvBl;<#h<1N96-BNV*#b|hB7enesCE?ur zs(f?Q8ASIMysuk`O_^c%%3Y*2=uhR3-`au(T>{Cf9bi1J0X;2RAVY5~uT2~YX(vsD z1)c50Tvv%MyH^WMwvXXemnwfI{T5pDmC<2if?#RNeW%SDwp4J=3{8?W@;U}m3G=q@4Ud$Oc)nx8V?dtH>V+%yNj zXniKb^Y*~eZ{l?3Z+Gt16N`iXc`*B!WbmTBi#A$ggDJ^i(qcBgefV6?k+{CoGX`CI@|o(8S3OtgQGZbjzKAHKKc9 zFjSguU+Ti6h5tm~X%fA;=r?qC=EEq>YLuRA#pM+V&J)$%6GbZY{)dI!xM=_%_e%5P zj8;;2|2W?84kM$w)5vJ|9hhDr1?#?K;M+|<;M@@v7B@N=y-KyPXK@b>dUi0odVl=> zqXWPHYQog}-rO?K9}2(gafdg9;2b1|OY&v7(pE*DX{k*^u^O~@TmjwIcW^CNfoESF zgB@D3^wYOI%*p%E#tXW#TLxk$W1X^wvV3 zR~C6Z|0>IU90%EB@8V9MgJ5yd8KYk*(#s=@$ymW5*uSI&LOT|Kp3ge;e|Q$mRg8Gs z^*%5!k>U-i1F&@YO{^a*MY?!6Ju^+4UuYMjsh@ONxP=|6#nq8hKVncn>JoG=(d6En z4FoRQz9_aiAD-SAN^|b?G1-_2q)IITbNr9N%U{BT8FW;r(k>5ZTa) zWgf$*|ByB4QuYBhh`PR>IrB(Gmo9Ghsm91R!}$40a`^V{0nFwZB3$+obsS>RZ_5EB zCLJiXR-f*9G9JClE`nIN1v@Vh4PBMm)OBnSD{1^gaxLYkTwS`rZS@f_%&%Z08x`qO zgDp^3Sqf%BqnZ7LKX87H9_};FBFkP72s~hoAHHewo<3b@UD$!MR}Nv7Dt#npMj9-V zt;Z>uDtwRQ9_Vy9Z0=~C(3%ABb6WI^#=sk7_UzBiG*t@=kxli8=@2=g!`STQD z&nQ>8A4)6xIA!mIgIz?^n@)tbKuSDL6Q|Q3Zics!<%1bwEl)NjZ5|c z#p+Mw^X!A<$Jrdbu}c6)uD6mo>7QYxVlk|siR6i}hh^%Uva4kVbWHhcY~LV9KW~xe z5(|&8iOK_*BtL@mOCG}0a?)t;q=0u$_d(u&NjRUxLy4>tKHj(j>xxvM>ry9FtzXG> zitAyEf&tTWRN;>To#BK+6IR5^(}+JpaJyLtUltDKJLWIK@?<@_p0n7n{w83f`Z z^g}PZKDKJlaZHXI&QR0ncs@{086ntV{y1B4m?P9QQJcIun zrow|mP9*G58!l?EBv;DLFlA{(?=RVKvvQM@?XV(9D;D8bEpa+IM&$c4&cRZTWY}k! z45|l3JlqB`=&jPDQ{Jv;YE_+NPR4NfZ+jGW_Q_Lu@7tv4+GDuC))?Z7P3fr@A#BmW z8L0Owz&E=Y3_HCKhs-=gB6M@ehxfBFXn{zdNs)!EzY5@^p&Zp8e1^X1iu7)tNb~ET zL{>d1!|*3R+4u=-vG0;3U9BoWt|`(58l|c9221vE-5D4axD{VY3^0Y$uduvN8A}#U zf{uIgP<^YAs0Npi>%a6sZ+(ggJElSB#tSUs?mxm^S|P*N8s|+IOXgJ1U^hn8!e2*8 z(Am@?9Nm8u>vZ(VbAxvkTD#NHU_lF66}cDg-l-E@n(-U{J^GCUVot2JFdqMnT?(gK z^O#J3D~lQ!%jU#1VdqX;NMC;++T@yu#jTeZ?*Cn&=-q^>huq*%LlnAgd<0$}B5~68 z7XqJuzVO0z53ItqWQI)xON<^)l}H4R-#H$73ly0Nt%u32)`GAvy&|4=i74y+Drh%E zgWctfjFT`=fO&K2a-y}>fc_^5>teMQb z6^X`|^dU>?D86YQ4^Ik|_=cw!KqbzT%dS0!A2s9AEzAX#Z=q71s2ou~;jNzu*QQ&I>Tk>M6My zD#!e#uY$rF6MnF35#IY8%f$aG@chN474ycrGfPDyqTZ^Gu_8UQ=ig!B1h-Qa9XmZB z(p3xWqBr53w{9qRYB|ast_Bs&G>DCmfM)q-Fkbf+y;ITzJ(58)#}Rwy?S=5jT{;aD+%D30mWhkMG#yO+X08t&I^B!6ah5h?SBc+7YV zHk$=8BAbWLvluKi=_7gP8;Sd#0Em*)LywK_;3z4E=R7uI=fzKW?uRBXU+0dW-rNyx zggaoA^BPCV`HS?d3U**h6LAR3Bl5W!Xmd@GY*`VGg?BTcy5f<*v@a2_{9X(;=|5q| zxtrv=``?7_oi!$(GbkJp7q z`=t5co>cnZ0i#K~*v>Xm@tASj(e^9Xg6zRIDhkZ>ka8gMU&h@Orw>Ng9S%oZ086L!h z3k%Ttk2^|ewt}i;2GZ;W(Br%t$2yu&{foaKV{0s_>_5OXcA_A#UR^Nn^%Qd0ycz9! zlfWlLoR4gt1I2q+ptr6@pfm}r3RTbi#-ouyD(PA2i3rd-`B3@@(CXkl@yCsqm{+#2Jgcc7K*7 zLwaZ`By5_7`=Ziu)C*EU2%^LH6?r zNl5=nw#_DvePvi!+O@ar0FR?S|FBbP7 zz_GE#WZz3Ed>*YL3~PE!N)PAbyh&HU)8Ye6O^`>eu`9qtDi8H-F5>)UGsv*o0+7v# zg8C8X;k92fc)iHOgndoqrs*X?)S(dAq9Mer&|_rj*L;EDCPm1mqwv`B3!r;#1XpxS zhvCB`K=DHiW-F(Gk&HDaUHOZzLMtI-ODBH%vw#fBi(~s6aaxcn!B@!H(Y;ywVORZL zVWiY+;TG?8F!fX|y#MNpBVALW(YTiMg^0YDH-)I6A%Uh7hr_Zm4wAjfe6IFuI5FcN zPG4+7zl%@DnBiC1)erB=+iy98q_H0sc9s#VUrDH16Tvh}ZQz4{31r`BC0}NkaN|F< z?2+AHynX065c^iR#*Og4uQzNg&IPH^E*vGX5&UPJ#O58Jz&YEHO5M}OTSj7Z)3u=( zJ+llOZTi{A*nAX!zgF=4;#!aldrd+rUctU>Z7$f`O0>F4G3in#aQ5^O@P?519)TFNvw8@hsvWG{P_8+Oj=JDRc9pL$e~!C9%Sj_M)@VR1D@i zNzgyC4W|Cl!r5(2@cmyk+@I)%U;QuR?O8T7-s}WP@{*(7*2_^`Q<@H3SHz^>iNM;< z;@X#LG&STro^>ySsKUd7NeiV#GbIcrs1*x)CcGt5u{p4&v=o0#{sBG5v(V2>9MoGI zQTd1+=-Jrepqs>Dw)u+RlUQm5nK7Yiy3t;3a>C*bh4TmJGL`w~&|TFWmA)Rv zaqk$WbXl|CM)uAJawVyLnG#n{Y!r%njfDSZ>#(Eaet~84cb0Hij0NN>LHe?=h0CGs_bBA>v?AI+VpmzY-`aYEbj{4lqaj6875! zv+1J?FkTbox)YAyNTI$6m zi-fi!bOKfW;}F;&ChM9dAeT%nC8568lq*Q>#()RHWd z^@oD%bFgTY5{-T?&0pxWkvVIv(PHod;E!=o{45O)JvfTtMQZfwZ9~2^=Q#OuZvnn& zJPjJR3(?r!9V}(zFmqnMu-Ne;P92pAwSR5EU|0)CZT^L(2hQXEb)oR}ZwGPe+KyJ< zU&zYX3J7{ofD`9Mfm6W<2z%;{>-RfS|DV>7b)W&^V={?WJ4%j=^sK`>t_#gorRkYt z$ME)-CoC-30UHmiL)TUp91{DE?aV$e-1qnioK4Yf}(ddU6zI`CZlTOWsXHbsnAI@OZ zl9v$dZHzyU|6}7)axlwz0W7Tz0au*}oICV59H=zs?Hjtl_DM9bVs%_FQPdAly(Bm@ zJPphHL)fG1S0GZ#S>Se7hs`ZbcdR+3NNeo^*=tn`a1_-kHRmcAvOWjj|5jjo>Wkpi z<00JI*@#*g4a2Y7Dp``rB`sw4^mTJ|s2za1p6|H-d@eh2zMrKY z`UsEbJr?m3!MGv%0IqV}%50kw3HiPs&0ng3gxgN+b3YCA;vcqegaqxoS|#{h>Z=;J=cS;jULDh)rF@YKpeQhQJWid(UvdB8*IawjX2ey9rSX5GRa58{c#t}HfYZYdkRO$)~iItu)L z_At$Q8#20b1Xq!;0n3Mr@$1n$*y-R*w`p7Cz7F)NWSNJ(7jM|5Lc0U92%n4U zdRrigoXDZ`tQNM7C0I}s4gPyADm=%&5eE28!VUATL42-A8~w5s>~lxbs}_ZzYo1DE zf)|5D)@|XldvbJ+x(Tn(7R`s0?chc(K6q112lK8h6!{;lxYy}1d~Wq?nEk zc|~?qPmS>6Wr=Wkvjrb09wFj4M&dgZ!|=*1Oy8kImo{hNr|pt->isfYG1Le*}#pEClbls%x7FwHOK%N&2HInE34@gh8Emn)H`Oj0Kwq`h!ioO6 z;5FnGX}I$VcEA73#By{<{_5S}6#X3vuFNIxD>-Qyp+rwVyMyC2^I)!i4;IGi^1OwW zjBXePopUTveMlkmjc$1VWFd6PbE5h6Fo3WXEmN*qfktY?zHT zj=b*3^~|N9<>yQEA1<0xjd3LP#)Tl<6N?8q@qkC=LvGW0uN#y{V6cy`iN zmMM6})(1W#;v(Ur+RhT2wRO?9qKo}Z{)KiSiC}H?hb&4+fW&fHx?0y6u}+;^H9kl6 z_CP*k+a{R5U5w{_+zOMtl_;9MhcUA|+3?dMf8Vql09_M#c(4L(Jv6``cv@g$a|w7k z>C!bBb@*vp0|>-bsL5~%I4-sxdZS0t^oBrE*sY2_`?lhSuv#QqnW%pJBxx7R$1NV` zaJF_GBn>uUz{6hVoMF#fhM4m;0Z&On-w~YtPc+AxoCsbUpF4k0iNf4ub(}Ak2&en^ zlXtC3H1zdXX!TLWrOZ6M{t`eON}E7;qZt@$n#%Hg)OpQ?YBsL$J{JCLC(|dE;rc0d{9^wR{G_GK4a~Dy*a>gv zD9;dq<*d_SvhOAv{l<|m6LGgAvmCjU>3S}Gxg73(zlau_rMXw+eBr_!6<{D)!Q!qT z6j-W11zN1m7qmHwJgNC)&dZI=Vb5ut`Bt>nwJ}uquk09f{*8vaxw2ejq@uZo_Lwf_ z3=>AYX1mSq;`ih_*f=g60!5f*qwNV$`FfbyZ#&Jd1YHu?b`0kYqh1j^^-OTyn2J7y zL+OOw8Th6?fJ{Dqilpx9zy||ULD2FJca~eichUJT6uPlPs$zWgPYc{v+kyMG8qxZ* zw;({i3*Jw^jxbLX4izL~&C(QD@_8xlsMtp2^Q7tLBa^Wy=CSZ-eknFgQerPwdazOI zHTc5xDKtv<5pfGEkWBq844nB3_Ug&-tI{g8=6f}JmoCLJ_K0v$*=o>nNN0~0s?z4A zR&4*SVd$p!0jAaOA_61TisSWi)T7Fh22Gkm`~pkikGu}oec1;8HoJk;u44FBkP8R* zZ^PLhX7t{{PuQ^gCiKph<{$4Rq4%?=Oy9tWp8l!CvrBcL< zpDYJX;+p)zNpU{ph$~K9Ch|c|bEE>92qxij6g(}TqxX3=zG}k*C^~%v*VX+8FSd)c zmDa_utjLnx&orcV({e!i+GTP*RDk)8vOLH0H+pQV!_-fv!gF(!`Kv-_9&+p^ZaPQ3#=@ zwCJdTzrs>k51LtJ$pxh~42-dKH>I3 zEo$QM6ek9!!o4G^IDW}%@IRmmj^d->tIuh{@VD2=-jmN@shSni83!S^>lVAAw*xH; zlJQ>aL7e2D4RbT@kOfM{0*BtFikTfbpi{D(^yXGVbBvjD&gF@Mi>hAuJ-!guJkEvi z*BoX>tH3e|JNPUsMRN)@xY`sK`qZNcf=0e!VX_mcnmw}jOS(`-Hl7rYDElcECXR!IcE2oVP{)|VzPKa9mF>RIC4VE&3{ zxY{I+eC$?*jFV6CH>$l^I7&%_~MG}lsZ^1(uUCa;o0e3HRtUqD`krT?$ zF{G8O3|8T@zA@&In1V)bhsnOfC-6&nBc!RNqj28hs32hi{_q7CY3SAFzDqHJlrnfqyRdvbXU^*%R?MBL3()c6XP;vY-n@W0MR$ z|LYH_A6CyshOT9D@6WLqpIl^`vQ#O?nJqoB9p?17vE|1fu_@($$O-8L()#Q+-12S1 zxz38TuuKKFvp<620eA7Q_;YYQ>M6J};wPE4`403aN3xWMk1^3)jVCXYgUHdZkvrJJ zPvas~HmicDl}8|^u$Fuk8bX=!6x`r91`58((&{%w?ENy)et=|8JU1#Hw@N#38EGBr zwB&|R{Ox`)f4LL#Ze1X0!nb6_p=VHJb(T-%^W{v<}z;*_yl$R+VapdVXMTK-_;!}4lgVDm zP7+9@&!b6)Oz3b24H)J}ATnYR|J;(tCOxmhjQ%c27Paod$M>9LKuSouqgABfEt&b7CPii5#? z*BA#JerXyv^tYpm?z^bZ*$7zbKAl<)7w6Xoib-axHgxRLgxhU;^i*yHy*t#5+Esed z0MBRWcl%QXs`A+#XFo?=Yr!$IcVjZ+`^IqMq3Qks;52ri{Hg_p$NDcWBm| z!>c_j(81J_yNmu;w}}V2rj>}>**2L^4qXo^IV*XZgcU#07mL4t*<#lpDK0JjK!in_ z@R(f}E{soMAxVW0yIJJ>|DsJ*9y|r^|Cx39Nef3Uo4{Qbgn*@iv4}I8Ecm?lJs3Ni z)77vERtFEKL#7bxNwhDKb>bpCroI(gzmMa7Dv{V) ze2BFt=0WP;-Sl~YGHrgZLwz=gxHlI&y0(5R+TUD4ua}$hLs=)$L&uP(-{`})`b#SU z7OUdfNW>R||47$y2maf61~hc5Qgx9Jal$8e>NKf@bX3UF>F!$e-VA-%d(aayzrF`e zv!`(6T>=`(oPZxDcKk`N3;LIu^97rS({qid@JwUaNx`?}xfInNoN^Ld}ks8Zdk&1;^1 z((+nU6YJd0QOgD5sjR8XCi08Pc~+8*7~3zelb12umxXn6_)#`ueBGrxSaW@3Sp)1k z%i$@!s-@bkJa2Oz@7k1J9{!{y?eXFcYh|bnTl}F8JLa1dyLbJ0%gn8xs^ly$@xEI8;9ZJ}W&P)*!k*EWS~b5> zg?;3_K6@a_g0-z-v}(0Z5l=N&lU=yqhQD~FWR-@+PF_pMJ)V2>Q(lc-CNKS$IRBuR zI)7`Q2(Mp+vh3e>FFx=p&N44ClUI8#nkV}6B+s^W0?&JNgl80=!n!Q+Z*gDfRo3nQ zF7ae{msE{6@8lU(ei6Q7!X1No2Uu1bWvtv7WBvy@9_x@{953gsBD?K{IDc8o1y-xj z6YQ;;GSAsFhnM0k%Hnj&usCb{d7$IK+q74ar<<^VRXr%dcCS)l?`zr1yK8iVW%xX& z$|qff?WpF>vj3aI(sF9!J?dRic{E6b|7v*_FGP1DyWL;`&oEMb$!fwW~Rb&cz%rcE-RhaJFT9#x_2^b zxz=L-q|J&f3mt}~SXE?MdOv|TwP_Onfc`G_wLc|1mRKyS2) zyl*wU!F7);KL%dl-S?hbDN!<&^>A(|jaqOYHpz{mV1_PzI{z1w`*}6_)GvCZ9yFgYs@F3LSNfH#&g;R_~429+RlFU3`@cWr0bDCmhX7M~~7ept{tLnqH6uUlk*6gXajEeF}l%U)SklJxWVG z)8YK3l@O+x17?ma)ZVp(>`)(st!JKd+MDl^(B)PrnROShtTn|asS^cUfghdt(vCwr zC&QPQmH1JSLp}71Xza3cLXN~$el*t*OIORFDdZL2cucoIY({in#iM3nL1V@nwR^Nhs43?T1st77P9 z8A$3AvW3c)gwtJ1@nJfu3++qojB$KEhuMxAKpT+IPe`)lOXsA;@Pv1*q z!_4!cFzEY(PMo+1vfPh>?5A=V-7C(WXzB(=i^AzX_cDChC5?gh)>xL}gPAIFa3|du zq=xgzQ^Ok0zLtQbLj1pk}mrTs0lH7GIme|~XpI)tvL-WK@G+G?asMbsryx6b` z?qxio9U5O?(%bDquk~!;{oC~X9@8@LXvF2fKWDwrj{h3-HmM62eTn8p)BPyGBGGFul1L-Gh zxWcX?D~C(zx-(v;iUNF-Wwg(RAtv^0ijj6gY)aNg;3R;ZiCaHHt6a76AJf z(1vYQ#40ceUhX_d&YufMca9p`B=XVqrWDuMJ(p~KkqjRkB5=yd*KpvW9c)_WO*Rip zaZ61na;M7f#J+4cu@!RQ_U{iugZT?-?%YybdwnbJ9(+pX9nwL`w?~;zA~GbOwFqAi z50cK&Yjo`>199Ht)NVCHJ+--bJ64VRd#HoCv*SIvUl)L_hST8CiVPxJT~13^8! z8BTR>@`!Y}4LEsSr%vBaf%_dxnyi+90Y|NHp-Dgam9raLj_p7&TSgXW*MM=*Mf7~p z4-chwf|*qzXrH-BUqU)EYvNIGIfit&@kmMG9wx2lF-Pe`8S+nWque0MA9SJp=tYiOnUH-*#EGfeAM)bcaa0sD1*`wUh(^IM6IsV6 zX(AdJvidaB*SMRk+!X`w-#Fk8Ei)?T`UI4kUQ_D%jlgo zNtWaG>{u%qxO1-+8eL|QBX1Tp!Cv{e5Sk?XoGZz2^uM z?ka$u)(1|)<~z{HQ=kQBni;Q?`ypwq2denW;ev)sboBdV{Ql+@jeaLbj_y=~1954< z8qmZge;2?#;cu0y+JrKfVnK6>2$o+k$9n_q*z`IV&FtmKP$wJOEQi3+aXPG>*bndD zMnm_WT_kSWBer{t5w=Lo5qkA|(91>%V8uF$CILG!>4rZQK#9zPFP95@2)_AFFt*osl}t2jcR9;!Ic0X{D^#p2X0 zH1p;}zKeS#+Fx7?>GT$)^;D8HP909^Uxs_{3h1p?d#IIcv`u{i z`(^eqa{ZIZ*8F@jd14Q-G5ChtyrXbUMkp?vW=JdFe&Y0osi7k`8SZ_1O=Y+5!21sm z!X|_B^g)&)Dq4yO>JnB#QJ4ZAa`r&SU>PA7QVF-z`N7U@%Yc)tB6wvr4c4~nbAO4u zqmW3)7#hWawD$};I7^R(j@x#iMgQ2KZN6=W{=w+$bgB7Iu<8d5^&I}p!;RElt^{Ta-}P@PC*iRAESU7X0K;y4*hx6Wna)63y%cNumc=;7>YIEecm57CvEHz8|b2ws_! z3M1cEpn9pC;MYq#JZPXuzieos(sw7553UE{ZFwTdre3AaeR{C)`(MhIoI#o$iXlhm z5!#Yq6zdOwf$37zy3rOqWB=i@2p@9K<}6Ca2h)OUBk;I!2h;~-r}SDYc;RHcH{!+6QG7KN&Rgixr2{8KT7rh}8 zhTMlOsFfA~O`_HGX@wM=*mfITs#Q3CuF`C`fB^7YkwdTNCqtN-3diT+b*LC$4({8E zFytr*d8(5zDleS$3qAWKt{%ra2}2mTxd|@4%OlU_mATQ*Q@DGxYe}^*HpJ+zgYDu` zklp`^99o%14V*kt_R}7Cm3x6U1#6+~Br`NPahtO;G#jihkHZ?}X1?k_N$xbK6c`kX zLEA6M)TLE{F*@M}RV(AMIoTgl!mME8qqX3en~0*>2>_?PKwEnce{#YQO-bEKe+-M` zbjNDUNqJAR9FL>Um?aL#iBcbjN*o3|y!WV{vZY!NIT0vj?-JnIbLG(sT2lR@ekm2MGpXO+Umt+lxi!PP`OrU@caX{B_BAOrj}EgTiL{{N1glXMH?

    mb4C$bPq>Iz z$}W)}ty_3>>1<4n(4xO~Zl@1zJE)ez8qjIj4ckaOnYWHd%8O!Yj*AqhCbb4b%*v2H zxP`jb5^(5P9DZavX}NfjP&|E|uKoBG-c|H7Z*M(?C)ok8WI&!=TGdJ{yZ!m6#5U6t zDk|rLMF|aE_becAW<#NgKMvYz~-l)kOPB|bE=xkIU8e-nGx>X(plI?i1{~BJ_FR-$l2lwUC>u7l4!QI9>H~2iPr3qB7*Tk z>~4u8nIBe=L$j1I_Eb9B_dO-s%$%s1K_uKQyTp_=eInsQ^U*Oa38tM%q1@6UB3)|( z%x5D(4s#y6yg?L)Uh= zcUm1Zt$k5;*$x~re^2&2lO+)m(Xa$^LEqe$*v{tAIvZOK?|~7h3z@tIDz(_h@uPvq z+OhWHOk6p>g9LvP!SlNtkm?lB6~83#clrCaghjesH0~xho z=&~(BzTxq2=y7V8tgrr$RG5X5$unm|y`uyMjf7!Hr6PUpy9VmdCQ{dNdyM%E7bwBoz~yjgi8z{C&!U9|O2j*%huocDi-{wxjn(2Vjqf;?wqtXm(b}Wf866+%S)TzWN6=t#OLr z%#>)VzJVHt|_z1qWt-xuTX?Qs=3kom3rCkrRnKJ>$;6a8y9$Otl zCK?SR=XnKaobiU#4@O+y`pxvBWe47Qu@OHc384Fs6iVc7hHL9AFg~3UckT~D%K1=x zibuaZ5re-y^1Q8wrs6dbby)xA6WmkK#XUNT0&lk&jMH9an4w~Xi~aAy&3g%?f5k=k zI;)jzwB3dVJ%jkdt47EYodLGC$vDwf*lE4K5mQ#FqTb z5}b<0D43loiSK>bc%bJf<_jWVUi)^M?;!xq0c&jOt0KK8g!x{RCVI%9#+8Q+xfjy& zAf#^w-pTXEJ4_Vk(it7pGhIVFwwls*ZZ+u-en&1mYsa0#Idt<)eJr(9#*}rT`0?LG ztn1wbZ5Fav@aQQM`=%dbcU^>eS7JfzMk<)BpDr+NZ=x$^o8wscMO1%Df%J`0D+d>x zC7KNXG*bx?Q=r?5b8&XuWP#@@F=)IT32O%q5(#TZbepvwhRi-uf8GYdO&^1_Ga>3W};096gB?6Uzkw;EwSbpfU_?IEPNPD+EWe2F@&Nr0+DQK+l}ZD845LZ0%Uw)W$K4 zgljna^;Vkh-$FMA>T$J`9cjb*otUP#jg-25gv#!{7?$*kb6d;_EE0k+c7i{cxqQX^ z=zLVzUq>doZK4iyOW^XeMml=nys&<{2w%KsV>o*X{XStK=mnO8;K2a>bL<8_(KMy| zTm)oepEKN=6wYxp9AwHz6Je{h19T2a;t{U`Y(F#`U%AZAhH*Brp1C{ z_W-l&y*Ss0lMaR7@SXP&yg)ZVU5WB}4pi|0W98IN^z( zGtetBBxJq_KT2=}llcS8*RY-BTf}*~!aI}jGb&(t|3vP;53^zRT^sb=b_*sLU4ehU zvzX+(V^HALg~J{3=#=>s!=2xu{{DqHC|yMjyF7&X%ni)R4Tm7*VAS|0iu03RkVDsm zEaEAl>?7tgiCo2dXnie4o;JP(U)6SD4HXeM^&ih-EE^MH;j_17 zL3S@S&wh$adcDB7dm-H_!^Sc_KjLiqA1Aw`2orDUVQ<+tvan|l2|#z;mp=oR$J;>q zu~NovYZYcMoyiTp@{|)jeV9DBbC)>%>4#2(sgPXvnI0b7gH10f_p z1Hdb0KQ1&}Qj(8|7O&iqnw&KGw{xOc*d9yZe;U)`(ucA0&!@y%@9b$>O! zo}lji!p zw?t2c3wS=;nqE*Spx`kPbrt5J?xL+wJK2`}5b{vnxz{1adl!lR`Wk&rH!yS6ZlT1J zt59>i1y7ooqaY#-MpX{*MaoK%r|85PT62f&dD(^r*?XuZuH+Z!E`$j$e^5t96M^B_ zH*!^702x=?sy^!c;_%nKU_PiM(f7X}n`u3HHCQ78OK!Pi|lT-wQ3zZ}b+ zdQ}^O9~NS}>lTdPQ_sZYju9)xpFEY=LSk94lS(l5v{2$GIph}2IrdbQxJ^HSdfv-H z`+g_xy{>~BbW6xN#SEwsASumAKm#>4J@n}lgv#|2htpZi$cuaAW<)I3=w<@9JPPg$ zH~OBJdkO2ywb*%mF+TRS7|Qa^8kE%U*-ImQ{2lE}K0R z*#w1^iR6K+3m7+j!$s$9NI}hW%HA!F8a-*mGa4~|OavEfm_fEW>;QGY2I~4{CWdXB z%uQI(iALh`{Nu5QjI6vv zZ`m9mY=>&NZ+Hezq;g>Y>={&KVlUcWyvMYtX7U;w703pO1>6jh!(4r+hxCuOz+l;G zxIJ$9a*{iIQ#^cxIC^zS3^PsqgN< z!{ZBZMeY!(bIXUkieK0n_noZtJBf3C@1eFC=Sl9tFev}L4Ale5skE~H&k%;7+>eX5n^qUKyTbj#>9Q5aG>-pdYz4eqX&8z zIUfUfpZ5R+?1SLlQ;b*rB=BlaALFK_N@Bp zUF;|KqUe5Wu-vSr~HdcI-`)GydWZBGA3nZ9aljXMlq zR%XyI28~p}w9uEnlj;6IE#^o5e!Rx`(++M7C_X5KSaCnxI`JTSz1fXD?MX1P;WjjE zGlPMm&CobD1nV|Nah6R^2l+oY$k2*-`YyPRVDe+aTfLf|5))vG44a!f^9sCm^M*Zb z0<4T$hzmiDyE#f3Tm)5Es96ZNs><+ysxbsuNx>6?6wXw)ZgM6t8A@I(66SyzX#GQj z`+HX$+0d{F3$=&A{K-H1j$zo}ou(wijhkV_65(7F^ zP`9t2y68Nmt*0kLuSyPa7a5|)M|WWDxD;vs^@otfi?Fg_7MJC~#;4;0sCH>F&hLCp ztu}2S&(~)`?D}YG$(@BFRpOv>c_T_ps=+#+GE{c>3UzZ!P-fXivRi&0Dsb1Jp|&** zESC{*0z=7!bMw$J-sC%&IZ-`0|^O`Y)u|A=f?|bw(D~c_*PSL}{ zp4y~%1^gU3M5eD77o4Vwq>l@c6326N6GQS#&m2N=hOd=`9%_JY4Gp1F8#HX}`aK(?U5a_B9myY=X#+ z@pI%Ay-kmAoC<~Qewd|NO2S(6VC^mynyqje_h%oX3s(6vb60AUN^9T0g z?X7V%@Xjo<-YkcPeiq}NKbJwu&i$eLPNl;B<*wA?s%lx2s>O8 z&{)n6I^(>k)8`_H@~onXwb|Hmxf{lprr?e(LvZ$-C^x{u5=N9lzk79fj8H7J49CI4|B)!TF+U4YurPY=6b05qV*xV@oVfr;Fh9^eLoni8c2Bo-2H3 zQc&6S0jI|3D%sGLM$ccmhGXBVsB!;p2zzf&Ts#u+Vft$_5ZevIJEoE|X75Rt+DbaS zrJZEChzeXDr$J|G6ZxZm19=xjLH@Ehs&-DN3hvtU#k?FmSsaZ=7AypR2~%hl_WcsC zYQhZtOPtHw%6U&@^_V9)z1Tahz&@b11HZ}3()04ka8$ROdE>N(m}}_C&NV`9^O$*5GX~ zbqG7X0-tmRa0J!y%=$c0YP~m_Gv8Hl=8 zQLwjY2_$?tfu(Q!ajM4@xHU8bR6KU!gd%0wdG9zRPHaS8*eDugWWaQZaiP>wKqj3^ zq1}F5uHm`cJsw6Ml2h6FyI5{L3jCG@e4K6mCZef-c_NM~C9BQdeHjI+^8 zw%MdC(3mz%l{Q7;rxXor4a$JMql0A6cnEaO2}Kr$Ls_E}^z4p6o1UB4P78r;CeB^q zvk=cXZ>L|Mu7lSS3n5mtjeJxIfXSf+=q|6$-E7iC76vq8uU86qeU-!Jo^Xyt&;yi{ zZY7fo!l2?d1A7Pl&@I=LxfPEzFe%IwZ@3=A1HU$+O`%Zt5zC@XMjGngI0Uiy8_3VQ zD5W2bc@68C`RDs#C}kBseBFSz%O;Z>VvU@E^d1s>F%nz$Toyv*xtP9wBZ_*wq-F;% zz-Xi!8Opp2zQ_4!d21TF&Wa}Zco>caOc0Enp2i)Wn1cE?A@sxQsc`JQI-beN!hPuz z1n1xZ-6atytd&%_L(4Caqw}J%TUQ5QLpIrFn1IJ@%1O_rNNj&Bf(iQ^;B;>`EMl*Q z#jP2_KE*30{YX5XIZ_50A>n9$eVz27 zmkM$AW~jlx>5#Q<8bn^KfxFsg>Dv2;LAfRXgfeFOX^Mn!jVlR{Y*_^+T@T>YnJB!c zHcocu4bp?VdZ9(Eg1*n=6N570*)Qm<3j0Ed{jO__=ZUHG@7Gbd^KS^2Ow32yWPcRH z9IUsshAmMiSV~<>LF;rTt^OrWr}mGre)#`|!^07gp=7HiRd9L&hC>M1 zUmuefO%-(B`v%ZTeM<6c>tX(YHtvXOW%sieVZNmZTKS$Ot0f$kwU)?7zs=3ct= z*bgi^7)yO!$LWZQ6nC%kFH-7z0W0P`q(PZ=u+foT(RDV;99zO!HY(1YmEn$SGJRn)*BnlT zhzk<@Y;m(kD_`T)71%hp8iTD}ATcH!QwJru8&kFM*klQT_^1N@w95vT|8l6z(uO02 zhBWiFJ^XmvjlXT5GZjydV&CI>T+>`kYR}1lnuQN^{QF4SErJ>2n>Xm0PqpNz^?l9) zlV5cHm#L^^_=>U$l<9DY2K*A|V&qvrFs!zR$X8P2+qr6bgexX68T5h4lQ?8CFPvH7 z)Ij6ptC(AFd2||chrR6Kb>ild3eUt9U~7U6Zm{--E!&0h{zW8SbpJtXgt`&eLyAz< z8H#r&Yk^^oH2Kkdfm8NJm0Y1wWJf|F-4}G7T+^No5l?eq{nA31B;5eT?!B;8v4I9p zrFf~3kU3Hwm=yIG$Frh9Pw_U5zP_Hx3O0ubL*Yzwg3ws__5+gWa}*VlzH{WIYN^LJ zil6VW1sZMEICvq5lDlazvM-7DzY>R^-I>CBaR=HpZ{e%El_VhbHsjXWK`jmkLuJqe zf#LB4EYqx|_Xp15t^G=Z%B!llBIP{p`zJ#7+{nh9@g0ENiI~n`K_@BH5WBzo=*O!! zab2xH?ADHee}8hxmje>mqtO8e_AV#=hgN`PXCA&h7fmM!mZ9^5mzZU1Mb!2U)A=7m zXwso1_6J`l`Wo{vSW%uPmfeB9^JCGlyN*8EE`mXGtZA+88`x421UI7g(~+weq2sVU zEZNk-Ij?dJp80yA$JAX+8v7b<()x&ly4SJ9u!V|#bOS+&G&%&lC#!s)&|bYz=w7*w z4CkuRDQWSTaC<+^G;^lft1IyR!I$u^OBCNtE(R6DSlIPc0$} z^u}YwiAZvNS|z!%-w=O`oa0os9)(*$4#K=K0{%ZP8HhpE(1cS9NoOl;UC0gkE~n#s+*x?toqH0n`aUjDMZBA+%m5I>xz-f?guL4x<@ox(v3PmI&c1-F1V~)OfKhSlRJ@0wB6__v$A12-D&=njM$w9UUMc2 zLZirG<(ZJ;8&9^ls|)_fjnhJlBM`Mw4_ybJ!B|-`6n?ynlH+mYz_qPp>aRJ}MteQB z3@*mETSD>niCUT#P)>fF{7y7~N29a7qM$ae4JxnAMQ)Nl#G574fBza`_U{t-p%aR0 zRHe8|RVAG7AEI&lv>7O=qsd)hdI9J4D3Vt)_h>}-2h1_ZXEc6v5GQ44htv~IFc za=8L(-MGRJ66R}dd#~}k-l~u-7d&Y1#7RuC#9WvfU;#>vAIMpWr{vgkePHd5rtfcT z!VY2opr#^@xji$7v+qhBG4c*$g_-?fTfEtWB8xu&BQU1n=XJ3Cz(VdcH9ZnIeJYAI zrvbZIA7@CZz{{^KxV-!TEIwI?{pWSq8|qSE9*YHCJCiW~vpKM?g5oX&8^#_I~cqDj)X8Op` zAe{nY?5YUXgG&TizCrLh{1JIM?J5mkY6ll2I7C*(n*3NT%rVqYLCWRF;97o_u3b?_ zWWyD4+R0B~V{eIeI;rU0`I$7U{y_)1FLb%uL2SE{&(kY266nXrW1C7Sa6hY(DVvip z$Kw-?*{ed!SF?zd+jF?C@rTJ=AR>?}_yb8|8{rcF5pBA+362~QrwRAst8U5MLz!zH zByTw%Yd?sQ-)dhW^vOxww!;)66pe9cUl)kVCzBudc~mg*6`7{)$5F-Iq<8KZzEhYB z(`p-_=A#(-{0l%aIRcjH24Z$}9lTf{jl;sbcDnOEIxJWMva^HHOhy^^^A6Ko15Lru ztBHc~-$V4UObKYek|Vno-C-^*oyIw9!;p@BGf7L<5;)zp3Z;GIpxIa;1;~n{FUxJ`vX6BXW{t45#-iX!mj7$nC`U=mm(hu9eZeq zV;G&H>jfP)-dHR5lD^x%1YTL*B$7vi(Z+TXUbYdQ33*e2YO@utuqZ;wjzggLEm$}Y zGw1w?*iNt7Pr$sDpCJ0xSF+1woT{#}MCWDYFr;rxYu6vbg84J(PI&<&L`Vsg*9D#HdG`H(1g=zTgAd}GMXz-_QwWC^L8oeZ1j zJwX1C`NTwtCD{IXgz}BI)0jRBzVFBsvaUdrYr#vwM-T`h(!Q|Z;3g=S`3Z@~%dzF0 zAETVEPdcW_VWdYj{h2d@4Jq@XI$NFFuq2esJ{O89CyY^d0+0EpaExquavrxyB+=o) zNBGF@I3|rPfGn*IB=^P=PWRJ$Fr`gYu+r8<(0W=8<=tcGq9Q&ND@F0IPcR2fs}$5T zScogU+ev$DEUPo#9EjmFG-Ld*dN>CX3iiXtnsWFh z9@1!F6x1`|%zY*PcGt^zt0M^~iyULv6+fZnog5A}X|%;FXj>-=r%kFj z+t(LhLqrqG&XQ&R4IhP7*$3eFhE#Z!`W-Ju{~;T0tHYY)aH^TOp8T=#quCi#NdL0+ zbW3a^d-5w5jM?@ujOZ@($QT1wYB)3fa2nj4VS-*egJFL4A422QVbuu(P$Io+%!SoW=e5uM-w&YxDnFy~E7qF&M5a zik(;5Va0?|tQ!$(W}a`uE6EcDhX0juo)?Wl&H3d5l?xZ}bon*R{%{lRFO|^)t=Evf ztCX6Ayo2R_revY6BsXAaGDM0+5tycjD}Jpb|IX&ZU!9#~hi(b}z5bh+Nc7U}E#sVd z8S|(EjRj9u6tt;mVdu)_U_D_a{&>6|<8H<<4@WS|{{ax163Tox^q65eJhj z?%+FIhxHwQ;pE0ez$z<&2=R#!qqY`H7w=`Ng(~q^qYj)IA+g|eh)3+=RdI{qKSs_V zfc)zXf!I4KaMJ22{4-cWo*Vdb=1NIGfAlr51M3y zp(qh5ctNAg%TXUZgVqQA-&A9*xT6Ms8+sn14-=>$E}=Ar1?CfHJX ziy1Dvj3%1@K&s&g9KL5k6`$$j)PhVHyJg1_+$@0l9oGr3$pl>A)PdXH08Xng4_4?p z3~&B@hoY-3*!iZKnXy%qvzGIUiCZ1Xs5T`+xYv8ybbmUCAG!cl&x>HFxdxA^Tn5z_ zbK%`K;d$6x309`=c%$zU-VScSD}p50d&U+&GfvQT*#Ls)U8n5>sqjnJ3^Wu)fQ%Nw z$8(YN!wPZ1-@h!p?-q<1^#S1BKEQ180a&OZf`%|n~vCg|^6F09{^(M&ko z;01_+nL`nMe^S`T)e?huv2XCs(I(uwH3zCI!syC@0y1^sd%~O&YMJU5VuExYoS1BY zC+z~6OHqYP_L4`KyD1zK21`iN4{`YV!vdzx|4xo&RpZ~c{%}Gm4NoMOz}EoeU?KsG zK0?Po_Yh;LtHhWs&aM9V2%Vf&xk6nef7A6#_-}m<#-=%8zrkBlT}wf0)nb(CZN%{j zQe3w4Op;ORN7ZW{&<%kp#CpRejL-DHKAL6r(lwGA@y{Q!GhkC zboZM5s51LF+#7EupZLGX%K2Y7KekgM|I3qfyU#&e{W0O(DU>Wxj%RPboyEM_U5`7G z#>nfJLLID78?9qCN_JHDg7}e5_~qFwO5+P5uy!wGPksOfjY7@v{d3Sdt&Y$ePDG)q z0m9cEWrJHORY}*!kdi`Tb1acuNH8Ymoxw0Wd^aPlb`k0vUxQUw5N#6vFC}A{L`J$A z7>I%~bq(&#-5UIq6$PYdi8KuS*Tj*Fy~(?wavWa$E5%1OidYaBjvhk2?PA6h*Ik>4 z%`S3~;Zu#D#{lxrY6*@jMbmAa0VwORhcg^6No*$tau$iF;!*n;G;7^NMZbH3t}9FtYRg6}R69#};E%dgROVlu)y zVS-@T)|Dnl_Y?8YiA1zkoDNCo5%2VVGC)_5yyyzX?1KsoOsFCP%_#hGF9oT719kAs zV}@gN;owyRY_gZd;|Kik0ndcKba{jirPttU#gB}!R|K(hjDX)bfcw_uQ@7J)pq^_A z{eSG>uWBF+od`!`_k29DrwB%d58+^RF&W<`0?PxJ;epTppnEOY zCfzTj*XO0;Pl^51?*s$)UA=iMQ&GD0^$2H6;3r%z&H_J;bRuH^6n7bok|jx-=*j<5 zN!L3+PWhyDMAcUUHXjv((!dFL_uM2h@;jc=PalZ+&0(B7MGqcto+t45qYmsJeRM%p zC*D}91AWiMkuH7$4*P|3Y}sl?Av~1Mxq4anCS+k_-*p*3qU7p($90$cBjU2n7 z-?Vz?cChF3N&AmCL{9lBoGL27KVmP4td+cgPSeMp3%PXK`bomx_&f3`;xzWBT!c2~ z&&X+cz`66p0p3O=gQAJ3K%cRNpJ;&VD&CXN#hTcKs>HbIsriHVo-|uHUo6)B2f5L6 z$m|`Xq-<<8+nrqvVkaWu&7{5fyZt)}*GQ$L#1Jg+SCH(kA&&a|I{0scInv|vu|%wv zq!)%`(RNv`$453MM~i`Zb}*d(tH52nZXQ?j+!3^$@|sF}<Tb{Uye7ziD>COZGRn zK4EYCG)T9}fYQq%+&5eX!>*ZF_4PLnA8UbC8xB%UrBzTetC$oVSVj`;qT$xfWOSG; z)SQT~#{87;Owq*qaPepeng>0><$C5!*W?;#ut^g3`Sw!B^%G&Wa7Sa6^?Xu48i?C( zuSb6c6K?t}0ekyf7jzz3gC~RPp{lkVJRkbu-03xV<;*q6%SodA2q6)@pb%ng^k9$1 z3*fja(YeR=@D@zJ!%y~*A~XDKLF>{6YN;a1z4hB1|KwLegv%4={%l{+i8x3t)~nI2 z2ivHzh&qlm7vT7myPPx&84h=>nOX4tI%Rf6(x7c(C_DW-Hux&z<*{dsw^%Q#cx&Ly z)klbF*8!r?9ZiC~rFnY#3NT}nE|t~UhO$SlL6k){JYOD+)lm-QpP z+x&{5=5H$fyE_xr7OVxY|3a|JcpkHIp9S31v7<7tba3Us9g@0W8SWD1{{9Jb1eU`V zU>C9g*WPfXvPY!3OYHVS-0E;9!r>>i{qcfa9?|0l#2-bk|D*QaZkb6MK2n?nvQCXp$hMlhhUmI4km(13|3F#EWEYSNp2O@tzBek#B z;kdmL^=NQ_t+OAZtIIMt*SJ-vwYtrpFf#&!u5HHgubupap9jHM!v`ihyrf(c4_uKJ zgSySzki9MjosX7b_n<#y_i7N`{d1rw>lR4pN(v-m)lp_N3kn>xnT*IxvhT=asQ8sY z^1WZu!QMG=XI2^Kq047XzPn1OuWG`B{=*FCg#^vZzYdq*3{W3_A6!we#W%DFs~fN4 z@j7cbM{wn)rHp0H^FtP8~Xb^1I>pE$@_>7#?nQN z>r(s+(nQp`d~P?eLSvbMGH38nPN27!D&gyeidgDu4wL>>(Ai%z$eGp8$@%DMVErQt z2Rd^Y%P)bTY$5_SS1aMyoCz4+CJzcaCor0`oU1**m|1n&0k?fi#{W4w^QfA>H;ii@ zl?Dx(HP5r|+54W|q>xl7l~AMs$xMTyM5v@R$Le%|MKKjnk|-13S2ob+Z9F3KgAE$I{7YYzINE!wSgsyfBeVJ7Ud zgFH_8*9C3}U2Dzsv_WgPm(yFPgmS*_I!Mz_jtiu^=?Kwh>_aI>uBd;29=$k5^TNAZ zZEp0zu(G?{<`9k((Fo-R@oBU+XeOJID&d^4;3lnPo=0DhFXAt_<-_J5wMM@LK8ekn zm!PlX)wF-M7%K5D<|@0Y*@B7+&LFZwpgRW9b%Dyxl1w@G=F3U+Va_z}R-#1PMoC*% z|9cc0>m`ha`k&E@`VOKU!fI^_rv4mk;`5ZLMMiIX zS?11CPIP}BT0|Fd-}Zh*CwfbeP2f>Mo=%wS%Sq-(y>j4MX6133!w=x_V`(mP?mkpp z*+mbJmT|Yf1|ZRFxmLM~2K39#3B3|Zqj#Il<@y+D6yI&n5rtlq)v=QAoP*Hqyh4up z$#71uis&yF^HHySBfD>ZDW|zL6YcfWM(ooFZg!Y(+h*O_&hb6}*cFHOb0s!+=*9hA ztmMgg+{_1t=zgIOdv&WcSAVdME1yh6s?>At!ZR;)5PGoxYTD7B3~OXl;)B+-KBSkV zwzJ;$IcVnO0&dW2F`B#NA)O*Lp9_9;61C8^$N(##C#t7e?S;dvAQp`d$~193r|RiT zJ(;LXejLqo=|r6(?wn`I4`?@eoeTSY4vDwUqJss#;KSFpvyJ!!r*vv9zgoQ}wJ0>M82$dd3VjoJ_X3yaVCz1?JG|HwG5v>F z%j#QPNVy|-Kdc}1jHYtT-Dli7qZIDY?QX%{!L!X|Z7G-2BaSZFP2&<8L$S{2cWUd4^6yy7;g#&f5fs`-zOQrxnTi{#1eQoQ;W;NDyL(sAB> zs7p%Hx%yNqJ3*zOPJstkQWLZ4;Tc@cxIen~{U{Q%mEaZ^Y~)t@Z9vru_mGb5Wq#5V z!OW;Em@|1Uh71jI*khD8Yf+bl4jjBo+XzXu4Lb=tw*n7vDtH+hoNiTr@8FGPh6OsGIRY&_P#Y?qB?3 zF5&h8dInubTP{-I>QWXVE0t*OsX&u+-XzD-8+>RSD$&O7+e{aYNVc7Q+koO+^uWfy z=Q#buP_EPAIZJxQ*e|}i$o-EJ%92k)pXZ(A_O2Ir?3Jp>$@tkQS~G`UXU0P%N@D1p zdjk3teU}cOoP$=+vY@YiI!ouJRiGblEKvCTR=z8l%B4>{V!hTqXbnx2Y@2((4&^LP zYu&Qs823?E;3-`@Ml+RjxkuPUyTKZXRGMXBk$nLwQ$`%VN^JQm_anGiwd!b(c?bKNh8P8}&@N%co`0+@^ZASWF*9 zC2MhB5py`=ww>$B&7qaPd2)L>7tZ^pQrjvKQMCEUsb^vwWqJ`ktht2@Q-9FvKTn|ao;n0Bm2p0slrQJXKU%?$Z#S^+4>Gs}J4sZ3<0xBgB+fO}KsHFM zpD&uAjp9#-wq~TNaHCH%xprPJdL!LHml>9|MjXgygLoWbHdU~dgPnYp2UT>vL!H30 z#$($@{m`2Od-&3YYTTX}1N2kh0Og0tb85;}oY1%;t^1;iJ7~LsQ@p=~9o{&f^V8gi z2J9Ts={p+{Z(TkL*%?Rs{at`=sflvkqVwpf{k81sV^#b`R$W~E+_h+v$ZFK{b1wH? zBAPAMD(3v^Z=v9S54estBlh?EK%`lY(Xs7b^d4Dd=f`0M=;lNXVpm*Y4Ntb9yxeWtui|uXgoA-dr+cC*^krb@s-RO^O+wZXdJtbV1fpOc^SD(4A zs9M&4ekh$^aEYyye@54ykm62%v|;sPkI;@b2hp*14K7{a`T5}x%6{}b!7k^n(Z1o9 z==IcWZlQ1?*M1?5o8fSTn{%cZosrMun}RBCTj44$F6ta-wx|(ZNZX2{#0t6b$J3C< zTW5X*4(7&MpK+VwD$!)aZaPX&7iHG{pf!aw*ye3b?CnNpZl=0BH|NAcPH4?hcJuCe z+!jGjXWc|BEBm&Gt@g`D5s#daS3lx}1$n{KSv$~-EvjheryPMdi_^Z$El>U^O>LStD(Hi5%619*QMuw@ zwA0%rnlGD6-wfD8=XORTVWG=xQfE1uuLjV{c$v0WM`yBMZPk&)Jz*|iWE@UN&E`Ve z9N1(vbvj9DHy7`x!M9#8&H3)eboz1i7{BtV2fI1taBKSq9nLLWiv664=(R=b(4V%G z!2K#_ZKma*_^Wbl`<|qs{RXeOg*8Wz^Wazd{m-kMP4ftfF!n*6GkZ{o{8>Sbnu4sI z=Am!0kz7uWDhgOvLo;E`e6P2=k(hEKdNXB;bLfN!r&=71EbkveDx+^(OAbe}edccT z(Y{LB<-rj9yjKdH>~*1)mbY^ew-V`dDns;0hZX0e^O5$;y@dR9kF_e>0Q4?mIh$GL z&-tb7MId-SD;idYxW<2+QNspyN9k@2zhl)7DRt+9;`h z9vk_B&|lREYIyC%>F<+3<<;xZy(0~ReNZK=aH0W!mf}6=GdG=XT(FOmn~+6kr{%Mi zkA_J8nOsz)U%{OZ7X4*lgUxrCi*9J{%b3(%9{I%^A1@}i~eACn}s z?E4`UR{eo{!A`ITe`RxH73tiOT*A)wa6)IE$FR#K<{`Z;Q#ebLtDHW5&UdKEVbfkm zu`QJ{B&XYzOZ-sD=?DMdziBK)`eWJrf|9$f22&=u@neMEQl`hwZEi(wTVpthwtc9e z@&H#DA*g4c{?ZX|LOB1u?@=o9=5nuva2o|VzT#6hXw~YEtVi7p?r*Ifr+LMT`;}md za`v4^M#t9Ba|QbZR`x2gsn)rytXT<4C_Tf?jDOGGVO5aw$u#ch!_VxZi~?@sTT9O6 zN;(_S+(qXORk2au95|IR2kxw)2D|lc1MT!#8fm{y;7&Em3ToCT_LFiOn!E858tz<) zw@*5-2{wB;*@R!nZ1@!0_N|!PJ*?}&Z;EUwAvsp(QxHD;MS&en+Tv~?*8kLabbY=&(t_c|A zKa*aMUh67w4pHIA_Jt2khv{+#kLB4XS@-Em+ih(e7(&9Ap0NAZUPE?Hx@d}7F-i$~ z!`WT+W>4LE&N-4qa0Nq&PnJ;Y?Q zXZ$qT9lLq#us_WOuFR#q*VnT<^`ErL+vT%+nVD^Kcxfo;mLx)6k7%fP5|s|_L=TpR zaSoZobf~o!_hGD#9c@-&+j7rw)6SW24`zL2Kfe>~`M4X4u3r91-|P-Xk$-&H^JfKr zqkS!{<8I<8)Zzeon0X#OnsSkS7S2bB&I?;lRh6^KS|#jB&KMmwP359dBYP^$oK{Q_ z=mZP(on0&_?y#9CYB?K(@;8}t`cq5j+ka&^voJ^Ji#O_7Gp>~{I(Uw={#8M{&2yt~ zJ>1*+)#4WS=G+mafV8IFB_5&A*n9Vf*>7)d($6k8bF?GHT@`9(JJ*E4GocuH)t=+Z zFJB^Nw*9EVI}vTs(dE=;DbS}Y)}rJ{Gql^ZhMrf`L*KY5>b&4r7XQ$QGN+`wg)RLt zhc#Q*gJSw@Iqk=3>@&y5bjAwT`}C(a=lH9D`@n$>*nk;*-8?<`Z!jj`ZPClqq1=?E%0F~xc8{7*VnkX6$8 zu!&pZbDaCOUWW6x-A|jI{X>gwQ(^CyzC>#}wa}GYS2^K$754X-3wPq|2yHtx9~FjM zaPQ`s2=a0ntcJS>QpvF4e*22IMO!`O8hA#iQPz=rW4n`2-WhTFqrS*0eg+ya2||Cv z3{k!i;=Wp&a(ag5+^PLvQTwteR3@~CQ+cVX7S z#gEg|Fy@{`Uq#u|q>#GN#C+`fn=3*1tZu=%Qs(Y8deX^dLd(?qGzB!TBt<7VT=Y8WZ7i4{c zk_(Y*Ll1WB{zauOatHm#E?{i`2~%k*P`WyjOg^3uyGDm`Q=2usN^ikpk}SBE_>$Rh ztQQ0w{Rw)1N#>90UH{kOi7YP2s__C3rCGF$mr(NkT5U!(q2doJ_Sb zZn3gZ)GXcU^;s!sHX#pJuy$mt>?Szpxi85aRE47(=Mb6qq3~DQeu5tB;?qao;ajsr z;Z(8t_>5rn8f$(SM5h~&PTd0PjNLNet{Mew`ZkmBfbRee1Yq&vGeGi#F`0IL0t9?p zhCgkV2bT8`QBs`(D_C847<%B{XRZR-cRK-fj{-9eYmg7j1?HvQGhAE07#`*O5L&|; zHuuJo>{TB?N02;@veUvn6AaZMBT2rwJ_T!@Ik#9z1VQl69%QPl!Le#m@b;Y!aA)8i z<_lO0^7%i(bHOh7m+hY9OW81(Ub=+TcLc*j%Z13PN?BO=*#J5u*uur%gqb}a5L4KM z?EY^Nu&TXeLFScOVBejN>vz2alb5te!y|Xtb0#0}Z1sW7PkVvF&T6d6 z>~|V7Fd}!3&Y=`kig0m9DBKpLMDFW35EBhc=vMKW63!|BN^4Sar`k(!`wx%Uu9*X_ z?a_d3DlQ~wSrj~cay7Y@B1()3`$18BAkp;P3_rWw2VdVF#;6xSfn$NR`9rcDO2RkM zAtZ2D3jC8W1HX%YiC-86!JV=D@T$HPjCX`n{l2bb)+c>Pp1;C3yOKcIK5bY&cm_|m zEXP#LK9C{c05{E>2A%iL0CuN@iBatW_-V&~I8(I+Jdb{i_c_$zzZsiBQQ`6CsKgSS zad|oXt*?r!_bP(5pH1+x9ZTS>V-{p#&nkFm!B(8j#luH`w-Wu&_wmrsAy8ItLYAzy zhoO;QnXwo0WUUd)SVhsKeS`){8GxL6HXZiC+qhuxB5)V5dCxC=iABuiskV=r>lJ{87JaU{W^t@!rp$1NoSK2>QMW7b|6HP4=5CB(t`zCMIJm;XsQc`LHRN7_W{d?($K+e4?g=@nHZFjuRc zqJQP%ChtfrVmZ>{mRCnvO6Wr!1w%;0EuqBv=h#QEmp4W$h?%@GhD&@Zf!sZ70&+YE zcQ1)ZES?Pl26y2(m(57&vV^l zLY~myu%v+w=DX|$iSu$W&psS~JdjNVsm&xO_kYCNr!;}ux0{Ueu|-(ZS&kfBs7u`Z zoA6bU+xW}E4Zv;u0*=~wfzh7SBwGfQVbjXT*tF**gHpOF_v)S4+iC@|eDa-nuzW5_ zS#JTyopQnIo!UhD_9VVlAwq6k?*WlYIpEigHDL6MF#Pv06dyC*N2SycQsW_55KHI; z6aE@lbFKlP&2LgeO>QvGDXZoE^GIr(zmd#r8wWFfy0u_!RrpzkBJCan_`>x8%2vR{ zqU1H1&cfM5fVji)Z?-b?R*zF>*%?4Lcpdy?DhUfN8AC-iSvWEH7D&Ec53QSJVBHZ7 za>TX}&lfT#bBq7t1Mkgn#`q>!v|%&M9!P;J<9e}asW>@ZWlL1Cm+5QO@@Vm5hY_aE4hHW#X$7y$``6ud6y z2TS&-z{ko@shbYnv()VPZB?k|d>5eq#mqozUi$HP|*j4L_aw zkjj3ihkqE!V2jPBBuCDG7|h=dD;Nj(Uz;uLvE+e&7UE#>Wf%BmRX>(DJcNz95jnEF z4RrKb^4`ca0$CSt_;leXC-;aTAoKbH=>6Lu;FH-_D6NIQU~?z+^^Zhdm)jWrROR{^@We#MW^H-p{<;vl)$8d}wwK<$_E@bP73AboKW z`F+0}*xxwn+Z%SrsF9-{)FExOs-h8;q&);@eeIGq6{{XwZ$|MnW7HQr5TVl+i9@< z{x)*3dp3DmLy?%$5^P(x4L3IE5yMp)q-$7%*nIUAuoi^?xH|~6Is?IKydIXf$wLo8 z|9RMXIrdyo0AjZnf;=lP@~o-^ysp?u-p%dCBP#+35&F&q&B=uJkHTPt;u?6~uLw}m zRxr4-7Nq3*LzxM47=I-bCp6c9YsGRfH+l?MjW^*Bn$CdRImQ^c><52FZ-d*8vtY

    d=pJT1Z)WhHb5qG!lM(R}-jByuoWosl-ekd}{rJf7_0b&ld;~CNsz@sA*pA+5+)MEk&mz~2qo$Lg{UN3})x6CH0d(UBZ zCL*mpI@D#65NetrVLa&+3(h^6gTE0o=pC;L?Z@Nbl`8-GD$3muO zc@-s+CP!+|y?<+)RiQ?kpm6!0l=X_8)j^4#QOs8 zf>YuvT0YL$0Wys-_S>fi&y`+b5)aBlaAG?Ab*%=>sBFZ$EuBejurGOJjDcIHAt6M*#G7N)7hy6Bn2#uCn^tc)Hgm+Za<7K7%w9!{mO9XPD7?B zA_SH;3b;iz*mYXR1LZ2(4(zcZE&M*s<`0Q>j)lTQOni5LGmHaq1HKhKhaXKZ62%NHY`10#vm zjbL&rGldk6#6Zu70-o&h|9B>Aa&g#DG+a5@ix)2OCO+BoNyBg&R=&6w#NsGCeSZSJ zXaaC-ehO?CuwF|Xe4wSO3DGT1fKclX`1yS){BK7Pxpngp`2O0s#Zu=t$li1Uzgmz) zQbSbmtHmCq^YuD7kR(rjy?+L}tHq($z+7mwBbfLI_WVzmawH0}jo`w*2*@;-2Ku$7q} zX8{Wom%)$bJ0Y+LC!eR+HIKJA;?j%4@S@TvI3Tz;`<`^g($}qs!P_X9)+Af-6O3sYkffF{{HjRt03DFyknG=5+X6$OMvg5tCGO4(h)U^0=E&q?xNS@g8n{e{ z^+uB9;Zp^iRw;?AlCNTqSE0DSaX+PfB8?JD83zx~z6Mu<7sKY(4e)xOCY(GJORh!U zr9L%zFdgQ?ByYPo{4Wc_gsIb^-Un|OnRFdrd+CHHD?-Q@UN+wOGY~7>2*Qi~PhwBa zI3N|MhaVK1!(A}9N^q@d%~gqEx%0dFUw5_m642azj9;i4Vx_|=>_L`7;CPZ{*2c+z>)@ttXmq1Q5S zWMeU2H8};&_k4-PHV#k)pLT&QS5=8}kpcXb8VchP4L#MXu>T_mX6(0sJ0mAY#>dv zpu|;)RE;RZz?m-?FdiD-u(szbbLGw~@5F38e7T1`;QehQ;1i;>(G<;DPGv;B1~B*jE~k z*B-DW(Z0H*OF*?7KIczjbzR^ew zy!Dh1wV!ywg@cNu-CPvjv*`m$`Sx(gG!Hv1+)hqETTW~?1d_lWQD~Pr*rG4*4&NkN zFiFbBu-N+#F#JyucE9AqtDA~|y;J~PK@-Z5;ls!aJIE8ISiE6nEDwhTFun(-z{VpK z`5N1S<4Zr`C4I+0YVCbYU;U0X36#K@TQ?H46IgwHwF4e^9FT%ny=4P6j~9`Ok zN$O;D&naHcr2!lvK?$CvGI=F24i-Gz0s^l^z##s89KOI7f58@nS~nY-Ue;u8{CB3s zE#NNpUB4cjR?4Lw8pTpEND}{PB*02HnXLTp8*Y|aMnXeZLzQwVxLJ56T)5gBZhI+C zQpAG@w@wsyXVRo4!3Bzmwc`)x6XEiNrI2AI;Gp_WvhBPu=}U?x9-HPu$LW4#;FBXmt*UhVsM$^2@vk12ZeVof#*zTz`~Oz;7QLC zqOc+bBsH3mb2WjaJY*a{9E$_@)ZXLKT0U_VFzMYx)XA_PX6hS%ffB<8@U}byq$?T1 zccVP`exV*drJDis<1T_;YZ?Y-rjcbc2dK$&`b^cfu@*)d4kjubl+i^TRa2cN#J_kPZ+{L)x1D~H8MLrqx zKzwl^$#}C1{>%s@9!LYo;Q(H3GY{UnEdq)(SCbd(I>Es`>j5zz1Sz$-c<=oUFnA=L z?7e3QAD+C8-R4dq(p(%9z#hW8A7=o5(QeY-D+leaWie}#!@<*>ESz%42G@u~_&}o# zWHT$^i4~q?W!sTiNAhJz({)>t{=$Qp79l`p%fN0{4EidFklWuB$S0X#SaennyZ73{ zikT0X%ZGP@J1zdCW!eOouv8^4J5=#fm6gCqR33&(gb;zw1kdb`Awo05;HvV4L@8lD ze)~6o=$4wo`px4Q8w}$X-vZ#~D7cqrOyU879@nbiLqfDB0WZ!KN*??Q);ry6F6omY zI+4BLUe!_DI}{2_^OH#+ae($O?gRJr3NY)@D;%XHN7gCmz@sDz-}&i8&FXd|*8klH z6Fms#`#3_gYyP02wH$o^sRftlo8#95eJw7J3c$6HH_ZIg+OXlM8QFRGd&}!^J@RUa z8;p7+MK*jAA<0wbk?HqS!0%mtP(iB^-!v2@uMUZj^sjSCf=~ty&)7untT@W-&PgTN zyiV}nn|XxY6a#~2&V$*q`MkqLz07m+6aV}>2dWzT;3LadKfVy6P|LthHs z?f!#BLVCdd)Mi|L-3VWvr34vm2iT)rg>^Md;er0Mpyr1kY*p1Fmr(&IE%OBZa*nV< zz}=m=DFcrcp2NuzPpH-?MZ84n5LiETBe-He3+_0afeSu_LVv-#b$Ejd^mN}(xH6io=3+5XyY40aG_< z0`CD|Oa;y+29m3xn}UGrTNVnZMUOCjr?YY8s0DT@*Cy@fFX1n_)8U7i%gNK{esJKE zJG5fYP*()!^XPF?^3QM)aVVFEB?nx|;!|s({V@?JBOFHU3HE`8;xZ&{RVQHHAHrib zb>Q5gAo8H70c^cn!RYFkf!h3Lti19N!$xjv*|dHpiTXJW7Ph3pyPi8>64L}8ED~g& zm-T>ofz{x6+jhXq-iPNHO(R47;UsvsA8C@fOw|+_;CE69*w)7zBrRJ`@{XE7T(bn) z+DAi^9SWfKfilQe41jAb<>0T^AHb2F%gChy71BNKPu4_f!9GJi{24R?+$@dZz4O}S zV#Z!z(b$A_i|&G#1)?pY({BkleEabwe>q@#dM3;+o6E>ZZ36rIBf-~WaZLQlTzuh% z2)OGeMxfgaeAPe|Mm|;t2gHAYWi@BMHkt4>(K6_)NBx*39Lw0OE5q8{fEY0s8(rf>|!2IPBLX-Y{K~)R_h{ zzYj~{_uuT{(%tsZ_f8P>dv*jUUh=`M$)RBRssX%z*=tPx9V7`dV&raUG^bf$^JwTuGX9&Zi(gUn%9&dHWvMst`QpQZ5H z{$E%P3EG~l&s6DrV`xA53@quHDPZSm3!=JDf!~U6z>vCv-LtPz!qNY=OjR_5sH_IN zUcU-5m+uAB&0Sh<7~Q94)^|JUMVV0x|2qnllor4pwnFgTiECKv${XNtW(7G`r3+t< zSKTh4x~s?4GwP?)Tg3SV83ElOUQ%@nG|TrQU(2ZLX#pcm$nk}EtPZ_!~nBnL-=g8 zCoDKD0o^|T1@41<@vXas&4J^~)8r*U`hepp}- z0UZmY;KMtyMD5Q?d^P?YXfKzAFQ2}mOw{>6R-8|~&Zd*x&rh9t>Gw>cx)WB)dBJ?P z8WJ$bFM&JnyO>}}St_~o7jR}m$jS6+?~f@R0>RlG(e`~J<9yn2sR1J;%zdygB|n+ za2c-+T#89z=A56-l&cnF+b0}#s2~{Li?M-!^lV|1sGui$5DQbQ17Hs?4tfb@;giM( zs52kCan;+IWZywPc4?HPZaz!{CMh%wFu4FCUWh`&;UZ9H=mM*9WSJTFg#>f{y;Lk` zinIK^;2NzuEF?JqlzKM85o>kKJKTt`9r!{8Roa83-g?~l;3^*JE^z7++*zjB$q~MO zA+^zLIeua}jig?cASnw4tYfuBV8bjXZ2C53|%omYWF)qi+I z1Oe;%3SjVZh&eCNH+~xY1_O(~ffBjDyi*RA@a`%PD0@O0uc($KbuHt}{O1GsKDt45 zR_9?E<#G_#a|Q2xtpymbVa%%z2fd#sc;ik2&-w#@O3`-<&?D-2{jU;ASzMZ!$=Z{r zC$60m*5QVA@uwQ0*HbFqYSbX@b7@ z=w^GOyhV$6O2`p&!7SHa^a1ntj2ezvp~Vz+2$Ng8JMmZZU=sQA73Nl25w%A(py2uq zP;w@XIX%~b(Ma0OgpNJ}*TaT@&W~+aI=LBqytx{qr8B5|%d&Cuk@I+s?+#%9{JIn0 zNFEkQBH}CVN77cO68~-GU`VzB9QD71Py1v6encYAGg1g9H3-h&=N@oT1&33b>`2#Q z6*7002H95e4_v=?1N_@vLp3k@0){-~NQB5XaL2ZolG$>l`Rpnw(xJhFGdVXXTN@2k zD;EoN$q=${<_LCJq(eH<-GDBXEL>bzX(bT1QVD~NN zyV*y)^I{Rs9D9qsPW;1rycBRXm{dE=AS_qJDUvnMd1yh!Q|)q1Jo_<8Bp9|09$8S z!^2V&R6V<#^hS6PkHdjPcF`I*w(1WkIdBnNc&P>FY!xN-lTO6Vq#5U@=#sQK3Q%wQ zA)GXDmYR1X61-A82K;`DkUsuz#{EDo-WM+iHY_m!g$tBPfVCEhJ+zluT_4vXfAK9y z?&t6?TDqn6j5TYl*l-nliR2Karz@|d^b-4Y+GH3+jhB;Ir7egZ!7~3 zv)6fjUI)PNcnwHNmlzaV-a@d=W%LWcoPrKw` zW?eUq7U~2ly?tOxn+(hdlY`5ym|zo4jNNV0u&$yV4(%!hL!&X2Z@VP8Tec83zltUI zFaBfPl;7dcts{7bp)g!wwX$V2!WjP4&IT(?lt@Y6C1$TzFbw(`2<>b>;Mk@xP-&-v zjSpyn&%^GZ;L~^Bnj!=PXmQfv^bWs<+5`pJ61TPfL~)-ryzd$Sw1-Pu{)An|MzR>M zHSWNxa%{-cIm+b1_#5!+_+x5k)pnk1i#d3nwSb4~#fkfzUS8Y78RYKhUO1BL*6fxa zL&luu6C>{v&A|&)NobY@kv$nhl&-CSwCo++nGsL)bQX~G^TsfyM;pGB(1BY2;^DRF zJ4pY~QIOVAh%a2(O;mi+@OAraEN1@$t4`M<9|}0oK5js^9`D0I%9zYC_JpanW7y(% z7`YL@oUmtp@jg7y2UVvBTk`iUfxF&tICNBkl)T>s3@(*3ndW|wO1}*>Dn#MU%ZdW- zJ0cHsYr&-mPdwVyjB8)Wkhcf>@UF9SL0^&rus8Zn#dZf^;d%dIjr9#FhE3M5QQLd$pdl6OnI_y25Mx1s_knm}!*=K3y07$?xrj7yf1_v$Y|w!mvK_DlxC%wRZPs0O!6o&bN7hj710 zVvCJSB5D5k6C5%y!EJtgfxaP_j~aQxvggs*>+=e#rA!g-Gx`qRxFi78!|!l_z!#Ek zYYj3hb%|MS3y@v;0DrTyhw1GvaLRLmH}B~?u(x6{EbDv)yxuM%-r}y%@RPukvhoKu z34DNyx*Nc^P!(t)b_%#SIFoY;jkzc@lp&7Jzn#SCgRUqZB{(zG@KHz(q22q|WO?c`})UxTVc*aXR zA|dEMR|l$*gT-yk93gcm)-DOBRmc+ieNwpao-o`PKMi)S90F^Mg-Pxb!F+I+70JAK z8{ChW1-tU($(lQPVD)i5V)0v%P@auotX2}5_=n+nD=YCqFTt!VyB1qGh(h`5E8wtI z2DoRHfwS(+#3wxG;d#!89I(@cV*Ewa%8g=rzH4vz6Rs`==K7l{- zzi?n;KlXp?OuGB6c(aQ21kAv{)D<@ZzP{tb%2_jreTO*ANuNbD4y(YITO^^X+alnd zlY)K4Y9%2< zeAiRcwLf54;d{8a$++b^w~3d|uLBN(c~#{>HM}Cahu0(UQT$0Cq^`}hz+prbZZk&Y zgUA5*(shS9*i#I)<@JGoR!Jb;Lkhlsx`U@bI3R z3*8;n$-ldK)b-PEnQbB!xNOLPnqG4q=dO|hW3Qg$n_q{3yN)O}ug`8aYE~v^w>pB; zHxGaf0}q+q&YH6tVwH&D?Q}|c&oUe|GZFI~c2M<;58{UfRn2!E=z!@*_JPq`3B0Yf zn($hRCCnA6Wky|}Q16%S0oVIm@yLSTAV^j)3$%ShwOLs~%|9BXYEp)rWumcrbuWJO zWiC8^tPH0ZG=Ol)c#2FYQYuCUFwW~a7#&;#|NIIg3w4KZh)fTVnxBnB1$s{9oDML} z-Jkkua1nf`4gV^Y~vkCwZ}&TW>Yb?jI6``sq4wU zXk%!iD-RE$8z9z836|IzlXtHLx{ihlNt~lXRBxAq+mX`b-K z!~qaj{0VfdGy>%79?)1)OVKmxfbL}r!ZT-qnJ$g%7EQrrU$XI`d9(?|d6to-FAm>Mfh~!Ca@^;=RUL#@w-KDgNe3Sus;igRlbAD)gkS1-q zZm_ZL0(D~Q0_f2#1}$DmK>L#v{Qc=JzHL>-*o>;e?~~!Mgcid;ORaE{-&35o$ODcZ z6(zaCowzfFj~|#U!X66~84XK&@?A%uE0%aQ0Xd*qUYuRXnq}l{>@X;eK}#2*ZKbMK5kZlq8f@ z7{l?NS8$77Bi{Z>gb}U>gN;s-_&J9GF*jZb4sA@t6Vscyw>{^89rGb{4)!H3>;3?7 zofnQW(wpFh_+Sy&s}1^3X#jh%Ujpm){b1F+8gRU_9Bg^*k29Rcf!$IK5@_27$S;xO`pBMcm_M=>e=3(Ir!)@Z1XBqZ7@h($U+9pAEkD8G`|Gjz(0y*D zW1>it%?Gm-XJU^J+rCF;j0)) zd+JM|*JQlC#(-=VkAU_n>hSc-XWUl)T!C!33XnbMLMHUhA)6=pi#96Naogt32RkqY zm4`yO{kIk1&c_%J?G?jQItlI;>6`m+j^oxwSyK8r3;bJS0y_(OxZ9sz0&e#Ccu+9` zWb9WUx~-~2BroE%dtZruj>oyq=JRoKh9vY>(IcJqk|1E~S@7|TE1dphJGtU}9qc;Y z1CGr*-SFKt6Zh-6f>ZKp;MEIdvPszsuH+^QMmUxDdGIOjs`7vM>6@9LBCSCXpsqud zA4q|;-Wwo#`cwSOOOIHHSJYZ?dE!<-F0DS7Z0)9EwH!59C;1Jgt zf*!9uATU}I_6wTv?{i%OQ{8*q$CH19y=z>^gZw98v(X=}$Koz5l{OPUdE!PUr>T=` z^LLRF-3{=uk*q*d`6oy0;Ra}}`wzSIdEuo7b8%stJ#5vvh%+kuxP z@UkR0>_93vVD%vP(+4G3K(fFN%`c!W-5#4?6!}&9l?bUE#(Vx)6JPmspqhUke-nF# z|BGKE(qs0(zm=!JRci(u9a$xMw!64lSr2U0zAs4Q?#5W^Gv~)wSK>MEqsS*bgQ!k1 zgVBn#seSmMV42%#;8i)y)r=VgU#wd>Nx@5CO4S^)Y>p4nc&JJ?ytswEWA)*tZKi^B zm7i-Y-!(Rf|WETAGW z5w=b@g(vn&lJohp*nFK7DOXc)GEiDbr|TqARlYCUj}KbWQa?EIIQ|-!nM*^2M({i3@X#xIlQgc zaV7U8Zn$nlEWaB8=an`1(MpPo+iv5-HlOj~S*5_(|8x`h^a!`_`6G~i{{^cau>jFB z$>8Vp*I+n@lWQcoN(1FHg7YS zeBTL(jo#rFW-NfK_DvxwvR+LGoNZzEcPpsX(13#^3%D8K7FciBHqllWDR7!wfwwf( z;>Y=xq|ng`ewlMfU=Z~Wx87Yw>OQT3hmQKfx{|koI^_V6YrmGX_;-QzLC-l0mTEWc zE49EYY+i8jS5+K2bPB}3xGngPn}9Fts*?cTS^T1)8SM3G0$b#ah;`gNBGi8d9)>1D z?U^-ASut|(sAdW&uTg=q<}y$h>%%_R#W?W#L;PfgF}N0$hZjt-0>ySUxXI(2KsrkT zrqov8#})~=ZQv!CtD_3@`|Jf?m*vT>Qd1zWKN+6utj0Hc1~{vpi{W)+e{stM1NcUy z+jS+&z~}{RK*7PI4Qv{pQD`B zU(aB_5j(gddem4?j?n@M_p7XGM|gtzFY1a-PBZW=v>nUjkN_n9YD zJbMLjiq~`JA6N+LwuKRAxz_?z+Jtkp!bpOJH;g^&0ymUdV%OO7SgcE#FkT{!=-M?f z*w+g>hGxLt*Y7~g853eLX9=O-%DKA^bD&%B9sEJwNN{P5AyH_}2R0#}1a_OeMZYy0 zaxmN*2P}JxTVJ`8Woo5BWu_Cr3&mjMN?9mV>_DV6x!9vfR9sT{7FeCzir>#G=kA-L zCW7b{c-;62(6S1H9m$r&DBvDGZt4JAwpl{4A{U~XGYJIS4`TlL zE`0Yy3z&OMjQgA0fm>Fo5l+Tb&=w;B#g3M65^kOZrdEFB_j47p;@xB1Z7_wHI?sZJ zMur5qV{YzNe_Vah6ZYSkfM?-j+{MZ31>J=Ju1pr;gIr~@KYck_F=`HPtS-jOM1BU# zgft-Ast#73%LJ;@XK?a1dvZLvQ#3oR6U?w%OUAn+NIJ&{yzrAID&MA)_W|Z)@eULC z?w>Xc@lyhJ6^ihj$kX~>7l4HpQ^?#8T>N3Q5#eE0KV zQo4K7aGf*o*Itb?zVvgai#VcTy*%#r-h)`_$3oygS`4b<%%GD`AhhV7PfndMfn!#E zK-1+Bu;1%Q6ek#SV=QVx@1=K~HR5N$Ku`ajuUa!&R&F;OW%w+=TVLc%_Rc`R(Wh*R09|Wi^H1{=^&+m!e9#jVO2g zh$~*;@(bq_MT7smWys_>wyAi*O~JilOBnp7m237$z|Bk!0G9`iNU7sE-ZaAr9!#=> zwUSvl;ZFlzT`B})Ko%a@{uS&w(S$)p7VcGXAi}MpoW;rptR@$N3smjNj+)I-@83jt zulNYoy~QU@mi3q?zYOcm9>G@YR*-iW7sI?Ea~R@t4oo-u40<9~5$OYZaB9Ii==CcY z{+G0Z;Hj%YWu%Ckv6dr$-xLcL7RlfOJ6oca&K2e72ElugHxJ1glKnNi@F5ZRaK=aq zXfIYI^^ydPw4H5oZ$61%6b9gx!7H#@sR{{q6yVh&9brAn19$3G!TuNff!4IA+_~p= z;`bKyz$f(=_)}dY@**yWC+wxkg<%h3bhS#5@2yX2R`f*V&Oi;d0Wv-y0U1 z=YwuRJpS|09C*aMz_~{O*;`^omMSbJWji?VG9ypYjvU4PFXofzJR4DdWIeQhwix6( zgaM5^UL^C-kl^RMNbsfA9Gp923V&Z#A{xR;q*6zTxSQ_=n|sCJbzK!${nrJqdgf0u zzPkdu`VsCmrB!gTc?LE!SSCn&Q3JTwvcY5CRcv-E9XD+KEqca5?sQ`z2n-E|vRj*g z*q!CD#w47qi5eI2;5i(vb1I~?Lkv13S7Lt{NKTyO6C+^)Xw4Oa3exAf2Su1NVTC@~ zYbHaE&n^OUkBg#eX$<7?2%g;11|H@95mX17!F!Q&-~x|@uwRY?-@UdX{eK^TUd>U# zMps$#Cg%-Te?<^cy|S0sBrGA3PQ75${Ke$Y4OQH1UnMYkuoUWq7GeJHO9H3F77!bt z47c5E!(NS7@l5?xfZKZof8S3bmxDiGdy(ER^>GsXv#sDnm75F1`@vH=mLR`D`85uWUtMnX0g0!>X9yfH-q4oQ`Q)>3!!-uoZ!;JOnnX*H<1 zr~~XCFos(rB#1)YHLUqkpRCw;28ZZofIYeLg2xlR$^GR&@U!b_+#j4_yfZqOxQG70 zQe!UM`e+5hU1kOof(6_OBAmz=|)B6s%B9MepzVR67$-96J zPJse8d>ZMtiUJc7SHri~RANk1{Zx?$(Zdc5Oj1h$N~mX z@1QZfDZ1(t*35uM4n&YY86WU^D+i+Or-L)zbO4t35m>L>3Cv^Np|n7X9F&sOvwQI! z{YRjElfOXwX)z9c_6U3^Tts4+J-8;ofV9WC;E?DHKvj%k_i=kDcRmvCiJSo+Op|CZ zY*|G;BPRWmjl$6eg?*VR`}O18?x@I8p%wz!?Ti4a3{Y0gbAk;CwrIUmGRec zzzK8mZ>R~3r8EhyUic;8l|BLdUUjJUc>%P_w1VTcGvFrAYH&Y*1732KaNQbnGH(K( zO#GJ(QuH?i=i4$6iujk~D=h?V`J3^9s4A@MxDQ0S=K!OJ-+}EF8~7$82nI{5;D}El zaCMypT&K7Op8k}NS2SJ)&o?ue)r|q^6Fl%jCo$;noD4o}R0Yqm5uP+X2t?kW4Y{FL zK>BHE(){f*IG`~Xez6FJ?`Eh%wrc;;P}RDko<2~bW=9;!UVU`F3Y zvO_*g;OothV{2{$!!;v7ZpL#iC&C6^I_U;oKba8m+2S~zvxz8e(!&pZ2)=y09_&|F zBel>A9&PuB++#DK$&Dm%Y|nGNT6_km-CzTH9%zGjNk#HW#6y3LnMy*1VNHh~?G&t1 zoDcj+E{Kg+h6eOESR%^t>yH1%zqIs7_62n^BEOZqDv2ZOonpxg6$p0}91;Y4NFcw= zIb^odEy2;YGdStv0!R{Sxmo?mzz1-!TQUOwh0VkV4Md?{jyf6lS0ts$6G{0x16&fa z0BVoVgyjcRAaD3H4x1ZH>iR>;bfF&!A2lV>{g=RR!x_*VNJAfyH*Cdd8feqwfTR&| zXj@$+FtnA#MIx@Rpv92Pwvr}qr?xoOE15&S=0s>ec{#KTs}tzYNfl^bj|X-OMcMKt zTS2a?6>uL^C715m;rZ=zh^vn}(YcC9{kts8O|yXR5#jJ$k1o7XQVHaDug6N__i&Dr z9ck!l5Uh+30~e?=Y%!Pznv7Io)o2SRV1_;!o+rdRwn@Ofj*&2bP!am(ghOk&G_J1o zAf74>iFM8p)|Ip&NA`)6D{-$db4Qj)##rK-bs}EuM-V)PjA7v)ZRnw?N$TVrNcB4w z|9k6<@6RMVg%k_B1jVFsN4xRCmJ(Snuo ze?&QvR4%(=9`_x;9sEvu1=hH$g6QszAhLcU38_B>9Jh(QMUUQa$Ma=~-NM^=#(4wM zedC0H`{_LQNJ$j1x0#G56xHDUhB6%2YvuUnH;twY>&4`IWw3zpGQe3bEM9W-1FpLm zLUw+hK}JJzF!zBpsdO&js;Ff`^J9TnA)*uPZS*EHqL&C>ao>aOcSZcvt3qrqlOgbU z<4G*bqOnp*23Gv>3D|e|5TDv+Ji9}Kth&3LREfOOL5aVxLEI}~?A0%rR~$-0*6~Tr zHUn(BnGav_17P-a1YbFR6l`be@#(7e#yyWD1x}5Ug0K5$ajz$Pk*;M$+zZW7Bx6lFQMfm_9mPziu=$$PwD*$X^5`?Eks zCKESyw}Sd@bII(PO0ZUO8)#W$2`8K>!WTr^O8ezhoY#{LhB}6EHE$;QD00j!Qcxpj zJ49I_I*sfQ7l2Ta?!WV!5()N{z@PO#;kud$@X(|(;CQL(Q*9GD3=@aX;M>J-aO9jHW_$eFcKe|h=}slnXn+q3XUz`4%{4X;wlSC+^=Dd zi)7DWJ6m%i0hHkC*Ba0+(+AG`(Z-E*oCQh#3q0oLN-Xx9({r<2f+H!;hSsDM9#WE5W?H2oQVz9gdSo z!0!%B;g&lr1P^vy#11*G;LFpE-1RMvWM%I<5LJE%8yII`|F5F#lDU@XP8kIHUo7EG z(cN}=C=@1E=s|qh6+Zm>0FSs-g3~9qlkhYh($vromR;n)7gi#!KxYs*{x_K@M<#MD zi}Z-<-2ptSy`F0+Z3e}!MS;SkUUz4e)3!d)TQ8NL!8o;xB?4&_13v+Ye%H>!bAu_Ye5o&>&& zvN>$E156WIk+CbA;pu4tEZ%gVb5`G55GYRY-RmRVzfGP&(Zxhpe=8Z)P#_WCMScSfFSskM6+A1EC(bcapmE?F zH+)+(ms4vlu${OJmQ2;bKXV`8A&sRlLA0YHqSqQG*k1v5LtXf9TPsKZjsS=4*+>@j z%fd;8XYo<35V&etG!gueCI(CTn~b?WFk+q*iP|F4@gDMsa7_*;+$5i?rP>BE8@_|R z(I*9hFLAI!(my)@LNLs@kSt=dUH9M`^N+R_b^wq3kkuVEH|Q{VGWJ99t0Ibvca6Ok9A1?KS6`VzYG_K4U+((f;T zpX}yC1MV*DqrM0hwyF`~R8x3Ou?K)cNr;l>kfA{pXu3p}ob22S5{74!T~61)*Z{&y zv!37y-VX(zb2q^CpF)V*+Kc$kn>L(f+K+D-Y7mZp1-Nc>mK&FvM7FkjlZNQ!q^)WX zwD~U(mWpywL7E8Wa6~iz#%f#@RK@kUq)4<9^nCIp%O2$cu}C!|xB{P^DRT%vXf}M~(sNHV3Ni;*q5AXfhly z1!f-k*x2;K39dml!0W~qVzxJd%FQbj%s&y!OjS2VBt&Q&26)rv&^%p36K)C=6D zX9W%3)->6HxsY&<2!!hSO{&gw;7pfg zmo&}~15n@g3_cg|gX`}W4flw?+xf}Q!0G)jK~`87UUOIj>U;mee?@vt%d9D&R>u>J zfMA$Cej7ioKMaOHP9iDU3Q+9FMr<$gSUq-m$#sYq;3Z4W;GS6t;C*pB)|+dOMUCGA zO{ECIr2ZDcOj8ZA{G|%G7X;vs6+Up)yXAPwuN~knM-fKvVnkek4M}2JL5!3?8Q-Hr zOqLwSmN&nE8{++-i=7EKHO?Uq+HE*SubrWhb3>OVP1k;da=0P>^#>y(poT_GXQ?a9O5XIQ||YRa^Ufc$>g{A zY7n|_3$(Oa4PAmw$%GO=xLuUhv2&dR=Z=MtfcYP1t4O{9O&cpghE^F?+fWLYG&sWH zH7Ou*pA4B_Z->9#Hz05KB;Z+>r@;G35 z>~r!at)a(Y;Y%f0Uo(|lYq23KDgbZ`ZwJBeX5xu1Z@HYdI1utz9xs_=0T?p}+~T(p zguePI$X8W|jNfMXdR7G4ch#Mwm6YHGp^(Ih{P%Z8o`AosC2T#r1X{^l!P9o`#Is2e zx4GuA;JbzaxI63!pG;SW;qn-)UE74`r#%Jx>ej>77rx-o_AFp~%N5_7aguw2XuzLs zR|LPp-U_0Y%i~k72BdF5iQskixF}5u=9*^W%e$ZA;n|L4<|5I)B?nj1Jw6N0snUZZ zE2qJtW3FHaKV7g^qyq*ww&QB`NksEwA0STF@Qtq({CCkDZs0BEW-l}$lGuoQcWtle zo^QkU2B&Zzok!d@y%oG!yFoCluT0q2iX_v?k+_aCa9u+j{-7_)a~+5SY6?q0(MgfN zx9b>qJ9-Nogl7RE=fT{w-#~lqRXikLi=(|AVEVW{G!6@dk2uSTyFmlE0R|~H2pWk@ZWDHz4pASARvVd*HZTRbxJn-tvLy+z@A1*<2$PVv= zpx@Yn9PxgPi|v%KcvK2rpLYg(D{aHAQVOv4kR_2jo`b{LPk6<6D4h1T5U+W(d3Nkm zE{-yZ5cx+Iz{xq~oLCi2fw*4=I7#)PVMa3UxcC)cG1DcjNrO1=_)N%aegsZ`J1e*? zQ4Y#>J9BNezHHKYGYFm?)`BUGmSkqtbXa|cPZDMulUji){B5OxAI0=z(*rdu4pZa( zZR%K0=X=RUf>_c%5BI)g2bjc1$gh&6wAzshc^FGb2%HTfwuN6{lyFWR*G zn5cK!hwgt|MpN8NDaWS|bv0KYhm6mxLA(?7S?0#PFIkL2rkz6xIeqLU_Y3ImmV@;0 z*gqJQ{(%WeUO**(s8h$ZDb!<(i!KzV(gpe<=-HbY%#(&9-Z#H3sPAVs+HuMsS#7z* zzFDtCo0gwsQ-|e|v94$r_X-EfnOnjh`>~AbO#Y8qIORHPkyXq9!CSgsY$@VzRcjux z%VC|2B%%gAEU?*1_* zL9zkaNK8ZnR%_S}?LCaxf;Ws=$x|A9%ZLYWh*5`?%6!wPQdazik}!NemnpdU2ra*} zp3Zs}Lzf5s5aH?$Bq25(ZK_hIJ!@xE8TqHo>c=I>Pd^9+cCV#rKitvtGa1M$Ru)N( zKB4Uoi-ptWJlM*Q3dp@Sfi6}OM`{WFs6?z6^`*H{-Je+fEsm8Kiy2am< z?Jg~(5{6INbNPj=YxNDZJIEjH&Q@x+)#K6~nV;D?uU@bYvwra4oJ*`>+>mg#+G)C5 zOPx7Ul#a@dtJD2mN7$Rt3KjL;p{MEqUA@(X{UIqsgZH)b1`gKJNBu9k3n!xfaEGq!rVDS#s34 zAO{88XP~P$PB3AUv{0_t0Au=1zzRbRMdxgSPVgveUTfB@A3T$`nE%J-x6Gu0zf}0k zKOJT2`Vvvdu}|y+#jlLYQ4hMuZYMKfYmc%;`+jwr6=+#^BocT3${u+WPs8aEw%ZgkbL|zF#zINDJFOK>leA#F3qP_6k1nwhz157#?k;+&>=8_| zc)+d|bp+q(xsD9eO4;1$_ZXK6KX^vsCxw%qsnLinw?$5zXf|L_kFGy!#wsegAj_F@ zREoyZu_%4s9&rVv*8YK>3OhsTtW3tkvYiGvy=QMY7SXyR9OPr;O^bCEsS3x2nVb+# zi*+PL{Q~ENPYqr%if$78l=a8ak9D)qJJlK_wz-PFc-4hVuA9>^87sPVxKrkxHgt ziqD3JEfY>pRi}2E63z2v$Jy5UFKF|kLzL_AkBu0LWzNU_p$i?(BN@ft%<0EGoKl`&7F#}OwOW6$iWx*2ad}$)eZ;dGQZijI7 z+$i?^)HqroCy&~aZm~rVC8(+j5p{;&M9-@aGG`~wY;KBpz>eedjDlkuHIa`*cv%tK zWCa|LGsXRo#m-6SbXh10PWX&EGXVJUbe#DG#L;&pJ9KQv z307n6WK^6Zps~uI5Q%xgg7pi9^ZYMDF-=pNe5sPRBk>hlBlMxpQ}kGCu#?STo7k3a z%5dGTQ13~BROQ|vx;lFjGSDm+-rCuLqV~)}k7Aw*n@oJ@yy{>ucjP8gTDO!QC&i); zuWIVzwjWh452VrB8MJTCA|~<{2i5+4$_q;lK#NXxq3i$T>AuN&+cnlN)P-(6cb9^KEJmuJn!2y8 zWWUcWqkHDx=QZBHz!X$GqPyR>qR46)q128W!Y58yC|%Q?I=@vw;ZFTbmQFUx+!)HP zn%*JD^{84p80hpRD2B zmuQp6MMigF1_PVpMBO|$n6>*<5MVl3gUdDOLSVP>`pU0p@YY8ZrQp(hcytfjQFRTQ z`sdQC^#EN++=`SuOqp+AH>2HQ9QrwIke!g2h5k-RWQ(8Aru**tp+0d1{_df4b~I=} zD1TxaZ-32E8oIZS4(Pn5rs7u3Su0o2wQYSg;BWv?=Sf{rCxJeEfn z`@Tkhx}=z?q05ok=m`DV89{G6`oLx!AnXS2EjDJ_TDo=$VkD0>(&#ZwCN8ZJExlsL zUiTYi``opu@5YbFI(z~e?P;THzys#`{A}J>*9T_QF%`Y5`QjNx&_^^>m8fdyb9&a(4lLiJQ%qc6RO!Npz3OcXywX{bgE7& z^@$o14j;M6ULM@e0f+uW$$AxUQ2DHS%^ zemcsIlSHm+#x!_!7i)52GHZR2v`V->Xhf$U^8R&=Po<7MH_vE z$>^L5K{_J(N%(y^8+Wi9-P?GLZ7EJgyz*kYQMp98^xqdc8Lgq*pqES>=O~)5-$RS# zJD5Y=4NQdRZ~8Ggnra#}Gc%uRF^+jcWL}WVMwr}S*~fb5?UPz2tbYYx_}#4e1}B$B zr};6K2R_08-D}i)@paU?I1L>LJVs|}F)UfO0)@xyVwSF~Mjcz8F@xIv?C(3S$ZLfM zJ(McVf)8I%QALT6JE)7;wlu2HQ;z1uPGr-o%=kaVz3CepLv*}u4;@jtN5}K#(i_s4 zF%GilM~%E>V&=#p50jqogUA4cgg-q)Y8YYkv6j^?n(2bC)!Ki4l3%A4kaCjOa+ zOkYo@hxPyzvtl>uXpo}8E?z9hx|~(wZ>PpGb*yB^G2w=X>zRWM8x zPgSsa7u%?{e*`<>vL!_oN0_6b|DlkIBWU=vDSLEW0~OssbZvo7b4gh|@*A?CqEG_* z82grv>vU5m-*fDQo4IJZNebieA&LHs-66EpQ=)_XUC6OZkeR5d_>A!Cj{#RQ~txs=f#_KMzv%aih@^Yu3s6%!rK*1VKOTW!d+Zjk# zw@slgTf|UX-z7Av{|w*U-_5Rk8-#Q_9y2?AhMBw4Tj{14E?uMFOt&kYqIsYHBCm%s zjLo1{^SY?Z^j!2(Hga1g`rDyL9TvthDo-+Kok0z~6c8es2QKkK6-7t4OD(N`&{Ay=Vw+vIJM+04JHRV5j`G~pb|DA0M4nR`{1-#gDh2|2I zv$SkZJ`=4NK>uW?GSe>}Wbk#-dwZywU3I^jhFM(VX*?7%k7Au^Qn@@Lsn-~1Z9~?e z^*HmyQjMQutwTp=$smU}Np#`8Ec)S5AN%vjUv`?63T=ONT_|t4gie!eX4B3$(TveOsZ>m=)4^}Hd0I%bxP67fkRYWGl(WEkrCzwHK3orf>8OjHYSga7A7lPW+!HC zMPk`y6!?mxuTl}H)2W#q(U7LjPRFU_>ruui{uwW?$Q7x6@Dic+(4+nywsO`rHg7G9&Rfi5oA=rB+^S|UwXd9*=KJ=nRf-C2 z@Yi4;&vT&FPu4tS@(FJ=mwBn#UlI@*NuRa-M^~UV!i5IL83Kr1CIE~sVUuHjBb7*|E9ty7B zO!p@3V(YdW2~EAe(7oxan6Ju{`7w$Qn64yw{_V&k$TO=LB~I!W78eb(^LclLgKfXb zuZ&X0x3Ge`Jq)FNDx2}LMHK1DG@^@>dzkiVWlUSw0GsnvlluPgrw(30Xw`jD*Ntr| zRezI$wjL6qK$`<>Mmc75({G{^uBTYHrgMzUbxj(OV2I{?k)UTT>9DW2sna3ZIP^;X zETgnTf?BJkAQKWxS5Hl%$`+o8u?=FTeClIlwy5(j7$u?6UlPsQO)t>OhAs4{>s2(n zY%dgIN zd3Hwf45YPIk&W3WhT^!Xs0pXis87Gxvu8qS;!q-z@z~3=ihIITJ3mC4JMPe;=wfO+ zaTEGkH;+CjB8>gIdUV`KlFo_#%1FB;P|1nYP=1#jd&d@{nP!!=kC{OGnU!>b^=hOu zpCH&5$z+5*K=~HYNL~018U9H?D+j-!Euy}^-Q|l=koInrDIG^YH``O~&r5{i#(U}f z%n06TnJOkt2b!%`#3${uo@Zcl}4s|b&SnlJt}vhlUb2`hMMnnq%oV{ z(fajMP`fQOhuSP(9JYGd&5e$<)P>P9ZX8I7Ms028D0C4PW|lz znW;GyXl$qGy(*tX^(svGpUW3h>wR+IsDe0O%;*z+zw<7Wzu*rYkzFXvzWx)fTz(NX z_o*|AS6{H^T1#oMr#t%IFqN{=)rq(2ykxb~%=cgDAV?XL} zRi0mcX%&*3`HcNuyOM4l&Zm~Ap7N-x1Ra;nL;p5@XF6kVp{zG&(XdMe-NaeJv~)~n z4E;|qDzb%Mx|Jy{wm`|bd?uh?G+SyqqS=%7 zA_XsL>OPw=PgeCZ<*CMuesM1?j9-lk<;LmZTkYucj``GTqC54V18kiHk6xRaNq=-? zpu+#EnfHh0vMYAZWX?GDlhly?RP5<2YNURQPCm({!#T~uOZ5`Sw^@P~MBHF?7-dvx zXN6LRwo?DD0QMNC5J{uG$Uh;QmAcnT$%!y#{MT9b*(!0q#qT+&qx~038v)c@{tk;( zuM2HW>V+MFPkG;YcbWC2FWIl3Ueg!*vW0p{v*{_TDn{HTif%fS$~37>;=8wqvw_YB zk<7(&ycaKyqph!|G^>|Hq2;lwk%Z|@x?~ZTHVlNL+PlWhv2WU&?!VNg^=}Q4lL-5{ z8s(uM$v5dD(_^SnI|p4H+lrPc=%ZZU1p04UI=wd9MZ0Z2GmZEzTV<}xzAxKOhmup# zxtA;G@H}?Cyq}!l(lCo7a|&tM z?MUP`i^o1_h(#ZrolwX-IlBCgM)UE=Dm2))8Oa&;(L*9xEXu^6vFx}@U%lqCdY65K zkw}ZKz9~f0dnPnDKlR{$RNRQBaFpplbP1f=o51vKZKfA3eQDyqqwL;qRCr>*x;d%W zn>k&zlt#_dMuBn@kl8Lh`k$>5y6_@|(Nr7Z-4@%zR;C!!+|QGlFXih{sJ#WUF4ACK ze`M3B#g#}&kVB352k62Rx1g17Aky3%LjUAWZt3Augq7&t<0F{?7mCiWnwkw(%igBNPF+{<~7g7_~w}l>3{KY6z9AY z27Wd`tLOJIJ9n6)ugW>pGBO)YEt5kFw}ewNu!Y8m1B&`@@w&f?vne`VC@OasOKSTW zF)2Uh(w=NO5Lt=D&ZNQE{kG_?n<-M~uS1=1E`9#DlyAYj`CnS{*s$s( z_FB+1dMC}BN%~}f0^>xzm*$j7c?z4Ymo^E_t~&5Mu9edtA-jZeS%=WejzYBIkQn;7 z&4C8*r1a;a$tc*_g?4JQ>;Mg67J8>L74wB?!R2JCHd{yx!z`&$Az^}wpD+f!vduAV zr3~{g2~B>jOs7|!qDc{hjK46B%@lFVy02Z($--EosVkr-q9!F4eX|k=TV=|ca}8pn>Dm@(b-Mw>3?dskaV(uO^IE^ZaFhTvr{Z- z<`-{zuLC2i%t1!sSt^@7^qw_5u$-;02tbzx7t>$z7w8pO7Utmc6g1H$o7G&3 zkWyR$dJyrMHmD5o?4~Co*$M4*^lCW0w#^l#z65A5qey3b+Rvo8OkvF2D%s$KMz+#J zl5SR+jRX^WnNJf{>0r|Yco?%d!l~t<)~x&CH9aA%ene7bmg%$q;XFN)pqS;Y|`A3ZNnBOHaeZg z^=dHM4L@n0+EnT%e@2);C4iCnD`5L;ZP2N{J(TVUKnC~LGU4Kd$n&EkN(l;M`AMf3 zMVAKVaGD$2J8v&@L-a2^s!&aDF1t%ty_DcX%L=sVVF9D4C&Le^KTo$U+=UYTjoEVr z$5~E;j&RG8%dCr00_)P6geKabW;MGH)8mON(58iQtXsf)rlldB-V-dL9-5!g{$~v5TWh&Dy?Z4c-e=bDg1%0}A9+a(2v2q`66)RKGM#yaZ058pDCC?y zvUs%{|JoIXEYlq*keox047Jj0w!^IcPA`TZDvM-Xg7|V37U)ELn21|arG_8cnJ4?4 z(WAi!RB7l5weQ-=Yn^ZeIT?*nK;=mI3z`iKo# zCP7>E4A^tMzmgh8{Idwg{}CKMYp!f)B1CI zH0t@pX1mjURPZW^<(xUjZtt2X{Jz(VDYTBKM?b~E`HvbI>lueoM6ESjV*HU+%`8G& zn%>aVocM*Qv43VdA5ZkcGnFcTak8xPiK-H|C z2-jp?W;jdasmJIny2-=_RUW8d&&BBSw~o(btwxobtsTEHB4Q6ZtL4%ixSLh$IZF*n z3ek=u2iW-SA=LI`8Bb!mBQ^626`q&pp_b7*WbBw3y1Kjw2}}N<5#LZWGw2ISQ9er3 zzini{JzR+H2m;vz|98xRD*@DC*Z^hT`^oy=?-iM=;!uAkVhiTyBF@o;wEg1<8@VeN zwY(i;Iu-@77miJCF00F--``lG=mcrxQJO-fh7h$mxu2?+g|L+;{i%)W4kVlBhm585 z(Uw$mI>Bf;TDtZ=YdzndHpwP2(_FNul+$CzRQOwjtYmqMnGD9@V<}2?Ge^$1WNBM_ zFRQl^(G@M@Y@ujJ)-8`b`3-j5iXHc%W{^}xD?!h~{^y3xQVLFe_f9^on zKYk}nkE-BZ6k)xM^~b4Gk&roZvYUPy`X_X2Xhs_xc~oBNg0MeJh{g<0h&tFcP>ja} zbYt})YCorz&fe9)sMXx!J^bf|M#uM|^U@_u_JRU-o5le)nk!Fp`tsPqf9~{AUlUKy zAceiNY94Rz@jHx8ryTvy#Eh?ERvidvO4P_dL}?Gim#JhLl&8?f zcMMH77e|wnqJ^ps@$6=msltP!_4E>+(Cj!b6%|-*K&!k1*l;nc=5qmsXwTXY?DkCq zNdL4pxBn*;sEb-rXwu6?}RbekVL}_ zw74}Li9Ov*^)8&GCtRK|J_W^$_{N{KY;rV{;`o(qX)vH;g%Zs-?(`)FQ z^6yOZFo$ygIn%6fqwMpc1bR|4kt)Q;GroBR%s#nq^tMI7HDP61M$h+$&6|!^;MyHzb#qDe;lXq?`0)Y7yEnxgH%`7$WrhcZhmC zxkaOlCO30jU1{RMFjTl4(BvOHX1a*ux97!C-A~idOsOyI%B7)b+^Uh}44Kfsj3xVi zq#ka)c@LGHIV1c%>P4sf_)<7~g!Mja!Tz#o5Po@@iS*b^x<%F*@k`>^g}aw?8t7W2 zXCuu>OK(T_8~^9%Ov9=A+AwaOGGs{R5=EpUa`xJ1ucJhgCRBtHDrr{g-yD%4p%Br4 zBx9s-_S$DJgp@IbP@zbYQW6@y=XyW-cDS6io^{{%bN@7WuJ;j4s4;*FN6q0Hi9oi^ z(HiP@FTx3@X#7`QsLSpDj(0D}!J>PW*qntMaFXLQ)?@KDJpITAc6WK3Kzg1ulos-= z?|09F1~mtu;Q1%Ur+p3k;D{W~_lSjMmI_$r(j6@6a1*ba#o&e}M5(G7;~NW=u-)ty zJlpp_R?Aehd^0J7pRS6h+MZp3&Xa}Q2Stma{m>2eirjRZm;aI5BNYVS%U&Y$MVi<> zPv3FPq_gqypS7@gy(sUw1;ft#{)z!%l3Wp)de%$gF+41C1ux!ZEO0#_jm1k=vO5|h zsQ$!k{E0cmmD4nZ9Qkz0!!-szzO@nS8l}Qrt+g=tJU;A!{*-42o{2;|97DvO!UQw*c z{fqDkuM=i89KrRqSps{0prH3gJ2l5+0gtwugnIS~g7>{IpvR~)D;HSGnwVUKKbPBc z_2@3@y|&ONb@gQz$4QNgk18SLgN6+F4>4vyM2E-=3|Lgg(P#%qWt)PniJ_?(l_Km*PY-N8 zy@4IxAO>OX9rp9n37n#-f=Pqx+#t6%?BVl~7?%dBALL9-5Tk-rQj zT_A(&rG8_vo4Ig9Rx)KoMYI2X8irz{26%m~d^tTPgF6d#@uvB=sdvYIvF}&EfT|lJ zxJ#NOdF1`KY?DVXjv3ejH=VrAuB@g6{bx+MBB|c&h6Srxc{>-(TmKAuZqmb_yM*j$ zt1XbcstAGqZR~fx2=g4{;a5qGa!viWID5GlH!*P?ZaJ_V=cR08{myW?``MX1eu_|s zoofIG2TAsaw=>M^mgR*%$b?M+@%U-EHzg^|#O!uS#d)_6;jROhsMBG8VArE~YV>0b z-tQ`j8(qus?UN6xp@tDldfL*6QS0)c55B=x~2dh(agc?eN&(zZx~99 z?&9vQyFq0odchCR&QOQGb+cDTov>ch3D|4bMQ)R|W)AlxKpWBl&JQ|l*H|XJ;hCy`zZA9Vmd$?c^|(G{J4$ zs84zSEunt3Y4UW>7*Xl21MJ5zW4M)*h9mv|QX}D-a8N&(`mX$ga%IKKxd$b9*KAKi z**VXtyFU>=wm2AOFDb&xTG?1+aVMLjUJcuG!(d`$DHCAXO&RW0hSQ>Nvr69Ypr7M5 zO4tU0_lEo>Ygacw`=8gi`k&%4lKj9uex;lGuW2c@#q9vamwt>5A02~-tCY(B9asg2 z*VnPmCq4;mg>Ug`xAU>%(;i%rZHrHhU4HhFBQ%rrasum;w;QM$EUXK z5ySDpQS1wjC{FWI$B7fOVaRt$d=K4(t+lljwfh{`X6;4%s`njs-`*qe)kwv2<=0bR z4oUGkKB@6?>gCGIy&&E?KMHo-_zy}_7nooZp@-e1#(Vu&zz$m@Htmff+?lNci*xf} zsbw^j>iq~6^fpn$F-NJHtu?}2T^l=fr4%1PLGZACENLKn5nJ39qI0ISabes&cvim< z+xMQs!7lUQxBoq>=_Ue(M8g4-By2Z7nO=*#aZV{$WvN@$v$h!|+VU zeP|e^kF#{jSgrbN0v~Bz%5m`r%EB!YZj;@C@A&d@^t%G+3Ky}`H>{wG^*eUY_mA+` zc28`#AObH|$zQvT!C$oiilVW!4F@ zlFjf6?mcSsK>_8PrH8jnOi@bxWpKuipHz$50F}P%Iuyk9;i-nJ)XhJyDNS=(s>aob zU7OL5ySy!-q<5&`&U6lbtNjd0Kg+}gN^jVgXT)HfM>?zJ+KwmR#!|KA=i&4!4V<$w z4=$abB-D7Mpu`r@@&`%wENh>Qm)v{KZqv-fMh&*Oe#{X6U95v!><+SLqBGb{{((4Z zVJ#&YFV5T6qyg_t$nco-eNcN!1CHM}V$)+IaB1^f>UC>4rFZHa*D53p<{CKS@w3@@ z^9CpOeU=90q38si#hl>YY`yZa^1%Pk-Se*d!pK1zTpFN7xjEV4sX>NKSf39cQxN`+ z!faJ|KR0J)nn06h%8G}*1Gn22;>4XUu;1ReiRf3kKtGTSWkrX-tPlB5&aH z8%J4mL<_f%DM3F8;Y_aw@UK1%R{mf-^;b|vwcNFb-($UC`LR@X%uo+C-y4fX^+t^$z$#v=sGsr)C>cg7JOe21f@eIVNZVnb}0Tv?#;b|m9E?uve%8ccYeiE#{b!4 za&Q;k02Y)tte6J#En}dI#XDRt^l?5%lyN21Y_W2!3VSJY8a%iz1}oJ>P%b&=pv2Nf z7~H-=Fd280d*iz~z8)Qf&osX07Cnm+{vTh&KD_~Wh0z7U=4OBRR^udl{BIgnIQt6a zz3D$Za~I}aWxLtn zyN_^x&rNo`S(0H2BL#bo6v3$NkyP{I1ia_od#>T8FR<@}1-vz94W4u70RFiA2=w0~ z%Cc%jaF+iFx70BYf0UXK7=QlEMj$V&8dnE9ILYwX^Hi9yJptFPUk2yy!~$w|G6g4$Wn|@|@Ma_pDOQf(!T)trIYz6M? zhvkChs#h^BY5=PP4WM7oR-9EE$~H@c!}yrF?EEZw_Wglrtb9otzLBg-t#Me5r$5<) zXLd;7^>Q`r^RGvt{^xmA#JMoI&U86kn;J=Rq!g*Jb`hB3`2bTv5U#s@43fuAK!rE; z@N&Oo`K6Toe^0J|I1);a$Yj^_hK-ftu#h0kqLl>m6xD+pc_nD@RaRr2qP|Et)YUs6;##H z`)t<^6KKAm5$;@~2%U`1;i@>%Et#BlVR+ky>Ra;JFMOH5I;F>0ZpI(qQoYoakY69 zTeemM?>Z_caLOB|-tM>v{q~CDmyN+VrPri<=_k2zF_R9~NbedpHE&?^-zxDI?2coD zUoFR*6FcyZfeiS`ItF__`bep1pTqvYmoq074&wO6&s0aEEsiC3!`%VLv1p(+9OZnW zHaf{unz|a$=u;ycbIhQ;_$S$K!^P~c<8n5Er0LmhWEWwKz~QfQ_48*Z~$$i1|_ zL12iop{`6ErE=shUSRiw?UYZ!y@UD`>ktQf|A^o<#YTA1w?qg7&%w&1SQvTGp6Y$& z!#+9toa$Y(8UDMSFYq=>z`CxjZ1vw5(&?FrAaZd8TzN#8YbLAlhRmx>sMQ>*^j$W* zC_J0oMv~!<$Bp>wc3YhOFcd$ycO4s#yHby@ds7Fn7*Dy>5N|(zh#J$1##`QqV6|tP zxF5s4F#pgl;dv+mhd$6yB<8aq`gA7yW~!6jxGIk_AY<8{t)Vb}XFGgg@ezLgl88&& zpRgWlJ=om2PIy|^YnZ8_4j+H7Vdwe(VK-Ye!x69qS3fuj*Row~P{?wMW4ekO9*Dx@ zfPww5&%W9WE=f@k08u^X3&Qw9p@xbjjAc2Uy87xvtOO|qT%`mhe3b!jy&`?U_= zT(kh5IZ+356C&}y{t!yDGK6~5X3XU&Sy3y;q_F0~0?O{vPyEMr6+3&iJytJy4~sMI z;A)qMFmVgU=X2)le|B$sws(K5Hq1yxs7|r!I=zdl$+v zdAPBD5p~{FRAAU|3Oi>!r4BkauaSl^=2dWyImTZ3NwvC zDPs7zc`CLR%wgvgs0hN9ZgF|L4XDf23fPV}3pRy#p`B%qxG~M~aEHYnHtmS;jou*5 z3%GR%I=x@UuG_VNrA>#R$4)-kGs^;MRfynd@Ex!Gy^i7TNWyYsE4gmMeTuj^1IU-Z z!B!c_!^saP;CS{87@z3_Qx`qOd7jR!s@F3(Wivtj9{G!jl7;N>W&=o{aDZg*OzOY` zY3kvfRA`uXf_s146&OTjP``XhdsEh(JP>xDQZ)C#(?*Ig=ie!KaDN%QR8ExlPO|_D znm)jV1_3xd?;Le3V~F~tX@>g_o)y&o&Eg)urU|?HMQ}iNEBvDGj7~>S2wrZ#hMmsF zL6awXm^E#nUi^0)4=uIFhYO_zvKm#~&(ghA<=Jo?>vUeg|FjxfE$*atp9H+%Jaydf zejAr3B;nDFAy}8c67IkL6DGf6*$s+!ah1Xd>vedPI^lQ}k6cZrgqJO>xGameY?n8N0bo z7K+WNfCnCma@$Ec+%aIxjy~q&!CRLE-@~KvfyZ)eR@HaB=!rXf;hPcfwf$pi`>|xa zNvj;QLCJ!svkF)-PJmk{t0~p`P0%%UHRW4f54XfwP^+{HG3$z{&pjGY;IIa-x)6># z3dMPM%$@Pn)m6}^VG*=gUrn7zT@2TW)KFiltFV&19JQuh3xDPqP+pm%+#*IE*McT0 zYNwyTA)|*~`?nDeY!QQNTi&w4%9r8VziGH*aWxwq8;AG!hCo&P2#T!PPOX|O!d=W- z%&a%ZTa%UWhxak;>y00wtD!V^o5X!=hwtJXCx|DjZJ=5I8t%DJOsP+I!p$MFxK73c zPaZVIHqjU1lx`^Ac&>)JqN;}{Yf~YzKSmXGt-#7f-|=&$r+73Z0-s+Th-**il=q0Q z!Sly*;k}I_(1*={zjWdRFG)YV%hilp?>)D?Oj1_3kD!HL`M5DV4w*s`pBQ{)hfMkY zS-liA8iEF)7SOgi3eM8F0?ozdurXDCsLDo!pB^=4Z*Q%Fk1~vLx5#CIdC*GCwcSd| zKVJ_o{5ww_RQdpSecKDQcMHECPleXk>`*8#b{9u>+6aG$pyFQ7XSZhWgU+@TOzZlE zQ{=R%jhE+QX;D?|+HH+3+E{AC`ZV}$sTt(H%!A&R?QmR>4(Gxl)^PC*YKwvrR`egm zq9?teJFy!sUT_mLnKZm6V+-CIaEV=$O|uoP0^DxVLYd0t;86oLoKx|fbvd#L z&JE0=b|(8#5x?KDDV@1c`4m^MEV!7;N2jBz7hY0uU}XOFSY$*i$uDpkt?!Uw}+aLqhhY#t*~eq(7A);mwZ>8FyZ z^B#%Nv$IW*sG5Ooaue{Mx<%MsIDc&8+?o=Y=?iy3Kfh{9edH20aDGFH#@&VI&Glipr8J!Jc^6cSI0Mau z`h(s~No?Nf4)qHb;oCBGZ0-t%au;@5O(;8ZZzT4zi?{EE%Z;_DiDHXQcz|WciO0 zQ@F>?Taw5|<~(3e>lv^oE*DV2e}iFxXExN@SRgp@K$V(X@gICKoB?G8YvG@LdoeBF z0((FGqs+v%vxMnYsCVTWS7i~g5B|taQBTAtez*%3(MGvn^NE71zpzo*kG%`S&YJ| zGui~<_ubj+g^#IoCh|DsjXu1T6~=Bq)<;o+OQ>5}dHB9(Ev(%0RuG*?;Mu<~;>`~w z@OE`ANW^8sfxP>0mt(Eq>9-T?6_GbMG_4St{L91FRm92z9CY9%#T-hivKpHgZO1Pr zmcv1zcABi;hy5heD20m`sEm^b@dK<|KE2~9SM63FMNEriN7RISJ81wuxT(zQ%iCfT zIVCpa?PIope;>Ch)?>BFW$ZxQZRF~0f-gpX66je4QqI zC8oa_|7%}}_j-6!rUujDakC9ndgc>yd2kZrBb=}P#oUM4#|x<+c}w7zk$F63_A-|1 zge*VlER%>s7t`n|!t)h5&>|C+(kBHX^KB!1+t3lp&mS7N;`tl49YV-y$RUFY|*Z`*jdZF>&nj3iNR z({Dq|n62=@1F`aVjk9o@O$qE?md85J?q+ZM=26wtW^iYzDzix)3gs>jcCoDozr*og znJ^ua_}SP;wm#rHBvzNf0F!Kdp|(&E5TAxae&yj4NE^3ZE)tlF6tO!DyD7y3^I4r! zJyefv7wi6EA+~6Y!y12I!OWxT<>k6oxS^Q~SKL+QCe=;~ZXL^nwft6iw<#BX-AQwE zHpEb&+JkJzlW(kJ^gOupvmSRx;92M$G#zWWKx(EVPAjeDN|rlfbz!pKq# zrc0j^fQWRsq(qFJChdWjb2H(r5m6{IrhpqecS4iL*QkUs`Es>GmvL0I3iUL0f^9k5 z13M~y!>2PsVOdus)SIo3gY7*bvn&)>H4nf?{=1-Drb4;@=lj&-P+2H`I+5CES&uu( z%h=BdLw!}DKVWs2Qi|(`mxMXZ*?w)*#=B;0hfX6bINU?6Xl5X9e+h+XRdJm@L zQ=sY24A@vH!ZWV!7cAN134?cUhA)M?OiFI&@vL2YDOHtM*u|va(BW9KTUP);|?^t*V03D6uQQ+AYq# zIlqdsIK2V$3y;loyI z_QWM)*rnqLN!4!X)G3cOjYY7G#TJN$dDy@$2A(lX!%;z%0`<3=Q0Kih^sSb|nS<%L z!Z!n7a{UE`p-Q;L|1@-IUW=dJG=y!3+Nidb7g@zYOPDK~!LAQ-X4zBCaC?*=+&A9E zmK<%Pre4h8-7unAp2P$tmvM*Mc|0B-i4(y#Nsn=({3UGJe2#s#7qSx-s!+W=f?8-l z%wBZN=V~2exFN?4puLMVES|2(=9Cgx_vBJmFUJJuhJ;g@GD>jT%M3Q=sW#4LuWU$$wvI@Yd;#5pnnK*To$uP$+iIxkLgEy9Mm4t|9=y8AYMRlg1{|wv^pvHz7?Za~>Hc+KzG1y_y4!VU&V;o=y z1O6mXO{b5+jCZ-*wDt_{fr2pTwM+@)>DS@s-dw0F`V6~@hH#=Pnx!B_fN>t*yS{4KFJ?g5xN_X|i*vjM*zNicnZi9~=$8qJG*hW>b-1yYA(8F?{F zw8Q5ny`f}`|7YG4^uuKe2$_xKNSi2gxU3oc`H+YnUC5#%w0?oMUMpmECJ-!Gd6_6z zV9_hrJi^%UEwNjBH}PfTU!q0zBoQ%i1gJ|Jq3#vS5qX>eW8{>u>okJnb=i=tyQ5DQ z)s=!Wn;bB6YYYk>RYLlv5{&#yOB8sfA52rb#t#*8MN6;dpmz(kNNXUsVOG70M`!NU)Be4hV4>a}!g#$1nE0g(k_4~mnG4^6T^?hI z_CJhp(>Tc2YzOz#JivhizQEIWI|1B%(SaNHP@5Fm5M-CT6@t{RyLJShfOG z`rJedk2ZoLDu?4=`UwpQruch~?WUcMI|8faJL%OaHt5+@E^&%`4MeOxh7$g!qFsMJ zBf@4apWhM*wy7T?WBhS?9KS7k90@`JN2HXf#B)6>B0R4@r#62%f z{tR~p8GJGzbMwCgEbL6mUUr>xzqkQJ+YlgM_X29K;-S!-V@R+h7k#<^k{J0j3m6<) z1JdvM@Lk_5MLJ&-L6VSP^LK$5Suq@ml+j=0>LWlOKfFZ`_qc+MBDo+yB_E_-dP|rF zUq>AONt9gGgJe!Ur!Q{L1ZECD>1Chyql(h!L`|9;h!55yk2^EKT|^&*T5dx9L7!2d z)N8chqaRB0x(@73Rgv!*EksLcgRtf#daU;qEhSb(~NrSt`F?(O$P7ZiZM3=UzEL#zDbJ&rxB}~oPmC2 zG~oCg1WPoQ(|!iRj?AJMAgWS>R2?oO??P$P+eVL`^CpBCfA47R{@*M5XI&@}&t62o ztzHxVp%BoaU=M6h{RLK1aYXCe0ibA==)<_e%%iacTim2&jdj{7YDKlImq_-NZI5PEi&bmC6IBJ zXYLhv1MTE=V(9H1)Ua}hu>C*BB$=>>wFIu{CfU|nbJL2n_C!p8oI?()63`|83Xz5!^F0qK>JltPN{24bx3!msS^!63> zn9eOUt)c`~P>M)S$l;mLZA1wX1*l#aK|V03L|mJp!;i@OkKSY#30$|0(xX1MWqW59 zqYE=_nUxB~X#Q#mI^=dM*ty+=jBxxxG^wP4D2b9bJ4d17w9)qPRG~5CzuAte}f9 zu`dVLbv4LhtsOw?LOmD^bfD>~uV7a5MP#MQ2OjG!=&apSXwPLyX8Ylr=tK53FtP`b zVios^WffEOFR9Y3Qk4E}Hn0 zf@GdO0^w9Q@$lRw@ay_jAbNhKb^bS3@=*uP`Nm5oGWx%uiik<%?$$sfM%cwXuK>uT zRS;qiDU=H&!NrDn5H_GlG_24@KMhYHrz<|7&`*andJxPxSGf)(WDX*mPmt+w9!Aj} z$-++VZy?xkDM>rgpu{l+Ol+Oa^z_|CDz$Nlt9ldVla1(%_-bZ%$Rt`|lfu6ha|hWo zrcCPQpY)B-Z_vXP;iy1HpXu*)LC<^_qi*}}!rx7Y8Sc?XI!|UWbHrYugQs*E8(|+= zwxTB5Y|)NNO7)oQlKv>{QUY;m^Gophry-f?sEU~I*+4Y*HTashl6Kgp7J?Ai3}+8Uyd?ykc3hv^)eHpOee4jMe5Wp+mqrVZWW?y<8+FqDj0Gok`Yb z27;*_Jw(wbBlK;#G$S56fJ|<_L4(I_z>;BIa?QREAaPuTl(~M7D9sxL-s9H5Xy26+l!f44hH00L1irpzrzu zQbr&}4xv&I=sk|Mp5DS_NM)hc@OWaK?M-l}&w`ZPNdPa6YGAWxln8P$A|GklAd!J4 zkUXlu_>UMeLE{RgI*;<{j~#R9s~R>$jQ=N4VjTqr^=F_XI~Fr&?FAG*&{kHrZHjob zjirCyb47o2Gl}10+33d|NqSvKHfLZaqI-WyfDPvmVWp>rrZ<$Lz{n6Zn4^wVCN=^_ zW+D2}5(PYtTmn{czfq5m9H~h6fkbv1irK2jkZSQ|`5y@IS!5QSKYCEexfLfLCe#y) z4eIIV$_^;G76ZqgL3)jpF^E!h0(WHhaXL-p!6xG=TB-jYI=Srz$S=KBw#%-UUUGC- z*|k0CMD>hk=#;1^I2nS8MaSm@yPgW#F5Ct!(vd;(*9HkCdvAL2wnT8mvYmgU382sC zPtu!Lq!OF1-lHu@9>}*Tp>ZxnJae%`Wv6=pN`8rI;RB2Sp1gvK7A zpza5}gYD@C@B+R4`xBTo#?uFd)8o;5G#zZW6^R?IMwQzi@r~DJ0E6NPH1m7}x^ue_ zZ7e$muAJFMS8lvQxFwtMlX>zqLCGRPTM$xLi$G%XTC{ifX|Q6)C;IlrEIMlkA{=W1 z!PY0|K)~oNRPL)n&$;G`QVPUT)}eFgl#2|Jnj(P?A!+2^xC+gLRzO3f5+!a{B79!_ zCS2_c3GK>0j!g}pIAbk=To_S0W?CakHd}{U+55!mxhcXfF&C6t*aF1GcY@HymmJsI zHNfs{0~o^ELXhGkVvUn0$lCA;8Lyfu3n@?li`;aOvZpbzVy7~lXDW2=FKW|fscM|v zO$8ikQw-|*l#@2Z0D5}7m`eG%R59sy}?akO~o3qrzM1dM%qfwaov ziO1X(^v!z)w93!}bf;}6s-a(T)>fWW7wJ5U-OmVCj#{kV>}<~U66mvTmT5@HDRoBd$l{ANV170xS>e&F#p3cXSl z){(sELIczH56D#gsrgv`@LFZr6{8x)sAd6*uAy2>;_1xZrnp5AQ9{saGwEYE= zJ*N$>ea;1SrxBt2Uc9UwWpOIMcoDBF@(JaZw9?7(=_rT40HyH=)X~{k=DRVE7?(u!*G&Vc)U_vlTz(xjS8 z2RPq#6Se3K@(nx|p!vUOw5{_yvB*sgHKuBFj_e5qh0-Va>l?Q6C1lQ&YNuTwwh(h^ z&66%5{Ka3w|A`97Rj=Vob0;{RetoosxgRlS$3`HowhJtKwHj&aWutxb!$6g;Jck^Q z1l}1wAh+`vIGg4TRLoV->uNxZoLob{wt7leaP^q9rr8Wz9}Q|uvuVFq>gfEzpES>z zM@x5<(4zOU=!9@-0k4JjbtS zxmYv(d%F#)jC(@d+0Cc7ojM90$Iaz$xIfNWw3;OnPK1Ka$WQbl-;*f)!7OCuRt_c- zqEVvyC>=4q3vJ$fg%}9Qpj}RO@o%Q~@t-X3qSr`-p^3k8z`bc7NLSs8WFJ`qZ;2Nm z?b!?x9&AKuFKvXJsf}P08wtwHtLYuhRb?|>B0#6`ocOc;FnFDA1QfoxA-iugY017t z#D{i2aPsdR&^z&!zFnXQGW>GTl=oA5-{<@ErvFk1Q=1XGXMsH(w%h?6T6`6)S}_we zJ?}%)r-BK=;~7NoH!&vwJ6)^I|bt zrQM639UUcR8*D_*hT`b&DMi$&IFDYu=N7H;EEFi}?5BtAoltweDkzcJLQgGfq2o;C zko=NrbVxIc&{-lu20w8^D_q7v<%L0dP%{I)-2IMYaW)Ihn_~>>_E&>ct1`N9tdqD^ zA`XgYZ3X=cU5J^n+C)T^0(w?|5qtA=3NF(YQ6^%iQYP9(PNf5n%(7e&t;Dxnv3b`W=lS0m4YL%@IfGWs0q zrdQ-V;K*<-Ko8U-{}|U0z71pO=yVxUTp49p|)4)Hvl^9y?gy5wn1nsJcrEM$0VFx9qJAM&UV!IPDa?1c~EadL`^2iyZ z!hU8UTjpBFS@86s5tEW3gZQ=4V2*((v+koUSg}W%$d>=YDf+8`wl)iUOY6_0y$8>N zxHA}aZZTqHblv#z3;V%{Umbe?Xa+g*p^Sf0p#^m~rh{!Cn^EqCY_!B>if~zYo}VA` z8&t$z0`9HH(A$TJVE1P!ver%KQ zIfldz{{%}N=aI$DYr%;D8OCX4A86V82feA_(IO)Kbk7?Gl}!=kgX{nhW~|6uzAj4M ztyCuONdr4c(Q^5fB9_>L9iyCR;E3<*)g$Jl!kcna}*Q1?>JOlvON z!CxtE4fbxXB0Ro$fIKS=j-i<(1L|kcHZN*`?HK`*yVr_B6Y2;HqLOy#^yd>zFF~@v z3~Xq8T{^T&3A|8M0jn01pt-A!sOV2b2cJZsE2Ry@UQYm;|IJ2b#i^*rHk}w2_I!F~ zi!wjmzoWAF546{-WZ@kfPAgxRXV%8x68n3rIE73QcrB26ieR zz^fZY1YNAmtWB3E!ya*w+x&~Dcl;`?aPS6L(Jbt-?eGE(Dukg@>vMEC02lrJr&5^>lP49J_B-%k*NG#4u}}M2HX@C$t@$Nk)`)+vgX1Iw2Fev zccrs{YBORU8=nQbim$-E*w_3v@EqhfC6##=YXAr8Fe)v~1QJe)XolkyS`bsr`MLQS zl3!m(>&$VWk8TmZSE^(Q`rd7H;}ehRedr9#isv%=se8eSm9pg3_1WnB962(PbCwv$ z2nKsn?t`3Pb3yyYT+lgbMg}fOM6tn2WT&5yHFKFFCDa>Hz4s#WeoZD)cw^02%vT1Q zN$QLZ?Tc&{>!5Xes_Ds&ICLgjgyboZWSn6PdLE-i_BmyvP5_yWQv{g#SB`-n3hCKx zw&-ucAL|+A!Dw7kAGt5`T)MaMGyPZ29_bBc@c)~cK?GRurst3Qff;8nA>SZ5A)ho( z$OF2MX36)0r(GuGm!08g&Tbv@rv7*E%brInw`QPXzl9{Fq=)9{)*{btDG(Mgj`BZV z0Qm>h$j!E+wAO8Da`i%G6jC}J{G1HsuYNZT`8dR)f?4x~45{7Vgt#cPm8)!xvZK&{ zGBHGK$SZ5z{6g@DP$lJ4EzogCN%Fd$7kt@UBB1Au zC(;WC{phKq&(VpF1h7EJ7z_CsO>`|C0#e^4K(|CRSiSTzG8Bk2r=R!GH@%0t6f31iUwHv1m(xJx(=>3hKY~^-aX=>TqR|*%k-5?uWzG6-KMefH3D^u@v&h_0Jo)+Ap-bTCK2Xa!nuH)O1FkJw^c64AUiU4rpyk8H!o-nkcap(A~1#s4+~MIkNga zeREiV9tyP!6P*-f_4Wk1>+OL83x1(G?q;y@w{WY`)D!Gei2;YIq!^F3ZKzTyqs-zU zi;5f+$=1kKz%Mw8KBlHlS}u}cmfyIAy40P}y=fCjJVcaCyx@vXwhW@TRh1z4Z30sN zJP+~9g?Hd-Pn7fpj!b^3yV3hD6UGm!JvqaRCZlLh(t zNNU!3+HGqXda~FGO?O)koWsY!wyQE^RgOC0J(iEg)_v#i8F%ATu?B?XR4;Kz{V-v+ zX%W#ryVH7D_$K=-WYIXTUJK;Hs=z(>R>Ys}PG_3{WUKHADQ$j-+V8!m^>3*liyM5D zsWOec%m^7uPyQoxf+B#MsUB0b;{-|wGe@_TMVV9k>QL{WugJPm=ppc$OYdJT<$MnK z2{PZFLn@R88WlN@3>NG2|2gUruik{BqZJ#_vH5bu7yT~ef9*yYf0hzimDvm=h4pv3 z3rB&d%zNUTksA3;$k{I0w-CiNM1v*MPdt>ie@hQ&5@5!VBy-l-k+n zNGEkUI_wYwPM(}Z-TtfTn)W<;+TAcT(=r=YVVinDN-}!ByqH0At*d}7YS(v$U){HA$q?ZO{B~~`A3XUP}MBdAui+t z$?4kphg5P3%a;AQz?u+B;cxdlri9m`!HGk*@) zyI}|<%l8nftp)UA5fdPY$KLde2;-vmqGySA@7)%K@Yun9wVA|JUTJu5$olE=y z-#Z&YaYhu7UN;5m>H~rNOlzhnBc8)(h6hkYyMW82dql9l}XXKdKfMQ>bAdAr| zy4mPFF{e`(&5Kw|H!buhUS16)fK?-y{C5|OrZ|DlStg+W-2f=ak|pNOUqpZZdL0}x z`3Ulb%$@LK9F*pzMZS{y4K!&nvhX^lWq%8Oliq(|SVxBpPcA^`pR|IrM(T`sm^yQJ zM+A7dqY%wYz6=CGmqF#%LLwzW52!cEFjrQZF{v*U=>2v*h>&_otXNhEj>PlOnha%d z^Sl*PTV>8n&OHk}-BcM}IpLe*;R~=YY(BG3T*!|5>%aslIWoqXt3cuV>CAZGDA*-o zNXFQ7BT0v2U`taou-H-w*7^LT9lu9_x~dXzc*}J#rc#Q$f>lVDN8-%zq5-68wVT#H zUyc;K&B*VM7n5Sa4^gkb1bKxH0yoY(5>funpw9FW;bvYAEF)>)W6D965kAN@aSkJy zw3L~DqzqhkA4fuT88i2n1CyJ}2gY;%A$j4vVb%5nTsr;;9TRT{CW`lfcR>``F{K68 zpRNZMb^GXbd5?fnWf!vAlMObSH=@!mRb;y|lf&6rLLC0;LhL&d3MyjW(nXDNg#Jon zGV-SyDRYNGoWla}(lG(-zFQ0aRBcCVx2cnr$3M_@XKT@gpdRAW=9g$gy8&~o+=O|u z&==TLiZDhZxkTvH7Q~C*g0i-X^P8P!GNX+%nT78?fkkdM(yn_0-cN*r)Dv^jtP@{} zFN@mIV}Dg<@o^6jdMyKOAWVURf)?1Z?IKF};)!y*-h=1Tk|h0W5fbmF!I{2Ws6bbW z*(at?Yt5HoMoM&7N3SYpXv$UOb=v(eedM zd{8FecWa^)uP9(qdyCL8^`a}n9}}JP(>Nwdmw|(y9XLEXhUS*^6MJ7J(s$&u(YoWN zM7{9~V$#W!D0B+v+%3I`+?WLbF~;POf-$)*)&M0>eg;XN6NtymWwOM3Ik~ZowA{0e zgwN$WXjx=2aJGI%m?u0zlGhpJRs5F7IUt3KuUm$HUh18>LXSt+JP`E zgNBa2Bs|tBkjL8)aps93`ah0kKYh(6G2P zYEU$$mn@nj6pa+X#dU^d$Cvja%dLK>1v1FOE|6a5cN94Zxy=n4Yk}v-Z{WsVOa8ZN zU8I?9LF~9N1U8L7M-95?f#zrsShm=lb63cRD%H{fwaYVrtX&z{BP3>O-C2ZMv zvZ|mVjzkS@k;Iqcy&xs&H#npggp#=D5gL9@e-;q|uUrqI--eTfb+aBS66&o#_6T)^ zW))gw^c3f^q&sr23q`v;8i?bvs)W4kKYEXZ1lcyt4(-mDMyeY&qm8jxh2s+;5l!jo#l)rkDJWqBD=D>g&QdLWWQz(kMk} zl5qFB_iUviq2ZSXnlm*?(nJ~zQHU}op;X2cB6qLj98yG78Z>I2)2MlV@B8ol>wG@< z?6cQe&-1MBsQa*L_C~z#@>1gSUlLiTSp#R^enz!1$)vSgBtEqYCT+P24c~J(`x;wD zA{CnA>Bk!UxuFNJ=D$WZVca27bi{ynDplwer?uFf@e{~~TNGEN1d;b1q1ZaMX=s7c-}a!WuIt zGQfEkG&ZNe^c_0*-=ycj-*;odBlTd-mB%E>?E&%Dsf7!*%Sn={5jLb%g7o@TwsP7D znDVzDY;+F+?+-)3@q9h;{i6$MI{%3MXkZ`4hRY`;2f^X!b~MrJjuW;;3ZLr;D@He3`NTBzVay^Go28+|(_OWKTJY%d{jA ztI*kl{Hh?Ou>c<%`M~QWOkL z4RK`7C{4EE_F4EFs3O%Vs=za^FTn3%EXjQX7G z@cdVj{UFCj^35GFkRPZi)!p8J^JKqafWn*Nv3LR*w7Cd0f|Dg*=H!D~)<>wDT!(fu z%V2SP7P&vPUH+hO23$+jL%q-waIZsM>bm8N;x1N_<|j-+wI{v!_l=KW;<3Hh$F&Hp zU$r5z?u6mV$$0L^Q(Wb5#vjUWfFC2?qhEtF8cY`OzIhmQE$)Luj)qBMruWBte-!;I zHxi%zeF=LFm8C4_GI~9(g&QmNA@j&3=wt7Ib$(YMIxZM5D)ddmz_BFuYAO6(y&o$S z{92V&bg!$R& z_&U&lPg~XnTh5l^!_Wy>JohgCim8X9rgXe%H4Y+d+Z?6`k42mIFC^kbEjiO6hjZ?S zuyF8EkSp(j0d9LBNur6rCte`j zVcVPKpUO7Ej0c6x{ZB5AkgD^lnVsX=SL@JQ;U87=Xtdku436!9_=P_-`|LP)hB_dqkI_pSwC=m7)0G65B9-)^W6H z)!_d;eht}kFX8VQ`*H8{Om;PHE8ewOr05qKa94*hpWWOFYNZ#@IWHMwj{L^Gz5)yo z6+CV8v0y11E4i>P1aE71hcOCPkBd$b?yf!p$2_*8lW&4zZhs05h+J+`<0d!pKM6xu z9ER`vQy3ntg|RJPaUL7MCpfl2sCpu<{4oJGCaXx#ZHquX-(o0USB_zd--l-$h2;lj zcw}fMS_>CZuitHO(LRX9>(AlV{7)!6s|7Rj+ZZ)DovAF=f^Wmj(OGW{DLWdj;PQ3n z8!j$^fBO_Va`P4NPI!h}bT@!UPh+X(*A}o2^5Q=9>J=O!^~O z_$fib!!hOu%-l#c}XdvXwUuo`M$TvC^@x zACl~qKGObM{jhM755K-;9%TKF<0CIM;ECpV>6xs4_~l-p^prwFX_c+v)y^|W8NPzQ z?tLG##R%!JDfY~@w~4gq*JUW)?9At$=`LNwYFSu+HU68RE-f=31xpg^;ZNfOJob1e z(^Y#0Yi54Mpa?&lwnyPrth5Cq(?eKl`wPzx)sWgB;$UTkAOCYhFMf*hdIisLEo^?V zN!l2mQ{BHKlwZ5sidnu`LN=|bWUrl*FjwJu3iJC6F5{w6*}oL;YKO6cEJJDkuSa0M z%@;hr_Z$f*BGUVB2lG-rbzYvjm1$m%;Q!M}Axe_1(lX^A_6zEwr56u|fYpfI{G#4k ztYNz@|61XjS-a3x`g68E25SUJkE53Ktolkm#zPKM9&hB=n0;fTOM;}o$KNJ#C4tfw zMUPn?x|Z)>eGD(Wh?S}Z+=F=*yZ8zT1IfE2{=&`=I5a*%TJicNzI-}SdM+Uf{F@i@ zXLkkQ`_8dahxO^O;h`&^dE1}32hE1@7W;7CayKm47?0wH7^%#agPz^u`286=I6!%x z)Ox@Tn3gr3m-Sl?UVkF_3Ds@rlM^Y;T-y_7HgVFl?@Dm4V+0?X7Xm$R#_^ru$@ndG zk2LSD6z_~uuym(HIY=_X`0LL5aa?Jn^b&mtUn6$$#d`jje(N;!n5i%Qlx4<)AIFSO zgh@q@1Q_{oDnHXc2s2I$0^4&=;9j&};s5EQ(E7Bc&4-p_`3f_-? zT%vSJz-7EWJw!S+Xdi6fzlEPD6rl69by5SFvDBw)J3noJ6S!7IN!u4@Nj^IT^KF*< z*>wX0Qn-6ENqRj7e<#*S&e?6{ZF|{cjDNV49^M4KShL(u0V%6Z?}YlOZP-*3BmL6w z5>~53@@slKqS>cle#zR+=%yYb{rDgP)Ca}$W0Km?b$x==eZ)@C4OznHX)v6ycdj(F zA{|0(cJk$qc4GJD<#@fpgX`I~ze7-@!%j`M^o7%m2;}7vHvC0H& zl|snIAH8tR?_b1V-du=(QbSHZnu>RwRwzP+_$6auq;TZhx`3_{>Dx~|rLdlei>*U)SBQdVS6q{GfC0>*3S#o^@^4p?8 zV!Tqm;Mq6w+TgBa_lP|(mZ`GvHA5BN0&ASx>W2xLCs;54(`=TH0qSlTCka}j%x3EC zCYhi5N-9%p&_UZ8-1m=$J!3<_R#%ICP}hXbzjEXSLDj6FI)S-snuGb=qh!d0d>pi1 z7b;C=!1B2qT=nxpt$#IQ|pngV;xOOX6==S%q=tzmYOu>S$8)S^Xl_E*r?kJ-B z?gd#ma{`>|qa>~Cs*(>1JwnXxErUIu?lQNFPAH4)E7$s>0X_SSki>8BWOH1i71;3W zX>oH9%JY5C9_#^x1wVr+Pu7D3GdPB60FP7{GB{%rdY})fcaxPzm6iu9g z!r8@4yQVK2bLWNpNp26;_jeWZo7I=SjeIOI>)9PvZRKFDd;utRXP%5^Y^S$GHF*$r?qCX|O$$`@y~z?Q z2B7Ah>A2Q+FO1WAiPaC5!t?2dBxC9VjEQ+g1}xLT;$%&t#}$*>`++3S%!z#}o()EeDnaixGw{#R zhs4_<2Gb%M*sRw7Sf88#*wuA_Q8OFdywaa-3v-5&ucwLov`G+BctYuXHrf(%rO4cg$wdE{6QeXb@0D+GF<4DcYLyWIf zfvtQt+`LtUMq#yNeq6=r8&!G4!h8XHyW%j8pH)a!b*94@&Jxn6j91k0Q4;I5Rru<{ zAy#uR3J-XmcUYnImTC0$kheD05Yy`_#I>*=KIxIgTnkmnu#x5L;iV&JOmBf-*+e*3 zb`04cBuX`36HF{2;szBd+r1f78O zu7Q$I!dg7Be=;s4^W+QDXE{7s(ZzD&ykS*U3f3j)!}1+l6z6gkK3sE&7|9JufzNQ9 zWx9)*UI>DRl_MqQnYOs{#A;Ak(~A_E0*vaPi3{X&<$f^}$oGGB4laF`LW`u49eUJL zK3j4Iz6?GH`0YAN$Vrg&_^u(@b0kSp9S})ujy{v5JT(H>Jzf^ zMjW~K!Ub&|J;+<#`51NPf@FTwI-Ik33Le`!4|_ceXUD{H`GAfoxTzu#w(UC%`N>zw z1lmGs^Br*ZqeLhz)4>f!rO@ORL_BNb<$p}Pq0hb^XdLrU*6#!{^SDRi75C&+<=$w$;h}tO_+q9fERsZ|R!I)+<5;qf4OpmUkRIWL zd}DLK?ORjzUE>aUa)ny^~n(wn5FFALYe;bE;?k&1DNSCS#yY z8m@5tD=E7a$-Ym@BhTlR%i{*VXJ7mMA;(uYaONGa$n5oQGfep96$P-^Ev;adtv>SVz-`GRw7g-Cv zH+&+g=jOAdl6z>Ybp))g6_BW;8vFNVY4|dD01Bhr*^}Fg@q9Kcp0QEK8l+(en=uVyp&(jd`~jp^utBZy(P8v z6wi+~#Bt8qtX6dl#3eqoPjpWsUN3TCk9#t6>NbP)wqc~P^B3E{&l64>$Ad%VMbfIH z#9wL~11nY^!1BpgnJ{*o{G!EVERZOC6`y{SOASu&daU9*b9)=9yS1GzhND`Uy=o1b|kQ+74FE@gu(XLVX*e;EWGup z07u-&C--DWaY}7Zlx;FW^Wjol>K+B#YkrXl=L%6R!3381s!H2*P%^&qACvFZk~WAL zM6%i*Z%;14NlJe5;Z}dpeyKfMePtKy%}YRwLK|M$SR3nmH^Jk1sjO;+VzxhI3(nnL zPux~`;PKo|M7b#!B5itsFs+;{Q&-2^^3NpB-~_xcT2Cg0MB*o-cDb@i5`2wnAX^V@ z!Jeg$NIwz{zcc&d43%=c*EfaizI`2R9kSTsX$VWrBH&CfL;2zv-J~}Umcb8YOOo4= zfRo{h{8+ajyn4HtynUo4)ow$%&z^<&rO#qEz=>DXd^4F{y&KD4IUTj1I4bf#Rtild z4@>vmV9kRP!AP8prz_joj4zv+=Zv0^Q!j&%i{1Gp={?!*pl5KWeice=i?FR|EEb%* z1&41d-hqhj{Da6{U=w+iIo~>gdM}m1`DZHGa^@c1IUh|%4jj*}y@-*^zD0u1noOu{ z{6egDKRj(8pNW?-h6U!tD*P{7;ld3$Y@RwEU-#XNx^Jy;zs4kT=KOI~Rprp*!~xJ; z_KDqkqXBmExo~yN5Uf+wIu_$=n0`+;g+_aY*>xF#(x_or+FFTo2b@8_JuT#|qHa4R zi6u@E@5zcRJG54F#^2{%*f)Pulr-&xBGY)3oZJq5QmoUr}oLu z%}gedKlftj+n?lakqOT5-Or{@N`|J#$I-5|k3y^5%1D+syAa_GV+Pq^o#8Ne7r74h z72HRg0U{IPmcw1a3O<2F4=7N zYJYUO;Y*Y~(&QCp`$6l&7SJk=f!*Dr@Y>K_u%_}1zVpn$wy_p4#;=7O+*wJ^+T0}D z=C5LTgI=-LmnVUIDwbUEl_H6WX9KP(fy)sih|x(v$9+xgVOTRb4gLT@U8(Y)305qa zw_(wr^KjwnYIrSoRJn| z?~5UkUN|)|4Nv-J;>Y==5}{F*q#Zje@0DBtkNwtlcH zv=6NO(+RuBM`Gf$x2U51l_hR?g<~A&uvKq*Lyb=ghE@mQ^tXdB@RAg7pAKR^MaS^l z^C_%xh^f5l0l|#;Dzs2|VD9o>&N9_t^5`$d^}KR*#&;;eAg#F2P*;4tj8 z^Mls3snDx?9@K9A0$b&AxJo=po@NEWj|_!waBUr$jjUH>p<19|{yLCL(pY%BDg6F2 z7?h^mga1}!${k9Sr2V(q;2MTFJar@J(j=(xx=fOWS^|?zhc|Ie5D;jLvd+HvscsBJ zJb^E2SJ>wAzjsP1wLOg52bq_;HPJ&@QS(} zBpj)M!8dx~<+gIRKqn1ODp82&sg3puPxY2*tC^por&Pagii};y0Zl$+L*`Fx>vf&w ztpZ$>sf9(^fFB3n$NHc{`1gl1q#v7!eZ1~FsHdmF%Yi$YyWXtwq+e27>J^H*G! zwDyZ*ue|%AME*8u_Mnc__p^A*34K&o(GuKac@bpY3E2@ox z^kFe1FW;5Cu54nys`2n^N*7q{wr1sflu5bCcrv5aLh`P;fqibSWVbr@k^j%;=luRi zn!2~Msj3-Rd^8uv_dQ2azg~cId)ry9K@4{56^)gagE7I)h~>3KLgIpJ>{s9t+;Pth zKZlKipm}fP+AnlibzCG^Cf1SWA~!Jn^V}Y<9LDCY3J+gFEtxm`GQ6Li3TfJYtnzRp z%da2Kx()0F`|UJ|{A3QSREuM`@5f?T_gUaQ$PCrLXXBY{Q|!6piRA5(4iekB9}}+0 zaKz8u5cSmu7Pe|&w7VH5R(3#`#w7XPvftHfLsh{pjrno2ifBFw+)osL#$}x(p$vd{mHIZBxVxZ8{*vJTV8vWH7Dyr;LTrTTK!9$s*=krM|@#P@2|iCD0hrir?Sclfn%QtZt~omuGEk$Hwz)eaTs7 zQN1AXRt~TQYq5a2;F6;h|Nc2ejFlvy-j$6%7nn${eIJ0aEgRvPr!!<9Nr$Bq%gKJL zYW%*Sh>X2bEDvc5K-0$25@Gct`F!iEl8T6hWX4%_FcJdr+4^!+v8X`3U!Pe`Rudcj zdx$)Cp(R*&y&(>6HmE$OhRwX60gDo^kfBcx+W%7;4fdhkFr@Y2I9*G97F$(`&tI>_ zAc+Tz%H0b+N@puF<`XexSs~t3J_HMo7UI=T?2NYkYS|9d8Rq z%2gru!x9|wqvRz<5SCoIiXXbHpYS;pQRxG|>>aKXiR@a%Ma)7OF}pBfH|T}p(M zq(E`WRO0Zm0EaD1fk$P_Aa%uB81cLggMYhX)I2MEo)^NVY>Ef#&Z%g=Sd-0k|0K6a z-b>DInnk_|W>|HSmyfmShI54H?EJw~@bhd>tin*Ne76fGzc~R#K}q<#`2cnqbU~j_ zYM3w}8s}|($M#0Q!!x55-hPitESonE@AUs9xuNiI7x?px8dOkA+^QltEN!|>{x)M?Q)kl^(X&6ii?+X{sUt>PQGnuNzHlj7v z1lschaISB*B!B1tIL2pU;IwyOozc#G#w}*Y=YEIrViUQwbUt`rJB(gmMzZhW`jGhS z5wpJYmBrmmQYcNoPcJU&B*u$Su7m9DHZ%ns|@2Vs%N36^y=1$UO_;(}N8U{@c(Oj}BDe{lz1DL4*}x>L~p?=xU} zwdA6CGdnRrAFn^mf~O&A;G1a98(Cz-wGLBx>c@hAn;cSOm>p9Q1xF9OD~_7A4@p}Th6>?<9A(_J2{NQ z-}%Q$>YFtVsZ&m}O~=m5Bl-_UQNEE_^*9fiCtaoUZ=8hU%oDilT{?8AsA2fI0=Tp= zi*VlG*tr+=Xsf*dyINPleuGxx(^3HIjF!OSTetA7=Ww#{dpI~W8?)VP2;P>xaj=?y zA0|mJfO?_g4&7r2mAB^MpiSmzthyamN1w3Y{O5z*ypJk44Xc2j9Rcin^cII7k1a6a zEX5OA9XhG8bzmgx#(3jN^h!$Ih3oPrV12Zf$%(X-GD zCbbPBV+UK{(Cemb=rJF1#^(gpt(ep^<^=>HPF7}VfD<3i4vJ+HtgDS z5G)PzNVQ=$>NJ$I7gHk1pj1PQ_BUrmiu>89VJccqn~Le5i@|*LUnc)~39THQ$%D?v zbS5)E$Q%ie$3b>tvOxGHfPFywgt~ zoLw>*hjbexH8J0hz8U8DvEML!GhzVwpwo&e;v?3rdIJWHT?YA2?^MmTyN2ePFNw5X zTNyZ0Kn-rE%>Hr>H@n$yv^)dgmLTZ+A34v{XCCcxY{g~ni#3G!a1eDb-MWLAK> z)Z6eNH10@885@mR=_z1(eL0-Gq08rOio*Us21&buR^Yn}eF8y^4FWBH%KrSws1xq_!LEYyhdTJy{tOB>g{9CSQ zd48N!*+!9({bI?#t4+l#xd*Eg9kRjbON;`)poD_jb9`Q~3HzJB1WCuCn!x2`PxPVq485E@&^G03s@XOSE;I{-|?0`zv6?s+e zde#}k%!b0_76f>wicje~YNB)sW`LjeuwO6g_mo1NOm$!|+S%@lg5`a#lBxJ-%>< z4YfYUCb@W$o|Pr?4=(#)#HB+l%c+HJZwM!LPvW5ZTqwJqa-IDe70Y}khq4=D8ygw8 z9HiCCByZDia?sZdVl!{z)^;yqb~l_f+0Mc>>wT(MWsi~QX)E*6Z__ZWmp{%jH?3BR z*2P5qp@>P5c&+y{Xfr!Qmet+BZu`bS*N;C06+Zqx{BiVgP~<#<=J1tsZAjzn_u$uX z1&Y3Y!^1J=_`R?ll0PbHjRJfA+-NO+saiTr7M-xOXp*!-&J+IDUCghea6h>}e{`fW z797fle+CImcr**QZ#=}ZGmu;-sW9VY7#@C~DBn7F9BfP6$##YOgzLe(AYGBIEu6Xn zC(YK79_`@)Vst&4*7SiJ|4jM3hf(PHX*cv4I+*v49?$dS8kCx;@x!-2LpEs)ZxnV1 zTaGxPkx9ry_vFs}yrew>HTr!70t@RoDT^_~{us4^UQ>@jUL)G{|k%X%cCRg*V!(% zGvzUsuGisTZ&&OXn%;t8gL_E5KS`y34b}KxP5q=xqV5B)T?=Hvgd55(GdLoI~Nl#Oqa^*|zKZDX=WBAUqqxi_% zHoW)#qj2!mQ>>GZkr3WP{{1!0etD_^KN9AE$dwDGH`Nx9S zn)M873ygWEjN-Atjj#O5gW3{p-to|R=)5#oYM9o9u9~mF zLTMsjb!P(qbEJlJdyXmJp#B$L{`Vf59`xknE0v`OhI{b;KF{Pc_kV&dSIqb*ek6-` z?~7t=0_>RL$~f1nIMq{!-?j7r{0S(-c#mPc-blss4Ty$w56<9^jXU5=#z`2f8cBx! zE0tLERFYpo0Q`2q*0&ty3_hr&WE!p3bIvHkNfxHhW^>rPCB_CQTIuilI! zm*_EHjbdg!>M+^zRukuS+l-!{&*Ij^{=AK<2&p=MF`)TAZoi=Lyr;eeRf#HJQtbob z$GSTLv2Ujl)|C@s5sRq1% z{~2hYas)j7o?`PYZsC2WPdF&skR0A|g^g=|!?ynMuU4^(W(V#J!-K=);i~-_;@N%_ zFTbZ`lT8t>Sy(9Px^RcYiH*cLLV(VI>u`WRX75xe>@VzuBYVI&zQNZ1uh><)BW&@d7f^Hf6ueaUd&Yf^0#>2J_eeBDYv;Sr zI&44Y#Vd9ov@L<4v)a6Y&?Ha#d7aHtdWU``NpSGkTS-o-BF~dG6ULOCB!AsMv3@zu zxO~AKW;-PkZn_ym;|&i8zSYQ52JOL9dzav<0}mk})|Av3mf_&NyWq!!4~jg+S*ES0 zfq$Y75wjH&CI1XmS)=R+oBwwUWW){tm&s=tzcriKkPpQ8-7u2;%?f=ES~1%ZfjDAL zI`*iXgq~5sz^+VCcqj`=<_2{Ze`FN?9zO%e^en^vGXikR#|N;p)f_gfswh@{KPADZ z;_&L63Sx0^1ZsMolGpm21}(2@@+{5@nwlq(K$Xji%&!v8jr4#I$Bz@Oe8PNRY$pu>cDh6Tgapr zO~L%US4^>w8d6h7W0g}TF8eYM4GgLwaeYr*pYenQOmAnC6nEXi#sQeHYB}1>Xk*3A zdmzMf4`J6&VvycRyz@g#I##NQi(0!e)$e(5tE~rIHP;61eZM8-?Iy4sy%0AQH92q| z&8$Q1Ff)i<2+pYzbe4r+YxigjX-6!@sgt@>ine_ZV<)! z;R4Z0{^Q3%@Vp$##`G!xf5TdMu(udDIZnjIAG934TuH}Gx00Z2s2iO9;3IGLT0s6B z`icShNzmf64jl{jP_^wWeN4*W@NpCq&|K%>fF-MM{mNv1v z%|()ZC2KLPKA&_Oq5?;kw&N!Cax_!Xkcx^r)ze(S>1t20{DB2Aw9$fqr^rrTEoSak zW6|mS0#dPb7~EYlMm{QNC)t&D27-%KX}O=L_~z_qvHvqaK5~aU-7!x^49r+5L|>mO zvd31Ac?C!KrB!c)dKN<+yUN6mYWr%&FRhm*AI=iP62?)#Iy?IE%mJ=|@ug}nVd~cFHOn?G7jBwesWJI>pyv9Q=k)5DquhFlp5x;ua>3-&$eOq|ducX8=(_lM#=*CXEzQ2!n*)f)8^?xY46dNlx_V`Cvw0@N( z^i2^5T6PKS$tn6R;RaprnMI9$nutdt3GLQ!T%2}Ln`WO{F7AFXS}e)_kCtc0i}8AE z=%5wr=(2f)%Q3rJbEW^zn)yl>Y2Q0Q4SjT}v(aTP>%l$F%yNjh#WRV<-a9BYQc3YneB;wO)EnmOdApxJATIBjMR;i33NsIk$dPd#)*&w!KSp^#lv54KQc zyK2s8`daS6K~J&THk(Uc_eGdJ-H%_U;DB6jRi#E_F7hU>UxnD@Kf=>SL39YRp@CM1 zDOPm|S-bbqXZKdsR{D2H`(E^PP{DF%Lx=HA5(q?dwT z$VS=>6o*vDP`y2N)OL0;RgQ43slIZAHWh}`<_R~1IMaIBRIfT&tzHBT?NF`t7{5-Y zG5xDBNBO6$yI4Z+XEn)Y-PIF+Zf+C`g6zaQ&3}auBZh0zf4w5< zV29U&pE-)2+S>GLvX!V8v!5OwF;e^*G+Nr~>n-m7zD{iUtwuLw=292yt8`=Z4eD!# zf>vsRs4>lpTcltoU#$H~OSUbc1;r6`xpMbf1&LgoXmnn1k{i*8F%P-#ahqxD=wwmh z>ZZzrE9kpBdxUZ8GsPi8lBnHuW12UujoY!^hz1`s2-&D}p>!3D`1h1*jN=)MPn z`0nZeA=*nSHa)79`5dql-WKQ5*1yV*<}Lf^2Ad*5SF@E?{jK2c$6OE}4h|8AzSneo zIR6dRjOi{0nI+OZok_IOIh~ruMbZ9SDuvoVqiO<`(na~W$DGtHOTXSQ=O5}cL`jARWMW6BnW+AFBX`SZtJurYQQS{|Qv08(bj~JQv7S>CytKB9 z^X8gzLuVM(8V}17f_G^PC620&JxqUaD1X6*U*0uhA<_5m%%Rmny}i zQRRDE=||mmUPx1`RqDFMMH)U3dJnayLuPK}{CAbh>>pkc%^L;_%|0&TzctTlYzjBi zx>fX8#mEuy_9QH?&OL_`kK`ETC8XodP+Q5 zpx{JJnIdi-pCZgJnaEw<6-ib9y3*lY%f!~eV}eGVHn(irIr=;|hKBbWQ?qJGop8=* zqPQ_(7u9}PBDnaLiznvE=nc;(`nf(=?3{X(YV_DEPI#?g-?cuGSsd`C^QPx;Q8{C1 z;EGt`dc~8PtxZ&bryIw+B z3rE#@dvGHi&FSxZg0R86RnY4zh~3@vWvOe`9sNuPaIrCzo?N$9usE_v-2HD2S9#}> zP}FQqhwr>3-uX3HoODp}4PVkA_Wm`J4wXgFN%xcKpf%G(1)aR6*|uItvX{|9mqPJd zw+6B7_H1FM;V`<|zEyZKGJ<=cvYl=%xFq-&{h-&c%4&;ie~N>D?h+3DXr_M_MNt#m z9P#Uv)uK-R7O`G&2hRO5LCjWFr`PK4ixJOvQ-6I|di_$8Y}AO$!gJL*^v?KDVb!%g z!jIWW+z#Ia+3DMQ^u@P5R4pV;I975{FnM;6c4RdOqr#iS8%0}b!SAoaj;Xbr_Hh@n zGw+n}=<$&n+i{C(9)*%b&deqUGEHw`sDI_g;uKof~C$pAQmpQ_X3d<$7^axdvzL=`N-o zv7uAkF4Lc@>}krDTe2!IeBy~exnAohFhZkMs`4}T=2_~(IIQ! z3*XCoi^Ff7DhF}Vr?GVMkC8PG_DquvJmpS%CS{3si>`=?$q{0d z)<){8?Ld{!ZxC)4{G?=Hvsi6>qGqPcMY?5jKe4a#CUM~6QB+tNEuvp4UEXy-%+y~Z zhGoiVFuvr*cRwvgZx9$Xrrf>ssrh)W;{8o3 zp6;X(;S1=aA5XaawelJ6AeV+_3OFG5k;} zd)?zbSN8r2m3zn0YUlsxg%$I}727$n{aK@U`f(-Ka6gBBUEd)@;T3Lve$QIP9su!S zyT7bQI*3Y-c8PIw^2Oh>3v^~O$6`dDpJ8JRwMQyE8`ltLLeI-B0Jtp@A<27<_;P_!e(4WuT{B<#6 zI6BuH{Qit~tt+H+7Q}KFbwX*@%t1n{rMTiZWZh5R2)Z69wC~l?WxI`-j5n+wh0>jO~p6*TWD9m8$xi|1u;@y#QDVRqF)pd ziA2MVGUY3)xCy_NYt__jxW(Nr2zhnK1fLCg;wzWWoQc6LZqMCu;*I2bDhc|*Z3%Uy zjuKOGqGqEk*gu?V22B+|J=YQ~Oh?g&wvV~@52d18#6~Xp!FI}&Ia%5vv)Yf7j&kbL zWYk1gy>`;84B7mVE9rs>M+B9nHsY@TqQs+4Wx|KR)znzEtH!5phH$wkT6{e)jNa&| zqqpZY(crGH!t0?6#fe&4j>F{}>54$b88X!qcb$=Q%d$3$t{QKJ!@GOb#*a`Dr)I36 z$FiSsehSX}#49~(D<%yVzB=YpzE7v%KeL$j;XSx72LfsKMMP2N|g>C=c<<^803)7Bl`3RM*j(X>xs+ z33TYH7XGvLPEq>(G8bEuE3RQn>8+h(Xzub~ob$#}!pvVg1=X%>q0^$T*!C|&%rx)e zxI9FQzhfnF5o4Oc0`mo+q8`u4DU;AbvKCDGdGIoH${kj zV=s$&&zi)>tLhZn_HqMX=E_#99uT+14im?Rgwfph`$XDrt#Ho#kZ`zSpRmUNhFDfM zR~*+phP$#qTI?}rl6dmuW@>lwNsV_^ifFMdM>vr&o{LxSBP6Cv#D&K`%e)4e(E4}k zbaiyPaMt>mSP+vT4Eu1H(vP!+3h$NN1og$@?X)?<@bCA8G1aAcs2=3?yBY}w=UgSkw1dwNOZ6unaP zj84rl=dOM1Nj(+(7VTedvg_7y^m|g7P?C0B?3uSr+_^PaG^j`y54^1r9|gE`(6V0~ z6QnEV|IXy%qs)cLnfdh7kZt0e#zLwwVGYf%(4^hEM$n(yi|JZh8!_os8Z}56=;*83 zTO5#ZnbXOL;p%385muJ=ptHU#7G;iALR$0`F+k;(Fm_LfIIU#|9eDG&Xs~IDXkFV@ z^L+kk*~jkl#8p?LxI6un=+wV`sg1d+*e`Y>jp}bpJ$0@NUkgu&<)d9{f|nL57#rHO zbo?=)GH)*(M8DFZC->2{1u#o!z}t}zMb%_%Y!be_okByw+ZR= zvoKP9JuOe&&V}DzDbCY!<5VAxU)iFV6ksYkbQ+#)w+ddle& zS9~XvY8?xw7Hv^fHuxPkYS1O#eNA_vA$X9OzqDA;+Zo3t8UGR%7+;coj@G6bda+{s zmPjE*Hj!?7uTdL1zlEL(FBNmvM2h=%l?#7L0>!EA|AhXtXNle??$R&%=V`P5MDhL9 zb+t{m&e4pg02;^c;jBYb#23AENIKmvIMXbqLAfcWX))ZV}mko5Y(_ zExFr@KKbE8#hoysfqrg}5uWr65!UT1qtp4_;_&(Bxy(HU!s1hZgh-b-+U;C#T9=U^ z{*R*Tj;s0o;w>!=iT0AwQoc=f?|sfiizo?|Jt7L(MWnq`QYtBx(9)9Dz4tk{2q9#a zR7NSHAsH$B?(gsWdfoeaeLnX*&pGdL-fQPRL5UC6_z3J!t^c~rtYaX=HK zoW9ba)=q2;ID%s(4Y+(kA`u^)plsGg-p};ubbMtad9)@H&pr)7g@3)^ykY@Kn&ZsA zKO}hnuamIVMG;ywo}gyU2H5qbpU8!W!Vh*D2n+HLB{uR%-jx{I5I{)zQB}wcmg0A* ze*;e$8R`%qj6KD=wE1-c`RsR@h8exXB1R8&__7~!thMNuOg*UV^hcfkO6+^pMGMc> z)7%Js7@plkrnjtwzZXI=tK=r#AV`9eOiD$@t|vr!Lnv*q3xQ>7X(0b=8W4FWIQdGC zo0BXq2)K}^_Wf2s?hEpZ*7d=>drUaI+>du$Mvm50#lhOM6ZB758Xbt!Mxlinc(t&Z z#b1y@+X5HzZaxczLAmv~Fy;`Gc`{kh6{Az53V z@YV}A6K$I!8niM8?X4qt9IGWzs+feMvmWqDOH*K`br4j2`UctlnRL-)CjPgs6|}!f zz^$+8c>I|R{cG6FTK>X~m9w=Fr61gbw7P1XbKDLW`Q5~#nh4C9DG9eOS!1vy8+Irz z<@@}c4RfDW;O$63rsBOz&~ixxy88-{b$b_fs$Zj)f*z{W;ybQ<{*Cz6oMT#Mo`Lf& zHz2!N6PZp-&6$J%;hu==H7O?j5oFs7Nski9|AFwL-mwi>7M$_l?zgSl7XO#V1L3QngvVk>a&+=KYNC5v2rl#dhX zgWPex7+hNZ1dmN@g_z=%bU{W2tkB(x&&&DL{i+sp$c5mGpHakH?lGiTte`!+Q&`G5 zABoy67W4|`!t_`RQ0nl9jQ^sLH!%kk>%vKJW&|~|&A`Y-p0vztJE+tt(}6A}5Nzsz zQn?%m7juFZt{i*T?#JNV^Z|p#8j!Ie7Qa}$#NmruktzBUMs?qSVRs#o4K0TppZyRg zWh>|li;!zDAA3)K0B|?N!l|>U|MW^Sy)y+3TJoUjZZ2UKOyR9|{7wz;tfbagj9~k+ zo3!WAPcCy0N z_cY<4^q}{JPtgI_ro*(8qDsx+*;wsqV-9p? z%+N5ohxLBTBAjr%0bic}qBTGCdG)vyEsxls;sHD0_g;jhMlZ-@+j+d=7Dnv({$zCF zFR1=gNc=>;g0}xWa$?a`5*io-MXg!!gq{W$LWQ%AwU!V!ZR8pP$_ ze?)h89QBoVz!N1Z#P#%SDD-(wHqE$1Vo&XYqi>#*ccY3p<(fVG8k>dB+BMO76NUHo zkI202Nn9$xA?MGNWhKpiMJ0tq;CU#(`!rpC@b6MglU+ynGV5`o+lgG85P_D!EBM+i z63<0ypBm;$-W{U z55`Ga(<;2QW*8Po7m@bqE5M{9h%Wmj1Wm$rv{XbEm;1bBwH!&L%01ci(d8^yFH=rG zFVTb@KI$0ZQH$kYLRbgC7l3RDr4zmH!1$>Y%yh2jy?A?@I3DgI_ZQ#BPCq56bMK_m z9~6i~vkB~q74%QNX{@h6)KXtq-WL$E8oQ(Pl{){(fMAe~2p)cf5$sk5^#fx+oahAcUXO zqXameH~BKW7}uPVVka!<#UqV6{KIqq@#^zLp>N_dJ+>}|Dp{!UvNiXU%l6jj7$}EY zImK9J;*Cw2SDB4A!Dtw#3sxoRBzYhnb#D#O9rAWCqiij-=Lxcrcb(-GtK^}l07vld z5C^f+>9p#K6ht1>rzMeR$$IN@((y4CtoD~d{^De;(yX9~Ir|YdcR^ZMJr3B+*;0UmSs|XKdJW@G)82oyj@6 zJ0I(}2IA~UNw(=kCAsKpz+ch%kB0Y{^56MRuzCx`v1g_<|7pA-TdF{p22>azqb3~5 z#SdgG>p9PF+6L+_{fgCdB@7JA=CF2sze)7DlI%^9qqP0eMHJH51d5Cz(CJwUO|_q} zZcRKLx>-;3Wxv4A;$%E5c>td+8lsC{i9`2{Z1}gs7+ZppQG6}~-mEUcac?L1=2!?H z`R!C>=^2`N&>lN?RFEUnRY-c)1~Np71sS@c=-(DhQv*ZLe}OzvXH|mA^D-RUpUXRa zcMG+BB~O=snu|NqZ_{DduW)wd0GyLt2cFgWr269~n1Aa$mgJO?s`Fl$x2h8-lQgk* z=1t`MHDDum9Xh8DkRMy)sh&z2d{7Ssu8R>eKW4(BWhwwps=VHZ8|doZ2>9F_k57NZ zka;yz_smSqcw(Qc+d7nzoAG#mBzc;3H`cEBzvI{`D&;ajl7`_KiXB6;9Hj zpP_iN@&)v&M!>&rF@CqwEqd(tRCc`6N>ZfR0?R|Pp}$Zb9iF}7MFkq5ll49na#n?3 zpS6jZY%R71C=#`2FY)`nGt@`rE3Kc_M$+?UgHoC&wnwdl{O&sTU3PExB zOcXs~Lnd>M(q#>@ypwueut?sFw%(Zzhxlhe<3B0b(x^(-|1N^{&N1NjWtiODJRMGN z@8TBDdrD0PUy?lSo7l+A0nSJbZ%N!rczSdedX;ctOW!0ZUm7dmx&Fej^A&VjQV8eh zv`mbhRgD~D8UBH$aG2G2k@wx}DzK!gA(*$GwSI*o?6mm?`=p}bM*BMU)aCPe(kG*7 z-U(ac^kELM_!EH`_J*upmV&O^o+ERxj%-kVj9q?ZWa+i7xO?Oh8fTAC^`&gIuS>w4 zujTlfDaF)wZUdRKG>@2Z0eWFRxF0M4)d($U9|3aOClT9)YDvTAnXvkKG)mnFLF=l+ zFdDW49pmjtfSWaT?0iM0p4U?il7$*|;RSrD!E||NV$JfBYntUJWQwGfdZHj)qX0@^?e=9JiBJo!g7w5?;}jk~8uCZc5qCdAQN&8qdb^JiK~eg0`v$ zS+=Id*iutR_pJ3nH!(5x{Vf{&W?Ls%d_aQ#H8>Gts$$?>e&O)8{Z1|`r zhUa@uVO8KU__i+rM!l<$cPI*~^Vy)+CI{|AHn=&&1iG$Af{*GOa-eP|c9*r=X4(sUJNehpB@e@c)N{fu}DX0n7M zKUv{tW$?VW4vXZ za1_}mqs1O>EW@?=ftYUoo;DLfPCBL%^aU+P7`{mxPsOF-j?a-4&#b_jOLN(> z=Oc*FIwP9=w~|J;{)9g00`DeKeu?D()7CGKW*SN3tb7Sxu|WIq^Tc8(Ix-iB`hIfL zx2Mqu=cn-e(i*|Fx!evcUETuQ%0?W_&qIln2Vuz{i;u2w%s%8@_Z(yfs<9XE}Mjbv>+& zYQ;TUMyXh#HaNwIvM*;jW5nDXQoz5)vfBQT_py=*17^Q@KOev2joB9Sl#W~n%j`1X zGy7prMK1S*aTMOauYe!CkD}d79h5Il0f$Z&Ef2~7gUyztZR1pQ%}l|vcLA)5_vc96 zT?N)`{Zy!6F2MFr2gwN=!Snx?gIT>@a6EVdw5z7^y+iYX+%5tBnhq#`wFP_1`e^Q( zZfZ3>i3DCajohuj$fBZb_*@lEjKjW@(5khSECZKFm~Cl++74$}2OsOhz>j+9>i>_}pU{MNvE#5L zJ_)3)XA;Mu5?HdU0(1v%ves?*Me=1V;M6FG<_^w9mirJrabp{po?HsOE~e1-&kP?q zdy%=rQ`wxjLdK+yW&KHTxljZjUQdCNO`TNrtr_fXe8hX!6+vA*mf)M& z?=kz~UGTi)jloLBWS*CTK)W%)GvCU=mrHxd-tlIxgn1GaEfr`V_Wy$@$7EcX7y;r{ z&*=6QDllhfI89Ts!iDc{4VVNCJ^>^YUiS{$H(^U5M2s%8VO4ZbdLF)On+6z6hR z8Hdv+{_#A~YB_$urvUoWPjJ4L8lv*>ZrCcNMc@CYi3vleS(3^fMDceT7Ukv%IPfY8 zzFnkw*G1~tpGiL0oxx(cF!(E94+jq0f^5Kh-fhoCc-&5&9VTiBV=`CZnQkA?NkgD< zJ6?g0xTgqjOFk$LHe*U&Fv)N7q0^NQquoYT@<+xAG}amOlOO3|#l2co_`FBJk>7xg zSFh7?L4NqG;wU(wl8x?WWiSj9?3~Y6h)~QiNHTg#x9i*C^0+WeE0n<}>EftunFGHo zwnN%48HhA0qMgTA;(sTVAok@K^4Tx}MJ;McukwC)uJ;hX1TBLHD&aiNo9B>w;{s&f zen*zY{=ssyPpn?ucruH$fyW0`c0`^&_SAP^AHxTw85?nmh$M`CEd`rDn$Q=d%on>V z=yz|1gIfqd>e3N77kLO9$1dQS%`W&hS{OIo4g*E-(^(;*le~68KKQ*nY4-W@Xjr^SjeHwyrw2Y? z0G>-XDVn(k$O{R4FcAk=I(*=cVhWi&$$^jvVS4)I3plt)8hU5?32HqD)8B1nT^4C! zC5&uB5@mpwEB}LEtEoe7b>uQ6<~ zC@(oj?j%Rxp5F%W3AJID#S=QP=m=~+B8A^OpQ3-p61JV#K2}IxB0OrzfIC5<=;I)Q zqq3XecfdJxo4yyKXMBQt?tp%hS=<_S1}}D1F4zd}*DV}X3?%uGJE#R`Oew^dZ71=1 z&ZxM5KZMFiz@MyGY?v8EJnUR?bJr6LjzM0**>jL@*e%GDj-)DBfa{Onrc0;#;J=jL zr0vs9B6+kNOKg?sYHJ;KsaqTkz9@-qV|~!ntOxPmn{h?0Ds((r$#ULr%wCaW2-$a9 zaG~NADqi3Q!xxIloR?zkgx&_;jh&x~&R{h!O8X&}a~#=8!BJ$z$Wr)VwU76};|fNX z)j+ZIVpu*JiKb4IG>$ceT~&X9=*%bs<#Tf|e0d_A&*DO?cN=p7FB2ydPD6LG=vp*i%{hFaNLIuZ;-l~us=Q3V zaYhjxd91;Ix;hMAjb6aWU5&s#;)D0q*MQv%A70a`df2?`EzOt_&g;?3Vr7r*pzcR( z*^gB6$#*3l&%oUZ%t2N!7dXtyrqPO3R3(}e+*IMU>?}-`3niZ7sIvd=4j#PESQ<*(WOeiNs!?&vZX)+ z<;texlG6+rmM?+ia(QU^>IVh|Pl?>M0XRKhoV}0Ffw-{>G|`aeS7~hnBQGKLDs%>z zBLgXE8JHqmYTaX_L9gsP3o!zW>cG24q{J}*4*r}<_Z)jkJ8Vv2&D{pp35VzOzsJ+) z=!`fxs$~M<_a*qPk7vTliKSrOAd7A10owN8;PshXL+1!zKqaIR^~-0d>fi|el$}85 z+8xjlvwQj&xl`xAD7Y zRuGp9*KuJ(BWtRQK%Y6^55}HnV7bt0s_*4VUoG_F)eAIIb8lZFYoBRgN{=z!r+69M z7i@s#?TBY8b78Hk2<(Zjg!?Xyq%{6LwKAW9n?BxvV4?F!OZQ>ze7itSTZmut$P5GXMbXGQlIE>njtY+j z+}O7^*y6hvHEadF<+i1`)5`+8OeEPA;f>hCc}EyuLSe@=8J?1@HHNe^zz-iGxzt9@ ztjBn*JGA&3QB(O3;sttB6=N7R38&Lc=0WH`VQhLni(Tb>6T`C7S<<%%aJ(gOdagHc zvJW9Ny`?+PxI^R-5p=FR314>~q4k%r!l;=j-$@aD~VxR15r+%%S^7{fK01OT-3#Z zrc-MLxYZB3+}sd(d-CyfJwj#hAEI9%KT^7G+4W9%fbGkJ6sdb!S=8gl3=n<&moC&8poe#@hK|>+_-d*V?ArT; z^m=T;sD5w!CD7-TO;(eJfGgm==Q%C88HbXurqdt&Z%B~jE}kO|gR>8eVap>G^jw+* zS(4%Ouz3=8dRCGN-yC=qn~ugTdEov1k7_(yh1m+(y!6`LcuoL>JZkMAyBq9rmrXhB z?mj@=$5U9D@fk4Clf}DuDxCZ=XaLo_DNw+?3pp2SXxd{dbT$x$|1>uXxJx!&_vQ`l z`&R}ph8yYBd%5(F`)%aUoPpV?vuW73RFbkfgf5)r2{)C_kuHBBGX8ik@RIJ4VZ(W- zmG}o}=N;T$rh=d2c(CMsH!IsQMBs(E&+`>%gw`E<42fhK>^ANv>s{^nAp+QE*$gpc zOMk~lEsmg3HlpQK9@Oj+hef-KShb8MNS$*M44clQ^@uUP(+h{1et+1v zeia0m-v+yAd+h04Mwi++keT;FaJsz;9+tkxvMgB*mrlKp&UDj2G-m7dCiTkIx`O|Z@RFb+3p36?rKusc%3d9i-Bw3ZW5KTYqYAL30mEP zXE~>ZsBWFXb6mWicFQ#rMW-9Q1JbQ@=WBgjwD}dD<3*zETur=RLm*&!2;Scwk77fY zY2`Kzrk(a?BucYEpH1=Tt_)&aTo0uZ-?{0Dfcp-Lft%<}m|n9I?;q3w1&{>c)VuVy zp(-!yg)FR2s;5_$n}WAOHAV@rqmHPDMDKGIP7$}KZt4O}*K<)Qzt{}PlXKBvXsQ5z z6ZC(F!^m5&8dNEM#@zT-53JnASY``{sj5*Wk-8xv&=V`M*^(-hio7HXDtu^s)Ho}6 z(wL=aTues{$06y@TgXv}V5tZMYA+|G_=4Ch+N0r&RA&<3^o!7&OFckrzZY~=i=h_f z@U-o-(f^GTru{;egxOP4c4j(%%}Os4F;)&Gymja&tjzyuU4V{0GHm+|hO7^*#neq{ zfYl#RNWZr{q7P4=fscD7k*aH9+OiyG!Qp3ca<(v#OnX-R8fENjD1aAl_M@kbJo`X# z6nP#kEYK7D!}mCZP3eNO;f@0Cf8jz+KD>fIr*4t62W9xl>2GjR#ZO$2Qv3Gi%ZnH^IkUD5VrcykdtmgiuN<1CzVGMLphvyA$CIKtQQ zNQkxYrIIf`BGj6}i#ygJ-Ea;@kLSVlPfxJlY8G>~d;-sNjvoK#hB~ZyxF21j{?Xpu zqU=vC#rS+eNl=g5(CE`SmX5(1GMQwKi4*%EJ)?^BJ-LMn&X@7((OhW#>4fm>3VbZ9 z!=`H+S+AF`fW`m`^sCuOV*R3UQDYA7s7WN}cZ~4X=qaIibP((0R$JU+h5T{sH=$7ayWG8a|v2Raf$h%)ey3-gZi?rljHkTc_y|` z>5RXK_GnIavSjdV;0ap#MT*8<38yX}t4LYyBz<*o4{svuG&n~_QC?>##%DbOua6r+ z8Q#NNRsw5a%?Pvp@MAE2Q9(~=Zvt0^y>R1Yxxm}A9`WQha_E%6D^j3Jf;E#FGgMXh znfd2gD=#GoFqj@(?(7NfsWY&mRu$*E%7Ojm4QTT6BCBtvE)JGNk|m>|R99P-j4{!l48sEU=>T58_de5R)%23w35ubIfgFDBo@Xe$Gy(v)+{NZjEiPJ~TssxO; zzDUC=dVnL_4CfR)X;s5!d>~>(i@p_Lyl)$6I57e1|2sir6{cfy#9ZDTszJf70;S@A z5RD`6>7z7Zn7KO)tJA0Q3v1(m*;Pn3);yu>D#xgAs16KIr_iuAk<@N0C;KeZD7h){ z|9~=a$X3Juq?Qu-wb|Tke93HiBgL1USB67M)$nbvBSaZ#(U~IZc+meF*S%K(!>t_g z#c&R(+<6j(#gd@j)d*q&j$(uUHk`Gyftbf=;Epr*@vc=jb~=Rcwp7N_d3j;P|I;|w zM&`qH!@pEb?H;Ak?o{`NBDf0$pjApfoa_jPFEO^*kTwT;6>~sr-f~>qA;Y_sZ2%to zbilHuo(3|TP$NWx74zt*;Jn#H_dkD5mI?R-oo_{mp3o#|wZmvrFKhugp)-Dj&Z94?(KW5<)vAO(*7r*hktyj_PBz1o1ys5C< zWE)n!_9AdE8#vvs>C@UqT(4h%M!PQI;>7Fl>d`2fwzE#qd+70kM8W`0<&*FWdypk2 z0jG6mgT{+_r1y~kzhUUnlvHv4Qp-(X9yA9YEjELL`3A&j>>4ZL$4slVsI8)9kZyUb2;w&rh}Qs3-E+k8lgS4 z@bM0kP(gGnUMS80v7Qs? z9eABS=;ELuod&sYDLkLAj_<`&1b!L;#$2QYx;lqIpPSE1VYcJ;nr)DwYfF}H6-Gt& zG0>Y%_`8l>#H@!&biE#ru;=H{9*z&HW#`f*Lth9{$)wu91X`6%%KRJMj*$6el9#iw z4-Ceo@u}T=2>hN(x`k}`(hb%4Y1TtJY>|UOnIm-O&sv;Q7y_%T58wFw8!5ldd&c1V&pkP(&2T zXkQJy>Pv>YIVXsu$7!^#GbhsXmBD>e9cw@;AAEKn$1M-!*!yBG;ka1~Yo?O{q(0rt zGx>A|k0}w@UM$C6XKsNTr>N3_7jk?*m#w5uQxe>gN+4jbJI=V7Mc(EApa(Ojqo!;K zZnV%vy^L8LQp-+y9FUtEaCq@$QMPDeq+q!^!xZl;GW6moY(6wt2nbD+K@ zokp)eMBc8yfhi_cpu>}etGjqmVY`a>m*>M?hs|h(PjHd-7Bm?64h63c;bXrc+AJo6 zCMlLs=J%Mc-K>WSQ-EZT7~$17QgD)e3)G}qVZK`&)3!m19=13S+)hcb z(0p>TNfQni#gXW;x%7uqH&>;(oW9gL%e2`a2xrRr$j>)_sb$y^I68F!>wWAunq%h5 z-Y!1EYB~Ozdtj>v`ly(at1`a0G$96?P5X)bvT9g7#~xF5Mv&oO)x_hZz%%8ag>9PJ z?7<;pFlqb`MsJy-P{spjbN)Xr6N1;;pJIvTFK+h+2{Lz+6#vrHQfT=miywN2(dnT* zy$R#kyU`wI)rsPj^iqzF-eIEECWlU@(%@B*g-14tLeVdM=jONKlvW7J!fhF`9orRA;7yg`R&^u?GK5$!w2oa%21(;kY^apOo{h)4}S zw_XS{Yt`_B@=hpIc|=0Oj$=#fO-OTF2KS<7us*qGpzDiB*fIYG+U8|IflnCeFa(TcGe$pD^!eH zN!a&_Sk*48L4EcL;@*7$$1+}{rORK|{On2Ae5NVfR&|5rE=tsQPc905ti<qK*gQ&(;^?npy*w@wm1(;KT3GD*W`0f(Th4OS-CXtCNd zs^=6&D;*R-KtIK1#d^c^~{RS$7+A z_bL|c(K;yb4ZfoKj=$(lDJA~EQw~~09Iw}#cL?qO?ME${lk`c{AxKeLif8VwhTDO; zV7}%ZoZmbZq;4i*b^au6Xgh|kCKsUdpcy@~CXj0~-wn<;OocCXr zN75E2&;vc3&)!!0l=bd<7fZ-sCoj}FOyFG##x;rg@ZQ!PW>ph7W51blZeF7!+gcH# zcEQuW5e&AIAS)&f$q1Q8)H0Lt(<&i6daDYBj$eT*MNZUP@h?HuaNuYqP_JYcTsYFl zldw*Qh3T2}mHBj>ku1>jcwZ3s98)p&5JA(T0hma)goz4&>L@z`kLT})U4@#wEiV4J z<4``N9cYB6cbkav_7c?FGKu=i`|(d{A>^Lz!PcEI!24KCH-3|b``^9MbdfBc{+7&I zRDB&U9?FAB!R$XYvxGIxsD>5E34{VQXCiUPgQhqtP{S)LDR25SENnUjIG#nTtY@Hs z!D||pW(c0CX4GZz0lLP@2amj5hyT7R;<3B2m^E4twzs8`tv!Xm0+MlJ&03gS!r|)N zIgjs?TJb{BR}y$z9s*vNf_rQ(5xqY_rIcdvt7H*2ZVzXb%J5;Ei4&|o)rdh$RN%vp zv!pYjgXRzLu+V)Ep4nMLzDkZl{|tY4sq28|8RuZ$@-HMn9+BH=1{qd?bU1Da{3&n; zR@i-d`hFCB*_B6*$?t&dER)S=%L(sD9)YnEgf$>OM|| zzfvP)+us;!WB&$rh57TXKCOCYlSk&_5H` zV2NiGbVo?>bv9RloXQlqzMDdSejgSM5Vi$GU*UY0IdAd5dk%3^Q! z8vH90#?ucgrB~a7sKep;toipl@xbI+2<*$_DyK`p@QFr<2RU}t{3`79P9fUw_+;;o zr?|CHk{>0Q<*tmbfw8Lr_;HEg|J0fVHSG#`LBOvB*PTbNH%wIeEJVNj*@n$(C*g75 zc3h!jj0e=L;I(NuH7K}*EJHv&#M z1g5pv5T(#Cc-S|S-?0incHEU}uj(g@98X~HY9}0+rpkX^m`{C`Goa}A6%^i0SyOi2 zrV@v@fYfVIwxyUrU-LtUeT}t~d}_@_zE}uJvT%Ym4?e(+v$?Rw+>W<-lL27)QK(h? ziYsj{_ zylDD}`kQNkr@kmZwxgV<@+z6uZiu4$s= za9m4S%RQXmt78oe88CPZb zoS-n6O|z*>Nv} z{QK2UKFqkl`uX${n6+8KR&y&9XAyMr+{v@ifC2WTF*g2H@-y@W9IffJC-y!!+F)%&*$Y9(R9D1jZZLaA={Cg%D zl=8w}LWpR60(SdfAR-@Q=&pzL0x!Zop65ToD%*-WvPjGnZoLzDePcCor~eZgbUGAa zuL66Py(LCB@kyls6-^(Ih`MTf4@DOV#Q(^;N9EBY7x zwe}uqHC>1Ac`K0F_lLCo5@YLM72&7*nt}P8UhFx*=4~rpgPYV{czM4j>D_6o>EFmH zSk~E1f`cpQR?V|$ob(o#t(3;RN-5Z_kpUZ;IhZU^#21W4Kv(swawsP{o0^)hBO@igEH9aGEV|soytpL| zbzk_97R8P9Nc1V3qGu-IV#?ph!r#Yh|rc|h@K*k>!m-kt_O(n zPtd)_A|A@?~q_ z(>77s*Y8cHmV`i}#X^*Esv%OL0ieI zbzt+Fb(C{#J84|!f|?WQcv^K8U#&n0>)Tkc!%KkwD+9}1;T2Y1Hw16S#Op z43gJ{!N3+u9EACBEGrGVXWavj_T`{;_X*ZN+5pMy9-_F)3|p^#Cy$JLApiJxI^Rbd z@5eoWr(>7U?D!R`@=1&zq?rR1dIyl1C4zaWP#fYbZRXlWgyr9e%6Go z)jEtn*N;$n{oAPVJB>K%e;^Aybyy9lqA;gp2HPY1Jv?DuVJV2;gcsE|SXR0eyDmG$#1?kRB9Mi^Wdtp!QxEaWxlkf-VQp`_je{`krvho%dB7k;=-Q0JNZ zQ%LKP3BpU;3z<0;)No4Re*bMet35%w{)JM0 zL?W&D5kYrMR;HtdW(*Tk>&uWZG+(wm*kF(ct24a}{K{9f!B# zv*F8Sa~$jUf)je_*asuT<&qgcyh0Sb{E}c?q@0zdp93AcFQbc^3omkuHec&q1x|2g zz&v9EQn05Q^Zl055anS~>M+LAR?23|_Or3_q!@Ussk8grG~nQg7rfC>HQ)y(kjRgT zxT`@7^AaOa&C3cBOpLJN$!vD=4jZ_+vxs!8E5;9rVbI+pMV@b4&ocFthVePGk?$(W zPr3Sq6_~aMg&OaWwoX1#KV3wE4eC+I%oyU2da{m&cHx_^H+gfNZD>~KCD0j_g3{xv z`0%%#ps$k#7uG{4xhV?w;_@jo&<}TL7$SrH6w^n<$?7G-IKyHy{P#Ku7R(=EZI(Dp z{X}kn|0iqKm&cEB*uD;#GIprMQH9}fJDTxY!26!ABWVKvl}Ps>*8RC(A7D9VeQWz` z#^3Ft-1TEWZ7!JJvyq)5&x|5k+~NzjIiL6kxY6Oex!T`Dn8w^o3`>Xg%$=#$%m)!$ zxdI~v^JiEBfQIAsh=Udlo5DEf~!-n z%hiW9+|}!ja#L-BnVln!)?94`?gx2&F83CT`DBD}b{&>sddk@{Kfm6|Ns`!YW`PF!d@%^4QQ&X@<*@AhQAseg1eaaT> zm<={!>Z??7ICqV?R}XIE9_IHjJaRQS>lv|}t6`qpJsdISEgQx9`rgF4&kN4izf3w= zA37?;*$`vNH8hCi?h_Sa+CCQMzSNN8HXI({95_?OXmZM}^P3#y^f-Ly+*mBftk5W8 zTwfy1Q5!#Bzx}mR{qJq>>dV%>=M?A)vgMgNHiu2L7+pKuI6t#x>R&DCtskGY)8_eD z0OQ#XT}G_kTaK5)EJk7VUxqf*j#0mP5qIddDL2gQHK$qlI!CxSy54zRHe=qhy-e#f zvCKH>NbbKw0~}XFW3G;kt6+EV07FGpnfr5|FZb~QSFZF_YvzJ=D;T;XE12!MYnUe< z?&J1{mvBxN|HqxR*p!<+Z!R;dx0bO*+Qm9BenWlzs~wEECH@>kUrml68M)qE_h0>k z10@_0k6WDFxWycU2_9qNhjts=-z?^@(9euxmU7%cHz8)2fHBY!5@PlyI&sx|%%4$*4deKM&wP8Cn;hYSNa*CgDyr2(h8 zov?*j=*O~23tGTc(E#q37i*cN@;pxbvLTLb_F_T4_Xft}&Oy$xJ5!nIpCy>WiPN|W zCC4}?y+yeBjL)1edu^Heja-`}kMG!+4h%5N%HDD^rUx;i=bmIpud=Z@xnwqXv^uZ8 z`9vy1zR!Xw60gOj`HeRB_n%>SR9Do`9z9#vJXp@K-6+cpnPbS5669Cw{o84C`>`%( zcr1ibJl}%xY5b9Ot>P3L?^mlC)}k{wPn`2PIemKE)B<(xn(J|l10&@&m5g|fUd<2A zvQsmeC4-)v+^k;4)kCWoQqKC^BxWgxZ+6kzt~b`kycswPy1N(|oFax=!bL{f#bq|< z>}9#f*zY*Xql>v>foqxeE~3_QrfH1E3tG&=smfeYXDRLl4^6I((ot^t_Fc?w5j`fa z&XMVR%ZvHRT#x&#@W1*OO8+>Yw7zg|kGy4=HnKUctp_+O7J73wez#zlrPy*zq(?dN zZ{~BedWIQ6uMIdpS;EXG_eL1nC#pCa?ePp%`xZuonh--{od_pddn)(EdTFk`l?6B9 zW-p_^T!tYx`GoU7hOWbr$~KD2$_NP|TO}ep?t4zkNLECpl4#H*Dy3brDU_t0_K;Hf z%6-q#UR2s>XiL&gyYSsV;okRo&iIYz7kID_|5<(!M%~<<Y~DYcOD3$rmCb8}hU0^2-oGkp zn^*#YF@L2dZ{_j7h#WC~em$LPs)n4*O<-ozN8oS0=)h_>x|QcFopOJgu=~Pc5b7U6 zS>JN8WK$;0ch{sXtINPD=_zHE^&~ypg>e7oXb67Z54S$?z+Fq{^Gb_<@L$boknF30 zy@ynJtoAx#%(LxuskfcDuQ>y{FPcqO%hl;h-XyVSa2ROJS__}A9fqFSy7)Cdjt0Fd z6(1%TvA)JoR?3=yr$29n+jm#Mji?=9_MZhU3k{=|->%%FE}4HFS%oX3dkCKzOgN{o zJJ*B{K=r2=q1!bp@De`>6Ql##$}~}0EUP<)Th(!0SOBN}YJs}r`=M~J7oIYoz?WL5 z38KC}r|Q3>BOA=I!r~q1 zI-whk>$dCii{#s4b?<@vz2Gb)_Naj#mp0JZ51mx+odNH{DkW13_riTv!3~FhlJiyr zT(Hds<%_*ACnJ`9$%LIx%;c?d9{jD)7XyYmkkytQBw4J--}8UNNZGsD9U9YFZ|5r3 zx|NFkUz_5rsUM(Rz7;%dE|Y=eAy`J&(ec?+DQ5E~=&iI5I=;tZMnekPZ%gOBSv!Qq z6dPq}iUz3&6OK`T!5K2^1AWfFdGil-->VMRZYlE)Q2`xjm?V}94oQQmL z_K@CBSL$6e218tAx%A_ga`ncUtZ{4!mR(*&_xqW1(1k(lEoX_%LknQi+8k=E+C{r+ zl%VIKB2gV=nZ@<0>~-`K^n0L#;ReQ#`DDK=$H$Fkh4&}lcr96&;ut==(iiRe_T{1o z7Z$wd@T7uSc=y3O;q`7kT$l6!f-;pjZcA6#fA0@0ELXse-{I8Yc?+uZT+#IER6G_P zz*Yu^41W0pc_T^D{)S-ccTY&%G8{T1cYu>c5#ggj61kqA!QWdOZI0{6zM>KztoH+* z9qWa^L!8-XXR!Fv^#qt0YeC90U-t`UUC{r<3qjpG1~|i?hTQlMRw$UDt&$b4^NwMS zFXt)ZvjI<;HJ$@&rsDNI!SJa)1qx(20No-I$UNAS(>!rfTs;n*Gp`1-<27@zry zQeUZZ>IqraYVZOWdeo47x5@LwcnKA3tEP)(U zcRLlj)oKI(6?~!R?^~d<{Q=y~F`9_hE>SnSH(S~ zihxsx;Ua~od(Mn znWI+uTbMiV4zxNh;qceC>^uIHc$eG7vYFq7U%!vT;?gmA@8=Y}W50^y6)Ir!V0-k~ z@e7hud+^K1k5a#(@5NESc0gZo4V@2i;Whtlr^#2_X=nNu$d9a`pVNv-@a)H*6zpm8 zhc5W?eKY7@&_ab-2JEmaN$Q}fPx0DK6pELC#QRX#uNNu?T>#~xMJ%WG8r)k?!v68A zz*EZ@Ha38`naAD5cC0Hh{(E8*$ValGs2&0K^*C?Lc_dMXt6>+?k%fH2Nknvu<}z$n$BSfIiCitbxA@= z=Lzy`4V4%wC5t*MOnJ6TJjL^I(hRl5uI9#c#XlN0Zz`r{9Szua#!J{LrwP{Q35E~b zK(cscaC8fXS#l-t&g{6DJ?acy*rCLl=hxHrD29*)o8ffAAY8LVKob)$e!cvlETnLf za4_pMjJlWsptcw8z4gMP3IVGdoVe?5Ls;|B9yM~0LFtxOa-JHF2TG5Ex_bc}ni@`? z4tuGsYo;i#*hZlZBVlXr1hMh*RdNjeMlfg)a;QGo$ohctz$sW?p3FHOX)x@2GNd(} z2IUhynKfUCX#ts16%^@Xi#`_II1B4mAEfCY8t7a1PBC-60-ETk;r7OvnAP!#yi6w0 zQ(4~3+kwhBYo;>KPnUrlQ`W(Z`HHytTQ22p+DOfb`S5K~J)qG^BDF`tpa)Ij?Ps~v zo>L*NbowI>{HF>}Jo|yG?RjD9M=4y}bp=k(`6=98+#$Y-GQ&l(J+;ICrZ~G>AG{u7 z$dR#Lye(%ke;MY7N!zDF=7JQyaVnNAE~j8m)hP0wP)O=?nn~(a3rl1U_xuVgJXh2g zcU9SP$*g{~u)8uYT0RW3_vrG>$7N#L27RtS)j+DXFW{i!U5Fb#6PKoqz!fTSd?vV1 zxL=e-fApLNJGFOKB=an}qU+WkIYY z&A?GbT6nIfC5DGvqG0dGrQ$S@+c^P-eANQqHZvMfGLKr%f29NK^my}RJ@i6-d==$} ztG;Vv`QlY9e^^aWjk+zi3=F4zmqtl%d{qNUz))x%o+kLZY2lPVPsPf<`^mA)jl7nD zoZiF=nvFIcIKE^Zn=@no<;HK9khayx2%Qe=mw!iG(y%`Fbe zc%^74)$P38e!cbvP^Q8 zPB-`jH_c4(U$hgBIBJ9*N^roWLF8sR z4FBzj$NdI_`HS`*_ZY7bsW3M~*l^}J@a~<$PtA6z@e-oa?k`1YkLzUpDhS+X?*Y3L zUAcPKF=E$;^k+_2UUvT%{Cj(!G_kxEWxI?H`#UX?+Ue#BHE}1%{@rU?j-DM9f2{(aj|0hYnVe|u`5ngE7$9A1 zhL0V?uvgz4{{CSgeeeHH%(Utta_ol}^1RUF`4V(!j7F(6mnSB^h1qgu*u1tcj(<6U zpMF~h*CH$_v!#KKT}!8gJaZn?MV2r9#GCJ53`1|Ha6zwEBureA4Vsd3)YhU0J(Wh_ z!6S?Dzl3QVb!UgLo3^?4-7ppw$!oh!kt*{!&+Fviq0H0g{)Fk1-_V+II=r*J2dD1o zjUH#FlI0Q|!U8*TEmOs7I{#o`kO^*lH3uDeK2JD^R9oDe*8D`W zRaG?13xgAJX9!L;QPtW4ax1gNNf8y0U+RxjFFT;JWEhv*#=|_vZEzu#Ve>afdT{9z zT^v7|y$xpZbx-jX1#D_{sR@zL*Zg*u()K+ zB{*zPkfZ;bp7?AR_Mdt~%Bd0#Iy(hdjQmA@dyV0`UL>rL+bl%J^ne|g;)GXiDwwEs zf-)W(Q{)7lKoXj5=82E&?g{TAL-6S5X?VJ30qgWT1-EX7pkbsZHZ-l^ z9^zJ-bPqU5Z3KU^R>PQz9P+Pn;cv~xJZHTR7QW9FS7@k!#)98s@RMvfb!QK)zB85o zdq0s~ucV+wc@Ss*3SseB45tqnjw%COK_fgJCfZ5i#lSXttFDA*Tc+VDg;?CaeJOi5 zj+9QTQAW93DH!Gj@h-(2h_K!TM|LQq@}VX=ne0qC<9cw}OB>#|!X9@A4JH4NHaMcM zJVvdY&L5thCRtQ8j4}l@`P!d*KTCt4f!VO6;u7TR>C@pTdDgwQl;XGBbNlx&T&JXt z=03Z@_ToC2?>q$B(F7MkR)f@9fL+Wx~DvkT{e=CFA% zYC{rzUV2+{r|G%)_x&DO_OK-${3^%EPyP7L(loZLnuUp_mV$j|FIjG6BMFwbsjnN5 z+@u?Hy{i&;&kV$N=k-uqJpr@2_+sW+cm6H)6UJ$*r-^i#EGHiV@#Zc8OxD1Enq!cy zqWNJ)9x05}2LBTZ=$5X}2mfvmXUeOh-A6NYx|mIxgQw8L8;aDY*8%BOgI=5#5lC-W zb>TsU8k~~Y0Q+t(5^G&VxYCTcLgL2e`wqcu|ABP&nH+CWy$jwm%<*=R8xHXC!je6f zeA2g>zWJV^#(A3DxpR#`Szq7{THxDDo_I|X!V04{($azcJioUO&uBNs&uyM`W9wU~ z>$)(o2p&s{#@Pb?Tt#ndWHaOTL3rf3E}P*q6K^Y4mfWRT?M3k8 z%3?w9&^ro@)+T!~3X|iFXj^?0%*u2{&k1M5%8xT(((FXguS$pHr~>F|FRPUe2T7wO zlpS`wgtMD%uvb$nOk1Fa1|AaLloLgFR_&sT6^1i9)n6_J1pwg11fn`y?J3Ys+@Qcij^| z+riEshFEE$56>q#fp%aK2%3*6&9NtR-Ovly!FNcRG>KykK1w>SOb3>onZW9!G}iwd z3_3Lv!rxd?ZNXuhHp3K^^4)RinZCFovN!9t+#<8#A85^#fjsEtGAKV+3%&8D@W$8~ zm&Y_we$``oJUC4<_;)$<2yUi*i*?v6P!rT$&wzsrxA$%Efu*!8Jq$+m&{J zoq;o4C=3zY3_#4KV-9hX_|1mZ8eSWd6gvlj|&~u>9nHv2HKWaB|K3YK^>7ge9~+fXH7}u z(dQSV@-uywWLk6XMIVk3V)1;8I=C#~1Xt$@I4;zktyVfhW7d9|cp6 zGIT26wayXW4m%Fw^(*i-W)`pOHCMXR`;Aa@=b`vjM}te}G|`|VD%5JO0j(Cl#K8OA zVX6I43aYOXA3s&a<=Ij6NVl4%4j9g1lO$-rVym=|`~iyjbX&~zFu~_OvuQ-LuF$$+ zo$O3r75@&bp!>hO@;W~!mTp)K{ofo0qZM;;gJTpPdNYr!&31@O1`Oc9h-EZ;p#FKQ3@Vc zWd-c+QV2z7iUcKtZurvsHEfTvtkm{T2HaZ;f6Up}B3pCh3M z&9uZ{9TXnOazIb(^Q;HU;P0H*bo@vj8P5%r;j4C(@^!xGl>dbSznS15&sQ+>Kqxfl zg<}0kBhDS^KsBiX?_RQqFN#C(c+YzAR>XK=u12Hus^2Kudm)1Qu@)FH;Vn(+ zQ9x~~H)Qyl5vwEMSIu@BFmViTDOKh64W`&5>#A^Q=0xoL}NeJlrVs^-GV&l4a=+yX5d`@!{lgJ|BqLJ_9N zLi4%~@#K#f@x_>_!XdYP;q8FDv{Y2G` z!F+1P6pS1%VLd^IubdmquTDha%A+B0<=9;QHGMdr-JOg-9zKKK4r?V}^HRYmaVyo{ zJ5ODd&XJ)(g1Dw(9HqZd-%3sTEaXJLfT8s@U{#|?&4w}3zIOI(T#9`0&Sd;) zQBA$i%X3M61--8ON_qs`R+oHFm<;>>dMVle9K zIjD_TC;HjR!QP)zYDf&Fkcf3K{=GT89GgPRugl@GY7-2q?~W%2g>s#4BHdNA=gXr8 z@xCY4n6Ubs(DFnTv;J6NzK<18uo)@T6ltTSsU5Dr*o7;%Zh|=>wV<@J7COf&@{(Vb3->eksGyPHflM3vdV!afyk{d&Y$`c@T zwj-ZQ_GQHd@x0F{3KJeDNc9Xi({HDnG~=Ts?3epWaz+VI=_ZfL)`@Vn`6}SV`Q*2~ zJG;lEQroSOu@vv!97bJttMLBm zPQ2im5q?ta4w_OqjHo&Qsv8uzf5|Vf{HcX!)Cb~-js4i-)dg`!oG}zX`yzb$T}xi) zipj#@(aDyoOLW6?9>m_h0!Kr2`ROK<<^OA!Iv!S}g49Jg8v51O8l@~ljV@1i9r-)f~FujWg2 z+~ip8>T|Nc@5NQ=uY@-Lp{)5alP9ES@TXBZ_;_3cU94VBCO!1IBx5mldpr-`+Qsmg z*hrq~I1dGjX!u}$PINpK55x6ZgpT_TqS^Iz(rg(#VASMVAjOg1sa>huEW z($oUE&7G7~DM09gBr2VdLIt7oQM=;`7?07#GcM2Q(oPLk8vGob%D4To#YLLb$1&T0;W?{Wb+qhd6kR=4G5gJYuhn^>Np zV#~{(&c++vw$X{TDdN_N<#gN877m;`ME==+td{A>FaBs^&bfyWb$5^8vFik+TyCJE z_D;e6XFt5X-WB)g+Hw5gnY2V*PPf)1}{rYpk9NM8)i^<$SW=5%;KvasXyF$(v8BVN@j6rK$4ij6m9xcWOo&KWe3N16|0 z<yd-LQ$wHCecy6)CV^4>I+CFYJpu3-5#Xic2@8LS~N*q?|C2!m6gzGMOgA zs;?sV8eoJw``(BB+8vd;cV^Is-+{2yxI*l@KnKoW%UypS>nKd zUq$l|FFQOHx{MBeucC*Esp9D7RZzG*1jg1_;PgyQe4%N|t?4_3prj4-{iiO!o9}_b zgR$bZ1V#R!r^r@IMxo(j6E^GP%ZnZc^Q@1P@cpT7CP@c3h8P3NcJ#a!QSsv&?dl@A5CZz z>ke+B4Re~|_1hF7*X=nJE}KWM{6@3clksG#@L5_t;{;W7?Sr1fWw`u6BWQ}!#(q^- zDZ{!K2RNLh&dNqA9NibU9D5}A4bVj`S^c8rNvs}ODrmW+3i}Ui6nz(s5?aB>mh(=zB+jlegs2!|7#E;xz>Z-<3z{cxyZ|zLB)9s^L-IC6K`_=x_Oo0?bhA zQ)Vyer8J$2!+nG$_g2vmH+eqTvY$SLl!3#KK|SXQS3Vhi7vF^aZypPojk**S zu0&5(eGuago6>vx>q7hUTVPahL)^P;Jd8Z>Qk>Ue11M3T%dfRCJj@b960O;Ciw3-! zeI44je1>=x1I}JIpGsT1(ImSn=r(mc)jwJV&v)E|`&Ngf(xE`Lr@G7LR=3LiSDj&k z>M|1cKOr~kV}glBgRn`VJKf97p~yo(Y(()!!UOnQ5=Qm~ z=fFFl3y#iKV0!}vymPz*&KD)&Hk{7uy4MO{rS807+87?%whT=z1L=Ue6)&oq!mm#% zqoU1KG2pTp_4{EY({(KuE;ss;pVbuzRPKT%H_Um_mh ztsbV(g9dlj-Fbnwry@RRT|x(TX;;QJ1ktp}>*92cdaCJj1m>?#f|y6kVY$|J+VS!e z#S2QjZGbHvlksKq6+`e?L>T_-lE~;eQc&4%f;f30I@PSFjI1Z(g3IHnm)cX(IyjzI zm%0P1S3*q2WwIE& z-iZx2uYxS8J3T$5%jF6_w0!PB)Utdh9#xT9hhx>> zUYIuWGMqYE2==w5qF2OZ_smIp_$<)_uYGc0wKdh0_<1#L%PtVyT9t*WJvw|xXP6iz zrzr`!W&^=_`{DFx9sKp>5ZGRK;ox3Bgo!DSAXP^LO>He%QGXRxI^>DddzjJo;dcd# zXWJ|7j|_y)l6?}pfHJ90_CZmL}@L5$RDdZfZbx#jdhkK52 zv1Kb1r8{u_vOu!$cOJT5P@?M(C0v>i#?kMRuy>*jP5!ftF8H03+CMg-UJF;jsk_G@ ze^4A9nqbZ@c8^G1EF|ymg5>wGZ{RRg0W6%q1FXLS%eH93)%9~=`e9QJGE$|Qc{Q-U zzcb7~ehR9nQ)rA16OMFi6?7#jxH2vn+d3BTXqry<&syLo``_@hZWjq*AimQshbA2z zP}%JwTpYhejKA}bDmqko?*5l>Ynm=}*>zW}DXgaT!9MtUmokpKVv4!C-7#g58=V=o zhn!-vMTHSFg%=(jf^BhGZkwt*JBL>fzm9%}5oL7Dp}xS;)wTw(9Za;oxF zp~G*FgN2QO;AUb2t@oG1w}0W(tKTvht~Hb%D(xid-A}4fe23mG`%ClDowquLVDb8+ z!qg#0;I6ziL=+^72Or4u2~Ac0v*$UPMP^`vj*|4Cksj>-pp8FfY^3s;N+?_ygO)9E z*sya2U%2THlX9oy_OWAe^3_Dny&A;E3rDd2q^WG|unKSNXb_Lg>lCuA>cJ^CRJ=4{ z9G7h!!7(S(`FUL;epr|)HjI+#P*1;uD94`sar7t@FSv#5B84HxSz{hcHU6Sk}q$3x4bgKuouIZAZdJUnQkpV>~uB9ccX8Gdz574+QX8>Wn)G zpMT$ivAb)*Yy45t8e3Pn#$hN<(2`)-8)aTFG6zn7X`}tC4~XHaouc!3E$X6Z!HfI5 z@w&~^FmH{j;5aIm4yc@?)Eh%_=hh~u^(uq8#i_9R-$qhsjS$>QLZxa4YlSD0PI8&x zEy;;pMwd5+iRYd$5qf~yuI*DZYbQk<|Z`E>ji~bnY3-xe6iiz zonEerb$5>vL>0FOLfgE<-!&y`_~gkOfwQ~6hxAr;YIo9hjVOFAJma) z>)y}qhLywYQF)vt7xYp^A5VRpRW%wH1TN!4!z#reIcoT7+B-0wtHkRogT#ombkV@2 zm=@=k3NJMb$gOY^&)*x!a$jTdU+huXJjo5Rm%fJsvG3{m-!RBGpDUU9t3#|Ty$6X! ze`NU#xl}Rc4n^MG2#a5Ag6DlrVVq1EIQ=^Cc0+5_zM6o8&txI06|leEb#dtVm$Ye@ zJSR-(j*HHnrD+?~WZ!3@fX7W>*lGi6iB-b8Y0CIWHXq-<_7%eJrV3^s8lbZDEhRnf zj}HysK~>Oq@LAs|%0JM9>Z>0_Q-@d7`@<<(P}3-Rx5rDmms;qxb`R>eOG=HqR?$(V z9@OoD5k8gar`s2`)7DNSww@e>UrX%79tR%Kw=AG;Z`E+&NNt>aUJ40G@8G&~7hY@q z7cQKTX|B!-IBK#NZwhLZ^|oF@w`r>-i`%jUc;Azb+=}CqqCH>G8;sgQli*z#N0B84 zV6iX*v_HI{qg|`>O;x& zc$pD~9T$qd3HNJLFES1r^_^I65i;bLz+O`}->dmqQLHcQq2f zd>zRO36^Nq=>|xYcPdr_l%H$;+}E+>zzZ^q!OTvtB-d;s-|C#d?KC3pwh zan{h&)FxLb=%p@_%-_@qOFZkuW|H*{{us{Zg|+w4mVA7) zezt)B?1SROBvme}tA*N|*XYZ82Y$I}5Dy)dj1#?e#Lu3(tPmmLruY%);IJ0nl__A) z*dQD|FoYT=O%&Z*gZQAyT#i%EK%<2wP&+1)N3<^G##>pq!!}JEDtA+CZht@?#lIjS zy)RsPpF=l$OrxXX8)1`7|NpFKUoO$L=EQ|VFtenZX1^??_JdtTC#^ehiVDPJm094d zK0s)C)<$C@=EJJfA8A9GCttr2gPV>TP{`;!+I!!I3pY#~q=8MiEzDF?LsxLc_RHw)_p)wAC3z>e&f&Sqd+*PJSKl}FwJ$U#BI?J3{5Yl+r z(G+e!l8U8W-7zZ1AA5SFV%M5fe)l#U)>VCk$7|j2@RuldiTDctO}RwRyc8*OsyBWy z|3)TryKuVKP_C=;N2psZseJT~N@ZOBaBn9t(fI~RpLRi=?PR>_c7tBDSHRH!96@XL zd6Cx|gUa&+dgbQA=@YH_o}o6@{hI;B54&JyA1%y^jS{;KTm=DtEgFOZA1J-0pm3%~D4DfQ4x=*&6- z4c)u*qoD)EPS-)uSyT>v!|KVs#(<6P13x%Bij%v9;A1!_=(^s5#GA(W=9nM1?#ZL& z#jZRrw2Ttw8X-Y2iAhM zuM!6ryTTRId0@~b2t$>0c=pc~^yIESS{AtBjHx}@t@m(vJvSLmbvL1ojC1&XD-2Es z4Z%IXqw&A~t9Xv)G17iJ93FhxKy49GFs?HJjNg^Ra4%KN{BFh4*W<*1L%XSCRIaJXM%Nk=rf0+a?mTnWnznH-}1#Pt0q>5JI?reC*LgoeZ z#+6rNa7T0^5Bk?6jLp@Bh3@(EpI)aVwIz*O%)Y{x7FT4Y)6}^8AS@Qbap2BH=$$9y zypF6F9vMxhT-pMEwq;V6$ubS@qZhPH#(DbQ7>zDXm&hwymlyjf^7a8kap@W}soKa7 zv_EqiZM1fRIBAQZ>!O6$!<}&bVpD!%RYR6NI;dD-2#*L0#y@MkAvgR29Qvt*CJ9>n zw~sGf)L2cH9(lCO(gG_sOX1(6arjYr9y(Xg;WaIzl=Hz z7Ivj%qB9bF4PC{S{@pA6zDlXu(m^zMUn}(OF%;I^{UNL@YNv%hgXvF{6I(+JPwzP# zdw*zxp0XJw*!n39p0C16@!7Q1{f?0LXe`}L8w`_wWYDU%1st(Coi$rCQDwv%=$Pq3 zmM8y-JDz3IC8aGeu|kBK-XlP+D~nsZ1%UD8Y;c<$Db!1{Xz7XBAXPZv{?VqCl06bA zMXH4ZK7_%o1-X!#C(}Hp6q3@wU{)V6jUQN!LW^4`sM}Fh{yjK}UA|XAYS{oWs47DG zXH5W9M!XTFW&YGWY9)Bg_zw!YtfsF4>ZIp2mfiGY*{)|l946y$lk|_nonFpR5v|PK zEEb5vmVFYB&j}-6J$0^s{7h_^_Z7VL&ccp`w<)DHO|Wq+r!O)ch58mJNY+x|0OOe~ zX8Q7vun@HWFF`cey<3d@vYf2qyNmOM{gAbJDyHrmh4r-yxb>;*p6>BsdB+rfJYB|V z-*%)aW)9dTx<6iM%9L~zew59#v7+ap9WZ6E3mDlhr@Zm%oSN>=_QPGVAR}JrIzL+c zy=DL?xAepwYy2U6%?X+_L?3SN*g!{)Y=p_81Yz3*^r^|^g5j%#q*LaYeef#G{iMdN zQCp!P$&$6byK%U3C#07s09o9Ig_q|;(~e~NUSCbEfg?DqZ8TSG9D_NtE8)6n0l2-q zDa?;QE0#Yf5k@HZL(8BeqVbX5G|ZtpEE#r7_@n<<*zz+RG;dE8M#eHkw|k+zttsz| zOQrcsvgr661Kv5#5T}b7&{*C9mmH-K{BAWZPj#lg=eyz6zHfz}ePrImg!OPh9E=rj zf5U>RWDfbHDh749^LA-(?ksPF#IJMU^{X>9@4iS6@{Z7q)Bzl^{t($MvSUA0Cp@M< zmMrul`OoKQ9`i2=HEv{seV2RS^Ij3RDpk`(s|^tH*%?cf6wz*=9UI*A756=zf0liDpkgysev|DwV+-g?;x~z+kSB^KkBdi+r9*|CBWCZKgkC{14d%!% zV$#j#h34AXd-9^WJIr z`$GyhH%ydaA@(${*-zM_-4p$k&B(8F7@yu(N0*Pb$oL`|&slDdi%0pOxuk_Q=!AoO z;0yS%t3R%nc>voUA17}ccb=WoljD@G!-XgHw6wvCO%{#dC1EqMzpSTBxE&|tA6O%( zO--ZK*RN3dKQmk!tcjQ0rtqCT6M4=s+3tK{9ZU9XK;NF@pmAm~qzw{Kqxmak+CLM+ zgI9E-a_zMwtp>th{tAMufi?1}4fw{Tu7b3Wt^Oux1&> ze5@86zvh$e`gy|T-8MM;shWU}XQ6alI-#KgOXe-+`n@xFadA2(?XVWjpC5!ftBp{3 zf&!b@E8>7h%g}4m63kke&Gv~>!S{YIEYH)$;;z;ly{V0wUuLr3<|RD+NfxqlIxPtD zouR2$vjy#X3m)Pwp1)G9nJe^4B@^qUVqP71)k+I zl4nP|^1tyT(X{EMXf)^_ZT2mvUs|VO?zI!{jp-fY#90NEoi=_n>6iqT?DrLuA5Ru? z>~w@%dA4HR{A_rhWdXDMl#pS;M&X?7{>*sYC|>H)8#k`I2Yc=sqQfXhd?`-kh4xC~ z0r!*eYi@}QFLx(3wGd&@n2|VP_5ch&mC8THb&%$@FdnOz%C0*3I9mP$MExp(-}OrY zuGP@+nJ$py)B!X8Gr&y+_MBJzNOV}>&O6l1`07hP+!%2MwnXiQ-q$xlZKnm#NH*YG z6LW>IkKZL_EtN8&Kv$@ZYlec8pJ1bFt2nlo0-DW~qea$5pl{^N!*V@L%pC>UME8mBVRLljx%3KN+cKJ6r5yp9hT( ze@WgY6%lC+;@930oQeWYll`u4#X)RvdO0sXunB_O5%3RzIIKuIwWXAFLr^X-lfEP<&NNmsk6 zWA4U#{(gkN1HTK-X(xp^JsE#wq08AltwinAk7N^E2^zL8xT)5V_7yIo?)4q?WoRcn zI{XUMce$d~a(&Ep8po|eU(p%w3hM6W#2?-`;G4KAG3j!fcs70vMLf-c?w4E0dA}*| z+2zMiTL)nI(J64mBLmzPJ{P2~`tZQZA7HT91%to$!Xz+|B*e#%cl8svky8u8d~?<* z)uoI7+2BblcZ?hw&h8D5#4A643HR&@gq(s2P-;Jyw!XLH3K`#WYPCH^Tl|H%#v5P| zTmXr;Wisc=3h_*>7tI{&ByPVvoA#g8hQVX@iF(xrV#u-vSe3q7e3Yhul|S4uX88!d zICU_r^>@X*kaXO6JA%8*6_TmnP&(~rD#X3;#pvqWaCl<}6@6S!Nnnm+Ps-uROkX@& zYLCJ0{_KA*SI}IP4dvB+u`PKH{qv84ipOW*)?+PnUv-R(n)it+7o({^Ef!ZRso}`$ zmheh1OnBO4iE+J#FsYkRlh0nbeA)`@*3FWp$M+%ka6eAk?#OEgW#aI6*I-kDIeMDR zhA;Pf@Rmi@?xV&V@~)yFej_&$f2q}wr~fhOv@t$(T(6iiG`7N%-})RoXFL}z>w|XL zD(o8O$a^MPaLTSua86L6dcR+AeA`i|`|QE}N2ucf|2~K>OwqMAmR*y4=v3w)mTZxE zK})mo^v*}(O?z|Rd2bg5hpVIfZSmC4+P@I(=7jD?hj4z~C$j9UC6mu@$vS)#x+FFV z)1~3K>SYGTHm;TV`_y?+TL_OS2;>93G&F-tsIjk}OS0$V#{SaN=)x(0f zseFY!Th!Qd>2@0OKSyUC74!SW@%9a+%~BF2Ewn0XW}Z6<30XsEk)%=)A&F?;le8#G zMHEE|)hy4Q(k4VoXhA9>OG~9lP9^E%RQcB$YVI#^CI#Y%$pju3=?BBuDZMoL+RWBMycL@#vH>Q2Jdw-!&_Lh z&DzwQ&l&Z z(H`-lGI#1P?vkJkZbfYicUxUGmpykL*X81E&bjU>T;}wrmJ0C^oc>2D%x~-16~#Y7 zxCNh@xK55gIrFx(GM1+WGB$Jw_hHBz?u|2t8L?^_%#?$Rn9&C{c+Um~IdfMPb8LiJ z0P3lqE599=V)hm|Gha&XW?I%B;Bi$lt#WU4b7p&naOWEQV@N%|RylmDnIp4$C0GB+ zkkx`2O3cd!3z?nU?3saXp1kL>0><;`n+#nacg8A{a&9`Wj=Oarn==$x$yL*m;^wo( zdBfQ|d3SQudE+S#Or}Ip)#*jaR_7X}d6Ke(lix7RZGPdwTO8!ft4NJxvcwxWXSS*F zeq1o)X_`AQCt_AuHN0D8#s4eDoO5jwZ(V8xx7WOmn%!wyS^tEUXJGE9Rh6||CfwKj_*Luxr+6_Obuwh~+z zG_dS=Fr9a-Yymg@UJxVMYnN5V#$AjnZ#1paHoW8796!T7Yi-9{7!t&jjtgLldWtgk zIlkpKjYV=8UuH2i)6ZIYpR{Eh?&L6h<&1ddYCO&Zs~t>LmMAmIZ!7QQtQl5Ldc%zB z{oRZMo4#_71s&tmUpv8^s+GiyxRuLOvG(S+DXnKbUS4GRCjTk-_17uPNiQw9JD+QD zuYG>RXt+F$TQ4TTTdQot4eV*;984{rTP`=$^B{NSMo(_Nt|RmPXaZxfeIE0R?2y&3y=)#X*~18NR%8ylDl!wD7V%Cj zTE)QJa)wdkbB+k#Yw>4hI%Dy}29C4&DA!vhl3`z_!Wd!uF%>z!%-Ls;@h*zo;;y&# zs2W_Y#z>k`%^g0FqBW(f8aFp~By=WP(y)A4zMjLsiwjJ}{2?u`wrt*RxW8RDNK z7~7+b8MTj6xru#7JTYZ^o(j*0DOR7&Sspf*=X}7Lck#caOtqgIc^ief4m(9(ao0Nt z_dUuc^VS6Ga_8m$khmI;cT75;_cSb;%VO(?rj{3vpl)yEZ1*ZJ>#>e4EJlh z(AVBN%6%Smg=<)0&P?6Wz_>jgYL(P*oT2$ShTB`K!f{7oc1XV*@51OjCQGe^%L-h< z4O~8(XDZ2L-sraCuBuqdz5UI{Dq+x*@%F0~qb8t;p&=>GbWRxMw*Qmlo)Gq#TAy#= zm3GW!4wO_|4bH!BmFSwxO>n=)HCX)B>dUEVOiiXX(>VK{(917mlu7Zpy>eE(FYK92 zm&PZIv9cl4D^E<ev|W4Z|Ef9;moGPSIInOQmZyhPa< z%q0nW%vR<`=3|x}Z__7fPJT@U*E%bb>)En^8EvA&Sn)-j_qnW+!#-Wfd2vmR*&JD8 zwYSZO(HJDjbGEeRJvp_9ck1yzUel-#^ZxC(j8^Y4t2>PgnF(DFIoEkR7)!2CXD0N> zG1se0a-Y(BR-)!=%&W2kjFOKwya|h5?yDoadDlb_@a_n6u(DG1c&h4Cx$A;edEsX@ znO7~gGo+5~wfa)4S>;xCkvq6(FT*5o9y4KIEn{igW?t@#cFP@&Gr5-@eCFy`n=@u4 ze6!5coXmW9WQakHba^;y3*!PwU=#%@GOVLNal9(GG3y5$nNjYcOp(VQxUq^3j27Ni z?&-o1ZdB-R#$f6~-sV?a?y$E!Z>Gk2rY+N$dq1I=lPdLvo3nQ%^9nX|Oa2IFG_?0u zN%bsaM2(zgEGS>iJa^BQx##<8p5#GcK0sL;*P*1Edso;$95$_G;J3e=^y&HBA7|tl zYNw=`)n!8r`HVcq6U(!l(U&uL7p?^I-X8ShMNSH3y0nIKo$4bP{48I_o%cVu%NI`L zYIpkb#&+29W@hhT$}hjq4Rc<>JG}{bTl=h-i`vz>+nRPQS+{>CXKg?yrz^OFp|&@e zxy~+z*_QH!>(>@uwL89pYuG8wnYr+n(eyBh;n2(DvS%IUdVRNGl!rERwFbCf6WSOsQNS=z1g1pyJf6Iv zkOmOtVRdLh9fQ@a&pThfr|Q+L-pa9kPK-l3nH;0^2(DA06E`RGJ@=N_9Iox{8pe^p z&CF?mVNCCOCEm|D@=P-aF{ZtoA@jDIBJY*!ZY%wD_Poi$T4T8N5M}x4R~n<4%JJN!?r~SxmM|JTq?uo8UvZSC zhBCIzt6}(cPGR0OlH$D#jpE8YSUOt!J4 z;<{F^ytgp=gxOId$!*+-Gi)AVhVY`}O_`;y+!&9gR&x*7)N?=KTJHVBe6G?KGlrId zFh4^@o;NnQp-Ot{XU?!mp4EUs3|Cbtl*3piOo(1@$qQe$m1$>wim_-}g_R^Hh|#CM zjO+FD9QOlnI&Wxj5BJma#mvg_O`H|GvN`Ma88Q~1udTc|EX9?yO||-A=W8`8+Rhnw z%12SwA)4566K_whN881dz+}}YxR;%dl_@K**D(Ri$~@uDDkr+SQxz<%20&NF7M&(r z!QqTv*5R+!RKQ;eCWnNq&6{Q9)ASXn;qnn&cD>+be%s3bv1BczHMG&k!)9putRGYL zE#X9TIDgrqI8+)uK(rjBAU?K{=DV>m=RY1?{hI(XcTW@5_!f9$BIL|gm4g4InK<=I zKFkokMbc6O9MwWGU1u( zUaDoL330W`=qKJx9d1rVnRn&zdUq$-C*{DlDN*G3=}D-0)rM?*@eS2=?XdQ>A1FLl zpxei-X^)y4jEaYl6vFF+ht$jJ(-ToXyyjbXk*juDLeQL;Eafn+EC=C@f&!Uh9z z!OQ4yd}b|;n-?XKT&11(JyaKSX*#qDS(-jA*U0CJwsDx-Xd3oT#hATw@Cjj zcOb47@LM4T%A9hT>vvRRv(y7B7~0PEjR?W2_}3(K=Q+Nz&K3HMnFR6oa_G*hRjia7Z$PJnT5_eTj@Ctl z;mh?)Avf{|MqS$r-PiWxx6C34mAC+x^ku+ae?ENjEFsCkWqiR8Oqy?y|)=%0s&(P~NXYfB+6ShaN zCl2o~#d`)E7!_57S~VZKVfS5}Z8QbMW@igCpeNH%buEw`t!7P_CIe-B;p=DCfkcsbm*Ya=OLUKrKV!`fRh zi>PrdGM{|9ZVu5x7Ws(M zSG4zE$8Yn$1zY9Ascp$wpGoOCrR$kJPTJIh-0>59t_M<;ZILar0xCtnD6yY zLyr4etlDu`$idlwon|X=v7IR4AHtBaYPvN-jU36gA(`K+z_F-- zNXm^<9qrd}zVRO1S}{sehL?c0{tc4-$^!0Hxnp3RzF_K>dt}4Z9#;PpO5Ldf_l z%U5l~vVF1edR`Izv}ysq$@)Nz!Y;z316de72cb!SCRG*!ou>>((u$HtO#Xp3>X5UO z{#x2fpC7mc(=@|Cg6zW`{-U^Jy8^bw7NW=2a%yQ^KqXt!SV!`@_^IWM7@KN~n_su` zI!3>O_>v()|CZrtY@y;NVsJ-cI;no)0-r2<$Zx$gcsyDNMN`mFn+Vl@<#4=W8Sx zEF0y=h^Z0BJV2`}c2KPLA8q@ajh8RH#PZ*<#Q)D#km|@F(i_w12VOE*Z{3QP#}7lR zYckq8C&E*g7#h4=2?tEqpvToptj{}(o}D>B>_pM#-6o>tvmH-8e2XywN^GyRFBqE^ zL|3$Hvkqv#fq|^wklFMaq82^H^FJNn^>+^JkP1cJ1tu``sSh-ouV6@HkT4-D0IjNo zXJtqmT^{=n@!C_e_zVFlx`dVeaup0GpN5}vy#6NI5Q3byrexcfo9I$w% zC};`NhM=%sc+XT}n-&1r_|6p+iD#hF;T+IAkJKpB9=t@;nD+PX!c3!lxTl{*HaH@dl_Ws3~a3-c7+e@Yyp8zFAV|?G#0h%_~V1GmewSI7oo@3!8bd_V8ixA-L-Jh^(@8!tblxAud#wm_PbW z0$wY!=Un+h_n&EFwtEE#vl4SyW?zm&;ZQa3r&PLU@0Os8{0&$n+rij+^&xqdc?)f`HSkIM zeb|{&h!0;evFx}FJ-q%rJ#A4*eGew#m5qw*+{@vxU|h&mx-*JB9B=qKTNEyprBZ{D zAYqtGFm7F`im@*nD{OP1sJS3+x5!w6#1*ldf3T{sq0 z059@?u=W?nV0^?mVxRDk#^uRF3UvcrA0>8-Hy_-#KgV~2#qjo+I!=HGD>>pROFcY- z92pd{ahqc49mgVi~cvn#h==y0P3RPwkj32 zNnV7)x6|>Wb|oI#^&f<#Yhc2A9f3%0bN z^Eq9pu?9-t3HhZmpGmfmbKElJ7PUPr13iPLc>K=@Ht+Lho$Qwste;c{4oZeZq@|tS zd~S~pCwGtw`dN6*D+2xvsR*8rtKqK<4p}vI5ApW=h2zhiK#YHghTYdizp@b2fA~zu zD*23MfB!(p*cmv!!;&g)GKFXTFKFG*bMUSGA;@?s5=~2Ec%(6ezLq2S&-xJ^y%|lO z$(dk-e=<}KEkgFF7VaLrL#mf;qwQr1e5*g_VPf(MGHYWdi0MVrdY>cwLya$KnU5Qk z?x~_qQTed8Y6j}9=imYrCcC|$3b$?Vz^>d=Ai-aTW3$;{6ypxPAH(Pq?Ob>zrGV4i z%JFHFGVr>kFjXxcI;IVh4P9?pOSj%9VuqWE{!w!<-|9uf>ps$7jxr?hS`PKu?1Jk3 zn{ds7KyZ+*K(B8#RA&&di?3E6h8-r+x>N_lroyvajAq&pl)qr>GhZx=G#=n_r0M(4QBy+$N znyZ7T{bB~QuQ=J|pvvI5)^AB9i$2}sP+B&bmTAMP#~Yv&Dgf5-#yc(94=I{t(H4L?raR7F5j=RvqS zM+N5{$H-eE&?gf^lM+YC?}&|PG4`Er zQ|ttf^`r$G|D8kD+Y}%HfiRYmgoWIHaAu`BQEz_+>oPK_*24<0(R_;A(mH5>X)fx_ zHx&3CiGx$+BjlO#bDWiy#V`A}h^9AYqg>2c(pPhv@UN)A;YnH;GhYskdCS;Qy^mN* zce0?#NdX-0$%C(slwkAEPAb*MrgsuA@lTyU1U_+vSO{kDc-lF*tJcc5)LjLJTPf>J zv?Ayqi9zel>afz&7_R)-|73PU|~A=UOWVsr7NNBaywc^4v||c z52K=&0Q>a|>2z-inwIT`D<8*V_2+ibHq>I@wbsUoeo^*{l4*3pw}TWX+u=kfmtH;) zj+(0^AYdinJMA`%dsqxj7kkL|&?NHoRV3Zprz8-2RES%(7+|M%oJtBw8saBx!EUu9 z*uCIU(T7SzbWk4$#Ga87N`xD};@l|1W{QVKm^E?+hqa&xK_LVuJoPBk;}tHFo_80J)V~5S5e;KYuI)=gN0<_c~#% zIQ)@xomU5w8H>p9w|k^ou9eJ8ctPJ~-iC$V8o0OiE4l2P!D^CUNyfYb*)OkLVh#N_ zLepjJ$(Jw9RM3(HI?ATl(Ii5T+*l~sd$|#}PC5zOe;J^4(npfJZYpY)I$)o6G0d6A zrM!PTAvY=y=3n0chl)hO@QoUNE=t6|`Io`5Bo7tb^;p~2?t;A1Ewtr#6ggj9gIV`w zkrzI}N|i0ftgcqv8LT?Yt9)W5^1SXq ztj;7r#x40zT6k zmons0eM%JMW`p++8(f8#arJv2JZ@tItzHuBmup1WpXy~%PT3W%XKbZh%M6gqGsF{vhfpu6 z44<62U0H0EKpS0J_}!JZ5HnGTj{9m@_D=Wd*SuTIBWLC!{!oOWJ07&9polCW%7R|z z6|&JQ8YT(@VCw7t_^p;D)Zak(Jkn?ID=b=Z`tU;%6}Ak0?#01ijsi;e3(#5r7F}l8 zLJpsrhcRIRthMtdSa))tz^dLWG-7)-9Eg|#%sdOM3kjj?rhSAjwHZWc_MmVBi#50? z6e2dggpZz2(b!QR!^G{OWPv6w^L-CYmmN6y`wVsk#~9L^|D&SA^6=GV8?*$M645ky zn6IJ@?%^kC;30WX;V1I{+m(o$8_j{UGz!lS?gDd_M&jrBkdKB*{1^Hxut>iRTz(Gm zbUshhY}fHikGJ50k_PChZNmGO#SP#dpVpm-$8jMU3qm?JYaluT1 z@jD)Hine2X@)eB#n1JSX7BuNuC~>~^ide6Sp$Uz8q~K31)GS?s_bp@aPIDEm(Ob^G zOitt4<$s~=aT&&Eo}s72q@h*FJx>~vfiGp+WK8BV+}s(;T5EQc=iJM~kV&x+Z@(Ef zt1LhROoSf>h47}R7MWWg(;um$th{P>u)CTLNz71GQ_NvmzNy6i1t+QVfvfOV!x(m| z#1OkRg)m5!1+Ga2%&o(zM7{VLC6dX&6!}R~7iyrUg@E5WPXf9_^H`=a6=)FA1cROG zcr#U4&#`8J_Xc_9=KFpiGIE4I@1GCl-Ld2>$CQ3>N`&KZ6V9+r&|~9o`oW-tjLq8& z+sxu1y;`WFoyUnJW>MMb3*^pCTUcD;f=lJElj*L_LN4+z;&i;5r4qf79Fz7yr_X2T z7Q2HOH4kvnjaBIIu@UOJl-cFtg)sYH6o$|}I7%Pjp2I@^s(cv#jdUF`Ta z4DZx4@%g6TXgV+(hu`1E==!y|@#8bnlW-C5InN|l9|(2NK3_1c+D+b<41vgc&r1S(V`5edE-H-&GKh3C9q9k9Zc`6vnoTR^Ry#}h3 z4z8EiVQv2pVg0k8wQO1<+$ev_%GZe}n>D>~_S{&QJAvr#p$BEN#>kUWFA_>jK|bIN z(pABDI%g(IsYl~YPAXhkCM~ermy8pNBdoJ0-C?neC!PG<3JEibuU=t`IYUCew((b< z)uv=>t6NOZS!RLUt+zyH;vh&m$D`{KW7rt<1(&UpgbR805O!lTd@y|j8Z&Q`>tf$% zv8d1w-JXm{62X(lM!#Scs+WC?T1@zuz;e+r+$l$k9k>{Fxk-&J=s!ODPPm`f; z<~68mjwjarcX9I{Z(K1i2VCn!*&P`-(Khk{$u=p0&hTGU)o~h~{ilE!Q$@j>OFcwz zM-!F(LYUWvW~1@ILP6z~cU0H>I^m{AKxU;K|Dad~GJfmA-ha8Io1;NHw4M{ojH?)z zR6uN8yx`XGMVQsG1FqZUvW!C?lYd6PNY1JRklMBv?)?Y2SoArox9Jmqm#;Icztjnq zvh%@W+=%EYE0E~7BYfRAQZRM?FS1dTN0anN$$S+B>cRWXSGw;7Me9=GV&GDI)j7eb ze>n-H&7x_#x-{;XK0zBd2sKLKMS5b*SCpQiOzwqzAWduK&>rG|E%S@tqA?E*R|)T` z1v%*I*NT01-$~1=RP1={3Q2w`u#sDas)G6K**l)VzxF9uDlr!q`_1PE7^PFepFOxH zdLD!tEn)M0e`3_l6nt@~7`29ff_dCbf#R7=*qA2dMU9M+iiuaC5H*zqwmreVW>Y9V z>yOnhX9;$Q$>9tCb)?WTigpLp(aWdwKskg7^ZRxJ=R_;{d$gIl3+o5FXRf&Xf*l%3 z+@>k2O|V;U6;8803(0@C!2M1RE`H{~N_m$qVby zNo;Q+ukwz#6K0$36P_VwsGGq|ENESW`}HD8!^&o|P%?|+)=^@q(uT?fGuW5=IcV1- z3vX_{Crj*eSYEzAsob1&*!|-pXk7k6n?%%UQtnwik(mL(`w3pl+l?P?#_^X|>w>Ry z1SEN>fX%#%8Mj6zhsCNzB?1G{6yfqSv))})e#uFS!300D_p;0EBw*6prYbOiNux;I^(W1WK9?d zeV{2s>+%ZBc|!O{|6b)E^K?ZyvvxAMxu5(iegbdKhoY5^7CXvV4fe#*(@G_nMS<3Dq83Z zOR>XL50PlgLaJVC0M}y5VQ1I|&>wPQz1^{eN^ClTt?RSlL{S8>sO^N<^*Vy38rtme zZ-%^`D_+sA@X5l@%){fRUx;<-Gh}dXg4d1n_%Hk$#<<9`<0bvzSJh0ku|AG2aT53f z74YO+XYgRM`T57*f$j$_XmiLX#&@G>(LYb}_fQqY^Uh-a;Y-+aa0Ywq-#XlBV~LRp zSFt}(1o-zA!DL4T{@7^Dp5(k9-hX&XuPu(m6$>xGvqNlDi%~$uvWs+=zYQ{6pqU);uM7xJlNa-%f6I@ z=uV(D8$E>Vv=n~LrJJPw&`xq}Q_RVd_v2=-B`}bwAW)S~;0Nht;v9QRZmZlQ zI$`*MIF}xT%WroGpV1La7IngNI+ZYNnFDHHSwgNY52lD|Lb8DtPTp||{5c!QoJaBk zDIE)}SJOZlBXOejClW#rcEUiT76i!EpvRK=Xd3d7p(ou;#8PgLQZ=$Y0D{^Z$JuwLU-%}hwxFA{g`8r|y^teh z4gTPOO+#J^Ii-svbN@!B66EeSfl74GZ`sAK8j_2RNWgQ7Z zZM7Qw?O)?a$1QksC7r%^(1U#|{oz<@EcV4af=lof+$HS{)6Vtarr1h)-lYIWW#;2X zUOkyLSrvAAUxbzw7VtW3Hi{{q!P9@$Q8IWCE3_kE@@Oa?ao>tFHRi$Frt_?6_mwck zRuR_x0kl+Z1kW4~T)X`iwK+c<2e-9=2>EILE0GU!#NmCsIqtJ}#vwjkjy+Utvfe)??6V^QXxp2S9nD#ua$5yLftT^36 zc%U{0zpSocB^{^*ec74pqlz9hw&)Zr9FK&G@IXAbdO4n&IYU5?DGDsK!$D5e0^=eW zur}-hIZ>+(PSYd>f4;mYx;it+>+wc%VM!kN>R-YQBP=X(UIcoHQ*gtd5coSOljgL) zr-@q>YXDwq7|r>#pg-thT}pdaqL>1556q}#NAdO{67}r zx7ST*5g<-HePuyTPz=%buB>U7uA|7;Ox}{V3DoIU1Vht{z(3nhH!oU*A>Ze~t{aU| zQO3m^ItRgje;e58kI|OEqh$1AD%~dMjdP+5p~kotN5uW%#Cm|@8T;{3^m(vSnn9!n zR3X6fCUz}#!9JZ#XnbLUNn65Mzkej)-c%2g@?D#Kp(qDzzCNV?$z3J~dz3-H?=*F- zcL6zDN+?4WPCem+-hM^4WWEG@+tO>~SG@rY-si&mnrkRIV2tG_0;pGfE*abrMExGe zfVcHYOpq`F9lvKJeD7x>zONSSIB%eD`B6;DeGXw4tFcwc;w=@KgR8|ov6y`UkNZpk zZIeS(a72o|ZIT9lAZgfnHXI$VsiB{?B}`vG9aqlnB^ke#k#|CGMuvHZuunSTPcub$ zJ2;M4wuGUxqBMP>L2#|~C;otu2KYsW!|Ct$sPdFddQC1LM*=H}CPNAz4)~L2dnbW? zMjg&Q+Cc-WvOrBf92$39VPDx!OfgR(@_S2Yyr&+9h}@y;>KBvZ;UYM@;2}z%%*C&d zWYDpZ13RavP~Q{JX>{uYNX_ElUhQ)J*wk&PFINH1%fom$e;-O)e!{8?39L(1^;Eg} z4n0_xN?=kr^4W`^T=X09e#F8=B^8wAO9+0Ps6_VGHB9cDN%+Mhf{g4m0nK$C5GS>m zPOg*(?UiN%lMP?t(~ek1YxX!b4i-Va)pit0AacJwgKRjYjLoach~Etv!PX7_(B)l5 z(o>kcz?Vt`kXfQcvNFXAICduY@d)Hg3D5O4V zJ)}Rziz*y9gKf5R(9NZl4i(hmGGT3B6b!83xL_O)RzS_)cGPVCc{*S)2P9r5!||sa zdLYmii@q#If1_q>bG`y)`R~b!(H+pW_ZF+ke3+&ReTuenM|3TZgu@GXSUxZdgF^zy z-yb@#o5@B8p`TIo#{%zOn<{W!|A@XRnS}c!*2B)ZqC$^vKLpJd(8V9>sB?xS)@R;? z>lR+HFtml%bam6TGfv1`IuJ66#GDMHK96QVRp5EFi(_Df zWI8cdE5xC`AaD#`h}XoF;gzr+cN=_7Qe|Ywr?hKy)2nDGyVp*(@-6UXR23Y&d5<>S z-A@+QNs&;82wZY0m`7&U!nVkFbnwO-8a7*5;7H`5zrBF`nD&alr~DsPpLG+)8?0FW z38u4dhu(u2;cOWD$Po6$K0u|F+B~!Vo$MT~Z@50W2`n-`@^`q1;MFZpsmY1?WM7{$ zmFc)aoi{dtnnw)eS$?B|r^MlCL=DKyO~J0`=HRLsh`9|PS;}YI(I!s|4bR^tTwf({ z``1Jitm5EqMg}?L`vK1^y+|CsoyET$L1fwU2y*b>0^}zjXYDOYq#E~qNFehlS^w%Z z)DnOG!#Pu7!LbUmpx+F9gO|g~2y;6B!Cz?oEDMU$TQT2pCmjE6gyEn^E2bWXiPTT5 zlc}{-U=_%Ua3~}Udnv)+ExK5ca*7_0)uYk&3vs{7Bd7^KMVGD(Bd%%Iv_xK1aCcWY zt95i1@w(%R0edoNbHZNIGp`oek&|$+uLtins|yr8D&hXREV3!dl;5Mb4Sofd5sea6 ze9Vo4nZD|npz)09rBDQM^X8!QrcJ0El8tJ~CGaz~4=uu~V062TpaL@K=CPl|bgvF> z9xI|3MMI%&hc$k@HcXXT-jO|JH;MjsCN|aFh0f%Is1rDi{xLd4$K$$apT2+^_66W_ z_(#pZE`@~TceJ}-0X~LhlVv`7bpN6cG-caKI{AG(zJ^c~FZoD24W;<@!JYION}zSz zV{mgZ0qfjRrukAsP{?fobeCXnP7+|&|ujSTn@(Xp{lIWE)ed{wLj%WW-OF78A^@bj8Gd*RWQnoo3xR0>hqNRP*EWB;}*ApiM4fPFpH^yJ~~ zSn}=^)P9Xd-S<)gyBBRBV*HC9_KtyoW2xj=n>jdm7QmK0GjYEAAP&E~jXggU1?#_r zk(@)<&};4xa5nj)Bdx_vH7z8^*%{8BzC>y*0;yzNAK9Yb%N);W#Es7=XyigNEey+Geu$?{+Zm{6W;tm=bHzO8i>Irmkx_xJAMWtBVR* zMc!AarC|;+SziksD-MuTrMFRPRSdYNC-c|NIgK;J-Qo7M6i`Y^B_1I{57yrqw7&mG z$3{P*sZfLcxu=P15_Aa9G!HMfrO|C^66}-nyNPUf7O{LU0X9Wl^m$1-G+zvbkV0ue z+I>s>Z1bI8s%iwHyN*yTkr?nQ{Ydgn-Z8xDx?p*nAzl_4CXuIKp>O~>inEv9tSG_Is7b|3HUdZzZDe^a>OwB0RrGm!?yW#36p#r|mdJ%bYao6YSAD09$|i)8O@@ z?C)VOV8cc!_T_7C)Y_>Wex17nWMw07zHT#Kyl)P9F-GXWAcXbLWCreV*+)*5l`uE{ ztU%we>ttWy7kVh)iW=R0!}|HPoTrnmgHAj0(QkAU>}&5R|zJYHS*&ObOlWVYoO=TJN~V|RGoTmI3Ct~8XZmK6+Sy(9rc<@^RUHGGo|6J6S27AgEJGWXh8l>40?GQScxx)lUNz_ zDlUf8cn##!ig5adFqjn9M_;_##Si)PiMU?MAl{pQ!pH1$#JuJtBy77+S6|wXu}6^< z3j0yFPDEkMp~-*}mtfSP1h3@1fiPcZ44IOQ`SR<8I;n>`)-IzCFDzNEUnO8TaThMT zoQi#MsgU+<9(%v_48e0%C-}U~l1*aG@WP^D5U`zS;hAREz_blCS6Tr-7uUg!4QGi$ zV=fLjY-G)&W8|~BEPF|?J$}Bv34Z&FfkwqszML2n#EyosAmk3bnOII*7o37RG6vgh z>M`k#ECeMKf$WtSGA;8KocP&Ib;?y)3BlSVGG#LR)|;E;(7Ye~Sy`9C?RNvpe4hu! zzjl#jyz7K>>p8#N?gp%KPk=wyZWD_|lNezxENUxKz(3}H4z6Gz)VDa(&8m;Fr-DaV z+y^My9!0OaQC6CB3N2cYMm`l>VwGzw#>RGnFGPHW)&l2YvR>%)`ra^!5L zC$COrgxK0E!;~$1Nxwlm>%ajQ_}ga;b0kFs6HC66Z-*Um<63tZdN)Fj@A%6ynB@T~ ztA)MMd6!5e;T09_NR#y3h&r zouW|wvNGJf9)_c-cWJg53wyqpfy4c37(AN>g=wc@x%6D<-ef>td)()5^7Dn1;sUbL zSd4zyq$s%SAnfOf$YZO~KKw1w#gA~@LHBZwkS7A6*J3Z!Pp!?ExP2R$YCRQvqcWi& zU>fePtj6yjl41R6!cROhlmDw*lWZkbn58V_bbj1TM4u4cs>=Wq>2iqlxW+otA3|yT z8@wk~h$8hy7?t>n?7OoYeP?aQGmTROHg7iZGxjZmz70~~Ceewn$AvfWNCCJV5PHag zHN;c*CGTIdKa9wKVZ~*f#J9{el#)w;@28sZ7CQ}UCY^=v^_nc5!3)^e6pQhTeBqDM z5Z_(26qhAGAa!*r0^K*r4@3r64kdjg|lBtLv zE!sfxVLv_ppb#on>_*E6!a2i|7T}+F0i8Fy$P@oX>|(d`IHLLsH@8j`%v}APP!T>c z8TVHme_Oq zAspN{6HE_Xp#PG`aq|T;IP;{Or~fGrEL!iPmEK`ev$Yg-I`_eDIUeNdza}`-5O+$b zz?{?f@#3X9@FgIEIMjy0OrAI2foTJG55=+^Zx}#SwhHk-mOu^Io}`qj;#We_7UEzLFjsiz|PCZFk5>W8EP%U+13wHN&XO4rg71-pG9gO zJpsp0&%w{W3O;TdBe%{-2p(_hXI_gR;~P(!%(}Pe3jDhuiSNJHk+w}0WRL9uNR-#Z zA*H*}UStH~Yl4LlGt$&AE>gIEavYTA97RRWQH)MXM2S*W5L;D^R*tHx*4M{lM^ubiD1X z0fTx=@J#j~+}0a_)lQq>oX#$?D^Z;2s+U1}uoWhh&?m*+ZrBv}rDV+b~inR)I>>9%Ya4%6IagU#3 z(jPAU?&!upDLMzT6^C$ZJLI?kP>QUsUE~cNwy6E?Mae?=14RU>Z5CnJU z(N81Ubmf{dIyUVBNZHw<&FS@Y;8!I1%~1wz-LF(lTLk5dwEy2@^Zhi3_8o5~n};ii z)wg(fzN8&ZmkYJqylI&Gtru2!-QhCg_v7>{&zK8td?HJ-Zej3_7XCjXLCW79#AhR! zv?ur;bS%HbIuM!!k+b7)>*V`5z48zgm<^N1ok+BVq1m?y1bsC|6}OP z18V5PFy6keA{A|vN>P^T&O3Kb*&}3&kVupzBE%<>RN7D?l@_HPskF_RJ13<@vXmv! zh7x5dTV(z2|M#yu@0@wh^FGh}JC$S4=ej&&*6vbg=BOYk`<{lu& z_g4!ICqFPgZEx<5!HFNhQ*LH=Yr?6ceQvLdN3pZ$7r_Ix=IQ6ZS$Xx`bT# zqL-JEa*GQ3KH;?Rn)z|z#ap*f;-xb)e=CZxql@I=#fT{xXE0rRm7e z4{;W~-*}arUYW(qm|CElfnS)bgW3p6#yG^4S%^+Pqs8})J)C2#KI)qFoTQiPl4Mmk z5;^89C;IOz^V|45l1aWRuD98!XYWq3aKag0(4ddb9=SnAoXe3{&`##mn!5p zvx~U6j4;imsYGe4rpUri)qy0Hl3)MT@!J(I=AY{@kFI+Y6G!Y2;MOXX`QQ@jX zyoZzvIZ)|Ll>fUUT*nQRA_5KTH_X@vC*$k+UYB#E9QbIB3_Fef0)b&-V?^jQq$q`63icn znN9{L2Qx?XFEP)|jw2`KIebg`A`(fLlA-;Be9es(K36vh?KhJqer_YY{)cdKZ^sX2 z=xhvLM|dEZpf&haz(H?e(0xp4+*49h_QonLr|l4f#f5R)3+SR@;xD%^With=mPhyJ)< zBVYfmK#ymS7v~Aql6v(mq*{d4wY2?xkexxLRWN^e4S1 z1u+$uhRMi*P9mA2$IEGYqG|K5leed(NVlsJQ7j8XrYKlAEGL5+e?;*C#nHseK9eNx zvnJc#bqZ(4zbCb3{ml7`C0v-}0cP*ZsZ3l=I0{=D!h{tKAph{m$kb8Vp=LOR7gK+T zgncZ3C{;i#SIp*D{TU$l>i3ZqTV$%*z>(P7HIW+AkGxsa9VFN2O?r=~lM^X+$jwol zr&BO-;HOR^cT8^5E9pv+S-Ki2KJ}eMDKKPF`3wHiu?Kv@M|X1az&E5rjM4VyeEuIL z>k!aaEz)-RKyE9EnC%Djc>h#MlKCtLjapfU!WC9JtZ5$R&pdvI4w7QVpy?v_X;~;S z*p@>cCl~U{uUh!f)Hsp6&=hUao<(wVTpX0uvZQU}GQRmAjW(ITCX1c6 zk?88hL~6v5$hyrX`_HM0=4)$+-`nY=|9%JGv*0x0RJ_qmYe{mpK+8cbv7VV&=E|4; zS^-TWa^e%o*Ocr@le1?B!`b(6h5yp3S$fLvwZH(iQTA{;+z36e{O;G@J zk`>PNyv4%f%z1A)^k+^cU!fS!Y-_Yere-hE-?Guno>VR3IJA$1tUOE{FKhAjrb>)! z!wvL%Ninld-2pi_e-XJKIZGm6C-C{T8dU+B#b{Pz4}a*@UH-^=Nup?KSe0{8mmrUH zl$NxLybrmH=7(F7{wNC~zbywDzD-6K9%b=2By&;DqX|qGxj;O|ycS+e*CIh)t-{UY zY|!D@02?=kc1Pm+Z2J)&odg-oo; zU-CK62CbXuOWaf6FcQox(v*J%Nmbf0H_wjan-!$_)%mkXo1QFVXET@ocfOe8(6f&PZ%W|D}{7n!0~J4^I&+Cg%AdlpH`--%A~heR(et$D3HSycO3z3N1V zDXA%mV=S*=es<^pYPEdKubsD)$h|j11v6Th|MnjyCaRa1rw3Gs)6=oyGfp3!`({t( zBa{yau^5wHf66rUY(GQyLTJsY~n^SrboZ33T3PEt%o(O4_&t zRD7|>9+6;f;{i35hvMd(o+{{Eb?Cv68JvoOU#+GQEH)Gy=zd~aUXP`sJI?+o(JsD$a zg6wBasVcd03#m6)BmTzds`Q-`7?Z30Xx>02udO^o>`S>wqUz3w`uk1MV&*LW%5Q)< z=68t%TW>;Z>rRL>{8FOqd4Lo>B&tp}De#Iva|s>O#>Cu7Mf){s&?37t1dY`sJ6A-a zul5C~Gu4*dm91yC_R9&!JJykDx-vqGJj~3Kup)QUUZCi)xhQo_BdS#h zTxqmm+)}Bjdu4yyuMXyl2l`VkG;U|L{DM z-=flh9(YFaTiv=zx~dYd;+%}~9u6|eE?C~j#Nf8})|Khs30;oo)KG@3S?KG!S~H&$%~EBWU@_+NcvQq$l6*^1xMfVNm zSLUeP1VjrO8Ly==M73r-*_RMX{1#y5%%Ky!?!-3cbyPD$`Fk;$)S$2?Tn(+(Kg>u> zJx+EVZzgZWJ~bWxN2tJT8RBm$GCp!S$l=fnGBqt$m|I(l%6}x1*`;Si^FIwj2@GPkGC?$E-ki z-JbKm`d{((Pp>c&$BX^hQqCyMJA;w_whIZKA4E58e942UVaWc72fAh8Mt0d_3rmn+z%be-+s#}XS&ueNYv+}klTeLKG4GMHgtYy*O&qRGA~rQo(3reTVrG4e zcm~dLI8c3#c$kGC*YiBH_SSiR>)p@fO5}Nd??61$?|B@tzjpK9cPA2?4U$aF%jrUy z6OYKqktF0kxSag)c0#{TS_rk|a{2AnI~dKe_RN79sbcM85wE=V1Zt_VsoHz|w9qhX z9Qh|N!m|EwPq?m{tdxbhJ?-`hftkK_3epH1!{3~}hc9$;=i81ql6k8Z)Aj9!yg)b?eQA;+W42V29S1)O z53LmE!*)nG1TqUzpU+z1yG|M6iT0719hJ-*wguF{clicJTKCC^$%bcP3RR?XwxxF>$)5H-zym4GF}(fWyUel^XGL4? z93$(K`Wc1n>qOumkJ>cikxrQ$iH_UF%kDAZ^M8u>IM4*KW30&;M}5(_#kOcxSvLP` z!BxKJ+!Ow(%Qy#5+4E#WcE4!j7FFUU_QBovX8FKd0w!(MFaBQV8k9S85_#91PVBZh zA;q1sjK7J9wsGE!zHd6iJX+bxMAufLafT^qEjW>9Pv?tH%qru%BL{i!&xK55p$b3a ziyUKCH442fNkA?(=b63R;*h)9DO6*%nVELAhq1_+%RlOAp^tqPl6g-@GxPR%kqI;M zNZr@dWSH4X&U!nMTIcs@m%}0yy0(kSpDBgXH>RMx+G6BY+Q)Akujk;Wb(RnAlqN0{ zEBNzAAM)3Y$2g2GEGEi=dK4@6qm@_xVthM)Axo`#-i?U!PoulYGpFxllUQq7d)pA1 zbmSwS+wHukW*WL){gjNmdtOAT&2=dJDofO^=ON=A%b4V+HON+`fjE|i3bogK<()hW zQ25n;VYA~=v?S*s8sYWO>UXCYt)%H_N1b?X9hoKeaH){4R4o)O?LZ3NZQ<|!X=k!B zWJw}ViR8N1p>dDjaI14ppv-Ai!t24~s`e*Yp_(^uiQf!{Xx}_TWOrrr>;AFiKlLHL zYNU%#I5o%z-CBr_+Uk)tw-g+v=&u#_ft2Uz|B0jQqBn9FEzF z!awXnnSD1z_tg8D`!Cyt7B>@1c=kfZu{n=)FU~=BSQ*~ zJ3!peM52KFNo0=NV`iss4w{~|T(tO2Gg8fuLcXI;lLAfg*y@NelUy>Bu4@b?8O^te zjn+nzpu3sukvNC^JQ7j1Y#*tRsU|*gQpDHsD@vYe>Ckk4A_~^lu3A&wgmj)xA+FD3 zi0ryLUgmNrKk?xtq1C$iDPXYzs(B=20*dn^so80JZAcu;wL`ve>e}2ehvSn*4e{(y7v?uLF<}E40 zBE#LJ@YXWan;1hHeyTgf)~+E7Rkt(AjpFZwXPxlgB*m&z?QZ0XhPeMe$ML#1#p`$j z_K?p*=^{(9CZ^`Sm}J!KBh?@`ytZz8fPWk5eHYvyBn{?bFU|a)hQzrZW}lplo0#SopzEtK|4^ysw7nY?hBth zM-lm`RHAoRf$3215USC}Xg#x;Y|1x6l9MF)NgpSX#@g+?+)))&Sv{UiKNiQ}?Q_w= zu509{>}lfD{zoWo8Sv9GUNFzb8ae!#+Jc@+8&_=}Rz+RS z;m9a2jO+~8U=&{IB1iG8ViW4&2QH=Y_>feUWvwj~hWFV?+h$O!@-y&HFlO{ZcBQ0hdYQk}hF_>MmYFoQE2)E+?ihc98S>hJ@QZ5AAsP5M?Vzkc^OS zAvf_1xp|mHN0KgyUL}o1uMY>I1rI}r{m5&w*QHFHO-LfgW#W+0gr&qM=Ob!s-$j3JY(Uyp&zYH{?(q|SMx)O=o5?=2a&)}Dh>q3QCQW|>gw^~UMqTJa{7uG^ z3x_YDti|J*_#F+*E*}=@T3uxxhCdS)Ei)0%aZLI2Erjpg)xa1YmnA3rFQKe(xvFoo z#M#=@XZb&hQ>tQV$*SuGwtSNIcP4piDiZDuA!ThQ{IGclTGkqlSoRdkM{-On}sy7Ss(dx=cLXW`FXu6}BgEe!Rv7Kd%c4^3>bThGjl43&6 zxlTjl6ynI4ns(tAwE$$XTG~Ob^&E5V^HCyGeHLv#D2*OCW}$4lko2e>W1cOQ;p?Rz z6T|blq;=SndG|`dtA9)u=f)F}6IqJZSHBSYUTxunqTcW=ig$?N=U?cDTQV8&wIWnr zGk;@+G+MNMEU6WHMmFjQi2-NA#N7w-{>E|gaQRBHpIDhs**>2HYwDx?5{0Uvy+w=~ z+ruyXEFyJAe^HKf2CrV-#_Zkkj=m9DjNTnoMe0??4l4J1nYZ>QP?aR40@XUWVd7E+gLJykev8DfC85#1|VH^Xp4Pi2JSe^= zr_4Les~)Z6xvCM$z1J6>`Dx&vBVXaWego@hHp|}E$(Id(Sp+j03-EW_3Y>qf2nQ&w z<+2@BalH04c$8Mlx+s)j-KHb>!G#jsAlYqaJnbDmqw%5Au-FIBoqMLju;wB~Q8KiL zNjh|{y9Cy!UFf-DuJ$C|;{x0Jx|Cvy1^0b(E<2>8j2{Kh##bUQ;N7jquy6W4yt;KO zTkuwci=MR*SaSxH zL&n17yJ^s9X$DG1sAHZ2)|!qBC#>ZuRiwZVto8pdM(9Hl(lGw0`$+&W>72ENJgTmlKu(-7jQd=zP%eG$7 zVxtCiibE7-U`)qXr%{UE^07vg8jf>Lh7!*Re7_B_F!?##{J0fYCaSZNr&+LoQS2f= z9lOrYv!RSKgpCcCsU^uH;BOrcw=SK+%Rbn%8$xN=IL!;nb+&94PGpjpqJXWu^yf^Zhz>h<7Nf#)q&uN~(C=sq@qvsp+_X z^Cwp3M+&>qupi&V7g%5Ga4P408g=xbB0iff3vRnrLA_l8{0r@9iGe2|AGnBp^E@7! zTrFt>)Kh`Cml>WGXiUE}IEFW=Z^bV?o>Z1RGUnb(d=waelcJx^a;?nrc>*z> zE_f*90^TOI7*`1zaG@cG8_{MeL-h=6kXD2XyFKyp2Wes^+!Q<*H-+;E-$5;w%)?W* zcCq=P+SoZp{OqsQS4>;+fwH=)Mo&~!#u0rb<+p!GfogmO6c2B~cdT3N*GMJd=pSeC z$&@It9{m=g&R3bd-X#+ zwHj*KU)exQUO9x{S;|wIEr%;cdX%}i4aQh`;cK?uHnKvsXgwYkeI5JHy@;QW|Akkp zCgXxW1+H^OCv`%5KBpr!ip#Ox4Sxh%*|6gZ^f&P?TaQaK;a=1rCE1`xPk4VFs@u}o zS9#%rdeckTb^1k|shf)9yX3I->Sg%s-}QLg-3+YJG#XEz@&z{(q~Rm)FN4W!Jvh4P zD2z{wtt8JTa>|u;)Qq2coY8eLhic3jI9J|EB~4SL7g)Ul)dJUw#kY@PCiw(DH8T}I zJo%4;)%jrmpFRHcu@+uEI7V%IbrKI>|H6j2yudHLuVU8a6|A&Z=6=)xR*2QWZNrzL z;l?}c)tZAx=Xqf12T$2!J}OxH`YHVFU;+rYHv?m4PRGys0rP^ou=K-NtQJXg+FQgP z@hT;Hn)+wxc2%HN|0vPJ!V$2UV~X|v=+di9c^It_%07Ow3D>-TfmeH_;=5_N_}G{g zurK}%4zs+4yCt{bo!O5B3u^&u+-|@|Y!M{vQedyT$79o84IewGe6caag^bKIX@JHNi-OkfyJpAyPq_iOO7pc=vtzQl`; zr-EAJb*yvxrXcXp0{FC4mg;lgNR8BL!EJXF+Vesd4!bF&x2zfhdGC|>g*_7 zzO1*hVC7xt`J=^p*QwDqH+kq04a3{!eAqZ5OK&{;3GC!vf@Sq3xc>7c)Hyx}uNPth zoJ%SjlPje4>!j#VtKazHVS&9|oRF(dQsv(Ko{B%~Na3HeKHwMHC0Oaw2wu)N*{M8M z=W1{Lz!u4WFi14BYgQU?j&XJP$Y=o=&31yL)qYT?bc@Z{piX;VEr(Z|r_ulFN70gF ztFVuxC4Od?SNR?Dtj>cxYLvoa>c$>vh_b~n;m!`2SuM*tVpY*U8J(T zOle}HPFoF?|o+&EH+9Tl6zeN0epQ^wuN6eR$zJZ6& zCsKjBa&T+YP1wH33cTODvO#L!V9E5oc$N85I3mo%?0__#a|7^3?|#8gJqfNVx`g`0 z>2W=Nuko?866{kj5l=yGRD^^czE|W2yE;;-`Z`tIv7(dOGhUv>M>^PQvWCr;NWuk* z`t*mn-R!wIQ+mqLc@uRoUL;saE<=8_-yLcAef;rtjUMo(ftANqzD$>+k@WJT=$=O}f%%LH5h zOov}GBUIAP3^1xtz&kG&Q&v+#@w9p3|7UR<^xoTsos)y{sU33od5gFn>s{IZ-duxO zi&s?26!Z%sj?*A0t)MplsiNj(zoec+B=tmw5g2W_45g)qpxOEsesQZB2a!pvLER3B z$(4Ye(O0O%`3K;hj|7hQzkw^86Q~rEulU!&6|BMw6E6FGDb>8^48C?D1Al31rEX^a zVQ)v;LcMl0%sS}}E>ZUc8b?pC)2Axp@Ps`uI(#dvUH=p$#jN|b{nxQWZW)$(7>uPJ zVXP3;h?82CE2KgZZaV*u;`L2nMCTE8ZC((hC(bmP72Kt-P5IKX;CDET zvb7uqvi<+4BeNI5%7Pe(82=6TWd~ITyp5!i{1OBKH)(v}KnOdv-2yxRS3_kD+t4Gc z^4aK?QP^o%fxVJ)j{2k7gAXYT;yv4&@V^Hc_}}04a5E$TP953`=l;7?S(bVMdp5YU zdwVlkOAGOLo0H}wwaY8FCY;B1Ocz_WZ>iux$xIkrQw-(fU&5NXVh*MG zVZdf{F?;7nC6rH~{$|EdpCyN|{kaywgk$UBU)VkBzpu~m$vrRO_Kg*kod06@xqdA+ zYfYhyhDK8|;WuGl{~DY!lF2Upa}{5l}BfQa11P-G=_66I8XgnYhw2W>}EA3 z6Dm{HtXUtgG&Y30jcv+qVC|7e-2c#@byIl?(j8e;;ah|wgRkP<^%R@_D1i#fvjXeM z#`sQJ6kc$y0-tu-hqJ?z*zJ{Rka0%~r+WF~)X;8-PV0lWL17SdAOSYalcVK8iM75#&lSv88bf1D0Ecwn} z>G)IeC+sIB#r5|7vY%)?h;84Wz>Nlm_{)lN*8gNW-nq91TfUZun@N}9+Pc-O-jF)? z$|!@1aF_)9?5?tZS|?BwuN{GImABN&m66~@C*XskZP1pKD|j7|jw5>u@XY6Dz$xGNZBZZg2)Pa`$}Uo@aw8O`|Ahs^Efpv6MCfZ%!#&HNj@l z)z>Y!SP|nOO)dE0r~^}ujm6Tc%P6IpPwlVtl;O7-wlHzCkg}WBE|?;fMjiQ<1F93U z1@Eo$K-sw#iaT!MIx>Y_6+Ru?)z7IMTRfRw`Q&7!OUQIco3RQ;YuRJN-~=fAC1$H1 zQ4{$5{aU%u?z6p1`b6v(S z+u6jz1f2e=nlf~)g}{O;#Td`C@%iwK;7|D5U**F^=^3<#lGetu%JZ+>J|RwZKp zl9Mp0?hHuVY=uf40Un$d1P6BX+5g$N2IAb+X+ID;t3M z{nWYFhv7}|D#0Y{jLK8RhQK)QR*0!J+}ty+fG>SQXxGK{`V#0$Snz^kn;!QChcc9F_4c(!~YGXW8v#894oioUPm<E!R@Yj)b~%Aw7YmP1 zCxK5^6fPEXD6Ml=frGO;gyYLF{?ILKbmh1BT^qo!rx(G@k6$1>q8HNUe1uBx=*smU z>mZW=Eq~<=a6T0RhZYU)J!8r}@VkSJ(?hYqwFF;qAH%J&vErhaW>lKo+lpVWn!=7r zbjN{9_rr(7pIDN&9{Q)KU`6pa3oCyC60S}QN6)15nxru5l;G|>G2yC)?YZL1EAc0d zJ{)jA9=o19kNexY1x-@_QNEJvanqn39^CraJ}k2g%XH{-uB)bT4;PHaT#*q}+Dp(k z?T>@i=mVhQKFB`X`NnPzdb~IA*KbAwTwIs*w`GW)N%(*4^ z--E@UT$m=^D`uEHhAoG0QA?Y$spiNgNLf7u>kiz6pDRwXYL|bp{aP9LL7WM`>Tr|N z8PLF6_f5y6c8m3g?_VMDhY6jrUzJW#Z)FQ7ctPLw2jHu_4+?A)>6-OY-0p>=xzF!) zxV`ts-~(-+pij|&esmwv9gUx4wu!Z5PZcAB7WT z2dG59_DbDweO51$h3NSzbn1esY~8Lw{9&yO7xL7Q^Qc}9xIU@Uz3XRXg^DV5Ye@w@ zZ}pz~YN>@U*o?;8a-3-?rC4w2q{I!&}$a9CiBky3I z-Zc7t3E*Ykc`()P1Y2PLjhZ0?wC=b>SbXmx%&MJ2Z0ko%K2m30r9iYqoH zP&tZc1fw%IfZdukU>_EPAB9bYRalR^d{PQKk1NOiL>G_nlaXEZnGnF0} zY{gA9S&Kilf5gHS7MywVEUsfy4Lh!<3OlpT+{bOhSnGs0%s!TkrPbT8^s8q$YM_*| z>pzD7X&Z1kGb-?xxkh+fBEg@YJi^pG57uPCcGh*z5PP~}J?nma2E9}}jM9FpM8`eT zqxar4Ylv= zcX9vew`UnlpPK=rJ_TYEfe%RCJ4XE%)Fnt1Xuxy(Jp9(;SEa_f;tGxJYTSV(C-AdC zEzUj4fNMD^=G#6S4cl&yh2Dop*xaWY5b(#E`gq%nQW{K$MXdsE?Zt;w`HxR{oN$hq zH(m=lWzjJCgA1HW>jjUN-I$ikVP|K2z}+dS_<(^K7iD>p`f;7YMe?J#?28WE#-6cRBJ8$vpxjErT##~aa;P7ekofJ z&YJYzeU)Jbr6> z6`$(M!Ak!WIscixpuZWzjz~Q^T4gh+YE8gz2687>sBe2T-C%{XFkF~2J1n`TaNRK=)wCrC4AJZR$#sH19bc` zrh9w_s7vog(}hhVxO`V+#iqVj*emKU)lUTU^hh;DCsSDpysti3^gd=l(S z5uaOUb-CQ0zxb?|ur8 zAB>`F`(9D5)G)kJi^ppRZeZ(;m#~$Xy*$$jv+_%0AT7faCy$>4PeyF$E82GS)LGZy zRE9FWsVNBmeWt}3B>tn4#*E{l)HyKk>Zda2HBc>@^7O{+cesP6*pkWD@t&!lspJ>& ztk+NiRR2uD&pCB^>p&5N#k4}7!4-Cv>=UTHeGI%yrGYm%C-`c21AaYGpuI(7XbGtd z=&*{0=D1NHpBn>L|2sj6I_;>ALs|H|Mm+2Kx1DWY?*GYM7Z1^^~ zy7a*)X?o4=K>TK%3EhzX5}$P#&DD48r3@dJVj^qAJ(?iJP3eD!WxuB4YikX;wc`~z zr8_P7_lqm|c7zJvG+ae>ygP~KXX=*T+x$e(|2G%g%=dz&qsGF(kB8L$bVIy$Ll0g( zehsU3JO`vZH-hz02OM|5hKpZptE{XWVaw&#!k7GrO0hW=zikCL0V!6<3__L1+*X7)V`~SM&YR5Kw=!6X}3k<^B=xl0^ z8fF#5ed~fAj@o@Af!&#N1iTm6;OVpT!RJE>d);;{%;Tp)K*me3JuOEc`{oGiww-__ z@lve&*EIa+RUmYi%Ro-1U!_5u9BnlmiVMCqQFdQ?1Qxw_px0g>H_1r@u_}PYw~Jxn z2TeRVP+G7#o60!xdkemf%yKm{NjD6%21~jENj4HZ#`ao#E{#3rVE?AeTJ1pZ{YZr zxscguL`$&!RMSyY&P?VNzB_*kSEbmA=N(!Bzh@^_m|UI@yEXUYILD8WnKgrTK4nX% z`i8)BTMN8rjyAMS%&UC6W*;MsKIBc>74of@!U}73{LN1686kl&D93p!pPaOEdAW!sXogGln?9?k9B1Ve0c) z3Gq4E4nehl@K~D^EYQ~F)_!F1v?unQv`si(d3gmVhKymuv=?CB{uVa4PNg%KtJ@#x zmY|N^t$;N}TOnQbG~QnkjYXD*5ZW~wkCzeG)^Al#q2@U?=jwS}=p2Io%3ITa*a4_K z=0MB)=flBkM)Z)N671|&(9`Z4K!my-t!)TMc%>RmicgfYXta+URBv-DO+Obl?TMpcE*lvfCgE>j9I< z6YPomDjfGL4Ez6_&uxxQ#3CCRI<);WrROS3N4$>*)#_fhsBa;4Tj?w0{Y}CPjg#2Z zOZ@Tt4N+qDtqWeJqe^=fWW!>~^YABRHTBHkGk*Oi7M}|+{TtU7PXSHx5 zcjMU+tXDdf(iJB~pRJQ;Uz_@Yu|X32TO9!#4!6R)HR*8FcP!0}{{|5Qi|F1q1(@9E zNQalzKz5=f?dGdbC+L30_1-BMO_|9p6+eq5=f=@L*+lpcEhrls1}lT=@qTkDZfM+eJln#I%S}6ocP~qYmAY5he)k$s3;aq6!_B$%`&)3< z;W?c4`p@`;g*t5@2!QZTnqFknMpb)Q(NZhKdd_@bx+4D;966v(r!1?6C(D=7MvD)? zwyMMU(Vj$BV@(aKcD)2=m^9)Yv#;O}Pt~~5Z@;mF$uwmdFP`a>Zx4s<}FjGVWZ58?@_L+uG51ftoGrF;*XYiRY;Ul0rPk zWe$#6GX*O@FrhTwrc{m;nd0t;6nLgfumLlp*(KX|f{y-jjA{0Oflz7ztXbHz!Zn?^x1_(Z;VZbLAc|I{6S3 zcU%BZVJllR`7(5?NnoEC4fy>CZ|B-9UffvY@`l9 zNn@vO+(20fQY)gOd-|Agi6Us?>d&9K8oF%!XP-c72>RT&rk+sF=lp93wg zrs1HLt<*dBV7z6oA)dXO!2+`%l$q=ad?s6x?Hkcy^V}w~`?tojV~Tk2Tow!WA|m0> z<1uh^Ln<}z-W7J@ve}rt+lY5wOQ$4Cf3U_DmaOB8lQ`{3KxKWkGK_z*5-z#RQH}k( zaYyn&{BQJYDqtj%Eqd3>o`^Pw_HGZzI`N+BJtl++NrP~mSYWU4aEMW_0io{wio!d+ z?6!~cFzBZYZJD-^cjXcF&gdFo$={VOVsF4`wauW?x&h!$IJN!Da_aidI@q*|!rQ}o z*&}W0@MY^IfdM)R8EYL2Q}HsFIhDmXtl({RDtmci zD!Zw`9hTk>24}lI0g1MOd`^j~YNM$eH`potgc(U2= z;#ie{i&$!(Hr7tUY_9T6cHQoef|%`7ss8TsSXM_GYHGr$4tPOlgdFE_@-zn#CUV2YuL#7fe zO^(8auN|mGM;^0!K}8_@8^M%a_XKBubi);+U>rFsohoxE#>anI;gfaWD6O)01O!bX|3u%Y&=AoP_FJoNCx$D}uch39?v_oD{xeBT9) z&RZz`#1fc!;4z*PbOArp-H7KN8qIC%6W5w~HUx=YQO8cp(8lYfXy2I;Fs&s5x;P^` zw&5op_cDN8Bk&RHmiMvo!8lyYr%)TFkD|_6{KJ11%FsAM1Dro!tvu+aNxx_?pw}jt%_$~wKn6OQFHJyo7pTVTxNav z7`%^B$A;1iVQs4dXTTg_z5iqglyEg3U@t-j{>1A2cY*EDX$R^3CY@Y~LA=$NGCvN*5Ljj(wPnCEun(-H}+xA3X`r7;VOBkUKbMnF7`?x8jE0Ps0=1 zX5pW?ci5w6^;lMN4du7X8}7?TV)yw`&|0I%(F;ZRy4`E4@tr=m#pMG&zbKg9Pg%2A z%%?srsZVRfeF7il04gAV9v#2KfOgd#1k)uBf}!{GX>+$<(CE4n_Vp|V^SxSh(YTw` zy$R-cn(;+kTDu3VLPOwPRTs4=^a40)2E*cKhTN!zEH+#B5cW7@$Z1AP(KYA?qOeZj`VyYb%7j|9DEoAA}GW4WExBka1KS{!OB$MxR3 z4QpP#rPOPD;IVEetQnEuHjlZ2uN>6JlfUS5EAGf}+J{{6Du>T(?pOnE-Z6O&eMw`t z7X6{lJkj9ZCMRH|oKobbAbsqEFW?bu&@-kNG&r!3dR2Wn`>fdqXn{TXOKN8?+nKoouO3b;7O{? zI|PT%HlQ9!xxi@25h(vRfhsH^u;Y0>bp2Y%&1e;CuUB_*_m=PF94mwzQ%!R*?yj8u zo0Xi#{|udXAXRT1#_dtb%81NDiX!2j_uTW6hC+i94TY$rlqMCDz4wTuC|T`r&wK89 zZ%7mkyCRiBsr)KMCHdX|@BQPB_q^Znd7h{KHCoscjXrvsGqrA0Xh)v~`NPK%)ZHgZ zb2l>*-3cW(U)WFntvE{td`cmoe_qMsJ_tiDv#z5!W;>X*B9Pgt*Mm|uotZbsy_v52 z&ZJh#5a>VWM5eoYl4n+&CV4VmrhB5{@F^t%~RPfiRfgo>RK&G}uKt^z{o*OA|a*}j!3?Z$)9stwCKG%#Cg`j*573;Z;PNcK(4080nFS%IiAo<(e z7-ZGI1c?UvMBy(48rBK_!?_#5(QZxh)TjjcuG*Dpy(hsGEQ>&ko>R=f;OB&h-5pN- zFF*7t=K%P!aWm<^G8{a9>PY@eCrRq_O7z_D0sYH)IwKhwjlQOdB7%-5#vG)XYQyg} zy)pMWEo&xeSt=C$Z0#hTr)n~Ov=Op9as^%AHIw;T{tW~yJSRL41gZ6J7U{goj%Gam zS;q+3glnIV(kknWkV)<-=jl18jQ+=3+Tx}08<}df*9x3 z%!tHZX5Nnk@b084sCexO3Ui%F6LSLGJuC+152b=n+!=&q&I;x@UzCw=JI2H}Ze!%0 zZX#!v2M|U-y-4$*X!1mp1G!>eG#ILOBb@|0NRcZaQ2E#-u~vNneeFsz%4*diC1)-N zd-mFqo*k0pZ9`wAaA6nv>ZHM38eE6|D+pyoOQaaQ){oh^_zV-`m=8EAO+fc#FOf8F z7H6%~1R*y?5FyhvND~nkAlY*fDCUWhk^kaR=+I+&&Ds;B*x+;U`Rp1}bZaabwq`%a zaOYCA|E!v@H};_SyPiUK(&A_<)mWnapBO#YXaP{N^WyN|rht?qYeDm?TfpU!7P_x; z1|)wC=CquFAmUddsA7VE)ptxh?(sl7U9KTNi5}EH6@_-X?nG48Wt#A+MU$K6Ai02M zy5(mkda;CyiWkVBEEQv(^D{Hr`)W7x^G~7m&n%(C+@^sP|5VPqoJ@L?w+m{|@u3~a zLbTW77QJl!M&W)>8YwFsMZv5N=dqmvYW30wH9OO+*WG3~3H>k8mW{T^?cN8p^`j;_ z^s)o2;Q$cqC`#U)_Y~wF{Yv}_kR-J&rjwucS%9%?59#NB@6r15bJ3Q?Z#mu0TZu3~ zd7|lKC2@K0LUcnd4K0p}peOr^=(`^C(ACw#-AnOLA~xtAeQa$wQVOc)Ex)IPx-RL0 z{ngIkY5^^)i3>`XUV z{vz6B6X={}vP`?mGBC=0jP~xr=;s_Iuv4}YY}nBUZp&-~nFqInNv8#%Yne1TWvB|i z343t0u2_`5ybJkV*n@@}{D8vBgT%4=PGbC&5yw&81w1>tgoxHJ2dl@@!H0-UVf|%{ zPPfG%?ITk>{%t3~eftocXej^_XIFrE2A;IV070kPB+=gU7Nhhhp~!Ap9P(A|Kw9nr zs9g6R{pmv>t;(37t^2tkqEG~#(B6UOrawk*?sL#l-ckDdmlBY*C<65CT~EgeKhL_Y z`w_bvp{F0Kd0NvkkrETjV{R!EWS}{+egX;chIZn>+Bg&vHDrC`Zxlk0U6Fqm1kQ)E zh_pMeIpwQb5&y|btn#Y>OA_-dj@^n>oW2xlR?k64uKf&GSD$BN7^s&74lgdk$IB=sJwL#wcRbGeb__5 zrZpSH#$Et-fGn|q)I+X@XOPIbT0%dg67{WfMLRq&3OLe&GJm+DWGORn;)W4eQgVgB z@i|1;8zti5xI9p!_tWz!82$B+Y)DjgK{T; zoLVAn?v@6a`*VOdzl0b{azo4C^jMEE+Q8(&Qs9a1fH;E{pra!YJo&Sj7!LMACwJr^ z+t4e#YgQDRXStrY!}+IZVE`L3@6l#(@MaOP zJ!uGnU82F4h-2vG-DGskH;~4uGH77yAaNkf5F~WIp=}hrc|UCl&K|{@n&73;#ORhM z^a3E zkJk7d491Gy5kX6XfMfR@;IZN%A!X8ps3WB8T2&udX&}+V%2)UmF`gf?(i=>~@`y8ebYaGRavuZB7oEndU z6}7>Ry(iI%1tEm-pe@ipKL_lJum?#_4xs;~5wR?3fG0a63k)f!6ED4wqxS=GNb8<( zwykm*+5R&@yB=^+)ahMl*Ndu}abe&2HBmrB7R*3o_Yq!n-bSMAm?ijQpNL9T-xI}u zMu<(>7Q~BRAv9m~ypZYQasDYP#?0CYvymmTRvSU7>SZzuisw)6z6n4{pmO6qj z?i)ESpL{sRI2LTS5`ePFbPzczMQ)js1k%$!gZf=xh^*QsV6yfraiM-M5S?dcJyjG^ zV;Ot3hO#|RG+fL@LkfNbcToUp`gw|2y}ZjhS3}4w zoh$?T%|k%cNjP);Pz|ak&Vz03#UQ*fpO`Ub09JYXqvW+3p!Bmb$VK{09-V{KXg#KE zRGz8sMQDD)`5IzUg(1(2F#0EXpvmY0F!4t~)oh%YwstMhn|ucDJ0}s@BJN;HUW1&L zbrBp4)FzAkCcvcheU#?Wk38TyWcaKXeJTnjZk-$erXr$fZg&iACXhpt>!g?sMIy-k z=4X1UI0;EDkY}V0XrR^F9rWS6Ib``{BWRY9CQ}}N1Gjc(0PLYeDsCU)T-;kmd@a~Y zh%^j?_;v<0b7j!BnIUK}Obu|3u1AUEibyl)5qP=j0eBbl3b^Q$gPFhNm__wn!aGKh zk!3ZQ$9pyD&|+0^xiJ^5yYn3h`Yl1`#a`g}wUVg26iO%A8lsny3sKt470B{o7mb@7 zYepCFQAnq%P{$Ji{0$d@ryf76&HpBVh4Tl%+l)a}M1-TGrlQO}i=V=|&je7L_X5bY z=7HME3&64B9Qu7&51lZ*jfN{%Ans-na_6W%Sbcf{d6_RoP9E?;4m~f>Je4%G$IKOd z2<4);F$Yj+(j$~@(~KmWmw|nS!aX}VL>$O@!FxRUnQrO*g|0N^1C50U&>dAE!z2YH zSxtdg>gNIVWSmGz{0d?UWod!5J8e*Qftc9)6j)IXVB4x?sJB&yHn{!{)jF!vt0b&> z*{5wt@lC%$U%f85u||$`3G@b}zdiWR@g_(~JqISvF~p$GTTX7L0BD#$07;jXX-7MI z^jJQ+M#6fW;I_o0&we6I4k_gRuSW1@i!}qIdksLlBM-<5=g&KxPJv;{(ke)!cod-f@o&^X2Vjz4Op_>s4hKN#BRjSU+ zvB(F7g`dFs+3P`I?I9xE(g#dVyAS-0yNGdt8*mJG2)-^p3^IT4!3no>z~fpM(7N-K z5M2tvb(0Nr=OYzTb3-#RsVz%B5NbcZu~P`1>Y*QmpG6g==dCkFkE459hr#`E2>8Ee zfyq)D%<_Cn^!|zi%Wbz1w~U2s-LFET?`;KroNq?|CtZ#PYb?-g@)aFydyiO|tAOIy z3%~ER`ru*0UU16t1{kT=1=EM_a&nFS(Ooey=#*0gL2b05^*u$v##3@+Vr&j^{!%Hh zJ+hvYVeymG?Bxi&&Sw+B+3Uf@lO=St%Q9l}`2bHJbrH)l=OSl~PR@*nZS)iFlj_uU z3>r-35lIz=blBihdiRR$ghf;${h;O`!14v0FSqUzt6frfne&}##p&1SjW7=AHD00> zmtPk01Dnz1;V7V;IFn9aFvhE_n~&B$2|%-JSE7)NdFb)?W?F4s2*~K_rWI$$fR;o_ zP;O6ywMmxXow^@b___$xl$0U-#g7MWY(&9py*WQ96SVeb3U71VP84t6LDwf`(_&k; z@E(O9CrVzQ21DJ4z@i;Cgz~j(9P(^3oh5&cr{Et;SU*_?EVMU);b}7TtAs>Et7V~} z!ALrdM7s?aEHt`_uHV4e@ zPo!r(IzoR=nTG-vJ8--Y3+HzxGFDkzrNFaBbpo_3Bm=b)fJKcyxqA;J{bsxYM=Nv) z|IvD&%vlZGzFQ;tGb>R0`wX<0zYc{v+(F+!6z`ygH2VJW3+JD>CwQOe&npGTICkRO zXm0B%!X~m7E!|*@LXKQUZqY$#OVL7toAHeB@zw*M$Y? z-=$9*q}I4jrlG3c!NBVNI5BuI8K`d!=1JDy;6Sr$M6HmIE%=i}T;~oEtsHgqRXmD_ zo1-ACEuMk7)_CccJLqaN_F~S$e*-2l2LrCDJ5BL2v4D zbpP}mwDXt=diL)$owhR^neMnm-(KZSyu}kl@{?L1sHF(bpY zL2P8!2Zq~yy%<9&^;lDe!_Gr$euE5DV76r@P;mqkU1sYpp z!S|aUL{FHhmA6JGojq5YRDD{Dwk*&<&V#8$dld;(0E&VUE z2XF^EIHlhT0QVBM4mp^P3Oizn*vR|zhr9^h(V3VIQYq$@42}@X{tHKY6(pD|cLnGt z{}%mLwHnO{5o5mXIzd<-Q)No%DrAth4fQ-KK=C`5aWv1Xfx^~nC`IEW_$DpnV2|*L zarG>8=%F<8uUE)&Z*-t#%NCJm_jH4K4nM)4moLCboemjrrW!m*`~V2YOt2a?qQkL^ zfS%zy^!c|eo#D!YHa8FOBIOm(Jyk5+*AAiW_A61HR29;*+)cYqFDKg384#lFM|cg{ z(PGynnVoy`P}dnR#2bHw-adOu49VXleqL3l{jD7cOP&N7R?!6XGx9*{s4syPMVRbP zKa}g&i=t1vq5cbhK)0X)D4)*$F(wtPdrAXB7lOjFiD~ZEGj_Hr+590Wbzl4XuCH zMwr}*0?{|tB0<42WH4n9N+wm1RB0jjQdUJ9eYpz8Hr)oJ@BRWmDLz;o+=MPovqc}; zo+Fi(Qgm)!J~)#lPFjcjA;7Xy(5Sl_Y;{!z!y41TW|fBkXFUhmQz{^|vjj*S>;xg} zPJy}>Z#vK75h0aSM>`~P!4s2dNX{VvD5k}OUr~U(yKg4QjFlu$e9H#DHp_^lE;n@N zSw>C$z+5zOtq1Jx4Fx<4VF=xltF-NxN??>)$~$;Sf;@Ao07P^z1Eu}%Xx-13iAPz{ zXkFH*P&ev{KIxsI<0=ztoWD&X&Ss(BZDb*uoL?wZu4MrqDFU_2G2piFJ>9>=iM4nU zIvm$e+&!)bbXIku@+5PjzpI8fBg`MfWG)~#Nh#H&JC6cG^)r8BJI(c!QR& zQ2?>=3z*5|M%38mM;}o^AgxgkL~`cyY$~48GxiyPrn;}_Kw=l#qy7N>J7~y@DKGDBfG)UwadZ%HLB#2mO60fKo6aB@+U{&)B`?#Zv>LQ50Ur|T_9H>+}p)SGSAKl z`MRwLg|}uCS*}aaL})DvTGWX~9@YxE{s^mK#X@A$DnVa!`-Ccww$Xp4<$xlU1t?4S z8$Ugk3j(*_Lf&4Nz&oxkV?T?Jbi3~%x4K|(;FbyF^|uAlax$o8uoNWB`;GF9w9(Kn z4d%g&S=IyjG3fM!7+|Hxkz7a=dXzAo+42;k^&P5AU~MI!gQhc84Hr<4StAiImWm2P z+W<#xJ$PN7MU!p!kXiXY)HLuPovB|!*R6A>dnz7*1s1o#Shg+U6uBL-A~NJgms?;J zgNO@?ip<(wp2$+zi?2`31^QDrfhT8#mT|jCce;y_&M}pA-KuJkf0YF9W^Vw$LL!ir zqBxnoIsxq6lLLM#Dbk;hX^;h47lE{|3^Qb9Li=2Y%$(kz=-7f8^h8(#3Uu@*{CZ4* zSA_<7MyTm2F|s6&^T3x6mD*r;!r;Xs~IH1euw$ z0N6C^1J7wr)^$W1czb6WaCkBW^740qqYtig+D1}=LuU|b+0P>9PGQ|W^C(g@mLbQT zeZWtPL|`W-55O%EV0*iXcWkLAIucqy&)^p!n_OSyzcG{cJ(&tFyzNHq*Y6{V3*zLG z&i6FEOM$HV%>~gL%Tau>JGyZ{kBC?O4yG3B3faB`od4>6qupU$Xyk1H-SapfdER{m zL#Ys8r%fY%g2QNP%}KynTuY}0DUq+dS_os$zX)&9pdWjV5)q9H!2ITU zjCu24;^3DEWd2JZ*qy3F!{IJy+3#X&4_*N6@FW6Vj;sY<7i5^wh$f^Z6OX=!hY;2W z#hJLN12yi~PavKD+~||`d+AT_T-503uT~2?KNJK@`Q&DSV2DoxZ0a^a|BRngegzYzkqW-m` z(4E7l}H0l^GwK$(l?XNBVw^0I3g;5cK$Nl=HiZRteexy#Evs)nku&&9j$) zv+sTq(r^0^=Ry@B^+N%j*0DpaWl7e5H80W;R|Ao~aNpf2tUp(8j6jc-)R3`W6EUTH zji*qy4JCd&i;Dgl8=zIy@Zpa}?f<8mHN zjazT`P9sthHW1wA>F9`Q1JVGN;I3CYz1rBSW-MrcjzEXev#+&8goF%d`FjPo9Ne2A<(5lAd&XO6zO1qKUVq zpb89g3d=Xp2^v3%^!|-RxfUXJEh+#nYi|IP;~(j%fm=kVzZLR3p@Pl?H-oB`cZt9f zMey&i1Ra+jj8sm2rhWWga}+pIU``u;spva2Q;aJN4J zoO9F2gWnQ}=5Zt5+5OLm;qd#kN5)Ci`Ah*_8QOp*aSBq(2?p}2mucVs=7DP?cfig~ zCyDeeGr^i8nmoB^WqM`3CpymAj1*edBj5ifQB5iU4}31ui+k3B?r*hVv9QPd`%|9R z_1+KF8;dhD!X25t+hmx%2~#LxgE2EhUWsYl9)~`ENke};9wSr5f5_+25E7~PLp6G` zjI6E%(`z(_wyw*e&*MV+_N8awR_G?+sngWYf>XIubDN--+4XCS;CReDsOm1P_m~S45o6(9{1P4 zY5G6nPGBq$n-(ja3(142pg!W^YAKGzj|i|M=c;vU^fF?`FpqFEaY6$=l0rL$6tK?V z9;kQy11i*&$e#u(r157pl9N*as7e)b>x~F-;mB_w{zjUdlcY{Eenr5U$px>zijdCQ zkQCK0Ank`50PoLilHVk}Pjpgg!?aL(-W3J1qInJD885+PepX~I4pPiyM>*$b&n@uj zWDfFmSb^>?UPSiKjRw<)$AIa6Q!-j73MflDAYUzM(A%~PSf-mJqPCgVcw~(Dh8scW zWyqMdzeJqn?MUmpHnS{Slo^@bhCXc*XO0%r=zWfGPklp^`FK%|k!ld?eYFTiPefIi zx%w2T2-#)3Tq9COuL%rx<$+!EG|7vLtBBORb42-4zL4`5C979(m>cf-=y%8qlSfP>3*O^q~S5a1-8f4y0ZH-C5cw>VP!!x9R|DCZSUma@PW;rAmrfadsNBz*3pXj+jp-8+-x7F2g`O zMVnkPJOn&GWdn0jEf90!4d`9>4H=l7M_$5b=!!dI=*#(4q)MAMnRjtixKlMF=eP;A zd|@|%V%C3P#9flS_*s^5`6s~~|8EU?oF~T|+%=o55zbT2Ccg%+PAZT|F8a(>6%pp* z{Ty`a>Kx|hZ6hXT^b^vgm7y)N}&1{juP8SUM^~GWlB0{5lR7%g^{VSgMfln^CP`qxx0I0Vn*=#0 zP5_src3y6wGoa*r!H3|5;GpUubjczSksn1EBIOD#eYhUHQC$ei9SnF5W8ECFHU&_0 zBN~`217IMm6!Z({&r4cnFBJ>bBsH)6BVPXobWG`VG3Q}O3)XDqe33&Dca^&ihAp4G_W{KSd-@=(eDT7FXKwg zxrSxPMONs~v+x8OF)2g$ci!WS|87J(MantXOr^=_As52TeT}OZsbsg2nSM(b)#if#VEqBz35Y!%vwF5=6Dp{ByCy9Bw?& z>J34apY4dEEe^=wyfLl({0w@UuSqat--#r-a&$#MAC4`ou8pZzAwxn0zO-kA6)N($0BKHeD|el=V&gIA9opXM85-m zRsm-uQ@C?<{|Syi*@M34&IT{l)tSEql}I*3fW(kE^Da~ytu2qF$sfh2(XN=t)j5GO zT8~y!XY9$t88sZ`+V$j2{~o{`*Fi~rQe@$Ua_fk+eaJqphWPocoZd0cBRX9yIMZ$X zXdS;Y8WgdFjm6EHhGnK`M~fUtkC7C5cpXAzDcX#jxh3fDSjq%fzei@>W@KDS5O3!q zA!mOhAN+TM0;LjbfY|g%u*g)C)4gW{8cnyPb)zVv^HU0K^i!7X>mQ}B-LoKNWqrYh zz9_(M^+NoJ5>S055-iDhNa*`)L-zyg>HZ3TVtgc)UJ90gYGEI8DmIzcob;y^*GQu# zEo0)`jYpgu(^*7E9f@=X%80GR8p7gIAHXw3!0&_ophYp4ke+h|F@Mj14~yr}Gtp`k zCs2t!*Z96hZS0%5mZ~_la7chG+p92R+q?u3eqmgXIOCaKe zt!GFpkqny%F73IFMwZLd@kf5pPL~f6JLqR1^rj}UBv1!9UD^$_Z;hf7{~}_YqaL%v z%Necqm;mPAJ`&=mWyr-Ub3l?sEMU|s2(y`uAf&qsBwJraw-$(z6H5b(W7@=);%n;7N z^N>t+C0g-$FKX7UDpKv@4qh#N)#ycLt>h^Z{#5NL>`q^+Z8sx|N z?}a=G6H0BiJNy$UNgc7ahj&}G;cXE~o9gl9+yR+>YP$DAd{iuuB0no($?7Cl@`f;% zUgH&XO!)#+%c@wFXQ{CNUO4wa%?s+BUnceV!!YajK?&v*|HlUCG*HX*v$>{uEH(V} zH|2IboO<{<3qD!q1+VJ7=bt%rg8$uSISlL#z>CE-@ybpPHD^;f7u3YUThwe=mHLZX z`#{cS@oOH`G3cj4cDTXDy?vCWu@bd1V?R43D21-VoR~|sCvk{_96a?{-6pQPo_}<9 zGF$N9arnYH4Q`d@Vp;Q@aC(O%>lrxH=G&DI)cq%e?Ch31Y?{42zG)E3>RgD1--BJ@ z_Ao2h+X&fx3$Nqc5HZLcNq{Y4QSe0SMr?8o;r0F%tQ?`sh&C6Jp)`b%y91~fZu)q+ zjvU-G{~-Ii(isMA>0`g`$%A&0$#`<)9Q(N^6Ao63!QH%FHYWX2&}a5#{)!c$*kYSL z)IGq#Jn2aOEAPcPxpbb*^F=1yw{KMh@y1^8P{I*d>*GRgXn)P>tzIIq9#+P`7I5I- zvjebAP8?oZKgx!SYhXv?PdKbln4v^JIK|!;U53}<|DYVR|l6zA<8FBn3by&$L0~? z@a5TH_K2Ph{+*yf0jbm6ClwiNzTIA&y)cA2zf_&l*(L{N!i*u`D2nXvX=jbY-cmOj zx8VlI3+(Hckr1ccrb-_>;I1>Wn7Va{%5oRSty@fS@+vDVxA*{?p{5KKzG8}Z@FV4V z?I{fWF^3&krVOVE3^8@LN|=x4$%sS!YJI^APsT#*U1s_(vy4GEhDqw8->NNUwkcpsm(L^@m&g=X?GraTvx%nh92O1 zpL!{|EjqAi+XE=;{Fcr7a~v;w{C-wW&3u>=z2DW|nmpy-A z3kLd$(C+gfdup4e0O@Ezg{?WTW%UaDC}uNRa4(Fd(!!@-hDOWpXZ1M0tw0NJ(zkCAWLX%bSXM-?@Y~fYzCy!Wu z)K@WhY2qsX_PrT!o8on9R`hjBX0i{S`u7ztxDhMN3-f||4*7Fc_}{owp9;B?(P{!1 zE`a}J58*A#XW~^cO;{#8l8t#a7fbyT6$nl%Lf3(C_6{(@cM1m~eKkoaGEafWbpNoG zgLUxwbP1aWu8BBLRUe;!6%Df<7~xv!L|g$>prpGYJnOOoCR+bs^W2=-o%jZJ@wUQ- zXRc9p8C86>L3Mb}&WS6sZzh(lv%=9OPlcZSAK?yl58UuJ3GR3M0$-^+!L~yz_B^75 zEj*^<%C2#E{EsWUFHRY*8OOpby;V>}Qvr%toxyydi5n^<*@GWPSS70))V5hZcz$vL z<&)cqZ9m7bUAdmuerOGGr3rM>_*Gq?)G;N3qaSkN5{*PY~Gz4(=slT1EzQ96uW7HClR z6ZUvZ_g#E@@+tg2`!@A=O9A!bN*z9bZIpU=z7!|NA=YEza?IUO$SP~A!?c5|@Lh>= z?$*NzP+$5rmGJ64wZlS2pkI-|&idzzt(Q7rYpE(ceZ-OaIyAs_Fn`YdU|7L_7H13< z!ydtK>+4wO*FJnUdLw(v_Y%KEa28sIi14=&-qf}g({SGARqWTs3-7i^cLPDee)>DiTdWAs)mb|?w@K9z-!S3HG-eFbpKV;v}I zwi6Z{ZHK(Y9#AbSmoGN&Iy|MYoaGNhun*=a~aYc!rXl?~b=C2@R+=-^8PbeQ@4&ahu%}&#A(WG`u8v7EBgeX#Ped?(l!e&Dn4T z%STLyQSt7q?&{SnfjZ%MMLG=S&UzA!dEV;NSdaSWtYA8@gNr zpS1U*#6|P5++-$K-?1F_|0?4P;)|iYOct9rvxSA1H?#KhM)16-99Ai}svo>ej zsq(6|{OsGdFiN`^o@+eIf?bO6h@Z0Hit~xPeP_IJh6Msn?3%u@fvp6h%b7>wl|8kY$ z{!4_9^ZOa?5*IjF(1auP^dZd5#D6QVlM!=f+ALMM0M8d^a19?U!P|vC5 ztgJY|Z+`t4imDFcr)LE4^E+Q^{`ExoK$*)IH5|j2&gkQ@D|4aG%>(eF{1lpS+5lG> z(rlC2F4#5g8hmwHTCn`k6n>EI4%tiZ*h3aCIHHltg3Axi;l6HBsJvVqO080ZBzQ7DWI;@8|Gp9k-$>T6Xrv>MJK7bRK$HJY5Dxkv;W0+?KG51;kT+%yE z)m7zkBgO=9%ZDx)vRac(Xe-0Hv;c}eyh063O5urlwlH_bLQI#Z!Mj`c!c!&dG0U1^ z-NdbMLPFf;*_rK7(?(q2yW|S>eQN;j(GI|^vt{`mf=qJMo5l}L&A?HdOs=@06?=8B zCpCVs2%Zb?p<3&>FmCh={yEnH=f@@Db34xCNr4h|`ARZ$`c()|7s?8F-`5E zMfxUO|1=HW82(5-eANami>j!l=`C1X*rokge;XbMvw+8fIMl3%Dsc773v6hq2%I=7 zBABOdO1*%VSf^PR2RQs;@rA8y$I|aOcexB@-nkebUOh;eM@@I=~tm zaoVZP%941|?`C+(R|D%Ba^ZHTK31J+qpbFRWN%s9ulezsno4`d4-DH$jfVTM;vKQ@S)?H~ z&BGY$#>Y^SNCH1E3un83|6v!r=V5o}ZvJbDSV|(Jk5USgw$a{G3EMLs+Cx3}E|mvRvyaibcO{ zqu>K2oAu9&sFKs4V7hB2Tl6^zmkaZml4P=R^dN*G?}u^6`z<(3EDI-PiBU;e*_8Rr zBz!mWCN9|H!>y(>S)X-Al>Vtp*!!|M*XN2iJX_0!FZFJ)pC{eflIR_5i_U&%f7JyB z7?W&X{zurt?}HI{2k_edg}7?5I@}^E$6u?T&a}!`;a&T`!!<@laJa`G)51GVRm~AA z-ChD6p1MGyH1~ z4oO{w7pGP*@h{Zz+^lOj_QVzJcSeDKG(a87%#VPcTMPwrTGjC`{zG`_UIlyWWi7Ry zV+YqryW*kvbZSit2aENI!qcHIF$pnIf}MiZ|6Ylu4w^%OOvq0 zf9EN=%(ra%UlW^=uTfYzF&e+H{DLb-Gobj6xtJq+6YsesVzadH8a4e5raoF};nkBZ z>}YQqwck?~$8!hRE{z8;T=oQ<5?{rg;%eIL*nbNzyzaybEZm{nhzRB5bRXV4P>bUX zB5}uM4)pQS#J)2{+3C}cK&2Vhl=I`0I9zs|%8q=5B}>cMPR-}|(H;&g+g}5(4xV9K z-|dIaiQ8cueGRLnRkDE#qVdJvVLs=;On5|a2>mJ4t*rHT7C6+@d%d}{fj#qhr5Vtm@ag;H4-NR@wh zM=5%*!?yfOZ1&hWDyqy3X0zw;d!0Wp=1Bt7I5t6_-c-QXkn?2s_019d5W0>R)D_|) zQx*JN_5oGSVX37}3#jSJ25^N)4!n8VO7KjTM=1xXP@Yw0Fi-C)6!p8sT4t8R8>J;w zaL5GCPt+15tX9Eg<7V86;#kNWI*5}7#j$K7gb~Fjxxt&lsP?WC*m^@3TWG2!&{>|1 zVK!klEnGoOC0P=I!|@!d%#*Q$!xSW!>myZOy-H;?OJOw=ad%Cds(7bSfn)hmDMkfUhzkN8L$e= z@Mpu)(^;%R{Y`9JXaLhDRzQW~k9bkP7c2g?4|__~t)}dO&emlw1Q=z+_1$Jna9{`@Y@5P0?b5Z?#M;U_`*Y|DvUzG7h%Yrl(wr*DzLv*$)a z*JqdTujCK>!fO{Pg-1G2*5(;|$g~L7{f&mV+GFANYcHuWK{@PP?1b?a0rt_@&j)u0Er&lP*f0`xiLm=xsPMZFvLIhu^*y4+OUVW1P;n5;YzJ)h*l(1@;2h|_tsz3lRtVmKyw=N{!|QRKNjE^ zH;6}8+d$E6n-%8ApB+bmOarDN^Ku4q^<-#!2kXLT$q-Ft()So zNz(%uWBULVsxQMr(;W7%nhcKY_klO7=L@Dpw!)lWuDI1I6Yrz{Vo^tLYUYBISX9{C z8Ed4&X+3pdn)Dd+eyj$Iep1D%OY5oeeTrnfPb)j!s+zS8et&EA(-_bf>`_I?SRnSFuH@Y@DS<~(aYcou*7uNHDr z-0@AfcvxIl%qE#M3;W>$Hf(ew_VJFR-iSSdYm(MLgWM=;?#mH2Xv;k|RN9c*sxS>E zUEU4v8mHqmwFR*3s{{5))rVa!DR8}0H}o33g=^*s&$8S#_RY8F)cTN@EW=^=4koeq z>S9%_sIF-9L^csFGAhD_*-h;GU(?_@M;2}h6BW$*=!%uMr(^z}6u!;hD7oT&5`iM zrbgEKZYiX9y5a$uVXECL8n&?&?Eeg%ha;6=7{~2ZR3yZxv>S{am1YBF2PD^R)9ZfHDoiP$p^_+MQ<=`cx0-(TpVh#%GP-MAuC(!Unj z3}%sKA!SG+y9GYac?whP_yXqdX;N$#0Ockb^snqKqdC)q2d8V9WQ)Bd>%SbbC$<+R zKh)q@RSIH+eg1c69dqcdDDrZ+#k2^eGJ*Ac z?7Gi|R5tEq{vAGmG^*CX>i-&;SDX5ftK(JX>;8visD{rhnX5{=OaRB^wLFrZRR)(D z9AM}<&Wv}$Bcg41h79&?Lgq0M(6{p#tgeyaJQ`myiw4^wiw{wt?!6LZb zxf{)wq!@E_g}L44jD94F@k{dL(SNc%WdEa^yt@Gi2HMO=uf?_bO)|FVkxv*oG?+$A zWdq2*h&jw6V@Vv;bO37Yy8=_#=J4 zeE#l_625t(uct$apMcw@6}t!)Tz|o=tXhJG*1bh0@vgAN@;nMIEhZnLtC8rD9Zarr zHlrvk#96Yc3bD3J@h?`fP%;xiN+dR*@U8B=@vs&+e$9#}Z4k-?92DFUfEY*SY#+Ix zHO+Wb$RW#fcVNk-7-pN_3gUCXhlw!SA?T*nB}dt&NLEq?y(##BsLO@CvWN-BpK&7h zjc6F*aS$&Clf@NUyj)Tb~7ha(e7jK3K9y(^tCyjzUs)JwK{_fw`m z>=5ycyF|W)H6xS8D)O-R3Os2S&MZQG>`lvuQTRPMxO=rFQmwoUtE|Q$9#Y{nf9Zgc z{u%5%iv$Lh)uEZ@o#ZODoGjW|1;RTXz?@rM#QR<~`CPCKDs~jYSN1w^I3}BY?s+{K z+U>?0m1!3+a&=JZ`dFsiLK_)2Mxb=jHZr1;gm%ap@r=tNk?eI>^t(@(KWKD^oLa2H zR0w#9p+=>ocw-gYnXd@kl3%c8%st8bIuTgW2I+&$KpO~sO zt4R{?6++j4Ge2G5l99yc%?vX98jv%F(I#6PR8rc_6juQU~`1E!A$P4})q;oa^*{!$+lq_QC z(fAr-z1xK;tN)0WFE3^1cr1iB74I@Z&lQPUMh2uf;Y`%7YNqbq8AjFaI+4gPhif`V z$pUsYIhMSe>?)Im0q;BDnicoRSEDsZJxLlBsvm=?##~b2Orf&3>8PSSlyrJzlf=x! zWJml;hZjttI0}t z!5OMW;LH0;=rPxVb3Iv-M6SP0_}d;r8_PO4OX&u@Yk!=$6we{vUn-EjB}JS-G5V0Y z4xMzIi#(^#Ae(_rWFOy}6BK7f8jj6m)UEW9b4)HWOO2BP*Z_C_-#uzMT(Gsc0P>hsba6Lc|`OD z49iVs;$&iykQc%YjHgB=Lj~(2tpP+9h_o{%@d6H^bw7OOr_Elz)S6=`a~?V`P(a6Q zKfq_S3Fl1Aa{kbN%4GMNJal$ZDjE2Cjwo2)VUpI#k}YK?$&VNT13cjX`)?m6AM=gT zBJb63WRnq0^7nwGdy(LMUC2X;#Wuzcii8FEf9<+<1iEgVQN1Wb9 zI%7`3nbK_}e%gfe(n_fHpAu(A^hsXfwJ}t9tQom{l|dCQdgRJX1Edf-2WbRXk<&l6 zllx(1=vDJlw(851sP23r-1qq?nW&V7%bur`b54~6J`RF8SD(TX>u7R+$rv*>Qit@4 zN8z`;Znlze7OCFEC5_r6uyENuk~JWQZaxS`C(3Jzz<&nVdmB5G*K<36` zD4zd}%ov=<6kKY6SDf$T>Mw#de{_+#9;(k6%6T#Ht|KvC%DphjSUs{Kl+g_Ifvtd|m%c}waStXZEDPms*WpCZn@^H?S@8Fa z9Hj4+fSSv^kwC;iTF*0y@PipdFGrN)wniVl-PVSdoc#a~3pA*fKp!Nx%L{=gI{;6> zG{&rJWVb;ea@m-RzCJG}f2P|}#Dz+->fSB3-mm$L;jjZKKHWn?y#65}X=8LMNS$9S z*ke(V4$Oo3m5iG6J7)D2VS=TeA=8U{$+W2qvn53xb`Cy;$c;WhFo*W#(O%P*|$G2Z!3$*kIsDLcjylsmab%Wtqw)5Og;Ru`zgD~ zkPp`#0^HXVk9e;Z&p=f&tI*gQU&8)UhI~~;QNtfOe#Ogo&=9qt_h*j4doQ`9<+wEJ z-7wB9QmJF=P5-dbU>|$6_;I3Be+T{hbCX=V)CD{3x1l#Y4wIAE$lkI48yP$PlB}ES z!#BF?M;>Y{N6~V5=t;s1V&x=))HdIOAL7q2v;L|e|Av>0Uer5gQg#G}DZFE5_81_` zNBxZFBu!9wHTl~XjQ$oZ<@2Srcym8I;`KU;B6$fb@?udKV)Dy)@_GLdBsi; z^&uSfy}UqHBMlddAe6R*b?XX9tk`4H z%&I_Fa24;?Cob7_>p99&G$a?I@wEMH-lW3n}QnEg=~`}awO~;myES$5-C9q;7!OBQ*Jj5Rrk1% z7wh@x+>2A}-G3Ial`8C^Y0y*lSug@;ROymRVuj*oO7nUkL5J69vF|AoB#jTkK#%LV z|MVbiQjjGN+wQVGO617<*-|9c|gA+{EqU-Q%ZV)qB zEXw>?@t9;RyUAp`Z9%%L#7OM|G1OJ!g3guCffMW?baCVf3A367XUGD!o%tvlH@FOU zuF)bbq#V_KDkQ$78p)}=X2$2NBEApAIO?*q$-K4$U}#Ahx$u-nl(|{x{Pnp+F#JMN zs_jsHa|Ky&;t83ZqmTYl*RWq}egZp??Cc{IwsY?L{Q;=WJ4zro#-& zEg^H3zvZ0-1!N?eBARC5#QDN{lzG@2PMS2JT?tK${~IsT)ij%PZEzi1p=d8c+UCfj zc^a0otYCF`67wWXiMJ}sl33-}Jl-2f=x0=07;!z<^akvY**^!AX4Yc8@gF!gY zErToEX^CpKX>lYk#q)T)b~5zt4$OMIhW$+A0h=FQ0pGn1C%>q(Fb|Y6U%j`Hn9Zlj zE!4|~3ulszt+V)Z^-S59;&jRT^kBkUJPrUD$u-MJ_|Hc`Rx}+E!>jPK z$We4Z{t>yJJ(tv8{|_BLX}}vFI)*r+(P({f26J`52rgfoL;M7qYMF;upLL`?y@xwCp$fkW+?ipAQOjlT4=cnG@PL^NVMy`~=R*2aK_nEPBPx zg!eBTB+=>)jA4ojx!|#w43#u7T**izzPAgi*6$+$8WK?Pq8<`S7Uj&5JdBD}&%=?e zD(K;!gQzcD7(TDR0P!0>`NMrg+>SpY!4mU{<>>|FNmMoaZe%mG(9~mwQuNTG1&dMe z?)OYVS3fin=|tN5x|pbKnQ+-sH8#4w7M&m(=)L${WC>KzVugHi`Z5bGzidsaXUjpH z{2!DIHy~eQ)sYZ>OkA%YL66#E$+-0fvfA=KOl+w^>k7Kbv4r)g`mQqiD!3!pyU#)q zb`|XCpnb%2?mKpZ=R5dj#cjgwZ)Uq#Tqn0a^fQZn+R#t?Iq>!64D>eV51eaSi%QdS zkk$!nV(X;AbXB#YGd|kr@AGQLfIrEoKjjnW34jXz(oEFaGP0uM9_m|imOR|tCeT|o z_yeKx{Kkpf==y}9U?$^BModRYAs@57jY3GNTnjU|sFrP|e1h5hwh;x?*0T2vx)Q_q zZK!B_5xQJ?i@crVi4Jb;L&>T%bK{>rzu@^|o@{s*x_jjh6MHfimFNN{{nSfjzRZ$5 zy`PP8xccPkpA+a{=^Ao;)R0vB3Uf@NZ!zFS4C8lmCo%UlBm)-aJokl#OqDs!WIbOG zf{56nDqWtGG&bdB_>->%J*^(-FN8-s1@lI%#{0!1pDv(W<1Ftb3 z(H-HJNhU{I&_Di}xE$U>$mD4h^Y1YX?yh4#S2VE2ZY^iusoRdU-3yV`yL_T?_7@aj z&Cuap&3e3wY(Ha<<_zv*4$rIQeG6Yn{iZu(iTsubhZ+O;E$x~RVFj=;UGI^o+d0-n#n)2R*RIz?PH>JdCb$0|Jb&AAK9|? z8O*=)7MxTcc_LXag{EU0d2W8oP)~3poT^2lE3QBb69)-Dez>|d`&^TZhJN@Qa_^Ml#F zW2Xi6@Er0nmP;m$l8LQ-G?}DUpyf}!k?gf0rc^B!y|MkvJ7e99>^He1?%phM*%Fnc1m^fWr1Wkte*ZO~a+asKLNaq+oz?slE}~I+!w#8DA1UTt`N%suvJQC^TRsGZmS~6oMG%MTf`flVMbo`%|ovWKJxDG>tp&$_M)1J zO(@9EgAp@bjBcM<#K_L^WVGW35G)=By_wo1C8>Zhte4}>m()ZhF3Zr7UsuR$`&3^4 z9C_sYJ)SuOZjgQV=8^#sYxKg5Fq6&wX#8tDV>*;Z29MJbIW}ef9;~A8G;laVDf-`a4YgmQIX+O0r$M*27`VND`=`!uV{j7ig0U zI7!u>h`#%iw?(^~R3?|e0`>E-SJn^xK0Sl~wA`GmN`6M(OllFSZ$nJG?0n>*wgOF3 zUdZ`y9NBc2L!>`PGZQ2Kc&+_Q(KdLFJh&%K4jh$0uTKhb(se=+SM4b->msn>kuVz7 zI)UQ#_A={LW}$Z-mw0_4Cg_LUY9i2VNu{tF|Ci2g=qc35Tnz1IvW{jWM=yvvo*zVC z5-&j)#R6W{r*s4qP2lTmy>K7tgA3{mQAmIb`t2KtR=b}e1y>G({5J?i#=Rm^ht1HT zMm;h*Y{B>(OCrn=KKfD^^Uf|y98 zH=skZmG&ZydL6WSX944S%ajoo7p&3raU>PpMxIX*(j9UO9*#T%C0=Yn2Nge%l)aUx zG5t4Bwe2ljwYrHs)2?K`UUXsxyc@}x)OXNK<}6uocQ#6i`Ud3!T=)wL7m{Bt4D+%@ z3|%ciFjn^rlH5~DHt6efvQI4G+YLn^6YUD}Zu1?Y_OBN$ObkF<)EfmgjF|*&+sE`- z8}sDF?xN-Ix&@ze2zh&MMxTPt5n=8$uvRTcVJE*bUD>*xx#U$B3`v8%bF(o1=dKi_H?|Dm>A7W1DuS5g-=46kBCd$<>M3di|k(cxtw9w%$ z;;g$!?D}j_&Cm?g^;(9Xa6gJ{H~9pE=RF{HJB8RmQ~iP-$@9$RhnXZf@(_Glo<~?w z3Y=pBSBcZ{eDvLXl4oib3(qiSO!Tl8%6qj3F4}XP7>mfE&#mWq23ApIeCu9j`qpve zE6|Yo9AaUoz7p99(oueGJ$6@kh~^9%q0d^U8L-|0g(PiakMoYAu0|yivhoJQ6R$wm zo4%s>(LAEF;TRb&Uq~*0je%b3HAuVnl^{Rd4{z(*5RrGUk*nk*W^a%rhvt8Sng_MX zv_>|WTUC$3&cEViNM;i8q~j>DP>6qXsT%QH|B2MLi;>9r1I!!g1k~{HCP}(n&z#&? zj^MNha_ZCPj5oP5e~QwW?d4~A0^J?O(MQOT-W>SY>LSlP?+MY>c@M#+HmKk^4=s{2 z;46;(ggR68NEY-lT}8D_fI>c+?RJ3dvCe=8?Db%y$t-lEwi;R2=#YimrHR?<<)qbc zKHq#t9$dj+LB`~xP-toclf9}0CQIH!qZ%g&TuQ+oCs*)ZtHz-_3PupR%;I=yx*?Ik zZ|t?dJ~RC?|Cm80jC>FZW*%9Wqy7SAbmo9J1KSFC&uB4pep-ylJC%=e*VmH7tEpsF z&Jo~P{fuO&w=u;EYe>kfGE&)Pho;@{u&pnbFcXpW=yKmFw8+l~t^3l=SabS`==cfr zsn-RzYgCc9uMd$M(pe;AYXh_9svh|la-5ho@Dz}5wbhvi;7STB=3F_Lp>m< zTOUJzPI{0(9;#%caV)7lZNX7uAY-d2%h8vS;nfYlVVAwz3QarvN!{_eL{>2W_T^45 z`fX)McH5W2U+MYiXHh@w7d=Mu9WMaIQ+Lr3R|RHx-6vkeb9-cM*G?2mb`Zs>oy?bx zah^uN9L}~qO)y|z8t`p=$+&EJ#W-@d!&SLg1oKNxM6=tBtgIK~w|~9GJ8|=fpdKcL z;4L$B);xn$zq&|n?93qhSmK-==ai981rJ_SmnSAF3W%33!zrj}L62i%A<;VmUAF6? zkOSiUwccZ}P&by;T~Q~I_k0Ku%>HGf1~lARz?;uL$y5higbjhw=whilc`9m4A^`)3 z?l{1IGos1!?d!<1iSN8S=KGm%W_hT&NRjjTqZ=B1Y|3ZG_mjPy9P}n55I#yNV9M>g znDF`qoN>ABC{0gO&~rVHJw8>z%s|-!ZCVtnvM)1-MP9<`TW8VZeG2@s)mz}UqFf{v zK?v8zi}bI(Kz_x{WzR&8s8Gv|cWl>hV9EtmLo#`AdMq8~E$e@-zSuoYj>=Wc< zoJc|bY3hm+?S0Vi{em;UWe;q+DT`kCJ2Q;+J(S>dfVW1Vp`DNthKt!R(4y7tFs{3f z_aWyd`#VpZOg7GhJKmO{OC|Y8D@uX?>6;`8mvJPYPMI?NFC17ZdInxQ-;Ej++8D$3 zB=}!Q0@}L%Df7pZ13Q)v2;PIODB@K*2@FUl3vUSeEr&nBktspHo4hbT{?{Aw`tK~1 z?mU2#g^rPsehZ%T>j=^>HpvL}KW5v|Rpv<_u$ak%GS}E0~`6&m`!%2wLQsLtGy!lE8!{=+%6JmFjXjXZlH&G^J=g<2vd9Mx$Y%U}2^V}KV z;z-6YRSaP-8NRysW4P_{Q?w(^6oqC(G-9Dl^y6~juG3!VtehkobxtB4OBFeGN9U1t zuU_8psSxNQSBuUsi6!N^iO6ka7LqNzO|CsqCjVuG@Qh0efJK5eGxFPo>|T2jskA=h z>E?*=UZqGezlV=7%fIHLIu=bbnPsFne-B$zP!G)sjU)fbq>>iBCFJJKAtcqMHPB)ct=ED3VvZQJn=Ce*Bl=}ic&{&0lZ+L?_mu**rBi$GVs z?!uF~!9}x{T_XI}G&DJ2jXau?(O8HC+B>q2h+o|drS}9dqA%AIjc+gU6920($XtZ4 z)2+^{A6!XlTrTsJ&;DV)Ez;oS=*)y^M;M~>`xE>7OD%GzM~I9~RKUn64@UP!GWqBf zf}|colJ{O5b(l{w+PY$>p??jzC!aw+)G3k+3a8PLBl8e^!$Vv51i@U7H>AVy45254 zIioWo(EdZ4kLbDzo*RIi|7F11)DR-M z@CuWkTF-y^+@8`(RxEiK@#SOSefiV02z9mOLK4aDjZG)-3C(*@V zK@8jS1!?&y^UBU9ky~B*OvlxeV}=B-uS`iEe3?p&2DH zNKAPKtm)}PGj_BSHq#B4S-l6L6P-+$#9TtkY{}~bhe@qmG}>tY5MGllgBSO;ky6q? z2CTLrGB^l-Z@oYah9yz^{yJFJBZ^l0#gJazW#rT+4ypN^Pi#68;Zw2GOlMyLV`cXi zGF+OhouN%8G#qi~+#tMlw<{QX=Zg#cMY!8WKhU8e;(+v=pu-%CacEE$PMx(2j2_6u zZGMaJu!jl@#iTcX5qS2@&ALw=7++4U-#Wx~m7YP(=Jc>^hNN)1uqF_;QNZ#;0uP?+ z8er}c1AJNf4wdct&2~3>KyApbrL$KZqbjO|sO8yZl)l^hgDhA?Xwh zzNLVqCR1E>Z5Hl2+d(}zricaJ0XY2SIPEF)i25~m9xjxahiA$N8cKp3>FD-o+PIsd z#s9X`B9RZN==&k`it`54fej(l&6msQ1Zn{_UtI?$?S&Xru|Z_J6*ij~q|Z0TQgu(? zarfN4&0Sqr2-xmLKrl~=uNvfm4_k65+xg}|U)l&8Oh^N*^-I9Aqe)=9(N3(Nybk<& z*-0HflLWRs$-=+tJitJS9bL_;r`4t}P+M*9(M7Ro)Vwia)@b`5?(FoL)TWdJR*bne zm?8!Oo{IsHas&W41cCzF5ZvAG1GFD%;?qX3K_dn?JlX>EKIKs}pYSQ~f-buC zP7SqfXC`&SeHSfm6-)hA>fn=!BWEts?_rdg$vVZhk;s(w!1{B z9llN}UfxAN_IN}w2NkH9_jN$iA~h_AyC_A&&3MJl5&;vv1U%`^!i&{;q8xX_bYR3UF#p%jt3_LG36AnikFC^wbDW2%C#VR$7*m- z+7s_dS`A7Q)L936M`(@HJ~Yk{4T=(>sT3KPFz` zI)tsIgRW`Xb~e7?ZWV>x*3;LxkDgZveDa>ttv=3_!s*BKx^Yau*2|=>88y;WbT$1) z`U<^SOac@YJfnATQt=MQbeu1q3VekUahO~@?%2B#=OqaES}~y*cO+xiwXt~fn?!(` zHsD3!_o>g61aNDX#{cE31HW-M{P(Iot_{)zK8HDYpN|K%;@^H&pW$peJxPM4d-fS6 zaT|cV1P@%GXA4S-%&-7&hO1h1LF5HJ+;UjJdDUMIlK0wRr^9Se^1&ZBZCi{hwrmBO z!W(e=#2cC$|Ap>hCaKbo(=<-pi0zV`@T;?VKqNFB|5*Qpt_-Q7w#KM|)@`$J`ZqJI z{dp0-e|ZCtw_1tM8+^0vk1C=h^$J)VRKZ={qJy8b&%}BQzEWvh1)k7ZTGaW9gSNgs zD%>H7ZkzA0i~_TFQ!6LUSysQID67ctw7T*|*0IpT)L6qo`k`itEflY$Y!aJn_ZvlW ze{alXX`Yd#FT4(+SMe@UMNVg_gD2yuh@<_qXZkB@?hilevVAMXe`SmvIyHgLw@AEW zp($SBWeAqat;Jiu7Xt0v5D@-78>?+e18sj|!BJHMAaqgi--g$NzWfE4GptFOq+g-* z+0a<7`7mA+w+~Fu2nWLz(%^RFLVT#x1}J_D2Je-$fc$7UZod)>5?U@? zd*&(!R<^`*-x?kkFr{j^W`oyRe=iqP7dqC_#?ODyYXZCIsM}6p1=k)+{kI*PpY+B; z3F+YSy;STz;Kn`vSOtsRsiNBj{_;CUm*EnqkN2+i2L~HGuw}P1cs?r_4Ck5Ptp{U3 ze&8u?tU)v7I917g?Mrcg|I5K+)yDV`F90;fgyX5pA^4eT82&Vr0qktDa1f*13_)Y#0{dhS7JOc}SPDq$S=j>S@qWo#KvU2LMO8{P)afud_u>hIFZ2b9X z59{*bT9(4&Ue?>)54qms2k-*!W*k57FbLjUf*WTiQ18OhsINO4=*sLO>SviRuCVgM zS9M)MRu2cu3$4USmz40fu@xXt#RK0HT>>5jK@gf}g&(F;K>cVu74xB&T1c^JcB?30 zRisiSKlV|#HfGXsKF6s8_jJHBmK~tS7U7EV*&zEsHfYJq1;bCWaf)&lh>*QVAHC8` zA3q;RiT+(f>!xhM`@{TjlwS!@^2@_rbS@R(ssdi*i3xZTCR9uB79i%63^ZIbv5U-R zAd#X7hW95?HL4%!%%@kW904z5k;f3NF~|W118jWxx*jNLx+dUE`C#$kr9j8Vox+D? zS&9K!th-<5QwC-IbolE**5d4TYUP+RW+_R6zu`Qp;pbD@EgXTQDi_MPxr0(M{zJzF z%Y(1Uu0ZpJCV04RGu{-H4i?;3W(Dt80+WwdQGwq1^j#HOe13BRy?ELW7`Db>SI%cT zbbv=!58Hr!$CuzpwkW9o<^%A{06OB92of!NVKz*}Sk&Q*>D7H56&BIzDFu_hlhIpyP@+2854 zuJ37#H^%85rzh`#9(JmBw zpV|SWv3v8(E4wLG&(h|>h}l%p1O?7+6$e*^#qeab1qd^C#%ns&@rpxPAkASX-tq4& zy){=9r~D^Q+e$vOVRy~|v7)ZjNvi-nW$z1;B9?=n18=D%0w2e(jSE0_?n!#|Rw(W5 z+D>U!OXB)oL-16UVeP#Pv5os;U}6!4(FSikx+4p42QzSho(#oqy+EILY@xxn`BbH| z0oYGWfWLntZB;J?Sem@%=`?4$Rlve&{@6y_OqGJ4{u@E;>tY;{Q~;`ClCX?cIIe%0 z4FadrF{|n=Rrz5XWqTB2T~|vmO0Nfmt~*m?l_=2J6g`mnbCN~5!4mQWmA zN*|oBL0>w@VsV6I=}o7U@a)+w+?t6mR7!~ocAM{v=LT`GcTWN^wTQ(kDV`v7*CXn` zg{$$y#4KQacP))}t>-#W7i{}&ZZw0F}5dQYnyC~aw?t*7;{YMTspxn}_6 z*tYn@QAp`V%%byZift$9+U6_Yxm5XwC$=(|W$`pmV6YHxg8RE2>Dn)e;Kt@mywNrh zyti}%<)S56Xgm){ve#o{v30mFEER+&#bNP^JG5fhEIP%cj?!9CLJJC)6nOWY%0qs% zv(*zSP3i}o_PURrv-vyakRgqSQ>Adr$2W9R_kGrWM}{^rdq|Z<{-P9}x6@YIr4*yR z2K1c`0u2{;VJW2~P!a#0dm-I}x>wS}ec6#q^`7$)pe3GAE59~1(;aoTa~dnKxnn-g z|4;$E)GG0zzgxhQdIY}9gkaHGpePZVdT5O_`GBP?y zcMH~!lY!{mel;*vB!k=jZ3mxjM}eB~EbPQh1dW#@K!d_VitlEC+e*#AuyS*A?%8im z$DZ>jAG=F5t6>E$@>Iha#@4{*u06ID_~D)QYoxhKmI5FD88}$Lh|J%#7+9b{9IEdC z7Q7gtXHJOXL7x&TY`qBXeoIM0NzBj(`UoeQ*4goH91$b9U1{jm#(C6gl zHP`m=>7|N_Ah^_q4l5qBneye(u76Ef>fNjHSB*&gcWMXN?w5;qb;VJ4qffDF1q^Mq zEh6;46TTq6QWZo>=io;M86eVl3vm3n2FNWfz|`Mt(5IaOM2>Ml(yb8e78eWj-Iq{L zzwc>YW@t>kxg$nfF1kd&&fIJZq@Gbp$FI^~B}4JnwM%hafg0HQV+F2J8lo-Kp3zdd z>Ojar9S6T=f#CE_%@4;E@XZN*kYn(N60=hVq9t`_$Qr z@C-pwvnYtk;$VFnOYls-IN2c^2ApPpECU$B=4=;U*<;J_UU zOU&q`CszJPAGkEgA zB=+(wp>K!0qECq>QI5*X>G4M~nCIk-Km6JTLW3ePMag0H340PThqMOYae6V1krRp}shSqP9xR|ZNWDOj{A1O#NefM(HI;Kb)3 z>?eqQtNuDs>fQfQUi}I5p6h}|?K(=$_A>{!0}QcLygg`ailP#Gq^PAoPSUAoo2k;T z)_9+jiJ*I3a86wf!bhV_fR2VE&|i^;T|T7(pPmt_^!+R#F;gDnB~n2Bt29`lEKX7Q0P9q*n2=D7!hAXxX!+)VqD*II_zdkEy$XnvH9* z`Rz!M%(Vy6v*U5%>Mfw3w+x?r?SOe93&67;8~k0*lJ(MI4r`>Xl#cioP5m=jjYS*b{b=yn@QmN+92QHcr&#I9K53mZiKz26mtff zMt^6Sll%0C!ei8`Er00L--&qpk3_7@$_6WV(fDOa zGLFBq8M|M|1e$?~_)Jv|E4Sbm%b%r5o$VBDK4SHrb-?lj%g4rvVmB_M3kyoH_w!;b z*RT&98qCH1JvG#_LrGNM_J>q#P$-=uwGc11x5K%!T);9lS)3QYh`zD^DE;*h^UE%k=j_u~NTvBFv^f}2UdT7QK)?&}6*l|sP}|5$8Z5d>Thn_~Sb zacm)ogS*OuvBJ7ixXSD)+O)R8EzSq-h9J*NBO6So=g6wOZ z-x3NI8-j#OMp$UqT+k7z0zUnng@fcY!5NMp4!PuqKbL!h)@BcZR^y1Z)joKNp*(9O^u2ck+flB{fA| zS=UKN1lVHJKZf|{fGarHR?J0$Bx{>(G4QD(jKLzY}jl`QC zM%X-kDakTB(b>H31ftgNZDYN?zmA(!GS~KgzzbFq2mtMOBESYqBm6*t0>^mkfYs`7 zFeQ-FDe3J_qT$c>7776C>cMG3IWM2_o)=S>r}(3S@iyx ziPd5Ksl6CrIYX?OZh==^6~TL&769Gb zYB;}92)9M4;=Hx})Ver%ym`tWH#U3WAFBSKjlBv#sC-T@XGHMVT|JbMd>cJ=Yyq|N z=Py>iusJ=|;zx}DouUo7G{6X$y`eN2EX+M0|(+qz+|1 zW5)_?=TZn(dTFpE{U6(|t`2D4IOfGw_4CJ%=54_XTegB=El0fV0fj>}^zcBFE$~e< z!?Tts;8a;v%&k`i9x+O|CN>F-2yz&m=v4gUW*iW8T#4&{BHZP}0&|S4@hlm?rm^BR zT+h=@to|)oTxGj5Aogi5@VQljCq43joAW11E2o{JJ6_Ul>+VuthYbI|f+9+tW9fRhgaTyx9` z7~$o3Ao4#f5p)2X3?2p}r%UncPy6t%%u*b5ZZD82%)!4oWAyaj_jH2yDJpw;H*L)O zk2))Pk~$;QP1~!rQCGVHfpCaGhwKyhytnuOA?E_XSI7r96$@~aZ94eV|CTmf*g=2U zBw%K-|I$xKx+%xYiYU=W=4`Q1csE8tNsW5KjG%Lk9ydE-Ym zzJmLNBX)?`32^9kASs%G<6mt7>gP{V38v?$?Td2hKUHOv-i=^zN<0i0{|&*}1*<_- ziz+^-y_)82S_O9c_~WmP8?D+^!@cb^K>7X?#{AhE@V}-_*yOV<=++6tL+ix_yc0v) z{aqg{=KFl(u2LSQZ*Eq@#l{w(u9L#%+hr)$({akDF$=f#CxRN~R@!iiXCoXX@H-Zo zqID+ISh;1kR9kB{H}>IjmREE!)mbG4e3G4T<_dS<;*kT+d4_`hKPVjccq<4oy-h1@ z8l=T-l)$Yf1)Q*cjJCTGiW9cvfPya>*a=;vR34~;c9S5i^e+y~-R%y(ER_eo>F)So zX&j)xUZp}Xrv4k{;1X#waBH*(gU<)Clvoy^b@MUiq=4LG$zVaJK%4f=2I>q$8z|f2 z%_bCpg0pfLKa5r?Q^DieL)6&09D1XF5Rh}S0PLqm__MSpa432~J$bu<3QbnSMb|Y! zElU;*c)Nl?e|v0@34wupI*!xN!=7&Wpm1zA_9&;Y(3Q3Lw{kjY7vG4-L)L?Ab_6J1 zI7rJ5$$)n!B=O6u$+V@BE%nr{iark9z;LfK2tA*JpDS(!fh9hmNbv4t_4s4ZmI{>P z#BtpV8~Wart8~@X97;{s8^k_w0>7@O;@_8D0KGjLjEK5|zqiw|n`R(jtM~$!tt>Dj z!vlxxSOdat7J^?}bHVnAeOOJh7z_oSr9vJ#(_$h9p!Ki@{&s&BW`&qyksv*AM#>z2 zxeC&z^pLW0FO6t52akFwZ&RD(=9J!f|V+v>DDfpT0Ymi{cHazG4)3wHL^%mi0?wf$v^$@(IRROOR z@#d}>52P;Iw$q2pK|UiYU%r?jfRNEFwdQLaCIo5}D_*5TX<+nw7YFxkr))O`4~#G-^&M z&BJ&8f%|x_d-hrTUBCBzq4}*1&Xn+gJj26ORqFu#`Z55LCs^XNMpNSbSCV*+j$IiZmMtq;Vy<3USDuzoomY&;gBb*VFOjx)*4JAEXay-7cTv^uEP0y5HE78L0VgRqtYg<=%2A*4jvlyKd8_4Fcid-kl`t zzY@CFbu#fVu>k{x6>vq_5fe|W0+Z|o^j2XW{W$k6ITO)C)h=YwU(Zf5R&~D^8%;_4 z=QJ$z&+((QWE6~A?1L9%o(r?}Rj9aXAALJV7o;8j3QzW%lO67lm=6=AFeOIRZgQc+ z>dfVR^ltS#YHD0s$TU5o`OXiBX#RNkyeXYrTb@ruB^p?DY&wY{2&>mcnGmyTnIL91c0j z;(>8~#QvEUJZ|IraZZ-eSCb#mkA}wBIkca)&jFlDf^hB~fOy~cP*W`84K~)N``WwL2-U?>j z+7|l$?Nu7|(+^th1GG%_K&C9H3v1Ym)sVnZoDP@v&R@>L6N_~+amsMC2cVa>781Ae4`Ue$0sU-sZ=f*d#s=K>rV!iPk~q> z(MU~iT&383x9w2%LwZ45)#|)_{n)Y3lZ5olYJM$?*gq4h>Mv$*{uohvp(zv)Z^#e^Dj;e=9%j-2XnBHMJQofO<2j=kojx%gcSc#w3>>=rs&Xb^z)-dDz3S1w5Lx>Hw z!Zo`?gj*MU5}Lh9q)(j>(L#&)L^-#Bl5P*!FYgJFD}wRE#wGA&@j03uI37bBUXaNL z|I)#y!IoiaDpY2j7Ih8T%lMZ0pw}@5i@n?-Xi^5=+r+TMUS6Zo&!mnW(Vb zAKQ$Q;Qgo+RIA)dMlY=)j@dQze4iW1^$o$|=5YQEZ4AR&&PYSP2@kLFqZJJtbHy-U zc+sPd2IQ2{{5L1b`niYcAHHYL_|Z+_oxo}`aSPB-*X3|+yR z;pO>t*tjqiWYXnvpy>tQ74@B*eyf1@ynD%*rh2ObmA&*uR~AX@a6|3gV=(ZAC$OtL z@zLW@Vs6}OCHK?A>f2;FVbq}%G%8w+lIvE&2hC{IeC|V_Xfo+{ETZOPc@2VD30nDH z@M`sDzDH>hyxZ(Wez<3mn3)E2|IE=$;NEz=d3X!1t=bHE?rC^ZAdPotzo6e_Cc%#v zI{4dhB4o?(=an@<8xLC;+A8 z=r|2SkhV6)jSiaVUZaNtrzV5_Bwc*ShC=hwK$vF`h3XG58z9%)bMWfUzmXuok5xqEP|@TB-Oy6;8~ zneJOh_cdJ;Hnn^g+9oJ5-z+B4zbD3^bFdiZ8jOao1C#jq!-!^0OQGNTmI+hx{tB-z z$j9$1qcHw+0jTXNz^tSBxU@7Eow`ckcu^9{HM_&4$11Qq#sN3S2Y^MiBPyI%#3Saa zJmX=C&mEj$<#aQ6dWuCJ427%9OYuxV0nfRVLTF3@{+U)tyYvUB;O8gCA)-h4>`Wx6 z`Ys36*_+UF75~0UxJ_)fH;{X_-Soz_Gh{@}3qp-k$S9Kov{XEgX!F{r@+&PY-$u!R z$vs-V=skIUP98qQy`zEmUX$f2pXiFO|L9(?K9cb2Ih}hbl_ZxgC(*u1bSvK#)YN{L zv6C)lCQhD570x-3DlJ=3D`dew+W~)@09*(eg+q}~skM_h2*d{H+v+g1916#X^~>N| zpcfuA--SB6ccNE!JzQ0)!0yrsc-%`KH@GQ+bLLnqezy_wHz$BD&qg#QErX+T)S0E> zE><*$SeA3Sk+l`j@;3*!41U2yq{vGmdhce*;b=A_-b zneAOcE+9VL51b(!Z4x}7W8@WjU#FEWG`d2%weHd57DWJYS+Fl*8*Y=#gMv4E$-lx} z;_@hp+MBH+n`&dxz%3R}{oV{xk5{5xk2KLpoXOap`z%CNa}p!|i%1P3>GG9C16AGu zHTfY8P<%t5*bWlW`ZF~7s4a|hbb+#3FO-`m0M&v8utv=bj!8LSa<(n(zwe8K{O2nE zSq<`We(1c)8&voC!NRUk)cxoNfB)D}$JudoP^p61E2%`UwkeX!A}6BrSDyNu@*#b7 z8Q5nYkFh~ppw~DaZ;6@E4cCHc(}!r%=2Jj7Ln&CKZiQ17W%#%u8&bBYpsn^F`fB17 z823aM1@~v+1Z_in{+5Bx9fqh(4Tb(|ORajXO^A{=&~c}$$YzyAf-O z6@f3>W5=hFQ&Owx+Ws6IpOt}AOtayv(kAp>k$^3-8<4I_hdSF>JYHT+$JCV5#xwP# zZdN0mceQ{%c3Vu%eGU=zG@$yPLP;e#SvCtK7MKRF0r~*=f-?c5NL}SAunDtQ`Z~JItqm>e5#H-?@ zzayYEVk9hMXYj7VT7AFnyRsIz{`Y$w;yGK%ay4_zCP(~ zKg{S#ULhr|F{q|K3#@EAg>@r~P=^0b{=6+fk?~k~r)3Bgm3Qd+!3X5PfC(9;H`sp2 zHCcG*?KLv?+-TgaeSnM)uofCTzd$|C7SY<-BVliw8_&Y{f?NZyDb+0nlb4=o*QE^2 z{JiLrtOCD|8{=_>Na)`ChZuc-NQ2unK{fFa9l2&VE!19&6TVG>GdGm+E6*&%e+z2O8LUz!r|4D4>V8MPf={8Ccj9 zVy63L#_FFcG|0cFvM;sZV#7D#+o4ZPYo7u}I6ZxLD zs6C+Hc8Q^Eiwa@??4+sUdo5pgtHRi@-{johXTm!wGs(F;ex{X^#n;uU@U3q;9YgBssT!0E0Ta7vJkD<52?$A(?-!T1<(3iZdh z_%0?jT7XZ~H!vFqi>yjGZTQ%Bg6Q^l(Z$Sk@M&+LKU@nb1z8Mq$|l182HNqbgjOAB zWTr|r)B1;pXzwEs9swg}>V`rZcEyp|`Ai8`{}+L{NDU@L`VhTh6|{RG1NYb8q0LkK zX{^#2TIE;3XGY(mm173zYT;(G)cz_X9$h26Y+_0!eB|(yM+qqiXr!mNY6{oFHYRuM z7<}~i1Ig8t#>Af{m@IXf(Hia&N}KgkuC<6>+p&`f+hlO%BXu$&?*a|FnFJqvqOjdJ zg&ET`3%2wg6NYSfPTu~sBFlTKXgYr%_@#QEgl;`XckoQ+`U%;X^$s8`bwBBjNrt_T z)A6t{9LCK&PyX7yBL6`S&dXZ|P2J~7R{cAYx3ZVM8~Kd%S=hqxuURnU=8k<69pLxQ zR3f`}6VX;aK=aaf6AxY+U+!at{&Sr`e6|g;W|JUW%m^mvkHEXrb>U~3IJoYUgyu67 z(Xm7xw$7`xy3l6GJmtGg<%_Sh&zWNa)f;ESG6OSwJTL=%e->bqMlM=hD&VoQRO~P? z#U#Ft8LXMa@fP82@D}8d+At z#Q$<|R$4ZZ_N8Ra%6ehfu8Cv~&nu7Bo`XAN|B_#;b#bALG5o4D0_ZV88F?evZ_R5@ z6;*I8kJtX5<-(l_LK5s4LmE~#Q@=fwq!w2Z`B@|2K&?F8bla1}eNIQes$@K2z6ENG z5^xYS>5jF2)JJtBNt=;NAN{EiJ|D49Y22|w$eu}~ zzJpWf_xtmS+=?iA@9GWV>J!m4XXq6(d%3IdS}?z-2up=&nko3~Mk;)cUrBU2hlmT4 zPmP&XMDxE)@L!Y%Ej-7fx*-irc7~FR_pKSGaudy7r%P0vtuf@`VpN@K3rVBw@yxxK z^ixdjU9YbvD-5rGjJ`z;=b%Fd@ho&q&z92E$R9_wEU;mtSiLTUr+VAa_(u->8&L}C7q-Fc$RfOQI~!Uin1Q$h z01cjr8U8cjxdYEYD>ChyzNV3f&29A6_YCB^^RT)&i|>PN?Pg}Eg z&&6vzOdUj(q3q>U__KEehG;5*noB%gTi{H!nE52rWdvP2eJ!5-upH}UBA~D#5|lj+New~i~_J0X}v-`+)Vv=o)`vLWMF>EpD28rWy8 z4g-B!ICftjBl}H;EI9VM{qn_FGWFg~YIpfRYM`~7MEUnq@8vs$wjrZw$0u8I=)J#i z?0!p}aM%DDJrh{Bn%78UG%&w(JpM?U2;RDral#2lNPl1ne^=<@2XjZ5Jxh&zDp){_ zlXYloWCnQ~HW5yZ9}nK&weZP|$q>4B931hJf=##O(brxAN`|N4xt^&w{~3RW(9y?v zW|Qcz*9AiDi?Tu+^#UPX`H0a~ZeV&I+0(=Mio{-1621(LhTE;fv~kUAa&Kl5jJDmx z2Ogy$8?YY6zPMraPp`54W@@hRah@-ud1yM>Ip9jRPBy1&6z(xrj4U?trE$MRR>Hq{29j&nx4=Oaa-Q|F@g#BHhI^(_td7-V3-PBcVD%mc$; z3@qT!Y|RvaS#RxNLazfXa z+utH`0Pm7ZvX|(p&qv4_uT-WlY?~ESO{V%6?=bNz$B=j?n+&Ipr+Wi~NZliSVtCJm z`O}|E{YM9pdFlKeIXVREW~_y?To9fwsDR&C4iXKOc>Wj9CeL0EZb>l^Rh3Z3l~ArS$cOEIQ-ZcJlJiL7M8j6yKM+q4bGx zX#3%X=jO+tvTg#dbdP~k4w5apxsqlEF=Xp=Z92beJH6%IO6BtXh;DK^b$CCW zXG;a-&b|~{4bEh5Krl?J3V?j`<@jVo06c3AB$^%W)_A+bs>L($GM-2b%#p1&@_chQ*;r=4*Hlp%Bq_Yj6(3t%MbPYcsV8Ng~r9o$N_vHyn_v}~VB=S~r$)23$- zgL#p3*Zw=gG3VwBn`+G(q_W03oJHAYLmm?<;Yp%nD#wHk9_=_ zDXdPap!H?hwDGfmY}$c^1+h>MbGo8`5YBs9zbQ-4UK!D?-&Ej32d z?F&(3p*=>m&ja-XPFOW#8mMPHA(wN+aB-s!T<4_0S2>fYZ#P0uWgU2FY!7xp6X20{ z06IT(2Sb?*Y!_EW*~(lvlaP)50iWnraaS7n%M)()SmVJZ`gl`wD!!d10n__uVp(bu zdR;igv(^{;%Z;hV$Yr$&47W9jMNzU<$cjxW= z{K0o#ZxsEe+SfF3?^AtxqN0i>jm&|JiR+*)It|AJWkdeA*VL|KH+?HF4adKIr7qHy zOxK1^a%tsxqWUFI7;Cp2?UVTa=!8`Gx-1H<{6cY-z7(piUjei-5QDr+X?V$3VeR!D zBzHRJN$pm}sVDip+YJ-&nKvIVu6{(0 z20tU($BfZLPZ1QW%h6Ic7pFei2JR0F5IrK$dx|#3R%Jjpvjq#c$76KOV!VDf5i%FV zVChHx&T?BFR67(ga*{4o+`2)xpU9z#(I(L9YK#j0VQ{a>6!z;Z$1m@Kz;uxrU)@T2s9`P!mpkRv)6~iPTm6By9 zYDn}QU|=k-5tM|J;TzhxYchd@@iT?~zIRE0nzM0P(_oo+1Q6R4ymu!SdM=>l zqr3k|?)~|AFT)pN!atLk2vbrx`=W3f#*yR=Nw~Q=7u({C;FV4RHoR{n?(*CC zO|lPtZt|PFPyNQc)a#;RQ_hjfQ4YYz?vPVf`>CI)1oS$ZK$?6jc|3a*%G}Ta50L_7 zJ;((8-YsZ%AQMI#6bRM(PYboDwh343z7h(*UM3ovqhV{CEgpXp44U(riP7{iz}zOn zt@FJ}iTybe`EaRlV3H*r__muY6Cb2iU8_-de;BNf4aBo2&QL~-bX=ZK4R0lo!)Wd`3_Ix4UQ%_U7&>2?O-#lbWZr&$0h7fQJGwl-XY z)zo~lzNJQs9C^%W(RI32bo82LVM9wKH9qi;Np?5JclFlTJ!u*^Jv777eak6Zt;A?# zSJG=AI5K!c0cSR^!OcN|FyAc+olOJb^@2E<=bVDuZPVcGmgCHKJKooJ^c!PbXxQfU z`J?d8JsW0Ba~GAd8U^`cn$$Bq2w%*N0E>*L1ud{RovZ2 zipCsfhPv%&iJBrathPazak+<1Eg7;*KEsjBXMw_^TS)WU^s)L)65M+jg8mitLOJpO2 zO0vo}iq0P2NXJwqlSSGY^xFMP^t2m;**%fasIvj9yNqb1@+g?t(L=vDii2qWZCWn< zk#2n&P0ZYX314+OlC}FJKyhvup3{khlm$nqOG5!c9D!XI)%d-{D7+>9l{Q{_NFEJK zAQ=gG`+gK|b8S$-*@?qHjlHDLVSp-Kx<^hf$RY>SB%!yTd-`sZ@nW9|BB z-X}9SE;$Ol-&(=JFo0yW_9LDO?Nw%xs^^y2utXPao2_B!w2in}w~#K0bM(Tc8sb;)i?z89_~3Xj zoU-)9pJFfQN6`Qs<@JtySoI%uFuZED_EG~=?ej@^O}EX;t9%+BRGf@ux28bHc0D|@ zCkp#j6LG}o^+0wm#|Sq=e7eODJ>^W{8m~7Prf5O4qdI7<;=RhZRUmrATe`&PGtIp@ zOwvtysavd&RM^}k;Z{N_8-Ia#$L%L6i&}_dQy*1HXs(A;-awCH358IhU3r)BN#lP0dX^@q4oHw(BYT~ zy>rvy&ca;uezXzJozq0&VNDdXo&xs^wDH2+MQ9H$sAlUATASwK?YbAj{gEGpzE)=$ zrx$01QIC9R0yBfEA6QCKBxg{w>~i?Gvb&x&dI^Uo1D=?-5*>)G z+vv<)nIyeuC;gz12%olW0;LOS__ca9l*`^EmvnECgMGbJZQ&6zt~!G5ogG5;eA3CM z4hLFXmrYX}Hq)e(Tr$wTl8#UaCs&J-Ny6We^yflfGT!$;nk@64&fjo{7f+Sm_+~IK}hV{m+SwPCix9RG?`3jfszNL~$(#aOVzEI~Ome zb5*@?-`gd0@Y81DM#;M*H$)v3m-6$F>q&MZu~ducUu;(KbMJq8|_n&kZHWy;?UQ@@DIq&eX_-8L+P_l@?omo~W(os>-Sk8Ni1l1V;Rv3RGr<=(|1fph8>s8~NLqh;;0FFJTct`RxzyJL~)*Mo# zW~RxO^W+xe*H$T9P?rQ>vsd7pdv}P;=yFTP6NhNN+zYZ-%K*&;CG^@pH_%U9hWd*K zX<}*sbJ!z|rrMS;qvpJ1mOfZU6*evcr*?iH=~Yf%Z#_fwjthwUsWSR?t^v-Ow@9e^ zv&70)UXs~fWeHp3+##zt058nENOCuJ5za1&iCXZ3ERLIqRz;aaX`DEG+AWT#^qOS$ zFv3ezo911QAX1iXRKKiN82snHaK!D^WXYxR#CWVboYLzMn<9D+L1k7BfMbv2rDRlF1y$4fziG{3t@55{dxE8(Z;DILBY zLSvGOgu=;%42Xtli@6uG=uI6pza|N?!AFJP)`gLuPq&fe$^k1w5=HZNXj3_!LEE%i zlbJJZ5ft=$!rOuV*lt3MwPfhSzc-l7IX`I2rVq4J|2ui!Acpei#h}tr z3htMFrJZUrP!PP0sL6Meh3U0)$Jkv&NmdC%T4YemKm`spO~fbli6DBK0>M6sSUMsG z++N%!X~8eaQ@Ky{`H8b+P3lS_9Xo-zw)s>0jnjx(eiXghYD`~b7?ML`v#C=+IE=r# z0cv;0p#8CBa1jh>(A@cS^%iTg=lcnv{m%@%7Mz7|9df~-I}um-731hp#aJI+3hlPJ zcrCTo>frotMm_U@)t52-~vM}6G*pE^*Z9C|-3mtL&dN?3@Y27|Ta>g6Ib;z%i75Vx6( z=2;BeeoG^M#tD_6?@bz4D@y!R%IT?7MR+%m{sT6+f4WsL^lc<`*KAU_V+pZ-7DT&>#7V~_AKdiE9nT7V;6aEBo?n~+ z4avE%=Sd!hu1Eo`sTz@^W`Rj`EWhjcNS^I{MPS}@Y8Q5m?D;AN7Y{uqA4dPC zR(prYva|LOd3Qd1S+)>oHu7E~WpNnH`$4`>8;!MT1LXD#mBwJUKR@%g4$*}5xmXpKgOwlh zU{zuoh6QXSi>qu&h_MTG{XLBcwF_a1ULJVM7vLJNEN~cDgaxCV@ai;o*u202uPrd5 zezilwd;L?$^yF{C)7qXms>mAaXYty@8C!g;B&6Bf2kHFWPLj5+jVeA(gT=fTE%bOA zPU+tW2emE1ms+fF z+{!;A7TBm{j_0M-;lc@ZG|CwRzm&w_w1*9jx^EBfKPTa2KAYTYTs{n5OvAO3V`$WF zBl0*xjF_LurZUanNxi3tXk~1p(W3^5|6yIcH~uvpH7F0q-l<|ha5nE9lEAI&c#ry# z6qGZQhGByg;+r=DQ*R}cf#6)Mva}v6=NJw=v z$%@GrnCL7E;e~O^xNdhYStT)GRdl(UHW=v8v69ai%bUt%dm`OY_vPOqak(DEvK9fi~lS3oT{6I}_G|ag+8Y6Ndq4-t|&Nw?C$9_xzAGwvdSXhkD zWBDA8+j$TdpM@9OcppjTcJlM_2uy!*mmKA*u&BN*@k%M6+rM)}E*9W#$~&SzVKZL% z8x9LvbfEH9ILUZB3$vt5p?#M(?wX{;Gwu#B&d3M9hR=r0`{qMyp*Q*)yFy(pe?BJI zfS5A*P^nhO|hF2j6}&FHP12iksxIFu1jzCRyBWG|)B zKx`)G63cK@OddvrSMc*&Hp(e>kr@R-W}S;FwXXCcjhE{A66r4HSkGs2S?U3eKQE4l zPPm{>O&Q4;RY>n$n?d&c{7OA17}Hx12yuRJ)~b`=1Ag1=3d*${-LSNaC|UeG`Ls}r z>Wpe8rPm8#`JN>HeUgqA;yiD+S&zEf_O>@J+1b8X2Zc<$|`L5I4m~VGelVKq2pa>59R_?>~~` z1^Y?Ns+ri}GzA)FZN#A^Hu$6=7Pc(ffUSZD?LJb{z;-)>Ku&m z_F}1GW_wcn>crzC$UtQ$|v;@ak&aovayEIc^ZJ#E2hGk5-~DCM~R*}qk%d1 zzZ2tyHYok|C;f7CCB&SKKvrowNHpbkdS??qWpyfjCk>~q2*dQwEEwj`&!6uM!Eaz3 zIUviR(Me~J&YDDgJPoMY-4>zkz8%bgIS0s#7v`XU-hfyp-)SE%kVjbJNkxk0Fy^r> z3U3^t$`)&cwFh zg4euX@#(`@8u+P?yjV9(Pd^DIB~zP;neATsQ@Mu}cF57AuPy0?inBEGO9nCLXQIYe z>JTwTfIo}o!Ym_as9Y=yo$V*7jD`VBI5R+=KXW6-^Sn^rK?^biwh47S8p-3&kBOsH z6&0J;N#$CHnKqYyq-TZ_x?DGd{61YU;F*)e*kD-0XWY5@^b+0lMk|Fwtz=2pCLGpY zhdb8Jg)tVqcQWiA{b|@mT_S#vq;N?LKe-T(Eq25TtHm%!*BO`2tR))0QDok`JM{gq z6u_lrm=)oMTV#Tu&@>o_q&7mokvdon&cG+OB-`*_hZLBjbj+;)u<0)1fEH>XP3G2KGJ z<}rTko5)+D^+VO7=rfPG(Kn}yH0Qp;Cthu2fBj+WkB?)x3E8Jv=U{dG9I4D5^QjS? zI%6x^p;5rP`Ss$XIzxffQ(f-q^oN4<=4JXd5ZE}#gYl^80Mm2bl^$u_DO$k==>dStC}e{ zX5&1Y<5l;#rdL-)8Y4ciO%F~BuDy-pzR%C?5Ierex@6Kj!Q*aK8?)P{tfAR1L7r3u zXLM#A=lV0!W|69<^=T(>K}EVTXCBfZNYGSZOO7ZSUy+WGR{~s^k=96}cHFOWBta+XOvFPl>v}-e%W7 z^x_JYv$zN23`Jp0CY_%n0$FRx4P4_KX|7EFzQA~Fw{@?i1sm6}+D2l}3s&4~OozS4 z2k!fo^PO5L9uJe`rQnoW>ugK};A9m$YclNRd!=*@WVU6?m za{KhdxZ$HL`|`ytQO)z;0@b@i?C&+zyh=B|^On3A+Z%LQP?9;D?-a@F*bpew`RV0S zc8Xvp+wC%vQ=7Dh>v@^S+Dq~|^$kuPSC>5$bj2?27`NMnwNNt<{c_&RE%dbD6htEV_+BGu+cCK_TWdc%a_K7e zXNHgq)+plIPW@ol*RXtca=xJYbsXoGw3&MxGR^w$or|!%eWV~`(=(A_ULpJXteHT4 z)@yVfX~vzoJAsYjGS~!lOK#GUkdEV%o(YEE{^f4WEMS{1on|f04~ZVmBi!1b5}lUT z>6{&r=GC=k(_BrGMijhB#@q##VSbki*DyX;F=t#SpRxd z$X%VRBU<@Nn|mI0o6#Sf!iE_95UkA@6f8@e%B?u^RiHDviVMEJk~3>5;R>tDxJT>s zxUEtm*gn|Cy}Z9iv^f17J6G}qTNEtENvLIU=AYx)QM2E%taP(zhaSuK=G_$;z765Z zXZEos+4`b$kDhajq<;zeJ)$`MqZM3cuL3*I@FM%}sya8BLAL*PfN1Wr46fzQUp`+p zpY0#DS=6>qUUcN4xXqub&)|2TH2WpGiv1U{meq7yBe=Bmhv1v{Z_)925rS`jW^rN% zPP2MNU)XK>BG&AyNU$sFIBU=(+v%!&m(|rf$gWE{Y5g~6BR8}37B+_(aYsKzu@{TW z*)JxFY-qFuyX?;>?)ub1K^mW3-;}q?hx@R=G zkae5b6`3i5!?~A59@8Vag@wDgmD~Pd!RQz4hLum)ZjX;5M;Tv%txJID&$o%9-oTlH zg5{n9f5{B?OmsYZX?LBVEV5S=+MmYC&P^2jd@67KB+IyCt>Q9KzTIxYM~gk8I-j$w zd(={PtC%FaWov_I<01!ct=k>W;ly{g<>e@jK3U36TC2f=F6p|PV-n_Bm|H*l-_KePH%{5j96*P^DfS!}&$E1TYVhW)1H%1!$`vGc6$+*i#QCvmKYkxX(UQxIuR_ z_TI~_T&Yg8Xs3=DH$w1Npt9jDd$FxwpnGGR$mHS`(ZUQpn+s2)MP+mQJ9ZSW+(QQ*Y+m*Vw_dY2i_P9LzzHuG>xi{qKt0lU0?hvL_X&_^7h0cJ+2B<~nn`?^5ZfzNu#9>Kp94&MVwG zol?<`S-pbRmI^NE?gei1{8;u=$s(@Y_&9gOdMtOAN1KoHCQwCdd7Hu5X6p*2!}SDq6HmJ+^%JQT-vQ=+@-iX?8Df*oKeLz&U4_tK=yt*JBYblPnL_|q}giG zO8W}Yx5F$u*wQ4@Z_Xzvnu3S zx&!#gxsbE@bW<=`abFZUXE#^#Ba;35y;-DwPJ&wtUpVYj=C#mDLCfU|?!mNBZj@LIJ9kXFwg09tL483rcdVMvM~@p66xG{_RCl@yZfT}* z+Q&qK;(Jn^ZRwBMuJ;qTn#tyDr1^1cakF=#15(CZS?*!3V|7Z$kSoX5IY@C|R_a;n zC0K9+`bk{=Pj~kGD`WP9(nW5jUM#y!ie(#HcMGI7{5W&$&|Mx56w$OB^N(id!CuMo;Pzw7A!E zuZFd`tjcknBTqLeQwM{c1w~nCnH$QE-=2|F)MbOz<6I7t24=zOJ@#? zUOc=d@HL-Dck^Kk6Eq6^<#{WbiZiU?STI*Nbz`^c57!JrH#LD-!HlCBtUy zlNFfCia5LT)uJ0eUvcl%g9Lw%#B?D&tto77AA{|Xb*8Alnu4?@b&S*;kcey%1s2nO2O@5RhaQqj}L3+9Mk6-iHe^Z_b z7V>}V^69tPw+p=3w6vYvj@>LPczjAwAC}JAkJ~5Mu&9drF3J$N>6dbc{9M`bBa1n` zLUArlJc5mnJjF$7Y!+Q!qQ|{yk`O$28X%fgr^PnC~Pd+PMG9mol#@Qy2;gX*ngHY**1>#S@}y~RlZE{ z&P|qM_x=%tC0%Cs_q(%$ugf^^)j8Hm8*ho;U!5q5mDCnQwmxG0jH_8WzGFf4-8+%# zkP+AJVaJ;MGUGZWa@l4GDYAvk$jr)K8AaiJ&NY&hq)2IK7o{SllIpwv!h1Q- z`K|NY!^GwklY@nhbdS{?8gRP_|5?l>V`3#djIabEPCv#+RY~Z>dv~0)Z_+H~E@pe^ zX&mWL#0|;HR6Q@6wUjYJPm>0e*|eC3uBfB0x{K-1!30b;O5lCpqtFmKMuN`HrfPf> zR8wLjh6|gJ;D+HnbGJJ5RZ9T*Uep2)?!?z?!%B@!6hd|>S4~!RA9>AP{~D)P-d?%e9@BU+DvhR|AeQ( z;_tJ#?|*0GPvt2buNAX7uHP-zRy$znR`_bOI1Nar~@}vZ9xxJN!UMX3qGZq-2S#w+Mv0WR|HlF zIMWnEBNFh5W&}aiWuT zAKpz}g!|u#n9N;W#U?!!g#rIklIL#*&%?**+S6lHQ0X4-ook6->UOZU&WR8!>`Er? zU4c!_Hbi+y7h>~t*iR#C;7D0Ke9okVps^@p+fvGw+Zmy)ZvJ$Y!E|GLSRVUH0+GIRTYWQ++?u`&EUmp#DWnI`|I3H&5_PEzCG6;>*$4TuXFvD~dan<5X>8t_E zz8`qLIuia)NU{83YSi9CD|L+VjIllW8>kG~mD%LiARjb*%pp7C2ADpj*PvOj z5D)cM!@9Wt*obOR6fW^VT=I;0F$b}nOy&5P72%ft8d5lZ8iqC>Vv*ksTR#-SxeNL9 zwe<+mIaLb_Jiel6$X6UIs|2a!Y*eJxRBf|rqvQDrnlwEKjRrUH`pHrlsjFi6RL!2~ z?}&r4-C1<mJdU0->)ykZfH?#uTma7}~(WP7xez|4HS&0S~U6 zicveZfySy(G<_R_>(y_8v2_uRTq^`dw`+ipqk%G(+sJ%DmIj4Z5}z4`F#lLN^4$$a zg%&3ee8tu#_$P zu^)$Tz5xNZX(Tac3+`T5hg01W(BXC*sm?fte+(wsoeK@gC&QJDaOonH+s=}~A1m_v2vrP6PwvJJ9;s5GKo};}a1cHk)q)No%(PS8M^C>n$fc*NjlB!Amcbl+>dTTkDtBt zaQJ-c=5-Dps{RM*rZG%aX*8X-=M|9(Re(pIuOg?n7B()K#fbcP%;p-1arc&%;?<%Y zyu3REyk+@lSlE1yWbYV}-+u&C&pf1UI&XOx{wL`XO=+}#u^WGDtJ9?6$HXGki1My( z)04o3d)6`-^E3#qzFmp_Qn7IMF_-$RE25)?sSRrt6HLDf1)=i65>RzkhGi>b;K>RJ zPRl|o7Or8rGn%^_wY1mlm%a%`2KlAAee>0R3; zkgA!+%=sOM1{?o@OXg#yuQn96J3MDz1~oIg7F@xYo;-+qrGdi{#l(1}6d3Gk!^1Po z*kPC3_+a;MqNsJA%$-7u;AC# zdV{!^RLVP)qzzO-)UZjTT3A22`aJ{p&Ma`tNCn=_Hn0{bqF)o0xxW?7!28q~JKHf4 zBPF+i^O;wqS)iVtU7CbvlchK!l~akdVl_73PebD=3UpiVQoL|R7%%76Q-!z|n!1b+ zZy!yfUB7LKj{XQy>^cFr7KOsw^EDt}?E?BV0PjVOq2Qf6v~8vyeQ-Mp#mcroVOlR3 z%Sdw`pW8(Gl#3X4A8$sh%0ft;4ri!UfVuBq0UhWRp)c_woH$ocU8ZuWhrIxTbpW_G zR-l>IUl`Baf-l{8I9s9)4tr{Hoz3@xq~bP6@Ce5~FHI2iEk`wl5A1%ASo*xjnyzX8 zLRQ#l^fXBpe;qhb!M|$nM56 zaO~kio8kh@9 zEH!nZaz`66b_vDdZAsuCT}S8q+l;c0-Z5&6)iE!`4BE6rxgG}(kco$)tYBCMTP%|f zGjDO>_;y3sUa|&{h982B2h=#z>>J4@`?>g2|17v2XlL9M<Wba`S~lmPbJMtS&8W5u*1@^HEpi7$o$bz)$gfAeUi?v1ts>oi>l77jFey^v~gN zXA?PG)J@{tZNSf7h1g9d5x$dF_~?@c)Lm!?@p? zVpM%N1UzK!kWp6y?sxrBT5PBR&2o+K)p#mhUaqw~pIB>Tf%W?dKW@}SZjdBy94l#WWXHna?Dt? z7WU;nVLtN55lbGAtu^8+8^qD!NpivOGUOG8VUdvlz8i0Z)K8gEtJHwKGgQgd#g1?zBoYQE zFM;}|@8k#@O4^QUp^2S1OooR+sp|(cH_f1ztCYCLo$9cv@ica+$#d81Js?|cW^-2P z*Ab&~WrCiu~ zI||x%uOb=&?rh7)!}NH@Yf{>fPH*caz&-I)bn=(uh}>Ai>njw&c}|yc?1>hqNXegS zscwVFtTkBg)5SP$h(@nxUi9d`8r1h1W6TgRTWpN3c)1Mc+&c$97bMX3i6PSVNS~S> zkwd;kHFT+87Pj*6{VXE}k4>mzZYxjMQB5^nnvqGIA1ZPGIfawNdNnYM&BH6lJ~MU< zgDPX0kQROqiq~k+++W*qNG_Xr#JwcN+#|U7HXp>Tl7e;BSMa6xQizV)gYVaO6Wy>s z%wxWO_LUj~RdH!JqnN_IF-uG~{X{kF+~EGp5xBO@2aCKV;MgxETD$f7d`U85W4myBWCffRi4>@$u&yNzP=#)-706M9W= zri1Tw;BvuftUh24y4RX`<$0Q1N%>Wcz1RPP{S|qja88o`yGwa9cZglJ=pT(*pw7L& z{wz-9Xma0wnF%HLx8RPNBWyf(4$aRO;j*1|^h3)_M%>x}EaV%2vnH5zyc13Dc`4(_ z1tr?HZv)+;WKT+}V{t#r(-(|e(R_Ls9<2XI#I;i)%1|E;dl~V#FCCJ;vmQ&w=D_Qb z^+fJLD0QE(1eepn@G1W(v!Ey#X82^o^aM@p{oco3aEPGp0zf0A&XFBwd%*X;8dL8b zf_d*AQ0=G){Bt-IBHz!#SoQ(gruBtQ>fp_13rF%-PMHYZI*6m~)}&o@cH_R;zSK@U zipXlNfF;)_h~w8Ss3k28CTuH;Bz`0-miUpzlDz<_1thk4%qdr;l;GxKYo3Xy-2gFZhNa`}Xl$p(vTSUVDl4jKZm`bH!MZs?=Tlb=NW{$2V& zL5Z`WMws(a^9ybC>}JDA1f)E&Lc3Vr{1RzL<&urC@a0AF?q@UVm7OFS&c5hr77y=! zY^J*h6VZfKS3d)R9bG zzZquda>3I{6e?dn<6(+GTvlucd`1t*b~8P8%w3KgyBvbzLQ~h@F;o8D>qllR@{C4C|KQ+T?G)k(|~-Uxz*r%7*q4{f|Soug83 zMsww-VgD#Uq$M7Kn+Iy3qdp$Pb*0g*s*lI7l@OP~*RWaXEUefNN%!zA=du+=nDo8| zpSBB=D05}b;f-oEuzNov!|R-!>${S^ys5~2Q?`~YkXQsin~Z6QO)e0WVzZ?R{(Dc+!8k^;IG9K7SHI7T!bY z^flNo9*GSWez?SRF05Rhhnd@7!uL&CxOeIta&^5fUO2fM3abWqeY>;iSHW5u7!rw# z>vSpw2Thj3-EG&&xgKt{WV1NG{Bydg1qY6JNqVx%TU)aEi<~MSCj~f2R zy9^wzR6ut3}W$#F1J7v$P3Edtrsn?Q7RFMX;cgvyyG;IddgPWO97&Sj<2b?39tX~!kn>E1^7 z)~aArQ7C-zIRLxHr=Z)LJM_WwmC&1E2?47WsM7{jvR_h)D?Mu*RqCh1{SgK9?KJ`G zaW1n%xEVzIs&K`!7`S!O8FZ_@p{xISl65(dm^W9GQyZ@|dWj{HHzw;bIOZYg_KE{% z!%JA}=?z^sV=yDR26dy2IrGldFdfiC3e9H1!ljqUAs$bAVty}`crumx9gKqO&mYp@ z`h5E3Q8dos@uCCHu@E(>Or=GXA#^(*Mw>6dx37~S=J{Iug>j93#peibUx0OsMBKA&($3cZ8@zQ7vp?=J z*02HVoXH)IF`U^Y2|cw=B(|-N@#^TuH8Km(C?FCCQ6ClTKhyJ11z_%&J|?;k(j#NX zn6PPfblt6SvU>F|_V=_TF#GfsT&3a&--B|p>6tIHOjHWH4SMj}$35)8v2H3}(c8G} zVhk8L+yGUNP+Vou!Q;?kXyc+CI6c0T9&wL=qnaCF%qxqu9IPgltN!A%#gAZH`%`>l zAA-e)f=T-xAv*H266OuwrdQG#NIq@Esd-6I{ZJb0I+2D!MtTtVS_vQM_K^0HoglXQ z1--ieHZyki32hC^0`^8f4c=T1wg>OghSasZzR_rqmplS;fgS7{Noo9E*2|uBya`v5 zKaib29eBF70JRoP#Y55rOY@8HZ&oA>XlTNJ=d~EW-J3W|o-T({#VAl+vWQ5m7DTBE zDNfO!OnTtkUpzGsjIJx!(7<$GTD^HCc_^|N9!IQz3!kLfFCP>*fASCEr~Y;-Z65?n zZreh$^$Jdn%W2%UH3*}EE|b7djyNVQN0+}GVYU?SVe1z}!`*agn6@b!Tw6O~P-Qo? zii*Nkzd$zqa~pL2P=QnKxm2tY$=J51MDG3|ofk)-K!Eo?m*?Wvi8GkU!*Fd|1Ifd> zEE*vBgKRoC9hT@mr5{`$@Vdt>;PJ^3x_?~;9~N<{O_nPg84#srvcM#ksl=79U1~)|{a` zckN)ayVk(5(Fm|BDWP(0%HXE`2W)k$Q8p(P?)iMgs&pZ;sz3!cHyh$!=X2N@ew;Ku zP=I-77SlDmkHebphpFW=e(t|SJ;FD?8`_~2FCUqTrG`$}wsbpuH8;UGcLLbGNf!_t z-_VmMo5A2j8q5qlixy()yxwR*HZoGpr1e3#X`;qb+ya1u>#sr}JYb|QYQT5ltEiaR z#ixa^9R2jx52*r{;JtLIrD`M7W-BpEBa5SybYgDw$1P zp>O&>66>N5K_+Q*dRGp1L^3Xl_ZxfAxHvu$pmdS}BToZ?))-BMhz&Kq9!R1Qp`O7>(8| z?8u!hm=YfYbCiD*y-!>$EnbSX-y_)Au^^hB8_BFONX09+1hH| zq}yDW^NLfzd14VP(Dj7*wkc%qg>?9Um+22RF%bP5i@$zOr`mltc=&{nuYCt$o0%pv z@WBX$?V_-IyBB;lVTt@6Z^8?{!a0Lyu+T+?y{yrX86zFEB!#7(N2Y^6k6V9#LJ1Ca z{)KN!vasUcOVZJr3f$r_9tX=o^M4jhbM`e5lBmNgjf$W&e;QOP6;i=MW$ zAV=pA>TjyVA0H>lD^G;qnjDUhHNvMPJ5b&rP6}qU*r?Gk}}S=8xYjFM84n06`<{04hTXTv8lRCbFUmhd8c=KN417)}PJ%z=44 zPtN9PTd?))5Z!z#1Y}mdBkSjHM2j(L?pe+UI>+iL{0oc#C7p#d{L?vj`k=_v!+sks z_w0wf=jE)=n)&2MqcdG1;)ko>+$KNt>tMZ5D-+Zahvqz8-YPW~Wh;}3_w>i)Mves6 zN6??dirq9d3Xg|-Ga~WjmO5PV#*0W)><61I&r!p#k2WolhV9}XNcqVQGWBc?gj~?! zEZVmd+y7=ko4O%gdeHzsZFWR5G!v$Xg~REwmM(DXUKT#wmQCPOCv{pf3)`PoY+N8-y%sA*ZJ`0JFN7h}nnpFy{@D zs4z(_NhIDq=7CdaD%t!iiM-ADhv{>2arc#eXx!UQOMi#Zm~AtWdutIzWqa<-;%9j7 z#A=*VxQKpB5$FC@%O}}}`4IWF4c1jgfvZOj;eW)EVQoq5Ju{88kFF!jS9}4>4=Zsn zfFIh#avKMRR4HuSjK>^4kjw~MUhlIydT8~aoykJTcOE5YUfn0Jm%75Wg`v2m&KR~V z;?gxKycucIkAb^RLEAh^uS9L2X?4YPq@;|Pp5IQZOh)N7yC`_NdI^-d1d&5y$5GLW z3#@e$*=1IT)7M^xu8lVM+-x2GoWn4It;5(f+J(;-nDOR`F_E;>0PBpaaO-;vhzcHr zRr6Be&D}ywY7NF#xn(d-KLbZsJ>qfl+v%|QZoK>BFjeGj!tyJvUW>o~hNK;>gF9 z=$oK3uJM!KLGk3yDGq$}5dz~$LGBD@2OThR;K~)o=1R9oD~}i9Sk+(| zCm!mxmAUU)bx7kYVc60@0oj5uu!;UePgWkM628BfeINbliw+r#Y7pj{e!s|y zyGe35OVVI*usHhFilgzSTwGRpf#$680%5OO=yUVNe~t&pqpKg`piTi?b$HKsjQWuM z8cpQhjAz99Hv@uO4q|4EGfq31fRBd->DL9@fUOtcWbf9-BYS7V9Ainyc`F4X3RaM( zhd8Kf50Z6Hh{2+bIG&jgf~Qp&dH#cVp==Gl8YzX5H627h&lwkf?}pY&=v$IPw}(j97FtJ52At%fe|!W@K`78;_Z+w{Tc9b>TSV_@!{3A*<9J_vuf9SeUI#*>)I-5RIFm7RW{S-+wPCN&;|txGi;N3}7hg%^S5hR(*2c@IFI zKM!|Bl+t@QWjN8arP1bPGSn~m44OiXq|#&>9r;y?Ex)Yj&!#D;SA7KHzC^?FD;q#< z>uq@0a|BGsLg0997Rgi|!0rnX(0iwWzB;doZ8PRz@kJhkC3lN%_G&`^U-B3ukqnzc z>Y-3$8tL0|2Awxlz{L7DB;{@lteKiYhWPiPTyp{l^K`SY>KI~QH4CR}$tHG=5TKM9- z5SbX+i0X+JczR@<`JNzD|MIgsiYz`z9BSXdajUD;a>_^E9cF3nmg(2gLOP$E+p`@e zuj`|9pg$(fs-u#g=|u2E0XPihkoX0vaQN^ul63PX?vDRMb2|izmhf}V+cFt$O zJl$vvGv8XkM9^opsJ0g~P=IqjbT1sTswDbtpGnLgdsu3{i`H+_$Hj&}NtGxU%?t7v zLBlB2dZWbE5}%HhrPZ)~>J6N8q?fojB6=L&06T2j$zn_PWuDCML03f$<&rnB3TrQ zy!LYl3KQT=97!a#l3m2rZaL@MhgHO9sst|Lai#_xH$imc2Hc)30&cQDsg$-7ngr^g z7xe>g=^ETWy^B3}{yOCa@E~%NAACG~jLkmH&)xAS0c7_@!@e&|NsGsAn)XzbyF2n6 z)tSA+G)pxbr(9~Fz8g*P-9LVg#|Td&uAYD&nO3;P+G!+ zg!9dXB@Q{{_X=gsl?C(QN|!jUR}+B~!AV%(vj%tVQo`dK^6`b~7oKm8LaK-$hMW@S zs_&Fz4!UPImVc;#B$MeJ6XABEDv*gWwHMG;{VVBDvw@vVHuG0Ko_gGl$DLmlVV}1$ ze$kc1VtIAsXzrnA66YalaW;vxYX@a*OSm`77fLTM%msekvmgb`u7ya^Tu8m*ga27fLlKuIy#3Y+r&KABOqDV!*4s;!H_d{^3{TkCmrSpn z;^j3gh9JmnM&n2W((aUf7?HjbMD*9gyhIzQ(Y`>w>wLi2$6Z+E`Hr&DJ8+JVFE1zX zgR0HCNcI@zV^E?G>(MzMTGC}`(gSytw=JPAPjczHyPe1e#4-WR3!pB}5A0O^=v*ap zYWYPJ*8daYXxMgA+o!VF8J`VPB+la83$t;k@;MP9F}Us8O7y!GgMuX| zc~3%d7gvreV490B7o%~sQ8rjK0k~3URK9OBuKWT=ke+TjD zvsf_6%z@Zn?Ic0D0KHE?B3Jw8u}2NoV(p3)yx!yvZ;CJA>fClJzC;=BwKCB5QUhIm zZRy31OXwfdh@Dmy=+b+T>f9E?O%avQec%F9>3;@Z`#P~}VwYltfGdpiO^}N!cJ#y3 zJ#hT02uL*}L^!?&MQb_kt%?$oeINquuHVMs?v2pG*^ggbGsvK62+?Sd#?^l=Lz`qi zO3Xe)B^)%#Q11Q)MDx=;IKPh9}VaI1Xy|h=ECDHI>`567-_S z$rrZ-Ry4~UGzG&k)T51byjzB$b6fF&{u-3puh|&ZBnR4BvbfTHIovL}ha_bcuHX6? zHcg*}-TMQvafJmo3RMsp|7={REKl-$|54eH-;C{AK`?4B2IUhI;LE&%5Jw~A+!dh} zVo~(Nxo{A;wU-w9brHq;CFHWPActeF02=oPiCM`R#?eF{Qr1bb{wLSK=b0Bkr8EM5 z^}J@S&YXn%is!I-gDq#MheA_83~O{>6W(%eF&OfKcH1PQ+_@a8Ad^Y6?Rb7m;Z6wZ zQiEmp&yx=09+($cL-xeP;-ZKe$~R20U!)ieL~Ky{2`?u^FJbE2B=$hNE3UoUN_RgE zLg&2Y5GZ*bHSFea+1eHOT5^J#(V3XmCfp$sm)*@ zt~Lk+Zp%(+{dE{iu9sox@FcYotw86U9CG~YK4#h9dN!{+7h`M+!SO*DVvT|ze8bD?WNYM!1{_)j79%cV51hs(&c&xA<@wpbK zMrw@g?DK zSvC=Q2*l^*ZXhu9DdGtRa0wi#TTG(d8lvu;TI-m>VKS_w_V@X1fJ% zlWf6xLUFL{nmu#YHyM@qx?Z2L#HwQ3q9iwMBto1tW+Q3$IR=3v~eIE-OFvZj@t zbjc16_~G~rN4^9ct*t70M^ngsuj zq~f-<7h!zn3Owu_i!Hg^=v{&Bbd#4999&rq&8xPe@)eV>-;y68cKaJ4PRp@t)(svi9f%utr{NNV}=4HcVO5c3O;p-jpb>?h%pdy7vTUFr4 zNIX^=<`K8^Jxs@_B+fVVfz>{D>2B4lWOdVh(j%vjSF^JqhPT%$)bsTD&5~#~V+Bqe z??sigvru?yfcfsDkLt~xbZ>St^-|gfYHJ=7?Yo<~HOuJJwsHvg zzL69h9VP;^-_g_j*7WSBBq+U9M(A2W(m16Wj8pmH*KaPQY|0@uHzTMI(cp-GoR9H5 zpYH!SSj8vs9wd|-y=HMn2r zjXCeHBlB$t6zA?leYs%L%FE0?^om4h<9rzKNuWx~A&>}f*@o~^`fj@zSf|REt{Bg! zpAK+%)KMY(4Q`P*@f_@nyG9Z&>a$8=Q#sv3BWyWu4=+CEMjr7a`&&Q-L$-clOe*Vf z;FbbsDD*4Qx?hen+gq^fcP@Qn{+3#cWWs}a{Lt7Pgmcxj$i9DXakbb5XkgFNJBu&V zgZHAaLTC*p9vQh4oWPXm3mwzRwiI$8H;VpKb~0S`W~J!`k%5cSCa3 zZ^Sg{XDI98%t5{n98NM%Bp8l&!Ntz=nT<=oLa^LU`umg@c$%7mV1pMvT4Z&5!0iKopGIK7-RZlBlx-%L+KU)xOOF#lqeK62G`#v zFF7-4+1Uk<>S|8q=WKw(_f63!@;<#P`jNixt;f@qkC?{G0vuDlr53-bkK#If@?F=QstjdImZ9CBEV?Lj4o33!)0>BhA!kYSx`og< zp~|&#HDY#Z0?Z#=L~1rv!Rn;hF!jP|yydbAuRpoNh+o!VKM5;vCO)jeM+IpRKHG?w z|EpnpN4Ut>bsldxnm~|x35wK8liRB{;zB!~Us7fU1fJYL9A742{=8&-sv1c16J=q+ z&!zZ#crVPUrM41(c?s@`QWvb@gi>m3g}<+D!edv; zL2kP_I@UL{Nq#M8Sn-SqkTLwx{Emh0O^_Ru2n##1;pn$AnjOMI@h*FiUcRi?r|b3@&z%Y*AhI!>&T|C2OwO@g3dex zkdQ9ILt3RU-oFt=%<}NHnh@!@yc%vl&3n<<^EK7pcnv$w&Vo5VMajpabgYPK#8I(s zl4Bl>hMuY%&xSXoODPpBpO^5o<@;>Obq~Drwi;S-bF{tdYPn;y()>ZLWm34|7gg3Q0aw5(Q{ySFir z=j)$=5q6f)H&EH|$i5k-ohhOrF_g^TVh7%i^U&>UDz!Mw^N|`^VTo-P$$qj5S}*6I z%vE!|R^UWc$2CCL<`LoVHlzW3oW`2C0+gQ_$8be&;I-~%)1cR8xHlESbmmvm)s+B$ zTl45$-;?Osv5V0?Uyg^W=5b!h3y}bJkhm_rOX@`u(f9NL`l~<_=RTXwrk@gkRo{4f zg0BTArEG-YN(0XQ?U^+EZx

    45XtBGW4uccIbpV7z%;l{-#UXz`!~Yq#PpxfWhP zPBf{qL*aX1RpqaQUkU{k0k>6gNsYmb8Ct$wO~^ zzmb=-|F?iioXcfb2Yf>7^fJ=5bS`d7y9{MIXYh5BALwg{lj{|6FnTu|a~$sDm)lDC zSY3mw#~+AtU9ZRw=W&?r@Dd-d+z*fPHSp=#kEYmd4`1szcqC^A8+%BQ8#x{f;(r{- z#w!`jxJMd&VI2(SKM%l*C7(%GnlWBKZ3(m7J@K1#HFhZdqmjPWjXWNM${hTS8;7#M zX~6*0_~t{6x)3gJIZGDa`%T9O?-7F!U5sKwA~tEvg6hrt!FTRcI6iZTEcNatuk;k~ zw^{_I3WmbHj37*twE{=Kmn7o4HZ0iYhaR$_^oz@8*v;P$^M6l+EYAt@@JSI{d}B3A z9q0YlA3i{sxg(r1&c+S$S;#5AOZD|1kb4Cotf`nBGK<#J;TSp2`XA3(9(xT>e{R8q zhpm)9rVC8DW{v-{2hPv5+{2pQ~Z-`I*cs{xOU^@4AB91giV(pL{ z8R9#MDSPZOa^efrGzh?(vx~@n486Y^{6 zV^}r*7T%rCVEJe!t0{MjyjrM;OEv9js4$l%>;(9`ke{oz{0vxbvZubI#zgzhEp}GV zUR+MI(NsPX+T(IDuk2gnJ)_4|8lFzM2_YN& zQS6Q!r-*fgr*qe!qJU!K9IGGrWk4IwjE#cl{xZs#`_r)hmXqnU82|OnMfuO^&8^}a!2{E2}6`7s#VEM9_ zUjB&W=^jVccHckx_0$ohzdhODpO!F1>n7H$_okVCzsc@IO?cnYfjtHl)N(#UB|Npb z6Fh;Zvh_Mq+dc;c-=>mn9Zl>$H3QCiTOreHdgbJ^)KVz;Yr*5!qTuH&C49S798O<) z3X3;fL>vAis8ilRFW)!`uLicD>;VcVW|!jmj$W$0ApqkSjMF7QG~luiPd`vz%#qGH ziGkDhftz?rz<)2q$=rWyxK@`Qf{C6Zv_0*Ft;4nW)i@b9 z{*Hy4yqgn~G1qWTN*=4`w3pYIe2vyfX`u2|Bl1mf9a#QpAq_QRoO>@Wk&4ghRPOQu zxS!C1=I_j~{7p9fZLP_De%}w*?$*Jcm`>B+{y3st!lhzAud}<4{UJUaGinxc0)Gxq z;pFJX(HADYaKqXVq&n}jj~*@Krifmps*EI_an3;7|K>wet0OsVT=B)MjPWTde^dHX$A0Wkfd1e)GOxNMpUy&f>uoz#uOoi%) z60lU}Ff{DYBNo@6q1QHk>{1s7|KT*SI^qHMJHo*Ei8~SRui$kuY5?ER1e{uRm{Ofa ze8}VQrFZ>-P`!0f$7css2|*;rUXMy{4z~Fww_9L;Hj8jayCC zATK3@scQO8Q~&ir&djaku$uw*?^$Me&ix#CEp^7OZOprp_G$go7}$#R~kJ-_qX~6*!m6wBh5uQG9nmfusJ# z5NsW8;IsgDQ$x1`((9E@jQ8&+3En5ra|%P1j=+6^BG}K<@MfxHqi$$9E#P^lp6UE0_Yc-Hw)!#j)s;Yq z+~x_#RL5aia|vp$UkkG(^5Kg06t2>UBe;5<#n9=!xOUkB&i?Ixu|}ep(Q(ga!)4Zk zg>NG8dn|&4qG|P$x3thFxf5+&1R1G4!FZuC2@iS6a;J~;^8NYQY~zy=`s%m|T=81b zm?y`_p7v44cL_aIP>eUzCU)Sspdxoi(_E0!uON}Px-nJG9CvjX!~9!jFq`qf-rX4> z9h}BYrc0vreq)YP-C9@>+eYmA4QbkrsocxF+fk?51tDjHJ{Z{c;=}L=n9uW!mhF?} zT#fI?Lp$z}fZhR<>%|8_#I}$qsP^H9wstzAb%`ul{Tmb|7qdt8_d%=pUl3jr2ybRp z;iF%hAefh%*x4@+GiHr4k^5d__ajQFM>_euvJ0H^#p&&aO!_=jmxs^gd5muy?cwgH z^8QWWAY=~?ZQ`7DUu;mnIh}SiF(6iENh8y|(QSn!mm{!{p3_>4&#o>3Vd3+p&EJ$@ zwvPxY9$p5|{d&mN&(Y*u#tck4`4H^SlFBH*6T zY2dAacOZD;Eo_V`$5;{G4Ac6J8uIqM8J|lQYPaLK`wcvLGJraSOoJP##kgX3Kd#_t zflpW%q%Aze)BV#F zs=>ApnS`rwGxRVWc@&CzhS3n>AWk;vyka*0k>m)M#n6_{TsXwzE;bdW;M~(`%veT3 zqtUT1C?Y10>K&dKJrPTVz>p@rnnGK3#W{=Yz36QzA((A=j|eMIBiD3V@LTi|BBS;L zB9}z*-j{E5vUHp|?SB!!hozz3-$e5Hk|S|f2qa5qi81>sO7SmW3B(nw$5IzD&cQ1O z@Vj<0930K1aXss)#REZZ>hFc<|4PoZMCmaay0750H@yITNj}byuRZcj;rVRWv()5V z9C!~8K>hW*n2@MUY|kY^nNbPowg!?JPYTKUvE!8gk{iBSxtf~vOhxhj*)Z{lk2}M= zg89C0IcheZf_b$csLvKc{+(z6{)NUgsrnYo2s=$DTjJo%wqz8@T|nBtUBr(c``CG% zXGzjnJZ+cVL(&y;VMys2zPH*7wFbf*gKHP@uUEzY6rFcGmhTtGjgU=bWGkbRNO-Pu zqe!BmQdEjkNm0>|7LgIjj%dk9(y)21b0aEABCAqgDWkMxB$bNa^ZWCDz3zXW=YCz+ z8K2KNZ!5Dg|Ek+v?Nz~dORMB_%`CXucGJ=C<45B*}<*ZI9FAY9f;OV`XQ z%C5Ddbj_+tb((60>0VaF+}D;~p9ZbYA2+U?o#|dtZU3+;CjU>RV{d-tvu@2rx93Jy zM2hEBjrlrM1-f=yl@)HZdOQ+RDdu>rlJ8QrrN9)=sw*4~i{8h3EC=*{S#3SE$ST>e zt;$z2u&VKn-Mvd;m#y4{>MibX@wJ-NGq>#C{-LVgN5aaqV4l^#tVLEq%fD7V;hI>| zfGt&S{f~Kj%Jx`^dInf7b^K{z-z<0Uo?2#=&H|rGn}<&=$A1k}Ca!#6*`YAIQnTrB zrOvkLm3Q`JRc$MIRuw$rYZVz@Wy$C+w=&OEUsRX>x>8VvR+=w8SQURf-l}s1b@o=?__(U7KdqUP(@ zs>Gj-mIvKdTDdE}x8g5Js^sBEsvi9lwmSJfr%Lm+mer!4IhESs$yEklUs^3Ry;v#p z<1npGPNV-NsDkOzPq?kXpZI>1!`8uEQrI1V!hI9u=-m^XHn}}e!wmzolbN_|p9&htadQKxWBFB@kkne-S-;1gRDTpWGRoknvQcCuHeM&5$Z-H z>DLWm=D!StP`2qi5y&_~u6&Z^WR4`kmb}?;d3F=mwJwrOsypK!mJ4g|_Hhl;YDn+r z2c&)BJ7^F|B|&n0^hMzrnmUxM-#Z-nSRg}fG3~Mf`x-MS<wZZ;U+h8MWd)$`qb-?9sD&2PMHL289z@UHAVAqCXsI1*c)J?6x^!Gh( zqm4V>JW&Q;N7Zq^jU-4ae*#Hodo29YWB%-}F886^T6jOQ34P~o10%UZI65xMIuseh zzy&$(R0TC0(kaK*jKkPBe1|lBs-&SkYdDE_BG7c~0asJj2cB+S4kCMYf#QlXsv74_ z#rXP2;+;vt$2SFY+_%xJKNg&3Ef%l)@OGDfzd$W=xwv=vJu=ro2r@0=I60kD@Q(C4 zIR9e}9$L48?2ZUVmy(w-TiXWWW?6yY2}MRNvza^pQyMk6v7fjmM1i3}Eh-sQa$c<} zfSZ4xg36v)i1VuEkY(*ee#a)dRQ4f+{+CXJ`V}#+F&)=MIbpBjFOG`oZ}MZ;8S>y~ zI@z~L8zqnCl`mCF#B&FRiBCSIAwD~xcIIia`Q;(dpCN-!O)rv|CHr9bg&mMRCS>I{ z1Dq){3uU7SuD(A7?n_<4{^CfiJ~oKet<7ZLyM3IH6;06C!-3})y}@yg6X)kNFYZg} zxu`9>3U+?>r9a|tQpwnOl6q2*T@+seC;ncg-;}Jdv-B-Tqy7^#d>ch3rX5f2kR$rl zu{fYBNkEoZShF%PtE)k5;0 zC~T^*!-FdNpteJsyb1CvduMkIJw;+5>Y)-w74h+QSqrkx5u%K;KpnkO`Wtt;n!u69 z8SI4Caa?N{4_kz9V%=vxCVihFvEDs`Qmqp7;z%gc50RL+C5hb2eFML4?uIIfS}=L` z2KlevgUjB@s4?yW76V_PKP!P&+dY`ou!TlKtP&H#@wACX6|&u2(p&M@bRP9hvwl#CUY3*okj0^T!XoLh55Q{Z|VO00wRalmCh-CW|qmynD4$8H0oHBH<+QbfALzHJZwpmVp|^7kk1#^Vc}YDblee4+dH~ozWo?!>3WH^vljqIeE?L| zd1s_1hSslrL(89g&?x2CD9PLdy%aun>JlaFx;Gcwk6k4Te3kLIy(V_juNXeq0>Wm- zn7jNTDvj)hQHR+yVf``ipz}cD@+#mz*MRo#Rmvw~L>V3OhgNSDC#wWXIR}FiseA5a zbc<|*>;8-3+2%}qtmq2K(eLTf3N6@_m5T#Xf~<&dIDrEnshWcx>EM~=uaZ;B$iW-~a=$vhxAmp;1RMqC-ya^6I=sXXJez~x3 zw<3F^cRdK@zlLu46ZDh(CfJocpZa)6mk&S61s?Yknp8GId&@n%^!ONQzVQ)_y*I+w z0zPK+zB{T|X@gbqD^!&4=H%+8gO5`z+3qU<8YPMJZ}J)L93sfR*i}el{HHMXFXJIS zxe;IW&w={wx7@g)@;zx=EK+hr@#_ zbMfxkJPcZy0Seij;AG~CIMb7Kmxj>y|ExjvuNMg;X&kmAf*!ClMmAg(E+pClr;Tzw zUlC+Cv{TIv5twiJ1SabG+$|B$o3GUssBOkmfD8w0RZ$4VsQGcCMt!2^SzT-;|qj zI|7PJxl{}ENv&`QRq9Oz*=t+D_4r!Yt+AflDe#0FyfG3*w+La9`Ylw{J_t|Md_n2> z1N`rIJ`{g=K$=DQSsP7?wUuW;N9Zou6YPYds-n>HX#s3cr9i*M^}$YJ?ZjPqO$DpUBGX{u8| z&()gj9rcAh&AV}MR34a7YkIUo01R8JiHfHfGP1j2vxp-Unti7a<&<#Na4;BnFQ97z zGtptAH;G4RS-JQ_K+JhrqQM4CU`5+2!DK$2Y-G++**AC{2zo8 zxyflT+M0mI96OGm(kb||VJ8OoXOp1iIo$OL*Ex%0Ex}t-lPrioK^MpyV#aVjBM($SaH)E z=H0tXeHJXi`@dso*uz6uQ6vZM`@`Y#^0VlmdYWhkN}zYUH_8epT|?)Q@bN zKT2N~-2tcBkMKs^Fs7>plSk%1IMpu_%6HTnq8iSFh6gr~D_eo_n>*;VjXUtdrUd9R zKZl(=_|fRQJjz60BYzds@Y$0<@^F6v`gok-#P>zuX}bnk-kt-oOP-V7Mboim-2&XG zm_X*reWCXj3&S=C9}@bwnm$!J!mZ9W#BNs)!Ywo*{X=$8t=-IBoR`RB(;Wo6zFpMz zwKzyF%A~gSdvUME3s98KA>CPIak&AbWa>XP5&KfxDhebf=7F9gDq zxD?3kkbwOx2X_=_qPlblR(4Dy(ce?ahZ8n5yl)R|I$ndK$7JctKUd&nTN2)_ONRem zWTT(r6Y6Ar7be&rXb2%>GE0ov?!ZHq|EMKeODYKaq!d=0<>R-4kKnox%MG?&#@6Sv z=;`My{5NM7+Bk@@YYqyspWhsY6#t8GUPp$0KAw%At%Fg!-vIpG(#gmyH*kiRC{6h& z|KL`zpDv9z$}RBlpeHQY9R=-Wqh_PGg(3H~9U2{ygx?!#xR$veu}$(jwJKF(UWmrQ z&s~DFI#meIUyMgb=UHGY(+RD2lDUoBhVbo(8Sb;|hW)+8w6Hh}4=e9M_~MRwcXklx zfCSj!_yTuL>jYID0mOixsFVx8({s(8*q0<|M2*0+s%qSR}s? z@`6^v@b3iRlCQ*7tpGW5jDT%Pgr>i39Iq7@AT;?6{jZY6`7b3PQ~xgH>7Jt}68drW z-&pX|Jccm}CB(E)n)z>GC^c$G$Kg$v=-(=Ju-7?`k$Hj8efSdR$IMW45czgh%+a5&jrbAr}4zPY=}x}#J`4%K`nGSq%M4hCyqD3Qr%H9-|Ysf z>>0F>9d(5>!BpBwXY^Jd1HwLA@@6vnrRq`gIX*r7R>3H#v;4q7)m;V^7zo#bCheKr91)FA-`+8l5t@h1>l^Tin-OQxrH^pm{tY=%eVaN^E)o1Eh)JRI@#NJq zx*}&0)SRrLOQuYsz0F7PI9x|pq*bF~a1=TyxnT2iEl|1f5?@{m!5I-kXm{=qu*wtk z>XD5 zK4wl$Aw zB##59p)hHlJpBlU1Q+3B(K9gbzfyDA;8HlAoB%iEor$#3N}RW0HQLRWglD{ZVN>n^ zr+->2QE7Mu>u-Liq0ao^VLhLIwDZPk4=T;h8DW>JtuMj z?r6MaoVx6L34-S@@@lPTkV;R}Eyr?k?qwfr{XPeeDX)g}(czGEY$vQ!I}gha&PV?@ zYazfo8SFdtNXHd1j9Su1%A2#HuIm!Mzj%QbHCUp|=^s?}H-*U)TfzTSBrHwsCI`YE z!R0~^Tyi5AqNL*?izhGcG`>eAO|H|#FBkCA`%UoRZyouw#TwIHZsCdN(l~9cA%v@J z!FyL4(4(mk9G9FgKWf@aeW%30*pwe!<<+z3TBR3M-g;85r|v{?*bP2?_|I%>{b+RNe-fodpvGy}XQwAqDj_28$}hWp$k z@!NrUOr*vYNcJhfyS}oN6RyI@Xb7=-H%CFUCKA4dGHiWfJiU9il{npNAf|t8F=4GZ zJLTpiY@2or-ghr%qmT80mBJTvauvei6Q<0c^+N3XU%#Lua~~wFv0z_)kVZ+*6~Nl8 zHeaMygJC~=aqL_xWOp5a)&J(Pi_Evu#J()fNRS0dIckni8b#UUTfbn<({PZNTEr^M z5MxdQANy5j9Ufe7!)UzvhM~~|Ku)TG*#>j=<$pC`)KH78r71Fc+ROq?K6a1xcgXeJ z4G}#YRxYIsr}f&n zHjZco?K7Qt{)Q@s>^5Me9QfJBj$ybSvmKHHfV~!AiX(PwK;?E8F@^`|_)wg=$=gF5 z>3$L;4=!eE^uFN0j@OXP=M1IyO;}#O0fTqHVDE1QbdWYK|E*@&b96P>eAZ_l zpAlnZTK}Qm)_r(=x;5h({teY8yCJ9D6$X!*u+x6?cu(ooIB{?t?@Vbj>*sC9)$ZZ2 zzBa~uj`MFkSvUz_q=ry%wi^nK6Gob^0xzZ1Lo4s0@v4z!gjGJU0ea~PR0=4#G9>j2*Ix@8IV0|2U)y9n4NFm2h;Pr(MM`A ze%`x~8QfTobpkce@_QkS@_0`=S013k-6ybO=O9V{t?!1;c*OoQ}m9xvIM)fl=|Sez9Oy2^h0y% zHg+X1m;3eEj+xaci=8tCne)@%p!|0MSf{7UvYAJi)?4wQ+_8!s7!YNjd^*f1XZvg0 zl^!gu6=EfxIxx@UgUmH+HJG=>e^6fG0p_cI1o;+;H zFUvYFw8V>NhT%isC&<}p!ahs3M8C8~aGS5f&Jqq{=GU7LzvD~UidTs!l`DZO12oaF zP=k3C#{0gd#~C;E9MBP8#diD6V6#uTFlIt#j1$?;Zs-1k*hx=zqB$G04GGiCe-4ft zTe2N;$FS5!jCr(fKb}gR&1B>?l$)i+V>CyLHL&w#9{9h3-!*pZeK#dm%_E4peaIgg z^VYKV7KW^I$pOY^^gg)fd9WKgO_+}}jeOg3Wh-W} zte6+0dhQpDtzX8@sgz;c*2pnCa)&`o?k-NWC@`8|Pq5z}ufv{?Hq7HxX-0>4j|IMQ z!t*g7z$-(IZ3;Ze%p=ZFhfCSGU{%!m^MY0e-JzL0-KceF21;;y=@MT-e3d%@R(}gH za>-E|ARUZf&S(+2e!#3aEa~?`1mn`e5lhT8;-=6 z*1uMTRhc{SDjJ%<=M%$1sTh)cL7rLHD8t6t@DfE%GC!b3);!(p2;)@yAxJ zY>W-&L(=*m*}Uj1PAQHfE_<4(!$EyoUw;$LW0JAqL^%emaEBu*lC-j6oGV+ghAz!d z<7$^~A)?Zmq{GRWJo1%?GZAi(srZv4dM_44ei-5ARql{6-b@6zPQW*@mU~TA9u}Lv zqhF`%lMkc(4kL4w<)M~VhWu7HcB!%PVs8dknxgN%J zOmSdl05RO4!&$cDGI{6cgt)qudX}Ar->2V@iZgBS`@#?ey0r4*oGRIvwv)HHj{%YV zZ#Xwem1Cm5j96sI!TB?bFmR1KT44$31k2F|0V74EVqVun-AW4mSPk`e%yF&1Jj}$h~SxWC6PQ|ScFT=_lJ9O=f zpp(JJP*bOlKHUC{{QUBO{El4L!a-A4MN+%Zv47@z!0AmgpiIm(}p z(s}b{bj|dI7m8^MDqZ8NlJE(b%~`0uSvYy!f{lFRzJ5hX<4RwxrKw79Ep zJ-~VGS2+heqw#8MEj_lV9*&4Na&uDu@p`@d%z|PbAG{`+ezqS+GanNsSUChDV$Y#o z7H?tR@4pZ)F&{(vzR??^D;mI0w_rH&P8JQ4e^cSAoD!{>=(^;kT^^DR!G4^ZJJBSmOViu0sv(Nviu$mbo zq`5&0rqwNAwHb5HNah}_0u~iyKEvjfk-+yQl%5jL!UKb1jHay|hAj?-_eFm2`KT=G zE!7P>o>k+Fb^Pr2?3GNFkp&I?@f4MnD{(7oG8@l+rBBu?fNhKpN?9 zfq67l44jVZQ~1c|TcvdW+1-$~U@JK?Ba_&~uZNVynnXSK0`0N21*gq!n0#NHJ;c{W z=Jt=nS6OTJr0g&9VcBgA{-Mm8JU3)kOnk;H&oX#9`5C&>4B5qFCEW0zp%CSy4RNM- zP+M#^OsWA|q0YR!Z_#{P|Y=~`# zg7ikpadR%xvqcLS&Fo~H@One0 zeV$AB2_L( z3O43YiF9Fd$61YROy5GsR;n_k4vsik+J`UR&SRwGM3}?P-uRi9_eQ?k$gEV~jW1>` zh0Xj^nSXK9*iFV)VU6NaSlyHl3D11!k|Sl%<7R|{U5adQj}7DLmqWe33u4D<8doVkgTFlTE+19=xlbuMMLr>{5v@2u3$HK{s&Q< z61<}<&+gsw4Qd>Hamt!x;?#YOc5GM0Gwp(0e~EqES|2yq@Z$|vVO|G`J$8+wT3^fU zeXkC`gl%!-<9z6EuOxT=Zp7P<&e0)G5=MNSLds{{BU^q)Qht9?^s~sruTQ-|f96vf zcmH>}%=AFqXd{Kj7mZ12o)(6ltit?9iJ-H1K3PhbZI75%KpT{ z>?AO{%5sd}tJ7kIa0oOdxcqkl-PnGCs&S*J^zS%4@+_1%YW^c#k14#P4kSu1hBFkK z3TX!GAb!C;tXO9aH|2Ub)7VAOyM7mJ+o}tP{FkBp?xi>`DGvpK$mz&DV{Z3#5(G|* zp@m`)KA)*iG6i0NW8GIg9B%`YMNQ<ajO zMK!0>=B^$u$n%W2@CXygn_K)RmiZ za?W0dBd^oxhKvB@i=M^I_ed>o;T%O}-xQkne2mWQ*T>mn7n zeh#eR7OFTwtF#$RTOmu*gn!eO?(5;^oldH{dw`zNOCh@jm%{m{AnNIG14_^|Cf7hS^>#(pu8IFiN!DShC-Cmktn$NsufL@(?- z2zyOL*dxNeD5W@r4|i!pPn0*#nyP^j%H1gUG!Qmb$8)qNB5~^K?I@_$M2we3Amd<#-9#g8acBMj6oN7{$v2+u`T)F3hx9hsoioIMDkY|I8eMDMNL1&ZC`R zc3hPCXm*d;z7CNcu5rHY;O~7x= ztNZ7i!xif%P}aB>nmT#ju8TI*VreEuSx147`V`!dn+@|DFTl@tHtY_S3gUV<2`*hO zhm7P0=%B0&2lssCG~eOf^MZ@fadR@xTDt%}b969FDhiH7K7rss6=?et%hQfjaf)CE zc`I-V_I_xDJ6VOeC9HyM3VXo0{Z^3ChA@(J*B`(3EFnJ~DSWkBgVJ@pcb?Y4pOhES zZ?D7#3udExlormtAci|T?!uFq7hroz4Lq8q1e-Hlsd`^9HavNO-WN=nLswie{bw!w zoxd4YHT@>KKeynXRhQ{);a~8h(T0^%m;_bHdB~qB%zkSZV`%sdyiv$t&}UT?xspOp zl@F36v#%WaxC7i}7q{Z}E3H(7Glg-KyASd4$ML|-IcQ-}M5k|9ZC*V~fUV{agl&K8 zXzbgwJimkh(>v`MW;eKE`5GtmP|JePc~Ryn%6mz9R3SC`^b-5pYru711fD(q9dT_g z9E@opce)TItjm#Wz8PZwJH}y_N^@n;DYM(*Lc|t((VZ8O|5eiQH2P;P|QVr7vG%J3H0uIv5 zu4(q>Ik|FJb2@=YNhq@V&H6OidM3WxeinO!d!Y5|b#%9RN+Q*MqkS>2C(QFkeHmDa zxpq_87d#Cj+1?wkSSI3u_%?jvx|Y0KSVqKGpG9<%WcA${X8+xXI2u%pG6)i5}^Q;y2aaUTq@6u zABw`2K|1j7cm&3cnPG@i58YWUg$F0E!w$cbSadcG%9`w>y3S3!Btd?<*x{8OJyfMq53|PCpFpFNWN8PvXRqXA9N(LL)h9nY=fu6K0e)#$fyyImahTXGb#c=5(U zF$Uh)XtHnprI;&HnmFa3B+LIe1*%%s@Z#%QbYmZ(>EmCdez=w_t^3J2_}&w>M;)P* zDI>ZbF{s}?fEMqkGed8;BCC@Gvl6|{U7la2QH{w^~f`IuY}kxpF!j}Yp_AHb`Rz=FSTQFb>jh?Euzk?_f=&r zPK$7^Sjw`8HPqPa>BYD+MuibntRUOU)ac)L`n0U91DHx>c2Z4-J-bAleSK4%sR=10 z!iQr(>24Oe-^fL?)k=_9R80OeISZDO=`gRO6^>;bfa6MoAQ5RnhmY}U;+Q$?wh>u& zlaV#^Xc0q8{GM_cH!1XpbAdk^`4Bu#!QHzSB#c$r73*>_cW5sPw2h%l(sGhGEW$j# z`yCcuQD+qNrP&J)`eDGc7EE_bFvDx5SY4%RIC|+HYm!6Y$}p?FmD$gm)EWPO@1XU^b@*7?fL3`%7_ojjO3CiV^qD;zA^+ph-|>rRyR3yv zqo?7ua|k>?eU9VOD-PpJ*1^?VEp)w7FWp|9O)ssk;d-ijL+*Y@2-}*^x#>F(d3&(n zXu?BiauH-kZS>jeEe+gE>4$We=rg+Ms}pyt$UK&F;1k|pb=bh%Q852gf>$5ELLKi4 zto}HaS@&!%do+1En|5Ui>kui&Ox@c|(uy)kx^@I;q$op~=QkL8IS4uHrZS_;@1R<$ z9vjW;OZxHbQ|ETSgB2n(8JnkS=F@Wxm_2J%$er>_!2KKtfeZPV>+@W(XF~-fna6|f zZ$ai-C1P#C6O?bNL|=|1`?E@zX>D5$%Uj-(cf;4<%gSRo^3@gleaumiy9=Y*RG@#A zIJ5Un8hl}fApP?o>RfqAdb54O@w^IIxKNq3jb$;;O@Phg#qv*#3ghh~&crO&i}TPJ^f#DMhYQE zB4uIbzgt<@pU%Z)68SixA;?7cNr8>VAQVsULcs@o?9i4@bH0RMu=!{mRoph4P3zAg z#3zM*icTV*7i|N91qpD_s}z?l`iSX;4j_H^5>;K5ONI*Uz}&HfG*oY(=|@Y!s!E8N z79N77ZK<&2Xf>v^@UyCW*F)S~Cmb(phchbYU}yPt?ELVE>+Re^(e*iQbUY5X8!zI^ zUGwRcXLjILnNKb{I^r_PN6?g8LDJ|cw7VNj-o~Zl#%1{&?k!%QcU%&ZXIf&-6j630 zZJapkcXPw9C!xar5e(^#1fh-D_(-dS^l!NjRjI}x*_ldGTZ#a((jd3(HFAd}$g7L5 z(Cb(`77FykP;C;JR8?S!@ntMh+5)F`$bxBB6&j?^XHUIzfs$eYbkUE6Ezin8(jp9h zeiMKPpK|cYQ*TIprA+jj)L~vi4)(uDK-rdK;2)9%LM`=Jd+-9j-*fXIzR^@>@v;~YkX7~bASDnXcY7=10 zmrnl4cVj8^Vl86Gr!tP@FG;vi31go*!vhfRt2Ny@_bKfm zmF7k0aG;8J&)2}wKi(LcWX!9pOR+OV)XcKJ3NGC{16OD5L&p_OSkcnWDRKeYxoss* zoqG2yT(z_dA5F^x<4+B6XHpjT zCtV;(+)}iCGmR}Ch=hK(Uk$H!6`b-^L}V2X5hX@+R!JGLe_H@ zlCDG!?9$1^{R0X@cXWt}vdi$GqpJz@KOZ<|(~I%E@$w zui+<%Iw<1dMxXMmv-Q+?{yH@5D#G6LOL0D<$>v}8MgylnNDY_`2YC7G7F7oN92R0g zlMWVs@1=RR#yI>*jybh&KEw{lGC$Xk!E@mZh?LbP_gwgyfA4%Dp}`K;m?*$q_72!P z5a9e0XX6re(J(-rjVSNKmXJwE+IEW6WfuUm=pDFc48giXy5Op0%qVTDhOyLVoHKka zI?pYwmfU)k1ZlA*!|&A5yYl?U zYCdEj#+0TBFm1wbAVqwVJ5tTZjCnr?@u=HmzmpMV|C|HEBco&@NQT)fH4PH^<}nHN zzd`vWg^7#Kpe8z-u|2a3^o|O%Pe=W!hO!WAJub|C2$E)O_5#WMDaDi=RcG=%qG6%T zEGUtXX8QM7!*vx!CR!w)Q~y|u8EzD3Ub?x%zeHs?{kjX%&)AZ$$zn{7krtflR%bTa z48zsXDU4n29gf3pF=jA%Ds!}4eB{G&0ANe=fSHPs4CuAdi|wA3-R_$P1Ylz05?_M$Bk>uF>*3 zCF1$8VEqu5XTL}95>@tl?K|3@r^jx5H;sLMU72-VW{mG$XR)H&#xQZQ2owBf16EP`lhXf3FQZz zm*Rt{pm820d2|YLQW%ccsj(hsGVn-q6(l`Vf$~3c?D~;NytyY2WQ<~He&SCEa2_Qm zdG*}eE8e*5lmN3tmbWSTaS|>G5JA^plFY!7WH2bbjm0txaA&$GGpl?bgiTMy*$$=< zGbq7oNBiQ{cUM6F$vUvP{{)>ci=pntZO|zvg-WI3O!gTo7^sRwaVK^BWY`1KYxQ7t z>uFTFx&V4gmDxYv!qDJoCL}bC6WLAQaEnVOhMZx+f3^g+`KdB)K6#LCQ;$(Ut-)PK zot4>=ftPz&h+1rdXScq9l{O$C9>@HE!%PL^bAn7v&1ErrZPJ&_<%)O zCc<6`jIL`SO4ViNVf$5Sit%@1oo9))2B+!$DGZoY8$rs18awk`A|5r%2dUSBuoe>F z^XuD0T)mNMm-9TRO^KlQji)c3HpDsIYcRhylNxznr?lLfO5a!xud{^MANlb(Z8#S` zPCCG3;$QqHegJBS?1HU(G5yCy zFfTHNSF<$PvY>1vx9`L4!|GTpGL_l8ClT^$Vvz5=0w~%y(VTxl6Z{}e9T|7w(FrW`VYGdeFg?Yresbsd)0@@g54H@aGYvlPw{pf&KpU;Oj%E0#sIo7Kp7UwoT1S1)Jn0~qrW7o-HhqVW6b5z6VH_A+* zd?G~cE5M@N>S(*3m%DE+gym(G|D)*4>|3@_Qj+c7ndb}@ElR7Th4xfmqCIWD`TakC%)I8g&pqdJKA-dE4okNc z$UrGqyj6v(+!xJ6WXx~KVzaqj(Y36L^@%w(?Z@6)N&lNW>D<^y?nHj9hz^33-*rQ{TIev87X)k<}ht@tx6po#YB;VRRSoi^e4;hb?7pj{d`rUf<77 z$b7>-Ym(y~KJ39hmfy`tZVly6PfX(_e^udcxCeqi%sb6N?uz>yw%Y1Gw`_NT)W@B~)yE9*j&MQD(V=0CexV}&;FWZCU4aZE zkmdrf_RM61Hm_mIb+&NhCS7KRy2)}XQGC{8&QLDkeH}CS_yaeu$(D2T{^RhmRza1P*!E4NgiE*6QfmOWHfjDmUkC(iyf=LV=%x7uaMMkSUo3ZVwW`3D!i%#l9G7H-3 zn6dBEn4GiL+?gfnBH81K4!3#txDVg%G5p8*jMn)>+zqKOpmt*(&*dq{6@EL+uFL3W zedNw@{ngSruJ=}6?QjvNYV?j9r&Yt`olj=2DXnEcwWM+X2L7=2Q}6Snd$Ae4pxa!L z#FGC?dgeM))R-YZu5vChiu}CQ9n3oT$gNnH&UlP}&E>>C?QKZ3e8_fB2b&t8dZJxaO ziLJc-8>Wda-&n}*{(FasD1X4g5H-F|hXd2(;K023EcjsIUJk!ma&fi z!pZqded9MrGyQITya8brjE#1zWIq-s8HdmT|&#`A7gk9iX8JuJ`&+6k2o;DI0FW@%89Wj(|`RfjQwBCey8CJwt+umd=pX_Efe~x0d`FU}f?^w>} zVhdB0>BO6uLbvEQKIagr!iDT%nUyZJ9NVGB471Y_ z)h%AljY>GmgtE3=^~)Bf`lk%+7ho~ry|^G&&qY16sPmpQy{84=4A-(-8>412X=IajEm!YmBd? zG6DT%>~dA9-XvGdg*3k8T&~D4j}OQ4wtuQ-6uwk32lmcpBSobg|CSOrL#jniH5q8g zZ!c$$Z++kp71+uum_~9*shS)gRxm?dHZU(0ym(o2r!nQaV&=@i2)@UeC{AJ5K5oXr zXS~#FHQYnnI+i+4Vg5WD&dH3P$+#sgW9miGTy|VfgYMR5?mE%p866zS1=eRZ+rj4aGG(`_GO(O$uQrJ7xFgB-DXK_wS&b0MQ)DKOK$!*edel5A**!# z1aIldjZELE_q@}^`8-L{EQa5Ikg04-KGR5&H*z#4zqE(4=IrBp=*tfnLxxNT1 zZj`GPb75z?L!@dj6E1e-xkOg5SGsLDt)uB2v)~A`Q(KFRyt9_;9sQbH{bZ2mx=_M< zbvKctHuJbW0Sj3LlVfbXpC0#PvIE_6j76$68+d6k7o9Q5;bGz9hPW4_S>^TS?DgCT?m>Spx5;!$ z1Nr`(r{x~aGuW}7ciS$@A$s#%F6@*X^KZpw-pKqMp3IXdt}xVxchYhlTb8ZwaMG)Y zdw;WpC%h??`bd{?ug}{te~#beEg9@#*tLoF^T;&z;pl~2_7Ho{c&`&@@VJ{-Fusg8 zFV2v0n6{G3+k2WbJ+g+e8J^7jJ{r_e+rFbgpJsB?bXz&IL@~2qaxMGejSX|lzlpVo zsN#xjhwyYPYPrn(;hg$T$}{9!Gb=WHV;ejV@^a^BaUDm_u$xV_x$I9WoO;S-UYYJ% zF2(c*!+qk_bH2^olLLPnRx0(g&XYYj*<*#AUdVh-`-m-TWgzCXy68F?DFtlS zw-sE+rSlEYq0W>K$!0>shw{HAhVs0}ujeTZjA9n3>N0~-GCb>9fjlo=d*=6(0>(5Z zpZB5LoY_slV{+n$M(&TdRMrajQ)RF9AS@gSG2s@Zhti%*|C=DfMzY*N8^&#AdgMp008h<9_p$Rj)N`u>>KAhPl7IE+QCUNcN+n6KWLq*R=dNPOo z9y4+KtoivZc3k7~m)!U98Qg@Q510jpD>=uZH@Lg{Gr0%nVwyZS}71Y>p1J^oOqsdeGnJ<4ZuFW@j zPx>D+rkWibPis2=RhK3&duLjGX%%0z?dJ#H!|;vV1?2)J+U*EaJAXF6Oy)l>{@n_&@*Jrz!(ub-{?k@fYasSUL@R)^7%enW?-`KG8>h@oQ4Mmr%hVp(y8Zl|tb@(o$(|K06 z&oJ9g{N|0`;=?^l%Vd_Ho5)i)e8`sUQxQ#UJ;IB-{*pP;m%!xT=w*G|dwIEc6S#;n zJ8seCCI``re(sW6HgkN}A?CcB48MG;rfAyA-|U2$daU8!PUb&FTPC+`6KBw^z|z>; zyv8MYT=tYVY_e6WgKXP(W=8N{?#vWrc7jhC`_ue2yZ7I0{_V@L%-+hOtVY5*QQyt? zOq*aVbLIKb2A91p-1J@-{>fD{8GDy0oZ#Uqk?aZ`*7%}~sQbxr)+Mf+IseyK^m(ba zeaNvihsOF^hJIJ!aA6?VCahpfh9t1_4Nr38XL&H^qbD%!=0VJVUEi3a8}zw_mzT03 zzofZ>BuDOxtBCh${!{yD=N*}mlZv^)(s(A|mljVYw~~3yOJrn9S2Dxim^-LMS&x&Rx0} zXM$V>)3f^;b6~VPKW0%gw|?6#E_9iLXvqf`4xdT56T^S9Tc#-R2b&-8IJ4bM=++K) z|EWTefKP@zu+vxI%}~h1{s{@Kpi6+q-t+ayMu|Ze9*wY zh~R|P)@+8FELZF%ol(J3F0ZAV_p;ZNiQH4dC}j9@>q>ojdfzTe_mfm|tvzqJHm7lH zna*Y|_0d^odlTFnDN(Z&aq#<*Mg}G>f}y7 zJkND~*v@In^>Q&moy@oMzO3W@wcPNMg$^l3WsFJbP`)oam5HbnFwr?4!YKG)67CT z>x^Jss-=2xx(SnU`WLfo+hy)*?K&Pd&dAM$MG zjg$udn7W4i)`g7fdNqf^8K-#v>>3-EFPy|QDwXqk6e4(aX4!UOE26mv7ZMoZI|(bx z88bVyU8Q>aJf^-u>WP||?I4=h$pwz?X4d=}*AP+@XCJU(E9bR4l9^}`#_=DnV77+r zVBXyO#s*u<=U%Q|>rij(S(1d4gg1R+ydZqAGmZsEoM{Ow>FrN^veYPdf` z3VF>Ns@N)%cAiz(B4$AvG*oVX$2+$ro_V@ZH|syU7W7e6K=_A{JY3g&$!I)PHp1s>ewGx^N! zhZ5cb)w@;R7%~m7LbyE+5sbHMO#Paa zOkRraC}#K2OqTnp#CvuKxUMHtcwcIVGx4svyh$r|vp>SKxQn;rnNix_4K9C`cn{)M zbG=m&%*9tf9L}B2uuJuM#cPj0#p*0)8hTQ*cr8tHq`r{3%to;$FFAMyJD~NO_iWBZ z_T3I$saCGR8)0AWFiCWZ(@gSjP<@`j9{X3rjyV;_ncsfqaQbPM!?*?6%=`L6o`L^8 z_TC0QYh|&Jo2zh`k-h(w?fN;5nY1XBt^0bM+4)}&yVJFmy?bmnXJ?$w6wF!M@J;1M zL);G`x4?cMQ$Jme``}7=6^4_z{?&`vorWd6Gn>sgg$?7F#bkwprKtvYs%;pryugBs zI5mrTcIT%3)^HQna-%Wlw==n9i)3c>_S?L;gD=??gE`#zU8TGc z(I)H=?d`l8I+5cGt}#nfgXxWJqa|@;kKy=-yWz=18MtQP4^xMpgKmcssy-!`JX5(( zepo%h-ziqYnA+z=$uf$Rmlq@ZAu_=%WQN`--AA;E!g>Bh_E{Arg%zLI5j!C6U=i% ziR`(z{FBrd+MlNiHV*QI_jG$9Y;`lvdL2QPjS|tFH3R46;g|rGv^y^rtk(JCrks8% z&}xLO(Fx*!uoMDO>JsaLd}1i+AhDU%SbgdtHT_^qg^s@=(rGlL{>-97{vAV=X`Nu* zc^Hfi2Efr3p7i+Kx$t_hjwm&4A#uFd@IZJ`{76p@4B9dUcQl?tPjoq0Il7^FNip7a zjfJzJH{fp1Ke{S&2AUa{fkJW>J~_G(4QL9;7%R~sH!qQu(~{xcDH-96Kr4uvT?Lw_ z%K6c|bg-3Y4hOtL@pRRFBKJiD{4W^^A01Mm-QIu!s~6+mICZdxX7V$plx%svo_v@6 zL!*@z!E@7xxFdZfRJ;ktoh?}yFnCd5K0!lb^Y$v;U;c;;`LvYI(|U%A3Q@#=rV(y9 z6iY@os6y~^8uhU-F5DYta%C93_5(d$ zszDB~q|&NB3Yr_ok{5=TfTxlQ#<6vzI zg-=(N$l@3O=!-4GVB+*bv^+Nf?%ykiuluj_H)@;~_$ZmcFvI^q&G|anl{FF^YvrU6 ziMO;)DHToUPbX@-u8O_5HXvEhEI*L zitzEX+)%1HHVQN59|fhi)~H^#nRGuD3;Z1O;6}qO^52Y`WPM8m+QgfH?zFS$ymkoc z%qjyHmmQFvJC@9@y32_Rhf4-(whQFV<^gtG7m&K8QB@$~0eU7Y$XNBnHG9A7U!9B)fnVdafX z6s2|HizkX?lK&uNZFq)FZ+D`bs+Ht*=3L143I~Dw1k!n>2U=a6;HioaE^gH(lJJ?- zy+041{4kdc9?vC>jJ`12zXVKN<>B?h;lO4mz{`&jd5}AZU|v3=NtKFsWgxfE`{6pD)e9@-AbXci{qYxF5*Z?Y2OL z!g@0Qri^4p%}XgOX(l|r!EhH89Z0mDr(mSX5vp~qmX6PDp#NT9!2fPO5Z(WMmGt~) zNi+QV1lh~$XjNt$C|mGw$ATbn+22Y~Uw0T2e@%lZuRpZc%Ny+i^kISXSG4}#N9}j& z2%GK=#ntZrsO|Jo!r-qj$?3yl!k@Q+mV_(f;oxD^;A0_-4oiY(juwzyyHmW&=Nq;A ztSS7+77Mo2mrz#po%&-Nar24B_gC6PAue6u+-wQQ(l^0u*y@b9UA(w*dOB=6S|n7g@Lke3q=OqQll=(Q4_ zHyneiW;NVc5l!yv_#V%MP-9z#ea$9 zfLR)zk}^ytUA{*APE5tE^KO&nf2ZK?_iqH39UtRLyL4P6Q-dXPhvSJ6G}{M$q>SU-aPjj?ETL_v}|{VRU>6HTL>t;Os6 zkVa%)r=2m`baRO%zPVhE3JTBRLdkxV-`GGEugHLEd;;kDEXKNPtt6JejX3x#2oDGR zq5Wxn#HGlJ+PDjeoRoPuPu`Yte;-iceLZ@~QW18*N$VWTPV>&qy(_eKle9#N9aVB{p)joEm8eI{S)ivtSwq=VgMH*95VP;Ntv z*xb+-8@G~G;OL=m|ixO~OMg{qi zX-PyozL1-_l_G^J5u)PHF=+F;N6JQMr@`uGVCcIFdM~^~HNy-Neq$`${P%<`ELjZ^ z6Jo(zEf4zc9E7@7861?mPydV865ex6!`CkFF)TZlB%~FS`0)GC_Iew}7;50wkGk}N z?*w6j-!;D0t<%*0N+~M*+5`tR3-Q;9Am($#X>|4(4QrM~Q>R5bn9+pffNU8adHY>3 z>EsFQzOWyQA1a`Z`A1T?sT=i8C&H_cZ?H=FELcC94bjg|V`5e%E@SRso=*fh>b;*D zmP`5Ib1Ok3;SUX6znoq2)(u9SS%!b3M!#+(*-Ab@o6zBT(_2Hf<8$1i3?pV2~MwA2*+e(8cHQ;)V#4(wBsF zy8Qy#^rhe}Y=%oJro!u;ry$_x0`#ThaQp8_*k>FKlh0a6;x*#o+S}p6;-FP9=dBM$ zHUU~~i~zY^KhW#IMSS~6Rmdxgr@NP_gZoH6Ew7kO2HezP$eM#F;hiUfm~vtXA zdg%`%u+%$5Wz~mCmQi_0%h?TNhFuVSa(^tnTQBulv!}^hn_hb7sUxg7_zvq7vvB>B zgMzkOoABtVy>P)*2-${5fI7{<^G;&AuC9rGyqu0_kN8l(P)#WtZVK4F;eln#Tq1Kz zMzZ&pL}2%1EuOz-3r$1r!{HyAIA?zev~ePc{WO=x&m95Dj|#)RW}1ClP@Ix)CjzED4M?M z+D}Wvb6~7v3yB+f0ap0G#$Tl?ajBoWpu=edmVW+9(u{*)OuGvV{-a81! zM%jrjZ%x7A=0?zdB}?D!O{b1BESy-r2ezDVrfqXIF)8+~IBNnAA~q_D^?p8sOYgS` ze=u6$c1&J)|4=_Y7BfJU2u38%x}ZiT!bp;TqY8FJV(M)2TA0_NJsL&uuCu*KUzlJxBq zIPMXX*P|AqZ;?LuUl8DdLyt&th&?SX|3m1j8uG|e3;ZkR!IPO8n7vj3_a$4B@M9-1 zZ{cRRZJ3T`K|5gbxCX2>vBtpJQM4^-8VFU>sq?MH_0B* zA1Ws+1}~!4Xd^HT2m+6f9@v&6gUz19X{2{2jkNZaa_Y1&x>?Hn?y3W|wZ+hL$)7aV z<&%{L>Vkz{VbrJoD$EZy#lvp0l3)25f|#CXw7=CIR_)P+i*BZPz|{t>ww!9cN29i|5#8!Lr7Uq~)p#@JSnZtWN;PW%J;~$sBxZ z6G2u_(}UL^?m>OQ0+{`95B>dK56;;+kxkAA@Z0_WAf|T=8kdcS1r1Z^9{Yu4?5z;^ z=iorHIB)#=ehaGI@}hqiMc}QFRG3}9inv(Zf`0WasCPpj|8tv0l}mQOQ=K!QF0Ehh z54lJAL+#OL8>z>G#kNkXNPShnFK zs(#c$-#MY^w$4z}Qt61U&SJ8He(pjTz-2H!^eBA1)q$^^PtpCC1L&yJ@5pQ&E7;khMYz1HIHt`BlPitU z|IS^AEKMfvk0YS9OcRF%3e>`aMG|#=n<;}4dpy?TzigW9Z$+A%IZK_ zCKP^nZN;-2E|C4247P5m6%<%F!B$g0bmRNeW%aYcY~Bb`=dqb~a31t+iaCt?q#%6o zH9&l0+z$TkV+#dxD&s+2BqJ2;qaH%w`32ySmPNyDW?`Ct6|vmZPpUT`!7*>7bapOU=r z4|hed|7aI^?y7-rCyMBk#7x52QT}kJLok1DE+$p=l1`a)NZBcYdDflu?4*ORJ2R7i zLhcOtXV1qBgHil9tE#EawC-AoEKk?R*ZQwSLhhOJt0Z=!SH#QFAH?~dyTKRl)7vW+Ndo5R z<2>(HqBHz29ZC#n%$0oT4EqR2-#o;kV>u|(s6+OPBWdRmYkXtG!;_}znNcz#t6HL~4ft;b3Zc8bn z(uG@8V^j=Hni~n*#y-UnR2My0`q9?0j=-C%F34=y31W{G_~4?VB&bpix*F4=u1*0n z-KGjptl5Ig#yO&W!zuiAZ!Sa$m%|MIc#QsbgM6AHh5?rf2$--FpFVGa=#ArWpVw@( zG}J@aiOS;4_z~DQe*|W}c#mjkOR^U)hZW5dl6%u%>~wno7iC!BB~LM|+u}Nr_a!N(D}H}ffOk)C}G6KT(z zYll{2bWz)g02-oaQFKVlu7$Yy%G}OzCk*8>MbVOv!(+z2k_xHgGWJ~So_jEJomF2gG%)< zn>{MX3El)%BNAZ0>IV4odL%q-H>acTnhVwDu7ojOTLe9WK^V}JPd0CDr7C0<4e0hl zkEHREzj^wYv{xOzE__ZPL!BNEK8X>N8|a_MTgVipMohMykMWNWVY{UqEH>^Ghy!2X zj?Ce(JtmLntGvgWiv9FaZZ!#0??&IVsZ!q3Hw>26`41l{LUGYG*noq?Qf@X(@i+s% z%k6N_-SsFa*T$kAC#qy|1r99r!#9sLz~NE^)wE55rG7cM>8>OAJ&^_e^kJy0U<)sP zKO+{4UeS@&R;aM8A4}Fok&t2jFih(i#Qo71jx7r!wl9hy+{1}vDy)QTSq}(#$x)TS z4){0z7DiSjV`iG6WRq|Qy^>W0nV+Tn%BA_R@F$X241IRd>;;~^;Pvdq zsPx4d7V~>)JDUny`gVa}xjT5;UEn{e>!AjxAB%^aTnq^|QVs5Jw^XyXhy2f(Xe84M zx@Fp+=yVIU_SAts^G0Nwt}Aig>;j*B1E^+J9L8*q7puZ^JT}=_Tu|UcwYI*3!d`t? z<2)VTKU#t-TlZjU+)sM=a2U4CsHWZ1Ct*W+I_&S(CT`P{X!g+O&``Ss=L-sml7^08 zdv7#q=6Z=szJ8*U*4csSicVaf`QFlVzc5_)KDBRKD|o@yc$9%bsCCobP29la)`Q5JxgBs zY2vnLZDgLKC$xQf4hOZR@6vQVoVVi;_)h21HiI5Pn7b>rzY~HZOm;%MQ4+N|FatDv z{fQ}gB)+*fT6Ci19$e|rO!8;;Pkubn`3nKA6> z`#}bKq%3HsRdn|Z4f0xel-!BlLn`v7OI%9dK|<;%nCR+(#%Fus)L~szRS{un_fqhg zG6BC&E&$WnTJ*5q8K^t)49iMK(t<6AK>Of2RBRhB&dh8hUu?ACKpGFW7aze@eaEQV zj&meLPKHVswV_P2ChGEXN!OX3f-H?kcy#h5jFT^=Csr5JkYxWE5D?ogcB`1@ypX!6y4u)H`+pt7n!%CeP^OB$nuQ}UWhzd8c%zCw)E1DDQfDdAyxq?ENR=t5$~8=R_O1V-f+Y0`o+>UDV-5 z?;+TAz7?Bu&d|(#n^5ocS5n=dM8$Sl*rCo6b5SY8TSw#KXA&qctj2wb1A?mxonV?J z66+l}fQoa2uxG;w9C5{oUif?)u5Q*5@hKC9kq+__$XtL+xeAfZkQrWT>FWLV-~@j ziH=|<=S3d=*Uw+QavL5FeL$Hrp0KUUnF^XcA#K$y_%^W`{`95`3PYo4v96HJ-W3PQ zZwCb5vO_=_(lNT+kG``L0Atc6o?)s-r|`zmxIACt9u^?>{`CmH#s8v5bT450P(76Y zxdhqvQw4GVdPn|Go*Hu6H99QeJ7Ia2KS^7>fm~ zPT`ByzeT!X^60%N7Mr!xaMaLwxas(Pazf)9ngpkTa%E15l5uS9b*{jXNoBnf(l%rtyi&ylmQdITxP)s=-&g z&Ee{{3QSe9ru$C)7X6EBqS@yc(^Ltc&iOT*9$3{wSI++mVNx#aw5~k3{CYfbwKE5G z(~G3z^I0-w=M>sM{%^u)5ArY)jG?8B)Pvh{j+hNLQYw&R{pt0JMp?1YTa_-k{n%i}k@RYA$ z=`AsqE)OD}xewT4x{!EFWU$Cii`>4?M}<$1aCl%e%E~%Rbv7Tk-C&NX4}Gc41fZAI zs%YPQOQvk-58_r^EA8#2z5nM_(0djjxLedhR?f7dHNKjttuY*JRyhD3`bWkOCJD&= zM*1^v8_FHJhau_@@Z#S%l&iRk#kw*4BVqw`rJka*0$0NaZYeQtVZf-pp59GNrs^`l z^)vMPbeOVwE3lQ~oG+Q&|erzaF5A)p>ZL>IMnbE{FfDLQz%z08LOBAyImh zfk%ug=xOtEnssa>8ecww>5WQQddv`)zgC9%+3Vo*j|=$AJRI+QkAj*lPw3O0WXKnm z!`yR+q`$iyA1jBEQB~Ve^`9DCNPfcZzoUnDKE;#mZsD*%s4aQ>{4=~NP=?9Bq;rOE zqp`@e0ve2`f_?f%IC<&}{#t5Fy=D|)Zs-vF`8*5v?rem;(tcX2ei^DMOM8yAd zMX+kzJg{`xhdOcgg3CGiUvrLh*!PKT^~T`wqBIP&t%S|;_lbr6W4thP5>5Ew z179w-!PB{`XwV7*db%9!xl##NH^>MBw`O9sSuAMCFtl)N8clOkgIvF>1`PCML#_Q5RO@OMR*1p>-~t@V1rMzN)~#(s`(lSu;)bNakPK{6a9L?*!^zt00z3 zHel4cLwL=v0AudS!byeea4T{-ULSplR+UYIbH8%&Yx`5Nd7h$heXI-ae;*>tWTy?8qvS%NKR|*z@`5Z1+A{C;P6^P@|uSYX25HHE$nV+4==#VCE^SXm#qCnPvQ1TD_R<$UR?ADa$?c~#i#KsP z!`@@}mH|{M_zLG7zR|D)s_;609Bh2}0WEfG3lAphg0n&;`DrhY78`~5e!n9Q`1QfV zsMU~}yB3odXM)~T0;PQu!RYlE*#B8ZIEB3?Ske}RcRq^X$L5jb#&HR*{#Aux`7cOk z;0VdLBl4)-h164&+z;AQfb+GM6t-pg8g z{8X*@iM%#h;6Fqn>pNUhEuBeO9BianQ(u!e(>3tL_#Z?&K2orE(iXC7_bMz-u>-v; zztM3ZnoJQJ!q@Ufar0XqT`Zjy-PCzP=C7y{nEF%V&$tRUUJDZZS0w|phA5%ahuaWg zVMWAOKGQdcl!YTM{=uJ?)$qXiIK3N{2D|hWg=*J2C-EM{IHSf{wzya8Eo#Ny8$71VI<#eV*Ptqfc zSpK+M5OK^7NBv!np>o&gezFEjEjc>2-H3GE`XRR6y&DFlvj$m|Kns(xH%b%t|^P>@h#wp&jvJ4egV$$-$a8Uw*hLh-T7AQ{|cLac7IkP{1d*p!illT71jPEHIw zPEf-~W3ypLk2P2J;0IQ8GveDn`o&{D7tqFFMQBVqNyq(cCF^S%#P%j7_^eSKOP>rA zTgfdUH&)~l&4>!HG(IV~S2JHaXS**jA(6<9Rgf^JB8jf+4RjBBL*9 zX80AD{V#{az6k>eXrVU}6Exan0JB^^N+tJw-I%QsTZ{H^F;UxbTVx50++ z4`fb=v<|rP3erY#X!9ddEc)a_j`tOjr4h-{+`Syay6%C_({=di{z4q7q5~b;Pe~7R z8fRza5V5%zg!il9oq)OU__qQLTPUKNj=6%dm$Zjh41ygG>;&b%#^SV%Q$aTE6AiZb zMQjyU;upUj`17q4Q!kGrVxE}1uri>nH;>~iducD9p@bi8{-D;oF!)xv3C}T|yedye zf$XW1c%yh9RbMj#T?fvSeRHnUMOjTMeVDXJ1BGYb z!lmm?(EI2$Nk~*i7e9I7y$O@)BlXF2Lw_VqDO@K-WI2P%(gWnY{b{^maT$vh^YNv9 z0%RJ-ZW*xLM?ItQc+QvOH-8O7(Ys4fM!-`t7Mru5K(RU`3yl?%qyc}cab&$ux%3&v-9 zWBV78EX3D} z<1oZs4L&$jp#8~Hr0&BxcqQ%M9KNcd&A5XYpQ8if+wW-0D^>Ve=0%0~&JzEZ<)pdM z7!6lc2xP}-L0yU`y!ac9{?;Re>yKu@w$y1t{<{ef^UWSVjg)~J*I+E1)Pk$R_uGY`QL_JDX66H1>r;a)c+y9*=-AzjHMa2+32kHslszo<;EaqQS zAVEzfWaapoSZD4J(W1k&uV5COE1HM}X_G}yrSlkNzk~QhQA5}r+rl3iE-SI0I01%r z#o(K^94P&Kow=v+o;)=#q_u6Y>7L9~a7~KG5mk)vy5CGX)FK;G)R&^|QmKcd*9I*H zRB`MqKBR7XMvX_y!@GNlWLAvB6Zc-&9_qdV@pK_8G zaZKE~<1Q`vHyN6?6+*`QuVnw68@T@cX_1k#2Ao)|1DzKq5%1CU#HHdTSU$HGM0_r# z1#R=O%dZSB%gp2o4;%sY%oK7lu@J8sOa$(OIjrD z51k7AR^J7ykK5v|&VTTBs|Pli&I8$1-@sl!7VZql!D!DGGF!HT2*()W>(DmzdNu@> z`L&Rzy4slK9zoKN7ZRuc9*IH^+y>oUuS8d8{r`E>fcn8Z^tbLb^x;RLU+V|x_g2PP zbB^O-_hq>E_f0%}w3ayAHRD*v4Mcl^JqBF;O%@CphS6*iF}mAKOa|IXds`MT=hu>= zarZ<+-Yb#B_@jcPcnv7_SV|sVFCecAC#5O44$KWQ$P2HPx z-jUn%jmbsgFE0;k#!2sx-r=bCX(%jgk_9f>AB?j?peJrQ8XQMh|D*#y zhV!s8AW&Qe?z*7)Upeg^nh(K|UvQ%1dBKupLQ;_YMfyIu3^Vt} zq1~Aaw9+7g4u6pXo;%%PUc0>TU0s);(=(8k{PiT~6MvD$lgiNLY>e%TqVUx%OQ|mW zf=2W&CG%fo!+?4cOdGcWhRG@jpROK;FZ_J5(C!kn;4#4uMTWX||AuF0=b_*0YC5ZH z8r%sUgTJ~L;*)*0a5yXw``%b!=ALk(oa7FZWG@rp%gHcxMK~DByr;207{Y%QMItN* z#2dBeV&@!Lh{*ULUJ&k2tfMnfbL$(jr8o=tF|~MSmIg46TR=Uu5cZqB#69neXx{Oc zkeiW!PYhEgCx?I(MXz4Z6qhOY%&7y|@jm1{XlbpPA&vTM3bC&!w-3JSsR`$BF}v zb%zLIiQd+e6NogVNM4Ps6zW-;gx!##M&$*xHxfkbcwC(A85tnfabIe1GZX@Z` z%SxSIeUyeyKc7#}N6tsJat%yE+7a++x_^Ft&5^ik~epVUo5AIX_e2B)~P zr$UtI=;1T;!KyO!_vipBDG25DK3I#Y3f-9Edk^V|0%rf_%A?YgWrED%>8M+IHma4E zLDt3^^!d#Tf-boNIv|_K1V0>M?pDT77l9TUAHK{i^x2P=D$jt&WEB{XPu6I@Ac*R{ z?&t13agOSSI?$7CM!a*u_N=K+2Ib|=L$hY)B709+fyk?x>IRI{kT7}U)zl+A0DYa8#Wrd4X_8zO^-7pT zPjtNz)b1^z$x*jaypb~Rg8@ah3!YH3<-6Et#j40*>Md%La+$dsRZpYLE-@SDZR3{C zbVmt;H(8A*Pq;y%-GIlOE)nifWcfY^S~>eo*Xk2p}JMqTFL+W`89w9!prLCo%aJ+wK-mfr0eW?YXYV~Yl|W8L`52PECW5r>tG)bzNJFBG$t)O z5RF_7Vl&ni(-mXmbnS(0bYoUEb8wpz?>4;1YCrvfjIkTCuG2*)e|FQ3%zM=v==ok} zHs?QG>SFL$@TNqPzD^U>!uW&eMaM3B-)aH%I=hRSZdy&RHz8)(GAC3dy&lPQ71EO< zNeuB=%J4oYFnb>t3Q8T#>12^UV$EbTS~t2FRbFJ-_dg=(zy}rd*{+zz-LXYt`$fCR zY#(N_Q5kBSn8&PKS;XjaF4Di2rHnOGMzc)_b0#iBaY1_+m zow&s+?de0}^{VKY+5nRj#BlqQsjVtt)Xr^xPzzDtb3S&;bc*VX^yUJeimK7#_{ET?Q6-?!>P9(Pe z0W0BZF61_>q5;1OliE`N4a*rb~xuu{z7f6&)<$f&l3g*_U~W=;WmB(;~CZ@h_edD6^JM>F9p*CKM+c{;EDA4LB` zsu}sQCi-uWXy#r}$*9!U(S5rGsPR)ZbMs9jTC_eD{S5fQ1}9x49v2p(*kpNTpV($b z;`C7z*KLi$2b_?dhJfvpTZH_h57UC=K<4SHTKeuNpIMUhK_KsK#SALS@HRi&jPA)# zU;-cQWV~10ZnGO%fjkXnBDr`ubki`BeI&Jjb=~<7=_wRb`S?u6+B%*IDOMBCNGM|l z6Q)q7x2DMQbun{jvNJMxy9#+7R-|qrEsR#@Vs_C7Nnz!WXx8BIUB-ugVq$()BmF4> zjM~%tG<*P{3y+-f;R2+DSWlMc~twwXRP74eh zX4dUhq-|-M=-5<8l)U;OQobKd1C7d-Sm()Vi?cK;vR-djCe&&j;IEz+1 z-b8ntvZKHHpP^idGbp#72w`wfu z^qkMkL>`|R%G`=JZ-0fNwc8k9>EkGad}VaVZgkDgo;Kf5L2Lf;*zFs_*sw1w(>N4~ zK1}vQq0uL3q`DvLH()^DgBj?!g(I`@R)C;l${X&0h1MyLT}kf>s_By` z72(O9huM^GiU_Zfq#n0MnU<>}olf9#q_$%Zn!oTimGioUu7(CsCA%QOpq9FDWW^jB zYjX&l5Bk8+Ic{`u!9#AzA}=OB*$de=F1gie2_ApAPMo5OE6}>3g{fI{TFq z4Jf%NNN)Qf7->+YHH}-*z^3br>|0FDen<#&)mBlh2hqGWDpJV9eFM6;b`|SsycoIE zXR;~hI8?<)6Fr^zP;h0!RYBDB5gM0VkGc-cp=00UsOiST)OhYeYLK#zt6vmP13#`n z+wDIwNzDY^zTk^q$}niDb`F(3UqqqmckZis>U8L{Br>|=&)9fa3m-=$hWnFs%#MxW~OITO_!HQ`vpN$4}WEMk4p)cl_>~cmsB99y3fp|qY-qX zia63WnZ-_<;YP3JPC^B*ov65UAljHNWNtaW5)@$%#@h-bhZ--o%u!M}>bjbO@40Mr zoDCcCuLONCi9<3&msri~I%q`AhWpF>DV1Fz%iHRdNEg&TLV=>QFwG_xWoPE0&sE*j z#-fB89W6!DbDpA7mEF`vt_@+?=ZtyoDR%CoL0X}Cl?jWt6#Y+U(=9r(sCQ^N<*&QV zl*34nw`>P0JX}V79?J6GksXYeRw~+Bxd%P5=b?Fh;Z#S?fNuI1%(S`$doo+mpE#i{+QvXvgnZVt1d2auuFcH3~=vR}71y>o4 z21L7K2i`d}Ygrj{YfCc|8>%eK@$sP7AHAnzUndHu1lb4zt#Z(l)e}&SNf$GY_MR*<_fq-_FuGq)?TeXuO+I5NES?j}nbl3|$P&>>1 zD|AC2H=Lkh(=T%KrJUJ4j<;yVrh~{(zY2|H^a%>=cCvGh=+oJc1SmyeH@o2KL!=ku zj6O69QJJJYH#l1rP3>Goqi(Ka9`v9 zii0$-^bgf~*~%2NeeFlS=2JU)4wGx%n+Ix^Hg;jhR$}EL|yUIO&IAzK$J==#NI1st(Z1 zniBT=;2%cCEtRUvT%i7J9oOJA7cG11hL{I+H1*VOlotF4tqxp`>WDR+l-ELom6ho# z>v($H`z*UrYYmrQQ_tj0XrTU%b*Qd!wm{~g6Z$B3i5nEw#mbJSFxOPHP;YfC{i3i7 zl|KuiiwYFEdgmuo(G%2i)hQK2JYrH7f?v?US|ElRj7Jwt6=Qd9Qr_WjK+H%qzi8v(uA-~>dwofj@NDJ zU*0$BAF6?NMfZv@i3C0F?2NuW$)W}=MYoSmfR8bBZm^XMfHCL zFAFTF?ZUEoy|0JbeDgr<6IN3bYYQZPLksDhYNu8KSCQ7`3Uu3>rHiL1 z(6yDaD8wxWy>OHjc0K-sUQ14(aHTP3n$_Dy{f3vayBFJ2!Nm}I zFLpZGb*);Uv2K{X^+B7~4yDoI_GG$PiO;s=oMl7LRk0C_HLBiTjU1lOMB$fGsojbH zXk*-Tc%l10YH0F}#*MphtK(X!!hVLvuGgUNwoYfXuZo84|1O{xW$|?R@c`Osq9`m{ zrA?zAI8pm)8MM7*8sl-`36nQ3688Ew)a(cJM*V&oX!(i!(P;&oJr+7ieH#JBq10%uY(sqUB5bSZT!q z#ym2cy~?^F*S4ipO~sbZ{g1FKT^BKTmtR545-8W~@d;!ww1b-Ji1A)(rl8+33apEh z=nghri558)qI@+yYAsXAgdRV@?w@^uWfoS@k(U?P%AKb(8EJb=nzoTq)b2(kb z25>iIOk(P)C-Tzc9guPJIAfDPz?_UZL}z;*rL&eEV=giSjJ8%MlN+)Il?|UpL%Lb4 zeba2=qkFU1-+SJ&nzMxT{$VLP*`!` zE?J2BP6n!S2^)B85BtRL9P|3su;7qE3({9nPizsGGE=Ezrz`aY zx#;=2TWnmYBbr~ei_ys{Lh^~Ff_Pd+SDNY2JiB~UnHerHKYN7T*ZYszbNo4~{(BOs zripZ#{Z(k9&>jUemY@?cV#t{@mmOv85og^dR9s)MJ{okyz7kAUqc zm{W+1xalZq)ErHWy+_j;#ZalRgu83qYeAD>1~W}hmUg@BqYg*UQeQ*S4qdf`R==)e zF0WK)qDTy~+uFn?e{yAS2odx6$!ykHv!D4K|6|4|>H|~_UsCl~!6>2GpNUb( zM&Dk=(pd%9nV0*IqaAZ5@;I|DGhdS4vgXsLAT2#l!666Hj_F=Ah3>ILCTJa6s`QmT zKc^J+Hq1a*ZY*KdF8ZO5+E?hS%%kk=8HPwCVPSjEcGEp8-m&IES;$GKK?5zU$j&}d z4Oz08`^-*>nPHa6y3cK9EoWb$=VWg(eyjGgEr&nRK(W(|%DhrqNzS8m&kMBFV+C_! z-fvWLXd+AQq%l`cjWdVm&ZHif7Sr|}^Xd5E9JE5v!)_A{3xeNg(71yc>^rYzG}2j| zr+)nu;~JVuUHR=q<-+A*~l&y1`tDR{q$lqfn4qH0>-3#i15}`ox!*O0m<%Uz_CrK8JJDx5 z`YWD^zH4F{6P!=IB$hM5dkeY8Z+;R;4A-%|Nd#68Zf8z&Tu@-E=>B4MGv+Ec(dPIJ z!N`9-JgkD@SSa%eg+-iI)%4lVIec+hKlfrZ3_K*#S(Q}o~LCFWvFPr6i>N!mw&gw9*htZ9d0vEWBUAf$8?~{% z$|zpn$GBLyqKS%cDHEeXjh|eiko$m9mzs}8Og>Q0`Io8f?&m1rzZ%q>^iU9R`5q(6 zW1#am)9DKH&rIB&PWHe|ndABL3j6~4%vmfcyHI3}JwJH;B zo4~kwilfzL+bI7}F-juz+qGTg}v}W}v^K^iI!>;i%E4ge_hwE7(hwX+w7z!#|M2 zF8+b($%bF3MUcttx~@bAqojlak!MhIcqCoYbd+_5(|ChLO-$u)cdGSI8x;trQt>@% zR5mxAIrs=7^{vJzr)h@Jc&dPoz7y$nwM*F(1N)h@C}Xtx>qZp$V>0Sbw?OJ;inRP} zH}|KQBt5k;5^Y&AgVD_3v-Lmxp?r{N4m(SlF>(>(IoGbHrLKLfI41OtXy=-BY8_b4vBTVqYoorrnA&R*+hZ$CHLqAXTG3)z9URlyrNNN3UdMvV> zos%_{7Izji(QglP(?9Fd%*7eBq3ADT)0Ks+_7|Wo5eJaz-G-jctYlVxJ%|2#RLdlf zb`ZNSqWO_{wFuW85$zOqvO|64?6gVmP-O0Q_T_eSX3KXC-ptKy^rVqDb937(6!s>B z?YN|aEDqkI`((vgr^98;u|$#X;Izo2Rj(*I(KdMAghz6RoX%@{~`HC%3f6QjN zJwmF>J5YSSFZ(4q4BeBKV4up@qFb$38LNc{P?OPOI=Q-$o^4eWp1QO|z#N@M?c}#0 z>+5F1;;gsGr)U=Y^;sSIOFh^Nr@t{9cdn!=$uWpmlm+YhO3?7i=_n;g63G_xnDZ?r zXwShqw0&0*+9R2V{@qJNzl3v9e_kN1YKL_5p4aS27d}I12kI8)pgkR(Y<%+(?t1k- z=>6+H>T{$TNqg>Ngvyhdb8^kd=01z&CGTVFrIXm&C0&fLv6^xxs?jaApO~k0&UCJ4 zIP#izoSrTWq3UM{ye1B!45Nt#r#BN4YY?RRT*AsZtE0x)x88hk0`=+RG zqZm@s%p&8Q8R)>%IXsu=`)Eh90S&1lY*|$_)j!OTvG}LV**(`-tF3@e^sq-Go4e?v zrw#N=2O2cIvo_orQFbxt=>pVm+Gt)(t9cS}IC7bw%D$U?gR-V@{+t0EkK#%Fd) zzCfx10T;h(M7h0}sYUZtp=_BrPkP}LLGgcH$cop_tVrL%{Eg8;-Os->(alyg`b|1L zd{4Aj+_F%xv2PYF`jEr6>l6x#JdZM`)XpN!y5mR>Pep21AZ@!T%_@G|!i^o}Qd(&VB%DKLS`C~4_%Q)!32xF`1nfO?`&67*pmX}eB4r`&O@ms;y zbHz06nkIUGD~fteETL?L4=tNMPWJziWmlR(DxLa{p(Q$W+VCEBd&&?kaMBe1Ub_o* zng2vJON!XPu!uI@s6aibI>LfQlhFC1xyV~gi|ILZf_})Ef|i|BLJLX^XzBe>`am{` zn%pp@>dx}K(?Lrqce?`%f7hcU%M58#)d#e@b}u`j^e?)xp$w&4Y@;7qba{mvAJFT2 z$I-S3JKF1osXU`?G225dRGzTtkUI=~G zdh>%{xWEn-JJ?bEkzbT@5HFPcuWizf2u}9NCjPg+A7Hwe3Ndq!#_z49fbk~>PWk9} zzW=$NHhsNiz<=*U&I(1s`Kp@%CLi&HVjI4+U3tWX_KY2rJW>ntPVK>7_a(_Q^E&>N zoDbN*=>z}q((O1^@hT_kwl8VgC54UW1dztA0jwTpMbv5+g81`cq{&)~1YOW43&e-O zwVrJJc=c|4V^{vHi!`1Yk6D4QF0? z#vgmQ2|T(aMXoow0oPKB{l_Y#9jc5%A=(hkR6;HC<(uwT}XaD(t}%hPO$mA9nqe34@?4UNM_v=jF+1b zACc#MFJntg%bT#GN4WjJX~kgvt&_l8(-NQbTMw8|Ghpp!|qwW;OJL~mp0CU?OLXI9X|{2mYWKVw{h{x>CeG0Q9j{? zK^}D3v6}y-%@`KB+d$pXY5Y0=E`nQ`YMjQZd3?)56N$(*0js)(Vc~8wyz6c%Njh>L zJkQ)mXh9KS2#?(Rs6^%l*^-jNc5GK31IZIzvU7S8>=fSyZ-(S@dW>h=OZtU@-k)si zqGK*_=719U{51ytba8^62a2)uJ}y+9c?5f3b|K9(gE{?jqoDFuDHgXAWjSO?!suvw z+$PZtKJMKB0=9EoLAL^Oggk>-S$#{+|Pk{dXNKeP0OF$tTRucmx7N zCcv^8S8!7thb+Q9IH0cq zFD8TQJL|#ZA4k9j`ROpq{vH1E+Z$%xiiTYaw1KW?FRtD25`<4a+A?(C3a;_8Ag})N z!Rz(YV1K_kJlHpzV?NOUx=U+A=TdQ~_w_8U)iHuoS6RbjOZ>qiwSD+_kPY;Rl!qVP z-N~_C5UTlYAj!*e$d+tVP@!{y?;`)X?I|Y{;|s1(q0oV-NG^x&MNzQq_kI4EH$9v` zQvcz%&rXSQk4nIu7zZ#=b{IsKw1L+-X;5CQ1k5i@faN1xs5P(#NH{iM&R&F4NA9Qz@OqnWJQ^s zJL1Dg`H?x~w{0)~@clBZ6XD$!XVT1H@*Q!8ysv?e>080kBffZveh0YV@Ej}Nm$VOdD<8Za?#E3p z6Y-_X(}~G{mvQ&t7O- zeyholoCl#W>!k)M=*xh9$D`qYpXG>9Y$B-5jl(Wqj$k*@Yd4$z1Y$}yL}lwv;#jqu z%>MohEdP{2LN-T`;4%YTt~JKl@kEq=G$wLk>YR@k>}TOt>#}3uWN(Qxt6Lj)u|~qM$x& z18#fE!1RYPxFtxG#ZwqXCQSRx*JzF;vEr75?FxjB@0Ib;i*T4c$ps$zxdgr;288&z z!W8c?Sa4blmu$-gL&gX3v(6q&-HnNu`xlO@kEkYC9>#8W`|z`x419CS2|VS(Mo`K> z%n!}T18=mFv1!YFdyd`)puDI6x4dEa{NA9}aow53_*5*g2vH?NvxQjYa|DTeS&GjE z#gGj=Q{pWftCQ^k9I|9h8z{N8p1gOBBZoFW#Pbbh2<*@y=T}|? zN*vL-D%g+bN3MdW?i#_0yl|+*&*MBRjE6p}=fI7n=keaMlVFv?8T{jS8}4}HPqH@{ z!uJOwh{u~$^69J$bUcxVZGFX{>{LUjcSei2?DzumjKfJW<4qPPl>yDM9aul(6v&^r zhO;M76|Ok;1zU_6!ro^MATo3X`KrtZ-ES?4L5>pHSa*_tHG4MsT;Yau7gA32s#oCT z4-GgT*bOWimt#Yi0q<^!AP(bka8x}R?ptV$vkMpFHOs&Ab*o0etdvMt{W$>?+pL3; zEfB8B*g|H#kmk(M+eVTMJV}d_A-Nxx1c10JId#aNyh=I^3chNQ=*jYM@Q){4C*8ph z&D4Z`WBagdsUI_3Y3ojq!)tHO0-8^5;Ah>xI3pGau9R)XRUs>2 zS~r9;gO~X6a~weAV;hd&hYbF|ueaKivNVbRxaS4yEQoYo+sQheH?cx zZ-9Td&V|=v)bXmnM?m(E%lNDF7yS7e3$l$50GJ-b5jhBu#Irqo73JMv?2ibm+?a`- zgEH{icU@R_%hk5G*^@c3kC%{jiQ@2ma2BcS%O=C^wxrGjMdFp%X1*y5I3x#`5I3MW{~CCD z<_h>!lZ{`CYvJY)9pXn0^S7DH!+V7f!7J%Gu;a&l&b*I2vc%jRjyY{2V3H&8^qmB2 z`x&4 zjo!qsB$}U})66$@uLM15mEfd(|2*x(zHqWy0yg#EMDETsgHcD>hqEG( zd2|(I{>=vd*Y@GywTduosw8Y$-xC!o@xF z;mFlqpl&e+%u*Rp@j()^_7}NX(xMi?Xk>aLERSXq+6x2B+nr%W7BR7i&cHeFDk)OMN&d`YoT^ zvJ|A%-@)Y_TG+TBit3jl-hG8bE^X$Kv8l==&h`tC7?vV_PjBHj(NDmxn2&99LjvLM zdXax_;{)8NFoEcuPR57U*ntzyYUJE}L)hTYA^C>xPYxd6PMBdkShc8>m=5QW&Vev|C#44)xX$9l z-gyYNgl}nmwn{_{bWQ_SLIo(l-vQ*;>5!YUrode4A$ack6YopaC2ir-By9HxXK9=w zyjkHu1~f(Zqc{yFMz@2I`;*AkCteU(*ua;QmBG(5W#GDI0660C)yNRZQ$*1p75p-mn_*-!`bVUO(MQTlI^byIGgvU;LsTiII{I3esJ3yzRMTo zLq6xgrB9!LB0DqkIr0)n{mLWfw2tBec@E^q8^g2{b}(z#a`0z^1l%Mg0bhJlhWDJj z@GMb3`D1G<_EJrQYx4bJ$)t(o*t9n|_>(nJ=>5uHJE<2a{+IwIDhI)nod$sEHX%u( z%+#ck5VE1hhm6+5^E03e+_LPtUD{*|qPXmC+xEo^a6dQ*6+rfYAGkKf0j@7|2392nkhB<+%gM#i>v{>SuN%U5k2m6kLnb0` zh#UAYkR<1N&jb0KQqJ03F>E-;hpfse#$4}2vcO~kF%+%@LctEO>v#Z!tFu6W+gdp9 zq7$kAv<5DDlmo|V;((Ir9{kX|7mTJ1fpgzfNk+DI;@*7KxoyR{u)#0Ni5BNzRQ%Oth7S7JpO!DT`O44zt z0M`b(!t;tfSdeZ>j%^Dhno>HjU~C=He-Tac*2xo{)|X)5pEYr*4-z@wErDC!T(WQl%G-Tk0NU@yWNzrn2o=fQ}*G2G)ofbyi*_^s<=9CLO6n`<@Vvo@db1gikt z#r%gK9KkrPUl%?#lO~btgP`5|1@LdrY9c5ygw2DFB;Q(_d_Ula>ntyTU+sXz#Cel9 z+wWjI`UxyI7Q=}xl4RfWUfhMIlNHz>Zy!j+C!aOpf{{G_yE}0p@4Nw+U3(B%q_Ug= zu{FSd=`JuvHj?(u>0rG|5>ZnMCtmZ#Igz?|K}<^sSiLxxpVemonP0DQ=(5dFzc>z_ z+^0aUn$3cKO%(r)Wyqp1GmyU94ET-R0$t26zWl#LeDtOQ++|~n$3y?&?Nkch8C{EQ ze+5I1*B@}Y@0q>BLKko^?+JGB9_K4>TSQv#J;19=MLqgqeNvli4V=_=;M89#aJA7= z5Emo_Ah`>hG=_-I*Ai$t8VNK44YBMqeKq83E{g?t%&F7IO89s1(xtVPBOdxvysyTOOyaXNP@A%{MbxD$30iN|B0~@i%OnK{1+RnO@n6_tN+6PnEKD!5;y`_Tr?q@lV_xJ+ipA(4b ztp>0$UIu=8DT)0)MZuT;67cCc7dZOgeC)a*2`>5uV4n7BvifZ}p6?e=u8DevIN$sD zr$-_`V~sR1$YQWl;bUB(SA+8^Aii&Xg@1F@iwxe*0OzhUWKdd;y#1(35+6&$U*R?U zum8Qr%hf%Bi2Dy*g3obM7R<(FYuh;)=|8}vieA3NKaBG3fC{tub-tJrr96J`08Q%zQ@LEWA9$g6Usca+-*H@Ap@gm@UU@C0? zeHF;+?E&BY7C`;RZ}>K6Ib;^c!rQ6@U}J)fj>y7E*$Oa8b_6)pO(aFy+HlC_6(~CK zjl)@|0+yPc2leZ6`7IA+Vf%tSa7Wo6D!E(2=h==hw&y3{#>K!{0x76EU`>J&t>I4& zm$ZxEFrfy3JF9nc9KA2wo9n9J_cA{?$tOF2>bpWt&74g9Sf~pY{8WZ@ z+OtSTx#+V7ZAnCKBGIkTgn!2bKtjx4DKQKO^y*jyK)vEgf5 zm6j;KF;kPsxSD`HqI}WdJ>NKsd?i{v){kR;;ZIN`wUr2`Sd;mFQ*nn%KGydd!gYmi zxTSItte8@Wy-%CLOqaPZqGvX)SS1aLJ!g^fp*MKzn_9l_Ab=u`9IgO1jcAP@rfl{oT;=jh^x5ymikt^(xmbcDfM zKXbOpiIZhpjqv5ea%5r>7oWSj5A?ol28U;Lg3HhPS_1{SILSHq`@+v) zN4o@+{wKgsMLE^$JAQ~}Jo#ARw?8>0^^spWnogF6&n4%xRH69rc^v$FH(XpF0`DB_ z1&4OmW62H^_-?^`xYA#XKpja)`=W{D`t?L^i5m#zAn<#_H*iDc71$}a1IRtpBr}W7 z0RM^Ru;;cDf9D%D@_Wpl@cw1sk`x;vbI1(bx}s0c2HwS=?{3CvJVoewV1$#IDTY;} zHrUI=ZMQ$xcaXnvcdxy-2s_57lwrX>(|IG0+;N@09r?6E4%&VTBkQWe$zxk5+~^_( z=9=B+Xu8Y7HGS{Fy1ND7qIEKC3A+lcb(A3GS%Ucc((q90Ah1bOBR@rRHi>YCy!Lk| zO^kZmMJ-iK>ht(+{+l?p=O%(h*&OK8;l*i-Xu@x1pWufM{{|~_EA98|l;e=QGqI7& z3{re0las4>5*X6Iz(`^uDIKo>diV69!-W%gf3!SVbSDTrh)~+j>5tx8`%k@a{ zDH)Q{U5VSe7I9u_U&qxCG4@uz4W2u=1CzY1AWBmSY&DYxyA!(b3!#u7H&_7P+G&$F zqZe^W_8fARi!nOpKzhG6fYbSYWW!`gep-f+S-O%${7ej~=rJL@mD2E<$0^RDxQ8IJ zD2}u3;T#;cPZlOU(&W!FS&Ezf^M@%%+{uyYdNA!>K0MJ~jgxeffs#r;_V_QFzxA9y zJQH&i|CANYI$OeEjEWY#^fAN3UtKeG-GGs>G89YZe zfPC5e6H6DXkY&@p^Sy~J*VT-AqpeS-Ox%)5$*C!U?S5wRQn@-#Z>!USc{jT4@uYGRYk+59+;YU5r zpO%6rRXxT2N5g@dBC6|W$$T3Z6xTd%UG>S5!TiUtcS&|p=(b~=Lx(#r z=J2-FSx~W61+rQGAeARgG&5X4iL)Is=(6C%4y^+w*WBPvF<0=h!40N|7(!JU87P;W z3XZdrVVRBr^#8S(yb3wW&$IC)f4_-(z?${Ybs&@kz1sx8D5k)!jg4Tc>K@EoSOjk^ z)q~cZ<5)bm3;cgZ+%l?5`gaAvpFo+Mnz0tv;{rHTa~5Zh@W}E}9XzZl1-HDPNsKbD z;|9-#BE2gIE|)9$O8pn3<$Gp$uH+n*wfGgSudGyKSGb6b*_NWlrM@wRFe0spDjLH_4LBO-CX)c)e*5VEnxgxDPU4{ZOg z52waefXFOcEIp2hgwt-kye)*Ni6mt^e6)$yuL=0>S$)zc>IL2@nsSnzOi4h|J*>C1 z97Ofcg{@JSaLz*?Xc6fOSIvXQU#D8gL?a>LUpAav}Wsw18|&D%@WI;3oO6h8jIiSt||3qLM|_KNE>fM3&nj`KD@vM}N&K0QT-Ul%7j zYn8gdUdtO!?)=74{x=ogs{ex*ub&Q!Yv#Ze(i(V``!p!^>K=c~lj%fedJYctd4oH? zIAiAQCcw|-wrwWg@y)S#{yeREP!Ze+WEu?NO~-J++mXfpWp)J2xgigY8!rI`?@GMj z^B3@^K?*wPyam-j6FAk%;}pXa{9k7|WaOa=@tyqvOtKLp0*`*s`=ArFF8so;V-Mre zbjsJ0Gvctiw#3ux0+xrx6#95p4!0*>-m*e@W7HHYxk za|_@a#bx-dcM=TyxeA774d7t)9KPGS+xUU$C;UEU83wAxqsyJP?W;k}A_ZtYBLv)-BTd>9#Ync`aBD?)6~1(CCthl23SFl( zamZ;Es1+gxRae}yXBMr-Q7a*a`c?dY$#TSbx(D8}MxJE6`-wGEB+2gOS@^uXClUNG zAcpg;NSEd)5MP-|G}YqC!(3@{ZSyd`!>`1_NAKd-PqM)kPYqI-rU>5QLSVae0en4B z(x&v<4IVG~1`PUEaR!8s@yDHC_(5wsM7f$?kX{vlgSGKcs$c?~{GSDBjPAjqMghd( zoGgj|GlFhfFJ6QU1~$I%sNkUr#qX|rp|&mota?r zD|2|mmf>uz21L1fF&X|cnMk>~5baBdtcbP1ot;g%>fKdNi2XTWd3^?04KIKPHP1N_ zZu)IYY$xM6Bl={+#clXiaUlp@;14yt7!vF62kq@w!BtaVVYeJzxI^(8{vyIuf!7~# zUML*GZI5$s*B15G-$@(cs(_7rIIs@BG>?aJVv?|9$2fdgOf$iInNzTQ&>(c%$Jac3*YhF!XAEE&|*T`9$~8+ zML6(bCw7b|!@adt{AJ&70k*&xRR0_TljIJAH;OE0)l&!1bxei7b;SeDhJ$N~Nkl$J z=JsY1w8E7nJZJ#brRqfQJOkHH7GzspvF7-?B}04hUOV*{9O76s&A@PT;72jHu69E6v@$GOndaUM)h zeUBv)T;Ppld${oaJn(ID3unOTEk~kA9C#M`gWP|mpkT=z`_}vlj-v)2^cjW%R|O}I z&NFeo=LdOgZS4t1rdo4$w-0bmTKJJCY89Y=(pnO@z>2T|i1YCBYcSE#5L~?fnp2$a zLNeAq1uFcVB-1;9?Ct#sF6@>hcb}y|+k1Ji^zvo=N_uqO4GjUd_*%-z3ZDZjBk$vS zt~=at8^H(u^FYk0d01}64AA>T36rdn1V0CJ9SJ62X0!7q~!T4BWZx0iwEm|3}ez$My96alD=OP}+O%NaLRK zxu=N=l@uZ*ib|4*5D6`rB`vFAC8VNzPWMI!$0~$%=*`+^nlc%oWs0ZnPm0v$mv6=kc@ld#|w7Nkw?!fGjlJp#md4 zzXLt%xn$RNX|Ski3E9%pfEnXe;GClrJU3;8d2-GMmjv?J%`U6)gT1$b$-R3(MMV<& zxlbmchrA)Wr4RR~Q}E5+qxj?j4;XTR?|T9}fNgmT=+RjK?%Jj?6V09wp1Fec;p9)hj7Fh6^v?L;`gI2c+Ip_(!5v>0EcvPYfc6sp(nxC({$arHKGtF zm_a-DnPh6pOD6rpT2kxKgpb5Mz-3C(Ff=BTtXEw|{yF-P|16H;&6DR6o+c%iI1~6W z?LR!TG7|83isx^Z-F`3V3KW8LHl!$5gwP+_eGKXmNy`xSJpX2)J}n$ja;xzwJq^%J4cDe=rUvc3Uco?IS^*8M5fel#U3C3 z;-WVn@l>01@<(qfbeXZ247kP+MNKWJ@mdD-2Tg$?JuLN3-k7XX>jvG=zu<@m%~&#B z0Htqrf!DjLs1w>U;L&AK>bI5+5p%kCU1onJC>yP&-o(5B$wyL|SLzGEotj&?X^|Q% zjXFsUCy9|kabf7*oP`71u7H&%2EfG;ZLm$)gBO$T1HQi=f}1Ue8PobQhKW4N+|eJz z+N)=g%!xekYNjs1&T5>i=SY^bws3Lb5|Uz|K?>%mK>d}|@y6g~@NnsMaK0>)*!&ct z@{XpFDQzo=jn@iNbH|(E#5R#_{f=avGQd)wPJo(8fPB(YCRS%2P`6*6XD*k>z-FEA z!24|zb@bB~mw!pSz`BWxO#7=d)R$Gl@X5B9pz-_%DznoJ?|dZ347ApRwIxE}P}ycm zo_>kt$6K6FY!(5p96NB;KR;0Cv5-iVdyziH0TA_sK5l!bamZlL(7 zIeDz30aaIelj#CqqTBBRO;79xXsaiznGN8&IUS%a)7C|^y$^&wa;(ew^%#E~c#79l z6yl>2G(HFQ$VJCpVBS#+qWRjG?7O#`On)Z@BavU)cA?oUHL{WzLpZl2H}@x!%2=dg>!gzFoS^v=v_hXCBRl3)<|6)YEi0_hc5l zec2HdZ#x2lE|+6xn`hYQ@;<;v5l|bq=|O+TjZBHvYmj1_104AMYkr~u+&HcSvqr1I z?&FTo^5b!EA)Y21e?;NGGWMh~T%H_VlZY>y%ag=)=kVvzBh0?yB$)ZA1D`P54mas< zgi+EMXI+t`PHXY|;?E6u=T{BLD}IQVKm&f$nGfNLh2+4pFlyG+4P+j#)s_Dl4sw3k zk;n0M;QHZ3Fe_ddHtx7W1>E<9zw{=6?m7Yda<&+(?LaW3UjsTdeg(O=OR24;R#3c9 z95!Ek0<2X`VBA?@7?Zn?5$%7$lzM5xhmVAS_?~&xmVeS@OjDBlR2auGd#~fx`ToRX zi#1khjv?1~IunPv%b`I@7ImO(J6yVDAyikb1DdtY@Q-pc2v}RlNJs0!k`Y}Z?HLEo zj7h+2`AgueL|w3}dOa++a)dE^wDHoLH}M&lJ$QxrQ7~Cqjx6HDU_ierHW0PJF=E~> zo!icUpS#NNtAKtiCB2mK)hNTM4PHRHYYi|yr4GlhsFRkYjj++t2M!pEfYx`~ILH4! zHPxZ8PPN<-7MZ-p-j!BxgNGq>YE{4ukqjtuRDkKfl&K>RbcvkrZan9-4LM#QOpHE+ zLFZRGF7v)^g};Kgz-K!kPRg?&7S6yVp|0?Rl&e%a)tW!-I>eK^gcIH#!t>mzX1?G&zS(gQVVEc5!tQ*i$GWME_Lj_s;`H$aK=+x$i4C>Gap3b#M&^TUJVIXpbI}RG}i3XgO3DaP{eu;+%R#QvY3OQO3D?` zWuQyW^$QbUXi79%^+`cOHJGXG19?LyVC`9iQ>hie{?P`iC7>51$&G_4qnSXcwF#T~ zdBTtXQSjtM1c^8#P2Q@nw& zwA1+CmEV984|k3YRV17Oi~H{?z?C)CSi{PmOlYRzZ_nM}L_a?tpezZv$1o#gXg5=V7i~%qrrDqDA<36hq&7UUZmEbL&6P$9xq=GRYGbnPd8H`Q52W$E*;cjP+ zTJqr`{N!P}3Kd?+t>}y?Y7wry`2iF! zKEw>46C;VO((ti}CYj~+5Ql7^NgXr3MFqoI%;Bl~z}Qtwh_k=pM(s0TyO%N)@tQ%5 z_in`h{aZtp&NAk?{N-@U0(tWCcrrYz5(#_HZzSKtw}Atl>xr5AGSbfX!`CJHz+Hn_ zg2UViXCh6;msr6%m2xbkWJP@cEdm2u?$vGFvk6E(?F4cW$0+I5%H+Yo0C@8Z!Oel>0j_Hp6vkK)T&1yoyPA5bovjZJ*afySmrd_wp>zP>dZpP91- zKUnd@MKoy&NKlHwHjSB-W=|Q<2o5tpPw7FI+plqZun+X(++o`9T`r31!lY5{5@>Q6 z2Zv}WI8u2I922Smld3AgQ`v2J%4il&7%0R28GX#eJVp4nHVb%Z9svg$zf*<(GGWM0 zOY-yfK4?;!30wAfgSS~~P{8wIk_j6rZ&M}czvUP1y2{Tp!yVz(=|lKMun&G!ZG~Tt zJ)tf=`HO#*I^g5)8*u0BZf0hp3X$y6ynghc2Ps89Wc-^3*v0RmKNlSa(-#r|Oaxde z!4r%9sAIOuCE^=9MerBl=U9BWotbB^1A~?x13mp1yNIemzZg9zW`6;=w3p-VL<`LK zsbHV$HN4mRHm)f}Gu~_ZZC$Q;?KXwfDBwwQy ziN{@e9R27!KKh!)r_#o7mT8Z3!t+L~UonB-xTxZklbaa#<9qRmcj6#Rd@~#h55{0k zIy~GvAF7jaAW zjRo=0`&JA*;9H5u?@WQ)n-AlEGPYz%v<;kyH6j*)5PrUH1E<-A!2KJ{NX*i7e*duz zmMs<|R_mqUEk#`t;yQ*0KkOsM>-`GK=0J{q1;MsMr zm}Q+6c<}>a;;$xx{bDMph_Bx%=Uv{wRO%*BP>*M*N}juFm;#54`#@VjAala^P+irn zxA^%AVTivJfR!H&$>I3xpfg|%akO4UM4vj5+&P3Q9b){CKe;RGhvMJnP*LZt^u=E6f9Tt16hG{h1V# z>P?!apK*EHH5GQBZ^oNi-e9%G3rW|D2h^6slYzk9l(B4WX4X!7NkwOw!!9a>k!nQn zz{a=0VO$Jb)O-d1EU)ABv(3nmjt96qa0P!ce-AF3s=&mp!vKWcWUNaM0?89KAgIrc z*lCEv&YWBllABKQ4S(Rb?URYm?kObOfWo~h62z)#%;ib=GprFeg8h87$RsU$;`Iy> z|CmUe{>=nd#ofV4$pJ7~d=>29EDe9PNRnw^ePB$(TBslVfvQ-qL$<}W;Qh(V>$*?7 zL-85s@cXVUkiHNA8wPYC7jyy04Ap_F2P>%X%WgQz<_ji;O*m#@A?Wb@ikD96=GnIs zxJRGC_rH3RGU0ygsyv%~aNNo~u?r$cqaE?uisht0TAw%s4q{Wu3qa_IDS5H$HJ*!g zDF=ZZDJbi6HhJ9w3Io35U9Jvf(S14M7IGe2 z?E_mHEdDc5xpvw)tA5TR1Dc}}~?(xnzE*5-jkECb~m=h}=Xa$aBiT<#N98&H-C^$}bvvq{sr1 z#gXvBd0!Y%r%pVy8}aIzAo5wh4j&e8Vph$}1FLqo<5SY7@kjca%g55o_`KMB{NFZ9 zAn;cvo^O`|@!7HD*?;mxbWD!PD;UQ=-qcW`ZLfiUCk3Z&c*lq;S-^)a3Q%c%8txI9 z1kTEt!PC|XK>6rooU`vPmAUHi^{d+kDY9rbv3+|L2bQ=Klfg;EU0)l1eOOMZ#QXxe zx>^t#9Kb?fIjXfj!{xbKF|%|168Pw&7>TNmfis39;O|ctnAwy&F6Z}5HtOer?C!%Z z3(ty@>F!RzX^jbdb-)46)X1fd28F__Mplp!*$p!)#_{%;y)bh7Cb%cefbv-t1BOHj z@ZLRTxK=I~82+d2lJ&iqs zO}NvBhQuP3={EC*YnuodTE7gcuGotGTjHVkelKWwXbGIEXiaoC`a(XJ6<)XkpuAoa zHaTMomoT1i_80z)W@?eja8p=KHGs?+F)&2MjRagsgB4E?LJ{Sa z(L0(2G9zSx{P%d~c!3v2kAH#@ssu;yynkYgI!rXRfX+o~j97*&49;l*AN(yqY5Fmo z>U$cTj=Bv-6&4W@aV40dvXaE+#E_r6go%0PIOsjDMj|u}h~C`!aEtSKFe{eFk*m7F zW&6eW1X5%Am{duVlAlAQW{8${2%1bSQr;N6Hl z-02|;rPCmelPkd6ht*-!pbUxU?^LHKS$Jle4w1_+g#rJx;G1v_5`yI5L3eZVv27}e zO;Uh12^7UU48t#Dlfk*O5%A9nL-L&31=k$g2LIT9X6gqtv1(fd-uMquceFR*+>bBo z`gkqXo$s@mhtX}IN!JSA?S2R*e($Vvbn3=sORE^0{SpLv@51`UxA4@<_bFo^7E>9j zq&B!34?6YX_$les!AbtqyB7{%?~oC6j?ttFzdvNc_42W$v$cyKD+Wq#KE%VE5`6E) zpU;d*;6`*4xOev{b^FiKy2hS2;I+dlF#1M{gbi*7`G;-r!c=+E2$sXr>2omgN`YF| zi{T@_W-#_$0UVzxV3)a7piUH$Dccu7PD6;8=1nJVZ$_!rqn5CHrzMbsJVP;J4fZbf zgc8Bo;KOZElAr&cxu~rNN2*21jc3zHx#n_sQFA5P);$OwL_m0ASv4^3t_PDY9pb-l zVLa7*0vLZ*Byp>vp~@;1sCLU64A0&Td@*Mj+g5(Y!pow_)=jc7ux%Zg?z@3(w94U}0;f}YLCx@OM){#3X`Xo)>`~>F!v%o6zv2$>?h_&7QA^>h zEza=G0U6x={T_HEsz-GA-=(X;2qsVcfSJ}*_)VW@KAug2nN?4iFkGz?U6g|9v=Bic95x+rgoAcrpb5RH=WU}Q%yRgq%_ zOL#rF+OR+TYSn{7>8UV&=OQSZ$=Be%5bhlEB%sNg7=5{iHH{B|%gBl7l#7uCr>)4L zoBNrt@dA9kYce^1HVAMIq4=BWHxOSv0+yP}z}S!Z*g3=i2BxXOlV*lQTB8I+Do2oP z={dyiynv*rHRB8|6EbC}k=b_o4Wrv@M*fUyGRvNv0r8q%aNVayP(E)loOH(nx@9Sl zsCD15KzI>plaR)r7ybvrG~~#|T7JK5@eG{xm6e|A4o(I+?T97S{dC$Ep5>7-!Y<+1ukuXi!e!X7-x%DqsRTuK z`Aqy07pSs&3EtKPNhxy{JN!BX7Fc}8%jh#Wrz@D8xWVC&#@S?Ns|Gp4KZAd-Nl*`p z){-8N4Wuxp6rU}cM2^2-gQuQ33Dy~glYyW;fbm;I7P%Rb4Z~hA(m_D>h^&Njr>4LM zZm#gc+?UK$(e-fGUNso_PZv*05F$b^&){b1Rwh^>8~o$M!R7{jPaI{Bm;L+Z{QQa} zJUKp@Ox`3#+CPMnp-)OsuQ!tnneHZa*Mx}Ji$bib6Ng_V{bAfKJV;BSA{+=_NA_ig z^BSdBz)Yn956m}%rh)lj`pzvdZNXIHTf82MTwVqv;0auKn4e!R=)?Jw80tx21;|OU z1!AVVK>FBja3%H!w!B)5vp!pry^$Q=>E{kV#d{HlvS65!x(=$8Tae7IQ7}zXf!yHL z_~+a_NuT;5@G)A6^ey7q%T5m41l+-a>@mDVdxY7zKmy3AijW*{1$Z&t27Y&Z2IdbO z#PY{q0clfR`1)Bss6>Z&e!URfP5VIgw{F9Vk9u(Hg?%{otu`K$7bUqov+(@*Nj!d6 z4~y>Y1X(e$)UrSG!BC0`IKHBSI%gjT%(9)q%eK2X+xjErxloq)UUC8N%l`vgPqhI1 zb=m;y^#btf()GhyjzqxoUE6jTks!fiY*-%x4_+^){=_Cjn`yIQiBmB7Q?nU(xz8cz zv-#frburlR^fRstox)h1drkGHXu!%Fa>P?_1MFWB03+5`VtThIjC$}L-?-wAPt`b* zZ30Dd|BpT)33qU}LjY<25d*5;@qd$j$gV6M62JQ(zN+~bZyhhj`N~q*@pL%&u-qH( z{H#PK^?$=VUPE~Cvp5M{FrVx?V-L+9+QFL%KG06a0Nxn&0rksRrXa3_S#!=FWd30o zTO)gLQA3ZB^0&i#uHV5vHJ1=_y_~$e-PLvCz5sGB_P?M4VznLFc(v(4p}X zymjveOZa`xV8JXPuPOvd{y7{@mcsyl2N*T!0a!SD1pHAlf+9_(;0b?go{3GKzM!pdZI+*cfGsX&?JR)7b^Ma-Eq7ntE!ufbC+ z44I|Y;A~7XRv7lc+B^A*cK-oz6UC6%cT-9G{CPzBhd%YR$AcIm{+u>i1G`gMxOKl6 zoW5EC8U_!5?;9XUJ|_t;=85A4gPTcjtOvXyxsUv$camG1-AP351W3(VMRqyTq_FNe z<;+~eZQC=zw}GL$@d6KW!08l|u+*1~$duzt8Z*K6Rww9NkPn11I&gl9AsiLbB8{@` z!1}%kG}XNX&L7xARm|r#E?>5TLRlKNTz&@di^WiMn<&h>t`D<t zO_u1G;h44l4`(pj*F*Azy<__!#dsm<%H1*vdX zx`3?I{s!i*oJOVx8j};eUf}j`EpoX(4Bj?a3gdr7;(Z1tphfW#I285~lRsm?$MFg9 zDwTl0cV>f>k^|s(c{@;*$ph>AHGr+58Qd3l9vtWQ7x`;>UgT~sNZtwHzLQZv_Q6Ix z_*D*u+Z2O0>%W4fol^j3q75S&E;HqcrjV9w0zaz^NshcOndYNT3X*E@9fSFB^j?RWF)0Lq&pNzBgr!+Km_)76*D>?4|2H557AkxL#NXhvKa_~P{5GWr?9yoar zw*&TM=!ps(EA=GPq#cOxXGgfXneU07O@qzvEnxSeKg{4KULRmz!OR%dBpJ!^@QOhV zRp1&AV-17gvu=5~ETj*w&s_`6G=1O|9|0*ep9St@ITEptx+J(y1HSLA!5iCFz`S*H zpt^$r-16NCocXLxn~ z&pLNTo}3@Mh=b;Yk=N%Hz!$H0vNLiyS!e7*jIDAh%P@1Y#YUJ+%~}JezdVJ%;x({r z)-otsss~f6<;mflDez-h63o8oi<5a@mGnVCoTo>VLlhvl-pE2pe;ab4)`wUwk|O%0 z2k@+4Be*kT72bZv4K_dI^*{YS(5gZas=d@FYA!NFvDJsH{oRW@s$b$ussEVcQ6i)! zd@~-r6#*`GX+t;fevlWMPF>+lh*R)8Jf+Htv<&c^`On38Y$L+LC*5$7|8gpEwinS0 zeh#*KtB?<20+Qg^DW;@P?QBwS_D~lpvvOLK? zScI)6yva?j3j8^;lyrr=let>vuuXXc*rbTV!sJrWuVaN{dbMGvYXzt*E5T{ekFd<$ zUi?9%9tYJf1?S3a$lTD2F3FvU{BpA;lNQ8S4&#@l(;3CcR+mj$c8q1f3Gm$hk+a#HLeL{4OTym8!o7Z5VRLW_ zIHRq^T;}~%cKQM!`-}GtdFq9)=V}q9rk=XreAoG>buBnsyN0ozgYm$~GDu^r3d&r-{+4M9RhBj z^l`WSOP9KsVvxFYE+wSA95nP@#%r8-KKA``xGU)kUg9^8Ja4flKCA)XXM2$uUw+o@ zxu{KATEAn%MhGPoo521B6zrE&fuHOynUHIdE{{X`uO;*QoG5SRaO#1GQWQaHrQ4pXhr4R6n`# z0ot;l?Fz4pa(M&#l}bUbd=B;#+5=YebDNi~QlxuU2pshb#^YU~&@$5mzUP_j+~0Pv zsG<{e#ID9d15U8Z{Qy3u;{*#(owAcj^DlK+XCMm=s5Qh|NC!%iRH6vu$$~`-$@DZQ7*Fjcwsi?)q2eoitU(U0 z{gaKW&LG@9y&oi0rQjI;F88wO07h3QLu2=^Sd8$qM}q~hsNpJ=_S+ZKZq~+Y+7j@w z4WGc?t4c7Wu^uekb)UL_S{_RM{Q;_k`5qWv1NR!0uyH#-|J~Qdc=L7W=f&palBYFH zznM*Bo}>~!Sr8`r)`HryjnJ*x57IsRnHA0Vz|DX{U_#C2IW$-5<_a5fD992t?h?R6 zavj?k<$yv5OMd@jL#AF41uZR(r0>=);JaN4ZV+jvN@u2nf#u8aeB&QD$7L!k`#pjy zWDKDx?t;Hq{C{>1B)2Eu+ZipnY2arz4E@)rWJ z$`-KOTMULDv4bMb8F;PLORy~CEB09z2W^^Ep%e+lCI@ry5v3*MZf*#3_E8`ftha>r zkzpXBuM~e$ik;_GW7V)?>6{d|AV8sz+7TIh9S-_x>uX1=Xt|;N8=UnWLcR@ z-_;0Wa$`0)U%Hc|jLjj&c3K2UsF9s~{}!$B9Q)iK26+Q&)TN4EpwqJtbTBz!0V9p4 z1!Myciz?uy@&c@_-H*@a>A)+8ZK3aeA#zyl8h{mt@iZ$yJZ5>3zdbWy;)w*}{CqRn zwBv!xaQ9iPGW`uG8fvC~#w>smhIS<0%^xPh1+cD80Dq}iGKY4qf}ecmLapuHbQX~fBI$8f+dPptKQ7Q9q! z4_Ax>SaZe?&iBs%>lODfHSLR_?ZaPSxw!*u?|ZDV7=owbzSO_+An>NPx$fvv1-QPZ91s0H1vG3V!N!0^n7d?1a!;ScVvFN|{>>(A z70iFusj;~1$1$+8*nvb{)`87R+ex_nYI5vJCiu&H{o;u#JnfGe&im5ASUZ&BGh&&H z{vZV=C!WGl2^GAr(JySwb8waQ_rM;(1n@I{3(o$YjO9Nq;yq=QNzKmr@COqFPj)4P z#kF}@ZA}pv`*@IYEij~_?hN1w6BXF(kPSeRGc)(?H28dqAFznJhKDb!gP?i)v1M5f z7`5!ATCVf+TRIy%%0`iSakGfPJQDklj$@lm!X!N~1IVc@ASs7GGpFY+B)UTOB>vfY z;B}$_S52-15@in5y!nw>d4B=3T>cmS8ND6vTyX}d?iM9+@pnPoqnF_098GwBmM*zk zs|H`rHGwO|@6^w#*~k8SqQGvMe1?6&9YXHY#M$qfO0-1RJNnP_z1(+`zvxcpV(usO zwj#sgD+B(C}NAV3OHMBfU2jbqm6Sz(ZA(G?A|^r6lSXEYIW3>zP)2M z-Q<^nZuI2~Iy_i(^L7T;vON+-(9W(GzfW-6UzD;_ZW%Y+9D0tXS+}vCYpx&}?OkZk z{%vUQ!CBm|MUxwV(nb2vtPXVJ?=8CcqZM~nYclu2r4|L>^2Bphwa}k57gV9P9mRDz zv*$-|2pqdMq0VCsoNwDEG~>XU`o~IQuFp4}bb&Mc4fiX#zo88QZeb z4xKbS%3&Qv&f%ks>&8-wUhXRE`qI&#v;V4u`fo_0cYRyvUy;tJSv*^CVv2A>na(}# z<-sTQ4=Vb(P3y06esxZQ)bJM8=VA*xeBdE{BXkxwd0;d0eB;lZdL`0uc=VYTze*70VrMyU*OTix@wRhFLcA39c=$H7WvOsJ=?mElBU#*1qnk+L z<2XBMkusZiSQ6E%C33G=Re|meVJ<;1o3k5LaQ!H)hn(%-vYc)K7kRRTUbLsZzG8nD zttEb$3vCnQ-uf2P(NCB1dENQknX5bKRU+B!fkVA)Ywv;j4d$6#%9&Z*vZLzgpGyUI z?PVRU^*b^jTo2d4qJFT4*$Q~*Tpw0GQf%TF<=uoSD^n2-xsMEECw%hJpKYSyWjaj6_ z-HFZQUfti1hI5_?R#_b9W>5djy?ChF@II5`p6YmTdLcvfoM>Nk^rj{98SO+}VnJ;C z46pj;P%SS1owjSd>j4zdJ&8s!_i54^Ll4H3aURIHQ;w$mx0& zEqzXj`+0RUcj=#~Ym-7YYGMAd+b*SXS&;$Uro;Eh?pB%lSc6P@uTY90N<4r|$_+se z9c5iNFFA;wXiKmUou;y*OOK%IsWg!I-b2ePh|#KwqTKh{Rdnkrz?HU#p^QG7Ro?2y z`36lAFb}q)>dk}TiupEl;m`;>(`OGWI^7rb>?!1&`^4+@?Z(lxX9}(kjfSX^8eySZ zX#M$1)#$$skh@oL3SG{dNoQ+FkyYuhkeT5#frwT<>UL7$mP9Y*)bqyB%%n#sH>H{` z%-MkQl+)4N`%2vFRNj*~Py+S3DxzzXkI_Dzg@Q55E^d3r3WQ{5a*rF1b6R?`$gQuE zwW*~V-mx;SzjAu%^W}!xx0^Z+51G0>oSkC+u(+l=}mSuE-6GWL|f}m z&pFM9`!3V6Gr=QtE^y%`ne zPC_3Bv)JYJ<_+sDJi)k!EYe?*%RSqlhWtH*kkzby+C<8bThCmkZ+fY7QyR1y0-k4} zDn1XjFQ9;%WqX>t_*sji)RXDss0_J%)J2D%r3n;=($Eay4Q!2&6ZhUbk_88x(7%Cu z=*hfqC_bWzHMf|KUM?_cn5^8xeUx}idmN5OAM}k-mgY*%Kj@19Dc|LGcUhvpyN@7U z@yG1yjJ=(#Cb}Nz`S9QG@?t*$Y z7I2#(BX^dhD=rcv-@hav^&Gqekejv^eKA-Z$)=aYSGUW z;fCkkDkSMH;c)Z@&hEh#Wd3V6`-1l8{0G`u`A0M9-73CxPKO*?eBhbDsW}$yNb06- zKJR3;=NY3pi~!jrPH>xK6~T_FhuQ9}wrEMFBDXlV3c<%R=*OvJT;!cJWIbWe;iJdV z8ts(&?VV%XxBGKYtl@b5yNi3-3#EnVxN;rmYLUhUk@YD4bt$slYlOZO53$z=a!}IE zLG-2l0b(R*+H!wneY?+_q-J2npyk{v6oEo~$qPG2wj7MG@0$j+&#C zIgcQU-P$}#uZ_LT1}~FB%Z|%&kLDcV%+!Lp6~FVjvOzCe`S&99;cGFXi)V4Q{(IRM zS3HnVauEIYvOJpl@h7sZ)8uk;BwcOyg`@8VN709C)wH9738(Cvz$MMS#6GZ)q4hug zt7m`{@?DpWigka`-%~Gh7WR#tRFNh(^O^-cdn6i7x0mORqarl3`ZE3g`5SuR>2Z`} z-Hdbw+qgxm-*dYz8`EiW`f( z54Y6RTXx8~-j$Yaa2*e%gX@&JOF^pi-E+U_%U3N0c=#-m|5Q)!_+!b91V5tnhR@Q` zYoocL_(o0;UxBKRZ{||u&U0JbZlThFK(-< zA9@f~%(?a&aS6X{xXoJ>1-p4)@&1~R^!T(|_JBtg@>D*I#`3Mv>{FXi^S-qDMNAH& zeNS+^%c|K0OtWCYqYJ3#(gXJP+Yt2L@C_Shy`PKBDyEOmibpFg>yX}3FS`1lXoF(V zG<2Xyp+0ecI8vT;mL2gHY3S&$K==A((YK09RQl@-dVW*{6i$7`WemD;vpW^)w@o_F z&97|c zXoQ*zOVNl_3um#{2PNiz@CbU?~;LCf6o`ddz>$goTpy_h1)Wj`$8 zI{uoX-1XUlb!n5(#LrV)a%MhT_?w}1UT+k{VZ|H^kh+vRExj+7 zQ(q+F>bOas+bkl|rR5tzu1PhE+Q{(eI%m07Zhv7wxUy)fduZqPTQ zPp}XCR$u46!)(|MbMCd(T~7C8di`-1Nmf!duRd|IT|>(JGsT&zZ^LY#>T>Ol+z9PZ*A9~2C zbVmJWABvk~!qq;JL)Yd^;RajNxv6cL zoayLF`gU>}vL9H3e((P*n3hMQ{JXo+_{?AI#yPTF^7|5wqJ`14v_Vez-bYruLz}xN zc+D--zQnB-IZh`7b+r1yQ98h{mkS85qCHlqvkUJFH)J^PXB{N(Bfrt}=zX&~7pHxR zw)>QZ4m~j87KA=RyIU`C`L{#SzC=HCP$Pl6v$h7+h2?Q#x|_HY{l%!)pn%OgmrFYb z?%@KS&F3OXI5+w#ggbD@n9FWZN5_BEp(Uv=&;`Hks3brgSyauXt=&}Fd5*eV-fT_o zr+XXydPWh452&)Q%~v9$4ckzS@-z0Bt1Q~tZs^*Qpv_IwosTk{zoMP%<+(TWmZ2l> z5>U(XVJ_pa63Hmh=l-U%?|-~!ruOdfsSGMx@Pq`>FFZAH3% zarEO>9i((EiQ6>8h%J@)k1Knfh;%;#*PGX}Sf?jRNaXwnF2T1NOYwP)>Y49p?dJ&J zxz&fRG&!Sn*&}T2yhmKjvr$3uj8Alg42#ZAG@$mY$LZbb#e#ZjF4B3hi(8~5Lo?%V zxRqxs=-suukX4-nN>-V{RWI4iW!c(tNeMEp?tI?ytp^$nN>h61*y?b!qUskq{V@+6 zJb}5pcI)U~ZY{SkI}}BZ+o2~;lTghDJHf=JCv3;(Om6$=OJqp}p~*`MSzWKi^(*+i z*NHnn1hyU7Xx~dyPP`klaph^8Z_P4PDlUPoQj9ll$Emu%%IE)myTS1vn zIJ%v+zPO$T1~+9iB8b>!Y;GrpO>)Sk3BfM zq;a--YB_t=i*UvBCpC!AlIMCB4IxtamTvtS#A+!Gupu8TT|Zpd&n@|{gdG_i6|~5o zq)%H$uqQS-bK}$BvnviBN8nu%ic+oTT!MnQCvmyx>GD1<@Wndxm$pJ3B`jCh_kufU z`jETu=opvR(#>5At>@|lr|Wllbm9B?0jTWzXU^yBcJ9s7Z15iM<{tdnjVdpm~_WYG(2PP0kJ?$V2Nmc!v(El%g#4s?vS%2rUGTsKR74N5P3fr6i2qK^x+tl`!) zZc&Q|GIYsD(w~g^ytY(iFg!-*Szl$ZZPG#kcA~62-~WVIxuU@)EXeR1B6sS-xQdR| zbdTo?x-I=6x;pB?mG5vt=AC;vUG3)r+I$AOHm|ZiDAocMm!3sOBmYC;LD%WBqsv(7 z%qF^!{l|X(_maJnW$2m}qkvwg@8{Z-K7rN+O>Bve4Z1(x$(|G{MdgF>^_31H+)bVQ z`sKwNxJapJw17OS-@RM~?Yx`IS*MNA>Lo4Qi=Rc@y-OP0e)~%TJBdEFX*Ykj*ALQ8 zuaDLLnySWKjlF>Od7Knj@9?Dm%L!#miUoAty-eD7Wuz2_ms( zXhM7%XRYDS28&<)A4S*UPxbf4&17UGN{Waisf>*4bDm35nxYaB^+iL4wo0-RAyGzl zc0}1+pYvQP8X71m8b)cTNVGJ5_xBgv*FEQR&Uv2qc;29}v>2b9;gN=h6PO><1PgRs z1(}|e^nURLaCvuIev>F{#_v3yTP7LGKxhvtsCTVcL+>WK1%1B*26!S zk7!*hxKqtEfHbGUwL@BP^xzYc#fh=_x4r|!PE5i$1x=KXX4t9EgNWt$T%3`;4R7Yn z!yWQTjNWWv%*v`FGv)eVzu;_N$ST1L$$yBO@h~3ssG?JrPNKErvxAS(P|5C%tP;oFiGp!+4BJQ7_51zRm(OHCjx{}g~i-B+<|FTol)dsy;h zGpN0@z+%G@k`>m;B%V2GvFT?SakZ!+XZ>=i-a${Wxm^MwQ?^5ruNU!|zkynO8imST zf~=u8GqB=BE?p;f3JtD%(2QL?GNreU^n5kIIprnv`=)Q?rR_bkJ=2#XYTgcBlRc@3 z#bJ0?HW!!7Y9(fZ+^#V!E|+7>Hw5AlWgQjRCl zL`?y||3Eq`uMNKK?WLidPI&H_h3YL1B=o^lkjaW7S!YE-++`Px_ULo7=g8o)6Z!N& z*)v>TI+@x`4T2ZPBp|!R4|F<%VN!3Pg_h?j9LvgtvkyDTIV5nwKLyR*^2i*;9K5Gy zL3<`4xo^4*T)&+peI40YlN1XUc?G2NM-CRQDFp5I2Vi?L6t=u}#}6`wa9CaqPfu&6 z=}JXpNOlbg&hp2a<(han?-z`uYmhsNlW@JIMM@zunql?OE2x0f!Cp0i}2DCU7QQLY!4*2Z# zFg)o3WVW^9hWf2!Md@2QoURGe)HPY}@h0&5dVt2ORzq@BHoAW*Axkc9!&R>@nUpmx zhxC9VbQqW7Iz-FB&31GA?idHLRtwPHzlF**oQJ>u8elv-2@MkqsnpFN+$ZQW&$in_ zQE&+@%(car2V3eF=00USbT-nU2v=O_EC(G0ChWa$Pr>Y*BAuF}4N2XljD79^IUb;a zx}w*J&sAOcx7mTJm`P&D$e_g)!I-1Hoed;0uodP9%;FEHt3lgb6_lQ=2KM3hxMHCy zdhc9L%8k~bt@vY+}GyM!fsgfK8VNO}co6URwE%C>p3Dk@6;+_3zM2aYk(Fok={s zu*ux_rWK^`_W)U!wh^iiiV=lHE14~YVPt&7g)S05fiYJ1=_8{-VjRhC0M`NJ8I~Q5<-=7ZX*^vG;FQ;E3jfAkFcPkD}Q z59P^)W4$D@{1+XXavAzYn@Hmw7C$&7f>h>l8g_OVs;WFNy)Brk&Xl*1yeE(NUK>2u zE~dNxtA+Y2TVTcE9>({-gUF9J;x1UWiHzxuuz{>IdTacLYAswiZ0>}sn~IQcF%_4; z?Zv|dlJHctkvK~8(Z;?3wQd!WMk7aNK-mwJj#yztQYGA!IS=KY3Yf0!0Dn9^@XsE3 zK3(4n$MUDcZ1MtMs1*_6OWrv5y#o1r%Zpk2IhE0V!6#Qr#Bjj&1pYUEn?#!M;m-Pe zRP)&dywqcYzC??AV}&bzEEQCv^W^YyQzA|-`wY@*sTiOu#gEjoq06(+5Gl2C@^1uxSoXmNfuaDcGV&+cQ2e*TY!G)iMUyN9{b^S z6i%t13J*;ZaBb`$bnF#Dv!UN4OOP8VwnZT3^;?cN^WV@Dce&8zDCpCEZH6v63Bi81 z!=&eT;q3P)B5`6D*3H!A2ZYLFMeQSc(L{_EELm)4Zzzfk8G44roSIfB4Ni|!S7WmJkWQA zjID-bg<>WJ(c92-445yY!LVk>V!-2i>|2%dMCP~;UhMxuB80DT z7^|92L&TJcxK>dWZ*DwH&EouF-kWaRb>lKM7M5mq!%e7syaXqmas@~G`xx~57wI3V zq#0`yVDx<~d9~e-^GeKFfL&d}POdDjC@ZtrUhIX+*9z$Mb?SIsI*9oGx`oS(i>ZX6 z4;=p%N?dM-VeLp4ac*$K@S{tJS^rlms{WU#oV#WrPo9wZs^;J-;Pi^#B+_?(ljz&c z{Y<*;<@%rDT4<`{19~OJ$gQ@+5GgLTOf!e){oknVX9Fy&zEQ8Pz78Jh2hir6N<6sW z3eAd?fGhw{wFpC<}+-3*SBPB)=8C42mJg;0npJG3Mk=X@bnR2g!Lu z5&pZp_vCwp3N9aCi`!m^!QxA&K`34hP8sYc?PflZ`+750{T9O88(u?Z=L}RUWT7|I z6pmy#P~qL7s7)TCN#FsTtPl@9wz(KC`J66_dPE8ZYi#f#1d?-K!?A!}tnPo`sqM*M zFmCXQ{!2TGCMT4@^w~q;Mx;^A;yv(7rkHtES53S!L+Qmg#gu8OM>m56GSk={bURO@ z(awv^2|>^A{`)&yzM}+kUnmgudGfg2>MY{prKZlD_yu9BnoCy z$v9*1J5!HV)|aSfjSP2OvxJl-e4x({DDqoNWVnG%WAuS`HAwC;W{=)7ApeZFV-sBo zXO_z#e@!fFEeHonvlQkElc${az)R@=&Y1u5`$ZBtH;gWFj-{;+g!t`Rf-$pi;%WFn z0d8(=43sa6R_PIV-}M{&ukQk`(%cWqLDOO7w$<=6Ck^P4hZa4vM4)d$5z*HEf^CN$ z;(qf2Nc?DT25zBd;3EvKH6XK-z<2)Fo5K3ujVtl@KAd_EqDTW@*8 z!Zi~xQ;^$Sc6ciUNv@+mluE#7(iG_OyFq=6Vwsl;4)j=QCbqoL=MM&*V zKW}Kkf(E*$2y}#v#A|RKgx3@pFKwNdmrfs-x;8*6HSj!`%X`h zL#Qe{2s3~11ek*u_sOOxrqKB##t)oEChk2wXrhg_PT8<0XE$E({zi^OyQBZ=4B+%V zr)q6dDF1RQYjh!)n59aRm8Z)Pt=?JW2JWO!@G$*;vZOvOTB$x`MH;yqex7dgyG-8n zq+=A+x;G^na*piTlFLsaJyNDV6D1E(7N94o~4aa)Ru z!+w+dJR_!bhAy~h#nXI;0dyO)g6Z~kl&Es7azA#16N8 z{apXz^h4tGumUTW?0`9stpz!lJLt9ge0ZqaM@OYzk_qc$U})V1Jl7aNZ`=C9u$4W1 zxZnq)IAa}VwuIo1bCF=&!ot?W)|j{`8&6;IrU{|@*eId~QQf~d&2R6}*GAT0?f#L@ zX{n@7kN>R?xL-k6XsB`7U#=Kl>xjMHLfjeVv%y$4ow%;LP7Yl7MfHOf;fMKjls_WF zog9*Yx8@4-w+bIao;VjfSNG!DT{7_WZWP|E;DgDdQ}}a7VSSClWPB`Kz$TkV3Hmi3 zSYGiJClB95&o`8T4B4^SN8AFMDETP5^$n-Iexkx z0eL2YFjGj53?znt&nZ6nx$OwaThvHm9Bk?5xuLk~=y^P|R)K$3*br753p5B#??}H> zJ=qhhhmqsaaCl}gV|zCjWg0Be?)o?xdwCd!E}X={$U+O{9m!BV`5bg~oxxXz1_B=8 zJ~cJ!BWvfUQJXe(LBD?s??aR+V#9Ib()eWgfQXJH{#k1Q+BS3b)9ei3c5&O2tV7*xu?t3ywGUw=!8Q)n} z{&ypMF0N16C|OjrOxQ;{B1C&{9f4QK^`$Ix$8L zr0%EOq^+bqSd-g4_!Eo@4}p@x8kD!q#7Ko7pmN9)W)k9iMnm?1BXCM~*9Sa!4#~0?peRKI{_WZb zf3r?PQ%E$vQ8uA#)r{yKg@gFSznnU5ddBRwTZtPk=b__}6}a?{&{35&81PmU9AYk0 z+m(y)&W}}C+FuEdw`QW!2P^(Pzf9U}y$9}1bi+Bznji)QI-Ir?!8f)HrGn0q_V0sC z@rH-A=S4QXzEuYAhrXeg{Sz@hSCCKanE(?FE+g@}gk?dO$oRopSdcG=FT&#J)QgRD zQn5LI_7w}f&@~z2w@!qIp0Zq>12=Kc0Y%nhzztu${{nqmbeV|<)}i;f4JgmJ0f8!d zsI_Gs)|Z~6ign?nF0dT;yY>qB%JbyCcM~(xJ&*c7HiHYnW2~;xRwyi)55LvUlj_JA z>=UM`{=LGI4mvkrka^j`27UGc9VTKy)Yvf z1CxK#y24#}QMDgE#RiCq)*sp)sEDfbr?ACp((Hl!MR42g7I_e(#4PQ8jD2-gV?Mvfv!#ER65ZVBZb0IdRz*8RHMPv`#01DoAGTX z?8V1XA?TTG2vv_ius^0x!Ymy-Btwfy=aduJ9I3?>mGHqK*JB{RAPDU|7vUf2C*+2k zEd9)ngxi0Th}n1v$X)2im)3re#8HNHql08is~${$6mB8&xQH1S`)9F++XbcDBNSaUl8 zo27r@-MTSSS$7^wJMQCzvqQMrzz90;FURMHtTAGQ5pXx>(U`vvzY5EcOWU*|(I5rv zRgdB$-+lG}+LgJx;5Bn<_yW|6`l8x}S!CTF2LUb|2qnJFWbIxT$cbttiIlRMqO#oN z&;)ciTm-XJ9?{mh+u47E=PjBB^_ipA2kC>nUMA4;bp3~UzW7*n4;YC%;l0BY_0}nX z)LIdg`tZlztq{^FKhSSyGOfS63(q!Z zu@{#WfO7RScCFuK=q>q4f87hm4(qSzbNVzqoo|8Zjl%qIG5TP26wv;C4|_y*n4KGa z77nlaM&eUmg6*GAWZi>iI=7Xr_gpK9xoele{OWN$a{UqPX??>cev-$d+b)sE-p857 zsa^H4UuMGMBim6?ax!YB96_%G&)LOVPFSjT5*r%NQgPo2D0epkav&Z*zRg9Ka%Wo9 zt`3Tu&B@pp4{OGwiI9^Pjun-_g^yY|wHTmX{{0Td z2fiL;QXk8_s3@WXo&K=npe#2#vJFi`=in?C)>$1O>gL1wiho3{-QUAYVh(Vcd)e^K&s=wU99QDAd!*RqV$qr+#9wIw9NpM5B4RqGu z!VuMuP(5!3dbNvSUHWQrZF4imNpHb3g5E;;+PXQ}F7hF(Nm)hIW1*rO)#c zAz*hsv3E+K-Wma@m0N{c+t!0pV+WJYQmA-bgVO)DVCkw}kTz?m8*+w3VgV1o z=nHcn4wVqg5F>0YbEf~gv~g>99_rnF%;6u;1NHO^7`y*3&3fnwn^p@%PYV~1!Bb4c*@a16e+k#vdU|x$apQGvyui3{5C6H(H7^GiG@B=h_uv||O?feZ?g~TmSXd+7TEDf5kzmiMPtd+ z5ItQU)1~rY+vM#S{f9966RaSB*#=S@&aqGULe$;*73|Ngg?RN0bR?(Aot#RVF#eEC z(^Y|;5*FZbb`}l1G7AlF&A?laUFbl$EA;-211Aq(lvO-O#_mM|Whp->;1%!5g} za~54MNcDYjM4TV=d@Ygb?0{K^pV1*FPc-Ryjaxqmu=3S9P|^1gx=J>a&FY_-Uk5|U ziI*MB3)vvrn=^o%kP(cf1st&Pub6@&9KU0KmL2I%%AGW;w`yk)Y&cs8Twji zJFbqo<8{<@5`xQ6WZKX<%1rYjcl>Bvj zfW9GnfVU@~RzHbF+fYs1AhsVIeJrtH(fj(5UH3__@J|bF)Ka7~ezTf!fMNTK>D+K3 z?wfOY7`-Tre)5dPU;2+}fEEYglp0@jryuz;@R(V2?IKDSWZ<`(=CnjEj%s>+LXU*W zuzz4DlxwB1nP)dJOpp}3i!Y{2)y?ozg(UyW1zkZu(vEh{%4AEd4*yBz8eA=?$6nXO zW6?JiY)zerJ8gvNI_dSa=fe;?T$+S04X7hfemUns*6^AX?7hynV=r88oR6(tc zD@mu53c6OG!?0|1Je2U2Sy^HMy7K^fpGD%VbKj{<{$)6_DFpZ@*P`vV8aQ|7F_|TF zpVXdy39s(^vFg@Nuyg5b{uOQ!xphNvJU5YICAF&OgJtWEaN&I z9J{JP@lXNYeqaq<%X5fq#dpr;i6lJ1uWG12Wd zUH#IYsJbabyFnuE3={``K{9nHSqWcW`_X?B1DU1`anxUD0z7-O3dAZ_p~hiJn!NH7 zS&-WV)2}{bCyrl$+{{R7C_8~G+4P;nWY(|`KW5Th(plj9I-A@NOJW@_T_H@29P6<4 zJmV7ELEq?R;%Dh?)GjaxZr7=jz#%;d)9%CfD8o+*y$)(y!kF(jI`Pe_m8h`V5N52` z#6GPP$fzsA`Zwm(sL>f)wBtd?z6dp=Cc&|H=|CGK(P^DAYR*s>c%Q17_Nx=PS|9DH zu9gwTEl)#(Ks{n@vlqX6%i(6{6U5)slR7O;#zPnPP^Y=EG_bXYPK|m+P0F+2zVr=x zvOJyIn2C^z!YI7bApi*-CIv(Iun3YryRGIPqV4o_d}Z0-J-PFuN)mwb?2B;#n$uZI`Qf($|I6 zoheE)wn=jDdDgQr<45qr78g8~KLu9aT1B7k|40({E+B@s5p;#kB;?JG!K0C{=;haX z@cer_6I8cEpm`8xva}-bAjb}_TWZ3ojT0by@G!XSG$oEtL{K8&F|#_a1y)8M1!2|u zG}QAH+*^5+obkHh(okBL%)9F82m zk6(69ffwjK@jax1C)$(XyiFxc`=Ef!w0rrC1sIEZPgrY(8Q?mj?+lGjX!*7d$y5jt*{< zz&gT$o^LPY|HlEXb<_E;ohiFXEgydzuBLAncd>iYM4(iFPu6SS9==L*Ty&SIOek9NO?; z5|gVN&zRo7#0=kvLOHL8q){Xey{jj~+rNQSymtie+zqDbb2DK`FlTSaDS2!kzJeyT z&EO$y%x-*;3X!q0n0P=ON>06op9xo)#k#uKyxa~J?lgg{o!T%VVg^3c=!3>MF|4jH zgL{6{@nyRz1QyMQO_P*hYX3Ag-()(@IJuO$_QVK9_e=4wXUk&eqBo$r#1w)<=3-*O zRcJDv1{(x^F0RQ=D4~B?j>mCq-Y-K^-5%1vQ^j!49BFtWtxP<)H(3cAFZSBG4ES%w zTNuAB3x8t_z~R_jG@dR`gXa$4@ZN+9u+vOaE8?%_z2|0QkZ|q2(}#ZW~?TbV$(fgR>pChDok37 zn(dmTR9XW^R|v40EgOmOsve@4)kIn|e$jQeuF{1&g6UZ&NBZ9$cp%W9 zy1l*xZR+nCx55(IzgUK&;A((xB>%wK@+{JAE6rVATthQm-N}jOo4Cz#CN6pv0^Qcr zU`pRA{Aiqjk54&5X-Wv_eC?p}N@37!SwxrK49EQbS9oZeHLk4;1&=N>R9YBCQfjxs z%$Xr5*3%5>#f2oew3k{deqyWC>(K7Z8QRo#4x4?;nRN*sf;YAdUIvUZh8NY?8EZ>v zZIL(@J#Hj>0@9G-8{z2LJ}MtKlO$|w#Qa<0Xnryr28+&NmC!J`r%_L?r0k$zxfd42 z&w|!fEo8RTQsN}KlFUD4M=wXG6Q>L@7~dg5TqR`iLFGB{-r0vUUM7c`Xmf{6((pyR<&^a9W8ymNU4^ zs^_5M@FecbnX=sPQ9mGJ@6b#e=U$-8CuLjeYDg0`ZcdC%B zH93dvz24AxD~lxVn~6q71Gv>a70-nr8vOTwX%_OqX-BGnm%D=;kqt$s`pG0+fS2D4 z-UDWZZ^@JwdD!*eQK~y&jmgg6>2>Oh8ULwTWGKd<^pGaeEVRMB{h_!Jg6T4$#jNFO zE&NlpoMbWY=|4SrxSpMe($}Y=i&q>yeSRtE)z?s8$5tx)W;f~d9o%3_Wn#k_rPc)A-%@GGLeAEBFza zLH2n4h7(Qckax-WpGBf5BrDWV?+xOlnrG&n!qME$gz$h&|H%&jY?12z40 zU|A{t-dRTZ^RLj%sy4c^&X85G)@^gHTN+UdXG3< z8b67`9n#<9`Qv>&{G=#j`8+2&>A;x3clKM);93oM&hPirUA1t0w z#qci){0)EB5>2=N*q7l)Veq^?h6uf*D{lMI_lGC&Elq~#$+d#oK&gj{dGx^&e{qyu zeGpHq5+e0wyRk%63xVkY*rSGH3lG5`_9wm5WC%Aae9^(d5=F{vargKT(RK^1FUhe6 zuW>zW-XKn%F7m~0mobaAGos1V#;qjoWIeP@sHwNU5kME8Zzm508uq#+3UKi*k5~#( z_$fC7n70((?L0#ob}t87wS}NS9~+eWfpO;PfTedfT=*dZ`U|>Yp3i>v5!V3<^9RXV zr2%}tUKVcsj>m_sad4wWmH)?1h-zQw179Zw-8XE4vISv~)XTvBBpLpHDiio63b}Y> z_zZR1pAXs=ny`D!l}=9MV$B0hnq2n)Iu7>Y@aAx6Q%%R3Bkgdj?>zHxktg}`=Qb!l z=g^G$bkf@|0b^+{q)ISce_P>i)+|XFGevHYRO=BOPw~U_+54fssuJTj=FlfzlnV9a zK%_`JWEso>nR2Fv$r{U-KUrBesH6pPO*#-Pc{HNmv zUFe5-EtVLzrxsNj&e45DjL(cU(!E>$5t6+XHyo7bT4?r=61#fPcxVni_6BflZ3+A= zDI-&jB-s;dQt1_u2@vA3AK%=l1}V>dc;@v>awV=4YGiY%pBId>vOCGDq1|wC zyCJof>L%V{-Y~6Gk1Hb93$Nl{pmJ?4wL4gjXEm)c*Q%UoG+v|u0TGxZ@IOx|n8}{6 zzXJyQ_rb=BTI#DZ!cMsx4AsIL$l#qorsdpA%-sGJLmIP*RQz@9;;zTdrJmTiu$La0 zcpPo#CgQ87mua1r5YBj!Zy}Vn3Ucr7r#qVqS(~3z;ry=S^v<+dFxJMf?s`GkdN2nY zzQ%y#=p_s5f95@iDP2O&-N!vBA(%*8>``4^F;X5b_Pz% zxPpVsEo22cf$;uXx;a@FzUM9=n|eqxo+}uIfc6>4!5V?gef!XAn;we!5lLpuQ zbZ8kdCoh=#2;=;;xJ$+nwd&h2`te{VV#?9T(wUBMvdd<|H^T~DeEP-mej7!2N} z@1@SsQ2387US)*R`j?3I%vc=n665DR8zvDlT6BPapNIzrkZlLL=!jS)#>|rA_b!Zv zsb?y|?CKJT%-;bGW@o_HDvX?5tVd?v%qRLA+~LllJiPBaM8)Y>uurYUoquZ~?`0kA z$+kk_qFm<8xBu8H>sx8%#cTA9#Wz~X#-gItRl4Cy3X^WNmS@mM$$Ishi0$3B!AQ77p8jZu6DAmwULa_3)#A2DWNsUgX~EY?OQ`K3dJ=U-B{%#J26Dk9gN zv&akE9r$Ye5!f5?c=Ar^gA%7R=08bWj2|d1<2B&u}s$a5kf5H<@23D$IHv zqPTKI7)6|0u_!zk{>^$q)bF;D6z4e5Y|f=BnM)z&_;;|28bH^pvCutaEAaozLxAmS zj4JLWDc#4&Xx(nOHeClBhDvE+Vlo{RCh*MC0eUn;AOeF)#mRf*)|&%ptr3HYqhVy( z>3g*Ef&@g+x8#h#v+*P8E-9^Df)W22f}5Qs`lL0}ki`rc?L@4@3d+wu3;dn_L`C)$ zJyPxjIe&J-^M8Um+37siugV}-+nPaCy%_qnP0&GH34dLyqMte+Q_;R7Ft`6YIypCz zw?(yVAY(^4cK^tQkB4Dh-(OZu{s^62{2BLsOCT-&nlP`k4^)grz<%LmE|tn5`(5sk zr+hA#J2jEa@E)Wk#i2wte?1=9p#=N-l;B%?0&QolF@Aj{{qQCmL+S*+-@giI^4}~n zJame^CL{$rZ=FW?=8vmK9)q8vFKxB`K_oI>(rMww^sOp^oB8X|r*1CDzA}ei10)?o zz658E;ydR;^7nyY=0na=lqmj6y;Fk8>Ail;+noZucK-vCduuub6{0jWAkztSgc}DA>?#4Iql`wK#oSTn($^PaMwn^aEvRpZf{Jt&%(wcd=OqmDn zomudD(HvBoH4Pe%rC_PxKdK!$0fti};MTQuFky!yN>@!Js{dB7He~>xU;T%i)_;(4 z`7rLTXQ=6|K+sv)%Dn1lNo9K>X8T_RX(o$2-+hhtzSo2PmH}LuDuK@n)hUU+NxrEC zV(+O{pvGB^Ya(ih$gvKn#vJm<&(flzw1fPo_X$FTm*9?{0Wj0P-(uh0RYdj0dAifE zpuUL1C58R($kk&HN$qeOrN?hFpWb8>t96n%v3deuEI^jO?!yx}_skpz8k6a9|1rTD zza?5rO3*al4E{}QVWft;*zlasSWsC*Or|N)qj??m_6M)xB)#`2mnMz!mJ;0hU7O*Z z`x|1`C`VZ3eK0%54%?;FVAt$Z0u16D$@^^$)q8ERdvy%3&jkJamqjpd>>6$Bn+6F2 zA8^}kz&ueg_&R-D=Lehq z-6hwJ1$>S8O`Pd|m)QSO0tL5t%9KUXedUsPtJ;xp1aIiW%$c}B_8oIhy^N^|y${|6 z%Sr0VW~_w>xVhsIrlcb;K1&9-oG&3>0x?gG%vs{%umLrz1iV(;Y}$3qfwjypg@l|x zP`Ks|>tWl59EG1$?}-DvZ;Helfz|NscRM*31W;Q4hxDu{ssH@(GJWrCNV&xZxZ>As z_%{?!E5Z!na%~m$xU-q8dg}%o_iV=LESKQF{_uo4Yc0kbA?j$)#K6+;65Pt;+NAqT7-%O7{4^cSI=Gb;|RSUbQU(*rGwtWW^(gE3a*a}W4EVGXG=4^PB|s9!~B3s`(T2PEdI-w2JVY}eUC50@8kYp#LGcg7z(G3nh@_> zP6F<`7m}}L;b(~)frg}sJTdx9W2Emu`+^j5rR@fZ${wcg6gq&vJsR;!1gz^8)QP*k2qU$u)^mSUf<>hC4CA=3X}0^ z)&$b^^(I}uWi7;&&W0&>!qB-Xm=?TN;eSqOBUYg+(a?ASIXKIe(F}M7G~qoBYaXO; z7K{<&*NO0Va|n*CpTMWzt7!-GnNh5g0okwZ*rAuh+&4%fjxoPk@ku)1_@)E&UBvNy z{VW*Y|Aj4lKfu2Jn+db-9)pi%;WXgK92~x%KvVV^;?A)HVN#f|vb75bC(1x7HD zE!)v*@hu$xJ&bQ(cVa@@4?1sQA)G8W1Z$CNWWHNA+i;_r{4;xnP1a3#sRzg+vq`w< zauju{cI20zk>_p`@ER_yzKo7wUdffjYP_Y|Nam1!bc$0Wqms!mS%7tA7aE{)_aCxW zP8qSpA0?(D>op!iBTTiSZC(R8QBy;jvtMF}ST@{97W9@^4+%J-M(QN1k1xGFan4K! z)RkqZ>q&k+cT*{b)$?)d`Z>^*sx}S7OlY!=X+O>H#>y*HlKH~^X4yx zqXIr|?qWSqjZPxz%AY`6EgPNJPa&o6yUEt$$)KpWlsP*(M0_@iV&Vf)RF-kTP!lEo z=WsD@SDX$mcG?G=b#vg=uTpr9^Wo2JT{K`OGikClq;*jbXr7Ye@~Nl!JvIs)T2JEo zY72Hx;Yad~zXhLeYlh7WJjs>1iLfgABojVCfSqqDWfo@?fc@1d>X~p1rys5+tn76Z z@BD5@+HGL+C)ADda6UDudusRHW?GX4b zf4al4hdN1l5-RZ7=Ah7>SZqs)_IAKEY$tOm?U156EdYB8tOG@JlHh-W$nbYbbYp4+UWqff|I{cgiPIo&Y#Qik zgJa+|_>2l!6HI-3jeg#eE#O89sO9faNLMT-E&Y#i&%+Dss<(0YXG9PCN_5z5x(q&d zy@>MrtV!XD7&mxO zB+GyM@;K2A>WZZq%ka;zVg~26jYC~8Kaw*~`z%LuV?%8f_jsM3RfgH}CKXOy z&$#(U1s6`Ll7pEZ=a6|gXPsGZjvLSK=8(CJV+K$0^$gz4?K`V&jxIKLu`1;JOFnCU zE}r3SoZi3@|J%&j+S6(BYUzksz+tSb+|y839&w-J5O;&Omn*}GebC0!@DJi0_^r#) z5b@!KZ#i$it}~VQKuyA8Pz-o?z8Z3}Bv)|c_noaZwOwPrr#PRdeD@;fh2IR`k~|@f zOUzT=q^?Ac3-1a~t}u?{cq*B9X`cbFZo2{}BUp*o;2_7j`{60C%m2Rl@0R(z|NP!_ z@MGT)e^MC{_FLlsC(As+Sqmt)FJexiI#fqtvg$2{~D5-ZR|9t2yh) zbN)D&GiBcOx;B)ysB1mN+x_7tN8z9;??CAyj!aCj`HI#)&hNb|czedo%qQ&(;C;L1 zZeFjg!Bc!D#uGnU&5L&n<9M%Oc&b0u%tN01tdkjN=G+{7R#!BBgZFa7 zVNT8hFHTR?U*4nd$9axR6zXE|qIpZyKi(zf9*)V(jlAJ$4V;oEA34dQZoCHft(^E- z*>ziT>NvY^p5lE>R1@su1Kt4>4NjzBetJS#7zZHVynk61&%vmi1Hr+(B zv;0`zlpECi+u&C72N#og{RQDX`HjzNqHRB#pS!@XOQy=aHSXh_o`V-TIVSpMVXd#s z#iJB?Ved`MoRsI)xow`!^EqBryKeJQ4&#~18`U&3kFtJR_t!JQ{8f%APpp^6d9Ev0 zzj~f~%_g6QI!7m#lW-!q&UCbnW0#oDS+U!Jml}}9xqR^yZ(~Rx$I42U*ZW7TPXEST zp7EC`PJPJ(PQB=}x;6GO=I8o@tBVzsI0vW4*IgRj%F+K;zC+E4&_`w6U|ZekmD%kGrZQiY);IX`J6Q`T1?}!?RfkBgE%!Y&&^ln`PIdK z?5=y33!H-!D>-T@T+Y6A%A8qe-As>sKgnBRD992@ zi-nr4T`}7{$g-&Jn|BuHZKxo7bE0OgT9-{c%k{T;8pjGbl9r*ollt$?m#SCr z4ryQD?cPhwBOSQBX`ckMsTGgat(kS#{E+HVPUet!ZMDdwy7_LhoYq%A%vWr!<;=~# z#0zZoi_EIyul#FO19WAW^N44sEx&2JRP+bJz+NKsmpMEgGHiy~VxGE(-e ztQ5&^XsM(jB(xQ2Xx-;LKT=wfmQWdyG|UhwjojrnaMfWu* zmyf~mV?%LfdK|9P&!IIwn!HVKgCO%oBI$nuaVIW`q2g4 z=Zq&G3~%Ciiw4+oUYPq-c#po#xrzQordY8t8(W_n5s%IPNX;aFBx2&McGLk>wk@+d z^d|%kR7i4D6tzJ3g)aJvZXqN5Fj9U1$@|==eBpLsda5OatZ^tN8*K`BPgRAAN-AyH z)1V-j+GftKTdBj`V}@LpcnzlJwS)g~6MtsfSClflfyKRgtY(Zo?5mJu8SmFnpD06S z{`CvA9sUCKLz9Ze{xPKit2g4k!7N;R&IwFnBFXfj%OH~7O#27D66CcsB`-UO7;dC;*6(p9iNd>` z&gAHuEa($(51Wr1#S|_~S3|d92ZRJ_|p&;-`oz8Zi ziGic@hM+92pZIVyaI*Vp+^?et^Zg~5Z+sr`woai#%g6C{lMvdc-baOemFb1GN8#i~ zgTmMKDkSODDW3UjS=`e3u&&8shox-r!>pYH{pM%D8i!s8l8Q}a0y3=tIR!lA?3J7eK`(V>l)Hhdxm3$D|&0Ebh7qj!~*O z>4h)cxxNKMes8p#y!$n8h7})j&k7jom#oo#9>4^zwKQgU8(sCRj$B^7&gyNrBuMp% zKx~i!_aq;&aNr9#-in37TV3$As)TwhyGN(z3%K^axg_yz0=<+h&ia1XK`PwF-MkJs z% zjktV{8(6Fepp_c&MA9*h{HLgg-Zp3P_!bd}*C+s)IWi!Xe3cr#`bWJ>{V{TWA-vms z$x2~o1AouQo7CGN8+L5aCu+$p#PstDm~&wwJAAc}tgO@|O_p=$)R247S}+^#ZwQAA zW*_m@+)o(pauFl5HqieX#aYC$g)r2Xfl{kx(iN9wpusl@^ICi`R`C+K*SZ+zzm?;( za#vx-?;!lLXB6!YDso*nZlD+;cI=l?UNZ&DBMRX}I-{3y) zLRt)zEfZrh`U_!VZW!qa^ylmESPrLjyt2T=VZ3iY_mO^tWz+v90`W zc|~lW+5@iNzw>_w+9R>M32s##@Fv9{q)&Q)TFz-uIral}ZF*^>XBI6heE}pg9itn* z!tOPFMFCUN;QDn*=yB*LB2RfVakPY&-S~rqv~7TnlZoK5FBKO0snnV_rU)&!1eABKm2FHxcB0#NL#r#?m}AS%8Q!=hK?mzQHe(G9>}v5kyf#cLX~1v zS%+vHPMg?6b_+J=O^}i1HkVZ*9^ygUj6k}1>U=WGQVthfS`Jg}UIWh}7fo`;K=-kF z;{CRoJjZras1}8x1ui7`VHN%qD}|~Z!nn;z7W{az{O`8m&_2TjoK(;ApG@C~dIJ+s zX+-d(VN_97BxcmveNB%{UWV*12vB{*sof_uw!82C>H*G`Lvtei@SQvXgG z>V6aRQ+uG+V*-8=ko^xZoY6iS@$ctmrUDGUO$=i=)6RGQwrnaBr@&^2Qn@XU#awEd?& zo&Dc&)H?WrE{_*wyObWHkVYx~Lkm{_> z1gq^CxV3m5vJHN)RP-fS+$#cS(+H^CD-T984A!TgBc-8L`0UJL*mtLl_xX_?XitlT z{I*B@;Y*B2z9I0qzXqNz_(LZADdpMtgl&VM}T>{@Q8GTatYX#|Llc8#T9)##O8F^1c(iY|%}$t@=2+^HYhB zXAus1Yr|HR7ZCrx4t3aMeEMN5TRkb7EV#KHH=ktXT@3`^%~F zf_N+~cBeVOcz!D;u#--CIGq=W7l&gY^NI_R9y1>=E-T0JJEnoq&T78Zuq)cP?M3@4 zRUjVr0)B=Fr@eO-o>#dHH>IaDtEn@{X8Q;Dch@!k{USYZxw8$&Xz#$gB7aeQs}Kh6 zmBjN;)Oc3i*|4cF)og3ybB>O^KXJ#u07pl zUPhLjt)uOd`PgCmkK_9`~Zu7aFfQryG20Rr}CHQ8yINGIQD!l8@u^xBlUR5k26dRr|m`lFbFvz*@{ zyJf+}N$o(J`$8N!x0kN?`-R3`)ZolgkuGz4K?Y7Juq&gr#HB}sO*-#`g#lB!Z}W8V zN2zj0P2DCrjqg=}<84=`#^86Q83JddTsagVfSHGqU zN+ZG8p^Lb-n-N!`ZEU@>4b+9-Ldmc5IOS(8ytnz^AiKtw2FQtXF2mo5gUcB>p&1B6 zpG4WFjd?I79PxtYKeUZ7=8SxMNrj~_=O|!w<*yH=6V_zIyN4&}ukL8@75QDX^5G?!Q*2}@Rgs-+vVH?@1o^cjYtP<3E9be z_B;l8&x=6*eG$4p-APWLFhT!uN!U>C%>TEz3@&@lVE%S8EK6@Ji^X0D%_mIUd?G|l z5N1wq-V45`)@(fzv$cnw+dCprT*bkmDSV=1r$-{3^;^;1V`BxOKoY{_-I%jkH*lQa6 zDFt*;gXz6E#nno9K+Cy_T*b6>>~2!yPUNoSxk!6~t`Y^q@uF;xj~K6I$Odc#DMt}DLo&EXNMiwJrPLzX)eOrH$*>D< zs@zW1I=I%r3G&IcEPoGxeBUpazGfHOylev|9nfO3;c}euBU7f_XDg6IRam@DAzuDq z&Sf-25~cICAQKz}YnI1R{Mi) znMdP4TsGO1?Yn;(Bn8|h&8uHvj@1{uXFZi3&dZ13PA9AsoPVu$FrE#v;qDscVZ*ww zpgC~`EFIhj&c2bj{#`8ZmXr{Cp`ghf>gyz<-#b7xJ%VU=No6KuNXIA*(<6=n#KWK#Lc(oGWnmD>kD(T~jj!Uuem*LQ zWk6I5x2-v1TN^!;1;YrfD?=4IU#RFI%(J%9SRQM!rKqP$XbTl z2&I9WtPyb?Tq)SWbBStysm0XABJAveL69YJq;shI@-=(M2}a_6vVCRq-91|77x9MTD%pSqFF5w9*xNsjzI%MqaSTK{PDff|mj#NXODE#LIgKoQ5V* z@3`qGHba@@^*jPID`^PdnTC$8{RKBZ)R0R9o*2#vs`8EVc(Ww>U~7E^R;1p5FOScX z%42d|mYEV~&dk}4FAem0yCS<*eFWQVd|`*KIL)|U2R^Q`IQF*;cZ7EF%iJZQV9r;V z@K=kStNy}Q9DajeO<%z|h1b|P?+$Gx!tCL+0>0-#BUF9n#_MyafRbOeSh=Ac+(cA~ ztAH1+`{O6g;Xi=hs5a^mKg^R9+vF zAC8H^_{<@mHR*wi3xD&zWk=w}X}5T9hA)AOlRan@WYTxFHgvq$S0r0Yc-m*y&8)hMlG80;S%avg@Iat1bcps zZ*}plG(G+IGfe*?&W26S)A@rUY~7tm+Pa~f8gz_WT^&;g^{vmrWLg83xx`VQJ>tyE z&)@2Xnh;DWY$_}ry?{P*XY;o2Jc{pUOOOlpHDLMeHvh)u!*EjAjeh*N7lU_tl1|B7 ze%GF2ep-1P8u)kOhS^O}V%)qyVeA~_qx`m+A*x4BH+@D%ZyUnZ3p2Yt-Ep zc-DTVsVvJD9zDRV*8?UTcbcn~)#JPt9cHqtOt{{IE^O>*8YyTIO3y^rGPtwXCr^kI(BD!2&Y$M%Q{vEa`s+?Ew4Gp^{Omn zTR-}8X!X16O*mWP9U_6=265Yq<8cyXt+nKXpzoNnR*mCx{ zLWIK|o7lG}+T1(ul`QJze9mO*d3K~|5c0O1WQ!u@**cST+*mPV7I1hS7ns-&Vm_PL zJ2z==(LHw-`}h{@xIBWJ$Bx7D$pRhGV;Va@<|y}b(m#xlcIA$2$fL$j=RkdL6HVAI zLd;fFfN0)gmX>f9#x*Y!@H&;bcmq?m_+CH!IU>bQ{tl(qcRrGet_u7<_=t=ch_LY7 z4cvWUZFZy6feZOLfqe@+$mtYMV3zZabC)(qa^i{3tffkp8~nbQ>3Cem^rXGq&)@27 zuHO@?4Hvl#kE9Jz_jvmUjpV$aeUvZY)HTMfJY*7;O4@a zoO-4f%N<-!WY!umr)LG&F=h={(ICS98{EZ(?94``fI52fnLlbRh=9dXKJ-G#9vFW_ zlf*l3hmr9roIJOQUG2XLSC8=6Nm(hDoodfnN4!Q4+{gJ&`wP0WwOM&x6Pe;~!Op)} ziSHkM0~57+TyWY4_?Ly*s^e;$ZcjZ{Pt)TR1-xB7Ltl<$>a*5RKW>5CSe9_XoKraX z9E+b4Zq#s z$g|FsF6YYA5-_^Ofuof(*sd$5xRawJ`1_L|*KcdWniK*!p_l3`;`VOto)2*AN(0!? zbPJAa@?$y&JD||kn$?#dBWWXM>`Ia)n|16cXOkhrChs`NbuG1Hb6vJ^`hmub?l{eP zn@!+q!HunQQ{Xo3IKsyJYj79&W0+G!8oV=|%DfyE*e8Yk-0)RlCNasCvs`k4J~A)H z%wj`0akH21><+-*eR^QfHWB}QkmJVHegIYb8^j(JaQlsPw0z?X?{?Y3m$=n1eR=~f zeoS%ci>u)4p)TkJK7-E_PD7?Zt9E3Ep@v_9r{u3gTviWe;34>1bsIle3vpdd!r0yU zmp)LP%=S*|f@sqqVt6hXwbt}vq>mZv74d-Q!8X|Rem-~c9J8{0_5sCX=Yf;WWi$)E zPD~9`@nUxxTrS>+97*Nr?oUPYx(9r`mx0;;nqbotC$eznGzhe5N8MM=_~d;a-FzU1 ze*P)VmHa7(O}}@-kF5^a*>xXFK8V3XVvg$`Z>M98G6i}%$ECtzUY>hH;nVC1T%j*R*`p`k`g(nwd;T}1GC_c zoF$&?Z-76FQt-3Gf?j+ii_3Y_K<`vCxLs3%(MuaJ=DZK|Tq+>y7o@oN>n*_K^hEaf z{a)-kE5+!g{UERS0KZm@KCm9)z!gC79_L4UAn92rC;Txhv~yR&6r)w>0G z+i%n3ClgUfaVP1hO2);>$*}0(T$EmO7j94b4Z(iWkRJV$)~idiJOACq>R2W8*%Jq! zRe!^QU<0n~-Z$#cWs{I_BPOOG%T3ujmmaA;!VgpYgAFHJQ2K8&4e0+z6O2kgV}b~$ z+CB}`wi_}}C&4{xP!woh%V4GX74l`vaZt#X=gM!Mz;LZ}_;SUBEwu2#g{LMG+0%A3 zV0kLhE^fgWA=4n!(;Q!X6y?s|mt(H?&w!e63ssk$&b7a&g|UY-3EKEUoXjjL+J_7B=HVM7?4 z_IL?O`8{~EXCtW-nTyqecd_-Q1sz^kk3HA}QxELG(+^CzAJP&K8TJ(?oBhSBsp8Cv zcMv@Kjc`j-F2pIC*&e%&J{+5h#)+%{0_Rb_Tc+77eKD_IW?=6 z<-+2NwmC#hU9VIo!yzy?z1~ zJ)YmpaAH_<%*?Ib6?4E1>B6hX z%Sr{chM(BydIZv<{n68+fDYO6i-bcS@@?zyqwb5v;5Rb}C)Y%hL(KSKY`k%bIRYi#a_B3m9*VaoCC zxT|p?NUc<&rl$3)-$klnLr6mPfvyEhp+3KHzcjBE{+&ygMzF^lC)lG52in z{`dv#CbshzO#Mo}Wkt}%b)EvvXdG2AXh6|<k2d63$<%*@kJtsr*~s@5qCtt8qY65-o3AV8oG?=$L*R zCc2Ko;3oT_whrxxB<;g;`J)YRnov{E5?IaaLeZ06+d((U&IwXx8TyZ0wOh zcw{8ZO0NfioS<&-`JOhxre2o!VD&*SESlBhKyy= zP%Ic4-=$;XjWpb-CeUPKi^+ERBGmL-$9~_9qz_|MLA+ZALqpBES-MqN(&~#3Cr;pe z>lHYgnzQ)O&kh?0Hp8d<+0cDFjb2gTjDI&yhOJrtXdU;DY|!?^+F?aV^%8;L`yx1~ zok1T$Alw_@jA$jottc|aiGg8QQoe|_$=-m@G<#B@@{DWWJ&25=c3cMsK_y zg4%8Js8*Z~PhS4P3Po#jSs@W-R*vG_wdv60_y&u8)X?|oe%!e0E%Yxgf~jj2d9@M! zgvwj;t=|Z97GKh^+vyx+D__DKWlQMsdV;!g*@&0hA#95x7coAFoI5MTjT>h`d(3aa zutFl0yJCg^I`m1qeK9l}UPoo?P#j6@g2o5N@HKJ=%nAvE)@x0uqF9Hb8o}WAVhTvq z2{8i`HCEhN3H~>0sE$B`-#*Ted3QBp%?lsAcxwhxO-8uq{S_2qqF^jDz*DO_1p)vn6JcCSzLQwj$<5?Ns3>I%X0dx(8*CXMaZXW}6xSlFKgd%SXS$?{4Jy%C0g zZok1}7u4Wacn0-e+72EY&XS(NCL(e775UTLk1Y|xn6!Nw9Fa*T~mTL}}J8yuA z|9tVP%u9Uq_!jg? zy4(VG{xj!gXIGJutzw*(j3uleCjv5$rsGweI=Eu!0BiTZ;9dUZ23w6La2sZ)k$ZZA z@69O-DIvhEdWC^*hAaZ#b=6j9M$xBbc(XW+gs}@1Blnii=T>$fZ@Ye{02R}bz{Ht(W=c<5rrk%&@XX?TD%MVD1`43gS8tBwz z4>4%@8Y1yn0g{%OV<)Tw{$wBMXukp((k~%9D<1nU1>iH2yQF%tKNwqWL&?M1OwhN1 zUoqwUxrIp(Q86BRxTjd>X@#3sI77Pe1=y~73vG5^#Qw5}a5gTG@H3^!^i2i)rAh^) z%TSnfAB=p~x(tI(k~+Ff z%*C%CG+}hmewJmS)P zG!SlufUz;4b$B+^tcMLy`U;R%}Uh~-TgoQvU#8nAh*EEls{4itBsrkZh* zY_Z-4IC}gat=pLjCE<<`v+Fs&FY2H#O=ptbp-XUHuE6A?Wr(wMgFqzUE$A;wAzSNz z!+T>Fyu_L?&}<@0%GN=TJE=5$$PLa-dVsed%tm*kHE`L!2Oj^D1eHHQ=&iDbN>mj> zQ&>0cC~HBNy!+s;br>>MJK*2Zk7UnJANWsso1jJa@4F1FX-wiJG+`c-Qv|e{b` zToCj-W7pty5(Jh$)mSP0ktPaidwsPj%;~Qgs%qSVHB)DDr)LLY=B3r>`}rV$k%kI* z@)ObYu#{9biUk0VPJvzQHE0Iqp zmJdVlB1u?xvkaE{UxSZxIXvu>P8X)7%e&$O zO<`93Hwa{nE3nG771YRoyv$H7w%qMLM$P&Ui<_jm{#{WR4cg3S>8JiJUI4Oo1D_` zf@CQzpl1ZUNe;i-ch~-7n8oULMD8 zr`-i>rP<(!B3zePBGlJBf{J;+anE_gX6H{3nOBLGt1j>q+B?zVxDc0kza1M?9u$cs z%)|MS=BPNS8s}RHyk1?VSQ%IV7iy)7J`SJcmwms8J}3PJ`A8NdZM_fPQn#V-Q!72F zC&lh}kK*TlSMc@y4w#%N!YXb`lTGQT@nyXqblGIXdX;J_`E4=c$GK2PJ7{uC12!Gt zV^vBZ9Q}}w4I*n`b+-KXc+iJov`B5VL>gvFo<7F*tJhp;H!$i*dNh9PBFM!T%((F&D420IS!pyx{ zpnT~VXin%v`K|HrdR+!SpLZHwj-;c|`P&d(8i`6_NuaU+2#xoOhWEAM^uo-?aMm;( z>LSLn#T^OgFWC#OXASY0%pBCH$U;Ad0qS-l4(>|^kgo0hVB+6|NnwGcL)Q+|KU;8% zmuBJ~@f#p>FPyL9XT~{>%ZIt50r)BIHGhIY|Bcy_N4{pO!m&zgy8qD(%oZOH>tal;u84D_pE020zZW;M+-!MW(sKIO#|ohB&K0TUjanHNFM5>;rIW9S`F* z(~1HIKH(Uc2PG+uU?U$)`hERj%ZuCKpd5ocMq*%EVjj(H*aKJhYOt7Meb7(ILz9$J zST|=U9e3(7U)58E@z&Vk%;Z&&sk8)ECw(B-HH%5uUL{Vr?FMgP??39fZaFruT?Xr? zXHlPL@i0>;7ED?hS-3Emls__NLJra>bVmt4^a(*$Sr9z^F$~3J;>>njj@8e5`Yh(_ zMmYCs4LXlpBmtS4taM)l94lXiavM~5m*g_gI;euAG*w{i#zCI$@ky95R}IEZTMWC( z*71cm72>sO6|jsEVV|Gpk*1zEsOnIMPgW{(-Dd@x+@2Y8XV?;yeUt@Hb3c=bn?8cp zvGF)Eww;tOSxNZezVLKu0G4)1)8<}F%-4$r<4eK3f3GB1N3&O9nX4!%J1df*-(WaTAveUTk2K< zXS@i7>zDE+j|buf*=_XXmxFM()`*<^x4B6Dz8))|%W(m6rSQ`K2Za4dz}F2QAzfAA z9hZ%TnKA}&%Ek~?W!F;Kw~O#hxHl9Gnc{(?@A)>r52O9|sq9>zJ{P4I0e3%%ai-xv z;M}cnJasyi_RUpc!~SDAqi##ql6n^tXSbqwrXR_D_Nr*Hi!RaMGM}tlGL>~T&){iqY?4kpCow5dBH3l!L!~h!whF#JiR*+41*^z$%VQ2KIoiz$Sr$$|UV!iQLwNc%0zI*QHeNXIP2?(M==Zw^_)C5p5c}kV#AHD<^hSKZx%(vr zhSF#>uh@w#3%kHmHI+R55J{reN26Jd824|=G+66#6>e={xKz*sN|9X-x~krA_s$bq zblMu;pS(a^QWucS`4d^zQsCfp4}_+Qb9ovWpn0L69wp+?dAtSHOOs(=P&VB!a|v#B z`(on6BxwI>iMC7DktKqBBsfHld$Y}kS=%M!UYWn>|0n~NFB`*Fj-8Lj9EWF}58>%Y z|3DzU!^sFk!81Z&#oAApd47|i=KV*THj8l`kNdGx_zF69Dzf%*=`b87!XitPsp;l$ zy!ld@{q51@zMV~jZ-uRJPuqkVZ_K1xiBh<9(k$>>dleV-h;n-lcaXSC*C5-Whf2Q? z$APagyqa4+Q2JAx87!H`jU96Xp1S`9yQ*)5=Xj1MES~m8r}%_Z~LC zrigRz!pSMSgR=r?$Rf2K8leqgj$6>*|PN*p|Vi_lQr(LH7XO0X( z)ly0PSoV*fvAO^`ArA4iPN%?3XeYrGg@#=H2KkSWb?1iq3V{Pf0)G)8PbS`|do|H4`^ zq4E{zx%TiKRByvQi%Pm~S`Jyer=MRm!vR+g4&l@G?fgaMee`stwO|gU0eraKXn!-0 zjyy<&xY^^m-)`G+JLgWKc!_v6ED#37l^{a66#5^VLg6nyo>GesUcToS1wtn5u5Tux^G)$t;Rd}JFwIMtJ##DOtyHr_ zmvVuxIw}wk+Xj>PUw+_`%jl$_4E*&}oGYm9r3-l~)WNHRtY$M{x9mr%x={j}8#I~nbYKlmDr}b5M*Dt^K9ls;k4TWIDOO~UK>8-Df}l6is8L@=zJh-iX4Lj z?~cRD5jjYC?};A%QMmqdIC)t4iIfx{q&`{%d?pT2PED2lkX8Z<^%4|a`2r6Zr{Xsa zYrMK9kzTz%$P1V@jE7^iKst98_B~Am12I93G5H^cO|Yly#h66ETpLifyix;ZEU4=E|&r2HIwJQ_(} z%?ZI(cMid?5M|bR_6_jWtx@PrJVr4Q&T@J@JrHmKXm|)(UfzyQK40jpLUXF|egRx; zmX@eN^Ic~*upIax;f91Ii1ta7}ygaig(FZ!Z z7M+(1le>2Bh;fV*J-A{shJ9DDGXBWNH1G8SPeK51ZdIyPxPmZPne|au%fEzIQ4g2* zw8DS7YjCH=51O9lPC_@ya&7ahaFK2UDcMv`Mf$e!z9!7VWQo_UPS#3f;I{Gpv;RO?zS`LvH=g0Uu7Bpph5e|0#a5Cs@B%>oCMqr7q&O@cuM6d^ybDfkSBSZ^nWSBq7{=k z`_^hYYfzUnj?#g>v1#Nu-<7nO+`{gc@1bkc1oot^hS(AbcKSscdZvd%EnF`OI5-OB zeiy-BClULIIn5OGS~lp;qh6^V&=I6f?jI_E2?qDk{-rJ_-c!SWSt!L_+#Z5a+!azP zm{Z*O;02`KTMyCY19;HSo)&5}VQ~+E6Iub-G++xsw-fn3(-z{^6G>q4PlmKeegNI# zFub-&h&wYi0gET^hTvEy*x`8>)ilqOXCtln#Eik`;LoUb;4?41L51bm3eI(HE-s#5 z!4H=DgM7_aG7w^pk%u>dub{8mH2e~73i8(e>M8I@T$UxzdP){{G|()=L|*hVVJOHd zqWk=EV4q5FQEJv7_+jLPZMW+|JZLhf6pG1;azA@z zsmD<@Zpx7^%u83|&P7&J1F3Obfo>?eg#8CR9d(?0UYS}7wB~<3Q`ob0t6=)=W%%LW zE3Drx!I?h(NcX;z<4ivJTRD`vfXUR0Bw^tP5E!O0K&lL+O>6nz5+Xq?IgWS9GZfGJ zod(^gIJl^`5gw&J=MVheLAc0r@@Zx*Y?qf|LV51I!P!$-pHUeZbeI6+x+j9W#yQY6 zp8^6qD{ttRjzD*dL*+B&Q2Ad!&Y!yger#7|iwAa~^4oi`P`Iz?;HPS+So(+P@ElNW z|1qD(o7~J*y>IfALkBn^su1Lk9Q72ma5FT&yJ-PPFWz)QiS$zCAsk# zuleOq)wryC-ywPZ4A!_v1n%zEWg5SLDm^Fw@l+2C_q8M2j2D2kk`Y<0KNg&V(_qRc z9e%gtEIgpy$k%&z5KCq(r}-l@>AzQZsQaNRJlyL8%VXu(;hr-zrBs&L=*45#PjN2W zp`T)>AWzZv#VC`h#FAS=#@OzJr+*xQ6Z1zeFAaz_mV@6Rsqj1MJHB7}%e*eB1Bc!u z;K`gW(6dvdJKSqv*_#J2Nq;6&*7JhXK&nujsydH>e-QL2V^Uvvc{Ry1o-e^qGx()A_MvBP3jAp1r2-H0;u{##lmz028Z4>)89q-G;oc>b;gZWY;6YXn zgzbNYrA|*_?BmNAUv5huONg@v!Hy2gs2M5#dpf%xKgNsnd*C(=r2Xsb8;GUGuR5$?hiGpi4@H0 zXrQ;u01b(f6l^qJNajrt%rFUN1m+3qH-!lg@vmk!w2l0wKdXMxRlU{p(ys{Al0U)U z`Rx{298*qyc3#DFM}#F+l}LmigT^*{l(Ejk_wsy5Cv_NQBgzHyXTz0ami)I%_6%U>>>sBaevlWx+(-9oA)U$9$hMFq3v* zE@yNhd7=VSKBoYdMrr)j9lyv?<#yb1tr7O-TQh?(yTDq)h@F4)A5;p`?AKT7;k0WE zYG1A)T0%06cXB86lnT5}MTgozy4MTIgbeO#~8&z_~!J(RHTs!e5h&W~9&RFEQDbqOl zdh@>;`ddc`^QFE*)H^fytPzB!sClf zA_-;#B-5-Ehs)mZd#?x4pso`B&LST0@?yYY{U2BraE|omFCo=0J;C4P8R@wb4*|DM zK|Y|E#O42r9w>o8(b?j1O7fOLP#sX6vm<9DKDa?tibIo zQ$(3I9TfTUf$aU016_wlabr^%uC;Xs7oV|QY2YWih_BD|AAJF(jy4kd_%mv>8shra zS77w99h7XP(RJ(qypC#w#aq9_lxvB&q2oMkj+5tdMpZf3Ovv7#LA0Lv88@!Fi4&hG zfR2?GH+Iz*)a!&Jxwl*Mis4R?@n*h+ORc2A5mj5DUTF*i5FwoK8Q( z^G;^;i@qhbuc%3E69hnw~E|Ei!UAGVo zJN#gtu>_{a3x3B>4kW(_JoC0$xcYr4X5DOuC3bh<*Su$7IdTmYxVQ9~ZUdf9eh2JN z7APiP#1k)vaEaq{aQr08@;CFmX$+TCD2liD3+4>kCUS-Y-Smkz;>FkV+4t)|$)MhEnsoafop>c&u#+Pm9`p-l zM>S8OX4^M-xkiPzV7EG+tC+x#wNC-10%e~5@-MXbsT6P0?%QC@6S3NS#t_2IRQWn< z6}TDCTZmD{2UNZ=8~Vg-;JIoO91E|sYP%`~={tW|%u*RWph1GT z;bc>d0Ur(MvDPrFT`9%za^^HFR#9W|4klptejmPwtc2F#99%Dm@j8e6F}{zXk^UMK zdCw86--j^axHMGjCqQ$_{|udnBb8qm$Bm542+^=YsmMsV@4e@_4T&O>l4z@t(%#v! zSAv#Wz^WJlx^NjEF{Y+zhr~F|i*}W0ul{;4s zn?wj&WR$o$ZlgHv-yGPHVoAoYw1L$!c4DJ)b~4tZF9`U@+Nxab>jdjeCR8`(5$5vu zMviY-6h~J*m_6p)#Q7)V%6a9zjcKZnWrx&Gam<@snB~jGnz{6BX4;>39F-VH_Natd zPiB6WQHymGOejfb10);SNX?DRrFA?(Vp%QcrQsPtX6g`g$Htg#*m6Wr?Dm7>)=%=i92W^lK7%s6tIQ^FY~FkNcLOkH?JP)vWbFMCrt1_yJQIcc+)4%6GL z*;^Av!|I%1qlGN{pN<6EC7mGFzPqz?+ukuZuC8Q#=PY4-*$;yAHY1$lP7>TRKDB}| zhqKwE+g@^J#$9F=!s|JQRpOYUs;joUcfMyz+m+e=-KzyR3x8LX3~96bA|=?+Jqhfm zVnz1b{Qo%Xo9sA$ioMv3RU5^-wcV^lg9@{Mi6LXK@D`gf%9B02e1h$Vd#S8IGlzX( zG>bXXIo>u&=^EP-wUhnfHNXVMbqO#)n|*ZaW!1y5o9s}~aZcg0BE~U2mpwW)hkbIw z+IDJf2Rr|bf^C)fd~HvEbL40R*3SAgHFEy@{9?{k*8+A< zsbsZ$O&v4)UL>1iwvuV-DPv+?<5|m$_R4>U)-x5u3e~;xb&RfZY?a=RU2JmLbOulA za|h?TF;xWxoWC0G0lRz3w{ez)SFrNV(%g=>G3@xPY(d(SVn!t}mHDN>uyy^J>=>5?+(}Yd0!~vQ zGwyX_RYlT8!MHyMnbR{S3x>qK{SV@gO+p`wS*k|7i7{j9hZ1Dpst15jzGf!X6X7-)#WYU{W*u?B|W?#_3N|*Qxj9KDeW_|F~ z>H}Njn7{iI*!^=`1Z9~iti#kvf`+$x)yi&hOj@NsTe3f%QSPv0I%GYV{{2C$lDD?N zXr%<_@1a`Gh0uvr%WoU8tqB`#<>rd@R(6-!ojp%DkIerFqSN{WmWw-?nUWFgwK4%S zI#(djVr`hg8dVM{I4p2lyirgh<;L1wQ|Df~wTAt=O@U>{Y+hjg*!)*3f}6x<6MqYWREO5$Q+9Dx81$>Art#N znKN!l0WpJ>KfFhm}#F|2>V(ey_~xKhI>=9300<=I>_~ ze+72JA5(#Xi@0YtejL|4&_etsY!@60Gi4tW11|HWkzH$3$|PjJ5rl*L#-z({8bw|DVkmb1Qt9rkY#cvwfV-G%M~r)Le!VHm1XwmZY*D%r95I~ewd zObqkfa-O+my#v4>aUZqEKUx0nNbZ_cr`@yxyr zt?b&tvDN2}iZ#8bV_8k>3xav;YZ!9rHm6Oun^~SciQRcjlHG4vg`I8 zoD;owGkd)EF>4>eW!H-JcJkkAY_GY=Fo#-~G3qDBu&ejGu%|N11Of9nRSm2T8!PrA zu+6RNdDEsXnvZ(?p0|LB#v`oYW_qrm#3*Q=4inozMOuWdJ zO*N`Mzz?#$6g81mDw!(?`tK>wxjIugrpc!!yx`%2vU&brzA@^u)j)n#Ugd%<1e z9NtdFg4NL)Q`MArLh!Be9;FIBccGsnk zf{oXwac*=z=7itPW%h4MU`Ah8W?%1c;Am~~Vy$H)xJeuJ*rh)wFc+SkVh?XuV-!Ec z3fO=wfz738_Da41Bc0h~tDlSHn| z6moCBJ=&^l!qmRm!q{pLw7xkO0-eW_#TBO^B{+^ac25<>*BKGflx>*lN#Ne0NOZ_q zjzMdc`1>Ry$cDh3_<@e)yKX4}hd~P%j3|IAd7+S%zD^};{-aAv+v$-{7yL{z z=<^IAIg&93WK(va#Y+OO&vnu?qZYbT?0b7cC57ZZS0dW7!6d8D7H96)2ZIwPSUJ`K zv^16AM2|w6A6<$WHrgg45P_8V) zH>ozjO$RmkA?Hq#_ib0{`9~h~-i=^v@Y@9X{c{K}dkOC1Wp}*wMC2ufcK9F5H_ONhsJui2X8*H}Jqg!mv zaZl?o`Wtg`;t~<9PTr3OVw2}Rb>XN6`ov)x5#fBY16e)kw^4JEL#k}+*z2lzDBf%E8RbmiyS z_~CjIc~$ceo1R9(_Jf_Ez1tdG?%%_T{wApQNQAkU$Ka0VUG$e;6sF3ZgQV6_{JFyw z4pqCNd`A|9{q*7eFg;HX)Kjq!aw2cTly|(0mMKE7BVBZ$eJ8nh(i zL5Pnx|S zo|PvDxFTQD_`@6{o2sGnk~Av2{zu&2Zh%prz3|ROeb}Tq9kcq5l22bNv9$Oy7FlVd zvD+xT?qdV5WRx)Xo)P5#y#kA0_OcU4ieS^`hh)~m1ZY)^z_o8fV54&m&%ZPULRRZC z-nwZp*ZCQdxqhCG#E*y4Ar=0k2M%IRViYWu=HT1hE#&*&SHc&&z)2qna4-)9>~4c6Yo{n;jH@lg`|A4a3h_k;9hv?l#GHQdsHzpN&W|5iaFcQa<)cR-mVW*GO2748`R0KsAo zNKE$zT>d8sgEsN7HC++qK8DdRXY@%}=T90jZ3ZgK?FOH1321Rn6ZH;Q;`Db~czDuE zeAvAa;{IC*4-?GL-2Me>Ub#TOzxYFMMb3l@|1G?eMURQp(0npSbQrdnPo<`Q%cz_B zbi6n4yLxY0A`T{R1M6ZH+$MZMpTtRtmZaZ;0ecm=<{-zHJ7O4Q zj|ZFl=fX<4leqQlTAXw&7H)XQVC|B;>Iv3vFr^|JDz-Gh73+PFUoZ{w7No+*9Rc*r zs+q9GqK>vXOvO`fE%aIRUSwqq_>mq6uA1_+Mnvi7PGfRJ?+TfpTYz&lVt{|*9O=MY zuu{Vmf*Lo{^#>Jkf!j>-c0)V{M>^xGHK}xB)F{z-`2cMB9u58yd8nel2`B!T0w=rc z@jo?PYWMtvFf#KjPeHCjcyruAsQk#nv4@&i8+M8)btI9veFCVm-vOKW0pKLaCT{(Y zp|H#zD-;OKaSVs5!fQAy@E3fTa)K=KnFoiZ7voL;0!#|CqtrW;7AijjZ}WWN_24w} z1#W?J{}G;4#V*Kg_XYzov!d8P8d+TpQO{l}{^G)=!cL46mWu9^L-DS(L3t1Gx9y_k zygd9ZTMSiVep>WCF_T!|6;>!Z;o-iIM8wVz`A`3Wt=%g0?1Jm$btH?UU)|yAmb44C za%AAov1K@_tdnOo^B#TEaRx2TmOyE47w_7-GEDoqifl=ngFIy|=e>`| zp%#HfVt(D1ajtl~!wHx>l5n-~8&PeJAis>a(%gR+2=9?H-^-(k+*uz52NT5%XW?^F ze^>>2?pM*gf8*%KN7IOf=P8mfvqCmytWO24Ueib9C?zg?;xU#oVNuV7U4u zu77bEi_d(ap2kb?j9@DRbFuCZO4NK;z^T(tO z(7&U*=<+6c40DRcL6aDGDK$n^cXzaCuU{-~=$Al`x3B2;Kp7mN>iBh02h44$#NSIl z($DKkXrQzuoVFPuI@wjoYlc}!b&Z^YOPQ;{&{IQdC#30?C? z!Tjns=&E-f-8-6*J+qyz`*0DpqV++0`ER`l8 zy7c+vr_3!ZEn5#7oj2&zaUaQSJym+ipo$m2EfP}h9i(l?PXLLm5W4Xv(`H&r^`axe z#g>ByL?NV&Cx>$;7Gjy6GhmhuNP!WJiCTuw^p=CtKZN}r^Jwe+4%piTSfmkyYuz2F zTlN8%rfG@`Y|JrC%uJb-VGJ7YRbfN84Q{V*29x|6u2WvLa7m$v>?n&S5@!-g^}pM2 zX`VkiTdqUyJ%n|iD~0DK`jhfbU1Dl@2z(bF#nPi!fpuSvW8Qc2^ovrfpKq9m(p6VM zboUo*E`ACI^+NLLPbI}0t8o6=qo}>r1n$!LaBRVA?w`#zSTo0#7?f?Ji7KaPjZ880 zx11vd&yz{bYcq&Q_k;x%$EnV#g=pF~9!i!^5}ho$2>*N%#Ei^?G%-aE$1M;qT?A>< z>?}7#I7pQ>Lut{E^+?0Gyv?I#k|i->X2hi_SXg8V`n@LAaV@KP=U&W4p;JFS_&OIu zyyWndkvV$aPJ}-%*W!gl5gj*j3yhRx!A`VLw6sGN-v4Q!_VzpR%y_ZSwANX$v6te{ z)$D;X#XX>1CL|B-7vP!`A;P;`5!3!Hr%kgR{IfSEbNIGCS=^N;A#_C0ddL_Y`f^ta>U zhyl;lrK5Z`Z(sL8VzS4{ZL z6I3PPhn**oGkgy;Ki1%d14<~n(wP4w(^~Y$R26gd?Rf{cTtEwY8S{uLF8E;un7c<< zY}`qd`?aah#3b0fXA+NyE(!xIHSpfE`_yw)9PL;h55%YeWjkHz^Y|@ruk{NR4BesQ zFXhnW9%qz$!N*fR#dO>kL;iC&XY?6SMa2n<@cR1|p5pEl-j43GcxmZwEP0bbo=7i3 z8TC*!I$BO|y~qbMSy$M5MUJcU*96`DFT$q#f9WAw2L}6^iKlBBI^736zr!AcQ}4s) zzA$_x|CqcHTqQC(S7F8e6GX={49Xps6WJ;uy|6}(k}D%r-jG1b&s>mdoNt1OWZ^E(l#LC5t)}aDwmOP`bO^w94{1m$V zx(?N0O>p&3F<9Fa2zU72gtZC9cxdlp^qLgT8)Zo$_Gl^S>9}&2Ou0cUQ{6~}(`X3L zZ5H}YSw)*v3&>lYG32;^1C2j2!YkYwiHq)eVN=Lb*m|}Ij?6MZ1NCjF?YR|s7e`^@ ztTXggPck0a(+5vtIoPVV5;vs%htacM(;pw&i5kb0_aVuLX4aI@C>2TgCGsX22A_pb zS4;-CD_l{`?ECZ9$2KJhSnNt{F!sELt}(GoaKaoZ%{6Myvz|M zjO`#wLzBqy(O;;}+J8K?EicHiej}Wl{ZW`ywv*(?j76<)VvV$-F_BC^2+^JD z_~gz~;PflwkzfCW3QP!SuRcrm&tJgI+9T#*jH)M_8?taGM*%HQ%b}&ADQOvBMAtW+ z#zc z44a<0lVyd~ptY`)tj@TI2c7;CW(ph0uCcwa`#A^3JiiQkE-8pgl9TDQ?+-!q^%8P* zZ?bU9sc`V~_CY7pR*d^y2&>Mo=DFOF=QGF6QNkve%*=m8X&z4|ChXz}amnNKZYFYh|8NbK*@N2Dst4p;s6aa{;2OEKoO8 ziF}!+E&3rmi)9-+Fs*oqEU5{F537%om*HhFS9gmLPshT|3?~c}HjqVzM=+ILNd~TE z2vwAw!S3TUbk5!b10AZE*Xs;FoR08*Xnvz#>Rc20+yXl2+xnGyLYra{g{7LDOK zI%bT67A;@w8Yv|4k3)!zWI1Lk#X`z=SJH8CEZw;5GcV#zE>N!`8nq!DbmsWrfT|n# zm*oi)R_eo{uPbTmKPmpN3yU!R?oJ}HIG-$TPsF8ziE!x26!2+@f?Z*kakr5XAGOS>bI!BVt_UWy3rLfs`x^X1B2m_U^}3PHB-~+ zO|?{Bt9s|qd;S0_ zcn{NbuOtL}KDNGersMm{p+0#oUg|bL9VZjaA4t5{b9%EVfvlLg72Rzo*V` z3(;fObx7-c1F4%8@%AJO$bT(@K(#9b&|&zlL2T$J-*H_R$TzXyk)qk2lc1pi9JA?gDvXyN+if z9|}RLj$+pLH`=Q(3v@F+0WOZkyaEk8H8FX={dtY1D@0-aZf%r5G#M6c zKE_LHUr*ML_v5$Sjf8@$`NG7Q7_vt2it@S>@zV8b4D*l>4bSyPwO9He|657)*0PS? z{8`Lf?*0bdHp)*fj}S zqbTK^ttCs^O7Z*Tb7UbEb1v)xg_glTs7t1h>^``d7A(1q*LEaR+qP~9HK^q6(MW=( z;g>}4Ck7s84-tzHIXXl30iJfRg-O~~SgH4#swN0&oG=K67wMCVqY*^$*?FNwL@DWh zz8j7>iaADQ+d+0ssqm?B2}IU7z*!|DdzlPM$AF_yIjU5_y2^z9Jy&4%NWit7AmHrV3CIu1~y`^U05`+u^U%bz+#A zPkJPL!BHFs^~X=f=P?WMaA`A|W$5q?Di1^L+Hb;i&Q}uEn@e{da)*ugMtDh^2gwD; z0q$VU9a<9dg_aw?CT20!usXLB1`nLUosLJa`%)Q%hipWD-6+zmwHHx8o#v4 zh=%9*qn+zd{B8A)O3YECfsRK&J<T0#B#9 z;z}DK?W&Qb1J)97r1*jG$WIs8BRxbu!yFLM4cw><4`Mgv8Z48KhVR9Hh)l;Qc)BPX zIcF|lj!6?G{q^W^RvKHQ#JevOZ&T~h5~359Pk0#i1J7)phCUOIb!gBJs$3PLUwLTCn{As^s4<@tX{hopK#K6xBaewzGf|Y#+Tr38#iqGvj_fl z){yJsxPQPolt&IOg5#Fw=#TU+3MOA+_S0v=0A&X(*scXW^C!Z*KngZ1^-yD=myDOs z7WPKpr87592K5<$(`A<8*^+vCai|^Nk17J^V>`tRDk^m$ESx%5R^*lC zNXo?TO;FSwa@RKyKF>)2f64P;nqUT=ahl-h)QKx!-V_?&UC6!AJq@;d-iKq&Ca~pO z7}SRQ3N7~d#A)QRQo{7zS9?-D8 z-gvpj8pCaqV6guo8unv>%noXMZcZAbxuj`x3{mH-$CTuI zu&%`l7wwsi4_+riS#k*8_bKAlI+$Qo)^gbJ{S(!8ZR5eb_3Tg00&4S;3s7jsPqo;I zoy)|$!)fQJgV!Q3da8r3y7Tbjw|%hUi3@t&K*e8;-&-F?zXn8+ftM|0 z+3*E=VtTW1=uH;AsV9jBg(`6U+->6Va@#Oxj3uc0{-w!@YiZ;4Gdzh`=fU{ZP4Ki&!sSzvuvgZIf8Gc%+iN9k zJl{-*iglrW>W}KyvM*$UnWRup%qEW8tRzZ*n}?sS#(|~ZY*^)90V;6L-B(-k>-?{2Xx(>wgX49K&qhm$Mhi=Z%hFq#Oa$ zj6b4+aDu4x<9Yb`_XYAfLwK~-0*@crhML!fs4qx?^N;(8WF8mgqb`71R|VewD|RJ` z%;$>E9uO|mkHb&um6&By4vUgANeXiS?*ugvo819WGR;u5_e~!CeQ^OVb#)cCwB98> zQre)gT@JsOo5O-yE|m4fLcta}p49HSbWNiWdiBRZ({sd+q9fS4t%fA1eWJDP;`-r) z1mA6vEZibW_%rtxow211+8SBHl`04A>ua$rMw48et4}_zu)=2jZ`5;X5*csN$>V%z z#2u$?`FfEF6u!TPu!ZOF)8eh@5@ZJY5)|vNJfLgTqPX&hPr#EsH^?u&wP4pAfZ@GI zdBaOtqU-q{E5&R)DeeyRwlx+_Iywa)={LgFC?9PP^4`8O4W*Yb+ML@NxMpY}jwz&{#XUJnt&lX06G!(WopLVwqU)92Gx zp`MQf{1Mj-Gey&J`g(7aOx5PST7MZ=?oz;p{R_yJ`S;0clONbVqeD1%!UgnD^QC+Z zJ!+b}fPTx!z}tyA6cs zHIYE67U~=jge@FE1)Xie`wmrj*j7fg-17>&Jj0_NyRGSire|G-5n+F0rnt;?92m4!kZ80jpm|FRppRyI}Af@69+3FTd5ISG_DP@3Nyt zqdYO`_FEEt>>kPA9D$k6sTfnd2u>`xitf#W)pu`gAjj%Lh3lp3aQ|;BOq%h5H~ozs zhWstZi;LT;Cm1&J#ys>P(>XC#_*SRv(d-p{Q&DDj{N9{=N z%tYIQi(oD0_+DbeUa9_7eIBO(CXrXlwZQfhrga?Iav1_fETX&x93UDC( zCvRYEUMfWUG?58?0LOdBgW88lM8netyZrdbo9zmY6T|W3(k#-Ra*1TBU7;n%FVYJm zYru5%2A*EO3n}BxE`$@%S*iy zM*bAY@)s>Bg`%N%n&l>%ZV^#mHH{2<$&DPq#^ z<2YP-la7g4ft?@pNovtV-dM3kb8?s>e?b2!Srh#T?r2K$WBezGnkA!fbPkYHX15`7 zX(lDn3AmB-5^g6Pfwo*Nc&*(_4)=wk^FD-!CqD~=#?FWOXMNaG_6f7%ZE*A>U9j5T zL;rk^MemF~B;rvF%5SfR^(WeRFX{|=4#vC4tz#{)Qn`yd&mYHcdv<}|YLVebhD4Hs z_f)aCU5l7#EJC&Y?$Fu)nKw0{n4T4@=n`|3>DG2-_;kPsiuW4QP3QUIUO+R}+o)pT zGatB6+YC{^OK|ciJq#TugrN2`yuiIh_WM}khPfIT;XVn22X@kB?lNHMFi5VOufVxo zm&u{vLc0AfhkrKR1P-4(N#hUghKu*t3D4~RhGtxGpEmFe%BK0_piw*|N}R;pki!r) z_b7ZXT+K=?c8ACNw_$SISkh$PNAGBM3s;;yPW!*_!D5eRWXbsgunbMYOwI;iLd%6` ze8=&#msQZ^2hZZ)!NYi)yA!p-Rw6s)4ryO91%E6~Ab;+7sZ?OUkF%z`NUkc7$3`yeaDnmQQ{lAPusP$@Aa^}J#z)_TOS;k)VkS-&A` zvH@1@B;dQ<0RpdF0>hqFboF&=z= zp#&53YK48}9aQ(xCqgepf#WeZbS?^q`JcyOT*hU1Qeh>u$#{V|-IsYG+1u%ke<#4u zWf%sYY=IpyUNF1i8EQTK20N1s;F5MQ%^l81gLprrADu|&taw~@Fp`=(mXIIgIw67M z4!bIo@ty7+q9RBG-?2fMY}^Ut+9PV!`Gvr#54;moC82&F7fkoBfFsTJSQViU57<@= z>hmP!&HgC2_$a(nIo>AL#J0LP3nXa*s7JXG&gRe8j(iZu2T&|)5Dyep$er!6*)L#T=MxHM%;|2|Z zJE`SgBhc&YgCX*QZg1NRZssk#=xy@2`ucRx%jn=<3|$STcje%Az#r7?c?PD{6!2vj zpwJ!Fn_rP_zG7okbtljtxeqJdcZlnNKAgQsUgTcq2}923Nw~ip9CaKf69&?OhOLGq zDKq}l+Lbu-c_aw@M)Ukj@*wzV2$}pbpYWX|v8(MF-F8P0_Feu)w@1lCy22iiSmBKY z^7BCVwE}E zr^3On-FSDpAN=jSB9v{K33?Y};hdQ9-nuWFHhn@c&bJ0KDwAwi z{_*Xg*M;HG^0b1slOWJtaSVG8y3(M@Yhn58*W`Tr3>aNdPj>x|5~|6C(j`_S(Bzf_ z8;50JnNdAS3MhpH?}fZMZMz_O?^Qa%--BKrn*_rOYw>;aDQ@Wb%Xr6gtWc>z61Tof z6lNw#5G}KR^!JAqH2uIOlrS?9O^{qfHz-EHWQ{2HnN$OP)wPy)Q`Q?3dTtP5b}ejK zm&@ID>KYvA*iKjbNnr2d^Z0L)COPsV4K0m-l5_kcxPO;Ez6)H1>tj#izh(gnJ&dq0 z&5MEVdo*LIc$Rm38u_UHkl0MJ;vf6A5v;$agT#yk9GsRzj_eKR9e9{dmv+U2d2&y8o?Qz`?Mo35u$JOqxbJpeE8&`(CfA&{vo@_xgcrLD0gkF z>UxhW-GzmPQ&~YE5X030GH3SsP4Oe43gbb;K>#Th>7VEt`<%sa72;j z+nu4RHua$X{hKf`Z5IybpQ8~Qv|#=6I0(J?k=UdwiS~A7!k>Z`S|k}xcWc~$2XAa~ zK<+rG-}J(mKbF8Yj^fV~-(`hCS8}}}0~|lyq5IB+a3=kJ zteuQ)UyCa?+@efyCO&jpkF}{i#HLgSq;Ktk5&w8@#_pYpMPu_|W}6lCC!fLHo|fc9PZ!Va*Em=au?z9~G(*gaP9edjja1oGUbNld8%-$ZW9mpO%?gdf;M|R1w|o{3oNB|< zfBp&cHqL-G=PzJne?Q5(+lt5p!F-=rFx7Jvc|#t+`RCt=eP142`TkcJn7PNAfPJEU~_qx=RX5_T#)X9^isav6%W`GfB{ViH8rn zLY?MMIAgE?e^@cJtLg%HOd8GGG0Ge|)r+fD$}?%kV-E~=^hdO=2B!rzNRqGOY*%Sf z%zx9takL#g?8>C86--6hBWvk`SFI4y?tygjGIZT{1%LdpL8Gm@u(RF@t+uWuGMSgy zo8I$izLvX)bB*No!5Ua)Gzu3GB;+i^1d`%hTvL%#PFa$+(6h6&N#5;zv zX#B)XJj>F@Jp+GePv>_saZMn$&1mJx${!~gwaReI%m(A$><|`isUxRWq{D|$Mf5Us zh3Q{qq5S4zkXyn7g)#Zq(sGZcOz7e5vJB?#>q+FLmk7xyOrbxQU50UWn*o0fbFZul zCwG+mMOkL5D8G-7V`MK0YgS~EUsGlXO*4IY)!@OipQ$0LohvCSs&m2hlacOOxExZZ zIe_P~as2Me>o~=D8u{IO38P!z!Md(gocS@2q>T^5$a4>cO#5G+XI2i>-?` z-GmqrNIT73@XUWEa16|-IPIo&zk;DL)d>g2b%Av~@~nn4$SuZcSIie!n6J z-5LsK3?3a?yDeu>k?pstqLqTS3*oyeJ3Wf{*t@%T6wcJ+k$)1N*M7>fk)$i z!yvUsi?QPSHoP2O+ewPxS2-_XrHH&LoFi&W+e!3$lVPsjO891ThfcqB8qCeFQRBkP zLd!qP(f!p4(m&%bIc_--LbGD%v}a-TKkERZy5|^3j(shhq8v_L{O(}&85h{;Qwe&O zC-CCA*y@Pbx8PJ;2#b`Xgc|?7q&<0g@Nm5`M(AGS#h4D$$!bx!YqJ_`99>BS7f(`w z_9Yl{)Q9u&MZB2-SE`Rz43IuQIXG}7Q@BFvB}e_mBN7JnJgMuysqu499P?|ESewy| zFj#^O+&%P0@@ep?v!Ig{J>c(#+a%ONA2T(z;AphBuw_XR{1nF}H>F1SBi0yMeToHp zo*vpScEyvnS@>3>nT)8`LbqMCFemvY>8`s>bhF!W;>$dQt+%;VDA)k|Bbj3-G|0}_d$sa8;-`#>H%e$y!|5(up{iir~Nh-}f zRZaGOxI^PS^yuv;;(q0vl`uJAG`}V}4%WRGBHkD0p}fu)QnK?NKKa*0LzcAB&{boA zw_k})sYM72iiB@RBOvnMT5RjT2VZ7{fd2a3p#10ta1~-e!(tuQsl;Pg*%MT`j_}20 z8;MT6M=jRXf|cqdu!+mSjH0XJ9HJR^X)mSsYB%Er*&kHa@g%-&UyS^cAEctHkct9x zaMv%&eKSoSdoDZ{ZY{Ck1x?ICzqR8@c>Z~wmal{GmVY|@w`UeA<=TM(Ca~=b1z2me z7JaU=xM;`>bJjnA89K`1ltHY|s<*_CC1v1pB?-qjpCt!oOY!vzQt*(r3(jcFgrP6< zu|iQ1OI}{Ao)}lp(;l)TecdfktGX3d7`Kof*?Qr|R|#aiw*}k?i-81#1dP~d3}(6( z)rUNk(AaxAMn3Yz>2w%=xp_lC#C|eEbp&nIGsw{Uh2%klHkOSB3mZB<|qXRrwZX3(~Ks`ZD?VyOez&V!{+)IbSQs0)U}1sd&{HX$xl`O z$gECybkdymy*h?92mQpFAPIicnAv#eyR=BX>Ywn|*@^rE&Ysl0eluKH?SM1Kg;I|| zC4S8`Oa3>#Yq-=zm%m-U0YY4^Q(n;`7%G&)baBnA$&uzKeK1Gg!CN@eo&okNEn)l= zWt#o82-E{kQkw*Ik-jA1{%*I2F!@z<`+pn3Xh#)Hs`$&*TYDKJjS^9P_ce`(+uxD3eL}@%K77eys}|&qASN*;eSTPNnm#Tp>0$4bnWuVCnpR zSR3~bcKysG>2D^$7ME0%TwqVnFTF(gswK2!{dIiEb4INd`_TA7IVkl$NA0NNWaRrw zvRpMBo>|nAcF79NTIdH!esP$?%SOA9-!xbMFRj+vhyADGi25ce$Q-i>UJ456o`a_#@&hzxNQe%GSz+O`aJr(b93EUaMB61TP*orjdjeaEYJP?a zKW&)C5BzNaI~H7knO-aCK+F_E`Za|I)NFsP zJ^X9sVfS1Qbhkc^U3%qErEP`3<0`Q}=_)cZ3vg-5FS2aQMA~gtLa+B97P`F4z{h4% zm{sS5e`7OoLfu&K&MGCUo8su`OSkBG6*cHTsV+L#Gz+fC3yA!(1fKHEQj|Dx0)KQ# z@x4+H3ANW0p<=2hMg{i3=QS6g*mM%C+rq+Jx*T)0u7FwaRY-eoO5KZ3g1nqE8X4`U ztV=J=TlF9H?=Z*yMZt9Y%}Va?s=MG>dJ$@*)I?Ca9%YjviS_p)%o-Rg@*PZt(q#?R zQ?IN;j&}xJc)S-Qqf2lxuZ0|X)=LK#%Y*&jJY00Z0zSOkiyQRslN5^+La*pnqP~iN z(W+j!VJ9p4kn$9APa)VUI^bij6qgp&Q~IA5J}NF2d+DEoVB~uolwDKt>`Wb4vt%Fslg`8e z18wfpf_)f&O^K|0vy2Oka{Q9f<3#I>-*R`XNWuHz&BX8KLHyVf40axl z=i9gyVbALtT=MQLjMOyK6Q!B3$uUdVsC$=e^RX9gOX#L2XWd|1CC}l*&KKl1_bP21 z?Mrt3YGEeFR}wj8J$~DHXF7JzdJw&};%PnQ!k%klES0PV*bg><;-WUxNc%^_^e5v! z!zOT8WencWs>qbO1oH1yBxka?rc%+$gxzHY=xy?y_=<+9!he-?{9r24m!AWAA5MVD zw+!5Dd70cfY>$~o-O1u3Eu{KAALbXHrM^wm>D_mdA|?MsaPm42W^p&L%tSmNy+2m; zWL1~&#JtB~Ia!rhs4Mcv?>2%rrLFK;+eFy8XFKuH2&2cje4ftFFzk!{N3!?H;9ujN zLYt`5IMGiEY?CzjDqjYKJI*AaZ`vk18)0o zljk2S(a>6gAC!{Jym#7x>-%3}v-BZUCtE3V{R&y9y#qg!ND^*03(^jK0i%3zp5?+9 z_Ab~4IbZZhf?=8-JBBt)hz7^ck-`xxKd`EifCo;x;#ry@ z@28O#9^5JiH98K+&dZ~7?p-I7qHf}hmRxSq@CAJCF&V!UTH}`+N_yv=6$?k4;$KUA zquk-B@mAvIGMa?f+=7V}C!o5iQXQiaht^RhOb z^FKq^;aB4qhf76EyEHUtY8TbL=iK*1+M-R-Kq_R9j9*5fNk$S)6d@x*z31Hblo1jw zEs`xoRtZV@-GAVIKJPv6_xqmj^E}_@S$Lj)IG0G}%dVmXxlEL22kD{$6S{*P$0e)J zrpp}0Gf_FakXx^~bM4Dr)X7hVc^LMH(_PMIc00r|<>Bhi)h{QZc;j94*B3cjKW!h) z%&unqQw&jX4WSK;BHAWDo38ep>U=YZatk-aQ@el*)aPLUDy;m*NLapso(`Cv>U5#c z6ikp*qC$0JnUu5fhKcCM^|$ETji=~tiwr&g`J8aTpP)lwQ$^!qxvef7_cdQoGJ$=NrSNKyg4|966uZX4uEJo1e z2kl3u=$h;{y2*ExE6Z?3hqk<8?5c;Ew%{V}fKv*cR+%q6GB~lCwx6L)qZhYzWT|tp z);p*>&yb1kd(Lf1*vIAd9i*RwB&|VbzQir*S4Z7TzAyq&9h}|K zFOU}d!0nfsSUo-TDNVL`Dv0(vB-kD`UZ~uBm6Ptuf&{#MekwlmvE%M`WW5ReT?CKyTkR0b{%?To)OMvHoDc6 zKu2T`q0W-)=t)Ex;@?R}SFzS0_Al@~@<~4ax))+jr6Uh__Vcog8y6 zcQbmNKT%jSnnL{&{OG3l1Rb$Rq+fqzQft$Boc(PVddhsAKyJ!1uI#8tr{O8jj8!WO z2SUm@8+$&t%5AAoqc)rQ_EH_m=_NCV*&zBM@GP^*X)@Y2Uxn6`WOGu-)vNcq-WRmX zh9i^abb5NT3tHU!4sD-ym>KnF>Fe2(897>qW<(_**NsMk)7|dKXWVVhFs1}$tNYR? z{>e1%*LGSHu#@pR8YhhLZ>CdL+@z}4?9hf5CwjcniMw5@z}T3*VJgoybKmyo(qp>y zOx4~Vk=^>2{)+Ah@bWF7sso`;SsScOjJ zG%>nYr_yO{DQIv14*G8LRiQ&=CA~NL8VL%li19e~Vq4R`6=u-R*YA&42?JT}V7eCfuK9CRG>JvA)SA~^K_Kh6+bG0a5 z;?)V-yV`=T0qarl#XGbRJmfr--@;wnkJG7_57E<$lF&n=r_9$fG0_gzFk>L2K@X<5 zIA>o=VeV9pM?VVcnG&Z+q_RcDId4)IXP`TVypAlPZ=*_?G(8@TJ9-9GO!h}I(i*GF|+B0(E!Sc@ch6Ak-pF08qH11 zX0|?a;6fy)a>L#i=#-N!Xy&jt$`}#tvNe_>Nsl{7b=w-mK3z|p^hKDpSPr*%eF^tS z-4Xd!bWp|BNnAnuA{6^CP~hV-AI{6w;AZZY=KT55%y(E$T@1Q~&*IjiG|47TaYPxh z>6U0xX%(Xp5m6m8HBNAO^F`*?uls^$pN?`fy<0hM&r zTJM?kQr#%%YnzRnO*-jgl*Xu^dd@|qzh>53AERgQyK>&^Q>bgCs`GI#PozELjzGBO zJF_==5_Rpo#%+wOqDwr_(zf&07z=$p)cIB1d3yL%x_oac=fNnTbiOB}8UC9&wQ>WR z8rjMjRV6V#32w+_T?F#27IBa*qFqf5TkgC{G#BTY$k_Bg;pRo}LwQ}2^h4%Vp@#nr z?x$NDk}2FwLlTdmfDaj{k{TgRliSRLMf*^lRx)A?X3~ioE12!qGSE|nEN1-XN64?E zi*s2YK&@w5&{qrkxx2gi1zIBAxarJ04E@waH#?j}xhu42fWddZ+5D}Dtv$e;EJL(L z-G{3kh(Oml#`$seTrN6k8(p{S9bI3MMZYftlszSh+PV95_C!16)R}3qZ)V*KOu|Jay1J&KEV??t(nF7XJ&F%9VtQ|=U}ePr+_OT zr{wH&rHHP&ri9My{mN7n+@Q05TojzWH7NKxMcetxXHhSFs)y`s{xJ?K4l&2q7NWgu z2lu)2AX>}+L+38B6xCZMGd}PU5$hR8$9)dsT<&KRiMkfFu=*-8>~f`xcLq_V$E7s8 z;~MggyCv*NvOvxE7mLQaPUMky0F|VMP>=UHoa@+W;nVI$x;^|YcT%^Bo{^G9M=P#! zw$E;J4%&=x!_65q=~V>ke4C54u^_#=;5asr>Liq*P5D82H2 zDKoay1vM$^P%-J>XwjJ}8lYQ<%6dy_K|>*uEGGT=^v!7rDW0!p^(7 z^taD3y0`xs)%T{TM!}vY4mHqTwU0C~`65Cp$A!&(67-zqb#Cyx61w%@No1ff&bjUM zd1O{_3VkYJk;^_iWK(e;)!II!Qoft%l~s8tJ$e#SLVbd9?LKts%GX@$iz4LaZ-c7c z+^K=Hlyk|CTtr29YpFsr*X^CojD6=&o#HxFlyaSB7RsSW(<{tqMG(Ccu7zZ8KSY6U zN4THo_R-+^7&V?Xbe0OTpdq|0dQ5+@&}zC!Yjkx1@=#KvI&C3>n@>{d#-VvMV#Y9I z=e-0~jMg#!J)Mkg{1$rhUOQu=(86@po?|TU&!$U+<(y(i0P|!{C6f5=kF@vXBI#|X zX@;IM`aWDF(A;Ya?V=~rg~=JTM#4srdhMd{{+ECBV8RTVY2SpL_??{Rb4ayHv^m%4 zOe7<>hl^L*&qRoHc&W8EB5ZCk{bd+xZf{{>A6`deEm3v zsj5n6UWIYYV&8u5=z78|9muES%kxQY`wphuU7MCasz9$)=b^PH?lGq_uAyIxHR+&e z$DCcbkpAhg;ZFD^(&&4s^uO*HD38H2AR|CcQzr}xo7F4o91YK z^>JjrSc2*c5724lw>b0K`CPv@7X1gw$TLyh*|&8M6Zk{VnF;OUeu-p0XS&yM=fteI z*%^oew-V}UT|kLDU^u1sZWkwf?wkzPcG zYIUFGj^~^g&DQU^StmB3F8SN2-v1p+K9UVbUX|0(Kl)r@Mj7{Xl{%B2aU4me)ga4^ zT29Yn5!I`2;#f&}YB0T(IkN5!H{h0XynfuZ6eH+n-zawahtSS9^_W?T8IHg+VQUMwed&fxbFhZ`%0@Smo8JVR-(`t=w z;p|Bh(7T;;xgzOU0sC-^E>Ibau>S#wopRdOFq7@EEB zf@oJl7Ns6pioP7bLLVKtEchLCkcMWLaI&}mLspi%(D^M2NN!0UdK9KtU8FKw@ZjVD zl>TfQ9TK>SSbIaXON1)~+=xIg`G=|9;|wk;#GKAd*g&P0?MA5{W%O2I9|Gs=k^SZU zXvzCyOrz{G<{#3iKCFI$V;$wV+oR2>VVX0FPqO{Bn32(2N`KO5+VHdiIhPxApBludhRYf*PCJ{f{J9@>f;1Ew zdys2Od&IrzTtwB{9jW1&W8CAyU&x`)h#pw{ksfV}K{?unD75xFwOF-?TAj~hnhw9_ zUM1E8n+N1@rwh6kI-fCAStES#N`&>LKcG^lebGbnIp}v?CN(WEL4U^Y zVa{GiL)mnY)>yislgj(KyyKztvu2@FiSgPvuqLi_&2G9_k> z+&#~KOrHC8)Whm9*W<2G{!lL(J{y5%^990X5YUwdGid1=Te_*em>Iuw39S&nE8-5e zpzw=l&>q-GQK*>n=L`+N)k6BfO%H9eHm9a$Cz&Njx==y~%> zgxs%Fag!3%{>lnXzY&XcZWhv6Duvw8@k+X&zmj><{S)Obn}lBCtMpcAHhnRDJO!QA z^xJzs>O63OSWGr{zS1t58_)TX#h>reTeqjtwrFwWsLs)l#zJJDW(=F|*9%Yf=%U)x zSgzX1p9viwj}Gdc;-uzzGsa&_=&k!ToVk1gH?w309jQ1Dl#l4347+WNo2?PLt5HTj ziu#sym$~z}<`C*|;u;E0eTA$Od$<*b2biX?Om2YOr>i?>3AA@DM4jdPP<5&q%9|5N z&-#3Y{~f>0EgLGNnmI*G_XZ# zl0naXEaAS9z0^yumlGBabH@@P8a=firMyq$R@Gg4$!$ZD$Bwr5;MB z%k~RXE50G6C4Nl${S>-W<}xF`?gd@(`XAjBT|f&&{G8_gJY=MDh}m><8)vItz)Z9| zPj4+updqJcAPbY0hq=dkI&o@km(K^PFkE z>q-yr+s?>YD^krj7s=SgXIxUJKX z(+#h2R)6ma9;!Z|YwaE~%Wf*sM6JDO$F?U_N->v_{BsM<$xG*J_*bFuElSk1bRYWf z=L9;1zmhhUyc0&iCSgcDL+g*6Lwao`oZ%or-6dW$be9d9Z+98_{nVpu+EjaI8hV*%SFMuR zO7GSM(KicE)8L#2I_v6N6!X!*`LC)Ka$_FTwVk_A-=7zZx~doIbuSilM&yu(tLtf$ zf*CEcEvFyqjOZshFK&KLG+Ojy51sn0hS_LQB3$9e(TQV;bh+hWdS9SXtvHCtKKUc; ziqltt$>nGGh;}W@>kuVXI`08N6DPnGTIh1x{9AP2zJ$3R$>1nCLjL8oA8ED>@TFO&WU zu5`=8JzN#L$lQYT&YlkCw1z-|*&*OAvmZ+g?FN4Ae^~wbTYO(31$3mJ0aN=V*ybh0 z*u?(V&DMDm(DrX2RO&Ltch4A+=Vne2XSl&Jy+2@)<|HlV@gH1~&!mbBrM1I;= zSo_wwKqQJIPUfRnk^dCPyBp4+m-^WhH!&FKBLV)t(0Wg&WtokyA3RWQO7^-k+nsaLFe(($S#{FTU*oTSXqk&ix;N zubLfv-jTys8Og?G2eyFl&9t(S-vKJuF9$XgrV_OeiE!JvRX|1Gi6|$@K*Kk3a9fWU z`AJ)_mYyF}EnEw|qAJ1O<-WwHSqI9z4~5c?R^aQAeDZgj8w?*_MnYmU@o=^-Z2nk- zzsGKbq2{7&46ltOTG|Fa*VZ958)LzyY#Z=?ZVO&D_z29q;|c57uk6ZR9+8sJBu(nU zVBteqm~_&N^vCWc<>`vV$YvdMjCUi)XZgb?b?^B-I*z2uK=gmf1Yp(k{j99$S#lec z!G)3M{J0yRfWo=8pxZ}`T(e)v_79;@$s=! z{LnusAc$x%OIItL6hg8fd^iX=!lnyDYnBxScP4VjY~~*9d%8 zgphiF8|bG$j=cK8CjoELz!66?XknlTat7`3r0+SriV5;$9@ZyTk<-YdWImDVbAqQd z=HQKwhS-O*)2cM{jk=?H!^9IJo!A`R){6no>*eW@e zr$lr4n~eqV@1PD*S^X7Dgw7)k7t*1@dLwvrM;^H3ZBD+)ibI)g!@#uYA-KwEz_4B8 ziPak?GAKTPX~KDMI8>Z$3z~;zY8R7L%H#2eKZjUaE=`*3JNVJ#KLTH`4Y=XxY&?2T zj1-EQy@vo2Q%&@vVn09xNmGO30=U$0dW}5d$J4fFSf!9+Wp{~^c=kXKo$SK zMi#Ky84B>0NhBn0jGc8tmG}-QlBD~K;V~2sH);pKcSrXS_0>FhX@?z~R4)mNf9a6q z!4!N&wI5qOn?$lb_rX~mu_V%=i9NNf1dPAg1Rmd1!!|`1@rOfS@VCw8aNb-WO#c)B z*~L$*_WPNVL&tyf8)YE){T|{1MRyWd?+Wef_rW(W04!}zBZ_bQ;2*Oe;LxxJyduFP zdyV#kHH$j2m{Jk`+?k8N?o)=15;w5fELU>dtr7&~Y{MG6l*yBG1$g+bKXm#%hWpos zz!~S1p@H5Kvg+DQ_J+09QT2h3P@uF)bW+u6nrQUt9YD3bJ5??9A<7H(@f3<^b= zV>#!2@X&cnXf3r4ehuFPq6R0BvbE1Za-J`L%0UVyioT_w{TaT0M;%wLRV4PtmM}B; zGLSyhj>E}XaHU=WFZr(tkPd0Qs^6Vh?^^~phnC=``-6CaSvb)@GZ&sQufbtQ8bN|F z0}mGfGWqTSaQTucbdsD6C;adyD{zjl-WXw9BNly=mz|2#LtyR%o4 zIU3reCUh~I6*3vBu2m-i<%{soNzbrjlO)u(yTo@l8RXsewS$|Y2`@EQhFHIgz{UIW z*^#5V@V2P$D2UK)l^kj8dc~a#e9(q-U-W?8hDG?9i3Lkj`hopJ7ijKd3D?O+kQjp@ zAQyE4?;F2^*YIBm7889n(P`JYzWrRrD%kTtB(Y`9lj_+YWmmy$kFTt9Xc@4Yqz*3+ zxRTj#Z}S=#Yz8yK?8#aWLom+L4W9O$hh0QDHU(-f@K5U?Hu2RYqi1-~OZp{t^l}IL zZrBmk)=<*$F%%xwyAHTbK`=&j7@yErCShj!M1*T%yLty$drO`Cow6Tn*AR-nubueI z^$f7!ju_E3$-==Yfl$T3lo+2p0xAsh!DZRk*x}h`FeDl36khNj4mh|Qhnal?Hoa=> z^_)`neQXe{7rw>ujYse@k&VPcE(iR$(p1@hOdp0D#qzUxX1LMlI3NoGiRIQ}d}4MO ztgDhIex{yS-g^&rc6tj;Jsfe8rzVm2G{hzcw_{JQV9N-8%;#!&``$p4W8=v37ARGh1&yGlEk9T zaPRhHJfk*%6srfpk|R^_+SiX*lbvetuh$+Ns zG$E>?O(6NDBVV~!6;#{}CN`5E@=Nb~u+PI)fc_*`Sl?qw9!7`o9uA!b*~g6eDt+&; z&eL%)Q~4FR9wqEIn~Oi& zec|sYYr$bFkKxe6XMkJzU7VJ-09xE@#t~zDQtQiu^5-Gm5}xN|A!`NaFCM`Q($BDO zWv_r4kG@pRyI6%^PCEtq*Ux0z472cBRUWu=mIsOEJpO#!U%30^UeI`97P$$^o2 zduUvF87xduB>4|Mv3`gB$;}jV5->CmUTE$x(GyNNz*+qD&!vgdI33LP>}JP5mP(t)RA`tfgWHfbGw&fdJ_ z2$QP|aO@i^$ZMJb7wmP$P8YO@%Z4x5bkYI-vfwh{8~h&2Yt4kR{Tl2ymHDJQ{~LJl zwHdFA?`Bm_4mib)_X8LGYe7x=A8_^bGxkfKCVzdvp~|pRdAK1d612p$f%39eyw^{Q z_-)jOEvLHxlVeZfCz+DRSM5OfEEX6}@`Y)At!%dHYh3m?2rk%o5}ce;13u0iWV4U$ zz>OijnEG{ql3UhrPSRZR$Eq3kuM(a6I5DVtDv_O<`U%wB+yl~*Pve>~W&Eo$7($P7 z5dEVcfBzW+(%a6mvo5-U1Kt^6?z|?vCfE^cEvw-#-7J6_4tw!9vo-L=LnD}7F_TPr z)yTFdMnlJg2f^g$Gf8m00W=;kV13=2!LqCqyl=zf$lI!M?B}PtWZV=T^2Ia)dMxoK z70b+s-VYi0?MeYS{w4{&?YD*{0YC7W>J+@SxdZs9@4|1^YLdV5wlIG0^(uJX6&hwG z^Tkg^5)%y_IBH}K5A$x~uh@t@^q&MXVj@YqQzd^bFBA?sxEOaZulKs@oNtBdvp(EE9}9W{ljpCsyd8GtL1DVq zBjA=RAHFYs02o7iqLT%o)vO$_vnH98SbPS`SDUcDbR9o9_a|GPQjaa@67VKb2$txU z@n7WbW#z#nQbd(Wc5ybySh<=Xt5k?*ZL%bP?_R|MS65gq%4o@q>cy{=Zeip6Jdk+! z9X|a;S9Gqofn(K4?9i$XZ#O@|?)MXU+AgZlvh4y}eJ=^W5}|F6cD8_`Bq1KxmBPzv z*?=!TZ3S61zIb}pC44$-HaWN<3bvY>!D&N{z(Q{v>GYI`ovYLFjIdv-6vfs%(|#M#CbPKdXJ&RN!QT~IOhyK@@I&j}>@2?h<~lf?Xag?H*w5az`^Q&S7Tw{~yx>@qMpeSQIluzdv%H!9STp)Es~aUv6gO02XBS2S4oo?{tJWqe9Ym?KtnR;r3$<>%>Z7roCdAF zs>3yxvqV|32p*X!4p(eeBDYtG!=;sTpth76#IJ_&dzHr^$kqhB3e*FvV;|mj8AADt znM7H6E?M^c`wfeTA@J#8yeN;-3;5iePuvEyU~1(!d_Oo0|Jm{rym&7l6MdHAyjD|k zcbPR%B3NR_9c*Y~IqLtu|;BPoMpcFS{WaIwU2;d&$jbE-~_~wq=c%kX5;PVV0 zA~W9y`ijqjSqWS5+@o{B2~`2~N}UZa zG;%OWg!4$4z@lS2MH8m>%jUd z)rI*8dMLiiUrvfgwvf|4Ht>vu1s2a) z3>D<=fM}JgJQtJMaOwKR&{Ihdw!MsD)%MPTK^;7p{;>yKPP>ik?7F}$?^z-<%UihR z%U8Z>%^&<-b{QGwPbE9bBl*s1lZXnpmo(0DBI-Z2;nb-C#PrBEV4k$EDldlxiXCn2 zs@6HYm6;j1xMXTo8n}ryJ@YF^r*6P6rcDDUIz;d8dn5Ag;CFoOtqy-WZ#7$cBAc(? z(}3rfjDrWJs)HRX2Jwzo9U@!&0R$Op;5{aF*nL(A@T+~z&i*J)N|PMn(SCo>y{o{f z6&UX5(N{F5mVn4+zscL4*lrpEit%p6NT# zmvV%A{!3<4?r6hBWy-`z@h!Hpx{03@y=K29_`%K{vhc#leq0*S1=fx#kw5L<@McvN zeqzv8Y;39k8}~_(ljgQy;z|bok=Oy2Seyi!>wIy{vdMV-{a85jvk^49?~W~I%R?o{ zCs^ZyfUJp(18pDnfbi`r$t>{^9Os>k&9+I9452?6oH(6qTBZcDo;BgYs)@w-^a65U zLIsX3iUhIUqgeISWHPS22+Vl?56kWzz@J8?$!OPFJhWvUc)^u}7x{(P106*sVDY&) zMZ6I!uRbN}qZj$QTOU_trfeYKpDp=h_8450xrEz}?u6=v+Hl&=m+Xp6eaKA@Cz4vu zWc5KgT>2v(7p#mUp}B#?`+_4-oEb!pb+05|7IwtmNg8P0%g6h5)rq0%Bkaj4^A?h; zZ1VXHWM$3(E^SrCs!Jym(|l9WoiPlUr2POVXDP$XK|NAAYYUdVIS$-O7{QwHE!bvq z5;(9m1rIOK!Mo0sfJw3+@h7oZaCt%rUWn3xvyKXG>8WPb);$HFd^y?l(uyoyv<4iu zFeOnk3*gX7RXAs(AK0%n04j0V**}{FWGo)A->zW%PHi_? zC+AK^?zIEwXlJtNizUBO!WHIN9mHpKTEHxmE-)C&gED{3iFvvWpzh(sXm1E{5@iZc zeX$#-{I!7gbG_iSO$TwgRxQ7{;We0hPspac$OnRWYtcMi%L~nl0WNv<`~!2ZfRcDS zGNJo9`tc0sX<{l+deM%&~aECD-T%7W^QPq=!@ zJ$&`OINY4!3@yAKgQwvy@$NKJ5FLM4R98j7@a%E0DP9qHl%K?Y8ERnte}!0J^red1 z;!4n1CBE?W3%E4yHa>gHlqCEAWA|Zc{+Vyrv1UjWeszKapGEVz#u*^~W(rJxoB=+s(;_cA z@wu2k@ z=HN}ke|fuiHdc~vEDjSt1B$$hvFC?E?BIP1v+`1K<-Jziz?#B~#>t@YuR2Nnz`z&x z_h9|+e{iaB5;UA|fV~v_a9Y7#wttE$+@taVY^B$LFDs5MUq*ptae&nH29S3dPqA*< zE8un_n9R3PCsDhGLm!kAu}7hWOJ+AD&nd zgC`n0;|1S~aG{Afbjmx38!g8$$@u{qlhVNSEFaw1Sq?m^=dyQ>NfWsHCfk3(ofouq zE*^0m;*W^3&#hO6-uSmg0~`vSh^Lqb;S=*N^M#ceAX(CqwF~uym01&j;`{|<^wch> zcEJ#Cw{ph^c5VcM;E!O<4|!r3(uFs)ec>G){fuopU*p!D`M`Ky3+UCjil;?tlL%f7 z9x43}qBicwUvJ&RDMQQgxl0v1wV^In&HW@M9`?AJD*}~&&#*bJE#T$bBru#UkLP5% zRApUh0q>==z`be4(5tlr&)TO-X0B+$F+*o@xJEPY#JxrE)=ph$JK2xC6&wQVip`14 z&PW3HH-Iz`eZbl{k*@WViRH!zpv>kq52{MQ+Q$;m#q9$Ik22Y}QD685cf>$1ssiVK zp2kw$deB1F1C&+u-wb>i2ewJ=1Fzrm;n)KSe&QE55>v1V{%5NWhu+U3V>=7M-@tp= zv!)892z0^p7$tJ>uNmGs`W8$7mx>n;kAs1c0YLU)FG%ip0>Cs1H#AQLLDmiUs;L1< zsnUT*1p;74^x&?EM?jI&WTJUQ7PrnItn3mkxG~ldmtC0eG-!VkUsz&@#f`ne!u!8~ zZl5tc@mZ4``Zxrpm+v9T6aL`je?HLYku-72s^=&C7Xo_yHv!Dc#t$o2LEG2zz@7=tQ#BVLbB>~Gtbv=j3zMRP~I#>gS6a8_Bq61NCSVdZkKH?8IOo2giF8lDZ zE4f@yh54Bi;TxUxIKZa}TrWFW$w(yfO9PxiXUGCjQ_UOU4JU_Cv=oNmi zAqh9OT)?3ni%6{GE37}+4%P(J;NA1>VaH4ZIQypqdGYWRJ32oZe9fE=J?`eSdI6(& z-L?x@zxOq}GjB3Xm)MVEdI0wIw>iY zgm9b@xKp4?wu$h^<{LKf>G?IVE;kG}kBcD5vP)q?(PGkZ<2d%L(}zA8Uqojb25pks zfziSsBJW)dZp=#ohffG`^risv`m+nn*ZhY`tw@R&JH%h<-^E7c=tDMpGe`{=BaTNk zpp<}D#RUE3X~ZYsHI|h?e2p`ytK3s{+A9_;Zc`^oQ=`~zP2ceDO|krFZ7p~v-3@z+ zD}ac-`tX(KD4r|Ir(Y~ToBym^l=ywy432*>k)LViMoxz-kl*&v@cu`6Ocxd6fWl>L zXX;O&CBiAM-RxoiC54fg$+Mx$?|0y5=QHqI#A^k8n8O|#9)}~Z%D}Dqk}%9|I*DrL z!R2uRm~Qy4%4niIiBpyUSJznKk8L{OwXZhLHwl7Wafu*$r~@nQY664S62$!T9H3LB z4WDQ`kUpO%TxoO`Uq0CkuIxL@*Vw#`T#ksrl3sG~d%QAnSC4_4(#zP3n^JI6s~D6s z^nzr^d9WNxlbrb*!9;#IzWQW3o@De3UpSr(_M7TfJ)bSZtGIE7?bo`<4@ik;A3L?+ zt2tU^_6b8csMrStqIy)>o`G$qT)^L+q{7<+0@!BdOyo_HfY()yKUg6LZD&0LL6;oJ z7M%#9`Yjw=W%41{Y)x?7d>B!$3k^m74QUY%fzfh#*cu9f?fDVRFP4By<)@Qfec>!w z;SUp!nvg9j#^ltRN7zNvndiM?0A$8aCi+Lyfhdm*me2Mhhc&jq3G2`B*Sk$5s~V@m z>3>t<@XkuSHP4Y;e76dIy)+)K;co{vNQ+26mqH#5EV4Y(fFu^~hLt0>B;|!aTs+4goH#atd=S@yy@%F8 z*#sAGAx43umFmLRO9J7;1%JS@(!IF)=^|D+Y!jU5zMCIyp+*Ka`NQ{5R)hZ*+k((n zbBNIt2?z^&aMvbdqUCS^ICSYi_U(GIRdp-ge8>d8z3xC(*+s&q4|wq4lX+w~^&yD+ zun>j?4dah%orsGOf*&Q@!GhxtDm`zW!V4C?0+I5jBq+lb_rBGGRed(3pK-{75yY!-R^<`k}V8RJ)!4S{WCqTJrC93KRW?zJ-F(D2hmym(PAzhz|sPKcg>tG?d> z*&b{0{j=kN{lr%6Yvjlq%}FA%B0a~rOJ?xvWH&~Ooto;(IEJGgpznvL4@mLrF4bXM`-$wa zs%PNeD>vYLn1>(smw`{w>7c$&nlJDF{N}THJMhK(?qt&a?cknwGtl!Gz^zF;L7bZa zhW?IVH?=>(9y)jMxfOYQGF1}Zx4VZwr|yOeTHfF=!4h z2mcmtB2j-|vsYhPLy2fra^vA7u%~*A-Bz&`KiGJkch7GtSt{ZffsYuxdS@wqST)9f zrRhv2EZW2F{AUSw)TWV#qu22J)eB(rN`@G!r9%`R1f}LD5ccjh(6eO^_C)zueCh}o z?oY?YDa-IoUlri?Jdf41F(jW{KjMz2A3$)c4tP(pAr@CN*)e86+Za%QC6w}j;Z+2) z2lZg$g9o_VSQQUU(8ktg6G5K363C1!#nPL!$b<=Fc-5!^$km?=U);Eb6~;$_J&FBb zOtB4{mClB%745;(p7Z#NX(r%bKMk%{F96=xC9 zhuc#pl8)pFc%hUQG+(xcue4DO9xh79I==_-5Pd=pl*t6+l@_1Ux(Cu~qB1{E{TzZDdgP>%4^-Kd z{LqhjFu-D%-Fe51F!|1Kox~VO+Ytu;X=Gq{2odd7pGBFmC$QIoJ7C)HulNQvg9DL9 zP$RVk3$rcw2Yy76AHe}c*K8x~mfp-WI{TYF&Ata4(K*}zYVk*nb3o;&h~pbFAjyW! zRT?ISF!Ff~|K&QJ=*Ua4v}ov61?g~vZ-f}#ECyc878YQ=ow%Q#&JHzo-2Ab$W?R$k{{cm54V z_vJX4ie1Nvm11yKZv$|Qz0MDRd>M}p*uYr|H6U=CC49B^E6!_t!S*ORU@d7ELT}&a zZ@$IAR$@=a7pQ^g>>>Wf*)qgu!EP8Ym&fQ}q6~Z6sqoD20J5T} zlf85f!pYou*8PZ9X25CiWq$I-30ba_+kUnfnR=eC%>lq5q0~MU`@R*$h%RDEt6%O-qeYM2e&ow zf#(%~zx02K&O4s!?~mgYvdX5AQT9wB_niAVR7yfJDrBWHN|e%4wv@=IWK>GDG$nk_ zxt}9_k)$-W2W?75Mbq!|`|m#PANTRN_whOBzQ^y5P)QY@+ zeGoXSHG&Ja|J5s5#t2el2-uQ`al4c%-22-RR@fR4=|vX=+fJ_lvLCWQp4$v8GB}Cn znm)lpTHgg7c^2gTqj2`}6gSdbFp5_=X~X`SBluVQCb(nAYw*x+3dwr(7UbSiCGuz9 zv5zNe!S)-4U~Jt4sHCnA)+>mS@VvEnke%AQHa`Rx&;FH?Xm`whUGqGHgxDjjUx zti`f7pW@xq`~>%Bi?H9F{dnds6S#8=KZlXz^9aTlI2iWKCSL*^Nn1(;%rwuyivt_6 zghm@KZqNtgrsd+5H44PGvJ6~}kswE$$CH@RXt*M25o`-@p6- zYQ;zK*IWMZvP&s&E+4>ghX%l#^}U#v24k7%iSSZ`4qhu>!_MaK0cksitZ-%*qbVL@7pH8>^1|Pj^bGT(iAwjA|E?z8Ns=+>0tGp z9KreFY!Ima3TWPO;A@hff_n`gLBIPOkn?d6^fb)?OADg0N9JXmJKq#%Ti1bdy&nAj zx*2S?X%%?uO&}7SI~*0t5l&)y(0Bd^5P$2@vPC?}5P_2N*T(2$0<6 zj}sJfKnG_{6#W;GAjTOQ1TSK@N9$nU9Xjmo#u>ow>SFwCGysD!Ch&^PbZjs=m@Jg+ z2EFdj!3&p3SgpbuJE^IV%N5IT;oNBGrRPMPtVRXX<+AZIv#<4T5nsTc&56Kx^Le~m z!wxjBdJ3|Hwm|mY1CUdY2=;EM#C=zSVZ{bXxcQDE(X3L3$y1`q>R|_>sbvVq1)1Z< zmy5~PeGIX_Fc%Je?ge^m0{Hb-0?!by0Erm~uzF_^*pVC#cK@^n_r|AS%J(h14WmfD zv^y-?Yyq!XX_H$aRrq2{X#I<|f7s0Pb`CM`_;}PN2wOWYIh-2ecd>?R!Sy@t@UOTx zY4KS=+)rfIhZiY=zXLb%U(YOfqTB_#*6Ne2w+O@?wIlGsR`8IVaxnS0j%*Ey1}6MI z(n3=ofNPgnkZMKT(nK3&w%yV4lr8KEl~P&3f~HO z1Pm4~!JE_n;Af$-piEN?cX>yWmctwI8P!kV&**eAJV_tiuXxQy)V%?fPA{?FB!2F7 zTbaDMa1|WbEeAwd|Nc7Wg452=!%69HJNsZ`*%ev;s6!On)cWRk9pK&*$bCh}zGbro`G z=mxO(j|Ke|>u{s36Ml1Q59qu-2jupt!-Fe!f{mFm_~D{C;Cku=csE5J&QIq1saDJm z{q-UK=~>`L(MA}{@15Ts<9F5OMntjy2x!{f3EB=<;Y|`ou=v42(6Yy$I0u@+1yz%w zgk~{Fa@8kO{@lkJ2Rj6ZZCgR*zjYj&GfFP>#x1=`)_zy!vL z3?7&c9D+}S)c4lxjB_D4oab<4Tzds~2gJz9&I*tjq(jD*rm~^P6W>Hc&}-ia)`^zD z7yC+p%kufeiJyUV#*_)#ecJ@HTmA4l2WvPuVhSp{BuMAj0q{*f9=>ifAQiWrNYPPm zIJwCH3bvlck2Y7;H&m!ViGPJ)u*?lyGxY?DRzKP1A0{v}ZVPy<8is3g|JHZim`Sqv zyTh{64Qk})VmGHu(qz3HM5b84UH;aDbyO!GO+ zzF^(%42RcQVz4-K4;*_fR!~r}f@}`AhgCn5VJ34GB%T?6Bf*Hjx=qw7-S$EiTFXtp5^Xk7!1$qusdK4KG6d0|FG7@@k4@g%SBxI8f& zm%Ooq>tCkg`%5KBS43C+p4T0~JNzveKTU;{>OICwd*Wb(jSLaJ6@c4y$M9w0E*w(2 z3hpR!Bt~B!;XU1MaBlh(V!gR~{)rfJ#@+d>{cy{e) z$1+%3q)N_?t|X2!elTumK2f>Evq@q+ThpUKBD1`p?r~!hYHUGJnim}YmIfb5c#u;` zG33{kG^lp*Gzj@qByh>kz<2+a1IOr#f(i=>!T9;l!Nb;3)@&aXEStVZux+<5>*gNF z?m6Z_#P3VNt4J4~k}@YNckpNA&u@_K&VoV}MG`djJ{!jk3QkX{1a-wZ7!0g{Gi|?v zWvgeCwSM}7B_3yR>JY{Tl^1|<^H#w~*j9X#XI`5%IUp8NE{G1lfU7^9$M2s=;KL4C z;8uzl6xg@mS(b}Q_BmDfS$76Zkj!RZ^0S?`C+5(k$eWxkTMji7r@)B68AK^(FCJ?? z1J`&>0_Ssf;i1XW*z=SxE>XNFh~F^-NQOj{$!@`LXM!KBqanoj^AI>U@L5pC*Cln+ z<3QZsfVvF<-GWy>xmft?V7=~^AGma}3JlAX6v$T@kW5EyqC6-`+CpXs9va*hyjZ;) zMp|2wxPMN>!OjN`)gf3hcTn*7%>ceUXaic%RUpz@1O7=>;4a}NU>_+)zF*XVPxkS= z@GKqDx8e`}@17i7mF`~eh4LyQ)e((NdDp*XM>>Wt~YCbJIUD_$$=Q$B=(*_(_= zgtR$fJ3fO|SLS0=uRxf4z68{YS(2xHNpR`y<9MS<7#UWa13#UzA>9ozIKnm-Hjs}v ze>j%x5QM{BWhXL|EI&7$rSYUtf*hv?0`?4izXMV{h(9iGJeJuOzir8;KHnvf;mp(VWjC< z>~C0xfBR;GuEW~|^2vF)m&lS|o2mqfN*`If^V{*8*U3Qt*h^sT^$RcF=Ztq~tjEj! zWx;df8sJ{I2#+={AP1x`g5BA!@O_*CNmu?0`p0D8z#dIpp zP9SvT&)yt<4pqI`44(VGfLw?-CQCz4gLw;mA*!5B&ZR7cfm8ouPn``WHRD}jdfg)O z`PM7|ez?G2y?SK!6+O~j>qxFUdcvEk2LooE4ITw03yaf$wT}bfKVSFH&AsP24l2qT@_2b$NpktK= zJTwq&TXl1Q7({aSz(6`Jrj_!yYF=fKRv&p>(qMpE$3Lje0c0122!iq%x$ zm$O1RZ_Eo&(0LD7y|2fz^Csg58rryKRv~~JEZD=lgRnrQ8f5blM)%P*SoxV6xKnNf zRrO=>ar6iOisH3G`!rzur7586Z)UyfXD7VUd0PFZ^|JWm{%LsA01JliH-ZWO)(bN8 zh6I}3<8k6UG4T7$a?rGjXDq{$+3NajuzT?thk4ojE+C{ID7^B4&iZzOdJR3|$Fm@) z?KgI+`3gX(6AslKi(Ajw!|lOQcc$kxg4Lp7exzl0~MQ*#sK|=Xy`X zPn>4MVb2LfEGXm=;C-7%}?uDI-fppqMfmw7ap0+_1yJg=I>7D|1j8{?rHIPbWb&D>rFC=kNra7nnDaiDV~*;< z*OzU8GLa#7+-`y=Q_I1{UL%;s*Vg{+`dB923p-xw2a;hnB(En)jPr;OJlH}*PS2*uk4yX-SA$T~Y2xsSvf?*d4 z(sijC>#Prfo6RM#QdtdPOd?^RgB|&Bc?|x!T%9DInGXHho5AguIUr~LP5e(vA6t13 zfuv{?k~B{j@9eS@=qv2IlGdDu4acVdc@P3#HRZ9P;q%!I^W?#(RUs(K$q-O70sJfO z2iZZ|xTnz`e_5A1)271^K9rRwv&O2EH7nDhsA@j^(7Kuw@H6{$H_gcEQ@sv7v8&;W zjjF_PO9Qx8FAKBhX_AuTPFOzkA69rcg;cC`gtm(asN-ksr8VQ=h{{4TV}>QPcsc>j zO3;L3bo@zO^bep_8c5odxq`tTvQyY; z1N~t8t8B&N65XNs&k1+R`3n4*YhrzOh%ouL6+h> zwqY=YbiLFfgQ{l$m=lWiEw_TcFOLP{JM;ww3tGX|cOyWry%OM(L-^H<^A4NE82I1f zOt9*22W!!2529x!La4YFJ{I3Vy5(h{rt3ynqUr-5|HpGOE6adQivsZ(HO402%0Y~m zBDtR6OJ1BdBWo`jz>g()xMpu5@oDiUTko%fyT_`M=OQQA_31qD;P+C+P49tiW-Y!{ zWk|lX^7_7RWdZ%JLK0#jdGUN1{5R(ZzMPA&*~no$XP53wgO2X{DR&IvPOU+pS~-cx zNL>Y67TWN?Lk;L04kh;6c`7_;+sJ@AVXA&Y{ z!FOAzotP%**=XaWZlNq1aJ2u$M`-@ z+h30~JDZb;s?8+W_9j~~YZ)}6DzGzT6T3h5Fpj%11L|zu1}8@ZLyv=7$n#)vIJU+O z#`^vd7~c4aDTxLQHHM*k@f< zK)p?s>>f!biQTefp+_1pjEW_F$7G12ohrGy>!g z@FCp^T5yrFDjqM2+FI;O0(V9rYKuTr+@Y z&PcO?hbEFI2YMWSH5tMiA2vbBlM!%rtT}NzI3BM2rU&%1pJDf@RbV3+1Lp$%{r1(Q zXmtvVGBYAguEyl-8W;Q^=Ca_zYz?yRawgm!=nl7nI3l&v2TJ@Mz=CKC5_#%4xO?9U zC%K2hF^2+pT}2nZBxM7}pEoA=k{a=a7x!3O2~Bb@H~%(`9Ho=jYGN6CA3{O|}f*aFp>I0WP10lOp@pCpDzxU9B=?8S+ zg_;vMeI9Y$rUo0M_TW+5Y4E9$6x^FXi&V@qgVE~kxOCp!nHtM33XWvi;oSIeC@@(7 z!)KqtHwHpM_zQkk9ui7S9=U_Ty`}6n<21IgH^kxXZ#^>i$u#&tFB*2aEQ5mvUSxWJ z6@1L^w@xd?7?+UVMlM~5bu?dJ91>^T?#*wrn zPvE|PGssrv*;Q?V2YtV>oo`g(>IEV^RwxC$>v9B7hOgsstCNAob3cKs#Rsha;2v1- za|GZ2@E1QHH77cr{{)-m)4&t6elT^^2qs?~#{V|Sz(dz|g8)xGeB#k0wj%WiSktlt zoKCcdQlI%}=~u-}r^pGUlo^B4HX*C#kW{|wNWXU6W0h=UdLC&Ph) zAoBH*1q?b4NYn{&at~U;)H6SEtNmJXuGEesr8q&GuP(&ugaBSSt_}q!BXEA-e^`>& zbY~L<_@a6yDZVQXXZr|baEj+NHZ5j8yH~3ZNR$oXF2xS;2dR;(CC((E_%67p)DA>$<4Ko8Hont-M{vx1 z9a|z71XC_P63lVk$reBD2Aj<=cq?W1d3~Re;KuBFXZF z`K%jv4&0U4k3XBQhx=@Ez%g}IqOGR`w@!`(g|+5D=a3?N0FSZXgjr;3tt&Z{`x87k zB?VRaS=dvxW8m!iwOG}?4Tshig54gHWa64(u;r8+a7~KG4G{-%#q$$jmHuZoA~+NT z*gOTDvZMIR%%=KTTE^t*%B8?`P61o(QHZ@JXu;oM!yx(LOj5FS66Ch2lip+-GSt@u zrno<>UoAg?8}k|RMfEZWd(j5kdwtlHv48RIRXb;nta zUgYDJ&-mVr7OemD1h6wzB>7-GmOVcX>U8(u!>g>}<#=tukF(~OWiV)LaU%Cl%9D^D zz~0dM2_}d)V{w$vK3F9WbIuOp&DQ?l!!bQ#*uN4?mFoxlAC0nS@92__>3YCEV+uUi znE>xf^L>ND6u!>k1asmSk~bxG*i>H(N|x)A7ulOhO0XF@^kEK^QrnMJ%@rV<`w6G- zX~iQOr$L!UF%s#*;nUfguxMn^L8*2-G~oA?n^&hPZ@%puP4yo(Uf%35PVDT#(lKJ_*pnFOS{&RJs z;PYB9EH&+bKq<9=-SA0+7~bQV-8eh=tHF*uDvc)0h9jWzzn9?PzeZr$N^o3FFMj2s z1+ONDlZ(!;@c0@8Hiym!$r=Yh-o3wq@6*%BnmeH|Vcrigykr18%do>!KV8OCcSXRb z2~IFv-jdjOKAgGC&IDc^uoG1Kyl1Ds2?jk;a>V3u8t_W8hljpLk&TVyJH5cd#I}8bYYGyABLHKf*sh<+JvIQqV^6KYVkQ9sbby4}8?# zOhzhVaM%eesDH+pEZe)AT(OEGEf;UI~V0;V5PT4CY`I4^W+@@sm_|qg9OUJRFQx=<6h%etKNZ48|&)lN`I`oDpd>S4np!+ zu#&LSu3%)98kA3d2r@&b!ODs~KyK_z^7F(xDBQLhHg;)~@9V!~vv1Rh&%ZR%I0XXP z&EYVi58$P65?owe1PojRB&OX9zBJz_FuWv5wl_?G?@0y;7_lJN+WW!9ih5kR#uhwE z)nwhPr`6qCnvM-b&fwwdSAzGm?gM}K$z)~3Tg>`o3I3b@0BjkaL=G5DheN;ifuxpW zcy-4I&~A1OoO6=^1M}{H>z`yvv!n$nTbaVHbTEgVvusI4N~=JxDi5FcxQc%qb%WW9 zbns_cA9CW&Y0z*}6F%FsAM8;0i`|&5!1`_&xUhdBY`?hxDwXkxu|^kg<8*uYf>nV_ z{43d`yLr7r|0RJ#$QjU{Y=ZBG?7=(pJg}GBFC4GT?^()!v5p{-WL64Euj~uJw2I-H zCQUM2v4||I!hkCd0#|$8iLtC9KKn}!y392tscFObsZ<%*|BiyFh4y5_fFyKW9|+eM zUBOl(UboY601UqI2PfA@!>If*u<%9}c|T<#9C-R2Fk#ZLVRb8Bcz6$( z!E150Jqsk|$z#C={=7s62a#DSGqK^OyI41G5~x$&3xe!M!G0x!`kmcgxuw9-caVVON73`$QvZ#K+ zpmI4lvursiGB}E-@|wM%d8*JO=W4yEFA~pRqyazYCqR+zLg-<15c?dmC*6yF2qa#J zu;McxxHcvjXZ(wVS6lYw%=}X|J*&sOdClMR^Y=!O7VzBU+6(~JJ z*-S2yc&jZUP6OXSa%DW2J81;ml-&exJkcTV6?h)wXek*OO(b82*Fl%ah4#bZJiDZo zPSTE>z@sfYfu@)-kv{VeAN6&I>5GnFdDSrTaNJmuSn*BZ`y&D`Pm{y@cz=p~BTZ7> zW`r+2i-f}UBVhTmYTP(x3qGtHBTzf}yZ+}zCE$Bf7H(>t3Kyu}7l19}v9SLEs6C|# zRrkw-H}@m(=%%Ltn=FHx+jytMd>68nXF^j}ZpGV{D#JwFg-umYV=R`_Y;rCM zj_o#wv-iuBf>jHFVZ1t=RIdXKt>nnR^$!Fi2mT78z1wm0qiL|Ltq}NBQfzkN0QM}` zB2UA2feSWbc$w}4-os-&sZ#3_X!3d=V#R9?N_xQcj|u)lWh$B+AB$bpGYaM`(PtQ^PluB9qucNfB$T@1jo zf54-qp+IY;0o*+J3AZ0JBA*UekR9^%V8hSDK##X}UMFoy%-t=(Oa4rPffg)wP!U%T zHGx1kF}#ESPwlv)3LZ&b5wyZF#Qf757;_~S8cN2J@B>oti>VJdNwC0raRTn^$tI+9+V(}_R)79{VUNz$6jaJA%IC{er+6!Sbygu-*s{op2>Al-?F?B?)Z zGil&U?PEM^P@dcloCJ+8NW)7?2)|AeCtQ*iEQs+ZNo`3ESCv)Z(y@Ug#d|3^_-r4L za5RGwlm24Ah8Onh{G!RF#GSz3zzI$&jwfNK704sCC9wFb8BvX03(ua*!Rpr9WW(Md z_`K8vmQ5}L4W-^V<3u93b1^~iEXIkIESGh-^ZT*D(M=X)G+hO**{;|%{IL1M0AtY}s7C0}1?0GeEJ?o=D+gSvJDaY`kT2qphc@J0by8=`! z+i_HqE6!K)1Dzv}**fnWP{#M4x`_{Q{-aQ`SIGvxN}WS=ZYW~c(;7H$MzFy3pfpL> zS0a|Gg+Q!%zaaZVSAA{|t^MOziZ@T@?~0iX;O@{}!PaSi@%9(CuqLS-I7o_s((5Yr zcYzQ~E!qst;KZN;y!2N#D;@I{|6Dj1yZ$T$?*i)u%O9BFn{$o;lRtVm z#J7fR)1HK18|kyr%r0>JlL*{_{_Jvx5E}QPerlvyH zt6B-PCI=Ab{da)O>9O$bx)sD;pO>o3J-#CLDUV9|%eIA_n69aYbdSM<5AVb%}phKGx_%vimbZzJ3 zW3xIzV#I3lBl|ag`Y;T(IsL+3Yh&TLx)Rp3Y5{3hafSKDfAFWW4A9-+53WcY#>Yj^ z>k}VL#wlkES>@SJ*cWyhc(qRkKG&mz7j}#ae9HU4Fyjk#B-Rs+qyMq<{y9TVc*_BQ zF~;9#%0LU#Nc`!#Hu-Tf9kg^)AY7vhqkJsEv%(B~OiCQSZvTdJcP${}W(EozH+=;? ziaq$AdaYob=2Ujo#Y_0I**qY7r5yNLN`bfXHfbO?=)8KaoMC(Q0bQ)*l0 zgIY?uh1M4196!#{LeW`!(RE)L2DYXmgE3SzrSlECBWx8enWlu|Kb8yAC;K5wC2huS zi6$NOWEqv~$LY9pzTD}cHYP1?3LR_ph?C!IL;L?Ep`&%vsf|G)4LvoBHVy1z_Q^FN zBkxoQd9YH8k?hF!fIhrw*Sl(eQ)HRJZ#p6RUR; zy*;18nVM8{Hc|&ghFeZ?S1zw((sOlC^WqEKRpln`O}`g2^W;$)wVP!wU7yNn+3cpW z|Gh&0B}$FBnWkR3%CsFqw60i>N$^Z%ZdJ@e%Z9!q z%WFh5)l81gS+|~Mp83Z6QOh?ZJZu%2>Zv#AOfYH4KeK_^^{Ii$%{SpjWUaUb_j6Ha z%r&&I8UEJ?* zF{b48FHy0rJI%PVhC&`Wp{1GVZzqd-78%jIqtBSvyJj(;*yBjY#Roc#Q=+=pCNgp- zw2^XdFV`VIMAfb@L(fz*(0#49qLo<>n9D!nQIfSRS`jTzS7+Oy2r@{2#Wrv&jy*!+ zM;{_|@(EjqpnM|W_t=pg7j&ajWj*~M;xR&^WM>Q2Iz=jEB*IPzhPKn3o6TVN4J_8 zG)4U))%;pSgMAvA8q05JzxrA3wcJf^>~J;Hmc~(|$k|lXGLcFS+(42WWzbqrIme9N z*&_S?P29qH?eK;15!C-J10}vr;N~wkq}PLAG4iY_vaNx_S#i&()cRPp0~)a^f=u<_NB<6F`6dGXVF7)ZH2}?zv_FwcS03)-@Q?YT2_i)Xts*^BPx?h!pJkV0dY&ZgS}8tK9IofMt8 z$Z0fuko6nxPy~W-dk~Mc0(3f<{F_Xs$I44l3OZ5XZhSJ8*6*6xWb*v%n72+)o0MkPb$c7 zz8|5VZ&BZ-T6#I-7|r+0<|3oYQJ_0TE3TH&rSBed1+pn9cHdH})%{I4(diyDGIWA- zSf$&bP+Cndd~Esuc#sBo&f@cC)M(s0*#^6JOAsSH#1!<8qs!tyiN2jyqbh4UxvC4Q zD8HqgzQ{Pi`}-f~9=;gjLgHG<`t}LvGQh%@a`QyCCWv z0JwuWDx85>5$#TvMjgT$Cd{J?N%Vx$?@v^iM#hPYqSC_Hh%6@ELW6d#y@6IAs$n8V zAblIu#{`V3(D+~PNyWx@qOhbKy0L5q_wrOPb1l+|tN&<=#)e#^{i5@zXoDEjeZP+O z>~j){GIr5ID>JmU|1r`sj$u?%SWe~of5PL3+=SfC8s^{M{WSNV7j<7!MVG$+!N^S$ zpfO`a%#`L9x^wLzRQufv^_)tlnot)#-zCK<$tpBVs62pJ$9IhNadB#-T`atM?=1aO zYK3HOWTDT!xwK+}F%9U_Vy@*};J!IaBCw+vRn4kKwb_#NZkQM+BPLFJ7HpvU>i21& zsux1%LpUq6mrl*g675X*P6J1#qyD5v$W1MUv8+==i>&2o=KQnh%HKtF_4`OVyJ9T5 z8TJ|7GF(kt%jTn~e*nb{`OsnMmuL;L;h2ZM%;%*>==tk5k#}b(;yk2K<%eBp%Gp&Y zZNvbbh)Y45skgY;RwHEKpGm6)m@LxJQQad8Dp8X8PY{^Amr-(DfT}BN> z^S`1M3h!yRm?}4Ahlp`M>Vncs)EsGiA@lQ-8v3+&7N=I0Pt8xq3uDs5MGmXoP)tb- z?cS=4G^g>oRg+{Ih_*Irl~~UEJr5$)tSBxbBbPR|B+`Ueb7-mkUbHNcW!Pe4YITIq zmQm=U^FKU8rX|W~#?UV69kEy#KHfvt!4@J?T^f86??TNnKP5 zkZy-H{o^TzRxD8B1aGgPiJSD8;=~1{I`R_Zf8&U7i)JRht)kA!s1~6^d9gIGz^LJP z<$R9J)}tQAy-eAfb7<1$LntMvfU0KYi_YHLMN|2#+U3&T=(r| z^yh;v`l)$<>2x0Rc!T1LKh4x(P6D?Yf_cCje+Bn4VixkBk3?1`B_5RcdPQQf0jO5cYexm7>u%ze&CwsN8Hq4F4%ET2g;^$qDXkzl0>lW;|qE9F9oohXd%Ue}LY&Kpo+Nin)7 zwjOmB$b;<(FOe{Eu5cC^M^^=GLVrAp8O4iJsq3va^!c1WZQisW&3~{LDJwl@#%Con zEz6FhsIV#~DMSq&>C<%77d2C_r8Vem<#we1Xf6s4sbqA@r_rNT_1u#>Gj4rl8zVXE z3NvqOH&@)Lz&x`2!Q|aGYw$Qafd<>;(u`Tz^x1VWIwEN@@Y>vn@J~` zBI7n1V6z(Onk|9P|MXB9Ss7+!_&YSNNsMzi98QgvZ=&WOyy#{7Z794-2EAR?j=ny3 zMvXHE(YF{2KJ&hk8`MleE$82HyBp=0Wh%=!-vyzRjgdyh0kVqkvQO{KLG}m?yfm z?Hjiz>J|ET4xw6|h3L|m_i%reGL0W&greL6(b*>p(7mWax&{10x}|$iKIx$s;w>5` zirwLc@7E)f3$K_+mBq}hr)qrG0>e!E?2ED#EUB$s4=QSTjnI!gdW2iRj5)9wZFpSE z%)E6H6(ybEepKETiKkyh8LUym2^(v4P42DGVAVdfpxKRi8nPeuXdGceP&R7R@J8ZZ z<@ELE8FVEp=GYyMl_zj{hLou z+q4V+d^>`I-ewDRnb~Oa(o$3|Dn?04hv6gB$%ypXI|lpJQuVNd%${5K=)l|MNOG_Q zsiZ#wt^Js3;O4ZhatX`LYaMy850`2v?G7+=&n4>5IlCoURn&Y@6p(qpTBX!{tL% zY)cNZTnQZ2EaJJ}O%cfcKg#uM@_7$eD!9{schU@#`Mlr00lk-Vggg3qEA#x6EqF|?IELQ&bdPBWPve%Xs^@0SDWSFptvPj-k4UPb zlKC$#le;_f6sntNOGnR@qg$6IqEkEFXvQNAEbHgW^xylzXVyZ|+lW49W9VkIU(1jl zJb0UNeYO(?w|8)bxu>Z4EJ->t-$l4Pqn7KJO<~48I7g4md4Va@QGJW)^0`WHX20ix4!2XCM@D?kxgC19)tq^Gw+>m( zFs6Qc+Gvi-c{-?VjNZ;kMLRZ3Mvp$fVN9craPnC$oaZ(dYA`aL_Q*Zpz=Z$kp=@Qe zjPHqVcN!yIP;oyD7nKubV<$9~?#778D3iuYFF{r?;Y`#}+eQYvt&cjB(Ucr-?f< zVnKuNI8gRL1#|mV5m(N1pm*XaRJJ^en?L<5t$j2GCA~=%t!m#(Z;jJICHITyY%^y% zZh-Py9#+6^D)PeXo3ThPNKY3gEvIGHV#Xu+>crf!-IeXz}(zRlC3 zh6e%}|FkJw_|^AF;!PYKoPLnW&xxViZ|HKF-SNVuqnpusvkP=UPlrjD=_1kP0m6gU zdr^Qz6iv~-$5<*V(kDi{sPc_ObRy^_^Yr#9`hjty>D%Wsj^9^vHM&#ivJ3@IHexUB z_dh}}gr(Ag>lbM8&_!fYnu4BBwV*E+XQ6qxRP-z)l0K|*;#zf{FcF5oMKVtJxr3Kh z(aJg*9)q;fgcjr5)H zKAOAb4x{KkL)cr0(8~7lG;%nXHnkNa|0_$-t{*nYV~GS6?-h3p`Sb!^J6g%x(?n5k z*Nez5RRwtzc+t;)`h}`bn)y1vLKtGt*LY_`>0s)6CeC#a4YRm`w(by|bgy9M{uG@~I}bs`(<_ef}n0zgCJmEBm311LwJC zl5&oPQW_1{{#>N$!c}zt>hXNG@=>NfR*mkKm!aolGm!PciS&^>=AvvH>0e7kS@YBA z<*KX9z6dWQv7SXnC1dGeLK%HL$C|$T%b-$857d71Bwh5+kJITZV!o(LBU|}OMo}yk z4efe`7Q7A+dcU#eb1s$WhkkMTwEqN`a(porJiEjcTWv#fPT5R`sEs+nrXu|v&75YS z5y~*R#~diXgY=VB>AD-bs48YFYPqH;to5Tz(PE7HHil7y#&6tX%@DdUcZjLBl65?2 z@CIcS{T8h|##7TDKQqq^ccBXFvD|3qRJ39A36hzfLl2x-#L4ZJM6U~in72$6Y9CmO z=6zSB+}9jtO!yD3{ZI$vyJ;2Gei_5OJ79qJ-?|SMGzKxMmA8?q=?BrP)r--Xe=dyV z&N=ol5bP{xiyL!s43QvS-kQR&Fl|B z-$v9q<0?P&merV#)D4^XgCiGlDADyx49xBZ$W}dG+M~x#aIJZXy zjGj&odLdmUI{4HXKN)Y0Chk3s)Hjc5i2Yr_8Qgh}A|LA^w&wzp>#5=PnqH&R9_4UG z(^oS|?~ZX(59~mL@k>}Ix07_fkvC@;v5Q7HDLN|syv205FF+qF<u6*0cIL03nR#ID!A&y@V*2_LX<@uPdiL%mGIeWbw!Pm?nQTACVy_+g>TXUKyLr%* zw~aI@Tb%|iX-Dg(Cey5xm%{U5@2POj0A2HUwkT9_Qp4&-AKExip1F}wfYu0ia?z)! z(AO)J9sgdCbNqRH7ni;H7@F^YiwU|s29<5CLd{XqjKL3YT9I2xy>g<^HX|oKo3nuG z4B1nOZMD?+^=-N;d$+K`cn@lfP7>MOPDj!^Sd>tl#yzA1=<#8mUHOv6Z2lNSA4bNZ z;)!dRF}i2DnDEo64%|o0m+vvJ_BG6$V7QE)TQI=97(9Uf>GI%3?nx9EP(uxxw$X`O zL+F)lru5H>B4*alEvTgZ7q_-0m+m{|i)?4S;e0CH>9?VM!l@sI(F-dTbRp9mi3@Jg z|2A5oN#%QxtleHT;pt!6e@_;*u;wU|+Hgm%Ohb}~m8im}-`pMTZd6*Y(C{Wbmr*sC zL*JZtV2lJSxKr^PIol~(wEy}fuIuY*v~ha~x|+ETZR+GR{`v~(v%xj=xq?1z%v^?| z=LgUhn?O2GQcHB7&kNojyov_?jz*Hw3Y?S6LZ;-$InE(l0$JT|6`mEUqW7-^jxl^@ zpQl0!3XKzo1pzKxT&x3n<#UPZX8&ex%A0{gfn0-=*+2RluB4VCHTs zoG4_e5%nM8XmU&jx45N(hB>8C{ViUorQV47DHDWlhD6ZI#YdUSHHy?)y$F@S zy60RhCwZ!zxg;pzHq>cT2K^MBHRQ9^3p{C{7=j~H^%_!-7%@s7bI{8Jcj?#23|J^K z7ygf-GmnSr3&Xg5iDVaviZ&{hEceWuGnLYhv?^)SzK4pUv{<98StDDr6cKIao;&9Z zNkn8#S_q|GNhwPG=I{AuKA)L;?s=c*d7tlb2`9B!e>=KYJwW#--$uO;C{4PqPLmws z_`72I$e>INzfUPc)OMFgcdd6NIZd@lSK~J6AGrVvJ-y(kst{z2OvIXFWN33%4vdnV z6_eG<>Ajs52&hemgJ0t4mX1Lpr*)g2+dhDbzubl!pB{t-#1~E)IE(B*S(DT014L1G z0$uXxEx*FD1@%7bL{d&4AsaPXtoGp{6%9&e!OsyQd7KI}nw6=^ru|51g$GI+D`J9YL1=aRJfyQT6y@-A1^Ocu$hKSt{fnB0uH9G#rRE+*(l$PHWQ83XBbANH z)BWJK4{In}5P>v?S&}!CPrdRRN#);6+BDyr-~afuXj?l$N_`6G$FLF{^%ueLJu0*; zC7EA5{x)$>8$+js+rhCOXGv^QG#afuLZ0{<(WT8(PE$C6hO={&<1V0r&grkJNQSjwQG)mqH4eqX@B{@RT zB|$HFlqe&p@wh4R4T8`;1wROk73s2DlJhEefV|pK$zP(k1RX83A_d_0SQM3wbwun-b@4nDNiu#p zAotD)RID}%YB%g6bF3vCEG4bkQifH?1F@Uwjt%-`|fG9+`s830~No$0_nXw zlpA{t4Lr_)e@c?j)2`9t{)naItNLDk))*VKVpJ#!-tLT&|N7FJnXz>Cr#3V(C;_c~ zeU5k;o=4)$2ELc;JCdCKj?B~cM~{5;h~@?nS}rw2jING_(Y_8y@%(4FU~vl^Igp8t z_bo*`4tUaUB}bse%sY^);i0a?tMEzX7}UcbgClUql`sX?$mr39 ze{PXSdE>-geY*VPTYJeU_t9v*SOy(4UnIUeM;+E1OB3tc2E^8*f^s=|D3q&&GtOV8 zKkQWL((x9w>Dzp2xl%}O@~6<>>vW)F#ArDAje%(Czc`U5`VGCFS;DuuisOQ1hbMv&E?;-I@iBnmM9#oOnBP}}8P6xiuN?_E-+Dg*v#Z&V$9 zC2x=JN+r{?@rU4PoB!aMOAW|$?R?9E~>|Mx-oe2Ea*9@&A`w$;+Q4jZ^V?=aG_%!Nkjzu@F3CwMOM z3(V2ehcZvTlAwRt#31<*Tt6B@2R$)bcx;4seRDjTZe0$~IT{N3>R!V?r|r=<^Y6ql z{TkAn=160k!bt3e47z%rFBMOg;g1~irAl2kG}rbqe70vkV!(wM#=x>wGbvYgu^iHqvUIQJlbq(h}sj=(Bj&Yf{4q8JgMLwgi_sc%e4Q>YvQth7k%{PFg;NFme}|Wi4@laP_sV_ikkkF zKi^JIbmGwtvdH=p@_KxUicf`-ouWEp`6v;V(4`NC;3lQ5&`|q6+5S9HB(C%#1HUd%cjp3XEAIx}Cx0c1 zLlG#*j}bBXnMi~(;Li^Zq=6YPR_Z)UAMG2YKimZ9d+kC2v+FrgHSIzkq90^r%oMbW zSR=jYBy?qJ64@zX7XN1)K_5!>0+UnuVf;J^mH&UHFjhc$%;JEeodDo5a^5moK(3Ubq zq-NGdTe`zVd3BX2{>FBy(&|aS`^@Dx>&l^AKt6}JbsLe0%q zwDrPrR5J?Gc`mjBpF7#e|I<@sb3vU>b6HO#Wn+;BKaX}lJ&!a@8qurS9Nn6bgFGk? z4L_ELy9AF(DSHxyO)*3t7CV!~AqP~iwwk8y3!t5+(_r+|`P4!)8EJP|2<&SOMJro4 zn)3P{|G!pUD(}68ItG@YXP3L-xPB<;UU!xT7P!(YF*dYLJ&vEaegl#k7CuG-)#RFsP)OA5!Vr z!eq2tssVmc6Tt=hqe;zVV`?itAi}0KuqmF0e2zNOWs5j^!w{gZm0Re-pe*kA*E=Zp zQ6kD&Sxct(cZ*i{E7Apv{t>?u&eUvyoIqA;2fW2;)0mgFG~-b=UELcGmj`U*`<5$; zZIt&@VM_qYIgaU>vFG5Cf~e%OXSF;k`x}GG4xEDKPAkYG_va#01ywO&1`JG_dx4xdBa8x6=sqcE7}zY5Kp zpp9~TQqYZ)Mnp@gfmpf82?l1%qwd@|Ds?X(Dmf;=D?JR1ec44yEo{)C%?putLJ+y1 ze;0kMH>V8)8<2fdvZ!m(ZFJ&p8Si+@IKhEE8(`@VPvpDG8`io=p=XmUM4jt5iu84@ z(6#0n)Go}0N=q2ICI_OaNuQpmZqgKS#NsBJC})GVwPeE|lAeyA9x^c7<0b#ly;5Q~ zXE*(FZ7Rt*H6F#&+w|Zti|VHwqapj`$WY4-Qr9VQx2w_vU+_dKLGvH90#i>@?mFy zWcg_0Sa{2&o^&kMgU)TcL|VtA;iQ_+{IwMUWec(S86j*^Uz%y zI*${jb?!w>*&3Uo47#QpY$qLD`hM909K&e?2Fy*XX9-}4V& z&hia3`e{dIy4|N|y_C?0wLKI@sR|l+gq(S@jV!U6MsIa#N%W*SG=5wd8d%TLrP_hy z+Q%i7-DQrP1CB^^?RBE}dYkCh*CUXb>_11*1EWLmvaKU+{_&ZgrGj8hLNR?Y!w&h_IZ)|+R@B_i2Qe*q^sF?d zcE1{t^X%p5!0mp%Qiv13^!I3iN9sPR^Vp5fjucU1GfA|!Umk|8olP#im!rQY)S_Qc z%~74Cs^9*(Iq~QUM!vUG(EeC;8a!(+ueS9m5(ZRJyY@t)prIfhs+mKbkH?Ub3y|7< z9)K6zbSQszHMwz5j}%>bNKV_8la*Nwq^C4MqL)u5nIq@u70+=<_MbaaWGeV()z67I zSetI&s!UajZo%uS7ifUd3HYUHA=)A1tFG7w5>J z4c=<#Thjnh_$ZIGiZFFwun{d`bKpj^#k4Q;2h42=ghNu#U|{kGlo8 zMgg7sXu{QgbfR|)ygng`yxM9{4xS!H$@+}%BWlt~DW6Ku6aXF{x zR-bvu-b<73Xu+b~ogt{oQW}*_{Yy4a87n?5zYy)ZFrFT+I8BcIn~9nYKk#C9E1yN9rFccS2ger z*cG&I?LTt=`W#6!j*=v{mqoSwe zytgK8l6a>-tBR1y)0tFshocvfCpqEqo*Kz^(?WC&j{bas>P)&yKl^T@aplc$<5453 zbLlT}3{j#fC#9*TR1n%%bsave?4jU?H0O~x9);=bN9D={J>314uVVa=KI_sXQZBd2 zpKJO2bhTvkwSOKxzjO!fSCF9>Q^(T3tx?c!%LJM;rGy-9Z5M@zThOfef$;bfKf2~< z64~@Bla7fBC$~##$*Pfsv=dp;8E=-+tp$_l*P;~un>7Y>?t>6&Z=6K;Npy*6>(vC$ z<)*+aC5&zb@WiXSr&GVl$B>F}JQ;kv3a>MNPZcG4)2o*W=wG2wgfy*@f0O~*z1AF> zpF2cjo|Vz$18b;g!w2Z}u$dlwBAG#;j##C2giM~LjJ_Pc#{Vwqba1WErj_Mdbm6xo zGG4R?X$U6>!cFvHiGDMcu8lx`<*xjHUtFlko*PigubRKYB$LLscQG}>{iz%7Dc(v&8mw#~L%?$pl z`8M>vn-3+&OHgLweWYb>2v6*|iaIs(h=Obzzb{wPzNIOkOVynaOf?iQzVe9-g^qz= zq%m0{tq89Sl=HcmPPF&;CfZrkNwqE{Bgf)5bP?-MWX@R%5__Q_AmKK@;L2Gl{q6>Q zpXN{Jm=U;Mk^!>+dlMeEBednhVUnF(&7IX_(Atu0v~|IEi3Y!t`VU#sCzj($kHJ*< z{g@`XmdBEd*;zEhcP5&&WHWhcRFC|;>R_F(5!v;&ifV{sh~WVZ4wcVh`^u6{16D7F;Kok6s7%n%Kf_%nI! zW06UI3xZdF!uUCLbo8n)a&lb=y)0)=Igfm_FlvxqdG(KP(`|;pI* zH5u1@6DfEU5dO(bWc02d{+$|(T$DWM^36`rBBhbscT%VQN;Z7U%|TRp;3&Fq=m51k zewWDWrl4nEr0H8AK%^aAZiPAg%d3esFUO~pM7n-4<~D3{?R-brG5$KDn(Ph zc{Tit2ezWOD)D@&U+1X*hbDe~cRAS?-2k!nWm>YP7+sjR9nIIiE6Mrn6D{0XPfrB( zz|ql)WgL5HaLt60ZfH9qU@uz5LYBbNZbEf3Z zyaPRvPN6a#H%P416_H|;#DlS&NEgg=iR!u$WpErgw4n0Jr_9kEJixWH_C!|lN9e^fV*U;g2AEEgdb=2H? z9t~vAi3Q|X#>q&mG4)N)`BJ^AOKbUqR3^h(a zLW3lGI!!w#pnGGUkq@_ZY2X5BlBatfaVBx3w(Snwx#BK#OM>X$ur7?yS%GBNZ$&q$ zCA_yzlHGQnhSWeI|M`ny(VxC*RF@gU#eb+kS0h8nSdBoswW6BXT$AID(0kC@Iv1_( zs~|4d?-7T4H<8um6Et#o7mWzpMiY5gB>C)c$)3eOR3Ys`OI>cEwrCHsGUF){97;ld zCvEwqvT2gpQZQ;4U?BVAeQsq1c+J4NHkLbX$n=_859>TPnTmv5J_7grV|K0V?v2p%Lo~ zp|jC^n%o*oL*6T*qQw{Jl3ltqNp}RTzNsqyXFgh75chx<%%F%gUL~9L8)%xz2^id8 zOlyTch`UxOTA#O&QVbS)!5gnizHFlbTNkXiHKWb(`b?%kVsO-d+W{E(}EP-W`W_bS9BHEy;r| zDn?#OTj?}XO$Oy8I*;{b5_zeg*0e{m*AMMO${P+)r%g6!%X4LVcCmm?jm}37_pYG( znl|vwY=T~^O@jeb&OlzMfZC{~i}Gsq$sXkXy{L<#B{7p~LPP&WBZRcY)!p}@_*elt{IipIAO6OFr+c01W#1qN4HX6K z?AQFt^k8_hyaCG3`6XI;H;`No%91n&bWp2CIqKF^fC_sFJ#tWo9=?4I=5An7e*Ju; z=MjmX%X*1U7hERGTeZkwYypzW4U>G0R1}jpofsM@(T6vS$=dgqQQ=4sS}w_kX;@{G zz7Ky;;S4M455}VDT{qz`(JC}vZ!u~$`-N^6DAQuyQFK|iC$SkLFSrdxL^^^rdL>br z0BA&GYt;D1A66li?h?9u&rH;DU@P5ndK-QIDxB}+VvNoV1f!K5=g0!BSM-OY1zNJC zj`)XlbJF{A>4x|q)IaGVdh?TIs;(VHkx9yE_eKW2RGEwHC#Mq0zB=jrH(IP$eGxSr z+fN%l21s`9yrHGBB-`W=jF{*o=#crB9`!#(qo2gn^tKxMv>Bn<_r8<172D|j0v^f* zfkeODolY@I;TvV`rpEVkXoy@kNtoLT&&)6MUCrB$;^8 zuSx7a3=4PNr=#;^VcRQ56#uCaWw@`SYxecik+NJ`JU5QMG|eU1yk*GsunSE!YeN?0 zCt>g<1+jQ;FjZsfL@Q!4(XBagv?<{~QgkVg*mOst@G3(hx=}+)btloepYV<5V8q3M&0aXQu$gIP3x*gPYMUgj0<@*^kf>hP;UY%UVN9| z?rDcYFNdKtoqDMH_##?d(L#$Jm5ILph(ZSc>?qf|mj<|;hHD(u34c^6vdD2k6TjCH z?~-F^zh680vr9K;l;4S@jOJP6VS1ZWBFnsLGym^hcCBCqmP!l^hj_G z62eYWnQ291D<@E%iZOhHGAJBAPL~~iL+d{6<@L+}bpBp_>QZk;Z6!IG5ZA{v@LdNi zFCa*^dp8Hm&yH_RARmupo&x(a_#C0bm`bQ6cnl@zW7#4 za?W7*EVdfHEz3udSIyASv@7uZqW_3Phc-0Ww*_vw`3^O}1q7~+L2awgT%D_6rsO1)p{gUSYY8XVvzd8pBw zu0zNxp`ZWK=M)m{|Ath@jF6EG0V+gCX-Q!{D)w_m-wH<2FH&-7$?iZRXueJ7DExxu zDqYl1z5?}(>xFkS`cSA{7(F!o6uK^*Ppc$edWn4`-JHDzCEs!6zrJ-{MnPxj&&e3~FR_w1G8 z$jabks;`)WPOJNq_W~laxHb*l7}2DwC1=@sePa~p`5!m=osDE?FGtihat-+eCeq%x zxyUTSny$MyhyMDq9GNQ>pt9FXVVFWX;f^Pxj6Yo@KK(n1uIm?_JUE*QKF6Z~g%pvy z6y(>~%!9s3y)ZPSl-xYEm8xpRq3Bs>k)vey~&|sbCw{@dX`IZ^b^yPh&aHfc!zY#{a8my;DnU2tPe*=HaP#E1*5Xb8YyNs57 z|3NeN6%wR#9-iK04DTCTnGRY;-y6=?a8Mh&6jvHvuLqIm!%^@nod9<=d z@;l76CU(yQ(R;NKq@J7#1G1W_{jN&%y()zIjM5V?S{gyS#p-<1AC6G3aH{z7>_m`c z0-)5yBzkP&M&9%-cGO79hfcV(6T#2vNMV~3yp=ITPQ0856J{6*(jKcKQHUEIE>1!# z$62GyUO%KK<$Jeg-ATG)!5}dSuczB{N{DvPXDT%m1s|ga{K_v*2+ci(eE+5)|2KiO zDqN1fNEt(T+vdQCgGZ3fy#=Jms)_i^%Fr*j6zE_55Hz9OkKfTGL-PiU$XJVqWU=@f zdT)1%T8{)#^Zf}#{Y?k`a17B{ISls(SJI>5W9a(dX^?A(qe9hb^x)ue@=fx83UY%q zA^Ri$)|1yFwW&u?p-eQCK9k8GKcdOs6&@_vZw#XEl3wu6lvvWg+9y!%n{4V9=|CbE zD5Hq9MI`n~5ouZ3D(aA&W%J_aqRC-F{LU*))OK7XTsOuT{r5Z!sa_5vN2JC;xw2NW z_WW}w9T$dncW9&QZ45H@)I&DGXVL0O=lGYG`JyEw0is=!9P^iUP4vR*IxXDsmQ2Db zY?JYQU~$41JKRgKG-?Hu=SdwZQCpi7|kE=@zcGfAjj34)j!P#Ue0{RAW0dInc|Np zc?aQ%>1J%UC<<(EO9MsuMnFTtB``ba37$q!d@D^E|GobJ9KN>|2dND)IWBkDilj-H&;oDwwzK>74})Ev5qPdyEFL$v9~j;n!@Cd_jN=Yo z$E$`ug60icya-N{HT?5H!oREq)u|ca;SMP zH(bExSCHL$bw3z;a5wg!d>wz@Hd@jdZUJ1KlJNJZKk&HarozEnU$En=tKi(2UAO_+ z;lmo&!SU-< zvfKo(F`Vy&JXS%J1~QJalAfL_yzO=hIQQuUe%k+njWfQ0)ocG_XW(mK|M5J$|8*iR zy_kk)&vU|y3Oa#;wI1)@NjY%T{|``;8_lhMa|>5<7I@&Zfp`G~ z=Hd`goEwQ7O0Si!-Z2|!UfYCIxbKYC89AV97K3Y2CWE%7N@4qiPPWoCM3~kS$&AR( z!HP2r@kiaO%)x{deE-BNQ2ui*u-xzkdzU}M74zrvQu-mUNW%hv_hH~748uDC6t z3mV0{(4orf5;}0P2NBn4d61pdxPX2AVGgL;o(8_p)D+I>55(H*AA;u#!||<`!T9$n zC45QtDfl?a1Yde&j#HDLGDY$Vyc6wXnY&-i@T}!IjFHbL(7o54oj7wZ&>wwC zvt%9<6)tAlK4vp$Lm=J-)qzD~1Ss3)4z3rgafbrtf{$`jLCx{2?5FReICVXo^lxv%JxOIuPNOz2?&S(w?~O}9_r(~dGTxZ8n3cnr*Gd7c*;Co~ z`S0=Cb!N#jeNYaY3D=r)Kgi=IUp8-h!59>jF3$e|`ItO|aA9tzGQH&RaUw zRz)L=o%=?S>r1a;@2pA#h3*G{<1%H=$NdR2{!$uyaZwB#BsBo0@BhjA&h*7|ckjVP z5yMR4++@}u={zo~h!Eay*5H|)m1Di{j>3i6FPNwyX`WkWf=$u(dbVRj2jh3>EUF`tf3$78>$0h&DrS8G-<39^2+Ub;$L%|jC=%6|oZyEcTe`tlFhEO}xZr*aJR zoMddZoTg!2XAh=z)j@zrID=gm|?xxAb0sBcVEs@VE{bRol#TzP!vjKu_UY z(FO4Js-|t;9eX@pUrFfwID#z-Zef-jpU2X< zz&lHgJNvqqx%}1^v_3Y&uWV&FXz_*(9j6B>s+*YXcRJigeuRW8n!?~maqQCNQ*c?# z5!=QF0XUe}%04^WhmQ*hTjia=)_Y!HcC_3BL)mf6@`>J(C$+qCSykTyQVPl$aWmYv0!~<3X>Pn!gwDT&ncAV z;s^568K_YXrsivc*@LIS^1L%lPDcqi?&HB4wpD}GZ}{xO(Ji(qv%|6FQz@QLf~D=P znL#-ARt+xwdI30JY+Wz9_af6~BFr317)oXgp4x(W0y#j=a;G_uE>e6g+X8~ovgiml}s$X>jE z54`B<0%!HbXG0ArG;g9GzkgUQYMT-cO0VE@4pe1#=A zn?DUGsdV6k{mX>)P$KujdH{2~K0$*zy@EN&yTjyKjv3W@yNV@t8 zSEPn8hu2qv8&kh9%V)cT)XmA@_lX3M5pfImuTR2@M?TqZrtes~_OtC4*CW8hqXgVL zoC>V#EOA&$1(1lR>`(g`370SqOYwU^XWTa6cH0S9dTDSI)l-06k|&$+xq=NT$zfeI zJ#l1SIu=zp;g+bYU@!@To>$toEp`cP$dzX}YrCtBm1!hC@05+}r)ctCT;Gh9wI{GU zCl=vVtPL+*V+tF#<%jLU`Mo&3<~q1`VjWIcvKgeGD+Cdi&)DOs5Ae-J-PV7-&2U0Fp0CxS3tnxSq?w?Kj7=$?r1o*D30}{Wg=p_Un>&;LR@l zZuCpAI;o9N}niEqe&pe|H2g7H-C~=iI}Yfn^}Rv=uk|=$FcD zKsbr}4$1@T@v|e}Y>)iD!Qf&`=Ed9h3@zG-<=GCoPz7eREiW)pH<~ zRS&erEdkr#ih-w<7B~B9C}>!|5MMYY#KyC-h2xK@;ez*0pt>D^fWrZRe{DNRf7UFV zx3>Y*^eS+I$XcP$IoCFC9BZ31?hK=OK$ClU+ys~wn{c~ib^(PQvYgeLaUju9kE4ei zKyK-D?&ji!peoUh+a(r)#V*Pc#>QA|)}q1NdA*f&FVx`O=rzRWdKGyS_sHVcX=*&f zEgsApv(dcNgfjM}Ob13{MbHuP9@p6#frSc&yg^4BE(Iwwkaw%?{n_^N?;v%(kbYi7kfsT#oshwn4P&hLTcq3O8rbQL)LL7R2R zsRn^Bo-n&2DL8NZ+;)%uQTB1zIChrrZ??{kW1Rz(d6t(fu*GXl-t@sSc(7p%cS=nL zsJzwYQhEOY%>f;r&qjHCNkf qM7%x=!V_?*`blfZ?6ql#GL?#4@It3xTb!DmZ$_ z1o+Bp^U5{V@TRe*yym_&xHCwaJKyl0dEKDLT@P6SVt?(yzhd`V{~qHl^vtnf?G@B{ zm*-gm7lSdp1-Cj`%l#_ckKnl==#DOD>+A@sl`OEDlVmQaRlxq@&&)C@4PFya#DDzt zdD7|s;rfinAWG||@cwjJUV8c?9C>&U7s$P5*6&f~hOHAB{c1Ii9lcG$XPL%rF0BQV z=c;hsrwH@Svj;1+CV|9D3fxX72e4?B8mB<6f`((3IOSC+Gv0JBK5aG=cqSx(PcJHL zyq0z`R>_UFAM<~K2M%^jm|h2Pt1$r^PEFyyJ=Fqt7T9tV12ZKI_Nkm>yB4Ubnaj^T3x@TlV3POW>0fVOu4Au9JUjGLAL6y!fJ-xcSp;ZkMqWh}yV{`#a_u@C;Dk zsmw@aS8=jDq5W7k_ux}dcUb|Wq#Kpu>8p3a*cWPqzA zJ*In<37%0_ir=KJz_YuHvD8EtW>CKk+vtx6O9I>QVtq4kbFvyY=ASxqM#449KXd_v z&HjN04@_r1|LMk|WIKa*%i`{%5E#xh#-%xdV8(k>PH)?M5ECfrdA4f-ucvRp@9x$y z^X`tt8fzVyt*_F-{)d_1pyORI!+m1u&Pm2RIh~2vWWh|{`a7p^VwVP|)Vu_ko*2i; zLI>bbWW@XMRT`HapU&Iq-GDbN+Yjzl%7YK9FM{QCgx%Kk3#e;P!_9Jhe)&LwoD$AuV+Xf5xX+6*Y9@sX0mSv9bvc}JjzYxyxr~qBxZnKTcX5ck%)N$zQHSEC?-`KG; zC0*My3fU8WM{sM_I$@OJTs*-o75CO&V-L+ZYol`Ep0Fw?62DJ7jYpTM;tyI6Z9T&j z7^^6M?0D2ycsOM;7ECH;%3Kq`D2vCoR~k>UE8DN*z6D!wZ<{gref14Pe@?yAnc>X1 zH_I^!a}MLDj?3^O=P>sAY%O+Hz)p}Z;26W`EcVo{NzAOW)4~(uV}Wb;e(+0I%O=6~ zJ3BWm6)%4Oj|~XUXUD&_!krs6S>jtzu(L zV(xsY%#;*1RN88)%wl_rMse1$d+|a;73{K86UVjsunD`h@te3T_NTiud*ISLJn4ZJ zcC!v+lb1}uYqs))AAhwoG9lW`)tz&({7WNY?YmOYI(U**a_8Y0cT{j$-$B;kWdQTr z=q+<8#2xH4j$$^GNlaDHb5{D{Zblft3Cten&m0(r*vl^n>mABrEBBW$S8b=`_T+7N z%3LaZGWr}F{b`2nJ!^N+NzXH-C69%NPa>dws-HPJMIf}itH`XLDd8Gt6|u?p0sz|A zBdl5z#G2LR3fFA6z!j@USf9mqz@RN3lqFtaB915ng}@wk+Lk3kcqtStopqP7^wnon zBU72CQAfaSJS_Y>>nv-fUx^h@{D-rvbTRy@0rn1CFz*c^K9^;IMt(6~AC?tiv>5AJ8*9C*vD+AtIFM<%i%=Hog3 z6f^Ew*eXsx{xjbEP@Y?xPU+;-7j)FID zbJz`12E1**J}`A-T-Zq$b$E9dE#;lacH!BjzXvWirFm-Z|MAXzw&gume*vycoGoDp zP2);Guj3SgM{(zFOygBqX!8~ycH!MNdk;MCkLN~?8gY}>I&)VO8}ZNe<9T+wM5PNu zWO?<~Kf%8ES={7b{&;gN!}+~^hs!oi(gqs>nIMXSTe*Ps2EO+=X)|sJ($82!M zCGkH1zoM>GM#6T#yXQ98bl#ZTHm8)mbx)VmGkk^rq}Suavlt*r4B|z(0od46nb)%^ zACyNw0Kd#-dGmIe@w!Il@!nf*<-Iv6;g(d5<2EZV<>ozF$I&Qh?$6EfoM+;5e0ba< z?yS8rLJ8-1&J<5g-z#1SFPkUh7~!vJVRcx7sHd8=*$cJ_D9ko zZO%2@&*gC8Eba?qz?~f)&wFw~z%yRDju*Lc2;|Qg!KLmt-0Kkmmvi(A){xw1nm)FI zT%Gwm`@>~m`JF7bGW{dI_EW;CR8ki@s|t9h25dl*7mwF2Qse#kFc(y1Y{!dB=75m= z2&QY2BDX_Rllx5toav|wSn2F0_W6M;_`ml&ykNBjp0-Pl%e;v5xTb+gT%m&-cRKI`)+o{7ntAeEJOo@?@n>8Y zsR3Bq8sL?!42)m%L9aICZ9i$jo2R{rry66x3;LzRn-!|gQ%|(x=~-ui@(2SibNgg& zo5>o^Su(plUZBdwQ66XP?Z6!!`x1Ax=7SBB{<7IT41(o&prd6xFKX#{p5f?CJWcHy z@X}>6?`xPNPgJ~xw{NX7@As|o9ITwht-ZXNQ>c51%e>5ZJL+wD<}Ww#Xo(8%Q!V4k;dSv3La>aQ4!v|#YO!GtIH)d9E(o{YD8HwgZ) z1h~jK%3b=02R8 zR}Y@G$?-IH-v&`-ZFrX7Ls0$h4p@>g#PBEj;JrZEcQK?$kZiYNZ zqOW273tiyarTe&k(g5g?aA*3))MJr00SckT_)8kg+PrcDkw@OMZ9bBM&eON>8YLy( z-(Rs#CwSAPY3k2V}KG>ji^d5IVM^@AUKs}By(ysi>IbZ@y;|=g5>B%eCJUuh>ovj2cB(X4Za66 zaj)&!#5-^CRbdpq>|PACTvM^rDOujNSCwE#;vmj?{}hZ$h{vHnm*9k13&3#0PF%2a z1a$o_1hW^^;^${dfKm51@KziHj_L*C+&w9v(!UE&@e9Sz4%GmY-{-Kjp%Jh8eF&)S zzk_?7-hcqig4uxxbUH4;)o=C#$Et2%T;mF24m4twXBWZV;`_KwVw#Hue zO1wR{DbRY;jT4qtfx!JTJj1X;P$ufd(LDs5nybh=m{SU}z7OL3P1WG-qF4Bp`xRWj zcK{T>D8y?!{(w_`aiHvfiq6Cls_zZs_ANvxBs(EdggNguSK1ftO4_uSb}E%NOGqgd zLdqVIJ$2~rzhVL@#bEVS3;EYI>-t0Vc|W1lgw?G=tcn~&q?7C>A`Do)hW z;e-uWamPq`=55|Hyy~F9tdL2925~kj`jo+0|NA&Ac{{$D+Xg$?FuZwJm2*gm#)eOS z;ag8BUYcJHf43XKpuQ@KW?uv8g)I_B<05b!at!S}P6M;J7redGprow@kG{PMN8&p% zr@{e6iLW5iaSlGOje-iHCyZY<4No*g!H$0loRZpg46ykFy;e=Q&+?GS{likh=%PM4 z-_MseH@^m@s&JUo!XcfK38#DilDGaW3|%t{S4WJ4F^@LlTbM`FS0vFndwvV+SmhT^r>{P)sf~2q$xPCZ=HW^fnzC z5>AIcEQP5vK2Q&%X41Z-8csZs<2ID(k!+&|I=j6RWsEHtDa|8zMN0o7)$1VVk6;9Pa`zE9bx{ z&0pl}mUS2@Iz|r1F2hkxEEQ^;2QnlB)6YqhGsW3(?=x^6^Ins08Fd1KMOm;V#evi5 zsUo#~^`t9NpDgmw%zRrDRFMBE>iwh+iBEO7mVg>!{of4Qw73#E^GS^R zLkk@6jwMfgmx0ox37qNV6L6-sje4w(z(O}CW{j;1w#*)&erks>L)L*w=k}xHmSwo` zWDnhTNR#PXw+;*Xg6P&y=W%+!6|->RXKK;#ip*Xb0sjP`dgH zNo>3bMiR*7e8m_+1R08cdhvM0NrNdcT84WM7K%KpPN90HqlEi;n#wE0kgt1r((XMK z%}>rDHMS#2l)@2sQs%_X_1A@ujaTTBAZuJWNu4P<6Nz~W8_1HZtuU=v!m_oy98qURXRewv7%evW67Y{O~nMIE|bFCP;$ zhBDSVGI*|F4e6D*?F2$MZd~6sct4G=2w*Rv_HH2)ubEBh&m(k|e<6CPJ28I~PEePQ zF*IR&5lXNvOwir0bov`@Se}0x_Z_c*J=6k@<4sCWpT#X#9he?zbvzUAOuqe1guQEk zi{*Mqg};N~!iX`jJ92;g^T|fFd@`?dCis_1cuor!5w$^e z__^jI@d*2e{z}TkcsRf4%18qljXzT%~N zzG(3;PeEfpL+tw1iEpsv*-PF{o!`YtID(1zD||9z+A|p$!jjj?XGp`pldyIi;6%4< z(1^)K(~K1`DP9_{glA!)ZajqOW@3k}FTkHvJTm4Sbjj|(3mf{$K+#FE_26N_HHYIw zwe1hR9Ud$BFM_ckC;~o(4bXMAx@dW>j4t~vq<3zQE&p&L6%DE`z|s*($aorpsQwC; z@5q<*jpFdqe?}0pI1@cA(&1@pC~kkG1h+I&AyyoVTh>N`>2q~lH7XaysvH~}mWT28 zT;V`Pnq+nKLWg-%!DPlU8YY(yM~`2{R~5;yrojd`jLyNQpNl|Bz8sIDD{QJu#zSU1 zA^qHOyi}k90S9iP*TF_G8GQf+W3ymHP7Kt1i9oG0#!x-!0)4T%08jWQ!j{l9-0N=x zw*MvLr}Rs3Z~71{3I0dgpQV87^dyAxoq%<3ss0>oFg_VZ4}UR(o~CO0;%ElE+;j%7 zC|v*>^)bkAJPj3Yt~hvzK!k*;8F<431~zUKWE{9f7FWz65l4=}kiRQ2Ez}NtY$bir zZ+xC$?ip2m)kroi}J#zLKH2tA#IFYb?rNsp5u|MF=J-n$jb%Ot7(+oLdS{sGw4 zZ-CL=6=ct}%h0kd6E7XP1e;zM;RyL!NPeGyOZDTy^X_pRDf#T5N#&TXS_bDJ5!a`t zf!~7+ta-T_e*b+>4-3a)f;LB__dTL-y46sAFdFy&3WxQk19bX<3NqqC7Ov3dVB9%% zT)6rSurcSbVVLB%EC@rDdD38XWJ`z9N`a+~9CBrR)2>kUS4xCEOG5y~~*yb6IL#%ed_6xFDwk#PYIV9qskv|NZ zV~3WdIoNcj0OB=Hp~ksUV5MyW6KvMuzUgTsZ-_75b6LVoy`cd$R(jYtty%EAr4VjU ztVcQBLYgP9Cbbtrc)E-X*4E>hQh9gj_5VKmPl=C#P}r=7EE>w8f`cW z)z;@Q@91IBaL_}2`AFgzv|n^~_)oemtd%@Sm#GLWzl5iLoCD(PiHpaMg4+&}AbmF& zyYKdrA7#OG2A_;ejice@huyerzC0usWWx6Zin0y6p;G*t7Hf*GmDxu+YswK^=c@oamn=hzvP$AS z?Gpr@enNhSD{#eI5}_tf3IE9(Fpg1eFs|$_=8Gp&>%L?l5Q}P0u8H=ITtoZRMl(m2 z^gwi$Ecf=U8=UdK3MDcLcw}}Hd9Y+DmLzL1s*#3F``01d$N2g1@UAwe9W;V_>Glgg zyp5whA1#>-LkahE_W%z0eGs0Wa$u^%RN=G69Q3>73|l|kp-P9MVf*|~^xZ~h@U>Rr z>OZS-#v8uFb+teEcao9dbyqql9CgHSiEqKPK7**{)36ke_d$tWu&>O|Q-*cb58F`&d>+2vH4B2ZuE738dCvKnJ$EHW zhiSHVhkTPaINJCvdbWyT**inDF8l;nf?6SOjHH{RkOjqoYtSku7>k<@LSobq5yx3@ zfwMaCS!@T?Z0Q!r?v!X9dUe3H+(GGg{dDGvK6t<38!VOZ3`?)>qthd^1&xY9P++)v_1d}!!tOnvJ|$g2om)()mR+o04JkmA^wgH#->%^*73FQMU$=AU1O4wFi4CXqIag@O*?)RqW&>J!qk}ZaE^RkpUx6!?DeCY#Xn4t}`R+N)3pF64f z=xVZNcp~)8vBt*}yUA>cW_I4&Tl7o3R&dR0tEls9p1`G4Of}-m$lMY8P|eMN5KlGc zh2R~iI*YLS&JytI`iU>E{*nxJUWYqABcR(`iKIvkBMX>~V7pNPjv2kisCk(CCXTqSwGmtB;c z3o;T!VcaOD&8G=2c0R>l-lHY#X#-KdxJh)>Pm$;qCeXiN1ocl&V1`8>c~PiLmYtpq z%?;zp3F-w?4nH9q_z=;s3p+q7vYX^7g^=?xbKvU*>ilx?c3`IySZplBz!Sj`b;1gj zKJCED8N-Reqe;Ym+a!z%Uxtp|ZqV-UheIc;kSCU3sEGa)*&g48R@0vg{;zqj)r}Cv zHRqGfm9I#*mLC;e91RcC7vMC*0GPi@kG`??#^$(C^5NwIoY<`lkzdqs!wenR<4{9y zOm(6aGINn}v;{SG7OwOSB{*6I9A2Bze^J|^{Lp)mf2BJ}{S}MM+7^I!#(-$lA#Y5J z+e+x1Ks?lFhXFFb1sg20M4SI8W32eP$Z6jK)Ueo0N4EN)#xgaqvi(b5t4=1XqufAw zgF3l2Z4-Qw524!KTcBX~Fbs6p$N2Ief{a~L@xZUHings2;fUE}I&T&WO5Hi+>#u$i zw#!RU$jHEmtqw5j5(DLSo5(EH3}SJ#gP5qCz7E??Zhfq4xS7Q6B+(zjeEMasOO_)X!Gw6(b=;Y zKP)ySo5D@dX|^lKIsGNlP1+Ur=C1-;y@Isvor_xn+eBVN7t;qd?Iev%fCr!2sHe^> zm~mQ&*Bpm{e(YL&+dKlknmW@HUynjzOE6tDdI!upJpwBaEg%|l5=|)QoC}-a4_lOr z=&E%<(mJG}>0F9P@xu|goElFxo%e#mkP32h(gNJP0x(pef=(EAk2aexqn`3X*fgh> zvMw?%2Y31a^QoH#PYHtL(u4H8gC`iQGK1jtT4ehrL%O7G9r};VroTH~F>cmWc$M^p zd@-@XyS8(o<*7Utnnls+K3^&KXC~@Dx-O_@w*&Lm2DfzuLH4!Th;jpj$TktVqt67d zC-{=mueWI9V+}ZYcC*Bn5rVGcc2>xq*d$Vy7t#EO)%5i}2G48{CO0OYL36*$R9QF! zQ;hU!UjKR|aeryPVuT=VsUE)E<%Ij!ToUAW&IMi1ZepnF3Rea#K`ZVLwXb-pfN$Ook{vT&CiR)B@;J_Qrr2p=Ms;V=-qK|FB`Bb; zD)+*@%+<7RoG%{sKmth{(f!MHBBk+()VxX{Q&fw{+kdelMdA+oCxp@9kvpLBLpycT z5`og~_p~Xx5c~}sCHE-;4rrVq>Aq`#|NDpTeyoi7yDY&+@jT|N?I5*-2VlY%U0k1` z2$~{S`ddd#va~HBsgHovu#a^4{(bniw}QS%2*JJ4&4kbD6+OGvP0S|fL6yWm^UN<1 zMqfLOvzFIF=Z?cPs4o{j{@#qLjiqpCtb)K!HW|$Z*MW2OMYQG{i1%qR#I))WIeR&n z{HqI(s~#XHHeGY+TkBczyi}ST^LE24TLZ~~Ee<%Sv!6`7l0-}Y8xOs=r6FR{9h#fE z4paVClN@g?eAF39tadMf$zxMpI<9Ynn`b-dXRTvokB)%&J$L|NZpn08Y82_)<^q0? z{t;)(IaKGzIB<1Zgqqz8pmPQlTsNx+#}*-;{(K(H4$Gpt`WPslP$F2B-2w}h6{*VH zG1zwIv!GB;5j(g-vgdduE^k=?IT9wOD5I2=JB#3YuocqXY7jUn7!@DQmGmN}fSzhF zdJFXhk*@p64@)!1@DNeu#aBh_HKbQfq980T8`+DAqAjir&W@b{jF%DC=&hoS`Yo{d zktXWjErwAqyJ=d_VpvxbjX!V|9PQ*PZvB}_irArO<+u=*rd8A0&QQoM3Z}+$cf#3Z zDZFV|i_d?lLZMDN`rgeWyCr<}yzqTEyE+k`kkwQha>(^CNq_S~5S_Klnv_~E#W?X~ zm|Aq6-noC49ENBR|7V2WM{7W8lMiiroCbpi415hv0jbOsjNdvG!dw_^$zKTTggGMD z54Ye&UMOCwYJ_(y%#i=@4jg#=l=_*sfp%Lf-T&@AY};}Q^~dGFeMO0{QXX;gNeOgB zd={;j{Uia75)XCLCTQRN2>Y(>0F{_;u=^g1*DT86&d9$sSg;a8r|VG7i~6u`2roKS zastoSPnCGGBhY0<22nmA0U5*Aq2+{Zh&i;h;?C7r6pgosm7foyh5t3;6*C1k4ftb` z_hC?Xr-IENV(8Jk`7XOQl+lh1V-ml6E5xPi;Lgw}Xc9%xgj2)7I5a{e{N@2m(hYEY zdmO1>=Si+joduVw7vZR_?&Pb|NN8=6!zUwzF3bK|fU`q`$Q;&!_0DFhl%9dh{f9!T z$8y|UCJUn2*WlaZhDg(d5Kxh*@WeSLMTCbW?}mhkXQ$@{Pe*-&zR+egk&6>VnVF)5N>-s_6M#W7t>o zM3QJDi66Ks%N%=ijjob;1ka4ZaM1h`_!cF2%USK6PSRV%KoGdXe&IL^qzk_i2 zR4&S=RN__pM5a2KfNI@8Y8&^89$Msvt2!eE|MrFB>PSyCDjf@B&pV4kkJ)0|0D(N$ ztDs{QheKUj;Lo2Tk-EtRJb1PUM2AjcHhC%9D|wF2_^iSbpYyQqUy|Tm`YVvVmy2?K zmGHQJ73J@WPU2ne z1t_32q`3ZuO=cJIqs%W@&{Ie~#`K`C8xNaG8u8$4FEGxkz~f7kVPfxjT-2vZ_v_xn zSG8xrXw@4`?O8_Bug5_~xGPQ?+6;9Q#?xqb7kIu$3nG6^5R@lm&_f}wsQ6hORkjm= z@rF>0JmC(daeeghz;(!zllTp6A7V{T7M!YSM7@>c$lv%+Fm0CzG5J5ZQLKl{RU#m9 z^&|Q-VUW1w9Vc^UzK2V%w_`%p0Juk3&_3y}kQ$hYXYB7wxX~Ikpl%|RENHLDx*3M; zpEaPTXou+jyIiooQGq+xW`L!QgtPmfJv2Y~fq#P=L`QuotXB2HRsWrUZ*ODB;FxyU z`Fsk_?92k6e_P0@Cr@xz&lorxc~>;+^GCYUy#eI4%OxHQ4w8lsBRLlF*g0DcZh4hb z#x$SiReXZ2Nx^u3+Kf_j7=< zXKzJ^SN(&K`yWM<*Tg~9P!ayf6fl21&Z6D++erS@0iR!npEj=}H#Sbj#BY;f?YUh1 zzNk_}0_q@XAQ2C|83P4sO@dv&JaBw~ER?w!qW|Rz(T;Ji&;*(RM^@pgmovfG>+z8F zeCoWQ5H{Qo!6RoH!7-?aj5&P~e{%20#KzIMW0fO*UOPaRb^IXho?lT_PKoNQDnr*l zi_p;h9-ZU&1=^OK1Xm)0ez+luj&a5}t7VvSw?3Tb)D2lb3~~OjF7!AOPhZZ6K+Rqy zM)TEO(0TEf7!N9fPx=rpFz`5 z^}adD9sLN{mvwY+*M8{v(*&ec7e+pq4ssU#5}vv|v*f*qs#?AVtNp)mwoeg6=k5}j zW)^|ERX=W9`54YrdV;c#F?Eu7^;~{ufc*FZGVfC;7)L+HOI;_>wqQAI$^JlR-J3@L z={nLUEv+(#^OPOv zCjF*EV&dps0S74(o%`B>E+7RLP+`Vn@}>D0P0HO2ho9s?OHL~u-xC2o{yN;r=Yz0& z?OV9>*aSA--UfTEWVnE{$=Ed}5hq<~5Y1?u)1I4t)-bHmgJY zTuBX*-G*zGDxid|rjs(CLrUIx{AHpd>Xe-6-c`HO`%{);SlLe)w#o*!7gWKDI$7>S zlw?k1PdvF(ktWeV?np-6Z@@g)p`a9hOyVE;jMq&%aCiP%)V*;6Prd1ZF<*UQjQS=R zJ};Uqk<1*mAAJtnQ%xbU>O2I-|HKjA%G@}^DttU>j;H6O0BI0`Q{yr6?L-x*U6$qa zw^d_x@Ll?K;3@8jeFAQI^=Rt)m*_P_()%mr@lwJ!5a~Y2mm~ZhKMP}z z%_T1{ZHJ(llJj)0o5UkH23)`G6XjGSiM$V0VdJ+Au;tlZoI7SWru_=QA_rqq*%Sq{ zx`bq2i3P?zkiks73etM$3)o*`EN3Fx15VMwo*;8F`de&M#{p@gV`0ZMV`2CSS!Ai(qIfN5; zwuAGM4;4p_-G@C{pYht;=U8LnfNNh&L{A?n=CJNXjJs%y@r!rS!;4>mWrhzdsY`?6 zjz=Q<$@M7pY9;e*u^}f)b7x}96u2Ly-C!G{!%4q2Wpp=wN14CMoLjCr^L+g*?z`z9 zxHtKO#Luk7S*STPRTBS?!3ljv`kx|qE!dTrzxp3`pElrh+s86QWf3)IF2dC%NsF$chE*($77lCbHJoE?p>P=)$vQxNiYt}Qb&yDA%uGHiK{lh(&D76^clT-PF&`THrT&$JQsYv5MD2F;FK&P;n()7kaE|63qRn(>^boQN0_M+ zg$L&FVcdA4JYqOCyeUQ-W(_%1nMh-W+E}+%6K`pphWKwi)M>;x`mM^48M}M{=P7A0 zyAAuX-fb^>JZ**J-rrDftp_wH%W~RnlOc7pCRe?t7p9C?;Bcb>)8qr(3@XJOIyp~p z`I0{l9*H4p5?@U0Ur+90_6Vl#$~3P3pb8V?Hkqr=8p%vvE97c_y@xkJ^4$H!hRn@z zq%vY3`$xlBI2 zxFr}LcM>Zihr%?A#W?+#DbrkS!!=CzU{v>Na^`ogqk{B1x|_8{)r9Ku$3DMskG%zR zdGB)0Zl6B0(od6{tG^T+o2)pQu0Y7<2VtZ~r)bXsghN9ksJe_R2h(gAkC*egeXorf zYv~z+?KvpAV|`W-p=k%_78)^guaD#oDxc0G4Zek`Nr6eB1V zq~PW;Wk7YVW1aL6?m>tpqdavn=c@Y;RtNRLjmrj{@^p9R=g>bGa88vuD;~maTW-U2 zy|{>z{fBY?oUNHBOM!Dg`yResnL_WDUB<-n;{vDGeHG<;UvXVTFEpI#$6+)DW|e4j zg-a*xSup(v7|vQjow3Ne56Kn3(9dE3ZXWxF6Jtkm8+)~xUuNUE`xRH= zvHex(3sT@BONTK&uS&2}U6E;eWy+mvp2YY!s&gy0=y11el^B<;CfxFoC2(6O3Fggu zLrT=tT=s_#!D+8sp3u`$XZ%!9*zit;)5AXiDDaEcAO-30Ewu@xb2EocLJ) z{CYM8o$Ee9!iFNac(R%}ic?6H`f;o|IGWk>u?(%AR6%g*Gu-4bmK*E8A135_;EYlk zMmF4$6aBXr#taLAC6&rk}OlWjBtWYfp9|iEXt&7GWpxaGvz+6_#d2t^|^|i z$;1INaYrs49Wxakg+}(-|AHT!-NL{WV=MC8vxCGzDd;}NPm@~^s zBQ|qppx(}2D7|INcpsR9qF)ivKI{i>2$V)w=_YD6bONlJe2+wKcV;vWr(jKgEJTe| z=8_g$a5|%u!O0{5O>RmtZs|7Mk-1Z0WXg1$boD1x-~9xCDk^ED|720<#|t72Saf9WC9?gde$d47PhD1 zI{grO{?;urvegZ$mTPfimv;~t9D?E3n_y=IaQePc@We0~hn*e9>|HaC`R}(UUTZu8 z-b9|0HFRa3&Iv~kjSRSZYdE+6of+f%#tz+%yFqJRA1aN_gFCZMqQ)~8#;H$3;q+{3 zG137IpN@p1MMp?^_!!3GQWX~I-UUrr70%P{KzS3aA+zqK!2?+tbiJv~%>QOf@4hht zqr3M}v1KQmpZJkz209?lu*A1B#xffY#Y*=5?O^y^hWl|up8GiSz2KBv5-s{%2ilJa zS2aH!+?_AtUo~yUUt%FjGTV*es#l73*z)iM|hQ~X? zQKd0kw%@4fYqAx;L@z;OKJH4b)CQ^4@!NEd|8d+~d=0wyT@cldxJ_NokH(()FJbcGzf@{f zGQ99nhvCCwa1AnW^iCeR_(PH29D0*_{EZQ{J3NH#xfh732g5}bE5hyUTg3a3HoTm0 z7A1u$x!TwuvOoAhU=}(Ax9CPws})Y7y=NGrJT8_-9m%U`zBZa#bOn%)1+ti(mrA=x zF08NCBbKvnV8QdHq~Bc~-ZL;9A!ZgSipCU(0 zk**JNf=AH=KaI(uW8+_gNnlX|`((S$0>6epX@eyu@Gl&14SgH4m2P zy>ZYp(wd9w?j`FBgj!PCL{5MI5WF1+l8Z*9ZzS)&xA zG9{TBsQ8G|EDBtf-jIR7*{y=A*DIX=l+GZX{@Q3y7trMcwRCYxGTKzv!DnHz$ai@r z4g6$|FLq02@N-0>o}bwyOiBUr$Hd`=q%gss;GfPZd%n{86T3m~-*J+W6${ILr;#uE z8My4N1AGebfMCN#pr12>OB|4fk^*NmdNP&TOLX|ykm=O)RtT~EPqxgqWFJ|%poJ*4 z%Mr(GTOoB<35LyBNM!kaux`>CP+w=rJrcblGqYqM!lIj4Fc&dtZ6axYR0Sj56G=+a zDDL{eeNu6MEKNOd3a4wIfH9q4DxO4&$h+wZpcoN{pFOH84jw*7Ue&)Q3c4fdho72| zH%gw^#7>7OiDp)LYP85GsGiCkYZZ9--6v;%ZzWfvP9pkV#!lxDniY8&t=3EOH|BlB zNF6&U+1ZQszZgN;mjsgWssZap9ER`unWSd=SMX3(6&-Q6)uou zk`%1?LyqWNLB&Ivu;IZxH2u_qpCN;cUeb!gOX6_1%~=dBH2~$$vH0VVJw(NJ!qA(A z^i9oen6M_+CC$*7ZlCG_aU*__v&v_oZS7OA7JMMHe_n=zudP5*7L%oooA8~`kTRVg zsmqU>DEr_IX(>Ql0XJb1?t`^^KT;D-XGssl35-Iem=^)TWc`u^sJs1$P7O%KXql~e za&xxG~rPZNthtZyps`#{7-d3jY~Qf zN8F?)uHE!!T_Yt5Tgb*=OR+F^8T^ve!08oM&|Ei)X)tzxvJeTYuGot{Ij2FV9bO1_ z+ge12g>g8VFUOlFO|Z7L1eKIcVQ#E1sB05jEtp#YH^%u+(K~ zpgwbooWNHb?t@~UBm*EV3+tB0Vb*UJ^BTtDYln-tcVPu4NqkDGZKH|)-E$aGatpS! zEXPel*MphvZ{ULUbMeE3XZSd11BT8Np_4cQ|5-dkez7{Le!Y$!QRBe8K#jS5^dC)b z`2z!1Co!yAfzzL3?(%6!9lrmdf>~=D(CB&z9M|s zBh4vB|D##D@9>dZJ-sIJn#PD?cFLLKfBs|<5EQqQdz$MzTqEx*E%zZbDT(Nr%3p?MD+=??`a3lj} zOZ>W4iPwm9RWfXvAcxb(KBRJ64w0pqqsaSyImny$4qpC=r$+~Vf<@CE0|qW* z#fH7~)~-aC7k+bSJA5Ojr~eX7@c#xz$E84XO9AAYuOZ^4O3e6Nf6``BiYL=kQTE3J zD2qKsYMPJXj%fm1(DVvwN7NJNlyHPePUx)k4vxrNC$WxeV0`aems3~EFmyvXm2YZ6 zDLoyuu@^v4CMQVRa2A_2H0gw!K{#9X4UHYl#ewx!sI%-kK5vVle!baHtuhg24$mgS z*ICrTl*b=(DfsGF8xY%7tQn=$>h|Zc->77 z%|p{gw=AWAsTJYunhvVnwusc*`ih2jsW9o&RUoCkTcjO55%YU8pkZPKu`LY-5nn?7 z$d^IY`8VX$hlRA^oD}hCK1$+jT115{k8tcW#75_4>}qYJakXY7s%kQY-Xfe5mO+c} z8B&=^=je$A3S1?3n6_J923@PY65ZVnFWyk6GZf?Tb+Kf2*GMv>v801~dkzOfnT@2w zb^zpWXu&0kcR!Sma=CBUi!)F5Qm2n$INjHXrY=qur2JV;k_tP)da@^EN4KNy{>yX* zDJ7E!54*U}yH3NBW%0M#BLvMDTp-E6x!jsbgXWI|_Ov=(_Uk8-s28+QqM@W7dPG~~ zf(2!ty~rbBs%q=gv2bZfY)J%p;5~VEUZtGX$LO~e(8KBp57m!D&aMm zvtuVXoQok{O`mDn@L`m1(8PChqS5-3HfsO%6rEl#$;HXbgdg*o!J1Dall_m0rk32K zcZwQu9-l$+y&_C+eMCIe{K>VSS7_|Y2ArUhgg$R_(CGF6-Tz+`u4ysF8FT%i$2*-k z`OSxVyG5Y4I0!P%`;+b$jo_tu09FSma~H2lWBqV_cy%EXHa=3P8m4u0`S3M(Zc!^$ zt_vW>i9v)-wj*8}%D}Sn8z|CeWbubmB3kP&!T0q*vtBkSaY?6dIvg=Z{yq2@bdnAG zoryzji_6h}vRwVhC$wu-7$yuYrhEFr;MHye2@S`MN9+FMr zbUStjNW8TBt8oa{Qy)7WnB=ZVoJZTu|EL3hkJkdVo^Twl9fIAd z@9=(F3SA<20P9V zX76**{}@TPsP2{cLEqtqJAE)CW;?yxIUQ?s6}Vk?>+oBxB;QGE0ow15hS&GCnZhR- zbV=p})ZeYfP0}#tWPJ?Cjt6U?ezZDsX~j#(4J@Rqdv8(x)pN{Elgyt?s}M+iNPxGq z6`1w1h*5i<(=T_wKzDo;=qa{h(FIw!SG$6|#y61A-;c8;IYuTaO7Qfi40G`66BzC( z4Id_az`kD@=so-?ZZ1h8KaZ`3IV#WL_?su-X(gs3M`TkoOgw)fCjf}O%=FIN+lLPcTsK58C+z3oEBT0fJDF3sIB7+#ZukCjJC&KiE_V3 zMbZl<9UynYi2bBUPxP24mXPpz^kAZZ|z=)=Ix4Hx0j))-?d#- zxaSROGFPH)9|Ry3k27p%%1!b zYFVwT=rXX5|S-Tux+titNQjCD&Mkm0|1~ z%S3*}4Hw5sHfd=mGUSn+!s zdtGH0t&{L$4hSPO(p>okCj7T=-^Hmxe}rSMDOO!SG2q(jeYuj^^pJPYzrl_=eUWz` z^HR9BNR@T|tj4>doSRL2mH5y3BKFJa95$&SmlgdzFPvQE&mVc6EL2Wf&98C@;uHGf z`7w)9d9MTCT)orV#A{v;XW!^J@X~oDtV-{1exoF%nw+{N+&O=kn_b`?_CjU~|EWpZ z?V8tI@i^X`zae)_EM;2EX8!ohpL`R~exG+$!ex|jQe^ImeeAf(7iBxdsfJ(p@3(x} zJ}-Or!{<@F@}O*0m(Fv(#!Qy?INZZlYd4Ee{m>J}yN%+5J8Hz1KYp-RsVCp^{+%#i zY8?Bje-baN&?`(kXu;>w15Kj0k>ma}f%BiS{X_r-I&YlPRI>hdg|EAGFdDbD^;h2xWq zS)pg8aQs0zwlidh_|3gt?9I(ie4@6sTY2AU*3>FnavwVQZaD+D4$Fh$I|oDfDW5lR z?tx3#pj><2FEo|i{9r8KW%)-qs5M6H923m1=@?{ZZyd|p70+hfvl@l&uS!{iqpAFa zwo7d8p(?S;yiDQ7H{rr)TMgDtDiW!wey>5v9P!Jd|}!f1=HP9bL@+ z_%XAJIsU`-NO~Bn`L&R(xGv%we5KviOxq>2m%ktkZ&hXO7o>_yb2bWBB|PC}5+AS! zcI@Zn^PTuzO2zCg^+o)FxtGMZ*$upThzH-uK6SO@PO)7L`Qmf)SMVqQe&A&`#qvLT ziiBH@*9xyl_(fgbH`!(h4Rf>JQ{mp)W}(SZX}5tfiLM(Am$6;ev8=IQvGCdRK2~_v ziCwh+4U-eizxw3edsc_~np^%$$L1?$X&NY0qs+-LmvFq;@iTu8Gvcf+G=InUI zBKGykLjJ#V|B3g781ibBLN+xE`I&QM#9gWim3pgf_!f&%o7z-y(i1y@*YoHBLD0z7}5}xryJ$mx|4v zm9b0Krt?20vTROLJk7XJuX!tUI+LKri|o_%t!U3j*C5*xS4R?JSiB@Ey1 z$<{wyz%*|e&-Z8q@~!%DLNA|W*1g$)Z%qy6$NmTWeBmfII#j|mlxnWjknloXoMVMq zDq4J2VkoZ}W6s`cjS(JcaH+DWkrvmxU#gT77qHj+Mf|&o%EIa9$^5R?Q|$hrX?#&v zg>b>HiTr@qaN#F^5&Qa1jM!uSf5LAcY~3zS>#3}sy42NQ!nV5dh3Eaf;@SOjlf|Vc zq}<#usqy=!0>8oTI$MWbY*;`AuQ>6mSly;;I zuNkL^w*)Bg`Aq?m_q|4V-(^0VYMmuymrr1=GjH-Qq?0OVadrIk_&BlmVMF$F&_Djv z9~2)8y(!6QeZr^7Zxc`4QO6ccyCYnE*WUHbOf}b0WUsKU-h$5=RN)6tYgO7ld|CEzR!;S?>mm2c{N=SQZ^h^EuQvzrb}@PE9D{9q+1~=; zfBziVlqzG^q46w#YObsB$(u#|&j*qX(?E|Oo-HrlOhVaMt>JEq)XT)DzMc@8{L^6d zBa>-p#DChX)%vh1zk4%Rg09AEO+ zfN%JrBeoI5@SpcTArh_SrKrDJ@#46v2@Q+c(axB1rnUVOOfO7Wq8 zCR~!|2Vs`ALY0=Mlw11E1a?t}l&enl@5(&G^X$!IvssxNzP$gOXZ+@FT{rcy75wt- zi$a}%PmJu+Y5e51KZFH$+Js9jqWRNg7^@)F!9Gq6bM-O2%IdERXQ!)`v)R0xaO%bo ze%hyOex{#_gb8+x-KJ>92UdUOgL>+P_dGASwx0_XTmGldF4gN6KU8oKPg!lsZ)}O< zTj$nSj$UiXp8D4zzH=(Zb-9%VA9m^$oBN<&eA(h0o7L>gPdOu!a46*QY0wvD&iS=` z(4 zv}ksO*(BlmtJABN`ORaaYzG)8gGjzpR#q(bY~_yc4P}d0E@Mw>Y+)bjof5wr_e`vM z&yx3*4(G3?#0&kdqwu%{_w&6zhV5mGSxb8-w&B@MaoOHd;booRxC(s+})fR6K+=(zX#^>rLXda;W(5w?1a&+8z9z zby95RdVBWc**IR_vPj&YoXpC6+%8n!p22(V5DL>Xz4&#T9ICcMEj#{l0RJm4ocCxn zFk3xkGC=Ld~X@Y;3m{P;F&_ICdc{?=b}K3VJ|9=l^(mAc~-VK#hY=LL82 zdOJ4rYPnZkzpM`8*UoAZkN9|neeAl3-#)!p7$Egf_(O0*T-4~p{`r>9dPpX*lJy;2 zBZ9T~(O-3|?6%DoKm2)uJ-FsYWyIH6tmUz}j9bVQ{>nC!%7OA@tli7I{0y1J?9re? zabb=P>mI4>Hgh%Qx7*ASJ`cXkdaTOjTRJC*i+`@+%|{iBlM`(h#rGG|> zWfK}Jx0#&c`N~D{2 z_YG?yJr^x@@E;|^p_}1|5q#@`UHs_@ zNqp_K_iV9yG~ZlvhV5@F;=7!myUH3?@$;u-yQV1`3j-WtSa#TG-a@kX{=VBKtVuC* z>l>TK${WhCclY*G`jFKs-Vm=MBW zd=?@JdMM$~u>nG>#bu04_cGpNORRXA%z4+CmtWZlTO8TA1Mh{?cYR}jd|uA3&h+BG zDLoL&mQ54h>X{|%A8+rvwqMTeXSF(iWz`mTes{ch>6lr3FyOdm4m4<2JM7?w@5}CC9O91Ma-Zj)THa0ha8J z%_rDb1HpW>dL8@V{t;d@X)CK#b&ehQ7A`(JzEHT`N+7N+E_J(fOpRw5LYk#Db|@GVNvvd7aIJJqU#Q%>V4yq85*`^i$aJ> zao*>OmW)!OA}y5)O;LT584;0eB9gtbIq!2L4VBq4LZu?5lm^Y;{r&5o>)dZbF9nXlgiXL@9zQ8vrW^er!s zeu)xSZe0;A4dO=?EpI&GlLWaU;;?$e6K;x~AsNT^(Gz9e1TStu$NE!f?lMNMRbONx zT|8*`H6`p&h=ET^dSK>q31dqdaEEOexIDUn0}dS!I3oa`hJ=%$`CEz6K6f;-dQJ7a zBjMBSMiTYwH60CIBz{HQ_Dx?wYv1<32{!3Yz>O}6~oaV zLAa#;HWr5mlC__dVdSVHo@&d?L0(flq<-Fq5GH(ir~kadD_MO zNfTc=%Lp}-({{tuSnWUba-NABcdmk8!f(j>sR*|$>WG2dU*f~d_3w4)MAb*R@a{k| zc<^%Y4~hg+=JO?VUbz!&wi@Enz7TvCzYcy`AEYmOdf@bl3fLU|6g*nLklCi*SiiJ~ zhTFuG3*vk5v({mn%$h@<^g_6E#u)F`$)TM?0EkNW*B$ENeg5(NxM}S$7RRW;+vF>7 zI{F=43yP(=>iTBObyw0Er*F_b$NtiKmvQoR_hM*SpF{AFI6d`Vm>7>qK#9~8@OYX* zqGor{jD!796()o`Lsw(?)OPAHO9tybqCs=10G%m(0vn~1@wdxNdZct2im%YatGjcs zcmIFPt5P2%G8?dWS2um2_mUi0Cd^$F-AU&^v*&mVmZOgLQj|OqL#_l~z!z7aWBFnO zP)y8$y?MKEe5p9Dy5>#HvjUix_svnK`W`vBZjiorQo>}zH)Ke97HY~>5|`7K;)Z#^06iw z6~^Vrs`Q;W_Oy6Z83fN@GG17(TJGJ9wO^MO+%kUmvPmUcJgGmK5d&l zAIA6HI5b_B{*zO| zSD8z2RaP^ECQgBqkq_|i_RNVVe5R5p*RBf3el0?YOW)ZL ze}9Nk4W=5Fi>dh9Dh*;FjDOran(aIomHNLA`N}nI~^2+9$LLV<)5G zLDhWD!b)v?BA*21hPTWv7;E9an~A8~ri6mV`{}IP_V8IV0Ia;kxJ`Rn0XwI{_Z3UQ z?8Cx(zk%8Ho`p{2VSo+Vzg8n#>f2b;xsLd^BopKsE`p2i7or&Rgh}1|0!1QQsr|7o zvkS4N*#27pUbek}jWupCTs?zh_qRDDKqa2<*XO+aKAS74F3u@cKT@aaUI!+!|53ToC?@%s68G29 zDR@1ykl9&o00<1Aw-WI9c4uCOyTyH z)sePuTAx}q$VthY&pQ&XJqBbWL*R&zsE}ui)q&UPoT@ zOpe_KfOB*1u_jXWOiJHl`bKOy*eOoI-1n<-NkIv1U-=De@^?XecQm;{RzX7dDMmh& zr8hl)pkGNU3CTDQjc^ecFIo!S{ohDizXF^niH555ZJ;2TONo3uSi4T)VL4l=PF_5W zeorK0UpFxS!D6U?rp8&USxB9v%J61F7zmn{f}8Xfyds!Rge;!(x_HKDVZ(n#5{B?6fF2^P18gE{D7L z6+)ZRCR3wy+6_l z{CjUPs~dV@m1q|(|7t?RcIm@Fe=>f(aSqPEPrK|bUF zDslelrzv{dIVd2z-R9#2g|Vc)nK(&4WL~&QyPl6j2~r(Qlw2cP4kOK@sljT7#c*i{ZfrOW1KZ4qqP(z`EYs zYzpt)GyPQ$30oCRqnEqTB%RqX_gn<)o5kR@*^z8!*HSRdi^M7s9pJRB2W6)nm|l5> zycqW)zv|oBs!#fuwsbZ$oX#U6mp?L*Bj+{=ae6R#E|E17TW|YW!AEX^?`z_4Ot6exz$3l{jE4mxfx5X5u%g1I)b8Lt9MxBX<9>Hrs#s1grWJ!+(-O^SFM*`T26BJTIeZH(I# zKU@pzfd51Mh_aWbt!NuI!=` zl4tgiL~#BDfd5RoRrYUJ>^r#IZHXZZcZe|Etez_P6trhDw?{-tH8d)qMXZ_ z9?Xo6>8R8`PH#nr5vl$C;C}BWgU@18>iMRd@+&XN}SEGgpZ2mS8CO z2xJ+aCvEF;KsZej)}As4PmLY$UD}Xa);t?FMLZ@oYbL2m_DqgxdK!33%A?pG8<6)I zCvS@zaY>m1TBIcr)%;%bzq`W(wS05KeW3%88SIA&iWEVd{cQK}6s%rrx{? zUIvZySl2Z$fB%?%yV*qBbtAy{lrl8z<^8W3wZ!~D8Bra-N@Y)s(gyWPg3&nV?E5XtHoZyXxNw0LW6ia_WbP?^l;}*W{7hXET#{!W4zo_(MBPz(po;w zuB2?(TIPiMGVw_EL{pP5y<{z~CvX0LBJ|mV*Hk4tcFXoE$H7r*I?F61kW|g;f>uI@XTEdHrI0LqXlJj zyE(zt1%X&y)kdzD^JCimUYI}63p7NuN!9HawW}ApvGX72Kzs63U^!gcx6qf`)%U~4 zvAdvq@(Mcmp9j^umq78I9i|zpla|Jtc+j|R_<0IL@+x8OkL9arYROUT@*2YeF(I^g zGXqd6A0xzqan8RbV8wZRztAP7Z{K8Hif`)J;T8#Ena&_OQ- z41FpK4U>2;0 z_8E^L+;;?xE-^6RoCzUIJV zd^oezj;ya8L~FGXkQz)t^VJ&M)5c{%SKx;Dz0N|MTbB*y%4J<%1QP=dn{Hr ztGhMb7DG;KfszB2w5T}>^LZW7eM1)@ipinrE!wc*$0h7o7liI7g)!+zG-JS74x$%& ziN&e2(EC{e7RQ&61FLyh#_u?swvm#3*N)>S?OLqSl11l?AGl-b6nrTgjo$ahNmO4R zE+}I#ZT+`9}}c<~J}4OkK&}WG&cS zw1u!*3`uYff^ojfxLq(5k0mwJ>thIJBMRW+T?eOy_JezR7qx3S2Hy7^(BhUKnPc2Z z9%>g+4aq2ydT}O<{0ks|WY@wi?_cm&Y8p&U*MWfR73g8UpWT~(47#L7!Cs7qwa)s8 zZ?m20F5d^__SQV;4SY?W7oEeAg1_X|%NV$LJ`7gfO~izxI9!mS42#!%qEDv1V;nR+ zaL~dY?|KH|>CR$C{jLVD54xSM7M_Fs4x!A?2m8@$GL&BZYK{w>Ycb=@7MSmn3@eSM za6etjK+Ta{c(vpq$d4Cbqqr@s<7U!^rUfuHm`iGdoKffdEIOhtk3ZK<<+j$Hq2f8k zMBC^PIkIy-?;Rk(aT(IWo2;RK0>J;1{bW>#N(YZTv_*%a4nX% zv#koP-}pNoSdfM7)kWmu))=xyk`F}gDUl}oqjgS8U2w(2KkP;ff_-t;cuvC!2QkxZ z>+xWiw7p0o76ieC#d<)G6p=x0BoG9#5+@~J>k$B#5-#&`tX|m2eTK~C1K&M-5}7+<8uSM$>!YU=uzEIl3ZG_(D*v2 z+P@*8S!dXvBd=-L^!aFWJrP7cNZ`kjX|Qeg9x#41Nd0dJ!;%#>`VH}>#1)e{QC2`PmYz* zVXt%Ooi2~l_l=>Pq6mf?1%z?Vta7N`~ zGKrqiPD4F+fNaA_csVeI-MK>sb4xxl3%jqvkA5!6lAeO!_?;noJeoe_zYJolUgGV2 zawxex+O%YD78ssafq-wrBys68=6!oNPS!fYdwxMqlE-W)m$62Wtj2Gz*25U?!>U`) zVOoA6lTvR<;Mf>>p{NJ;t%2nDX*r^MBLRF)3xdM25vbfJ#GcbCr2C$E!;w`r==tv- zyZ`L zIlmt=;;%jMuiFaL6#h(pkLZ&>{#$6%{v!BkmWfuIGN{~T6EwDKgRMK`@sB_uPKoux zw5Roq+FKud@Szvx$a8Qam7n!lrH(x#9M&YN7<3$i8Qayz$;^Zax*)ib8idZrwzgw% zwK@~qpXzbm|2qkPE}Wuml`r9sW(^7GyMi5e-hl6;&p20cH`dZ{UYDsWD>C6oG=@ZJ zX0ip^+j|kc__bIny$3YjUm`+dnb5f52e}ZWj609rLw>$QOr4Yj$y@x~mvQ?bG>Ae4 z`46kTV_3EK{G6ctbR2)tj~_f@VZ+&IR%zY@6^#5!oOZZT-M$PEyrYU6x})KuyfrqD zhA=57E66_ozp&=tJo2T#-b_~0h?Tgf3#ZC#@M35=Sm&kTKh^*5t4GPY(|IuK&Nv-- z>jw#!6L9_>8{Fl-31eQ>;>qwdsJbXiR~-4yW+^=Y{<9-yB79lUys4Hhc+4SgDUrl# z^Ce>aAq0PN#F5@3 z&834P+^(E&utE>m`)R+3%&jzP+p-qO9}S2MGNBqNSLjXk5D-4Lgj&R=&k zdUsvMd)a@;ohQZ2_JJAP2{47(*iHZPGcdZO2RI^E==ir{THxHvD1YPWSe#|)y7Dt{ ztJVTOy!i_kvn4n@8yEieru2z{KdogA(EQK}XnX32ORtBbm1iA}-%MgVUccdWpi6QN zm`~yIrF@5;Vmdmv0)@GrlazY^H8+?Djtyy1$;1* z{gt)==B9<@9(f{4c2aL1g^iMnK%>lOW>2PZaMA5f|@Fc;~nRH|>o> z64=Cs8RyW;Pd4M5ntUt}osUf>WjJubpkHWDf7Xhx4aF>Ed~i3i(0K%Q@>F9@DMD=JAU1+!-&T*HMdplObuCtf$g&={Dxvz+uV-8&u<$y1(lE~Q4 zo7C96r8Xo_9UGRf2P2V%FdVOsscQ1v0hUa(wO?9J!&VTFNK54+#1T%OO8bO?|8?zq0 zBJ|oyUGOtMPtD2>P`jNEKqTiPx{u_cN!}`OD~w==1*7O>!6o*!S|@MDvw}6-RPkZo zY#e?RMUOwQ0Tbcr+@debq2w(W?Q7T8HB63?zOH^#AIk!;)L%%lKYHN@>7|f6a2sWd zrsLJnD11=!1(uuWrwh;SP8 zRI%goHl83VJaey$G$p}Hf8(wOEMLTT=J{sQ; zNK=m`z_ociUP`$L_LvKTXXqO8W%gh4#-bBP52X>YcV=inp_y}D>IJE957dZi+lGLl6o(O)AzcK%1^|Sm5H`keO(Hc zEL=u>+|I+au@pE=E6Ek5y{Pte3z#}hhl>Xiu>QwOwo1>F9aP98N7N+ApKb0m^uv90 zZR=sxKKsB><#Q@MwG&MXXTa}FQSwaj092F+u{Xq9!FxJS#kWRChfqhfiJ|gv_raL_J86ho4wtO>hSMo_~pKJ|BPrZW8#d&Wa|h%q2^z zH$q*&9tdySiRv+->08%#wBe8%^r>odo(w8-JI69eAMbs?VkDZJPukD0 z>Wb9&3M}%>C(oN#p_Tbl`ZBE%Bs2)-{thNqSLTzedy;72idAsd zLI-QO`55lM64q{#=L-BRho4#oI96l~YYUy}XqP=Ma##V=GtbkjgBNgiX+0crI12wc z*3-U^{M5#PAr0T+AhEBpo~yADbz~giPelv~4$Ft4wX3m)UT3}HwV}F4n7+Co0Ac^C zQLy?nIz?+j*@gt{7IuoZ(b98m)Mn3X)Zy2M z_U;KXUo#T^eRN}=2H$4BtUU^ORd&D;F2>(MyfScQ%$?`ydW&$BYMMbhJ^0}0`VTm#N1mG5Uqx~0{dl+Z0tr`+hR@0~;faDP-ksWt zTYE1vh2pchM!PpbezhXbT7QvDdmxSPrEd_IS8bp%|1r@`z74GTSL&Bo#}wq7qGgLI z2tCP1*^(6`eBN99@16iXDD#)>e5K8J-HBxN+5)hr+#4+{ok3-_2a1Udf#|0wSp3Jd z&S9nz3>FH}W{vG6>~R-c+VqXy&atKXKQ`1KD9u70y+&A=5COJ_3hN94j4>qQ8K{gz z0sp*s)E(dunU-d>>^=)uUwShoX15`2+IxH!xDcK@`amcfOr1b$jaQD-CkX0UpO+BAQPS9}b#ec4{P*)VoP1rygY&E$JeE>k;{;UV_Jm z{8`VoKe+Q1g~7OS%nZzc(T)uG<(WtC8t(>m@k!dZ?FqiGaKx0IGpO&)WvIKYg>=+k zfJ2Mt!tLl>Fpjv3v0HT^zNnqHP#2VW9Smt3rf@Dj7Q>mUBL7NmGc>2nb z8MUiGr_5Gz^y380QVoD@><%2A|B>~r6@=3QlQiBh8#XN351w|s9n^Ii%@I__Yn3C^ zXZR8x;)c+KSI}J?4WPPc!pP3#`}(=nGW9xr4F1b1MpkoW4+7y(|>k#|JAS z0?Dg&cd7iAb2KD7jj>$vgD7td16kK(s+CvDD4Y$a>-s8Tj#?Zf7yL)JO>e^Jqv`m& z^CO-9aD+@7mqw?bMo{x?14yR_!@P;RF!HmV#uSx-^`JB+)@87w@rhJ~Pla>;?I4)1 zQ8%0E=})YBV{m_zI9LDV3XD_E!l}x4NmBP0%4c8-d+)zuiBIMdZY|rT#Kb4 zp0n||iy&v|%jtM(rz~E-H3g)9UdK5!6Ji&)(>S#-^j_f)$^$O+$e&6yaN7xbfy1nw z@e)#*@|NBaYJipT^SB3M>>;I_k5lOPiQF|mL`pFs9)6I7L*dI>? ze^sK$JWKfY!-pqLYdrk-VlFu( zlt2yz6Y90{CQP2v26Z(lkmKH=C+(I(m+~o|Zn6>o8!u&~bS~1_vx9Jc1n<1)j>7gv zfUiB0c-f!D1vyes)MyIp_r9Y-{7&qP-BR^$R_W5p&swCbVFN3lZHx;n_fzi!!QfX@ zf@9^peITxZ>8u{7BNKAOsOdkD`}Yeh4hnJ84R*jpX)D=P&EspQOK=5NUZK8u-KJB~=Y6fA5e zNkYwLSQM0leAlkhsaMuATHnp;SLt;UAvy;*aTj1#h#Ll3hVd}V)AVGnBQ!T$#kMYa z)aknj<2{|AFvR-~?-o+6ICZ*EEFFsl^ih1CEqKg02f{%sc^t-PBAVomr{7-%6W~K7 zc!6fhGhkqr1Gq;Gl5Rys@b8u3p1gbw*JwqeZr2z!8&zhHN#z14HR5sSa(s2Tlop?J zrv?w4sJplb=S;f}xY@>MC7B9iOHV%%feXd({^Z|Bm%&p%UBnFCZF)&Sg zI)2PjB*Jn!pbNoJ9(NRvn)(s*2O(G-C`5Md)F6fNm3ZRDERJh;3Her|OM6_?*zG3{ zK-{k|d~S4!ZdqpqP3NxQn92#%wTOW%`)Aa<)fglzJyAF|8+wIAVSUA7YJZ}Eo?6g} z-+owu-I`;t;7}cz6Br6@rPH|+AseaZEfZFyM27I`+k#Nk6IA?{MT5LHpo)_Ir{k}FeJJBf5b4$q>#BDL|rxBTwphMd-Oz1Ad zyR2=K7$mc*v}500@+d@;LozO4|HvvFR8N30<8Tmk+=OSoO4GhokHBw{C%qRF2$PHK z;KHB7WcHm53_CCYvr8{xaC!(lj1YkN2MjRfzB+B(auI!ko?)lhT$m+%7GuS>)%Kc{ z5;!{E=ZJuspD~PXn%qXuXc#xeZjEHF;)$1SE&WN)}2)n+2dwvBGw^0|L$6ZZ}C zIHQ!RC-0!pX$Tfky{NM@l4?43;Y#Bq+%iV*eVp_ z8-m4=b+|Po2KY;!P=n$y{G-UzPQAE@yA`DCKgVX*{~A6An)Q=}Q`d)EP0Hcq6b-Ia$E( zqQ?H08;B-T7*d^F47om6p|AEYeGy_tl`SjbfOrZ(ssz4!H%cPJE(0WsV+K6M z!;7}y{X04M^uuYhY-s@1*btnok;LqU?&z$&2sfu40Ii8MOeuInE2FC5@VioseXT@- zmh-SW0dY=4Oa_m82%zVigBT;ZS)93tUo*qqyU2zkqB!M}GMl#b7mCiaA&%GQp%;(i zJQ7CfSG{a7j?6`FtP=X(lULU55@`RXU_L+EU%omSdoW~xU_L+_ATf=TjlEUNv*~6=Uv$#FV zQ#giqTAB6&Ly*QMVrcx7gnscSo^mp{SS*EpE+?4a6%5I1ufp}MfyCNdfZMn+1^8Sn zab1=vn`7qAe67d;535TcHN1e+>nu*i4m`uZCys*fkA9kt8v*?Ux$p8kz{ZF}@5DAj zyp0Y#w0H+9FXKUVOCG}w9DvwoqFnLR08G*Ch8uB+U1jG``K}cAtkY+5Q!@_b_wm!N ziDJBXtBY8PY(nkp7MOB;Czbh{0=Mj5v&x#^%tD7Mpk@AiXpP;=%yp{3%{mkGvuZx{ z+YXR@7FCoicS0AbXtLrBkK2k_fP6FlK}ihHGu1gkIJ-835q}=&$(z9!O)>J+Z8n^m zagb9fA%F(556A@HCdT~3J+i%O2X2ToMXjj^@%o%GdS5016MWQ&;^*rau&$W-dbf`` z`CmCGcvb-RTOm;kA`Mcnm@{9~@b1VBSd!m^t+%qxhJGDq1rV>Y<&*v-*Fk{@ z?)m~*1ur1IE)rUEVlh`I5q1^kVduWBjK9)i`d`Tai1GY7V_RgaAPQ<_TG zg~y|j$XqI)a0!hayWcPh^lA&qKj@&Yycj+8 z?Ts`=$BMc0^E@@G_(iVY-3A|08-Oo;kobB`!C9g9FmNfK6*wCSGqj7qP>IFmQuoMn z#iO)9Qi$u6B*@Xcd=;~`qsS9EE}q)(5`vzO;ICKCD53s^^0Gak;(RSJ(+EQYvtrPl zcL?QLUNQzYs_@m;8eYnm($}ZNF{?dkr4QN zU5)zg#qcZXDWn~{O3r8tb3XqK1oeO0QL)jC-i>=t>mUkGKeM+Qev*W56%aDIiTBL=fYIIx=y3GI&h9ZH?`}z_dF&=f${xxU93cRaavu)a|A z%ZaDinSWu}BZ1W(T8fNu1o6J|nYizqG>iNp0BsK~3BQ>qT$Ayq6N-j(w6B$_Jw93g zR63GeN?Z>PQ;^Cf?uLh_!%^kFG+mLKVD{)<2n;&uquR?D^cou=kNOG;D!GBv)BAXD z9#8*pIR`xSs~{1t(+T|&B-)8MSG*ARuH1nW!}$o!2W97cach2VcgN+}P;=E0a!=rT?zt z%-qYcbf|E;j_ZbM}_m<2|mVTL3p zRqrMW70t)LMv0&yew_@js-nm)LaXlPd2pyw}b15PB3KkF~*yg>yL- zDM6U|nCB_HF&*xx6%bY+mV7(N;{(=|)L&T@LAGZOVfj5twtYyS*{Kz0Ho0{XJ_sow zBF9qjsdWz~&RNBbE)0bQ^-F2}kwU1NTm&m69H76MhYj~Opxwj!Sk@f^HPI^wYaW0P zRDH?#SALHB^e)(WQ4f^!a$zF06DzXnNl!@$1iyPq*gKE$@X967B)tm{=#4TjcBjHa z&u%meO2wQ)8CbTl9wYlQ@I-hdeoWtr?ou_l$ykg;P3@#3*2;KcvY606=RiR^3GaQ< z!kZ)Ur2EVWw%t&GqVgNO+D@J>(<}%=YJ;iqpC4o}&hm4k8yy9^sA|-OwV+(-y4Nq;;JxU@E$GXbfs?#2`dNM>jprn( z;P)pu*I}IU-x$LK`_Dk!O5S`MRE!?R3ov_$0o;6K49i1KlN0f&l53VS@hPyT!!L}wOH9g_@*!&!O z-tP(CTabZeLLceww<C`5OWJZm(GKEU!KvWr)(hW zZve))PsKi~MeNKEaX4DGhsF$G)=uj~XhU zL#q>OuvlLWbO%R3+WjlBt|KscCJiLr|IiMrVWOl^OfB~ABUgmq5o=`|5bxST-<@5J z(MgZ!w9gme_PLcfLz=_YvUp8y%$9=4&TN45S zZ3-mbfX4+LGKR**ugwDWW#}QXCsb|gC=}AmR6U>=OBNM_-%n*6W#u5!NDCe66u8HU z2)sFbm#C;zP)$0E^CV{&IPPmP`Qb6}YO$n&<^y5{=z5!mo*+nbtYe|4pIyvw}3p-1e;#1{d z>MVNx%yo;l~X$%Uwr@DL}a zn4rX-RG8%$OhV}@`uy2JIFX;n?)vEo!s`O)icSUmPxuIE9zP7rqC}V+zkRg-Lj_%S zsvMvF_Juzu+OYSCDP&lFW^aq<;eO#(Djj#8HdUuVU7j^a@~h!Z&LZ0G)r>MafL|^t zL0fS%-0*iL9?#!n|Cx5Ee?OIEt(lF@-_L_efgqgpY=Q1dKN4Fvmnev>>3uEEBgA#W^o+c=%8sfW;4P@H9v*40h4kB@Hs4ejalnrD&?q5N{qmlJn zTnfm#FIDty!dqsSt1UXH-D1x!>&2#@JWi-Q5vR_6OH$|g^Y+P8#I1%)yF`=7$U+6q zD*uaPmm)Rgn{XQ$+D?ga5+Accp4pMCw4``uiWt1~)%% z2VdWV2)dFWM&!6}!W|(dSsdm!R*`9cnm|5SoO4n^7Tn*= zxe0NQLh^)Bq?n~f1^`+5Eq2{f{eLH?E}Z27vWW=ao&sopMrR;MzO ze7Yt8Q+G-6_Q__VU~`1bD|v=qe{NxI>qq!CH5ej(>yiaZ^{mqgTXw-N3F1~55ARw` zx%YMck!R8wbiUjY6rSx2cf!9h@m?2@I4EGPgaJO(OMNTD7C`k^nQ+&F;(%)d4C~c9)t&4l;6*3sAn@5d?O6;$W9BmmSwY z^)PGv=C%f-bw#;DOA_llt9jhTdwm+?cm{VgaN)>@!|=s11LxS55Q(NOM6vw@O@Dk2 zQ!SS>-_FgfyZ?L((RgD?f6V1Dt~>L=tizdj4+wHkv)6HT^J}_y=@`5U>BZ5?GMY5w z7M+$fjGxoPSgC`-@N#z|jI8c~q4?gY)!MR>ZXcaESeXXFxEji#14?d0336)b|8Ip~sf8~(p zatc8IjnkHg7hokL#&PVrN7UO2;o;VBACmn?f^kG54YPmiV{O?eJS*dkMC<3kwk#L)x|{@U|MbZ%lWum_kzmT& zhEe0qnb6A1=y0&VPU%Dk37;_JcAT`r?qVxa$J@n1m$~m}4u4mJp3|4~^kSks=1XeoC55`TR-B5{~&4tMt!qwOCR$k`Z5XQbAV4T zvw0i`zt;^&l6gus*AMXWjE<5mvc81>uqzyp$%EpOjZ~?1k|AeHfU#dnrgkYVJLmBS z#!8FGy_Gps*(L=WPs`9@PMDcdVK9x@Gz)+4XhzrCpY-XS40uSB;rIIn^jSQYhnr;J zWg3Q;Qc7!`(?`r)Ci-cDSu*xfo|e>EiTSf@kc^Zlz-}o38RH;WC#y&{ZmK0&!X}hC z{}AU67+~~{_tdlS4E}y!Lj6lN!FVMfS6au1lqdz0nP~%LQ4rFJ6*pl669oGGo0%cK zcFcJBjjALUl53B*kg`${F5R6%2JHNB%AXpr^<9GJ-(*8M$pDc9fw*r8(t}rGfG_Yp z%n%JHGfe~F$Fb!cVIEeUTe1UAoN>iqp1(#n=_}d$*94mU0#W@Z&x6po1$Vs{#NL@Q zL~=tg88TUe^@;7E)Z+#=p%i8u(BlpsPr(Y;0Q}r;MsD$PX>cL|{eMWKZt^LRf4!8K z$rH}%*lnP5=0#xj4t{L9VL%IB6l3>(8x-5$On)rigNGZYlXY(PFwd+WchpvarrlVl*Deb>tfS%_ln2uc)u4KJGgkL8q%M0wWo~o*!edBR~uM+CyoUa52nzUP`ddS~-$ytf!3>0AcnhEh~x&!F*@o$!>cA|glfS>*%)p4QhKVjRu! z+pJ))@vA}0Bz>|!ikH?tR)_&Q0yJv(I7-^uu#AWR?_T-~6U+z^_vGn!FBZawCvGHB zWD9I`b_e(CD@nM-GWt4wHB4M&@ixaFFV|(&wY;^iukENNDXYrh!;)XvVXsW)HC18F z&67mn(HpW&Xa~pDV-OU0IZm(o>OrwO64#e&&?kNuVV_GT6wbZIszr3sO}|3%5#}(q zI)9lo-2&8dk3n4jy@5Dt20qeV7k#BUJFkIPm^Vaqsngq!3HmQ=BGn5@ z=O9)<%qvp{qK0-U(47uBG{RGG3QT3lkS% z)0i+~aUuW;K2HWFbg<#Nmwb8C;sX9~*1=u+K{ca?23<26>)A?s@Y3&f!`uL`h32?WH}nt!PQoj#Mg54dFiLpqI3hwsvXL zAWFO5`ybrz98j_dMMrExM>z;iG6G{h2+j>h-P*(qPk;0NJ5!($n9!RJGncSMh#z z2KU_hQEEIZLmK3)Q)QWCDPIr2I?!v7rgYAQ@T%#%_0N``HLp;eu5xz7L$6BHk+#xL zn_g61Ci!gCoYR3bK1rl1F#DOr4Za@o?*e%8S!CR%vf-mPQ`wQ~CMGkgDZk7 z+uMN++4TWq4GvefZMZBgKW-NIv%ozl^nJCo@%%06e{1yq``tB80wLWZi9APZ}8`pQ&p7L>=TIooA+2A883PGl3M$+<-_Q1Mc7pvC2 z-cV)Od2ZE!6I-NTymm{o*PBWMdYqTO*eKs&v_!G;!-?d|o+oBjb(LCHY1n@Y2zYz0 z!ov4$Mf8z|${AhL0}rWml@>n9s%%W$Rn^pGy!4Me##LQ78{jdtT6)uE9oIFCuF{*8 zUgUH89fTN`U@GAL(I3m#Xbi2B_7sUl^SpfCV%3=-wYCZg`Lh5i9%g zoT9(7x#m8&rMDh`+IX7`mZehF#4t)a$#n6QH~xzLOylf4Fe5pg6k7VT z+Lal|w*xS#_y=^p*o_;SPYeBKUnzgnV;dEO7tjE+pZ=Hgq+D};10~sxYH_e z^P>!~Mp`H9HzE$wO$Lh}Uv|Q&CIk82>rni#>@zJ~(+Pj?c7RUX57FOcRl?UNo%vOy zJ1Y1o@}^azC}n|sUwVcC++3YXhx*!y_SgOjbK~QA;okrhqi2!l!*}qu^$Vr!=#1w- zAQn$grOV}B{BnLZ7_2=bTX|yygfCa-`?6}7==qvFlM=;EO#^Z9vz2^m-D`1eybGs{ z-9)pKG_d=!&XSI)`Z#rtoD(_bt2lSc2bs?ARN7Ph1ct@=a=KdqByDiuZV$f+zj`Z* zS6?j#!^H{Zm0|?6$JCOFTeeub{1JuxC+E(tsfCP)p;%0>U=~_q$Yd8*xW7d_d&Pzi z_;sSSr($T*v3P!#@|W7eQ?FK%B|qE)EPIf8_I?sK9bdHYwqEp#|x6{A^guf`nvv%(E2b67nddS zkM+jvW$Mib*ZFhn;lB98<~ZpJ;dn2(0_Jz=CVY7Cn!0!#h2`gppf33wH5atd0!g0G zvr`KUja25@@=aLpXKf|}x0B@E{8v;`Ue1HZhKhx;7vX-tm!$h7OWfRD^c%PpVd2GE zxb)2gzt*Uud;A2b%j=LWZQD%`t=;kVhoRt>5DwQX$6(9xKGbM0pFlKLaE!ZvgHbMAwUU(S)~`$y22d`kBDr8ah*A5Y^IOmVRP6*2erN=VgI z;(`Cwljlus-ky9|=y=(QBgBJrzsivBS9}FrSVi_TC(?ptBHgj?%BzQkWAR2e3|;#f z8ZRw^=VMf;HMd-*B>N#mRsDe+TXR;rq{ymfuH>~fkA~X4hx{KsM6J_ycRb%Rv;|a z_Hg*05})-cgODw2pr1=m92ujIb3dEo+eV zp-S-XCW8gV0j&ONJ+;5pq_fh0a9zck8>0+);UhP+8kt7z$$+QpSmT76R*Gv zs%J7*B*VZXa>iS`06o`);`qNkus6;W?uQl$j#jB;qmwD{`5>+vE9d1s4q@ljb42Br zPT1$2HcyXRS3Y;lC5nIK#F6u-p!?G-!Ywv9xPtjexy%>c!^ZH7LoE~=%Fg_t&JtuxDA$XYG`Oe9txAU>RX&lf;gL|9cZ61tyeav;D?{w)?F8LZ zZ_=|Kk7V~<^MvU>(d_zI2V47h<&M%BxcOXf?0Q`vpH3S`T{qaGw#Q#cw5bBsxvrQ} zJfF|CY!HkuOW5|SC09-fK~2Ytw6#JTe|Vgs`{xqjyo-zsceQ|YnKHkgWQT#`Ao7(M z;K-ICxNGrT{L~*p!Sr~95Sftts%Z0Q%<^H)`Q-@)56%vHz?iQ z7rowRh~BFJr?@vlkeLH{Y4xMU6;nVMaTTVV*Wg+15tM9UBdH$!ScD88a(TIzdUe+4 zlM_PlLEBK?-!p`AHd(UP$Y#=ZaK=&h-cfB1;2^a{l2wH}pu*`AoryQc*lc&|;%)-< zedMf!?`fdwFoJfO?-Z7ntOdz~USRW5!ul?$s1tq$y3Y&3rkO^NH~ay3erSgSGq1tm zr$JD-{gAlq(u?w>xD`+uHi!p|Sce;w-hfFd$O101jN+Vq7W{fwG0oP9;}whD(WGd`aZiyE&5TmQ+ZGe;CVYk7dxJUJpN(pCIWfdoFJ@!+BFX^N0f>Xe}9t>#T>t@z7}~ zhR^3mE5mu+=m2&(c!QF5`hxlLUlhMd2X0(>Mg#R;!p~cugvtT;g$LWUFlCV@f3wtw zgeOt7Dfou0`#D8i7Hf%L#(jdb+2<&`#TjqFPq1F@f;EFKz*cWRyb}_Qrs0oZ;dc%8 z*=LA(V@KiUE=OolLl<5eJOH(7w9#&YF)AB6;7w%{R%tsVtc%)0VaH2LDOdN^O-1kSF8yr$C<{Yo{WQkc;PIzIB1Djhshu3Mxpd1C5H!uZ=eXk+aF(S5KeguQR1HI*`0 zoLW!LzhA(AW(CyqhZ;I>u;c+dizsf_=JHvZn`-@AhyWsX22Pkc`AtsM@<-YM5 z(EZ$Hn$qzLUb*(<%M&h<-u4;t{M~?O{x}1Fzq@kaCIzs5*@=>>$I$avgL&VEheFMs zJPNqsf}T&ascD!EVtx!B8f3FT(ms^RR=tm ztkms^kKc~u(Y;ggl-eNdGZ^uuNfq_J?S)6mhoMD+sl=`>POxd#Wi`G;4SiF^*$c8@ z?i2Zc6sE%jV;B4$ehV`G$eBNf8zAaptLTte06(?WIQ^A8j@0Q>`OY&!Q(Ph?{dMNN zl+%J-ljE`>^RdhPIGB1x1vTW10w0}Ql)Fa`%9g5dkCEy$KsS&Sjl5YcgVlewP(#dQ*rBS+8@e<@pos@ZN#byqMGg)6p$y4$!&q}r z0B+l4#qJlr!%WMUkgWcMd?(o9e8((GnsAY>dd@?~>}L3&*BAe(t)uqZZa5^dAEqq| z7C-9&%^VVd*7d;{w(b+$k+WQl-3?j)?L>cX_(`uI=2O|C`Pf^6+rxpfbO?v7-| zrJsb6su8@q?kRb0*bPStW1y%lj7Is6q4%e|Q@gW2*Q_@MBRdiHe0fc=YS$?$IbZbM z^HCf>#1@a5|Ag-sPQfd?IZQ26F=h1>Y^s)EFAU?REX0W1yQFM{xYD=>xn~=qzM?tG z<~)Kcn#0)TSRDQeT2H$cc1G2xXiQL=$F18&apfd6p6mWW=vuW7aOCZOh z`*f=JR%qCHnQ}~y$tL%;7F;KsC&9Vc(4Fq z`hZ?oI?-KxvdkJ;6(1-u4_&}kpnyhhiA)dN67ArPs;DFD$lvN%B zo6Qnw;?YNtv&0eRTc&{RRvUEr=awnXar;2Ku8-guslBne z(icOMu1U&1YVvqbNB?i_!$ut19j;^HV+3#TU z!IhHxEeFZOc@VqimBX5Uj;MXfl=UwW70KM#=-GaV*0p5)hf4UT@elRQ3dg*~aqJ&E zkcZ9EKFZM#PwHqGGQ`~C36#kuHfHiH`N zC-e4W$=K>0kME3s!Q);ZNM-G1`nGE*-Tc;r8=o49BSSLiPWDzvoM6vGQr&p@E=#;o z&;(|gqrhGXczC@L#7|M>Z{}Snw0bDc(dfBD4+g`V$LGOM&Y^QnC?=DwcVNOtdH%3X6+>dz3Jqhl zu>Y)6V*CPSEd2Ne_SUR{8J?c(8rKgW&btiWRlV_JMHuSVII!7TbAB@6D&2F_!+n4L z(6`m{+Fh%Ou0Jy6?M^K)FuIM74fqXlGRs-vfAlFAtFctF9pA) zIjw%=r2b_l+{NQ5Z1vkoyYBoH z?>H|O-#(iPyC!Ua>=S9Qd5tD|T7IMMtISY!ktH|Hmh$MLarp4d9?-IN!%Fp@tW~Uz z=Z(AbB00zN=c4tJ5;q?X(%wirm%DSaXQps~ti>0OM%?c45=yT2#JHPY10hwyL9A>c>-V{4yb2m-C z^Lq;fsfE+r1=@JQMxU1ie5NI}Bk5^dAst&?CHxtiA;cQq5yxA%l1^$hY*lN7ZQ38i z^l>*R=enMpDU=4-@ebzCdq9~FoG2i+jf{(%#5ue2VAgzR*lcnZ0(YQTdq@pJUlmG< zvLo@o9A#c&7Dx)U=isX17`&cPBsxv266fD(gn+4&F?;#;F$k$h8%wYF!bZjTp@rZoPnB`NsJAZZIvm zwnq$KpoT6>K1${e8cav$YhawyRf<+nz=-80-1EjX4jjJ~f@k-Ix`o%Mm+TGwd0#>k zzeTg{bYC15JPws2-0AYBB&@L+&q5asx|dc)@2CBR=;1;9eSDg@ILMg0ry29^;jx%j z_Ks{W+Tz=oF}O`<1@E3WOY}ODz(vY`VDnvd;wA54K)x<#y=Or?A_Qs=4#49kDWy>d`^uD#n@C=LDWEOcF~a0;PssLL9`yTl zK(gqLF{u6Pg_;vP#H-oE;7q|&S=)Di%(l2hc6!@k@r!zzT4;?k>vXvIasaMRj^Ocm z8_DN`CL1RXW#6QQnDQ%~Dvt%?OmQCnJhB?!Mj7L|52pCDTW4X6;3Vd6^+rprGt_AG z8TP-5;j5kfc}STCe^GczJwyFb>&Zf#Iqe|aYyAZ64V`(hdlq@lkD_bsXJFg6EwFt- z5N3rs!Ja|NP;_J`T|PFMeva5i-sdCHPdiO4TRH?UbkT*n5=Sh_lE*ZmfZiTek=vP0Xej&u)SGa#z%^Z=p#O%~9`;CTES)#8(Y1q~x^^ zc1}~{ZqLut<;Hiypbv&}jeZjhl=;bq`<)RQ%VZR-)Ck>Is&i$f52%Dk^N^LhL2}+w z9Q(CZ@SbOaen&IK0?YpB-|EUA2Wv>OSCFXV;s;hsN~o#lHTXOtle|g~(fyNKg$*k$ zvEZ{SKMV~;+m}&Xc+`rI4NMl+XP1%x_%e7s!VFtDjZTkmgV&|ud}#VdT2f`jtGmU5 z+xlT}{fQ}B=WYa_Q=z!Mv6gGfsk*);Xk^( ze_JhdBtE6)j1!QUuE2Yzq{E)fByf;!6@8C>7K4XAlf|CD0KG=|)8#;Yw4Z+xzByR( zm26!Ma{o=f^Nnzm)pOdCn*)`@cf;wA`IM%d3YQJkxa(SH`VWTF(x3aG^Tq?>NyUB= z_l+;$Uu!U#f&)!EY=whIcjhT4qd}qY100B)gv(Q$aiGFr&eKo9zLUm4x?(&&HcW+i zHJilsFEsGScxTbS<|4G|6zhh=v=6vUcPUPDp-0HwLVh+0AJc!{|12 z{veZcuzj&OXArwPD{|pS6;40U1s;Y)Lvivj)ZBatZVeA$nQj2Kmp`K?xkq8<$+g1H zWnJKhoIU@jiwtVFw!(1p0&?DJFE8h>iu?Y(Be%hCAou7idVT*c9Ni-D&BhQ+-Pjv5 zpW2|e{SoNz@PvXgt_lYSD)YKmN<2TW24w5!(}UHu%KLfa-PEP&ranB zpEl8P2*y7#A^cI_5qeJkM?XyLxyH8+a)KTRKm2N`wbw4;RJ|+g=&dm9xAt^Yd!bn7bal{x;CbRqMomqjdR1kRGlrKSyCchhTn{6KCIk4&(eS#rL@? zIBN24N-;VNFKTzf#Am5O?K4-7J|2pn$Bx7X-+hp=;T(N&>WN}M104VCKVf}Rnb^DF zE}eC8;~{U8yZNN9OAh!M?<${$+5W621UiwGVmR7>%A0y#?&PS>;KMa4a zw32qC0d9+1C77lB63_1FLR-x%NPTGp>sB_wpBeT%aNkkk=H&idZ5WLy4!JfM zdWy&PoTLwbta)3TGH+2E%das8Ckz_{Q@ho{^NaJrShknku1k5?rnjPNbtES=+<;;I zhS7BCUV4_RiM=Ev_^?5+08I*fATf=itH(ir-)VZi<{AuXkH%{fXMTC)D*b2G51LEO zFr&W(MJH>Zf?*%{(UdBDYuHBpZ10j`@E%|siUAkGxmSK6IOaFfg|3rO$8j8&4j#)1 zab-KKh-uTDZ5i4IEy+7rPigp+S*TWw%z!KJC# z@E;9~?qrO5TfRY9%L!UvWq^}AJq4W*MZ})(!BNG5H&knIh4C%=YH7odB7D(wf?Q)c z*#%7J-lmYoTd>m2faC6ehrPo+@MC%x_H6gVc{?xB^M}3U`BXTMos$N0GM~}4OGVIK zN0~lm=g@-SG`MCsOPC(4h<^fB(Tx$kvG+$M{|TwTDBIbE$A|u-vlNG|a~E-v>OlS$ zW5=aclhCjJHl5lt3hu>@7y9M)L{o``lin%u>vLlSjk2pC^j=Oafu`t`T}k5yZGs`| zx57OOcZ^Uy2tC&h!`3t>{zJVnLsCFR@8r6jOwQWfV1u)#o`kd>Rb;;OAiTP_jRM1M z#kj-^^1JsAjF&p_jkgkbZ$FM#FZ09dhd0S>TN(VjsV%y>_e1^Uc#Ma^7?u83vUSG+ zdet?7hs$}m+Lo@cQoXZG@u&y7d{*YZca70^f4MBP!3+!U^bs~M?MEX9$&LO^eq!ur zC+u!~h(Ze*AuQg|KXba=f6?2EdOq!hhjhz@?Q@+`qw4@tpSA|_cg?`Ttxssw`%|#~ z<}cDTI|ajgX>oCnoAhSYZPH+8y^#HlTJmUumd?#S?5Xeq9(3mUFZ} z=?oLfo(5vOQz(Bqw1M`fOrbY>^-(Stu--8h9CSd+AIrvJ*C>I7Gg?^eQw=xbyKv-* z0wL~eA^2RgWtEfD+4bdo%<`O#cJVpzKEFUXw5yE-_XV`G=?i_!&!O{ap)lvJ0rpJn zOWEtR`0Sw(7&do{@cfE1ie0xrj;uvYYLQ~+=Q*^&*Bc!xm$94bOPX|F!nc!q@U;C} zsCV+9yq^{UbI-YRu8@eM6T(rs&K^s0{Fw5(i`Iut`CIW&%m{fPJpS&8N9a5(Kfic zhT_z?OTq!A8T9vCoA6)QDzY^zrE|YWiBD2?gQBXcJd%_O>a|5sOD(i^hA)fVj>*== z+v4WtaLy_EM@RN4@&1#y$mvgrJZGMY>S@iuk5u@*m;=4Pq|3i82KzQEcG_r(*;|yk zbE*W@t_BOs^IRbKKplO*t%}d{HTc$7CA2)=9kq>*2@XX|IrHOE{BR;mG*WUEO6Pos z(pi5f)Zqn5XRW03r@Qi(bS><$OB-hv?Gc+5Cc_lB0p-gV+@<8OYGJ-xZ^(JSn_h2N z1?EvV!~vIrSu4dAtDos{>G#)s+sPwdWIaRMDzW51i)lh;H`#096a0Q@&+2 z>}`KW4Q;bvqjVKKSZ9LG4hP}L&K0t(Q@-vPu=AFibgjP`rA*VIiueCfefR{{-@c6;?4=xhYA~jmETE?NY^dui1JAv4$g2J| z=x+H-o~u3C!EY)mEa;9-QZ+2G{Vr(7T_E*A7PMsFZ&Bmmchc{B1iT{*SZ{4TobJ*Y zD=wL{WNSD6vS%I*)qF$R9UoRLjApBQhcP3kDet$ z+21m7^afm|H;T*hOoY@fhWyRbo#*MT#+akiuw>~Us3srIS~3;Cn`Kc_6~h zdza}_%8&AxLC3_i5B;grn;WE*AHz8&RkEg&UX&Bk2i}JHqwjYGUN~=&%yrL7xb!}U zQvd5B?<1@Nvp);j{XimrSMS0KT?gTW)M-5GkPR>IypiUO^yae1{qf-CPSifw7c;$~za3OdRD?~AFR1@=TP&Qg zf;42>IPY$V&?&BwuBQ9(Uao>Z6-F2#siNS1&U`Lq11vka56(vIq&FY6V6dgRI1;q^ z&eLeDwLDG>MRTm2{Yv=tayVU(?^zGM_E@f|NIAjK700#&LGN2{!D^!$_gLbH!51FW z;51{*-}4Zf47FMJY&VJ?s>_z|{n+$zES_D(FzE9)c)9iE z_Z8OV2&`7>iL>lp3CF9O;c{Rv)OT3Hi*r?A>-jD`{n>L!Xc2hZ3KpN*YI6KzAAGP{ znI|@xVv)>&0vdnHk{dpWEr!~Xu=+BJO+6*N4qps0OXc|kj-{CvzGzsc!@as2;qNbQ z^vYxrX01u%m^t@lh7)J;cpp6sa1N3=Tv`YF{VU+@wugXQZ8-3d3myAw%-dtOz~W2& zIp$C(-}l&0POA-Q#KPY)BlKpyW(7Ry>d$@&L3n8Ha8!5@0lV^>h1BWqgxDu?y=G^- z?DQUcybL3_565v(!7qAeVZi<|!T73Xhj8Ue4|!b>fTwPTvAjnMm3f_5uahCijGc!e zz1>lbOwrrzAAG;6$e|tIp!4r;{PJcj<$7G8YkA6idXEW@{nQJO930B-_k&q=c@J3O zm8{JB$4;ErGFP zy5Xv5Pa60_pVOBI;)Q!r=xcXHNM&v8|2~6Ff&+L>yB+s`RYpydQsK@XJ8sMV2N%z) zqGL^0&K-XaJhrc)!6DPBd8CX&RxIL)=M(+CDzA|+e=s=K#X-e&3-My@3wl<*4Lm}3 zQq#^7Swqc4`aa!y;rOD6r-dR(&S)@iyKDWma;xzO);z9e#Z z0*)L0i43Rh64KYZaY^h5R?R(4*A{EwdJPZ0lBS4;x+=W-ZVL)9^u@cw7tJ4*PL0-wvVsu@+kRM8dW0X_)HX z7gu%n#O;SiE}`4cq!z-20~w3_KA-3rw8x;2|yEG47sFo2UZAX5OU4xR-GF z=`*mdvE-sjOCfvlFABZXjW;=D!03JLhJYlTou?% z0ZU!+dO zkFVVSnW}?57FLk&h+Xh0`m8v&+ihW5(tGl{R16dP#nFQGYlV?t?D?BnD{OG85QG_* z$kgy6oKYc4;&B}OI2$64n)2X7J+Wi!brSS5frcnOWj(2RZ<{wcbvGn6KoQ%uG;QW&ixyOMu+z-%_r#{^5_*P<2=!S>> z&cP{i&w`m{tN-OKmEpIrgVJPj_|0-P!5XYxY_wbY98D>9LsATa#;4RWUuK zFQ)IB&EXAR+%?jerww-G5%=xz$O=`_b!CX~sq=2g9n>hOM~_2)%Xy@@q&si0-U}^5 zw@GRRUvXEl3YPSp!DX{=kaA^zcCJ)~$1ifra~{@F%!o~7p>S7NJ7wAoFJj&>(X zD~0Es#;|6oKRbNYW{=S-r1z=>c9jK+<0sb1wmq=Gke%w(@%i&%a19K3ebv2H?O8{IVYe`_jtT=vV{J5 z*h5tGLI`aeir-H~2zv59+{g#NgsA5uaQegh!YM;jA-2;R(n@va_tGq3+^nlGdBSV@ zlC_r#1YL4X^k9n;FRW3XCQN_xPO$L4OJQ3@$yeu{bj-&HVg`j`tC}m{oOKM^GxkB} z^z~Ud^ndHfbF&-C8 zwm_>;SDfG5i6@NfhhuY0`G-80e>6oI7k0DYf}9nim6-!eDrRiR`WCCq791Jw9*XrnMXmE(sZ6PU>GJ0 zc`Yn{r69<+2D8b7&e;CEgLZ6ffjn17elN_z)5lMNY_TQI=t$;UC7aRDYB7|inPUbA zaB_VS^wd`8gW<<$$K=IO*!+j;d##}`#UJE5u!0sD=;5vS8xZNMMWa({AgS+9GL2dS zX48TFUUY^LJ<`Z7TU$c8HoWq$1XXN@@aAzDbW^1}zcxPzSDa00aobdJMM{l0R~$+4 z8G}WM%6XFLUxX^z60l3or{jj2IOowG*=6g2@b7z!5ScYjGM5(8uwP5$vppNQ$6$en zt(?LK4IS8_yFKoA2d+GM4a#qf!jQI^thmY(90z=)Uq_Yj+;|6|jwINr<-hj{n zNmLUNf_L9Lu<-3IDUI^LUAN!Et4^J8w((Uc)pqB@GY5#x*FIBDvJyr#7;~7h3I3fe zqqu}_=(1rLhuk&8?h{@3NwzI++8V|!Lrftj_YPUO5+ona7JL_Ogv?Va=<{Z$L_u-4 zc>S#dmTj6KxOwk@*mc1)FVP!p<9whrWjkG%R6;Lat)tq|9xVM}M}-~vLTHaJT^R^QQ|olsGs+bI`>`)5d9w|e@TVtl3r|^G6NUH z%)$XxV}wtQJ7~l~J2X(V749X6;-h($v?^>J_+L|nPgX}^oBMy18m^0~*`b(zs~eR2 zB|!7F9N9%1AH4mvgd~HAw%YuF(a%?lf7e`r_eY)atA*VAH^r6TUp+}{I`4t`LDy-G zTr2#XXC?I9?8)c;MdJ#`I@!wE)v&Kem=Klh4j$PVU_fVJO-!xL0eD0hMOjKgyFm4_LfYE=UQ8Sb!RdfMfS&je>Q{6 zU;wP&d6Pak7Q)ke{kTv&0W|(9vE8rHY~r^}@R<0Uj-+W|O{5vWv$Vm=vCUMh-W!Vb z3^=rZA(Wi5WS6E4bbcEr`u^}om7mMVcAOMSP6Lmh)(w-}Tg9Du_vpc@YWP;&g?CPG zfFt>~ET#zDd5tTK(by<@FY8aoht$ZuRNZ;OT{GV8oQk!!VkHN`*Jz2T0J6dHd6+XxBfYS9HaC}@ZelPbu>344;#i5$w z*b{NAlyAqqI*0O_Ud7<}VlZ5_8%4{CLgb$IQE2S3n#+R6($*KlxMEzLII_WvCI_2I zv?e#wzuppjld2={eRtyE{$Jo#LpbVx8OyCLjbxUi#eNxbk9UW>cY7|4h9=*FNx74u z!^DGPKV20)T~@=utJSnu-v-BiDFQ7cZwwhzK(7poVC17jp>l~8MxC*vyh-ZdxMi-e zLRVR?3(DD2Gy1Uhur=bD(9^X2%2x_rt%f5;4iHv+%MlVzcyP5GG>Bu-cZXw#A$m2m?{(aYxXU z`+L1!_GZi1h<`43!c`4ckSUul_Y+J4Lk%^oIenIfSF4Hpr4K2gcXvVPerp zx^cQf_Gs}%!AK_^rYGhIi{1r`LFu6!Rcp&?kGO-Yt!>ZY-Tlvc-I}-=c%h0912P1XXn%YP}OBJXH_x78rB0 z%vLriQkBP@aKV<>T{&c%Bf7awrk!=>`1_SL7nJ&NmG%(oUS^HGgOAeF*w?h^k0JgW zTnSAd7E#AVPyAz~%V`rMX<}88a3^aLgq{0LJwJ!>2zf8=(6BzJ9GVOM)52L{!EiK~ zV1b)wmqW{t3R)j%i3yh`)3^_R!FBjlck$ME867yPs) z8>~K-(!kMM#I>z+#9P;gqOtBI_L7a^KzaPUn6MwbS6mPSCXB<-JKyPWmtan<7I1K1 zBk^4SXK-kP2mG5q6)r0)pu@*IvYwwM94z;fSck48|Cv3IlAW>Xs3q6@Rlvr-wtT+M zAD=Zhk;UPUG(f=&tvi)bCrc&R`z?e=>DkbLRBxVSp^hy9iaaxA4y;L57kjTzL~(w9 zJk;Ax2&+8`8L8{(!v1HZo#_Hg+M>y1^jNB|S}O!vyRxdsXne3Go$I#c(U~V+obm6Q z(E7NJE`&SN@kA>=+NOa@SAjdWeS~4ohJ5zXC~;Q%S8-3vcs6{U4WZhju^v|P#nY-B zG1ic`=Z>XDzaq%rph}V>g;1ngF4{iZ3)dIva%~S?T(ab*&^FbD{3Q3tN_~!~;CLE5 z&d4?WbT?Aj)ro>ud9i++D_(wOi>LJzV6=rB4l&6l+wgICa+V>Py|CrcH^!m4sDgK{ z7C@ebC%D<9!yu=@2>q6cgLGcd9Fqo^J=-39`pATk(uee@#Dg-IexrL;azBLE2rj-o z6Z7jAU}ajg;N=kmH}`*}H%mH1w}Jpx3UT4Fv7<3~usvp$sNo5>b@-CfxT&iPzg_!4 zexI3gfu|u(?K>6pKYNpB)KqdEa2#TbjQMweV@$1>&cD+SZpnen{UjNviPOrva>+S=oTj9{CJM>t+47Oez&ea=Jpe@6ZwVy_dE54f3PZulR={pSiHVsETho0Q< zy}vMQ%_rf$@E06Cdvlh45a+s>iq}S);DMFHxRaea*M4oKGf91L=$k=2r6ZMYjsB1P zSKS1$(USXLY@!SDS(cfj>R|4E3*Ox22PvFLqQ_U%vExx+4*7MT+U&eIwq_<@@9&6j z3P;i@H8T{wZd2o1H`o-N0CCc4(Ii8GcX>>a^~)d2?GtT;H|E`O(q9+ zm+9bLi!dHCCQUF<^5$921Nr%dBrI*R#8Y3pgY^1kiX7~Oj`tr5>pK;JqeBE3#eb#B zpeSCkLn1VG^1u=Y9sZ{tj@!20q8OtF;fl5a_iP@)`3kSa+s*#C>Ckri>18cU)>c5X zJ6R-qw_BLq?7+1rUFGk_k~^mM$6l>B>1?|a;*7qy`=J#-ocBYpj_X5L|MTOKSypH& zQ=vseo=XbDI-_~EKQ9Q3%2=<&!`CCQtlhIkR+g#v_w=cn_B3qiIxJ#CHr3;?4xIph~mEmndB?V8C zdx7>E@}G+T=zdMD`0r~P`R-HZ169xI$hiqH(Eb{AE*9artu^)8_E4DY77Y>A>nV2o zJ8*E&q|)2kgvPAjP3-r+sD@uC8FpWaU1@|<;EQUUF6=mjQ^e$&7eb8+UXr5tH* z&e@w)c+b(-FYvY_?0$(~c z9i}<`PtkeD)%5>yyg?#O(Nv@~P#PNN+|TmA$hnTe6y(M5Un=T1Xj{?m74K zzAD+083|cmGZ7(_-~Ii2{X8@dY)6Pghc4Rbm^NJi#Jv^^ah-YkLa+q$P*jY;D7j zdsN1K(fP>~pD1Ql9Dl-icwOXjPoz5ehKF#6zi2bO%^fCv+Xqn)-ek`1t>Z3RXs|0I z-1zzw2Of4!-BQxucI~V zdfb^+4_v{k-n%9`E!Eimo>DAI5E!sWZ5FcbXQwbb&lrfx&nt27%DXt@n}0-K4+Cp! zzl`5dD90-vXyuG6m-21GRcv|eA@1x^d*)=9nN+Xs$M*fy;{I(jWq+3(~y|g zhAq3TV@0_}bRUfLu?ieb| zLW>5!C`^TIylB8m4ey+5>=)+7xB~9Gk}9jWt%W->VJ+9OCE4lui>18(tt{qNgT%?> zKdHX+jvZ@g$20TOJlV811O8O*dL|$0xcSNBSO@3j{M_zf-t2oGcjT)dKP)1io#!aW zillz}`I(!gI{r=k)XzU2$Y4Dyy)^IVf(@NyWNWtGm{a9PGw z%`)dkHyNi;<_XjMM3dY7OX$>OWx-nMS8^$ii?lbpTp zJ*O33vb^_nZP79L2=2uGc}%YDZ|>iGJHB+1KASJK51c-t!9J|`#}Owjc3qDTd$F=u zWI0KChF7d)nIZwZ=F?PGu69jKE>L85Cr;-5J7@F$E!ym5w_QxRe;#*u++x;q zP$a)%u?e#*ynxGZHf3L^2J;1pi+JtCADl<$Nm0;>OU(P-HB9@M#Z2+caPGpOEK#GP z24CZvDzeIW$4!nr$prioGs94UGqaz^e+i0ZD=YNb*Yl+3)5%1q8kR|^xxU+Jwf6`%c)BioJ4L{Y-VM}0-fh4;YZG=;c^t2hCE?JF=gzY~IfbYJ z#>TjYc~aQtq>^3WR9EdPGB_B_`CE-(a&$wPw)P-qd;`N~9l9)9vM+$u)Su0Br+;u~ zyUug7_RZyC|4KGwTP(*c@MLS}Y4Q6SpED)pZA`7a37325LH${{z&J}iG4%>vPEOx# z+3JU(eCuQfW^b({|Lx&Zr(Nzce017QF2${d6H52IS5EOv(pPzY{Yo|VhJg&b)-a8G zEB}v?|CPo~OFqK={VHP8YwXZXAWa-_TQ|3WM$9ih0bTbcp8be$u8ttg3fVE zxT9Pw^U-lo!%faw?vUt;_b}dNK*$~lAHo|d9^lr$N#cC=17j|9m1?nq*tXuaoYsLY z%#vNH+`+7cymMd&Lv_4)!y8RZ;3(<-ZBQmRJ1fGeeq|lE=G!jLWUU6*TRnvxXS9yb z5Q?}ie`|iUtQKFmp_5ThJIUQPP+>l{j*}X)q?-SDS9bYzE9TY=D?Z!VnLT2*jD2+B zB1ivei4@~Cn6KvM?D+vbJ{v9gu0A!kYlov#aU9RYHjUG>e8D9@A^yK6HA^FDLk>fXG}`}lhXtL~M~ zd~(%f1NNFZ$$z=Q*xssSxR@P`&8OGg5rw->BW}fV=GV75O}@05t8V$s96a%q+3vfK z@fkIN|Ml}H6O-)BUZ1jsi<(-;eXZALW3{B3hsD(l#?z@6E~u@CGuGbC#Ch2BZ``w)7*AW)SMMD+^KzQVUACI5 z+FHc@?%u})FL=!jgq>naF6y%eF@t!+RA1iYZ!n|temdLOx`Usw@FO=XOq(})@`1CB zHfL3Aeb^}-`OJ*+e(v{LW%iwyA*1rcfn7Z`jcXdajj1lx^Q&fOrx?v`D{7OYL+%SE%f!iA&x zo-5<|t+Foc+fVbEMXm~L%w_5MK71x$TifoC6PU*Jt;%ECauoRC+gq7wec!omn`Eac z|Bi4!O|5x5my^t(NjJF#^>G}dI))FK5YO~%&S8GjF?{w|hCh~mhdZ=&3+LwQ!sM?o z;(ZP6SRJMF+_FUnnPhDhKJcn4JGE7deVJj*>o1?hh-W5q%F0@fX4adSvR}aNkj|0o zvSZmJv8#BcM=?%ErBJCZ_p1(%wp|lB_&V}e{gqe`B~Ij7Bt2smJG1NGX|i4AN!*ZV z!j8{e$I71cWO_Y@^A>8#tnTqG3?DDgEEqqJoxXfBzsfs_Q*)Zn$@g0FW)Y_tEvXN} zw(LDq5%Zi;JnJO&ifrKBbfyGb0YZ<`VjTb9BctZqx>z={zCep5&>qk(aJ;$BPmfc5Nhg z^t=L_cX|{*V?ni(sq;a`IECU8ZC%lQk5F0A*jf6TJF zYocL06Ifemw#qM~f85l`lSJm@LRh=1mHf>2Z$(QsNax4K0bHV{4qNmt(aG|`N2YwW zKmQ@ngDn}wF`7Ot%%73Nx%4ZqM8yyHGA(-r{Q90y(TvlZIdWO*ryO?EDQT00(=|TG zWUrdbnjduMViuj^*eDBjc|!|RcU_b9yf=yue3{RTKOf4!n3T!*azk0=xAmg#l3+Fi zXNuNH_YCtE>a!1q`EzQst}`;}mi)`DDs1btUS^?-CsWD{=6f8h*zZ!WsEyGC*72<` zC%^VAXDc_GkJOdn`~;GKMZ2_@(X#Dt;XE)_%P;do&s+((9cA?kov1jzc_8l`R){L?at;M zdB(k2=D{r;k;0AtK9WyOy~6!c7|hG0sIcbmA93|NZP{;0%bm8~ZFQ;$RA$qxjyWFO zTE_flmvVuJiWr+e`?>5->a4{UKYo>cA`|B2$%l`B!!#Uuz!|zqpOM#rP8On*obQO2 zO!`7~Hi+vMEp7kHeU^foE=^y^1zBe^Q&Zj8QI7t6j+ZWbuRKNC--U|;|F| z$>pw{$lxT^H#yVojojBDeTS*J$;_qPY(_l)D|60Tssp+_i_~*6m_G<1^BQm8@RD_huE;8+YY!`8N|p=L0S{Eg|Fi z+&6<++lNh@AoLJpH0>_4|BlpOa{n(gc~uyrH0d-Grh1A|mudi4*`Ic7ED2#-Gbm>% z@#CW`I+=$StN7v?dH#!5Iiq#@IhTK?qH`ED>G;~ ze>wFEBV+K7J0)o6?hox`5uxIYUi}@E| zw`sGU8-#2?l@EVpx-)ZiZ34INnl7hk+T^4uKY{%=^gX8_)p32u6tdi@W-eiWH*@{v z81}+eU;g2gO3?%Bdd@pMf){++#hz&$%i8H$vlnLs^Y8uFvuhhFn2>e@=2X@DvMCh5x*&giW+pXfe@Ic8zQmUl>JG)vFd&w69S+Yd=~atxiyOzPAT zogZ?NY4BRc?Tp{U>^nY|{}*+MiF{_v4gE))nl8=fN8C_j3(st1noT>nKUGq_xu1x$ z@)z=vz45Guz9r-Ed^NjnXAl!Bf0&!_dR^<4Y^V!n-d&8f(LVb&L_@y5MoTz&06Cbfhrd4Cm%KbZMhfs)$;{c96N@4ts}$RhAMJHhyCSFos;&JE{j-WX}-I`BR{OPP@_ z>deB`k?dcE=}hv22VB)lPINMEB2NvYxsY1|_FqjTKYFAhue!yOpLavL((-+Ar8!w zb1$9xbjR>%ae17PkC;=95-~xxdi-y@L0qdapX*+nUDvpKANTQ3ET@@zhTE0t!|!bg zVfB{Eayur=u`4!e@&lc6?CX-5OzCGgr*qMI?12r2e9G`c%ub0iU)N^M8tt3HAAYcv zvn&2sUtsr_SzWn|SE+H~xB4jY8?)5;)dQAnrdvLjVLXBT>${cTDez_=6>Z}dMjdA+ ztqx#o?4hKqLqoeR`*=~6!!6$$Mx%Bz zzvhb6E8qH_TcG)xoALXllX)M*_H4K2M`?{`A3g}>ldgI5|Lm;zCFgB8GZ{U8_M8ZA zGVjGNfAxx4IC?Sf(Ne_iJfy_Wyy?U&HH+iCW$gLozKKqY%W9cGsfRPRPsI6tk>z7D zPK(m$HPIj8Mds+K526hzNlf}0>0GMch&6m+#eZpT;H=zlIr&ffAX@vZTckM8hi_)4 zves`eaB-aqtikL6cJ!Lp+;CTG_RCriCiSp2dsh61(+MmQP57k5CFl*}-KSPEvf(QH z#F-=5kPG`c(;79dI9iuKHZ*`+@bw}iO+?`mBOLiu&*99(Q6_BFS?Mh1sS)$NL4jHG z_9OG>u+*zr$T)p?p3l{P_Hml9GScx)!F%rW<5;I#GbIcY|AR|<`<9tDh;Tm=UHH#R z;jCYGI4f}@X6FBYuKCHK=O(+Q`Wa3OmSTTQRSIrnUm_Fp#8^M|k?bO?^y zn1)NlYT!R>wxB)WI9Lw8B(DC_h8=#kX!tP_HCD>wT`7Mi;DnR~`eHHZoxTQ2*IXyE zqmNOoU%{;Wq7Jsw+ySEMW<%|XM7nsebV(T-N`EP;;p6rc`g7tGI_#;Y_`>+VHgr`m& z5TAPo$JPqaj%I=TjdoIH_L(@QN5d_jnQ-RjJ`{|&f%ESHX>dOP{%$tHiuwmEb3zu* z^u5IzK0%Q3z!b8F??bPMPndbwo!p%o40^q5p>xYVJQ7e01>q8^_B;)5f6u^UYxkj3 zjUG6q_@ZuS9JNoIjrXorkhiz?LaApi&I(Au@(n}r#kH%r=3y4GwMm75Ki_Hdx|g)R zMIC6*5}aKV1Oo{j^w`lww0FZX5+67o^p`40lyA+1nfC+9wIq4Et8^tzzUc?W+Ox1B z%Z04X^+JBDC2Vi~0yA%SDe~8TjLi7hAyUm{{ClngIpU zJm|qo?Z`L1IfCbM5^>IPDPyrYQ9Q@&DPBz6kF2GX&5}?+4(@7#=}whIf6QOn8DuAZ zIAa>ru1|*I+-z{(nv7%F_i#?t7wVOB!R(>*nv^YeX7mgcoSuu+@eI}crZ3nyP(dH> zP{y4dL-B-35ZsA93UeB=rTOW`G-M6hI>eLg#aw&s;c%TO6Z~71S z`6LUDi3afJCS~D6=PIHU=}&jrU&5qXL&>^Q5ejcj0Oz_GdjI5Z8oEgl)K+T1xQJ}B z@y!PGf_JdlR|Zs?b@9`j64+s(i9uD75O%u^jcu#xrI2?Jos&d=JdVfJv1jnbPjw7z z&=Kakbco;o_zwjE%cMEZ8*u-Dt7tf?js5j=5AIs`G;chQeA3zi+6_5^i`Fqv2GTBY4z~#mKM*aruvLH2dXpP>RTb#E+?Hc62?NcW(u> zEU2gRm8QVWNCRps9xT{#yaxt<)Fb!Ys_^yZ`~0H1i+JCo8;cX>ljG8y=2!fD)Vwqr zTV{>Jj`=mj``;J(Re6UvEMz$uU9OF5m0WRU*b6F4cS4HSXrz6=@mHoDv|Su4zTy8A zEb$j!J8TJhie?aAp@VnjDsgx&rSqE-@r+d#S)X7@SM~>q7wY(d+wT7{u8W-ZlPu@Jc6 zc=)hOTXJohKmBMjAKx~Fg2B2>cJ02IX!mV_lrurMRe=p#@z;5U%JcXk0pW>GG z7!2NEjW%zpp!={H%Dra9kNhwdI-|FZ^vl_!%xwHbK*Y><@MJ{ryy zCUocSmol8hrh8P>l1iBxgy{m9mXipc@(wQ zBirPPGjctN=B8{s+L()OitmVHnGPH<*apd7>uJy9DA+Po3Fejh!2L(*wB1Eh4F2Z$ z*y2LAi!b zGDPt!6iGQ5_K#gae!>VMI{llldRtNW>m9fCnJcC-@$kva0H>a;gn_!7*sw4gk9k^( z_ieC1dE;K(9Q=dGjW{S?GV>bwy*UdO*B7H(87r=SZwOnyDhmgM_GEOdly^UW87}X6 zN+d03VET+cdhfw0aK2YA2=(y5qSg@j;d%gnX(Yn+hqpj*tO|cF?ZudP$-tfQrM1s& zNYq3NN%(hPtgsy{d>`0K^ZxyYX)^WXpnM2?Of`tr(_3iP3!`%(`bN0I1GW8v&Pa8bHP{puG>1Y$K?hmz#}akUXM?HnJ#poc z#R9dj@-Voml4PkUNgie`z#J(z(z9p^u7ram_*E$RdFee>>Z_%NQD>>|L6+Ru|Bcdt zW|n_s0rQ?U^WV)6z-%dh+A{b6AJr5`Qw1?ZID0(!OtgpRQcjMKccWPGAz*v$4Vd_1 z93)B#Q5e$zMDHq`o3BA)Nd&?RBdE6sL@px(erG+P^B(NOeKWtX2l`e+jz$II-n-Pe zJrgh7_)3Rg-iSH}$Dp6xI9PSo4-2Zd!}N8DST779>N`WgP4+ykI(ivP75?yJwz<#^ zq3^{8X?rm&eH^Skmy4fAw$jT26=B&!5iR`LLi;b(2&zN7#mVeGvQfqoeeMs!+#w=Z zyP$)<{&56X8sv&4?SDw`n;$URWDPQFd7Q4l8ehEKjrGsv;9373boP#?wZV@eMJ57W zS9;KrQ&(yCy-$=3xebR;7vjYEp=d**VQixbz7n0LN9UY}m{y6H_4K3P#6AMorXfPz ziT{Ddib?RGd_T?qxS9^nFUAZ1?BUwS)2KD;qhQAZ8%Z|=fG+If4Q`ac@Wdvv*N~Fe zha@=E;})5CR{=dWYv{B^m889(nK~?c3BDc8g0|{1%zm+ocpX0=nEe-VMPoKG6gt73 zHIMkK!_3Li_F7DDS%spmXx#d8D15*R;wu|#sM!S-!Ich8ymECj7?}FuILo1!DqDfi zZXAFCFLnIsS}R@^ISD#4o)f=y>)_s{0vfxa7dFc%2q(YorZ28v5}V}jKqGSPF1{v#z!_ljobwq@j^xD%AUsfdABf1`3Wk_8UseF)WN*tKBWnl#nv;_ z;L@Y9G|1yFZ2MS)RjLKV_iZsImhK~V3RB68;&G5YSXp>={2N-3?T6Fy@^RBQHQZaX z0v{=QgXIlX^!a)PpT8)_q0h9-@U)CPaXD>@9)Vhl71`}U|>oXENxZ*o9J>H-W33P+x_A9%{h24=sERL zK1L2**@Ka7>!EOY77SnNBHn+t6pi{#;8}V&-5}*0_4p^kdS89=DD(_gOKa}V!5?sQ z!8)8PEXkxufuFOS+V68P&ek%oTO#VecDVTj)fVwpbzN3b99Pt7hebIn)$dQ?N$w0!|6 zI(U+$Ys>LwyAIw~;E9ft6$Uvxfrx2s^($G=gJFp)>P6FdS$Sw{0auF zJWsAgy3qrJ98t@r2-jx&!}^y)aP!YXlquB|Opr1}HWZhVx!fiA=|=HsO)yMZv4S$| zujACF6Xfch)l|!*7@ivHK$oeMZMq-;LY<}ewZmS{cS#?9dFTS`6<^|Ihv9V5Y!gXN zMyl8+WFHQey@w`G8gZ9RA_m`3A*VdF1ja2J5yc5`C2=*?lX6cWYwCQ`Q7UT|FHM_fjCQ_mltQPEQuV15~uIWB|R-T%ld{ycHEEe4Gn1%kBN zFeh(m?f)+Q9x1wY02WuxMRkLAFwUC{PhGPyZJCz%^N7WQn%RBeP`yrYxMYbSD%cQa zNc+T{v+s}xORqzAXC!%Kdy{PPwIa$kyKwQ9Jg}MUO-??21r|zDe#ILVEO}}oezN!! z>b6=5tJQmG@2)z`KM@XYJ{$4;?KGHhJ`JB^5;tY2GYq|Gii&Hy!1U*5I(w}OUXD|M zt5@HGo;VjBrsmUBo#W^mH-~I7Z=b zt10|cI7J^e8)M`>ZESyk6r#78Qths4+N0r3RQ&?T?z9|qvP=~xi=WZoQ@w@fzS)T7 zUR_0f^Ln!N>?5k8G+Nj!zhfD{E_^}#o z2vETj;+f=t;#}JNSq<}s4TV3chPa$}z~JF+fe zg9Udq2|w5lJFgask1ak-4$X?EH)e09egk{xDS-gpZN5_|SWjaVrqZ9u+VFn;R9tNq z0jGz&Bc)k2@H%xTv=46*%o)8Jl1tvvmEv;x>0B2zd82^onJ(~WXa>yI`~kZ~s&uDF z4k8!F3f@o3A^$3$!#qh7cm>-Dy)sS6dC5Sh%npE*+dMB~d%cKv?X3(p_S7DgOW-5C+3LQ2M#*5yA zB-e*k1vf{ZhMH&!mu7X-w@VMfKJ8n$Ps;B2<SJ<@aBMopVCzZ6S7b929M@osLd>@^#kto#;ZM+WaeS#e9*K~{duuFk%7y@ZQ;n%PSkMv;$}&@6 zo^hVoa83tCyptz=KKfMnXCF@a+yldgC<`SQM~U-nElJy80uz22qlx~1U`?On=wmtP zBUVCH(@?xWl*ReMIV9$GAN_6hjY{5GqmNuA{Ku&Ya}z2^bK!irG&32-MXTfX;KA^p zG!%2q=eL+wJ_??$=?1;Jp~4!MLv*c{EX-VRR2f(RWi$)2b#2*TwV65{RJ1uoWKke&$)1A2Y)0WQq(Q>or07P)SU*pvuszq3yGu^* zRpyDrS)6{!A3iHt3u1iQNNuCGSdU$TVaq=VmW3RkGZs73s*Jy+BKt5r-Fq3t)zX?# z*UNnvah{oxah@-}<%$*Wv^`8kce z|GJ-Ml2A0>Hv#=C6@`8WhCo4jJawfZ*w?EJi;~BqN3sT1;6JS2zZIOzbiw}C7AW3g z0rzK)l@#g6Q1kk6kmKEta}vkm<5%&dExBC$RqZH#?qtB<`v)oE6L44spf_iN7u)xV zhkk!1*e__rGfN8vC;$5c$`fC~tc#jrwX^~{v*9fncGnb*tRryGx)JbT*L^`@`f-~7 zm&3TwWYCz?h&{@lpgVUg7Mv)U%OT=`Gjx0DgVp6elC^CRbT!Q&PXY$RKK4HB zxb06zkN!!`R}{id*+1k-KY%1XhW`3^3tOIalkQOmi2EitNQxW-QX}iI32cF6cg4-PPn304WDEL;P%{+c*3O)^P=NGsO*f3!VZfs zhK0cJ%sXVk=^ZfRy@0H}?neGDeg?;mkCOEM*of~h{uF;XkxU!qe^8HOJRI$v00*~@ zgAPX-;kVzD&~yE0;p`trNrUW2)VEa;-}g8#D7KPAgC9~$-HT`B!5C#UT{J}!T~tH- zuPI4fCx+vF6>t2rAcuf_ES;+^fObdPY2u2E=Ql>FqeEdo=+D z9oPzX$@6UZwik|Hu^iOC&Li;GXybgzu9jfg)cVzikT5 ziccb+GRi2sGLW?r{u5UX*TGI@Rd{&H86UmePDVM;z+p-TklCjIQLl#4?ZLw&9Rhln`S|7II6868ew6#N7jN7fE-uZP0!9^j#J;!`r8`a` zHm(ATxw8Z+jc4g~u%h>h1H>trlhM6N`Yy1Mer97T3?`jib#WwCjhGJOp02}#V~&e& zuGvN7N81UO$(dkkk3F8Qaim{gFQ!k9=n1Es%|iPnA~gMM00oVsB$}jLT#?-({`RmE zc9mSgDFd>=sq8^@|F^`6S%RvQu3>o-D>>l8gT+WS)E3^vXCaH|thAMQXUt?WYPts0 zF&%VA`DPd=HwT)_xqC+NAaMh7(+ly=r#Rv`dAP*?lt1}tLg}Kt zOX&A@}Ey=Y( zxpYZ>K5m@+2VTU*WBZL1Joo1gthr!_|LyyLo9xdJme_;sj%jeddKbB4EeG$<^w0v= z)%4<%N0-O%X|BGSS6X9e1>@he?$d z!ja-TB;$Aqy_qlsdpq)B>o^VkQRO4 z4Dnw_lF&Pk$chP7(Be!m{>ooW*=h!T+m_?;{hd6^B|`4o9e7{<6lxT$AWz%2L*cV* z5*Cq#8wCxp{JVy5UO|fBes?|Dl2b~~&ol&%6yld5GsSIyrRzIX0rn%<$5nhIwuJHHTS@S zkYsVac?Ej5R)FJ{Q_v`0gRj26qoXw%i8Sy8H$Sc6>;08sYMg`QnCuzMiws4xvKc5> zR}8^Hn_!fI5s}fVq#;HZAwljU9(=b7N~Ub0E>=h2Ze2QzSkMDsWBpK4pT! zY;g8l1<2n}O(J%dpxMGf66;zmcsTVcx!>giGZvQ7kd>>k>f94SW|*!dsn`(>N5+Ab z>PgZ%GD*BZsh}nE1S0E8q5Eb7 z>@G`z`FWRFmGu^+b`Fo}gEzvXZBcO61!!-a247(YH0R|C zeh&HpI|m7%Z>1IXPSg{6dCLi#2Q`wM>HBH#@7v<8TorhzI}((p7vtBeDNuIdB~4~d z5Y1DKAhS_TV&fG8Tg`?MJ?Y%iC-NqEkF$gCPmT&s`uRcGwi)!)oM;mMz6HZOsm)zDn6javOt^i$I+`cb@?zG^qYaolbQR5(oqsn^*xg%W{5t{n__EGEM%ZP4XW z5?xQnQ1ve*{Ivf@;iOAH#DQkRA=JDYTa^dFCb>mupTdHXI32x2n{fIbV^~#@0DV!_ z__=L9)Li^Zx2%g1c!oR?n?#3@sJ|ni^Jf4rBSUGyt4g?D91T?(_4M+kJy2UYpH9Pz z)L*?6yQFg^y(n|??x{a+d&*(ZhYZk-8$uYZ8fYDLiY!0A4%F)B2nK6;2zQO2D;~Pr zSaK&gjL83T$3M|Ws9~>}q;0Pi#-?)8`k;*L&s{>CnlIvlL-J6$M4u#;zNf>I3*cJD zWiqQX8}v)gi*u`YV4hYQtW`aO6V2qIdq$UdbogQPHCDq*X>a+$hr>zE;EB+4c?im- zuwuR`5nRq^qqC-*-5ND0pF zlz!*YpDEU zDFj{>DdE~dOW>Ph4Co9sCNB=`B}o}O=(OvHsCme6;mNPsic$%v6TiiI#KK2$t7 zp@S^mv6ru(#o^Vv!FXg#0WF#CfKyabG2Ub~EekQq98NM124J5>cAJjxHN+2#(#F!p#BI#CFI8oOWs? zS!LY=bNUnDR)Yx7{_?_$168D>^drs6UIbPV%0m0k`-#R#BWe^{3+GPkBKjF0X^iX& zp^S5mIH&zD?fCfs4m3`HD@9ot=X;v8_wa&1kdrhue-j_5{Y-Y}pT|LuDg=wCZGz{O zv3RxXKD?Xf2X-~>)bHdaD6uR<;W-&`XJJ2D#MpuVi6{&nGKU(P-Xf}-WHIrxfmo?@ zDrOjuBxjmF(`_{v^^ZO@YN%SWOhVKg4UEi3%I+Lg|*8ie28&w_!98r0VC z5Poi74HIS!i1}0nsGjv6GdHgnXioElykV+1cH4F`azYz4oR~~j8tQ}M&?2hSei&B0 zZ71oI5m9u1$p0q-n-VWR6Sp z{*<7y(t}=?m2%1_?#I0miga9E8$T5i0GxpTtsq-8(VSs~BY&lkh7?QWR5 zrUvGAdqUj7E7bbUXt;iR77Z~sLk*c;{Mfk$5;9DMkKR#YcjFaU-HE5M>o(K>=J#+( z*Cgby>Jls%t178e4aPxDRj{w}>rLp>e7uKEIp`)`BY zvwNVJG*;+YXGg5xD+%u#xk1nIB=Xy6fV{i&4~{1qq4h~IIInpNPpS@M(Dzwz&7y+# z^*@n5!}&N|hG3b?AYuOLIB>822hSvt&>i6jRI3(rjlPh1EUFoUf&E-b$WKjwTUkw4F{|CCo?JnK=uTJ``d4liM zS~_o22)pu!ns~C-5SX)JF!axqkrZahiDQ+F#2v=1#JXZ3$SHYXZA}_H_nJwnDt5qF zn{NEn7=!9bzCbqGyWMb|KIOvd$%d=+zby2P05TfO$dTPjlJ!kDX!!e;FjQx?xW7e%cr8;!*ITOi z^SdWhmdOaV#&3fohR+1C6K!ef*dY>=23bMqO&4Mx=!p%f=5V4j1Jd?JK%iY5e(Tb!aH=%}Aze)#jnm)SD!KZYgAZDxukdCYTU# zj_CaAfDh`*w5w2Fl6Z6^7`<#0>+BQb*t{>W%p!xNrBw6Gmn!hDDTPpL78X371bV9D zFuK=DknepD+@DzBA@Z607CPfSbv2=d#{wAjUlH8Bewg-a5zMJ?X!8ux{2-Q=-SZUa_LS3HqfKx??=tDz+l-s2 zbbs|x2um}M6O)TF;IQ2jSMJvW+52(g4@Td~Cckws=a3wp-n>y@X*e1*9{1BRA8w$Y z?{o+hNpSza9K5wMg@#tW!*|ZvG)7p=Ppt)Vr$yR-E3Tlew>)8~Y9?w)YyERi`sh@J z2C&yZOkCES#JY3e(d$k+6l6IFpM+FWFT;Pt%{B>^>)62v>3ntRtf^Q$>n?VVQODtD zq_ckKEyV6qF$APsBHIT{(YaSwpxPS`b!VMP|M_vmV`4ZwJXTA}p2YI2k6eJ5`C{^W zmbp}8!{V%dOI*6FK{|u{D>fUu8*d$*4d)!o1(AQ+NTbpu$yohQf|p5(FuYnBKDJrI zt92K#{JS1JUX_X6lWsxd1uuMjESM~*jfWRex6yEY6%=>wrML3t3I46hgb&w`KAB*dj>A6t^+3% zHHnsm9J#aTKb&)D8;E9Yryb(A{Jje-SRT?7irkXn*Bur|nl#eA~% zyOJaKUeRyI@*!(+Jtohjbk*Jsa6DojsL}CoGRcps7=9)-N6%8bjRjEFu@E>blJvq?0BODL6a1*ZcY`+wZ9_{%sdWjwUuGBQxJ((mXQ#Jr(#_& zqm>JcLFM*+@_Ug2K!HCNkMe_KVF@(Az80U3+l)oA2XXst1{XJF68mEhal+GhJ&$*Omk~qVLlD&-Mqe0FR5MJ#`jL6!#r5xCPM0cP zKaq)F`VEDKOWes)rIX-ur~oe(?!}@eC7>V!mKWsVq-!eaoiQJ`v1_3*V3hEykp?Vz zFcm&<7sbs|9nZAE8&G(BB3Q&`(70WD;J**w#47KyvCKdgy%(I2>bJTC$(mQG=tmQ^ zpZJWpMJGe(&N6{{_F0;iSP_!m7@Nc_?uqLsC1a(isn^98TgYF(EeJlX| z7wRZ`x>vkp+cUB*{}7b^(4k+}*5YCBk#OzdX{x#*nnv{RhUna6a(AK|sorCT#gnq} znKqK)V>EDlX$5@Tt|v_Ss021o?dVo-ihB;r;NNFTsB`x*b&2^(!j6Z5%MWu{5K&9c zW%Qu-yCQ(p!ys22h^;4nQMt=I;rZ$;y1UI5kG|l*b745BsypE3Gdq#LT?WTYt+72R z4_eNIlZO4Lu`#p`oGZ5DZkHQqJ+>BJKA1#c&w6_MV+$TQas;wQZIm+Z6UggRb})5^ z8?1i01pW$l5|2BYSR+0NA@n-ymXJdu6A#f!eOr)MO%@-I>VobrUu0jLCIQ_caD9Os zb{+91cVa?l$;8=oN^v+UPdWnSB_XI8`V3c?|0H>y`%$D|j&@qvXg=!~@pxGZtFH%1 z-!JlH(RDN2J!U6mpAHa9`%+lh9gQYO^(2#9C*!bky?FHh7&`MnsJbwW+n2ItUkWK9TPeBc&OIbcp|Ie43w0`jejE zeW9sLuHkN&Bv2+z%i0)gyHeuHF~l$I3~t#q#CyKzF)VawVn(dbk*Y9brnjyLpEnrA zKSjR#jZb!BgZD>Z^__lR*Si}~{reMAx8j<}KiI{T-Km6g`Vyc*xfDTehe+2mAF`qK zHryb&gyao8#`Dbhyf@Ml;La8O%;kcM5ahZNxxi}j;9>>5YNYPK|B#oUgW{{-)UeqdCeS`zIe)A9En324wXnJV5`j2GJ6#fOa+kg7Igcsb`8 zp6-~;xRZ&{Cd-ff?Cm5zsT-ly=4LK_!vHxl@DC$34;Upe0vB2#UXP&|<6nCMzUpko zq52ItSwoIoF}sf2^U8TEY(5bi?Q76EJQjwonn?UyHW0(B7vSEt8O-BSXP8qSO$3+x z@EYeWWa|bkLFCpO&G?RGvMPybb0n0F)`ZE1*>Ig}BwX`}k9}KR@wPX!;GWP0*v+mU znq0hv-+M;Ew(b_p&5Gv*FT2m=7q;U^87-K$m>|44(1zzIPJy%MT9HdHL?h1I*`k{G zKFQ-w!pR!{nEzfmiZ1=XjA_bt zlI=Pf&ORt4i8+;moVnvn(lSY$am>tN85Kfq6d90RyS^}MwNn{J){VSNVF*}so%lG% z5!J-zEP2 z*BO#^9BO{?!7@B|l02qQ`fpywPeX-d^^D`hTk$pV{UXCe*iFSHt4=`mJ~i^=#YQIc z;6~y)^VD{x>x=}Mt#%bpUl_rNYbW7sg9>=@+wzEnw&HAzOe@?9bPuIo)?MwG~#h;LZmu8ioB737J0qu}DdI&wen8SyLB zA(rB&=wo(f2+r5rkij(eTZK0+sKE{#*H^@mp7yKu_3G3dJWHf`mW7pRb`i?jAO_)TY>=Vg$-Yjw=?-6Mm{mIP(l(4nc06$vM!iClyi#@vkkAxqTj{&uzqc`h~D}p);I1CmoI**C+crlLX_(qR2P#czohRC)_6+ z15bAZ!YhZfnN#~MNMJZ4IGo#tqo)DhLQ7XDKJ66brY1vmlMvFrN>>0|kKy#Mh4^(^ zDSq!`KvWX1!jz@IV6DhQ+|?(`RQRfs#+r5_aq>1+99sYl%Z|c&13ma~8;|V$S_u{Y zW(gFu)r5xIn@H(~GN$=YGVlHG$IQX@)68Fe2lAhsiSUuJ3=tcygppZJuvFnDBd(`M zN?c1wuIB<=wtNOm&Fdti)7FY|M;l3Jp^AX<8^DPogIng=B4*>-Hd1>uldRvy7nJ5N zVQg-PlDSc5p~GTdsJ8LD;Nc>XpMCW;OdEd2V$zS`hKFaMgYYz&^CAFR$@Vc{G)v)8 zTERhX@EcC@kA@X0^3Z>|3}GE7;3R8PsD7=9+}ipSyA51`_b18Wo4vAF%XxtJ?rJ(z z_LC*{){a;rxSpq>>41H$_Yp6>8WN$^0++1|AyxW?Ja8xt4i{X7E%j-ns3eorm__6N z96WjX$79HhHRs4D-AHKGIvrc^%gLFWhhY8FD*RkmkF0mf6O^3(DEMs@PR`zvW|9v^ zlKy5x;PNM4aF$oz>@4H$=Rrx(2ajnA3Y(uELwGr^ta9BDnW zoU}!+$CY2dGEvu>7zQQ?Rs~(ax;s1Yz3yjlv8g+Jd*(Fk`sxg$-0#Cxmqy@7*+-`M zC!d-0_%o*@JtX*Z=o(|~o6poKPjZM3)gu4VdT_bm4l`S(l9x6zA0FV{XSzCG0(iB*J| z@s?DW#}OE~1Ftq+&m8irfOj9fCL3cJCRU`Qjpna{nL0j1uy!&0CCZ+t2N%$zfBv#u>uHZ+ZBlg0dj$hqQy*#iitgHJ=pvs*x?1^&MVI%98%fd~#v` zIGh{qg4M_a9QR)ue18Z77vK-O3Zh_Kk^yseW-{|!HUl;btl^v7J~hn8=f0~zqWqDW-y#cpm^8*w&JzcQErh|<8q9aK0n)qb5bWP2 z;ovrMTI6*JVg@aq^MVxD5HY239-CZ_S4+1sf7PBa+Eoh~sVmYB;Ako-bt!|rU*yP< zt87m4*q>RSvYlA6*|aHg9_Dh%-?6(WL?Kn_-M-s^8Hsdl=+m5ts`2Y zf6@o&DeC844O`$E`!cx4Ns0XTE)w|H7IVRK-W0{dk@-_)Uq-zD2_BLZ} zsOjKSc#gZHl}4uZn!>=@y24%mUB+3BlY~d2jtG*5|1d)u6LDuxALiA6;#G%)6DnW_ zR8UP34D(J9gIaHrT>J{YFfnIN{jz2XkB{Q1IuD8aCqQaF;^B@970_sx7}wUe-I;u2z))``yznUT=DCrJNt4IF-|l>E81 z74HAFf{bLGA;S-!GRE>hct5TFgAebf<2i3ed25|sGP!{g4kOa9aX4p;FZ|pLe_mIG zyF_)bb4NFcG~WlS-<`wr{FH5Cszm&tpslMdPH+d2zh$S|) zv!Pa@7mUb=fUavx;r*!y->rK99Z&hgw^ojDiqSouQEn(?RpiLFim62H!anRb9LpH0 zRl+CL;*9iWb;4MEWO@>lNvi%i=(D{8l`^@weC~ase!!U=DX=7p4c>wR`4VjT-!*vV z(02SNWsu=tJW4zxPm?bxVnoBsoP0a>UbkVL;#RCWv{jU0=*5-(yYZEEh0w|Kj_A0W z&N~$yAmDv+f!D7d=k0sE9i}{e$AsKm3Rg?!ks-4op26S-*!jzaERRpd_GTHRuwNZ( za4PV^>Q#cmrAj1v6J~xj-xB${wV_$w>Lzzi`{ML8eeb3bz+OVlGWTfe-pl zB6-gr!%e~!rz5CHb>2oyvcXcZHsE%Zbt0JZu%%G2bp^4~ zx`O*po8kS99%QeNAGGgGCXsi~zy-Fl`2MEJxL}*JaG@x-Et52h7&lyo681teOQa#y zeb9h`^LCP1weyLw51*6>B;cx`D{y|9$S-@lgYlTTgsJ*|87}{~9sij92liXeBj(1l z$O9jm@E#-)X`IE>d{Q9mB|FHS&sK2t<$FYZx;4`!sq0|1^eL2a4kI#mlW_d&8o_5u z4&F4aVpgvGiRUgqMx@-$nFE?dWTA&M^R8+$3^{*G@N(S(SU7JV8S_{{mKeIj@S0kD zlOM;>4Kv7ne=A{;!9G%`QY>)(oy2Sn*JK`r8aV`6XEQ0h6u4IJFYm!iUY6ij#Vq4E z8)4J%xuh#i68h~?AhJU)OkC6l?7aL9^m^zfOwiF5J_}MOBeq|N^y-W7!MxdUbml*1 z^(0ZQMbJ((eynC*i*}~Id~HD*)0e_Ms{YI}(;T?;_Fi(Pr(NJSCxJ1MFDK1d66Pm= zBu`ccz<-@X*z)y!>^OZYaS8c~!PVzXe#}`x^zsSh`i5Yf8PLV^thxtZv`4`91uGcN zvy>eFbsS#N5QElS51Zo`Xi;YkorcR9C2fD0;K?urr5aH2cqMi^ zr$9DUc9IG1Wh9{4i`l+A3wr5sO#jW}SaVSW{4 zG3KJkYp`%rA+dbv&2veM$B_xfWWj-j%tf{7#OtLPV?W_0)U&K0@n+|_BR}mJYI6hD zSr7vwTx&zfOz;l5qN+A#5(1 z0le~2bqI?e!4IqZU`@qGJeCqfs!a_Xr2dFz@N17^wP_R_yX;RagR-G*(;0kICXh*= zSPCW7yy2+>`cNj_4}V)Djm>IylQe(8RAe{8ug8*kJ{Nlg^_KgXVR2PP>4l-tcWNIa z?=}-|wQ_~Vs%@nBb|ACJEsWfgaDmlZrD33!9lS|og&`AHkmzn_yjL}b;H+4zZ+e1! zY1|7%i6N*rl1F+6HQ|PiRb;rY59XL=5#Pp*Fn-xxrf<75t`xmfCcCyVWd$jC7o7~> zZ7L$JbG1n1<_SXok*lQ9ZIoxTOp!#T#nX1)58-_qbK(9J7T&D!a`0y_l5ZJv944>n z=aow35v=7Tx~Fsn#7UJ$1xi8L(u>R!`&{zYYp&)ou zTLbF-InC5uzsg%#J_jc$$>9CzrtnXa9lZ7FJ~@3yQnVYv2A0>4Gu!4}!hTnHBveO~ z<>EP#7>PHeFTS6V|D;XC1{H+apGWYp@-gzy?+8(Qp92&3?SN+j9^=5q6}&qWyO=kJ zN}={Us<2*Kp8-r9>`t0^YCg2X}teAq%!#g&kK!_wN}6-sm|(>^^>*O$pTy$R%|Xqk=A2_dua{vNEeWFb5& z`J34tI|PT1?to@had6J-Da@)82^?_62Uhxek*QzqlkK67aM`PTxTxEm1b9i4@zux3 z#c_t1uXs;}rst6>FD0Q@cMF;F_JSZvl(l(Y_8DhR{seUgp1{c9)x>7ln;0x@7wz$= z7n~W0g&&{af?0;;BxJN3CTv*D1pTSvwtrrRC8LVS>|<|uDt(6-b-bVVi&sDh4kVyEYf-l zO_@-*xcxX;g!G=Whe2 zPI-u*G4o-4^KnRHQ|543yU1NsE4~ZTa<37UEPt4NtBTRio#+rHdkM;2-2|ci1M)QW5&3#}D-i~Hkw@`s8P>2I zhKDQTSBllda?%!POudEYZ%R9;G;fDHwp%jZ3t~t??-52SHj(Lms=~m*0DAknJ4C$5 z5;iVd1T%NsW=!4x!{^(o$(+z#c;LrkY?#-IyGkCz;Gol3d~ylt3*G^L)fYjpo5!Gh zm}nN(v68$R+6=9i386%$xS-_n5`6oV1o?hvp~x>4$%OB{OacuHh$goO3ir)}YkyCN zUTVg0Tj*W~kEKPVW@kIDOL~N)H>@CgD=Ha&l|EzhPnsE5yDrE*5=?lLnsJ~&p2S{~ zgzaOEg3*Nq(D&d!va#$G>G)JAD4i5Y{6HG69b3*MwC}*<12%99|0$kz_9Q%}y8$kH z@EA@I?G`G3eS;DE@e~$$Z->(78kuX$mXTq-K)i414g9Y{!eQ&j5@>luj)=8;LAQWH zrYPEo{EQG{>F19GF`muvhnhB|Bg^qn+6rFKbW?aaKbbK;^c-H8I>X_yVF=k_2T4Ro zJ~3Ej4jX>6&}O)eTv?|rR5)9P)pFwSb>lhx7I*L!1 zm5cP4RQP^_Dv{5>gKzoSlgBD8WKK{%Q3?GixbG^!=|@Dq#Uq{ArAA!%)c6>dUR1}7 zTP=jkw$+lJx9<4w>`YQH_X6DgR-Vi-FB1fGzGaR^$MPygyFS)$`;L9jutXzaFS+TF z%FG)|Ci+t@5jB@u;`G@DXBF^8+VM}aM_Y}ECk@l`fKT?mJwo~m3dx5oBjLHxXt;5B zh*=)qfb*1_@tiH0?0y!`@> zxY(1rNx|gvu`1Ga^*9U?<#Pgzvfzn#iSVFNHeCD5pVxd>2bS~<;&j(Urt|kp92E7H zK5lCYyFV(zZ@Tx%;QX(!HGdP-EZ@g;di#-pS7%{baT_x&>@N)b*MVyrRN#?x0eSFk zHYAH)3xYKFVKDb#Qn}4YSry(Ltsvg;Xq_B=qikz{Dz?gmSla zNqmRM6QH1gKi`>2-m-u3@(GH7RK^_r zC(o!w+`%WCPm`^)W{{%);^5VfrMv?&m9+hxsY30T<4{}T8D?&IlHR>mjH7)r8BIG4 zAO6_~ZIjwiKt(>h-y{#0RD_Y^T2n~G&)qoS!#;RyP#eAyI|>i_C*ePqLNYdbh8eIn zhIif8z}T5jp-r`?!$$XoFw!uPXsgJR#~Q9AL$v2$#+gIpoQa0Rx5rV~zBLve)W472 zg65Ih$3-O7Mpl^NxdGQdSVQDAePClw1Ebb7M!J=FjHyjNb7j?Tii&li@bn^2gVuYKQ9IQ6RNSGWhq>|!-BLl{DB2(qIt{S zIQ-#h5|-i>k;t2O@ug}XVWz}H{7281k?T{&m+GdHXM?Gsccl&=#2cBrpVp8WkHGBxxl~U z4;PcZiIkmv$LsqL2y0vKq9=*z;8(3UC*xd2MNbfc!p_vA$fMuEjPRNEU5d%56(H&Ka0`*Atm6ws@>joQz&&|;5Rj1JsE8FQU;Fuvp|^R4t8M1S>SUainV(Cjg4RY z1)NE$Mi07YUTL%gX(<`*EUo}=}6Z_w+XqD_82EdT434PduJ9&j|71!g)Zp;V(wR9C7P7ceD*x^Yd0ogBik z_2;%AlSKmdu{&gU(MjNjRVs>q+W?G$`l<5kkAT{OZ>TO`kq(>W#7d^bg5Idh)UjPO zkYhd2CdUKd=Trm+?o9?wo2?LD$Wd)a?f{=J&GXvbRFQhzDdc`siq?905QWgbl+Iy4 z5OHHNRT+>4O701$nQB66TEu5y-J`*7vGPMtst*A9OZ!=dD*+v!rKmyU2VnQ+T%@;& z0NQ*zI@`XOiYZHCJyuGh(h1kly4wUqqC2c95rU%`0L95ZMfbMs1-hC2$imkZ)SEnD zt29pmIiG9jRP`COLd5(HB#G1sJ;!*a0<;#`bil#^`Sp0 zk672uMbzt85r_(2fR^-MM4A8oQrM~I1`2fMw`$y zH)9lWeFkv&Zw-}~Fox{PGr)}lrvQKZap1l$6XYM81@!w=kekyU)_LS074=yhyq|Ux z%v`kw4QG#lI=$PdZgn5KKIbaX+$UmW9C`|p^mf*8cfEn%>nb2hdr^ur^U?ZF3u^4L z5ogx?99%ql8q7cR5M|$s1MPPtxev1>!Gt-6DEmh%+R`mgiJu7r9UaNw{EuJ|-xmsg z6%Vn~{i?wMp$&TdpaU4`H?V83=Y!Dp5u{wT1Kbd1+-KN=Dl|^*;nJUo1ugB?G{1g(Yf!i%~+EH+XB*kCquYfr`j0=%wr>@bAM5 zl$#z$z04^CXKmFvL$7R5Qtrr3d-IT8Svd~I=kQra+dL}XdIH+^Nrpb+{v6$zABvP+ z^w9m{ouGKoj9siy0`4nqLGI1d!S3)INJm0e^t?W!H9sGLd>L8p8n6c>DT$grJVIU0 zV8G+Ke_-`mplUx zU7e`CKRm&R-#fPF-!gPN?+H6-stuHOy+&IcTM=FMhjN*rgnkCU1O`FR!IkCNtbf@l zP+VaLau3L|hoJ&;8Px$>+t;&Kj2uv6kv=;8D;k+icueKqRR?~%+u5cxLogoF0%k?5 z1eMMz$l>Ba5D~78Dm?d~@p~FTVof9Koht;E-)@80?a4rOnlLCUM1AvvXSwqi~wSljs$HE!ufpX#TA!YOtrI$;?~tJdLv8rqMxRxAfA^`ueD z*dcWCg9#eDXu?LJ7W9K5sO|7W_VBKG^LA)ABJ1jM)VRTdeH8ebb?_|)7j!Rxtok49 z(OZ$A_RB5we(!w*J(Q8->>{9-nu#n$oT0zjc4&4l07@~R(cGy+=(0rtb-gYfy;<@L zOwAt!b#FTO{VF>Ef&%38{sC$_sEum2#{#|I0ibS$JpY*S4shX4HG5mW6%?Hu1HO~u zk(XCX*;)$_bG;r#z2s0TTE?#kOhAso21rYBHD$AS5=h&;5G{RB z434ZKV6EL0a87m!+A*aAR5!i_8=q++!QyowyzdS2F#m?E>u4~4=5F*C*@M6mZxBpDlb-Y2X5t7_g;ScJ2{thH!d4Vh8UqD=Rxk#asL!Z>lD07Q!swF=H-HMk- zhAk^lGqypK4_HtIWIrn_qC$G8dp38_3#T3=&Y6R#?OTD8@g#6t{3qWr(;ivANdV_1H=wTy8A!gMpJGJ! z%JPq=!HnO0;I^dY*1_sH@ab_e2uH=#iTsuPDhUzSuY3`Dc6^GxV$nS|OY1s_eUDJL zZx`wm52kkN1_Hx<*HO;$3}m=rB0Br;7Il?*iWCEGA&-A2MR!9H_1EYI>PwrCg5-YM z>z8?e^@Ru|Usnc0qCMZ?`qRM1*P`9i4N}~~U67jSkpi4mkE3hpAt?TP6gY3Eg;wu> z2RiiS>F%= zO3AtnZe`e_=}8qx@$)0_J}^}Dou}YH#a<9`EDB|4J+zM>!zjwQ1cmNt0!H-*P^Ehw z_-av#RBvtq^JcnG)o~ZVmYoMdFC0KSzP|-kH#O1U5Bk;wjcl{yA_xQ;mAA%G2h0ZD<9X0tV-v23y|NBRkz}@Z-K0TCpb@ z{C7l@ogd`DE?+0~Qlbu}E$9PBT1wCfqax5Tr3@JN45LTlcR>mcLi3*N0D0!u&<@xH z9NugPlb@YHrA?oK;o+M|BfJ#2_6&fx8#KA~&Z8jXa0D8)TLAif{zHR`xgg- z6Di$Spv^AUqpFrV(0}$eQt^?bxybKmfr$Ch*{BMl7Cm7N`yeu#dJnB{Bk09LRl4Y6 zH!_e;26M-+0r%e%I9G`w@Hxm&6t*lt+id#5mp$2Ny(m4tJxQ9Y`8byjsM8ej#?J!# zc{#`~@(!qdpN^)_N&$*R1ytQu3(CJP13i3Uf&MlxM&Hpw@OJPVW%%win4s}&u7s`% zdL4TbOi(-o{+>OC(9t~brYIEsowo<99O*?n#wF>ZqxaE1)tjKQUjlh<6bEMyYXOCg zHQ-U}M3BEw2^}gGf`&*>P}Q6c5*oguH>&MG_T~~)e)1;Lx%L2@3~NU*4%Wc--&WM9 z{}hNFx`xEB*ntN>Ikfl9B)ag}9h8=62g*FAqQB`+spB=%&_%Q9Aj40Ln_c-1w38B4 zzwAH$!TG_!X+MiVul<-uc(C z`i+m-xwI^oyh4Vn+L{8Ef1QRjLN(}6<&R)X{}2^e(8)G#OQa62fRtlQE#Sg)*eo4m zdOy*l-+w9Q`$}q2PUs^F3Q0oMddEN`N~JC}$Xj+5D-Fw*l zQ<}6yl0JROasqmL`Hg+`f9jlX$P_Li`Y>g9Y#Nf=B}u2ZzZ3Nam7r#k7F%6pMECLy z>6$r9*pE)n*ah~Mw{n`TLHD1aROzF=y1e^c4@H1PrXxKi79$o(xHLQyU&j!V~ zD2_u0?w3IQfH^v}U=o-Rvjx0gtB=Ar@3lWo%X5Cqrf}g)Qot^&mAA+gEqeb?Yx>rU z6JUqxZ}yvy0PGk$1;&o1fS%petjKjvH#ut30~HpC((t5e_ACIlk8gm{{}NHwTzSy$ zCVEzA0+(aE9=z6kMV+d92domlfyqx95bqbo*Lkc@A90#ZZ}v9>4-#%t9`@pN>w){A zVkCgQyU!fmm_3YQwzZ+-veiKE7R!45kf(o{PoSS&`fcyAu*ZIWfHK{gI+n5b*pkW3vxYSmrTLSIQR*(sJf+{aMaCD-}M_fLTvGA69=LIL%>Zv&bn;X&zN zQ{m>sOycIaonV!gJYtU=5a*^hjGHv5%Qq;w->IBK2(w`cA! zWz_VrrqDZ+YLQQ*dN)|0H1kzpR-7_dT~No~Z4N{SBA2sE!eqJjm@{ZB=?}YO%V7#D zJ_m|rZ9r4@I(RF!mQ{%=27<$yAbQniWY*Tejx12*w35b9`TNc63bjmXL8bxd{qHpE zAiD?zT@cml85bzo2bM^4R}$s;J(_wpB*|LF?LjMMD&Cr3G)CE z5SZ4gBjN6gKx6L((CO!h9*sbtRKE#XXWE15ZTIY=m2OgVPaa3bR!_mLQ9q=t=mO?4 z$I$B4570YrXCU<@0{vK742)W7G`D;T@RjIe-R93>4>^vq**~h;_Z#KdgC%*Ov#u3H zCay-}-J*NiEgyZKq(D#UmE`_hmjH+7#-ZZ5vEYTdIKsV4L4E zbKn&Coe%>a9RMh$+7gTiCf_>lya~K?O65OTPbi*x8B(6{kWH2TM15TE%|1E!if_9T z*>9U_gCd^`sL85pz~bI<_K^EcR>?z}6;93sGg~D&YW6u0&8$QtARk1<2B7-USi8IF zIjolCCD7+yfX2gYKwhCG%Cd|D`sW(a>4+sj%WEszj4y#bPPZwy2SaRAu?(0obHsjr zhdF8qS_SSr8sXo)be^p!@}Bdj<^>u(UIN}Z53`vg#lWFY7Uj;o%>M40!3s-D z!Qnfzs9lXEvu+aV9%OnDkUR~68~B?ul!*w z(%SqSOj~pWnfogMvA5o6ao8%bH%*DGv0+hRVZ?OeX=w^OA3uvr-L8g%fTZ5 zIjG!ARI}RcL}G(6sP~{1*mg?+t;t&grnnqsgFilG&Fmi4M9lFA&y{q*(l9IPyyY$` z{cJ3H(4hn1$^v#(cQ9qDU&^}tbmz_$O4FVdVlU8qQ^i}s`8?X?u?iPjZb4TYN^vtPc(i(eGWV-`CSBwu z%|RhUUx`%Zb}hD~e~U|V2QSy5_u5AQJop^NE?ES&y>aBG%4*R3FdJ^JyF7iXSHx0V ztsVO#CeNq_?UEi+4ZE7&5#mYy}%4Tc&O4~$kAkmP1 z{-6PsPch+=;-u++%3561wgRwjt|k|`;V*FWQRE`3_JE||IU+WW0=I6f3EkiN6^Z=^ z=Zd5 z>^Lq;i#Ac1L$Bv6a1$L(=+yiDXqP|D8F(tw+9zjn39KaTSY^ubTZX_EqQM2q)q@p7 zCfxgHa&+<~9nPui8hAG_mpi>)kp@8~+-9$Spi^$bbx)F_`MFx0)*K_qyp(13H}?tmYWbLn^|6;4WVx`-?L8X;)J%~kjbG$kf+){8jM z6AroRPIa0DT5;Ne5_E8|9X&X#!fmBZ>5tW)(Cy&KTz&0t;3Q%&t&J%H$FD~rGmERh zYilZMp8SGUb+V>I(i691tGkEe-vQ-i2TL5ub&Z9%1>_O!8|BB$zZMi=Oh zqO(+IKb1e8htO{zQy>r)y5b9}~IJ>3VdEumnBU5wV*@JUA0s6E4uQ7o>a6 z;OcYaX`{FL9GG1R-b{0$%Nx|VRK|u*PgCM1zA~V;R7m2>$gMRS9;TlAgaILV6UbxLxM zhkt{?D^JlDk8Tj~(?X7gSO=&{Wtk7o71@x98g+{tj*2P*-xeXaC!lio~;Cm)rD-*rh8Pva0s$~ zo@GDovKGyqRYYY`z>f2Y)}+B45A7K+fqJa@dvx!o2nXoAaW+QtLRFXuJ&dwg&@?j%g^vGZ8!i zvh25paH{ofef8}1PubkS7yNkf2*Bi8v9gVaDb;&3(9o|v=xVaY~> zv-dz*lQMwQx^-;iry$Vpb&~IGeUGw_<+D|8XAp5YMkP;)L#BITz_A8#u*Ci}byVji z8;~*=(Z9b^J?$%~Ki%SN?77KEA|HdnYgWi!_c|C3+JeqWuSK@0c3@+wE&7nU6m(x} z1wE3QC=TUbr0geQVwY_Q&2uje)xX|LA#8C%*=1@4f&`*`3HD ztpM5Di2QcI zKL_V>Av#nm0Mq8@AZFnF+>MXQL$*F+e}R z0Lc|ufNrN`BzWrtq+0|itT_eLj6Xx)D`i3W&4=h0FB~}ByNi;GLcvRwb}F&_Jaza{ z7rW!7D$=p!z~ZHw(ciaa;LXc#YGyu2h6PdfjuZV*tJF63+$0-dc77ZLN2qZ| zQ_N`BAQkTMhATiaBpW?hAwlnQ%tDFx%aOJ94e?`8x}I$wVL<9e5w`~xcn;UxYmn=RlOiSL6%F{c@GRqeL?Q7 zGW73xD%_9xVsu~Z7Q7@5~50vF& z`MEDT+2ONSfSLDB^lq^{?e7|m3ae$g6dg%g@{KZQ6(H(uF75*X;g49QkS3rmeHdw# zsnB6Qn%setX7n~rft$ZO6YUgf1Xh^^C{dw-4X7waC%+v4C-(k9>jPs!m-hlNNAf!B z-*6nrYKZ!$-A_=>pC)iI&^EP-L*8n!$d4X;YsM0%p8 zAfezRnEz6Y3p{2_du$#?Q@%)nsKd4Psn6$tPx>>_k}`Qj_N}tp-m(dm)a(QQ`F#NB zn*=xDK8RKbERIO z3+?jUi;oKQrrt^1;)X3?=mSe>njJu5hF(c+{UTZr_`h1U_y} zLU-gacwp>`E;ug*%X+u6=H)p+Y=10TcYQK9b)`Q2xqc3JSx1$gD)R{x0C^72Goih0 zEcVmD%xTrEzgyw5`#QKjf(zy+Z7>L=QgTLd{1VK8~ZH zuJfRKT#j25s6+qC(&sGlMZZ(h2Q(o`iQbs5!!2oi2zEt>fitxx=uh(_pstjGdfVh_ z{P_#o5o$rp|KcOPdUL?_C8D>j`C#&%uj~((HK2M`6*~Lh6JR_R4-)q;NA=tyaBle- z^t;*|7-aRZ5?leu6`ck5YgZwzG8nYgze87FkAu6##;k5fB9c2gCfX7xdcUUyv2xdK z(N>=%aPkBnIjoTYj%#^HF>wpHwjv2lahnV#dhbCevO0iXz;85V!WpWiF9Jvidy$H$ z_SCtr%hIA*Pu-IH;Npr8X!n7ORMzT!fHoXO58lhr6NhxDomE#v`h^a)r#6Qjkqtn# z8%wAwk62c=`b+iK?JmITy#_k0J|8rDB_c~VIq>C#FQ`4E${pY)(CoTPKs8vTwQrE) zBpG#DlKY>c^YEwY|Kqs5Hz}iqQZ%G+&-*w>X<3y>OH=wvQrg-odnRO;kwT=*d*0{X zBT6JnDH}8LE-qJ7j=z+`(V3IET=CK;iDif0*Ofx8S}# zsqjnuKF*)eM`*4gaQ~?iNS^W<^X^P0va}At_5~ksN$MChl^ZbT#h$ofvlQ9tDZ@JW zykI6DF@i%W5%{D=2vr-TkvjK(A54{8eov8|LteqrgXUOF$&!C2O0d-ZOg!Nc49#+* zxbwFh={su+Kkn{A^tO2nC$IP7;g%7mp(YObac{v#@h=u0Sx7t--wIZTL_S=w9EnB# zAmeF|XS1(?@xey?_UjWdOUV#bq@>6zH8Si&-6Lp(n?RylhHcW8C*@Q!pt0T?>BM@_ z_3cBB?@nwLn~hCtw}9Hmv$$<)03Ox5!qlDlO$>*7gB?Xm(PV{9y}j zEE0uup2QF#6Am%n8|K54$e0CkNr6m5>S5wYiVn6UpTp*rD&&Z*I;(rq6qEk>Gr9&p zcwvzr8FeoLjUN^;pXoGW$8uZL?feh5a{SQn=$FqeI>f}Ys z+u*(~3|+%i$w^__?8%#Qs6JsM20W7`ubGcwlJ+s2SFQ{NYm3lD?JcaiL_+<+?XcQV zg?%KZM&e6nI1{@URrD%hZ%!vp9SkF4qAbBpzZKgGKf^yuJ!baIbC7hm4`<)_1AA=k z34cRFkXamvY#5CUhjpb1cx8ntj?DwuS&##j-6B2m+#+DL zC$jSGUy*CPM<_)Uz_6}8TAepw4BNkzZ{c0yD@+irfBbEDCPj>>7U|uW21LwMsc(4g zl@jT7`6UGVK0xWye^5tCoRqQs4U0S{;%}8_5Ux(J-S3KUMP3p%diUdhbHzxp@!!Gn z?pqPBSe?{}(PoDm|6$-2F<2kmLF^bhhb@F6-q*fOWSf0JX#+E|$+iRLZioOcYkg+p z^%A&v=rEXmlprS$2SZKlR;=`Yf=S~u8H?y!xZ>*+^3ah8kdIu6`fJ_rj;$1#xUUP! z&)$K)RMCH_N{!XFS76Or7h;^j7*4cMA@zzUd z%{~t`<|?GXe+E1FM3UXT_!JC3TM7H;p2Gy0?RZ8l3PwNlF@G(-!jc=NkektlbK&#8M zk#V|9*6~+i@xWWy zx9BU+-&%rw8Dd4|Zybez<`VQ?k&A&h>fo-}FIckp2eBbmfwa9qvLzq>;+ov2czbCb z+HEd`k;VNm_gMh?_iaTl*EqN|!w7sDJ+R)`k2xAQxjcU)69y+S#9^aw@IR-4ZW7{{ zbFT+yY}tt`1*;hcsWNciHx6#_C@3^mWCx!QpmabAxUH2WCodQ2fHv)T*WoMV4U3aM ziaXK6`a8yCCSwKJirr_5VbYOiD3+)sbW$bBeYYpGTUPa>%-Uqsy7LnUsx`^QjpF1? zaZh~jug?DaH;Jrs8HR;3mq6iB4Lr)of>(tvK9I2mc@s_C-*JoZU)s*(ZH&Xx$W&3L zDFPinc|iD&P+V+u3OxF~aOCJu;?b0A_|{02^|On^L#-D;ole2X+#|3_^qZDdbP+XA z3c%THDIV_H4LLfe(XYajiO4Oe;W~(#_l4PZQwDxpx}#XT2a*PHFwr0cpI7fa)#YD3IC==X%t^Z(0gdbTyWkTRj^}$ zRc;mhM}=bZ^;-TORC^i z%*omh0pnVTBV{2Fay6D$Coly4+@e~Ye&Jk0RCxh~@|8TS`6GOeW z;l+XV*lt~m4_-?UZ(GZ7U7!peP#0%vU+;&YaZ}K>{VXH(&=0B0p%^{&816f*glS{v z3Hg|9{Bv;&!9_BjuP=Urao#Z*&P=c&zINw9t-BU{KduZ8t4&~oek3T$`hi!V4cuM5 z2QISb!SA^qKCFnxSuboc!siV$dvyp*+uXN9hUS=sxU(@gCurmaqnO zIw&+aCh{25gK@We7@o;mim9XTD`th;g?E_Bz3hsIs@F!M?-u!kH#>4r!>nExg>Y{`J8jshLb>YA|+wT!j{*Y~&t+3rZUs;jq$Hyci02 zuF)SW9_gca{RBe(;z#0Q&jlhzK^fvS|1scv9Dl0Mz~x#?7{$nseB<3tjKP)3gk1Ol zv(tJd_=$9OCel|ocUJrRYGk6^@EFt-C zA{gW>hx4|@L|51nIHF$xO$LkLaZnf>9-qYYrMly-w=oR6r3@d>at8Z=5Lnwg3*r^e z!N~RtjNY1HsQphCY$+``Si6}~GCxjeMyg`{eiPor%MEy@)gB&MRb!ILQlfHw7xCck zRp2?yg!chY30GeO+`Xy;@~i_;0_5!r0D-D4F#JysIPvC4=*;trb+(Hc#-DBQqHjI z{t)BgrveL)--6Q;nV9jj8@?{BLUfX5*4sBSG1}wde#ui_$RjsE6%Di<_JnWq7NP5m z$8g|aKi;_i8Dc|lxw^PvrV<|b59d)HT(px+<)S;YkR=|dNVlZ3-L&U5_vM4!Nu#l2>2Xbdes>M>Na0Zx_v!zrStjUt zmna1y&0TZ{?!AS|f_I?(FC2Hc5>U|FQO?F_f}d3+vxn&crQ{(jsg)wb??j=nPMrNACZG|3HPc^Us8%0atLzzDDSjug3Os zAq3Yf`tJKb;R3HWP}j2^myhqmkGH?U>2hUu(+ANW)-{4FOr*$-^$P6xXTGp3;}B|W zj)6$K{kZDdUf3pi0dJRhK&mK9@pM)*6YTQ`%qGjQpU;Tyx4dMWHrfqEgWpm9{$t3C zbH#FrcksbRjI}=b6U>e+$K0(s*l$(_qnQ%yy~{apdBy{%J1j)$gP#FQ82pwUNZ9Il zV6#Fx3}>ogl57>~nf?I&u{S7?m<6Yewu9egXUq#|hVgq`h|SMl!bpQSTXgyt%=S8f zY*7>LRqh9+6Mt}ftTVit(gY^eL-@C`8>GYmd#6dTWzOHA_O%wfn!E)oR3F0KbFJu+ z-Ux4RhT=)9pQsn3OqM@TWaBq)f?U(pmdl2$32ko?j!80cu6ZHLIdH!s6T(B&;K30tl5ng~*0Se?uwj{ei zL5p12Rm;exyvN1zBKyo$(LhGgGENNeu?U7KNdLl3x?DzeAd z%aF%YJrRBRqWAe84Ah!%7dHd?);+~9>Vsfc{Si;(uYmYR6WF#dV&rzW&$#@n3AsJ6 zA6K^J0=ZWVuk6qz72Z7uf5A>v$O~f%&OO1l2@33m{$H^8ZwdN1T9G57eeAbY64uCd zK*KRpEcC0wFZ$b{zKG9To;QPxG*BV?#ZDsU@Q6`4Wx_s99glin?h~n;NXM-}SQ0pl z*U6_4?vah|rN-nwRXI`{+R!;?6D037WA%%Ez`eHOdi=^~o0sv_Gf&z?jnp^noc--FCbU%6eBaI)#7w#abnAM9vin= zncVTG4i&2%m}hs*Si>?kcK*mG@a!?h7ab(2C7p|XIoY7)7KQGpOdk9q#M5iz_-bvT zAYc9iBqtz_ca4X@v!Z9Jy(Gre>)JRFYA%78ztGMGYT+IV;;Nx*<$F9cSfDxhavWt8oI_wv2_btVdIxP zkj^OzaxH(eZDE#DVI*99d^PE8O0#??+RR8l?1T*G|rr81nPYQ*t|;v7oSok>XSI4 z+&B}g#_B=(l34M~4?COfn9Ep%OR zBd0Ey&)(fJhkSTNi}lpBAhnD{9?$t10wHc{KtFl#=@~lVVd$84Ag*Ds0fL^E*UNQ@S%vWg@Z)z(* zmvte_4qLIxtQF}K{ujq9`IBQuo!AN;JJK;&kNq%vEBR8?mLrotmkfC%%lg~UOy0~P zCT}c)u-Utpw?tsV7Cm|oviv!0nDh%MZJom&m?1~nOUtrZ9eofkd7R`ucVb`H+LJ+g z`Yda@id`Qt1WSWwuro8>!`j5X|L?y*f7&g)f2s=t_0{mXVK9-t^DyQI z93m>7`;e<<^VsNRwq*PHDeUQ=E@aX9d8{2XkMvwKgxzxj*gw{5$)Dd`*o!Mg>-imb za_2oBo1|k#x}H&G_aCrk9~!BX0jkPuqgXS@HLqu-GA5FVW3$<*yEI9iK?62reKh?0 zF2&w$z6@dWW|5K`2hdS{2B{KKjkOmKuo3DMX+D1$`zcqG)EnDFe$sGeKliwhk0$D} zuRb4V7uzo-AC#_Un}RgR`U#uJH}eTrR(>(rp*WcpNb4d+IuSN`F3iDg$#7@$N>-=A zkTlDm$y!y4&H-EPSlcjBR!Vz5YuPyfhmOo5Khr~4yniD3&b_`GJQ>nJZw@>1 zT!y6Ety!HE1_HNeu?jg)VT0E?R^jGsGO6EyeWoNu9!hb9f7+*Tb1n(io3ArUfj6K( zy%Kr-Juq`#A=7za0yf*MW(59+aa>>)Bkh+5QU&u-Zdo!k9uG$4kpt|Zm__8YqDAcU zPa0(MPKw>vAxnPun#Asne+*+URN497*Wq}`6D+SWhVz~Ou@6`q^17ET8}v|}412kP zJ)o>i-g2JJHat)uw`~Z*Zx&H-O}_(I)Z2mZ<~q`1WDXmBY$oZ#>%-W1JvK??r*u1v zV(xX3rh3bV{OLEJHS=0TmU!s0-cw3&Uoc-33Tk1_1y_K(=?Zr2@HFz*nfa`5%TMrm z{)H&?-9vbMSAd+Wv*5%84>l%xDmhR%m)*DTH{9s_OsrpYoSAG=M|`%JU{BvhDF(zh= z4w*~9(^ZlA?r@&*8OXqSvfD+-iCBD;lMLY%p9zf<7vRT-PUb;Fmeq_oXYk>%IO51_ zOX9_csZ8F>_l!z;IvTB_p!wS&=BwlZ@U(iu+Z)%-H+BkQ1@NCn6Jsy zSTDselNOM5j6jQj)8TgR1@v9p%6yejgGtg$8Rqz8I1p|N!?hEzBh!+oof(9yWTv9V z@_ZuQ+Qy<&;KRJ!Z#<+>rBs}bEb9uzY(-g{RZbm-x~{Zv5=v}nSbLRs{Hn@qJ^HxWnMLxWl_ zpWeBTanS!qgl=(#{uC0+7OWO|vkTF|^ba8uF`f8Z7`HX>@K+~kc-d(GdyN zdV;Xts0e0#Rgk(P5${a01kK2B)cw3#H23?RpLKL4)3!enWwxdOH@%MO`TR{8H2!Df>S$d>tl zheju(#Y+YwbB?LBx=ZL3`&r%JvJgD>hw?Tqk090!?_vC(ck@lweIT}<(!>y@J|ewt zQN@zOJicJR2mk8Cz08{5?-|b?GrVsr4^k_pV_r%bf3xcgY+haf%cLh^OvNow61~5H zGfJR$eHDMt*);I?>qhs(mmrI_#hlSkFZ>!4me>l&~IY;2p7pZvH=?}4P&mx{%a0HkWS>x?v}_=xe7G=)U?bR8^M#={Hk!LY^QB{MHa zhj^fS6i>_)!&LGM(J{P(uQ7KeP+hg)GC2~{v+9Yjm%>ovu_g}hn@otPv&6x*;qYJe zF=kfG5i28)SVnhgFjH}`pZWQ23jg<>X3<*%M#Zl}E$fYsXNxoUmhWU7 zCtL%*N-)k@GtBGfl@sOawYVuK4Wvd-GkaUbK)Xy-L}lQJGh(!e@oNN^^q`% zAH(BS@tAQ^w{eDBEZ8*zf4*`yv*<)CPa&a|(D1*u7cI~(h1!i3QW&IK4g`qp-yiMacFxd@2XNNUN{yG;91GDzdiW>I1+sG zPQ|{l=RD_sx#i4}r{E@k6?<|Ck%^pnTcom{SlzUO8^g(;~V!rbFPUgQox%hniMd&%afLZ7^$n?A} z0^PU>lp9(@*#EJ>`Zsy7UZmeI4;J$8o9$zUEmp%;CY#wHd7p56X^-v7vM}wwIF7!p zXCB>|3&u~YP$<3|=8b#IT-}F6z{7I**%gEV9$JKl*g58c6(47G1w+zF3noXX2j#D0 z;X=`ItY~c^{+(FBpC0*%n6C=BNM}7ius|0I^O`{JVJtRD=kr$9U*tzFZ$a&tL~t?5 zXWm)qGKcRa18-YA5|&1vks!42$=jvW6{j- z8KQK27Na2LQU2M^0&G3XM0!m$#6LdE)6MW?=636%;a)jlugfwv<_?e;?1OcZd4$>w zB(g6{GdEO@pyS5bu;lX`qVqx*Z|e3^xIY|?TT;4t6Z}GWkB^2!Qf?SdlqanEKkC6_ z)fdF2-;+@CRR^K$Yl%IIg`l7lj1eC~iPe{{GR36@_@^KiqIZF%&q8Vb{ZAI)KC6XE z^LJ**t2$5>C{FxIEv%URa*+8}ycizKUjeV~J^1&K4Btz=pIPhD2r)HTI1+n+$Tg4T z-Cf7P+!!rfy)&CwWh&$!y<83v2QMIxSIRr_W+QR&@?nheQ-H7as!%&Ji7)7yh}?A< z?6}!Ms7;{wg>^f@_H`@@%3p(lf*n&DmH=y|B#Ey-v-#;KjF_a?wS@mhF<5b_ms$44 z9e0Iz5$&_$aZGe4{&$$*)pWIC+OR4)+MvshoixYo9vAV{&Y!T*w-#$&C!%TX1R~Nx zG=nR19}0}#R_N~5BnobfF++O~fphTzqGw(T@xJ5^Osy4T<5$fV-A(0)?y@G4Z!XRz zFSS9b$OI-v>@w^P4nppXkgvbtJ~lkfB9>oL!QVC61dqvJN|bAGNt_>SJ!Jx8^E6;p z#wBp0!?0?vCVuy8f%B$c@ZgSA7#&YB&$ar9^-CRL&!-)@sI-JXGggc!2y%c;PyNs} zdM-0%coJT6RwS%0l`_8zE->l|QiQX$FKl@6nn{ahnZFZ?A!?^2Pj~t~zPffHv)1=WpLkxK zNdz8wN1$6I->>No9Fi0D737PdShNM6Xukt3qxR@H{|cUZ5y}+IYla1?16bS}$rRiT zX9TT6+*S1kHcadx-kO~yP8mMM1qI(gD>0nti+eyE{!5|8H6%XvykqX28{$R2%0L?n zTR2wJL|nm3MrupV+A-r-HvNhea2Td_Gn06Q)ygOPTP zRl|=lj62c_5u?Xo#ry!=I8F+aRv8ejDYHS{ULI!HB|(YD9=sH-iu=~&VU39^E^O)} zqOTqXu6H$I9{mzGJ(eW4Iy0T9Uz$nm*1kz}Ww|mtu7!b9@H`li zG8SolVrcMoCQR2&2IW<)_~70&Y}vOEyM@YR=`mgQm{t(Fu8bp=)A4XTHySsO=`peE zj$+Z*NL-TH35hQ#Sgv5me4aZFBZtp2VoVQiUGyLEtWz0Ww^rkv;dVT*CI{Ds&P3yw zTd-%zFJ^PzS!~F>j69`$(0)+CD1I}?nUCdAdxBF%L zWw@x;8n%Uvgz^U5D(a(n>Dmt66>+?Slo3QNO@Ou~ z`FQ+FCI+L1qS zL{`x(W>&mBObo7tOI{&p=CmKLH++JrzsG^$X&Q2(j^MuZv(W6_1&7M_;mHT3#HTm% zu>77c-f+0aJWI-gar-Ta!or`heeD_|z(tn7^phF*ZXbf*dq#=j%Zb>zIiG3&_#M>* z3ZOsT1EpaLP9Facy?@O><2qITBF?yc%9Iz_X1kjxRJw-i)&v4M(Scp8xIna^n3>Tt|n7^Y5gyfTI$lduGoS0eQJM{yW-FeD5*EkRv;WsU&?^y(GflcsM zT9XK7{NUTtW#NyOERM)Tp%kbC|CqbIi?bi?$S0IgJJZDScc zOqNHpXWMYzjsjR{7RdZA{zw?I*Wk24DbhkYn3Q#f*cOosHA;uEX+Z=S<;mc_8j;6* zcsCqe@|~|7m&UX|@h8@t+JcIQ1~Ica8&+lLnLR(*SmK>#q&C^Aw$@A04dI=<)8c-(ifM5KGJgWD}p7&phAId&!r zBzvXHjQ+iVMtLtVTJ|1~{FPvyC7ffjj(g!;=~SYP8N}Y`vqaHQI8%RU9f*zVf#kV z2>BNd;E@yEppZP?(#(x@|S`8sd_ArO+brtwm5onE}>uSgIn)pFjVhhV({(>tT5P;@A*5^`)^Xb9k%Ouz)&m8cH zS&XR02Oa6xysSf_>|qAWG?|n^eZB`cUt{2)MHX_rY*=6)R=y~05=JITFcQnc@KHb< zzAlTywa<=0uUiAk{Q8IGqHY0+gGTuLrx_-wie}4&bueSKBr9tnLr!_I2H1nh(|%vb zdvaeIGKEW^)nGl0K90aji)?VxvWKDoJJ=Y#f_eN!nJB27jQ{+W0H1dd&zg`#b2twN zHP_?L>ldM~t{OAjKjMwJGG0=WF7ydgi4BY<5F@Qj!QxnW^4gmT(sv_pXg>~{W3Mznd*dXxf`R`SV=4}eFT-W9^s7# zN@R~hCtR{Uj=xCJjQ5;7SUUGC4*yFOd6ydCwIUWxiZhuBQ$wLSEEKouKZi@A_wymA z4(k)IgGOIJYNlSsHnCTb)gsQ8RZEd&FMXg|L(~s+Ta=5s77UfUZW4oH9ZbMWcUWZ5 z%$E+=B9y8__{CM5u+PbfnfyGNIh6B-*jYKq{Mwoh)(M@k_xpbQ;F$|=-o3-r{?FJq zzYcBk^5NBJ7y7TU#)isEuqD0@T?4b=!qf-ERt*puw$a~Dnzxi-psTH37D1)Sq8eHQeM&1aKBb_uj zeB4+K`C8I!x46iUcpifHWCFqF>2o}AMSwfECnHE7fUSk?*!QO%pZ^z%8J815)JlWn zkKRQ05pN9L76t2cTXBt|GMn2S04p|S;Waw}EON+#GC38jeR&pc$runCmJu*-syz|D z{{l1d!ULu(X&#%3S<~VNU1XURpUcfPOjQ zhG5FXG`i~8c$?u18a8j58icMP9#mJIJ9lN)RBrtOZCdG`8aHQN5EuLJq_AFYj54Z_ z<(6*}u{u5*a)%|K(?^bO=FGpv(2hX|IMWdkb0;;GmV6RR5!0tsUgS&C#|x_|t)qPU z|2@vfwA}#-eN{1oOPY4wIx>VL!^e5kPa#?O?#WG>-Q7(;n&Hh^ zMPKFG7w8HOIjUj+3Z^bY3WZ zQ>Kjj`!`XrZ`l^E#VC*7Bp%33iF`vh)@svIqt;xs@Q~1B|7==4zeYIqwH5vJIYaNA zdsfKvImuZqjuu2-xn^xJ>7Fp?rX<%TP_jwr4C7?)FQT?wNT32|1#&@-n%w)XCc@e~ zo^+a}h49t2c5YyHEr$zzXs6qg>C^jW(6>LA3oECbqkEe_a*E}DC_*Nevsc(bEmR($ zGnkWt$Cxb4l**umiKTSMn{G;cM8Mhx-lk6bm(!jZ;k02yB6oOOA{TVv4!usRhWb7D zo_?aMMYlAcqysJvP){GuCha~H3!C#)>7D9193i!h+u!$4utY;({jmE4rMTob6_gTa zoxHMN$iHIC1t>KMFSn25RIFRLkdohmI)laB=|&~0?m-Z3C@E8U`AsQ(y;FvE<}aq@ z^xe5#w!l??-b1Hvs3Ql*->2Sxv7r{n%3Dv;&E}+KUBGu7MXw*2$?f@cj{e)dgSKl< zr_-fXgx}&a=?9ZssNm&Sxlw^KJ!WYx+(SO%td{9>gP-*|#ZErg$y-e`>OEY1hPj}{ zt<XUx%R1If!1`V^5E0D{^Wo?LzA8I(p#*Bb$nIZk4VFo#}~QTHNwi6X+Sv(>aU8 z#necwh~Ku#M(DQkI<5Lgj$XaZl;f(5xGue$-2F-^ZZ!R-wd`tz%H|4JE`6U0cgMFz zxTk!4<&B;s>yJ0IIJewMl+WQC^gEdV;qK!Zw5PHgoi(CtqcO>xGB~x7mfh>d4HO=; z7WfoXrtA3hc}pJcHS*Znab=3N?HbuiX%Xl7Sob<^amg=XN^v&r*cv3%`VvUTNIKE) z*ER^hAC97fTo%~uwERPDzdfEFe&of4*Uk`JH*V&HhZE?S3^yv-{hDy0)BrW^eV;IV zWGQ`Q$dn+nE9pV4JHo*&v0RY982vWwmQdU~fm2=;CU_h5ok~$o;{I*#qW2Gr`mcG? zHu`3l>98hmflb+O>*d!aD(=*jSZ^yxro+bdbJnJ3>624L&-`bp&{f8ac8hu@baNl( zEDcmBrHWy$C*`OxEs)?mMuzBfbG?M`0{7D6zrW==8br^gV;_~$wU>Sn^n}WpEb5E5 z5##Qaq;qRjZ_`_%_H*Wi8@M)kG1{|!5qV^eJ^k`xhEVBil=bljYf5kz=@gS^w9UL< zf@A)1Tujsh>TT*Zy6fwC+A}|z+AFSM^FAnou5UTOaWPv3)g!5#<|{)Rt;lzj|Me33 zRdSd$C1M)JvbyDPktQT+V%L5^(dL?VwI;8qu_{ zgQLS}x=A;XmMf?c{+M!~&ic-$dRD2@)!`e3WYcsySV#ztDjcK_E(oVTIUc6x{Ch#? zRpbkKKdR`hE$2D9qLq8PS;W}9AjvII70^e1ZRK`}gt{X((KMKQ(sh^gxJ49DT-;i^ z<+Tp?TyGwCPsFr&bJ~VZJW|Vf?Ubfhf7nLx-YZs~(vhIMRa1pKN@S=+^)hNKdxzlc zzc4PkWEr>ghzh-8Te1-J61g`|4s$LdW@@YbDo)Tgk%P2m@~zV`u3C#vy{#@0NY9O= zrE~J=XRG^ZhwAY*Vc!}0_+vdzaP<(^zDSZQVLx+6TNiU(1!Ek!Ba2$wI-Z`Q;z8T4 zKFWPBc}&mK&!*wy32u|CK3CY4OMiM9!}(fuY zeZGi6TV%)mnJh!u-5e9_c`?ieEqy^h?RgHdZ?|xm-O3ehWT`uQb?DNPwbT|V5vS$P zJUae-8KrHiQu$20ot{&aWPPvbJvVLg0ufg$gd^|g3Le`AaieWSW&8$9;g{h{w8_O3 z&Q>9wlR5fIIKS?su&>wD#^#WSYdouu#xG%uvEEdezbQtj z&^(3HEchtAyz3xsHbujF<%P@EPoE#4ZIo`1QNJEjiig$;l%_xAPHR0UZ!UZ!toI1u ze0I#>eoalKyvjGx(!PDvMXgU58Xiy2O^p#8x9q1@=XYD{@sSft`i;j`7g6)V2Ziqy z9|*m6eif>&(6pH-Q$)!NLIjJiG*KtBDf$?dAnHc0uol=W(L*_N1giVC2@~T^a+6#l zXu}DYxPKSd2whXIQmI#ZD2e1S?oeAO_w08=cxXR3X)SRyw^u10aT5Dg7@V{>|mH&1g=W2c* z;4*rGxu#NUuIckSZs@~z!4KtC*1OdwQ(eK@-1R9xs9k%n(6P!Wd?{iHO%e#{wKXRM z&E-z?%=-*Qn3PZtDrwsHprp+QnaRR5Hu#Yt^Ir|=`ChDbk4LhLiX8jVXN6u>!0lkm1mz!q<0m%3LAdK(I2P%r6Z(5=tZux ztbfTiaSz1b2%BFKoa)OgQEzn}J=UL2ZxG+g{hMn;g?_rph1;3a(j)oY?UrU*HsuK? z8LVpEU=SxHU3>&%-YPa9UE_qiJ>O8>+I3t`?FUZj-(4=edn5PirxhI|)TZZ_qzif* zX3^)S=nALN0i0p<40=Y%44d&KN`h)LU0OP^f;%FcO}hl`pk>=Ugv%z9+yjmC+^5^> zHUSm&)TS7kyV!Ps8oPRp8*RBS$b4PFo#P3t4W5n&`JuT&qx4d4>sBp#SNvtFdeoi1 zx?+dWF|L3N<=)h`2P*_S=EZaKzo!Xy>Ym`b?}XFk_6+BG z>^4`e8baHBQR6lwPpRB$DsJO)p+R`8U=3$&yo_@is;5>Sdqi29ZR0AQrV59)7}LiF zy@hRitLVf0%hayU49aM55xpwn7I&^Dg%cl+qTlr;Q#FrtImur+oPU3zp#G0Jw_9o^ zo&0`__1K;;dI@in@NJ$4J$h~>cUNAGtN+TUWJ^N09VgccWz7}16>B;h? zbc5-fdhcWU<<1Q5rMxHITOwt>|BtNAhGf0U>JbmF7SJ_Wj?R`q^>W%0861r)(^1RYsnKc)2*L4MN=PY8mY{@USX}E zh%28ukDJ;kWfS^!ozUW9GTrb~o}S+vz%59fUHR%qq+r*?C)|r$A>2QAipsvK&0Xme zqba>1LE6T9TzFz3)ugB^JYgG8#fiIfk=+l3Z|$$q@t+Cm$2ebZRh68L(HTDdxwn+g zc3H#uo}Wl(9O@V5snm1bhWeEcH&xL&b3_b+H4jB=h=$Fc+y2~?=CjF}l>WfZJ1~PW_bV;0_E1avkjfRGQ8Wu4M&DhkPEQ zy81M^c`*yQXoq;p! z=R}Bji5fzsyRtS?b2I7gi!H*+wy9iWdm?$X>oL17`YGk}Ntb(LX+nQIxRh$snaWk) zokKfEi&Hfc*>vKKQSSV|Bzo+GGo9W%p9_6A6}%2lqWxy=qdkH;sWl>YX|8AwlrIr6 zlf5-L>3Q=xN8_);Es2Xb{p-g#WkCtu8h@3``LvW)P)HHJc)vpEnZAjeXYR~Z-n?M# z-2IDo+B1_gFP5i$O-(p&{cH5|keAd29hz&pDq#~%C35-?8>od=x9NX+iPXp9V9rEO ziat(x2`AW(P~S~%TN`YC&oxFgaNn-qrDvsvbLJ%$^yt*z!mC@GtdrWNaD>8H&h%`F zV9Q&{$}(Og=l8ssQ(I9<3vcDnp|ic{_wy;bqdSDtwI2|eKR-?v+-;;v=WL_TpoopW zHAVRIql!(4#B%z+bP@IH>?u0L_=m6|MaXF#5bfm$rqI1vp{xdDO#4-52%okaaCrtM z+~3tt=#j`v)P*h8wEP>Ivs{}^2YV;ct|!B-lUGwB=2ikNHB*{O{gx`cRoTd0^9$lM zMt^Vu({}FSp*i&Im?TPW#DrEJ$)iopi-j&cDO%e{%x37eF-Kf-q`Ua{xu}UHT&wmd z{bo@l*HbHR<2tmIroQG`JJl|vJC{V$VU8x;X#D`U)P6EOah|*lvE?=8oxGDiJxP(i zEq0#XcytFB`1T)lbKiP;^SK(%+5I8++FC+5SF@3FtP*mGcFSnbE7gL$OA?h{Y@M(w z(VIJT=Mi_>h@-FM<eE~a?xSwaV{neK@yrXzbf>-c61VPZ!EJyG%k zU1dFDz2vbHtvOIf_diB%@{e?(?*RjD@a#rfIXjTk*?x-(lGo-ky|P7X5_#cPqE}<7uJGdSBtFTP)2tvZmZ#?xqtPD=Eco=A5$K7TRKg1(z$em39it zmq2G7a7+3T&rlY zl38>_f|SSxOyYd~chKuYRXEe(bYbBu2kvV67!_xJl6$G?MQ1)xxA}e|oonMa(l8iC z_ZD>tuU>4Ujz!I+$A8SHe6E&r&GXLEanfp)&mom-T6Bi~q%NQ)=ZDdPrinE5?jR@j zpEXyOW@>Xf&yFrWHlZ@`{c3ts;}Y)k#6HSJpdySY_7SQU8PY4WZ*i8YSA^otySTuX z4A$;OlYk13per-;gbk}CY4iQ&^pvBAgkGXMGu!`O*mp2pIO)P{?jkXRo|`5{AB#<* zmz`TleR$bT$wn4Xri#0{6T39HJAJ7_{e#D;!YlzL(SKcNeR3hE=6#uKls4o-qNF*! zfxFa_yJv+FA+?lwcN=$Y?ObkB;dRdOZyVQbmPF|XmI$BhT1IW;r_g@a2+GVjNO<{B zIK6pwD_uNdO0Ct_rAOataJPCpgo}QDptgFraI0*rXd??f?)w#KdYhFvZSqx$mS~z> zsXE<(YddPrnNEx0{21S!8%nfjCv7B^B};0~ndhD&ib5$Z zNcJUJenNKIM7tKGqD`f;CN<~GbC0E@Y?VSq*&>9ZY+2rU|DQi*J~Q*&_j6s>xxRyO z&WIQiyzx0?xur_9@9q|QUid?YUAd2)4kobl(Fo|uWTh3$CQDPQ6Y$y&3p8585a~-o z3Y$~Oo=q+|tws-Bs!vnRt689>u@*vWeh3ej41qmfEo6D5JU-&z!0*qwG~$N3)Z*(U zNN=%#Db4Lf-{%U{SNX!vl#8I0J{2zRKMN&p>bSe0gf4jB1INq~sJ?UrK6`zQY#ctB z#*FfWg4w5`R&fd}z7&R5Q{<&)y}K|<#zAvG7miEct4NB%1gK-v!W+Wj)1
    HdK3>o{_YwBeh z-S0ss)k|ghya-9Q?Ma*|;~wvb`$#@_gyZ+ri855{4SXDK59hu|lH}m+5b|da*p1JX zv3dhw%5o71$RFzQizT8js?ZZIA=kI{($nt2^yg+*sJd>6l{1yWIlzQE8wu!`f_jWp z_D6RW8=5R$39UhWRC2173Wmnwik1}Ny6OPQs%D{dz^p)IHY?)ZoKN&ytR0Cml$X|C zd_j9ooCLpvHe`su6aM*cqO@2tlysQtiQ08?;msjE+%xzZ34BiR*7q7{yEc+;_GLNk zSwi&w;Q|SQN$~N+OS*QbBk`L!3GTd#qt#``FuA)M*VUclj|MmkWsgz9&0;0#^@CAx zR>lGT=LeFE8>wJC%L$_|U4g$ob0Dp07SwI^2fI-xkX5mT9pi7{N_|%{CsG?r-sDm@ z%R$&_>rLkT-Y<#0^b^?B)AVi0V5!Hv15lE98OPi?i0P}U;C-nzwC(N0QTKP#m@)Fu z-g_UGj}qgnuT_xJ<^a8wHI%oxgSX;GqyEGs3^=9>&xe?a_Ju~k#$qcle6R+`R_l{< zCF|)a{i7uB|7^a&*|1eE3)O#{QQzIoq*`?ft=H0$=KNU$7t*(b{Dpd4H0ut{2&p8Y zj``Rd+eQ;571T9h0;CF5A$#I#blzen;{bmrpl5{%lgi-f#A~2t=0!cKPorRQ8;#s- z3y)5w;oFQULTTYzQc^u!daLye{ys1XTAnxJwAlog{OhBlzcG+M_lx9M(@kpfZ6N$m zhysJvdU*A3BE6P22|Sy};_mhg4D{Md)?`?b-yUb-;av|1j|t?vL%QhLb@{^nJR3Os zLGVAX-&IGQSmYQ~)g;~YxMt`-Zo*WMtG+PlC=sg-hm z4*2Mu8+5GvL&Po`qH$?{#CG-wSlyXOPfYkiI%loH7UQEN`>{P4@!B4Ww)=raa|+#U ze-*9^exs|`bu#WIN7C$9;hNk@(3KbgFXww;V{IAy?YBYGr`d3!a4ubI>`&&D84Fk6 zP5|?03(^v_Q7EB~Xlp3br?qxcp8Uq`b_tFDgzA)bW zJA6Jn0vDxuKz;fHa^;b1@0&LZzioNN|DAD?e!DN@a0WQwUHt^I^vovk3NOJe-Jd|e z`WYPakD%8JSHaJ=6p6M|GO$iNV7uFVv}>4+nPNSB_jx%K{yZ%yIU5C^x4$Rax+(DI z>>$XMW`I`(&(5$R|apvQ@*)SJLeLPALB!E&BLK-Fgm z_{{48Ej%4TlYXD0y^J}K1Ff`q{#0Po{b6R-6Rh8q4Lj=$Y5e>Ik}&QH9&hl#xxQ{_ zqL(Z@@cBC~j=V;+2P#QxMp3LavcacMC&8h;+l9`Ju|TR8;D5K8CAA^3pzyq$EUs=P zv5gmC_5CUMJTQjTC&fb4`LjY#R#7xJegx52?g{I=R3P=vP5RI)0|U||FmjbI_%Vf4 zHE%e$ZA`}9i_=N%lP3TV96_a_p4!W19bMZ_t>uuOE#kYGR=!wTYNaKM8`KblBHF0XM}sgNw^XT;Q8cV&rB*d!PkwPZZ&4xs%Yn zIYX$ivWPlr#Na6lYuGw+Juof$csRQkcm9_sd0F3wU$1m9ngOd(`xBu5wo)1!uO<31 zV>59*6(VDqR^sCS4#AnL57?PCKcQWxnm9BppyluWkW)=Evg0cB?48ENs2S=>`V#{SA3O)=5<@mky z(BEC?^MLZt{<&duZ!O&uq)#sXx{RajDkTHHwo)0VojLk&GWx&#D(U}XhE0v*A$ImC zNcb=rdmS^$@1W`AUEd3G{a!zqw>_p`-)oSSaa&3F^VQ(t@{4Gs9)Pcx{PEDR2a+8J zhvK(IF;LLeh9dJN5tW@64 zhbZ5f2El;{Va1W)Q__Ik`)b@iei=;9i-YOybwa;0KgmS5BB)!DOn#hLjoyc&=+LM3 zxGQNI2#g0}?N4L8(`kYyR$EFoUrq*#+5b>!p*i^HUL%F`L%{srb9les71F+@l4q}z zu>OfM^rSREEgnMW@r#L&h6oJ*OCndMR$-6W7L|tIB+(nEW6AeocKR`0e6?jh-nFoj z%!+zTY#hBXYzBjoEj^NB>*i6b@|pN<>rg!H?+9`4$v>p$<8b@C5Q&qFSahl5H9qU8L|6U4 zpr#j3MSs5Fcw0@_>6r+R;wcGF$cJ-&rkHTT3ae{nuD@R){>^QAjz+u5*%+Ql ziT$ymuw{T1&g@i_W-nD236APZ4_8Ov=1FEE>5)&MGu{&K5C6x@RDB$~z7*CT?k3yv zG;w^-Y3xzgfrI&tlAD!Dr1!NLL+AKGQr$>8fRU5N@Q3;9YXZpN=c6Re-D_ai#jUh* z&`$a`_X8cPZUoNfy+FS5Ff0p^hhc;KVbIGI2=?`+tq(n@`)?J=b@N7C{HkAKx!@Aj zup2C6sDCG}Hs7hG%|N>4SvbycJ&o0)UBTB7g>OwmVBEz`FrmUwG;3f!QLe3{XHM{B zZsRt1b+s3Kj;G^2#V6pc`GACO+Xv4q&Xdb8Ps5r5W!N!8Uv`MBg(u%8!}=A5aPj(D zuoKF|`6^SqBh*4G-BehW5Kn&3TZsnAH--O}Ct$W!gD|FiHynAnmfRULj2?}$$HcKO zVEeU!cxR3Q)_$FeIcyDaHf#}Yw9SS!4Kkl-TOX~x7)VP-^$S*nO+(*nPf%;PDmb~? zQuV7Jgv-AyLItswl(VeE@NfAL{zw-W$UGv)RbyfHVwwM8>WRnNjtd+1@$~WL9$|KM zCixdyh2wVCV(suW+|W1{gYIgIzE~0TygC6(^0Pr9%u_Vp>u2XwCwl~ga$Af`4$;A(27 zqbkUxhLa~QB zYrY2Hzpid*aET=intSD5*g%7(Oo0@q9>3isht-a}6-q ze>T{Ub-}^LilW1bov>v@Dd{Qb0=Ye5p#AL`?Kd64)8`?QJl9}Jx`7kEZjB%tt4zS$ zzXV^rj*%hHtKe|mQev!8fk&cL(PwM~c+Y!+d#)cRt3CS2>9gfvIsTc@f5SkK+mo4Mr zrH6^ih;^7d*bo!aZ^GOnZBep*i$wL}T5>?uo(x%Wos9jGNUpw6l74rwC(D*Bzy(eJ zaHIVM;>&&~s<-r@VZtUHAymSs#ypZ(GacNjtZ~P=>*zRaBPp3Q8m#Uflj)-;!PMOg z?|fSf@n=sG+R`Sx`NmH4=R+o`7_$r(RVYJRu^PQvu?)}UU%@dBM=`H)6S!ndkxo5f z03JlQM>r^gdkyEXtBjYf=K0X#-FuvHcHy*tPJ*Qa;w?Qd%6;9w&aqQIR zG;AhEEmy9Y;k;>{H|XhuU3XwPBM zD3vrszNgeO$O9QN?!_aG*kRNk{ z_?-8_u(KM{SyipVHnnU}suR**A2z^@c5isGWDENKAdn0Aw9?f8q6%q`~HY5@4RDu!u zMs(PP{qVr|G@gmL4Q35qBwO7CzBSw-6D_qd(sD5Dy8ncDnHCZgtNEDgZ6ZCkco;mL zQ3O*eeKGn_0(p5@N92{A5B*1SNsMI*^esJ)#wUwuRilb@UfT&!sWL{p6g!dCnLIk{ z%?UV{e1~lOvlB~3FDK{bro*>NIXq_knZ$)S%ec0QbX(LvYP4}GY<7Es)19O6-j`u; ztLikl@@opnaY;nOW;mmjd>UW0G?NL#F2mRxv3R!Q6f{ngm);n494u87h3(6F$@VSx zSe1wRcz;I>&CNSc4@|m+YnG-HOM{W*p>7`L?Yt&jJ?5j(eM}TS*rJAW+=^JIRRs`p z;4thQtPh>@7vlM;N|-n$9k~l)RG<3{jK0WAtD~2qYHA`Za4Dg|Pg3yR{h7Ejv!65- zchmQ>Y+v@%pHyvgx8&oG3GiX$2XboRF4$3a3eP)ECeK`_VA%HSWY!=9TR+$Iy@Zc)k9cM$@hs=N-z6yAD{xR}OB?N|zpF)O4 zMv*?RRd7FLv~;P1`rA!7x%F2{=5`tVdKgU(9z#=0YU7`AQ-DU;lWdYRsp zJJ}aYr!0lTGh!g3U@`0vD#M*;8=&dzCipl!49;u~A-eTfpugP$c)KX!=8RVOe(ft7 zPB@0&EIR49(0+1KO9_{#O_W4M9>py|&2Uz+QZi+3sW8dWk9f_mgtl+7vbl5=Mtg)~ zetACJ^lTwjg4u9bTOQYKOTj9acu*fdpY|*A*s^Fn@!fhKAG)TI!N0VGb;h}%k(xoz z9?L`S&SsjpN=kpIJ)`?&cNz^nWt4xAO{Z?D6dqczjW~=7!~^9zByRXeeD>=Wyt{pa zERGL_m09+X*OG+C`Qen)e@pa-@TAkz3UXcn+fKGW2}-n1a6 zoKEas1hzqIaJ_LLJ`4L-88@(yyaWO)Xe81-5Buq~YK+w)cEof|+kM(&ekURGd_Ku&skV4xH(}sR_nIWx`g%MGPVnkJ)ZyxF)QJseG2So8O`sQ_(NisQAS>IZ^^f3 z8%TI+08D4YVOcdH>7yEH;}Kal$AUwt#UP9g=^!B2M(4)cV%_Yw;BZU@?naI$9tG_X zd#ee%EOO|Hr%Pe@J2M;;SWYHfU4d`bDM_ne7)aY}0qfNlQh&|ks4_g0ehx6kncsd1 z1$%E`xZ7Gu@${1#<2uQdDzP5wir`TIl3X<~FOq3= z3O@BWhhhKhM5o5+ORt>YjwO#K!Nohec>HG{eQQ09=#mT6xn`1VUKK!4v9joFbvyl7 zIS`nmk;2|7+4Sa0BaED8i|J1*B(BeITrPic08N$f@t1w1f$dmbnFIM4(pvc+)=NkUvF>6d@coDx7OjE^my2D@gRQc zGl5e}x50%}RlK*>jC@d!hSVVdl3@e+-#a?sg|!HleDlJPT`JPX2q!vlusv~V9}JW2 zWf{t4pGn{T2~yp>BDktri58MG!syx%*ui}#t0pSo#4F8wv`-*7zU)A=pM`igzzU|^ zHj~{|hlu`pm6G>8XCO>B3Cx1qVZ1a3_B7nWX{)x9E@dyc5g19Um^1ccy~beOe-oi* zR13z>HH7q`qd`@CkbJQ<1HaKJxa5!`sPEp6Q>V;B)lLVpeb7F#sLhiWH07bmu6IHY zwY$ReT2gXx-ZvWXwvBE&D34pbdSO(n8kV0MiR1pJcTTTJVGC{Z_E(tx0-;?NZILf z=qUEioWV_w0o7xAw^1 zHJ}I8&XYU&6oLwmLPGChd_F>rW;|U_rWJRh+ixj3dQM8l*-}ZO-dXrv@{1UiyNZ5| z_Mv$LZa{m(RoqsQLsPuh(F0%CNs`X#;oQ-&psZ?+>kgF=D~UNwaY@CO1;3~pv{L&i zWi;o@IC8wM3IfLu#*F1r!Wp-9MAyn0v{n2|Hme&#{`zRxbo(>})&8dQ=V#Jk&qiRt z>Ng}&QJDnx{3IC*Bk+J!w(lKm2K72aJW;*`I>8odj=iBVA09*ZGCQg|Z-6wV$p$8d zM`G>UIMmWhC3b7qLd};KsNC5{*%fMFy``UIk4r=6^WVtzf;P$ggJZEID-LcuBtdeJ zipb2dn$$TifS^yI+>VZ3iDAbX(l>rS{M*w@E=@g79Nn8qq`f+(`V7RIbPus!>4EA~ z^wCKnkxI^~V_%yTPZ$OhV_`fvX^s$ix@Ay(|17AsY9udB9r5bZmvFl%9nihj2sva4iggasgoh{T5*!<6Go7bIUEd4uVP<>DlM>27KRqM;&(a^-c3&w=KDo~ zxyl;W*=9Cq%um9`1Bs;7%o^txCd0Xcl}d`7UGc){p)}~{A7R=#Lusqp47fGyB*+!((X&RQMS1C?L3&>WNN*5b zS8C4HTlck0bLIo6!g_Z?L(gB{iE&QO~`M&SI3MojYd0U@EkX+o5X9PCsYJ_MT3rG3C0`ZCxw3hoPjOd<$&wG}U`6DCfCQb?c z4i3VR+1Ef@!xA3wb8voblVsTMcVN}PL6F*7cpbL~em^fI&vYM9d9Q_{f(h$!oRR^4 zI<^Pz-8F_U+egFPX~%I49K+gw$!PN>A2x4jk@-#%5XgK^kGYCa(HKE}qpo3+av#pH zkxnfgxfM8mDG$ZOXXSpRY-EFN)F<-BZy{l zI=`Ua0&<5P!dddN4A81pkghewYwtXucj`q+U_=4^ULcRgm_HqXPTuhlJ)j5ws!qTT`|arX!xn$(UV{gQ8|Wo8 zCQr(D!NQ9t;j)`L%#>3S1tiqN7oAKPRImb4E*_SoDH}ti?PuD4{sm}E(#NUK^XTqK z5n8CF!N|MKq-LxGs;bVQkaHFS`?JwuK>?%ewgw+1zM~fJ%lNgE!f@E>OK7nbzqK(lmMAOB1;am?bP|T!NZq>N%X$0(a5CbcYFhMg_3WQSl1{0SP$PAqSO`ngMipHYtPhrZFU!hs~S=?HPT z90jgj1TwqV!O7Ft@zl>K?Ax1!ET2W6ypNy{`XqEqODuhx{g*tMG89KPnZOXEAN2Uz zIiP>$050`Tf*rk6VZo?W=0a*PI4|0b_$LfL4QHr0Dglob_(AoeG~wsT(;(-9EVp!_ zjt*Ebl+;`)C8h?l&tG&Bf}{>Yt2utW@!tV(Z09mcUNk~bzZ10W+W{B1ydmx1gmkQ& zD`-@TiOIsL^t4GTXqw=ffnlcQAmul~3vO2m(0=k3qD=6w~Ft30>_D z;?87E0>zH-pm-oY+`a&!#XQaU8v>u7y2J4m?(?+H*7sxhgpF`;#^rSZ!=oozW|#CRino! zIca_AK@jhj<(k_vQ5s!>J1tzmMok`e^$-z^tb&qPRb>8}Qc`@e6n*@6qFU}QtQol# zZGXI_S>-oy$HQ0H`7i{25@%!-KGP8qOK>Cq631*CD!gqI1P|gS!f2;{$(CEApxH?F z{M2R^KCW(u^;Ul+eR*qO{*2eu<5eOU)Oq7~FLT^}HVn3}%|!n$3GEL_q{9n3Nk@?* zbeDuu>D&V0>&Ii@&D;~PYo00T*&UA>JD-EpB#HVqYshx#ue7{(APDE@P>+f2m_A@M z=p8#NT&J`K4gU)#A&QUj{IYy_TCOO)e7l_b{s@Akmxti!UlY+huL5E(IYn-bpCh^S z!v`NUX~PG7Ju;=soLabtldQ*A@MUO>WX~u&sf*GJ+;O8@qWDMV4efo8E(`1EucAQw z9)3vTUUC`cYfi$1E3d%r{UEBib|P^!vc~>Tz7TsZ4Y%jE5W7g(d;S^+w#y=+=JaPe z(5MrO-Nr~8mnlmPrw);<6Mtm4%5p-Lqy2=FzlPzJ{kJ83GXsDI`@(#G1Du$tDDmJc z==c3o@D1sOmAlhu6w4#0R%>p!=&#iwim?srfYmQmkX}{bCi&p1vNhJ~P3?OJil) z$$6-@xC}}~NfFU2TMGFPXt_gf&0HkfP z!I%tP60)OSc;4s)UYM9F%R=6vA=cO7^ya(5wvJIa)JFwMEQi7I$#x*pGm_4ygTbd? zHd7ie!V80spc5HQCKe8b1s}R;#i1KC&+)J1)3k-S%Q=GFRyIW!-#+4=wG;LST*Znd zmt^_TRA}2(0*iu-@Qu-0Jo{7*U4|&IRX=ZXZt`bHNoxx1(u*X~GrI6df&vy{D_E#f z(562yRl5?VUOxoR=@ZeyFq-Zq@)(D1;z5^Ld>(P0`ICN|zLbAML$i6Bd*qaGlvXXg zdv6Ml>+JDEvySjzma^zM`ay+dBD=z24=Bf)3qOr&!0C@?Le1GQnXprZrxwOxY8pej zv{u5+9)H;WA{iHKJP-e5UijM&Z~3#rQrP_8Ct4DiPsAUDP&z+}jIcd{J@3O|@#j`p zn&1dudetFhUKS=#JB0rJ92+-lJDoa{!&TL~R6kA^lPou4S85IF?jC^S8l7?5!xv=v zvsD=2w3jGtUQYXhjKQHm2OFNQ0<%*iiBF3xbMa0?C|~pr-bcTule3SKvA@2dNZ&>1 zHc)`?{q0dNFPQAUdmi-c)nxOFC%3)lz{(}B;K#s4`1sdsI2RNSxgBcK^f*7T+I|J! zzrTPnGuz30e^=O2O{sFvFX0WlB;m<_6G+Ii3@l82#+Bvkkwr>JQCVjLTuq38zC2$@ zw7!hmA75ghmOJ#^Ttkd5dXU1_T$LImk-pS(FPC9mwCo+ z(iON=S%_~V8qu^f2$XCUMf*^jm~4#2*ke2B$7v44VP7uh>$}5|2kWpt-4I7!Gl73s z%%HSh6YIz6(!d-Q6z$U>+vCr{M+uLYq8p+9#{r>|?l1DFWHlN;UyaQrt}sa79G=qI zXqDy*H><0J17&((Rq|UH85K#Me6XYo-tB~+$%pVUSm07~A9C*NVA!{RF;-rPg&UW3 zVUnvgb#+Y?o_CZ2`!bE>HLB4~y;*4Y>IZn9wZWS9&5)%$43=0&;Y(R2>R4G4J}m6R zZIUWDdEW@MBJ8QwH6HgiWPxja3N;nV_UdJo!X-;&=dIFTqGIDj?~f^x`DA(I!xI%) zkbhN}UvEK!-nR%Zxvhor-eF|%&TCZGwoRVWZ}k29C&IW+5jt76!1DxEoE}_==0Eb` z_~mNa<-3$W<7_Tnu4aV+$JUXaoE%6s5|L--??_(JW=LJ|h3uSkn$LQ!XatZ6)z1dcnER7(8;Gpt4>BteJ6_`m`;=%(6O( zCub|t={^V#lxE?+ou1GUkw8tFIwa@YXF>Nn9&Wz1!{A_B;ZSuwoSE2;*5M<-(#jb+ z9Bt8ZYY*-<&?f)nhKmBy21wk4oAHcgF~XBBs$-lbywG-;ejTYq&iqQnJ5T-br>7i_ zbaKLu@(gnP({oU`^%>{S-ADBfJt0A@wrH?LOkV9@gJl{%bj@^s;x|nM#=b+~h0GhD zBtK9(`qV5;+~$G{S|8Gt;xnKt|CxBzg`&fpCG@(^8QJXmAT++T0()QS;il>(?8eVo za9-R%wAyO%N!KF+)YrlxQWMm>P)Y8+lRe+5eMJ)5 zzKe|34aHq-4jKLQAv{PQE!_~{O|Gwg2_B5H@Z^a7P$c?C@?P#izakIm{G2|z;z|%T zjI#mvvg@$_Cl6)ZWx6-bn#9PmA@gTkK(ByqdU{7JY}*$HQ70P7%|q*j+4p-S^@~ul zbM+y~0jKTod%zzUbL%{f9%7EayR*nH*K^pAXNS*^C`iwq*bS4u%!UZ17V@^F1Ga9t z&ijqD0mWX5k>;itH9^Q{II4h+oi3hY`_N|i9@s9lfWFaJgqQN~!lUj;{A_WbHit;@ z(vczHW8Vw9epjJZE{&Y(cp)scFC^++x?pIc3D1_DAPt|-3HSJC0Iwj+=efIs>fJ`z zce9k9m!F6Udj8~q6J>A6?FVs^nl$=v4GH^u4OE63&;^T?L}~etFt9I+zJ2Ko=Q5S> zKg(TE^F|H7Dsy!H-{+X`@m{ER_Bd+S%;b*>H}OjQ3^-TgD$Zlna`xwTFV^H^0kcoF zQXrDCwahvOvwHBve)h&<#>He4Gf&@7e4^$6^KMB+MG3hsu8&t@B%NouwD0Gc|Gp5O z&o<@#(xsf*xHS6}W=?|t(j8^J8aCpKMup6^BzJzph9G|7)tBOPL+xapu`Fjk?4)>8 zR5drrAet$hQ_Xa1vh1SG5^=6$Bhz(M#QZvDC^p&iR=mFb4aXD|30h)$>{W|cX6C)c z;(HVHSo4Kj+4J{pnMUy}ChWY3Z>dn^Pya6B))gADd7&ok*IP}@p&Ke}aHRz|FXSK7 zI?S0pvLl>HIbg?XbS&jZR;=gm)!g7BbjI?{qYT)sx3$^jzosyA_IPq{fUp*)*RdxH zN|?C&Q`vzJwON+{0qYR+#V&Q%E2iSlDdzXFoU5+$MlxC2AH>&Y9OjQd4dtgkKF3wO zQRS0=*|0xLrm-6Hr!qs94r81i=ZItLBE(BvbD4vZWX4jVf@zN`7Ay~3!T3tLxJS=q zz5o6C>}dZmSyG}?95h0WFOsV8zlITJuh%p#@y%IoFlWg>om4CsIyuR{(5OMM$;O>i z?UWO|kImo&A(zC%{+-~AwX?YEGS;rjLtB2-Kqr2+(+zI;vWw!G+eY)7&s~=BIBGac z#U_T`zMDDO|B-q8&YGEE@|uYezT{>H$8(NN!+1xJR;E?cjIVvG$p;*7;y&z~Wq+@w zojWK<=hUzL5ieLXhmpi8@S00q`F7PD?x5#H{>8T~vSNxzZr;l~GIo8Q;PUE)yiThr zZ$3GaTe@6}(O$5O`F10VdHQ*=;FVJzw@$`DR$3%4D3e8z z(Mnuy@^I!2TgRzI>#}>N2?bvxHQ2d(UAP?majbofA8RF=#feHi+3?iC;@~&I+;Ckt zF4Q2D6Ux5dUuhm}Zn+K%nk(&U7p~Hnm>E*YsK6RW&&M>M81DZ zBHu>m@K;APGskSh`NL{^`L)jt`7z7YnAbNexdF?PxWI}i&RzSqKs;k9^J&XpCcq+8 ze9c&sUD>10eZA_=2(?YbuZ;?rE8nZR=^bym1F=UKOTV*Rg4Hl~j;t5D+w%i6iyOzc zSV|e~!kKKy&GpQ~fjWGK$7a54mp$J;W;6e7(MC45^cHs}nrG~UZCrElYQ|;2GS-iA zVM`<3*`FzEc=^VoT)^FI#=1(A+q6s~SeG_|H7M#~1e&wib=@LfG4DU-Yf>y@C!fYl zewe@=kMH3A9ly)@s#kG)H~eH0M~&y-B#!6&^B!{=9|yD7rp;!BJB93yU(?t*gS`1G zoE>*+tb2uSkB)e*v`w&if({cQ>pKq893%EJqnziE^=!gf2lj3IRJNmhJ%2&8fmM8! z$UOYx#@;#Z&bJ=T77U&H#9m?ICc*BxmG&Doqxr_b$$a>|o%}16aMo&Ali;CYIJf6V zi(t)(g7Q@=-#PV#vl!z6ecW*E942k!C;Q7$d%1%br!(8@#Q)XvUVxIL=>=4`sJ_81rg7xAG2~ zZg8ug1oAmQHnC2Ae=43C>}6Ky{Sm+VdbPsjV>zdnX~~o9NY4Ac78|B7;y*S2<~paE z^P7&_u~F7$tae|FxTi**9T}j(M~}R~ymvGZbS(E~8jc*{<~w-uS1!Kc*3}#H{x;Lu z%pg1V@$5OA?Mi1he1jUlcE|{RxWx;5gX5Zfp=DDyFrZMcV zzMo8B*fjR?A{+kq_l2yMjQ9Jw%#F2*_T=}N2k^J5oq5aY?!5i)rR?R%_1w{^mi+Iv zti3{MGZPDoxz0pQ!H?P1+>WA=%;MZ`X5D2eGw-Yd+uzX7t+^k<=B$3q46_bkt$&5^ z-P0m@1JQJTQKd}7ULVdrjf&?t-SXi>3U=`eKP0k8U%K<3-`McM=Lv6_QticJ#(=d{A~g-|A&S>qg>XO&FKBHjXo|RS|q$ahM4| zYRNxbKa`)-^^psT8OOfptzyW*ScZDE5B^+blvZVMuRm6Ecm5sY z{sk=*+!!;1U9#JS9j(2XZ8;w))_FOKcS-lgAHbgYa8{F*6L zPvqj~#xpxdJQLskvxv_FIVJ~Nhoy5%s>4Bx}AWL5Q$`5+S>6Hs(~|GZ&aamIhh`x!Twu&6{nBNU zV-7Jte~w^_CZ{tOTI0DFZp}>Z#0)0wrz+!^nZntB)8U^5C@_n0XCG1nh{lsl-NUojwX7W+uJo_#Ral8v6O z!Rzl?#@377*z3M7?A7tDT&}`3@rVbN%-a59yO&xc1xvhSjGEy?_>(J+a31N#Y{c&O z%*^#8*r($Lva9^cld;Js8cMEj+_Wl1_6zFHjuipUK3XEUsvcPi7(yvyDD zyjuJP@rXTdxtj+SSh*z)+$rU4T(}50I?0f+Rr2SpPL1NHp7-ISs`_LstYXG_`6dR} zcQGC#;u!6BBUpj60~@v?hl`e-i-lXRFmpoBF+R4FI4!1x3(bC3@jdQ{n7Z5%Y`2~x zetX1}jR*khFpqm-Tat z%`KU5&1L-63D217Gqd>6U_agn z%m^Ddr^>c>;eQf!*qGiVH2h!;QbF%-Iav%Q#<%=T0h}W8RF> z)#j8Bn*;c*Q^`Hs=L>SUrUX7RkajOC+Gne9u&g-VRM+yE7SP^ zonG!*^(4Nzg|HoeCbQ)?CNR%ljAMP;Ci3@VmhmyCr}JS$jkzbs7O`tL47Be`i{om0 zfpvQvYM(wajGeqQm^BU!W!?*R^S#bnn5MosM%dUc?o8RrJyh7hchtM^>eQI8_g=}) z8uOjeao)~FMX9nYE-CRlbu$=$7k#E~+*;<-s?VJFZ$EzFTQ`1X?;ft9VkukV6vF7t zk#V_a|6o{2T}5<=G5_WK74CrLXKt(GaCYFBZTwp`AD)>pi0|IBhaFnu!qm98a@sEy z+1pPoxs=3U8AHdHAJS&VTnb&rrr!w_d^m5(J)hmdMCi}pw&*GGlMjyLUCcf(iwO!#&Dqz8pNXI zq1>CLF^pTP9LwC#6TBFu#lF?8;gUj~d3oUiKI8^6-5cFl#o3+A#H}wFo3c;b;AyY9 zaj8Su8Rpu8F~COmvoE<`X3)iEX3&*q%vSn_`_?wLqVC8yvDb#njJ%yK zvm)7>fArg!A0M-lzj}Qx>*e`Y#)rGX+?ZU>1e%s}-Q@wiidPDEE$0Upmu0{T^``R& zRc7)-miX|_nR%RRTNiU-#}szxXnVF|S3e_H{hkS}cVqg?t}q{#yyWIhUch_2S;DL9 z+VRCso0-y}t?ZCxv)Q?m-PlWEt7Z4ANOt#-ZLAJAgMGTmgM0Npj;%fg?Aj+j>?7+r zeB2HLR{xn6yI0Sg9rS4{<53*NEgv?ITYou^8=I8MnT)>649v|GY4Am>H?g7tcI=Wz z-fWfj9WGse4BOqC%1zE+!#z`WO?4?9g->bzHx~;*P6!Ytx{qCds;JNNOJ zxquHXS;2b@iDhou1v4K^o0uEvEsWKqN@nVuaqO>pCwACo5xd&&g5X_N82{{oJ3n!) z4S)RjImSy+!Hufc(?YEp{ z#&suhrepw4fY8(blO!2g8M{}a9P0X9=i`c)N-t2Q% zcQ(!VAXnJ5ioN`73cIMunZ5kt9_KaqnD~rFm*8a1QSs63x{N~GFm};%FSa+;jb&;z z7>TclJFL8$+ie`neSbcOxzDd)M~<{)<5v5#HJtJIen7H|MfD zr<^^9cQzd-&g6b^pYJK~he;bZ#j2VMPZ`hW-8bjsG9x&RLMK-A@(R=9&9a_V27J&O zIrdR(E~l39l6zX>%#Zvvo6UK!hbb|T*w<9K^EdWi5Py58z-Y3ij9p^_6YygW`|FN_ zxT)|2^XZ2lA9UH06@+-RTcW44F?+HF)22`3Hu|aY*=PDVjn8@9{j87Nv%t-q)5eLM zTKhmY%;*#MKZ>rypQ`_jM%yKRA*g|24{Z83>9C9UrjE+sAviWKBZ^y~1zDL|y z{(s+?@y5qYamE!ksaA{g;_cw_b$w`^FAJi9nzUr`Od750N%s!Pq4UGREMQ3>#2>Jw z^5>p(_QG0>U+)YRPbblwuQ0df%O8?_#`G1 z4=a!{iln{vec_?e9enH|2Mu?0Xnk7+e(rOa4^z?sJKbJ9oT5b15C{4g#kgOSM$-)& z4Jg@bN|k@q>2+~F_g7gTM(fyc>slQk(@mE$PdHF`oY0{+)dr?qj-q2RV&0@wo&tC4 zQjD!8i_#kdc3%}C%^?m_UXG#T?-S_9{-X}I{pV5rW>5I}Z6>@Mn#tZWf4bAlAbP70 zTyAMW8R3p!;+gOWhB~EWIqSN+C;4^y>7_T)3)&Aqb zSfa+rEYANVr47hN;j&?MZU+CN4H&gbfqAO8%!k86|oX*xOXuz?*OxiC`b z8<&aO%P#ru#k}WLT#wFaJmzA?t$emt4tif!QJzPX^Hs0d>>ErW&W z>NH3rnVglU(Yd+_;67?59lG0!p6cUC=2|?Bf2|Cw!m^-mlO+Ur0VT%Gf`&P(QFiW8D8p^9RzDt72$K?hm_fOvj^jspQK!;|KLX$WpSUrF-Ynk)Hi{?UWcgo5xXd zxhCy1ACGa7lj!JqTR5XV3_i~v27?;p@Xb#zkXJ)JoI?3HJqo+nm>IvA;Yh#8}dfl}@mr zy_79$`pw5JPBF zAuGyWroq&=+Te^ySMa5NI9VQag;`d?Fgo-d^BQJL%RWVdVuL+qe7KQ0f#!Mq8=0pckO>+nSSy_r~^%iy=RUy*#B-6H&2b~55y7%fW zulD*J?wm9bYicyOH-_mH+rEG_Ph?PX%Gi+2nP%NP2tm2I~XK% zAkLZHg1l%b-RNk-rY~QZ)mBq@|5O2T1UyG3+LV4ei^Q+9ge+bTwdUkOXHzcxESn99 zCFx-EG?cRb-vOVwh4XmwiW?cw%#XP`myUg$MiZjq=*F>mVEinHrhZ&Swfpi&uRIT$ zvfuFjR%3B@tPx(iS;h_XSPt*PazIl#3#=bx2{`akb^F*lPPu1=$YgpW*F8j&eAgG_ zK}R(zS>7Ka+;T{FZ8qKS$S2KHx$yd+EE(*obZJuK3ZQl2dxgKznmMC?0Wzd(%a<`FA`-P0t30b&DXNdjZ9nj(}QCGx(ct z2EoUN(Z}FXFy-_p*snVV_IhhmK?aZ8lE31Q`aP(9>OQk_a|O$}!hBe450<49$S5bB z>F3C>t1Uh(PALY%EqZz9>|p$u`HUY`s?D7*DrM%ktC?5fIp&bQ8Ydt5jXBvb@Y|Da zv=z6rzQ^D3zV%I*7o&o4dMCJ!h)JMyTLipMFeLc6)6(siG0g4{dWPIXXSLOARGKoV z$*)mNsGJMOn`}6Eg;|+F+o+IC-5AlONL1GuqZl>+W%ht2t z`^^R5GeHv;EtyRgvs2jR8Jn2h?Cz=o!V7W6#&PEkc(8xQ>F}&D2PUl6 z1cM9nNj||4+O6KR?&cP@Or$~=6@uX3!Nu^OatkZYvZiF&op>zxG)_9O9=F_;vaWD{ zXq)c`4VxmWFMe~Q@MJ~e0@P_?rZFCURK!+&%waRtwQ%b>5AIMzBfp~bHB%aX9F028 zvVNa_v#am!u|3yrq1Fv~PBQ#l_4k2$*zXQyoEw=VbU+Pa*8+n$jc9iqKfo5cHp#-1 zo;@seh7S3Ze8Gn=zoBCA2EKO6b7sI894Xzzg2%YRNc~Ba-J8aZ$dT|TbIsVfW_5n{ z#jTinsurKve&kg_w4^TtXRKj9)j`aL>jQn<%kX=; zIjHt}g30AvrqF0iGout>!f*%pw_pSt9`q3ZKD)+lZR`UDPZat0$D%RM-jZFO_JMmT z&SIxuPr&T+p0KsT0cvA4U`28OsqS6`pYpOH=(j4ok6TRhf0S`23>UH3C08);ge}8j zeQra}aei6r38t<(7?XRf;QQTk>_eLZY~H6!R*ed@_qrkJ9?QiQ~`JU@SsMVO7Jp`Hb#n^;Vj4btf5%+;X$(I4a&#Y1!2W;U(ag}OHw z!YZMM*Zg+}OGz@ND~qxz^+z6^jUGkkg5$u}*auF3@Pv|;mZ1AOl*&$+ILR)Yi{GY& zGy6X;`DH1;nB#n*i+GO)$lQ95S$7WMo~#QPvSEMqznV7oYN-Q7l{nC_oUeGBvx5fp z$<%MPBbjy&pvi-UnbhhsH+1e{=41O2dvI6vr?@LP{rq~AEzm&wC&Z>7TLP1^L*VYH zvC#7(iEg#}Kw7dq=(%rWp*i-n{GcrA#o0SqI4;E63q|~uuoYx&5=GyRSBo}3O@kK2 zmGH4Q26F0_V^raMO7K>p+Pzj(P&x-)FB`*$yB_o{JCq&?-Fv%Mj(}0C3fZcyZ@KeB zx8n6ZDwvz0OyhQb!{i;CxRkSMFlDMp;8RVg@v1q?{xt#;SBzkWhn8>+mtt7Mm4R4d z;f0eI`|&L+*K-F4H?YMOPPDph6b1WQ^BuvSp!Qn;Ztnu%hC>VL7BbqC;Y0OXJn(Ma zFgh`mgLjpxV6s{ZmI?Z(YQQ=y^LmMv=jEtZ>jmHXxi74i6?AnU;L5jDvB{f{IoW^I z18-Lou<&fg74v`NUai@ftq8bpjTzoQiGF^GbUUd1s3OT@bmvuby`1c z4(E5%g!M*DWADM5>uZpMogtb0YhE9A4Kf9IR*Ka-&FKAU2mayANvt_w9%{e%Cz7Q5 zv8H?}iyyueb%m~*53CzsTkON()3Pw|$O|@X)*WWc`>->uvW{sxk6~$n5}R=`1baAQ zhmBNt>rMfSb(+Hdw@1*Ut_!tNhk|pWIrQ&TWmmTir@q^U!t`WinDD+^bau)w?Dajt zROTLHReBZ}+aN)|>JYSj(8ll1@~GPL)s*#CIj}ZJVxKfi@x}NhxYqbO>#`Zo(u$L&!4Z40mEeNJRlzM9waoCNK2LSUHVX#9j$hu20 z`L{dUTJRrQUdhJD>)|*>CXc<_eIMJlJ;$)}gKX=eo2>o3GEV#P5Pu5V#kfn`*-ybM z@cFh4*=TE5$G;APE!7gH?EZ`?cne+DuV$dkiHWq+Gl2LF$nCogFm&f$k!txsUe<1> zXra(Cd%{-0_D@y0kLI!LaQaa6;mqMo=}d5)l#TJ5CsEsyE^J*U#j{NoI4?y5u6LEt zjqrH7!yB;5Q!7Emsz%WDRk<+nYWAaeGA2!JNByD4@%4*5+|TYaX~HoqyMIc+AR+9L z^gR1zypjrj%pzOEnVfz7T-aa7;N1{oxSp4b4?BAB)EMEs5Omb`zFOF+I-bg9{;=LV z$Jy6qO}yI>S+ZMgOsPsbl)59k>QksN=YI|WxvB=X!JvleY*<1A$6mvE6GJhe=(OnC z?#Ha5S|4r-eSDW^?d9A*44`A7+t}i15?0Q8u%g}jF(mdZD=XB1rutWGqwxVWyC{Nb zwKHM#GB=oZ#))iaoMvxTRM^IuR&2=k6R7-4mo{BY!3B?>qR**L_G*i``p`GGs$~Z} zxzh(ke9gd7EY)m2J8*kByE{Y`@4UaqQa0K^=mb^xYV`}_!#}b?TQ2f{E(NgckvGv- zaWYm7Q0EQ|`pjS6`j?xoahe?%af>}Y_k^vfIFI|i!{OD}G>A5h1w+S~)Lw5!8rjzL zXPYto)3$`cw=|)3lnU&hY7Nf%BWUO7-)!JJZRm3C4`)Bik+r%zjhtvr*~1*jR}Y}J z;263uJ%QixDsH@0$C6BYd81t;u-`u`JZqJ}e%jcN1dy=At zh!i?(q5nh$O4{c_zke9ffY5=EbIgvc2ChQ8c{nj7;Ci=tXay1C(9Yo^~Dnme}3hTM?n(+>;7`kaDjMp><8y9D`k{jLZg+Ka4k=NRZBz2ul(dyAq_qmv@ zaec(b&%DY^`t$hF>;tbfM1@4ZuHv*+wrupn&A2Mol%~A-gKitvGgY-%vR~~*KcB}? z%eHwisdy@#KjlOwr$cCC+zcr3&Lp=>@l^9Xi@sGX0d2Q9npVar@Wxb9GLDAf*^9~C zu?u&kim3IQC-B{xRIZlpWb1SZ54U%)qZdAK>Wj-Mz@;+|RgkN?A^+2~b&8M5v8RT-w1NMc?qUugT?@){+lX=mg@ek;! zjU)X^^rfy_<3TfO0yrOwhr!kUaAx;7a$j^8pT2h>TRmMW+EK?2YVX4#r{w5KMnBp* z4Vi*XYxVRwRotFo+W3ClMA7(cbC_vq1dnXY;h5l+vb`0=eBEzy$vHV}U4tjKS#(#e z^r&Skz>J-F_l1|_FNJEkV)(Z^8$`!*XxMxOnpE9{azAA$%TNQB_U2HzT`H}AollVo z*)VtGQVM;ZLpgI+Qrd=*ySqtn^nq35voP#Qk;y1^bD)?ki1 z4B5po3+RZ_GE&Q1OxoQG;YZZ;>bA%>?u|5FwEW7Ws=}%x*idVYzKvV)nDFhp&x;{l zHjOTvPo{%6XG6JP4&QR6jl1&UO*Q-R)G2;d2w0Skg`xU0Abi7A`ja}Fbp8El*yu>;H|tCppq%Q@qEroigvRjO|5JNwc;MO2RF}M ziL!-TQRmqvcI)aucynnW%nKa?yH;sX`4+8Tx&vw@(uws5y|Eae4t z3$!El9+LCBUs~VhyrQ@4<-KFv9-r>%>4gSXI+>U}@87Uw$FbwXWo<+aT z?Zcm^H{p-T&vEVgL+s~7WeUHeOz(uV@2|=Lh!9^v@ts`!u&5Rut9GPlYy;y*`qE-$Ys#n(AjRB3Fsf-m ziS9!@(^`we54E!M_HhEg7YvF%@z5stDlhDnfoBeVV4|@SY&a)N(T(bO!P^^OJIeAS zmDJeV+A8KTu!3E?eS(=ym0;}yV}kyp$fwzw()L+EV#FmjWYSjlcFsDsbnh8_s3~;L zpI^cnCT24sm4R)|TCn%YFtDGi4|^>QXviKzIPqE+9t_fhRcK6KKDx7{D{d^TYcj(@ z-JQQ-4DidvQgvOm`bu-pNjjIW%DUj#2esRF>ZUJnS%aHmV9 zZs4cv0qtRd&|&6DDRGM+Wm5v2(OCj#JX1-o=>{vg{fvE*K4c9ZPXw>#JB)p1Oz!u` zK|FVi->rC3r_1n05#G%YVr40S|J7J(Q8c4-U4oX^qzU`wO}K^4 zv$)eAkMdwQm-}}35oY$$CFNHRFfV&1OWAD*Q#EA4)#5I;zW9Tc<3FNH!*|>_UlY=1 z$}vy~0lnVoa6U4KD%bR(-X#KF|0H2H{zm*Se1^F@ezPiBy{fR47r3|a`DnOQ1Utt% zfrq9seO#(Vsw<)?Wnvs1)1M5^uRLIi_jszGtxwj`d)SVK&#dLiExzT}TVA6w85i}x z#8!@823^ZjLH}_u4Tv!o^oK-j*>jmI-{Xr5qI|g#fm*2jZ6kM+Qt-c9HfSfjw>%QvA1onKz>&J z6mxg(;EFA*IR9J7+?HyS&+h@`=Y5#fX~=-A+9!5n>_3(j(T!hMAH)g%MR=f6jrniP zt{xDj#InC6QfCpOIOL{i*03p{t>Xc6F4ziugaN5}3cTN;POkr{!)#?jp77fYK-0y2 zNPnLb9qAYiht=P(Z*A3_VZ})HWJeGxch2M`n-NF*KH>9{*0W@53%>XGA*Vn2G)U!JlxnmP52u(%s(?8qY(!Qp zG$%&GpFL9{eO&@g+B~03^(WIF{WvnXHU~Q1GX9G5Dt>|c(5k}9-P{m`MbvUh;Ql2O zXu9Q6xEXtnd)+&fWy*TOExPn_9*Z73eJ zkkvfe#j*#zU`0I+c>InvwHPI0gUu+K(PInix|bk7q8`tBjlmq34J_QHyV^?489fGN zR;LvV6IJaL`1(a!P<77)Oa_@!_-ilt5pN9-+a^NGyg-Txn#D}c%wy3eHKMt*7vnnB zwRoUhmTYTvQL2A{{r#8=wjDX(b~zV{ujiAlYX%l}RN(cM#<;eA6Vui70TV$JURXK- zD*b#(=K4yoPss<@x|Q%`Z~^I+4xwsKc@nn7k&ONjII?dDJ-K5@X6CZwGD#P%lrN&K z|8gj{FpkV@vw_=j8Pj%L!oPAiFf(76_iOcG;B#I0HM&2DKbTU8!4XtaE8trvwc^FQ zn^?gnQx-T{%5@9N+kZ~g!asYzpyR3{yuG0n1NdJo-FP#!`E3Tv9fc5BTuNGp4d@Bi zfx&%cX<5I4aHwex>@JOh)vKq&@HY!7IWP*kRFc3ebsRjhUPvI|hU;aN>rhmTfaOLHvvx~frJS}<~>l&4i0MP>6W&)0eVAAf{pt#lsR=6dS;ilVs(BlTqC8Cx;xwn7|?t6C12!9WoYXg^B@R{>S<)awb0dNKR8#~43U%wbh!=J@TvMXXvo1W(P}g{m8_ zGoMM8AU{PGZ0bCrB7G#K=3m2#pfxDoArX2}Z?lvoCz!6*P$;oaW}CLwqouVR`(9-L z>BYONQy1o7=x_be%7M-a>(|`>jCcejC}fDZZ5V)d^0#GXs}z8Q}5v26KF- zM_1+vo(K!!eQZY4*)VJHFF(Sc8vmX%x71-W|K4!s{7$w(YeQ8f8o);H8<=u6f%FBA zWU2Kr%v2r?_1DD-B|PWy?ABM3Vfyz!@C!jvR7OE>Cl!?N^j$_+}<2^ zRefQf*S=@h8qcuxR)ndE{e_hfvG^4=>Fh`a*qt{V=FFVIDto!+Hu}vRu*e_{mwph*fNE`2l=lLM?-MtH)iF)z(gU4zP}htkN#y)hRQ|Eu-1dB zvyV|Z_baaJ&Bo~7DmEd&iWNGml1okz+feuvN9(*~hw^8DwfAFw!Qd<$f31LD>8K4& z-*4mUD+P@IU`6iptTFY4GF1eQ01p>m7H4n=4`~dAp~odCEd;AZkSALs49luAE@U;|23Vv*%FNSt3 zuf(^4R&sWaKZ$+>lGq>%qeF+mYPYv+ypt37?HhrAJs)9P^hFl4x|H4cBjK-1y(CIq zGY>24UNS2o3$vKEvQZ6!FEM&NIE^0(UakT3V&@BXdshU-CWpeBH^G#YQGgdyz3JXC zZC3x~5&wC?KpI#55O;=u#>~m@S@ZXa{M(k>ysDxKULLrnN<}e`jW{=+<(#?0#&bI8 zZsH4vmc>BF{d^XCMU7U_ZB`_BL~d{W#AFu=9*%v*Z0?)E7JI!&hYa5gf=jQI*hN7*FI?veo(h9V z@54N>`nLp94~>Te18*v*)M4sHH*w1AJuI;%5jVLxqhifd_FnMdCIqWM+LR<{vRe+< zcP?WADe{EZ^uTX-->MtC6(P7to7xOxAhjzCbW5&tD}vi`p}Q;GPMQpT0%c)Kqzxq~ zpT~-!KXCcTV|c;1k*U8hCCkzUq?J60)f;57MK8=DX{{b8<54#I%rN?+V#NiPIMRjs zkq|0dj&qI(=dO4@eK8lYlDra8?cR9U>YEFpZ!1~5{6hL{q(rB$p5@tYLsIe*zAd+I zYz(lZ8;5_eU$M_w#5gbbygCl7?hF2->M-(tkc``t58%fUK1{-mXT4L;v&233xxN1H zxwUZ}_{{-`%eN%?Dg8IPmKna1 z?T9^!73-70_)R1v?@Hhj2PDwbb}epJe+4#u^#b;N&YCK}MM55EVhV*lT8*+hGazHN zF05$k1H+oGace&sQ0Dg-NQn-DJ&n(qL?rlF=NiL*p?0wGQLxB$h(0Y|CwS=Y?qE9M z{ZZNW0l#q9tm^)y1$d~q4`|=e#0RA2?H-jr55hvD?qa@YqNaK~Lw_T|lAX0N!CNhTJt<+++v z`S%_!p0gJHdaq*%{A3+5M&!I(ol=%_Re`^4puvDg_V55ktr?0+v#Z!bJv;gp8BWcE zMsp*F42S)&>@fe{2LJmUh|-=f9tyO{LE5S>jg;-US6VSLhL8ooq}jN{eG!66FxzUlC!H4Jtw zRTumhDO5T22s`DB?Ankhrf_pPCj05rT2~c1ZhIW>C0GKd77qF|Goj?*4;EDakLex! zi1iTzXlz#luV@~D&-<9s?utMoh zXX!GDY;VdwT$;EYKm6`X$!)KN2hWG^>*8SLjV4yupvqsXapneZUd@Fq`z*R|IUnzR z7zP(D#1NZM%KEn|!`n?mVXTZUtUGbeX{tsWA7Lu^_6%nMxG!OYFU2vvYu1z!5J7jh zZA2}fjacfki>*FA5L8Fr=d7C);5cUthRtIEr&Y7YmZR0V6O{R;2Y>nD#fj|ilvr-Z z+6CNkM|Dxol-XRqtut@iy$bsbsOG&49&sM`R3vs-Y*v?)4J3BO? zgqv6Ri#M#$KyxJ*N=g>)vEe#oH|rOM?(IiWnuEwxS%l)pRNXdz1IvFkmhqzDtg7=S z|8wILR(V7jR$m{*yVsvYhv8u?MrfQ_asL~)Xy{Wee25OSSenkxH~nIdlw?6;r8Biv zr_!bY=J-_cC29qZp(RrzsBC-=TDDZ-g~xh)(00V46}D_^{TNPe{0In)wS=o9H?m8` zgIGvg3C1s|z)@)~sJgBg7e7*P&d)ecP9#^L@(C-!1o$^SWUsJ68N zKJcB$Zgv^>ak&Ct^gM{y_wMF*4XuA>joV{FgPdOSH+u9a|3@s? zo{t9gE=`tnQi~f~AmHJtb0MTtnK^aK2}*7)+i zhr78SZo}wix($gwsgk>kF>DKRp^^8+lquxkUcR$~l`|b#h3w1f%fnT$Wl=STIjums z0k3#1QJs*bmvEi&zM|E3eoXG-J1)Rlfm1HX;w45JTvwZ<+Q21-yYH%j`WMz=+lwoB zORZ3JJ~afI?|DP4vJW&iP9e$S1iDn8KsP(1=*#zL&`&#q&chyKouG9KRTE5Q%1iwI z?lZ3Zei!A;9%#rh9eO@Z84Nj!%A--hn_6FT@bgW)IK@{kZ0Xr#wy13{JD7HWT^Pu4((~!;sq1G>Bk=|ouMm$K2Tgf- zv)NqftP%Vf?KH0VToBB;>I|<&c|$^M5bZK*!s^b&s43HeFLqyJYi32TlG6t4a1CP% z1}(q|_qy4Kht5nrZ7+jcsuVa)liqsh(cIZXY1eW!C_Zr?d#`Iz!m1XW-tr$iaqcKL z^zb%bvFt&H3~|GM(HHB|&MJ1H22-1D%?mOm>|;`45=DFNm6idGkZ@ zV$JO8;kol6aLo`XkXZn)PtGN7TM#@L!@-5#AXwz&K@IWE)n26~xBXY~n>1*5F}#ZS02z^v~`Ba*AJZ z-P)pH6P7p$4_qsngG;gDJT48;jx>knYDoI&mqA<`oLR2_}Q( z_oYx*xd8gh$I`1Q9b8bZDt=trmlw`qNH%fw$Q47XypbVDg|u(s!9pb+XH->qI2%UwxWiUw)QZI_j*RN zW2sW6H_eU(hhE}WO|W4JkA&RGi&f0D=_P-+$&EA>?WwQqC{lC|09Q31iWM^MU2lL2 zv?oBlsy6kSs|)i%fBJZ-KYX2ZjXkWXXV21hvYVf)a7o4@+}HDl&zbkEI^lRR>z-A> z-|NKm<;0~VS#6u51Gz$+WF_c=OD z@2!aYH`0sCp%P|s{TlaLNuSrs9|`B2hd{1|2}Ivkqek`$boJFKJ<&YK&z) zU&h?0^=A$mCYYIen^+F&R4I@eMos$WWQL z94O0=r41YGsqNQT%KSMN8uPQ@+R;4NJvalVTIbS?I|g*~lNs4fFruUbrjXP$3MP!w zfGrcOAU9!@z$?eWp*71vIxYzu<7QLjSaoRdv;y%XeK7kmh^mZ=F(LL49-X!gS5z!# zw&zx&@#ig=YF3CHl1(g=XRv(3C@2nghNv~1keS|ydtImC81F)yda;02rObr5Q379Z zDgj>m$I!UE82WK6g|6pCQ|SJA@abJT;Nj&E`7#$8M`cmWNKaz-tV!E!BKZrs?gvd) zq^zJu^J6%AQZ*X(omvH3Zf8SlQXX6^TSZT%EM*^SySQ+TwX891DL(xfN`^Fz$~vZ! z{X2JXRo!0gvAT-$(^eBrI9?{&Z&JWg=asUnSL4~sBTMka>?L?GKnr93n}tfPk!-J~ z8##1)kmojQ%GYDyyzB%j%__j70`HY_6^H62MyjM{17+r_Da5?axcnU>5oCpC0P2Rgxy=~1yee#VETm#@N~NeeOJu!&v{f3hd>`S0X!qANK3< z5H97Y9`$?Mj-y@^=@Fcz}^BWk7zX3drTv zvTN#tAZOzj{P0Z=4}%cs@G+^hR&16x==L zAj|b~!opG4s<;4mTA+6$L*ds1DWwyTpDGH7v3)xcG(zViNQ9UcXBzN zc~ihUemcwA!k@FD+DDn==_lx&cLB8r3t5@vcW{VeHA@c2#ly!qQ*G5@+B=*m-`$C-X4}%VkE6i6Gn&SHi=wvBX|yUR8g}0BppbWgq#>N` znJLDga&$8uZXSdm(x;-=g+**n{u6eys)+rSzr|Dn`qLQQV4CDSly)2ule2<5*he?A z8PerUb;Wb$`o0CPt1f_3?QvjVngl6#L^#!UG<#Jc_&+vhsj#UwlV{yr8J;=S=7Y>~_gS9^L#C3?N zROndRIG54VW1*0}sE3(dLzelZlhx}r;3ltbcKMW`qlPxH&HH3Y>xm24wi&@OOCbWM zDW=qE&Qu;1Nskkp>5ZKc6b=eu$%;+vtJs~bOL0K+L{0E5aDv^BRiM(c5A`X%g*}tY zuxnx?Has2-S9SE@TG`Pw+|%MvtsyHDR|Ova`3XqpVn+NW5Wi%cU*b!r0Btu z@p!)MKkWO>hZ7@eyFo_2xYlsAWJBLok8S|={5ZN+Q; z*Eo-pW9YvrdL%ueLnZG%aJm6UF`|1T{#d^otL`cCg}E`L`q`C2J%cG~fj;N5))UsL z*u%QnPS9LBomE`)q-ldKsY>3Nu85jBwByAaiX#jZmD*zzSsH<|KK|O7v6|}e^j!;9S^yz>#8j3h#nh~naILRgk1BZ zI%Xl$&eXiM*q>Alk{_)~7yI|4&1ygST~vhiZaJry11k@8s_}W#Mt5 z%!ZFh$LSeKSXX%;cf8KWi@OTZ{Mi{+P-w@RvSL`z79EznUynWTG=*G))9jvwETl=^ z@}JLtL?eaI_~PC}j4P@YRSe9ckHcc9{m@dXR-Dh@-9L>A1l{86q5v|yHIg3}JDEPG zkD;AmjudmkkX_Ch4O7Bf+4A{)A^lM^esXDG535|+)|~xpq~IymiO?d)t9>a{w?A3e zHgQG;g17VSR~EFZAM9VTnhQ2ggtN85u<=*`WIVoAZKbiE{3fp?Q}G)5JXn|C*#94z z3*W`;GYYgdsvn*!u4IvZbJ?89v8-qQT(rJ99dBoNqioPhESYJFmrX~*yYn(ow@eFW z92|u~Pl91Vg(-+S9AHIFE^kqy1Os%MSkj0#wr$7)j2fAbW8W*_^RyW_EIgv>Yxa5U zSGQZZr}txSS5?)<1_{fPJYfT8A7Xz(Hn7g}TX^aIIoud<8v`G4XnVj2S?FLaJ1^p+ z&&aVqZ~DXMx;spr|0mpayyF`o$7dfSWQum`gZ%OmOgeUeZJb=gKDtl-QSnhj^(c?xVE8^OxfWpEX)N0^-JVP>7U zjs^OLqnhzIW>fi`(VK7Vs{eadapnL&qDWN~@^L6%_2!{ycB&4n%IgDD@At5uO|@ul zn*gnQBcL~T3haG95RVLB!#yyZEGo@Y<@$!k;!%8sF7s+ISI9WYR2Ws?vbn@NJ1pm; z@>cLtxv{*S*?OGWvkxy1&cv4Kn{cDsJ^t!kE1YWll-IevL!@CI#x9wtvA)_NZ0F*Q=6Q zeO61)nkh@UotLFS(JC%411hBbTXV!e4PC^4M>~m+E89!WJaVP5?Y8(4-*@cczS~@$ zMMO%UeqSsNUyv&CGn5z4Xt^z^PU-^3p=YGii)Krn`pu?(Qw_!MSLKWC?&r81eOfNP zXEQ4-Z1qQ*WcYQ9)2XIM&@dqbfx&0RNAxN`QNhxE+v1; z#hVB0af#54lzgwg<;;W~o4$!XF7-l3`?1YtV$F!H;un$O(v@XTT{?^1#UI2cC0$>S zyL??1D_)M`Q(bLU*#w`z&IEYn?%Y>;!A4_+>&YrP~jr+t}o_DCeD$$aI{Z_cog{QvJnzRVY0%$y;$9`$3lD_ereja?#hVjNq<5z$ig$*M7w3O-5XVoJ zN_vET`-*C%v1x}D$Bs2llYXx_B>ov|>oRWY9*NXJUK0C!r^G45SbWgaT-q}&Qqpf7 zIU5}7kVuBVabES{gR^bcIO!6yk}l z5Z4>qi4XtlD-C@sGuC;ZmvqXNZ8c-m_qeP*zR5Xof4<9x#uTx^fgR$;SNY=Hem;^S zx6$I*76<7HrGqZBj3-I=-~8fI?j+~DIi7dku6|t{`8rPg?QuuV-smKkixWb{t6KD> z^S^`oKGATJMoLo?@3`O{ z5AhVQIg*Rt|2iMuG{Z&KTt*tWHBB0H?6LT!>oOOSW|3G71I2Tr=DV~QS4g*W%95yz zI})dL`eGRy9r3_tFQu-LP0nA(8;R|w=ZVe#$IzMhQ`HAy9LZX!C?$$ggrrd1^P5`{ zMJmZsp-|DDO0+2pFi-haU5zs|a8ID4^C0 zQ*eV^_|fr%U31ot*?f5dp4|+hD9>4w{(xrXW=c*R8?$pQVQE|NYpcfsYCNrCQju|%2 zjfM_aIZ*DHLu_aPgt1w~a@GNw`Zp9#pWFxL7PXA8LNW~z35EY?5`LEDGe_1x#P}O; zupwq2cytdiuDYkOWvVKKVk){Fk)tM=Q?YAhCbM_RW7f(@8>$}J!|A|-^iAMJU@hlR z;Wz=bxHQh37%_#qA4cpt&P?QZ5c2I^4z$h=Cz+SjQR8FxZXf>pb_>3YrP4nOr*bb8G;xmYdCYV_ScyO5epKC&{0;}-l>=PR z1HGssT(~$G@&pAj@u4&6%TtD60Re27^agpZT$nwq2EWJG!?_?E5T7B&HMfm~@Lgss zwylIDNdfMh)v;i7<1=ZGZGfzYU+F@FF>-Y{5zmK&Lq(t={MEU_Xg=OVe9D#aRg)tb zwA@bb+RJi_{>BltmiIJFx|7n=YcMEL3eA)3uscu@+)7s<_p?3C>^q9D^6JU#B6Iw) zpoG})S>h?lv1qJS0?pGValLl#pkiS`^i@|GE2q{*({gHH;(9vsDtS3dE+1xfH%w+f zz2@NLfAff3&teidxDnRO|Ai;^7r+$@VV<0ts?!0nu(L@TC7!YBz-%3=@u73o`u0A2on-w)-PO2zS*uqCy5+v z`||*&CEC*Zfk-rLKFL~4oyVE=>KgHa`PjKQA6lArLsgs^4t=kM+1*F+!6g^El+ULS zg2`Y#t&)bCkHRm)zpE`VAB!v=k#FO3d8heYK)f%*^%HwVd;`KjvRxNFk9v{L#vG9U zk$@w|XH`{fDaK>;15yoHn4LTs4C80wE2Gow)crH?S^FT-%__#7+mk`iuDL=E*5kzb zD-hHwjlaAfft$KH9O>*Ls!zY5%hxcf^|u?mWA(`csW#m8m4RBh@~W{X`si3TM$N?d zJf{gA_M|$%oz`L;Nzo^*gF$HeXdQH@`+(Z5n_$zV%&iU)0ttIJw5St@iiC|M^J+8| zygZq=q0$ocxytXgb7r;Ywa zOx}pDPd<^(@5;nHGZM?`N1OWwAdk3K!^CljjG^Fy7n_pAXa$dDl30Ql11%3Xh<5p=$VkP9$6O z`#$?1(37~1SCiI2MbuCg!jSKJU~uOav9*6jC)*sxr@0->{Zo0kSH_n(jlaN-&Hys~ zS2a!2{*T@I^C4**C?(6@o1w<*Y`DKF04!ZBsl^#nj(YiX*nU?KkBf#t*_%YD{OiYz zy$ohMv-ZNJ^lbJ=ZZh*{MtZpe|)9{F^0+foC(R zOgMw~DgCUshB*=F`37^iYsixe>lwopiQpHq6_Y%BkcnIenG#%RxZ(tdqswvsHeI^+ zttmWe;F3&{^Yr=Q6I9bLn!&ntcrjcb+DeX+tUv*7ZrlYdeW`{M6}{jr8jj-$>S$tp z9vPcA>~yzsWM+kvUXON=7w7XuGhM(ya)7nfF@g_X`%tp1ohWIFaH}`nq4!ce(Bow> z+&<7qvY!hQ0rC*zPCUmSeH+NrO~&}xS`;Ro_=cz3U0{RgMu>WI8FZ9I$ww2M_c^y_{jbuC+Uehp8U!#UhNg4#L%!i*! zGvV8`PF8fRmwrrZA?!O2?$lTeX^Q;G{M-!Ow4n`78|8w=dog5+_}ua2MEY>ScSyRx z!rU{mynAYjX!?`VNLd~kJLrTnL(*untr}kX`H0VRp8@Z;%)lu_t%Scq2eSin@K{G2 zEH{lHs_hq1AiWmmULBz2k}HY4OL|q&8#B~&d`@2~dqMk#GNK#A((ncAY521LU}RJb zj~bmI>q8SUGVv%J2!78VN|xa^Y@LB3hZo`pp&+P}69Y^2dC>IyDQnm31c}dU@y+Wy z@FvX$d@V(wTQ!Z$7UF=3K_V{g7zVE!$7sw@IW(S8<;nzJMA6l<^w^LZ-PV>3--ZgQ z{gcz=;pyax^+xWnYs7`#$O^`b-(#u0PzYS%_b6Kqy`eI@1bFH?VlY~8gzjlmVyrcz zp+Dj#Z4z`vnZl=Jr^Yy3`lQD-lncaZ+g#xKGYxzhkU?Eq_CU;6fQ>b~(BuyTS5|MQ zLm3Rr>JKAnO{*Yz+gunZbp!FX{q&)-GjwiEfzRy<5UOH@=^qY4oU=UrAb*|qY!yQP z-4mqI`z?%4y+sF?%3?hXgWR7mg6911EB#5;wMmdl> zokp~7J_er7?n|iNw0reN`FfwFzeFo$@`uL)-6>Xcmcfx9tYv2$^iT* zX$D4G9sK_2D6D)pAH))lpjSaC+*~_A1x?1t-ua0z+cuMKlz+y&Y%PP|UjF1JXA6FP zdW=TRtiz&2d2&;(1{{a8QQNu<4A#o>{12|AZJwUgT=fKr)s#X-iS;x$yq6q3cnKWD zZ-Pfw9&P8dipJN^k#m+RAX{AQdjUfK zdBF*7&)jnjkwfoK||Z8)L@L+3{IN0?Uy2KJB~JUWl1r8{ z^*SrLV^>os&n^T-+D6dnv@J{uWio+5=$wNyz=b1qRY*awvf(mV+>Ntj`=jGjD)^x(=gJRmec5A2G7OVum! zpK%1-Iv5B07T;weYrV*yJN_^&;|=8hn2Ap>?j%3?>a@^)Z)}Mh!M%t7k{q)J&{-Tn zPp{^Z;76CyW@a!h&Do5hUU%S2fG9pl3T7g!&$0R6gdm_{EjM)8AU$*-7}xP@zocdk z20itMk>wYmP4gx>HK_#lteeG+5Y2+K-+kfTu3d2b;RmwH-~x!fDI+;aX}C(~7MW-d zA|KL*u|Jxr!JE+-?)L)f#WlF)G6 z7<)h8C8C1+Vc(7nQh7g?yz{NZhY!*#hr^{AtEN*5g?C7}iCMmnNXqkPa5FT0@j(r*PVZmxA$b ze`+S}3^}i*AwfYM1)rF3W{ypRr1)TzI;e_^grxBHw{h^DpTY#ZEv5|{b?AF=Vj4DV zB}axUn8coiFuhA1qFr=xX3}G*4G<4+xG;Ax0C8xY1x8Bz<}ho5K=my52fm z%;&h3-IPX$3(H}~dJgt{dW>1aKTw5zK{xgY(`@NtNU9ws6$b@?x5ounocW2%CJXU+ zVSi|=++1=_<|OCw)I5;#7N%cwq`8-$$a4$DdqAW>ls7)Mg0>Xg##8*i>(Oa{lJw>_ z-7Kkqe2ghkz7`45@7*!%cqL!E2*qzDk7=)ZA?O&N!C8Ix=xg67xGp1$Uzj9x^A5-U zPa||;(K@o(%nl1fjd0t!1t=9!!>(Q)$N97)1NDzeVXq0o*Q;uvu2+P+GbG5AN=Ksk zb0rM=@cB3uEi@%xny>#gf>1^YglF5}ZSlqIvhCN&d;ZO+uM1^yZUl!4iu=GU2mUnc zlSrh~FA}HvJ+MBag>~31&h@%|pGeI}V^`JXarmHU?$+R|q$akBcR8Sc`m*aj1P%I7p2>z$W9aT&lSRKVJ<$hEPrl(D z!wqw8V|cCy>)V}-b6!zwv5yA-`O{HF>o}-7_(SK@)tEP*&%Kt@gdgKm(cMoF+uzEN zmlalw@|`p?}CSu09PIpQ<(Pg}w^pQm` zNx5N3=WJI)(_W8a? zu~Q&S{5D}oVjj(zxq!RSa}}d4xfK-ps=%-97~7a*gLaxfD!niA*-$^4;Qg*sARq97 zEc8}~!ls4HXFj90MmUt+KfDI!iz#7k!f$*LPz3d_q`88e>rmaT2yu1Moc_=qAhRNUcV)^2XY$1EW`nQzEBZS96qT)f2F`-^?Bo_61aDR5zK*oR z%0(7n!0)%_rsa~fHy5e?>p)oibv1a5xf%LeY$iRC1w`Tf7I;`>4(HXY!O-e5)Nfi2 zaxKlI#o-xFUH~Ypc?Y?#_Ef19@1?!^os&?;Qis>q9gR@l2_2zR;8w?an3cVpWUbu`4s!*#=W_^&6YFAs%Cyo~ zzcg{Y1L>#IY3LPo6n`(g#%?wkBLB(HBxl!@!9iUXw(l3m*w1A&*v^}_C&fbL&2orp zL%5-8hmVUNfx^da;IiBS)HVmPi#IKV%~I>p=-gE9erG3oB`%sqhR4C>-koR`mjSwO zc<_t698MSmd5aGTJSEt>;*(Wo|GWP;JdpQ9o&5Xl@{&85GCk^Tz zicG0)2%byGp)cHS(WO@gK}k1|d_NV9nfX6hr3g(p*U`pycke|^KKV}7kDD?n=kfkO`<^&2df;H@L8@)yXm0# zI2McDhE4P4;y!;_^fZ;iEo5GH?SOCIJ|yr^ z2wvYh2R9Vt!PM~}IGe<0R4xysi?<1YNXUNNH8lebyR>6D6hfB{(vLMyS_i3KInP_3p6CqQ0gluaO)AG;xL1okzEMP zTMKHZeS&5f89=vf6Zm^CBJm^RIo-aVv4^8%7_Mc8t>&Gra>-y?#Y9|hoQJb*fRu~qA;?WzBPZ;aAXQ3#r_ zAW}D8qDRXyAnz)eQvWvA_PHKxsTw8+%Q%peD2>~uhQNpU-xz-6l_u9Xk@GhB@a(_6 z5VK?e?3x%u-UieIERF>LvYciV&_`kGW4+I_>HebU&4b>D`W? zU)~NQs}JCmfzQmX)QjYMP(RKL3xj3VlfkxbHbjnWq?Tbqq(g=S*Xo{82?b+PWUvBT zIr%g}R0D~#5>y}A3V9OGQGL#Dj^dXcR8k;?awi+2t6vN}pScIkSEtj?+FA6Oo(g$6 z$pS)VEkfmi_vm~en_1avjur|g5VXgL{7E`jxu)z6e9YVr|L%9QcT;9@lV0w?hHoC2 zn(qr|_2-kSBR$N96|Q97Cl}~AB@c;vH(>KYDMovbG-_z|p)2#0l=cc@)pZ_t=+tB5 zAfIJDZw#bE7NC%uEbA>znUziE*k^A=lg`voKUGPd9fhVJ?3Pt#4B=d zxpCEn?3d_nC%`jm@+Epr%Cz?UB(fuH0keKeJ(Q5>8F*2k~l(Onw2 z%NvPC-w#INzgk?C{SqSjTXB1rHM64P5EgohvxK{v7G7(i_rJ%X-``2x7?ZE`vd(Ot zNQNYhi+=(RgO91c%OB@1jm2{FF6_)yhRB_QT*m{oWFPl3(DYn%5;_d6=Snd+$bdXMy^PFP zawdbRr=a%?!hwI1+>4PbVd&WsGQN5iU3ldYEIZ)|M+9F{`q7mcsVU)jIT+G`)p^*n zgrIEGD|Fr_4La@m%$U_{5*D}>Y(LxwUc4sC#3m3+{&VzsXD1Cg-$j+<>X@A)Z`n}$ z=~$~33yadfaoV!R3EP%Tnx5B!-`maRJe;&UEdT2sU zE|!GN#^2XM>6~NM^wLQ^@C>wv6`W#Dk=IK4Ir0)PeZ3g|upH#gO30$19+I;x0(3pf zap1y5XxN}d>BwB1?7a?}R}DbiXfr&zYyn?5A@DbN9mu9<6Rsuy8Dg^@PF4OU`iuX= z@D&3z_}MHH<@SeGM(JYwqkV9F<2}&fg`%ghJ;-GGL78nEj5fQ`j{_^K68L$E=5kf| zqjHWECP~0bx7X}>2T_RL8AB`58`-h53Ha%Y1~lByA&;X*h?JlW4Bvc?xs6A$rh@~u zTLUYl{`-Q5g()4Yy9ax+`LlqK8Q0A^0X7|#g!Au;s;>PhLh{dn?KTR)A4b;D;_eC2 zTtTje)Ew@#cNcLUy#)5dcj(Z`NYsgL0h>pubiV6GTqd&>2j@S7Co^)fYRyZuykkm( zy(6)T6``^{(_lFkg68@FoMRn{@xgnjy`2x5>Sn`SjxjD_&$3><25`R77UC7fkba9J z-<@62qI)hLJmW;7#3o@~@-_Uf`rg3++_-nKLz<6tJnwlj9N_=fwhWXAl5`47U-Pi);tvC>fM`4F9!L;4H+%| z&;AXa$}{OHTL)(CYyvsvYw&5d3@Sgk!uF^5a&DbE2YZ&kXUEgUc;!{MQNtD3wlY=1 zt(nP-e4Eb9+OZb(v<2{D5@4%dAbj*cOkas=gYv2o#BHys{{2;u@+KS}Jo~^TJxhUK z@vouX%LL6#RN(N=QJS;R7HoJj+yVDXI5*%wy1V`Rw%$NK&5ZQ2* z6t`!A!0c39{o)j6sPxko?`7~L`!ha{P=?HmjZEHP4_SC9tjgq>D>?-%C2lW$akSl% z(UP6QZL;vg`u;8SUgAsA`t1fj%YDeVuQhPyscF1t8Y{3m)t{~ZcZ00!SWaT|SD-*z zDqYRk0VUlx@uk&)ZiIH*|)UKT3_t zwSLaF5MK#(>xvb>pHB_fo8yDZ0Oq%rwUO1GC zk4KkK=LgZ`1}2fClY)uo;YA#s#ifv2a|HiDEkd3mWNy2L9^;9yf75=DY?H@XJM*x2 zU@}qeYUTT$9w1jblkuo*!Q|XksF3>v_xX0Pa|b_T&%|4DX+6Sn>o-h^=r2}r%%4i@ zN8?1D6xUJp3Rxi|OWZ@3!^oQ)_Wn)@uJ|w)>}JStE8GK-d%qlqiZ-F1!vvXHb_k*` zwS#4s46XT{fVXOkA#clWv^-vf>m@`ODL*Z2l_~{2fdI1fPArIkKipZOL!D3;zC8{j z5+4qL=EWKuU9lcR>0ae>7Md+EIZ*{SZ#YnrzjW%(Y2gz`S{{Njbzwn&1S%|O-YJs@k6 z_Tb91X3*Ougqk+f*nEp=XqY}qJ5GsGfdXM1maHTD_r<`}bBgd%%@Gc3?55YG{V{4# zjmR_zbGI+s2`*2Lb6%{xMZG(Yfc|Su^zab|#kNWkf(C3`V-hR3ON3{hCAROZmh?F(P&6>-w)ID zkFZj9zwnXTbC3)s{q-j5KTL~y4kMv{Up0}GXBw9 z2jzuR&^Sc~26cTvhr1X)SH|Kzm0KvNArCGB>KIm_0MmY^kmxiQJa?vvyjw9&dVKak z$Ki)GExZoanutQ0NH(3=)=J7wsAG6TGPtYtvbXj;CAW__;<^p3bl6mq%GxQ?+M-`* z?32y@9{tITGnNp$`aYE@HlVk5I-x-2IL!C!gzcia&?h+$FHe%f_p?-S)8i?4L*YH~ zTX_%M(%j)+TmYuTKdI8bcONQGzNghomJ*#df4WM_kA9z}3idfZ*z@c+QEU*vruUH~ zbfkmy+3&{c_;7Mj%@3zoJjNLs(qOvc0>+H3hsE;z+~SN_7^0P64d`zE5^5vrN_D(gdoEeHGlZ7#%CYtVX zkw&SVDzsNKkkq{l!via}f@F-(naq{-U%$&dFbIC5nXs0AN`8CD@=zorBCr>Y=?40dsjOuQkkQ44XS z_8M&d7yvEdwIpqE2HGx8AxGPUVSJ7fe*O3eY&{NxY}G3=G&2$mRTlFC7ruh2W#;tx zw@?Fb*9Fx4!X)CF^BA`TY$D4C@*z3&CQN?*lfKCdBx{e~!TmBpWUUVi`KRYoMb{{- z-8DoCLKnbj=w+NC$@l#}T?EM)v2>Gu0FEioLf#1x*e;$&?W#*b;HfLxjqQc{U?E22 z?;{lTx(Mrc7J$i`OK|YH8FqY_4iz=wkW;D)ala|7@?3;P=bUl=)d2YGp$!Lb&cP)! zub|rEK9XZo3^FNltiy-fu=3hnG*jYIfqDm=xyA`CB}35Wd@3&KJ&#Az-;=go?zs13 zE|}h#4i|ZMVLNzYTSNn{c{+pkr-{E6<4j;pf?{HiO%@th{`>a&5aG;L6PCxvCNd)UOYacI%Q1*1HM=uTRL%fATY zny8gD?!YH97CL~YJ4E4Kxe&q-lOz-0 zoE8umki{Iw_mn-ko_*rMqZgVbP&oDm#IB#qeaz~REw)iK%x4yixgAI4ZO!1+{U1}l zjzzbgANXWlF2oG}p~ZRiYn#2{(OWqjtlEN_hHt@GJe;LIon)_hFBxQfVaCP@ zw)s>pWKMsN*>6`v%?O1~{Uj=-JDn~W8z$SPD?rWA61enVFNh#MY%XMmE#w}AA^gA(GvoxM(zlT- zpt5>8Z1BpWXVQKW*W)HwWHg^=eCi$=xJ8lg7tUbKDiu&#P=qcAcY^m;F?jyD41UOc zqMg4~=vMXJgfsUbbQb)f{JA4J#pf{SMfmafXeuOeZJ)RWZ9tqM$5Dji{~5W=&>^gZI@``Yk4h zU6Pv1WP5pn%_>8bY2Jrl5;cf2lM5}2Z$Tjcf3Q1U1h^&{)Hi4aeqRuYy;7X2c~<=X zV2?NSto#A*%g6A_A4KsI2_24M+s5vr zrhz2oR_!GVt4?Bn$^{&NUi41>%8@;vKxHL$n7j`o9AP&}FkTgnp~`poeWw{3?72j& zr%#0?2FtLtM2$DT<0QQ!RLm^7E64p@e+-=WabVpGTU>i(61p1R#AEgkv8y8hK4`AP zmg%opUymd@*{Tw+7YTySli6_Z`$3x3CcrHji-7Br5~$6qAWK5ZF?h;k?h1!y+}d~? z*>~~aInS3&5-G*k*2dghN+qy*q>$>Zc*<&DHKgscdWi`e#LtH`(Fk*6O#1elPTd?2 zQ>4?d&62`VU0?RM>}}K%yiYlqp5$su6$(c=z{ZQx+~aplz;Z`7wv6Y3>Xk<%Rc;}@ zWmCaMwh554uR7_T&xA?Ve@dnvUP*M#uHu~EW$<#_A@Y553vQCX4mWF6aX8cgJyTzR zf?q3b6-AV-^MTu;V%!kr4b0OSN_gMzCJe?eg{fCpV~ukPe(gJfU!sa(iEuVC9-5$o zf{Db?Q`K<#n&Z$~eU(N=ZG>NfdtvT6RTMdOiD)d;LF2JlDj{abYR-9r3;Q3#Gp9ki zdcG;dCAz}oPcylDY@gwGOBZZSal{8RVz5vr6~aY4@%xN5FgsU{E&RHj+Sw4aH>zix z#f5kVVzEp~`3t}uMx($(bVqP=IF6%zhSL#AOh8A*O?eoDV3pI3@ zY7WA)xty+&Uea(TlBOE=zFY{l16pWlAW7QWu`VnCGF^wOj0d&BkAE*H`*?u7^9)+wu4{Cz766h}U;kk?5XfY@PltayRw@T)M9TYGRSF`o<8) z@!x)!Tl1H^`YFuK8CgX)KH36Tvl>X;QYGB)n1Dj!@2O~v2|xd-iZK~w^pcE&!^?B(aODtc?Fq=aDAr@UqC zJu4mRAXA7NPKVN`nftIoHxkSbrlPW)BsV;u8HtK1O3d$~Ek-SnRUl5CN}s`5VFmQg zxk|*p?kBRdpMjgwWRNIYgYQ<|gTdgF%wCgB{8M%d%&YprvcHg>_CA{A{OJIz()}RQSU6MN(fF;{CA@$j9l+zaC`MnfCMR!STJ9!5Tz9#W+N13CawhO)+ z4Z-5%CMxPzLo0Q|rYHb!DLy6ht_6U2qZoPxtD*efi;$n-OA^ctQ8NDq zu3n#re{*w~hNTawc|if*lx)R#_hS4nsFEr^lt$G}$;3=ikH>v0jx`yMpq2C=h~5#R zT~UWvUTZBD27D%OtbIx11AiL&^&?uo^g$o@P}&^xo!*{1LOnZFnb8#;p!PHt8g~`b z`H8QYpB6G$B9RJ$nH)MEmx6T?RW#l=2W{`U;iOy6iosMCcKQ$exG6UDr?= zW5F>bfwFRIKsEXUt6U`qPmeW0%v=fHfdw-m?o}B&+Uv23Pj=!WbxVv?6D3*Bq14U5 zjc6GP!<>Xuq3N^Y*iuPy()||oNEsxL+HYcA9;Nn;ap1P#D4NJVzz$wI z>3Tj1L%&U>|DGwsxq&xTLayE9?%Aj0BwyQ5QV*o>zZs*z_fMc_nMXS1_n=!G z`fmps9TLOimHZu+lj7XX&uZa;P$LYvKLecuZICT~sY=@OKKY&(M($bNW`c@-GYSEn z9Nvoy#C6GOQe9d>Miwl@r50QqN_fe5rxenUS zN(H|}P|MuzmmFHIB?UH>o?5I3WS^5jCH+Er-s1mO7xlfw-Gtp6xL)xUH@(J%l#wK*C=6*<%`h#J(lnw{W*|Z2WWX{64`UC z1qTM^!!5?2&NBIgUNyV$s(l#z&YXcww|8Q-r!S3KFb&0Cu0@rz_p!tGKi2AZI&l>8 zn?!}ILm>#uA0O(Pl-PA8(&j#v~9-+%{)+VLMZ zfSvE}#$-K7COvLdkh67!ibN+P{j!^!QSnFf2b5+<@4}zKarjdhAoF}SGh=oD>hOK) zX=+w%eA6Y?GmWf-Y;}CeUMSBG{$20PRz5O#O4%C#<6Q6R7s(nHNJ2V zMe1|G*hPzOU-la-`FRDs-e7F_xS{IUfIa0?L|}{NIK=N)hB>b%^N#U&u-ac04%jEN z<6^rYMdKoxx+9q;Wkj}zD^`x?rNJ8K52Pn?2MJeK$G(A9oG(?#*O|>R@aJS&(qe$$ z-k-w5`-!z87N9V!&rp}!<|4U>=w9-YNo$X{ijKVxPlLGI*W*8)Xzb=_?`>V9+JyRs&5st5sTo9!FgJ)uTH?4 zN48Jc;jIT7sL!gW)Y_$!wcYFo=94Pm;kyPJ*mV+`5=1CtuZ-oD%khQBMYL(Xfnd&; zaMhC-4+%-`!K_{ww%>;dCj@wlMpG%qe57I5&7m}=2Mkz$);BYZEI2EJ#`#C6&Sq&0 z+w&29{=H{)MJe~m03>?w2 zKZO5mIspY8QIPXn5Tqt&;R}NZV4O!GX3q<9O3x0y@O^H#{5HnL%l&f)KF>9DHNc+x+lbJ`gh-XC2-6NcwW5Mq&H9l@%VQ*mts}u(B4DuDtFjS_ z>Ab=mdM862)t36uS!Nk9{az+o8M}ei_HydAX9@h!m_&8vXW@i-IVu(3C%{QzIgnE*jS{~q0kgen*9j=%11MHmzLnA z^f>g$c1J&r>16!dJ`%~Vqp}Oen6YRQvY!3)*or1tYa_rDe>jy__BWme%v;Obf7t`= zWBA#0?Q!;b(+OO?CIs2)UrgMbAh2FOmnl^@gM$sOa9-RQJByMa^zS@OHwdFD(~F6a ztUm6T`>_f;Pr#7w8M4SJ1-dd;!;02V%nEKEXoh_QRmX0~ND8LkQnpcF)&kjWsxX>n z&Wp9_q`6M}A#r3k(RpnK9!AOVOy34KY%r(04~0_+MPJV0;76<;QAOWajfE zGDACq$`oCMu9q9}&#h!Qn&ghIi9YC%FG$McyTRx=vionmWG&@yVU*!~=sDI+-{nlF z8g7?Kt$-G1vSKR7R=1ISSf52+=8eD-+KTrlLSeFpAZh3?!-?omIJfyFZm-jXhkQ@I zP)-iy|44Ih^Ca*?pdUHk97)d4*Tv2!ZusM8FkSk|3W`u}8>*1x9NyPcYFG2)3{ESmFEvmXq zx|^%mj5iMOFW_fD<`hT&LJJzL{LQC)X0G+KV9`*M0gwEG@2oA2jyQr_c)vM5+T z52V)T&(kNHOmSJn8EEYXGLqXvBWe+iZMvRtsX6E08s< zD!8D1C(d)IB3ISzX|=y6G@DjJ!w+FXPfK$j^j?88!zakwWvy(@p4&{iI}f335hM&} zu#QuNxnB!UWBFTMo@1jj_sQWf_P&iEp4!O*w#kwl`C>dvXW~awTR4+)6|UNqK<_UX zoEYS15_U{xmy}&cQI0m&fEl=+naUGAn}#CO82Sgnc2P@AoQgv>Jn|P z@30k4DGI_R%~ObOKow-PJ2ID=Z$o|eKeFV?7rNHM0#1dOL#WL{Xt;Zx$ZM~~zpD?! zN$U`pVsRML`z{k(4IWwW-&083E|0Zr0T#Aq!us_ISZfr)5`Um=uoQ3qLRj?VKIRS2 zrq1tPklZ=j&~Zi|V{2zb<)k#o=AV|(_e+yb^B3hwhpoUR3Gd0i*Pn^&vobuobrSYB zAm_#9+o0CmP3qB@8ry_W$$D`f@hTu$pDJMN-F-}ukVR}CM|ycb8m?Xo$L#Fz_sNem zSoSWMYLj<{BM6ZIh+Z);`6iRmm7cjp0!6a?XCt8}h9wP)2KvDZ0 ze#=&Y@z0HH)8=$6Rno%W4-3fN@-Y&%{XD)8)+PCSLCra~XD6@!vP|OX1ilalG^LAXq4WMa7CG zT=QHben#gU{b+lV47;`Av%QwMn!nS~{#gV64X*=zt6mrqZi9!LobcNH4h+j*K`lq_ zkoGOLU^W_r0YOz%HC%{Ejp2Lci!*5B?MXQGR24i?ngntOuaPs#Tj0NIiEz~57d*&V z2s?xrc<0edL|vwl+5utikOzM!r0^*2_?E@0PP>4aag^l#p2j7w-l1`vK9xxIg+jK0 zhD>jyHwL{STx>J5XTAwebt*^E9g|^n&I9`0e-d4;GDIx{9@BMrhwNIR1(8>u(OBMj zeBh*wh5|m2mbV|zEc<|=o3%0Ef+ln1Ujj~F$ z8TqxY*c!VBv{VGS59oFb*tijnX}f@i+y~`tHR+T2}ak7ufdr-Cilv# zAjD7&4)4gsUj_N(q0R>q`)d_7te%H?+6&0{pFC*Rn1WVc3^9L~6r{$l1%)S_Y)jA< zTrH=GDu4O>riLU;i!rAE=}m!E6Gl|%jRaWe=20biH|ljD1&DhLy6l<>?`C@dC&hr1 z(Hz6B@N8jBzDsa_+Ju7a;k$Ioby+f8{Ez-F*@NTGZy92dN_}(Gd0jsvP{qof>|QvF zxtniHHy1fD={`5%Ylk?En zK~gBg9qy8bhHnPgm$U%S_gLeL4PQuP_dQN(bv)hlzLw+gsRGa18)1TSBlFFKMl{9q}Gli+}YK; z;rs1_*yEXoj@=57(Xtu-1RsXk53Yhm$a=_q;*XJOuNm%wJ{r}x6ZC$+p{i$oFnN1a zv9Ns(7RSfq=!gOe6*rO0b93;kK`3AM(W+7qPDTAiccJ)*EH*!lC8ph_SQl`G7*@Q2 zbA?F zRJkm#N^(26-qvLc4)4P&)64K~Tp$iKABRsD<=7HsC1@H*CAOb9yr;hWT&c}l^7_>R z-c?gyqV(GZZ9O;Q^uL$re_1?IzR(183-8i`x5MnaI8!3ND-%X(A-UBU4qq$-=@}yr z@b)eMd9MbtcZvY7P;(0R-MB9)N#bHJpWSsLM}Vu7=ZdqovB=sO;$P_?Y;)H|749~4 zo^A(y{N0I_4~t2?$8Ff2nE*x(a=eD@?5Z;#C3!(yFMu#7nxJb6rjLvHwQ7a$U6gQr ze;Fhkcm@l82ywPIOoqtG8Mym#80P%Uf*@-p-d|__KJc&xjg{I=(kDX5jgoTKtmFo) z6-Z{^Wv0P33mI=7Tz227!pbFC0`P1|9;1W(ApW~INc=g5w~h9}@xp%oPG}oG5-S3;3M3Dd zYRG}vg1o`e8nl;5pku{7M4NdUKHIUMw$H$qN^^M5ari>I+%d^cRltq&Wr~s?kL53Z8AN##*`c^svA-GH>)TUDQ8Tl{9-3n|sHM(j76d z#*c?nyd-$9^73fGzCoh!*M@kW>!K#3=W)Vw23OOn6tB$O%1d zE_)Zj7I}2g+4WsSO|FIn7j2`D+NSXIx6}}?YCn2t%Sp%|JV4!sBOpg39y;zy@wP7) zf|D}{q~*(VQm<^suubt;oTr0RE<3@m(2elNqKGVxmBBIdSoAAg!ac#-z&%e1j2Si| zcU!h&O6yeU3l$+VpLC$|O9msF{J}|0h_}Pi9m@VhLjRQ-;<_w}WQ0{=o$O4IK69Ij zI;=$#1s@XEmxNWs-(rSPHeC@eY>LEpsrp!hm2eX%^2OxO+3?^`2C{b&aDu@r~h+oDln^D8_& zv6&Pu^C72?{!h`Fhg0=^VVsP~Jf%qHLP*3tYrBV#G?0o)(jX0*zl|D5B@rnq$y6zk z28zmA+dYP4EM+K7DkV*%q6pRR{{HuPJm;Qkul>I7dOx9TXm_nJW$I;-UGgZtd8vp$ z|6*)O>FFw=Th?x}cFiquR@R2MnA$C}FnA?wS}D)J{VNpubO_igU9rc%<;MGIE7 ze2~4q`2c&-PgZoqqLpp8D-+B}Vt*-nanTzmc1fdSY@=xQ#Qe_U#lBv=>Q*N{Og3LQ+e=z# zJS9&Qm~%?BZ*`+E>9}->;$js(ai28n_eRK%sqWzQ>z?y<_LF$ePwV)GZQDd6RZftL zlaGqb`~QlznKbiW%g?Yq=^sRgvwe88+pmgS9JE9m`Wr>*_a%7Y>xXQ`_G`kVfP?(N zF?)W!f{n;8P`#wTM$6ebs!1rFwVQwbY#HnN@gTdZ%90OD_|7_eoo0_ay%zmt!0%9PLY+%oq3VLTeiKlNhG7&$j`HXA$l7%o|V|B z!ai0*K3kk0iqcBpKa^(kRVOw00sar0uYHNXH6@Fmb~ce+9FoG1@qXp>+RU3*C|oV9 zP>A5CzswU(-E)(jR`gZWaJN!uGV{Agz0r&9*POw>J=4OLKe;H}vDa02w)q$vtFBk_ zdC*6cd}bqmB*BQEw@^yBI7XK(^GRccBwaXrpAz34;=-?f-yupk7%9B8X$sr^P=o(( z`ZQKmCYa4Gi|4xphxl`kba=Bfc~1MSx{ALhMX?`>hVlN78b$9^FY&AE2ShfCMts=u zYr+xmkTrZUk+m!{a$ft(xi~T=f#n~V6qlX2&PQ(v7Oq+PQ^*fX6iHssa?;(HFO>Cf zU^zrE{B_2W&)RLze=VQKuIm%;<@LGq z|7q*;`>Q_i3d7Rb1HC??*7d_!#i}`ciK~)`cw`GD8n3Z#b>oFPiaG4KmK3%mQoQe? zJ%vr|GvzlvmS<0-e&=fgzp-PP4iQbNL< z7rkcLNs-okV(A9fyl^@%yS_z~HbNXzrR;>IrENa?ck8ePr8sltRBi{|4C$5?*AZc z+94+#bMPcPqtc%pyZII7{~ach*fN^G+p5CqPlZibwVs{Uk|uJP&5Q2tv=HuiEiKHLIEnRpvXF0HY$5s}e9qgC z-yy7h>LBXeR>x|dKFqI64`iQDPbqGyu3!Vh-P!cuEqsRBFg{o71wZp=2Cwb*UG(eV zD^bZ_Ws!TuCbre!h}g4Ebcz{S!LAFphz`YVJc~ik^n%rgG!{kK8H|`4mj@5V8 z@DlKOHVgT#LltaNZFlk7pWAuaRq;aQaTi&Y=2d*wJ)tPSXeVp>Mc=94dokORUn`pN zK9cSAxWQZPmMd}JD9-C8I`IX$<3wpn8LYIkl(7H42m3Fc7v)Vq&3>LLA$WsV+4eVo z**g~toz5NEEwXNYUVOPMLZq_YnO8|(E(+efhz-m(D<)@Oh}{1wmy~H+@V4I{2|bup zKIi8yo{?)7T?xC!>YW|VD{4v>?>LsY}3vA|4I@5SkWXhide;4 zJ#gb+Tpq>F{jJQ$9-Zm*c__?jycj<&S+2{zH&bJCCfAF4O3w?uKYZi08t*#U9GPEy zH1k)n(=ku}(N~5SJ(|g`+yAK8c=i$g^XIed(EA<2NpIy#Lay{viw)~p`JQriY>c&& z{qA7?m|7ce@nRt#8zjm4e>l#%>Kze3w=pF@URek`V#A8l4Jdy~i(|9Jdm0&bEZ>BAz zgwqT=`7UE^;f7nk_@}G`yCZ2HvrU`>^nBMwm2%z)6}KMZcYL41n|7~qdQzhx^ggwR z)vA^k=10o0tr1UI?P+=Jl=riFMXdqh?Vw=ZD)^!(3Vrz0#fRAw6*pNlS;H2uxXIoe z@5c_+G>P{8$!2G-$>aBbx8=jsZt!k3D?}bKL!xc3b_v(b-p@xmZf0cZlt~k-f z(d=}W6MWtIRjft-E;c^xvq*L1Q8raFhJE+vqHynK8}_bMF^{)Q*|F2u;`c|Fi(FqA z^Zm`+S>Gp$td6n`e`3x?_HAbhKX-vEn=<(&Ta_|P*#EGFU9X|T>Ko1CM^CsT3@^OG zu1)P?)i(#RW|GaU!KF~4+el-f#eQBie*I}Sa_@VQ%bXx_hR3-irTGgRkr>Z+@VX_5 zIcNC1HHo5A+ir?(L@1ZI$4}?WSJ{?CIDgcb;u(Q;fg%-jF zk<6|Y#m5~thviOXT(g<t6*6v{vWY42Z(@rq6DaSx*b7uWoz+sBgq<5p2<6Prn z9Nikn9g+`bGO|nrg7J5_YyOX+czPM`UXg*mi;HmeIyLw_(U|)AYtroR#av>FDLFj) zDrPsRQpJp9DDXBS(_d!7^3{7B)_&I}*3V{$`_c1pd59WW?sN>h=dI$J>$PCXbe_{2 zY{nUDleo1t_mY{r;cH=F zng>ekbRx!Cqwto|?*H?9aoYNE#KP(pepVR|fz>0iI{qB&>CmNbCFN;xnIRRGN8_@! zk&rM=k}NZQ0!eYBaM-eR7|d{`gY*g0yP^SZ?VmuuUEPW+OCE9inrG66uwmrDbuHrC zmyf%mjzOt%HWO0tM%)3s7W?(ZpHJUbJnF4Noq1hsE6ITazuOV_nqyGx;tO|qTRNOFNnvb* zmP3#Ac%l}18*Tm*(%yGfT%6HW!LX-pus}EUzhc}C9;fd2At>{E9y)l4mV-nGO zuL^0p!jSGs6G*SPV_hsMWBKnPR8#{@88?EY^hDx_m-du53T5KU<3L3qENbg>f{Dwm zVU6H9=D;23KM=ZH&2#Y~h~y9q6yR&loFb<6qn6 zJHLY!AWLZ!T~_Nv9P>udfu%ft49H*()vYF<_P;=#BMazKy~o&iT#mr6XioHPAL6SYWV=X|npSU3bM+K79_9NxC5Rn%qe4CqpN#nfz$ z#PnDoF`EE(xU# z24K_XhhE+#TxGs3Zfs73WrD?^bZs&mDO2IppSE-I1Ak#z$ON*&+Kv48Y7zPcX@l&; zwT#f5gQc?%a?Mflbk_P^+-Wg)$MODo*vS4s^N(6^*4!4KZFj}#`_2KEIt2wo2IQdp zaN2!TnnabY1d&1slzU!g0{@GGKlU0#dY1>eF5X2h^|bwb(ONyeT;^<*C>)prXPSKWa5XY9!IvFh;TZX(pll?&X?C*m;fDOf&TgVD8Wv|98H z&nOmP>mOP2Wuzf-`esM&Lo&KgJiwH=>XEnSSrU?}Nki5t!~Khi5F*tBDeAFE*1tx- zPrJF@+jwsLnqQMg^oOA8i)|QF0QA|r>0C)co8W;`FZX8>Lfpb~v`Fm$4;xRCmbHm+ z8$7_(FB0qmN72vqi|OH@)HIf7*ov2Y^Dke<4)Oe<4!K&`44J3A)Q?3l&0-XjY4csq^Zr=y_8{{DDRV0M@mO+r#Daa`(!tT@3boks+(6S{G z{(ZlN%Be@tVQ&j|YPW*%N;%T%;!Cz2?}WrlBPg+Qzzr%AbfozMX6n0YoYA%rrs+A5 z;q4rFRwWl@{U`3gcX%vV+B248Kb?RjYAaAuMVbaA#I50jF1R;m8TG9y9>?1IfKY|^{lAtndKYB|e~hSohd_i+Qk8r+=MF+m4RJF8TQA1=MKH+z;J&XCoPl18P3mP?k%0mrK+c4 zc8vozZe4=QL}BP_SB(>T+tA{UIg#91!+cu00IVOKU>@8l5_nz6ckEW&$3$-oWl|0Q zGI}T4pz4U2x%RS_NsUZ^54Pqwa{e!_q3IDW4wq*ZOf(^0EpG*}JBCn2@CaHKyynX5 zGeG&C1m>J8fQ-}k893To^tU1q(vreVxeL{-P-S-rG&PbAg*HiHY^ALIyZ*Ys8(ijyo9@k#k3eSA{ z@bE_^uKd|{ZlAuGk5`w8M?8GF=;trc?~yz`m{|+8wTSiux_Dut3H`6`31d~T<5a{$k^=ebE26sW04 z6*H4(LC!*Th`#%k^E(g?nQlAcaEK_d#uathT=AU; z_%VM6$m(aJD?1xLtBj$Yb(2Y&{drg>u?fmeB9VQqL*5P-;jgh*F>jv}COo|i9r6+o z)|`xA*6e`Wk!jrQwJSg_B^q+(2XRVO<&e9s9wzqP!$7_bf|UyK(-bObeqIfCIvTkd zK^JjG(NFZfy&wKHUMXt$wgYjy6L=M-LtTSB9df^qYQ^U`E=?1B{y0%Nvq#XBJ(6^e zd4kKrv*5tS7Vr$31M{bKb*ap_s13O4z;1<9tD#2Z7kS( z9%W+Mm6-oLrs2p@zVIZ*A2hZ9zsTcR=KhA;;lQ`(;yp5wz%|wy) zQ<(HmnOe;D#|iPt_^4$hNw_OSAQ zLbdNqxFBX#-HJE|s!vj|PBa6i#GVx7e2wHT4OA6950E4qe-7cenCDRaaw+&5idprC zSK@|($z5!NfDD+uaFS zJE|F5Qy;LB`HH{IFW}=Z$07WsHxAVTyt?cHhH6DHCTlX0mA=Q>pGpA_M~1nba~gMA z#c_edro;Hlw?HrB66Y1W2k%^30i%@5aN<8B{PAQN9NMif__D1I;z~}y;h7(bo^CLr z6S@{a0L^jq*2@vhv|cIDuq%O=kKTgq$s#mc0o<$E>risaR3`kSE+{!Du*%xzARn-WuOC-Xa2S%9v`Y;MkmB5n0rOXMb6QGqa4=RhhA;Ic1 zXQ`}-jjS^LJu4esd|a_94Z(4c1?{3XlodZqGw);=JnBv*uiQt)_h}5RJB`;gpW#_& z8%|yGGi>}&iHmRThP2vH>@z(D#k$vUUO_ao*QQF$70-sKIW?Ga`z>5wmx{wIyWrY! z39_Wa1I}emBqnSjGtX`nRX4o~9eRqOJNp)Vtg<0H!YiOy%-gfQR0H|HSCF^z?qr6e zu3*Wt22Mft8U}gY#p~BSNs@Vrcy11&oiRXz58pxQjD?(fuL&8F>VT757SPLYGC?(D zE2(*N4PuIgH2>3PM%r{e3GI*~v)8bs%IqkZUEM_6u8Qa9&CAfB>M({!X^`Gr9gYYl z(n=jiBG9}6*>U2}`afMd{7EmixvnFpziK*O&P<1iHA>`>ohcn9KZ+Qb4Ps4@3rV$_ zgBd3~&}HEutX%&DF3nv*zxZ2GvoLSG8ZAvus30DiqeCl>8IzO^?sUsw4P*S&9i3(= zQO~Lwa9Jq~qDlub=;>(M{PQ;K*t3p2bT=c0*~>_C@g}NS9YO^|bHV@5DW-4ISR%B# z##zly!KZqvSg^;ADN2%|lWonZbLMZXkJl$d^cSRgY=Z3(X<~-^2j+Y0U${NPniKCj z5hh_SwMr(&Pa89i6zWcyPX&+UXw1cX z;D38KnY!2+&wOcw?x=F8`SOVCe>so#xR}viWmVEt{05>6d}+bX#W1=05Nwfp50xc} z_~}WcV9Dy)_+``Fx*8F(VVk-qd*;3`s1!y&NTMDC28|hB7RYc zIB-=_T>6#8RaE}Lzf$#RR6PPj5eCHhdTP<}IWzEvsT}z-bOpAFfA1lyk60%@+s%Az zNu41NC%c|P{R10X_S2U}{xGK*_aflKHdWH=U4TWVZ{TFxBpS&+V~%SsBx6f-No`dk zclVwG=}KyXhNmy?v^HhI%BB)vPX59-Lly!j;RA%R(df5ag6#Q}3+J8@vglY2&Ximt zMtrymR<*9CL5@?&n34@7R`E4#ydNYGPS&K}>+V9XSeJQyScR;RoJmL&}WyC7BNY;Ut@lwC1!e9lA1!=Iy@@30Xnp%`H=+ zH2)4HYjvT_s+lyv@)^3z%A&PTBDjSMh;*|WUFH)A1!92e>Yt{xrf($O-*6F+ikXIU zG_S%!CJLiA4G40aCzEXf0~l}q3J)DV%vC1b0?*6>yk`CkzJ46W8SE1CqZ_AE%jU^c zd1oCn?&mS?w2uvUwatQFO*qRXt1O|{y^P5HhXsNiYF+T|Vk(y#`5!jf21AWpviMm) zMdkPrBw?}-xooi>EVjDQV@Aty+1CS*75bN%FR-WgD-Xc>ZxzsSH&a}-HHs{KJsmdc zY=KKVKQp@XQsAb{MaEI78|D`9SiSNWjM3c$)iw6CqG=|q8Q99{8!6H1QS+!_=Vur` zDgrX*AEZBQK5}-Vzs!eO+qiz~Oo-Qcf)eXAvC58v(35YO%Y2i#)+GfCuc{E+Rb8;P zw-A3Gy@{8_yu|>w3FMTZ4KzDH;c?@mc;wG0^cVB8x((0ZM7hZ%3(ANgD+EHyyy;E&&eP*^_$byVw}( zLFD}3FjanM;9sZ{BdZZeKIoeXJ~z&TNoish+SifvV9#URliSB};&(Bsg`quNvxwoN zR;F@k2$)zs;Z`nxhMQBvLHheixYcJ!Cd@LS!4Gw)Lxl$2yjeho=Z8U7P%sWYu_upD zOeDwiDYmZ-AkAwxi&;Cq#DHkwg%{qiwox5JH>{?^eOVBWm!g@9dog2jD*iQJ%Q&5S zh?N?DF>t#D$i_X6_f*9xs z$e_L{GybS0mAn2JK1>}?%8z=3Yi%L^_wqNRmb99f?`pu|_F@+I`UZHl?khhNLoW8qTl^KV0_+OL(CORCA&V8@gHs%)lQPhsNg35fYXMjL6YuQw zB;KEO7=sc`vf@f7H|y(5MxiMOhM$(CJBy#e>55=-NY0f>Z0$lTQ9e|Jc#;w^Yqrwf zpLDKWOlHotCKnXnGyke?6ydC1Zay;(pKNR^8a>{M?nr+JbFKX#Qnj8L+`YU=;C}+A zY`(&6-nbN;{W4&=b|hqH)j-*QGQ==o7}ruTj0DDqFdYULv8E!&@w%ZVZRh^r?Wx0Q z#`sfkb=_w?KDPvg^`Y?5^AU4tzXI{YDk!MDhojf5W=iGcFzB8ktj`+(8ngE?sMiAa zs#aw47%>;FQX7KIwTbVBktAx#6WIDL7Z&|zLh6K?`9J?8WWn1r}`N_u^ zR8-;EW#fW;y$#G3xjF^|x%_B40$zuR2Th%_d)Es>Etxfa31l361Bh^T7DWw*?N z2V>@AWBW($wDUa({1XGWqEy6l-chu?u1OmeW#OQBUi+@mgnf^U=yaDD2nrDYT*^vt zkQqlJJ{Dl*qE76-QGkD{?{g|11z2jPf~#5^xV#t>RO}i}g8jD(7L*Mq!&fL0cfV_J zuzhKf@7)`m$)hds^j{2a`jLVU&1G%o5GMy}@W zV`hjw0$J7l7~fY9g+~5Gp2y#VlKA|VE?SDM#}(-UBU#!r?GSEHi4dH*DGySnwamIx zd*HmWBt8@D0ju5?LA-nM(#{y6RSE|`CM7L3epz|wd*vUSyXP+wUOcfOiKO;-(L!jC5r_a(>;&2}bUK@}tG ze7F^FUKJ^QOcQ%zjW{nsno14Sp|qL?F<9~vr%ce|wwj;7G__sWn6?8?Sjf=CK@EC( zLXCuz2b*wxSV2+Ej-B`^U5;7iJRY9)i(o?SWE|gl4pZie{g!Ws;lAJk z^JGaUs26>Obg~r8$Pn~D_=wlv9_K<7YB4S95?6oH6CYgk;&PMKAbD^)r)odc$(%7PBnIGizgsVqkR)9AAv@t+Sxg@Mu z+0N)3ZNU4s@ z5(3|Jx&-q|eqm;GGo;KPMqZ6e0veXi6$T|i#;(1B3GThP%WDe$>8pe9pO10dOp2J6 zuuHh_{Vsv_jAWDwjmMyk+wgDl064bIgdTrG&T(2IsLt5}qJOE3V~Z2q9^t`EGm3^5 z?|M$VAQ?pc+hOaK4t(|3n`zgtaqyY_lp%j)=sWlfXSMWc+ldzLO=SS1GfkRSu(5dG zG!-|rjDpVi3vkUl6MxPgMGN04fPQum&<0ON>wzggI^Kt=pH{(`tsfb$LsclVWe`Rx zh2wn7LO8FqfO+|O3}aRw!vc=Pf!$Y z30Jm==QC%0%Dir2HjN4vtX0bcjc^6h``{&hHT2;A)5{g8n5Dt#%nUHS8p9o^+>8!~ zlmt^P-hzXI61Q;dECI1R1pj_p(}w-yh^E{~;gN<)MM&bboThPl)y;+7c z__GYjJB!YDB7g^D@IEpc+Uug3eH9x~ciT3Melrh0h_~@vpBr<2S0V(B8$4n2_!$_G zFqK@6n*o=6J`}0?Ov0DX7K-`iibPu801ec}frQUw*mznBXFb0FV-Ig;{_Q=$2xcy3 z(qGSFjLZX}XZwDpcIRYz-}o6W{nvm>TF)WgY?I;_a{Zv3lu0+yQWJ-j^Lr0CMGMa<1_4lrfvFs9LSFN}zM zBG#BYu|H}T+UCpRHj4*Ew%#{j-}QP&<*ht~bh*)MclD@bdMEl*JE$Cg6sAZlr1EdP z={ZvuiW$A|*4~CXYk%P)dOzZS4SuBKQWW-PT>{WHESll6ga+@op>qS)Qn!cJB=V9M zni;-FR(zMy6pSKzHiww3?t3uJvZCnRvfCg$mWdX#J@K;zL)R}gAgosuqdL+Ti=xj% zl=5*b^9;gAYB~@s=E>S}E_7*AGH0-80|~xuMdp`2X6jAmQPsurH1GU({HF5;Cw>v@ zotA4MGFBa?H)@ceCI#X!p_PexCqu8DeushFsiK?442}1CR`kK>H#!H}5_;nWur>9t z_w6laTLMGmBeUVxOk)y#M~l3^u$Y$pv!O|0GW3mw9ceW6q>s;7&`n9x=;lSzRHb(w znd=JhqX)skSccB^WQf^HNiuGEGm#WeNDA@F&e%N2b)1TrxnF+qG^3fXx8pX!!F;tCOt8}z3$i{{b+l_?O~X-{=r zGQdOK7k|enk`GBnV7N<{wt_9KoF(b#d)XO%GWCe1@_x9S>P*ws{m8`90d8&iR5Cgw zfX)mw``o*xQY<3>Z6-7fHtolZjS-=PDU zNoPv0rpXFI((WAs&UVv?PrL%@zTbh}qbNL;(Sy+tM(My zhpCZCU)E5A?6q{lNrc{tA2=buLF|G2MsF2WvXzF=@UR}7HrR))(WRKStp#PooYt9r zAE2Tr3suLpbAzv^b4gJzajD;JID9INQRCFaJZ^8I>SjoFWvxis$;CwYA%ZL~)ubIw zJ7CyVhV;2QkhK!S$i@@BXtAddTQ=6?bM_`CDha?Kc?$U|C)SD-cB0y?Pmq`J?)IB; zuLW7k)A8ladstBY6rI1m#0@J#ag%tiet$HTX)tgnPd9(Z)3P4aIPWU{(!Y#LljhL- zUG;(sgI6Hb*%xES*^<9m{qV2)GoDH*hRd>l@KwaBN0 zduUoZog_R>z!7G1sKLU~RKoZh{2jcC3G5QQ{z`)m+n9)o?)4bF=olP(cM@OBmBxK< zPT}0ptyo`{g$|=!Xl+9&KEAC;`gJImzi$`1ToU{0W6uj7RK#NQkQ{k3J_P^djKn>= zN7CtC9wg8@i;HlL$NV}5Xa2RNW}goL$hN^)7Y#B{>_P$}w}7OZHQh620UV0E3-f(H zfnl};)tRvy4sU#h7qdGM{sTz;VubDQhSS*g2s|=e25;xKVv;sXf6kP|gTi6dVXiW~ z9c@khRUTvTfyu=3#YMQkU)-PbrW?!GRx|66!Ju_17!zNP{{^TJdBRe;$T+aq-S2SO zFCEt%Eys~9VvXEX9b$rhh-bV`$UFF(`xW{NH@GA~&wD#a{5c0-+R4%Fd!o2ScIs45 zJlkPHDeAE%=s4C5VmoDM{RVB?nuN@5O#@mh%EzPAci_PZ_F_-yA2a2y0Zck@0n|=@ zg3+r@iC=*oX{ZguI}VDZ`tE8l8vG9Zcdiu8_$ub0@2ln}72d?x89;-YVt`Mzqb5Fz zBx$K9CVMW%hWo!!Wr_uOK9(Z8w8UC|?kuwHpx7%j$tb!iQw@6@&Vxjk8ZD6ejK)>R zAna!!(2Nn3NnM32zCDASt|Cr**b(OMCl_33oQh{`${A=|#Jn^8jyjrEp!W9?$}}ec z);ZDBp)JgBD@8iJNs>yB`3SO_-CtZtvnV*6(awAE8i3ZUu>){IT z$zs-7Lvj9T47p^P%>+0~leb?**q8mAX>ogm@o9DFG`7BI<6%W|wyB(xpZHd+(+h~f zD_0ECQYG8V5%R=d-*#P|^Nt=%W7l5B8;8F@k=N)Xg=A_$X58#TRo$p=i3_SQ=LpbdX#CbwKZL2I-U^qMy^rCkV*w+VuWBc*?7s3 zWOUrbE$4&r)t0BsJjYuo_s)?dzPbcU9fYvmuNM+-sE~Wv@wm7CFHX2R8Lu7R1+Ns< z$>C>_oO=6fc)6(U8~G)UlI^wxQ!x3$I?xVSa1Mz*QM!)P5e~$uGu-NVd zbe-|1(p?kDJ|>;Ha76+mj;YXi|5UEO9E)DqPXjL49UZ@Afy)eni=w3&ACqFV^w@xw zC(gj>Sxq?PdJV?zvBnQ$#I+~+-?{H)3yJbW5AN(V`#eR*da}bsc6)JX7#X&Tejh%ae3q_cKKwJFi&w`n zL*C+KsMZcxd3-q-h#BUa`kQdIa2gu_-2}e##}bJN@42i8(aeaxJ`g_GNF!}?A*-?r z3$tgCp8X5yZYG4NyZYf?Gk4he@-uWiDhB82=b3=HYoTxpOHVr-<0c-z#gtcRku$ol zam%4w%#u~&`3OD0n4tGT)iwj^TxghOJPwY7$7-p9a z^KBoXgV`#&EO#0iS~PbADL3%VJGjBv?UYXoc zamPcxwM@8!GJ1ZHBVjqMSfejRtAmB)kHkGJFRf-W_5WeTh_z&Erv+WOr-fK6X(b-hTws#>EgLVk+>Z=$L%u?gze!w!AUM2WIBwg$rf$Qeko?G z*BOv!#;;NS%_zt(Wnf}nQ;|b*1?1R$W1fBJE()~YThbqx750j^A5#kud0Cxf;@ z)Gj29N!u0;iwYNjT3##H^KqCs^ErT<^+I92>1FVq=Ep3!c)4i&ofVAi)f0I5?Lv%O zzk+C@9@^Xqpr@COB&mJg!-tdTNl_ z3t_%DYmqfkzwyA`U)(iQZDO@-4K5kd!%+H*VRn3l1=Ann)0_`5){mg&H5Iz7sS<50 zy>LPL2j)h_KW5#8Y_w9C0gw6uK=a3J)YIW`b1x5n?2a*aWJc2P6)mXN+W>a@43lFz zj5}rIjv9QT!}Kx>JX~f%Cio{Y2F`koLb$ zltDOSCY0T7Vk8cUu%z<~H|mlWR9cv!&B*P{*HdEM;=hkLb;D~I?eGv(jj}=0_FK{C zlhK%4UWnGaTxlF>frJ8vc#Vz)-R~@E{&ER^U!Fvr3s&I4E<%>AQbf6J3~lwWH}IB+|J6&#rgI71!rt$C&G&cv@2IyY<3~nlf(aK{HDK%)yPvmoh$Q|KbCq zyQtD>h_~Cm!I)BG=J16keC0EaOJ0(T(=Ttr&Jk*)(XbOT_ncr1U*;fZlFs;iRCP3b zrbwE`MT6{!F;oL~Gm@{h$uiXA+KML9gQr9B?yFN+dd3R7Hfw|XEe+876_29wTg;TA z8s?5~Cfa$oLbA6W<*z2-t6_%3?92pm@ZuWezKz4WN8>Qea5#iHd13Y*Q#>(Utf`#a z1FO!C0lgWgu<^%jR9ZBwXkDBVaKevZlztKvI-TgeZN1EQ8FzBr>nJY4N=WAgblog> z?rFzSF#c;vyz}?r^S_b!);FA4`lFlqCVI#%xj&8DQFZ}R=RV?&O}vk0`k%P$J@dGp z+$Rh(HUQ4*eCL|y4{`EmHsc*VhS~hA5R4smz@%7xav@Wa@sF#P4VZM4%Q)DM_Y&&hWXpJx-1VBX zcB;X;loE8S{R_*4x@fiOC|-HlfKNwnWQy!_u%Ao7_t|kp0bP+|v@b)K zo7PU18Mrx7QmG?+%1bdSmRiMZP%}G>*Grq9qGr|(1aYd*owo> znvx@RM$~1I7QGO<5;g`-CWTWou(oj}U8-S0jy%wzX=80jh4Lh_ZTx4XfsT+WcO8Nc zTVnO4qg>R|7o7AcPmpcj0Qbj8;t}!z%Ic)38VwdCjSL35yb9Jz+7}s$bHF2J%mqcC zk)#Jy>6<`pS`#yzzMS{!jnnJ%pC0~_60OxR*8Qg#14 zljb8nTkFR$o9{@IH|_S^{AH)mxv7Rp4iDyZ26bt`ha`}?Y=qil&g1laSK=CU3|{!W z!m}?P!2B3xGVxzG+F9*}nO}wpLO&gV@HL%`yQKv2S)>iMs@*u|?ggfQj3QMClO(hL zDwC@g9NOo2Ii6b?2S4pTVyf7ual23mBc46Mu|sY+(Nc}%&#A|u7FF`hMV)x`OVCp@ zpE7bgzl>kyK7)}FEi5w+YH!w48 z?BVHS8S(@YaHg6S)i?SJ--QM^b^9UseB>5--%ZAxe(@PtT?~-{BT#epL3}{$vC=;e z7sy+YYd3WucJc+3++ zEx@kZZe&V~ENR#1#iRE|68qsbD0s%uW#;oBMejMbeyB&=A#*Yi`4^_1JO;m`C1A-< zX=->2VV2?v4l);ki-{#o=m|&P#;Z8N(}(POIsh_%wBf-?8&0nIAnt$m7L3|_xL;~@ zg2mdJ;PNvZdZ$#QhE65-sooR{o_P~Fx3Rdc{5o{Uy>dKWupECq)F!&z5;z^nG5K#4 zX^Ql1u06IJx^KROx{b29(PIeAmmG()R<~gFt;1Z>gB^_3EfLdWq6j-TwBeXUeOjfP z?O67+lq;9dR)vI*W;UWShfW65R13q}1QUhre= zWRh-OfQPrJ67y>uMsRBMRO&N$_)!_8-u7|D4JV-R&oJ_4dK{Qux(7?$&fr0HLwa*T zEz|C!O)f+q!^|#g`sAWA@sBVieLXtR({=$oUC+USmUL8nGlA+1H-+IQs`$V%9Jojw zQe{7$oa%Ut+uWX@`wc?gj@kxRf6ha$=OxI^Uk10|WpLN6%b5UYbuyw~lHLm&LiGeY z@GX<3vIpep)Au)+(s2?{c{2{48Jn=P0tKjPT;HJtk0>2Pb89a*sA6m*}l0lCryxU@*b9V?%WRPM2CamvVYm@43(`pHTCU9u!;Z!>m)jSaxIx9k$8Q_`wu*0+sXBqF?hJn z3c41_QLlgvxZ!^corhn|UmV7ZmXfA4lvISILgSwAJ?B!9$VeixA}QGvYca=_Rx^BTPK& z(QVr9xeqC=UrDD8ZV<#p+HtL}Oz^eL7kz42iej{cXl=IwzhFizmHVX3@yiUwwrcX+ z^PzvmiZgCf{Xzvk#$OZVZ9vGY==RALYI5lSGLpCC-@MGGtMYS^)3ooh zOnWj4H+OI(hSz9~K!q!vkNM}BZ)xxxp`(?0)^`hf@}OJSTVJ0yrFwp|iPl!~dvg|ld7a4X83wUS@BElkk9U_3GuWr^48 zjH5#*Or=N0wxP=Ex8l!XW9iWW4t%GfEm94*M0ZN&Ai?TFRK0o$svey|A5G8~J0E%G z6dHe5oGLd){6%EWpL9Nm%wue5@+2qj-hCk-EE&mJnkCXvf7j8Z3FEoa-9}u?wbjVF z^gY$~x{8h!Xmgh?hodIRAVJ1rAtzlmhL6=f?AW#OBBg_@WasT5`ciFzQ&vkE4OuDR zLJr&VdC6U9#;j*FruQ5DJZ=P6amGw+tLV(@#1n43?H5YOW@_BfgIKpO-;JknMFTyn zf^MtQsmG1E(EcPRl}Gtb*ED2WcT|#~>zg*0*WQWzl5F@=u{u}WHxo^B&qQCp$?nCj z;pn8rbiT$a204uXAr3o2#Ke6px2?euE&aYyusLs_VBf>pX!-sXs4{qg_fSz*B;q z8`F^PxgosQv#%)Z>np+Bjsm0?Jcr-DYcgm4S52H)cNvLKjO4DJoyv&@oT6RN+>ya@ z#Puf1GF~$y#2Dei`p182DI%{-`nPJE#S%*OaF!hF_5NxQo=}u^K08 z(BKO?Gm&)bSbpbnO@54D0KcW}glpN4mAAo9ICg? zhzEVb4J$O|e`J=@Ba`Y;P~#w8Z&U{TqYUDG*+aN*J-KxFTrYmjTLoJ2@fAANpv~*% zIrAp3-jyd94zBuV^qB_B{iS0q?+N5I=Al%@XQ+KcByG)5;!3`b=11JK<+4v?pcUf` z`P}*4f`2pA_}H8&{IK|^$YTwnGv6*o8%M=EZHrneZdn~quO_(DjPiGmHdUj=5j&nC z^}0F~Klh%XKB!fY-~5d32p+-}3-Se1@-|m#9ZGcSSIR>-Zu{~Ue}d?$LM^nR;1AUv zI)jTF>|A+#fHgnKW<19|`zm(05P@7S{6J$It_rrNj1+HrQXzg3mxd;8^rz>VE(x0X z4K#avI?6O_M3Jv{BJVZ6e4@uU)MBPC?g^0>e{0flO1~t>FS{Qj$ear(XK*?_cD+-4 zD@mu4znMhaysn_WA17ry0j~7JlEcWY^#)zwkuO+s@(|iAibA;>zfoL3g?Lia3-r-* z0Pho~%b$F)hu%Iuo0<+7$**tY(30@Q0{!s?RU5~yq{N%hQ9B=?zeDqo$zL(jTcyG8 z4*Q7uZ}-ySk8+UN)vKuaZvi#vxkMd9j?!;a_X=E|L?TU#XX3%no}hrgLpY-n>I>F=y+Ua+Y3U410ud*3KXJriJ zOpGU>yj!MRi&K2n%N3CV^FyP!EMcv{H}D{>Kb~B9_u&_G;i^<1_t$qmz?KgGrAZgg8AzunO+<=!(&+0nP0q5xq-qGS!Yy-uM=j>A7A$^w zoc_o^js)@^eC7J%f-<2!Qi!*rThrs|fsjsQ6!1!XaI+%uAuoJt+>se2L*-v8#?mEVR|b4jN=W3IF$Tt zG|Ifmki5kU8hXFK!eB!k?H*l<21LkH{c>|z;ifj4>aWCUi+A&fJ{&~dRXT$1tMN!b zZX|#0^+-Nozas67`zCl4S%FUO9zhf5YV${1&1HLT0kqz{mrm-_qfhp_quHVX+}$k^ zf>!aVD*v?qs7YB6XMgYsonCPWZMbdBjog|bR=*NP6C{VJ{J2(RQ?7s_@NR)kLZJBW zD?b#N+kxI(oGxCsB_EYM{fe$X{E7VHFNupcjiy$$3{_M_)3dpGPBSi+p(+z)?#R;1 z$aS!SSW!nnzfH2@?zKHZW@^K@H+VR*U9gDmA7RC>Exv@(Wg5n|fEZdQ-a);GS#bZg zVw9pX8QBvxe#zxg=y&n2hnCCwa=AQ|Q&=)+DNk z1v8YWg~oKgqjd^Dwt6zZDQynA*=j~_|B0jpWfQnrg{JiJyh0Sm#&O#f)cGwrmE!nN zCr-IuHj^kGN4Yn7bo`YCXH>ib^{*erl?-x2v!eIVuKIi8&CeI{L(wACxMUiqX|j~t z@>Q9Cy~Kn&`Bup7tBgk}8wya?pH`}OTZvzOBaD8k9m!4Fs=;@EH{;4)DRc8(M{rv= zy`aXfWBGst8(yM2k9UymlD$}H%s-eom#_7GD%dHwg1-3saFaDeT>tDyRR6n;R{T-n zr7AXj>#Y{r8&rt0dkH-=PmKCam(g897yj9b5QKwsXvaxIF5Og)Z<-gtf2{H62fy%@ z`GUvL&x9bseGx-?h9z|3%2Q&m|J*tK%0TX-`YLXn%#**U-H#p`2)KRj0#uaZK%MfI z@)1*4^FbpH(}U;3olJdp@hb|$`R40B{LV;6f%m`rsQr~Lm)NDpjhaZs6Hgm+vgSI@ zuLyJdZjR-4q$TZRYh>Tz@C4R=)#xCiW+VoGUgvPOy}onlp(UL z7@fHsO_$WQ(CkedV*>m8StOp58*o^$ME`wdyzudOxjXCk{_kD zj(>j<=);(|^tM7M_j|M-msae`eJKzM{O+4_w`DqPllu@^hrwbraAz<-^VL#*se5OY ze`Yc2_-_p_uhvH!y*1@|prUth$0b z0?M5EvuO0<>=(K)-;lTHRN$S`BT!O^H#b5ypO@-Bq^^Gn?SEj*ZOBQI=?i7D|37a| z{;ebTqDR2F%x@CB?hzxeDXX}KdLege#sak5YZiApW;z!>(3bOV)ka1ZEmSGqn-A1n z!0U8RoL`HA>U!zr zBRW*uv4QR$=YSG-A;Cv=H?-=F30L#Ol3)1eF{+a7Bpy4SPu2H4qBCNKbCZi?x_0OQ z^k3I@>gZ)DF6%lY&I)`g{_#SKTTp$Gmc+P=2O%APe9mDyyypliHa?Gbq#UMYT84a| z<`GnF=}T>1%@h}>Z=}uYSI|cbk>Jp~*J%Co$>`NHKjb;-yI6aoEjM)4Q8dze2=Bgl zB;DlEfo7|IMqMXT={UTg%I{SiN-N!quIgK({24;QxXJG~|* zm|Tq<%bL(*|2|sccoikOm!P`4rO5Vxzj#rT7uA27hN{vWQOU_bnt0_8J-aT7zT3Z+ zF7V!k^kn`!yb)H$0b3;yQ8X-H#E8qHdnNvBR4 z%r|_F7TbLFrNI}yXv=0_Y8dXo&nrw8U)W$vS*Tp5|-)G46zYL^Hy-ZPW zVHo}L6KGO?ytsFp79XivLoM49oyup{q4@)riJ!kUp&#N0Qnz*I1&bGstD0z8f=s^# zJDqbKj@mplc+=79sMhB(HJWcN@GjEkKU<9#9B?=&@Hk*YmnBa^?feO<>})}oE3Xt+ z-x(^fbSi48;HCdJ&&g!mmh*~b#;JH|=tt9J?XzLM- zHr^#3;I{{fDyr#-i071A8qv1lQmQn^mfjs6gAN=H7Zh9^%MCGUp?N|#WZl0~+}L%& z$^X3X4 z??)D%MzpLz4!Lv`()lM$sUYEscyEq#)%uwqWEoOle#fE~!GI-`xh?nZ(|B3_$8_5Q zp5O749)KMv#UAmC_jDq~6ji~V$vJ3Uj29n&Zy~iDXvg)Mc+r*Dy?8dmf%9AV9LY&0 z@gwq`ImxT1bhPtK{y<+oI`wT3Upz>_3Dlla-^>Qo?vsf6PmST0U1^eKHVyge3Rwms zz>05QK8ZtfKZtiZ>GL0p)^X~)E+EG(AL+KKW4Wj$LVk?1%-@;4SL|&vkvEvM8pTvH z*-qwRn!+#OFV7msNyNqz_9e@9#Lc-! z@i{ct{gRV|<}*6~SfAMc(?fJpXB0Q;$asF?jY?V_FpypU@gug#y)RN@9ln4miu zHFQDTXl{VKBiEw6nN#s-q#NQ~`9HUFkwf|{u4eZ@?v24beuAA?wlh70v$`YT_r3Pu z4Vsj>`{hdf%L_i-)61{v_JPfUC+od9VTT!SI9h|h)ZoB-ovfpa&Hh81tUBqb{5%@C zYAioCP3FmLu;Ko!JcR6*#?e+7z_Hu{>in%laA@5SZdT7U&L%30t}W<88k3xdM z%QWP7%_n5JUBVSc^&{uRVro`XD)Syk@mHGMIPr|vXw0V7G;%iL-dTP?)zx$OeKQwO z@7J=t$+iq?BQW8Y91&gjBUckYp)1$t1bEfyv$=Z+YhrM`!>Waohye>*fn zaHnwuC%@nf8r3*cynBc_uYAK(wretsH$9nx{5tJ9yH%NJ?1g0dXifk>)Wnp#EX#s^ zjXjBGuJ+)x_8zBY>s@6T0eN1lVLI}NFVOEVEG_2qn zZdvg~Ph_+3d=Kuqk~!aIG?fonVa?yaJDy+C_#92;^|}6O$0H&F!*Z?WZG%*mwIMSG~t8CkE< zYh8ZJ$?1Hh$xnJ#xRP(5CdV5rb>`mPO{XcvF5Kj`oA^(1W?a{iBAR)17{9eL3iTXy z2xkL>=)AgQXyV^;tbMh zKO)S#=@%lNR^9ej(*!b;`N1$2A1Hhnk#wSYPjnsTFFP%>W){T&`#Wma54 zod(^KX>p6F$L~mb!$gYI?@!<>rsSaQ9fo}9iEmDm2A-nhm;9y5bN%@bmoL(U+-{T< z(M4ldW>)pJt)Pdy&QRfpttfbAHQmNaoZgzI)3IDM%?(Z!M5QXw)r;;3@;?=zQx
    7?G==Bz6{x(cHlxYccHf%jriXuX7DAl9oe_f zev8AFs&Z3yiBZ?9adi9C8EBXHBV?8qP8Y*%y2IX;&M^!_4k_CFJLM~a^FG^@Rg=G1WY0#c~o?N~6#$Z7EXe}W^|PoXUfT)5B64Y^mZS5Yv3fOb5X$*t_YNKfiM zLa%;3q8Zu-e8^TgKHt&}&1>F_dOfEJYQu)`I~>J=q^kqCH4BF$`>AO(^jr!uzdD^x zns%nrY-JI>sG3hFS=rLvwwAo@<`VSdczIQEtSzlq)^j4$Q;>(gK3^92!AW}gt-!|F z3LVjDK&t7A{J63@>UYe8K2Z#CIyW_v>ihhm-Dz*cHQib?# z{$Vsr_L8tl<=*4bNUT3T&T{ArtU6*CR zEaiNu@0gRQthfeQ4VULzB8PIPAD2;b{kvehc{brrZS$ znLqYw8GZ6(6P+vbOI&sB(Uff`XvdbV6kqlhgk1dK6rlfxw#YJv+w7j%H+K;J+`=qz zY^puId~_bYZgL$}`G{3? zWZg&81%1<1xgicO(Xr-TXszKay5^1}3iBc;bb=b+uwyL>*1IJhyx=a~{h?cMXUtXf zbLcGb*~y1!ZTVR4v2qWMOE%-xzAxfK*QcPc_bs$xd%57d6^PwTBWa+_cO4VBoGwpP zLw8=MqC*}>=-%&tkxJfb+EG0YJvX+ce-sW;-T5QwyWMi!vg-5n)6$FL1$*v_b*D5^ zSyL|!@STAAE?lPO){pR{QPpsb8%vr_90%*EIe6C2>mc~+NK8X+FtZX(*k(EmzC2h% z_N$Z%-w1Qbz=m%eO3H!YbM|0hZq6Qk>BZaLkA;iv^DuiKCseN(ChOe1$;Ox);T4HZ z+_Rr2$hKQIn2P-zXknJ@*YQDwGx*7OzltRe+H$Z?P(#$y@8X7mPhr@@LQ?l=6u!(1 zr9=K34O1pmk-sj@IQ++Gk+&u3XFlNlxi`qL z@Hk@UPx0~z=fJ_XmHo~g&mzJvFueUbNv+U7OCtN$P; zY7gkhxv}A{Dv}0CJic>a0iJWVk?9Y#0^M)cq&m<_`tjf+RvA-(JC#O~Nsex0iJt^w zcKMO=;!F7DR9E=E^8y^aw~HJpw}N%+yve(VPb4lAFGA=LCz9E825|lwVo1uK&Za7(f7JeL2fg6c4NQ752XsyvBR?YVzc8eYRp|pD6nsLx z$hRZSpuEr=4=$bsoq?5-e=CxRd9)Ql@)%O*>yfoaUnPkbT*%?Q(ZjG+!+6rPD;kz-#6xACkbFG<1OoRzB2!)3 zNXN_sc4S!)n5vbtpq&>aQ^6yC(H8pZ#?Vu6B};DKD{_XBy788ZGP1HXvU; zix_Ws3hE1zVcGS+Z1z!l4k9v;{`L>1HQBUMFwtE|N7R+9JQ~bEId+lI2LFaUX&!ZW7Zu` zqCwV{B=ENryJa6qluM65*WJ;g@f8=a;%x<4SG)~ux~fV%24=8G)s^Ia%r9tk^ns5x zA7PaBNZ46gf`33U^O@OlQs!lOtcqOgMTkUJ`oUKMaMM$x{jTSL1JOs8uMnrCxrZm0c5F5E)2P~F(lJ>QhzxR=`zE4?rWoS!?unS z{S1FY{(g!fmqQFh+Pn$#YsiB0&sMS8)ayho$OGmJQ^*bHOHAF~fv65$#x_K?Ff3!m zdvdSx7W;tcjgYfiN^!hsbU{hP?KG@G{Td^QI#0DRUsS{gBYu zI+^@jy^a(;_yS+fErP*|V_>hsAqZG7p3Jpx$6Ci+Na=>FAZfI8_AqUgj4}#4V0XMWmLzFS<3KW$^lD4};@Cvu1V0nKp$-AKig+BM#Mz<_B7q2J$ zX&2(SZ>7+0*jdn$UqVJ06vMsZZ`eqX29}r2Nc~t>Xn6VnpKQ7WThE2b_>6y~m~0nU8acg4`G~ zrsy13P;EnYxFwPq>(b%naaFQ-wmHjQ&_G^>UxB-BZ9-}29&+>ARiajx1eR--;MdEu zVLO=uF10E^cSl01`vfo=J(zgjGn7U9li=o<7ABaylC3_n3ij4Sk?@dH`1f;HQp;T@ zyE2tv?}b}HrVWzDIo*?W@n^BTHRsuS@Q1Tg3_#V!2qt?Dl(g!^kvEsumRt4nj z#Uo^Gt(-J1q6T|UQ4nr2*u=iAN@nQ_+T_tr0)xH3OOh825S@q+!&>-_?Xr1^T|znh zS`#H1F-sp-OuY*$mOqr;Q%+=Yt{gezG6|q_47a5`8FFJo$)v9~uq;^$7Y9W!)3|(+ z{ZUE!)AbGM8|nf4*{$}khMdYA-1N85#AfHybN*07&VuBS`@JJ^F>eo7vSi1    l%)e>_w1v>-<;CyMTAoWc74ydg+3nN)04#seN}f}inuco(NG z>M=1Oshu}r^|BKZRT*b-rL3bJUoixYKoL_mUkZL)Em>^*9UgCK7G_FcU=8&s;uhb- zoJ$@E2N%nW+;dW3`|7W-U92h%KKF~oFVrR{WGuKj3$|e46$>(IcNi)7?-sMJ=mYis z4CvC*W_x_fu+iVYY(w2e{Ba&)Z@4r*xk+BCJ5^QWmZm1ERW}d?DP4nyD+{1x!zB_Q zy9KV#HI_y`qTHFlC2Y87BxGLEVAB#);j;A|Ec~KCF125k?EfZ*T@I`P+s?5hy+_7v zntzv6=nQ33T#LyU`VdNHD4h~P zRDFbGSKC%1UK@j#ED*4|%yjl>#cPQu>^$vIIxWdGS5kg|I$Tk25-yy+ zjx5qtgz{-q$T|Dr5ToD4zEwtpmBnJ{-x@;3OmM|Zmsmig{#z2fUh#mTsCXBrmoUljCFN!J+~Fuv@u=JSwx6dauhN{(ZA#?7a14 z!rsF~d*npYKhB)|9XJkN46K#34ww%g4-6p-0`ketM>0n2;M;`hcT46j-UH_h6{JHd z-$VB{7gFhx$XvWmy)T9Ie1ipF75~@1jV)ta&Cq;2|Sd`)Q2n~6+K;KRpAF*a(xK2 zm6%E!L+6tHNm87Ab~HKEycHar{;-WQCTYc>_c&(ZAV|KFLXHo)LF!71!1I+eiI?%( zx2*Oiv$KAH(>i6=>v)uSER5xEczlC8@lNNU8*XL~B0ifhvWmS&C2~kl zD4G$)#+&Jh6lVXC488kR=-8n^_BiOl$M$5{{d>pORb?J**=t}o=q?%kAH<8`=_#Z@A{elh2HEg!t8hqtdmc;DuSoYg%Ever-Oscf+FfKCO z2yvAOfbX>9(u+fJaP3F(JM9&V8u<|N>o?=Iy(fg{Q!}CD*h9RnIE6F@{ATwS>a&=% zZ+LTu9bPTTl<~Aqu(hj7WA;5X>Gg2Mu(OP$0aI3=S@w~Mr|k1z9i2Mb@sBdZ4<~W z<8#6=S)-Z3_*W!sdpDaSV~@J{tYgM6)JfY#JN#`)6Hc6GiBEbalJ44Y{MaoF?%i5M zHcYQ(H{Ene{VXALyS;^oZDx2nm%Z1AiC&Pn zo5!=;?|{A8H5|MvoyB|4#W~~a$VJmUEKocu955&a4qp5Nn))_miR}G)$Nd@WnlOrJ zZ!(3S&yU04N#;V`Ochb4`eX>k>6|!jI<~X10$$| zFA;je_BXR3d!07>d<99elYB@VcZ2M-8Z7FIR1xhT(+0O~qVT9wRJrm5*4;C@Cb{Q-Ab_>g? zGKUwXhgnv#BlPvYl}tMzn}t3aLsY4%$nsV*7Jkfw^m8-#xotQ3&x3Qp*~XQGE`5sS zo!^n4e-27QCf0%5BMYg{=K)0Zjh0aL{w+x#I!ICtL-9`cMl2`ig-`d!V1Az+300Vj z^L&aRZ`MMhVp2$of;DjNu8}bQ>0{E%$xBt8qlw#(vv9a071qv}M9#jND)HamF4-VJ zuzul0@VOZd6OUR#yqYpuA8jYQOMFO{LpRwOZ%c~jl#;L2&aiaQCOFv~Ltc&xhlF?A z$Pwu?uv1cm^#j*4x9u3u(3lKAZ_I$g-~CXr_Ykyqsz`k$Bk}&!t6YYhlIVludI-pXF)?zCc3OBgF*UJ$ODI~ zq{r(XSQxglANw4kx^6$&`=dy>udkJd)M9uNI122RJ4z&rZ(?1KMEq>PZ9G4Yrv@_-+c`vqDu$ zH~x@x7^#qdN3TP4{#g(hx8sWRaYRw>4gAp0C3=w;@Zk2-!pALoMDe2%Np!1$9}i~1 z>F_T0Gb@?BS+)#;}a(uL>e-lV(HWm*r$?!7>tF6GMLP?_=s=Z*bP(Ka&0>c@Up_mb}h? zNlMQh0k@5hB@3Fj!K}*y*~{@3yOWndy z1f9?Vs$4|ApE4s4?=2!n zY)Z*HuU_0c$c9jpdXhHD0cMVJ$4keqz>~J_BIDDZ5Y4(swtC)GmU%%3=8muy-8$(j z(Y}`?EJ!^k40ULid^|B60vG(iYWW%P@l`xT_b_-JIEUDJ#R!}7CxA&;5ZOP=PPBNu z2}#;FfoL}7;C0I$z?dClv7_jL(6h^tG_4I{+A>ygu)iO9lztnROelp@PqcC6(gsPr zZ$5E*oymqqloIW4JA@I}DcB}YCZ<0-nEtN4q{VZLh_p<=v8U~rN7rC!^4C$&J?$c! z(l%HmcI+Xw7k?6mO~2T&)5pQ4v7Z#jwczr3?aXiFV)p6Fef+(-hpn7?n0$}APGZyI zNTFLhULzOAP7X~Vo;8^ya8!_F`O_#e$G9452if33CkGQIS_=nNUcx-NKH*O1OISDe zDtnZ+54e!wWZG#*_;XfHs@GCTjJ1lOuxAdb8?%yFwpo*B-!BqWpNH4lI*Fc9H#|&< z!an~I>CsC2N?@9rdvsBed8#OGSr1xuM((=Mhi<3AI3&L z49Dn+T|JUaq) z0@9mR$4pl|h642@9DLE8R6Tq{#t2lTf_?9ZjMXAs^Iad3CPb0n=XxRe>_O5tK@W}^ zN0Wa~jL2w(?bv;21zBde95Rm2k@#tR7Iyi#!TyT1u=|Y*>)xWpjCUM?+Fu4z{ZbzO zoAit=mvJ`r76?dAix@B1RKaSGG_afNYe+_#h%9o{0F9`jV6o>G&K{n@&XgH~gQy7~ zN-@AMm%EVMJT2kKU2QnF!&}m@>nM0HC=xm=Y=c0}t?d3f9TGP04VQ2`8k)442>0Y8 zi`-x>`l7Od{MY3Tf4o!3eMS1y1xZa7>D8O(Hp|V1H|&mxnP ztl7#fSunpclW~hW*reZxSmidciH1j@q9mNi>5hOE)rQ3M##**h;|SX~buan4=qSl5 z8U{a$S|wv5BEd2*pEx%B414SyO?#FhMkqY^!e|vFIQBE;C9BCn-vX z>-;2Q#o@4V-vk&tz+9TCE5WIyL&z1Q7kHS(M2NbulN64v#y91>nW6SUCjYrjIHV?z z{Vmd!IxJ}C8IllNGhHYdKNXHGUIjPz7LYLoF*3&YNRjX1>w-~^`NXBVft_~SPgtK$X9dTVR4~fD?=DmKiWb@hO zM1Hp+5Cse3S@a6b!?&@s!6qW>X%^D^p^V$(=SK8CxRFJ1nPiOEM`*I<6PxnV8NT(q z60E1l0=R1=R327N4;EQq;d3WV8c^6(O zRQ=Jw1?p`FX|^8pNju3p?-z_+`~sUlMzif?Ia^+2O@yxXID6qtm^~Y?RYoJZ`EM_N z{phl=*kvYI$vEJ(>a!?Ou%9vrvMTk_)`6D!euXxS4 zdNSgq{JbIua`8jA3~d zR%Dx12w|IB33V6@ElWyB!q=1J*S3))WcMCe|5&Ek{PKsCPcB68I~m%>tFi>!3$l6t z7JGm79{D(F3XZ581~cuYZ0dRul)hMjGZrJ6AG4o~E4+^VHX2H5tKY-j?mU>UB9DXn zPm_0BPk@Uk3zGJ^!|{2(tXamr{dn*&qzxWGX56uq{`>1mR)5q36R%`^h|eC!EaXzybTWtgp)3Z-Z^R!S8VRig z)8Oaw_uyofgTLCHgH7%K$ook{$id1!cC$!9!mlWTn>lC6x61`=?-n@{u*Zmu6V&5N zt@*fcv$p8ha(CQ3^t9yL@rPK&S0CgzJ%N-oQ~cn~JvK1ijEpiJOH2lB7O&Bp!h$9q zCKaRZfJ15(aeV7dR-bXiT|0l{Rqf$$XFveVOMQwZ*;*{Q`xH!{ctF_rtOB1-dM;cP z_nc|}p2twe4G>+s0oCOu#9`MH@E`j_HY;v~4^4~7tB;Yyeu^=ShneKW^ZoFtXE<4H zvmWTTFxcW61eeDjfs7GraN)o0Sl_*e%jee<;oJTCSEP!1VI-pds)e z#6%xqzs!n=ve_c=Fj8X1sXybXGM`+#f>Cm=57|g#addD-N4Cj@rHBO$8ge( zT9URgp6Cg~ShRZq8RGj^D5^J-vbcewhMyGvdo5!#zgH36xt|RILseN#6(Q%H){~JA z8j$3d1nosLahBF`a%*EAgt|nMExq^H=e=I!O3!!_JevX=fzmpIqvXZV4AN*N&kk+}l95wD+_q?v z8L_^2_ro!8G=2z_#KywITl(1Ua1K0G3kJvHUy{|z!DP$nO4wyM0`xD(Lcw$u;vP6e zARiQ&y; zH>s25V>a(uGl{*WLOMKK$goUfXiXhLZcA)LHpl((mBUWNMs7C=?j0r#u(g+jnmXWH zXPim@>dO*NI9N8rnqwooLr@;OQK+@5m`(Iw2$MUii9z2(@YpvBUTA27`IMy)?QBba z#5OU7o(njRIFkgoJT8B|id2%2&9aAMVa@hyxZtIhXlX|ZT&VSdws&RZ^%*_UyV3o4 z@ZxZo(b&N`nU4aUMN&TY!y21~*x3#bB%%u@)bB;7W#iiSeN*84vwLA-Ji(!9`%9Pd4$OtgQ*lPv8ZxH+y=a$Rr?I>%apNU#7O3<_rWvl?lRQzZji8p-TE>saxw zc`AjD>=uB*y6;2!-^5C@M4R-I@ zB&mOuo^(^yBKYvz8c^(AZv3HaQxEm)r%I@b@_$eY9(#lwF_ z6X0>42N{zQ1~F??;f2jKNve#UJws8Zv7au${`=Cv{8FY6#VW|=&;+vJHDmSCJQ6iv zkVq#s6(%g)M^0SJAp`gP!a6Q84Y5}fwsbus>bDT+`3yN%6~i@)ILbWPNn=k znh3r9N64&JU%0R=1sl0k!Afq&na-yZvGazvpVUxPsxt-HWpt&~@)+ryyQ)im9om?W3 zbtz%3GGnMSJVOHJS%H7hWxP4#1aUoAhMRXBBcbu8Wb(B+{D>Mu(aX*?AZA&3_LV^} zzeAA>y^&6yr+I*fzoJO(Py(!uTL3$UYLWF7$&w!dzWSPYoct{@^jw4w)yqrwRs6s+ zD#no4x9vf-@HxqM9Vk*-GMjjgXvYV4+=tGFN6^{0j=OB`$V7@Eq*3;!>d;mq8L7ir zmO(t|+0KK7FAtHaLuAZt{YBVo>uk_DNQj(y8h}v=i3op+=Pd5Ta{GgcRn`)yZyiJ? z*Cb$3@Bw0I8o>O&U&7a~FCpbOY$WU5UC8IzN>CRo1;3zA@an5JX*3!RDb26Q=QG)? z&^ey4fL$QIN`>Z!AL8I^7>}ly6C+qPb9h_xJyO&V8Qq>~qdq@B6Ox`HdtU69wner09u8U%J*ho1XOu zqIJ<_)XsSZJ$?8!%1YHi2VBPBR!cG7v{;Vd($BrH8#vRsyd#3nu2yuyej8l2SznaY zHK2Js(r=`>B)OFbl&dB4<-ad)*Tz>vZ;7+2nf zs+Wh5fET%F%FI->NpBYM3`hZEkLpQXUof)W_JaCOOrtxEKEj_D_tTu83(z`1r~#*r zbLHk6s-8E6?lG=I8@@>Mu6cFQk?%p&bf-EsOX>D2$w}k&)>qM(qExO zraf{Dy+A|MW>TZT+31^iG5!{RSn$Qiiw613ARV71k$+qrTCN&^PFo!zC@X-TeD{O~ zs7=RZ)-kZ)QzlWF^pnON&_(gaxwPuRM(9>p0=tXNQ0T>U@Vf5*$!BnjNypHT~-46e#|9~w|Ka-gC^5~KFHq`352d1m4k_i`fk-Uo_oF+(#f<#uOb2|J|RTY4AxNp!D&2u#VBYyE+&-vnGTJ@ ziczD=L}5(i2b?)-%v)x~k`C8uJjYa^a(`ZfD^gjbV~4 zsyiKS%qNPkSD@)TH`4o-6A;r!z$%T`B=}n>z4dE^>g*C}yp|_~Q|Y$aN#l z-Dl`x3)}gT&o4`0Z{49Mp|D)Y1f7FEv>zW-GXcd4F$8igxqYGodkW2BLMK`N9Q=A-7~QtIr| zO{<^`O1X5H{_5X}5}YM?bD3>0ox6k_^%+Fnf3~3@<)fn9)=AX5LIRaQZ(=gJ7UJ$z zw8JoiMo95dN}M<0T^U5fc|TEdsz}TKxgJi~eGYClm7yk^S0cc>1~mjK#B^8#smB&l z#Q}fno?b$mI7+nRlRvkrJ)Ly^x03{Y*X81^MaXE!Qe?j)hqTQshwifX1Vu(Q^p;FL z+B~8!+$=h;^PiQ|b<3>KPjZ5kxgq2b&_X`T+ro@NM=Ixg8hJ0QA=ATd)0m7w@@7&# zQv16EO=kWe&zyWR{yK?de;N@$+d*QIDmpvwyijWXSM=Y^O=NG~H?sfq3|d`yfI7&S z^W1OiBF~p+-~+u>^m1Jx4KA;w55&8ma*!Mpi0i<_vFF@fQN?J&k!;jdGF>=uwHDpA z%O~9hO1$V6N%YP@4Grr#aji$yAv4~^4LSP*ZEEj8J**Zoi>K5+&=-0&?m$Mj&LdeT zUpl^~j@}OWh)ylyp;sn%yZ+VyMj^X1)WN8A#0PInE8 z;YraD`9|(4*GmEys6%CM_|U(-uSoFFDe76#0JEB%NaoTEsH$NCeRfI-P4Z-f?lxq;~L_#TubZ_abs6p4a=PNYU|CRD6=9M+smL{Ik)BfjYd znBFB$OL$Xx{D*a1<19s5{O%^&Sice7iB2Zt4{J$aW;xz?CX$YxT7lxsHj};u0`5Q* zr0xaE$oTtSI@NLy+O7WvUKlN-6GEPnlHx3S`KTw?bV?$qC2}L(+?q!iCD-a^zCpy>Pn))3D1## z%4X`g{~8gKTt}qlnIX46Em*lpoc4^*r8A{hqm%3q@u*cqm6ODHtKBY>x93by5%&vJ zDH<0%J4ldAp&?2WyGV^aE2z=$Zul%X2Z=w{fa5U^+#1(F+DsY*wL@Rw3;!aJ?EjS* z8Qu}7-ZZ7Fe4J?in<|?3awA>6D1siG4^ULqcX%pU4K$pJMaP?uz)$T@P(^npx|J40 z%^i)=H0M(C?b=>aWhBNMZ0hBnO4O(1<4ZEXd^)Y2VnbC*6?O` z0WCbdA1$!{L$Xh-Lk6}ywBpqXdSj9YU0N-oxQ-S=pKgD0=+R%oDx|=7E#JuSm45oB z?~UNS!$yk6Rd_}bTLcYppWx%`Zgl632?E})8(ii_AG+5$m3P{28`02lqmNfe2_2s| zK%+NB@P)DuO+J>%{pGD7Z1)#X{~6!NP?anCb0{ATDsz$Tfx9$o$e8NnKj!`kIZ4a{ z)p>{IGhzF_1=Mz=1PB)CQ?(rwEsd>(e+ovpc}9_R*C8=#pQuYs4^9vc?PRI3TNjFL zRV1-c6&1+6rcYODlFB0hJ^v+!`|nEf#@wGmG10TWy3P}oMrPBkgZ?nE_63p^=Fz6> z(e&>A2yhDPkck#8;GuRpiZn|=O~(SLeNYiu-04KmwMvL|T1UicEkc9XjQis=L8kN@ z+;gj&nw0E8mM-Uzh0_qVNKvPawy((aKl`Yu-c#Co@f_*=rzuF3T7gtf1R+g}0T^QO zm>v}Cr+fQ8z*ui9q?O`L&3+rux8ysGZqg*r!)^<@kDa2IOl48^p(2`Jr%W|hLOPT4 z3rVlp3-8~KXMfvrh_h2BdJ3M?fpR4jP_dtczfM9?IWN$#bOTa6Gz@QzTtqc7b7_uP zIWY-%C0H8qkhbq{AlB&%P`h9ru~I98kw1s1s^m`cBfXa-$StFp2R^{#jJWX9mMLiE z;6&QmQ%hH8R0>WnD}u#qEXWGe3n(%=h;+7UA=>~Kv|X~B_+QyU-9&w&vbvJs@9RK; zlh0Y2U1bf&lrkXMDaq9dUJfVkp3W)^D2%We5nP7o+1QKj7g3ET^fv1-* zrI(NWAU=O;>8@iJ$niz}ur}ESZoi*}UTnETbjGGovroD-b-_cjdDs-)^*KW;6Pw|K z*8NZ+$rr8ge@K^ak)aY3Oi<5~2V_iCM_g>LLN^+Epmp_XiktXk!`+DpeN^H`{dv$4uIJOpTx|~f7`jSNuwizgPmkBS?ppBLW%!6}UwRk}jF4F_IAab}j#4XIfLcCK} z&;zrj=oXPbt(|E?72Btx%pM_P5BU(Quk+A9lQ24uo`etcv+3fkMkr(X2Krf0MXPqY zpr4iZ$gh@I6lv9nN{q^AYu8flNgW?JJ!eReRGW)d>Dr_2bH|Xvc{!xIGmutqut4E~ z-{}R9VOT7xTQ_F9(2bJe0+XeC>EzSnWIoq{zKCeVO`q)Pp?|*Akfcsi^YYP+F-?^J%88Qg2?`vh}l z%|SATdZ;XW6aABy1NHCmD4l14)K=yoGaezTqsyV#UQv&obB~^oPNB2iWvMs)4htse zz)jBEsqXm+ydQ~g$nGuy>3FyYIb_eJQ_Be&;>p|Z9|InU` z)hMZ75lL(+LWYXZxsMF5BH>0CG}@2puS2q=dqxCWza*1p-p&^6*FJ>iotuy3`o5EA zPomJ_R|=@}&_teHYCEM}`_b`>A8D}1A9}~~ltANW49&H9OkX!2f?M6E(o@zG=spo! z_Tq$T!W|v9hq7?wARR zYzAnW)G9K(x}3CdrO=q>9Vk@oL??#ppjg8q^tD6-*}EO4^SbU+yOB~jsCE!7x}SkK zE{{;Ur4RL<>_PWbJw)Ccizv@;1Km4x9C5}1p!Z)hsL{0-?fNZG&WD%M)0zXE-l9a} z_WA_9;PsCBRUbfhCgMo@Ybk2mo=r-=>?J2pRiPkYK(6q1QLgb%a?9Z|lJ+`HpL9MT zZKYPER?i8Ub|;XzKoxlpHNb*Pzu{(SKKi$}5=o^@5iVaVfnN9iL~dXrP1LFA4 z1eC6~pMDqRWfPuw(oa*@T; zv0^>dv8X{`^Hg|s@n$^Z=HKw@Eh9S3D-jLOn1Ci^zlJ>vrX$&Xk0Dno1uhWHXJ~ED zKtSY81UyhiGc4ul{^`xId%*$ppM47&80jE^%YV|%1`JnB@fbQ1Tu1vQP711O-++m- zuV7)j6t7zUF#Y`499^Y`Bzo~_nAf60R~om{gC|uQ{`#= zRDC*8G)uC6?p^9P-vQ3cQALKwyy*;ak?$3|mR`H|0;Vl*jcwP#Qj(^7JwSL6XHw9_21~ncRBhb&X7*M5{Sa{l95-BChy^51@xVh zkN&9})8`)NQTNqww14;a-#Q%llC%FR=o`EPD)3$k7TJU?>v17R--!_ z&O&qGiR_+9qjULbB(T#C270^Fch?v+MQ1BLe`vPwn$0?@tzAPW*s1a)<`>avKejm? zJl+Hs+lSIGZb3Z#p2r}5JOzmj+=Z3-E6MPpdZK?y1D>8zK)i*eo`lHfv<{l&82}=W`NAiUR-lqcTol&CQtz8w z+Sa&{DovIaUWoD{AxD2whEs+1_pPSxnTM$K5(8v-DS{-bzDMBcY<#2RFWsD^h8)&p zAWLC3)Ce77)yi+8I)z^36`w@jarTo}3lKWEH=2IBX+_xX1L(quFM>LuD|D!EL~c{O z=;OCn;e2ojpStq}IqEc0qthm+=AjLK=;{WYr3;Xy$iKJl-$+}XG|}auy@+UgqJ)Xx z1+7MRpmO3A)I947Z7RaFMd1jF)Lck~|EBVu+`0t~4P>Zxj60Nko=8nXJ`r6^~4f z?sf*F+3h;$;p`l`oL@^Ht@0B5cSIAbL{CHAVO4bBs~F_eT#2gBagln?D7QcP1$pGF zPH$C3p$~G;VP|#@ck%rKZbCv1s!g0lx-at38kv0V;GsWI?a?*V<8OqTd*&kJWq;wx zl`8b(`cmZb2B6eOIYe{p6OCafpyzJiNlW)FflTxf_;;=uG;|C{85Wn&5hzCgo7O`* zDzl)K#RPOCbw4`xwVpVXU!Y8#4EZAbkK~D~!{HANuoZDojf^t-w@6egzDz&|`{vPi zDJtklW;f}|n=Gufu!Ztlt*LQ;1C8FO1y8MifJQtO(ZRQMsGv>-UOB!B-G3(X_#gj6 z3r8DC%eRwM%jG8AW1>g5KUjt=&)Ja2Uv-3I&rF5(>8t3ob1I}O_YizATtSzSG1B)n zfg7%DL8rJ^qocmrDE$3jlKlM-yztHee#tL{LGxs2(%iZ9z+WxYz3dV#SXM#e%5C7B zXF>F>Sr*mR^@H0a3TV0gnBa}1D$_)jQCM3Rs(0QEQ+Hma*eH=0MQWfR?_0Fb!5Bq0 zl+&?HH?(@KDTM`%DDGYySdg}izGart+a4FW%j?h66ZHqt7V%0bx2hGcJ=!S9jJr!e z=nJ`vPacB%C!IoKFYm!CH)`k#%{j>GDVI)uJU|D7ZzFK}Huv&WPh^okNU~Kg(Y3SM zMDmO{b;@s`)(cBmebsbi>#`jmNXQn%wB14<1qOoby|VO#UM_m_(H3sL>Otz)TttrI z(`nG;CeaM`B9bD~tdu^Bv{r=}Im)#bI+^P~KMhDOw5bPtAwPAu04; zA%-bG$vMYlYV$FG&VAU9dbbwSD8p#9X>zOJF84flBqu@ioMy9ljbl7TX;Z_wiNH_)pf1?p|yLF;Djp~w9n zpdErjn$e~tTrATrD0`<4*RCC#y-r$lP2MHzc4#!|UkS%`@nBQathHpFwPC9En`y zi+tYrqt9V(XnaXK9s8X@E`J4C&APN}N-a`0dxX4;R8V!>4pe*U8ntVE zioR+5hr(WLN1E;o-J37c^gq}-MgHI^!2CGh=0bH=g(C+li0s@@kZU`cEZe)2rp4_=%5yIW zl#e^1l#K{|UMxdpl#if2^Oi2~}^xH@M z9o6Y6`C8;2z5^XGaH1c(b4ee6mB83Ci~1l{WT}-#D=1etM@MAqh|`%XWbr@-r5XWf*0Du`i3!@M>FQc^ zb7c)}4mG1se^WSF*%7`N|3k?BTk!HGC1@<+NP}aO>81{rUcReA)tV;qjJyvZ31J$s z6ec2{v$yDj2N+7qetQ zB%LMdZ!C~@>VBZ~-VBvBxuS2J5OnjekUW?@9U8oKg-x#>aSKB8Xh-A=?mM@6WMp-; zpuKAwth@6>z+Y!VPdfad^W>Ds=G^0O%6AKN-rSe##>pb$yV5{Okp%tz{vm7%EJoMO zRB8F>2u-G`^xWEbbgp{`D&JK~_U3M;3802Px>ks)KhL2(Y1e7!n^N?BhboHM&`QLL zgHdO>4Sl`O2?g7fQSpxBI3epL{99;6cNrQXgL-FSvxcIu+sltTmNkcNaH)b0oB9M& zO+Qfa*dXot{shi<{sg<6X8OZWR4}5{~2PhF?VCdPvD0HVa69sgiqDr|M6j>=dP{tp+KJ^Z z1hw{VKqfs>^sW(u=54M;ozvw7sgI3Nz`aFusLzK^bMl}PB2D*Zbvznfy%_ZftWd1O zWF+xJfqIJca=q(Z;mI&@SYswS6W!JHx9%SLdua)L_bP(yO4?1NeOpmM<~i=D%oei3 zaT%>yKZ&j=Fy(c2tfTI;JE`IQHoBxhVHS@3nz zIDQnh*GSXJsr^V>OdnOdFC?>EE~6Wdq_|Tn_rcclS@iWION6=!am$s3mcLW!+>AJM zE|WsToqLFv|2FjLc{Yx1yH4F76d>a>|Hu@z-$am`h8~LcWGF?q)4DnbuFLv0P&VKI zkydyIee3QbF)@+0aif6@$jM@3uTkjTU`8)CE1D-}OII>9`ofTya(QP43*1IJ*zBYsC%KxHZ+5}p;u#g_l?x)!mCy-a|2I8-|k&5;i zaTBTzBI7@HsCQm7J%5X#o$9h8y|;k$=WIaRwPF!CAA%m2iSk18#^5DM7vvh7M(>>8 zO0_CAk$#gVUB3DZ{Z{t@EmoXL^>X4#_Ay_i;5?Jw&-z4x^KBH{e}--jNPt^98@UTC zC!-S+9r3EwZv?S(-@`K-8;R{2HQ{={0;<*1PSV>hlA$|qP$|ttsxS|wecwSOKK=(E zY^tEwE~KHn$YXTmz#oWXo2m1EZK%mm0o{AjNs>fFzfaAn)Tnqb&M6F|F7gKSU(gWz zB;!USzkPz9S{2B#RR?*T{0grlh?0W+;3hG7Ug*A^D0HHL+#ixc|N89d3;ac}vw0uY z+gbwaT$F|R<-zD$N(q_mW+I$4=OvA?7R?QoOhM_ajBvieE7D}On!DVCOSg)@p;0-a zCGxLlkbS!kP*>5MUB*FcH2GOOT0O&nx^{%Z+eg)*j(#|@Ipu{~j5ZIqQEsaOfMbBx(<9RRg{kn&!eoy6g zYUsh}>=Fx0fQ}f;o05Wb;wCO7n2OKrvem z3Xj&&C*?Zu=3kb^es&_t3CZZ9;t4QiodXb?G7Oy?rz7v}GQ0y@Uc-0a_mL6{efVzG z0`z>vLvDJ+S~AkzgZyS!(ogRgy148<=s8eAf80nX7UM;Nu?q*#^jk%!_Wl%JyuTSu z!w=A-TZYtfw>D2*cRKpDBL>}03`Tkd&&X0y*2v37l`E}b1sd0$;1=k%305zSMtfgv zLc9C+5PoNv$Uk@pgXgLv3RBSx*_qTes)Kg-ISbXgtzc=>oJ<|^jXvS3`R60AGDIJR%^-)uo(gt^ z*igyY)98>$>l}gkR4wchsMd5xPMW*W)=)3x`)51NJ$#=G?3~KWo%mR=>54h}5TYqW z5t_WbSA_BtkI)VC-jdfvTzabV2jNe;hLlg=gi&A>vL;%hT%-aiQBfc~$M4kAEt=eJ z)e$~DTueu|oPuAr1<<5uCHgkWn#MUz$0r#~>lbFAsIMPUyoWhOxyh(}w`iY_e?Jmm zI|pe&4ccxgNzXrtAw4^7Xn^<@)N%A0t=YdAT39SXCnoUe1yLS8tmrEF&|LvLPFthA zWxB{f&l6dGyCP7jxDTbazoL7V1%owS7f{fU2RdG_Azo!8~J%K=vcOB?qOmQlF#K)^POU-q)&|&Cw%70 zSsc7@n1LWhKCUBg&ci)3*8)l=mKNeBd@HGPSfxXCXg&q2S>j%`8_$Bgj_E2-tz9j=^ zPkKoA3+LZQ4psZs1?{FzMB3{f3wSwN^s?JOI8BX*eAk~L|0YMFr`4ZOeTq4`6mXqN z59rZcsSfs+x)3Xhbs{-mUsSookP*foK(ROyx%AB-g(#y`{0h8ed6nK}jN z?}C5u?e#vSynQy!S6fJ3y>}6ZHwWpp1tF-$H&&SFbrESC7N8LGJhWeaGpy*AL)IH6 zqrf~VNxu<#~6))g_Mf>kc^-({hFWa@mN)FB*w*rXeWB zP)BG}=7=&vM~J)sS#)%rg>a4YVeSV1B3e4{K4ONOXzX@vQTE>(*)={V>$oe?`r;LE zzr#IpHESXF=k?o=r;xbzkoulxr^aB#FFbNIYZ>4YD#>gZ?d2%MJ7UrLP3*`=H zqDeXmf^3f%a%Wu|Rg1ib?DbC5*H>(~6PIX{^7lv4(e#BVQ~D$=S{VmB&SsHQy%nU5 zC(SK=whF~;uAmQXRgivq9Cuycdi3gXGOc+UOn)yRqn*Jx?y89H+CEV&Wyfi_pqp-uh>F5-%M zpwwhh?YdInB{#S{yF@J&v*@g}bfh9_fbw@7p(b9%-a zJNL5OP`mx~+q5U>P4FE0lgQH#;Zo?Cy$|j0+d)hVt+^w4SKwjgaP<3h4s91tK@P1f ziby^$$|5d+x*rZBb77KbYeNvUZ5^hcr6i!+!VUCC7kXPM6X`6tL|W2Q1YuFSRQOn#4u*fD=WZ*}8z+b8qWThgO0I=^e5yxhU%n^r z&j~1T3!*dLY()QB?r>W?cXHS4UL?Gj{0$x6l+LQ*5P{6m^Yp4)KRPJQ6WE&mqfZtn z(wi1~y#GYc(P~aASEphYwHjVS;*P1&wJXPHgrq)5dpfOaFjS!{zFqZ z0VueYBI(3qDC2P+Q4H8c&M0M5b0=*kbwUMOn&}PhUcbWbyrlrz9+fgpbut)EeaftI z`^J1oiDj?Nn}dJUN3mnt$@pe1z=B`PfVr(V4qBoK{MMdlmO)8;grf%}*O;=g>!TPW zDIInlU&Ky)?2Cu%y|7s04Dgv}iz|8-vVZUW;|m{@aXK@va&E|k;_LG@@tpGZ7T2n6 z>?;33rpHzu%=%Wra?lLGUQz-2E)Kwd_%pjw#S1w1hVrL3JYwtB8d%Hf-ORf25vFQ< z8t@2}XSjz&zkhm(u}rFBzAX{)nK_}DH-~jPB=3fYcgkV?XO`^8@E*Q2BMatVeaDnt zEn$X}y!hvzTClOg71$teH>15m8q_=t0c%VbGIQ_S0=e3`prv#bQ2H*98)mHlHwT^Z zrLGvfvv?+utarlNGn4rVvU9<~i?^8QcaGqXK5-ka8{oAWW51*9!$g2EZ2a2-dsF#)C+t%ageQOT!io5orhD5l)yc(2zNP)fto{d%olS9JgH!e?`-XalSjGu zke?}*>yW^kGP9ZEQ9Z1gYb?`wn~R%8_uB69TCDqV2Z-Mjha+xCu}Ytg@#kptFk^aO z*-h~&tl`^EHYAAKtTs)^zIdzwqE}mki+7FLZI?Vi+ALYzzMrxOKg`2xKO135g|%2K z#0x(Wvj-I%cl?*wVgI$#*m+P5(7Rl`KB$>FP+7{vR=wp&TIn=zGSI_M=Kf~2UcBI> z9h1e|n;){tKB;*1*+}4_U(ZI5lykzq>VkqJ;&{7=`&_@O8Hc&wVuF6vGPwsF!1TABOw5vbpvTV}Z0ehhM<>q* zWAm=E^X{*}qVN$2@7sYdxoH8Fzq^?WT*|Q9^Y{xBgTcp{e!$qm3;*hg6WOUd*igT# zjDGMGV01ymSCZ1i`#Bq1-d|ja14Vo3mzKO{&e+TWW@WFLpjU&;=I2@L<+)bu!HrAt zj^_5}gbgjsP}Dj`@!eHsCCg==-wwucDI7-D zR51UM3u|~Sh;x-D@Yh*S1y)gM%tY>C=6Bg6Ci7f6FwEZo{!Wgx*z(>>Wm7?1Jb&&++faNaI(M$JpW%E^KPa1%A@UV{At5 zU5<0P7{4YZn-z1K$IOUbjg@zbm~{23K*{Jjzhvbe@Yh2b%Za^WV+L$-Ttf~gKVkw{ zh})Qk>NA}6172Xmt8Z-ENFV#~n?HEpZHb51XEF&X&sgSs3J6=!&-5TQtl4p$EqWip zSn6frhX&G2y|*jglHZHbX$P84G~x|!MvrP_%6byRg=NPGf{Z=YA<|J#Cn(; zYXFjdF2oUv=b12nbG%v62|SK-!AAPgpxh}Gm=7m`*=Hm`f~hNb89xR5^ozvSuXX@< zIu6w8F9Si_Zm@@v*8+Oo2H*G|i6#0QIK~@h;eA!Jn3Z!Kar9SBuy5fJHqcQL*X$Q@ z1ai|@v4S!7t4kx>Xt|A5Pk6v^d*$)?y44`$i3c$6?qT0gn+?>eCW4NOsCcml=4Q4HSlj-9K_1#Mru`R!G9 z>@J=JPEHBHj@SZRJnRb=+8<;+MSPG83fI|xqI1@4QO8zI*JRGS9OK{mT*`Ra&A`W+ zELj|AgY~S<@Kc$WY{%$%_Hyh>FqGmW;x)wKRa19^&m0~8gmgW|%;ArSVPVHVZgi9# zV-0Z7Xt+qhT84#I$C$V!+nDOxi*QH70HeG|4X;d%XJ&8^0%rEvs0JH{P4h z4%ltL4+Dbnf8D;A^F0xmMrz^(?YG&U{BNvXhaOP)G7CSUDj1J1#XqlEfK2%utR>`t z(`!#N$~zeRe8VE}r1k>qqw$~XNtPF};@%`3t{O>=s)YP>UR=atFEG9GN_ z(|UIM3}u{>Dh_0KPsW8ezOxz^?U<+w&)92oI{BBM%Y%l#3MN0u9iMy~!v4;a#(cOL zj|YW-%GJg=Xz@kHkEaJ7a%!0kudgwdtAauD)@9(#k3;lrg0e#I1X6c9qz`8#eHS@cypTuvbeKZi~MyO-oy4`FwF$G;F>jAfF z1!yfW!g41Wux!*3=mn$!;vxqgrLO|2joU%mY6vdO=d&x)Rsr*g?^#P_(Z2rcmcT<@ z1#h?C&AB}GhE?Hj09C3!_}RQnW<Ghp((fw4Q{2NL*dc#q{uX7{yT)*)sV)-&DRJUs6-r>lJmh?5*=kE{O_aj@;# z%`co8=~!;NBWPYB1IBLagZE9!Afa2t zVK=pK48C%dE&bNWX50y5WT=V5b>D+p-P10S+mSlhevL8yuwIHcnY)Wt=NfZg*z#XB09k zi3FAjE9IOklmkx%0N+-a3~mlh#n*b4V8yJ%objN!c;&)TcEvp_>~?$_IBIQ#JqFCd zWtpk?rPzMP!&wi@PPxDj7_eZfqP#F1Uj&YtY{5mQzIb~72&3+faL8#ZEF0;K$42tm zu<=6n$0Z?S@_sH;xl#rsYbszeB?y=}JA6PI>Z!nQT4E#xWcm zwa5av*~R1N)6yWkDiS2f8-mzv@xU%u7uP&!!N*EAG{zLmFOCKO)g%FKc`Bn=ln4wNPrUQ-D647R$izD7VQ!s1yI8k? zwVPyv%ZwA5k$MO)@x@uOr?w#Q@C0DBqLA?!6mee0dRb3nTYUPP7m!uhjeF=8{3bsc-(B#W zRaQ`C13xUo!3y(n^-L~UW)zMmg(z|~-}O1=N1b8+Em;J*>$fqMDe}zkleb0R0nAxZ zaGvQ|XAYEtFmj7$48yyl#UJ@IxX zs2vz*Wa?dj^R&aPjj0Y!3`_%_&y2Bi*ed++V>mwTVZnU0UWNU_!r0!6bJ)D&`&xir zD`$(fJY+TP<*7*uyj~s<~ zPSRyI!}SsC_DjUIdGdiV-&D)G&i2B|K_YJTx)FxamSWC7%W`7gRxzu$m(K2a-^v^@ zo{LAVr9orLbdW8o)v)(GyynqNz(1q~R!3v@;pDq)<*&Q!)5dUca-J8^-oKaCDO?VA zRp{pkzA3~yynr1g*Cx7*=jcU#b9<;$PhW(WfJDT9Y)5RAI`f}KYGV8~qx$VBUb z!Gn|7GvA#-qH7TD^v`Fb_BsNsExVa7BVM@HQwdzVJOSv|9bp!TH~^oAb~3_|9Zb8N z2cK&{nJI>MnZ6VK%+p2w;MAoqR?R91EblX6+FOpXA1_ScEW)iE-I*ayD z2J0pH!?R|CE$fy8IsPu3$;<)s7dV0VnH@}#n>b^*A{I1XTEiL`PY3&aR6vDwIxsq5 zgjJ?0;}`p!K#+_jHg2%u-><7|8Bs6;`uhTKW4nkq)@F`*ykyXOb`5rC%)v1U4=1@W zWtAoa1M&ia!K6@7d1n^b^>6|6|M zzRLjTd!&Na8yWaYTN*ar^o5aW4Zx=@+gKkFgYtRBb7uTsH^;bM0(cB+uv5=OI*mRK zWRE^mWUq~@W3x&=+dRn%>zXgeS3l0jH)g)!JUz1=$KxpA)VLL>89L&ivZY`U?8FNL z7vdS4*Me@fE!gg|fUQ_Ai?8IFx3sC-V(k}U;GdQ|*!(Mj&B!+b>ouaW68nXX&)Eo0 z+J|8A^)qp~gFG%spNz?tC0J`H06#_ZvCmZ-AW)9OdABElJNHZ3NegGPdmb8No=qTY zySt|5J||27IuAtbwi+k!h%W_}j{5_t?F=Yh;|%b?L*}y22H+yU z20t8%$5M(n7^R7cIO)(<_8c3+mR(SB+7<1KZzTtShV*=9;a6)mV(DQ9Eox{neEx)4 zGryPZek~2M0t*@a-PZh^Mj55#s4`sZLN#facwzx04 zjo-&x0OUT0;6raevUue*&Z)XVrr>V^o>r%dbCPsH=PANkTAXBOKDFcbBrn4?ftsL8 zkcCz3J~G=*=wml?5nuFOEwj_3nmOI(0StrU@Dyte@apPk_Wp}lmZfTkGvHnVdRhE5U~8&&|iuR}mi`(iMDaX+)N#0;3SGeA&*8jF@gaBXo6 z2u_9Iuv;zr&%goPG4kOoQP2R-TMF43+l|1jkItO;F>lx{b%$ANyYq~3+D2^c?}k(K zJwbg^IOg_ev6*?^%>3(b*oLD~jE1HWSp7~7{5rvae`X?f$t@?WKPwT>FI^9^;0ryTZjemLHu=z+JkZD7xYxnq^_LH5-Z1H8lcEnmB;oYhiX#?F>#V7iAC znX$Z=j7_u-xZvK;pBk;hO!9oolyD7k?^RE{dP6kWRv3n*MC|beTYbS~|3}P~4~9Vf z&~otK%k^N@;dyx9B?oX|Km+$1YGBPyJ>bvF<*4gRV`E7*{JvBjl#h8~xq54SCj1Dy zYUDTnn3OTz9VgCyUq6BI^qK0U)Y!pPl2-|%_Ra`Qd2oqIs<-0H4*hYkC^G;bb3JicvKm%af67J)ikZRSU(A+- z681`G8Y@sX=g2Nl15&dWF;)En*7%VzducF=c_n*+Z850h*vC9!b+n%HUz|(9mws=- zSal6hUcU)XHXUPI3gQ@-Gjl=I#|S`t5U_|)19OZ)8}0`JX(v93*7_Nmw$uuY$4l1AnYvYX*#SB5~vmd9e0lFO#_P5py#9C38%u1Ey*~ zkuIRbEdJ~cY-Z$eRyyf&o`$Jm_iDmMj)#HtlD~YL*$wRaUH0ILyDjdDF~J4F{j8MR zS2n)>7W?p3EIWeSK{`7PZ=ZRE^NnwhqdR8fmYG(#r!tFmNL2!*zn9?93Qv%8xsCmp z9E^JoI5T?HI(S`-FWztQl)apFo^2NO2e{`48*UoI9*LC#d!DrLZMymy?*LUW`ppoZ z_)i0*aCjKX?TRm7yP!O8R`BSq7jGsNhgx?!ytF9UH>034Y z;m}pl{dZ#?99#$dm7SS@ItSpC_@0sOyTPoF! z<1v3J;3t!a|EpfgCVGW48y9uX-n>j1Jc_%`-i#FWlzz5A{=i>Uh>Z8m{`lo7UY zWde}1TZz42h+|=+GGFhw85gZzgs()BO|XFo_iKEovQ*)dMJa1 z`Qf1J{2kWg=?YNLJIs0MIt}}VdSUt@9?!@!!M**i__2x#c;J|eEx5NCex8TZ>&p3j zjdxj$+X6B0r}{8osoxRbirmXKH6?&p*VVwag2S%yO9io~l$fhq?ZAP-OqM&M1#0J? zX4t`b;7LF`lltx@b7RR8;4J0?*b|en*v^aX3Q%1jB}A8IKJ!(bMxjcwr87=y}DxwfZPUV z#v6OKPhiPZ{$~#g2467)SEPZ{iDE|K|67V@ZUsjlTH)6sZ{VP;FLsXf#r&Up7!itv zF`o;;qrnh7H-Ycuvk!nr67%u5wzW)%`7gHp<0yY`=pE)`HV?#xJYs0L8F1Irz|pfk zL2jNAsF#rdb)g2>@e}}o1%oXaGq$q#YS!X>Uu_^eew|^jh_si-CcyfKEBJn5kg>+f z;8*W!HdWjRJho55Y@j8UG%jPB6*po#$z1>%Zw5kJRS_FF3Eb02Wq01sWxji9#NgLr{A&0{q@c`Sj5NbvnylA9K_iXkzW0NgENllI?Se;4VJjH_dcBk_u34hZJ{YRtvA7j84e(=YXG%zRg7bm4kqkAMf7rm zafWLWZuxH#J^?dKTW(BWF5V?@sPdBibm~T2%$PTgv{C?lpa3= zx#C7TVV$Nh^P&xX^l`Irm3S$!^sgf{TmeS$|GBG63*d>_KSn>Wp4l*@j;{i*Gc|H% zXmvUYX9gL=&zqU}W4baX&sN8)a!L?&r<~SH*)FRleE1cx^o-B`ZozsCEJfEa7?uxMDhY?o!uO(t9g^awL zrm(d;3yRIe$lWViG=D=qjqPXf{Q5rfFwOua5AxZzaZTi7;Ydt9ppT=%cM5~jZNPI` zGqr34YN@CUohv6}U_l0qog0JiICZ=|>`M03{i8;E&(fz}QfM&Q23K70#GHrPWJ6vl zX)4}LN|OgjEzkFrU2dcsXH0-*%MCPZ*;I^uI|aihxzJUSrr4h&j=R6QQO&sqVD{Sr zv-HA9@g95Hopy?PM|2WD32W4NW`XX0iWr{qmCpDSAl!I}&t2`GORBz{CB+9TXi7>H zR&oPm`jff%eO?z0o8U?wrr#x@PaM%K^BTGTa+sMiDwx=J5}-oNOqa z2!EI8;VF09Zu8%NI!_6k~{68lbwf`c1B_I zt03rY4a5cIsUXEHhDqtEc<+5A6a?QR=BJmF=}9r@uuGiUS?p&VRPK=NML+16vJ9w5 z3I@NozHr^b4+Adyz($^1QN8nC9-w`cWq@%<@F#YOngfb!< zIQe`Y=CPho*zE#SE{>)zmZXv3@BGa9!w2QsXVShabE$&F4Dh@+6{d{2O#6d**O_q= z%Cw~80)K7DUbhHc@3@1^%_w*;k&d!&3P82|1)0Co5*+po(8(2jbWgkqb9Ry@cHO;4 zY$D32bB!`OrSKest_<93T8HgpR)frt7udE%la$&-&|O_h>v{%-za!+}RN*4rqIi<* zIpPD?oy@S{*diFaGm-i!8@A7Syp@q_VxfC$5p2;cg837FlQ+K}6Lt5gIAXFSlFd9} za9{@B>M2CWQNBoR)4(Rw8~t==g5>uy#>};aUi>TpT+K$Bl|Py3;sp&C3rD;W;0u4{r=Zq=B%di+ zZMRAx3wH*j!h}WZ+O3Mbsm*{G}NM={DHOf;!q%J^5@)icsu>1tIPBkTS34dB~)9^^K|SQkWRY5_qaS5 zTwy}G1||9I#b#Rk1i;6=K$w52p5|^%KpC1z18>?f!;gI#y{;qVgv2?L`IImvoE(I1 z(?gy7@la}*iH7C*DE6vCIC5DOPPWP_F@Vv_< zVpeJeqp!=O_)H$8TQwBcysm;ft*-)JHiCipBmx- z&%@n3(l4CT`-VBV?<>7rs|q{)>KOIIA+Wrn%kCYo$;h}BPtA8K;n#xAmnh|KVZ{_&@n z-z(?BzD8ZhoMXz2caEo1W>14dPb2Ya#VXQXTSopIJWh6X1fa7p1t)i`fU9R#pv2Qg zW{~Ft%+D<&?=P9qS6O@}@4wx4IVs~{M2|j^S>On9hkf9-ydzBf(nuDB6!Y2auSwh; zD-t+C9%3w&z|2TU0{6JV$Pp7^{N);w^g#+ccZsOy)4fzQyNem=u8Jz%YpLAM*YugV zI{He6V%`uXi;TVS)pk#^@#}YDE4c`+gx;rXw_Xs+1dl}T^O~?{MGK?Cub*gZo)fl|3(m`^jAesEr5mLJfgkD~O z^i^CQQ+@CN(Rtb3Ue@}GJ|g*acic?o_?C-wlcq>G>{mioR-K~PEuRZL|K!5+J)Yp^ zA_)=GQXoTOAMv*t0l(J0WJXpVYTp=kjU@JHK={J>_$X*1ELg7xL%as)$>`m}UD@`a zWG9AGn`V-4lTyjrN?u=+dzv<~Z|QtPPZq=vXlmZ6CGRIebpS z3rf}ywoCz5f1E<@tegNI7=dx=MwrRZ5!;ofqxlOzJH>60;8u8r-f#2-#S`jy=cX0j zlH&0%MJaUrtb)c?;PQfWw{y2;V-5 z+Q{XB`YJ_O`kd!A)|)_xfes{o=I5lzE-3re1wX1ErFr8DXlORCtN7VN7V{YWE8W9P zT=#nVzRwSPl1IX)0lqFi(nMRaDcF{Ch~~>O_;PJ5PMq=dY-G;&R*`9WVtDaxB)nYCb6schQF6W$XkYn{?h5%#3->nB&)q}9%uQRVb3#lz zSv5%R4zHkZ?LAF_9xlK;j*eKYKKu9lcgK*XkoJ|G5gg8mGeeZCzx0^L)6wvyv{DFH7$780Muj zx0oejF)&ohP<_T-&cefpp(M8j)#q>vm5!SO)Q7SeNO(xkQvGm2))iD59) z30+#P;it$2?rHxf=Fdz|PD%HNdv&IGaou_nHV8@cH zq-LFvTq+qPiLcKxDL+n-=@T_c=D`%u@0$vPH}^8To9BRLxDLohYs2L2qp?bS7LT9C z!?uHWNxFFmZg%aYFDy)`q|Ptq$Gj~>?wB??P_B$)rGl_*rWj7#B9B9k#U#03AoSUz zj1MZbLHnfuL+6K}^9Bi=^&o?em|TdRRq1HEI{_@3H{o}yK2m>28gdHP({szy!9HvX zD75OpvJdxYg{cP&Dch6!+*r7E`Y0KS_k=#aF05Lh18b+87CKKbgosC0(7um>W&d2E z?9yGk@41seL!<-c&nJRcw*y>gJx3X{8^S4Pj*?fhdf@D-1>3SL;E&EM+QT#R2lSe0 z%JN=vec(Dh)LKeh`LocGK2F|NT9DK!P4vvmcr+Bu#v*NH`gQL&x~=XEu@Hr~yZcTg{P03*DE@1nPt)#bK!=YX=$;+{9h>GdeR~H0+d8PFddiP3Z2 zAlXGku8cZDJJOYL*tM1#iDkn>rL`cnxtKoRvJnoibHe(jsd(#0C)xJV7aJITj5gmV zJRT^E5$cN6(<&cUhPcDeIbP8AAsbw7kHVnTd~^)e1joKnDE02EFgfxreY`RQ-MeO? zTFo-({&bp|vmu|1s}{hUMP|ao^A=#zJCYpCSb%M9N~r9fNmQD{ak^9~O6BLF z=Kif@?}19}j|l?Tp>x!1VHBLwD5QU)7Z9Q?M<=dou~pkrOINGEC&%VhF#(#YjKNfG z6brw|>}!a|=D%4~MLi0i%)UlTMtPyz3P%i@9EOAYZ_tSPV?wXiX`o>^0T=yogA0Sk z&~`Ty|C^r;anIGzgU25J8+3w7&n_B$V+&(G>ozk~F-a(9`dip-w}#5^Y!;rZ{6=@3 zd`MUP%7Ap2e0VjmnxxA5Kz+z=(znaM#tQT`Cu4B^ zUHavFG)-R>O-2=lL4-^ayjiig{b6Jo#BO&6ANg?53q4Av*Gz<~>Z`$~v>0Z2E)m}A z+W@(C{5!LI7WM>eBhl7NFoN&*BXw<2^m7ZH8TFS|2Z>R+7LFcCvqhWyXAInq$Gty8 z(SYX|3qrEde#s%yViEw^eS2xX55Vcwdx@?=IOSbrLc2Mha8*i%^mOf@+OqCYyVe-r z_E^(PjU`E{&S7G!F{33+*sQ#KPll!1ugi{+|L|$A4`urJtHUfI*|jlX2>mYfN!>L zm@(oZ^_?J#W>G)spT9iszhY~92(ylk^K*bFu`|ir_eS_H&5S&feq`6a>#^`^Jx}yq z(L<}+Ph?T65AhzipG>emK%3p3Q=eNu=#=hh=oMg#8mR#1s`T)2xIDZwlZK@ezA^9r zRubz%Q?TQ4)#>~GkwsZM$fvSFdUo4*Dmg7$xcDKVf}C2qunKVLlvDZ6PDoMv%6?-~61PMAJtM3x%=MaOLiO zv{Fi$W*vS>zihZIblx%#{`-E6%I`T%O5@yMy1ycwB^?42%9JsBd@+3!7oxkcunHgh44eC`f4 zMhuM6bfpVY<3WGiYnmldOqAK9^z5JQ!n@N9AvJaiefcXBw;Fg6jVT)RYqBg#aXh5imIW4|Um}41aY~A!pbax0@{mrAluQ z`mBadod>+`rUDZ47eiY6LU7J|+TQGv355lVQPXu1PUNIey)6r4Eq=fwcsIp6Xcz`Kq4v^u$7ip<_e*297>gn;*om6U0 zBDQ4yVRpP8VV6)9gsGkuU}&|HNsUnl0kH+idNXk7w#GHy(%3i@N6zTDp;N|QGUA>g zM41PmLvkGa+<%tZ+E>wlLk84Vy@(oqUqwn^nNiO)UTe9@9-V*HQT49`XAV`+LH}?( za>X8BIK^OfN))UecErSo7m2z|97t?$So7x+^_;R zow6h!GLHyXy1%6NlYE7#<{Ox_<+JFAb<1f;-$8O`M<~wZv1e160;c@dYz&XsLnF?z zc>E@|mz^Ai?=mLg7e7OkS!ROnm)-H&7E`>=Ovj$HJLuanzNEDwq`mg%A;wfmAAY~} z#3KhsU|)bf24B}kySuvW*@o*#+L&7;b?6A$URpsdZEiB>+pZJeq)Jk=P>y((WS4pXp1z%K7iVlHbc{)JGjh>7tv~Pk!c4 z#RN78-BLX9k?I0+1%2?R%yA}*=lNsw&C&R-2QJ^7j;Ub@D7o4Sg4E_9F8)Oi4KK#Z z_y*GD>xbhF+K5g2c=*)eM1Jx2OkX@2a)aYxXv}x!8k-D<-Ns@2sd_rFNE)UDU!(67 z$AdJP57P~8vF7T0C>AMUw`DetS-y;VopHj7mlE*WTY@MlexnS9VX2J5rCXeA3@_kcPcC zg2=9FzNU3k$*bE*kJfD_?qW4z&WTv&3u{VSj|c zoLxP_8(-E?18+65-Z+hXR69Vv4iAupf9{guhGw!TqmA4THHALSTC%xU8T;b+SuwDg ze1Dx!7AlR$SQ5mT=M+HvLo!Ny+DI@cJSC8xmZr^aOKT?-t0 z*cub`Hjq{yeZ1%B2Gy@wSQ|V*&uvko^EQ`Yvf=_fv|N+N_VV!p8-qWa^YO&OC*m@)nw+G&$QoX0=|0q zm#h?@j3dHsQTFdFP+Py8wzV&V9sV)Uby^wwhl*i=7ZQK%`{bGQ9pS|jm1O2k5h1Q7 zAa$#QI)~gK#XF{>t!Amve_9o>viw8^E8~Vlic@N9WPZY zqm}k%8uM75`Ey|dS#6>LpO1tH&uhP<(`)#-byE(+Yi$zhxGe)O?J8Q=;y^r4nxOBe zmvmFXNAjOwCsV$S=Wet&lVh1N^swbD>c3z#7ABMkZ7}1^u(+ETgF@A`X)eGXH$P*p0Cf5S{ujR@zV98HVmbyb_?@QK_z zGAH9W=}Vshb!$g~)MGI_O^F+{{L&{He!7;fix`jB}jE{tdDSj$6$@WZlQAghWKA&pOTjS4cV1u=MGDayu=6%4$m@oZCt^oLo*`T{gz~ z`)^a1ntrmONQyi&b0=|=7LhMbv(ar4gUY|!gx`Mep~jj9(D$l^j84!4-_hDIG}Q)& zK5F5M9%-gxn8kM86tb+Km6n(bP+ZQ;&OX-|53hH`wS%w7JEdr}y~}esdXuqzT@9I# zJ_`$GjRLp(148j9~Dy$N7u1~HXz!P8gN!D5ynt~p_jA5+CJA}_f8nW+Kj>WabEvo25_ z>H-c5*NB6iE9`vUK>J0jp-#sM^sXoIn(qa8>%0#f`#eNH$C+T9yc7nRxnQeA8!ei# ziyU(B18MFe^|-*oUx5t-J$yrp-mHL&<4=&GwwJ_coDFS?cLJfQ68Yh644XoRY3R{) zkQbFGtdfqPIh)^-Tc#3NvQv*b8+y=Z`;b~|my#2)y#FY15+*%uC1zWd(CT6{Ilg%| zZXRuji5*+p!iV)?>c=EIv$|(=MozWR?`$eY9?V1bm?vbN4n$drM3Bu21kEs@aUc)6 z+OweP<}|2R`9Z%2M)5skP`FIPov!GP1WAwAOwM-;NGP|0c^lkG!)9riDfXV3yV{Yi zb+ZHWn_(!mvxm;;HGmcG6u|TZgB4Fz&?=*umMxXVZO_M`%Bm4KuFj2AdP+k)r2wrhjd?jZ8lw z7`os@FLM``!Nl7SslSN?-+R}JGm)x^kb;<%y?VgHJ z7X;90=ZTYQS7OJVVC<{)gM5CT+#MtirM$k|>-q#{aAph`eBsFNF9y_SV+#pMO<>N9 z?<6IO49woIg6e^zp=iA>?pgVOrp3Q#4>IPbr}KEQ zD?0F+bSJSH@I5*eLQPGHSDgUb3=fl~x9ustA45WHo>9%y9%!?8Hjat52k~xyY6mBypK&}Al@I`7j{sSU3|tCh^-yiaxJ{8H-;K z#U6F2{IrTbkahv5ze3@O)$!n+!Q;q#D`B>@7l>ucz^6U@-7U6(5jF0pepID>*%-b? zTO6V;ty58WEF8^EWycxH1cO z8R=vG?_9c(0uHX7P9^-UNb$#4RLZED9=sljD^rs2>>*=F-?SL_Zhb(-CzxNH~mO^j&qT3_ohEpn!A3Vz#_L5blN`fN~#OurS5tdTNS4K3m~kld!f{9Ry& zS_~lXMM(M6L2mM1w%IOGxNJghd#^?+{)}p%kl}@Krw2%qxi^-J$zfZ064u?2#sAgu z!HXi=^vevt9Or9JG(!hBZX?@gA16bOBjCcJN|JF&l`NPYjr;o6)5uLhsHk+0w7ltV zkD7FxG{xt^rouDAhX?Giu)B{L6)V8HS3!*D&mpqc%nC>3^7vGiCw_CCqaxLQev{ZJOHd{ai+twv9qG=p|am6mit*lZ=&fx1EZ%IM_G+BALM{(Eljv6W78&?<_EI62s(#dtjFHQ}XneAN5``i@wahLV7;0z`m1i za4GmP?ceWB6~!0fmvRjp^s>aiuL?2oWfIW{aK#Eq4W`Q}4n0hf4){;OsJ;l?S>%f+ z&s)KhxpsJCivf{TxI{M1b;qw7QgB4KhPvsG!V6Zai$+Ywbn6Bl*S zl8a7^#Z4q~e$6zjKPKN}3UB~@Yk5x~!?QP_Q?Nhs&yf$=LEiMHfBn$2_D$L9`F z$(QTM=lz-|J>;JdHf%LDpPvVI=fWUh&2`(+zoQ|%$_;wE-;-7?4|L9$1Af{k=$C7! z=?{Y}=1rJ1{$^BoT}d`o7_!2Xy#LhxuotY_;s7;q5s)(A2aP+*=(!&O_{Jg;MAPDM z;h_<5KY9~+|IQg3U4r5Buq|wpGz6a-1KgXBL^aqP)@Ch$oD6?-nw|=gU)1nU(z5mg z^5*;kt$xOT!wNcXo<1H--XQcM<1lJ~C27~zQ-AX)%(#&bt2GqRW_%IdJX;ILO)grEoBHs9CyHnIg0qqO%XnYxsVkm zx9Ptlc_gYu2D)qApy}XV;T7H=n75-8c0nO%bzY@@GL;aXdx3mdy_58vdq$M1`-zK1 z0UePpj^OayPI>tV9QaQRbUP+PMXEXe{LE`Rn5A$>BLHMe0EfNj!x@uq^11ysQ4fs9 z{1OY{mw>ZEKb>QwYr<4=f2=jEt6D>rZCVTWttQ})tP-+BaS5E#P6Kf`b4O@sF8JNxP4`drF8-YNMa1{*sP;9I>l z`0aW~F7f#7O|dbswX~NQTS~Rb%nrinV?Nkz5D8hv!I=HxDYfkU$BaI+7;a9VgD`0) zsjnLi*B$HWmCto2bX!M){;YiRadjU(Uox71w-ynjW*yvC=8F$$7+&660H3n?cVlda zuy|rN>@xJh{%NY%xPB$QfbKN8?E-yvekA{VNU655YqfBN>HHjgE%OG6;&|Tk z(P8F$#2M;yKZzu()Y7LnTu7~*D#+j4Kwj zED7UyK$6!oZy~dT2I;1kmhj+-IZB=@6e@UV2xV4oB$qd|GG>OShStZA0`Mr^>(U5~1elJ$hon0orR-NGsjdpn8T8KHe6I z>#r?DYqx6hes?zR|9zPGxb7I^H$77rW%#MRM&6OMwRMmRUnLCTJ(~gHF0ee6fiEU^ z7^>k6MKy-7=EZXO`A!9delyVAPP39#pyF5WrphknbC z5DSZ7>{ob7jp!}JT zjVXgtWn&Ot&xJqtqCo0jG8D;gqp=HB5gu%!>6H&@%gb`IInB0V)EXM#N87u(Q1w+ZA_ny3)FZGvFUcQ-(fsLsVAhGOhNBW zO1P_u*J>4pV@$FVF4alE(WiE_*Xs$;=a4C^KlGYWaaF>GHLld7h37C6J!qe~9Ofq6 z6YicbfR%YEFkM#*zh2FQ7Z&G);`i3mf~`7mD^LaArOZGtGlopiD}e6nlAs^jMh+yF z(SsgZ7+C5=-e2@2UzY39mpZ<9zm@kF4xc0S|K`KNv~-$deV+XH#+^nD{MX)EG6VCY zo{{ITd@;pl3>vIyArJae@wE6xGBLp()~^gk9os6>drXSD)SnQ(_S!&v&1LCD2Ljh* z8_B4*(lAc8kcj=~Kwr-DNBL3Y4JnE;xl|9Fa+ltHarXx;lzo=~p(@5S$iZEBSSOpDkN->R zEPm5=B~7S&;t9*Si6}He9RrEf~PJj zq1dna(D1d1-VEs=#plMt7poj{{D}pqwMN543w_A+`9e6}yV!HRo}|tRg>99y(NZA^ zg8utWH}<}uA1;c+qSrGpe4;6wTJehP6#S&_31gs2(FS^V-6M{dWgz$BOQLv(*R<-H zf}lnL79TQ%-)To_AeIQt?-9_@{M+6vKNj>de~=pk<8bv@U0C=K>C~!ugnPOG3P<-) zKW$5TGJPQycaNjn8?x}iC_9FGS<9r%S0b0n)-e*B?a2ulS8Ut+gTFI}7_pU+ShFq# zf95pObLfkuGG(;;SQH$HNu*PbIYRk9E%0Bwk@#T+dDd);GfPsT@V+}7c|IM_t@nXg z{<8(Y#L;JiBA&mrgnCybW4;kVG5gnKl}{F?94n!WCW~`tONsy>>F zlXMtZYgJB{zAC0p!OGy(7{w?wEhphmG(i942%!73LB&M{q)yBRO}hvvv0(ZI6uw=&QEa1>MuX-?gyNt#j_;v&%Jp1yWI-DERg|23qJM1D2X0yTh$hM zErr|+@W6iujUlo;j85HBOf7jX`*#e#w`1}oVPW%B@CjQhe536E-fyL-{600j@8p6b z@_MM=>(P{)5z?ve^NG=#uVg}Q785_-1pbqk#VpHCvMTU5J?#65N=@5CN7anOS@8la z9esr)I~yVoi*LN86fZ!_n~ z?blvtd+q@3+7gPxhh#9uFc*(Tj>cg-M_N68t z68|r8@b!f;Ogl3LYSnUZI5`Nc);}S79j@^Ga6Eo6$pnY78ZhY%uPq(5jj_7GYhUjj zBrc36d2jKWevclc4TE6*i!)&J{ZA)L}eIPK98AL#`o~2ja23B zW_sGNfy#a3=(g_jkRjf1VWbT9gxP_ywlUmKQHBDW28t~@@K$paDA~Hg^B!F) zH*OrXX*4sVXB*(Bt$vu~NbsYpB^3SYBZC>2sbYdZ=wIaQF3ej)9_2ZK{~MA zDe>v8x3%e|>L{>3yotKC%j5n|H=&$^CsFeiCkK;OlfQ}leu>yu!eaZD`1W>-C2vD6QK z@SZ?}sRx8%eIsyrwH0P0t)m;y5!8u~MJva@OttrNtj>vmf!IdM81mZC^!xOtoizBm zNx`?0No3vE4Cpw+d!jc+L;Ibz?Gr7t;aZ0h7LVIRl6H)R)z2s3{SBUYvUeg~I?@Vs zuV_-I5B#qBM|!ZYXEBt!n8VkL*08E1inM5-B4K;$N$JZ37-O0M3*Z?YDH#TpQ!Jrn z6-RPb*V(<+?HA?;%;Y_DyJ=^X6%~oDFzvp1RJ7U|UAGrAKHd(v>cP0yGmV(c z9qSDOPyxn;gz+WvIf= zk+_L^pox<#xs%n)ztdmSkE+IGVYCx@bSj3>8fTpGp_fs(Wr%OL6f^GLw`p^<6$)f# zqH&`bNo!n&=jtY4IA)b8GY zGWgBHgVC&2Mn#_gxxHN<)TYUU^B-AIRR1oVch78r=UBM4kCwpX)sb zR(3<%q>xXfKy^2PDrsQ;DdB%q0IFY1L2FAm9MO-*9jUzjATkE6t88#iz8mPbbgnh!@oY!cjGba%@jU& zutBMFYzN_HPFTvBU6NtT%xu}PE$7)yqC)n>@x6kB4fnZs?{0F%8L}dqm9^}gxDA}T z&qqGxO{>nBHNF&arIRV7utC(?a&}fkWKY0$*s;HTyJTiJ&cd0_(-+Uo}q-WNX8Q1ocO4xz`8PI;DlGZ2o0$cB1nDx3+8@*RM9n zl_=ijLIWmoYbIRk=(ng4ENZC{+<3Byjoz6jm|u2<6E9`BfwXO`Wqz(;xU!gY581_? z`S?*}6KgF9UG-9Ms=A_MNoNsPGOd;!dOC^o_`Hq_(2*Auo^a(frpAdX_}uh|4&?#~ z%L#(-kB$ix(^j%Kc2shU*2J@3onZLIyG81`mbEW1R?>)mv=>s-F$AlU}*dfZo`iV z*45vrb1ezHO77>r#ENq-KRghP6E|UlUtAPi{w+(f&BWyu0f~=l<>Fx@@+xzhlKjrP5p3#1nr79qYH) zkLe#~yWab7fA7B&^jL4<+|*+Pc|rR)!ErlI;&~;`SaU$2HOhq3dX_1Q>sZKsetcUb zRqVyZYE0#X%F3N%9_n#BGW!Js@@u(B{SjQ;tG69JCF43v-|$SO%fgUcJBPV?i6=r$eVMt&}0vE z?-MY8%~+9+Bs+L|2FE4NV+U3$JNV2l6kL|?V^j9N<3hKu;nvJmv40(uhJ#5bI>P2` z>DXt`#g5*)nbTc<1U|ZrV5cvT6*+_zhz{#0vh>AykyDjZ=jHC%?D^~_(csKif{JHe z+}A-nc1L=VeZP5`NNdb4LB5Jghts`b3!_TLs-!A4RDpl5A6RE$gB5P+;CR8>L!WL{nUkiFyo*+17U|4#pND zQTL%<`zvQpbN}7j!oIZ}?$|wM4R`yTCHMJY5Bp={2X;#Vu|KE$hugeAfeVb2=?o7) z!``zz$Hraj=L%Ph5@^klWSc#kILR-c*qo^ig3*n=+}G-Q(IP$zy6ST&yJ#8}Wv}w$ zdKR{drhTquXUNuya{Eiz)-@sQx#&%HCix6vPjU>E7#e+{VXfz6Ufbbv`ch0K%6zsy(w6`qnKOH_nV$+ z{{#UtcRE7PcMFF0I&t0_v)CUO_WClySETP6_fauH-6DiMVJ-PkWBvExyD|ioGp* zBZ}PrwzPub$8U=T^u-#kpeu(P^XM|CbvA+h^H703ZLY?)oA2TT*B5gE zcNTM#-;Ea;sQlpmY@5dVRP}JZvb#hpyx)jIk5f_okTEwt*|^iFtcG5+3}h`@=W;vW zhqC`_I_%$Mk8zkkRK^OHq;i^G<|3(oeCDf8HG6QjL`UG%M($taeAa2$mi0)i;7~n8 z&>B(2ZqvzO4VM(K{oPx+UkWa4?V{`4;pJy}U0MV8HQIvxJ6O(^##MGaVcpp8zE$i# z^ALd|S!17aZxQ=)$1ASJqp!p1!g01@@Rof>_F7TqCI!wPc8QJ|$+4#n*mEi)lv#2n zg6&s5%*J<>v*||}Hj*?5K0e#RB2ysx)UwIm$IqH=Q5Nq!_Fj^$nxewpp0?fol(?Ql z5}%*AXYndg(D%*kg5d$to~#zmWn+GaSmY)4?V(v*s^enz$7@%?W2oS?j=13Uhj_tu z2V;lxOV)@~#1gsQ=oR+D31_)?_jOiS?9HilqUBHTa6e{9IJBLq*s2QJBBE>}5bExW+_m*B7aAUE|`l_(6JvuOj1 zL|J$K+K(Nr%f?z%ay`*&1Z`J41pRFv+3e*y><{;M_K#jza{GmsMcd2AusfU$IqfBz z1G5S+EdmwL6uzTqS_G;5!&hu0{cmAP2C+2J_@>j6sWPPRB^YDY!cYY_j8llQP zBAmUhNJuFq zR7!?Kh6YhWKQhZK^AsW?k;-)Tx=Mr;l_8o_Dh-OH((vB*bN_&Q&)sM5y@qE!--kg^ z@w=0LPl~7IQ+06p^<-kxa0D}(FN4O8K5Slng6=b(1ylYigO^Y-Sr(H0eO51Q zNGK&&8{+BRKW#MNL;&5k?*{EL6@({yBH@*}Cro=k!YXNg1uIKCYWq|aZ$-F4_s1?c ztz62WYZobNUXO`x+d#f(6tf#b(Aw_@_KgM*gVn2bXNM%gjKb^WgV$H8>;DUj_r{>? zwD0V3T8N%A{jnpu8MAnt_=YFnaE#~Yo~L>rJ3os;-nXe7lQMmXU+a(d|E1xEg{5TX z=QYGEK|9;U|Uw!dbXBEj? zrh>ORsu_ou7MQMb9Gm8(A*Gf0YwQVGtDucj*>#-sRa|(HVNb5lHw3@UO5C*@>&RuR zYph;u2mLm>fi>871)h(0F|$7`!o|BSz>kj~wEN@mutz$0F0TU(M{`*8aRqfqvVn7c z+Bo>nlvpnfp)cNY(4EI5Fwd1kH*$t-^2o-Xz$IKG!wh))LXf-L*8sf|C-rpA7vkNg z>H5||ziHl0Nv?O-aWHFf#s0Hh^o@!<ot=`h7}7Q$M8Eh zK26AP2T8@F;M%4R{U4W5<qzVYQP?~+ zg0VK3&q*RhaKHHt;SjY#9MzaB`1F?u@e~TirrgVP-eN;?k(-UuLT|{Hy=Sm0YYKjn z?x9k}d1TXmA&$g9Z!+WOD!kYu$yP58hOb&l*nK7!%Z|)JzB@NS(N-E9-mFKzRnb@p zXQ=&GUFiHK!STALjZYf$$d=UyiK?_Wv@|)RN+@qc<3}a*y;Fia_2&4`TZ}8NaGCj* zK14dVUjfeUXHd((5`CJr!C-GS?mo5#0MRz_WbH|$=b3ma(z*?g)~6COWqGwHzXg* z`3J#h^eItuNXMXi*^t#62!HmEP-UxHqE(%RCvPd>!d7)Weo`I+-e^<3!&^|@?F!uw z^TG26kHdFkA^EM*1{bzY$DxjGAYK%K<2!^gZ@wq{VVuXGT09Rv^N;DqcntCa)Lw{_ z{tu4L&&49e_vq66o$%gBY?)bwCEbxE_k1j_$x4F<*}>@eppiYGu@Hj`=W{Df4PmtQ z0^TvoMzg-#khoVD=KoShr{h0i*4v+CnZ7&7Y3I?%>Ph;leH|=%cn}-IduTkrBFL`I z$BVED-z26o$wHExRriiH>^WIeZ*%r4N@SeGWo;Mm_>L%)Ti^hm`vtkbj|8y?LaHII zbT65xt!CbAJj?3+km9zfPX`+w6WRUW2de%godz6vMKkZ+f$kSA_>{+}ImfvLS0)a_ z+n2dS#`F>Hdn<@LTa}RC3kiQYA8y^|fFn@@ZV~VjJ$$KZf&yA~Ez*Jj|&wp$;lC_|s@Rj25Zm z{NKFweSTJ7HMxQ)eGSF)9l>~r=gFD2wGd)PuRzWl3F2DE&)I+C5>vJ?6+^7Eal}6q zr5Eg>34i#pJ!K}In>$L+^up$TH;zz8j}m zp)FFJhJgiaUS$`mJe$RRS0;hm;cqaG2$y_r2va7+tXa=NCEJfM5rPwv>gG%cbQJad}ut;_qnRw()f93qZ zoBwUXIrmagrWRnkP9xFbokud^BHT+iMKSYGGZAsVg%yK9o>+O4#C!?;10P(;l@=2o zmQDgwyk9WO<+!*Ya~}Me&Z`?Lj8l#1#kf@32S0SYgUZdzusFvGx;Ndx7O8I}_F+AF zG(Ej;*J({Otj)sj8{A?0>fNyA+z8EWo`I`|T;O5tIJp+>kD4MYA>nK}C|ppWO>w1U z&Ix@e9{PtH{yB5U7d61*Lpk`)tPD3v-+{MM6LbQ2;X>_n4ytsJziNsov!WgDKbcK_ z<%W>h+em)c9D69kNTQhw~+D z-CjqwtZ@bByc$rMkqgP{JSMcjV@P+K2L_)Q(0+IkcgD{Lr{Wa)u}q46xc)H>qCEeK z%eg>mHo?_Pi_q`96y!y0rj8rmvU9hwbj|!5pfbgV_Ds1%6z|x={!I(e{qRdVBgq-p z{xiielLRu#<5ybL-Gx7j9BSDbkLH*8xVCXzOlk5ZWy7ZA@?!;9kUxq_s}8~!(J_KG z+wq(CCAi8J!qO?bXf#tvj?58*b{;Dz=Fn#F>2st(#h38Uu0Cq%{f}z-FXV(LZvcgc zV-OTD8&==F0JoPUk`Mn{cx&^SteBArl(UK2&3;7vEjnSh>~-=`rUHJwSxD3$vk-Ui zE>`afMG~51Z}!`^S{ z;NA^jIaQUr;b$UjI^M)%K|g`>7ea7HsntdisT zpkHkux6XfOhqigsFRdbQoOz6=%yLm&`6N+@mY`1+@4(0Br!njPEbfZNI9%kn4=cxC zz)i&p6u3=U55G>#=y=Ghrwx#QhvLbbwrD7^cn2uy$aCR9O4@ASOl{|*Co-B^mZNO3OIpDYH05O+)LU*T@K*QNY{Bv{~KAg*d z-k*cG_EHgu^xU92lci*q$TsK`+l%tI8%HqaQ)!y%6>Y`<#>LjF_nY41EZ zcg_lHvQ?2QAq(4net^@tO+-(9KhS;GFj+?t!z9+=9v+v_&}189XU0z|VvMnRG!)+I z7(-F}C2(q}pc4^58-qE}_J|8hZwNx-1B6pI;;{JDS<)9Un^T_3qJ?hu7B6sUpLUB0}UNh+9*H0oarOebFAwUAbE4INhA)_+J$F=kjAzpN1jj`UQn z^!{R89(A0iK3mRMo!)|@4#~u5TLvhE{)aqQRVuXq4~n(FfcUl?G`ik^Q{Q$oo|}?z z&unqL{KNnU*6^6Nzn_uXB6mErEdjE-&ym+64eYn^3}TR%M$!{B$nE-mFw(LD`z#9- zDzQiDS>ePX>^k25l0;pihDkPa5G)pqGK)%L(aL(5K78?%ns15)+0i0KuWK$o3C!bh z(7lLt%mVoDgew?be?WiFRDplzHiFg`H@H~f2Afk45)3s!C9k*i`A>>lZ#-nj=5`~7 zSOSyvoTiB)SQWWL`Ry$bK{LqBXVZz=M>$X^_W^eD8@x`{fh9b)9USapoN}ztC3hq3 z8`uRBR+-@BGK{CkFMwr$6i6-PgM}CMq4AhM*7aT>cE`Swds9T%SMRFnBfnO1|DpuP zH+_;^o${1O-L|JN;6a|mXR`Mp3a9(TV1jo7Ns5?YUQS5BrJ^)ArsztIYtk0D>ue$bqPe4N&6hCvTDlf62+aCdtajtx7& zUkM@T-IavTc?`?OIpG*tbOeXx%0S7)imC)W)Ysa?yNfmk(b||iP@Z=Vm$g)Ym7xVK z`m-3aR8|A&n$K~+c8>7-Y=tT6FHj&NlUzFg3iD1rz#g9$m~_Jsx2o_FAB!7se!+Q^ zymgT-{Aq?036HD#y(VuGaYE2y%v(RefHS}?B>226i$o{MkMz?*NfPd3^yfZ5jOqe&YGF*)w&5b3R zvj^zRRjFvNSOjsrdUe~h)9`8aYIKxuM7zWLz^|hMc#RXB`qhyB&f3bE7I*>v808{g zg$hybv_^cq146d*bH5xNBfdaW5 zjtalm}3UFgP#BMjCmR+i)EJ4;JvMt zJh{|K1a(62ZuM%|vUvkpm7K!s(HFt>kuWMo6k+V;Z>m3YCiQ9Aia(X}$=4J$IH#Tp z9}XPHin(Sm>)sM*{5BVjI;PWeTWg4qWD<$~`Vv2R+Tz{cMR50^Foff6-bb3Q)nAL1 z{Gw#xt141hRDnJGGPtN+6=$B0#22Sd=@0Z(;G57pV4Ca)u1bjf`o~G#wHlJHodM!J z6S(~&QYfJkgAM2Z5SMjR;cC=Y>UcyL6*gS}m#7SSBHI;$F8ri>j^BmjtF8l890Q~5 zI6Uyn26t@{;|7})W39$+x^j&d-Lv5Y)Ew%;z7r0p)}O)-^bS(z(y;oin_Q{Z%xBOh z9f6ijJ@{6b*T1~Xhq%fW&L)I0<)fn9EIJ47eVB%I;>x6U(_*+b5{32~>f!w*Su!xJ z2Fen`+}qLJkorXo?ldT&o0>g%yCSXsok`c`-h=?HAgo)TK`Z@|@UqJoop>%tZtPG- znND|@<1zzwM#Mtcp$gbLs9ZHE@02lkBq9)89M8_KB`!MrOLVIA!5*iOh|6R!T2G1 z{V_`y%{@T79?Ij3fF;r==ErK605&dUpx~zvW@NbRHqY1!(?b1hL^1x%mU|sN*ZL zdB^~`)6KZ&n)T7RcRAL(-b3?z36Pv?1SRIWsLAQ1!5Y74qw#Zg#C$#cTd@SoGXIeE zPBJ*sV1*J>?to`pAolwY4aUo$e0mtTtFqwekR(T{a3L(8F&BNFSiwv^QM}yw zhFmS!O669*Wr_{=1KT)-tX{5!oeu(t@(*1wJ!l2ZX-jy$=bgAf)03nH2*MoAU(ofZ z3ayp%(2h?J(?yiv;&D6tkyA^)@b99W@k8`Mk`Bjqm|`?BC)a*#K=B?UR4E%HUw)f# z13!eYi377h@VOB8*3ltS8|R^8bm z{iyXT6Lx&ALB)HMw4k&Y^-oIE=$Q*Jf68~b)+@-BIP_3o#O?&pRlK-K`2!hi;_cUy z<#eKd2Ye|r!c6ycBK&DPJbMxhI(oFZ}}kh&I#|UT}SgH7a=+^g&D2e17d+QaI0GozA)5)7}FKF?9m$TKkjtA zKWjhr-xG}L8ym&AO4_hO7Ke?C}|CA8b`)|l`$!wycD24Kw$r!#ji+2Yof>*@~S~Q77JK2qy z`bZg8@9&`tpQ}*AA}Ow8Zv=?C>u??(5v8*>tj6v4Bsm&${C_6wyryW)#r34aF}7^#8}!e#Jk;4C!%?Po$R4Y9XyAAIrp!2Ueq4yRwo!|U$7 zr1r}{R&jpw%Oh4BMoICG4t9ZT z2)$9Oj3IRrba8AMXsAlDcXYnspUc(w``RzIO6@z=ZvF!IPvznF$FI@pT^+s*(So$| zdAO(QKlo(xfMidWL2PInRBOkBi#Y=`d#uTuC3*Nd5(a04XAI- zp#u3MIJZlI6RwPApari9Tuw{yUGunU4-Od;W#wrH&z4w_=|^p@Wc zJ!^OkwvLH_j^-r!U@}fp|3s1(KN}fu;R~=%Tacq28wMhe<~FFBG=lrGYFd&1AM2GM zjz05z!S>ZlvS^tUm5RDSg_p5->Od;-Sa}n^D2Wi^GLTUUKV+z6xJYh@jkt$D^2qEEys<_Fpc@r%1rzjA(!;_f$hr|biidAo@>&; zxykkLOU@W(_U)ms#!pevU%x3|g%{~w=f)gsb;aY#-f-IUHskx`G6{1!4qJAdz?Nl; zxqPdx5*4>KkQLVpB6KR1)7*_$AD3dz#tAz3sh26Rmr}NVw4g z^vaiVC=bp=-?X>5&bAi81~dOGSr6On<*Gl8f>>R_$HM^=zO1n(96 zqzwW?u%&`awZsRAt*8x5A50W$e(h!^(b(YO+v zl&_?I-&*Ni>#bO(k;G$}$0D_lg^D+PoJAs2xN&jo@SRdV)WHwHud z#AfiH*#MlCcfirr20KfIKym3G2-h!zzt0m$#PXH+;!z7GE=i(&n>28?wl~?Q5Cy93 zg)nsU9lLPuH6k-|4oeJ%XyuF+d}cERldeXS;grvC!EXVE{Y_(;*7fvF<6}@BTuCLG zjA+3WZH!)fmNtesP`#z|XwNwS{!9@9ou~5fsNpnBouk4j%G&@BZO0hBQft^T za-DQVSwQu{Gx*^9ZFv4JkZBpz1)Z&z$hx|lnAvjzZpSOZpk@WiWVm8bS_I@v#FKet zmteGE0{x9DdRE^)cqYBR zJR2oHTqJW$H&BH`%kW%88;F|dyPRfm6jFD;} zutz+wF|7$6J$VQTb)N82`2($P`~(`>*XS*2JGw7*oSgGlr$Tk&;B+&IuD$&U&X0Z6 z_gGMl{x_#_Ebm5<;;>tg{M7;@EB4YC#r)jFgWGXKoITCzas#oGi!hG2_VT^F;8Nft zb8*`f*1o@hwq?(!Vh8{YN4>=jQuQ?n&)a2+j^a2Jhh|`24E9l9ozsLc1PuRLf2v@cSk)wB9A^7S- zJVE#3bvtd8IFw7)gx12QN^ElR1;oD9wB#l}eMu(cc)qW4Kd+~?;S?@4S_V&pZ9s8qFiu(va?(}WseqCru5}-wEC1H8!D1V+?ye+W=zGl5 zXblm2pK_dgOo~$(Qb$~*)`ENc5A@_|T(0F+!u;bcOiPX>n3&Dw_`2Cbd{-xVI&mKU zm8a09UsI_3Y=*e{E#ig?2yi3gPtfUQl{A0FYHZtD#vI7u#Y=L^@UA-)F9g`ZY`%7U z&)*5}*F=C=SQoDBZ)ROyh{6`0wsp7B2guS(pfQK`5ufpmoYwqQn2^>(x5sWYPo|Bn z zS^5|ih&RAtaV>PysU!FA#gI_(3iJzoKDYpRlM-BZw~ssMgZ=claOL&`sA3uAJ|5T5^A2)CBb zu?Ult+CXPpXb`6eDYzQ_0uvg4lgzexOhu+JM?SuYPHlZgAF8bftG4rK zVJ3sq>`udHp$s^;A_^Q{B-8w>)43+G3qbXn2DXIngW0LlVE*JHbN5{i)l9Y~ZC~=J zLq!>i?p@AQ=WHQAo}Nbg?M2MVmB00_Jpb8XwPu2zz10Ndcn974!i^0MOX3;W=g^d^ z@7St;8Dzz^>*PmvG7-w2Af7q9z^1&8hh3I>XA5ec0s=pa3v zhW1+Hi(6gr;py-6QCqp1b!#yz^8`k`0s%!E9~6BdL@bn7pCA-;EIw@p#k_fo8Eklp*KgmoB4GRCA$F26-)fdE4+^*w=$!taqAxo$8T(MJ4B_vmYe zj}lQM4t^=-X_^KZyeH|x{;{l~QJ>q1_>yPvNuq&<*6pUMj^Y>`<4vm*uaSVVG%DrU zj~4ljz;ubwUZTlj-z45trZ^I_4RR;#L_ny+|=*UBrTR42`(YHGYm@2pCZjKqQY7dtf1&A zI2~0;jAS2R@Z(X`eGrLPWa7vb88e(AVhIQ4w!rplvcQ#=#FK5UL{()LC!uu-?`|lg z&qV#O`IZ)nHkHHBi_avrg&*_PH$eFkJE-ssr%RqGb3SdB$34bVIhAH_v3&hH^7MHL zt0WQ*Bv_1<)LaImW>=X{d#X`$Xgc2En-z+%` zwS0X{<7BOVe)Z>s~nxx^SJ9bIu}$}kKD2{Wgz6g3>=eUDk{BJf#U z#YC#fgQg#^54~}-~msptfcso_acQ+T?_8Qk!afcW zxH$tl&s>ATIl>(8M1NqtOUOi*3O@F$0$1BmwD!7(hYW8~V?`NI(JR6S>CQNBZyjzv z9}H<0Iq+j|67H06px>^Upk`t!N-qp#;^YqEg@QE3-0?I0tT+r+`X;bwl1nztw}Zuj z2IyE6g+Y$iXdSM=O^H{8$#sWtOw$qM5`8e-Mh&zKs&JEX(!uv3S3-YLR)y(N>Zh7xv7?|c+#EP(b+*04uij94C9OU<~_QXR0sMXkPo{=)oIQ07^XN)jvlA!TzQw6blrl zb49}-)-W4fiu&>CVF`$8=IMU^i-)^+V(F7Dg&5j#hh8WY!!ttla6us+ihWw3Vs|2u zl(-KY#H!Flx{lh2M^b+CY??hgnv}Ia;OV)na3Erk8p)|(llUv*6R(DcLZ(7kfhgfZXZ~GM{p(-0gSzs%I=Xzcs^XO4%s0(V>{SO1e>E5sir# z8?fh-EV{XKN$aA8@BuW?V&-v7&98xPn=ZqLpfqycCX@E`EW+8jv1}A?KA~#!Zpv&` zpy>&g5MrcAR(R#$I+f3e#nSo#!z>*+O3C~}F)%(i13g12^V};PaKTS~p#e3Leqk=C zy9&bJTN-fatp&i+Dts@{L!U43Mri{(y8Vke>NfLX>c=B^eyI?=>E3}tM+9;7tsmZL z<%jAmdw|chlKJZ?&;2z~MNXxz=Gc;W)>kVVhXpHemDv(#t}dikUWsB%FhW*}OoO1$ zQh0F3j9F&13X5%f$tI;B7+;ct@-6EdWc+;bmHIbkihKeHz4`=+lKMo~`V5(Gbewev zEXJiNGnjARe$XwF8_9pqcK|oq8PBAOG`v3(Ol_|efk_V%U7Z+a-NLnOurNO-!{8q| zbay!(?Ol(XrVcY7g^FSEWlaz>y$9N(HH_XBife7pLlnOeIlN7i-22Y+X5ZtF)2_Ut zmv$|~?>S;zzITe;*8TGE`Az~bI=R&GauIo_#v&X24Awrg#nw}i=w*5o6X&lc&DRaF zHzgW`d-*vk1xU@Eop3ZV0e`;v0EQ7$ae0Cb=B1w|#y&yNX)j7Pu3r!5rqARoEKVbV z+2-`E(i8gQlM%UCnHzt+N-<&}UUun};v!!@q z%Td@dT?4H~IIIwfJ%V6HVc~pDdbu4UCB{!TdVM9$5`#!S-^*p|z+u<>?!rPH*ji}%~))(a} zIJo8CzlM){S?HFYO?`JuLz6B$ST_oD?r%_pFM%uIU%>+qOH80)UjOM&6PN}OQ<7-#LILuh(FV?yTT|%CU0ZRZ z++40;`*X}xO(G5({m?Z25jb|op>yga`YHrN)~+He*O+#_(wPGIRE)wJ z8(*}Xk_FoHjW9X*0r_FD1NGZ1Nh_ZSocU9WD-F#-)yJ8+^ym$Q{d!Ld9lY^aK{9DJ z%mh7YX;8QOK_i1&X^{snMh>llq}~i1zbL~kH9d||*HWmuQVbl})rnIJ1vslc_JWgG z8_B=qjE%7)xY0=y#s1yJyt5sk`ziyDCI!LU9}QskaGdlnyn#tcOYqa*&-8$>4ENl? zBI<6>f#5xRd73^)(sjHFUfqdjzxDHj!YOUMD9}tzgx^5uVtcmR{3AKLs1xQG%t9|? zIVxkOghO+3Nucc^TAX?i3OAQAUGHLHX7(wR?`lGs`<_sH#2*Vx)6gj?oW2U~C9fPc zz|8~byp%#3B+7czpE%W9T2ss`L~f zyJ!>kRvd??FV>K*1*d3J-v##mVKowbFdP>MT!wQQp>%keGAag5hwRdN=xf!+6kcD^ z=vn}&)8o>8dmqrFRRIhelFr5~?bU`E^PviB)fmzfGXnS&IgW^{+Tqt^yZdzXn zRfXv^>RApmYxWyKF}@44Vgd4_8CEF*xSn}GOwsZ#v0UW^`}9&9VEU50mnuLLV}OMG5+0f z;z}^Hwd5SO4Hx2Z_94uSTS?seZxR9ia_Vp|31)`y40lkHTTo*SGv133c6$|(h<--x zhE~v73;t`E?|78VH}HmO?#Do{+zOS2^&s!>ak$;8iv@j>u=lzsUKPrNH?cMJkrRhI z<-gusJOS`;ro=ctsSGy`qTr zA83RXRbSbTq_uEhx(jG~3F6Yy%@F6*t$$Kc7*d)tL8s>wIU1x2NhuGA_D&BJc~}Db z&7-d3gb)H%JPYsyY(a^M|0;%~W@d4#m2h9T0a@ zgyU0FMvP29lc%BKgpSKV)Py$ODBl2%XCkq7r8a5!(*ri!3z)-|Lfi)DTB3Hc3VcTr z@T!&r@p>eLix=I8@gD`GMpBb=^(IR;-+q8^4yXbwrZo7GKUy$OAUl~?zb{6Wn_IY( zY96ge>HhajZR%Ib|NSmK`t>!fO3Fl!$xO1*I|2vY_rj^Y=P`AZ4^Pb%M!hGc=*z1G zCP$>PD}GG}jlvviG_->`mL~(VSuyUextUZ=RgM@i&+vX^07*R&4Pzz=?A06UP;%LV zN#C#*e_KYv(~c#0sEz8+rNSJaQ~vZXL(mT0HIAzs6YOya7Un{euq+$Ua9T5jCQXp;}RcKSr59jU`;uwqI_&wd zTqdsm2GO402$TCaqhOy3&Zo!qpBLI;`!+MM%(;s7B9uI`uB7cgF7PN<3HA=nVvKb~ zAv>WBrM^l)Dnn4O(vfbSP=n|ViX7|K4%~qkDb!EWV^=(IlJ5>g`G^qRW%)2)BOaz! zh@gnEBVH=bC2jVXAiLUxsegJN%%aVytc?Y%Qx+t8E1lugF##NOMmTwUE65r?fx)s< z_+N!Iw1tV_ituh6^SX&=Us&OwYAD#3AH>2r%fb3aIkj6h52S{Z=;gvJ_GLsVG2F?^ zGyJ>go|#W+=N6t1K=4Cq{H&U~+SoQcev$%f=9{N~LR=FDM^sVU z+yO3JmgD}W&UBfxF!z{aB8ERL09DCkJgo5=z5K3Xws;5@ct@h`mv3Z(ek1SxeP#~+ znu8A_CGipSnM4=!X7Q?H$P>F9N(}iyLyI56-Z$~QPi!G6wwpTKQH9`KC)B8y;k3<) zg_EkHoV}hkAQ`}m6%OSw^G94E+bk8U{+QvGukuuVp-V#=Z^jn%oZyX0p8k zMr`Y0$`54Xp-yAs`E7(MB&A|El(Qd`J$Y2EsYK6V6R*B;489A8psvChwD=x@NiQ2o z^y{5qHF^`KJ1OJiW2aG~DxcSm3dNh{oph6bA|#%3rvJs4kP0RasNXr99EgJI(oAYp zoqz&k#i*@gk01SZqw41v+TxiFZlM4UXIGNj+G4zUub1TZI&V^P>j%-zasim_3;x^w z(G9(^cxQ(IM{c|Y7fiZf^t#*dS+<1dof1L(Hw!a2Tu*>*#zlxYSwl)K(#dWIEdVoX z3^|^!$Nz<5#<5&Fjn_|p{~-?hIX>jSL^rlJAe+4`kxjQK^MNlpNS!iWsEceT4rQuf zj=&@Ednt<{@b&giqW zl*$gBV`8-~V%Gsvsv)Bhs$x9-buPA=z5f3C`$wHWX+bY$~S6Z zb`UQejw1c81yH444H{04AXe25&Ho)ljW4Gg7JaKFiWgR~!e3MAi`Zt|?evD+EVLwx)F2Ewzhn{~ow;hi|ae-YCxXTSmR zB~Gpg2xQHPg6zeytAUo6Chn7iz=1&Bwq1EFum6PP+$ zjh5^+crcuYdAu2mQ^z!j-20Eve`7OP3mBp#Pdj=bC72t($qOa!9jJdLYQdywZvw-& zR^*eB7f;6;jOyZ2_;8yk+?*RscKw=(mHoVVRR7aN{ND~Re3uO>M|MC@&;>%n=fb5- z0lK!#hS{EP467I=(m7)}zTWkhD2elaGk-O?aAGRwm-JQ0=#?P1XWK%A#B>xpb_tYu zdWx4fmy?E7nRsIPZ#<>dkD;A+*&mC(u%CP9GyXYJ+%>mU@EJJ{Ieg8amfei!t7p+i zJ)dd%<{ebmcXNdmR8Lc*{E=43UNnQCdEaRC;qyf6 z`WY0pIs$7>SAlkwB~B|e)_YW6$=(OQ^78y8@!P?Ezc{FPa7c2G65L%XLS>3KL1BF=C{C9o=C(&!`M4t3 z*}W2Uj+c@T-~vvXQS6>!DX#890kqyPh?}K6>G1T=IPHfJJlCfr@Q6J8h}6dVdsVcT zc?XAo@WaU^049SB*Ew<)G2dW{i9#)G)=M$y)ei>mq!yfF<$*ss!pXNQvzQZVM<7!1 z808N)1ogFT%-nV25NxAOwl8%-bK4ah6a60QW_4H3`C1&1kwduVQanz0F2&0xg>aX< z48)^maU{xYAmV=%oq0Hv?-$1HOZHv%J$p!*_dE=RkdTNugP%LfeK4*a8Mclf*@N(Z+;-xM=+adEgRx9oR|D5 z=LNo|PdeORpM1EhCobgY__}djHyZHid|~e7J|4fY{+J6-Ihs2@Z@O0Jzn%K|(H~%2t;e!PKOJR7mq3Iys&ZD&I(VJKoHHBsu zwRkVCT12d1@4)od24P)S*C!?XB|d7^$xl{0*GSEwT?WVbLlXD-c1L5l+HF^=ol7LR zFO-<-+*32TY{3!!>@`cd54~Kuzcz`xir-S=`epz6~wY2`~qI4*eZ?eqE@zga)6`ps)`{_{0o`OXd>xVUH?*DNZTzcgQ; zKcyj@Uz~2nH441Q5B?j<4NErW&x>*e<+aW(dY5&%okvlsXeybn_4f)(9ZeQ zTU{T!tZmYD-ogrI{h}hb9F+Q;#eHS@nY)j1t7Uch%4>$I0yb-KeeDyuQ+NlsQ~v|* zgW%!e^*U#y`OuU&%Ymg+5fd$_VSr}^!_#jEZ2DsZ=b zc~vbw`E8YyS_l-Tc zVev9{Fg_0@6gNVbRUc}6SwI4H6PN=nQ|OeEP&|D?jeI*IiaGy=*bn-S*qUAgk?$T? z8n3*Gk4+r$M$bFsHVmLx#sP@*9YJ9Y4*}n60vu7ENsa%z0}r&M=&FPfG;gvcrs-iWSKQIr&pI?IC@0)lti+0od#yohjj>l^se2A@I;)wpgWOzMu0vvz6 z9eTb{NMAl5Dz|c}os1{O;0Rkg;s%>z)M!vp2->cG!CcB!VOy?#!{_tVNY=|0jLB|W zy7I$4X4^(JZ28m*BVIe%n_uP8*-=K2eQpG$J73^~9TRE$rAAhA_cS`z@sKKu*6r9`g_>W`ooim-vN2>|9BZUZv2Z||IQ`q>s#59*VZ)X zsv0@+wcfm5Zy->a4 z3@-kw!uyUgWdGxNbXRu+ZuN*Gf6p0_+x9=va_1gMo|Xx&V&Zhk{7qzTw<^w3olBdx z8ZslTSrF~52fKw{;o~c5xMPS*uWMa}PGwi#;paZ&nFFeUf+is^gvm?c9!NXp0PDkIVA;=S*wr~5YL=*z=EaL(8$S(J zmkeWtAP1Eu>d=^r+hC)v4tnp+WoHa-XI_`J!YTc~=s$A`Ku>=^XwA6*ocUVxScNHFdhjv|e>ly|U9L(m&#(l~i>JX( zz>#a-r$+X;iqeYK|LEY;Taasi5@qar*fqOX!$Z&W5HgcXL4GAl?{T3;AAE3LwkcEle;<;9U7Rfw5>PXCkyCKnr!r{rljw6+{c!bcI4Zqxpg;@=udExK zqa5kf34fSfo~>BxyPVZHy%RBFt4^9#jTY(wC24L+8I{B;KGNv?b9gJ zAfl)NP<<-hLQ3WK87(%JIU6vv_kj7oP-1PUHWvfM=!Z^i5<5D%Smj*{=nR zHVHitiVVkH*)-|85R8{43{hoh*-W1eRRx()a?|aFITDx zGNDb$`!|a*yPQkEeVd7Yx~9<8Q;)z-BM0iXOqx#WaHX-2&f&<`KLS=&D|8k}Q_nlP zMCn>4HoUlmj*SN7h|(I+;kAI-l2yFqpR-7G8i86PZhcT;BmK^V22UZUZnfxv_ zqE_7uO<&xhwK=&n!(5UF!P}Nxv z>!SQP3qy_}JGTa;x5|^W0&}Wkwu#X{b(kldButL|FoJEQ2$!EVA@+r%ywFog;Qdw< z)NW5CpSGu9@rTz;$aW>NHGT;$9li|P8^oFG*DLWm{+~g54um9^!_u{4G+?oS!<*d8 zTym^}xhwkc+@_gCI`pZNA^$Mgeh9`e{o~BSmmLsqZdk~XmHN6Ae5q%KUxCQM|SRo1}H$iE!nz%{Utb%B226u3BY z9uDk`LD8MFA#Tb82$*XLW1{~c`n@EmMG8THuRm@*#~~k|S%I0uecbN(2fT%}>Aj$t z5C%bbO6dr8zc~YY6PkEUpM=>vLPhXy`6POzDU0*#g*DU%yoTmST};s1PN(Op5~OF( z9Jo5SA7Uq7VQ!y3RcV+~1rD3!>2U=KQm#6Rqm{xqe?=1>J+O^4dSVH5mHWUx>0EZu zJ{0ec2vIAWE^vK2g{J=s2kn`|aO9t$CYJqxwdJ#zHr;nvaYT;XJZnmCEZ++TU#3HV zt~t7Vv>@M?uE7&U!MLJ#F^J`wQMLE6tgJ&2^Rv^M9=bo5h_26LJWF$dH?tYTJ3qsQ zk9klI_mIE+7)QsS3%~Xlvp)}M!^7-#DDFNBgPoF4w9);5YL-_v1zm?VFfKfl6;D{t-Vtkn2*U`h^!Fwk3fIH;uW`)YtPj|uYD7ZHvzRl92f=o# z1{FJ^LMBaJfV!=QBzUblS@q{RC(iT=UcL^jN zl4kc-=U@%z5=6WXryHZDF^#hfX}newfSVY79<9%0i+)D$a4ROIW;*4cSkLa{??;UZ znINv^L)QmRK>Lf$X#2K;Et2Na(=!XPHC6$J0{Y;D@l5JXIyeJn)7SqT1w_1RSTtCDl%?BMB2{Z}OCIzJda6tYg@b~Rue5S>5WMid} zS6TuJ!YydzCPkKfod>b`-2gvg@ri)tt5;dTt5|Unl|~7;)CRFh-3y^|#YLXu-#s`} zL!93H9tOK6iPN7OEKyvZusO4<*{zQ)Xl~wH+#=}HtrqvdqfHYqP9P)q34ejB0Z-6w zk00Y5=Lq&0N_4(a49FS^gU+90IBDxgT)5#d?3*}++KRZL(XU;w#%tWE<(n&Av}y$1 z_RK}kOdV#;LN&IqNRzH@ZDo$jAAm)Fdhm|dR<@;BjYKNcWU+>8c%wC)9+0H>-0gF7S&YXO_ zt<4VJ5b#{A2HEAq;rREb9bFsvi0z6}p?5Y2k%HB#r2XMD=xC{hbyo+O6Gf^d*ieeR z+0SDfrj?`G>J)IByM#SIa|7lpWx)a!M^ec>0>bq_(IKOS*&8HAk0pJ9>PT^_qxOoq zDmZ6sj+=`nlOBN3{2!QIdl&ZIQA8bOTeKfif`G55aEf0joGi-5lBNl0GT=sIJN58F z)D+kp5>#nmXGv<$5AhgXJ*su~HuGSPIbMi0WjfR)$@||=ao4O^_;|e>)BT!Z)2g@7 z6DUuf9!-ZCc8}ry(|=f`)Pv$FA8_U1byyyu&Dpiy86}@=g2B>L__C=8-O82dzVLFG zv->T_ahpFoDsdANOAp~H8!c!tW!XO>p^#zt7$Z4+%#YfSzlH^zL~BhHx^oH(l2*fr z@+;tl>SOe(lQ_093r6>v(e3XNF;&Zs^j?p`N8jg=v`!Dwwj058REz2zyoj@2h_Gjk z9SC!(jrDN9T-hbB!In?~2Yc@g6!pk}OQo?8M5cnYObbu*k|a@ADC0O!cEOuH&Y(4R z9+d58;EEj*WLt+go&9hoUg9}3LF1R1p=?v$?JI@2u`7jHy6+MjETv69iG0OrM+N9^ z-IGkh%Dbo(q(()=f8p5caY!7V244@$kY&bq*%c*KC{kVx`CBH^Ey~tl-kAx48YYsD247Gs+@9nmzkq>xH?U~&0gkbg z4LPZzgr7%ukdX^zU|ZhJ1kQNGb9)^@?}W_99gA*PX5V$f#M>_T!#W4oUK+#II-X49 zFJIJ+tb&ZFX>^iv6HE;ka8TbFp#DcsDj|FU)J-PLqe{uulL$=d+4M)28lMyv{G_oC+t7+z}` zB`qB?^JyVZ#oLYeTIpi+lNp3Yh~gz-1>$1D$HQf|WagPUn76SM3S!ne*>AlJUe-sT zyMHEKm!=FBv*RgqHxeEF%Fvy^40RO3DAvqlC&jtaOs*ms3=XE<-kJEyPLr0jC9qTX zdO_>O3uu>VLo_$Bu&Yywwr*6xwM!?EVoN^HM*kHvd^-zX+*2j}3tHLFaR#LLs5Tsc zZ@dY2DVwrFl%5`5!T_Nk#Ub?ygOEi$JW`-sx@`;n=bV7PM!z6{ve>-x z0OT!*V(N}PLSC{Q8reotUH@*TK30vU-jN}*H)_x`Gk!sQd4rb1%*{Vq*n1Ic)w9p4<#qfFB5zh2S>j<NP>`I_SJ&2tYIGHZHq(*n_jAG}nPlZ|NK&{`^;h$VX zsxwoD*x&S}a*-c_clouTfBz=n+vpRCL)oBYkOkj1C$KZiXOl%Ql}XDzak5RoYnt+F zHtA7vCo8tTggNGX{HkC=%!Ja}bAJ@*W4*^PpXW^H)M`=_b9qu|`5DLEM&WjB3>^3! zMy~&qpy|;Oq*(h1DmDjEt-O3FSrbD8->8Ai6b)jPIgx%Xb0q_L@7en85x6ch7QT4y z2EUHQuxjfNJ~&^%=$1;rrLI&apY3 z(dDp!q4Swbd38A;EkBOSmrRDMlU->`kUX9}rcYnW7D8*96V-ceK$0_jsCgd`-u&7? z2UY~L^S}ONf9==;j;6Eee{U~<*`8-GcxE|A;j23ti+zhtx_>a>^b9h3U%*BVuz@q) zufb>QHFU9y2YV|eI2P6lOBYJhj#^c+bj?jB?a3VYOfr$Tc>+n;a1Dzyiy&Gmn3Yre z%I3UP!=knExG3`!yMLt?Y#ja!=7NYIMM3=!T%Ne08WQhRnUI+5YSy zkdL~Dn{IxC+J;r&F=sWR%XHwI97Wr+(sZtXt84C9gxPk-u=IgD@syoKllSe!oh3aL zD)AxUE8ty)CAzYm?@qv*G;O-T)`oEJ<)eK?Gd2{vqUoH8^ye8%7!`Yjk)x$-gi5dzXtiHWg6kqzo+x zQe_@ZPk=|Qlc~tFANcwETBhs@LdUOM6b`W?Cq+)<`_Klk@m3_EYHRVq^c*&5SrMz~ z;)zL0hv9i|D1?t#V?+HPlw9%$Q;YZD^#Tr0ewPWkv1>BjdHyPX=$(o87Y;+wK6TnH zqDpp)*|6v9=TUx5GMt>T1c2Yi77BJo!Z3tPsxsqLpYCD~oLeuv{+%pWk`S&Qd&$%=0zi-mnc1jNiba&_THEq(tL=-3ia~HU6^f$9ZQYNW||m z4D-keA3b)2ui(Ilv^hh%Q8D(O^ulNM#;iDD8U1&Y*@iuy==5EJD!1g}!-FTV_!JjI z{s?rxgnf9nu^i5y=)e!3mFOOG9?meF4@dr*zzPAwxGw(&1iuyVze>-rnrB6ku@l8a z2@}$`N`wAUu3>U41hau@U$J(v40%{BOaG0ngjvqo`=A^uQL2Klb_>|Ha#Z&SDeA+`PxeW?@g;27E- zdj$Q8(jVl*~U18V1tz`_U5Ici}{3B z+7FsZ-mtMenu#&I36s{FlC*oL@!erX_%=Ai>ne-~-;@VT<`HR<+)>9C&7BW!k8?rx z_+;X_@)kQKx*d*r1mRzycT7}454*2fg?_Sj!2bSNeEaDs8hb2czNqIh)%$@>FFpwx zpFZKr6E?%h;K^8f0|iQQm(M=kUSYgOEwe;ppcB?AV&Cc+Oji#x0pn z9E(%%pTjepx6%ORIuEe-1KsKQyT-&~bP)6Gi?MihEbLDY!_@3t5O05rhQs$TUo`;d z7o3E5^>J7-IKqyONs~Q8U)lTVp)e!9N1)eKK!WizjJq)z5I(WiUdCiV{t1j3C*$Ku z0$%01Y$!i^7jL|}g>1eawvb5nj%qu+e7+NZ2k2AIf0}gCG&g42oOd8)`xaYU`_S*c zEfY0Mkti2bS5-tMg$P z;ezi>@t$U0nQa-a|6RaII#LQSD}fzT0eB}4s2?r@%@I?{NB1!dsMRDF7hixS*UKwc zhS)N>dR~}i`~)oB48f(hjk)uPfyARraMhV6c(Uv~w)HFEiV;yFF1nN1C>w!1c!t}| zZo(z=80PLOF?y$28{>=m0Qu|4VZlpEf-+34M3eF4ocjp9@>wGY(JB0&9zd&_F z6fVxHti0zN56QOTC{iU0VcQUPw`$OS{sHz#ej2%Kal+1=Gy}=337QuVbP8_HWB-z1yTogVCqk`5H*T2|K%3uv1@Fn4^xbzYjLC}TX_zf% ztO{cU*VSK~84!*w+oDm96US4I`^L)l?_zx;1#pJDnQ-4K1h=$)gbo`S^pcuJJmth_ zN6;DICJWQj55Jh3*E5{f#TycHD_x2ao0!PQ(af$xc?B0g+2X+;Bal4#D3+dj#%8=wWJys2n^|d2%a6vw*byW4zkkPJ%k+b6>7qMq z^kscCZWe*SsAu?Xqd?2KU&qWTu7t0Rl8k|y6ZW>eLH#~A=zKbiy7KDu{^WI)d(Or( z@vU_jU1~~jM?N@;K1TeiNNv^zL2G&Srve$*X7{bw1;?n%Xyf* z!v(q&YS}wI7NoD>E@&?ssr;|)4pS*7N16|=!u3n@@P?HcInbquR$q_9w$;d9NmasC zvGz38m|^DT$kC@o6e6*ZcSENS&I}o`2KOs5PEwZKE1@=KmiEPAcJ6LAYU^jv8?r&2*^=bfm$}%KFbglTa%{1W5HZ&3VtbX>vjJki zE8BluWE4A&vIbj=d6oe^AanmMy0sl=dR{a$C$3B2h1r3i7kvt9j_l$6Xtg3qSF@0> zcAGai!5h^-5wOkHrC-jUf*r>@VRecJtgMWO+^hRB2do7Ba{zs@vW;;Hx(vt8{^6`! zw2ad>@Ru>1d6=me`2$yVX40oa)98|4F1&>sl6W(-B&d);?^^xa3B>D?V7bN)ELl^I z2XapX)lXrDj~&6+cem2ly=@RFtPG}B-K<3SMB2GBOF#~^Cb}t~7$S5FHn}C^;IAVVat7RK8{<)6C z3_7EsXgKpqx((EKx{|P68)@&lWn^8XFfGhhB;U5Xk$KiKQ-u zBr~jfD@mdw_fUnjqp-Evn?@^t#_2k1Nn@ZXU1hu<%uT}BGtW(lm)2p<+H@{)Z6K`u z^&rBo`wkzIoammVoowXl>tIxVkQ3fA6_RwN5F~HG_3~bv)Jd@5vn1o?5J^KJn0|PE zfcM5*z;ApKNEfVnhkI>Pk!f*5u`{PIy72~#3YihbQ4P9wU^;25`V9FZ=Cmbs4`?Xt z#n_Z+sOLTb6OUb3dsGKGf#&#kbUG1Ul!JfbCSp*RBf56GvO+0@?ym`>!y|UIe9ug_ zKjAfu+S|~Tiz%{SJ1VC%>Cu7*%ZYv5PyCbol-VHAL@HMMQP(Y7Nbdlbj+XSn@p-|5 zXQC&Kc)ymUF_)O>AGT5-0kc_px(@9%5+^(4LP&^!o!b6&C7eHf2dnQ*CB$5uE^}Q@ zRS(Y~MsIXU&cf%FBJ}Hbo3rlI)0WD&&+mAHwHzLKG!U!i+gBq2N zK-hQ+Gsfsp?ZNEIGoIn(c&Qz=Pc>qdUT!1r0=?;v|DuRWNdVn$>Psx1=A!K45=_6I zfn|rP*&Bs^^zi)%QrCG6dKHvux|n~a4LW=Rf_Z;%zcp$BqSIOv|k{};DEvVx09Q1ql27^q5Nv~Ed zru|ex{V*MpbyI=~f2~MQEZk2nIfYUExq4W7o54Yd!#CU-i1qR%*T;it;6F)t5FkNv zXRT)Z>Q<4Z5wUdL_D#fkltXQHuAqI(b?NeT-_de~JPer3qpc}QXb@mZ9bPRV7AZbd z>{c4=J)OgO+iF3&hL+IN*{O_hls1vtdQMQ|BdCk~7oJ$G5$Vt>VLevr(g~V(u;R`Q zj^olsR0=hqSx+CsuX!T$ZO}uegZB!aGL3NTVG}MP47+Z5IDC~{!sOUeU?THyhmJdu zH#a2zEt!nQ8yp~GYb7`+*1!jCdFEz&8_u7u1ZHi=F~!@R1dLbUOeINDa6pu@n|H8e z@m(m}FrS8me}U^_qo^2J&x!dVO}NiYX#PD4?w1zvk}ror;#zrZZqE|T$Sz~Y0()W^ z?aNfW?*@s~OxAvKDzo&tDu_H6A>lo@F-G+R3=fpBj=s0hU#fyLx+)3Q|L_K45kMnu z>XYH=9J;6Q3+y;pj7v}W14ph2hIHk@>yO|&lnT*J>Wbu8Ogd}rsRmEK@4#;NC+yV` zLtNhF2dT$}*eAylv8(+dP75+e#h3Erh+D4Ncer!LV>qo<3$tc*;+Nh! zwm!oGzhp#1)ug%f=BE~nuIz^Q&N(=(#179`D}$|%Jy9{0riE$m$gXlZGO#0va%q?s6D(2f_hHnJi?N8ogo6g#K> z9JCiYGBMfp*ln1>oM~!>ZOIpzp%z)-dN+Yc<1+fzX+21%y}~cb{*|FNa&+C30F(|n zi{t)Q?0?_Cvu^is-4yyHlv9~1scj^VRUVz&C-97}mMN{e6m6Z_E zn~EpGuCfm~^`OD6!0#|0V>{k*rUtuE>8eI{y=^*X3g8vpuk)e!wKfxL@sdOBN7$Q+ zW3ch)EsUGL9{ut+ zy~pg}{bUPV)oTK2FDK)$_IqZkq8hOuFk!p5%i<2NHb!E{eJoIGhM2f^nxtX^{IZ>c@$kWM2e8}~hMu$J~(5^m!t`G9yS+Pr? zd+9|+`@tl#xpI}DPgv_Dxgn8ptX;{@__+iZ*4Z*$(!QKd;}O)-kf4O!3Hi6*!b;l{ zsF2oylitb`BU=;veTYHF$&rlHHVHbct&RVVZ3L;4iR`&!w?W6|J#%oq1-Uyqm*Fl^ zWjjt);5A!9y#n@vNmnOh|2H0gdA-Ilr;D5%x#fVB`@v;Inq-{XkMxfMz(VkiNrknHSLKbPK!}Dq>YHs^N`u`XDz^i|*A| zVM|v#61|)|yh^{PnEEu8y|VfYboe+ioPBBVV1hU#oN*wDsoMBX=sf&X_z#>7UPH}8 zKYSsv3FO!u92wpRO=(AQ{LDtC<>p%^Ay1C#rA+6n(G}t)PrV3{=j++SANv`OMIzR8 z7=fh+3sQc&AzOn(XZEFWLVw%S#JRtje+BwPM{+Wz9F`;zrl!34JN1Zj-Up`#7sS{* z;UctNTAw`jiDQ0dy#dDEf*kteMKc*?_&sVwSK8Y#ZoI?b)U3n&`0Yco1Ry)7bZPu> zb|02(`GP!|Yiw?s1RA}cjVXWa=y|U=c8;V#pKh59+r9;mf+`)%FO(q8MF}vw+YnYK z-edw-%w=|)$3ugsB`scS0sh|wKGp>nm>E;GaDm}=M%~RCJKL7vx-m5xFmVp9?a9F* zbceD$KU}aw8c(LpVjqiiusve8v1*YtZgG|Y!yY|I{^(8i{Jw+dT@J$osYk4@-3lUG zSk8{S4?*5!A2eGu7Zxr)&omry#Ah+fG2@*eSM%33Saba`T+dKu7Y#Zy$Dy@1Hr2)lYRVQ23&~rCC-{dPTqUxSLL9L3BMWz_v0JtP-5DARxzgsi>xB^t8H$q>!7OgSktjJZ z{(${(*onT75u<18REfWDI{v5o6!*Ch+~Lx~WbE=`mxlm+mggCG*Nlqh`6WiKpt)m1GinYB-QEIvXTGpePvW3XL!UkqoC~~t62UA-h3F~;(juEh zFdJV$%qkyvzRifr@kMESCz5{!(MKdiW9U-&zeH z2NuzKDM=_!Q>TB5YuL#PW|LLAqD0d2EFK!%jB@`4lNnYL94jn91@m0U#=jh^h3%Lz zqCw6ynZediCiKziI>3FC>0%2FSkuEH4Qb*uQPY61kE3Aqa-a>tb~HXwlcp-Bg4M5a zIL9kzO1t%luhD;~c-w<4S=kIL`Zr;v+eJ_me+PH7^+>Jo2T(sV2co^sz_?Zjd#J^e zvJW=n%%ZK3aW5I(ZdyQnT4#evkv%Ot2%>H~AJ5Mb^ z*Ddckzk8*qYVHTT9j8uzO_zf_a_S@g3;IU@{oG8+!Z)@6M&TuP+)t+V)%99}3yAjGnClj9V7|`R& zq@lhKmA+0Vi-e!BdWFGcPqIHOxbO=;TU~>|wsu_cDBJ1KA1jg(^BrCeR-*8PH0C8Y zh%Cxq47!U|sPoWGwrtjPqBzun7jJn{fA30QFYjbDn0EHeVOetWRwI1>aRBPhu7mG| zhj9OuZMZ{0n*1AH1G5S@Qm^1PCP8~1{W!_i$zAY2^zJ{-9EcYp=eGFr#FS0Q-fcE) zcz+@~_!Ypt6}#b8unbtW$P&|+RiHk7C!%LFT%9+a-0twl@w^OZoFGijehMXDUaC`v zXi>1*q(+LWq-eL98VSFZ!Yen3W)gL~LA@%MNfp%UkSTi9v#boa*^EN$@H=cC*umE6 zELUbCZJ z6w0FGARzcL)EgP&aK9X0bueT#ENhrYBmvas2vbeR*WkNBp2=6@!K3vTICZDCF+IDo z*yvAkG(PY+o_6X3gMu>5HU7eJQcYn*Z1UkmWfEkC-Gx~yRUkgQikZ?rpEOoXqWvjP zIB&+)iTG(%C~5f2zV?2La}%{ll?4M)gT-t(Zh&Y1BEe20oIEf|25C_iLi$FLySWX< zcAv!C&GlILt_q&f zJ=_5k?MiU>mriEtEh!?)+$xJuZX zN(|>h%2ywD-p4*vSnLk#5|?7Q>0XRpeGMKxHlzCvEr6O`T2#2^5f&R<$5o#%uu7Y< zV5~)8#Zb_otwV2M;)JEpTzHWYvW{gpxRkMnZa=}&m4{iM;b@-I`VVlVW)Y5S=+O&5 zK7y{?J|>Q{3meaCvbA1bG^6(zw)eENGWXA5XO<U!=o1b9Z)yKr?=!_CL8v&1KT*ww9aUk@y?X*W>KbIx+En(T^do3oVNS(N-(XwB0({D?;ap$t49iM} zq4A#q=ZnKku=6;G5Bhs?MT`@4FQ~(5KW?#eUd^X511<3G+jP3%zAn3Xnm-*LGl2f> z)x4%$GswJj9iH1igg*7x?3Z8b*!Ej7;Pd?noSAnF*G8X49jSbf(Vj}JFMo%CfL!ot zDS{K4zp*UDpMB6&2wDOkb#YuO#I_j0$Y>WD>09ISchlJaJpN$D7JodvWe-)2kcHJf z{@{`GoRj;Ki=n3aG<2;PPha3~ma1bIkA7{UePKQO;E*Lf`X|3Kv`ZEhX?~^lIbXK3 z*PA}8H>F-(*Wlo+ZxE^3hI%&|LAr4Ro3iOLrzgaU9&!!EgH4ZM>p) z$d7M+@aNKd_Q8!i5F?(16O~nI)jDN1bfq4K_o-qjX<>funGD8hz}Rqa!5$ZYmN&3NhAtB9IeBVLC7njE1pQzo&qUBy z(^hXLOe_|+r^!ITs4n(y=YY=+dmMBTqV7^0qNIKnT+()6SDC=?X;22~aWZ&UG6RR) zlG*nt8dFLb!Mx4N=A)xAy zmJlaYpyPV4c)<^Ys88c-NDs{b13Ocgk#YgQd{QE>NA9E3iX*rnas$o!yOQxZvH)Lv z2xOnO_~4>@Be3OLAzqQZ&#*SrDcht0<0{q6q05!{qGu}p+APp8?NdQHRfeE?8G3Y- z;IFj`#A2~Kk5!SRw7`tmxgTRC^Y)`Q`yK3V^x~d=5wgTI0iW;#;rsRzNSvqO)0(-M z`;iO36%{ec+8*@uA45g!FI;gzgtD^l;DF5-YcN-xbXMDt^xB!&7xo@AgID1FJsZK) z+aG0&GjYXf!N%9Zc4#`f6|YO!V^*0D-PEDX#tte`hnFm_ur{GeTU^QU1`)bc5VDeS zrV!-()`I!!0k(6V4VYiJ$h=vf4M}d2jKizV@MM)RebFjQa&l|gz`wQZjRjtGk(VoG zsYzqPWiRm0nM#&53qeQ3Lhw_$3*uHn^!EY1%Ki3>P@NUbcsUNFNyP(A7PT|xuMqp* zgjN1n?*<*hwa9%sj?Ye5l4tMv%sjIu)*M!%m!uD*s(xkcO>@~-o8O_vW;e>LIgS&L zTj9ag9&}c@z(g3z;yUkFP#;Ku?_<-D{&wKd9M~%PnSXaj-EtfQO;`x;9DwV8aYr zET+1{7L3|S2ER4sz>z6Jq`c<>^S%5l3>vgB+eMTapLB83DdK{`nO)4ot@FWjq8Lmy zRwVx#0?}x*5$pO-o=xDN#?=@Ld9tGHh2w{Tj@EG8P8z|36l*ZzisPuSH)(r21$=DJ zL8VbFM^|(^F`G64Pxh|FDb@?w8#m;r+L`MDzr`}RoMjAyQNctyDGhUj17Mm}C~ORN zqlR~iu-pFw6dES7pRE3&xYIE#YZ9luRed~ zi}KX{gaTOn)TA5gpRn$}TJ-VO9FF%5J9=~8Ww?@T4&1hXOpv1t9d>@no_-!iqqp19 zNsom|p2BYk=on$X`HJ9)Ar&TS=U)`ook&VQ?8lwoC(*xu4T*-}{#)7D4#H3NLYPn? zI_u9TlN3%cyA!ft`zA^18}9&5&WVDrvOW<#+=|x^X2W9hd}vtq0y4~x3-q4)%1OyL znMsp{Fyw{|S?Aq@hyNrp{Gbd3fex}d(34s5&V<5EA}K)UChOIqfM;G zR|h(CFr4+-G=~oNI^%f-Ln_^?1UC=nXh5APnWD1Q@UXcPN$r#X9M))c~( zKEQSgJZSf_KeO5~Kky{E$ze!P(;P7g$&Nm-%DXgA0U0FnbQ4Q7PMieQJqN*C|HT z@8#l=bHNay(8Mm9y9~G)8jwz-m{Tr-eZwIxocgUq_SU_FZHXK-kac1znriV%q%yfW zN8rPneiD98Ife8~EhF6OK+lwl(mq8OR{7RXm?Cr&>+dgOXKBmOg*HaSH9>*N+^$4_ ze-tC{KWVbh#Cq7}1$mh9w+e2Y%7t&e^$_RWj@cv6Q1;7TIKRn?INU14;N%!oSW^p) z83XW`pNcj+$C<;QXQPtuH5|>|h<1{MeX;8ZUVM3zsT^{n4ia}b3r4x@j-R$Tt`~yu z6mMdL#vnwyE78A!5`_J}06gthqvyf<_~-KsYTumBBwxvdt-6+^%)ALgX5}#Rj)Y@t z%pc4So<`=rJ&y}ROv&y1Ev)XbUM6__eWiW98g@+zXLq=a;OULaSktQ_m6Nnoi1zvl z=KfY^)DY|d-MS@5=7p^$>l;4d%I7bcAvP8&zj>4Wk^#_dDew)Bh!fu}PWb!HD^AYV zbFA^$b+A2POk6%p=S|*ljZw`P1=jXAJn7)UW&b8t|MxB267>emPCDTyhZo@2{{Uq& zSJP6ZHf%jD1csibxZr9b7>Kpw9nH0@i)S6vX4MTNk{=mou@k^nT|&RjlX1V<0rYU+ z4JP5|Av5J0bM>17bQK0LV{x0X^?_g>zB7!y93_}(?^gdGMd#sHk%k%4P#I}yP}+)iMCWoSMlexh7#LweR?KEp1R;uy)*6NCTItv-kTaTE-zEhdi^{s-k zj8ZOnwgEq5n+f0ARVP>&I*^HT(_yE$S#fsW+T6jO!55B}Oku~AJ2H9&B@AOXfNA%& zWxqvsG5__vwe(jjU?T3@^8E37?uEq=_VQ!dnfyt_ilr9>SCTBa1G6}BxrH+OYT+EF zZbrYjc%v4d{iKqalsk~KDjmXj-ajCor!tGJd~k&Os#L=yt-C8eyk|16kh+Yy`Q43c z+V+wQ=wHe%j}QwwjyG_2S>tO~H+Nk8=;$FHTQa$3t9l#fsn%69rznaWR)1VDVa+?{ zdiZ9hKTDqPUn9?VG}{VZ_gJ&XxgJ3x_fc@7=nI$7vylDo>}kR7=hcG4@hx2Rl4QpC zuLAq~wt%nj_7_JA=ko^AezEPPy)_Nh7a2QW6*g_RDQ9Z4%VOt`jm*_uk?iiEVSL%5 zn@nQ(Sk~4rhby(ttx3Ioo|$tXOK>~JpXC@!%e05u;(^#97~5~pE;ASK@9H9$#vSw6 zCf%_tvurSbAw-R}e!aA&(CG>n+8oZU8ri_?mdzyg>G|=yza3^~MZaZU$DHFL6*h=H z&MXmI{u$5fNuDy@b_2z3H^%X|mn1QlV}3F3pYP<2=W6gCuVb0ulNF4a+Y@e&;|*?) z7RPjq9>E?yH;XM@yo>2BFJh#pRmFiRo7q&QPNvB;l!-+H@xRdN{DwpB>=pU#;y0Hp z`ThW3Hm2bZH_gk)@=1UNQ@u_$*ZISl4|W{EjuA{^(XW!p9?A>uwG!s+^G{ry)=Kuw zB8GjXJB~ToZo#LvQSlyT9Q&``o-diC%5Mxam-QRgF%ox6@m%d_?oVGB8*Hh|w?4OK zM`e@==DHnbtTW7cxwTiB+N)#O!*)~o%)J8zHp{0Abe>i+A76yBa|)6K!xoO_N4BjJ z#7nlZUt%J-ra^<4?|;JtIro;Zc88+5y6OqyT-UM8%sobYLeFt7Vox`7ba7uzU(xEC zh}Q#|r{x|TJ7KzPCcu{c*{vY(KBQ$Sr(rD!kzdEYYOR>X7une?Au?rnTHBye8Ra^oO8!( z?x)fKR!3HMH9u45Z+Nd|7s+OqPCjyFI?fd^ro;F0_a~e4eka~BanDDw)7>Vqo{Mb- z(Wk1I%F06m&l^$vxhdnB0wtFk;UOhP?l8y9xnsa2-}}j}>Ke)2d}G2p{F%o~2hL;j zS4?E@I^_h z%O*y1LjE168xX-tW?bX+v$fgsEs26AD;NH0;%4T&@BmkF?=QF0R*_XT(--GIG2&N0 z-N{_llHH-_Oa-gY9OVpiH!;Ne7gH0R!I`h_6w67cb5Zee+=;wsarzT2{#f8_et?Yz z>wRD>^Zc+et9?#aJaPMTZttl>T#jfG+W;4sQ>Rq;=d%Jhud^d~+Cl7$zKh%d(!>N7 zpJH4pT*Mbg$cdMB>$2mwW^*qlU*l>%Uu5=X%;NPMr?LOF-D39J2Qup&mU3~ocQVVG zZ*k2dJD5lJ$~e!E7AE4S%%kV9jnQ=+%pYE4D42P|onN4{i5cBJjyLvb;3Cv5S#J6; zuJ^la{_UEO6WC{RN4Dp2W)G;~pw|-S1gFCceejlE0ulGVQ!fgYXu7%34!1 z|KA{H`DtTrz_-Et(3|U-m#^ama=k;i!Mp}%xWr#vKkq768yd~rsO@6XccwA#w=CyE zClrd8U1$}jUNvUJ>?=6B>=`%Isk+9IH{o`Nt`dAuvtqB;{IJaJO5&Uw7IIE94P9mA zSiyfoE%>J4hiZ1nJi+(35hm;v&y5&$n@ifR!A?ym;);4!Fjh%@mOX2Ka=x0?jLlX> zPOD3s&FwnIEuMB$ym*BnvuE-<#_er4Q&4-Ai;;Qfw%qZ-bACZ9C1yOJy~ZV+1hKOV4r&JD+gfCyd#G zC)t9o*OPd84=eVLcPTgij0!t<0x%dxsT1>E~L&a(Y6gcrrkmM8xdZv9IPI=q7RD3CF*BQfah4&nS)KA>%$B{&nV@a* z?5c+M+|>`xT=98*hToP`6ZGN)V}FAa>!^HXwx-!JwvSivIh|52_R?|YxAK1GbVw;T z5^VZmx#o5w%YIm6j+&Efb` zEA|WP&!2xE$hUrF`H6u}Y!bhYce!WInG}YJch-AzUsLAtgM3+bpvhG(uSLZFt6#=$ zdeqHL`Zb5&J8M20IoFThp?;s)6f4IqPWi@7v|7NvS@wZbhv67ux2K?ZN7Ut3;U*2I}6f-Rd&dCpA1I}@HOG}PeV47|YoGH_sXmN0yEv<>(2 z>^8woTTgafow>|+CS=y%zr!?N`@-pOt!MmnY}qFp)-xX63cR5b;cq8+vfU?a__r1A z>~&dY$)1sF>~f9$0`ZOGOpU8EpSm@QB|dM&b7a2iC1!^>y-J2(Fn1VxL1P;0ovp*$ z{BUCvR=aT_iC+ab1UBriduqJqiy$u1w2{-xOXK{e`LL}A>X_2pE`ffUh!0^lvU#Uv zc}_d*`D>?K*;z_~e3iyPwzqsTE1E9L?)js@{z5G_dcL9f`h2?@SFIn6$@KwuVeY`o7mVVt z9&T3N05*2xQfBgR5dXc~&#;SL2_DI2c=eC2=jJ(kvJF>SxG&GNS*NO{Y)hplubEZI zMbDkil;@<0&6dn&=XT8&H%WIgT}>mnFm(@hvnPtTccW^8_beh z25d>nJYHd}v&El@Guc-`Ui=F4No=9rF~)Ge5qs*)eD>hw2bS^O)A*1;iFnCUS2n6+ zqS)89Tx=ZHBp$pffL-x+6&D=vTo5|hoIN+;7q_R@j5TZ>$>vq|bHf!b3wAy*PyEsj*aHhR?cE4PMXRuSn0w3oO6gQiqSwO5|k zoi~d!&MdYJdK%8mw=ZFa_C69f`CD-VWxnhmq6ds{#x&mMP(1hXKP5JDw;F$ITO;RX zEbEi39Kdfe^kZj#y;C!wWfRjJ-p~B4oWr?}+9K#On9aU18ONt3EMdp3xWn9+(bJD? zwB~OFzGMz4|Kcu`=yEn~HB7jTHM?8k0~g<+&uJycafgbkxrr|G_yyf%-1<|KnH27w zxUpA`-}%v<*H&#{>Tdp5vs&*xhf5j+?`_gKxA=wprqEob{@Wz>WneP5D0C{nVR;j` zCpd|-t$BE%Deka1YO5>r`5*18IA;tK&k>O&FJ9`S*3-!+Q8LN!_Wz;yBG zC`CSPks7Nmdt==BRm~LL)#i_hNAUNL*z+zCBgMK6maKb5GpBylls%v-;lBPD&l}vh zED+bs=HG;_=RcSDvP`NyU!1at->E;4aU9&s$PexjpHTGXAG;)2?#h%EwPreejQ$2r zv|<~B^qAn=;*WxtB?(O8_VH}SCE0ze=_WW7tIapcGE?^S_K4BJM5c8cv7aWKIQLVrw!#Tk4@sulZ<6``Bf&{s*>BL zF@m>rnZb64jOPDNyu__I?IjMD8^exUKSJCXds*Cn$;o2s=yL9eu%qVuwQS~nsFYc# zxq=^U=EFWD z67kCqB9`IbF~`a@8OdsGSwCqEE4}uSJ37Q!{7kxwyY%@bb3F2ac!2yj=3+;*cx0m% z>(#Ez{I}^1lbsO3z<<-&UAN+yA5ss-w(AmiQ0@`)-C_)X<(dQE(BsJ}Cb{u1D<w~C!;V~f>##*^{m zCqfoJ*atT**I})n7TjvE=BKt##-blG&Qi@SqS6|PcMo5q7Cx!4X7@uHu;v>qzkY)3 ziu57opJH&}+LMY6~P*pG}wqCh~B!G6Lb`wojXtM1z+Lt2SQB9bbMH403)Zo zChoPe4Eg50kl+`NZWq$=M^^;C7_I}QpAHe%E-BGV`H$JTwiOSaNub}`w6QkxB8H62 z!_l`+VoB><;b!e7GQ3t7V`O>wLanGK!#E@3)avuYWPx&)0#P z#&I$wJ{6-)CgAxMNuZJHCam0+h4<@qMcu9}+}v>pe3u3j6Boq8!IME{mH-aqEC=7^ zx5?C(`l#(6E7ZF1oF4t}3a&kHkS31PLa*F7BpU0`Wvn%fChM`hC7Y-hdvf+(O}J1Z zC%viKD%5SibuNDgns;saoPgSa>CVPW_YNPhA1iq;J070G$EUo7SK{BHk=#J?z-bx*vvu_ML{+%Ep_(pg-b&p@su4GZ3KC(cM+N3y zm=iM*zpnZZI%o8d#Gx@LzhWZMbYD!vTF2r?{X5VTRt0W%190uh4|MFiEF3nY50iSM z=q(KJojCSvf>dm?Z}_(K}&X^)TIe&fAVq=ODt;je^EnLeUXf(=DGSgJ*-sy}|D;`s zW#rDhZNk;pDq!TJVWN>EVkEzIN5GOG8ec&tEtmaQNMqi+FGbUl<$wy>bWGa$|El{<$7AoL7ExB_P zN6H^1Qb!pBSv7~w(mq1NGEP#BdV*)?G1Moq8#CfP$@~#_U`{qiH$6QCCz5VqXpbiQ zCSfbNSM!am=(i>D=M*K&~wQx7bH2Jzj?6!=Op0*kW4`+lzO<$a3K;=Gi_h}h%HAR@Y)ffj%(gBfhJGo}>gOjgSV9(#f zxZ*;sjE(yp8_TcJf4@%?adRSwhCHO>!VGZQEC&el-Ua&yN$^jx795s7!Pn&z$wG}k zlA)dZ=#!fLu>JBcX!Q|E{MsnlWP4L`qwx%JYe>b!iFstS;3VX~nFj}79mW^0yGX#T zmo%?yBN=tf2(MmP18w&$$XfX;!kG~x;YpSdta9~43kS6zRD{5ouSC?~T8j<`Zc-By zH|QO;783NvqSf+t+S3w=<`(K0yF3`87yHtdk)g18TY`+ZbP(=8O%~P`|APEUIYPw` z$>2O+2j*)v;`YItNZ6AuA~anLK{F%afnp;5&JPDW*;(!~KNha1R>2LGBBAqt@95wm zs*)JCNOj4?PFBJ1lD5UqYwk{zrlrm*d@>&A2)} z3ul#hkoju1aC+NHm{2(t6pM4Pc0R#;JvGt$UxP%0oZ5v$|E7S}-%>1Z+XoiiclqG) zv+-WOvM559cdT1~5Y{QDLguPCIwtN0yEmwxei~p&FZvKTV(|+W6xGwVUq)b|b`o6s z!XfF|1j#-7Tat2l8;I4sfQ9d#08Ois6n%ArN6l@d=+-iP`C<=_@P7_nHCkBmc@&&9 zy8tG62H-hFRhGX#hH9WWsBPFl^i*y`qlKm9L{lVM>+8ado7<@W!m42QW_%TQ@VtfbcZ9rE_$ z$&-ZwGEn0sxuUs{u1vL)B;8y{=Zy1(nnhFa`FR;%Vp=m^JTMlT1}mXKBY~fbKGH1} z_OS2XJ<0a`r*M4aIx7G84Xt{_V@~S}&>LA!{JZ1efI|!}x*bm@YUsd;d!3kM5rXR< zZHMsLwy3@GFr8-k8rsL51g(7+=y}8GG-ynrjB}ET{uZWmu**i0)v*I6JWhetJB}lN z$%0;JafT&}KH-}2d32E8Ml!W#qR6oIJ6%~h9Rp51p+kp$5NcGk3C$ZN*fzxr-r8it zo<$MhU))aTPA`K$*0<41b0};LSHsQem*~IfE5b3uW!e1JYN-9=nk4gGpJc?FSeRgP zhxl%`gmjle-0N|Q*uAQvDM~Tu=qh6;*={Bi);kO33y$O5ReyQW$0Km&u^x>p(t_$& zt4VdStY@lygnVp^A#`*oJ$Y43 zcLWb4i_cDmQ9#%Q8AH*avxK}Jbp{$^gJ5Vu6@B=3jA+!Czi9v32}cW~s3vz$=pK~< zGoHN1m8&lbzPw+8=i3}1eZXMwUFkx4bmLKB<9Tw>`vP@8aG&_jcE+S*+M1yt~# zvm|n54eA}9B3)9DO3L`X!mjL@xZ|EW6dC>!F4cRDJ5C*8i{GUdYqI3;|hK7(rdJtJGExX`zP1au6ai~&T8mbzsKGbiq*aidgG&7hrpP`XVzoNCao zK@IXh$~eCE^YCn$7EJvTNQcchNmX`gW25eUREd91f^*EFLcLzHEBi9)7s+_%YW47w zn?lmB*TT_Hy0~bLjB`{bW3SXLz++DFFns!8JTs1gfQ1$y-(x{dU&NBGF@=(WnsM~9 zj6Kz9@M)fiTv`%3qRKd0rz_vpWHTU_~Z0JhHl46((x;Ii})6eLla=zIda)*QhS zM;$cibqDwB-I!Mt0CKrrSRzuzJ4yfH+HP;q{CkwhN2Np9lv=21bfX592VmTHT~WY> zo6tEo80%vuVSw9Oc#B)%b7QYCLgzZM9jQwzkGBhkJT(DX;FP|5K) zxP1g1S8qYR#~*>TR0rVhIT7bc!KA*r16?;;p;?F_zWaKXoQYILuh%E&=%^fQx;q0u zxDUa7`$psWyecASK0s+kCNZ0`AHqs!!H-STp+z4hc1>4=D!LocT`d$0H)P`uCs&DN zTrWEz{3iYOdH}82F&+i)+tJNYo$el?h^^o9Fm`GT(NmuRedmtBuN~14n)6Ze-E=V4 z>GDJ|qfWBaEsEaU{edRF2*fpe>S_M_C}O$Y2v#Ilph+5v?5c=!ti{Sx6sm6#wq28lbPmGI3vInnmUKz9HmBdy>$eY@Ai@)RTCVi=?_cl zo8bAiWi%nV3rzPWOB!qE!xVZMZ1>)w=}O1I^iZ7ae7%E5`WC~F`>$#6Co5@C{1tSb zUO+eCD=1e}00W=-Fii3fzYT4bbSyatBl_f|!=9$W?^+3z=?}sbcXb#L?IT&-w*?-2 zuELYu^YG2@32>>n1s{%_1y{#Zk;Jd2g@^7FCawbCPx>v9Pa;G;`X5_3R7lqL zBtnMeP&lQR!bj_^g^zKGkQA^Ran3miOk0Se?k_ks?+vW<&4<*zLWtLXf?Y4eP;2FO zcB=duG)w;t`CTUeU${c$Mqf}UTR|tvcrdyFZ(#G}Iy5hvf$GJJ$=Z`o;7a;aC`)s| zI{R{3^jQ-ctNh@4#zq+v`zUUBV+ZP~Z|Riaf6(oiOg^)oyiU#_dcR(d+K-tb+*p>2 zNnh`gK5$Qa{+OVqNVG-?I5JI;hp358T8Bc9$d@PqIh?-hB_y1_NsG?9KBYAb;}3SgA*lSRX7Qg_|+1o$ynmqq5^V# zQS`&$G_cjok&GKO5f*otqTDvwJN%=r=o;xH9b-K4l}$OBuJK8Der7g)bC`h__T3b^ zm5jkna~iIBwNqlI zQ-W)cm14NpWMSdMW?UNcinYr6BdO?VmW(twipj4B3Bwm&mvNjv;G-T3m|V68L-iIx z%b-ajvr7RewGP9L`*#aBM;PPQ&n=YMGaEN&byL9tJ5)|kCUSjw)Jf?Y8E&Qmfm%{> z!1RH{^4u;6wjF`LhsWcQ61h`hLAkgz;N_&hQV(sDOIP`-z3tyS|4 zZokRUP7e~{ast;5a;7KtTt`vbcoI_GP36}7hl5nY;k%48T2%FzEY#K%$t7N=N-Lj` z&Bbl>iD?a34OIrIUVp1H#Gw=YP?pix4f3CWQ8V-)(DD&b{4APc9L;hv3RvMKi+DOJ=! zyUV#?e!Uz`R*yy%6=$dm>Vs{5TX5mZ21%#e5V*CtO%fI#f)`6uNMY6q9C+V>{;BrD zUoAU@pZZf_!fZqAu-BH(i9JsK${L|dnRr?1}SFaqtY#yIJTC=F7<=y7tE;XI)nqe+a*WGn~4rKdxESq z;!Ybti@N&q57?&rI`Pm-2h7SfU-W=nk3&Co!rtoGy(xFOO!ab|gB_>YZu&{F! zzIc0`y@9THF6Jz8o;D9=op?z!X6b;{EGGzcyg<8-e#F&*XTa&ZhnF{a4Vy+Qh%$ej zBU*1?LP=LCIkxK@?UU7uw`ZKik#TvlJ4h4dcbbq{t;5lGv=W@!uOKQC2gBf8QTWOSG!lil>QWZ$`kjSW$Ek@d=Nk?pp={gX< z9V02|oq}33BxJNfrLajB<|lLPS}T$+D{3<}!`UO!Gy<6}yA%x(bw z^u7gUm&)kCzdPxXoh~4EL7A$&Yms;k6_cRn-Vl_r0}3bgOJMRz)LIlpb*8O=e^(xW z*~+ifXrn4RYR^V>p9BcXJt~~P%8gF>TZ4-SZUpUJKIne+5S~8WMCu%uLZ#76QO43j zGPdyt^-@}n-646H^ic|S9%pI!xt};}f-@T4suxZ^pAHH?X5m*QJBgtwp6MQp^nRdQ(Pv<3nq2zbS25NCxTa_aYL zFqs_#n{Rz4SC^f}qTq7+VBSjDxm6KV9v(%9T2(4;EX0a}QCQc!1OK>Gfd4BI9#8Ux zrNI^CYTYLiJuMwahaG?+N3$96`Xf-kK?`4WuERbTRp@a)i;Y{HG5AR--BA(^nq3YvZ)hSk-iX?7sZ$#g>> zUm3qP)f>}FWUTBhVW7KF9~3ozOZsUEnR@;$e5#Aa(SZr{((f_g@Zl`P1*zbweJk;I z))n&k!)U&H(*oH3-yQn6`4znyS%F0c@4-mhQ5bkW1HA3MK`*WsT#Eg`Vs;}Lxp+7x z>o3507V=W}YasOX&LSlv`^Z9lUsSD{j`1ti@o~~W$$%t9csoE58|6}Qt7I%5+Acsg zuAY8sOeXe*Cuz~D!=Tx48GGEjc#Vg13M4l$KeOSY(x?M?%IuGKU|X3QvuW9 z8xF!{miTzm8q#5uj;=?-m_XIhpucP{Mhr}bYsqnN^K3YJ7M~#sC6VYF8ADd36~fa| z*HNqE9%fly!9P1s3&;7sq$yKUkQvuX-rCD*0@WYr!+++gMYSaSbhpI(wGbN{q(bQj zUv#_~3p##Xu%+LKOsOr!{UJ_RYPObq9@vkCK|n7KyTn%2_<)^@k;v|v6Y>h3w7M}4 zu2#>5IUch6Lq(T7jNc9;-sDpki*!&EErtm1LsZwq9fwW62mMo{aYt6Su-K+jSYLUE zD$Hx77v|aFW}6W5!{MK>;Cm|!Ut9n?oj80t-57iL+mN?aLE5eUnf$7EgI(>_fI)^B z4~Aq2lP>JEn++Y}>!i}v1tY$<(eweTL?bK)vYbL8`QA*_sZhZcIdAAH_r;Jf-J1Lj zJTF-?)dmXYn?di`5Zrzs3RJi~sBap8oY^thZ?K6(by(xuqI&YjT%P8B*OkS$IfIe( zh~(*)ov`)$D(s!}3yRh%kcDy!sEX$t{4vi+H0{e_qDBkgSosR z;(kiFF>65L=mff{ZzOA1ouy-SSxM;(eQa-G$-jLZUX+%B^{JH*d%TLwycC0{c{$Nm zi-VA&eHNacw!+nl(?ol}{vbP_Ul-a>jRJ)&Q?but7Cb#00Y3GoslSo4&~#d!@Q6ki zIz1c&;*wc#a2+em?yaPCY4>qWRSZ0QVT;e-H5yF&}MiP zw&?#S9DME!?r5Eg$9W^1!Mq|N9YbKvs1BOzZ;GkUqiO3Nmb6^^BQ)eJ|uB3LQA7GWW1o`@j_eC?u9+q_ERgO zfR~uhbjrM|FleJQITXE$+{tgD543+^)|njA-eFD4>PH|SUm~l!chl-wuZWN08gy#C zMhzZi;DCcgG^gPdWGSwrZI|6(gRMGDz88uA@CkK3x)`s%^TsQdmn5+pqmhII?sN2k z{*Tdc?f7dH&Y2E=10u0!njZcctpeS(BVlRUI;@`*LA@_%!x!%^zR!I)JW?A*o$qdD?FEKD+uQeX~;fK3Mbc5mG0N9qc1WbkM zqASBUVFft_d*%Z5eyM_Ov&UlklV`l&DJ!zN>K6I6DxB{3o(|0tYoKmt$>_%%F@81` zv_JdO9|qbuAV*&0_S=la>+`VujtjNQZ6Nc;r(peW6_L`71!&Z6D7|g+08G{9fTejD z4!aMy()!3oBur}skFQE-WFWw*rg3x_PC~Ed zO?YqfB1qnwNC!#$@N)J;2vgRL$? zA)UQ5M=KC}^O9iSRXOPft@Qw}u8>i!!Qiwmh?taW!e{R% z_{Z-8{t7fiv+1))ve8lOc0PnXs~(bLawGBW6=&GB^@1=hy9G4$5tra;_+@(=3PneO z5z2`kpIL^(B`49O{18r!nNDq-lzCoV8{@upQ3b1Ql9Vetk|M|gvt%h4OMLOjjB--) zx1IhNmnVsM{TY-Ee!@m>9tpC}5O15zqG-!_bmv;>y?S*p$*_l)kKd4xKeg<+ovGx# zQ!|}XG5|Xr-hjt%F}}A=mN6h>QD1dB9^B&d)jMpUNtK~}XYzR(s^-@0MaYcZdMk6Q}K!E$fNy5J0Dmuu*g zve}}GzD4+Mzq}|xzBtsa=Hdg zN_h;b4KIc7waW4Dp}~0bPCM1`K z*wh|OvkbH7mLuv^V|gz2pDD*7E(gifook?FN)$6J;Q}0c6pK^32TCufIzjy1>$J>m zEw1Q34()APxarRU)SLE(oXzpaCpkZGzjO{}OIT7b91ZK{2w{t=9?mXSg{+-AnD2I* zWEb@lHSPQ8cp#H5KfVU{Mj2!9vnSB)eFqD&0y&=P zIAo?R1k685UaiZe@WffTC8>di+2_z*`9*a3Pfbz(nlNF_^B5R0TxL*BRDv)6wn>hr zp1}r-E0EdsgCvF~(tl$-LF=nOZ9EtYxi7QG_u<#@y}d4GO$&liwL@vAo(T+BF@%y4 zmE=`Oi)6Y+4C={$1OHLeh`wz9sRr_;Bjvho3gNeJ%p0oHfByo>{Q>{TTY$ zsaSG7^OeL!el(he9>FcAdm$sWUD$DPmhf5KA^I@27|eQ&XoH^ze4Lmn>DZ`&A$Q8i z;p;yonsN>>_dyvxdtpeP4~~M7ixl1NZw!nU64KR(CrLo(rB~B|OMBaa>RP*nB6c;Fn!VCw}YbO?B=*syt z@k}As99BcL42K7oGx5sx8oU}lL^@G7l-#HGaOJ!P7<|^;QJ=?4yo0`I+Tr<(T z(1-AROeAQII7z&oPsV(O*`!8tkR*uGsbc3zvT<0QutQ@FOfoiy!9_RWb#XE_=<5>i zYkipYY#`X5G=Xbd-wIzxUcx`6Ms(-QBj9o}fqpZ)4L8hQu{tuH_q^(LF!EVMu2q>) zrnnfocRzzkY4x-w@hI4x)e>gfO{9*O+aPj`ozQiOlGM>W9c`EOg8SOvbl)>CbltKD z5(l@^%RMtiYi4y)&59^?*W6B0n;#5+`aHqNX#i0gbQXdwCgFRx4;VD;8d;=Zi+?cJG4EG*Rebn<;$1zz^5E%fX>ud;GFEK$v+ChkJ0WLrOf*1^GM(Zjg^6d60 z8h_UaH}xF_f$v$W+PE5Vr3*yX zSa`D(6T2qR+NWZAWTB9ZI`Wl#iv211Y!2Yy`<0C_8w4shx`ji>w?p6Z-x#B1NbTxo z(>jZ-V0BPll>Nbj)UrqLqvlUiS(uI*@=ql-{xuja&A8PjEEPNksNEdm1z>Rw{G26Hm=9O*4V~g(ybKH4JvCdg+cX~`GADu}@ z_4MNR?-L>4C|vftw!>q%hU&8Zr->*VeuXwmwjA&$`!*lMg7p!2Xvrn=;Q3Z~^*ovE z&}ib)6yFm_Q=~(l&8L~D0y}c{;HjT${#*~BPUFi*2KB<(1eC1HS`V2nmOoYWs9x(Nq0A6{igONQiIUHsJBWD}K zcjl?Y;$j8fP-R5h2l&B{yD9KlMM=8Rryg|vBp}T!mwCb-fXNFx%s;&uie;$8*l%v| zii<<#)wk*5H?^G24maVrlcq9F!3UlMPQxwx;(_t-#z2iV^j}jdS{}a&i8@)3(t8Zg ze-Ob9KY6&ca3k1U)FR%Izd#sq5#E*lg@30zz)^lPo+%hbi{7~jqbn&jW*VsT&wHq^ zbR8nUh{^h&KH%g1Qus2_jog3#jqrgve8#jUB6*sM*PXq=Gjk#A8a56d56Oq!%f^dp z4h%-NKM(yz?*WJP3e?w1p0( z=z?{{LuhJ!HfrDQl5FxCLB2VFbhlh1`L!#PoOnH!T+WyXrWFtAhu`)1P)){6emN4} zZdwTA*IQ%YlU?v~)MaAOlma?RW~BYgRJ#6E1F2``lZ})?)yzvUZ1e!>E58andtaS~ zte!(HTl>gh)kJ*VVM=~oRui=r)#8WjOYk_nhvavyg0}~5k}dZ4CGnvbvD+#PC$-x1 zXZ51Va=}hGBlDg-Tp|KZ&p7lkK0`g)3Q6$Dchs=2O7bH5G+eeGAYE+plMD=wgYv|; zB!1p=$zzLExZzi;q+p!}ejnG1L%#%I@8idKPHQ{PTzrjuICw(h zvCicGT8Xe_NIUtmYboq9Ukdk|d7)y(FM9rt7HL^|g-0$*Z@?)jfrn#TE&l8{8&qwiQ6+vL4~4@n#Z*aYN~#;=xkGse{n( zeIlOD@`a|M_Q>xlB&PbOss3VXc>VJL2&@x@k=m}9+8m48GQVSgXA)2^I(4HU$P~+5sue7fxGh!Y+7C@^SEq6gDe#cl)6IriTm{B3d-o4MbNwX6OgOE zC0xX8qk73JsPW=;kefdr4-RZ6cgx}+zvd!7u;_pt!wrbq*@rZDgBl+FFF;adcN@R% zccH!95{T41PUgiNA_rED!-v7$q@-aCghZrbnr|_ieD$b|4Gx64(L}zFbt45KGr>yW zMt?r>#orr!@kzN5x4ar8(ib+65swx~w1!k*$|h}Gqc;WTUNWbo_Y9vm_8b{rph4#9 zzlJFXZ$PRsgMqP0*dnTcc`fg#&*VOQ{__ou`ENCh>Rka*z5+;>ynyFX6Ct(cA+~z@ z!;Jea{I^0?Y0&Tp)ZOxt=E%Hpcc-SnUh7y$-JL>uK2C%qffMNphX|UdTuc{DT}G#> zzJ;%Q^reo??KI}hOdO6EVJ$gJJn}!2v61iT!J%?EVR|J@y%-J;V+?R?z;|+E*lY|O z@|qm7Od(e{E~RnPJ@DYvEZmnh3cuv+z&jxu!F$hL*fi6KxH6Y8T`LULTon25E6ZLlSzI0kuIJhgsor-QEM5*+K+2|xa&@O`dh>Fbm8NxT14@+5VDXyM$QcuBngwhTG~i{qCH z1NRgO-Sh8LiR(hz@JSo&t{sCFv4a@v-M=X(i2|*kt8qYKG?@Db`E5a*Y)5olAC9_gJ-E_@w7- zso~)Kal|MmhX%bW1^S?m7SGospSEm5^^S0K{Fei@j*-Gd?R=QB73tjm$2iKz0j^C7 z!LKW4V*dU+^o!XSnwPwS%pSOuvZ^z%NqQa9l`c}}sW(Z=w;6DKauT^QAVsEEO@Q#t zlkoQVQ@A+#8{V1_4(F1u5<^ohyw|!`<^h_I2PEm3zxO^U)J4OJy{Axv%p=rkl<3mg zyXdv>9J-3MP_D@h_6(f`ev3meD|rxPm>;1d3fBnh+z;W_VLimY{fFOWr@bkZm zGF|*5VbtT<3h$rr4gaH4;zH@$F(t5k{!6&+Kw)fbD_PUofSz(@^rXo+P+EBt7j+cF zH2u$D6C02BoNRDomzY}qi$dq`R&&Qk_6zSdTRS0?vBWRX{ybmhSYats^kIo`iLjD1F z_ud&u_Q)%C-DPR2u88S~q#lK}R@4R^V?4}y*`I2u?2HX_=zb*$)}U7i71_3P`yV@? zJ@tPXj}u|^tNIzbVa+5>TpNqn@-`;MZ8;ii3S*X?9Q9 zz44>0+j2VuPTr$SK5s()5!X?te=M86RD&LkjzFz$=5`YQy`W}Sg{W!eYwn^M9}!nY zp3$*Ch=TP~nJ?w5QAKz(?L62-Ej&%>h(PnGRsEIj$uCA1ho#VRb_bGZ3ZWV!1U1&@ zpxO5un2(npF|Urn!X^dhmv%+n>UlcW1JQJ z(2a^_m9h0xUNDwl^;qc}OA+IAh_%q)%zirPjgC2Zp;LKfsDJMmieK1-qCJf%aLu7t z*M_n=*H@yN*zIUsRHb#9S|jC|d}28;0J*9)qXS+E%&vK6^v$Z%G`FyhpHTRk^|~&H z0(1hpcZDU?9|2{%!lek7Xil}vYJ<{|FPt>{mYCY?BO2c;KuAkCR4 z$z7*O=2wXVZCMBTsfp$=c}*rNR>()!MIxCS3q{ce>n;2@S8s3&)Wn%be=86hlSl*n zvKZsaG{hb=WWTTLf&XJOe{xC;wLP&6eVzD0wLV^>j|4lQXdkyq<-^TGY|KZxAidbYTlLg_&8+n#?uj z3RJZ2z2LK%i$)TTqgus90={QEyJpjNBpx@FiYz~hY~^&2)~4xH1sI`NeO-k1n4kxV zlW68afquF`fiApRNQ*2A*~a5n;YRf-sL)CR{qxvLxo;ckPKSKDLU%K|?yXNF+V`Q+ za}Lz2xC{PJ5%iOWN0EeBB$Z2j#m=m+U=jy6q20}Q*{K^|(3viy^v)`Q7T+m_i5kx3 zUlMpKd=vp4+hWfqywRoK#7nrt-WSpG@(OD8f@P>+Ub*0lFg01-M9x?@(M&HnT6gai zUD9If$ngqRRi}hr z)ZwANo3+$|&qW8MKoc=xC5X{qZA&xstb% z^816?14S3PJp+8??dV8>tUA52f`f!#4&b@w&*`*mO}cS(KKt1HG^=s1iEf!1i~6Q) zMVU ztGLr9Lt5Ld_n{*`uMmIIkw(W%XKsw$K>RiVZr{EZO^csGFC5E8MeT!VTU#+Y61;={ zNk7cAc28-&eeyUTVS6qwz>q3`z=^G*@3^b3XLb>XJNG;$0F_4oqg{zigwC2SreC*lB9;jMC<7^6c%Gne5z*HOz}K zhUrLqgl>qQKp)F1km(jfJ9ig-bir7hnU^htB%Yd3#XH(|ej8h;_Z!pJxnvUVTjqj1 z#>eT}NnM()FpuT;Ci7={^O=9XM_Dr|mXf-BbkO%18aGd(eLfY;{ceb~`%TeD;YzwG zp@0JZwAP`iawtQCu`|ifMjK{lAS?SR%L^>~{jul0;NpplP)AT_7$EVr(>AYgNb5-n4m!9M19aw|Rz9geZ za|ucjXm{=|eamWlq%VHKSE%yuF8U6!fk? z{OYM@xiI?M`HPm_{=kM`i=#10afmr2PhGFQrNwTNbW`zGR@@8FGC46e**%qwxR}oh z9aKdJW!E4PsE9JyQ*@5YMf&`!FKx62h+j<@_ho{6M(HNgr;&-IT|$xKHzCwYEUEvT z3cAnQ7R?_`LAwN4sbXijAOWS4$?0pLU$1{hL#2jx9loBe|0Vo|@2cJ-kDmY7m(jn` z@#E5HTHR89xA<|!a#)t1Ff$SB;kJ;-H201O}*#YA2Zid;bZaC&!7i+2=vL#d*-5V*Ozg4c=yy}9xUGm$&}l~7LjN$~vvl}tI2CB%_{`QtRU=e}%R>d{elg|U7W7}GEE?H1 zL3XDrMZii|wBQ(?`|bk?-lY_`G^=HatrtkY~i z^w-#ou}h1g*BeFcRw`Q|btsh-n_QYFu_b0586FYkWE9x(|*=R^5a&aQ@9cBNCXk4I_nlXFy6sU9`-kMr;7 zO`(IYVpz`&XKCbfcWTi2oXOq5r#mHG>6Q*DG%;+*)vQZk1bWqzf|AOvBdfR8 zG`#XL0@Dtmrq@RN&8g+g`5`XV6xoSN9p_NRni}dEyqn4yA7|yeW4P0#a#13ufj%#~ zOWwE@^Ka~ZMwk2vKq`(sOs3ctws7@EnK|2$*VZ-V3&vUGQP4trYn1oLk21v=TV znQo()N`6_&RP@!b+3)@`>*7S|qTM>QJH<>eZ+V7%qaCUAqjo0!)d3ox_lJAevKjdu zGN<*a^V!pk4?5ZBg8B?5Xs7;Sx<^cvHGJtzqC?}+-<-WPxN{{PdwogZ$>UOx-_6ZA zZp;5AvYq+0^)>6}aEb1?=ZA)5kJ2P-DZA`#0W?3?1jW5_JUF;Rg-XsYl5wqsld{r=B{ z`LkSs`SI&8V_2q1_hn9HtvJVM{@wGaspB<`=$_p=dyzHD7%D@e>Q2a_4x&3KnN8er zi0=5J$C&o-p%>jZQC1;{;fYnD0H;Fasnbut1g)nAdqimTSOH}QwCT3XIqa6$SS0fN z6~DiFKYCiej_Q4n;IEKXZdJUL!#@|4O}qTJqF@_OG{7z7Ub=sbyT!kks7)7Vl=V;0 zvbcRnIrTnOIv0vw4l7d5db`$=;bOG)#xQbeJ;>-MS@0wE#kk3pVyGk`h#&aHK;Vt2 z<%>jYq@Oa+Az7e^8t+l6`&){d7mLs>cA50tpa#_*nqYmN9H+vzS;$m%2`alQ@a5D^ z(DCy5?8j$I1$=d%fTz_&k%{5xrll|{4YO&j-tn3~%ok=GXWd4J!ctJk6Cajq$6+c| ztdRKorA#xg0m+5lWMv;z$u^|&7sJL60|FHQpnmirP zoNRbSV;!EL<^e@P4HNMEV;8um8{g3JfLhV z(uFS9DrZG>_0fB$y)-Z>9nBP8h`Jv=qRC4x(At(Jlp=frolBWQ@0dn7fqwXLTnUNcUy}KP{sUlh;_Mj9wJ(sDY$j`3U+?fXW84=+N!&s5mznZGXQR zor+dN8#ChQkJi`xbz2AN15`nM_N+vk9DBJ9Ha0Xy(vP)8xXJy2*%z3;t7OLkQ;&R;la`?VHi7tUw>65G(vt=+J{ZwXpA_boeSU&W1X z6X@>$++f{%vY0Y=S9I4?f$qtkf==IqXzLzKLqv{KS({d>H!BbA4LZsmKi|xCQ!_(C zqLwsz_hV*m5@kzE?lY<1W+ElNJ588;dL@>oz1|vH?XcpVpdlHjF*7={`-DEJSBMy<{);D4^6q9rUVn7kkjg z5`|ufLSz5LsSaPX^^`uQM^G>OH()n9D{Lj0`Dkfu~gE^!Yf(G8D(Q)&By4QRY z)Aex|-7A>$IC5Htky< zX05iqU5$SzbGLSq9cZ$l&B7XJMcoZHbGZ+9fTM>Fc^*Rfzo)l0rB$$%1IL+V=Gn}K zi%rzg_#m@NJ`lb9d660Xww@Zt*&)TZDQMz@5Xw5Qj2?9>(tEWwR8G}{mNndD)&=;{ z!;5pNGH(UaU49&e-pyv-UG79`x*t&AhN;M6Z8_hk-HEQdA!OGi7s`xHlSb#C9;Z?Z zi`lZ?JBXti%34&opoyXN$SmFoi8}My0$-`t=Q^pBO>{vik1o(2T7zUH8X3Rn7C{fO zgnnN0nT?I`&NPjfB0?k=uF9oR0T= zPGcZlD|o-gKpfpadkqb~o`Lqni=gp{%~WC62`ZXgL3?K0VE#TgrSgl-xQf<%W@_tE zq_A!iicZ*u>9QUGIHk8t7F&;F>S{)IAe&WZ^X=uv! z-}GhJgkaWah;(I*5t~EVYW{lm&^j>`6I?`}#7MPHNIyjnB0lh2<2$%Fg9SKqN(bxV zz@<8}Sx*nruiZ26udHnsH>a#((ZJrb8o4Xrwv ze{dc1;*&YL^)(sARYg;qmQwVfSd7|qRtdCAf*Gd685FVP0h+zdn}$i1Gs%Cg>2bpW zzC)BFHx#T!YGMDe&8nZ7^_59zR3wFwT35sNoJ>J-i~P`chdMStXqLcdV9d2Yv6QB- z-$#+;4to876x}>>pWE+Pfo$VyQNj8U?x53tv~n;X*(6=zjvPy42Ob2XJx|8j_$`G< zMAww+?5UuJ+h5VIepBh1r+eAb$B`)iau9Oe^G@LB2B^RpAl=aW#H(%tdYF3|6*YRm@vSFnF)8+tKo3JtLNi{i8p()v}8d=eMX z`u=G4Zn7u3x7Cz#4PPMHDkU@~rAmKpFsA0oVYD}Klu6jT8>uY!r>Uo$*@vEuv@%Sl z^-q5$dhdRn%Q}t12QQTEydzpz&3QcL=f3Tz@2PC-M^(YMYkdWEXFX#E%7j@?b`5jP z?TEnl?2g(e-lD+$R&*$Mm?_v0%4%#EV=DqOA%9&38gvh$4GIQ~bNDq{D)x-~d}I%b z`gw-Wo_om#hELLg8#8GDvy59yZqV-uSNWfYb~9JL{bNdhb+db9Pf{zNaFisH&kxTi zVv3%xr>Zx42oIE^JE2<{-=%b-c zc5I^mepJ3Il^#{P&cE^PD!L^Z!49Rbr-hocsGm_R;_AOYtiT^@Lb zGTnl{3bnI0MVt7SmI&KjzV?-|QJqe2(Hy#?@-7Pg?m}%(32;dMB2>Td0Y5uinAx>s zJ~DANpfeqBuvZJ@k%vHsjb7Pu4@dE--Npn)XoWnzlJ_3zr+#9e-_PMbID3nV`Ru1B zS1`2n;#GFHx*HV{PoquX5FPuqkjjUP*ac`C(OYNEu?k)KR6M4RKG^dSS%{ahYu0n2c%V7gMZ}AdjCg52kUw!6}%}_uM>q7+h zcR9T{naTG0R7C~If+2?hNkY;KY((z5nJ#h`*NIQE0l_}I(d z=FDu(9ieFJ#mH9sRB7t7W-baCO{H%|GthLoR(h4GVJF+)G7nDwF6?Y5N4LE>zfMcS1|sc7mLQ#IPos%o{f zk2Hp~J!Vm3Z zGkH6h6-Rd>$-6ZIOkdA7EZ@!kJhz7)eK`#^>{6j)-AmZ9jR8#DsYP_Eb~cq7;!%-o zKxG#!qV0#}x%uJ4?1jDOsC|wIRleEG#yn1@n>*srz4tln!rfb`TXYwiu-L(_kJ-%4 zFnCLDEzCrgVp9<#QiD$DykoZ)MxZu5BXscLRJzdM4Bdb0Hk-HQDpGrIPHX9ATGi4rWA?(FVkR#xn`)pd$n|j{6?Si9AAC?l+ck{Ps}@PSpbB#|>~@+tG}*zJ z9M(s5s+MTJML(0`*UY!y-_1t-3_&GDkI|0Rk5I5sDw0B8%+{HA`D?ZnbEgC<&^5!_ zXfm~y9h|C5k4~OL2^N7|b^nv>m#(LR{Yxfj?%{=K?Pp!PC$DDH9_bRQ?K6)KRY|eF zhv%b-Q=*iAJ{E}=l%N?WHIX8-hn@vDXzZ0yswXeYRao(mzg%nrY28j>E$eDg!29*I zfmg`Xq>I}v3yVT?({gF`_#gfU`3q~DrGslQk!*JRxt=KK!_wmm+N+?{LQdMDMNxm{Jyi0@(q`%k{J zpUrjAds|ViOu06k{%0POzdMkcEF7SgBbD^ubwzYytb$!7-%WR)>*kh7?B{C++=Q0u z0y*D-c*b-=397x^N39Gz=tBb+dg;?qw2sKyh4|+pCQcq56pWdsU(O>fYbDtX&x@>Y z@iumspR!%e%pSoWtOrb*$6w@hKL>eyNJRnK^HHJaBV;jLN^ifGwevo+lj`@cVt34F zK)c@fAgxLxq+x!Qxn8-GR>Xt~IJE+V z(RD|8Xw4i?fiFm%Hj-S#I}Fg4-Y;;~iXL|A*I~xKCJcFq?nIjU(kONLDdsx$L|rqI zsJ){WT9={5Oga0S3awbgTzaI6noFiovDaqr3A$Y`X>lTVnCbAVYGox)@YxXUr4 z@64d+KGbwbv=tw|hdkYXGQ)4}=&5ZMtzV}dMaAp9>3^e7`IWD|=)T4Zru)V(rj1?9 z4yS8V*^Fc~XcCJ~95~5jy-=fFZQsdKOd!!<)_#=YZFn|kyni6`3|P& zR4nb$D`qOx9nfdXcCOD10hd~kieh}n*^a{pky&IgqyGB{I%1$r4W&}i?Zj02WyU-H z9Rc<A+^1fA2bz{BSvXET}yf6pzz7Pj}YA&4r3??qO`+%x3w% zCz$d1Z~3RB%&5YnCCsy*?kp$e0xQGKML+vI*z13-_~ZQ*)GTUh>(KZCu13>-X8o)w z=wh=OpXW4_I{nwkx4!!tC67DN9|G;kKZzQ~8ULc+ehb;%XgrD@U%5+{Ro`OAwno#M zd-+K2_C8c*YC|g~%UC=9TV{`iAw6QelyU=`2z>xdVNyu0ou)-E6S`Fe*LqlrEFb7CcX&YMqkw zX2m|Fjpb>|a573s2xLF+ZsPaq2>89q85FvOux8#x$fm>xb-(z>?w@$W8ocsGPnFzh zi{C9;{%nYOG3NvOaKsxew*N@~-jqV(S90i2odH&4_EFj}pF`aTZqj2%I#G3|Z0qo1 zExXH^y6D9q%k1^|%6Q6Mp=&mEF&k$|Gasz|xDpYWG(CDR|5bk~ZAFAlxDem-={q*do zVt&d}Ddc#ILv1)#+d_Y3U%MfkZi5}_{BUHkZ`TGQq*QH*97${l$TzCd4>o5PWU>gl(jxA!a+UhhSH(Pog@JI;_jN z_2(u}iTjp!O0o;s>1yChid%r9>y(zxk25(+-4DS(Yy)ErImB3RIy@Kfl9T4%#_?8B zBF9ZlVdt+VaNSRYT(f=xy0x-!vFbc>JY)-zJN+K~jDnP$Z#Qg!s&#hcXq`9QOLBmCQmG(cmJi56ikB!maK8EFkUsM~ z`0GS2abIKt-Dd{F`!~#p@S{~w`_6rkpK691H=4lPSy%C#YX{(=-)T^|B8hOALvYEw z`EcemNqGCMBbGVx5vUJ^!hZ|Kz)deVvc}99hCAv&A#puo@9~rO+iC*4PLCq%Qx1?{ zLeX&JuMBcqQJXw3{{rG4jq|h{=5g}emXfuV%7p7O2V%Wmj#=_W(5x;EBYsK}jng(z zOk*W5-X=#pj1=ID2rcq?rVd;Z90qqUHYN5^tD(A@3VgU;1@?4%^PY*7@(y0y1r>=T zQN^>N&!4 zXyZ7AsOx{gPi$vGnMIbQM}7ra`Z$*i_9a63*I6X&)n~l$zAL)*?LknnL=}r^&LFFTU)sifQYClaS-=AWQ^4s#!JhJ@ zGeFZ%oA;%TOA@B!^6aImf$2vTP%)(n=o~yC*z=oBy1BU!$o~hjA})YMQq%DZUN+D$ zp9cT?77k?#9LXNZaCpdl4&huSc*bZm7tHC+?>ewjRJ7wjef!yE6CH& zzNGf#F|dEc7|_M;@Sw5^+}C&>_lYW#@B`Xp{FOaywtbCzxPdSy@)OV*8RDF8O~cde zJHYZ^AAxIBGdQ#6Jcqv=Vfx4lc7EN5yEf0jW>v$uSv&~3PHw{ud-L$%pLHZUcN63u zuz;GwkHNW3)Nv4mDsm zWC^X~cEN)hUqEP#4`GXb;N|bvlDAb>oMj(|!BfsHj?J-EF#CO|t^6GoIIr1&yqu{( zR8$Y*yDQSLO3FNVF-DmT|GE!O6BV+jYAJL-Hl3*0UdDg!8o>9z|KYV`Q6zH9Dl#a~ z2Y#-fIXb(h0;~Nzd@jNP4tuJT#>yDTU$_l^Usj5h7U`1Z?ytdzcy01U$`XF`*C+ZG z%5Yus0{o~x1&BxJk%Z(h5`Qg#JSYeUZmwOx*-!+;m@UOFxY0H}qzIoD?7dZ(h45Lx z=eQX6;ifx_NOSN!+tdOUpIyy`<)4*d-Opige~$sigSxzMH9>Ay%VIdRu?WzO9k|xm zA3l-xgZ4QOfmOmPA}`I5L!AwHE8c~RPQ3&799O~_cEv4&VO`*{>@@PHW*>N0eHkCP zs|Np}Ii&id7U4e`!m7SMIUi4K1-ArQ3|U4`fpg6gvP2;WE=7RH*vehm22;gU!N?oJNZaycr|r#AaR~(6z1r zFQfM1PctTzn)?!_br2ja|}P< zxf9!27?6a?S(!t?Yu=6T3goh(3EWhkP0pU( zPTXcz0(syGS3Zy=`u&%9x&CrQzjQ7$LRR%Og&m*7g zyvWWO^LWD?!LIdLJfNE6PZZbO;Ypu(&TFt4!Jka5!Rckmu+b|V#_R1Pe%*N>U3n&4 z9X}7QKa|bu^U4FeG|Y(W!b#jLwG^5~zQ?iYhu~UsGpOSjN)`xDBYIPY!1$mPsh%*x z?cdw5Q}I_Euv3+^Du~0M(i~hlwv1e@^(FKA-heIYrTA909+@nhOFQB1>+3*?v^4yx=|rk(7sEKa6>#vI6&x_O zAc0$IIHes6iO*;iSY5k{D6U9`*ie`pG**XaJzoL^$GLFN-#>UF>IYUc)gmj7w}5(` zk3dgY1vH#(!_IL{ywZeOIAd!f-uhbxzW28#O&^})_}48wn@}T|xuFSfx#k0v9M{_B zg8ytAqh*N)F96C;D8W%d2nOzR&%@LK!=vd~Q0)wKgCg!PTI-4e|;XPK7#qb>UhoK|k5I0AASr9Be*c z2XdZggM4*a;>WQj62RAXZjd0S_4^A@{qGqL&nX4j2HhOb>1t5dPm6>fT!rb!Enpp~ z2TXMrZ^a!2^1a6qL~R+u){Pn9N#>{^-|r9DvB88iWf_p4v4&7PHOI) z!R3dFap#9$;IzjN+;J!b@SB{VN_Q|39o`5>1k27_%3k9;J|i5{(|XW#wLEM;t_$ae zbYX?PkSG0M6?Ag)!1qN*IZaAyNcOW>xVFX<0Yg-(`GM2>^xSOSogvZqhhQ55`8Ckmv5( zpvQJ$UUYZ`J|f7}ZEpPtA_v3CO4sGY`@kR=S-T7*NF1{jRi6s0m`w0e%?vEFE&*j1 z+JL??fSt>-5O-Viqi_TSA_y#E`kl!O-JzC+qk% zJ5M8D#D9T9%b(ym&CZ-2H(RJ!Kbv?DDZw1oJ-{n+Ecb zUG7UF%4MMW$#~ACw=9s@vJpT3!ZInVV3Z33zYfnP#*2cmVV5J>A7%~jmM+9W72DyGvl-;usZb(h zwGbW@xrSpj5@F-^I?Z0*NV+0Av2s zh)moDs57Z>2+( z`6~EzgAJUQ=|fPs0`%I=fhWplkzC`C;DY&lT-h~?2q}kQqxjaAHGiIid6)|~%sYw| za$G>`%W1G_?pj`Uz&4KYqv-@@ZX$Z~T*>psQk=Cz3|esHNSU?@*z_kItO|<(Em3Kt zu-^{XT);f{4*{@EkU>_fC5osuCMueFZ6?T$Hj z>swA6R^c}VX>g@MI;Sh8AcpY{b#z)E;%U3(e(Ta?&rA?ci&YoqRNk4 zzsTdQov_Y!!qs6OMEh1UT%5In45Wk- zx7@Yx=A{=n;_@kQZHW`~JEesu{7X5%1@G*y-$i8k!7w6VF8R;73ZMNv4VGzs#t%2y z!MfRZ@!<#&SRZPSYaFy;z%CBQTFM%HIMR&oMxNu;u^+(+y^XNBY6Gc{^Mn2V4zNS| z0QA4TlKec8fOEckfQe@hF!yU9+<8HaOm+N=*MBMF6vi$jLI>Z1ME*N)KkOsOd5mnI z>o>Hlu&e_{55_rmO2*{=)0@EQb}}hnm_~rL3v85+Cv!HxJTi*s#nGItVX=YK@3|l{A39 zZS|mQYctQCcLS@r@-ZiiOCBlS#`)Wxg1us%B+)07V5u1Ly(tPNgl;5XEBE6Av(!jW zu{dzrY)-6mJ>lG42;QjKKt!zq;J?T^ER)&HP@23g z|Aimjmm{Ay8bLaA2Ro;Qkwy_`^4i=LqU~N}yZASdUE>8^?e~$cmW@zj!%EKmdQljn z#pi7bTLiW|%_UHFD_OK93Xbv;Nv}aN`A{1}(o(j<((GjNLOc;{_ws_tU@qAOO<{ec zG<@@0NANuv5~Ck0;f|;2a7tGk?3(3Hd=_kh*P4%k+hLN#*P;wJnb_d99;NV2Y##g` zwU1a0rNdt@eaVxC6f&_g1SY->B7+V|Fugh%<{@JeWe^3+zNA6h%48^%>P-B28F1m- zeQ?Ijjj%h=okSV$fNs(8Wcs)tiT)4_ncD5-wbU-oi!?`KZ2Sj&E#5>twRAvd!CI`H z{sWwqipK>BsWAFQ4t!G)L&lF3z}5t7XghT}+^%{Xf6`WfHAx*Hecx4}quGH~-!6e} zk;R;no;TP+LK$wmN%5WG!4`GpVl4YR8~Bf;VBOb|#Q35+iJr9*21@NBC*z7R@5)nb zWakQdcy1)@bd9ZSx+l>-*UppKa|*jB|G|CREx|SaR&ZEF7LQe42N%!!!FThHW4DO) z5a~yeIstCl|7t$W5MY^;HQvN&wI8wgDFmIXzu@pc`^dGBED{j212%t1AbIt1&;rK7 zuajP+W{VR9JLSN#wV7aV-!0r@Dh1&{2N?Q39bUaFM^ulEf_oh)WQ|Fn;GSCzaph7X ztepWHKZ?ToCVO(Rg8}IRyz^Nd!c%vANwmpanB|oWw`=7KYZU8gPyHQG6uP5&ES6!>PBq z#N_3D4(F3U$^3i`%zL>WdaN@f+Wce~cGZQzlZj;cQdxNMM?6_@y#`C{w;;w(F93ny z6OSBN1EmsHLDyzCGAH*tP}n|))K!I&H`b%T`LqN0{N^raaf0BS89C5xXp4`l9s|8+ zt$@=!N%FFy7V~ta;J|)2+h5Eb+_H2I4vu_h`@BM){81L=nV2QvekKXtE*b#)de)Hg zOdW7=nE|Q!9sojaJ?DL?l7O$OwhHsa9HR=N)Duv z4MK;oSfDM;-k<@utZTr3!;?si#13fYlSq``d;<^N&*HrBW*j+u2fvB=h5g(#h}jYf zo|MlrBsSzrh(?wx0Z(fv|9&Eytk+!k@o0N#HyMa(?X=jNWYNXzn_UG|LUMT*t?R)=unANYM{w5fzkTV*@J>x-i`e#CKk-MNX(2iKPHu3&VSxS;? z!{LJLQ~1CeE}7D+1%K)bVV<8HY0%06Pp-`b_p4>0uaqnaH4h-ZLV~Qsau1@gW-)oc z|0jsFt0E?RpbQyUBKEGH+%0}0N;g|EUp`8l2R^a6v8J>qE-r-hktyp)h~GGue4r492*r z5>cs-_*Kjlf@Ubegi}*NsZl79wL&DKe+umVl1oDW?d3f@c?t{ZheG+OSMi0N3h=Fd z4gPIl4pV#MIMU0N8LF5F3sf^x|l?>FE~(=!2= z4yr_6xxa&+Z{x4%z%cl+U zM5j#}-gX_vGvaRIPxA*kMyLiWIB3H2lR+@4HyZEgT><;-?8)>VJ?M0*1kd-(~N>)E{XWO`HX%RwV;&h!sFHRlt(F zI^<@)Ig~4QB9G55g@5+E01EB?SnksmJZRL3+ZI0pLsQ!ELth(Et(*r=CB4HD>mVp~ z_9K?g~C%I39m& zP6DqwV++ys0^ZM=TyJoMEhm)01Ls+guP6Y9$HXa5G4pa6Ks%)+Og#Gsjx zIH{XGowT2{!nL|*aYe#qpmOjGXkHfy-*pOlW>tGw_AC>>=P8k$IYKarH-m)9D3A~O z&SYKGQW);Po(R}7p3qKRk}}=_;#WVzu{XbPDn2ZM+2cMWU-0|t%vpGj`eUBtcreV| zDn?qS*+G#>BQmee4Q77|C(BRCf_8H|uzQaSp^Ybb3XgoSOzdBLsHX+{hkV45GeXIa z<8HPI;p4#!|Q5RaKaMh;59K3vLW0W-f55`CqMlIhC7wWucr~PYhyY7 zWuZ<^HyA>@`PQUGrViA0SP@&_)o?P6_o^9msE&Htz9y5q6>{y5nyk{t~rl0xQlj(e2MQb~vg4G}6DXpog% zc7-&wHyXI-^V};HNlA&KUEfk^h)TPD_xJbndOd&Kd(Zg1C+JH{;R6G+v768Zc^K_S z)7&KSwrkC7g{B#{4P`*KG6(jYZDyMTuc5!*#yI}LJ$g#K7MB^!fQ>s6@Ul$=bk&mK zc}YHw85aWf@xyRbcn~^%KnXg;>4;3v$NGgy_|W+{IC(P|D@}&5p_VEkx$$R!!3&+-S)o*CGn&k;s$nhu@G%JA&RNhFr}fv$ZY1x*jr=$D`{_QsPl zRJmg|w*Gh?eeTakX_=k$Rcj24eVc&xM4m^_0?x40BV=i!gBPCfafvRR>MH6VzW|td z1~A-C8L!;8P&D}NTD^aP4~`tOlfJnxk2ijGz}w`N@sk1-&_App^o*{@&%>hemY2cM zl@*OJ6vbft;EJ zKK<7cM}H5e8@>*pMMJk~YLR4}U04xa_`;H2u-J;eP5y&ibUveR#*s9LUxGi1lW=ER z5LMYbgjW8vz*&o?)7?fKy7nr9wqKnER((Y%cKHKQ3{gT~uKI!_*GIn;6gzswpA^Xt zFGYK1Sz>twHGEJ0B3qYjhK}=1Xilvb2!DX=io#h}O=oaw4#p-~^T0dL7p4j`(pJ?8 zxI9iD-bXB{Ur=rirGK}hQy>1&$(jjpIW-v!E3cv8?{Dg5SKV^_rC|zbr8c#6uTNNIb(wk{oRgrlJ&6RsAZ_Bc>**J zUZ-QCY#r~IX(OkcYR=t#3YD*Yft}Ztb<8sSBlooI7L6V>)_IoIBXfCi8o!jO~1+y z&@8iT+*CXoPrW`FXFI6_Kgta@i3Pn$iO|z3d7Zv?uz{qpt;nzVJKd@E9cas!rG-G=Jhi==i_i#wmHAP+oV{Oq{BO&aSg&Z{Wstksy zr8<^=Y7RXz-l*t?KRuK=5^Jupf$0gOA^K4`jncG6q6%Y7M7C(Y>K^nTHiNgU30k{* zBserEV$&y6K>nT@c!d0*8#ZFpo%;w4-BUxp5|Z?xlp9|D5re_jM{Lj;UpRSZK8{dN zg_zBn5H88m)b@VlS!IUKzL5oIJuSG@VvRR?+@{-u?$HE2Q)vAobk7>w!JG@@QJP=A zqhf$MayB2yZpxnmzc2YhrNHGSR|%|ObTa(*4WQhAXOPRxF?e$O5gO`YhS$wj!u9tQ z>9W~dam#&cjO&)bjXQIM&c!eopZ6b~JwhM$SNq|QR@-rDbt~$Yc_4c2FcC6zf6_9a zV6c#p$Dt=jgXtjxvCE7>={?4O8~x#ir3ozTw8a_qp49M(9bPmt7Y6)n=#@hQ^>>H% zqk$i3bi%&7sNSiRUeA}pC7V{@m(4-=wuuBxZ`(~9RN7FH-59JJ5R3F;dT3H1@4$|c zh1~TaG-`z`Yp!)2q# zixR#HoyYqguomVH_=}kW#0q>~2gf>OoGTEOE2SV?7GsZ24V+c_o2^ydM$5U$bn7^K zNWJ_MCGVL4VW0Kj)Y4qh7{}potY`tYFcA3uT{-m0a1+N*<*vY79FJ$KO@iU6`q<8O z0kqF4z%#yV!crGgsQEWJc%TyxQQze0u-~remPIzwy>5j!JPD?{!TZsXvHI|UB*SMp z?fPxJ8lEDow+X0e>4}c z-g(GEZw2x`e&=*gZD@GLpK{*bITh;9n!+pIwBzg`tO zZuhYj>$2$9?-lH~;qKIaQXD>M`3dP-ZK^l-T0~nj4pY}6Gia`@L3&gF&|5Mp_>j#x zcFwOblypV|BGG8HGFp$FH+K?#)$)@q4K5JP+*wzbAXOqVoq3c-h%>RRu{-virUEgQ z3i#%=0+@GeJ``7;XQVt?|i6mi-P@$ zk#K4A8}?7r4Rmt!c&g+tf-M5qGu6TsR#-km9eK0xu-ZV;gd_J+&Yk6OKFAXiCC8z{ zPxfH`n&8BJmiYc;b&%U=hga)r;j^AEsI}fx^D(R_kEy;n7gLWj|W@ekM9FDOU$Q7I2JHTfJFN)ip@-$sLj9lJPKo`8wFt ze2OMsR)7&oF4#D%8O^skh!XPEpwb`$B3Fci|6nNgZ%c+lgL(LMdkX&LC8pmKqVeO% z0=V`y5sY8X$02?x@WZ_w%^mzghk~R~mz6Ip??|Vb4LcpX>jy<6##Yhqk9FYro?+OH ziKcD8ijY*_1aQ-IWw-B7p$h5+blq1I>|E%8z5WRI$*#EqhOUpE<#}UXPaTwdym0sA zXjm{~HK=c$itmR;!CT2sX!wvW9{iIb+L$do`|uTL=bsU{q>}-su?z8KdtDgt(u3C{ zhGFF!E^yBv4T{D|iH=t)!%X!B=!Nw>tj(%`sUck|BDb&uvES8p#MGlt`^?c?B_`9o^| za9(}H2yc4pY6$h#F=r3X8>DK%yHI*&5mXh9f*r54;rk35{BZ6Kc7;=x$YFUS`t@oH z`^7^D8yNT|a;p1>Hsk>{ef*pjr9Gk_b_!Tb%ryLd+#hyO$ndE?CWEJWszF=*LF^q; zh_|%&(zT09Xs1;uR$ttT9_8(?w@5L7r9mrk>B|&6LF!`NzpKYYhac{MDqUgza9T(g zyf?&Ot>a+qrB3RpF&o=GX+X6({nXZKEV$2=qiL6f{meEE_N4g2!QVRgh9QSu9bAsf zq_$wit|-=2QW2|E?Z&<3+whM^2Sum)tRQy29hj%Rpha)qq6;5YaZ1S+xHM}uq_p&- zf?MHqn7ku|TeBkbi|XJKtA?GNwxAok%&7SkJ6xSCyw9+=blWa}r0ez&b*nuQU73}F zI+ILcP!o%aPK|*NyVOzJ{0!=s(1`wh;_2&{{isIwD4o`)4>w;fMDvfP;Y)W{LAuAH}BNvDXHX@qQgNOH%=yi5yhTQUGb`2S{tJJWX5ShFYD+L6quznsCS*u6s|x zU-rbK>Pt`9CEF_L8JVl}?_7JVx{msHU6aH=8Y00m#~myihfuTn6Sj2tO;mF(1&QuV z$HyF7Xi~5~#)V_?Y25zF*U1o`=E!+_5?+~%;CMwPrm8&7Lu`u#qdR9Q$v2FJq2AL?}eL~|JNO9igq_)nzm zNFdqD51SbTf!5#|nzOh_(Dg~<-+5A?ul9rf8Eg{QTCtwWi?kC5(* zKzu^r5#lcCg7xE6WWRV1E7$(BzA{O`6BJEAH)|A@Kh}#rOgu{~4bP%wad#Z$vUS*t zl{?VfN=8)tZnUt!Jwj^bhFGyuA3|?WfISkjAbvlDi$FKN?mZcZc=db!uh47qDza2v_GjGtjCWc;!afQz*I#3?|8;$*~ zf{twpftgj4!N}7OUG24}rZ;vVi}u54%a7-xm)E;#w|qF8(f$e<%>9p=&sd2EZF1mY zeegH&#fGd zB~4BvWzWm>>?9SD=|mMUYOrm78 ziH;AdC&0jl6ErNofzCc3MjN6|u%E&oir%Z;VAcck7$u?7MhSeKsOX5(dq@pcyHrIoN!DYHrjaLOo!>9ZroG9 z_+bM|d_M}`=!q7oMOuJ;ODReysHLtK)|k|#XAzX=Ck*+s3*ByiBw zMUdUL2#4?Y!h7CL1XY6rv=}GitXL!bqWwDkYgS5QoXv6P+SdtwC=CXQAsp zig?@n%jm@3$AVTsL5P1o$Flp}apH&^>ey?8GwN#Shhk4;_ZjG3y9~$OCw9>zSx3PiW$Wcl5;O5S;ut1eQMc1E;4C=xL*TJ^TmI-%rBY zz55NBmUuu+$Yd1j^dDW@WQ9kX71K{^97KMd<8X+_L3;3pA>B0UG~F6_0;O)gNYm5I z>EP-il&0rHlTI|FoE6(?%WGA~icmXIoH=;qjebY<2K_^mY)ibS{3YflIKZp3!FcKJKD_|sB+;N=~9Lh&e7ExRb{ zEevNzB)X%8oFshHnGQZHo#^XJ+t?mg2`uF=56Ybg*Dmpg{pbDgcOPjqwo4z%)i=@S zH%FkdH5VyAI0qDt%VD3)XzUSw82RezgMG9%?5i`P`r}`r(F94G1D~Yiz!k+p*6mbOoA-=rKVJok!EfkFo8#=T=a#rI z;{&SrSAupsC8M-Mt7!90c?exR2_{_Vr&8U^P~eO)aN*oYn7gzLNoSghxRu(VH>XOJ z75N+WtPc@sU6a5QD$GRb2XBkyyp5^xOF>Ka_$6}Nm|8zM+!kL>v4(YdUSK3}{~CL0 zXk~x~tamiS8Hv7dv)2NzkS=8(#+;x}J$l&lewR@|`$q7VSO9GI7|g62jT<+6VJAj} z8$!oJ|GR5wiIo~Yf=6S$xsT~gy^*-LXcpl3X!vz(B$nLXNfTYx;B#trc#XOeY?|)_ zi#OS$1<$_FyRMe_#*@eNxt~2qznDbVH+-S4@4d0cnPAY^_>`sxD>+s)nB(rl#-jO( zAy9+eaZ>LxDB9UuKl{yCSUG$uzVXu)j%O(0+;xqjH)C8-cjGWH-{emV6>IEYqC<4w z83RYJga}BU6ioH~)j{*OA714;7bh3}LSAbOVANnTB`t#Y=<;f;bo(0Be&{ZW@f?q7 zh#@`w3Fu43K6(KyfO%eOpuK{!O20>mit=8g>ebV+ag#B~%9&$Sl8sjd)v+h;&89t9 zIoeRA1Ga+RwI;&|)h_*m%vC(`WG@lUY)*nVinH(;!&n?(JqO-Ds6gE#*MidA8%T=g zp^ahP=(y%#I>R*%qMas#Ou0F3FWW8p{&Ewvy$qnSBSW$3RS|gV<>JcO4@5-)yC`E2 z!4Ar(fv#f)8vFbQ^|<*7`Ti5M7W*CXR~uH;?&g8-onH>#$+KwdFBhzJRp4>HSz_-% zMDN)u!xdD{R_YevN8e_Gui7MBDz_Xor3h9y`2~IOvjz>>6jazF(+d|oSB+@x14q2#eFD7AOT=>y=b|T3 z-Y^$^M;V@lbg7mI?Ed|s(g!nWTUw3C*FFIzrS3<^o!4T6iIY*mxjn2?<#TGM?toXm zn+n5=cR}Rc9JXK3_Bk%n#fOGgvEe3?c%WC34l0XqvTOiY9}(V>mMva%dJ-J$S`Vf9 zm+9Ww*HPF;mffzPkL}gg&`}YLD%!AXtbd--n4%loM@Z_;}kwo_m?+O(^f*0%ag(VVG4?20;q@J=eam}1e}<2 z97*_>h@{eQB86c@*h53$NtYddsL@ZwHs8?_FQ}7O(mzki|QO8up7&=x@!4)+sARe{_zvz_0 z*6aWs_M}^67VSeN9<G&oPc>Pb26d@ia&%^I`c(-L!sJH%bfoAbM1@5Vd+C zBqR6)1NV(U$2=a0?A-RD+O2Y6;Ag|0Y3Hu^WC2 zr_T2Ig5TvhdVMGk)nx3Z@uhVlt8sErxcQ3+`KPk~U7LsAk!y6Q>me1%NYS#<%Gi0Y zI?$R{BpFjoYtM|tA6802SJ`&x)e3-9I@54l*#NQ}+kD8 zzmF_H(UIDyz^oGLHzY!IN-owL$_DR(0GK6z53PGVnQFGV;J>~Yx?@&SgPX_M?&4qQ zyssW!vUEF*2ymhn*<&E6=#Z#u=UBWV%^G&Eo{E_$b9~*W6UboU|C+%_IC376y8N*im*244O1%cxO8yIDA30a)_ z51r81h)^2f(GNv`j`yNNlR8mFuLQ3_bEra!;MMp#4z#BkV{?;V zq7t*IctI*heFrztu7k>K;Dz0^q~|TFjI#vhxgvD@tU*)NmQdBn0kGqG9;kXo;{8!^ zprDl3-J+)NC*xLfmhD)c49liE;REqg zA*U8Z^YZUeLr)t#|I#yRQ?7xn=J?51O{Y=p`V5&wBnX3^p9<+Up%14Ixe@zU+&Mw{JxRUKT83T0W*(Vz^$Q!X!9%&G$!o=)rlDg!^eC@krk66YK#ri z3fh8K%@%O(4UYIlFu}WPG+?xaEnXks4NksoD4P9Fr(_*wEq>_$UmK3L{Y=Ml>}Cj$ za>B3DP4L~`Uu?tve01ZvJ7|7*&qjQ|=iHm4hWn?cNGB z>_|2a`ez6e%FMC3f-PLr^?(UE!|=NYvgqTMUnqA;D}9{3UUYrn2{n?p!fj1~xZ}`7 z*k~z%3QnkE^;MVX?Vn$fp45Cn(=#3CtyqK_y{E7nu3A8Bd=xg?J`5i;t3(ff2zR*s z6LI#66fAiS!PSLDNc~n$edxn!wCq7qy|?~oDz_vSWZ&mPVCYipdQutk1ih*1v_=}? zJ{c|AG>%?a7Y{jZMIzl-_mNfq7~J-3D*is8i68sjr*F>Zu%=J4@VwPOkXw%j?E9xc zCsb#`_>TcJ$xjmBbF;_mJb8LRz7&NT^|2|71rBYR;33PK1u3~tP(uuX#wp2=+@S=s zYtNv`^RJ;C7oMGeGz3B(D&j2;esJDA0CgJ}V8^Q-RC>(|l#?6-V&4>4vkc+KzntJj z(K<+dJ04C5dwpd0FUR=G1nlFx0>{7Ch1T;ku-v&F;CgWZDE*p>eX%wYZ*#+WFGBT>c)?VPa`c9klzJD-2$;2E*?xJJoElh`hGJ6P!$8BzuxgH)`pBdeN{QDQMv+Ror|-5#N$~MBPWuM$4@C(YK9qP&RTjR9oBPUI%wj9Q}eyeZ5FK zo~k1Y+hc6AVj3(CxFMSP^|RxpENN_hED%o0c+xx3qw#6|b4YcXz~#t$VyRSRNIv&c zq;xg|%eQXA)rLW^alHjL+L46Uj|HS&@Rs6@hebXA0a~`_GW%F5imvLmVB_vDhRDNT zsh9p3d^T_xXf6@>yG<(vPt0WK548emcXgO4%pW#Q%TZFb5iFd&2rMqT!G;Yw_-RHR zRT+@Nn^X;PV$KCrl(~y?LbZg!&XstEh84t=y5jcJMo_-M2>e^eW9Eh?D2h1r*Fh0) zjIf02weEPaV>1JBdZ!-sS`}Zv^5b+EH;F-I zM-*W8iazwpOwb%AOhv}mR*7aE-%7XTYvILdg~+N}8m_Iofh5NQ{Ycir2?tC1KJGVK z5T}9_yq5?%0!zf!Ym^YyCF|+b-RJ%A}5<{566He`^;j@;0Zwrbk7KR}6>VmUJ3WoQJy1 zgKEIQ;+hR)p+wEQ)vSnpM~khK1tsOy9oov(fzEr?nHJx*KTrf4OWlHus6 zgjM*=vugUJ#vX4AX23)Ig(`N2AloHA5VOV_#~(rXfia1LLGH%e?`4sJx39H ztg)5N=sL?^68O+p5yZdUjY3=&;Wd^WR5s%viv3zfEC1bNr{Y)C?S=@1kOvq%s3oe% zSAgnpX$-^1Q*~{1^de^(y{i5MwO15VMU|6|)53y96A#R$%kGYc)4jE*W3>$}tPaJA z%arx7QnWy{tz!HH`4Q;a91JSvnNiPKSz{SJ=M|E6^FwBx+GF_!y2hv*8C9 zqr4s4aJYa~L{|Nym6l?}UHw46M~|XW`JdR~)fEnEYnAZ%(E})O6iVHA^($(?buuZq#&?nv5~tb&As1VR-$hpCTz`bGRMREsD74 z$Hvyx($DW_@9c}jVCVu{%%&v5)}AX&73 z$1OTz;tu+%@;@|rbtCq3g6F<`r^iyw+^^ zH=-D2efEG0bA7S*MpHC5M4Oh1kI-#f@1dwQw@`nfGAOQihm=GE^q)0?MfS@u5qt~k ze|+)b;st0;pP%5x>PItc&eEb;$>?2%D{d0J%!WUw<65o=E!WedMotGyk68-6e#*&k&JW4x(FQk2x#)vgoE1Kk)%{VmBBwm z)7EdJ_U~3gOG!Ij88cYljxHnf<)&!g%i$mqBmt*n{zGcLS5d9yJ5h>CD1G&G6JiRi zapU`O*h@{8s+3e9>*0%WMXC?ZG$C+!NH|yX6Cl!9n3>0aK)8Jdltx8C@wFhhtM)=v zSs|v9Y?I?~!8{6Or+hO=> z`8V3~@;NnCafK~W*J#_?2lQ1#2kl7j7tI^(h(Bh_;k6rNvFu7M;1>xRNv}*KC+NG| z^=6=>v0m`7As1fqk=W?t61Wg*jz?c!f~Tofi+Vmt9HU2UM}x|V^Xx{&d)YZL^^Gcdu5gly*f_xK1UN* zR?LOqAF`kuE$Hw<$HHdGqo{w08d!aKkBr{-(p9A{IAWeAYjuAlJ>nCBHeFjGdY!Al z=A}wQ?B%&Q?}8qCrdCBVc zarwyc_{4%UD7sY_J}y7P=1412_H11}-;$1Qo1Q^kAMc|1!M^ZGq7_MB^}+e8T14k7 zR-$7U+(F{LKEzh~LG6yYxM{rww9C9jf5w`DnUJCK^np9mf0c0Slksrt*jya( z-WDe_ZnS>XNSLFh4PVl#=)C>}Sf{rE|6l*GX-6{NW2K0FjZe`9jsbA!aVJFqvhbhk zZ5m)FWc~bJ3w0};z^z#i_;I~R&1*V7P@s%Xbs3>SVSbl;F&_sUxK5qZR$=yo3S@k_ zLERIyA-k6W$z^M(fs_OMvyOn>hRg8ia!a@j;n@ABBK}e~fn`-qVfCa5_^E;neqG*# zj)gO9yMTE(&SAjwaz6SqXbv6UZqU#D6X9jiMf&9y1CMsDLx$d(Y{1D*bmIG1xFz@m z%tyLGpMe9;U*1B4%~bI5yc!y4q==_5ujqsgztAffqUmp}1ikne@DwoA-D%4BtJHRp zbN)73Y4?jB^DU*pi~rF=&0Lg4{GeoyB92?IleHc1gQJw1*}rLHp>~cZ-n4frD4-a8 zLv0P-Ji!ph>s_P+>f@oufN>!{Ny+46V}hfqjnJJ^d5s<4rIfQ_z^Jis~J{P^c7W2lEsZ%J)uz0w^wAY2CKC? za9qn5=LAKA&x{$^@^>)YsFM+Wiv7brdOHfgwOI)Jw$1~!lso9)#ah(bql z5t$|7240H)O zNQx6oU}usHE-(B^m83=@rm2+rxICygI&6&jdk3l2zG8G~uP0UzYDsS?6q`rmloAc z*h#yljioUQSHPgld>AY~jx70RdgwVv|D0(>*Cz%+vAQ?DHQfZONt*!nn1olP3%KXY zr)XP~2&)v&h4?}poOODQsNrbf;(6Hl@)b4|AERv}cCcLnzwzVaHaca8 zHJ1K33e{N}(zW?Yc-YgMRQjzUUc7Z01PSZSXNe*1`EMt?^V@fHj|qX24QhD8_&i8f zS%b4aYf;tbXVB6V0v_oS#JX-3mora+nILpGyp2iZV;t+4RBjll?6Tse{>@T#vGFMBBlU@+Xy;ad7zsBEV>N<=0{jqL5U7p2%kNb}X z+&RN##8r|B@^ktADF^t_CAWC4--B`Z7{c`!U8-xGWX#)-noAmXRYUXnujE~BJb!R! zGbz$maV{5fd;cxba~8=RWq#-mCm|DZ_vnz>z@%ZT12Dg&f#y)qedN5c8AM+3k0qC&%r((^o;Yr_Z4)8fM)mheTfH+kDS zL@M7&@>ixN@sG9@VqJ@LUjOh1uE0#e3F*i-gd8^D70ePy^M;*FX7VZi-(QCFK1aFC z?LN!Bvf>A)wE z(C3rjgdggYpqNMe?1{xpePJK7>gz=E?AbHM;ZKQpgVGMN=Z2WE+BAn`Jq#qtLKla| zq-n%rwk(&S)#23ZIHDohYZkxnS{2b!JSR?V*XCE3SBhtr-{-$x*v%Vx9wwPVz}&f3 z&3le*Vjdd(V!E8eNWtsT+^e03 zrkv5Xu`R&Ea+y{N$czS97>2ggK^tmRn-JhIB4h<;K=baF({RCL6cKllg0} zFuy#7eoWQ1{1fYgr1e2H^IksJX>o`yIa(IN`|UXCw4r?}XT0P)r{9D5kPWH4%z@kd z)81r$QQmF7=+R>5FmWldy}XWBQtK3Y?$>ZO|0!`{Radx0W8a9s$5oSD>)qUzpE;yM zt%{Gz4dknDW$?*gSw`Wm1zFpdN46W9G`v&G=8a>9@h?svWVR@dAbE1CO!k%&oVC$G zuH3YMlncLqd*#4a>zR=7o)5FM;ho1bGsvVrXt%$7Nc)+P{*l}jhvB~7q`U^ze z-<8a3j3vaZlS!QZgWJ4l5=j}6&A5-+PE@ngoib)^B7Zy!n5vW8c*}wxtw!q3Yr6Y5tK02dW5g-$^xYf$Ore9L@5**UxJD*4 zz8P*>JCbB+R{Zus0rTTzh zrz00f#=UA|j3nBK`J%&4SqanjrSd_vSaJ zOE`ypO6RYZyK@b*qJ_R38%h!j`Qi%B zJOAc0(iZ!evf^)Kwo){2olkRqb;~y^_Aqkag$T6^fqp*hO+bT>~ikX z7)-MLTFJJ=YI0z81sNGJhtIl+dAM%GfNwdoCh`sUFykfXz5f6S4>-)Zu8eRhqlix# zQ_PY4Hpael8P9xPMD|D~6Wi9!OyVuG26nk6DdhoGF;R_ z=F8b8UEWL{zHw>z89tGx2}d~BYp;mT`A5WhYX#|8{(*V*ewkBx#0BEOH4(Tw z53fI9%Geb?cS^B5&*jN~aSGmjk+i>W=Yt;JBDVhu`B9r~ombX4kST?GA?!dE({?z8 z(}?-NZ4)2m)_fYpua?knxZaV$S8xOo+KLirZ2hSu}XB@mypV~X>HxD;l6<4oq5S_*%xL5*)}80LH}R+A4nFHa1S zGl@vQOY##rrto|K+1GW7H?>o6=JHL+_o>cgt-~ZTR8Yy+%#B8>9}w%se7C^^r@nDPVq2pbeBmw`KB2Ob ztlcly@WZE?{EIlj_=eOoEsv}D2t|&6_P3Io@oa$E(IP?G!gR@crk%U)DhM^^v^c5X zc)=-r_u@L`+d28d24c9Y2PY;Ocz#xeJ9#uykE^pQ;-$5QxUM;?$i-aX zwI`iFX{X4mzO55KUA2tqd33;uy(`@y=aERR4bJ0Z zE*J9+zV=M^XjA^q)+6Nd%}nv8qz_C&;yTbneBj(9W@)qsn~Vtvi|Nvi`yZ@tetd#|FM4vV^<6(Ur8d4RB6l5AZVA zjm7Y#gj3a5;x?w5IwxKTB)PpN{FpWgKKrCDsrBk6ht!WT9kSmz`4hoTJ@RY$`#-w4 zsgr8SqVVe^B4j(`kRHXczxDX-E*0eO@eAbmm|U(e?+LRx>lK$7Aawi`%;Z`YuO-J) z5ArtR%gmmd+uS+zTBbP9Lc9$2bCS`mLibu69~U4+p57}b6Vq(Tq5@-P*I}W%b(|4N z3XtK<%ZE7+B2DKcl|m9;EaN=2EI-!l^;8616eRppBEkd zPaLJ)LK>?#GwX%>5)IAb%L0xPGh3lMMrj=3y;T~1$EfqqnKV*-p@V4N80LKa+7Z5e z*go>X+LDA;p5m-~?}}M;kBi)}n#j%^WNz*<<29!bIhk)5LDG{1$I`$3!U)l~WF(sF*+p(*@T{ccW5=&OCWaT`BEEW`iQ8OwKSs}VoP&EkLGS27W| zdYGyGIt^QXn30ty{260~?YzgvIR4hw^W+q>kDnaiNCw9jGbfiGCBDlR^To#1ysCdI z4T?=BxWSIRIeU%KPf&Ea;cCn)DBBR;WD7UsaDdY}*T;3wODAVwKJU;{Eq<(Uh4J}x z*J+#SH>W8>=NZqCZ07FUD?BgZMheD-6Upm~d8xOG&Z~a`S!#;-owH-eB)2?nji(|% z%UP9SroG}@Rx+Gzg|f3oa5pz4Z6OKcYDw>cT0Z@6F4=p&n)~g_au-J}W=08}cxAWM zNgNf4U)N3N5$j^#;$tOCk-~zULkxOr$kksm1{K%!nWO39@QWE39?TC^hR}Ymk``k96 z*L~8?@)JFsD)gF|SY!Ey*_zLY%(+aSnG{L(Z){?|tLZmvnCRf_Zy;vmd#^Gl<{u@t z2EaU1_TW8#$~PqUM)HBf#$or75#km)jXWNyEj~J>nmL&+^xNtz=f|y+X(+mJnSA`} z%9os;PESkLGO|+)gdW^hZrg2d{?VxxGQPu)R7Io_KE6+UZPqUCPjo8JHJS2{%xlQI zg{{JVyqiz4`Nl2SK9120i{l5h5AgM(I^rJ(+?_WZKiBIrUr@Y`-yQbSspI}GrlMDX zSloB=fgbUs@52QqCr-s#H)%AZ_K{^i>klx~^)8S!J1^3ixte4@&*Lp`E0fA7$p#CH z8b&Y9mOltpe3*^}37nTqdaMG-mb64>$|_Tbm!Aw8Os`}T$01qLxbQ5!G*m+V+%t1N z(z}<+{AnaBU%BvcJ(L;W{e_J1p2dvVU&JfRogf1fo^k1g@3`H2-*ScQUUGZX6n=`i z3h{9Xo3AN5cZ}f3;1o-yeg8RF5I2+OkSdwe)+X-he90tlmf<5e+$FE&U-GYR+K~Lk z@(qWoesCS_56IY#uRt2&c(41pTt&|?q2sWTjO!dAbMocMv>qGAaKS|0_U=zcg)Jhp zT^-1=Q|b*%pQ<~bP|af0>#j5Qw?8lo`(`sz{R1L7O@oHNo$L7GtX-tJ?Tyng-&4fw zRw7?4lEz8>ot$&qUxI}GsJ{nJG1q^1@+aS}<4U7tm}JR#J_qyUSGBs+h*A0^)W;6FcB=ksG%^Jmu$FqTQ4y!X;WWFcG5Z+>IWh(iA|eIJkT z51xpHbN>%>=ZhzQVMZBWW(>^ZdR3yEcb7k2aF0CSb%BePzsScMJ-{pdkMXy?hjICn zGx(m7O>VYypZrt_sY1`U1NyW(Klp4d@FZ&SToh$pU*8slqh7UtYk;NDD=C(|KOhIp7VUx`~7*rzls1@F+2^X z-24jRC7)3;E*b__8A$@a_mcU0qUqgVvmr6!2RZD#hva=QK>3G9;Eibw(|PX-sPB48 zR%zZPw@klE%t}Aw##>*=G0P;l{dEOAKXx3|G=fRUb{jloK3-ICF_K%cJqxzKenz|p z$)ob>c?9+u;G1aeDcuvN4`#F6(ez&_tl}De3 zxki;R^5y|(uo7S~`9+?JvS3)AA^2=f<0}j)(Ho)-J}w?GOE8B{j}{@n#0z(;>yk(A z%@QNuM*6%zo_K#4hrOr2Vy)3lobmKE%+b*jrTv$}#w?vp2ln~H(Qz{|f7ecYaJ^9Y zdS#Zd;#WJJm8wq5lcRCnrZ?o@uR|c&XGChWmC$Cij?`Iq5f&;8#G0dnVE_JoT=K<( zuuNq;wNpqZSNf1>Mc#v83s+pIAwlneA2hQ_6SEqQfP=vq659BTx;`_;FP*x0<3|(3 zSiU3^Mz4ov>rL>$sEO`(Fp+Ur)UmHvg^ih@K;<5ef!FiKN)N?Eg7KZjuxaQQBBMZo z_pagCZ-0%Z27dvCeirAo?(g+uQ{(Rjsp+|^M-g!@8q z%ndu>iw;N>t@Pko`DHL|bA`fN31I%`FzLv4rr*{Yi?V~7X!PY>bjh&uv}B}`G;+~q zvN$Ch!b60#p(cVPD9hoXefA=LcexB-e-o#$3PfwF5@vbHZxf`Aa%seO{Sh2Je;%qo=%ovfMWWZBIC9ZbU21agI2C`=z=hgX7v`-%8dRkF0l)z%XL^$nLD$KpEFS?W}g%ty9Na?Kt;ZVagXuRwS zqn;iGHbjHWTpt1^T~^4)nG=)n6Yy(M6!Dt>LgLgNP6ywp$Lo!+`Gj%7bi&r#5PT&M zuD6CkZB#!bmK?;dHi2+!@;MxSuaPdeJQixFKf?4xTiCbgEmpcup;@w=TkYFa7~i=Y z#b4fXj`16CJgKDvyoR7KI*#MfID@)(9}mci8eEMXhij?lVG6Xuh+Q{9fpQ+uX z^Af+mkHGxbP6#;aODxUj&`gm!z8bcQ#&Nm0QWOv4YDQsc^G1ArBot0jZCMSHKr?0Z z-aDaDxFWrcto6wza`v}{uJ>)xTxfOU|RPz#Pnd_kC(1%KDb&g0}j z(X%DG^wDHfTrp-LTzcXH$GhynMk}A{#>CUnE=lxQf-y=zI8pjt0UK&-!Rg;K>eAXn zRI~Px9WRfAbg>h3-8>5N+s8wLydLW*pc2n$Ot~cMSs!7* zBgdUApJfh@9c}Q;0!O^DZz`0%y+Z^hXQ0j550)Hshl5Rz(Q#)HuAf#Ud3JXa?sAsH zfnhVTVD3VZ%htcJe)K)6Rc=NOj$8*7mqx?=4ICEk(i3ex8I23OBj}hB84}sENh>Se zAa&b5$oqX9g+C8Nf42+@mT4(6I;DlfG7lrP)}y`qS@iwv4o_VP+*?_X-Q|TO_QDjX z9@mU{+6!RQ;$To8F2@A8N7J5fWvKcv124)SrcqO~>4aBdAo_g++N|fwYFj;N4`>x_Nbf$$f!kZlVPmZVx(%+x^PYxMhf+1THDok- zTKZm?{Pe5vz{5yTJvN)zecXa6-2!-NkRao1FBB>^{N#r>bPIbU%85*r4u{w@ve?uV z9&|1tN+Z?LBYdY2_i)s9;e2Qub&!slnnka?{6v2}aY6I{>R{(O8Eg6&i(yTGH!^-p zPVSfr`j7t6&uJ5-qZYIh_O}vRHy?#_O2g>&{tBuw#hog=>LG>+3&E}{AKrE6fX731 z>DBwH(#Y9WSZLsZH#A(R{_PboHETK?8a@Wv8^?mrW>4JwyBk!$8%j5HUBs&PLYgT} zq#^yqe4ENMIR4}krk3{zO`i@CUGmqJUc7V}j0LlCjL~P|i@R?m8CLgDX>J#JyU_-T zERX$ft~2h@Gsf27@e=OYPYPeI(=)OAVUyQljB!vUeY(9u4;feWmy05Hx*E_o>Q8}u z_yhYs)x+fB=I|ZzAnfo|x;F4Asoz~jzwVoZD@PupA=fjZUH7m|sP2R*zaK)~@P+Vv zlpYG+%z;X&3l4~gfEt}Y*sN_wlJbyDXA>o{Dow&M2I2TYc!loTE`aOx`4UFK0?p3r zQlX?DT=jWSvgt+bnYn0{FOQ$sjDwj47r}6QKHtB*1wCCqp#987@T!i$mh@ui$lrk@ zuS_CO_MO1Y0%c+!cooc!$>wK2_0ju#5!|1XBr(4?4~V`5-Sf}Tpo;ydJVb;qZhXRb zj#IEhEdxHhNTqsDN8qc?u0k8pX{zsjl785dhrRXFMAx#fVux%FBDBvVx#ttf>gyGl z(Q%CYTEIe4loqWDh=DirlX2~w7l6!Hq17B!Y&x-%P$f06n0ts?Et-#EuNBdG#|aux zD8Y+$meM0BsS->03v)vTK+8a(G(%1c3k#l;xJ!S?)5R0W?MYv`t7~?n*=sel(Cne* zW!}Va^98)J{1UBeyG>@8Xu*nlOBh`{SvuXxhUj!16vicQg}bW5fy4%quRktefp<7W zX7>`m@5dlccLTk;&I-%p_X>5iYM`cm6--vUgq8$O(*NH%I=Vqk^q{el-mPh5 zuT9Y>-Dl;bzek^fj%O9n?LH7rPr8YjCU4288_ke^sR0uCEPR^t1Gl@WW8?k``a3n8 z=6t$<@7pr*bLBu$OG_@=AdMK>OExoWg}e2)5OVW`2`yYSll@?vTSpo;ot|vM(v(d}yg+$J{94GD>Mq@e>&;T_tBIhVP z>ga>K<}0XUnL&5woa4?7n~mXzjltN;2%Wzw;-;h+$PErB%l2--A-t+oU)i1h={g4+ zdlTr~iOVE!lkMSd2nY9CzmQulYoKK6UOYGSDAr8XMT^BQ@I+}id1iEo5%SbiP51^G;Gd2;?fZY_xEij zJt2zHeea)OzpSp*g=8r3sF9R~ccRPYSaRrZBwcfUmaNy}0jr)K!N2pAvG&LWoS?dg zJRLcm^w0H!XPz&}ld^JL+S(~J9`lI`0uP|p4J8m%mmpU>8J9l1Og@L)rUnX9nmJ`T z{aU#RZ!Q{wH`9O7g`Nqp#z&7X?7jtBsflEd+F}~+xSciswwLC_j08=_P`c3|Qes`Z zOyZII8XRSD`s7^)SUlAT@7cUXebrb@Kb1l5JUT+UckIH-2Vz=Eui!$t6{!4;(x2emq0iHaj0<-RVKxgs+ zd^^V-A0#|Mfle6U40Y&SUMD=(cAacnn@fAP9~Zi2UL#*Rb5Zn4OJvnR>B`if_{Ccd zo>%J#uYWxO_g=lB@f)s_i%VD0;8;RRoBc7$@ane=8WxmIrW+jxgHOeh#tCQtAI9O& zPH#Hjww&fT)WW|?CzNOv!@(u8-}EP3a;C~mblfc&GUtU$ytLaSziPI@Ono_#x7r!t zhQ5UfQ5(tkg(`UA&K~^akRu7LxDRG}&9q6JjayFtgh2=HN{*>l!ZEqgbVEoY{F_$` zlRXM~tBae+>+8B8UguA}N|)j0Q`5of=3$8YB%xn_{vt~~qj1J%PV&%6S#&Tuk4)Ux zNA>ERLG4)_e{a`mtXh7ItO`_~Ow_tRjrd)NAb!^~ghAQbkh1f?>c@EPJ1{ zGc!yed!f~U5ke1)}`cjXAiJTwu7zB&e%l1G5G z?qu*Pav=s%+3f#(2Q1kxh0w02wBQA@hHr1+@_$!J7cNC^@KE?rGK;tNWP}cfa(IpB z-ym^y0iHb{1;@3+@pbqB>7Tp-u%Y53tWtUbopIOcy6OS&dG}rX=NF9|5@mCu|Kv#I zg%U^^{)wa(8S_S_#N8=Y*w?Z$k6D?_`uy2Y$Y7K{}Q1 zf^9m0%Dv-IzTcR9jhTcUzZc`$Pxt8RL^;uqiHLr~HNn_8joMteK=TU!(X-{6z)tDG zbKN#LLh&6rP@gVqkm*B4&|j*#(GfZ{Gtfch7`#t(mZZic;*g5pBnu39^d*%a`5mfW%!IQx({cX#=V+KAgn<Jh)+OK;(LhnqSta;rIbNuLR|8Mi zk3xRR5FEejEZtt$Mq-zzVS4F0q03U)`+0<5==75$a)Ul9e)2}QC*5?xoWZd2SOIAa zt|Qqtv+zsgVu^cpIYb^;M4{$Y-2FP1ro5j4T8eo zy=3C^CgIUd)iAMrl4#zFM0hIG5o9;kgKN(pJUfD-pHDxe)FtAX6TReNfj7}zUIa_p z|5A%RZ(uLz0vqoNOH0$JGSfrlf5`e1-;Uro-#N5TGap8kP6CU<^JKMTH(r?<3c41H z@Vu|KD6V~=^kwrM&{-D`!-=YN+B`-4x#=!Z*PRHCljfk_OQ= z5IrlO`mc$E-JfS-1Al@pF+44i3wjEhZRI`ntg)YuqpxLA8^dD4Jsd9DF?A>?{x@41B!2=_ z1t*EU(FB}TdIhbP9fwu%30Uc`grDO6;G`^!ZL;Drs@s)PsqrH$S2bW7O zXBOhrV~fy>UxrO<{pmr)63WFNL1&dcFpR!{+b4I?;?wIeRp~SM*zUlN$WiEPw3I&g zS4QuCADC=5QgkBm6)FC%hmH;!W^P zdm7lz+J<6-EcpCO6Dy5^$tmw(I`%)Q&?53Utr}85YJXNrG~OzMWWZTSDUQRL2GvN} zBzU>EQ5b2LN3#rLVX7k%Np&>zdYEwES{-42S|O}3eL*8GCg8^vuAng@2l^ax(C6o6 zoWV;;Vclueec+3{-4FV{M-y{q&me<$UclepE*R9=0K0#`7q3h^!X26L5KQiB>zbL2HEnLV+(ODh?_%TNJ>AMqJ4r$XPrmMe>U(kHH!qZ3_H*709t9!0{xY- zczDK8^6}Xe_*m#m`!!r;#t2hlUET~@8~#bEz1r~hWN-NAb`@j>;C7N2=?nl+Q zDdhID6Y%a)7HHn^!CfQbB=M`9p=su26f9Jt8mFq*Zh<=8y+u~b*@R)!*$jFnJ%hw8 z&ZluNT8L9Z0pDi&jT+p{!5`KQWForY`K|M*|Jc1?KKTVneR&nl1U$|Sll9?GDav}Z zBfwSe5e+Oo3{#(u6E^m^mEdZf5kzZb8m${d;NZ3_wBFhh6mkLvYLSXyO6B31(`!LYu3ijEa1oPjSzzn%Av0bKtv!sJXi;n+7dHyf< zg}wr-wL2j8dn~L`GX(eLZm92}2a_uO(Pdx@b+f-lEDGe2o3aMGMHf)4I-FK~8!efl zJPY?kw$QL00p$7Ec1in6ZRx-Din#KCAKtCM3w6aK`7pIGGVf&(8Cn!ioR6NMcNF(Y zY(|uz%_C>pId}nFNH~ZINow?j?MFgn?_XtQBr(oDBWW2h2xDI;VV&1SG@XmG+Q$&j ztvC-$mCs{Sm<^mh`Wo{z$KtHKS7flwBPw@b0d3~TqJMoL{TwXoaeJAPvz!gY{TvP} zek(%imM8SLgC`8x@|VB>d3+l<0&eV-&0OXyL1Fh5;a8QtWR&j#l}l89#)m)ItRLIW&GtG8%h0pyD$zse5`2V=la=1u7fq*Q`^dIa4ZeSvn0P zd>kN{m||#b5|IsE;iCBf=(XR5QIpJI^}G3WO8in>9@|MK>098~{Rioqq#?p{zaN9j zg@w>N(+cLw;`ib)O3>#T3S&jBc=O6v8VBa|WS=tz{y2bX8?5Q=Ku>u7FPMHY>nC?w z(&;ZP4e)fXB5^`9Ee|8Ob{2Ys_i6Mj$9;M3Kp>+4ZIk^8Ik-6+B&P=K!anpvu z=1Z^W&1Wf++;Jr|Hgyu3JiA06j=f4eO_t!r=_&M)hCURZyNZt*|AE&pAE?jygqJ0O zp#1D6xQ?@h&c2Q0>dVRSwnI@=t=@#c9q#eLgG+H$oUXJ@l#379Kah8H40z8!4@Hk> zKy;)pSWVBCopUM;|ENgjJS@a>;{9k7H-H*nO+)TQiR6jwxs>O~VbP!+q-dcwhwaRS`(QNr2{9%wZ4I&qkBjbwiN1ohdau;xb~ zxV?^s$1U?Prv5rieWiXMEl1e8_yDL@ZpY6>*D$b`huE?i=#_LE3YJd9MUJmX z^1mcF5cQLWdc}iM!fLdeaRkmzYsWW3&kFxt2*GAkAr1MWD>UC81N&=a`aS76xc#6Q zEamp$`GW?cd#O&OGd-F1CS}rZ?~h<}g*>8$HcN?a`y;@-pIG`@imfR!|HAs0w9n}*ZuPOoo2;62 zz42!7%)dZG?pi~7;u?5t#L)}ts;Dd%N(PxE;XiF9(b;?zym<64{dA>+HoY(sompp! zL6U{^R`Ee7s?nk+Uhc=D-D1h_6LX<)mnVwLoH3}hf+%J9(4jS#>DziGc>UrGKz{(Z zP4%QDv0B&_We9i_>yJ|N z!}~FB#7KNMA+^VUs?d;Wi^cR zkKvHkxtf+P83YqLIh^z{3S$@bla9YeG&txQeD!>R8=D@&x5oL@K<+ZiO^XN7fmagS z6D};dIU9%0Y=FB3xv1fzjXOl=iS6$ss*@$6b5x+jF-UucXG5zt&!-cjK%z9 zbv*kcQDPaACS+&X@Lx&>LI3JVsKQ1Z#>A3|TjbDd22c0QmXhZ8rL<(+0L-qkgVyiv zP^&i{Vp=ximi7~prsz}jWr{Ut7!XeeS^l9Jqo=`&4@IB>X>cxR9zE)(PitnqMRV6K z)EmAHwIbhh%YA%dLcBGcG1ZgK^f-hHzlM;$)9R=s%>*}F9T*LjaNIA9{E{w^>7Fgf zU)?xN$(TrT_WR4@cBPZzzv<_UlV$U7td-A6Stm)pIfMXdW6iCa{6%0%l99u_k6cL``%d#tjxI zNth_C{#=DaV?AMkq)zx|zwC}&Y(sbaHx_cFx}eA$1LAQV@Tm%R9qXl8WIhTH*5J8O zFG!N|U}EW$hwHM=67i^6Sg0-2Zl1Uc+rHX>g~=XrdC6*!WFLgHB@}x8s7Ze|EWnR_ z$Jn2u(RBC-Wzks4UCEy+nT|>-4@uTpq^P*;pM4m)t%bY{izly}6eQcbZeqceN@zN~85hoKgDp12 z!nLMP$=+85@Tf;bjGkMGG9IYHo;&_HE7AaVPZr~!HG%ly);REPyNT82M`^qM4dEu= zG%WN^CB{yBq4dHw8dl&$zZ{r|)e(n4XLcO1bUT7+(vkE~x+if7^T&h1dr9Qg2;p=y zOS-kw2cloKLGbeuvQ2iDPc(1glBxtr!_cc>aN7WP`6m!tZGVw+$s)34xhYO_O_Z1^ zn4|4u6P#>12cPO4hb`@P7;~^isP~|ho@(|)>G%_*KeAXD>>3YqWizUrQAXtJd=*iY zsXQ5|JPse)4a3qcdZPc*9fa+nyJ_p$a>+)yLK?6s5A_%Ytc!_5rQk1=m3bUyD;vsY zRD>)Y){0EeE;3{3H!Rz0ftK@*QRfs{pYfnjC|NNXV$0j0EKrT~JZ&Qz8+YNpPX=H; z=nM%FR&d`OE>rHyDr&f74o%{NAXYZ@Up-;4Xw+mQ={L(L)LFNldO6HQfBppl;bz$J zu7~*UpDP(N)e17-OvAD#mq^qb0n|KK6G=OiP%g0w`2PP^uyGtyPfZ{nK#E*ID@9{Utn*x52>= zDcqRhMAV0tf&HoV;2&X0%W5s@vjBHA8RHHmSxK}=Kc5V>trhmk-0`|*mxMi{III_E zl2cPMusi#_WOJ(+)>oI&2Y+SS6x@leL#oKot!Cugp3l${a~Q3?M}w+IAzc4_81tt- z0hQW5QXhRB?_bTsmPLdv4{sxpR(t8=ptI9eOJIH1& zK$qHb;xaw}`|NcgYhWNW`3{GDW24DV@AEiZM;i=+JLs&M_4MQmp3KsyrV*o7g5mB5 zbZ%uls#c0{Rqj;sex?&S6gCTg#T!c$?*Ab@@q1{Jh84IL&HzS|5BFqVpII$(xZJQG zelASG5~mncasJ6K6E1`sc1^V6WCMgT57zz+&tUx`)HOWg2LYd6e0z2?2+!VSaoX)G)80 zN++5ebQl7WkG9}Dj}^qx$X+yi_dDV>-3WG=-dOQ>J@7c}|$s+zBDML}=pl7WGC(jioF7tw~=f<;*i?4z5)T0Ia@ z0Py`usrxeuLocYp+8sl2e|08Yk6%Irt;dC$vBhN0I}5t#!)>}Mje+1#qhUjXI%aL0 zCONjOEI8{o>`Uxc*$h2q-bnA=-PR**b+GlU~@dNIT+`wV+p%|_&Nn9B5RGjT%KZ(1$N zLgtvG(2yJymQ{SD>@0UUH8q{Tll@3IwR9B>dN0%ah6m8C+srV^Q2{?l4r06AMm!sk zjQ%wy_-w!pYWr_0o%^;9O82Xx%Nq;P_%HV)KOR&{)?MqOcgS0EaNkif!7rA%>gNRS z0t@g)Od-7y`3OUs;)u($heGqm&1Aow5A>{QBvP+nGV7xcUGfjf(6KA1TigrMsg;N} zZhsC+mc2cHOsGn@6m)qM;&{Wy(%0ef-%@3SDd z6oG~NF8UGcwOzJ*}>`A*HQFv~k20B5BwNc1K5}fw4LaBsD_i+6e4-x_~B^ zA3&{AIq2@v6-8f>Bgf6|5dDgmcwAQT`-gYHIaD?Ep4oL8rcXB>K=YPApOCT7sdf&{90uA5}eQG!p|df41~n>uI$BZ}4r6y$Vjt^G19KnZZ_hIHxV|20 zby9?0ZL8rXE&x&f5vUib;*)8oNWZ2QY$;5Eq0xthTTaEZCs$aKPen6u$&@|R+WIoQ zZ^=Z3y!(=X@mEk+vnpf;y12#tM=^1B_#8k1yD z)lMNlsxIJy;ZNwg^xIf_YaM;E`!)R6=0L6v7og<_dFeA>MRZCnrQwTXWasq=w!03Z zo$r2<-P_KP{X4EuBb{=XzhoTLB>O^%VmRh!8v zVYIv~4&~}#R7;*@Gz#HTOfs(d(nbQ4(%|{YZhE`O5t3RMh+Ti4T3JT3#}0ff zX?h|DGml5eyb-y$X0Ekp(F4FL)rIu&Ljw2LMxlA;G2swBH@@4}7(AXIhgV}{UBQq{ z^6O%vgh&jb&0L;bnw<`9lg>)U?tBcv9s)FL*QZWhm+0sT-88lC0jv*yMm`LXx%49B zq~%$kuv%Xw^y;BAxbmAfOvbvk?LJ~=o z=~BMTYA42P>)`j^Nsz)1A;)x{Av@>@Pz9N$r=w60kiW5{}#-gx_Yi zV9(1UZ23D7``2m_(|Zd+`Q1FIyx@dNi#ft{DMP*TAZfkpM4Dy1Uec~NkzV9up|fNy zy}09waR2sdQnplr*_vWpUz7t=-~A$CM+akZpCOH1>yPp4o508479RwSgY2)1fJyi+ z@tJ%Qo<7c%y!2j-ro*Zr{g@?gys-jq_&I}$ktxir6Vsw=w&WO;kQ@sI(S|uBhcDjY z_AG06NZWBlp#LRNa56uWX&<>joPO(hwaVDroO;4f&U5!?L6Ad>I4?SnvGE{mU%?qJ zbG|MYzw|FRR|fvK_EY&NBzWolR>Pziskd%Y782xn5>=da1A*GCznv_Pi2Q z2+Y|2d(+uZ3R}5L3bVLvvFjLzZ3*1fw$aSrc^+cDN- zf_J)~tB>i%azSG)`SIRmOukc)Ab;`)&V*gYOBpdUcl=|=Wd9;&W-i12OY-E_98O?D zecc_qy|*w+d+eA=W1kD`+cG&*WgXsVXp%tvWv-xohKRX##)aK@?lRY~Lsihxca*W# zzr+bxnPtGk$Z_EK#awr&J)aSFnz=MDN^EOCh$;6z!SpQJ#r(=T&16J1ROub^6>C-c z3$`wt%Q-YZ6m}I4esgH%M;iZ#YSYW;QcJ)E-bJawCfZ18*&2ufztl^co zc2GQbeXTaYYYylho_Bwaqa-L)Ly;n@o#f9wPAPKi8#M3d!AeB4#XCd!1@)y&o zHeP&e+E8x(gN0n!@(xCEp}BaqFq&KO+f=qU#*sN~DkoSw@v*?ouZ+1hU>0X-_Od$t zLkx576vJjjr*S8GQkbh%hZ&7lc{VEbrC9ID2FHb~&T+|Rp8VidwM+}lW?toQVbp&Q zV0Q&s2xe=9awU>`Ri(4PF&)l<+{EbC>Ju9lFweAP7GKqqoG@h=6aBA8P+s|*5e}>u zf4P&vjd`2Q+>^2KW=9r_Cn{Som3bZF+Mi<1YJUuKdDl?xSVk(-qsg+zHBWJ6-@^s} z#%MA*iYB};Q=f5lTE=X2F<~BbJr|pP|KVu$`Y<>5aR|5R^CnL1WHe{p7{}R$?&KEO zjpVa89A)mzwPphOZQN=(+0IQZ1@`oqVD62nJipebo2i^LmAQU7U0nBgnLyXZoPG9Z zhWNx*2S#qo7UmB8V?G$1=Pn263YL6+U&qPU%7%6cex{KYWzzdb#_&uJm(!zA~;eV$Ba#z$-Bw9F!@zl z;$>O|%(R))*;VE>T&7+oGvcHgn-zOOY@o85+Yk`P?W`Tf7bA+}ie$6|`QPFkK8zbO zaS(gl&V*C=8_oUwSIlY7m+=djaOV5N&0O2LY)x?r?`5Z(p6sWT(&P zOi$kB#@+rcK78M_+QBbg@SrQcT3F=7zbW9DEnhmAH)mHe$qRLO2mia(r>n{wpR4`H z^bgZzjXnFgtQsS3&9r)PVU#lSZrE|Ad)^4v%3wCz zlB~pMUNd3p6pYyE<_hddK`(bQ$&I<5mm=fLPUM?4@&uDCmoTX_O1R*TGVaNnOy-Pg zllWuMz-q_MH@N*l7X+?JIRdK(IgEIU?EUVq=brcPXMR84fz@5+p5aKr`pjIx*tRoFmcKI_ z5Zc3;lpYdScT)Re}AV%As35b8dB%iY8_B7NxuJJ2p zMpb!;H#}8zY+t#HxgTE24B1~QNZxUV+vK~RD={9;dpmgZHs9pghTUG`K=&Qom$5eN zpiF!5y!l(X9e*enm7~NT-@Sy{{ITA_{LTkXaib^Cu2o^rr_W*zYv(Wv&D{7gvqa3N zN+YJ^W)Tx4BNKHQ$`uqrpx*roy@4uV0o#cgj@N3 z4!6@Z)Nx^%5m2I7@j0&$lt z27M{66=yFV%BbJFEtvA}AhY&t1@|Q@mb)kHVod5IIio}On1?3}`Rw2XF3?)WQyl2b zMim(G*TyOGeZte>?X|jCN%2aRyrg2Rd34Q`zp^%@1}7Z+O)WLgD0>)@W(Om z^K{1ZU2*kw1556ja*bd>&m}HYIG?#e?g*?`&0`kc{~@;i6TwYSFXKeRlvu%40kf?p zfmtsa&F?<)i5VSR!Pt9F;Eu4P`J6`w1*dM@X6#>$;!+y3xb^{K+5Sok_E~+$l<4cg z2fF^`-kuu8XWerV*wqYUhxSZitAmHK>junWg8P9#J?IoOZ&xy-7iqv{#>I0Z&+D^Z zi_O^3pa{qBy-vJwyBRNBZOQ-Y+`_Ft6Uc^1MsnI;?U;v~>;)}8^-Qv+J>NIMj<;}| z!CkQNX6vR!bDKAu7yMhhlv6pS#CpXKX4ADMuqVUqs`J**;W2q6zipfW@2viXJNMj~ zz5KgY?5Nu(2$Pu(OUFMKJXkqNFsW0Ytr)P3JM!kFxHlx76Mj6(Nlp|CDjyCNOjvK*t#8J0;C7&weM*i3{ zulnR88|J#ZCi`RcaCY^eR8Au^hHKomoK0}EWABZKWZoqFWv0AY$)3IwF58Qy$Jqw| z5$HZb&Nrt*Kx4Kr=_%$K)J`WrXyP`N*&F?bU*>#8C*qg2RV`aYQ*o!P^vot-Y` zyna;Ml?~-T%-hQNW;zrr7?6fTcHg2FI zYj>!OIWKi$Ib#`HGU^D|=jg=`y4uT)(w@h)X$<79-!6G&53)RPBE9Je&)<|2Jk;!dFJ!ZZt+4-NB(qYF6Zyy&KQ-9=MNN^vp)`x zW$xWk5o{^n&C>o;oUrCOBQ2K2o^@9I=4&kPr4!Da9=n>Gp|Oe`PW4 zq4Iotpf!8&!FfjIr?%sF`v6w7Op|Yy{;KZ#Hi-FhJBUB&uF8(I&0@;>OBjzbFV>@R z4i{`5!PFbwWcr_&@b@2=bE%dSdAIccxJ>4;K#3UfxVVbz61)=!7rOBSmL20_^}aLv zo@X$#qStf1*UweO%9xD7Wv97!QyZAyJ1UvJntzP=v>CtZdl94kB3ta!*u$0InaOG; z{oKiTL7kam?JVS6pAs2-e5j zk@vmUDxM~~&v}j8AUIs|kz2T8IP=Q)uwZA}2!3h7H1^p=ZL!AjC~jDpIjfmAfY+Nj zNNia$M*Mq5BjZw{Ba1y!X0q{F=8D-^e#3FuUF*`o?0I>STlcw`$(ekE`}jPMGu%@v z_RGy<_GFpyWwtfq=Vlq=1G0VRazk4L-67w^HT9m%jXVQB@z8g{A%QuQG*y!w)+Xzj zh?Uv>x01vuNrB9Ywj10j?J6MxIKwK7l=#HLBu?q#a&E_M6zov2;Q|#ZnWbmG3-0Aj<=fm%Sb2Zh zeU~B!RvCS)=KuS&k-;`78|fuickANP)%QYHo|64QsW%S4;x#i=Wb)TrE?^l!xA5tvF;8}_V9!XZi;^~_g~&*re?|>?qNrV%&jzy8(i>M zU?O8b<;cI5F>G#%t%k*lSFIT?ejodv;JcA4v*JJ-H)pjoo3=*8HydB$O8*OUOkE)3 zWo{EQtCN$taHS~j>imDq{L@X`sD}pJ!BdYJi(y9mW`_&Rw*E?iLCGU7D3)QLr2=m( zn#xHluZYV=Ph+ph>Rd*C6el=eA-*pt6~DSAW4D7Dmt-Bmg@=CQZvBs<^Ny$TedD;S zgi7{E6f#1nIQR892New&l?bJwsio3hgk)BPWK>2nQuev8hl3VM3S}iLiM|!>t>5|m zfBrbHbMEK4uj~4JuFqT6AGT`C!;35v$l)ZzJHPK3uqu^5y~mngB^Co4HMXOD@He<88A~%NRr?mr?7E&0t!l3kw@-X2Ay)SbZt+rQ&$_t5aE%@~uSmzvPHFbA-YSr> zT*SY}D*ut}liD zXD=ahqd(nHl}_9KRKv8jmdw*cdDdF$FQ@llFhBYEVb0Y@(ImA&j$UDm;liYJ7_$pU z!3Hk`dF6Y!p6t>xf<1GK{@Lfv`{2Ry7(|SixY2`C$JF1Egvt&3+ zDlWkpy95$a{*@>iSQJIb+7s9L;&^-X7vDhb1H|$=z(%(eeZy{G5DI%$KtiTP{txtnGi$yr^Zy1=) z{|;_C-RP@rLNv5}V6KLYkehWAkDbzhamEZjQMf}p`ftIV%Pmwae+_@Z@97x7=BzNY z`!ftQ4#9ku;;p4K7@M)DnD|r%*M)@A>d|7Xi7{Z}0AbN9Rq*ly<{~4!Clx=!Zeh=} zL+&q3ah}2!M1Q ze|C`^^XSGiuv#g`>{rPqTW7R@MxZ)8PFsX|JU`lKyMdZs6wcXpFJzdH`@qe?f$@Fn zjyGIKX>a}>l-4z5|IM4iq#7$SgMUIvNv8*Map@d(y0IP_>&ze?XRTOn(JxrJVLanm z_=WcR$S~7eN2uAZI<&77Wo4^Fagp9r*!f`v$!#Cte?6hYw7O_8N`F7V(sL#7Y}zOU z1*tIa4m7~2xlf_An#0^n`i*l1Dy&}QT0HsCj9s84&gvY_!LFZ@=n}5yyIKS zDs+=3{HIWEr1P-RW;uw~pJEQg4@d>(2;_xj{)V__k^h}xC zhS%`#U0GRltf^(xQ6lI>>CHorJQxk79m>9SeS1d^4&z*b* z6LkvF$uS3vx4NOgNRM%b2~6opeP*ucZD2}FnV2glae~!Sh@ATzFZ=w2aI4A8p2tXT z22O*6w+$HYA$6wtXB}9Kiy}WhE@1u(5Mx7H_hMAiWz2O|WCzx$v$>n9(C_g)Dy`_o z&ao&$rt2Qz>@Xx>-p`<)JezQ%+ZUpDq5|J1{XvN{TcJ%>f>CBf*f+X5jLpaI`1!sP zxPMc^rDpAzHdPJc7v$n`*FBc)3U@H4KMO8;J|TxJ&WmUEeVADN5?mW5;CtWjUW;K*SUB!Jkq5Kwz@EvR_ zc?uWgyTQ-R{%~=;1vIUH0^b~4p!}~RGJRF}2`&+@kKb@d*)RM&G{(1{J`dOJh`|^3 zBJl5u8Atc1D^}#n;Jw`nBu31H(R81Kz8_cP$Tl6$_>SqUlvfcJ?EjByrHQjurb|yJv0u+ z^_-<+b@6!E<21=Nw_zh~U1`Vr9x9>ljkYx-u)IG5;N2>ych!PFl( zWECf3fn${_>_O*hIF)@J?A^vQ%JFxp%+Nb*YPkvtExn*HVK-jP7{|^osRUkC4VY!b z&?n2x*@pBI*HC|AN7_i4^Jr{92$kZQ_GkowVyDO-X!tPh#@jj zc&%hHeaQ=>I}9v2s~3EyN-F0#|H_@HgO&nq|%Uf1A6!Z~=b_X@1sUQVQw9?*^B49EjTMRdzxM~RJ2`=|!uhhv1WcKbg=-#`)3V_oY-~C#?`^>2$;|q3vV`r(PuW#X_{Faofj|4h{Z^;{`Zz)4Z91k80xdxdg%~kvL5dA zWl?316eLw^z}ff7*KLI(rJ#lJbF^tdAW89|&a&l*BqJDig6>Q%MvRiZUx=b+L zn2-j`V;+Fri)Y08=vk8s=|cr9prm#v!RML-{HHeHKTPTtw@Jg zh4q(8;M!*w#$xy<79TlA_bti;g{WqD`{^{k-Z4sceZCIspdGJMo*B_zg?sxd9fKM_)E73PG7NuIZ;x-K@Ji<>Trzfrx>W@Fsth$6=-&Rju z+dHB2l__&9btjgLGeh%bTR}poUw4^&pc^VfNV#Px%4OX}e|*Rx z7Bn1#WU+m;tWv++>a9Cj?mHe*8q6TI*_)GjK?K%ZkES9*^MP-fEps@U(gs0fM+^X%pKGfj;E28XM!Csn{B2SVfW63m+RXE2v znG>>S7hF$12V1@?VswjmS^QyZoDI?#_-Qva;dGW*zKZ2+uMUE}5fxBel+Hge#G_Z+ zl5zICd~|hN&KJKhMmlEihkMC!E0 zpHY3sU{dyE3H;*7V@2{Yn0r2rl=hqi&Du~hNqPePFX}c9Ui89U-(85Z?@2PCUdUH^ zt_phUa*W3BEHe9W34c?pJoGxdgSL7C;g;4Ak$JZXFSm#IwGI=`u`>Rz&Bn+%QiNSI zbm)TR_E@TU44#X(!>svr)cuw>-Qu(hmMUqW}~tI<{@@T`rdMZYUZp58uCU)u)OjyI^>=W+07#2Akbcj7|+I&6A-8iu_4Kt#3_e)$Ag zZn_YN0l!~ib7CW3X2(IIQPp3*K+=HtTw4ccZr|j{4)~#;cQ_WLMPg6hY>xb4c}$xB zl)5KeCWUt6p<3SvR~i`rCgkz`VJ&KA4$*wQVLEA^8EWr#=QOPNj~x0u4K{BZ#%OCZ z9CUNH`u*z&Znu;tt?3oy*@bi>*&YCILo47Ddyt6r>5{U28Du$)fU})9Iq8PtIJ9X! zohkK!ct7i~K5jKfV;a_iTrmTs|kGTaC<)nvQGIJTTa#llZ=NK%UD% zyjgw}PHa<$FBV3)e4_$=r4ot;xh7qwR5%)&O@1vYlH%zEUsE^MNBttfvFY>p!Itl zVg2(tF5x`($p^*jwW&x^2OY7O0!Vgzif z389vU;`wlS zTQ%M>ACG}sA91_~L%@?$O6?6-fXCLwctPq8e)?Akt;KJtU(^HK*O`O&XM{km8i&gG zy(e_LEj(l|qw+`=9=%bAsuhs{2`X@9TNE5|oLsI^Itxwsh4hkh4(xND1Qz+%aJpLw9)8!{p%0WyJvvDZOfzqSrb9_^<^^bdVtDTN^#znF9hTKVv>?SKo3}a#hfM^ z>X6a}w=fnzu8#nbyJ>Luu24s|=%(X7Y{2ltjcB;!2?>6D5ip5?-znl$*XRO@s;R)b z_ZDP8e-bGECkvB=oUmsN2hbrR9F4r}sbpI!STc7o!9^ZJ^d`^-fdsUyN(TO`gD{df z7jORl4;Su}$3HU`LH6?#(Bsuk=RI2wFCR<8ucM-HHHI&D zJ`LAOJVB>J12lG8B$z$?Ks*~xb1ro5#?B|xVO3xd@_j8(lkdk7*WkgzGD_2f{m4Df zrNqPZoMlNx9;z5rV9$xK^uAsN`D^L}CoQAF!=Z+RO^LuY$0aebCpmOwl7}XYm$sB#OSRPDEYldmmm&rJMyWHy7`!*6=qYLU5wa8!d z5%g?(;AaHO@mbc4`>!%sbnG;w*2TmAyd<0&8xEK6oCd$MK#zz|LFXg!@X1yZ!r#q9 zYq3yzUu!X}4w%XWnbpz0nSEvqEve_ukJ*FAX6NU)V63Ut6jg0b`VfT>zSp0vXZdTzKACXerf1HU!d ze=CL5jEeJcdi5-3;QLgz-S{F^Gfd)m*A#)hP{S!sY~e^g+laz#GUmk@6~?z>F^ZY& z1|c^BGPTFE_(cwn&2!-om1pDhsiI7uo-L-go`Bf_nSABJrL4-odYa#OoU`R-7ky){ z&BhqllB2CB@F1(i3=3JPS4Iw!x|0*xv&(m4!$f^%R5X|bxc0*Mq4%)m)i53vez(Q) zBhaayjIS3FcHgBbK#CW@<(KwWJU+uZZ15l@sb4vfkF#l6)v)lsFoTieQamk{O!gdE z#%wh+M|X#7@HR1$S{!m>PaXg;4jsUIiAt;?_dKZXokFeFsxsf!jX?N|V{EWtO?h+P zRHl4?H#FHDV-vhJ=!|EbDDk@xd*>Q3LsBhse?&jQ?G_KQYLv^o*~RC)$oL5Ia=#!X zr5V3h{zvV)CbGS5#i&%jf|2R9<7j1`fXGR9a7yR}>MuA&*UpRwqlwYvN~|l}dB%~` zG1C`X<)Xp<<#SB`po!DJtV2sxU7Yw?2ijU?5%*+bwUii{dQ5{cOuEeRvVVdh@86P| zP7RlxJ;o)qeZ9~$Barr-TFR~%l!adFPzdUFfb_Q+7~v)K6I#h* z(eYZW@)IL5UuVI-%Maktlp!ke)`1;38A)dznSw7nrP#AVE&KZ-Q;d*!inWicFzDbc zX7ZOv(z8W}Y4~8qxUMo|58Q9SYt}E(MBzSeYFNM+3jUGM{z3@WO5o2wuffhKo{Pgd z6͸qAi?GvK5PFft<(3Uiru*6BuQ!pQ7Y3~VtRDPeww=kzG~qWV%CdReKjMw~F3j0StF2noYB3@E z5u`QiGk5RhLd|kH{Au)n6Cyo>v9FZG15X=q{|s}cx8V;MXBLqViO=vn!Q+>Tuwj>odNj|18Kc!K(b(6>!w+q0xsDd3ogltr;K6G9k z$zSxA1?db6@ZDv|-hb0eS9e~<(|=~bf3@07k#idU6f!us>;DG5ifiDee+TvlzlYFG zt|Wb-kg1KlH6LDupH^lonY~IO-hqmbuwl$NO2t9kr!q1(4c8hL^DJE`m_3-u1C0HWVv<|7KLB!HC{w52$zXMPB2%Gr1g<+4poQUO(E1q!BVh|sbK(_<8r+IA zWUBn; z`r}r5Crb?MqXjrvABlUKmz5!pBH_^Y7LSX%-@6>U(EHlf{1YEqrVa0|YP}Q6Xs+)ypY%GR9 zW4a3*oi+(qI6R;d%hYh+E=6`sU6235eiiGp=Qmj0PRGR0;&Aon99Fu|0%Tv^6rP7N z*eft+COqb{Rq<~yoi~OjdR5qF^FX-&MTY$@0RD|$JC;;DhF?2=;-8RSA*Hlz5{)+RSe&t2B$v^f&a!^ zVbkCwX7f)4^nVje(?Yu;%&CeKqOU+Srzx?`MNL?KY&T02SC=>ZRAQ9e9)sxG6%0?e z4n4bo@%bt!emY1@0ExpX$;_+FGv()te1jdI|&*AhlBI-c!PKMz;- zl*7x951{l~ERA~lAMAK@9JVf7#Vq58p}2Y%i1eJq7Kg34-X|EQ&#cDYoldB$<-uxH zhC|Q-BXkc-#4VQ$+5F)ll3**rnBS=ZmA(zk)qRN=IZHU-pK3@YznQUgQZyKk8wE|_ zT=Dw+A-w(mHxzA^VU7KhIZqa>U?;S;!Nxn^!17!$+-q9N@DN# z=N+EigrK`|F@M43b~Uf+LZ!W ze)SBV_mRW3xeHv=xhZx^r0iZ zx-gr)F+kz1Y8#05Y@sQW{)4L}OE7CF7uMd3LfM?zaQ^&U{IFY+l|3>Oe=O^R?HUiL z$(uQh?9vK2aeh7HzFU;tx3>yh8kd2*mpP++pa3n^G?~E(BJk|#Sup4}#oINBP~o2o zUpD69?Q)^drc9nb@_3HjvnR71ACJJC1CGp&J;U(1WHIBYT}#`ZCxKJJRoIxoVdLZO zgXs-ny`hy1_bkD*XU zM}4*<&zY2u>;r>PC$L!_4~BM*B%zdk5G=XYP6t!w(b7Y+;gGxw{w(Mr4?{!fxP{sfkzIjC6TX&O z`uV~38x8dO=t9!8v==r;iZGuG?dZ|{j{LybG2ETaU{7BKeE2w*oTJ~$uJglyxAPfw zvzUmZAi|E_9z=c3GU(BpNdMH>V8dnxsX-Z(zB&bS0>8r%7j33N;~wq)+l13z`{GOK zyD&{{CT#z407C@l(Qeg6%DQL@{TznK+t`k(1>f+3y&qhi5&*|`1)_<(KOG(Rf)vN~ z!MRhJb=6+E@_^}j>{_~GDRC$Q?ZtwE87LonxwF5bN% zjvrn{f>trZkLdRT!O~?|rA{&1TMPTs>v3^?2yD1(4QEQ)aYalw9u)5Q?0TUL-IKKG zoB8r=-*F`rJPw09xv~5=O7F?hDnFcj_bNxTegQ~&&R`x1`R6Hb&mhFU6TTbTV7u5X z&XbX;v{_b%{L=dYo<6nE5cMBE&A5cM-_1aOS|9cD-Gzr+5ib8;hbPnyV~1N8aj$TP zr#p4w)Z?dMnJfXvI=+FkSP`r|KggGImE{~C9giliJ^4-87Cqj-%9yy*PHangIr-zxlV zd<>&tykHu?`B?-yIq6|(P6CW+q|gIBxtNn|2hQFS>_h8F)G!YPCUYKHr&NPAx24cc zD+UyHB|z5wRtWJHXUZCVaOC?Hu$yp&cDwJ#-xCJO%(h}wI48lD&fkQ$m+;|DdnoWz zu42&HEV_MNF~=jo02k<2Is<@fF7X+ol4jE z4Z`<`WGIsnVFnaG!>waNouFxu^I`gXniZHq+e%S2}vJXTQ)te=!Yfzl8{AjhEmKYYF;+_53$pbD7>!MRxzXAQbGb zpjRI*VGU&$F^iakup)$m=3_6&1=VCwTT%mcLr8zSuVvm&pUV~${UXClGeFNt=rxd3 zWs_Vt;l1Y97`*i~R{aU3(>5=K=EplR>fShJoNhGj5UsOv(Kg|)nmLON|3{epv((8T zyB9+{_F|BJ8c5lUXU{ohqTh8xR_1;`%&h)}6OOfF_;`kugNsmIa1WNpCc?8{&Ww*u zKk9d!L-*f07(tv_8I5^NTuc|3srADRXMIMaZ->y^FcoFxqx#sP8{KwC@Q#J#31>i3!44t~|TkCk5`l(STP2doh@A$#fqw zWxJvfmzS3D2i8pEV00b?X(lFnKM`tNMXm-BI|VT><$v&+)af8I-RshMc{!07_GtOv^#M z96gDdGnPuGwn{RJGdXDRZj2hzA^dY;lrl1_F*`{X9H*ti&5TC!#CkIXG}lq}mr0y= zTB}*h#%au|^l}U|egHZ~cTnu+2swWX$!aYVaE#Mqv@g$MU6d;DjZFnQ+s|T;YyN;8 zGP`hSZ#~L}tj6Abt?=ho2Ug}T!M7QfOoFgCog!igCuennUfflDVlop3|5Jv@76X`f zPmQf+v>=L~$iG@(4BFjS(B!`^uy?r)nc)wJwXP2{t4x!&L!G8L0hIVPk1jeV4*JU|&!u3|{Ji z%PVBrOGTvlrj2 zUONKWHD8zcq<9%`JoSd|1vcn<UDWk9Hh2UtC~r)vMr z$E-7{aC+-FW?O;-UH>%({ya2-z={8naD#sGW8ev_op=r&-;!bfefk3zMkP7hwW~<7 zwi(m)Z-C4lyM)HQiz-19e}lK zXM@s?O<*c99g13taB<5bIB2Pk2iojeDVzP|X$CNu40#s2k30s4369x0;_3@{T(4 zT46z#At)b2kqr{JaX`es7r~W2mzOYiD%u{mayuM4><3T{D!v*k4WB$rhZcdK9hQgwM`mF<2}5 z51uxJ&@HB}_}y$g>*Cr?Qns1G3!`R|KcNWXDnuE_h+CL+cO{*owHv-G&SWlAKO$$D zPuQyWv_t1ShQjl z4yObh*spX1!bE-HPaqF2uMuVL8(3lD9V;SZ_Z~lMBL)RdX2TO7W6giBV4;@_>gs1f zgfj=C;-uKFvgyo?eJ*&_Rtr+JuYgj_4eD%`4vkixFnUOeo|B8mq?3ElnuzcZX6NA* z&kBo1JS&ujQ_(2n_(6o;NUCNN$s z2;w(S!QWY-<&qw`=(jh}>a#`~RMCD^HfW<8o3?=TuOTQ{rox_b&Bwc9Bl9v2$}``D z@63kB`%(PN2NJZ+l?gstgFjNjVD1}TbQbQ~X0^sZz2zLx51ztr;unx3(Vlok?!0ie zyBq_Q({Z|YI(aE}78=K-fjj>w%C3<|ho=tUJ$EHdDLRY$6Aq)j(`L*uv87t)6sTP9 z3bM&}B8DqWv!TqWi0{CyrbRJZ zaG9kr-($i8kY%LM>e5v#Hi|^)-z+35HxMb^1!S9L3B9BkgTJ7hE_5)%qaO`0%uv-z zI#3y%!=rE`xkIf_xq<4*Kw?!`%3m+F7Ba_KbKWU?mJjvMfy>wTK~jh~&R#14rY_jkjhTf-#sjuEQ5ZKpEXCvbgF29DMRLh0frLxBUq>5!JSfIYRY`{<+*Z6aI~^5!c9JGZFHj76K?-8ZsM|A&M=i?nmz3>sw3!H@fOaqL?b zihd_(H_iq&1xC@e(@ZdZsy6Oh=|;q)oj8l43OFZcWYA;#PV*&1GpJ#0I?h<=z`vPZ zOa?+n$pbN#FH9%IFgI&-JGG1GZ1w@(^$gHuE|Fz-EyxbD6}Y@;DZVHiCJ|ejas5a( z#&=|MMCv|*^qY&AQIJVpqZJ`Cu@rW{3nlXp9ROoyJhU3j#Pm6UMlm|*7`+(1MUx)>ruWf#aR(TSTv40u72Nl>}Z3qi^rkH$WG8^OP zgN=CyN&NRxsuk~sJA0x*D{&E7nVf-Z?|h-xUbo>yPYLEZM&a^d5q4#58Ez5!bz2_= z+TZ#jnhAO%vVM5CNV6RvvvOGvW` zL4Si~U}7RqP6do(+CD6Wc#&H4xpJQ#*qO(7zh#GH77rd*8{xP^dN|CNz_{)*;*&g& zW2Y+ooy%^(`7SB)>qIF!-+h1wZ@J4z-_yd zagXj&B&;$uFO(vA7rvC&p8kl*?H%~_$1TjIw*faML9>N2(_B;mUD*@yQQT>a@)o5P zE7QsYdq0sfQHJW2RiNo$CU*RK0LDS9aQ)vT46Ip#BCi7BPwXQ&9i+yrs{IDH-v*-3 zyT@ofM~b-j8sNr4V{~}a41Y*QxxY~edHFG&h-jyheA$z5M@$}94ZI}$q8b>LOaPf0 z71H{#fb-nB2E;RjI}>e*uqy063eG$uFKcq)L0mSFLQ&Silf~E4-5B)c1@1iU32B$k z!?g7(jJ6vW{`lMBbDu{@1#hw1?lErI;tHkd;rL)$3;yLb5X;1N!U7X(7h=}v4Im>k`UD@*F&@z)YCuw4V=^PF(=x@_F0 zCPgdm_Tdd-UXbp+<>clFFh93{CC6T#!=RkKq_c*CqMSC|oUcQt^;$xCPAjx>g&7Fb zb*SZ17NwT|ptb%2`YZP<=hl)dIIzit?G$y#ld0V>(7{KZk`BvlE5s<#7GmNMM>j1z z2OEr2pf`omf-Ca$>CKJs`04>DX_|}LnoUL509TJ1vE{*5>vQ5PV&*vz!au3SQOa&vw{g|FuiE1}SsIA*_p|5f(hFfrn zRaGD5^;{xDYl`sh7DG7CnagPI3q#R^`_T4eKDBuz56{geVYiYOk+2S=n`4To?%ye( z+WL|6Rak>%mYs*vxA}BR#3k&_=_6fhmNM7(Ex^ohFI+y4+?`yE1G`4Z zEVXK24^P8|p;@r^P5_>*2XF*dv6CWqPBLj=aGeN8j<}H}Pzwlx%DnD>R7rVDa zMb#f-Wi7mR9FjHCV*p`G;;O zPo?3nyCTeMkf(cs+c-O}d(gxHCsOLTg(#DWDB0wJEAPmYNs@tJTPMO-nplCiUMdsW z3ku8=*%*3zZUe2l=n4_T$uO@z9y<;PqeA;GxH{^>Y#w=zUaO?>S$-3V>J`oswfCWj z$bL-KFoXZDHo&*J>9}!!3GOS<#soP=qeSJK^xkx5$ybfJrH|alfNBaHQZdkl107$DZU4L~RjMZh4HK~c^_MfK+ zYpF*1=!q$0>bJqI3Nf5Mu7p??JMrI0$KsOyUF3K3bh_U@ikPPOfmfe4XT{}*P^zC( z?rJH^))dJ?MLL6Or-iw3^K)TLShI!1e4?NGFG5vL8>lYXgq!ByMe)p|@N>B-(-|3$ z>kWf(rN}anaZ_Oa+=$0}^QVvw!>t_S?N0Ev+JjC`jzn!`D}KJJaDQoyDW;7^Q~As* zoZS~rqksQ7;yrmI@Q#*{KEJ!9H2MH=F2A78;X$zB`3*9qa3;=m=aPk|&+(JD?WA5# z8!_fw6pRz$)0S%~`0VK_h|O~$eF157d#xSkgy0Ijlnf6SD!>!jy-@5X z1HZmhQn9I0oak@kc|F6ymCv)=lFbH1 z=?ea9JDz^X?#fftoz9z}eu<|z5d?J- zLXO!IJ;Bg&cY(%MVMg?EFM;4?RYlj1`pOOAYbsYO0V|mo#@qHU-TKqpb-aZ`e%776 zk^(!YETMnjVS_Khko9zjAMe}$TNcZ-%16OpE?L`swQxUU#e9-x+CNQdy4SZ9zInO zE9AQr&zxxEcITwv@&+BjWOIhMGIvmLMZI32H|Ig+!8*0d>ZD_pX^v(*PyP#D&V%qu z&$3EE?cZ|2Ny99jfLg{vQXyYlj5E8lTy`ewlfKOVv{Z;qIz}m)`d) zmMygssNS5!y?!N;n``XB9nGlcF1#+m4PCpevNTi3`wo@0nP;|CP`f8ZPrbhxdEq;x zZT>~97kmxh&wbXR&*gq~LRME2uh3kXS85YkQK{i!?P_GqD?4%9Iyrq&<>7J9tu^}t1ovjo=J~yot+XwS z;0lc1@a`YIF0eJe#VgVD;gxK=Q&By7w{pv4X`AX#EQ#i!N{@QXccJmiG!)JD=k92Y#z?4-d2^ zXAG(;@g&zTHC%9T`R7V)bqj03azjB{elEAyO_($nWg*ZK&#V-kpTZ5Rdc#w<3E>%d zX7ak9Jg~NJnJ*}En9kGOnkX3m*Q8SAi@KnA>1(c*p*hdoJka{$K~=$lfQh^gG6}+2y@uI5IB~+6e zx??F%rSSz&TvpVlQ=p#?GHyDgEW#3kPKd zT~0>@GqdWrrQOkj4Vw;CmL0!aIp^^|R_07Zr76#`V#%*g-t@nn6>bH>wNODBFMhUx zAo|AxZdmt9!CW&xuFv%$!OZDJ){A=!D+-dzE8nR!F-vWX1nP&qxb0KxxJT~H6C8?M z$kUrEYQsOhQ?OG{mz%stSuk(W9)YEv6>t64qm^G4ZL3@v9wsnl1_iD0C6%?)@+(y? ztmXyvByp_`4R}-Mj#@u|+g^EUccoz0$}QaI>k+0WoPY&l&~AY@)FQBmXW zZ&By@e+m@L9&F?#+}7mXAE(OQ(R@J=J$azgldC64=$y_w{6B`yG@hz13ghM^Lx{{t zDn%hg?%nGmr4o^$WJ-xdDAA;nDf2vqOhqV!D4e}+Qh6)VpbTl!C}}P!>Aj!Nr@eoB z@3Ypkp0)la7PMl45qDr*ofTs*+3IJIP95%L(zQHp&1+!}qwpvKAN~>q=1X1MvH>{| zPtS+vUg|h!yCpX3%JCQwIpoHh&7hJ%X;PUh<``@xxlY1u3!B8Do?{M|XBna0U_2=t zJqO9Ww=M5j2fJgREZ9hd0W`#8ayME&`1m zTtIDNg7N#TjEy^;iK=NUhO)AV%*%xslUxL?h1HC)qb)YN-N4?rsf=vBFfzvT!MLCu zB7YvHcj`}}$2vt4XP*MURIboQn^-u1x`#$Z-6x^0{y6?$8ojQQ$~N^6N0+8ubXDVB z(#LT_%eV7M+p=O>YFJ3-h7B@rFG>RAZw7}KNr9Z_T6*-rT}V*cgg$e{Iikx{@z{tB z1gqNO9gaLlFT9B?{_zbO3noa?(=le^=3V+QFc%JQIuB~E-r|c%5pH%)45w2?@-zyRe!_31JJ>%GbwTr_4z}og58v=I{R=F8i z1v8iA=|F}UsLP987kF*|m%AZLbz18NwGp>1(9biC$+smdnM-v6BGH@QPbE2TIN zj&F!?W)m(5yTRz}+KEOQ@7eWkytTMZ$HhA=VcygU9Fxt2JvvY5!*wCFO7IO;-_e2L zygAi1uLV&kWdVLYwg4YKlg0CIb&=*BU8rQCCWb@rPjBf;Mxa== z%IVRYPQd@LpIm5}0JWF`3>9J8W*kXHjXYP}qo_r~i!Zk=^5t=WZ+f#7B(~y{=sNsr z>yE#6`e2OhUFLe`Q~Z~!!hLn&EOSn~hj;Xgu+B9JvT9av6~D<~!OB{ap!0#W>(5b6&VS60p$0s9qLBVN;e$N`nP@RO58YjgdHnYds1h&5=;b^P(vDcLTd@%zng4W8SpG#5R=Mqi%{|j#Rr*aL&=F6+RW5*blObAr*$Wrv z#ek{UD5iY7j2|L4F#jTNQE3@Ij{LO}V&UirOXuVgu{Y(oNU#j_l|}K?r%oo7H$Ohp z#u8dJT|xeRF<1*eqYY8VAvj}#n!FI^t>F*ZRK1bPVhbLNUx<^?&(>Pp$6T6xK)nO1 z(Vnvr537tqhrlxwD+$8ULtga2>UTK)+Lx|;`hloA9mIE~Nf0fW#hg8S5YxvOvFEPb zjZeJZVOYK&&G>?FZ|_=+T>FjcLIcj>F^|u_I0l_rF(i0n7`gSXfvlby1BsC<#8z62 zJW`1P2Z6Y@zHZW%r6Y=xeAK0^f0qB1U!QZ_tq*GFtlmAMC znRV|Lv%-EceR@!o+YnI1+&_@P-u~b$m2zoCw7N_E^D6L`%2L!D)52~4UBn2LZdzVd z#i|>Sh5bvel6Ijt?EQD0P{(|dF50vW@73r*%dBJ=>h32o^3t$wt`aRYGoyK0`0B41hm@{94WA4VzaN)`y02Qh;RbaI>Q(|UZ17?p-K}dKhnn>?$pN_svbo1cqUuCB7sIv=;6Mhx$G|4t2pw5fWYme zBt-WPdy#e|T94JxXtV3!psdSb^SEG5TX>X*i($NTl!^*#rg8eHA$OblFWfJDp6)ig z%v>*=Uy}FX-q5pRl#x%I!YI#Z zc9EPocTHY8-nE>`ixX3Ej+G89mp=)&qaU&!I5JfIM-GU%{ABJeOhfO)K)igs4A{db z&?0bx9bHz0Jp2T1tx2Iuv1V}PXrE!qZ1WbyeJ^j5)@%LX zGfSBp-?{|7HP@1lZQA6~odo!5*w{AS`4o*0m&0joQ2^RV!`<$|i4{4ZKOzFxFIaQ$ z>G{FwS4Y8Gvl2`boLe>G9Y|dMMF>tw!QUagcf9K+e&2Qk72497IV>ntUb z*HR&@+lsge-2yGhu4VX|gV4!s}qmI_^V2ZLG7puXS@39OKT$P4^1uigjsLa*Z0 zcmIe0UB;+MRl(zKW5|}d0^e%x5iR~y($N)%6B{z&in`Q?c2%a1O7C@NCORio`tB(WpLtK7r8xq zlH|BnvTUnskW)N^lm5X1rkKYNzUwYFUR};+Zy6#-*IprO43fZd-gV*^JwPs*MH4LA z+*V-JhjLSwAN z&Yk^=Zuk4m))QQc_v3Bw&2C33A><0@I)06Y} z(W z!6urG0c))xC%y%eq*`$M=4f&y@*|2F^0*s@d&rn>9dp912*SF;Xz7<+Y#LJMRzDns zHUA#swrQs@M}LC3(t3kFtZ2n~Z^Lo8_dePl(nAN?7P3lAnSk7DhU|nx4KJa};gf`tnA09uHwbGHk7-+}p7B-w6j;5vSY6Tu)Gr8hwVPxF7hyFR; z3Ui~hkw1@v#kzA?bF&(u!*nVI%3P&8L2J>TKPwXa8TrvyiNln5Ik0I#JeM@{_JFqt<_p|4?_`-NZE2#O1 zQH%8bmbtamNdSg-(3e&Df|@C|xOo53RKvvlpxNQiBc z;$+5}<4ENgjkLDImn>6|X;rOF#QCZ|vI6-F?JhG8}GLvav4Qm7JU* zO#Ur9P0VLVQ!#-jc!;$g+@nune{vOG6^uj@Sw!a62;ku*OF*8_7j=Esf!pL1iIS1# z4*dc=yjvVzExbg&)*HaA*B{v-p-If{g1vA-Vliy!yFi@QTw(vYa)$mFxeZ>mXMtcK zFV?TlVc1!x(8+Tj{7f>$M^0X_H03?p>!K>S@|Zt=g4B5I&{gm&z6(69g;2}E4Q2Lk z08t2q@u38mFpOgE{24|_i6g)CuM>XD<-oVok(`;c4_`*4qhELdxjknEc^y0q=c^w= zpIa=J4Xc3Ny4mQ83N+c?0!xjKK$Be*)aot5jfF1oNsvo7cg;jS?+qAP)=8ZfC7{0C zE{M?=K_-!4{?|Dq@9O|Ds3}1;Q6ISbX}C?3s|zkXlF4cw!#riaEL!vqlf+Z=*+*t* z!=IZ^scMEZyYh@KZb&g8kD8mQTF*R`4md*}>ih*B?i#i}(Ik0Q$HBimj+DC3;T-G| zB_TiRL82y}=Hzr^z_;ndXmB31Tk~RN6=URo;*3Fs6ZF+T2N=5&K<03M;79K)%zszM z{uz3QXdPJ#&mz)cewhZ9;;#WdzCAGQKAWVx;)9Zj$9U8x8ivXxL|1Aawk!I-Exd_?!`-qkOTK2{J3B)J4liZGQ#L>m0 zs1%ih^__=c^`3G_(VPa0hUSrIQ&-yf{5f%7#oMq^2WZOL2BJk`U=p0eq}zm0oyeu+ zM9gWD)42l@dKl+7dmOsFGr%RdyX#a<6z{u`DC{7Y_Pe-&s}I+0MpCeQ83qs-Otz% z{@HV3JcbYItxMR)h2$ak#{s+&H5Ttg5G}%`9}WddhqiOCp-L!f@KyX5@R-2yXT|Sn~1=ywT8x#liL% zB$G^j`}LCEkSOwbUjrU1yG%AUd%>XdB|L4LfhIW%>6{6BvMOB(KF?5Ql%s69p(8)w z=arQ-l#iERi%qek>?m{XhdK4wmJA~jt2w`T8u7l$f9y$F9=kN%xUDszjK^Zsg2ZcW z^r}cd)z~V-nSAYzQpUSk^_qe!n#s^w3JRfGjMc7r7P9~8bPeuKJD3RIt zUwJ&*b)P3PGjrhp*$&?aG6*_fC#M|`qMv>QW$9d@K;QpH?Z~%fB$@24XqHPh?xcHcPEGoji^=B0*svQLJ?kXD2-A9Tm zlyKLIwODVl2C7E%P{^Z$1iy=;Dl3b~*5^NI&Zdnx)B6Mt)7j74emi%7N#`k4NKC`<-;PjL@f%Xr4$!S7S6IeO%mZau)ToVj0>O#k;A!)Li84WNA`Ub0?CRRQn^_l6yieh{u@O=e{%+gS$56H}OEKg$06>OP$n z+D@`wtiwfrO=+T284Yz3qu1`gg5<>|xc+GY%33PHW1%$S|0N0%xwoPJB?n!;FTixe zLX377!s8PMVAmN9YTEdfRv8I#oNXpy{p&>hX{k-2#}w?3`(yr3W58rT3<}(fCf8Ev z=((57KZzPlwT*#yQLbSB_6*+j3L;;QUjf-|X7uKEJzQcbjW_hB!{MoqaJR&Wy*gJ1 z{jaLvx>>c@mtwT!Q(dH?glhnHmOE$rO|xn0P9El zYItTi#Uzvm;UaHoICth8d7Ki@V?uYDevVi`$ENWVNuRyUjKmrelyH&dG=Gr3n<`;9 zW_3{uStBxawu8J6ROGHT5vEbr0-OM)^Tc@QG4WW1thq|3Nkg?LH+qb~^)NZ;UHy|z zeJQ49!?KV+BLpQ}UlN5@TlV_WKD_L-nq7828Gcuv$BRn*yc~cJjg_;RJzIk4@zP_| z{n!X9SD3=?+r@CZ^C5I^z6>WG>%jBGQ}Fk#1kPXZnVif%iALuxv0IL%k@u@VVr3`? z+Q(-@!-9|WZ24mns5i0NJp+b0rCB@MiRK+K{Etqm~2Tqze;U}KP^+k$HUe5o=zK~S_7Z#<_ z6Sj9z*0ve8=4Y`tpAiT3u>sV1bPf+K(!|E_I_i300lVl+8NF4dkAIg*;nb`f)OY0> zte5d1epZcW(bS8puJ3>sv*zMux!0KOT-_#jy&hNe4O69td!*H7Cmqk@=MM8??U0W$ zu(YZe&wuKI;)EnB>-Pzug9SS}E7140ArM?tg2GE%Rmg}blwqxpT_KlMQzlztKcH0eTGx^5-0_Zy~rCBHy* zTpy^PSb^IDrBOfo0_FSsoN5?7$JF0xu&UjX7_2VCIdZxv7&)9+Ab-Zccl7+-W*aEw}Bk=v~TR0?>%giVIbj#{H_%Nym z1I$v%u7We@T42byRb_*2O9rOc`H&Zho545tGpU0(lJ)oUFO27e9V0{orG``*)so8+H|}KUR6b)?MGplyEKcc{a}Mx zU>)74e3x$J)zUwDUnIW#rC7T~o_U(o105w-+uS77;d5RuvKAWi7^>;Cq0t@|xTxaO znkLF;G=Nv8dxNWu7DOuP;XA7cY+U|@Y#E;iAO141kldiFhiw^lqc8-^ny~)T74Yv= z1|`@`4oCNblCmwBrs)%bd1K7A)(xm!v=G(^Y(NWtPw?D1PS#33qPY#eU|efV2VFGj zp{?uiaAYnWG+~i~tbGv87lqDh#*p&)K54E!MCVEen3mtXOlCWMBm@3N;8>)>alS3d zn^paTs_njxPtU)HT3uO=O=%Gxy>lio?c@C7EzrzzfPyClw4!RTW_#`JE z`5a!cZJGsPKzD$IXhvbtN}$T<9}yV2FkX#>}ELSx$hn-L;Qd`PLI$d`_WN zs~7xgvqAOvGB|so4$aM0>6`Uf$nP1-do7tTe=Xf`>x{+jqP}8M4G$L&IFqe zjgcaat2A)VH+D|HGiW+qq+(hTjK|9mc$htd+ppMylG>e6`A(A?FYu3SYEr_DuQ%XO zqA-LPWRm+K9c1;=xt#r;doWKUl98OMqCu>g+-GjuSo5F@UcTNBK_yvmZNeNSC(_y7 z9Sv>2i%x;pxDMv?^wfbML9*D94{W9r-1jYuWWS9iW)9oIqkATN{MzR98sJJUpZ+jHnvx1}t;NCA$Kc@t)> z<0r2@mebp+dBk;S2zTkog0!qZ+6J`JynE>|)St!k3$1~BF7kL#{3*F)@4&hJVj8!4 zl^T`@sKIt!d*Z2WkH=Hrkya!cUB6>oF1V|`GYYrDw?Ui z>5Xq79(@M4Qqk4^c&H&70!}<5QJwd3c7!#~bO`JAu@7i5(qRi9!{pM8Nz zzTS)3rn=n1o^~+UxD)4|)MGdQILlc07t<|?v*3g?AB;{rM3VR_vHtsZoOywm{&L8q z8(p(|!GHLnd}%zc z-ZvND`}H&X)|S#QzklJLOmARqAH{R`ezw_YisJU1cKrT44KH8x$D6I6=)?sv-uoPf z7n`KH;UgN{$Db~eOM6eD@uzCGYitWHS`rUiKb66Z`_ded$t7e?;3c-%9Rr>h$e2Wh zZ-V|WyuH8YjhctH!7v=h>bwd%s1b)Q1Cc~qMIQA97I9x@O$WiN(M*im5uEekAxbtg zIC-uZHi#T&%KRN5OxGF2XUP%=8Gp2I9wO71I$*w_C?2-Wq?cNMk)Y}XG*C_>ggc9C z+1QSu^L~(ETRAu-WCu(8^Dx``D7c2a!nxuJIConZ2~W`AT(UBQpU1wE^$|HlR=gFY z4z6Hi7oGrzf>@l%d+Q&@4AWiZ;@s&2xe&M10@DA>Xa9DwA=jijuwrb0sMN0jrCd%7m^IQ0sP9hbtMEqQo6P7H_sX28geN6-?KN307) zIpHdcIlF`(;PoX#Bt%aEHw|9^r_HCyoJUK*>Qe$1j7VWmg$uSzSkY&<|3K7m8Akk# zL}4`x`1F1r*?s5~-YA&>ck5whSNt^Ex@9-sV_#*9zcD0DmJjggZyPw)Crq4YK zQ(hKa@1xf6lFJX%#(+)T#@Ly4(lWoRU?z zBK{K^SD}Mp!uZ)Hd}`67ax!=YdO$O&!X{{B2mn1 z08>s6!`mP2)c)!$G`4gFYn3pX6kS0kjrz#Pd?S4RD-LcdUVz7U?O~_t5T^e-K=$kC zGPzwR;9+|gUep!mZmV0$HvDb@EZ+Hk?!aQ%d^cpP;UBcFoQ^AGf5Gz%Q`oV06?V5& zupP${;$EG8OUBVI@4L*aPwbPif8955!Er8LI7ff@PpG8Z2FbA2_q= zY@a{Gs_zsT>y)Bzo^Hg%*e?23%7O{LewFSr@1#4PKL-i3D$+S5hi9L;!=KTuWd2_d z;ItWV9g3%c`fxmkyKGisShi~XsfqpK zZuDhqCC2@{&hxfvVQS51kkc<^`?+N@70fQsxhBYL=CJYASS3-YZy`=`BAhc1>|w=C zeiX$BG}*WURGPA2{QNy~c)W*ZmX=V(MSA3efd&!&B*#&mbC86W*-;U}2wbX_g%MJ9 zXq25u&2Q#H6Mq*CcNW4|hs9VkcBPOGA%baxreLCc3#GIyabv?M_#StodvlVZQ#%9p zlnHP;YsYz>9(%Ue$`X7r8VJ0>q-b(F5x(sahUcGj&_h83>fPmuc!Lp>mH7rFr}fZK zt8ZjwG0zuLz6yilh6udxrR|cRNob)ad{9~d50wNtK_$t^za$t`n`ffTn+LEwa0yko zya;NxpC)`;lQC=bDRWSGEBrn~aQc!SGQ~QJzZTEtsNB>5`;Em=AbgBj{6>`A>{G<* zx@WXwUmLW1ZNeRw1|aANWz^4#Vp^R6MzzM_fBCJ%s?Cl1Bw1p3{n3p%7Q@d8X~y$1Ecz4E4)e6f#4)_kRda)i+GUkZ8r-hghs z;)(u>vFu8Po;K$N=gG1quJGJyDVp0G;E}_L5YVw3_l+5oX*|s*Ao(fGU{XNrb_9(& z{)pT?=m&bH)4AV%1wr+ZouHJI3(1!oNbL}Bu%2@@)TfF=c>5&Wta?JPT7D&}+lKMl z2Y%vSYz?L{fn+_Qt{uVP8_%w&*t-U#Co{EW1X z3~X6k3K<)$Xu+yhxTF{do;L)!E~_W$iuA`2TeS>NuTvm$zV$eb)!7!2C`N^yhESVx zj#hkqNf-AAt_0UCdr4|I!qf6L0g~s z+~EUfaKv~7vk#6F_TPTC?&KLTJ}3|Ko-t6oyc``LUm+uoC8Tnw9*Z*7NQ9FzclCck zkf+1b2u2?g_rDU{oVxEM=Z74)@ZQRmq4Sv?A6J0aEFES|S~}J&>!rQ+i4Z?Fo3rD| zK75h84jKUJ^IxnN9qTb*`3wxzG=tdpN>n^M2ly9yv1>Dck!dHBsZBQ*yZ zVu!~R`QWOiDA%Q&3tI~tsI|&#Qe$L;OAF4zdk-U=w(1m0Yf6CYeGia6zLlo`Z9s{$ z)5(lmc6j=FGbRWtpsC(8;C_w5lTM~o;m#Vka?2dPbi}~|*Xv;5C5=Ct^4UH$1hzda z2I(Iu^ubUhoSAo>8e}g+neDI1tL|qg`niLF@I+j*zzF@1XCsMOjqO3Q=y5{`E`2tK z#H+86oBfeqc*gUzwpT;HXgNC)%DIZ-iCvy6IHsaVq*$701TvU?eLY_mAu2 zduKk*@AnF5@=k)Ac={pU<@bQyI~$lTo>r&Y&&A#~W#s62CHU>ujP-h}iF#vd+ml7L z%&m4&=JuoaI}wn$G5{yat-#PEmS|X-p!&UC zyu9-<{!rAzYJ<1<>|!B)?ca%tm5m@Nd6xIzum!8}TX?{zhS+`cCmq$d=xpU@bmif4 zR^9T0gtQEiJ(kCD|C%@W`j9=?*%q+%mS-`$64gnS%peKN)q=fAw^|R`-J{h4FIp3a zCUM7KCHA7f?Zma<1KO^BM#>cy(dHl-NSi$js~=v0=Q~~DMztVnJsDwd76X9pG}iW2 z8So=jn;E~bg#_gBVQHNwI8Ba{t}DiL-qc-sKw&A_f8qpbXw*dUB7dxoV(5~wEAZ5K zcH4{kC|>L*$9=i{7X2LDgzVXJQ2r|#x$Blg&bS|}R}Z5a!w*PpUkdSD7L87)&a#!W z%i!RcFn1`hgg%IHggrMxq0zwuM0Dy0e-`K1N;K{k#N_;H7p@c(A?!(w4; zVkMhD_X_N3ZXp4Cu(kHbJ0{+K5|@Wqw%N3YQFpf+AawOT4qU3BnZf~Z_{K@rU(20f z_Tw6eTu?>R6aw`}DGUX0@Nj4*Nb8NURmbv4dC+gTT)m&MFv$W|^e0&DJ4RFk;_&6; z%{Xr)lf9|pCi{r=anxNj8wFOW(g&&DJYDk%$k%G(tpZ9eM|R=Ef+n1AMXBfZ`Ivm> zKioUei8HfykmMsPAlNvT*uGwhwf8TOFFQnN)~Vkld_o+Q+}7~=57OYoFVYsVJPZOa z22gHT5Belz;@Ur^Fu}ac{@caQ#(i+irUD=B~=(?c@mgU%CvB^ z8Qhm+NJLLB-F&J6CuSVsWc=#~am~wAFuob*Rk?xOqk8apb`#To8&Ny%f8^E@Z8W`} zOqySVD7VXYk18PeJxelpZChT`F?zx?WPyU3GN4u`zy9867_htp>Pkl3OdN2t8UG;^K z>nzj4QJz<*YBT-Yd7J04zDNJ*97K&|d+InA0T%W0*i^O@m4o-;?x-pJF{Z-VReP6M z)W3il$0%Hp(ExSv1F&^q9!_iiL03uFVyyLhjMg*7fv5v$q8NhZyNDJT4M>m^FmNh9;m7@6~1*^Ysi$ZSqB9!U)&+IbALuAc^0 zEEjNjFAWmRCFoI$16!9AP!fC0K5|+g9?Q#e&3zBjUw8vZh5+U*kpQ)6)~za%$B1~A z9c4X=XAB zT?t+JEUCmuGkce5H26OJMx{HVsqBFaOxVur4RcBXT@yu)!Kebn4yU5+MGr8QkONJ= zJD??ROMTC_qTrsdH2Dob=UkE&FHbv9`OiLpP5cFP%K8Vp;nnXpn|U3yUX7olOxKXH{;F&>DabmKXWcO4o&VvphxL->OIK2=N(P7 zZ{j~t7EVBuy)ihpD*-NO{sGzc21xTtCtW47$alF2e)Al$|0a0DU9Lso@3Snfe0CPi zbb89X5|aRT^L%V|7eKSKT^P@2$Ko$ZBa?>)$yXUEP&eSxaCtV$8A#xhmlx=x##(%# z$LqE6>?0d4_p;Z7{HA9N1i+zX5IwY}!TbdYkZ*MpgEO>1W`h!uete$(I$R8_xJtIn z@3naLo)`YzeTJNhDaPFs^VyrX{{yjnadx=x38;M22%TacSQ4N>oN9IPx!w&(8MViE zUY#`E-IVIBRY7lIUFL`OJG6KrhXJR3F??qV3f$dKwj|V&6K*_hm2YjUS(H6)_Wwk$ z%(2DKu4mwm+e_(MZC! zB09=RXgMXo5si@pfmfcW7A{LZOMIk@%-`Us;UboZ`EvT?vIM6)bRIa^HQ~wHN#Y}T zo9;CjBuP!XsYF2lW!X>XPS1`Y;MsiIJPAaxNUmZ!tl{Ll1$Kr0&V`45v1 z_aU(o0N;)cr0-xNtkw}h?csjfx3rX{ca$gG-_0S5wC+Rh;B?YG17QD6E0VNR6z^Pk z1L@}*(a+cu&lX(51~qNeoeyZT&<<7oPebFr3f#n7=ceZ0M08jb4U4|vQfURe(Wc6_ zvSDE6zAETlR{)khiXb#?51#4kA;FK&()+@?c(K_Ot76um)pjM+dHJ3kY}g5ocKgY) z5BxN9yFY0Q&Bm0oxuET}p0r%fXWcQq55F?(p+zE?oZJ%$0`w>Om-3U$sa^#sW3TBp z*S&bRB9G*1snhu-L+qnLGTiA3zE~Ez5v$F;=xF9E7#@;ApBQyKs~sO9mo|{@RdLv@ zcZ7;rD6;SDe#v(0IgaPNj-ly=T{!3xLErAKhJ9_rm|Yx>OZncSNmMSKaqj?FH-Dzt z*PQXbq#hOF^u*Gd_2M4Zw+wI`WNw&V`1vwE%1ECQKGA^f>x`oacutq z^e%P8opdu^zL!tL?(75s-aD}Wr7<42T?t>m%cF8tA5r8F!c`5YaO?C%XldJr8v66` zSL-WWBgoKVlNdZ7zp5=f^f?yRZ$=-l|8PzR3ni9q1MPLYiM{7~*tc#3if2~C_qfX# zBIHFBR%p?$`wa>8ISj%36Opx69)C-kfsFQZ=u6F|Em!j3neR$)uM`7Ik4x9kg6wN8m}PPt7lfo?)9Fi?ANd5;zutxgR$E|7po6I7{iW+wBMHv^u%hciS7K0#7?=xK;H`r$EHxc}Qv1x1mmlS0 z#Yhs#s`<*;HoZWFqp5U}1O?NTfPEFB867hb39r zY-`<3IAzjBtb<U4SY{*#` zMMJ7Gc(vV2Fk|94iCUdW4+@_mJHGKWaeoz%Xjp;6sn5V})-9~2nJ|2@jJ}Ug1CeBN z>=SOnIr9^#dQTE)7>%LM@HAAKW{kfD-I(=xYTVrZT|}$vG&z_V2j9Lmkjd(6jLi#Q zQ};X_5dEZ#Gey$j)SX!<@n$A=*p{M8SS*@6(uNr0GE7?m%Xt=~~+8{FT|Z?J_f~l^>_V%W38lAGp1tpWUr^ z3}UmN(2V7Z97Ag(2yqC6&I}j)xaki$FVK(5SNJi!dV+q@x&-eXmqVQH4IC?cLj~o0 z(dWfseEH`Z$ll!8cIfXV$PVj;S!Hi&i>o)e{X>PQ+4;ccZeA_$ehk#zO2@1hQ)GV6 z1x!1w%JOr%1i_Mz*&o(cqJGc?c3I+Q6u!O>)_#egbJI(3&um+^*7QHb&LJ6=Uy7xB zg;LP%_F?d9j%801eM9W_tp#EIS-5Lg5B@RKWj*-25T6%CV#8*BxKU$-+l#j1_w)Xs z_kAH4EjUXQ&PjpW)q3iY_K=-*E&;;6`(g6Ir*u!)4@jx_k4d~VPS4I-z!^8#Othq9 zA@9p32v+hl9o`cJR;#x_-Gddto79RGTdnEaOT%Pqd@;e$EgmA(|B;(7?@Sp@xO5Qq3-4Hmd9hS|-xAboZmuK8^RS3mp0hYg2l zuwFQh#Lj`nP#1hZp$f;Zh;z8yRrJvAb$FMmK(<;aJb5le%)jbk-miAF=YI;SKg*!T zdk*frB#33Pf0_MPdgL;~a%HPMj7?UIyuz@ei(>mNT}O&S6ciHqPDT3!xrY2zN^f+2&Bf#6OZ_kGxw9Ap`l~Kfi@e zzu`wukH)uEt++}QTQwkUIzKnXNCz(-oQ4ure#mwyfpwE9#FujgEOQKDL6t97{@jN1 z*R;Ui+c~gIegJ(A7qU%;!(gZFI6L;X4T?k@W&9X6oV&UK#?)M3W>7C?u5}?2nGc}j zQySChr%z3nejqQN^L(?T9XL^ckaU}L;L+fdu@A|K z`#GMgXdxj~MoFTCN)#2c_byT)ZL6h}`#GMgGWs^OwYRp0wg&y4-(Sz4&${=1&ilOI z=e)An^4J?~PYk!V8D@mBK3h!quWq8g&BGMlrc1e9y)cu{OE}73)43x2aZ}{A3{GHo zt_)`v%p5QLcUZ%*dyy!!cKw;C{u$Y3_Pv&uJ2s8I{bxLH*mhO;-_j!XNS6`6&~}EP z=h8WLbdpK?qI_`D6zyKA>3htYoh%SuH{#fm>+<|j$LYe(r={HC)+>V53l!On8;aO1 zp>{&g$&-aUdK%ez`LTR&njatTHG*F}LWkXZRGR;B+}KfV-cF&q)m1*Az{qJ-(pmn4 zqNm{TG0{D{-h%ay&KK@kQYkDuH%jn%t)tMmte9_T*eN`6UUWU>G5oXEX8y1W!&k;v z32uLva?1SECfM@4p3VAif|F6-R@N}El`Ry_g!sL9-lo6bm!Ir4-l;sIn3c1YWXGOr z;Ftcp&G+kO^WY)PO0Re>T$VYOkBn4coBt`Y2dbyB?N^!w1u>@uXI$P3q<8hOmcRVi zrzXb*OJ({6ANJkiuUg;aH@=@Q+^aK#PboKM&BwkHn10~}p(S;^%%wW^{P^wci0uOQ zujvQDwv4la@<*j@2~J||LuX#{-kz)33vq_Rm6nNo-O-lZ~Fp)uZj5bnkl!KECf0EP9oITe{~v>DF&# zcg|YHeuyg;ZZnJ6;Y_*xw0jC_E^5%yUAB7Uc0yKqgcGW+LtJD>6BGh3dtmAxANPB`&oKEFU^ zfW5nTGQZ_~8r!j7!s(Os9f9?Ks%t$%NAVot7&65C-V{e%=mlP zF1HZIobw%1nXpr$;RPE-QHP*^>SDp@kPdb7OPY*9b!Y$hE(Xp2V(O z{7jJSe3Dl8YCBX*TbP0Zm?sePuEcS|M1H_=>Z zv*=mz{AUOsJ?ScYWdA2tqL7ES0%YI+n9K&fljjYJ$FN=9fqc4$q#)jAC;usR*zvn| zJ$p8_N3e`W@++hZ*r?C8Z8Z-z39X;>3q~715}xS!BMcrni*Nj8$G&vz7Krkz!ldV4 z*~mGStoQdz{2k2$0-M|k!X9xS-pA@2o3Vbgu%JeXpZ2Ij7*MIuHefB9Pqo|4K2?@y zW#bk*z9hp!sW?@EZD|r~>FXwx4fPj1br)wtALI*Om55decXskx^&WiVxUX%qSH|)R zw{Hnt+IZ?^NU&*i1b=UDv7n5p7u3%> zk4LQ1*?RLp!Lj)RY?RDr;l1>5wyf}u@bdU${E0Q&Sd|&)dGFpFwu{@x*U$dNH*Lvf zQ~OHUOTaY|I{+QY>;Qev}PYKj}Ug2cJP(j%XzK3Jbszhe0IP8On$RonqZ6lMc&)`67w{z zlFhTv{m3Wso)hd$?E-f1`#s+FC3)60?BMn_n*UjXfjK z5NRvD7o9>a*y&i)`-Iguvvay|sbApwT(bT8`beSblRC!(zD+!*I@EUH{YjyOVtd=D zzVU3XCmRW$5Tx>O*FHU_B@3bq>7AICbKOsxL)w2CZCm>`c=sNRcKEd zLjL^qXuf|{fzUfE*U_?fJ)b_cMc~&npN|Mz%A1UlY|mNOEzl^uBk;GH${&^%?Xl`g zZyP_Wntj6$2pT5Hv155X_Hy=Ew%h1FtL7{&9MHFBdtC<{BdSf=PhUd$m~}RMoX#@g zciS>nzg5~PaYUr!CAUg`0H?DSAwz8EgEzv34~N;a0kTdjn?|-zF3b=}K9k`e9Q;L> zH~!;N$5+zbS2eL&*A!~DyTNKzHBhgZ4F0+=NVAzM417?B8I7{Azrq#&70Hrphagn1 zwMC8mMPOv@0!O|&LfGUm@H(~$y7xbzsi&vG!pAWXEa8pI?xm5%Y5};ybP?vx{YKp@ zEF2DO9io3CEU1KyGJbs_4-tpw;`@<(RDXQ}v&z;8FYAqj6#4t~{?YO9F0+~W^+p!* zX6b=$L>ipCz5v>*CPTyRO<*qO0v=Cxk)L`B4!^9 zr2nMTTSXUi?7TJP>#Oj5e+Lm9RbD-A;k85Dr5&tfCZjksCbDV zD5-uWo};~B`n&rrDT6B^|JW4x_$~oIRvLh}SSq^Aw8pVR!7$HtDlE8^OJ;EL@Md!Z zdDb(MKAvw)j&J!!o+^#R0uP}d%7-c+rh$;qvy$sS+~fAn<+R$_d0RB zvk>hsuf&Yrv3T~_Vys%IiLMUT=)GYaz4~txRVXc?9*JdiEFTDGzv;q@j)_n+)fa+D z04i;1vA^nGL-XrTQoj#5m?@);v7-tx zg(W7gGs5VJ5_s&>W_;C5xC`~ww9P94ZO)yf-yYAvgd~4-nKun)tXM_*u0JBrZYjg( z%n2a#Km)@znL_^-1(^QD8zss|Li9~%y!E4>+QpWWwfkC0ca|zi^X+CvzuF4EF$S=4 zei}H8$c4d(5NtU|X!L*$S6vu_zmmTbj|oz^zef#B9<77VE7IV=LT}RC{e>KGb%5jd zBGFcIE6EI#Lix6PboN{i&L=LH{?fD}u9tq%i1ZRFaYshPAO1!YdPBG_@k7)=#NE-` z{EM!$SxYh_k24=<8=%FCN8I6|ugspVVDd9Y5_MGu$hxP7ko>3^uC`<_;TA>U`6mNr zeJg{bYbT(OAQ!%%$qSK4LX$E{#X> z^)T&kT!0q#;ka}5a&WwMhKR474DVj6!y&N;)KE^BjJj=yyrMeZH7dhh-W>Iquom~! zm*T&Gi)2Ii8W@)EqJ883W8Pep0f*Lo+}43@#N>J%IbDzs4K9P^P5CUSB^mIpr5L&u zj?+bfd5|aX4i*{f@S(jn%80GR!FQ1uf53+cPfQ`_j`@<(pgJxv&yjhT`hz}u_MM3O8xkTu?GzAhAQo(d> zPHV!|c*xg?hy6pvcr!w@e|l&m9t_Vyn;+pIV|$c~4V(vuR?md{CKK^Z;Zw?5S29a% zF3{~wrjTUxmfECD1@VTxWFg$5UgF-^l>2}#yM3fpL2)UJGgt{?hPz13^qHXhA%=_k zqzB<)9du#IFXrTfXkhxfh|0oH5dduq6x_)qegorRd89J=Y4MY|?kL2i3)JzQ$#e|2 z%0uhrGR)rfr-@7LL@xHq9kTD?V%&D`A^rJ!2|kH7Lr?c0TyoJEYId{u{)0JYEnI-# zORmt0BVpJS9E&eAH4=y3I z4ht|r%pc_Tn2~L-f^omoRr=0s5-MKxz%jvFuw2`Qj@a#sZOtOS;6 zspET}<+$fbfx{bxIT#nS88xg5Fny&me#pwh@!#^%p*$CsKbZs3mPznlA`doxdqblm z_RvIED~AJHUUDC*{J==s0~bF0NtTBQAhy;HuC$9&tHBviw(hrRcm5PKF=n7%+5zv5 zoPcAEPSF>6)>x7GlTK+D$4k9(%sj6kCfdD~bVYU$jqkd6=)oA!nFx5IXaia=x=22L z;JLO@+T4VmDa3Ss4tyCW%d{NW1pn=phDe1HI`gF?%zwBU@7j~JHe4s0e#Jv)LJrB+nj~86O3A#M>*T_wWEgp3Cb&zjgq%mIVBY9S z+J}G8r+?$RJSc6Q6LcUEk=NS;06V%7S!kYNB-s)9^`F2w0RBk?)ccFu7G4_SzTV zUily#2z5j&^GsZsn1sHzI;h#HjsEvG;|HZl#3S&%!|2%?XiD)~vLh@I3#G+TX(#}X z-}l1Jj5+wW6g!yI{vy5=@^HiJ9!ZS4PxTxxlUb8>MSLe+7!@=grWl+cXN*;056Cex zYo3wcO&wgJ(_&KaJqt;~d+IP4kB{p&U~1}gRQ)22PM6P8{|!sf#Csi>M!zE=1yS(y zv^iKWONGDUV$i&FC0y283t0`XTH7zLB~Mnj*$bTqxK-b-lk1##ZqjVG!8#}(qQ_}sbg$i2hDW#!BnkKlr8J0uf(Tf)H)S7tMQVd4aG$N z!A$UriNQDfZK3^DJboSfg>F6HN&1`Px!2tlM6b<~hTc=bg3^C<UH|-b`-r?aF9t^ECq+`&T&@v z6v)00#)S8NM@sJ=C!ZD_AKP-cWD7!xZT5ut~k_1nNlMjG_{Ho&vrE1>w#8In}G zwzYnDF}3v_VRee^tCb{M>>T0rY6DEYDq`I!@1c`^y5f(y?@00X z4raQx8WdigLt5n)!Ou|z#P!zEjotUM1K5Y@H5 zmsHT-*Aq2wZ3Z@YHcnn23Txc$;h#2x+6Q&1V!tKE-V%L}h9dNxY=9@C3UHc5HqOb& z5TWc2(k$cOboAyBoE(sf^T$oY)vXQ~cw;>V=UL#a_Ck6}QyX8m+u+FqOObX6&?R<& zGCKi-`A~E?bcV)XHb8Cc_H!@2&O-4ge91{pfCow(202(dD)Ow=(oh;s6R4+-Ci zd(|OQIz{A}bzUUOgQk$l0HbMY0QG-uz&SUbMA|*#F5M|7H?t0s7spaz!EawUQIQBc z7NfdzfSWR5ls0wrcT;mSR-9b0{%b>3-GNGgI&^rhBl5i6dckfUJX}z?IXugPt zyzH`^5q68gh?vgSQ7q)b^G`y$}fI&F|UYBOT3++zw{@^)! zwWgXX#u`#P#ZFq0m;qZ$<~#gq`a)Q%O<=6+2KIIBM5SIDgy$zgWI_!wpzRJ1_vet* z@3G{1M_c_G*)l?gqw9}>y9RPYMUfI-((d~Dmq`7cYu2Juu3 zXjP-iRf%M1D4#A_P(*q=Y$2?wxK*>n9c~2uC5x9Ap-IJeYBq5`lKaId_g@+=cou?V zoR;Hu?IqY5nu%5J>uK1EHQ2j68K)K|H=%sZeL2xUMS*TSywc= z_=IT1n39cg_qhj$m7(9D8l5Fhkw1&d@S<%Yj(@xe)i>LqOx`@aR~3XhJsm`*Spt*x zpXV~qFD8#OJ#NRY``7b^U?Rq zCalP~N&Tu7(cr=Yyrc7$oYmIAmFG5+ktLO6Aafjq4=6%>{vbJ1b%jWv4&5Ysg4=Of zjNUS6V76yZrj6mXbVE}Kx!m`fHVjPy|7$6n`Shb?PPY;KHQYfipZ>-@R0<_4XN;vK z?pa`QEP)YhPJowvX#g`1(e*YFSnW2C+gx*sjt){U5#?mR~9@{911h_U!X zRG-n(ct&E9EPQm(g^HC(m21X>ttL-LR6nG}i=yzO&>#0r9EX{C%khs?AS@haa-xG_JefW`%y4mP8qYmNTKQ8i{wtrXxNgzk6bxq%Nd4S zf#ambv~#~IG&Ky6sd$|3IwVFF?$=OXdjt^YdLG7ikUxmkHoEI*piYk_YME0He%?spd0JxZ!YeoWd#I^95R(pFzZkfCU^CngOoxe3fgah0AkpTGFY@PM#2q7izv3X-x?_<3 zUX{i*Rf(g;WyRL=vrg#f$)b(382NAQEV%quo8f=1;i?SEaE5|0Ikhqso$KS!dHG7* z&Z>c@)LMKsl3?Z)V>q5_0j*IgFva;5DND11-UwY%JVFh244fx|=Ns@==nd){?TQ~S zuEyOXI!TYtA1+<8o0vYGN4(m_!KcF;)1wwax1S+wS(-~K{(dKSt~}-hdcPh1y_}D! z>F!))btApwy5P3dL9R;_)Qj23AsE-;!2OA7T|Dge` zy8WBpI;)FgUWtKjl^%waji7E{T_Nb@4U(H?4Bt$=Kr6fouTL3?4#NxaqE{B$GQX(D zhkxXHwk7TjT!ej#m%{m%8*%HQ{mmh?*kuK!X zib>$;jrjbF4;&3n#@%b<@bkYo(tNm+oWFaBxwJfpR5)2f)p03Y=KPs_TlAJ=on>*{ zpDJ>CxRF}6U8af;{v%&^DHN^v6DdSVLXgdGC+kL zk43eL7%JTwq9b1ZregC>F~JQH4(FB{;*SUKX;t4E7<~GdPW-uzdE&Vg(4dLV3@2c0 z9s#$C%(3^WExA6opRS+hgGZ~c(JK=exLmJ^E)5sC7?qXWkM?0w;-QQuQ_j$b?=4_m zYd<%BgCu0G5k1pHoZ)jfb~3__1`)43m$sg?$B#G4$*7dYcs-F|e}E*cOwOmqU zxgO-@e$uQ*CY`C(_v6>^fmjh9-aI6Kqy=@DV!_5G;nppmNm#W>=NMvV|3J zaeY4KojgGIMc847c07KRr~uYU26~5=f#k~_FxPPjED8395hvZDBi09oWpqHQkl@T9 zML6HPg45v6(JG^2ay18Ni#9T=A64PTX_?e3J_T?1l;9ub>9DXf-~P%iOW2>K30G(B zAOllJ!R{Pq66G{ZHrrJ~gU?~EE~XsjG?YN!`cgVYbaq&Bx4B{UN1XJNb#!jc7Fx9E z4L#t~Ox5%{5PQu$x@mw{Z#R%C$; zVp2wfqt{|ZU<$&gBEXL`TYEoe!hcc>80d97giO6ad)~G&KSIx%Msc{eXDFS6OoP&#g?dLs4QiJ z!y9c;+<5_+a%}_Men%GWt2%;ajWy95VFclFcj$yWp3wAjKW%?%k7+OGVXmPyv`O_- z*_W1xCr;A$y942O>_{wC`@^jAGJ;eU6SQqIg5+aLuqrteFB$hxi4JYdvp2)rzs}L4 zkVJYsmGEE0bbOWQ2jTDUk@cJ{989oCHsk}VRnH$Vk@Zb)lhmd#0w^Mv{JVy z4Gb~L1m7!%xlh)c;rqmFxD;oI^2HX!&Q%MgFY4oiXB%*}ju=^&c!&;+dP)u4jG!zE zutiB9WGO9AT;`n`w47SGYz{!SE_;ffEoh`TF zgtm<^L4GZXmFOcqFdr&r)nLZ29D4YD0s0-@ik+{2GOs_sW4c}{)7@qwvb1dqfor5}0c5v!dLayQRNc@oQiYV%@Z(bRO%g4>e z?VFzvYu800=3YJ-pYG=H=1ObpL9JSPGEk41T_NI|ap_PdI}N_wUki-n7MLth14kt* ztb$2!uu}yr+XU!4@epUwIvsnh^>O6Ga}4QrYBh~bp|V}QMCaK^2d+>W*QJbyWh>1| z!m|P#_pFOpirDJ49fmljviY>i4kvJ!F`W}A$q}vTsSv$bg)aP+0*(3WU{CLE`Xfq` zth((&E*{=SZZ{TS>S-6;wKo&RALL+yk~g-??jqa2HqrI7=VJ2A9T0O?#9cM1fbCHg za7jsxYEI$FQ^|0;F77^^+v-JhUieZk*K(59Jf8T@oeq4?c+40!gG=6;5ZahRqjtFw z&6f`t^%uURV*fHcy?PREY`I2n=c-ZNFk|qkn+bjCos{2hK>S7?A=mEAh65h{ty)&Q z$&T7eZpokLB*lC`Qxe)p77HY(y!Tad_-HC)NQ~&YKwoex@1$R@Yr{U@8E|HWA$h!o zqk@kY$nm!|M0?paiic%!Oh*T;uGK=57Z16-wT__IP(myFHV z%)Fd{H7jP|=jS^)&*(O0Cvia)gXLVu&-IC?Q)Ijc*SR(^N(bk1t<~;W8J=IF=)oRclD}w_Nf$ z{VjL8R18O@SCb;`3lneg8;Tun}$^zCx9R63|lqmB5h)ZW3VZ5odYuS8+IiX*&|7$*C3 z&470_fps@Fqu0(i#O1(vlrQo^|Ig|F*WnH&b(`T^T{;AoAQP9n0JcoOYu~?V1WfQQ zgY`<{P-`dhmR1*n>4YV4dD;Xz<3#|N+Aafy1V1d)F~Sey0P8FiaP6vX_%Ae{xwSeG zm-Qr~bwdg|j6X`&uRBTy<5ywB+!QMCT!EYQHOXa-5-vEYn?@AOg~7->^t#1E`nNI# zERIM+dhuG2pcxRjCI@$RWzuu$xj6lE8qQWI1&#CzH1SjhBuf>7sH$P+UflrVr+nzn zHRHgt?j=35PM_RxTh1MSev6vy+=jtNja#cXM&T2qe0;E`44xh7rEEe0q?sAQGj&tk zbFPWLtgz;!R;%It%Y#fKXN-2QCcrGOA0&RJDNGR6RuQeabjryxlBd1|eydMGiyK>E z>S;%KD@?}5OlM3i*2m<-u{d}%6(ei9>C%aL_*^jyT|`aK@Tx&-`=E-%&pODoDo0Yo zwR7=^*cMu4R>}>kErySrGzPeP08X0%`|J-n_!hjSdizGunr5@s(^*wi+C%|@f17Z- z)@q;(3C8UwPtiUfP24m_w8^9?6E-B=pjjfoac*1$G`SJjsAP}it`*EmHH3Ax_BgDe zft{s8bXvq*^o}Tp=T@`8NTN4AHPHh9)!_KM3q&kl0X{hIqKj|z(C_?HTKwlW9Ube0tH(%U zK%zRP9TVVFr)<#jNh8W<^MQL+2ws;*fJ}%Y$-5QHl-2hV|Itz4QEUutjWfXEQ7}aL z-eRm;6g|#2pNRwYO49 zpS?T0J)}k~l`Y`qZ#^9Ib|xq|>ETU>bozN#J-u0@g@wPnX;J+w6!S~P1N*BurAisv zQILt*c0rJHJCy9Im;u)ZW4ZhBY!nO@?UwtB3?@ zh_2g00j@-OS`{Uv3gzzDFkv&Co-H5)rvubCUK3A1~-oTyOz=RH8p->|5wvj}%`&a~~5B5W~hr|wdAoT2twE~$%Bn|y!niFrBQ zwsQmIq#Pp4x6g-|ea;|!T?;eaCV`~vQ4#mE6uvE7gpHfllRe94qqDUdI;*^6DwZ9j zK0h9i)wQGWVC*NR-Jr(d+N)|}8+?az88b+87pq`dG19jQKk0Vm0G#e=gLnQ8(rdFE zv7% zgeM=JC69`)lBn$6MBFl!$V}SJH8}V%ojNWUVf>uCXrY7aorBTtff%*FDB_l0^Fhz+ z|7h$533#n%gol22l6$uKRH3rXOfx1@Dp5=?d; z3mYBmAn9;F`EI-r9xixCx2puh>nXO-nivab6y6h$wyCg5H3us9Wkc$exzKh*2L5YK zgo>B%ncU~M$hxbC=%^(j}nZNN2kzeUZGzRaHn_&yl{lQphqP}Rd$PSl%aYUCj zOBgHK%!svJrHNc5thuQO!grSNM|wS!-d&A}-( ziRTqh)VGes8B1Pqeh;KT$NfJ2RXLu!Hg-HVxrQ@(+#lN6;)RdZb~78F_t9&5F-)F$ z9~Y;#6|`;L$vrJQ*sb=EJX~200^jGvS1%p(`wL)B<`3F3Q`AS&+)v9?wsE=pys6~& zBV?J2G^U$N;G8g7NNrXkZ|77JxO#|a3R=0Tn^4uvd-C0q$Ks@p)n`#p!5qI>UanG}pr9tpZi6X4qXF0x{d5=i;mV6$)( ztaLX+Me9LoGxUTpHkE)oQqMV^ov%sFIR`Y>QKFk9r=i014|J;yE!m(Z1G~&NsWa2jl z-koaaj;8jL3t_snVABxy&>$8|gcYPLVKg3|(@Pgh=#gE23%Qj;BKC622PR3>BMx{K z2a64LsPKI+8Mvtm9-4JzRl{ruZy5t~cF)4PWxdp}%@B3;uWsDG|`!Q!6*fRr9$~VxTBH25n9)K?=ra+wOWs>Nu59>Nt!EU*GHZ2@{dF`Ni;ahI50v4!X;LdaI|dASZ~WA+4dP~#@%Fq_`p~Hw%=C^E z)hVU0f)~|ytpU(%V@4-E5&5Gdy`duU7_B@fi{0*aWamK}OcyQvbf}1FZBQ{KH>?gZ z<%uG_a_fmmi>8o2mH#;yHASMqupY)nEJuZTfw=IR4cd=l(cSn7)vruN^*D2UVD*>& z*;YhvQC+M*w~PLDSAh?Gikw-p$fuq@5t`)nnemZtNaW{S(myF0m%JQH$j&vWKb7Eu zVjT#XRl?d;jg><~!0FC&dL+o}^F7pzChEZoPR`i!MK8PgOoi;xf@-`Z~o{`c&gM zeX)KD$g9dhb@+F3S7|Cl%o~rpRK5`@YD$}JHZ$Em<>>H3jVWo^hQi%DaQun_CQ+xUH(QZLX9Z(cI7f^h57C(Kfe`)5o6fEu4e$Ss1Fg&UaKv*fk=pAA?ZpN# z!bpj}f2@l9{u;vFj-fKEazS;gKM@RC!E!Hqh%Qd1`gdLMnq?WCzq*O;7=BJPwIyL` z+cze0SOV^O)jAZ4l%tdJy;FKQ-9+rLAKX8>3c_oQ;PmowAe)ecUF)Ba*K4zJo%=tU zBI*;zYFR?eT-s`D+`|M--ASJ3tF{DLTigrwsfF%;DL^CUl)JvXft-rE{OttkO=8Cd#Y1te` zCvzhlH62N^KhA~R>}6oPZ>RFT%f8u)cDhAfV+6kRW+5T9X(?Y5JN zccBJG862lYeW_d z_8j~wkL2I4<+$vJ9Dco10Fl;Hh}QWnpx%`Lk6X;Bk!udKpjsLi^pr4#oQRi1y|7Za z1x;_1;+p4mxbg8hPN07P?~bU(>&1V_S;s`W&e|LxFB%ElJ^7htk2=T6=~UWZ=IrrQh7$G6y2-qW z6ZtAPHj|_2WhA6t8Ln#N!j92u&_;q_PIEpuu(mMOLm$KRZQ)_bE^^sf2EM$V4LU)0 zL}%ngIB5T#G@4F=5!_x9n;-?h-tGd|%`>2M%XYYWWGkFa35Fj-VyCyp>45p3?WAVP z4p4ECK7K<{=DXY&AvQ*TonCe4Y;u@&o>)(WRw?PHnBixYV zUOM@!8s3!&!fSW6$Q}zZjC(&7r~7Qdnq^9OHgzo>w9>`87)Lk|P(sX-m0`7~1MGg3 zOHKbOq4}shhCO^2MpK;8}mk!zY&tS6Sw1^i#8ZF zkdJLGWw>NwKAhX}ffOcZ!QcEGICA$FWioTA(?@Z~C**GRh9b2J zSg^kW^dD8h4DBOS+us&MM0n6!S_E^#W9XoICLH!&4WZk%qDs0RX8$Zf;r%>(e5R6l z>>%=DW_i;MCm9lUI0UCMdLZfGgLJMW&iauH{eFcGsz=SC+j#@Lc_m5X>>@xc(idw^ zYH+he&-Z!q|7fz+4rU4NAf1&mV5B`0HAFtB?Sg^R1M1$m@KX<6c<>WFu~!C5K4(C< z@o1P8lK`oAvLQL>D;??VN%^o*wCtV~Y~8dBQvTC`Pn|yCrxXGp%2OO~Z6NMG>u_w+ zQoKGQ87}oq zd*Gu)7wp^Xg{Q7clc54{G=AxZF^b#h)uy*p?ngeCTk(k*=dFV7H)|<+#nFYQ>*=Yc zK~l<%!pmV(;K(^~_|1xu3AK?Vu|0#|$vpu3<%Vr&HeW&n;et(;c z6LZRG((*{$dDR^wADkx(YpxM8E}Qnu=qLUW-Q@JocjR}y0x>cAO01I0aJJrT%npph zi?IcmFgKdfG4O?YRu=VNhm$=b4Okx~2H$fglTYnpApI^6=MEb%>*r?Tnx0r}y6lAa z$1i2F3P!a(+4`d+9FTCH!W z){(a~?^7s@l`Cds9sEJB#UBjkjl~@w4d9ivKk62}p~<5KwDFyg%9w5?Vr9?i=z04Aur!!)wDVyzuN55#CP5Ei?)XGV&R* z=0;9)e+r2i-And4c5?kq%9s#t#c0~Bp7 zME+q85|0#A7&8X1b9$)#IG@IoW_t2;F?N4*#3^|j@zlmttp6^`k>W6!K79dJte%gj zeV>rhI92>B%8||;+sph+kwPm+S3InJh@M|$kIIE5s2=o^rdmc|a9cJm5%n*xT74xB zZ|tE*EEF!Qj>kDDiS6A5piXtRSmR3-ab}BXdxXTD z(J+wD(&7zGB(Lfs$vo_UUVo~I)N5ND#TlW&0s&-HR5DxK3}EvTTTqy-1(%zWn7zim z1Y{18ze`IHlOu7ht1P-+%EBcdzR?X&RN(Ih$yUWn|B+wiaai+v1f1F#id)ueV7N6; zHs5VL)iaz+m;L`-HXey1qr9*!FO5?f;<)6;IZ$CU29-^nVCPz8aNVIs?7}uvcjY_G zo#w4%#@_QZ#q!q%b}yinf)jMK=#V*G*FhSFi(&MYM`YKUtuW<55ttv? z$H-21u|KDmOinldCaDJw(ibCaVG(PH?`mh$mwf?ncaaKq=EMSho_adUDFNFioTS3V zfjB6=4$T+0(xwMr81XS?*m<*(zPfmiy8qS2w(eqw9=T7nNhO*~E;J(w4PWTuKVyh( zMjG4mKW@X~Hy$Kl zIpAZ@G%UH{3pYQt((UKf;7#phD3MOahAH{2T4D}(ZjTq9=)FQS)~G{td^Me7IKb8C zRpG*CUYvU5E_4&yioLIrAgm0y-S^hRRmn8?`=^eoCm4Xsz(o45U!CqneF*^c$rddr6%isSOG-)`Y1bl43RzM_D%9LF_cO9aizG$1C~H}=RQSg4{{HTd z<~6Um=bq2=d7kIZtg{nQh3l5$hgi(scw<91d_GJoA|_F#sCXK`%7TuZKbI!W4Py-o zd}vvS9{tc`Mdj<91WSL7roThSRL{C0VO-*jX(?TVjI#5YE^ZsOI8nw@ypr<$iu9zD zHoHUZ2K&A^fju+Kn0?$TkEXpFi-zm#3T{2^Vs7MqU{REWWpw0eXt@llesCe(ICMIn z&#|CZ`aZPk;#=lr>|-`)X9siC>>INqh7S!hpV)8zF7guNc$%i(!yL@fMoe}k<2wHk zV;r)O26_vr``2Z($J&;z`ICS)6{w&EAN7&)g9!Ae)r#twrLyz4=CVDrN7D-%Md*va z0Ga%CXVl!cqXG{HI-_p3P1T#NG_-deo%w%Hq?`+5m}Eu6<>yh|ndV6IDMmI?a_D%t z9SU3(Otqt)GY6*pWJf=2WH0_zq_=JRm|f^K@YVB8yhv~VDm3lCa+(d zb@Ex2%OrWWZB?S+;2kBD^I{@$u~{enG;b6-9kCE?_&8qN(&31X{+Wv;Y$Dn_@2dFn zL0d#ST+lf6)kq`im3V9V5~QIp2bsrC6igkdjjWC*vKjj6OeVhN{qMJ|rW^I;Cr?zA=^F`Rrre|GcY>PX@Mp?mVqn<9Bn5d7ARJ>p=p59K~ zCQYUjt#(q2bD`A6DU-V8US*dr+eiD}rqURvP3Yf86`S~0{iQIw406+BT7 ze}1b@o`i;GZl$|+FQg+!Sfg#S)97qHJ5;welQH{NDlnXWoz<3cWL6odvmvda6qLGc zHaBnKeL_C8bLU=kV=3=_f4&8!@LmN~Y(@_W8d<-~?t<9&C)oPK29$hTM1u;dnLj-f zXyUF!^!KMC%GFmu(tt^5%*80gR@E{S>gFPct3k;9eIAWd$zzY+ili=8Y1B-_?}u7k z=+oaTkaoo~^v7y9(l(GV8>hvh;juwzWWP0vQq!g*hFUO33@0(2Z)2JJ(MHr|{ck3C z(tUQYA%$+%UMMKO5zEg%>nPgoPg8sI*i)XQJq7GPJ~^gH9v3H zRDbPW`YB>RyW+q~X5(_i=I&c44se)>%9o8oGudZ?%^j9#ML-7|(6347%J;Hgr^N}@ z(ZzJ9B!<_kchVIJI`o)dB5DclWQNpiNA%VXH2-!geHQYaSvTK}YWvNndf&FuK&$hN z(`jdVb(l5PZd^-$PUgfu(}&RZRAA3|CNLj(@A=Z#uB_s|R>n|X2mLxOgZgD(F}v4% zW84;d(_3eaP{1Bw&&@Ppm6TJ_9i=A5@3I*h6%>qKJ+-u%^xYI~>a=05U(XPXtH@`s zCT?U8=CrV5cMoG*e%sN4?Pl~;*AzOV*^a6QIxsOq$2^-bG&qv3s z*&JEQK5COg)*A_u$lPFZ-&ZiIRrzS0`!nWkYBVZ(!DnCjefthx@0w=P&JND~%G!7K zGMm=FW*#SKuoqh{Rt3%$p@&ZWjAH0Mrm;N`-C22+^{8IOHkPWeQhrAFk+{=2&Yeuu zcN#0{r9L>#JjiO_{7>#B=7rn;- z%{vuCqrPlqQL-!Dw#SIB(%DMAdB5sIo#m814x`?~jp$(HZ}!rO6JoaHF?%^pO>E?Q zmXSU!r6bB@1%<7tHs7}dv;JKHY?JgW8+P~%8y4t7Juci4jC$I{uD!3o zl!OhV-_k_%iGl*vjnbw{g=6TDXx@+NWR6OMo0tOYu}D$p4P)e_jwIo|jQOs&%&LkU zRB0!RK38O+pZ$A~Z|_R9e1RTXv~U@+ALEPWM(VO+sSY*Wa*v@w4NUJdXX;=6fj!Hg zldb9=ES{N-G}i^O2AYZJcS{@^Qa*yp>Mmk;Emo$DtJJC6$6Dr#TAg65ssi(IU>;*q z{+%^>IE9-3do9jLk+L}^$;cd+FiZJd$#mYk!45Vv`CgZp#tIPBH7pSSH=437lgb&R zA2-CV`tJlwHQ$TN+|B4H)o@m>%$70BIn3ll>}6ICs8@L%Y-Z#}R55e2_1MXmeV9M< z9x;=iKVw7X)M>EAEwRJPF!AnJ$`l_Si*63@VDs~Ht5yXEp)Cc@C?H+~O+LK=owZXz zvEvHZf1VnsU_C!){m4fp^%Lm1rIBdpnT>o80I%bhUSyTRA25e(Mljqle>SRSE;>AN zj@aA#JhQ1#z{r}9r7w%`3gQZWR3~juKzSjiblib`XkvCeI-ME^qb0*C5-}9NsEXr z+c=+EKcvjtMYZhUs%++EfIf5LP%Lxu8}Bg+45NL6UN&y;r%{tm5&byq08+F)CN^Yq zkT=aix)IOVwbyd^eV+&GYc4B3b}*CrOcu~Jf#c}JN6B>Bu*dAPPZiA3sfn!i`J?Qj znq*e}&M0(V|0S!xP?vpP6NhsC^Fo8=Rw_JTK0kWK^4Vl2rr#K?H}^r;wapM&u?Ri?xryGDA4kn!m{5(Z zRdm_8c6Mo8CtGJ5%$kfZuMTcsMrR#yrzN`-X=j=ewSO8Vo;y~^++12_{bESJIGfM+ z7Kc=^N3v3>ElQ!TEA8mxu@@O*i5rzuUr05E-etOMJdnfabL>-ZTjsi%20ij>H5H!- z5^PwjMBm>Gp_>n`VV^uHXTvK!>BCE#sdV#1x=7xhDo1Xi-+T6miwx9JyWJu-e1-|@ zEU@Bt@1xNAk#Cs!sk%sY>U?x>VS%8@k>6*$9E)1Vwy_ruZf4>S-WM-1(4!i+lxg`+ zgytGNXEz+zrNaAJwA9{?TJK4s)qTnI+A)7L>U|&@y~7S^|675^JoZOAr3T1I#t9iD z1S8YW$*6ba5aeTIfg1b?+ScujUc7aqubnLU-N7y9-2Q06oT?j)yYXaZVp}FV*keO8 zEFA@}LnhJPrJbyJ*CBh&hJN$ z4`C)MatO0})Ms(umpjbnYYS*!cN+8K_8z*?ZY*`)Z(eP2_Br!p>wNJH&jhw^s1EuT zZbz?OS3`rdA2A+2+35A*+bmbKA30R6M*a8F(ZakT%;A*@DD&2CG<|<5bN_@8`5P1n zqOy)N-A0YfNvo@DNeE>&e~}S4qlrvLs~nBC!YHQl6ys=YK*d{}ZC3plV1GLIv-%M; z5U)_6_iGizhshe$Y$Rr`PBLQFS|u={#ODyW~liB`Gz(x2iv)e}ZrGDaKU32vwG`sq@4{oY#Ser`0n`q+drYhDQ6KGmzfv`*m*yL9+Y_Q$tj^rn9@dwbYtW`ABE zn>sI&-I-!RIV}NWnb=}I>1Cea<(61FiQC3RKA1`S)b!~3`Xea8tcVHawXt>I(~tsP zz$z>ztcPh|wdM{Jwl(SsEC1#{@y14Nns6f^)ruD4cJ|A51H_7W{Kxk*e|1#&&l@Zr@SldJ7?{!Enu+u|dy&!JdYq{?$z;EM?_zg0 zWuq&;KLuAp7ox23lTgQES31sKmMUzvp((d@=s(w2OmE^(!R0X$=6#+8!Y4lpB55A8 zedjH<%-@y$+Z{`b9jaLiUjJP=#hgCs3Za3MSoY$Tar74N$DMh{o;u$7z*+>1q9>H+ z(6SA-jK|d;R;HRVt<&FCM+cj+u0vAAcfab>S>Gqn)tX_{R^E>KDM9;kz=7zwf8lMTE~L6|=2X(aaRSr0bx* zKK0X_%IoGM+5T7^=Gu`CrpahIirW~#IJ0xmzVo3-<%XwV_2a9|Vw?GBN6ubmTZk@Q zLA@D=_Yc_dnmgIATXj)N=X`X!Wim3BnZkSXM96VeLiO1=V^s6gs(MRF52NzDojv={ z7=7$dss3Y@&lE1LViMlVi1*Fh%QRR{Lg$Z>C)DtwlvQqg#A^U&JK_LBsiy$CEh-CEB)`5o%!?8;$k)UEd}Bl*M7aaa|VM>;Y;Rr=Y2Bps%>FO7Acrcdos z53;7izq0Fh7BXk&DKnc6Y@mq0FDZ{2V26wJY4FTnY~Q8V%$UAncADi_Hv3L1bH`ea z&+wmO9O60!f1YhZ!cGIUKV}BHXXuM$n|m1Mj~R-JJRw$d))n;bD`wma?bw4|ETdU6 zz53(XLH6!PiFjz(U$)%ejJ5VHWGvRGF*={4nMXU47`wPVkV|r*VClPQ=$%|I zyKGoqRm1)`dXB$ip0U_Tzx`cLNA60aj};@B7F%EH>gPwLVcwL)$#ce6En`x97jFc{-@O zcoC}5-_O)$#4s=U(joI{W6)`<9!5Clry#zML3>Y6WEy<;vXUb~sK9Cz8{VH!ZDa;m zVL%kc%Z^f!yE`@8p-$sdy=Zp6J@xt5TlH{CE%RF8Dsy9JCfhAchfIs-uaJ=&^%jUR{Mq-L4DAQocoM8#1>uu>dJpUSJZ1} zgGR1*LSYH6==g$RX#I(N)^OJhWcvLwbED3RE~t{F68Br;nfXR^&%sC(I_p2dD78)K z{F?yOJXL`P*$+W+8^%z9yan|u%}3>L{88(o_2`alK6+mCOmI~7HJdrds!Cr$zyx19 z$u5|j&wh?5XNO^yRT+^)jqa7RVfO2($Td7IFysl>M1f8ViGaQ+w zgnSxYR6w^D&!NBMyu!T58iK+utD$T@*Jb3tjxNdI z_ckx1Xo6}qRr45%CM4WoyylKa9xCSOtCm0VoEgMEeHDOWTg*{wdL();`XwIwbQ79x zyae?eR-uj=uFTS$e0J+2zI&=fovJ>YKs(2drd{E#^hjDXif`%`Wa4y`a!3Q^Ok05B zKMEO8>0?%&9E;`)_n@&`Oqt%t@u+xcAbS7qEK}?X+Gd3)t(h{NReZP1y#weN+CdH9ZA3FuV(7ZTl{E5K zgU?l*c5k1@wQ(z zf)i_hvu@%4UA3zWqU-cF&>1dP^v72rwSSpO9hf#Y+bMzm+?hdtIPFIz?@QU13z?{A z#C)VYK8?OS?7&wI&ZW3yC;b~V8(B|y#Jsck%fzMWqn$Gg1o1&0NHk&+4OiVNP}(_` zjaw7J*6ayot_5ea4-Sk)2A>C*(P6>p);&{HHPVwV8tufY2J7)z)48;4e?1$RRKg}F zeGnh>Imte#HlTkBr_;z=T2$BnEt|VjnJK>3%7pG)#2E9>P3xQebkBo9-b)_KUp#hG zEb^waF0t&3t#dza%+0cN zd|8wD-AUe?6y;3GmJw*bOg)>kBbW*!7PD`<6nKffm0fnzgBfyX3N^|o65N%U!nQ2t zeR12DAQjCa;#qGE8Mpdfj7OXgUHDrO6%32AnX6SQ_D~yxY=0-PukH<_Gj2{mH`VRM z@*Sqc*JZRK6uH+3KGtkEC_f<&UCn?y}*B0#v%1On_P6YE(gVM5$B zc$^tPqSNY#o~u=^qghjfmVt41bM~48JVmG6s2W!ayQ!QJBrmO}GZN6Ay90 z3o7tAyWvn$vWu&aRuJKL)&%c(gjX+qL_CH#k<2OP(*57c@iE%KrQMaqbK?KQb@&m_ zcFE>!rf(ux$AVPb&IOI_m&uU+5u*3EMZ_>=I9aFm0_%RTBCSX3@cNi;I4U&3(NV`> ziCr!3m~>vWq%Du zhuxPb+TD}PX^as1C?4XhBTkSTniZh^&XBWIHkF*Zv4eQk9N_fAHOXwpG!nVi0*){C zB1eu!ljqMq;tTO#@XAw*$x@_2CT#Bn>FrJ&9_&N-T5WLo)&PlCnlQ2M9G-3%49YFP zaR1L(F7Lr#PR;Z%e4BlgtT8SI_cL#~i_OQ$M=fJ0&Qd4aHo9`I_HjcCwg~$@%kY+&3;2G1jIFa(AUJJ`lZyzrMrQXOMio9ZWm(zCUcSed>N>;PbQ;#t6<(Mo=4pn01Z3? zXtiAy7!12guIZ$bcYj*o8-EFtR;{FB*?8%x(V3j;wrxC9_%NwV zN(b?hLHr;g9cHc1C)Fn7ge#9V37aOKhKd<)BsLw($X%m0e8*cEdMD)&hZZ$b$~RK& zs`Vmo{>4D^3=5(<%bTE@Px!UWK_auo8(?;gr1HpP&UN2HGB!Apj0;>#=4}lZzH@EH zaq=+gGHN{@(y>(y|2!baSG`ZI=`qhV~^9d1R7sz@Z-1&`bIl7G&7;rrVHNXdH2 z_a#>nozwS0@of%hZK&bo8z>&NTnwvk=)>JpjhsxS4<7B;OzO|P#n-m3;jGq2f~C?J zVy|UNnllH4vx?dv*|q_W$sS?NdG1=zZcl>N{)f|JhvR7V6Odn|EfOk(Kn;@w^F}I* z4vXzU$8{0@k?@LRlw&cfy9+zi_7W9o2zUEnGq-0~A}Bf>CT=SeB}10$lB(s$!Q3E{ zYhGv%KZ>{E6J#1VKeHu1gKr4>aS2WbMSU1;0ns*X zQQg&iq8++`8-3?&7S8{kF+D_9#QVGoQTtyTfu6*F~xP84PjrrGFmj= zh*Up)D$)7#n`b(0B|gtrlZUe$VcTz6@?2A2bm?#+!JcxW>p8uWuT7Ss!#3HRZd$4& z)}j{b>!y>qoxZSH{wZ7(4U);*){%7i(|C@25;?v715UW!hmt0FNoL(y218TCc(LmZ zu5OtM@o8E|CU)9$Iw9xr$@a;h^AfmCH-8JS1P9@l-EWDDdLMUT)M5NJKSE-B!;NIU z8zlEwP5evp2!h|P09vyQUl!`4Zj*~-$d`KDoIaFW^v6$lyT<^|#JwY;&v|%$MIQ>9 zw1;$Q4;Nj0WCyvvLq(^&PvLh4+hOB*EzM37 zaC&2u$)`7&c$Pyp%(FiMV-HqwR|=1U)4Ao`=7{}dWn~@ru=y`{N|Fz^tSC2Sl^92R z=91jV^W@s=Dm+pDEv9`uv)A7O8bbYrOJAu=v()!-E!8_nR>DMLv#5ug+G{}~GN<8~ zIqu+Um%(Yi%jRy>IKu)NMLbo%60)baaw%3?{ARTURAXeMcaFPr=GR?#M#eTU-<=CI zyPR`UC@1G*R+DX6_qe~wQDobKLM}3JE}Y(*iIXOG!eDh3SFo#-dnA&Omc}mJvPE9% z`7r=|pL*lV>oUp2ZTlq}1rbng)Pb8pt+vRX#NQZ5DJm+x9c$s`z=uN!7UBwqRy}@1<;SjI&9aet1!tMCfgiqRM;tn4_d@0})=nra= zk!=Am>bMsEGyfXNStl#a{d9-tQR{<=xg#mu%fNe^KGG4R4!8ekigveOBQjV?bk6-8 zmuGzto^__-)z!*Uu|SS2D)HnL$7Vy*y&_I)NEI2Xs7OkFkAb6}xA9q?{Z!LvL0TK~ zV0dd6cSY>UP4%8lh9CJQ8JIAV{_nSbCD89($W-aSUbEys6YL#zflI0uU|j(_4*J+FZJ<9T?;`T|@#{To*L zx*oz$X-Ml0u5$gGS?qnGgt(RGL*2}DF8anz*f#o%u*%Pjj4a(qTtB5kL0cG}Uo`?Y zncwHW4nF}CG(Y2I3Yl<(XOy1wNFsG@BOp4gkgV%D&+~=dz`%4FxUIW`_wil)23)(u z`YIxdYJQ~qi7Q;HxhcH2>lIeIet=k==HFjeY{+?oIr#3R2RK~*ys%h#95Jw*M{;)j zfEeRuNpi(W;=f}S%-Q&l>pF6oYiTwSPB-l5MqU+hs-NPxSL^>kE~7$Ed_0a-a)U@~ z>1~kNvk=Z6)0W0xx{T)^947L6ej5Un^`(aKw%Bw_JHBHV!)^TM4v}q(h-UK%9PnTU z6bcW+y9k~O^1K}^vbMvT<#NF7rQDbuUnJlAp9!1JgmAL&wYkh4#c=59N}lIa07ovU zNbD9V!*itqIN+iOmTR3!7p)h%*YfP^oo!s`xk3CS-T=>N`6zk5@jaL9GL7hUD?&%| za$FZ@PL>*Gz|8WEV4|`fTJBAQfB6IWmv1I#e>)m%2eLr<#6)ag8~{?;DCuGN5jjo$E~DTA22ZkMEhAMBD9fUWCUILizb z9y+j0sPk($x2|kE9uoOW5^gtw7+s7P?$rB8er%j5t=_L9{lLFp`=2i)i+_H{im!_} z{oa|@E^^JNzMzCE3JIPfXNyJitB2 zogv@12&w~56XT}}5b2hN|KoH(GrXMCx+)3dJ?nADs{@>eVLh=5`vtlchw;om736uD zrf3wbA!i<@!ZXDn7=QQ!EJ}%kkX#vVfnXo(2v#RccBMmMMw{^Us_(-57;EVBJXdDU8tAt;B zPQV$XRAT4VCiE2VA|(PjzeysZ-bPj%c( zt1ytv^oQ}mpTWaFlY9wGAq#p`Ny>jJkZO}p`tARLFOlP#=XgWxt;t09?lZ~3YpcnH z6`!C~YL7!6e8i6W&YbsyC7he`ZQ-wKOuV=3C1)E;@N`uZY1WD)7~gmwu6)c9IwpM~ zA>p@4;?oZ}&(8ql1baBfBN(P{Isxq)syIu-F%pM;m&hp@Q*idb!DUQ2gK`2(aO#If zZsyZQ61Czuk)0!i`&(r3sfGoxMtB`;YPCsUkp#?{HvIkkc?rbrfPGgE5TBPuqIY&q z+>D?uu>ZCck_vy3eWQNh0EYuuPHmDXJ!ubaUNVD(k8^-(`G;hs=N;^KzlD50qfGdu zmGt1K?s5gbyX$hRAN+H${Xd;KS zcEBi$7H~U$6heK+iAHS*CCN48IqwD!xU$R=+QkHR8D8gHov~C!E)@D(gK^&36h`y3 zI)rx3Bl(7}N%SEJep#6UUFGWtpF<$3-jU>6tu6__Ia^}Udk|;IwL{6dI>-uC6&3W4 zfk0a~?%!f3@}V~#J|+GItu@JH{maK(w|6(u7@o_?s9c62zm5ra#?0s0(j1{DIruP) z3cEzfTvFE+ki;5smge=4HP@XCPw#?f1~H_=eHZs#wjWDhRFV_y1W5mQpE!k5VMfVP zuJ%+1_Ow;z%o=%?v-G;8Z~OzO(oQ7VcMv%??u+ny=xt_&=P2pWw+ZCM0~c~vZXdL* zbs*_eu2B+lf-7<#D>^HG6EC|S50$&!NZR|+WZB&%_)L5;`WdDOHeY9Rk1Tk(P_K|P z?wf~Y3y0yt0U>!N%@L;Z&v0PX4WYv?S^O`2Caylod!!^fqSuwHV9{U&aVINDN6t8C zS+oV;*K{KKL_u`pI)SATlY~28*l^Zs7m*L!PUF-)cA_!r2O(Z2k#yLm;hF`Sz^ctB z&95dv=E^cMezXhcF!}{pZ<@sQy|IKT^N&h8%hz#+)~8_jg&I=HQ23eNL#*gxS`)|X4l z)+mZ5B>GU>ZXq&}8i2EVg0Mn2fW5} zJQ+{++9==S8%0D< z#xoHA$Q_K0G&o%?HN3E22PUpd<-QnJ;G6Q1oc}*J@@Q%tQL!r`%Zv9&WY_}Irga03 zhHfGqJontv-D161(BC(JGtpBhQ%=tIai1C!l`}^pqy|K!n5m$fq6ce z;U&v0eOby`No>fYmdh07%SwmOs=?0|W|H6IU*Qy!oscm78hM-%266XPh;`)`;q13s zBzEfrQR&85qS97`KYu>J`PHn1(0qAF^%@Dnl;I*@+ug)=m>2P$QA%nrRlw)@3AlN9 zKDjXbCPZtTuZupJU`1!+HTp67VT)YG1B^P3oX;-mX#A_1Xb(w5iZp4*-7Lj=- z%4DqBT{3IA4R<>C8dEvX7*x(Kf%t}zq?BhYMZetywP$KM_33IvP3Am~epxE{`gMSq zTyx>M;XAq0QR}#$w<2+@*I5uh35TYV5~#YXNnTEz3L@nk;!ifu00^$FUA_x9KP#18@wb}@cL7urORgh z;8b2EN=`m|56agIU~a)Vi2wAE*sXEIn)8y$mQn4nYNCROmL7okf}_Oz-C44tZ9j2U zzKMUy6Y2HPQekH5E%MqXlOz;<#4nPfg>5NE$aH}OM*ghA!yRPF&5?D)E3rv<`acaQ z9v}eGEnKBE1lrzJVN`Jf(j`FBQm;Wx{$+T)HC?E%V+C}y{FIdKcYuM=|0L_2XONQb zZCv&>UvfzE3UO<)B>KbGi2^R`6Xi#%*sh3hE~j}bm*ZhT-rJtT4S#IN_BoCaB0fL@ zOjmHH+Z~DNomWEGXoEwt8zm#hE+ODB9IAila&x+$arH%tuvKmf39~+jD~yMd=CiRR zzBP~Ynqxu+rvq0NCF0zF@1U*)jaXE6fZN*KfpfLXiSgiS7!#v_FQ7pf^VEgJjnN|) ztGc-~u7`wu1{hiBgB3id5W_EPp}h10dB56%)7d%?=LFi4^L+RvPi8sT{i=t}6B3En z%3N;u;|1{WQ8w52Cs$aveT0NF_kfHp9~`Y4PiiyF$(mCq(Y_lugv;X|lL~WH+}@-u z%B>y-EoI|G?WekN`l@K|O=%0hdtnGkiRchMFH44kBL0q}dWyt0B!JA}Bob8gL+H2r zi169An;6TB$wAL|@aMS`IFD^5E-oc7eS#A?;4e=i?$(i5Gc_`NVmb+~K1BX5vm}R{ z{Unk1CgSnoUtn|CMqF@07gz^Bd?MQdM!ZXe7N6aCjKeL-_bF9ecgR=p;u-PR7Egm; ztNPiZ8TTcxYR_`R4m~0fok8R-|Xc6S|_v3EK5Z-$KYwfqV0 zFzOpd3*$3bJ^u|8us`ClyOu~ZWnNgPq_kdQY9AF!2gFVYAIfzpo$;K!c^N!hP1 zXg`__N4~v+J3}fZF84#Q2S1-XrRsq9-3!?Il$=zxYy+0uDu6<9HtD^i1I7K% z@V+f8VVA^+TR!{;e)R7Xrxn(TAH6!x<(997hc9|Kr+FD%@bJIHuGx@NPH-U3-$oLh zd!s}remRf~8C7mb)=~WaMhP@M%_0X>#-YZAMxv>q%ZPE>c{1_zdeEA6l=D|;fT6=$ zAT=r;u7(fd#KTH3MWYs5n8xsD-6;}0V>L{4E(6!Q#iV)NS`vPA7N@vU6B0WLp;t8@ zylt)v1CC$B-&X62CT9APV^apXoA#z;rCc38bb0|PS@j?GtR5q}8eful z_^G6K>tfiI`XXQy&V)tJQPoyLe4%O`>pZ?Qmi= zcPLq&bstt@IqA|AKj>9k2)ZsCA;xkGzJx3#lk?rkU%3UibZs%rJllg;^ZS?`M=vwC z-V7&a{1YT}P#umgIY%Zhd(HXEZv%J!Jr@sd#9iYuzTrs83$^Vq{lW!s4M@f<=bWJ@ zq7oD)MUlxE!J-4ovMDiS9>m~6bY{cltIGzPxzC=Nupn&AUdi$lAEEt3KUr> zE|mRF9$G1iP6Xw{gYOdDTUbF>tT=+#b{IfF@HwzyHj;lY-6V64Q0V-UMPy!@;L3#y z$%+ZR+^WeU?4ah##mBrO!$Ns|>l6azM@1y^Og#A8e&GV$#=}t83y`EB502{(z^pHm zA?8Com+hYeO7uG(R7)b0_qmd93vUn&n*@B}{7<;ZySUWXDs1>yq zYd_-k>bKWP0zyP8Q%PoD{EWFGP1v17$kSg&M8)zr_obzWi`G8}0c-TQjxTSaquc~r z`-tJ*`aUAFHkR3&=|QBoQ-sHJ7J%u0I^@Y_S+Z$%FIN|w#Vsy7L;NpfK>uuQ@_zIx z{`WkItNBk3Kb&%gC~m%wPxj6NtA8B6pSy+lwC0kWUz4Hm=rPP}JVTEjQzZ+&MUZ<< zNOU#pIZ0Peg)Fv-+f(RIj7vgDVDldAWS&Td|MMVT(^lZ9A1pAwqv|4F(=Ud4gvI!^w$5!cPr z;3gaWz?xabINo;CG0=V!VJzoVOQ{*y^CCY(ZlvMI@MY{r%r!{EX2-CV3^ z5LuHd4^hEah4ah?g>x2f;(lIg;mRrup}tQ9w(ZB@FxdyeR%6JWeGz2AYHhN3;%CVz zd>*RCpNG8K4Q6D}{0A}ag;7=@G2f!d>ygFe%yt*By)72zvz0_zCL&d<&yrQ! z%b-+lDU6m^kZOI{Ov)z}bJc4GK-+yUoaUL>g&|AGxHpfv#AW-5Mz^}C#%r!H^!*v( z+xIn4Wq%UR>GqJ$g!{r-rKiEYx(>EJ8b*AzEAW51_0Xwc4q6{iOZ0Xc5MLt|GG}fa zL^f!N7J2-_JAV2LQ}k|O8yP;I;<^HMbid#tA`#Z(_flY%Mt)f55S{ILytWet1G+OI zXx#$Ij}R3Ibc`WOdk|Pd5v(|B4n)14v~|f#n`bT&jvJT+?)S3EXv-w{(R30gUfD+K zTW@o@${g_lMe@E-8+IPzpVzW#j2_J)+Z(q*ckmwYQMm;1M-@c<7Y=crSN+J%!~^7Z zzkrk&t{{o?I{~#R!aHsnIA|UrJ@wHf`q?e+=96;Zaf6^&{0XM|+TzbzUtrqw-9%>e zERc6k2W`=8_#ESpN&P|aZ2w0V4x#WumFJTe$%vHx@XyF@J?OA0C5xp-TtS957@km; zZkZzoZu8xtYv&U<*^~kMHslK@7+aA9#|RoXu8Q+YI!e;c#1O$S0;KPp@QwHg%$T_s z?{&1m6~36evYi7jrL!34l#vhZ4D8c8gI5UiK&QzC0$Prd$;E$2&+27hAbXg5eS0=6 z*4jc`b*spm1b^syKZ_es%!W6&T1b;}JP8WV7OJ@hkxw~qNL_t~B=qJH7(P>*i@3fI zzT0b%8yg?Mh6WKhpe*k5MaunIEDO~R_MkAio;XbTMF!UR3(uGRz{50R;VqLz;JchgZDPClI~3^+ z;Fiqm1PweP5l%_q{v`{$uhheT3&%mup1H(Ue=(T$W^iA-^nsn44vh0PoVLLX3?zGr zZG{JJmMg;kUUDL%UHgcdt{!)6F5M7V8%-u0c#8cOzs9GGin)OYOX2pgLKv!l3lGk&;BK2R_<`$EGR9+)bd~7|!>b=8LH*)>vSnT(XudWB59{lauq+*N@r*V8Jz|*D?&1Z~ z<2;mPwX7u#Z?m!2qcLzXOM|55Btc5&3fM3`AL@4x0sWZ;B+=dwYQ7vJ(fUdv6Uk28 zRXG73?KXkss{_EQ3y5IUe)#z`n;UX+0elJH2p?n8fm&CCzL}r!%oifq(p50@YXA$g5uz!mSM8(nU=! z*>nP4zj_;q|D8>~JDmnanM34LaW>i8{}PLTci|bLJ}xE9j=VTOlE{SDfx(}VqMk+B z(Dhpe1cC-!-1CdAIUCdjYhcp6X43v>1{pPI4NhtY zu)2I7Xa*O%eqRs9@TP^9x=C<*V8X543TDAq%&q! z7qL@yEB5T^#qDL2A?y<(r}zEie#}R(lLkQh|0p`|Kq|jKj+2?aS4Q@13->(tIX9H{ zQbt9oXla*9R7MgZD|?j4j!L-ax#!%h5>cryC8+^X< zohJi3&ytmIG~qq(WVmln6wgL)7cJeKg==VS&ip$q@L|Y)#~H5)VF zaP(uy{t*Kw(?;md}#jy(N|ZQKwI-0&}>4BpL(6(Ro>cs%{D&qIbsJ_JyYg8ND&fT z@{_mhYBbgsT}$hnO^3AyeaWc~a|mTb;n=3@FfiDe7pBRk5SHpB<~@Cnji&Fyso8O`TWlVT zbN`2l_8a(e{cBRCRDeI_7La9{59s|peq`@TJ7~}(~CZeD#5 zp1E0zFYKC4PW0L1(u!?x{EZ&_d}RbI*{4tDDU`sES%q|hMFG9`Er53){vt)I42Y=Q z2l}p9C<))Zf|j2tLTufnpxcIfMB&nZ*uBAsNQ7)5-y-$-9fCXQiSB%SeG6t!`Oku4 z8wA-@!xK_!v4Y+5av`*RFGM=W%IMna8}Nb;8=m&br|7*4p{`b+z>6&n&wkoYL~CtG zt12W4`GT=nJHW8kdAq@f_IWKcL%nw|M4PKZK*MPnF?)NUZ zmAQ}kc5Y;ELmS?8dpEh2mkAGN#lz=F3OmF_Lh+z-{F1UITb@co30ZM|Aa#vQ{ZZyv z$rQus;d$6Y^ezdWzl6jewj`@<@~By?H@L_{j6YXv5SE<3O~S)U1)@)JsPlFfj(N46 z-2GR@+vzY0EzC2?PmeBgnbSh558Z<24K~qt=k35O;|e^*_NEis9zv`A7w7{uRrHo( z1z2?t(WjOSkKBd<5Tz_~K0H&&h;8Ph~@DDwKD`#Gcs{GMgRZ zE=$_w9f_iUI<$W(sPjU#h^KWc`^>s5x+F)0Q`obUcy>$UQ~UM{7EVjT!*YL^qv!p| z%aD8QB?8V=V($iQDd_7M8ZM+Q&NSnpR)XKOpoAmftGl?wJv!O?*_e zVaB)n>{EHhIOg|!ax^uDmzw#2Rtk!PM@D@iTse)oZ82nhPLv>bJPxe3KwdbAyr`VU(TZ>2fmkc}hB63Fg3e4O_xg!ING z(6WMOa;5znlDqUGY>L~%n~;nmO$$pgZ|QQV);@sKzD3b(-g%^ca{-9}Dny&-N1O5& zW5c;`=^Y=1`5%~2(lf^r3TgbJCTkt=f)6UBIy;iSC*%Tm#j)r`*LK6!IX7Wej469C zW)u9_D@%X=&kmB^h7cOhW*4(!=xfJI>9?z;h*HR2QrrBH%t_q??NVIH)Y{b~X@ep; zRdka`HA+D%!E@1_sm{6Au!L^@(1AOp)cLD7h0^5(GLRV;hRa9VdGn+Z(bsfDo_dD6GX-_jZ7X2U>Z%eIkjhi~}rk>W62E0^ByRj0oi#k;%2&$*|iE z7=3IxnLd?>lcoec6_=M-Ikt)|6lP8Kb}Ntt_r;mpy4u9BSLFY_BN!hOOWDgk#IJYm zgtN_Qwur`Bcv3|J3hZjw{<|bu-^!;oj(();XfC$@Ap}EH#bI#Y3o^gT1%_=%CcbeN zbm=xe%+dc3A90vN-+CNR2A-@TCOVaLhCwsDka2?S@5>+@*+=Ap*+Ft)#YN&%vro7=JyT)j!g;V!)r?=L8UVdk2y^-_NZ{X*yGT$@4O{(9GqewEAnQ|V$y}IHMGs>MO%_|8<40#tX5qpHB{;ijJAUQT zOPl-)!4JZX=vA49kRdAMxkw&~x2hxBnv2LnB_E#CGerS^d|1#cIs>2mO%P<(QCK3O zhu&P4jW3L6!_zBYuv=cMaP-b`dDD%NP_SZ-$KDsjuFBs|;)ZTO%WI=#+mBJ4_2V>L z9&rtdTb9G0t=pjMG70#gH;2vs=Y_fN8Cc`Nr+>-6q2uJF;e+5O5QinfhaUap&2I^| zo}oH?INOQ*+xLn0Z>W{-bbk+jWHe&4mI)lWXD!_I>kJVs+sHrfUC+MoN}Zh3uY{9J z50drc%}`{V52gLq(BZN+*u=sPGkYp{|6Hxvv&aYmxAD$wPbRG$Z!T!=Rxb8s+2@)+LmP{yKrt3%4 z4qK27(r3u@5C1F@1PJW)6`q{D5;}bV-GJ3|@9bnE%86 zI`Ip;32g#*litKH>~ov`vVEcrIf)@R$<%U5*uF)XK6kqtKD}cGO<^cB?X03#34J3Y z4j*v-c2yXweE@#<+6MIkmJ+ATEZle@gsg6xOO72lMAj;uhwi_hV{k;6+;f$n4`w9s z3RVo#kN4U!J!6ZY@bbM-q-quP`TmkVv*!;xNx&|8-ueLTMVm>2&Q&_tD+I2(14zw< zIkd{ea{7UP5^3_ufq&m-ky&o_qzVne*MYkHf@c(rw-)AiNZ!L+{@VcGx#Y1&woNlO zX{9*F*M(SaUrW08P0~A7pMZ%MrkTKi&2;?KD{}ToDsT5yGsu+BfOg(yXV_p)!{!UDPH*Ta|G~Q3YNVdydHUEAU-c7;`c>hUBlk0lj@eCTw(6ftqU^;Q$j! zd!M*P6~2ih?@IvG+~Y&`Avrv=R}LorETc2E8sKxIzwqFri{$DJeO&&cpBUuIaa0u( z@c8fF^!zu*aG%2x7(XcoCAOzSwvnKi`)Wzg`t1%2f4t;nHAXV^^9x|<*Y|YD+$45Q zWgHHC7)yU%Fi!U;c#@$RF1#~GvWScIKKMt!j`lsh1kOpS$NY(OLbQ)V-=k9eht5vq zbdow$HFqJC0=|BD<3=)cl;Y9LBgoL*UvQ`X2h#QCEjAw9LG*)O@hUpqnaRb^p|WiO zSzHuJ=3T7=5m%FWi*Ps8&(NZe?EH;Q%i@S*`&aVl(_Q*TJ#pFS*BfKQ6 z1eFf_q;sF7lkco~oU8FWvGk2NXduT_KR}AzSD<?(hpx~T#NvovAh z&L(&S$Z)jZ@?o@%EFM@Y#97{V95%R}q%WTnoKqs9B)>KuN=)U#+T3+8{;Mz9-73lF zyfeVX!}r-cJ#NxlH3kIPF2c}OmSd!wja%|P$lFLM(jmBS1}OnJ$5Niy7Y5N|t5)IG zv0VCJcOuqq&miVF4&U~Eg(;y8aHHXIp7%*nKGy?6r?sv0zS~dWtCMadSz5rs-gGCW zlB+ovZ|TCckM+bYDx7TbE{7#^++p;J#c=viCM*llj+8YLh2}Y6TQmf3Hg9N!x#k< zIFhphHoAYJEpB4(q^+h5^MVqsO7PYxO_W|WAFbZgYS|cxBe%b*1Uylq(0D# zrqdvuUQRDn6~?GEhyCweCViqgg7@Rl5R>V80T{UPVCYQ|Y%=VK9S$$T=ffV8+>}kc zFISeqEi;B`;nGpu{U?(Y-`@gL@&LBlvyM0jA0vusar7J?2ae6_@v_ACjxrXOC&4={%)q;M8Lk%}@`c+kaXyZ0~@oynCE-I#_J~R-uphP%(V;?W? z;3oEmAD?-3MSGy*>1zBi*#Sy3ADO|ACuw>3g@|huk-|A&NTKf@_%7)s3~jzZbnK#u z)z9-po_-{l{kTTYa_OXD@KV@6TbF3h)8I4f_TyIB#n|AR4QzLpffL7mkSnFmP=~F{ zr$*<)jk1@ZeSZ{fFqi^Q4)IBi_;>nn>Pcv#Bg>I99VLGR97u=jFpkUkOXmhgksq(; zayUIG26z z=w*`NtIlbBBgyaWP=(TKQeeRYQ+k~53N5Ck`Q_4Y;O~t=ytQA?Lv%`*uiLzuB&=*E zDr$M8+C?1}oc%;6do`08OLOR>npUL8VGCB486ihb9fCKnD{%VU3L%rqCvRsb^6uMQ z!6(E73~bkYcy8qqvhYa+6ibHm?Rzb-e!VJ`OPzx5c?jF?BoTuSV{}KBSFBq-FmqKpP z4wL7hsBj&=px=RA4NpN$xla6Vt0p;PR>NNLewfa>o=Vc&eCVaNv-uyRf}kF1#_N2x zkTTylkjXrOF`Y*C9@;#(TKl^d?#H z^$B#%jHD->#Gv}6@9@KTHU7;mC1Sfo$z4ks4s+`%u~}UMBc4z3{M|0luOFRXUSy{x4XE)_5rMptLS z&U=NmT_(+Uyf5ic5vAForCmd)!K)-X`K`3nlD0wKG zX5G6_-l?C%8M=1RW_29aN(&LxauE>kh=4bazJkZU>99XsG9?dW*FwwqaQMkFntr!8 zk@nozMv4WzfZ;+fsCMQp-G~av1D(h86ZbVRW3-!o*K7~3gpQC z&52Y-6rC?%bU}L&S!d)19}IQU#_4y+ZT$^oqjxijtsf#vEGIJP8$n(+ZiY_r3-P)5 zZ*b426pT5JM4gB5MXXCsv2@CwjaTK^{~@QZ~epB|n5|8!08SzWN8*P8r$KNE`krXvpI83qOr*O zr);a<2yD%N3^zSF!wbv4O5C+qv2S+8)7ACh z^#_spD<8otvL9fDUpnqJIgCY@WRYfh39{l?Jef?>qMxTf!t@Zys`Y}|T< z3_eyNhvuvz^a)+c>f2o0*+G-t1(kGJktQ$StQsbI|Ar60#1NxYX*!Ox1!gpv)6PHA zXva$$oZjscg8!E=St3zLY7_30)2#ok6V-=qs}tPLYa>Xh*nD=vJL&0+hl z*+z2zmf<&7my)>aO4v5~4ej?^j3d{-oGg17M@p7v&;!qIV}q~-*e~iPE>Mpl%S{|% zlyVdHs85ECXLIP|?;PQf`ENJNS&;l^MiyR6yj_e@5hf4en8hxukho85YGBO ziD$jKhb6yx(1t1xaB}+sc&viJ(P{ zeg`TFvRB0x7iQP3rF6PAo0s3EP2!Ez_;WXO(f8wW2=yO^iz_#?Elw%n8ETK{>v^?! zj^sbQZ%GO^>J=xe?P3J1xfDm+(3*}D)`vl9GVns*W$5qROn;nroE&Hq%$FIhf*gYz zaH~i${qflr{J{Dp{7w|1sljW!Yr6@N8(%}Lt53pb*94hiW+u<?d|3hmw zjFAn3=i^t%dBWSI4u4*H%f43|3&m#LfX1&r;grJ*+0J*#friItql5XBwd`DCap0I%{=!KaExajA$k ziFztb-e&ia5rg+IuPulNB%jeW*?Xa%vJ;d!_g65R8iKXX{^do#SVC<66+v76ac=Ed zadP&q1QGveP4?U{rB&P(;3?&0r0UThlD^)IEVyz6%Sd$*`f~{O2$vxV^%8L6zsIzv zl?c(l?*RAT^(7p`S@80{8mt@k2e-;kVjrPkvfN0QAY585BNa-qT5-%P~zRlUso(6|#m!hrEUPQne z-+0LDy#AGV{F8=z-wOD1!3>AmgCFozz5~ok2_)MBssvoC6>a~ll9sEmiD9xcQsy%I#?eL5a~Qwdd^hTzYAv+z8l4xA9T9kTh?F^Ur6_`RM>!ySzz&{-L} zRA&&bLkTep48nIIZCi<9c^GU#UQElUn|S9%JMu529A@{dhG)A^vs=trq+w8)^Z4B+ zdZC>jom|>LkBt<;Bc339Z!zBXUm-5_N94YTE!lHfho2QN zgER0oAImq3@GVb0gU4_ozP($R^ZJPrE);IVtF~!i&g5ID9cV||t!?1gYa?=5GK?tP zmEp&Hz5`zh7<*%haH!CeNHO|d>^1dLBzE03Eaj6(s#_eOwfG{Qmx}^C_+L0&=^jA0 zdl`}|<7)80t4MfkSrM#io(&hwe*;Hjcx2(U3as4x6l)~O34Yg7h~s=QN5DiPjbNyn z>;vcIUx&eRnRwP@Dk+HzAkUv4hZ-G4bo76z zpo>Z);duHrn*Z1oOLvqJ2NehMcJdZ95*dN%zcomZ@G)N5;CY_mA$^?F@``@#Q-*cC z&cWxSJBhILRTyb^owP4YC3R&7@IH+gS~_zKORAx#k|3`3QWi!p2p8?~Qp5jjnO~{9ze~H~XHd!0ljMnN;|FVzw{!9q`VtNDXWk=8jX_5FU>kSY6T}pQ}+hFNqZ(!1` z^WHL#baIx1xcxa0i-J0`;%!?=^#|L-Q8}lf5{l{I{-p$ZgH?|U4r%)p6C&Qm9IAgmj z9bx3=l|1!HNisVzkCE8-hR*-65C&|}BHk@$;CGbOH9*OR3; zml4I_Wpv(@A54l%!Baxx6~V3*Bx+BwD*CY5+Wlc_nd z@#_T`sKZbsT5Tb?>UpaAk$>fm`ftw%F=A*O14LiE3Ar+ zBu!Vi>`B*k?73aSM6Otryg29!g>zDIUY#&J@Wzy!er${#vi1?rF>{Wx)F9pabs5&x z5as-<%!Kt`E~Hh9kM|wB1p^hc@bo%OUSwuEp^AfHY+DJ=*;PreJ6?^y&pb`~pHAXl zj|9jXz6kpodT^9PHkm8P#w*R+us(f_tsHcTWO^IGM}-YItv3tK?8?I_?bak@|2O)9 zASWy2$#F7CJHBA^jjXCnp}QZ|;6ue$@Mqvp9BR^yrLRum5uw}U+cqKct1=x4foT4B z8ea+j59j;4!JijWu~m~do-BdOwY#;CPnD2^)l3FC`zYo z_{meQX~78=dCV+vO_)ywkfHu$C4F?-(`b}&#jlJ54OWt?xQxZ~1I-oNYByptiz@b_3fBA4HTXMW!Z>wc=! zPh~z~*=S#QIo+IB`_u-iSND*+Ss}#srUTShx`~f&drm)evWHH7Pw7S@LA2kF&yH|b#SFHlzHHu>7Kftc>?fqIov{9`{};@TZF-1kur8h>tO z4($GjwF3m7VUeJ|`IioVB&gFNdF3!n_9rnF*~#_tH@3p z9opvHXVPexK(2V~#?{&@d7b{vQ1Yl1%DJ?WNM0Pntgm_y%`L=g_IJqfOS0JJ6rlr^ zQh1*mgg66&{S(K2t|XnCwec}6FS1c7htzA$BnL$!@v_KAly|r+@pBZMo9m9?l!0{e zLi#L;?`|Rz_wNAn+GgTgs{*g2^2o6y58j`7DR8a!OBj$BM+Re>v46w_`Mi6I?z>V9 zU&&a|@4j*2to(X1;57rbAN|VMJJ;ggFZb|TVQCI~#YXmzfW7dDw<3Iy_>%rlX*H4f zqDX4@mc#YD?R0m5m%BvxzcRXzZw>b-sFB=P17x0-9XnbAGXG9dU{prOfN9foU6-4ZgJS-A0HEWhKxLMAO z{FGe}w;y}KPJe!t+_rv2>KxUeo?r*Tf!E6P!TS_#w5E*?QnDn8vcb4eYZ;^2hDiTz z7KgusO`a9qA`{v*^q!+9Vco(dP%`5Y)2Y$w4Eo&jpT$P=WSGAa&OkGGO@5{q- zO=m%79)=~8XAr7B4l9JeWmoi<(Zexy&{sDaUhB&zQh*DK+(n_^4uP*hE`xycI{X(+ zUBASB5E5+I7@;b@cRmBm5PTq4OU~@zq4SBe=j}X`UT#8c7Sw7 ze8#uL>v4PgLP!PPhD8m6TJ7=x5jT8G51R1sbVn6&R*-ozI=_V< zldL(HqJ!X9;T_OZ+K6cB`N8U+@#Mx9O^(NOD;`sl~N7C#DVd z^z!y)#IG`rzL)5Oi$!hWqvK6DdgToE?M5E{G3vqlykCL;t-+mqjqrsYMcq)b>@XQU z_Jvrr@ab=lTJg@hmsniTYaX*tCnw*I)7f^Bydf1g!X&IGcg(+W7lajn_)$|ZR`(nj z$j<=t)3sSX686A6MS?L)73}wYoQi5fgqf0EOR1hsNk~_Hj-8RjTqNWD5AbJ6v6>fX zP(?(CO5Z+$)Z%ZU(0Pd{RN#dy>sm@>EPD-_w|0W%lniAduEmUei34eK8$i?TmD~Bre$hUcdPp3d%MG%cuw-t2hc8H)~Kce#{ZLG9H6~ zzk(gz(*~671qWvTS~1kPNEj^~?cz2%-2{E!Y3QrxZRESbl-b_JqO#ldnc>Qp=&_78 zb1c-B+IFG?nU-8ao|QsOe~SYp{`o07w7e8qoR(+S^~|F>@IJe5SsTImjsL1km&ywK zYqJ3~4`czUG}e@z2(yvb2-fCQfSo~qfJggmCRX5#`20y1%u2Mwad;?`3V&kBEZ>k?C10ZmOs(ue)UsIcx_%~eZRa==&0fejiPKbKMkrcwC==zJ zsX%|vh*J-I-+|-Dlfhx0I2HCrgGpK#i$WInpq_%+jO6;2f*tr~fV=e_U^jGtBQ>2! z|BWm9s-=d~ZdRf*##R*X-C?xh`E}&azKFgLb%7w(Gjuq*8}%9)Ft1iSG4i05L$DDDE{s>|no&oE{#*|$M zn~9N0K*hJ)xU1dnp(kku)KqFIH^byR%2j00#VS>5T}TmFDxC+8?U1JuyX+XL(@z$E z&WS`@CfaP3^J4+nm%wF|pR#uO86v4#X}kU{$3eOKBd}ZT4_HxZ&3F}Mfj=ANf$hBx zAYCi)ZuQrL*ynG-qu=sW->9iz2W=Y~s7XN@-{hGu51gp{$!N6LR+G8A*p%t8wWGRP z>X6*M8BCA!EavfB6Kc<>FG#3u0Q;6pQweTnjLxYr1mrQ=azmPNJMlfyN&7~k zhd29J%lYmU-?bam=?gP|{0@TV7)_?`k}cQ5X&iJc3PokM(v(pu3q0Fcj{q@4af}RA z*vOzSA`*<-+$Y?ff83bthbEP$1v|%UH{VA}|J)ePD{-@;L5cppeHUC?ElF@X>Syn#g@@%PmCBN zm2w3|H7WwtPk({WK_9^DJc~|0$U-fL4MD4>BRZt~8Tlk&uxj!p>)N3Qz{-1sTXS;C z?tQ^0#H9Oi8zk$2>S!;jD2YXC53HEufBU#%wGU8$*CM7f_XpY;CrTBsb>5gcS@U>sTZ33(jKwJrD$vRBzix zlskZleREhR78@{oTWYzTpOmQOeC&63pDTP=u@nnG7Sa#$1FWEl%{C+=;F2Y?CQ z$F-;xV?<;VxTX2u!F}Ar3gPH6QVx9XWBV5%G%x^Jyrq$sovKJJM9x4LcZ=Gt~$ox ze#Om64Hjpt{8gAE9ZON}(R)B^c{n&I*8w(cvICW6 zvglLTX_j_i9SCUn#r=10%w~Z0 zbI;HTa~l+DCQaShI0VcBWU24BY{4<5FmArWMKD<>PJOfp0cX@IkVH%e@HluARIHWe zcI=+EyLyq&<<>@`|FkcmwU_F^+qKtGRE{QNvv5AjL|RlA83c>ejVQ0Le_2G}4NY6Q z5;#_$2K6((FTP~!#66dS(7T)J%>JTiq&cktTq-dVEibM<^dShzOQxak%_o7^cqU-t zb-_?#2>3nnrg~Ry1aj({2j;6i0AhwT%TA{p>8Egz*WfFV@qQudyL|;YoOZL_p?d-J z8Cr4mqPSp=%@Qu9^%v!5okkVslDPvjtic}dB2+bL4!#BnfoluLtJH3%aF5hwA#W>z z4=YEDbtFub`gHd=u#ys`>V94Z+Q+3Benby~E54zq&>^mf!3Dq^p9Q3`1{fMy0aVM2g?N$t7w0bs1Lw?cKx>BpThYqx(zAf=c+N9qu zbKgZ|x92`+68Z)5YhG|wcfLV!OKZ{D9C5}zYax189|bBz1YXe}H-Wy}W)Rwtz{+eE zxPJ}1QPBo}r2R`9xEaXAa-_My+L$+)#( zQF^A`$nbOEGNKIlhoaC=@PJ#?6OFd4Ux$|Do4Hgn)_Au4YXXS`lfOxLh7Gu@F0UYz2hhl$E4)cOc_ z%`Isv{Mc7C6}^sm5NXY zH9O203F3>6F4m`hcE_+PzdQsF4MgR&n(}$f+?EXVY&DDOo&5}aOwgu|2<}n##v7T=$v70{v!5%r^D5f% zun_cH^q_*tn+VjcM16@XsUy~ZK)%o$@NKI);P;3z71@75dfyUiqv|ks|I)RL(mZ?C zzRbU@*D4|CMRXm2O)9AQcodL*I>Pd%eYxYoV$8pBD`u&ZEaN{ooAQj2Vb*?SF_ndH zx$*~gQPS= zwZZ=XG^lTyvB+4%kP$BUg+8|mGcZ+yIiEZXo>EPAn>*6?TA$rUlOM>;?-O{L+Ri<=e;DO?Z(uI{?Lt*bx>S&V0{Y-= z4lEuxpc=6`!0))foy3xBtY1+EOo)38+P|j{xI3?)9*SK?)ic&GcbaXuss?(L=D7;g z7FC0oN1H&TtH2ZHUV?P0N(FvWWfT#93tc|+3WSzNgMWROz^gK6mhS#~V05+|6mm{8TN8WiZ|jm}`xCtFEB5 zgIWwUd%`jc&_EC6m!^z1LL+6op**EMADGfU?8_UpixQ2=n2EO=KV2mZ+yvL4BF z`D|6{phYQi{*MCZAMw$F!-~wryocaP84t~<6J?}g4{(+5)_|$>TJ%TiF52~1fr(r4 zj@x$MgtgtrAKl6~RR084sWsHVRM_1k~0Do#jZt@y0Klzcdk3>yvY%Rt$2Ch)03{ideN%o57tS01|z#1L;9UH1C28rS+!<#Cy`neN!l^ zI+%ePi;cNqG5G);9Yd#R2QdG)9d$unnyORlK+bosqK}X6fVT&vDLpfW`|g|qc$Fni zU68p8Jf{oMoLnAyen3#y9G*hLmOZR$ki@OXoX0Jh?m=$T>wz|Wh@Q`sV;(PSL;D5> zKtw<;$~~+jaG04hg#!PM?J+4ZaiSXK3+j5WRk`4)r7us+=Q0ym1as$ zxiKqeyhI;<27?aFMT%i8W^&AwxnzJ?)?dm%+frXv*=7Y+`M>wzWP20K>sb|WiV)mO zWiN17xaS~4AW+Q;>oCsQgTU|S5$?8g(oA3V0IL5u3BIMT2g+Lri&=gF`6QZv-u zY~deB+xIH6TyqhW%{#!Ixu%fYm7D?k%}QC0TLtgNbYq~K`-nBqc7jU_-)9Z^p8%=e zn?SqzP3{S$Y&$DQ4p`OB<9ZZLp(6hU+$f{()yrzi(8wY__hW1sYq(sE6`Se<>dw`o z_Bkco-=im3>q-uCyIR_i(b;?u$^VZ#s$c@L4_)InEQw?7%2lM+1l57cqz=@ip}^e~ zdIg1TS&epQe**vc%;FYI2!l5j9$@cK30lkE283Y zK3D$b5x~hw<+=<$`5vb@;`v28WsVy>C1xqrr;g6 zwV1oNgk~9hc3|=U46+RLR@=20-$LIyB7x?J3tG7RdUc@mNf50r&;2m#IEbr00t|=a zP_U3ZO7RFn>79{w{f%$HzXPqT4uFBENEHzJc@d5LPG{9li!-VA<1DenZZNq?7isGH z1M%+H)pO-y(Cvxum!cJTBpkv5kwI8cML4{KEsR7!VI2zPOY%drSiuiu|hnXpvRjR3eN_vy_3tp+>vlo)0WOGyvSi+qsWFj-%ns z33Sp_f@%(q;0jN-*bTY|BEIK|6Z*v_}SI)be6lLDr!ww4bpBWR(*P916Dt+XC=y*vn1?K z0;d%l0p7X*^~siTHO)hjU%CfZWUq1ch2^`D;u%-)F-MjwG-n@>zq}85%n?Dc0*LYA z$R}_%aS=|%s+XoGqGt^L3d68o7M+*kC=XL#q-s+VP_-J z)zf|4!^5#aBHI=P@4U#GJuwTZZOyNWTrY#XUQ~niZ+3teAQP==SGBvC{f3sG;?>jOWNHja0H*a)}R=1#O-NSLEiTinCh<) ztkm2kRFKaO>Phn-a4Eu(`F45-wco>^3Rd!@%=UUS|9M^lW=d6{fqw{jxikrS1zV|K zyMw6k*Lsvl-44cK;u}!DDo=?7wu4reZd57vllwPnJ$krJoAKtR+ig}|hf+ZU%APC5 z3eJ8EQvUmhx_bq#N8J%*=_o=i4XZSPpq zdIg&Qv;)M>H>VDc22i;wj{&sYz$_{b0VO32s6NAR^)64L5j7!d1nl63o!2HgK=$5NK}7M~>cg zXoh?OXpsyDmv*QzORk+}o~IvR!Uo(ZUngBkMq~@szG^;osD2Z3dY=`gd}J%7NHwx1 ztZbR24G+01ZJLqILGLIVZV~y70=4nUQ*sXC2F>J{PcuvH!A_ z8FW)&XwfxP+OQ#W@U9En&n)f-)X+dR8l8}90QrU@UC6X43HZ59+qEgNK zoaRKKM3#~kvbB&zQJ+G5^T+%**Id_`Yv!Eiz3<<1-x~+!FtX>CaR%F*ao*5TTw{e1 z_p(ZkIrmhFD|_X}1q~W=$r6llqRm34Rmz!BOkc=sb6>z+Uu4fIPT$HMG+xA=U%!k= z9n8VjxzgND>wa`ip3gL^tYY2|Y-X;Bn?PsXBCgCU224vW7*h=u#wJ6b8&qG!ElHK^ zu&)`)#c!F)&^0<-B72zocq0|(Im}_SFPbr?o3=4h5splP;&JXOYsR(iTEabY2TpOU zITLAj9NX%9F=oFWS01d(BraEFPG1u+uj)HMYLf<6Dl>_>UVn-adl@qG@2%oIhD>1u z>T8*LTQ?@))jCe+yD4KRaArEDIxvIJow@l>|AYNWTNulmCQR0OSMJUDHB9~-PiDuw z7qDdGGA{ht zCRPvSgnxXvsv)=WdC_7fy$Qv?J52Kz_9ty;MFhk+}(&WTs-^& zEdS&JV}6>D)w2~C|Ck|M?9d%pKV6&iv9`d`E_wnnpM^^_LvY=z7Kv`y0FPUAA!Sb@ z&aFwMNvZFs@tRCp(0>PJ-rNUA_m$EG5)End;XhcxbmNUR&S2HA2fJp3h}>2dfPzge zouc`KF1+-BI%Utp@e>kBvD0$wYu4f}tcb^9_mkmC<2gW88RkukDpL^u0Tu3@C#^H3 zXll<%f!vu|^0C)Z;H)dT|CfG*XS(~y`m{h$c&fnsDoTXl!&mYCBqth~J`(eWoTQeA zQ!v8i4Vm|C0t%xO@YK*pqE9|SD5E=(OSS64JzGXFcV%tyXmc9=Q;fmjbDznri4oMx zJ|B#_OdAG)w=nuD*$T!K8jUN@D+rJ5_Kjx7G zOXlLLUjlljRTtb*nsN4#=e8v$f~nbWxVA`_$$MOZ=R>LFJpZ09U&Wwwq!pyR+K0;9 zKjFI2%P1SI2kmiRiH{(I|RbC@gsZU*2X*I)~%2--ZV+LmQPQr*>z6R=*>Ai(eswnHXfiubLDa8%W~Xh*a4LXZlmR} z@tir;wUuQ-p6p|aU+|8qm%*@V14jDw14(!22mCrqf$`Wh0Lxo7 zne^j@xYu$&-WIE)pWIl?2^??vA*2(NHR=&f?&C_^QsC_kETc z=lO3MH+A1sE^f9K^TkR>^6XG#-s??ZnmJ|s9k0yApMQtb$4%q(FBmZg(%;kGvb}gQ zF$ud89AL?USR%7uj1C3zR+@7R zFlBJ>O2#_Jg-OY6L8CQ4P-)E+E7))Tg|;?qonVH=;!A|_P=<~5|1keS zgrN5D0mMApCjvb$oPos5 zDV(d0Gbc5V<(jgknY$lXfNI$sth*ROt6p3Z?W1Au_)o%6D9Ce7E)ur!y+a{&YAJ69>-a?&xJbeF_HvJ7vO>xf=tp7?zW__^^&GF_o#FTgRWzl{`f`A z89!@gA?-oO@R>}CtP4}U-;S}9@51{-jG6tdE=-oXBcsgxhk?PS%=bMjm`UDCnDmA^ zoEUbUuJas+ull3VvMT@#E%FMJ?`9Jti{-Gx`YIWA;yeuXQf2xJO5pr31+Jl2ALfR2 z05g~m*S+e{(dQQ3tM*JVU;=_$mY#5J7Qza70AtyfJ@$(}&&kjp2ES7NBF7-omVh(r&e}OZ0jS$;ajISq7XHG^g zWeh_dnD(=e(Ia{?7rkvO*YI-zxBTgC*kUciwf>UyVZWZt-MaGxl2b-;mfgd+@tMQ8 z%~=n@qDF^{kq^V2dw*kg#~XN6dIRdhM6j1SK+`NGZ0<3KIj>LPYISYeFc1&7o_FKr z7acffj4}65ei*aj^d#=6{~}H-u;KcX?nBt~I(oC}Ieg3T0`oj8Jo)PZ-N1&Sfy*OI zx0;Vyt~#94^L3niq&;VRS)NI*&Bpe)-(=73(Ji6OB*olNe=-+`u{TX29IioFNr zn9#acBD04}=#0ambi2zz{23aGVb0OmUYknRz41Z%yAUR+$}&0C^?Tk}Jo>{4I5VsU9$q)3in7mO-wiSG@ZeC*_6%m9c!?y@M|9T05l-z%CfemX zFf}TlTzgE2v9m65zG_H+-Te+aql+MW!&2-#upSliYH;!7(1+7n(;S?>b*E9IkEv#?`vcG%5a)mkOnz!$q@0{LC|_^C&t{UA=<9zV2g18N%*>$ z)K%(Z{;UJIaEmGVpw>%H@3}_ScxJq%l`~HYme(bq zm2WPj&K$)!PE&zt>X*RYeGYovtH72e1++K(BUMf9!Jko=v6Y)I;dm%|ESrhC%?kKZ z=c;IFY9nr5(?g10pMy)S#R7r93GA+xM103QS+O}f(?5I(a=kZlO7#TP|ga5 z6`rPC+&MDo*D}FhYcJ|-`hz+Z-KFREiHjq5&4yQtL^x{rH(VAtmg?-QM~%up;$z$h zTY_zgiM9fCWaMGRil4aR)p~*3G*wLN+yMo%{6Lg84wfXY#B14RWR0B(*35K*1G#b- zrv4a)Ur=L0)CL6xANq<`tFMH;tEAw5N*Db*H3_GEtHY_1lv+2LPI6WEC^iRaVf&7{ z;%AT51@o^ff!wG(+OBR)+bRiZ*gvJ{hG8CdY8m2%qd;usev;IH3*@WS2crH>0ZpZ5 z;Bn1x9M|1~BOTRHNixTbdl*HA#Rt)ke?n=OmIG`_Y)8GJ=kTOiuINMhSm-=s1hRL% zKqL98=%m3$+$G6(aGl-@fo{i$%&BHDxT^t9E`;2eb)WXyRuZ%B9WZzz9~Aj`;^HZf z@r60K_Uli4`MFv&ZP9vgSbCY>Z583fcNtXbmMWwEy#QPv{wAru<~YrMDxGH)PS#a; zf}Qqe@+w*ecB{8Snj#MiH`(GHqJlR}ORe_HO1z&t1z~JVBp49 ztg8tT^!G4i08{6Ssn$>tjGog>Pj9^} z!d5*r_jIK?$FI}+xh$0q{E6CDlOZ7Any5`~7u|oH0xf3`gQ;;k=9f%_OQr5ut1uDU z8#8I&VLcc-V=b(%B=}C}mY_T9CcgGM0F8b($xf?W37fY7YlgMbV`1$Aw-kNR8MS3} zQoxy63B#Y@s`GyZ;l3*1lywD)v@X)D-dyq~Ck(|u8_4{R6QNI>O=odQxa?z_VBgt$ zXj5?eJ-J$WghpeaPz6^zDS47xbFcD|zY$bzzNmQ(=h}WNgAZveCkTpYfF`~Z} z#AV@R(ute&$=78tqoEj$eaAzwhy}O)1ninyLgrmqLbvZvqV?~su*Ne)a6bPe$?)?K zIR-ZgzCSX?cPD;{OvmKXr5_XMTZ5(OyF(3%Ltm0pr3VCuzx(3%>REUxTqG!5?<;!$ z_%(Dc#L~v~328a*Sz;V%;5H+=(s&&NRf7WIsVJ4-U{Ah9M{%(O<>{(p*U<{=A zKO=r>wlv7J2zEq-Ku3=+$TO!&0q2QI8;66H^HneyKWHWMdnis_znD}Vy+Ya>O+hi5 zp(>NYh}Yyk_!V}6CJcm%l&zgv@pWP+=!#2`=dKf88zD;&2Tq7kJ(J*aB zA((EuOdO&`WcHsQRDvean(hTKf0qknCoiInnKSTe+!x|OM!?ggXp+z}Rb)P?i5$9M zA`;*;Cta*s8<~!l^uz0B3-7nahbs8-i-cjl^ z2izT^iMVqH$)4i@PwaSlL9?J}cxZ~ic>5p?{xcI44NgMLDL*Lo?S&_L$zEP0O6`6aARi zsN1`oGcgvS{rX|dII})DQNMxdI4sMg5C4FBJ4#U9Nrg+R9EugX8u(k^odn&@Cz~!! zK{I-TSSc(e!(KlS?D#nhCLi8UN;XHKvU)D&mBo|4qLHX*!J(`7B5t_-8`L&xflVh1 z@y5)Bsx!0`vS^ zB=6#RlB{aU9f&vMJ|FuJw*F@b2c80>rlQWUtxi}V5>RzR3oaPUxk=Wum^$4M>Xs?- z)^5$@&UV=|H{M-9c31-4zDtM#$#sTo%z(-zUtq$)eiE!`z%7cn4BAT!$^4DlRO^H` zxu!WNa*-iS+?L;9;unch--C&NZZDqePQ$K>R%kds2?rEraAms`xowU!n2gkHbP})N zI!zTgNxdUD{wHr0;L}6W!j{3arj2O*w2SEMm*>1TNjTIwmmwi9kT~abP$9n&bEx=QhKhi;I{($4N|%a3lwZ2R*XpBlv8mmz#X#| zaYie;(NI^5(TqQd)4z$~)W%b=|L|`-d$o*Cm#xC#C0TTO`)Tl4a0(0~=P_#hGjQlo z;a*Qkh2aLXnXjY9Gn#`B@b57ZFb{Vyzv>K`-=%XnVbURV7-GgP&DQ4@Z`5UUKKS5s znRmo^b{J@e|DjQfc@Pet!t7!1g4&e<^jB8{r4oCF?Y#y@etO(ismD|_^poU_-9Z?E3Vshz4eiT03HwGQQ zo`=OYpP=TAFQzP(MMvEnP#egEZAQ|ZSwJ!TdhW)^{~5{js2g$L%;dSQCT&i8<~uaF z+6Ar!ZS-fiC0^TNgVpPQ!@p2(IQd4IiT0v^-U(oPbJ?tHy6?;L`wV# z5svubfWB8Pp||w}HGE+Q2OIprbjfVyX$LA2ji9}!4%+UY zA@(cPAujABjH~!S+d_>cO#e9YNKeA5YrVp)tG7VjSqpGhUJZJ4kSbg+Cbl09@TKSj zj&DuHecR1(^^V1`!T5_rGiikm`&HyZmk^!QdD{#N1AnDI++C} zOFog+S8kG`qI$uG{IizZS0^ zYWz`jLMH_F+DD_g;6Aln7fTzvhI4hbXUQ0eu5wYs2Ht%8PIs*}1KEmP2&%dbpXG-` z`_i>o;_Zu}CoZAvD-n&=HKvSKA{KpTaD;;cY(EfOJVL`=;NDY1c7CFQExsN&eylsZ z9uSf->ZZ6e-4%*Pk6?(=P-b|@7HlO?XD>wc#o;qqRrGMesbE2N2$+FVST z3}@%G7=A_{LJf@;lzDv#e|s0voCS}l^U+8Y{kIfbXWpR^e{}JxuM(#omyHXD>QT33 zC()6%tHiLmgB0mrgiP7f^jA?U#C)%V(B{8n|D)^RcB+i5HH<|wxlvFNTTOh^I^b=m zJ>(bU(YW$XGH!o4bf%w$WOD(247rEHwi|No6W?N``~}iiQ%|D?4Mi(2z7%0WoG5j? zAv}BcAN)--r6bq_qQ(xw+NmEP?Rb~q>!J;C>RBdi^_>Dy%i~2^ zwK-5F`UDyCB+UAvRm3oSBD|`XVoV>Oh7}jZ@OaC0GHK3p=rJjfc=uyyd43dW|9J*i z&A5meSz?A^z1tYNIFwWeCc#V3q2Rf$id03EknB-!sgGVI zrl|4M*HVKKKB>Z0()GA)@d$2U{13Q!bO?94RF~WLQh_rtdI@Xh&fo^6rgJZ@vYbqt zF+*N_1|}sRv?n=s7z5y17HK7n86y+qBus$9>+N3gHLlKEi0 zjZvj8Op=`jSCRM*wlytg+RnU%)XtHdf$Na^8J3f*Wzjgh}@}z|fZ#%!4hHxbGQF zBt;ge*%NbciMm8RU83-t;#-pS=@d%Ry2wH&Gp4Y9EA#%-a>i+g6KCiA48Gd-LH~AX z*rk#~na=$Xydf4k!Y05@TX`Jy+K_X9NI2Kcrrha3J;tW=A6lQ(;pU<)=PR7dydG5v z;JlaV7}pNp>XbNUP{8PWZDDp?aAFch*l<;WF*s(1fcrPlkJB4gf}rIN+^rwaDEu&F zyf;nZge@Q8{9+xZ;zv3h9yOYKt~;A?@N#AD$!=%bQfxRS1(>k)F1Xo@ zWisCRGGFa186o4yDZZ>guWhbeNJJZ2tk-4GVkon0nj7=Bcs{d7$CRs_F&wMcoJJFc ztJL64HTb;DgIl#y%#zP?jC{-pPR8jq$QmzT?2afi+qwtP#&8V`H#A{(O}PLU)n{?> zN2haTh3mPC+vacyO8U&P5r=Wr%<)#QwjM@tWg_u8kc^H^v$-F)ExGw&Qq05urf_S{ z?dI0iOEl>RM$FJwSuSF>As1pR$3}Rbzn_AE{9dN(#3bgUOL7ju7=3->&15w~f-3KQ&~ zhcOZ#Yv=U4sD3~SeA=diTf8hK7q7yr-Yt0O-Fi{!lxm`Q#2zejpON&r!QuqR<^@aoUN!Nl) zyP?EQh}ywLL`~xUfGD9OOV6= zfD_e4k~2>h&j0xm-@4myUEgK7{U^Rd#m-P1Z9bLL+T4bo_x^z5J1?|5`jl2T>EKc? zJLvuxCy?8n3Xw~k;OMRtY(8?5P8gpEk3ak-@z)7@R9uIB|F$zpiwePGnFiMw)qv(H zGFZC|p5Nc!jo?HoQ zfm#X29v>M+<&Is4ofigRdc;%G5_%nWu}4VuE(=H~T7nP9S)-$bAJ=*I95)W!$G_=M zQBt>s7fsHgyYvRQyoIMOwTUF!%^Z95&!DUD6Xbu=gOy{dU|en(oRNzWNpB5;eGN%e zrspgw$OlkOiJv#NIvdFgRZbX}gWtzpgrLCzcz^vTtP~!BX3=CA_BEW`a&08l6Zcq! zIRpxp7rTk1Pws~B;!|YQ^Ab@alM98Tr_j~!^YK6C1zmihk3^(y#bmt+M7LxUT3xKi zwR`r^*vT9YS#C_GwLwE`g@yRDRz;Z5vpLskb_{e={pgd-N9<54tyAKT;vdZgamr&VP(^3 zuG`%eKODFLfs1CM%=$M(*Hsck_$~qx2HUF zFtAp1N%BnnBk>UTuhl`rqw`4p@?o%!>Jt6j1!#UM2P=1+#_<}uxahA4P0z(s`}>A8 zt~M5xyR^7CA3ee+&%rGVO6YTOD9-iXK)=m=OB)nhNK*A}*q7-9shI_Etz-%+FPTbR z8=pwdPoL59yDbUOxhlBTw~Q*tz5`OLhaVVy7_-;{X2n%tt{@%Px+-y@b23oYa{~So z*g~e@DI8|C$U@hX*qv^MW7;p`+mV;CrSKMw&RhlaLdKbT5>CdkmhU&@^J^a$L#*bZyxZOj{f1vlN>fd}HQ zV!^;@uH3E{@44mDdF#hpcBuUot+q(PWAk^?t}(Omh*m%1eN&X=6yTV1ui$OBJo9Xm z9P`5a3^}Uwj=syAU;KLg5YBu&r8{S2Lh(&wh8qzD+Uh1;=mJ^rn;Vad2G$W((J1h- zRN#V!n4wJcR80AH1d?Vq(U&#nVTq>!oV*oIO)W&Ax}!>>vF{`&<|%WW>myj9|ijt=9ZFbg-|dyQx7)EPUo$H}ZTexV z^AM)P@(rYn6L6xdufTO~5eOgTLqK39s@a=BSeXKxSLn0y*XYIXM`FnSdJcE~SBFwE z>)@iT2Kq^~nA{n|Vc48~_@!K0w0u}Qp6VTqiZ>;ChF&@P`Y3Vrmlt8LggdFzZ>OTa zyNJ%6Dh%Fyk3@Knw6g9vjbk>KTCEH1lI+EfN7>zuX!tfy^r5((T-qVI#u_zdgxwkN z?3luhE+zs%wsC&^A4Tgc|fJbX?pC?C|s9SOA@3u(cG&U@YijeAVout`)T6^PYpZB$bou# zDoq1q-yMb_k6(a%LjfF$ki{3ScLW2sPU6PWm6-FNE1V1Lr12Z==o`iqXH*Q~ zO8k59u8SFW&cT#voU1AtsnJAt6;Fbi^^%Z~_)avKtHiz9eI7GsFJ|s79Uu*U-LR*9 zl*F^k!6K4}AL(^EsCS;W#+Jb%4_%z$ABd-peiJ#C^N_tPl&E~&0dFsb;Kw^oXmT(K z#|}Keq0{1N$*sSz;JqmZ9gW850ZJ@?Uk8JC7H~(Rw){Ml44pxeI;CuF*x}v=6S5A% zk~7aRQSK4VTzgK?&^#Y&_r51J$0vaIb_;r>Tpcwo1>v$8$3T1EKFCb3r;ie)QTyLV zjA}OE)Os@|e31+zGVdn$b896#^nsEymK!kVra*5{7RcG!!IHWvoUl2Omi&w>YqH(z3VI8^;{%_W+|hGyapHyvjPrxr zqU(7BxZbvqI_Il0(ltYIpg$7RWz+!AUV%d}1d`@nhn2Tw;NG7<#K7ktDQK%h<6H0P zm1+xSVeNg0DSiu5ujjy#Fnuml+8F1PA6Qn|Koy#r;q~9KcDkuLO-Q;%rg?Kc?x^$K0K`6Rpv161p`M2~nmEHOg5{oF0E94vx$!$*<3 zbxruIn1Qibi||o=8q}Y^O|Bnzpx)P8FeT6$iXz^FW&pt`OP1IivWCDKQ+%h8gT||F z2`(&mMse{O`XaLSj>3U2;l9njgCL3+M+R*vEE+>8SG<|CE z3M$vzQ4*xXjoSK=YRoH!hYwbQ%49P}a&tc5cWWl_h@=V-{7j*NBU3j{)RiNSi9F)J=KxV%lezSAIHVH4t^rndJ zJP^ZJTQk)0kYx(vqKHv(5K5`ugS4$)*km^!#D4L`_sVXQjPC+8y$K{KUyCT*xC~P# z`Qq={@eooPM5?pH@Svp`=Q3LlOC-A+xi62?v;Jn>$C5K5@r+8ix>FBT-#37|;%cl~ z?TH>|1jN!qiHRR^Q>6861Fo=&BxAlR!RyPhv_;VccUeyWdbl06SDz)P>q5Xc>m+PD zsK#jB{7Qt*IjE^K1~2x6;Heq|8u;frELQh|H9nHhj@TloI^mAPn0MrsLp7vLjU}rk z{O9Wxt{B~jaPf7LVA0@+;&H#Mp^iC)d#iR4KRYvY>^VkNQs2Vn4|z1@V$QYB=sMSLb31g0a7#h%((Fm>z0| zo!i&qz02lsd+kK@dvt*m{5(KJcYGwfldW`u>?^GN#(`S)57-~jBQT18BiZY!6-^yg z0y~C&E{-xQz*^bc=s8)7E&oWmUw9jeOYe)p$GK1uFhUNcDYkym8t%kgyo znF&yJ;}YCy>mzlIK@hb@4+oF#fTvC4VZ2D~EHSe`;yB zYdGvN3IdmpO3=8JLc!vRM6+uK$zP$)4c`+%Z~ZBN_1TeRn`0*`&RuAgRig<5lOE%L zbP3jt{fbj9<;Y$|D`dj&;f&aY0(aG{~$~fXTNf;)_8k>=&xT?u;-& zP3;K`a7%>LP1}L{>J7&4+{p5+r;9SQ+~JBA5hOb`lC#G~(br>4s6*-k4Bnd{c%rU{ z4_1{+80TcsA>CIb!uBut?v96R&p8mT>ZD!^E8!(y33Dg4LU;Ri40gN>YjkB9)A^|| zt}Wl{evBsW`}7hW^EQ!hv-D_4mpmqa&cOM%j+2108&+R?Cj(JZWY#PSfs}J8#LQ(l z*XU}@Nsai6-IhjBI5QFLcT~`dX8}0fT1IsAohsx1tqUSXwZPbMSLwV>?eKl@5o|Z< zf`rRE>4SO;sLPXMlI{-^)K*`l3s;$g?UQCf^UNXexmb(-O#J}H=54S=B?!t6OX0Ac z((s{TqCie_0-Q-MpsMCO$IdVXH7*MF=f(;~e2#$V%|}7cwuGFu7z+6J zJqY)VX1p6R;HY9YrWNVY()Xu`--F?F_xo2xj)6fiGgXIqR=WYx7R;jCN2xG#^nZat z{ZVi>`7MZk^8+6B|AOq)Quxxm5|_^J5g1>;29bYe;A*`k^vF$bt;)pbx~pLVi=Ok(Du}Bymx6l_KqAb^51zBYwk%hCRFS3Md2?( zqMISDy$>wai^0q`0KX-VV#Wmag4^Pu%-P&2IJis$hHcfSM!WjR^#hlP^~A>_m)T)B z>g`in_;x&1ZB2r}^CCgny7dr|-9zULy$0(Iw3(W_>BLl|jW?Ax64{9R*b-d)eR%24$x5tBG91GLu7G2RkR~>8dbBiMd96ie3~HwC71x0&y|yb zmIGvK?nYGOba7a}KfPHzk?yc>AbIa&X-eHxa!;n#s!Lpl#RpAs!CVu(_{jwgM{A(y zc%)#wrzY`Xu9Cv*R#+~diW4KxNHRT4F{Qc~e4BIetb-G7@B1d1GNY~bbU&rpbCrwJ zo`ncPZePcHPfmj2wzc#_d?QJSafg>O!%=(M88RXLFMZfJQ*bH&k0^b&JITcBg1t4K zs9H9f+-`nHTVp!uz-&2~)9Oe5tkD-d%|9az7X7;Bn=v*HDMTa8eYB|@T~SpyBbRH0DsI4xRzQRJ5qEc&xMkna3< z0v?xAV!FGIyx-vl@nSt#zpw|)oib1_XdG5VbI$d2B;nt=5T9i$=|`zj@R`b4mF!!M zv$|qXD|{)8Eh;3&4gwmIID{7FtN?ABXTEOJ_ko$B8DCWk} z9CjpqVgYzAtCRGuQb(PW*U&@m5mb%}La|dMnW=7sGpvV#LT?%g<&{x!=?$VJx{Kcf zB|KTe0Ikggse?tV!uokj%z9U`Q{#_IiRRu}A zjmVa1Ma0V?iln#5b0kJ2nixC{eQv6tiLDCUI&OqyV=oOzsG^@P-KT+PuhIF*3yHUw z5k;!ZCoe>w$n=tNIIY82P`W2q!pbzzKYvV!vbQs~ZdJv;;z?*(J%|r=-=o#9Shzba z5Ee-0N*Sw5ux4L6*5_5zV>Uc0{A|XGn5_^qwup$`Pov-4N|BRKi(vKlb0ju4kr}uYW^aI~(AbQ5vLfsio56cMHs~g;VdP?}^?EBe-st z1G-=Jn0KEwn6i|I_`{u*?EOrHYkQmpH>cdCy0iAco=>Ntc=btGt2G{zC3A4{_Al5Z zNW+Plf5=5G6?8c$%kA@fjq`5GFs9kt;D)6%!{l{=<*yce``;Nsey|lZ&?&BL(xN}!{l(pp`?sS64q&WgdM#$sbm_f2&#T>ZsP@Y*n zybKPs+(5spAv7;pj`oEuzg;_@=oHw+dvZa*b> z;D#i|?l^??<`b`!XdHg&8?iWP16N9yz@}Y$!Fag@daeNZ!Zeg0F6*GA#)PBD=_fDq~!?VJb8fXF|@z z3*h3bf>{OL)Wl;fbzOeHxMtW~QWN-woPMsx3{Q|{x_1QN8ebPO{$L8|oZUk_r~gO) zQ@6*4v$EK7HXLKNZlT7Hq)>K&Kba{PAxN1WN$w3yf{`0Fn3ebMz_IBMQ6t`w4xBhn zJ<}w-O2i%V()9vqu-_x$^<1I7XC}6{i>UmP3#fe31LGGP!|aR@9A!BJ$6-8nEiuHX zq|Bm=_O7^3x=2to))C?ln`3zE3Rr3Mfatt3$ApzbaQ3=exLBf{LPI2&y<1kSBk{V< z#PQ_rib86fBp^FXWpTlYFq#!v1m7;22;>gr;)@L_=rlhKr24gS|7SU_eA#zAp_B%_ zor&?-TZT=@TjL#M8zC0s#IkHE1$ET7H(j8>yxSkYuTkEraUTfLeb|d&5n~$*- zhoZ&VO7;An_#W$mr^@_`L%l-N{@;Any2Ijwe@n$hnPWmPtpHX%pieljCrc{3RdnU|t%i)#pzZDlxSi+ChZR1PE&1U@{=<%$-9sBIz1a|e~Yr-7cO7@b1 z1z$E~4%?V|PiV2?4XdjtYh!1tSXwnzq0~HMsxaxH3_o2qKzO#XmhUK=DGXWLD%{j_ zhwbVcFMbg%=H*Ry@Vm-4@=VGuwpdZ$rrLD_s~^0RcUPXo4>|u!l5yW76u*oU!-yc( z@4rUze}zfB@!mZw^X|C#NKFxcWOyfk$f-{FUSTDlFW(~`a&@vbv!F!y!D@mqMq!ww zXTwkIQeegV?NceOIHfMGb(i5s{$9qe@Glq6?Q0On(LC|`(*|PivxnH8yCN7jE}HkP z5c9{mBJqXB_xzXbZLCh0D|_ERguk+~itmgZ!RDV`E>u51fmhLw;1%ZG=jSPYwtg}C zc8TH3d~x0a1EE4!G{1Oi2-eVc+0%zUHO6(9LZh-}G1@4r>|0ii&RY-PQMSglZ3aX!H$nM!+cXhf**0 z0Jo1{pl`tk$cjGOVWM!X{InU$X?v3$caz;jpjG>kK&KDzOmlZ zH&HBhY-C?wP2gQmB(jq>sj$y94Tb)DE7_vOyI4u!w)pSAkK#jDKUn|UeU6{`dxW@( zdCumCc!*=7f&cx3D>?G&Ft2sy7hjyA%|Eri%s2H+<&$qDu*}y|_F&XSR_V%hw&HN1 z*v4K_nCE?0cp`BRzg}aqaGm)!;qYxs_+aHm#; zc(VV1IH>*xJ3cIotsVJL`0zh(-dJ-F+jD6P8;~aCSE`KWU!)oHWK=XeTkeMSoCDI5 zTx46;Yt|cnYRYN;$Ki|OT}RZ}+`b@oY^98imWieK>||w|hE1ffh z_f$N^4=Typ=mu1jJh>q0WcTZ5Hz_{hox}ab!A;LtW%U4lALqiq{FEgA(kaiD*@cw& zREP5u^<3CymyO~@&)mhzp_;;F-+l=H+%sSoKReE@^{8YIw?y)qlCbfmk%c8Em-Fnh z*$KSu5j|FagN zhwOU=Gq$76f?qU6$m_n;u~|I2#rlE1txD+x6|@rOHpmw}SVvJDU~RdkYi!?tj;L7w7ZVe-8}deNLX|n=V`w z$~;)cjxn_>i9g&WeDZ5Kf8^L-)=_;5|4T8IpYZINaO2hc!XIsGVU51zf1fs0Jn!;0 zVe{|-u^qW6meyRszBAd$mfd_OuDEZ?2fIyT+hvEbYErpu9=@?p)c9D zjhpz>+lTXwslI$wxsv!~o>b|UZOUxOeJ^2|NtgJC#S$9@nOp3i@>vq(c%RrTLnO}B zzsP#eB;x;0s+Z=^E@4-D&64z{ZeU}bBUuG|snXx8pNk)AE@J5%RlZq8U3kp!Dt};6 zuyE3z8{&K6%h|L;D~0==a!WHhJ7tvD^fA>y4kJ_&DJs@#Jfs zeC$F4w*7@Hzw_I9;TVrj{_S8Q`%vYG^+U;Xc>FFZW^b-zUpPuKf*Ve=CAAjp?QuU? z-CJ2=x!nfrpCuRBA^ajXu1T}hBqb$2l;6F#G7Ks;+mvbFET zF?>K?qj=we(frZXJ*@JRcK*sQ5VqVpC|oPIj{ooVHF4(O&#c3;3*ykZk__-u=Dc2i zgjhz9&BS}OiX+x$3!6qg~=PVYR^XjnBldYC6RUA$fd* z=^?ggUbL_ydlT!MUN1JCoX*$&2o@^nY4ap4V)2#6|9G3tuf*2Wh8+?f#K-b) zSgnFS@%i8&mM@(uc6=t})3v?%qrbe^Pw!&IhW<|cMcoqdrmzm)NYWvf5Y^5mj9AVt zIBzdLa#5OncGZIAcIEIM>;3r3-*!r4_oOqtLiuil=Vs`9-Cf-VMFYof@Gp}iRi@n#O$u7GR&96_=!?#?P{HnX8hyjbHsyL+>?&Ewtn>@V+nam>w;;*q6E;*d%m zzAobczx{|ezhLuf)??XiVf7uUQo{fxHq9l6by-s_7L0P@6~@VzdU^-3IdP-d9!Zy@ zXH_x#L>wU=Kf{)f|MY|{vru963KQ6C2MyT`Qh&u+w1;c8*R`Q(@%)j!;p_#iBH^4v zi9${F7~zqBt?Xnu!e9UUP5jpPGwWKkLHLwGw(p+`pO$~x8q*Bey;sKZeZFmc`4UC` zc9lMRP`{D4eX1<}?o?_0O>4OLX8to?|KVBo;aD&6hIvN(!G0N=ISuY?g?gq?Z$`J+ z+Z{`G5D)&=&LjM%I~r^vx0&y~HJyF;A)9sHEX`J2s1#amk!F2(b(;)bKVECJ5u5p2 z!WVDTzuc(xz35v?4{eq=dG)^PFizO4$-B`@SzpzA1k5 z`}e+H^VdB0KIeQspL5>6>G+J)0T{SkaB zxl3NzrodF?MR0e^c}Tn-hh8hv;CT3Fyy@IdE#EvM5+4kaf9MAZ*{uQhfBmH<LwaMspl?d0;C<;vV)#Z6=Dj+^&5*Fh!fl>p(qBJt z%W0u;z5fZGzm&$U@)sd?n+p62674JZ{({FZ7J_^4E*w5M8IMQ1$>cbbE)`HO=QoMcC2ERnG#5m%*Kveg#!zk}{H z$v+6!>Tbb-$ra?@l{Ppxw~kmg6bVWmoI-U~N#1q4N%&J!)OWmDO9uNN5S@vM)LBKu zEn8y_*BUcmgQP7Tv^__Q2e;zL-RH!xW;*!z6+)p?BUvP3i~j0aLMBaZ7Ca2#;;Zxu z@JUk_MwQi}f_WMi{fNfq=kZ*-0!OZ|vowg8ZlymL<#Jbx>yWOEEp*?iL@-Oqg567|r=RMADBw$DEWd4-KICm`oJM^(x2uYD6d2b9sOQoCGxT?|4*+OpKw0<}h^p`A3t%B|TQ=#u%HI#4si2>@D zf#2>+k1zWzaK7yik`B+PLGlSy5Fep$vU0ie6c0lATuZ!bERC5-u6Xw9BRbs!s3WJJ zK0L3Bd4aWL#^DgI;hS<&T(}97%9^nzR~8F@2r=)u8m!}FK>F4@Fg06_J{7$$S|%^y zafBD`A6>~!HI9Op$1l_E<08;G>@Jt|R^zRzJ2a5lFSyT@g=KEqIN+Lu(bp2O%f%2P zCoZ5a<)C;?@X6Jcu(#-7ZBy`7qCuJ#E^8~j78xGK=$iZVl*HW{9Vuh z|Lz?m%QpR@Q%qOE(CT@DRmS6SA}3?^0X_I9R>0XjUyH3myO6kRr#@*{7B+^3Alr>$nSFjt8 zM5VyHQ)6Lc?gI4abcbe%YQ&4J0=13GaAxWe>Rp%v3$K1+OUJ#Yx!G$`FRphR>QvB&@elE!)=|lLcMe^)o%O>1R7EU+zh= z`WM6Oqa$GTMw_agdWVX(5zyT9h%Q)v5>ClZ6KqbY#m({(7;xAU@@73DACIL|h3oFL z=qSQD?FP_K)&%oys|C?{JnS@?!+Z9021K1-NNUIJpi`gUgMC9{G)YSa77W)xsmn_^ z8?}P0-{AvJ#{X!W_z${qn;D&TbP2gNRuxS5z7aTtNRt!8VwkDcOO8m2f%j5HuF=QK zu;*zb!h{6!t#~mOO+Emp$21Tb+a_}0Upr1u?}c;u9%yEpjeA{Jk&~QcI!|bD zHS!Ngv4R2E<&HzoiE6}nUoCi6kH-mjG;saYb#Q;ucY(?|b%<4QqeD~1fpm@qtr%9~ z)dX}BgLRTb?TI{ZWXE}obc%y{tyhTpm^ARNx(JhsUJ#o_mf-O&0XMOeq5Izl`0hPQ z?wzfMb)hQ4XkBfdgrtZy>3s@r4vmGEbFyiYc`mVZN(5^I734(!As26$iCD3<*jaXr zxZVw;Tc!tqPyS7qD7Ooe=Srfal0VokSOV!rQ=rteg;iYpn}!OSn1~c(9P@gd@W=~F zN2M&dO-O_u0lx+6(vqb0EG3I&$K$!to47&B3+l(l;iT2!AeZogEYP?PdzVV^Jnl>J zE(9)td<|_V_D#hD^KEJG$5W!-QwQ4Lt-x%hNczM248;1c$CSys&_3Q1Z`K~-<}9BM zN}p<|(Q-%hDQKZ}+793nkq*+;tGOxiYoMYb01uSqkc*}>!2M|;m2@4C`}2iZ_3ARs zme!+1ckkh{d)Glns|3G#u7G1cld6hn`aKYSGSh!>+%Fka$wJVCzd+U5u zjDJhwo$A5o$VR*qJ`QRf^I(0)IoxGF79x!`crUG*$c_ox7@q2Y(-$1V0sVVq*(6Jx z+I#}vHavjriVkd1Tu$DEw39Qn&++GR6IfBZ7(WPS(6*U7P}2ZFajqw#h9UBr_&5@y zi0_}Cf*sz`WUcyQQm14mm@4CqYhP$$z4Ad)>Y0cVabaXp{v9Y-bsm0eA0?6875F32 z1{W?>2GvX2T($dB+>FL`kZUs$SEiZ6#JA$`W>5)hlvqJUW(0nyxJ`N7wRFs}ow)Yc z0@UR`#usLD$yNnpD4u!^du!F`jo1HZj6hbHw_L;;t3QZwGesIq&R!DbYXB45m%>s` z8MKYB5cP#o^SXd9#rHG!rsiU<(MfQ> zaS+z}{6-CpW|Zj8CRg08Q0$`-3375H+F@KgqMwL)PwQyw=0#-3vNIshJ&l!*3h?=m zFVvQ+qu6B;7bpHMRviqaeW#P4nNv?&dL&WR`HVodA{!qFwDFW2V*l~$*m%qU$VxHZ zlrQ1%xx5T@PkrQGs#paJWRx*tXD~(>greAtSAxp1hq!T}H|WQ@BuJnC1@AoHfY<7N z(6bHOp;%EI6r_(~fxSMFVH@cr_8$0M?k4Zt9?>nj`ytRiQ&d~z!QQ=*G+DR^MtrQv z88K}LoWv&dSp8syl{1!hVJG+G}d8G3Ud`HUjBMi50u zZWZC%xFztDjmM&Ao7tlHcxpF45)*ARnaZ485dV=W7=E@4v=VQ_v1fH;&VLEqosH7? z^0hqAL&U2*Ix-$)e2;*Syf{>iycE?s&$&w4&NzItlAf+Oguncf$m9njtkLxmQr+JR z7f#%z-7nX|thIhbynhr_Ru{n?eh`t}q5u~{ld&)@9^2k`u$-(MSRAh+c%M9#cQ-gj zkehr3WJk)WW_~M#&YXh4SY_Hi!wEHV=lzRP1_*Epe${aaS8^fNsbvv>H^8T zxmEbcVl9|uvqZME5H8QX!>y227Ot5Yi+>Cppii|NsVq&56V0p{D3tZ}AD< zcs8Nx;Q~<5X~m>T(l|Ho6nxUip`T7BlQm_%czA+2t)FR!>)%MytDE96tj3tv{;`Sv zVXXv7A1f)Z+Z%%B$Ag9B7!bRYid!GK!{GUIWS@}*hK+V&oQPS@{W}Kjsvncsa~@od zPc=@A1iJ8O8J_*B2Uj-S0PE8Qw7-4@1|Hl`!sR{S#t&VTpE&`3T^y)8S zE+3Yx&4ZeY^kF4*gRnVz}ulG#Wlgr0>3H-Kb)xo(!B?ca1gG4 zIEof6KcQr4I`k@5!ND2FVSslN9qvw~1G0G_6Q_jNHl-1>P$!IYpNU78W(&UjmE&1( z{RL~3(lF%NYVLm%l415DbFy(*O}H%Q3vOBPo9q!kicD-aAE*{w644SC7bd~RCmTq5`ggkAB>_Ll&Vo77YB04Z0r#EHg>y!6c-_MZ z!)l<7v>X`V_uhI|V_fbKuAcKl1;3wSCB!3D-Xgo)Ix*+rrIscDw^^E2(3v zMr3$PCOn6;>iMv8^A$|Ymg5z<=8=Va)Y$IqzhtIdXp_SuYo_pRQZsXQ% zbV0#4!Rll4ao9|VEB8Oc=)dc@`@dbLn=VeHy{F`HeXk-fB4~l&O1B@@jBO&OTJ9p< zuAht;KH_HnE+t>r{{m_8a@g=7oNv&12%i1C0cTqtV{0xK9<}ArV>c|YqO%iL7B2*+ ziZw!^dMoT?1_k*I*4Vj;gL7^Mf}zqA%((H5+iWq20WQJVemw|Z`njUc#1=B-SuN0A z7zZ<#R+7Gex76ABChm@ZLqpC^0M)BWTiz{2l`!*`}rPqlq6vony=8oWuIuX^;dFoO%Ga$?p3~4 z2vj=G#_&xCC|Q0HlmjRlYg~rQt@E(qtSPSV)C0|uOxSR$595;d!--|>BAy?D^MN!X zy{rO%jTOhX`tMZvW(5~N8qzts8{xp2SQxO_gHy&4G+7voSB-}0Qq@&pcIg(G@@#_e zbAB33_;4AQZXFc7oU6lA(@qsgh72%Ew@rcR+DPYm=O(sz^^WYv{}VL=xs-=mbgQ&|2Gw1`nKcjmtt7{Vhbr~pNOh0V#3|C z9+BCx=Shp7q%b=29d$nWhg-ayu#sj~XPQpkJFeF!L0K-%=Cyj(?0fN8iztd;7uj zViz8E(GVEz{D||}@nFB6Qt+fqHr7;(*#MlY@hxDRV`ZP`Y&C~sgDV@Jpz zM*}z$9|RL@Ltx61i&TgGhqixuF?sL|T+1>MSmyT&_ExL`<15>!)XV|$_)Hp}REVVc zqf2p)S0xz!{?5fIE>JfTf@$gQcyy8#cE$OjhVda-C~pZXUFu-RmJ6_lc`X>8H-WdT zAO#9-%gLA{n$*K&9NZhH1is@Hh5J^YBrPij>DKacSZL6I6OwlmqxwXyp~GQ}*?X9E zbZTbCj+h9`54b?))oF0!U=(hfYs0%H_(Ogv*I~uFb+G+{G4%7a1oIlN!NTsxe9M73 z)S~(oE_`qi8qdcG6xOQ1m5gK}Gn32z;3Er4Bb{vAV?~~_8W)4d&f)$I`UY7qm($}D zDXK>gaND?du-7dG!+sVJrKA|#qPH7L$LypMw$DM<>I}VYQN=yrB#t?OCfIbv8Ai5ukxX?) zXhQ7pKi&r-6?Fjje@n-A=Z?|}^8&P}o+>ynZ82FYeul^u6vETL<=p8DjuE5n1{gAn zhr3n2a9iG91&cqbaQamY+zH-HN5>T7RIW3Lo9l)TbS-f8u@!i)t`Z#oDhi+bmXhEJ zdQieoLe275g3qb;Bw(}{-?uHtVEsz+e9iz=2rWpQMA9ZSW95TAm<(-noY_P5c`yjZZ2l7qgS zde90k#s;JJ^lao-5q|g@_7od{Yf2)_c;ii<+ztj|>Lm1jF_C8uvOJ$#yG8z-CIoO5 zXtd2Y3=2|*UZWK}`;I)=!8t29!d0WMR!-tARhSJqD! z1MY#R*7zZH5h-8M0y}rjKzEn%#P^E=8t-HUA4X4ecYJtE>mm~A0 zo)Cx5xE6TITc%DPlM*XId;D|@j$_AGSoSqbxZjT7ohL<=5Xm;uN3 zwBd1^bGVsTNFHmv6Pz;A<8q!Ih0cgC@Pqe(zK(v4u3e+hG9Lo_-Jq zDPi%n4v>8?o2rN1hvk}@vnvcc#l}C#NooT zS8&QRDVTZVA$Kd4Eeo&FnZ|lTBj0IYuA2bIl@X;cjX<1t7JZ@Fh4mdg!BFK0?R@zS zl+w6ps~d!`>qWk6?*XpXc12p##s%Fp2eeQoP<%cd4lh-O*H4YP8kVJkM*RfX8swlG%NSh|=r>@YJsa z7UBhQjquFJI|+%PH+Y3_CO(yDtX zHme$;GaWYc+`#-tkX{3=eCqu@w9fJRicA!OW30WAK8<-93HRfgx?cx3Nm@pc%tz*riq@UK|2CgC%8`o1E3@?j0Fyd4g!KRc6By$0YiI|K{wUWS?nj(Eo8 z7||0BHLXN(tWV!-~R*8oDshS`Q(2tLdB7 ziGcalT<)t}C>dIWq2FrZXxDo-?WxH3W|JTx>M&ff9D=0q$>^b2DXNoHx$1s9Kxy6( zaB5A^zt;x5CdWc)qZDsL?@#XaHNEK7qr-Fm+RTk=U%~y@#>JS~)$pxG9Utx)3zE~i zaY02nq@6E=1tA*f6z&SH;yD;FS19sFYf(}16}FCvK>uY?BxKDvY`gNAK5RWgEv}c7 zLzfXfS52e}UUnp@Ymhr?69N~PA4FZ161vG$5){hb;<}%4=9^qGk zYv2j;GdcmLUbLa}LO$cI{}!NU&~q9%eu^+OSc0de6o?-dxqyRoCHee$gd{$D1>=X7 z;Vdt2vhCAzb*uv*zW6L~{!fn_o}|jt8~VasEt^f`?n~qR#Xq3ot|I;_HwH;_L%5lG zpEf?3NE$fuaCPBAjL$KJcS|Zrd7})gW!y*RhH60V@D5VFhT!hs2QcV&59aBQ5$gY! zL(g+!QM*i=+&UkQmzPWM-1=K_uAc)g?#|+d{L?|-QZ3ABz6sa88`z17m&wvAGlaE% z?X1`55xTLV2pSL6z=aD>h*DJ!UE05wLiGnI{pJp-JBkHjV+vq)R|l~dHzyW4y6`da zDl$*^<0;<*-1SlqNTPZ_Den;F78{R<`s-{#kkAnB-FOF$wa=*pM8lVOW8U%^rkFNs zGW`B~9_A?W2w$U^TG+g!>Y;KpA$beo1|O#TM%!Vu@~*(EEDPp&SmMpckI7x765O^s z9#lmdsft}Dnl-a@B*g%3{nOw{oV^X14Xv1d;{)vID#TUs@?^`zh1g>}O`vqj7%L`^ zhnHGu)GXBw5(jPz!hfX-S|W?-+Y_q1%36KM?U({R!_K&`b_`8w^Tzbd#q{u0EtvJ> z68`zpK=uDtLFK&;td3d(+dl@9nU%^ARkaZpsa+*wT-12jD8XAb-4}vZR+HNztuy97 z3873}r$Fu38|bT!L}j^bvVhJNZg{T=r^Ua}PP;UsAw0;PZlZ=EOZthTMgn0rr9!g( zT8RD?Otz=Z5cT+vxN|?4U~-KI*k3#kKCjn`@|5!UuKOGva(ltQc0aU~JlV zmM*Yfg&ki+I{t_C5c^Gro%^PS^iLjxSC;IC!9G(9;c9!ree!J%BFpzYP0p zD7;GBgpv zG*s-wjq$P=ktt6nZ4C!!BPqCF?1J(u=HQ<552=fp$ahSdCRiD80tUWjV0ZEr@;4@( z+$=GLQ~rD!Cyd1f3y*;N&;{72KAR`{gyB042Uw|l5R3$ggPJ3g3SD5 zfCH*r=QtVoUAX~ob<}Yy<`7}7eG#7QIfTK1JMm$13i<1D42DZKLq^Fi?DI-ye7=p4 z^v0=>W_FOoy6S-Cm{1kvVtC`tMrwM{h3NJEBoDC+oPS>?rB$DB@6B?UQ7Z#> z8`c3=hex}V{D}LJ#mL#`2uVNl>F<&@!Lmphxci`y>=}u`dF2tvIXVV=qyLf*PWMRD z5*_e7bdR_hbbv~#sP-=m1V*oveA%AHL=Mlw6#Ii@yjeNSS#CqxlJuc%#|pA|uN!px zws1GRb%X0yU()}~Mu^G*15y2w4)&AE@yCjzSfu`$E0L{2j=I`Iyv0{;Xh06`(>H@^ zwM$?UHwM$2pOVpKt9hMU4%1WN%i-jG(VTu>EBekiqLXGS!Gd>+80nD0UDEuCwwdlk zgPG-YdRq+?SKR>94<%5kC-TMBE4ecw$HPK1U9i|BhNV@3v{2_88Ta}KY?BVafz)$? zgeoB_^e4c{5>@EPya<*Jm&szkEjTVWmHXgrw!mY!3I|(uf>~)WDfM=sn$Itw{`)qx zo1+2SjLhNtQB~gdMSn5t$rajKqeZ_=E`zHr;&l4~TTr`bMkl*w2?{28;FKQ$G;Br$ z^qRPniEAbaw+!9^pGrlz(c8;-b|^D$=aIy`%ESoi*${tF8iu3!c=)Rv%Ff6a)o6#Y zVq-a(H7kMJm7fjv(@eqWj~T67|C$=+{uKDsR^x21>!N){MR2|*n0gGE2{u9`O8zUL zXJH&39h(HlK8k#WwNrTKnKy**Q;epI4&iS{2ariV1>2J4g-d(P>9gnrf#-yG7`t%{ z^xc+56}M(A+H?$`cy7hg?)lLA!3maVUZzi6;-L0w7peC)CR41Z3C25_aWC-pkapcy z2szGlY={k6*On((WFpG5Jrfu1y73YJ^D9Nmh&s$PJPb|ylz59~6hpJ&Fcs!nLz0Oy zd>`dwecBNi(dk9+JtI`^P7zLia2oUNFSEZk$f8mGMvON+fREp-;c~3oM45I~k(Rm? z$MMrJ{j@xkzmJFOav4N@=Vg3mssVGRyyTW$nk}dw5)%qXf6_O0E2w$DBP>0W3EMe! z7~fb(B<}qrHU2ZfU+)rmHbY5}x+)4U5dp~GGQ;S+Kn%QYMRz4xLZC<&o^x6apY_MX z8R~|1A(zRaf7ZC&#DW$$PG^t znV2cac@PBhC*{eKTY2D3PSJm!62iMqeFEJuiZ4oTpw;?4G&H6Fa7iR=>D!1}1uJ0{ z4udS@V?RkiiKgr5ex;F`SCoOH_%cxrXd3<}(h_7^*9kl1H-YXFmE^={EOBwE%Mj$&om;My=fOB}RU`vrFS@5-;E;>(ftw;m4 z`z8kW5;H(Qt&JXPSc-Qxj>Fp_s<>CZ94~&3Om1#41sRA$Z*20JvcU1*7 zEuWKP8{d-iUyot0W(DLt?h~{vrX*fA94tE|sEkE6w?pKQRov+#9vB6ef>?n9)xIUl+>BJvtLyIKpYNB5ONc9U_U|T5+nb1;-wXPE@h?&y z&;fV$orJy)6XLdc1+G{WgM&4lL>O+y+v=!<6Qsn1?%#sYp`0bxzF(v>dZxiu(VUgc zW?7^eDs-~D7T(?vha1EWlaAVLq%*J{)W6Px?fGMgSZNpYb)yf7F>J!jb!#E_X9|A$ z0@P+!E9t44fg#o3S;dn#p?}>h$Sc`O_OGkt9umGMLlS+s@VSvF7q5%E%LmBGzr_%$ z$iqMXg@LBnF?!vi9xt3UM(>hA*vs$0#CU615@pKs%KZSB|8v2eKNdkwsig4MdK>5o zn2I__%<0GnBjMn#{h0OcH)y`EgzX*~urhHJ0y`GN=++}l-a|>=@#b<`TBZs|B5KLn zD-(Dx_mvB5g-_wKcQD$!UckjKqrjj~h3EfWj>J9wLY_>}1^o`-SI6|yi(< zCoRG28Fj(m*3;2nTt=u~^_RLI7{!H^fJyxpT5Qw#|g0rF5;m6TK@V-?@f{Pcy z#e*XK*U$#^y-TTEC$>-aJo2M;> zs|8$Pc69>eWl0b>kQ17Dw%K*8NI@*uq%j^0lNi6tqbd%zjg?CjC}!hZCv zn}WW(72wmzWA=235oFBNf<4zfKr7c4Iz`#Vm7A{NM5#lh|EnkJEqRECPdUJj?ftZW z-ZB(S&7)}vXQ60Rl=(BR0zuzOuoL+>Zf(i1e`FeO$*8)}v{70pd!deoYL$^<@d>o} zxHy?q%!6$j1I*i}JMo0ZD@fw{(4De@w8=_<)!+FKv z4j1X{#v|rB7_@u^27Rsq6We6`XQnFB308vDRdsT5J%QG)Ll|&2hU6;rk->)@^ug8B zFns(4ebS>sPb^6zLmEe+Ohp21R_(?Kl}qT$_aV^s{VCY5c14%zmgHxJDNMX@lZHld z>BGFKbn7-N7<9cw5;qlM-rTG3?rSP$nz)0vYb|~A(opDLpAYX`FM#|5O}M9a8IQQF z#K8h*!0qzFv4eKh^pQL-xVHjLs`wK+B@Y4rRttuFT8~#X3y^DQfsXbnSm(V9zihwHbxtZFa$0&QvWa0!)j5z> znU6p2Oy<3eiA6huT9Oc(L-xMP!a-Y2sJk^A=e!RhpL#zM_p49wyp#v^Q^>r5ZESR>E$I+7-DbOX=NCP#J;BBM=EW6%C zldRVwG1U-?|C)-$kH$j3(?3$YDG)!aKgJBjw^UrFNih6+kbCLcd@PGTNS(f|$6-z% zMrX~06ZZC47Uh7Y$5eTO&nsy5wlt8FuBExjj|lVoId``h@OaTtq-wt|%pLed2X|#b z;BN=y9b1dl_rqxDx=bpkZ-OVdjdY3 zmfk)fuu^CvV`C!8fuH;Fgi|Vd?oz|kK`U@8^NEV)uJHJ+o%mS062w&N;l!0D4A(8f zw9}L6)d!0)Sam#4oA-d0*i0rEeS)+|HsFz#Qm%i0J;wE&Ao?pl zWBZ$}@NJ8VFwuJzTI^f`PdS#n%umU1Q&j~+S(@<;8*U7yigEIk0-j4_u<>Q5NbKen>w}`5E)}}_`0nCYC9b17OzZt zj%Lvg`*5mYr-VFVHJVO|g1nUx7&PS++#Q<@djejO-!*}#QGbH=9?BxdXGG^s!Ur#L zgs8W=hv@W-g%{0dXvynwctc7F#b7Ro{}c2Qk%M9>J-VU<8 zsU3Qf7hu5Ta44Sfi(J!drj-UaP*ZIkmRO|_2fjK4{}g4Yri78R5|lc4M?81U7T?N=K~=dKaqiztZU=lL4u6#}dc_gCS%&zI$*wtw6`x zyX4bLdAul^6~Gu%j9kHRt9MsIOx-Y~t-J!_v7u1IS%ivHSCS?6jtKM>i0yj8{Z$#m zT`NBspQp{o>sjJ3KV>e2ecu9-EkpG9W=eH8p9PilMi|~P0i3@cgxyLNaIg6zJ^9Iq zRy}<|>~4$)_pNsjmb+m2dpYPm`WjN|%WWe{Nd^Htp{f}vRzn`vkI)Zf_EXkU&94Dm|p{D*s z=0`J+ow`tm3DL4(A4qSr)?Jpvk9_vFzP{Fh?;w%EjPcH8|NK3`92&}JAEz8-OHERl zYYCaGads**wAYF~HGhAD-mw|1+#+dCl5rTjc2W?->)FNjbnj%YUv*)8S$DShRy3ot zHjp(i(`2$As<9Hb6Pcz=Rd!QP4l__&$Oa@9F|YEDvR9%T`AVN|aV!f8_|cMesiwJ-BM1<<0MY@ zKmcc8Mglt{J)Ifq+Q$}*&17CZx8mEq?B|%hp3^YTP0S`71dU?;1HiOfrbJ#6~s z22Nc61ol}>Bd2Sl8rvrts`;7xk0U=Om=%29!c!G8}f6UEj za9!=yFyyLjqtgG%CL>>sjr1tyUpXPh63n~_~QIb|{ z^(IyJK;<&djl+h_^V#z`v7%V%lfvChf$JfD>vK=$gS`}c&B~M+j?!mk!HDrn;IKXH z8PR^RrJVPgmpGGei`le4PGhcX*s-CfQyB8VpZ)MHp6}Sdol)PrzW&8tPsT%L7IS*O zKL4bm6LVzvF8{%V7QU;=AI`KV2l?~9_Hp?C7{;qsk*)tB;x(Q&WN)pG=Nz`4$uZj; z$Wbo%VWahFJtwH$fMs{*ay%aBvRr-+Q&gMB8Z_lHha+;?(_T;Kz4@lh`cAZL@Y27- zAFBq;)93-d>@OGQ*&0(eXWx2OVUZ>KL&W2IaQhkm>^5D#&gY$sUD!+hl#(sX)b5iU zJxhP~-ZXQYd9MC!wSyi5OvtF^wJ^JGe`O;cY6=litc`W<>|O!th1{ACkN8LvZ>Z=i0$WSfm=U*BBBWNk8J z>D5_GmBB5(nyWAKZsLD@`&KLK@Q;6Olr1N7B#+Ovk-WW<{UX+6qc+Er-F&c621Tb;h zTiMHBHnDx2SdMsQ02_Bvf>CLWWIrf`FdroH|tl__p z{EBsUeDxG@w&tC!&E>0W*%_4{%tp%qb_e6dC@-^RE*09cSCZ`+)%|vC@)S8XD@d9- z6D7+o$&zI%XI1izHX=vzT039-hJX_~(}tZ^HJ#~on$JGGIfME9%A6exH)gikn6eH( zXE1WV4cLsnNz9JuDXgiol06?i7~mpUhObCb3$fNlY#$l^N8CV3p3LGPZ$H>`Sj#95zUTJ^gZ!6JR68 z=6(#f!I1L}PgCdGM9LO5zbHw9`e-rhJL{Lrn!ah_-G#| z^UP-c`uH3Of{6(-$ zxa0bUq`&nx*HaGEXPn<>vqL+DQCgA6hU%m-kNpp@U;IutY&^cxreL`QfBgIon^WDX z%u}Nj*4H(ax#g3@%AWnmU;phJ=PUD;KX%avPEM}}b9|p0d*y@=W9GD$RWO^)?g_JC z_WZPAKN551m(Fq~bB{CYSmVmf>|DmWu6AHMta!{zduLX7+kx?q@8Y=aM1H$6a>_1N z@<&<^F*>!G?8Ne1ChC3``_y?gBPq9nZDTx{4XrC!pD#_Eg*J8kYsWe`zZ-=7MPaMh zotdu8wM2LJKWi6e@XmYw+F4II+CQH2|73S?EO-N)_Jd#fmrlIoH1qrTX*)%0cizrn zE7Q1)f`c`y7#hS}4EAGBuieQ=_V}}*&WAYREywt^uMTk5%ADX|s*7M6mxZvFQu`TU z{w{W%-+uP*yC_CaKAt`48O>}nU(4P*vYK&z>cK7wU&Ty4(ZgSUr<1e$wKzMyx07?= zb{9X<^Ezj9UB@(P zuVz=ptz=HMEoPl69`O}T+c_TX-}qJQ?~2$|b|Q`whegL_jK(ZSc7@|Zeyqx4j?BlK ze6_xA&Wp5Oev#Z=&eo#${JEr`Gp%NjBW(N3Z}A-Ar0p8vt3Sq{uHne8Xn-Q!##B&k>u;?z>spfndgT>wfm@tT5)Ihy~KW$&>jv z?9Zm?ZDR7Q6*%%U<@hVQEjcUan)7#@kYiE>QY^hK&2*&w;q#YxbCmyX<1ZfX!ue*s zj=$-T8*@zAm0irNVGiDKVPB&OXG_dfesrq7O;-EOhE@k5XVciLe6jn@qFwU0`1aC) z>@;=@Q&+K*{je^O8L>UU=Bp(#;R_SkqCN49W=lGwv_F|0yD^Jd;g-T?IjgWKZSzfHhzB;AP=u>_6 z=%kl68r;r?7T-o|=jgPCD=QDNHup0atKE6*h5MOI<9|WS-70_c!%hC{no5pR*=2rV@3*?@{2puX zX;T{9wBv1#+#j)tI4#BBRVl-H@K=iO6I#o8SX;qQx_FcGDESiqV`erx+cSgNCv}(= zp3G!ESKh9FGyQAb?FaiC9!W2?e%&#TRk$~YnN-MO|NOCJI{&2btHTO75D>z5iA&`u zxn}aewCmZ_p3`BS=O3P{uWZZozMaf||7^%CzOKQ(?jFl%`TyhJ_OoI3Dyy)sFHU62 zBO_Rocds~sgXZiqEqex-dF&Az3uZ16am2T&Guf)Y`J=Z#aisfx@V|B)V0{u389I9x zoBMSSlRLAV6ZNc=ljHD7QenN!43eQ||+HhKUR-J*de})1Jjxw~Mo<6t$T>q8jCx+;*mr_lJM(>T0Iv+$QEi zt}E*k>COC)p{oq5atXquf&$VKDxq{K66f0y=@Ln45Rfn^>F(}E5Kut`14S%6XGiSD zR=g&5w_w-({_{K>4!g56Z_Fzy;*aOnZ#}5=ToW2B4)gX$2ttfn2}gdzYVzvKMv}uk zWek24VeiKzIDRM{k7*^rSEmQGO@m8<_AwZgxR1=Jcul_NU!X^F49VqOVQeq)!ko96 z@F#2$_UulFUGKW7#%ciJn0&4{zd9_qu$kyC$)~5fe=$Sfo9W;uJ>0t@9Y%bF@wDJ_ zSl{x39$6YeVjtAfj$AhTWmg9Lx|xEHw#P$5Mm{!kFA|f21sMJH1Nm%kfN!!)fY}hIF<;Jzav5+C9WT^AWrIbURU+?u;uZ)8ST^ z0~*jc*fn;ToJ#pXZsQ+foSDNt7RA=Nn5{%hWeT2X+(T4?{qb&nGMss@fbW0!!K?;9 zJXigUI1dCPZq)?(3AQL@L7b&0zWSB{d#+@Fyjwb^EH8lB z-nr;Iu@FZ(#o&&ZG#XF&Njk4i!JTg_iONm^bX>iQtYF_dsp5EWpDK;SeZHhvFgJy96_rrGsH*5`MkQ2dTbpcsg$u zR5iS($C_1%tnVdW-=P>{(B%#r-x(uQ&4GYiTMTMZh2mLkmciX_-snL-$8BaZ9BG=b|Qh6Qi<=;+kjM4y*TGAA;K%EA5SF~uHeVxPq|9(KggpKLEt@R|Af z2+&Mii@dm0jxWFDz@jb1i1o$rtbUX@@Ym4odt~6m$`AC$wMi;3EyLUn=109hs$@~8 zF50w8z}0Oc*uwXmgvxtC!Jr4)`FO!bEe~v%*-UUsKJ-+{Lu39tIAsw9&!!*eS)&Ti zHQbA=b!6-2Drt`UYZthpPL z1>~E%haTH_j%Hm>AUZLZ>D|EPv{qIc6{N?he?0x5x9n-k>&Hjm?O2 zgq6S7!jsc7_%Kfu17Y{xa$JcKqptit@+wLR}*F2NQ7q5*aa7g*jWb$j7f7*ps6L4vnsmDv5O0 zk{`@3sqNgEc_+!^Tdq{&;}nK@>V;GqtPh%H8B;PYdanOizIz1;C zyHqixr+$(r12f^-gLKFoLG5^3~nw( z^R3V6l>^22Dex>Ee`t+2!Wd+x9iqOX0^~tn0hA8BXY+0r!tHN2h^IQ23hQ1cAEgh| zTt0owkTd}I*M^v#qz@J!2B^AR1L+b#>d)ySE6zK?+B9p_`N!t%u>hRx*t=SY{>X_3X7RG9EVNuI?WF`kg4IM-0m8fMS2K&Oj#0LhDy z^R)^#{w%}1eRc3{X9aqs&O=UwGK}w>kAZD-Am7jnuS+k1pUpn_*Ob=uII zXpGBQyn+ALT-?;51YMJcSYW3F?{Z!6bEGBwS?7T&t=8Zveu#7&I6`G2PLX)glk`qU z6y6Yu0NA+(rK}?1^4c=$mtI7w|7)kFLbXJE?mUdxrUbGk2Dqs}4YWSK;aFR4H&)C$L5bKvkx2mCm&7~IF_K$VCx3MXoTT*@4jlWSmX zthV#U^L8-<|C)JFpN^il)d?B8k>;dQAL^-x|}G5&!*d&+=y>M zIZYN=PRu{%(c$ikWNzeHIyU7Zxte;67S^mnu5matt%*T>-3Tb{Sx=pec98vlYU!cT ze)1;Kg<4N)6Sv!$RF!8+hF5AqLXigQ{?&)0q1t#orvV3hYQZw48MU9*f#kw`Xemg? z`2B^j;7leazNkbGu`<}!QG>U5<#6$e3I6P5vmSq&9V7b-L!j5x50z&| z!h}`;+7z|Wrqi`V@kKKY`qM}be4LAU0?JS+ITyt*s6bU5gS)fM;J&Q|c3w9JM;T4{ z^JF&OeyIURkE^1f#%U_H`xM#t;6CkUGj{4FI^apbIutSA01~U)aIC|LrgjDs&D)xE zkEa)bdx01t<_%CEgryvBh)di>gyr{8sqG`ALiqqq+g5~Irxd{EEoHdGvIs;}&3UH( z8aO?wcD%8aagM!Y18&Kyh8Hc(xJbShKD1ZjYoQWY>|KXb<;&n*yaU#KUH}71@pKNq z6l{OEpZ--?1}oEC$=?unwofW0gZ?f!ZND>5{evu~viMHlvsg^Y{!D#RWzn)(0ut+E z(f3X*=^fR=ow~nh%$gJ+BVuGlu>~xv4<&lj)tIf{&XYg4yog9ZA6e*Xi8n5EQRR|Z zSQar#mJOa~=lkCz;o%$FC~=0j+fEo~q!}{f@A9c$nm7*Rt%0*HBg}A!4a^GDK%=D} zXleCrI@p;>6Owi5nK@#-b7R}6s>31fw&soWyi7P)a~IQ{Bfd~CT}r0+CDZE(gCwV2 zhbgNmpwsRrz~F2)o9>DV=H%jITvA!kr&L3 z3deyd1RrR|0Xn>-_w5qk#^Y!j7P*P&S1sU7QJYDwcz>ZM1&gS}$@}zNj1)6x>w9{B z+!x|!tJ0@dq7XEtn!HJtC8>-1h`iuyMt3-dKK@uqA0PQe%g-f|#cR76^9PTp)q`~A zXuw(K{-C$=(-s;>Uk!xk3 z;gc7+$S}C_Vl{O6wh<@3nQ$j#f<%4{M$@QKD$2f9G$w+Ci0}J^y#EwEL^(kgXtyC(9D%bn{{u=BbCc^#)3dNo;96H8(YI;xpMla zPnhdbb&Q6UR`GcJh7j0XgzD~2bU29*wO9Ew!jayv!dd~YyzwA)Y_8C}bpv!o`b_vT zT^?WX_?h#%DY)~G1pbnh!BN3jIA>aflRjf4xF!%IH;AI^)f2pl6^roPpQprl$8_lC zn}zKc%*nC7=QL-57DisQ1t*>|cJ7sg2yPZwaY|5k^B|e)5r%dly5w(*BKgK_A>#$c zOw7P6;;NR=Vc;%?YTz}6J2O$vDT_Ll~`jvO!N(Nu;7&dOpX3S z^~`>fkUx^>AR|m}Y^dXsPjbK%Sdka?1`wbsg#%H2L~{XQb{#1oS!N+Tuh>Imx9mn* z=g(#ze2b=n-owOwe?9GHM4_{4I`+I0f_0DiFkWaoO}1V~xb-?T_ZMX@xGAHH4WWAlUg z!EACix+u>Cu~}JjR1)jl%=OP zNs|pTP3V|KBiR)pM7b^o=>6YJHv3i{M<=F3`BZ*9>GOf?y#I#^nSCN}uWlx1Vn%6& z=yl?=>NPdiay7p*TFO{F66QtB5hJ!QL(p=A9~gCc;j=@lU|#bwym8PO)-}4}FJ&8u zbamw^Y2W0I{hj0(%7q?0^4J*77wf@)?Rxn4fCgOVOUCb7aq#_k0s8)z47OfB$gQzS zdcsd1%0=Z+%%cK2_m*O}W*zv;S7HI1F+CIF3qkJrV9+5*Vz3Zvw_-fL+zJ;s~*iRvA z8yW8%px(NfFj|?6FRe4+>Aghc9Zo>$iAcC(9)iQBfw0zagyg>XK~*#5VG6$z9?@Qe zP2Y@I9>xfFd2zr@&VfAHoGh_OduG{U8qS{rItV>QZ#Q?f`RRoiNeH5z?QiqL0{2_PbKTD19l| zzcm76vX^69ODtHvkHC?Y8K7(ykBffi!p)P3c>6>o)4PGqNPkdFL=2`BH;)$Tyvb zRHo)VS%0npueH{Ize*GK+^PqAA$_>Gu9g1Hv4t^q)||gd2|uLu5k*r&bnN{=4rzpg z6xR}WDhEP&h$AkU-eGPbv7OMOBF?U%jpVr7I^3~Q4Gx@KhwHZmgY+wV_<7S0m)>@U zfJ5Fm8wF7QUKNDK3S#ElV(@#u22(OuL#tX6-aj1;CqCq(>&qe->2k(!>r!^cj)T`m zYw@PeT4=0E#E~H}aw938*LCkQ)1|zDCp8*^2}h2S(M#EwwLk?9>AoduZ&Y#Jabeh> ztcX7D!aS|CWAsY-HhOnjDP1lz1?$?!$dd|6b39Ly<~}nxSF!*fR6By7nK=%H&mdtU zo2cAdUlKZP1AX==6d2(|+@jzGzB^)&QzD4`Qa2l}16=np6 z3SxM+oePVgP$|1tI7PkmgK_jo2ZaAA9+o9d21v4uC=1cA45o(wVo~&bOI{*mwp#616#uqe8lz$ zJsT?Ujhi|seEC6FOjCrRPl7mBvVm;c&80J%4igVHzoj~V8WoOb7x%7RNhi9G@Olgu zqwl#GxYOW_HMELh$H7@nmLhCL7{U3U44Ks)Twv><+M~8WaOA<+Q zt0eTdi2=TaIT(8|0pxpyF(%OzmVUTR^RLc;sUoHLOSlNk1sd>4TPZvPI~#b^_{5_bCJ zVu(DOH~x|V$9WpKOxprVZmq_n?<+x9a4pK;D2FKJnHa|9z{N*qI4b44+99A$G@gplKNd#8eMLxbd_&Tow_p@i@$>Bn{^QjzbUKfGSu}1EW{S%zWidT7od>M>;wi}k(JAkg; zGW^JDAUM}bAXm2-)$Pl`@L@5I?zBg#HfxAJYK?BMY{2w#0QSA}1rg~mob3?^L3Ty3 zU`aNX@Dic$SORY7D@XaS#gPB88t)gELf5|{D3vY1sZ%TAb#Ea`xP}1s2VhFya`iI&v?2ro)f@OHRF&`A$EQWun zNN`4_rZM}%CCr09zM=_LPl)e}H+1)+ak4n57>vRTaDc@cx8xV%;gmM&>eEM}HdWEd z=x%Z*WF`3QkHm1%6qa{i!(wK3IK<)*K09pCX}1;l)GK1<3wc;rI0q}YD#EL%YPeEY zg*z_Q!VTdnbU%2Bp647R^&4N(Lbr2dJpBL>@!3s_{<0Z?en|a_`LKNJCo(GjiC&-o zmSlY^!vkANK=@iE=KL;!UB?O`eKHSkgcJdl$j338rTEOy8T2KB@xeM5cynPE>`rBW zcZ&*K9Gru?mmJ{Ndt1!@XAe)7+u~CtFI3EShr#S+c;C()t z4_&dL$r<{+{m}liD_Bqe$8zC+i6kS2JwZY+;yRC-UwqH>iOr@Z-IL6YXVtLeWjUI! zYJ}FSm3YN}ocx`2m-?I#0AJ}R^v3KadgECtvGysT%$8L|L#hn#^cBHNuPU4tRsxzn zh0rFE%ht?tNVF}$l4(Dv_~>`yd0ZHCyZB+=-gu~+h{uUXX<*5ph`Xhq6X%6bY1h*+ zvVwb`PUfW1H+J46Q__R()V3$jd;7S_{&CE7OCyeKxh%7b?;rJ);{)yS@6>O`KhnE? zHj&hkr!r}8m<`)!QGSD8L`3l|4d4?5UFVN9gT?DtlsTckmk%g}J7eZh89Jtwz^rps z$QQu!oXbu!=AK?W8N()X?^{o}GYw4f=2kPXJ?nt7;{=vO)zc2GT=IMEF3LYqLk_NJ zz#=o26aCPPw`A+!gRVGzebJrx^BdB+77pavsub(!6jec7A=AVfT^aK~9^KT<~WV)ABE^nl|Plt(r zXb;`k@PU&9w!9f99L;lOZFm9CzR;gSY<}Ss_WvRLn)vmVL5EiXh6vWcpS~iTUtfsp zt+L_5nnJ9O&IPjr5qMuL7{*;tNADCGOLzhdsZmakXhXQQPE1q|3p1he*c6<$MGO|t7Gi^P zMB$H$F=wh!1jkq97e_(#mU(^c3;Hxp9psN>QIGwNf*095a(Rv=ZW1&%I)db8+=RwnpFHJA%mpo)Jztld_HfuoTam$(A-)T1z=Ck!U1 z4->;T+o{f{Tg3bDSsJS#NV$(^5YzmfyZ~Vlk~3=)ozk$6EHvLtjm{5~cj%3eWL;rz zwKKMQErrpW5s*4B1k;|zg6O>n{3#;``r%X1X_+Q0NuG)(`K(6a?^im}CJR+&Qh2S9 z$9wQbi&UN-=l#%?A={Jrv0&;qB4+-a`j$-)MOF`WIztVG8V#UvzZP~(y701^?YYPM zs(D+^R2}q_?jdQPn`v=pAkhpfq&l?~*b`C){3~m4?T&JA8^{5tvq?CzJ{zcXDo);? z4O7o5;(TH(pBK~n5 zol~`f3{T{M9xoH?Z1dr%U^d3Te@@pqd?wyQ?$m>gS+CM70QP1|k&2af{ z18QHd1kR6AY!t16>dj^NB0`HyRkfvOUx<-s2Mp<<%m_46j)a?2<8iUcN_cs_fGD|# z)2BLX$(fS1qEip5*)=#e*$6l4Yw_{@0ix z^KRC$dg}_jc62jIc;89okMoH2wSHEce2muSY$L&c&r-qg-K6J5E#wGR;?3txFhjW- z4Qg9ZC9?rGWVhqIJ&ho(H$?RU_L8F)xYX>x5NQfnj{g7r!8C0J78eD;Z>}sJIwk>C z-TK(&Jrnx4ImqXf0^xUyF^WF}M*5L+J8}+BWkJ8$_@3FkpZk)?!H77j@M#%&p&m_L zABUg|FA(+xMWQg@a!|Pz4hCx@@l9w7w2iGsj~nH*Of!#oyLM6yy<*bUXNe;Z%-|h5 z;Sog+d{7j|AIra!g*+k5fAx*@i+tj42`puL?&tBiM-DP!H)dZ8u764ue@~JVfgh;#7F+m!*a{bnIYNn)HFg>=!qsMGP+xC_(^w6Sh=@6R zU(LbwKA>;1Ijld$V%>2+N%CP~l&j_giBnBP>Qxb~4Cx_X607Lq26;Hg>Zk>dsY6KD z9E61ebVTnPld7OhFHL@9uIKZ!mXMEB?))1vkMo}Tv=b}}=fJj9J6s+@*jX+UepaR6 z#J@BcR!L#Go;LDdWCy+dWj8t3xrOGb*wSkIbBt$DA6>OXkDQ8efp<<8n6KjoE{WC{ zY_b-_?tk`eF>f%_UWWF! zETDM86K9?Z#^vkSb77HzgU>>t@?|o-u#Ur#<`s}NvL8dF#OYfPe{2zA;bM_S#o z(0^+xj2_QME1@iKuQJ0MzIJe|nS=3FHej%*31pM%@W%WWP&-zS^Jm`Wx!mp~_2Gpy z^A0;p+UStO7gK2KUMu_|BB>HuyPRCMI40xyx=d(JARdR2hEa4oN2oIIolk4%!mIRWT{eT*56b0d8d7Qsb9ImsNjr`_3=r=3FKoeG%QQHD|z7^Lctb=>= zTG9FFL6W>}Fa5nuh(sRWKuec9!J{d9xNrjlm-_WFv{w&~PyI=K5357UKOtOsSQ{?f z)WWydWZ^;E9DKYdhXx$FNwyfTp^tW*B=ds>q1Bkd(ZZSF9BhQSBn>pX(=dHB39b%h zVCBZe$P5`l+9?yscHFhhoAg2;eo5$eJuwe3FuA6zElN|YaXN>FcdoJ>87{li8 zhB$*`3>|w5m}X;bqFb9sl3iXMbYis{i=5r?9uomya(pnpwG!fKDT?G&!JND@yk|5I z*4|XXBbDkP=%azv0_#b?5_=B3XOlN9FMe{+2d(SZK%q?#{;6ICK6bP4eU~J(is_=l zFGY~qSq*pH**SyN@b>6bVaD4)`o>v}px-zzd3zA~k?N1jZ2~}_)v|pQSPrE=E3qIT z0zxNOU`}5syo?vaGyh3KP0f26S1}FTMZ-zEg%zhwxrWRx-^I%viAU|3fne_Jil5xQ z;NEu+RJh{`gJLc?N5&mUkp=GE;0B&oL{ZSt3EtXXqOt4Lh>`JQ`k*m|n7(_!M84;v zEem%M%kahYW@{7uHT;Mq^jFf_qMJn9*Mq42+edFasUR{SUUENj2!ZW;%lePf7uo= zPuV#~>HS=?>~a{2Di*=MkWnVnu>f|2FT>V%!jSXoI}Q0X6&w@&@Y3Q{aD2&SUj29k z95)@I8%O#n=4|5Kx@Ji=`;zgT%^cpAAF(*Js1@`*|&Fld7Xn*0TLd zqz>*%{7fH>G2pcUc`|SAl53l1(WaVMsO_!he(hfZX9S~Y^p`nQBRv9!QdQtwrz$72at0x20C#0#TauG~0v4zhl3lZ~zQRf{oPv*%bPYZwBj2kYpNN zv>_}+1p~`7Y2*eq98ZV>_q^peuCs($xgh`>n|~1NZ9VjTI|CKL`{{A&kDI?gwbaakM5fBBpK+46-vdo50lDk8{Mn~PNLYBJgW^Z;$F=Yv-X2j~NA=Y-aWeld}zm83Pp2R7zjV^j}UFjKi9psjwJ zlqVa)_?bj}k+~MYMgb+eqT$+$ISg-jA(n2cpyN-Bu=Z&>>#rzAOYLsr=1_{gDQq7U znh9AJF8JnA4h%h*hV3txqi3lGo_uYA1sO3^{zoCxFUg0EONB^^aXOmVl)=FHd(_}` z37nX!0F6e|;OVKwaA?C3a_PVjz3-+B;%0NI&9{( zuzHF{+}F_x<6DFAp1T_uB)FrCyE{ZYy+Tem@1aX4n@CDp3tg2Fik$@kz&q}R^Iil% z&DJ(}>)V79g&mN4v>7Y9uhVPy&JnpyJlf)Po+uAq7mx!;aqxTCVNbs#FsBVwM z9c?kNzakRzB{gyXkQRI_V^C|qA*_5{i&Cd5;9E26+w-mlf$Myb`|=KTzBx|59=l4X z<_BYPr#s~P`D2`cJ9K1eqGX94Jo##h^1g=f@>&=i^bf@CCsx2$461cTRyG&`D(T&2{M>nQ&sF;|!3Gu zrv~6Ag+5|GQbKDrYRSBgN}Afeo2k@{;%Pmd!nk$^@q}}g=~b&W686l*tXvl*c}=uaMu~mSx5jxP+%=~TmMOF5`C2!`w zqgy|}CtaVHpp=sb2sis7W_rMd`8&y-J9}tC(s}af@d@gd!SW0CT zW6N;+A%KS~O!3_4QrNeU)$j#;BsH(U(l~|dZk$7eU>w{i#R(IU|7$;D2|0w_3Ki1VGd&|gL4 zF|}!#1IvzvL4Z>_Iwk3WL|H8B3lxJdkMGeMc_b>A-_t8I zLs0Cl7c{Jk#C_I2knyAiZkn=OWJWRAylTMJAq7~a6AxbBR^h`LaqvjZ9^2||L43J0 zy6mxq#)<@ZUFVCiClju3UW>Oq?$V+*Nsv?(!*8!9YE|5i<1npILh@zv@=czW2t1k zH!KQ0y^46`YAGDr?TB+fexOn(`M|1!4<$VKVTi>dytp-Rqr4T(#Ofg`#1F3vF9Szi zam>7814Bob;KoIsa9%VNcPtEo;Em;Au)6{u-AW#608x!%%nOq2DG01JrrRbYm0hdRs(I=_~yzH#O$`dFD=!XZ0CDa`7Wbc$K z=vHduCzXW|7h{f^8%;rUAOZDNqG0c^H@4l4hD~lpB=3+a6;eCTdG7s~S5j!ih+pmC zS!5lhC->MemLHnX>02GVOGiNxrV_hGw8)h`)->j9 z9W&wfo2M`yh0z}~;4k&XB%5$3GjE4zlXl!|RtKjW+b~9}jTZ;|n1N|2yp*mwCLz0u zpv`jHvnrE(iAtmIeo149v;y3Jt%wOt^6-6?H|~D!2InIEv1z{tJgTh5Fx3j+9<9ef z)hdYc*8*L24P-v)LhE)-s^Em= zRr?*9)OCb-vwd1n?GRDezMn4rdw}GBP=uWyl+cjvRhr~yp~S;T=33_(-kP>p4xY*4 z4axtd_v1g4>578b-uH{x>DysavMG#uTj0{WW-xX6adJiT6fNvMObR}oqH-!4xZ;%} zY&tj-4ySl-I{8ks9!Ib0Ia)FC(1J z44P$ANHl|5a9>a(%($}-(`uUF{019vXWwztH;%CTrY!~-RG@`pDfmRzV5@aGyjYlk zvFfYg_Kj5ZQ;C7jovfzgK{m)&m7!32K8$J0;LC6^NQ;$3&FJaS6fhgolw|S5W@WGr zmqQbtF4?KGj7FSEA|rKS^lNo17WOxSS#~SV`P2xxWF3i)-axOb4-h-UF1lq(m)TV7 zlbkruBIax`t7B1;0+enEKuckN zP>5WKxv7DmduR^){H=vxs|$M$Yhv2!VzT^0IlXkZoeVkG(>3|3sBSn1_-f~3n2;JY zrty(*{{y^=i?^BHyL)(I@uB3xiC`L4QbBeeh@g5JrZ_8*^)?!q;TP7Yny@kgZI*?= zhO-fPJc0F-Wg66e5y3T=mqG16PxNEosCWaG|B95rg{x&jeV+!VPiOr? zMp~TtnOVG@*D9IZ#5FwLB}Hftk;JC2O7Mo|k(60<=nJc}7P(i5_dljW*mGrgGD`^? z&KN+4lPa$9Heug;SK@to9(7i+CtXgBynowTnZ;Pgv)(bp%)IA@PBYx#Sb;z0J@N$o z?W}iCz!V2+EWl036vbNo;XsEk>NL}i_lLt2q!ez_u6F@YLo`T>9sT9=`nFsLOGD-N#c>tI23rbhV{#2ppeH} zFl-XWkQY;7J|lwKu~^aV>=OP+eJ@Yp`rSFI5(Jsnf6(bpV~mp%{Gvv2URqFQy0mJoJamX z5vL}T@0n{lKX{Xxg*YIZ2hl%^akqN`Xo}6l#+AyTpTdv4X%cWULkq3WszPOg4z}J> z1##)sc+!xatrn%@A>Fm``so=Wa{dBMGCNG>T)#xu{K&~_w<1Gd-B!!6Fu_c6M0G{>HU-A zOvtwB^z_Iz=2y5V-i`iC7FK_uXP30`X#acDu?^;iH-*qD0*BkK0{#tmrq!}0Z*TJCneEgtd2$JKhr;xJy zG_Lkjz40w%@cS96>baL}c-D&dbs9m)p#x8!ZGuR)=e#aC9kX9azzh_}4Ssv*tMW}m zXYw+gKV^Uf3`fFq(@<3YF9J+8Ls3vH3A37Gz+)f_b1$xi;5D-#f4=}q^YKA^A|LD7 zJj@%_vL-!#RlNJ^(qx^Z61-*iYm~kI$CsiBC+acr;ILmLop9>;S4Rme}d)2+O1#&@IFY#zq}*RLBOn z7X~@svv>1!RK1v+CEIz``DrYsmxP#B1P$5gsPdd2*Y`{T^|)#HHEjxjXB$t5JIFZ| z$;Wt(^ba4{JH~zWTArxgy2QI( zQyY95X2kAgu|&(Bg)shQH+R}TK3@7eLnf(7kyrjUjs7|vOuDwz(2u<<$SIdcWVQc6 z%DZrd=(nGs?;`8T{a)7RcH(47j4~j!Ha0) z3uOm}64~#x2=h0j!sVKLVrX%YE;?U93=)RvQf(htV9)M7 z=nsQz9d9((t|$AF8mZ=^7@{gtOC#Tp(R!J2V$gGi9$I>nO!upSl{YK#i$^_-G*_V> zvl=Z&{DCtg7H6=2{hnDBa3-r1TS}@yxTFjhF|n}Pg8dmz23X9F!Th47DDSrfcq`rU zI@cL)#?|Bdua%&w--1>t)o`dh70WB)AmL#qJM*lCKPD}(MWhMqgIeKpa}#Pq1nx9h z0gGK&oO=2ynBiswz3oQ$GoAJR3LBt$t`8jRbj6)+0pN7S0~OO2k`g^j`dZzLa06{< z=ZjkW{iOl#-Ret z`&3Aj#_DP2^(t~hp^}dM7$C8l8T5JKcJk+9IgMIZ2J+8YPxyjb2>x7x$39d;^SKK2 zWbqnn&nna}afNR@Hw?=6hlc|0_%LP)h@8AeLvD-0U*0(FPh>fv8w8_VbRlnv9_nA5 zjl+hjAbMstnr)f|@0Ql#xBr^qNpJ^#JJAXOPt|~HHy4BVX~44Gb1_mVAH?OdFu|Y% z)Mn-2c8(-n6Z4Z}aXg*(a?pusx2T~5Co0Gs&KOml>?Fbyc5q*x;4A@ima{U$MgCPR z=3R=>k1N4dqa61=YdFXs`hw;Ehf?)a@51ogjjf}(r}pzB!;aOnbzk? zyvAiZJw*xCwo1bKA{CtBDg)b_*}YlwG+>R77QVlr0q0pw-t0y%%*+Xerv2WyYCHjr z>tmqHJr(a{#DUG7NxJdYdy@G58*K=CO{@~y=$L#XX`AS#k^Zd&56bXX9ZY66I)61k zoN2}A2%8V58*k=@Dpzu!xy5shcwRW#hRFo3>Tr z;FD7Lr&^2KBFbU%fjS9Vu#nz8AxCai>CZ7xx0`F;`65Vhsr+tXE6I5q^xab7k`ac#Znl&ozb5(nc8X7m8)u z{2}@1Dm3a3h7*qGX+zmn@=f^&?bo_U_7A7AKI=qSv>*$~y%cz5)5r4?9$^HAp72Ds z4m0zEXTq@#Ih<9c2=m9~u#;a1Y}y1c1gC=L0#=hVe;w;#X~aEz9U$f0j4xG8AmFnh z3bu1#s>6IVRLf>o&9CNZkJvINB=UIk|62t$vEkT|9S6IMBk_O+WpemCd6oZ8F|W4l z;5Df>X-_%-4-*USrM~F6rkUveoBCy{hDfJKp2HXn)N(Uw^?fjQYe3dFw-R z-8}TwGlq4E+BkmOpI+;XC1XqwmCs#GGPE+GA|M&F!*W3{BNg|2PX;&57(6j3!{e zS|*6CPD0L6KA2d~hpI7rU>U}TC;1658XT0o3}8FV!70YGp_-llRNZF5t~@2&@ir3d zr-k4a(`fiFJ`4|*b|Btc2PZr?pubEzZ26y|^YE+rjpBHNlr*$7&{9e$O40p3=ZYp- zAsHbdGD`NZrJ+H4Pc2HjMDzQct6^j&BqTzlNK{5czxyBD*X#S-^PKbfe9rS8N~n39 z7`N=$^NJcH%v$ETac|w&iS3*ZqDSgxC+_5ox~yH+78g{L$C zB~O3U>cj-sxrPREI^aT`^7ZqaubDIIY$9bi%I>r4&fJjabPw|Vy*<*LZ)1{mCr7Siz z>OwEtaJ==`*TLOYod0qY>hi>5IO6wG>e!@Mj$(v|Y3;g0wV4}^n|{5yzjl(Yg70gmnY!_UJ7D$gJI-0w zNmlO~#jeIcSenGbuD0W_&#(~I?yyG9{!fInDw<7k66Sss8RavBUNGpn5*aT)mZ|+j zx+@7f7#87`{mR(Ap@MYvyJKOfFdGvWKqFoGe6@N*@;qWLo3e)sAhU;krXb7J+xrlr z=d8u23i{;7rn}HTcAMN9Zld*3F8ER93gjBA;w-Haj19ek=aL2(hw*Z7+aXRKtWl;< zV&mY+pS9>^Zp5q{o1jalrlJ0cCey+$6PhhML}C=nq1JgBW<0wLHAk9oSo92?WhVd& z%U)vcnG)>yy^P~=cPU%9dK^|AOMu1S)yZD-PAdKSB54s8=UwXhLQOJcU@UF`E-ea! zFB<;%x4QsedS%m|`K@$+Y6qJQ^aUUn#j~ zX36_>*pmp~tVDO+t*m;?CXyePMLxKlpzof&rgF0qFlR1cN3=E6dmY6D-9l3K>L#)I z=!da0#Q3c7>j+Z?k+U?3+?hHF!w(-Ml^^@i*ag#cBvhi-a9_8u7?z z6RjJtg|+F?%+7&Y!aF%m7dGV3%+e)z-b4bWM|0R0tM|a}Ye`^GtVee7TG3^XEoT1L z3FRl2)^g`Wz_Lw(yp>xYLcO2?h))yYjqVDC-V=e~7T`&3MT#&a?h)Dlb(}eE9D?=B z>&fNlH29>%qw8Po#PFnNq`UbH9%IhnJFhtS`0)rVOuPo~y{BYn ziypX8CY9NkqK@|lmXXK^IX=JsGApihoEv#D6WLdS_)fTyzKzzR{+}M8*sy)ws(H&{ zivK8SGS$cXS3i&)oeQw>tvC2hlZBVMZm`p-5nqa(0{i~o^ukCQ&9aEar2onp-DX9U zdvFfl{?)|~Ka|kyv<2>)tAnGGhjFit4VE@LqT8%3AS>Fz@Rua8P2@QnamF74oOP*O zfg}VOe}nSiXuQ6B6$Vffjr3rq0n0wVz7+oZHAH{3+Q9TeopfYdZ? z5Mj?Tt}aU4)k-{!y1Eb_tvZE0YX#v}jwUR6v5{u$P6m@V^U0jg%ZaW0I9TmJN|)t} zp}E9f%#t~aF#*vS>1>I6PI;1_Vo!*$$UkDbtq{Hmnn77Y8P3X7MMaTZGG|i|R{s|W zS#K)QXR$Qq&SvPNrGN3`T1i4H$La74DX1FA1(6wncyA45IXT-QZ1*5x4s2s<;sh`- zR}U;M`J%AJi#pi=2DW&KvE{QkF!EiEoQU0k;)Yk~(ZD7Yed+)v8Tufb5{n>km)z1$ z$A*MlJYDsmPT(;YPRueTKF&|kU42YWE)eIfkW{j-oAoQG$o=GX0iA&9m1!o1lY4a7}A4>yMB!TVr7XYk1pymfmj&)xSdoEyFf zw6qGPSC!zw`M>DdPhY5(&Q#=BZih{3Qd|p}=Vb0AFXFAW9ZNBn)W z$$UoI+E$oovWFY?mqFut2pYe8MBef_Sr2!eL2nt5P*8Q6>EaV7y4jFOX z45R42o_KiK6b2Fecf(!I{=j^uhA!)UMdHL4L)$$kjPyB26^lc0&{+fJPEAGCm0I9i zr;hJ;9j`N8TZ4*Wn_=a)6z0hJJod%mdOFxOjuL_+)CR@4CrKn*5LS;C_5N58T8+ZC zT~zpK1(BaVo%^~y21IUY@os)J;AU)nkApvo`5dNFoa-wKL2nf>J&A9=)vpD+*M&Hz|2i~2kf3%% zi4FxApvwKI*a^MSLUU>dy2D*ZDITT z4X|1EH}hz@0O*@k;KGTAbX`^&iIVXqCJo(0uzwT1Zp-K4&WWW4egk-PnmQi(R0iT# zc+`9K0s3`-hvi-WsKB$S#6>EYz8Z@FpAV`Kl3+#;9hxAz#iy8*HV>F%@sONO`b9E! z{w8mI&Ql>q7*qZ|hu>k-dBFl^Fi+tI8RI{!u{)E`!ce;a@skbUgJcL6lq|;|2lyB7 zDp6Qv9glW{x#U~zADpjdj9+ivhhm>1YH0kN_N6u8oa>dW@pBmn3-*Rdb!$jzyc|eH+8(P65QvIz6xuLlBeI|R)iif)`T1jpA zGVoM4MF-QF7;Oxgp17LcwsM1oAz@6&r%Sj-!-hP}Ok?EAE`Vjx8R$-#Nfg_4p)JV` zWOEB3EAJe0N8&H}w9^`1uTUo5S1gE^O+V%By+*VqDbR(A6HqmCKl*;!1dpS}*k3(g z@$1pEIN0fnI$mMa_GCNT{@o1bzk5PDlLGLKx)yAnX-0N~3Eas^hVGd!$oKwI%vf-N zmW@pzndNISzG5L3lm*ZiK5k$+lVv;IYuQEb`cQ6_H;kDt$Jx_`;J+O`sHNZuGZriJ zzAGfd)m>Mi`U?le{$0ltdzBdS>M^`AiU%b=zxct+CHO-1JDQfPfhs>Pe2Z|wB;ReM z|4|aN>pfxR#f5kem-?a0&H}R7ClHwr|AE?%Q8==qv2L~N1;|O)Arl)eF!J5u;QAmB ze52Z_oJBeQYyE&5IJ#WlK5?E$U_AWsG@u)P{bX;phB8yR%h?F`PF#Mp9)r}^<2F$a zDePK@uJ_K+=mHTm6DXxs%`+i+au@#LNnr8RA7t)-#ppAnL(8A&ARC~@zeDx$s|3Nn zb5-dxK{IC8VS>ts{!loum;F%mkT@->1(}dXbbqfiw$F&f^Q%@tTcjT9H?+}jwyPmz z@*ter5>BQIr;^voy)Puv%J)Tf#&^?_3eIt$Bhg(r%;t9eLjO2jbvzv=&MRR4^=6 zjEiB}AarU5{`SabMsrpZ*L_)}SRfEIN1icZCavI5nnTZ?oQDFR&%u*3k6`HYIGsOh z9!)P9BgHAzU?Nyd4jpWR4uevVi;1Il*X)p?!eozNA!hlPL(hqD_-W`18-t|i*|{RH z&igd;y-bu_oS%VA7whc_}oB=>lkT7QAq<9c3MHQB|`n46?m}r z0REmChrz%0;K4a%R4p(W-yEvNuqrEj_9=$;ss^LSK5tIl(W6xMj4+QkAPy}fBUs-T z0=yVoP&Ar?@xqs={7w<>{QnlCQ?V$n%Y1_}L4U|60Ymh>bdLHTw&Ol{D+(uG^0_sQ zEVI(o1|Jsq(+{V#aQCMg_HGLYPhHi6<7pM}BRd7IO;Lf5MoIWPSAaXXGXy(dlrYVQ zS#n>)2vU~hbIJ#mcnu$$sd(8%C3U3W2JSATAJ^4h1JCanu(MLyLD1 zu2T*~?+J#O_t@g{5iJn93b1n6gpH*UjV-K^E3h`O@p%{N+A+)aEh9;43=!=qCvKf8J94@kvo?Ah=P?2_o z><0&xT`(b;!7SO^Ljy}hasJ%^`mksLQT5A1$Bdh}aQG70n(dCp3U+8|Aq_+J*J!t5 zCQ0PTk}nw_soR%Lw3R>WTxl+VpIiOV>omjXMLvZCId{lF!xdzr=3uPJBoNoXMmVuE zSpU9Ru;AAijd3fYC;QdNm&y!aR*SF%^=52WN-6Y;?D*!D}hJPyv{gJf8*&I{31F2t{F8o#^A`bl#aU%#?`48x7a!FRM7RdEEtEHP`}u z@_crTIblv6m*+O5oj~c+qO|1m3OFfs93GHGC~iNe&O9d(ju<8r7v%!@o3<0Xl(vDu zWpgZg7Yp9e$~?chXQ4}34T@(k!^6LSQm4)D@QwL??DjRJlMieMjjuH%$ZUf4-*4ly zfj85{iz!5XS^-=VWPudV!yZpT?kNQ=pkDrPQ85Ru+D*{0;jc8^`Yhq3_L5*ei)vlN zZ9Hro%&dEDM2$XBkUFrHHZJ*2>;|qwhtqTnKI02lj8B8>)jYc7Of;@|9ZmL$O~q^C z4~g>mD%vm7g~c0=V}i*a`kv1&s0=s;yIzzLrMVQp>eNuHxb?)-XCX8?9mbe-gw-js zr@c3o*_zM3WV=N=)j8sVH6(%DyktSElNu_KYZHA=+>kI5L+Z3xd-Kc%*NoFU!C2cK?mX0m*z z!zM6gMi(}N*!K#QO-y4d4ohSHZll_D$M`(KwL!StaR53x-hukmR2sY^8xQM7QMhX3e0?e7gMw!k#BCn;3+A*B?q^; z88BNfsAlO*KF|2_Y+7Och!$+hB$YiX=y}-o37zqZde5W{FWPJuRX$5>2PR z7RSwU>%hRH4ZnZa#%=e)$?&K+j76k_ub~abtvHX?dTZdtwrh~VxkDsX%}l?q4=1-R zX5owxem#8sgVNyXFioVD4s3Zw-N-1B5s)Xnq=tk`TfqLRQ>5{PCTQu*!TIW@pmNrc zY*#Fzv$!Gn*yB72oO_<8re3Fh`TYCq_hWMAdo1j-QYDh<=RoG1F_fJt2f30H*kxgf zF3*JM+FU-5bAc6PsmQ_klg02=T$m@k0HI4&hAWaHip@7}Lr2vo5aOSg@pTI*EfVB4 zRBfdOb`y>*S3Ya-Nq_{-S+x|-Mje3v?B;>=#d6f|+e1Zb`HY`m z(qQw_hF#X^1dIJF@x0L_7>OuGuiNKv%&DJs?z;=`H^_272F`-Kj|xQN!7tk6JWjg> z-eGzfMeE#Ys1h+iFP!ABj>?;`s-mhP831V%36&E$+})u`{?5Coe#+O%|Qpq5^kMza?8Wj$p^T9VmM)j81I}qw7X* zW4W(46zh!8-hv5I%#j6GM=n0PsR>2XazS?KM%FocItl*}MW#iYGK1p2Ao^GY+VoO5 zOHWu6c2}l~>0cvgm_-*7O`F^JaXO3%Q;|H9|22G831zP=3_?q_5#q9JJH`|Zv35f>DAUkI+Y(aXW8oco_gn=?E^q+5 zl%86lX{~5+DH7gwbdf7t&%&LB&Sdm-EF6571|58cQH^#0dj5;3-I{R)F(tmrz8H4TTXN!{>#-!kr` zJQq;w{lp46*P~_fGy3k}ICG#-fTXtnfj|6t_uaexIX7H^tm091}KpKv2$M;fW^dz5av)^He zoX)Ezqt8E3hb2jD+`Dl+T4GJ#D+F~EV-0XTL&sqr@+tYKUmi`kGrYghlCu;CTb?XfZ;P&_mw5%>wSl) za=sL2g@rmb+Wr6zI7Pv^TqkM~aF)4u{0R|{c?Y%^van{7H@@sXj2{2yGYKE^P)s!p zb?y1=xWHS)GR={*>`M^XEERHCqk!#IX&|LwPIY$+K{qc4=AE8R(qp^GQ?ebp^Xllf z&y^5j8;)ZJyHIlEEouPab+&VVvhaoH%qD+ry0LhR_MPfAA+}EJ>;xJusQqe)l%V zy67Yc{2&QS4C@$+QG0w57sm8_NyHJWL1M6a5jnzEkiO4$aJg6$;e~C6(RMR-qRp3zn6#msGp;f>l{OFNEmSmU1*{}(sFn10vTR6;Wz1)Q~ zL!HrmNj6ICea)WPbOLhipTf39t`IG{5*(U2 zMC^44*crFd@cc7y_H7oOW%P$W*zgR!Z}#9(9YL@>-i@<4`fRyEAZd150BU|)KpfYi z>5Ez%d}vsEch-9FEHtKCk92sKi7qHvJfXJoDsW_bFw9JfBeV1F!7089HtVc2?h9MO zZS!m(&Uh=zleLbI9}E8#)Q!cy02Ime<)tUt1X>mlPU*zk8iPq zVRlWP>%ik{^*c9~02^})#}CBf&!=5*hloNO~C!qTeNc3 zf*EH{GNB`pxGu7cG(Jed`(2f6(+UT;vGEsn`&}YxJU`GattB5Xe?YU<@^C{f6MX^~ zlW+AISa0SC=a)O7$xktS^uipcesw~Fxg&)0bcoX9JYWj#vE8wk+<$SAPE5T5AEQJ- zBu@!9oD0M4howl;$O!RSE(=d;uaWvPdB{C{AAZD{;<2g)_|rcKF&e-zkk7L;x1j4x zZ6Q3p1fP^zlFuFT%=DE*WZF{&%84ATt9)LMYI=6KakQO^=7@7Aja$R@+ky~lr33}i zH(+GwFO|E|j9u!+xc$^(u-1Hnqe13i-6IWYkHomF&jvVpOP<`&m=0~X>LImT1r~%U zLPPmQ+%q5uQbT39=(`raOLYcA8+GpDaR=PfvjIX4(jnwqG@1+FAflg@@R5i*ToZ{V z${Tgb@`51n-G05!wn!HK^b4VxZyX4``U>{;Hz=>pjPefZ;91R|u%{povXl8dVj~Z- z`(`BG(+bA=Jwf=drHrP_g+hnXBeM55-~Tx^L?pO{%=z*hYPndDYyPkr_z!sC;fEra za5srl*!+$R^yJVLRY{b_7Qj6F{b0}ZQ<-Vzd{%BRNp32|D%IcgOvMaro)-mWA*%4T zWCNU$I{;=gipcn@6&Tas2bVum40^ki6sKLq=E*8h9dm`wZR(`PU*oBqoeW5y%!8C$ zSD|!O5FUQU|5kc=s5)3n5f7+kbfLs|!T`$DJwBy75GfzGQ|G68MY=w5Xi*9ACmwu#LI zE0Hdau#P?wH*w-_u>egP4%4ik_vq1o+tJ3Q2Dc5|Wb(AElexe!`BH%uzT||IP>}!#2R^^jMa8Xd@Ms}pP32E zRCP&Xni)FGJHbqx-VVuG6~xrn8kc&#pc*k>tgwk!cJ`jhwPuk`&U0cxPAyaQtYv$NefTR65yV{uMKcr8-3aqq5EYmI*mp6*yTcE zQu^3~X#rp@9}YXu8j(ORF>-wK9~!bu0U8SYp!#G!O_8W0hC!hi8<+&5EmLTuT`ca9 zjR!X88tI8!!}!Yz@k~u5c=~>SQ2qN|7`k^4tDGj`SkP5mt}~BMQ_#e%>*8sHoFDSe zr(>qaX?j#|9b=#Q9s2(4!24;2@a}#vJ)X)V8o|jVQuPwUe>(yW-ro&(3?GwE{hCCn zJe+b{-ZNH%pCJ55H4gE;gC~W@(eueAQrqziQXP)Mn_W80dWUCVp%p_XwPrw4ygK}~ zR)9IlJ7MyzIke3$k#5eqj5mh^$rLFSR5X1}oWD)xc78O(krVaQbz?QWz7Yl0QFdUk z{W`r{Bm>FwZnCyq1DtUw2-M$DBJCWG3IqGtvg7fL%d{(~m^KTGvZ4CQqOkK+Ki*91qubo~GDplm(GN-s z$fTctG047|O<5L$6~WdRQ*4a0B!qdLI}XCbcSk|5#D#4eW@(&fCmPM@qDOA$5u+8W zv2}VX)i~u)=Y0wpV+BQa@zQGeVYm<8%#|g(jlxLlj49x0xediXG?IqY-KhG(1~)o- zL)FSPSh2bp3*uWrLE8_c09!&n6D^GetV#Mm8prnoJK6*2 zd(kj@>%J3Ax~q)NISFiHf*`ki{#!n`HiX}Q+CjAQOrFxQZy0z|4^CS+6Yb!3I&H5w zE_mHd%XU}boY)H7uBwB3-|U1xO)79U=Ok>IT7_SvZ=qUa5)rXXBO}#ez~2*uGL^4{ zp4f_iOm>k+Qm3k)BT1c47X*u|xz=ux5uQmz*O2#`Tyrw-z2X?84Jq z_1Re4^|UNq8x}D!?3dwbbo2TcwAy!z6oy}b0&b7%Q6rHYXB6qd&RBrI&EYsS{AXlh00;)xmaYF}%C^CGP1sz;Z9E!zb_ipz&D+ z2k2cWZf+-zxwA=^B};nKl|m+VHZ2N%#WIEvh?eNSblykb_W`Cy~oBeLltkguCl zpufA8jp>=nU4Q>PoVOLD?H2;Eyr7ZREq{r%*7wMyj6{gB`~#vlGr>_y0*`Fc<0cj5 zg0H_coVc_c60ViNG3C6vwf!^szilDyUHTALn@WM$FG{oJ=QGj#8h`olf1qu10MbYd z%vk6_)~F%;HYp_Vv6-$8w8m4vBf)Q(4g394Ih9+nldQN<3G>YbF{7#p*Bm=a9lK_r zGS?Fe?Q6jPqb}+ih~VebcOieGo62b@(FNyQz(Tc}wz<5AZbd<+IPE=o6&sDJN8;%e zS#{pxl4)Ee?=loyt_qIM23!fBF!H_hOWjY8Luma@ljrPU0xPyV!v3v4Y5m!jy0W5bCD0UOdN6u|TjxX?b;D&})|2OZsV20wc`Z1H1iFal^GnC<*YRV>AjG8 zx)7!%n31%c9$a~H6+Bv=RY&B%kcUYZ(S*;q4zDPu{Zf)>9^}utb>t$pS6gG5%4_t?X@ho}fX)Jvm^$`9DhPpZ*lN+Pa56;G^&-@~=mYqUlF97uSUK+J&}#;$)7R2Ppj z-=<#%pCfjZ_wgbGG3%I#!)tJFh$#87Uxl3n{pjKvfyecn@a38TvUIN^92<0o7kppV zK5`b9bt+*izezx%zAL8g@Z$G#lH7$GU7+;hR@(M0iAvS(!b`dgWFO5!>%9+YujWDe zbVClDSv3#*&bAU)zHUy1C7`oChpqRSgKv#TP_;G%%w$f|%XZ7q%wQ0=u#%v)-3@nX ztbn-pSBQ{hB(`l>40?Kn{Ie;*)9?9CGgdF*b=ahlm-hEat->k#zON$r`?_$5 za|2~Ib#SKrH%z4Xc}mWq7|^sIB)Y=`bkROLNF8d&OJTD~v3D(Xzbs3AGd{BI-458L z8cq|`S@!SxBWROo2xF*XyFH9Rr!-lKAys4{86lvvK*la z*8UjPIu!;z1E|TfO}Kw!9skTUp<>T6RQNKJzMozI8M7bL(!+1T#Fzhe@Q!%e_Fofh znc)cjPo~1DZ8u2m?P3hC)x|*72w3C(gf`6_XT7zjf%VI8sP^j$`hK4UMbov&udj)y zF5Lu+HgASSb%U%(=6vv6auFw*e{kyD7SKL15j=T|-^nulN6{x?-q=D+`B_7~8tSB(2|Yce@}YdX&=_#v@qyGzDG zZqTPLNpSzud;X4I3%P2kiu%_(S-&sdumOA--_`1_ zhKN$)WH=^a44>bKL*980I#<&G-rsJ-rk$p+Nj4EaR(v8Iyhf(^0W64JW?+A>Jok;p)&47RP15FenN~k6i>imznVRm=P)OGX^sYPhe-S0s1zT zvW@p4Eq9n6%Uh4GIY_E!Y@zZ~tnp5W0UI-X9GG}*ZgSNjINRuidL>cVFmi)a=o5xd z7N&yQg9-Aytgs>#=+_5vZ%PBGBhp}2LIl*III*6Tk6h}WReZ* zf49Tq3T=2Wb2@ovaus{RX5n}XKU`RzHz-ZZPkv1`|`(=3>7>d@o&T*2hb zyl5(&)`WsS_n=E*F075=pkvKv+Nv>|+ngFmRy*y(@r%<;^G~`HxuO-^>r1U^!HOuF z=@<#s14i)oj}fydJeTD6e6^c}IqAYw?YK2;bdcd~ za{RyutQez#<;%#7t$`3zbcYygZeqn&hT#;)adI+M0-d|7z@*n4g)ax7%e8F!Z~a#Y z*_KS2u1%(uXM%`~pAt9hj{$9u&_?OYV_+fEPL@U5v;PWZF!r$?^i=PJ6UXk*EC&hl zqj)Rqbk~Kh=l|H3ll;M9?-23b7X{1tIh1pHd~H6O$9y9fQ7tP4o*C!j=W1u%T9!o} zGnc@K^jj)cV~Zk+N%Uaf8obQkGnlb?9z2cw$R7OGLaz&)!yh{t`|8`U zTkHf`^50H8AX`NK`bQ9%<{)Cv4l(YgqD;8PW++?aN~34LBKlvhl3@3R#J)=evhEl2 z?;df}q<4aNXzG5lE=rLc`=(7fvz)kwA1@R2_6&OQ-(SpXe2mxbU8W^c?QpZ}JrSth z56{$uvE$B6(wIDjToiO=R|dq==^JaP15-=nYyk(ZNWv-O>AbbSc3{XoU(yiB&jXos zQQHTS-1RydSn=;Mjb}q}r&BN4q<<6p4)f5DKgX+hU4>7M7r^47Bd|;5$T_~hKlH4V z)qd_tTLxax1J})Q^6fihtHVHTmbw@`+;a_XB*wzakZ`=?w3ca?c#2bMhM{fsOqx4& zGC2R9&)bqa57szrhlis|c%iY3YEB=ataCB_S005Yb)@l5;vQUJn-1j0m40 zW77LWY{SFwXk9vqEdK&`L<+I4RF3EC_!KWp4k7*Xy4i(q?$Kj?b>R55n30!Ug5r9W zFzv}L+WR&EYNh#^n%Y@}Ga^GpwC=NCv&tz7O(OYmJ9$;|#fId{`<3Sueh_w|7RMO2RF2e`N#&g!B7s!}AbQJ)7BBBFKHr$$>|G%2@X682-!+ zgj-`%(Y*I1bSDq8Qu3R?%BdfF_e2rpniF{DK>-nVFGj&C8zSG64{tvH!MrCSq<4uH z*N>mwH~J%k!xz%=@Dvqhp-&=-4b#KJoBd(*`#acPDZmZ;84Gits=*R#S**z|pkF)o zq22cp^6g+3`S0=uR1#Ez>wWc%o$`4yMTmc=n#rQ3X%M4iy8Cu{@y!Nv-PSlxR4VwqL>dNnUX4zs)cja14j%RY0p|967Z?4@BDRY4hn`>NIQ% zy>BLCpTMQs5(8NhF{7Anic5hH71znO&A*uTuhJNQc|PP^SORBm6HwRON|Lu;#q8FN z=&tS$U%eyXlL_L-(;?K>vlbuUh`<4-HBen93(u`jVd5Vi&I%SHB0&Y+t zB?3Zv3h?D@IsNV$0~hZp@wBICVA#c#_?4%N8=A`U%#S?zsnN@3(-$ z7B%ip$9&wqgWs1Khg0>`TSPxHn+i;Pr?mx(a7)5&%)Bs(TQAB%FcLwM8v=e|PvGi^ z22P52$eiWRTavQcVAV63s>rP6^k_K4qS*q>gH^k5Sobke^?H=7^B;tuIuC5hIYve#){?&eev+ey z4Drg-*C=_{l300p!n3eoaI0NO1OMaaQ%&EX$m1ar8`z0UZse0CHg8zs+rR|m9>Ipc zcaRJxfYU}-P&jmmc*Pi#Pd_8ckXHdLX5BCa-jK_-a^TOexi?g+#mTnyGy(`4k4)X`z=X&tuJYzEM&jg9`X(To>nQ1HJ5|v;lay!hyDKCg$qtv+Z z_Ac~&2Y)B;&03tbJ&_D!FwXxd#VdR>lY3jvl+l+-K^h!D3TG$Kg$B|%duKeDNsN;W zfz6aV=@$)HIfI-lb!6Mh@8Pzz3^WL>p_Wz@J#^xL2uZUK6In8_V-XI0ForE-1<<-c z8VWRaL2iW)tm}!vmLH=eeBNEME9eJV<=cYYV}7uI;TSGS|4D8)EeF+^LFiZegleo1 z;F-J3M<+`!Dtz-eW`3Fs!lwg3Jm4SWqvC>(t~P<6RR>+q*^WD9#c{6kRla`q1Tp3c z_@vHdj?2s9;J%N{$UjlC|42Hmkraab;Ey=_-7u=?ZzbR22XTF-Ht)H&806I{f{)iU zGJU-)c)oao+rN&}-XgwVl3GWmSErIN*IL@*{Dvsk9$`Wj2GS0`r_^hkhVD<~@I$3I znxF7w-YQOG`qHYwMaU0M-A^EG0iw9;`T?9jI{-dt)e!fP^O$+-0&Fz;O||B~g1`-a z)R@0p_4Ckeq8F}%VOM3a;^`mu+jSxKQBeV$&p;wtlY1~F$K1bUar zU}8oC7HVw+_RdN?tiA|dX$iCQPL+|hB?jQ9Zb6n5XOm4|X7L{Jv$9t%wLxF91Qgg= zW2fdlreXFP>TE5`C3b=wjo}3B`*j{eMqiQFvMWgDk4v=odJr7GJDF?yuDZ^~aS~)b zHh@!RKf!TXE@nqx0i8puF@0hM+?}==Gi!sXCQRueP(LSbs9FbhZdf>018UK z$c@G<_>}YtD`FE-Fy|xWTq+?p*X9$?WpR+-{It&R;{)<|=qWbVg%hKxcZi4SCFspK zLA!+X;c!7Noo^dK)Q9qkqJj||`V)crCW5@InXydS4a8Y1v@w3gE2>z304qDLC%%ChKedSV{~Sjh+eFm1TEN!^D(I@zRH9aNk8aVEgG&p) zQ%yZ3PUb;Dc)r{lcJ~#-z$NE;$|o4lQdSbq+sUH=hIR_KlN;67#v?meSmn;&D_k z!VWuAgt#r=0p7-#a&HZ$kz*Sc!TrZa!1x|ZuG&n7jt^6KP7OQhFCInoxr91e@^J4Z zMMeEGbh#)Gay41hZ*B?W?-vS{u4gcEcoMf&wuiCEz5;Hp$uy8xNu=uRq35|Y7T-Jr zueRK!${r0EsUHDe+TytXLIK3EnzU@?b#O3O;${7ABniLX(!a*yV7X}~EHEfxY!;^c06 z_(*hvb`+O*&##HX>b~hmKA`xPbB=FCQm0NAt^ro z5x@McgM*ooj9W?z^SxCK!eh@8*$91hTghBnyzDnQekX%n+4+HW-k56oqhbj;A=Cp8 zzFx)S`|mUP8iF9QK#=Z9q7_hTid+&Il@{lU$Vsa=OD8(=PAAnlC93Lr;_I`Nl9?&Ys*$9-}uM z!@yK;4{CBlfYUe$rfDkjBg|W%aH50Ch-JaF(EwakFa^Cf=|b7u82$sc8r1H6DpC4W zN_+O5qOK=8@k4t9jodR0_fGhO+p&lAN?H+#c-;f?UKOEW^+dQSz5gl5aK98C6iLn-Z>B_ zyLW)D#5(?5v7A-@Cd8BG-z~0RpELdwfF8=*Ao7Jcp4%Tm0;QG^*Ovy=WMcup_VG}3 zZ8cFoa0o{htAn=37|T09jr7jyqs;y!_$|5ztJS?gdUzLWa`iv_&$ALIhSEsOB^`2i zeG<%eH$~sX0#cE6jhsrG4A-YNGrD_cgMQn2^2RuUiYgq&UvryK{&W>+E>+{65%EEP zi@7i@$O2x}HiOCGN!)(9C-^@@=N(Vw8-{UvX0J+CW=5oN-urQ0m68TZiL`fG+Dc1i zB%@!F)lf!cuk+rI^HQXwXw%+lNK0G%&Y%A9aeU7EywCGo_jO(0mY9d`FKaFeH)NLb za)qCS_BYSE+f=l(4)v0pNY{4WNH>H3fSqhoQXsoq@sYScFPRPTJT4wWz6!Vf9w~gA z`ize~{YpG5Mw@+hv9jwvxIe6p5l+&nX8QvN>)l7s<%g%W3CCO#2*WQPAVVtVt zS~MSL&rI4+{BMJBU3(mRY(y#R8!hjl5oRN9$f{w-$sSKYAy zVq)0sNn?9rRksubdyiS})njAIpyy-jR}LXNT&rx5hUM zJ7fmM2PQW1^FNN|-#js8m(=D61MODv7b4CJx6M`N$9}mgUht^iebl?oddH_?;gWz* z{@1~H{(E38`$TmG+x#ev-&j11?AJI7nV~m?^Ry1JQiZY}#VLi5XF+moH_ zvG6{A;l{IUaMHAT`~qAl*UNaCMOy(&IBY%DKOPZ58+V#e2W`mt7L_<)`n!FEYV%Uqo1W+S@{By8)U1W> zhOfJYg@$KX_nFna!^|4@lQW>9UXopQncU5f?i2DM^5?{!wU**8i$lCk*%E$rXcDja zXcAw@>aY*WeZqzwo5)(EwOgtU*EaTtFF^$HOHRjKYXj=S7C^F@5w&lit&_{ z9p=h^?Q#$r6^#}j?;R-){_iha5E3cuPOA~F_l&OZQx0`MD!-k*_^6hRNR;RwDMh@3 zx|hU*ufXqY92A-_T*v00Jc&s#)YX7}Ap7A{KEtxuOi z_9J(QJ#8e-E=dX!-*vt$9CzR{`%y`c6(;49$cys4eVr0-dQqC?{5J_TMtQNn)v>;F z+Yhk~TP_>{6X?tjIc#dkVOBKl7GFK;vhcf|jeC}AVLkIBNjUebBzv~fnqNQry-?j{ zGymj9HoK%|7GFH_HM?O%GXLP?QTTT4uQ+nr8aCw@!`@jizQIkiPxw|&TddtRga7B? zQ~xN)i8o)nQLHIrYHEFY2C-$7MO*~Wir*HzEV8w05e8C=XK2okxJnh+0_gyo#id{C>*H?cy z#j8zO&1d8+JvwGi37db#CI*r#eEAagxdShu??F0*wJd) zyz!n{;w}7NFv$HRbXbwdTRqEl|Mn!7{cr3zeyyI1@bR>Y`s5KEY>cJ?d;jhR{&I(t z``<)C{fxMC;;lhO4dqL3u=%12J~T0t{h)e+|J=WTZNHu&UgmgQe0%8=R_@jrzU0a= zal*a=_L8P@LudYZwm0Im`-R^xg)iEkOFYZ2;>k-Z*s7x&`JL-ih3`!i*=27}2|L=I zdC5*IKIz%OJ9L$^-NwyAqr^t>Om7qRR+tXE?0P)=ZfPd_!$wOiR~;(cD9VDdTfNw% z!6sq(;s~~LW|dI5P?B4lc$-fywP*j8#jx03%0AFM#b1q@DI9B*%CF0L%(l!p#dGXHwt0|DC{N)tjug1B9~sZ~eoGM- zj42Z@PP!@Viil&qzuEBaCclJ!+3WZhe1`Z~IpzQJwII$--`InPZV0=BFY<%8Rrtr@ z3aru{RX$Wni+@q_i#miS3Qe#2@OPIx@rL69_!Q+kti{@wLU!6#@ta*dY&v*CtWX=s z%EEnqiTDIRL=yC#7LvuY%?_;2+8g5aFGS?asbj*lEuO3l(!-hgP2G1w;*Vay?zWxC{}@)z2F(d}_ue4jFJ0B~NHp0hHcNcOXVu&m z|8ucmm0HHKNnTfZWw|5#JcC$%g;5T>R;sRk%h^Klqu^LpFiKyzeP)8t_Q6U1cT}19 zScVnvAAVF=ogy#E{m&HU&G;^!ISTpRT7|sorzrNxsbg&4;8ND_YOnZj`bZDe>ACFV zC^ZkS!~x-dTpRD4QpzUW-^d5f%;U$MHDpy>ip68J)WlXH9;}as5no-oM;NnQrUCWb z`G3MBUSw&>zx1*c9|=3m_ey$3RvIcctd0@0&U5|<3tc7%|4!hA@?IGd4~iU{ps@*Z z)74qg&J*t2XV2ojf8XQf?CgcIIc@x>`FF+fw=GzO<;7wHeKWCe{d3{9J#rE@LYg>P zk`ZftdYBEiZ)2SY=CX4OKk}v98psxj|08otIVv3M5&mquz;|D|BA)wdsIcwYcb2)C z%no}rhIi&0`M*cB#eTzg2vef0*w0~ z2U%yi6Ete`eRfvo7_oIf1J|D87fa?&>#|dXOQV+vO`0_t z?5C@-C9jM5TMNwi+w&IlD%FYXY|pF0h;`~btsl{_vT;m(r%r%-!?rQ(mHj6CvGOtO z&gaJN$C?-MJ_;vT$Fh8Jif^tk^t}t;!CDKS?Y9xTJ>ACdE&nJv7niUQbBj+?9L~-e z27*05Xk&DiAe4)q62T==}xCwZ#GvT+lRivQ|K^Xa5noS=2q{pI+NyvEx3ymQo2 zp_Zq&hqhKDD|$Pce5v%1CEA5-`# zJ7lWs0|jycE}t)ZlNK#Imc4Bp$Sz(n3E$C$IUYP|{E3 z#D{7<;a`us#~v${_;^-akZ7VC*_B!+#9Kpdi4U#S7f!iyMR@Cv8EWp})*2(;>reJDwQIj{9 zPew3bxoI>zu79F%$ijtU<5My0V@a>kKk*9o>Np?%c-J}h z`?qx7Mv{9jP>}a%PG7`Url_;7V{F)mv8n7I+3P}s4kunkf2h!Z%vd(gC7wNcSHRzR z&agityTz6d8pSrI{;d1B5$wDe3BP4+D*yK0bk^d>OreKj1^co24BxwN2T!P2=pCHR z=JdAnB^jx#oUEj)>1Yf;SSIPKQMe~AsJ|+_l9%KDx$3O&(jj~Hv%rau+f~DVTk^ww zTf}rpMsbf=X;vL?P`FR1^g)Vk+K|oa-)8DT&tK?c+{q4}ej%BI#tOAU6g;lH_>Y}E zbqjCxa|$06Qpfgb1@Jd)B6$Ou?LyVO@%$CXBkb`&mM@mK#ss^1o;RDs#|O~*)!oXx zUo0!MuHVd#zH^wjKdj!c>T{F0vbKf~P^qogIKG@18>q5-EkCdd^DB8JmuNn@U<~^u z(}Q>YRn2?f8o{^tn6R4`?yT>fcUZVNGF^P*{5GNQ-@n4A&Z+D@D`k&kVRzkgY%1LC zbd?$|xb}-*a6GGQdxo`NU%?0Zd=n0_Jynh`OhnU##Skh3I3~j6aIy}@K&j~@aAFhfUPX+^DRT%)5Qx-pGbVR zy`O|u-jeL{ns(vaJ2%|-`%kLBzn^12eo7H98kI!WcG&WJz0KLFk`Maj>~;6B&u!xT z{vf`i(O>K&=?8$W_xy#sQoLFr&t6+Ik)Ieil21FIz@E4A6_)5^NbZCVeEA*)ewwi( zuQ%JaK4PXize3THT_7!H7c&Rh&6*e3kt#>T4HmnE*-Nv;+Qt%((Z%KbrzsQpFAF%D z;4zAwp8G_6BCLaDwG#NUq3?vhf;somZ~o%wUBTozc?H?$CeqUz8Zm0yJhCHq0%;Dr zkI4_@X=wBiqVH`<{S0oPnPm-FC|?m=o#Dj%%TXYEKT~cv??P9%$rJk(%ZP_VHfG2M zkYjyDggHNyOP6ycN|9rz*5Z%YX5&xx?Ho#~9(z;Wyhb#QG9}v<2a!EpPIRuRjHFXF zoV;^!#tnrp&_cZs|1B@ZORW=W|BPXzG3hd%|L8^?hbohfLt|i*mo=T0+73_mJVLA0 z?xcO%UeUA-JFv9FoBS?ZLe;-((&S5FMDDmIxf^Cn-`FK^$?bo@G1!=ET$Ihk>=2OS z%4%dt)N;(!9Z#mje}iWeQn-V2CrTJta>&%yj{gS-*)6U0H{w(mWV zn~iWba|~RSoTvt zgkZX=LWxW(n1VUYTWK0+ftRw)sDG;_J?*`WyS77`{(k34B8^$1R%1*mZJ*uws$u?t3&9lJlo(?95Oeb1_?&My=Amm$*=$5bG%!b@Q7R((Q8L!YW6|p#PRe_%QsLAj}quR@297QKri}AxYtsbxkoqm zQ_}(^YOrS}nLJ6vnC@Ic5AFrBb{9k68yeGLuLDSz+ALg~lnM$WTT;LHA#B)YMJ}{| zfQsw}CU;2yIeSuvK7XuAw%fje9Lqzr*~NpR%rqjmXe#Nxx|JTeFqK;E6_5|gp4qx? zweVu2G5K74A0f9Lf8;Czmy>2(qq#19mG%nXye-DPF$J(aK%KbPACLsTvT$Bb1cpr7 z$k@C*0WP)2u(tZ1=*ZACHhabKccz4TY!Po>#;{8yGXv(>Y;))|NJz@{O+9pR12@Qzc zN^SBwwire{m8TV}y5NDl4W(V5F(_vV(Uso_m#$1e}m>}s~-ngwE+JCiTc*<81>n)_= zU)&&Pax%lg|iX_~Uv%BF?Pzydgw-mJ2 z*wF>o7NMfZj#Qtyf(G#cRE3#j<7`>!Qa6tuP>84<_Kr?u&Tp+&MVZq>G2| z{}dfnG$lXR&jC-6Bd#}Y;`FRSSf=cOKQ6mS7`~4nGF^czDjvWKT?`8D1{3dN>NF)~ zGqmJ?fj19VL&5Gtc<{v}axXx_6%@{(wT=V0v{XaFr@w}%bUi+^F4QPZi+pgu{(^Ps_b#fr}OaN z`ey9>EXDOZZ^xOgg?Q!Z6Un>3S@0-#CJDWK5C#@@aj0%ghXdI3WjY1!d9{iLe=?O2&MS~Uf80B{d{C|hQ zTO%Yd(cPcf_ORLC0pSY}k< zPB*EJU!YvS5~sRd1i7_axaBtQVC$Adj5;2U^H$ztCZv92W-6M}Z-JQ*_AUYpuL(gg zrI>qj&!w*D;c;v`6p7&$zA!f92+Ay0hJl-^G~=lm4X@e5+zcz=Hrq-Z2L`9HX}^T^ zSfVHDP12`I83DJ9BB3%^6Z*rYs7=dI zI<otG6p}UnJaGAuDd_)I0}Z^_z?%^V@n7m@rsK>F5G*i* zInPp%r0ig#8lvzqUIF1&L!vVM8YgsI25V!o1YW-^NZGQ>f`cDa$b+g$I7HI*{8l~? zGA#C>O>8DCUnNUUImE!DaA)YY2MfTq!+{s1y%<=eW zeCX(a@3!mF8(;EJS&cINCJJlcFWK<5=08Zi{17+fQA~+{!L2tw z#m$+v0j7Q^0DbQa0JFQy!6h2x@5=>vV5Wq}Ti%WlKX1a^ecoWCNVx4C6fKPJbCtUL znKZ3Iu1DG(En~()mRB|?EsW(Bvx}iBwG!BE#i%ju0k&*8%>9i$i3a^$FmuIq=vgO* zFTGFTb>1?38qa#nta2jkk zG%rfS4co`yp6*Cc8r=%(6LpD?e+jeA$cdDfe`ZEG$umYfb;+=Rv2Gb+gBliDfdBOL>UX1xXBN)dh~@zoPps2f9afJjlH;B(GjDc=W0x z^HuE*v-_(pEhsq&Rxv&BdyXAYVLHSfZN*>1-I4rKq?Nn>;=4|stE*pvUl+|qCl_lH zF|nK(d^m@R2r|SYB1O6`Es-frU+Sbi2%ZtQ~rod9oxQpT2BB z9jy@3H?M^$?L!DzXGW$t9B0($_>qhZT^uNErB+GS`X{Jc0-Wq1dN*I#oU=$20eb?1e0F2 zh@!%@$sWOD$vT{c?YYCqp|3ogc<)N=54^>WwhG)hFdjtBdSsYV3!1%5LxmUGa3!dSFmLBH(Q69ehYvkTlhQqtXtKIr6rjf7^cL_IA znx1g%#1y+w=6;(YtorLfr!XQ2Id?_EQC26@6F$S|$&0wcN`~%IEaS#49)ssr51|un z2i#O=zk~~;M`K0wG46Ot9T(+a&cHWcH;2#+3|M1Nhb@ux>F?Gf_mxfYxziMSr|212 zHoS#=3nQA}sY*iRIgClIz^lz?F>~Hed{pd+8g{lAzRZNk#g3-ioUX%%Fkj+B*SVRO z+=D|aB=5=Z@|t;qTn`XpO7lC#R0hA9tD;m(N(*z>uD8`)(A8_ro$H@}DY-X)DOP*Ej1yGM$a zj?HoxC|3RX2EFVYhY=x7Eb*w1VjC7=02kVhtgi+ef@p7X<`6r zt5YKbpJQOzvg>ecK@{pGO40i1M{%B34lE-vI9WCjhK0F9M8+?4yx73JA8$&sOw*t> zrVU+)Ett))V&>SMMEU$aRQwSJV$XW`eDNu2p16osC$cfG+X=_6xs6ThK7l7KN9k|F z=-I_hP}XrUY)?o=huf#%WpOF2+}OuN-X7!*thOb45@fK!bO>mccw^KGIg)&! z36&*Iz;&u_Bq?GzT{hR8IcTH6=sy^d&gr-w-Hz{gyGQQzzMJ()7Eugn?9k!bNdH z6t^bdhPe6tL(gM>QR!s}r>OP>n@|4*!<*so!p@q>`)7&K*J~M9V+$(xvz|M6{xwcd zz6o_%!)g9miiyRVf?cP_pmSpvT>2gco+>|?IcHp`g=sTaBHbZqm3aq0y`DhjT|>}c zSB7fq_u%;*Pw-1c6=$HC4~?GZVT)S{w!vAhM)MG-zfF$Lwm%BFH|wC4zl06Tn?*s} zZOK&f1`bvMBsexfKk4RjOz(j1R#zDKQH6hOHo*PG9VqAQ4~z5$p-fkvUR!q#xBDLe zvq%jZvvdH>Hl@Stq0tfs!w@q4iG*3b!I2zvSq0M!|HF@_HE{f;J*_j{1y)tVq41y` zB!vHj{4W=A;lA5$={*wdgISCJwq@Yhi|LS|Q4HJGUjUV1n>ZgLMVfAl@xJth+U{y+ z&PrE<%4D5^Codji*RY9bd%%b*Di}gyU%wPoETK$Jy9?QC`5kv3*TZKYRAAqlH>ewv zigT_=5v_qzu%8x%C!z$jOK}EW`E?Su8>$ec!4$4gT5=w?{}25-f5PjCJ79Ea8JK%z z2~s8)Qub~Uz8(@S3cu+nN`ET(u5~uSRJAguaa=gQZx~s}ocs*M!&T@^gA^J$rP1JnCM-@#Qd57u&9$0RU>|Ns)Eli8sNqXX<+9Mr6wa?$d84k zc(VQ_Ud!kL`^0JZ$u|dw2d(R*P=TSNP1~X^NaX3&XO`$&)5AOek@5fk@aFGs?WqvS$?J_X4 zzFM%EiDT|gF~ZE1pSd~21kM^UM=>acr$H`i}fj@MGnVR$?bqJ#{vu)7a>gZDsV*kKqFItOt{AdvzMn0APw z=jy4W(%b)_zFd!-s=qH;-z+xR-{X{>B#dW0pc|%5MH3f8=3ZkdCm(AAcdka@ z9&;mjkZB_LZ)gbwRbRx>FB(M_eNynTcRHjr&j1T+4K6oSp89_}%RSwtOP?8qc#`19cz;L(7{EN(fPis^juTR>ON1)}8Zt%{Y4FyWS(Wu1%zRk$P zsY}e5#*<|bm!Zs5F~>P&pL|ZWItybS&0``hXTz95TUcE`5r0_!a@!YfO^j-^VBMxK zD6%mkUf<&wIn~psk$DP#=XzqWegbT1YevU=2O)UdM**E=O=1r=;P;_ z)knOz+ilKV^V?jmJx`m=b#Wmo>56n!v?fk9JHkn4<-pt(OTqe=8je_%$@~;-gb=+r zoHwN#g$~zI9DTB`d@E(j^Sp4yO*`CnlVU~3Vc2Y{C|LMVhgAFBh4*8+phKjA--h%t zOVTKa{g1-}A1T^?`!JN}KLoYb$zmb0ldQ=XlA#|*k@!`ns4(#)Y|yL$dtHE@ z+X$-@)DUJKgXjg_-0nYX@!XsxxLDH<1tXP7pUoZka5)=KUcJTy^<088TMJ-3B*B<# zKIGiQeAu)m2Znsn#oy)<#`rKjrl9vAGq0kAx#sf^QfIu7JPSM*vY`^PvYbeG=K%q! z)T1`Jz1;4kR%DVsF*!G~aKnXnQ1f;)s&rVB7tSSkqSug?3w_8cpD*xD^#u2y)o3zx z&I#Q4^)sq}_{KFU2^ggdPf^zRCEQxoDk=TZXQq#N3)(^}Fl~@%7SsX$7z9eV?&qNP zp$Y`(UxxK}Yhb3;GVWDf3f`lauyl1VEDhA9Ove(A7axQCsOi|%r3?;7x`4#ph9mJ_ zxTV7c!)ABFIrFh1XNC35HF(anTT2M&NTQ@#d;&M|K^(qTyo;KJVr*W!8*An%3pV~z#q0Ybm?=uGkaXh<^hAhJ zuT~R3{1P#t8e>qYR9CP(&X4YI9>Tyz190#6rnUc~aN-(XbgjPxKZq1z@#%LkT@G=| z4k!F*yP4}W9YtO(9)~NdlhCn2ku*d#!Y-*fToEV4u{$au?sO4~PquK<@^_gkJ%9&s z`S@wsGH^U?K`OQFKz+tmGWqZ|ur9lVDM`J|t0_z9pYIm<^wlV+-PO-jw3<`@&AS=- zMfRjSI1yf&h0>kbBk71kO(?B&SwLItsL$2_?E89}+qH6%XvNVT8j{() zg;5$anO<8fPcNs2q2ZVs)OuwLpm@`d`HT5)JiF@)`UhNw8;*F(N&8tf@1p z$AlwMw8?%HZ5x<B z4ke*0c9ILx#>As|A1t&F#{Eyeb5oSoLSuv?2|2=G=!9+bjjJAYzhFSb(VC=>ji93@ z>(emFp42t-DQv#pjCsL&P++D`Za43OiYqIbUD~nOILjBtX9a;`=nN9kf*`wUE?US{ zaH1eNoN&Te?d!)vL%ZwNkm@eaG35;0KUjr6-a z!N)&@CW@coLD%2hf9iAbyQVo!et!>)hioM_(Y+8F<3Rr|pGNl&&Lmpv>!DlU2aMfP zA$S&L_u~n89HSv6~-lq zPQ*+jg9|Uf&hVWib>TR&_s4v^ea@a{w>Zx#;u4myT|H zf%D|%l3tCGPVb&{EroTLT7bw4RQ`viK*8!`FXF|=E5!u;Es z&IGN_kZ7&5@%STOvep{lu9Xyhbn_@0j*}&W#t&e2_GiHy!ymZHERH;rHY5w}XVE3< zBk9-&tLSFIY1omeOJwhU5GDA8ld*}iWLdoe-6E{U2e*u=^cHDYTcas3|DuTbYn-WQ z_Cc;7QJ-`)dy?+#RYX1aBOKhMg&X~|khq6Zfo={41$&VQ-Lsqr^Pyy3*iJnb@+;0)E|* z!hcgEQFhS?qPkR$`OuRJ0VTtT=)yE`zg&oFIXMvCbp!`){D)B^e`6M>jh}Dt78Uis z$DcWMc-h2=S!*o?rc0HG5AzargPwA(o>p}K7A4|6oWa*g&p4m;W3jbY4Xzd~s2ltE zHjbG4mFbvX2y>DPp*Et=<-fxw;QTEIpFdQjbI$~E^J^9Gxy)v$NOXd@{F{v4K6RL% zqfUp;m)swNmci!bmC!XeMDXkNaV)QY4P*B!iuUd7Lu0k&csF$>7(SRt7yU4#4Y?7x z;k-nXFpq%;qRAL!kP?BRb?<)bch*su3C`$pal+%bAx-W`|y?3VOV_L7YbwRaQKFzRIxpECQ`l!?9`S)>r##H=1D-H9D4np7cTd>+gmVT^capjI1 zcyXRPmd%l(JI|*=hNl`c+kX-BI*SU_JmN*U2X#n{lMVi7I*6}Kw*ozy3=OKtt^X*% zqNkCNxo`}G-&w=d@6y1j2g|uL9@}8Uc7%q(R!G0m3fDCQF+h|BstHBpSvKq{5Of zW+0(c5=J=>UcQ=6_S7xIK;eG;nec}z3o8eUAv@uNlMI>T^PC%B5eaUglyPVe1ns>u zp(OeWd|Q!%#bxH?^=}Ju*ZmfppEU$-%=0BZ?0nef{hHZ#=?Ule_f&1{CN1K(_9u>e zw_kAK@ilDg+gZ2XLYr)t8Ur^ZyuP<*g5Y0q4ER@_#FF1{!4v1bbHsH%y(ZT0RtybfiP^JU93~{wc_9dV(hpj-$ghUxDb4840$Wz)Z8Kz+CI4aBPko z-4}2MdMaJXIX@~o)?-c9s4f8RkpidPI;}QMg)*ua9zaZW0y>&y;5zOxTvktqd-Jj- z_a0g5UC{<6VqYaqao;t$FcY4HW<}b0ol=);Cy{0Jbo^@FU&WjG6tK_ zWwSdBEZxtQZ2pDa2PE@(*ZGk6Vi?uECQFu2uq1mn2SJB#44SDHF@ZxYa=Ib5gQUm&7@y`S%PP&!xH(rL)PmO3(!WNund7Y~&*o@O&Bb*8?5Op*S z0~P6XxUs62o4M^BN?PV?LJ9HYn(C*MWYI9R z?cRo_DogRwof`1)a-*_~Z!o9JCKLUpe9)@j0sn@{agNW2(BAe}Oz!$Fw2sKeNf#%B zuH7XF_I&`~M+~9$)u)i`P6KlFA@+kDMiN~Z9JvTQ=c>{jb7yi@6Rt3!Cdx#Aj~Dd% zkEG{acEB#nXA6NG=hKqdR>9HUvHhja#i)Y}k z%12n`^*~^qEe|7XXJbUh0qlyCMC-Q;VUd{}sUCk4_8fMg&ZR?e}FU zhppVY!bsH6R-|pM+UO^|B#3VN3+5xLn1`~Tpy-sNzE zd?P5W(W0R?%i!LI9>!L=lS?jCk!U)FIP!cTJ`0s4?V*v7Cz+xBwm-qx{3yll)}L6~ zMn$=0ew@If)g`8Jm0Y25z zAYs8%80E@QvF9Oi&(<8XvkdH}|CSkmMyKC~ukIc=RMfmVJ!=2qSgqs%s zn)_Njgx0Eb;0eX;c=pveEZ=KRrWPH654%4ACPQtc!2xsW7ftSHb1 zt;!js9HU@7P307qea7I{ObmVXS@hjjiatqjCwGTD6M!7!3k>ZxgX)0>Oo+Y+HufLU;fgwF-AUlOpO3Cfh^Mfustl$Zw}Q0x zXxNpm!)T^B5~ZyN@ZeJ)h^RNFu-YE=Y&SyE-6{C`H4mTvzQJO>ci?E`08IxT;iA?4 z%;Qv%N$xJMfc0=a7OJ3_EmMaIrV5Ox{P=Sac~FC9PVEO{vlkGu=ss>v zzRs2HoyEcHBwQk^#O)vpaoVhQ&{5H(HV*A>pIxJ2yU%sz`b;xg8QTGlBcq^XpJcY{ zUy1j3%0b6>zAIu4dMbNzBD=Z!U7L+$-BUvg%#$QiEqI*Ylf7dmf5Nkq< zM3>Oup+4vY{e$bVd3dVJ5e#;gV)6@yL=TfZ+j|YLYojdf3qA~Ma%`wkYl|p*!aVwY zD1ms@9B$yTIri@x4zYtjp*5w8J9#h|j9g^M>?v1)s%X$5-pBBFp*L;U@FE9uy}&#^ ziy551g|1q>6M_S;iz>ql;LEP>uvS@?-ZHBZv`;E@b2j#)l`6k*M3@nsGW7^NDDfr= zH*N7(gAMg^QwD8=3_$VeXXLmvWVW&CHEe=!Wk8JZVDekAAMB*c< z(pRp-1zWx`A0`BV_xkB%joD$`@F^ZVzkX(PrL4%U#2vUR%?iwZd4e@3B&QN$@mZ=4 zDVEN~y-6$RsQa?CccUvwk{*lIMi#^^B@27=9%BAGMf#=tE+=~b7g(V$7bcwpD*Gz9 zHHBh0J;?}nOjV<^&#a)aX^Y5dqj@4*hy9%H1$7Kq;6}F2d&aB}mKEh24J8YEKH=3} zE=2j#DHLvxuXz>p7B=|3!N%9GVUN{o5JnSvIN%-Zd)<#W^d)=g@Zsd!D+Ap4cNk`b zY|3!Pi`fflf2$fE+oeW-ckRGKUMEr8HyF0% zmoiuLlR(&SLzlE@()P7G(aGvQd^a6Is-q@jL~aBuOVFW%hsS_?SseUV@d@pQjD{

    aINR?bf@H~mlbjkqYRlz65=@Z3Lb6*nzIPSOwQFoW~25R~W;Ywaol)Rw#TYLqBU4Nak$EVfurU`1w%+ zEEUCJa*rIbD4vdzW?`&q9F9v?X_4!Bv7*3&6I}E6Ai90pFo1%3Zi?q$$Tbh4Gwi!@ zx#S$ExFrSdxO))IofK6o49T^9>+swKGcb5YXtHE2rTaFEjw$Gn>lc%;=v*t;nDz$x z6(xE1ZVO5c55U>+cFY~+@gyr%1(kN&76fcaReu`tF3wO5fCZd3jR#yAqA(OfH@pA*5Q5)4BHcS`-;X z!ntLg;8puTFkqwzt~U;1&^KigR=NsY*ICiNi=&u(0XZT|-yJwS>?RsaGo?>_ZgZDa zv`Mn7GQlQQNZoGF^k~cCs_tWGZtgD05%9Zi7tBhz3*V(BY%u=p%Z4@mxByXp1ZvMK$z z`0Fd2@TX72U>2jHBl z68GoKbl4EP1#X`oM@Oi*z|A?AA+Mu}Y4SAU?&xWeCozJW=>hhlVYw4HozY2*{L`b@ z`9$4q!ZrYnj5bhc+bIxE)n+bF&cn+S+!@2tc+roeYIMN=88}&8s-AWA750Z+z!=59 zOv;9tP(DJNa~qxr6YNar8;dfow?z&2Ck^8`TMOc9eFE2e-NAV6q4;KU5y&_g7i%f>vRl!WjAi*UDGUTnH3Drv} zN7+(SY#*7*HIGn)efM23?&%d&4U;B2RY$|lt%uxJH^?xpQ!2qs#h6rvpONV8Nz9{o z1NyY|1~d6dGnens#B4D*4i$6ch}@UCu<-Oq@O-clE?l{ca-W^hf3jqcavp;Zjn&B4 z_lH1@@o-yKE<@+AD)=DIkV%^=ix`*6g)d$JvzJHXCCzrYwcD5RY%+$&|FSu=czYPt zsEZL%eUK}*C1op@fK0TI=&aiZH_FDN*XvGZg3M$%{(d?+_~9rR|MkMJgNU3rrEi$VFTr5rq=RuzO`zL(L-dEFU{T{@ z?vhis=$J_o>L-aI{f8|E|K5wj^~X5p&OdPG!g@SX;|P1Fr{kX?&hWCL11iVX;Errzj(*vUF%nvId=j6)p^Nz>Dy&VUy=+Vy230P&D2d9Q8 zL-Hg8lB;Wh^_7hPSu@a=op*7vfhO zg=Z6=L0zN|Q`Bxw%$uL1Cp%f>pj#@s=_-Z;Hv4gR?P=^ZIL?VIoBwC%Jp8fx;y7++ zHtcMoNQDsh+~=MryCEbcO;W!WO-dzX%gCyP2vI1dhZR{pjt2#zVZiZ=E-7=>7d?pVB|aRUq%C%j(L+~f zGDiZdz{S3A=t62TXbH?i`K6^OHDw{wR~!XS#vTDr8ybw!&~L9;+^&w+;-+l%fGiXQXo`4n^(U2(I5PMDD0ymbLg`kiVWC*oc_!wRThR5xpO1IR`+4_aQSs$Y*GOHoQ#;4oN#b(btZ6r zGYXUn9}#zsi<9$Sjv&K|nPkA273f}1fQ{#_q9dk#XuYZ&5-rFE3y+io-&NwI_tGsW zhQ0#k6x{^}Ta!qu(sQhSZApR-3*$PZT}y zApYKY4fvnqfkn6(I#(J83=dAxYixd^T}o=?ljJbar7;OcJx2(}Z#MJa0tH5Yvm0cz}8BzN6m)9uk)i1CIop zwN-y+$|3(y9cEF;33NvBCEfh(JKD5f5gkrCY+aOi2h?u60#aXUkca+Afo{SDEgkqr z_YGH~?LT#x;AAh7S;j}JYdg@NI2~Foau&1yk~O*XiYQ~a;RD)ma2NS9EDLqq+5d>=_V zh?6qU1n-E+1~Ol(3%zXDAh%zB3zqoJVJ6n8lUho7VDq996a<}^M^KcEGB#t5M1~;o zCt~DsXobui?W}`xlt@GSF_2la2|e~SCF4WHNz;z~pv1SO+H9~9Mq|4{*3wjA*T`q-f#7>fd=RdY+>P!ra7|(3)uEM_VzLEklN1Smc40&5547zJxvhWn}&>|J{wSkg3cM7ac{u6_zGE4YJz z#9v@2`X#ZT=OmrbE6KRjjDwi%$AHBCS>#~tedII7fQk`wCi{aIlcjG?77`vvc-Vwy zKBj?gzAUrS$qjt8cmigqJ*Vp%J_%-V(SXrUN1Z~R%&8a~Mmy;uu;2^)ToqZg>+LmU zwz@6wJj4P1$0q1QVow2EpvNq|=R>FM@dAtfh>}H`qSemNEXh7Q1K_jyJIGVJ2t-dQ zl84s6Lnrq+lJ#~%)=on+nW}rz!1-1YFpD+;n@=kuy}7MuR<$Z=XgiD9`W zmMb!sY!;IB^kSxZ%?k9y)rb_9Luk_JI!g3ePKZC%1Un9vpxN*`a0&i}tdDBa!>4M;*0Du(1-*IF=jG7TNSVZ;bTz0A#dgMi+m%+yQhGn$tjz%lt&x@YBdwY_K@ zx+%tCo=@+ji?%eOW!jx+)s9XS=<=~jq|=mnHLk?;mm8Ayr=*x)iZjWG=ppcMx*WU} z{e`j*hgx?h4uC!GPw6_tTVVIhOjJoZpcP7EMA``giJgf9$F7CaAHPfkj|pW)#=8Qj zxxOcA!vQ(>vtVY_u0Xr#)`GF=IyBd;gFaCwnE4L$SPwOOS=n#92i6R@kw4YhHN9Q`yIv_(Dwd#<~VOmIP#%ubU zM;!V&qYU{UIz`VudY5>1$)9dJ*hweTMTk4Uk-o~tgM$`vM7OXEIkjRB@$BtG+BY z9ndf60~7_hf`-pWh@JP%>4Bqzs7hZjQ|?eg)@LX}Dqk0*JmprIdPJZ$n{tpz+n~S8 zw5m3pIS-zGt)capiYnXtR)nHJ@28j2hE56nMl+N3K*=H@GVMwL(Wvpb+F78d{+if8 zq_QbMx*dVEA2x!%$5!-mAefLyzD{uZ?SRUfa8P3Z6v+H!&<p*rh!|F2WJ9BfZkMAd^kN$FD_Dt?ES94kgEHhlj|?KGh6{F@ z#}O|~?}MGO1J&NUUl7Tz>EMS)9`cY?LSZHTgwMPMXtk;-J#^UDdS=3WbgTRmXe}X; z`4B-P2ThRel7^@`b)fB_I{5c;BU%%!gx)r{6Wf$N(?XvFF#sA9pI%TVg+rly6%H-j*Rx?>t6udP{+%CZJajuB+Di4S*Hj9lg91X!XxG0KM1~L5H*l zpaf1gvJ&(gS~KE-SBXD*_C%JxF3^Eq`;$e-WSmDMTdoq@oOS^6@DSQPv>z-qtE{&E z7lay*dJ|XYT_lXABk1*~GLerEq>bitp+o>Ln-o{mTW&EFcWX^L;?moKDo6=mVC2$fFzg<$;fu@6hS@ zBQ*E&d}Ml3oc#Ug6MApnPyCua16huW5EhaG{nH{D*v=#LT*Q2^ZH}o<8HKFrf&?{(qD;N|tfwVOjh>$1u!GQ`~3dJBH^ULU6 zf&%DkG)GHymZLyUBC*PECZq6aIioUA04my6RrkL%1%1a40MW6#;LJ=H)Ru3Fl%l;r zLOTbDoDyfyqXgod)OV!2yAQ3~=LvqkiME~*-V2IM1)7Y3+i2VTED#VdpWfY30TN#+ z(t91u$X}15h@(@Rh~wv^n2;0w^qG_AL72obw*h2d#hWXA4G}*ozh*0GGttB41PPl22Cx$P`auR zIlW(#vE4X;PF!$9YoCvSro3zP3V{xxI>Ci$b-V@sn3WUkW@Ykn-skG&uT`t-(*Y5X znvZUMqQT()hae{MRh8@MH=x0w4hY^+h6x_E-g-@rdG)}SwED4_$qY3C*M3Vd=HeCT zYtuO47AQtIj;kM3fod+LgXU>va5pPbaHcL1Hv+DKML}}p%;!bOBvFewzOw<0-Cu*gU#La9r0YTR z=yK3B+6D^RVu?eSEyyDeONfxnJG98_>mc;7HfotKPS(uL27*M;NO;DQY1l4IHpYaY$XVymY|k9<{)i|8?Op&_7KB!J>7cdu%t*1&c-l8P zmL4avtM|`UCfy6x&yuuRS!g_?lU`FlMQ1vwqwuA5Af(#_Awfzz+67as|jKXeM5f#ZAfKLq2nbmpp#x7UzWVREE-WzKjGa{%JRh1ZRu0FF* zvJdyn- z&%S~sl3(b}&Gn#E^(IOkyoAoI)&!9PZ_v6Uvl+=|#_HWBBW6vK5V`HoUjAK_QHy#6mivB2x~GDH=YR4{_?(*{L%oc)3_FZEDnA2Dt?vXK z5>L09Su;xnzVLiQBa{|COjrl%lLJqWf%i+o(7e5(%+qL4C^nua~6!G?;59oP215}jB15_^x*3SA3vi~jsZLKfR zNUs7Ncnkt917&8=O@^3jqk?u8ysO@^evs~|)&VD0r-2oJZxE$;8Ss^+k&Zw+gMt>5 z8LdUtRZq0QpQ&sh?_5rwRtpE!-ci=UmvxZ%x6{`11Rl~{btz)<^BAqUD4o!XnL+w* zX+XWdWS9ds$%Ns!E>rty6$;2oK@I~9ZFK(~IIf;WOxR`>C+vrzWw!nv;wBDsZgLb}bj3OTffn~7_?Pod)fuZXbf=FOb z)e>ulGQkgrC^VPo16|j|na#xxDC0RK^gDx*_E;HGuE;~H_cjTtfIigf<&KPWqOCtY zS`2=ON-&?MOMyat4ziT47U(yo1fC^#@We8o_FXj-Wjd}WYhG_e5yC!bGR2YRo@xga zk084S-_Rj91XA!1kwo;(6r?s0hYn!D+aky-*X_DkZPB5Q;sf5((IQgh5uYj4ec8D> zIp~9R>R1P|?N+Wnx3L5r{NRh~_Ag~3>`tQt$E?AZmx(CRFP_kU?!cU_@&R!VoIoS| zPB*Wc4PJ+Mp&8N+C^2IYhz$xnlZC5*#pFj|Z;ipdONOY=>pVF8R*L+$Itd8HiGisv z+X-fgI%zZz2S(8%s}(X~ARQ-Jsjlq=eU-n^ci~84SD_O4lBGaboykKXT020DR~WE= z_Y%Dm5+#9;KJgP+0ekx^wEv^CAjGyFojKP_Ck@Y~qpfw3QydRmFLp%q-#NsGKu6&J zc@#~Zeojd7PM|NwiQw*1Va82-9KG1rkKXQnKnHuyL9^7AzyrfIMAX(xK#C)49Tv9< zjM)nV6|HKtVb)A~b|wiZEj>o8Ufo^ateSdc_Qae;k z>sM8wAC=|Ei>gI)WmzIb_5uAlXrHxW-a|Sx6mv=*6Tn*{pmVbf-%(NvE^Hkyu$$y;hsi4t!O;T~(ZP%8dhi zYihyYJM*d^R!K4))6R7F^IKIq+ZGD+I?sV%oh@0n)P)gU!6!zah>$s+{Y1%WWwLN( z4#<|dOZOSgCxzx6UyC06n^#M@a6qBYI?)gR6T}h_sL@i%YUnr0!Eaa-W1h zvJmLu7I5~WYm;VRP;VR4^*tVJeHV+m^W>OuR>XS89C31ghdkK4;Uds@C_#S8tEN3x z7Jx?2g-BcVC5V5#2!(8F2KdHVjQHYV>3Zm)jsarwH;`iu{t^?TY$RV<)tAJSK2Ik1mQCev^10ETzVJu!*GPl_X zfu`yP3Rl_ASVz!;{!juMv%3c@POkzOumw#>#xXx zw$(c z5E?xSuKtNfD+^-iY>ozVC|`~AvZz4#wh!p=c?X6)TR|vw2aJIwV2i20_1u^#dZMO` zI4$UH&wG$g!z@km&W@RA?94f2PKuM6gews^*+=LN>5`nS;b3vI5wjdF29F%_K#Y9~ zaeUMr{S8wglZ)(8!@o+v(cQ#6>)DOoY-j`n(@M;ecMF;Ihb{=}86%W);|((Np36v0 zHGw7Zvc%o;w(8lhhSAoy$w2t3zqU|1CM-|KB(DASkU~yw1*>y&jIlWGYjFD>qmlUp{2RUtE@t0FbOps^x zTI7SOzAWpg#(&7D{|~Xs_XfC`+>9LD_0iskHK6fYImjPNM7ihd!JYD@NY4C^wcmIE z(I!;{ZWQG=hW=Mf&mU<{@P77$dpNhmcB%xjYsi0z5 znA|=)71{It0dkKT)37NReO#D;ia*$qiSi!=do>J!l*l-`D)*02l;xp-BooGbsRRoD zSBy4%&ZEr>qma>b9$^=>59uu7kx8B_-6$arIOApl{cjRj=Ee~H`8&|N zzY?IXJOzDiD+P9+W6;?mCsaJ$LO)OS0@B0EOxGm^;KW0q(CItDT=GQ6cJD-x7(r_F z08Q5Kp@Y+RqwMvUfl*EgGA!-}d2?Qa1XnHa%2pWIdR<3XT~-s@64Fqmg+Hha(xg=< z^{NLBokxVGCv9?03C!D?4K{|(pj#y@(W>D#LiNRN(6%K4Y>|;*UQr07f1tt73!A{J z{VMb#I06iMbRyMAyCEQQVi;I2j|JN$bI^>9D`>RCoy@z}P24!1h1535(qNYa*}+=VFmBX(^&uay zA={qpT_w)Q5gfWN=s8O3F=B>}OaXoCvZ|whuOQ!?Z@}f)Yc%Y-9({YKKn4uWCPc-X z3D32CC_Hlxb3>*Q?C!aa4Bl{w6|!6~zRw$sdc3UOdA$&nti1y`-1UUQq&2NK){d0? z#K7falZf-AU7!&Z^#99DnbXOAh}5e_to3I?ZbX&wOSMHW*=#iPOcDALJ_0^P%_2)z z@R3eN95SuGiLz9c$@vpG#LpZxf?QN}e*UZQ0 zMeHOx_?Q7VZoNb63LDYfH|x=sGlxOVdI93b`2}5nlBPdod7$LuRJFp-dqCG~7}&m~ zK&;jmbi8bvb+*4GBbh5iK3NzCsO&^E^*jJ|2z&`&cPRp+n z*hiBbFBBqOmnbpwdkqBNuoA_+UQG@Td;ld`yO4eJThzUC4rME4xcbjX)lf?kWNA==<)09;>TM2)wAd-CSYt9vS> z`E(7D5p};>qiY^ByIPp^pJxj0Eb2wMCdnY_S^;tw>{r<;@Kwqj*~Y-FLZsDzG%>wv z3G(|vp`QabK;)wgG17Mo{g@L1zR+Bxv_OmINIpOouMd&F=`Ku`PZ+v3s!h_S(Ma{b zG`j8DQ|pr1bBSc_7G!YT7QMZH29-&Vf!DeQpkv|+eQ=*o_2AhE!uFFCdf~kjiL<+i z*)L?t4MX$L*6l3%@P$Rqr`(y8ac`ys8j&F>r)er+IkF=pKya=S6W^Wz9$w+2jHtPw z*Vz&{T}>mXN)38r#4{A({T;ZR1cYAzg{=GA!B|xsdjIq@%J997YRyDRf9^jb<%12kWlcI?K~eIh z;QDV3FnwwqJ^S_^eRq{0f**T;dCPX8FNu+$_~TjR;q@7$0Tbl>I~o~_9Y!TlE-2vB zc5rlx1|gbD$t(88%x-ldRB1g;)OQzxDnU)vYjPR2C@VAn+!UEeL7ep2u+e(^uLAmA z#|QA{v;h-U5=o!&EfMI2IAGpC7gQ0W0*oTdt^4<$0}m`km>2jKikOZ;;G-+ZD6{|z z7Cxr6`#K4&)IXqoTF`q{96$^6XEWb}B}q+*V$^qaH~L(n2I9&qiS~Ub={5@nJRK7P z%CR?yl0El{=Qr1*-?QYAYQ_>GUT{V}z26NEtrB7W6KLDk7Za$UqY+#Wt^$e61he8> zTj|fKen?WH9@tj&p!vIGNzS=G6csRuS|;a^#B715=J-!^I6?`yER+X5KNP?ct7a4u zRDqJGw8@#l(u}@!49$s8L_TKYKv-Io88%Es&+ksqod0^P7d?JYOI9j@jNUlVUo?T< z?~3!zd{O5T$Ev59b8Xpbyy;Ii+f<=PE?lL+J+wa*@;;5?_CShSIdzy~j!NON zI}fl)I1B%SE$sT)yI@1X1XL5Ai?466!2vm;u&>G)4=?qlHt#QiC1>5Bgk?Sc_InLy zq)-U|{2q^gSe}Jxi%#GZeK}a9Gly+)Qi4C@Vpx^!((r(95-S3q3oGPUr z@gA!TdtgPAIQWOqpk}F+uw94F;U#t<*zj&9%s%f-k>3lklGJv*U%r&r*O z!?hH0 zDod&H=ezNPv$t_{R~);!>=fM9AkNva*$uBL8e72p!G9WHcu$`}l9VuQ^T zp=}C}YIqpP8ny=Uz8F8FT8$6EjkEJ1){2AMJzwJwKkC_gHr=dS5Q2d(-;?(;I;eq! zb2#_#TArTwW}dCXHvH~h1b(&QCOp>d0WYMc!bc96wJDc?^ICJMQ)_PF7cBv>zVbTm zn=?p${cK6Svj5EyD^`XV?{-i}tM;?S3%^6Bz*uI(+9DoboC{Ap^M-S`$#T}5pMrIY zbr6R}zygmN_|@^SfGzbD`#zeWrc=K1lpI>HPp%oS>b#)7!Abw;?EV3{(39?QyrK zD*n&31grf%53kM<j?BG;Y5$6T39&LEe8`u$cB^_+WbtYxOi4o9R12?^_q)%>-%oi=PVh>4Q7F zDL@)(1-2A1~~dQV%+?R5MG>5J)GQ{1XuN}#f{4S)WTX5_P5_r z*sqrXi(K>Zm{>KOTB*dSDBpLkSlan5ezEiodv(=C+~;lx)1v;P$OGRw*7_#6!2AiWTv?0<_VPLQ za1qQaYovT{|D>XZBcbdaz}%`JQ9&1szPaZBfEAW-o!k)6&@WTs+=idWllz--4ySzgd3nV(eSy z3lq6j*d!tWO4fTot%fLS`|=!E{OB5``jf%ese4#~DaF!n2VjqB4ZCOGajZOf8BPjc zft)-qsNzJ!Rhq|e+{#93fA0%8F!vKHRab#WSIV&2Ga@)X%H`11(vkWVo5p*Xew}@X zo8jK6!|>usfcwK~_Sv%sFl$hlTlun#a&SEWpXE4E8HM@0ABSf0b06O3$lQ90?KaDB zH=nG5buZOPiIt*cp$nH|&k^Bfsup4m{bXp}b_#x}Tu8YNt%Z*JHn5S4198;)P1t)~ zCGWQLb!um57<<8EDZE+gMTONb$89w+@Z@j?7I)8OZz#{C{5Q^p8i9AnCu;l1P}_rS z=tL=Aywwmsmj^KHKt4`0sKge%Hz3!;iET?c$YZnwT&D6v@JOu%bP|4n%|#QjoN_&T zAaa7LugRy%SA}jNILir9Ya(#pW}(p9kvMSNkp%G1@5jjWqwr;A0ZJ8o*?XQJ(}NU-ml+qoA8bUa{>|NFpW@1CX% zLfY_P;Ti1EkC+_|*{pDXH=d0*!et9yar#a4@rSq1uuQ%c+aol=#E!V(zkv$`@2og> zbNS3EhIgsyR|#y>r$`)2{bJ)j{pMY-tmWu6N<(!sFW7gX6R#+HgKd8dvV1cIz9egh z?c&d2t#}FC-doRp%M;@tw0{h*d^Ugw?!AH~s&c%IA9(QSHEOd6T?|Jep+EUA|ek#rB{b$NYm%p%)%l32b zhCF5#a!Q%=^2hL+?=sw$Od)PgQ3;M)uoC|`UxMwt3$XVR9k|4PHGB0+9Co=QgzH|J z!wX8Qpxmal@VU$^ZiIj_pJh|bN@st;iVZD-yg(7h*iLft4qH+kTYTX3zFaD&{s2_e z>VXqJVR*9Q5L|oo6s54`0$%gGnbO;z4Ec(Gpjq`B_F3RYY@m7`zF02@?=FsJgWQ_% z9_pB1Z|9oddlmAShBzFfa^v_|hT-wgM7aDNe<+|WKb}P=#EP%G> zUgMUoU*xpydi-pCH}(4PLC)!)H{kJmEd0oQ!b+>Vu--`kXMewer8u*( za#9bq7PvYyxVFA)D$dCD8ltfTH)2Ep7Ds<8Az0>^m1X6$@NWevzUtz9^TxcN4ph7;}k^(j6v_s9FH|&wN`LNmN61Au=0xCbJp!X^+)cr3C z7mpt17_=n7wmJ!)CX@kqtFnB=ZN$5r#Hjc250!Y=Png|w-QhL2&l{rZr8tle?tyR2AITM^XA*eAl zSw=`(kxhw8W}79Q*(4buer8_^Jj4^khxDD;Wlkw&*B%PN&P+HSoXZ;R(4tOO7xTQ` z75LgJy*x?ZQ}~rJ#AW}^QPbJ?cqJd|@vg>o@J@k0Cb?Sh+Z}y)ZU(_dRhr|ATc_C9 zE6SmU!$Ha;aDdYoHvnt%V|ix|j6vne9QMggdA{uOD00eZ2NbEag}#1e*m385c=pm4 z?B4R2R}=6QKL0lhGq!r%4A(d~!=jV=HyFV__laRQ$_-MoZ-zK0uHB}(N3X&rpw723 z+|1b{PvJMyp4i`~1CL15;hF|ze!}BCRyyY|r}xEsyyRga^|J(1_wPKXk}f84VkMhk zyXSVe`F0dLo@oO&&Txe$YmKpZt%~3tzF{-gEQDKfCh>TLCIlZ`*xyxmVB@P5IMquE ztE(-C{8?4h*Qbnt6DbK}zcRSu zPT!Rk`17YZ*nDR>4jMFtkLD=i@Apr@fsQ5kMcNf=e@Z5Mj=Bri23f%;P=rrfs`GqS zk5O??AHlc{@7XoP6=*EUXa5Rnp7f?o?C-(t)NZ%4aM8Z=)S)|@sDQqHs5jGf}&0k)ZLyastgxze?Dd+Tk z?D*?J_Wk(_l%MK8d==)v;gwr>%{rghMe}4hQVr*M+>tr3W@s^Q-->S}x^|3;2{{LH zUKnEbksNHL7xO)Xj!~QT9Kl^9F6@MTAr>w%$14v`u$$I=g#}lG;IH{|e6?EwF6_5d zYTWS)uj#@x+a^8}zNoN=6DeQe`iOH-S7lt}Foczo0du3jt)E%roWwJjQW>qQUv;y^#OXu7*vbt)0gGZtmdN6aA;6U2^1 z*kBhbhc|dx2bVsX1(Q(=fanF`0hU4~D;Y=`%rM(}dPuEC(RSp49iJv3B! z$p#jev#RxiZ(A8`Sc2CkkQIFN6m@pR=7OSm+U04lkF- z!LyTY6j4P6{dM^rn|b3qwZ!lhdwib|@APmgUZApTpIr=#&O%_f^dYmIr1Vnz7V3jrtTngiwMokm^z@`ar{-iI%XEyV2#ck!9{+ptRR zC0n&|E_KIRlFGM>V9mRCVfMEN-k<~66=r?Z8m=Pr*!!M0Kzs7f8*YL*Zxg84lqh!7 zewy_$dOBKeIU1c?dqhO5MIo_k_ zLTGpQJWN%&NUhXSfY zK-r9RVJo*Y*mA{gsv>m`mRioTE{_H74TWjIp}ZINtHrSCA`xr;1wDp!TYpP-IRV)wjkBfACxh{|1awJAyCZkNQGX z(CArQo*4oii=#N}$c?yg?g5yP)&Y3{~N; zkCzqh!jXDe@SCwaZc-sQiPoL$u+$c)V;lzqU*E=iUzuRb;v61l?_1pOlFgo0zK=os z9`x$JcI+FJ2Ib?k@#f^U_*7a2uZOQlT|I1ydty$pPBw-3*xV`Vd2uteC~w7^SL~;3 zJyYO(?rW$m;35wEDx&Jvq)-8dKiSL;7`m=AhRt^&maBXSH$2aVC2DVQ-LAWEzf}g* zckqUyH{9UUsXKVB=ox(8`6`y)>xji41GvS%2d?zE1*bmDW$V8$5wP6kS>w1B)b?O$ z>=%^HKDdeD6UAbdFX-o8>>Z+VYe+~w@w(}dY$(>HtB=#=sH2e?Re2U?z?@3~p z$v3mUHs`3+bQ$X4hypI(9LK(TasWRaJOLv`&2h`d3|QYO!dJ-c=PiAlL2j;3#FvyW z!0qW}0`~4ro-DtCH&ay=X4cQ;GMxqZ7W*B<9KS+Uh03z_zScN9D4klnbO(HRbq-g( zaspqXuCw#M^VrABGI{rVcEG0T4oY%e8+NgHO0_B!<4uM7@I(0-7`a^=hSw)U3Hv9k zZe%$}L-Q^hOZ=r`9P2nOP>VCU;X0*W-9tU>9cM#Tg0ZC6PR^&k$84J5cPGuSQyXW> zVUJ((+`-qnTn#Tl&#v?zTcI0)ZwGwA?^SH5R_BN8NrfEvGD48E{Ci5ZJWA%s80=yW zMl`TKj{m_q*=^L1Lo6E<6~vypWdcFTRY=V@vmL6PPpo0fKX7HY}HE=6$E?@UL zpW`HB00-_JVJn?oVMBy3yL^W^Rdo6SHZg5xH8#p}QRyD`2E0f~H!HGN_NL>ri*7>U zS-E%(lMXEe{HZ7NM6sxCGL%ARa8Yg^FZxp}e%^5ncV14#W^D;DE-4e2o`N*&2*dMo zld#*@d3bavmX+SFf{)*u$2#AUh6^edQ$_}2_)kw5?{vr&=8|hFE+h*es7s_2*FMJ1 zU6JsKcrm;%>mrsJ65{`otfDs7YCyf^?s%clC-&mEOdJ=a0E2C~*m+MLZ2RTH{tG?K zp0vm8Ze2V0Xk7(XO=`!1`Fhag!3Er?;fTXeJ!O}g2tnC1yKu$tP-?PafL$xdS!xd` z!;yyvu!~F|$MR|^B^IoLOBVUF7AuVLm|6{+_%sf>*ml9+d#_-_=M}85Z4$L)a0Z@N z-$}F?luf{9GTLts4KTxP}3U}0+!p^z^ydcRCKa|-B6Vqd0-#rib zURoW`hViieR2V+gro#6~1A@BB8aC(N#s-5L94hZ7`?)xjGqgyA|NY@s$OetD3tNh) znR~mFcHJrjIazYUKi-N#4UOOcyQH0MzW!Jg`M zVtvxvv2jZimivfsLw_kgwq1n(&C&&b?B7Tl9V(;FNQgkSutX?*H<=NCcnAAh@bT%c zi|pBAGwwIlXm+-bH*baU1n=Hs8CEv858r&?ilaT#*;5z8a2x3jl_F&MZ~azaIk5>? zzr_Q8e`<$)I)_-%es95?BObHp2~R6U7%x4Pi08eNhTl`QVclO{yvgGnTv)XL@@LG1 z;@&2(^Y<=RIw}qeje2r;r^ayS8Z|07Lcl)t>7lBWX7DEpvw7iQDfQI#D5WSQ!MZ=Q z#!gO4amBg))DNv~Y=6mqe5S?{T4c%b!`*w>B|9Ea)0*pW+Tb$0;PgF?`%(?c#Q78s zR-5M7-5G&@tlmN;=Q3<&)4v|NXRKlf!@vtCC3dVJDp`54%JmtI(=j~+i?QJtz zGhaJu-?s!jzP1or&1$9W7rlYTq1R#HZhva$yO(Uc&men38B;CRI@obooBwHqLy7&_ z4zpYH@S*kJ~;vLC5%W9>&adD?|KNxovWrhQlr|2XIF zmjXO%{4N}2TBxcVb@<-{FBmf9u_bT^=;Gelde&$?w`fl z&MwC!9D$QpVxUy_M>zOrAAEN-1r`S8(y^TiaLqd@Smm$fV121dzWnp!(j!Nz|KqfTB-z{!y-;H?jf`R4OPcmvD6 zvZ5!$;X~nSyl^pKZn=8_yd{wx6NRaQaSIr`YU{sL;kJ61LfshQ(*3z=KL_aO~0?N;vH_)LSjgjbHsAT=ImWf`6ZbYbVoS zo{s_^`w zDdQ(M`7qL3i5*y8L`@}&K$+v+?0Sn$KE-( z_+<+_6=uXLrXu{LGYbocF*qq%6hF2q#l8L)S#3cYyz`tSuP!B=4V}klr+$gzH(_Vt zTsarK!{ie0VX+`ri{Fkt#n$8BOg?_p8jAl)1Yw!7i)^uR4BYlI4wJUQ@K~D)S3jhZ z5`CfuKRBL(+xRy4CVi4s8hS0z8zi#Y+%#fD{9VB2edHa(+?{m*PS&etkqx0MW0re!;@z0q&@=A8>Pa4_VB4sFL) z58J5YVn^AnVNUq3Yzjo9E8)bo3#`oha-3!>0<{MMaDRgw>@hfj?F+^DxAzrL4H`Z; z?b9_@Lx9a5PF}*d2S2I$>j}`bHI@~*+X`J>14&wT7XSPl3eRtQhkw3NW0QaBbG0ce zR)2Rk^|@&lZn-1LKYVT>=kqlMs25O))04wEuzE48Ju8PwJEujhE6jo#iZ$@P@DYx! zZ7T(PA5+fyg0)$W6>$0bEcW%b6d3R*5}sRP4ZjO!Lq^vs*bGA+%$%NyuO_D8lqwyZ zT^B(8oGHss5vA~)Pb9vnNW)9+O4#J%G^?c0fb%R&dBvPNR2})8@?3EePU=TMe!&8K z|3MR${;EgKul9w~PS@FcQoq=*(N2`|Vr^_Ek;`r^aDuU+zt}XpSorC=C487B#dqrL z;|ZM&;pG~c@X|J!WB2!u@c$H@c_38p7sh2Lr9vu1)|9o4xp&?(NkrOINYef-S}Bnh zAzLDntt7G}Yb9aso%c*BN{b@xX;TuFHtoxAet*q>GjnIT=RMDPp7S~9I)hqnFQDF) zePEV#8vdUDgz5t~Qbwkg8}9QWkuS%=jIMnA_Rk)_Kd^w&6(tff&K}WdEI~MiVErUTruTd5K=Xmc%7ww_6L&q-Qcd)gjbcHSYN53G7_e;rfe_8$2%ce2%U9em0ApGdK3Ul^I zDP0naJHFhY!&kn-tAcnkYyJ=Vb=WI3Tc{y&x7bayJO*I%x;}bxSpr-cpGra>u92vG z7>rx@pCDB?5KMLqB?j|v)1D9OF~iJBGB(YX8VjLWY&RPayGuZ`xHi> zEJ1@ohEPx*4X!SmFfjKzxPN1D&3G!*_N^uJeVW;G)2qneFgrNdb%;3E9g#Rp_mEoI zkE28BT>QPP0MmSeVU5)kyjW9!vxn5tnIj+LGP|Mhp?xRm{Qew%8$S_F6CFbRlngpe zBZ1g3e#y&$$Pj->9QG5_xj^`@qy>)!4_ME z5%t^2n$LPz-=~7bCxV5Cju)c#TptWkjD>dxYH3r!30&Cgf{i20vENu14Bt$}kghcP zTa6R`DeR`>V^5-oN;OW;KPa2!q(bzJNT`4I2`Yy|gU%_qZ2_>adRiZX|fjq`D;SY7l+)_Yp#hk&SW3bt*^ zq~^B+sp7i7pp)B1cE2!&3&sa9y|)|UVt%03Y9(o$*bPr!*T>fb#I(9SfEFHdMTKWS z>64jC(iiQHk{g~oaeiGS-80V-&v!VZY0N(G+cW@FbB2jNWaxll)_(F}vbyMgcp71L zet%eAg^da8MKJ9nQydrp9nC#03`gH<5d%?$E2p zZ)%f4rNWGVVk_PGJXJ-2g~}1 z4IM;Yv7K004T6>k4=hspM>q7`qQhO@k@%1dBD$T3|Mks=__>wP6W&cthHJu-6OZ}S ziWb@V`w=;@ClDgKR7CYwdRXQ2k<3^mN7sjbC9J%Hl<1m^TI?s1q>IyJ_lF($+bI#o z-q;C`^NL`-v%%AN5t*3G%r;7JN%g;RY!}X|E_89ZWtVWC9Z!q?4F>Wl|2S;B`qUEC> z38!|v0gpw&V3D&AeYLD%xa2kbGJ8QvTvni|F*H$8rC}+w`z>0@VDiAf4$J zkSeDJuT_GuAnzD8caEha-98BGW%tg_O+o08w-BRz1_N{P4Na-Eg;dXUVoUCN{^fzTVMBJQ?C(qZ({Cexb>)2&jJ2MlaEHZ`M%};d2n{_zk!F<{|<}U7i zSP!~0bzynQ2F#aMLev%=(ZrVP^mOJv*yMc*)Iz%?I7UQQU<+i`_Q|gfk5ap zTwqmz3+e*NEORd~Y`Q}K?TaPj)o#M2hbutg=Mg$nT8wpbyRfcwtL$?Vl5bUA*!_A1 zc1`XRx|Q|A)9#Z(tDj~}nvyR5Ql=zwcohca%J}kQ5{TT!$0U4XBT?vA#+=I&Fe3Z{ z(J4rTk0+EsykwxLca<*gY0;IYnAO0@VeM4@!ATmc6fT>qxZsl^M})TF`-CfHGoY@< z=_vJcM_d1FNm7mh77#94V^yHo<@1NYk zv4f98D0_o#9H>Cf7|n-VFLl^H$q6(A&e626d|bYzmgu_dLEVy@l7ELx(f9~TwB0U( z{QKK5caaZQ3h>3ruXhW-jwvKxe-)5j?*znaQViMckOH&! zHiNpFE%9x23y!CTc-Jv>mOsxCuD30uwUc zq4r`I`oJ>?&fk{@_ikBjPxl8aJ1^wmAyqw-1ezsAqW1fi=&(MURs2#pF*)1`R1V<2qIj>Ub3&A9cZ7mn*#1PyOb;GXHD!QtmvdRf_#M(TB8 zNx)p3u+fWJ+AoHXZwywt>eIQZeI&6sf#h=i)MLR6{5oL(z8vUJts+05=EQ3tU8;)3 zWEE)`$I$hX(X{F3a2}$lb7h| z!MiY7KNIu6G~mkz4}^dFSllHZPm}J=!d#6;IBRzj*4k)dpG7FtH-Cq@(?8MmCJ*t( zL`RW%%~2Tdy%tk4)A2{zO=?$mmsV!nr3(TE;UqgVqM%lV7l$VjO->d$ah`zv{|xB! zp-V{CsB6&BGoDWv6GAMu<_c_$KH~w6nbc}*4{ZxshRfgn2Z1^FXkO7zkqnocrb=&ZZ2IW7`f!XLxg=H(Fg+8q9d#gfx`_vr);0$E48!26I?c==lrzEr#j z+g4Rc9v>M-M{4(y0Cx_%&b+7pR@lQZ4O3B~zr0BOln+>iU&p2m`$^ z;=Og6Y@eV6s>fJom^;Gj|(FSFIk4hA(6BoWXOfm!v_!DPN55+(&P3EMZsXX3`<5 zX3~B2!DzDXrEp0=16}ek4fhOIg$^qnQPJp1Onu*o%Z!b%^Ghz7;1h{~3+I4$e=Ne6 zRg`^Y0k8JP2-hr-&EKNh$u^%osQot=9|bql%Q3CO(Q)N?=V&4LZ_>dz*74Y>+6y`c z+vt+oAsDmtCK;VFhfY>}MMiRGC1bkZQ0Gby%vaLG;M{q{MzSA===staH~Z*k;RneO zp&yC5)FYWP9CqN<&B3xw4|1Ao`krMQ#`NLU39u#9cZB_ir4)m@#8Ob&fYwv?bu0@GRMT zmd6L@x8nDO%fNJ}A2;G1?Z zEn9X1QWaw)!C5LGuYCg|y}v*%Bg?E*tdz{Ws{#r}mod8gg~Vod5g0VT!gSAa^0Vj` zRsAT_l>#S$x7KvHGNld-qQAiX4+89hIMT3gJ5Yay{PQ`D(bq&|OJIV~Ip8w+6H);) zhDS;CHNTU_SXZ2tlnHBvaoD`PRFWol3^FrbV8hl5`pBkDIH=Z8SiF-+{Wgw+XwUa_ zZ@n&5R_rAo4~%0BSB4SB&zJrz{|Cw+Q|X+fbGW~MKlr()k~foYV4g%?@?@t21gs0e zbhER-nxw*mZdIHWH5T|CR`~5$0nOQbfD|fe;ESmWv}i;k)*KarxyC8Ug2M}iT$Kjh z+r5&SNo8L8eFLQ}SM{KBySb?LWg1m*dx4rB75LSz1=(OnoHFDnbngB@i+#i}RLKt} zE%b*x>v2T)vJ)Jbkws1I>!8cfToha?g#*(w;nP+-X~0J%>EiDz`K_J<(B;`)`o+5j zLu&4$lFZW@capNo3yd(cFdLdZ>d3HS*?i-89{C+15jq}p$HFP`lAtxlqJI;8AT@V2 z-Br2@-xm0Q&b?l?_3$Y$FD%0PZK1+RYXa$D(@L!Q)(g1~RX|n3aMytZsGL>|txcYI zPwEJ*wi99Fu~+$jULsZ{K9; zEd^H>x}xQno6xW6Nfq0!QU9L-sQX7B-rR8l6Pdp}&xXJhJOLYyc2V_1^1>WgNVEgx z-~;!QeN?%hj@ywhIkr}y$mLUcgH};vZ1j0 zq=w8_vkod0-{RW+6L7fQLi%)4EDo7Imqw{uNbN_RfN{t5q%)bzLbG4Ls0Sw^nLV)* z{qk7+dche)saI*f%~M!?Jr8Xcr=pUnJ`@@RVawUYP$P%|SB33(L3)C?&TSMv&>qfS zN~#9krl7JTj$WD?}z_(%o%!)ab0dsHQR+jgKv&eJ}RW z=LLOq?sa|1?=3^n&vPzJ8nqDOe1AzAVFz9iog{-3V~MNRQkb~)1Z~^bDfyhPD~kM) z2^txbK@$Cmv>jq$=K(?t)6!%)xEVe}%25WP1{6`h;@VEO11LB~{x{i80E>cwj~q z?2C(}Y@M3qxE=@Byo|8j_7dF|YDn!{hof!oH#CT@#6J%gfqK;t__lioJnfHzmMLYR zI`k+mI%G`w8=APtPX}FnQ_)E-6g3vd!P1Ee_~&*z@P{fe*!wFDs=O@A{d7nUKTV>A zX{N%DpHD++D?q7W8lHXoi=LJF_>)JypanZppnU5iTBNd{e7kZ6m33a?2lZiKlvxd` z35SJ;mpy`ecF(Dye>;Z9DM8LpO|X&wjpdvY4m+hvjC2o?@X9#&tlTeoIqemBB|9Vc z{^_Kfmd?jJHJ53c+Ie6KhEmJ<7op%v7uoZkqC)0MAPSmr;olVDkUOTNTrmZt3mZ{7 zwTAkgm_gzOrAXd7dSGT&9DI2_4pZtzW6u_Sc(N&jcdR)}|0q0VSE3H~{`V7ZC0xW` zqw`33MxUhpLM?vL_d(D9_TbhM2fX!8O%%~Cp-U~B@XAUB&@eOy#|nxAFA~hocZ8D* zdhvCM9|p_!!3mZ1@To*1%ynE0Pfdqm)IwKyyW}DSY;u!y{kIw-=Go#W#pRNg>HBe- zelbe&kK>xQ4x!%5vBC{W3xPS>fzw;tg@W!E^pj@;+0?QdAL!VF{Az6oGdm*5Q~3}6 zdCBq-?{2{Q1%<@br zlDqsAF_Gz^$3o($!^Q^~su_$DJ#TX2kAd`tb34Sm8AMJh9);SKYr*wkDLGbGK?lCq z#>;(`Kz_%=S03rg9UI8F5%nU%I1UB!_wJnSsZS$7W(FVcZ2=>ihH(gZv=kCV)E`wzcpd4cnbbPT!2BT!$R|d~^|lvoFyBzgnna`8u5T=o&6P919jb&m?*- zmXJ3}0S-rOrrpE0qOo}vsHNV4pR2>@nVS~i6ctA^``vN1G?{=dL8(~rRPq&i7^q$#~*U` zzoqytHX0h@W650-L6z)F#ZO(0AmM2VER-ClX=CL@+cx%snY;-Itw-U%2LsSd6a=NC z(un8$tE6Q^x?~lp6n?txM03nkp}cV>tk_;bBEApD&jq1)<)M^(j`hSd1MdpmyOLp- zc^-V2?kL=R{;AMUV=#JsIErr3rm)j3Sdy{90ovYfNB3*nU}SU^)?7UyQ6Cvk>)qmH zxsNd9nhi*Wa-z`b_bjq8=P;b~8wfc!Z(wuMHOxCY3${LA0y*RIK)>@jXat`^w+>gB z`=Jc&FJ8t$12aLi@imdyS>oBw5eWV^2rnPNo^&}_vDgCrHC8~D%v-bkfB|YqZxc0> z5{PQ>1J0Kxms1&fzTrIPUeSPJbycC;%@o|dT|r8V`YG)hEy~R65gP292nH%?(EjBp ziEy1p?RI=1#&L=0;C~&rUH?rU{)(jGJcEo}pN~DQd3dez5MD{D zpzq>p@yB2j9JFc^HvetGL-paX*7+m3wj&dDIUy3#duyXiJg1Ex$-z`WWOLanQ(={CO<68E z7MuTo&}IkHnuHYcRjd2!bN_3U4>`lDXN-K$KMky$xnKy5@-Rxq1+& zs;FS{=2Td}Ef93CJ)v)M?&0$+9T{dm>neCt%eLfc^CiXX49QRFT$xgsrZ@f1dpA50)y=babt=v8vL4wF8lhZ z`{?Bu759t;U0g&?|2C)UujXKQSUQ@#z7DTuHq+^988BqaC@>g(l&Y*gNtIWq!zr2O zGN8x@tWTHgX*pmt3HFof9zy;zPGKzLQ%O_iI9Uzu9Jetd9 zNZgYa3~N`#t1i(bE@_P9@%UY6RhI{M!V|EeXbK!VrA2Se(8tIr@5utMUBbwR@vzK9 z9@{PzLh-N(Xpo=`i@!%=r$-ifc_*4S_Y~p5zC+|k`c9|`G{x&Lwh`^|3yI^ULb_XL z1*Ak=qh|B#$^Cmpq}+9g^ospa$++5+!1`;(4KLAhTLwBUU5MB9{o%B%j@BKuFtlTA0tTxBdt%(rgf0cT+n+eB+`QnHoOEgnFiu=6%>9bv?IO?(^BrXl2 zNg>+6Zui8mKYCfMMN#ZImqcuOosECBRdI3J3!GdM3pSi3=x5!8>{WB<*_1`#Qo06V zuYfPO)J%-3_G577U(|}PCDm?&v0{7~s5$LH>$e}kZe$~!e)J{SZ+ZkfM^_UglM28I z?Ih)JKdG`%kv`9PKtIL+>B=hPt^VH~38HEDjBK*wzzwRgu$DTl-hduf8lbo(gf5Oq zrmHuUkoU5zz`&rvSYRd7jpgQ2bGKrUbPlD1P2!0&Uxxml4Kew|MwqKvOs$nq(O8lM z@&lTMu3r{Ow*F^BLgT`*Bl!1WbGQCcDid5>CjV@+BWJ!#( zY`?^9$p8Hh;=L-N$;+4Sn=%}CMu*YmoE^9a|AIAs(V+U+4z|iCVD3;k+#4p#XNIbZ z4n!1?xaU*xL1{G|^Y;u@Od)Wgcs~63eGC1Ci*dT7h)fPYO8e_`AnPs*)t#qF%^pjf z^|K7tTQwu5^0;Y)87_9qq>$WA=B3|+VR<|X&$84i)(FS4@o=mn3-r>?;kNKP`gQwL zEK^1@|J-Rzt+2;CEl-L2B@VnWP5JFwXR*XIcx+bTg2g`i)F&G%`xa_+)piXx4@M8UvS(n5Pq1C zg>5M|7$@_wzOK8E$(5nxo?sAI%lAWCmz*r$5R1JgZ((7bB?RA8Bd?>9X=deos^wKk zaw{g`1w9{(pSqq%4m%)gTS;_7=FvaLbkXGTWYBtLh!Hlv5Suwj@*(F5t@u$0R>BTp z)3{E$^uvXzMY@g;1$w>uJbN&G4 zZ!JQj+Ie(aXSZ<8X*+ZmzJO{I+4rwH6gIpLgPZD}LRE(zBDyda-+iA2!vcSkS6>Y< zLnn)S)lfrU%;|t#n@T04n--vM>q{J$AoEAk1e`tT3oLTEC~T8E0S$F#_$JF5779g} zfFC99uCjX2mV|RoH%6*p>F2R_Ta&sY*SW z`>ly)XdTBat`0H64Xeh^!7Err8_SdMOoXb0dscFYyqu18qfX(py2Y4V zE0we~wh%ShJb6lkF&xc0hSHhYBxXc6bvNA%Pp{9$&1V%w*Dt4_NxiJUY@CfJKlIQ5 zm0TJ=?kLsY@sd2hFcG{B1h_oG^nGJ_HkX>*2N}Kwo|iD4r}sl{->8#+k!_=MN!jvz+9O-g?=-3|Z~4yNmnUk7B|N zbqql_B3pRo5sJ^Ad6D<^MldaT z7F4ePz}PR!7co=wB6&I9q$tpvJGE1w!H zDU!%Ch$#6BAXw=O>s;legKaNCflCPKxFt_)X9hx$lcse0@Fdb4y@wJVS-xY#IxLsX zCm-cnL2-T}F8y@^!qn!$1~b_;jmswHnmttgX8^pkErqb(YoY&gn4~-^6Z3A56m8dh zN9v~c= zDxSU{ewSWteM)E4CZa;Q6Ao_K1$z%ip?paVYQElpMK@a6ZzGyO?UOmda&P>ND`2(5 zUoy%RF?2#R*sq&Hj(wkn2Q|~-&rE%U8WUkbXBEF^bpv!{y272dgXG2bN0PeZ*{~#Y z1Wf&_k1wxz!}d5?Uv+aj6_+VN$(IaxpB92sHk>EL=?)OSD*+^z%t5>JJ#BYhOD@GL z(5P<{Fm%rVtd*-JKbqR%7H1>v8XTvaf&1SsPl?cyk zJE`O>gspeXz78M-$p~UwH8q8^EmX7JBcq#C-r^%;Mcz* zYBXv+ES)hD+*a73VOa@I4X#A$;zGA(K4-gI?$Q7cEp*g8fz0@!()5z8w03?5n5HFx zO-2FfnV^f#vfRg`ahjr+yUTESumql-jNwD>Y2xZPJK1($F_gWZ4Hp7ekYO*T;%DWZ zICbD<)ZOL}uP^)19onf7IwBGx4_l&9LJ9fL@+omn83lj3o&bOC0el|r1fy@O;mX0D zI7v1aKk+Vz6n%HW8RrVnzGvO4q!Jv>^ zML+ZNp!3OnOdTNy3SW3qe)p~LPnb4)W40YWYIu%8=G#E)wh}(q(1r4}La1ssgfpA3 z!UDCCq86Vn!DML`9Jw_U13t<0pV?X%bSE05FqBjkNnyPEW~fsfK$o}r;iN_dTD{!> z+!RFUW!Eg5Dc&VIN86DoOTnc@4Nqq)a~~&Mhs*(ht@erVEE|Zv>t5`s&L@2pN2z1W z7D?_;e_#xKX@|^5bld+R(Xz|IF%QI;Zkfn;mtKd_PkmwV;EV7=5`jh+XTX84p)xaQpcvxTzQex9*IPT3o+{o?yoNtBeB0!Fgo%^nawm!xTQ=+KD@g z$3b(c4(;#!mq!KsnD&Dze)~gRP7?Ggn2UlWW6}77Huz^;1vod3BnM8f z!#nP+METZxD489`J5Nj}GxWw%N0mZ|G0}&>2PQasvWC=p<#zH%rt3_LHN>K)x;V`< z8LB@hQ75;3@MUZnj?PWRGYbadjbD^>r_Dj5`ZN?A)kTf@G04jHqeXrRka>gNV9Zbf z*;b@WnyQw}8TQjhX% zl&wp|K?~X;vS|cXu5!Wyot?yCzz(6_`7e?|#|n{NcuG5271TQu0;c1u@w~}9I!q~x zH~p9dqOTgFtOS`=cGyPnp1Od3`eH#mbqA4Q*&@;SZ(?Em&vlS9=ouJit;IoSOd#L! z7ggA=fYb6f;;lv{(d_&pX4L9guv2jqnecWtB-tDX(XKyW?AOI_o>XV#()?r}0Z$JSkrgM}BX6jq={p@NwUNr1FlM=#yS8jST>MIF9c;Yj%^PfSeAp+#>)!R2!xse1mMYTn)gu^B^9eocmC zVv7=09!)**xTe(hbVUfV?Tw>0@o;Qu<-fQ!zPo^<;#UAXr%tPFh zBfjipgGwgM_5ioStd99C8Oukv5O#xS7!$6vn)grg<`m+02=qSzdqV3W_rqSB`O$Hh zi`-htjDD}rFS)*gxoBU@#g&$Gib`?ZMSERdAitSga){&hZyCglJ!Q?vH^*>WFD&7M zb}nM`W+iY(KKSqtmnpMkd<9n#J(4|~_LqyTC%i+DuK3mZ*PPO!yJAMV*+#SJiyvHbf6$pX>76jwzQpui?)*PHiuJ#p~dOWkge-K~eG=a^oYGO*f?bu%-hOEs> z8?MF0MR0zxG4Gz>z`h|a?A9CixjoGmY|uzm_Quk;V%O{GjCZ*?pSZe8{37GIU{!QC|jCtHn z=5E_)cGTN$PPhFr_wWy}$I>S9&Y6Fi_Ps3czw4#|{AC>Jcz=G&Eo;_gwJx`2;y`v% zIK!?Vsl~qg^M@H!nZ+y#9l~x~Xv(f@x+<_JRA+mH`fPCCUcT+KJDa(D9zXlIF<0V) z+>F9uT>7c4;%8M`1b=RnbH-^_4i@SY`F$fFJK8oT%Q(QZ7=xRkoNcle`=#@axb?gq z@7wLcx0vX$Bl#LesZ5u7D5ZiQgJRe{*~gh0^;C|36UD7x<;;fptMaCIM{*~=PUbOp zG<*4?IvdyKA*dP0^2_`GFhiF==lsJo*iGqjym)RBGqKc)ebwE|EbX#o)yL*BGvmke z?SJFB^Ybmm^M_dQuT=|}+duBrN#iFmRFqMFH2kSJf7l7e)k=+@Au47X{@8NkuUIoM zr=|X3;z{n{92?$iPA50O@fH_8T#N6UtiwhK+?XFJBDUa24!5vm1?$H?nz?-<_41fMo$Zpj#tHGMOg&*I^%yN4HF5Ua%AKOM*v>f7_{ z_GWT2v4M$M@rYAh)yst~Q01Q-{J~uLyOPuDzszOK{=u!DY{FNYE#rT-7_g;b1KF{e z9$ONjcWyZG3n5Iicu*dy+I5Us80&fFD{?od1 z;>Q`9OrF9+amWAzruW%qCSl31R;PYc;}o&EUDLkMO^jy7=DBWjsDDI?h1rNh~j4?c{mL62;AzBOxYr@;;kpUzdvI1GDU4Q2gz?Pf!Y zjaYN1H)8wzDQvpgNZw=f*C~7FREo3B%=vHc=5Rhk|8UD3?3j#oy-dW#6C4{vIN!lP zxjmn^F@w!NaPgZNuH;V^x5@08_^5fNcxkgPQ&bN8agA)|{M>n*Q9`CzjXB8ef3LzF zPAcYVQ%V@^|K>A}A0KlbUo^$XJZ^K5w4doc9?Xr?2w-(8MZAHvG8<-V%dYXa;g<|H zW6l1KV%?9t;^wS%XW#qZWSWeov4_t(v2})pjFF*`ku$g|@JJlN=7ekWEw#Yvq&G7! zcFOZwIX!~Nf)3`iiv~Mw^K4GhZUZZ@w&lkcSu*45348ajJ{v!Dl8k3B&u-gcDA>1u zH9t+ehkG~Gix2(sk~wk3on1Q8juj+|SPN|__q;HJ+4D`vnk}BhluIXbsUv3c&O1$+ zm!3i%9X@h{7pt?o0^bUT@91%`**=X;I0dYAsVj5j!C^*g{cKjw+K2a@_>lSUDB!&sZ#+9MwZ<_($ntS@4C8TIjY&!=XF?iu*!j*Y z+4ZFZ`2DlgxEJ47veSx{`Q0kZm<>}Jxq_bc+>$HrxmO||=EzzT-gfAGrfxCIJ}|K3 z(mr2f3Kko(8|Sa&u3de=P3j-cR#6$-v2_8@T%M2c%%zU*3mQ#!5h6Qyp3%!Q%dq-?!e~RYv_s)#v_Z>XN zxQ5E}83yKjuH10eLmT)Ei+paPh6Zcd@5_cc8uQ%?{Mi&Q6*f5BjPL2GX2zF%<;I@Z z;I01ZaD8KGF$aWngnIafc0)d?QPJB&XenBAblo|?3inLB72qqHc9@g4kv(;IVw zDH{Eg`?}x<)4N5EU%6)hzjoz2CiMME?o!SOzHO=+Kgi8m##8m-4a@2U_O}zbg6$fD z01E@=>N}o0G)9}x4!4?9=Dxhy$1ym!I6?amn0}@+xN5gZN8x? zj@O)vu_0TXrNkL|IWyl|k*if6#b!>C*4tI8u_anfO#ApL?AV2rBY_iT>|R6mQFT8v zGt{1WGU&Y^Rxpz>uyNq7_fF=!@5%{U&oywB4}zJq-3n~P zQ)azu_dq6a-56G)UCTUOYsNUg$mX7O%;u{-9xy9@bTTLAZ{TD9G%`nrKjI{J2J-ro zOj+RvOE%`zFYeT3Gyb#V4E8|TPsVgY3wJec9Pe@DmV=tQHItli#qn8aKU0}Cgmu5B z!F8osa|;)YWXBFaD?TxH9y?~(&)SXmflb=YF}szTm``*wH+|POF}=5o>zEM54!NMm z=W5-mFZ(c+9sX||yI`{{0b4(r4T`m7_lrj`ox*Hxd7TA+Uu6#`OZap5w!dZcU7w0W zwf}I-BWG~q>>`;BJD+eGuU;|V$ZB@!<(~SVzRql`!y@)##0ow{R&y<#y15O$Yxtpl z1K9;jmN4Q)$V5D~m));DIIWvQ`Hxl8`Pvtn?C(Ph*;`xm1rt7NvR~}yvUQC-Cv8$? zcdQE*y!<+pZ*E)1EuEXp+#jjW1yvmv?0M}Apwx0o)}V4D@{F1{%#eX>>E0=P@&lH0YnaYi9aZ5|2Tb7ORi?5NQry^! z4eI>kv(EgTMTCFMEM@TgCGN+iovizYk>cu_Wo-CsM>Zzfio0+O#VY!4VnKBSQ^G9d z9cGSXx9wZbPd6RQH~y|eF3r$1wrd6cEB4O8d&W%Rd%ZXIrx$ zcDpc#UZycgF)P@h=O%n&vmrZe_D(k1tbz;bbd@m#-!kRR^0JL1$mvIp;jLQnX#Qi*@lilzU`6*|2zM^xZg>Ik9i%zO=)uGkM`y+{8&S^Mq^v*VKj`)7{u+|3P< z%rcK@eA1nv{9wUFc)^o-gF*UMhtxDE#+fqlcNx`G}d7C@9n`sA_A6^@o zm?!G&2|bn(c=)iBBeeOvWmK>yNW?okwR6S-J3e?q8AB$X=gPF!aG&09;`d)0$i{si z%%=}o!{5&S#XT`TC|EVfiJf&Zfd4Z}#_(`2W3*3o3Is{>1d0cMJz|_8hBPlWcdM9D zwG8C{=q_O!KlCtH+qUzMZpyLu<41}8;yoGP4wg0By_`Lz|Cak))Wwa>oXegWqR*bL zUBH~bdYjqat;?kPrf|MZs*I~AwOZ+&M<%(pWZ`^=d4jVGDUUP|oPDhabqJ(t;;`-ah|vf*br1YgQUA$1Qfe$JsPAG8aBNaHkWES&bcDOs-C-;MbA&^~LAx*pDenJgIpv4q6TD z+(!z0Y4~bZxzL;SD*qz5I474mecX*bbm=EkU^0UDeKMYx+hfZnXSj)zUudzf#;st& z+O}}!v0JzUw}p%(oTERFxtv!p~xzfhWnm}n^e-)PJ5{+^=qJ^ z`n~`AT=%~Bx#v0OJI*)u0XQ24W6<*{Y;&XIsNqlWPe#cC78!P339Xf^6M=`S6F z;|7_+dM<=!?6k%i^>OHXB$O$Rm&f_9hvS&jN?4|%%Ssk)P`!i@kAv{Yp9)YQXGjdLw)243mec(#_Ly=n^)TB@Y2y*66^#mT@d92RTiR z`?UA`ACUYrU zLD%Pzn6#^urk(c31FDwrDNvcBI(g~e`R{1FO9XZwPY2sWMHsPPqr}Vb&P$B5ZJYoyM0|To|^A4YZZx%bySG^H<dco&d(3U_p2?K%G^lIIlU6fwuR%;quQ+ERDbL+GDfwSX4t5`K=jzy3g7-Y zO^SL(+-#Hm{G4r$I439shsN5V^6sJ14TVZ@HPIe#kCajAly`LRxEp)+_7im;num_T zo}6)WIL1kn@VMgRER9CEIcb}dn z{{9N)y}LgOP7S6ggaNn zwS^A=w+2NVV>g6$i|*2eOZO@6up&hyyTkdACr~oSgDtj5tZAC1D%E-Sj|@m$zm0I1?=&t?$eu0UeiG!TjAgrL z?gzV#77QkiVf*hOMn?RSE|CvK+v^|cu9h{MlJ+Rx;LK)M1d)n{ zE>1h;$x2VGp;4c`*gr!RR$hv%AWw`@W7L`YjU%*W#sv^JRMXV{6R;q86{{>OgRP^L z*;v~??D_a3+%V6-yn(L=E4^mLo}B*<+njpi*}ubCZeuvRlPcWXx@LGtF@q*9>xTov ztoS6SYrZ3mdwV^f_Qqf$peeqjc|i@1)Upt zgZR0*urufo%{t#(+OVJ>Ten8evoItFe#vdX-Sulhq@+XBKP<+z8)D&nuL*RL^iXc_ zd9Y%CLAP9!U)rlbriQ41w$(GLFxC)hHi_A&X=yNR?j9(fugwPdjANU`{c*qQSb9+% zhgt1P+_Bt;q;0K+Urro^GvA(r-VHHx<4gIqyPBkYs5}1MI!a{Z-V6KGH3B#HB3!9b zhlo29@L#mB>^7$nP8mkxAd4KhW~vVB{56?!?-=+pU>yDUdW=>WB(Yg1l5k$`POp{6 zDOx;%to3`HLX9_&k*2|Y*<2*O^=+kBHxmcL9lKbNwk4-GI zekpz^wSuXt>)4vNH|gr!cf18RmCHZ!6auq znedVFQ}%-ERYjOL>H&BJl|XP`bBc;_q|gh8sXO)y$zE^a7F5rs);Yk^Kbzs}_$t_( z^_^arm{6+kEN<4Q1N5fvC0LShl@$DHpW$cddh>T|beA z#1YcM-jG7|e(AVdf8d(&WM=nQ4_!y7V#zvrygVj|-BTTl;}cHPvj@fyEOUp<*?z1k z^8ueaeKdP>5b@TY=TJ6$C1-lV5O)s0C)&1s5dQ11Wb*MZA!dRBuY16nBp)l_TJXJQuqf5?Me|f3((KMN57svrV-FaBBWeICOJ2wFr5ZgWqo;g|IA|RW*S1 z+xr=&SzU%qEd?AX?bx- z-X~M)yJ9_T2{wfEhdON7g(Nzna|EKr1vLAeGkRN%p!<(M@Q;SbWB$c7YVY%q%x@~N z{Q|;NhsNQjx<9}u&<{&e%}#@SGMz1`ZgZ>kF{arE29}VUj?WA8-c+t zGud6QY!Z8U;D*sgY+#2Sx;O^HWx4*iGeC=-*gF_+?M`M@1AMu-&HoAf>jhf;{RwpT zAH-5Xf$c7hq5OFPETI{_vOf)AEqZT>d&>BTS1;)P=fAXmXB@^am(lc@@xa+BOWR49 z>4s0kzD7M<2>RmZ1PeT3rHHa<@0!ONib}2=h9HATKIGpdzG89)T(59oGqav^$vf?^ z&Sxb1R%(Xr*1ssTeISboQO1|Uj?=h<1z>5K%2)d*QBzbObWmGQH+(uMG+3MZ+fTz! z!<)hJLMpW0()FqwcYqphj}VSRev5ClG~uKJw6XkfHcdLvTQn%G z460K?G3)6SK4rySa55Q;`ZsLQt$Lv7ob>~+-t~#wI7633-O=Te>}IlWZ(J}ldp5)r zf8@NDPXTX}$#CpzB{}}JK*cUKI#hU!&fA}(o!3uupX@c*f*a=a)83tF&RR{?-Pv5S zy9;{}bB_O%t;AZgvZec`Sz~%nFE9fimK!b1XDZrE`IZEq=#;c^$ZrrABYh0T> zosC{6-aUAfSO|)S8@ku zt5W0+F_svNrsy?ys4q7b*Pq_Rr8V`#V`nzQlb>$vqkJl*8z1KE7W85gmqN<^lT{GeQ6HkyobW$n?lC! zj5v1bP8FL~GhD2ObG8W2J2(=K{GG;|sd$5h%#hi=NClherO+ba z{#$3+GrMv-COtG1Rtg!Y+gB^%1Cx9*Xx>7vG-_d`n+x;){FT-?dSjD%CMAvWMwR5D zIP1I>yC*7w@`~@IXZw&IU6(_n@p~wXdkHSFd#U#1H+XEIjkoPiz#m`0yoUSW5ZcVS zPnm#S5_5k43=iD$HcK?hUf>3tPQaAIwsh;k5k6b9790lT!H^4^p(SQKa4voZ~`97-6;Mt6I{ zk3H+)^_kr;t)u}8PpjkY;R082J%#p32eX~0O&Rmgz_ksdV1VO(8b4to?sK{cj#h^3 zQoIu`zI+6#j~{}M5r?Q(rxILSA*0Zi(KO6=6d2wsrkqa*4xbFLX1AEqOuAv^z$^$@ zy$8*AE?@(LGa)x^JWd$lh_`;FQiJP6A=h*hq#xb~=jUls+QeLV`er%{&ArdvH&n($ zf}ZAzoehd#L@{6EF?3>p0o&KA&-?^j`tqQyuqRIjlkzwww}{}?FkP0o?lzS_z6CLw z!K^dG2abL4V;B3%vyLxY;p*~MNLU?!W{ZZf+Q2&Mr_mp8@03Y*RoOB3pN}BCHkse> z%?mE}%7V5WWn8)Z3%62tE{zKcK=-T~DtbSUJ1so_w(3%PUi5`qV{d{#HvWa+y!o)P zS_YRx(&@irI;{6e4|Fi7fRBZ9VOPv(Ebdp96`CMAL#1UVmrSc7~>)^=yEmUab4(Apq z!d^XhmJF6mb>UuV?ZgN${X7|mAKOk#w>{xrKGQ?#H#7cOY9hOpPuORAWv5MKMcXPvI?$PpdDWK>&fToy_Ww$$3@a^VH zd}X~k`|EN5&i#oar8!<;|Kuzmc+s52OdW(@J0+y+)4|(LP=|}LUNpG360WtK{GevuTVEPyO;u|GoL&tG$<8M&XP!~*HnaMqE-pH-V(`TOPMNsfBoLm<7V%LAm zGmCT))U|1F@VbG9#cZTYn?%?*HW6BT37U;VTDZixl&rt?<((Hez_kI+sD5*iG_K^boW9;bz(o3|co-TayS|C-_a@K$IzZNY|Y9|E0^hvON)LvS!-AhxVG#q*<9F_lve zq+k>Y3lzuWh9%bQ&v`}Mj7hL`DplFL; zDW_a&CY6K-;3|D zG=i0XHpasp|KPe;x#<3-sgwl+anbWVG&FTAcIjvFeZp0lQH(89DEkj|0*=r!g+#L6 zB4Qhl`r-K25B#bN66QVU2E`;Kz~MA~0qaQS{|+0%hW2WJXg6Q>QeBOWy--0GSzlr4 z(oR~q^%I>bn!`*S(>UGB_H56_V-(PxLpqOaS;w|VoZ79m%=}#s%?$BG)BARCW>^{5 zKG2nBa3}k=T`jFS`RNgJAjz%o&q-0eSNS?PIoXck6xL+T@ z%BvKH^>aq;XZ~zq->Fm|vIykzy6NY*PAb;dLeKljpjOaIHl;^JdmrqVe*9oaFXKG$ z>L*VYzQ#nrM|9b<)NyP>O$u8w-h}-z9KrfmFCmAc)7ik1ez-6m*8Z| zW?KCaYGWQHpWj8Vjo)(*?8mcQPb(IduZXWMdDHSkftcf_j+!G}S;|-kSn%o%t?M`g z%MbdX&c~hP8QdRFIrhQDI|t*GAYBOC_KkMDiUs$PL)BlFYO;B0I*mb`+n%J(G9aLm8hSdlq;!~@N zeV;&>`YnjtIF(N3hhmubOx7^14Q~D&B=ldljO?#VW9qZuy1pg*BisX{RF~1EcTMnb zNT+DdvnUP@*ND!%vtTjPckm4>O|iSPn=5k*;69)67uN7tv!R}wu-l-MZdV1;8>@IM zI--dedXFWqPhH%}6*chtl`U)$7H1D0RSN&DtmUseZ^jC<7U*MuxSLbLpbOVv-{TM# z>v5f;CiLT8G^^9`sEhoT!eV%RyoHN5wL-UnKlyi?wK3$C7&A`C(6#p-%=qUM?rzZ+ zsdY9%ir5hzUYUf;G7MSb;=XLMxh?*8eh7;;aAc4DA3-0t6mC((8gle@#j+_@blSOu z;#6EO*0?Nix`ZONDs}=gPKtI7o&*p}{U0RIGOy5+1w(x%Rtcw7>|G zrz;Sa68rpbcqI1s<)*rv!b`n_o%?Nw98r$cJ_@4f=||Mo8Uz3ian z^F(y=V>?Y&{|hsVAHlIt7VOUKFg!V@mINIL`+oBn{Btp)%U}O+mTGI^?>< zeXwuKU8-!Fis6UugJs<@`u?C24m)>IzGxYJouLg{?@ZXZCEiRo+ZIz~ec(V@3gp%l z(T9UaNv~%VzHT~8=XPk5e%*NV+4_*I7ro_H>nO0TjuUC{$tAEdAqT$g$>*dWmQtpG zVdy@|gMA~8gY2?Bw>GB*-pVR$c<0-kZqYth>s!Xm7#9=p`uDRb^G; zDt@p}06q{fq2}{r8GrK>uNE>CUxn6E-+ylSx#TXdX_ZUYt8bG_cq-LSAhPwXqlASf z?Ed3%=qpPD1N9g-t$HoIiW@4ezQMaV3VNu=@u-%Qhy$BV*)nZ8 z`g0-$3-nrP&w7HRA6!xGvpw5qP|eq^S%_J7x@a=wEq&L}WIY-pcFbDDrru8D|GcT9 zCHXhVIeQj=GPgnW=3W{*C{e@Wd^NHtrJ&O%e!)rHGz-vP^`!h8C zOu45At6mlkcQ1c~(y^)F48tM1pcR%hHN)aHndJB|kG?D!%-oKMSg-oMFymY~?A`tg zX0&JmKUvt}yJb8(HPVT;O`CxCOrlxe%_cZwN-&e#Zp*sfeIw6HA>iLzpY5*v!71A^ z(0bj<{|NJCkD3mO0s}12H~A)iGkrPq2)aVsL5tbN4kuirzny>gD2P=ztbpwwQlR`9 z3iU3AdNSi^T*P{K%$c$Wkpobp&n5aLHDr~sw#*%ubGaECsNtFcr+Y;M*B2_Yb75SQCXV+E!hj!<%xRSzoQ%q(+VzE^iPue8LPcLRYQPJ85^XY}z?uHXO^JOpQxDnTf6~R){;eS4ZM$`gV1E_`)8KI@^KH(++NT ze_srGW6Sn<2ji~sBd~emROaVCj}4s?K=P}7*{a}f+B0f8YAG#eU8etF$QeJ3zk;GJ zA7!ds_JyWRN#_^XjYav{0o=uvfv6R(fo8L9arDb3T3c(uJ-Lz#=YRD^ztdOAsVx9^ zoRNnPw?|ZLu0(0+O|Uq}i^Zlr2aR`nZ1fISJhcB6UAl9GJAW<>QWYXt<=lKu^?5ev z$-jhDJHXNyd$vM#BkVqt&)@qz4Sw?GoQ9<>^FDPPR&_3=yD~ji$Yt;kLd*EPPIFf8 zJdYA$ma!qn9`K2y;z`ea4ZEPM3hpuGH2YXTb`oF1Y9FCK|I32#+XL{wiONu9JVD?P zltfP2{)|}#;l6X{p6;6wON3vxz ztEAqO%sHzO$0$cjhnwkHN10w`?8Ml=kmlinmcdSJ(5s)+;TTB6XYYsJy&c&4=EGjr zZbR@&`+CZCGoU|x6Wg!or?|#b+@$bzvSvG?c*8vk`QsV>>NqeZ-<_jqQ zYtOzZ&%yA1cX(FW4NH$&qtA64e0M?2&%9Pdw^k*A+3)ZC`qw4!d_xtldOnofu*#Ly z<}2ZK(PBD$tr0pDr?G8k5;1wUJ>5%8W);J0$oZAP`z{^IZvWX%{lqW9yYV^y$Fd)} zEb0$;<3GXS*=}sWAa!W=jYN&>^`hwi)`OF|Hr^N-N(NEQVCXcHW+XV_+2PZf{R4S6 zzA=FL1p{m0x6l(Vm_;jf!T#90^r>PSm$fvK4c_XDnM1vq>uL=MnjVJgBV1Ud!0mst z9)n-6Ix!y)H%z5W60I$xi0nYD-I&e^5ug~JX2Q6>u`o5-UfOcii+wRSl2$Gq$cji9OY7rl+uOfz&n+L)_Svx=JFmc{Z{=K1!a+FdY{RAneuMXxo2Vnij(%M2 zhe5T5pCjUW^v`in;uG{*!D@B+6o27uQ%L4G#;5kVDD!I=299k=At2`<(MpBI1Ng9W_3(A=26)=DjRIzGhqkSrH2IMp zGwoN!#gZfG9zH{2={V`qP&+hpAH$qpeuMco2H<<{8yFQUA>X3I9h%9whnf#zgo!B{ zbpL>@FFx@eFU+vxOFPYcKasvJPk>wJtXS6?7f?y)hewZ0p!GICV9paeyb;g{lV-bc zn=gK$DNcj2r8$@04nGai6JAkbKtAv1+6P}){N$x+LvTn;KQ`d+B{;H55z6)ZvY7H8 zaAH#fg=AOJUylQzBH-{n=L6aIj(FHUb}+l_UCamdje(}~<0z@0JbQbng!{5no!v-M zVCRZr*`r_E;PdMgFkftl6IOl!kMIogG}sIwc}+B}XA&wJkH(8{R?&}1%FuQCIr#q& zvlRV8_`Eli;?5Z3W7W^lt}q?*y&nLK=?6(EkwPDz4PhYz*m!em7HU&LkpUh|b&e(7 zjn-yY(`K-!_V?08w+62`lehFjdWn~+Ov2C}eX7im!85rjxM;OMyZKff-@ht^aVAbo z+P$271uS2*{}V0y(ifL`t8-a>hf&Ag29aCjdKlR`nYoSH&&RX*%yij&*wV`^URg8Dvd=!Us77x+{gM$|`ww91Vw7@ZHp`^uq0#t>$cq zj}-K?tA)B{0RNV$(cIfeMXSG2|Hs-49-i>}GfvE;>CNCVZ!wOXXT~kG;wrpU5S_fd6?mxQYc;7~#e({n?A)Thv(C3$1m1*o01I*3v@T>lu-ufZ^{_ z&4rsgT(IHJcK%r1G2Zl;0ylAPDz7r8mA5PJgMD%usi(Y&EFNwFLzj91i|s?RU*4h= zZ6kQ~Ug#atMEK2nFv>Fhc-K2QeA}K8xN!Ld{POk)Ij&rU&T@M|ZH6}!t9#-%EE6=w z(^#+nGy&6A@}VY%$t&y)l_T3WZrp{)o4gHw4fj)Js_%kQF z<6du5#$$2yL{{c>33fgngC>O+VZuvA_F#S>`z+|-BKxGlr~a4d+no$DF1<`gZzQr6 z%5UIDvo9+@yNbz*bNM6leNeMf9zQ-;WBZ<8hLF9x$YOyB|MBZb*t`|l@6}sr^0-ly z;5Le>#0_EAr#J+m)*szBhW)UOMrLizF3r6Ld)*U+-pvGC?uF7`;{#-V!y4O`d;_y_ zW&9gQBijAm4W3uWV(#VFq%+_YWF8PS7r$b#)W;AeHqIgor#Kw3?E=_zCDV?caE9TD zkR4tPTl{-r&ATboC}^UiZZ*^S$BEpRi|&l=1$ z9@Uqvj+%-eo2TJZp}#)WdWz4fH^zMfA42FXC1ipw((2?5=q+Hkw}RI5C*Iq!V5=c; zB0-CpeKLY`H|N9e+-TP5tc79jPf558xHoNPIMC7xWBqDrMZE{srb#%X8S%7l@iH*_ zWCDY?*)g>Pv$1+}ENePBMi8pS;u%FL6p2h|O~g99Q&|kZdnBmq0_;|oFSk~89QOX@ z&Q^MskZ;~_3f1q=M49%enKzJ^*S<;y!f*3Hv?CtZyi9++EZ8zzT@=Z!f&Cv}P}AQm z+IhzuF3Gi1+!#4_H?I%7*Y=3&EQUgJ$V-^mHjh)-r;T&|OyW&)-ooPksgT8|&=3Db zY}9XC(iio`^@o?So;UwN`%5iWIJlK-Nc_ZUW@M7vqb=|qUemW}-@v1eF)C3>4MD7o~+_%8aZt*l&)Ak)@%2@ouoJm*c|Coa;Vm% zVQIryh|VIA{#Q>6s>kE`l4W3e+Zxig{pB{gPeJG!!2gTh3%&9O($`C7c>cH!%gNP( zJlo4qZ|}{oG!1Uk-J$QW?_PH-(tA=~y z-q0f28k30gwlt96gO7q%AQH0Pl+gIkm+8=-Jh;YZ(3pYU(ALrhCI*LV8hrBL-6lJB zV~h`roPV3&!ylw;4g%lP@s%?7I^q12H#yzaEP_#D>P2F zg%QfMvU%LSbbqG$^Ep{LZ6mqoQwPTYbY;jmP@9xe~a0Uv?Gn*7NK z|Csb)w^qfVOhKI;&Ts+sOcz{feUeKE{|Sq~9RU}QZaBNqfF)iBc3x9}rB2%N;|-j# zxOX`fP3Z?&<7Ch4+5YV{Dbu7XD^ zU3Z`H!xa@uCVd!Wv+xD z%DkPUnE#3qD7!3pjv52mtw)2{P;U|2-?9m=Muf5)rQuxhT1|XexTz-Yo-!^f`OWF& zjwN61V*2T|69Nxc!~WH`U`Y6A+)$^;d=iW?@M{xfPYlNP(QBc}eH5Em;DePj&DiIg zjWjZ%jGMLRA~nwb4*t$=AR1>PU8i=1w<$a#U1DTOi}s1w!)hhLL*fYYPIc4V^#N?} z)#ngh1dN-SPlxO5S-i$6K50-d`g_ulvf0tm^MXbmy(L*qvyi8i+4d+ zAsSt6+UdX7YNTqjk)ItjiPeubWdmN%rSM;&OvT(773_jovZ*!{eh*=BA03$1c5Ae_ zHICou7zcT+G5Cug!jy;oql(Axsbf(Z?79+w%Y@n&dSe(a?CpvDLaxD(A$Le2p@pk* z84SbxCn78#!O}hBL=VlIV5@=+-VarVeUa}WL{EfctKzY*pb_+MGQfjhkJHmB$RlX?_dYfBd^8O@0=f!9k_T&!f{inq)<@uqGptp9N zm;wHhSlU81F#LuG%iR4`7zgyRGq8=0ZLmd)Gp}J`i95cu(PI_HFJOpFoqgXCi__(1 zv9Du{u&B2uYutDVN&=RW^|=4I`XNJD%$%oWpJj``AB+`UJYj(y0)Nno4(vv?5iTfo zrwA{gy+s;)$t!uRExqPdb$797pPt~G=)6NuPq)*Rjo$GPlgMfTyTGOlk;7e%GIvMZ)nDQ|r` z>{fkG(%1+zZ&yIm#hPsX{5z!FSqa8+Jk&e*QE}E0l2!cY=~|}6BHL7P$E83tnrSHQ zRE@%o9p_-_F-45{^?@?93*dWlIt;D4L7umT@99EswkWioZu&2!9N!CY-xH|Ib^{c4 z-sRVfE#M+F3c)j?6Ik)R$ zuz+L#<$4xtFq7M-s7*1D83iwczNwFBQn3hiO|zk?GY^L5#Gy*79EN1JesfK<_Ueneb^Lfjq-O#QdQ-1*fp>WxIbm^mU?CFGRq8mjZ6`=CfSU z@N8+Upz~F~^oR^2OmJ&i7U<@>&>+_`IG}IO0r+vwFjnxXl&`G$3fm6oGMr{Wlaoi{x)U1wkNnM$_#pxH{7-NL zo}2;Ivc`#EX%FwcuxSWRxz4dnb8v9*@1*m>E20U7JS!TYr#sQ5!V<_9U|l6G-o;ACsi` zVail(Ow~;ytrOia+UXHI^%*K)@p*8>&;lHK3c#Vnn7J<2!$mu^S%mLK_LZ~aZ5`ZU z@kMpEVyrusyqrw|Q(Hyy7WSxG^&gcT5|PFabJiRe#q{c%!1rhj3Rzfex{@!x5WFoK z+tSE&-YahP;8beyo=#7i1F?l$OG)phb0*8Z@rCIZs&g)+>R1Upe=q#c7m9S)*^Q){ z@}OEairfV*Z)|f{%`O4&D7&wV4yR91ZG#?$Zyf}SivEMyGncY8TU^o9Wi)HCcVfCh zu{2xP6Q^uAEs9fWmzt%`pa&}XB3sMvT=VWvv~+9$TkrK0R;<-#+ahy#*FApxEx!T? za4My%Sr*j&Rsk*Bm8tzoru5gzLAdW`Gr!Jv8~is(O3kAb(cyk6_*LuC^g1JW5pTzG zzwd)N)`6s0I9;l=eK*)2FvjT9g?y**cR4n7kR{;xM$Ox4T(KqlGFuT(gg>FE23@d- ze#VD(K7v!idb8lRD|DjTJVUm%4!mByhYOrxO*IF@&~3OG%Mmz~dzJ#`GQA$wZcJbelZPW68wtO(`*8XJo|qIZ z!45^fW^w9l{PbcB+c9$kZxI;=-!!Cjyl^j!2^of$n}1Wu_}d`#N_6LaI16@CLc?LF z$@bT5Hq81PJ=ppPENTj*1!kq(AiW@{d#;VEB9n1MK?t+&%;Ig2{}FKe^_;o-b~3p) zg*{0;3&SrBX7T&Bq=mUh@5M}H z-7q9yof%))OafTD*v z;j`1aH1J3YbZ-73sxNv-Rz99JvBxB|e7*`(NHS;lc8`OL6K+uYvjwERJsYBzT!!cT z8cMLUBG;ZmzP;-NaZ9x@pe%^}ILD~b^PF^*%X3;M^tfv}`l3NwHPl`6#2MY1ko(e( z?d4YT8D~YT*mx)ZP2d6>6a47+uuM`;dQH#$&OnUIL()A`ODR)aaJdj_Ddd$yW6E0I zd-@HS)j0&ywQo~=CgM@48%dJ;;Fx!(=}20C)H-^foTEnLjQIKN;FCz)@4S}^M@H}q z{+{R5AKAb}^$=EUXov1+^eA7ZMJBH_VAjI%xNw}H^E0r4@zrN2@%vJInf#ZN-+UT= zd~v~RBXw~8wO^FF#SSlf6#%Pl7m@c~*x3HWOVW6oXXWMe=~o9|iO0eG+#pt9GW5I0B zPP`!Oeo9&MnyzjaaO6>LXr3l`OqCkowb5iSykw1iyRN~`>Qgk%&=q@BocM)>Z$PwQ z0DARa%$ZNwOcVaIz-y1Mll$i#^!d?Nc(6hnbCvA*rHhnUM)qB36Wg%I)9#Y~c2}G? zaUP=snyjk23Kp14MX~HU)%Cv*m$8*bo~+?i8&5-?(sc-(SuZ-$co~Kr|3~}J%HfyE zns8u46HGJI<{#U9fIjR!AMjQaHVr!iX9mB3rL$g8#x+G8@O2G;z)kQ8-8DjKtRla3 z_jW$rxQoj@8^DfiTME1G9D*7_2eZ%Bor1TjV#Fv9W_Nl5##_gdB=tG$J|2w2fQ_6m1Gr=zk%oLC z`zP%G+4!Urw7BIsERvbAnzc9RNPQw^I6Gnft69upR~y9Ul+w=G1+?R_Avs<@3GdQI z@H$VUq}Fpzg3fbS+z?^RO`f=jLY2fAbXNEtJ1$XQUk$c){z%MO{1E=Wdd4q#*qaVs z1ZbUnkQ62w(I0l6_E_|TDSswo%Bd%uOsfS-YM*dl{_UV!1xf7YPGj~+Q<&u+j7PI2 z;cR<7uq!{8p#Q!~ny)K`@rmUWTJFW3oBK1a{0rPy@`Y{7+}Ms*8+K{44zAnZO^Z+U zrbn3#qJ3u@$?Mm5>fhA^;(1}RAaj4otDysA@xl)G*4S<0caPrTuNk{!;$;%?GON0} z!chehugp|QNW(DiQM-;yREE2Ij|x` z7B@_rA@e#oh(!(&@~4Nk$~=75h?CMvWb1b~dY}2+P`BD?kL1mdK(W=4wP+i0QGBvr zibVfzh%9H8OKpRlw`9j81xatE-{OmrCnY;OCrbR@J+7-BH%nsOF0FGHKbF*ci)FX% z2FgNB21@L96w8DlL)ph(`y>Y^?~*0`*etX3)|Vw_W=hr`-Ym2GeopfE%xu}ZGhwwx zy+(Tv@~^5(YI`F)I;*CxbxMHDc=NitO*QS_s1Yk(ek@PYe|xld^xXC0ihsR*QiUus zX~iM&Cgo$b-s<*}$?ZlyzuqkK_K3PF^Q+t{{^j#rmf5evK+0c zGJfuMiAT_w+Jfa8J~@URvU^Hj>jt$s*WC*GCtg46xa`5Qf3*)fP3yK>z7Tu<>mwUA z?6_q6c|XYxJNd*xH5Z2mda^5O6FY15=Zy4ku2ujWnWYdiu*W+$s&H9kw8?gSgTRT z$2>YtW|4kIW|BQheD2E7ogP**AJByU)ET5J<|RiY!?CQFYvERh>>K@uwD1+EUvm2@8zo!nEG~Se_~>5}R~jv@J@!f2C);JVWIZgB)H>)&JXhpO zwjDm;eROSsr2Ju~*vVv7UHjuQSybzTy0s%6CD~US>r_`PmhCFIRVO$9Ox>{eC_B{|F|v;-+az9^<0RgvK(?%4i}=nK zCGU)*Es}nPF_QCY8)V-jW{FE5{Pu<`hb7iaQ)O#cN7SidRoymcRf*-i9kN52nb;WwWww%D@>F;9 zbbw4(s5d*Fq|3IXxt=Zld{~x0dY~*_O|Q-U zL}5?Ps_MA~YNS2Cj z*SQVOk*zq=E&FtIhD>qsW{G;~c8RrtwxqjnRNd+|1M5s4?*@r~@4EaWpJdn5^(9j~ zSIc}?FB4zUzA5XU)JuHf%n$K_HFIh6_Yhf@T(bCVPk%{%XTJE0bf;{dP)mE7{KY-4 z$7H{2ndDTux~zGcz3ic{nj~VLwWMmHPu(2vc3D{YbBWL1gW{$=1>ma;9UGGzZV zbRF(gwqe}L$cU6timXV4l;nNxlMm5>$jKKDsNB1BRY z(xT-t*L&{q8~5+YsS$#EwoxaU2d4GbC+UtxH(qY;bvn!JE(X?qqlYJ+ zK!S5GjY+>pMGfE4YDY>R{TU?shh5R>$tc-zVLzJL#F0;R_aNyJi*cD7LF;iwEU!tpXvRUoHa5`tO2zMR}Wdo9BxYw#{LE)hf=o!YqqNqU3 zNe-gUVTM%bb0IDgT0u0kRY-qP8FcU40v1nwVbR$hGXJIs>sGUnE)-aVebTrr<(ljC{Of)IpS{Q(k@X#12%>T<>s z6vc!%dc~oY3W1AB-kv9LCuSAIE#c+Fi%7B43(nzt6BWAO?hov8@rHS^cgW$`U(`S7 z9^0J|2tIP{WJAm%lnk8?dIu1)l6j^}y#ydHYk=QzQ)OSaH;i>Ea`v2fikkb9sQ8kLullxLdbKgw{o>7Q7wO|8z5AXa14ghg}$F$`A4uUCe&-R3i9E zn{23Vg|%lhaPnRaQJGDNS)v=Z{E#qp+h|LBdA-^#`vThhjK$_GOPHQ<4Ug}NCWo4; zpsvb*jCJk6n({z$wI>EL1d1`=CYTKn`H#JyV}J(PBIMzsFt`?^iD#sFS!5T~pkTCy zKH}v)hL1CjiQl}p;|#IQxKJbn&wAfM=1npdd24aH>~~kJ zQ{m;s2pZ7hA0^naGzC|R*1_J{f%FhJggKz4O%vuf(GLRcP&fG(oYk60f>#-}#;oZz7$MU3anu-7-&VrYd19onK!{iS zW*{?D1DqaYQ)un@pJnswh+M` z(r6|V#S~=K)03-uVf8v~cuR=)4MOkvKd!Xgu*3JXtMk;F8}BRCl9AFsg4Tp&#V~dpPpb>8;e8Rf~h8VriO#a zP!Ozp7{xmmaRPQY>tRpNGwe#eOPEMc2%K(7HB|19c*m=-(V9bTz8YZjur2A+l83R9 zb68R-f%|v0QuXI^afk8;Q|Y^M-0`k|)M|4dEWES={u>&mUhtKSbkBlnp9r*DT10LZ zkI`k74``c%GALF|(&dWbjG#px*?ej(dF!W$Vmwpz~`a>BocN)pVd6Bz`+$OXgI|$fx44)i{!fA#qG5y(tS65_!d}$S{^QDDG*`(r|gN=1jaQ$o(JEd3wQ@a)5ys;!VDX@dSHZfzBgDEQ=55!3e;6chywv0re zT=_E?y3LPHCks*UmL9fQw~}qI_%QO~9+V8WA^R|hN?FQtMY@)79;j}|r!&N8%_d14 zJ|{sxy~{)C#W5J{H~`XhR^-qROWdB;fHM1P;lt!y7@q#mw0=C$bkEyhe5T(=4+Vb5 zIh_q~Y3+5|zH5X^KH=bw_AK1|n)il|)l7YmD?Ac@2YKqz*!D*QilL8M#4f;%;d|gj zMK?bGbd2;&3UUO7${_x4JY+d9Mrk1}qRE@R+|C5T(KAsbak!K=b#&kYyYG0Tn3uC@ z&!vqzogjDQA$-mqV!kI^f`3LP#B=8mp_pRSJeo#ScBet`$9mi^xs!VOF2=>YT&$S; z?zpsMJ{?~-7aOZ?(-+g~*js;Zp|r0g4YZbnZh3RAn+YH4e|-n%MV^C}zXybeo*^yD zwxBIp2SS$KaO#N#6mLwV$0xIK@6q{i`_VD*a%n^_KO07UNg+J(<$XgDskl9|18JuK zx5O@-NckCXrn_CGj`=geCQS*JL>$NENAlSnr!>&(P8Z1<%4OwB)i}?u=|XF@H#7=7 zf$BZMX#HLkM)dpf!@8+tWsNAtS2j?0MMrpfS{#Z>`l$2f7j%lg6zK7{(chW#iQh;$ z5ufTuj?Pm>@@YAhz9|U&vV0te8Y!avoMJ-RS-7+OIivNp5;kWRV|j%?76q-NY1M{s z!dM;EOZ2J8uQdE2vVb$}(n_sL^O*Aw=5x;my@jvFay0)2A6I(C40@lwAz?kGaFxFS zY?Ni`t$F+DnvNBizGXGn`oJ`J_|g^*s-zR!DYvOppgJDUOd&pb^7L{@4;kL&MHPny zSR*r0z(j$HeQmNFdqIDk*>wa@^QnUE8)aH8xEt(_^3nbCJgG-h94)X{1}~}gRBY)x z@-pEi*|+yNy|sQ5oZL`~%OsA%7u_-P)^Z_MOxsU9xU(=*Srlg8e*!uQ`E>5pyQaH# zrC?|v?@;T85PY`jDqVWr2{KZWaIZuR$|WV!L_JCF<}Y2$EqWgxI%h%J5*89J1`yK} zD|+Pp3$jG>5zJBH;~Xn%geU)Yg8lnEIBT?pUJ_CuN`JPHoWnA>U~r6vj{4%5>|VGd z_6lJlhmBtUn@lzv(IeiEsa9wqe9N=Wg0xOcpfEcmn(7FS+jeeNY=2rpm0YSlN~b(bHL+@hA7IC809 zwiRb!&oF)2)rAIr!_@5j9aMYPjz(`!(Ea11H01CmI(-_K9-p5ER^R!!H-t~aOVwsP z|7aVw4f%qX%uiecrN|P20U^=l6>%Y;&Sk)>tATq6Gw}4pQ*CdCd$)=;2f`1RzyIJ{dVF4PQN$@JN`vM z#($v@_^Ao67*&FAMl89Jdz4ZeXV5uSjw@;{ahm53`f+zZX%bNe`L$QE{Ju2a3F;*H zB^=Siisb%X3$gG0L0noDmgsD!dn=lW?7LhD8<)e1hJ%VC&VHe&` z0pBfo#IiPqt$(-#JnG!Y>X#L4SgI~-C?tsnFY?%A*%Ta+@x_!y<@h0^jRskthl-K2 zz_q3ryjTw(#a)7+!2OtDq{NZEehbffm=TfNon+9>6?N|kz^^@}bn=crHJdm_re%zh zD78z-uN(~%ElwaEGYiZ%9b>QW>Vi100$9VFV^%pZbnUx%_@~+gQtvy-Ma~lF`=o?> z1CzjW^JCPw`H(JOABLS(CMeo81Zq7UBsF_JrI)@{_9{Q2ul<3YUM&Uk^9HH+^C|c_ zAPLQ0nNy3+fe^XmB5se9W5#ZU;J4hvFn;w0i2DZ+BePU`u(XCQnQ4VWhf+~&ObGOE z>e1Kf!|3v-k1lo020`6H5bhVpzf1aveb_M+-ed&^5vORV>L#2gX^iAqAkN(;fVoOX zAvwzh=bZh{s>ZUo{&Nuiy`###tE*k<`=ymunoF{>6}+6?#CcdFVL{h!n?mI*YauNt zmmY5|r`9POaVSlhYofcE>ok85e@6XB*YmOr#ku`(??W1<@MUAs`peMv`whG4zw6NK zc^;DX_K>b+&9H6lB@Db?4l_ox$f(5{kgr?}vJtB|;@?H#Xl4@h)c>RFo%f-#`ywXb z89{x^-NfyzGQN2=$msQX(T4+q3$8P(iLnEdW6T>qhRMc23kI! z>Ad0;^vtlqdH+@7W{CzkkoysTSLI-1Zz6u(O6ZCWTXA$|8MySw(?bF?aN>vzc`a`b z?}fM1wKH;w_nImy{%er+)JlL8@usM0zKP~drV+0UWvrW8fbS~K!$^!GjvfML@K6do z(duI=?;4RHn=z7CuZZ$|?EtIyFjH^N1}&v1toc}qG6Nd;D^eMsdc47F2h2fY(vWo$ zA7`I>@`iq*OB5G&q2)shY|%>vtrIaYyDbtOPWO@i@Q1)VxXTUSagV;lIJ|C`f`@wE z(Z%2ALB^dIa5{kxI=`KTqk8|*$2+c*B)ceRzF$ww!=N>_ z*KmC3In#Eb+7x0M*(pc(xz#88X^oZu)<<8iv~oDdzE$=k+p8G3e8vst&RM`U>lTB7 zVqR|e8(~;;{yfnVJPL&eM^UgpiDuDzF!bUDwV$yLb{H8^iwpInBp{yD|0l$jE{Vs= z=UeFeN73*+w}`I3C<|uZ^U?6~5}4!S4m;@ z&lR$IeJjx%oJn7PjHZF{-lXUAIrQU7)205hoE?UK5F9;0O*Q#2>0mx4zt?16ZU0T3 zlFdjp8wrMYN9Y1IN0__Cg*_wu9G;{{n$EpbPVAS@N5LqE%9n4Fz^rHu%C3C^VvhUi z)7W7gbJ~j0d!$+U<6}E*2D8|uSeh5VT%0XxPcCyek6l&);Qd8bs(sle9 z`*e2+to-zmUa4=!3xXjiCpZU})+mv{JR9)v^MeYh+WWE_2SD&f6I^(71gi5tfV$un znkpZUvsV=m`4!V3*Lfp3WwICYCl#SS%bN_CtVYG(2hnYb26MAF8I~>yg0Vjq7@EF} z{P$-YnL1%aQz9->6~}*U`^ZA>&B!9S%)b~9uc=_!TJ+CXgd1!Bkgjt=xPjs0*7R>D|DYRe!luK|h{uqAe>YZ(^{}9H zioP}f!i+5s2B~Ml*t|-b>(a27&X?#V*VPI!Hs}!D;5J0E7QSR9mho}-9V>wN-09$| z+yM$tSX#IFJ^d4IMpm~!M7yG5+G4$$&R%>MJ{0<3+2S1B7CnaVe%a#8)^PZ$CdL(T znhG{iWw5Pw3w4)Q;(kk4!`K!PZtnF1$nwQv@<0#FiXWz_aFIMPzl1B~rg6(V_drJC zZ}Pavp60$|iB*vZKAAj^)y>DjyfPNUCPP?Xp9U!EJ<1%dPQ%IiG}i8D2sStH!^xMc zm^kwhB2PV0cf~An+EWQ0r={X|*LHemXe+$^Qi|H@i*ehrNg}^%AxxtSpya)psrh<$ z>O6RtIriHNRqchjl6H5{Z22ObO;S!Rq=FT~>Wtwpf-{EhoNW|@PqXf&2Qm*!lR^#jSV z2bB@48i@JZt*BM?gnBLPAWsZ#lgj-Ks8II`FELNq+r`@OxZE9__9eoXqzOXD?(@Eb zDtgl)fSQe8#7`~@nDrt&7jN7_Y%$NGXQY5eL{3M2r|URU?#PCTpQkBrdSJ%kj(x>w*3{a=BtFz0f~50n~B);z=#On{$BYpa4WGFct+eCPQ$I*2t2q| z1WP$CqiG@j_h50yz(I^eA~yq-hL2bhFhUMeH-chDo?KIKVTY?LTFc+ zFxRa8BKRJhAd6?XGt0Ix#A@7<6S<`rbq6#+y6`b$>mh~TIg8=QgdlxGBf#22oo0os zCIRm+64$Mo%u^d1ws&A3%uF@K==^3nZG9T-3$uaqB`dI!cTLrB=LS&r48cwQ$9eyM z85C2y0#W0kq-{WfE!w6EL$}r7_%8oVVAc;iSSM91&X#VK(9zB>owk&4X|v zBaY2oHj}DduOg;>CG=p!V;bUL4Tez#R47gzhf9{C2@Y;Fhu>EJ z*#9i{Li#yhGBDJJ4}=#o{aLrzFYWE*Xx4d04x;w1esO#pF+# zJf!asgt#elh<5Y?bdC6rw`X3&>PN*e7RnIj-y3j$aT+;7pfTSfpBnPgmJ;FSN*T@_D)E^_w-(SLq6TU82lpWbff#yDbmBr}&_{R+c-k z#|U=Vy{0BAQkWgPn7o}MMdS{ihpMNR5EQc>N49B$#!xH7`{vV2C$5mZd3wiM)BP<|Xw2(fbfrcUb9(D%(<8HV;ewkT`b`RvR)r|A{~Ly4 zCv!|&g=M&RYy)xB(uaJtao`S=%9Ht9gt=+MZ}3;(+kY>ID@A)q z)y|oCy8I~q{B@2Acdf>E>yD%0fmm|jgB={clt$a99VDeEUZPZ7B>BVK#co#t6tB<5 z-^NY!mS+_H>rKV?=bAxDryJy7$D(MoAV>W3FY?XY1Q#33hkmHR=XjDCXl2QaC`H_A z{gU<LmM!`!(&5cc5+iCZ4X!%m;F zhpl74Lo$JGHR!G~bu%l{Z%X%!aK_@dB937U4L5z?KqAx85hZGQF=!_8yp1yuvsQesUU6feW}bQ?@P zAx{5GDrRq`Jzx&!hv79{aj2J^hqk5)++39s2>kq#X)cc^L;ngXOtr^3isEE@aR!)v zZy--jC}8fBb+BKfvr>0Bh#vMe2HsQAWigXvO208T_?I>4T(HNjQ?HUig}_Qn#|jv` zP)qj~XR#Zlc0s<%UhJ$8L5DAYXjlD3Qqpi9*{V(WAvBWQEN=y^bIJJecVJ(YV+en9NjMk4hlAu!OozJ4;%8mf6L3oVT$(M&X)+!uWe zQ=OJmRkj&CPdp+2Ebif&t|Htn-aumh@bHXuANWslF@CdK1j@JN*oS;cC}h4FI;YR) z$p4(d@%Q$Icd`+1zq*{U*z*)#yj0-^aKp*3XQfoyD4(&soD729F;Kaw1tWH+faHt0 zT$@1+INueGCo3B8^SWC2oX7_a88N83NQ#8|ZAYKlH8_y^7;2`x$LhdF`fI8o*=-b! zjjAtA{R8ge;ap!D)?S6)#yTW>;sI{87lY`MFgj>iNt&CQp-AfzeI@ahrbNe?UY$Ht zu|&L$Ecf|HH%*=3VcvY)BSp)If|v)6CZ@nwl{afK;s zgj`z*5uV*xIA%y*D1Bh0f3^{|mp70T+Jfs{HZUKDg*Y*R;uwq;IG-CqReJ|W^XF|O zN9-1DG~jWXH}q)i`(*fjwiAAY^RcPdMY+4j9pINw9t_KKO?Qh-hmadFcwSZ-;e9l5 z^A-C= z;$H>dOxB=`NGrNjxud+X2xsxy6w)sg$MRb`V0uj+@YWI{_j5C(``e*HuLZn=E%>yc zfi$$=#K+D}ME}`1Valt?K92x;;*At%W7h!4v9}<j4 z(LlYN?OZ>LduqLK;VH0HJ zIk960ZNF~Znvx{deuwLrir|qC&C_)I1WLs`7~qcO_(2`gf(*4(0#!S2>$O6kvu;JkrMZD^?~JN zPJ$qJ;qfHUR^ft3wmUiVkqMqD4yM5)0BmRa!&MueUa2^YK6*Tc+fx#l{R{q9 z9^Q8zo>#`uSCQ>_VChp#K3>hZ#3nO4&zy&o89bd9^PWtRLioqz(>F$v^r7PgynRIr zRlM$kO5h99vU{*{?bZ%@dH+`YoEM2-9ln9v9ZmWtuM!T19l+pw$6R-VqF?&H@c%uMq`k~OziLYQ!WrDvNuWl9<`K0Fx(mmP(sg(CQ9(grGQ z#qdYe9m>rJq;*4=VWRsFsKsq$f2dKSuuqAbxYdqE+zF>Hei86bvjT;Ftw6ttX1XS_ z72Su=qCn?ude+eyx8F7Z$F*fR((cPP@v^anwiVNyL#3$VPyk6+rf}5w({OIa2pe&k z$H&jHfpH5%y7<&9_#+%nvkVu*sl+pMeEB!5EQkkd{weV3@hT#D@DA9XlHg3Jts#fD zMw2IQbMWpI0nYP8W7sbl%)H*t<3(Z}aLT7)=-P0WJf6p^N!MlJ&}ki#_s<2g#By+; ztdXky$6~qZ6b@@92up?hq4q-}?On45oMR4y{9a>tTEoxDuybb0-~AvX9>TaE-qQ^a zjYwnICdx(#VOxqlw}5@J|ZR9LOztV z5tVNicxRO?sHE+pw%iHgasM419DWH&U*E7gI={)hf}NzNYZ9y8R#k*5sE{k~)>j-{ zd=kZ|FiCz?3c6xH=$=zQVB2UZHL8AXnnp)xh_*i2@M<>X1g^&?zN@irX9kw}zax2} z={OkGg2&c*;GCt}c(N}MR+m3RP2*XR6qt+&H5s7cbsjwyJ;3P^!q`0?N9`Wyfc1HK zOi6qT0jZ>oqICcNxa#zMxa>ZDHpiPm985g(ZUX$&{@5_`2^rR;?dm z#?v_XM*b@`9B(4e&SgSy_c?t}+|eUQ=;f^C7aP&YFlzI`hop|eGpf>-0zp=T$`X=%ck z^JNHga6(Qjho~&oz@zIgLEqaXFgh_wZwPNeX|d^~FS7-s4|G8AnMn9FEDKYgjldfl z4`$Dnhq$-Zht^G=0+oek_=5GSxI%ki(W{&ArehzpM`R%nxusjD@PmF&9lc%jnh5-i zr>Y?rAo9~oY`Sdi#KSzsUxNnRS&mrwnWcoqu_2(4PW@AK`&N?xmD&xvx@?; zY{5q6cY8N^K5Z7)+Ps)>-&&(nKnS^VW2ExEj2TCtEXDjVEu6B!8Lpks2aP?4>5Mxp z+;v<7<%$s+#s&GA>u~FX9k|JME}5Jl;jyBAb6W0ODd}&LYnZj-qjI9Tq74qdVr^ zf=M||RJmAXy7bjsns_>yE}L0PAL@nkaBe~9dEAA!PsqT`rV1E-%D^sG47TM+LSTjv z*D6B;O5!pwYu9bivxvhRa!&9)D+6x~0)Ce`g1zpEAkEY4IQVZ7N6=D+ z`xgD6x!^9Uwca6aKd+I5$QLMo(-mp_7woybpME&I3QP5RsX{LwNo(pMLF?q8)W{9x zzT8Kt;TeSMsD#NEjfsP31m5(`BHJ$Cg$D!6pm&uXcR}D@j9y9E%R4+lsdp!w_4mV* zW4uehdUqM-wj9^uK?l)^(x9E+-BIzcHz_E-hBs5s(0}vQ;MneVFxip>C9kgGvL;#X z=hRN3%L~r%L@{p4w_h-BJe8xv>4rp$<9I7Pi5QniL)92dXV`P-pxIn5MRsLr{LM;w8r}sh4kA-xk)e=|#I7ZKx zZlYN$IIyzJ3c8l}vKKfPKs`Ge`x;~D!suDxdqIXHCOAN1eXg^s&k4Zu-~CXy!I{9* z!#wP!2|hdtK=;k|%=KRhG)g9mT_RLXa=yyr&GH0bN}cdqLlCZ;)de19HL&fErD=ET z8fa;Wf_V-*u+;cGEEG@%*D3Yn`rdaW)klqvxpO#WAzx`vmjRewl>&q7BDl)Z4a`2& zny-OCyrL(Aa`iKuzYJw~~H17}xm4)!+o&^U^Q7ZI zcR(M0cXrXIQVNFs# zTXm+I>^O0gPKyu%_qCtswuL~}Uh;>HW?^`vkBdR(5}5JB7S}yZLcX!Hbo^Tt{zzXB zcZclZ#n}~Ds-wfMu9o8beK!YW6?)jQsipK!)O9o~m8OP0s> zv2hNleON&D?{5Kto9V91RPpww1^#k?6J_hB-rYyZRq z`^D2JWvDp8``AiLCQcK#STWf2AOfV9N`T0>|98fM7?FxTJO@xvwURMq6o|rtS4r9 z?zm%DGBXyU4C<>MVPv)dhuJ)Wbs8h2;o2B|`ymFrRu(c{Hw)0tYAM*SE+%1DHP}mi zH}TOgaR`Z6g&PJf$lF^kaQb38Hr;KYkEah&H-&hTQpAVTDzDPL5A#86mITQ1c#y9ht?B{itse9Q1(&TUiOSgz| zO7fgw|7B&!{#FcAij?T%tG!U~%&YyC7ujX(eN5c7m&!K&!JH&vh{;goF5_XKyS3Mm zm|yke*11R=PmqVewg6bWGXy?-al#-uH`Kk83?muSaoRtJrOz=5_g=rOr zug*sAS&68)A{iK7FfKj6c5wLiarbR`u*Arr_~3P(Qp5)LALW z;)|vcSqdQ9egjg@&mxzLH0cCqCHCytfd2c;(WQSFC%%es8=9i zIPB!8tGK=@gZ}blV0>jHUG>}utCp=q(~QwdpX5~3`IZY_X=h1NXAimEz{i>GLGYrL zCHq7@5yn+Zpy@<5=tijHwqPByI^i4o2fd~rW(nfsf|qFgIHLSsf7qNo9m7tq zgd0a|a5!t2bewMhW`7D!baL?hLLrXRDh+(Nc8n-Rwa^c#;*cox6veMC1Z!U3n%lD< z5~g{f+WjyR(ou$$7Yi!JWG{h&nJ-m%wF);`Nzf!)WwvDcHmdj39u93j16P97fomX3 zKHP4mcVCW^eFtJlrm#1zaXnAF9!rosZF7`L7^XR9^N2&oEHJQ}2`T||uxf`pKNOuvBKqq_sfznSJil@rKIu;5t{J?=6m9Ot zTrVql&Xa}L@<(iRse zV{f6>qES36dL7$sJgJ#VFe!Nv2ct8$lC=kfXpCwqZ}L4Ng~x5c`S5iz zyrOi~f_$8jXn@wrUzxsN!E7w+fEWG*!G-Y;R6zC~mG|bLZOkdWHHGJ|l6^2~;VM-!vC`WmS#Sp;nfj_8@W1RIBeU=i2UIB$oy^Ln0zf<8+}L<&3U>wqaYYf z&BhO5eatAw4aCZGkgigp&(p48qKP@zXU}brT;2#L_MBr&9^41fm|(D$w1D45QPgSI z37YXkACoj6p|NT!)oSd)*QIvk>}kL$;&0efQt2?N!Q+*+rc*l&FI244L0XfE8N=DA z_9`0vUh`wysW$R6Y#nV|dlKJQ?t;K^b^Nv0-88dA5GK=S($Ro1*kWvf%fus5``jEN z)wzv)X#EDe-Hl-X>IPgbpbf3YLD<%}4OoeE+_ClwqkL3^qrvlA>8Q^oE_r!qUr*r1 zt9uyQ;0|v#T)~@;Nz}$zfZkWmrTDsnIW=9BMn7FjH{P)Yr?q<+m!slL;nz6Rn-_U{ zq!$6;thby*&%1|~4mlXSEe;D;J%^ICX+-9_DXZal4qs`j!fKO!m0$Xw!}r1}V8+eh z={uf(aC!uuEOUUoesgGCKF%Z#8$-T}668HFg1f(OK!Jt{xb0HG16`I>m{(S{9U~vi~l#`MZx{(4WoF(47RPSOC8d%;C(c z{KE#dm~utM(!u5P5V`TAAK%+3z({l;9IEOg6EzBu7$b}g{>|tw5k_C+|ABj7u0v;# z3-D#m<$Qdz3v4|V+4&EIP#}3b96ejk>o1>33%sQwWf?H*y$MvqKb9OFK?|L1EYx(R zIRQh=r_Zx+mv259Kb}H*47ix``vJ79+C}8fIivc&+3+DY0=g|!ILSuKpnI|q6?SdJ zlULqXsz;{K!xH(JS@)cTJ+cSJDTpoP%Y`Yave^5unn*-bp4Kg$iDBG;hxU^ExfS?0 zFcb}%Y~b9MT&S2?1fNXu$;Z7%klWFOJ|4Q5ER~EUI|A4_=ew!-wG5`=@CFXREatM! zQqnfmO;mSWVX_y!LM>5m8m-byb{Gi5iL;fJVNL!B9^WA(sDqJITF2Yj{?U&q=U~Tw zVPsZ)D4o%-gBh8^Ty;rFkZI0`tiOmkp~-al7Zdy?WQs?njG)&x2;B7T@t%wjT4tSt zLa$Qx-swne_1XzVyML1<5ATA>#bC1Z(mq_LRZSi8TG7tBu<}sz2^i1c2bwNrkVAQ# zN5lZm4y?u~_p5N`LkWA|MGNfZCww~c01oTa%v<#ex!IL0Q2fkd?e@~P)RofeultGnTk6V0^nv^6~L^~a4h z8yUXQH%Sts~}D*GGo{7j-~ZK}%|$kO0$D5|BZoG#N4 zFP0X)zQ*Hr(}?E03wUQ!J6ZBT8A2VEVYkmJuvxwl{12Odr@iI05n0l8=K^%U7{?XU0)W`J(&yP(wjkIwIQFxIjNwboi8ejX_DhZ0Nw}H$*Zd|`1g|!%npqq zEpZAcRAGpsu{W@HER_6IH3HD9C}Nb=l_581HBqf0GE#s^jfnqSm+&nxtBt& zW(-yKxKICkcZ2R?&EV}4d-zkBhAn+h@b{EvoRSg_**DIRW}S5?zG@GsDBMLcYe8ZTpI7S4m`?A1$byXtmx&cRP+7^>xFT#BInLvI4&0c}{h;azay(7S=5P@Z zuJ0rhSHmz?=mCl+W#W*^8>bL;Tw0KGs;vA6smY=gUOam>sIi!1GF`7CZ1lNlOKord}NUE4v zFSmm9nz@`4bFUE2IbrY$%_5U)0X}!l6u|WPTp1Bj*3IAl@SfGx+&&XQ(%}We#9b=mDmx_&V)o|1m;u zCo4B{96|oraXfQB8R{UG9`*VGwLy&_#`~QuryF7SnJyx3Y!20>4)kT9JhoIe6pehZks@aTHk?gauZrBKE>_`UD$41K-4q!@ygSWROZhz zs${o|y>zUGxD0)u;T62QYbH@J-Cz~Ecm~7iRvB2c-Htvyr^Nlq^E{hgcnp6xEC$nQ z^U;0g3_9=LH`0_z=+^NSaI?J*LTc=x_-h@eXZ4_%s21IF!iSVj7sWnFL;SHu;KJwh)Ao${MEeLobv#-X%}J2rw$sx&ccxJeyYT`ofJ4mqQE_CaMzE) z(wDh-Z)zxytMq17c>bc1NIs5itu5Wn86gqy1DqG0xb+KbCzmI`g=izORorXr5?JsWdC0 zNcXJWJ&0722Bm}yh0Iezl#-GRiKGZcGE1oLS-X2uAu^=&)g&{OQY1v=xxeTC`#P_4 z&fWX0y*}%+)|=fm%!+M|OXuG!wF%#A>9IZsrj)grWUwNecO?x~JiGmFkI-VMyZH9l zL^d|%9KXiFig&rJ#1iu&vG;kg(51bQU!RlCE8INk;-K-PG%f6yOZ1GXLe+MOCVion z9Xw{oOCS*Z`t=1u-&Jp1))qhEe>{ojtJZ83UScl@L$zO*ZuDs`UD*Ah^z-y$@q4o! zr3>HI31zFFmHK4WlzJ#VV~6jXM7O3LV-?iQd6`k#{Oss??4yJOXXaF(aod9b7bF&2bUJ| zOh^*DrP)}#=%KDSK(&j_wk#0_YdzvUAF1(cYR2({qvKpg>rN5xkSP;K%v>&1PAL_- z7pCxBgvz8A?It$ETZX-5CR@fo5woTrEqE;*)3Q^(-^2-d-B0OIslOVb(=XH7W&$^IrAR=j|fGvb(Y(<_DkkEP2Zv^0cs8nal>i=%`QQm0uZ zh4t(K4%70wUN#{iq4a!v4c!)7 zEgnY>3jJ-(dE?0?Z29xEY*DzRZ@+6ZYo_YMV|N2P_SFQoI{qOakd?y5T(aYx{@mu@ zu6@9+DqbO;)8fOQJhxHYpMHdWQ1_f2k!B`LVNUT6pS~1!?q1KDCNzszuQ|ycV%tfO z^d&JnZ95-G()h7fmw2<3Y5a0NM|?-Gh;K0e=@Pqj2!HYOaAB2nCSSH~X{oGIF`IV$ z7ioXAO8ATzOJesvvE$$ccE|V0!rROGSi^IKmpL8EUTBIFr@k4*?mJV@FYy>s*0tJA zyxlQWy!o3Bx%?=N&0F3rKdfBLhAc5>ZzcDORX^16q2rE;AIN0#H&gmq?Ja|> zf}1m2?v*As;AgQN+*~%ZdN>Tg#XiNTUHn<%bWe&%bu8hj6Y&y$?o*jXA4{; zy+jd~rLQ06l^V|=?3MlXE;F}JWTz_YuwPqGvfgVpi|<_bC>+j_J7Uc@4wdKqx0R_u~!C!TOFPY?PvWId(3{~lGApCKe0!ZZ@ccmi`<{{oeTDe ze>|VYXUZ*Rv$+(P>Ib>PFG2poDQa!PDVma@;+{o(=t_&SxTZ@kj8dtv@6=J=tFMSv z-F{IVom;}X|1+92LcfUj^1mt^+n>qbnVBm7`};1xzV)SWQMVU=GT24p&%DAv-(<=s zJ{iikc5LROJoHN=1CO#pxk$co&T>BD$Qge8xeBrKyk%^i=5AIT<|TZerp(6Ni($Rq zyyjotjuX3oO=7P)ag>$hCy6+@H6?0uHE z*mXhdFPRB*y}=otEDE@-}7d@10D%aIsOpVHZ=&FdsNG2#%>lrS+$$> zl6mhUz5WQRXILfNyQx)dogXPapzg{qnV2Ask>uxl>LhW1oiXWfRXyvXV>Id5XNhjF zyj9}i*uy%vN3e=khJ4}l6Ku?%Wy0aF;z}=nRb|h3gs^ERGTHZ^qxtAVIedn=LzpkB z75j+~vCr=b*^vh4*j`CarZ1;6Y4$7&VRP&(Hsngje;H^gbH@6(;1iig(>}Ev@+~(Xw6a#WNNNzP^b4?>h2a++M!x;3<}`cqjh6RfBg~+gTd= zAWX~#4=rnt`6V_lD&)i7mM%P#U~#kE5B!hMbKNX-%58NSu3;?JtFv&4l=MmH{d_5wb1V~I zn3=-=`jN~ZjqPE@8%xC1`d4@&c+4*z*~LEgS70^W8L{PTU4Cb$iA!Y2#8OAS>eA`6 zB()jwC-F{xB^zSo$hWT;!)_bl!p>Ka=Z{3(WW#s=C){;ZkG-Mwjg5O~URGJ($PT>w zB788>nzt;!EY4SO=cB)8u_2Ebmt{K|OFP%4&@8uj*0x|3AC)qjwJ6D8>s{yZCVRyE zj3sNuj;qz!aeO$xd$qcF&uc4sOHNjt8)q(VQ@JP>OgYc?TgsG;9sDG`y{|-QQbfej zb(#FH1=q!o9y#(oeT_ovpBs7Y!hErz+HUr&k1jhwVHST_H$(hoP*N`u-xT^APGtL3 z#`98vW6LtfNKLx*nP-1YGvgDIs>LaRuDq!80-u^E`986OrH{Vm@{O*_WeffWu&cd8 zVZpvQad7$+xWC+p-Wl*DwX22_Nn;$I>VAq9ipk{nW;aqgA`F&nvw@d0Hqj-4Irw0I zCWyK%(Y{EL>{~eu&qk!+Rm(F7yH#kkRaEisCN=s>JOC@cj=)T{W(@z54~yJ4af->C zx!Zmv%@30oK^R$Dv4HAsw4@I|9E7riH#j~p0be}$iC@sLbEJf`jP4J58Qp|WW1bXyr;mMOL=x0+eGVGDe z#2>GZZu%W?W40?7&6=rei^IOvmguk3O3f7e9M zqC8Nlo?yFSEohxwL#Kz0C#&9?5Fhge;CXNsnX+s~c<@g7<+tG;lwd<1U zu0sUR7$D&u%jg4s8clt8QPA~0gns{T6FHggO5|f}xw!vg$w=dsbj~~r8k(^Mzm4~& zZ;2YY%8VyNu7%?j=@*cHj)&m51vvA07O-27Jv7 zJzop@y%lh@bSvuSCoue=4r(d-(52PGiOU#y0`XFmEmR{0rsl+qoP|Y0rO2fqbNZj= zK63DqEkX7xZu+p599p@R4yd}(9`7}vcsQOa6~+-)h3Vw7?M3+ZwGNjyIT6DUeKNOm zGENULrrqC@VE&MUV4GF|Z)P6iT4#Hcj4hXO*2BF_G8Uq9+b4LGst#-BkDxzx2a=pw zyQsrNhPrK6hj)H@bh(6g>RByebU9Xomt#B{%)ACs$KwR3HEuwm61z#**h7X_Y`9N%&`Jj%_7+Will0 zR490D?tsYpLO67;hI=+WlxD;XqEX%_Bv}U3=kNf^nHWQ~>SR)$Dn)JVTfNBdhx_oIdN8#eIt)Zj=G52c zD;RG22{H0x$g!Rz?(EPcST=Pf<{Q5O!Au+KzukaZS=@uF)+vktBGKb$G!=!-p$3bj z;6zj|JV?9`2DZELSGF5I+L4S^*3p71vAe;c(UBJ37)R9aH**!cTABPMao{W>^vm!z zJlTB+6hv{ja}~wDS8wrku{HjqkuWvPhHkXZg{#+v!ql_B@{K@32A;i-EHnabs2W=_;2u4TiAn1oIk&^87BTJ@m zqf?sUxOp|MURwy5I23bUWTW26k;F|$5u`gjxur)OAVr8EYBVRkDL(Yb)V1`Kphl2e zW=U^^DdD*LIdFE)Wt^%VKrWBn&F%T@On2H|!dov!lEQ{ZxT<3^Ew-p;(iXqN5ql)F ziyz-aw;C;~mLW$+sZOCQZ(K&(-eJUQ?p8cAUWq)q?M%e}qanC_EH(ca2S(>jVBSz0 z`m$Aq#Q0yt5(%I8P`Wem{yU2PJ20wfjQKF~;YL3$c`}|9b<5C81yOYDouSYpwUEB; zUBXCx0>t zC6ig088vC}HA{$dhTVX|gPNrGToAccXiL)H=0o0NbsEMYbH|(^0l&1#?zx$q`MOF7 zJXnHZYGyQt8%6$!*ODCE$#@Af=u##)op=!uOnm*+IM;F%30#Y{AM|aok+Vb1A%4qO#el z!{pux)asH6O@FCKmG0=$i9>Ut^!^zTdd(t-60Vi}Ib=YWuGXi|8_z(az=6(l zH>C&P<%6G;6VZ#dBukz+kqc=Cbo8STT5a+SmCu}Kl*;x>t}g-=&E<@8oe^X;&xdin zn#7`gKUeN;LX}r&gRSCvrtXCmxA@N^{G9eeWInPA9?z!u(7Oqiwq>G2hyyLWcLslH z-^I$`+o4S>kR&O%lL&`?&>z1GrnSxi_e+zwOB-%u$09FqUSmTQ#JQYbnJU;H+Y2Gx zaqy{cDH#$ppKLHyB;UT2fboS`Qn)jEyuzU_TR-XsM zJ+aWXzX%^29>+?x{h-piA5BAg!B^jm+&DBE4*%W)eT^!D1K%6r!oUOg_8}eq_%}h7 ztSw>Ntx4%&19IihO}OQzMVfl15r?%dWJ7o}{=U&K@S7Y6E5}IDi_7Hxe~&$I8%qZ^ zOYVss%D69@28Ey_;q8{8QT%N@n4QIWhbj?0p%>ZNJDM`?6X@6nU%53!6`UdW9}^MT z#i%v7U~12Hqj&3*nLm(A?9?ZFvf6Q)r3~qHS0(fBDUzJ-9AFn% zlViFg=$JWn=>IGLSO*R+)xN{nqXQVe{vYT&3duofDY&xo1Y}r9V){&Nm=-sk>Q9oV zCC#^BnrS*C|KI~7HPIdy5BrR3NAiANsm?rM?ys&gMxMsh<-gqWK!z_S%ux; z5!Q}ncMhQAoirS7;Z5J%bHm%0r*rGYwYYl75FC?Q%={hKgzwwzLA-evESdilK8EcQ z?Z|zCYikx`si*_3FY8dP{y}&;s6oxF-I#A*_h83>6wzDq3~pUJ%xQ0Z&s;SQ0dpHu zRPl~Pv(P>GmfYe#XJv6AYR526O9~e_^<&WuJx083JHB&V#QmPE0a@#VaoW&YFd1nT__S`(X!j`jce1madRaOuCJmB)@gtYmO1=e_>_+Uz|7k6z<%kO6=KSzi4>tGINPjyFs zz>rEaqMl;`kx!&he(PU6IXa8kuGRo5-3;n6g-mhKM%d&$5>gh{BG=(S8_Q^`v(MykmdgmQD z5gG&I23~O`>keU>Wmbu6s;cwHG-EO|Nec!~>;}I>nViYarCip;g?KgfI*KfhV^+fm zoS-`of4Zi^LceO@Tiy$v;Tjx}?dLAYWWt+KuFi&c`*3oSIX&{vl(}i3N53bXoXCXB zA{!ouy6$=K&ss&`Hq`_U{q}>a{~ln@n<2%=4+P?$xo0tOH$%c*KBEs~DC*AihI0#* zQSfRDTs`|8ERPhzkLYhq8FvB1w;i$M)V>nyHCa#-cOUc89wT4+3Df7>lU}b2V4t%| zRFv0^7v2sByHU#Iv+q~%(s{_;;+rt*wlR+S^%x$_IK$ZY4Wm@*k9tN~5> z;&mpnQaiy#?-n+ieZ(hm$=uB+g?N8a0B0%tne!WF3^D^=Apf-zTt{>L)y&BG zEHOI0^uZ6S?%--yPq2%)jVtwxO5#6ym((87rI|zjLBOU1pe=I6+jCR65#8;eVL2Pd z&0fPjEiQptr71WVKa|!TTLYuofZgeUd+ z{{4X+Pls@Y({nIy%@i=n-iHg$W@FlDKU}kRHtH3OFhgSHi;bAE^0p2pR2M13Jf8gaFC7_iv(tgl&*~aXt1`|7HHfS%H0Q5-b?yi4Hyz=G~qb%xC5R zSVYcZoZhx@t$a9M{PY%=xqQP>1}kCS_Ei|Z>N57qZiVetdog~?7O?Sq0w1oJ!RM2A zVCMOwqGb#I7Ks~>=6$}6V1@DA?wHihHT-=ME_Cj>1kfQ{1> z>B4EsaQ)C4uq7Vcw7V*V7126?Cu%F&;BP59R13{(%TK&Qx1tX(U` zWg7lto-2Ez;=~W|{`p9#8nX_HMla!>%o$1dEK#MFCm+JuSHM)~b>=HQ13n)LC))6neH1OqgQF>{^- z2|)_f&|RDAPJIpmSq50t_*&HU<`-l0^&4~T{4FN(*f?_9Q=WvG<=|gY1h?$`Q^A4- z1~{zT89vSVgJJor(JbSH=+$^xuy9D>>L2xk)r${c-O!GgR*i-jDT)3tT9(*vG-i&E zkRk(J<=poAMflD>mK(M#6-&hj@Pk$zhF_Quwi7gnj&UOXh|j}CnQr)?ki&?Fe_+JD z4)|kH1ytO^O*?#$vGYuaH0=P0*>M#>y^HG}LL^L9F`N*pQ}@9)xF=5*J?A}va(zkc zvbGhR@v&x>|G5L2853})`zLgfR-{(V#>CCr8n!+^&YV~J%dEcHgjEl1iRJ1X+^Km1 zyGLf>3Dt5~_|=FG`S1Z9%OkMDZ>=b!x(C$X#KB4RV!R@k$9;G!N30TFgO1QmFn#QG zPI=l@{Ge8X+viBCnz1|J%=v1@=6WIQoMH%$zgUq08;<<}Uik9mLbw{#%iZa!g|s<; zAegxZrYR$d-_AtL5u7Y}^?R3K>53!pW6dMM2;1WjToA~mCBMN7*4b$7dj_9bKZ4<# zx)|@qX525@$n2ev4Gmf)aJHg`i=3*8aXRPW&qspQ58uGbPdDNDyTcgEHZqSNxWnPO z2k_anulVhX9-fz$oK+Tvqjr%DT#J?P$Jfh}4eGMEc3T$YtZRXNWeE^O^iXvqftHwT zocFT9*{5GWHtQVZR0d$3O$FYpr?_E~4D1QnC;Bt9P2jZj z0IVO{j!ttfaX)p>V&a-CoVl{-Y0&wL9 zb0&4qR46T0hGrETy1CaISAN*QxZjM#anqEjLWw2y)Xv6sBL{?)T9~YH2UAyBgY&eL zaMi*dX7OK`IbCj0;rKHJ3t$);?IWwvsuU9u2=lFX2U?74)vU$lbLY3SP#6Q0H`> z87w%9`N#f=wpOd)I=&6N|g0ro25y~}F&aN`g(ksgkkHX#gKpUfO!7sH{}EFAv%HBPd- z&K2&c0-b>;a9_oe#_Rk-(avSuhFyM0{x+iW;4aRe(ZN+#GWc=GUcvd1eYnVCDt@`6 zj^j>V!ZSLlTxyLF7I-N#du58TOK^=FymlU37B;}}LRm&-rX@b}l>E=!DmW|AT8=y{ zf`P-XX%MLeFf13e}^!Kj^P;B9@SXvgCWIFzUib#B8T zd-H9^lW5?^Ijfls!^h#q^j%=xa}MgZ{U@3re;E$WOT|O4bl}e?X?S^~3V#ni$0=L= zQEBrf%t|g7xxFletEQv4gKIq)or5Ld=3+tCkqRbg&O>gt{RP;f7Y9cMnqlcAYcjbg z7_@jplHh7ipLv>5EuCUWoEd`yFMBYeBN;k}?8V4p2YPOiB1v3fK`H}xVC|7IbdUN8 zTVjk!Z}DWzT^@)tuK)&0&ZPGNlXUoai^D-Eh0aheR0epjn%OsIv83luJK{H?NzM zvFkM;H^Q4Mv>FE=-ACe@+|LqidKvhg3MN5U?C6`u4RrIMA~SO5N#q$bLevz9Qt@Ig zA=Zx6B}c)lv}staX9!339DuHq{muzb`vn`qzTl}VC3t7dN@6relf+3h^M_-m)8HF1 zbo^sYT-R<&`F3wI=j|dwjycfQ=K1u*hTSA*MLhZUIGc$ZCr`2! zqv_z^(X?QQqx16P7@yLs(7*{&Jc@9fL?3v`&eNAeb7b#1r-N@x(Fp zJ^Vf8M53n8qmFAm>1SCxtdwcQ-4X_`%j*s})n(1;MyW!(lr8N^d4c!md%(+rTzLD% zowlJCm6`C9E6_FI+HQ10@Q*VfD*VZ9I~0dY{ItpY<;qZDyAHp}oB}r+Ir8xHB1}}O zhgZad`B{5|`T4d0u1xC2v=Ub;{XL8{RD}_>84dVyWf875u_m9mG~7Ak2N;H!k^Yj6 zG~tIAb@twXb6Y;bup5MW@UyABN~|Dsmp~+0MzHqULiqOUAB4>2p}gf0lCyi6g4imq z(eMlRcV;!-HY*(qFjXBtsBHW5`3xo3HvY4r8x%!XwEHd z@^|)d68|NhPI)_tUP+W?VxHTP%nd#?=an~o$eo3(u2JOBl5l#`WD3o{{G2n8`v!|= zGPH4%3N?uHg$B7j5LysNo0U!Im7PmS!p(JLsZ$~?jYy_b4(V~8_vGpIAECs2%4QN7 zf4aoyMH;k>y$+A@EPUVi5vy{8=?TG1a@lns`Q1C7X>M|$v6|t;dyWhf^uPXHVj+>&7 z$MYcn`)~058xF1Mah&#-05tet_vEkC{WGVAdGa4fSL1@ zFs(EjmVXZe&1w~DEy)*TwXR{0U=GTx{f@u7PcRn$r0CIf4l1t85dV4KAzQm1#Up3n zpm#ZZ${dDE`}8Gw`%=;>e?g>gxg94<81_-Y!)Vz|QxSNXLgUSBfl|v?fqv?4FbEIM zmT(Ny3sj-6U60;#^M`*uMtgv`d-R zcX6mPpW@3}RY6~E2&kCuff>DL(Qf$5iXDzD3_+o(Y0E`~BJ^ZDll! zuRVr-$rrdp>$BXAdoLmOv^S%ab_D7bMA$rQ05=A8zzMqwfH9e(A=c}WswqL+`*o=L zG8J~^d7^HW6nQbZkEy$nAV>(l2SZnZYa_p zz+~{dydE+i{>1~fmoe@ALHJ;+LtBEJ=<}V~aPP%L=0|b_(%B98`e#2E9JUvw&MI)W z)2724c{?0z@)ndvvNSn<7>%&|#D)7Cl6mzD$l!n)sYMU853YnoXXjx(-GO~07;Z_U z283rtqrv^V$gLVf>{d@8;($@a|HKon+(#8_`_AKRC286;BazvpFa>=}?n%yBa*V>I zY)0YSGv_0o^Cfzl0gdmIrFBz9+}wISeDGu+x2~RLf-gCOdE^kfxM~T}PYxkkx359H zzNDTmoyJVE{EN96weauRXFT#-Kuy$K>9-^o_|rX%^o?xB{$tJ9?QKlfFEAv}9=yg+ zPNuY~VmR5c*o;KKaO0jdWa4w~5m-l7VZa4h93>jd#WznPt%~Ig6Vi#36@?g;{ty<= z&ShRLd5G6BHIXq<8>r<7_+Gi}d7aik5^KWIc>Khq#{CAv>$`V9;o zd5&50JOXFAI*I<9XGl(@NcQ-!{b-a?hdna#gyPj4{Joq=d}&yxB^gi(9EjisJiB%moB2IfbSUJH5ha>rzP;q@R*Wyxvc0(I{Uhj^s zVJWcT{3i&^_bbVJ>xOCm^S1gnYUV8L9Nal4hfy66xibIKWu ze|+QSojL?BL-%9V>C24qwWB!9-V=Vjoeh{_h8vC=kxGFgjX&o~pa1TGqmM?yy34+B z{beBN&Af}+;|ytslq}sjI1H|v_rmJT2%E&UK~x@AW63==@=bAZ5|o6L>%?Fab|C%FA?H0FOaVBDrh;un4lWZsBJ zfy*2C;82HIFK^=YNsk!yP(L_v$(|N7>DVeoA?VixuFEeB_WIT`r-iy0xNR5y+n5g@ zygqQd8{XiD5e`(Y;2vnnCV-WNJ32qnrFzr7gj3Zq!gB#m z7&n!1sjtTD_;yqXHpciUgbfch$-CYgV7`RG9IIS-`o{sPy8D;_nuY4c!)e9`6WDR* z3)B5>5pFzL3At-ZIOi5O=HLb~K26SG)Lw<4!asd_Wpo-miBzO_RF88bb}wcutMZr- zt#U^8{9L&C^(EYj*@pXbayS8(FQ|7JPWH^ups4{jdT`Tnw247s@UR@2vTgkb&vP(c_{Z2!#Ti4NB zZ6b;Os!ArOilAZVO1QcD9QSL)4o2x90uRlY- zkDP#CZ%i~5@^PcTC1+lKfZ4F{3Y?qx9^H&jqSueR%qNMTDr@{GvhkcI-EgrJr$?y~ z^`O&C*9sT>{mc@#v>p@Gy#CDnx6=ZKcdL=?8=m+#vJ>uSZ-Pbfr2@5*W3VC7k?wpq zfzT1S@jhcN%G3e*;=B&4q8Oop9r=kRI3TLobO3 zW&fB-)H6f`+eh{?=K?>YgKs1^q)dy9czX*?zNW(_M`ape^;{6Rqzr5qACh>!lHmB| z4y?IymAR$RhRG8hp(!(jjxx_=bkEjsF_KzuTf%0{dM#pl?vBS(=|jlNv3<}|ae-U4 z_PA)tz)5E3(9<~WwgshQuYi>Q4Q}DpcC_2Xz=p|FsaaRK$aP2{^JsoMJ{Y8Gn`zU!tyMoEMC95$w^i!Vc1+9l@z5emMZ|cj zFlaIWV|(=IS0)|0v!hXAR0*z5K7fHg4M{{m352~HWIkSVaGu)w5bkX##&wP(sF!L5 zerkUTmpT;a+ovPxg4OMCsl=b~w>2RvGtqg?RE7?=0d4U(05x;Eu|(5MvUk6tT(vi& z>2*!;C8!F&wGE*G&DX)$(hANB$AD7GD9RuHhLg7_(=BTx)iApTnEv(xw_a^LasI4I zG9JGTE;~DJd1!U%Q&iVPzN>t34hoKer^o{-|j5)LyZbnQI zB)821-`mRgjEMyFwj61J?gc&7)(wafYWkb#4dFMU9f82%>9@67(-7w3BF9rMVpsOWZN+%k}UCiy*a&^OisVgc-n^$&xO`F zA05L0%7F)XK<)%HNAiwtt{hHm1uEo3dl@VrZUzTqKXO_> zhhxNi2YS`@6TB-hfWzbhSN>oQdHw1Te(toRNwJnN_*;)U@QEJi;!_E8y2dl5|^k+Fm~27AnA)p%)2nsoK?@beSZV4=0CA`m>(TJT8gw6)G@X{ zmeQ1akMRDAdeDlLrn^>dV3t0e2Vu_Npv`m!o;8tZU{7C*cCH)?Co{s(#8id2`$&^r zWkX4?ByW6g=tciYYV_-$H{;^ksjxJz3A3df@y+a9SR<=IBgP!V2Twa;L{Jtt_P=s? zCrAXfX-=>pW(a+Fs)ZXVSIOjG4aW>+EqYa=kBO@O;gCBa^sLC3mIWBnBV|rx<;oB; z=}9wa#ct-_y`4f11<8>^TV5cyX*x_^vw<5P-XzfbuM*65m4U_|NsiklPb1tH;U~|J z@W4%%`UT9OW5*>qb?@5@O@ZYI+e$?@MJ*t+;4L}~4kKD}@}#{=f;isr8=|9h>F$0* zD)|1Fd#*W(u2MY!8{a>JBNM#nONF`QzB8eVSIwZhYwX}a#0+93>%^@*yca`WOrYbY z%QK_@Y0`#Sr#aM@p{w&1$@aTea3W&{8ov}^zS}n#^4x%G`z@lCH^#xcH(z0n?@PGQ zEsu%89bDt3=P>1CH@;5F#cju}VB6K>;Ip8P)0#OO_a|?JasT-OKG;Upugs+*|N6j| zxmu{J(t=Nnas*~wozSVL0DT%QxUq6S6r7n(l8u#N?%{p7vq-}VFL@8`Mk zS%i*oQzNIUo48uQ7e&Is(Q4du@Rk`)oAFcIxt(83e^G4Fwo(SgE!8>MtK>k8|+7e<#pl0 zH%0uxaL_rs1LSTWLFse*p-X8h^KO^{d|UAwoOXrb%)cQ}TUQ2A#?kn5hzBf~TZ_FN z()7hSTe2|N0;f;ah4502QCnw;wZAKO zpz1yD4NFD8$ot&W)>Of=IaTmljU{2PuR~N*Ez_KL3f^tph1EZfL+X~>;Hi+v_^jH7 zf9myMW%vO6%2R_A-JiIm#}@D_uo2GY58|xlJ*ZO2F-01YKzl}Td)|0q`-bCOX3t&D zMQRDo@(Gb>s;0!o;hEr{bRQZ;eGo_&OQS2~;mLD1QFc)_Gd8RcHrj83AN94&!RK3; zWl=wHhsX_o)_)HOPT&->tM>#^{9NM4!`curC)=BL?s^&f<5C7|IIW;E2~84yO@MF zb!~X`x&z$Xw*;T;`ye^z3Am)?)4}Y%4AB(-;l5s*C(z%p3S#qq!Oh88sO?$}y$dIi z+E+o~!kaK#g(DD;enaOQ57FxHHE{a94@$Nhlewd7;hW|w&gsa1+$+;Uu=kDy*X%hmjSz8HGTVLbOeh<)iB~70OoWS)KRcK;+y<~0RLbz7nRa_ht3xz*x z1$-z&b4vSAdZZR9-t<@Wa1n=Q9k#Udz)gYj6d93ecpKPg3^4yjRN;5=6d0j*ij)5` z1mhh4!LJ*spnd#Ci78u*FS-Izaoz`}EA0fXEwe*kdwt^d+MWA!_7uc0UT}t1<20+q z_`O$y9t^64?hlQOvFvfYnqowrb|~WCgHN&7%?8({OT4cFQyg{lJ(Koe6tspTH%*X? zTf??<87)`2!B0t`AU~4{yyPM3v$FyVOCC4o>6A=QJA&m#E17{pGxFVD8wchd1jUn2 zMJ;=$(ank(qW`2e;X2s`oWu+XLmCd?9lgMe3k9dOGMDV{S{tr7tnl(2j*I`1=HI&3%-dO zu+Q}ZIxNl>JWdG0u>LpbXMYd=6*`k2f2B#=<|1^pnt-tE9p2qwL2RciBnu-6wWlj- zV?YjiJKIy2m1D4S@m}a?n2!&0$J2_j>SV&lT{tG>nqc0RbKG!$ZNk})BZIOUOs=&8 zS-8)W+W2`L5`4+-^=O4{p;vvZ+Y0!T+OU34x$EE9qA#vA5bns=r)5X^!na0=xxvj?nkcy z<7-Br?UkZ~3a6Nz54^}yPj7n7(^XQFIRkErwJ>{=G+ij1Nzdq5k?9A#Xy>j1Jo#RT ziaUjpI*$Ua-Q_?mc@47EVG54Ov?5KtSrZooig6+@(cL}OXu+yV!Tj5{Bs|!hm_~*G znP)?NojmB{O}631%k0$E-QJDGy&Xx=#3Q{c)G5J1Yx_c^hTW3h^Wnyu}3>8fL zdIiRt)xoi{VdT>oi6yIG3_8!+^*>LHWWc{7y)>a6Y>EYBM$Zps-e4n)`ejb(A}bo3 zXGoc2MeuRAFH!HaCQk0!WPSHkQqb*=tJKER8pC1aU4RjpYHmqaWUZzbQ%`}_1-r=apDU0n23n*6)B47M8$N1N|5C`Uz6%cVXX#M9xJ0IA|3}lQZT?aNbRh>`J@|ujU3~ zX`4hZZbhR(Bs}W}2&kPOTY@maK=~MkS)<-UzEhgd|7M!#wI#qCGoo z=(l2VN&4~wuuEe4FUb8MxmOjMm@$&%&t{4K8DsKqTs)SpcnD8F^>c2`FF5J{6sWD8 z94Sk`1NCbc2u9p%5M4KL#Fpb8q%ztKgfs)TJxzgqo25xYO&&Cav~YLdx^ccI&ZAk} zFMNIW{}?(GhZvtQj(3QPbm<_JR3ef}we#-GDkM3IQp%Y#6v=)4DqV_n-%(MD6rp41 z-Fa4$oJl1T5+TWzBe&oF1^e#qJ2THb&-eTNXz}NJiUoEN!|8U%WoVQ6CfXI5LYECC ze2tGD9~uyYEMH%z*DIA!^aM>_`$%5(f%}@=pLg1HZG0ix>v@;nwbbQy1a70w_0P~0 zp&tFb=w5Z(84*|gBmp`052h}+?+7$3a~+<$8uKX)Iw)rAD71IK62E!tNM8NVaTN3| zRp8}h&W|~6%FpR4rychmpy|2hv@>M0;PBOtbeP`?@yPo!Enn0G@vl3tk&eF?{ZN&U z?%FLEe0V>Z`ze`B)eqN*Bb7eceTiI&;uE{k1ZkDv!>=guv32I+*$QW~*Zpgn&eV^ITNf97u2zn-17u_mQzBS$9@|9ln z-R=vTbuXCP`*9=J@$HSEvONoVsi^U3i)+N6%3Dxlsv<4E)iLZvC~UtM7%P!QV^Jqj}C;k#->GSl=!$ZL^_^Dix@-^N~PBLsGrk>M*Sco>BcK?N8P2M?r!a z>F4RsHCNF;!^!k^%Ujg=GXyDz^r3B^jZw>sFR1sj8amJyii(d9qWjM1@y$*nc-6s9 zg0Y(Q;+uYB(1(>(;%D|d9OiueC0=YZl@j#_V%M|fXsGV5Y8*72yPc7MPRuD4Kbfd4 z2wGsqyLjEC`@Y?z*J2_mU1`McN}S9)ZdTz|VqG*PXbRt;bC@>CP3F%|Z=jC4jp9#u~}M+flpkU00Gc+Dhjly$OFP;(7LUli#KY zCgv{{A9{aWHYe|){__oZ6Irb?YwIdBRQO38s^TcFt$KmnKmMg=Be$Z_h0%1w_KAq5 z-lmr~zo(_zKe?^O zJ5>3izg5X}Q=Kgue^)l6uPsJ5Zt2ok@lyWxp#gktT@U^7WE^#!5QhG?NX6=-chlj| z47gJ-4LFU!x8hEj-t^G4f3$LSJ=IQHEJ!$8K@*q16O3wkRBgIo5ErZ1U0oD!L4^^H z)#p}Jq8B*EQQ6g8QczmYKKGB z$0K>%UwXzik-QLlQX!y-zU-BvwzWixAW-l zY&A4{h&!Kow*e_{IVMgTq|6z#hti-ZW2)PgEB2E4@uK$Zq?0>4>8cef;>4hF4r&K3 zA)`5gf^wfX=tOZK?f5pAcaZg+x0oiPfq^s7>I^lm&T||sIXQ;=rS6OT`~Fa{mh~nb zWoPZV{Zz%o3`uk?(Eb83DvDO;`0ndS7@|Up_2$#^H8FzGH_oG>vA&$<$4xYQz<9wA z1ug#5qx1Cq;EN7J{1TD)q%rsDi#N5sv6YLR{D!(M|4LmxmsiOxQQ)>)`J$(m;dI%o zLaONY&f(&BfA06Xr|6<(4$W}sq))jw)F)PzD{|83Ev>!L$nJ4yfO90heYA#d)6|n` zk*-aBdA&t^IQ;^}yW8l%qdSo6Fdfe9bDLQ3MxPH#ZKqB-tpfXoO2MGn0-o!*EO7oj zn5&c)q3MGj(q-NH+~_e~C@D}=;QvOB>#f*8YYfg)y9x7YH0rT`2tV`PPde(|H)_hV$r+tXGh<*SLx_d)i<$F>>V^7Sr32a0DIKAQ-4Q`Cy_Eb zbGs!P`_7qjJbR9sE{>?)rWR&j_ehSPcG`fKZkfdA)fNc4oo~^(jaGuQ1M-nqeGS!e zFQfm-^svg1>F~PkEZXX{l0G{hO}!4@v45hKiNcm%LDL<-ARoCWNSJ;WJvsx`CN~E1 zb-S|Z^u1N$`4wR_!+EcG#OWlY=QD&GSr2sRgM(=6Q;9>MWDH8agJ_I>3jHC|lIP3r znCVj+=@Knf-tDG2f74c;zaPCG<@Ua(e`~IxN2=G6sZ7WEpzj(Y|IHTMFG~^69&jAx z?!1n=7sVs5y!)sy+?YSVWF$&1>!hYK-ShpBN*bdlb(k{t6;;!k#=qF_P9NR7EFOw1 zkfYs4!I;!0dNfMU;b&u8rk=51PZ!g<1v9IH{O=WhwB}+TYXY;e&W{le^SxcY|QGdjc9=?uZtt z`5=>vC(z^5x5fL;#|v({jzE1@NQ!nLCT#`n)`6zC zdivbR@9E;zjasPc>~F!6_ld}~fTL$-)QdaaTWP@65zx+p8Zh%ay zt5i-0Pp7m`eGsSgvw)tr$#Ur1mPEO#0sP<0c85hDCegKF*8IYanc@JMRxytZ;bLX) zylBLFvA&8U8lkXR@S%1e4J={w_JVg}>og<&aa}!ak)72Nm%VoTt4DKI8V3BkwNrSP z=u$yd#%-i*{t^8;dtO{9)5HF``y6G@+RQtj7w}sWmHD9$O?cPXzvxoZIBwI}7UcT; zJ31P{)6IR8`IXUi)Ns!LK3>g~+x1JtwYNPGcPy0Sk8Snksa%?P;I{XIkB9Bhm$jRD z>0T|~chVq!%AD({QsTpJt(?nyst%%ynt#)CqH|)~MnN^H%0#>C-U)o)+jGiFkI>M7 z2D)bxrdD%`1IZV%{y!fDhXIiFUWnqr2Su(7zKd z+>2BD>GLV3+&;fnI%}I2zj9kOl|hHO?9C2*_G`?CpV*2ndX41oE;r|%Ot<5dd=4Vl z_cAT@^#HEgZWHIMcNs+`81j3c@8z<8gmO1JehGG;5^(pwALKivyZO7?KLpkK8r<>c zne^QB;rzhwd34ykB{Hw>IzFy<7O%8NrkRTPj&>@p<@e{z;0MobK^uxE^K-|q<>m*5 za>+6;ec`yP=#BY{qovj4(4gAo&Rc{4J(@JP~ zXtwxMPBDEUP~_aNU89L7YiR4wRmkLbnXK>a>=0mbTkttogHtxxf#x6mCiqt+J2$;Y z@%&R~exRE%fB4`TI$)SC@9pD{zE=<7+z!>EW1>bh_2g)W6Jz&LgO=T>J-~~8)igqb zmY1R->q6(BF>q<@dF_{|N-hdFAz!JEVRjas|$O7sWyt0a~9>%XuokQo`R-^knpHk~GeSYRr zeRMe^l}_7tSD?L2Q9O9CDzC6yqspj5wy#=zMT-)}XsP-WWaKb_S9!jb`+ajcw=3#A z?HKSv@apGM{^DeNUPWh&pz6wO?r`K(>Jzt`AHPhFS1r9v<=6S~o19ho)=k-h^k7TG zubwYRm^_8Mo}|sYH6IfDf7r(BwaD{se;M5F4wM6Z0lvqceXC%XBpKay`dRY@y~OlwDJJ&=C+EDTrAJs z@aYrB8V=%iYu-S23K%loSwY!A0YCIsI689y=*kmD{IcK^XvLT?4jnm94|N}+Kb~*k zCiyMpu38&%=Z$IwBi(m#nj1u%s%Bc%_3}CN=-&W%#$ zl)LWnH!vHQ%DB@XjD^!o-tn+`f{p=f{p6>)jX zMD9<3wRqw@E%bil1;HAvX9zZjI!H9y>3fCmbas-JV6|%tUGeD?HBal6=|D_T#^S$p zT<{eBV_pZX>XF^U7JHyP51@R@0YPx7BLCgW9=UyehX%~LfnFvX@?l*?;^>iYsIrj~ zFRc3~J{@}uaeMy>*m`YFKg2_DqU|Yi4Q`^=zQMF*bfn;ecPqVNQ&JT(IE%)9`H0xH za}KwB6YZTlABmU9a$4@+R1;YEs`0f;vg}jlBb0M$GBW;j8kH!WLQ!(RQSpK_hmsXR z$cnq+uzk>2S+>v^{z|?kHzF+>UEL5Nc(GcY^DW$uE-kG`i)(G^2g4zJzs?|juFoHF zk8uhi9kG^~^cX<3^ z4Bz@`K0Vf>z^z;tOf@u2WV)wZan*>QRH;GcK=bDh>>2?(7+JsJ@)8!|wEk>#@$8ziEpG0mNQfe2p098&rNS&T3^AeMD zsARl~tj0}2X7q<(>GwbO$NKK0v+`!#9+wyzT{MhSRL`c9+}ml#5(n|#JF*;^MJ0k; zPfcm-ra}BznWylrVF7wv{TU@JQbfKs8R%zmhB)(zRIns02^qVpppzC|)GNJ9?Dq8m zI%H*rlc$)wd$kQlswZ;jtHFnm-C|ckOVNYsoGrmrd3`M6<{5H- zUz~TCH?0Gy{(VOsqe|$NEi zdOgC$VaqbaHY*NOugU+R9fwu9)IeAIQR_XLa8#Zf`ezc_IOMs&VoQYB=$|_Gb?6h+ z-Lpq9GiMz2kW5F>r<%~N#aq#b@)n2OnmM#>_G+XzU0(360SlHV7;#-gR-+T$6=<4^ z4N~857-eL}qvxM8P(@A-eeDx0j=UToxc)-1x}*58;Jv&+e5yyEuMXc#%U5Z0_uk9= z_zuH){hM>dk;W%c>Ac6NahEm!oCVQaOLfp(Zx_1$;6&;ca+zlAJx5O##nLmYKF}87 zEIR`mMb36qr9;V`F0?nO2Yr!cJKWnM%Q?ELz=szw@t3pmoWhq;G_o`D5k$-LzZ7pX>!!_Y4sY%k zhrUlOL(y)Hw8wG@eZE4K8|6XmcPH1QjqIJ^pZKTPl~=0%_~JIrmd+4wU)w;xkSd3g zQr`Yyz#w|)YP~=s^e9rZ(x!_%WOc%`{YYnJ6q*vO!#fAfq3+G9f)8CIQNUAsTHtSp z{xM4$o@0A+r%-;^4?B=-+WB{$u|^VE%g^I-&fj`a#V= zB-XR!zc+mqSG;NKts`=a1)dlX-gM;@j$staRXpBN zuN_&ibFu-s?;_)D&tv3d$RRip*urgDRxa#!eg=E>t}>BM2ODwy1Uc_>M)K}LKC#$+ z3(Qhe$aAYT*ZkDU2Fh?{*k25D3>f~96(O@N%+`mdugiK zX>2k@lgMjjLA{bTI28?*y6j#}e&k)odOw0;U#v2&(H=)0D%HXww`j?lg^f5y{Fv2h zwc>fHc986r58Y;|c=dxu;`2~lH0yW+tp4$e+q^!NeRTUux^vfq`q20M4v#3d?Co7* zpm`07{O^&4fBJ+y;mXiu8Y=8ejV60#TzHZy%mocmmFE{!0qJmyk!(u0UN+FL=N1k~Cb1BtM%5 z62l?)Sjzq;EV|kvIWumwaKi21LaA~&JDDd(mW^H@w4L37^Zm~e{Sh+u+46o;b@?J$ z^l~4pP_=>V7*lCULIVVqUx9te56SBzp3KcPjQFwfBwEIsyyf|pzcfS>GT+Z24FSs7 zquPjE9Dk8~pCcv@qR$gQttfn=BupZ7uLtkabSRu>29=}xSWI0OSugwiVJ;&`S<_%L zDJ52NIxm{F@18<@s>?uOU@xA3Fr4|z49A(WVw`BTj~v%s$>JY8U>!SNlf9nh%(~H!y#0F( zzl38rz9yTf|Ir7B`?Emd%yAgFeJC;K>fl|AymY~~=L{9=;S2G4k^A{~W%Y=7lN86JhqrS={5S3o=I21-4@2YZN~`9#-C5j0>Yq zvf2O4a7>0J4oSOAL}Mf@*7_Q&pW95}0qwu#?#!`75 zER1&7Bg?1u;ZU+1ELuL|mDl88#kJ`~_dkwYZk51%@o*BCy_oboxebD^a?qdghHQwA zV9o>mNaUI-_Q86v^t2TbZ)B0MMoWoVDM!IV^K9b1J^{XC0j^)Nk$nIBjJ&UImt{mx zV$S3o@tQ`#v+O+7FKWb=Keoep9W$~%z!5?Y#gdUPI>8Z6v9(|J!FG2olGeQ$8>Me0 zpZZH!#t0MX*XUVfn`krnuBj_Jqbo@f+a$D>&d7H*1Un-$j+LFWuu`Y40YyhB9TIYbnivhZ1_vWGNFWxxO0ubrFW9U7O5~-c$A6EO>t9c z0#<8Gg=_Wc#JFfJ`CR*n)K{N_qs_mW(W_}>pVK0t{KyUn`J74wH5-Z9gx6qmIu{CT zC&Mw@!QgsQMS6PBbgAjzQ23SpfN9Q7z#bY8k0l4_P8Os5C^;EKtj?BXo?IY zHyTfp4;>uau#O|<4WY2vc_#NKOpP>XTw!&IUvTsAy?Bsp{frKqL5@rp;v;`%!oG#- z(ktNsWJ!@N$*VpLO*Y!1FPa7LF=l|MC5^F;@w%|-x)Ibac}?_%vj0nW70lQ=ixn@7 zlbq{SA`QH~bRaX4&K^4lZ=0_zjs1rB+YjGL>}#A!nOqE5f7K#8mgj?X-A(f3y*nJs z2nHS7wQNYWDfu(N7n)Twgja+A!y^N4;(Kndh*C#A8#+0Z_;{(27E^g}RGdSE+t0w+ zSUr-_Cjr&JmtaV)HOXyu$DfNo!{!kmfu)#0yh|a3OqvJjn!iW}24WwODcrIufJNt8 ziN?$|fFD7D@Z($!Q@J^n9MaB&5r%4T(!>G|Tn->Bu4qEnnnH4*e~fUaZan#Q^bGm7 zp#aj|e1yHda%8|73-Y*X6Z>u$0IIcbaju#tBn1zY%1u>~wmIla`!Ac4&s!cy`W84! z(nUFBppi0B?4Ck8v=72`+6sSq=L!`rzb4!Y8MEa=zohMTERlSEK=|L*;Jf!K*`Bfv z`hI=G*g3zf{c*XNH ze7rhXh&xUZWv5BPHzs~C>2Qkh*2l#-rCiFDuZ$v3YZj3hT`}=)Z)V-yX+qJ`M7($H zV$%IeN2+kb7V6BzY(`%^J9MG~E*w2BRGj=0i;8wJNyIHl+>}Mowzil1@he6WGO(Pu zl;0DE*CU~<<@uwz;Ru`qE5m&tL&HNOkb*GncF z`4@2SP$@iH9!q8)wG*X(k^@KIBjk6@R}vX4%V;i{3>)NI$&-L*@ZQZ=_)2R#gknc* zF!LnD?J*Ny$nl zn5rB^9{d7gtMHn9i>oFR3)DsJtuZ8{HWe$FKf#j}uY!*+WsNUwAv4P#e;5}IwQ-2O z{=Ja+EILj)^>+}>a|NK`l>`+_k3-t7U2J^xPwwH~`%L<^o2{AENc0RN!96sbndYs5 zY1#%PbKq2V%RyPXI$T@$0HRiE?CSZcfCQH0r(mRygSv-W~-Mog6X3KaI zCasdO4pU*QaThiltt0B3Tgyo=jf9a$^O)=4HWDf0_`UVN&6-LZ*_D~~IO+5X@+)~U zx&6kF9N*Oq);rYTPlY_`*ImPYJ`Bej@9bqd{$t6^*qiv^jWoQqR}JT{Na6-uea-y- zgvxj=6WFh72e8%!V_{I-MY3Rqzl z%&rw|!ly92BkVal_fQ$$F4iPL3-7_V0|_L{Y&u|M!2o4Zho_!MtW=Q0QDq z-1_rKaDf{pO^0CY7I~?!dJoz3a1H3p&B9q+CX9c26dYm_K-2LHGc`XCi5vg2F$a&} zX4?Rg9+d?jGTZUXHFoT`feE%4s0ZpY*7CF)kHC2LIdbGvA_;9!!r}w6wQ}jA@NsV> z`8f3k+hMLsLRLM-1C9-Z@P;?^*ji08d~OjdOl!k69fxo!ZX(mFet}I>3n`I5PAI2D z?wxuojOghhOk_$fm-v#l1bJ8&dV-7{^%l(3ZnHTqGS-ZB7nz;>iwufu7A`C7Bg*+Y zBsS?AWanOxEH8YFje}&+`=MnlZeD@#(|;djqCFjdlB1hEKNQrtO%#C?T@)l;WL0wCkajXorpv953!_`=`u^O8p|CA&=Rgxa=-wy*X z*>I6AL&z@5$R*Rqe05a`M5jrGPpz7W-N%P8Z*MUSdk1i3M>s~A3gEHz2Kl%dlZhh+ zVd*Pt^4!B2Kh1x~M&6HM*RDE2&y7LS;teu`T&*u@v7Sq&UXo|ihonI5R|{CQvr_16 z{1VSPU(FP1m*YL2&bZ)qCAb`o6W$51gv?ww%-`$7e(ky3=kzaZM7yHY@aYZa?KDN2 zy3ZNY?4u;#rh#;Q8AHZ@ECGu_`-wvo&la0Rey0`Qo(VTZ*IOFg= zw)kL|j9F7kj%tj6a-Arqa>$H)csLin`N~Vai%0RX4O!%Kk|Q3LWJCTI&BZ@EkHCZK zEK*wJMEnAtG5^A$nD|{~dhrEVM_-vlDQiO5_#G1dZ4*8@q6yE_8-|bU84EY+&O-9| zVR+vlH@3wmi)arXg)Kihkrd7AB>0gl?0fly?0EVgDSiJBuf25y9xfhE)|t4$7Tbkb z%ls?}Jeo!VFPW1Kw`RZumkL;_d7NF{uSD|qjbk|lrts5r4#-J&lcy0+%4f4LdC{=>p$^==bCCJJ_k{AM zFGS7fE7MN#B6Al$U>-H|$)!OxpjLYfG>5jb1<%&8A@8pdmZ06sKZ2Op#L=E7$D$V(@&#hhP|9TRPUmr&@Ro((OuTjQlp2n_yjw1iW zGJfhtS*c_dz?AE@!^%CQ2=AZGA8?;6sohGTC#MJMyQ|pG`Ti193sup+9V)EjkOf&i zB^Q>x*nw;B7-P=^r6lmBB0FOzg!LoKK;3{6P5pInIR~>tay{&DnpB>6!O4vx+yDWeX;>hFtjXX*qpW>^YSp4!4*Zmz)V zY_76L)AI2nO*_!7wTCI{ZNgvYQemJZi=0XHgByCmV0iEpEYB?_x(lzteTjm!YTF{L zYyAfQ@yi15!*B7`Lk6OP(3xQHWu9y;889pF``G-IJbv{1Qq?oQ2&Bh838$Odlfsax zu=dj^QXcY<`C7FRKPh0Gw(9|>WU3-iMvG=-ZavK*uTZZ zSYtPAvvzA068u1WsOAi}SQ>Ve*+;wy$+8?0cvuoo*CG)H^o?TN(j ziWZ(fBnW;gZU%p&ab()LLMXZ~2O%+;pt~s>hW_Kga%vax+vf-t7H_c(5GyhjJ;$qz zq9m5ueZt*$b%|wX8XWROVC@wTX{Wd1NejY>^zj@LI#l+1^lk$UW1-}AQ zpdblnc=qELBi-QQ$lpxuGZSvOFcGw-m0`}Nl6=@4z@7}8$XbXFzWQS}%fE1)JMvMT zq-`7xuDMnGpGZ|$x2{Zb<9!uQJSHy;d~sJ|^XLID)));17iCMc@eKo3 zu!)Ey&g`aF0JD)XZQOZVu>Nm|v|y_e3^Tut!~P2v30I3~Y3al5{RgW~J|5!PGCuambTwa@?yDw$4}!+_M{! zy2b_;dhWAO3cHAss+;mhcTsb2Z&|bU^X-22sszI48NW{NIEjszMZLnM9W7XDoJFTs&vj9<|@FF z^?<_^pOd3djBlqha;wckn0fLO?0k5F97s$QUM_zLIa4Yj<-EGoc#jukx8xEWAWy#2 zOe_*|AiQ`A&M0`}TQyV29-Ip9Q*N^fA@wZKY7Mzj+mB6-r{jj*&m^Ti#_*=YmsHz~ zg7(D??2Ahbv$!k5u7R0wYd*!fX?`%qpd7Tm8b~kxt{~UWjv^B*WM}pXC3be#5BMo= zBHkIoC%K#;>h22 z9k%$C5Sw4pAp!daOShNj64gn6B#HAX;F@SJxVT!tt6m$4^0S?=%Iht@{O2r0?H&i3 zvNf7)c^ju1T!$vz9}tG8kZEpFFkJgH)PL*1%GXOF#q%B?<@lH-OmidJ&35GWB?a(; zE{IFqjfc!qB6aSsV5{9E;<~z@O=@r@Dyg09_z?x^Viy_Tyy^+@>X|65zi18_Tl(0- z&t}B+X)v5xdl+uKmcuoD>!8DX25Bui4qjouV7K8_oL(70-X#3MPew0*ZBY{;yWWs| zX^WS%b_Q@ont{Zs={N}OG+?(dl$4#C#VS?`xwC!E!aLpXSz)j~SSC#(W~#H`W#v)W zsh~-Y*cY%}s}_?_xr0IO&tc(>uqU`}{~~fZ+LKtehrz1?E21>82VYzjO_~zIgje|> ztQv05Lbl8%N4*6Jd2Uo26 zMm!!ELeP{*5RGwzB&!2pJ1myG*>nOXU$n#0h^;K`M;sx)Rmq1|Klu5nooS5G!&Sd-f8Uk>QEmOV!zGrEV5* z+YAeLcrc5tRWSd73OGzV1>Zt`^Hv*fK+v>W){;Ds7~NBsW=#>w_|^fC@Ol({T@p&( z)s(`pb51Nb(~77qCV*ZT;*%i@AUX9O379V@RsLK7f0wQy$-Ym?Yn#o)>47r*NLtG3 zo@)rrb3EZ%{8v(%*GwMmR*)Xf{vf=Ra{xbheh7>G(s8%pTj*F?NlGQIus||{C{2+k z-Fr>g>-EFPgPot5+ATGRba%xwTs0wkX9xtgn9Er2DJ&$zfcti17k1C+V10u_*_$`# zV9?wIHY92ubFx1|0s<}wZx;2y(>X`rk(NY*2llTLc> z0>-ZQC8LC|*cI1g=)aK*b-QLl=tV~uNBU)VnlyHB$O@Jgl7v71Jd16L2Z@{(^nvE^ zsW|tv8-#hbuzk<%VAd`Z?DkJqj~31&#=a56Narzh+MI&!%x2v9EdnkZr^A)Q47NS% zBI&~_h034T!_TG-P~rQGyxpb_(x7i-;V>R&ta~N&a6Ct@?9nE#a?RO@!|7PEeHm-d z>4lM5vWjoYdh+Ush|PX=m&HeA;)40tNn5l(Ink^jHUG~Eu5VB0ukHu|I~NOLq_RpR zAMl2iw!FXrnLn_)tB2@SygeUDevx&AH?<5d zowXsSBNvh3?KwECJ`(O2p2vRbr&++H+wgTi5T&e_Y-e9S-f3YjjgDW0HAdwKpB8R{ zUd2wfN7@KS%JxH&+7(u_ZZbJwy@KRklC4MgG%{ns91{Gs06g7Xd1cE;5Z|mN=>x}; zWtY}U)UKT&rzhr;S>u~is`Nq?YpfTph~pr`FdHXKP-oJ^UU=f$6p4+(J-qHgJR9+G z6g#yk1RJdI#_{58vcBq}JvQ&EI+$f_A#);>NRN>*9C1EO%r|_6*u6_g_~H3v*KG&r%6m>0?vn)( zuN?{N>>d&P-xY9CNMuEMY0#t;0h`1cqIqKvSBFb zo2rS9#*75>J*&v)@~PzZ{y-8KcnNlzhsgW{7EF8Ya8WnWCRss>vRTSCHf@;#G}g*9 z(Md(TN_%h7}ja12mcb^l63V^Ph&0{qCQpv*L#b9Q$(JrCd5iR=rCn;GBcFdr z+KrS%DvySd8*TA$d(;P*&W^InKfBqaGXt2pI>JhH z_JNu2ac)CQ2`&xpVkI`~L2Rlf{cEvN^53?F>f*Y=KUsR8~fnMdnK?h4ud`uC8?;~08cI9 zv3bHWJYKYsXfDhq#ArOkDmucax_Z2F*$MEsO@gUIez3}Bd02MxGz1OJCP&7uAv#@w zc>d649Ah?ETIxIx)_p&Og&PWGd4(0Y&cYhLna+msvKr^k^fUO`Pg`Ph&WuYQlz^3q{|z&y(1y zR141)oF{L#d6WO>3yD(2RCsXu5DX4HhNnLtAe!E$C@M3aN-BlN@TO0e@Zjo2{M6+I z^sD4bqP7aiemg7C6P5y{l3-H1>;nAu=_3AnH!y{~*KD==8n{;) z2~R#&5|2nF(Lr_zHu`qsw-$rRxb1P!f9yTe_1Vf4Lv9j}cr4tyzZ$09^OrmhoQn0v zDng1Rj%C`0!kxRGa0~{4)VEdgw`v?r5ha6G{&|u-$xYJ#>JT)<4F#k57O++rNGh*6 zVWV0NDZXC{lV_NbbMaYhdzuk=PfBOYpH!2hwff9A{2J^Ie#>kdCll{2DNyrg08|xU zWWVat;FV?%lwZ_<*rVIX%Gv*fW&1rycDp^^6Y9;1165(u*L!%Xd<2XsUQV7=-{X?& zcEEy|qr^+;ESs_4nCz}Af#g4fggq)V*}uQLN&n12u-ZEqw!X`UUSR~W9sXW8>`Vr6 z`a6@Drj8NG<*Spu*BxMrY7y~RwFDn}>cfie-{71A%%J|&LfBCgKw6RxliI1z@$`!k zmJYyQ#xOWP&Qy9KLJs5;@`TD$BS}^D6L!G99D+j+ zu%lb=k+a2taCA!rkbCj0Bk&f?{{0;kr#h0mUL8d0#rUSL3s}*48{%G<$(qIm%H~=> z;J=eHR;u!4vgpJdp#Cq|l7WLsXWDn17%vY$uGQoIN2)L~F#rxOOputbyNcPEa!3yz zCiOO(NM<-_LFCCjIId_p%q{d~bH>%<*zq0AWw%6lQO4F?Q>iCCTz&?h>~;FDXdVB=f_$D-s#=J8vI>Y;Ub)r~rs7Cf5xJ7~bW zMJ|M?P|&>G25NPyaaELxbWdwGt9DZ$ru(kKb8}00eRL~~^?ZqkNadjS*&HZY*b2tQ z+X-pwW#4QfKxgj}aGll_8&EmT+!2m1#_F(iG&OsbaFNrc#S20s5AK* z>$T$0?cvjqa!fUR^ zTfBPN0qw)!m}(5k?GB_`HcR%&m2s$7tFyiAG@GT{FUi<(o_w|KXCSi!i6118pF@w6 z50w`s@l(^uP5W@NCCiDe*^-AB%LEI?=k?XF;CZ{)S@YkLUD**VZj~juUzQE>r-sA1 z#j{DS3Woaa7s!+Aws=is6@(3EY`sKY)@KVQo36YkhW8m>ZW0Jz=O>ZvSx3pSC83f{ zPQ&rU(6M-MYa2kWE}Q-?gEi=-l2vN6pwRjv8F980kJtFX%8e={zOosU<<|i6eoBnw zS)Vg*n!XphhG;=Rdov!RKnM%!#tVnW!mB!UQGu-sS+viR&Df?5i9HUf@GCTcXR<%_XPOTgY5&JfhlmpEyYwuK0VRM;B+BqFM*3t>Z4UJNB#_&xW#pjtaOhI|O-#5=WVO=_(86)#VCcv{4s+AlTWdmcs`jeo8Rd;&LFk=Xa1+?%)?^*-Z0*(g=jCSXw{-s zsyS!Q87)%DmL()oBw11@gtRIpOC_mL(W2E>&3WfNL!pSQCEp|zEr`gL%5Q#u&s^8M z*EQ!l@AKTx{oEhwT)SJWmhk{PeW4pLFsmo|BZp=`^kwSmeH1DkFobN;- zlFmp%&j+WyKF%`lO4-OuC$NNj0Csiy4_o*7;Ls&o=-HMmZcnQ%et74LAov*-)t)nO z)UvZ|$$$)fe>sl+Ff?ap%sEVLe_p5M-}cfJvrXuwNW&Qkj>U1FTX1~OMpi9b8NJ<; zKm#zg)8#65_&hHVb=>J-O+FK>Fg1ln-3mqFbzkw-KN1yw$nqgDs=yCAJ(YK4fmOs(IA|OH4JW| z)$ly6xp`6m_KG}m=Dw_-(j~$4w=Z$!uQS-AGmr`2&aw~k*3-W3$@q_jfL>I6ic|v3 zaqDIW-V{e={5_dLJuM=?h+8s^NW4h5#wOBKr!soH`!MxBAuc@s^tGV1wHB?KtRnPs zet??C{n>xsZn*ziHZ~sLirzfo<4pG|yslt64wQY115Z7``nwyD&URTm>HR_WU6~Yq zntlfzbtUM@hr<^&jpwAMLOpZ37qrZ6u;b|BrH;}qUwrCXmPqX<}5T~ z-r+sy*ya+EF0qNGWsK09%R59f2vr;w`Ve_(VcN9l61&AwilrZC;7|8Xpx|*tZU5WK zK8PUn@S}s+eTxMBC(_2OH^~SiLzZ!`CM1cT6$u;Y1iI2unpP$srfq`tF)FM>)QF}|3-d7Fv9_E8$5yBirU}yq z)KJX>6Rwv3aq5*7izH@=X6?Il@tNnxse{{5tf!btH;#GJf>RTb^goeLMs6J*xVVAN zT}8QQ)p6?h*q61&6pvqgfZ7*f0EXbjVu(- z6CPe25{m1N8}TmZ3fO|u9Gtc_9eIN%;l*LPq8Hg8LAy~s@>rlkCyW)-Q`|jHhwts8ar}64GTsY^i*&F zi~Tjkfp3Ejj#^IJ@J>dZkB9$lClcoIF@Mg_U{l0un^D7tINPjnU+ zpaSJ>I4bHqp4F<)bC{8g3$hGQk^eK?n7kB=pPz*PEIlGR2MO5aXbxKPpdCBuZJ}Pr zwefqWWPIyPJ)Rf;3-KOsaa!6bdR$6csF0b$HCq;mR=kr$A!wWtWOLYuPj}IKj}{3s z87>`9t>qrOYDra3uEve57do`f7M~e-hZeo56b$TlrQWVF*tA|7Uv#d(Z(nAi%OX9| z?%aOd{yvmiu4!dOONaDQV?W8wZw8HSm*$=9 znui1Vvv8uZ7;oHUJH9^hh?rJ*;b~i1uv~KrKEC%I5{;3Neeo$4E))IM;0xpzr7m1G zTL&eYS74XzqS;rZJTHR>@uT#O{AtZv^j(Q5r{c~;T2WPmei;SR*{++Zxojq~J?zYL zqkKB2?J2jCQ$U9u*P`Q*{&ay;8Y=Y<#y$m=)b_$ju5Iopx;Ugsd!-Yw-1O7zq+_9I z4jV(A4{NjYzlrhgO?5}h23FHAefjuaGe%3K_E6;_cPu%kN|%1UBslhcgue8cN^80; zcro+-3S`EvvTK9C;-ro^^v85apb~NxM~~b__Y7mHbl(WKxMLD}+wX^UtX@&cY!~kJ zeL1ve6`?Y2Jhalf6xlZmsMVFLtlwe-+&a7qzwVY1)~rk9n2WvVo-ffyx?xvPPRt;6 z->u7KZ}*{#%~#on*2ihh(@A*THiphVD2?4eHNiPSh^?zXLtllKA>ZSBQ8p`wNo6el zt}jM|tR|vYGq%%d%fxZ#Kq~gJ7vo75Sn>wDF41ypU*venh*qv|V*UHhVS)2K4)b)RXvNq$f4Iw2l7ua;7qC zAJg>s@AQwe$jdUwMSIWf#9>A!@dG4_*Ne34hP$pb^zSgY?6$b@f^HT4FfRs|uDwUQ zFCP@N?Gtc~_I~F+EnJRiP!%&TTp5>`tI!88I+1AFLH+a^kiAt0{UIfZCBrMZvB#w7 z)D}Klq+de!4kV(;eFZq=*=&s6)=+WLJmHD*5&Ct<9va!{LR;!8Xk~#s9V@*>(=U&* zLnsKVW(wjG$R1>i9tZGn_b9g1xPlP)FG^_&R+JgIPAG-gw-BV@W4udxk- zK?$xXS)|Dx-5!Ah{=A}D+zGtcrv)g{=7m76;|A_=(c*m+&7j7gT*v+DS`^6lvq}mb z?0Ltjctva#bu1Udc>~3?aY`@t-F*rbd^f{a9={WWvIT66E#MwJhw>0oF&yDCfy$G7KUEz!((j%p?vTJu{F z8P$WDER=b)z!0}>{}0Abu%IoX9SocFSUksVBQ0ONA1R+JL>(!I*b}E^vGt7GR)gL5%1dE&)szB1=^jLjyt2c4 ze5q8KHQqMD>aXubk=iFkp6Xs?9=eO3zAu`$Np;fK_g#dOq>ka18?Vt|cs3jOQ(f5b z{RNdhX-yZY8DJYnC)PIeGB%(3_kBn!2wR@nTwV-X$yA zd*Gs|9NWqqd3Z};uyrldxBm;O7C%DUue@OIh;CA`D@j<}&z6lF730kr-iF>3o}q?_ z!555=QIB6ISfAJn*fwJ#kC_reeNrXpd69osTFI1tDP704S)74uYGw%E=EmeI0q%KIgOS6p2e;YPSTiNW2kB61gc^oBh3FC#Afxzu^V=~vGGeU z)5N)&)MCb5s(5iN`}5)>?wfi;+E=xao#yk4K$u{}zI!ohfpy z|HEzR4#BtL&(O!e&T>DUF2nf?J~587r_#2M`)TgrRLqUZ#beh@@pgX=O0R6gJ=>}T zS~N@`tM^XuR%1RL9g@XA=8MyEzi0SxZU!rtc^=i?y~^I4fN5B879Rd3CGuMoVaIDH z=&0zK&e?h3!T3!S{gmdpbU9PnY{d(BmWud{x+a2*yi<#8_ z^ECR?LXzG5MUv+)zLdIsnuR`0X+zGXNAQI=$55a0RXQ&^2TA(=WRa>1R^Z%ZM^i7* zWfMYim+(4G(mu$Jy)9!;wB=Lx*_W|Y_AxraL6%xSJAn^ry z;+%>v)CAz4QAb%=`hX=JHZ z&CW8cLb4luXwg|EYP?}PO<5p^10L+gs#9iC`>X1xTz(=iJL)!T@}U~%S8c{V9@g~k zHXXb!B8ry#i~8`72e8|LX}klOw%mU@b?8xEDH06MfE^Qe(%vf*@Ol0Nu0#GyY$@7A z`KNy$RmrFdUK(9sFCTJat6#*iov}L9y%~yTX9oqwzaJvC=wy6U^Bn8nxD^+ju*C&N z$?Pfliz zp3LJWNgyS?3bs&Y7M;536h7a&mCm29OD}X*(wZ-&?2I}CRPS??2FKLl+<*G??O`X3 z>lNwEFfUl%afa1s?4WaN9gu0|HMm90rDOF}u4}2l#@d6{#j%>QM z!wZGUpXG8KRnhVqWvseC2zQhe(Va=2Tz{udtTb7gH@HwhH$F^aeRdx}ZKAtq<@W%5 z`)wJGo72bDICqI8-xOolXPm+c@2{cGhzb<&^BsDpr-0XM^kUf$Q@C%rDeUV7Mm*Dr zT2%co55KGFz=mhcacxm8*Re-{y~HH2`^-Xw6@_?*&ly_ITY`0!SJ8@&Z0x^Tjk>&f zit>UAaq|x<(ChcXsjmxg?ACQ`q1Gp~US%mNEZE7K8LUQOOI~4j68nHTWu*VhM|BUGsZx=SC&xl%n z*^25W-VuoF1)^cmCGD^%m-e)+#ToIN*qoY3YSNs`>gJi?74tc`v+)ZG|6IT%)mTxN z!W?up9O8@LZ*jGybLb_(P2`teOfT%JqLYg!(a@zb!h0g`gr&h-=HIEKI4|N9&JI<_ zZ(1&~wYp{K$)jQxZoLLBgny2omE)u z%D%}HAj@uZVZ>C7Z5AoeAfFgo`=f;2a;=vJUsy+<7B1j^7l?LhqH9@Ol)&!m)S(ey zqv^EU*Xd<`1?`NgM$6vr#EaP*bVbKM^e#?Q^uPXM?MLU}lsSIX_TVB^YvRFuT1>cB z2k)?E>}hH+;|Ok*vY|Q?6Is>%EqL&m7}jfUq@#s-G$Usob`^TC4d^fqHhzUC#GBHe z;m2r4hBS^>$iP#ru3`UYilSb1j9YiklFr<4hW%4+LJc$iLnmXm&`AwVc+`3UjW6@2 zpBKjBGvFY0b-zZ>xh)vhO_Ulg)prs(@wf<*(t!wb`ikbQl%oQc~f-8iCi(GU{|^1>MXYFMaS&n+%nc&$p z0e8B&&;UJe{Lu6bTmNr2<$EPBDk}4AQHo2I1h| zbNF9PF@A0kKtF`tVbkXSK_7n%(S6ZVQBGPYN>xkXT6&hSHT*=n>8ma~d!3B1>XaBx z5BUZ^_dccu#@lJ^r83%YDoL&7#d&rQ=dxdiTkxj+6NL5oGQuxW7txCR4QRs(75egp zEpCyP67I?~!!O>7(d6ors5?iB=KFiHJDNkN({Eq=C^5~4a$8dH5|PIIA9{~gM;qgVY71DE#W|wA&phlq|2G=<+Qq&%68S41 z{$xkqO%)DQq|hUKN^#!z*@AB(Pw`5}yR50!e(uCy3c{k@4uWYSo$1t}B&?{;!KI!C zba#X{mAFuhpFTIn9d9+!c%CHBWJxOf=6W1DH0Y@8%+@jT&hEs_*E<3O#&fiV{5t1L29)ve+x0VQ*NpvYS;s@rLr3 zbv%b?`eox`_WfohWXYbc>N`wpRPFK)pk=YA@$Gmc(& zj?(zAzj1QedN$`{FpYel!;Z^6p?6ZBvf6*F>EG~4JeP@-MRC0(U;8|McI75_U{VFT ztay(7sZz;C?I}drb2yy1cRDyGLJQ|_vc-;`we+&z50(=qpufhgX=HLq> zByJwP{t?iD8_j~;5-q{NX#?6Yet;f2xRO5HkVY3p*RgBb$52OZIallTAFQSyghpy_ z3XYt;j|Odd_!QmwLDcUj0_?r)lHg@c11qL>fc419#5)s9apb}otVQBc+LXEO%G3_WkOSnMws8vKccOXtLaj$t?Z)oqx8=1nb_fHHYz^UiYI)z1Z5+yQ_u7?L0i&3 zHn;gWjns8O$%%?0k7_QSb<2tBi>+Z@9vq^&&x(=4a5}z;E$O_og)}cR5l0Fd@RpYd zztTO448`wY-(Uweqhx^H6TO~Ya93SeH$5HYj*8P6yKaj#!o4)`3&Cn)2HY7gZ_!eH zdz7Hxhua^j2$R0Z3IF}or}S_uYj$W2!lffXd|XzD4%dhvRV#4Th5M*;H=;Rq2dMn0 zB}Kc7=y2O*v~tS~-aU!CG#+fCAIGO+Unf&MbEzaIYy;Li_7WT3Sj-LY%EoT(4CVS}bWSZRZiSDZA(B*#_7T!>2#X9Bj-Fpr=zkemWK{*2{T(iJVr$5r&l1-@1 zLk@?B`O=oa6!zV%XEY_#Ti`k?7jOPGoxX2XW^FUl(WQob_S6zR+BHL*XW*od3gtvP zhV=wIp(~I!yeYs^c7`As|s?bK}D?I?u(uOE|y6SN- zTO}9DKJZe;E=&boEpr`vgb=!4-wz!ssmE93*I{S99DKul3T<^4qamBh*tVeg_@JFQ z_VTjAic%c*u0L0(x8*o%e?^{WJYAWsIP;S|Vg6Z^wIPR3&QTPO8?bbsP!rXxZf2*? zSw@`<{qa|YyT~Pzhf1#=r85J9*h|7fdUQneHmF{Jm0u}h(|>VTdOV9VM)O(gpPsm( zS&hYwr%-jkIxP6GkQ*bjkuI+{M)j_-IN{ACEbusjyxjNFH3zrgiv|*O*yA6bfa+*X zN)uXF`4yEf{Uq|WJ5gbCp}^wp1@`9YI8My-L)7n?Hm#mf4S(#@MrZt6vFQ#g%FZsJ z<&8W#x}J-9O26nH?qa%MelB+7#KG&y;+T6s8%a-lkKQW0LcgC!a!>dQXxjomnC;QV z3QcF@Lr%%me90^}xc@yCJhx}B3{PQW_|w>(UPj#YAp z4#(n+Z!V&XO|JC6+j98*uA}tJwVmvSWHq{Gz?IH9*NWfO1yb!+6JFH66PSu!#4W5f zPgkalT`#d6FY`&n-_Q23hngk|`;^Yo$b-Xz{Xad}?@e)ZaKU^Y>*das=hjkX`xH@z zOFcH(x*L10SH?!)QfSyd4uu03xCIkZMV{+&dO=x6nEd@9UR`xa@NVB9D$iX*=PuOd zP50P=b*?qAi5u>*twq!6;CMav{4q7&;L|kv?`@o5i}hk=K;fc;wzSbl?9Kj>=Z0idR>n zA;()ZEi9UuZfk=A0@biw!USH(uTa7Ix)@qHm@c?gJ_%R7D8xHvr=XnQm3VpdDjcC( zj1vl%;=tAq*x`!`^^6Xn3F9Z|vSoAWxXW?uwN8v3@KR!fx6NaZ?~R}n^q%60t~01| z$S-u`urIwiV1Xv|Jf@2cW$2n$Ke?-JcvGdEqj-ErDO;!EOoI&f(Dj|hG=A9`l-eG^ z2JK!&f20Lbu)K!Gn6JeL8s=fg9bD>CwuPN#%@Im2zsqh>EW>Uhf2hHd)AY%O7$lVP zU_ZOa@ctW76NrMLaQ*A$+;#Vd1jD%}1rME_(AwX-4!P6d%aln?9EyK(NiBJ?M;i$0&iM|e^yuDPs-JGO14=O!hgi5An@T8(Yo zNAHycXZ*Zq*2#r*x#v=PRkaFxrn%!u3 zqV7&2LrMDLfwA7camqGOxL7{ob($D(=uTyDe!-+AG2G z<4^TKO3l~L~nVW7N~Z$lg6|1m?QYk4%S{k zXKDknvA-Fv*mw~?o9m5NS*zn;8yfMfr)yYry$tu>^2YPdsH2zdOL_a66oi-lt7N6^ z&a!9R`qAMdzSL}~Hdfv62VZ(~3Qu#9rZzH<*cbD=a9c+q?zc&1?O+R)YRP1u8QWst ztVp!C=p1g@GX<~ce2o^5w$s4wr|8Os!vd!D1eLes2#-b<;R_#9STDE+ofy`jh032X zpJ#|Hf?u#|_OI#P}y(AWk&7xbVNreY%JoZems%4m^lLFDMut#{G*Bx#wDyJV8&c>d#n67Lzrmu<( zahRVL{`6doU8bah0}oCYhR^lE%KifMVbO7nq7`{fgAPV`dAu zpH+LXl*J=s^cujxnpOjUc3ZP>UlridpjF7R==b_F5YLq>CHvZ zbT&l+(`Z#xJS|BW<0>yk$mx|oxcSo>d1pOk_pCdN5(iUo#B?Q;a%l_Rx!+QF_rpT^ z$nm^D{B2L)({qPj?jUZpKy|I z0hf3zU|r(&;azfB$UMLg$%|%kM?084XiA3yol{+kW8BJECe8*gxop9U$Zy5v z?M<{rayvB!_tBqUqRgG@4r*Td0RM>INcTl0(0jm`-dx#9yVNeAWur&go!T%FG(?{(z2VEicEIfpU}6^pi1Id%K=CGjrt6a@ zw10UIB&Y9U%DCS^ZO|5`%u|=?!!dBpwJpqsCjdT{7Gu)B8-a}>X{0j5ADZ1S2H}TF zzrz|EjU9n7d_^KOidxJ#lok`WK@YI|_Dxc|PZ1`3 zokf-pZz6wx>}J*m%!3VMOPMtLIwF>7N9;4Cf%hT;I*M3FiMCW`Q|@jkd&r6;UU~!) z`&}5h>3iX(tsD>%U&R0Z)Ew?>Rv}j`U7^0Pj5wM+mb~l-dYz_X$4Ew zjG2@F#lg*?{?K{KdeU9^n3Ht(Eg+(Ereoj4q!Dhg$-n zUuOW*+o%l>%l!mjB6l;7fB3+e#d?mX8g`Q!Q(tKNE10pw$)IrbB=GOtK@6XM2kJj$ z;fCf-oW|Kx;Qru;Bz5_GC^`u+E<~K^G%{ws4zFhfCq5+RrqbXve}jA~6OdpAtt*x@Z3`!hIExl=OTi2N{C8QP=lWXaJKfC;eme-0 zCm`@;RFcVV=8(TMncQA0qLgm<4)Rv-gQFrA^Y;W}cuBSiBq^L`^j;l>t^19cwTI&Q zv-Y|(`#nS~H<=*j{vvw@d|wKk(v_HqkaCb#GMBi$xCMf*Fl5(F9Y)o`5ipxI8I=)V zc;>?h=hEv(;LENo#wzXzY&!&)t;Y@%b;U#Qoq`3U-lQvHif|c?wUxlJ0$Xk6Lc6&s zWMZBrr;6)CzL~NS&P8}A9iq)ThV#|4Ox%et)G;e@36i3LSh=xKF2*) z9SU5O8J!QF97UBvu>H(0Plb3r!^L%A^MJF7Tym$$k zuZ@A0M=s3X7t7$7oCH%IR?7+WP6E&EOqd_pe30ee$GNm;F>~@_A|PEyN#{*15ELK5 zME_Y1S6`ULC?uo-tv#EGv+E|1IkcUqUp3>S15!*(c{zWqxrpf2{R3%RGJx-lc(Njx zfGNe-I5=;Js6V{`^hE5#+LJ#)sCx?dP-(}?v5ot zFOn=70vv}$T+jAnV0YSee#>$z*!kfaP@km@M`fRrBKdRVLy!{FdT11+mh5DDhBe_Y zm8r}nn=ue_e;bUw5yxaz{RVwRk#SpY3lH24Wy0H4iT#Z}Am@7w2D`0hdgT>i-;fuR zHKT((8!&>3v3bN^#G32ROo9c+Etz>m6XAlW8$gRuWS*OfLEB|=%#&HK_}Z&qgU-)0 zi1_2D;Bc8zZMrCz*?-n7=KYK5uqyT)xE?S>Ca*mLL-*Ok)QPKs`sQF}gu5NqDJwC3 zN1l+i3zMPer$mKAi}V9qE&PiDQlBP2=bz;5Rc;B}5e?3A2f{^B06=SnGG8SDee zykOw9sTnvOb0<~mcfnC@ACQ{8k1RG8F&o;IVA7Oiezn+4`0u4OTzSCKF*=8ksRb@| zW+&cp$cQq~sFNiwPk%TDmUZyYU)#-pdfo{BRd68xo_r&o?)7~C_8@4yPzC-Ck4SJZ3HJ~CXltC0NMR{Ds$&r2}o^GhcRnMfLMJftiC*h_;z_g^@v_je{&Hq z)gC81#D$#wJLkfN)`vg{WuVd~0pIF{12cPXCNNweN9-*p*Il@$AYvFCX2MAjTuBG{ zOPh2^qQzZu>*_Dg@B&R@-*JX`Joh212cuxKyeJ{>rU{eY>;igFJ2G+YEI&JG@T)?85jol zKLPaiQE*X7z{#sSL#{sHG8>m|X4XHz(CW@plF0f(!Dt7ln?48DuRTd_SxjLb=N{%~ zUwl=!RQ@z)-(MNBE?ojPUvXyi_8TyDH#m&iZBOVXVzX7*3OHb^1_asfL0Dl8NKGl= zILx)JyBjuvX!8hg?G|IM{+teH@!o-lGn&YzjjNgG!CfT%(-gRM?SZn*}mnbYq0 zCprL1J`96l&%S|zlzz~dvVi}0u>!bIaRI1Z4g;%}7%+_q+RSFZTJg&;FP1`;$bpXjezB*_eET) z>b*na5|_ZsvMP+ZZz(5nl>n@+*hlz_ z#`#ViTL@}yBlo{;VT`qmpnixMvxK*p*b5{{%M)Ej*X1e@IIaTE0++yV$txJ)vFT)- zPGO!nO=kvb`Z*ypJHeb?GvG3hX>dMA25jG8LZ16iVKSU!iN|>bz9qoHEYDC^JX zKiYU5D0O!Mg?*|ZRJ#jUO#e=b(wl%sbTS#Uas(xBJ zZ6Df!MDQzO7j^}hgd%2-;a&2i_&?%2zaH59E+z*`|BwS^x=gvVIn4I?M$RO@0MVL% z97?N0VZF31)4O*m>~WOfQ6l;CGn|1LRSUR?8;tB!=Nxz|3zPF|ISao7=9$SNI8VgrGq`bpL?lgMWVTDdw9hv1Q=2R2 z$+Z`pgK~CE9C-{j1=@qVQ}2NLJ{)F_9)N10$H3$}?VzW1B9Xf*4JHlW1cw%PgBKo@ zyiq#BzaG#_WOr?1{`^`4TXt?E1{?+E#g`>uqv~@0htUV*e(8MZ;bjBuay^;VMLtmf z^(;6|>L2Hx)>h8!KN_$;X$twN83AJw<}&WR+K?-24QgI1GgpZ=Tz$!hM8)=!=g&l( zZWl{t+3hG!Th%leWtPcr98`sSnkB&M$}P1SH+C@g`z_$}ZD#yP5#RHV$2X!RFoPO~ zLL&djojlE}1?dv6L0H@vUn$s$`5~VT8k20nJe!B0X^}O^P0@sD{G0p@o9_ag`j^x? zZ(^2gHizTEix|~Sv&qa14QAB>EAVQy=nmmZayXx|$+jdbP~E;6Oj~NlS^n-Daqc|_ zJi4k#Y@#o@9^OFO2VzAW_YkJA;xdU9aqA{t{zBa4rb3;bHc%KL4<_i`Ajg!=m=#CT zNkh(Gj(+J0e#xsezIXW(&TWo0^G;91R%@6FY)d)B+)x3=ij5Ha5-B(oMn|WN~>88RFi>{)|L;kD*^23PFOP0vuJ;7PY~{~{ zEsp1#shuOau~N|Ii>AG+tp+n~r5*JOf@t1*TKjpRw!GA4R&D725XV~&1l;}?&+ zGcOIsfwrVJF?^)K6tt9+`2|NwZG9mr_O^n>w~3=s;5!h|W)BT(CxHByTi}Z=on*pa z3(ZDg;zL_rl%3koV8#|z!D<8b|e3eLow-9^&kyVW8{{kkaITV3E6N)2XcpEIK)^C z#JgvKrd&OKeijS1>{`uyihT>tR2p!ez49Til@mElIcDTYm;`@s%PXMpLBxX$xpMhe` zA)>l-z|rYnH(>wyKriV>WOwFAU{$6GcYL$~%Tq-CvgsUVgUUwIW|v1I&s)NSVpsUl zVfZ(= zl%U602JyggNm=l@flCg!+~tpFnDS@cFmOy8l7V#T8?wLX5y(H?4~|MGGd~+8;pxf- zuw1(r=;gI=YEFfcwnY~~r>rQ&@1ZxeiZ=t@2@-Ic(HQuiq{aMPWCeTmkAN0070$tO zQ+V8F4Zr#u5B98;gCSKK%yBOzsB<)o$S!E%DD6{)HY09~%mEQE-dBUU6coTbw)KY1 zUo08dj;W+>NDcb$tm40mSqf~jtvTP<93@D^?oQE&CyLADNYXw}X2Yum&|5zVnB}X& zBu1pYtzHYVoyy7X3@73zRAZ*TGJ|o`^+Eh=EvT2{0SDdh>KGYS6=l6iwB zjJ=O4jQ#I25fs&d4S!~nCl59;4&W!yXx~O=8pMJEpC*#KxR~5IXaUc7I6|x)!+^)y z&|qREu|MU_|Igtvu@Ra8@1Y%z5iLS+@TC>p#nB{=7Z99w_91xZ=>|Xgau}I@Ur-{U z%_ywv1I`hP!LXe))K>rF7?acmDnEYa^!KST&klbj2G{vyP;(9#?GS_6^`_A4_cSu& z%oD=Kl{jj1ghaP1+R<=uGTadzODazclReso%uGu=(fy}Py0a^R&5>11pp6QQ`)fiz zST=*oodEXp)L@9U9@#ZbjxidKh93tPGZULl;onJ*$iwY_fYaU!M8j|%{PUa#LvMC+ z7BnqpVx00pe4mhi{oFUABVt-6J=smR%0MPe-~n@1hciW~m0+0GiJxw9?3J~cf&EQjgNOmDrd38}+xRnAo_2{KBW}#SwLZ{u;SkAeD+kJR2l>I1 z9YJ&EWl}cFl2P-GWBQKAz`(OBnfKE!0*9Pc@L8EUv*heN=v3Vf5~w|M!ek=czIJ&n z_*e{@#uGu5Fp+O4>ZulWTfmfwzsZ?ZUU0?!R1mZ8DlnAwVDcOsA=_I;Ud-4DWPi_s zT9OaIzvP2(a!xAoejx@=>~|(w1!m-z?-WAT9b{a#M8Ix%lrNQ_#`KG{;8Z;WvZ-P| zJUvg$QE}lCCiHM4f0;;obI|dGt%vu*1hY*{yhtxN>-2$0tTljjqY{kr&oiX(2VhJ* z);PZZF#^1nQgE?u8a(N>jdZk%fzYOuy0K56`0nq6fxPP~=BLvcAV19>l;kNh8@>I& z)A$8UQfvoNno|sd>}E1g@27*h6)Qn+`&Ope_ceH<*-N}Wd?pKBPJltnYQmkQ3AHr~ zLEXv(FsxreZj3eY7ytSV)?CqHJW2#$QBW4&=R-WmpEL_TDur;CdLWd2`I7%odoP?f z)s4yQ-U4$D?*qNg^Wnx_d1ULqejunQ7HPXioI@rWB=dA2q)-lyB_)x0Hm2}Y>`m~U zCl5!ebz#+s|db_%)$()!^kAd{6$;5AcKIiW_3Gk4U0ESHCNmBS4*ejp(KZedUo~ka4;xdmJ zLuAa9d3Mi!++&^!jm9LENT`tJWJns2D9KQyL7JtyXK(jVDTRb+(5NVH8YC5^dhfUU z``s`1!@2uB`&s|B7Lu+Ghb`;f$)T{B!2H&QBR2|o*hH5sNGU>QR<$Ts&|khM<_#K& z??4XXznBBL#$fg9C;obIC!|c>fCyuO<}2->Hp|_|Rwv$ahGOg?QdgZUU!g&w%JPw& z$aid;?*bt)qD1i>kD2GOlbN1y0ev|=h)ja)I8X6E+#M_fx$E7)ymc40E9$_ZR0v$o zTn@I|)Rga6TjtT+=9o;*=>f>Rm6zZ1q15ZUnR0iJBSg# zmJA-LONjEseRTPQxl^^!cA~Ug8uB_SarlR3)ZwT}eaO^B-b$)qX7UaFkbHqklX@w~ z)CuNE!e10E5lcLd|3YjS!3%>G(R86`H0}Hooc7lgCbO@h-Bp=R=*lQgys1nQe!s=C z&p2c#e-Er1HH0%}R)WuGc{zExcy{GlvcuG#gjQ7{#jF8NnT;8UPJM&WW*rnYDMgql zVai}~B@#L?i)ck=f=+Az1f=<5|IJN|-GwxAf9nq%dh8uK)anQ)RGbmdL>sd0f>Dyz zU9>PpaOWn{Fcjwp!hb{IKqW<*D?3q;;(wHE=WJv?*OVM}Od=LN7PzrZf<(6_!{L`Z z@Z@=U{Ga3-H23j(P!(GN&Kn<4`5Qc#3NL38yV#1nwGM{0=29f>pPCq`mqoyeN% zMk>}j6A!u-;BCV)=yg*v>5i5mn*Yp6fFqv?kMkzG76*d-A1$)TBoC__hoeEa&sdcM z@G*Ikx*Ar9zIuw26%wM<$w^64W3^t;>$(}99@K{`MenimdM8Nb?;>{`TquK$&B)QQ z4DCI(1Nj!qp@0#4v{s(Me(&wUM>QGQy|Kay$N@hcuq7L7%t@i49XXkpiLQ%%Lb6xi zA(1)9c|-Hl;qc{jqMvTYi?{NDb9FLc``ef7&}*QKFXb{Vp{sF#(IHAZ{SCG&R40{J z6ye)Sb(m%H2lf3&F`vq&lkJJSfwXP|iMV6PbhkGID`gPBTw`(}Fp+4z^uvLducCdc z!id&R4HCIzH9Gl*MMv(vKsE#Zcva&9_*=b#*q>A+;XP&uS0n0>7(46=>l)i!sVJ=BvDlwM`6=Z|_wf)p{97Hz0=cAp%F(hQL zk=Jc*5ASDe#hg!v@veetf;~h5HvVT0Rko3Y`|CV&kn@sScgqFJ>&H;c`vg$!`HeSS z-iO5c0>SZvpxu647Dud=AgkxSMk=yfaY&363HJ^poG;n%Y}5mOq+P-ceU{3U3W236 zIxt${LzLTN@b(pU=tO%u{<2>g@@raA-V-fis%k^r_p0OaPt(wwN_G6~Ngdi$YeLRf zuOdjJ6|G$7Pxy0Nu*bwl0?fQ67D%2oeCV~#!?ep-*Eou;}YXNw#Ju~Ho zK2Z_g3R;EnFsu6oB~j{2&VFe|tKS8ZVd*}c?_Gv2p4S3f%fFN)Hy?LiID;nkg^=Z8 z%gE}1T$J>@9i?X$FuL8nRKSv_SaQb?-JF%zFD1{|joTX75SEV;k!6!uk*Rz4r=~eAJO#Ghe{)&#l2L z-4WE?i6TW2>%lnL8}cq0Lx9mSJaPC6;~;t*`D zr<}=0{%-8KWSo*3>ZHC0?8IS9I;eHVJIK_tIpFnj8M5E%3_)GJ=y8A;sk)|0>}%CX zihvb3+^+z~Z1T|k*@!}JaH@R^`jbGk5E|L`q!HwDn2dVfa0WeiCP zdPXaMuYeglwg}E3P1yOq9M#lG!5b$ZVvh6B=*d#fzFZ?5%d;h)*J{GoRp)T@pe3$; zGKQqpjL3z_O#CvufWh}u&`3@&UOv+SMT!f4>oQY#Q+tN0uDpwWFZKqB+fCF=Ln)lb z4aTegqmkw;Gw>Y~L*FMK;x&d>QN_#UWbA=DDj+xThu#uuaB33K8*~G?t52A(*cFLy z^%ZEHchLEw&-iiu8SK^3#qoV)2Zd^CFn?GZ^v5dkk?Wc${Zs?m|7rpGea{yjs$@X+ zehU~xyQnR4HMm<^9Xhl9(VvFhsH$uxY04VFu5V8vlc+BA`JNq|S0>c0%&E|6E)7qP zmLdh`DWoo6hooh?lG6hX==&yb68L8T$41G)AtyO-|8y43G0(^6KRJ`N zexI>)>PUs=?^nc9WlvJd= zQ3YP@5{0*OC76r%-2prY@mN$C`4?e9<^+DgQ|$|JMV>PJcXcLe68}#17TaQ>qERe* z@0#Eay^d(AXMyl3N0Qrlk5^i)2X1dJ@un?IM;$8Sf`YG5%2|M$(9c$JXIPskRV0=!)&r+(^EWiuJ9gplrJoM%Z%- z+1cv|6DFtdSa&dUdWAWj@@yeodiM$!=2oLV4F$4rtr#+2@dKZC)y1bmTk!3NiTKb8 z!8~?hf;sE24_>po@v@!x;LAMHP{75CYFcFJx@Ra%VFLS!_fx0C z6-l#56|xrWTC|TXWn`T)QOgrq5X+Fo!fV9H2NPc+JK-RhEeddUpA~eaOvO@0bIIri zJt%LYQIKjqHauR)Xk>|#5G6_QyJ$>ie{jKTFKFzUEgY;q zivgED4sA+qrVfhP;0-RnZzA{pndBJn zIFgD&NPqJq%J{lF@pvE(nTaRywkQcKn{|RY*-P^d#;O8jX5*>TtjTjP39^#Yjjy*m z!`v%XSYNjoUA4LDP?F0LXzjYNE$1`3^HUNgN`*k)#{*=n`>C6iM=^@DLmxXVApcka zGgzcga^|{$;(~T$DZYhyIpqZAf(~r{VMzLyMW9>h+Ndii4y{c>csOVkQ>r7_EhMf) zyysfP);$^RwK$J#)D^+h5>rweX&ByN1JPgpL&uckiROv9;3r3cRMI3-&UOQMjyB6O27hBVZ3yo5_w_KgXZf!LlW78%(UsVc{g`l zVa%mG@PBoCv2gMbbMA5=89n?Bi{#6bE%yRJ_lG;tSaAqf2)mJc0bi*zqF$u_dOvO* zIES}sID>eHEGZ8$BMFd$qaIkoo?o`4XTXZICkvB}qoPc!nm#Joq;O5$dJ|K))Bx|A zF@ijL%Td?U?Wp2>4Ano-f+KQg0Y;95yZ8etFq%S^r0e5a@oU&o^Z|2P;x+F4_YCU? zzC(*=sF9&;DNxUF0*^)>kBsZXnx#LmZjBcDlyU;Ee6|cOEc7P(8`JTnD}w8e_6#DL zFA9F|ykR`59L;vnfuIUil4tV~i*ERXEn`mLQXwH!^27#wdTrsquM}CcY=m)DyUJNS z@wuSJm8ZUA1L z=WuD8BrN@O64fl}#A?^ILF-mN)t{P+ZF0ULr3FnmYXRVef0mOMC-8Ow8qv8w}fp%p9QSa5OV`12)gO{3qwKu zvaX=(S)cUw-lo=Z5#Xqrae?Y}#n z{BhPqpQiu8PDN_OPTdcj=PV+h=FBAf|IH-tMCZVJO=l$EbscL{D~Y#rGHmYnfiG8A zG8RFh#5i#s6o0#gnUV>-+FO*oNL!2FD(}GezpbSl;$4a6gSX$l znoy=HJRB5|iRosJ`hz3hvxO4%u9EZ^d;iHO~Utca2iTSK^WG4SQ0l zCDEw{08R7>k=;MkiE6(z>4;wj#;gH3+UU#6suRpKtxJ%X**$!J z$REqh+6DPv9LZ#X11z1V&jd*RK=;n9pyK+jp;<a{S<w@gF96YssE0`WG zN9PVNMrwVM@Z#8Uw9WVts=4`+>9#$I6}qMn&)vuHzE|#8rhZbew$_8Bg@3V!YrEjZpnn( znO%65sT_ItSDNYQQy?R|Oi-ryS9Hi3k&=BP_{%Rjh|4ZRLb+G*ovLyyeS98~a?+yS zpEZCrvS0Apoe!|Mq9U_;*GV+SlOzW}i<8Jkf5H(CBv*rTVL{a{2;H$7#%K$&W9b?g z{kRKmw>aROr-E~=Wk0g&4hD??S-AW~4=zgbu&QhX5h<$1Zbz+2=@~N+??26~Hw;A8 z>-o$U>nK7$T19?SY6L7_GV78Mnvjpc$6RKSOG*Y%W2#ENKc5Euiq2qm`!4>j5(`DL zY4APh6ID2T0{4_@!Xj=bUK!DZlsNN>LNx{46)#y}C z4vtlw2|I_rQW+OiaZ0=t9($)pcAU$?qw$mYi}wH)dX>a%aXQA)y|J45Y`ul#f1CyH z-euxYu}tdOEI;zhEdVM{{K6vgnbc*=%h*?(i$C7Df$4J%IO?tw94nbY@~#hXzA`(p zV!$Yq5SdN+ONLR;eNN$D-EBx>odY><9Lq^ROE5F&3jJlF6#u9tHTWSSco2EBHmu{a0Md#=;qpH( zSf>?+r7UXEn_4+ob9FIjEnWt;QsLz8eib|%r3zcL}rC|DEhdS=1H`c>q*fS>sD;R$NDYetnR?$$f~$f^i0bDa=l%tRNgGX zd9H;1bsj|@tNQRjY9K7v*1~g5t8vd+0T-Qri9&t9c;W|a@lx4#torT@vr#V(mF&4k z4WPT|`lKH*D3Kt|PJnFDo#>Mlp%YR*L_G{=g3zq6Kk)3A+%Y*Ytx&mnvzr4F5! zw*zuu0xvvm2b)rMq)I()?&HUi7dWp*aiTAr>d1_S>*nq!B6KcL>+@UO@F_ zcbWJ{pYa2KSvY-0n~W()lFo@%+>>_#U4aW1ly%{4Q!T*VD|{xgZsuVP#h z!y!%&w!^*fQRIGbn76Q2lWhOP0ZlI@@-ur7SEV?>aQb#KYu9|ZpK=my@`)hF`vSmV zSeLk%%ahwqHMmUJ7H!aef}}!*knz8hr~@ZptzTbIMRzH)gtw8}9WR)L4FtIZ&j-=H zlj+E5_%gmd&5!rgW;XhGTN{-6D)EiUrBvX@D=00>7s3zu5p6aCwWBLIaPwuVzfp!% zg-#)MPRkKLeYNB5seR1dtfP3Hj|kcF6_M6H1XEV*BpzC7WO>{*>{8{3!s`97$eClz z?{x~W&_st^-z*9PE}{5k=~kpTqZ+-v{ReIH3jpr}j->wc0tjCJ4>tv>5aECT*cW3? z>hTuTlAcAS*O#Fly8=8c>sj>hj0bKI6YO0V@$l3`p=9WQ60_iJ z7IF%o1)dH*yjnG1(s5uZw4N_OJgaovVZI6#ac#(os1H=Gv<|OW)Q;$?XMkjxKA2o7 zMW+tVBuCz-AfX4QWVcEO9@ho3V&zG6`Ogqu7cNR3?lB;;_tn6DjVoF7{wdWo`xaI` z?FH^Hg2-=|IPyv(oO}&m46Qy-@rK_!9ZeRB5vg1I8QvyoT*=XZrRs6$^BW*@jl|(& zqcJ2}O+^lSEulYY2+99pQEEjTEL-D6vi2uHf!QVIg0CFNS1m&OcFe)Lo67OCLrG-9 z?Gc<>p=+Kf^FJr@>uneky({M=Zt)I%=e>SRdg7Bz- zzo?$}m@#~+%@|vxqQy1kyc4N8j&m+K*rOTj}X9%-@ z36Yn1I>4(E!$&MsLDF54kSHmVRAqt#urp}2TapWRZOL0dA++Y=N%UHZLi$$U@n2a_ zyuvvasOgJ{?;HhqYbXg~@-d))Wiy$tL!;iu;!teqPj+pX59dEQK*%vJE*T9br8DOM zJAE@psW=iA^!4%zc33-|eti!u4|9Vu2@6u>qYq~v4q%t(&UlJ}FAS^@^xL6Il*+{ID?!!d%+y4kNHeU_YrIZCP48b1c4FD}I%G4dGN9Jiqk>fN! zvS~&hm05oaAIjJUY6E-8nf)o^bR+hAibzuH3J#u=QHi~?E8oQ;~lMPYM zjKPm(@c7IuVzZS4FSPPeq}oz)l@kbG9h`~BjvM$n%!Shmt4Wz%8!o^LV(edhHTfH=_g_M6=*PHlIG>r>9|^~@ z1BshnBbs%&9BGAQVd)S9h%T*WRQ_VDWabEMXZ8@&O|j@!sT^2nb|Cs`B%U;>LQSc9 z#MF@kgBKjhqZP^!BqRcM_fxR=3&EU5Y{=Z5o_N!lYU*39J8|oIj(I2fSWq2|Z!Px& zzvJ!H;t>f_xMG4*o??MVeU_uy8pN8B z7ENgmSwoclImX*(Ke}qPl$3oiB5i|-R9~MYL5CFJ;lpU=Md&kXpt*$#EiwSl6o#{E zn?3~mEMY|aDw)mBqS)omeiUQcOx3=Rg*#`iA>1!cbf}Mb^?VT+b#1~Hn^ed_MM)w) za|uLfIO5IitBB#l7nGgRAWm8ANq!u@S6`rJ2CF^VhIM&@@t; z-GskS%OKls=ED7S4#zA@rIpFq_gZa@=%FGJ*8E6d#@Cg2j}tSSx1>U$ck7B zZHHGk)}YFARh%{TEqbfAo=7|u#&H~7LGIN)GJP}?#N8^XD|Y5&<9TC}SG^FJNE+YE zf5G7HN<1R)+;@b26rbvqnj=(IW5uCVSB|!l2+tPWZp{P>gN{7FjAi6osc3yyPn`vZ-9C3 z`Is>g_zbqeA0!py0oOZf7|q}mIRAAsGJV%f5&v6gwn+(^jUP}4uFk`+3|^wP%|nQ* zvkQ5B;-k6MD)1%p8#;A92<>XthULitq_T3c)1^P1c-;w0l67=1{2V5zZelw+av&US zomO|fJ7qt*{Qfv~xm<$G-uIfZ+NVzDTJOdj@2~i|nmm&n+J@PK5=b)qEPigW85s)R zSL^1A!z059s^nH3dYN{O$ygYSJ>@iU$?F&BZa@}}buu89ZvChyB$&MT`31RnD1!Ok z1hO-`3fb8JBR=yJy0QKsrE)EsIovmn1lb>Oq`QK7@?;m@e>4Me!l#mtP95mOC5kl8 zNXH@8O?b@r2!4M$2gwSwwf0e#aTLFTEDIILE^-ym+S7*k>t~TVizP7WG>BvMOL%UU z`B>|!Hk9$D$y<@_WYyqVwg~b%7=ZB97{g0L1Fh1nOyQ1UBA(VS^~eLl{@z`Gu?y0S>`f|e6fTC`x-;k zABwpAy~BHvu@2Xayu*V&o!DWc5e~oi5Sxw9g17&a;it$mbU#>?_$_EcH+TF+Pb4MD zf2m$%@6B+So^DLa!rchHDH^V{#gaF1{U|qGaQ6{trOOUBA**MRf*GKYiYS;3_M@Sc z?2<+B;=%=lji*33nMO9*`y#{D3ZSkLLVDA`q9cmSF)R0)N@@2b=Q+*HRK5mbp2ksa znk-ho9FID5!f?#!BDA_K+sXY|BU-NWl36Pyf#Q9Zz{tO)B>3zZH{G>dO4kLx88;p_N5b8Q01JgdN!QVVAAZCj{(KWCpO>!IYGQ}t|xojPL z^vJ>eZCCKb(Lkh_xgCvMy@eL{D1-BQRkCRV2PRT};zY&cSW;P%_+FEM4oOc^8el*q zep%s1E@Qa;1rU>|znHj%Ga=A5jy&w0&y)Cb9yvsLfk}u95nbQ|?|eMb(fuWi!E0UI zWiyBexgXKsg{>samkXs@Tr#?bLl(Eok(kr6xaP3H8(i0kE=$_sBa058-#-<|s#mYk zgN`7m?OH%eR$f5S>((+dlJnuvq9AhqgFke1Si+Kl=Xmw(YW(=5Hkmz7l~ju&(z~Y= zIb@v0|2^1*viP1<{o+?xkI&3i1lr^ZZn1E|c>=#kldwNI@~8-R>Mp|3uWZPB`<*=P{wsJq z)ye61kuqHIccebBTR~;30<%RS$*C-=0c)B_AW_3!w4Sjb>z^yZ#+7R^b7~h(R@NqG zy^;}E|07;dSjS9!I*)iGgE%$Q8_td_WY#ayN1JgdnEEJ^#sXLJ@p=G$s5cuQo2^dP zZy%rzw|_$c=QPNt1E;A5{Rd36x+=8X+D>BK#VM{wCN-Hb10D)yhJFV_Pzn=xn@YAa z_A*{r;_+f=cI=`$v-gm_YhB>aK~b{ZtOIX*m;nb}?1*?Prjk~Rk=!RH;GJE;Q4pSs zOlD8RX@0xVp`R3y{n(4|^-Tr6mOo5_;RAeXiZ0?E)&|mGK*ZdO&}di*Hcu7kp6Pce zlixZxcGhZa(S07zG7{+SHz|~zm;yuJj8KMJI25OQle$&c(W?Ssu=xHAZ`|pJ2RV1C zS+i`Auh0bkr87go)a=Cx)fmNY=wjZ+?t-t`;^5Y3Pg$uU{NU(n%JS(o%KMZu437|e z92fIp!w=)>4m4>IN<@DAjh!Sfy+L2ECC3u35;aLLs9 zB<83cRsTR3jrjaVVw@Pf^RxyW$s48Cx-P`JkNcVV`CD+8;vG&=M+#P0tir6bnoijk zD?rKPsU)jt3|%OAjx;YtAf?AJAH{6&kZi^iXmDbS3^?kZTvi|1y}BAp%ez2smg7E(6^mM z^8|SeCAnPEvhyruzNY~RFYG`^rmZ3gH}{e_p&Z;T{|rsOyoXA5y_Kk))_S z9@>(VKsS6TZn|+tFmrDpmQ(WxlJCXIRqNrj6qn>C6yoP9dw6aEnb7<=l~{UslReE6 zFsnk09N8xDD!vH<8}mF=cf|nec1K{pl-Jn0aRdj(7*OSo!sN!v6=c^x3o@*pM9Mbr zg|}+}JnkeIPMLSi?s+D#bjeEMv(=LHgs4IDq8ePy++gD5>_~?f zA79l`B#Ud+z_4mFY&E=%hu+PA1NB1akBlEsRb#w(WWEf0hes$s1W($Ca1~|tYC~&`$i;MVs(97^Y zSizLR!`dP6L2@0;;)j!utv|7m`WBA3s{w2qQXxwlJ8(m<1EqSb5&ymW2R*EGC1*^} zptL|8a6WP!j|BvS)!#q(VT&t1)Ne$3l4jxhHUVE95=Ml#?EuZ<<#4Ba49i_ofF)CY zF>T>Lu!Nc*^I)zkxh&s<K^2+#E7 zN#wo37yNR*W5?fI*qp8bKf3Ip>g;bkd}Jw%t{Ox3y<(u>)`nLr>thj46OLQag;tr> zVU^TRjK@bQNV%p0)W(}k<&~M(x#j>$zAl9ow<|$zyd;DQysgvc+v5M;&P69et;j8R zXR2L63EbPwNwf1OEVXnOUOutcslD+Bv)Lwy9FIDNRd31>urY+zbsFTsJ1@!MlF0)yFM3X9m=OXe0z94W)A4-R=`CndFXhp zJLR7qNRnnofpD!cd^6J{Nlo>r%xww?3G{UFCQTA8xTo#YaD(PsIULq!KDl<`0BV=z zIE}U%U?UYNoVd9cJ5Detp-mOkY|nA@vUVan!R%Kn$XST;T#b=+IMZq;j~1@?g1^(; zpj)61FX+s}RuNLfeqS>uOv4NQNtK~h_ERAA<{>06rUSRyAK_K?TKKkKGnUc%L3PjR zql$0c#|a1bqSj(ZBFC(RDpPq7Yx{-M?z_T?)pLoGxddijXruNy9ix)6MwdLEsR##=4WwV^6(ZPbVk<(U)HUVk`J^%G}} z>ygxzfU+Bm@vP|@K;_S1#508PtgFt{6NMuDh&PIt|FDAYW)Aso^A@|6nn7}FDfRTy zdpzy51SfOEj%t6JL;mU|z~*HtMCGbIaj5S{z3$FL?nxrtk8VU>Iuj_o{1z&5RENUV zf02RJO`P%g6DmIP7ay+sfWz|dAS&JjFLl{Q-P~XS!Mn?Gri(D?iSQI~vspOXW8y-)*#VtNL_?U7bQrac(XP(Kx1%^}Lrucl68h!{xzLKTB zDCZ&~T!wqTZp9y5^U&uIbw;D^HV&CJjFu`GF=;l*Z^7Bau5gK_-g$wRV zDUTe8JNASxybW7a1&KbPa%W5GeQ3FdsHa!Zz~FELcEkU8s->rrtKp*Q#k^(ebb25KmhtI zY6)fCW61WhJ;~V;0TR9+k^iH8$i&l%B-fNji^Fx=DfCDSaj@gIkoP~R$Gq-Adomd| zu8GD;O1=UQ%?+d@T8AR!^ zBKl~oe=iDqQ6;%5iNwR|mC+Cs~oQT*=l#G2~VKkS#Ict6@ zkVuruf1#vD06(ji;QgkTc$trDk#$iWS~6oPGknjVlm#3@Lvwru!O)+Xc!7tEqbEzo zo=kyLNpX}xY#B<78)k_30X*gKER}cAYr~Onp`=;`z8AsvpIvze-o0V zfA1*)zc!2Xo|S@znL4)2#&LV39vSslBexgKWdx>Sr}~Nn%3#ZCuw4Hh-57X` zHXZ80%TOxXGJXl`HaJlQ8;>~nHr6tYO-j`3&poJ9@)JsbFiKgRC}jrkq>+S(dGN4F zh3uFoMou~3l^ujfJ58t0ZwN zrKr$3UgVb3M`n3EW=gMhqZb*1|F1lUahrC6%CMe8WnBfV`?^!WBWyzlL&gwKxB=B) z^n;0w(j?%MChTukBuv~Z+%wQF(E1PK%R9az#5bZuejVi%?dL#YdN zk+g^wx%%1|_0BWFZ@v{_$G7IN@OTzy^oKrZm()=^s_e-Qts3fwZyySHFAW{L4^Wm0 zk;w9o23d{Wv9JGgwBxlLIXX7ZQI+?B>Blw@{H+PMPr8AFArBczCF1Lcq(RtmF0y?c zL`sXc!N>9wRFzK!o-d?JI`^8xrYwDukbM~2SMiZ_o;sw5+{Eoul*pxnBG7)@5pB;H zMmqP1EN5AXy6LjU+OCUm@RuJUOTb^Ls%72@S-ty(dD_F}5hQtK4!5xRobvwy&!C)?j3FHbm+ z3m!P@u~Mv^gEzl`G|*--HT*3@^Jn%Yu`;GT&Ms%<*xu?Px^4V5{j~A}|LHGZwzOLrZdjGm?}l^O zx$1gcQ|~9-m6!eK)zZ~;XSERfdERdBGO>?>4)5KpQDqFBrCP{0ng6T7`(!s?L}VYk zCc}bVoQ&D~SBvSjD^_tgyO=aIY*1;m5v(pIb#dhw26>02cOur84<{UFXJExbu<-xBA4 zSewsZQ2vD*KAg+{6YNUQG8FW9^8#px?efk~^5k8X`;ODTYs>h1=4-LS0T1b_fGqAe z@(UD=dK!F9q}h~x4($6ZGp^xDVCzbj(SE0w(wo-?(k@=TEZ{Nv@4}1DN0euB!|x6{ z=QyR(@xGb-WvW+c-LSj#@ttn8-R#VUu?Bgz@yB}h;Feh0_K|F3`m4jv#aB$|5S@kW zW5HgBbr9qrPr1eYx8y&5O`r??y?-t}YMS2=A7{ZAl`ZAoF}X9I#N=E_PY_m za?Qo)Q}#RfySF z!G=}l4t&pzd-$s*diWXAJLwAV3Fja77wM@NAJb3H?xO$6$G;>*>AzpBoJ+Ic@(=rEfpX9`=cf0Kto5~G{^@!<`hsE#J+QBuJMr}( zZJ*W8-aBu|ZR|B`xbR^HTV!9$eJaSAjka=Wc-E}Q4nbIL#Le(cBy?h${y$V@g1garNf`MvHx}7qiycGvgQtjw2ET~Un==C_Y{9WKVxh! z?V(phkNd~d(%zHo9j|5lQAbTSFJFkp`dRG!ck1lZeoflxWFvdLzk{83vzvC9(Z`o_ z?q_RyPO{IzgDw5zN4wp$rJe2wp1<=v{>X}@k@N`k>Nte&77<&U?LP3Q|hr z>EFp++&{)+H@@2@m-OgF)fGfUNhlbJG7c!x!%NC??ozG zD=;eAl4RlK*q_9yaF8Pj2M7 z6Lj*c+1m?4;}(zI*T}9U(2@ zB6jkM^L*DKeuexq?#!!M4c+(mvhs$QJ(48B+Ey3Qb)NUR?K>Ul%@=LiZ9?1WRcaEA z_2qk5nIpToOW)?Qk9{X++~=cxrhW!f`-{CUR) z&HI%Nh#BRJIQP*poCt2!$uh3PyGz_7k6oOb>~rX>^8c<>L@?=?R<8> zn54_(qwU;1`ZdlHiz?Zy-S@e#N0zeJ*NQa0S({3ahCb!4i3N7I&2gG9vYXyk^u*a~ zG>Q9tmM`Cc9BdHvmv$~h4ctG+v*2#^Q*NMd6fNbqnV&ukd|?dW3z4L z*b0Sn++i;fmc4U`6|elp-EV)a!DPse?(*0}&-=HAHH)fa54XtkeWtVA@#a46DtJx5 zj5nlT1&FX@-W(Ub_?6BR8-DX|nD@|YYz^5tZ(I0Rj7|Al%+%RO7q+owiRtv~16|G@ zkIv9{c8a-NEGnc&tK`|#R(F0JQ^qYC%%T@I-(v@#d}j-nJmF5W&|oFnOX#Js`?>YQ zv*~LRySW$A>ew^a_i$x-Q`q%wZuGNp8Jh9rv7L8|*b194K7Wx7|5{5HyNu~;n4)={ zz3VUMBE}o%_aG12KepJJS$B-qS<7SB;;pRl`aCw7tIB@2u4M}k?_x!t)zIgRUp18W zT;b=xKgZhm)bqPvWpcG=O44@K`i(Wm-*PJqUa|pB6|~B2aenYaE&A#7Y_@aXH+EU8 zs7t>cYP@e##=elb!zPRu!oT@~UW4u|b}*rszSR4z!C8=*K9>2B`|XJ#>#5IYr=QyD zeE-25Zmh&_ICo#rPnz)`>oO;UJwGj-ePQ;A>!MiVyx^`Bo1oOq)u_~=H>cau8ZO=3 ziF!}i*m1O>qHHED++@${btyS_ibZh8FYILF_O|h#zJAH=4ST`{Ek4WjuJxwftk!Zb z9KO!q?r+H+fA7QJnAyX(5j##Roo;uIdMQE6Nj1|KB+*`ZL@)jvL7&ydop-tO_u8@h{U#fdSjz@a-f^}p!I$qf zH;*4O@|ji^O5@fQM{q}tYq?pXf!z3tY-i`-`+V(3vb5fBKX%EKf6lkB&Sz`lvuL!q zOVIH#Nc(eU(=DaT`1|bsXXrfqYW$-(E^Sg;q^V@2jI6rPz29?1M)@HksVEd#MKs8$ zw3T)$X;6fUG<2VPzvpTgWrYS6A`L6MjMVS`2lw@T?sLAMb3UKvyu)}t{jX`us>eL8 zgpFr;6R7zoj z`+x9WL4aja3vtyNJ}hl_;qrLnqY{u8joA4~69U>LTrL>dG(L0d7u2cFz$>je`3m4oH6o1g3AexOJ*3@^pcif45Uy1) zZgRYh>%CI(mef(+y+j36=?%t<&&r8O>@(Qe_8uu%D$abjM3z7Y$i)SNIoz%10^j;$sY1^ItPzSKY+>>kaW#h&5_yRpE_?ze&`UE~bCRDSZEF zIyumK9|n%I64uHg^qt>`Txph^_*@BDj@npzB}w34Zw3R8_2G$XC@OS0K+3VjP|*-Y zhm7UG^x8=D@s~y2ldmLo=Z>gc7 z7`RBDSHpctuQ|K$7FdMSn)%L{p=1o3ZmZ(?ByWgI*^32xGpL@xnv_TogZx%~TDw~r zul>}5QtM0bv+y>}du-0XO|QebAG)|C=_ULrrg;69H(32ULEkEUCf-9swC= zy}U#J-6iU?kKpu?7ht`66YXiu0L||&$lc~%l&(F6PhC&rA9s87^VI>3wj5ZU+eEKO zbm--&8YFt8Jy=Xp$CKxD`B}Zjbm(HbsA7aNzR7X~nOVV<*Rcf0r0HRlniaYApOpAl zU>_Mf^&_n*xQ;(7p3!#2y_A!kh0+$m(6+=G1FI$cYr!Od$B{r@4pPykM$v@7XW-#` zhI;s9fr@mAqz^5D>BBM*=5T@L%P$h`i(WzXEg#YYV1ttvu7)<*$Eb8bh2P&6gt0m^ zG2r7vxY|5ceER-vdMU(^KD5vxjnW6;-M^o}Sv~@lFTVxXXTgMYb78~iV$d+=!tt~y zw5pGWhp`HHwcs|1$}xjQ(d&RHUxam+`mkcV3henFEo#{55Ayp);OIZAV4?9;;7s#} zr5^+6rWdQI*W6-yZ{>b6@B1TIa3CHR$NnVQejTD1uW+2&lE6Fns2GFNM~PKdsfrVV zR>106F;Ks@i`RBnna>&eLz*56fFyznx)_hSJB zwbo;}rYwy+pbRE&J*eK(WunOONwDSQDba5M2hT=7Alh!J(7)s`41xuXobViH`uwJI zI)~}>;J0M1-4S}ou7Nut^E6L5@-T$nKhK+?+K5j|2$fkZhkFi0(VbuBpxz&IaC3Vk zdSCXF{4t#g9z%acU5-Xr+!+KP+%j<3flD*Ac8C?V_TqNc7)Vp?ARqU8;bXNM_;dVY z5akEr(ar8qtMkIw^@ z9ScdZ%YHPgEfpPD?@K!LFM-kX^JvO8q3#iDoMto&@4reE#S2zp?wGsRmo0-s@*J|W?;=`6^@>XBpTiodov5OyE;gL1P1fzZf~ldFqt)=o3SF{6AY< zU@c)dN6TV|#XfFU{w40}i#y@JNny~ZrH!Q<=fVWHIlKltYn&}DVZJG_RBp{TvWfcO zKkwJX)IlFMMlMIEoF_y+c>_H!I8EL}%43>uBaW~=i|ZDpk+6s9upz?-eN^2zsoyH; zK&=f9ZMuXG3(nBIQ30Skw~bo<*p0u562GoAg*d%ZA)RyM(Qm|9I`&T_-e-$&nd4nN zvN?{lb(%o4({^-gdrEw#-iOD2%J{EoDc%kA$IzTGa`v4oD37VZfRWoik!@jiQ-PRg8i&x zv|V2p7JkgZ1`iqHet#$VnX?kT8)m`fV~L`@Lt|;}0XYnCcuZbgdJnEIkH8b568*ht zKJC9(BC_Xe!U_wa=u_Y|e0xg+UP#M>evdNW$onCEe04sYdXxrNW>0|c?R8|OjV7En zS3@1y3H;l7cIf$PIvNPhdTNI`7<;a$b4ib)KC>qt0D^1#iuZp{u35@LTggayV6u&R+MH3X85low^IJ zBX)=?POO2I;$E1LAdg|8cCcs5dR}a#0@#VpL1NE2cpj<()j!&ZevmHoneIo+AbGeK zGaVE&PN9lU0))~@Xj7X3k(XS^H1jYjbM6!jpEZPZt>tvG-2)otkc5Mio#6dxXgSuPOZ8o-t^#CKNJ_W|QAPhiO>mXuj`=x4fPx zc?qLu0a!Qe!bAI};~ZOQ@T+R4&lhdP_)q_6k*N#rm1}`P^9afeC%{|ZGn8>sq4%Pa ziP2#b;8q!9tlkHVo1}wM_c2^0ku+tW#ig4iT3UDlIy;I$Zzz~r z2kyjg$4|nzX_Ty)x*A>vYQS67u{6_eG!Cm=6?jdONB2(>4)wBd9KZQ9)my2LkB&E! z|0Wva>9^adaOgK<|H& zbDJ3iL9uSA`*bm$w{4}4omL<`Z-MQ*H{g!SpX9@YP@HtA3FhEo-kjtU&>b=eyFM+1 z05??{9h6HxG^CM?LLPiClCbOT)NuV$CBDB&6y&}6&HUL^MV6&zV4Z9?J+k=;si->z z7bb*2=+O?ElWTk1zk)3v(&4ZPPrSRih_1b7%U$#)1e<+& zVOFLJzMow{U!?bA{?otI(p-bD*jq=hDYc2TZmB}a-&JHvL=IUi8-NODBWd%qGTz(6 zo}#2@0xIWJL?UAkNSCvhT?zI=-WlT(_Qwd*6Mbp-*3ciswx9*UKZW9f#qN z(Jf3lt}h<&*vP%{;1Nt=QgL>Uv!o7D@?%0gJW2O}mmb=@5@|o)3Z2tXvhXx`Z9amnMr&!T zeG%17VpDpJtn6icL=HA3TTP1UO|8RVrf02Cl?gWjS0kkorhuC_b=e2bFgINjf zm^W!5$SFnOnUg!rvaE%8Kc)f$RT$pOn6ori6ff!ZLvZE|ecYCz0XLUP*l(xxWe_7HO!ye)5rLz3ZYAKjetj;h0_m`Bl%cIPRIn24z7~Y_|saRz{CHY4Vfz<(?xW@en z?~Kbn(vhdY{~KCM+tdSui%bYi(7!|E#t)Gb840L<^Cf=mc?hNU%KQpb9p2ILAYSmj zm^`@}OUfTCAhU%@usHAkEs7YuE)6edSc*c0A;dT{1X?}6 ziJVr4(REvYlbrWycq-roR2))*4O8Bbv(oX{b+Q74J3r(0b;_W3DjQqKAPu?Vh0c~b zG|=HHE)d(GT3)rt#Gw%i+^e9_HifRebcs7H$qhY2CgZq<6Yy=HE?j;*7h0OH(@*aw zNw|JRIIv?Q*`XFe>>W11-;d2SKB5lqCEUZ?4IA;Xoju*Q(-ekzHjw-4JT+Z)o_tN) z4bz_}!YfHXsS&jc+ZP&B-Dgy!@>=5k^qRtpj8)*}v=n!wcTvd&fl6hI$cVEQy!nqc zkykYbvW#4y%3d8+r7n?4$rNwjye|4J_X8eZNQX}oWuX7qE}p`;$9Uz0HA(k+M9Td# zQ5`g(*Qp-TLZabd2jHEl{y@ZYz{8OuMm0ZzXUA8=QS+a;-z5f(55B-dKcB*4^HRE9{u)&oVFt%4O>k=!GD8oKR!~Yny9%Bfb^mhVehR9fQk2%3%xeOjBU@s0P?zjD+B9)b{k1^eiDz zp*~VvGtA(w9zC>);FF1tTXFKVu^9L(9W0hc!w4xG_+FEPildw1?6)$o>**kKcZOmA zs4O~4h5^IgFVN;Di#40QL|J%`?)35$rDWBRUb}m=EOrWPJChDo;qzhP&P-ar_yJAn z3dYGsKSV3r7sCtDA)IWm13I^V7cJRr1lbP#dqoAH1GHzGyqpz$iVE`%NB?xc`SO+NJb$a|5r_V?2Jq zTzDK&Nf--b@Y%H*_NOfXAKrT0I`oVeX0{cr7Tbsv$EJ|C=NrhhZDBB_`#sd|w#DU& z<#epaF_Ln37bX}MVXCq$ljtLMW!$tS&BT?+{ z1br&4)Tx6@uB2^1WrMZknfV#mV)Tdl?8=7vkYG49S_-_>8c2KHa#;60hW9{AiXS{- zhv4f92iza-Aj#)&XxFh{U_5ib=wX&BfAEl$XxtP_D00+83|B|z2oZ0(!x`F0cvy8Y zj27Pwg+texz@}8<>%3TpHAy*S)_>EW&2&DjzFY+3J7JGrH(2ZaHB#S(fn?{cX1!KXEAc;nH z#I|yYAG~QaxqdDKr_b4h4|e>ahW|$_?gp?#4T0!pU3^T-gP8O*84-b>8HX zy#n7LY8j|!T_q-rs`z7{G18ba$S?DQ>ZyL{&9Q*;BN-yUV0EzevmwPkUJ$$T0+l}; zL{pts;IB!FWQph^*?8O*-VBG5h*3tcC~O&?+_D+7wq?=w*Q-#?c8I>UFs1udZc(e} zacJ8lfM>#Dq8+vwGD8KBex#hdF&2~Ia7CPGbO^U~+X*j6?+|slTm!9T4#N31`|$hk z3(&gAk%Sf~z@4FkU~hMhs=sR>`TmXYpRu;6z(xuqHDi&h{SBqP=FpOxK~P^r4#|uqSxGe-9 zL>l7YS7)S2uSCv^qlv|tNO=894^K;G)_K;>#Pd&?Naax#vIfr3Lq}7ybt&|x>vWLY zJ3v+s%MgvV1#p9L#qkvCO zW6`sDfUJ7(U6gU~vm~?HghTs^N!+qPOe-DF&G7ApFDtrfqNWmkI&4ATGYP^@@5ksf zt%c0Y;K2ox(`Yz40~cwIA%6oSX+)(V-jZ91TRc|NS^d9g579=wB@>C1!GBPH zp;IKX-GSLZBQWOK9xyH$1qNaweqWe^Hol>7IF>J}T{(g`OHhJQ;br*k+*+C?`~WM| zf^byGJ>J8j>(HNFOCAx){jT~0KYujCk-94Y^Txu?abggr1YmfsE$x_8N!OiFfZKVo zF#VJ-(VPESsQS)@Tpz1Rr=Feyciw*@at|l)zp01AtkDlBUf2#-_>?@GE5#ofd>8F^ z-G)7f72wuR8yGBH!njV)K<)mMu)NS!w6Ho4`%Vr~e|=g0 ztCKegJTw5CjTV^r)tIkcr-7{42p{VD;FD2#c=g^aqB`v&Ol#E>znmBgj+a!#M@~MY zTVDPL3R`bbWA*#w0#i7?2be*BQKB^RUe9y;6T1+@Y=q1#20lwUx;K9|-hWn<`V zC#YR}ke0g_qiCdo*uAv_?oJt`dYO{Ela3HN`(BW(51dI8uLc%=FBe_zIYO<}r;1K; z2NwD?EmXH1gJXz$QhuS((dw>d<@^MRWU zV=+c|8+N5^gVMVPVDI4&(x3Vc7MbT$rM=#ux%xA&OyVJEaKc4?8Bx4Mjab+@HJ_{) zK8l{d)WKE4W&PkAPQ=|A;IT`>b}g2$l2ec1LQ5_Fk6C5d^$4*h>N>Hk(m}J9e6+DK z!DLj0spW3B5gRP!U~uQ@<}_xOlD{5OFvZq`I`V+@3SyN&V6W2mD+A-KLCrU{as z=9_{qN+7W4_OTJmK20NAVsFEv8}p!RAqQC9nXpFs1TwY*)NF<(KRIe9+Pk?*d@m{J zOPGd#ii1HX_X1>hcGEeqOOlOv%{!7%Nx#XKzz2R4xzV@+p6}A38-I3?(B4!yF;|(d z@SzP{UTh`)XRL|7ygrehol2!=+$8)?4UoRm0ke&6z&80AM5@h=ghdb2gU7;PXwG+9 z&?^VCwNFD{wHvQx)g(MHDqeIu-wvQJ4i57h@bSYY;kbrvcp_<#cVCf%`($T|=M8Z1 zpNbVJ9PJ96cr`KXiz1ho`hrW}5eV{ck#OBNLScZk*k|om@GRHj`>b^1;mQ_tX1rmK z%6N#H`vkdLG=Vd^9Q7tG#6>ADv4XZifA2`~jPbe8``*x$LpFSKZy9mLScSVQY zGV#-XXEH2x1>Y*jVs_3$+M;lW1SRdF^)=e)cm{Dz$JV;OogH_W#@oDp8VwJo{l%Bhj_Qh_&=BNk~qI8pXl3uds_$aWh zcf^jnvi#PfJXrkQn8fc>M_P6m)VB9>r-f>va7`rcsfq>L(p2a(`bSLmW|2~ZVeXcH zQ^020N6|*PHhi7kAllS10hDG`QK>2}LYf+0nk7TOzVv{+kEyWL_BiEQ-Xm(ccZD@Q zerP=yh4-2?L9u%p_vrgQU^c@5w!L46&3==?dS5(v^X9_y^eRzz{ZH=ZlGX4bNEgp+ zNkrdi7e!spwKV?VUK;8whpm3EiLgf-O=5JZd(I*dj>;0Yd$p2@*=f|)lcfgxNAurW zHsFhh9D2z485}Kdg=U#>(dCR9-qt6F&`Dkw*U&0Bx~Pcy%(jNH7BQG)mWJavUZm<{ zD&9D5N6Id4z@E$5^gnGwd}Go=#GVsCF|Q7kcj)qu7H=fuZKr{BLNe6Nalr@I$HMmg zopi}@N!G40!&2(kXx_lx6EJb=0kYmD6qL=+lbF{b)IL-hG8~`s-tOB?zE(=}e;=ET zM}O_Y|2Elx%mE|(`ObredtDaF?2ST`?L9P}KTbR5g#-|70&T!=Ur4pRC0 z*fH$^tWLd+9uej6{Af7XI9!Ibca^X|pdVT~BjIsxI4oT##ee3UkMl%!q8foZzFb*O zW;9B4f9e!E*QT3<&`xTt(N6l+|L~U2trfXVjsY_jC%9~)z{f?NB>IUtNpPP7zk~Eh z-10=;gYA*%mbMVKB}wv0?X&RS)MwNzs*{fQo=MlJpG3uDV<5`*gJdCA2+{d-4&UW$ z5HGkh0embjVeD8({@<(9V9$C-gSSQ22MIIcATl#J8)t=w`uE4~_YtGvFk$zn+RWm41`JGZM|w`dt+9@)C4xt)a7) zRzg)m4h~4wa#yH_lTKGvk;^1ESp6T+?jlRLD)A5vU4HYv_N{>EkcB8N?W0qTLy2_E z4H#Q#C0d}i8{SD5!k&ErFzh**7FYyheC;Nvf3^`r>SHl(tP{kpg5u37Psr|t zK2Yze0kYM#WVw7JaoCrK>3;y{jWR+*ouwFg`6?By^`K`q>VxggNHQpY0wNClqD3yb zyn@nuWK?A~zUq{LE`uQY*P)7X58l9U4dJxtwwTPU_)K$TXYzi^D#LuuD=^Qk8j^!m zG5tvxXg2nd{)SZYV{i?f^;?dvJ(vY=$K=u-ak}Km4Q&jN&yeJd4r83(pxeKT1 zT%b`$Wg&TU4PCzHDCv$*faLJ=P+R6h(A0(evCn|}*|uo2u^e55ZRAayE3A^v#5GdW ziQNqmhN!uc!Ny5gQ?{Kp29-hUwefUbr6=U9HizGG73B2iZIT`(3gXA^LTg(S@hgpb z@-cQHY|=?5O_DwQcB;2=-gHgy(`p8z`V-_=niQ6*$bi6gA0B*io&4Z_BkEhFu$a|`q7icx!Mjq7-F-htv72Q}a<-HM+b@_0e|j-cjME6Ot)0z=$Rr|!86t$SaI z#tZ8q^1?;l(n)7fMx}*VVhYbAP7dd5CBnnZiJ0|J1*Y_dP;KqqfFI>?QHTL$-uCd` z&OQL2pUUyyEzW@m-a527;R61jj^TF2Nu>NlI=aj`hXF&k!Rb;0IT3JA)c!{bbVg4H zA6yOcBZkRKUoDst`ml0r1ieusMf+vf@e-~q!9yl7aAW*-SXE#kw%lk5A)k|=g~KP?GRoodfHvPO&kvN- z^Ko3x0_ws~VW#Onrwb~=hn44Ou2Ca-2b|w{5`$J2V}AP! z5`WtkJ7)(&-)2kFDgOf9M;<`4_((YYDgt{}kE2&Ft;KuUH;8JVB!hcAmX6LlOz*6- zhj|U_;e_2ilEJ$J-X&!?Crbx6q+I6RvN$hN{+9-y464Cfd=j$H`l6h#5m-DN2@9W< zz>d0561?gGTFyL0nU^my=FLbvv$~IbuJ?c|!O6tG@;TuO*5meOMS9(4JU0E&AuoN3 z;Mj+9e7`n`G69;<|LQRf`7{EhvMRto{V%VvCQ+my$+`K;+DYC!9xUbW7af%w!RPVA z@n+t0h#uWWX8&4Jym4sSU9`B91ak(ONLAkqToJk# zW#-z_2dlpG=G}D?=^NEhx1XA*-LM2~lg7e}e@J9y3@!Z@rNF{}O8m%)o%G#cHYreg zK&CwSBRX#|P8^)|0(`j7>7s}s4Dk2C;PqAH?^aD5`CD3i*suvJ7rd2Z3NO-woH!z5 z&cXfuZZs|u(6MMXMy`H96_#Fx{fhI%)0D1b+|yZ9&_7muYK#$ip_qdDE6$Q_4$EQD z0Hv^yeO9_}WoVx>FKhP*(`=H1u9D>(s8`{c+c;YhKOsVWqgIdMH3?CJ7TF&H4op?OQs zliJ*7WU5}%5ocV;yE=1moO32zc`pxdk5-EO62owI$4*l8(1^~kjwdg8r$iH7FH?P; z9@>%g6>oW+1oC$W?sV6I3zN@*YwBpYmvsWX&{kyrED2ol+o5ollGwk!hn&>&2JG5M z2lrlwE9K^N{9S+E&m+HSO!8y=zT*HYRwuxV=d&QsM3UpH+78z?%mt_ZXgoMAj<>-# z2khmGaC!L)G)#8Hb=m_oabqi9n6(O@o9w|rj{>rF;2jAamIZo#37Ph8JYTV2P2!c> zkSN!9sC*PljU<`f<<~+;nB^~$s;~=lUX=?U9wJb&dm?Y^D-%*Rq{>&BR|Dq~g7I>M zBDP-E#V;e`sri4CNP<>3n5Vas;mJAF$Z7@I{Ye@A{`(J)xxYiJLj~v#c_f*QAgj(< zbMMPuyO?mzcQHQASa5#)Lm4g^b@2C7U9Q~F$ zW19UqUiPWm5PIe|Nhtk-0k&Sq$+!T0X#tq!CEj&=ut z?O8&q7t51*ncBSfZk}XvnkL*ilR&rn8WLAChJ@O#hDQ^>leD2^;@YOpo0S?uR&Kls zgX09S?p7qYPk2MmnzzuA^YzfH>K3_QRO!a9Ni(d~S*Wt;vNL zym<_7$;}y}i|Z!SYbm!xG{*&*;ws*~w@PT}BumFTT0)1~B%W^XHTd@K81HvbHah>A zOPAf=P2?B(f!2k5>@9mnNz7`jU7iEWE~kig?V!*(lm%|pHSj8P9dWn50@{xfvC5Q? z`;LpxImg+ua6g7=ixZHad_2c1gc+>_?hq+Mpe(l#tWrn$-W9I zI=)}xt-HZ{y=nOUm@WKH_JGuh$t2)z2@d_%#J|R-7!0c7nMn~a;ffWmVs1j>@EBaz zEW?u#rjeCsf?rpa(Fb06Fyi`EOf;wz%^KOr>-y5hn>d|Ay{3%E%T$%7N5zmvx)M1- zUEmx&6L(C|fbYT6X!JNabbioBH?(zuTXs3(*y@_D>{sr!`nY-2->?%F|bM^w+8%RSZg+ROM0_E^Y?+{lv4UE^(uB2`_RpK zQAERJ4qC4XqkfgM@Y?WUQ9z9rPV2A&8=Kp>x6TCYJ7$1sWTVj7IEg2YEFj-}>#1Fo z26dK^7wevk2YbVC8n>g2XEOdCH+trBNZvM1Y$=(c^jk7SQ8^#oCE4uh?MBq=#u)NA zVGVHHyxgVTf)e;Z(FP8C4v9(n(N3=FSU-I|^c|>Di4RZYa>LO6y_HQg2u_w-+5g zxstvO@>pnGg7bok@Pvs2Y_e4&hFuyEW}if&zG*}J7%STLCLgCb$>R4n+i-H}?v5b1h2b`ALq<>R1O4&aR`y7>vbhY{7XigW^XBW%8~V$cd$LXpO?xzo>_qI(zj(U5_wsM{-p zx-Ct>uQ&nq2YkSMWjsW<-RE77Tm)m*hVeKXD#(`jcs!(X5<7JnaPX4U%LzH;(!mA% zU83{oxmGgANtupuGb%`X-Z&b$?;sQxw~>iEG>Cz35>aVV2StagR6pzyE&pOgN-KPc ze99}{l(V_ie7ZUH*l)xwtI$DjyD%J{;(nEUTIc>l!o3wCO_fOj((H zyl@akrB$Kh+CrY+=6;^Tl;toXun_c;qVTS!9R}U9g!7GSFkosX?n_#X^i3dLS?NW` z^R(c!+$tPCeij3^DdP|C42)VegkvOZ+h<9eVCtxKG~9I(_#IRcEx4OVa~FAnv))T$ zx@U-vFtipePkjoCf4}i^Ryh!-Bn#f*OKY&?{(Qi~1YCL66W{-k79TD8hnTF=+Jlz`&9J zFozurQi1I}v(w6S`Op|tUm6eVLhWhb*Vm}=UR8YP=3ml#GKE-meI@(k2XM^M%hY5G z!MY)>M;czkp1pNTlALmfYa1XAer!jMUG^BA(p{n*wJjMjW zan&3czFiDG6Ozd66$5D4o`BmV*^0BiAw<#V6ts(-@yKvH`i$5Bb|sJKMDt84d8sf{ z_=m{NcOa1$J;CY4Z=PO4CaTO9@f?KFbf3=(`0{%%iTP3h6Nl7M!m<>dJBal9OiBGV z>?Ed<4?tsN5I+5qLe*_@!J{Gv?rzrwq2$}SqA}|r)OrGBomtOl+>jNU+^@sRQCkV; z=0tMsfF4OdUI=MT7XsEX zxwK7wjQG&yVoGJl(9sE849Sy44f_ZPJb8|$B>bc=HEQv9+F1x+qQbX(dWD+JY@=h< zvN7q116kTr2icoe;8xQDYUJ|`c0D%0hL~3v^In55-}nKiWaSZy-shOUbrwpk$l^^2 zoeMFGWr);X4;pmyEIe;Kgj&+gFi(0RI9RpwQ0^4&np+KD)=S}XDvu`=L|ALM2zD>L zNPpd(jIRz~q?@y4Kz_Lqf4cP()Z8s4esiXjw7LEzU6ZmfGjbH4JRixQWf}rQ`&ihu z=qVMn+rwd>b>xdx0jxRehsuHkk|);$E2Rra$nJ1>@m81S?>T|0?~dcU<{aWtYei?A zJx_G%b4g5~JLDWqpf3W?VYt7M_{7;27`3|ue*O6&dSWv~o6LhnLCxXt1YSVVZcWI~ zm`$HOxg#n%e+~9Y59!%{E;qhwL6Q_ zLxsdKycVZSn@t5CCulKPndOI-v+Y?olxFoq;I) z$p+lCw1K3$IFh!s+hjo8#cltv1idroq1k{tWwtx;R$PsyODDc3N8bk^r_qRHKY9zX z3u7Q><1RFrun~6p)!?kML%gT@zO+Gp8d_P|qw$o@)LZugmQ9is@4Y1l@?Y9Tt^IrO z{DlA#xVTD`J6nyv!An+>SM#99_9Z~cc9Y8f_np|i=NhI&A>@Qkq{fODz~#s}aN711 zUpl*p^z}NRxVjq#mPLWF>29#UbOArx5vb&z*zt295+;3UQ`nLrOeeaKtcXaz9?5ya&9HOMHC#)RQ` zlzQZj`!iKxvq~J%_gVpQ<D>vK%&i5iZa<&dF zYR|%T{a)y=Glx5AX*M<9JWSa<1AIPUkDDYOwa@D)Fj$ZY&EG2F#yNSu*;orArIJM5 z`VL`ROe7dAo=1i(DP-h1<71;NdOS`BgoA##XxSyv=8qG^e*^6CLeNu5ZsRgJpmUy# zY)Pjtjr?K9xLj(c-a?+fR1x{kI0yTk7h%hfk^F0oBJwLJm`u1Q3t2tIxZciD@2Jq_`ZyDtT&FCdCi-7_qUeezPiu(bTOStvfalt`bqZHP6%e# zTvuoE-koChD9;zBzqDXu&tDUy>L1}ejx-Pk^A&{SavNBY)&u6Z@H%U{%};Rl^pNG_ zheNF0h*#|XmL06AV6I?ef(4r)VYYEyvY7H3d(Q6SwTzvQk?>q^AiHb9z};<_-)H`AJ#c(?C2T8bud18%P8eEl8DWW5YoV6{Po9lvn|6DR>Oebka+ zC!H)|ucS1v!(sh`pU!E_P(U%8xI~&M4sqcW6&zrkG}IYsJ1MSuX%45}*nnB~1%zY& zJ+*v0vq%7{q0HMKZkW%Pz=7HmuK|V|A;aZN4WEsD6;} zVoM`AzF*9QvFrQQUj=^;NS^1LBzFCG1);$; z%AV266*S4|GOvzLxpNCiqqfE9F21K6caDOWwrp4u{N6Vupw znLehz{x382&p$@0<*8uQ`gy_`m3IY;mfmMCx_)Cq?vLX-r+gITI1X24OgzZSZF$8Q zTE63quI6%FZk@MmZRucV6kKL&PkXVA6YH3YW0GAr(wNTh)U+l2!?s4&}3L^G?SLYUUYr2@}mMz5Sa;nPLEy4yY)Z5;YxeDlB%yzf z4G`uq^;*G#7f0%u!Y`qmRI7A$!wwmyKKut`mv)*XK01j}(CV{zt8s}*|7*Zx8@;(9 zHBLt`kwwNKDUu0wWf;S9EzaEg7nyGd&q#JrmoN%rf?1`tX@cn+uXAc7yBxa-m(?WY#pmoJ>lW~PL*-xAKTcb9%)G7DUowa43-ZkJ=0;T^4lr{k3?wN!d6s#7I?RygJ`0pEDjSb5^JAUZmx@Q2hJ~8U_0hwendf#fqGS7*O@GHT%P!?uZfCYK z#++8R;Y4QTnwSJONo9p#jOZ&fqt}M1+Gs2og+rWKUa}G@PYh?2+eUU*KpJbH6U9C< zT*@vu-^)2=RLv;b*t3hHt67!SQ=G)8a)GaD4=3r20;@JVfqALFg7fbFbGA6Vnay5d zz%Erk$td)uGNYaBnE6(++#@PQ?EI`TT+6LD8S_6km?jA$_@v-5+j(+?&}ml~o2EUF z`|!?dwlE-tbLZv@*7^Am6EU@bsaoX1)bm+J>J@A8IeG?rdYpl;B$K^fyq9Th?qrG&2XhX>MW$Hyf*{o1p1l=3LU7dN z5!?B~i(x)4XJ#LIz;P7ZL6P}s z!OEOXtWWYE&f!CetZTt0CQN-aH-0tC+?lA(4Lcmg-d|wDREW1T`GF&tE3w*~w*nq> zfl0a%>>9vWqcu}(5zd6Ad}Dg{tzu#RVs_2E{erryLF|>NR>4t=Kb)2}JI1B&M&(jY z2P2%?BM1JoFcQEY)6C{+t)WzxGL{T+KF%UYruj%!xg~?hcM(2kgdFUf{K`*H&C)?h4YF%2OgH)U-kH zLpGhgi|XtvaS0<<-OucPwt|^)eJ6WhQZ)0Vj$yA{ooEq#xrycG=?c`nE7%i#Q`t`U zQS9j_{{)s=%h=@2j)Lah9-JL%!EAU-tHnq^3rB`W_RY~ zva9!|v9}cU*zlcv=HRaD%$W~^f^FTK*`Sr98QJ1o&fdBAE1IGtAwLP@`*!|HflN^; z=R;gr<>E17OQPtmb#}fF9XsAb7njgd@%gLvS*3} z3!+}LvWzV2dM!v$6Th9g-FT5x`1UQQ9EMn^}Ewtfj@|!Aj+nI?iC{Il=8Mb6A(a6F1&DjbnSmE(q?YyyH{{ z=G+Jx|3{E&;3xQAoF;gFE`ami~|ew<8-i`Xp!1hjc;h z$^pU6*mO&InIXZ`R|}cPZEEb3b1ztZ-7)O(n|17%DUQs|pp}fuo^*?AF;2|84Xpwq z#jv|RtY%Lof8fmh>%?XoB~?DW@>Q_EI)-u5(qeBmn+R(Bnky%!mNIL`#50NW4>8rk zEKZ&BF+qA)635X)N3cIhlNsR0R({BLV92N-fvW2Ui>fhsY-G%Q&Uf$iOvask%p6%y zOL0Y~<;z)HS?#@BSZm%CX205dX7{Ms%v5nRD-=t1*!`8TA3U8I-MSBzTjrNB`AoYd z6`8Pksvr<@wFD88yJgY$+Z>-EW6p288SJ#Fr`YMEl-SkL0)cq1CL>pVNuauC6elR{ z2wQzEllkSH%vwG$<(M5=$H0^MjMJgHOx$KycHiw%cJ+lyfp+j0&Wy76mS#b&%puV^ zNu3U7F6*6U`)AH#O^-M-nG<{kJ?+m0+d~xDmW|oW?bvZFzdW4%bhMl^pu31Oro@D4 z`fSGxq(!swD+O$MbuH5*ILL%lp5ok;&t#>tte82;quA8iC#-*lg7DL($IK0xZjNW* zDfVD<2D|X0IcM$DD1lq=Wx+S$5@whF3Pxe;1qoB#im7rcsobtGnOR)i#?Td)n55ww z9PLZ;tYNNf<+hlOOjk>eAbnl_g3Hzof(UDScEZ|Bws5l*n^Tl1*t4^=GPH17rKQOt z&b{ZJtnspZ)>r=k>-6^tdt7T5yL+_(=gzPeqg=9znc=K2IGLBin(+f!85=2f&ec4Q zk;+-l8{O&5F9Q*K^T}V!$h;WF=UOv|$p~ehTbl^pc;{G_c_^^wYBw=YUB3u6l(Y*@ z&KWCU{4E*QER~rsb2MZ5KSgIASJT&paixhOl_pIpX`u2$b@n=?R0yer3<;48nF`5R znk5Y?(VXT;qT%j!Xpkrxl+c7iMUpY3;=S+x_jCTZ_uReqde-x-@7G)4F<}A*Ox?aZqO!b zzTrDP)us>YRu(|Sus>WkRKxsvfz18{Nr>jUcyJsIVAc09-OUwl)kL}M@7c`@&2AhI+7BL0pF_lIJqq1 zy=DS<3JUPflxCx6Vhn2M$KY{h33giEpjHaoFgR3+IBz+KtLxTc)SgdebK->X-?IzE z=#w@`EYyR6JKt%sQw@FC?}>&d66rCC6qxoRmzb#sQak%AG&Z?`TvvNXv}G1j^?T-+ zFrk26dpL_cQVJmB&TYE?g*eAyUI zCw!0O4HxmKrDOx|s^@VgVdNM2PvH>8ebPZ@Zv?RuJp)ZGzR+eFN!zsM5KjSf+_-&= z3`f4AaF5?xkQCxJc7pCw5&^9;JN#2{3j_VlQ2je4&gIIq^KmeqO!|+$VA4sZavEv$ zD}?y{lGx#T9`_bY5c9IRJU?w03_4bUGyfXWou?&BR&IDqubSGEzEdvb=*=lIoHuVO zD7>0R_7oG^t#SaD*+j$YpcQP|HijK0p^!Ma8rGlN&1XURaw@NdIqw-=cxd??_V9CE z_Nwe4WQM&b@vAp-1@m0E8gD_)f7U{-wY`HJni`2;o}}QNWwY_$RN&aD@|;1#Z~Ezn z0+;qqk!wwk!c?LASo`A<1l==WH{BnFH?tnWkum{xX}kqHzjzV*G2jJcNw&kiv_;$_ zc_r>fVijJ^D8>ur*4!?63(nl>DgKNe!xG~I94FwxnTu(1N#88-xcF-Jiu49{{Q(J9 ze(Wph-Og~Uo;q>aFW;kAZ!;$!+W0#z{A54m1cJFhGfHu;>Z?O zMatv$N=_^M_@Z=QKlHq9Us>aML7krV)~+Cdkg0`Unh{!Hbxzx6u5A2C+8yO&B+E# z_`;CR=rVA6DPe4(|26}8;Q>c5`D zJ`(xP?3I|uO4ZC{>y^qGr79~hoUwybw41}tls|!&X4qnydOxVm-U6!(7DBYeccR&3 z%P#$7!{1LT&#v9^3XD;aTVT7Gqq~A}w0$wEe?G`5+xl?dgJn6rmmO%VCcrta7v~D9 zy70}E#}F!P#A$bY!Q}6Xyp^leF_>dmv6&2;Vekh=kGk+#>^r%<54*Ta>NB`6)uZSo zYRD}}o5MZ4as!w7T*vmQ+8i%khg);x2VQd;B_6X08@$tu-7h*B-a2|2_s`qHzL4}_ z0qABcY1#|cDj z<$7hGU~^8X&yWt#?TY?}qqKTgweCWgGXQH9f$TGTGemej#LEUn_#f$Uebb;uM# zE{}j%We^c_>?6OtszAm_6V6|Ah6C4~$;N@>WQk}iZ}0mKlQY+pz|!L*@7Tjluub6< zyqx>7EaT{IaEYuXb$h`LdqO|+^x%Vw2PW^|M)F@O znwV;ZFu!j}q34Jc+{>PgK9d8`rDiQ`-ZK$8xh6FEEd<-GEkI?S9a+oQEZ_I?cyhP1 z;bY@_-qV9&eBPlGrsa8odF%?XjGBwHw52dS?@L)?>=5mTH7+XNYh#=J0Jf=}Ad}z4tK~OWC2*I2; zMo0kU_e;~@8NN9Fb`I$gT5pNfqk?itP0bF~)yylcn1 zov&Bc{#g|}m(44!IP66CEt0`M!u);ju43RUeT4iy{Tzn(Ou&1>o#eu|GRiGJi{D0< zQ+stm6u7pX7zN}Lcg6lPAwI9z-eof;+uqxNVm4H+XB_QW!|H8SvN%QZSVB9a^xeu0*|#psJ4%W-dQA~>ChVYWu_gSurS zByLxA*=kV%xa$~*=^yMM`_&lpapqf-yfA69YVQS7^JOpETd!a|rYoX%maky!)wM; z&zMTg+=-(@nY=;25QtZw3XZXf;5E+@@0f^?DTjYiPo6cOndeLG-)Mlp?OEJ-; z9BGGDjq{;PC!OvrbYMQV^9_*Pb>dW|j>+1`$mzj__^W6qUL5E1JlSnMZrG!G6^e5#es|6P;)pRZd9a{h542f;*60GQMgO?~#Pho?Rqfmkr=@ z%Pb@tPmvwJ+o4Nv4nzuV!+Zr9aMP`UcU#^-8utpmDGEW9MK@%b_(1K7t2{yOD!sWu zjZIi9%x;f~p+nOpps_}qmA$+cmYtJfll4S+#;Oh=xy2v%O-My(^QR-D65K$oDGlp; zPFDrJr-`zq@J76chHZQR*EWaK)4?-QKPnRsrkUeJP74ew{qSFx4mSJ_fb7HxaA-j! z%73_l2lsv@3Aq)e!!Li~6Jt?UFwYXU)Ng@Cl^$@lSc)FaKB(5i->n}_Aggf!d=x4s z*-06T zc%D4oTN6M6{+=Y)lGV7Um*>Hxg}0zBHV8tq+o@|Wz`K8PAaw8sd=06_zRD<)*%E>{ zAcoqvkH8IsJz$hdKxj!FR0^bG?1e{sU3V^=&ewq1Pwv37s{IIVA28^A22Pne8TWe% zutFCy$$b4bp3YrKR&i+^doglTvE%S=tqTv~u)T@h2L+)Rg%i^b2g7zL{)K z=_Kl6`naq!8edtOBMGgYr7k0yt@!Xgd#Do={`=J^N_6A9F0R66F5~)78iDZ zq(@C}g6oqAxV`N>WG3E&Q+$q2P)Q{^)wE-b{{phrx)^p=n3Gv2?ojc6FX2~R8^ry0 z7(B!(A?Hd5)YJw@HRe&cRR~K*WL$&^P70rS5-JmEk%S89T9iUvPb7W1EgU?Tynqd%d1RHsWE$6Dz#dsTfPWtF zP$Ks~Jfx#czxsrNV@@y}&B`DVAxUK2xk0RP5aV9*&&AOrHdv={oNhbwiLsrx1Z{Z6 zxZ9-#bmj6u)z}0)s?Wg8&zwn24Dx&9dOSVf0h8CChk74Bnk!52^QCSEG%LuHC?!Zy za6(4%FwWlCM4XTCAXoPm4Qfz@$5SS7N6fm=yte^XHvT}tm4ei1PTfK$ky^Y{`40YS zIl|xTE0{s!P}FShCxvHXaFcB!{oy0duDakw#lPr)h_^9*e>tD(?#hIJ_VMU!AdJ2~ zBLJHinxk?Me&sF&mzTSkr&sJ?w6P!a>o(#OVLmwOd;+t}dmKuw1lc7K8=*}544KrF zjYFXcL|9&j2AU@mmn>80b&>WCb+4 zUdKZ*P5v^;R(hr}6J3th!-bcET*~rK80_Lp_j%>vj8PX%-~=%|<2EYjKf=O>Xb>*U z$IVH1@Y-Jj(-PlM2rOrw_dNzDb%rGC1km&T1@thVukHPxKUL;a3BEuMoJ-wA#Xhfr ztM;j7-M8M8rT3%g&FV{FzxyXWxmgqoy_ZvE!P`_?H57+@A~9R(AkD10jHgs4pwwbB zV8U}q={#77S^C@ia z>%&AZqM3fYn}RPyuVB=KILIl}MXA!K)M4Los#e>MEywxVMgqSV4++ON2@LrD^aRVu z23-I732E!N3dg>7g0|cZoMkdbM6V6vgPMt)RNX~L>@`50+Bi&c?cqmgvrGoNmSctg zUGTCv0F}PVH04)2Xtz5-Z+bk0I!S@#e}iz?@e#TSPU0>o?g1&&C{!|(gV3Y%;grE0 zwEPhTEy@A7>3|rl@p}yWrcXgXzHa$kAruxpAK^)tDFH7z9#=(&;)-MD@Vh3Q9(~eF zH&v{It^_As;G$1IG_J$jx>3;exDZ?Q4VZ52N}jPn3eGUUNelBQGrz9uQgYlJ;wOyL zs-_>zniZ39;?8VRdS(F)xm8Jh@(B_5$iuH>1R>JQl^qJizO4Bs#rGBRWq#P^9`eeJ_*=qszD8fNutz> ziiOUtNyw9LP6;HKolwu>A}3=ZLt2+dK_AD zhyM4ih;ct#&6C@aO7pAsfw9q4a$4vT^liF^ni76QKQfK{=<~(Bf41@he)Z5R)l&4W zV={)=7eQiwC0$?=KxM48vHs(1@J--ritm!)zf<$jwO^R$7}*Qo!;6W@a0QkYOdN#swHQKHX)vW4;&9wt0?dRWsQ-s(7+1*0o&|#V zNim-ozfJ*PA$dH}tj278m_SOe90Au!v+(n=yZqVd1Q1KH1<%s`czMEcDt>=9X8mzw z98CwQ{0Dz<@GT{yEfUP@RrS=VMIPz})#0AbYTU6m8fskRakyzB-5K7G6XyrfhCz+}HmNYPqX8dA$dSYQCV&kc*$BgL%*h;Ea1(9z-H_+Om06M%?DI>|hcPHlK6(=8Xlk%c-o~}s5M*N7DD$is}&Ma0@D7ezCo z;Is34DqWBU3Ae|I0*5r(QVEu95-05FZY)}~A9kvJG+|V#x@I6-YJoMud3#QDIA zjCAbN@dL5%{8{Afd$7(zh0dD34?+dpN&O{l?0)T!Z)RH}SRg6(k%EN$%~+GY5vKO| z5iuJ(eEwY;m-VlLv56OzdwnnQNxUZszK&u&UR2WKjj z$+~|=F!RY_GO{Kb*XI`FkDL_VJWW17`m+^itf&GJ&k~4SGzLeWEh8@^La}b`O!EHR zIoSWak4`_immIfnfGD>bsvmxelzs@s$hvrJzf?!oI5$CR#4R|keE}Yu1i~~M1>T{C^tSQOtAAiAWc^Y{YWls^yo zn$1aEFjW}8Ynwv&XF<$p8zy0VrS_AeB|QA11yf%Zl7@g=CGTEKqDcY=S7!wi>5_ZM z6{=wI-f7&Bi$4s-IMEl^dWgTv0MwYZ@Y&{@;DlT@5lOlSjz4eXJo6azNxh4wN8=Er zEYSU^Ag(;S0ZSSmqjicqDa}qnd51~7d!FGM!=O5+foO6oTpMYIx?H9o^)p z3;jj|WL`rU%BW<+_7E3--3Z5bhCEbUR*vCM7ocCx5Qgv^pc!sZo@gir2jyeF&^t0v z-Hn3j7n$wbvoS=t7;E=mrD}T=skqs1dhU}w{@lJ9rC*;%uZPk&Uwbfm${w#kHL2~W5_*bRq`waND@CB%Z3GX$Il=D- z?}BNr3`A8T6xTnX*Tr1%Wz+`J-LwkL>a4*=KM9^5w}!jdk3#mz4wAO_6QL?+p=#?& zuw2LyJC_qA=Tkb_VY3VR@8p8e-)BT<=4p_cD9Z}HiiF9@UN9ly9~0{~28r4qanIiM z7@(Vm^sg8ixlba$F^Ko)vm>4D9)Q0$p2JW}TXY;ZN9Er?NWoMGsI_|z8mGfZYwvl` zIwK0RjMB)9;263%eJLnAiDRvj6uexojTXz~n623X(Ck|X?q3GTKwAau&dT6>92%)| zmmxA?eI!>S9;J5P!OMI$Vf~WNIQ#KWs0zGFj*aZ+-+en^cG*Vg|E>;S>x}7>B}I_- zAr-sZ0&s?LGljw#=sVv+UYz2|al2Pq>EO{t4LK@O5 zX^p!Zy=Ar)1e_bdz^T_nt|yUETCBl74i*HB4qtLxtQ~vT*x=bi<@oFR09E{$ORg6@ z$4*`qehH35vGeIf`7DKoLV4C~k22c&=97Isx3E|DFW%aihUS);xMa8v)*Vj(g*S&u z?PgnuXN_R&pbl&*KE+F@2?5*iJGlLzCC>30!45?+wtsCnByJA}z2)h|IMjwY_+SXd z?{p(R{D6(e=8=o@yyaD}?wp50Z=QkMkz$}C@w8v9 zgVB5^z@4c0hY>ajxVmH~InyA*u95Eq7ytdRb-^@ld~PTH`tkwUorkJu1~^0}c|t1HlT&I@!eJ3uuBzu~=VQBHSDX=(O= z2)YHl$Cu567+4&O4=yf7{%^v3i#8(82Mp;p2N7cO>KvRt;|#AJ-h%3aex8FvA?l>P z#psYMG+NHT-@J^+@3*q>$C-Ro7#$#!pFc7g+7w84sjNc1)Ij4?&v%z)%NszYV>0dL zKTEs+1r}_0i_bDzkxjk9JH{pBo=XFmviKg&Rw`t?yyn2S_T|v=ES`QHF~IC;LQvmw z7nHFO0w+z!J@Ah2%k#$a|2|Tcl()>nq8Iq;<3pVPVn0)LdI{Z@_ZGwz_#Vu6!LTje z2O~TK@pzyIx~scmYF8H1vAr2@ZyUf^>qxvk{DX$(w}IR1D==>~2Y$Ef;?S~>r10Es zI3z2DHNp#_cxyI&+vJUIjdr-43Sh{jJbduP8uf*{$-b6j7$KAl6Yht=QO_}u9S@`H z7h1!$W9E$4b0;$O(KqaQ@(FxA`Rw=|RD#RVZ`s4_NxVAa?}Xh~!8(+Sw$*PnSH{^|TbGH&W)q zw7ev?Ksg!Wk2?_@(=TY2#{Ud(0kRr7zLnyfP zC7E+j67-ZC;pV@MG@?eIOfPGsk`r&ivT*)+vowtl5Pwh1gilh5{$6;rI~fi+t%lBl zL&X1p0Y0v&g|X=C&}HJtOY#$eC3@PRJXs8s4oAY@Z!IX@5RH$g9>#QUJ{a~apR2k! z0*rDp;GN!nv^GhD-x+P7X4?fIx)l7rG~l%^p}7c z>o3yXeBbffpPeK%^fZnx>4MILJajFR;{I-(2p0l?Rvi9|byn}tE^{#sE-=T61Z9%D z`a7>8y@(!rIRnqdJ%oXx7+5sU_aD8!1{dA>@JaJ+TvV0_2I|kyNTUkRuD*oto=$|H zf;l)d%@wZB$%PP2{w&LD0^9K?5wHG>LLng)A|fk+)3Hc0uvJ{S2`D3Gd zJS^$^fU~EBpv1^uvMv2H4KhxEiR3&O?Oz13B$3>iI+4AyU7jtJE2i(qQs8cG1Y@IR z0L^lDVakjKgg0*>X-XUfJgF`7y?cw7=1_}fTpjK*1tu?{nbr=8;FKp-BeEL)VzpZ9@3Ul8Da1 zA>3aV00KU{@pSGPkQ=;6UXJ+FT0Sq*d}I!H_;ni7AvhTuYyBwMdYd{2hv9=A|Dn_7 z3AoaO?{!~#1$mB27!n!47_Quc?8zKZ^ScZvxe#y4P`GrUj%6aDc# zjOT0j+rDl!5f@EBHAibu?^lJSCkFJBa5;UT@(*U5kYp`qMPq=qKHJjYiHmBqap$B< zKPYDJ$wN+uXIDzq-f}lDMy1P2l1-;IXbuF0=f?^qlfE*=!m))%+ak!**{(I zYEc2)P?`h{HCeFHt{yz3bHJci9hX^%uztoL;P1`_(A*b?b58RhJ39n#NDq;xOTQAo z=W1A}`;-QW9;DxG`|Ehr^V zg1yk;#8J@v?*Qqmb|_o!R|m%T8X1|hnfSZ}e-s_USs{h7j&(N(d(b|UFv&suxJ+WRO-r>H7SE~D-TXQpGSIZ{xTmrAENNH zU`RAv3KmWR7U5^0a}a!~4ND!n5ZfCEaFXXMyS_*f=7r;s5Fxr22AK0BqB(G{Nv+pHByT?M} zT+~Y@JFSEtg2(sR?P7aKrqrQL}{ z0n*eBYniJ}`ygrGRM;-pM~;l?Bk#{Kyji)84xAY$NtqJx;O4M#_Xkg0kzff??njErUFa{hIf-!DM98+caicX!T z&8!!`3_1H!_;=U#lC+K#y!yrn?S8uBWuKj>o!bms-%GQP+j;4NUgZ86 zee&ebMRd6EjylXNLZg6DP+Mh1Dtp$`s?SeQa8o+ukMp_lD|p28Ya}!@1cPIB2l__~ z69=mo%$4UG$&$-6G3fbcGP-^`)v0M`3iva=JL~P~(sN<>_)h^2PfnteW|??H-3sdG zkJ0!`c@SQ$iBI${;Kv*MTCSr9H$0AD`hjC$qLDxoE?y-n#^S2TLmo0uc0ZX=g^%mcs#9*!K{BDXj7FqeewAf70m;DyQ2^i zR&SLf#4s$*`o=g`1gKypiz~^g=sP@$Vm?LxjN&Z$Zk|h&~8j(5F z!!QUYHJ^oh4)Rd4T$WdFm4)8VACbVJpUlzv+00`P6ZCLF(&+S)`Pr9Clt?p45!(-z z&%^k9cqbI#YDh?rE3CWx7;^$optb*X{Cs#aRqpXE>q|{#9tu3fl~Fg~;%GgrG4+6B zice@%MH}(+-T=GTwUE-^@g%#p2zzd;)2Wiv@xad`xHaK3xv*-8Sid?1wt6NInL+1oT>4;O>C@E$ldO2km1+aRb~6YL%kWD?zojrm-|8=8~8y);67S+ zIRd|#+o0sS9Z+k^=gx*ZQJpncG2`77477Hp-Iku9zp;SE&UyiwMlq=MG5}Y;jlh#O zA!smm10Luchkb|m*{mr9^GeF0uH+!!CppMlxGomEOeoZ;F9+qO_0W339i5%hz{y^R z(KhoT7b9GGbGMb?qK;s=`>2md@9)N=+v7lH6TsTo5_l3uWnT!Ev<=Dseb=YVYM>ikb4ySjjql#QE znyhw%y8~swWQd^90$D5!BGkZt87`U2(b#+~T45ak6(Tp$+*T1>e4p@c*={CnvQNP% zE3@qUhWVH$6onhu5ZK+8PcNDIqv_~Dq-PM+?I=EdL>dEeF2vJ>9Xy?$Spl^41o{`_y85GxZ-mJ1?F(rW{76 zE6RjEQ-+P3lHmc9i?TO&!_s%1Wa3R(V9iXhZgeTO6;319=f4IsXATnN-vhJt96nAx zPL3F@M?W)Z$j5syche>CsoZ!uh&B+c3!wuP8y$siji$jbha;_7EE;A@ZjF!-zh z99}2MF2CGG2H%CE-r;Iy#eyN&b)XfBFAd}U41erN9tUY5BWiQ(Dyq4RFQ5p~GSMy=jg)ww45aLX(_Tr+k`Ml}p#8@RaXJ$+zkao-e zAq(YVfu7J}R*v|g`~4go3v2Bi501(T7$$wXcz^0)@O0g!#I3hvJ0pBPh!<~DB<{Zf*J+S>2$Auu;9uLaB6!7$JGwd z+!-&hYH>65+wlemhQ`USh-7$8f{9n(H0bJ%qds2;ac|{ul#s~Ag60}}c4aqI#vOp1 zX*`JW-baof^~E=tVffPc4Yq9vgE=c2K=gq=dFa~4T%!MIR>B%MRn|q4>mzZx#1oUl zO%K5S8v~@^D0r+gLa7gvxSfjyaOv_inE6%+mT+Up9Y2RFqGpkmawDWrN05kJK=6Bldy@nD&Eb<2z!STZzXF)bVkwB&#C73nRA0lghw3I9?wE z6DOU;kt2JkrdkH>dt(MC9yFut_Ip&w-i`O!OP)1w(!>#}jd5%E-sh_gM0Hacy1aUW zmJ8qDita>{`+GO?3M_Bq@Of2?-KWV$8n57MIp>)onFKIhdk3x65@A+DF4MH{8gk{` zOe4<+D6g8F@c)nAR!}84(-kcO#i%D9gX4$zI{Q9tu*?b|N0JWV2{S$T^rar(yePuy z!Kv6-&3}H_2io)92#ga`$iMXEC_mvgUfz2i54#&M37_g9?ocSOL&3CUpp*O^`Us9$ zWw6+^4hH3nAorXUcXq!C7XMC1{f@`fi2r^Y`y2t0c}l!h_*w;n(cz@WcBoHui?#c6OJ^+LjxjAaxy?2O?bN3qPcyTCi8;B4{0% z520cgQB|Z8eYWYcbEn7=Ufo4hQ(l0*38(losCf|Tm`P3^Y$UZ3-@)sc4(-$B=UVPW z;S|?Wh;06W%+fr5|F#c>?h5noQ1_T?*)Q?<)pk5yr_7G8nt_!$wxHp*6b@S%k}Y!t z(IWK@OkGn+jQTXV`Z#Yg=aMwJTG)$+H%YQpxia*p2A_NGDgi#9XK{6hTX^2H6Ho^(bj(h9fn zzTrivlA6W2HQomeH6Ho4>@@M<&)s{nRVg380cQhJaQ6yb6gE+ZC%!RQS2798`L3SP z2=b^j z80T)yVHD@_|LenD7&plep0z7+(t4iwTQ`Fdswl!~v2Z4ZDF8STUa*1_y-fk$#CHje?j0> zC5^CG0iAD=xb)^HsK34pa%cgO{dkAmKe`tV8?eNEID{A%zD0`*!LZPN4K_6eQ^R{# zV9||6{&{(ee0{5dts3D7lXnq=%0#FQGQr!UfzZ&{21hqf#HV|0d8e&JIU~OkFkrS& z*-TZORJRNKv?}nTss!A?H`%T;cZsNt`4vV|DlIHV4a zW7n};_YxF!A0cUNQ^|#!N1?^JoW4@)!$i&NP*%?&drAqrh2N5r-zPDA-~rAMafhB~ zM<8Idln#si=4Xd>;j302G#{N$-SvCG-*qyvy_gD9pS{J(XEN+#odh)D_bEF&w&62I zl$-KP0n*n$2lrEt@ym?MAeMF-MQxYi#gr4g!YlqXYzKo~WrVNkhft466=;2{m&9i8 zVq&GAkf(u?ARAvm7wDDa?9?b2)iK9`YatMm-^_S*976>w2RM^y0bAGm!qoQpm@~Hw zwuj9I`6ugf-?9_X>=;X$+LCClV+&3hy8@Fx&EUrKG*Mv6M|%8q)Xds92$Rwd)=tluD` zvy%Q64kk}8IZ^IGVZ-aYKV~r`;X3Ig!uQFce$ZzQQ9RD)tn&cBA|-Sf{6j?EFGRUD6z>ebhm5PaFlL@-WIv#bFZNu9(g#79 z|Em(O9XG{)hd6La-wditufehJ$Kg+OC|x|PgRSNkw5G@vUCwO7*WW}T_PPw@SgV^T z1n1I*cMsr<>3TR=8bKqX`1;`q2QYj(iLE!&0F`y(c(ncrX>P6sg+&!0{O$(&sLp{2 zSK8tGl2N==mjSnLrGxmYNVw5>htAq*KrjCggS^)_(X(BUo%iqsN&S9{37k<#ywB(`P=^41=CO_ELFb`G)l`v%*74)O~S$woG1I$vKaEJOTJX#yaObSY; z!cRr9ta<_3r6rTo(}N&*vj@?4_XDG-0O$#MO0Fk-p+`?v;THFc#Czf=GR68V3_kOt z;-&L3(^8187A_&XTkqn{)e#73d*InvJBXORfx-hRI61hB2$qUM)rI-^eCG$A!@~|- zE7XV+gQU3kZTCsL@jX~#s0L*h@A9HWCQ;)NXX3G4hV6BB#fWloNbHgT3)f~k=r)WG zwr+*1bNAz3wk>cMU4wuI9J|1%h z=V4!X(&qsWV|QW0&-XR2buL)yLBT7G^{0m z`Fep`%W_aXG7-gw!}xwU0)ehg@MudDo!XH=Wh$q0r$Sly9Ageso|W;!21C$sr!UHl zS%JMii?0TfAoJS;@+;sViHX>ZlBYN<8Lq)8*W7{1=21`2FV9etJyc8RB+29R%mCqr9n8WbeWAyYAz*A;xs8xD1U6(<~@3|wysxuoj zeujYS{7in2Eyhl%O#*Ej8&KREMFgx*^6XD-fxG%!$lm}dcxa$};hVe-N*dha2Xb7w>@3c{+5^_SHfI-ZlxI7CRl&MlXNZl4Boof4 zp!)Q zmJtP!+qgc(lC=LyrbW7U!R)#dtC=Xno(;VYDgQ?3Q{COHKI_D$gg*lJYyG_Wzx3D_ zU!+*ozmI|aGy|;S)39|k1l=U&pn8cDUDG%LXO&$kiyB>p4tZ5nKy(CCoNK68)q%1@ zEt|k4bt$*HUz7X0J`W#?jZtaQiJV^2B(&J840U3B|NEm^+|*eUxLFTQphZt8rrWu& zLEAR6k|KQ|ZJtH?9QC*p<ri{hx;&V;SR|BahCnE--_+GoW#l;DWFTY zoMbg8Sh7*GKEe;P^R(4zKKI*Ag;SYci|r~QXftZX7VZt8ucV`>t3^Kj>8!|FC4UFL z=nKzv3K2CuIE@xl&R~2Jx|m;rTcx@f;Vpr)FScO&k9NMFSf9q4q>x76O3D`P2HAlt z;CAm4WN-#}%}a!6zLUm{ye=|1C>6hc>!LbtyV>MMGxlQ0W61T$r~l^Ph5(o8*l#3) zM|l3k9d>bRubOioB;TRg_zTc}>BYk4U99rwzhJ-jAz2lCpN51E;Qh;Ak=&RHH+58T zVURQL#mxKko8C3tKi10&_D#c&HPQ6_tPK3;YKLpt9XQXdk2E|e#us@A6&{MHG7^E} z7QOIkkq^5?)`=aTDa^9ZuFx{sU-)>-9b6kA$H{lz=Fl#|H?!V5kIRT5v%1>tAO>(4Zf!Td{;~xweAc`YZ=l)`Wr1wuPK~{sL~ex{s`JRmS&xZ^5G23(&0>rG{G5 z*aoFan7HH;v<}b5+;w`K^L%B_ywV2^lb(_lW%JpcPCcNLo6DSg-$%*8h1}$FWzJ2g zgeTl3L!SRBC#FS>)Nz|TiCd5b@5ZgT^Km+yS4$W6K364vX?k4a$(h`E48hrd7Q@4B z%VAVzFY%JirPJDW(I$N#sC0YC8-ixKvQP-7_?UC=gLFCd6#l-Buj{FoMk*@G`eU-W zH-3B}j65_X~gM2dZuEZpbM0;_)3;(`r# z@v^BoerV%oniK0W;i)$JbNg5DPAeg>el8eq66X$gi-BuzD6D+ciq&iN*u;C!;kaNw zQ9p1U#GQ}fZCf6h=yVxgiPh39vuDggLm$u=aRm9ETHF?+z}2oS!aBiYM0-L$8u~Bi zHrsqbdhjZV`F)43Iw!(@y%7&R%FVp}hn#_D&)**y77yu-O(f-45y1a7ZTY8Y*H(|x zi*_))4L|vJvdgF}VNMuxPreVho6K8yo8(0DsM$Vo_Q;|DXs`C-=V))y{6J6sda8vS z!_QKdKOLbq1?G^Gz_1Sjv{=zWX;68nN}mpEu}A%-S?iWM5^-7pW^6Fz=g%|7Veqc6qHmg&$Yzf>E?*#H|_}P8FC(}VCgnUIEbYD7`JI~JH^1Apn{{9)p;)pgE zus;f2Y)xPpbWoQ%UAD&VoeBB+9|?XO4{7#yz@f1iwB$ngI*T=If4!Ki?b76g<2GSl z{S3<0UB>=OzE8(E5>zJy(G;~s-1wauG!}Kl!$Ma{;6Y7J!$^QTUJ*)9o_2jsa zM-$n7;(tJYa~ytKV#d$*{l~@S_R$T2aYSd<9B#LD7e<6O)59{#O_qYho@={=z)A z(_we&YqP70-Vl!$x@h3Jh-+;7kE@K+rG{5tQX%;`Q2ez8<_Xl3p1MMqVfUNaUNMys zr9T;`w)5!lSPVvOb7{wl5`5MZO0^3p&ejaXjGNjppI@WwqxMjbmJ+6FLISBxIzrlc zuc0Km7%sUw;+i5A&Nn>?<{VZ5Nvj%aJWYXFsn3JY{k7zPYz5sTc!A!z>4h$rBZ&CF zZNN7~^O`%Cf)=?6XI%+7dTl;V*)9a@Pa09{tpPY!#uFQrJ4tM-59~3WiFP%j=wsK2 zz3G8;N}+mLa@1D%r6z%wl54PH@&TB)BAJZs-wIt<6Y2N}#hG66C=%@lV{4y~MH>9< zr$Hwv2$ANerq$5wQ8|j;l*LKa`N%G^!|+8?FpqZ$DvM76qiO@G@A5Hp$0FL7Gn?!X z{z?WOib44B zxZ%yH#kfJ>9G)lX(8#j_;gM|e^nfq3(kPFvUz>@V`6INkD-^HJ?xIRr7og$j3|P9- z4dTLv2&Z_C78`351+if|ED(rpF)MLq@LW14Z57YAbTV!@8w8Sri6m9QhWPaHGZ9m) zK=RcQNMCda^26e&MusP*FFOb|e}ve3VHaW8ArV7Q608Wk%==_pMk6*bSSDkMSG~{U z;xBy7PgMX~JWPql-vz|AECNazCz1b>Wr_7*HFZ7klICoZ2kiwbk&fG;(l1?dVbqso ztrI|pWmfolRv=FO{FdY>>yZOz9)j%TT_%HjdWq2T?YJgu67IOjgZ6w&G%hgZ=RU9O25Cq`(o$43w6wnW&-c&w8Rt3YI@f*u8ozzAY3w|iY2LD;z20)>&4y1x z(v1b@@=UsUJI$^`JkOMiqxo;267z$!pi!eWxG|-AM&pYgLA>ZBqsE0p`x9&HzQ{}?E6(&`ooQpLt$?|<@L5x@kTUZ(P3O&jJg7EbamBPz?4)gDPN9o=*!@8B z1ag)of++cJt1&0%k@k`^?w)%`vZK zGtK*wc>B-pndVz#dK;pZ>lziygB#CnSk%y^>}3A+P?K4$kx0XQ&!-KNPEHMX^QSa; zcCIjg^u)@1o?&`}@TKI&?S%^(Ki6fM8cUg)J8aEuu-;eQ$Pv3$M`o0mK9Ubmq4k|V-P^5LX@ zXw!b_1{(G!1U>)6!i#r)xVx|tx0~D{DMx!q_s$kP@zaMn<{ zP`mHtsMDK{e0$rVQ&a(EB_pssxdLo*I>@)|c=Q*l$LU}F$@)8g5#Q^>?1y%E?b9(@ z6g&WYL;~B-Pr%UU%gm)NMDYkN-2D51wDz1N&I!>tac={R{WjrD8tuoFj(iBSm=5DY zmzhZW%S@Vt4cbIru=pVf*zI!->76a)-_2P>_q;5c*BD{EgCYJ6;8;(n0vK@+^*l)sk$1X zwiY2{w-(}Clz_<2VFEI160lsefevsgNV z_rERrC7~qgGaZ@slx*J-3%6f~S>$=;!~Cjm^k-ZJS%2&uNqVe-y3U5QV%=qM0%s7p zvIgq6&cee^d?*v%1G{7!V3Cv$HeL0G5XTIBJrMyakEd{IAFN=G*9n8*ucathu7)kQ zuHo?BsZeYrNuAf_W9gtWXYF7g3}|kH^>@?giA6v?!VBrorq^hDFc=rSjEAIH6Jlo| z0c>#uev?9w zJcYjFhOl>!0m>cQ1S(%-IV3CuGu(eRJ$`=O&*fRAsXynl~8cJdl6Raf5t=BQDE}yB1V~Kz2n z*$rCoR7M1~bW`B5bOtZWUH}%yXf&poW#E-V6>w(I6g=&Ykwc%9ajn~JIKE8{Tor@K z6srulqY{jwJdWVE(J2@heG~0eGHKBx1kaa$LD!%d*12iX>D7Wnq%WZ{Q|&0+E>MHt zNuKa=pn!*}IE%582Q1pJcA=x#9tbdRB1eZC>9X-UOglV{&QalIT3>HQJ8?Vu+T9K| z?KH(D)t~TBZyK_HyXmUi@icT!1SX&UNjNG&=#`|xUF5VJANT>5U*IE`KZGJ*wI5ya z?J@bO=LG5c&hVCx%j2UiK>dk@Y;}++UXUBa^&(|-{%%Rmwuw4SZIj^GS_yIaloMHt z!b|kU8VQJbElO``20*CQHk>RH}F#W4u@bC47qT{n`0=>v1SL zlvk3M8xz4auLyQ#uYjg8P0oqU(%gF?ygcqS2lC$P3u`iB+jO|3iPm2=v2IZPIn zfYcrx`t#j19z!}6-TZ{$lmf3dYlK0#t_c{fHNwN4A!tv6*?I1B(WhDpQjE?+MebQT zQt6BnC3URkhHTQ^P{eLBUqI|b+fX3a2w${zz>@DVn4-mpoZ2Q2eEUq&KJ`7v8W<%MXyiH(iXcqAHxw4##;OQ(>j)cQPis zi+iwD9!w7{C!@t}ng zRJ%;)rdUS6v1B92l{JP5L0x)p)CW9c-;wFX-|_HwW7d9p3FvsthwZyALMbcA^xibV z@MbsSbH9TbT^0pS%caP7d4F_$Xo3$7CSXB?5N3FNBP~Q3t5-(St1qKr*}gY;(>VoV z3kOJMcOiS`*g-rn8HvgcZS3eqe(v&v!)W~W0fhg2gDM~8;a;C4_s_@_kX81=6P30Q z*kOsSoC50YR6?lEX^8V0VsFiV%ve4$2WO!!=9P0Bs4T6+C>Jr#B9qPV;i&{Qc8r5> zGs-dgsR`Y@X*)5RmjvtWC3 z1g+$5AbGL+v?8|&6AstFku#z=bNeK={?;PbA2*`#%zw_`b*iV8CI$%V>+t~$}$&llfycW zN6J>(&x<46ThfkyUK}A=C(e_bIa;84d>)Tc9|jQtQLy69F>r}2rB#hPQO@)pq{*4U zg0jOPr8h`z<$5gc-uX=)m9|6s%1mOLHXRatUqJr!shlb~Jv1#@PouZ=f>cfo+`D-X zyc5J=Lq|X1Gf<&dvXj{{^Gpf`fjHQF9ge=)K|dy5gAn5?{5X9P_c4dE9!=ZudqFj^ zn4>~Rk0i6|?~U=v;SN&&%NP4gnyL1jX{ftKiW@i3N^!;vPR-Mw@FBUERGx4|d6A>+ ztA?FeQ(Xm#N}=>p$tHMLcNz`yi$KCCh)&{jX8f==tQd)*DzA2-9l4D4OV7iBZ*!Q9 zn;l@!hIsa_O(qX;yn@{rv4m4rbbwawN^5wwJr>5#+#w5>>}OvEo(2Vtb5v*VW^}!S z5aN0l(`pb4%mYBYQ>bO`-ib;OEm-EeGgs#{rN8ZK#LG8^MSgz%V zn|dNK()wved-N%lYlra95}%&G|RAGy+a54=U=@xnHwzN>BOE>lDJ%P}^pvJFb*B}Gq;npfyG`e?_nR#6i zQ?sU^>$U`yieD>kN-+0Id6tT6VnifQ4rs0o5uCc%7OCWY5wS)EtuKu{zb!@Ov6na9fJXD!a*M_kXmu-4I7j z+KA+{Jlra!NQ`AFp;#-0wJ%!@>m9o2NpS;i&)-?Ft?UiSlXV2QO*6PsYNfdMqb$a3 zeZB&SGpk z=m)kRXHmV)iFB-|m1vjxL(7p)`p5YTtyC+ZAB*eJc*$(?FO#3U#pyZl<1Q-s=sgMN zF_ESOID*SjmPvh7jk!?8#47}2C3ldH+G+s1(iaEM{-b_-j5sYbjB#XvDOOeXgQ)2a zOftSkpIP>ipT*0}vm^U)MLLU>H#6w+SEtFe?V@n?a2q*)xDvzmJt1F~i9+j+@8tUw zMPkmR!K?EbbY%yRf9V;IujfmEtza{GJmWjDy7Y%Ek~W7J33)uSQ3aT&WB?IWvh>p_ zDwvZ_=M9#UM@Nm>!8$KktN)Q6erO8wa?XH8;uhS(c)|HKQ<~OXn+b1uJmtVo|7h`! zIj|=npF*H63a*__qpp8oVoO}ncd9gWUJYnA-kpC`#|I965@K7R(`Om{)^Ad92G7Ic>L_(dF z28d4P!cn`WARzIbT0NBGOc&h5rNdsZ!|4x|W2CwN?el=m0S2_mZxy&^8L(0=X*4Ic zhA2K3!GE_}k!|yY#1BvDDPb*0koBYhU3izrIruUf3gQLVnpVA5hDA${l6&V&so$L| z?2fnB@qpTDFuLspHWlG`=3g<)4A_7%$M(YSH$52jse=k0lj53nZ6f)zEZFzAW^(*= zHlVcjBb_K8AzrUgv*qxgm| z5-m&lq2D?H7-b(+TPn-hHn<+t=r;Vd^$9zW+=d;h-$C}=9CE<4ldAe&Anq~cOzF)| z?9z#Zi>JDo9f$b27iw>l8{huI@9JdwacLx$T)RmX+It9By@kA4uY-*{yC5_wmF^sG zV|LAx=R}mBqxF+k7BQX0O}T1Qp|j>8Ga6rudwUweVEiQg-rGkn@qV9+sezJ!Fc1vQ zg~L14VRr99$g_&1!!P%cjlLClSS=8Cm`H=wST=puz8MYFkE3vA7^@fd8?&uW5O z@~^G|uKAXdHQu|?c9|Nix-gXpi(bNS+6q`_eT7_IJOv)OJf!=CzQU(%8_?%w8od5e zKxgd{!$^xy^z%Pe9G41#CFLX3;P_WI;%hdnEaqt~8p}B2Tli6F{z(|y;||?Nq_MJ0 z77s;SCA~pc=(d6s$eSHcR``UoUn-|l?xT%hxWf|nJIjGd)CQdEoeIXR1{s(=pS%8V z2KhC)n|B`?aG$6F2n3FkZ>5{)4f)L&oqL(NKK_AjRQif9mc&D@ybRZ~;x@Zost;nD zTbcM>u53)hW^_0~kjs>Tw&@G}>-v){*cOR7!`cu&@ew@#NN{y8yrBHrhv0QlD9qZg z0~WeLu&kWN&+ZX}4?k{^;tj&A^WN<+Pt=9H^>sp>OJ3++ss|;Pyurfv0GfzDVWWOu zhHQ;A^5S7PEB+)2piT`6-}s@Cwi+heIzrXdmE?*3QS(FEy>^jgrGfbVhIJ-3h>dW3p}?P;f+c+JS2XVx>p~AFt>{+Q`3wxD?gx!`7Ycw zvlqtvl({mVM|j*YW!xh&i~KyU37KUDG%_TRcpOBc(=LzK8+3@hR52VB9YUGEN+9E} zLH=0R!2XeJ%)M<#HtydDIi2re{jW-DzPOFYNY;d1#{Ot!W`J8)@o|MBy6|9o2c4A~ z0`mf$&_lNm%~of^{EmG3AWs858tyY|uf{?u=CewMkti@{IsUTnz|lTM3|itz%nq!9 zL9J6*6??tuV{Iz%Rh7`tfCnfRYJ(;YD#&NB13T~9qRqkqT5dRpbL8$F(Cv~X(Fby< z^Y&F_)!#(ydaaEwt%{)Lzg}wTCeJ+?_nc<0P{R*tVN6<52{y~5f%<)0$SwRxkFME8 zm3Q!F#mQf=$OOJAtfQZvcE#7Y4g`S0E}(0S}A50yU|0Qu|*Z92F~PuPfeS z-r0Afm+dF=jlDr>n*x^JyN<#6N8nGY8t3ZzI2y8j6osxEkaZouNgRI~eG@eb9%Fa$ z?L!w*FDpn@-+!dn=M>O3FJ&lOw*U;7fckQ0y0NU3Roa;gl@>v` z!|x$nf3Os$&b)yW501cL)o)PonZY|T9kfDp6m0n(;>r&O)I~#^T&N9#=rb8~`l1wk z(j5&WpC)j<(oKu*%dI%H>?v&BFc{HAMp7^=kXJ)_=McCz=t^=OgyFzcF^;{FG^AX+OQ%?M(Qa=J zyYy2VjJ=b?|9#gR-vT@U*5A%;xB0g+K}vlsuy| zR?VVrtNCG>0zXHSTiCSdtTLXfxCwd~J=u-kDw*K1@F@}D9%8=Lh~SSDSQh7(~`{h&YMDdpEq$Cblc(8Y|PQQUMq zr+NZ=gq?B5f;;ru))DeKD~qmhc!o2>HE|A)|M%WXiKFmFhxG|az>>HRw2#LvUvoQ( z2;RAdd97=pwaN!vPF`k?n;r$;ryTeF@Ft^5{&~Zx)`swk**D;pL zykE^~R;SPmH!W6S=n|f>uE9Ij@0oorhQPf2MQ@F7$BfhiaEQldK3A|6-L8tkq$7tk z@GFD11V3G&@5H;_l5pJ20G_M7hP(T6+3hBkkh>=j28!d@G5c}$g?lB+59(9PJs&_V zHU?d}k1%eFCDs^hhKGOkVV$Nt{1V{D1-}-+qS%$#!|39??;RLm^8hZK${=~Ex6mLs z4?T>(()$NIadl`bO>B<`r*nNozCjZG{)?qrR+f+~x0^jKX-W3-7@Nz?EZNcWKAtX_ z&zAUhW6#0QM0#EjjSn#9&dNvyrLS?U#$xec`=-QE6K;dsFE60*redW7YtaFS@KuBoGeNq-Erv~5e-ovDe;yqqCkoVwn*}Y zd`89gescCw5~Lk3z>|Nf@vhSXaOitSa_6^_H3_Z6rA(2%GJXIvUM~dud96?%xCz(p z2qetXK2RN&YTDJF0;9*|pv>y3c5$vXP6Z;>SyrzjqB-j3vNBi)fs+&6E~= zec5ES%otvn41(eBBv{eXM6W~-k`n2drjdv(Bzo>@xaSg!DV#m zx)7Yx9|nI!K9GQAJZ71QET`)vA1*IZ?2ee8B{A-aNKl-8rsZ<7wcw_XIh`=NWB@YR9u1^ zqAsG_P#{j*K0vf}q@b)s7Z=|!q#{2%@U8w;qNabBDY{=nx_RtCF%Jjc40oFhm?>i9 z2`$XJk&IHVTx>nb({{X;!`R*`$k;0f4~))}f6-f6-~YNGw)QO1@>apM6Gv&|Cl^{B zb&Srh%ZF!QpOH$NP8{UZg?*eMrd}bGs_gwnq`9*=qt2q-gn3`sIsEb9TNTV4DYC&o z?VHKb^byuD;XU2kR0yVgQs}76kG(CwY5b)@oHh3`?6?{Y-7QA!zi$jSR2k5K*}v%G zX)@>?F^}xtT1k2pMLA{8OG)yqNAUUk57<_>8crs@B0J;Nh>G_sk|LZ$#Is7+%WDUtuX4va z-+6F9V-nMUeqa{e(g!WEYDmmc!Y&In?iQ_IWYk{*cJd+^S1OBf^Gv+q&ul%^zGwxU z;dGpllY>z{mcV>g#jv;=bdwQFFS{y|r(<)d=YkWs^lv3?Fn3^}T!BjpHdgvp#tsf!&X(p(^90KWMhS(mTOh)<5@M&kf zxz)f1+&WtdSmmi?;@(%%(zToZ-g^&!DOAASo3Xrl5`}@&znLGhUjo7}CQ1ICOStLJ z8#8S= zMMB1_F!JF{_P>xZvN2YG?iiW|ZKr%#>-{2}IU-?HwX&3ooD$$^j7xFV?Q@Vkubjs( zR)X_fV)#JrKk198tNSI?3}N9TZb+hMpX;0I%uO~J!)J0w^nkb~=I zBgS1MW$CJ%g#KlCzH^c)n?K{}MN4q+(bbp$=csCjCweSU#i3Xus9SvspM0OgZjJH7 z%AP&oa%MGZ|IEU5Oe{pTRDpROua|iu1pT_t$#DKTcKWyhvtMNfxXtOKjyh@VHohiO zyfz5!!Z(5GsyZkxScI!~slv3!ZSYd%0+gDrf~_Iy@UcIK-IbaSZ|ooAU)4jf@aabU zfXJ&~Z|L2CXmU!yh}sPAq(1Jy*qdblt=R^++i@u_|P?=ABq ztcrZStOaLAn}G^lA{*AW(PPfM8gsgi>UW89O#P&({k@N5SdQmAdDsQn@1}9xl`j(Y zFEXT6m>;scL$PhYKK(j$5C=q7Gixmh={Nh`?Cgn0G*BXvG$w7wzn$J#=^{*5tCcaj z_Y~mBu6k1Kl}pC8FW~bMY51VD86{uZLW+eC`rfv}l8^4x$TI{levyN@eexWU#7@S+ zPmYZq5n~Ns>>-OUFT~B__aItcpFB~~pig}$@{c(&w|U=|SA0ptWn~gvm?eRilH*W1 zHGnE@E6lBl2Gme}u_%10nF;G3DN)nwofy%59!E!f z!GzC*1nc~Pn-5}P)3>W+MeJpIjJKm~m#D=?zX0?#s|NXD0g~|I54Dn9jVhD!oQ#v( zpj>8{OvhHxKbnAhwQn){`o*~9>RlqKejhG3?1OCO3dVTG0csojADP43qcjH?;H^JS z#Ms9Q{$wR$Nu(7{37U^fStX9p!jDAH@`T0GkK3{BxD(`+EFx{mrR3P2de|Y=gQ`b( z?^@6w+U%;1^&grswzC@i-H)-ps#CaX-=bKu{03uoOO+}L-6t!i@YB@S|Ka`42)Gy> z#2!l&|v|qQUTdSPlUEB(&3qJ%woJZv8hF9!sgJU?&J_o06)27pZt%S|m zAI(pcuArON{{bSF<=-NQUrIDut6-giLpD|+|#U69u|q-#&`bX!F!axiKNU3>j3YJQM{ zsg?|W<}hT#?o5n-9RrC=>@fDwa|=&%-i|{q0G9iE;~KFRVr>;nlo|`cF76`sojd_W z2H$AZ&l2jNXO5p;hM_km2u$s^qQ)8%EN^y4_xfqTUCU1vjGqTV2M3z4C;|Q&*}&Cw z*22efKRT|jgSI9Dm6=JPa;pKCaH7zZ$aDU@yv}I&@L0{(yC8YX6sC55IBa?DM1THr zWgZq3(JUVg7<@ewN~NAdPGdXS>{v`MMGHXnTX8f?Hzq;JTand}qJO8D;mhAXWMV)A z7r)en|9HKJ(9kc?`FkA_UNJDxJ%l$B56~}L12L)l9NGGFgm^}7p=TD36U%=|VEBF) zNb_{;Y}-E479zvkOzMKA>+VyZ;xaflN0PWr=7V1MHE{c~8QNb>MT5^NuoIjebkiO(}hj;+{F+@rA=J~@~}b8jSs8D0gC6YXq& z4bKNtCeE>wC2+Fl3jMn0Jig3RqeQ~DDg1pcZB%R_^eQf^C*O$ ztM2xZJm3A4oLMx2jX%4f)#w@;2u|l}g+wu-)us?w(*`FLJ5eUE1%lN*VVR^O+G?jD zQ_Y7KeAn2Aag|WJ&jJfhU!l!*3DA?40N+_t%E^|*<^4MN-^*NFl5~^0+~dGH*$hI| z)YvqMT>8P<1Y*$!71yNW{HhLiRZ=dQjE{y?2@zK5)D=|D$c2i3`>?=$6G(`9;-@Ru z@cEW#6!0^`{NFVoUVjNV=W|i0$AKFCFbAvRAo8%_qD2;W5~!>J-MGOKmdRSeOg`9%pY5O0-DyWat!X+ubI+ri ziu+)mwioOrL2y`FnA5j4njE#93In`-&Wj_OoNWW8T!Yu zw?ja!_s}c}7IO8k!~V5zshHs)JX6lZ#nqaelh@{Qq&n8aqq$vp@$F(D?OZl(V*=54 zDkiJdrf{V;J;0X7Ry2H;3C_Z1R@ZPoapL#Kw^s$ZJM&#|`iW3Rqk_SwP6~KhM1#)h zI|+O4TqhP|)=>DAQol>%FdS!w@--@O$2AyCUdxhE7b!>%?qR2nEQguA9prhTPt?=; zFuYc~LGZ94(O78>BFf=7sy&@92v`UkheptEw=Igx%Aik)7HlK=RQvlg)O9Z5~q z4n6JhkMeV-;)$~9wDM&bR`oaH>|Rxn%)G#CZSH4XJ_a_;5tAXg5+!t*oFK8$Y$Ly< zJaE2KCTcvljgERm3$m@SXqGjEwP=C)>2wI1iSQxJ2$f2aK5W;9 z2iqozP?Q8^MmEtAn|Z{4Q#07FTmp)><{5HneZY4KBt z&mKv%*u_V#S{y?ErbD>$ZYUo9h4^cYJl^%4U_DwclJ5H_nR1;P;@H_mPArJRO}(=~ zRXK>*ou7gGYYfqh=b6=NC?TO$A;jW+F{B3;!-9b#@Zqn=fpl9elgY%aUrTBGNjE}o zYqJL~9R}IEjkN6aR5)@Dq{Yk_eq z&r%be9c0md7MF>CBw?=Kz%=I}?CzG}W-S9Kn!g*^&8xvnn4g3&(>NiQAZmR{El#n3g+iF52Wcxm~O9x&>&2)!!|&-GfVML!1}BoBhA`V>&R!=ZICk+5N7 zIk*;8Q~UJUaN2yFO#Ny~g@S!>GM)jK6+)=L^*Sh8@jQL{b|M(KXENen&cYsiM+Sc?;O?eKdOh?u8iwgHTjGx4`^!aGKb%L-R+Yi@ zq(bajx)-e1*)sfp-jEq)yg8z~!@|??F;0s}#*XZru)=yFggCur65U6!f|0;QLQ^>s zkpi&vN)GkSs(_k*0wi|MDEJunLGP;H^q^EO-Pt=Ic3*r6((MPyS$T?N`yHCGK9$^B zeE}mD&Vntg_&5j)QIe99lst&zl?`a>+=%dbtuIR{3J7Xkg;y|_km+Km{#gcZEHQ)M>jvSy$pi`8m;u%;S5fay1}f^+ zSafR+(wt{C@S^q{TGU>|$MV@U;#myVwr|5ykmK?vSz&;S9^CG@Pr5~4k=u_ENCrPF zNXtd#)e$HrIENGLCy2ua*`VXy$V@n2fTi|6Sh(&f74r?oy*Cx<{svcU8jD0}6;=2q zz}vfr_<>Db8TxUD>BWl6XwKCGd%-X~#K*~+M3>s@R;pv|ZIPv2- z$v>*WoT%Z=O`1A5JWT~dxReXsDmQv?y;*G}DD@q_CELAdc=6HXrDL!DTjmrOO9 zO4L%aZeA5OKJkJpOHwdYUXE;gosQ;(sd!w5pxjO;sEs_1`(qD6U_%fV%wCAy6f`%Q@by5g()^(D~ zn?rQ})cf!wZxNZlV34Md1>@SqO5DlWk@WRcE7X~~vFYJ5d)&3F0wqNW^4Is_*$aVQ$oh2kHl(P*X%CR$H}novI2e=Z0|=4`_c z${f6UY71rr_`_<8g}AV`8aW0&*mInrj%5#s=XO=>jY=hJRO5l#RN|1J1|5lBO${}x zan;O8TIIA1_|5ZJ_pO?6*U1~MyJ~V4!5#G4T?;xZt;np;$>e-6AD+z*#qI8!(2Y35 zOV3cKpI=1%#ztUEnJ{f`wZ~J+@*KuO2K&;#)5_#%Fnw{OvHr+4^cqcr!1fZl(fTu- zoD+!6ce0>(Ssg6WXa}?WKKi(N2Iah5hy58YxO-k8^xBG||Jy{EaXkar5@|YiZxHov zy@H$Xgt?buvf;kcE#y|-Cuw_%EzE@%K-n=p97zPWduAH^S9F_}U3rf=a;fy+%OUpC zn>^rG&?O%axD(UpdPXy+2!D_FkjJM7*`#hGy5~^_E>3MEjjz&4cNG_|XOzP4dvocF zO@lIIEp)`K<24Z5PuU4=AFlBkx>Eo-pwX9_cgIEwiHB5wt?;6 z4S1e&A6}5-5aaQXjL$D)L??Plo^S{p))!^^B~>s}ssg?D-Y0z@pEi}-4Ks&q%ZbBJ zVP>6;FUdcdLabkjaV`o5p^eXb*zeO1x6~4dh*~MQ-(JgU@Y2CV?MKYA*7s=Xaher6 zG0avpz68PIbLec=PopkxhuNDo=~??9BtI<>Rhoj(-o6G-&%Q@ov=@Mq-*%kSVnRoF z@3gw01R6aEApFZ?AZgcUYSu3e^>#vEeK8&}Yfp3OR!Tg*t zLWCLz7{;fq~OF@A9>>Zndq<2fUAD_WLNkR zWXG4IWk@)ANHf7k-Ol3Q8%kDk&yw2LJ0UTl7@s)Allr%IaMHPp_VFb>oXkgiuT~NRF5!NiLCu*w3AofX|JI$Pf5|93)XO-qaoB4G{ zlD`pux&Rp6pMp=E596E{9Zj19b-=siEj0|{)kXK8bjp`Z&=_3;H}D)N%qpQrb<}ac zz+td?AWLJ}5i)r*1o?CNNFXmscZ+uv`yqTPZV>;8A6i?W?ov1m^S-HK+`m-elp$z+ zOu&QZcH#T5X}BOXoXxwL2fc?4OuQDDmMS9OS!8J5ObUr;Q`3OFX7O%Vp`o)C$N};llCsVk}$qiZyeX zg{`a9aFMYlwtR|$td1z^@}Z1Wymo<>umxNRGZ!%0n8EDq;nL%Ea=g9M6~^G2F^U`( z1PSk3r0Cu-BOi2uY=4zSq|5ZMb-o*F*m5Z^K7e|32Qq=aSICE0F`Qg;fjZeQhtmrt zn|7OKVRFGgCiSQRH7mY}uQ$l@JS2f=b>sz{Nh~923mtK>1q1U99H@o*A=tO{D#?s| z#q-FmfXMCUxHfJ!?%{qW*VX#adqD|s7nG5Ydr~QXPwnm4{8^G)VEhW&mBYdOm^p^8SHZ#lbGWpa zK)?1|s#k79%2PI=;_i9SXZ;%u74qos@U2*2n}BzAXF+()6IQKzi0Ko4%)G0(PP0@s zz++bgc|GScPMfw8HuyU+s%O2>H0C-!H$P3x49_wxk6+W*_Xc3y&lcV}tt`A@m)ay8 zy@Qy%VsJy#Mu@cK?F9S=kXmfUcUen7*4YV;?Qfvz0)iym=pOBQ8N$wZ;f@bKJtc$h zZlKl0LVR{8kmm!tPuk-7aPlh`9*+;u=MT{Lm}by;Bf3 z3;40+ny1iP)D?BvOYlPKC9J(6hC*5gK}9VNr$7Rn|FM}lasLvN^+pp84eV-~;Z%+h zDQ4i)*F#*QmFeJ*BHmcP2p5fnqI7^gTApfYdc$)W)JMoeMzq{#VMQ) z!AsCFAp(z-o7n@|wy>lt78mGWhRD=dSXLqet<(R*Yr2(m*!>w%8I=TmeFJXo3~f{% zw_t;>sc|Og3GlzP27Z6K4BL5LKQOPtOGDlew?7A^%U{zIPZCf-A&@wg?O;4!?Z-HW ze6XvP1*}rK4eZJey0&ATmh4*8nF1`J37a_6W;Ki-P3XW zFzY(+nE@lL`F;}?7EY3ADW0~xpC7Kd?!&~%XY{wDDJWnR-P%}%g=6k8(c2BA_AJek zTLOnt)o|jE51W*(OKLrm;Q2@j`geYZi}401jXSdHRRkhxB>?s zMZ@@FE$&&fT-bk8frmCSB2)NX>1tL4G<9vs(TRTAF_D4Kjb)%Rw}=dnXOiY)*I;G- zMGLu?y?C*59 zxDnoqjKS+uYjNm~08IX{hIwTs94ohQcFgGosCku<(KEepj^_&sOLc;h&(q;^LM;{u z9inZX0X*Ma9!jO0V{iNyg15th=)yNb5dLiil&id@WmQ*TcU}|tYMv#h#XCv%twQF% zf8lt(KMe*_UNAEwL^(YX?=7Z#3Ul?nFQM>YHCC(K0S!kX^3pCBwE5?NMpFr4XD?>l zqjCcXDH z-Boas^1{c7Samww&2@yUx4P)YtTg&`Vm2sToI;oJf=u;d`z?)AfljpvDr$eSkd`Vh?dV*~4}n@Fa^f2^YOEKco`AYhDS zP`>;(o%1G{ePd+F+25zf9rjklSIT8@Wbz_CKcgA_`{kj<{0d6mxj`MQ(k360u*1Vl;&3eMeEB^?6vI(v$^l%@tL=YIB@v;u|HudH6cQ1q&J7y5 zh|Vg$MDNF2;*usoRc+UR+(`+}(DX1W=!SZeg zbjz5)Z;GCe28*&#g=UhJ17jiMUTYDHIT_qWnEeoNw zr5K%-T9J9@XL5bl{A3wxXVUP#(c*!FJ?6i31lQ_fs#nuS8qDiKXZ;K&=!zC7xi+Bb z`UAMgyc9iFro&a81gL)blFnx&(abcSEG)Tz2S*Yx%KH?Nx)ue8-;ct-dqPm$&HbUVbT)GaN{z39d5Mqj7e_Y_`b&&y(u~neS7`QDq}) zWu47VKbHd6%=R+}%pIs!+$ua#7YuVt)xlf(FK&p7!a|>N9Fm-emp{sJ512iq!c7WT zJy}6>lFPx{IuADv0jxMJ!dY!406lVt7~e&%P@^USFT3PHnj4BsUY}#O+|R)v&;M}G z!spc6_7A)2uqQ747lCu1xsiG&Nsvi9K^h-L;oAoSpmko595D1H%2q|>`=Vm<`dtL` zS-*_wRoqK<`n*N?UysOp`$}jRSx(JbXVdwYTyTofU6Spq*Cb&R3}$U}$?x7-cyrZS zsJ%5CC&ehDk_1?_@y=bg7UF>wZRBT48U1s7@x}-UC!pl z=B}nUhbq>kKpnSs{Q@R~9<24V^x?nLbz(d}i84z);!)LnG3L;@&)nTYOQvAm;wMw7qg35Tt{LN(kfk1MtNO->SpVYOvz?jMMfS|kJCFH(@dlirh;vGIANe8Ejxh^{@AA!s4l@&@-rSX% zV$N>gZN9;lV*Xl>B2L$UaARS~bi8g2l{8nuI`W-pxT?0RPDH#M(pmcW@dve3@lmDoAbO6#J0+2L7w`srI@W zn0fZ(E4TSmBHd!On0rgH+|!^Oe!ZS8_qpN&BeEoz`FYZgsT#27T5M0zwTafuwymzU zuf|-so7D<+jsM>A1$nl0L7FjKPSpV4C%%Y_j*R1P|8a_Y{ZG{1Tc9Dosw|}CI^`Kn zzXMEei!?v&b1Ng>#4hyCFS{0fg*r05itf4omCQ{j;B<2rF%co6-23la8N0R51b3S0 zTr@StwRFuwckW-OqpWK=p=WdHhW#lDR+MUvXy!uGKEPGr>P;+{}zS+uHrzs=*Drjq=x+bkM>pV+3c0GOZ`p z!L@f>uU#K|iZL8JO>en7$i3607^82_Ou?s0zSh6FjIeMsSK@ZE_JZ$iMrBnZcT4Ff z|3g+6lk=jPd9(H*|Ngf%jPdDQyAO|#@PiIKX71G9=XX-k0v}5nm-5Jj(a}D{?W{NA zr??IB9}W|`L4Fx0zIsMoefMU5ic%}rzeHHz)yQE=@&&rf>P(LFapAI?_b`77Hqvqx z9`+qKF7l_rkB5Mt<@`!V7eX) z^z!{i{K;E@n{xVL>b>V>FwY z~hnaiqa^s>iiYp10M zauV*`V^S7KGK){Aa4FA5_(cYPN%pjx%#jfXZmU@=H)TYhK`Lh%Rn-H6z23WaOHd;B zW$7Bb9oY`tXrYW<y0X{o5a>B(|NZRv9+GvqEOntX^BjT3kkMr7<4s^`$Q-wrWI zH;qr4TWH}XVQzC5^xO> zrzVz2-tOAPoL;z(IseR`@iS57UlZHT#CxaMX)IeoN19gFemb+3IlVrgOUOy)zGzq0 z{#`$tvwUdCm6oS7&VJjuH2>dp;$3rYV>injEidFZJXNwE`gfLgx3FM@o9{CV!^3G8 zbemQ&xx*-U=JOZ*Yo*1LA0d%6fxl+eCBDkpul%cvhG_F6cATNWmpZY`jPrXxk4g0_ zqL*vOapF&oGqZw^(LV-Dm{$zNclsjACI1+xy=&<$@IK9+Rm!vShZlzofkK%mI6zRxavW(5nXztIH6-@hP2PS`~Qr+?^jojSR>uNul z^f6RKCKrEjIsIUp4?U!H#E$y6j;7}#ZoZA0T`<|fOzDl{yP5}aS#sli3#Ieih9{{U z7im~$D_zZ~uRX?TSY3-(aFj;jcNkfpkyHWlC z;Tg#v&XDI^d}2A}*NL?slCII_LCcxc6V_bk3{iT6nh&!*KaakDrOYn;Mh@Mc{fujn zy1>^QROVg{PT_JDH_?lS7BDmBALI%Geo^TrRlQ`M)Y8+&{F)gZE zoZD#=I|=-ra}sF;<*jeI>M?olkxfOdL3JZ{^3!cQ?S;FUHL)@LloBVtWEtjbh30c& z4#HgOR7qy@z5C43+mdy~4&BTQdtq+#iD~wFjVz!4&ym@aBUab<`aNx~6 zECe|@S=>`Ef%o)Z675^>&B<(-;O@jEGIwT=2=-3oxlp&`^t|{poaWSKJI%*dcCDpX zm=X67#!sY&Gj53E6s$bC2D36w_fgx` zaLncud&X(|IeP_j`p1lthy*h%bBf6_oGI|r-9vO_E4|~Mp#So-;QR_Kn1Vg4nZ3)3 zn5oK(YVUZQVsyg4)^>}l*xOtFCXYO)acicYd&iB(xZ|@u>}IOJ;cUhQcgmp$ zjN)%?zV}?r@z*@2S9VTfR;9k>cTXhIHX}zFzYRfL@`cS@a>*Fx}>{y;}2Lp_$;!>-_Z?2a=tcI0uF=zsi1G=~XZ@{=A-f5`-JB6Y_j|Kl9x z1^2?bm0aWK1I9binMs#Qq_^3hV=OLA<=Q71bIS$#tWlUJU7KXjWk#hjZA<>+XULwS zixzI<^2-j{`AB}G*Cr3uQ%sD$EE(Q;2w^IF~9fkq|>?z7}q7!m{mKB>F&v2n9n!X)ZQDM%*CBZ zw9Cs*N7B zmg+YCp`r_P$=i!|vKzYio8ONh1-qQuZqY(|Nv=p;sI3|2F~6O;S1ZN0zZt}Qm>IwY zICt_@zST0r>0`{qzg0}+1`S3>$CmDvbKrDds&E5Eql~t0BOSH~^xlSWer{$4f77nXbtx}Qx%-BR+^?aR+{Y%p+GTMPeDm@{OlYS+of`R*A4vX@ z#bv+gf<$SW(x1;b#3=D+%`4<5jN37hPs(Xqvn+1ynn`tjLuT}aBE)?44`(_D8@c6W zZT#3(0>2@b#Vvj%$u%0M&?T$8Y0nwh?pWDx?y&q6=JF8BZ5%$$^=&-Cw>9kGN6NkA z_PG&e^tE6=%)O5f3d?B2u0h&YYnYRKbjU93ojd2z9LQJNk;mUPtB27X%4Uvq2yz5Y zmecvulDIurycqYEnaui|fBDv8Y24Swb<7FyqTjw)%T#xkFeKmv{b}?Gqh-5_tNMGI zYj`rbcKkvK7dEEFpA>MIJ0ta>R=9;ve~6gRu(}7iX*nmjFV~v*p2r1wn=N5nU6Yvo zA>~`lx6dui&8egGz^xu`Frl{g(ZUqwy2Z>|<05gUt+IfN6_&62e58Q+UK-C`RumP~ zq)6_s?tM<-yTuh5Cw}w4S zFPyuY*}ifwKei*5Yj|;iD^>l+fqfJ+Wu6LCW1Y+R@w)h{GIQt&(KNg0lH1JUUybz3 zQA~hq2qQH^o-_FS zg`0P#%hAh=nHPe*t99SH*~bVnr;>uYnJE9O z{AcT5(dVy3F@IM&Fgu>Rb2}^l@-J6ua#_NkIsR!w#@BHzO_!=Ln`2gT>jU(eP5W58 z4G&_N_w~1G^;;g$CY`PPj}=Gg4TASM9$L(Qtw!@jeRCQ5+a=oiR0{L&e5@T_)JC4I z+sn*(o=2yq5A(0g+|TTYS7ydLM>+pTpBZ&Z-@ZZeCEZ*h@C0~w^W}Pe@dKO%J2U%} znUgC1wPrUm?1bMW@J(M8GHrF~{DRy@+ImRf*&aZ4_j=!O-`Pw2+HOlOBI-D2Jv)jU zTYZ?Bp%_M=CdasGr;_-8HVrTq*{AqxU0!fv+Yiz)HK%D3xsanLzhibwXfrtx`~iRFYNSo=ij}>53-&fWxn7(vVfKm6M^m_|&u1}zukB<$d{eex zy|I9KeWRW`_vRA)-1#uq`S}HH(cVOdPMJe{GeLG)I^K-7R}!aGuFH+jxX%xlkhSNE zyZJhaf{X^!KCZx2h)XbY}So`c-nPpP32s$ zeHRKN8$)qyyb@CHO{ca?2!l+8Icp=808dAyaPb=*sL1#M=B zt{F)e7ejIZN%(AR7wC58ux>w7u|xD;+p6LeoHcfr>aG~V_FhA5qtHX!isv)IQ9_s( z?tc=B)(i4Z1DBw6rPX-z%_R`}Oc#B+uRvbi8GxYU_wj^%BV~TquEyc^I@ohAj!ns* zhhmau5T~4e7|szPR|Ra&(OxNXrF%!1nrIn;w%o*8l0U}j-kd=rZrM9_! zVoR*kKsI0w*;z3YZFx9^LoZ08KbEOv|B*$=M1Cn*Ylcw&(RA2T0wk)plbY>bfa6z6 zlc{y`NSbQIje0%s!sQUFAjl}aaIlrS5MscF7bKB|4wn8r{hwc(_p;Eq+x=)oWRT$bcz2=x9dVrL@rf!4CYiQjfrZGum7A2^(hGAq9mg z;5oeMfQrbtvio`SKbtHF}=gWu9aRMDlWBxZ3QY%HwD zZ_GqdoJcjkzT`Bli;Ty*i+6+T!&`7scLy8r{349*(ZrACmhj%6;^3n;AOE?v5l0=~ zgiN0wBo^5+Xn%)4c@$F%0}t#4yK1MX_m0igg=0a;NMb3GRZ~K4N3F=sbNOI;=mQSa zkA_wE7UDsk4*t1I5VUrC8C+G*gJnir$>01iWG^0o@5Zl0bqW{RDOwb7Ndk@C%WmNf z^YwVGipg+7BLEK_+6zbP6JXT=S+X=b7If9- zCx1(YQNZmvgiUw{w^kj-THm&yEw<6bt<(c`A6!9nZwQmeWzw*6UnA?WV-n!c3^ADI zfRd*e5mP67RB}v*EEJbO{goH-U%C4rxgr;e7U&W$9Ru_$C>=klkw>NZ75KCez@nxv zEZAw2)fdiTzpKGG_?otWAv}vL)Dl5vRyXnPb<>eaumvg#lqc6d|Ag%VR)cxW7x0YJ zgw_dRI5aUr9hFx{>sSB6|7E4Y2E}6h*x8Nj?~FqCLd@}{;sek&Uz(_I{0X+V<*58@ zOE6nyL^Wlr;y7LrR?)1*|LMz<`4_~|!UHzA=d2nMY^0(s2AF-_o=#bx_=>HQo>6~H zUtslkS!j#r@Pm7MVCfWbOpKqvMamf`XFsT((x8G|mH)$IQ7N|BajHBUe2R5hWP)Al zPvT83rP#=@g{qFYO0BGu15;%M$bNSW2dzj2b-!qkTD6xH{&)uk*>j2YdLIlbxEL56g#biExfDHL_qWj@qP;3!j_@HK)aR!Kf{6@6>~gE0-}M z`(few1Mpyh7_p@ap!Y1n!7C8@F&s}Oy|6_aV;o7O=oN6&zJ~u~ti#&_J8|UbOmbOV z2lUfckfiC?S(B-`II}VvW{RJ|FMphdw}OnmqpqtIC{#^Mabaz-z3lz_pvJX#K%l)lz1Gge1` z7XHRgov$Hsogz8D@&`_qw?scbWZ_f4Kf%H3uQk54&Ft>3n_yxhgtQd+ZN18JeUf7F4UMX z1i$ccWiL&ZA#!^FeOoF@J~aFWg_E`T;e$%N?i7tIv&D&ljxOqV4#n5eBw|k?s*B*A4ztM-gHi}Q)`V~GVZpslkF-b7 z=+gcU?3U*P19k$o!5veg)9pZsTzAD5cNU>9UO{AgW(S0<^dzqnU5V+TeW=d*2^QZt z7tQ-9jA%(yQeQI!VH+sY!v=u=;w`v5P6N%!xkhoPE5Ngv6|gY+KzMlq*2J+CIVTP6 zmy)T++l<*I>!QhJekh`JCX=lHz5}~@D%yG|8~1;?M45FPu)=e$VwF@y_`3Zl9C6VG z_S!J@ht`745ho#R{Uzw>OGQE#7L(S$H=sLc2cb@G$MI)3QJU+W@jUx1n5Oau^sMHP zFFG@@f1E85^Oi<4@_mUWrwRpK$yiaqK*L|>AaPwyB5!gNl$HavK5H@=KbymfXgRaY zcs!5sv%_naSF=uXiCC}k2lz*w#&?dm?qZN(Tz^@IBFM zKOT|4p935ATjP+umspvv%P3a?f4OA0B|7VEOvZ=agX9aEOe8pyzI9t5RzjRisNDrQ z=^&z7VuvooCqwMbE$HUlOCW~4i1@lzcvKODtt%o>+;AY7|BOcGn%pavklSJ{#iuU@(yTjUPW|IJF*(>$H3A;gIsQGqWXi*!LKLE zqlNZ$jwhOITXc7JXZ%ivMOPf~+_P z_sS1LDUqY59x!L)ZFPZ}pFr{y=b^+;8svbDC%XB;8zCPVqGcn2-VVy!p2i}C%%!T6}oB`9k?#m32Z!m}gExZq18 zC~P&MRCeqKqE(MG%a>BMG1ul^Zi|DSyT@_=q*3a>g$}S^c?xowIgQ=;(-)+rrLpe! zO)MH=$dgZ!sPAKRz+%|w@_uR!+1 zLp;+w4;-_;;eyr~;2>Q>9T@km*@$SQAiCZz_aee8q3{WD2;uQe&XABdXPm0|gOQ}o}FYf!N2 z6Mh)90esgj#g~6>7CciFTdr5e7MA_RMVF@{`|XFICMgz{9?8M~nmo`Ut_VNbnM9fM zj^V05Vc>K3ANx|V89rN_g(IfQX#GnG(&zP-YW3i7kMk@PAO8mLs5wDZ2XT1Te+j50 zCyscgXrNX1wv)J?&1j%#6X}$fL&2J^#Amt$+UAgmJ-@5K@?*zwnDJSxcIzml&QvK)OVrvzVKjC2QsO>K6$SK^N)``tIB47gTNyGQMx8bbKrsPH+pBh}V9bMSBjBNR+gQ&h}l3?0_vp&6n@g?Cn zQkI8mu^7@?F%>qd0yX>Va`<>hnLIcxgH|P&;eV-Z&@x?xq}M(NnL}noE&ea1`|}r7 z9HT@IIM(3dj_#}=zTLw*EP01ONDtNX% zL%@Om!d@+%PaSD>Bat_9SnZXW5Oi-G<_A5a%Isd&ME*{u<_DWlRh7l~*%4WwzrKRa zlGDjg!MSK_z86mwsl@YR<^#X>9`r*IELOP$&qUO*wEqQ`D#)t|>6+l3f3HjSP;GE? zN+A9;uos2NxZ(IY#%Nt;Io|u~4^FcgscGJO8`fsDgXZ$(AR4m{uRnMI_T4zmhC0h* zS;++0EOgEGhS?`5Q#8hn2TNegz!_#g&EaL6e1%8YoD@rSg0Qm`Xic}ljrFoXnlDgG z{>tEL4N++I&W7Q>JWTb>f;iD2G+WPy6vtwSi+ztD$4HQPdI;Vn*0WM52>z;H!6By6 zkTWwCPYS;U{0qOa@T*N|&zeezXlBq>>&3*;N(b?36tSt?DrEX85}njJM4WyYqg>@s z(#{)W52T!69R-Z_o}i0(+4>d4=FeJGbJL16I4?)qBYLP;b_kzeX@!>m^}@n?58!}n z%4p%}3JCfph0cvnLA8}i#Gc4NF+PH~NAE*n7uCsYr#^5NuxStFr{QuZ9k|imfUjgc zrV=w}k$GxvXkA__>l*$9zUgO!OKcabyY(l;c5TH|HT_8ayj4gv*pYY)StG9;IW#3A zk9zs+FC6aB!dt3ZKz^YSkU9tfna1N$9X ziR0HPlkxlS0WNEi{K|Yd(He^GZdpgNr7cl|q8CZEP)6dbF5*|uHIeB}C#uIh875>4 zaLuuu0*77_zV&Y9Lp0Sj+q ziZr4=#G;J+kJwR661A9);wC|ztC%yFycy=9(cd08x!MiAQnMk3DXs{EFzGCpCz4r7 zpmCdE{U@4iq?It5n5Rtw-z!tk53IvZQ^nA`=rXpyC4*|SPR8$-#esL<%^L^)Mb~`e zK2d4APvKYFeuJo;3v2vD6ZIzepxUoj@B_Cg$nEhz+;UbJ=?Ayt@@G@fq2;kSq3r}@ zspe3IW4_SxTZ4ozd5a6m&w=)zcD&`B1+;DH#F9UU;pq(*sCcmgZl$jzzr}6QoTYlC zLTCz7&6`1X|A+;NvL&qV%2L>DUkKG*?;xqd3tUPvpxIFw-3zlK|E>w)@IyKz9UUOs z#&)6?M|kA(a0JSf)SuhMrg>s6^}=RqSb}Rv3p@D&b12!N23hvuqzO5e{?{L zcG;1SpLC$MQ;V$5kVV@68ZqS*LB+mQC4I&!By~0eve$m#%aTgS^7SbkIGu-FM3&(j zS=p57?FlM=jt1EY#W1J648L5z6z$9QCRb0aMxn#^ar~JVxLW)&OgLb?kN+M{rl+9p z6~{GY>>rt#zD~@j&!~q}N(DPtl@-mnY*olYjmMWsI>mTsnfgoOA;#;t) zd(D1!KTn+wzs(w+s%AxR$H3d^9B9l5_&2Q3UpFUP^_}Aa#&B-U_WPOC$@y|v>hJ*U(Q<)OSurFZ{sjLC>0-4up2z*N$|#~kh9t^M zvT->Nv8Rp;dGrR*G_4YxxB3i3b2cRLUK-kQ@)SEWG9111k|GC$bkK&s_QYJ*7(mcHwbFQC;vKsy?<+n!J(1Tko&&8v%dy4688G8T91xa6zJE9D^ay8b-ip9_nGq^s`z&Pp{0;<$*^*_7hDfSFifBbuP%fg% zXwQ#WY;Jl5=6ic#L(Lfc%9Mx7rMuX?VgnTL*BIU^y<}hZ`>_qrGbkm6U~)lE9Z_Mc zNI=h>D=G+qXZk z$iIzRw)GvDXs6(LF~_NxbI-8f?6rbB0F4{w2jOq3SJ^Y^B_I`bhRUoeh3R3ZtwA8uvZoGU z{|5n3x%V1wY12kcd966jz8e; zw~qvtjDLhX1Z?icDq}R1_5fF@sG-6678t#-L8nce$*+0|^gwM9d6zqt3>z&##i`vm z&uAL@EFX`uU+o~ud%O@{??IMpR6*C|lPpp8rRKd#!!Jn*uK2zeFAEaGUze)_%(o?9 zAKd|Nz7PpV$#C$UKWfP8z$awXQIkO~R`os2CJ#+#YrZH^js2yz#!ZeyuXz_LKL8}n z=rkmM&ZBlVuV+IX-hfyJOBJc}kcn9|QOrO{{j)IvZ$I=bB#_)Qo{P>MwI|mNKfuaa zudpNOXGcp9;FHTY;vB0+e4`|oSE9C-J-Bu|$O=)Ak=Vj^E$gH*b#Ai;x35CUo*4LT zAO@dZR+GXzk;v~q5AyGfJ-WAfE7s`Sgl~EGQj%9GHXU+C6#4r5B{#LvKS8pu9 z)b)KeFX**MC*O*^-eQQX|9ghL%1+|=e~YO2UH-U!=OBdTxnVdW0#&y!;|1a4&=+gWc;HkGA=M8-_{r+mg&Y_mYQhJ=7$(pN5FN}eEhKO7KE8Z z!eLfir!-C{@Q&O_o z4-|+3S)4o{Ieu_q(+efhKPHUSx159do!X>x)(j-VaqNLP7D(o`F{O4W6us>XCiQna zK{a!#K>zawi?&vHl`DnfQ<9-%WHkkmeb}j25yGPTSR(lp$tz4PFqm1s?b+kbL_oEK&ViUD}&< zRD-42z5S~NJkBB5GnR|X?*0OAm8sA*B#YmRTeExb=-?McPau6)4g2ENVX%>(Og-)H zt(lRc3Okv(?9&a}_>6EkGSFB>UKJ>y6&HQUP^B>&9aMua@6W^s3dG4_tJ$oss{{Ou zno3e5b8%dp;6-f;#>IVI?EZb)*nPGS@*i*~S9d6&r?J+gvbmidk28VbONXG4I!TT9 zv{RxJ2~be1feN&((fFzDNUP6-I1Za4cDgdT+F1?5Pk!Mamp`!Miix;3RgReLz5>;k z+Oc1+8|Blzh8+E}3khxVCE`kRk@HLkNUaQ{4qrG;-LI*qeqFzfRsTJrUY=>h#-nkN zr!t3Hxl)hv+U!q#v6O}(Nj+G3>NUPxp$EZ995%L6t~^R zom;{vy|_2nXVNh$nrBVCeru!j*bD3&#}sUwHUY5;)$G+rX>g}YoXm0l4`$jwre2m-Dvteq;0JJ0N={ zbu!$w44q4?t0^_MAxRH}(VeB2aKk(;bXPDRNd4drYZNUgk&}+FCQlhLavyQ%A`ZBBItHLT9JlqdD zJu<}a@hM(Hdl;oXbvK){<|39^tPgLEr$H>=3cf51#ObPQ@YG9Dl-_OykX+@6_iUp< zL2&PFe_xJE2dCgClk;HR=y&*h%^Fqb86XeyHF#`eC!RE-g--Q9XMg{s(96TUY%}i! zumMTz(!xT@ZTutbG@YBg+DA;T@V}P~7T?pw^K5NVh|W21Q7|YZ31}`UI@jAKQjY#~?>R2)#@>&aU+|K(nv+uuIER z@I6>XRhmc=PlM~!f=_4gS%nJOas2y_15qC_MTg8~$o!wmNX_mC9?$p=f9hiKlGidIyag+vB-LP4mM_mP`FDTe2$n-#AT1b=4*fO)Z&ZyBG(4qB3-Oyh!ZrLjk3l!`q;Ve znyGBdYm~*jJ8*4-3ydzBz`Il`v9q=*x-4Zz6!m3M-J{c3`GO<~e!^0>glq89){E47 ze_Pa&dIfJ^l>wXdy$O*l!tF-x6mM%3%uP58Z7J7r-dH>99=!lBJNg3D@D<#@aT>e6 zd_9gxDPkRO8bVHYEq=YS6Uup(HLDK?k}11G(XR9vq+Bp_eX)KCTCQJ$<24xY{40%% z^7r9&V#lEWoFQvbbrI5MjpA=vG0^32KIbapWij=)sj2;>8X765Hj>?}{lbQ0KsQR8VyC(HB7$lgWeR5Ms#nS>vJ*7`9 zvb2!%TU)ZkcLY|aXW-e}O;JVGYaHnBfPY=Mg>6dq!1B1a_}jeJ=TA}cx@ z-OX&o@)M5e&pG1S!kppn&kQ^$_!iEcISjFmuK2``i#7BlA?irMT~RSZ1gcR9*0e-gNHWDV9=|V_3u4M zsdi{1{+%HFI{p{6fC#Y-`z*;hx92c8BMD2kIzq@UC+J8yhnHUa#;#X=hV`7{QTv;9 zWLKzw$!M)iT3;Arn~|4P)0aM2{>BxKrPz_~rJ?BK9>LJ1WgIpfmttq{kz=nf8Lx)p z4*1Y)kSX>oJTvoP*g1$HC!O zDQjO>hKE&eLWtKoB4=>{AM|+wI@8ayN(VBi1iKS>lfyo=_?Q<-%M?Q*JC+mqhq)A| zeGad38Lf#9EyL$MuG_LhqPVR56T37~n-~VSv&Vmy^Dg-Bh7=1E5LYdN_X<{|M#dFQ z|1pVB##f>H@+hV$FPvIvMuyg@K;`pkWP(2gx2~sR^RYxUc6crM&%qtpPdJecea>X{ zehbttyMbKFZiX%TX2e`?J+k}u9ZPSIf|Q#VKrGr6%lwjHjgl|o&V-Y^WaW6O<3k37 zhB)H~!ESKWSpvlehNys@*bPo?Z#ypy68olpueQ8 zu&ubiZB2_V^`ZC$zB5k_pEvTwXD&^vncAyO{DL*A%LvriesbFx?E#FZV%489(tk zgJc}=erJuZ$_2b;lPbC=lzGKlm zMs3LH_~W2(r5ATD48T=-E$|{T5yw7ghc};A;_zcC*c`6oX6_dC;g=E#oNtcgn&;w| zi-LHEc^9DO+$8i`cnX>{y%~p_R_z#ILt*CpY)Eye@z=f=awds_VGj! zhuz3LC4%2ld5|tV0ovEoS--Y}*yUyt%$Ru1O8x1DA6KU!r>a+2KH@KsnObCSg(g}y zV9Rb@sfv0>Pf&HRiT&|y5;Etf9?uj{UK6#ZbKrDxOxKmGt;T>PBy$vzY4%1>M7M3Ere2x z-6255mbh-1g=j@J@>}LD%)7On==TYutA?h;qsba2-JV1;I%lKn%9pVC4H`lOH7Z7+ z!LH36$1|jA@S$@bdGEVI*nvn6l+%1Y9vhl)Q`e&8;8G1AY@Q$Myk{GkeNXT zo;$%qdwGr|<=S@myg-7Ot+d1F8w-~gJCc_H;*`ru3&HN?2AJ>vAO4F((UdJ;LA_%o z>nCRdqM|*#oSU<`0@B14|W%rW?AR)7zGmw?aVC)lp9%4qCQ z6>bc_0PYc@yjQh8HdBo&bp+ljfhSE>KJ9&9nJe=8V>`rOQ?q?WjFq*w6mL@&nF9h0xG|+NV3enL$-N1 zHtK&2HILM)yHyKdPT(5SWOV?EJ#r^r^5Q6GcX~~YW+Ik2q{05?jj+0h9mt6pz9?2= z7AaHELJ~Jrk>|Xnc)O=G8n$r8TW#i%hark+udkJ0d1DN=Jza^cZm5&P*Jw0mq)c|4 zD1&8sQ%Tsy8A$2z12&vW#gf8%VP-@tF8ZEJy?L+|{uV@FYu9EtruPK9HE5CaaZ}`f zSBoq>podQR0S+0-!uj@JYVLj5g-g*UG|_EGMnzho$zup>z0xJm1PuFZ#cI6j#CF*1 zC#V_5MWEn@1iI#Fs^CKcdfyXH9$XGW0q5OG!-;KZise0K5#nqvj_zwTvyrz)c)A8nc&x$~b&p7shXa2gdYUvTtDAwoXbs{;`#;02 zb9i`^oE_m z4|~JNbqOQ1UhO$n3!Z`md?m2Gt^+P2O|0^@$!zdCFB|izPN183K>%aCjhF5!#4V>T zLR)k%E9qMWwd?tK^!g_FsuGW*`nEt%*&zhOkwmnDLN30`h&$!LhT4deTM5?q?5}jJ z=QRNNyV5bGBZs7>9KqI)ywJUrE##lT$D{9oh;gN94ewV3{_^TY&2IM`{NBKh(9wx7 z{QmKc*_Hx-s0xLKJf{%GBN|@bQ6uxRFJku@8hAe673*1ZtjdQbDz&K)PrIuIP2Jhx zh^CSoadS{{$y|YU`336SG!Xi|0)KoQ%8uwYW5;H7cJK0Jm}z1{C644%s*exje|#JE z+K3*t&?OcUMi3Dy6Go5P3pmEAmIn z6!kY~qFM9Sl1g23WGVO$O;p$#af{ZP+?PF2H8%-R7)H(B!`o{+i7dOEm(-+#@#@V8nZWU~-SxAKU$f)iRQ zpNW?*>%uK}L?BB+m`oC@2leo5xV*oOJ=na1XZO;Sa^56MCYH@W`P*OM-yVmdcz*

    Gv8^pJzK&SR3bSmo#_O{c-B2HdZ_iqi*vY7_`!x5m@ zPzxy~#<1huTYUH3RVt%>8|4d`=#tA0(rA~2yi~W6(U+^y4A&qck>i1eYJTB!VV+3c zWjg7Jq)^Df4g7cKO&q7Fihe$yiN86!foDwv{5sSR+X~OZ-7R0J>0$mvY1|pDGt?*E zmnF~#gI@48kbr|I3a9XYRz= z&mO`+NG`eCmW}dtR}=n;tw=4+0=d0FWaj1lFv-!Gbe^F}Nxdy%oqF&TonO?SEwf1S zBV%|hmdQFw4X~SLE~VU0$Pt(P3EPQf1#D^cO0xM_EON*c%u`PtM8r!Nt+Wy$d*pne z_q->2=D%91Lnx)%w%{|hc6KB+*Uc9PgEZXR-i{?i6|vW3ify>lgy-Lz1AdoGSRExV zWYwZhj8e@}(Gf8+baMyk?g~Mr^+kBwm-*=ThI$@W z4j-*f;L=@Ef*jsBykeyzPwz)K-rnOw?!Wg$oe#H>$%6~gHs`%mTh0|+i~Rx%AVZ)3f5njBq5x@yZgxp4kaA{=PZBfgY#6> z;?j?w&p84eDiLU(%qk+3vJ}Z^%_j50uH!d<<PX?VtYF{Dn^$i-9&Rn+gnmin(C zqG2+LvFw7u4nZ%RBk=y{6YRRFl|7KQ77{THZ*5$wEkx7c*22wDs(u1~Y0)ImU^Cj@ zr$}P1%As9WS8=oFZg~Im5`K^>iq1q%LHC|Y6H)s%tikicQzu-gVslk;ZQ?M_GEZfT zqBj!P!dYn3fll0*DY#1~29k-?El52p1LqAcLY@Cpbms9;er*`HMA=H&vXeDS8)43w z=O`^G*?uaCin3LTHnhl2*-0e3k}OHt=A3!XD729zNlGg1rF~C%=lyT~o6mgaJkNcf z`?~JyyXYu9mJ&-|oG^u4TTPN`6O8{JmIkr!BS3qmA6B?N4aZrR<4e`wI3WitIMo~U z;HTd&v3bEAEKFJ@AvUs5y+nz8tulr+@AQaU?;E_iQWDCoE9bO|o&c2*ok04W1a{qg z3&b3mZ1b$jAAr4boE_{HUiQP^U~0GzP#x*ueP|B^Kcp{!lLvADdA%JkRA>aJ=Qp%! zPcg@4|2_b*au2K%?Tm{DEn)YFA{>xUkHU;PKz-vXA_eaMjQ&|Dn>>lT2Oxeh3CXsko%O>JgG7IcgzbLer z)Pcz1t>l2tdN`$Ak0jhMhG*-!q-c0O{5mn6DBPS4mtDvQuYZhyk0LX9@qT7t=Ql(| z&8%R#K$%Q4Re&$T%*f-dZ@}UdJK!Jh181KQdZllj03nI{!K3)&AarE{zM9|&tYfu^ z`z?KZ3?{+j4fZ6$PYcR!4NRiwFtc(8uORp3V zb?Z1r>xaQ57gf^vvYB_@@Gejr`H5S%Cg$mW=Gx(?@^ql#Q<0BDrxPuqNGsqIN`B%Ojo~hqMe4eYq<Gt^{WY(&MD)qD4zjpMT2?HDRTjD&P(3JtV7_^saG74<)^XpNGwOG zm*KP|OOhG!l_1qg3;30^1OJED!S047IIvqB`d6ml-sWf?4KKp6h2=P^MUyo0qVbuD ztKdJ?X(avWE1_rEjU;c}1yi>LV9i21IA?nhG&I;og4>6{dl6e=itO>hXIvP#n2+ZO zb#}Ks)1W}uBgQY-4_1BY!5cQ*1wCp{_eLKsx5kk%Vguh4Xj1B z-pB`w-=Dx7-Z>y{9E6h=8R1>3<2FMX{ydc%AF*UzQp@`TI*=o| zOeJliTHb@F_bl+c{9GH?Yxi+-$tu`BIG;!kJ+zUg3&_0t>&RQ}Lr_#-iX=Ktf@ek* zh`wkh;ACiGh3@0vcVi|e@L>Yyne=_&vZ4WxtrUgjqX7RdI0=rQJB8WWhrsV_9vFSz z4B7^E;*v21V!BTOI)4@?A0y6kUM<&$rprah!|HUPJ#I=Of2IIVQ6})}S!1&Sd4aT- zr|~v34S2_6#lmwr2Y^_$ICKb>A}f2+L2`*XUNL(L6rC4>O|MkgL|%J|_ufAVS`zE= zfAH<-tH`{ebNN)-zkcB2=6Nm zuQtN{c^`R%*ubt5AF@($IrN*lm;j3q=yxKM=XLM~ub!iZC9D7N)_JsJy?E{Bq~KLR z)K8XUHj59IUSEr6M4ZFTV(;;#h2KD4d=36|K?0sHz?401oqi~;n|Ibz-#|Dkr3<092&cbNSgQ(UHPq0eW3}FRI`Of ztG?p2U3}2_-z=U*&M`17GM_i=^Jbx6AsA>B*8!=!-Xu}iN$4f&#Wo60K@(JiotA}o zU~)QVSKxVE!3w=^I)8xLrDHsWz`NilzZGxcOu>&%pX7a+7yx6IF5sHKIEgRPf^S@Z zV#j?ZL^}8~_<3zRCYsCP1r-ah@R1A&<&N7}ywbpji>H(D9C>(&KhCqy*W&HbQG)hI zhVUIzWw>MJcf8@9D-037FTw_e{-%64lJ8&#Bzjcfsr|}$y>TpfQqG0mT06-1In$u6 zjseN3k%f+yDn$L(AK)xiz&TZL4nI(d09G4MVu`M9tUpPp>z^@zNBjOG4}Yu#PhTIy zC57T-@!#hdpjkNjE0^d0;#I5izu6#Z|3$oi?H8O;9}JYu&tubLDL_Pd9^|Bjkp{C{ zz%FJDS%38b2<$(BbqntDG`+2`%aqAv`;#O*cJCox?JU6=IX9DN2>ny{|EiI*Su+7T zX9qikdT~!GAGoC*#AA#6;I?DtWTS)$TyK93rvw>uwlztDqRIj5Me-@Y@P-xPct}9O z9Ytba`vC-+=tH$tH}RAEPH@)Smo0O`*1$v3hUE01E>yZELoAYiww6dJ;f)=koRcEr zWTJ5dFIp+ofzDgtn7MU8r__K{?A#0c8@+5$tOoq|)*BXUOOS-$rm#V3Iwo^dyYBA@^+9yC!-~=ALUI+d=8x8{4bGY8m z3Wxl&fN`1xC)No-zKjMbSn&(&h)TnC|Ej^?(_8`h->r#UtON@esyhy&Bq`SzQDop<<`Wm)(!6WT}eVC-hjE+ zXOq#APr@@I4GYHuGFcM!+k%2{f+(?@L5b)SPo2SKg267_1pE8-LA@#cTdO z<4x16X#Hn9Y}0Tt7dvoL$oBOZzXxUWiRYL6hvCyZiDvkV&E>GKe@#D4BF2u zBhi0O1GS$)c=;pYj_F_?Z_UL}+^w!i#>ReQMfGg_Db^pVXK~4S8zHuA z2Ucwn_*t_YthM-H6VZ|l-UeUb+)NF?BEeR|xztxq=9?EF#&bTV5rtsry9zwo3`vS1 z1m7>*!6rO6m;sfDhRzJwdqIbU&Y8h8TIL1bRX*lf|NH`0n@7X(#~VpNP&UYS4k2O} z7xFH|E(MWqYB(`MzuSQcH)vR5L~^_(pzmSf$h`6s9^I$||7vaIl}XCL>n?Mlv6UW~ z)({8w99u37$Cad8C63fA9;k{E-BMfKUHy{>H|3M*ej8nSP0}hY(fL`@h zp2X$rpn7*37T~8XL3b~M?ehVs(ksWg!f>u7@(wV3%`pC~rwh+*up~tf*Fdl1DrCoz zU^uJmGDnbBf-UCWDl(s^4KcEmht$dOff}0#V@I15uSoFf>=3@Pnu~`fXX1?oUxBM>ACU8U-#YW(7}m|vCToqG zK)R$US@CWhoE1t<4#Isf=d079p4&Uzf4vQeH5)--CyV!`3poFLI`F?#4{}*u1HOB| ziO7DJg(~C0_?AjPZD8`%SvbTN!nL{9Q0b5y$<#H5(!26_ zCKscN1NfTI59DXx#d{P} z%eygS3LcFe0e6<|!SFR zK&WXao$9LG`t{<{4ZhV6~OS@)!k zmAn{F^oBmEs(1&w6DYp$C?9BYQhEErGl1C#tll*I(2E%R{kW-?)oHIu@0Y_AWX~0&n;`b;vULyw%FNpxrL+K!YZ4eIL zIR`#pyNUdXKoGZo!oBB3z_yd8dB!`!I2W`#aDZVFzG*AkI@Q;ip*(ltX z61B?4M{m&9Z=5omvE>8r=Y3y5-&cW_dFSw~0|j_*qzT#R>rS4m zc7_xdVMOdXSzE$aLR#q&S4DveV2`Ymu@2J4>v%O zLLV|>`3dZCdk#(nPidVS&%i9r1|Qp(1kS(siSN~#@O-oVz`m+uV5xc{ z_{&Ws>%}+21gJ~GZ%l$6aVBKsxe+XwDuOMueJN*T_)JOwytv;ez7UtqJRm{X(VF4UzOY@UVP1v5A{3gWT z&l8S;iU!{OMg+b}%ENQLG&yJQT*V^k!mNgk6PyYaSs41u8ZHQfWbLwfu(0J5?k;(S z^Q>%Pca-2&C zb3w)6bL`-?2n4#Sk^a_n-o*3i!jWh$I1)3AuXoGBKn+vg_%eI^ecTJ2T&xUIOx2g6Y;L3T5R2F1Jc^oWBZv)!QS&na7(8pXWONA960g`q$k#5Uv&qNx17xEX3!M82gBgsJoc;{ZjH5u5|U@|SFQWth5kjXIBrGw zM>K3kaurGA@)KZd#u2O>wFi6{AIHo8E5sd5W?&w_k@Gh#0kAUyS~7+6qMHFrcp2yN zcqc_wK(@UUynkalNp+WnW+&{hsICKyk<}-SuCZX=ayjx`Qxnb#pG{86wYK<-Fht;y zgu|!G!amzAAZ=-S9KY>s&b8Yj77 z)-@MIj2y-y(aQ<90K)U@oQbbVE(m>C2QClY$FuF`z?kwmxWDK%7+jyiYo1;R7EY_i zZ!_QG^vLx%Wn2xc3{v8Rm-b@=H%0j1)D;|XR~w35Z3eINmlN$h2p(M8Wiwnb8&tdf z!DrPcwQ@f0#!p^-0v2sf*gSLsM1aY>=f86}SFV-dci$a2hqxWs_NFP}%FKrspGpyH z)nc&lz&279zYxAUXhAMm`hdciWx!@xDUfuU%1Qfvgg5QKlVJ8;Q?mKdKHj=Z^2A#{ zh*W8=gL%%g$@=q;KwC&Hu$4c}as7K91RVd-qPs~N&o7(`az~PZ(cBY&InD>B;3X&Q z{UEPQy^i;Fk2=)JolQy<;=z%uyV$SpDA~EwO<1=Ah>>u2N6(4n+0S+c3tqbcHaF77 z{O}o$-QQWb>O?I5DzXmiiu>VXD;j}kjyvAXOaVI-wnEF){Y2bT9KNvFN={EcEXW0_J+pboO@zIa#}h33iUIqhp5xu&@jzTU zvvq4tAZ))qjogS{3px8A<8#_CK#*M^zE`1yOBQC~Kkt%-e!_SdIZcb09QK83yPsqE zVntA}?k?WjYYuM841##YcAF3RB|u~}15do&Kq6LKz^7#fq}^f(^hx1kahI&t@0(}g zwF*;t7d6L0+%yAHxoZX9Es~1!Z|LJXTTbhi0y*-Z35(VACc~V{OT55$l3?9Y5zdHY zA+J+-$JqBb8XuaU$(wma3zqH;BDrb#pzqKirdAR#VnGb&(upkVx_mC~@hZb_lOAxK zw+MGv@AE-agBtOiV+YagPk2H99&k@P3G3e%fg}H>5YH-6nD|Q;W^S}17yLay+?Pef zu<113vTO=moj4z-=Bhx|=;_2#ycfKz^~N`(`GD@U0JUq!aNw~#&ee7OxZ%PqV6-%g zBRet9sXCMk7DP!9M@0j0IBs@p&%7%@f8R9XEfonL80eE2&F%2`b8ooaW)gW9?+gT| zATeK>hmEuvh0RAH$UK|PlSxz{ITB+)?A~8I4L=0C&jOCoqzU{XataI=Y8c68;p9KN z0GPbMhXff3d;frM7;cq>A5F@ztKEGdHc1igt8c;L1IF;()gw4)su&(0j^;H~?g5jv zYI!YYX+-q#dN}afmc$jC!NH!#IO13*c9>CsS8kKS-St6m{;$Pkpvh|TW3|(SQE?vptkz3#KB)fkgxG5PA;2AJQVLfi}@PaqoeaPnl zH`w_~3imm#!R@?2&hg8aIkveO!W^l`AiVYhj-D5UOR{*p3zL^axf~hNu#*QSSBa99 z2FcLi;2QFGZ@)0Zz=bH<_kivj!uQOn&z$v#9HKD6`Z z+*j)b^hHIBbd?AEsPzoXXV-v=Rz-Z*G6I_YT0tt8X}~LkbBK>L4?D2Q;9gTaF6sN$ z@@VcJIO*I*5@@|2zWW|PGW@#1oP!7NU+mEa6D!u^zOZgA zOpfM^3@rmqfnUI#eTn#_$SP3x{wiK*ZwQQ%s(6M$5cJzL;wuY^0lBghZ;WAJcAoq*7R;eUZ_z&JLnFom;sJ;99+2-p@WLB!{cV!>UOXWTyvKQ`>*ur`)BYjGEz{^t~@ zqfirkFEt}?KTd`U?Zr4(FaRW8|G|9*&#{$M0_UKC3s}Lw3fihA$$|@;IkDF1xH_o; zTwi!y_&s|-Va5nPbH4{WO?wV>2TXv#N&tR7@&n7PJ8_kG5m+O>3QyYaNq)_DfJ$|# zctOl^sMjS0nLJste=7yy!$zbscs{wVHWPkXBuNTB3&3JI;ja3qu(zMC0^|G&Io3i= z_SN)-WZ%*}4&$53v7Qx${|i2j_wFgOnb9fWUHhgCZ%vv{1VWuv?5!(?ZyK@Uzb=mM zy#fFQ*7$?3J(PEyNv3M*L7UUF$f1kTWXD+!+<8@!gsZHA6`NLrQ~i;6IA#v<-o?j# zRT#|TKgJEKLxIFVHf~Fv1e#pkaArp$h|76xGfCaGRnJQfH&3YZ?D?BoJFR@+u?J4% zo4Exn|0a#kUk-r#o2{9;%=sCobFViH;gy9N3 zBX4-AD`jBUQW+9}9pIt=gqoG-U!Z?*Ir-2z6ApQCNZrOVkX#GNM>i+9XqE&yd1WK` zd_!6|OG(A;;Spe2z-RnN;t{ZNuK_;>?162+0ng&89;}PfAkPdFIPKq>vGH~RXq{|_ z(U}CErO=!2?xzktmMtNxwyuN!Y^BJ?1Dk+OrX<;&+|8?W#@MK43XFVvA8Up*0d+YW z-jA8e@Z@b-Vj3I-Zd{y3)cz#GyX&?P$ES{PT2m+qSe6C?L@L2uk!oNZA>eHKSAs`F zdclkSR_wOT6>@x3$S|A%1-aF@e`G?0ChRx4N{6|D}Mwo8lEnA}A*;>A&~ z`v_Z;NDxJBDRva%!Sx;UubMT=QFN}6Ab;rQb(mWwrcn`bHbpDDw{rKj1!Cz zqSkX? z!HL5{mgfvD%6;fZ^%*((>a~RJvxEZLU*L{HdFT19-)fl}qYx@Gr4?z$dNE4zi|Fmy zO~`z7HXAM*Nwa0A(`qqYbg^r;z{fWer8^!%(>N#S>$wK#;ZG07ov%-CXp0M~wb!C$ zjxOj*R0#dH-HLmmL57tJwG^PKhmp9R8(KCilz!M)!I}zO`JeO8&|Hm|jGpQlWOMl@ z6TdkTl^Cn>kFAuV_l1FamKnL|hUObOnTc@etB9X zQmpT%66vjM(K#7Gg4}8vROy8FwV$B}V_n!AGgBCaTat9D8#;}(4M)V`!4ZU$rM!-t}0t ztoIz-5+)^hP;baA{wHJ>I6Kh(zgHO;)5OMQ>}K4=uF=`KVzzE-ubF`vQS@`o9GYHz z1leeZvJyMwsCVxp^y`f?LNj%c)QX$zfRM-hVrUndGdYK?i4714Zg zxiU6*{%Tq%%mM#HW06&O9`zQo+4yDRRZ z_xw(7UGYt>=baM9KUIfSS{q8|nrPel&I#tuS2O0?IaTp(va{K-Fj3^#e2zB8#G>1& z&NN$XjEv7)irnrwQ2v2usMdHNeSC;d#UoeIy0h}N4wWRo|kvQmY!1i#jrAnE&c z^x;q@bI1G-`t?g2J&#OhhBT(pnN3MFX{6SgDQ=57?};Tke-hPv+3b>bTL%SmhbnDwQ>#= za=Re(-rT`v+@HqoekO0bMJ9yCU2bO1H!1SX&n8mcKxgz!w}e?_p-5E%RjIya4Slmg z4ZRm>r%T5!GHYZe(c5z4^pj00I(^{=3R4lMx-JInzj0l(HeeRAckiQTLN(Z&U7M*q z(}-MmJF=!70es8A{mgrIA+zT4JR~qGM7p1<7`yq&bdB8>=2>hql6lxbx6V&PmJTJ% z=NVZvE;yg+ospvfSL$g*?{V~aM-scD{unLc8d zx;spO`yhH)bAoQj`oR1Uay@_SNM(j&-|}5nzF=3xRiYQ+G6I=5qKy83TFmbaG6Me| zML})+YUEAZDCf)vl;4no9w>C8i|Q@N-&Bp(K0821KIAf1?F*6q`+4lMCl+jtYZ-jC zxR(3JdK>LiSj|S=S%9jI$Cxi;!_>n*k`?t$=UO_+qBSjv)HcSQTl(9E*(XUcf0l^t z*bfmD*H^$e=s%$CazYyb$z$DG9GLxMXX&ev4{XqsUu?D9Vl->s2utCF8?vpkj18Y6ugU$_o=T(zaTUp}+vWWAA}yF2RfJ;$|l?_)C; zBp}nCUS|D<3}&zF6K2A_19^IjF!T9u*&e-zXoZ0q+vaJ2Ow}(Vf6p9rYdj3O6;Zb2 zbuC}=LmktrHQjc&?FL^uqmIt&OruLV7toZ-e6F+43(iUQH{Yiv8=hmc=&LUr`eMdG z8s0mfz8CWJrkHCY_h*jGj@|d@P45WC);|iBwT;n*-z07StVUE-W;fC>4xul6)lpF6 zAS2TIoXHEbMZ5DxX@!s_xMhmG@SJQ$CErieX3^*D@Xt(ky8C2gUpg9 zjvmyuxtd)kw}jdANkS04s**7i&qhD5DhVF!mOxE;+31F!4a1vR$%vbJFe?<#AiHrP z<9Bv8t$&h1%MRspCI7oXhZ*77cu|JVxdK#i@d{P_B4g{xJB^O(7BGY6_o?sFBKq?F zJ!IKajaKR(W3}!8Zs^mgwo|G~+1EmTwDq71Gx6n)dFWL6g)Lmyj9>Eq8vs6RapO)-B! zmkDd)z>jgZG4%|)^pFpWMW@m$+KcJj_7-&dVL#&;Cr9t!b7NAXuChN~A7OLFs%i6D zQxqlg2Hm~#i?y~mOl8wfP{nII*dI1wXjxArZFG{Oyj*pGaaa=jQ7??0++oE!UDsej zrsgsRxzhwK`61d@H%g1fMVZ&$!*se-12g|)GJm#h9{L+LN znY9rHH0gyHb#{5dwOWr^gHB7@9Z8WKqe@*;tNFJE@A9vH4@1)>Yf!A95pfoqp_$8W za4V!wvk!|-pji0}^z5UXOx2MS==Os#>a?l>wR!e3ElbAP_8=W3F}D%T=iR1yl9fpD zstx<1P~3L&pGJE2VFGjS8li7PvuJ<1rtNdL25M|m&F(pP5)GB_V#d73F~7!%wuP57 z>IPBh(b_hKe3!^?!P8zhKC`OhC5MmWht67mVg%AH(@}?Q)a!lJ9@nC0y*Kp;!aH$Oo}a=}72453}OS93RW&oYlh1@yn|m+0nI zE2*FiA&(m2G~)1ACVXI+8`xPu1!AgHY2_=b+$n)(O-*8(#?R9ItFCkR?tRMN+x(sK zXSBlFIO7PyO+yk)U_em9qDvHr-kVa zyvyHmdK|qjEk;(H0BW%P2h)2u1*uicLSCa;$RTR1}~`#Te@T8Y}0K44B*EJi+xRV-)V4|CAsIGs|J!}R;^ zV$mIYroZ$DyAvtUO_M{AJ(Ei9UOJ-0;c^tH=OP3 zBgIpa%-#hWwkZ!x(YF=$+~TJN)ctKB8#M0x{qz#(im*-8_>(%_C;66o zTlUbGJ)Xj-hwXe5sZ^G;8l%?V%IvDW7udwSDU8##epXa}3o5;Ol8QS%VNb2mrIHK9 z>CCnhsQ&6owEs;T*VyA9wNDg92VQH@(_AAIICnMsX3qi|?la5|`qXkP?i@r1SEcfk zsWt639^jq}%cM6O{<8j_GPYa1?=p9nx1k_mt!Yxor`&>iru^$t8tz<0CErRjyZ+U1 zo103|+FzseL`ebuLN zNIhaBc81W6aU-;RVLZL3&q22TRWj!`7^C__8&E&vM!Pa41Wt~rNb>b2^wQLo27er4 z*T3yVD#tJIFO9us&ujW4HOnO^OHGt^GG$2o&ON3;HkuxfETa3rb+AQ~f@yG=I=%U^ zm(Of}h4!Z>vnSe;k#TMrv%H?L-wn)Z!Yef-G4Y=Ns`wMV<1WU`e!PUyJzm5={vnn7 z`b68mYER8EeWN zrF!YTsIAJAb`9)8Cr+#r*hD9zB>8r9s7n+1JR;CYpg#q(3Nz!DShch9va4u<=LK4Ocn`IV&8N!tZhV6jaem+JA53BNHEMkH2P2lalH@6u zF`*}=sJqX0v@j(XNiWi7I`*bALc|(MXnRJd|L1~wnIXdXpgKV~_X(f_rn9q;c z5rd9NCNuBHw$gI}4v02)vD#}YXwLmncDG6v6{Pd%u;&e`lzap=Rd1o9Yqzr}KE6WZ ziP|(S`2=dYd>X&B(?-dQ&hod9Z(@#$_o48AqV$MkI&(MOhw4;JriY|gvfl0AxItAi zbY|{VWajygHyZwc|MSx(`p4B8@kHmLgeY^?yKa$?6@H3cbnOb$|7ANZs8B+!?QZm> z=Q=htu7KusG_t=&N|45;5(bYCBJ*RJjO*uh$ZO|9Mi^~FC!{3VH-0r}*^&Wfz6YOn zna9#{*IQK4kx)q4=so#fITY3Gne*Fpw{8M5==iwS0Hu4T?k9HwP{sWj@`B-=;njj&_uI_62t zAKIxuifp%paXmaKd%N&G>%93YpI>*5R{yMFbRYkJVJ4cHWj95@4*lSo#Z02Fr*v~O z(MkT`y=~}+iwpH0l(f}4lZI%i2&>j1VS8rEPU^)!Ox<66mN%--hAcI?#3y-iXHOpzO?q?QzMbkMcEl4lWZz zH_ztt+q+&fs!Av6z{M?$vap}B9Fk||_aC4OY-E^ehxStu(VggJQX$$Z_=w(pPC#R& zzZs2daa7Xo3w5nrg5>zpY_-@NYH#+P3Ch|?N0kQ|_w{a6(QSZRTinf@{ck>V<3Kh| z`R>l@G!OHQ?(AW=9-7Z~EpcO6m#Oq?RU|rj+mkIat!Cav?4&s#HzK$92e_K+^4a;} z@-)R+k@{}QW+TNzgv>;i{lTkb^)D(4bXFcgQSa-8OsPZssnY7~glrN!IuM5x{+>rK zr4O*H+jcU0x~@ z@_($!Ls5bJP#wKcQARcHh0}TdvCOiOWmMf#8+{(EVD8)Wq1}%v*!p>usME{~xqLlM zKaK9DmSPwU&%Dn5R&Hk&?@(j*W$r?}mP}@jg(wQMo+Ka(8t9+z>2zjXAl-hijXKT0 zPZc(+Am8?X{9L)q)Ky=fdJfny+n0BvX-Ebgd_EVgly>Io3_7vDLQk+)r>mh=zirXl zE6Q9e7)M)%4$|T%dD~xF;ncd4M<*NCvdSjq%(lKpCMq+Belpy^=qR2=$z>2N@sJ|9H|zb255$T3DX@jl&h4xl0T+4N^K z#d6CxAYe2J8HePfrAp_RiZMfCl=PRnn{J?cV++vi+qSG~5wm48gzV*Q zv-qVy)RE4EBTVzvdFWe2G?VbYUHClqQ7?@uRIRWch5I(s0hihIRMisHCS-S)|4C$> zcHTmX#(wCX&n4QGIG1*2Y(jdicj&|HN^~e`kV%_b$@iS?NJpInNb$NC3b&X;lg<{g z%Z&jMNfQ&)GtOw>LBggUqesb z_{9CKro_5fsG*D-Y3Sh5o2;G5IAW)&AlW<#S~gSx_dUGD4jG6sEn%&6(xQ8`f6Yep zVTBiZ#(JR8)1z#P!V~U+s}E?sX)kF5FWG;7HQ?sPlc=NW7&CQq8VMPjZkxLFBzIi9 zl{NU`Dbz@cxPAZLQ0~b%O0xO1LL-*FBz>6_dlaC1KK>~9R2s9P^*U9oF{D>ar%@HH zGxU0G0FpaajHas}LV5u!>8bHg{F=-wls9k~$zLj>{_g3>8Kuz1b(5%qgbI@Me!-o& z7@)|YwXCg63cJ8`9V_7>tSvHuG@w<5HE?x8?~T?`1+m$5z34t#!G@r3b&uH?=Y=$V z>o0mu`aR#izK9K9@Qk0`c!Do~xREiedCHz~TZgiOI_XV;4vH-*q?$i`(L$~-KWCXL zZO_RgGj{bOc4-phF6M*;Q|=;Dsa$$sq6%4MEMflXdeElWaDHa<9{%CuVw7W3&T1`> zM-pXE8T*nt#=-tPGHLh$EkhU6H47Wj4^>B6`lAS)_pf8$*%zWb&#UYP7jZh+P(v3b z#BwbsT}Sf6)$FX0I_fo3QSe~ZB_;wqfwvkusOKk^Mm6nLusYi3$AO7HLVN5 z{O4~C(WK})`n0Q;85D0vCPDd(#U0@@LY{NC&i~3jcd4N3?HthOiY0uNcXyd82{|-4 z`yz8Jsh(|VX+=iuF3hU>LMruoJAbQi{?yPff{GrO@xAA+rEz+i>?%tReZH!bX+P3M znCt|e5xCXtq&u6l(SffuO@7b3Ld>Yg_LS}6gL%IXXbk4`w z^nHjdVtTn;pM(pj;P(q=)N6vj^nngNU{J;U300*H>ZL-a?KRes;j_E0?nJ$#`x)vc zPamy+z<$0^%GmQ8nHQU*Y5jjaG_d&ws(o9-Up5CKy{Ko1_c0#TovueB&ht?0yLc4w zWgi_4oku;DrRekOT~to=3eu_APLEEjKuhc|q78*|bWK+zH!SEJGLd`FR9cKuDYJLX z*Sa7&5g<+#l$?-%b}{{;TZI%90-4*p7bD+>V5FV)kVMbwW*cgsFhbZDJs0LrcWsPC z-CH&z*`eEv$h#o+rn4>DuymBx+`Y(J9`dFk?@AFBgfSLwpU{lorR)YRH`eS$EnC?m zYkS=NHhrqS3YEk!M&>b9eCBflt5$7Fk0}(eI`Ibd=}{|MP2`vbd8X)l@oi>J{#n@O zozEs3SP9}&H=>=L>yhTAqo{PMh2YAq$!I1g7iqv%%)ovFI)9lWO1L=0uKb{m^er^e z>TE~$?eKhCRefF5s!_}?DBevE{r!iY20!9=^>2rnjyGv*OC00Y6v{po%SDF;F?37P zHLir#BWB8-tw^~pm&X3qK=#3g)cIT$HFvEe=dW-Pr(aeubYPSX8%hbl4ejjIPUn4MmVXv69besszS*4%6!dyse{ zJ0YvOjc0_K-c&PnGJlzIT}{NZ@aG1PykH(ZEu$r(pZVdZ_9A-iA>Z!VMU-4H#Qhf1 zLHF|JqKrQQXzjv4T7SJ4neB2y+cd|R?nEJ#;#Cd34!6=d($)O+cDK-`6_}BIdWD8u z38j0Yb*cB~Y}n=!%}$eki(EE~2(CVP&!oLKpfe`P+eUW%W8duTW+w#o+^!97+>aJ9 z=ngBK6I#!(UCs+bi>AG!90?iQpn`l<*PM=$0ww4k-TzT^=J8OzT^P5tSRzY^NVMA0 z&U4S)XNsZ{DQze!B}qlnj7mpM9iL+Suryumij8yn^=8o*&iw-u>>MqRKfqd4LXhzlDUY>}L9O{MFZaP8XUxm+aK^bt(L*zz*hDO4KG%AO=NM2Hl*n#LcLWVZ)7U91I2XV538?mgY!qxp( z(A!4`%1hQIfn9?G?EX3j;FNOWj@qvT!HLp?|Ah2G#-PO ztizXYErtHRK!2{cVx6+zZ0*a@cw$2#oYnqJH}VDe&*!+%E=&{W4!5A%;hEGkeW9#H z<1Y3r`44nf5OBI04RyEd@Y2p{RBOdMh0aY_PN}lcDnFW!#lK3n9qt zI9gAAP5WdvAee;Wv0rm>Rd6~ zIbyq_R-ImK&(3Wy$HujzFvVLPyv_Q{fBqhfhllON;UgN@t%_qZSGO%l|4XG4P0O(= z@D~$QUtuTmmc!qwJ?vqC3I^YP3SoA;uvYyB;~USSPjL-6>=-3VHXe>|#TC-ztdsO^ zH;)Hbq~fgbZ$kB#W%#On4!(bRoq42g10ydb)M#=Mribmu92I9aZc!^0`P<{yUktV^ zETETW?T7CZcggPQ>f?Eh(a>cci`MV#>B{Lw@MOCc{@$Jrlb`$H-{X(ihHNDkWP2Tk z<(|hS+UKw(guDh*M|Y{ z;)4OIe3*kSfwjyzX*D=4SfRKNeuAhk+hn!Y_bXfw0Fl|aurYqC!(USpW{4%ssA?J}q6al%g{KhSKk9-Mnk z;HFV?wi5LK8T9{{G^8mq(IfO4$xf_50mB&g3;g97n0n@ccY{y<`hLQa(&Hbh{ZiDeCn} zeeCG0+G)^S_2TAy3=V9_I_bOBYsY! z9iQyv<8C;xtCudoh>=d%UA%*_)uE6TS5MpeHG|QR@$BJE4oBIKW)-nf(C5rSocioC zc{@G|zpehnl${FMsS)q7sh|S3_qxlL#$Nz!^k=cQrueUQHMMVT!r8IovG)QcQJIUF zahqbL&cpkHDDyH~6tz)!U|SiC|9e#^c<&}u3@^bk%BhO=CWQIb&)9!MRS}$$XxOO& zsuO&H3AzJe#ljp69$=5_sxQ)MrLBs7@LDRW)?(kQw!_fJnxc94d|=`DA*|Zw2fBBi zVZZAHV7Jy(p8k9@*sf7U6U|I~yK5@zSMnDkRwaPoy|vutmmOf5A+8VE4jYyNy7r1^ zy6+Fe&4r)f@Z5 zmD)-$#r{4ne(#U=doF;hh^K>=Bs1>tY&bFE2Yscx9hMAQL#JF$rkA&`!+?fjaNF#S z?^@Kr{7=2`zGr_N*LYIU`K|`jC(K6QKf}Q1MI3v$a2Ohzi{ZoLi?l1Bg31bIs(-1M zV8b;=XDd^dp~zy7x^NXfb$h}hOJ5NFnT^Xow$LfRmH_{%AGO)M93PEShjE`DGw++@ z@zHP+?C$qL@;L62=G|9 z5{V1X|M&Y8Grspgc&YTHaMYd*W?ryF!FW}|bHm-(q}l-Dm$z_9uOzzUO)*XXC}OcG zrC9LM6^1V>k*x`th;`|Sc<;s)!Ed=Qv~*1q{tQ%t=!gg^J{}UB!^lbPLQ5RY9+~Tk!HkC)23Q zlr6jy&U}<+;7_+KSXQ$_c3JN_)(kA79;$a}JJH1jw`ReGX(QqG-B{tg;VIOzA{p!c z8wh&&2kFq}-`KR~Bc2Jp1*S6M@Q@IBLtX|G!>J5+X_(f)1FM|OseEc(U2D$>AS zHOpXB$9|kVz6^r0r^1`;(`@_8RN=dxWi%WbLHGK6=up#!bv1b?e5eb|Z4@|m-Uk&A zWtdeE&1{mBaH)faT<^(G+1Gk)(XluM=i;D*dy*(UScz=Kl&)XcL z)c7PFt9cP`ULOdpW3||{fccp8-3rH?y8zOGhwwyTA-KORW33;iL1KnKZnqLZ&7DzD zHOdXDof_EtepB#W+ypvj>i|(&iY|!a{RBh%Cb05wnQ&TsCeG|Sk2Q(Ln6l#)8&zWu zqJn+Gxg&y@{)|O<=FlUm@==uzZ1=}d((pQ!)Yw|HoJacY%5}?xQQyV=>^{BszGgfzbcTZMHipklw!T zgwbh5=r!j%986W*1ND?}L;VvPsJoR73fCcet#&vm?kr5c9fU4sAK8t1Mvn&WSJVK1 z>DiwKxG=;Q+y14|6>B9@bD@Fk%I~YJ>Qy>5*61sglf11RzmjIAF>w< zGT_9Q?YMdEZ@PF+8g=vV6g4f=Mn`!T&F&rr-|uU|v7x4-q=$1@tlWi-&5FZxfugVD zCdd0xWiY~BT{L)I2HZ(Z#l`PNq4>-gNIde4{o5M=LVgpLl{<)X&U?_DmMO4GvK?QC zsz6_FM_7AY3gFrXwhGpvPi;J0`n{cYZ&#M<2A!bhL35aX?>Vr{q!3TG?}1y@5>)z7 z0M>^+1#1HA=;8I_;dXKaJej^%u66*!Q22E#mUp&cacO9j=-FVHofyzJPd447q+##SrNWz*#@ zbWOG+{1~K|1~vCO3+EcC|Asu6w5=XLOzDM5(^6TlFS@uYrjF&DxWq26?IpT0z>{8X z2*W!SA;=yVr=wl?q-zX~S>Xfwy?u?XBE9IF+Ovc4P{q!5sevOzlJqp+ao%0)SX#0A4bco3O;&a3*MYP8dLkcqj^E;P@O-Qu4KDpvd5*^Q)UTA&0ay- z1#QqT-3g0#-l5`$Ytd{;9R^Q%$HL;2ME3VA>CiGK%&v{WWlNU|$$@P8=}rUunREd2 zH@l+utvt4?`fji$ws#y5iJyh6jQD>VG`M%-UApMCptp3YS0c>7ewi0(-I z*oTil@P1nwF7xa!Kf7lP+H9xHCaxB&Y6_)?#xKIMtr_@jb1v-+wu0V8M<8{F8l;^{ zW|j&D!9BJSx3~VG%clp(7UV93N8LYgZs#5dG`T?I_p69jt=dX2ojIuBXvGl!&x3ls zi-uU=gKYSLZWfU~5ic8R4}s|J)@xK2v>bNMYoL$brJ}#W7jfNpBt|H`V4t4) zgL?0B=58KEOK~@iFR_AO>Qg}UTX8>MIUG8z4zrly7jc%CBSamGhmE?6L{nVPqMf>g z4S1#kA8zrW(j~?F9j&qnWp1!_&I3%{s?Dl~?_;NT*Rl1jZv|(q4e?@uLX+4x2g`!j z<5SlkY-ijJb~e-z?TxEx>0&vnI;zBMk!}3DTV@}6 znR<;f!|lf!F>?B0TvJ<$qvj9CiTrb#APzzOy8DWf^DQ`?c*a8K#KMNg9rVpgE&0W! z-l*QVndW7B;$8O$_%d-3d}tKoooBm1Fi%Zxcf5;zJUSGt-YR;`qx6v9dzyLn-bhcv z5vH+gKZD9vn0&+n)s>&J!p;P$R5qRNOWK1U-^g)A9?vgvwPM$cc((O+F-!4VLoZEs z#Lt=UWp-=?d-2?tRVc?`$jW>uH@ra8Z)-s8Pe&;BeM1u$jRKs>v&g(`oVzX-U*D*~ znh!1XC2eC578F3^Yh8M?;2dhbX+~M!iFj?20X$DTpg7lfcxiNn`E*x6_bWX-el?wm zLd#&Qpr3rzixRr4`Z5cQTLv4{WRNmt4P;ssqD|0L7&Cg4(4~JOPTEug+nblNzUzlV zom3m$?1#w1SImX#07JQZPBtr?#G}dk1>mD#^_9tVTz~u_t1CkYerzGS_-!C64_>Dj z7;DM09>if}d7Y$rSU2TJCaR8_2br!HF?{UJxe|d>&DDK(` zN>9lgjI!4W<1niJ8AjY|l&PH?j=krfrx2Hn^Tt2KRpS(O?zopQRBZt)${m0P z3C8$qTq}Gl`UrjYeWvZd)#WD^hfsm)b*eI^7uHF6tn8A@1_82VwH;9ki)d!My3`)7ag;AYfxI*!xw%$+~N3 zw09)C5<8DJF5M3|I%mSH=Vgli=|HBMmkvo*FG1(v6?imQ6>l|OhVN73*{2N)@MV7v z<>~|nKm3A^yv2A3mJ7G|mb0u=%TZOk3Y|M%(5UyLhoG=a#1We-24Zg z_jATz)A!VUaXQ|gI}YU&7o+7L5BBLs1XACn@*cB* z#YfoJxgVt;mgUm%o`XfD9dFqD3?&#VOvN;TjD@%!p`-5!uur=d)Vq9wH;aGMl{#rK z`N?VWV{;7ceM;fWqV=@8_XgVW!T_N-Tjp(X1NZGsfs<2)fVPf=j!W7{?@jh&!nL7P zeQF!v`Q3dx*-cAW<=-1#d)Yqwb-};P||syl|ll?(mY*(tZwL zZoU?p7nI_?Ws1*6PC?ml1N4u3%AUD;(f~6K{g%9AUz5${8LLF{BacgIwc$uqew`0q zeeYt%p-Rf9>WMhB-)!}Pi_~P%E_4aHON*}O3)2+-j$=>WDD?2x?3M0L+Ak=MEsrn7 z;%|r8c{^nslAguZSzloOcEz}I$X~W)g9DCq^Mr<`bNF`PNBZi)0CNDm+pa$ArI;JwK*8>s|QJgpFqg<0iwJ9!1+VHzCOrC>ee-uR*nG~Cg>`sh{(evTrP>>F7N>)-Ya6r==!0g#oAJqyv2vf~uh=cw5UHAi zE!~o2hpWFNp!=a|5Fys3X@Tyv_r7t=W!8Dzc-|T#m#I3Qvb%v-$CyGz-YRrfil?rh zZ$nGl0WfUW#A5k!4C&EF&g?90vM3aKhw3qjQy6_w)I>$kKheqQb)aVcpI}sTKKQ8! zae!qy2qKJe)!SNJ5%K{v7cGYV&nCdInr$#|gBB)cOr}1>ROGw*E-TmUpt+_IcxH4R zl-+J&pI5!1VGk?d;yxX@%FHwDO28R37Y;>_L<7)sJ z?@6S1Z=j+4{r+!Q-W?;{_IxdTJUHAHy1)KWI|cQS_nGG;WMiduuv$S zS&BBz&w!>(eWw0?FBDF-ft2x;?5JxpjLJMgWfh~@mjEw}I>sY=ZYRIgq$S7d9GKX2 z6FlZ{Y~JDtaA{czO1DaB-lhUPR<#u_Ep7qjkwxL@$Z*5k&8YIc zM|j%H0Mhg?2zEc-iKf?f!UWZ~ptb2H1b_bwmXQ;|CUZGVa6z=mEr*`wnP_)<9j+hv z7>1mfkEYg7@RHga=-4`y4*WI_f=WT1-V5vT4vaLJ0T9|CYv`TMBNX zXArQu7D7{(z@!Z)VRWekMxKp@pW{5y>;NxSQTRNJ`<%xvFChj)D@y*mB#2k=Lt;7{ z79YrfLNkTOaid749k?9(svL%V%^|eKB#mtpFo;oncipF9c;@|NQ3Jn{vCqR~PS&bS zaO(s!eAGu8`%y{c7MF)3J2s-vF(t^lB&TnbRYXrT6R78)-&h($VBsnoJSk0wV3XrG zt9vxNs^$bkwc0^w5GeHQv4feeHqhti0eJIfK3wNU!q|DEn8$uS zoZ@h6Ja^eeGXnz0cI;OOci;mi0y*$nj>Mcv)Qga-|9 zP`oPZF*9Q)$Iq9Y)|Qk1^a*%(`(kvpsr-V963Qcw!<5T6SntoLWm(?&Y^IqZo%-jB zuu52jed}(Ji<64rRl@^%@r{D1`=AO}*PDyXK3HSOF~z&O(;c;rJckwg>|uO<5nFZp zGTT7a!25tH7?g<6xoQ|2Ze@kdFZW>C->>Xqa2^=6)Pc<(SLk1nh)YMhV*Cy(`Ha}7 zG;7pBruH16kN6{f*GNxEX4kA+$8?WprN8yxjJuw(FTX0YG|zO6b& zm(?vt!P`X`m2i|EJi8GuHEYYAw*|pU?P82N;~?A_mx2$MJIm&tPNa59Eo@d*m+W-N zco<+20X+|-vT26~!1LXT(M)#pWr->;*~a%b*uM*BaKpa{Xc6mT^^$bhntoJx%+-v& z+W(UA6*AO)S1cS6pbQs(PR6QknM|i#N*m(hF}hDG7DV(C%~N@b$J=ya&~^gl4u0SV zb?6r30PB`*W2PNGz!%P>`ELeGb+Qi95bartF&sP{c*aa#I{ha2{6Kcx*%p76Ov59G z;^DM^DeY`@2kVI=>Eye6AZ_CaNImX^BdjafB7@%WUs5XCtZ0&*xuFkVzAuHqH*M^x zQ35qyJppBrTi{1W92BPN3fJXpz=e|c za6=-%SLK74W3ZInxPPC8bUa`utNy_GOLov-aDsV#_oh}!L%~be0vwGFVPyMN+44Ku zup`YEwk|jT%6k|@ZeGN~?04grvDaDC1T#o}6$I~|)?$m=5%y)|If(A_8q-5ftY$_ol(aO=!mAbgF(vps{WL~XAo!At zZd1?VvUVdxvvatfz7ocjK0wDa2ica;IWW$=A3hjlimD&t(BSbJ;ZCb#s8%GzX}{*e zpu~|-;k|?PegB=Um>K}{HW|RlRL0t+Yk~dwA&t^c#);?S@vqS!H0N5V)N&u*Sepxv zyOJTVI~ya+#^Vfkf40$oH14q7OBd+OM90ohnoZ`9NER_j)G;Z^^)9 zwZAMzrvU=nf3tfXTGTINGzv~+GMA7zG;gSat=%r*UF?W@1#PGlTZ@C26vJpWe}$*x z2Uvtu!t&_V==1U@{>ko^wVgO7Q(54}WV3hUh-KgLUAZ@g8Sx-pKb!pvMf$;iqs;E~ z6Rb<#4LxqBW%q|v%U;e51Iul{>Fuw3*|`b%GQk3|@bB*fG;v5EhWL)4C08EPkD?v$ z#v_KN&HT!?F7IJ8Y;9O@gb*)DwXk}_ai(M8i6_^qqg{h5PPl)Dez?^}^9=jbi;WkV zPna{To%NbZm<`N-^A5KwSZ6-5ACA8q3zd%-2~!qN5WesCS7ueXi?w}v1*Ee9{g zv&EEJ-btiykGRlsu{j;gsVO+VT)0y&X2qAE2>&*B(uRc(S&y`z$TDD}f)9TS3Em?3 z{&NtkiFaZjotyE@3y9U4$*?6&|qB3QwbPDTK}n$6x=nsbShy?5d5xe^%ibD(#InL-R3z z?rOX_^A6keqc_&vHkPS>4Tho<$@KSIEj&6Y2~;O!Qu`2P(Iq7rnD!EZVEHbVs;>%s zkrl-4QtD{x$!hf{l-qbF8c61jCBja%0u$Smg z|IpXBj=~_@K=k>hgd0!IqBG+zQ`+N#<{|mi>^?yIw!if2D@{?+wtU(dRtrPkn2Sd9 zFJQ>NH*84DeyZ|)35I!zab3(k>~oOd1Jn?CKlh@k3ICu_>;StiSFwj<%jm@AuJS#l z&KN##8b;NAV$&|{rq2f{e6hd0F)$VDI3GIR8sMZRpB^9e4*nncCy{q77&enn{&4=is{KVNf@65yZc6 zK)uH$RH%E1u9-Q7${!uV=T3VeY|L{^xxIkRFxVwhEu91hzKM-`QYqc`-Ih&$ z)CgYjyHI0G91_V88gn)pEqm+0kaVQ)b`*oz%*@(LE-K7%%6*zKFNJ<{?FGY(chdxI zeP~&DLpbpBbVx|}0{2#pz;@397UcV0*y>~`+F72AgU7C;d#Me3nQ)qA49JzG8~>pt z6?USf;TG}@bK4mg)*Jgz;OJ}Oive?v()j!i2%0|>&-H#IG!inF^H7C)R-4Eh7Mh^* zg><@C!8+udsK~Fm|3&7vNSHTtgW^0=;Qmu4%HzzAgXge9>>Az+ea@dozZvGR; z*P4w6qxYlfkHM^E+aETl%UquA9EAVwm_yDXJKPfP!+xze#r_zm!>LLA(dls$v+WZN zf&cx79ix}C>$gILsmaBl^=`H_qE|FMGi@h49W)QGC;G$Bv@!I;5Nph?N~iU@W>{0+ zfyRo1Mf7bn&b2xN&OP@b``!;$^>VsA@{tGL(ilVQ4*wADHC_iFhjXz1vgJ}Hsq9-9pM>sCPLx}hv=nhV_jQ3f>)au~bZgR17~ z(f775V7;F;8zUva{l*-|#KTDsUnf?^# z4BE|lPPoH=2`#XzSPNABQdrl)8BFp~L#}y18;;$!!;vLc?5uPb7W}6wUwLe(=(ab- zlqE@U@wXWpV;F;ToVQ``-pAOmN50^b=Z{OTw!+nGZEUXQems866@UND!~BjSwpgM0 znuja1Tq_Pvw|&Rwp9D0bW*3eW)1R;gCGamu)j%D9pd94)vk&>>4wn zZbN+VDvhJvN%LiM4~J6p9w@tZWGkHWIYVbJh-F6CfDlrJ{Zr_3qBL`;R^O?PG>n+k8Gl@z4jM*`j(V%;IC4H~*iRqr>;qWCB z(flrDG(NMRjlELFrfk;(_oWqVdgf&mHY?%6$1-$`y9eW4&f?~xIdE&(U@VwcOovbv zIyU$)9lt*ta~ooSwK{@WrIdMeErqtJ?`1oZk78%RRH&G|7yf=-3Rx#=snSn^<1Qw! zBxA))i{_J5q8lHT0qw7_t zDSVf^!EY1AMJax;V6q9y_#tR7*2G*B8~WgMG4|_R&l>+qaQ>-Hc;HnGJZ#i~lJD9w zm1B%vtsjV6R=db_Zq(3>Nx8CrJ-ad0e?NOFc?@LF8ioFu3@%rm!n|_{m}Q|a5$+AZ zj=6!LY%hZC5zA#hve6i?sM*HOJ|tbHw-Q|D41?~wcj$P0pp3z`=Vt z)Kj(q^xMDELZh>=!^9q@81G>EZBN*<$_BhEdO-JBk@pHMC#S=T@N|}Z62e`a$Bfb==VNPfVEw$AF=RwiJO7-m+b9W+Y4W0xS z)&7|#zyzd}{? zyLDLh{;LYUur-A8jdsPU!=YyeaNa5rGIm~rsrM&fX7)@vu`v=} z7uZ7co)J*fdV_9yx|%kWZGynHooswU8U2~pAsf=V4)=r=(nldjX?Oh#X839$&V23% z?=&Zi)<$^4zwxTFw>f%B+V`@Cj-Nu=OE+9O zU&8u7oB=C#O%{e#ufb>SNwBpl6nDxckgM?>YLBNVvMU^A*$ehE;nJ^kgZC_WH@%1& zkiW7;A3KGzBY6$QXTbRBk1S5`>&FYF`8@Vmngy)%LA zIrW2^6&fkYp74hoa%d&Fw|yEv)9V-cSC!A}WKSbzR_P?x&QNgWdAsE6q#Z=*j}{ji ztC%ww>%)!BXOilOsoWnoE$ONDr`)r{75ub~$+aU&ECuz0WB4OZnIyXE6zO|)fK*r6 z)v@%yKwda>A;0a6slY{5G4uT6ncB|sk7P=3kzncE`+TimBZ-_WC(~D{N=LlTBhP<~ z>DdN`vr;MRw?i3afYlecjwM6ujWggD>xOgjBJ}8 zHtFlJbG&V47PqR*Nn&s+l?29rnY8CI;hWFNx%KL`+#P*ye#)-t(yHTQ1s)zR`8SJA zrEAQDe6mhExASE+g^zPehIm{E%*t`AO%i{4Ea z%qcRI9!~j2l*5J#Rv$Owzn55(fz6iu?!koD58qBc_d3hRz0@L-9f}VQxs$QhrzNuv zhB@9X984PC-r(M@i{}q}#}RW48}cvBn)B=RjCLYlK!chSNXM) z{MX!denq znm?kli0t}X$zPC6BJnq*q;ZeEAf~2+ca}zw zkd6Py!JJ$&XJaKl#*cAyk0UV?I&-IF*}TcB3tZg%hmKYGT7qjY7f(E2RKojP202E& z+fQn((s`SPo!sV`QtqiyI>{89@&BrNB)9dd#aT&bi1jpQ{%uOT!<|9iT+H(z?r`Bi zN!9OjlGWw6xjRLHq}$25womgSUUg=QxTI6aD-YJ)OwZBN_bz9f+&09B- z_~=AV`D2=7r2RMI>9(2Iw>%?OPFqEcznAec<-SB`K#lnK+d6LRm}I_T$TVWKa{&K3 zave9yb`-Col=4x|GA3k*~%@*>8-A0maCn`9TfIWO#xsl{z z+b_qOo8Ki|PCd!(j+Z!`xg>r&HBv(K-S}0_D|=YXUwZh+)T;|r2^RhvBcRw6NcGlgQp-0VdL$X<`MSFGp#@*4 zHj9jj{VY!S-J5?XHR8leu8Hrj3n${BJ(7m^Uj0& zbBCWClx)m8Dw%Qn35n9S;=3075Njw)I6v)){QG{lIs0C2{Mx~p{KiSoB>{ojw?ukRl4b1hUIXGorM565c|xyu)_eceaN9Pe<)A7Lp( zGBA_BuwWG#ab_?n?RntPFnR%5_E?!SDHAF}C7p7_GF_maTY8gZMYJ*oP8nV)s`1gGMtAszAlGAZ1W z#7|oNk@zR?;Q!U8@&417lli-U@LyC`lY3!1c}cmP^wG%SC!MY*Q$jzJx}BN)ld5_W z`MXMT=}0wccSlEG&wr#QQNXj0e-63^8NBARq z){{%QpLmzNKjiT1ES_sel09lKZ|XLiNT*#P3yS6u$3w~jJziU?lsAlUAHMRLH^&j~ zNCRJP^W32@GFx0`DCHW?^_NC{u;w?f9Vi_YbzJiJrHypYcrQ-wtRnraGMoGfxxx3z zcjWJ0`X$-2OCa%t(bCmN{KZfIe&;uit>^eDtwbz2$Tv7nCZ9VP?{}e&4~R%0qxPQW z4~tI{?tCI&w5NzX_L<7B{HaI&{f*`$Ycfc2>lyx3;3;C$Z6V#bFOu7TSWBu~V?l}* z#Syip41UhsDq=G5KEK1%L~!iQE6M)2*W~L)J>qEgi+IIZ@Rd7;6Tfeh9futG!etcY z^F0km#6EV-{K}()_-`B-ZgGPPdW=H!ksiUW#k% zMx5iV9P~KR+XQ~zj|fudkj#5rjU~T_I+8h8^SQ&$j^x{=SfXBU#*Y*6de8ISqwkU);XdwL zjvJ}6?#~Bp_{rJ(J>~1;d4BVey?pJIuRLiq7QEO}C85%CV%QSI-y32i?VD^uoR=6# z)uuX;`0K!NjqZ>nPxA_q5%ip+r6P5U`30=H#>2m&)`2lj-;+tgMvv1+ep1iU0KH@zpmDhtz;-j^x^#gc^xxa|H z#VtNxM~gJ=Jk8Iv&Em$}wD4u&ZDQJfoezBKNV;=?pE_GzFm=;4-Za07l;63|oz+`J zg4ub+?D4+bxz$Jc-nVt66Hif&jWCwJJTE6+k8OyDD4V}<{ygyTyc^NFBvN- z-n^b);JSgUQ&-`KfBD0&y~T?Q@AZ@XUAvJNZ#I@}=@lYQjn3gld+j3Dxy5A3`JLRA z)=@lpbb?o^%a)i7a3ERhhw-OfGsxA1A7rfm4Nh;+PIA0Omw$5UpQG&w!ks8qCoue& z*pSQSY@ZzA>WX_LLzlIafFUQyQ%4KlDRBrHHs%b8+&qZ$d@Yd3ww93zmwmW`^?gZp z`#;A@vmTBnsPP4bnc_8VzMS0^D_+AVgY^D5jy$rh;DZBZk*;s8j#)dF@=-*NxS1Su=^BsbW8Q8yT2`B^7Q`)9N+x|j93a{z?-lnV7v4E)3$Lq|!$%91i8w8TWSRPtlNR;l#c=-ugRI{8+ zpRkvB%_^4|h|Gy}<7J}22e^%jjFzrf_GCaF5-SG_eu&JT|9hccQl_vOU60P-#~mvo z$N$T4{Hxn3@!3$p#lOg`HTuwpv@W$I#}&DDWh(zk4wMch&O^#1IgM7e{_kuh4v$=k z!q3n5b>6_+FFGb}lP)7xFW2z~Y3||@?}_|=6=%*pwn4ED96852vE%2U9sHr2x%}pS zi4ry8KE5*~h2JV)z~{DANy66-6}&EB+>N6azyDQE~=1DUp?lUfPo~`PF zgY-PnOME7gjMpX3^-^BDUWNaBaTIyzY$}Nj43gCDQ|J2$6q&hxN4R$`aimbKePYN! zbMCqKGj5R6K}kzXIltKCwb=J=Czo>3kpD6D)TGu0&EkGRZ^@gHZzV5&wQ#K+nh^G7;YlES>@lf)@m{PDfPjw-v;B&CBIgv2giyj@7%9 zStrkv!~w^-_4Oce>DkG@T5^E5nK6hnnVVJXZ~BRg)PB#spLmF?chx6)95}9wI7@z> zjuBtKI!uy2{~~d5JWW1bP~%58PUS~NujjoqfALj{%&|K;y#$`-v-!B$y`(w^!vv2K zIOzn7z1-oFX@YrUPH;gjnY@ZvN$Om`l>}T{DSZ~{Dd;vBEm^VAQMz<-fFSn6Lg^nJ zU9LEGyr3=r27kk77rCFZT9SBHU3zf;L9Tr2Afn?{OOCBc=2y6n5$GywNE0{Za~I$A z7Ffj1knXoyM?y^Ncvg|krw_~G#r8hpVVQc;j%~X9H7zq~&@x4RzsZ5WFyK8uK!1{S z^G+kdj$Dys|Mg0~=+tuQjOspu^(&>4yqa;+x=tIZ^0N}h850TLJ1CeJCEh3gqGfz1 zc}6Toq9QANk#ybGQ37$r3&pcqDgC)JkrY=v=giWtbAnHDQYW7ym|ieix@*D_uJxF$ zVA!O?+?Xgg?s42za{b92$50Jl>FM=~`*oTNxjx2PTC{}l#@?00V(_; zb-R4>ZM!Df^rDfkj`R~4#mtose61wdGqsrn`)w86vZ;O% z!_IPpU3&|hifp7M-Ez`>v>)Fq)IvH-qpzS&K%|`W6w)XW3KngiBRFeeDV_JcjywL% zO`tbwv~<{wQGzi&S>$$0AL-)sF}x&uH_2@bCUrIO{DIM-(${LH0;{35wef|*Xo`48_a$ceXVg2iRqq&ki(1;vUC@qJN>EbtpyT=`uc zsUiQ0d+oy+%Fc&ukkjy}6i??TTkEaaNWF_-qqoHVdWcP9h@c z<0H5`kO+<>sYx|W%=jmI_5#~Gj*7g|T2g1k_`C9Vyx4VxG$eMSpj4$z^4s4{x>ir0 zSVjkv&t=a@L%+|GV{gNy1ylV5{okeWk%^0?k>dlT)+<~DYKnX%BXE$8IAv0?(0_;3SfI@i?D;$xooYPAFae>?yhQRYUM1bDY4y zPLa>KFP6XfqL;v8-Vo{3nh}CUTb@dO*qcdTDslk}*T?W0A7c5tL3pZ-RJLf&$^`T9vHuRLSpZ?uJ*_REGP zlD{DSuRga=%aZkb7zBE3DGn!gpxF^|sO~qVPv(fDQ_%u=dUq>pQoV#*`>PW__Byh1 zX}j1bC*8P9L)y`F_C$83BA{gZcY4?9K>e*@OVC{!gq<&j=pYd&f`f6VQjucf8;cZ#8Bo}SA zPs$_n7pmaDpQnjht{!H52dq`G1p}Kpj@nlP+-s@{p5FOjTDF9~e64}vzRqyqRS=DK zuOj7+{}J8(IGlJ+3?J0or`%*e@-)l^Mt${Yo_RkVOibcb|FMQ`3VfP1`kKDYKMXJ4 zmXHISX55h7!Ki9lkj+2h@NIVh6mK6SV}h!Y|H1|Tjh_ILBd_STfaPTC&Oxe`SVO-K z?qlwjm&1DZA~LjBhkohXN&L2k(~mZKc)Tc(#$CvOF%KeeYuh{we;ZGOWKziWXX~Kk z`G1U>#UbVf@gbHNrL+Gk-gfqknQO-CX3bJ7UZpnBiRLVa5LB$g(X~^bGngwUi!-1Gk(EYXWdU~u7%j7CeI-1O6TY@ zj}6QUe-}ETDwYX7yd4(J7v(f#E7_nui_A;9#jIB<$4?u5nPXXTL}t%_jE`~&wf(n= z>{Z#StWl@6ts9|k>Tmdwwslh6 zrP*tk%h3khkH043?1MMqRDCv#c!~11+!CqYK}`}oz8GFqJx33?4C+TGV|Aq*o_f9r zlkc{Wjip!dR_SW$w33G>a{iFytVJYg*dFeUO0%01YcEUOBlL~CC0=arMK>=8h`iWN zZC#)4h?Z&bcP;gy>TN?lruM+O6E?6eiQ-clNE zR*N#u=3j%+|K3u?;RkRnp`PTci~-H0aQZ5^nk-9=g6l^Wxp&pqF+W7VwYCX`%$Enq z;*;ZW+`tWvM~w}6zw9?1QuYS3mc_8xL<{*B)}gfTcQQ?CD%OrVqD10Ux=*o^EZ)5l zJ$6lm7RyRp^7AI0F;GL@29J|nS%1jjcYk6u(gYoS@uceE1scC}CDuCD(F<`q!7;}ZE8n|`3U4B8yJpNDuocpm9m7~E6;17nMyb`K%dkz%87%YnqT@*mTG^=tXF9)- z8NXYYYwun_KvD#GIrSx@G$RXcOXYxKM>~-E#y2R=n%CrDf1&H!WA6>;Q)6X(*)9nBnflD=D{4~&zjXEB_Lo3FW(2I-9??8G^;`lB8F>~)O9 zTsZ_)cI(LOsu)o{9!qnzSy-N{K$y2>Ft&Os$ZN9n=J7jpLe>gAbM!5Jy6QQ|EXbor zpaP0wK9K`k`XFyKnX}=N3b9yRPRi}g!F=mc{AV7{ObQMM)vHh7?0fl{_d=Lrb{qdoKMk%@8T4JmM=Gr#gQ^t{P}i3UKQe@HxM~N)g(MT}&}+2B zXe+8abFo#FUCx}`L=LShBKDo`u%m#5>Z%^n8zw_1hMb^_W-lQ3;6G|Ic?>6ERx;d+ zI)J+W$$_6@3M{^O4s*YRg1&MM#=l3bJ(y3_ZcZYxe`6p*y%5qb_JZ_88?w%GE7`T_ zJ`MkqN2YrGhOhMzxVP;E?yL_cgHj!IKoAGIR^8-YLMUloa1v*YZG=Pai4Z*V74Z#u zh>v$9LT_^ioY~F8mtr00I5!%C9ra1-55%KX6YP2ypkL%X`XQ(Rrzz~kyYeTY=Zyq+ zX-_I%Pcg;-LsdLqVoke`HPYNcSyuStDU5Q8V3GP|{5rB7l6unU@GE_4;*$g^1|8V7 zHSXe^CY#}lza!oTFZ7$Ze|ls21es<5KtY~paT*dBC!SpvD`(-Apxs58MA-leMJfcHYGdZdYS=~O8h z<$k1c-&WH4<-K$`Pqf)OE*)2I_kj5|9q{J<2|9bo1R|a#VqJ$bTsL~be3r{4hVQ+x z%C`VznQL@Fdzr|t>K5nVNDNxed~4Gz=E{lcR;E?P`c$Ur94tN~#{K+X65fA*7O7|! zl@_JS7VZ24=}xiaH|G)Z^tPglg%Z(=4nFm_w7X+)SR1UarHaS>gDu zCLJ4n6Yy!!FS07z8J0iFgziakm5RTBe!LBYCTi9{`|lw3{VU{9pIUN!%`RdW zkV-b&%HYI#Ju@+@NWFIf|PdSfBn)}9DU zJBh)gWALdj21A~&1ozGicr?;${q(;^vQM#rtaANHck+gbPx%3qRItX~;sR=-`JSwu zc?h~+nR8AkS<;*uaT-$bg-XQC;YbCxGw|scgr5(lN7nA+bnoAVX`hba)|PRgMZ4&I z1rxj$wG6(bEI@q?Z_ZBpb!5ZJU<`T?2c0V>fz-liypwhqCQ6N~+wmY8EQXsvY0Wz3 zvep{4p20O zPI2MVzN}ZIoN%{aZM-gOa$e%15Mu&cV!sKAcbXS+sQXB)l4+0`iYbI3_k%u(g_j zjNw{3o~}XX&T{JMP)>i|h@gg=CRjQDG?4Xv=xfCRe$a99FTjXgbS>tTZHWhEi)K=m zHIE$J^P+D0&srGtIZp1CFU0QjLC)QAFNlS{U;TbB2@<&cCdcrd7Th}8%!H1OWCqsH z#u|kORI~XF-IhGU2`XMrBz%{HY^^4IdpZmKr^!)eQ&&7!zmOB4egJO;cGC4}V!8CSUOZ<}=WV)Y zml{N6IFg||y>!-Xdyr}F5^1$Zp{;Ep)WzvRvQ!mb8R~>3M|Ft(@>TS1k&x8gmB3vI z?e!6>?I89>5wTslke=B*jo!N4MxVc|rlp3-xG_2cudiq#=dzU1sb7OmAIT=NQQzrH z`@?kMUQfEhc#tYS>Y;)SYp81FG0e(XiCd-gH2!-v7VX91b(e2X1l^*<^E)PVr?49OECvWy+#a9jD{Ed7@9hz znd&#Y;h)rHbWHU&T49%tS&jaVdGnMeW__EH1kU(Ss?63~3H zjcO*!V(nlH=Tm0^v()G%v+Q>kC(m$($SmtLN9UFuO)d zH$)$oDSjsb8&{L{|Efu3jxLT9^8$UC5BF2Q(8IiIL|Lhd7EcTyb%hw^pko2`QSrWH&SOA0TZ6aAyc9RE&Ci{<(cc`lR+Yy?Z_m7Ny<=IG#xl% zk@SbuG?4ij$MnD9;E_Y$=s%lmq9<0#y!8#YX>k_A1M{Ea<0e1q?tB}!7jCC_o6>1f z^l=LJDzL;|35P9vv8#F_H7c$Hh5Lu71}z1rje)RAJPUPv6|ti>7dwh&$r#NBTOGw9!SSXdZ zh7aEr&|o+h7|Akt^)Q#7a+QTke)XKG_dA%Xhd=1HkT%@mHN@17-o{&pR4}?`52-YJ z#<|*L1o2nB$?DU~VeH&?dS1Gl6DZRCT7+ickX1u(M^=H*xQH>H^@M&eXrRXzU!x}v zO~xXllVHSbrn3zsxC5a#;D>xYGqR_c?2LOtT9$M%rY06dHu)iuR(U~}ot37(4vrZ6 zBns^{?%^NV0!EdjW67ftDkZPR-u@AeId9F#pR&1}!wqY(A%F*?%`z}ER~x0WPBKXY zA|@d=1oo6!kcW#dL&lpr(63nw5AD0j_wI7g*kMnubr)mmt|FR#Gap}cc%VVpO_W(` z4zZkkSmmpV3g+V>WyF_xE9%4Xs02-y&LEOySs-)$4#-}yL!!NbScR&AegYrk9*28Bq9~Om02D z51MtlaA7Ed4$7ueV~f|+-1r4Kc-I*6n(|QdFoQz5W2AV+PE2_kf}{3{n8b`S+at5E z#xV_5(#AvE6@VabcXS%N2|p?ah&X~^`uDySs`h2q_q501uX7!=)V2~=UNzwCdp`?j z4YyFg^lrF6-x7B$PO9HpR0_J*F>rRI5HGpTzya-AW|_GmKJCB9)TS1rl-quMdG;7` zmK=b$Z!BTfo-mw0cOJPX?|`M#w&9qsSC~(?r;yEw>9EPslN9{*B=-4|IJ!O*JVe_4 z(B_HsK>Hbv;k<>A6W|8(_O+4S@4wOD7C8~Su@8Nc66>ou5jYsW4CW7Pfy}w4m~+dY zCVbd|d%_BE;IA4_XT19SF{-t`JV^Q_2DgIqF56@gPu<|c|hxbz_F_#Tp zajO$c)=fDB<3zLZ>>fKPyn2;$E#?%-UZ#Og17!%$^1*DiXjoD%0YSuu{*m)Xse3;- zt%0+^?sFCtZjYx@sjqR$m>2Y6ekiV0y-18i-z~wZDmv-;U7DVDg6Q_1WGa{0!mdRR zp}a*0^aGozB)x~8Kbmlj=U0;H{nX||$X@uVB}3*tUW{La@9Eli0@|n-LtFP{5N`Y_ zm_G8EOy2JgOVuxNtjEf+z3tawCHs{$&0R&dbnc_q-KOJBjiU@Vb30M@ZUVCrPg3;w z6eLEv!|3OBI%}Y3`+ZEp~DM8;OJ2}Op zGjXd%BI%WS#96#V3FBP8!O?47%+tb2aB~Y~#+BrP;HhZFm-QKrY8*f#p8}gTxyn$Q}$Kwv#oOwO<~SkUw$Mw7MQP`4`m5cjiJo zPsUm+N*BLokE7=u*5g*melXv1nO>V*1_|4*6O#!`(Z{C-y*XX5eZqB|9hgV<)W=g$ zFa_^3DQGjV91eWu;X}_eN-2zmg%0EXvydiVy^j|}kMq9WhCb>%1&ZV|j$a!Jr@KW^C$E6B+QJya zIjYbeDB_6rc9T6f6VNGsDTI1*P{+FmPMl4~RbL(9@6;H&e#~VML|-Nf>waQT#5;)9 zzlor?6&4C|!QjXRMq_#k_TjYU{0y>KR0H?ti0=GgeVp&0 zQ$JeO$jEJ5LM)pyA)xd%QF^izUmT7C?X*zX_TvS(UP)xEn=fK_X(1#`J_8ozDqLj~ zH7KwX=PuD*Po7uC)AyWCTCyyTvwgxOuAja+;rWQcjj3NmwJOC$E&do@C=0iL-@z^w zJ*pFU0G2Nkl?w4)jPSHI`mUY_IWN2E`J}hx;%`N+zV`z-GA@RBv_y{kQcy{jsD6QS zxnXefR3F^ZR_0cm*Mk!EUnDU32?$rbCgx3lNkGwCoRscI2em92MXoFzZHU3j7D<## zASibGNqx_oS;*{`#rHq|)+^uiB&TgAz$__q<1qV`Rz+#BuPhQ6krf5r9D9qlXL;f4t|ug0-3-JSJ!>-u^Z&J7K~o{nlWu+>1amLjgj5tHMU7cyPC>#B>y^fl@f+*t+4o5}*~o*A$xS5UCABzMwSk^Asl@cfC-9!L z0IyW7z{a;B^z6-STsa|@s5#GsHkD%>xx?mAdLas}l$N0KoWpcjO$smkRVCtwyU65# z_x1bztD$)PIoJ>*%{86hUC(O&hU07ZLDYwP@a)$Hc!s4QV{!rTrYrfdMjD1I%HX1@ z2^=cyg8-k4&^c3)TT?lcRLG}*=H?x==+pom`{o^*88@KOx*s%h<}`Lpv@O{jCeIoc zn_$DGEKH9VF(HpT;nH4p@IIu>{rlhtU67>1w&pPO@Wbnv7TW@<-w%QE9y6?rybQZm zwBd2nb}%|IQ`8%rC0X;{Ss8Db3*~KHoI}S3IR!(v;IMKnx@(E<>m@5-_q%S`kx>ad zZhGM3(KjH$HK1$np2yoKTZxKlEj?wkfH7+POD=pJA(4$<9F@y6$XDyHkS!KRzh7*C zrqnK!kvvah_kW-x@sb#;ltSP2tfpUj!^n%`GO+P_f)@(K*rQ_EBFDZOyk@-^dqQqu zN^l~vRcXbFk6P@zz{T}Vk51DAv8{~E+%c^7>ltumiyOJcYC%ie4>atGuJ1kZfwt^@ z1!4I-B6(vxOxYk#wpw*z+D-)`XxoA25%=h2D>u;)HVMY6ilN#)LU=FqX-q~LWcy1) zxyCXaoPP>JvZB%9Z!_sgiHC?Sg%BDWOgj%sGwKIsgK^awvV{KyWO*9sztw~!@o(DB^{ho_t(m2-B0Zc!6=JOVpG%Nsy9Cw_ zB;erj9J**`4^e%~fk%Q@Xmw1~8IFyHg4sGyq<;#1HMzKOl`$$R{s(uyM8W9hO1ylv zon93_rHOm{>G5d^IO6aZyFS&_D|hMR{qq;$Xsrc!9=pORh{%AJq$Jw^mj^3-Js~6f z60wimk9T&NbHx8LFf%8C2AzA1 zmdF3Wv!##FxyAy%M}*UtO|_)xT^uF^)S+~O7`!O81EY&VT%9oyj1n$zG$&qxyB~&O zNr^Wuy5T||C-Fsyus9qEyTW{5F&<% zMR}U-XHLB1GN8i!38uvN!gK9>YH?~Ss;L^_s+qq?&iZzIFwGOHzFWfBFTQyCLoo(y z(FTvny5#EMd=UG3pWZ#VhGfWopjKBp=&_#{fwLzNEoD_$Cv69KU_KejzNa&Wc06dD z?~70Ul*y;mt@y~-0#oS#btyBS)Rc&ZVWZ= zUJO{R4ncJp=xmyU$1Hj&SG%Hrc)LKU#6=8Q3PWgw1Uro$FN4 z^s5wmW0)5#4TWi>-qT@p(!KIi}@Knh&qU{r97R zoUwqG@@f>Twu9Z8XKCr~1{m|kk)tIR4J#hmBNp;eq&mTxrz&i||4Xtj4KPJt3Ug*y z;=qzm=-jRjInwrIux|!_wpxNIGn+_(T@gvHoD!MSo z@wXR;Sm~6I@tO^^rE?AWoh$>74=aMFV;-5gppZt#r80L#wR_U_GdNMK3{>xX(HfI+ zmxDWnE@K)ay_%Tj~sdYGUy5>(y{=q8N>y_n1Ce^8&0amlL~jnw+oC0-+C1 zqxZsA;L*G!)Dg3Gan>(90SnD^G03`?c{X|PHkNuO-qX{6VE zRQABR_LlIbbq3AquO*QIPnk)EU+5a6$K;Hs52$?8fIin+3~E$`!JbBXS8+1Bd#b`o zat;E`$I&gKy6ARN4j$a`f|{&WfZ*4M;KKqD=b!(W_$RYePXjq-U9GT(IHQ8{YdZh_ zCytbw2EL53WXexXr0!1Ba9Vfs;>y#$M%50~rrQMNTwVG_p6asHy0%``| zApgd0gT{~@6JEvpF6MVYvLxq#2&Sz=tN46o)l ziE1Sid}CEg3?w*^SrZAn^eRZrfC*k$wvsr9YNChLN-*m%CA&NA;b7ARa^JH8;qh`1 zx!vNU?hKCD?<6$&WQ#jpMFAB@cZ2iERQlk8h*vx#MT1NWVA}H>k{>ie>${J`pTDok zU6&T@Ju?o>C(Z%+lqAS$uz_B>hTd7c7Vd~Rn+3KrK>i%kKeIk^R^$}Ga`QLjeAoic z%eEe7-{QaIN^t>MdQ+M%Q;9*1hA_;`86f`}r@(^z1NbxfGc_5}6!jqs@n>Qw9SE(& zYG(%A%5~_}J6dpd_ilLK8I0n~l&RG7$9VfqEV(6m@0ZlA!s+R@+~a?4*Y8xBOI&lb zxY2x5_PKfqdK$-}#25)S-zW^1KC{5dlk;(8WeFO_PT&SDUPp2E8PYvLj=N>X5R8^M zpz+&C@+0paM4t&~c<1NA(lcjBbk1b_Y+=pq`K*WzBhjG$Oq(k)D8+jGl>yHS(VWf* z3G_YrlxA+6itk^#(R{ffDs3*!9aHLwUE{6c)ZSsRebY*A^<0AKtrRE9)N+h6s-W1& z2O4X;sfr&L6vI{Fq6|y7uI__L8#JKhb17tMTcC~jMfA&*263q(e6lT>p1a;h6QvJ> z!6QR1({lzU>=lPE+%!mC5(^eK1irr=2MRNcA;e}WJHPKK{klrD!*qNM`(xb`kmm~^ z>zyizoAHz=@+F}0&pHy7{~Xj(!^rdnn%wsqiS*UnGsOE=B1CQFu$46_U|boE>aSJV z4%;}Wx;F{-{9B2m@AN?Vq%U4PVGQb5?~q}M8r0n;$qilphK3y6NQ`~Y!1K;w+~5Ut zFlP(re7-ooi*w_0&on^W=~S>dq|WtP^dIDRIl|n7l;(BqqpcVCT&oYGr0A9`cSrhj zGCnmJR(u&Ia&Io;7xx_y_L&E3j6O1xeyoLe;p!ywP%f3q%!SGve^|X@BNgk91z+Dt znCA16+Aj^Ien7ZKcRV7izfXjJj+F zjk)P(YROF}PgKg`d6gb{;id>`=BdDQAEeAOFEro%j|tG;2PvH6IPYC1yy=+_Wm*Gx zGE)G@b&i0Mf+_~I&IT1wPZ_Wd#o}mX zK8kYLH}Hk@d#EuE#h9?^@VeR?@-AGZXRae~ldppC<||YUDWXlUr_h(G4C#t;Wd?{D z*ti;?(s_AMesKxO&JAOfqQRtmWa_7=Sn@7<^F6wf~~dIFipgM?VmLT8mqK|&#=exo zn`Phejbu27D~y5qhv)G0kult{$K>grZ?Ev`*`tu3;z0eZH=)6ZE>!H7<=Q*fpfR7z z(ZBu^Xns3hxAs>SHuVp~8TZMo?MEK*`Xpjzx7)BjcAa3jyUWJ@jRos}nE~0FK0@u4 zm>&U|G)Qv-d>JTa+~>N}kM>^p$NfL>8T^P#WG9g?X8pjLw!*)Jm-t(^f~L-kg9Tl` zaO~kR+yc=mkk73_RGPVk_KIyG5h;JE^uYz3tzBa1D1&U4Y;eJRZsbAheM*eZx z4eNEe=Y!Lzx3>lRy!k#kGW#RU8ucI^-p(lLt;+3B9Ke9YVNPm>EO%p1D@;1=i&j&< ziZr+z(9#$KCEvu*=2aLYDcapGPc=chtQL-C9HT*{doX3Ah>z-1;MRDi(18%J)yetszNPd+~<)M^BH#hSY1Y>#rY%OwJ;;ZuZ>s zeFtF2?=bScdNE82%z=pbDy#xkuK(mU^w5uCd}(Zk!yc;8;`NR>WiP6O#vg?3C0e*R zTm>rHi=o^~jl0BO6Pq7g;M`3ZfeF4Mtx3cYjf<&dlCL*Hn!Xt8I$N9zU$>GWcLpaO zu7PXj!}#K0INkJFo_iw29XzJ3!=2xb5{Fl}!E^~iP1+dt`nq%?c5n>Wt3DEnvX(N> zMCYwZcO#a}ygu=}C0F#l4}3)eQzTJS?*r4Q{|!#= zj-=D%F46n(Gtf%j82oFe!@jkxL~XkwKKPu7DuYAJ%KeLhv+p)~Z2t#q=ycliK#E;= zm&e71dqgfU1^Sal>IuKz#`EbSjCan)Bb#CwvwJ*lPuz5fdKUl{?;ns%UOc{9ynv1_ zKS53kH-#>Gg4*QO=_1Iu?-e;!L6c?M*cKnpN{t9$Lw1CYTrj^J6(dg zt1m&fxdPYHTOLm6IAHRpIjqCrGtQQKB2E4LQ95n&Fj+_Q=r}V|{FZ77X@lt~H)0N- zl&+wW*$6RHwBlZJlmjMj8K~@-OfQ*gawjg{fF3h~@U(#(EaNSMBRkKK{$DICT6qUw z#6QKE%KOoK^Z@AWaRI+}aqdn7d2YAUdZ(HufzRBwX_tIi5I&-Q4?z z%BRSn#MOfyvSo6ACe8K_@14b)Zh6 z1f_4K(Dlzsu2$U!(A3bzxurjRA$#~+Q)3t6){^~ zTsErm6J^d9qD+P~<=$3c_Gqmp)oxk!K0AV#`&}G%R$vUw5wU0sBI|Kxn<@0KpG*UL zVySxHcre&+%RRWx3=&S9fIZQ>NO-|SHgt0$E)HA9ba_n2L{6FL8&5$!GnjtKu;k=E zx`sM^AMio`UGiEcof=Ahfq=kV*mO7+ev2PP{T-FqQztr;%m0ui>l4Y!h{deGk}Rt+ z_O8taS9{vbGvdZhF9OBClZl5&Ln@mxo2wW39OR70V4jREMAY}8!X9}P%O7RJ`sB#` z`4(*6{u^|gnkMe`(t%{59GsIaB1?v4NXZNrJoMiKm@lEhI=9RQcLSE3R#InW#y%i+ z-3PHjJ_nmVkH?Klkq}c~h<4v79&nw?24DL~`d?9cGev{_c2kn2v%cV&oieQ3typ{^ zC*t@FUqJMlFp!-u$vRJrrbQabkYf3ae7=Lwqkfuhn9>GGZ4+=>)NAWtQJ;plA%DIC2weu5G_E!IC_9Bc9#bUp+sL=+|7pjKNw`W&%g1s3j1r(; z$B{k$B%SEi<>GB#6|TD^+M$ShD60L&avcsv5bnRJ?2Wa{NTK&nR6pv6(_h) zllf8j@$Xkuh?>VWN)&^i$L>SJJatlKe*+fvSd-ry#Gzu>vU-<0`eaydjm@)LE1>Ca z7COuD+4x3JqP6Zca%TJB$X*9-+rEX+oO%gvvRdf>Km4+*vw z6KU5^NZ-IHxX&_W|MnNqh$90SwD%+(9WI5Q>^ySnW;&VrrkL#iZVuH(xv<;GZv@%mZA8;_0$80rOLi=>XA8yr@Rz&^8~a|1eYLZhIu&?BrYbwb_EyC%F0MMbh+DjMXjG zq$`eyG~Bzv?AYOYFgvNuN`8@I=gJ>KN8_E4ykb9DWH^ORU+cwA@7M^lUhC39y+0UH zq{Y@P5$#2ckzuu0IH2b9Aa;Jk59r=#!!BzvWaXt+WA5Wvbh$6$^-6iT`{8F;La)Q& zkh4%-vx%&-IYbq$DnZ-pKJ9%`fJ3j{Sef*G^4mv|74VC2yXrjd-t-i7Z+Qxjn!;cy zW5(_rKbf1nNsL{kKFZvE^8pV$T}WK}R^vi%9}&0X&Xr5_!B0-fFr3jxynQZ#qqQP? z>)&)VxiK48O!Vb0^BIr3=Gbs`pXb3>A9dIyTCmhRxfS&11i?UoC)< zAnGk#_V=v)aLz3Z7iyo!3H$2E?*k0V{81F`)U`nPwq`2zv>dLBxX97d7qPbPDn8hi z2e!|3@MTvaT`H!>+Lgspr_IM`yp<#_UNfD0J1?4kYRU$E`3taXr6JggiD6H>8Zhli z@JH_{?sM7*uhjdgq|aAuYtrMsx_U_T->ATWI}xxlS&Q|$v<8MZIU$%_!+1Y$uFS$m zXv?dCp(S^~cJE&FE)Sz&-yYJd*(IFnVJq(T(usIDE*6CDYFt5v0L{H$V92=LpmTzd z>AczO)4&IGZ@50Ts`jD(gH`MT(MCeF>Mm@)J4DBx`oS1}u_k?+fa7)vzi@u}2KaFT4j2 zl~%Fd7h8ebUWb38v{^H+G+@NDXys>H$e$C!S@%_eb(yNg*)t;#f2zo`oe}f6U8!bh z(CPr*J59J%K2Kq;UpKmmcCTgPjsZ8@6+4D&nUD8t$bAh&%CtLrxpBu6VzIl3H2 z#%;!{K?d0GAx)<1m0?*X0|h1`FM;BIGDhU9Fi~tgcifCr{J8uq?Ee_vBx@(f#aLA)TQen zx@%lT)vGHo*(H^;Ju8A*1QcK=XraPFfc(b?iR@c-?np!%o^-to2S>!X$8yi%EB`cH z7&%CcZhj>;BcAkC>2h?M5CJo4YT&}IF1T1Y9?yAMLOMutUDO((BXco)&&nb;XI9ep zo>q`H{WJOKI+huk7J=`@x0CpvP0)NG7~d)$!3|I5u+*H!Lh+$B+OUN{&7((eS z8QSbI6OD8hz`v$XWZ|Y^j zKzCMJ!=V|?bgD@gQ~t>ab}V#;?y7OPQ@5V@aK=(m@({W6Mwv*=D!37Gbl9tXeKUV@0!rgd<>VKAurk7=x3W$IbusL=-9RB%!MNDrY zY2j}nQa2vo|8<1NBEOLcuN2(4eG`s6@TaX~;z&W_TgJJ*0nR<1faXVEppvf|4Naec zui`jpDv+cn>s!d5md~8~V&{lfJ50oPNq!( zetdR}e%!wdsI3dmeDBV2KXQV!pK2qge+1$N-g3CSr;qIJUJ2^Xzv#2kdVD_RD3f{U z1SiFIHH?XFqIPx?94TdgPTBnBXlfl_w{BGw(XHQscg(&q2ez&y|4s<#<0%5-`XLO9 z(d){cRj+6Z zF4@jm_EQJ=)qGeP_>9xLVJl_GR$9JR9=BYJ<$P-n?+7of=;2vxWJEdjy}ZroVM3CgB)IdjNI3XCh@bYt zk{9(phd1baLGUJdl=tsRq(J0SV7J~_iQgQMD-e3y3;&9r;6Lx@3Fc-t2oL-2;otn} zZ+q5%oN%yotf1sj9KU^G4u7TkS>fi_J+?u9hxs+uLp(MxuEFOB5x!X9&esie6aK5J z6ps1UD+p_suroctw;TJ?!`2{T4PSGHB7ZoqLHKu0kZ`OUSGZ)G5uaZgEu3d)EmU6= z#@Dod#Ls)^Crpbk;aUH6ja|_!oM*t}t)3spKN+dd*IHvM(Drc^ zDCwW#^@vggdRxv2gKGm9+==2-sj;U+<2_&R}}TB~h(O^8q;=e%&pd=kHE_iNsr!Iy&M zDQ?1&?3uRF31Pgm%TG%mwpDGdt8VLl}y(@)>)ztVF(l-2g z2fo;@yVt?HBr7f49@ERebSXj*tCGh0%j!0KFe%{+r(4@)w0ZNt_8ky1ll&U2Ut|l% zwVMn5zWMQwjO^sw>|G-qdDGTlye*6W*yNzl*7c(>GJl0|ukmSKt@6W`Ut z`<+=O^j$l|w_LY{|9aX8&-KfD{?h4J!zLVm;|q0`y7g8HEyQ1n@m@2`Er_V}V`;g6wF+ugxw!iWW)g8yiq za7LcHz+f5j4!CmoXB51I9oMsXPujxxN;{YENADjN@b6CLw|>wNp79#Tx7N%QtXfqm z93_8*!qhz$GR@%c`Kdd3w0HRrpqq#-tANrv_6U9hpsvi>l|F@G|@X8c~^TO(DW8;KM2LL@&s)4X9P=MV2d z;d$YO!76@RToi8{*CAg@esquaZx6;jGMRK0 zFA+?H@7Y6t;Jj50$kKNJ=aKhB$MY4Z-}@-u5e1ww-a&T!*FX-tSYe2rq~Q35Dj54O z6NWNRqt^W)R0BgY`+?;WrfvVR{NErZTaGmzDPhu<7wE~x!5O5KDr z!Yd+Y!DyEzHg_h`gZ>t9OLGM@+2o;_cm{d=YZ-3$H2@nYKN$I@i0uJ$QRMeN+TF*c z&wS?N+((m8t+kgFzm4Ubv{B(k=P$sT&>LGyj?Mh?C|3wF#)@E4+(GMV&Y_V5=fqVpkb;=F`|6c7^WA7V>>veBn$| zDF$DSfH4(wjEHQ*%+zq)`aK(iHk9CO_7uEdY{5)@;DYBidf_F}t@PTjURu$z6djnK zq;s-1qyO}VDLjCK10C?v`8ip;_dTjE(#FT5dpO>Hi6|d2ldUYe3h9(@ zHAH#PHv8uy@)9aLvx6xeZ0wz3~?KNJL|ymJ!}A)j?Cs8=!6L zPQH$vq>gqc1Nf?HLwr~8K zk0z1V4=;jaXCAcp-=g>Qy+Ot@59|gs1x(#6uKb+6^sYu9Y5%vMxRq2g2R_^4iHui7 z@}&)k1-K*U+iHA~-~ta8=F;@&M|juiG-S&DqUF-3AfC0EePfmeKdP9Evc?e{>MUee zwZEr@Z+*zy>FqGS)d}z1DTchM7a)F18-CMY15f&<3gnb>K_OL6&@E?+bML4!nQ@Cb z*Y0yMxN3}ivd@Rn3rhGXw3qJZyg;YcS@^4eGE_&;!1yc|j0rTO{tfC((;*>$_k9Px z{St~*b(Q4($TBJuu>-2sXX5dPb700vAcP6+cTw*na^iN z&1^ZfR+;#tCJigM_LBF9ucE725cazklSykjP|5Kv*1bB%c{vn-=kJB$-sVcoTssAL zPW=$9qr>%9G$%wj?sSCs zcLRKP_e1cxdIH@4wVnR{I~}#}T9WJzd78007-s005Ya9!6ed+*zv^9RlPDs%me5Q7 zW8`+-Q+OwPlYeu?3%v0BGLJSsW~8)Gi;FUT*(U01!KbcP;xal}I0E-gcRiqgr; z-TT;wTuNZmjdfHqauq)4;6VNUd;E{%AtZfJ1O|WA;?J~f6bpaC|1?t){@O@##e}h5 z=T;N=eq<+{7`;O0?M)`dhX0Z5@)lS<==i4y4nY`7iiANzEv0Gyor3ZlbSg6viHNp{`p_(bmdys2s14WvRJn zv&RFk2)P$m<1&d`qc&VyKN~GtXVH)Pt7y^BdHCUr2i+erm&~@$VGb9Xz=D@AiRq8i zc;29#Hh=QRKgq)94WsxA!}76RssVT3ufrytWayTi1b^fvW8I~0yb@xF&YIte$YguY ztcq~np~p9=V)PW8vZx9FY;U0@nve%c%%-|P*jZHUFV-+)eDA^1pdn4KnMN3nXX(07jn zJyw>$h$l}JaHetTiroTG4;$jFWR&o2ZL)BG1mZ`93N$IcMwU86lGU@;lc2r=sN@t9 z=hT^aU@;qme?4pdnzI+YMkr^ccDg;p74(xI+3srB(|M2F!q+ulACF zUJ>l_D}l9pg0VQ`72e4U1(_xhZpwo=(AC(8H>}PP!8t9It38MZVsrRTy*k`NCoOzZ zI1`rV7xSy;RWYj^*MNOuHE#cPf}^y8VETD+{E>c(elp3zS(j#ms&h0H=10>gmoV7< zT$Lp5Y=&bglH9Fra=0zVhrBRcPwyMN!`-`<;J4Cw)Q_7DMcb<|$$2)$R(nFp{j<1Z z>ScO8Yn+JgJ%@RUrO?xQ94DH(LspLlH*@|Zm}8Jb0!#nU>vuBA`3`wOdYu7)Z->4uJhZtxPVRW?z>QT2Am*5j|0xLhx#CUa`^9rG zC4MK=-BrM|Hyt6Z_%dHz$Zi;^U5Q3Q*3^Ts_vA1qnQ9%Jfc&IfvO8cA9Y`<47w;Ph za!zwTyDJF_bzfj_pOCeF)`L2AP2?&r`N}8OQ*e+G;FjM%Vf<$i9&DBo@>7x__~}D_ ztI|}kj$lCPgEQur>eE40PY4lFgz|QzZ8APMt*MOO*6Sx6mC0n;z&B3y-w@(4uMf$? zd;Ad5btGFV8*^s&(Zrv>Xv0G#fYB(haZALQ2Ss$fTNe#HWQ_erA8Fb=VZ4}m9~b(r zfMWfZly~<8Zua{?t|qU=mDm4K-CN4A-)#__zS*PO+o$w+PY{M~2?ys^Q9-@MU0M^` zM%Nin;e66MinG$xz-{VdP@i`jw(4i$af^xYpMM7SBsbxSI468U2zl%`74N*fLKAn( z5PAJ6ShV~DEiONSa(6t?$k7ko4uYKz2uFa zIg;c|^7(@{&MoMr)t5|A%J&-aGIqfy{Wd6>5s9}aWD^Z_K7jx!PHtxctbFH&1*yM5 zW}T1`)-^;88qA?^MJd!CIRVm(&tsm6B-W}Y;l}AAFrnHMDihM+VoE33Wt}bLs*Aza zWL2^|?H1?M{#5GwEF4ui7J`R4b>MaP6jW-yCcZZJsD&>N+|UR8E?j1-uBs*D4I^a2 zb6x03+`*CXX~nnKBFV(*+E`n?1HJy1K~3`?ns)I%-SR7guFUbrJ>B+Xec^8Y-&TLT zeqb#~8)iaZp+2q&h~k|RYOM0|*EpU&XE0cW1M4oYgBrU$>i%RYu6=L}xN9HqtGeFN z*Nawzro&oP*6_sr7zobNsgUq(j7Z@U$CUJ%i9z$Q$U=ErkS~3Tk7}$S z;#UkTc`pV%9Wl6kUIvtyt-}vr+v(JTE(lv(Ndsh)NcIk0uBXf>mE&k&XTSuUSFnkn z`=SgDXY}EcJ4-NA-47HNwL{NAX4W96d-dKR9hEB||r?b}QFitz@h?E9D* zVnvg7!2;Cq>BVz0d33+;Pn7G4rSjv~V0)}Rwv1oL*D?3NUQrBApRa@UJ4J0`8YRH) z^ii1lSO914p9iU=1Gr2incBbAgN0^uxRtWSq@|*SF5zxw=JgiRejzGscaS&vFJuyq z4BX<6EXs$&W)a{l8B0B8oF`sAF|gBVCta_)hLnT`^TS5O1;fW0aKh6C7)ayEXJc2g zyF4BIowpEJWq?Hc8Ok|%lh8IN{@vf#_#cdFP)e_x({bc1EWDe74KGX3d2$m8FK7o_ z-`}8=*GrPz_S4z|XZYn=i2a!dpd7xFneQEN_$Z|+<&( m-U8^b*vktK%8P5xU*q zjsG|PBdz{*g*ps9LaqFD@M@+z5pPfBryFj;ovQ#0w|7xB&Uz@?sf1QuaoExHmhU}F z2YmxFh!;;3<~dWGJog%1zQ2RnVA_W9S0~`_&t4F2KLw9T*1+1*S>!>`3-Wl|7m&@n zM{d~6CuV1gu*ytRu;8^l-r+naWw-*v`bA*A`x2;dy+CVRP7~FC3t>iL! z?~pHLf)`&eAPKCeET_yJK=M*?%HV?o1r4b(LHfhQbN z_-}a-uvsJ;OoAvgWMXK}RTb~z45oxy(HL5N*b0r4Dnyy$F=p^phH z%v+2PU$}$gjis>CbuNnfte_RoWKusPD8)6nCWlSy{K>oxqr~LWeOPlZ3|wTR$b|lr z_+i~;oMihP3!NxDTyY;3Ol;+kd~nAn+hTE*;v7(@7$>v0d?j|rl0ox+Dd*Pj3Rv~A zf|Qk6Ky*u*uLXt0#?*hgH}vM` zfuxo(>aD&itREZ&cYR0LazcZ2r5Xs@4cmy?mo>oKE6bfCYlVYRnRk80KNxFSIh^7pvGKeJV2|G!|^ z@aO_v=W2+*PXS!623X)(KsNi%LNWc5B+0uN6lQU0$=ODX-g=h4*8fD4`?Jn@?8{(**K27wbA1ZBnO_4BK_@IPrc?-~ zhvu@Cgd948aj%4&!aZ3y(X5Vc+bc#lBv}dfR(Hsb*&w=?Y%;GrhlO`Wcm0=qaFuin>bRIsW23Kfd1UqK=w%$fbRJWqO<2HDAy*# ztpDm^e%=St_e}!Szg#0>P9tELcZlJhttPeMwKP&*M0kI#N6R%b;1gUAtu6j2`)Vp# zu;d@vR%4IbTC(^(vx4w~>^NR9eS_r(#c-a88pekdgL&W#m}wJ3A7{G)Q?!Rp|6ESa zcr~!j9_a(`m$UJ3s}b4b@*jkrv7=qPW(h8593Uy{#krz-@5tLHLBM>uK$Cno!|a`I zP_$VGjy6w&+ML;N|NJie6||ao+!N+`=ATL71H!c*wuiXC@pQ?lBOvmM&F`v-hXw!2 zaod9ia?fQqWNefWtPnHD?pgOh!gCy7!AZ=1X@s{vTSH@V8gr7(fj-Hba3g6p7zJy= z)k~RpOZFD^Fw5nH7fgjXHw`RH?bk}n(n8p>q`M#1-MqHw8#ZP!QIu){@sva{ThyzsZH6I+}LP z3*Pst!6y+#R6Okm&QbU1y_^bk{&X59epDCo$l5^R=_I^yKMRN6r{l`mH;IeYF9x68 z#Ke=?RGQ^Z+SZ+hX+j-n^2QMK#GB}}l+vRo8|aP0n(!xL7z=N**@dt4Xv%{;$Q!>x zjlEu=N56pHRH?$`JmgrQsbIBOBC^l96Wp>+s3+Az*6FL{@VF%Qbg|L6FN@xo#3GCL z?Iw*pW#-z)G+aNijMJXiLUcIC(6UiALu~pG=bLs zDG+QtPQ+pwiRFw?u-8kYAG84)$TS>c=fahvF(BiW2-93*V83Gol|Bk+urZK1P;#19 z={lt0gx`nQDj>hsItH^GZTe#7CE|r%^g(AP@FxhPd>}krwbe#(PFcQMo z60+Gs?PhZ?w>DC>#Q8+pArof&JBc>wR-l;l219h0VO{)XP<9eWafj_t{^K!UCv!T^ zJ)tZp6=^3Ix3tg)RFd02l#eGDa^XGwMCNNK65G5JF#IzB-nmo~y@l<#`lJzrB@21b z6I)TDSsJXCFUQ%V+u@^C2xgyMi!-W<$>@eqd|PvfobAu26U8TB$`U>N@0%txRoHS( zQ|s}D?L^>|v2a%|(X9p}dygn-8~ ze}3F+Np6Xdcb8l`1s`bq^g zndFaSkWj}tfrnBn>HAtI^oupaI-5t3v2+-x>ehnah$5#;$QpHcSWVg+I&qjkL>CIV z#|A;qILg~g@TG|ty`I0B{%6(BB*KnfE;U@ZT^oZk;)~JHOd1CpJ&58%2g2*T0!(NJ`7SR7yLaCO z$!+V1SM&?o{zMtSKNzPWen)V6SPt7k{2l+%v5l}g@(j9~bD;L-M3D51z_#_qm}@S^ z{j7c$9h=Tm&W6YI54nn$T6XcbMi-*dw0_dO_%rMr^B3xVrnvgE9Vz;Kg!Z{e)5Ick zkS_>F*YP``e&ZYex2P3b>_~uhHeDpLAdk9kxd1!xA8owE4 zL1INChZUJY+A5z4(ZrSDeMt<{F096N7I)#trAx$ejv*FS7-Ha#F~0Qni#Y$S19a#r zkn%}w82(`ij3!D!RqX+sm8*y4?d8z=JRD66eWCxwU(A-tfdz`^sF>Rk92DQhX)osB zj~r3>7j%M?0c){l_8W4Vdjfw->;v2$37Z^`Lsf_jH?FmU?kL!RA9tODuIy3Dj%X*{ zt|FYQBOLjduMeoYjV(!Nx`0pzcRsB}fFVPb`maZXB z3ffR3QiYu5chIKG8!+_x3D}Y_g}cpN97M7mh_6EtI5%owWSAX3{V<8M&TEwEEiEKj zJ|*~stxn_3H)CnX9e&nRQLe3u4BdCWi@jxj8Cd=)N87$oxS_XrD;u= zr8ruK??gy3knaLk+=z_M<7IYED0?&d{ zV17<6od3Or@F#RrwVY^JUOT{lFO>kFKlG7v#~1a@>s^bg=OvIjhiuTT}U!}#>uaFYPht0IrPn6c3;^WR%b3AlJ zO%U>IFFtzL&VMa$0*8c*XAo32P~KLQULiof%#_A3}XT zt;dVk#F~w6R#J~2WjK7N64*Z%K+^sW(p0Dn=Tx%E*Lz|hcG4Ht-(Nsv&b^`y+;}j1 z+efPlD{*sAKg_pyiO!1eh=1E7lsl=4>yE6U{UTc-C(jybb~;VD{EYNI(q>8p4d{>+ zP0sgtqK-k79q_e6J1m8~MY_&OpX4XXH zM!o^h4+xH0ktlg5oLp}@fc^R7^k5{V?;n(roKY1h^a$h6B!@_hOErBF`_g)tnFd8A3JOd$9OOKf5z82OHCc+nGTY(y#1!xtpM+J{ipct}ooF~vLUXA8yWi61TXpfen*+!PN@35Wk9@%-b-1!*Hul-Bp*J~-B>3wwJX}=^ z%9B5%+x6MFEa@|AZ}J$ODRL7#v((^96dzW8{zLP>TcfDF0p5B$lYgN?2mEIpLqWs| zu*|N4c}q&k_UHG|Gr~ekrTWTU< z)pl|GaOpg##9zfeg;UTt&Vt3a>cGb$hCVG9z@+#0A!bH9x%$`-ZQfD{v?X|@!4-bm zYe1iAG+xwJ!sK0BK-DvvKO;yMbiez;ZiD4?$@vdd?Mfu|JfSM+46Gr!52MjiYZ=}h zjs%0dHN?*5INimo;_Phv2r-tjxbyEG&djlixOT3b;BnOv&P#RzRzxj=TkIdibW5%Iqwg6 zX!962_Wvb|9oA#BMmxv48XS<;J>WA23OCfP* zXuT&2Jq&iz-`#n5a$h4p-#LYf4c9^>&z>g#Hy2#3g&vuIvG_AzfxAmTh_HGJaEF;J z-O~1+v$`ph&i9Z)CBrl#x@Q{n%t@qE7wrelJK_9Wy2)5A^r*eOT8#J4+2BG`2T0vE ziECLT32Qb)!KwXrIQK~pJUzxi8`TPUnvz0A2IRo>^?hLen}jcR2#_=FCTBxUJ_cOc zi$OOJVO{bA^6ZTgYBhJ$i*LVxmf1HN;2lLiYpUYzp;$X^Z?i+j@6KyT1obb)^R`HN!^6HhXK84`BMIF$+RBwjlN!n^%5 zKgD(m#w}O{`Ih&9$PwRN= zHCD$$oACbCJ7kXvoF(v|i4^>2mx!N_j}hL63S1yo2-1!NG~Lvns;;|E29K-(lcS=7 z{AVT*zvvZ8EPg>wtYDE(eaB!$xEyy@Y6CS~_7o3)ucA+uh||)`6znn|k^3*zL5{Z* zA}Y)1(3UGu`GX;eQ{@GPnJ+n`^Od+UqHXB&P7Lqd|As#KGJ+4T`DD=%9hBd25taIm z!*|sg{A)@XB<9_IviPwEU&FEv$~P8+_T%WMHZ8{oRk7IDxCUC}9N<4zBPPvV3r7?tk)ZCUBxY?9_~%c7b}|h1 z47TQDPU4O&}3IZ}LTmx?r479Cf92~9bh*34{Pb_g z2Mn9_RG1e}z-N2%pyR47R+RQ(ub5C7hH(mKmO#Vo4OG5KG-#sRv_HqNr+wg5XN@9I`tnA%SlcyO+Q+ncMP z9W2IOSGOMm9B#q<(0_R2&LpS`O(*VJ7l1YI2e~}c5qEz-1J6xXlkB&XVAbaoI?pBn zzWq4{6|qk_x%&`(S|flzwh#53rr_4m{j_23EdEWgY8al+fgKyAQFdSxzaUH-lf@2T z(FpKPagT2LhA&qSLehs2@O>Z$_0lWJ*)Ot!$=7>f|HBXb zL&1;fA}ML^;#CtURttS@ciL#_o5}D~qm`xfUK$UwZ-ChBZe$G@<49X3oH7Z(om%o_ z;GGrjmI%QQ;^N@{{4HLep~Ag%nL(zqoTC>Nj*qpj(6)yqaQ%TNd7?WHMYBdwPizo_ z{H_Xh&k|0g>pybiT0gKXw!lH{b!gNz3v|0*5qP@}&p*pX7lRBqm-mo9d~HP*G3q3Y z^NZ3O^<;YZHO?x_<8b#E!9R0*U|E0!E*diC%2vH0>EdVc*6;wxFFu4fAEeMX$5(>G zp%6mSZ(v=>4bp$?Iw`4hhAfq52O)sa}n)7^V5A{pLU0ah2mZsQ?&vKT^C+4(xjq>B4d^yrHayJ_d1cP)7j@XRm;QuXDNY ztv}LKg(jTMjU#im{ib1}3W9UKY@Ge050zRsk{jAZWYTJJVmdjQqjyS*HuiV%N0c5C z$(pCMHK7iiJyf83Wg%P+6>97{kLiu+)fwA?P-=r(NCVppsih$c8&4{?!-$u3r+VL7o+6mdI*@m4VCL6VS)Z}DiW&-u8+jHGU0h}!TKKj zjz0-YZ}x#|-gP|wZ8C}W4abz1>&fZ84Pbk(81olb;WZgAqUtr5yI;@_q9)$>ymNqv zikGqNTsMIbPk|$A9GU+fW}vl6D;jmMAoi*xQQX$Y*}d>Rq*u+wfqN1HgBT!?`joa! zJ`YaQhv=Bv1ChG_mN5qrAE>ncHac`6v zZ-x=^f1Em~Mf(M*c9e&kdrvEQJyc5ldZe+yNEfUwRN^dCCBff=hv|OJ9&EmO1NTb0&;fN5 zf#&3+aPB+@92YwyTVDy(e2${@8YzsLwG<{*i$Q>N5^6c0p#23|Fk^uNJRRLdqNmkB ztg|t05jjb8eoX}5FEZG(!5i1a>yndi3~-QIa70EjY2M~j(57nv%RkTH8h_2Cq8k^Y zR&70=b1R@qE_djox;oe!F=|-^$55HF(mhH4h%QjM1u%Xj3zvn9VRN27 zd&Sp0XiM%z?Jsi$oZr@PO1O6u-TPqojU0M-xvbC&cmh{`4W#Z7CLmM06YZVP;hFgL zxNcD%uKE^_+Pl8NnzV8{r%XbSe5ws5bx8_XrXQj5qAH%f{Qxg}XM+8=XP8i_hw3J( z0;)HW2;c@~N{`@CBT=D`Sssjqe&aBW%XF9JQ{tSGfj+0i1o>s* zouZrQ?zgMaN#hS`+8={*Z%m=~@_BgmMgeM%T_s6#6=3DQO3Z4S%IVhTLSt<_jGVed zw9P2qe_BSwBAjWH?-QuY62=K6fi6{Dn!G2*s}roxFOqcB0fk!X-BxcS%wa-nMxX%CVS{P=9hDk&)iQFT}R;uwco z`>g2U;Klr%V;sTUpSjpQ^D&WfSO*UkWa+j2$7x7x3rxJ?M7QMJBT3v$cFd$`oxJ@qr9NlH;!KYa*RxT9S1Q9dPK-ZrYQ+VR8ryuCLg=Ymrsp@M}85c z{8|LY3KCE+Adgs^jF6sQdxV~uAQhUAZ@d$!iwPxHzbXpCCddlX)N)9P%?!b^tA?Cs zN3==p!UKSZCPMg+yYRFGA+i4%Q8XJuH_Lmxi>D>9L4JU=R4+lb+6(weMxX0j!SH>< zMwvCMi0#SL2X}I!Q&&AxT;zeHjeP=eD`N`^Ue$!tZ#uDp|@buMiuI*HwQH9 zTIqSSQYP0rjidZ|CAxJ8&%F;kVt(QleK>xRm}cIkQ4=M6Fho4%74aaU~9rZvhZ>lzwx&N)L0ASxv_CPpKuB+7qzmr?nn}W z$ws2kbdMhWw~oSBV-&rei?*fqOpAIR!NJQYHIzYl3iTx5X9HQ_`irg;&BftGd!YO1 zA9B4)9&T4y;so9m)M7{Bm(SAVXNET@{+ePm5N1?R_++7jTc+X?9bmMN zuorNDQ3*S52n-kHcZ@wJgD(yct7mfsVs|Hzi*L$s-bNq1Y}$-F-)dpwgst$&TN1oq zC!nge5BZ&_i1{0q(MQG;fLXqQbEN1d)mNGhD|5;)al=X2fA%y)20SGB&WAZ^;Y|>5 zag5e>w9=RZQgmYKLwM3+iA2K>8@?q`ZKo&Xcj+Cr=&S;)c1?v&{q68~Ya$if@r>-t zZX;8Yevz@4cR35DT!+^2CThk@$A8`BB&O^n-v5+9<~`m?+9yQdA$Jup4+|rWzL8L9 z?nkp`PJz&wPuP#{E&>6Kpwma?1UTnCyP#KuP7!L0@=8K4pGpbYeLR_d>Zpd0Z!0is zV

    bkVTH%o<=@}_@lXx8%}g6fzuAsf<-s>;$x*t9LXzo+|Y({+EF>3mI;ToWS(`x zap@==UE53_u;ziQ#c%%VzZ;?E7#BkBEy5t-U3w@c2Ca9R0r{yzQnWg0xXMZ_FdgRi z?J$O2!rqlhXOGkD${ege8c)oY??Cg>M!Z*61Ixp7h4H`{{`xZ;}&G^m`M`p2y~HHgt!D zsg)QcBLaVPV+21B6bSw4pWp)PJK7!pOvb~N=_Z|Sj&yPkbkzK!=PtY_hDIyFqvQi4 zxm%IzTRB3;j|I{%DQS?hWgC7I>H=0bWQeHoS`|g#1 z)V@AM_<5G#<-x$q=O$1U$AaP^8QSfA3GM}(f$m3nVI0k)Lz|T#*w+KX&nN%Czk$0e z#va*WuIvroO&I*l3#}(7fptp@4FY3Gf7ykfgg*V%vLb>vu`MKNOo{jps&lWINO0$D zJ5Lt;nSjcL)nM#%n?3J>D8IglO<&ve!EvjDxXV?T6UE3vS^gKPn!mdGtLK zzF7F(1|?8o{{UgD8o>|cNYoeVj;$+Cfmx8U;B*_{%-0Xezf@^K>gtzd<&#{#k-`{0NORvz-wtHXKLI_a(qOce zrMVHrPQeGu)zb08>JB1$c$&~dmI1N{R^fhC1Jt?I2b`MugjKl+E$8__?=@k4?P471 z9g`NMOQq2y{Qxj>jo_OMu7O3T6QTM0WcZh`9qcPbxzh&31^r{s>2QJxt=IU6%P;Go zUdK^*Ka`1qdv=j$ogc}x06+Tc@ki2qCYk1TcVNyS8!T6<2t+q_ao#RlLtfRlLZj;| zqRbA!wHKz)r$<{z+=CISc7Gl;JLd}rB8lQLei5dpr=r(aT^#$XkGFbk1ff;qV3-mI z*SYrCBx7N#R-y{Yz4z&wC-*^e{{S%+#v)@43~)w|gSeg=XqM;!??(%HGGYu_9_ehJ znK#`ND35x%ih?DIONqYzQT`YRGc?^GR83ot%ltgS_rhmreVh+RtN8q=o7T|kR)g9k z3~J~oakI<7W%8ZO;D@!$%6wkz&?TcaUJ!3^fl$+?|^Dgw9ye*9w zuMnQ4DG+&B2JnqBE_2W$u|GHA*U#69+p26vb?GSj4RylFW7)9sm?3c9A0&xWD#*iN z6WD4c1uTgua<4NJN-g`z(?y}MesF?7M!pQJgy--BxfiJPG8${nL?*|cq+a7T`nskdUGd~WJhgwO<>Tc%!&Z8(3_?0AB2=&>s zv(WEY85QwKp_i)%sE>>VKK4_AoC$5XEmcm~^TgvfE8Zqmhn_*C#yR?=P7OTPuO%w; zb>PEwQ`-7F6xbQkNK&TY+l<%zB<1Je^lB=m9#rO*&8?uS>g^nfH=+FNKW=j%yAaJ& zV)53$U9h<~o^!b`3ReUt<1_t>a97qDUw8_&@;Q#Ms3!o2^oIF&jvmAI*{b-b_9A9% zQb0R>F$@>YfV|EEc&$GftXtE_Oo9r1@c4c&e>h=`{yCh__k8k^%r(@5kX~{3)l?jTDiD) zM=ccyXKWlGmb`EAiacK4pT@^|&Ag8eb*x#l&#<-@wF&2qy0S&0T$nXdz{vEtF^Ll$ z*mY-?FxFgG#{PyOGo0wi%iYZp@)3U9j+~X}_0^qcU4D3pXED;jTQX45y!VU@@4Nb_ zZGr9v+f%dRY&ZQ_#S*XG$CJw8@m8DhS+>y|*pvM>GDdz|nX9=L%+AGfjEtlddwT95 zE2=MvwP#uaBgso*AM8BNZk>|Mp402UPYQRiEpe23$uA& zjKf)>m(p01<(1iS;#!P*`c(Gt^66}&{U)r&D1EkPsxfnIhaU6v^E_r_ekA*YSTy6_ zAIr3jY-Y57EMuC!FJtd{yqtZ^brt(x+zFn=3+ujB^+}YBJ-ymD$$wp0Rjxifr@=VcvK~vU3b#*y2C7vlm)UXK(v3 zovFAzjoG>_pI5&&f~lAn$<}R-Vh2Qcv6rk@W;`dzu|4OBv9HY`EFU)m_O5mvW_6by z^F&0MVfg8@S6WE14>kVdZ6Kd{tGnLux|b?5kxklc&Fv;^=S%6VnBQ}m zQ;X)YmGc&})qL|=&$3pqXCHQBUdVYd$3p#?jiVEp{P4+acXe5IN0U4|hwZ}ty55Be z6`Rle3EaojSOu|9#qD73DE`OHeG$gwnS`?@muK_bH8OaLbBcKUQy%PT7uGQ^rQDfq zXQdgP4jpFqVSV@;BgU!4kokJinwg+n$+Ew5n)iMEMqaCuI8(8s zoTa_0h8O>DfS37o3Twk70)!VpXc>*AN$0< z{fwd35k{m*ILj)09}}Z?kp0Bs2>Zy?c=rAC!R${ZdztccvaOkr#S2?vnRS72Z1M6~wsu)E zyM(90c$KKI-3({2y_F8JI4PD)rofEt7de}4DrD&d*<~>5@+aAnRoU$72`AYjA=}w$ z&v!DfGxsoJRy!G0RyC`%xsul@^OSdI*n)R)Ne0iVBb^melEK=fFi|)Yn#HcVGoRVF zW-;@mO`aLbvS4HyOxU>{^Vtg~h_f#Yd9YjF`Z0ezeVNf8Doj{JukDdtV!Y!E3V7_~ z(&ldUGS->$9^U(8As76m!hEZ@j;uS2xxA{QCwQK^uWUc=UBzA{x`wHq>AELW0>&JD>l#Po3qqZ|KnxNInEQkagsHwJCqrC9nO}Yx{EESVX^PqMX=30 zVwe%7NT#jDhDmB8Y@2zG%-Yv0nX8Fe&HK0AX2pu%;RW=T^3JE|Fj7I8O*iJ7U0NNq zwAowbBkN(qTvpyCQ(pOjaNe7jUhH^w0He|v$Ud2|i7n~k#@<`>mvz;Cke5Hboae;T zVI-0@n9{Uq?C=^{cIwDm+xE<{=Ckz^SYq)?wvH9PEG=0x)eysH(C zy!c7N8R93>Sl3@F+PwRFnH8kf$Q%Au##^@DfGKoX%Bx-O!CKh3m!%X|ZhKH&oIPPk zooRV8z)J{C;PC?IFjW%9>{SUf*=B}+SPceAEE{f^aDMPr-u{X=wwg1nc^v_EtX&?i zthr-}tWC$v2E`??&<28%+@Wt(7jSN}1Ege!(E+^YC`T4@$NB1s+4*LwC1* zF#N&@G_4-a`|mmO)n`fk`R*dx`boMQEEwvu6s+-Ue>HlmZ7DhW%Nq)=r$~-BWJAg4 z40Yp?3kLrQj<-2jse+nF;at?Z?uBnfopeLw!1PGDa z{lwKyRv0v7Fpn!6$+jrRek!(jKhd5~8aVOX=O#Q|`tIlK0n|2gK<#o1>8|w^g4sJ& zaoG$-@-gE}u{k15eN-Ikc+}A^MP5~~sRKPZMg`Bu*GH3i)ELxHdNIqtQ_|}Za zm9OPY-_0XEP;iZmDC$8=vBrU_-!n#)>t zxjaj877y%Y>^qs8xKbD>La`9klH#h-% z>W!Y)g z9tLYLD_V=}E~!CmY@6UBw@b)cq7Pj;s?_KzLtV9FsPp;<3Ql|tRRf+u-;Zr@IKzmp zSi8YX#Z9m*{kV{5SqV9Rq>PTAuGH{?=xsp>Tyok;7J3z=w>pw`B&V|9+zga8or!V< z6VUF{HOO$fLBGD7r<3M~=#j-7T=glN6C!e1aZesUtC2o~_`R*}27;d21pqeht0?=YF}t)dlOm+9-idLipVp}2LfpQxs`N;2xLDf+0j zlYBu3CExO4Z&h1%erJr@vR1gK-Ws1qr|{O)6dbv5Dqeat8_!qvtHA8afgUIy0PJ~FK{QOJL->A!2L6}(%ZsX(2Y6_>DRA9=Yd4(`*SIr z_E`hRN8|{)-8Vw|mnEd7luDhh3u()91(XSr$6rW9*VyU&(1*}wXMdMQm ze9j>SpA=8VpE_~4{lR9i*4{upK9^DI>QnUXU#FCl+Jim+c4sxS^EAIX219Pgb9-Gf zpS_yOTg4{WaIl5s7D_n|NkueMOOtd4S5T1dGtG6%W$pAlo;749$8XK! zia85$fo>7nxt3s?_xbv0qe-IT=v|_k$#2^w&~B zKYK-7+TEKsl{ul4G`ijRH5yYE-GCKdzo`4`FLY$*392zt!?d+;=~U|px>YiO4F<1} z=(+x`*SVNlzs~bP-Pf-zpm_Nqx!rz5zncG}SCTq%+kOPJLJq-BDbFQkQ7JjUS_YLx zRWQ$^8lGOi25|f;jj+5-Bf2h-PI5XItja{i-r4Ayk%vP=4KSd6Fz1NIe0#qUe;VkF zo4kM%L%sP^x`g%a?-u@aTP5~ipCa8SKdXLlk0j3gk%|c~rlE6Z2A+9ohhxt=aZIbg z{j-s)e{G?XuDxKWau76BPr$2|c+7r~!c_^=dF+Br-Wa0BS)bG~_^~>^m2$8yz#o`= zqB|RE$Z+-bo^126uh@02x1d`pFZ5yQ(v_7V;@}Cs!px6~!hYTB5{>S+CAYm-);GPl zQr|sJSvd4W`kp$L(|?AG=JuZ`l{y8{fdm658Jp+&D zxG|pez+I=jQ6GJ9gX$QVZ9b0Pro@uRXnQJOyO}(~YoKRo6>O32%WZFdPB(wNfV9>( zkaw#cw*MF`?5u09kG%Zfv8CHiNX%{~LBg={RPQ~Ny!?Xb$Elm7Q{Do}XWF1@&r7JS zSqZlnEhB-K&>rmqI=`$xm@HKx-EO_;@Be_+EHuL@%6TCx_N{ zBQYr^h_jnQ_`~R6?yjc8b2sVXV(-B?W{D0)H4Q_d(vQ8<{Q0dAz#kI)@#guFd@UlB zHEc)m)xtd5*OUuOU(SaCOF|(~x}V;~#0ZpLn!w9fhA>ZC1IOP|=j-d$_^X~eKU^RS z=lf_&cY7PrYUxhOhbt4|)xL=&YadB2C&!cSPHX)9%9V9oY+1d)nfo3tl2+@PgUm%JQOo)^P^`yY{&^GgUA{2o3(egmf; zwUJKrOQ>{u4pVNmLWB1-be)#Lu9N2QgF|z8`85yDOz^=Tr+jhphaj}vHhE<>=P994{WNK&1~;G4a_9o@hUpHBGa4nqxlC+Px5mEHB2qL5mQzG*MQ*A7{Vq z%cD_?jkQC7UkxSw0v`%B8bo`vI!PGv6MVM)gVHxWQGZik-dU=JR^bEjMBPAK|0fCk z$E5P%^fZ>!nZe;7jVSuUVtDqX0u=VQKuO6<*j9Cyrb^isac#kvv_1e$hsChbba#GQ zX+)aSBfr*Cl^ceeA-$Y-eEP!RZ`-C6gZ?G=z0wTZN&% z?cyY``80mfK5r zp6K$-1!kD{@)1qVRpgBHGxbyYIf%u=av@sw417JO%=>pm^Ms&Cd@hEd>O2qOSFW?r z_Sl+SeVXc@cFDm0ON02Z(>G`+Rl&W|osN^v{}dv6ofB898WS{UlZ<*B6`5=Vy>G`M zXGuNnZB@a*T~Ekk`F&Cg^J5S7tMzLpDc9%s$&nP?I|D&ai|~>O%>uOpU0@ySV~a)GE< z&l31*XwAc1O>tR|zUaJ853CkH5k5zNC{r?nP9}%(RqrLJsg}pRBg{c*uo(dU{P0 ztK9AR*lfV@@&S13y9^JHUP$>4wP5vJ3!3J?q<3Nep!}pPkI)*xCl6Uk3_B*^kQWnq z!_g#OJ+KEhjy*4veA$Re0~dI>C7Y^2CMcN4HMEnV%GJ;70|XB2nNr6$L)rx&WhY=kbLR3)!oskc&R|VCRi;xJ0_& z|I#=`Tw&(`k{Bu&+* z{5FUigF{&&eUBHe)I;sPdOTyY0oNQh=N|i=g_pbjP(@YwvGlCuA+d*=TO=CjEojOp(V>-)4MB)+cp*U92R>HA%mBFE{jO58lh4ii2* z^87b0{AZ59)4h(0qet!$c1U*^|CL`UTy9I?y#J(U$uJ!kewvBVfV|e)9ld-#F>IX| zMh$b7GG(L0z}fRfkBET!ceQS?TFP-B(=m$LTW8ab*;~m`u?kv$RKRwp#n7%WoU(^` z!H#q%DL>Q?N-y7oSpm1nN6JR2Jk>yDx5sjvO%$rP#bDRCM10f!6E@}jBE$dw(!lcX z^t19MeeZk^#vUEeH2oWBo%Z9;lcab0?NNBDd=#Eo6~`OzC1LXODJV9~#P2QTg71ww zaaeAHxM5#6`Y30H*U~Kb$u|quSFz+Y881|+8^#l72Joc`qxiX7v*3LBi}=7rhH|z` zIfsKsveBRry!duBPF9G(tIg-Bbl(V5YTki!@CQ;q`-S#iZ>Pswm3W1>D!#fQ<>Y0m zUGy3syQD4JCW-UY7nbe@945H4@+=QN`*$eUI4*~=yUNINTq%w7&!^#@vRL0P z%Zv47xYD5;?|5K}KMkz-K(!6qe0JjhUc0kXl_%=X_QCXSzG&52ASULG5PGlF5H7B4 zl<>2&)Ud7r7Dn8GAh$a($l(U*Ro(>^<%e+W#Vg491TbNU6Y1SXa#AoMn}42oIopqu zPY>mxAvXL#r-Zhe=Yz@B8kq7;6{1RGcudDc)I6SyvCk&qEhz^gK;8=VDs54+$raBU z+u;EhN3PJag@_I~i26gK_@!+t9GZ`cty@KynDhxg$#S2ekHt}H9;{Nb`|NuH#l z*Y3Kz=dO#*5W~7RlQ1M_HfAl?!4Jiuyj5c?j$AYWca=}UfeTtedu1~PO==+3{3rBp ztr7X&wt?$X9!d1DPlCR4Ey;a12cWm>_j|j9)ah2 zh9JfT^TjP;?DJ^?mo`~(mVqr^UuBLR{uXGKXa}a}tjIm3A8AHA(%8{9SetIeyJKv* z{q+zwdpJUTIsUz*Z&#M&!I}PIm9$o!limxLUj7Fcdzj*tK?!`+I~fDYrenmr4AeZb z7`8qtCr#HSG$m^dHD&cdDSZwT)CxM6dYb;+TTJIBeG@J072!&t z26z?QL#Fh9o&8Y;FAg%mX6MT=;7bl#`WCQ`ULk)Sv6w@b+VP81PWY(Q9bar!!2Ge&d0o;;CfoZQ$VZD_vo#}9aBOdmUdTc0M43X!>Msm0-MFBO3E2H_<8lw0b z@PBv!E>GSMzK%<9!;htG@ZWNF3@+zM(=#z*-%Qpj$l;3FDV$j}L@4)66V0mr6F*VBq%xZlp@{%g7HTX}M3A*Ds@6lZv3cVj0X=e>l8-4& z4cdpy}lM27~;>H2J zuv?8B-c1;UF5y-6-W!KXvgTc{f4i!WU>W&|uDE}MXB}M-@cKJAOHar4kMnUB<)YpE ze5|^>3D$L!a)v8bQ&qo*^mtYwhRKyMELp_f;$k)ecOI4Gf$^Ds*sIS-+#yX<_U3wE z>1DuK9afmHH-_iEj=;?_6Y%8dXjF~}V)Of>@Ss5mu2v01s}1eKnJXW~B~{tUIxcZiIE!=Ujz7IRC8pwvrkKv-@Q#iC@B8TQD;y1s^803(EL+{Ln;JO8Lq#~c>FZQ4> ze`llij2yNq%;VC}h5W^A0NzX(z%Pn*dAzD6@9$)BV|0YjKha$HBwH>ScYY+!`WVXF z_D1lf$YfrjaRE++6U|$5j`~0&%~lp^^}5|K#cUsNOEsv<#iGW5NnAA`jW7J2%NrbZ z;p?r(Vwl1&v82aUv7x{XZ|1o2``$j>crbxyc!XixrZ6@;5yg%N)A+vSDf+wd9JD3W z!STD-z(gSkXDf%W<;@6=hzsG@Z?DkNZmnR|uMM=H^v2T169u&k_2QeGyTr1g@%4|+ zxWnh51WF!|M%7sx$VYP0;hK3pBDlO7fn!Kq>Sb{AxQ0x27m!uiqNd+CYK-Nb4`_ zS5u*>M=_o6h@m~#O=+ULHHOO!Va-eCtT@h!jjItuwju|YxU-3Y8^3T;6;}C4B=A)b zI?~d`Art0e+K9P4wkC^hm!@%OsXC*k7Jg9H#HBy_;nKOvT;-vLtIn(7vK4)B?RN#1 z-6oG)b}M6jPEWi(+K2;e4DgwUHLi@-!4K(veUK+Th_ zEr`RpO+WqDlk_;)XH@Y{jT6T8QVm*1JNVfP^zVrPPT)Ev;J z(ggSVSW4?S8@~O`hB4aH5rg7kTf9mp;2+MK305Vk)YNRsFFwA)9KUcCPb zdGcN4mL|(yOM7x*uK>=-2*SN7{%GHAG=5qiiOyaVcv;UFUMD8<*7{f&HZPi#oKnef zz&J`&4WpNhiJ%=m3)((SfeWS1)T|>wvV{fQ+u{KCr&>|YZ3%d&4uu;Y+l7nUoVf9i zGwK<*;HUo1xO)3MTy465^Q;TF=t?2`?3j!7H*$E|o_vmbxsW%vkL189A=t+t7{9xY z#cqop37t=#ivR7GrBAmN>7M>}8vlAPBs|>?8?~Fjx52#L!$wJ1cX_Z-GhvS~%u^Z0 zRLhZI(~~T#Z;Qso7U=lSnk{3U`TY}BN>%BL_ zYrlPTs$)C7n01CWm)`=HJ`d<}+I{l*Bg2nwhVj5nV^MW;H0p=PW3MrDq01tf5|_o3 zze_T?6<5Ou_nq|iC(}K}8VWfO3yY*$X8oN!I@=?citoIo9>-t8_J$YGxaAROni}A{ z+s6F7)s$r?*zoB65}Gv88pf5{L-)z%P;V^D`=0i|ZT>xR@fsO)k}RY1*OtNhpCxc; zKr$SNa^cCx0KYi{Mqly9F7rLGAa1Wz6WmX4Pp_elgr~43{2BF+dQ8Kj+h|^nlu;^`ks zcvTm0=RG$xvhhN>#jg15{$SkoYcS`sAuk?kz`Zki;EXH1c$rk|z8cYkhnUyGhjquP z=KW=ID7a58Zs#Qpudmma|11){s$RPkI0o|f>7#Ja;V?`qAA_y-gVFQKVAhD$XW8fa zJYCTr!|wRA@2wGhXuxQ`zVJr>pl?5_tBWc?R0*a70vzd4W1`=k$P=AwYV#C z@RPkXD0eA@INt)*kgHZ1((hLAp2Sn{_zRop!7g4={uB%>>tA=vQe0sHxXB6^u;81 zWgNUlo9#dL;rB^atYWmb-bZTd4!NsWKQ%oS`fJ(3O?6i(Pj|Oy_N0Q$#<}q6Ri3Ck z(+y=F2IIC9XKCoXRz6jFcGXHD*+dp_S`teHG6G`tRt z5R7dWqxi^>KsE~t=G}kf>C581LO|a8ddppT!pMo?XmuizPdJ6L&GcBds+1Rdy*ypN zelEw*BRUY?S!DYwIuo3 zL>^sJX^@pUYECp`T~l*5HURcgcIFSu04Ke$!|YFp3;*td8z&EtOkfcG*?OFevIgKb z*S`EirZ>k~+e^8Xwv=)t9n4{ppr!v1l;g*v*OPeu`^SO%-HPI-MmzdyF$4yl5MYgD z1AM$)2^I$NG zvVSBCHD*#reFXd{=ZrJndE%hG za`@x70_Sg%!+ISS~ZxWgF}h79Y4JjGFmWG+A&$bX|sIbY>nsyjuv{ zKGs3Z(Ne)qZmwkP*HwalSEumh#V>)D7~~uvHKgFuDjE@ZGN&PmXzt2V2l==L@ zJeQ;Ac<@4T` zg*%WMA%~Sp%1m>20R>=}%#QYa?j=$f+MZ-WFnZ{1De! zJrvg`uA|x|UnGJ0qlAX*vJep08=Wmfc!QMbdh5|RY`PtfT8>F*>YdE%vl2P8z=694 zX0Z9rOnelOfjc+*;K65mh05B#^!o81arujZ9Cvak%-a+}b1!F*Lsk!pysU?_3yoRr zgez;e#PZq27e!6KV?uPzaM+}uAlkGT^N~{~7*u40+qxA>@B5J$aW;rY4UOQo4V^T- zd=g$VjNy=LQ#p47vUS53x>52L8VmnHp{@=5+`NKD_m}~Tx5oh1JBy3AD)WPFAHY>! z7l%g};MAZn&MqH~K7*oh>rsES39{tA`L-zg)DnFhm9W;hg&sznhuTTcVA;+#=(wbe z!|LR@>7*K0np%;$$8fwpHk^H}BY8o?Yr1%K2GsmWr=@3>5Z~(%59j}c?Cx)9Y{^e5 z@GGa9A38X^w>}@}?9c!8FlNV>KD=ad5H9K#i9@v-AuB-@l6tm@hf5O0^3m1w)l?2_ zWfoI<4;hO7bdjd%s!&7T8u(c8LGXFr6PI^xh3uU5)EXO4=H2#y%ccRm{7!G=S^e;Z z*%I+k)^Prt>W>Z2qfklT4WXbEbApy|O2iUQI9tGC?kTZnMuM=^O%5{lY=9?r6LH1w z2!5vl{8RfM9d=fNU0KSMyXw4Xbk&2jmHfGO*+|@N7=ZaFw6Vi!6yEn3!}Wo&{NT%H zajUj6H_TE%>#d5IFJ%u7oSnoTUebP{kdBT@ldc zL2Wm(Wu<`2>?C;4eh_92%jMeGT)c291uyDF;rAnF>B5r3aQ@CMa9o)I)<4|%_fJiX zJYj-uuC>CCVu>^K#VI##3qKkwg_b^g;*{s3*sF9DsvZf!cPi?* zTr&;2*3YBy!)vM4Y!$gr%jCglX5yvjskk~J4VQ1|$;TtQpqK0?I6CV$wDmG#AQE-xJW2DY^lJzunnRJxW5MgwB_>+3jtdoLCvcBo^J z+eErMGaNj0mcX$&Ti|~AG)(oK$)$PIc#NI{H}rF(Znsk)ylosjRlEg(&C1xhR+-<( z^yGcp*VB9*OO&xN;vrK1i``|)d|QWO!{KDeMFH?&7VaJdj27qZ3n~=r>?qe zl&O;Zu8M;3N4@b;Pj6mj?ZJnWk!SDD+=$2nZ(*g8SAqG4$zJ2y2#p|Dz^x&XOy_+c(oO{Ow%s zIF!kMDUtK^ibykZH~2WjgZiJzG-G@k9+&F3d#{J{6zS(zG6z%IaBy+gJ|h$dyJ6^i zD~x$!%r%jAj1P_x`0KK6>tF;uE&O)3FPcgH$6f2n@#fRz_$$N>AB8Ex#XZl&cKa&P zaeIGgl=%YsBfipOgU7UKT_r7Cn2SF5^ZEPd0-kww9;Z&3%W|J`@a6j)lqY*sneEOK zCJaZpJ>KY>q>YU&N5JmmQ`(SoiCnBV5LxNs#!@p5G*sj6im#|v2C(l~S1#x)@c!@) z(mA8ZwW1Q1q?`u@Cvz;S_zit7zo)R79rR4INnBRZB8*SkAqtJ}#e^(t+F={bWmjTx zk8&KAt@g%49ZG1rPL<_ypOC_X=iK55kmeFp|H+fFJ*a>9hh`}{>9JEWC z7mskq8SuVK*yl)?DUG;Hb$7&JT#vRz-p<#t&-^hA@p zo&8BkuR3P}+FSa$%hxOYipk0a1cm5#t!JW|U z=RHueI0{}5v_+K-;{;k~c+BU?Z*igAGVr{=i=G_XO2Mg(^&Mst`OL;(EOHJ-k8BGZ zwq5%B3{T zt}{cU!?yVOni82!=?njccL{1CUxgVZRy;?r#bRF{eA`VQ&8k&6=BzR{+w zQ;%o2^h32N<~aU&Uwl^{&0$w2;9!|}?A8>7Z;wr7%@xz|S@aydd^-a}o+e|%qA47Z zGlTm!&*mO+_aJf9L-O4IhHl@L=hIW}(CTmZAZ*zQ$oIVgJwEkA!_De!d&ZgvxS4Rv z%AVNOrpzBz{!sP&fvou;6+e%i#szBg*rRX?5APxE&3O~W+HWF9t;tknQ6$Ad{wMK$ z=_GvLb53&KQcIZSp&}kWrYdUB52r!l3-Oz45$}6g!h=l8c$}^YyuGDQ1!8sE$5&wdMRIrhmRx^ZeBW{f^RnTZ^KzN?E~PN-NpBY$b=BC?Wg&To_T1 z1(65J!F>Kfc)4;vwe1jT!MyRLqOV6^+|A**s|UPK*(QvbIe-T(9HA0X>7zIN~~33hcKy+YN;%aG?vo2=9h5H=0&K~UWP@zm!pY97tMbf@{gzb zJb2F#e&L!dk&pRSuTV5j^!I%s*7fVgGY0&I{893_@8(mO8(2f>Wou!L>kpnErBlCv1JozteT_N>!DQ2P0zI;6PqX6Fw3+HY{5$JwB z8vCq@LO!(5Y2VRM2?45**6@C7zQvq<0Tg>kFpK z@ta^pto|kSbKAdxQxl%T^Kp-<*WZWqL0z7=zj`G8xY;QfM)!w*Yg&Ypt}$G7G#;DN zVsVmTB<}u}#2vF!(BEe|w)UBY{U*xepzYG$`v4twX}?3)n!*eF)ebzi|(kBsFzl~Rwy`0+TlF$PEO+y=`Rt|fhkRZ@e;32Km2 zT?EO*xaFjn0ur=em9kAMd7rwQT%003@a3evFv?azVu@t9!_?~ zcg?NftZxQ+(UryVS2hsk~V^Lfe$%bRcbJ4y;j@))z>^}O0FnQ8>A>NPd6gM=HS&Y3emT^q-GHI`J z1+P1vh^thlaMzOFlg`AwswtabO;7tmz< z9~sQPM|tV8yy@u;a42phz?BsB=N?_}S19=QS}CqewG(d&0b+BMgqtS0WA2R+nEK5M z?Nvtb$Jzckb67AQ+7gQI9WFq^!7G#zD94w=meYcSRtL1d_$_U|3_uAeKEmUgU=WYVoN7AuIZ`EyFTbix!t|-!!=9H zOiyMzj})AIN~+C{PsjPAa)tlG)`>^coG56CjL>(%4D78rhmF=|bN6F;+-8-5V*)ez zYTYb8buWj{k91|-C^ytQ=ZhQ8BbHo0CXCVfEb94`iBsymY57qfRy;5aU6RajB`(tRKH-08|U3WjrB{WMl5PCQ|8+SUhCKj*Z=h86KRSO@h3xJvBK__=scHC5=((dGT=*~@{+Wi+ z^dwUl4SlG`stA~TG77sq{P?LfhyS+pG(BqmC@PL<5R@OQkaQuxkg;<&n2)KY3fteb z&aoRjQ*#0O;6jtPgi!Y4O|ZRvg;YaCC|V9wZ%HAZN`(-s$6iQGoNjXTnjN2Jw;QapBHU8~V_>AF9+3O-U&gVdpBw*` ze@NTxQlx!<>HI@=q;%#h>Z6pLy4 z;k+(RpL;6X@X%}7nEN`D)%>J&y~b)9Z!;Xvy$EJKeLK#0)SZ7iU4bLlDk$3Cf--Kd zr#^ir@N~aaT;OYf15XFz(~igD#&cTmxTgyJ`l>JP`en}34_c!A=qPmRSRy&K`Y!aF zwuA85F|oL!7i;VpiA_l%yedHIH7Hq0mAX&h`_m_sU+Tc^+8U4>+npzm_zl_Z@8Eh@ zE!6zy!Py$aaQFjV>HL5J*6R-x&d8k?)&G)ku51Z?^eE&5k|O*)BoVvPO|W!`8wPm$ z@!(P0DMsqql*<@R23bR)t-A(A8cCTgbHdqVLJ-2d2}n~e!I+5&xZ5?J&!=Q^b4~=^ z^gjiVf08<+y|iMFN(y*%pL$kbfXZ!6@YD3L#Cgm@@@jE`>g+n`xlpq{?(SwGw&j*M z4)4OD-nQ&wqR#iFeFf)OEA(&fjz=@p*!QLqMh-B=Y8QD7kBUb5^jLnqF@{fFn8s6r z#-VD!1P=QY!~LHmux;i+;Y9UVaccP*Q8Kf#e%g!x{2`4ci`Rv6MCeF9HOi2(PD)_m zNjWHRwu6tW3V5-5A@)lsLdPYgI9_`H+$LOV$I8dG|Yx>|U^LH?9(IC94 zdmn1{zoLOjS82$*+w|nsK{6On2hZZ3!L_w_!O83eM2=~rog?m$h0`^PY5W0tI-hC& zn_m1!YbD(@%wVmZv(R;20XFPRK|N`FKH1(6A8u2?={!?k~!`_)(GckwbvS;$nk$O_b>|kDP zBAxXbY{+dTzPL1cI7e>qWusC4Tu+*)y1qZpn$(A{l^F6D<2}?=Pz^u(U4uheB_Mn+ zrXu?iP#!P}7T8S3u9> zrEqIZ3Ao+eLmIyOVfxag5HqY6K7N$rsM}JVyyi1F9M!=d2Y=GVif*`5=`@T3ef)Pb z4hKI^=F@XiIdf_zXS5jbnPgMk{K635r+MP1=T@A!#}54sP0%pj11*ksz_jOIsrHF1 z58rp2=3U+bhErBkQrtp1Hv9o4O}a{&-L3)a+=NLxm%=}dUvN&d8%M@<=k7DQNb|O_ zAUh{t9DGp$3hf8c=U=;kq`C3!z(@2xu}L&JGE}(RElIpxH&A%7u|+)kZUZ%c-U7cu z3jw>=!NYCdoMh^a(|&tmyN)ZW-Y-GNY-tauw3JJpF5|LBCAR6Qgn3QcSWw>sm3#PN z)|sI^|0uHloj`U^J_2|D9HeDlYslSb4UMl-#7YY#9(bcWuQycY$hES}!Ez`*y$9wU zzhLErVtTT^7`z|ELv!FN=;5u;y<$wTUzibY*7rnqQou=*d-4OB$5i@Gj!$dav%?KX z40Lh8UiaOx<)JD+i0O-`RE)7G`Wqx24aM|F;k?v-6esvZ^5K{SdTSjA-;OMWBkJ?u zp>;goDNAM9`HB2;M=qq`D@vK^t!k&SJ&;WKVRV`M6S-I=fl@P@U#`;O64uWk9BgQ!uNa^ z~C7otQQO45{J<(l}BRS`<`(6uQaxFyGs3YN?`HmMKo`JwYaEP6?OZx5*ln* zQ-yB2VEgwQ*~uJ)MJMOdjrq65%~1uSn7@nSSA2*dm;BR$>+;* zwnC>;3AIRPkOwFA!T|84=;qPj&^eQu92W?tIrr(4c5hq}s>MP5?RcfSn(&te{xjMe z$L^QP0ZK9uvC$JxN&UH*9LRX{s$|32={WRrBp=@%$7;G;s3L3;%!pQ`g|qI_lZl=} zrkOgngj;aKjZ-x8`XrLu;DTMpJ=yK&AIkL9W8d~wbm?0e#1w53hBWSg3z4tHfyos@ z#*o{B$G0-U=z1xpw=U)j6Bn~$em;NHJxIBu?!e*5Rw&vK2E8s%@=j)5`C5X1D<`##lkvMJZjWQb5!2IxHeU3{aglv zm80-)iG=4j>9g;_6=1b+4nKbGf}NYJu+T#Wiw>#@3XQg8s5lF{pD^I~mx1)aRBFn9 zvk}tPUV&pzT`;x9iHBC%uzo=`&E8o7BQ)yC*5UxgU(j^fuw@Yy#2dhE1q0|?D4i+W zsmG40cLASQ)i?Stg_F%U;icV08g%jpC0{s5<&P8a)1GMFJ2a7#XK8Wb6f?Z4sLFi# z9t9NWNoO8nc-yLIJoPLRgLGA~c|sB`+UpPFW-7SMT--|t9iYM{R+{KCr6gR#_9x z@3zd~nZMofN?&U($hG0g7L)0+!+QvckWO7{4CfHzA!vVlDqnbzh5M?L^=oPNp6b*6>4nU#G$V)NP4e7M$(<_y!Kc)4j&zaB@M<{dO!`< z?m9t(U%zlk%sEGK?;QBi7agpgql3qjUO~@67wX53GZqF3PsGPRhYDX#Z-U@4e&o8^ zg0A>>ihgg7LlFF>iTs>As~o|4r!Eh6HNY8$Ls2zLmERjar5sf`X+PsHSbW?{*Dw79 z{@$A{)o+8@%Vn_ry*!2lYxBhIHPplEFVv?PqI_x>eOK?tnwCR&)jpRsYO`dIZ1O7YJgBz`+Go-tYi+)Mcx@JolJ~SSMR@D z7L2CWep_I@%~GmgG=MbIA~Ic%?Wva^!GHa^*jSPO;?4#OAINmmskBm zA5}c4vyRGBm%_-PG;np?1TWrZ^MIAP*zI2$8vL{1J8@y$P?#Xy!QhW-U$26B+%f39 zaG4s?Ww?(dPuTo^IGwT-g_R9^g(EYgp(@0KuJ#@dYm?33ueb^-4(fCMnJ(z5Y>LNv zkHu-9y!mBwAUb}r!nw|mplS0Uq9z1%g z7nV#jL+`6G)GIFpCXD$fs`hsPCrNkq=#b;mKn1LsuY->twNkWiKF-)x!kH5n@{wox zFmc{TIvR8hhA*&1t<4@nX3R{E%N&A#3W~61ssgs2o+f$T(^Xi1QxvBtn&R4N!PvNa zH23!$$=M^!@YQo4YTTRzANOn0T50SI=o=4TepZq8=L%YJTLHK4h{tn&LpgZ49&1i; zL+gfqJxN)RU106Vr0 z0t?XhBnadrLAylM;)w88o+;p)37#t3irD|o!#_i z36>ij$i>ML)->0`dTTx0d+Z{qWgUR__+b$GWFxFjT?k8>&(XCsP2N7k3f_$f8#199!9NjU%n2 z)y%xKX42ygnZo(>^;FRTRx4s zaIuAxHSalpiOxAbXpLvKnK|0nCl8-bO1m*{mxT;p5HXpfy4{O^LV7+w#%P$+k)gyq zC>g`Pu$;hZ{SoP`F~BIOuMlZauI%mQNBBIST}b?4;v%$f>kaqXXm)A zW;51UunR_}F?Sy_?5Y+k_Kk|7P5p~X&gR%?zOP+0Uvirro6}Ii$=uS&zi4!qKV{25 z&TFwSX5#a$?693^U!00y}XRFG$WD{r*XlJa7M!a^DDjQ^N}SAjYH)Q552 z21fHt}SPHn}t3MSWIIt(iIx@6sIkV)oJCj-XhR^x%fn#>yJx6o6my;Ji(wDFZy|=`iT|$fbj^m0r z{t*=%?zCK+=F8^n#Oa2Na@!;(Uel226WyV1Wu<(T!4keBKbno)_=| zZuW2LYo5_($yC3c$-1@8Wp&u;>}wEtN!@{*f~C$J?TW1&*FJf6m1tf^)-gp4|pD>2m{Z(tO%%v~$E)i}ce>BTxl5wI<)#m)> zr^gbTN6#E@Uf*fQR+ic_AU&UHlXYNf9?x%{elW?VxHr18a9d$2(^5*|5r;d9-5<6KyHO zuKpp%XpL89&b`)RTyNWP9C0fD=RAM@xef>ZtJE)?>HH6TUF&cB(527$ZcdtV>Y zhwAZs>%-=JcA*n%)ak)SSm%vzi4Mou=HZa&)_8y;q(Rkx_QmM?X| z*QWJ?%=XuuNvmJ-#eY5GTdlv%UwTuP*(~b2ywE;|?V2ji#(iGSG~Zmw{+YIxb$Gmv z)vxSnuBrQI%#Ji>LK98cu0qkg31bsh<>Ouc>KC6l^QOPz99r{(^Zs%# z-{kHi&g?UfIjX(nA6M_rt+f(KVI69v+ubs=kbB}Hbq^TqGp0D z)><})O`n#_x+R7&0+$H(rDY^rvtvKI?Dm^;YtQt0D%=&fenio3JpJG|Iq zegQ1+?mN!@hhO=VI*0gObuw(+TuYJ1J%_1vp20ND*JJ#~aGRS_;%rjr4V#r)pW8Hj z&Ec$T%H+4yr}BR;jpwh01N>u52#1OH=XBYbaz08MG4Fc~*vVkPs)XsWJ+m749VU&O z`l;2NXs@TO@tr%Do=Iqq@IQLPV~kM9p>o=(}tUc0f6&F($G9zCYRH0c|&^N&qp z(<3d|A2bFePX=T6!c2JI<4VeEFTvdpd>Z#{9Il=hhsCoJxSh3%r~vBtC`Sw4PE*A{ z8mH*5u9NWWjxPL|4m9EW5zM{m4Gto^wD?s$?eft@r%l$_wqqm8uSg`~<=;5Dr($u3 zeJ0A(rQw8Q55eHW*neq1%>0-QYx{kOPojiylEf>)4mm?m9B{{g zAPq9h%bc{juEy_@vKTen688Kv5Y}wlD`=ERVl(7c;Prh|FyYKLJUUp3?*spV3w@4z zCVU_lB(BkGUsKUe`!e^?lvDImw%Ps=j}vqddQn-Tjs)< zU}u;(uo9CFYy_K~cQNHr39;Q+OD}X;@;r`AfGaCK$?Abr^lCUnl!W)mo%VESO!0w$ zdy+h#KZj7-R9rYYXqYI>^&@XJ=E0ToXYtSKbQ0A1j$ZFBgrwAF>axxf6XxjPvaM6# zk?tGP6aE83+p;mWI|*}gW6&>ZB|QJxOP*BL(+;y?I;KF6_a!qPl0gGq#oh@5T`h$@ z77;k%PKt=TFbDT=hRD8LNuCESh6zPxs6Eg}6}$@ZxRX1yJ)cZPu|ICffM}-Zp~ra5 zUQQ5uayNc#(8jUr-DPORJBAWixD!z~ z`W1KLKQqj+IZK_E#8I;>DH^41Kp)PX4I361L1kko`1)3oc_Q7*)Nv=-q!LeOsmTK` zbr-)`bcd|-Da1a}JJ+~)8sfAz5UY3Pq|Na$eZsE5ujNa)>3hU+%k36MXng>$6DizR z$mPmRmJ`lw5zVXb(jsDmJ!Hbg?ex>-0jig6$eSb|0pGH$$&g(d^eo>@CIrO@Rz3Vg zVzs9eMLBWwmU&823YUP4vmtbz+X1y+YjM5PC|d0g!_VbM1P>f;k^TsAVRYU*EU-ER zK2>QrqqbL2T(+H#ey@i+>e@VFxL?4l*o}#wb-7Y;NyO$si6HAv2EafPY`eaJ*5}^E z@FfX&`l%%Sa>ksjxzYU%e zUReToXtD;5{OF|bx6OqzQxS7FkivbpJ%Z@|4EXnKJRP`om}D(Kjpxi9!1>BY*f^#Y zhl3UH)7Y`NP?(QW_uFx|R1}7Gh@)t98nKB|hA-8An7ArVU~tZlCR%+aTZ0s^{r&{h z*b|P^xjVtS^9{`D-U5fC{eXASA6Bk8ii@m#!TZg04E?YZo*b_h$ihOfep3W5Bc*BB zCvm(fVsf7O69LNN+4xQOI(1~%;PhZIkoa*9ZYsIJ`qlm@7nX%fO{arcx-Bq`x-e#k z1<^Y508LXCk<}hsP-DQBlpUKPnst2Wj7xB+kdu`5`;s}W|)T&H^NcOi1| zapE`06&u!_!SI~T=q8~?<*dbNx57g5ec5bY#bg~Jxgy5gY#f;hcXAg(lNJ}JNhk=fv@RjVCCVtSb%}7r-N=BNI`|pLK_+i5 ztUp>HAVWd8R-u7ChW@DWHv_wzK7wq<6d-0itbLY(4;8X;ju($Ki{=tnUO!9X`-(~4 zwGeW5ojywQW3YYb5z$nx$L1S}M04No5Eq`Gt?IDY2H=$Q4f%&D0fZLErhSA#u}bVjO*gTuhmTN5?&-TU^HryRW#z zj`yGOXHOh1%zXs%0c}KlI0mG)D?kckObuVJq&YhtLhYxYbR=R11{e_x?#YGEXO3di zoCT<{^eawq-va+Krh`}gdV1Ku6kSgYqW{;=^a8hx_*p)Oj1WCsTJwMIR2XUFZG&1* z8Q%S+3OM6aHp)DW#K*IH(N*K5K&#yd3eW4)T^*`)@sCOompTrrJEic1=uZ1{^$2WG zs3wJeNpR3I9KSV7p{=4ZOgL~9hKKLKz1fl?CayTFw%H^ImGXp@o6ka`sxoP7+D%j1 zC1{P`HB#jjPE+T|!^MnOG-8Jwrf?%M{nH_IkzGNwUmnH6#b&tQ@iI~PD*-7567c*> zFLZ5}p;^~|&__N~kbUruJWl!`;zVcQVub~CXX8Vdwl9{u_(hb6(|!*O99LnHYA-4L zFi|Kdw#Jh6vv?Cv)=|;GR$Myzgp}58!o^ohaAL?ku;FjT6Rx6XF?m6I^D1b-MsMt| zZX!(i9j^aC6kVUW8r1?MA@uDRSXSVQEBiRKx$-`lw9}qu zyQucMCD^et2g(!)Dkj#zXl)()R`V=VTT_Cccec?8FLOwBtR#|ca>AaU4fKX=I2=2E zj@#@sM1HR%IAfX;TpGRzO9XwS<=JEUM@9xa^5tlUr#tZht?(SOCbfx*fBXr}&ko_+l{Z16R}n=81x}P0hV=PI zN%7`eV4^q|HEPAt;QCeU(b*2ywEqg^`z636{W#c3-=xw*_24SC3Fl33$KVs|VN7NS z88bB)hG$6#m#-;;lN!}n6zUEQKk5*EU*$fL?-pD%NP>l(swg-3nQ)Ic(EG~zxWLjE zR@^%YhbtyyQG6%Zmha-`uMPy$m{9%%2Wx=A32^r-6cfZMwJFH-iHd;=R>W1 zDxTsD;+e!o{FHtTgB>Kf`Vz%x^ky9USD=TU`J2#t$}Fnf7y5)i4w8d4P&Y3K zeM^*tebSS0uk8s)w6ljD6KBK3OBL9$Zj^poxSy`Hn9SP{;tG?uSV7&mpQN(S7*u^u zpv9#aB9XD2E_BwyPIIR$cye*mL)|kb%NKOxAC3tGU~G@Pk_PFsM_l%;&6GP(#7LcU%>!t z3s*y{^-nrIdoJ2rT*e=h*Wfqz34$8ai==$Z39@BD6e{HAlCAND)N1o2lA$~RBX9h0 z^jJEsnz|K-j^va4$>H3px#s~b^6_1Mx!TmZPFv_0=byEyTr1d!J)>Z|Z1x@t6@d)kxF2QrJbis&# zV{r0u9awGON4g)pB1+fY@#p-5xWHa?mS1#+3jG6creY3A&MC%_>zc&SaVmMV_YDoV ztD#?wHHC7`D{-R2Me1Rxg~!&W!G6zB_^|n-fFPf2@(zO|S_kR56AwvC^;UXxVL675 zKZk0vES5F6L#%BDxgLE3<;IOc$8c%fo{~q#w~mDi)0e_IjZ$*=v>tKe$l~DqcvMFxGqq@3)6;)|1@jpxtUA1pH(5>v<+a< z5BK!!XxzHe1QrEkpkmY??)OoBh%VbpemqGf7cYyL(rwW+t*4P3lQPH09;;#K^gn8H zLxb2@Oyb?Lk7AF<>eHDAM!CsvE<;nrX{us!jB0tVgk9P75d1O`tdE96F_8q5g?`|h zxgO}0Q269nhHl?mxfK>lAfdGmM44{lm%Wr+d~FJQXAd)CvfI&X`8_yPH4|1bj+hlO z5mFk`q4VM-9_y5dh3A6N;j#}79egH8N}7xQwZ}y4OCjCUw;6Z4YoSFzCRg)g6EXR{ zi+l)q1NX*@7}aTC!0LexT{1j{{v3P*pBz+CrcoZhe2*jBogP5dBx7NT{Arr|u#(%c zZVznxBMA$V#UPc_Os@t@@I=sF4BD;%7glZ{yq*T`{o!LcciJr)?S7T!Y;Cb*)yqMcWQugeF(4wETzB@;-z4EbO+3OWF!0iiD8rg(19(K|S z_k$pwm4tmQ_o+wi6XIx=jp>DpX{ur$3FU=TpT5gfIrtJTtGGeVW!(o=X=&JU&zSTr z$ibHKLGIb#;&9ewP*B}$As+QC-kE6xODso8$=|j3B_@Wx{QHvl1lhr3y?lWT(S(iOYII6YCXIsj z=9AZ3>DT2fotcvcwjyT88%=$*z3mAnw^>r*#c`yt(F@LhutuE|nppd-0zX!Cpz4Ly z$ZI==C$uf#m}(eyO3cI`#Rfv*gfPKz2PJCqf(v8cxKI__Cusgn0BW9Hr1!NhcsC|c z&Yh2R3kas7xK4;ik< zIYRFzXwY@GgJf)gD(EF1LHRu`l)qAroOkrbu*7m~la8Q1_c{7+GdbjAE4iPFc&wUd7Y3*4TN)#{xff2tG4j6V$lSHHpg1Iq;C-#jC`t0L)s z!w#+|E60|6Jq+cqBw*Bf6a{r5#CgGfkm^1OD&eo_Ph1FpvPJFABLps=>ZOHyMNCq) zFhP}w!5i&T>#Mws*g2t|+VIn@_ zziF}9TU0>O2din)I#-O1Jq~(q-q>Qg6+lN0K5sZg{30zt?4Aj(Rl7>Q6>f&79%8~% zfk%n*LMvF(lTSAIYT>@0>e%v7Qz*MX6bJh+U~j}moEh4OpLq7zX;y9{-(U?mLhHzqP5FLm{)mOuruHy=mbjR-j4w=EhZIp^(zHq zpLWwZ)B0()W;+Brwvu{nO{y|4O(53$g!}c2J>Bv)5Wn&dqEEV`=v)z_iC@c6zP1Qj zIHr*8z8*h}dq$l74iRt5b7b+k6pVO#orLwpLr}XiT{lOEUhuvEJKat+mzm7P;SvvO zTaGZ&DvO_+4WPAJ2K$z);LDXaah>^i;O*axIX^_5+WVs^1bE@|$=Ap};{nu}D#;6| z&%s^qj&i%Dd__7Qmn3YBfs5)7K<>&5?jQN{=uobVPZ#E3)AB%~$Wx(F@`cEbsDt+> zF&ta}mE7}qEs*OoMOOSIt?Q&{#}UVCch};(+F3Auuo0SCbwKcPCCRKQqizdqVAqFR z�cQ@Pyx0*R25{`9>hy~&cbO~ zHTr47HEx1k3we4*nr3!gqyf@%NVtfbx29hH%CF3ROuAh#6MfIXOx19c5F$u~k-{B1mt835%u`(WElRXB0x0_9>Wcarp7Tu>Utaa2+4^(t&X7tjI&wMgr0z`*CcdH#sG%#VrvZq{D5>LWd?Jq&vm%>DuK~y~Lb1lC_mP zcDX!U$dVK04J4rZjCU}g?~ki1PJrY?cNl9g1v{^X(l>oMm?JI+{^Ke@Awb0H)t8g_a%1N$0LYK|$?P znmP9|72Jvh)o3eNZuAKotW|LKe`hfL?^80hN*=bBAB59sOEDxP63Syf;aiCdsQ=v` zT!Z_W+Tyo(<+V7e(6~*b8e$w;9NSDw{iC(dUI1h3D~a8Yc7Z* z3TMKwUX<+ay1NB_+YF+d=1Dla&73Z1%7MPxTKZt6Dm4fUg4AbKkUA{Qhx} z7F&bKy%4G_9}T~H9tsY|d&2!uIo_-_ao}zHk36+|PhtY1(JXBv89P&ouX)&wh$8cl z{y`27JkEk|GowJe*@H9{9;4OH;!xrFi)J_h&j#8i>VEqh^uUO&CxgUxD zrv^&XSFr_`mf@PR9QwjGi9RXZ2cCA?sIuy|VDU&L4fA*kryfU>lI*v1t-dY&^?|}; z5o`3ndJ9lrzDl@4|2k^cD+x}1>8Cx_J|Y&f6g>QXZUV{x@&$*2qQRM9R7_z1A1O{w*1Y$cF z65parRB~;gMfN!MTh+k&>RPr@9JwMeCd&5WJwb>h3lP-t4qw&xr zVa60ErjbEz5w`i|!VM*Nrh3yNr&U;(=t+Y=c!Igw6)el>|>L7+bqVR@xx_k*K~rI>tx{SQ}3vraS)ENTmrmo4w>P53Om1kz^4v# zA#G(Xvd)cApYu}iQT(}x)7*+qPiqB7mpWqQZyo3kSp+w`i}3TWaFq3qf?e;TASz)U zoqE`nEPZB$?r&y+xw0uvaL~ueW7lA8hZsEc{7AQSYoXj~Wk_~d2nR}k(S-x*WbLn9 z{H=76#EJp8_H{LJzMw`Nw<$sV*bJ=SAqS3YM4xk^8n1F(VR_tVCg-{s_&%J6-!-hz z{>U>h=;Gss6UDHg-$BGqlYy7vf2qgsOmfq^5$xoW@%jO4SgAY@i?vem&8t^*>cA?@ z*!_mA%-)YuBOj3CUJkn!*-*Y|3@pA~hB>RF&~v&!o~m3Z(lZ2HRW(<5vnLgkM1Ek7 zgBAX1QRa9>fb_+aBa;1{doiJp}plcWiEepiHL3Qup_4|W&z=v*}<`kV-eUwek~Mm&gL`vp2H(wh|C?c+{NnS+z1zmx9{cgVXb zr*WpzEL@?|k59LRacthE5y!3=Y%!Y$Hs2D^a7G3=#xXQ&ToNhB-ia^297hG8XxuVk z7MUeJ1{{8zf(M7ic&%}%FvC+B<6_>RAeRqk!#|O@fBoF+KB;KaFd&$)|2nzW)d(6l ztBH-{67uc2sHt7T6N(>;YVgg$aMofQyzAaYZofAbmZiHQ_3I*=;(U?v=-7S1l;gR; z>B++z-E%-kS_48)>w-utVtiHzQTDdsgDaYuuZPJ*S>`n%= z!?+JxrbBge0eO2ckgR+dC#czU4tM0dhWzW5AbDUv_UQ?6>`)T&_8z4N2d|PVN-w#F zhSS;Nxw3Ssiw;*cVjXTDY{Iu6BI)Yb1-PnkDlXs-)0CI@1!k{Wh&tB@HE(!xyY1vb z!dIEQCjUJ)yo<)X*cY(mkc(j9)Kx?x?GOCLA5<=+pT7T=gRQfck;*krL~-F1c%Q9G zSIaNM+dr3*Cx5LkQ(?>zd~YI+fBeVrmW7E4m7{YY zbgB_FFSnqTf_QqiDpSx!qTtTkMRek%BGP-yorGSwfy>UCve(RCL$u^vL8H<|yxJuP zKLhqb8%Ks0>$HJRxlqLav28i@_?-h!nfIE(9ywBU5W5m4A)LfKJw7xwX#X;t`zRZB zZu+H7L?f!{u^i$>b>I|{Cru9p=l<(67zSkDqsU#@0AytXsUtNQc*TQ zdK0{uuE4vKbq6GWPv#YsrlYKaF6C7f2~x|tiK^^Py4_?0@)Jh6D$TcuZRvW_l1kD3 zi7|H9BoNL+0Y)s+!k;OMf^$n-L2U8>Iv!=$t2rHT?%SVv^v8 zdMNx-u0SOnb1?b(m_GINh0d-tdfns=$*kSQJyEa%D?Cjhf7VI3Xn%~kFqp!bInIX! z$tctQ9g74KI(6J}g-3AlGY)o4SE1dT6F~4FN^p10bbhe+H6s0!A^vmrlkGeg!QT}Y zXquA<^Hp6?YvdgmdKJ)E_qn)k?0lTB8;{7e-wCsF2y-j;ds!%2KA523oj~vA(wyMBsWH;l65N`;rAGd0Vg9N zHLQ<`92tXOH%=3+Q8(G=H%^czT?Z!Hd%@sUtY}>?Q;<7LWR-?RK0qt9YxdA%(k{H) z1y4|*qzrEBB!q)iecZ{qRS3F!@j0&uLa(b~)k7@^vT3L2gmG;Js`xj*CXR-K?03 z1!ZwxrCEdD?h`mTC7F~oP2rC8nBgXMNzjlQCC&_&1gjL`IeR`mQ<^}e&6bjm4XRwm zP)*qQ>OWSzt%iH^{U3Ux@)@0y84JN-=ZVALV04$!gYcWv(CKjjy<2IGshlcwY02hR z=}jZYGtP*Z&Z&6r#UOEVmBmgcTg+^Y!QdTToDVMDsQTRyLpEH6eYXs7)?ghvea+!^ zbJcK9*degGp$?D|g6T_o$%sf>+jHR&(LUG+aw49$m39`c$xR|!uUbUEzacc1m%@#n zi%F-N8hF2J0NXEzMOmCCq91L7>hG4~i=PYNc!H;(%jP_kYfCW4x0w>VOAX|e(NDo6 z-f`-Dwu%}&mqX1bVsLq>9CcS3AsJiBNZ0ih)bd3huAX}hoIcht5vPmc_&+o5K%gmX z@|Qrd#PN78rxc2Bp9gcB9Yn(UI+_mzV(=9qqxdiyh8L(&+d~HMdK^W!LCEJBjrEA3fZ5uSXc?`_{LD> z8_PYUn};&CsjxI;vf$>!a@3sHPc8RIl|C`bfG#kV%a}C+Rp;F3N$@m51P9vNjG*kPsg= zVBPwbYNeLZuU|x2h5xGQu=-dWDDNTIJBG{g6n*dQp;X&XQRwhi3Oub>!5DiX?P?C9 zGhS}OtIHWE8VTVBRDM9}EhF45@e?n7iX^+dRB6}!WHjWC6|&d;>71ZUYBgMzAf8B zPl8DI@VbD7QFa%6!gnw|85pi@Th zr6&gUG6$)zs19kH9|Ti!3+bQLP2_2vi7?3B6tC5a#L7fP+_vTurnV*!!)2`CP@$P{ zQBXGN9RCQ7{o9}|;t_h*tHH0rX~f$%9H-a{=({81;SQ`KOJ_$3d?#C>6yJg@+Fl7G z6$Jvu+Z~3QMLC?3x8(ZI9ioiQPcq{|FD-gfhxPl`VNBC*6g%Y%h38ZR%Z+l$k-{PF z-y279we(`{vDg`07auOMul_}6E!m3EGi0=S3QiQYVDIFNOJXZ&+`)zvw?d48IuehC^@q z$%*(YSmp8sj!CPI*%@xiSF}EEwI;h$fSM+}z z%vGg1H2zR1TJ7i{obV}dMEfP0c54VL6i$-eX5)D#`+Y%Ws1&OlA7IA9QhM&*M7$+7 z4mX*Uk%pArI5u!1=*20)7q7c?+GrMVByvR@-B{XWyMZ1+6RxJf8@wvwL|LPHlqriK ze?+;VwG!h{$|(%*8jJ|uR=&g~Qt7z&dILPRs>7s|WSsiY2wz?)A>*Y*IW9*jp|bl? zn)*tD=P37>tZ}I#ZIYE-X)wa$AMQd${d4GYXajro06M|B3s-#5qsO~kP+3(d^1^ee zM5P(oCRl+3xodF#unMkM&%iCe{6TSi9#oZAg5$h=IIl(pj_Z3!vh`Rv*z+B3cKT!H zV@s6a8wvNF*Q80=Az+oqK-6a=+*&+Bzkk?(`eFs3_+Kgp_5K9G-$Jtdr2*{BJ_1XY zOb3r=1noIWz|lEJzH3Xu(W8Grym%JYD7JIeM7g7j}LhpiX@?_!= z(OZ0pT-*7B)V{4FNt@$fQa2?vf2-(H)l?J&?<3RP9LTbx!Q4$S0vp_uA?IHey>FT< zuq$jqEjuTO?lFN&yEfotKY4mCG!5+1?}GS&Q#j+98{Xc3k*K$r5E8W(hnnvbp~4qp z+j|e3R&@ySBunVbc4c1e$HOpCv;rq9=<@t7uYixa&$)K)$FSkM0ZHl?WxND;h{TL| z95)gU%3f=sLDCdlUQ`nPp;~-&{|IrJnk1;m$Or3}ZuF)9E>IpUBEE_8;5dl4 z#AI&$pTp#vC_U47aVCk{dxiWdegT1giF7ztQP>(hM9)~C!I0y+)Ou4T`Sf@lSZICc z9(ayuyHOQSRW*||aaq`9!qWGNcd+`WJ)O%xBAUUv0LK~T!OVr`czWn7l#Y%U&Mx7? zxy6^UC3!EowQm_>!D5J7`kuN5o`h37L&+aQ1FHSho35^3My-n<0nSU0cgsmRF8#G0^1p$IvM@y-{c^N8h_Qc~_GwGFis(AT)KiXf<#r8TI z=x~@z2em}!MDZQU{goxSG1^B?Ja@sPQe#P-j6GcPXo8iY0jPFQkBnxyl7MUt>Xx{b z{CT_^9v*Ihg)^hUUivKHVlkfU&I*)0Tm&Z)M0G~xEO3!-pw*M~dFxcip!U}r(2`?; zpO1bGyzTqAtpdAXwo9cbu^#*{9_S?OH2naj29~$4gXCf8#)0;#uPAf#^G1oL=8# z3OR0v8T0%1=|HGC_rKsY{Gv1-+jVy1IpZ@>f7XSW5*Zl$ql~;L|4PPL0X+RT z%DuZlv=?6#;$U_;9IXA0sW+eDOSePlt8)iuDVpOi{f(fdt_y0;7wC3HH?+0hkL~d{ zVa0F;)jC^CJ@0a%G(H9gx)u<3^HrdDcnt(-7op*vPPpkhKu<1TL0)U#MD2UgBw1Hf zPbchzhPruVLhS|Iq8S8tB9qat#T*m+@1fFX6WFmkpQO-6Z1K|-&81P~-Fl%ZpOA{-;rElOK%z@dz}=rwN$gi@5SyC?wVB@DBO5lC7Q^X#HX?<`)Km z;>R{Dns9u->MXLna^Alio@*?_WXFJ$^Pl4YGJ~*jFnddo6j|`~Hfj6Oi zIyF(27jPt3@al3gJpEgVeOJ;2Idi^3=JyGpzE@eOtEf$m`PsmS|GshUK3#!;$2QYHiEn$O@0`)$D@CmZwA9?){J z4*K=J549X?CKQh;gvr7_;>WqjHF@`o`)qVzN`2oA=Zh3cn9Qa{XB% zb@&qLh5?A4`UqOiHlT)KF}+`BN6QaoknOgY@u7Pk&`D|Ft>^=juP`LPI*=SaFbg|} zL6GNv2R`B;Jy;};+vfeI#?~9*^~p^b9Tp4{3*ynCD~`-d%RSTi0LHD=KvA7|{z+5@{dY4~ckIVCT{sF;5i)#=(mJdEt=)|2^y zaV|%ZeNj$(lXl~^i^b&oRbyIoYYHq(Dgcv=r`R`El=j$>PHrs4;X*Q6QOfMQd@PT$+RO5}6)ZyJ^ zbV+xrtk9_L0_~{DqoprT;>d#taND3lv*j)bUg*lRCMy?kLmLfvA}tnQr6k~_eJyk- zTnz4>kV0amKz6L#g!_NDl2;f1koM397%7$zO7)ndXJ!F)EDNItUz?(|U_8$;=f~QUybGJl0yEY zMkEG>jMtGTBx$QWPVjV~!5b}bxk({ht@p+6BbxZ%i>07ldlg24yZDb#k}SP^fvPZ# zXx@zo_{_hdJC5WEh{Uml=;b_qz( z*hv80%`o_15tvHbL5*00VBgwDWc@l%;MXalOZ6KnyGKdTH>Usq{x)1?_LjTI zP!0xEGQrBu9KRj>$Biw^B$m>fvF@M~e)@Tdcz=n6gPSFg^iCI;EiNWS7fYaD)O+!~ z<}}=Lam5F-3?XuSG~uXbppVgi#B!fEq}YxXK0UD+J#NL3I;TmzS>N|U*#HA(eNFIR z<5B!N{E~zesNjurWoR97m?#>qLQm;D@-#~t!h*KIotGzJTU{Axw^~a(IwPPwKA1`$ zKR}1F42hN1V{$dl0bIuPk(tLU>8PVNbq?0VnzU(cq?_e~KzupdB0lXK+R zomA-m6^O-``URVJqQL9G3Tiz+05?-J9N#&KyFh}E?~3)w=;S?Qe$N&hmuZKeq$QAE z8p1J2p(yki5&T~xye~G0MlWnpV&4K>5Uehg&zef+4N1YG~}dZickFL^3V3iZ~fQBjZ{& zfPb_l9204254NUL>EU*mZ^6OjF^93_tqTgYBXQz?z`Q?t6y{E~Akr4L5IbunsCV4O z_d$l}Tc^mzk5ckXUK>jKrr{T_FTjaR#EVjTC^*~<>1XdkSFa>*VwW>%8x6u#-6s5L z;tWdpv2=UBkh#*j2)L29f;V?M@SssStUhQ&E|U~bL=)Oti25#FN)iS{ zKHg_rI1;CimOK-%_Bjt#Tw}PLU?R%nN%4LtyoGxbL$tW}1o3cR4F2zC^DeAPgW*^0 z_;JrmL1M^0=#rCx_1mTCCFdqg*?fu2zF&#A4<^!x5KAh4RhqXo^aJ@6_#bv(J_t4b z#us)MtD?i9j{i|~=FwDrZx~1BsU$;0NQMwHp1tqcQb?jG4OAKs4Js zlBr0Xy`3#(NNExx8mJ5nN~*7l`rY4J_wTdrU1#0B-}iZ+=l$>=?vCh#)qWZzn^C2! z;l{hp?BhF4mI-}brTi8bTl#HcDF3AI6d!vdr%F#T1eH2`U`Ku}K+g-c>F~BqXphjd zB0GCK3f)kPc9xtIxa_@K1ykz;lDlL1hc6-9^=n(zFk{iZs$b{C+!c{cq#x|4NqPL*Vns|;!5riFA|WD;sSz@lJ_G<5%Q zD}Sg(fR5)JM@5f{>Hf#L{2Fu!ewOD#+!uH~PDb=lM1p`u<-%z0*~M7HyKOKL4Ve7)&#ungdDH&2}5g zGw-7Jx^L5*5JftWp-ruS&ZQj{m(Z3p@#>VH`8*%0NFz2c7WT34cx;o7ET-I{etZl1 z<<-M4?n|V5vKF&iOXt#_+6dM@HV=gzYv4j^EzsN%ttG z3vOK&Cx(q4?5hpu(C)kK?60TC=}5aS-)DV=hFOWz7z;7vrn6J%fAd25o5Il9CEhgH zyNI8a?u?2R!-P4|hN|pKLG;U2Kjsj0*h8}VaLHHp85XY|&b6z$dLReo^JU0A zIFmIx+{ho9zf+i7S;S_yG4z398&8{bDd;&s502>3=qrz@5TSx%+Pu-$qU}^^y9W)3 zIz`T1KSsZ7>Sp(k+(W?QKL6tH6;fmL4dwTq7OcK$W&sCL&9_ zOm-+d7Rhf2K{k3hf+st7v3b|_(606jI`xJI%O##*i<|!Am-^bHl7vjU|BZu#{Mlw2 z{wbMG*)Rjqc4hy2`Y_xSMv8OVKb0v+_vrfy{u=%;)A zD5iWR@)~@G_G_tBdkcMN{--0+?W}CH!}U5ff85N=eyybkW>nF6g~9YQ_ffEL(K+-r z+aFn+eTSsF8tL~qEBZF(AsT5pLqi^%;a|kg=Rb{)6PWvqL+Qa?RWnEa@m&fG3d+i0 zKUgP&PQ?xs@a-6C;MCCRz3b_%PAm3kPMRP#ypONgJRS`=AEOJ-`k+TM?CA#?Z94X8 zh9GU}OoxU4C83@FCDBg{L-;zEUv^iUcG32zMOAWM~EwU5`(FXwI)?0*#5P830WWIl zKSX#`l{#N7;!(|8foG?xLtfHXdUCBkzd4mP`hB`Z z#F&b>C9+638?C*Tj=t>3q477`_ycK`Y?*Ti9XI!Rm3Tn`@K^2NFIz|PsuNFAg`i~q z$v$N|H^`ByN0cEs=Zk33e>^){;DB5XCiB1V9;a(1vsoF-FMRc|K9%nUwCM3p)WDdy>aWF3Z zTV?lEID-@9(vaB`kx6kU>dT#ku8DfmUHd!ucIG9G2|7=Id`PbH>6aGv(njdC!3I?F z>mmE|+E1ifaTz%sm!m4-)--*E(2ugXk4@7wq3o=1KK-v1^^f?x!i*%V_bDX7pX+EY%2`iEczYqMScR`CPAH z>Kv(xitOX57!jjWg&y&?_?=bdIflryt(%_^oll-;y3yaS6sp%%Y9q<2KDtK16e(R> zLU-0pLCf9F(B}6UeD*vg^o+}|y1MKSDq9uBpOdISH*k?(UjfDR`K_W^jcvU9h0qkX5v9jfbg3cVMa zRSPzVP$^YM_D<9){`xp6s!Sbe-}@Gn5$;XDPYU1-*W9FSOO>f!>1pI}o8c_Z6LUzJ66BHPn4ebYWXzsKs!Mn4D!sbRSoJ85zu+IV5S!;TH_*H&ubvk>0C!c=G6=iq4tQr_R#dj26tvc))O3!Zbq!Syj zu@kfV`R2)5{MH5Ctew>Y`sCp)H1EGNe6Dja@3~5x?!G&-8u#Qo^?GE&Tj`;UomAm@=RDe)NG*mMlc`K-zL;r zPOzJC3vXS}AlNnO1HH1Kf-VIG?0UUO{zktYn^|^-_WRbc)oSMi>NmUj`R?9S;B3WL zF27PW=aw&uiZMhI6YA;jY773jcs4zfvYjeAiZ~oB4MP@7s_2fu{itE{Eu?RtMpe}p zv-01sqdi!gF1$9G{T`%NoxAJ`-D>>>j#vB6s^$L{yt7%tM#>zfD|NqCMOSV{QD-dC z^vhxnA4aBDH$)|}g_DA5$V^3t@r~bk$K)x9NlZuJ)MDCjHHTK2h@+uLeQfwsarF23 zDVovxjo-7g4NabXfj_Z+C;ImG7ArC_0S$B>pw=D>(ZPy5IxL<^!7P2csQm)%8?vsx z(o)Z#JC;NB-?^hnCOqpLl}rsT??PYZOQSWWPm#j2D88^WhTYJ$nx3+KO#>$~Z0Owm{KH|=7Il#W}(51a%PGoUFAKG(hoWqrqji`O~N}Bj)KWh4Rjs^|h zW82&Y+3WVtiQnur#LTDYh`oiw;hP81=dsbM$y5M6?;K`wQql<^K1g_-c# zj>7s99--c$k<^seb(ngLqq$91?48|Tk>5Uf-o6jhV^I=p?1@?Iwsu3oodOZ05|K!s z|B9tfXfod`yN&nN9pt;iFVX8*s0DONqp;u#Vyf_jzWrn)SbC%nO@5<}CiY1Py|%5W z{^w+xlqk;I(dq1(2sh+n0-=_Oq zF!6f_Ya}nhIv5|NUd%)qJ8+tp+onga9~kD>&eWn8qlj92s6;)-G@*?>s z%Tv(FtvzheQ{!rvg5yX!qYTOGVAOLay2_$j86`BGLaU80^36X@DH|8b7au6#L)Sdz zpT5{gU02*f&*fH7IxvHt+igjC<-=&nM`MR7D|s~PQAQUSg|Hp%Zm4~-G##ts+3Lm% ze1~vF^DWgZcGYiJc6_}JZ~pf?yCt*&4L4@+S-%eRbJJDm?=2<5bLK2UD?|DDn^S2+ zSRrjG%i(|5`OyVcS~N0AXt{`cB3SJZNv$R&(x?|RQCqq>4p=!xTeLp2gVu$}BEzF< zx@8`}q;4i(u=*1koZ?bFo|h$KTVnV%{XRkfeGJ=nGH&!^jJrK;Cob*=jQ;xvM3-stq1S~RgcgW_QkwB7R> zyLM7D+M+d`{&-b{wtV@A{!aYD&%5V{h7RAS$RJw9q5R$(tmF|yHnxtY$K}jXr z7M4W=292T3fhDy5^C6g6t+T3T5uB|<<)kaLS99`dAoy~QI4D~y8$nx z*M6m;eu?~Pggd#rG}PHZJ~aZ*QxV%BRZxvhh`>Zu%T0$*oL>mY|V-UR?PhX9}%N~ zRya;0&F^~z3tt{4b&JCIjed$$*XSx+sMAn&e9m(IMC~nfN$7+7Z-q8d5caCe-V1jH zh@U|>B@`)YK8H4}_ow%l^s>4|>(J+-5?<$}2?{R%kJT+0Ks&wn_=Cv28?@cI|SH=mNCALf2$%Yxpqdo#lXal<>&L)}?acUx4| zLKuS_4trCLjrlBkC-j@2+JJKBMo^b?!c54)OKi1w8fx^}jI^IFr}I@>=!of2)^%+y z3wx4L&OL8x8l_8XW?P|i7vig|4&tgv6;t+b_Z7NywQ;rijYafp{3t8!^oDK|&87LP zX3$!f+X6-}8=Z#J=(tz5^uD8GwWpjT>lS+)6%;IB`;i`-h_%rn=!bTu|DexKEMzm4 z?(p+&j`2*JII550(9k&(YSjOgw#d{XDeELEd(x0U<{X2D>eJbAAOG?x@1|8j1!Y<( zmdd^~*~s2>n4$ry>aZGlt}7<0@fzs&&x`2K*~fHspDnTx`crBe%h`-QUHo8{jYIc%H?(Q$ z0an&anKveBXwz+}YQc3Ao|=Rr*|02hF;dB)|JZ73THr}5p9i9A199}j;#yR*zzlsh zx`bW`HKZr?kIcJF3O%wF?N++nI_GL>HX9F0t?j?r6Z#gKEw0n}u1oR?h|Mhyzp zk!Q*|WG0ZHM{QzR|FExUr5S~Zf^N1=&WN7*X^BRjeq%4Yb+Muq^XQSpZ#?*W0WGlF zgRX5JPrddFHCv?wBom#07LrxSV)GmNBqE=mZd%4C|Eot5^FLDnd(DYTGi{G+~VtgHqnpESJA#WC-zF=BPx(cV+$;g zQ~kfaRXeW<>yP?VC@bJAe=)v^hSdD5f?2{l^sNK66^TQMlX_XP$|P!%@|B-HQ;fb< zR6~L9yLABwshh{)c-(`5ju@*ZRT?EPg#?Um}5WFwYB_ikN z!*lxr@XpT;_{D-wyl_Sg5KU4hJg#{FTFAD~$sCX6|C<91nUDB;b0*Nz zZUw@ejvXj&#>Dy*##p~re=*V-CNB)x$OX+9Y_PkGD935w-Pq& zaNr{1FW|)N&%jdw!DC|ba43@{U`sz&|LZ+x`0On1JCXv9&in-|M;tKs>=&-yFcFH@ zW#T5$vv`hzDB08sA#*{BB+hY%BPWM}_+#M?^J(qi{3lspW42#tqwpkk%jS||!vs!q zxDi~NQ-)`DPQnL2EhZALM4*4*9q_L6FP40Eh?IYO1df-7LW^h5aeY@Xc|QqYlPDk9 z6Cz8V8LPp)XSDDGog{4ib{ZVASV{if9|DH19&k%HAZiLg685r>`fU-qtYtUxksvQG2q5-v^Zz9RvPRW!ND>1TKzh0B*}qaKT&sU}nf9?0H$p zDOB}@!5zck)2ZK>`|ki)kl@M5o?6Qc`ey;<>4aNl9*Dz*2-bhUe8@M`alqd<0l5B= zB;|{1fS*PH+~K8+BM;5NG27~~BR;}h6CcI0`zHg_h27v@{$$dVxe65A@Pjp}sg;Fk zkwj^WGEB{020wSWffRLTq9D-#TwY(q<+JYDm!JQ^2tQD*nIW&Di}4H9V5sSR1Ai}# z#g!es6?=zO$)Zn@p!vX4FnCo5PjyKF`?k)jQ2n+-k?n>g7;v=0)IjQOGb|kMYosWn_=_DCpH@V0-)` zl6Q9owqd-1(PIS;^cNkbARGmB)aXp+a19pR`!1vq6HMHHT% z2Zc8`LhaTiBqhTaY~7OroI3V04{DBrR3Td-d?UfHpBC7@JEIH>Vgreb#cZkm$BQ0?$eo!p@teoM<0QhRu!OB^e#~ zVV)y2U!g_<+?Ao#{!2{MkGGgnzW}VcFmUMWezNOz68I2n1^55kjL+RZ0GfZvk@4>) zaZ;$5E9yCb{}lfP_1g8=>4Xkg;JuRQhaF=MU0xyNW;fwT5DVdVZ*n*?o)}$-f={bs z2;E=_GoB^FvyK5i6+0GB-9Wj!O0deEUnYmY_{ zq>})X*G0jP*8b$dXd+oGWEbM~D`Dc1r6e&&mI&@ofqlCuE-|(yb3NQ($}}A!GDjW; zgqe^$UwOF0!;)Ywf{aSI!rc?%;MSEwR*>p`7?hEN&3qCGxfl%tD@$?V;C#sV9wc#h zZK1yR8u-4)g``(6A`#{K@al6dDEdX7^sjgfnhp8`u zh1{dtoGnS(J^-FSw}o-zClMzFFSy)zKCx16KHLCZQHs37A_LJLC3Dlaiuw_!VM z8}TL7dkf6AoJuZxuOq8Zu7p8bHj(N7ErhApHbBje738k_e<02JDn6hR!W|d?jIF>v z5_EMoysr}mzgotSPeM+v&iEqyNqPslcx5-tRA{Wc^XDPZebs|kpLNHd;#Rvq<#!Rd7khZZd3T3*Fko;Hzt~!u{@1&}nozng1xBgmQ=Ay2M>1DcS`V zZy66go(af1+eo-*<#M7VmqcvtMM632-B3JUiwK5eh)UjG=yd`S{>c((cT~uv74H3g z>$`$HFB<}}>zBbQ%b6rRV>+28WZL8`w<4&14hT%#4&8)YKRv4u@^|YNI8~4c{z!iV z-#J5a`4?mw_r^l`(}84T_%RY@eiTM~E+Vdz9l+d=9Uw1%Igz9Q%GkINNmCo#@z4{9 z`bCk-v-9BB_lc0(8b`E-zJRpGV;5_i<#w=4be;dNGi$vL&*=y1*IvMP#Tmnko7F8!S%F!eA@| zJkIRFR?n)jOQRx8aBRi5^Ru`c0|z-q-v#73HrOwA5Qpo1b3obKQy@KVIwp6Y;nQNl zSbK&oBO1F6Z+Cgd*t8u0U1J7huiyq={zwOQw@xJ<$ps+bZ7R+$-U;?Mnt<*@mE66% zr@+6%EdJEc2})KpV?1IC?^zwMT;QC7eM%;Qg)ZZW>XBPuZp%9GE!z$p_WTa+QwRz& zZHNwR05Te?#CLilXWcd#erQ!Eh68s&P1quCmCp=N?7j{phfcsVF1O&PMIO*u<}B`c z5D(UPHUVGTsl?#KCosFzk-WGnPC~rI;qSmytSqJux9CNJY9Y(1ZhHiN7JdtdpHhHp zrb*+22ZHf+H4Cyg!Vi$<7(7%_2@0je$@EX7%n}VfSnFa;j)u1a!`4Kssrrkd3q9bM zD>L95FCLg|d=Ac5J;TB5Z|?em6wq;LJ^qq-7Lq7N#6yDJ0Pl%M!dO)O(BlaFP8DuU3XlH|2bEly5e zgtzZpgrns4(_3P!}J?KM91MxOiN zvculVQvvuqm4sUyim?AXXBcuv3a;ELO(e6%!FR=qC+EQ|?TN6(!vKybO(0y&IQUKYpF6+M0jf>R!B!HUa7WcOFmbv7KUPC< z<+>a~iH9aVx3)ncUKlP#9i(SllGUa+A_ zf|MFO$1ZF#SeyP1j5Ho&YCBJYQG+~AF~*ckwwx(s6xV`PZR@aFxjgJ>o&eY9mf+vl zNAPI85^=b?jytVY!E*h-Bk{`$%+D3gX) zWJ>U%^eOn}2XXTF)C6J^BPQeyy#&(-PT()rN~A~FLn!G4;%|vM%rq`6<;rB->9=cb9DgUnGfJkvHa5aEZKPb>+4e<(NTR=w|fkT-a zT%~0~43HFA;G~CbQ)S2s(@G%cUjQ~9DdMgPR)JY&Q<=D+7-qjn4Zh&k#e|M5!ApAm z@oden*d;(3U#b0JU!kHz^w45(zaNn3|BeCe#S1`&OgY$F5(QG%Rbu0ZuYh&FEH;;( zNbC)Am=|Zef%?z4+_sh*xbq(di=#0E&orLlg> zRvgkb0zN;t1u8R)z}vmw@r7F&aA%7&?3bB?BRgjh-d+j(BmSVjbT)C!e~Qn}wP0Gb z{{u&WHU6c=fD`L~;Oeq#c`_cA%(j&=((XMtS+yT$t(OFXz7844^*2Fv)1*vtDcP(341^ee%OGAa|+?NA!zuUmp_02#k z^bugK-AFtq2P`H_!pwO_@SDv2hSy8tHINzw?wCW7;P`4&#T_W-P z-#z%z)5qWg9m8TGud&D8iO_9?$EnpVIKZleS?BS{KJzsIRgGW4n~&$f{UzokYVQSv=fHXmeJ(!EjNIIC9{ea80rRzO@yKEeXst2}%T&C_ht>pR zvhEY2b$VW9H^877mM# zVRbxeSNNO{HV+1w2*bFIX8KZC6?E#qE-r3|s4k{I<<8*x2e9_EFgnQryLf z_NXL|wO)q%X8pvGYHzr+AHH%QByWIYLXOXZJ7#3a{yMmGFdB#?ieOzoS2Dr51c*Kt zg~fY`y~M|6p#S9yBi!6jIc!#pPwDpKN86u(nYFL*sy1h0vC0;P25i9zB68esX*Kf1 z?Gf;V7a0@bHi3^iS1J;9&x3;kU4jY@a*E$4Gb&9MU~I7*`0&FVx|^7R`Q#)1FZ?Wi zZ74}}Mu+SVdk-;7hW!|g^;f}4yFn&gBpVEB7J=yI=it%u0H7x1_k?`W0=osju;lG9 zro2%UzEiEk0p=y3D?yt4-M$eU6s`p!DSh0vxQF0+CdEmIl(5v`3tZb&0Ni$_+vgP- z*}tf{2R1EGg!76VaoxSA*roonoxI>L@J~|5YvqN^uR~dk&!7V~OV049R-T6ri0m0mtm7Nz^TzMEzGed4v#pN{Im6-zlf@uW?g+lJW*i7! zcArrVJ%t@ij@eHefS^5fmXJH)$ds520a=AoEStR%Z(Hb%|7(#WXC6$)kMwn+ozq0v zKjn{*b@+riaIdZML3Rn&aN0ncH@Sm1u`Y1i22~)wPzsJtPshGqlkmkavgAU?B~TVK zkvu?qZDSwD;Pr$BUmcZ+OG^CznW#AK#6(~qUJSHzhOpCR9tU)(0RIj3!1A#XfznFM z`j9`&v7?i*B-79Mo{+$%GI7lG;dbW0xELnk`V@TXM*yCbwt#sO^O+gv^9FBf{t3#| za)Dmd829s86tjOI5&s2Ioc(7JA{sawmtL*q>a&-D%}-}Adp;TBslwk^`85dtnJMHp z#YN-l11GUyM1y#Fn8CsB)nIUMB7X2v4({75f`7E~%;~S68Pkc=I1h_8Oya3Ocl%o=Nh50<}xU+a+lj3=5Tb=Ok-aY`7THscGIoag}@OOo;F zVpGyPEztgw`yKGdDUfTO8OdBTF5!+@24JJc4fv?76j<#$9X<-2K#HtPD=my&xewmj z@S_nT7V<54_9a=kOnNCUj<~|v%~D~aL%0$jN7 z3i$h06z~5WiO*cG2ixOJz~XlkiQjrzSblC5*j-x#c;+LBw(8{~p5C(GT_eH!vL-!`aFy_|n8XIB|wD8G8A{K6AoU zF6ifY_^(2igqc);QRXM3+7}D{3<+y%2_4uQY)m}8&*9O~)8OHXB%oVT2)5MR0Ez>E zq;AvWrd(8nZu@U#ou@i)6pVORN@hH$>xr$P^`FZ7bwb zzP$^|ce~?+(ITK(ln6TZ=@a7^8B&ux$X)(BiAY+y!fwqe#OuOb=2Ag3?kHeM6pJr8~*d0!J@dgAOnGW_Vn?ZTUci`uW0pF7m3a{Ut?;*YaLNRfRqBfqX6_;uGYo+qDh(%q{#_wm+v zwXH6JHKoAu@^2vI>EWgebAW{MWKuf$5{~hhKwe0G1Mz`UP~p!td`#s72p|&JskE^2 z*U&Ct(x3!W()zevO&whLwL+$8y9r3WxeboCWP&1*Ggz^;2w#oU1WTf=IQfg?n5&=f zf-|B=@Wx3Waq^C2X2o2g0=2J(F|+W*+UE?Rf`krun0}p`v)dRd+QkC187gqxb`=<1 zp2ry*moao&kG=E62}D0g4zATwf&RmfK*$Lj*zoc=Sny;u4w7{z)3zMu@|%7E@z?Fl zmI<9W?!g?`t#ccDWk``CKU?CU@f;jIq(+QZ8&HWF*KCLJCA#1DpSdyRW*$NYIm?{rZx%Rw*+oEc>^~s{l;-3%W!JN4)Wq) z2#ij>Xa6)&pQwB^B&FIBjB-F8&f29!P9B&`M6n9YgBNkNa{{;iimLGOo9yW0Us)jXq(&bXDy=n&j@1YG$>X<+#ut{L+;w@loeieTBpa_&! zoyGO8?l^jrH}qQ}M!H`LS@P5Dh;nQSXN>Hjjcf&%S?7cG)_{tW1Wjp~fatm>- z)D8PZYAkV0Q-I!*E&$kO<2eH%K=MK<=*i2+Dt2}DTSJb3AD#kuc2Pavx>yr@iPj_H z1#iIcH+9$-+k~g}wt!jX4R~-~A^s%Qhh0v`Vq+UKqF(6;i;kFq$JsoS)SyZ%+$({4 zuo~H8UW!kvuOJd9*27&!Td+k{AE$Igh6G29fj_4{b0%BAVdrl##N+sG__(bW&v+t6 z(3x~BC=rs@_YZQ(w{w~4H{^)#(n2oxU?EmgRfIo-e4(FbIJuy}3+F{Tc<+ae@JY-R za=R-LobNab#uVm3S)xZ`)5S@N*>rd}jfE=$da!xOc@X(KABZ)W!FF*Myc~3Z>&w%HionfBrP6hVMy#SP0g*%r;s~fz!0Bm ztXyycSjjk%%LXRkymuLRoUYDI7%~9YeO<`QPfqaV4c|)bv&VpppYXgGD*}@f)JXd9 z64IU^ORPTs0rk^On4Q1R;WH^&AVd6a<+T&*@T&SLu;<1?qP4RTo99R2C&C&jFU1Ag zh<~+j{qYLyI@XLET?2^uvB^-NHVyAfP$Cz_oiWlm4w7d*#`g}IlZW37;2x)yfULib zeRkSGAFX6y?PEsT(@s^^*vOOZ4>ib>!=G@-Xb|Ii`6_Pik|h^iGr;$gdRR=~7_QzN z4X(utNU?z|#7q(}9Md7+M|-guCk2lkGlj~ZV}Sgc0^IXN8|&#U!Xde4pzT}71T;jg(>0)+&+l-XX2Nw(AL9K6(co#Y4Ak#cCr8EBfQ_EB;TT!Rm~3!_l{X}CH3P_|_eVf!{933w z{x_IO|Kal?@!Te7S#q*^Iy|)*aq7>!u(FE?d^IBzD<-gvcj|fYA@LbFfBgck&MILH z-&uj*JDPFkqMuk*q|E;H1HjZRE5X}LYr*lHV7ygxG1sjiWNNlbfLWS6Q2j8U8Ch!$ zxg0xCa9%*Bi}r9KzGm=MX(nE-?M9Zp5Qp#I-DjLLe&9!Mr-0M;g^b%+FL3Y~#a6Pi zMA@^A`9hbIrk}?4lNvk0p6BbZ!Axy1?Dz#Q^6IKw61AM^C8zLSza3aUTpsj`*5c^y zc)Jss3h+jp1#vfBiT|B3C!Hp<@zR@0@NCx{V$>r;Y{gfT*{)0Aoq5l}n%aGw-;;Fg z)O`Z@-LfV=WA+TW*anV&P#_wInz;D>P++Jf4YzIDfK&cilUJd)m@YMYuBP}0SeX6* zt1LEy-O2GdzF(bOyG=k*=}LTBZ~|PK{sd%MrU3<2C!&%i3=?BTs1e-CX{zi1&1+8r zWy$FzvMHC7+B6HCfBzf^ZoCBRns3>{XMHWR!HtL26Z8KQq`E8h8236^D< zgDXB7053X=l{UA4yL#qW(daz0PP82NeAg86g=M+hF4N(~!*lFw78{Y!=gPn-9%I*% z_rNIQBlu)CfbY%d2C`0RVCH^5Qe$0$d;ht^>sBh@;4e)wdi)INxOSYmUn&l>XTAgf zM8m;;*9frP;y4zUkcVB(uYghPCVXV5lgq6>0mAQ3;VK`t;;VsAv6gc$Zg5iu_K)9| zPnvH64vs0232q(sDR)ock-_tf@0cyL{wrjAUzvtgqHf`nqKa@zj0E>_?iiSpJQw_3 z;7wlNjQ|-tEl72%DL{M7;kg5$c+awaFiG__{?`2k2VB<0*(pX!E4=HbCb7%?I+$b2azjR7a~FS+4kEHY!LR-7xYQ(fk zQY4(+ZpZmo*WfxK+cZ4P3(M)};9|oTeEXy$6Pz~-r!*1fyC4qidUGECSiF=O(kTV= zrVRj1F<-1un1?mfW@hm@K*z=o4&2hfGuE2pm!TBjv@?Jw zLMW>Z9pYnj;wXMhFhgyaM@|=?b~-Jav{M2cpQx< zoqbi6P6ID1x9>~BrHXRE<7W=GJDrF({g^}=Z^YYA2xjo>ZUEm;InJ!vu@mpHOTs~q zlsFY-O?dZ&Hn}UjgU)AEfNT8|xrO^VFdun<=6(K*nq@o__~JgZ$}bX(xoUt%w*Ab$ zj%PS$Pqe-IZ6AC;Di2Jt>jtGywU`F?8Q9rDgzV;)Leo+yV5~97Nv+!gtoro{JbI}z zt+WXrxW5CKM;MXTXdb%`EyN1H5OmH|C0`fc;!2UJJk&cZhFD)FK(o;<~6o?w*YiaQzF{$`@o@}!n!WKgY&GA2f0hjaQcl2 zWJzNm_;$~fxY>_`M*3O6LNc;)5kXkpMVwrQRUj=V5gh+G1Rghh#%nHq0W>24RE=1Z z(RtIrKNkn+xr!w`7ljK&)M4(M68yAANjMiaF~Vfj9MU{f+abgT_${5Hnl0zTp+X)|Eky)|IC$sBg9$_J%#`oyK|Ja@8Ek!-Mz z0piQWLF(m;STETZcj))v3kuy}&I>)JdL#=!H&!Ojx?V8g#}KYLq=+{uPl8t$d61JP zemM5mE3n4B1&4^AXTlW=nLojajC4&Bex3G*>0NIC@%GYT)p zZc3oUYXF~LcN<%|6@Zp*OE_#0M${7SGaJ0XHU3uru&);w(m| z!ea+>al-8pp*P4FIxQW=IN(v8+?-D<-m7*ASxPAtYDkNgDb@TAH1y}F@cMd$%6W}VB1W;WvAIA?pu#a^0 z1s}W8IZ2P5m6rmI@FSJacyw+(b95jXkCoJbrCO^&>rPR4+EYNLuDAob!<1nU@~Nm5 zwdQOUJ~IpV_klapl|YRLwRe7d8-u$up^oG@A~D+^KF#c?G&%GP>vr~GZst^?n>`!X zUVM+=PjrHFYj@%#jn9=I5}aW77Ju^AVLaIS^@Y8~(N0_tCIRS|dTcxY0j^lq$LVTK zgM0P~d-nH}$ky41KyXr`ea%!1b_~8|E_<58jmC4hxKq_&G|3zI3ZG}E9*JkoUVyF4 zEOB9BDl1T+GEbalV z(`j7Z>lS9sm=pHTsAX^4R3uN!TpUeB=?2q{9Bp#U_rW&@h&?Vj(I-EPb8#B zu-H)$7B(K*>>0%e9rA#2jw0-cV)48w+QdreV@ek(06}jgVBPP{K(ss>yWT&cZ z3k@ldSH6RP>#PKESAT#VLT-Egp^f0tSxI<#y$m?DDGS>aj9|~ZQ6OS23zA>`0Oy~I z;!E{6L7mfcptEd>eeP^=A}K3I4!g_*zQ?sl&}lokX|#uV^wSn@F`7ko&B7pM@p|xf zOx9j2JPL#e@0^htPdSmf0DRGZT4`DF65nJTD<|A7!-5GPLB@C)C}t-`0{2Le87ZY; zUCtfgqPG%nt82wlxDgb)3ZJo37F@P4g4<^K3+ITHobHCH@NoKWymaJ>kUxACj1QU! zqY7o<_Eq-8q%n%yElb(y|0R3%9$~K>mJbx?3}CkdJwV)Wv%QaEFOHbA3M{Q& zOXL|zkofwp@EzuY3nBi{U{e+7y~W`-uF|)xk7|P!dLR4>O06`@^&_WB#Nb~8W3p1y z7(dQkM@FBh!W1t_cu7aYUa&m|T)FZOR7-Z?#rha;xc(Fv zRpyD!1Q?L$3>}*H0MFWY!uj<);_qoB^g2c0J;UjkDZP)sa9W@?-+-L+jAb^SbcL&Z zOtE(5Qy}A0%ZY3q4=VKKNoKt?Y>F-fGJ4;+Wm^o$p{IF3NL*pSbpTvN@8j#=UjXaP z@9=!Hd9d5#1y>ill1Z6Y#oT@;0$po8iJOx)Ar%`rNmn^oGEju!>{71i*a^J)w*#pQ zH3qT1n&j8^`=Gzi9Qx^K5`XoZOmSu!E>_vaxpaz>`uFYJ=w%(SK3|vYzOF@nFZCpr z7c=qt=5a8j^AdN>A(69kw7}D4KLX{O`Xp_WC4M8e5og|81C_;>kh=P3V0(Qp$eHL( zqI*`r&M8;mJ5%&;dxG>TfX1OBx!DJ<_T#ycS&-))m=iyKF z_s4OWk&%(CQW+sDGCt?r&mo~{6(vbB3YAJjsjQ6bkcvVnX{uDX=iJXxLPLcJk(3IR z<~NB_zx(?KJh(pheBS5%8qe8qOQr_w`<8{b^|WDsRSA+b>k^+?7Xq%2u7_f*HQ_f^ zg9P(7{`DCw)VskFtpkJ1hHbZjPmv_?ledKR^MY{DoxfPBc@5b2{k(7oHlG}7{>ym3 zDaHzyrh(RJeq?E{0{(hdP>s1cFuQXPxEg!{pPRk)7)}?x``gC(pnAM zE+^xmvl-ya9ieYk$CqrG{fz&~Ou&druOM%y1;dId&gA`DmI?Un1Cs}rlRf8KnWN8@ z@%a`ZXS_=V&fgJ;ztEmnDYr2HpORV7GN+IvNaWnR2vw-Xe z5!mzA6_n3@fLlJ9!F3;3lGS&j@RmnU_ze(2_^G@#>-5=7v{}? zTdqNDc#Vb4oBjc3?@eHltSi3qdLfb6wGY3o{mmbBeTr|)JBIhq-2TqwmdS7Tt!6g&8F^DCenlL|H^nZTg1Jf>4j8h)*l zVcxff5|f)|WX_a4aHU6`oR8ZB=i5$(WAHs*s@04)cI}3~&u!qf5kqE|#R%WsH5A74 z(?MT-CoVj(6hu9nM?4G#_|DB(((A8%(1fH^9z#C+IrexLWSBED8CM z4t_75gcHtffD2@=fy#|S-6U>1S;HQLVtq^D%zZy`)$t~rk!}xDi}S%`?VWh%Ek$9z zCx{5nUdH)4g0=Tfm5ZZZ8^0ejok6|ru`nBWLRt#iMjQiu zYO&;=rwv)TMHiNpFM?-I#u4RXt4W!WBUrF?7U(r!25LTsfpOui>h*g9G5He(cR3ov znW`PQO6c_#ZP^1$_n5#{8}IO)olIauBZO*qpD?4_2>5$W`2TH9Ab-R{$ii|-^5>i{ z@wSbJ51F;l-YX3|42lu$_1mD7jUn8>+yELrl7dR9TZ#Co0OF_h0|#7;gcr^lK_w$` z+!$#L-!y&!K0&qofQu&h(!+m@oaX~@rCkkv-4_g=59Bgl8No37od(!!phpbfD3I@B zVWeqyB6N*m;CRSy+w-@gaQ$;j?9qB1i=X-jc)$Ij*q1|ua%)NC>peTmkEXcnZ8n_Wa22GWoG==Q(A<) zw;^`9Zn)q92QJ~~I6FKS&p%ZJv~>0nnSL)gw!Ig$zrF$-N}Wh{gf@H=;R&=4188Q# zk}RoZfH^P;&aI2YRb9i3s7e-oub@rz0zct`zY37wzY+(I$&oIu3Gh-(i23CAK&N9J zc*Ka4gAp5GT|on0(|rglE#pCpY+rJE-8b+dNgjf6Z&JdY22Llt_$DF6_))$dR2yAP z%HNB_(^Gx}|8vQ}&CCY24bC9Ea*1j;m_v7^?AhKU=-Y*qzJ8ZYMA1V&$!P1CwNz+jcdFOggSR8 z2r5~Jt>Px)##nQ*KYt$FexeqWB2`j$w-5h%sR!L9F!pMzzz4U;!?qn0$jj3O;Nm$) z&~{u2s{N29yDxuYO!p9QT5JGY7wut662-_gmn8m{z5Bua1a%VpX9&n2n#d3DHzr-p z=CC#d>i};|u@brsFuJxd+^8TTbko7coYo#yCecnV;P13A;DXgE!ml2`lum?p(GAb{0(| z^;1QmV%H5^mJ!L9w=sj8UbvBx{VzdksUqb0*ptVB8Q`n_WH4QgAyo_I@X_aLIKX2q6$J8hZFZB6}y>2JtAHQ_*j|>@{we&vdm~x%}phXIAmDhqlTJ(v`c^BAx z{xP1muc~@iwJNEPw}mdxh5nH7{Wx1U8GG3&3BA<|;Y2NEz)cZ@vxR5>$FG0Df1EaS zvYrSH_L`E$C!c^;Sue1+p9{l0T!~A>IncdU21ZFVGF>JA_!ky+<2lh+!Lj{bOoR9t zT;x3)OmSR+;W=&i`<^|~^l1aD>JU8oPM%C&lMBrD_3)<&??JV9ci~m$&0tyh2VC+` zjRf{=!4Cpv!t-(Ou%uuepL;q5-ie)z8^JofkaU3iOONCGz1DDV%QNhcYHscOozP2gEFy%v|`&lHk{&ptwk#>|Uw{{dCH3+y$XlJ;@3CYPaAqLw8uU&IIm{ zawUJL6b&HS{@nbx|-Ew-AYgE)G8BmdKt7kuSo zUi@7{03Uqvo&ScJkNaRBxZ-u3`5iWqX<1oes~O_S=Y*Pu$F2F`N`M$BdwLRVFjs)7 zkAm^r=b5<2?l73CmI+MIGrryUFQz~^e>7;W1nD=bv9FE~gYE=^{~lN~ciT_$od-5! z*-j0hRig(k`CZ3K-&Kiky)fUiHx=kCFoxT+1O1%piE11l1tJ&ea6N#e7s z?t-rCQ*8sr!+}ii6@I*CGTwNn4V*SkVtj*TA$@Ae6a~eA9#3hQaa11H^v2*@)9rwp z`x?e_ha~wfB?fA}@Zr;6+vyAmYv#Ha;yX^iIA4{3G3rfuA%V;A+q|C!%B?cMW*0?Z z{qZtZih6>>%yi++IwRcR=m3;@6S3pU0{rr#ELim~3g~lEWa`D8|!C&T*jEzI=rx!7cyw@@K9=UcBj%D25KN-jJW$JJ62 z=ZYGCg1BSukp?M%XaW&|mar_ll`+(+A1tv|r2OLpX#B(A_!J|e+9QfCUeAZ|I z&B0Pcet`^7{IMGQ@wG_eQQ@8DRSfv2y%xy%bmD0=u9}Cun8wVcareBtzxFZcO5MK@U6l7!X4_kn1i41(_ zCETfGs*zPrCS*#?Gu)A|9NP7E;X;#HP|YC}gp7qi;rk8mx4nkX&MjapK3oGg-gn`# zPvO8lbS0B0s{wb5q~Wn>5pt+M4HuMj^Y=Wn2To@rz=P_EWXSLUe&YKcoV;U4cJ(O` z2NPBJ`JOJ=@=b?%t}%)EHqjJodfo>iMWcMW`doEN_A z4&q-2Ae5rs{J1^R(97$z?UbbV%($c>CYx)R$ID$n%Az2UH~kUAQ&z)UZ}l^IDQ57P zh;XMOK7))F1mgC9On!!I4zPINU%lPyCWy9`C5@lu!Q6$ljE2WmX0r_;@v1`p(U zw&MqTPk{5TkNHILG`5dy2CbtS!c1HbP@--yuz4apXZ#4~Zgs&;i>%<11-kH^lF*wb zv5$OuJOqqF^x?833G(~yAO0tY^)U0E8uWIpsN|}Gh=GFw>Bv6L_;VcIp{D^mR|kTG z>KPD@m0)FIelyE?5-1Y-dt+6y0i61TUpea=etJ$3zVwR6auqXS-ucO-UsMFDMts6g zr-Z_z4Q+VWOc!7tS%hEb4uD?Y*<|PNQrIGT7=O926P%Y0B%cBem|quUf%fWT7+Jt0 zCl_`L@8vnb)oC-3DotZzqMb>UqaIxHKpcL!;Rp0q8A0m=d9ti18?Plt_$8hN;K$a_ z%yHkzM0$M-f7TgOVaJy#7^@6xDT?1P8tY!X#TDW6a7vDCK zhG$=C6RDs)TZKUly!BTg=q{d3IwxuJALc2;roR)Rt%^J`?C%2~tgXpNH##lJiA}q?Fh(6;7AH;@>}obD1|R^=t_0S{p&t!tM z@+8J_;(qL)J)aD@i9*-BsbsF&KU_W0n8cii@RNBVcDi*AFT4I6_eTtYWj{7CXI>U! zCg37IB2o=n|NOvTA4TAgUS4oRrU@~~RwtEP60yN4eduN#itVF{`LdcJAZPp}&=`M> zi>fx0>2I9yfhDU*iO3X~QU3zlZz;pu^~_)ZR0A==x_l`m6*3{APY4(@ZS|xXgfI? z{}Z029cFrPqg@OLxI6(g{(z)4e-1pl!<5L~o<@9s)&ZB9l5mT`JaYA8Drhd9Ot#I? z1*&`f85%5yD~>KFeM!Q*d;!5Z=n4OUYBWw>T#siQOa(b62|_%2CNHv z2%a5tgU>A&lYJBA@!FqPn9_Q8k}A-LV#lYzs+rIDhf-{a+07$N;T2K1*;^cZFxSB% zd75yPTmriq&x6O$o`ck7$MJ``zkpbf8thoP0-ic!LuRQR!z0{!ydlT|e+n7ozc2X; z0^2YM4htZrQ%vAKdkwheMI}C0ppW6cr$EGAm+b1kfUj2k#H9t=V6d_QBz!+&OaFZa zAp#$q`EfJh3qNxS<*8)s)sbokH***__YJ-q?EpNIWk~m$54d4Fiz8Fo`3tTkF@JgH zP$5hlKJE4(6NLc`^XhysGE&cXz3&25I;F{S2WRqmt}e*bJBXQ6ql~HVZJ_c*886}M z69u84S7+f6W&$7M*o&#)%^3%KR2cb#V&Qqh|o?EN2n*-S4n|i5JdC zGstuY8L0QK4hUYQfySTVP*v*+KCUWD%;WPxt@m=WVYL%nynZF9HaNte=c@-rmo?eW z`2GjG6l?_Bqw_%<{Dm*3>BEO|pTOsEZKxLg96yfB#Fl}6AaU<@C>>jc>)awpZFVId z8+9RQT`b>4X*qP6zYsjlQH1V~tnld(3$n>o3Wxn!3|n@zgPo6AVQ&03KW2(NE;Mx~ z{rmGOPaK3;eacy;daFN_@pHv4m1a16{tE7z7wS=l zaNt=a|IHnDJSFUob%^bBsHW2b-e-i_j@0LYwgyM?AxoP4(uu~)v?oJ0l7Uax3N?}+ z^Fds}5g?-@D)a!y1B)56;G#?6AZ^nsXd>M8ZFyS3Pk*@(tZBT6;d*DxUe$uXja5mj zJ|OpNwu5{3_4%2~)4>@YLxK!elI2ciSm~Yw!Ac*%hi-Y;cRYgePN~AOrxi(LNG#Y@ zp$Yr%PJ*^QTcEA)LSmH{?W2`)!$b!POulK$ZV- zn-NKgQw7fw70Le6X^hZ42I|Xpz&5)v@L`q%Y!dDhW12sKg{CbGv+yf^sI?S&s7`{? zM;_r*TJrqO@orGU)(J%Hv4dalmf^s`a(?08OW??jEF3ih^8I66NQ+1q4xD=u9NFPc zs&6`x$|wmieeG&epx?_sm$-y%9@vIGL?sv;ra(SURfAo<3jyA%j|=;Tu(z-hCQehF zOnB*nw{NX|?CH2{`GRDe)P23u+Bpm@D6YVRcDm_=-P)?EiKPe~-UylOWhb{2qV9d&?4l ziNKu92*}5wUjpE?kR$*^y@~G(Te9m*7uN8v$G5)`aAe;S8}TAZJW})o7f+u-T8-s^ zY5iuVsrwkST6p#>NOUDtt5ZQK|2Y^maU@?Wgov3r?%+;;C!TV@7(dyg4X<6Q1j$pK zq33#teF9j=aTVq~u zHkiTx4-`#k!_o?K!HAJ6u@2lo8rFWpH#fF{>Wouh>BcH-qMHZA_Uwe6O@8=~pB^q@ ztRXll4|!tw%zA+}__x*=F4BGiyqxNA_j3st{8$D$PYEWI1FV3@-wmMm-%40jm(E{y zI|z^7s>TOo++lXnA-upHpa-ZIp|=cKm5-lwS~0wloxhGb~;^)S5U zahQBL5DkHTI;`+sL-Kot?+#)BJ$HwY_tVqi-H<(G;@c?T`7jAOU)@HI6vV@-_Wi^O zr4mIpo&0wv6%KzmL{v6rlV!b$@I+)X868iAGCny_HhCHB3H2e;X5KL4{30mRodwUI zJVXZP1w$Liny^{tBHs~S zvUh?@JKXtiHJX6=bW3tHa~9EjokXBqGAvTuOUfUlk_3^1q)0g#N(g_SHT~)2Ls(*U z+Mm5da_Lg2f5;X;d3}Jqja(1c#4H0Rm+vPU9noaP?t}1>X&kW^%_b|V5@D!hA`$%= z3k@%&!)t;m!klank!Kx%dr<YDt0O8xqO&{KarwbU$>SA3=@@Jsrc7^x^6kfkg4W zC9bGB1m2w83iM)rF@GMWkhwb#LBDUC$?ez-a{PfZRYiSZRewIV@ zWm>_AZ6;L8&LI6!sZeq-ia6(Ik_(()5ZuX|4l|+>;l;DDDZ}Oi3mQZ$yMX**F-VwpExNOD2V1GGM@rY`C(?nuJ=v1kFzJ(9=4V+5aM$Y~{s6 z8A(m-{w$HaYl*`f(rwA-t}XEE22#{Ay#hmvyMR$i82OZakbG=Rf+CVh0A^IpTv9vao-ZKd_+9VyS0-nA3hE|Od?@e*h=yweIfa!y&85;Oo3)s(@AXp9(Z?2 z9^AsYlRZ*?@S&y$dH0_b>7^F1KVluZ=n(~T7DvJ!MK3r}ZU<>uwhp>yL_^uyLnP>I z1{B*HMyk=yIciG7j>)DusFhgy?K>GN1<(0T+`+r^P5&tjptWCHnEoVU?`0oE6gDij%bn@@}KZKeF?a< zX&278l?#-133m)aAL3bcKEALffd9h27aUsi3>OzllF-;|OriJ?E(-4k7cT4bO&|Tk zJ&UYJNraFnVjqWn0&@Uan8|EfC*WI&-vGr?6NGc<2{8Js8e1*8#%SAk<9>EJc`TR= zgI8R^3L?hv{kaA3_&h7@oFxKXON)Vd>_IR~?FRl~dl7V>{sZPlSTK`UgyHh_y6_1aqnVbp1JKI z&|6T51CQ_EA1qnJ|Dm}Z=w6i~9~MU8hyB68ykrE-_LauLVQct~cO=1U7Xy;yBn^G& zYWz?CEtv6F30z4Zz%NB>8ROoG>7l-bq2;#OK)L3y(41 z5sLRITJpXBUbR8T#+b1Wnjr5>I==Ms2eahKGvMgfz$liU0cAZVkRAL2m}}pG;zlq2 zi>r?qv2WiP$Fw_)*>+{2*HRN+pN+A8;YHkL$KnxY2zwMHfKYXUXAZsOtDM*k5`IBE zy)G5o>DU2-{ng-f+;5N@r3O`A)!~fSd->rVuc`{+2QZvD$c#Pi!;O6+q~!E{+fx(O zNZqA;obo&iXKWO9Dx5ll-EXKeu1}XRkM0XI^|2RlNy!cz;F$!@DJC%}<0Y_&Qy1#3 zwT#5=c|c`?KO-NK!hbN~5PtCYJ)Yp(%bY57wY9W!1%-3vNNChUrb5IP_j>-ug(10k z)unposgEq!Ya50g^c=Co$|`WP^<%Yz=38)VI|jzK`|*YqS@@voIeyf!rTE)zq26%v zDVP_diN&Vq;_xebm|}@%wo&3{cwOOJ+*WduZ?`5Ldk5WPHibx%f_@oj5WfK{o-4#& z`Xc1&=f-NC0)uMzC1=50om}9w_a=A~vKOdaw!3yDi>2qin zHcPq&w#>GIT5IG$k^Fo7%oXrI&zcW^7S96>_P#`;Di^nz=)w)QYDE5i3UO($AfsZj z)k}-I@bOL^n66}oKYqD|r<)WBeU1O{yVKoR?m+~&RHRF;xvnPSUzhNw_-`hkT>`*W zt2waNVwAa?zJin+-oXzkm)m!fsr=laH<{96zqp78>QiimTp_ciw%H@*a0(x zc-SvZ2Hv@U0dyAbC8^6X^YB{_SdbXW2iZO_=G_u_?w%}sw$_GRYVss8>VdemR!%q= zr{D!*VKCfI6Ss$%iE@&^lp*MPHu#KZH zVQ*NIrZ01==hT;AwbpH9^)5B6z$!wKc`J$SY#zC7W=RZgSrfDGzU0!HXgF|D5e_)1 zle%0*Qo1h=-Y;tg!q!&2QmC=|8(I?^i_?rkHB0UV-Nx;&8R+=69_an7tST#50+of? zA5Wv-_(z}}xo~MS6t3Xlo*8y<-S1DBCl(L-?yrZ<_a~7>>}jLk+fSkRptsG9nhG)7;kujnJ{?-`a?QyltXo4h}{N@W7A60^(3vLU~ zyFD;6&k!z?X#tgukFesKIplq@u-7Ot1RHHIhF7n7LhmIi&?fmY{uG!9{tEw&KVwGl zbo*S`8NCXNyKEr4Z~Ksm`f~8#p>ceG@gyy$<`Ji`MzHds2f;55$*D*Md~VnPeQ2IZ zqb+n%kdPO4W4bfXq{2?v6}W=7N6MgWg*j;B$vF0hY9VK);Y|09{$@X94YFuo4kwju zfkJ#j(bHYQG_i67sb48a$G=}^H!jsdL4%i2@o))czHUeF`t!NHTTgQiDaCXo+J;-- z;lYb*I!bRu?w}3*H@V@Z<$^ z$;QZ0zau;p(rSw&;aw`nJ>>izo1(JPAQYUnooZP6aW9&y+3Z~d?B54&=v7P~FGuD- zPTc+}{k!fvC$77Ix<#}ghwXo8g3((x(Qq%~3;P_mT-(Q;+LlNMyz;48`yHA&p@mkq z5LR=47rS`LeEL)B75zNeNJB0E@p%u*P~Gm?bacUG&Lrj&YZU#Ptwj^i^=)BDqQsS6 zub53IK2K&htK4QSK31SAwI%Ec-4J@c{{rV3s7+x!;>KT|MUH=XG=Fmex-HGp7GYnm zZL&1mS$vu6xh#)9=S86K_bN3`%(LFJtGrds;x&?x)Z z(2LvGfYXJB4ls}5_SxO_02Jzze3%FeEn>_2c zT4;aXVI*Syixo36XRp0mK;vb8(Fe;E=+c*aP~UD<-rdUGX!!bXG&d`rR$SRgm+RFa zgH?0sgs%U{TjSY?U!IAc?Ab`a1ubXSSD0~0^RA%JgPG`{L^db#RFMV_Z((P253y-; zGSDSih?EZ>Ked8lnGpn@-C91WuZi=F)Y3emz-kl;kn0ANz{VfG8o_LU3mU4-%mAOF| z&V9){KYo*)E%gFvh`wRH6#8lNtPki_`v|w{TnFd=?HSU6CwK$$qU^7pL&6?r1EjFV z4>2qEqU~E`SWWSJ=vP#qAl`nGoqF#SJ7ja2J9npyJ1*=fAANk6tzYntE3}cM>)O+h zUFKaH^D2OivRO)$BVP$JLXD8R(`6LWGn?k$YDQ-cm~bmYcGK)zm29=08?xK0NUc8< z(`~MI*-GhGf<-s)a2`EjoZn4H_Dk6`JK3XNXqu)Sb@Z*~G6%b;yF@5=;<#K*m8X)B zmn>2SJs_KZEpPG8RVX!w3LE-qt2=VI9*dSt za6;EA-l3Pe0?zJ4JiB^rAT@nKxa!h}EO{h~a(_0nGv@OJRjHjcxxAKIrVel`Jujl{ z{t8;^>4@@*m+@}CdV%Vqa|Bym7twqFT)0RfXDR(yE4SmQEHVn&gC4#m7G&^%tJsMgt^PQO6GWo~1(p6VZ&1Pr^u%a#J@^)2JGhXqSwV8^7}e+n*whf?@W>wp`9I#vkc64s&>)fU~?c zj;0&PQMq5MsHB7~H{pg8oz(V<8<%jX(elw}_gFsU?smRr@9$kt8OOzxuVYj*OJ0N8 zlB4KW(nHo_XfK+Ulu6GEITqee%8~Ti5=4WgsMnhpbnbs2xyZ^e_U!f=Za;e)869e) z+ZV4ByjiDZ_sS_j$OHJ!RqM0~!iFxRjiN1GDb1Wsam&~#HC`f;d~3)L`ZT{@Sd zzBkQmSMgy%isg0s&r8_jJg77-E-qIRQJmsYt9Kjy|uJW}{{Q(I=S& z^rgQNO58V>t_RM^!PlBiz!;;R9nRWL6rgE6r;#qebLIENDAZHPTmj{NNz&j znd|7Bpb(_J`H$eh4;%EzCj#-m<)Zes)%4Wy9CV@dvp_oQE88`$PsiF!Irefbdt}KO z-l^Tv6s{t)VqgP`dXd6A)nCS*f3kw^S-gV2|F{&LReO&v_Vl7(>S^fab2FM?pp1@d zUZrBa6vNcWw+&Qt#r%U<8(%sK4;$enZBOMBTe%72ZJu)b2j-C555 zo?Od0It6gr%pvrEHlU9?5jAYuhfD?&>2WV(GFi75sd&#udx~_B;Rhu)s-v75o>eI%5b86Q2B=TC9PeQh1b?oD% zuV9IgJ%40o3)^rQbL&!TY3JGRoY&H|+<{&4f{`^R(E=S2dgaG)?)pi$nzdggxRKkA zJbm{^?C?YbJK6jW#2hWB$CnEkc_)fF__~z)J-3d_#|*9c7KXaJ^XQ)X74$)szFn5h zQ3(1)QI%=~x2*CIdPJq@wn}w+)Zh#E!PkM;YF)-zz34zYwMV&=f36~)u)n=^hZ-%K zGXZ(5N}$H!)^^6Oo2i6tGK%a;=8jx1q*FS@d8BJP&6KS`3aJ?=D0VhlZQmzkyl`w> zNjdsRrcnJHSNiFmd`{oXenRCZtAt^h}m$L?OHUs zW?Jb&T5#|e7r5>h7yPSJU}mCR<9t)Z?m~AqP1~!);-#YWfw(+MPmZL8DiYkpXHCe> zT%1ZsE3;9Xev(+bnKkYi(^+J~(nh`vYB94#PDghlvD#I%zaRjGu9CB>KLxpL=^ecG z>E+yni=pi98I9bE+ryk9cU7=@cRK6D-lNy&E~WK|R_b@V(IgiPVsTmQ~-n~w*xOHJj4 zjJ56bk5U$U!v7d;x2#5=KONy@bYkexdP($g`EfMZc$kaPF-LiFNo=RTusdCAA$_;& z5T(BBk09mCb&`+RB$MZK)y0Wa(BVtY#hyk#yw$0Os1`RE zpUZO54Rp+ZLXE3uF3`yF(n!Olh@2Cf3dM8qA`1scY=CW$kQ|-Cx0E@BP$j*J(8L>kz&B$ei{cwx?6oXYpp=>EIp7Jwka4)}o!0 zvZ$R>2VK%9VdvX%j~$yQSsGr*ty$}b3LkZ_T_Y#h7~u@zUo4=97fz)@ zhb{6e-a@~$l~EPEjy5am*?AaC*(2b=5fh(f=0 z(5MSeXuoDMHBM|~pG%i=Enn7AfzwYeGEJYZsTttV)mpAt_0PF9JGP9{!5}c9#iOOW(}I(|Cq{N%S101Nh9B_ z)@XCzf2>)dAzHb~7Trs3fXInEauvGfRe)fLLdUt0cS0U4%dt_~wL+{%d(cF z-7DAdVzu6IZLfa`=18AqZSv!(^?(U_J&L*O{(;oI!V9HHv~eaA6VNVoUlbfGB53yv zpv%{XQJ44$cKCNT@BR!GWV4-PB`+J>y*N7`mDMz%8^`mxrOp}X?UGyyBC_a5A$LH( zj$pfeQqdco^CLlt2@Y)w6r<_0zA;ZK%L4QgHIvB}%eSqk?^hsms&1g8OT- zI9)eS)KP9v^~h6Xo#9D0@U!TH<5g5DMvo2ae#Dg=S3(Wy59yb63ABA>K3efj0c{^J zu4&wPl;-=H^Gs)FqWBaaw0kL`Z>z4++dI#&YlXbd!BwHuK9fW6WHFZ-E@7vmeg|zW zKT9`BJK0rBi`Uc|TeHl`yQpnsOb`&}!j5W1pn@Cbc#(Jomp5gZkpI~vSov$YVEr}| z_TmO-cG?DI6qW0PZb|=P-xVUd%ioYryb{E=JnZBe4NstdtL7v1!K?H+mY_DVnL~Z==DFaF7B$H$LZdR_XcNzfS{THj3WI8D=chuWKK*2O_{^&@j8sEg zR{v(x+vM$LEI5PYzrW`eS-1*r%{qt*PBgH`11+fL_y9KdaV1`lsmrJmoiq9p4I8`D+uPUDO{Kk@ZRZVMZCegC^4UV?`h=jy`5y$z^#B1&jx(hFReo%Z zi8>wF8PBe|GC)U3H@&IkgkIiy#udGuh+Gb0h0#CB}4rBzY&?5w}* zxG+qHNl+X1(WE<;masMyp#UE zlgDOSPT{>R>);j^mT*gwis&(eR?2GFp~v}r+MRiW{L#IO0#{C>nXWH5M^=CmOK+fi zuddN^F>}%CY(rWQlE?9uRHC%>Vq{#SUSmG#2Ws0x*g*MucHg>StLxF7Q@R4z zVsG}6TQc>3ypNV%{ml0KjpaRz;OK{p{j8Oj1(Nx2O0ed~15}3#sn}&jL6py3WO*tD z<%`bXZT4r;sqfcV_RLIHXO%ep_$B~lM9XqN2RlQ_pb`B$ads6?Q*o z%W_kYh>REevpJipYuliVpa0mbENv7p%}22QLngb}`U*QAo+jH3Oi)~G8N0uI9#Y&} z%$W+mv%@M;RP63smL{8_N)(N3$M9wr|=obTRk{=lJ$K_uKIXJ%2ro%kJF3Dr`+a zW=-o^-)|arUu2h|anrBplJ5jM;t|N2e&5J`v^;{YR28rb9?Q_omJPmvBVOTEe?A4-KB6n!8 za39klYbW?SoXPnM_nrPy>a;f{7l&lZq0lj{8u8kC&gst)wx{DbtvNTuHOMz{UZunA z!YAr%YIhvc)1D-_jY7Dluw*hXDFumE-WM`%;?X0=mE@T49-t?I(cNhoGS8n;*DJFj-fQ5MtZT6#Pk$*i?t4>bi+ z!5=qPJl>m@G^Mk7Bi+c>Xo2uZxNN2b4Xh0i66+D$9y^aJ_miFY6!(leqb!E}0Wmo;8FaG0Cg zWr)TmpGCGu{=nUb+)#dxGkWeWKv~aXc@EajblVhTgum2qSAMy2?Vm?EtA?$-A?p%k zvVo;9s>SRI%QNV#-S_Fqw0e3vxs-mdOs9#}=eSz2Dg>f>*z?Ut(N}LJyVJ_{+$mXe z-i$^Wy5RFDoAxvhE!nyqZ9Zva=bUpC&G#4(xaoS+@QuA(Q*Asw$+>bCleeMX=C9=L zs_oo;ud8Uu=~Sej8d$(a>YxeZ9nTsX3f=5n&Cz6Ub(Rx7?p)Mu`767YbHXanWf<1cR9yg?y+k)WyF> zBfbdGLAgXOc5V&b*sjcKg=y4y9Vp@kwr>!OEK#b-{h>-PiOKO!iwV$;|77h}hjgM! z1vMJ9AQD}yHL6+n#fA0Wn@H#FwWBr%(m1C};#A5}fMzUAqUJATko<`u>Nm=v$3>pV zxF-h5@(RzOn`jMyR+=c$VyvxOuaA^J4V$N}8hoE0?PR)E?183vr zM-}C55I<9iefc1X)pV+&;oaph-f=Dc6%t4NMjq2ao8R0Avx%JD{=+Cm5a;WlY4bE%`+3|Pnpw5*}ET^MHuXzXXGS6wUI@=A|%k#51u?a6|9oD5= z95DJSoYy@c-4lp7$0FkgQuOrgd^GpuE~LH6Nf6#qz^QQCcxmx!XvX+6_UxP;yhDG) zP`uFvZfP22yMF&A|CpO}aK<=l@)ppI)z2tXE8KJXs&oE#MD6aanql`mQMBea>HZ?SA=T0p+iN1K~pwa^|oU2MDo!OcyWW0@Wp;y&ug_kxf*OpC}7fIQ@sOe-A zl;_i@KKqeID4=_N%aQjr>6-nLe`$z#6I%X4kyX>RMmO%gU=L^hV*f|cna5M{e_=dC zw#rf{OJoTpd)%4(IX5LLNkt{vC`zS$p;ZViNJL2?k`&TnnYo{HkrtJ{Q6#0LO`A4} z3cvgN|Gr*#=FZ%ibDrm%=RI%eA5s|O4f{^k(oLNsMOA}_NWz`oVEv&q=qz%^J>f-= zS0*d*>yV`}x4fh^#Cn)ml7h$VS7M9JCSgF@S+X(EhGxgK(gk)mh*fDMe(@GVc47^c zRq-bewkqKK;Q{3C>BU&nC9O}Q+R#-~7sni31%GV6i2GV{@X-%T7^;*4pDK*d@Q4|x z>R&|5^{&{mSz43Xz7d)?oEENmXCQtjKNnLk=;GeOOjwo^gk7`!!6bd8#O>w>{*R|Q zu~N;0gWeLTIbaJfezjxjs8e*wdPmr^bc}HEyE0mQKoQrkK7%FuV^A^dIQ$)-ERLU^ zAr7824_8+;(!%8>a3k3Zl%=HTJ(DieX|7bb`>P1ArLQIhlN((8~R2+*U(E-1gG@pQL)@ zR}XM{Bjteh#p0vwx#Xo`D3lqVa*^0pu)<#DpP<6%$A9=EO z&UN9M>mlIdwFY~=lHhdLLL4*UD9C+wlyV5~(dTb(5dF^;h-|R*N4>WW}`UzK5p*0mFEzIc3MKi@Z+7fV1I8rwj^j2Fz{%k)G@B2gNTPkDE=1v&9l7U@5 zwRkRO444fYPc8Qjm%MZ|qwg$rVaoMLjA@(#o1YbcSbP!-6+&UgxAXY=t%aofn;pDc z{Ze>kTn%njdWek=14-57L3GovQ7FILimd2zvk*@TQT5I*_w2?b;&PS5Ab zX>Ww675gAAOHOq6`&hC*r-)k1O@jR&?h=%Jfd#4O>91u57*uNwjxDdqM3-lzQo{o} zH^_p$X_hdfe>x%jI5agMgKHzp@#H;aC|??h@1?zL#=$0{_N;}J$+b1opO=q3uc&nl0Ht zJgwdd@9Qbyp#A6gn4nDTEt3bX(-=GZZqPboF-l`5CGSG2$;e4XsQ&f}BouRigMb>E;ou?^N=5&_kmPIS{_;EdB8 zES+$SWGu>}J4b2J#`Y+T8D536o^+BrCI(;m<&pd57Le|$ftps~XlreVy2}>wm)~!J zcXj!=ZSQ$pGjc63GDhe(u8w@NHGtDh5tJP8f)H^LO`;$Z>7EV(ow1&AQWbw8;;2ve-pF0kFlW3 z3bJpGfzw};sbtg(d@JpvLvP=0qsv2~Rp?9fuD(L0cMN*=d&7;m>o~ILD0ChX zA){`L9tZVcg7p$QIH?W{pDZC;pQq5w#o5R=&cb`497MfXge{f*bb5!gSWuwC24}>> z?eqXT;lOIpsI?N+G|r~WjGSOv$`x2+HI^29$*0#1N8(NSXY_*aVH$Te2@_7#zytX_ z+Rqty*)z9j;Mpi+%yoVje#|1_fXf; z46JyNMU9?rMXiq<`7?Pgep;=A`X8Oiun*~ERl+M+Xm^sxoIOE>_Awwg(N0{K7YW%G z+4$mW7h#?5U}pa~Jd^7!H-3 z0M&bX9FnEIHoRO!^nbpgN^h?b*X~kifBzaqzgc)DGYHqe{Z13iu9MseU36of0Xn@I z1s`QAspq=`Xr>{DqW7;!q)Rf52OZFwbrw=zuERMv3D&IM2Gs!(IQgL(s-3$Ju_h*1 zu3`?A)<3Z|;R5&^`i{I|1L0>mkfPrQagXvZnz=V!Y(-|mI1@8?sJ;;Ei}Z1?xtZwI zhehN_@Bl{6DaD`p|DlgY5{z8`kz`&(G|Nn-U-F-c-=Fg)Lb>4>mVTLRK6wDv%+4qI zC6xr#J0WGZ2sG8VWBe-*xcqn?G>>7xwzU>I6STnPq9biD9gH3e7L&IwUx}N}XX-O* z7?9Ofyt$zarnK#&l`gC48=C^mf+Tzt_Jn3EJ`7TX3-H2YSg~*gj%&)rHy@Xwbw?V$ zeEO4CUW*i(1@{qQRwk(HW|9*hTFI`h=do+QqvYnvrO+w!26wf$lAXOzNz#s!MAPml z{5N9;CV!D~T}6lC+%RJ&|A)$wqQW}3HzNX1=_`s(88isRU(-pM^K`i9bAZga*DTgr zvX)M>q>xtdg~S&BqZU~qcy`cHaC3-4>9)O-ucnw(uxCL(QF;#rWWv$1!B8?P5H9M> zgQx>HQL`&g7-(Mrw>DHk#i+Al&#qM7$e@VrF;Anf)|bGt$>T`Jv}sWHq=ZJ?K1?q= zY$v<^#e!q*P8hfEBemWq^&+PRlDRRn$+WI=nET`h?27h*bB+Yo;7C6OvOXP%b;;V zAL}$V4D8g}!L~~VSIl##Q_GUDdWV##7q|3l^3chTpi3$9#!4_V_dqTbL>e)M&b!summ_RbMB_LK|UvUy5IJJyoA z!MRki_9(i=$>Dm9{czIg2qsUkf>O&EynAn*wB`}BW$7&Z(fOkSX?e4FsJVX1G)N z9vOXOJgiFM=mXzux~=;O_zfE-3U7;}^BN23xygOl`<5f}u18^MlnIo8K8jAMiei#7 zrO(~~IDK$09Bm1b`qCTGJ-?SW@W;@kcr&b%l+d#0R>J1#li=Fa&)_w3E|p)FOFuX? z5|jIJxZ`N5SofVj-t726%(uRUaPx9=l{J+7iRRS$L@{Z*8HYpCXMtP!BXXeOB5WyaqII)<=+#w5 zU@DvlQ|?DWtgRXOT)7{PoH~K;OOL_d_Z%M3H;|0|?Lpm)Q{h<@CoZZ}Mb%@=Fy>#D zaCzemxEnYg??1mLeyC>%lRN$K1J>~G*x%IS%^=arvRtuvSr6(2|D|1vDP%1igVz_$ zfvWj(lC-Zk==Agd(88(}+cZUR;c^evoOXZ!A(kHSZWD_aPE~B5un}X-B0h%Q7 zK%1`_MB8={|L;b)u#3n_;xFGJAq|J=&(vj*4$Ej(hMC zdigT)t64)*GgX8;yc)o11y4`=zM(C{b7+ochq(ImebO5ig9o#-ak_~ttZX)h%Cecn zg3ZLW(!Rib_)&2|kPXpinasH($x8+m0)N^O|ZXPLmU@N&8OX z{~Q79v=v0SAeQLw7=uHfU4s&vKf)=FX*8PI3;FB~DtXu|oPR=0Mm#(TzIT)9oTo0Z z=ITb86fqQr{#{RMy%W(Ya00%&HJ_8x)zHRQ2G^{9O>L)li>r6Y;I^Gcq7%BTR0r^o z44R&Vjn||s^=tb0`nwG#q-jW|ZuP8#=RAK<9Wd zu|2j03Ve*gCy(Sl(4(9# z4Sv*xAM5)nW?g?p+`Csu?{gu{j!L7|Qa!11S`C}E7pR*di$7K$CU?YFNSGj!KI==w zw;ttm-_7sz^NC$xzG#McbM;$dTr?AoysJiqm&358^fI~>kDwkI2^hO67T@dNC6m^GMYg=uO?Hv7$` zExYdFrJwrntu-BurW~gQV>!69WeDyJTZo62h=mnnRd7_FDn9Nq61As4fFTara7lJA z&iVXato?Nc9hr3#tU?RPm!(4J?`Pm!=>hJE$eApE`5%rPZVSE#FA7u20wDWODb2qg z2WJa*(2`l^AUGI+a_!QY!Lt|0?p^{u!#s$^5e3Sz8;ISeW#moSQB(=*A!1Q9UN=#P zqsJcMGm{!LRk{O1@0UUCx8vlh%Uvv7auD3R{{t1y2sKapz|3$NyuM9LzkK#Yal94S z8KmRrvTRt|BparV2B-Bj^?S@Kt zyP}+ot@|L=RHqQXIW{6A2kDH1RJ*)arGd85`*5>O6dAKaiEMY6PF_^Sk->fLB>bW_ zd{KN&Y}V|dK9_gW^l|rvI)TbqEA=IAXvDHnzD;Ogwg%l7nn2{4A8;vQn6TkSkFffU zISh+AMr5~n!*}C1Fmif7UC=T~G9aB3n&vfudS=|iT@fbm#C07Bd%KDXJ7!XC*)m$c zyMrvS9R@3}KZY+E4y^lfBdCoWF52lGgjPqdlC|%nF=KTlBx#sd|Y z@iSALJhK~DE@>wo-qqy4JY6iFmIHxT%IIwWr*Ne75MCH@1O7fgPUqVXMrUJn5DHVV zNyAFyBJsh3Yc3c$uMk!>`H`nR1Hx;UztFX(!|1TAQFy6C6Mf@Vgo+_wg*~~Al%0DI z`+_RrR#G(7wC#kSdnQ5m>wNr~F+eR3%Ai%}5>nS3N<69(G44$l>b+b{4L|(=Zrpe@ zx}YH{n`lunp=&d0$16(w0`60_@fk31tOI8rjG&ogPeJ8SD;Rv>Al=s3Pn7jE@Pd4= zR11|Q<%8dnI~zZO&(2$P_QZA)GVTYAHor;o|4EOhGKECiJ^){rA=u+!0H@49Q0qH2 zP@3`@w$2QIM8`hbqojpZZ%dJ{TTi~pe+S2>FG=*H6R`VfDaqL%45f*0aNX%rx^&EI zl6P4bdq!BmZH9-@)AkFys#Ia7K^9y+Fo&$PDMYSvIZg|h1zTP{CPn`q(JwoUQBSG| zu76ZRLv~t=J)J7SDE0?Y{ILq7Qmo>! zC)?ZT>I=D86QYN`b)g`m8bW#__v2)hW$@!*6UrTm#@zfOY`C-@;(m9*qwn?joj9O@ z>tWohZU)|{Fa1xfX6KtP_#Kn^0%vk#WRC`ic&`(gZTJ9csb6P+z5}S#@$62E!Fhg`!9l3-*w0u=_2g%*bnmK zMnZALA`!iHN2p?X9dl1Nvp$`!bl0XHVeytMQY=3ae^1rHn@?qN?0@ah6m){Bx}71$ zV{0Mxh$GwieJ_pfdLZnGOycauM&ar`O{7`N8jm^5BO1w35V>@tcO1V_C11f7oLnREfQb36$uID+vyv({pdbh4j&kA$IRmGc+72%Bw1NoVr4!7 z2ZCq7&aMOyO*%+Ix}xFB(gY%Z!<5=YQ8Md#1lbzgChVJX7oIoS;3D^9&|N=EcyDO{ z>#=?X7-YxdnUxke+Is|vwzR}gU;e>I!wz9&^;t~W7)YKsDB-<5C9q=aM(A(~Mt)R1 z=^Y^>v7Z=+6HF%J2!AC|Nf<@uTlS)C*J9ik@CexLK4L4A1TtosEu21klvLO?L2s!% zWCtG=E_${WA3SRWll=#9`eRf2+CH0Z(6mHvD-$s2KZP>FNNTw+35?YJp{#Q_%1d?M zR)wn=apej+5&n z`)yyS?c`)|shE#LT?b3@b<3cDV>BfTdJ z$kMyVQ538LbF4;yszv~HO?0IfZdH=x^n;LaH65$Q#<8DEpNh@OyIH~2k;0W-n&5E# z3DLh9N6%?&$D?I=RDEcJ@bJmyc>g}Zgt$oY_!UL;*Pl|-5_%m@?LLLEL1E$*XI-(} zK1DeB!BIjC|KOmezE6Q$3v(dYEE2wrESJtQTmY4w z4(O$R8Xk_%LP1v`X;y5&@PLyfto=R>pX!WZw)!HF4B!&E=OkCl4~K_0kb$jBX-%>X zJeAfXzVgvzn_Gx@m&6i^7JkCo=V@rQULAvN-OxPhGN?@&3p;Px!Xd*t_;)RXco0{~ z%)@7}x%eKD^>-xcny)}N(+*sZ=MjhYufk0aO-0tq38cm~OK3jd7WK}~L>*IAiOnl9 z#6+GahqlKNjVv9gA3KK-6>sc#>J32`El7uT0~qh977u(Pe zVNgyN_4?zD3-sRLvCTznllom0OFh88wlJ`M9!00@T*AofEEMBh9?Xx=0djp8)_4Ta z%Zq(_AJ1nDl6mk?qaV#w`4j*cFBohEx9L)#At!l&}ixKPqUD)m0lU*h0QU!>DS?sy+1WuXUg}udJ(CidX-H!dEt>5sUItOE?Yql*~UTS zruiuRl>y)D{fWVHhCY)0L3g~3gWa_+@wC=Py0~g3ov=EdR`-WN_w+wBs&YIrKYR_o z4;chGd#(t-PR>J_Wa;kC$DicqI2Sk@+lGrZbWzRp8|HWYghtsT!X?GE#Pp9f`TC=U z>V_xbx8J9PqEQM`|Jgv4=KY+cXdI!3c4*Sp;Ei|%V$Vy=d)-1+emg;| z2ymCrW%50`5LrbnSbX^y+P6fZxu!WtR#elyZg{05`r_t6*C!4pvKk}cgyK=U!Y2{Jb1#XXn+D+A`4wcUb}~Iq zK0su*mDpo%89tvfpKL1uVEnF%>u+Yzq0MvfLcju27dikJZ&;$!+&COrT1vDu{}8{D zH2f9(9B-egC05lhs9(1|Oi3&va?c-<{z+$v_l>{!;=ojBkzTW3U9(}Z;t-MVmB(c1 zLQC>w=U`epUIP#8Nx*>_mHgz!$8_u6qxACg|Ic*QL#FRN+8_bg`1U>Bu*VlGd-js4 z9$QGxWK$7)JzscNZW#Q#u@v16eenFVX>eYtnRs1HM_0Yka6h+;+EnJkGC>cH`*;%T zq=9Dd)$U-5j6E^@b_Kd!Od;-N0yIC6a+p(H$ozxtaH@yL6B2pwn3D?@VY9>;W3=f- z@g$%di^NYhn4+=z1#n7A5}xu_p;waMqv&ZC9p%y?R_{?J`SWgx*A6_xyszI$%3Yq; z1xWaOg$R5%{1hF&JO%!&He=4kIf0k^RpFMhs}vU&)4Ug3sDA8T7&mbZE*-BRYJKDf zetL;GC1Vi3Ex!YFJX?g_Pi*nP>Hd_@52`GEcIj-a z6uXR#g`h%#_>s$d>)l54U3e4jYv#Au1mLpd0+v7f$_Nr9EN zV%VtS4CnUo&@NaAWsOn|G*btYS#8vf??R*Py>v}L2r9m+mTIuCadEv8-NqlmzqR2Y zkos;=JnEb+jDS4c?3C@k_62=3}M&LZi zXn4Z|w|tSu;?$vV?eh`x%b=F7mCb+lPR~HsG%x@_7D`nn>L@ABN1=m8?ra zKE|hmlHR*;HuN%aiHN1g##oTrMs3N{z+&iIc#~|s6$L$;&Y+55IVKByVCu#bAa)L- zU)Fr30Zv+y>BrZ?sat;`YSSpf`KI8BDJL;9HV#!T8)ELXarjQt63*GqB**vG(G$T# z(C~nCKSl8Z-#lqI*{HG&KI`R^9Ys2Ld0!@q5{*z{e-XI0tCEVN%L(7@2uX<{5MD4E zo=XQ_)<O#X-iUo+{ln?n)D#L!P;lWB!PFx73g$8p7((BCHAA-s1V zA4#I=zc3T%IyDKa&hCeQtIV-K`WSiEwFPs2Uxp3kvb5&>9(eZsHLQ6n61^4W;|a4S z^t>|_oCkZsFG4_W;Wji{KOJA0H(^jTidXxxaOgZol9%XGjd2r1tJbQ)kb9>w=9L%g zuy+^zed8;>RP>~$-SogfV<*g-tR~TRyhDx!Wy3&+2R_Pn!&>Efh%UVeilcWzv1J}r zl+*}4?{aW+r8C%!498V=8&N^Uk-l@Tq$9sAhJc7mG~C@?>h+Arru6sqC_lX{D z81jN7Y$>HtLw(4e$1=D*V<9Yja~j56R)GlZ-{hjI4xDHj4jK>cpy$(Ea^m25(AaWU z$_1{*r;`uj!Z`&vD0mM(5Wb-4+vk(z?$&TH@&`8Ru0;)T`%@X!&3MMXPC>$ytmMNkZa`pH=9w0*gs#ueWgBiBDQjN!ss;q1!Gym%l-khr>;fyDM zo!>cpQ9G5Ef8fZn(>`=+A(HA;9T>g)E16o>j2Rw@(DCT5aKvxvZooQwD$dbB1(^~& z>J*KC%gzxu-F$NKN)owqrwdwdxr#1mr;86(-GMLdP zo@U2M|DP44{&N&eEzpPFPnqQ3_UrK9unVw7d{)>qSeu+NbwiCd9!rORCO?c@(QNxr zIJq(lwxm{)?=_L|sl-uOb2bZlgP%%!n-ehk&r2E=t1tBy{OI6@lVp;sK4yIR%G=wS z<6gNS*m7+HdERsu6m*``BytE^bq`Cu5vl(CHW7k~&XWe+75KjB70LHdlg>zMKwt1` zu$8=r=3D{3?P$lD=O2-nd7tRh5%$<&ei(a$50Yh4ZKr4U4VdvymYB^r2cOmm$g^3d zF!jhPI8y(H%9!`kXBH)pm|qJ|Uu}c(Z4saq^aPJAN~AxG<{VhbEU$6TaTrVq8{ zUzl;Pjj$o{0%Xf5ibnNG_lsZVNI?I7o)`V4jb+)dk3ZsU!bk;J$A z2nifkj9tc75dTXcUeI}wXfJCLHl$yJ`JN@PHfB9!m1;wmbT2>4qXfrnT7ix}*U;JX z6w;Qf(wbim7_JR!PW=#;Ux?@Tjuz2vbT{#HX(xKau2anTMe`4S#h+VtqJQ5Kyy;X* zk4}uiupJydb)}6zwreYLDfEk z>g0&U`(EaW`)1d|cJ~&v_EnPT1eil^wFflz&%;PRYnY#YiD*RS;i})o;joE>O`L_L|-pEVYRXHSAl!;BEhlt`D%ZAksy3VxT( zuniBwub+|7*?19*f7ZbU-OV6Oa3>awKZH%z#sdusZY#ObmlB)9%96Yd%; z#;n&5Xu|SXsM~)UM1c+B-rlX{PgiU9|Zb1d$IxCid2@EygPxlG6#4~aX48# zqX~6iT97T7x)44n1dmqT0)6S+G3;)J{A>xm`o|S&WzUj<8&^oghXxX}d<=Xtnhpgr zi;2Qxc`&-y1Je$_f;!6xdVcyTy8U+|@mU%V71wI1@vo<_@6}?M@T`FbWxL=o!6c9~ zz7C@gOb~`dM1lH;6GEf*8rXe*4|tIda{s;uj!<2SKSFN_!wL=1yIBJZmVPD8nz_(- zWF3b0ejuZ+-U3y#<0w1Zn}pD}5Vm%d#5a|L<|W#qibf|&3g&=pVjg|@%Mm7S4TTSi zXX)+APGR%@Drh)1N7|c4kUQ`1!$gGzcyi<=u!t+cx^OWR9(KS<(wUX#%TAMcO)cEh z;|PgLQDme2H*$Tc7u_;90yo^LK#x%=5cyDn3Tswje&uN*5ct8^+l6$s{xzYiS0-Fh zC?z*HeJ7u9dx@1=i&5vN3WiB{+?^*6kmZ^qC0-+YXke8hyxBX4%$l%Ccqum(SEm+( zL)=2*r`b>3dIC`Wa5zpbR+h}pw}9oZ7GOuD49Y!xgYQ?IL6>@@YIC(v$!msKPR)=Q zuig$a&3kaq&4=Rs_Hr~Vs({3%t|sqX*Nv4-{IvsPXD1T=)fF;IxCUQtiozUARhY^el4IfP(EG@4GR}#D*^LSu zB-I_SrR)QpCp+-m{yi}3+cUahZHaV;)eMT2g7A3A5o$1#!}3jK#Q2>xG}y*N_#=BX zai53I&Gw=-ZxZm>Gi?Y8H-YsAI+DTSJXjlh6>?T2;6E8nm>i`bnzH@16! zL;5;d+!5^r1Dj&t+Xxvld$k={vh&c4jerORIhZ&ihUeFsGY_qGaGR+eowR-e9^a!v zt2Oj-gu{2(_4Em4_LzxP<8y^^F6~0a15?n%`4-f4KcL@wMoNA>P!kO>pULZm>!9%X zFv*2iA!u~!2o5PP;R-!Y&e(bqG56!$lA|KooM{UkWvtR$Px8*n;YkLj^p30Ke zS0BjR>SZ{q`8@uUJ&&2`eRSTg?bve?N$SC9>X5UOo{zYJ7T)9FowFSMJ9xgh>{S$Q z*Xtm1dg-t}#s*Ez21z!>9)_D8hoQ-~k6vv$0k^I{LCZHr5dQK6^u%|`lq?{l(vr3VZzos2>1Yq3u597ujy2$$cy0C$v&;LCw#9J8aCHIsfv%WH39 zrvHk}SE=D2{Hccdt6pGEMhlewIfN}SzNG#6MtXhrO%%nd!E(=HI&+5;?(Yx8&n+!5 zD?&`|$V=fht22<|@E`cv*kNX0Cr+E@2g`Q!l8gWPgv&2ggYU#4#3;4d(ot{sqBCGOWQdly9Zk6AOyIfVt_X`E$K;J)7-S`n` zO5b587mG=KZJAJem^0Y_$)si}A{;4>1J$FlFzQPa-KslQvOr-sY)`&Gm8742{ke;H zwk^W$_|s&*u`KOhn}>VX?!<|oGvV~fPO^TOE1Z9JQ_L%6z|s2`@k-2YjA>BCFjs3( zlV&0PR{f3Ld8hF9u4p_geb;nrD#MgncOX=95JKKY)9A&i(s}J`aFwjac=T28f8UP4ue51M8ep!FqZx-cyn{y0Q{E1QgO)w1xVfi*e4RR=$`JVNgkHA2f3 zLt*PQJvdu?lFs|&3ONaH$izxTN!sy0XgX>&sGKdwmD%>S z#+wUh?0f|jD`&yO(OvXZR4smCC&G?%Q()h{B&k=aAenzT1S8jM1NM3czP=Cy{GD-d zH(Qsez7)ftKWSvmqU&Psyry%dh=-#;%-NUkN3l zGGY=H(?J&HZYJwm=i_+0GICr=S+s7~FbteK8NVt6KDA1v^G6m47pYlNwoa8K%wH`u z_$xhT>x<%JX*^5|DxxKA0w{m7k;>%hz(y(IC&PS{DDiVAx#Ii-Y_88FxgYA_*Whd@ zsVWke`prczmo^j%grIO`tRQgsdAPn)1*#@0pte3jdbNi%pDRgt(Rmo2I%A1ZuP>uq zu%6_BH2X$1QBLG@xQ|*(<;3be7qIu>RdDEkhOK8BiK)3Qkqz_1_Hq+Eq2!6z%#NbZ zs|#Wc<}lTMzJom1s)1({TXBYwn9LnNQK|zy#Y_hvi!R)e_CSMRe$9C7Sa*dgd>I4T z@fRSs`vK|PGZy-G&%ujFW}%Jb6Lz*m(5P98R5z#xo$Z=s#*zzYah9pMjREx>zpnAbB|IIGWEn1Gi($FvovC`8_6{w#+J^ z-=`hKh?N2ob@hzUtB8_gYjf${_A$biC|R)jxm5UI$Slar*2EVdPNVCa33xX$R~$NY z9^5)7#HYoR2r2r^=S+{J-iCg3;l!JyC2tmVOm%@Ns>Kl4Xbi#MlHhTC1}Y8xj7NVu zgVvcTcu2egwQXXF*e?^D&R!AsAGJqeDoXdu^|8K|0dMOzT0ZFqd3`_)4arIAzF!%N zjh4VCvkGKeyI{hHGi;;kP3c@aLbI$1d|9iBhpG<4+?k>9d1wdmvGW6!k2k?a`fT`H z_y7#`k3)LCj^v#o8PgIsaL9|=2TM+S+O zS4O$Fm9XDzG>VUm!TWIz7^6wBa1=G)Ri%!`>N zOzq9joVj8ix3?`)u&Yp7+t>h8v_azRnE!;ayfT*ES-nik>2+hhKVRcc&h%&Ne?Dh& zBpcb>G=083w3>T8Wh{G2*MOZ=m@P=#d7}KAU?Y>yNVxdcA>68WtGJ$!h5WG5CERwy zC9K)6B5rh*2K&DM5)|pg76j-rp2}0$z#X>*Ue_B1=ami%q9!C&)b!>t^`cb4H6g>jb$Z6MrXFBJ zUB5B@#a2w?gc0l@-%rdt9U1oNJ5PRhOd0nvSef5aDdqE>eZo{WvAo}xnu=97l$n{6 zPH{~!LO$@?vGUH6-U=07nQPx|z^*zYz+j!woOOi`EdvvWoymX6qYl$C%CfSuV%9S(kx1YoLLnSo``Y0l0yiseFc85?|5uky&wg6|?-}EN0S^sIt#K8@X2*d7P0>SH(y5LtLdW zf}7bLQ*k?4hF@}M7XSJ7EG|Wwb3XT#3%5IG81H2>oR?b=Q^8UV-c+c@>t`FWZ6~iY z6Z=ka>&F`MPYpH-+W2^8|NacFoo51+RGr`C zy_lbOYdq`u^$zn4WZ0pyKbYuwn>dC11SadbJ`e9^@K&3yG64&L-TNSjjcVM<9*?w> z#!$Z$d|L2QpefkSd@Hyn*d-Xw9g*_aEpJRo?A5mIwFn$01If>c`HJ=F_fOeTN%x)8sBZE9Cx*c+Bmdv$z7T?`DjfHnW#Q z_p)P^7VHRST)RhEMbFH+{GE5|?Cx!|SPLmbHG6YEvsWAVigZObaP$z?^j`&YrEwE; z)O|Xi^}v;%ImCgqteMM2$Bkg!rY_-IbG&%`=f`2o0WKka7<=|c z7Sp02&)?hng6q9iT;a91h8g$@{JEndf!E1k)=py~>n1W|yEeKpq3@hnFR)_=)=3#T zPC|aqHy>WsZV>;70hZY?rF>4xY&NVzjx~Lhz%Akyuvz?AcK!Yl?E8eV?9b^rf|@cp z?t0!gX0h={?r-@}-rhm_JGVNr(JN;2>uQbJRa$}kE(58U(E``qZ)Q~Beg znmC2Wh0H|5S4{2OW$dYtFwXYb5Js~|kx$#OiJKhY%eEhPVMD8J**z{9<9z za&Fo*7d~c9G&jNI9P@pR9qV{&I{WYy!~S=zi<|sy6u;X9nQhsr%*-bbrOoDCc7Z5` zGfDbZ(WHEdDf@W5BK}q*7vO2cXAEkrm{R4#PBu)exN&PZyS%W5*;i}D&aPR^hpuwt zP4D01LNlZJ@2Y$FG0{8uN+}0Q`OypJZMrO5xy7AHX&l6+$%U~21-_il_;xNJT8q6N zI)gnta~@-)8eY*EwSW)#K7!wN^-9I>k*O6M*7a48)`tw~abSzmoLDl*k?mft#XsZ| z*bDtJ?8+Pa`5TtetY+Imu4dY7euBkAE`Naxn|yFGw@*QV)3QIuIXy}f1g+o0hd(&P zKUXnfla{LUSAN>DW-m&ZQA-b2Xv8txZ=D#Hcur;`7k*$i#Pu>o{wMgXLT@Q&{hO1c z(L`Ry-H(0iu$ygDu@ZFlv@xS|A}jVze#VtWEob?gZv}x3%UIc_wXCUz44?eKg6-)U z#x{4qVm?;hWgZ@p=JT&R#MN(2(VF*~2^ut--!CWRmL2zGBa_`&3$0=N zmF+cLN9`0|yJb4}{Im?$G_s{)#!pYaDG<33*$O6a#(&K5DYNBv%h}Evs*ufu(t+warg7gxVOA_xyETV=M}FkS^p!2%&OE2g3<<0#^Kx! zuJM_e+4ix7OZSf|AG4&A>GhIfr)OjdHaEI*cB-Ek@g^_6&1VbyIp3KrpC{r+)qdpG zbq(XQ<@9-_RQ-6B?D#D0P6K8|@W@5PyaTFa!xyRbRoGuWd|Q&`((L;l5jRd%88B6haD zCws|ZIX|alDcgT_72xo z|Cgyr%c}UDFoHMuEYGS3_cPBnoS%nXNgY!$@Yi@GD<-autE^Dz=QgRjPGz2s8TPOm^iNH7;=nTM?!(oe`M-b!Oki za~ld(`9XzKoZGOHYkPi%`_Jw+*Wdk&DcZHWyhG1~4f^WD>Z?~X^oA^#^G}b}Ycga? z8gFyT*#lhYm`Fy$<0dm<#LbE|>*ny;^L}zYp68sS?*s@$`MnigUptuFHa2YC<7#fl z`?+iZ^dGpx6IiolYW%qk2N{L;A$)hkQvThE zAh!AJXjZRd20J496tmQj;f>Ww7>g|-?32mvZ1WWX|LXY#W-@$cu4@~y-c!t2?`s0y z{cdG2*10$tOSa&I#O@&=%o5P$QSi)SX{>)^p@?}$g zN`LQf6L|I2_G}iji_=`B#O#wYTVpduv;EyA%=wq^nO-R)wOS#UvG*R%FH4iRXDhvnEu;kHWJgNk-3k#^5HD$>+2QYj=P z63NI4m4=kek`WnMAtR~h95b6lMr4n&$0v~yzUTkv zx;q@9C;cu_`jiK?V;y{v(ns*@ziIrtEK?Z2a2hAS594(jJvjTFJYJIWr4KH0qnHK_ zO8VRbhpUCd@U{?$3mpOtbDVhgpSiHvu#9eZH=%^*r$klrvRWDABXoJpI(lZ+lVw6i z@t!PwzL@-jwrQ-TZcmY?`Koi^cNuPbbe{G+(C3rG^QdIB413*uNh%wj3;9FU`E%nF zad@E%obFl)0}~leUmit{Y71%TPd&UfNtP>|9k|1{i9$|3qT&Arig#;^MZeI|{3LQb z{iz(zpEXpu>$Dc(+o^4EpvI8L?p9}SI|p1lQwG<~j^~p~p)41C6|7O0r!7{)ER~Cp zncPg-arvP1)gHDiPzQ~3HRA9F89sHfFMm-O&k=Ffw8YgzIKIUPr_Oi*-OaZ^-j=ak z@@72*9{vkXKTp%opo#crvLULC&w$m{7OX93ag&r~J$m0Op=Hu&UVJ{2)J!s{`=4o) z@n06qGw#J_dN|V{bq9~kS?W0V{v%K`bmr|7%gAJ&J}%sx#{Dw%_)hbG^rLGu?)Nw0 zD(AH@{arCwZ#h7=(TSpLX)HhMlE6)q{e;Aq%h>hvP(H{0d$1 zb*?tHCM8hWx^rT6;_LsNb$2NC;j8ZlN*N}-$#}K` zAAOjFCyp(~zR4AUtE<4&ET3P;74gRBvUv3R26#Bo1!a3UV2=L=k8i;T;A8A%kE=4t z^l`=qs#M9L{zhpS^JW?bja)~?(!K7qMi(!s>Y)4G4#?1*fYXde;=aZWQ2x6I=!{)U z8B!*e%p^~q@X`@gTRy|NZThUY)`(UuQA7P3zI^^s0sl#v%l@ak;*v}QmMgeL8I$~J zZtw5l{`NaH+D*W)w1K$poe!V!3xhoYmbm{%2h1xo#~B-+!RRFu>8Z>*A^f2g>6Qj# zpLPvQnWKj;Qs2A!TMEA|4nU97uV8lN0CpM%DC-r2KMU!5Qk&*_6>5nS%~i4=bQq&wL?_=uEYXDPl1=f_U) zNG3xto2*^C(6b84->Tq`Tnp^BO%GMKDd2tED2S`hf=rj*_}8kOPTao-rJ;7ZhYTsqj&Kkjr1+}zgYb6bgnap(_I(YVZ5{~ei zj+&Ql(J~1qd=9eWLo*ckpj}tyfAe8q+h4dgz6V-*{h$MZ_n^XI1uYt@P5Tcm74zf1 z3rCDoJPthRNwRfY=)2D{;pFZ%*zoc`Sgy6^az)9q#Oc4r_1tV0Vw=Ju}30Es?OsYX<6DyrPaHt<>jb5zKYa z;&{C%^c!x5Q4d#v?Tu$p`dbcHPAP-5#tf*)ZKi2T7HBVJo6C$G##gHwDfj__l{FJ+`Xbi1_I z+?p+2K9hT35U13;^X_Xp{3&oK4xR0YZhMV!>hS?+*IOHVeCsdPbeMs8*MnljBpVK_ z2IP5&83kA9qy8b%tTX_Jm)7{>YcNlm;mQpUq|BMUayVsTKDq3SN97T5+;8zz_UU=6 zHYsi#<>@(cz{v?TOT`O&*<11K?Y(&NWmkr+MX;-(0s`WVapGYIy!mjwI4?1XBg1W3 zS(@>jm~>J6ct?7U2b9mt#iGvm?4WmJ3K9(AjOIctVK>AiJ> zhMGYByk4D$b??q^I&x}H%Uh7fJ;xeHwKrn#4x-0b9th#Onds}j9AnPe^YcCisO1=j zDu1JJ$y(*we_CdkXY!q@TVB)N^+j~?RX_e1>V}<3&e-**8uyXjh2^@=m_ExIOC1B* zD6W%wZNCZ!(^9~FkUMTPTn}1(r5fhapAV55K3e!e>fe+&uGEZ=G(7@k?GD&fFp^uP z{Aj_p5;7v@@V-l)oOoa$AM*JtE-hS)L7^!0)RjT+cZRs+k2HJGO-l?s@4&4k;_?XCH&T4pNo)dmNULh3wCc-AC zSsWXYzujSyckho7yRPNH)bASYdUnDSA8qp_zJbU`OwCvKW>k7R0*Y zm z9;}4CaQIh`szg(4d-4Me*YAOS?K)U;WIcIovcZt*1~`+Zg8g@H6nffUrP{~OsCI`v z{>-~37~Igqn6aujSeK}KvL!2fjl)}U!FV=q2VJt<0{7l!!APlwmmL>4c(E$ZA8&}) zJGx@~vJ2GTM}>E+jOMrBCvmEb2UhGH$rsi{@U^xOUL)NKwBc0wft z1ch?o;riNJt;(=bIvdL7t*}dfDTiKP%qQd3Sk`T*@TkI`tE{fls?PDC-R&2|<4*`R zA0#|gSWh8#LvZHGHd67T^%|2EBZdP?{mP-UcI3~ zZ?2GIR}Q}t%=zQeE*yC$jNPw|z!Sg5;qf3{c6XdW&yJ19i#sNvhLn%)WHy|CFYbXu zoBQE{^xf3{ayGUs+(OM&8e}o=JyKuK&93B)mfD*>h1{WQ)%JF8i zEP0-!V~2A4#_{@Y25c(Tw|?in#V4NZ;j%TB$1WYf39sJJoWsFf_IDRet9&N7T&STQ z?r&%)X=68CGaTJR0gm{{ab%({uFk4~gFP*{q{^2vsvpDZ5FgRktCiwfb@)K)Bl^i% z6dT(DBYHYrSHQ1HU4?rY3D|c*7JF%A^VU$HE4iJt zge;)Hku}DP@5CY2tHJhcsQ9B`DCPZF3Ob8EgH4ke_DC$xewpdRf7Jho&4xcy; zX)_sZb|Hn{$W@1xxwE|&KKs%~tiEs@3}XKilU_OSWD^Uli*&|O-r-pP?!B;Su>zmk z<3{e|Mw8+BLF`@HlXHSADQ(bv(emyVI%zqWFYC9^p!e_T-sXi=wY5Qbyh8^S_Iu!< zZ9eS%qm>435FuoeF7A7%%?7WplI`JdG;+%E+Ls9gH}?y?QZ^6E6LZjSPdEPZcoJv+=njbo2gqxSyqm-FCFY@wEP&HC&IuDGJs622io%DmweTl6+5C@By3s)Ku6^ z&J#>|PNxw(T|NMx#~4t5H&?N6zbfj_@4~jD=CX2bB41Q6#E++pdG{(Qd(X`#{OFjl+3d`WJYu({>?FIVcUkwYs*y4?| z26*`23J@NShnRgjyv9(6FV5-%jjIiCbzClGWgR7(v@9?f9naH}s-R1qDIUzIg6SU= zxpQs^?mgcZv!^7I|8qZd>*>r+HYOb0G!zc3FQ`*hvje_#F=uy^hZHCAoS`38aJTC$eksi&Z`g;S>eDHF+xE0r`B(m(Ow%#ZgB;YSOHvX_S@4oKCPTBjOswfX~wJ|6xa^J2!|I4z!VF$VU3UTZ6}Z zy-hdm??a=^7-+IqV@r;Mz~xuK(Py17RK^mgjJ4s<$F(G0(TKdl`?&| zfx38WF^|8Ks?8iuNR_aQ+G0ApDw=PR4PW~3o|f57hU}^_@HlmcxKpTt z$NS~@fu0@Z4YOoF1zmo-yBnkr83l(o^yPD3wE4D05E?J;g7ce?(=n|zWNZ2!yh5E( zsWk+T=X+zbjT$@7?x67!Mx8C;@+UUkVRX(>So_G8yR>@nl^W@Do8|FP{X~A|m&4WO zlSv`9ogCE1ao4(Rp0w#6&D@^{_v8*z)Yn@SpQwX6+jUT7V-_E5%;vm{o5lQ7Wi)Dd zD;@jq9gT_5VyFIku%c@W;g@3Zd(vLIl~+s+Qy)Nj)0SH2(g1303Fd!AMZ&ue_SAXR z06VJuc~M#nkBt6JawAq!?mue@?<(?5ufx*aG-xHY+kvNf{%38x}G;= z+c{e(Y5r!=$#mwc^@^PKScfGi8h!cr90u1S^ctgx1&to`>5?KY^OI&cPwjZn&KpAQ z+y6kmstNjUZ3b~cq5IxF$m*6=FeNBJTs`%sAln%Ss%MYXwvYV_1O6MudEfSut#J~r zo9c{9KGe{tg15C==@GcKCKM|cg`zmi09{@rV)y?h;q-5j>^d|0_-EA_xHNwzs@SRV zyJiLS4!R-6;>+6jzh|Lh;UeM6@cul+!-LnXX`>Zqc7b(b7c`uthu2;i2+iRwG+Ez- zKljpQJ*!yU-@2TvMvS7{s=+i>u7=iScdc!$HNlln>cHnfme_asAiUVE2aiacM=JY{ ziqj7@Q24>eFyvqfptd2Mx*f=FU#wA4I|%(s7E$ZlIk;UD&^6x~FIt@??Y0M$e*Xr4Nu4kY_>Tup_Fq}uNdI4!>XXw}Hwe-k0ld|p|CYh8WeED!No?9)QnZuFv zK<*j|1;60A^D#JhSDTLpS5py=<9qAkSxIgXX0PbU^Cb>+grk&iT44r7<;ThWq$N4m zoDzE3Z?9ea!W1)~Y2(Uj6+BUE%Tc#ybKmO@TsGpI7%lgN(&9QjeqGuH30IcE_si#C zaqv^vav+XNV`EukGeM^OFJWCv3?BO#g0~OoBe_4NUk>wdWdHH_ZdEp!zn8&*qk{ND zRc|gX-XboG>`57hABB4p7eQDLIUHu`E1g?o+^MaI!|!U)0u5~%{7ef^ih;QCU$%J8 zteysbwZYwWdidWod2amY%HA;|WewX;5$oqmyub^Oos+)MfvW{jXS+`H+cby021c>W z*FcV=<$#xm^F2ocC^pX#+H^|EMDYN0AJd)h9oA#Pdn0(qO~C^{O!1RMiJ10~aDlM~ z8i&a6vuDwiQgH`5E-hmDX1Q8h+cj{j;UH|i*bjB|c94PM6Z*3;Le$e%=gLWwQ8wux z_|5TT=N&Sj965%^6%FGGLwP)snIk%<{sX7Cm5|f#81#tjBW{wo?J=2Y*lU^+j+p%d z4!T-`oc|gz5;xJ_BquiXYXIGIlOWs9kS`ah(daL|U}th8gd9H#UR}Sy@rr&ZJ9RLg zpREi(Ct7iQRTg&i$-+~=-PmBvO7Qb?Lv=kLEDL??(YZAm+duk}N3&k-Jc)bv-fzxP zzQ5pMVubkYfIV+KIR?G69tjN_mJ0zAr`PM(K%VQ>T}q+GsGFNfy5RxogNGQid}>WMkdB z;rMjgdDwYo2oKOOCM&;8K3E&YC0=T{W$ZH=tXv}8uifY#qqq{Pbe7Uy-7ub^?8|Xd z&1|aOL2EZS;eyCwnDXO~kkX)m^WJ-j>ogVN;$=(t>pPLX`K!(O1g=Tx0A>`MC`(y;K6*v}3UGtO`yKx5hzB&3Vt+c;1_J!y~ZQC|Gm6 z3x>3+v$j$&+Z_}*s^%2cZ`Z>MJxc`cb4$1qxRo}U-Ghr8hT`~*4KU*TX)>vNMJNhr zBx8elUGix`gb!_%VpVR(eT7G2htN9Dfz7w~$7?s5VPJPnR8uwPpJ|^3tsZM(^v+`|Isrc@+YbnA`cI&<`2 zZjOCj>u9IAzUJ||2Iy;AMej`PxMrunDBEs}_eU;*jcet&=}4)#vC#z|T~a_t3CDjL zrirh=gz&#FdR!EvD#(3xq}3%ec(>6E-WhIzB`Jz_j-f-T@z`-GRiT9nfPJ&ELlJ#n+oL|(R`7q6_e$3eU9iIW}Yb8p2=em?RK9GsCv=9V@b_x=%SrtF8i z374Va%1!v%)B#;H>nJ^CA{{*a98^9@&+L#4XS}PZy)jaQ&8B>(=Db7HDHqQ3=3Jp^ zzrWG*R2R;=@Qm)%>++s-DWkeXmM?9Wa>VlQ9~bzzYdpzQ**}gL9e!f=0$g0 z+VzRpykRQ)-iYPPtLB2Ht~Ymo+W@PRe~2zB2E13l8^+0JbD@f~VB2;IoJ>29S{3+^5t8Q>Sm8YlP*!- z5e1%jwI?L*592AbjJeoBnwcy@GPtjY8NY+^$@0lipqj=u#jf;i+#n7=-$C<-e}X?} zwDCST;sl3JkQj0T!gd{p7C*~_}#~f3&suLn~klcGsYcFm6(zx{n|hKk@#|M zcRF(PE_{eshC?s|O)_kF_1k7hDqRhe_xK+_(D@niM%4=)PAMq%3&HwWXL^0c7>j>x z_t2el2!@PIgjYu^!L+WG-rvt6d)3yQ(&*lX}}djlzp7V(tkP~1MF3f9k#1cSxdSaeez zyJ>pSUeM#q&v%2t_p$t^ZYJlYorM`8`78u`b9T32j+}Y|UUcpe^dGzOpd=eE-IYmw zWsLB>Lpb(-;D~iE&QR=)zI;tr1!t<9gyfi>tQzjjT|O@0u8KL_yx^Jm^X51nTfd(U zzM4jrgH3h=9!y!*IWGW{ntD?0GqCmNji_y=trBFzE5 zPC&Wo8$irof`gA=5g%W@LcJDCm~bzVVt^8klnY}c$^CF{{1J%Q>&0h>$D+S_GKNcB zZCbqszkdAOV^*&De9FxH1f%}F+89BpuZNKH6gxFfdC^2c+&o^C#m;q7cTFA2xOn; zk*ZcClbFEU?atB269ZX$g*|WX(v!>HeioP1$YHsC7XSN|$_``(zdnxS+iqLI_?-hS z?drnomrH&8v}rK^bSda4d<8x24^*MM7+;TDfk}=EtUhcyZcsiAHi@5L)g24oX>w2) zu-=8Wwn*C1eP%f4!)syeE?sQgs*R&(Y=;h7D)?GX$Mxn;IO%RUPxmOLm<)TKt7pu6 z%p-BIoWSK7xkOFNXw?N(aZ!mn_8B%1d`m3ObXCX?O`|fJLM)Rf` zp{aQy-DILrs@&p5HJZ<$SI2dFpI# zTi6}z4CipKXNx%eU?9%Ct$HJh>Nb&2Iv!Lef%#WmZbNG}E z^zxGp6y*=W=kqM^#Vk|4KW({dg7-HPR0r`ri#Qm6eiSyPrJ)c$1s69o!tGFf*1VQ0 z(3jbepz&w6 z*e5IsFL$xw?>Yyl{*a`f95W1UYOJ`wstWDD{2O510PMHOiVI8PL98>v@n@ZJYR7fz zZrUIF9=uJ@rF!7w|GPGF1=HmHkyt1bidL6eX;9cJ!7)6ZmYh-aVr=e2UVK1-7aJR34^?OMcTB0h z{;rJduIWS1$Wvf_XD94?co_zq)*~f1BQ`25pc5{)$YWxERBBYg7avxFLz<+2)Ry79 zVO6w#xFjLIz(5Ar(NQAK+J#jiR3-4;kf*ExO+4Qth-p& zZuZzG1a;}lDFeMh--j#3x>2$sF!z$JnMLo$8#OV~XMCC+M<+0vGIdAY->ko>clCRwPBBuaTtFaZKfs3LCtB zybt!hs>t7aA0)?27oKq=mF6t%fpR`B+*taR8sv2-Y3w5iC;^^kT1(PUpEuptg_Chn z7(Fixp1xZNZ+lnOY+2lo22SY~5jOYw2)>R@kUHZeHa$lfBe=n&0vPZXb zuju8LT^_y#Peq8(rM&2E8DKcCZWXt*%UL0B21~08f z^P{Q#VE*4SQ2lwEGLN+h%hnX)&9U+LYS?38gJV6owT1GJfN(xD+7>(NOgPuf7j%0s zhhuI>z<0j^8uk84jyB$G-78R-H41o^oPfE9r5V2NF)D0mgK6KEVE+EWSnju+#^tBc zfhQ@PeI$`BjZR9vnhD3&ord~1c6fB6DIb>!;^2Nwuro**uY2q9w#voA&Yh~98<|TJ zy|&T(5#6Ci{uoVv&`uNjcR_RI8o~XjfYVGj39D}P=c-A4q3;rVObz$J2RD15vl}4o7QuO?Q>mq(Cp&shfXBBk3z`osF*bP#IvL-gF#7@z zceAzdyst8zS#p%rqC4R126s+PIzjE;M!29eil^n|@hOXiyeAkic*J2c*!mkzx%`7N zZFTB8um?RzsRU^YsL)p83Ps;e!n5D8oT;{-9*6qlM~6nyA3V@|{1;fMQ$ok<``{D( z{;~H z{~Z4#X?itqM%#YibUj?-@sCFL>&}K(KGVXtA8CiyQ>v5R@lAg&2yNfxIAs^$oZvK6 zyc~fMHw?wgy)_A(?6KA~3zHv2)Lf-CFxYX9VD}}M`faVFhbQNgfxQihY34kaIF77=-k}}!mXUxuQh^iyw~)IGPwwLydFNSeUo;G*!?2RVXN!(a$%x%a0 zXh5(IR-9c2cTESu-PRIeEe^vuCkDayZxT=Q#*g=ncgGQuuI=m8SXkdEPa&%gPCrH=IRjAklAYwAcqQ7ETJV%xpXmNqdvc9zdwJCeyzrsE|eXkw=C+WGuC0*xK9c4CXc}W?|_4(LtD-7vS z;M3MkG`gS|%+~e8OIu%+Xg87ie5Pn#; zk~W+mTEBb(-+MQMS4|v<$Cc8_A*wIFl9_=wAV zfStc&FuFv!wtIFCoLD012aXOw_`DZJjg(^xT}ys6QH!rN3)rAJ9J|Q>rKof6EH_h; zl2+@%*#E+?n^^}82zXetK&{-pv&(o?mz{-g?Z4Bv`V#kx4@RPT*))v*u$moz?-@?Fune$$B(wzg5g3XR2*-NX33vmY-LvM&3YrzTQ9oy=}bv`etig@ zxNFS64VS@{m~MP*??rn3>pAIJFW`uyi*Uumcx2;yLlGL8fT7eF8Sll>AP|d-2-bchFi=FPtgcE(F$?^PI2AXnX0h5Ibc@ z?TG2}5c)TW69rf39+^P4Qye*Lv=tjk&@xNy|ROAOX%-WY#9l zm*cGk`#iYJ1f%;7#G!h+I8!u3nM;4&@%VeVv+fbCo_7nRyvA(xy(o+U4I}cx*Q>?314C@!{Y&G>HqvhU5U&u-ZstCI;Y#gRU5U zubN)wl+ZJ!-e_ZHfrXRx`QgqgP-)V_{+TbJbiORqC>222>HfSW*_=%-G{Dhc8(`CN zb>7(0pC@*+qS>KXAO% zkOpoZ3x~pHVbYXnELd-j;bWsA|9%L1_>IJcuLev0qJFsO@Cmvx_#ym%yu5nc``QLpd0% zgzq+oV@F^d*N#o%|2kH~tl#>0GxH^&g&eB)un{Ul4%6O*8KAf^0*YJD!a~!x(DdpG zoj1P(fdjka^Sh(@pZW_LweTg4ljfHSy^U~Mc?qP-DZ_|-O|*Ys2+?<@;M>s#s8afl zmb7K#x;#HU@~3MZBfV4rcfM1R-u zIJapcnch52o%vSWXV*vapL$K$?&C*U2Ef@gNvnGJS=;@gcKcr9-0}zwUKYYXM#GKkGVl>b52vm)K9fr z-D_cYdpiI7l)}U6x071<HXo2L;1N)U-T^@4eRNr$_7Pj_(WcoVGp74tPYZ8j4gA zQ%K*ts-WJIS=^GTj_JkS!DrA@>Y4olHvXs(&-(d+%GzXc>{4yqch`d>=NWQF=tH=B z*}r(~dLSF`&-UfiC8{+z z?7pxT2GfQ4ft-<-Ea^49`03f9TrH!5R`r(bov4L>vkdsW#6O?%kbz_2D}@&4XnHg( zoX$UOhnyaf+<9^;TWH*(l5ZtcZ{Cw>z(2~Wc|r^JTo=uDtQS(--EhU3FSXs3ZqkZx$#~iZ5F^aN0aK;51I@p}=rVAW!=^6d9Oa=SWm!RZv2J(zO zv2PzIVU_fJ*$(-k?g<$E?Kdf1u*Yoc(@?H{R=jpMM-bjAqVcb>sG@WkK77lA3mtK2 z9Up`pjS^pOW)01|<6&^VEM8rCjq-}3*fO<-0_6WtSd%W+F45wSTTQiICtt#$i+MHe zT`l?CN@YrVU}r7=JDkfX$pL`3=W-=P!P zr?)f5O!+GqP|4se$m*X_2 zWs)B)n9cI6rBs<|qIFdY%={N6=zG1Dc-sVg+vGl?=)22m&NX3cj>yEyQCGkBE}tFL4M^wsn6UV92TC;vvWrA zy&JYX;^Pb9U!Der%f171pBZ9C=Sfi|VH^w_Ci$~oxsd*u5=hGEq-hhzieE465--ho zP5UEG2-nj6-DCFsrpDX^?$#|2cUg|Zq&?Ru>_sXKi8JN!xu1XJEmnXr6qvT!>m44<|M&QC8y?nl`$D%-($vKDruH%ddFO49%y=24%dN zRSRL|efVK}D1MxkNIO*?!`|$kQqs?T@w9FNP6%D#ewSCh)?@L1Jl`N z*i?RdAqLk+Nx9E#_jdt-MONiRF>1eM=91cPHzX~%;dWK!ZJR-y`b zeZO1swM^p59_swYDh6%8F6Rch7+kb>7<#n2@vb?h+{^s1dkiO9T$9iNp=^ zSD^H(3wl0q!l8);WWH1ruS`(q>0K-(J?J`8 zE6z>7BDkAuqooQT;oP+q@Kf?%O=y?FH1iwOxO)*9ZS9~g6;(7Y>yyxBUm~a<+D=nL z`|{2aNjR!40I{&1yww(pSHBMD-N!w+r&k?l|7wK|p}N>!(*})?Luu|X1q{|6!aoP~ z;#o-wqDjbV8uDrw9=+>}4(BT=c+N;v3yI_Wdn5R^(hW*@AjfIFbl}2ndpKz^ljdxD z1Xjsg=wI&-6yO=d^H+JH#*HXm`8S>ev!>t~IWMvC>JZxR;!K-e262b;6Bsq%H3d}I z!&zy5IAnbd?X|VW9^t)l>ylwYr&3>Bdr#oUyqoi95f~M2}_FqT60sp_lD081jo8=NPQ_@81 z2VpdS&uMT^m%+!eqv*x<2y7d(6e9z(G5wD=m+z3~01p>(;rC?Lu)hvw{#$6Op$SL) zxk+wAlSp~Z1XP{n$nVV@xFs+Vss>fV=;&}5a41hmaN8*C{t<%Ty7a)2$%C=BMh5D> zNqWTy76MhfkuLk{a?;d0Hv7ba|`d)-9Wv5{GKLv2wJeeGZ zw?Wa3TH0*gLVlk*glk*sgtnI%6xLf86>bG^im4|IkyFH;n?3PKo}pMfO`E2q1z_JH zig^2KJ&o|YDC}tkyf5|Y{dH%E!_u_bu;Lu$9n?EMMoW-w;BBZ?{4}`&6 z<#5UAzf`udCl{Q%4s+(Of?I9joW5}Y56*W3M;&uqS=0c2wl~1-+*>#l=SFkYMq%ik zFf>?`C+H8!5n^5P@cW+}bl)uS<^20pQ?!y(;}-CIEg!UsjiXCmx^N`&lps6$7j<5U zWiA}eS8Zl+t)naY=N7`Qgg}lwYt5Pxj!Qja!mZv8JoL5;_sfsxB8$H8>lwn&Dd#~m z+Cr4+y_;rM72(VD<@oi-05(581|L4wMwQSeu$|$~b$>(&@3FW&YBwa7sG*X*G78SY zxGW_UYfk-fHxB(A!=ZQlsko7@ej?Q)?;)sy1tdSk4+(6csMzebvu<$#!*1xj->GE41&4vxC` z`RrU6`hFKZK{d!!{w-8r>;YZ!%f+92{z1B`HcwtfH1OqW;X%6-YrkB`51%jN=;FiB zrnjEv?fL~_V}r;mRSaj<@=JZcn*knd&6TA4X)O4)39{;AX*a&dQZXm$9c{@5Wrod)0^EM$JHD-B8?TxRf)$E=3o)VD?&h zo+iq+)!O;DlIfXQba9go_1m)vdUw2nIoyb%L(mCpQZeFAxPe@+UIQfObi1N-MQ6Ru^t{L3B1p4J{eEj?EZ1D zn&5F`8QmJ8$knKT>t@`6U~xOZn{3$@pG+-!<=Eq~+Hq!FSbh_&GlU^MwMjV$g7&GgXU^Bt4{` zu9@Hwb(>`0NSai;_poSkB`I2(@wQtIcs)m0MsgSkJx_+EsGS6AVY7)5^Mr^XIj zRbff_PI6LyOs4-S!VZ`Hu;iv47AB5{m0I51bKw#UNm+zy7T`0R1vt2~KvZdbMEce0n5rh}U4OcB z_mP*We7`K~%B`W;!ru^;UJWXDj8XT!E3P}AM14(GLw$J?U(D{$7f*M=Y_lb`v9TWb z+{O;=qQBFRMOozjxl-J2E#Nu|<_*UYZ;qEI?QilNd;X~yI;k9e7+YEU6 zB&PPo)*u{V5{BEf{rF;{ERGKx%_;H7_1BBRxa)ejJV%j#EhrH6Zv3E$i&LQHy#^;; zG~}?S8FXMz1ou{w#a?|Szrw^}wCH13-YxYEk^TGN`;qfN!(=L-Zkma6w@k;JjYGL; zj3L&-G-Vz^ZNZ$QE>o9q2KA!1aO$(hpMZ=A&pfT z9;e24;tx`;nKoZ3<824gwmq7~mA=Lg>Z?Sk^G zE2OjDk7>B1-!ydNKN5d1{KPRh{AZ+K9=F;f#0zPLS|#PwUZ*2fl2&l_F3@$~@;^mq z8cx;MhH)~4LI{~Mr6e*$!d}m55@jk$BMs81gfu8J%RG}QB2gsCl(W}!%222z8B!|E zQi<}fG`{Ekc)pzLT-Vua?PuNhz3yLw;gGu3hu#;Kr2PQYin|_4a?Z*WCg-E&sudh9&+z4Ha!Wxg#LH$WOGl z;icA4L%g}9g{pm?b}Ks~4@M5O0{#u(-iyZ;qwlZNCPmTQj+59mOrzZ`x~hP?JBiyC2}1&(`LS z{F=utzp;S3JovIDJXvympW$@N=zLqtreO*0E$P_}3!f%8%$jC$op_(J92T6)m8suv zMUU*YYPcY08NZxZbxK%q-jgisJYWTbS4QB-)+XY0&72o< z#_y#;OleyWu}#zk5;%k_8$CcyWEq%l8-Q@XFg(rY_SE`6Wji}>khx8j^li&UR^g-^ zte785@*j!dBcpj(dD)I^b;=@s3pK$Hh^Ww_INl^kO8Sa0DP`1v6M zZ>}WVx<^JZ>yaqj?dwJO+eQ};mSA(tD)bgs;cQt`jM0ym(xFHh5@#U{0+ZdqJ!d{N zrIg{`*V8!XC%>W!iX~(|pK1R6+fSk>6N0ZqV_44*LEte>4L*j-@S<0#L)XYlYMbOq zXDeu;vaCCrcQ3*+6B9h#B*ckc2;lD_%DEz~#|V`?f~$Y7(2VhAI7k#AWuG_|c;hDEiD&1B%o3vGUOaq#@uM>c7h& zL)C$FbW#URZF~wcpJPGzPZ(y;P^I(c7GcId7gCzmg9qqA>={W1``-u2FUFF@CW}G; z`*K*YAcv}3^-)0|MR>8m1zu-IFjTIHC_U0e6V*waySH`7u$>4dw0?zGJ4;YvPbB6Y zI83yxqnK}AQIOkphdcR94fbx6fcj;>sI@S_jGl#{Hq)8tJ<})6_dl~g&fFu-staMF zy%u}J&JvB5Q+PgjEq1L9L3;%xsp12)sBI5D#T(^5IH>~j1a+~@cY-8Ojf2&Ruj$s( zHLP)pK180iV^{zENc(r^LEzsGCLm`7|K5v(og1g}xyt)VV}Xao%EM9AzR3t(JjTh@ zm;Ly(#trNpW}x5sVal$|fQ_Fe@bXp%9Geyhb@>ADQ^*}UMva(rKO9Mekt4VcUxz{A zi*Pl;1x*tZth^TFV!ebY4gYZo@=_IHY|Tcp|5GRor)gtI@-Q*%jG{cvpCsUbI?6_W zB^%F{^LeJz$&uP(THG6n?nANoHoq1m62x&${X2qQ-LOujfUMUXrp?yMyvuLG!Rzci z-kRJjt2@os_+?QhF4fC|xc_F;)g$(denlMQo?VGUSrc@8!CeT7b;tFbP1sSpkE%q+ zW6}E@Xvo?E$I`WUk2_-+PiZb5s8FC;{`_9y&=u&uF@SwivkyjWmH_ueIibDsys5<% zj84fq`u5NgJaFJBz3^%^2u_j#_m4tw{$mxp!LE#KvHeG{i@w4DxqPT5$B=y%iMjD@ zRHgPH{&0K&n={(M)NU5qev;!g-pazD!V&tu!xkEDd19W&EhunN0pB(^RB~7ZfopkK zS#t?D|DBH8ohS2rpT}a)oHf{)R!nQBXp^y3GQ84kE?44nCKU|*%+?wFLggxJ@-Rc4 z{CF7;yA7J~;)ego5AD;Kud#uB{rfrg$C;BqGZWeK_fC@meFWni)=IU{0*m`wY5Y8A zIP`$exEo%N;YXj4wP83Q=25koQ1rJ1AT1hwTUQNs!AR zSuN3wV=LvLDocp7h%!ECZ7-Rr4?yPd0`5bKFYcr@(1l7@Hq;?AjjBWz6k%0Ai_z&!L_ zkG|uN(0^$w2^-Ml8O~dW1D_({0u~dgZaH4Yw;DWgW*(?aEum}8A{iwSX|%XpNt(SC z`Hc5-wh!!V1w#-# zexF=gZ;pXsX)tD2fs1t#P&uNMOiz47#{}QfZC{qst~ZsisIHH8Y|KH`HL+xyUnG$X zcgI?#txz&*0y$wpnC23J!Fo%v*=T~6Cq&V&s-tXU9o%_>YFIhMh+%lAAub!k9W!6GQ(_g%Nm+z66KrkD-EOB@DFZ(yw=QKupn(b#97gJGS+Zq^LK{VY8{6YtO83pl=pxIo&0< zCL1uF#5-IRfg~{58}cdGa=nDhR)@`$Flj)*vU#oC<@WxJUsawb<>W*yVaV& zGrtMb?s-FwvmcoM_lJ2gV>vY~6eNON*2)a#Aygh7^6Pf zN%XAuO!zUWg&PPAy|Ye+@i|Aa*Qgl^@0QXY<~?gD=6u)KFNF>8E0Ax9uu?9X|TvW7e6~iVfHx>I@oZDt}pch zg$G?C z`017$pOF>-vM&nR|2Dm*-ObJP{1q3}eL597iiKf~Zzj|UgjqS>6h_hVLaKI^L9sg` zyz3h`;OS;@QaM))Jop^1)$?v)=ii;o&et+lwm)WqZBj1Gsc)dht{hmZ>rF)OSW^j! zY9hX=o|JK=aHp#>$9P!`KK{3a7JMv&#s6`M)^Tkr67z!U1g2OzPD^4_VzoH+#u?t`t^aW7+Xb=1@CBa+$YY1Ox zGvM(43^@>%NTgxx1t z{5jS%jrmZp7pkn3d7n==lA~j<@wUDY=eVU3Wb}&=g@xIy{LxBuMSl{tYZ~ZW5r*+7 zHEhtIfd8zWa3o(G@v#a7Gc0p zJ!5Z`36}(#K*RbqNi5LAObHpz8Mkme$M2rsvfC03PKDAp6_P|*fr~TLzmX{uKS;3T z9O(3%4PWQ!Q7N@Fx~@+TXE@5>ns@C)WAQFJB77Q`dFhe{0R_0frX9wAeItvNWT9)y zPbS^j6Hac?rJee6vE2IAIIV{Jv6cQ_xN{3oHYzCYMyI79ibW#q=|CfuB~9~J~= zVBTgqA7w_$L!^ZvRRYTw@D? zd$?4g+zU5`igQG=0Oha55Utmi@M`H!U}Bnyyo4`GzVas~L6#6NF98?t4#3J=+8mvT z|Hz%ebXK*plDiKKU_SGZY~9gJ#4PztyYlnUUGs++8lNF6gb%_)hD**o%A|Kcr_pEe zzOe9812qj6r(B~4IOivV-naszpSl1WgKgkX(K({G(*fG!6kx*aC3|9V2AtpP4)04H%9-VXlq~!M8+3wAr$N4Rbs|lES^v2hcV18C;KUAQI8XX`TQJhkx~>)XgiTfc|E-UfzfIF@w0DxQQ2EKAaf>=y+GmlOPG=#@F_g*FlHmL;RiepN7wJMO zg0UHTv{Q8wdbrIb=Q>o-E^0SccZSj@@mbL0v6%^pj0B~;Cb-~s76$`E;g{@o82P77 z+jZ*6MvskXVwnSG-+Uk<`3Qd4Ai=3xyMzDi&VZ%2h0#y51Rniz0InUM4R!M~8OgS$ z$L{wrKX}sUdGjpnnrwu9r7y_P;oBraG8C0E?!lff8ye>FnauV@qCER;Q8exOK8(29 z4--x-I@u)9X}PCKhPXA3H=d`PQ}XCnSi!8>69pr3#-Oydp4RqDg2RLjs;r2_za|%% z^xVa0mU)7^JY^*bTRw@@v>kzqk6WmwS2()g0Mhoi7|hN_z`U9MIIklChNuA5YBS{Q z{3On^;t12JJ~4Q=dpE30$YisF>hQnO%S=m62BvQNN&=55L1SbKdTd>Q-n|+)_$3dl zWoB`%%s)(--$IOBPbdz$+7rQRT_hr#QlDMQpd-BmOjaGE)xG;@yf{FBm<;8&LBMpz zAIqo~w3I{>`@ct!`CE*GW(lBrXc|=SzJ^O4i9l2LWX@a{F1n7VA>O_XU1?!>R#Fg?jT`_&;N_zgmP>I{h~G`rpVPXdvl9!Fb^(rTqu4!l%XhoNd*LoCo?7Oz0kC z=*{PA9ua3*JLQ=ic6tc5?8*VolS?3E`~yw*XMto>BaNRGj~)?bsCBe}eys|Cq`GD- z%j^e%6O+Kj=`1Bx1S&NrfzU9Yt9i5r%jXI3ys}=9)eojY-P#*)^MepcXtt-f7aqXe z*r}YVr^}&lJ|z~v!%1t@6S`6^iKydus#CrO6JO2eoEd9@&9YKZ&7XUA2k zIU4I~Qed)078L6Lrn4{Qk{J&Uf=R?iTK_bX+$s*lef-4Nl=GUn)KP?{?;&vi<3*gW z&thsq0Uc3a1JWJ+Sin(bGwivqpCOvNG0_@imI)Au;REPG!^~G**vamNd;~n=QDKpEWhq@R!X29PyO?v684$w3H7aL(`U`Q zsoD<%im&LCD@AOIQaBoiMzi~_F!)eNnjCctq6$Z*QHeuN#P6yDpS@!V@8o((nAS0} zG-xR{xNaaNgW^C`ykYcA8XkULM~2N;p-_)9yzP~tT?>@Kj8q_9UIE^olfa|PjFU4b z1wwQ$v%BAgk@i(~_|72K3iAW-qUlF!^CJqrynIcC=4Igxhd9E|RYAw5a;$1QOI;^t z5piXGOm$fV11nyzbIo`tkocL_WKN>He#BB)hwF5QOd#Y*|0brMAFRsmE2G~Fd%8;L zCCO7MprM8Z$TlnDS-m8zTXlpgdY(toYp&e3d;0hyx`02=wL+PC9`0MYnD%s*z+ho0 zt@2L52M?miV~c0dKXMOuE9}N>B|*&8Pe;`|{Qp-l3T^*OB2lHXAi8J+?7P~Cx@TB$ z`XJ2Ym-F}T=7abk<}MvH-;R1dD`EY4DO8GKV3AW7`7fjf(j=C{9akYtvlGNBiA`{L z9haOqR*0~%0y#sKP;1wLKEnLHu=6PCi26#b#m%VBa0eZK_nz)N&gTuQ4iLeA#Smeq z2s-9!Jh$#75LMNO;G#FMHC%v~&e7l;^}WRBq>sa$KX=enRF+J0`ioJEL^+MGxy-!r zsk|HMIWSLW7O9>ZNN4@INu2x|L0prCCB`lE%`IWHWDnsx?|4*q`h}%`9a!TXbD=x1 z4HWN3QW?t(5;W@~un)qaOJNVTnk;4_Z zW-ABA1>?Y9CJYj@pAhBQi`W~hhG6~a?IHQ+ABc8XhpJn&JzMv!5 zIhcosALOErpkn%uXn=Yw4uq>>%cUqb?*nBY zs+OVT*japOGT$n72}NzPo9+leqfu$n1@?IoAg{c-**?IwVG+_1iWa zIQEWxzvC}%s`!o<&XnSjzm~+mW*TqDs0PQ>D+bjYwNcnu7xtf0!SAw9$oG+jJRe>D zENbX)FHA!AZl`0NBGDorP5{ti6TLMh(kzzSGh8BO2bH=xa%A<1x%!6y~SR0pt8Z=GCkm63G2Ph7*$MwSrG;wRX7xDXSR^g zr!$$wGfn6pKMN4?--oS7GEw%wt)wc=3vu!Zn7nTRDbXv3x{hA*w0$b>>5gOsf*z8U z5JGhLbL;bsQMmGp8m@cTO`Mgja2_X$4qWSE7Z%&WZ=EK_J4}&^ZGMhp_iM0vbrDhg zs)P5MMd8?T4H9}f6%&qXVCTXxP*7M!b2M&({|rs2Yn}-gLp*8Ha0uw-z9Ogj`|P>d zr%>zG3s*-+e; zlnA#)&f%!g02>7B$)U7k5Vq(Z{yJR-rtwL1eq;OhjlB zC~K&JWK208kjjL}L}U8iLJYj({~_*dfz!`^5w>bAocA)M7cYuHQqp~zq1M5KcRnEl zWrHNl{1%xShxszv~(r>imd_SC?gleg)SxsR|-@)YGn zxuc=YbI`k%%ft^~qs1va`s|GYFH?2_#{T3&mSrLiRay`S>pE0gv(M_V$ci^Y5O|XBTG{p_UxMR=~ z6jdsich*_t(UK3WVMGcmPK9}!=Y-Quf*0`o)nND(Yl3~rchIV}6F#pVrC$4A(|H4eqEzzr4$pl6If%d->)4iRws< zc5fy;zh)X!`jr(hcE!Wzb9Fj%GoOQ&z6B5RFN#Mp+MmTBXEjv*o= zDX_S@oi4-}dT!-CGReysuUXE(n*A zyk}JqyHYWp%y%a+_^8KG-z|w=z2cUR&n8jHy*=!bwJUL9h7@N{Ul}R}m_h&3Kctqq zh)b>~V99@xS%ZGS@b~N1ZK4@uiId%s%;?$a>7+IWN45B0p=%wC#!HLcnzB zc##To)>S}JX*gfWKPkW8Z4dP#Z4T#NY+jsVIPFm(qpURz~CUI(tb(! zY(xAZ>Fm$9jUkn+b7+i;o~P^A@4i+`{)|seNMn# zD)r35Di8Wm`kd8my=VjSnDTm%?(JZ}efkozne2dd+#I4Q z`W$rz$EiE3&KVC@ftfi7D`!b_qr);8MeP*IX$&HwOQvu_Z6(oGD;(w;ouyI*H^IC4 zFe*LFgLxUjl+CWkHg6sFnf6{-86QeoUfkdYCuw1WWdM|ho+IqG?NBWn!~JtWi>&&6 z7|V9urTHoi5RP?Na>Nol&*#!NSK0`>-HQzU<}qes^%ys065^p)lKA-~1V{+;Rwd|Q zjDrX-%A*5 z@Q^L*des&15&xmdHmX3oT-3`0m)Ur9|K2!X*uK`4(dp{sANqf#H=(kPpA zDE563OmfzuB04wOvwH+M@<9)15`WF!-q%W$dRjpy%n*lc^5NXt`|QF^VmRt|15UI& zh2^@k@U1q1geX`-hk-nLx{c5#e_cAD7Rb(@71pp@{UaWqvKVKcS_=wVvUs44@3JYG zVw`aXE*Y2Qtv_x}zk6OMQ8z<@pBrYLrR{?^3A?d)Y#Auoq|uJvuehcBGTMG#23;P# zOrfhDn-Fo0RNig?vlCxnxs4?r^fHG}{cR9r*++}#iKFwgjd;6T1_FF4aqYhl7`k)< zvbV-U&)NalC~1W1`n9auEMqwSQG%(=FT~JMX`Z};5qRx5O>Pd4K}4z;`L^&G9^SQ% zObXeIY4PHi<@1Rd?>9mx6egR)v{CT=258=14FBG@bT^ncG$iJl*dEKl9aumaj1_#`5v`1y3)e2_x&;W->fvoo;d^;|S5D<`XKBzUgYk~nm0lsV&V$SGRp4njkg*fHG+ zb)ILU+uL9=?ac@NzITz?<$99X^pw&Btzz6=ILMys>BqTOFXFXB%J`st5{SS531XiT zVYo#L7gluBD#ui?m3KtHlb_jeZ%GK+Yz;M@>p5k67ogP8SGvc_5t1HW!JAiV>429s zvdirF+ENOJCi7cmdXY>ti2@rPMi?r<8y+8FIX(BOf5BF?4pi9)9~?VIWE%+Bd?R!;-YtIXsxjt<-5b-ink@ac8!MZ7nadamiM8kbsmbl zZ^4M;uL&>n3+&>(0nr1kWT=3H4=y$lX!{PmjjrS&y^qt6G?AAE=Af-ML<4)j5gqnA zwO#R(CTU60y+HxEC(;e-51z*R;=aU6q7Xxl+rgG8#jv^iH_bM`4dQd()7|m95X{%_ zMTZptp_pk{c$YA)+i-u{d$viPfKZV;Mn)CkcV!d&J$a1Gg^!T*?>;!EvFLGW8n)kG zMmn~=g8Z3X%+V$X{<$TTzB1_Hp93$DdXF2ZakZu4ko7}sYtJUa*Z095pD0?rek=S= z7)ANiWmwdf4v$(cqu;gN5LA8;ua*YEp&v_#wNVg`{o28n_jl1ZQE@2wC>IklEI_C_ z7TU;6l>IrMl%JEwYH=|R$8sy&30e)#aYKw`yC^SXd;znr`3(N2tbiKzWhil;i#s%; z=tUE89&gSs8o>XZQ!@Y3WksP-^WGWWPudQgd`ZY!wH{A?=I`6{k;aDau-bC#GB(-h z!Ck3b?8%eDVy+q{Unzt+bBbV*odL%IYJhHk1v*`pur5azRRoWK!Ld{lG_n_xGLK>D zw;8CNx)O3Wrr^CjAFy3e+!bdXYIA{uw1_t|yb$o#M3hq#-O2<7-;& zgUr7%Mff(-O(m;Bt;iW$D!fhxJXdbjSwc{y4C}Um$;6CQf(;V<_5v5*3nC;DsW7>neJ^mj=bYmu;5()D^XZVg*8mE zD)|Xn7dAmuaywu&*8t3Z*3kMh(cIZ$F@)!EnDu!SgR=g`gq$sh*4TP_EIyYsHDwUx z8gKSCM-|O`O~{1RT-;-_7*D+(pd0vl>5CC(+@RG8BG(vR`m1R~uJ;{T(isBkVwVXC znUB5x*I|HqkjqnTY7LVeE1aOTQxHUguj3*=1&(ay4aRKO zB=i?=$1?lp#Oqxq{UxeH&GuzmHRq1w*o;4UTc0&<_-2;=fET!dMAlLH*BDpPuHW_fh+ zaWp&q8Txv!kT>UC@bT1E3+04xC|vG?btg43^#!let9*ll_k_guO`elPxl^#`t?<(Mz3eQ#ion%scu@+mj_==o$jyJ%2%L*cmu?lsP*L!f{^dDwsCo95J7B z8FgY8gAzZlCu^#KZB07lRG=pNwlf)ywDSF?EjO8Cvm8;wdV9GGjsIfY}}hu!|GlB$t^qK#umnxgLIz)W*GTWW3>yo!9NEr)+RBYGi+h&%o5mk zoeOb30$AmnM_&m&# z^8FpxN@+F{@K(E;7EU=O1 z&$>g=z|pw?n_3#^S=pI*Qdtwbk~?VV*d=s3mIpm1#=KhDS(KCY8uxum1Nkg|KB8C| zth`&vY7&gX+e+qm#w4L`xKu zen!E+{hAQ{@EO(p*UeRYe4Ot2P(#P+=0fmW8$76Tix|jtk#iE~;lEvviKF%p7;Pcw z!COYNIvmLJ?W(AyvYwpS^ossG_7=B(ilhT79qhujI(&!D7V<;Nai3Z<3ukMt(X8xzcHn0_e>TX&hY|LO z^KZe{5@mS!YLMJl)@A;B{v)4UF5{9ZyK(eOFKXSlXI%0`IQ^1WAm+In=6jl=%YAL) zwo?X&6$PLjHUV$D3oJKx!OH*<`z8}K3^j7E=*iM(bhLGnidSpT_92%Cr!*(G} zURkUT9J!kSzkgWZOW_7$RP~9fw4Wme#+l4yzpD*`n{+^Fa2E(DsiDdmEA(4f15XQO zI6X$nxajdAOkY_>?#XIs52}UtV@6Lpn5#|WwP`jYgvQ~G?r4hDWqC(xPlC#F3-o=QN{&bh zVd(8F@+}Op_0{I72NeGB0^d#fn)iL z@#&jMZpC;(&+*69t@{heo~|SsO-)qmk_L0BwwgSXSc1C)wLrn_AWH8z0=^cpq}9Oy z`ZKd}jg<@*NZul?ljS)k?Sw4mxBH$CuV-r~%4qs|CE_NThLQX%bisKk$TZM|kgmme zr{N9>_fv-17Zs3`Zw|*gEOGwC5Bzwtlw}->@g3ifdHy(sHQ%a4{FCaa`Dsn;oa0Z{ zybxi=?B&UuFj1OjnnKvyqTmwz5?hXR!kFqa^0HBchPF@TC|V*TlsE%E->4-|(pu5B zFb7MYF5#_l$%I?!XEArH7G0_4!$iK7Wj1?Pk;aG=a&t*3ee|S{hGuL=fp`1BR#=XA zT=gEa0Wx8;+EbEoX&>Of2XHN2ma_5=L?!NkB`Pk1TZaruiewmUsSaklLtV+6Wf$ln z?o!-f^a6+5ORk6;Hr=;iS13s<6V6aW-ZN;qbKmR^DMS+h5~=S)8VX=D8*`T zC8lJ>0Q@eyfll@P?9M+4cs}3*>seKbB9nzE({h&be%~dwv1jR{m6f1kuLE75ZldJo zTGZ(I2WFo{c@^A7_%LY-l;0Ai3v!fjtyBVelP3#}oZFZZvlvN%3-^TiT4+u$Cc(as z$#7LC3a=Dqz3g>K%GEDyroeZM9Snv$;d3D7RW=>ZAh?1o#fzyQ|q zZg&pB_~+6byrhW^H#o$}#tsHz|KLXTPav^(IX;h-rcNQAWZzIZwsx7Lb*3mxy_pUs zPY*Ir=Li$Dx=S(}QW@RBgAg>~0&a%-*!?^o=+H2Xn8oAxab5cP{6)6nUmE@l^Teuz zbgKV?pZmUJ!6TnS=+3iW$hHz6JeT+b?<#DAtCceuR~SZVx(=>?G~;X0fAP-IP9kp{ zOggM3sppkQn((@sRo7O+n5biLYC#^lC#2HopRch*f-l1OL%!-7yxj^%KOi{c0;JQViCJb1>TX>DcDYNiXnGha*>Hkx8q^_b_vX^r7!x*Bb~4;KQ-<3Fe^3vj z$7Ei}VjO;-hfr5j_@CI4l*U>D-mbt=CefW#6 z?RRIMosI|ZWLqdP7lsQQZCsyggj(CYsaKjd2A#+!hb-)=|1y7Y*ei--R+D)bXBUF7 z$ZNQGa5^VnH~@^@UeT~9U-(mQ4AW*Wz;Q7f7|F^b$b7_7kL>Y&g`SnQ#A2j`8T>|LJiDU70}|lSzwy{gC14$WP;D*)N>jJzx8;{@?juU>nmx6B^&JjgW;w7~w@XzmUxI5kf5C3^XwGH;+p>wxz z{~sAtl@EjW4-%PNQvN2TcSB~3)Q~O-Ed-P7r;p$ zJi5~!r2h3YZ}P3k^Trr7_7LFc`(DGV147KZ1NkVha{|M+=8;!h=0m=O864bfNv?c9 zh%>TuFl+xEx}aw-swdrogTo)7AoMGyBoD#X!E$cccn^-ejY6TnIi&D@G90>8Nrl!O z!Np%cvEw2Q^uxUOtXi26Zh3c$sz{umH~dkWi`Wb!{0yQ9-y_dDqy!tO zC&q@`V&eAw(9QLt7O@d*+Nwi%Q%sI?a%~BuoNK`BlVa#8>r860W8hD49-~`IVbUHy zSk1qCBWH7gM60>rH*Z1S@O4)V&@iRUuNz?e!57bH-=_c8R1?kWWK@_g!Txsr1JwtW zSnpp9eoomA8K1-vdM)r%oFE2%%%#KOr|__+6TYweNe68nLFJLZpw3O91>)<#G2IG& zRqKN%w;3*%Ot9Gl(zvI|0wYZmNcJ)b&dVTqn0~yQotcu3%k}4@r=%EqC-&2%nSErE zR4b&a-9%|4Y0i;$3vxzVnsZW%!#w}whf^D_LE_gsu8Lm~*XZzF{NN>v3u-gS?Uxyl zwPrWexVgfP30bV&e~EP-XQ{6-2jZgn8uKm{bo?1cvun4IM$xW@AEy)0eYg;R=<8u2 zm+x=$vv=pKqFCt#BUF4w2D$382F}~L!E&$3+{0r)U;1C9@7h?r@zVsN99rN};%1`w z@hQq(Ev04e9Kg9C7%zsXvRg71%fQ4~b1{sKFu*+GTf zHF%n7f-RDkRJOqf^#3b>*}ki2o7Odmwc!zIA7t*_6NR}#j$rhn7=8RdS{bZ!Ku_IF z%;_p4sv%$KhNp!P?!xziUgU$N;RNkz42E8}9B`a%Li1jR!2#bA+VuB6of5kZS3HPe zPBkq+-#Eal(JjQkdO!Wx=?CqGD(KT9!08oFg(v^gV3(8;n9itx584*2-lsj#an6Ix z>a^h3LYfheEwQjud@CINPlM`gpN^+nMnKZzKHlTC0qxVp`Yu1n)yM#)`*mcBdk4D4 z^;2(8W7KhY3Ac6~WMqSraMJr7xIeHIt-a@2DR6W zuNCb4McW5b@p<_Q9AA@)j&)yY?-2=TS=ma&qPC%JQ3-Q5U6-uBY>0aN`yJJrj}m)U z2G1TDB?5Ee;hg$hm|+`^eapj9E`0zO&o2O%q$Na}a5u z>i&=kZU;Rv$qsk@5#=o#b;qgvvU3Za2K41EggKVy;m-_FcqyMvPeqxNU5cf+af1^| z{mVeP`aF`$CzMT#DrV$WBwjIugDp6lbkDhvsoN>7nRYntCT0j+QIZZsZa$*>mBKfhu0& zNx}KB5+Li{;epU>;{I$qdX+@8eIhyJ+ASHlSLMM7zs@3-&xOGIxC2@=1)ykSFgdzB ztDz-n4$3qFEHnbVa!N>{&qDfli4$^qZ$oO``QVY@VbF6aPgJ3@`5ACaI>$xs^+ z1qt@MQ2cHjiCfB_{TSQpW@ z=jYT{UI5jjjp$|hiyjdy1Rm<4sCO=iS!(irF7PD}pR6aLuM5E0M1)gZU%~8|CBeJf zFUX00kDMP6U zjpR#nMCv;~&YyE#`^Opfe%E@|v!1)ugB`m$jeWblLYQ1XSva`&8(V6bQ;}!TARN$I z$*z|BfpMP?2}^y7#p^B1-JH8du|Ey^*?q|?*(Sws_Qi=6{E<_Q!pnhK?Cqexyn0j| zd)Yr#?BbOq7GzFj6^+eUN#oAHnkrX$O=lS&%#UMtWaaZMky38!Zmkm@)F$E!O?u*@ ziNoCXyjB)|I`N&K{i{TLv*aP$+wfSpWkwFW*E^bBHqMEUqLTV8dIUQdyP1!w+r=tK zYT9)9mwe^saeNX==66o}*je(b?43tL+%(V45h^aLWm9reg-7G(2pb!0h;LddUuS5{ zQ?1p!iGQBhd(3J6ZCs1cBq~Du;ISIJ_FJ2{_k4jcZ*IA3d)+lYHj#6+F|*^xjhn~s zDUsuMyg$GWQHiT4{^TwE(Xm*3X#P!M?F?UD&iN=`wd5#!%rlOEJz^Of@UcW}x6Ouk zTAwJ4aJ?+PYn0Cq{chqWd;KtrvlIFK1sOuCTMd$of^lxAGH$aCK7$qg261e}m0)p9 zbOwL_g=C@Z!*RAF=bJG6y?o`L7g4TfyEluCcb4#H4rK7sDJR5QziarH9|U5(>)zsh z6?$$bR}_K#r#SYE%`%?-x0O{m7R^SJwY)+8a$$8qTE!MC*^0j=693`+uVU@^-F)u0 zMqW#16zd}KcL!|WC~g|_Q7E>vVh>+A&t9}9{HoYIR_>k^Tbg6!c5K60-fL?j+v=an z_qms|gWc+tPoA`hPbPGVQ}Z_P3LRFi##_V1W8;!sHELwtHrK0ENd2yoBpe5^4u`k0 zw_Nx05;Cbc(xsEn81_T_P;)(B^L&?Z@HWRjesfefdsqSMs&JBjf3974QqPpl53CTH zwpy~zaevtta+Unx{$khDFES+aQEgUgvJQW4dN#j*$u#~*cM$7;L6glIo6SCtxX!

    @=EWP@|le);^wvcD}sI>V?C|D3m1K;AdHzhZ|A>{_nCf4_^DwEyL!P{KHINP zywK8CxVOwfC<>m+Yuu1>bCv8t}enz-?iz*xQ<|q4e za2UI?tey{h@JG1Pc|T8+E(#x~k72_ScC*tvPPqPfnk&AWA84J-RzlWhFvf%R`}qNf^d1tPuAdIGV3=|-}S|a4*v3sv%KnQPuE_Pdj7?ZB)({G zKI?W)rgCgZj(Bl_4xj7K@p@Hh{N+)Xh4pbO*^*yf{06-+)}UOT&8f{2Up?i=%;&nq zyRBaHjpV2+XYoOJqwH^mEAyCt_~6>)V;tq z&fUp+yA%n{R7dcCeU9+6ZpgAPt1bC2XicD1W$vub6DWuGu~YZDt# z@moCbWeeZ(d@EmAbh5&uVJ9E4AwkT#4CjBZUL_u~_cFi8FTO&0$rRV<^!MW7-b8#b zTW(4mE_56wOD&E)Z7(X*qgN;29 z$G=lEtc-XZK%_Pnv+K&$*yJO3`GxNs#OLQe=KY_a7t?>k`Piay{NrlL{!GphKARX- z=1Ug(D(74mQq5pqIoyw*aiK@-sWOqhIqDL>86(*WiRb-lvx-~M%tw62tXOt!$SL-o z-bvQr>}9?^=a)ni`5^pu_#(UR}-F!TU zpWszWUaKtTHUB-}KgXVxcu^+sH|G?x!T+RK#<7~e=M~8-c2u|qC3v#!B`3sr&PJ@( zyzlIB>vHz$muEbhOlKPcn}ipNDtVP9zHEQ?EcWn-n`~Ux(99^bF7!>^h5R@`q{%%-va?9O#@tjP#Z)_>z1{-jR?zbP|Em@soce<=El zu;TTA_?x6xobzFS#b7DVa$Zj2WciKks(aIfFQaP2v1|uDA`xNpkM(7xbe+=mt-;If5PYziuOu2NL-@9%kyU=_*dmy%wwS3SlcC-5+ zwtIF}nD8%~?f7z@js0W5?)-9t4kYJPzz{?6gbPw^+J^*ozw;CxT4ULuOmi9(JQcT)jz_W&8WZ54jUUd=g`9bn!1T!s97rU$i=a5Y`a{|!+V4l`&?MpPimF(US;!R`(_E}*~E$c$0Ud& zhx=AkTv;t#W^dy5`124pApfH4a1{&IY+RQ3!JK5aBwcDxoFV}lO`6O$ zj1vi!q@~?%J~|>^`RRzOga31W_8nRNfW~zCcU}bFX17=PHp-kWJ9dHnaj#E!M?8*K z8FEehZl^JSDg2f2!i~e?shXDT=MF8lS~s3IADGQXsW0QJxx4(%E!Wrx<#2w7;Yj}X zx>v68LFf5N1BK#Y>rb#|s}$WruUE0dzZAHB^6eDfd=$mHH%a>Lqbz&e$2x7rE+_ni0`n;;G3^y3j5O(g&!SC#Zviu*$KUxZiU%({L9DsY_d;f zg{w~z%S8PVYSl@(*=!t9aW6ELA9S7~tj|2h52;(o>TOEo@BI5Gv~^SieT;|@x)TlTuL`8%bBe{>@GyoPD~62}Yt>rYd}PJg1=@`D57 z>;E>gu}Wpa)h=uKgDVXyH>@%dpMJbXnDAbMRk*s0f6>qK=fh62*E-wSFs&Bh=Q9sj z)2ctb@Or#BX?Gl%$L!$u?0fIp6zIXPl0U|tUngLX)#R{#HE&64VLI`coWthN^Jiz| zAFX&);z*WFogZeL|(gt*lk}F_NCrC9b!!6h^4E z@Rtk)JaaTptl++y9a&N+{4nwnUw?QfAGAWFQsb_~i8${_#hqEo!o|DhvRcsz{GlV0 z*s{D+tlhBj>>Q0-tkb7*G@fo>QMj;~AF)#M%yda(#b+{vy#+Gt>Ef|&`%@?Ja!D)1 zF6T${-NIy6r9ru}Wv4!yp|O|!mQXFuTByQ4`$>5jr4xMc&n3cbH$DoRqqM~plKHad zs8Tj;mLhF$v=fd9~1KI>8=z%B$G&?4eCk>`I%uiov}F;>8p4*s_3Ay!jMMHs%T82iM!Ob?c-m)82g% zx9=<9FJCBQ{ojT1ien`ynl;5|d3`Keg`!qaaR z*>8N=wzHeaJjG=8XVMmN%9nHE`^VM^tz%C{zyGxn^7dcd-O#% z?Pp4b*O`|UHLM$3wj@rh;@(OpT}|S*Dzp48!9M=dvSjwOsf??A*f^ni#!t6k(a=GVt_j&Pb z+0hJMMM~Rkm-G6tt&)+m0+_Z;vF25&si5OtRml6=mI$-Up0l>o^w~>8UD?_4huF_!+bTY$N_N|$BZHF2XOWF8mCsOYG0~d#tm(bft1{sWAD(FiC!Qpj(2}AX_=Qv!d*%Ht%R3 z!^ef_v+Vf2yos<_=>7SW@Nv#oc7I5&n8`{KkJf8r-{!p*CvH)88>o69)UWg4i#c=l z%7`eo-d!RXwP*6V&4a@2YS#SePJQ0Ed=}pta8_u8v3!e;0sku4n7{l~hE)qn<$t_t zBUe5Ag@LcOu%lkv3tOJu=NtQ1u=kFz{3NN@LRFbQc6f1g#rdk`{Mr^F+cLzM=VGjc z@0KrOCl9sdM@CojlC5dJ!rj$%RB9p{_N}bqTFOUp>E(TV`irl^p3)St;`k|S@YzMK zrt{9QcGF~qb6=k48^e{^6|NK5w?~Tj;2jZcjeG)M6{8^ZeBCLwZT%};yKD^WIkQOU zb#Vs&qHXs!!Sse$65!6OD$7;&|Fm?gIAhDZYhR7 z=xXL8y|w86`sw0&+eTct>k5u5m<307?1Y$?8{m8CC)}Z{PKRt?jMj&?i;Pb%#zQr2 za4;YPRlM|w%E}RB!O+RfiYx<~l9!B%WmO_k>TTw1*JJ2Qm4_R?Wng$}0xV+kaByBD zxX#oj!pRR&;NQoLOufpfM2eZ0^L)z1mUm&D?N!`#Ig7LZnv26nzkrGp6F3hEXa4F% zLuT2GPf+h!?TN;np^OpwDI!9K5dpxv|+>q^|1h$Hq?XVgM^JzgJvv_67;_f9d3PfvhH z!|vfAwlVX%rqUWeTY6O18`~25neB;}VWU$TT=1WRbK0vwR368ip1TXu!)tLy%^2pu zA<8Uqoy_?y%!Em1h`Ce$;myUy^zZ9*Q9Z8(7u6DQQSes0;#3H#XEn&TfB^~9sF8c* zXhvez$dRMdvT>Al4|ct;h5jNVPF0eTfBN1&=JUrSTw$+Ck~@cySAl_~&3+e(@_pdn zW(N3QV~JMrHt?JNifLYW1qWPLpmAU_1i$;hy?^eF*PG5U?L*}0q`BHeTCoPqXKR4W z*41#TY&5-gL=XHAxN`NwjHxWKr{6coqF(PKL1O9zB1?F9Bf4=a+4@j)2%NtSB- zmxs}QshG4~mK+N_2s^HXqJ>EoCvW);&fgt|>&FIyrS?lK^0y~*th2HE`T)H1TObhV z9OJHO-$IAGE8)1oD`?d|4{P7JQSsF?&{r@GA}0UkO!L&S4%frX%8%%lvr5$d%mHFNdxXSPlgvR_~gc*^fH@65O zYY3*NW+;IXmml3Je++RHuU#N{ZmnR+l#}X{0WXSQG$FMAo$MfFU*4nfvZzb-E{RkRk zO~_LfGcxyx5{c@bOYQacbIaV_iTOJ-^3y?&aPI|l!}Hf*;`0S|Pnb?()L!AYHQsdT z9!+|wU-(iuxXOdarp^$^o=Jwm{pAB-MOUJS%oqU0;+6fK%a(Br1yOdiD6_dL~R+4 z!Y%Hk>(wK)x7iDKB6f=o{$9k*wmpUmrUsC>5J%GVBL}88452116RCxSLIL6?HMg(2%qZFTtl0MnJE@cjmyv&!QC1JUmu9jbzRL0N%07iRj*NIw;|2`UuWJ z&7dxwb7u*zJM9aOwoJ{{n; zALxRi4TYSy)L+RNsf$++mVmj(ZzxfnsWHRJ=x8}w=%Pb! z&e?-b!@J>~xeK@Ho)MW^d=JN7n~p)N7~I6(M5hoXQu4G_G-Fycw_}Snsd67e=k;wv z_ufZ{|m`R>{2&GlTqN$40IjNm)Fyan|@ zXE>%g43r)0yT_R1KF3J=bT~SBKWvOwV)C+hfn1(Etg{F}zjaygbG;EMF53(1-QGUR(%ynXEzg*PZ9` zj;EkYiWwf(a3%X5C38vXv+(cZr%)Lwpl;t(@nO+8=9hB?j?;Y)PSJa~w1!ledbkJ{ zzj%v>WbzQl-hmG)^T_^RQWQEBX=G9z-U^YXZASIDdgxa;^J6(qKhukMTdsmhaSZ&- zQK7PWvQ&2pkM*S zJ|pgPvTx&XVQvtbhz>L9%?jifu|@f&5*V@jD~_>cry^RcV zSrc~$gW{{fa>!aPf4ws)&ssqiTseycX|+fOrb4!F0^?|7K}(m))>3fT4ZI1$wbB!P?Ukiub2tf8Yto2wVz7|I7gO?d7;KQ;r15?_$i}CqqQoLHO%D z6Dy}B!irOK%j{1)hH3!|J)Q<+dZr;gx!f6tWnJbX{8i`yMTGm$_JDT%9nME%&?Vpf z9zJuCgZp9|)U6vry(UC}<@U#LewHu#Wj8V-mmb9F^0DCYs0b%T8`1Xp2e{?xs$Afh zUwCbf4UKW@g77!rT-IMtz{3}2gLH;AecAewanKx(#=Q;#W5t8q&=b4CaUc^^{oix> zPDaGO(28aaorliVj@YB{$)#(f6;n3lI7BtQfkXL{v$|#z`syfA-M(M=d$$@59+eML zcKcxGbP2~?Ai>Jd3FOx6dt=Cx-;mL)OaHt-4u7P6pnW)lMNSl(K53Ce5?bDJ)QULH zO>kLwr-)I$_mkOHs)U2agj&kVGZuesX!ho6ZeV=|W)z!~&d_3R)&9d6YWo-(r>}vE z!}+ja$!x5B>H(n<_pmOj53WVG!S+f=s_Xm#-&B6Wtz`-LeQqC|dN77eyzGI@x_)j> zV=Cm&kVBV2EhrT~fF1KD)2Ni85UM`|=EW^Vrxo6e-Gryu8#;yh_frs$&14EyWr%}P z0&GihfnPoqcxTET41c3UC*5jATgjQ8P-{R2Eb?KduOkuNZ^VkW`FQ=vw&NyDIA6_-3p~J?+=+r`78h~<<7!5ENHP9<`iL=Ha9?yO zM+~1go1QV}5%=;oroXHl+*c=JQe!j>*K%Sme_4z&FZZGReF7zh6t_u| zJZD4?;^6O}Om289W>y=~o&J%6{%l8Vvv`AUlTX5Csg+Dg-$ooYQxl67Jz>Wb7uc8= zfNjgS!==~PAtT!g=AWr#LcRwuu~uVAVLCvaC<(IpARs>co=!;!9#+AF6rNx2@7^? zf?MabY22q_bf@_p&ibM@RhH5t<2zKScOs#pJxScaz5x)ukkl`)8T!lWEqDbCA<2=e zLGZv4pXpSKJ`PX8xE>8`lGOOoZ-VKB#$e1n{U3&mxPPV1y96i zkdf6!z!@l$?1dxA{MLN@bG8`655?dSkQXt;GI z-mwjX6W`Cn-xc?utUI17c$J0gUvI#(*RFEods4t;!Ft&C(}KI;+sJIW)`^$eK9ui2 zO_)yAX2DgXDrT0#ZYbQmifIYl$b9vCh-TUKV3mFWs`g9cU%Gu;DD!II6<^7uTV6j23t4ung_2uEVoC zhLhu#A0T{qC-cs`5fa*;Vc+Q{r2R*k`WaEcjdsBk1;e-=^U`vt7l!eK{SXHPc|M z<08;pc>-6jOcwpTwTk(5Qk(hgJ%iKm+z#9DJ#$`F3J(u`1}8R{F z%FbJ;Q;o5vv{&akd|O}-ZE6YJ!lnKUIne@t^(^VQbamP^s7KlJdtl68514Sb2HT8`tC&mJF1hyob$F;ppyPV0v3}{=TWi0;{{AnQ#fp^Y!RGg%~t|8oV|75_iej7>b{bC9|Gng5%3X zm@&JC8*|YdH1*}^g5G6Z*vRehR;mI%musxtw#smAz`8@o1 z;T{?*`~cDC6!7ZLVnRPCV6B#w|F;^9;@k0hdJVo$*bi+hPh(lo z36!|2so7?r*^iE(`a?sespu^uIP@Gp{5^nL#nKEtuf(WzS&^5v8btNkEWGyBfO+}K z0dAjUFy7+~H_Ryvv{J1}_XSg8G%kRu*o)Ai+L;u^$&u{4PthjkE2;)o;AVX{x<4@q zx)xsogRlDZ_9GG2t=WXDlwF9C?jMND>;=Kd3Ye`ji+=c0jWjC+D6TPWmr&<7tv0eDm`#bYs|4B^lxyDpIlwm@Jc6hfUTy*l;4(L~%1%`90 z7=w*f7`bm1F7XeCJ-Hl=HhYiF+D2s9>k{U}*)DYY>?s&HA4WIWJJBiH?$GEx08&Ap z8PnW$sGoQU;!qVe!izb5NqwySLzx;qT<@YbqLUfThQQl~`C#m*!0kP_foPpKBmGac z=<~J?e4+G}+urt)dHa4X?%QKa8ck)eH?S0+=beNn=>uHcCmZw~_7vSOM`6TnbE+LV znJVVU!>B*QXyEHG_*QZj*68RUqv22VU#x_;|Mg;ldFvI^@WeU`0WE=0Dt@ zF2^M;L<}6}&#XU;(7euy$Qw>3{FEwC+clf+SYrz!*~)QrAGzIH#+1R%S=iC|uCVDMS-#Q5fKS$$Mdl`D`{CNDm za~a%QqCn=>{1YsDRLPB!mnDK6O|opczi6N?RT~`@ciI>O#5Rlg@~9PlPWC6`T!i%Y6BdSzlrXg%N?~cL7~zUGZSqngcxDFaj`O2m_Qd1$ zts~3V{Yb?xyW?Q)-Gj*MNfTEGWk&nmS2Wvx2B%7Dxi4G^76v}&oLoQR9N;;R|8 zQyW1xs~FHFBh1LATT@72>`8e2ZAz}}Zq^zn3xCk>a0oGT*WvW%99u{5Sz z=St$@?=)cI>EqD2q6r__cXL)6A8=Dm7PIH&G@|M&D{|9SCr_qN!AJiE6U9hlCgA)u zSSHiU$-L8|rq9gDsS&XpWXwm)bx)X=7XA`9W(BO4Z^wM^M>xcG6sgGk3X>M6!Q%N1 zFu3?RcXud(Z_+l9;xwCXG8LnKZw@}nk|J+Djf9IEE@IbfBf9>A2I+oOfhEs>VBV>v zlGvvKSeAGeKHoQ{cf)w}+j0{w2JV8Ch)*coQ;JIB6nJ@I8u6THN=+tj!!s!;I2Son z6w;Lmrh)#9Z^3tF?TAgx3I_#Jk>L&YlLVlB#SW+RHo}C{30VB)o3rc{1F~kRDOv40 z5(<13@zdh{Fw@i%%pIy=$LUk3DHA|PiWLYqe;oFm*oSpyRhX~&hna5n61A&pxn~Iz z=zpVLbIpgxVVmh7zL9o9gJv0GJXwZV>t4ZQ>FdzK>@W1|ynx-mkCk7(9|BM8m7sFX zHYQtZ4n#P5LFDRA+*|iuG)d#Z1k=1lbk#jYa=PI;>Zj{*Z$(CA+z?xOf2Sr>v$7iI z*sUVThD+&+HM^M7^YKjI`bG4%=Rasq`;0Z0)#-+gO?Yo<84O=g$P6!fB;u9Usp;yWu~>B+Zn=KBC99~z6Ze_zG}ipnG~cn?#ZJqzNO>M)A$+68@D zHW(m(mU;H-4Rf)UvX5u1{U#&+jRa?^P(b>%B z6bEn}lEEmJf50(KYv57VXH-%Gbid%t+4`Sl0>>C&>Naa^U3LjW5-PZH^Cr@scxAeL zu{Hb|H=b;)*)0gH*QFooiz{P`-}ZygRPp55@Ecoie_Fbn#hji#OofY%yVfcls`I6h=H9E>Q&b2*=2*{rXE zyX~qH-_vPWA>4^eE%oTTTz}NrDNos{c643oXl`+=2^qKAm?*RNFfRT!Mr|6wJz?4U1Nzi8aXGezoQA@X8}Oo1mA3k2p`u3%=%wh9`8~SSd7CcH zc)AA;MP)%?bQj}tN5quQa}k_9c@NFD_2ahX=JfcvYcTy!6RxeO=h7$i!R@!NAby+y zs%?J?C80WGm5DhO>HNXzqcv&a4kh~O-BJu2^AEJLGN5kbUBeR#2g=U8F;Ca%I`S33uL)3=Q6{e|bL6>>-SJaavy4Td;YGQGKF%*M`txbBjz;MYTzJUrF|{F!sO_NOM5 zX|6*f(KF`u(*$%b(8+Sz5#HsJ{;Ox~bVdz)|ZuHVbe0FmwMAtjgHbW13 z{IL$$udyX@KSq(-raZ`T*C9%tvuJDRC)hUd2&#_C3EC^hQ9sX6xS2KwzU=&rd(B)? ze(QZs5HN~{ZsdS%0Vq3XPL>$}S%3ExbRAfSuM}>h<=_)+*}4mt`Xq?9T?*y0J^sMP zXI{{0eH&xky~tp!4NU8AZX!v#DHB-^O5B&so)ad&Z$oZI$BhI*z zNnXvUW+Y46r1yfLP6KxwtwO`~2QX{YC@jBP2amYpaFXAS$$yVv*xpi3>FZ&*=6DN* zQ{zySSPd3NaWH?~aW1M1F-Gwiy318?V`qKmHisQVgW8F(a(*egTKj{}kl6B!jcf5} zP!XDSB*VhHr(k2g0$rk10EPlt5GkZc`tuw(-9L}1FucZ*qd@3LaoDaq)7u#*9YiTeFgu`S0^FIrKoJ^bUIj*gl}gg zV4;;O{yQ{*)HH2H*^QB4rh5!$ZMXotSIbb*%#3?X9se6g}NZ{?jG)$@ltFa z)qrig-oTYjkxcJ_;nZ5biHl&2(9nxAi-ii?e0z-WRLF-oTTu zM?~}2hmh$$cH~(4P5dz4g{t`^aXTh^leBNk@Z#_TSTnv5rB}V?*cGy5rd=3hL^Zh~C9YCr!&2ag2SQFkGkHy|AOC)N_R$xAxlGwXp z)Y`in4G+hNMt=>1FU$Agmr4rik~!wWd>2S_Uj%nV7TnN8Z+h%~Gkz@@Pd;NYsby0(E7h6!_jqiA5t}OZ4hVbCb8p4mVBUj(-$N!eQP?>L?pdB&@ec^55($$1y> z^@2d?Z5;wPt~|mxoeVgAUoywX+2G&8f#BaDe9Jw?sb(3`v*9-6K;?UIufD|P#w*cp zs=17lkq8>+EWsB}aiX&ai^2B3CuiyCjT^?H=yTv^YF@aA{&N0_KKEvm=J$To^kEnJ z7t8=E^$8ZdaHanQr(v|FA-=E3hQ+%KNQ9FLv3sZj+9um^*6~-&)mBSFD|0X+OcKsz zC4$=9C3sR}}i z02PN7;L!bMXpwXcY#+;^op=}>YFo%HD>H)CT6Oq*iBK@nd=`B(-k5%SX+y$(sSqVw zRkE;YCk$nBInRyK)bG~@=B?dDC_Ws3k9}0>+QLa>+uJCN%K0wfH9c`4O`bj;%*OjJ zhr#UX0Wg^8L^7HWF&{5hN&W+}%(u!+EKnLjbu1Sldyyp?cH`;6Xc4}9Yfq$iMZvVE zk6=f&4v}lG!kS}A_~!RkOn)vz$xVH^TZH5!q+u0CnohYcv zbm9sOzA#l8hQ#69Gm-vRM;e><1m`w?MTGQ!b{g#wyxB6#O_ z4j!2{6esH0F*C?eP;&Gpw*-}t-aP_+9vs3w>T@Wo+KzUmiJ*DOiuelV12aJx){c$E z@1D6XN9S~7eR}|2=?^6HW{)L~@3hf!!BU*_O$$Zwr(vf-09p9=4VZY3z*fyyXgWBI zGw$018`G`nEhSmzq_zV+e{2b+7yJ@zYbb{^+dA>BW;{-@2!QTt7G~8Q6zz}eWa97q zMCCbA5IIkePTlQH)v}~Xu30u-Tj5Fi`}~+J87cDcKLMTl_yipDe-C4>W#PN&uh2MP z0!_7urq@GlA@%r7lK9w4v?ntu^KQYm^1EPr@F%w}%>lme8i7ZP zwsV`altpBH#H1`FRY~v!$PS~mkIeLJ(EPI zweB(7O0pZ_$-{i?-YQKFhPI13rYsfJ6=&h_&lm9R zPyr1{aHd~n{sXVy1I&YZ8#>g}g2v3q1<|eyCZ|E-m7Kkvl=z2G9lV9VLxO1DA}cz7 z`#2=S=8*@cU6SDG=iH8uDd4cER-`z}f$X=ggT@j-nYwR?esP@rxKIgxH;1t}9Rm+E zD!H!3dEoVHId_+lcrE7dV=O9GLj8vtFfM$7>ZjY`$>mYx_uEX)J|Tp9DoT_1%ZtgI z*S<8-OBt7zPa-chtcjMv03Oa7O3S_Hkwt3J#I)O)=K77KGx~aQ>CIjYc-;Yl8=nM| zg8qW0Bz*tx=Xw~^1$1~omgw@F5!mNph0mAiLxX`YeMINcO#`;1YZ{R4W=rXw^-gsE zse25j9%c+~xRF-DaH4WbKyP3Fji1}kK*O5dQ18?#7#x%*ZJr_UEXb5vw`-9P^9dTX zx#ODfATn%y7nb`c!}!G^u)K9Q-hPz^A>}tEv#rUfkXQ%T?Ka@!T6^w6Vle(QjK{b& zd*J?ziCCHP0bWH+AZ=@|z}4+3SS)ejD6b76?Y*{iZo(*h_eOzqEG%QfKWO2u73O$Y ze>>hByO7acE#REje`ezDm?Gcg2}(|q`9NtjmJd48MPJ?N^AUbTcbG1X3Z5uQkkFzx zif?gyJBE^p=LCewDF)H}kpctbT(7iDPveL`wz2m zV=lAisWrK=uYqg1$ubeAo?!FnEavkk9%qGiFdy4&=-W;=+MaX~`Y+5O{S%Vl*Xy6) z*&dIlZ)?zHx|2x77c=s%;0XGJ+tQt<=g?U6q(3V^!froJ@@&>nIcKMo*+F(IHD2;#r7bemRira?6Bu z=N3}VJ*wep(i}QV?vNm+QH}OgThZaNu4HT1DR}jA6mbw-ebY$BlxD8NzX3a++ETIfWa5`zUY7jciT*Hfp{H*O$fx@8uqMoyoSr?E z;A%B`w(2|X+T95&l(op541Fp;@Dzi?)M#i;4eHypmq&_O=EL^gwD#i=s`Q~7Oy=){ z&~z*MfSg6cS6|@QXBB*Uuoa??7}HrdGeCumghL7QuyyDN?vm06E+F70{=1%yOVTfJ zHd_wBVzLZIZw>=zdqe7VQJyIJ?*)%CDRS)gGTip?9k;b(4>&#U1=C+;%*c|HXmM$Q zOPsMUoqt)L+8qsM4ypeKA*1JFUV#ZcGav-_pGQHuum{&zyuix+*P!K+6v1LJjT(L_a_@;4{boo8o}Lr(WV^ z18D43iRWhKU3mOKlP-$nack=_oFiuoe?LjI693mIuNpu+4yMEN77aY8e3Q$3e_6D6 z)g>lUW)79glg#%Tzrq8RCdU5HWNMbG4i7RE$>ixZTo*HybYy%1Th(b)+o=faMlT@W z&pVPE1*(#nrY==jrAD_5F!XTdXc9YJ7ba%e!|ZuLE8QJHFwT+r9^``mHpzorbSwAd z-5fLv9YX{otwBQc!6bxcA9He{6YN}e4s3qjhUPO~jMM>dR7^W2u$+DzSDr8;;%GTC#Je6XKHH+) zkR?!bU7xJSJ)CgzBXnY4a96ZM816EZY}%zlHoFs2(6WH4{^c;{+7?U>x&>LC##nan z6z(m*ivcCa(Ps5O_#k70-8<~5Y1jaEMNcCyVwD;2tz=fHE0OCfH0W^Gb6n9Y4$5qv0OY{nQoE_CSILNq&HlKYDnIUubU72)Gu(;U7}H^w=u%4n}}S2HQBPi6#rT) z(aaa}#L`T1zs^;m$J-bzw9mrOkaDhG`7K(#(j}q*a}smpH^=QbhO3Rru=bGy9DI5Z z%Cvo`V`&69*p$MC10Uhy1S5J>u}iSS!do)C7()&pF~@<`-ZXHDKDEWMTo^wCvHqfAjmXOj!CEAj` zrHIIAn4zf5d(QowtEI9M5=tozC8N+({qFBS_&o0Ye9n2F*Ll5O&jFqc-Z#4icMN^v zPsr>7F38vVs#iI`)!&nJ?H7k@9#}$!w|~Gw;RKS5cayQT1=b;}Zh=7U5=`PUK*aq2 z;6%55U}jG)XM5>ZqG|q>GuN+)f7R}EbyQj{4ty#O=lcbNpWeAxXG{zKIrt3(<$nWz z@AAl!6f3< zt~+7mp>Uw7`X3(WE+;OBWkLSh=U`sgD;&2jo+GJy6~yn>fO_dxxJS7Sm+VuAw+a=Y zSHChDjW*>Q1YQQc`EI!7g(j)BPvpE7o&VPMD#F)61Wy@tzy_C!s?TjUhre&V2E!Yq z!CCGG?EE+u-;6rSUtNC&Xm8g6ZI=sx#P~$w^+FsUcxYNJ)7gbB0;+HWYX_L-U9gzH z76#82C(Cb$X66cQs1d9J>mMnSv-Qb%WFQd;d2exTbs3OKcIMoku^5Olx!`qa5e|7# z!0(a#2u|-&fg|7V;3-)loS2iMd*AeDKrL8|_>AZC|9M>EAD{k*&jYfsdFoH=v8^Jb z&37?45_ku%ciM?hE;qwZY{T%|&f8$pn+N=gAaU~bs~9=+ej;4)RWy$)4Rd6JWTA|T zF$_7-k1=14uXQXI1RPpH!W#GUE8~L7jbI8f7HDJQ&evq112oy6zu}11>)qj$vaB5y1 zPnn^ zV{S8qcO3u6@fh>Pfv?r!KPDV6{FVj|FY^|Sl8NAb)KO4#w+yKMx17`-m;esu=vz02 z>*ImvlGtVDHO&2zg-h=zWA(7vfT!^h`+k`YMhd$?^|CsADC;_ws{OFgr#2K<8gJ#R zO?-`IO-!&J;}7VYG;6nCUEsEDFHndRormWy#UZCkfxdD!p1Nv1Fnu@)AN*^|(SN)U z&*9zXA8CDUUFkFtZWql2vC+DC40c+7o2ARQuayKh1@o%&I?jWWzmmYHw>OTgmLL=Q z(||`!F(|#y5C!Huc1ryR>_gT=6Epf@)O zEX@V@tI-v(v{VD^Xgw#=6)s@87pt(_$`+1${V33*MvWP3Ia|{`0znjk_;#LSCs7wTLNjV&ge1`6!m(UNIL$b{F!8rEgR@ z+&=?$M!A8|O;7O6skX%M>eK2IoJxH5L^UU~T#J+5Jq}tXRC5+zzXBG#8^YVROTqsn z&+{vUJNeJWJcv#1Jy4Q=2H0<@;CJ3MCkl_Iz>|;Tfc!xV958 zEgS=)t$2%b`$`sP@KggZ9+QM2HD_`1`C$Bl(E%%ak5+w~KMmJp9>={>w}9(qw#3%0nx(oHJCH#9qmAwpHTXqpV z|9c;=d3G0^_FD|s2dv^~X?kONbB8FW(unH@Ch*t1;c!YzRpGkDiX=||1pcV$gD(t4 zaAqu#!${af&Z>t?0w-D=lZ{+9McetZ)8SsF7)eR@dbNLPd1ED+QGQ~BGKf#Qhi(4*#pirlgj-Zz;R6%x;r;A= zVC20e?D#j0d|y$C`6}{Id!!xT_EUo%W;4ly##;Qlcrpygnn~!}5F|FRVqs4&%} zVh%YL-3~S%n`>RBPy*IhQ#_`!32(?(f$6(WV=0R`@ZW9|^7GkWd~1*BSy%J|y`9To zZI2|7cG(Sf%-bXKXy_4%K1nYD2i=t{rrhLS!5J3I8QaAv_>WqIUpz z_U~e%c;^&H8r`TquxJffq&L7fdNq&J|78RWKVnE?HlQoI9u(%-w$Jk!PPZybh)3a6Kp6uZ2_AN>{iz^8~_9o-{{F^v^+EcuK zx@C2h>mD%s)_43eoIv;zTm1%5Nf`Z$5}WSi6985tAVQVmdCuOUTkuO-O7z zKyYRhC>Y-kHIB|82>l3+fYi_uR3t|A3ZW| zy8xzr*$GvYMVjh6E7JbE1FL#@0E_rKev6kCWUoeX-kz3&+Xb89MR`a%XFkMxrBbls zm?8|gsz6pAHiEt`QCMSSEjgOF0-oy+<#ccE!f<{(mO2`5T@k7a9tTe1s1B}%tM*7( zy+E`1(G3RptGNIVPx%HqzP+dpy6FWn?x%51J4ph0p2#C1B?Zj8EBVbrYv|D$4t8jK zBdk&oe?0`zrzHsEXRi8-DYZLTa!>+hMZqpctP1zySEqk2Z03NWk}@9`GW{gQxTdTUkrC@Ns?7(oDh6?BL=tM z2mx0WMeB3O4D2-s$i*Lr`Te8}&^~7n@K&45cag)E`qOZ%PBcz$nnwl|R)URZ>VTct zJI>o*7VyX#C*ts;7vB}-j!PF^1}}zJgBtHh&a&qoAW5JHGp;1^M-(OS4Vf3%bW%2s zneD=t?^lKqpH^_R1E-OBxm$7GNGX1{IT@c4owN0EI1HUi*1rr)=tGA@j5qYc@67`X2VE=~)myIcq9WaQK zr?`lWxvVF<=d=P&lmgT$7vj^`oJ4o(VkDX`4u8$d2Qn58uro)F2;;}F@Ru2X-o$l8 zPiZH-aQQCo(@X@qf4Z>uS1!Ey`X+u6E)Vm*Pv^X?V_@RgYNF(0Kwd0)jNUl+_L%jC(sZhNBIzXHlf+y&q6)q)2n4nglOM{@IZ9DlxBIry6* z0lz4n!nIT9f+$B(He{A5RJphZm#;d27nL2ZK6>O3blb6#2=1N(w(jbj^UDQzSE@X` z_kc%`xijC$umZ>1h(oDoWBi~!#r$c;zxlpF2H0n2Ch$~D$41|kVdVC;L=5 zuO>_v{Nl$giUqF6&Vx!S+8dN);Fg6u@Z?-)Qgr1TCn7O`bg9pRvdZOn;>LG)`@3Oa z+CGk@*KLJJs~=mlZ50mB1HsqshoEB9FK}|hKU^xg0JfZ8NtD&*a5l_611yTI;liD@ z{J6+@FwZcFUo~+J^qj>d;l=47#F2rQcP=HusaE*N##Hcot{+HBYsTDFh9F1sDNw(= zy&CX6K_%xP9@r@c6{!85p-o33hYm;?Yg2 zL_0wh-~XCg#fZL#1qn(p*ioccb+z-iI(h)l*`4^}vTktK+6*o-6yVbbrx9O=BAor^ z06E)d4Tm0hVu|rz90}$vVZE@QT(SJaerRF*(_ScQzcv zG3G~LM)E^k%$-ZT%r}ABPs`xI4Mi9ybcX{RO`@be9h~hpBh1Qh&@g04R*wh3aIIbp zn#~BbRe`_Ve{p=2epdqpSEA&m0NsBp0E;Ju{9ATT#AQ(A-82m6mxgZU$Hvcs&zHC3 zFZ}DA?6dM@|I0VHEovRuKhpqS{1plX0ZyV0fc@~go)ek>WIkEdISYEW%qF9JO*kV& zkvzRpQuR(Lp8p|P6>3*U!$+}UMCIxUIN$1wNch0u*p70bn;XY@{vZlB^BZxIx+i?O zW+@pJ^$|D@#K2GbN6EbcAGmt|apKYxP29i8!(=mCqUF8>-g>wSK6wyGy3TsS=Br<@ zwDn1nr!|ZGd18pyEm;POqgMje^kGo>;yjr6@*V$SX(WspUQc|oHpA?r3&^JbBB7(% zIR0_^5L~t76mj0`K-~P#;!BE(Wb#D@Dk&c1UjoMD-+Xm=Bj69;NXH$@?`_2gx}>50 zmJ|3+>2w&Vv6KY3I>0&`Pmo)BnqzQUf`q4@070!c@qhN~2tJ#~ml`bsXF)x;;dM#^}KL!B#lo;x>{wXDx7P_Jo zMHDWcf|*PFNRW9H%vlglzJ79ngYVW5J9!P!`6~p<+_(nxuGRsI1=C5TeK<5qIYy40 z3WnWT?xe)_C=BaALOhmvLSd9X`LQ7ZUNv`zw=L4}{9#FgU80D~x=czunE`zs>X{C9=`*=X4&4x84Gia=(aj&SCI$au~TP za~S6LZXljtW8veqJQCa)2opgVNlOVK`)q>YWY&lHF`c01Uog4za4tL&d6YZ}-42a= z5}@QrFjhpe)maU2{#y!rg~pVq`n+VSQm#%^Xsj zb&%{HWr)#?ci_+$YZ#v(MXFvnlJBsaO&QgJ5vV zL9#d^3d-d9khev_(5_-LX|ml9dqi`3r!0UUy9GEt_&bh%co=Srm`h$bdcy-#w1}Z` z6bxS#N@lklfwI%1i40!}bUZO74_AxBvW#9)o-+gvNP3Z1S!VFC^hRub0>EhPtH64= z4NvJE!&;k+NY5J%u&CSv^g8vRXGRKGpVh$$GI+xIp#M?SdL4}W7QN*6CN6-=LBBZr zI__~UE>R|rhL->ZSsvcwjEHIWd}4Q196p_*0r~3fc)Pf{^H}RG z_G_38Zmde+lw~Z!0pA_~B~gJ#{#-SzfDFLPh|3_Y+)UIlQ39;K?IJ~-3B+FdFXRbA0!TO_!~^~Im;H;frlz<^$MHMqJTU*?GrWA&bAFx6OdwTl25P<4V5{kD zQh!{UXeleh&kjSNxxEuC4PVY*qBjWgJ_m#CoSR_j`drboe&O5iy@D^709ZG;5aMam z$Z;io^1fplkxTuBAE#FE`wzSZ6BIUq3hKficfSiX&mAY@ezGuGoy9akl z!p7m*(D%!0Ja_SBJhw;-^!Lms(*ARyPWvJL{@qPXzFxYq= zWU@M>KwWgl+i@ElP5KL@EbrpSL#l95Y#pb~W*i($>&3O|Y5eKkSzyUT9gcIa2K@2Y z7RH`6C0QPGiL+B4c=4batA=KAp5GSrKrI(13G#En=CVAXdtw-yWMo?3vA$c~eEKX< zZX5#*BKALWa?YcTG2q)o2{L~9G2hqc7vEXtDi+i(AhRYHgTLK6(0zkA zsF9N(^IC4<9dMlEl#q+>{=11!j=sd)#UbFr>L(ZPv5{kTKLiiXPr+BFS3>9YABQ zDXjcbgtu`wfji225bltNjt4~=koF7f3!-k2>>oYABu$g3`<>+{|2Bs*!NCxFEhEt9 zdA0q`UVg1M7yNu*UZoVlf&Owc_)AA9KTY~_wVgt}mHz@=_%L-QIU>q%d)}MRxzn_Q z9P8Lkc8t!1A*ORc*ZgM~i#)VWKl|~jjgz7Hl#OIf!d%jPG!1m^CU`epN9OKbMz+*R zLp8NV(OQ#?8^b)|Em20YDj<{pD5Bhrx}A&%aeh|a!+8+QH4cY`mjQvB6&WrmgK!z4hJf< ziTI@|euTUysam@T9=`Cyy1MfcwjA3<-UcTD%io23sbee1+|#N+H>e!X*sG1F#%(2A zGDShu73ILdyA`Y+lLhi74zN0_7aInD#rh#9z|a>9sHh$ZeiY>4?>ZCVg#s7&YmX1f zlRwA#9(9PX=w5?2bsNCysG#bM(tdDYzY31p@EC7VPQur=r-H4u^Wn3WYVg&q#`@}C zEl#2;;FKikfo;E4pzMJNTpJt%a(&K;GQ?ki_=;tqbfy-0YinKS{6w^GtU@ zuZNKAuuQ<8L|ro(cr8>}y9}6F3HgrCv|;*i9@ed155{9o<4>MO(Db!GOyVdJI|Fl) z(3B0%H!B0_>cd#ryqYt06CXz^sFQ#KQ~sRH;g*?m;1wtV?T=gn z+Ibi8mb@e{e^=DG7hM2P7{}#P2WKlJZhR@?Aa^NQAZl?cZ9QO{N=Rm%)GF-mQEf7?2|` z(^kN|xH&i|;~UWV8-_Fa8_CP!#qhV=LLg^lf*Tb?xj82@oH;%fzqt1loHFyT*0YlW zUMH+bXzvhesJ+T6sM*nXKGM7l-*a?i4i^Q=htk0<1#JD=Euvm(9TBstkbV%C;ch)@ zPRCY6px2rftnuGZ%(wX}NO}EvI&Z0&;7)ZhJ$vmYGYb8Sr;*b42$ zuhX<@FJj|ui0X&iDgCS?;`Vs3&aY;mDLciG_1aF>uy2Uo*%Hfs4xYo@JfKLop)geX ztqAAb-Hi;*-!L0Pov2t}0h*GSgx1^5a{7&n}aid+ZSD>fh5XI^U5 zbka(?=&wKfVZWR(TP}^x{aMK>^FPu2j0E;?lMO11HbZazZKf6Xu2HoWj>uG|a2qiAW@VC7dvCr^(>jj>~p!q zo$1P>txFZyd%dn`(KUjSnPJxEr6HrYzD(4Ao53h?-?DLfci0*E3iQCApWJmrCETUo z`ccK{k4)%DH?urv3rZYnWsDZ5(jUr}Xw0mS?JI6)IC4>}ZR0I8_n{7&zsHQZ_4OTd z+&zF5OS?v258Wo{-zt=i11ZyMM!ol5XXgnb(eWMY=r-Ft`e%C&bKp%s%P|aNOzev3 z;ZFk$r!7>(!Ik0~1k9ib4PJsm|KI4+pL|%pL6zcLWz;wPHq&nL7lq`VLvdevNbbRp zxG**h?dM0a1A@Q&-ZPibM1x*7Zlxb>+I5cZtP5q!WOLE85Id%-jb&bED$u-}VazMj zQfAZ7CR%BFf=V(g*&Q`*+M*{jA2E>-3Sq9^`t2p(>U| zY;7J32h<;-x^wc(iroN3NB>6$S{qQrKpnlJK1_{$Q&>AEP4sMZ4ddp1i7C1df~_udPsMOotqm;@5@EfZQ6-w)89P7 zS&2XF#?edY=UP1)^IE%BrKJzP=G8)fADbw66;wVPSgpe>jg_mNfq zl0fG=<}qcBcHCB(bL_C}GA3w|ka^*Egt||XgV0_=5LEnb^scWPZ8#?nklY&i{?CS&UwOy6$j=~1^a%eHi z@SlLvQ&zLr<@VCq+$z*|=?)z~B}-2j#Ix)E6wx`hjf}VYA;#nMPx?sBLlE~#oj${1 zNa^o!w7RkpHMvg_n7v3rGum#jewsDNwq2E%ucnEzTicjJ8hUh^h_}E$RZXS#ZJ>dU zc5KL{HOz-Zf9{O#_b6v;6;rjdg0g%!s~PvE_a1syUqyt5uird&`8+FT#?La=-FXR}=Nm0J(6WgA z``Upy=6aUwt4gP)cg9(R<(JqK5en#KgEc$!=`j3cnnoL0F?P?%yUgGoN2(=EK}UB@ z;4y*!Az*I9ZqN;+9${zb)f>CeY-o$5`-+jf*(4f9i;;v?JT+V6L2o2%V#@~4qkoN) z$gbA z#J-by#(aMH3>7M8pk>I2w`eC~?aCB{lEWz|`$aq(=EX&|1@WlU!Vs+}U(ak9ccOjB zKv+F7o-KALLTxvtg+of$S%<`>bfWAibNUEnCWj^DABs18Y&^1jS4an(Lv`tDlpL( zd`VG53mVTbcNHfHbN%?-ixUU9N5!<*e@SI@-6Tn#Zufew2B$}`H#-~s87*TncScc* zhcd!phYR$W+eJoVpDDd?Z5_R8EXjNN-vT<W+3JKdCac) zxs|5 zCOzT$xz(W0Z_3fzzN0ApL=XMD(he#ZeUPeuKE{6Py8PM*SrsKSJ~!M&4ve}Rl}7g zYL~Ie&D!YZiy1UJ{}}z!d6{jRxIp;2{VwyT#+i70i$k{uv(VwWqILHrLm$d%Ao6oD zbvRLpVrtvSfPFA}P%G+6_j6_z=(uw~MgL>tyoD@x{vxi|%mie~cV$;^enozpl%tPq zIa_qt2~|$CKqfb$Y3ky|bYp8DyDQn1NqW5*Rrlx%n^hyw)-OB}<23+vy?VtojTQ-B zTsntjAAc99CtRRE_fcf@;VtxBYE1ha${1nnGIJwVo`#r7m6{QK<)ECF{$;@G{yZTDZd?^{qV6g3 zmbBiW@tR9%piVn0Jo=RWUgCzF$Idg^D&lBm{TMTA@+a=+DHDVxK0mqnP8O)>%X#E@ zqLl`k+M{11B^2*zrH>r)(5J!OjNMN);hD2(^r2)r*DhCrJ8!xlTd;K&x-v9^Mkn*= zt)DHlIaHqOFips8|22i$KWeAOVT0)0&im~6LkV`dMk;N{R-xlc=InK?T83Zn5f5oxJgE;(ll;bBmGu`f zHsn!Ap*%g>*hud)UV;Yq_uSWhQ`pZxHDT2-UbjT6`<)h@_n>38<3u%5XH43P2Z zJ#59hO(-)q31v=ZSRc*1lr~?dPlHELMVB+{<9e3W-%!IfNK~P`FIJReS4)x}?h=fM zePx04eYz;ggk?6(6YR)OWm-JvA>?F){PMS=Ut9MveQs}aBtORG%LYZmvI?=IjQoJ6q`zXq#1Kll8Vg{uOIkUua zQQdL{_L@o|Gk!0U2G9J?4C8!3{HtY)s(WV2aO z67)jQA5>C%nOcfVvD<5tk>|lEdf4C*Yc66%UEaTxNvJM{0y%f|H9v$oe{3_IJ++XL z)Kf;s_vKTEh|@^Aqlk{TD9|oD2-eli~^T8(B)STGuEExn7dWWkj_6d`uTG@ za`p*kQhbc)+Yb|Iqxd)WmTNjIo0^9XENnwkwT8^_dKGl0-;*ZRPDDF!Gj-jwm1?L> z5stL{W4Qye!jI5Ha3Zyf9#PytU1oQ(=Jy{mdf$|V^%+Vu%ykf5Ubi2;lK3YuwF^f* zR!f+bNwJJLP2jpl*w9*SX~v5cm3W;5EEe!do! z{Nyun!?O^RX+c+9P!bw>defpUaY%F07xshAG=$}r(KYoUjB;u=y0EIBeosqAItw)E zsZ%agp=6kmFusp|o>CWn@SVw)Y}DjN%Cs^U?xv$Nx2GX4ZRPrATQJ46&4?P*U%u266Nbg1MWmnznW*UDuA?<`b_Ot3SiX(5L@QMl~el(5-n5yuerA9#i z!&#I>tw)LJedy^5$8G~&LR&T_p4+SI7|B6=6qi2T+!p;Neyo2DZ2!?&rT4{=G{>uy6# zPmVveBUU2ToEr0A+zoC;ycsh6qE2tu*9rpH-6a_&?*$WQYcUIV6miv8ThN;aMb^=| zT0A?y5hitI6VeLA>~;w$-qJ@7$o3W&b)9Np?v)nM$kClFw$nk!k|BDK)Qa*oVwkxj z7tkq>0hEF-vTrg6khSe}I-}p5=B!UZ*NUdXrkzm&=TJ!zd#eT>t*T)rx5cvI=9ISl zw&m8}-5^?jVwn056s;~SVK$!cWPxk}!p^B^r}_t^A-{!@tK7n-pPz@m-CxN%^xtK- z`8v>#?UT^g@4?8orjYLXx&qa=+^6?iS=8)vgq2!6N&^c`=(@bKRPoL`der?I${G=I z;QXA>+gxTmxUI_A9 zv;fJ!@nX`HqS@r<=TOToB^v7Zjy>Kd$@^fVje_17u``}#vm3v9(E}mpVcH!(3QrqS zvz{2*Fl0opPRL;o2fbvq)Uf*S!jRTB=pBSo!M%+ zncI1=lhKfvfxccZq4s65NZ8zllD!Mq8k5_TqqabtBGm_N@r)U_iC4clErq2Xby{Dwjlsxq1Baq=dKVk6u$N&uZ*?na%9 zr!jM0M^Nj&K@@Q7G&Ka))I21KZM0uXhel=4EU9qxMAwqme%(sdeK$}Gi%`Lge+FE= zj`hq{nK`U!)emmg22FJ2Z2zPIhLGJdgJ$f=1Szql?yy_UbumlxwjQ zdHoTiM^4A0@FSII+o^Dr{H=&PV@?_R5xNo0yrzZp-4+O&CM)pdCO9HNfhP^G|Hv+m zlBJ{b2U*QGKLoCWEvUqAoj{Ivll`0d7QLrC>E+*9f>qsj;Sqz!=<|+~e`E{g7W|=_qi#rW%7LYOPcWs)E76NMaausKsGPfuP*|DDY_}U{ zKD$k#tNfFx0WXohRjX&qe0=EV)&ypH#tk&$qQX4bzn!*a??Gl3=JfTeWH#QYiM}(r z!bF%HMt4O_9aHNh`a>8>qu$?VqW-xfnGLV#-uUzE-ct_rnT8qNpG)%rRecU9%|rEq#ailq7_A2OsBy#+Lm|?1&O$`_cRSq*3hKY7hAxU-!u>xe}$epeVr@{A1+&t<5IIn6)$o^hx&LXY$P zP;cA~=FZ*(+M*%ia`Pf-?yIL{aQQAo7F|Ja1r3bccOzz~>N=Y;V-Af^QR2NF&tTRL z%dqW-Qt6U4?-_yiBIa4FXx8J;<8;PO(!#rAD z-GJ6Mb+d`R=jos4Ic&zANr;ytf#md`fbs?dbawAcWdErI9oauX^T%chUhnK+bxj`9 zIzMMRb!8M|sM$%gGiNfzoz`fY;Y_r_;UoLN@dmS@P7Y0UdPQx=O{h}eeflq%&+K3>JRD=U z{c2!X1sz_%he+z8IY*H2qMWW?_8hsrPNa|a7_*T^M`@O`HX6Hfi*^NU5X4K335J@s z@EnG?bkhYNR2HC!;@XrbkoKY3kJI4^*L3F0T#Dz%9WgBg645G*T z&vVC9&oIe4yJ@3{mAhb{NL#Pfr$*cNu&u{K*pLm=(948ofs~XNsz?#1KaONlJ-6Lx zk!}G>*foW>SMLsU<uCSsGkL5-%m0RJmyGFZnRa-ujNh)&?-?n_r+O{W(bL?g+cqrIH;S zo`Rlwb)X}grz6b^5>z<5oT_d9hx|@&qZau>He!V?uQmQMy4O&EGJMmia^oes!2Ky3 z?UT)@eo;bqlXTGI;WgYV^W0J7YL<<&Zf1(kDkASFJ+|<&CDUpAfVsJ|ol&~2O{))v zGX1R>$uhu$F|G)5hpQy4?NHdMJW zLSV2k6*<3N!!uYLOXo{Rv#kpzvMSFeA~};Ll$F9kwN7EQ=xHCb@aqqDx=$9J^Cp@C zk^XH9ZqrPsLh2}$&mO(6$8KsA6W+7FfSmVgP|rpgGD)|Ky~ zjtM@rqxLq_-`~QjoyZlq=4kOWOhcH2X-cRqL;@v>N72z^(JT(OqIGO8lN0!b8jUQV zZO@*fi?!PX(d<@gDTnEJXbfrPnTaf9FR2xXxNFtfETpmtpt! z82VH6u9k0iW#nr=axazEGWxz83QxSCY)=}pg(-C7B`uz$mM(JW{!FjO3<&b-bkP09 zU4kdy5>S-KL3r9sRmi;pXzgKrG!V+tmD|S|g@Fz#ozp-S3|pv-Xde>4na+LQq|3;w z{$TGVv@uhcNzj^0$*98n4qeB8&;A>27VH?4<~?`!rp_}q)5AYBm_Ht=%-ECfg2zpX zsPViBI`VrB+ISG5b@iG^{Gf;rIV%xWi@!u?%>ScxOC#tbl@=yC?F};`Vi#W2t)%-( zIp}^sBlov45A}bYfsQ9QAW~?EmiMHiqn4Y{4AtjMVc$wNynHV2W3>fpP5gl7-S$Rj z8d9m$`W9CIKtB_G%^e*{t>Y#cO=4FZm8Xkz7BIPP7ipjNS>zldBm8)BFCwYVJo#i7 zR5bawVCrx^^{h9fSN5x;rSaoz=#F^Q-?|#jX}!v{UAv29YmfLVn&P3*S z$Q7h#GMRockrA9=ezMNZg-k+KF{+DwL>=FBQF!qTDtY9OI=A&v&z$E}cV-)UeJh%+ zxFsfxqcv##wcE@#!C4lqxyF7U+lIW?CD7MPr&0TT`OJ|nQ}li?o!*$GO5b(Kv5F(# z=(lakypq)ynKe3Lg2yg$!j$iE0@f@T)oxM{>USH^(w#F=(vyp+;aN$|ib&O0agO zRm{hxczUD#DLeS_6T7FmhV@;d%rrqYVU376NSdS(>$QtLX&puXXmglDli#q12h`Xl z9|qX0ixnv7wj8>Y)QdPcnF(d24;jb!+3Y7QfP?J+Z7T~2tOoz5H1 zaYp_*7AWMRhEQ>6KG*4$Hg$Y3K!N#1G#f-w0|}A-6gmramOG$+&zH=a5_Pn=aszVY z{bll-ZbHB1T)}ObN}7|ZNLE1C85FV11D-`TN> zmoG4xb|QWHdponQaT2ewemY&aB$a-*KhGu_?V*~6(dhA`a2ik;i3ax!AlqVTbk*i0 zv5?e8iku+s_P|uZ(Ou8kQOk|AX6*xdy5B(f?n^rTWqyo$KD$D{&Q_zp9?YQoUwwz_ zovA3U^g82}r9w?@m(#BLV!BM?C!5lwNdMy%(qkdwXx!~AQ#t=3>ed%WJnl!7zeOBH zMor}TWzIvE@2gqaXDN)Q!8Bo9My!aP8H7IcwXiANI|4;}W%`$=hGxYDGbfinq5eq> zn$s`O+$fZ$vp!1EHzHR16SFWu>FF5s-E=K|wq+ZtzE_RB#?GUY1;%vb^(S=JwG6E^ zZD&@0kU@X<7t^C_j&Ut5(%Fv=8tm)|%A)h$Becc!8GES58BMBxLpyK8pxocVC}v9u zdoM?mW(GcGy_?O^`!yM?+7x?~xmk-&U#iYnue4;IwcVuUTf^D>DSH`?;imyMQiV2q>`gdQ2~D1L7`ef8c1{RqmYF(UT6=BH>dc1(v>MdmUiLuaX9 zz!l`TqL?i(A7`9*>_SDFLS)skRd5gOqN~gIA=j=xS}UfDj?cc!U9nS{+AfV`G{kJ^ z_;5aQR`+2|g{s^xx68;djj)E+^QrCzGy3kpQbYHeUa zA8zz#xS!404iJxy@=sHVElqUtw0Usn*GObDpkmGuJn@(IK5d7~6UdNyVF>_s2)+4EYJPi8ml%db8Nv zu~)RDN1XC6x3d!;OhIKE?NQa(D|8?&gW3j8McX&*Wz}Bz(Jiiu!ehQ>=(Dhv{?Ryv)$E-$JA@_Y7NWbeEa;WI8f29i_93v(OSL zU7~#B3~Kg0jq+qC(F-eg(F5IA(eJ5*oFwL!?6W6N>fsE6G9_0NZ4n4`Prdvx2 znSpdE)MV#P^%rWh!(NZrpH&Hhf3j+*=$DKSl8u_nG#}SRd9L%(1OJ26*n5Rg=Z+ms&Qs^* zg;_EyJSv#yr!S%@$;+rk!y>xq*e@jWM~xYO@`DLbA7O)dVJM`26Z-8vh3BK`fD(?B zp@-bFjBiE^s?N$lzUH&3_5)kQX%zV?#AN7(NN3cmmx97hsG@%^UlLL)yhPTSu7yq4QEB2pe4dgnRHF=+@^Unch+r~z&&D0Zu zO!4h#XR8}cs?}zrKKx-FN5UEFx8>aUhyglMdW#6}8!)La=c9!K!>q8cj;&gehH^M}=&W)r^s8F;> z`>I8yNZEyuh)766MA~p??sINhtSwq>X;IR?Dc?#_zx(?K<~6T*Ju`F8=kqz|eG*>Y zsHS4QYMAm-0}gpRAnRO*M=f39$vK8pD#XFqelcwBsiEzVK@Bh5fuL(8WMkK3`1^Sg z{%y~qe~+4ifxorr)R?oxYrH;&U7C%4)<5xl>lsYdRKUn@yJ4PsJss#V4(}{mN6wAD zK@)~NqPot{=m{kTcWlam#@w}d*v^RlzSl+P{BEY3-*$oMnL0XL?V|U6?S=32#$fTm zi*Tqp8h$97!m6xO(DzIiZ*3icHI}XP@&OIBZMsZ%EE$YmLyjPQ=7a~sKH{~NKGbD^ z2~IAkrsm!r^!N8h`sU3sI&$e4;+Pyt-aZ>i;}*{%v3su)C4oO*ENu`!*faAMd5HTa73N zL`$YzYp1431;q5uDtMid34{7fq3201-C3Z9`5H~^{JCR!uS?Egbf}CTeV7Qn!H&aI^cAq<6&{e6=QyRJ-fo)V6MN zLDr;){AHi-*nrdfJwZQX3EnHRCMLgDkPKrMtMq@8MZsq;eZLjK^rPAO8My17j=F=pelzBF`J4;5pKT zzP~$6wC9loT9yyO!3OfU`sD>o33^7|uhhV~z~gZB^B=0#JR2%qr7-`Svh@B>Em6FF z7CD@l3QaajqHnuHplqxax8_O`bU$gO&O(__8mWi&kN@L8AB&gBh`_Wp#}_zFclxB* z4)+d-2S59AaMyH%fR+$gVLKU}J({2{m=|8^FQW_6tg*-`1w&u@gY&a%kfo+16|Q$g z&#PgeEuVtBC*G!CMEwg(NxGFPtnrE? zGY{Iq#)dFn%c>b}uFwYWjtMy7vy#Yk-Vi)kRwU^=?E=|Tmg2tY!>~Z^1U`sCtiPE? z_P;rSO?RB|pAt$wNrpn1?iFEtRx>)6xnNCck}zy`G}In?2l9j6WSVg~rYXOoV=xZW zrf?XMHD1>DsYn~z)v-7t6K@)R!cJHOejg(6W3MX=vM8X_pHGl3-gKMB>{O9njrt{d z)IO1(*W3V=s}GQZo3@yF^$OWlN5H$-15)(z$j`Q7$Qva)6AIzN=V}3H)$a}g*RO() zavKTFSOznMDQLb>1drP{;MgQx-Y{MnsE!fdeVR`T6}?fuV-u`@W`}F`7T_%H2P8#$ z5+WBl;942W*?GrS2>+ZT>lwzPiee|d#|;uyvIc1T^fif4tiayAahPRYL|+)_!$tLB zWWIwjxSTPDk82aqs^f|zsP+Nf>!%F+?61<1Q#DEazvDza_!^nMOenovR!sIy+6$rY z)J0C^W6|;ENeJ7$jRc?F1_>JDalL>i)+P?vIU$K|(1;;9FOAV>4iECVh5P`w`8aOk z0FhIK?D{Jv(pC96_@Fim=gD`$$vY>7H9pO>ATS4&?QQUZ^gm3^%wX$$hoZlu0u5}- zhkZL^>4QZ}$dN<^(Spt&WVLZCJ$w1B%==i}zujEAG3Du~Q@>lrLYcFVh913>*V`#5# z2{~-N8s|Ij0L#Y1xYRzM%ovslmtBW}QR-g!eKY}j)2%=^?3SeXQUM$u`ks!tWPyS0 zD}~lqdH6EoH3{X0z+j(d$UT?=|8}S1rAJR_cj_b@sp2C(zFA3d%h5oZ*Yq4~%${MM zjO%`R&{Mkdzh1H^VT|PKG&$7;>bUiW4V5gz5@X%9TQa zT@eb#_fxsdZgA>+0nr_!MJ2l1b<=~`@cG$D_KKA=ER6o!cri|+)vvt)`lyrmm74HIFWgXf+%Zo~W z&%#+&ZzTUFJA*-yJeOIRgZ*iniFr>D)sU9Z>o+@T+cQ-x>??z+aeZ{fzV%daJ^}*h z0<7$)C*BtMJkxZPw0|BWbv4~iBl|k2+QglbXIFmFxC3_Rbuk-lu6EGzzP^xay9-{d z3Blg@ZnD0&3H}ts(A9Q{q%-pz$@1Joc7%SVbE>9*9)@-TL(U$ z*Z?tuRM2r+i}3t^=6LGYY3yum!Sdyl-p$woQzy%j_enW)!nJJNZn#1!ct4)cd7ecs zmu<$L3$yU6YYUB@8Hc~!ufY9oh8+903{UxdCH3QY+M%ciCtcUkOP}M>*{T)GTy^nx zygGIbJq1CQ?yxn$2V&Y@QL9PHxFt*o=EsJKYGsz|L;Rqa+ zWkeq9x8ltW+4x#bA0xIPNF7W>FTY%+1|JmA_3%s_q`C^6_2xm^j{(wE8AGUcZZ%83 zgu>0#6d@V;2Il^3ByD37pu~FFq{2QRVSKNUHYD!^V|hT>_xC^Y8XbY8jk-?mrw(FnMR*&f}ej} zlQ=tamulu90uyP0_EqpLDfxj z@Tm31Wp0@e^IyN@ru%+j@wZ91ajOX!T~jBcG>(JoMm>;j8IE#|@nCVp9D^**z>QT| z^twklzS=buO5FS4_<{&NROW-cX;OiwNmpR**DUJcVgS9ne<3w8#Th^Ju+bnN#@vj+ zg?~bYX;be4&-{^{=P+s=d`gz9-4BiXLdp0;`%(232U{-sKvP2~OzKjExQd(T&1mAa zOXr}qUmhPM@@Q=~4gX4!mbyp4#B?)ao1{(MyLocu;stcAdO=s7)R)Gn9|d)tZ0aPp z8RxCD0F!?QBu_{_Jk(|ItXB-tIJFHMRGvz_WSObh2b1xM?MAx8@dPOHo8eD)I@z8( z49ZQ%%g#YGoYlO5FTcm&RWEH(fv*XUbqXQ>F6H8N7kymz{vkR^^pUyWMQyI7Lxx%c zbdT9jd=z8p)s-q>ePW1+AH53lFL;qF8kLfw;o-zJ_cl?i@*r2tYe=|PJWk@a(tbe( zO#R^tU!Nd+d3hdrcXhDeau#0}Rbp!BZp_NqhAt!lOr{Zo*+7q(GU$a5ml`CNL&TXo zxKpNY=B%j~F4z)Ji|$mxjpVDS@yP>5*d`LMkUsj3xdQ60^>}KhmdI`QOB|DIN6pXQ z5~djk!`1QEY4HhHs0#|CHl=@QrA;&3njwIkmICnoV23FkQzb%!yKrIlW^`xkNw>Zc zaVu*emrUeDrja)>@9|Mw?J}2s{!ve!EFOh!2RZtrv`#p{;u$K=PzQ^y$2j`^SiBHd zDg1FvQ`DYQPj<^P_TS@+@t}1sUmpFI-IHt%8-xVY{_I5KMGweu%XyOX4c@pRGy}H1 zj)1I1^)y-k6r?vFfrpQuLDm~erim?~d&No6mpe)ti%$#n+|OgTcPN^<48gr>voY4z zj~uLR#?Ka&81zLBM;R`nd8I1Yb$vQ6&%8@a6d!=vT@{#Nln*JdqDhaVD>lipCM)Me zW9&0udY~`?KKq1YzE3OU?a>h)_F0Y3-&Uh-=R2yFyBW)!&cZ>hH`Mc<7T83(NP2Tz zK($6o^Jmt`v(6i4ICeWNIUeGIBytsxI8L|8ZLC;d~} zNx%0fOONXnfmN!!w8u07Pmifb!Jh@hWX?&_`7Z^+WOZM}xbZkd&l^kqqLKVRvyxaS zc~Lk_M2pLX-q+&r&%$@~LRT_;(tZaPPTIKh+D-`QyMX#RQSf+9BI-<1676a7!-4J^ z@MCElY+zSnO@ubdI((N8x;28f97+d=p%HXP+BM7&i81T^V3d2h5GYY&q<#>*j|ATH;~{*>~#uIi#r$?66Hk@yC{68d1hVR;D+7 zt9*q_NztO6bGPHJWxr`i)K+@qo22-*YT&6{0KAc@!3U#a2{E#UOaCcg-N8UQ zY}F;=jeAhF_B`?IwZYK7>GWy71nk_ys8-canxN5w*`xFI{PAS!t26hYqgXd?-HnU91`piJheP`$b&5cyvd=;qeze2+OBj~A`*4c<{Zy)H28bznVC)X%`>i)_gD6>aGi+9*?ON)MCwj z1sv6VAG9=*apna_sm5&cAHGV*A(CNxHThT{W` zQ8Fk7M^9NwGg(dIckY21BeQ8Oy%wLdStMv^V_(nO~)bP04OVbGJ8~_D!JcPRJ3TEoVsD zb_ZN_Syl9tiy)TUr;;mGBSa=0I#PS{4Inr%616Xdz-l9ZzJF%F@LlC(kdM=37xfPk z{XHm0(_gGaU#FjBs=|HYzvhF4|MnYt&gKcrmcJH`R33}4&4`+;)(}b8~Z$3(DCiy`7(j6EP<^=o4XrZ#xb=vQ` z0yU#Dx#%So^g+N1_+_z{`sv%^zPkp(`h#xJqdNk-KYC$vz!kYkTIdrv8=v^6;!4{D7<$)LwvJrZ zmlsk>)KekrTaPf<>?u7J{t8s=XG;H6J*A;LYS}VbP4actCjLTHJPoYRq8fqKu*#_*l33WN3%J%_3CvqBV%mjlRB(AhM_zkIa=z%3 zU%zM5jk4Otac~J4oa=zPFPcgH=30rQ;W2Gr;7m1&1z5W#k4(9@1Y0c6)4rZ-bnrwc z6kc%?>Uuw-gGOzjhTS5W7BLM@U3|ztQg{l{J}*h5=~zsC!oyZ2d)!*T9WGZXNIwsd z6SbRWVv%tVZ9X`LhUj?0z?p&c_a%AQ`}ZX77&?`hm=2;dMhwOosRpp%-#g6pbR>R9 zec`#$60|xWLuVO3gR5VqB(^08_Qk|P$9;k1bI@@dbY4P}%Fcn;u_M&aGXsZb4FTIx zx3IZ+ldNuAik-^GB}QMJaPdbG250f`ddMv5^GzK$?H&pnBd+td^#et=3)RsqIU9m+ z1p{Wj$H6%SqGbPT!@6{8+#5rT4@`x}R`IAc$qUxaC`6O1J0*hSp=7UN3h5v97Mc7f zsOnxo+TV_Z!VeX!UYVY>D7P0HWd35cK{W1d-$h(mFEZXN5tmvVgS6AXNcx5IFh(yH zuW4VvoQBD8!{#Bnl-YoCX8=s`Y{1(`>LLC0UA!^SmOl1OrR}B^vx+W&f$V?r>!=SF zpD#fY6aZ_?r?PHiIhaOHqDS^dh%sL*>F*i=yN%j}-|SWJS!KI$(@z`hn^`5x(xt)R z`X1q&uji=aHw#cS3c&v?KC#G0(&@QTIHKDG|6Tl#=Va&SgqA)G4(fr=`!C`-RSls= zUMS4{`(AQF;{p8GJXrL58{(PUv-n*}6i+G0T*(0!>6qJY;@_Xn^Xl&}Qt#U<9lN&h+{k*~%{tgCxT zjddSV9oe^E+n6ICc5S31W8(0uixsS|*a#hyzv0A~E964G2H9+zMhX<%VMkf9aM|B7 znBo&j+u^-*Cl<|nPv=TP5WLm_ zehooelSH~q=M@GT0|*=3XuqkaY#whJv{vsS9S4hWaK~SgF)$y4g<@=qQ4`($?~BCI zS($Vu+rS#dHsRKWc*J=uB*xDb?YRC5T7SB-)1+^NmU|uPtkYk>PuZN>pZW=Jn*52@ zz`K|<{3S3gGoi=S59U7o%Y$^BG_RpavVG`yjQp4bX~kQ>KWHS0-j+nit`V{d3+k}V zI33?!TM27JrlECBE$Dxli5sie;Lv+DkX5!*s4f}{0>cH8LrzD~ae_01O)?Xz_Zdhm zmIXl1uj9hFj2gn)?7$zpbFre-0G4V}*d}L29HiQ8yq-JGR4WI0BWuw&A4QSe850?$ z_yLB_JO{!TX)s!r?QV!{#Q3|@U|dcGP3`ajg;hOZY5kd2q*x;vI#{&bs~RVm--DK^ zCmd7XN=!oYu*F8!3l1nET`P(NeH)MA?i&>HSI5zbKhvrE=-FW1m&lKC zJBRy|I-$uv6kVzgk=Ot12pO9M`_??8Hf`g;>w6^9$EP9gYY!D4*A~85+yoOv1rncE zC&AGw0)E>I>6(R#cuuCJc3xfuEf0dJ<*uKkN-`CgM-}*KnwBWSaTf|NnxXd!RcUx_ zE*$jE2dCY35V~KJygum;4xwX2f5ky`o3jEg+4hD!+7=4uq`Ksg-*~95k!53a7C>a6 zBB~q@fe{m2(JA|zuw~jZ`Xgy4tsk!pn{~bT4rwIJ?Z}`xBhNr_cCl}#;JMW<6v+JcV~H6UWLGacvlaM6vPP=CWsUHJ_8C<7TL&H8 zCuLrHEN*sQPxW@0L5--Fe0R>Ji5g}Yu}T3~W;C*|wbJ0YQWKddDI^D%2&wI&b@23_ zE55V-M&21L$Cu4AYSz`mkREy%tt``Fvv-OtBM=WhX7XgzfaS8Rff4itzJX1agN-fsOtOxE5=SGG;V zrl0%a;xQAlD7+32UDm~vv9e5hjyhVAJ9PMUPgs7$oeebarqpIT&A2%d3)}5LxuS}s zrK?KQKl$LFx?lJ~oCp>k16+%u<A zmlXU?AUh&mVcMxDr1z#3o-@sb1KX3(Y}E!luv(TYEO3&zC|wn<(3%9ZOOn8$%#0r2 zaEq>}F@_xBX82IF7WXXvM8`cl!~7U3pl6vJ31;jQrnG$}b5H%?OPozm{oO1OcHWm{ zg^d#Jn>Jd=C2xUq56xipi&gOL%W|w}$bjyEJ>VXnkBYI28IGund<>6Z>tAhHbK@jE z)U|=M85B_?#gnwf=^+vCQ3jIJNhbb{AjZGs$y~2Y7`FEac^#OE=IfPkxK;p!G@c@( z^}S)x!-4doWgyhszNdO8>|yBE^LTDR7fzKMj-MRLXqfH*$+3z;QYhaD{Zegk8G0CA z#~Glj;3XNlsFw5&cuh@r-a!6eG|1l=jx#p+lbX9H`GnC$P=0JO)hjE&I=92*&h1(n ze`^TIlxdV&zVUdlV;A@bWZ)B{2>e=^gWY5HNJePTec9`K0uv$k8 zZw$C4nRH?cIh8A`FBd6`maacQA1cg;TZb0X3&l0CuTFui!)}JkuMsu>}jbA9K3+^;0!rjrnV7nF6@S~uIFex^e9eIGnVGXEkO^~ z9)@W-QIq!)To)Pv{DE)eU~3xP8Ss)G%^NG7o%xg|6dfiidsBsnj4jZ*=poc-6_QYe zyGXa|NH3)y$7oFj>^J`l$D&S?5cMRXg_b%Vy`l%6(Zi)(1Mg5p3wcpR#|{{n{tEv4 zRf6hr_i?S4h@JYbAD8$kkvW!!LHDa0Y~%^X4H*tjLk{4#!qZS6xgK4uEkQ$jIP4b; z0oBn8w8Fz$GFVAb)D|{?G)@qL*!(b#*Byhg&t-cr-c{1UA5!3w-vXSbcNQj_SYWqh zEUYPU05d*d)jb^D{!4V@D z&GO^$@4Y@1_IS&lr$h9EySN2?>g$J{Oa++*;ine;q zhWlkl>2m8D$(UasAZYABA|Jz$=eeC^!~1HIZCFlrD{aRA?wa5l{vs-Q>Z4A#7O2FY zh6%SNq)y4LNuDz+c%j>-0A4E{(*sak+eUj^Oh9|HyGMx7JVh)1&k zpC9OeAIq-cYu_C(S{Mn-#vQ@0Ti%fEU#%tUOnHgl&kW(y_l?9qX#}{%0YvVaK))I2 zVayU)947x$JD}C# zNBgwWLi+{ct$`231U%iNnvkLSWBZiODlZ47$FUni^$d zUdJ=A?dig03%nrNIgHe$MTt)*7sG)UsgQCb4-;>?qpfNPPOlG#=!RrWXtc+R?$;?y0GUDZo24E+Ztnjh)=H?=4( zN`c^lcaqmfd3H#y9DDCrHY{}LBBxDu!Mh3NU_EpiejgkLUJ><>8Jx~b|1^@_zh(1j z+OH()(Kc`_vJf7!8=&=23cTFA6kZswC8ZAYF{ED(Vnfp~d)RndzU>k@TfH0q_F0kd z?+Y>QN2jpp>`z=zase~~V9S7Za=mR|WFV7;PX}m;T;h%pP#=XE0nza0a|((0c@WPpjszW_1RU$G2$7wmU_gBa zj%JRL=P8Tf@xLOHvnClmq{dj+e4M^49Sv(1ET>wY9L7JnO9gjgg~L7mzl*>GS6@y> z{XN;Z*-~EmzEGL`n<^#eWPVS9M|0`+rYq#z zon~G}icz8QOU+RcfePWCvo;HrNY%? zu=Lw@NK-O^OP_S`*T@@mZ|rEu^L;?(37_NMHZ$}(EQFE{ZFp680BftSfJ4w+aCO?pG%<%VK@U+QoeIPVzE5G%vkFOMV}hqR%ap(D+`GY*wZJMq2dd3w)l zBt~t^q+yqeP_oPds{*yjkHGVIX+KMQ3kFC>aU)^BFTvBB-{YLBd*rrD9evyP8KyN+ z9C*QoIE%*MmREA9toMfy#q${9x`Z+t+k`h#{=?}S5%g`rQ(2Cu6LFRgJf^A;?=V@i zQm2v5DyhNz!LRAfz0K6PDjf@|!)V;JTzboKB|TYJMYDce2T8R&_&vy= zZRKT>L)Qt~KN?9Jb$Uq9-ba#|PLXhCgR0d1ND&6xe4^KD7fQYx$)Q7&r^r9TSk_az zN)2|L0SWhvEeTDMUHdj(-8zULvVSS~-7jHJ6spnZ=^-%9M2^rhW4Q2dgvj>l5qPt_ z7>p;B&@87Cxl za`N^OER<#J%vTok+H+)Q>URJ%7mff?T0c$-^nw3E_u|EYs`T!a;pFNiZE4QN1YA31 z2t3Fyhw&kPU~$(H?^UM2BU%c~4r4s(lSq#X3JIgJPGl#`qb&3vAkvAAl2{HgT6r&sx-!#Otg9l5iUw4vw)8!z{KM>T}LD2q3AKX4BNrEfV zVer!(#Hlor+>y--Z>Y-#ui|ZJsjUs~2C*nin;~`g-imi#1dzp3l5uu{nn?TgaFVm{ z1ddzcfoFdom2Cd$rkF3%&}2NHpIUhnY-0|Dz z-LT+vB{1q`;MTAXFKZYh%Bj+8y(w`1_zPS#cM8#)-{vsxl^!0;jb-+mY)36U$o=b+-<2oex z!aUNT-$n4xCY)#f6!TuI2v_A_lN@ttA}D*$7tLM?Nm≈aIY8$AbUp+%i?sBxX7W zPc5N6U#($;FqfXmUy9ED)wuJ+YUFl3gjimiO#jamPX4C`_6j3#WT*w1H(Cy)6VBuH zdNWXsd5><9U!b=ymQnE$p&S$v{U2p`)NCI)?($6HT)&)VO$wm98+N1fL!SI6osXxw zRir*Kzd?HI3jNv^hU1L%pdp#kTb<*mYppsg^Hjm<(IIGJW&~}gs>x`0h9_3Oh57UK zaPn>;2v>UuM^6rfsYah@b$Bb@e*q*J#H_koHIUBrUl*VmmIT<6ax}=+{*f^fG6de^vj}73a z%+qEjZe7lhvHEPnAVYRvd@8r#O)Enc#HKEq zVzD8+?xX@Ak+Gf+)11NojXuCC72IRuW;ybjIVzPim4~pa^JZ~B8pg8b?Io3J9zOi= zHM+cA^&0khUK*30c&hT*`)iC1JdL|Qd=Gor6q%EPN&K?4EdOVl zjA46!ExX7fTQJLLFw?l^pWw=0!fw2}iXVDWwksbev4?6VuoZVFva#|l%)S$T{PuUj ze4DKY%al!Im$SkA2tN^@qPdhgdSD4}=rD)(VOyE8Yk(cL+)T#PUBcNum*dwuI|sSrznAC;Z{C#67Z*N-1yj>lT2T45Zifox;X2dA+vxeuz$S| z37&hWGUt{~XQ^Zc8+7Qs!0-EIra)mF&-Pn$vBvi7A>|@YHD)^R*|d_8yoh2I`)$}I z9S6iMhI4quWz+f45#xB-BB)pAKX zFPIm@9x^Gv_KBZWeP%u?4`d7uIPwZEf4E6CY^A`vmFeo=%CtEzWlM`Kxk;hF8Chn4 z(cR_1KNwKYsjFUK4qw^FZhPdy^hieX&6?rN@1j$J!dpvu)mKx*4qJfFx$4Y$&E}cs zbSK+f6VGYZE#=>79p_rRJh(f@o4B{(m$=RWJGk~Nb6$Pk)ylY~JJ~d+Jnn9lIa6L; z&OC0Y;)m?D;CISz>57w%)#~Wlk7nOm`h_&%d#(0^ps0`wiqJ{ig?XOHw@Li^Z zj^TeA-{j<)C$ZMw1u`DjV(!q=b-dLUU+(e~5B8bRnfDec^M}4U%4Sz0_?6BTOqsea zuT(WhFj~e6+vI1 z>+D|08Vj6HRt8dzcmI59P;qMerW$ zvKVGwrI_u~;{ra_aBoYdajaV?yC>kcVEu34`OHGQRcfy60WC|$P2mlP%?aE^(@w7O zZ7U~kuN80JBVq^a_`@l8j^#VmCiAJcZZX-bxAS3cTiN}F0W8&&@yrd5GsosEV+-xenLmnq^ zHWv>vZ}l|TOodm1i5J7T*DlKhn}Y3`NW)3&m0@p~sVfqh)nZ+~DRwRYR<>iea91cd z+gFoazBP(T40_D1O8vxaxsk>!=+2z<=U{QZP+7`~k^uB_sU1q1neW5%(=ldmzVMZ^aeimWWse!&g-w4Qeq%T+dLhcKH;@)?=1!S&g2mAXx4 zJkGkn1U!{-!}h&n68=UpX4!|hW6;R$=n1zYCAS%WvkLB+w;TUGy^fjiXfHc3>5OfO z007W`*@Q{L)#t+-7giTP%Eo7r@D6^juSg847!^G}5q{MWTB*arhA zan3PUxtfU!S=B>syq^fg8#23@SOrgZ&%9;grn|GbxSZj9`RwnU^09&Je@eD|p7&Q7 zKkyIt#B&q7$Z7%KyJ#sN(Y2o~lyTy3B_8C&!?T!Q+UJ=EDhIiVb-KLClI2{bq5&JNMM6#!EovS=F$%Ahip2zK13}RMxf zJ?_fr0M_hHDtE%VrDEZco7~I?)49zaNsQ8|F7CrRE&fP8@CtW_a4Q4t_=}c}g8lhh z*>{PPxkN>Meox*txtglN$IISf^%ITaAMX_Tam63GMZc!=s`sX|?ZabPr{{)j zYO&y-hBYUi@k)p)s8mTXn#PPSx^3%jO3mA4Mq z$?K;nFj4Ul{E^cJEVFGkd%sCG`YcIehGh5JE^o#We#@Ibtfi z{Z}(%n6Fg+YM3b>xJ1N%wB5k>Z6C)67r$fPyvt`wXGC+~!>XB+LFLToz7yOg>3%-A zPQZ>3hp-m&=CNr#>sW8^WN!1PeY~q%2sd}E5wC7HUMx*{&fU11S?QtpUeJO!n9Ut3 z>`JE=?sTq*@2N6jtB)UIUsq-^hx%`D#!(ab6&qKx?kS(ek;D6#chrPmta*%iQ>4$& zT>V#Exqd%)%2Ty+2tChCw~XN83g3%MtQ2{R4})#Lx3zLREDtkwo$Fb<#L@h__(0yd z<`I`LHi}Vw7|T6)bey^PUWIKb9mjV|s>BK}WVM`(NU2fC@-H4nahK;ZJlUbkAM}67 z-IUde(`F~Kb>aG4+rKoPvz^1AOrOZ#sy)t~5?3>lsRpccR|0oxoSI;Fw>GPlGKxK- zZ^KVijb@wLg>0FCbe?ptEHF$FQ@8*GQL8V{Xhd|DI!?Ck%zxd$EChwsL)12YaX<~`@QjX}vwMOi)< zFgBWvV&W> z)m~YgqgfpnT#&(x92{GzTx-C;_Y38SNuixhQwQ_Q&4LGi7yf|ohgbqetm#k}E;VJd zVEwx(>^94pto_&#j8f-FzTDlDZ=1b`Reqhxpv70Q>ET$;y6j&w zv)|kPG8gBqW6rreU~cWRW3T@54V&o9pB`)}XjI$kbk4>i;=i^5d-X}k28lYS%E!0IWi%_|dj zqqh?OPiYRn_e(QVyVH%eDgySh%^hyTj+x@|HyD1x&CbfNvi)9M$PH_9+HvVyD zH5VA3#2nSSEqJW4nVab0#XSG%T48n7gDYN`#>EQf%J{Jp_+`e=I8RkG_Rf-poc7W0 zT-eCT>{o?3%$%@rX6se~8~EQzLAz-TKh)_V*L&D*vo<79+Uz%eZSNTHVWW9**I;z7Om~LXHP94t-9hl3Ey06E^a5LEw zqddX(a9=*FF;o2E**3Q1-xX%n%_E%MjRMY2-9?74Y+yDsV&=}VazRZ@C-dp8x!~&{ zP5#)6%gh4*T1F*eBrmnH7MrJ7vwxzWF~dYH+<*Uam{FH91TWX+a8JkOac=R(?6D5Q znY9n(H+P8GMN4$~cjMIgo&R*%wt0itJEJ$U4Y#A1DTkGKTkadvF+fY)dp((PK4{K9 zZQI6u+^EaFT|JIF5NgQkzgxh?9pIV6GhTC*j+;44k8Eb){ZKySgA8EN;=@)rjbY`E zxwB4XKbVCM+xUK``!ZZ!qM+o+Kbu7@d+a<%591rl+qqW7Tb!oDAa3+8duHpKMf`hh zYyM;3eDeMjhSi5T<-0snzNVAQE;Heu z)IVju8}fEnb#0h^*HrnPGOp)L3=`};Y{zV<*Wj-``@#8YT;_Vlybuqxp2uH2JYQTJ zT+1nJlGXlYz<$w}=SC@wVs*Z@Gy21Z@dpOBGTOh#v+GA)V44+rxw@nb?oewKbNyN@ z1HYr#JC_b~*98NZS`A%x!52ejRl+d4_OI>&XMTscaqL!JDdPfH;}gMhBUZ69N-g=h zect?!&CQ&G@(kW9cnmYGZ8z_nzn_v31+kS<^8in7Q5}=Ej)~tm20nPA+Xe>!&`N zeMBbmCavYnCQ%f7-(Q7wx*o?1{gyH>1H$-Enk>KRffqYM-;TEoFXJ3HzGlkqE3s=I zTCxR)NAnK91~JbC)7bUupO^!CCiCeEKkR;Y4Pe*xpJSGu7{Xf{6bM4^IkLAGJ>hQX z1Fvr@V)tv5+vQo;Ggc+*dGW*uu5bJqKfZJ*RI79tOy` z)n2pN6!$%hj>lv+zz^8lS4A=|$09y(&pJMF@mjmdGyPZxhbgR8@jdR|xe0vtiB0VJ z3vUF9!+O6Dnp$~6(&4u5-p^WSL49psj7mRq9JGZ53E4QkooiQ?>!xogP$(W5N z>>NCGI7hW5`~jIBq2s-m*IAM(SZ}(M?dSa16^R<`r{;YO_okNnT``uO`fnjyVt0ZW zZso$C|7gKa&78xvl%$A-C9bSNFbG~U)lC0GbKX5xgZHlKVcIKZ@sC%J;oG^P?D}W3 z*wM2Ov)Zmtn6fVcyw0F?y!E|oZm`pE{`dfCtmPSnq3Lww|$PJAXRxvlbh0IURDWlg(Hr&N7k>>33m--r@Z4Ay)jdmEq#qhb7GH zGY>iWF$3BC)S7o4-!0yjqROr)HDs4B^k?R@4C4)LOgXjQp=^|v5vx^D$0UVViFKNAq;O`KQz zAiGN*ja-({hpo?@%pR+&5El%d&p*GU#(j9x&pf&}g&qCgmS14Dkc%00fQ|Or$bKr= z%)iV3k6YyL#71mh&APo>Dv;azo16H{iFIf>%lVF5z{^>wa+bOd>`Sj@EXtV6?ne!| z{X;bP1%9V1mv(+-`k%0@>dtZejm2{Ok=u<-M!`F7(yMwdu=x(R?@t(OI{JmUMLmMw zw|0$q-+I}7bwn}sD`xPE&kx}H3-)logD3NXqyMg_O7+Tl8Havs(Mp*G zTFlvPY2q4<)!1N(HZ$(cLFQTNTW<89aL%wXl{>e@Obm4sm`Qhf1onk#T-(m!Y?y_o zc;*^MzM7!96 z$O$!`oR;AT(=cdTAVdvuLAM2AG}Sf^$2yOxaaWzst#<8Hm((A9i|T3WC`%rf_m7t8 zPr-tsecr>;?urhx|J1E}JP8L&b& z+itl1jReg~wa~mWK)lrLkhtD?BwxF}8}6uiW6M2Vp0KEh_C@JoTEi~#D4oeZ#Vbi} z*L%@bKa-LR#^Qt?=fvk_>3n=fC=Uwq;Fj^fXno%_e4X73{!6+jq+GJ2idXNbsQjR~ z!U}L;eRMr7RKMsAL+4KA-{lFs@5TmtF{K-J z_pXD}AL6j4Sq(4BG6pZUzNX0+j9|j?1=zdQmgZNrQ;#t@(5qPki;KIlzi^yB{PxAT z@BbhHw1i22r-I#>da}CELA~QlVb14P+T>=&$*056A0E=Zsoh!ba|9>Nvc`vXPS{%u z>5g>_J?pjt;#!%;S@gocJOnr9+=047R;=^9A6GQ!qTO~i+^al}D+=`Z(aM4B_SS`u z?Nj4@vu1GOVHKMB#t-8I2a&X88b3}&-0^1~-kX#uPD)ebFXMA){Ep%1GO{-;a!$h3 zM^|Zo@pz6?egNl;!?9$1G(2w73_IDf=mCipotf4X^5+`5OqEg&UkEs(@pMt zE_fV#xGl@%+dkY|E@mK}FI*`s=~Rbco$hqG`h?*9PlPo^G4Sk)Y}SsO$W9}6%kVBa z)ZOBV)!CQ(sj0S!uP+y3*Ir(@GTxj!XLpKqc`GR8vN!vi&E)HL&*0ucLpVHY5U%j( zf(tE9K$h)7ajFbQP8VWntKM*K>Z8K@emr zoP}jdx1iNo5koFtKR%D`?XYb!V(ih5XT?bQ`=eB z6$2ZlLwVMEA;!)P8s-Gz^WLs_)o=;#8FW<$+7pX9twy*i+Lljy#hP(7# z(5JzbH(O@&{D_V4Xx?-*>oy#%C=y*P?uvzDJ$Sc?8dn-mMw9QC;nOGy7hF(;VjUy2 zT(ORz*SFgA%?-2{p3=R4gCOnlpxVyaci>J!7hdvcFrV8--y8;`%bL^T zcb^pa1%S^biadOo4Sr5h1^L7n(i^;AOj*_n`~M50^+N~psck8E?sXe@9H|qsjdnuv z+W*9t!S+0JN)&s_mRYRRkHunlb6lBijK;1Xsi9<(5ZihKX77YIV>0zh0?KYM@TQ_89fQTDxO&A$k%f-;C0tOVtuwHkC_sK8JcYnHDV8R zyZ??d)m%U!*d5i%w}F%6FADm2jhcQc;O{BJF^QD;rolAQFB=2R@5At=?^re}D?qO$ zAE9Z-A4+md60++*LT0x}{4stt9zA%13=iC)M&%yt<7UhmSKa9SIWN|H(?s4+qVV$3 zQgFX%i=Wi~(1BMT{Iy>w%-`gMgPm*yX^(0c@B9o>dTpibtV__}G=`G*3>CwZv$*2L zSFp4xz&lIjxL9Q|w#=W2`5sfD&z5kS6*`mt{8vGCYA+yY;1Tc~l*s!-`|>?oGfZ&a zKq@N)J|!Q=rw#3Sc*jeqdMy*VH0zV|gAz<@Z-%wr zIW5zn+mX4#4J}okn68g)=5c5+-&nY&?t$|h0%D*tzPtvF!B3t0K_F05SFS$FElGSPB~IZE%}pbzrF_&Ver ze7wB?-n6^1rYuV>M<Vtzgd>!>EE@kXr~k3dV2`+b3?H*GYMxL)P;Xat6=A8 z1>ENIhk`>*D6**&#>~~i>34pE{-bUT;QVP>6Zh%~!6STk0gZ9}@-1pWcdL+xLzBm30&1+M~z=IdWE!OIM zFxa1aRQ1I}?v0?H+(tY0f2RRAOtJ2woA<%t^4xLCi&vVd;ftt&xbvt+b1`kx+@$% z?es_6yG!`_0ApOZT~YYCM3e4!Gp1AZ58>{*YjFLo4c>NW5NzEnFnsS2p8hz9>DWv7 z(qs$H8F}Jn|500&w2{pqrqEUr!kiSp2?5JPg%-ri_xs(dxmr)mZ6fnJsc@~ z0wImoJiCVzUtS!}vC(63XjdoJ=<3Ce-GTl^2w(WyulkF_gv!XnK!A_W41#|*}D#kmRGbg@4=s21SP;ii~Y*(#Jxb7P%@b{u=D51d_o3qHL%BIGSf zLkt@+=-Z^f>5$P82TWE~okN$LaCz!Te6+C`e4KK-p@C z=sW%n%{e#V)02;G;@q!ql`@M1?;QUUC)Vo9jJG$}T6Be{OJD$*56L%NShV-4? z;LLq1+}mu3{cT=DU_>d|w!fh?3_(+v0`bd!Qe)5#I%b_u1N2T%^XT2QYP6Ru^L#6X zAGO5Zy=H>VFAF|oZv>hg!tzkP1_#!SyBElM>3eAD<}c7wEtksKiH-*~(f2E=C>9-p z5l$U+{+~!WA&a=}Sq163EGJWq`*pkKRe)FNBv@D!K#xme=}ua?5b`X77q`0#Z@VkP zi1kvsxoVlPZPYdRqIr`BH)b)nNAtlTV|4cJ!lQy#W79V)P>S^DOQ}o9OHf48Fd@D7 z-Nm5m#~?vH3!dCggA3Qlq=r%QYQCJYr#$9blriS>ipjoXHJ`SRW$JPnREaNl|S_okFy zysQKDsrTqsv@RW(b^^{B-Gh(AtD#Q67j3e;46E0gP;=66&~27SzslR{5K!e+9>@_c^++z*U+^rH+RXd>NTMQh#dxN5j z-isOWQFu@}8dlC7%I`OcRG(%F?N>h2mnpxXI$4YFW183})k1uIqbFK;X`@NK?6b{+ zf4tA(%^J(S&u)(5QDN1f|4a{4wUxPYOaLz^&m^Z88Kkv4k?VWBq+l7|rgNc}_`EI= zryTwbHqu+9uuTu=CuG7I*PYa5fG^L=njq}f83w9%j?jzLd_KRmgf<+L$7bx!Q3j7- z?uOZPLSX>vYWNFDj|b62tqJh(Rh!Uj6_2!dq60 zZz~Uq&kt;YUsD5d)6e-R_tu!_gl6;c=^J1fNAaQ=y}2Ux5x}!DNHI6Z6xAlMplUJj zvlUwYJPkUlV8nPZlGryHh}W1$Ixek8hP4h!}6U4qN&eQ zYVj?C3BF^%da@Z7Zzl0#=^{E(+YXaPYGCs4A#myX11hwfO&@MN5v6Nlg%m}3JZ91h zr*>kdqUY@ZnH_HitpJ|xd0KH0$*?IFBBLzgE_?15!+F?jR$A})HMgLm&P#w*IB z#DguBv~!*pcYgmwma5&UI;so4{_z>I0%zdD9UsM2V-E@adWYh_Y+E$Gbd5G_vS7;9 zCWQ;zgzi4pJYs+Z{RDWj5(GC{Xre z4Kz6Aj)`!r?#q-TRIY0Qv&?$n7BxAZ7?viyuW{pc-#3)ge;jTJbHo6X6uSBO3MDS; ztn2>Yds6m!?tOm3T`>Lmf<~+#ibk(|@MQPHbp;9!y~AsUa#){X?CzfpH!Hf+mSGp* z*X79+J+4W3CgVMyG|b0$>$R}@ha#UVsH6A~HP9t&DO`FW(+9lV0P|Y4yw_x`fp@(c zgmD`3T<|>rC)`cK{wrGRPWZm0qwlR@cytrhpAF)ChshB4HkzDX_2rUaD=yj-4VQk6 zrTMQulKTJ~yfWYmrQ0b9m0N=`U*B5zICCiPeD|4pAKwhgvRq%!e?9rxq8BiF(08!1 z>n+UsJgw&EBn_UpdMrbjH_!d~SnLs?#lMPra$21(oQ~Z=OFY%_a_9&?yxWAMe5rp7Qg^?jc{yhUNB#*~7Cx*a=yHn;Umhn_ z2an!Exw+b!|J*+$Ojx@V-_DV7-lqbv+xCr6*hh&KU&wRY4IeRe<|G(ne-mnUPZifk zDTBqmPzqiW44aQm=2D9}6sn*{6MP?djT&pi%?b&$*<+v(u=o)5bJ$2r;4-9Bj#ewbC94o^NpIcVZh**Dd-+{iY@y-rx6c_U?gR#Q0>$7lsLNUzT zs{j_GJ`FR5>*$6V3VI2A|$)cJ| zJ;jF1!E_{E0L}ZeVehE`NK3T99iAruvJ^3Npgnm#{v-N)9)SBRVlY_!0_c7`EBv`D z&rkB75xm|;_at+~ehXD_=*8orqh%dz+uSTR{29S}w-4dd8rr;9uMuvJT12N4vH&Ir z<52%G;#>Vu++)KKu3D1C$GocPmPrbBMBZ<>QLM zR!sW`S?@L2eVHPx;VP=KZ5E%t2}7$pMv$_uACFAv378Oz+f**Y%3BfmwdYW5jSIoY zHRa;%*q3lny^>B%9nWoQx^Qx)4(1-Psl8He#x!yWW9?-8CF2s?b7yg$UI70IH{cNM zzTV@%2ja59I(Vd~34Og{#jBj-d5ESN8#w#~=V3!|Q9}Z)bZw!q&MWYCZyp%7=fJhC zx9Pw28GQfB7@j^?_CKyN0J+)2_;c@8lAD}Pk0-9AAC9)%7`&SdryHVJGY1t0EaU}+ zt(5f?FmK9!?}pF*yms?W`XNhF(Q-AVqf1(0dYUTUmTVKx#)k94x7UTSuWUGCg+Cj4 z%JZ0KOR0U%cChyI7Y506pG9kK)DAoHi7L(x5JE;@_b@iSG?vk9>-Na&J3>|S@NHTjVT5ef_qq7lLOw8iD z2mN?Z@-&=gG92^j^6A~heD;?uWOIJl0g1Al^WH<7V8(;<@Zyyr2ErM*qZrSJ29`t4 zsVn3?dS5>S(HS+TCGqGHzFZkN5jD;PVBrcKR-d*^xH@tO>RQFYi8o8oQ0Ibo=b(k` zRHDjrYIcKZa}dTg4x>#;9xPi_N&R;!;?fE7B-cEXf_^-sd8_p3u#O58TuZ~e*^cbB zARl6M)j3@|1w)ppnJd?XX*Fiq+ttLa%V7>?=lfp?MKqt=kn~**aR__$~?)` zmB%eUB)&F^z^6YRfZl+foO;56UtHdjke(XgRd)w>FW$!$W@G6Tl8 zU@Wa%2R#m7rT6iLq^fclZvD4`9v#-El3o&VLyRsqKKcW#$NSK$XS0R0L@VBK%A73& z-qMFXw&KS(s{C`4Gf!E(O}w`6IGqg}39ajs@abU-F1aw4_gp+7-gCVSi7JZJ*~bRY zMVsJ{vjJ@9K7$tokZ5)W3YRC){&&Mz=dEhpw=vyt zTh~Z@-S|qdZ`}r)w>xoAT`boI*OGW32Qb|bT^#zu!^wRxx_S*Qiil#9tRpllB%f== z2lR5C2fBxt@aH>V>G`cDAmY?{dWOO^;4Zwd0hyW>y`N47j*j;$x1Fe4z8CI|e4 zs_{pJMRkj)pnVk?&dbETf5v0z5d-!qpTpKpwzr4ntz$f4%6y|5cwU>s<&pC+`Hw%`nJ4?MwxM8Fd!1I>TXEoyQ!v9`87-2TDhvoyL7!S zk69H68=w!*xwnpdgo|+eNtmdi(vNRPFC~Y<$HIn(A1M7+7WkGBoNHErNvC9Yl#FA` zukMGFZl8tCW&u1^qZd?P359E`oM|=};<&@{p#AYPUD@ak*0cKJypkLmZsW59N~vihsgPko3NR@nb{$bHfZLPt?V(#b#_1A@J9QHuN-Rr|{lk706UT z6l8?x9HfKm-ws8&5Jz0p-;-4io~0SvD#@&=o?eVw0$mdYZanjl@@_4_ka05pA}Jbs zT>C*w>r(meWksRO5gm3n>i|RRT2Knopbxc{{N=C@BwjuR3kO!vr89F#t;vh~-?nD; zUTNss#em-hhH$UShhWt>67}v^3J31=V@IoSoS^T9Q#Q+U#h^^`p68C5d4_QN_+vpY zMG6lwBPSaUsL;Q^hi9-!g!;Pg!`oEIuX9P434i(>wNADbvw=X>n~v zF{O7iO7w^{rprW2ZZsT0`RGUOep%YCRuncB z?SWgFN4>v2Z=ljwmnqe3x@?BYfx9|3AT)>0Z}mre8&w`qKa^v#KEm_g zW4TM&c$S{r>-FT=IGm+*7yj%18cvLx%|ivxJe8uc8uT~%crvD>JBREl8$oiA8Cb}BgbSqj%XrQ{oq%5z0?@d!I~1+1qt#w4C~MebZXb1S-mDGM&)I0Dw~`KI|3_bnocKr7 zAYAmKD@IZ>8~fCVX19{C!Spl@+L8gwI>YhvVI3~oE){y)`jVEn6`p*S#;QjK@mGf? z@3I^d43n6lUiJ_8BGX>m%3mOZufyR~u%sIT~!5#2Ex7K^nD^om`1`r>1kwVnPV=w)WR~ECUL8dHQZY4FFf88#lya}3aPh(*+IsgH|xh>v7sh;27jSh zea?(tW|_+4)N#98Lyz+wC> z_bCOgOyG*}Z{)SFH+KGxLC8^HeV@4mcXa{S(LbbAV0S^=8mjg zw9wR^moK~nFDfs>ifhKuHCB~c9?yp1@hZGv-G2BvMhSMwsYBPeJi4;1jjkSvVy_!d zVS!32iJO%<`1)q}yXGZa`PhwJRL*!G?b9FiRrZ6wtlsSW7YNoRZ^Sltk$mK@!dDXq z&U_coeIlCaxSW-&kDW)3DJSSp%4IQk19D+qqd2KE3O!8RQFPr;pBL9qi%SZQ@EnN_ z!$!is7)`Di7(o$FVqwC2QRuNTj8ZP{s+*-N)5$dJbEHvA-FlA4h{=!PTC6`@EWHHr zSO$8X&9vjX9F9&p1udP%*k9!p#f{a%gzVW=6!l7|&8wr>mo_v^U4gs27)|cs6>z_c z4EqVpho!3CGTy9(T<#=czJm_l_?ySwbY*(fO(>>&)xzw#1>DtHl{=?)L9K^9agxq! zk~idW&pEcr}ZB0r9O1p(rG{JqAEYcE;j zmSZ899y1wxWk|s}b+>oNz7jaFdk#(6J&I2|nBus56XD^wiHysyQ+G)WznZX}o-5j8 zpJG$E8QTCKCK+O!zK%Gxpr>%^?iWz`+W@0m)bYTT4p=@ThUa%p024bUT=dQpKF1j>#j(xT>4o!RG?xs87>{He zKfj&kXnYgr{7}K$w}OQ`jjt)Vrw@II;O%XsLGK#sA)r_Wkjk8KTdAvKpDWK?l0e>c^U9 zD_~dWH$ggiJRgj=z_1?5*tErvTL-M5*tu@h?xKh>aoyl^MmzL&3gg~yPSYOUqk`(i zK^XGy3~pj7nzIQe;SKxn|yHX&?Gu(7sss$ zk#y^eDaNcRgz(G$ICRpLI@cUaUY=PF7e3{{qFyR8BhzPCrg{qEUJby890&L>EDTFL z6X@HpmAGTgCO94y%&T{;gN)P1XH1**JuT^&F(xh zuw8r|@|C8n8btRGx$?FdYIG?)1iPfV@Vwt0VvcSl9FNf8M>dN1ygre~_l^^dzm32p z7w?h145RO-A4MvKOYvY{Uo<+UFR086hO!WUgm@sQUt#KR4DLuM#UQ#!_?9%}YzlFkwf2oxH&YzbLAabo-MnkV2q}fX(^zL0R z_HEb%2ejSbecwxTOiz(_73`t-dw(ETZUtTae32UVH_*)*HT_51_K4Z(j2lwMu39?x%NQr}wjOMhmP&)8lmf-HF zf#ycxICtz!nGeYn&Zo;mpJ#jN(h76#err5W-?fKyTBhQtnTgO^`8W(_ z`G;k+@2oaH?ErexrVc*;5a)&Nf#M~e9DbxX+sbrp!wc7fO2HnnVclloQr9=I!Uk#B z=nx2dERTiVdvfwc2Ugx?L%x}5$SYbv{`o{6v1SnKnSQ2$+CkWLYH#?b-k+b&(Bfr9`X&B;KLA%D@6dcFxgrs{;xY{KFn*uh$H~n5P{qB2;Uv?K_?2P&A$70@eRh~+J z?ggb)L_YVTxqWA`*u!%;J1qT9_P2VXSHgO6|Kl>8_sx?2#INJ1h)sfNrz0D*MS!|} zA*z`K;m0&LVS#EEeYa}_-P?~z!Ega9yUX-qeP4@fC+I<@voHQC7qD&e6xMySj2-`d z0i~`#;bZroH0!ekC#qkC8Uq*3j8rFaKsDs0FXtJNgE_W-qPTomAh>Q*t$SRxhej-w zz;Km4^lj`$eA(uKrSI-h_@sQevvel@k~=E&5Ab30<#8Nip~l8vdTfm6E0Z z@BglWOZ?0_`RCgR+}6^DswuEj!H1jA#`4+-ZyqveFxlQKfM3J^31dupaOl%w;FR4W z4@n?Hhxz!aZZ)M9$==J3NFK5DDw&b4XgIw;K7XD2MHX5s++RZs-EWsUS;*)k~G zewqZO-{3i84h}gWK*yk3w^0*afY5{)T>oM@;(l13Aj04EE)XKaF(;mumvP3IubW-FN8-is(jh&%Pjo6m847O8d?B?y zlyF|Qx!-k$IHeXE0Rc$A|e&fgYvc8hN zzag$2qK-Fwcfi=l&qChIbQ%!^Xfm*bKFu8m<_E6RvA)Y_L-XZYuQT1nk(m;lZe@l& zKFNAv^DT7QaXJm(yRdHX?vv1?|7EB?IDkX*O-S>52D;DxF6{1n1CE3IaX`s5%$UK-&`Wf1x1r)O*R_JO{Q)y73tuMrFs1m0Y=7=__0UWr>kYc+u!S_S@Xj=OfY<>^s zYcFf)@E~2(s-8;mxzq7p>wGAz^yBCkr|5D|ZLDyPLyx`v@zx6$v|3R|Yi0c7vGzdx z`CS=mpBnMHskx}_UI!ZG%Q%SM(3)Y_z*-oLdU0WtXR{ItuI`6($4W%cCV}U4Oryab z1F?&ZIi7i=MkoJkfP)u&xu^;0s;3_Q$SokZjW$@V(gM#u`f>Ru1s*l?HBEXji8`l- z@G8j0$^Guon^lJV?{^-=PfEw*iI3=l&OTW3VF)LF>kv=Kd?fpy+Y6dE*0FbYf8l0x zf8G;TOCR=Tc)DCSu`tpGLeYp7X>V65IbL&Qo z9fT1(?Xd7=AgjHOc|Iyv1@EpNeXt6j`4*H|#$dzm~u?yJEn;B9B7m_oNG+7hz(XGd}(B zg|=8uG}$q3l`ug+?7wmU*1t zie)Qrz_~+t-kbY};HQMa@P4fX{00;AJ6wzd_>UqyZ59(2^dU8-krN5x8^h zNis>#rWgL}NOj3I(SB?ybbA}ki#NT6Fa6Xou{a80rakncg1V{YV12R&&U$x3&rkMIY%_ z(-*%5+xzQ<@lz{cMPDB*)UTrB2X#21`U)i9$-*;x%PB`E2et+!p!JwL@UHX`Wqxqs zWRK_cwWysI8>!-qGkQ2_`*tav-xPu))$o|@9noHe~v*kOkVYQdbRPzDx-0Xj#)aHbEYo;a z1#^9O4;(uu3A2jYMLP{Ea*fSn$IH`rp3@^yKE4B{EOEwX3L~-TQ-`qV)rvar>}*kQ z${#p*QTAI7w8QWhzW73KFPy5b@%GCK!JfMn(D!jRyL#=ZJ3X?PF8upVIrF+=L)Bos z;hDy^M%tXUxE>rYz7g&&?u*-}mVo$s1;(M^O7Ulj{wl_;-Z~E)f)H_HrFOJfH~wEb4>f zCVquB&w$$P3q3fGt@)Ua8^)DuqS_;Eynf9NuNR`AUA^ua%eWV(}x_ zs}IHg6#@Ju`KyQ{+9_4pkGq|9!(lzJ2!V#seF`Y$sKl`?x`@!`=}Vr?mQduqaKlE2YW zS&zE)izAPBn!xv}%OES`2-&vjqun}1ers>ST9Fdm_4ft*+gS#GrV|(WcBLGR`1_RQc60Y1McdITx*vje$i?Y*ZQv%WiBF|%OPlT>MlL|`3gW? z6W{qeh_9`Ovw_+u?m7M`Y1g<>!~Gig9_z?8YN9SRwWg8OiAX2Vx{129gP>ImS7!V%10-t3+8Ft$Uf)3U|ung z6&I=Sr`Lz*_XI2G;pZyTIJ)D6`97R%ktNRRDu*Mdxe8O}UlfL>9;8$Sdo&nSCF*Di zJlH3Iy!lI~nyZ~~Y_&Ezevf8j z$3OIQo&(R-YlDy?I~;lQyO49~oj5CgAV+4@(#Ve7x=SD5f?C)GxE*tkbQCA?f3yG4 zo|VPWJVjHijIzXYy|i&e-xT3Uql4ftUZ5&RIpMKpE>teAqm42%zCrFq(cAoNzwh!g zE#=e=Fgw$d%c7FV@K`)9&^N=nG1ovX`xrf#m@n>E=z)I6B>1s8PfTzzgo30qCgqlua$uyl+KD*ouoPEQv@pEs4zG&-73E{Ws%13Kto3pi5dnT~38 zA^m|}>Cu61G&9ndzx?u~!k&9Xbs9%ygOxdYVJ$6?;nBYfPKbu7mUwO32KaT{5eAKk z;VFv8X>#mq;ni?WJdiVhR!5ld$Sd|}_A&6JY^tlI(qU9`aY!*jsHa2cF=90h$(SRfaNgVZjYGE$0RmUSvyj&^}_ zGC%9?q8*U*ES5Lf{-aMO196I@KA!md0LDkkbUv5wiQchl;@D3+;Qd*1jEwjSJ9+{< z^6iBCKCd8aPA3eR(~Wz?k7gsUhr-?uV{l&ba+ucfpExhKm>fSWM>roSth$}ehwnb7 zUl()G^3NjVDY01g<5pcxpoL67`B(hecRtT>-%CSe+@|^ArEv86VWGe<%iBA!4+rIL zAlrRQP%UXa&Hk~Ra|b2U!;?Q{eyjb|`o$KlUTp;LRZ*Zi_Y~B{U7>yj`8>W~JSXXy zLA&P+EbZJR&a{t#Eob}jp3tRi_i2}4;uwj+r4jhA?KSPL-au2%2lKb}&*4c_CU(f4 zZRFY%68rZc&jJ;!&CiDT8~35JPpRO3#G1Bja>VQ@rtDRuS+^@~E*>>L3P&p^;jY;o z^f5^$uokm~2@CYlb@&A+)vBbfalOb+YJrdQ^sw=#GD`+ef{qpLWN$N=tMcaZCYNN+ zy}uIVS5M$rvm&9gv>AL$H6dPm0GdW2u6?OQ>HigAlG+K;?*1H3IroGFZyQb%vdJ;e z8xNi?0^@7p+&bnLx%b`KvVW(;ulHvf^!+ zu}O`K<6OAsk$rWSQ%B&#n|FnAt4`68ihr=S;|U$m%i;q&|H1JBRT>f2jk}t+Qy0a@ zb;qXOgHKB`VE%DKn6hpY4K0b4;Z9>P?)D`(Fv$RSpLM`v&55+v+mmD~w_s1i7_KVM z2l^_9Su=CQLv_c=U_k&6Z#YK^&GvZhi3Us6bg4-YglvzVtk)4ipToSU&!>@i`S)wM zY@SG}ZGG5YA(70VcTjZoX5qoahtMAP2hyhG;ErErG(x5YSo|+Xh(08bXrYWfg-G1< za<&jPrhw1jNLfCVA@9kq6>2+%aD~h@E3bN4w!GY*eKd2~9)_~QMF$w=E8|Xd()VN=*(Tw7I zaKcW34>oUyZIdVRtCKqT;)Euv7Y^a6292<4p9RL}7s$9ifk!_J=Ji6I;I+Re{xpPo;*}ihU%tQCSk->KD@GH4hv9dYDU%V@_kc+HA?q z_g0cs23I6zbqYuB(#9eflMJs#5Q7a<+e?;&lsYpOT~X2fQ~)AAXwA zuw2!*p@^3Ix?cKN?|E9O@xfPv{<|`dNngi3^tGI-2Su1UtvER$^Aens*zz)Ye)DNT~~e7!WiBuiRft|oiV;nG66xxVk~ zRHT}Bo;7GYsn##+7~dcnUfSUHb-y(3N3L%|_%VrG-`mm&AEG3$jH0A}*F;L3PlQNq zR~&87jZTz)R=Mc=uVJ%v*|D9{X?EXy&7)H!5$DaMamnL-Bb#Nr^?ip)mY9S#%nHq>emkBZ zt(n#+aq6{B%7G>D+h0p^tGZRX*}O#hH?+ccQN=3BiWC!Rsz+7DIv&J1Xh z4(>joAzzk(xIb~XZ`U7NB@6$1BF(sF(HMTH7{^U{BE33!jYJ`Cp=^(KaJ^k&r*H7I z1nCXox>PYzSL)aLLh`%!bzj@v_k3IYwWWg!)f*!}uacbeP?okDx=5yL*f%_I?jyPA zY$`d`y^pkNkDP25bajJ!eu;GK`V`5ARAWiM(gew%WN*o~juJ`Xwne_7wjI95L$ZB8 zZ_1Ts@?B~0?7h;w#vKj*R!Ge0zBJ{>8!{@6&ew9-Mc{^h2I)1T!fRyWn8SDd>` z`d#kU*z=U5k7T2_#67i#G~V0Cr+>d0(ntRyq!Aw@rG*>+`PBU>m$pR9*Z-MnB3=4u zP{W6lBc)qaF8JDnrbsoX%1PFy8uY)WvQ@HpgeAm@hMi9s1CnWd=16j$oR&x6o29nD{3H{%mr6IJ zuahWNY;9OC8cVNztM~1ysV1?Fmq=_+cuH3MH%v10k-X%@h;!23tsl9`$g1Jl?X!}! z`O73fAN7}%&aCt;^sACot@o3>5ORIDHlC8sd2m5;R`-I0Kdhzty*?}u1?}EPgY6qTdNY9 zB$3fH?!C{s4M|dzkrXYX(4x`~Nn~%?A)%1$>-(H*q^ydhMM)`AM)Maczx(g~>%P9@ z`JVGRpYwUcqQm9Pu;vnS-L44+k2b&v6G(p=%HgbSYT(^qg!}vjEL*)WcBg&?p7-Wc zx7`(--zq9_ynHFnTQ10VKh;9&M?RB3)92F%=aQ&{%N;ygtPgd+{UD_NCwX<*k_g53 zaa?bmqaISt?81d#=#R&L1$&`au}avGn|dJ^k0!0h=m2dfcCw&zCpvI-lM|`byu|c$ zZ>0M()`H!^V2}~LNH;hQ6R@>}mraA@=)e?y_6A{Y?2A0q)T{!>Ic@k*SD1glpHJ+U zTaz8VA&~cejNJOOlRjk53OI&(aLVWrm2V(i-U@k~Gz|1asTqjPU5_^{1Gq*CJ0a10 zDull51gT6zw4Eo5>W>z{70DlPXY~T$u1=&H`rc&sR|ei*eMpy!UB+1_Y;nzCBc!ml zL)H~v+~d6jV(S(I_qZAEJ0=TrnO87;fYLbsKz#KhoxW0>&hHHlhdYmwVYH%!F#~z* z`oHXACaybIiYGioQEY<}xU0ROGUm%UO_Orq z*1`WEA^$F0F113yE-Dse;3h!zns4O9e_>3u`6s%&>^Gcy5QY*{2;4Y%i}U#7VQh?9 zh?l4rT<=z4TmS|Qn?G7E( z@yBG0)7RkEccsCcx|y)z#~loKRD#=exkNfI3s0Sqfv4WfAfi_h4Ghkc=QoqVQLY*- ze=kOv1_ihra)P{`JOOVL;_y;m5z4G89gk0u9iE#(Rw4pd{&7Lo3P!+Hcno^Qs$j3u1FQSop^up1 zi#K9erk@4pO}*&m!@giCx12tle*-MPuBC4kf6_k3qk??l3j(Ige;nR&AL!Q7KvN4j z{%?muu(18b>_G;gp^V)9t^{!-L#&V|FfXUWsC^AxNI&Cy4>7VyQUYZ6E~f`)*uaKa`Q*$lVO(st8oXa2duNj~c`<51 z;`+9e)WIca{lydKOLo%TIXBTSKbs>%IZ*UV3|H(t&pxzxE1b2uOZpS?==k58cu_GF zVvTy49~b&y?Z9(fBDIEQ>7KxZ59;W_8b@LmLZqh6;6F3pNQAlXsbXO(K1rOyc)cn_$r-ZtE?Phym*%1lQ-pzMepBJfAB?*V3-8M~Gpd0))OC3=^e-j+ zy+T*$xUxIzfsG1LKC4X{dQ|C$cejZs?*!iSN=*J0Z+j9=Hd9o zMU3)=?$ckUR@7npWH_*UfOD_T0=(rVVfoUPpjBl-ugz^Ik{aC{Go^I8U2+p_Xr50O z;55GOPzUGf$p_%)?h2OjLiEMvPPp?-jX#+#gB{~MpuS=7??5`;D`N+rKJ2DN`+aDV zY$J^}T}kFVl?J;tFX*v1>%i;pLp(lk7$^T2Cw~Mi@0|T-a7pJ|XyZMBtq;~ixw$lX zTkpitzmWhLtFzI{-URu1=W+X69Z1-h1hGCceDWuauq|9MH1sa>>*O4M%!XC`l0}mA zg_H`txlSVg(sVtG(CSZ={dBf$O@ie+$2m3t&&{6#$t(JMjXi0phE`t#` zJ5P>WTcwZxmgJ#R_X_GY=@zk1afd;v8X7a2jfqZo$eq*IVCmXxoE0}eb8dP(;#hYc z!kgz`!R+3pB*8k0gqS5z1M^ODO8Fj%x4uiKeZ#ai-cmzlGx`OF~Kg8YEOfcnm zLmTs3@q2(Jtd&-$bAx@b?J*^e@V zmFHCAoxQ)=`K=QWecg(7{_q$4PZu)a!>78U`H*#NHL=-t5S?r)=-99h-!A?Y=^d^i z?z3!YW1bpV2w#M5P82K%SdNihyP(BJ8LuCo#J{BFhka5yuy#%_XYPUws=p`_@>@M= zR*WV+Fnt*9M1F&C*Ft*!vlO=27Srn9bSO9$&eb)W&N1jlp*-==gzKw5iDuFC~kVEyFD~Z_0YSdf$kvdPjgu>)t-21$TY!K(eoP#q- zS6~IKwA4Wvxi^rOBVf~A&LZLB&mnR}6mEI?5$D$VfXz?_-Da2!r{W9oOo|p~WPB>%IiA5r&?!0RTFjP?gnNLeNfvmGzu zf=%te&)m0hSg-x3ru=fO!&HuzZzac$ou!u&tM=<(DG`^BaA*^I24QRM2+SB&)OCGhlOH;HoXBQ0&GKyA4u-sv-< zc74j6&X3=zZLJON&8xyzt2B5oaz$`XT*G9c9W?V4t_59iVQj_`Q+L@hrH82Jg*JO|*#{^6v zg9+HU=@~fd$J5Q4hK$He4gBzoXF{CsJOr{c$G;9vq8 zg!<7y%@KOXKaCO4L7?78g~Y)YvgF7V{xaSDC_O)rNJAxTIU%TbG?g*gp#bEnYl)0^ z44mzp!rxc%nKZUlGojZuV9G{KV7ES{n;u`qucy@^`0!3ta_OPlsx0vRoDE!C@x^fT zxGMZc4f11!HT7yzX8f92q-l35onkMI|Cu<$k@QoT`-*{IH!JC+Zx>+m+GN~Xq76n} z)$o?<1D}_3L3zPh{8Q9`a&r?=e$^>7Nv|X+lX-Mu<^AAzvWa0RC z*kzUr+Bn4N-R6t3Kk~?N<;zTi&;nRekxylB#=+gh-6-nOj2kjqpfI+SJnwVIh0&$- zcULQ8@>7`#Ev*DbO}-dmWv6o@*4cnl z+XIpx-AAIm_E!B|<;=`W^?~)?YB)_cl)N>(1ey(N&}Oh0q^J0jDu+?H;9t*q8)iif zFZscBy$ulKbp%WjV_>GX7(}lqB*F%Xuy?u*WMqb-_`VmQ6W&Muyq6(e(Y&K|mB0Ih=-uw*{JXu7);PQ)_fI!s>9(EtR@@ct%pC;@sb|#am>I6t zA0fGM+v%v15P3J;gGsOQaAoERDn3_%TRQbM_~|Y}j)2c~*+mFfGh;+4Ii)gMrjZIw zu(5G^D4BXZ7j>_gz##WEOn*E`eG|ptMsOHjPWGk7`UB;gi`R5LJb>JOcG9bs#>aAh(USDE%$xI?2Lql=JnN~RF za)~~fl1(fB3q!G$+W4|%6q}U&@ot6^UoCkmWFO0fECD7hvBjEYZrHR%9RIu=CryWPAb3q9UKNigE2Zk0iSO&F45tHQy3BAv`%eW&|{Zjw~HarT*847FS33EI0=kyATD7TeboE9X?s<&jSESigb13jbau z2sK4p;Xqn;elj`?1>xBvL3EZ?Guv@k4B~r@@yM$<_@>oKE;&zSET`Y3`V|T^cufVk zJC0NHrVm7+>n2HBaUEI$7`S|P6a2EYfZ;dxDC^lw6!wRq`|c&EdOZMt;xSHVWeM|q zzCg3`c#O-F3^_p(N9jBPf;nf>3N%a>a7mL=z%5&taZXO7H%F&Z=bP%#@-kAO-Sj|C z`D?rqRE2eg`M4$OC8ubu5F9d!f>Z-FOqdX~Od6ZvLz*Uc-M8iJuI9hwm**{P@ExZ6 zyUj>1_Z6KVBS+4qDMPGabA4o!5WOaGkCE({gfC9nf!NqgXkOq3=7$oPb%}~_A+M10 zKvEmN3)s9z2j9>^xvQXdIuf^)&lPZ~9+0R^Bdid8Nk-!t%%|G0DXM__z$$dt{HN|8y|0q=S_M?Nm1djRg;PLSXupn;* zHNMsd!L9O;b|D1dcPa5)E{ls-3UgGv*Mnnu6zHGOrx)aNU`;|fih10xlGPVRYi=m+ zdsBt4T1RM^>jOC1aS%@^Q!GCkj;mf@Cej%Xpljbz8vI2ZEz>QS#{&KG<+6CR^4fr_ z_zcICn~!G~2w|&AEG({bz{mxs)lDS0!1 zN%NBpggF#SstwMP-PSLO>ZWM;lo^2k=FQ+vE))!mBfl{=4>@2`+)mqfcCfRKDpKBI z9Sn%jhh@zr;4CQyl0U9-p6|$qgr{B9_-7-tVc}C!nHK^loqgy_^)Y&^VIz7!Xr{X2 z_Ox{20qicgPxLN{AzMj;6n78PV_B`_{NN4F$k=&W=WPsIc#XtdE{WO}+2YSD_c&<} zrQzP1D5CK=fPJOlDugLJ!{xnCV4lcy2xKM03T0tD8*u~TXJz5dF##)Tq5@q<{b4n= z#Gj>#pmFan^X>RaYBZ$=QpUExqS2k;@14pl+bhn`5}&~dcV`jL`$~KX0XO}0UQT}kf z2^vehz_F-YC}GmTqCA1l>R*93_OB-vH`kIs0k97gi-N&HyzgkC7T1l12E$l&8KT+=v>-&mgv~Wn~#f}UgAJe z2CRE7MBl7C4-1~}LViRJE7IbA$SOg26a;MoZFc$DG1?I% z;A>X-v(Bx|CEJ}RC`-VooTT7QFKBv`#X=EiYof@n;eLUQT8~M2bvI{!eHcW186;Wx z-#~Ac2qp{^kh8%Taf@FnCiFg`mOFL8I3bKmn+9Xi>m2ldJIE<7q8M^;0XH^PFsJGE zhn1v9Wmq@?a^aUUMq<@@*OZ9)(zHNiN{P ze~Ioy`K~ox3YB_~uG(|2Bg1Tv7T+ zHiQODYk~rG6_i=v3ct(zap2w|{Hak*7sTxWBTYw|w*MBsTX36ESz_JC2o#xfV@35 zB>zSm3E1}r=ZqTD#bdEhFM5R$@^yoEfAmo@?k8>a$)&;DR5A3vD*8tBVxwpVo{I?p zG`E34R@FfAOnuDv0)c9BWi(zWycD}oO9S!Oe;;*zk%Yr0>SrHE{M&i*Ls5cp9_^^AlG+UzYXJ2|GpMP~EG!UyhK(aV zRDaqFnB9DmO2Ru}_MasiMN1*qNtA!Tl?5H#H%#5>7>Ju#N@AAOlj)z5aMuQgEL*J3 zufFvh!e#{EsGAn&`Ormpqu=gK#H5 z0zK@Z^*XK0=FltnS5KWh=s8VJJaNZ-a}C^A=YrSQ>cGx*Q{kJK3=VZ?z+1NvPL0@C zxRGlP)%%x{u;O|elb#GQnd2n=TMo`J_rMjgey~pPgT^6oPG#>EG}pdLKJ9sfiPMkZg>$p{lSaHj{_`i&u384Ng8RPSw42zy+JXwg zL$o&HDuh?MpxLcexXdPl%5Oc4>GvhMTizB^M?2(%Jr!{9-WOxQcPG}(otcXq1dG0nk}nfcRPppvvbL-d zrT65b*zs<%vH2GEZmMB=dL-e8Cm#&Ngt;*%w!qT`&Gf6uGGtvSrUClFSov`nf`(4w z(`&|Xv8t5ryB$N!Gc57X%t~;4wUWE)cm!}y|Ki98&%~~%UvzZyDe%_w!b^+_9C&3* zpEe?VH_TL9tL+7gSlm3Vef zF|HRngR3<>G0H<(P}57G-r*0NQiXm_;zk)vyS)g9qB5a+_Za>%e#fY}{UVC}m2}mC zU1Uk{aS~~99WA!oW5XIh-1RjFCyD+;`wuIicd!lm|CCWfrzhavDg^O6BQV$CK1cIj z8&yFs2w(b_^Xh3h?D+bczBd=l3T_CYRpc=Ch#y9Kg*~{qq=_t<8bvRdt>CT|nGe61 zOZfTLeF7tAIAS*10#N3$C-7h~kBsXe=(wq-nKd#y}9#wjjD9u$boG5%fa6t5MA^h}zrEBR!h2WYyQZ1U(-! z(Q0aVaIg_oJSKy9bR*fcQTOjw76R0es3v~ zA)*STs0zd1_25C7>3AzkJ+6$hN)5+_BAIw;}I|e+tLvqU+$kfhZ9Ik&!Yu^PE&zrOPizcapMsqru z5x_=8rBu4(#RkZ|EXJ2)`O#H<4X|w_1IMeSxsG{OxGvHH?!9>kzkB4d(|#XW@!}fk zyBW{0GN*EcD;Dxs7MS6jjr%G8@KO+VEd`~~K$!bllf17H%)FHCNSc}x+NkC*s~!&G zxht)3M1n71*v^8e4d=mDF%4_9Lg4l`XXsY&feeiSx-aV}x|>Pj@-b&pF;T~y?)t$Q z=?Fo6QJ79<^*&_Tr_hM{ma>0OcCGJ?`Dh$3fhaWP@0C+F0$%}dMblhkm`4F#& zLm_J5ITp_w{N)Z8%cNmLdjmCd(4trD4}pGi6~487LC+0o;(?PTAmSUrnE#s2xhyNo z4@ujNCA$nUklO|-tTfp4m`4sd$x^BEaWeSa2mJQ|IqDifq0Jw?oxX6AB$1BSJphBp zf5?#s>rrcC7ioR)7GsB5SfCIEdv@gDwmI|ZrCkzm_nqLL6_(OlCFSg=XTM@c<`?p= zDwh0Qkj7bZtC805_=Fjfgok4h;1JqQwF2$Ipr{jaS0Md0Ii6!Zc`ZI-1fGZVr{vey zRs2`s57GV2Nvt{1PV4`O!LtkFWX?T9sy=Oyp!r))cwQ|ODx3qKq7Y2WzDySEosKu0 z7_3~_1gq=WP#pXeN)(mBiq6CL{SBC)+r=zQC?xs*ujtHw?l?1`lc+!52^Ei3(83N0 z=UO6gCL%zkvy2`v%)qVhxUeW=8qQCw$MhZn=Uv$g%PqticmF=}X!~h&u(%58DGi+I z+;h|?<`b^a-45RuC~&3ThhT5VA*42uPF1JOGkDqiD!MewbYMi`^-->sEuh0Pb;58Tgi>qK~a1Jv+Ash|n zexqvlq9G@J6`DBMQBTD%vP0oFC>`~~qf*OY)5J55(zHDw{QNY>E#^Cl#XlnfmQK{> zq!S#9_GJz@UxzH!GAb~|!roPT$cNzqWThOSa@_6YVAglC)3^xoyAxpb^c<4-Ed~o@ zHV}p8k5FN}48=1G!T&!c!v7}?!bY)d7b7lh8OUKOespjihFXF0zgT!TB_11;Zo@N) ze1MZrs%9_ogo`2a_*0}Ej!b(7?IklI>TNu`w6=_P6tGeLhdsRL*iCAJ#Ho|h3FO|a z#vi}y(AnY-)pir6-HD!%>QoBLw%sAmxA_tw^+KX!8-e?E_+)6W7*!je!Pj{xj>hYX zaj1=rZdZ5H(}Q)4&0G~&OlF`0hffR-Uc;x^8T3}yBaWrK3#69!wd{r$SH$KW=0)qLFqGX;s1FW}If+d(z><0bjgc zwwU-8ZNXbtTj~AqmHbhe17wlfPjal-g50vQL4n>%?sK+){eT#ndAJe}ts7M*y>D|q zk`UBct{ZvOw zzsmE=Ounu+pNal)1GK`m$*Y!Wlr55i(&rNB%TJ%M{!9n^3P!w*k3&l2a4~*W5K&)YCFpDzt=;LU6>5fKTkv6qw`R6A{Ca! zJ)x`@@8Feu0@#fNQ21^sFk!hTik$U8!(#a)G;{Rvpun8ms5_nPSsokd?h zIwW{TK49B7WAvIi1;6GBJUnm@e#IA%X$eMfMAVD${K{uMdFQT_4D`P9h7hDC77F8?s$G zpKhtDL=$1eGlFL%uv!oeHhq$yCh5dhky_N+HDB;tiGuJGDHOZ%k~8~KGF&wf5pU*QiQUL<3l@*aHaEzgOmj7OUnJ(U>_i8S))J}Nu94!O!#aE|x} z`q%m^IeP3Y84;Pnx4I&w|EY_O&goORwlXI8$@exG9bkyU>t1X)WJ}9UW5H~F3~Bj0 zz?_4kR9y1K`6(sf(6kxGJRLyRAOp%AH`8b4$;^}3S&+OojxBTZ9kF{@2#2OThaSO_ z#RaeFnG+qFe#V`Pl8woig*q48w;4kxg=cp5dg4AFjEN-tZh;)y&5 z`qH$AvtA(x?~Li-*aQnc=mb!+@srr&IvF9S0=jlqVs_DHlo^o6%^XqgqS+xJ`{4n; zTDciDx@zIWw|_)&+gE61Ps2aoMraOO8En?8knI+=SgrdH3p*nr=$#HO|NR+d+Vr91 z%_q>#RlVDLO_FTP6dv}7Xc_g&njD!HQWw0wi z3Q8MOAWv16sXK`9psNr9pS%Q}0#jNg|Cs!nFNCQQ#dvL@1V85EWd2P-diAQ>%Q%r@ zKw=lCV!QKvL7moss>x@-wLy#~?~B4Cqv252kw#oXV&KS)52RhcoMd<~bW*xMYP%i* zMGN4poV5UV&E5!C+tP?O`yO%co5cM6H40Z_rD4;BYW(b&0yp&QY1r@cP#=>{j(-`V z1u}|wT1YUbt-Fe@0&h-6{#5=O#u0~K+JM@wm0YQ_WBHAhjudYRhx+D*30-BZ1J;&D*xC)YkcArjL+^kfNsWL;9CEOGd?HL zc@l?dFK0UHJS;|0OvIUQ9WkwU9=uwkfmvs#V#u&Cr#LVRqq8qVh1!*>SLK|9!xf6>2)zA!n2I}1D@Qz`($ zhIR4%)=F%$FlNH8&%=!VjW}L#9^~}eK>z3j7U}l0EuAjWU5~cHW|7;RWm*}qv6a(>+gNe zzvFILc1@P#sQ+e;4~g<$?>|Q73qcrFS3s(`9RF7VOzEZJ3P54>Odq2byU zTrl8`$+Psp%gPBHN7Qj^AU*X~w6X=x^0pSK&QnC08T{zR%AeI9u$})CcLC6G(EB@tvS1_7n4kjwgpf?OzmW-}eR1FYyO^y~zS! z!dq(kI}$EwbI@z@Y3Qg$W?*?9%8bh3gM+2CEpv!dzkfA-?;%InR~5J=0uS2mUlLd; ztbLDg0TGowONLAKrjXg1M4*SO$suegmuJH9>b{ zFfM=f6}-r0+3T1qRN^VsV)^5#zG#^1R7am^jB%t_13s7}&0U=3#+m=w z50r%4F~Vw?WG97z12Y%orHYAZpbD9xu?($STo*3Fd;S^p z>107a|6`0J6jxcL_+bv2^6WUtt8QbB%?l**q=U$qt{1F|WYH>fH*o)C0lQ@7(O!23 zEGRyQIVtaOb?P-5@FxX6CTl@@Hbd&17*LwIg_CMKldsxyon-gS;V<8EfyyrOr%zkH zk$ca=N$Od_9B!it7-b1&al4ebS90vIC2c)Qe;6PqczO__B252HOoMHoeBnq6m!d}r zSdP44a&Ld6ZfhjCp0zo&<@jl6%6GM|1H%^FT1#atkufd*UNy9i%w#^wyZs3(j#yD#_8P2H2%&H*x?ytbVJm!x2?+uFBsXsM)&%cfPHMI-dbF z{;onQMQ35}yKn5h(!VgfJD4iGZsM$txx?t6lj8jIm~QIe1gkdLa5$=4XpmU8Q%11a;=gN;ro1RvdHHz^@%T_TlK?;p7C$sz7(gy z?KzM*kMgF_5#c2J_S-_)O;%Xc_Iss|JpkxbrvXnUn^Yp3z6=tFc%n zatL4V2}XIvb*LE{00oOm8L>u39MVJj%x4B}?9oQoQx4Gm_ZKWZBm$zQ&RG8~gi|G< zO|n$((?gcK;7Q;&@J$t5lb2Iq;kp>QylpFdPhCSjE03bu>YeZb)?s|FDoCdZb2p@j z(q@wfU}hEkrvUpZ2^X~L}lDO(R8L_GZ%U8jqRNI=Q9G*;Sb}4W=fA!NnZUQ|Z z%N7+E?jeqCgLq71Ilew!4>1jU;Y)5SxRnaBo$l#@OS?bZ_9H+p$nLS{0 zLl0&5R>3iwnQ*{R@Qt^vCu(D_9V&yI%&@7I8N zMHCLHxWN5f7wEs*22M%&nE%OznQXQK|Lf`}?K$ZfWB82YaU~t+#cP70rD~P$kO@V|E6TixuFZwmS#@w!x{N@+ zkN|u$JOlF+2>f*+JwpTHIz!Jby9FpO=B9`jS6!xa3L0g)ixeu5_c zo`!QYP0&+D4o|-0agO}$pqfqx;lapzJmQycbhLablkf1AI<~aK6k$)a75jpQhP!Bc z+ANyUA_6|k1RlR<6`1!imZ-bgSBaGwz}2f4u}5qjytlb1I0JGy&ILvH$rzuVfx?6R&u;z5rnSz4ZSmap=w+nyF?hWU*$So6j4izYOj*x2V}u! zpF5T--Xxo}4`E4^DJ(bv%&jg{UY5Zyb7(?*4nFJ|rHk@! z2=wS{(Bdu7h4$ot#@emSfx*kLjF^&h38=>@PR5uONTgCi{7W}@blLk&< z!=&BQ>F2ouU(VevINhxQ5;ms6(Q}q`-3Bonzb)`O*84+)@(=ph&<%X^l<1a}O8n4e zjz0yv0v`v3vB<@jtX~j^Q3@LPhA+bP-*(AmpIN4W!(c!OCvCb|HxYt5J)a)Br>Lf$?B1LY=-v3D7Oa-#1 zDH4swbE}5uUcs71bF_zk`13vpcdL(3;gmIa!Zi~g91TW(^e(KbZl~Mx=aU?qkKfed zaa1x9Yp<8$ql+JbyFQhs%Z1W92per^9sH_FfF{y{-+ z_GVNg6uQ3n&y!&VrOA9~8>tLqr zA5?|AsG?Ixa?`ZADUTj8s$EO?b8_F&^|wWE_dE`q)ic4$1B2w>@kRJ{&sivRO2^G> z#Yk%NCE`4+Kz-*VFi*KTFw4-F*7eN7o=6*LP}1Z+P#R*-vrZ*ne1>3i^JXINE(5#L z?~$tOufRCihK!xLj*g9g=^=-?ReKZHKxUUc4zMSKUC1cOYdTJcGvA|Tmnjph=g_ z4x)PRWguci5-(1Qf?I~WK(lxejxSchsm3hqy&r*z9;J*>F(sXar964@No2~_dyMJ6 zS=j!r4AzOKqkY^n*!giOHPM!br=u6K;7A!Ii}&C`7h71l`Xkx!*BnBfSr9)d3fDLF zpO=$T+=@yles$+zWLSTl#&XYmm(X-(ocM#>U?b~eoYz=N`#`J8Rb?_xx46zZz8 zAi{^jbM+`p){3O560cxf<~eia{%z_bGX?m+r*O4aH;~fjQ*eW?HHLB2@D!^K& zi&sIAaz@?=c6SzhPDg%rJQ{c2hKPU*7%Z4!mMnNo*YK~Bb-Tkr^_3UQ`Ll@ILhh5* zrfrn*x(gZnTj&<93}yZfAR;yeYuDf7$vQ;RsXC(M`sRaB*HueS9^A|%3iNqeJDbX; zEg}btOrce62Tm{5;_k5TB`(TopzA%2f9T&LxMp@6|I<80zL!{{2lXJDyG~MJ%SL+b zS}mSE5eMrMi*bhDOLAxpz>9AgklR{AG?Fqf^Ve3ie7y+=F65!_#3@?XatM?wis@^f zFwAI~%n@tuOpUZO{pvldC$hEN_D6?>Bh|Mj%T;qu&YXp z=aZ}Emx!gL9o?oD038txX#OY$r%iFh2jMBiannxFsuANZT)LBF57>g<9$z@DXH{k6 z_!tI^?^2P(Y6hp@M!(0}Zw2239WP zw^f`yms07_|Jo?qXVl1HeyY*!hiOI*r*v7FI>&h{M^EwscUkg$cgpdqgNrLYIhF#3 z-tkI*lM!BOo*gem_LGs_cr@#V_>fU|>nWC_ratT3>$$A8WAUs#hSynzwMwj-llP3K zEq%;N{8&}qVxGjitEk7bSr*OP6lg@4q9g+|n3W`I$&W{}M%ENoG+ZlY{eHfT<(PPx z6}Y9NvZyqQ)p=2!W!7??^)O{COEp%3RU_cSPkwycXqoD;k(H|zD%528N_g9im+>yh zUgtH|_4BUYjNpaY6;?Ve9OSiD?J(M0zLEFlft=ADy^wOY!EU3QtEE`&+a`>H-9svz z;@`0>hFw^nLtj^hC#+{hACYHyrk!SedO2uRVc^RzcYN{?o7JT_0@Z*=@VR zo7kaRxk0>@C-mBk=d3=16*?|&xbpa$O5L(zR`=fPhA~YFyr})QtV7XDS^SAI-tizQ zUaN{?<&diX#qZm}^8b%jdGWO^ z%UmvjXQSU%F*(kU*Z1i;@0-2|d*NJh-jlC>JilFMSeFjTvdrF%uv`tdu=bX9vL>a> z;AtyAW$Cs}X3fof&N}^PuuQ9P3-6KjC7x|{GjFsgk@vZww{nwS3+sO7bEBuu6NXAL zs=S@w!;Q2S1oIR$x{RbIpW$T;Y~)EVUe3$-`hcZhvydmz>dU(_9L`dH{DyZOw)z-fkA7wSm=Nq{S+fk7K%-aOy#}Ka;RF%%X`km`sgMhP1Q#hLzc?{AyW=>M|9RS59J^?3FSc7jxmImcL`2cd@H@vP6k} zXGD}8bC^H;J9|69wR_&veeuHRS>q@R< zseaqUR-HVBZ60{3B3^A1`$&)>@5AIQqyHJY?zkG?DBfOLDwVWIh(bvBId2h#24$5< zAu}VH{p`{ZO)Zs5CEAnfKIfHaQX(Uwfl|>>+7$in|M!0Gd+&YU^PKNF-;v_M;IRs6 z=2N~bQ*8APR@uek*|1#rUhxwoTJxd)odQY)Nio0GoY_Us8*u0KVt5`c!`A6v=h&9K z;XEp-<6K#m4gGCStjiBxO|#tqXZ{JXdS;T0je{}wjAaWEW0Ihu`wP6}%K)c{Bz&&4 z20NBovXigG*};E@$kgLTY~QYBP_E|*#UHbA+SLS}A6g-OrEEdZ7ABJ}OAqqeQHvFB zYvLBjtcKqKa_rju9=P+2E4Iyz#M#^5QpxwHS@Ren7WlV9&zT*p(+g`h&tM80;PDq! z8rQNP-Is9gn0Aq6JU>hAO$wkKy_i{OGlePG;!TN75}basjaMVh;2{%*^lID)ngWSQ8aUm-3YYTh zLS$7bHOZQVL?;2B#5|y>4eQZueiSIIj(|HU71(O*17|jA;ndY{8Sq&>BbUWRI0w>xP(K$5a=~*U`qMLAWw z{&b3yKlj{%dvwPoYrJtmgY0nh<$QJ$rrHO^usJgYSNmLt+@^F=T0BBsIwuxS%U1zEw-zit12NL|E7`r%5%$A+^&ss`tD(n)J#ZxvX=LGYoTxYJm9P7r31C!sG+JHnEv&H6Lc||cDxSa4*TIR zv1=$^>U!igxuh)(Q+of>AKC>p`cM{lj7L#h*{&k5 z^s97VacF~FVI~8~ z?Gbe5)UD7V7mwP!T=p9Oy;wZ&D%Pr-(-*fZ!TaX|uzT!+y^ zaYj)|SRrQljKC^Z2;%LdQEKiiM$pOwCU~_)amq6Af0_YvG=-`E9BopqV1sw2Rpan$ zE1LX5iRdpl2t?Worn#mY+L`jh@tN!4y`~BLuJ$dmkjOIJTfk7cMdgNCv3%62IGOC& z7>`lEEYMnHf05CkKE^IjM#uWIc690ynWlI2zQYwQ8r z9f^MPE1Vf!gvAx>QGD)l{JCu#dVjIS6!kDVeW^IdNI{xZ>Zn6s(M|YjD2+BV&XAX} zm&l;%JgSzz6Q@k)c>yveAX~VO#{Ij4Vab<~F)2iogc%^He2=!CG$7GGS3ua{QJ8bm z56%Vb0Oz}lP)tY#ZI+7^_5b|MRex7cQf$m%<{l3WjgjC~p1wpmz7ETWEk3e z@%+-Wh0)V4kM=4!)6y5ZxXVr$zZ@H-r^}K+t^WWJ-r2cl`#XGI^%3Q#^pGcBLLl#U zi!>E9(rsn^^z`-?h>DM=`vpp9Nc?j8%&P=DEyrQRTAF!X(GN8mN#Oc!4VmwC1AnHa zW7#8FSg`gw1fTdz9zHOG#HjmVb!0Jw^1kax&=(?f#|+@1Dq5|Yfv58?qLk`ay1AvB z_&Eo_(3lb~5zK?(!-7<{a}Q0)UJmP~OvR|onIQKf8sBWy!cxruJR7hNH+acmx%p#` zF{cL0rFN2lWr6ev8-#QH?;mb#Z@~*8B1XGiU=C)r&*^1TF!YdNuCPQe8oF|_3+KI;w8)EMl3sC$%_ev^sX(G+re@YWIKxn#t1g^dU@`YM08f##yeNjKyM8{Fq|Qr_E-uP$G>ql zrwYMbjSaAV>@j_Lcq>=cG!@cwvZ*frQs`KhM^_b!@vv+rym*TA$Nl>-F`r8e{D09E zcX>T%ZY%cg{6lipu7T^xW~vnW9G+el$JvU;7}h4mIX%UzNGtO?`Z>8^$TulgQ0E2w zY1hSqc1z5wS4QLU$%2c-36_WI5dBD=rzW2l!dpYgQVcOXDy;xF_dkX6yY~X|tijGF z1Re=aVa%S%5dYt~`1-6j%~^H{E^uAQ3b}hUWAhLRDy;^U!R3SryN2OsUQvm@YP#|G z2#zX#;)1|0PVnk7h+CkCpB1|BS*0#_&dql)F0V=i+BGoA%ZYgH%!Z8Q*OlR)9bLQw>cZmEf$T!pwUx+T3TZv(pG0$sw z33Wg4n5*8r7G(ltv9n~H5NC7n+F*^V4TTux{!Ct^^lMIbsx+Lj=-}bccSK-I4Ek7J zgSiHxjMA7q9&MXSh5id8rKW1I;`)E|`XOG{Nyc=jds>Ps-x*+F&q~<$sfsi6+%8=5 zbQymAT10M~kfJ^>SS;R9h<*YJWc%!SM9?-1{I|=&u2to9*^ex6sC--$9pA^D7`}x1 zp4aGrPb=LXmdwp~sE8dh??~-HDAlTZNLJxqXppi%ziCU^mJ`cxDBKP7(`SNnQa_Au z7J!{U4REp{A0^@h;q7Y*7rS=DTE~?{$0iN+A9&!rD?;$(MWta`Yzbt2)W+*xw+-*f ze&)8+HsRP`KKL;A2c0V^MQsFfuq&1j4MjtS^eeQ@s*W(|(*T(!GZNjy)hhE!Lyb9CM_(U+|H`Ewpd6hOA$I z$y~M1dV^RT@yMqu-tx|ks$lNVNfmt9RzfaMUW3kG;`lJ@BP8Fl z$M&l>xMd+fDv{gNdFx+N_2D^@o%;wK<}JXG!wZlR7Uqe?g<&Yf6r|V{xX`+dtX1(Z zn)7B79u!TI;)oK=zc5K}OSIy*U(?WI;uu{R^qeFy>p0n2C!t(70uICn62-s##G!pJ zhHZAi*fxE5sghdMCmcbdq%?8)?R{{jC!9X?@J21=_v9S-7TO%ihOsZ|aPm+PSa18v zjeEEYUM?P|Uy8!8Ip7~@T4n)drsinBARoou>q+v4a2)WkqP`m@h-DBTY)X@0F1u@! z>BBd1gJlc7A{&S{HW6gGM!4aXV|sXhXzqy@;q;qE zGy1fw;Pw}lIF6DbSour?vRfi~+3T60 zrt1yliwZs>55Qr68%Sw-lQ?ZNaKHHpX9dJzb@fWzxb831o+XU;v-MyXHdebdv z6!;(P7n{NGoy{RndZe+ZVFPrI*<)jkJmr6J38g;1pnF9^vA>l;gA=i2hi?dc3Dv;h z_#b4Ifi4^~awKcB9)R$ISmL#8h`ce3hU;e{3oq}#!poaJ0V^8fAu2}*971zpx7vLC zoOy$L`fnC`>1twUt|s`jIFrk#b%8JUFf3&oA+{@zF1Eagp|ADeRJ{)@bC-t~u$H^$ z@o%tSuM8paQ=u$93vDbPle0&{XxI~eEc1N`IdiQ^edr+gyH}G&P3Iz?!8mX&8^D<( z4|s~6V|3|QCu#~KX9>?o**#Yp{tMOxmA4W&?qCGjxm}!wZTF~$RSVh;iLvTEIVc~~ zgBia1bW9|Vq$quYDK0Id?K4K3SAa3s#>dHFbAz>w*j}M}HmOt+T_q`reR{s6vz{r*X7e z13~s#ED_qi5b9DM;o>mIqU0al+-*2Q?;TnM&xD%k$srl;*~okF?vNAeyg$Ho=uZa) zt7#Cj`3X7y$O`5zo)3N_g7|0ERL1_X3J9&JfOOq#j*ra(NOUiTWfcd}@xgJN?#;v0 zmNK|ed^?@b%eQ2oq{7F4H^_+L8;}T;AT>|Z@aoT<;ISYXFCMKXqdH+^<2eBwQof2g zyxiN?+DMozNyh_o80Ptp0bcEpVr5PlF^fl7a+mRd#WnXxu#yL@)Ek5}&TP8KGaYeL zF9gpEhMUjRaevBD{4ykrS*}vpu=5sN>UV^1o6=y}%?%Ls+8>Une4u6P#F?$W`^f^T z!j`Ubr5Bz&#j?G-Y4$dL_V0No)Xy_w-=>e^DuY8L(bopr3Iy0Ht^mpbBu)F1gz2o<2-p^sfUw&0^*<~So1^A~shTQI#1$%dGwn{*eOm4f2_Dx!NTw8ut z9{-xW#;PxMTy|k2d|%p*2T%Wmw10Oo|LafO%=)2uK@5skQU&Y9pP&%au*Jqyoq9o+s)>vW&}v z-7ve)9eHMX^w_ir^leeW>xGR(WQGE+3z*KtUaI3pK8{D%zt-sKC=LAK$5@L;aTsfp zi;FhAK`SADaK3sN3PmQ-sq-miGPY3N{Z_E$K{doCoI!EB6v$TJ1SaABxJGypJ2No} z1{50LmAnkPoBqSb;arHw`3+lpRJnPvUewhcvCMlvz0NgZb>7G_^A_Ggfp1Ezkn<;) zvn7uA9wV$=IRi=#zNhmSyJCiB5cko@SD-IkQ2&HHTL{LCp_n9NKPbT-trudOb{e9^ zqr0$Yc^J;n5@(YuMHqA0CfN9lMGfg+Q24YP8`=?<7wHj6CIVu;8quDQcQ0jzgSKN9 z$NNR4)xfX_ptU<-~GG>@b9#A#J2=a|+&*jG!X9OChVuh?R3m1^K!ejFbK-j{iIk zMF~?;&&hyX^DW1D{pw8bEot`aqgq_5vJEb7&wv2=nFOkyBIyqX%UmA_ZIqxVeE3*{ z4>w`-f*VqWbk>OFEC+G=O{C+Hx+nUkK?%hrZL7!R1nU#9pK`dff^4zkZV;6 z@4bbYb6*8nOXC^r-q1?ODDH*s@ZNv};j2mzUy5=&P-`21@+ElP@aw-!)Wi=f8@(=cOiZX#U z(paL{j8YX&L{r+4=E_@PROC38s2kITE)PLXss|(Av$!tN1Rr?uc3pP{Po`ps72=uJbEmL?6gs2j`H2agF^%O;LjZ}6x3iW{^+pd z2GiNvnU}z9gu@){dxriSr?DM-I^cHdJBYWqLp^wX$y%9m*znI4uiVIi{BKP~`{zf( zNjphqM!_F!Jo5^Fu_dsZx8KvaT#o~r^I`OiFeZ;>V&G8)j8FXl6^$?PRgeTwIdyov zONRZC9RYlwzmo6wQ$XzfDU@R^N%{gl=0IaT$ni_EQw)EBOIjPu*}MdfY3;>XcT+*t zsR)&lB=By-O%Mo=!#v|52z&bD8dr;)im&|EQ+qU9A0Z7RLhW@Td8GRBhH|{PHUC^oXnB`SeOCvW|v~sonHpfC96; zrVs2nOHd_5oO%1Y3r{{0z*$8)Oy}@(bT=yj3+5W!U(}0JV_re9y&W!S{6iCdi!hOu zSr|F)26KImkohHb_*^9r?5~ewPIDM8Em**8_%@EKH7oHEYfjsFS!Yd73B1>rfyvHQ zAfDfXA5VXS_m7NV#))UdxjP0$PCA3CvjI$8 zT!CsdTfZ9;?&{*>E?s=Ec>p&piYD&)Q56O=^ylnOL53$&^njXyaq=fgH zfmwC1ck5pWyxW9l`ikIqrXzg$JqT_k%G?iWn(+F)6kJ&7LGRFinA~!HJfDk0(caOvVjp#@xRTl;rf*x zbi1GnGb7Gn(fvPIvBsA;b{ZBf8NXfhsap$dckI9lP=`$?GLRWD1$nI)ES@w6QPI~_ z(~RPo+w0(JSOI*#WkDD|Mw(y2TSVkBp;Y#)}mH-hbhqG&jU;Ru{1-!ZGNr~i;`8O)6<^r*W^T}KPt>jk8J!t-B&lxhy zB}cCDajteDn9jF>vzK((wZme_asLjB>O@)AV;+-ab{MjDe@2s<#mv@)A24N?4sO}= z7<8_WV*j-!XngY-Sfw&-$ax5h-y~!ITM1To?j2qR^A;Rd-$G)#myqfx70$BO9>bQY z<1m!=5+;*U(W&SQSf0{mRL>YNt;N|G#@oy4HT|Y#jmW(;Cj?42oJUUCW^R-CW)O|p z1irT~LxZ0#wd#M4uPi=LJ)amVEAs*Z2Hb(%e}N-AvS4~!Aouo^Va`qQYhn#W2SdIxIFxM+JJw{aSHmTeLi;iFS~6ANGJof<2~u6lIm|%rN(g40FPb!yJ~E zW<8|%nLfy({Z8wFUGSKQq@5%JBHobn^aF+Mx+pzcm($MM=kQxhWp*8MKrK08NOkdo z+1-c1N5hb_rSmX@OarBnkLhRx1E7)XlsIdrX9Gx z`WJMXc7dEgCr<7cXF@LRK)b1Gto@5skSekd{nENYpeBjjI6BAh&Z1tjo$o0wSC?R_ zA07i^6en4p`Xppy3?1qWz;zxU@n`Nc%R7ekrI%85vDOu0kp&iP; z0LGVi8R^CQFiU6=R&8p8=H=Ej>5?=??nnWs{|zz`Um>J283g~`A)8$#U{Q}C7%dM6 zmy^+kGDYvu_2ODwlG(|rn&wj!o#lqfwd=9{{CAKSdx|S2FXPaBYtFefa?H6Y(&+TG zlKR-Sb56gRhBv2M!&xH_=x;LTPL7>MxAK0@Yx~QPJjIWsM`hx=r8{s>W&+uHy_1Oj zS%pGx}X*aE3P?zWR)q{x`;1l`9Cd{f$V{l^ZBG z=EsdGp8{p2<-q4Jz@7@64yp%+iOt|W@Xfcvr;Gn11MQoj=k0g0+3XOJQP1H1{H_IQ z_KHkM^b$O{Bp4qAi-Fvdtz@_K4EU_=2Ok%N^Efn1uxyel`LV0%zDoZWu})~SU9sYyk_U?a@9 zH-uj&(=bn}gd1MB9ae6Xh12uAaE*lvu4)*e9Z_?a&zLC%!7^KLnv^XFRh}c?ZhE7> zs|!f;W&`uuV&I&XBBUw&!NaXr!0yUxGQsO@hY!oM>fcV{6eS}#ePspOD7L}eQ^M3! zVLANLZiJR8UT|&iDEYi`kh@;t8UA$rL|0HN`g?dL91dGdKCCb2rr)-P(=wtYyHf$D zJoU!&OHR@`8@7S=zHy@0aUACv{-u%b#`LVCJv@ISMkU6+k%-N8Fvpws?0io|?)(Q> zP;eXSJGj^)nFHZb%`kt*D~$eA0mY()@ZF*Tct#0mtyKvsYn||Gu{xUSDZ-g#V?1o~ z1^zDk4%g22ppRw`98RA9&0Rg6P z=}#Cx+>HsdB^d6EWO#JwCv3{|L9^g&h?#L7V{+eNE8PtFdjs&?b}!tq`6m8T@Pjvr z5hySZp{uGH%=X_!(cMqLL1G=(VG|z|$c2*mUw#>;MwP?2=>p8t-CJ-6|4UF%f6Ofm zp9PFKDJ$)VI-y5+efD|YTAA{(QS^R^=Uwh?_;+1S!f&C z1_@e^&`Ui4y6;UTa+^I!rj`wt(?5e8_Fcnn8C}5E#t~-aVk=g6;XRZnF2#!m(J<6j zi1GufIMvP_Q>O|*taTe*{O}vjDY}h~=lWsO(=wdsNXMpoo>-k(21_$2s?KP~s?&UI z`H63^QMey3Dh+YZ-Pwe`2K^K>`?zH9e}tK|0O{Tes`^?0Y8OAjJ;D!BOg$6wURR;q zhaURy!7S?a=K`syu*EyK1R3(G9zg9U{sAcY=1Epn0*7m!D@JGD#6f2 zL3rgr9c;?91iRQU*eCLWW{A&)x;xRJS@jxPHYlOxD}UH7AdO;2ZgE5g?r|&4CTP)1 z3fV$+Fy!Npj#n#SZD|AAUyei*>khanK7{?xBe7-8Yuace2hV%_(CC9PtM53ERd-p& z3}&uk)eCQ+-2y>YXq_w^^F4zj|4l`&N418pWSV&)Wf^qj*mux6ISC)P=A*`zUbxTe z(L)@R8Tz^(XIaZJ*Lw}v7Pn=r)r1ZcKhuWwzt95La>rrHlTzew_yW=fQQ&J^1%3Ph zsOXvkX}oLv!tH^@XA{u>`dbKfS_GAO<-|u^5$7+x4y^DSbkOZV>4A@+Q}6{B(Lxkm z5RdV{u7g8nA(lp);nTLGRCm$@UbfD|38yqTwL2HSehb1o_3^Oj);&CC(uy3FP`F&g z;_>Md5aFxH1er*(M+b!%oA}rGL_m$T`SJ)JeI3OcWr3t)PYwB*cE4!ip%3Z8Y@Bi6 zJ#k4Y2H%C__%h%Y?A`qf-?}HDNZc!U9~TX0_xz@7?WYGFOz2l$u3;ty@411+cfOJRTc(3(%1IvPK0_7;axp2Q2bS|V6+NMb zWcb+>;$ydttToo>;h+{CXM=anRb7J-=Menwz!jKxL;!-%dm=~iF0L1~CMp?Ca4D%5 zElqyX6M>sR=~5gdPgbH**lifT;=*lx6^r$6Mc|!sGu*o@!8?=s@YI^;5RxwnNm(1n zu9p4W2NF^c(wm0D3f_2Z3(sIDJ7A+8d7pr?n2P1Nz{@hA#MSqroN@D-$Z1K{1_f$KFS+3j^xnfa$9xdkQt@ z#g{xT)J893FzReZv%(SjKItsCttA*#t_tF|uTiMku@A~7ieX@BHJU50z>w)1VAJ>w z__@P~#vhu4#x~{nJB1G;@>jy)v2J|Zy9#dn`vW*Cz{37mGX3xxR1%nj3h#5kJu(QT zUN3-GVk{_BC1TC?2#7Swr&Wt@q1L(*XuU8E)+x<~&u^}fzqf94yQ*Gc+V=yr$Jm*; zKYM~_cIuIJy{b^u^N5b$ybSj@-se>ML~?|uC}Ya8+h8%nmbR@4f({LHuoKL{hf)Th zYJGz~e{lxoh7-VGtrj?R&Lv;eL(y0%50u|^Lr{n;^sY?DBerK@b$QFtlbS`D2Q+8ruWh}nE=V)j! z%N5@U+`*>9KTtE{4>Wly;NI6&)sIqik5yg9#gj3H^=UVtZ0&SQVK z2(Y$ZrLf0sE^Gf$mxTq9*gWV4vI^7Km16O*w<`svzGJ=xR?D5}{Ae>qM1OE<7(uHn=SoKGm@el5V z=i)Qj%UgPYT~i0&Y6jt7&FccZJC#Y^EU-DnBizn5_13TbBPjQ60;o4b@)oy5&*GR)WWH_=Y-D^^c%kY|jFdt_!Y(IcHO zUpgIITT0ee;_y}J*@K%x0 zF3@Ku?B=sjUQu}J>>5^cwjskG^cyr2#aLPKMa*uiO0++D0_#&^IF^$+7#F0(wv3>v6H<1?FmBQu_E_lAbMGN#k!umIb_;&6y921+4?rVhE zgI@1ZTAjz;p0yl=I|gyG){Mum4=GsL#n03h{v*z1&v6@X9`WX`H&K5$jHw5&;h}_B zwBTX*h?QH>Hs&P+`ie7W_s*kt(`PV$Os?UO?hJZz5lbYJ!%(O~gju~#lX3B0i1Rm0 zW6u3t#+W==z+Be%87ySZxQw0T+mdgD~vm+sGL^m5;0XJaKEme(-kf1JR%s6w7@>SM~(Z z@^9IYwxkaqU+c#Sd0y|W5J-CXv|+1aYEk2n^Pn`c3QuH<6Fv!klvPy4pn@#&$0C{z zwJasGw;#n@yqxHzs>z0P$K<&xQ{u29>HtiM+JN@-E0x_h7pAYPAB@7Ti~dgQ zE;JpkfwiLNaMp=8blwh2$lWv@=c|MnS}s3LMx0FW{fP*iDy7Rc7K#Gn-LKFp&Vhbn znqXD6Cn$H_gVdd0xej9=;qx?a&^d5`MoDL4mF!hG(Ye2kJmqR9ylY)Pnu?8y#}X#-0{9?IYYVMAE)Y!w!se*^BxCM4qfC#vJoi7^*~ z;l|T)vgqwftd@I&8Uy=b$>p$LOZ>PvWg-0|GEUvo5$&>x(Nhq5}}-DM{%+ugL-RF2stLe$Sk-^S3dN^&$73m zUVfBxuQ`e@kA5P8>w<8lDo>;5okF0y){{q&bkkJsQ?_VR$ z8GV3>BOx#%mJYJd!||kbB7{fm=51Rq;Lf>zaM(2tjykM>3LST#=N6I-t+RAreKfH< z?S{BN08*?}QA3amLgSmsqMrX?wWcn+wyp`%r}mMKi)v^fuFM;N4%v{2AnksUvzguO`0l3?WggmdY|Fq=fGjeE(XFf9ie{z3cg8`#))N zQQ|Df>r~RH)LZ1gOasu^zYf1I@ML*n?gyb)=Xme~mI`pmqX}>7C z^!RUTX7LFAC2q&ts~kArPo%*>&r#m|dW`J)?naL(+=N{-bx5@RYAmW=!d?C|mq?Ld zG&7t5+iv{_H^<#jzAgzXc2|Q-_)0ixa~a#Y&x@|C`U!hCMZvcdF5C&_Oo$xbhWmKG z%e)*HD3|BWu*A}6Ss+ix=d4Ve)&d-^7K8RvA_R+sIbGqYxN7%lXzut;$~B|#aK}BE zQSN}`=20pecO0CKorCN|TRbwEhpyuMI8SORxtAl1!LrG8#pQmgbzGSl(%*{{L2Ar7 zTYg3(cr7@Jege}M!c4j8e)wTk3j?(!yqUWyquAI2t%n*Qtmg^pB>BNit0mZ`698Ee z1(48F1(T5*p?+NexXJj#f;T;I>#r~5T$5nAXG01vO%~u{tu*XPK8FXkIKma#leqFs zFsFT_5I5R9#qW?uO(R-y+Zz{Du&*N4P%2 z1oQ61fyDV$5MBHMp5{}ywZq!5T{RjW@#eb?5)WX+J^_j<1ey57QyKMk0o&%5l@Z1lVFVLA-j&6(gSxpwKZ{ z_O6Zsdo5=eoouJD)01)_o2Mb^{^N+NI`etDt^zo!zYoT?3&QwTA=Ycq6U^4=vneHM zcwI&SJ@$>!Tki}}=GPBuKdb~T#}~s=1KuvLUk(d+`@Z;#$FcHH2DX>fqoBVqb7ml# z+G=YtdPh1zWmhd;)A8k+GY!b)VYKY6=2$+fAKg1o^7_>(?#SKH|^ur5kf{-qu#K!hrLv6iNxc*@y+2#L_I~L=E>f3CgyCfREKYIe(W{H8p zZ(H;_dk>hcW8_HdyTbL8hrr;gKXqlc(TCd&fjY*6%1v8Xx}_QdMMaofHrGJ<;}3ip zs>LyE(gLl&Wnik^N1g4igHG22cqmX$jvTs6n80X44ov~KVMkbFAwlAy)_m4h2ZpZzu&L5~X;A46ZqyrVMfWOB|VWvbXEPwG9o_wf=8tpIi&d0lW-PIL;$u7WV3k%Lo z%F{c%_U2u41-$Ob)20Z^u={ehfr(Knp1g7vghj&Pc}NI8AMJ;~Wx1%YXUGfbl7x8ktFY7K z4T}Hj!P$3e!D;&zEdHenD-RyQMWeGx^rIpgb@C4JdNZF~^$Mm=C#7KaYEAatl-nq) zaUQ0ZOma2Zt7!T?0fl&bEx-S&ib_5|#Is9ANE6o^?L0K0_D3J;@%xkNgjVv1*wLiF z8VJJQ$nx#}R8kG-rit0iu6=)Dzo#A+wOetnTg*hU&RFQ5sSN`g3+Q0SV;G@t>0*N! zkP$EkHIFSLUi(8}$i4{gHqXH|FC%cnZa&;u6AE1cBKW0Oee$p{nM{h92m@zdB1YRhjV@o?vyZRaGnqOk!S0DJ+_Ye2n zuEnUHli$)Uv?LfO(*khrCR9i@oaokyu@@fku_Aw7LTcV@^vpE{WdSYb z$X5w&-M0X^k+2+Zm3+n2mRle<=N_Gx>W3+v^WnkyZ1UpHVVpX|2QS(mfzPgBbd~qz zUVRk}r}Z4bqO^kU_LYH*qH~4(FY|G4X8}}??19bRe=uwAEOyfxFK~GhO@j;^!C~(R zaLz^Js!$*Dbgn2inimlMh#S;@OcpLYPomi|DtP&cH(l_36&^jl5uYj+qn(-$cV6*z z5aJl(hR7*+;9C$}loh8Z7PVla1s}WRz%e+`X#@8KN>MI)I-5}0&Uw}SlXkr7!cUWB zD1S8`hHH$-;O*y}L;M`vH=pO7VSWwU{u5-FNWEfL z#N7G|@0ZlU+xA;n?>ZMY513=(dU1SuPLM#40o~r&fqQz&QKqSq7@3_1e-&Y<8x%yL z#7%JF{U%uSPZPCgN#T;FM5wran|@t^urkIRP9=AezZ;ZMn*+4%k|eIJy-S5fR*?NU z9#}J>%<65i2X)RZ0xA5go2)Xr<^+%rYdvYfrwcgoJ_*~h6JdFvKG*5XUTSsNhvkVQ_L)A8UEN48eruqEi$d_C39t73TSIHEWMS_VhGZTxMK!M(^o_O? z#QdIu_CJDYa+4xG9z6}~w(G%Lq1_ze>nb?cJ_4ugxW(1Itb}WN=Rt)ZQRf_mtexb+~B<3{qEY%%DaCz|}s0^@5*Bb zb1wM6bm3~w#CmsPzFUyJIQtPE=vj(wR&HD|`A~RVu7QP`n)uR6hcoNYayUv9fMn03 zjXZpN$#*Vy%~2=VGptWIZ?Ds_jC+tI*$G86%E*K4aw_HH#97;M6-}H&>3v=w|L1Kt zNGZJHw7$0k%cbHlZ5I&d4TW^_cpzxG<&Z?Z`{c8~HyT?_NXw?`kyY5Zqwc6tbUsYVzdKhOU`ckvd`PivDi!6asw3;IaHa7lPFR})=Ua-Os z8!C{0wJaR*yGHg;_%H9De~2urJqg=d>xky-&m8?A1MvRH!KnEJZ;fsv_0PgE+>;+` zmhyD&x2sUw_&bJm=|FRq39*&?h~mYc$%=ECoc3$S;r?t_)K74v*Ndj{u+4dJ{ysve z{}4XPYM>oE3ORlbJsee)UG#LjE3Kbt4!sc*Ak{n#W`)<0+&Lv=_8v#Dvpm6>AtH?` zTn*~BcpjvK2pMu2pgV_mam-HFQi*5_=*w&3h}TNs@!rdrsecRCUuH4z;skY{9Rg2X z)xqFiOVOScW#pS{Ada?d2b+Dvf9nr>YM1G-%m1APM@cR z%R~doe4^%S0J)Z_)XH=>d1PuugfBKMKecp&OFUg$fCtAg~nyAOBL?^+d*a_s~)*C`_+N}sSWSO(PsBysOP74TGi2eYh= zKtwnSUiD5vxwYSk!1^j&Fb{CD^e`ST_QYusGq5C15$_G}K*b}sV8uRjxGZiA=NJ9? zpP@7Hi}8!Xc#AeAqJ=1;q$pBUGw(e^38AFOz84`RyGZ*^duiX37DTFfXYQFS`9+JR zMJZB9Nw%zse)BKf&->nc&U2pgeEGtMcKib$Y%2S_A}u}iI@s5a-JFN{QUaSPh3u`- z?rd4zGmhMZCzf&a^nE7=-GmGwMr$9}4o;s@8Xa$y$Dzs$H(g(o@P5`&yB$0&A3 zgdF#w63-G_y4j%RjHTz^`2y)H_gUw9WaV#u;sifd5`=VSRC*Uk2)ud(mM3a^*}q-B zmTBZLtFCpEv!dufe&b+h#n9X;b{XSsF}djw8>V%UW4%zFeQYv`doo#p4IA|4+YYv} zvKBnHvSqP=w@JhndzNw}3up4H91XeUtHnD!yyxt^_)F}>L2;JM%~t-s1=XC=yIq#f z8$NRm`CX|BK216!%XF6WCSa{;MFsffEt4h%-$7@agf-oNtG0`Je8{2o#q7u~c|4O%S8=OH4A0 z;wYV*$QCag#XWBK&GO8tF@ne3rR;f?WdcoJA3J%OK0mL{ku|yS#ZqJ+!fw*u%X()P zS(3Gef{Pv3*{M1ymBWGMoI~671Xjx9Sak(p>-Z)d#$vx^&(Q~#!oY4;O3#}eJzhL( zY|UWLP26cwX&uc5ue9RKs&;16K5SvXU()AH2(x89_m%N4==@?uwsDrn7pe$S8guv- z@ANrI+7~N$pJ!XH+%}&5&MjbHgdX6}I%q1mxki_jpL3V(yY!3WmB8hfT28IhuT0?2 ze-^}g*iRJH4~^%`KRC@R@x}Y zlPYEZtC?6CesDWmt7Xh8H{N15`r27scU{VVSapiC)ue?p;a?Ify!eS#|79UC%l*Q} zp5=0OZIKrIoF~uS+TCh7@1X=&sVSMWq@Ba9dep?x{FcSf-W(wfi%$8B?&AR!lzegbd-M&ZcrT#sfy)g=cw1`Q3yX;PmorVwR`F$SO z*|dbyDRY^PQLP=R*M?LP>yfo7>eyIPEAT2N58Vs67y$qG$jTt z7iG@lgkCS1c8Obo?_Xv!ytB^Bq`k<5>Q{o3oXwqc^d;GZQT5)MQ!C{;Y2KUJTVZxWA z>sxUi$cC|bIttt^vXrxDY$ zFT=9WvyML_InuIaf-7sVWtN3AcPC%*%b|r2KTWfoHd4aM$6ROQYX|uoRtnj!#00)W zls$VbB$}^t8u&JR;G8==hI?O4*Ye2cGR~~X?d+4Fi<~sQ>zw)fr?c(THgcxs7g?Al z_wcP>ALc9@XUz_mp5g4f?87;)Hj9ls_>$FWGvY6px13YgwV%zm|8DugX#sn;#*G~b zv*%>JtFcVTZ01~scjnpSS~y#N6|vt+S}N_GVmL|f8CKeH0mt+8QK1KRK17I^UYTu~NW> z_Uz=`*q|)nmbj)v^`W z>-e6&^?a*b2Ttd?3+%@Wt(>;~w=2(9OK=vABv`Hw)!{EYGh`wCEYQMn{Bpia{U%P& z&8wWDnEiZ3#T%TCkQ|Q6FyVN3SMp;nKezn*=o7!)u!!w9&E>y5>d9`oe3mmjTAn+; zej@)$b`$5z{w!9SJ4G-f(vfvJnaBEVs$d7+tYhOw#&c5QezNn^pIOG=H0FEGyU4#T z7R1LTXy* zVduY5=8te?IrqofRw`5$vx?U`*`!TlEoX`6SguP|*(bMwv%xUmGI?4o-!?vqv$as4 zeOb@4ldp$xUi>uV>>#H(hqew{x;*RPc&azh;j?BUsk(a7#Y};&tKv(a_v2Wr33=LKl(p%EJs;QLHhR@ zRAbQkWHolREJf$*@0d>-FX@w12k;!JWYp_Ji9#PCli@BL*d)Mf`dOGXUxnH8<}=Lr zl8AeYrJ-h;4K48Sgct2{P#qf#+|Z9WZbcNX%n|b(UcDvxbJU4bixAGqheE$+2(|^R zhwe5iJUcOkG!FZNLF;E4e%PLR+_(t)f2fMm{?&8W*XYC0Y-_lr{)3$|UmssLcEHD> zLoh2_6;1k|kx*TNOF#L6S-UR`Ek6XiP6puBo&v&Meh>2pPhtH#BV6%%gtpn0!~Wqz z$h1Ag$jEwn{OAw-spFss&_({F zu)2)&t`=ujcJ0S+neO!9ig5B-<_E?slR{mSQCPLefcLTK1-)m|NJf<`1PcW%k+a1T zjEathu``pImZ)6v_8_3UULS2v+=tJeAv{$Kgh{-8(6>SlvVu)W;{KViXiXBavaJHy zsyWE*^TAt9)%c%q5aNY4(A4AvjFcH$!Mw*gO=7+Xn~TMjrqtqjFg5&>0y5?9L`}@@ zJQ-{XV_r@I8-W7C<|(2@-J{WBXAM}DKgLEW8_4}BW|HW*z{{;uh>XlvI`!NiIQuJ} zhBZ%tfBRI0_p>i!b)Yn?xwemflT#OU%0I>`g+hFw76J`a74@yF1YzpO;LPa|*j2q0 zw)v&uj_j!r_}~gKbyw-y<&T6r?cbu*yfvU#q)fN9yI}V2d-T*_xlbEB^i+67|5}oSXAiE_BO(Yai^P`X%uB>CW zs?8*Q11o8W)*HIa^)UIRG)$fBdWD}i36a$_fp3qBVd08rpkQGK@gFqdeQLA@OvWEBj`+mCoUV31N=qYb!T!y3xZ-{d#xF7lzpGqsi}qY}y)lkitQ$f- zKdO>5mvu00%O>J1tI709|D=(V%JJr5T~T{t2pI2r&V1?@XF-qmWgO=E;adO%b`H)g@-tI+4N@50Ly`EB;R_AUNeA zdK$M-ue&DjI3FSC=nF>WS~G3c{ST^MPXYd-W+L>J1|_@2XuL5M1wZvrVD*P-{%%NZ zCwW8kv@-mJXR+{LF4bdKL1^F}9D0`oCI2eXv_GHJzE8k;i!xyMgId(g>Jv0Pngu8F z;;8HTF`@-=hPYZu5o6^#(ezv?iJhy4wb~R6{jBhj<4z#gZV@@xYq-$Tn|yVa zinHX;S^Sy$knWqpr)e&K=m&=pGEs9Hgr$b!q07s$S|bx9Q?>A@iw@=mjS&C03i#zm zGmS237ub**IAy1SD!FPn!6llSPjnTHCEMwVq49V;O9NljWRO`7mgs61ivdy%%mTZe zLb243?|w`2TDRUrbDc9_G;k48FV|7?y%N~H`2a@h=hEQ)b0KDofErA2hm+Q4@uEW( zIb)~_Unf;imji*g@ty)%Hcywn9rFmj|FnWCsd`d9rcvPgwUDNtZHIdY)_{J5EN`nj z4|Y6ML4$dZgw2zOnXk$wWcBoLP+s5xCwBxfg%7^tjDUQcve+IZ)7)_P!pDr;0%ffF zX9qE_Wr4Y53x(Y|^uWe9`xrx(|9uM!^`o%1&kb&` zdPobKFTj6)lE{3)Tuctt7455?O`66Qz=cm*G%r<7)a*YMgtJYd>da44@1BpY%H5>I z#20V%KS9NID>!28h>icQkP~j{@a={R)*fsEkI)KuQDQ)ijQ^2 z1yG--Js3K&2dwyMxKNy|lwoSclPode1tc5c@s>7FuFxczzvYm>%mGHU{(~aM77ou< zgOScW%-8Ose$@u>vug|W&#EC0mc_%%QAu?8q&SR9E(O`23-Md65uOcP01IG@=-1T>Jl{S9OA9iDK@%=w z^rt);RQntaC&}O{&k3T>NB$#MXGUZE%QSQzH4}d;d!gG#BhjpDrkJb7fM&o8Vtz~j zhL;}V?D=Y1w{`bzp{p(3WW6+_ReZD_F|2uq4@pyTdnXb9`0?C=-(P?k@x zom@v>Jzgqm4EMr`&Ocz9#cSlPillqESKzLH4(z<1NIqhpz>pQJ@Wy8^K6!zvPK;Ya>kE)8<8cp`sNgLN)uMA<*Yf6F8id$j=VvM++t$t2PUvw7W~Ke^6v)}mjkF?8PY9eCW#1^&(o zfJwF5IMOa=KAo3EC*@FjJ!P|)dnC=sFVVu=rwPc!MUtoD`r~ZO7VLSv0CY^R!n_3P80{%dh-j4D$!N=o!`ZVnBzu%`az;h63jFHG+prfKgs3DX_k;?^g* zIN`ncfB*1{UcRjdqo-a#i$4;~(U+xU-QGCNxiN!yKl4LcnF!<84wLEDC1~3pLv`3} zH1RkJ`zDOVaUI*h-X(@e%IM(De;b7}%KGTm(kXPxVLMWF`YZ^3+hTBKI}!Dt1&Oms zXczTa@Jn?);Tf6m+#ii*9u6M>by+?fYih@mhdR`+{~q-g>;R|5Ik-S{g^cn%fzih% zQ6gsB#!b+H3+J5ZvBQ4wQRW_gkGRK_W}XN0|I%;5B0MnyYO!m4^9C;Iq z>+(b>r_o2eyqCbb)$8z0Wg7Fm*&j#DOkpx#LUdDg9)_xv(4-SxF!gJc@Wj7FA!CN1 z(Ud`L1QmiSFJ6(Ak6H9;yop;o5+HH1EV_0ap@w_3K+$|L_7$n&l2K}+(cc|G*>VgT z7suns-|fUR;}Znz*p1u##4MBA0I=GhLlr8nGpFnt=+M1u;BLDQt_C*YVvAtmj=Mf! z@ZU4~^7>AEG_?RY9p>=z))IJj{~}(!90jxPF2^|s#5_3L8pul61v@wBlUt|b;J__C zcy?wAt}XsR-Wh2L4+#Qc>;(z5TAc_St8Oej6~}dNJ&mlfBlOOD2Y)sG&|dR2V4Pdw z5S>n3b_B!c!@Dr;-&9Dnnhm^FE1;k?38?E-p3&DO1nuW@owqDUrOY1Yc6l65YnS1* zufGHh{j1>LWNAE~EyGJK^ML}%B7El(Ncik98N*i7Cvz@AsFwo`hzw&i#&F25srSKV z-Y8MrJs<4XbENUxpP*m33dk0Qlc&8>+>Cr4Lm;1d_kBBX?khsiTWL^l4u=Z{3$dbP z0IXXd1OLz*ke{v!|LP<}HfzjDU`7qRX-s3bh48ud?~U-SUN(+8t^fnliLl19f;+sj z1{7qsQ=JE=(QaBFEV?Nm&n7L!!>u~_hZNyHwn?~lW+K`KgyY!tKZWH)g12V!Q*!j# zC-U^&Xx>s`0seP99GhzP61VRiv|{#HIwtHmjCamQMZ-%V;gwF`D~2<7!hGnZDc)Gd zOlKZD3Gm5`y}&tfmvWT*@bR4~q6*$HnV6J^`pdG(;*;9M)^97BTepp#;EckJUNazW zhO(Gt_k!}G_j6x-JBJ0YE^}7=GX7^U zW?E^}$L*Ss6Bmpg9^&w@_j%0R(lD-~{#tas9)jnNTq5oD&hSWc3?84);P=-Xv32!d zNZcX9aH{}ecY8fdmXO3b3EQAu@(h2(y$UOjokwq@8uCHLAC_!16Tazb$7PL^z{9!# zBSQm0>3hC#?dk$LxtN0fgfl38GmlJGyFo5a4i%E}ERr+52_;vd?`sLSt zAb(|0J3H_Pdvg^q%A-Y%TCzgEu_ay^CBnB;$HB)zNBn)vjQWUslq9p`Y zlpF|po`c!|k7sL>18Wz^z{EXzI9l;CRe#V&nVBXi>^uv-U52Q+i=i%?Z(-WCY`EuO z4~%IOk(@D$HhbJ57Yo{HY-c(ou=lxN*=e|~HJLOHRbo7Mh(u>v!d`=NS|K!qUy27Y zb#x7}o0H70ZJkUrf6u00x2}cE8(H{b?iilju0?QHLLT1pc zouU{h5Z~7wqxK>m_)S0WFNIcNJL*)4-}_L6MXmZKOkOeFvq4B&$YcI z?b5e}QBB8)%cn7-)?qp9Tylzp%~N1{Te30q^EJFIA%g~H12}Z@7W&_AqamLJI9J*Y z=Kl)9mdpcaQgsv;>`}m1-fC!a>^zBBu@g33RmLy3Bk6(AXSB~T2H(GPL-*WvJP@Kq z+jK_gp!l8d4{}5QXFuuethdBW)Qjeo>)_SbPxS1uV^EwFKs8Hc@n3HdIt`8DxvDq- zKWq_}WyWBQz>s=bOyDis9!XDma0HH`JFr384jZOyhevXe*uVZVntVz|pAUt&{-O%( zsn`HZ%3N@%FT=RSHDLOhNrJG{E0D^GrOqZH+|0mF%#!TqFopdlOz{zM_bn-=yCs&; z$B{Dd(^d;N4VKd0m;gAgEP*ZUWmLI<%TrqUUfAq?AN6~KvDfn!xCUH>fvq>m_}|OW zT;&PAr3J7+FOMGDz8TzqXh5AP0_V3$VVzrv_eH;DVQ z8pGBCBp+N2a^I`4cDfWKeNh5k{ZyELJDwOXFCp96NZf8DMdh3suuHm66hg;f*@o5R znZ_DCeP%v+8~oP)?4l3=7IM+Kf2;6V9d;bG@M?7n`E zj@L~9UDdhRH>U*5W=O*Ke-R*8J&O2T&LC%d{Ak}MF>__cCHVf@9P8h_fjIjh@Jvrd zr;iyRJ$f6yjT5tHZ}`)-mlwe|)eo>q)}B#h)X=;CBu!l!O>-5bQAxrSCq{WO>oX0- ze6Dj8n#Jf6yK9*7Z5%Iu*I1tIiNAE}&j@h0AICEryAp2)_<()9Hx6e`fwW!?=n5Le zi!krP%=tmcHMfR6A;CnCtrv&)Xp_SGHpGQ@5=WVBrhg6iWJ;4*)2N%uNckgO&M||G z3=Mqzw1ITq%D~q_t7$-O6&vlN&xtU2iO$9>XoLF+V!jl;*t`X{$CgJ#x^5pT7vlz8bhdH*@2lgC& z4V}%}*xemY`<|HMT*I5x`AG*l^tu73C!L((Y(|@+OUz5jG?bPpCyj(lHh%g=RK;UT zh2$4=U)GiPn?d4%{+*y<~GDQ6X2Yh9s4inceM78s3yj724$aJgeC^_OwmNon( zp?W=JRX_;yO34>ib{gVekt4M&%EjuNk;3m~Cm}0DNLrI0lOW~Eq~+XGxWIjj_SXb- zjMHhjWk85UdNp|$>q)1plwg9J85X>_LpSKH0E=%|Abi3yWYm%%;))h56rO^TplHnh zYEI5RHG?mIq!=BKcrJMrLgI#_VSUDU-p_(qxS+iaC+wVx|9#gI9xqoD_3t;r8QZJK zq1{Gw(idIm9}`R*TOJDgb@eg#_huM+HX0)%%3#Nt6(F>0S!hz5;IKlTB z{dH{?tUSAd9R2YJix6m^sSrgsj5N!XGG%BbK7C|>t(`%Q@d!EPYpdG)&lL6 z{}2ubupp_AKDB*F?IxQLqnu8#8x?`2XSSl*xH4)Sv<&{9o(gn(2EIS_j_}>bQt5(x z4BnM(Rg)^!p&E;$3QH^nmHYk$%S7ga?G`$F)iVJ)ull;xGSw9|Q?ikM9QII_Zj z1ym`2rS-CLaDIj?eoqg@$q{>SbNvExHpr5G8Ik8np($BrX9;GvwQ=}mK1q>{=Z35t zfr9Zv^RM$!=1{2a$SH}{H|=OjMUmt#c55t{h#*kqbqs)Himr#PnY zIX2dwgvm1ka9Cc<(Jl4F_sX{*V%S3D5+n;!DhtRq@wOtf?kXM%%OL-Z(*zM(FL8R7 z7FvIjM?roHJvzJ+Tjj>k!hCN!S~CkK*j@k|x8>xe$^r}v@dnP+bVwu((B*Of>lWN6 zo7%gmVT(JlcPzo_uT62qyD?aKK7_t{HIa05a_O=^b8$wXKiJMlA>YjA@mvqs&}&;h z)8Op0n6+h&aLbJqr2U=+?sl1nj$D0E_D{#0n+ITTWda06h~WH`4PuThOZ85?14VJ} zX67}2*qU;Id>+{Of} z`N7(g;WRTXjyktGV}S7@xMUFqmOf&4e^fKh+j<1T!&;2sva=KxX~)2YS;ok;b`fhODPDWGF`RE3 zq%+q3MkW1KF!szK`B``i=7vV1%}XO_^mtCglp;_!=eBTq*iXSL$tHSkPaZtEwH#Ep z%)*6Kk;$7R=C}v=avhyFg7oxQP#iglvX|cqMJFPezUc8_0G8m|P>ZMh!@%gZGB|NJ z(oeiCoR zt;RSNKb$c;77XZftR36}IFtj`r&d6P`d4^aG!Y-3`bQ6SX!6vbrVGDs9m3O>>`_Pd zG6>KgPddcXg*i$%-e3({j6VVnzQHsus)c#*P6Ir6H)zpt3^jVS04-lnBI#SbiRha< z_!#NpZ|)AXToMW1vFjn?Q8?i^n8TL$-jtSBLFR|Eq%I(lTw832Hw=TR?@WSp#0>#45`L#km#x6B!!OB1 ztw_>Xst7M6^WgpF55mQY`ygV06YZ1T2HICALd1C`QCEi&*0fAUvz9=(Vj2n4QnYZz zC?`SsfHw>|+@$d24%wu*6t$lQ!paGMh-(*@d9iRW$j$R6k5f8m%vCSg>^T!#ovW!} z;3JldzJmr?VdC0L6JsBZr7vF%(ESI*DY}7#WIgs5&S~$ZY;QKctp5&UECR$mela;& zc@xV+B+xS0MQFLn4|k7kpnq39B|d(R=pKHG-rSW0`9bkurWTB&OiqI73M0{AdkwY( z1kk#P6_6yE$7no#ORimH;rNd@m$HXS0gEuh*nMuUy*0F`k*gVf~_ z>~Xe4KgkAst7t_HLy};4UOn_LT>yFf*KqS!JU7cC3V++rr_*Nzf@8Q2PTvqI{4Dv4 zkkDY^BcYgCAFG59r0VIJ@Dwbx$`u+GM8lACH{AK!Padr|hR8M%iYM=6AZ8*o-6}*c zoky5a)-0T(oI_qG%454~2Bx&MF?YPV=zPnG8~Hc|)7y_St;=UZTS5vA8Iu5e-n-H} z%Q)oAUK@~}FGo8L>tM-?bTThOk9YQ|w#a)_fMD@qf2!f)M%AaDhnt*-@ON4kY1*y; zMaOQD^A3-xUZoml`z?XGdj3>n5B}M};RBpzk zY4>p0Yyz$@{f}JsJ4MD#-%0M+|3Km6ckneL56b()sMhgf+Q;sIl(55e{(s5TW$tM> ztZWBAZr&u@m9#~#hsX0;3ew zGI3-f?Vab0AuF!Zn))1SR3pht+H{tjFsq~%ktL)mwgId*cEVeWA~N(Tnk+V8;Loy? zH0|INkoogRtjR|(KRh6fvA4hAk8K)Yb$puOqjL({luw1WH-}*S_4mlH2_{=3 zWznHr8IS6}A>5H1TJq4Bd@=iv>_3zzp0{%7_6?C3mtKV~I%#MW7zL8!DY+sONVg^U z0{gWKzR)b;3BPfoPqVk7f@~|^u$hI23?+&BV6kwO_Efq`s*#C4Hxt_yPbE!nmI^=T zP30bwP7{{Ro{cNMs)O1FSvqX+NVw%p2F`r>10K~4lh!?HUjCf9_~V&0oI6pEe{F%L z4=ocWiO=K6>ncH&YB1?P9s(IAv#CztIxzCThnGvl-}VL<8a%WShO_Te_ryBfv?K(_ zY9vCb&>xFvI#h0Jph4@bVEfu2^6REIzI-Hu^IFx=dA|y8P4_QaH8&F<3@A{6)=Zw> z{y##S9eZi~&~q|vlqSr6wqN{yC7`F|TT(MqRg~DCKqQLWk?E*K8%#m(@>W(uyN|ey zgyQAh0%|>DGTyN=!`H!mK+3V!p92D!tByfuFwM7G46Ar*?AVBmRI)m?kdZRG>|4xNy~h zi%=@H(pnw=s5eEr~z{e0mqwGjGIGzuC*s*r!Cl4$#$#%u&ufyl$6tf zh^M}oao0u|wEH3%H#m-`+I$oJyY8Zr!f9fzZ-fW7^Kj{mLcCeq2zPz#$e+ew#xA8& zu(m`FAACxHC8~CyN8+i!#D1FO{70z1{tI-yT*+ND+73k@q+!@(6IKaINb!%i;MUSe z)$7$sn|>3S{of_=++-Wit}hRqHWZ-5)pE#v=7T?OT}OH^kLkd(@V0JhEX&D-P5e|mT`53QqkOn&Cyx(n{pq8E6J)RuQO2l(4CQ>mzrFYI$jOI@NxHCR znguAgkH#fWU(td10n|D{Yv@(tfn^y$4Qh zC1h~s?F^u^^wDEy?yh)%4z~Blz!Y6q&7{g&Gr2lg|zakpT7H>QYL@y+{%NA60Ki7y*rj;^`z z&A|XFT35uV z!_iRTTLYS!--Lc60YX?UC30VQpIkXVo4l%u294egWMf-8Y9Cw)E!iPpw&pF`R;lpn zuJ$qO+Eh@F8;TlYo!x!43h&Ca>j0XBXc*@M|F#8EbjL5-ri0ZPe~e0vqjtseC~90n zQ&f3A@l?PnDE%= zPT)0Lp7(ci8RUob39ADG=$hcG(6qo0O)VDE?0;2s+kb^P@IaB5UiO*3T;B(KH@t?x zW?5LZ=oERfG!p)1tKe4mdcl^w`}D6@FeK)+i8GgTaC}P)+}IEXa_6F8owpPgIOmap z?Z!lJW-*a+lOowSr$d-S0Q!jE7^m4uV(xb&9u5~UyGFNR@`dI{VW z=ZYEw@u=C808dNrpuMlyyL7e$R>Y3x?GUrexhdz#wdD;E{OtxvN$eow+|tOHwf~4o z=|$QY9Ye=8N71TBd1Qja2l&Y!pv@K5)Q|rf{nq;6mA+bBUH^eRZSWx9&#j=7gU-R` zNiEQ{rHR)6)gcyxU+5A4K*Gry&)d7#7JATzthb`gPz)tE0bI5WB^!TJv z^7j>$wR9Jz76swg^E_}0UITN*wadD3O(ga9KhmJ8%$ss;fE(eR2+A@SU~|2mXz}?< zh|x90?wC^iv@%YdemMi2*kaULX8`P+T5|CFF?!~A1JO!eL%rKKL9R?TnU!&$xazCW z`G;4MfOo$HS63>7%JKEI%;h+`+u{mu4M$MecwNX;SCW9nAp9r0 z3Nm!Yf|h<2>^A8pTdq$a{t?R0J%%D%S-4nn;|pjEgBYwGg@58;_iyBlN5+Mz+P1%vHIFFGl=e z*Z7H``KJmJx5bhNKbJ!4za0=2zXSe0xD67UgNVxuInlb8sbuljzbMR|Ao}{c7lj+% z5lx4=SaniJ8y#*j-`m#W`6Fqx%Q=h=WvXEEpB0cKu6ffp?F2#MLM(4ICH+U6$lFap zbbRI_hCQ?zylf`n)tlnBcT^Gea2pU4VXC~61LrJhe?wzp;Y-SceV-_ny{RiKcjaSDYu{x!l4@#=5MZV9;kqKQ8Klg52vS3&<}X(E2i z zaQA^25;-;qQ%l_iM>zM1=A|s|;2TZeTK*+)3$q0C2n|u%i`x*l(GuqPmNQbF=JhvD_IfG_t|A89(VHxAov%YC&HQ%oAP? zoGrT6nM_Q-q{81l_Vit*GaL`AfR!P#j9*_Ed=hk$##}r6Hui^5RX|}upCT6eln^HM z6)ZTd2yZ=4akn>q#JHu;!QgE!?v7mnV+)6A9?2xuK4NX@!*iiWMlLlr?FQwSh*yhM z$9Z6*hT9gWk9! zWMJ$j6!~1H2mQ{%(CIJO^0OLm+No26#XMMf;i_=##ai5ZbUEZ@*%lw-&03{V?ig z5zNlTioD(Jg>-vchtNiSHZiXo#Y>tB7};(BZMIFYcEw`&pcIUOvh7f|coH6X`-RD{@WP`QAC4^*Yf%r0XZ2MmJGCA!`dEoFx`M%0po+R{7?Mh~ zDU%z6D$jLsW>*k;?>ykj<>vI z4EX0?gz^)G)L=z0NEp_U9ZpW@^Rb7REchhs)(IjJZvV)Wvkp*TQb-oc=;FZ64X7YJ zo7cIr3}*L*;)%Gec+YwqqgmMxiZfNeWZ$Dc6bXHwuOWXn zd;rP!H%MZD6j`*Un^gWb5}tF|O=@1Qft@p7!Sk%|jOnpB;kN2DW@Ky~QEU;JThkCf~wItYwu%hUS$kH zN6r)fS-Nn(?f{&1O()SOAJB0Hr!Q&v`U|Wc*#znpGcnQq2s3niG`YDw3{4QMh24BU~~DQkE-4x})p4hfD;RVK^CN&g;Snx%GI9rc%iQZLFK+i3f#Z zy>IX#1m2j;w7>G-cxggsFP%xXGbHeD?n9I$*6Ebe22!k;I!m+Qo+u^Uyt`<0;EwUSJo)mXEp6vMr$(I8b-RC}ZtK1Df`pBb;oZX zIglAOc!xG zXA8XCP@JOQPER@cL&~li@^SqO{8B!e_fIbxHZMz|b8G6@q;)%?z(|jlJ*g4U*)|Xu zF-U4A7UJHhCR`NnLAXIhB*Ew(qqyTReSbqhS`IqF!;7B3(JzeSf6^-gM+**V<=rj?Jd7mrn@PD$MZSn@zOqo&=^=6w+(!UAQZsnc<^; zWwJ#(R#>gNAAQcL(xJ23a6r~u?0ft}4#*}m?%flqL+Ah#Ri_9Gvi8E>*tH<<9gH8} zd;*nSHw5j(1KqVWNw<+6_&83*AmfXmG2jKsPn=+dijaPZ{!IMRLtxssNPtCMP<_=F zpJZA?YqmAUyPikquf4E!(K*m<{YNc}zL9-$eR$!c0p5)xSaIwPRkqB)Mejx!v+M}` zZX``+Ir4>_J_@+;qlD-~XfCCWg;4h3A|@58gW0DDkeIv?%dEG;>t7ony?!Av?}|X1 zX>y{Ep*x{ub0}S!o<|>BHo^NADLPJ5mi+kMMS|C6;rCUZ!U;RZLq$gyp^LhhZJsgc zqH;$#pEn1U(?^SaNPn=?;4)tSehGZV!F;mzAE~t3fKV+k8xq1C*H9Chdt zHj8%`j;(t@c`bhkv$hl5re)zedk4l_qLWA)@1vEfS3&VD4~NIwkpuD+@{JN0{cAk1 zOV~-?aQDy;yJkRHkG$w~iWNTSi2#53ZgREY2w{_ELeFt|==d1~dh&6Mq+vG`y;ly` zF0!Z1`|qP>wl?qAUq2Z8GaFkxWWhrj&m=P5_>MKA}}pSGjlz1yjtwj1$(Rz|aGzmORhj>6A#=`fIV znIwC^1t&{cOso^{7_7a=EZ1I*UbWZp>W)aF`97RnP?%439YoBnJK<2XIUM7&-l3xL z5qzTffQnqXur|XPLu+2Zmegs&48dBVTY8vaO6Nf;pCJz!36}87{|hYES^>)*O2P56 zV|coJBDm^x(#ws@iJU<>q$l{n?f!n6Dc()EV7?u@av5wLjK}}Be&Bu{Jr1oOU%`^) zYQp%_9pk{rZ$?o^gYqL#}X#v;$-cT;Z5QFSFC{J@d+U zJI-32g0Jt!l8%V$!0n!c+f?d_%8m)JIpYETEb774wo7p3sT62+-o}JW*r4Rc`SgC_ zLU=EpgW84I;IE$y?_KRVRNL1~BjOIhnd+w~Y)l|eytcvr6kT^bmERw?_sZUx$zF+j zj_2GEN+c152$cq*w3JyXgtC%RQpxzXq`2qY=ZMnKqDWhVXb%n1_}$;1_mAhkp8MSM zoX`97Iq#7iw}ro@@FHt$>4l25?D-D=BoItcU}4)0#J0!NDyo=ej0+ zaKM;8`%&iVHC~6BEDKnXm-Fbvg9=pAD$-tNu%4=$<)QESk?gc4Qzo8uV{cusX01y9 z(&EAly5?I1ovolvl|1G%Uc~}D8frj0_MRXUD>fiyg(#G){|pUvD52sF&Qv!}@SY!K z;kAWj?5q`A`1O~^sNsG-+V(vR%}m~l(q5cF8{&-VwB}5-w_+IG?@~g?z*Dxu=$EUK zFlH>zC!?RXsmMiF2&Jhi(w40|*s$LZ>HHm2nMrRM`s!*uBm4U%N@)_MDL$^u@8mzO z(smSCeDbA!N34-WXFF3_w+o5)CbO3-a_M5#T}=NCTlU=bX6S4Nhf18>adtyWV;l#C5yFt4n3s)*BI(MdKFco z{pj-cNOq=B35|(}p-)mju}WhZG;3W3mHener)?}@DpeKGzkiktRA0_?-9EkfmQzp#zP>4JUp3v=Q8O%kp}phD;ne~xM*O@3X1ddv`wj_pE*PDhxL%&Y7c zG{J;>>L9(d^)&k1QGV8HH!os3(f<_h-;vKIi1nPr!5OoP{I3Y=;`=k>UhE$<)-y8W?Q|`CX;9MipxRfT&57x z5~a2eXr=u-zSJynbZT2BdVVb*HF+*zZ`Ka8+s$7gTN82G_%gt?xKo>P zj$Fcw)k$$bpDssE{|!?vc+!Xchp5uWaN5^95B2YnY_ADg zKaz|lJ0i|y zhbisXvPb8eV^Q?s5?0xIoHhM?gT9dAQD5;&idrO@&7GF0$$teaJ37EVP8ZCQl7FG> zh(gpS^p84T-^0FB<)QwFvc@e?S}dEkKoB&PZMrviFYfV767ubMG~<%$k?a zX{LaW-LB82Le315T~o&x=w#DV(UVMlqb3?rJjm=jY(ZZg;n3H+b4^rmPXBfim-PG^=;d9A3!k~CD$`qFE&uGNQ6evcAEa#p7y8C zVfSyk%D8A$B6AZ=MN;|%I+u3#jyIxZT~#Q&B$Him{yI{aKg4($Uxdv9PTf7@2~_R4 z1wGDaWLe>*R9F1~b;+2C)T2+c|4!{?a(&YfIkSXae5?qSG&WKdVJBMlFx<6!+D$6@ z{)OxG*3|A6U!Ucw9A3}ZTJ5AZf;RJJU@DSUZDb>~C8>4dDs)DB4xP7Z8U69(DYIlcW~cS# zGuIZB(Fy5J5@h0lre3;54Vu5x0-`VZG2`b2WPZlkd zd4+ENsztBkBdJX5XQXCi#aF!{&Deh#q04PL*mGy!(_Ig=*%e90Sh<@Q(1S;*$ivi! zS>Zp6RzH(R`y<`x`P4dBg`jNYG~t2ZdCC`;FhF-z%+UK|pV|E<{7^Q zOQMZ`4N%?oerEi?bL=cFcl4snkr7kt<#$}MqOGH!P-0OxKjK;msCp_^->(8%yn^yzdbQ+6wkIn<>>$Cc-yF|7frd;0^U-GAS8 z2CQP-L;j({3TdiweH**ErV(w*$U$rVS-OsR#`W_*^|Hy^J-|$+v#Y};Z^jKY$mcepU*x% zmd9wPwz4i^Bka4|b#zy|E@Lco4{n#-NlSaH1iY{X?3LGNQP0}gWm6c#Uqa0EZUyG)^H`M7 z>CYs8@uKTOl^Feu628^wDfXy$E}w}%gcje-K>eE*GUH1HXHz#ssLY2l`fIi^bMbp0 zN+?uA(yl+*czr(YpE%C;hxa41E*tht#BNuYxChjJrZVbynUDU)%|f#pPVvL#&av)e z(MbO0aa!GAOSLlpxYoy=Wv~sgCRqNGY(wkSiI$1BJ z<&`Ip*GMy4I)$Z{o)4*Boiegm?}vUJEp<%<{sK&1!n9F!`o2Ppz5A`0p8s%x&U?h6 zhYZZ9@LNOrK+zt#m}W3ND4nk2nX&&~@R@&lPh4Z#OzF#qgS2))l$*QoKUyTyLuYv| zVU>T3&|gi9K%YP>=U$|bhWi)N?YqAV`XG5E`eYdGFsnvU0)K@w-k<1=U%T0XJGE@d z9F6vG%vW?I>N6{T5x(iCRdo58zfe*acXrBhkePu-hcYk551-*yXV}S6x}ZF8xHmNE^x9Lz|4SEL6=+SnDp7I4$b zIv8GpG@Vg$0qJ!mP;+~hxjM8Gss0#s-I|PCuNV2CtOSoSm-z^OVt*pFdeCfhJ~e$V2q@g#^R6iE_s-H=u*On`rtPgjT4_B8&bD zY?`qH-JCVXuB$IXhegZKRa>?uee9=7`@`tKOo69B!yn|Vx)Ze+Thf1N-ps|{ zov!=eUO;CJ^oM$^pf0mZIPeGTbT;_kvSjFD{?+NqaW&%|xXeEFBTF5fPkA&2JVpeH9W_9fc z>ADaOJ8ni@TaPs{R)>9Qx!q5=Ibej%Se8lC8e7=aoqN&XkTQ+zf5Df&9!wW*TEIA~ zZfC+QM46%G;>?zfKbec;?Nq}(jM*i6j#)HU(6^2pVSP_?>6MVjG@(u&89VU=T6%#F zUt5H0RFKWAS*u5*4r{Y=yE52gMsv}YkVkaptk3N9iGTFnrYNLm?8bV%4nTRA?;@RV zWi+U%g=&f)qvPYf?CHfe?b+*R(Z8T3Ddmh0e$&y7YhCEF6fy&Sk<~L zMC^XjKV=1INx&Ya>})Ol0?#1k!&~N=?HAPLd6njFdPHwGZ>FS{PcO?`Aa?gc6!}Sy z;jdKS7R@PVN2YkP70Rpl)gc!csnA@(=P#}yOYOOfOor&*1tuu?{2hKm&OJtMX(e;f zd>(VL=Q%snW=l^!5oV0fU1R>NTZ?{wJczs>j3C{+a?DL5X|&sA9vj1HU{@YgZeQy@ zlim7!2b)#i$X~*?qS?AdtmXE_baihp)nC}g8edt-WSrKdyY`!*t-nkJ$m|aLT>Uqn z*Ir1zXSTY|Jd#XzT-E2VOCM+Kr`Mp<|K7l_e@?Me1vB%4L@{n)(^Q%s=1xz0S5Vnc zmhCj|B(40h7g_l`x1U*Qjiipq(5CMJbZ$@|W$Fay_0^g*$NoE=(cg&5W7Mf16mV>; z9gyjXOs1nnn7S`pN#D0#5GYojvxj!(p!-P&($)ja()U5Ex6yjW zGc6Z+TS~UiW$RG*&>40c{7!}AW6;>jLulPf4)>b0bbF)Vyf`-RCEe_Im6<8C1kJ6^ zrR6uj)2p*Y+K&l5POj(fMK=VvINSad@4l`UBVGN>_08{N%+lYYsBe=F<89BN9da4S z^Zp>UHh9cL{|n(iG*v*Syd}^o9|QIwe9jL^JckzVm!bOS6uIF;@l1oKCp!9ChDmun z$-L56fbv|yesXC#qb^~G4%0Sz`Kv8G$FD;dwYXg8wu5M(?ie-OOxTXjB-A7^i?;3R zVCVTtQQ!U_u6Iw?up*WT^vvKXw!!-kUAX=kV`Z`x_4wYR;a)w|UBij$yt~S#{g}cB zM#HFYp&WB8`ZXG@$fGUZbu=)XAdiU}#7-8_<&sWZ@mnLb_bQj&_sY0^I!~f~-Y#A4 zg*j`{-Z_8J&cE}R(Tg$MNXHa3x_p9}zPgF3#OX4FYAcz^?w8CWL7(pOZaQJerH*+nplM<&~^g^wR-==6v6|_7&oV6G}PK{O?A&sUeR;G3aYW-+}j5H22 zlTpi=dJQgpC_bHyKf$2~1w32hdu!RCd`e~w#f6;OLm34JAJBp2;G1Ak%AFVyxfx|_^RNLEF~qX#wr10-3w0^uIh)t`lCx$jVL=~l8#hy$o41_$tdvBtTgu4OB{p2Wg{D+UCX{?k?{SUh46;q1 zB)E~bhSV|r4YdE=kNOu6GaGa)`TwR&73dj#*lW2>uH${~^z*ho^mS1?jelRwxbzmV zC4c8LoNLqQo$XTXbEW1{%~Rjl!O}_g=j<9r=lC2X>@(!5<8Duw7n5il8q9oC@PboydM| zAwK0%N1xe^viCLO(5HFIR7vtQ;~LP)e0b!9?7XAdB!f)KuB+zj-}Hy>s@eQKrJv}( z?ijWuJs92kpv^wtoyz2dX2RhVHFnS1Yjm5A1mmeu!qhJ0qL}KL6c(w|4OKR*eo-AW z(^Qy7g=I1!vvt`!mS6aj6~`IE(MMyeB3O%rDa@ws02-B=jSh7|RCoRW>RnogLY5w; z!NM^#X*`O(dn!RNe>9_k_hR^8zA3l2P5()K+M?0RcgJX;wN87ucN6nivRJ@_%b^Ol z3K*e>W%Nc>HnZq`yQ^F7IF-o$$nN^OmmZtufY#T#vxn7X1RhO;4Eo_gKifS=7ZVN9 z$yi}p_+y^PNU^&z0A9_ZGy9v z+pc5wrnFA-5Y;MM!ESO)kD`+wM=aoQbH`fdu zC_8{&E_v;mwj_^9EB_2H$&T^eZfK(B6B4w#dppv3+{>5?w4ch2Hk5nM5Y2ng#;CMK zq8TA!%spR4*N8`6?6lr1=xt&R8&;RdZg;FlDmrJ;QnTso+UVD=(;H@^7ZZV~^87<~ z{rPgnRN#vvG+fVI)xE~>Y~R!KDdOCx**&y#$w#Uq@rcfLN|gaky8LxBbE$F}3htkebSIB+lutbcDFTn~_-)t0jM_iocV9U! zxRj1#Q`JaRy1*-c`inN(tVZxikY!cqWKaI&Qw)W);K##x_*0Drr#k5)(3W?{E~PTC zxl9gc`^k{~XT_mjsRXvV9|}|6rSRlnINmY0f-~HH4J?>*kN4udRokO=1Dq=_Q$%nwql?NmlFA_dlfju`D0nY=6c#fkPFxzMu4BHYy9{*kpHv|!ECee-m7VQC@ zvKw*n;V~>O=|l!pW#GPDDkRSCvy1=TJf68%5xD-h569GJ;-^*L@P6M>kiH=gpX{4K znhzV22IFw3W1~rqw`=1LX&J0_R|@BL&gTf-JKA<|Z2|B2nKPi_f+Xg2$`Lu?ue=Kq zjo^x?Fc!O8i95e*0*AK^9I>kz;GU)wtUN3SO+T;2>Qmdn;VS~~>I!qBw|o~z=(jpi zw?76n29!a+b0B9$UOm|R-S^xvnTNPCx0^HiRi1o5wu*!sEFmXO>cNRArbIT=7t-e4 z_>J~_FvYtQpZRQXLZ88VOFWm=6Dazlc>d z7ed8T5AfspToRKb1l8Kqh~ww!5REK@xAf$Qp_C8Yn!1zpzRV$=#_4eBrhEd$hkJuF19y5K&pv1dy`EpkaoC5!>z80g7yBpQktPYlYNkGTaqu45bJI>79i7i!U zfYuf%-k;04Bs^jt*`u2Z<+tr1?*(h40y`6fchjLwFo*p1u!dXI_2Jo_H8|knLO7xH z1{Z&5!ZTY9;i^PQQt-);%qUPICaebZ4Z=8~hH^GUpU1}ky1=0{gl`C06HQ4#zCH^B z+Z#XNvQq}oXN@$J-t`Pur)~%HKfA$GvsPp2+b$fZ3r}#fUokm)WEXk2eNN6cv>sCCigJ4TH8Wr@Sd|Rop$SHx*;s!2sUASZVkn zHWdtw2SMeeS>!4_tXk6huQFY@s5unt+5>kX8Aib2M;H=zBJ0$Ke=2m8+mB(53j zLCjiR&ijXfE|P18@vw_KIXff=rAxV_q#+B|$M%5E)@tlkKNASA+d+OC#E`3Tg53FM zTS?DIbF1b>i#E%nf?R;RgSb)u7H92B15kQj9t;^)0!?8%aPOK0n4Z2IABwevp1ucM z9@(pq?JK3AnU*Zs(q;m6_Q%0|=cS})g$-;xUI^!Y+XID)U5H%reCT&alsG=Fz%A7d z@M}O4D7xJMriV@fU_Hh;z2zZL)xQF|{GM}cRw$AEKKWSnn+m6`3ACB_M7v}ylqaKm z0tEqw!9-wU0Owz}Brn4&$?frca$|c5bWhJ9Dh2vbF~tV%l@lR_nL6-d*Gq8V?{bdA z?DhCiXc;d4c?i0i9DwsY@<>Xa2^_bHAZz$Wg1tf*dOuFVFB0~U11D|BjO}Y6Khll} z^|-?xpDH*uzmk~5uZK=W3yFAGCh=Od5e5sT5z}cuaT*kcnJ4{u$)=MaR^Uz+?;l63 zGwtD(syzgrHY9GRJMi?We*m1b7RSHwgR@#UaCdtaTw7%e|FoNfV_Q^6>*8x5 zwEs3PWW>m%rUsm4-h#~}A#6K;4ZOkQ&f;PpK)RhJwkQ{5nFQ~|CDHbzs>hdD&$1#H zR#xMsF*AuNeE>8vtH70KSFq^pexB&X??89Ex=X^G6(EdX4xCD}pjvYXO#Es~w9lEq z9z92LCu$8Dh;V|>cZU%F2aCzP85X1^Vh(f*R3t~PXA+K@8!;XIJ;{B`Sz_E zKk2vxy3!wGl}(4qe(4ml@mVxf_-;fF`nt8P(w!m5`C>WALy=fAxE)K3%_a{;W&;VV zMDE#gVXgmqu*>TLUOs38MtePYmu@M+AqQzPxWo$nj+ja=N=zYOjy~{MmO#wzDZ{y2 z&w*3pH}KNFK@K}+0TjnhxV&YAL1MTLbUXHr$JGrX?^-Qkh>JNvQM*ZOQ#1+ETM5&? ze#BoU9Y~3cH<{*Z3ftG(l8sks%Lu7Yz-dUta*!SYCyn z)U1TupR&+#*F`X6(P{h*6@!%djX>(!PR{37Zusvzb<(3h3qOfgAv|>L(?Yj_eDMhKLZW~N>`&q(ODQt@x;1p{2*HWl zE^)5w{)Zo0Z30ZXDc+!2Ox&X9kX@RD?(yaZb>O@DyFl=R``AY228fx` z0Ot4C<64y(kRjVF$lX~>)?HixW!vWvD>ExnrDaJXj|xHAsWK!~^)Aku_Yb6ReUF!E zwgHtge|Y%84>0#vJ$_>72=f%SLt9lVuuxWrY^ZYsd#=xg8EaaB!ZvkM)UARG$Qp1| zg9RxQGHp{)BHRsLoN1}kaC7&3tRsJub2Or_bsqf+9xV~W zg9(!$T#)5;bKPYibtVsoM*VZ?iM5AfHA+zPlpk+YXDSqaYD!k+IS_vLf51jllI*G( z#_jfXxZ{N<*f>i|kYgGOf=?U)E82>kH9hx$cNPHe^|L4Mj$H){9)*+Y^{?=yPjVzp zJs7Ml4kQvO1@Lh4S{PK)iG%Et;epMGB-V2f8;wdr_rjk%G2Q~Qd07fs%Qb}&&Z*>} z_z!F#uT3`2egRINktOx{da!&I112w>!jXoV$ko zPDOwh-4&c^#dFBn?=ED)_M5<9em;&EPyr7_>UbQn?Yv)ZO5}Rr9iE@40|}WW4O5=J z#6y2BfFq%*P+Y8sCogIM<8sV_Nm3H;=UyXxB|rflye36#9H$eUJspOPb1~mH5^pp2 zf=>=5;)u#AaF2N>u50%MF56FW7?UnAH$D#rXYGdRpO%xgg-P)9Jtv~PTZh~zQid0X zhVVJ9bufADc6gFqO#ZzE&~DQr@VVy-s9in6X*|~jV)hz?fzRP&ca0xhzdV!JT6_kM zu^I3IH;r^#%!5`LZops;Qp+;YSe4g`slTGvy6aC&&3 zr&)ljXJlZp;GHru&w!zqw?qGH9wan*Gfap(2m-EGfcBRMc~(}9z0rH}LJN!N9iD2Ols`#BnhvuxXB%%fhV=WJ@>YlqXOyIADY$=>m9o z*>ZrqrP`9j+i;Mz7QiOsV5@ez^Vr0#HYedkFiR<@ZQ{Kw48NvE4i>w?g@e`jdR8;9 zwCGLShpW@T<+r(TP-HWFE0aZTRqTL6cg($YC8-AusmH!2&_1j%$q^@eHyg?TIgd$f`-K6YUPyh6TX?7WL!I=x7+WkV`zS z=aU^*9AH~g9_cKrg!4C7LW}!$#B}ODsHW^jPJdrTelBx{D|14~x9F>w=c^2xiuu?m z<{u{}wTRdpE+BTo`H=hP0NE)}ExjwM$J{UxNH^T#^y#IL_cP~_l{;^O{FkmoQmO`? z+wBP_B+N)zn*xmeG7RAUgTP2U3?wlpIXy>Qm}oTK#shgaV-f!lYa81~$B4O)j?0Cg1Cn;GJeJDNnNJ zSoj>lWOWU$@s}bE6V`BN*LM7NZZ??tXg*dv6%=oMbf?kS}#i<OQ-q;eU> z4f;Q^$cY~~cZnT5CFKQUKADsC8Ms9!1!5+zlSs{sMPQ6+9a?ebOc?_@BDPV3hnrT$QB;zwI6d zDqX^)ajieJnwEhBQ|G!Y>dWI@isgerFF{n&f71AarzjcvyqEX5&Khi}`UW1|u_VLK zA<+y7B5Yw(hdI<>hjFTyHpIzOprW)8 z^j|KDe}6K9!Ts?>?qU$h<9NX9hLH69sKy^}G~=1o=Wyqq>4LnOaKh9ok@a`2;Cmf) z@<`u=^bVVm1I8*)c5qkoY5o;<_v?}b~n=L9uM7uBH`(ubII?DXjs*oPh4#B zNz{z3@W8GTQXzf?oRbs7eI~K^y(0%Km|iQ$b}xep;k!u#&Vw>;bD)_afXbO%qW3!* zZYnF`Y2@VaivC1kEoVN~`x61{gcd?6CvVcQ*BM55^|d`g7G&{aUHCpknLNIKha(a4 zp0mI+gA?m05392lk!r!S&~{M*b#@Y@e&|_S_>XsZi6MvQwoRBUcesMTY}X?N3W6EN z-+83Iz7TG@xDmPyhYRvsGa=`~TzH~LP@k-C;ueYfV1LICz*{{FPHV5n+9!hH2C7Z! zZY?7NO@Z)_hZnii6azEA&4%~K%<;=Z`taa#F=DjS9EhZk;N))`+X4jN`5-9?8v8CG z)+gq|7sst2f0i7))NW0h)_nl`=9UBPuv!p(N)by((s?cO^f<<84QQAPAZlap5VRp8N%$YwF_cmYo4Y_(dFS=L(tJ5*%&LK*e-_ z_^jLwzS=a53xyZK4YQIUXPz+pSZYQthE0J-E_%a>20d6bVUNGApASpw599Ay%4B3q zJ81e9N|=Z5c~fRIfE_}Uykj8;ToPYZ!;Gc?SZd)(&b!G#-;oP|*?0*2S{sg!b53*i z#pU4>$dW7=hycn$qU5E1C&)Zm24uZHxl}fngHsy<;q7iw_$+!HYhL>T^k3iLoF=F7 zxi8Cc?xJ6~T5b#Jp5{RO+s<>=;9zpiG6=SGZiM*-#-!!dc9^EA|p_ZxU8hZ{M*HFBK38?`PnM+LoV>v=GyI)ER&k%v7mqT%`s zS2!ZMkT_iv%&dgGah=@`Lcvn_J+golh8%}i(o5j?Z`mX*cMmM#%_FmB$GF_PzMglN zj^XjkR>ax$C8n>d!0r>-n9=%+!J~RyfK>7Q|4fOJaV?lUH_?_4WI5btg8$~jCiqY| z6c&A%OL&dua7XvPw#e-RyhCGCIe*T!avEp2LlHsTrZRU1ITXo-QKtHM*_`_>NAkT} z3xY~Ozf3SuxOx;%28+Xkm37!~aVg-b%)=vokvzHHleof#L(FE%a*nMyfa#Da4t3Z? zn6eboyp#(i0^`Y%pA7=Uv)DzsbuMRX)jWK|X(o9zVM2}yvb;xk{J}3_670RP0s0gN zlMD$9c=p9=D6>8a+MZ4#UY|puQNn!OyIzCL$sOpv(TtAa7}AYu}m>P_1lf=&o0hG7YDE=cMgW`5xDS~H8}TMj{Mf1Ndjv$;p%;I z_VQ3Cc{eO6|i}^ z4QXohfVI{hOcs+$U){Nm+6BlCS zvIx5WDZy8KQgPz6X3opX&-kOqYmT>=CA55!!{ezP#7|XY@e9v&eX!<1E>ZaF2EWJ!k?Q5=@Oq~dj!yXw{6=IC?r?s9 z@018bckw`sTpf5A9s>#CZJ0l90i41&VgmKl+|M~=e!zX;{PH2l+*=KF z9^T>g`Mm;$FFoOl5*NspRAXh+FnC1D3XEp`1(9jBJdc{YAa7(3)~m?Dk7gaq%7KzJ>ffi9=@Jg~hvD)SjJ^pPX%Zii9)jvA$ljs^UQqqKVzcBdsp6?*N z;vFaGY$ECXx`4zq2=1+pLkebm0+w$r;oO9~7?>JBv(+bX!ICU2>=%afny%p@?>x>_ z*$iwXQo+%@x*zn91dw9}No0+bC$#*SLGluP;JfAO@ZYm@yeVtk1=nNVX#P{+u;2v_ z7AXbdxAqVNpD1D)F9x^BE++Fe0&s<5HqUkBG=BPKE}qcIfH(Q;*czi ze!2q`4d&rA?cI3i97~s1I}hWcRd+z-m4i-7MLuw;mL5!>ti|$wfw^AASg>uQuEI=xc&LDOd)ZoA!E^)k21%J(&4WF-+#@)-jU|~ET z%r9yND-X7VgBnZ0l@}{H-QH8-#R4s|pz;+@)W?dP>j7k5c9$T7whn(5P9jw=ZAb>` z2Z{R^lF>FPe0zl(F;n^sM2CZj&ar82>!jvDi|fkdHnM>R9cnN>d>7bo^8Xoe@$l*y z7d+;oMf^X10{`Zll5IbtVgDgzxYLQU%>#J z`=tQyEcJ$)vIVvC;RTS|F$0p)b?|Cs6&4>}1~++Y!RR_|{Bh6`&#+U(3zuz&yY{bz zMM996{kHV79>tpB2ZEaC?UfBPNH!(T874o0#gu#Q^MBwBO zf5(QxW_=mn?A2fJ!qzBU{w;&|;@eUjy(1pSZqp?~siWXSpjYeJklR>Rxsr3n%N_2M z$_8cR8aSs|0FrCI0SV39AYiBipBeqgX${~I?7Q2+LU@0D65C_uJn7ryer0VOt~0u40{J!>Ysx9 z>Jz-Qx=)<_3SHR7wGKO6zlYDI6$7>V)wnF71S}SM2tIAPj!Uy-;ldeW&{x_K+RZf} z*`yWJ>VLZv^@A*@LWFznR#( zE`ws%t%=>aT*CcOE5N2ja9-d65|IE&Z`y1k`L77@4kr@DPIquKy$zo+WqBKzySVqA z9lZ0D3ui45Cw_k<;q1y6cfcyt zmiIOL2WM<%25wufOpGGj;F~xP@@jP~_`UW&d^%nTq+io=dE1B)PKEvMV!ZwNn_M_Mc^vQANwDBxI1HVtL;P-p!9N%B zfc!~2=ge{9f}YDl{Z=Q)F_M7y-n-+Je+E$HygAu=+MOKl)`vO3oXot@2pXH}!0#t( zu%g2SFy%=`j2-$ zDIYw^vLm*SYOEGuUf!$u{i2Lrc0UPzpuW7bxB4* zv0)GS99u*-$L2z*m}(NAGlkgBk0g~vA+X-lp6nM@!0$%CxwPC)<#A_! zkLcV7QzDIce}+r(XNPJR+e>!N%5!Jn-K|oP4+7zhDjVXMITv#U6o~95Jve2!&P9B>JxmDrg8QzGf-^_{fZVP(;2)3S>Fj?6{8e`o zg=@3HpkxSso0bMP>y*Or;|pN?@&IyIHV6*XeCD;d&*FVZddkxnThO-kVKh`%vxY-v z3M9GbJ6O5tExs`C0bZ00U_@G>%ig!EcwytYctmd*mX5iGYf|jUqU3pGlaDuC)Tl>v zhZd8YKMYCKRZm!OrxI)0wSyNXIr#6)JYdMp-D%QG=2^=I(FfB4Fke!m_Z%{>A@>6Y)Qpy9cP__ zuPzCC(|AKY!q|EBH1dcY$C>l@f=wb*Nk?HA{yMf8_ji@y+yRrB&)2I!Bz=n0k#-x5 z^tJ+IdId-JwD9V|Oqlp`5UcNyAYzIv@E88YnS0<7XzBWjPhN}RWHxRB@#!|ekRHIg zhhp%+n$x`NvLZO=LLfXA=L4_r$->4V+Hm^xVtC7YCscSIOTPK%z`SP~#KTg+*#duPEH}wj@VBwUl)Z5J=tUv5g~BGo}{{I!iXteL}cMH{6e?@uiG05p=~& zljxJTZxOhNxx@tafn8d8;KSYukfqqcd6~4HSQWStVe#$2Z+s?^P^%-`&8o=3`NYE;VNb3k@Jcwlau0?w@|AQv}hlGa-(P<3S-i9WIfhRNpR(FYvf zY>R!oPER>_&fA{21XKcN#PuWR9d0uEE!`n|Yo*IFnYdJqawEWF;og6!Y34SG5P z=rP^`R=yP>1$T~e)K5B+*e|A}?X)KJoA4p815C-AC8fkh!5KDJ?!5A6%EpHG^U8x5vHk5L{DVW0B zU(K=6s#~~dg)*=xR3f2sbKu^V1SqsIn8f+$L$Q~2xV7^q-m>=s=sR!08)tXG#jB3P zp^$PS$fba-cC%o)!%E&peVMjdBTqQ4RuV)zQJWaWi~u9|30&Dx$P0f~0OFJVK*_Jy zZSwIu@e#Kne87-^d*Cis*|{9HpNZn>F297gwH@WDMv21#Q5%>aEJYR+{sQLTEMS$0 zKKx|)1$&uUL*MjF(3C@giIoWu?S2Wc`3z{$G|8!8cC`M?6yqc;EFjyZrAgw0Y#@4P z9?2dHhHsr`!B0n4;W2qXIOkdmmQYO~c0FcrX?+py?s<$$e>;&wk8Xlm{#+9B?lAdh zQbdmD7eSYrd~*2Xx;Cxsa*p@SM_6TLBJJNRX#2*cLg!F~N3fV$06l7W_xg^RYp zTRKXl1SyfBHV%niGY2eDb|?2M!im&`D-j46!7cq#tTfG*m~HkWVNU7B(|4hxu>MB$C zs4ESp$J&q`%jAi@zcf7SE6AUI6$!;Te$Yh~l39ne;j1VAIPG3~ZRgeQ@y^=(kD=>+ z#OjN~WF--mk`WD&hA8o#^WO7@q(lp$rJ`hJCZmk(?7fmiGLjbFbKZL-N>Y7QlJ-zi z5e-e%_x=I*hkNfCpU?9=pXXT(9P1syewkTtN0Kxt!La}q+mYX%?!@?-BOGU9L2IS%SA7&!kAZ@%gSGH)1QtHxtO^i6^AdM1*0hooU!y%PQ& zU=Qaeb%5{=SyIswXk0XryGQS;kT0L?@IcjE;>9Wvc<38$X<7xl=C)&F`T(zN&@<7W zH5p%%9N>NL%ZD=_y2Gl`L&T*e4IVt32%FNPVQzF7Sv|5ImWJ9w{AUpm)a=3y(_(Su z)L3$F(jJm_c@`}CXG#3NjNvD>+?}{ZgpWyb=gjnDu;EuGT)HKSOzlsDU$&U>{%N`M z4u5Enl(5xCp2-vWe`8_W_NqhE2E9 zU-GseQ6}x9Ysk$eKJ5H8fF&&A@ei@rJhdrCP%|lqx2RMbGF`KwX30zZ%WE1uzwlL) zzu|OpG{6VzwaJscJ0D?#T6xkPIu5o=eZqsr=H%f*52AHn6Ye`2K-4Gc!I0J&a9-X3 z{t!GH9texW!%Cs#dd&oQK;4b_Pc?xrzf^-P+k--@HL-wN4PwbaaS}0pI$V~~gVPfj zqMbMo8yVGtE#BFfn!AE>6HV~#@>ihI^a>O-x8sPjPNXS(E!;e&PZFe+;rv}Z7?aMe zb;Uqfiglpm+0{_{!54gdND*e7uEOOK$AwcZZSd9+U1F&nhwCn$!^hpffv@Usgqix1 z@OJ$?IJIXwSy0mpwy~?gu2s>X%uHH1A;gGheRl*)tdYi(lf=N-+Yp>kVN320@=4RH zx8RVL9a-GuO+LTgL#FcO;CLrKS*wFUyT^C%ZI3GXy4DbCrz-+&cLa;oBg$XP+DsPtYSNPjJX8yD#{0U$45XxP%d15BbtFALx+Lg?${hL34L?vd|F{>D5lKv^AXUbI2tpMgd7Na)9R=OELN^ z4z{*m0OgL=c=)Qh(9As+tg?v!x$Dnhf4)Og!3=Zq^x8?x+ui}**(c(7KP})N*(}W5 z_kj0xwj@X#@*vK2jo85WEhs)yZ{o8g1zz}S4|O*xkl(I{U|K^SoSv8s{hKUEt85;8 zxwC>Ck2*$xL^zBf)#Ur(7;=d?kUHr*U_YKhZm0XfkkgOB#4VZF!fYGw*3`e)?nVqz zyOIe1*~^jCZb@R>JAw2VE5X?594~b)6?dyk;k_NIP?OUMWr~%M{Z6?g?qw33?vO`P z^y102v^!WiW*$yk#7~r0kQg|ZEc1VYpRyyM{P-oDzWqDz&AS`Ik%P-Y_O6dW zB6vL%8@B;EY}O*)N8MlmUz^C>VaPfA%}vqPYsn7*4~~Cg2JaUn;%BD=pig!o$zPd9 zg8#)pFqTSY40ZzTwV#D|hRtzT_FJL4$0oe9@+3Y|5@-Tq^0D0>Y53Jd5uQAJ7Pomy zLEGda-s0ZRjZ($|JluSr*EH}1U&=Ek5#N4;&*yc>4Hp5gDV3{7PZa~Axdtmw+d}qW zJK{1p9S&(NASnj!Byd9n>DA}1Qtt(K^VLrz6)DJ9_POUB|rL16h7_cRX467 z&bO*?!V3%XrPqS^94G{A2_TLSQ}B%6Aw0fQJ$OB?O6U$nc*X5Buvur4u+`26B;SMN zMeGt%b!`#M-@6e%6Rd<^?%4r_TxD{0o;pt~S`BUqP$JQ_1;A$iY?5sy0xNGE0p3M5 zxOIjEY;#{lR$Y%E%YAPE|MAgev6YlC`j;P%EsGc044=Ud_I}29N@fw2>I9zkdkJDR z{vaqC@7m-u^BSIEfO#u}1jOs?93t{L&9n3vPll)MAlV+8WJjVp)C&(48oWvZy6vw_ zLIYB9{$LIWY$N#UkBj))P(Ape?@tbWTMjD_fG(2^Nwr8E-VwdPBh~_vpr8m+mI=s@ z7H{Z$!Ut}Om`~*YI>Fv`J3(v=gt~Vda2(kL*YqjiEHhJKCF2A-v%7h-eKz5NRSU=& zPMdKIYLo8CT4c?w5}$QitEOThL4sk|fK_~4BH zVtn7Z16*=87dkkc1^51n!_rq5asI9*&{jB(gshVwyentHK9%1%XTomSv(ynj4KyO| ze;nZTJ9{B4-U-X&?Z}6EYuMXq168a|@W8ISLPy_0;rmbeVECFI$<|m6f7ShmA3fRx z_EgB>laIRafwwLAzq?DwvxCD}<@QJL{%I}#=wyf0Cu`$}5n;eOCSsI6b>>}sn zOyPGSBcVfmLQ@VqG_%U%?dE#SqYtz3FOQ$Rxw9Vfbwsjf#=z(DGqG@D!Y71U)EE^)e_6(@#q-0>{%yzT`a-SCz7$=DEH<>no#o}1x}9BVjz@Gvx#o(ZLX&msyc zV(^pYI^nA?CveR6-9Y}90(msJ33|PCgg&ZPgm+{mbQ|b5ndkKbk2{hB*oi6}UUP~R zi)E9`e1BN*Fq}BI$H0-yNEmjbwh9-70Qz01g_ z&VBIUQxU$C?<*7qxEMdPio@oolk1UNs=>fe954^1N3|!NkZNzz<1VqaNpjS_)nKH^q>#H?jS|DHufb> zc2UMHVIJht7I`vbToGVjpTxPh|KXVmh?w3o0l&J(lZuWGu>GPB6t8%L^M<#<_JRjN+G2-UD1ZH25AhhVdPkfXA`byv=YU*bx04Z~G$xVVcu~ z=i}>uEfGM4?G?a0W(T-fX9EX}tx4f{Lpm|VpzUhgIH*}0(`a{8@@S;o$Tzu<_U$u3mXoS^O=!khX|!_I&9_(e;+$?lQ4_^4AL z9v|Qio_TJ>hYC16{JkE$IJlZDtEvILw1UTL@&mUTi-mK$?eWdG#X^Dqb{x0vE%@}! z2)x%ZAd>eU33De2z^lO&u+sgv@z&*gc;iOILDrQKp=P-l{Lk!2Q~N1nu=D9`I7IdW zgReJ%V7)R?bnOLqEe}D7H}d#xlArKN{8HWm4v>tNzsJ+MX^O8auf3_Cn82dBO(*7+B&$VttpBpO=?{dDqm)ievXd-1Z-!Oz|P^_^k}W4Ze7J1~5Flo%Bi5Lw?G zlDRyH)0vc!+pip;qaq&`9D0r~tE)jR*%r{-Di71o6kw)L0y@>52UpZAV1@4hw#>4G zDl>P%ibhv(>HG=YzDLa1PeqR0Gn`GdFAf%8cTr`IYTn@7~f7s8~3EZELQllYf@ zP^Vvkyt`*Y_|^*W*GFUGvCoN=y6hxLYq@`Wa|O9CI0V;hvxWUs2CwX!4<9U4h6|sW zLiG!L(j&GU9u!3o7kOK9yW0z%JNh3kc@|E#dNysdHMuyOfs03$9H9!JCzujFLBz>z@{q70o-~`>=O7uEBu( z9TJcY;xE9xPfJOr=5-)b*ag;x6yj+GYeD3bZKS(6o;;bE1jT+DkinSM#OKdOk}^>l zKALMmE{7!H?0H*=tmAHIJ2#Ge-K0exSF9s#S-(JC!60_6>%dQwEKPI_;(=S4k8sBu zC#8B|~UScX4Row!QSN;GV%NFDB`kP?z?JszE7)oXmSO9jm(Nz+Ih*P0zgbpiJgLyyxzGxJvIO?$162+RnKUl{afiLzWCI zeGx>wt}i9lgN7s_&ko!W*^=*SGQ>5z63<%c44xi(f&a5wiBnI?l1o_pJfjP2sJ5;sO3UU`I~cuOi6@1+eC+I61y|4>`Kw6qo*s!HCiUJhLj4RQx_b z<{UDIGNlK}loPsSe)v4%UZ@Js92!T0yVD@)OND>9I(m_D7MvlnBsFWe`8mTW(EqCf z@tyF5mr$Dn{4xx&z-2mc5WM4oLl<$7upJxfX^<OV)a+eTkwg`W(kbx6~TX-w07~(!0UoTgy$YB z@U-+5;AgrQ8kIZaFTy?0;lmVCrW#F7%+ZABv^Em&`yS+PVg}&_eFn5Oopc#2BhkB+ z!0oUMJT6t5XIHu!4sCe=1QWkuX>S$i6L$-MtFkcirXFuu^;9^?XF7ao-^J5i{ztf1 zRt=;UnR7E}b=da6bzxth4(~v+E1AqnlW>7M)@hd@fA-X47cV|Ju-+MT%vnRs!fj!D zt|?r6NQKPOoDVlwPlB!35?^*M;vu#a|2c01PwhyDw?wg|Dli>h?20GLwfEu6or{HL z$Plj?DFGFaKLX{96?jwDBq#|k1Gzs4=$Z9AS7 z+S!=MH-!59Y9KI>f`&EX@W=WUxYm3a*k^l?!{T8?(?*`#a|g-xHgnRZ_sT@wvJB`K z^_eJtwcjy3?dXZuLB;V_oDqz-MXjmcDB46JHp;PixQ{9OLJaJG79({<}%yyVLW z=v}&qG%TG${z%@y=f*CX1bywqx&;Tw#!CWn&eR89IP(hoxBU^$iW0*qFZ{tk<4pYV zjtBX%V-I=Xs0)>l3;BF8iFE%+Bp&OR!ZMYkKymPo{N0?5mEMzHJKH@x_~HYu`x zf(=hX4uj6Za~G9>fD8UGc)S~FDJCH1fF(AV_89XgO$AQP5Ao<99k|KDoF^vA!NT}8 zy!T&iar)CcxbD#sD6?f4*E=TSc8* z{LT_Q$2JqFD|`&(r@zKC7VE*P-m~CSqZ4e&<&n_h9nj#; zankiKndqL5hto=OI8H5A*m`XVspGAHQ(r#AtGSvaKtGKf*AIi?1+nD$b#b!z)EPY9 ztq7RcISa$J^vT;Vy5#$ZFDA^W6*)L&fwjA{z=e4{Ue$wTuqw5h+)6)2RK=5Edb>H9 zmbstz=!XILWtuLu0LEZ~Ukow6y^1`)zXXJ?&?ltAoz==1uPM18aY5$6|e+*wAh= z^vqBqLmrXf`XnC`ICmkL9XtmvFPlyrW<&u6RcWXb{0k@FspVxJG>5#S^WmW965>@Y z1MPyJ;5<dt_Rg91ilJtyKdE9hM>Wy=6dU;S935#EK`I`q0F3ni{CQv>q=+ zcd(NG)rO`;&v1~B8+mKv2M+4CfOc-C{p#L4IQx4nOwM*Cx3(OCY={ml-ld4&er>}E z28ZxQ3+^89eU8&KR)9PmJv=3_8VkqdN%DO;`1$Zk-VdB6ychfpKYVx&7#!`xBk`BP z@qL=G>f1@Ix?K|<;90_w{yeZ}Oq-~Ej0M7PA>ik38=SCiEwFzVjlWvj!r+{Hpf<`} z=rL0Yz6|8nfLIt2-x&nEw4zCNUKYN-c`i9KC5C|9XuuPX5p2uKFT8jl0(-T^j+F>po%asg__js=-yeZ1Kc| zGhn2)Qn*t56pnz6c&7VWu-v~8Hx26(jcw87W7{MVVMnCZ?}U?jG778`1+BK>-r=vLY*k|t%6_o zx{%jVlgOy?HY{_wSy+HC#<&F@#q{C~=sEF;HxMf=m3SldKUr=;d|>e0X;Ym(MN*fgil9hg_BOoLR-_Bn4$sY&-CNn88&2wqcRNHxs?Q&C%}WhBjCKbexz|M1inaU1!?ZXI7RaU zSl5<}<+>xlI5G{o4Doos!=3})lRPj^cD!mq@|;Go-y|avl{gc2a+zRzLbk&?P8Sd5j^jFKJGh}suBrJ~w#nc7 z$9U=neLOq=x8PZ&7JMwekyrS79$dAvm?Vo9lX)H&LEFC=vRy0~hvlEg606I=6dkUg z%z6p>pIhL|moI~T&kFE{r+i{I!-0UbCQ!e}lT=k*!}C3j$fxhkpi5yf**hf?@6~#S z1xFUb`>PblQj5RB@e?PIx923FH{Qz{o68X4j4NRY%1-3rifM-ef0iP-h;AN!)PMn&zbX)XY_GynvfuC>yWu8_VSOx{Elqa)x077cQG;Da z?MUaFV3P4{J27^YgLzfbq+ZFBR6kNC9}@&%;FyEQCtW zQ+bPuIIOo}Jp5QV8Akg}f!%XkwtFhPG;0tWaJ<#K zL;r+}nwJxwVh1u`I0m`}+sV|mC&@XdG*UV#7;68qBhu+7;c#vq^sCP#hcZH8#r5+- zdaj^panmuJ>~I4+*vi61H6if6+FG)(JQ*&Ma0gM*hntqm7~yKq>c&r;Zkq89hPOpt zWaR{PC=RaRhK(#PdN(5c5^)t@3yLP|HzX5NcSqQMbvL=`B~GM-)*v^n8XH#Xfe#l> zf{5%}n7r167b+%@2erG2{n{wpD&hg$0 zA1Lf5LuTAefhD)5HY-5ov2|dJ*gTm2WeDtAbP=THEW#I~{6U=cBix-ip9nX(z;u;x zd}+sSQt+1Zam zW3Y4z{Ij&bNkfzcnJlk_Epfz}RPJz&neZc{c0Oh|gXGJNtMr%iY56Ix!30n%*AkZsOZ(0Yy?*?&0@9#l#OC$2h>tykuOsB2MR zt;ZCyY{@0iP9xMlPvUbA46Y}`=4Rg5H62K{qpYe#ZI&4C{S>v~LC;d`c25Jo+F?i{xgLGMT^{O&S&>X%Y2M=xJQNVU5cdKjwB}cj}oo5yP)UPNU|j@ ziexN^B~IQ^uxmvm`F&j#c1TMSmHg{WAtwfOp=_!#H-mi>$DF+fq~Dd|4RgN((T^`!B1n&p53v+yg z{MP!WahZp4+Q>$rU8{tBc&6lK;VNRSo)1o*6p#R3B)QO}iC0lEm^aqU!)Yl*+%Asn z`ke#~vXh8+`#JnJ<0YOk`5*64>IjaQ>p|8e0n#223Cr^ilasej61SE@ViHmeGu`6J zT7`QcoLYh993K`mr5tQ`)Wgj;+{vqm%`kblH`)1JjN~+Mb3*rpSYN-i@rwL0*xkT! z()!^fv}_AZ=e#R46{AUCgeOe68Agf|7m}93WyFWW)D|K;a@|M~`Xyw8DP49ZExTCY zr?ZF16qu2-YnDN?N7KmK8?i7hR~PP`a7B193_vC>fSCJiAhkQkf$}VMQh3ao=N#%p zY|h9+ug>knU7-Qo;%mXz3a7EOy)S5*P{ir;OW=XE2T57maTsptO=_a15yu^yV3?{0 zkA0m4$7aRA=M$nx@QPh+bb%I}acDL9%hlMKSqAVDHy^y~ z^d_i(`6Q9(EhP$rMeqVTP0oqy5u20Ac-;3+@b0l5R#CYJ23N|#Q|)SGcF;|5>qZ}t z8|Mv2jEAv!QyT2m*a+o^o#CF=6=c2V5x7MV2s{0Yd6Ru|@%@B3;Mw6$ur<*PR{j@B zY->F@4+M3PZ9ksO^idM#{m~=#sm7piSst(Y-P5MkV;Vw3?3^{!319+PqMV$VNAhmLw55nI#vOM-D_8omG+~b}K?#^k&UzRT-Z5PDA zBGDP%i}sxumhCl(lvxR$y#9w9%g=)gJ_;lx+5v3uP{Wr>eDGfjPA?wE6W%eKOk({z zFxWVs1obT<>I&Sspfj1%YCDsG_s+!km?nH7!w}c*crZb7Aw1i%8!x=I6!vqt@m*&F zGQq+OzB98Vhaww=A(5+~nVuBK`LBd!QR2dhH^xBIiI3RhyeEiB$tK^0?xYGDz}b_s z$kkDIXtT`+PTjedjQ!aKH%ZKc|8g($R_T4hHJ2`cqQGk24Q2@tZ@h|s38cWtpPeL; zcZj>&_QQoiK_rLES621wr?Fq_>HK55csLJWJV?L0~tf0Rld{K%hoPDc=L*vn)(US>Wd`JsN!@3-SfJyPRF!CO>f zP)1;zsQejAjXJ7DnXTv8>jCA+>Vy`0d@-C2YARzStrj5T+{vt-K_@Eux&Te+kQ7uD zFQrQh+nMHNqZEpNf%cEaBj-cj& z@2v0V*<@M`LWaM#@KgScLxvqo>D0t;{5uzi_-|rb(1Pg;kXO7t`XF|g-qALsmZle& zlLPg1`;R#G;u#aFs`-;HSp0@L^}!!S9T{LJ{LW?fYc;d4T#lo}kqmxBQZ$npyM;E) zp2F02hzm}RDDuxu-@;|v)UmpIK8SSxoMC`Z96os^25BQcJ^sKIZSrfN;U9PLSIpN$ zKZJQ`sOJ*W+?z^^UrL}j_t#uLHl($JYmA|oJXP%}p;{hC`TM_?i#WdsS{<9tkngp$ z@$(d<;L?C%nFn%PJ7ha^g-6f3Aws+*~opxqowk`8Yss#;N|6)Aa6q?<4TCi2+ zy?m1~gi_`{Wh-*a=?UfItkH!{jGnO)jae*1ZC%3A+>t>z8`-k;A#n@wx~mE8!r$ zef%ymGM>nOF(Z8Mxh4E*PtGtLnJSupKN=O-aT(%H^N|+1$H(7g1Y$6YF;iFN-kU+0 zGg4{mr!uB5bs2rAI-d36C!qCRA?U2THY1+!oL08or>9*-j9rW#x>bPDEYCZ9btOysg- zOzmu4l={^Wxoiw&Y{Y9>*$o8Fw-c>tkZ+VeLlhAI{}=*~O#z%ilBO_dG!V zT*Og<>Tk5-SqoEfN&;ETbz}cJ`qGO!!EEBc3)DIy7MYIOplkC#Gd@>r*yi>BM7q;s zsa5-J(z^04WA}rhy^7wn0qp*s;`gMY_wEj*XMs7fV*Yq>3N?)0`VXycDd1~~xu$9_v z{Xs{UB=K99S2Ci&JM) zjtmVN@~{Fq4zc`^3ld0qOEf(&#HU-YtD_YeUi3cLkN#%*pj$py*!9m| zu&dbuI-LK4b>#lVjw$2mq*GVehX!BJ=)ES^Phm1+WadKSj$Yxfml9F&n_t*VjoPR| zy$BhJsZpJ8_i4ZjV|uqX4ee;jp=Y-gvTk@Y`g?FPt9sXs4jq1n;tb}~KijzcEw={d zMwushH6%`dR%!}D*Vxmz?tI3VIIv4@pP^pAPO{Q2Zs_MnzUW_J6;eKziz+88qt~-5 zQH#qrX8E@YdR^WgojKOX-enV*I)?>JshPS!_%@$zSm;F;*^ba!s>Hmoe#cx{E03`D zCNz1+ReG}?(3oe^bj{ONHt^9XU5^aV8q;)!0g33sUr)H#QTZz3eLpFBO^>m*o9ei{_(=YG+bDkYj#`$H9psB0 zsA6aTy2>nHNSN9747xCJHO%*|K~oFg(-O**+iINLfww%qUT0SAoDnpX-q9* zT}vi4-&8!vRI1IT*ti6RoqEETdh3k*jf>IirfY0&CP69cUUVb4&OGC?-08R|dj8WV z_;QhAbN$U~G_+2f@!D#~>g#fui2wTOVVP9A((M@v72nL{9P2^e@9LSnvqQ|w3%Ah0 zzGl>wE-Sde)i$GNm61#fLOJP|n1T*>YP!&jPMf}rDR;X^lkdD`zjp?rk;w(<^N&Yt z`-@8ELHi=QaWoA5@{?tDNM>_68;iMTm$6sQJEH|D-7K5Cmu955iY^X_3sh$n(i(+$ zYVg*APCLF=&=Mby(j*m;m-Yd4tWaE#^-Zk#WQZTDVvxnYzB7~Yjgg|MTTUUv&#AQj zOEJ4us)@KhNn@V9Z5QqR`Wg9@JwSyLlnrBVu-E%F`3CkH=v!kZ`YaWV_78j&#UJ4d znx{QSpPK{FYd1`9*Ph@fKdC}B7pv)i!Z<2x=s;!7me7F-ndnhX4r~8+M z|2)r*-t=U{)488;L_BT>Y;!|WQPv-JJThwLo%B)aCz3A9EOK;6E% zAxC*rc7n}Hx+Jia9u-!gi8C{h3f_;HHgozpZ!NpOj1Mis*T9p7N0?!hO3#MRqAi2< zteKh*W7(O@l(%X$pPFTkTp9()*;~Ze7v4vY1IrkP^E2so&WnF}j~bUx+{+ijd(8ey zJ;WI$u@*1?6SdlAQbnC8v^lJpE-X+(Pq)fbow7XEUi~x;ku+xF7v$5>3k8f+jCAv2 zUjfpXahT3*O`~#cAJ{kd6$Q&KuSc3UyI6^u@#t$w0#!J5l)c#y$q(Pu$@IijvIg_+ z@H=j`(62}3>D7Ig>9+%?+4WL+j6uXH67`xgtZ^2-J*5j-9TID1SIxvY-I%toCv)PCSEn{wa^&Dwm5TGc+HH7T0~554!W$>K+lXWTNj>})5Sf95DZ zGF%bu)T*SP2OhJ_9!pclu>!if>L=UMUy5AUC^yghx)rIWy<+|<2hk&f%WS;B5k>MY zP~DRqD14}gj0+1u$^S54!Z}NHB6$`quwzl}yLdWvMiiZwl7u3zj4-v{zNky9T10>D zMT%#)p{_GWnF;Rt0-4CCOr^pkw&(H$#`}vGgAJC_uT>J%aV=o;QtSm!)mBgmC12FE z)QRzQo`~WTYuV$66KUrWt!9&W6LhlAj7EfI&}G`6_@1{q*pdyB=xTKZf2(66Gb*k? zjq7UYE^9NiIcNr2HCDu|?u=*I`R2{K<~*bTx6n!Fwdt?vbu^CtVwTOwq5i!ukTXw- zc~;|$PL(}Hv#;Nvrz=h%gQ_I-^K~B`viT*7(hg#V9oEqGFB_=&KXXA@!wTlV>_w=@ zI+A{{=ti0TC#c1bHS`Odi>72$(R;S1nQ<;oDDuA{6g4}U-<+q5?z+dLiZ4&lVy}-( zoIgWNz_?~@eSg}UEQPe6?PIrnibuo73Cv*6dwOJ|KT~~1R?u~$mCY;*;kR5fp*0hw z1d(kD^j`-L-P!erH7(MknL5vy^XC)j>5xw9zV$xem&;(9^6wPiBOrx>lqlBXd<8w| zpG`YDozT?Vsx-UUfLU3)8qK{SiEvRAI=?XnMOoQW$ESE-yCT+Q6Q}$jz%NAsoY2?9XPy- z?so2BJQXj}gU3s#MW-G!7_*f5>vaLyR~(^{MVnEA#8bApel_jbVo3`PwxVpuHdJ&n z5IMZM$4~v+%jo`{%o?@dMHkHtslldD)H<26w}!@;00XW!2<6iO4PDm2p$h#PxW|ec zG}FHSPOx&?QdC}38g&fqr*k~_^Vd9CPkr?vdMaj&x?C#J>--yxi}Ewt>Qcx)-yjqP z{Yyrp9*%78?qIb2#R974zM1lO9cSN86*SK@&ZOd1vFK@c5gPki%$A(&U>qV3Gp%;5 z%*@-{Y2Of(?2RU|C)Q7Z?OG5={>ZK8YSk@HZu>lEO#|yc0-P_ z!b<2@tO`4sNHveQlB7FROsVfmE`M$E5mtG}EHjy{Ib=W^v6_>;~mkULZ5*%q{(ZS>ZnuXV4Z?#J=$Q0h|j zbSRVkpz#omT;iTZRg96k5pD16V0Iokg68PUQDwGVXi6Lf4yktKYqQ3hU8vG znKghpbFqToboT_BH?xg>0U?qxvZh|2HBt6hCM~qbw82N8UT!$Rj?6zG>hYAKt*M!#lq8cSn z|H1U|Wawczc{Jt13u+P}FX*$8rmd|-%x+gPs`u81HUFl}^fqs0#l}w;Y>G^#=L!KG zTQA)l+;N$|F<~6ydvNF46%X{`Y9{)YyMrFQWkcUh1*my?0iEMCLGbFceDn6;GCEi@ zfP5Y|qEvW?FTdXky^4Lt)zk6l-lEyfOV&p%^S!Wvpc zA#4##zie3tCCg9H?Hy9gGHG%4+0!-@(Ylh&FHK=?r7z_#>D@zT$n0epCV_n&ijdBK z%8ajg8mhQ80hN9#MT$$dpw$jh?76!Ok)?ketCAr^@*(+1>FH6bvMqzX9-$}zpZ)2K zOGjz>ju&*?90`yLVi`{A-)$>w#6cbPi*&D5b@ zTX4NnrTKYE3R4>qkBXWG(2Oku?D*-g&|1@MU_ zGY9KTm@2ttXlH{w-MWv5Xx(iT9;_np`)5e!2*#+jS1{8&Z4#ZQDM?jx+fedl6;u!X)YZw@QC5qg+vF5o_R31*IcUce?u#WGI8U5stF zI=vz$-#j?~CR+KZjNJf(L<-tD^!^)tB#|aZm*!reO?}6ix`hhpOVt-f;OIl!j=y2^ zC703~b{n-WY@!xplC)A*Mo`PYz#PxrN}c@Gk*shJ+IBC29dBKSwze%q&fOR3)8e@( z`kxrZ@uhUT{#+Ur^O)@}&8CSPlm%I8uSG-8W0{%46X?~Eepc=6D>T>L3nkC2;zu8x zjp~;vp>^Ld+7}zZGG&iG1EtRZNWi0wQHiNqRT2f;j6SNgY z(!la^cF|>1I_X&`+jl{R?jF0yT2J+)la3_we~EM0T2%raeG`w&#Xlo+Gh>={X^ftK z+0BS6gs_zw(o{Otl@6`bV!V5=(X!`z(Z}g>w4ISO}LJB zyZckSo)^f#&WrAMzKQ%QmQv|+o6*#GKho-}j4X{8(0b(>I*@LM7CV+Rv@wUqEg|Tt zkBlH<;Xbr7XDbu8FP6>ab0QJXYVR8w-E|MciX z=E3Cze$(^i%*o~?I!#Es|K($p{mjP(}P3R;%?o@^mW1*-n#Q zoMyj1nbmx1>H)OuKm|R+^|DiYOK70>a|Aa(rCw8)qiXBxtXu0oRJ7?dDp!6)_m#); zkKRvV>Vwi*qgXSv*Nu-#FDzzeDX(CPZ|czu-FxiEo_rRq=kmQ4R`T|Bq|kkoZvQZ~*|D&XS$J?4Yic@+?a&=UX`Aw?yq$=K zIA)@HWx|x-k`y?+T`GDRT1O>wFEXC9Qdrx`llVbt9qa@1MkdSa1M`T>TAe?q7!9tv z!^liCV?d)byF8FV&(9x$@1@2uwHZEW5yv}iRlANB{S2bI*Ds4!?Rdq!&JE)7bOVso z`im6Il3?8UNvv!JVYh3HG949>RIP!-r>h;y%enqQOz@R&lE*SA&L-b5!tho-g!SYeVn;7=efV_{6s5SC(sIY zHSznFLf&9;0F=Loh9##DQ@QF?(a9%UaHvxed>MJ%^Uepft8S8>(f;7KW*ce=V_=W% zd7)YbpIH1?OhhVgc&}ecgP&&_S$rWGlX64h-WnAM_VeZMjOx9cA67ORTRsV#s>3rAs_oGy+!TZ+@T$MIjJEiQ4~jB6jd z;f1z9xVPU5zMXsu*CuJw-6F{ye{DP2mUV_yys^dMjS1lC`h@;I;|Q&0zHqYW3|YT$ zAGDqc!>bN9C|T-D0(Do>>e-gMRLV#InS!+CaHG3PIp#mil3W~$IA;Pxq{kI z=vh;W(q$E7NlFpUJ#UC-1k*uOd<=j7R|f|=1Hd(C4%mfWp;v9y_`1{NM4e3rkgXsO z9c$9b@H#oBRaG_{6%@-HA&Ih$*5 zEZ}v?D`DLmX>f_&jkjmWicU>YL$9|5*sUSQ4~#xTj!7LN(rLcLY_bQOi5U-X8}4Dj zm=w_+)puYa^Mn`C_?W2v_8@hP85!Dr8w+0?16JG)E2N@v^3vnrU#tSv9&vCxD;V@n zzk}yWd@z`mK%Y-zpfTeWzL?_(?^DX*@graAwfZ>ZY|5hZZAOBOj|n8RE##!z9gLS0 zNRc(4$qA!}v_N4Cd^iw`Q>|n`>3Sj1>DJ}I-(6|QHpD!maxzWV6-36b@RzdK{+4%AHBKT1ngk4ATK)w4SOf-?_ z53D{)%^lW(=}|30<|IJ5!gBobQbjzrX*Nz;^o_C9reJWNBbM4Via&lu4{5Ld zO}ws{KwDHgYTVk2PfJhGzbkVwvCsq+%x9BHo+|j|tGnpeIt~8F3T0H;xewG{j6jF) z>EQEuShPAxOvVZa>5{RDFw1QP3MD*TMyM6em(M1TW9rfC*dLLSi;kqTFqm8Bwh;QP zR)C3PDmi;$1~h2hB6jCPq4K*k*j%|nl78rb|8YxE=i@(gJPpJirE|i@n0atb6bga0 zVu)UI8=HINv1H&UnRvudY$c;Ymuy)KDnlHUX>S3gH4P*=B9nI}_%PTdk3(hC@ASN* zA+0}sf!r&vMdyWoVQ!2(#f9I{-z%UMqSd(tnx{Mui;sYG{e~ zyKT7JCUenlc{>sAG6dPycJO6&VQ8}(W{L>)m{ovg2}kM5*Ut%YONX>4^WYD+5%wxS zpwr108kx2jn|02@ZZ3s}{dQRU>AvWzq$hMuY$$#n&_i|CP6#Vb$0tW;z@ke4Ya{Kj zaQ_%ktG30fev=@wYb-d_SEJD?Ne8wBQ>UL9!*>t4z^%*POLmVPhaa0=K~;vrE%o!H zYJw|FG3kV9in{2xA{IW~_(n3e@d;;lDVh6FjelyN89K+9Ve@)DzSKu`a%lH)awbF; z`~s5U?u`JP;X4x-Tkf@i0%&GY&`nZG(g4vgI1 z(v|i_lDQp;#-mhGx9tU{g-wT>4_0&O=;xw=>H- zRE}d$Z!EaXdI`^^TX;82%W;**DlCfLf$OzS(@VFt`2BCTV1(Nk{9DpaQ_LPwlOl6| z_P9+FtuleOjvp@$G0Vc9S(-?N&3Nb5Etv7RoE|cqhBj|l=FcsN?n7$@?%6FaS!qTw3RHCI|07iH24CYGenD;ih1w!u(|&h9oqbz zTXDAvwjV!;+ENR7O3h1fXmk>BS(Prb2q~eJ`YxDjV~2mI->04t*B~V5GZ~)TP0qDk zqe7iLT015XUbb36zJVGo9hfYXi(3!1yTjlCzZITzjYctB0YSTu;0w6Tn`kb8$xo(2 zu){=jij9OUZ6$K!!4F6Yj{@8ERj7XOD11obfT7DN47xoE1s#dZ)`rDUwl*Grq;*r< zLo(R36{x+=70i=7pRBafQF%r!h%buq?h{RkmU)YuF`I?mC6n;g;dW8|_E0?6FQoEg zLO^4A6^#0m3r-t#`A&@==&@lRP~Y4{6T9lLr&UaHv;(lER*o#Q_66;4T15N6WvV0H zL}d~K$Obum+T{5T(!C0Cz4HU8`fY|sY>IieMT=2tnq&^Y><34=v+(1H2n0 zMaD;0!}_k@q<+^z&<=?aO_Wh2Ex!WM@X9Kp&~7RWQN9L^1%F6UurUJ z+Ve>cQ%@J;+b0FMw%LK|4Bp{ce?>4%o=;cb^nsZ=qp@k;T-c&d>E>+=OmotXB zbg}_la8?(e-c|^sJttzw@LV|X$^+yp57U3&^6`^f4BV_pBsG6mk&=K(@SyA=9qq&; z4QG>4yvr13EqY4td{~TspY0R9FnCE8&6ed4%v%S8*XKgNLp0QWPs9=O<8aV^A)Wu< zBJ{p7LU??H5&E=N5rft%IC;u-QTy#*(7p6CF^d^VE}mbFUF)ag&%Jx`iq%|bd2>Lr zdOZWT*iA>1gVV6~#t9f*WP+2n^C9pdIOGNkd`b!DRTu<~NrK$@su*oH8rD3O=9kKzr)ysAz&z`{xM78ux6av( zTafY>>>nn>9!oAn3e9oG%FEQSQGvcBNi^H?DCN{?gB6blsekgx%D+e9%Ml(FUOj+Q zhbH3PB}ZXZG^IIvQ(%nJUHYjx0|vDh;jH>BVa>$rAp22`+P*)AhsXDFQ>)8;Y6us_!mu z-f%6BlHDq*mtat-C5tg_10P*(KcI@@?fgs4akx2dK0M3$%a!sG@UHGOhYyB#VE@&0 zH1PfiAv5>TiR0&yxPv=ET62iBsg>coYB8)G&%;*r^}t_{4qfB78oSDwU=G9dk#uQ+we{$YLV6s|78eebIln3-x<$A$lg^=;p_$kq(swqf_^>Y!1P!E z42pBHK4Ce1X3d8T*YPM@tpn{U6Ud34a?!0L?a=wijDMlj4{AJC5dR4~(b-)E{Tz&O zlXMA9t5wI!A3?mwnY&4?%uie?7f#B8Rba7X8_z96X^b{30TaKu>(Frb&tfs?kW=YBomIb50ReC7W}29EbbjpCy|vL94fMe&RxFL+}ntMf`1f-n>6sl zPZgZ>Vj;*Kf5Hn$(}UO6n6%*dpKd?4sGo*^pDR#ujG+I? z1GFv{(afG@XyN#rUbyc|4rOnJKyO9(He&^*NB7a(dVLU8HDI-e8>%(GC33S;Mcped z;?TYuR4_SL(rM4Z`v+vip(dfiE0NnE)T|!lgk?Mr#~2jlpBH5-$6{&v1Ky6oZD^35 zPM&YI2gZg=a<}gQ*QgoLpFbZC%PB*1f|U5r<6FQ>QRi)o>ZFf~e8|s$2;6?&5%WAn z^n4&fmX-om_WQuASX&S+lp?FpAMW3ZpnXxRVIlK|@b$N0gYy~UZc+$oKeS1txixs3 zOyosA;ekiURroXd9eUIr#7*7jaD-6^=G6N^){O}0(_T(09p+$ZY(BcS$>P`7vb<03 zkI7*VS#)WU%$+sYcw?-3VSBtTnYb$l*J*F%WxPzHhM`x{=E+i2tPqI2FhXSFqz&Cx zN_2CmqWILpG!QJ_3CpxM;>LqxP{k{j`dF+7$Hh;9+g^#aDZGzy znqYRe7$#X;poVAye7gIOsS$J`*G&QE_|D_)*nSvVqOM8U$60hk@i$>jv5f?CO+ueG zE7+MO#h*T|pC)kz>6mxLqKHK@cq2I*jMLq?KgXZI?yL-$>hp;N`$ZxdV~h|mlX~b` zz`Nly)N8?AQaJmM=&9mNVxv}%KW3SWZysvK8Ny0gCqV+wt;Ab;Z{9?fd zu=+QfWTw7>r(Z)b)bbM9{UaOINg4LEJ3^Bv$= z`q-nCd|YvX2(_n)JFhC>$evVa(728BR?Wq-9e?S_oY!RNnHGdCtAHt$Bg9b$glK1a z7E_E1@#TXy=y@lk)H;nGUws)|E56~p*&oP{Yym3omiV0~9z*v;bs`gVALds*rKyhu zaL&jPO`9I0%~@%_lbHsdSg(zBOYGsg?0C4KyqZ$wLOhzT0spjjVdU7?T+OrFQ7g&| zKbdcUZ3Sn*Ove#C$~OUfw+~ksOa-mu3K0ER6E-C@3cD{lL1upi?0J(yZxdzA|8yN@ z4ZI`^CElBhPAz!#?Hi1E_Jy+VSlaHA2)V=KQ8POO8Wx{}dm;7s%hZ}KYxRWR!#vo2 z;4=Kxh{A0P{!*RRsl3ZMi_y5c1r#s3Vt(pb@dc`C>ZJmdsZLQ$- zaRC&_OoDCM0Z{!?54R^2iX3vvKvKdEdl?;3@XQX97cv50YwGcBIGcDXDup7^g%j{; zi#c{KSLO!=OZ>;)BS7nLB)zP3ly(m0LB3-;dUK}C$N&DZIIf`%H}A@Y@xrfU26e;>uV;%gzKf{A zOikEG<3(CoHRv>I6rA$e22wRQh>P_oQGA9W9hBg}KASHzLE$fS5Cy6^P8mkskVm8c zUc$j*dHhg)0#{y;!!IlL(fXC!VES=uEaLIOd8au_Dc?pb{WkjSX&#!tJ0lw3uOc1~ zwdg?iLYCfDw#aoZO-*^mxx8T~VMq0fLY6fUcWxRH&DMqVyYG3+CHJh?ZKmQq@;c(g zjhS?k+INU{)}_Nr4|wzY(xB{P1|976f$*GLbYHRL4vsM}_xcC;7Q77AQ$2wWW@1p} zYEb4*piEK^midwC4Pely-?Lwi(i(#BmjQ{bH49Z|t%T``II zK$qvg;aRLlh|4@qmQDOB(wV;x7BDMGLY)D5*cgLZI<555Zywxv5eX?eGf0=i<$CcLpt2j}mEJ=1mV3l?`bVhK(V-u+S7TJ$Ij{{-6BoI}qiViT+Xd@$A=8sMS(T z_(@ho^PfFf%8w`Zrx~&(H4vo3s(DU33vtY?>r`>oT_STg4&5KDA?D5!-6wxM{PWli z0c#kXbMrE0%YPTyojL~@_c!3Z0BOptxrmEK{w3E=E`r@nb&zsA1(&bc!kc-<6Eja( z;_H~H;*h0#vCri*J__*y@rJQr|F=^HDX`TUc+T4-3ahgX^^d z^6bWFNGp=Z)iuo!VquORot{)&yokzgwkMqIv(WI{jLew)gI=y3E6%#mOdsD_2N72- zaMLCcG#t;t$^K{QS<5XXxWJY;OxMOdc_nZ*4xy<5Q;A`2Fxlb#hPIVG<7My?@#^(I z*navlWE?z-=Z%&S*-TS@>+dP#`MhKZd9nkluBwq!&m5q&(H7V3Y`_WsnTgU`rLenM zTGU#$2Etx{r@dw&_%(MtJP6qiYN}@9^M4I-c-l45zT#ZG8G8&CTU(38*RO$wI26zF z+o|!$AefcuPk(hbQXCOSrycadcck^!5S;TZF% zn&&>df@hzi2LU&oVL_`dUpM6p?*^%)jtg>#bIc*?J*tX`b~(YlifU@aXURHqFPyW( z1J_G@W+x7}!{~3anDwQWe!s~D>HFre{L&ublz~?GG(k+FBp&B3+mS@g>nr{_Q-m`Q z%|MU!T=ZW{!E{0Z;+m62~TCo1( zZY&6w;6d9Ps)1Lgye4(Q56MX#74e2A zc_2E^!I+xUpyvM(-=6HE@w+}@K*}Htk84DxGXjq(9YSsSJ+vZF7T5M=p;wbW)b981k!98sn3q3C8LZWEj~gN ztiKHVunv>swD687971h=!nWRrxMIe8^77LzESaf~$FFpf_@L3a*ha)3!5()Hv`5m1*jQ+$$GFqfd0;>SZ2~JXVc3Eo&tP@6(~W>JR-> zwhuEJ>WNxup=jM-6V%{|MWgn&!`p`131r}r^Gjah10AS&@R*uZ zU&1{y_H^J^5)#``v1trV9S#;@W%Kp#@hTu&)igUcx4wx`(+Zfthp$yy^q}Tn+XbM(%@V~7Tx#q8_DiW zfc38&;NhStexGVcX8In-R~grMJ?3#d>%zBmyQ2(tq`%>*I`qTpV>xug8Cxiu`4gVi z7lWPRD|lpo2wzEjb9FCH<0APfboqc1&iPLZqPARz(L*kzs4t&)=hr!ydigfVJ?BJD zi4K6_{AARS+}F2GIDkzBk!ZC?opel8#{EBX@T5lx3F?udo0m+aH{U;_WA+Ua;geG^ zVZ=Bry59q7%4*`Bty58c=Ooa(mIM0heCdtP)-W{b3}j32CThMRZ;u-DUwzX={qc9< zLH}=(@$5Vt6gooR0;GKrA^79JcnJ18hfTv#Xu+3;e?4pAfG~~NuUik*UXnS*%O=6` z`Ve6mPfzPy79BenMtSqqQCIUOH`H^Gco*I#o<@oI;_6=VVzd*u4a@Q4U1MQ;;aalc zhzdV1V>v7@_=&4dO~SbC1_TG!qPdhh96#SnGkz|h(;ccnI&Us&yQ`BquAZ<%Zz6d5 zC!))EF3E85hgYNTq0Ad&bX`^iq4!nzg$|xj{QDzGDqqN7?WrlWV`f0WR!#Aqz z^L2?Y?H*AxN+$tsmH71!C2FlK?AhUhh2J?OsPGy+Irty_kSg&MBoq+Wu@UgQcb_Qe zrx)gNoKPVJ@%g%iu+5=FRB)3I!z-Q0k2Wv-Pa&OtM#=NI_M|9!&wO&xkuUOT1TG!n z14Bb|>B}Vt7`vR$65QwjBPCgj)3c`I;nmZ~u3Ki9QqV$PJGRite#z7)Fhz8zeLDo~ z9ZeJ#NqjPgb?|P)7tt#E6c%1WTzb3^luDGKBcT*Vc=+JQ4n_Xs?G{+-HVXeZr-7|F zhg=|KbnNky@I7ZK4rlKp^@AJfVs#Td`!NC&Pa^UfJIJCNC(s~jfE3M-B}vhfM30w+ zN^rfU=$gACslPOnGzQfWEr}OAG)1h4K%rN7CH17 z&%4tJLO4k72EE|9hnb1fXUJf+e+|BVsUV)K(acMHTP>NRDVW-_iG1j2BAj_E;Gx7< zx_sme?i9})?&>RJvG%T_xW_G>d@kRDA&Cp{!7mT=9=u1{=d!}uD{gdaav9Cl8-fzr zMv~*@`JW#J(q{@hfDa$h&*nZhy$!()Wn4%t2!d1gli>66MA+4^6gfX{(!szQTAIeC zF$49&by~~d;_iD`=WGs6)AU6zT318GmpEE3(MR@#yFyscNm|95h~MfL!I#=j+BqQ? z(%P4bcs8!oL2n)YUg;w7$8?K+xB7}IFL>a$nOjj~6@hF+IsUAbli}draM8@VHQ;#d zIVFV$Fmhcc>in_;y9z$mTr?s)YbmPhE{`+RRp^%we|Y*WmKYS90>Z@kI8IK7X6@4f z6_@3dxt|SvPgL;Oh^hGgWFf|nyN++;BWYT+HFnL@0fl_~rL)SctON z_eSDNU0j2o&U>KG>_V_~=ZOAwo8XTEQ*wWrI{v%NplMqlRjif4Yc<-qX$cX9E55^u z1xau@qk~${{z=tWex#jUO++pb5OIQ&uwXbHCcl)8WT@Ky-I)=Wl zAK;~WT&8=q)X1}ldZBz%21>o@A@AO&;rHM${Lq>M@6W4>)layA$hw@yweJ*t812e` z(NT#1QX9xZCs%m-R*uINw2{|N%XqF%O`vT*g*b{&!ju$UXrEF}7TlYT3yM@gWL1rY zv1f_f{0`7;zbFq^RSP6=h;bg)OBfE5O9~lh_oDf z*hEF-J>e|u^A$j|)EnXXu37l}Q6^a0)zC<lpZaI(a@ehHOip2Dx7Ow8uJ- z7J1DCopm-)zaxv7)E<-gm3=^8;#C4@%knnC=&aGh+jYZQnTf<2wicsrW##SwEE;9H7U( zj>AWvVj$q_CGsFaTFmL#Nh^kq@bs_>(=Ii`Z2wqln0%AYIvfuBHa&#^V-eg5bA}}U zc`#=n4|dq@CF5Rpp~asiNG_{`Pq;gLEUJKmi_)OfITwe5yXmBMV)J5OVoK#J2@E=R4Qn$=^^3B zZwVmMJA;qx9l?HkANhNHCweKSlTT4fMA2(G?%!nzr!C#N?w*t2i*6f~ye%Th4O8J| z{7BwN-wXJ~Mwb6GJOO_zOTjkOq-O7W(Cmi^ejnXWbSAb!oXath&*1^uU-km^AF7IW zeUxCn)|=2eZW#u05X4>Eu}5tgY>yj{%9)pORc}1T#GJ!_1xZjT@#jd-D8K`IpAl|` z3{USg5oal_fn|=#N@}M7x@DS8x zVZO!)@u(SZ;lGDD=-oX74=p{2&ks$LOtrP5=>etC7rp^brJun0AB%9uYKi{y<}e;g z(8q7fuEPG9#SpvZH$4@#6RKnP;gPM;n5itlz=GGL`nD7%%?k(bH;&+S@FAUNeHYD+ zW}v*L5!rCT(>*n(d$a1_qy^YE0N zDoy=j0V@tOR8x8>Ua8YT+Y@Ebv%>`FoeKCnJCc0cWF(&Y!wbBWW|2mhL3~i}OulMK zya+Mo_)r3 zZb7fi7ZBd#f@wVzuZ>Z;@5^vn>{_q{}geH*FTt3c9~Eg}=2n!#%E8(wtkPINQ;L{&ew z1DW+vxZ6Nkyy$rWo{{*8x4Gt%87q!>`Q}vr-~*8`jLkz)tI$Wg&O8Hp%!Ne-tEvQ zQ_lFoGhX6voZMoh##b>tA)Fe@QtsnI3=-4ZEqe>nvQVF40%ab-*cAmtV5N5mjxAP~9>N z_wYYsK!qb1_elH~3hUw0=SpzREGBJBec{#Z1Q_`}9=xYLp-HX=!Y)Zx^V>X2y7ix| zc)#5gklPuFmb#K1F*@<&?~sn@R?KzSE$Skt#q;s)Za>f&A(@dXVd!c88{?lA!~05q z+VV99W7CZROyj9uh!+3lm9Zj)&8~D&^DI1|_JeMmUjxBUY|wGbI;hd#3gZmiz$Wht zoaoDeubW2TwfH~e?zg8UiZmjbx5R!7Xe9S1+1@@T=?2QawS2{zR)$Daj5 z?4q=2_`0tXJHz+Gmf=ywlhTLU@bTavyo}L;EYCvx>7XjX1)k*Es{{1nre&~h<5bYj3qk+Gjih|RQMeMH z1_qKn3!bh*ZtqGt>{S~96I6$>F!?@ym_HB0Ehk`6Ob6_GAC9-CHPS$dzTcj{AAY#& zk_~g+A^Y4hQ9;uL^!z%K+zF83|6VGOD-_+}=qMHO(x7-?&Z}c}lo(BWIZ!kv4{fZ} z#eW~h67#F7@N4cXh&IcXB0}lOB({Ga+4ZoPK2{oypBFn|_#$)OifKCJ=M;C`YuJf9A8HC? z{Y|(Iw+CsMu$QiuWI&`)0rh9iflcoPgmYLC@4xkcJyGZAKSftz)X727)geiKQLc?F zt}N!w{1=6vrGHSrj1XA*ZVX;=e+XgAi?P>nDu09U2(*n-!b5wXK%Hkvja0A#4;JY9XXKn zN1}Zxd;yJ)%V?pIhQ%EnB(c#1{$=*VVO>T3#p#2*nR=eEcn&3l(@aVEHaYQPSy%MW z&?m-Y8$sx3hEm7QkWXg``t;V(-uBs;n5)HK`Z1fDWy|8bu6-b#;tH-x#yHna1zhdT zd3qh&aYx%JOpLQc<41=i9#&h>Qcgw7kv$R|^&ed@MTZs@tisrmznHKsmZvrH7{9eSuk0)n7kb` zfz(}ffsUe6^yx_#5L;JJqxWJuw!aXq)coPr!l|%y>;Z{J{DbP{>0^()HSmLL;KRyM znA4Ogl1Rn{v~`s0Wnx|H#Ve;#gntxeSrN&Fadbi4anIKOOPbU|rffPYu*G58yl2 z1S}RDgI#f&{JP#IQRJzoaNnB;`e(CvQzv>LZjB=S*EGmEzdd;UdpOjK7UF`k2@ubh z7GLyyPM@gUfEzos@OzaAeE(g;>B2U0(ZhiB9BG92k7UK#wLB!|#`vr65sa#MOOlQ) z2iHyCMZ(;d$!6?IC7&p!n9$yIMO*b3^=MFA}?>^bc-1$UScWR1N z)pE!S>ufUJtO$cVJIFMvyELot9$mS_8`Y0Z#OOOSz}-9$jCu2Lf`=5U-|8ak4|l>w z!(e#(=mt5Vvqkco8bvRvnv=Hf-6Cm~T#zxC4Q9j2^pmkS{k>b6W;@P?v0BCw+!zYS zMl8Yok~^St#(FGz;*E^`FYF3X;6F0spbAs{9$~`E@v+!+$$&H%%hJvdaTplTM5pahLiLypnEq%P z+%&6(uy3nj+X`c`(X=)&o;nrJ8OM_~i8CcTMk4WpLK8Mko=u0NEyV*vr^x1SbHU|G zASm`o$`$=xMRK=udF9)aNUqvu5?*>6s+NS(+(bRx`MMwSRT@dKr9N!lt1Dg-=|uD| zTTouIgb!xg@JzZF46RxKUMp4kVWnGm#Tv!%I5o{E)&Izo9+ZPfl=5op93{-Y9Z=h1Z^_7L}}Lx zq8f4mWSlCdqf#{q;^4|Iu`o}i#bXP0lH5V-`&>bd&iNomqJxQ`3>Mh+d=?E=O zy@a2S9)#>O0^!)zVX$LYEtOT(go!Up;but{-FP}5Kj;fVecKY4H(C#DcWQ~r7d8Ih z)oJ9%;T8BwHJZq$_mZoO7HCC2rXMG~fR+;iHg;q&Q~bV|g~*Y@rz5f$%-qcOIgaJ- z4U**=&v?T8tSD#l-F`E3gu{&Uo$(w`NtgLLw^(Mau%D?mxX+}GI4F5c<3mE$qTw6MRBNzGd&Ah|r#BfB)iCyVcnc#x`2zDTaS5aKI9m|-cORo9 zGm>Gh6tGgZrEKBP5*GOvS$j#(0#VbdX-g?$Tdvizv)ghR=do!5tF`L{7nMe`^4^8) zJ%xP6dF4dm$;=+sTx%hN!#4%xO@)lF*(mO#+CMe)m_KLi^SexVdMD?`#Cyz<4{O-k zR!L`0k3RR<@ibQd<0{t0$Co{m@|AU5IhrxOlq#6FI)jnlti%<{)tt1n$PfqUI;d5N^zIf2$?OQ&u$)45G=UT`R6;_M1J4oyC+ zvgZ@Qs`E1i4H_-Xj)(@P;c-3Z!OKf*pwSeLOU6C+>0UMA^)dT7J+roQHWtdT0mjK} z_GlNun1%>uMA1Wb{BSm_-1L~4x8xza(@cpg|FeyGKk7WY!P<+p?El05+9D@3Kb*uY zujVit4^%O9^AzUN4Fgs_sE@Ob+b$?0UYzT624g1aS7?{H$n<}l!(83Cj@g73f@LR^ z+26`rIT}L;m>sUN!hhO?an)YJ?5^3uxU9RuzH@xf)O;&s5*8OS{S(w!p4oWr+)xd! zt6UlzC}J7oXIU(ndxf?Bm&?}g*uja7R}|iUU?lwHK9POzI+B?hXu&#n>j)A%r?ZuH zQH%-U)OehhVZO8q;%kr4u-E~`smUH>+LCzoM)z4B^ z`13yN_r;UdST-mq`YSCo82ruN_6T5(T}fanH-8XFyA`qHTxvL1ycTi-1p+pC#s)#u z-$~4~yFWN~Hdolxi~>$2M~8h}HI*qFZ_XU>n`|dDSB3fh+nDp(UXJ^D$!Kok&_~v2 zg$wh=-bm=RVT(YLU}i^q=W2+V@pgZ53E2l5RjZ|u3N+zeXcNI5t%ceMU z_on2tGD9Z>8~=u|8ji9oOc}*Jq%OxT`sK#{JlQSClAghRbMyo=bFl*V(9ULte`^%? z!PyeVJ>@n>Wz7wao!oKG^w4+gz8^N+s%{;wRn%s7>54sqCkxe>^@o3R)NK_Q=b?0g zf3p<#^qRZO-01;V=ABJprhj*0GM~+2oX1TS>Q@^Iuf7goXFvYSUVPWjR(B=~o{TpV zN>z>$Ms4e7tDdq>KGi;rWg&?;+o;lG!Px$GyqEM3^;^;)3=4@CeWc{p;vbB%WIITa&3rc@{W^C&k z8TLmn`zO4V-J%%bkXb*MbN+fdyH@`n)8FhQv`iW;yttjq9X#DE*wAXvoppNx_wUqZ zfkJB)Gucm8*t9H-9q5SV+(|5Fd{4;>EzKUX^W1Vd{CDphCK?nolk?9om7#p$%Om-M zk0#GqS7Uj}zT0tJ{n7b?;`*(O?e%5Mk5!dSs4dIJhZ=G9KMXQQV?j__QY^^Q@?n~E z`GUKrm4xxn8wL9=DGLi)#t1{+XbGS9o@JBo+p^ZYo$Lgc1ZGaPq&L;XkK3wh&Gmbu z#eDm5k-5kj%WOTehFN2_mDOI*Q`1>*EQ~1B6#l!V%-(EQ;#zn)2}Q+YgbNBJ9S@s! z3s%mkVbXlcnfpDTIM3<>S>9V7ceRZr_gvH(`{pVsw&rdg^Jh^Ola_am?bJ$SBen}z zk!0q*PB-IDGRYV8``B}pt&O-Fhfgri{+2SHA+4;OSri*ya);Awb)89mILJOf%4M5& z-e*=;Ut}jc+HjvJK4zL$cGrw*=Lp(@VwmcK$C)F|&jmL#P1sZQ3GA&y2U)eu-AwO} zaE^=g8)nrHVfjz}EG8hMTs zTF-7^Rn8={(Oz!^K6}2g-Ap$-^SllBXv=9%|1Jeaui$a@aW@&Rw(~VXRrxY*O4ST* zN88An1{FJD$M3~l?=mB=Z`&pk$YuFJs zx{`132yR}zA$ux%qEKk{mKhrz%Z$q1EAWt06i(npb52(K3XaqqV|;_ZGV!Gfg5?k7 zIbDn2Gn)P~+*w<5Ii8Lo?6JKzOnr7WBRXNrM1PgcE@d9OT+%bTU|goauj-(n{i6?) z^UajI*>jP=A^JAw!R9={pJ|VnAM;}*kxPQVb00GH3Tez{w_GM( z_YNm$&tjJM<|C^$eHwFZ??GnX?=ymwx!2j#Sp|&x=tZ0()$+_Nk5O#@=}2ZgKf30f z68F5$FmerM_{Bz^D|$j&?!S@W>J zh55~iU^V}HBKW+cr`rE}grw(Z9ox3bTd?0Pfyu7XXJ%)lGI0UA%z%s(r|yNU(5?3Z z(<2|pC>@LEJmu?hPpJ!7I}>1;5)JlkMZciWDTrh9vXt|FX*uVkzZUoWXHPc#b$?BQ z@T6e>8&gJV>UEY?>}82w08{q;71J6RUL%^mkX^5p&06197AjtyAP_$Zb!cB9!!(vj zaZUZ#u^+Qs*gaN91!5Uf?w#g8?3F*C*d6P`IfuO0G2tifbBbSBFsfxktfz~raM_(* zg4EuT?4ljg?D}LLW1xFiaOZp~r#tVj{30$gE8|SrJRg05sOz6#c#I#T@^m)K&94z;gx5J7>YvYMf;6)vY%H^)G>5Sa znaq`kv8>w#12+HFABHFCN^_39&&e~5WdHel2_CGL6JC56&D1xiJFJ@cg$dW5!L^#Z znRSYibhfT^XO+n^&gy6FED`T!gAU(dKc-j+ek4_~6UICeOz8G!^WRmmf`6Tydxe)A z2BjX=9Cm!pT=olOC;d)hje~bG>n-HD?4%sQzleQ;k~N_m$NQPgUB4uUqD@I`*>Eni zY`B?e)_yGMX&lK`*8k!By~(Y%KGM%|SL8EsQR|t_<+*GxC~|j>sbtEP69tyDy4lv) z1m@Ja&CD^A62Yo;TjsUj6n1671%au;7JfC9`w&Glr9)$A&Dh zW}W1P98uYH?ql5xY`Mx(_T03q?2$1n$Lakew);de+xjD$(Xwge?0R0##LV8!L}v;( zLusox5&2`-dG*Hyzsj09tHR=$k!c+q#V?AC>)Awmm)lChf4>CG<29Wejry%@cGCkU z^2&DBEp;w?+f#)zE$}a!;IWz6>?}EdYFd3xX#``%q;Y!RCvbM^whGLCmof7`T#@uE zybvs#qQqA0`o;N}@|qoDs_gl*S{-f`8?j}vlNke(W=X5PO(>hP-br@-xC7GtSr#QALZ znsuv{bnAs)U`ke^Lt0XcU}$MQXWh;W!O@Ns#(%OI;~ainpkc)dEE>ZEZ5D-0uT4K2JAb$0cLG8WL{Pyq(c)r;J69+f2gDYP$#|29vz$=K2 z!c=&uHkp_%OJa#ZE1~VlNL&$B47+~|c|0W%I6v<&k+D=|w##*4=bnl9af3WqSmeU_ zi>cU^FXR@Kc)*UyvoYgG3NGFH1IPW+r>)=S^P)XEeA#3}v>j#v32F>C3=9_={#@a_ z>7+egN}2`gJLQ45y&>K+E|AHK)5%Eg%vP4gVPel~2p&3wFW;cgZ#7AQ%U;Ib8r%}O z%+RAj#p6+XtrgbQjK+KWbjVWw4(23eEfoH}j|$g^bGfhtkvv_DK09u}JBJpr?CjqK zN%EGcyW9~M{SATjCXdABYgF*cV@;mA)g6ztNMOiEgwp>ynew1iwy}CKuFb3znG5F= zXDJO@mUT=~L!XAuTY&q;@?`X|A@~JUv3B|dEOhB0gC07NnXX@$xG)Ybo%urOfThs= zLJ4*cI|){?(L}}mFn;4PWVURu^Py$3f_42av)I_nMtD3qm7mySj?0rl^d!+S33*DTNh%L;|MU$sl*8TiB5<9gu#Ma zv!TpQ0k`~Jta9^xA=C0a>>owIVQ(5Uu`mXs8y7^Y|8~IA_z^s7s39%; zHjF9+PiIvf7sS(>eX(=(QCL>BkKG+53lm=0g49?;j2w3syM3*3TAM5cofNYm=TSfu zuZVLizlpWN&B6M{RVduNO*Fm!DM%Mdp-d))s$nPK;{D$^-?xn{IrxB?zf;7=RY_Pf z-;nT?GxLB!lq9{1TRialQ61OE>)@Y76+I&DpZT~4}mjA{^B8*ESa?tKYy7i!t; zPfobc@*;^D5DEsfuVdfe+3+Gb0tdZ2&#q64#?>F);n&|1jC~S~#Vu>#!QqI|;aW;w`9%kzP8oWB)n_TTv;ugEB5fmc;XNQY*#zsP6nG{RcdWd7+ zPJs_0-e`PEhAj;Ii{1yPLgDx_{4diU4>b*-(=Tl&_qxrg39E$2WB1|c9)iDKsMDD{ z!(snLP3rFZi#429r>ib^WF;fexKPA zDJ@j7kH=%~+n{l69(F63p|`vxZCH5{tOyw1vsIoPjm9S zpwaavtiE~{4QHPf^6{h~hmAo?$1p7Rd4z!~jhHw=m(D9(&ZcNJ0D2gV%QhoTTAer`+x&T28^!;ch@J9=u|%=;R;=Nr*Q<*MZ1Y*!lW z+YZZ$n(@x)JW|#4375$0@aE()TslV!CZ9QovF)j>;_Xs)%<4WFTfC8}UUtWpLHpp3 zn?3%DpvsT82;?o#Cu@w<&Ge z@Eq(1M&p~32k`9NID8p1N-QxZ2(H|uu=jHSWY;IbjagcB)Qesg+@nP&HkN_IAt8%W zI|g4TUqs2q!#Lex5A@vj#-lNtVBn?e#CYdn=5RBd#JyaFa*-L#!p0O%luyI5%$GuT z&hi~A%J4$c?luE7mw2PZ z_XfBS6#^QWcR|Nx9Io!L0V$tx5N&e-W6=bn5wHlPjiS~a!$zmJH{3mRlr;sA2KL3g3Y!* zgcEiTgobJU%tYUZ8Ja&AC+~a$5B~0hf)saHw?zStB`jimm>tx3WH`H@JPLEpEX8r% zob2@&4KJtPh5_U_4xW>Q(a~)pqy6Jp&eVS}^@{_1GAGVg#!E467kR!Y@(lVrPZRwd z(@rjbdIOQi%OPp{17eiTQTSx?5;#1t94=Xyz_Nr>aCK$3DC2SvZ1%VTaS!C_i3d{P z_Tinl(LNWWhP2?W{8qS9aD`YjwKJub71$Zx3@iIIA=_XmIIrA@cWP_ln8{-{t?M)6 zhmT_W(k47~{}=gk-VGMH)x)3<^0ZER9-O3>%xi~)P#aBPwA)quq@0h6zn)|Gn|in% zThILezF_66&*0~hKxoR2$8+lwP;O8?MzS3wAu?603s&szqHxizG);6d&LeuY1KHw+ za;ES*1A=EB0voNXZ0xKf$log9oce8eICUy^)E~!QmCaB;yMP$STwreQPS{=R0>3LF zA+hxqx%txuLe5>pKR$<;wO2H8`|T^t!XaSO+r=(b?qzaan=x_MWr%1H`WaPY>h?{F zj**Oks>;!9OPVihUv7*FP1#^qwTO)vo{Vjd^{C-khzHN=laae7v6uv7Nc$$uQuSbx`55n zC$Q8;c;h}_V&RABC)J zPZDSt3$=8q9ZJkUg)p7WnL2Y2*{zU`w?}>>uuPULVs~L{%Lv#q#S>T0jm5*4hjPz>_aJV{ zTK@XgO77KTAn33Zd7_dYO13U$yE_L_$EWqAVX6&1wnvM4C_P7Y3w0(RpGAVpPeDZi%GsHdP6V8=I@eME~XTD)I+5U+rF#9&tv?Do6P zG;}=CQu!Kcec3^`B?MCylRx-pn+)05WX+?+Ui`gn0{I($fYfD3@(Al8{2h*` zu1oWAxXnho$UKOCNWVpF3nXdBu3-?KbQZ+T&eV000xvn91fy3+z%m(SYQ3r&+RBV+ z=>ki7X6{k2cy&-zbV*lSdg=fY0rT_<$%HZKvUK&s40w@f!!0-av3&IsbUt;DslQGG z!{8`Xx;q1UPkj4-4ZtBuP2^lj0=y$1fi)gSwSy+CzAl}epEd&*Ry*?TjspJvMwaj0 z^c5BApFnlh2@*B8QP5bZiZW$J!xdXOh;y;RJ*D|*F{TMckFVp`IZtq|&s&^v*ol3B z2coM-%S5|(jK%h>zcA8IhdRvehXXOgsLbM#^x9wHJ#;$**5;*PknDxp#@wjKbaOgmT_p^N(V%WO zY-mSkES#4=4O2TcsK=ktv?y1RZ+6~BJlCw`!!#ChBkz?|$;AJUX6v0$OAyD=TnjN zVT6r?scIReis`q>6pv&u@h=i^I~ls+=?3;K))903%&;r%JmzPa(F3c-qgUBBxTRkK zkNaLQuV>~kdr27VZ?9oN6@Ku{st|5hPusSA3vgjX)xJ(=j<7dQ>V(23~ zFCNALD)U*NkRjQ(JdW*?{6{h_K4B_mqoL#LUnJit?(*}66Z_wbW*&ME(pT-#wR}4i z{Z~eI-n}8NQ7T8>^aNHO93)Punur0`GJM)$F{Wkz#!kHvAYs*4U=`blGC4sg>o4TW z!+reqRp>?L+mO5kchOF{6YN6=!-;!waA5HfJiaaokI1iwc~0X%J@5!>dc_iBqZl~c zk&In8ufzN>L$*CB6P>hW^e>l!K?`GHWYsBH9r&L(;%o?+ z(=u3WZBJnHdn0_*beGM~It>%=neot~p^*MC4RQt*vzEYa^h%0l^BzxwcSi<-SxPuSz}Edf{`~QPg1KX!`3-5bQL)z((nuLASRHp|t53p3d(glOjw|rc{wn zz0yhw6Yr91D`dG=%Y{PUTz$-`J_DnRw}Ic>5X=f5jHkY*iRTXdhJP2Y6%Bhl1CJ=G zL2vMRAz$}6yP74*jWfzY`=t+>PO;=$Z~Yakm4v{_LtVuAb^?x%yJQrUrQm%l-JcSPf|(yc_kLlTv9KY;s+OqiDy|=ZmEFtp4JmBe6$%Xgt~BeWCo^uutVdI7sQ&E&Z71vd-%3kg&z_{if81; zg0Ffamfar<>qc&57tQCf2NBiqqsjvVB4e?1eKr&ykHnbm+Nf5uob2ej&6=Z6ptN}e z`!LWP-~Jqm>xEu$$NVvv;b{y3ZZF79IbmL!EI~~tt;7{7&ybrefdgN1u|aJER`*fn zkmw_xG=3qGyQ9gQ^#Vm}inUStcp!`@z7B6>_K~cMwxZZ^M!0d|2RJfWmUgR^iE6%v z;?@1Pz^gZij2`iYOjLagN$VO>ey=n}I48lJB@wJIHXG}_TG^};p(pR3K!z;(%7)%a zgTz=X(2=+RA05sWl!_@{DbFaOK4!1txmHErTy&xM${XX7t~TtH*QFIrS~P3DCr*N` zo_z$81FLb|?|c|xv7PmdC?*x7$70!^g_!SA%3gj9z}iUJNE- zvpgQg*IxyxfM`Z^;0y zI~DAB^)Zms1=P!|V(QkTHp-2i57QsPmcnla{N1q|tbC}3lg$o!*TxYwWn^PPo$@zWTXpR2)F2z|0h z`VZKya>1O#+eju$g8`3SNwE8EB2u>C;f9aE&3`4FbUp-Of7?KJL^EDEv;t2r4Tg@Z zBeC`HVD#_2%Jg)P6<(cIhE_$dM33&Ivietuk8RSh!O<0xGY61{zFDN#^#zVPEpQ$_ zH$nO6S8`(d1{9l*hE0DB3->?1g2wToOt11gkrJ&HRqS~#ZXcG4|LqHg)pBn{H8V6| zZrUBxNHilSnjFcY)mNZT_`kI1W9({G=W7%`kW!Tg=&SYvK6HM7q|PX?J1fD{{`(4< zMZy`!HnF7X>3H8mg_@nO1-tU^Z2!XZY*b)0yx2JsR-L$x+iPVrD3@g5e9jlzdK1c$wzk28%j#40|5&c)dVlr0~#CWB2x%JK}9e{TUc|1d}> zwTdb=D$%Iu2>y#05Box!aOv3)*pn1bE^iZhYN5xnmvrQ$#f1k158>L^`(Z}oJA6Nr z;K03&7&Ehw@ETK4s0)SxIDl?ArNd*jMfBP!ElU2=#yeJ~@O0BCeBK}yrF6LQ7jLdW zWsfX>_2DC${hbGS6Wdu^Kr?K3H5>wmOYt`)@t`Ac&We^lQ16c<_k1*%-jbxeZ>2iF zCiIU6m5Q|S!WEpmUXIH>{tryX=ZF?|p2f2>@-V{4kZ$yv&-X6yp$k;Zspl;jJh|!~ z4EQx3$4^;-%PoGLqFI(R^qy$ z<~-xAAump|rQKVG@X4p`d27i~oUG^!6W`ot#!kY0erU_v4iefMBF~qt_`vq1T>`le zDX4Fe3b&G4;lMF_-W7Qkg5;iK&3YMLaY(?rO9l|Pz92MAxQo5?H(GTW;<%TI#K1Wc zEq^(|*hW286xT|o?s*Mr*F8y4(iV{4k$^fq8u)RiGm6uvvm*Z<5$L&){LLEF>VAqy zK_d*FzPSwl*c>n)p2hSF_rd1li@~eNUs&LGi>!9lW|ie9pxw_9s~0;$PL*)}2Qu)3 zTMxGTzlZhhCj8KrV|aY|0rY8F4^M_Ruv5bafsj8;hTk&gyGI$(`epUt@x>HQ?K%b3 zuBLcygcSWyX3MLVcEXZML3`2j1K)ij^!E5{bVWfkWS$7qpQ_Vuivw}R@(4(|{u;FI zEr8`)@|btZ9pUb$LS01#D@^mkuz6-MY=|yyJ>D#``SqM=TRdkKl_Q}hG7h%Yo`;>g zo->Cd@z6DQ7+)kkl5!I#KF8RE2QDs#pW%88b{Te|@AW>6?Go|isn@_L zREhs^(B@svgD7oK#!)%WB;)!^_POYbaAx`Bc9IRQj-HKgHd=7ov%1teM3-ynj^z6r z@4@8*ZD>XZ&{dZrKzk!64(;E_2EU=i>-7;VxR(Y)M?|5~fb;OG*}?h01F1|`eKTzG z%V8O@`s8%g7sv=WiVtfV;Fsf1ST>HK)u0HN7T_!zWv~I+TI&!Mrc3=eEo;+YyHFv4F1rUDi&Uq~TdnE8%J-vrYGc6_C?JRkI;iVZ9^q|+Db z^2>9dVynqjc$ytbKJ`jM{_SP(Zn+a>8;0}8GbYf(>BFfLH%HCo=fN}f2mY|izzLTW zd1#s*bVp3!UvLJ$t27s{n;c_TL&Gt=_zoV@I10b|y-`|jEfD!|VioZoS9gIZRy7_U zKfQ=^mM_Dkli@6}Z$AH1xRKw4>G)r|ADqwlNt7F{!DdqqzIrIsOPyHExbh$7$F;EK zDy3w()D5B984qGs1lkp&i`sIt5@)n0G~z6NduQHMRfGqLs$6cJ5e&Jp2B(zXfWAewaL;}jQ`1}n^O!D<_Lsui4>5SIlfhPRfv1Rd zSdha3eyx|E+i}LV$>zYi1&^OVx``u7`Co-BK?Ah0I=$e=2qT#2yBWyF zPYtj!<(IEH06t=N`V$;wE z_wAm-dvi^C{XSxB{Q>Zw{})xw&3V1vAlTAmE6hL-AUa9Fjz({}-8)|g3 z!eqKd&xzg~YXS|2YS^67T`Ye86p%Pq0)a|Dv2k=Ctc*3Mi$|D=PK(6YF}sdznJ^m` z`hFk}w!4u@z?kElDkd?~A3w}4LVYVuZt=i`cTPTwe*^YFUuHP7+PoRhq)iuH9TLt4 zcWj06n+L%Oe*lY*SJ?5RJEFi*4wK>83O`cp{wIR*Wf{TS1f0g{W3n$ZnQ_qJ4wuc~4i`n;^kOZVEKu zTqoW>^c$sqe}eDYk+^kB2Tc3?9`tgb!IT{)Jmazi&3~mr?;o3880W2pPXZN*ea>N# z^m}!BDcPFWk5HgXYc=SVYc04U*pa&4k>r&IU-7^0+n|3zhOhQ|Dro(_gM)cB9C!YP zJO8QB&I%=}!Lkc!#CCW-CJ`$}Wud+LAxL;C%h&v;2!WT&Nx<`CtT-wLq-Q?HvFWm0 zSZ*U8rnLc&DGh??&qYLA(7v5VZSGX?031%dfiv|Ae1!TdC}{YIGb&`c#Sbm66k<;0 zhy6xYqrexeD_{ybJ20ezllaE-?8p>FI%x7!Om6RowewWy!?Vwr{(&J>e)m`0S=5Aq z)@Pu&LW2LvyaR*Hd!h7W9!%8y%g!a}f&0i9xF>MdFytb0uuMkB+qcmB#8!^D^ z4;W1|;S<)G(6f9X9UcA@%_l0;x{JTjKvI=s^f!D1=Cp>?;=1!4II~ldhb*I1KD8S^ zRrX_j={a)k*%;hh>qmyJbca{v1M$_zTDU!Xv3O4Ha8$QZ=f&p@sLTDYIKEYxFO)k* zVl0Z72A#!bReFj$<0Y`;Z!FWQ&c%~zAy~BZD*8_=gZ2gIa7XwZoV7j(#oHC=bL9)@ z@HrThNBu!qhE&r)y#MaW&hnp3*o!kkfuTBU$um4 z`!xJBPLjSp(hQHDU595jH*ioxHSA9Q1*^W4lTE3Hydpmg?#skL@=0UtaM{n2`%BUJ z$!M&Mz9=r6CJnpfCqcK|ua1ShA}T_MPj(M*_Ck8~zWj$cb6H zI2*2{Yw>@>)OhInRigG)OYpngJP29vAKZQ;PunyHar-Z_)bg<;^^p(3L0c8+)GtR+ z{dh6nT%*ZTRP?C+&R*Q`Nr_kf=Zc0k4-2zT8Gy-ECs=(kA5NSz=Ni@0^zt@o`XW++ zHgC1zpXc>~PlPh{zBdy0xk|yrbrgSJ+K+vh2`8H((ZO>c44oMP7w0K(-?<8OSeGu} z|J;CITOm!4H)-+c>VMERP>1HFrsKhk2)MT-9xK(G@pJS$7+2YW-%y&`T>1zpA3tL9 zM>G0OLAYbTGjU;&Jn{uG!Ikbv?@ z=p|rwmpZSawuL^uKFWYL{gFd+k9hVqbs|}GnS%9|HRwCQ6z(*X;jM%%==VF1-C-W= zwkjiQRiDBh8jV4l&y)R@tMFjtNbxL13KsT?Odwv1jM^A=t zbC=?-n10;7G8AlmWa*BE01Vl18OQ7Af|1rR2wHIzd-ZN%?u-a_V0%AAO_kye=Cb_P zk!!eTg$&mh<-sEr4LYuP0GPGN^6*!RbjjV{xJbC;Ybe3$9cQfJskx!|0dxWbDfnk)rOy!h-sLIC-QNb-Sxi zJ>HKN2Q4mz!Hai+(GzKKTQwa2#mLZR!p$fczaW>eDsR7+cfO3yf0_TP3 zp(?>o7HY7SwNv1GTsRZw8{^_w8x$nLuzS1;zuzv!cY0*uv|Xp+$mtPWa?@bG!0I+h z9cTnCG?fh{van;{dRFgUj(gEb{o>(cb8zT-cT5uR*5CXIrzT#7HkPnEK_`B;AHiq3`f`djJ^tt;4n*a&%;(HLZ>+WOw7+#r?rgp**h})Eg4XLM?>$ zJQ<$!P=On1bfDQsPyAdR2(Jxtp>v{u85fzeUq7qK@S*aM755AzzT5;2_p5ls;w?^1 z7VL7G$kyuV!Y}umqU0$W+;zck{PCs_BxKa-00%i5lP*Ww$6ZFFutp3Ds1v^k*aX4p z3Al7)Bu@7IgRb5xe8rF|sEPXlem5JSt7R30)yAXe#^bo-s1k*z(p1Z@8f6>%aY}eA zN_gJGPd9~nUh)Jse30S$MnA>*wHaV-_XUP^1(M}slf=oJQo*yJkG$D34A0qnGq=2P zVEJ+q>{w8bmzPdsxoE&#W+sTsKLdVjipSfF9)tTnFF{+{!8WbQhPsGT_&nl2MAaj> zs_Y^z+!F~VgL7d{Ob&jHP$WuWZ8$-kf!7kB<4&hQ9P`zg6a++(uMKKc&EYA`Se}Zh z5uLbq;(2&hH4@BC;_&9}Fr0Nlk>0)V6Qs3U(2^?g38MwA?vyU98C(ys7nJFsLo)p6 znpRYsdlS~X{DY9zVK~r2lJBv*1QXJ4!u7clbcMeLyw^X0IZwpswn2d_jgsUJKT{z% zrxtu?#-sAwQ~1#P3RL{N3)LAnVfW5LqL{Rcn6FP|J4XwAc&iDQ%8;aD!Os`n%cNQVi#d>RNfWIn<&n0T zxn$1WJaK?d8jO;>gmIM-xMV{Z^O3Hp=|AGW(L~&imFqO9=d@I4$PdGw{bktlokP%vDWGtp5NDYbV9$M3 zuG07kN9pE(^Zrh7Oa6pPo6E4Q?>K(TGvr6Q-=kfeG!N`ZfyP%~Q9URayF-hK zlm9zGd+dmsc~dd@uq3suQ{aUv+B8-_1K!PiDR{`{z*DJqe(e1_aQSLz*dgBd6fqbuZ9>6V%*v|3e*lLBSA?ptZTvg;5W zmfer`riN5O{Rh~-k>e6=7SOlAp4%MK=RZCUK`Yq=(Vwv8B7KL|XymKOZ9LPl&m|RK z%zA-eCbR$#ROiuaT0q|i;n#aYZ`{=;({IXCu|pLsfBuB{t+N6(&BJ(o^Jr`p^^5;q zya(IUS7Z8~cCoaX9TuH@3u}z!$uFTsE;jFgldX`rfGq?h($<~HKbo)3I*JeKeMZY+bu&NTYXGzh{ zHYti#|-hz7RTje@*G0mw`( z$c)5KZ17BF9(T_M4i7Tt2iE+SbFWU#LLKcBz{XmTN3}!3UD_E!X z7c35V0An4xgn2OqC%Y=~my+IaXPgdS@~96z9pmuyVsotU<QhS^P91ao~e*GtA_%QgID9ue; zU!i=d8yiyS#LIahq~6;Gbwyt6%c!}K7?w+dR`;-jTLo_5-6p*Htq0|Hy}}KhFEI9w z;4!?q9UJURv7mDmG5+$GL>*DYsGcqslAeT0Zz~z-t!8UB>}QJ#vcOc%2cP9{gHzXv zNW-rlw&e9(te%l8E_75N8WB~@t4f8RZy7|V(pQjuHVW414~0ehjQ`w%15s3&YTDwpYaiV44c8ALxtb{?9>A3N1fsZuWSEjJWx_9E+%X3+hT^)w!*Wgo=B9ya};W^%|?DerJuujlw$?AsT z^Hl;z6KYHvtu6T)cS-(6w-n{|YEWzRd-Qpv#b3&&GBfQT5@6VgBIp17cf5z$mrQ1WksRa4%QQ<2&$she_+PKvuW>o#>n4C}RKaJZ`Nh zC0ThIJS?jRS3gaLmCCa8+~$Mq!-&lB{S9?BE>T;@Vn^^ke0j%k~4C_E~*{Qw{(lUCdq-txEG)o zex428r$Eaa1wU?6HL0ArPdwyV0w@G~z+;bxB=foo*jbx`hqnN{Kf1m*BeW6rA|tIDY9qfXB5)V46i0`*J7}J;V0E@2y>EVqw8A{`STv zi{FdK#j4|!SFdsH8DrQOSV^qq&cUm$i?CHM8y(uAa7x2`Ouu#yOYV0-%iUT~3X!0x z>e@8(iV%&SPY((@c?-lLXwU zFDG89Q((i={{Yk1;FzKi<`U*bB0e>sEFQ26jfR9{bLl9s)cB3L62I`e?0Edu@kjJ;K?)Z19zl~A1u(bhA~eb;z=LV0 z1Bub zj7<*m5VSE{nC;zraHUW15KbKkM?3d{PV92DvsNbCl%KJzzjJV~>wTQ`p%L#t(BUeR zborvqzOZC}Hpny$hxuprz;Vw~NGemp;^Xh2$?YC-@JMGNcT!=4dd$MZxgwb>ZpB48#mg`hv_JhasOCPQ{R!Jm6=P`%X|&Xjhr zHzwO*aLEI5?n@s#yjYiBJpBmU#=K)mE5Bj= zP^xSMp2{YC$M!x5+1^vQ*#8d+8m&g&9}6d3pIIBp-$ z*oDAOWLKW!*@g_3?3oH9zN_*vPm}SiTL{b$c&L~}L;8JsDRi%ihKPIm%&_$#Y_v%R z=a*jS6=py^Qa`mF8re~(o~KGD z`&Qv2nP{vFG2=Cc_fTU>FGh5PVnIbbj#ImgXf61%weDil977ak{S$N}HSBq=KiQ9s zFiG&3o;_kk)5TKMVZA;6JvJC%WDqVpnL_jw3s`c-cv1N=E!10cs&MU@v1oAXA33O0 zh*=KqY;9j7`rUa)AkO6M4ci2Fo*^Di37NAdk@Zy_hGXt zMhkpL5-PNw#LN>rM2T7wJW9S8HVm=>f7gYgo}DW6sEahSbX5YABtI6%1m1CMJ~o7w zi$9dUXNp&pxg+@sWU8#F=(9ImI%Gf_l|#|$_Ce9mrc)%*-UVKK%x0qXi=Z#KwD8-9 zFnD=t59UsDBXuU8D4utfd@jC)+w~%F(Rpu{8QmmG^ywz^zMG@lukWzXL<1em_QSmW z99*_quqK6lsFOvZ&m)iH4vKRet72*wLcVR~c2-t5anm7HxV}TJOuM{2jS!N>|Mi!x;-2&Vz9fxao zZ^47Zees=nA+hK-f(vEsH8i|k4|*J{(Wxd}~krTLVV(@Cq61h>tci&%0~ItCwOI!KqF7&d0gpl{1qQO*n{Qt$bgBovF8$^8gi^*T;$78=CD z{%FD5*a-M}G?`q!DtOeC=fV7Kwb-toLKaV34w*fXXy_G(p$i6*v|Lk+UOEq_C8=;f zmq_e*bOA&EYbSyA=is;A2MAl+1tpu6Fs)S`lE2I13zrz2z9xe$e%J&xXB@$N(GIw( z((SY~e>`@-pNk5;1}Gmh6>7fMI9Cpt;k-k10~SjxAiCr=yBrw~E36Y>h3*$(G(Qy_ zieS4Wcc{{uuM*qbTR?b|7627^(S-Jqx|hdW#u2u}s)0XrTsZ zCy8#`&m>G%nHapzV9W3{NgZ_v%!POL+)HCPks1Q~OxHlR+92@^Egi@ab|1{|>SU{$ z*Ta^%XJCnmFV5aIgIGGmvH?2BoX+)hl9M+qaP9yXP#3sS$8#(2*~5+Kx#2Lr8Xki0 zv)_>iBeKAI#Ro{fZ%Xx6Zim?`qLGa_0_5}^&?$c{N_Gf@g7JwkXv;ZR>zKj3G8eL% zYZatqfGxgR{f}IFJQ*)YXM(!LA2>Sy70Zzh#Hu$X*t1oY?};_RV81(9w>}LcLbUli zT{*u0mmKYJw!{Tj3)zGGxnLq-CJyC3SR=N>1EXa5-xm=~WhJ5nSL7}V_C$A-B1qhq zgQrRUWb)EFzKaQJ} z{IDZlj>&rc5LFy~0a`*GC9A?vxTWO$m;JC~^a-3HGQtH3QlwqcfUHVbjVT@(IATK^ zTlZIjm*<|s=wZUU>R=30?O&ky`d{`g%@zaHm6^ht9xT|Hj9Y2d1jRWK#$96UFx zBYmMo?3%R_1eW@M_wRKuSi&6hmPxavmPO2UYbb17s|w#GH0Z5JIqD|aCR!KmQdphj z2KpWGaI{-ioS)wT4a!yUEXI*GUD|{F4cYi(TR7$iDe;5n=iqLuUrghmDLWv@< zQ{%z5!iW~!8c9=HJb;w!!0yv0V6syo%zn~C%=T}<=bH=3-PYH{*hC8EcKcxQk;yc8 zPXp|%bb^1bwQzi*JUtbyN?WXspo(WGG=B<1AHyYRr6v6C#ZGuqW+E3|apir>qL4KB zl8fv6(B04#4<0{@&C)IyTQ-{~FB{ISp@@wV>pB0stWb08yS9>3;wTscBlLS znk)UrqJRfDr&EeL@Yh+MURC}8d$Q|MzatXE zVkYqYX9n`63-`c^<~%TQ(WJ}tM$(_lU2s;)OI*7DHwM=k@?dK>lyvljk)spw{n&4~ z@!wuZG5&z+m_d3+N1>kCUR<|h4SC*`j1wo<<6nCL!~1g(H6vJoW2HiFgi-^^t(Z5V+C=^ zy#nWBRY;|8J`PG-OD2EH#q$LhaNzF@Jd=}-S5r%HWt#yn@$3sfHeik2O zm?C=gUV&HY9mS!yU*PhRCQ>o<2J|Ls)4WC-`g6c_vc)zL#|ys3Njn_K(O;X;sre|3 z(EWkZ8~@;QohhW)dpcWot_|v58_}45p0M0Ff^C*u0Uni|@agw2BEP){i{`3wR~04x z%$2gP>R1>;Cc>}1p-?bw8FBj=gWAH*iM!y5w)6dP@Yo=>{izIF+kBZN?Ocq@yOTh7 z>q~T)Rg974JH(QK>F{M|3*`GK)5OeXvhl7l`S{)t&&(K#Zb>JJL1q{!k1(OT1W%%Q zUJ1x7J3&TV*iE-D{g0k_avc6iy2GGJ9#n6vDP1zKmHgQ>926Vx!vm9tFk<#XxS(_v z+8izE>51m_-MG0d{nZGleprryBNVy6o)`Q2s2txMZ-*xe!)eQ-8XPiV2@Lq5$0J&M z@MoYBd;cW`gU=>`WtJShyYCpA>6ngLLl;54JQH>lY{8PmNhGIW9eL!k6*e61CxtN! zVZ$+FP_q~U5)aDRDBEo3Ij?m1W#6IvM9_9pERI2m=fPk(u?+si{f73fWl+}n48{;;b}0>VC$_K~;@*=zf9;4}L<6W;WTEp$PBxJVmFaR|TzKFWi}32ZNgQ z*np*jg`Em-Abwmgj*VW*KKb;buJCt;t$bH#xH=HK=UilG_S&<7B}O2(q6wxqci~wg za2XThFvBt$0-ovewsl7Q^{3f5wy6*|&wtCVxeB`I6WslrQivcuO7hV3Qmv$wJnfwVlV{k_6NmZ-T3C{eb^iZeCE>O z+&?uR4SeI!@!&T+mDzx6=9o?txqFPKFL ze4Dr$hISG--98MahStLTf^>1h1Akod-&t7kZ2neCHEw__9~pa7Q59 z@B!Kz)@bmu{u&r?eSIdYut$Nl-Q2it);X5pxo^|;Z`78t7F~H&Z~Su(>t|JiKC4UG zD2%k%(TcO_$Z+C^pFwz1T>Vy;Wkz0?E*dtTls8&BSZHW#d$j&w(Zl-rm-`GD8i9uGcLZ4u zFOL`u8d|gT*K4tIgeMGAUaT{`=AT&i`#{f~#?Frh-;()RT1Ht1j?QeUi+@de2#NSq3}#Sgr5M3{QRusB_>VOpkc<8x#yx*H2~n8pNr{ zvKD>eW~uKu89ptQH0+hhHavEGx1ps?qv1>6DnqT$I)>pajry9i$%YT)<{A}wac48N zMy!~l&4&9;ZW*4PY&A^(X=nIgB)ooglAuwSloYGnlF8b8xyDecX{P=~wUm)h(?`Qq z?76JlULo~*!a}TyRPM7>_DJ1N?)^JjwV>X+thruk#2obEC&`yDl`wgr61BLcG(|T` z@bKMay6)gw_#mHvmE8XGxS!e3_cfWmC>v)lwZ2X2?|ot)46(%X)f;fYO&re3iy76N zi$spsL9DPGrdH-l@zu$>0FmKFfmVCjlhl>JCKawmh8P*@oe+08dWEfQ+Ufj1+a;+3e(oUeQ|_ zTH13V%sdYQJdM$B*IBxt+ZrBxuA}9@!|`QSBlP$&n4Hld;!~pZnc)Yn3 z&+-PrFV`{lLB~dF-{yq}<)778E(*u3T&__>_)_$U*TAJA&uCVs1?+mm!M@;;f5(ewxHWEE+iJJ-F?|e&WI-j z+~TFL*70HQbUNYu6bJ3-NVad32R5wZh3>ZkoRzwJk-cj#9^~>i4QVTW`I|-LZ0vxs zs+_dE>LMKFQFeO`FU)$L#g-jN#k`}t2%qgic-O8?T=QGWLP!O7?@}r#K4v8H_#uqF z5k&{_9atvSh%Moo@OlH2NQ@?+#?TA2^Q}j{-GaBg^ES1N5Fga_AeBlWJ#`j#z(pN^Q*G8Dwa1B2SB(Q%veZY*48E7c!rYD!YBDNEi zXu|DsFZ4M~i#*0jRM<+e4E;nKbJw8z8wdL8dMKXVb(0PnzN`eD+>=5xDs!=3U*2^MgEr! z#NwtqY~!9M<^I|bDQ*TdvXxFgAA~y90I>ME0V9qRoEE)GU*sP`ZgV>@3ufVx&93-u zW`x#yNMU-hJGy_?r0PFfpyuNWs4%_?wP$1D_^<*SViO>>yP6KlI>M~D)%4j=D5e{m z;NnCU=@{ss`)137>t#WDs8*Ox&GJXd*qyNLl@r*U4g&3{t%!^lJhOqYo63YhClf=1(}hDw)NMy!jR)=KW(BWsI@~Z;jK0 z?RTMY|4t}(y9KR&p^zlsjk;WpNJ(i7p3Z-X93mG1%H@k{XR)F-3y zVLi0)uft#GJMmW5N{n1q3lX;WG3reV>GtZu2gP@2#@RZO*O37Ue>Jd%%m2G8JB*Wl zJe1e)8T*S&8`SSN0K2t39IMbAdS)FzlK*ka|0dD8>7kkYNDF8lEde5ht&EDbAIy`qiT;Wcj{-x@RP7*#&67{et8D? z{68K1aV3|SE$AWpHS^&*TL?e?7$m8eH{sl4Nk#{D@-rVj{>L~y+YdTAm9cz&HK~60 z2LD|Cj4kgLql$D1?jAjeS$f$Fm*@p#;of4>f6@>-4)Ji#f3gMxw&$YlYF&cjt)Tqa*lg zFojILH=-G;f}FW&H_4eqLR{U{VAD_`SZ%APt5&wcEs-DO%z<*YMsosLcI^}jc{I^W z$z342AsD`>8FIO_LRjOJ26!bL+jLFvx8^(;<;y3EAFk2_n+)*(H%4_gX`|kV0w+3w z4^@^_lgYw%{9%`YKOdaJ{JYbn{p1hIp6^b*D+%(glf*go^6W!LCD3qb7%FrPG45Us z1JO2dJkJS3&z&yh(v?;;-E^AzXga|p=P|WQw=%SZuVkC~IkXQsgIBeN=z_>!MD0#B zc+V;(ZwsE$otsbN`gJb2^w~V4Ros5>O_7OktGE=qBkm&qp(^yjTGj_W7Tv6vOY9~t z;BSREGL?B0-qDBN~DI4z91Y_v$Xx!iFja*UyO-pwN z9uZMyNU$`$t{4Re;u*}7%If67X-#D7yug(1Ab9qE28LaN=(rFs{Uy5r#`ZmAFIe@C zkvJ)bvs3D;SUWRe*`(04iJe{cn+znH`GbKR=>bU=C5_*pe0fl-qUw9Wv+XVex1>Suhm` z<7bzV&*7~o;v0$gA?=7IU6`80053VzBvhI)lWq;g$lWy5^46P@V z@ecC^wkT09o2q~+SnP#`s);Z#@SHsBsf5vE<+Og?7~Sr5n{2WEOWK>*;J7S^-k8`2 z;z|lkiwHSlU7r9-w`GV}&?9{J^fg;-{T>u%EGCPuCc(^^2-Mo103}26u>0vE2z}VV z+M)Z62y9=#^it2oZ^wJdO-n0W@je8jtC!-!UVn%hQq~~g`i@&3mj&}Qn6M( zIHed2zs|W}W5#06^}#(jw{$u3s)*pisAMobrO4@j%m;7&eMeCbdn6NHYz_Bnn(%<) znRa8kmH#KhP2~=i6EUT=mTBlUR}Q`ho5NB5JRIRrm59Q|$KW2{O>CK#VJa4bsdEi7V4EV%S`7%O&ep-!1QcJ=3#VFx# ztDrWI&OlV?KQcdi4o6M-3z6eUpu+R_Y^Gr^$TllN#?Ek@I6I5PsITFy=-z~iFn{44g^a4!(>A`k~-9PG_A*p*ALL{L{oRFw3D9 zTiJA(tt$p)y#$Nczc@Y90t;VzVBZaYh`jwAkNNf!m3e_!)T~Sf{*R8JDHBcBzQ2VLBnot7l2EJu9!XlTA4U6FB)HcS zhx2cfYgguBvU43asinfPQ*2^+>Bkm>Xs{Plzyqh_F!_=V z?ffP~W|Ez8t)Me<5eB^J7L`LHrO0L_|}fb;SdRsH=6nwAA&-JcN9H5;QM zKR8t3?omcWP9^*m%RwFQSR4^lg{AUOiO*0MnSNJAWJ(m+L4(qmamtW$h+D9|aS-`rSbn3IQo|Gsl&~|S>IQTCc zBt?#LHS#XFCN)FyDfGzqR7L{`*a1W#RRak$Cf+EpQDh#$Zc<(SDczX!rR*axDBF-Jdv*vpE^b(S_!W)iRQJ<7*(aJNDrF z{+GmZ?gbLj@s$x)P)ash-2lI!Nm&129!E@97*F2j1GRN4IsJQvk$r9}>z~Jo@oJ(oA`F=Mx zj_!rA0zuH>8)xIuDCkobWB!{cMmzWK}AB0yDZYC7eF{AV`wCzffXl% z>Apfyxbf8#xl?nv!b)RA)Q7==gFS59>W#QKTpWKcUITysnBZuU6?^C2M0B~y<#G#z z;NItV$b#wD^s8<;$-Giyq?wt-un91M@(EtjynPkUJiDRM1pSV6I#Mk zg$W+{XwtMB(_D9Be>(?%YCa&^`LZBBYR~1T{-WjA9LV`Z9?%bsfwqA(2y6%jLDxO> z=>{3jkpd^w)fR^x0uDHMh*GA~5h`+e8Rs5MvJXVvg(Y{~QEELKyftSSU%Y>zqs1^R zzITh#&PjB3sAn%+;|M9TT{Jbw5Z~1QBPu6yS&0XCqvh+f$k@IXjQCb@Qtvk5d>>o< z&G1F_#OwIRSrGOq55UVhe&!;r3k>R|3d@vAK)%_YT4lz8tkOI1DBMdtbfRIxdJoMj z(4~8PVhoEn3IRi-5-L@qsp3}#{57t?!8x+?z3>(HMalP6@8=L>4`Kwk^+*d@}zu$nSdnc$*kS<9+l|(jV#IemT*ns#u3pQp8 z!A)~D;O60J_tQ^MBjZRXN?N#Fa3%_|gmH9q10?)1L-W?zXyC%Vuidev?(SRBM^TZJ z{nZm@A98{1!KHM#PmQhq^8qTv6ya8rjSyO|2>W7lC~uiR*0?1TvY?D@m!wbE1>VKV z$T_gRS{sAIj*^pm9+Le9pAEOIFJg<#JfJfL^5ETF#s=#?P=(({zL~+O;eCy~EHlJZ zmF1Mb{0TiEvIBy&uh1p=b0C;?02XH{BcJ3^y1=~$t3IzoamEIw`Mm(#_RP z-H6j%UgEULZWbg^~=jN9z-jNE1dE zi@ipbdzER6TN`7^-ZK0P8K5nG6{St$;Y?KqZPY&uzMhUW^-?FM@g1f4ALm19`fs}8 zk1`hI8RCKrDW)vg-YyEJ$##XpB z#|O*Wo|6q>T=q`eHJp;)M%AUy!&#vgnC9}w(JEA3;bZ@ZyEyh&uy|lTaunhO2+gGefaP94mxws z5XXFX(Y&;14AQA$x0(#obRIQSob?4|a-wLfH-~QH^@H&~VdnDQ?Ii2v2ip8!80ufK z;d0=-@z1R7c&8v04{21>jsPVT`pE;ThgX2p{B~08r2*;!vABV+A0+D=(eWIi^ze7e zO!^O+tPZlThJ1v&azUp5TTA%#{wZUrku@@?6*|AT$zo|sFjp7tf-XI2*s$aR7N#qJ zYM&TgJ|PazPgn;PCq?aET)1j$Z+OTVjWK zV|UT}$uN`}@o^UHW5b}qHF~*okld_ajlF7j$lBkQ4EbqM&MB5X9#A*O0O_q@Xk0)< zC0Qik#Amwja}7z}W)JR`y^!_q7q>b}wf4d4f@&A-P8%}wGixYrOz z?Nk_h7EV7NxB(;PW$0eeMFe<}-pVOMi66BvDqaMaPTnV*ojq|nZ$6!D*aEK4+cAh| z7H2?V9h!a2fw3AsWI`b0RA>OUtkg#Fuhp!Nk@C36X93u6NPwQ>)SS0{pEVLfmmLmpd?{~@-2MVU|5N3na`=97-6?;z^c zMdB0p4TK$}@oe90rha4#)ZO9wWiMlKfw&{Kzd2$QCH@in+Qi_{zj(A!n#10QYV?X6 zmv#2R6HeG%qdr$3!04<(>ghI2OG0DO{qAF`!~Q|%{^Y@6FA+{%=yKd>GzVQ}Ga#M+ zBm5No2yu(fv3M^CVc~!@n*Yqgf@61yS9B>1Y4f1gEn(INWZeV@^}BdcMu$OlWW zt{`f&?6F~1FBvlLA{kFpV4X!FeaNx~`F0s_al1y2%j$u3_;(s^z7fc+0T_Ajht^Zq zFk*2qLDwIUX>}FS1E*-L>2=h9Z^&dOeIkKbu~cAtCk78#(>YDftoQC0a5F7I&)ZAE zMRN`e&ih549-E>*n4y9szcDW{fuB8?w!dlm{bn9K->G-pdmcT{Xc!e zcS_*jxqYPAtU1W+;Mj1WfNRvSWhd`{?JBSMYNiD$9~6CL0*+t(gB%p@N!r})r%L>BHaWq zX}O5=*QLT}-pe~Zmo4eefBc*kS$<$ut;;pIyBRt*-k_(VNK0Ql#M*DG*u;=fFFrOz zYD(e$zzI@x(Fn3`tHbjP#k7=rcWAi5i{(a_VZ?w5yOdR!$(Hf3$u9;kIh5eXy)hUQTsz4?Zg2f+kNcG0u;LksoEVvEDWXC6D!!twuahSXT=5 zF?t*cp=<29viV?n=`$+k1>ou7EktvUG~^22!wU8k+fl8G-DWJ!Y~(PRo@Pp%-MnpB zQoEmlzuKwsv;VmMg#gjuPcVAdwVd3NTuGA({V?o|EHp&}>~@U6Su_w=c!uKIb~R|s z6Gi=_5)f13h_&ATNcX#Eu*1Qg9^&{wq{0)%dUhkxsalU_Le@sEK?(TqXD+GI{6+(V z=CPmtsK<%H`4E1N3E|Q@aL20&Ifk=2(ewG(m1oqT_gpcw9~Xsm-Ctx`^h^9%zyo7B zZaC+!Kj;UW;I8LPn%bYoHl68*-4{}6rIR}+F~ALU-E1&@V>0@k^`I9-i>S(;boeK( zUhh#Rhr*;xuqV(iFC z`3?BEX)_3A1)_jf3>|1If$t@=nRbm%8A_sj?jggVz_5S<%Db7iJzvc@2N0uGq=s0|-zG*G##iSV9pf-n!qqVng}IKDI- z*9}fX#=`Zqs;HWbsZ>zU59+|0<%Wq<=J-g@g~Wd0Co`W0vCggqGsd+Df6q7io?n-4 zT4aZByCdOmM+n$0J`6oK?~tLHQg9BxPp;w~RF2kQsf(q;aR+_8c>gWQv>c%x0+Vdz zGYT-z=sM%%igKv?j~CJ|OQ2w10qpfI0e&3^sO-K-?ISNS=1LS}%9$&4>(y%Pt+ONR zx!igCwk%X_j73M$bcUX5Bl*6!o;qu;h1NY0AmIF+rugVHrPs)CXG76&^l=PalpV)) zE!S!M!^5!e{s^@e;?D8rU&7|s=kfdP8ggQv4DF$x$i`V7@IkQ{7QznTwz1>3?dQnp zf)+4V6oww}R=PC0l*+FN2W7L}P<+rFL?3*{_y2NnAn+pWT9XWX$yxNI;0>-;H|E@2 zC{64xJjbCq{LFK6yiu|aNn7Vz__VYU5|7Fef!J~yG!=(E>8UvMT^c2RE+gjYe4uqx z2BnklQA3Se&~UvPIyXE(*Ng#1?&u%LPg+4(H*V7NmfqkQ$q!#&A7X!h{hoc>PnPt! zd?N*sN%-yN8&bpbkh~T>PxO^O(bI1q!Q{sE5V`&%9XlHbnPaM)1v|7EdIbxZRa$xw z?xsoB?Mfl5J;R{ruLA_OY==juBjBZx2TJIKfY;8?m{C)L3IFcXb6jq`s)QkNekuxX zH!i_{U$d}D(-+M)RYAT<8uX6c<@%%2^zD~pH05hQPB8=7HqR_US3m?aiUrAoq)k{l zubPgArjx_24G=H%i`Wdkr|rV|Ai9UEYeNcQf1eygG?cL;4pf7}vN-BvR)wZ6g?Q!` zm$z(IhHs>=6W7FT(0JJms#V$KpNcx%n2aYoBI3cqelvRQW8lGe!DRT^KGLs#m{y$< zWM+zq;+YkXXk#{;SjDz1Ti~0S1=EjxE#kFeOE~n>oqLiqeF4}8hcDUhkB$wCvPe$AmD`u zq}@>@)+6$8?AcPHZqq=7uKXkZsgGex{S%m5dkJs)KOpA2{<58}-y-rK)yaRq-D#l@ z*E2n-jgjl*p=XCG+3sUUPOUZ}!+~7Sv%wQiu6&Hu*HxjdLYzI;?2Vy1ckuS5c+AnC zg9S(Cvg;F15EBdTy}>>moo@!AVDDAJiBDqOYv2djCs82Mz8y~E66SxKmcu{0ZaNn6 zo3T$>o|dr^z;CMyC1b`kaq15}Vx&iV>~G?HkqCBbj~N`@RznwD*2HU~AIOe%qf~rC ziFroZ6Mi4iAZ2A*xTdfds?G}uP3huctbp=Nq_+!kP7Bkqcy$BcLB#rZ~@#5B-8QwN>24e*kn6*bMsz_j@y81Cr{J_pOGwVpZ53N6JY z+$_Xa$zA9qD~J14<(R9UWTUs9FTUP)6Tb63;N61!yMHgl|FSB<#x4TlZx2y-U2n#%F-266 zkOdX?L3FlX3ky@OlVy@SalKP62#tnOyM9O5a`6~hD5?d$`vfTOt~B=0)kzqd;{lSX zJ>;t1L)d*h8wOO~;r!t}#58vs;VuTlMxX8Vin&*ze|;ETa4Co3^gtfu4KmCN(xKi$G91z8yP&f+{T15oA1HW zxuXm-txhV&^1(L}g)pyfEo$x*g3X@=ISUs)0@pPzF!46as8VeI=&{$GPPfmQmhJZwT-0i_m zbjrXNo;fU&xEzER2(0*P%=N@c2+5bzSu)5GUF49??Hu)katJzI9*Zi7s)GjSr@7A!{n!A9h);n3;kWUPwlfUeR3vN5oM zg#Fz~a}J#Yw@4wly5%YaG|NDz1NU_roJfJ)HPrYF_|hwzL?|DF4LR|YXDAXk-B<$B zKG(64_ps6IRnf4p%a+RJm*Flg6L!CR0WS7XhKV(N%(%D)T(|86zO_CK(el-3XPk@s zYOYba^@8wqoh$H$yaBPY%TRsu8*v;6qk2}9n*3dY<8qylS$PSprk>CZCk1dsd2B1a;grLsns3{J(+fa@xp z%Pp~#{q!|==4Vb)tc1{FfjC~c9ZKa=7o(&kFPT#%OLW$2(}#1VuxG6^84X(w>wcBs zrUlK6OtWtIGWdr3oL7aB;11v&`HkBS?nIjtY48lsu?^KXqat0+$Tv$trQ^xqm|_AQ zW53D99)0W%e+}+=G8}bK=A3-MB8`1HBqhTYYbLfZ3MZZszvZv#jQV~&ow^1}x%zHb zbQ|~`vL?YQvp6HmeL;H3b=X(_0sUI|;rG7?Xo%yZ&qfEy+f(A0QT3hd3^br$udN~@ zE~Pj!H;4Aki$xjZvy6<+3*66m6|=vEk27d{0OwdI0D))3L~npbizT9M{Bs)oToibC zvmj_^9$3q>Ff2L@O~+@`w+^Y;_VYWWDvFs2pac1hwJxD0h2caAhRJ*MK3s)_}$S>iTd4&bt=|^et zj{oq1&`;u*&Py#eTxM+S{z+{e{t+Xu^H6I$$xsT}4aZ--!K|Jsv?^(0S6?3@FPT^I zm&Hz4wZ;vNm)v8U*j>fM<`Al1wE}8l$LUbE6lvUIf`Ju>>8i6;XkR^x6BXpi*3yoJ zlRx8#{{9pgeSHi%T5HH;$aWIw(LvvZ|02!_#k6x5Kc~dZ)acJmIq=W6!B4rFctNrh z;-03^Op^*oulr3WbppvUYL6S2IKjTZzF?U9fF3Z^;H3O6!Pr~7Fh%1KdhPrU*IXB3 zfABSu_(`1Tq;6%Oh^F|*&xJg;WZ=NFBgnpTgPM9<5T9`c%;w(%Ua$N>rfMxU=+A*G z+iFQ^?{XM(jVAJ+W^*jp{G`@0ZA7*~3C7JE*oRKLVeRRy6h-zj_iGmsT++$7YP11| zH#{ZFBZFzQ^$AdSR|duIKAaxW!R)8=@V^ROw(O(_*2TW0&+|^uW9yosKxQwEJXJ&j z;@>a?xLG8Vcq>Bgj$&i<1JFApNRzv`S&na?;mZX*I`5edzRXoapUvN}E<>BNH3l#m zdIE{VKtJizxJtE4Y{{JC4)o*V*${4-PnO)PV-Lt}hBv+iQ1pI?bzQn0qWDHBpJqHv z%!$R?U-vN@d+Dj8njm!pn9{t|5 zf^1Q}NK#q@(J=ijxsaxboLS-Q0X-M`C!v(K)L0NR$8cD?cQK(^%jx5!EcE=FfFFX6 z;%Dt=v?5c1xm2{AGevwm>p@@Fd59EDf>-@UC_o#jA3 z1^a@{@-*1Do<#W;xK=X4mmp85R8ubv)#OGNz;@#Twy=G;6_??cfu+nO+YowSW;Q;^cL0mXC<1R? zjbfd`8J#knM!U0$G5SzB>&L4INca{)kDbv2g_2uj2PcXZYZ6L&^S%K6o=*8!chDD? z^q8%4W-~-nxE^r%duqCMCvKMK$Fru#;r$0EY-MbQjtdrO{_8ME-nvJ?Hv|F>DWT@b z9$L`YP0#axW*>KHqUTQ>r!Sr6GJ{+?P--#)`?~m8%FzrN2V>y3RECUyB24QO zYS7%^&9L9Z&$N7<2JT|M44%_SS!R+s?jTNt9Pr6H94j%AA0jZ%C>VwYRq*NDw{$=|hQ=pEG6Vxob7v23 z#Np;ByY1yXddeyee)UI_Lv2jvs;H%mf&Wrij_O-c?$9i(q@7?DUO<%@ z!`3Hjm^&6$635e#*y?>0-60Mm=vQi}Q;46!LdfEZVEQ*T6gGt30l#rc&SgJ$dhPCB zSoeE74rj|lQNl_3;SP%q$X}r|Ug2=}ya#sl3uCe2d^&f$kYT&hoTx{K7|j{Gg6Z}G zVAf_s-1vv^xzjev-fs%mQJPBK_)D}FOQMC=UHZWKB&Z99!4?@&oD@pKzD`&Cm9Iu` z{oRXicN>G7^*S=UJ`jC_a^T@-P3H6W?kLLDlHIu)*vTuwG_^K@n^uKb>$VOK@mH{P zx6Q>9zQ5q0BN<;TAJ5caHN05t1N2Os4=0x~=YC`w_qK+yI>GmS2Wb5j>KH!(@-D}Ttz4uJddp15VtDpJeMX%t={H0yt1yQvkCFJoT@drRm!^G5AiOT)oEK7t`oDB!LrX^6G+CvXK(}n(D9Mm8sD0-2fU_b$ew)yQg+5yFUgi{@x|U1+WK7^&c}F(7 zZy;fjNxW84g%mK5&1Q-Z@ka>?-gEBMm$DCJ974HGu9G`d<;@D3qk z|0TfknFuo9vK=CA-_kOdTI@V0%W*xHKxN+PP@CWuhWd-AWVi1fGCP0|k7z7{nx@rg z9&rqfE%-3T@FMabxk5*;F2*T)6=tLJ6P!&R60u(kIE*Q2bUb^HHq?G&+iBdTl1qWK zIO@Z5-P`!O;WV)d=I72@$7xj2V$k>;gG;#Y)$hzGY|bm8ciu))=5q#YN{s-9^fvnZ z_iCujDxr1_24vY2RrKFw4cp@WFkad#Fm*gQ@TJHbmF9^O?|n|tAhI0iP0a$!q1&`v z?HQg5+em`WN-|$v=EIrScG$kQ1CC|#!W>Q_2*)bmy^tZwZkJ%DX!sLm{~4D5nm7o2 zmx;HYsDkSvS%!yg6mC2yO68dc@Y2P_@OVjO@+A%(coQ zz#lHnc;D}bmkLYChZi5n5U(5t3+EE=>KVdZr;8bTO-SCG)i7SG!ibw*jA?pH$T)8t z;eTsIy8BA0*zh{s_s0&}yVjzUegPT^7{j&-WjI{55XQp(X9QGzdi#Udl|0Y1F%RN0?-8O2_?FN(C0=WbDYBS8pXQLd#e?(ADoDD>!=hlINJ%O&RpSe-vXbM1f%=&J2>KD49A`` z(PGROZ_UZZ4^Fu>;F1`Jx4elowTHlbt~L?n=E`y(3NRnM4JW(rZKU-d4`J5HRV>AW zl5lh|iCDfDCCl3*sjTni#tC{~Jep=4y^b1R&ypjUjFo2l6GP3_czZfcFCm%+TQq z82pw+cRYGRS$B_)FdZg_o?NYa{QYOj;nw zoV`hhGcsY;gde7kSdi@oKEO4D;dmEtVU4kA7u zjuOGYRdC^T1k7-G?0vyQ3t$Q)%Z)FKhPfzo^D!=8o)eC$yG(pq-7O!hlD zc18$h8@Pf*pfC!&6UBo&9y4}-v7_N7#-RL14h7HWf~BA=nmfFK^PH9Ny=Vs*BxEsG z{MEt3&3i$%u?~A;li7mLozXEl6Iwp1;@+NHtWDahjkXN0A(P3D`1{RcR1#)D-t* zZ-W5_+zO}j=8i&nkuj+ixd_2NCm3-ejo|vUmysKP3`gEyHd0)zgtc+o;N3zYG7_$b z3EI9O_*5O{r7wWp*>mymGCtVp$AcEyE2y;J51LXYME5P{p~AxnBaP zEw`r+AOG12N3tBiKk^JwH(0=lWtL(`vJEV&N~5-$I3y->!l0rc2`qfoakt@bdisDb z@Mldz@`C|#{CpZ5R76-dc^Q^;ZNYFA6HwA}!!5|9e4M6;J}AosbF-X7*=&|HeZTq>TiB3_e*Y#t*g zwYCy5u|+7UTSobxgQzQ)d@Cr|yMyosKut1bfS zJMY1+@DO_Ij52ITO{PUW(!E_9@#ADKB$TBYb;E4fJ#_NIsoiCvT@{i zE{GnwLcF&YpjdYrwhwFK?OH#2;MFhI-#>NqE(u~O#wJ0R8y^gXZ^kR!yT7id13X<( zk1xgvx)p4|+czuG)|czYt@q;2TW`|VOu-q2R4!SjdFas%R~Qq;^{k^AMf$le zp6F8+1ofjCRDb?u++7ic*%-y$+nSBxPsgC=$YHYAcm--bU4^$!?|}T!M*2%Gn4NLM z3uEpZL8QzmXILq>i;Ov_R zxS)RwhcYCwM#>sStR?Bs)=?CxZl`AT+u42B2&%q$K*paxrJ5fWGo4$W)4BY)@ZD7( z=bC5J9YRm>b{&h{mD~r;raLg_KmwMfri0_KG=$~kP^S59Mxg0Dq{@q-PHX{Km2QJJ z0a36KeoRx;Oo2al3nV!?gJR7)y8TNv_>C1a96gLNvD2IiOz@zA;VaVOJ)c=Meh{VS z?!nS6XK~f*8fc!0CjRGFz>1Q3^5T;Whws)7Yz*5^->Uj@bwwI+H&O$xn#JUh2lR%H z5FOw<3nnJnINK~AKb$@Z;v%nLVL=~^{K`UYvrxiNzK%v+D?$2QF~e!l7(Ty9xc18-acAztMf?eIVubHY}M+Cq{+}Aabb%?(Ojc6QwOE z6y%Sl(vqM&{0c=cC1C2HERO5yV*}p^UJHLpF3JU9T9yRUQ$Y+53Hi{d|FY=C&XbtD zk;BkWct(@fsL1HJPy2}^u-{&CPMH$v#zlZy5tYM<&C`iOvKuq{Ec6R15ZWjF^Tl4KF zh!*-yd{t$c*8D8C_EIhIcbSU9HjF$msIQr=>`5#5+;m_6k zhH+$NL}V)pAtTx6xig~3%9c$=C?%1-XSQe z`JB&l?&rF$`+A=Zt?7Dsu;c9JMj^``x_oDE?i#S{;P0~7kZ5Z$>leS(hqP9Ur;oF@beBo7AA8nk`3zcK5`bkt(}8!%+(-A(iy0{E0i)$IQGvOOvG%luhSY` z3T(i?*A~Cm=AgS^I))Esz{rnmw0cxZrnl@P<6E}U!I2%LefbKsKNx_&*CVwOYXW z-*EA=0WADvh_Sy7LBV=6Jjve#v8pwAbKYif<4?hkrUd+KxB)KfrsAiOftLtdz@=mYg~{&;_K8D3Oz#zP}mhl|D$}BXv%r3I+Vk-EfiSKSdQlXkp7X*co&0Ffo#t(I*RHKsi zk8UTX{E@Khra!c2t-yCl;UIe04LqEdg7OC^RDoqsZXpiW{6(R|TMTn-ML|Vp4UROd z#Nd7$_rXPRE*Kc?11+H* zSo=E&2A24OBYyxs#Q+%AO2z#L6Y+rK26!+p6_o1Qt=#)T} zmL-$a#5|6>d4j-16FI;8AYJTmg0M2K)Wl*jy*uVeg1&iCcP&*2U8M{%pJ$?Py()aT zngh!A=^%1G6RV!&z}eUXLH|dV%2NXBmCFVEo(A9cwAlB~$8C5Qr=;H>tUEW+D zq93`q!jD#k_>=I05U^V73!;Ah9IGw_sIEJFU*QZ7i=5H9&>a+yufc8MtFT;i6`X5{ z!%Xoq=&CM)D|d@==H^oP_E>@hiBBUT@zdzsSQWBXH-#vzDeW=D>L>+;b&cBw6ouO<&0jIBr#g9G=t+xP zW-VnRYaW;Ko8%IvfpGPZv?8rZ3HF|Y&C0S58K`u{( zAfIRiK8*tF%&&lMH`3}lgVZ^d;=iqZj4?7bUz#UPJ z1=PJBHKrDz{>?m?adr)^T0b451bDG()-N(+LK2Tg93*RtGl=-5OVq?%3QlmJgP5ln zT-BU|egTd!S8@mW+EYS|c7)Koojl~Fq%;+H5Xg=vyzBTLTSQ-<&_&^ydU$?f0rW=7 zV*mI~w(8M&w))*UrfqO9E819r3$~SEo_RS$RaT($Nqt;pHV-Ry=>YGLKDJyE#fkkQ zSQjP=1%HI`)Wj7cA99P77vHCU2k($lJ~JGTGQ{QI4B>*M8M>2PNY+RL?-yxkl$-Z-4OU_4p(;=dQNwa}4l;RaFzBR#>IK^H(3g8& zR?J1OS}m|qFT<|9jd6m@T*9cDEo-?xX}+TTHQ9L^C#PmZsc z^OMZ!8zWKGzv*ueCqctXP42gvrmH*|^dOA^52gjatrK{dstpegT!v_%#h zi4@j2NWo4e8T8I{L7gp2ao-|G__5IiHQKh*Dc?5HA@wc9L9>zaiQFOV*2jd(KBUg~ zIXUy$M zd?J~`vU1kpumdr@Y);Cj+0nW$cEsRb1xK(dg|8pWFm+P}1PFM+Gfh`mJkuR7rFy_d zj(J=Co*$p)^25F)eteKh@B{Z7dD~_L`mPM}9KKI0e%_)d_TD3jviE3Mnl_GJpM^e0 z=fIusYFHql3=*s&ggGdo)(K^}sCuQNnRj)^t?^gpGN#)*NcJ}T61W))Vz$8M;%z88 z(~U~sh@wTFu|!lOidrmKL%+=zq@BDPe1m zb%GlGY9az+lJo+P7(HE1nZ?sJXn)uy;P%)KP$vW^GhL3F%ftc0mP{cU53!TQ#gNtEWYIIrO?@1*vpA zO#6K7@bI@KxW3X3^yhdW8E>N|`;peoZX@;Pr|6Um%VA2)B3N&{2wRSLLQ$q9u819> zlM%nkzB`ljq&I;xX_nCDVvWO7tsqkSBy(0xh1{Fp&)Q#o$(WqCgY91yf$~CUEZqT+ z`_Bva4lKjrGd`eF?~EM7i0e1jAr~*o(ULPy8KLJIV8pKm=YOl?dz+b1xsQN|_9FQ6 z+8pg87lGLbn4`i@0Fep6y}c!1bGj6cTr5Fy zyabfv3+bt&WmLf~m(2 zNIzbqguejksbK@mWsOn$?>unxVz97Hx5Fg)Y=^HzMaK)r(;ahuA0h7FZgPyVn>3W` z0e*c*3|H=*iZ10G6L6&j28jnl-!31>U9%jA9|wYs#2~TIy+Ld?jnE$}M~S%KK2o)? zhCF**PUEc`h}8`b9Qv^gW5T>aN!1_wH~ZuEx!$;8wl~cA9E6SwOGxF9Bjj@I2^zNW zAQ>9n#dx0H*WNQ&$*L-zV0f~&5$pTuq|QB&-jXdPGd?#H*tMDX>^n-=pWj7{cc@@b z&0IVMDlihOisl6bg(nPg3$qYDUa~>IdMS>Ls!u-OVQ9n~bCS=O1G}4|!E0FzzWkR4 z&y-8CU$h8YwMu~fy%F_Hc95AuB}70dj|Q}yAs1V<$g9l9jLw$nG{o`=hN{U zX9q_Yo(cKsgb_8)_&3uLvK5zM$O}c-DkKXXev&xjy9zM7(`eYHL^{%yMAn>0qwf=X z*h4PYSZ@bQMy=!;OF6#ePW_9FUGqt{xUYq|6|F{=U9cik!x_)-uLY`|wv62Yr29VHIU zg3WuCLHdk5mcNpNss@7C$)MC30y{qvJa9o4Qxc_caiKKC6w6`7n=N2{w+6W29@>1| z0=01s%uv4pQx%@aE?$J4z3c5yHR!553DyAtyQ? zGUIFHsAst_nbCchSvr?8D;aSr_T(Hx6AzHPJKD$=p#yYk&q>lSv6KFI5JyjcWQmDk zI_2B4lXX}w#`-Kd#{8`JVXc~S@xZ}Mbc{`fhcTI06m*{Kx_q426gAVv{_~`Jnk%q9 zj__sUGIY4_0>{yxj(;}rnux2^0WdmqXf;Km8nU{RRw}b3;lM2_Hl>qB-o8(3u#*UHxkV+b1K@F65X5K&V@G%hFm_YX)s$n| zr%wTEQvnoMSxUO%3rJ2}Efv_dk?gT^1#3BbxOUMFCqf)yr@#s*YK(+eJJw>hz-n++ z2*86*9+D7i(jotP!wsQ2rPTbhqe0+A-To?-ZkrDs+Km8s0(DEs)M*Eou$$T4-kd= zYP9F6!Y!9-U~lMVte5h}_ER4CvcVlj4tV25??jZpzYaf~iihVWN$B1bNKbF+MLZnYG0FjV9IcX=S`O%h8?%3+UH&J!a#o+ib+8 zD@0>z4+;N$mfmM?ke)tk=E}xyHs)#`@eC}dErAb7sQMv7?a$Cn->#Bxb29O$O$rLf zr@)@N3=BNX;L8>h)LafQZ{|Y${MH!SU(N?NVO5+6&;~h;9P(Z)gs2>_p_z7tMEW47 z1$|J%4c>}SSs{m2=WUrOP6y5J8((1B8`jZ}f245-%Hr!Q67VBU8oQP(1VK|9So_NY z8)jOA1D_u3+HD5fQnsiXW(RXV=wRF{6I8vT2V2F|&}G~f7p+~0aXI#|&&3(X4{suM z6$~kl`N+N(FKC}JsspuJT5wxH7r!;lg9rcD3BFJ|uK$IM8-Atz3ryh2UJWScn~IBe z{UAYMOvWyf)e`o&~L=cSsHM z|1SMrYJKqlX_5cIY=1hBT-hW~*UDKES64v1diwy9+uU&7us@uzu>$FIb70plK;0dN;Mk)FLXO6;|GF9a_AzkUG6mNCSr2b? z(@@wr4Yrl!kjU6$d9s5(yqN%dK4-zAt}Jxhnh($TT=4o=j@|mf4*qUi zihn%gQJ^dab4H^feoZXujoc;Irq&TN)`w0gc#&?!ji{kjgpITiW^0z?kdg@^zdcSr zX3lAK3|8*jj#7c!G3x$yFn+uf^~%;j=(I>!!A9eXU8|s1Qx-JZg`O1TQ2E%XD2_%)J=nfi9EP=OavZUZ^G;T!9(&?_BmNe(849u=g#sFk#caCgR= zr4aqR3{5!w5g8sVT=tQ^e9Q|n`hxhjxQ6{9B}R8?dlS9|V{FY#e{x*VkIWhW#a=xb zPyQTwLrgjza2&M(dhJvfsaIZ)`#ZAGR4)&<@65!c?m3w5B9CiDBtb}D4(-D0>7Ild z+B5$&3A}xh&J$N4w9Et!}kN8-{^Y+ed(?n;G)Mk)As`b~Q1=~c>j+$Ddu z_0#E}T;cNMQV4Q$#I6iiczf2Jyj--1Ov|>V`#zbFkt1_Zq;)nP*PaUzvvl#w!e(|P zIFg-OdWb1F-N-JJ=w^y5Pc!MawAr3_d`yXhCKjwxN8g8>)}*R|J6aVWOIHTm=gXn> z0V(*G!~@5}Cdju4Ud%1$2jQEtL~GQLG^#7oEiMvdo8=-L&gAZu5f)(mXd&)#uSS`q z3LMm}gkrJ@V^*&sy}R^D{Q5BJ`V@%bs+nFHEPpJ{4uWrY6PqB z@{BFmt5t(}duzbKq!vFd=Kim0CFoIk_;uWPw(*Y{2HQE$ZFu zVItYwgwQ zzO5DR+7ODTp9f&3Sun)31*3LA9UAZ5j6uIQ!x^is*j2fesa`3^+>Hrg?{44047IH% zOJ;kKr)J?alvzs(9(vNRwgvR(;*CTjIft$bE`VVDTnIEMz{yty@UP-I={t9q{NC`C zzVCTMgjywF#x!xz8W6+!MhRF|Q$ZzKvgxgP$4KD1F3K76XmBwXO^)QjgYtazux=qa zrW~(!_ActZ={QMh;kX%#m%@DBrMNu85p<`0GdJZ|=?I$SW`dMR^J~TbNSEGEvT*n( zeVi@`6}5u!>6suHZ{kJSR(?pz(1k>0?s=J~f?|%UVCy#vw`@^ITNPEP09E`uYZX42 zh(z()NZ@_F3ac!0;XqI-JPJ$0OZqubI@-+SsQqJJ{1|4xS{-A)mE9!Xr=Ad@H}`3L z@guTlk27o-T?|J*xna)%SLi*wkov8Vr?nokL{(OTZf_^%`j2EmBc~~4^MS^+zhpA$FP(j# z512a-X=LRCYP|C<*>L$jWmj-_ySH9Q*7(3QJ|9$S55S4jKG<642kM=H*lHL>+yBd; zfg{O8b#OI3(^d$7SLDL&h&(*Byb#_hZ9s*{^%zi{2$>g?(CW?=I+%HbKKOW>*xl%( z9yyyy``g{bzh)=xb=yKj3ue%lf21hy3`yeEBuH)2bzr2I^Z%-~@w2cF{5X7xHR>2+ z>qlQPLi9cRtMfn7;dPC~oxVl~9IlhUYqzrsYjjxSZG`(CtJphuiZSUPVv@hVWa|vu z8D%j~IK9^qmT5Snyr4U*%ge{zQ*toxWezx{<)d2rNn)YWOt$TArlJ!qMD=_d8P@71 zDy3(s=7V<97_EV7i_|%fSsm=&tK-&bCiJSE4;}V*=NPVbl&8@jZkqeSf6@LZEE){T z3OC8koTp@i#uHk`dE5n-35;ICZmyZVo?R&I$9Nv=C0Xv*h;P|t+Oxi!417q2sarNc zvUw6NK9LMF&M$|hjovV8!*c9w_J#ESknlve6U`f^sqV)v;`u}mB7UiZqofx4cr5_2 zQb|;`? zFHYrq;qgb!bb56iUG2Jq+?jruZmk%mu2O^aVZvKdtoEJyt+D~#Dl4d7z6hsZwt>LT z19WN85xTnIIBAbKNSXRdtWGY&n)#exTwRHSSHF`NZj&T8ahz6u9w)rdG%#9pE@pF{ zE}5y0x6+HyB(ne=>I>jy1?QuDT}~{_qsW~5D>+VlEEx|Irpc#eskXcAsn@&)Q7K3ko>K4VY#WpnY@ zj|^N|kOMzG^Kr_HU>ut1k6woZptmR(_f1iPuA_<&n5&2t+ZDl7Iv?KzWMQ;=4u~Gj z!*X{K&}$b2A!T9oOBaO%zlZF_B{FnRoEY&N8)my^-X`XreWdc;GpZlZM-r6F@#ou( zn09$1e7I7I??j_O;^PWfFlz;79}R`Tg?aSOSU#0gO(bQH!l=))B+}U*NB;Q+(JL)$ zNMf)mCOXKX0-q?@oByCXyZ$l>76xQ~pAp@uY)5=muafr-cgf7ec6xRx$H7c(q_fMS z>4;Yj5uceyEBbPvc3T!S%+JKmhdHpieH#p~+zJ*Ow&I=KI!HU($nH0|z;2Vc!Ca0t zU|T!il66l9iP_=LG}ihXxj%4#v>huVX@Bc!-q?PkY~}~)LCZna&KFCzEC)T$2Dogx z4ZM%k;qw)ZVC?XKB~uhAuap=unR=RCR6Yw+uFK$FtJ!edRSWMHFQw`a9q4t=bC26$sggVwrGH7g2pz64?bH$(n&Sc$zCdFlGs!u0K|M#gqVJ!|)9H7xTD zht9HCERk6Yt&dE=M#~6VOiZ!PX%X1wmD0AjO?1IP9^w05LQ9l?GAqK9h^;e2kFRkd z5}P9NihVdfk6!^_*M{S6i%9z6l`WlEpGwN))>ByxrsH=I4$g#CSgl8g@o-(8lX3nHb2Ii39lO9y(0Qb8N zEd_`zMdvN$aNB4m7Fw%fb%`?AACSevd$>9MdNx8x9vq)6K;iv?kTNHL(?WxA@J0|M zd@<=*8F|)JZ~r>hz4s8)mT-@Jd2oZIy4;~RuKq`q7l-3T?LhQg6U6zv;mEg|^SKSV zIU;Ea5|JtJg-4f)PA=x=@kA1KxSUR2H-slo&B4so0y7NsA*JLjyUhOwyZj|T5uEjz zjZF)s9!15pesLi&=vqx(n-XaVDWx4sYq*(ZHJ!GnpEO^4L~b;3y7KQ|L{M};iP5hi zPO)uNSztREJC#Zf?kXm2GPTr5BZSy@#lV%zkzgPniN}|%1}Up7?B!;%vi)iBZYTrW z|CUgv{*Bb(Vm=8JDWow1u8_IKo@38Dq4;zcxXZENrOSEf`2rf^T@Xa*2GN5iN618M|4OT8GpWs*v4R|^LY z`hfn^Wia2t1LsV5!M``Z$sgk}vd^3k!$1BbCrtj)UAG0X;N2h*F66`am%lI=^^4K@ zaIb@H|H1ssGe!MF^RX%04EF9cMlVwbD(v;09SF%IQ|C^j`~Q5RZ?oT1Pd*`7Gi{R7 zCDQOsavCPfW9w%+jWPEPu#S=HVuD40dI(Ez1lxw-Y@5je!kQ%;45- z1`~JzWDDnkoQf{+mg-=Aw=SGtkqzfU(_nHe6E$_S;iSe=5VYmkf8DlReUTIBr4*3A z`<9c*i)A#neI-#Y)M7ryZeen#WU>WazsyaK>%zYEnjjvbhVgc~z%SZMtHw^!87~hL znYd$AVtFY_NtGer?gH=%FU7G(GI)w_HeNq69saeeVCLMTWaZmt;uBm(3yW%raOV;_ z5adBK5;u~S6^m%5bRNnvSy&mH1CC?4*tfeH^JkRdYyNW3kgG(?Kh`i^3gF>=D-0Oq zw4>@K(tNs%oO!>UKAxy1yu%t){eu{lTdP7I9o3=ZF6Q*tL2Y_2QimL!GM6q&R>wGT zRkU2D2gm%hu={Z=9@rC(asw+t?Zg_q*IxiPCJI6NeH>2lD+J#sd?-1SW2avhfERrN z_(N?DG@EI_8BHym6qy6ozEk1Ad2zT`AdXu&X5{YN0H~Ym15;Q}d?gnEg}OW>Vnz$2 zXY-gHt$V>dG#lU@F+Dg#3~_2eGxeQym_~hXC0BnOrFzdL z8ObUI^F5vN%*$_~%y);cOhaxFQ)DPfTQfTtEs^=;;_P*VcaaeF5}ZXO&K6+CKt8rD zD}dWmijhYr2FI2}AAm^~ihUxqH5?Lhvc3-WB4 zLVqPI(#MhCnWKXuw7N0}uHH+5(8hE;9>Dq0Qjz4Vg*}Ow<4kkAgUBT)fKK^*NU_LA z@}~%l)--_Ye|7NJwH_CRHA1jwB6`NHMuW#|;IqUA%pr#C#v^;%XMd?0>cC2IN2QRQy*Wa*uDFKYcY}XE}DfaZQUL0*S zizVJm1L!MG-xUA(nCeZP3S}OOxaQL=kW7;Tn?j7oDqER(961!<6@Yls)gT_=?-Z~9prU|37@@Eox&K%QwjIkii9OgbU!W4He ze7f8N&vIHxx2HR{NKTMZd`1+%kJ4@Dg~0PyIy{tG3*oIB(AzZ!%0!iMfrly{OjiaS zDuvU&ey2tuU+B=LuOun$FFoq~mC^jb5Vced%Jf}lct460s}Fk_=O5FlNtPAqxc8ZU zHyWmR|2?^r`G&^taD)9WEOhbF&*8+RG`>L z57Dd;{Ec(qiC!M6#utJOX+(>!Td{@95=hV4j&Wm~aMY>-7tOAQbCuOt`Q;76cb$(E z&wav5WmhuOcz@Gv`CsYKlL@kaaFj}KGQ_CVdDu{_2M6O!aO#z{oG+R}Hs464E>)T2 zc;qY6D1DO1fA6Oe6N(UDpiC+veM!c>cdW=ZKPKsN7#zzEg#Q)>q2b>!cx1*BzvC=9 z;dF!=kF=8M+gCCxPWPFs-`v95ZBAt_?z~PWFMlPDf>&t9WF84OS`K%GJR!5n6D1iJ zxFpv}#VStH*~~d|QsO@b_y7YKgihAo<2Fnl$l{gd}{ zCTn$NM_>6Ob`NhXHikwaFXywR_DA98sz)TWd4$ZlK1lsyy2-AlV9+V^hMbiFIJ9#G zROxEa>xCEDoKY?Vk(@+RL?W>JWjwx*je_2CckH#CO&cQZ=_fmT&YM2X-dZ*Xf0?La z!(tU^JT8xagc8uKI0EZu#6$e+bhP*$H^CQedHDHj78bJHv$ZrEPrTkn3bvjg!=LLpetQV%3h+VEZbz)( z{G}YzAiTmJA*+M;bG~XPEoyHkIbvea`%DzB-r{(Q`FxNcB#-xSDvFQK;Iuq-lzQDw ztZFusqFAIK+FlUolA(ulxg1ZeEIBv2k~Q4S(q~Km&{U`Egh#-ILg-EMpIbhuT=|&Z zTRKV}_gv`c*X-;VmE>XSjL&pDIvt1`F8km@i{)^>ArL6+|ttj9&CACZp9e zp};~3mRBj^P=hjLS~pPF$t~2*a5K4^S5Mz28PK(NO=!|FO%k@tlrF2BFrS>E1~S~fnFyF~rn??*Cw3QFNPXO1a%LKL?%KAJ5?>`~l2(NE8dx|dVbm$ zCqP6mO{Kf-G_g8L9hcf`fWkseJUdq#jepI--xoCCz$|U#xtxIm0;xEDG7X{*W?+QT zb(+`RL!0w@$?52uly`nSR6bk{^1IgJ&!Bin3;RTyqlfAFgdq~3|A`7K?jvfrlbG=~ zQrVb;q%&QYD~6hfKRk4xvs)LZ)*92H+d6dDs6M$bYDA;1>OY^f~9c&>rxcgc7hE%q#&_U6xvHUo^XW}80Y?C(`@ds z&JUh5jXVFa&1zy0t|0<%bwu!-q8N;As;3suYv}OOTC!JY8@(&9&&u>>u<25<%#9)i z*6`{W`7SX=@WB{u%o`&{`k#sQ*&#AGYnZlJeIY?l+biC zd1m>DBnQ2vxSWq(J|;$YQU}d0%B>QUhcQTZ-gdy_J$9JcV-KHA9l8F_0s5i8pSpd0 zMvk0)O@CWgLD|(ZSf5#rRHh1!EfIn}k=!g%!C84Z)_wJc`&1d(NM-zPqy#r}{?UaYG96|IK6Px|&qtRS zts|B%ZAroQLbB}yAWu4v*lR6-2^ARQk-&pD_L6BkExFwSJ#?3RM^wd6(yS;~%pCB< zJJ$oDHAWdHK1q}~?;3GZb!5tu2`8;R(za(Zp3 zfOPiM<9)$xI6bBof}1y?CU;Lcb6y)&j1obkKL-nUYNAyTw-Z#T1m7;HpsXf8{xu(_ zOWP!1tA_y=C(45M3_e)=c!GxQ5C$`gCT9A>X=LbEIXk}cEOTonOH%CX$OVtjbTCL5 z`djyrr(XS}(q)9gwMjBkBF*%f-D3P@-6_9YCOLU^3c0#`j8XZ;vh_x0jB_tPI3E*+ znTk`e;$#VN$%;eo?6vrPGzeNl^s(Qn2!i((!-cDbm@1hIW$Nx&+wO~gZ@5Z?(xY_6 z@uldSw-{%ywSZJ_26yV@gZY76(DTX1-{!e6Q_LBTp0Ned3r~n*iCo&zOH>wJ zqzVlUjN1-hklnf*N*{3WZ!AKZ+OhTV~+XG%tLph1$E%P&&+=~Vs4zEi?%uoQB=+r zidF$?DJYqXV-cHV&_a_=8-j&?IrJSZ0dFP`S46LX#wm9Q&&?gAuBnHG2gyV!ql`La z)Y6CZ7892gZ`PNehisNsBHP>v4dZrO%nD@4yaj25UJ~ihogqmlB`a~2ObIq-ltX^c zCTxk1BIm_-5aDtGS}1&ty?aU<@6DKx@{T&7%ln*$gyk?xOf?8&dyZA{*~&_K*yBSp zYxKyq0-;1JjF~b@m+1&$t+E~{znFoV=PK#N*aW)C;X9EzB7q4BubbANsaXO>q(-v>LoE- z_l6Gq>mkH7ix~U0l9ad$RIMn3Y<2oV_J5lKK^>3iN!71JWK}U8-&{dwWt}C;dy?qu zCs{Nx0_pRm+QgD$kq>{pLRLzACKApfcsGyBQrH{9g9DhUnBe@JVzcs;rV?S&gbpyfTrI>eUBb->!2&EId@pEGh6xW&K z$OU=a(Ps!78gOMvGUHyHI~u(Qqz28-pfB#66DG)aT_5lOW0c~3g{STb@~6t|5` zf%cEriTsfR_>?qNJGV9r>B!?9CzNWVv zqsjT_(kLCUn%fmg0YC0ZH*uh&t10aH@=!Ut!?gyUDs45<3I9?j}c z(f7Y~DUIzVwacE-gO|LxO#DV%)}9M>zcTQ$L=K5qFiK9RaWukg0q|6cfe)+0LHyTh zjLKODt=Vf(ZGRf(7e~O)p>#Yi69~7@hroGfceLpW2D?5Z^7mjBb4&Ip>$yvsEc=*F zWaGb+brtWZmBbB>&8>=W7IOak4tWr<=kgnswj}0NHMtbiL9u%e>2I#afgRZLif2ywZQMMXH{Q3&ihw+Iiu;fKLc zDNfH8$DL8a5dCg~9@suX1tvAgtas8kl(M@(V1(#Y2T3$;$LP*zvVw9O?KBv zd`mYy^yvW!^aoRm$I%XSYaQ_U zs!%v97Xn4{Vd#A%9M)V|3{x&LVB}+g%%w#T<+p^`%h{1=kz#uEfC!15w*ajr^ick_ z8NAgq!kU`-Oa>dxw6xDs6r9lmmi%S#arQ7V?DVf@zT| zm^_~ap1WpaMA%FacygC`II(2M!B)!K{2%#q7pZqhAKj92k?`8Tpn2L8^r6OYdPZ)X zv}lgf&pg-Zf3Hr{fm>IJ#L#6rg{BSUnq5B=*b*bs zS^J@a+!Jzm!!U6e?W4Bs&&h0wBh=u72OZWPAPRessxA;E!A7E__MA8q?I^*F{GEmR zu`=9loFuSkxZAqo2fC+mkRA^GM(B$X+WH&d-g^u1me9nMItEHN*x-pW6NKmUL1(@( zw(j<(&mKn8+$Iz9qi8V={VxKw{R7dZDiofckHE9tno!@S0QD9s_>0RRD6vS3SD&N8 zHxClILkH;Y91+kS5{Hjv;#k=y0dspd(k*B5>29$)a%ZrDidJoepPvdrb9)i`Uf&2} z-MdNWs~U1ysDVbl=pX@o1yHi55aO;CB0MUFkAK}D>#hrAXojFjT=@SeI`2R#zdw%K zGLvL0Sy42|xX-!g+=x;sqe2=;8rnrkM0Qq4Nl_AI7D?Ra+;c8zC^RSyq$QPxZ`vxq z=lAdZ_j#UkKj-uLyk9Sncw`@TuGo%4d0}Au?7g@?m8#^ z$*Fg2<(+@b_x;0c+LABq6E9_8si=gTqO5`DPg|gnX^lt2Y(d1DKHm3-zPysH145a< zS9yvhGR%Re8QdId4EsXggIO^57rSw~1n%LU?bF<`cxv|`>p1w8O||{b+~4+_?ba-0 z!=jI~CUM7^(wj$Eou6NLN!?d@r6Oygjv?gX*i>*Z>JWIcE)`#kPX(FMb3xgG`2gNq zi2uD>2>5H~0j0QkAhW|6Yd&=aFLid|sSdlbbmUra_rDN)X;BD(mo|Ww3t{+vb_kH1 z%+r$Li}tGX5lAF#oN)45%we z{#xcx=N#bu1%WijIk@SGBarO>%qngC!diI0W|U5RV8^ZdB)=|&w&awsktWBPw+}C}X(J=d`91$Jp<86|`+jM#pWCgy zxNruJbhiTUH_XPa;IH;!JAGZ>5O;|E$)=I&(L5{dr-Vx`oL?A_WF4kMP9XlI`;H&;2!1wib zTr!L6(MxB7xl_|{X+xfXng_9M-TAzvu4DUFX(=l^eI>e-feisQoUyta=+*Qyzwor@Rt29Heh?^^M*s_-eOx9o7C0SKz?SYB*hzOBI99BU@x?=U{qzKUsX88{ zjU?lcp-1+Qg(||tNdJbQjTsHIO1NB2{9$nVR3_foe}u~ig#&BVP|)tb2Y*i42S&CX zW2L-fS#xbk#`a4oYc(hhPU}qox2|g8O73pmt7Hmhgd2nOI3xU2*$9Mw{LG68TF3jL zL4}jvLuTu{C(O&J4(3PGSN8hX$IRHW7+`%o2KWv};ah9sKxj)U?(R;;F|U$Ar*8_@ zf3XXPehI^U?{|P1j$zo(&>E~%o(ei#jPY|{8}Q#vJKP;)hu8TzfO$*pvCRVjq_X*7 z!z4$X#O=qbeswcFH+z_djeYEt>}QO(b2Iy>yot3LXka$4Z(x%b0Knhr0(!3V zvEp)PkTij?hwgNPctZzl4V;ltbXf<;hgrci^sfA=r0t3z)WX7oKJ6 z1)4t22jaLTIC0q`u6A7t6g0g+x1uMWe#09`I80{c-lco*g@P0nR_1{nV-ojVIDaO>b2bRDejg9ya5cI8tE^JGpN$D|b4ad3ZmBrRNT_5~FGCL;VaU@nZ_>OiV#Uf(c+& zaTyf#vzpNM#P#%jT?Y)dD%51QD|D0)9^+4+327Ib#Gbm2o z0@SR7fy$@#_322p*IDZL z&DIIv*)0tW>lOg_eiy(@ork|qTL7Lm$Kn`=12|=I3{c!1i>C+wW_B$aWPB=ySnuiI z8Ae>dn>VGEcYon!A*?>h+Z7N7f?sU~s^7L_4ec;cA^DD}Rei&#Z+gobO1@=^pAIlC z8H0?+{x57)<|pPR+zGsFIo8y~E%@E&c98S)n1lSQ82eJwyLCD($qojMfbIM3#Aa*} zFn`Y2v-t*d@EVZ=Mi+SCoecM$cTd9Y()w60Ru_crGr+n2hd}R(1b|*}{4Bub?SHi} z8@FC#dcE4%7x8zPwxzb*uKYqqq%@!XBIGfZh5^jm#2|*oZenlDTF+Rj9bs1p^_0EKEv#peTMA|s$?!*oCV5N zXM*UiSvam|78u+S1U~RqfqzrhU?qnj5VSWCi2YZAl)5$eMt&d&($oeOKXpJ*fezkV zq75{zzT+jGdBJ;dHT*4^a60k?EJQ8~P9utncT+I7f)Fy1V zwg%aYX8|=6YrJ-;HPD^ygZCd>%FSq&fpOey%8IPPqIsk;I@FYFIbakVa8Al^V< z%MU;eCXx#N&WADhItsnzNpCJYfJ*yxX+oDR`c@f02UB}Qx;u_je$_cT_uxDfo3M=z z3=~t3xWD9{u$SL0b%lDjyreHVo^bS*Pc+9uQ%t2#q6a>K)YsaXF8ie?TBYzxG+v{T z;-S|tymT?@k=)IHG=2hd%9==z?v{nyS4Tt9=9@@6dL0^R@TYt0>JeGUipB|IkbK~H z2K3LP`U71d8GEDhe34mIym8Lc4|E$ezJibbnqa=kwO4*@n5u zURxJ^(l|*U*$$JReJyn2kTsIN_=)WN5l?bwT%cb|kMd=}4pGL>^C2LnM0?3E+SR8r_hFCIsqhA-qh8mu7x-W;F!{uJugbP^^k6vL9NS~{XWj`jsSBpj5K zE*Vl6DD0d~v#O-T$K%t`+^IfLdj0{DTKW+N9N<3nOOvR1#tJkoYA$-Qxt%UNcnmd! zp5o8g(n=L~tB@HE9K&|+WNIAunm@+Ap0B#zo?O{C1Wb0j(3LVmG#J%SraVg`evbJx z|B(xEsCr1W-1fu7o$t|xsrGcGp9eZ#{ROqv2hrSr*Xa|-7&vyD4!ZeZEV?vD0@)-O z(d?*sXvbe!l%w|mItCvS>C0&%k2P6OdWmzu`4QM6`U5!cfj}hrQEC>7Q(lS3F*{e|#13Ta-Xc2J(ni zd>YK3Ux1<}KZC<1y5fwAWOPRFE+L(@RHoLNP7~FjzzM08BZH#Eb-tXNaRKelZO7X< zx8J3^6(Y%H8R)oIKUK@`AS17wpkCe+A|ZH126kVdclw?Z1>-C_cjz#3ZTu`M4^cpy zd@hhPvt4P2+$%bLT?JbHxd(PD+Vf>A?cvTDsc`Da&8Tr|4eZe`ryhBFC`RKe-|AT< zxj8r;$?v#1V(&ULqTK61Ww5kM+ zAFGDc&xX^QEtg4=c`KFen#_Nn+Do*SjT5LGJV)`dpD?-h5})q&rxR*tqsdNR$%9R& zVdC18^p)CgQXBps{CeaL($f&4t~r^=TGCjs;y6$6Ow>XfM`nQFgV|_6Rh=%@zfGmo zb%j>f780wC22g(D5n@AlNRxDk+?jV|II>t2qfv{b4rG(wGXZGX4;vV@e?Q$Z{y1H? zt%D{V^F)cO6_A!u8oHu<5o!$ip!#|3=*X`hZ zp~kcz!HZ)S&!bjxN8sk)e5!pn4^3-TL7ovdXe8N4G%mqRptPg`h4BUStnfUH($S(f zXXzqQayL}xc)JP@RmJneO$p6kMk35BQA^HUTJ~Qw*vGm5l4omB-LX8XVm6uH@-(67 zld>qN8&Tm0U-*9SX)^omG=75ZDN%mM6*_R}x#+EaFTLT~OU{MI(t^m1(CFSor2Acs zUbtx{@G1vr=E*=Z>BK6yc6KDW4uoXW@I@l0XG_&YhhUFfCTh9V0u5g0(Gu^|G-Ak% zR+gNB!OL!uEejW*318Eowao)KxmptK{Mv)Y?aQV$r_Drd(qZuB-$J_djym+Tut&*9 zf4~*HJn3MmFFK!8MvuF!q0eu=7UgG7;6D&%Amg#hg7x|FH15(wq^e^=^Ue>V^1*P_ z5q*@}I_Sc4)8?Qawg&m+*`WF<7uZ&CcE36jMUw97?O?D5wgUGq|-F}F3z+lD2ue!z)-sr-$<x^gePU-~zcZ{UCHV_(6o*)6url`>C!%0bQLXiI~zDVySwH-Z)#pkI6hv z@yl~=4&9U z`K4r4Z81`BSD;3l&0yukL~?V{0<>w14ZX0;6LraE(2Lsx$cvdP(etEjRPI79>TYf) zZ|YXh}z8Ysuzd1E_v+7&_==5GQ4Gnr0{g zn=jjulKoex`ZZ0e{wDzKTJ{@tO}|V-A6!SDhh6Ck_gUooc5P}>dY9%Nw4_$c;?Rv( zxuQnTgT%~v5nSLimPUN)LKA-rQPjvN8h!YI);XLY6c3|bqZh6a=O>fti5eqncwi=aYc!wg^yQJmPD_!S-+%nS!9rvlFpGK_ouYd)APhSl z1(Vl~htqdi(gpLniOP&}TJ!oUdT=QXyqs|xX>HXM1uj|7Ir)-l+4yck^-uFHXN9A& zAt_XOMl$_{o5=SrT`H}*g|1r`KuTrCpq4W%y|O@s&d~+bC8G!$G#@3uzHFyf%S+Mt z#V6^x>`M4{a11RHuMtIrX`o`x$NYSLH5}lz(RP?XC4>D@fo3}L{@g<^jJ-wgOC6*; z_kZUfofJf(J`GW`9U`RG(#QF*RFU=bD1Jm1=R6oMCWlN> zQ#2~{;qNlLjTV&7rQ=ffQ1yy>+P^QK|03upx%ck_`DwSDs}r6QGt>RRVx9-OEy$$j z4P8-O?KX1yM=^pYULtl=JDA0Jnl2@JBjhmwzSWvcK3gTjf1l-P;kvW@ZiBPvj&?9S zJ@qiXt&3=fYcBj`t1XCF)rzt$glOf@CR%2Y1Is&d>Bn>bM1s5Xkp$<&oUUR+5<}wo zN^`!G=bYqXe81WRkP?2+jRr^s=5dm*HMUls-2WcOd6g)Xw4&@C@(7O6c%3o`PtRqIr)`kh_ zt7iqIGD3Ld+d25OQ;UY>$e_JuR)RXM1{iqWOHfhQf$mIxfc~L)GJ`6MJG!?bj?%|* z?N?Ec2e;|xrP0XH=QLa$GM&!MT1@V@n4&uhE&PBnMpU13c1cwflg5+FNP)f!S{iy5 z&D^#DEtb~dk8ZohmvKG`Wu^|&Rp}Z?tw&n$Z_Y11kuD|PsFvz%eg|Qp5%N^LOciFx zp;eo=)2zS|D*5sx(s?{gA9>i)tJgcBVwVd2X7vvKi#&qVO8aP)cNXpInNB>G=c175 zE7A9${lsSwPDp%9+og*UK1kCODFAJrjqVWQvHvm|-F#B}w0` z=%ZPm=OTAM?rcfENF{j_=!nQ!U}V@tB~?@4iwWbXZU!HEihaaomJXJk1k$`Ue?_s#OgO67zL=;9BeryO59fb=FcqD6xrE<3X(PQm^$69L8$_GrMYO`2b9{VzgtVt+ zp$A%WbRSpa{@_Mr>!CKa(>et~R02|Hl7Ir4B%-ta67Q8t1vtQNCIUlGN&3Q?|-kUmxK!`rp@+=q#KdUhc3MsrhE1)qVTeZEt2nUuFk= z-5w9J<0=2q*#u%N<0)!C7f(@a4Ut-uLNAKd(bhT1bc=ZoQVb}8xrZO4;76QCO*}}} zCg!94mHkj+)dQ$-WGAgG=s>0hMM!_QnK%fYMsE(*p#YaoQ3 zoqMU_h6w2KN(!mUZ9)>@Hgx-T5G~3JCYzKRq5jY?4Od=EPOp!p)0Sa$Uqw$~HOGX0 zyk$>s$L@gJ%G2OEJOTBlh0?2M{)4R*LfYDP5P_e$==I4dNYz@Jwoa*~%6(R1dDTl4 zMV5nP6<;dpc}8?CGl*(Lc9HweO0>^ef?Ma)L|WWgmK8FNsK1&{3t~>gAFL-Wo{A+Zp)rtgz7I3+d!o2a%i!^zTmIgRRqv%)qC^A|NjpQ#zsqf5)Ps}D1 z+o6cmjSr%z*Ime}GaT(t{R0nlZ-7b$kx24DwCIh>7{Qi6OeNBLV1I=WOmF)52_eV9ys&$+?Kv-bo;O-fUA^mfQqasJx{!+pOqE>9NG-(jA)68iTUh4w9cw zilE$!ZPaO{8`+|Hkxsqyo_tuC1>FK)F_(UK(v(|5BA0xX{HidgEp{*XNLmuTX{YGw z!UOcWFrCiS>f-O;d6YWV<)NM(QAqZKD#|z;hql;ruCikqWOBZ-AVZi(ePANBn4-hF5`P!ZAVhuz36;}QvT}dByQ-aDY%yL5$f4?Ak{CIX=&{v6#wKX zgclFM%)%}7=*Q=zF#ILIE_5^c5@1cYnJWnLH%q_=74h`TId%HP&jpRO>_#EsN9dQG zL(s65bH;TXhNmoi;k=7p;vdV#p*g%%eqy7=z z*?xw;4U4B^^_Ic>2@{E)TQwP9^+n|M{SEYInnZO^57T2;bBRo!7v21GKb+Ccc_h2y z(4PhGNc!GxdXl?qmCu$%h0Ouvvr&a8LC%uqZ#_#(?w61SB}>s!9eJ8S|B2Q-Ttsgy zHYQhR^2w%!AE?~UJmhoq0lb&Ai)Q{7!|RTR=u`i%2v#Z3aOsECPe(w~dhd}d{w1ix z@F6jbO+@v*Ptb5_8d`N@gy^frQjfG>h+SxpQfvoNX;Km#GWv+~KC7aZDhm{VykW_= zG8BCHJrrM!rPWqvXt3#VzQ89L`Nq4@yB7QC&#=$bE3J{F-;YOcBD#2*|9+F8=fkLR zixFyynLxFkUZBM#y-c=-JDl(JMKpA0I=?1XA8j#K6vyR{gGWQBh~HJcgG+r3#aS~q zqTSioiB|>>O}3XrVb-&$ON=|!*mjkU`=*G}Uu(ea+s8sMR)>Fg-c^yo(EwU$ERXb; zPN$Zf!+CvW6y%g8bf!f-(z%Lh?uy@(1yA|mGo~Pa&J(sSM3L`z!;D@!_8PVn6w&K} z58)|8DQeiC0AH7k5%+ZbfzhQ((BsJ=xbCI_n!-0m;|K1NAn$b4k|d&8P2J>EhclH> z+Jo#OBm~|4tz_4sG3eOZJ!nRkGP-p1JbiRH4&93#B>PWR(%G4cAvWr zr_^Q;&AWHWDerxB)oKQ3%2t25NzRb3!HW&`hp9|P4RwzOPRK}<*F zQHIGWn#}s4=uSy-)R7uw0oI_NZysp>ZwZ0d@GaVEI6|~!e~6-u;?U}WGK3%6LHi|s z@bS$%{8hP=(C?8blvKP8c7#JT=ipJ=dCP>}swn3>4H#0rvSqYoxgFixlSe%~(uqlL z5M8y@h|c*HhTQwFlcb3Rcbqaq*LPXb4?+on@Ru(7y4C^hy3s+8Hi}Vm;CWIglSRuH zRqQujT+MU7CmwT#XC#sP}={$|L@e$r4;_t zHx!Ru^chv8W{^w9OQ`%Uu4ma4gH|q_g)Azh=q|42_PT3}q|Q8oSDb!Ae)n5=p|lX4 zGvZt=jd!Tv=3-Rwz66$SmWD(9XQ-0dlH$=GkDXzQ)Nl*jwV z*9$g=4&G^0^Xf9#UClw-f|vEwdA3VR(s&Y^y{MG zGyiBG=L8I!r3(!`uTd50bmASEN|)BpKvs5&Xy?46=(R>F=SJO(HrE$$&f^92y4M1N zs*O?mDm^Os=q>HHH5YmQs3WKPmFaes*T}TonMOZ(ik#kMGqWbkP$d%=X!ZUVIeu#~ z9esO&yfK+aKfF*7kG)nVN=(s5yI;Mf^S|vy9sA3W_B>U5@KXcndU2I1iKAhg%ym(J z)oHT&QYIRDCqV-x>&XevRkYY2px0SX;HYaloojfVWZn=Wjn{?bnfWO?WqS-AcjW{0 z-y4I(<$_pT%5T4(z2)=YZC@DUs*??51VfEH}Bg>Mzzk({bX^jn3{Z2Ldxs5bJg?j&hj6UnvA=X|}PB6xS~H-5vq&-|8sih{XUs=5C3 zSrjKPPbMFpME5IvqJ>?VbW&9b>IB!3*&Zos4ojiW)qFA_loRg|BqIBod{ODo&q(}y znTTC_o^D%Lie{^2ka72Ap};YO?x^h}5h_BWy!|+mEIEWat^Mhy5k+c|mq2$aUZSE0 z&qNzUo9GIs`P6Z6l&%?bg^b?3fVywUQPGbg^8K+2+5Mv&O5U0%Hu{Cp;g?m&;ESRd z-VQ|pwZpqS6)dB1@?zDt~!7VKR$|%`13=Q*X~8DEdS%LJ+4mI{Hi6_YBWgqjCAVM;f(G*^+A>Hzv-Bzag^Pc zh3Y%Lk_?4B(Tr19h-1zJY8M4b?>H6oxZpGH)g%0bm+q*HNrH3Um5Vn1#VAzer$|th zPIETY(77{Cl0Clw+*7g(CbSRItau5*z4v`0FQ;j6`U@#qGy4qvc4!8gDgL2^5t{ODyueRt(n>L;5BPr_2Jw%3{f1*of971D4&!H=MgkX>Xzuf@91mr_~ukM##SqG=2#G!K!tPc+eiGfhM`;|EMPP^LfQwW-$U zEb7#&%fFg^5{2)n;GWu>ZzjNml&ZFo_Q!;*&bIs$#j*UCf>Kz?4(Sp!O*HLI( zIg5Wn`874wnhW8>hwx-R*E`6PqrUwh-Ri_lNA6a36UoO40-S+TeS28M^*tdReY-|sE#k=Z&~|GG(FE#)RmzqzISK|-zv(AM$D8EUvZ0vC%#c~ z_|P+GLd;+!=MZ}s)dv$zx!#$64jmL1sqZb}$ zQNv13B;iUxX_2mYNIdB=}9ps6?JkI}}ks^AsBZ*%+ z;~zY*(G*RaW=_AYZ=g2U%Zb;sO6ZED1U==)`3sv5qel}DQPJ@xJX=DUs-8cJd?K9T zz5`lNNmiSB_C%3sQeWuIy|bw8!an4rH$p08WZ=44oL_Uw0~%rWoa)g&6xw_a^*bJe z_0J3FyL;DYQ_d-9w^tw4*O^nV7s(>|6Khfbjw2$`_g?9Hr+Hhv7;4JldarlGf#^3tr8Yfvc(-XwLpLn)+c4Ewz;8hxUnR zr?D4&7EpvI|M#9a-+v94Z0v{4oU^=pW+QQ5`Uw^v*oFS(TqQShHqmpI)^OI%b^NG5 zU1Us$9BsRBo37eA7Okik5x2wJ(c=zt(ds^JY7l1y{jY9AVRrxx39N?;$1X>bMa6V> zZVol9X{9z(Rj9q`QqJ9Z2en<6MK3d-5;aAx&g)ZPpB);4vjf`2uh@*38#D5ln8y6%Z@S8Zr~me%lwFGGVsj!&`}drvd_5q#scuHp zXVjs2zJ+Lbs~!zG;)v{Cte~G5FXa1TJ6RvNkG@mYM9wq+5Z4>8NSt0M7 z#&1Mge-?8+xff)2w-dV5nn?HLej{&x-GSGW+F-&oTl)K~59yMhNs~5=p-RfP`O2DW zgn#0`Qf9D@CjQVxQ|(3Q9Swr2>Ve3B&86XPN;E9Aoj)U@f?h3jr}mv@O6%^{q>QV>ST&HA4lpl&r;)8mV|d8oyP7p;j1<7LX{=5 z0+seL;xV0_e1(H5w6C9qmA1;X(kzm?Br4J;?g{9HQoLxB|2G(*l3Xa)EC1(#2Had(Rk-UrK^pB#76d@Uztb>}3ToI4>!jUbW^z)w~ zdc-~zUJQz+&n-LQ7hM-3IB^o4wD>`Xp7)R@aDujGdy78)7mSMKwFL5?XVBMX3&?Yo zsmR0hFttQusCj`L-O;6vlGsSNm>q!{TdPH2U>=RgYl6w0t7+|DOLY6e4Y)X{9R+Lb zppv8MXwb$Dt~rn`N}YFz)V}tH@z08g;f$lG(=Z!4dGnBY+ZXy-LWC|ak0$%T=5sG zmN6!gy^=`lTq6`2O+r2@iO~1$eL5>*6{&rHh`W!8NbVmu1Oiu+mVF1vKN}uB{$n=v zuL*{BF|zR7!?FAo3xALaHlcJotcCwV=AyFk++EY^rbyCBku;Y_(3(e6_&>rs`CpF3 zQH$7A^fsmt%{F=tgCczCY1u*)Vyq%4Jamd~S<;H84ax|*axBp8h{JTTU@@^Renj>i@;T`Bs~CD-|1In| zdKZ3@R3%pK^(u{sW9n{s)OKwK=)%qhj7xkHpq)rqo z%w3J{^?xI6=l%d-Tue8XX45$;QnWi5BU6n_MCaIflD|0t;{OcA`Kfr_k5fRqXwhUbkCc} zfwLT69eY{ELdvI%{zKzh`MDKEejfhKCXQRXX__Y1I2Lk$Git+Oo&Er z(pAL?W0lc_q5)Lvab9#prGrkKKP+0xP8EpX+9EZ%2J~#wZaTai(S*xx@O$G?()-vR zH60i$ZfAo4PJ3{ zMM|r*(T?g>bf9V>>i&D3Z@vqo$CcHHvgdKJOt@&p=p8&PD*U?;V2O`L6 zKhjB1N9K}JVw=J*C_h;NPVWmvCkl#){)4Z4{BArgcTz#OczN{VA!X{^X+YVbWYMvN zV}z|uLqC`_c+~MbT~QrHJy5IY1n(#Re)&F9*la*E{u_&`;$MsYu9p?ab&nEw=`GRT zCqq^I%*C}&?@`@Ho9RHvI{IOPrQo?yBeHulk2r1XC)*CqhT?OnsI=Gz)n18#(Ix#v z$NCf!8K%>K69F`9YZ=;cq!#g>$qPD%C6Sz^G#YK3$k&W2qnrO+LnpbNp!n6tNu0t8 zSaaPQO@EUKO|g_9%=r@GKRbkavmht`U_;SZ#P@NZkRu*ypC@{Sunu=pC^=J^72Nsvq{e)*9g<3x1#Y%d-8 zZ!VFIu|d5xR&SyTw%KEaIEIzQn(nco2n{#iMO+45+)pA9~bgAD!l`T0>yRnbYtvAK>1;ZiVHEsYI~+#-Cq_6oaF`v>oP^>rqqX95V}nDw{o{O zSqc2`nk(+Ho()=7RI^EoYnh$-WlYbWMAj}Wo|U`Z&0(?k;fc1|p!$_Mi1lq`?)a|b zGS9MDx7da`?mWOQ51fQIC98o|a(ZC1(kF&*unrvAa**@pH8VPY&kBtvJ!F*Yzp&3$ z&ax54LY8A%vnJz=L2?OqEGNz5$-qGH&-1cSq9B(YtsP|TCX8jTX!SB{tB(rzuX)OV zovqA83l*^XmLj*id5Zb3@egxbQWJ0WA?)j_@z`;q3fmkV&E9>T#CTl0&7Nc&*iTCP z*@(Q$Y@YQCHpn;L{#SV%TYg6gTf19h>76~oF^zTXE8WAapP>bQ8=24gO(|f)H>=^s z563lT=*~#i_a8vU*yhRz{R;5eqkNq}y zLD^w;7Fe1Keb`r_kH@fW&e2W+-r_YwwmIYV=LJ|Kj+|ma@silSR&87 zGl@C!PLD}xlm=TuEwIJ5vBGw?lO<y@kvwTfjyKeqskxHZfsK`q;b! z&zNTkH(5E?XAL3V9M9#@czk`r9DL!qDmKMaabT1z_~tpYzWK12xf?j1eeEaDF}hRP z19_D^^JSULZzXNea@YXmzyB)Sf8HFpt=YruOmkp%Od1rPU*W)h2X`42kK2q|*-A#@ zWet;cmN2h)azMIm7t{RL04zwg#~KIKa7W2!cK*8Gtk1IbtmlWiP402$o3#x?evjx zTA0ttkN~Yg&9FX{UiPyGi98mi_2D~r5%(yDVu}bshK<4&~4L*J@SZc%&$9jv| zl^5l((gs_cTvH+S`{v!x_O251w!X||EnOM* z*@P1I0*B>3vO^Z1;2&g<%;$V91$P;{_-Bkm=S7}u;0K<+`z^+=O&KT{o!}YXUdRL? zGoZTt8{4R6D7;qln0HJ$hiw@?%eLF7FgI$qvJXZLv1Ps+^Lmdon0Va`#7Y_B9ZD z9z={yTCSgj>kDV-i&Of0(fjQ z#ETutV`rT+#?m<_+0DU9jFVo8aLzh^ggjst($21cKwDV;8K!#cU~%)*Br@^sLl;xKIclRy4$#^#2DmKS=764wE-a}rJYj?fR1!<{iX_7Fi0Pgy^w3&C&K=E)RrbBTlmwM&(S6@MFVrxlC4l+B|%v!UM0_F~DSavHKl=1&H9OkkFTf|#Ic-;VV!!m*iXsY8GotqEb zPs|0%n+$P|jVCkHdn#l8a6aI{2-dyKpM9nLnwh=3g5B}t8!6sa*ViKHYu~RI)aK4{6 z$EYx4$LTtO%$iH=s{&0tTpAr|I8WmiT`06(8v;KZpx zj7_l)c-7m)tUaTEUz~Mkdq1qfzM<4tZXiJ>(^ux{HNhB?%qN*u_+D> z+RaLxz3reMeMs2G(*{?(JK1YRnn3(6O=!{-0G!J7*vRo};Hg{_qhyrGD^J+VE6DWc z9r>Bg?pgkuS=K%t=%s`+^Ov^^V?=Wp4Z|Ri(vr%{*Rf~zJ(a|rr+)F)Burp^z7zZ8 zcV(>W`8wW*T}kZMo>pe`=pi=a+hibdvV-v-+{fcTnt|snOlLR#P{2QQjM#a{kFg<> z<$y8gDf_LQ$jHe&<4xJyzznEQ25~#pcpY}Xm<5xnSfc}zIL`eWw#V0%ag!Z`513kl z^POpI*;5}j<@6Y^*~5T?AgEyd>5mxi=tjn3%UR*aXUaIOwVG+wG6XmEBG^LNd5qU5 zYbKTdoY~+L(Gb4)8?(ez%xLU)$MeqHF%PWd@q8}V)H8i15D#aucb9TOdyCynZigw> zefEG!x&4LFGUDnNmH8~2Yy{qkrJ2%WkJ*Z=j~GKPujp<(0Zd(+DJ<}o#KFH@*?&d9 zm?I{unEP+fFz?vcfgKLxnPZ-{+!D~~DUxVtxAK4LCcE91Y<>X;oXrrBT5vSYjs zsN=tv{}V2C8esUN2-Jri*DM?;{ib(?EAU@*lJBzwl+``8+fQQ(oc3ZT*zMog4E=g=J_^kzf=|z9_@hjN~VDs z9(=api7DpfKVXRs=TR^1W>^2r;cYLAW|e+uVfTn?;SaN8jD4adv8;W^Je=TcIrNHi3pp&>;j zP4Dw3-0|G|`+d*(em+&EtOD<_381p>&8Aat90O1GUqRE-?YV5vHV}VZms?U22h1M2 zp&#@7oDxp^(B-P9z=u?6&fv^^6u~<&4SbeR4c?>RMT#0-H{X_S_n8=~BbTol`Vf*L2#r1ih3I%(*7(@w8jP;&SbuHA?CS6BG~^%KX?=qg1{ z>*FL&dDT~-9KyT9?<&%7ws9cl-zi|?<4TWF>qHyWr9jM=X~>!RgIeC7Li5LcM=yB~ zmb}(Q-sjnjOpC|S5_0=^rl}hh=Q)5h|CEC7f)DI}rrQ9UeFqI8d+vKg14!-E;a1%J z!!{pW%erjW<|I0=0}J(fWZ?K0xvEX%0@FJv`A-L_NXsTRC!mat%e;!V9??V+>xFbn zeIGS1AJB)Se5kvPS?EU^2bOn^qR3}z^z=_>*egXvPU%H6x4f(rKbT$ys3C_88>%N|i3n^WlPdwwl(vJM8ENziOp}50Eh40+bhQMyJ;H zfWNN_P_5ciuy#O-+xK0a-a$Si-)+V;Pe!Mdc0FWkqhF&M#otc97jz>_W)yYZapyVX z(!kpppcOl9(d(o6$c*<>wR zp8M=ZTX(9UWj)*4H;lIFG@x}^rgY;sRqj{vOW@qp0N$Qy0iPV6qTKBz)S7i4P-wd$ zeKpnxefnz&o>-lsR&;(tkJiMZ+n;Q}2G1#A%qYvYlxTD1nTqseuN*{C$ARlTRjzP* z2Ed1}vFSh6>EsAkzCZXD_|KA{V|seoNBYiO%HTcJ9N!4yc-~q|?>gGV>og*T2PlQ^ zarDD3$;2ZD3EF4_IgFKF-p{=3RY@>VZ z9E~4N>mRm&&rwTw*4$LE^zdtTbXOtRzd4%9Ir|XxcGptDN^1fB;=9qdUTGH;( zTana1OFGI=1MHcv4i@!YLBK^8NS=|P?UuRGBj2JyMT7}B-E$LBmwiAXzsvmonSquj z2ZHrRjVQn^A8j)41`ppzaqK5e`ZAW_6#Uh}xin9Z6~gl)wyCi1XZdj_LrOr;>nCXc z^mNd^WfObxvJP$2*AI>xd;>pGEt?`LLgDfyAbEj4m(st4vZy-D9=KSDLQEn-=ei;^ zXSm7fY5Y3`iYicG`9E-R)l_cs%tvhN>Ql(bg6BG%pTqsNGocsldjLG!)v4>W7pZs4o+6TQ2V2)^+So}~jvK>*tU-qd_$ zP4BgV7sm?F(~U*oPgEHCG{uJg}4%Upy!@3>3 z3#^kHz%fR^$*DVXdXcuA&3N8-(#w`>KbxS+QSb2*a^O! zfYnJ4w>vG(ISgt_uc9Zl&UD+YF7WZS1l{fKNe8+rbLa9jxZa<(+^jYq?pmN3_hWw@ z=nszJR<2xtBsz0Y<2!=V@_C2xr8d;{u>nXPJ_0H;eXFgh3EbrVbawXY70CLO9+)O+ z!)^XE8?gM$mzA$Y{WFy0rpQQg-)2};6;Q*~`=!UAK9gl=jm;)%%$m1o#gd2UNo*hb zvrHb9RbOXsLl16N?RV6iYRN7RDXbQExN~BMC9Jrn9@M(4b3PX9s13H0xfSU#AaFp3 z(~cQInJ}HISuLbLzwky&Evk`J`%M(5D?xinzeNo@)wnmKB|xXr-DzurCMCS1&Ruj_ z$~k`R0vilFk@m4O;KcRuEVJB=KDb?*jhpxfFn^`!-4fk=k6MksyXFNIkgLTVzj2;@ zTA;zX|5^%6J>yXE^X0ULiykVtKLI)1u;TVQ^rM3X?dV^U0$tI3uX@qsMWAK;4JXB( z`zU;(A6-8q328ssMYqem0j~Ssu(v|Ta?Vp{a}Af}>7nD7k>fofxHU}z+%q2r35v7X zg1f80=8q>>Ii>H^_rC^6s1U>@$aN$6cxO<2_h@wnkfUXP`GU7G=TN)3KDe+}i;I6a zj()O7K%Z`WkEERf=)ID-0DW0Bi_@I^1{L-O(X_KOxCxMp%b>NxxV){e)KQOvb8Wj_3`pBW*XugsQvPeBy z-Q1x8o@w*abY>=ScU7hb29i)`(G2eDdkJoKtPK4kLynGo@s7IaQ$Rf^D`#bX??Od% zH)t{TLl+NzK$_ez+eu{u*CaL0#`-!q5L^kGE{x|4RMOG7joX2oUlEvP-3jVF{#464 z%m?AGwjkGZO>Whq8elI|$6o!Z#w~Vsr@zkXM(VNx^tnnG=w5mSjz8-}f4|ME4ofCT zBenyi^p53hzDuLQKlhMx`DFUR)baFUjcw?oSeMRr)8ux{u;eWLN~!+TvFKB_2Z-n0 zG;Yy**~bF|==9-J$mw1qII5(?ZSr_Ry^IM&shI?v8dTv1jqalEZZ-O827{Qx_t6~b z2#_SM1y3qJQq6gX(W^n8_wn`}l6){gRbP36rg}u8dw2cV*QyM7G;9x?mO&?0(itRd zxW*QS-=sd(js;(j2D6iIS%HvU(V(DEhqj3BK;O%qDR#9CRe0$d@9-3G|BiZr#f%{v z7UxZE)~`mvJkvDe>o347wm@W&G8YqJgAP=9gU`|gO-Wt^7JoD1c1T&Gcbqo2JA5^3 zl&#E_R`SnGp(&74v!*jHh|u2y7r>*DIJCg)3$nVn6x@7#7hJ5g;p_#w(MDhX_rQN9 zC}m8LlD#>ta9Ey;?so#(uli8w8*SRI>kaBkcn&JuA0yNFTr}IJ5d1WeW7qTlBL`lE zfwSdP(9a47@W+qmP{|#1da|SjExzssd`_f*pF#t$IVO_w8FLFAi(Q5+QqFeYfqT4Z&T=P*xN(T!4&loA0J*PIcCLLu}@ zNZko-*G&VOZTr~B%{!1!q#o}t>;eI8rPcewzp@Wn_5*_!6+rdraZ`OivZ_Z%z@Ek0 z+?>yMk;?28;N7Q2Gh2JnzYcfWfp-Q^jcQ?M_|2ks_AYa}a5V*q_)JcH83HO}&r~;! z>_gw(<^c&8HLkACnx1$zh5azr8eMy30=m8!aCK6j!S3UC!Li!|X#d(Vw7Z9p-Z3@? zfckU5)HcCsw#h=ISB6lYnKH1c&IJ#7xBNbXN@NnIMVC#vSH1R}A{~182|8^l2Y#O4 zjokY8p{Endz{p2gH1}3H(m#~p^m@U2@WsUy1kc!phNl^DI$auQ;Pep+yb5E#ejh=n zk|)qw`y)`bn>=VL)}Y1P#Hf!i7><76_hmj`ko4M!X7;}Z0fP#Zx@tUSM_IC;hR&kP zl}nMziq%wT%r7u2b|)(RaGxq(%bzXy*jK;#Fb{Rcr?MTwe(?F~jmk%5rKom-A@Go( zxG(>5k?7M&u=jckD(C0Ks`<~sc+QEcd-DwCPt*pxb;oc$GGo}p*GAm6)PqR!sSU^- zIu6XIsd1AoPUf;c907)3*HAa*rqIJfQrvv4Y(xZ&zi^w@Dhc@DVjattLFA7^t)Qc>_&C!q5) zfubbL=+qF+stx^%K%2KC$Vj{9Bp#6CA|77=9g~(G1-q5#p8H&6H@l8<^_)tnWM-qYcEA#r+~>TQo*h)X>PhzJFB$Q03=*XMfbORg1gs^xdkUj5$$dVxZMcUEaIK_ z4@QB!lm}2#HsQYgXa-Ar2~gS7fkwYi@(=Sdv}V0c~$ay~SSRw_x*X6H*O(eph1 zyQ>UNR6j&{seHb3tp**@Ria-v`ErLE6}gNhwrE2@CrEwSgc7B{v3;I*z?2AA@YSpx z{dd)vjvZ-2E}x~VWq+G;60_=@2HcN0hztm{J|eOOK(mDQ~P$_t~bN}5MlC*E&cdwUKV z!}0WtE6C=o z1X%8O8m)X5j+B4)AlEIntnEZUbZh7sa*w|bUK=ZNhgNKK+Qb^5vDwZ@=ka`S+Q*r_ z+!X+9uU(}=ZtC!k-u#a9{1t&eqi~g1Klp;{eVmQ(Lkgpxc$umTxoYb; zjw&RyxfVNTRs)#Vk)uf+nq9BpB*MNdPkhX&EMss5??cPMR*}D^jl{8SV&JR=dN7U$z z`_j zls1Ety7CD71Ztt# zY4+RpCCKQqEEk!iO#5eEK+A(CqRYmMtLZ)ZNKdyHT&Fy^myVkBt@)Z<{Hjyz>Mv`- zrY{q?Jx*o7rTzoUxT(>Ln(VpfdY{47Q*Y7lj+N;5gd^ah{V+Npt;TkZ4+lGs9RLyU85)|=X=~0p+1?39SDT{Iu-SNSS6V$f z5|hNP)@lW7tQFA$-sAmt)CO&QaRGg7*5O*`E`1D4Sc9MGnChDrteOY^e_lguC zb3X<&2WTKQn^|<`(n(->KpA+WKLVm7ec8^n=YYymHTp*d2c+~XS-(1(?u;7Cl|A;L z>pJgI(kVFz_?`i3y-u9m>shq->oT;(vK3j6oTp40DyiJ$eqdWZZE3ZfO{XoW-HL6`evpwFCUzqFKA-8^sx-N`@XWE*M5jo(;&qgdO3-k{rq ztaneNdk1a-ZkrZ&>*q$g>*FuP4JM)EWy`@DXFl&}QQd>5i1=g2X@XIM9%!P zbjo)l>U`kCo~tbd1{zc7F_ufXH>PEjF+HA3XiWimqUE6bU@>s&%>d#zd|px%iB3&7 z0`Rap*SVvPJR9zWxF#w}m5LN1koke;zQ# zQqBmLY1Gu$LA~RB!XxfAsKCL6BRsb~%j71~nDd^`3TW=Hw=X5Ew&cPWIDn)a zX|7ytJ3I4{Bv(^4h8wbYi2`;$0rPB@P{*5!fnwz%ZurG2@GAQmx_{Llod{aYg53+rdXw4dgRd8ReY61J;yG1x1^) zs3+w|QLU{K$O!p`QqLu#_?C^7bMs&D*5L!lwVFhyoY$r|JuF}+ug^tpo0Yg2RY|Zv zH4{ufP1qjEYQVtjz;>Ynw>Wqd)EqG9ybq~z@Aw(C+4njUy_BTi>(55}Ei=%xfqHg_ zk_zXnag$B*wngRk&RoH20&KoY(_{1vxeIta2ntfB-jtP60Yf?9VS_zty7vs6c&E+P z`j)W5+6U~-v?$P$=0dsYWKzi@Q%`{q^-CWG839{Rm%5=YQ3rPkp}wjt84^)d6#=22Mlp z2#(=o6lR0YCJ}qOJDVMMSdCK}@j`267$MWr2I^ngJf#2HoSto$h%UuhbKJ2aq`%t& z<=m~MGL;sfS=;)-zV1x+;dL+6T~L5#C^oUw6 zpnHGbQyGsX>3w7Gfo;E{(baDssde*LIkt!21e#S*;JxDtD(l<;I)*V=;k_0dT-#HP z?b?x*ye{G%tVfHERDkjO?y?VRUxIXpHmdRW7I1jT4Cud@jvB8Sv7R#$+0TzPz^4dB zI!{@h-W^~|=bVt_{>`?e)01M*+Q1Je#xolo-b^T4rIXcGUyH%zL2}aFaoF^ zROdR#1bWNE1+;R!H&XN|M;ezCsNUiG)nzj7wD-4s*SdefckQj@{T&la5Uj~%bNPe)BB-XhP{rf3B}tF$f^RD0FRqfb_* z==Iy9VAs@L;JG139V=Um&i{6xx98~63Q42rbwmuX+T+O`>8V5y!`CDEz-~ z0?(CVLT_~-;@!t!;qiU&P-X(WU0X<2HfzDjsyf1*2Qjnst`NTcbcQ)P{|EfLVJjK1 zSA$?E7WQjakf=Xmm@KT~dnVIi5?3q8S+Ec@9wO#)-Zb*dZwhHoF=D2Su7K5#wh7G& z%%I+Nd-&YBhH3M>jf+g?GY7Zv_jHYpFyVe8CiLAXUKn#6hKW*1O3rpsyEB4rAD%4)pwPd#KnJ0=+6-@zczgu=JY<|7g)-7KKNXGl6Z) zF1h!Y2i2OdG3L&%Vcz3Q zjQMtRk+xPT>6j}^vgeJEw0RpD=0A5bkM)Lz3?@6Yn#plpN0ODT3)z_y8Sl5}M7K+jOyKhlO`wNZ2I zAXo>><`*!=UR6-v)|?q55WtIVxp3j~MB=kCTEHKelC8@(-~*>Zq3!fKX2;-Zrm&xo z!dJ#5V6rMPp1TPypHxULG@fO)>)gW9y=%$SPh+9$lxC7&-bQYps)8}|mB`oMlbIS5 zd7_e5&m6cLLMEljz@rPhnZ7r3NH{bWuCLC8KQ7HD!RH1=Ne9jgCQeht9%Q(v~M-X4lLa9?Vkj);%ku?ay`QC8uib{Mz%Mk~EZX{Ej z#|R}QJ~Gz=jmUlrb$rJ30a+uzh15yuF#k5MhrTJ_m~ox;Fu3wOSs_=64MV&5YQ;i$ zc-db@GvXY~ON(Tr-7LtX!2QL5!t zqVawq89M9-#(%MZTI*VwH3jzI*xLrEvZR!p8+S%jlNCyuClrWO8cx9XJdg8(L?6tP z6_fdg(_!AUGjL~(Jd6}yWxkYLhkE_0;;enug2c8ovOH#+FetzVO8;nNPWu5+&q?942*}NO%&4!;!QH8Wr0$R&W>-myX%%-kKga{#mQ{o& z6^h_$jUPC9nG}8((Lt0P*TEM%bg=$Pc{0av7$(a=X4@6MpR{=el&ZQ(R~)?n%`WzW z&@C*v*q4F#FYSgB_# z)=>qpec3H&98pVNlTbLm6_9^@vCQnX#~7y*o8X3jIV39XCP_X&mH2%5jALIL6Y{4I zE-2N&-|?W}AiNHx^96A2OTNc=K%MxRnUaUQ2Jln8Omg<62FZ6yB@<>$h1d4Q3qBb> zgZIo%!k>4ZGa#FSRXN=-af76yTcI83jFu6p&9<+fi zXAd){63b!d(Ioi(?ggwQxJXJHpTp@F%HYXmvha~y3mnd00-b(zLDQTp_--rDRmhq~ zJQ6Nq?=SLDVS^HB_VhZ_<@!w52e*DjH=T!uhzS{Uc-T)_C6+mh1W zJ6I@s3y&K|<9G7?g4^$na8!CR30Y}G9@NLd%vC+JvR@> zE03qbgTC>M-PiAA<;Z`q+oT1~e4r|pffbB{W)caXVgya>dgvaBT28g>5;HsHH7pO( zgDxSNWUniOV?HL}3Bzh|LZTiSQXdw*94aQEdW+!lk6~oz&1tyabG+D9v{qoXZzY!c zLBaj>MmQK6MykWllYw`!&>|%eu6_H7iF5WRx9a{fcHN!0{OJwGvhf9XsOSZ=aQGx~ zZaM`YPjDb%Av55|u2aw_wSr7N^-9nnGJr)%p z2Hbx51lj!9hivIqgT1Z!aJ7mdOs|P2qt7exlY?E*{`e>6hUam3%tHwO{eCZa=HpKq zEuzUvlWbxpJ4(pTHzJK)Yq8gX>Eyb~CKzHf3!4NvGwRRkpw6y7lA@rW zi_ezI66L@>aQCo)6h}Hk^&c_hO;{%U>|D#FyD7n%ASL1B&z?Nv)EvKe$GE267rHB? zk_zK|92RUx8kf{SdV3|=EpjBK)5ZASkS*-=2qhJM7#=d5Bi?&&9So2k!Z@Ike5*JJ z4?1|kqkoK<`nP4w)dmxi<{C>}URe=6z8hP-$Q}l@JrSAu6k_%3%H&$0yztnk-=g^& z?l8YoCqO%%y%LlsgwwvX;qpcsapasGWb&O^@Z#e*(e5dW$tv?0n*CLT$JVyO2^JEt zX0IidG|Gg2%U_W#KW32LmQzf=)LD4qc_Hi_^oOMdf0;R@Tj7aa=2=B_}Y~SAQqjc~Q`Otqd8Mcm6-nl7wR4IdDE|tN;Pwz21@E7~ee?Z>%U_9pCQ}}M>6f&|EZc1#nEsnsFITPdj7YCtfw7P(Bd5NZr& zzy#w&XkMWS<7zwUDN}cdre*Zt(5)A_fvdwX#u~y$Aq49s-ebI^l1W;NA^a~c03JH^ zkz8qsW$rIYffAnP@U)vf`6ZFeEJ$o6DL2Ps3#1@^b@V5E|57N<=2=QrikIP>xA&MS zH(TKwQ5;!h+bKADpr7dtye$f<3d3B_QR3b<0Pp&7%+bq@(0xf19v-O{g_eh5vg8@F zKcSDY2#{h5CY>cNg$v2>aWmLFHUe5SIKiSu1};At2zOf;;O)x@v^=Z;6(+vG!w==f zlh&ySZ?DJ@%rWg?ERK9+ZYGD4{`+N&Ow@K}q3l0F{L**uzhD!1{%fnqag`_h8E=lw zQj?kT+y~fTiw#uqy3BKn^YO1`3{mjl*<+*op*YMyxHIP>+;~k9Dp?HUnM#}Bm*xKv zpR59MG;|W)yIzG?-amlXv-3nh^TU`Ug)U6kd_Q=h(;jNM>EMM!d*ODI5$Hd^gwb-k z%``^6=Ry-CNSIS1bgR6{Xtc$^TbpBG|8#AhHIPf#bGrz#yNFlG^%55wS>cTI8&G=d z1EOVr1ggJxf-`RYfZpp)lk}I5$ZdhKc=D7s?6m4OG5IwahshqqRa?uMrYAfH*6$V3 z{dSRrO-dFRj`bw3P@-U}>L6pi!b!{&789Rt2*H{xk=oY+xbxF<#@sekRBd(uZ}@31 zwAnV1D7Z|79}0QSpTtaJ*!&-p9#kW8H}HqF(|FP-@WNv%DoMMGkfik+i;m7dgH>OS ziX?b8%We66&@@X4Dz7_3{<^OvjWwR+c+g_g5E(Cx2mnm#bjm0}eL2 zR*Cv-EaB9iGLc!?B$BW{5s!u>;9dPbLc`_N1ii0;5f=ZTP*01v-%e%%)aDA_sv3yv z3m%I&H$`E5b~G6{G?%D&WJsR= z6T4eQlhwXA;kKX&a6)(&Q^7M<+^YAG;C)HhWECQ6g2V9jZ!u%`^dAoT=R>S)KMRV7 zc9DCZq~Y?rCj^69jPTm42hguM9O8@TiGA7v(P(-GTzPyAsd#V&c6D0_rO&1^u9mam zTiDHbsnwHTj0tpAiKU7^Z-Y<5+64<_c@FV72A)Yf0iUnE2djfb%-5-Tg6It?#AT}% zwEfpC+E+~B#k;kkr*s0j9=}3#}J7oqxugfAt786~Qcv!l8Irdefh!gF} zG~SEHM{_9C?OlC>sD9;uAP%pxX%| z;iachc(z&$!RKc{?XeFn&%k)R8p^-DRjeD?%+pmZ6`p5O|juJz!bO?f=8{u#DN z`Gya1hT`6>^F`hJ&yx2Y;qa_z1-@65gliA=Vv%7uEb%x;W@*U@r#KA}r&R|@wp$~r z(7VN4?aX49`tK)S7M>ZjG?J}e>jHNA`j zaS2Jbw}f@uoUr@MgJjvBBXC#iP54>T6W10TfH|KY!dG-H&p@k%LQxKR^kx+_*{(=# z7$q`87v>Y8w;fzH`6^tqDII1@pT-N=VeGn;n|*o;rW%F?J=~v)fQS>y;`wb1s?*Fl>T0 z_p0#1n~-_9?H)1m-A!)u-SKQW6EatK3e=oq0&R7BnXD&aq%mq9aW+(jg8VCBhT%yj z)WvvFVW~c*#Qz;g?B8FdNS-Bj z7%#CaWfC8Xp;Mp~+?ZFz2pYbD-TjB)YiE05{Ei5spf(FXn5GJQb7jfK?Hh%EzR3v> zgf$5s()k1==8|k(R%D)_N#6eSC;CDODDwDDEU%t};6wtWVp0!JEb?T|IrFZSU-68$ zF`pye{RKUjItV1T_aA+xRvVS z4*{EqM}!_pc_=IXj|fEPuPTV$(|z$vS54v5k^+eA#&EAzB@;*CG`OOT#;bOg;z4%> zX3fe167Bn0Wb(_2E0|&fC*{Zz7oH>EA0#LKBjT7-g=ZDm^sxl z)aZBSOw(e^?O56g)?J6E*mc`Ps9uNJJN24Hj$?{wZ-SclNjrdX(H=xGwd3& zm5_wb#9)C0-t=-Ml*<_hGtOiR{%QGO?Ed?uve z0W4|GCSiwl1!AcgOsruC-nMQ%)-e?Voxwxoi*+tMp>_d2=hh3FZQF7F=G#zlkF+@9 z-vV5mw+dG3A|0TXDxe%V5)C|X_ zT!Xe=e;J{D2@{vlg%8gSC6tGxur@)&-6 zyAqe((}JqYtl_o;?O5-CA<0Y_#3C>ZuUsm^Z1^?O8ypR*+#BF$+h1_>fGKXzN+NGl zC1B-g3Fx7r!fcgvg3>{$@LGc`lY29f=GPK2vXqp_{RF?b<%1#J60$UH<1B=QP^Z+Cj)jYGrGBenzX z-FSKZ(KM-)jZQWkH+bD2g-F-!nSD$C&9gNpRh^GWh*4#)kse6OE>6aMj{HBq4g3wyKGN&#$b9zwf6(`zAv; zR?8jg)IG#|HFjhF4{?IVfkjXzDV6Nl)<)DTl9>=CRe0gp6;aPWS#e}f1+!vQM!5U4 z5t;q(E+buNOLPMFF?;+jiv0KPg9%SEiC$C)`Lbps6zx}K@cYGNN5pwzm7pcumUWO^ z{CW^FlbqmOnGwd$Iu%D|?t@PE>!E(mPB?j97vAxxm0q#M2&NY6;Sh-}Fll-)92_%; z95VBQI(8Y*E#^GGCrpC*-?f?UJr2ZTQyY9{rwA+j{|NTZU613;7Kj!_ohN}=7UKI- z`LNznMR>&R4Se5W4~XTfoJGr$z@c8bLBF7!h)j~blp0vYy329tGA0=gx1V9A!rFjm`# z1cOf`*6KL<9TbkWZSq7j;LL6vm%oh|=zd`qpG#(B zl`k?bKi$a5Kmk^AS7scI;;>_H965VoDrujlj7uA6^7T_F5g0w?8HWyxQO8plcUXpK zy_yQAMpX0cPX&0fbuBr1B$1Hteax-emGIc36nv^M6P66dL6xVwAc&g5gjlQb8KWf@ zKlUWg9V!HS(v}fTEj^MTy^AP1jweIGeK7yyVaSCaChcM;lCXZ1QR<8*OShdAX?L9^ zDLSf9ElU>`ek~^3{3PL-^o3;6+~d${r9aGY;m?-#jD=J7xDoGuU9z#GlkCz>!2LS! zV8jbRvcrE1E*U-*9KB&C{(VVNI4W2Gby`Chj(&yz$gUEYjfk-9gX^&0F&pl1@`ozX zKS@xRz4*YrnQ+k=12XT~DWWL53En)$F_%35VDrvOV*e@}e@R)7m2Y2xw-)TgME^DX z{J56HUD-?gH!p{aRpOcYS0y+hFdduwh7mVA9dhAo0#WmkhQgX3y!$9YbhmCIb8@ek zw6uzG@Ctb{c2x;c-x&;}`;1}ToojILiy<87YvJ6@+i`Z{G%b7ad$Jlm@gp0CQmzclti^}Vv>_)c^3>E1ae%$J3$u8t>f)Iyn% z$YZ#D&u+%#_B*&#s12Vb@plk<5}23#XYjh^h+xKhJz)|rxbl-eOZJ5R=bsKUmju6Q6joP1bVPOix=hf_bjWFD5+!HAL7P|oNl6P7rHy`ob{rSWS#@9!0; zRdx#AT2cz%{0Jc%3-@6CXCl$t*EGCn{+GB%@Qh>kV#q@Qu<_^dIHBE(sV8|{U1ca- zJT8QQ()su&xXfr~pTXaiMsU$2Q?b=)1=8HN8?t9wFs0Fs_edI$8?)ST%b_Q5M|+*< z%f&~`?Hi8RvvM36Zt25@v)stp_8<7BV*sfh*hvES)sTtOw{Q|JhXxUWMB6%zgnhIk zU&bGT5n;T$K%$EQ$8QkHpQhpuSy$oT@+vY@F&TP#DlkSz!)xuH%`IdQ9@$4l|#kLb1b=JnRtgAIWYVz}NPx zLLZqDm^^Eyc;9_3=K1_YQOxVt%#J^DFgUprZY?cm)Pv@ezRn7$pzt2(X$@+MJUog;cCrO#a5td2aj4CvXP;rOG%45T>D zo>iVVo%uG_P5ZlI*c<~xI{Z_>%~H9Ts(l`fKXZjC&-bO)+aC&Nx$Z$q7q8Q5#Vq!r zyfghX*AwX_C(z;lP9P@W7t=HMChf|dtHHLNTTSUVe(&@nanY1;3Kg!Wo zU^br7rB`kA9M-nXLrqI1S(|~+Y*;HmZo8hcQ_OAYMgB7S{KQ0udG5PWv=`?W^{if) z^~aAz8S0{4Dg9i}QXkc-?-QDy)TC+OGWaFgkJ;4q2T=(dkGyzebX?eM_C)D6Dqc5= z*!TZYt+$m(clCL;V{0W!Qy9-AAGk^reNwr1=MVcny_}KNsG|A4zl3k(v#5e)7Ug%# zp;_l+*+NS#he;wUekjhMHd#CA%jFXE$otR2bze8}d)JM3h>|rxd8-bv@tGWp+D6hL ze?d1pwKbov(a=OU7Vl-f>JOuv%N-paE)JnL4tQ~Ml)HREU@z+K>8EF3m!icFbx?$u zfVSKTM_FfVP;c-i{%)P${3S8m9qW{W6kp$Abv~5P_un;yo9Avutx{W%avx%tQ_XhRp zzr*+G#e}WsUuz@h@%n=ah&(3zmH3Bk%wCCp#7;u&+da(qs9|>Q=c7!u&K1bt7|uqk zDxvwa$I){Ox6#L(cSwKTY}y-@hg#Ee(X-JQB$QL5c}6R!aj+6i+*3_QeojC){BJT{ zH;1@9qmTN18emo^hp{ThE}|*cYRKu~7V5lvJFOZXfJVB>NKCSd-UyVSQ|=u?JZ{h8 zm8L3MbygesUs+E_kEheq&k}^yYj3hvW|s7UMmihfu?xlC*o<}@HKabulhN)vIVwN- z5>xmgiW%@sr5@W8QSYI6)cQUF)ffcPZ^7m$clSPKj?19X&sLf8UfUu^O)9Wuv9702G5K+3PHlUKVh@?^Ok(YH5T z@j0zY@V0n3k$M;nzuU{gKEDt6XgVLdXFHHdtT!CK77w1>9Ro>om%&F-vLwg37mvno zhiw)Hr1Zdea>Ct%=tdWS!e)K)z3eqGP*o;}ZMBKs*LL8!su+|esDs%jqsZq02=9G- z!J9kdEiRdU2d^t~$JqF`edp;|Sh?Q>`2G`ze*5QPRcT35p{@ztG84&`1*YVKf+NxO zJb+6Q=D}d$AD)u~r`b{uhrIK` zW;7iTtdSkTc2*u>qHzfr%P(_v;aL(cS^1h7b) zr{nkv{BenZv#cNBo;E*t>zEQYmfj5=W#*8rKkVVQN?oEG;Yo%h(}2v>Z&>WN2Fd6D zz$c6cfXuKssrPHa^HX1ezk~8n=Gp|3=JB~ov`mGB_{|4qUjWb*lZEzc%wc1AD$&Z_ z4%Loq0N%D!?fDsh!J2W~1dC$>v3H|9k+U%(YSae$rz(&sj;HWRK@J!=QVza%Uj;ji4harim4}jv(@BuOH96a20;RuL6Ws~`!CGBn z{?vvXI8e&dHRQNj|D_1548@6yNg}UY@(sp!B*46xJHh^GR`8a5CYeJ110T1Gksnp9 z;CN&`zIm(}`xwoJb}b9Z#WDsqq)U$KgF(cVMJ)rnX z8jzggjWbrxgAVWe0Ceys3sz?W^P=?X;&KzB^lTmeI$}&rT(;v0OJ4~t+qd!djH!~( zoW5_z#9BPIOBd$;+ey?L6^QJJ9@!!agg5i%lMZc7+_~43gdRE#j(^I)Kl*zGsgVxk z!8UW^v;P1d$_~ISwiLT+H(;&Ehk7oT1O_zp98Kr_FeMq*t)KXfF&9AHeAy_F(@pTiEezH5mQ< zSFrfaT3{cFz@qt-8GD!KbvqE+k2VWZo192us{yB}u*DW8 z)^IuJ({r1ukXu`;LFLA^R*f{1*>Xs=u`&M)xj_-aw_bonLq~jPJ*Qt zhUfPB~Hk1JUTjSv)vphWM)>W`{hBk52 zTnW?qKVp|T8rc8pB(gSWg1|aROc48~3`e@k!s(kY;>Ot9y!tz1AUUy{_iU#zX-?F@ zaV;*;CH5$9!DLO|-tG`^&?^(~ZQBB(Lcao-Hy$p~n?YXqJ_gid^C!T_SeeMUXv2O(b#m~u0sPOh2&lx&A}@-Z za8r&FxUOqVir-xV;EoSTN~7R!W+Dtax`-HthQh6#kwByPG5DEs7KqDm+Th6spg7Hv z(`Jf*ogtrG6)59vZ`{eV-E(<@w1@c7&Pcr7$c+@poCB-#?%}}x9blW18T{h>3*5}S zA!vCj1vV}dhrZGNq<8f)xXtRJK)y;2?~PIit5$HaOSvEXPs+fj4)_9}V*ti&2*;~FI>4%Hrug)gF5q+{jJ&c* zge`ZMLur*HqV&TBlc%<$JdmrH8UXvi1uw>N=N4d&#cy)w)Wd&)x4ODSGUYWdaO!e=I#rY0FZzJ{C)~x7MnAESmn@7>9RPPnD}b7g zJ9HL}hubv1f%<$8;#7YQzjsK(yQbX)5y2EMnCni|G|Q@O<2yi1U;=y-k;{{izm7*{ zx8Y!O3AiBxgD@*gT#{l;J_n6~HK)zV8%_`RPSb&GJS9)YW`vN|lybawnjBoLr3z!C zO>yf{WwLrl1UVyK1TOh%z>5?3z{7-#J3G&VH&rL>ulrAhJT!syj~PStYf9D1dWP`+ zt*1EVt`*Euo=fgGg!87lZWRnYlEBH$?I1Br09+p%lc(RKfQ|b-P}a5<+Ap6*TqB&| zyu<;lXrci(Oz*VYblM&oe?zdY3}`?^mYe$*zgWa`k0H)ztsV?=XZm(R|=}?IVs-fL{+Rl&zVS-FUGU- zx8S8s#kl0+aMhk|X0XWdI-ad&Lq5HqOU48R_?^!pd`#vAh;;1&$M09-BNldepRzr4 zRc{q_q(^cOez0UQXsf&=s8PRY!ibqWfxX)B*(iZr181AS6go6Wm#+Rr;` zxC*}yGa(x$`eB)*AK=!8Ygm1p4JrR24P~z01-W|jN%!mdys>|;@jsmbp6=D>c>aIO z!11H~y3hEM_8;=tef?7qsX*RwS(O6e^GLYe^CW)! zp9}FStOhe4*7D9SslfBD=iwg@{_&Kv--5-=R>4j=ITBOti&fSt!H7;xxY#Zi$k^#a z*^QjGHiYHroG8GPzuLo*ju}w%u_8>GZU;`2J{&ep4bBMig7z;~k*%H1ka_a7vGRJjnDfDjvsd@Vrp^O6p!Xg~sIw&7 zPOx}Rv<@JL zVB8C7GNVC;gne^^>m|P7zuFyGkspT-w0ZFyMT0=gQ;)br#e(}SX}IUla_IlS6jl{U zz$ltVK4kY}#RFe~ZU1&&W}E_%zWE04x$TYjS4<;VL6HE-)#Qp21_@1mu;=AoqNAXK z@9!{yx&BK?LfS5vlYbM`4msmaj#W5p5a_R{9-JdDgdG#*udbS<(ikT11olauD_7vFJ+5wagyJKj3 z3cUA~gUZESVAI4{tb3VbiN&arv-+#Sj!%z4s_k8T@0#D|c6jWXo3rYm$vKmR8a;ds!2FED2$pn*EiEv}?E88E{ z8K5Bg1Sa1m4ZB!i865aZ1itU|_fvmsV(zH(RFTOYP%v-LX+1S9%0H&g8yh z+ic=B7y$=9X~Ji86A<2wh9UEOq5ad9Fk@OV7+zW@a9mi1r5eX$y|(+feWnL|x%wq9 zbJl4vyRjT7Ue+R^dQ(XA0S8iD#4#BfD}h3vDS3V0g(Q2e13p~cv`aFl(tM!^HQK(05_(YK7&GMuqOo_bRdKEvL(gidUZ{UowL{Q9s$2 ziJ2-}=qhiE?Y%0os@5LR^~aou(!+7r!{<0;nH1UK&%kR2&aj~k!k63T;>!Q3an`ZD zV21lLvPn!1R=cpc(_I~^C35roClMs}ryrE-K8;)PGGY^V0rVkT(yY}9>OX1|$Ib0H zI6$7`6rILvXW2r9Rgy5g>pT9p{3BQuT7(~j-onn)6v)!k@g!Dyy&(Hm2yCyJf_E2a z5_BpHUr?D0?Hr~O>3s~K-?+p=HXB!F`~aate{j)CQ?m5BDO{))PQHwu#REMz@t;B| z;(zTpekcuz1#bg*T(uuQyU4?ZzYcjc%>WN1%VI^9$t1vQ8wiy=Bk0RpiXZ7)`!D;7J^L&Z-5j;m8Zn$k92-TiZd2-gi@$B+|w+M#qsx>CLd)cLVt(SuF7U zV2xj#ngM&Q;soamP6<-m+{?!Vvs047oK(Ag~@+iSf90o*0U3!-iA)Fd;!N~ z-_p#hy{rWHeOwHsw{UmR3KQ&P{)OkVdMBMbb z;o#)16WHL_LLxoe1%EM@C(^y=@r1tyyoIY~k{i4-bnrq&YMCKW~LqjDCQoRn^Xx%$s7_I$AYQr;2-Z17MT?=uqOYvlpN+fD;tmw3i|RC5G;d~68a zFG`WlxZ}K2wu89hy)nr$*af;LXhF+z7xJObjx68lMTqPMd}+lS@Lp^b6PX`^i}~J= zKmH0n7LblTW;qi6#sE0bC`A^_NWpWwM6ywFJbX6Uf?NpkfQDk$@WuWX+&*bJ)XWPf z<0EW{yoV+9FZ2Q^XPMAvo!&(b)AC7 zm~Ajz&q!dfQXTv>u?&$NZ5#f&+Fr*A2G3icS)Z78_#o6i~&nL&*Rf10)PAy1a1^c;%6qKxOJ*Ld|1sP z&#te6sQw!KRdxZ)-xLjf3>Xrv_fhcqhb9>otsw7>LZSbSB$6_cjX!)dB~RN%z!$d| zL1ViqF>y8n-`m%L{0H*zA6X9^Wn6JDBw)qMOst=m1xDk7vHA5#a9B{x(_4C&w`^h% z2mte7O4)Q`J!D@i1v@E3XAT z0v$&3u61J7byE?(o6dt_!oB37`G4TyVhfn~A_+|CHbZb~$Gz9*fnnn(H>DX(n znO|+r-3==eo3M6W9)1^Pg(vRea+@nJ@V+ajc!E6-tKfGX*t$R+`fuC}rrxrGhj+As z%bSM4oRfSq-yst(PWuO(Iswc&UI~7Wp5y&ppifQ=O@=q`W&pWo-|_37LtyczN#sgL zF-VRo22mB-@Z$NGyzh(F19~kL?Ebl)47}S8zvT~tJnQG6AZ`{ZoN^gF@Kp!Q`DFOE zL51kg@__F5E2{T7_+y=K6@tnP31DK;hCiwOvCD;8q@vXf_VlbJMqB5S(LNU-Kff1) z3=?wok^>pLst2!6$rT(~8-iQgH^I?o)^OI7LEDBC-$1p`H{OX!oPX6dEnt{r0x$ef zg3q;_32oU5)0~Y+vy2e|cWXhx+BQ%luSzzb8N@fmCXutTQcyy#5(s$70PXsWU+gl+ zyzd>r(kKvEJ?X}^FRtPu(;QqEj0BhCCE;)!1CDMp2IihiNPxZp)Vu6K8VtP1hB%I) z^ld&I_^U&r-YS!KCUanq;mc~;{Hl6!i5th38VA>zNY;$MV8m%U zEbcXc4%ibIPg@KG{(zW^`GM%dMts<=6Wi@f0K<*%@h3k=z_djQ0uBVgj^C?DDoz4x z8;fx?;#eqthk5nkW+eG&hrNBI9gKU>&D%JACwQ4-1$&_)wu>JJ-(Sc97RNaL?<);r z@wXcrtq>!jj|Rb>1@7eBl|H;#QGxX7O@^bAO2n{d6zdMO;|4-UUY&uu%0E zI5~8>Ds8M1NH36uR}A}rmxdaNl=6gUwGUx$MHk#~cM3crwE-WhO{m_zXgUdOcY_rg zMkGkT7VC`Gg0V<7@_Qo>nra#Fj1}L2Tj~0wT7Cu80`Kr`Um5(=umVR(UImUV{y5BC z2Fh#9Bi&CUpyvGma>`tnNPRNG5wpv|-CrsAzpu)$KQRM1?Be*4yTqZbr8)fNGncF| z<9a=6e}RhhBoIq~3)WUX6iiuZPjaQ?q3kiPM-ml6H1xDc%Fk;2cFR0s8>`BDXW@if zZVACwWj;Al91UK6o+7X>SPTO>24lwOt?;!&39eYL0nWEs!tn*tWNm&hczFc_e~lLK z>}3d9m2Cx|u8<_x+XQ&!V^d-&*oOo5Mq}5V$H3;^X5h{%5Xg#2kcG7g_M2Tl03VKD z_OFB+ zJAL5NJYO>Bl0A8_Fc(NnEyiXl1z>y0Fy@`mhuuF2m}%_|3g%v`in(|gJc^rwMdkYV z(04KH5ibolbADnA{@QZ=05v$#R+gv^S-|AsoA`NIDNpBx6OsP37C+hX5KHf!KyLKA z!JC`)U`J3NF3)x4wJ7P4CstxGa)&9@$P2Nzd?yFLPK*KpLw2xkswxp}s{o><=fIO- zeYj_Q72col97Mb^CEc<%WX|yi06zZ-9!Z_X9YO2h@QwuHRs-SZ{csvX#`$ixdb--(IE}{Yy>;?{{qpZM!fvD7mQVufm;H1fCkqSV0h+KVwbrN z{w!&=ZxGWYXWyuUedT9>*WYpA}P}squa_ zTf4+|hm$346&Y~jrC6eLm77b|9KqArO6>T@8}4}T183#l19F_#-O}j|ymmc*sJLt; ztl?t%lfoF7(k4cRaVzf**XRE?sSRvvx`Pi(WP!7O&gAHb4?H3Y!EbHH!M;xxkmqVn zR;+Hp#cO2Y1l8&A*(YZ>V~z}Yk%7p-UuPKpU?tao>cla#`>-?j+xG{kkjkh3z&1T` zcsu<#_@l1~mpWOH@psk1F@8AQHZTrPsanbVv)hP-subYmt>3{U+r{MUv?x;CRtGAR zmvAh7EqGDn3c5J%Sb|hH*5>Bgm4Y?UVUGd1Fgh3J^gYMuaWZfgaPy>7+Mwy)Mtq{t zfZ%X@95NP!;W(l%t#U=YB_x*NeF^h1l*CO7b6$@d~=RG`~mbs*PqYHU4;sD1M z>OeKlf9X%1F^=u?gPV)`K!bN6Na6DNl&`nJW268(zHvN$+3Vo&?6>$u5d%(o%z(aU zd$B`^0z9(85H46FOOjm7pp2Ry?ud{j|GWoy)>C!K#`RJpRmuq-G4q3`wI{)wy^b*O z%~ot7{~3=S1hD$P8Hq^}0k8k2!~06{c;&`Bf~Lw~JNIqf;AE&H83>IBLoR$$Z7%{V zSVc0HX$DX3n+dCmRWNS+B4F#bLzSTU(58apcs-kmPb~`oiy}kd3NLx6rltcg6)Yn0 z7NKNM?kPc&i4U;Yu$)*l?j)uM*TcgJfCbA*s4RCWbftOZW5lQJp za>dC2ZhmJ+BIE@EVss9_%d>&9mj=M44okRUh)=AIm0+K@6CuuTdG`4(K#J=v&8u+c zYNHO&e&9T4agGF5r*&b$24(m;d;+=Y8Vn0wOo7L~2|?^*Gx9X~2;gKUklDr%c=n^SSURtY@FsC`hNBb2R z+&F_gHB%$%|5V`yZstW-iXAGgrpLw`5oM&Iekh%$mn0>7cY^{S}!xVcGWAVkF-*E%1|Fauzzhe%c zsXqrGO&i7^YQcwIJjAN8wIK8XE9gk(cEti_1DWDo*wi6X;JADSH0_Tg{vTHpp2IlO zkimIT<@#WgNu~C$+7iGgzhQiL?f{r{LKEJ(`^r8dBMT4aUc@URK689tPi!c#!#}^a z;C=G5h<}zUdBpi~t$Q$w7>E{=*NHuLfMeSiI-J29jHZOK{Ho8aTb zM10bz6kmBY2yS0qjh{MM!55d>*i(;p)At)ns7l{YNsRq#V{k?*SWse31S)6vud5P@1c~jtcC8=q`XXT=d2P{A5Z!-I$dz3%(jvw{s+-_vrQ$vx4D?U2|a{-%6C&`t#zLNmZH&Wcukd4D6P>M_LvNq(qG)O(8(gK2uD+>3ieoW+JY~} z)-Ofwlk|}M)OIFpvp-$snaX4@?`5S%f~oAtH}>ZNZ**w)kkF-FMRZOrnY~~wWH#G4 zq7)|&`Xg75xgJ(U7fQ=9dL`o=6jw_j51*~f&QqBvIA%Mi>5M|54*#L<=NYU|&{X8% z??Vs29p$T*dWn8dd&iDnn1Bo|&Qn-fz| z>%0x=p!rJ& zk*H>zz~>x;=IdRhy%RLit%y>3oYP0RNI24R4HIVJ*>v>QM3M4SuTlxS3v^#RpI%B_ z&+gQx%>8ByIvTFv5FtLqKj>CU|Mfnh3;WYp4V20hcj(Z{=y@pZm=~Lzvro8ZTpcp# zR!70nzj?+tIlaM=WSW^Ejxwz`GGR$-NazqxO?P}okB6It(I!hMGq#-7SaFw%D{{K` zK^b&%+9z0?AB-G^C$ahal7yE#{n2da3bOrlG1}l+%^!(2qHY$4k%huVX1&c}#-;DA z@ZRVmYMH8m+Q01L*oq-kbwexTdFu?4a?528CF-D#=W@c6UHvGcONKtFK2Dn-ucKm@ zjTl|M4u;CQu@43>pfi4f$l3ZJHR24qDmSf0`KwH5?90{c(L>=>YkUQB@WEc@R?Sl? z`1yxT9aeO3I`#xHeH!Sza19c}35-=nG;=5955toTLDqfC(BZ;bravy0`n>38Yy2R4 zCGR2nmRL;xcFQQYr#w1B_eXUfTGyjHz(wL?@Vq%yl~*bg)>P-J(%}QtSZo ziE2eWr}@b3$aT6RB8BR#a&wqeI*$e&S7d5}_0WgA1|omg59|}EiR^S{C-bw}57kc# zLcfM`8Q*wG(a8xpsPz#>l8-GJtNA`m+>Z~!&`(A*PW}_h+F?rVzB{9M-&Eu+bfvUm z6{2&(snjlGdbaEl9qf9Ezm*!%`m}mR{9`)(px!I&irIrs83ZEzl3V0LTs!}P`A+7; zg>$rLM=D)3UJNOVN73dRXV7i4VibE6v-P2RqN)o8ROe5Ea8X-6)p<9ONz0I;-lkj7 zh)gz{qx6aFnItQk{90L9`@xO5DKQJh2VG`nb*$rEI=q|dj5*0rZ9N*cL0l9ZW<=4c z2G*ndA8Q}gz&39OOs8iz;$PE3pv1`G{*ESo*clbN;gte?H|mPe2Tl4RIhUSSS&vq% z+)L*tTcP6R3u)O?CuY0WMPWkVV|K;&f3y!iX4X$rM9Eckbd%s3)2m}hGp@2s{=!G} z$Az`XT1$&b*N~#7>QiXVuc!2Upc;A_Fp;^=>4VkYC{t(~OFMTBup9qVcKG+Z6kQt~ zL~9SvWh57Ap-WE}(xCB&Q1*NkI!;rC)hdyr*4k>Kgtvw$K^~*0ymBh>(w@3ryN9L^ zmY^tgNzvApnXGL<8$A|EDMNEu``|?4X0wczsAjTVleJNY=?fYeHd(Yt^*Uox`INpq zs3kg8v4$#ry-YVny=0zvykfrOl+%O1H0kmyQuOHYL&)yzbgHsiff~GQK_1i7QLfDn zI`ElLrDMUgEWLN@DcJiyOjn(ntG^kG1nUF&=V#UN{Z41oWhdUS8hO>Uf8Rtp84ogrXZE7upJT#~_pA9|4kJ`_ zBZby|xlMcf%9(2gw}rnY-oamqbD5Iyl%26dMyR`6C@i}APRN)QqKcxaqLlZTnZXRPgy-t#gm%cJ~ zN7kY2%wzmHdHLvf!*r%k>kE6!J(m{S6tnJ0wN!Z4lScBhg+XnP8N&{)-GUwY&r&p4 zw!r``D$S&CKE={2mRH&16MqX;H%mD9&3^@FTFIfH&fTn5{cc_=ZxB`oVY)G678@Gw zjAl5>FnbQ)VYUXivX{j)tN%%d&KqxrX_r|P=IuUjxbh757CC%S#;{B zNaQejm5H6#%&u@lq*+QVtZOnx;pP+Ev`K7VDhsTcjLbQ%>z&!DYW zOHf!jpRRrWm-(V(h9W=8pcRrT4(9f5Y(|YHv$XRyo9%a3IMHVj-J93W+)Fb@Djn+V z!&z_8IG2aC;JY1FjTmD`L<>YkYYT;Kt1qILM=ziq+A2s&$s0A4%7}tSPoe(X;z)Cr z3`)Of&dw|DWHug9M0x*-vzx~^F~--op&XqoN&-%!zTT6_H6n$C-<^m)Y&^)Un!l5# zHn-5|1S3}AHy8G%Pe6S>$;d412d5dXquq}lGI2v`bg{~7IKkG3Cw$gadCLcdZeQOd{@ zDkhiB8Lo$@UXKE|WnOgvq$dx0-;Oq1G%J2I|arW}uEgteb7CR%qQnQk|}gbL7V zWc8_>dAQP;`d!GPA47gKW4CgV+|ox>#@(E5)7i^D>pH;BPtHfH9A=^3uUpZQRpRv9 z_Db4TTFfjht!DW*)0yKk`9ibHAK32QAJ`vAK%0V&pc(5DxM~62Y&=3818SLLS;^>b-E9eOWFQ0+>NuH!fe+9ITPG>!iE<{>{$7$`iSKiR>|W)Bnh-Wm<;Yh}uM^;q>E9sGAa zMRZbQHjUo=Rj50EH+BE=9?9q3ME6@#=*5RG(M9-%w!IvOd~2fEA7!SfXSft4@8%;g z=r62(+>exHryy~|mHe-Uc~ts>AM%PnBOKjjOQW-=FdIg)DSLL5x~%P|mI~FREFh10 zZ#^Dmo{~bV-_^1iE|ZW9-xSRba6?P(U!{AI^$4u^lySn~pAY|6=yNJkcb!q>R#Pr4s?SFMzAdCPx%ii_ zS&Uvk_{R?q zo};+6iypiaK!55WLlsd9vQIDzBU zRywj1+SAdV?^oE%2hUOWgfdz`a)C{F*eMJ;euPcFJ-|G@Ym8=D2e4j7T}b$3ly%7J zq#e2fwk>pkX>CeF>XN~}n-v zG;&UX+4{1A4mXw2qc0Xy->+uq+3py$EvTA*9Vens)zQr6aTi#LOeyB$?>mD0;VD-+hV2ZxOe~Tvd=Spwxb_zD?tWsgy z&#_f0_vpe)OPN()>lkcR#7w+r#%hnhh=vPUWXd$L*1qqMZF>=OW?eLO$zG293y;v{ z+b7YJ8+Anz!ha~dP>$XkZD9VYZKdJ)zKDOz8j-z-e(1iB+=MTf@JVI%}F2-DzjVnD2OmQnqYnCJmRvuk()}$!G;pnDIrXDEB%Nv(9GD z6|F!|uI8ZG6W-Bs`*yz5fGJJa3t?YL`p_B9<_=MjX6VAJh3HaG3Hyd)^L*Dmg;YHD z)9{?>jCHXU6ES(R$mXq>$eF~T?LVAoY}r|QB|U~(Uf;khOUJ^oIeN6waENh9UC(^l zJBWZ(F*>y;iyiO1o%OjBN{4zPXkAVg>+H9Q%Q4cq|6wNG&Mg*SE6o@V0myo}%di3$e7WAKL3)^Qd$yd4=j3#SdWTb}G5r>2q*0?<7kCg>6)4~iy zBPXAu&jugZ8!?5{sGQPe#zpkY*j}_!%n|<6jiy7k&(WBnIMqLP6g4aAQ^kxf_Eo+f zb93Df`lB_H{rPl+nHYGR-M{`YT?Y|8^PS7t*PGBQhIf&=v=KAEkJ3%qN)&zr=vp8j zWlWok7QOk!eEqKwCAbRcXv9(0>4ZLGFY}t2uVsr&|M@bFzn;-?ZnuO#H|hwDt6wwA zcVg-rc^nnt%Lu3|(0Tf=nO7;Z_{jlwG`Ic|I@!9G=`g5c-u-+<7hP;Zqhve!GB^z( ztNq;hUB?(K@j(0Pw1o2}`=Fxl2sP~5&j5`I8Wed`h-7ZjeFOFM(2Qs7YZYr6`nZVl zyb#*u_?jO2&w`PdC+08^XCo?nWr&uPwo|#Krs%i`QW020XB3}A-J4FZpX6nQ{Tx4L znu8ymamj}hL0&@dPk0~&S2Z+He+d23h^4XbRYc*Vvk+D{KxHq2QM+3an|vb?>Tp!zSY<70+OMAxu|=T@L+->m5QJ$$nJwqGA&zS$9D-4fAqm&vT#u z`+WgjeDfGzM=S*;_ZP4i`cy@hN57zR$%Xu=bK2-@w-fq4+XBrSdQV~T4Vr6lf~wvQ zqBpwMpod+%n2oJ%)IfbV>$uUA{ZlUx%DrB|IQswNH*`U2xvzxXGFHwse_epaZW*EeW%mXA zITa0TTH-q4x% z^w66@Jtkt@08QT#gF?A`^Vne_?aV#L4F0ErB-)HbsYj$~>i!hEQ2!6(+Y(0)cH4=z z4*p{`#JAG2Rds05DO+?u^(OOO?F9Ym6-YG#tWbrD1HDnJME9r%GnKcX>V#>+_Lx2BDqmVb}tzjx=~NxURHSnk8FNy=l(R~%)XgC*GR z^^{$sewIopyEcj)Ap-OBpq z=s}D!Z4R%o5(3>dd>I=sCg&B=*I7qWN7Q)LJQ)!&yJT&LS3KaeQ z2s-ufo>1LU3~f7AL^Yd^a^}ays7W(}uCy{{43?Fk$LbuP`jHblVVF%PMINPYzBd>f z=jT+F%PE|DFC(dO=a|c}-x7n(V6(w@V#-oeN}0dN_$k2(mLndISHwR zL<^B6@wI2o9$KUlZOYPSDP=8Nch0$UB1NPHEh3Z@k+P&n^1Hu(;Lg16Yi7>#e4gj? z{=~M6rXEmp+wd%pwYoFHZm?_k+D=lRziPd!kx}$26_uB`fnlpC0-%#OtL6u~mt={Dot3ZsX2| z3Nl5{_#wYMnwW4}H0qBeB83a+I_G}YC#ZoPQAwxEDt04PPRRe8bdyi@^rq?q2a#7< z2%Ye)foA_Kq1l_hqODE71XYF{y5@TL7 za6nLh`7u8`w~Xrk%jWOoRMMVpk~D0?Sv1)!AA0X$MG4k{yv!OEn&~W47N?oX?wM*& ze+Pfj7BtCYY$&F-k#qJC1^UV!E zBDa_={_jUYh2j`Uq5o}Mh@Fk zE=jwx1?dUKWwP1H!D z!e6rVyR;*Xd>$bBz4fL*<3KI{?)Er(SW24)yZceCFJaVNQU^_&e4AQ+$>Gy>ToxP; z=|oQFhv|0(b826U(I?yI?9~1TNJ`-r)w@~3>&m=h+rl5Rna{21r@N==#W?9Q)$Bs{ z%QE*I=MU07BTlpM#~bZt7h{Z&m*W;pk-NE=XZ4G zgCcT#kc^hNC9{!C345vMGG6fVrl{HH3!iAaidww?$IkliH?6&Mgzt*4;eAHWLQ(r? z(5*+a#SHXEv{Gh(mAa@-6>C7*`Oj*6_pZ{@hL@;|%>h)Ddq}kLi47kf zqC;=(K8zkbIL~q&#z?~IKwyI@`hRr3Z_U;&V z9lA|7i`f^3?l1Yc?1`dBRtd<+G=)BXx(+R9zRr5bD$%un-O-(a5^6BJoR4bw&Nke5 zOe+eyP<5Z4{&>RCogpN{()heT6uiB=hy@o&2!tEabxe%*GO4LExjT~o;9^$Uea z>a89vd5}oo1x-LI0&5ie?ksycG>Mnk9gMs-grg)Q4>3#WzF_W^44PqFOwG~*sI6c( z8teavw`)9ve4XY|@2P+ekMX5lKOVD(Yo4*UtXHAm7s9BUhdF=7^f0~Slt=A`7&f+J zgmtJ-X9KqBm1)Z6@TcC7r_D0?bjIvabYsXb(UXJA(N{}L_SNlkym^ltdOkS`?SIUm zZ|g46i@6sl={t#>|32aUS3afmu5y{l{$xCH>-{w zQO%--`}d)9ru*rK>ofTFs53~TJe3cF(q-r2dQ>!AOn-bk&ELONM1Lx&l${EDjpXf2 zQCin$KI*QrTk^7SeotPpC_~BueMp?+7KWb*>N;=JN6yhmslAn-I6jXx%*cT@k?}mC zzDRGJ9qm<1;b&d*5NqiQ=#;}mnl&>CNy>d-*HsTvlkHd0@ckU58ZFM|I(G(r%AA1& zQtH(Dw`7^I^BdYzxs5gco{b`7XQQ+;&Ghuq6VxWOl*XNSfGQQjXsElTD8VUF)E@O0 z-aSnC%JNfaNoG7+vOyaCJCcRe)}BW4N_o8X0(qphay-95;}{CgRYGY#&jkBt%d<~h zoB0(Y%KjUgilX$o_zCOBpteV$=zG2kuk8?wQVOHlc^j|O!?&W)ogrgt&{aYecHBV` zQ?e1UFhT9vm)Ttp5|N$cO!{A7CYr<7qpXKYyu$%)YP<9fRo{G;4jq}uTira2qM|$b z4G%TkL^B%MtG&_uv|rY=V9Qy)ogG6D$VZ_>gV$thyaH|f7R}$2IV+wYK4*J$M$t99 z3TTDq2&;55PGIpNmVdpfmRBkK#eNd-$mq!@k%{JUYLan>?n#)9dZ#C_TWA@re7u8p zCjH_|yc1~AY7xJy?-9>tsiQ-O37YL;M$6hq@r4>+(5_@i%l}E21wU*?mY0;#dFAT@ z|GD+_fSW!VD-5TN6T11n_&I>f&#GN39e+}Ml-(yV`6Iu#7qkVS5b z_~h;-DC&(PuaG{SI&LaKp>B%EYRYTY@Jk#uTyvgur+V{l!gDO$7fvCNb~}12iLI%% zq!0eKqOi)ns5|z8z}aLo(r`P@KIrtKw=`;4DKVp8^WjbQvX2+461*Xfj49m|ae@DJ z%^m&SNm+JXzIZ+vhkn~JDA+I#NvxD+U9=C-c`a{{*L`;sE!Mlbj84(fdgsw%mlafI zffudcE24qt^6AVw0=~P^9qpSY09~{E=-}cOHcC5>O`2NAo|&wLK9&9yJiNbxk_d5c z^Y#k8`Q;^&6VG;oj9Sq@of(3l(K}G~)B*l?CSjkoZKfwQOi{c22(O`Ziq1`JVb58l zqXYF4^z_Fk^!C#8g4)m|8W8J(_DocQ#>oS8wV2(0Z@xLQNvDuq{vSKG?jgIh22tVU zuQYpJAnkmfA*i((%O-WIiPR)&*mjjf>hbcGw z!ZTW8Hl7au$w2=d2-eurhZKvsvqQ z(Q7R!Xud9?@zcb*&t@%ZF)t1oxSymx;4uH^O$fig^(qn%?b*xEjO3OGJKWpm~eznTJ(&%$cj+MzQ26vhI#zOrCX6*)lE^McN;rg@QuB!QGjyU ziF6>+TU4>QnZ_JjPA4RsVV7QxVp$zCnzGReNpz29hgT$^+O$|&bw*rIthb`t`xM+( zx0g^gs7JSdmPe|y#auH}Uw(ht22i~C20!t(7cwcAcGIl7Opj&+(=)F>in63v(EWNW zZENZ!4Zgi-_?==|l&?B{pLdA$-+Y+r+*m~mC&{5(cl*h_VgV9=dITxgrqI5~DQ+RJ zF0#YlGN`SbM48m`ul&)&JUkv^>_(n@Q}$@R7>w`=8F@dUg28I~#-@TUHVzQ8j@Qx6 zMF!}`!yjl^Mvm30E#Pw=M^pbtN01P#VmohWL&?L&G`r3n^$2gUCtkSH7x%ZJ$7`>N zx=+fZp(-bl3&+!uk*n-L+#tJup^|_%$`@pveZwzQS90633R+56D@wqQIFB@XkKcANR%2wv(OAN_xmau8d9PSb2O3f z#yWI0axAZPV-+7#AfA`oCJF-9*P-3DH6nFkH2u%w8?QQQrJ(G6K1y5>O$874u&G_4 zC?bGGo_l)vN{5rQA@3V4OF4-Arq+l#v#kPq?0~Xg%DIUSW}qIA99qy9i{hPcu+rTY zd~n5WK6Lpx!8h&_vY%y%PQ71-uK!djoBS#fDOEC5-tIj05B8*!3_R(Hs!8;Sydg?n z7=Y&gye-n0Go4<#WP<)A3hBF$rL5r1wQQoVWXhUs2`+DRV|LVFU zFM4taMcGu-$7MnknR9{<*rvdC6nV218PjN9QkE!O@fWN9u?USmF3rCy^%P~BXd?UN zN@(I>20#10AN_Z{huTQ@@jpBhSylBW+8*0Y*SrlzoWU)A)1GwXp<67HyZxK~6#qxJ zcm6_bm=T+|O_R!HUqRJplu)r&6F+iXAkJJ#U=3=I(2m%(bk~3<`gp5|3Q-a3AS!1i zT=%nmYvt%r^IcwDPk>x=?OCBdgXYeZEE~0CH0$}kjaul%pl839qW#4z9k5QNE?S3J z*@NTggJwd#pPv>?-#8zgc$11g+jbyng9aKL)`uSNbVbL)SJ5N6gM#t$8T@0LTozv+ zMct*P(Y6qC>Jm0c3mTHq<}JB2E+7VJEnkBkD?jDG_S`{to$`6r=jLdd`VsnVk}VyU z8R0+IS5pn6RW#1$7*)8RfwC986HGYL$~P5mX7_Uk=zB|BI%H|)maMdkOzY`>^9Y+@@E4l@)%w<{q zW~w&l9a8zKK@(1HrLExy(2&7VzDZpIb!YtL+{YC2$=hs@S-34 zb{F}Ag`srx{yg^OwD$SF-O^U(yHf+i7!)6901aR9X-sL;dUz zv5j9X=#GXDNMEBHs!QBsU7WU2W#4>W+VU7BUPpMx#!T@7*G*cW`HSCjHwBIPcZmO) ztW3ZBxhC>mltZU(TqkBpdLq3S7wHkJoAg4{7IwzEcJ@{LCVJr3O4L5CiEU0?A_^a$ z&Tdjlh%-=n*^ei#BE9@;Xisw=JHOhH%FnQ* zhc<A3}V2GiZjawzea z4@&%afG+8r&ib%s$dAa-$P&W46?LKgwWHlMr)APMo$Kt`iSPK-pPlpc6AY&Sf_$6iuW5X6<4rD z#{_)d)vtW0ye29L9qnemHy)W~t)m-tG~Irs^sr+0A@!^|jwi&LmYEzgqh5c6v`*56 zj_;g>o|;+FgZGQ6c3Km1R$771bS~k~-ik)k7yXCan@`}Ip>`CNw6XTa?TqNvS^Qm5 zlVlWexcVN44KLOSf2vu7J_iY`&MXH?zs=!_EOlt&J09+rUkSdKj)4zKmIB#JYVg_m zR7Tn^3+&k71CMxqW4vsJz^6C0uE&OD;Zd_IxVAkM&^~)OzT5;Zd>aY&?NuV*DqM-r znsJP1PlDKE`-D5XXA)3+{Sm|`%>ro&3Ea#>x@7t*6EgmJDOQM)0hP5Xu&yB*=%;Ey zm9CEfMNT08%BCb;`ZzWg??bO$(}O#_E;Cx9OWZp5%iyYm8?>BP3BFyQ3mT1@!1^+M zQbVSa{EzQ&$>j@J+BJ}KRXxDm{n?504;kTjQz^3hP%qZ?YR6oX6Uh&8H~%ki7}xB(&9 z?vM|D80Bz-qmJU%XkD`Ni<@|-+X}o(mChXduiUs`xfSiv5 z{x$xv_vR67vNjzoAGnE8QMmAT-YWdCNR1p0U5oXU?TFVuT@W-7%&E0{;c71}FuA0b z+0rBnyA`a+`k|X!gN0$!P$gCDA% zD)qc)2_G510QMd>AR$m4x@2qP#q&>rgBq`aS2FGlao6 zB!PN>F)SI@#;3CFVdPRDVwj>h z8_JRhd^<4h&^+J}p zyd(sV`(6bk)>*=;;*wI2r7m#Rk$8ME`w@6~?>e~TWJF|hE@JW6j;znLCJ|kiuxct({6qdhNg-4mu z@S|b}SRJ7TlP>%Qhdgq?TEz|6$DshvA9o0J>A65JsmDwPbRmsP0j&6O&Xr3x2R~DE zVEW)Au>I~?D>Pv3)nw=VlBm)+ zu`lA!&ScOiW`Cx(AHvQDREbF10D77B-f7Vkb0th?!bJviz<=zgh$CB>mtyXTy7x3z;zhHZ+ zF>g07n=c;oN=>fQ=J%$kM%v%q(LGl0IAx+&7hT{x7A- zz9)vbu<|%QmQKL8ux7lh%nDEPk|h)5N5q^Pb6nR$@K=w!Sa>yoOU`z{R#%OQueBcD zzpx7wKEBO-eh~_*)QyRR9|Ke#w&5Sg7J=cKA-u%uEwkW{4_TdVL(aM$5ogHTkiy?u z@SeUk^sL^>sf&FaPW%3XQ0a%v;3@|aeZCSOW)#7~KfVAoWtO&W5|C`eDR7PALAX}- zF_&mXaDIp_u^2rAPm0Rtg3A8jlh)JVuYor1hPxB2UN8!dUblsT(>uYy1AB14cpdp{ zW-9jX&nF=t6(AM+U(c;{BqPT>arfS*IG?G)Yh(&Rv4c90sIPKytHuHtD_a0PG41!D};`fe&#YjCU*U`yPf5v_As<^u0^n z3M-gh+Yi(vyul{No0#Joxc!3H_;0{==5PtYQ}Yi2ZN)+H0)K|k@PZTkRDoT0L`Xtk zA8FD#XA(KG$dUAXm<;llJi&|n-e85&oYMTZnea_<7q|YjDe>0$0ZzpxgOBT^N&l81 zF^Ba7^LtYPwy`mQKYZ08Xsp4j8>JaqAVo4JDuKD%0nrdfF(DVU$jh~p;LHWHp-5sM zT&JQ7s->4P&v&QejneH*&2BZK{Z$9fv~CgqrVk9Qawcz;4B_*kgZTNN1FX0*jQ^Wd ziVr1Nky2C#7Cez8#o5-R?NAzHbo~>y^8U$XC#GPVdshMMo9ANQHHvWm$-recQsCFY zD6nCcHQD0618YXEAh!!_i0k_poR;|tJdy6lpGXLi?0x~R$u0w{HJ)-$|EmU#B8<20 zjs>|&^pS0e5Y8LxOjczq1#>;L zVcNKpc%0utMrD}`Z2PhkzE16Cf)E4$c|FIQQxsvqwR!OIlqhViC&WvlR)IUGAAxDc z+F-^y@wNXs5Zj+qpxbTA8;=;q?x;^(jFiaY5%VosL5^MLe=1DP{n zG<@5^GZJ!du~d2`1FLnRSJXRjKI|_e2>S$rZIkf(l1i*D=3Dm88^)*7K7l1)y@iK& zkAdY{L%8=x6gJEo56;*-z+ZC{vBZi+c$K;tI5X`f_*i@g=oe0bC+8pGZt1y^>#JhG zim6jzbh?NMpBO+Ut!M^g-Tk@ZWjgS~zK`NVO&Z*G_=6{07)`W;7sB8shGe`4SjAt7 zYmc;mBgKk%`L5kyL}>(PFHXaAc58B4UIHRFp$&KsE`u!<+9Y_99vpXlB6&3`gV0DFpZ_#+Y&!$Bz8(eFT>RnEI(c~LfD0Tv<61gHdktyS@&Y&3Xp#wX8YEco z0MAPr51(ufAzxt^Sbjbeysp#X{*@>|(Cb4Ix+3xFqcvbyVFf<%AP^Moui~U*&kFZH zktb_g#^a!qgZM^_61-*Zh5L@5#H;e8pmlmE)G5vZf1|g9HPLH1d@Tw-egMe$KpVK? zUkrSx;sj6X8Nn;sC-FYXYVcv19$9&F&z)W?ZG7g;Lu@%O1GMd$%t*#u;aE}*w3K`C zIg4g)g6ANJ?@}UWGxf=fjg!ILSN=flp9)O+BnMwCY3433IS&4t`x2;K4FyVN$+)kF z!}AP`K#!QEE1Bg)+-i~ltEzIhsn*{K8tF##aITa`@7HH7ko zC%}o~CfCv~FFZl{5ogyknv`#95+={`Wa`HBac|eEf|T1^fPBa=u)NHKtQasN2`LZp zBhA-hCSg3t{W6iee^r<8+RqqO1o1?7o^c2@B0H+GfSkrz7yH~Fc;Rdpu%`Yf7%DI4 z)Rx`{$+a1v+%*_vj(-FA{aJvo%5%wMm6*VVs_^TB-KAlUO`!jUFT8T>0&bG-xKV=Kn4Ii#dRZEm z?)DoGzIn>Ag)2dvnHHE>p+X9l>TpupC&2U6WLy$-!{uq;5-jrVC{-Fv$32>tai-!c z{A21E;#}W=V;hR_Ec+7dGu{Xusgwbscp6ijn$MkXoQj{Bdk_nt#Vz<01v*6@V0Pwq za{9(i(7zFav>+SY>|qI)6Zaz;sF`R-XIxrmy9CpzT$u*=1FojCK~u?QVYo1s=%FT z32=2QTZn&J=t6G3xXv-vg*#*0@s|KkJf`(4c-2=4k^{S$^A;vpb$%a?Sh5iIw23(! z>D`!(4F`)K1Y^~&O(st8HO0YX z{Jj|9=W0` zZNsxcT+Vl}pyVx>7HsF*Ugp6aPZ&)=@kF@S^oy&nhb%lJW){^i9R(|Y6a)3|o_PG; zLHsjQ1}lM;@LOCv4Z?T5FT;~B%Qv*jW^UJ8wO@^tqPsov1Vg-}T8I2FV8{e~;Jo7%d>&fmlT zuG*8UD*Zs>O%U$9){9d*LmCuo`a!1r1XxwpkG}@p!29y;pzwJq_@Ofs zu6^nu%qZD`tx~#ygR~8a^Qi)##oVi-=96Gh_iHfgyA`~qn+NJ7oEbTx0-R~ImI-vP z!V!Ue%;djY!6bceIM=0E9DigAr|)pZZ>3IQWu!w)mBr77OgZQt{EeT7D==SWQW$xm zB9{->a%YdU<8KkR(0Ap1{8XwDloYv=&&l&}u7wpe9vlL%eeEG~SB2rHCcsZ$4d7g- z+aT=jM0j%3MWB-xj|;^fTlcSXK2fErYHe zj|#yF@tO61hTZsiXIL7g-~N!hSUy_JNlL{J#f+BxBuirT!h@K6al@U$Re18TM68`)PbBLC$s((( zIL~($(UejKr}Rf~N7)>h-eidn$*o~#hYaH%@H#koJ_+pICq)^F$pV(L+oZyIP)0p$&Ll- zib7_hxNejQECr@pO2JiJ#7tcp0va@Da>Dsy{=V&fusUjx6PVuwcUNs;e6A|v)~^%D ze~AqKGEW0$^8NT==qTV)8;XmsT<12l>B0r~TDaOt_d%qOI$7rW4QKq?guSzVa}82~ z+*^sK+}xX$V2E>phqmS5P=)RAhTA$o6F+fZwes*rvK^E11stDG@U_AiFn@*@)>wO* zi|%N`#xHg8`Jo_u{>2t>;hzmCxMj$Q_8sIxHEVHcr)&iw^W6<%o4=mD7 z2Cr^L;g|N}K61%e(z<>m%W9EZ^PVy{R>?u#=2W4xa0_YS zP^@)qxWZN0W`M*D8GLGE9rq~Z9`mr{K2B)mg#P;nLDS+ke4@|_j+uQ6oK$`RcKp)C z73p%WH`Fe3NhMmu_oWuF(_D$8Kfl3?^m$O-?g500#9E2G6||2{2djd0;8KkPc-z@( zrbg-=_s;7jD0Ny3j6UlCZI2Lqp{;*o5k4tUGp3=?SFW+0Px84VX40{Xi_Tv%!uXhnvT(k#}%;k8!n-$df zQq3*aQiFRXSAhSTz47W@rOe22Rjki;;jpC(U>h+ZJ5^8MwnKq6b zlkz#C?LVB+Rtk>cDm>w)0r~mK9IEWpC6gD0GZ|?NG5!{SpXa{EP52mSXzT^QO_ktk zdkI|QZUG+X2ytCW9r$%bAM^`kVN>x1@bp!?FxNEzMq-R(hNMZwf^yfNNA2-PP=Om*gYC7{woa~jNfC~#yk)i zdJjxDkHedH598ye;+}HS1T1_t0`|H`fygQ=xa9VE9A!5E>f}E%r%^S28MqFp_2+=e zaV217$0FCs;<{>IdKUBU?pMakVn4I?;TZASdSa5FOS#|!>(sKL?0mq28{33)JX z2K0psh;G3-&~=A!Z;p;Kw86LEj%y7C6d$_gMm_N`z%f7oK_WN{6s zC%&8cULe!+1YYdS;f_{gSS!{|(idrypqa@)XSOx5c|8jB=c+>2!gnBNsV=#-?+D0q zu5b+xti_JrOTht!VGeJ7id`bTIR6KxxO-;`-e;B0RL5V%nRNgP0(KH_GJxzjQ3l~~K}5q2sW;hyD6!>qn9 zc#D{&s{CpqxN5KvoUpMXt3sXO)rYIW^(q}$)V+)xf7%X`M{AQ!W@7eA(-dLusT%wv zy^3MmK7yzp^WbW$M0`~;1B?u-lVq8EEW2qF(|>*xRGM`fcL)5%%C6IKkVZ1sc-jQI z?CHm6yxYM*wKRz6r~qvekHOA!Dd6jDBb<347d)Rc4(e=~LJsU$$5gNC$9YH0;hSU` zqBqta=$PHWX4Wi;UEjs{shYu*U7xYRLLCyBu7e+}T7&g{UVzbRtDvo0Gk(>k1Y=J^ z@@&p>VyHJC%6tA67M4NyB>4!pZc`@s_jVWPc;+HzoNXkrg=SEu8j{<7fw0xjhs>H& zFI+TPnxuS{gA40y$d&1>pnAJI>52xhe_A`&YHSH0w+iL6i_IBr<;%JlHgQRFx^EBZgLrqr;G^%4J!l4ihMOLDD(^WwYmw6iH`<4yb?A* z%fR7JDIjWa2Ik*dz&Se5wLI=FIMMnZtF;-B^ojZS#qkQfx@kVR_4*f3*tP)-kZZuj zHv{_(=|Od{mLyy}32wbGB7JN0V46ZZINtJ4I6LX7aHR49PSoiEB8_~EbCFoTF@@(m z1Ms==ukZnrM_@vc3bvYBi?x)(nAM(*+_3v1Q2%i`(L-ZllID04CZkUp8kIkdlsm}7p?wc9={4m>wnImEvNCw&{?4N zeIwYp_Z^low}lITodZ46-uU~0Agr_a3@}p*0|u6NnM^kWEGeN5;}c#1yHst$9_@ zE+T#6yXY?HR5B%6r?_KSkDy1 z-o)$dhOy~`nK+?pAJ+&B!DrcK?EJ(IMyz{+3uW!dlI4Ry5LN|7_1l5UuEnrSBOiR7 z)CER8nU3E-Z^t!D69I2RK=zewp!DJhb8Bh=e&p8<4)-o$vdc2Tgw!ysS7!j-4-aGG zVQnBiF#?;v9>8163Yfvs*&yVG6nXS?EcjAT!~L!P!+icbo_v~h5~M#$#ty@8@$R@3 zS4Sg9c=`58>~Q5Y7$as#543pT&^B|howec)EN&BOE$qZbhcj{fS94;zIe^%92Z6G? z_qb~hzhUJQ;*1E>dm!dj98Stm2QxmnkbZL!w;?niulzd`6gfY}qXqZyw7t%sm+OB^Zrn;SBbC1-q$fZl)Z z@V}!8c#^n(lsz5Jt^Gb0%1%6tRkrqH|6Te7EJ>OGe)s3>I1Qe}N)vDWGk9S^I4)HRgDW#FF^M!FchYw-)$f;qFYitWcV<2YkNQ<$ ze!m83+0q1*T&;;^$tK+G{TCZ=p3OO8OQN6q6X(4g1fhax{Qk={+;uaaDQbRHdRsxC zoJ~I^?$3%r;%!eNZITai$B$#|zbZq1#V%ZV$`~$ma|BL}qsZ4?-@prpWW2KaDRW3) z7Eg$7U>X&yaCXypm+i;y;Csf7u)1jy*{m`Q)(YR^#s5Y?SceNqYth1LbG3!jod+4+ z6JqUin>q1$dzoumJ{eArzkuz`W5ENdYCP+50si}pfI)HQNY(2B;2jVN9$8B>vf*!W zwc#0RN?gMzv*T?oR^2V(GOBqVJnC*qmbjS1 z9J%8dtyG6!#hDKQp(pX4@HWot8X#*glmn@{v8C?(RPyQ6UYsKC+uwXr1qbhZ08}pm z3PqoB#M5=;V2%U%IZ7M;U=f*wRG_Vxl_WQ7GF;a@k!ZLHg|Dw9Qt$=KX9o5gR_1xtHc>?Ix^?6<@htW zEMzVi|F{s`tz&_Y-4*<$UssrRO&vbEV~HPKie!8m#*kS`0vJBpg9y5B0qwtO%$EiU zGW0J1Zc7n!(07=SMB~*U%Uqe1j$|_K(|Uv!Qk(I~fakcD_Hs2Dn%tLjzQ9Ih9$9;* z3am{z$Q(4;4p+SPCEHs8{CmQfY(1e$6ps?{xkpc|pXcI>YplqVyl7_oNF8VFSkIWK zEWs~q=itrBP0W(XLKmYUHQ=!D9V0n?Ay)W)85`B_f}v}C$nmOv@T=62i`f_e9TUyL z-lr8n=y)CnRRj2T_7kD61YxG>>Oc`L0DEWr0-|r0W1jJIU~5< zc_+v%bHLSkL14)mQ}Sx^&0WtrnUk`IcI3(wGg$cV3I6w12uhasVn2BX zt`4>#*`A$P0t7H+bsMgOdDR!ks^02OTfRa1K3wI0A#Ega$yZ+ zuF3d=ZIe{MvLAE67?*0iu1XnJWnBQKnPb8AW&NN*PnGm(z65=~3-P&&^6+SzGmN3D z$R_0zpmy+sP-;#l4*SdDK=lja9GEqn?A|zL``pX8xk(=ft{KbtHCw~y29S(dZzAUW z_2Ep{g=C661kQV8NLWq*$lg(c1IM+1N#dN%{W)SCv)G#SZU`4j<#yvaH;gfxcZO5g z;6?UWFJ~MFX1jiguE2i+zk|`I`>xEC^B}7p!H7@QnxP z@beR9tu6qUB4=|gF{WT6F9*xSsVCp^Qox4;CRi%r9tVwdxC43l0LS>@4AVERIS+NP zjrus!w_O$6Aq`k6&QTx|E6Aa3>%o;}Y9w5470i;|4~xvkgNF0c*g*6P?>)YplS z)0^j!7i1_qiLP>3}UUbMFO~YoBNNu>GC9&!;%tKXEFlli;o;XgK{HZ$*bP5ZB-VR$h+rX4; zB1$kgavv7W$OQpyhsn_$>QMHiqVT#v2%evP2+F3*WBvB~ocFLEe09hT%Dvu?y(dNi z#q@go<+FI6ni306c8q|X7x&^WWh-3$uNp^NtHJLJmC2!Fhlq)Izv4~V2fT64b=Q+s z+ra(*D#4ku{rH>RG>FB|oSIm>Y>;||)5q-kia%%etn*qN-SE*1sQ& zF0|!#eb2&1Xb~_87T-thT;^YXBnUs73S=uJf&NZ8tToXW4=g+ec8|Np<*ST>I!D8y zsCFEFP&<#yGCATpr8o?8+a7SXH_m~uZKH|C!#ObadpwlyJBO#9M=HuJG2efk1+$5Em7dbl!L^1PdaI?R|EAH(LnR| zJDivL6&r4`!#|%XyFUFi3C5$ta8>3PZdS!~(y;g?zUit;%pberqvzvs=2?j#UvmjK7k8N?bPGH{LYLa4pR2tEqaz`!#Z(`qgJMJg8@YOr;EAy0Aj zubFt)KJkLRTrpG4T>^h34TzsI#V4J$N!RxKpnUmtAh)gsEQo9u7XK{*UL7Vd+%g7A zgcUNf?jJzbvFX_GRU+xzP!5DY+&Q&mL&79Q5}bGxTgFOrPeT!qi&2HuFYn>xlwnXH zbOZSxnt{U1N#x~}2==#ZHZyKkH1bvKN95sIk*m~6zIESzI;D4XS=sx8qCVTZG=1o} zsQ%+MwELz3l3w|V8b+L-Hrj|dV9zjda z)}jMOv3%+Rj=pY-V(Y}a>=I!nto@UXVspF~E4Ak)do4jk^EEFE!mY+4%cUEslZGCj zYxIr%RePIH^}9-2Z>I7ZHn&j3y}N?VOJt}Vl%uj^F7W7JHmiI(hd%e0E}Jn%zzSZ> zrZU^DP;2v2)+aXxdu&yqm80TVYni3A+}{bM)q9|hiw#so%Yfa|dT==^Qt9$%JD&FNzy9X1Dt*qBm5SugI~)>) zoxZ|rk1D56uic|hT0bJWG<_PPCWGLVLDoImr)+d~22#G>!mIpbckhO$u@<%kj`)ZHrvl)Z%``$oeC7^Akv?aTeM(l!xyB)o`0`kc)IRno((K2GS{O zM2E+{Vv}dLh(2n@p#ujA>wLYLraYHJG3R>)jrn)cN^XzX*Rl=S_vyR&4~=E#Z~jF! z(!*$%w@7fNm19Tha?zc;TPXj-73mcyP_?*x`cCkWJ-JVlFZ43zKS|}FI5RN*O@P$)5@l*D7YDy4WYHAQ&CHMHcH|Spt4n7B8l75?7Y|2 z)Vw%^Hy%|R*A-9mzgm-=v6(3O^B{G;lZ(!pRHBty zn@Qr>2vPQ8eX3w6L&Y#V>XCc~Ej{*&O1ZnbWeR?*{-mAzdu|Iu{LIkE4Yf zf{^cLJvXTy#j*{@chFPo^3jGRYp7pHH|wFO&To=C&#vt(Kt12H>A~Ms{FSxh@%ztc zn#b=$I>zpT1qlLr{$M7B9&gyh1O@hT&>ptV`#LK3v!l5Yw=FN@c=a|euQ8mi^;q6QLn>9D+)g~pkgGWdBgC8!l`>Y3W zRY(l#90_OFw~nR!X5+HDH)$v!<&UWQ>7x9<*uR6i%+3fW$#$shhfzI z-V>48_7>p#G1^f|&=W(f;5VwsS;QFjcz(vFQ}MOH?8CJl5QTFT`Gs9 z+~sMRkEvUu$9r_LXM}B-e3ni=RYRvb9YUVNF35V#QI@q@POl8bqL2y>Es?x{tOvB{ zoh?(*_|0lGIAlEf#ExP!SA?>khnFM6F;}Q#$9X>YrwkfCQqB&qnn~-eD^bMnXrBLD zC0ewwjJ3Kag?ju?^T(&kiI(P+(PP0AkapfRG-~H!8fYHP=LCDBh^$t6d7Bm8S9J&_ zb)-`#r5L{T;}Ue)^EmJS&VrgrtGfN2oy<2)xz2ChHjW3!3+SSwc}V+_Cq9we&LZ4^T$5U}Es;IRf zm$n+;M7wrm^7p@qePGy@%1ZZ%yq1nw`(jz`Mq?W0yn@=_ZWOF{=h(?& zL0dA%i>g>XLNX`z(z!vud50Q5q|%>;EX1BUlU)-9dCrSjqlFXLRa1-5{u)ZZ&ZB6} z-7|E)Pa_=#@V`_c7N4)CVm^U%-YG&nV468q+@ zKku$QhK{d4h~i7v3A~ptr8@qj&@}A*KZ@==oT}#y0Jyc1J!`g5q7n&lXYLuY3rR_6 z5lT`KDwUKiyFz5i(ynYRO76@(bK@%&Y12-rloZ;uYWtnvA0Gdm=RW7o+?n^Cc|R{r z?Ocy^B65VGi|){>un%mj|93_u(39CYwHI|Sa6)ZyhRnto&e$t}j78oA6ewD~ z>S|gU{xMg(H3ppVAhDnF{|=xHYEw|d2@}pK5fiC~b}}hYj>_@U(YVL{?2gn+^zUpx z^yiXOW)Esy$nx+0{J@+v6SI31 zpUgzRT}Qum%Gm+qhtxkffOWRApm7okBr+@$?P$J;ezmxu5C6PSW8_ojN#s#Z}wEa%mn>Ak;Bg^J=q_LI_AkduPM=bajxx$3R`Uwu!2QiUKA!nlWQ$UHAJMckL+r8((#USp zWVA8~vAL(>k^8=ntae@xGII8zo^pHHfZL8${S?H*Ov2t@0d%60&k%@D-qgo zS{^0lpP`<4s*LuEbY_F`4D>>ZN5vmr5*~w0-WFu|1v(AHgRE;@j)IQ7p&ymCsBXwd^gNAQ|8;Ed$W(dSA1h$_7sQ;t0P8T8zG9i?3DpH082Xf$^w>O9=aF27X_9Su5I z3%zA1>D((}QScdh;~Uow8up_Ndk3oWtW5v4DT!o5fAJd>_`;cYHZ$L@YKq?R z%hBlJ+w?+k0XzLz1lyUro@#|~b@E3E^KSK5{z>Bz)_>N1dbM?jaE;Cn=J^yKdL&*J z{dpy#+Y008wrUOjyK$MccZ~)rxLl3azqY4PEr|*o$|zKBLb0+1C_$x)iL91D4;%Ps z&h9okT2RO?Tjfuk?wv#(-cq7iw3p5~P)jF%7PqrIlM#(It)M-Ab|iyM)?`4W1)kE<(wEu@~k-sq3|AYqgw{rfhsc`M&4CLMSi_McPXSZvIFxAW;{aE&nzk0NsLXjr-UC*MKHxuc| z^BwGb;}=wls8HkTU3{t0Tf(@PElhvsR$5FF>7UjCx^C)yBs1*=n)6H?dCEDlnU2B8 zg2y?$VPa^3lC0?2{@+OE6k&d72x%WIMccq-=Ds;+kHTSEa?YN;=uyVl9Y4n!m?;Xa z=a;Yw4h^U|)(b6u|AGm9A%P|}yP(?>w(*BHWKiBi6IvK_0?Bpg(|$!6I`xz_%~Ltf zlx_yB#W5);AG}$Z(HMx%EqICkJ^0Pk{`Z~%yC>3dixVlf7lWD&f9Sg4qo~ky0)N5+ z9(t&4L*qUh@@=Bqn7NBOnL4josO8yoYB!|JztepQ)y6+(t6jD2VjMT2=i6t~VR;cV z>tYErXHN<`bOg{1UEAos^k}+KbrQ0dTR=0GU1vvFq^>mk*YWeICsp zN*7+<22hM(9vxoQ&BUF^KrY=v=A)Q0{XU>UcOE~6-q#kR$y2-Oip4kB2QE73@6m2* zJrYG1KirGV>}%QWl@FK$0WGX>xgB+S>&+awol6yTKC*tgkp6n3k1n(?qfc*NXZ`?h z_Su0G$g^oR{h~NPZMB=xj?{L>H828+LUwST+Xc$gS;u#u7mhX>*3!ut0W>~-4>kK^ zk5-xh8s~0f_q{e8?V6oScY;mm{C$T0Gd)JzN)Dri0jB80sdC0=uMvFuF_`gfF%Z=( zt6|kg#+Y2CGqmo?AnQB702x2pfPUUQ!{(+vWfk=&G2(l6p@Yiy%&#7*77 zq=AGqC#Hh!ahr_pwqB#>vLxwToWQ#OI)Lmrr^8%bU)0mEkiWy!1=^{oppU*PG_O{I z>Gu;ug;SbQ&m3Kprs~8F7iChz9ldO$W(S+PxttNtnZl0iSW4gZ9AUSM*@@=oWzx>g z`=^CpDBAx04zg2RPXCVO(ASp%ZC)DBXm0dCt?%D55m6m6P8e!P& zIn1FA#=^$32YiDyr_stxO}pa->g<^Fch>e_6zZ9mz}*Z@tYZfA`}SzN8ElJPtG$f}Lbe?l3hyN?k9eJCQ#QIur7pFN~- z4o%oL#+s>B(;g9IemQNU+e0tYthArPsx7nWCJhyIMMa%n`7BNY-&ol7t~*5ww@9Jl zMIYE2{FVvh4CJ`Sx9Rm$gF>-(agp?n(`<{5rl_Z{9O22XtW0$pT^{X#-i>f`MzND^ zO5MvA-+9Sz%|A=|P1E>VP7j!XTX)&SUGve7*B{v5Ufkypb~0|S6+Ea!^3SHR%a_Zb=(gGP>xvP0!ss-L z{N2Ko-aSXv6AbBbvkKax^OsqDN?90bbOrs|a+!VN)q+Oul`s`GDJZ@?5`lbo`i8TJ zo_;B%Di>GLMeKh}x}6kDwyx`OMY2hWsT};X%6S~THEtB$nyGWR6P2T#teJ%eFq@5l4@Xg1lQ20;}=kVYbJUg^N9cYdm?35uBYDsx2Ajx zVHyK(Gt>paVmGcsQoUE${vdan!*)}dxeu5k2Uqe>^D`T5 zwUCBhKZ!Od?q!36_fUmL!AQIL5iv1uVx4ZsvCEr|qrxvAI6HGXy0_pW>M=XPTH6$| zDFqDe_ISu%oOgph($AzX^zzZ(s}fXoR}Z_SG71&#?`A`EzSG3|ldN$6W43F%AF9fV zL@Q4C^3#9*K`Yu02=6}GPa9m!nd=V+q3`KoCd_uG$Z~icYMlItSwct-`5W#k(!}%} z;rHcd(2!^!DrK$-rOJ<>MhDF7PPxI;Pg~3w+iJ5VpIKU#l8%0ZR@xs^!)6O7)tUALE3~XwyEFcGnxN+`n1)B&!VNZQsWBEtC~qRdKZ&SQ^Rp z-MEADo-ah_-+g8885nc>%eP2z(Frv8^)J6-{}Q&ULLF`7KSHPPtw8Ox8%>j+W%toy zn0YYk1pOQTk?ndZDQZ!AfJOuKklj)>mUJy<&R*zZmiK$`M-sg0-9O><@@B}&#H~SZ z+iOr^NHu$6kuKv`5`)TqdNabAbJ+kc-l+Y2giTZU%~U=6#T;>|5EkvPpfAR2GSAy( zL{Vick?ns{th1#hJ!nyga``{et+oDW<*`;|eMO8OHatNcI~Fi^kQ7=MHj(w->rVfr zn4`PHrF7>|H@#7MLfE>nlTrBtX|33AR{6qiR@VLmbGqp|6?dqc=el+qWBghd{W_?^ z<}c_GdME$nzuLNjfAg0Lt2=HnQ*RxF%|MD+{J!PNW*5Q7yOBt=J+SC)gqAM?)r8_r^3h|=3d_T&#=;XNqc_Z6 zrzMmwxyL++s-=%jKcniM_gT}Q8<~vKZFI87arE1Grd=(+kU6N}$9%9U5pI^rW}5~T z*o!JHZA?ugA7X!^HUHDtcSfIu zXT%bP<C~Ur&~7qw<40WWp85J)C$H$Oqm^&Xs2q<^(aZT z9W9`+*7 z9`&i=(W^%~(bcmjX{uijs#P>ZX@44!jjt@S>)(Vvj|`&q8xhU(R^dx^70}6TVkjfT zfhzHn(C>n6RQIBQ_3~SQf*cI!vpfOYQ@e~V|NWkEnA8N#FP74Z`hUofcNAIK`omw3 zF||n>rtObU(?0wZnKo`?)*bnerUhr9ER7~+?)emCy2X(0?JnYLDSkx9#SGa{$tskQ zYD2A~M}?bJG}+Cu2DE?vHh$68RQC6~Q|RW+BD8VlT+X6zW-qpFp+`Q|l11P1X`*r& z^HSBD&e+Pmw}@6I;im;HxN?~Kxog`MK28*jS3b%fP)+Bbgq_GKA(Q%z?L;by$56{l zKI3)aHagPT!F%e+~{GPxesGy~-@6&H)Cr`QwLG<-69>(BuDBN@r}O;Dxn$eHGA zXVX^82PnX66~$F=*o}2X%#z((>4lK$z-s4S3Z(=_s#mDGWt7tS~fds1K-j#ReDH{IlFzd_Zm@it1)2XIn zc8X&i?AY`I>b^jc{>;2d@2(xS5qrmJvN&r@6clfMomJ!m&T%=}*JDiy}`jIwK7J7_IPg4~hQxL_0-JB7Zrn}M|AbCGS2IU9fQAdRw2q{2zfOpx6I zc4VZ8UPPp~pPA4nbHLZ%GC--a^XZuWg==lnIVhcmB-4Pen zANkGP_;{E-w#for&-=>uPhP-Y=)8&gHa4^8Z6}JpE}Di0p6_G33b(TZz2&sBCxJU# zC?`B7n+8lNWvk7qga?Xxg)55^sqW5u{J@9mqIa8BXh^a=f0sOG_$i1cqhC_w)wnPz~(iewjk36A1hi0P(-<0gE z7hR=S(-za|R>$brCkf_-^I4{=GlNbV^WsNn%G)XD6wnUG3^ol+p??z7_$zBdg}yx@ zG`cQ@of@(XU9wANb@lt$kPr#9?sy1 zjTaeNAgZJBnmy)f%|_UEBeM)G(TuTu=-@Fe=EpBCkGwk-T~^$W7KSG?Ev@~`rm^ve7JzVvtL8?7S=F#bH<@bu}ef-v=7jCf34`nNKO9j$rV&uIf|-O zHY1t9ZKD4U_|U1h#Muq?57~o$9{hqQJ}5%D7(I*6p{CAn(fC7enP|;SdUQAqWeKvG;IUCUCo~RqYmBGgZtSAfv!0@~ap@>> zMF%@hrU`BQSdA7sX)y2d^=Mw;K4hrc$i2H4tk=$)$aCZ}Yc+OA*ln4Ps+3N>dV#9zKSmiFp3@tvk26J+Qqi#)h(9}}M@U8|p{MXLd)iFePBJ+Y zRk(H0f`@fjRZl@QBLuL6`w&)QmyDW=Jd$rlP$NbNTql`Dn%6zlbQV=eYPjX5KejHt22@E?b}`x@OoyN$xx$*R>b*-BqEV z>|gOaA3vbK9&6B8$9m!3v&Rwd*EKe8&sU~Y@Q3-nzKGd1x`&DRBxFoRiqM4i9DV|G zf+iTwqk&%}MQ6AOv2$Yx4KLu+XI}Tw&gw~suNRDLny<50lGgFH=Cz>#oeHR#D#w0s z52CBRBSrb!B=GjYr;O~rv+VJ$o%EkY8g;XW$bUGCI%nljt#Zy*jlV;$1n%Z1x1Zpj z1r{hgYNB0qmOp({vw(_553r?r(@~wWD>G$GPK2Y}kf-Y~jlY=BxPLz;4AJ*R2^xcR zy{iqqdwl`xBUgb;pA4~lWnK31z;V>Mxq#-sl@YnS88git?kIVC13EQ3k!I`Yi%#~g zW9s%PqaI6jQQ0LaB>!JBwV3sgN%e81C!fux%MVpDEpNrBwLvK2ACN(o0w1=7GW7G(8Ax?) z7b=NqrqUlapjk)u(Fo46?MpABGa4782Ro~{HT*kuUEzTazI3H)E+(O%G{STa$+PLV zwlmV@1E^J7R@Bm^Kn+zMFox?q(NVn>{F2?dh&}t9u@c3weO3EtKX=akIf3%WjLy?% z|0vR4Zeln7Mi>%1*@p}-xG_f$eW4SS7oY?D)ii;#?m8C@GF@Zq*l_b8s&jZRiux15 zP9Bm(-$IaGM9OtqGED{LZn($(aL!>GxqZ~aX9ZLkWI>IivxM3~+*xn76K7(bL4WRr zu^w;vqH$NQ3U3L_QLVxsh8Wi}r`un%=?NyZcaSpet(okEcdv!1uC7#d=|$R8P)iRq z>M>eJ+k_);?ClCp=AciW#z?RBFp^X+p+Bb{<7?(E=DWWHZO zl`pFL!lt#w)0V#`X#Qh=X5^fP=$Aw`lK!VG;x4+P81^dTb>=1#+q4O(FEtUZZfRiY z&(pM^N17(`vl!|SOZzt;W9r{cWObW7nQ^a$>< z^zV-r`WJK^9o%}1eK6}B<1QsGa!FpzlyUhW;|M>dTFsCy*&)WvQXEGA)omebxxQ&y z4+Bv|Q9jZ!JchJQMwo|34xqKy4>0?Oue0pgVOm`HlBvxLplW4J?4-d6>M>-EM2l;Y zmXkZ1HOE&t?&&zRg{O^({gSdluexlh!*Dw4-t&Vy zVVZ_!|NY42ZvfhIH3uziOhhX_ma*%{x{+w$CV$uK7IeTwpBA52WTxt>q1yE>qA3}# znQP((NaA!Ny1Thr_{1TPmg?ryhs!fKLwXuI(jmcwM%Odr?%rbxkN2SZv^R|N)%nya zK8`iMY=sgw@1d4Y-qH!`!3d>lG7I-^V%k;jpo%PUT5FI?XMT79>m}US9FrP!*#8%Q zhRzB)J`vM9KPpgpX(RKWRWZ`U$QFCbQ~E|Z3z zl0~}mk*wO2udp&~DrztvV6rUysm!PW8h$g5S*^w}FZwJecPhl4#pzCz`Gc^&_H|mbU@d=jq5%pPhNGd~8I-K)rS{IdQS1XHda4w&%M2FLJXp@2S!|EW zi~{K1$2=;ZqCNcHwj znC-dcP9vXqxtaRdj)UxRYQ(rn(e~QY9`LYRUCvsaf0qysbqD4zy zv6mMdpi2HF)H(J)cJRnaUtj=kqwTg4mk|Wjh49G<`3Yw_KJ%_RC z$znu@RZ#p&Nzto(1%`+&2wNN1QEUH1R&sO`J)k~Ogyh|5TRIc^Etd5(C}n&mr|?Gl2Y`M-0>11ii;ea*+SUeM z#N({SLxreQIKzD$+5Yhrt|%|VtJ|E3ZAcslQ+p5O{H&S<2J*O4_zo;Sa|-{S^oTba zCkCIb+C`+t&mvE~m%^Jg5j#ayalON*z&|;%#3SdYKulIj_l>t6i?8E4f0?e&a1iR%|!m~51@iOb%JZ|qyel+Rh9=U$Jb%s9CS&Q&MIaiw# z`^098n*fkfSLmtX1Kx+$;|3KYXc;RDpHH>L13Ia=_MIZkP<6wfqiO}u{SltE2J+cPIa&5qRM+Wa6o=CcH zR$`}q1-O>06MtIw7U%2v!20Ga&~{IVZyj#IhwveRU(5iA|8*CA$#&1m-M&`Je3s5BuhU z->T7gcy$c;1Wtgz=QcvC-L5d-b0v99XF%D}HKdg1MuNHemCOSpc<0mWU~7{o6^m-ZO5r1LS2+iq6?TIJ2@M!(xd<0jHsMzviD2gst`B(3 zeL;mw71;dH8{0U(0NRFIVdJm@{QY(iXHhjc_ZHyUc1*)VGsbwc#wo+PiCZr1juF^? zv3G{?8JoD>eK|f{Gt%VImVuw$<&$W|0ifY{1^5*o0?}c*@P>~u-1KNasNwGXAN*0t zdx}eO^_zuY$xb=QUY-gKxr5Jtqy6~Vq;r5>G!A^IG9`Z^ISWo<6}(uzPhhOQn%sbk z$>Ohp@b}UQP+X=D*L#O@&YKN3yCOq!J2@{+${D)#ts;LcuizDfX0XM78T^{lh!4yd z0@mS&fvwOJ`VM-L@=M3?49*kAPl<&P2*K7S52ENR}-6 z0J^H8v2MjzYaPA^IhZ@2%zDS-={GFM!q#yhX;VDUw?iN9oX$Cz41iri04;Lt!NYnR zsP{sg8=vw->c~E@tGiC1v+_ps@+T2+FGj?~VHE$ny@@z}>;wayk+}0x0O`ABPP}Jd z;hoEpfn(ff%y0=Oeh(HB@2r(@p$5ejVv^AFg*s_7>jifUuK^iFYck~Z8EY?0hF_j> zH5K3O;nw4>kj>U7k#A*S_ycwLdA%tZK5jwMqR)a5gU7)8juUir8pKUXX9V``RwPX7 z7EV`L3v(+nu%g#|`1`{S*s@oS*!*yZtmiB&$=#_xK4J#;-Z#c)Jema6tJ1KN{3WbC zQIaU>y#O~ij$rYURS;>;f~_Z8IQvkJ)NnqGZ$%;=pRfk*>8k=tmjGE3X#$mf8+k>0 zfABIF7!!*}`nV#q7&p8)C>VF?tw3|O9N1e5ZT09^LFTWWz<+iTuy*D*7hBB{q#CGz zBn3XWlxQiadwK=5`zesU9}}>j^Kv|EbQ@lHAQfwLrQxQF#z3lf27DT}lKhF7OeWeH zUApV7L7rO&LP7Xa;BYY?|K$2x25LGBe&?|`)185jbrs-TUCxsGung?GwH;VJ|BR=- zJ_+ufTo09{LP>#(9hunh0^Im)N*=%Y10GMA2xH~?!RA!XYIPaKH`T0(|2;Fjy0s23 ztdS;%O$EGdi$8;BA*Te4FP%#=Qt#U8-JS`hCuop|hI6p4-Er_`yh!lh z$2P2aeFjlD91W^#>je%%2~uVjL}HI7kRt6MEEA$Z+-~WSf4x)59z$nvYULLkeclqjCT=y9;l_&tlpZu9 znum_yBL&hVdwUO_ll22Hems!~BU7+;T_7e$0Pr;$#tod=v)pM5fzNwE+xB;$|I=K! z%;FCi^|B)xjsn4E-a;IhP-Z)EWiROc77kk0wSncmvtjVc+u()rOenrRfXx2Bl6?QP z1_UXok_ncNz(R|3&|oVK&YIi89eaB&C7FZ(FFPL5%8(X)7q6fUeW4}OI`XP`~ z_f8;LYDq3QnPBl1YJyohv4VB?`|!7hB&eh03unBRCEM<2;L9)&9!#DE?icdOCv??T zsxBGc@cIcBI=bPzhpmWk^*9*Q-hw0NF9k8%Qel1TGSWO+%-h|zl=LrKPh8tygSjUk zV+o}eUS63ETstugm-}u8lVm^eeAYYy8{8DhwjU*Ubk8)>aAp%>OwHl;6_xm0a{$CM zt>CHGM{sMq3>@aH%I~LQ$PqagNKaqD!wvJ{q|CE|*uRcs#e!bx} ztA4~5m5%UG2?GuFf8qn<5_#HJ)nUDsCAc})i??-&ITq{K0}LO$0kKL|_=%1T+4U+2 zhn=z|YrFYS`?DJi1k+AYC#kZJld}7=W5cSyyrtX)9 z)5WX7y}JNbEmnf()n$q13>H|c@}B@UhP(ACn1T={Yv1Y8`&lG|2;!RwAtGY*iivrNgYPio*s<9ag5_$m%xm_o8; z7eQ1k2mhjBoSkNc;jiO(Yp^32_hlUsW=$tXQ4kWB9VG6HJ6P>=0Z8Rw;M2y{B|ZKP z>L&Q}#@5Uu&)R0dlooZeJLS7A34M)SK0ra!AvI!DEsLkVdB_ugeF9HcjUc$dhsX?@ z0QD+&VwJxeDr+ReT``Nu*QsZ19geOc$9Wq_yK*Spzji8~l^sO_7q1{c8slM&fi~QK z*M_jD-1hSPZ@_ncCEmTqlt>$m@d}a|f^rs;9fdBS{!jqCIyQ}z^z~uqXlbIDH4Q$T zHE6rZx*6l-0J!GA#qf9gZ=fgl2K>Gd4tMRJ4P}^JMAoa+=0l7Je5J4hC@IUpuBQP| z)4mG)@S01`2QG&V29+RQ-JQthZz40iih#qC1hD;3ERdeNh;&Q?xTl~@u)Z&ed!8OV zYcmn1{$~NVjDBp+)fp8Gv>M^_%8o?XEJ5s4bclbF7@0G_9H?#L!$4zwV*J#OYzUo) z8%(?jZA&5<^;7Y0?UiIWdJak9F9TMSr;)@Kcj73ujM!aE=lLE{B-(9Wq+x0@nRp&H^2E%)Y!{KztO(f~2 zBs`ho3};;lg4rjWp`+qlSe1Ace0}}}JRWm}Q>+Jo$K|`gxpY0;8|F_Q&D9{Y#w-?%z7!rBc!@u&a{hDnL@XT5!iY1FRL>s5yVF&$>Fg6=e$fe_o-mDc9v8<_dTBgq zuD&ZqHUqltt^%*#6oBO)=0T~FZ1_X29Nc&mfhVcO!tz3Oa{Xx-iT!E<9k)nf#gav2 zsk|G}W6WXb_bxo>$A_ZfDlGkL9R#7)u$%EGr*xlUrO8sfjmNDCuXGK}e7yp~NIhb5 zWEEW2cNoaYIPzZUzsH%2j`AvRm^P2dd*O}Y`@j{?>9DhCCOKieh~N}uyh{2BE-#CL zUw`)EpdWrXd}a-3vQHud-Z!uf;Oa@2=?TOOA2$c>Gz7~xle<4Zd%-}IBWs4Y<-`y~}?M(2a5`>Eima|CJor%s#}?t$lh5!8cMeIi*`n*ce2qwHaSFDH z>IMz2W4uf2CCH5sS+dILDRw#i5NC$f3wrqaAmQH=Y;3L!`nZ_LU}-(jT2v2?ntJ2D zsaHYLNIr<~@g&`sCWEubFM+5}g?QJQMQ{`H0bTjAM5o0!lxd&;)vMb91K zmem_z(Ag|}xX>6EUYCTjK6d151O>d%Z(xVvf4p;MiMS*73-4WTC7$Yc2dj-rA}`)cmyIHLcn3DCZS^O=!U3=iPawD|vYB%`#iX!`T9N zFogFC*5iA7_`EaQWXTnoi+B_q!DROvfDxrelyU)ryh8~^i3hW zN`HBBJbD~x6WPKmzqR4ce}mY>o`LIL9R!t9uXtrGA-LF46J$rokc%@CaJ!NhPxXx> znR`x={LHl{s~@%EM^6Cx6qieQ6-lJ=izS@gzm!~fHkG{k7f5cms6mVNWZa(5C+4aF zB*|M5dR{w*WAzuosu^-H>~IQ}IeHeDo|wz|Gdx_r&>FtFt`7~Jjd+{$e1S>mE*x<} zjigTO~B^M=OadR=wh%zK+9!fyRq{HA(R|m*EV?gk?!vaOY zATMTGE7s;+1O0i5z^+z z%Zt=$-^C4Y=E8@!CqbKq*8~e(Z-QNqx067zC1j(#4|ylY*!NWN&kXR5@hAXpl@~? z5HoWogZn~=l`#w2y>v*oqBrsXV}(23+ky7CHUz8;fG|-GS~!;B`BK|qif$a#ER2G; zF1ZlNXFGZSu=bNQeEpI%v`~{E z?lDu{CoRiPP7nclSnYqz$dq*B1 zenI4Xt0|nKy&Ydf6G_*t4#6a}5!-&s1TW%rNd~h3=$M@XUr#%djg1DN%ghk_WX>d# zE5zWr_bR;5x02-E?w!CXY#AxI9!wN}Tf^3Qk>KlNEfSaOg)429V8!ieWQFfy-=v*w-bFjRyAvw+%cIm#!>cJk!>(FL%8v#Ih@9ieT| zS8!I!oow2|&8dGnrTUI-2kn*;JT5^z=!RQaI^ss3!+r?{J!t6>WR1fE3NWjx8a&<32l zhHaN!>;t_jYFzF~fw)@L;eV^M1&o#m%$vvUk8mcQH*X`{_6EVp%O-<6))QdP9dm&c zXMZ19djqUGV*_*RrE#uRbaP(yzvhE(6Nr*(B0TP)4!hJ6fmu{2XKQwX{__ojcN>P9 zS2B)3Ldy+yn9qi%!@lBd%`<}ewKj0~&m`EhD}o#^8^p$+?cjVxbtpSvNX*4f;gB`5 zWRc4qfojyb=F)lNNuM4=J~rAC{ShM=d&ryoIZLo+=oEqzazLVM^QCRAOUa0B3{n2? zEV#k-H#oC*DU=i_L9?v~1^gU)7`;@Pchl`UKDOQxE?8{_6;CH&>nWSS{3Uf5M+{>N z0iPVPG4?GsVoEIk@4IZTAg3kqyn&`$R$J z^wnV594Vr-zxC36E~hun(;6oE@?daT3$H6|2&db&1F2_Ucyq4Lgy*iVfnQU8;^KMA za8_EqpjJN$D9m0APX(9bw)AVbKacB=1Nr2yt0y^ABu!w>Ea-h*R={;)BXh@DkPmi? ziMQ%!t|fdgXgFX2@iAA0G*A+!EpD89*LVX<|7| z42~=@CvaIP*)XgGO(ay|$7|65Xmb9S=S0Hitd3#v3^MFoKvp;=!Yy2WT{5{4gw9pP z9;K>y)9(BD`=b=1RW+Qx)vEeb{E?*Tnyv91m-S% z4puDHgHuNzg4ubW@$;Gx;CnC=XqtraG6I8${X~^!R+77eSm4+EHvVMugtIbak}}1$ zE8L0QBqw;uXcBy$GJqe=lOv+EnI!E_I<6>N2Rm0!2D8G(VT0Pauvc*g%#OMaW`*YA zWB2;OzS@Gs|EWejket{ z3>Yfr1=P9Wc3FQM>@Eq1QR`>H!RejvI;H zr)X#xw2|EYpvX~?oWIhvhkUtRh}FCXZ3gD;1aigMWXyX2k7hX$iEdlc@_R0H+3*$5 zp1G2oAK;PwJBsjpyVWH6U^jNqR3LvZX~Ntd&dj`j4hVa!U`g>#;<85szNla09r^wQ zYx}$f0cZxi`&*8@ezk`z4gCUcYpuakT5WhK5<;-n9>Su6-+1L#OLCoCizVY7xIA+a zh!R4gGpbjsH{141w3V_k&S0ME0 zd$0_|2}Ummlhm&!B>3DJ-Ue=cYiMvKsy}zbJ*k19VES3GJYNF3j>O?1zbdSJXfLj_ zbmD!~od*MSmt#B0+k(5{b)dsOgXkM9A$L7%z^Q=|@H1;B(Kuj5^k%seJ-rLqkIQux zS<91`4imw}Qy&B$LqoxHi(FhbUKcv4hQW{_E#g^kPXe9+vh1QRi1yFH{NxoSA^=4v`kW3UR%DBCnC$aQAf^TKM;caVBgJjZu-t>+n0>17iXm2Ro zn<)jA_WuWb9lc<$w-(uI+#zU}-$E+0v$57=Pgwji2tGOg7_6KB8fdG0$MUsuxGrfU zyrQ86_c(jnE?i*>FLSAl-w!03vW`lV#yv*FK_!7qYDy#CR1WS?+(oRc0Mws<7<5){ zCwuPQ$1lvyp(w-=UP?71mDBV{`cO1BL-R@0ZGT%aTVvv9+ki{8E${(%bJ)9gIq&;I zF|x~f4_G+!5AV|A+n`)?3}@|hAg19;WKoe7X_CAS?oGdi3x9gTopBzpvcbvLUH%l# zw4D#x0b6*j`8hT(Rs;w3YJm3=ZUVtIci8B>09-t0PnKV5X`cM*Adc#o1(%%3!=o?m z^WOSo!Ko|xFyx*psWBMB76z^O_Bc86aIR_d1Kk04vhZ=ApIi~GRy4?oBhps3XoGg#esIKHG7c_w#D(mor znOk`pR}c(6*bG{7Jh3S64DaI5L9ltU33=C}_cWU6BPmK7Bf=XxAiH7asw~w`cOAHze@90+dPR#r?o5 zL4rgF_X=D_@8hpFM{K2IRAIj?ZC;aL4L3&5h12ESNf~R(%hTaN1W$jG<`)6ob<=I* zoOVHy76UDRRp4(k+ktz#1SnfPAM(Y|;`?(s`$?r76mR!{bP?CN2!@8pOm%6)4zFCI4E3VdZ524^}APk_$0V>qjK{?D7vULkEDn#}smv1jBFj)xa*& zo+R(n<;iG&1|BYk@Rpw+kvi}fuXvJfGcjf&xzQMjv)*Te7n+lJU!rWGTdz9wN9`Dx z=7CMYtDw^dTaZ)J2b8@v;J+(@a7IHDPSYCZbsF^xN*q+63^6D3^E^oMp z^n@O#oZ;P~eQFm$j`&u1NG%-{#RX!! z;Pr4-rW@Q8`yH?RA46y2S3?`cacPxOsR$*#rKCuu?an;sinOS#q0JVOEs>qls+CHM zLM2HlWr>!1?{h9iD3XLk3#CG~q%7f`|6o4%&dhV>ch32KM;&2eC(aW-m>tiTGyX+hf>%}( zneX0fm^UYVIUcwruU=hWxG}<3D14{M?0oiuS5WW3%z5F%1ojhVe~2q{T}O{`l$2+J z@9PQbuSW>^QOlXf5em$ZXacj^^#tR&Q%~5u`K!Qtwkq>%^)ta0id8%%=dD70 z10UgWNoVG|WUb&hW6U(RDlkm78xwSP1?M#w!xQ~?T;TdciitlL%9BX2X4oB*g`h&1 zLp7evAGd75#R5sjP}@c5m$F58*m#f7sc^ZFRh47Te^?_t{wsk=_;i$+Goejz^}kBq z0eN4c^??9k**>n0Ua>)-w!?#=vD2CLDxSg$z0)=)KY22yTtDm0@m0cc&;JR+pI8Wg znpIaW{i-DB3e&Ib>4+0No@^|fBbUW1|69%LI+)GdmR`d9_iqj}b@eOn2~*~(O(r*i0n;NYq=Jf8DP-a3^r%-UBH%%F-R@7Wev z;lTtKMpi?c3E42fIY(`U=e(O&6e>fqYSCOD8 zEX_^gu_+RazFTf3KYlUOP_>W|W!>VLTr&`UpWG*SH|DR+*orPe7RT!g*zu%t<0BiP z*|$o80h;l4mEYxky&x?V_ht*uh)o6EhdBSCy*$(T(3W|+$B4UbDPebXc%`fHKimXEN_O3Ikr~3wK`iWFkaT!diDZ;RTCz!r!=#`JQLND1R;BjrVk81fO^u zEONbYBaUDUK1%W$U8V^y9H|yORs1994cf{3=dI41_}eImebOnI(Q#bBa9L#5Zxdnj z?Fqc_Bl84JmJT)weH`;Ix{tS4ET|l7m@bGqYsfqDafXn;6ojWZS4?K`)JiI0#$-*E zW7w!}LD5-t-itnMUc0djlN&83{1^2@kSZU{TXOPxWv+b*FJ?BE31)u~EYgS&+Z+%__D3N?g*d*j*Jue7D3TPWK3>ffwiq$lbH@w% zXU}Go+&}Yn^%?UHHjZaJZr$O%`T|Uej~;WPX}<7=6U$|meBsV36NCrES1W@X#|Vpm zSTfgE@8j}1A3@Xa6yEt9Ri4CoJLcBxHNvjMy~5zFE=(6#F(Fd_;nm8q!n=3X zn5sjggy%9BFmcA4nK28Tg|n3=GTtAS3sp`S3SW$uWxBh$S%ojR1frOUOo@`9l07Ib zbiX!7n6!SA@XP|vvw5;!AV19Y_BLoSrUMGXHLN}pmQqF1o$a&d1$d zwRB+KGDsmu$nll|Oh~P#1Fr3uu*#3>mzQJB4@YX_7Kzu2vx$=Zbu4e`A$wLtVdT$c z%pV;)ER4P(G8cN{jU)-IUNI5I?3cpC7wIH7DUqCB5&=_#%Q189T(SDgDR`6P4m91j zqQ1B4Fs->4h8J6sW}AbMs;CBGrVa)AhIZaXl#Sk0x3>bhtH=|kme{*4hdm`?O zWd$9t=fh(yX|eQR1)Nl=?E_+;{I)< zyTK5`l~ zC;bv_CI-Xj+()!cCIi*9{Hf*ASkm+37X95=1$BcK)XmwO9?F~vt*_tLn<+G?cNgj_qO@ZdLL~va{SJY*55+jAvY5tlhR8PNvMz`w0D&GO_q_|_e zAD?c1Eib;mQ5FV1nc`LDop7b|mT2LS0)OReQ(QQ21h8xT!Dd+)v3=7?>^v0s2fiPL zs?j<)`-Usr{6yjE<{Gg1JC-_4pM%?ZtuS>(0STMM;@PG^`n9bR`dXL5-=32+Mz4_! z?n31BzSMkYp%6P>pWCpj7&xQtgp4d*1;oIiA( ztesm9HqUst^zI2(J#IUZ86gtg9l01Aen`WjxESy}Zvoy;xe#KcfEJ!%uq?cREUHq& zt|LXL5HTAg={*{uIWH9FaNo1k<*vVfcH0yu|HnFbdO9-CqiA(yVAn zmL}M>4N}1^G2S|N1^YCXL+^`9w6?5-r45$gIC7YIsXZDu3?3w=i?m>szAZ#>y94)* z3(;HoIV3jd@n=&fSYX~qbPdxXTwsKeMw9TVbv`N{@PR!Yo@`q@w;xthN{T|Nz_4i! zhWCu*w{C7Ej~1OFAGw|GpAHl0#j-4vHxj@jV`sReeu<7O8v)FB4Q^k%0M4Hu3o9$m z!s1enVHRq^eC_;>7FnHC9M3_Gc z7adcE@4+^7iS1{YxXnX&Mzjk`S095JNdo$Va{N2jC^~n5lR2J`2OaGsIDdHqk+I66 z-kGmx{EiuL^XVQeN>L*-r2mpU{b7+nLLUoWBFa0=y z#g7q>X^n=Cxoz0BGyzCvAbsnx2-J5HXjscb+q5=1_Mw3E-j3l3-G0%XZMNu7ZVPse zD}sLyEXDOQ^N7{XP<&-M6+9j%h%^TOl6?!q$=0`#w7c>N-jkk*r~a&jd9NyAbyfmS z+$)ApC$rG0#Z;&=n~C&5B>blO;!h2S;OU@0ermB5HwK+W%Y{1NGwuP2E_h1o zzox*47o+ex>0$M%9$`2^s@1E#V*g3Ll$`h)r+d-SARzRTiJvijC7~dXILe=3bFukJ@s^Vv$TR{kY)vgY^ zyidbbHjOmAxsR9UPN4>`@1R5eXOfvuiK|qVs9|_J9NoEyi9BfIjRJ28oEL1kF%vcyf$ z5E4v&G>xuM<~S}vIk@w~Ybq>NfjxSs@#)2bWdBwHJnoed_h|kHukQQ@Y)cLfIk{6e zjs>{;iUD8Z_8Ic9 zYZKPOFYPDX{%8-aZ;nFUrf{576hpJ4W}@N6bV$#4qIPdj!^VPX=;`_%vLY6efxiPp zuXQ&5sa}BNmko&&lE=Zjo@+Gg+5|KRnuPBi4WMUpIP1OpGkZA56+QSNH0X#cUE)%J zTV_w-?>g}a+P3bdGB;1*`6Vl8v?v4K&CN&UF<&4?#~6QlM!@g7Q?N3-3g+$4hB4hT zI5@NgW$*gqQgvl|t;7Pd2W{v=e^nSV`$coAX5*UftMrdk9UD~H4Uh6#@np?h+~BpE zmKe~iJk(ym@#n1?5pTXbukXq4v=AK2zG{|3c+(B8rkN}9+n`0OcMp^J-fKknu@^Qji^Mnkr{OLW4QNXEh}vUYaJOzb zmOb!>G}j?|tIrZA_l+kr>aNlmFJx)C|75!6!60*WeHQMIy9f488!7tV!Q|Bu^xx5) zP&s&$-D=lFZ|^Py#v~3$-fRPLu`1cV[-Is*CyzMwzCL^P)<5^qM92@N~A-6w9}(M9(P?ep%V z*=x$-s>>vyrJ)j$RZ>bcEpU-l&oi~i=NTQ`OS;WmHlYCB`g_}Y-2BB0RY2SN-KD|=`bVC&Axr`BC-HaIY zB#D`q5`z_*^U-QlJ6U91C<<|(NBqMpAa#rui5*-+H+{Vb`@Vf4gM%mFoMAfEG)DH} z4hjBPpGB~4yE<{(e~5J-`vZo{evyT|00^GxBFdk<49wde!$j-lF!o&@Ha$HEQp^~f z)E!NyFjr};+Imd=mP*_o9>W0ziVjCSkvY$U?;YA~vhre-tD6F@fg0#@iceg6i}23l zxg^nEoBvi`hdHpPnDT}~aVB38ZfXL%t3Xvoh8h(2k+asBR6PAL?o}Hljz2jCWbI;TSKMFGg7`=> zxa=7_aNdME|Lla|11u&+|D*;Z1=0M`5IDmDZY%#1Oa7Gsetwtg9!6{K5!zroSP- zBhS)0mjT@9@sz%1-w|t>VCX!VL{^w(qsOdyc;CenUl#kq2EFD=P4%;65EE$U%f ze#HpW&bZ;uyB&lP_7eo8oH z_g}!Bpo^zA?4U~|z2HCNOj2~BfDYCj#f<)b@_h4YbgDfL3Hf2*miiXtuS()O{h4rf zpnxV!j)wn|V`$9dk8E_E4GG}V_u3`%sg6N9t}k0I(ml~Ywiu1USNBKq6Gx80eSZIl z+u>wPs=0?Yc8Z`k@Q++oion;WKjQfzN72OO=k&=*eLk}*obv7%;U@h$k+qd28RfMO zL*(0OpcfD0jZ2U*j>0GRR>7)1b2?NI1MgR;?pX!)$BNC3F;SQg0`X z1Li0r7KsjQUW2V(7olRg3V-xu6_B592Hr)+An41%*<;mkd2kk9TG~mawA&y#tx;He zZW-15c%3v~7>jF5ZE(`{5L(Ky<0|idWRC5wBq?K(NoAE2{+L!nx2&zfCH<`!7(PTV z_T=CaH6_~0asCyLMqs>mDap5%6IFU5ZT5bRkxPzJ7rS`6-gFaCx&Op76b510*#hWN zuK~$-nGl%B;FpJGI6Hqdmi##(xV%@A|2eFV@CWz7`h88fe#Jqwolq#sh||E(q)?E2 ztiU_HYbNHt*Tn}7k703J6>(OnMTPlQ^wVxFVUTGJDc`;g4hJ=hs-KC#KKmRL=X+6) zH?!fBn;dL2TMvU%9#fNBZ|U82_gJd0iTXC<@mX6YmZfhL`Q#*FtydBBexdA`=Fe2( ziyclsaSS#@zhK|*T?_nM3UG1rE8?&`8LtKg<93e<(iu4whJ*jZk1p%r^*DKv#W=th z@izQI$xyUZ97W5%4r9GS9DXeb$CWv{}+I8grNCX8#AnYH*NLjv9gph1n90x(lC2;O{B{qI5p}xP5L)ZgCRLW9GU`hghGx&;^o3c^W z^)yyInhduU-;j``9J2S9gt+JaXgXKy2cOR6;@FsJWXk3yNSnG2Fgt@SD03uPPH*wwn)ig)WJ~s7I(Su?!{bvYK=aWE$hA4mx^Io5^E6#i!r%ot zJgtJHZq);ir?oJg@P#_X#>4mW~AioEn`CY&yce`zQt zc?_?wQGi!d*OJkDHbUd4oAlDMiQ<^V1N7FR9vH59%@j;_gMlCu82%K2hPMR7+wM2U z8vA16Q6YVox*6$k0CgJoNmS9%fpf#+MQ+zV;mXWpC|j_apRnx;S(e}dbTN0oY!89! zRpYUCV+Kg>E@8j!sv+%fe8|)OUQ*+95clnq6gxiuNVKFU;Pi$m~WD3O|6N-#5)(@ z`*;&N;QN5SM^*Iw=*a$^9Ro{<23(ucEYhm^MeBN7h{J3Naoo!7qz8Av=(~!bTXPJ; z;@;4liGXPDJ0h2yAhhmphBPfNc$K!BC~(|6w--t{ zIBgsJRJDhPkw2+z@hol$Nfu^yC_%l~ed2yr8twYbz)QlM_8qT>`&Y)p)#PcQto)Xe z`IEq@O&8=ECek%g3CyKHRguM^oAi*II{#7de4?iE8vP7o==#t%!pgNlQ01Hne;q%9 zo@ovK)!P6;(M=c&Jz1pZTs!Bx7y!P(^tnaurHFXmD9Ly#hRPgujU zY!r6+AA{>djr86lS6Dx!#LC~P!#PJ@Q{U`JIO4Wmg^c+_lC=1qK?wUBwY4YoTw?Lbwdd_;rd7|7WKP zdWBhG!7pF9xMweQ{%%OzoU0L|a?otJkVeS%QbprG>;ySW%=6v`sj8Q7$+|d9w;4vK z9Yr8>Y!?0S@eECLN+K1jyRgOCn5^Z#$t)tZ_|}K_!Jj8-L`$fT`Ij@P#)O&B8M*>b zlQdX!@)WLW+kj?clDS-4fQvHSiSBnFkZLI>&gbW%rDh3Tn8vV6c35JvpovQTSTu!`yJv_~ApvDL1fUq@*@(pgFb+!*(cf21w+~nbjWGKuZw+1s_R8zYl4fq+B z0&C_TU@2_`o@14v*yJYkd$xQfD`f?tBx^4WbTG^X6k5eD3-n8a2kKc44iV?X5ErX}7>BO(Gf10EKFmpdMXr_lkR~HX{BcoJ^wxhWKL3*} z3M)Jh+t(e!YgtFZOu3L%3y{DMspYWNxCif9PJ->8cOd_O6{^oWhpC|%VEl!F6Yq^t zt#~HxIyM4}7P-TUye=4}FbSiCLOT0y5xhKZMn9;mCfOPK7!n&o+Lx$cdrt>8eVYxl z%05=gHZNf_-Wwv3KTq~ohS7qcY&LVEKPE`6#i(gx$zji#V0mT~{tWITH~(xSV$;#$ z^8HgNj=PL4n!d;n4CemJx#ZoR&$NS|2aUBcAhX*Y-8L-1`Zg7K_s1Uo9!UXJ?touq zE(1{;*F&Gt{mH+C;8D8pm-yZV;be{qX#bGhMMVO|-+QiU~*?hjY(Q#m!FJ z$qz1zT{6BGPPSdZ%los@|KSz%ce{i7%O^tCH65{aE{bmKjsThHM&LHt(f+?~7_{#jY>!+5 z$3-cS`0_2vN6P{Kq5^6AH4{um4ACf`WE9o!6b@zFPeude<^Z z>!N6CRp$@k%mk9A7fg%i?!vQeQ$TCJA1GM2qqX}7deYlQto&*uTxeI~FI3}t6E{B- ziM_Y+g$Y83(Gk36u^+BH)Pe@{eiZx;1J_&Sq}MP9TdLoY6aO5scWWPv+HOYd{suwk z40oJ*E}71Jti>O>^e^1ak6?MpQ>gQba(wqji?)@!;JBOLY3Q&qs%yoNqFw6nBXu1f z@aOSg<5;-0;}D#>WrvHtoACKh$B~;~W3Z&j1)>5u{*UP}PPWv*`2*Lm|Hou_HuN2? zp4PzBN@p;U7t)gp!)a(yGPO;T^?NBNe4^v%Zan)|6W=#X6YX5^ z5kI^T!bF?V82sH2wh9B-mY-qt(duvT$D;_BXPk!5o6?cj-b|bqzhZaZ)PhYfzM^Yc zE-@3OlZR2;v96|#F1nC_VSE1r&)Qhh?H1^!*P?& zDw@N56BZVa5bx6oqekCC1j`=z!?LX%RFk;^EZ1ZCNv6riGv)(t;b=n5FEQ!GrB|Y@N(`_VUOF11pKM!ZEyg`f)d?bl7 zTcFW8n!dhoNg{^J$m(HVMi=#|rMV$f|7ir-b@>wvADR!+Q)4jjb`xzBts=pbx03wv zt@v%q8`|-)5H3-|H7qU_22~l&Ob%tLa)HYwHBV$F*QuV}xkVP6LtY=^9YDIY`F_mZ3tvKd!2D zrvI(*L-q17D({sbvhVI8-Zg^WkSQBS)ZPQxDw2XZKDDY_Zw!>JhtP<`|pbJE|b2IT7sNO~#`wcQJNNvFJlWCv)$i1{_s4kI0*=?a@TcQB z8X=cb1Sp4pQj$SC9UbVc=u#0FCwuXVo+~AJBFd zMh{Gd(Q2o`Ph9{_WjrvLU?$!!Y9q5-=Y!?}4d_;h!q!-C_}Jxv6VvKR`R7Quo<9$6 zJy${5(k5#0ZybG*um_rge~D(iQ{)#$-J#a)E~pcBi&%#IfZf`X;@Ljqple$WYI%Ic z|1^j2-Iow}%S4Nk_P+)1$B_`?CBTJu4#M{rW^i?>x_HilMBLhY2Ig8W!|;!)>>Mu3 zdT0|1*;RA+cW<1f30^5+vOpTg&yj%eOjnHAyAcd01;O)mdQd*`z9=E}0zlqJQrQ0% zlVz@hV2&EUx;;s>tZFU&5R;D*f5$-a;*nxqzax0zahquLs>4_)aSV4?=AdrdWilLa z5}q^^(MJjA$f7j^)cb7)S-3wGw(a@OYW`~>*L`E@r1%^vIxGplKl@^vQwU0h)S+t! z(kdSlZYJalt=N8;1Q{R3ua8)cNwjBWD^$2C(J`L4aNZ>) z>^@~d2JOD$6O&p|-Tm{}Tssma)-J|Y@mO}A!4hzEb0wnri}-p~9$e-gf?dbPk$h1+ zS*Cc8_`i>ahF}NCiAp979d~KZNd~vP*@@+W-^mN><5+ud8s*K4qAym2;tXgIzH={v zcZpK)aFHddrjHhf>x}0=;d&kQ5#cnw>HvKa6@zoV)x_(qw$a8%i5O=yQglc6IW6bU zL6<%?%v6!!FXx!fW7oUm(|%9%$x|j-7rJ1#U?gAhr83w##X^4VA{cwGkJJb0l6&Xw z!}7crK>tiTi;86%~HaQoLiXpyB+Fx`QYQqD`bSpOA^eJ7SFwKis-D* zC46{Kf2kUR+~N&5)n`2{vv~(>!$CB-*hL~1?1g`_7vaH_@5J)-5vu+*8W(YXC>>@d zITNCa6({=02rGa`iQ&GjE4i@Qk;!LLp_+S28a$s;Axb`PP zdBg(L8s`b#Ej#dedo@}g8m5LeA@s8CZ}RSnD}Gr%8G0_a;Gw{57^avVeyF`9P~iFTr!ZBBZUdtTe_y8QOqY~%s1h5 zF6!DurZJ_*)UlQ}vnEuQzVkaxfA4R^tG$~q0{DhCsA;E+1bb({!AyTah|s&aZ!shQe6IfD67%lVpA4i|k= zgEvJ{u=}40crhnvTpnHMML zVE36Tkm#@wJJ&7~Z&UPvozMQVgFV~v*El|A6#u1ep*4_u%a(~fI1xKbE`Vrp3C4EZ zgD~b87#itiW z!U2Aw*$!0kbB6=hOMn2hUSPTr8y`!lfBTc2eADn^e# zouI^I!al=B9Cn|LzixBy`=9gRY^R9Cr9C9!lhR-bvxH{*L=lgXnrLYD1pPAJ0c^TY z&pQ=T6S+s!MVvF0nC;dRERN$=d>1F4x56CXS!-xnIvw4VxM z@8_dQ=d^U#78Vco$7|rSg&lcne-|dr%)x}Z3Buk*iy@bDaDHljLM0dO$M;uGiIUHs z#lIoyL@KG9oHD6_$ueFr^<4?=7EPybRhwytQ3>gKJrB?BI6*wOj}vcSlffoeS774% z7JTJ@ft~%P*6$^J%~tjv4-AIk=pfgX69r zq^2X(vFQ2*x<8*stS2AB#FMA#f_8s+7PJNK$!rsq+>wM;6)$M*o;y&uatXvH_=&2! z8tAMROFSf+!OF(F!Pv!od_1#LM{&g(2K`6Z3H4la+Y=xW~4Cc*LC%^y6qmj%{ z_`C8hgxB_=&P!!7+rGemx4|-1}~8ZYew9kjzu*dxG0esPc;|Pr=z&N@!mm!PUjxR1MFg zbXW;`>s#TL?1!Rr{ib;G?L#{6`xOwI*yHEEiTp#R63}D#fwVpO$NqOJ2wknVLe!CR zB*o`2db|bPzjquXA6lZnaT%74kEXjW@gXxl1l#cuZY({?zOZ+}FC+9I@<}*%#uZYF znGuK`*NAq2E;t0eA*)^&k*+9p{PAoXaa|Wj&NSR6e_xf6lM7c8q0~57YAl5J&J^Zz zIrteLJM2BM9JRjo(Pi_#Q`aAqjQCm${NL}1?%r0BR7EJ<-Xj6m#K1V?;EI)_s!!eM!#GC!kO9?hSGbBBRO3*#`B`&FWNCq0^ zxNCI4pZw{loudlLzx&BU)gth&3`5P{%v?qmxE;vP2jd6m~;IRQh*+7$> z_W1%aE6{I&ORsJ+b2sEr<*$oj{YcK;_W3QT5%^(a>V1qhieS~>97UOY5q)~B8J%M! z@u^-Dx2wXj+oe8`U4L$q<3B$`yq*K}XGWq_$G;DAM$)5mXmUqULS~z4Y9rFAlY=JW4@-q;R|o@0nI zPSZJmTt8Sgj~4wj*n$7nbN%586O^m!fVO&Payik3Mj9!y8((XSwa?xq6HEL+-K-cV zUVjO8JwGtL%vRW-b%F3zjrhM)DcA_O+-r*?JQi1D&K)<56(7M-8M2sVo&x<3b#aEw zEzaq+6k^?n>55OgL1W}XCWiZsE^;;Hhb1ad=CF<)|2zWIJNDuZ&q$m-B9L>dfK zkywtB5*uu^<@_?PXd{>e3eN!r+c*h{8tiGYte`P9J52$p%>rDk{2 z;O~_qT&}!>G&6B9^Y>nn{75rMe67VQ$4m$98b$t%L`Uf3JdQ6HCUU-eBYbPufr3jR z*eG`hLYoUfDK8tnk`}@4rQCZrz!3d-qp|YkSbnIQA8I}u&0o=~fgk@wz+dG%qOncM zSbQM@Jsv5d$$ul@xuzdHntfR`eY!7vaL^I?4XMzf@fvt;+(SrO`JM{bb8~@8^T-E- zba*u}2KOwV4%4@$VqUg9xxe+g;GT7#X!W)WO67BY$C+yd7LH7kx=zdHWHfIPyJAa7!uhQY`l`4Zp zey`{evzPP9tO5V`EUc0}Pc>J*V;4+)MJ^eP6c)rMQNxAb*nq0bu;Q;Q=j#$uqlA$%CQiLG8`#BcvB0RP+1;gMMn>0GDI*PfM%ZvrO5 zC50~Dg-NzJXswCvKHM1{9RqLq4~XYc5$o$3#@j9ZftsA_gfCJ5Ft^^F^P!pZAMI6x zy|Ff+FfsycH`lWTDYs$k4i-J0*VBF9Zm}BH=^%eAgRb9t438*HK*!@Z>Ee6$>C~qI zFx0^#q7yAJxOhFJELFseORI3l_6xLr0q01H4P-41=7|cE^WnmZ^H|{>iAh}*5UDr| zBhPYKg{}#eneYlO`R9`YT@SQ9^-VNU#ubDKjgVC-EtXY%PVTu#fY~<%_|$a zB0K=x*BfKfqs8#W{}a7`mBZm$hoaBU>CpVH0(63lVcugS@b@qiyT)n5i^(+*>+FQB zt~zjkY94W!e;Ru?-(b}uAF`h|7sH++A27ZUg>z>-6}35E$0wQ@q8j>_=sXj%@Atf+ zdqEH1pH83-zrT{9=d%btp8# z)eQW`<YP+xM=S~7{9KLIB$PXeA*LfUzH=Ay;n(m&=Nko+!u_JPeCQl z+husR1H{s)IN{cLtgW~YZ&judRze6yBmY2P7Kb>Xa|zEJr>7I%B3;)l?K(0%U< zNl-9=V+9pt^&AVR8HEt+_M9G{l1NHtYC&_8yqK=~L&g~Ak^_^rM` zRrO_5UN)TFH!T*d<rQ^XnkC3tsaG?8w9Byy=E0Eq zAvUfA??jfuclmix+i{Cc)Gfj7_2=-D>J(g6t|FF`S_R|v`CKlfL#LP~z$DLP1S-Pu z^V_&N!#mJDXAvzQ?~7GtT4)zMgPJeg%a>Wajt(wKgSnmDton#4m~&nkepfM2FyISe zUnb$Mr)9J@;XCldilF;`ES^^SM?#+l!l~ADlAO8~rj{OqaXZV<$Z;x+c^Zl8@h8a~ z{zdjep)t6;l%&@4j*8UINc#L(z8p*BLNMYH1KCNbq%kK1p5$m?RDvA7a()QAtkO|Q{77gfzXx>o9wQSH zHbCWpIQVyn^NeCBn*9F3tbCL}W^U3GCtQDrVArr$`geDl zQV@pOpLfH0<37=iArIgmj)gv#LUJ&890VH`k`KQ&;mYUdN%ZAqR9&C~<;!Q`i?nch z`rj2$jrdL^D;EoIu2aJ+>hW+Se>TY4-{xGjd}1zFh_aqq*j|2=cJ3~rVPU>7smct5 zuMgvr`U_YZHXjAv9%^7&E+ZO$)7 z*ZHU5)y1WlBr~6Nm1c3t-6oRb`yWZ16^u$>ltFHVmY7+uhJwwFP@}S!y60M z+K_&B{cS_s^J@<0^l%6Lg_6*dGg=%r^C9_R)=%$Rk3wN(Iu!lhL>~`~#%)&P(0JW# z{B!3C=-;@>VRC_V6`Nu8v_z_#AP<*xfOwg{B^Q$-Fh$`suG)K>_)a~BLc1&YyV(nz zs)tFl%ud?yyZ{}ypM}Obz4$*vXC4jZ7sheBvX|^j)+`Z;%zNK?FHwYF5-PNdv`}f0 zQXypDD@(E@TN1L(d*6AlHHC^6QqrEZ?+a4DIsTgCoH=Laym#*N-1~e#AB|5Y>>7y@ zI7?Ux);`h}_)w%MMY}$BAcw=|IA!26yTfqao2~rKBcCZy83$kezRETH?8P+8CgGP7 zCakgH16Wb-0{x@pC^0P(hrLLK+mDyQX9FczG;LO^w~`=hr!hiVNnBzj7Yad-IBWJ? zcq1OZkimZ#b{)PeJ`G(|_Tb-I_1!XR`(nXtIen1b;CP1pa9jZ@>=(lg9S88> zi9$HkKg1e8-UhW3?ob!jjKdbIW4It|f|@K-#EYIt!f6){%&SbqFYZFP;_X|u`eX~m zyJQbrAL(P|;P0??zi_MeF~OYqJyACH-e$o%7Pc!ie@_wnd|7bm5atML!a4;_?)Z;! zyqE9F_t_hVKRwOH_X<*}qE{VkN;`u8g6^`bC6>X{!(Fh(vyh#YJjZV1%;nI*=>TP$ zQVG}QeZVaRmT<^tfXjZkipR18phuT6wQD*ayPPzI{vY$%pgWj)Q*f5m)g3@rZrQim zzn{wn%{PQ6wgzJLk(Ib7<`|4u|H;-bvgZUpGQ}%pSxQM`kMQA9{|*rGL6=CYZ9^5w+1{@FC6y=?SbYd z_hH>nNBCes%1#Q-gw_|&!YgI|I6dA6-^*@-6Dva5+R9>FYNCjT^#8!>)dg@@z8pMV zsfqtMMd5ETPgviETX5u=SgYFC<@j+}7+a>O%65H^!PZMY!ecXCu#s##wlBCw1+Dyy zSLQXdnTHlr7i*tm8-asn*0v`&vcQauw|@+m<(`FCi6-cFVg|+~j+i@A1Z(05E_z!= z-R>)d7o2smkdGggxY`R2g%5DmCdT>ujBMcr#r^Qzy;1yYPzOd^`{GTJ{){5<;O6s- zp!S?fR@3+@yvOj^#{rM1%>38X+@@+=dYa`9+3=~1sprrySq#?fkb*U`#cX`t8UB3L zCAdZ~AhPcAZ8mvx1@QDJf-A|_OqAV?=UXjr|>K}3#_Fh z2a6<&D6!;gY~lG@I9zg_J98kCDkv+Ves(di|ML=TxvU3UZ;ypvbaPp^^a567!w8%h za}mt_EoD1p*5e=Q^C{y)h$}y2!S2XejCbBS%~+m_#q0MDQB5~l>el9V+-29mq)5bY zTTFFv_&RGWY*R(0&3Xs(j#grodh1rT*b7up_bq64(TiR9cMTtH%cVRdmr|D7d6aha z5cN-aJ2fxF2tRGGh7&0?Wsy`)sS9$4vCCI-pY3g7=NOh!U#K0_js8ol{5QGQJ6c^> zUPlK13+aGP%xgAPelb@*_yTN6uBQ?g>}CsN?@>v#747=+2dCB?D$quY&9X-dVQ!6uvVE4O`7b z12(ro@F``G|HqQ9mwk)YmR8eSZVS=Rwe`TEC)prg?jQYGvzhlHr2y>L$O192m3XB@ zfZrj0ba-1cr{&rVM#{Yq9Sn2?8)vAHVsk}6dHofjm7q%;8qgu$7B~ZH?jz!oD^2&g z$D#y#WyWZ2BjPD8rDf!X!S;$sAo0uriHCee^LKQhcUcAkAId3SQ-u>PhL@o5nnEBW z6A7*aEkYftH;~|%l>8T|K*#9uP$O_)%r^@$Kb;;UrKAwRZ=OL)9I8k8yRWpg$)6-b zV{?eQt%hLnib^nh*9*eGJeS^SJ`XtrE=KLkUeNN}WI@s+Ehg&CA>{R^1$1wkN4lTM zL{mz#pmgC`&?IyQJdyZKt7dXg{Ie6_v!W3guo?x!ongqn;39hLGmklXr4b#7{6|Ng zw%~9Vm4jsx)!>!wLprlr5tWI=pv|*N(FKJp-dkNuV7=-U*pRCQ(nezG_Q%z zn7AKB8tzBWZwK>E{dh=B&GtbWGc)K&Uq{}A zAc}S$B<3XbftAOS!I;P$Q1+{rPPhSqSBhk9LpfAb^U>G0pWz zOI4AHHj4*i24_H=uqsjtkwlXjt;D~)QeuC37ICwfi_)JPp!gwk@Z<0_@l2-%+`X>I z7(R_&eDFsF(0;8$ww`kX<1WG=HrN1YkIK`xJPr_io+B}0UcmF0wL-HDeuAceMBwkw zp!@+brm*-63hD474`)j<707_pI#5Gw_sK`*mCf{`j~9sI0$~O`{0zLem(liO^T;TN z2k5Ds9=T=H2r3H~B2}Gxfw9N{J$dyY?QB$p;GQ86_fz2a5epYEfV}gK~M0kOOeT)o9Th(K+wE-kawSN zOV&G=p}iV;AobctqE5h8G4FeaF6+pm?nN)b^>8`n>~jsy?h|Uve2y4tW1z%5XN!U0 zdKDSx=h<4W*g*`>c!9=b;&>0~g-pTGK(g(43~+StBAdFo#P);cjPo3EvPLcsUHJ#e zrHmhQYkdbQ`KHHwiaJdcT9kv@stdqXtr^|8(+bLjhLOF1<*4U9hTgn!1z)Ubw7d8# zxOd_>qGY}hOeaFmk7|&2pT0w`!h>J~_cZEuYNy-c?r_xiBEr3F8VT!60P*i4%t_Nu zwEBh~xqh)Zu$gfM{CQOX#6-lwhO!LAD*xv6yj#XxP7NXb*6WZZ_O>MVtU0MV9>Dw; zo=FVdcV-%jis;31TFi;%s$}|VNwm*t8EL#clxgxY0S<5UnYHV^fMdXZU@DT2qBh!t zgW-1M8;RA-x2|zcKZizLz3KGDyWPC^ySj)MUBf6g+lajCWXlvRtf51Od(l@PQSxC1 zWELJ=0SXI58B^Igg*MwsmYeMFji7`Cg8e-}9YiRIi0a(-~%ae7;;-D7Uu<0*4m%EaDDP@lAgB-|i`*|dX@nOEK zO$N%(OqhqC?xJ1dB(NoZhHrSs961m3YdS^?J zMrwjjPqpd8f3~3+OLfrdjHR@TV;vfG??*iQ0U~ST7I2_1jtE^Y%sjXljC_u05MoDc zLG{Q6u=}MYXGP=^w4nM0ku{J4-swdH^S)?cYAoP>j)gL+0r|kh^B^iMlqB{(&|<#L ze2i@0Z6x;Fp9Fy!55TU;O&q_b03=e#wmf~8N5~03MH(-sSghdjP$E-`6hQ2$ z0W`?*1>;-f(I@lOsM5Xy{T-VmlCIqYF2AL~5?&fXuZmvm$z&o|Q&AKrcMd5qv&ihw zK(yT8Hd?Paf|Ppgnd&fvR`EPJZ-69u%kLfM@oIfAg!@rmvH~NsHBImv-~bL)9|u-) z4+YO#16nba2IlFnCf4ZQ0Xt$Ffcvs%K=hzE-MH*3I=Q3|2<^KBY*)@<7RT5zmo(>r zi&L>+rjiO8jxpmc{V9dqKeo{kVO?PGn>4vzYzRI5P}p*Kxh7d;21zURFtm7nDN;Bd zN=H{%qr4Zv=-gezF?*#$&b789FM7;oa(v5Bz*2v(rLGz@+X<1O-8Eo5SpyAOiwWF6 zXVB}`UNmliK=6KHvgo%dxz0p^{3H^I=I>m zW&tX)dLrQ8c_J5sJn&Y91$(?^k)G%00bWKq$WTu}>CK;!dfZuLOU*zja>Eh*R8a zOePKZli42UP))lR(zOkyFRuMX&%e}0#9W@uMJuRjl7A6W(# z4$G6i#$!abZ!{PyeohA__<;XHmLR#V<#eq29Mt(*80ZM<744KrkbdYC=uc4q_lLiM z#YN#DO|%HiwN@qjraVEm2gN*)`HwlGEyR=xJPisin`ylcF|y%ZIlBK(3DiallfHAB zL5<8iw13-T6dAz<$HU))HxJ7}cYg_RwW&uHSq99m|GLrlA1lzTjtbzWJ_$Z#r=joU zEmZi&4n1?{afBazN2zu%QLN)nuyFe_GP8dI4IHRLs`{azR%;UM_LxVDt26*=DiPGY zN(X&JBDk~RC85)#g!-?H(H)kZ;3H>*zSb2_%W3@}-h2NfhCfH38Jh+5zp^NDm|uuO zwv~XAvLztW+}n0yqbS3(ZzHb!o@mLfl8Jg)MK;>{dt6$a;dskD`FvEeeEWB@lD=?xZswn}BpV1MpQj1*Kj~qQ547r#pB4 z1h0mZQC+VcO8I;eJx_~6Dw9(nYwQR3eoq-ZlC`5lxc0R2Z4Mx1@=*8QB5>iCphnyH zn9daW&~h}ui*S4x3u<$2&|gdfiB|);;6(aQTK`}x9rw}-af(vV@;MRUBv&2SWNjhb z{OUnf|2!i6`8G76qK!N&CTO0t7h)P^!P0?vPT`~{@;o_$^yhIrzdFK`|%^&-5##1e0Y7ZPiOgn^oS1=3qMN;KZsKxbQ2(2F$w zL!S*(>ARwJpu*gYjGkYKARUdA0H27M<3M-El2LFIBI;^1u z=(Q)oVdf*JalwBmxb`7ZDMl#!dja?@VDEigA%oU`i>JSd%A%ozYlzDt8)(*hjE;Je z1f~YR(S}wTXe{s$5>L;e>)PS~Qi>pEw2tzoC$FL->M9^{)oG$G>@WR3U^Q|z+)cO3 z{~!Y1ekNLVPtfnX9@5@7eVX;#JAu`1NjmdG4T#iRgU$ylpxXQjRQ@s+*!%&Dm1LEP z&;SF@(!rU8&Aw#LhqaQxC%_rmJq!bjJGK&sWU|p?aSknfrWtU9S8&Q!+koDRa@2j= z6s#c6p$(!3Xsc(N!1cevAit&_s2C=-?0=lX$(-(_^BVO~-!~QT6_H@)5fhZF;sPdS z_;FO$R?%r6uK~%?-j+9M+4PTaeoMvETZkyGBpUQja@qx!BLArQZSxt|58k9Y%ow!3!-_)UvTYnJ!1>X>Ud>dR38wPLn_~3@25z=mR z219MeL}B|5kn*MvJdNohww;cq@4eyB7rv*1=HE@=ROVqY+;oe!Dy&B$JBn$qYgg&h z*-b#C$^vzVzeN#2PV|ZXP7wI_ELy#KCpx{PlJiB(9T)@*A}wV*pz3N4s2@LxpU=Od zJ1Z4PCoy|cG-4^4ATETy_%QUgS^%!<2{@isx6m%|3oMv7o19xW3?5<*xzSIb`EPJ3 zvoVWgc3#h=wI}}4)bT6Csu|0G^Er$Rk86{s$7YgG-b#}DWFq{7#wLj5k3VVf_&joehVO<93jCoIdU-6N+^{^bN z^-7TMhV;oj<_hHBO{G9$ejD<0o5ggQr6a3Nmk91)Gy1Yaj1VSQfV6}r;%L`wr1^CY zifz?HHli1R*5%7+_(45-w!{I2UAAGK+HT{lzPJ?p3o<5s=t5dUS0u;Yv+@q8_h@}(uCapOOo*vvS*5JQ;cgkhva{y(aHK1%;vQ# z81qZ+OtoznSnJ?K^3}b_J1zT>cf>ld#o#Kq-d+TvC$^ELWfT*;VGEY16ti2a5O5-e$g0%upi%1|dTjES6Vf3>mU4!H>68wm zB6}PhJjx}DS1uy+#1;sgO|76QLy^3)-<))jY@x^h3ApL8a-<-s0GN_3;M!L|v}Mr< zI>xO=_uqv9`Hjh-MAHeyugyb~-i2rz_cEAOAVh{l3p_aCrexiX0aP>Y!0?W{Fz*w? z(2`+Uvc=MfjQ%={e6eE@qI`GJzO&4b(62~@CasuTGw*}35d*T-U4wL;iNM)WF7x@9 z2E)-CM13I=Ovr{=D1NuwI#g!e@#yxk?t`%g(3tiECU?mt+T~jWa>^5d#qM zG!BI=en9WgT@FSv6j1}=jI8czp|s}H^hEe(0V^AWxzhQ-Z&Cxabi@ChRH+fBFqjCC_rM&AG-45f^7l!eqdu#%|E?#}laL zJ_3T}hSGMYb9kG2QN(#c4{DfAJk{txM?SqH9=vlv-U~f>I&=Ray>2JuAUTDM3!=b@ zP1@vnUjmS&mEhbzA;#a~DDqGoK=a$HP_Ng0aN)%e&-q?7I)0Zz3lEkdyJBIcD%uUy zm&l@ylyyM-oHVoOUogT?AEAntL=c^I1r_M&p((u^X#2Z2puD68G=Ba>{OnmpoDPX2 zK4f=uj!d=?s#}bK-%tr?y1W9oY!U(T^9BK(;)N=7{eaKEYT}t1369!+MYBuRq3Rtw z5S-Y79G;y4S?`B|?i~oMJHLT|`}Szfv=q1@9*>B~$EY@RElAjR5jbDV1O7;bJeX*V z7KohzEq(8apB8IC!*?3_8M=b`ZO+8w;!aRKn1r5<90Q9rQ;DV0RbWH<6-3|n0b7mb zi0M=D!1F{kde!n7Matc$xmn9mbMRMi$UGlhKe`oo%UOdfxq{y4!+rFWY@&r<&tZtW z>ZopFCTYcCY{!rOAarl8Mc0$21?RURpe()#xVTQEiQ^e4Q>`E9&|+ZL++;L-w-B8R zo{xm|a?mo{KJa$Q0g#K|@S?iw2(_3xdiK8(ARcp!rrqN>TVI9%wIA+?owors%%)J_ z_e(rKX=$cU%$;~E;)V9)j)7YG9O%{1K&Rd(1HDQ)qS=2vI(VJ}12=~VT4O!9w^xH1 zM$N=fRv|(OW}run1KcI7i30yRU@YjBO^wCrpRo?8=FSMcbz~G&T_kv~yjep1^jc6Z zdj6r&6<_3| z5sAdj8e0y9+@muWo$wJ4NqCX2X2`E=cF8r8+jBT7^< zfcv!`aAsCAc(YkskbU19UW^1rYjdKIlZiXWSNRFh|0K?IEEM#-0tS9dtroe%~M`elgkA1*9?<_+6*qRn~UO~$v@lJH`>1W>525YjlL5BQ$RU16ap;2=EVcWjK zagbVMO|G42BNV4K>3!vwfP0QIart2n5z1BM6=%O9u9VahSf&kpIarVG-kU;6hq;Kg zNCQ?V7HmElh*D0EqW*9RaytGpr+8jFn)z)CX+5%_OOC%ouear*y0;a)m6@r$tVwI+ z7(GRl+CC&iY_hmkl4*ocx9I_bho$IwR5OI-I005S{Z(Z(h>RJwrW zm`VIZU&QVK&yYzliv5Ap?Hn+GbJ0%jU;4S%5S`H7NiTBI0@N}g5Rj_}OedPaxex@l zt!tzctUA#^@?Z2(h!5^_-UGc7DN=2>AV%X_ieA5PKz^G}(ks7fG5Lwxz}4-|pkk>x zDSRQ8m}*1d&A=5x^``^_8%6}Z^=#6{@*{CGU7HEGGlOnztVQx2b#%1WEU+l(GV&}@ zXTrH@;Aoi^Y3OG}m}OoEUu4rkMd=34?Z8NSt>Yy6QFtDml;t9MpF;%Fy$X7T!ikIT zccRKKxhU3QF;W$rLp(X04gRhgLH;M>(Aqbipx;`We9MX>qsBR4K)e(z&szcFH>;xM68qKvGZ+Q2Mq3HoJDfS3!3p#FwDBPAom_;_Wb%#ZnWh03MPgj=sR`_ zFvo=HBrz6^4b&sxl!-KWOF+d1PjKU86tKM?2#)UP0SD5ZkVX72B=f0%BM(p?Htab}R{b%LS+r!q#T6-0+le)+oipU4^y~4rIF+jw)-KJ~aHG}$l%Mi20 z6?7aFoFQIr=D4P(bF%jAZ8>n-5&WHgK@&4Y2wOu{^y28tmSY$4(3d0qE&i!HIcrq* zf}_47^unX9XryES<;U(pXAkJ0BVV%Vf<$c)L&2AuLB~x5ohZ`3O#x_P0x;I=q)lI(5Bl<(4(2fh-^wm z3$xZChv%gr^v^kg%WO7q8)+cUZV*Suo$>_3nu_$mvLmP^^(^s9gymU}T?Au74~d3; zCGg*~c;e!dz9#*nKD?FpD>&`ZZ+UJt+Qbl73E-h_WFu3Kk~5M~dT~AdI=B%jr&@s9 zarb!(PA);~EBA7u*X%^MuSOsRd=AO?WT6Gu=7HAWYv_33bo1<`_P~1=U$7TlrzL_P z(GB&|AlOcw(;>4BbopqHu7$Efu_WW6_p%63oDrGl-zqAyViQ5b4fE=R_^g*2*IE@8}=eSnD(K zydwq}M*E_20cq&@yDGj*B?X>%JG^l@Z0|kDIKy0I+roDF`=pOa}{J@9EROCG9(fmX#`+gdjXb%z5 zQFW+9%9D1o5JqK%Ektn2CmW;aOmOp|EbG=3w;h!k{smnD#oiz4*DQYiBbLS2sviQy)F5G7=bN`!)mErsjB zgYguQko+G|SYm=&d`8g$b|H_RQwIKc2$O{uyg_cKHz@l0fO9(|2km|!1g={sA@}?D ziObLX(8p{SP-SRCH~bBy3&wKLY>joanS?Vs?vn*Lmj4Ky?9HHO|7ODNpgf&BF3cz? z#(;qiOK^?4vG_l*9Vve!c_03Sf}XFPoUdu8i0Zw9?^fLb89hCV#BPq!E&XbM1{_qF zZ-O4z7a%jy4$e@lA)voM;?$5b^uB#Z>EE6;KxFs^ajQ#@sNUs*ZYgp=*e?t4=6E-7 zeDI8SaoiD^bVQ=e`W^HG;R#|aF_LyS_ia&?9wZhsM~Dge8|aCTII6oKiOdhpr>)Z& zbSI^ij(CDNzN{ViJktccjbDg7+7r+X^>%u)g%AEkyyA3)U8Frti)hct$G~jD3LNd; z2>wlPBb4Vv0uhx6Vqf@Y!gNmyk*BwXKIHw{R!rv;ab4sUuc(LuHQj;0plvRgcjPE7 zdc%O&`zQ|VyRZh7ua^QNEp}*!W)zwma|YcvFr(%A62Qu+V?2W&xpdFaH%`&sTg0Qc ziNGb+h~BYLl0Ne>moAJNB|`e7z~1NjsOYCNIurJS_p3yC(`R}rI765$8>MoDxy4{=GdNhqBAee=7|NH zfU5u6i8uS}cqWIp(BZlg!1snYsQM5Os=@^R-z9qh_dkKQ<#PZq*nO5hV?0e){8R<) zg&Cmvx<73jy%n5D6yrJnttafGgy~Yq(lx5pMCa^yLZ?+4_3YFEp382x%)6rrVyk;O zwuKhpXjBH8bV)%4OOK$PT_GU)UwCs+ZE{=N%+C_tTT;9`#|mvN)C&jEUJwDj%ThCVmm$ z5<2O3sSClXxdGx8s3O8oSa9m*+W_sm{-B&v1bb9}bM|-xU@0`t8(yA9 z2iF2(|_Ox&ac%wxgz<8k!DXQe3i>OPb#mk!>%$^`Lc=fU^e?P#A{5}h(5o-n+y z0_>g7@}Rg6D7q(wW;E<48Xw1?|MJ|B5^pD`e5o90=pQ1yZ2SnBh%!PZ*al4YYy#%U z0BIc0pnpDp%F%cwLcj0{0FPcJ(G}7$#Itvqbp7~3UjJTsBHZH-k)zX2$mPuf5~?Q% zv)VFVd!R8b+q?zn9*L!$RRsyU9BGuJrpMXVIYGxst>GA`E+qC!`GTeLt2o;)Y~nd? zRpzuXHb}2yC79t^-{Lmu3idkx2kdi%kx2PG|)J6!-kUUVytSA)Ooq`^3 zqG)v^F1U3>5=1%8LmDD$P-dwt?P<4+7*8$(L%G3(eS|cftLugWx1XT>mt8@>){f9m zWBO=qd>s5TS3`B3?VPP;UZ5h=n5gyN1b%OROcWiAr#)ropn&hL9KGK;#Fp$^#Q428 zJlFPOdV}#Q@N%&noyxD`5FJ$@ zX@BV>UiWC3!yjqwyZXdDFF#a`P7oR*?qJm8FQIZR99;RsMgH66(Y}{av{>>Q7~1}b zYb|PpdG;#!mCFlUqAOtD_1QqF@wIH_!Dm!)_Df0$8E`LKMZudbp6soK%b>>d0jg4> zlRFk&hHK^8C@J?+wyYu?mb}hnukjP87N1bKLn;rqzH;Q3)I6gWG@ges>U*K#gAb4} zTF&SD`M}Q~5dWE06uZDb3^(2owQJ#jXI1?Ruu`WvU(Rhk8`M8cPY>2%?6QkrW@&>{ ziWBkiqj%V}>Sxr|20?zfAPC27lxU3!ejs3`RZ&;n)?g56gPUs)z}DIWl<@iaR8`mn zTNF2!9oe=TesLY4=qp?CPHlB4)cu?P;nywhMyq1#<#tW1{3VB7v)2sIkFV#STs;6y zehXOt3lvy`+XBAwveT4#=Py_s_ZDW&?SV6oFT|puM$jxS9(FXppyGCf!8?Uz?1HXb zSSxh`_g(K}e_KejK1`kof5@i7XSSPh0KwQ6_|LFY@#}-w%m}`pqX!ST7PCul$WzmT z+{5;yPBysE0ebuUL-~MjIL2)*-ej=@_8Xm~(#~z7@UeLIta=^XBee=w2RlN=ljq^P z_FnkjE0_9ruY~Jl+J;ZKDpMA&D`3Z90e`QReCw_`qqz9reoQC`+lh6^!0k_Zxmr0+ z`1iWo_|VfRTzBL%)tFTa{eLy#iYPZK!9JDIAjhzfTM`y)JWD;^?LbvJd|_6=F9P;^ z3Dv*WAFt0Tz!&m7F#p#v*p}0V<%ZI!E~5-;&-d%p>e6{|%`r?FAFPGDU9_p@g%8+c zyi_=L_$769SPpK=HiQj|9k9Yg2dx*orUkZ?qW~6Cqm5* zbr_Lx1_r&qL~JP7f&Z)*%q6fNp={@E49Bw8p8zy14hYA8XLn zOKq(naRLWXoo{B?)qFq4_x&COPZvnD^7UR&r62(wU2~37EtItLn|lCzJKknh+7@H4 z+oz}jNJBG|+Cs@OH(b1;+MOPhK8WJlG{!CMv^DR6f+b#Tjk_&I5q zdZxApN~FiJ5ka@$)QlRu_UPi)4=WV0*bxu-ZTO(+mp)8_ zQwby!TCvLq4}4N<0UY0^h5tr&!N<|*)GIm*XN0BWH!2G#Hy1N_h@CIUAqx0XcRX>M zoe@6Su@@dwT@L@gjf3-lzY?r z>;@YFe`-1fi%eROZ*Pj?sG1R~T(<;&oT&vB#n*71tTtnfug_T%F+FH~YY(+&RxsSR z{|LN)Zyl_Pmx6)sr`S6J&(+zneAYzI4JYK9LU@G2?|1BkuIB~$+b=uW4);dN=wuqJ zUwn`?$li{(`i8;f9go--r|O`O;%OLoHUS3wbB3S90@>^N7ulq`d>Cq>2elXbbC>V# zV!6wXLT!Js*3K1@>{%;e9Bb+U8HGgr(r%pku?1p}O%qUMgrSgp3w*6Lz#fxshMXW8 zpHdOz^TKuEf2kK?&D}cOFQbhM=CbU@R0TT|r*vxHZROUq*EU$AYA)tXxKnZctME|6 zMeLwejjKv(DB+cZtjm`QXfozUjqgx`$rt_b=Ay%_euyzU;ixM3408Bhj=;&HbOHWV z6vIvd3OHFblvQ0Oi>r?`Qm3n~QEk?_xJ(+W;|=WI?^4F+K1ZCp`1)rOtaEG7*qh0Uhc&kMw%)w8L$Kj+{D^>O%k-(Gk~ zB@myEtby7URglm&gvGqyu{plcHH4o#OPto+9(d<&ox2%xM26%d;7sii9VQrp4>sN<+D59qhtG)EcssfH!>hsO~ z%d7U|m{Sp~XxT73b~l!F`ov++b%wDV#b?x`uu6)x-wA((_rZAo05-_1hZ}3r!m^Tw zFn>=Tcx&+q>i4`;usi=gYqaYh_1cw-Q%oMg2M^ET zCG$?QeRs<6Uh)X`jowXd{9w=4y6mJv(+XkFm>x_FZl*k?T(GXIr@(D+h#zn^fjyZ0 zik-{u#Jk&GvR@6%FyHnr_Lo$HmFD+wtssxyyXXlnx$zgxBX6@YA5OwEH=1$iH(B<< zh!fP_Ct(-I(WGq8_h6x2jxg}p8GPr1fZHGQpf(8RMJXyKhk467yI3M(Q;*riVfsN%U^@a|9~R-E&f?`^psS7$21FY~kD94#jp zCZvhu&<)D;lR5l7TgooOeu~}n@D$fqqlx*Wa)Wj5&xS`uXJPRUed<4=o367vH+VzS}zo2eSv6o{4>|5b}dt+=c8qr|j91 zXD8rW!vd_B_>LW~G!`(ogj-YGrl}1}5Z>FP0DJ1SxyQy5*+;Ld+1`hzFz(*~0X~Jb z($~Ps*Lpa;!5cqQzXVs!&!>dm_EQSF(pd56Wh^^K70Wd5gC(A2crFrxQai4a$#$-6 z!c9@?`(b%JI;@HVB{G?be@g7v<$IyQEie3Tmk0ma;}g){Hj&-=Y6bP%3}bNtyQTQN z6W{nq7QUuIK`{Z7DEhZ4Txz@>23h4n*JF3s4AI-vYVd^`&-15hPaed*s~1DN%6*jF z?Y-0jw_v_g{83mNC2S{t_Y36Sn&hwa{lhk0Oot}1EAf^|b-wCsi*pJgHP_umvzVrGn_kj9Ed^=A9j{1(VTOWSpQ-5!j-0NcC+@51F{I(6X3rR!gWyS20WrtzT z{${9E{S&*J2)L8I`>}CzGZxL0!1n#d`1$sH>VT#-KBj7pH5OIk{oY;dxOEKGV?PQL znwltstuNT;=1U-v{T!}OJ^`=Zu)_z%H{oY~vGC7iGP`EU9~hz&M14#i#ol)txB-c$ z@IleJ_`6`QU%urU40_UyB`=+z4%c|#$6m`};U&_aFmwEVb0T=#$ zcNL#cX<}!DU*#`c69Bz^eo>+w|3UF3GG z>nT;rYPwtDd8pWMT4d0fq3VtSudJo*Bvh`go-Kes_M|3x_R zl{emHqlCjH^RbhpB)ha&pFOr#7@A&7r*6FKhWM+booiq$t7^~&=jrRQGTLQum{^5} zryKCCV8K<8NjSa@mcUb79vt))!{zHJJpXGH>skMmT_1v>qCisTFcprYb){MjoKC_a zONOn~>t%h2VXD!tgq`!l1wQ$^0-g>tw5ye>V^5gpQbYh$18sKN#X&)L)&%J|`tH8`L$S2D|VYw#0#)Bs4S3U^r*XO{A#Zq<_r#!G?p9T(BzD|8mF@Q;%cEiMZ#@xzX zA6UoGGt}&?dYoRL$ey#8r`q0VwHB8fal2RDq>8%o`6-$o+4}YO+17DqIIN#U^)!BF zBYuof|6n(}yhNHZ(7y)H^{vOT=jGvH^E_C*YA!zF-AZjW(|}b&0xsHqmDV|*5}?*w zWtcj8k)5b_f}W`l*=jEp?$ksy)v0w4ve5>3YIgwar?#_3pT1Dj4ce@jpti{W-Hg-9 zQ{YqM7*^=yO)8{b*iIs^mm2I8f<6sNd?s}Z`=LRtwYE>BHTzc~s}L>=Ev#=)yDe{` z?CJMdGa&#c^>k94D2m^DQy1s1X{VHr#^K;99=}^dyY(C|3wA}OvWeTz!r>*`@YGTZ z>iOmr>O{+TY_1Uv&6qao(b{@EICK&&TMdT z?_!_UudLwg12@dRA@G%v>_xd3l)x5@EjDh)&eyI{(@J;Pw>B;CMoSEQweKtr@fL7+ z@+i0_eHS*{QH>WE4pJ(IRM}F6YIyO{6})+(hdr2L(E9$f9&QRe#VWOLgx4-)QRT#M zY$b5#R2%k@Gk)dZgqj^tjSK{+c@>;Lms}ZaGML3JYpNdN~zd4 ziujwQDNhl#a_GklP^7!!q@Y^!&|#HVELO-_`UR9>dDk0 zSi98z@iU+GNvUs(qmaQQ#DC~JV_J(g1@ z_aAj0)w3a+HTc?vS#YC#FSaig!$0brSVg5a=sdrTtpqpV92Zac#dD=#LyLo#Jk6<~ z%^&b+X%~A`eF?iFeK}5znAy5=GMG&i7cfyG)a+)bbiq@vFGEt}2-Ruy0$L4>LFpnf zylB@-_+qXV8F;P>sy?~RHCq$ME-V0CnZqXRwvKFeL#-ox8b1@duZp22i$1{lFI1tL z&={V{%z>{i%!JbXvuy1mJu1w35PUZVlj2 z%RIQdI7i^U;!(Bnd~TY~I(X^7C6va!qwGAD7#MJOCp>s70&4!eh37`E!%i!E@WEy6 z?46YP_?gUIR`+)RHlM2v4-0C_^6AUed@*DAb{&r&u+9XY(6?qkN;JXOj=C7Vn#ran z&uld?7!vd~lNbX#SeKCr>#?B%{}p6C;S|+3BMEnSoW+U-r}4UcN9e3f;~8GRaeYVIME1^1ln^N^eK`&jccz50_V**0A?5!lZ-P`iqraOvHooD1%dMcQt5c|T znKK=-*o*u&Z>Oa)qip^yU20a;PcM(GWv2bsWIXmSwZ3t_pYDC&hJFprM|Yof!-I1x z=)A{4G|FLu^}J_a*+VPh=%Zde`XMzHsqPv=I&+$sT_Xs^i5Sz;I1hSf!!cxhq7up0 zb)cHY_sDTw7bEfT2JnBl{ z!>vp-{|MDNQ_nv22^M<4&Zl}O9qGHYe!i^b7P?n*1!ELph3eKep)e~3O}=vhVS`yn z*pkOSF_NdJtr{7d3u$QEioG-<>MRQR?+pK;`|Nf$sE4gOUx*T)5A*#^*U`wi!KmQL zN)&qF1e)UWkhN7bVjm8DVRMwuGxbZ4pbzs`uyYzWpxVN68hXBjwYw;Sl-c*_IhJ6H z5{{vFMK{^j^v5)5pAy?yThCN2_{C&Bt3@4m-?5)Y8rcGP3g zOFi1B(ayB3o=7WxZKpb=e^IrXBV7?{N|gBNFpUCAAzI{S{cs)_Gzcmks?Gqk=RFWYJvF|M-b3ijneI zEuBQ4u=K!wjs+!wwoYlFpFsoO*DMVbo#INjO>Cl~ zQH$xXqi5OREz;I6IG(uoyNSr!e2CE*)XIP&JA zuzTx$x~uprvwokFwd&EmX!5BJD&Sb1gDuDCv7Z^VUFR|D%(J7Wo4>KwUu>dYi3_N6 zSD28M%Vp-jn~Ju**^e?8--UC2KA@>vN6}8{E_(Z|1`8J4L%TC~(CN)0y{Rs}FAkt^9q&M_Ds62a=P8E0azWpUBmZ0nGm^Js<1AnidM zdt##seWt?j=k=XnJWpMqb3rq@-7%A``iU8xPcqg`W`Ailev0B=HZZYnGHBx5`Rw#| zXLR+y5QcsK0vR}n(v;WJnUVcLXwRbQtZh^~qxk+Yd+d%0`f;L!p1WU$j@G^w#_l=| z?WWr^=1-02KO-Y{nB&+buiVROeOu4Jvuk_X>r%uFONSvd?ylRu{v~rp%bGTA7f>^= z`^Nx)ZHMI^xcCtSZstTdfU*8*UHuoM{TIq;0ao?M#Ng^n8Aw1 z3^M1e#_6H~Pc(cmm!8xf65ju4#ZEYQmHAy+fVhYp^&HT6t`V^`z9hY&riQGxog=@fAO?t*tlciJAm<{nZ{;u%kg(VEm9tldYw<2Ja)? zD|N_rwict4U4-0ssUhjG!*ui+%ba-Ij|#Rma!v$!nv*)4COs5sA6@3nIt5puV>4a& zFI=ApH-AuRU!OgbMqkOND;>{K1G`A3Y!RiJ*OO7f=}D*~m}7%WjH61A3Oar(m>yQ1 zj%=I7+T|8pWzIhJN5hOOYB305_cB~9mGgkOk65GhdH0c8iK9@g>msr}mqBH;0P?Q8 ziO#N@jV7L&gUE(jB)4-DO=(R=!hL46&UBeI8?V-$ad0lZS;o?=>BrdbTUu#yd}LTyKP@pZQy!oQt%qtjl&EPwA~L}G{esgjAz-DV#9?}aV%U-o(yDy~HRjb3b! zW(DJK^nfu+u|(r{^C&o@M+3?sO<41upFVYfpE6I8f>)_CnUvv)dWtCUg6RzC7uI!idM?=sTXR-;~${zxS55c@_WlHU8fkbaU_ zj1EW`pr&gjNDqoy=VzqS_1l}--ENQhecc)8oAo{Rw)5V8}YA!#-eHJp(G^P6vbg-w6B%zBJFHn1Z1t#Tc5uKnr19fz%Ge(;ausiMt{# zQPgl9lK742Zry|IZQlq+E#@*6b6dnQEk)XAlw4+*7jIVu~rTuT&(C^VyZoNJoJoAaMZqjT|I-Pc8d+v=QItF0F(+EIck9GB702}1hk zEY~1=;|(j46ej%ao6e*mOg&Bhvc=yIVa=#O+A+EieKMVDJ>z*0+Vbl*`@Q-ZD?a%) zSAY1;l5?jxmdrCcYhyCX*m;y5e6fTdr#4D2{aa3LZirh?t5HEC9=&L#Gsgz2+QnSF z)xd^7VvXjoPC_&U@CK~Ly0gBDHPzN2c_JO-q z==69sYd)t9#d}VmqnxLEt!@QwXxJjo9xp{#_DZ1lVi_WXiFY{xzxwL7VcG{^hdZmXB1$z}@Om|{mgTi>wV zS2=Bv+A7*`=m6@ya*zK>J(GVoay1-`QEjg_)TDE5-!qZ7T-nw|&WzRPF{)n?!5-w; zda>_zu-E3bvVYTc&>n>xB%Py*It5uMT2iKc>efqi(%-Ml@g~Bm%i7X%;UiW?c!qy5 zlqbwMrD)x=`98bKU>S8T@uTuf{ORk$cKju%ob&f?r-}_vSXKB=zolP$ZUNF(AK8zmu$ouc~12&;Dhh8!^Vz+5UQ_Z}+ zNIEkC@fz>2+F5CAh|CN0^R_Y5nU+9L?LC2dKCh#J+hv*0-52Oxi{rF$T|KM*xsQJ* zyo@%qEkwuWBrsO%dg)EwLAryzLd}xQ;T%UDbm2=Rdv=K~ePUb5_R5x0C-MF4iGnR; zs?Rstdd--cWGPXt!!yxE>qd@KX~c>?>Sw%^R(xiM?$od#_rzfKiJWqP32s8Y7uKAql==iD^m>(P^s zjpSU!bN0AnN1>+;P=s$Gia4`~Z)aY>*s@a8Y3vL;d~72H4~LNV&r{3- zJw57VGl_a{bz#>o6{DSuEV4SoXYvmIVO6XBnHl}&%x}BH^qZ3eIuu`y6ZC@l+b{2D z(m1~3q`(R$XQ-FjK9i(A>-9MP`7pa?PdN=?UZK1LhtQgLqbQtq)4)@?s5Rs!?eu3* z(=8XM&>KQqq<>JuEfLi9o|sUwPOtq(xjfo#rPOYaTh6X)kZSKzLChr!?o5%NNj=t> zP)SEmCd6lw^+DfBG^I0zMIQ(0Ka2e|vg#t|GvYjjON`h<)SR^g6)3-L9}{T51^v2q z4|z;06y|Wb>bHNVIF?B{jf>>QHKu1WBUV-@SXF}K%YJ6xo;0AxtAxUPEeny_R0-se zt76?W=1rxIezCv7B)VDgAiBq%WoElXvZtQL(2U)p)*@$SqWU}M(aV>Y={$|usQmXB zdzZ@+CX6W}(;p&;-(`-P27Ty~y4^IpbT2*Tm&dym*u*&Zg>t`@DlNBhWwKmKXsDwo z`&bu|>zA%GFRBkQ0T2C9>);G)nag7IsjL@$@hFU$!Lc@ZV`iLRH3qpqS4I(zWynQj zT6<6A6q@%%m)@`!LsPYGB8`MJ);w9A`F499yCKS(@jBPU#1RYX#0|~&2TntuRK)4L z@W)J^cOZJNB|~jIve>-MyQtL@2unF;S%_pc+n|z7dn>ll?0;vOQio6^ExD7KH#-YO zCJ52ml_n_1Wek~jtJ7|0Z`8VcKkF~>=K5UT@RQsMX~UvibWB`{o{w%pDF#cKmg%>d zuySp__Vd%GIgS-oG`CCjvtaz`(b$kj%kGhecnIvs3%?R4r=x`b~Yu0-oq3utDq zBQ2TG!CvLKx!(+x(8|(8;eQu*ApU1LB-T<&m*_V#_?>Kfk<%d%yHK6ZQQt|2pJpSy zH8l*+`56=KQOhWM|AkWoPpQri#r6`#disrh!<6rlLk9-SsDh(0l}eCiw0ak#WNjQhemV!u+Hb(B`)42*wR)s|JBrFYY+$b`GE{EgdwPNM zv6b26qn?2R{$s@h^o^+-GMQM4k|*w?OaJ<@X@2XNpxryE+HO(gV4I5^NA-}Udj{HM zsEB?&InAUUZV{?+_sg1HmyjarM?qb8slLN>>jOSItj_2hq1L7#w9#*rX_o0_qk>e~ zj@qA0&4!z7;r-|QC$P&DfFJSDN+#I z&ct$c^*hoh>7{w1?H9Dq(T48H?T3~7S$XUGY`J79``LLLD%`q{`sN-+HwB_pHqwP% z<**T%alZO(11ZS#$Z=%Se+YGOUWUG{J^Xt@5!xFuhkgd;=tAFHR_Ck)vhhE|ruO`0 z&X+G{*1y=z^fWWbZQVHIs8z(;7OJx<<>#rrjIZ#hw=sHf?YgiWN73W`bu`N209`q~ zl8SQ~{dk26WdlU$yV5*1>qV{5#WDq1$u6SOTMdz=74}Q+<%$JsIkedI~u>)H2fJWoW&H zCwI3#z}hQ*VjlZQw~KwUMOS{E<@5BNsaaGm=ld+D2@kd*mBD-H<@M7v`Z?#2mNT?2 zy6=xRhgZCRgU$MO-A30S5W&mYtcFmQ*?8U8neQq5@;CZqqA+P$QmxEyx|3W ziLWy41=$AeHQC3QkA;(j#u^vUdf{8NGxQjv$7QnR9YX5(iO0s<9AHMI>ljXG#AdB) zWE#hAGc}i|BMZip7Vk}BLUtCR4+EQ!=B>%}eHX*+DKd=V>2ant+yo6TDrCg&$)nV( zA!szm4Y{Ugqb|o(q_Q=YnnmAaZ|H|nT5e9K?YTtjB$lzedVVpTGX2cGt>v^}CSxrn z9Ho)cu1rqe19U|GF|CP~q8CQFT2SXP^v!W2O8p=~c|)bt|1FobESGOTuIfZJ^7b-s z6(2L(&An*xuO7w=yk}-5UqJJ2cQDq{!ThA350TFG6u#CaKH4C1mR7ZEqx*-`kl@Kg zCNga&H5pvY^e1zk#EmCu`tE8pe&VU{V^KE^j%;M4IsT*5lNu^~I+F%Xuj7B)HOhSH z=|u0_?dZ*!xis~LAKfQDnf884Ko)NCmwWt-gK+dWY0|Nm#s+`# z{=z!a_uG*ampQ`GA3u26OGtZe%Znc3rMeR4@mI9hybLs}PxaK9r2-^2pU|_HbiyG1nhn%Tw#pz;l}1L9$US z(Th$bYFDXpR z7Qu1W3<@^QheMASlF$-=a=_e&_^srTFlA#1uatwHBjw=8{G}lFkp?&Wp9zh!vZ3qO z4Y0|_kBlYGg|`>ZgtfWz;ihM%WaVWGES2<-*T#$pDvc+DGYjL%oMVy1)pspt2@Ho? z{0I_S8BSW2{K;JDPm3=9VMLF1res3MsTQ$pj(e$i~O`zOa` zoU|BxOpL}IopCTYC=sU11j6Zu!im_|T-k{EoZ)h3T}3?(p{X`KU@Vx7O#Zk5g~AY^EOg%mJSD; zGoj3j1o(@WNb0Q-NxmIV{U;<|F++G6g5;N)JK4U2U$bdfrXJ}vR z59@p#p|^oAkz(rb*Ed3Zqv{eyGoOPQB}v34B8;4l@+F^=!r+ZSa~LvU42Pd8z}dpr zxTZ@98z%0@ZwG>LPn!dn?N((8_stW$(jDaeci^XBd|&~2!RQo__=6ewSdmQgmp2t%U%mvPO zDX4xriJYo#!MaE5adKlfzQVB*7Od7JSHxV&8JU^HxD53t|^2MINrvC;sUg#3FtU%+jzWxvLS08aE~T8%!X{ zC?$O|m1NVqo#cdgCS3Y&6Zh_+1ipj z`X4b+JO(bNFKlZ$Kcy}5(J9`p;idSr-y9M2bY7<8 z9iCp40zBb-9h~A$6xI*q31cKpMmg?amDyBcZZZu%c*_dbZ;Zl@ZNqJ$ZBu|(pB%K> zF$C`Y{0GvWUBeGKl-wdcL#WTpf|4s|6RWlIaLrtCsJUAjMmouok>X}S$M*cQN7Key z4+U+vk^pK@v2P)Kx7!H1sx2m!Hf^BR_7)gTeh#XqUcpYM6iM1<6~f>D2%k805!@Xt zgq=mj(90$X`nD#Ms9rr%uo{rRBYLE1>s+YAF}UU=i$k6DqEMQvl{%g^B6oKL6VE?( z#N)mvlzE#A8*w3gcRLAwASq<&95qtcZbW_*&n2ym+E6Xon*{#}CF|EOC2@Q0;aFN8 zDCrT0GftiW?J0Y4uS*l&!VSiKx$TPeJ$=Ca)IIpwvSQrXeH@QWxCLG7-2NDGuG)iXZ*HhI6a_1FzOR6g(UsiKZlD$El7E)77^IXkin&s z;OevXg5_;0f(Ol^IL`MkaB*=ESjg`d`h-7R z`E4!Pw<-?kud?R3s$AqBBqe<41SXlNe2|6|`h0!_g1f7roXZD8Um^v-I zwJQfm%#6dv?}sg;w=j5T(guz#Q%STpA0XHA^N0d1f&-g2lP4!KNnqi6(qt6|d96|8 zx@|6LUAuua*lmQjb=}C64li=&^kQ;bAHp|=~&;9|56^d6f6pLEP8L9H{%rNJ4*JZl;WP?3SL z-cmd{X^!huQw0uWUc@t9rxOvytN4~&6P}uC3EEyM62)uxxU<6xY;*EB2=fzzo(~OR z%ZLQrEGl2%P~;`iE=6lW}i^Wk!6elrQ$%#Vb_4awxs zla0_mCkA$EwO@Dk#WV(1^cJ`3V%gX27n$ zAHm^wa>Uud9+uiHfpgm-%)JST)f{pBXZLn|+oTj{7z;q=1X~iBvz#P$8IjN_@^JO6 zF&zEm1ExwfSoT~bSky68h`#xJVxhyp@1xGd`p#;ST4F|aiY`jF*f9XCrY_JW!^iC(U6K$a5p(Rk{4}w`|7m=oxF|0cvPJR~15;GlD_{B~Ms{WOO z3NoV5*7`9%x2qc1TATw~((Pb6ZNpYvhtWgE1JB=f0AJ)dbk%FQ{kYQz2JSJ1a*ZaW zYs44MUgiQDTkK%_hvg(^P#MSXuEcfRl+{j21<1{fz}+~Xlf^ox(-eRVhR zneJXZY<3-NT%C$n9_zs&PjBGsM;`$6#|jQ_!^HI6(4sCJ`kdQJB;KuquV<};a{}zRNX?$q=4z2!A(IK;VHUC9&x6|i8RYla zGIA^4kaTPs1$V^_L955-w(lLcc->yIg8A`5AmQ8+@J<%vWN;LE1xC%a`AhOHGnXEK3B?H_$Y+a~{V1CjDUg%P3V0Z5c{&IZ{ao@%#i_kJse8m~& z8HmFB`n|y9;AbEgej3lx2*gfZdj)r1b_on8zvV4|tO^4%Rp6M5IovqOhdWcxB{3>V z9Ak7fF&hkkmD3!F#WzE8{){qNuxm1`{x}MlGgpAZiw7Y6^E3SV?n+n_?E<;K4cz>A zDN#$0wW>Tm35P5U#i`j5Am_yaJfJ4TFQl5W*Q@WK!j<}F_F^BH(m z91Cg>JAeh&Uj-%SGvLIBS#Zs%4RF58HnL%hA+eaKO7x#8ldI^3nlM^o(wO+!M(=sFm90*^1yi>>>19OnxmkhaIuLuz!UqeDT15kl|S9 zU!4Hg*7(D`ubl72b_-8Ip<57Z;sWg6-@u<5T!`n^B}DS2Hpy?{SkXTwLFjR#}6fdxB-WGw; zYz=7Rrw=uU)S&61>4amMfP(^C&>Oy+r=i#t&AM2iL_&YKsiaJLCr&s$AiX zxhvolW(7H#bP-o`?}Lo|Uaal*4x9QN7f zvg)zfoMBuemxHHqVz0S8Ych3{1Bp1NMJAk`3E69liS-Rcj?7;`goAV8iy=)clf4ov z$y~xaDy87P6aJ*}eiEs-@h6*;qTzKKHX#~G{MlE}NJ5o`H!$tzHmd>a@H^nl$fZeYzlJ>UV; z4PHCFnM3wiQVgQ%af@b2j3Iq_Zxwgy9*${-jH3up2Dgx!4-EBLDzTx<_Kk!h*Z7h0R42q@2ko>+Fl4s&gTtj_fRfG@( z{jLB9%Wi<~nKjr@4v@I*49V6vBrYFKVDFSX@bmox(5jsRJZx<77uO~5$RaNoG|Ld$ zy;(pm+Nu!wL1l6(UxCDNb(KyVN%&9r4ZM_6fuAZYh;++Y@N)e{U~9nfHP#=(JsCls1!nE2CjiKZK*JouKI11@NSuEjcO~LTq06knRc_vZ;76{1uQu zZp7>$mxo=+Eq`Y?zN3uT-KikA#4|}`ZUJ1+aXAChn!%3NT+khsgVTa6VPUfs^vj(C z+lEZZm`)#9XEO|jrA~sv;5#@=sfp)pUI>ch#&|K|wpb^#41Zg*8MiGwg>}phgH~r7 zXm33YJ{B{9SCnnZnks={r^s@_RMIKRAdDl45ZzQLsERXFVOUP|K4-ATuJb26xI597ULh{fc;cx*tPxb9JfjEe=96Pb%Q zE_o?%SpG~PaoI=E&~*VP=jGukw;Dm3tOmjRBzb!8ALI3VMnQ&@Joa^(iA9g(3;sJf z#$(;8!M542L7q_#Shr9EyVlPmGavuN(mHaa{HGk0VX;+D-(m1hH5Fv-*o#TnB51zU z5n7*}3$+qw5S6K_u-w@lUhC9=ZTl9IWl2ut>7PjQbGtQpVZp#IvYEWTv6T$wt|8?W z4$%CL6`cRq6~25j2O8cnCKI0B#0%V7@w&D|Jaa<^P!>)l+viCT{sL*T*X%EN888N# zcMNQ*Hy#6ft-}RfL@UZGoR8`-20Uo7+T^BF-QG7L8|}VzIR3H0VWr zC5LD2Crv@QBqzB5W<5zFZoVD_yyB6Y(`;bSN4~(xHUs;)9KqYAo50eTO{8zWE17q5 z4KaHj0`-2fZP_UtYv{yuLECjTfl72dneQJ;4h$uc1*(}aChsap9R3LY4)ubb(>0hk zIGH@}pG5wbE@}+yN3~t3l@d6W}vkjiX^6`SyJe(d5ODcSYNw zR=`#KB(4glum~{ZZt%jk0igXLHDa4d{W{Al(U@Ls&<_{B1Hj)N^KN34Lh^!*3iRUkWxWY&m_uk6_E3b6({+p^L zFcVP%YeZYIrRjd$71#oJU-G~*^)}!Vd>)inXW^WZ``FC4A8Q?djjd!}0OgIzWS?Xv z`S@r(i8>SkjdI<%n(i`k&v7YfKIj1d3&T6QOmn8B{Y$1i4{Z;A2u) z8*gJ4$XZs)G3y3_ox)$Nlr0bSKF@>4d?jIPfGAm9^_BO_{~B-jcrp05ts3jnN}RmU z6QA}vip7i80-@n*(jByhtSnhh0t#KA72^Siw&}neDJyv92ajC5CBV8)3$V)5r?~a- z6>uaxlW13@lZS(wNQ=%EsPrM8$`KKChnh20vf=5r;QVhF^X)0g5}owJ7-}{|wll*o*znX5b^YuVHt`b3k_UA(Gy`pHvkUkp!=OuzYp8Ks7`QxBREV zi#bu>R$Zz-We_XbacW!SH>1e;C2&j|vW zLFu|4@bN!w=y6aCHXoKDZXTgTB+Q9uhdGi-^E}|5HScg~$W^SB)rUoLy1=3}+n{>I z4){CM3+^2cBpVjAfj7U_gMZ1ToL}hxmfGhFpFMJcHLFeGvnMv>*XA-19V-d_)>VV$ zQ5Z{ZIgWc}pW*Zhj$yd;o?!d&Vsi9K1vy`uM+OG>Lemo$@#9Y?@K1#@Tp@QBwCrIB z@LfjEoirn7>J8wZeLui*)tONAgf{%KSC7p9Pyj2A7Qv};@i3z#nJhc}0k968pt1KC z0CF$!<}EKlQmO>>n)V&cz9mN%ExwMG!={m|N3LSuKYM`ffixft3E;4O3k2e?9l@cN zB)pu6Vxi1&!K}r4Je50fq&GR1oY!?Fi*LKaB`V=$iO)vzx5S48%v%W$JUfHu4_(AZ zzvsqRU!MPEh1ku<;irLF0lDqJXb5* z3NL0yL(8HaMDq3w9IbUza29dn1I7MW;YR?8=!+zehgK4M{XnPPYfeo?SrJsie~1i@$N+?nZs8}#Si zgC9IS;l*eE1fOvtNnAJ8^?jz~^Vj*X{YM-bx*SV(BzP0UL0>p}+ZeuDFcUiWGBC8% zk%S``*!p5UeB-a=BaP(_cV;>r>ndR^l_9&)|}cEB2Et;2o)G0N;;l z!)==SaMg=tWWg5)^8U;s^7rWiqBDmFXNtHGr>TDA?pqroC&guniyjJs*JNQC`2?)% zl>-(nKM2Aq&4AIIXmGN_1X~RR!ZQ=r!jqy7Fkaq?WL;QK_NgTiUzOEF|E34L$|1(S zT=W3ncex8J-mM0CCR@mpk8vdNoFAzv4~F&~UwD6N%E0#~j;WT^DR}-!nM|B&K@{wD z$h{47;V&mGu-oV`*zoc~TYXrkfY;+s=u3u7t+pmJ51PWS(pvCtmL?q3m<3IGwaHzj zM|kOx^?2NORN%DaGEZvhY~ms6K_YIM6T>ibIP`A`&#FhC_f!0cU?8xLr&#U^WpmfT zo?aVRIKzwMnC}BkzumyV=)1P;>N8fnRqJ8Xcrdh7@q|tNtBJk&MyTtw0ZuSl0~;QA zl7k(FpyuyQU@6)S-2ZVISO6cmHJt|3`WEOvbOkTy^TWsYycHxl&JyTtXl`pyQ6#w@ zN~FkDi46QwhKJN#fY>40+9<6FI=9;5$nn*1e~KT}eY+TjpLQgAuUiBnVOdz$W;J$O ze;sHB2SR!)2L6%qfkB~Mt-eT`s9wK``#!U{=34=%(3uZYSImQTcXXk+@*MJ-|A%K- zAOU_Zx(Bq>CKH3;uL7?wjQ2-J;p0*DpzuZ-jGL7QMeR00yksZ&UCv-n*+Sm6aTOf) zRsqcYEDlwMK7un788x#t}v`MpFB=4Api8jiCgDZE(f@ZCA3p;&)ehp zfbC{r`_>#5&NGB@<@zvewJNEJ`-E>5-ovT~Hf44F>e^pwb(0xV?1}iL(gD z27fW$(z6$@FkA_2ro_W1lQ%=jq$Tjv6d%IwIt=`Zm$vr~9JIaZ#bw{#Q2zT`(w9BP6V*`z!=2IKNs|@kK=veBUX~25*C#Us#&E*q zDX{x^H|TR~1^#j#IH;=(6p#D|raW%|Cz@(FiDW)(QP~ZZTaw`G52<9QQaj#Ia{y0N zzKqWuX#h6e2{7GmJ1jcr2J=PT$?eIAToHxjoxTBq#(I$Fxe5%57lEXaT5#+7S=?Qa zN=&xpk@)%uQsBwe9fgv3irgD~x@a;{3Fp?KKnqUto(&rwy#z|yV|c7Oodmq~BO$

    Y@yQ8N;8bBS ze5(@x-+r@!e@-nWW4pA7M9U=dQ1LJBmKg%kenxm;-(}n|doAa<^9C2VIU6I5bTZ@` zNKD>@!iuNjfbY47r)G1Xrxg89@T%KM(7EH3p!UEqj%nS);~iT`N<;!lR*Ef2n(G3u zot%uX7@TOkX!w}N*6tJdZ2Ab2j_bl6B{4W|FGaF{{U_Ld%@H5DS;_m5F;h^Z7=tgR zbDx$boZ@*39N>A9)2Z%6=8PfN8^L8hIa7%H%{O>^_8+`2atN504T74C_h3)zAkdc< zA?Z@-f|cDlycJq)g1)poLHk%B@mkezz=PW5o?v;rWAT_ac` zcCzhfX({i{NegVCZ3QmApGmsPL`a3EHcYV&gP$YfA<UIW9C1E6~QEnKnN8)`WP!A&BLu#@Y1nYAIDXKs0xxAV(J z-krL5L2bxFvY^a{M87vBr5o%xcg1a>npqDvB$NvVpUnWdKI`B%KL^->jNxnNStRxO zPM+6{aNe4GB3M#-gy$$V6V~srgK1hC&{%y2vC&WcZ_-l&NfOTn)}LJg zi#RXAd@Vyb8m~@HS=p0sj`rl?MLp7Kt_u?sH1URkE%<|3BTmTA1XercNs97pvj2z} zd3Dzi4s5g}+u9+~5So&ZaXvJ6{t7OY^@GL$320j*OB(L`lG9cpo{xqFzmJ1@S6MbN*P8~?&_@3nKb-~8(4|wo; zD?T`CgF9y=;q12%)UL85E(T_#Cdr;KFIGUXsDkA150ixG0%AV506Mtl3;g#>;#(_6 z1ktAYmJUx3laDu#l5+?5kihB^_`)p>*1X>c_teJ2=Vv0w-UUB#@0PzYf?(I50J2nZ1N>1C4(}wo!L@sQ$d@7;a%u^ms7RQR^#@Z3AXy@4?s3Vc^I0Co~52VAtMH!19PB z+^!`D7k-(}{Vn5wbnGRNu(}qgEwI4VUAqWVu#+U)hLfCmo8TND9?8<5PC_QAk{|cg z;ct6)C@Bbla6f_-PRogyK`Mc_3J6&nPugDvLrJ>=(7!ec96wtJY|~5ev((u{yjY6F zwHgug878o8ESt>8$s^IuTgiBgFPyabJr?PjK%$Do$?|qJsO@h?R=a7F&$|st^fZ0w zd1V3FbX1cRPEzISR7!B_)n1T#ekL@Z^&7lunm~--=i!A4LHL@!HNGON3H&uLfO*r} z!F$WwKH2K)sc|>-`PhK}TOvvNf97E5l?bHEbfKGuK0H}J4Myci zldQnkV69OJ@HQLa{5St_@_awy+8RqrvjEw?RvmWjb0?R^!$=ocOsov~Fh(wk44qFV zPjtEG{A3g?=F}1X&s1Ti!ZdjF;SX%|%$|pL?E_+Q&Omi+7k0X0M@o*aBL`yVlTmXX zRC7s(2XPuSN{E2q1t^<9x7rZQa34+)Cz^&q)bSgiT zj(Y1;iN;5?(8Qi8Vjs}cSDKM;sB!!DoPUgOZ7sPY69rwh z=ZilZL&Nj^$<$@LS-X=m^hK!*HG5aeca1oK3MTBL4FV}Deo&VhT$zY&+GsGRwad`9 zS2tNy8jQwtKElB9iO7Gr4Q-03qLVX@utqY+(61Gubm))`)n0UpK20lNw@PnewB$q4 z;kB<(qRm-Wv+@o*BQ6e^&y8c(RH&gzc06mLbUI@k_J&Cj-a}V-XLkad$|#NM*4#SagY=Z*ZdyAM9wrvmoSAslxE~ zR?}M=N7+%YXKaJbap6dIIrGRY6g?OZMo%wGpiM2A^!*%J+9?x5qc$-7rFtneLB<^2 z8ntA~;=R%OnjOsaxBF@E`&Uf!hI!1`WeavZU==F-mdy-|lu@bg_UK^{Hy(E5HL4lg zz*s6Px1S`H)FO6>jd<6`j-JyNEIT;Gde_-pTKGo-<&W-X$6h~WtM2V!kG+1woYh=G zlZ>U%2C|jr+xfE9i!vD-{#z;~xt-bEtjp+p)kUrW_2`LEhhFP!Wi~aLvD(M{*=_P| zboI?DMt`#r6P)&jIkQe6>~;7e3^}HZOxN)k4|`pXUvw26-$20P_G?t(sS}F-*~9c= zEA;K!J2XQ(iVZ+lg=%uDz%ZkV=B9HM&AjQ>DM!!I3yrnZ=F()k-$#MoecZu1R@8Al z_)FQvVg~4lK|YOf)}Rl!&m&I$QtiLD`qI#VTqF^AoyKK5bD&%k`fi;!9kJSnN`8Cu z-+sD+o=e3Ghwf*vt5>!07u=nXK5V_u&+N5e&F?;ECVR#)?w&J*ol81UH0QK_#jir+ z=P%JM@y~_X;!Ek>OF`)4s}UO0lS2QRo?rwDe6;PV5_)?wioQ%fgtA1nP*`>yY26h~ z*R7k}o_Tu9khQ)i(71tzo%Y&D6q{4?v zA0K6a)2PCxHOz>75Yw;u1PzbYQsX~d5Aoa$Y}@<2boRD7 zWXjFg=!Ry{UUxB?_Q?a)9m!{`n}0BCAVi-=Ptn8En$gU+)2QOTR+O#3kkXQ7mj7Of z7RIE2Qfs(%T3SmuM6=Or}wryknUr`{fYE0I+%SW8{9ix{0J#q5L!671U$ zcXrrB9F^wo=G!&CV;r|mYgfLqheafT**|9qu;sC3%y>N~Vj6NHL38T4iS>rmxMK5jJYrdPy?;f94=Y^|@3 zL;sXw4dtsaI|)O%dlzv2nWeD1I1WF(DMGdke4x+j_tFg-x#Ufe$7JPuK#mtttf}X} z>7PBn>C=tJ@Vwu;kl9?zy6F@Tzxz7F2nQz|pXN`W5AcS0np#kC`C0ghb(UUnJrWvd zULo}@>~Vp%Dw+M^JLb?j_~+XQd`2W23jdXXy%RD}-7S)C_Lt>Ltd)mWjxQSzRn8%h&~0ajhC`To>xJxTQeQvAPg60 z8bXe*8(HpELECUSbUb+m%J}%ewx|=B+UgG{N@B?m$O~so1>?>u$+)Lw1pd)(p+}Y} z!vg~L>E_%0^s>+(Qclm3zq+Fns_n7Bj8BL0aK0G-naL$G_*EUP{V0L?it7zj|Hrylb0wd@CBwN;PCbh$+@wv7+uw>DF_$1vH249FJ*Ea8j z<+|tKiQD(MU(eZ-b{!<{HK@XZq%Q2fKZL&h(+A%nSMqB96yl?8@vy`BIVr-P!tpKo z*yO({T9DDjcd(RTugMI@j!NeEzM^1u*; zATNX-J;DoEkZ z6>a3fpO;8E4U9YPZH7X+Qm|~Z82jFVU$kdoCXB7QOpfXQ;0>0@(0Rfa;gP3>f^Vpl zw5EIU@vCoX39VVWFfNlmmE6gD7PgZWnk|fLmpMRx$pF})6Nw*uUVuXrFDVNx>c zJ}FxjK^~303{U-+Mpr3b!kRnl@jst%`np&h?O|R`GKykIpQ$gr3xy;2$(GCi$~>0X#OQDn9^_+b96dzt>I!^ zQ&a{AWj^Bqt|5JLuPi)c`U8IMgm}#E2%g&M2IIKDcyfax@Iit*mR_OH|FuaQ|N6av ze>6K2-nEZp9lL7=Pc@BW`J0=uZp$L7Syf!8jz!J@X!tnZim$UA1Mc^`_);km17aHQZLDf9Owth;xcZb{B3 zZOnhr%|lkWPw^d3t6+#8S4qODV!}{J<2XLnp$NYhpN3%v(%_cFK`5WORxr1E6`xrr z%vFEe0t@_Vv4-$!ye3oxGj27oGVcU(UA(`L3%mlzw?hwktIKnsAHl@4!IeC&k0mrt zE#>wq*YP}lY=nM3+VHy{g>R+5gnQdG`3*;oQ@^g{LTAk}+U~!7JFOo~o>cPwL0XUI9z)idr7t{qa7nVgCZAum37wn9YY_@q!r~s}_98r;nw{ z7UvJj<OwKlE4l;pJz)>2B?}oBG1xHEt^Qek}wW&uqkgx{u-Zt^~vTIc+?1`FEsmaV_Z;b)6OpN{8o?lJJGG zE_gI*C%)}=0e(*K!;8K@Cndfo!}64AypRsTcONS7_m26(obGhGb4@GRv>d_YU#={Z z>}%vNXJhQ#y#@|G-if^@p5VJ5jrb)B`8dAg1@1jzMye4fu=!;Pvc;*Fc9^KdBH1Z4 zY`smYM%co&#@|RgpCWo|&o?q1OpzaaFx(b!6WU!4gKwXNU`;1~_H^YCjd~Zt!w+Qm zIXP)KP{43e-W&i`XOi*YdM#*uVUp%r8q>B86r*jb;RVFjt)Z zbp@ujapCT|8d7Vv8vk6J0gr`V#wS;P;JW7@VDe0+;0~WQ($Q-l&NI~_eP8lPXHF99 z+Q0*Hu-*^9UmMF?ydeymU8Y&yMQIRGo80d@V~~1U88?gKZ6eP z|KUKx2yzZ74kJfT;&pE!+~OyLv-aPHrK9H1s7jH}^iqcknPG7CXaQ{&}uda9<}^Nn&q+aX!io>wj9uRGndm_`Iq6D zEI$4(LV+#1x1X#VHz98Y_(8SzcLdzgQo+o07L-(yW?P$Qk=d&rLv680TI0hG7;i4% zUvcKp4?ncBJbnI>ky9}g|NJ%RsP>19E8B?Q#>^!-)9dkp+D=}Q?X%{-uS;O7gD-A) zr^_ai2`dvk=!S!dP2nGT?_x4AyC?o;DC;kcKl!aL&RHr2TOj{*p6tSn{z0 zwC#IC?``;i>r?x&duajQ>RmuSOo)Z7PfOwSuX%jY4FIOhuEeh%WkTgs$>e2b8MJBA zBG)WB!~1$63};tnv(~V7V0eBTPIuix%O3p*|L6H_E9Ub8uJQ$S&x7VWA2b2-JW8Lyw#W!q|cu@5drPqZ-oI(G(N0R zj3=~Sl8P4Hki%2QzwVYo5o~^2OhznR3gZkcVt-b z6L<%gu~hjojBvcvD-AALau6m;Ucj#JV`+ca8gBF5IZVg5^KtOUb^JjplZ*+!!NPa! zp+VLXcEb2W>}j7!uX5ZExBu=STVJcfU<(xpLv66H1xZ_eJ5G8(9)xq&Q7nh1+wcTw z3aib6p`@x4TxvT9>pHf;3fFMhR_F|~?hVpcZ6HiK|6RcSkHST86=}Q~;_FkB5~k!V#Th%o;mKGzT*}CziPA7=CVB^N`c{VrbwZ(v?nS(L-2r^@ zem>4}*h1EadSm_jH*tL95jgm=fQ&1bCzttr=Wz~p!M+pyuzT1H|NXcCbz0xXj|}(X zBlUV%f-c53)84e%(h6Q=$th@bdK=wUR!Rox+{JBkXL$T@b(j{EkA+A*yh&HU3wSMw zi>(^*Rs(1H*JvEPtVYsxPhR1o3XFB0S;6IFT&QzJi@jvhoj!-$S(1@;SaIW5*3wfK z>1UdLFuHLD&(v4rC&x1A8)YqYvP2)9rk5_rZIFi#l$Kz=fKToeU<5q`_1b6(1?#6G z;mzUSSV2A$%b)Fss?G5{0QyMRkIVQss3=;dW;f68a|o8NHN~5cmzhe`_Q_%Fe zB;O$7k3y4zQX?>?f2)3=Ia`TC7G{AnS}c36EekH`5ACylnzY45DuTKE8v!W>hR2Lt6;`^l)T;OO~{5`C&9@U=uk`IcYjqd zsxl>0C+^UPZ8fnk*5E}6SX~v@oS@cRLt5tMT=v<_z4+^kz1a1DJpZs>J=`BwfqPEL z;pD^{WQyKtdMH30JNld9Rf?%lWnm;1_D)3ap(vix$t3?+&!M%qUnV~)r;vr)c97GZ z`>^UdUD`@-EvyUc#YZ=+Q;$>CbfVU=qEly#XGxp|Iw!M3^hq$-BA_VtU*O-@B(s ztLDBT-!-11H-0%m=4{WWgZ4|q0I6c$#;Hu4RG&wd$@1xWpSoeb$71?!gd~3Da|lY; zCDU6Z1GvVI+ewB+IdnL+45ytOChLC2(;P!_C_D58nmCo?6 zN?7xt0(;FU@juRWAfMoIA2v=jhvS+o-n)EPyr3>DdjL`2L?t0lzN=I<|Y^*r-ML29g~i@OIua=;(O@qru~_7EEKw?IMZkVlT|(Jm{l!*_jxeg|Cvlljm&XuIPB{k64`=l%CEwEb8BAMFbz12rJKQEd^wYHlEW zXQxHRd)C02jgU+{YJm$*U19RR-Q<0a-UgMpUvN*e3TzeG2c4|MS#@HUaNge(TsY?% z-tWJUY<3uAIaVc;_jWkqtEU&>_hPniO0kl3I;H`Wggtq?1s+bYT^xBy%!s^i;EyG; z9O+e)6|h&!8>bg5v$UU=kz;$NX`{D0@Iljuc!RPiete&T%hQGN>H-nI(Bc%Hc!?Ay z4R4W4^vYn|9Vu*7!)Erp^us?r*I=hEQRoq&jCWU?VHX2^az{fyow|XIpDc2R2anH# z50?shW7pN;4}Gz82FsMZcj+N^33q}03MsJl_ySzkzK(StDZ+UN0^w7?T*wXoNPcD5 z!1W#(_{2hC{yD#B*zS@}Uz!ZW7Ig@Uw2NYKXBB8W;)Hv56!JpKFT>w+^SB?UW8h*Z zSNKeNf@kmX9~82^LZ`G?K$_DBbysZ0JHAKZg$B3azQ!_mG%yoR+ar9TLK!}eJO{@N zOz2VlL3*9uQOu5iN6QQ+!u*$MSm(I{JQco#PL93^L!OM&@v=9`If2q}!QFD~@^~0u zH59O=@1BM8&S+taYxii)gYyBzOVvpYSO7nj#&&Qe6N0xp45QD@)%o0p}RFvpRkh z(6yEh@V)ab)@%hKV3+^K*SZAdj`~k>NBnv`6-vNXzrV1gLju~IevgkFYQ?X_8@c%c zPPz9sHV!y(k+r8%51txd1}()Cu$A!)Ed2cvtNwE5t@I6r!vtH9FauzVN((HP2xmPs zB0PjYCv>>&K( zX)1o?yb200xzAlbCji^ZDX>k?=dzIJBWxhxiW({3Cl9{3k9ox*IIheWT9oaCgPmJ= z=RU0<57Iwar^6=k=doe>eZ^kPQT@xB)J~=eb1v{1T*hN%_CfX)OQ_Z9f>~Q`LX+$5 zI8u5)((_a}YYYuo!Nz2i&>_-6z=Nt3}GtP|QsVcP-9lSoFp}mgu?@o9eBDU2Cp;IfHixf==%xzv}wd~_%CoV_mJ8My-}^6 z*50gzJ9?F9HJ4LlPux!4@?Yu#=HNf>=ahVApf`;-KaOW#(ft`u@$!%JNJst~xZ`g+`Rma` z*6%DuxH)H*POyxjOT;7SzjKaaKbf=SREs}8yIWSEBOZ|3tdVEe$!BNMRvvk>o% zS|sVvyb60X9LH~$hS4^fO6+H>Y&@sU3!l1m0&2a#Nt)gNOIAK}#!ptyg9{h3*cCO+ z7`@wsMIE2g)^?G+smTIXh{G5?GjtJecij#UU$G3ER^77w-)R# z9R@G-64+NVjx?)kz#H97VZo3;bgYlXqI1XTL#7EZ^!5vK;rgHCYne!rQVFB~O*qkC z54n@#fpSoPjwn7WdLEAZm%)T34DJax7kK6TdlEDDlF<{&>`z0r_|xbZsV=2NzPWdY z{;e=hpZM<=Y5CR)PIdmF_XYhXH`;%rPuPE9?Rr9zEw+Vl%eWv%!P;T^&|{XG)bx!2&?Tl4s8uFvSO`DdV-&msZG&JbFA=fK}7@#H-5 zg}CiX9OlG0;E>Zw_-~^zc|t~-{w!d=PrhrX#ryBjf~5m@Ft1@kHW1Enl!2_#47#H6 zHLd%n2Uf56gDpBLpmBKveHh1($Jf=6cdY*6;Xf5*q*^k~E1e~MmMvsEWbejRzt%vX zqp|Q=%`;lX=pFs#@EEy|NQT{e&7uCQd_4YBjU21>g3>}aXf@FR7?%UF)JP&uFf0uowC)}?SKW49!Ve490(wuyFq}+<`l{%yGJena94WWBGxw|W$u5Go$=A9xm0Yvx(N7_j0zsL0Px+7%XV=*jk-#}`7 zyG!Z>KO@IERj?;74o>PW#aMSadp!^~@N3|CsRY)-ut-RMb|ojL z&yyLdiR9IHndIV)ujz52oA}!EDqOu=i{*F37lxhamEr;o4|{uGQ8~%`~~ap5!;x}dN*N{T~wX~w6YHOi$jsaa}dk_ceYT<;_ zN_4Ju4X$U`@Uja;v1P?o=;36EcU8}W8DICn+_DSMh-*NW^rwI{u2o8#GByGz^gPB+6zT<8|b%7Qee{e6F5ON z9PT`tjStyYz_a1A@L>09>?`(|mNmVGFBFH6f}Tg*oxy{ltTcS~V;yaNumGzq7Uc`| zh{DsqqF9}bM0oUEBTUL%No#)kNsrFUBbl-rVABpUcyXl)e8^dcyLnM?Y_BwT#YF|& zR`LUr>J}MQ#JlS+?TH%d_KYbxJgu>yfwYPbP1l(0off=+$yb?OB8Aa<|=i(n< z593`9NpyqlRcQQ42|5);;e^@{_>dn8L!WLS*@v&=WL5%vq^nB4c=49JAlru>Rxq%L zl^Hg2djPu&o-*Iu%H{d**C9=;KhtFe4!Fyw5nEq+NlFx5rw8}lpl`ljj4d^6X}PzB z@Xc5>tk`i~km*(dO+3HRS1Ap)xL-EQM35UNb~uYZyJ#~GYE!^s`aW1&aF=`S_jc%V zx`*U7hLL+!1uV}^XJA!M7`=5#1ollxgrd8mad@FHYReZx@kV!$?YyH1B;7~sdDUR^ zX(h1Q-3djXg@mcO6kt3&Oe|^hr^2!xpd)v!DTSm;V3TwV#27VFc6Pt0d^?U{^5rrR z_t?OsMc1N*p%(E+*&|lPBJ)*@a;F9D{rTHMi-vEYVI6qq^Rf+FV6V`in)0L{Ov5aY@c(EW5S z_ry_KqSrbHoHlbnTej4r2!}>u>C+6rJsk*=NEPl?2QS9pTN32WY^OxZG=XkHJy0Jn zK~1y4Xxlq+kn1N$CH@vi|B4~Fey0V^-!(!^Z@UGKDbVODv4X2}{63>rX8;roXd(BC z3JR}X1@73+qE~8mXyLy#NYT@PxM=OlhTW5xhVoRS_E*DDnqBzV}KeI2Y-%@pbJ46jA>_Sq`5W; z2_0%d;$02s%Ly4~ot7^8c(@ka^M6L|xKf5}&uswr)g@5rpF|YnQ4Kt=MuQ#Olh87) z{N`JMgJAJ~SyXP@BVd$V1na*Qp+bf2#LB;SP|c5S!exU8QKoSQ>39A^<)VLy73Y{h zocg$afd)|t_-NYW=GBJ(nIpwD?vI~4eEFHa_-b;p`$T(IDxuZz)|Bh za2p5#%&`_utNcAm*F*~Ggtmfr71~JsWei$s)Cw-~Fw&2EL>y6229jN>sPeuv$1_w8 z9QrrQ@E(xkjuy92Aq7UD-%1Eg#5;i`PhCdG<^c4>+m5g)Pe9cRPost-&D7!9Rme$1 zl*tJ|Xpi=n=9Gj(AbR&KDtEI5n$I2@^~52nwRW z9SF|GGicL)${@S@6C?d-J2Dtw2i*3EfFj~OlKUdy#CeAUxtYyO@$?EtI8}}`yUGD` z{0hT4^be&anAtKON&qb_Jj76xK;O>^QTL_ZQ&B%wb6=R<0j}n`#7FD9DCr_ZeOf-C z?La$q&t)05FfI3%6SAKRDTy;tAe~o+dVb|0iB3Oo-1{%_<$*C!4LAkH$E(qmWJ&a0*%O)C zGJ(W?QS>yXAH^{;k=}MurYPNmb}e4Z^|27;5>{t{y0HkryQt7M*N7VbVpLlm1bl@55SP4^xyJtD+}YU-WD$4~B{H%o%~da`iN+7avrp$y z$@RI+6AgZ1i{PnfW#p;$eymOO}IR-WtRtb+{90tH9=SPf*sL z4zPKXI#c(mJeRapW->KsWc)&gixthlLS+uo93jQD_?`)ZZ7zZJ59+9VrD$;XD?)}J z?h;#A(V*$kI3n*nM6 zJG`GtzET2Sx$R^)UU`T#6^g-og$__yL?gLHufbo@ZRnG1#*NmqLfpi~MbxQxvWR%I zmBowRO|wc9!@9nPpNy^V~+J5XXqAaJmtEFn30sE}N;tD+m7Y-yJi81vG!-%NxSQPkMk9k~ZE3>+!6g+D9PL)mRGZQ>ane!{o5Dhx7QRs$Q zV8fXUS_v6${$w?%-@OGXvz{XPzDe*D3c;@rVW4!k zIXCZwC3jIr0eby3irShBxQ~nkbHe^LfPTtB?`uTCvIGoXv%=6x^b^3x8r*@a=G@lr zo4_x%CKT7X7-V*B18ek)KxeWKqpda__&=xuk}Bn3dhT|#$omdRl6wOBo^Pf0FGW=7 z#42XVqYl9{7lRzT0d#n@20WsY!0R=sR5Dr(_Dt;H#ysE3^-qYTWYyjgGfTC&+u37C zRUe>eiA-ewpJ08>Gvulr8bZRd8^IxF6N;^xLfK(qV3mML zXjF@FhchVj({!489Ug@4(Q}xdbP1J7V3aRTfgev55kDdxRK5#G?oH=`*}Xhs_uhOU zg;{!zUZjftBl-$LHBF-Acr{Lv`(7R(~;9sfpxf7nOik0iUt@N)|FG zNdY@ftAoBGEoSneT$H+Amg_!Q0?th(qRMCY!8zYLaFtRd)D8wxDIdB4um1+|J=her zMEyaIxe)AqB*YxvUkJSO#!#B`a;D+qRm`V)qNp|@9BlRYinfN(uBTe=`8G7g=1^$b01 zyg^LuRzME}zA${Z>;^i~Vc_S%UFh#+ZFGH`A!wDzV|;qM8z^)4Q+~4%NH+NrXoIEb zll>fYWu7jz>tQjqTX7Tk`RyXH+2{+B^S(s5EDK_=ufCz)B_spBHUg!)<|0R@8Y0Yc zl<@LekL-+L-GbqDZ~?mZbO|Lb_7AO( zQ03OE1|ZM+C)DzO3(ytyABbAnN4*=YVm!Gm!);nQpSykY1;GKyL;E(hGD6fJQOk-( zP#$#vg|3(6UKB7-Qukk_iW}-t<{S&~enNyP*}4gJ&uB8`Ha$e=WAf3O!?(bVZCs$` zy$5A{sz)!4Y>5B%W`chnvPfusBN5ns9IQF|ncC~#$uY=Yi+c9GM?;6A33y?IX!x8+ zL_IhRN-o|2-!k7)8`~Xe5p8|Cm7Q~*4=Sb6} z0nGR?C_Ps(ut=-{l%)AlGX*`!!YdWMEk8nWm$m$nvNt7;%jwPF*{WWOcg;5yO{^wC@T?Vplo>FK>W8qC{V3^tTKLhpM+n@hb+(6dF~P(szT;eiD)sBI}ktK@egRX;68&LR>m z7b-&|>CT{1_Ac6&n8mob?LUeaTZy(AzXhk`dl^HO4}nzfJZ8@?8Oq6ZBe=Y_5Cl8^ z1w5}*u#c;WCM43SNXZ3&yWtcgxv3i*GF#7VKYs~KHCv*$SFUqzTCh16}Ul>WQ zO(0$eKLotI<0xfkAt7mU1RO9q#QA(U$8Zh#ieNnaOvMVmgST_dDSM$YAfeL@HdTdD z1Ez}zo5w0(C{`YrIDaNQR!2sAlI6>e9hIp!<0jV7cx^qZTv7OMWOwAp42u{AQw1{|;E0o=Yv_ zY(V@TN6?tK4H>Q}r|<=B@VVF>98cEq-_wgDOE?W{~iAS^L!mC%dk38 z2s+ZXQ{2tJ81bt0sIUJBLZ|)_BOjxhg>5WRfp0sK#^Qv;aV=s#Gl3z;OD6iY0)V*U zE|e553Tm6~f|m{bMDF4k&ToyM4C&Hu#HM4*(e9r!g!;zYR63)Zvrc9{GW%x$w5GJd zKtBuQM)g%unO>dB!HAhBv5$}Ozb^* z0<`_;qLi)=QY#kf-N-O>NBC>1P;JqL6*A4A~HPNaUi1DtT#L?m?IBn&Ud zfw#BWlWU6g5<1DTh!NuMc97>54cmda8@dGA`vHgz+Qf`s zu7H+aT?FQR`Xo?&c_{1oZS-g0C8$GAsMtW8Yjb2AjGP=mp_Q9~s)ZU;lGRCdAIJqt zWG8r?@D`nOh>z77xJP?kY5Tj>&C$_6;QelH&e%e-S4Y0_L9;ufZ;r zf5@XboA|a+li63729`(VBE{8Hl+i&6rt>FV?(T2%n0IoQqoGoL?#b%k6#X)qsH*sY zs_F>v#Z?|16S-~Fb%75i#<{>IXa$$|SC#2h7KiqK)!~-Zh5?CO2b8m`hcYTj1zN%) zOk>_ZeVWHrdNmA8Yn3^EZY#N-=3HcJH7d~D zi_z-GtGF`PB$@qQgUD#^W>6j{#r*a!lUlp~GZAI{0_Cw~x&81W@%KRn@=pyUzJ$(U zZhJ4vZ29mJjT>+Ymp69I|6=vgzUNkGozESHjiCgyHpLq3zS~b3&~uQCPASq1^#j^x z+R&k0;plGtH2BZMjo@YqzOh}|s85zl?JZl)4F3C9F!!wtM#is#q@WHW=xR9S`#2CS zR4!#`@r=Qs`~hH)QpIr8mgjor3j^~TYUro^7;0Z9O)NgXfH0PIMNgB=snf3{m>i24 zaJKat8aoq*7W2iK_NlI5>|r_T02l-YY-c9Fv8Co#CUbiGB2nyOO|El9ELGT6N<2#I z20uG;(3uy#piOcbO`N`q9y)&nOSV^o@0DrD8LdMr1ofa&)aUa|qdj`7O zYmlCJ5_l^zhzb*5p)xahL<-Is$~6v&sx$$S2aafyk_FiJua;_3G6gPbIt;^d19W0t z6uNgK85jkgMQ#0;L44QY@}ok*r*&t*`TLQ?*#{EbhNn&7U3NcOsG$SSiAoBXl)^})=`9i7UXRW{ zdqlYv-)Op4HOr8dIE(Ck8OVuQfgFGC1wWZF=w4w9XV*r*J zhG+S}rC+A`k&h=J_VbO7?fr`8aV?n5J0GG9i@m7WeUj6(@jcpib_`A13s?_lZO~{( z7Fe=>fan^Si>R18=yiiS+7}&(R<+y*-#+dF&sKF`?en?w!ghf^kxnt)j4KB z^s@n>EWU>tvHOCepCqHY_x>Q*t&FpHXfwmUv$xq`z?eu};zl$d9z%v!Z@}i?_fhbA9UOYkv9#+#m;bB*Li*3qt{xGtwrnAg6Zu1}3R30j zwb01zy9 z?XH)B)S424eajDAs_G}?%`sVH-NBqz~epjj48_F0^dXXCxtC3H(pn0tuLczJ3rzLMZ{@v3L!q<8>8C z9M%+MKNy3l&0@^P^;^-lfJy4C|2^WR>l5NHHy?$Sm7}QTebn)C2aZivoxr=!0)uTX z;7_$GwNEHTz&%PuTfFkAmtx__zw8Z3rPFLsQcDd6v|Zt5jN$(!n)X~{>D7Qa^)nDcpnA)&Pt#{e{HluES$J~ zRt-e<7ol}*KVWM&MC{kgCOU=Mn$Np)ftp$juzNWlX_Ep6g7Ojp)UyCD7f?3znhAT9Y*Oprl|Ja3phC|x&`m!G+1De2)bYt68*0hXkBCg zI5Ei((cVhGjFc0fpNfI>b2q87R{*^)$%crF8NugV2$)ELy76Oky! zu&X)55(CAYXJA5I39UaiMYJ<1q!KFd`G#W9QI$E=@1wSq;AjTII;9|x%tZ;GgTSOp zJKzjTGQ_8ifTejpigT9-dnI{5eL)w;qvAH&RJ*@?+gC}Q_pV$vo|s*T%;SNYPkW|)$PR1lAB2U zcoWsB^q6Xpa|Ma}-Xb$rG!pZdKojT72$v&q;E~}EV33dxM6#x+xUt&iEf<`CPF60E z6L1#24)vjJj+4avBv0^CKMk?WtAO+d4dnGnyIHbgF_BXcg60;tAjbpgjEM1W6zOx9 zn9}kEz4ad8omVhOHSIN&X?qGfwq2FjEbvy_JLgm98609u z%CNvwQ$)vPI*`OI0-OxokHi-IK?kA`aZ|qd2CsiHu_(k0d=1Z`c1}4EW8N9S{pBZA zqY)1ld26HHDe8!2#$p!KegV^}%wSDp%A6V@ zuH4rHt6D`sJ5P|Qa=RWpcM?O(_zRJkaR`wj=xFl`Dq(C8ZKt$e}YsUuAFYq=uZHu8i$6Gl)&q@$>yNLAn zc!FSb0>D=rfU-#_SV2Xil$8Y0Go!)Oc2GeqPi1gZLmP-D&qpiY=76mq^@-{-fd)5U z4xZ0k;>gP>5y$J=fZM-hP$xbJzAmz-Bu%5K*J66i?m-#UC&&~@I_yi$cJ>n;`vvuP zbRCg9=Mk~ZGa5M-3)u7<>nX$F+nivW$+&m)F$fgeg688J)EjjSe#mwLvxrf&``!ym z=x830-!FvjukHe?6WByc^9#ngQ51@4dlP(M41{vxJzcS7CD6g9vgQOA&| zr5IWfx{8^@4F^>Ny6Df-A~a`pJ@5?(N3Ca>sO07%;2{@JMGdb>r}d<8I(@@C@bcJZD6uq)mwyrCr%lN(tF_LdaGqiG&g{*F0ww z(xMX4rclzNeNj>0e)sne%045!BI!+vCp_Hjw7t4&$n1_9eM zFwQEwZjkAfpUz6I`XlW9J;5rW$(&j7u9DZtF>w8U#52(^HZdiq&+$GgoaS_1Evr%6 zgBi0Z3toih*VXtRturJ*n@Z5VIGix{=^y}7gCZYTg8>Ocq z%sH=Sbvor4Pj&VQ-aj8{_RLTy(|4nhiAGy?OV&{~IZ4*)mx3o7GRK_BviV)3vs1Ee z&gB;eodI;W6r+Px*-%9Ph%zw zY+)Mv&$9dsWj21TIoq~nm#}R`1Jk&WVYN2?_{8}z`NzAd?$5_3^ zA&jAQAuChk!nR*K%-GG&V3#hLz|O4YvN$^Svco?gT4Zc-V{}6w^F}F!3C*RJu+krI zvv1e63qNU#g)?5isJ&GEgiWoEVHV|N@!XeMGF=A`uxH+iSdZkZmUd|~*}?5ocs~zE z2y-|MR*>Got6t^H7M*>^6aJ!&7tnH>J-{0w%nn)2yYnPmxc&GF;V4C5 zqQ@N*UUZ3L#yO@i1*3d~L&2AXc1P5hQGX_|b#*4}#orT|Q?5MTijDbc2h z?58+BJFixbZP*se7CWa3_xgP1{j9KHeFvtqHsRxWQO^u{hqmR{RGzpY99p)7H`+Lx zX%$>%E#Fag-ZL%(L47IDRpm8f-tfe7OGFv3(epTSz+Zz|9@Nc-Kl{yTaV4y8i4`*x zS-?{k91u1?SjxVTJHsru%w(+ZysN%8G>u76)@OreEMQL!q%uwVW~}qzA7OIm2BF!b zz#9Gi^LaP+gfNpP+w*p~wD6*uQiSy$>b#kaU)fiq!iDVzoS4diJf8hKE8!RO6HI1^ z5mPg>mf5SxF@=*2nPUC3Ldmyzj99*k*<~+dF@AiMrB|~nv;4(uUfVNuW}fd@rm3Ww z4Y_cG?R{#*OnCG_s28cnc22KkH6mSDJNmiBIa6lQRsm}jk;CPtjbVBx7c+(ZBZO+OfjxUXU-@mv%$ZJ zC#$uRcXQ$xMlIBqsrj1BTn*jKblop!t6SW8rk--lMSmk^?06-1(j*0;-<}LMSD3|{ zaWsH{;lq=B)yV2xh+|Lh zUR*o1%Zk}lw3!Y4{eVr<_F_^GFK2}3)-r8t(uMyj#|f>tbENj_7T%-5XN7wYYv#y;BYz-aqe@D3dV;pf+BwF#2~8Jo%|W^h+q?aHsd zJn5Ds%k?uu8R@O5wc_(d!XVw#%meG^yjMH+)*d;)3ZLfe`O~0#&Yv-W+2Cxp_h#<=*W-b!QEVe12) z-wQ`!LvcQ*ldjrJdN7D@~ZLvloT#e!jv& zQewHfH=mi1ey)}{-r!AoyO(Ekzl--|*qM3Sn9nP|vw$75u1HwCbQ|;RMXzwqM?F@f zv6>gRQHf`^cMGf6Wy5CK=CW0@0+`GWU}k5xu{l#OGQ6)Rge_>#ezU(TY+k&U@i0nY zqC&u%>v)x&GAf1_y+=hD7Z%4F zxGJ*XI+vL;W;D-H+K`R@k<8k}88b%?rm)Z3ve`3-(|E}fb6AZrABD3nEob_&Wtr)o8<~)-o4o!q6<*m_ zbDn$9R$-*mRJKv=o$%P1xwVfB-1YOlO_Dx9d^%3GT0 z$8%l4@YdYR5GEdN5oVVIBQa+b^K3>Po9f(9YkuQCZjI0IYV%=b5 zb~}w*-p>1$M&RpeJ`B``VvwSwxFsqCtqw`^?ZfI}mAL{b&yFHPZ$0Zc&y1CQQx59I z_=B@c6{d6ksol1|@JTP72pyC_&uy$|NcI9WdTvAJy}Ver^)@X2G!q{5UIx|7UJ@|% zKK@hQ1`AUs(t95HBsT3TQWpvF-gn-h{cs=5pIHii^V*5iG!%V*d7S8bi;3l{nW)xP z1)CLgF-7>6o_`(3Pw!eHYFaRY7Ps%mn({mvZM2=FRfoei`)iPFwh3Yl*7LUo%aeNF zDC%K-0pjG{VBetx7@S^;8_f?vg~Jmx(3*&vTcT)Yl8RvVJR7q6pon_j{!C+)q;Ol( zEBaYA1?T_0OQ)-iCvYPUHJr*I>CQeh-O))*lIK9CeJp*QeG-1q5jdZhfTyBduzO+x zu#z+Af#H+FpcRt1?wcECyN{(V=g*Q|Obj)!6JTE1edK@dgg4Jl;oOl|>h`uQgPxoq z(3=SSo%3bI{r{q&C#WBkee9st&{I;Sh7S>`=p=c@w454X^?&KfLKIspf7O~$Uqe6k#UprXe>{Ap?^ z=q&6MEw!9cH#aMjtl8_1UVRe;W>wEgm4Nd$jP({QCz~Pc?Fmx!#T(vl_NJ=eRmGbs z%b_J=CcHXg3QsEBNW82&2u|d}Dqae{&n(8KMSC%+Arf3!C9&npVmfO}IJ8NO#i4z( z@Z@_{ft8;(`Qu{&$D-d-jfymMPb(m6OUFS(Ry2Rk?qyiJ<`PNT?1&P^d0=3>9;5Xx zkgC`6;s>i*;l0UX*s)4e!QXl5jAuoY1BVO!F~DDyy0=DXib3y=OHcohsgL!h~ zG0&Y&9c-ez9hyY>xd4lGQo*Gm1e`1Fq42{zIC3wV-ipqlKV{UxPFx2k8~4-NG##`p zdy1QK^{BK?iPf8XiG**}LzK*vut)hbM5Mihro9Rf_9Bb^3<*W^*u%Iz;Wfy%*Q30Z z3p!8I!IHIC-~uS&;^GdN&)-P%TP}%iyN83$bB?#EJqn}ZE|ImB8=>)GI8ITFwDjtqf>qZ~fse+kRRX8mn9SRQCQ&Qbc-wVeJf|~4+SAH2)Dx~4o zE*E$q(?boF$KtU79l^9oyRh?MIxNWJp+crEtoiwqT=`xu8p-*UnCw0r*Y}Wkyq$a5Mbo9F2dVUtiGs(=jo{XOT`0JW;BV?k$E~oYs?nFo4!dleH`+(+ z`tk~H=qaVj7w6ZV)5s+L5mutatv6}plWn4d@d8vFmkiGHMhR+CI%!?OJ}6R`w>kgN%Fkor z{f%xiQ%4`&!V5S?gC(ep{!3;Y-3G}@5%AzeB55>A7lrGx{B;j&G1JV6n5?e>nHjTb z!kP^HXO#lF&I+6#Bp=^>f5yN5GLbGAdjn^k>V>@zPlG?lrwiSliSO#XiNf48QBu+b zQSYrX(&rjMKV4spPx2mumU%xWpS(wR{u>Z!mHg#6pMxZT7K5}s%P*)t0dpD>;K|=v z*m=$wbvE6_@$%E~uggvN;8zW5smtNHpcF7N9fFD}=|3`=s$H!k-u<=sVoeU}9F`Ki zH!;FGVI^I%sF!}>*co=K^CJM%VBNw`5+~>a5Tmd`;&>QLKoJ{uYlCQBLsW> zm(n4vUE~XWNF_gYlM+`o!KR>RB;t`ZxwCga$cr>!yj+y%!^2J3T;NU8zYS2=-OE9Z z+LH+l<#1ltjiv-!;QTNDAj0K1-1k}rG9ypX!P!T_y3Gp)Msh5yk$cIF5-s#LS0R1f z>+xev6sc0Ifa7Wv^lsQjXn)uyI{ZNywdG#Y_eFA0VS1Yg{t{ZWDU`p2HWA~{67+i# zPosQ#Ak~yb-=*iFuDcYraQuwaKlfmLsT?sW)r9Q0Q~1E`2Kln^0~o%p6b0$6;+IHo zhW*BI_@%@EWuB@LkfozO@S)ej8ngyf zh*K|%=Y55Qbu7nE96wZL_jCR-kr1BF_Jme3f^JbRoe-RbME=R*FDIYeHonovu1xK*BocPH0F_|cL43t>NrTHADx$5ZgA zy99PfH}PjUQGTAfD!9n_Q?lo1bBay@_biaSzYzJPoV4HFG=dkudCC z#}|mLz;1R3lq?H^_LOSA@R>PUbKKz#NBm%u+c#2b*hSK=%hQKr{^IP@$BE}RKXiK3 ziI0cxK#^SlRPq)>@l7*a;J%0)H9Z3_EiYn>Zy!d7oP%#~c7bnKERnOzBHxodh}rKl z>ZNA}6J9k@j&DzLJYIv{Krd8qOp4TVFNt>DZINH|6)Lr{0!-#jfvrCRX!Id~)q^8& z=Y$}PUVMof7=I>u95ZRpv3Q!+>qUQOCgH7ZXE8#{7_(b+L|YxxVE^YUD zlS!Mi>55aS{9>nv#4#!ni@jE0f$SIzJ^U9O&K!pIavZZMzK?v+`DPUysf6F19}o)_ zIdt;(#TBypf~;{BxM3rOf7zcg`nrTTMbHTb6AK^`tI*}$8o0>y4u!WVRT#6B{3lFGE)RO!S zCb}NDam+?!Pv_GN=}smsv6CjWY$O3&jo_rk2GAM(gAA!(hxNtzgf}^cC{|6yyO#>7 zP~DaP`({;L>cnhO>CkD8B{58Mq;_GiUpg2D8N!2^)Uz#EOo`XQw*Va>`zhanK-zne3qOw*)Hm`)fZguSq6j6zu|_b zCXt&p4sNX*DQI=RMqe(9B2CS4bX1A2Xm{*nD6dZg-i#=+Xw6J$U2=@(%?pF&ocEPp zccPQ#%m@Et#}PXNsqU0$+E>#JmOE<59mpe2@Arephim*nE~jvpRFCMTM=GiQM@g)~ zIrP<1h8=$dkT5P1Vg;pCoSJT>r>+f8mllz;9y8$NGwylnuOWOJ#s_UPK%uq(B@(r9 zDzlCxxf~=O)s>>s!*BR~Ap_KRun@EOhD3g;0iH@^V8ZmpnD^BW#U0~t=+1XKa%DO? zE{Md*AsHf@}^@VEu0JUTsf>dRqyc^W`Nm z9(V$hYu>W$rcP)kj3b^F1@!2xCB$v$7VRD$rh6{>(Dr|$sJLr97>u=nYd!@;?nVGG zjRuf6cN@OEDvRT8?j%=aH=@Jn?cit{19>O!(DUMT_{IGf+WfhQhr*oEzIc@2-oSB` zx;{vgn`=1X-)qt^4~ii z^bK=NsBLbjE3b);Hwcm0Y6y82Rd_nk4{Jnw(bUo#W}3x_Mn*b9gv2KPlt+!|I@=si zYo3O4ohm@jX^11t)TqhHLbBdV4nz-5V`qIKJX4K?bAMNpX612MP~DB=9NxiopL#f; z^#dDnBJp9*4_Ne1OMGY82IcMhMaAulK%R4sN4R{#?&~ASjHesm(JYR`o^J|^Uf&^0 zESCeLagXj=AwUP6n>5*bh)9bRKt^>ZIF2g<8T~fWyI>;fa{S;~i^|B#2@}Zci!@6XxHbBpyITW?hHSNO$JNI-K_gKxlWU4ow^Ld2Mt)w?f4irlUQBQXNu`YriSMahk?siER2 z5?Gi*XYFg{cY7#`J{|T)yZCr)-sD7rLx0nyPXf^5zAf2x`Z)jLw-T^&K$zNJC=w|w z#Seu#0!^cXIITtiaeFfB^xbDs%lgY0=yee1ag4aW%0qZX`3#DzZo+Qum!j?*50aBy z2=)EP=@ssOU_4w*tyhhP%-;skyTu3h>!-oApJ||+*CEm$65zOHs^Zuk`F!1LV?e>s z4OAU9>8vycXUy@KSt#eX2P1`%AJj_b9e43nel6CFHKz2ne1! z0YWy96a1HSj-zcVh@Zt8;r$;LWY?-}(ifJCO0Ku)m;I-J>6#BlxAu^ogZ-psTLxZz znoV!!Y=`ykiy_lOKt=px{K5!h!D^0mxkUCq`lxW2sB1w(S1q0y5nQrK+-x4YCnLnU!}1`R0Dti&cj(% zH=viR4ZpQ_iTVajKxDzU>eAK$MUw)Iel?S5x{rbVCCT`u?JQAxkqSq39WeEJ3;zlK z9z<<>gNH1-Nb~2(m~ktK@@FVO@aSB0Ih0Q}>qX#{ssyNhrXardP6VT(kBe|wKTS%} zB}2Csk^YLU;8693I2n%OU;7@859eAF*T))y(pz~jdirR*IU|J*J5-Va%_{zcN0BtI zcQ-k_FcGo}LU76HR(Mn{iKitWp~=4p*qGqXzxQh*NjnjV&D!^9NM0FfaZm=&cnNXf z+R@@$3hNMp(xKd9C)Dr*uw?5jG+%9s7XzQd)m2|OO^@TC{<}+yPt5_h$y|>-`#2n$ z*^IUd?a;XY5sfr7LjJ>WvgUd`%qjmtPR<`KAPSRdO44}f9B+?BhRW2`(M90BJRX+j zh2t5uLCiCfAV=F;sfzr4;!pJjPGg7Yu!bEn9H-0HI|8mhFoIR5J+Rr`75xrz+E@NN zYInK-m(I|~17p9?0}`X?jXY^`f72t8th5P^DpA6~GOjLcjiHjaU(wa&-Sj`V>EKkY z57Rr{u=GGMdRd-eQYQVNb$w%C^YRlSnbu(b_%c~4InafwXH~?lV~fCiQ3FXGeIA}Q zj)Yg&Drr_v4E5Ftr`4yc$;GZ7YIF4ru6Z#Vdza0_nB(~(XRa}7aU6po`)sgd=@sNn z+Xx%;?tuTrQsUM$04e^>RJ-^*eIeS3YQ4XR>!MfGt>i!0b3k9D#N|We4kUo;`6k-Y z*a(j3iw1vdsIrL#b-p-LP?L~^+tdZ*qQxR|(8&+2hpW*=ts3{oO(m_`OTpGYl!V7^ z1)uf==vQncXTO+3c}xgOd3aElY3HFQ<~1DS?)a&#WyFrlvN;POM*+ z&R-&}1V7fZB&JfDG^eD4nkjeJo4$t%&S-pR$yj{fB8i6ko)GWnUbyJ@D|&dHB8<6d z0X-=^{NvD1ww_sFrJ61alX+#-T52*V4R*rth>4K?N)>N>i^jeMtr%c*p6cv#gV&0y z$U4n%zSZ?o(lY!Qo|x#+{N0-|aP$!K__@DGBFPA~-TO(E?P^RKF(2JaO$9|Vd%;*O zl?Vz-_zg$CgLM8V@LJ>nub<|LRHbro)C(??Mx%`WoF~BiEQaoGJ3&)z6S0S@bq|Fc zBQvt3#kGoskW$gW?z~`(!mP6*dFfIz&fOZThEIqdsXl_42J=yx%m<}9AJF-aL61Lc z@J8qZ8dR|ldBqbjtRNcIcb&x90uuuFRR!UVvc$}|4P_p#;$Jd+M<*#>7QHkwhP3Q* ztSyiO!S_?RY{@$6s!~foEN6*wLK!rr>4?wXdjhe=BC0%A8y9T24n-Sw5bv+85Va)< zlHQNNQl)<4+mlRXN7s?ul_Q|wo&!{j)rM6Z+t}MA0JH?5;J)An`L(zo8lDH>q|Y|| z-8<~S=HqOP%3OfD9Q&kQE)-_nEhl>~7LYFX2EJUGhnQmRuf-Cf>RH z?N%%CgUk&W++K{=)91kQ-`mI>CY=6$@eV8Ycj8CmetdnQ0$wPGQQo!;YFu;}@;Qz5 z@Sh;EVCWQHR$hV=OUGeaaVtJ3_rNcD4Br2}g34d4!1c|AkfSgUK6lr^QR!y>vg`iH zTgUM|b24GS>;dBP(+`az0!3S=sN%fPYjmSy136mKgkD}-z}E30rY`wJSKE4`e{l%e zU33>ePKW@{l{Wat_$CRM^bzgDFJZiK4*4@}3qJjBOxN+};mhLdqQa{Jyr36K-v3P% zdF!TgSyAbrbcahAzE}(kIF_7RnHXe#TfwXTUFh5Whvo+C!Ik< zVycKzUr!Mqt0X#i$Py<`Hzf^kRzP}2Bs32tqW^t;Sn(%}ZVA|lzMZ#8<2?^Bja7l1 zfi}=QY64_^K0SAI7d09khNB(3(Pm02ock}2PPn)lbMB3zZ)crDFJ2VgI!hlXL}lTa zRUxPopGK_pUJ%(b?zdaqCgKbBkad0&aP^B#)KMjs>^SHP@$Z*HmD*amXt)Wx(#zmi zU@Ug*S%^MLkKwNEInv>)f{Wfh!arxC@T;4w;LBx6a{K8V@np|j(AZ{xW$)jTc{L1f z^&5jG`aFCQ)r1l=x$pnt4@y$o>9|}K5_e6H(^TAX>{Nfcbayxzv${<*b|gz4jkbou z$s+p7v4vyt%!euGwFE)eWiU8cN}#u9Ek5NYDdT6_!K<4dM9!lIUr(6<4)4wJQ&2xn zzOn)g*Gv@G9DK+Br)>;+4n3rJV-~1XJwGVN_12PTT{1ac&alz)J_`Ss#F2}3Y2639J+@C+txpA`CT=oq9 zl)4XXe_f$IRR(%wcZphO#=xKA^H4c*Gn`!?3fJ}K;{+Omc~6C+4aH_+yI>8tm*NaX z$^~%Yg`9X<(`gL66pG8{SksKsT6D8qfnT&W;7!s#=m_MPxMjCN|F;B`_9=<^9S6~T zt{2uEd;w2BC1cUGP@K@Q2)b58W5>~UI_ZQXeM@t=|D!t?$T#A}7EL%~mx#b?rB`0h z$DMld85xvwTBLX+CsK!T=IHpo>7)dZiysPek zm6j-)JY5>bT-k|h4UNE5sDZ&F2uyjU2!;-=N_cn*4oT6 zsYoPy5K4Y7!oX!wGbouI2_hK#g7g0#N&o*;Eu&VkiN!$053VpCoLK zk|Z74?f3>#(6~V!=b8G}HGPUi*9F5=adt3Xnb3}5=|{j&$_^JE9*zc zN1*rR96fumoSrzQhJT7oNVj|}d3LW5rFvTE6FH8x9PURP)(v3^RfEg)xDNY<~ zfF`RgA;85R+5)A(b<0}hXBQDEMWD~Ors6c$3_`D7Mzz_gWW<)cfcMqWJ7ffmzc`mX z8mC9M7%e4}iibH0o+jMndhWrlOjI~5hMRh7aNcl*sB6V29Q$P?*{n20oNalYesIu1 zbD1K13>**mlpU--Aw^?BR?GwGm zacgrMW7yI&@fa>U6TPaGF>Gc%{MLOaiV!UV%ivI~nfQw6uJ?z6Ez__uUKcFkmt>pUJ>ka?qq>I-1oWip@k71jpA^i~3ZDpu33xD=@gK+sY^jyg&pWm;5?%iwQ-QZ5JR?ne~ zZ4r!DPA8I{Ir#Zz8ToMZJ}&>Gi1}XA$z7*d>R@PsM?Ze7TS2bTVgD7xH`$ur=LJCK zy-^sr!4}rGydw3AA#`n*D`I^OcK9cf_xt8R&B1EXul{Lha%dMYm0E(iau&p6wmq(l zF2J?BSE19?U@ZL-?dqxU}D z`s)%^jEIAM96$T9hhE)|Z*R##<47<%=SZLXgu`2dc{ECU7sSRdgl(VmXa~*Zue+iv z)>DzftDj||U4H`bv}#G=lr$LGlLNmFg}~ugn_wFqgLN5^KYzZh&;O1ZUVPMUeV{WPsBxLY?gsV zV^r}68AUU{OQA`JklWcBFH(t#rd4g+h}zU1eRPXq$hViqHip6ynG8Bf?F7pF+KKC) z{U9ps){t}U8Kj1L(e`F@STZ#p`?$O)`>0N0bbp#C%0iERxLAS`3+?Lubl=CP1(9HQ zu^U!cyeD_`V<61;K0T8!BX)_Hj_V#HJZ#TIJ%4+Y{922(uWeCv%@R6!zAicDD37aO zwnMJlHGt|4uz$UU(D)LvV%Qx6yi7p*!V>Yd$D_oFroS-qodeiJzo2_uLU8vMV=z$R z(y*-};LMvy^8L6a?s6GNRVSXp{iovT>`Ys@P@@8PPagKDtc6eGb>Pr}$>Ne*1=u}q zI#f&O;^ZCKsHMM;sK1^>V&3?X8_q@ogP{_T9BB&wz33oycim{t>I*2ao(Uc^In7|= zQBd1B3Y%B7!LQXnVS8f&Y*_1n11GI9=#Ukd#hj?CNN53myfSLImcz=^1){(2$HMFa zb?6r?#g7LPMOPh6al&#F@l@5Z_~VtB#93_syTB75ziA{y-2DKTnk%ry@C4d~YtXZ2 z47ge5dibcUir;2U0n_03G-AMl_&5iHdIC2yUVD!)`z=KGS8AZ7!v~Rbp_ag|I3Jhu zDn z<10vTc}>?RONvJi7|@g1<6)i|ON1{9NbBpFpd))5zujI#o}6Wf{CY>)&^Q(+{_ul_ zp;rF)k*BGdZ~`|QQ6tY4MxfE{gT$fy4*5DqMu+oJSb|_7=^#6I_~km zN)*)AL%6~*8rwb=8#Jd9ZSCdc(@6&R&95gV(uKI*I~D~Cr0_~JhmO>oM>P}1g5~u{ zoO&z?4R?NpJRN6HeBx?L9EPz`HOxv z3%T1F&79_n4`gY=x%=Dbrl|$6_Sif$cy9rlYPbxPs&!!H$5<^hb)~OlB?VI!zJp8o zVOTA+r<2E83dWnxf>MKFcy?$H_!!K?iH0%M-roVpsr%=^JtU0POmjpKB0wmAsRJ~`y6 z-wc?c%82r-THxI01iEok99RUZ3oNQ^kiDuQNbC-#>iN^L$z!sh;${+kK0*S=&%1%% z+IGZ$k}^z z*xOG}`M(sc2v^2^V;7SHcj93XMDFh~f(+*u>YW={vMCNe~yBOmeCl61Tkn}IRLlZjMD4uA7) zXS^q!g@F>2K%!nrAY}fK3{x3M5`6@lB5CltdJv2{W?^*HfAp@&7zlY#0v}r|Nchw7 z;-M?GIR5PhtSd{Vfp2W^*}huRGFeHGJoODooiV~OZF9)&!%3o=BNq7d-(9|e+XJv` zB&a3F@ra|=K0LI)W=RP8%&W-W^_6t?BN=!RK3a6E zAq(!)T39cT5ty~Ma(h`u7}DcG)oQ-f!M!D7pH@q7vp83`Ro8&O#^MZYnq&<9pKep( z)G!k8Bn&17Xku05UfN)uj2B)#qUTqd!fVNQ)JAVuq}qHP9!FWj_YcN+FL5{R`nQ2h zu6m4n*Nqg9QYU!C`6T(*JVaB@+{5b2r}>)aH({r(KUP^5(8W(EJb0jhJ!gu^vV=f< z`9w^j@9lu>pyPPgJCc0O91Z=B_sEtDsa7-83W;OgNRT}^gR5r}$R}0|@0B|sadHT# zxXb}Lm4)QoyA*m>CJl=l-+|A86WEmTop>LUuu@N*h4a=fptidCWR6u8nfB~0HU7B< z0|R-~(!v&8BMNctqEfIh(2#=L@FeRma-7oWz1NxAs!qc+kJ%HkPSH$>C_jusCEpT9Di$fv7g2S@h91A6gElQY)hj6wO`;vSw=te@QZZZB`GnyE5p@ZzC{V?I-k{zr*dd zkAx``b`Zn%R1E$Vh53ha$ldgnQ0{dS<`=1=9hDPP!*zJYMh^D(Xv4&%uZh>O7_9Tu z6X?BZq_Z}9lYGCGWbV=5to`n2YH47A|I!%IVp;t`%biO_7S>nmL@7c-@;|rH}jldxx$+>so0^VB>0_QVx^~?LeHBA z(2B2}ut3ut%3Bb=7&b!x>~r}3*jYUMMuz-a-bqIB4~rJIpC--bm3*zUiR9PyO4?jo z2|kYT+`K3Q1yxdl583{Bss1BP7IaaGT@UFTH+u+l@B7*M~G|0k&yoou$l@jJ>m)p{v3t3|`+4|X)D=@0$d6o_#Pi(r+iGHTt;B3mA9 zCQfQ8G;l*J(PIw4jNIcGF=q)LirfXB4pnqB2}VEPRXFG0XxLsSBN)ndr_(#nqOx}s ze9oB;o0X)jIzlPmu`dbvQoZEnyASl%Vl%kf@eFEr?m}hxWTG^Y+tHfUO+|AhuyW2D zSoS*+wy(*@L7Tm}O6@E-3{-Hra55a@xt^}rmjVfGN6D4bv*FzG8ZchCkj{3I7YJ%2 zNVNBC`d)|AH;2dJ{*{54xI&)0pO52{vyKoG;6=xza{CC=ev{CBRUkdB0A6rA6h3O{ zaBW|%h>Q0`U9%YI$`-=Mm#^@9UnD%4kxFz74$+Z16Ob8wnI1mZk6y1!`G;$^fvtNr z+Hn2th7Yzd>J2vw3)SQ@37oJ`5YO$uCqdBD0scpB*FV558(-Wv!^*Dl7+YWimCv?t z`Pi;>#(oRjJnV;)PbY(|&Ia1TFQU=*S@3;u5xtYr!0p&`n$%nyboYHt7uF?F^+%&g zuYVc6<@%IvztqE5uyMwLWOd^HGnYI#B}Aw7(R65~8%UlCLE{-7Sdy(Ea7#9UFK46C zVAd+UJ*FPMn|o8Mn;LXac|OVFj`YVjW3curqsuG%Au+$7{>RUQL;KZf z{lOY~<3+nDGQtVwuNQ%>x{A16p$M0VC(+fbzoG7(YW|v|arCs;4N+s92F5M<%wJ+L z9#)^5A@Ulr0qz=P;jSm{XkQi$O@~LqKp)~h4_%nkT7Y{Z7vR|2^7P8&si^xV0=;i9 zAm>3zJTS@$K1d~E&ghv`BQpYyeu#k76)VBl{R}y@WhotT@)6#+7Y0FB`)Iq&bBxgD z@uN!rkO5-@xUydW8^m_F>-l8hrZ^b9&A)DI>|WS4;yUSh?+qS1?Lne22emZ9puaI0 zpB}yrf4F)t_*4~%{H=(FJI2Em6(#iZOCQYTt6G z*`5Yi%IC6GBlf}+zs;y7eVX!*FQq2jxoXcS2n^09g3HlfdM!gyytH^LrX17|#HLE( zbu})7h}uJxq9Q1KS`G0|k))(96oQ(vv2N0MB54_4v91 z{pH}Ls71#ruBIP!PtnN%2J}^bJ3X}{85z$o(v%xc!WWFkyGc&;KT|!l4N1mtir>J` zHHQoqKBUvd<`BD28b36DB&R8%b$Nq$tS^Et+^>U6`zF!4tGDsu)qU_rH5<#!B++`{ zFZo@$2d>>|z$+$KVQ6bV{m{1ye+0~i75h}dA#e=Y?3zHLQ>5|2_G-))i(#p)qImF_ zCEaSgA6Hy|B05UO!h}^pc)KAiov#yd#JN#ZvRAoLVC$^FqbHz|%G!hr|{iZDw)A?mj=aFk{ z6e?;Tfx}n*!RJseI36m34L=^^_i-uo^uQln+jtJ{j!PiVKgWR2-NoqY5<;?1-xQ`; ziC|&(Nb!umBOr5HLo6&0LcQ?SSabF}Q&Ot|4$l8j^B{uX9tZL-K_pr_`z1-?<|%rw z_&9ji1Z)^1daq<8xAWU)wdJZMZf~r|fwZ|WE?faqX5~=38Dn8zUpC0xJPaA?UQj>I z3x05YT-LVZ)Wb6Z>P+*%>`y3sUUD2${^RBriUGJr*%n9U#?tCES;0mp7ch!77rp+~ z0(0BX!?;`iFg183DSN7kWj48dlZk0??rSEoxN40&krX^GnFr$6xoDyoPYfJX5i~C# zD4B}hpV}w7cR3x>z2wBJ)0^o1>7&W(S(Bj$u7IEK1pL@3OLy)VhwFx3;PMlFc!SgF zy{&%2gpQl!-rS>b>UwgWy;y6_IdwppOwKbu5dND-(7(ER$4!yfD`9*Fb~b>Qwdk41U^E?dpNx1HX4d5-$7h{KZV z7`&-=3wV3SLBebWIQ1|F_YI%1a#Gw$poD>A#}`3B_I8n5fdc0x8bXPn{rKyXF0Ipi zO8@K5g3Ch($Qp8oKPGq=eA39IDYcQHqw^fnk7;0(@fLFGv#j`tsVW@z`;38C!=bZz z3YP`La!8y&;#tqt?g2jJoB1&Pt}ua=efmnTX(oXI_e|bTkKt9ND9|vR1&i$#qIl13 z7!Kb8t0Zoc#zqCm@_0ys_M5}t=_jI(o_QdtTu75*Z}86_98Ida8!+#{ct-8TEB?8b zI7mvmLoR?WJWB~6bKKX^%mXp7Uor&WJ+A_j|AaKPS`}q~J;oDWrD*SJf%}%8;m575 z6Zzjx5*;y;M#s2xx}f13@h?vx|6}M(a$dZ)h%$#SArO={&a)btuJeTX zCZobc!6ZIsP)D<}H=xAsBXq619<_*0qn|%X3Y;J8WKeGdd**{Ytx~8$>NWAqsew7P zY+oDmvwT0}J5G#!_9&ki!knwHBs*g)0n@%Hq+}LchGk8AoRLt8(X-;h)TXc z#rX!8AmKL7gT8wYee_`#ayycQlFnozSyhNRmpbf-eGlQgCv-*`?qm>_G!*%|B%}DX3?7IduR-}k1`5}Q1Zhs z=<4o+bcxMrBy(B^-Sf4hXSHSNqU@QpOYH*ve!2rd-W%G=&G#?5yk*|BUSqOn9H;SS z%jutjYQ`bBi%k_bK#kI0*f4cjLG~H#=3&nX=)228E{Cv;mNd+ymxG>*0#o*t_=Uf44i19e&RXo_;hzVbwoilAP8Yg zu83$ocdphCZ=!X*-25d=Q}Af&9L}A5f!%ID6FGhGLa^!_BVjhH*?%S2eonrO?EkAae-CmM_Zhy#T9a>k?TsL>*WGdb~HZrj}Oume@$(}`L zPtt3?#`U@YosO)B&oMLeyO4R_M!GRvgckO=&_@HMXhD{P=!@YN8hUq-{*dWK_I*RB z$nzjddg(;ZKUs@rtxczz7b}=|ygp{A=^=xCE1Bb2oR<$fQn_`pG_9)%eF{)+K0L4n zJ?DCjuia%u+YO7TamiFssaHF4RXYWfNtoJp0P&5>^& zmqS^7odI%4+?L`pY5p_dr9LJkP| zB$FPqpTYHwJSe~OGF9Nb%70@Cy&3(0s=C^sUpfy_?%mV$+t(k|(?6Zow+GR5uhsOR zSU1zOrxhLT+=$*UhSd1;M6^&`4n1$uL~wHpy4e`T*a{ADQ_@qYYL*__81R`L&*amg z4aW51RyBd6L@e`YaR+T~7+`;mIx%i3XHoS@H&J9kHoCr`h%xdrMB7?oSy%4e!SO>g z(8Ep5Xi9Y1UVO<83AA>G#K6|6K(s$t*#|haOSGhpnvm zz9Rm-ZR461y{@B^+vEk~zAa%TR~Dm@U5Dv=vr3j%Kab7zbZ2+1Ql@)|7ODuIhaN^v zpufb%v&X)s&@{i3^o`^^k!(l`-*8p|vbx#KoK>=9cu#NA3-YC+0(Kun-T#g(o zIWhLdM7pxu5}k3LfI`!>=m)7?bZ=k_I+@nZ);=;|SB}ZBYh@0ieW4PJpYea}BxG{#7%aDs>4l3Gc&b}O-N|ml#um_jc zF-}XlcWcUtXpzEEcGh8Qs-v`un=RxbhV^9XbD|L6r<6Vo6{G9ha_G84v8?h(Wi)xw zcV=txB{olSCRNQ?i#}etPxS?Jkg0DAnr!uy-Bi4lzhj_-ndDl8mMKg{bNbHEjFLZS zQn(n^_}awmS-u7RRJ+anZr|9{_xovhXBKwN(w0mV2p2Nzp2qMme(z;ZZ0TpN_c@930GAJG_h4R_=~G$LVrJ_W z%tVYDqfcl8yYbB+y4$UWeA-z$`P^PwGq8hMQtFRBoybEGT!zV~XIyjC^HBCf97ApU zlbNTrzRrv}neNb_1Rb!jVO`Kd*n?Eq?LZ{g&rmw}-E6HnWuUATQyFR$SK1blR+ z7rZ>Un5eH@3Ue2&Ahq{qkT5xQc;dPyk$5>1t~t?x`@6b;iQ*GnBi;>M#Cg!7+Jc;w zw1SJ`04c63fW~fFWLL{hSY6M};=Zcm8X4JNhn+f!Hf;7@q}SNkSg%N0?{1!Fl`1| zZ)pLYe%ca$3r#qF!W44y$P{?0SBog_$|T+T3Gg5H8*kd43FR)%Ah#lQAl#r!GQ+3C z44Dk_!6yOg?oTE1NTgcxjTj63CEt0cY6UHWMl9onIShm}P_$xZY z_RFrs_Qwjy#=5}Y!#1R0H4k>#E+i-Q^N3Gy20X{lB727N;ns*w(EXib?OtsMLt0Pp z!1Ory;!Y5`;Jpd5N8^ZE=3Ok2cn{o)xrd`h9|Je*TX=F;C-~v{5I^2@9c-f6L{mEj z)&!>#%jX&JN^>Yre_&KtJn<2)Y?xLVj~Bk%*V19Gr^nb8}NVPpMmn;k2tY%7~FX} zf-5-}Ny)wnaIH#!^<)ZxXfTCr`LPLhzu7?UH>Sc(j`>1cz6@yf91|`~u>@bmG;m&| z1^Dyzq_8|u9?TfI0F16}!BhNy^J1pW#yJ+oq!@7yh%h4(_jfidA3vMiJU$z$XY)u9 z*Gj%AYX#44R3(wqIM#8K9I-gy4}15zk&99Dq2e+xQt>zy>dM9vxA3hHUW@0H1V!X? zUk-GfTR@7w?S)i{Ps~#7;GI=G(wxkLu7C5%>9srId-q)OV@(O<8Xs_To)bB>U_LzN zX=Oi{Aa`O91s7ki#~qVSO@wga`EzrEb^pk2h6{nLXry8V46iIkXWKYwrV*- zLw#FfrLp&iy~p#{Sfls%t+X87EQFL zea4dxa^Lw*IDQB-K(D+OoR_3d-i~U*c5?%A&}0`xYCB2Lf;8y=Jd-?BSr7hgtip~F z)!^2m8XPAX4)4raN=_PBLL%o)INcv{csCP%8L=Z{Y0gknH~_9ibm64WcfsSVM|jRs zXBgnbBcEpoprFHzOwW}f#n+|ak}ZlPcCiY)b|{6IdPP9t=4i4)X9pZ}EyWhzDt3P| zRDp^_9WVImVq2$6;@CQID)>8agyZX^kfjA%;I`BVQZbSSU#t`-TQcOK@W&*w(Mt&` zE%V_m_carV-0>Tcxhgbc_SjwJ7|SBHz5y3o z6Ok)?Yse?;B`Y}M<3tFXf=yoB1*5b7;#()Lfe`t4Xt^+en1zPIKk_@s`iq*jtL7i) zJz%1Rf9F4L;)D)l!~%Kt{@tke6|thcpl{y-H^q%4F-67KDS{<*$$9;{0gqx zcLn(B-Nq(0U7$>3IUUM7$vt$w`IEdVqt_X^ui%m819CxZIfn{nifP+C3cx(|h2B(R)>ZKCJqm}?KYWSu z(0bVRF@)^!oC398DUom0$}se_GFkob8$PnT2aJAvfWN&M0|r;O!41X!WO3RC__{Zm zusg%x)-n&0_}v@M`VvU8f+mp*_7mZpJxU~aNRd19GDuWa44fAfPaLCmLM_8(kQO+R znh<9g6z)URS7$*~xSa^b524>0Oe->+b zU6Xc^>Ay3emN1pT?47X3X%KAM*@M@N41&nte{k(;F*tMHeSEL(F);onPTGgM@k!JJ zCb{0_Sj=T0efh1X3JZ>NdrS#KeuLd%8(APOIRi_p8Gwi(SE1aKL9D;<8wfo09wW1# zVD`FS94B86291*^X)OlucFK78?e%ZGNFW9eBPH_m!H zvn8#|_^|z*9dS6C3$l6HxV<+ORB*1f$Uo`ezqgz5yOlv;>Y7}vVXw=ZkoQ%%s^|&t zWtm6Q@&q?%vw=rgZ$32AT1Y1AkKt7sw?Xpb+qizAID9@Yj{LP;5BD$iBc~f;pz7|k z*n8FuFfO7Qzwc@QHS@EG?9@1@aB>G(P`VRlF4}=_a}I#f2X6SkU$$V!?m#kLuo^D- z=}W#ug+Sxz)jUU|0-@#AmpoW_Sy-}T8DS(4eDI4$4&ShWs|)s!L4!iLWy3DA-l`Oy zk4_~fO%X8sP7G=KFAh3ha3byVtzhK=QzBmJ0Po~p0J}F<;6b-?u+SFc7n66w;^}c@ z`JV)supym9KJbJ?hEBxkr!$nxwjy(+B;fM99GAR820s5TNs=7JVSJ(t*)vTVRwPJ} zIBiej!Y+VCi`~fjHczfiJRi&WEd=K#ufm!yG=YcXHPGL34`)^PfE$^=@W=fS`w!TF z7nTB?R_OrJb3=)D-&$yN+n)q%2#33K77)3^c5tyhBr7u<;HpIv$qFg~t>NeHO@k$azAhYvD92dvNiw10GSzA#2a3z?C^^#I__G3Vp2L_X`%p z(Q7tLFNNf+ppF+)sf=H4kq4rZ4B^zFIq>lY6B57A7%C!Ta@y%Rc8h!kbYBc$y}&-8 z6K_dcsvr#Ygv37I3C6Vsz~j)JB>4uwuUyS(v@RTXsaF88lzeQk!U2RV_#`}XFOauJ zhc?9p@pzL`&B>;RhH#py39)uCgD6y$>d>*$CO~E@}hQ*oCT-=O%CcXy#ismG^%@0RrM1jI5fw;qY19+Q~L6*KthO=~2h{t+vFS36QR)@CX zaRc4J;mcQS`!XGF5=$o7Hx>Fn-bpMyuQ$2gJTvxSS!gJ*j>KSR9?@J^5i*CxpOYrT?3%{i!`z}I~<<(k0GP4)8OC#h5+&% z#s_Y_203dw@hgYNLgT%&K+BSe*y8*Wo>6s^@HsnZm#l9i{1&FnljP};6V}==aDyg! z|HlwYuR9F32?dq ze>9(5P;ev{x6OioR2_-$w(Zc6ok$W?HDQ9ZF`1FL0g5&FlBt(Bz}j_z#B$aesC8cx zoBCDp^p3t2<~_;)U!4|1-`xW8Rmu?>Czz7r1}kWG#DY}o&4$MQh}_v>f|JkMfk_Eo z*fZ)s(AT#f9(lK%EMMmd*F^ac8^J^{W4Z;7Y|;jwmuF+I2Rq25m?+p06HPp;!lCB} z9?5T?4Q=5pa_5We{eoi5$|Ls0qNxvag22hSh@2e z_Cgmy@P&yk7jd1s5c(7^B8Kn1 zU~Az3Fo=4EUn{->GpzftpKlN`UE~90p8AkS*8^akr6Ig6u0!n7bfCYPF)`WW2d}(c zNqRe0z}#cp8NsbFulsy*S)75%j)=H*Z-hp7y~u~q)o}3V2J%a0Gpx4pB}?Y{!$TS2 z#6oj5N%QuDx#I)Lt+6op`N4Jwe{3X6g2JGJAlD< zV{l0{kv!&{rfb_j;m1)L5b#zKCrf8|P|=xOUa}evg}M^!PDgmS+>y8?@L}SEBrsLH zO}KXQJ>KuP{&>eMJ)&Aa5#FCYo_s%Q2$w9b#zo1E;3EGrp4(6ZdT)A>!)^e+{%S{} zD_24{2UoDNr4SoEH~^{^ZpQY!BRE)@^JIAJ!aiFM11n`Ype%M=SXBO(mk1k$60ut3 z$s2Cn-8Umzz2?w7I)yy!T@95#tsrk}Ghp@K74T;2QnD_{3l17CAX_xe$q_UQj->O5 z?eZD$UYilrTBT21w&=mDQHBH;{K5&vufb%;dl*}O2jYTioV)ET_;{}xw_2Y9HKW>C z>ETpRt#w7XRXQBZZ^(h)LX$}JwiGzR)kpX2GKDj~n}dYxWIU*w$eS514K0rJV2vja zfu7`elFL7ZTdy{OOK%_Gs_;ADa7zw+-IhcG<5J;4xopCl+XdFB_TXUc9xy)cAvW!^ z5ZXHlo1_Jjyud&eyXr&PB*kSLl+#Qkdor`3k5?Zry3-afHqr*sDdX@AU3b`!;Yy_c zEr#n>yOW4N2GBU1^I|$kKnDpU!m}2EBVB3u*p4X9H`<7ge7y+<#q02j*A+l*ODoPV zuqP+p&4ui1OY%y7IaICPK;*5~z@LO$i>^ViA}R|cxxK)HPLlBRpG9~oI0TO+?Iy0% zi{bTYhe@E1IM&Vf0Pl2d@V({AK)ELqZgYwz-)mx_2g)J`zL~-`T_$AO3rlFb)|6a%3?JfS2z8O4?cEYE*b(rjbUAWA(7=J(F4_Jc&Y%{qQoHjau&Ap02k8Ud7uf%yh zuVun@2h&K(>I^95lSxe0WRTxV@o<7?B6&3~1Ii@JkTxwPxV%M)ik{a9LG>l?$4{ z&jaaj`MCsQ-IM~0+cU^K3uD`&`|dp3_vKCJ57^m_JxzuQi?f~J==n#s1aB8%zaD~VFW|JUj z2c`ebBfEYLV;Axov<401{qQ3&vl{`RU!3q&e}I#}&SG?=f-Lkq0Ee%alSNrapfha2 zU8lRj$kPXSDfm(~l-&1Hq*YJGMgQXx&3m2f-3GkzMJ+Q0hCRpKl8$U6A2)d1~;u~X; zaOvJ4vN9+XcAVKl8m1M)Nu4`M8_b2fj7vyDavxY~Qj6nyE&<1g*ZA}|YuGT)l*B$W zgU4cRNJwiDUK3pnHuH{QnVsi>fnN>qSI)%vZ3!6Nb_u6U_Z9B1`D=&V?p#{h>)&MT zl12mUCe{hZ&JzM<_vgnasi1vqYBIGCzIg2 zDsbSGI{EPG0{D3QCEopE2zWjp#{4ET5_HWN`fAT4gYwq!%%h~Hjry|gw9tM9u5e4OL8|I-J@tA5*E+n3E-HM2)(-}95F zAvpoM9hg8SO`ZVlmdKFHSz@H(hAiw|#ko!E$G};11YAo#i(SK5@K51A9=Nv)=;o*6 z%M!`pKi?93^UHX$@Z|_7*8hzYtu^3L9oBZSXdRDcstI*h8FERzRLGP@lJokTV3VOA zXI5&*e~s$F(H+Nd+_fffez!05ZeB=a!rh?wA%9YG_!NF_dK}Db%fK0>Cqc=S9mJg< z0iTselVypi+$#w+c;MG0BKk53-pE%Y#Ti;~Fhq?+>#4#?O$;b4O`8CTa0IrvP=NwU7En{>;`;e#SPGzdK0GwrV70; z=``(e-pu=Y^DuAqogL&x$QIb;6+^B?rNDBzuR!kES1kQy82pYBBSYmDq(jA&t2fQa znE`8C=F-JOB3mc${pyYg5t07B1 z9hHE=?<7en=Zc^8X$BYpb8wxBJWz;o$MsHI$*N-;pnY-}>6j7=k;8m4w#o_~nP)@h zU78P@?QO`dw=ZHWVUT5)50U*g(O(^VkddJ+G)R0qbddxYDoc7fM|Q#kg? zE8zac5AWPB50|#f5%Xo^V8~k~Vz2a1nA2trTBA2%!>#4K%g;aK9lJh(8S!86%#&{c zo2Y^J+!*4WOTHsia5n(Lf5X^)**mbW<}ZG8M+}}iwiUiM@gvFN8{mqncp{M-M^rwB z!RNu@#MyT{6kSULOA9>l&lB2!wrAm8r7OsxF<0oi)SH+DEQ51u72u4VzxXsG4*x!t zA(xWNVVJOloEY5;)4~pt1Eu=FJDHDzcQYV;iX1*4VGi#c(A$AZZ2Z4sVuK5KW?sLM{k_0m>;Km_}7E?sk7;#7fPo0Kmx`>`B~j;u}w~Rane-IhcFU5ht#41TUBU2f||%$=2G* zP_6t2p0HyO9OwG<6ZU@xw=?_kuPie-so0nlY%_zpZ_G&{D7Rzp9}qr1L3y5=jhil? zkr95&k^uiXN#f;}MnVs7D=2MZO75(j30HlxAv!Uupo;SvlGYdk)Bdd>>Hhnn#gkp+ zZDuj-Ua+66{T59Ue+R+W_dEzOj)tc~3xqvuyLm~5rNTSUEA2FBKG~I(3i~ZGh^$L4 ze7G$GZv*)pKR%Q5+V2N~+%LjPnQM0LFNT}W&3MT(o@_w&=&8eGr)ea;PYXu9S_zMP zxRS)m+;FO99BsUQ{R<%N?yucP4K zb!p_-3_g+Zw1Q2{9AdrJ3`+WklA;kWnD%lN*>)fd{`!T8k)sp5veAX)Ok4n`r264{ z#{#gaAPWBlJHRMU3|{yDg1-fQ1C34JaeT-{P`-B`-mhN-CiutTfB9PQ@K!Z4d_o-- zZ&D-LUaI)dpcaU(`6gT?p~Fi(CkdYkf8%Fg#39E*z#kqS!b4LJ0q=lPd~|vV`1{ls zzDac^x5XF0{{nppKAlVM^`ybrlbPhHWFGwPCPf0bjDaV1W7u!6IDD2fnUws}f&)kN zNyb7w=u+?i9K2V7lWi)%*Q|D2S8had=V`*=KbqwE0}EL3$eYAzc))piD@pYo516+l z0n)3XWbAqbG&`8W-6u@px`le={F>=7vDTQpaQ+Ko_lWTO9gV=b^(`*F+Jip~KLXo( zZsN}gx4{{wwM6~nBKX6|jVye+3O+070pHtx;^3G+od2>HPaKhjTK`Ft+9WCXV1hK+ zxnDpA$63LBl5+?-kDz{q2=C@C1$#=(vFgkw&auLS>(*P4VlWFPn^=>m1;ym$=K?5n z+)ci}-4BxwEFxbo&4F_b=aPj@&d~7ke*E9u8esf;i9?&q(%EM{J^gQp%Xjt58f+ac>W&#R3k&Gnctvf%P5|m zCkqFEXu=8=Ir4P&IJhWJm27u#Ae=)9mO9TMi60?cr!fsH9bX1szKg*9X5s)G^?`dk z7Z9h;Mew<;CoxrC247upB-v6f@c9xqvh>YfxMOw!DgIpu^+kKgl~*Zn`L<1DaAp)V zZ z&=HQ$+68u1l?9CdIt0X(o2c_hsyUeW?*nS0qik`wi_> zJ_a`JNG!4IDOm(Ruzd0v*u#ndH_|oTnkea7K=;I1WRcJs-h1qScvgLXx@$})o7kz38plGc-!u_$;<+8Z_(xRU!Si=kUJ@ zYvK3Xec-7^Cl){U1kf-2c+x{|K44)_RvdPON+yfR96FWkJv9Md6e^K+uC{yXqzV4B zH^xt$48W2nDp>03UCbnQ0=dTzaK6J`PF>jG5r;TW zm(CIS8ikHfu(WC0y7hqW77P*qie`}S7=>xnJ6qHzMgTK0!`se`*mt1E!AfQz{L z)-mvfJ&CJ#3&DxTY`YiB(uIAmyg7)((xzi6`oy4L6N)}*5f^&{XyBGhrc0$jiF$5t z&d-DU_pBlpKf1y_+uX>v4L;C2el5(kTT14vc88yud`Z=uHtgr$0i*}-;^ZT@K$qhb zcx;(0(H=J*+Ucv3;wwo|>r6PQo)`@;X6+zPCl~TygtO4Nz>oJuElYT{u~n#-QN?SV zMTD|J4|#cu77)da_At-gk>u@K3|WH^5Z#o6GZZp_d)P)S_*~QUZH~KLe!fbR{ofUK zyRTK^hdyV3d&Wr|u5<(p5leWk(TJ39HikPJtVrd9T&RcBh-GgE>|U5hTDbQz!%K9@ zelY|1PtKf7z9E1g=J819O#myO3W#d64(zN_B?imYp%v95E8e-1vT07x*>oPM8gzxD zYbTIGMJ1?Mr$|PYOn@i;I6#RtkQ}(d=iV_mkuNGWxGOyx%-WcVV>C|zqbwJwA>xzi zg#x%=pW9c;Cz6QBiEzfiM6xJk5}XwJj5qz>KcVx*-@N-8&xMJmUSOJEHg5 zj_{k39(--5N_I4;!#p>Aa(B*pD7(Oe%-QJ$MK=E2e%lWgHFx2B<_QQ$d5*tslECsi zZNRqtnOJb)FV8`E0*6I3gIgLxe0V*p2(=~)N+TpH}s+DW?Ic{)c}z_Jdg_@9{S;G5F?J4soBJ3WvSZiKb>AWNAiO9ZMws$=NI%MPYJ-1Z{ZlC-XyNv~ix2Sal3vh#=Lx=X{vPmq z8klEYNks8R3`}}U7s_{CaDV=e-WyGcOA26zK9CmB~t16 zk*Ma)8v1g=02S40(l*-yE?c3Fl!``JMS(wUzng+|>u=CjZ)f8C`~usxeV(9f>r$Gs zC4t?QkLifqRMht-3oWmyN2rFPX+L9_g7!ir_UA3t=`5get;VR}@eHOUxLCCO=}p!& zsg9ZLHi6!r+5h>Iu3cTj+LV+n-DE!+U_KA;#VEz58?5~nOcAoD>6kX`Y zhFrVKUUA}>VwXOkaSX>x{tr;C4qs+d_#Il4#JOV54X_z$@%)4QC-le7KGrTwU$kQ$ zL-)4aM>75Ik@Y4C*5wN4KJv)na*FEocEbs#YvyFqzilDieN~6vyE&QteefXa9B^c= z#aXa>!;2UxZ%l>mBW$PCPI_Xni&js%j4D*J(3|F^^ylx_?BBv=%%ztB^xdOSc4Wdi zCW)U%72n)u2Zr_OYK1M#p~pIO@%uA!`U?c@>538cA0(jWBvmDcBQft8hHskInRr$8E-TS|2;-Xg%L}W;0XL&9RU5J-7_>2DbF+O1{sv zWOn_42da&D#$MXt$#h=1MW6dEMkg!N6xepV1JU^Rd|9bKi#{yGuIMn*a#&DL|wqwaDx zVh6|V{FcB}8_sBUlr3f5y9$x+bA}e*I6#d)7Nh=qD`}t01&R->rg55e==1|^LBnkw zcEyxJ)IF6GyAvaU?8&Lk+1p<7?KehH^HOu9F1Hv(tyqn|#~L?pMjgyD$!SRUUj|*z z3upeiY(VSgYtVleLlE!VN8sE7+0LIB4Oky!jLkEUs%09|eyq#}eMx06CFs+dFS}4p z+j%y>^A7!dK@0T{`5^1EI;!(UiTy7jh0T5)fw)@?YQ4_VA*V-FD0nCO<(Etkbu>_) zf2E?)wuvHgYBvg!bf)fui%{t4Q}D-ybu?Vtjwy7Wg2Y^uQOATV_U5cFGP<64 zihVxFadOwEBa=oOnm*!5GusQOWzYfi=MhUc9OL{?c0OoPwgzhNyG6Ae6xo}BMB(cy8j)ahFt#mEI@o-czcQC$L)Qj%uAx8woI#%hvuVR-EV{YAhuQu0 z5F#O*N1>_&ow>b-A@+BuL0K`qshUDd&c@U9j(+6PxE!gBRigTDQS9se@@VHL2U^uO zA7NK9G;`&1X6>9#QDA@rtGdPsU2SvVd{Nb`Sg-=xw8##9#ZG9!yFf;{dpzpBKF0PP zZ4`YpmTLAI{ldLIH{+Nwoova1E_Sr24YfD_rjLRyAql68s9r;vIpbACT_2>uc7?7fAGuvvVL`%Q5(_W=O=G)>(>LIg(o<6sisb4Kd zwO1`goA{VUr7I#+`v87i{SxLvPZL$pS%)hBJI=`>U8&{z{Y;6Q8pV!xsMTx8hPw$F z_1&3h<*!Qi*|Kta?(}N5anUx!zp@j3Jtx+@smur+@}smeO&raa>gIQv9A|?UKtf(q5soei-(fqkv39)8w z^kPy4%OPUu?-YGHvU(*uLHZlBK(rH0^zI|IYxQWju|7RlA4gpjwP`Rb3(J7l|C|h@DHI==KKr=Z<=D-z<9)EP!DDc;~i8)`#;2^2g2X< za%m~Dw@pS%wabyFQwF<4?lZf8!gS=vYo-RmJ8bOCmFTQ)FH^d-5>QL zRw?To-BZtT!NP3lCB1c2+|`<@+j%nCVtL3wV>|Nf??HKgThQr;&*{emN5%~IFdczH zQ4%viv)&%%^0engY+EBb;QEM-Uz0=M)%{>KEV!Hq$KJ0C2%u9ANC=o^f$aR?70B>< zB`Zo>;4x0TcIQR=xa^gTsel5*=MGB zs~g)`P)y(MT0?_^pU{g|EzB0R3{(@-j&A;LWR2d)qjL9a=!AnO+IaRJTet8SYOIPy zq1{`V|I!7tb2pc{nf;etvU>`3%gAS~^NUdYI%W1@-Cq7iGr*GKyXYUn=(xN(5`F5S z(JO~VtDy@s{}rS?L=iD3BWU6QF}CPS1Je39o6-8_gi_}`WX`l6MW5I5+3Tw$kxGa+ zbMC_+Q{t)3mt*guucut7y6-|%HW12gjkv_FR8FQhI?tjHzhmggr#5DWgdJLxv>P4Q zaHnmmo@i3bLR!C|o<5t(_15Bj(St+d+2^Tz^zHCC!6V)Xz2mWuc6sDezJhr3k8zeX z)b0f{F#8o9G5X1bPmxECzH{lR#T*yyT|H7Ml5hSq<16)Ycc<2`-?DF{e^B_`4q@w5 z{)u-OE!d^ThNOQ$m2YZjsH=-;dPF(exS$dp^UI^Mt=w5P)sa=pf6J;BPN&XewH)&z zl`+zvggou9(Ed~|g5Ftz{uWiU2g<)9)f5X-%SlUc9I2YbUMTFuym(e_rawL8MP=W0+ zR=Y5p{UFssua%vp|B;)_JBd$>^CnEz#fupIXZsm;v4~l;@(Z)P`V7DLXbLi)-hu>M zHj$;Z^C{2Jw0W_qLUZ7{gN**rJakTSFZIsKrQP;Ck$kFBvxdF6;A9z}@zdJQ!bpx+ z`7a$ER%hsH@e$fTNe5Y#EJp4V^62Y|AB;k?L34_F53Q)mW;73)(P!NQbiA(^+G-z3 z8(tYOX-m>+ksYB|+6vK+QE)(58T1?k>Ytje)FWDx~JTl|5sYvC|cf{{W zK++R5=_;wcOm^@v5}P=c&DQ=(=N@XIfnz+TTWJy6dasK6Heslc_KzuNio7cQW_jFsp@%`5Dhtb!`H??cKCwlvl@O(5a_i+NUbk;CmAQ?laT{UJ^#-*}O=C;i%bd701VXdesWjDeIq6-|2Hj>ORhbyVDxbp_d2Aq}D7{I<}1NkN3lU znZIdC*9F#6Jdaj1bhCAviqYtl0u&)DD{zm?r3-3>?D-wf7`d6QG$Z;0<5PZ+3H>NW zzpOaQ_g0QVuOt7W*rc+njPn;-t!Rk|~HOJ_jFKW%(gPyQIIG?5sNoQZI zD5Ou_dr+E-9=boqaDweWOwq~!v{yr$I=@e)cZi-~_k|v|XS9trU6-XX-cpSFuQkY5 zi)pS^`@(nh*#xzEKBE`j_UP-iHN$AKeBc`lFoWA@VLI(;O>7#=j?^mIQiPn^1 zCO?}`qk7d5b9tIb&C3BLwm0z|v{aFQcr{%w%4Us^uR*#gciE@GFPWyBS7=GwH~PWt zI+}W~kBx6VM_ZT()bVC9o#s(ZgI`=hX6Gd6xowB}2W)1tN1x=O6Ax6_1L`lC?V+lu zsfH)mIPMxFJXnKtFX^$(C#IqgrzfB(>tdLr3PbG6d^hI!Pj|X%%S^W4Ns|-ES#!Lb zedyXN4P>gkmg;>NWKBor(OUyMP}4z07(B4BPjOE$=sQuO`GcE5l zQu<)ddBDCg3T+Dfu85UL;ad$F_TGfNYHVr!*l~s`h9gd2!MwCM$>t~rveyQD(Oz9O zfud=G*2FTpJja%Q{_rmJGH@xqHf0s-vosfN-e61j*3Dw4_cXFEUQ><-^Z<=Y ziLsYUmC)_^LR#Ulo7xMPqCDR;=E2pq{Dy^TkT@Uo|izH(z{)CF9^`hA}LrD9`Rho0plX+#bhkDuXLjD8QqCWw8 zOqxt5U8LrStet+NTqz~N#jGNvUYdfA#jd9bFNf%LcX6iqy$Sm$q!@W*w6exmRGL?L zoTRNwLzoTGu5_A2K2to+ftuGoVggSTQnB*$Y?t+SFuy&WP4=s1RzyV)2LCi8JjzF2<4bxLz-N@X1GosiFfX1=>HU*_aj%|AID{7 zkF1dF?49vG=Y0>MGLw){ib{zF4WFc8WR+D^Dy1|;MH1fUyzep6KuQ`^w6s*xl=k<2 z|APC=z4x5+e4XdxG4nQ?nEsTVtLVw!TVBAVzdeV%Ew<8o@@lA&Env^96|>i_<{&Up zjs0e<@479l4^7*|puFmfY)-H@O(~UWj(k3w&W4|;$&x`KV zJzZNXj-hcyTeN@W3^c#@5)Ezgq`YopM&rpbep^xo3Yof*nlI>Mzi22l8)-{XiHH_@ zM!lQewE8Q%R_iTFl6Im$bc5)_w`*yqTmaqf-9nuO*JzEjLvz7(15WUKgKg-~p%MWQ zJ>I*J_IsAn?!zDGmUoZo13Q-aQ_{mPFWt{%7XRNj5X?@AddPf#H^Fu9gx$=h$(s?* z?4@ZpywM_y(@enXcgUjf2y&O%K<^#mc;H1>83Xr?$YiBI?JAwe{1&ljZjctGwOb~z z|913)Tc>>4SMeT{H?GvIHvbOYm0`jGV*5F_WF{fvj&9upMg^88eUFjJS$U^YnE( z%~oo8+?g+oLE_^+UQ5?b11Ybiux~qA`~R5phP)7n^=I5(yL-7D!GLp zb!9H|UO5aYtWriZ)GLLzf)P^unT=}i-)7cLwjt&R8)YOe6m zr?T5W^7D_kGAB09M)$Q3(WIxt?1z^VP;^xsT{Du#n2Fc1f|S?DV)!=7sTW~>T{}+) zWf!nVf2Xqp;Tlx0ypWzzutOo7<1cI171UZ@i~2;OSmmRdbiTg|Qh(jZ*Qnhje3bKy zHT`s&ey#n$N^xt*oXC^tE67F;v*Ov$CuQl}KVOBjHYlNU$68V9M2=gSc#gf7*@cdD zrO|!(S{0tyKKdE?OOUjCnaH4arD9V&ac-j4+P|1YElr&u_uP#f^)Y z8Mo#6@|+)T(%350@>`R}D;5i{h5zAy-8v{-@OZB9mh&`5pJSaB-Z+O;cP(N<7pYSq z5rjIm_RvYFjrRMwqll~X=vn`1tX^F;8ojN-bpE-5j{bT=KVMU&^KW-EU23V!^_xr3 z556kx56h)v%LbUW`y6P`9qHzJ^SzA0q1nt&n>p0jwU{-MDy=Ir2hJE*!?md^jm!nqjWwRtxn&|P7A)qem!S| z{Af1oX~CoF=fALFXG+oU+T$oy#EcEJVwn}4d8nydNNUVW>FNEp^miU)yx=UftP3KS zq!P|IVL{QtEM#=Z*!4@K5t~!~lo?%=C%o!o$uVG7p`RZXqY3@X_#uyXF${AKt%(gm z*6x$Z%UhYK@t8R~Soey_Ybm0^iP^MwVK)0f!G;YiKEpgIn1#M3IMDH*cNm|KJQ`l9 zN*mYQBI}zCIbJ5qC_NlyreEx1lDpE`DOqidzET*(8A)RKA|`qPVX1$5JqSSmY< zhi(ktpih0osOwu9*KGj;B;%*g_?(bO%h(2V_NplR%4H8bFM~VpC}}n;40o_m-gqXM}whU|VlhEZCVOrHjQ5bA%NDcqsNPqqcOeHDYD|2m0pGH+?$UXg`l0NO3snNYn5F2wm!^yJisi` zpG><}`J#)w~(W~8pqrBh?mP)qer=A(H##X3%| z={0Mcb=Ldv%ND%mYa5EuIN6=ZM8gt2pWuKVo>vqGx@vM<3twi#j~4c_l8o>IJA=jv zUO)-g#mIExZU$~Sh)#X9L3~$Jv?({A1|0H2St6ozMv)2)ZgNAdQ6G_S+%0yeOcotx zjOl5~v$UhVgjU(N(xP-k-PC8Z!NVGer~itzT0RRcogabTFaN`utMnnKJLl;4#QD@E zt{=6p-cKKNbfW{G#p$c_Ip{)@7oB0t(6O|`YzM~y*+u$<{MB=qpCVf5Xh9g-vu+nH zocJ2$*MFicjG6&S^w{S(t(ba(nqCh^y%#s2dmA8i*loh>*e6dr z+{Tg7z$vzUuROaY{wmw6c9E6o`h*TfbTXmKEYaLjNAxOUoXKB$m1RCrcFXQARP-?r zU8vQdxiXv!ZRQgyE!>do!jMmI&H(`_^9YK?Ma zY!im;TYcG#TZ*pP1)FI7^oMLy>>gHU-yR`T6Gk`d^r5#tM;JlLeWoR6KN9ybrGFIG z(x_YCsdDCDWLzA>d}cLh{_Ai?a>7#N_NoY=p~8<^pSd!YYSCtug!jg}-8v%jZ$xQdm}L#y9!XGK*02=`kzBKA=<>v-OR z3KHbe=DA|@?)``8)SMmk)WHlIb7U)e@%ji_I&LInF7SjePF$j|#ahtAAt_qewuoj} z^rMDrSJ@QJZd&-TkpcPu*;q?c7ez0&Ao?QRgGG>~ehu>Q)j=11b?Bx^864A?W6-Y2 zr!_-o>9&dgnAdCW!;wFyS$&xa^qavTwO1>lxzk#Z7N*x|LlG^5KwtVUTy-b`v!C+}K1YZL3&RgCgxg(0sCfoMv~5urGL zfce+8kPZwhpwF9+qOTcqIR;xGdcSdyd2gPB##Wp{^&A5=aM4G)^@}R%7V>Cjk|JGh z8HU`ZzGRPmQ>I!QI~jOERhaq8lDV|PoIYu|$Ertp(@QV3m{T;BrraDx!)n{vlKGKn zD0(mRFmo>}DtC^q`FxSy+$+(X{0^pG0O*`nD|Y9kJoK@T z*CmM+;3?^VpAUe;KA0^X@V(Q4>(Q{TxJA_^@`1 z7NXN@8FVp7g!%nVh~8KhGi&EAMnfB}(Owf(sw7CEzn_Ov*A1?yOFfz`>^^|9Kc459 zj!#*K_>+vPj~*K07t-B-<(iwGbSWcI&FnZL3{`jbX zM0yI4<;x=+2b5#zo;O8`FLRK(=4twN)mv0NodZU$;@E5jXP93{I~mC~hLOKpN1ZEo z(gF2x_S)_!I=QxidfFSIT+W@7yr`KSzb@WfrSXF<{%6n3eR2Z**|eR)Iw_j|Oo>jZ z5_4UvK8gOkx`|p)2WnYg&gdPqM!&&c^y9LC%}n?#^zVs5vk#^)ViBBgttMUgCC?Pq z%M38mM#oWiR0f)w@4c6!>4E zo0q%Mv@v`7_dx`V-I#{FMg(+w0%Xh_2U!E5C(XAvr&CSWqrb6$R&Hx%3`47F!c$$e zU_lxZzafX5U)7?Wvw|6sD`w=4*fk{m*@RBJ??L%ioF6J~jCpfBi}H;&vX_%Net2jW zQ~vxpvv%ce_J!6V^!s!bI*@#TzE^N!2j|72u{$s5jHiE@)(LyKTz)B{=~A@%`(@_8 zQEnD7n1NiS9`H}>2w*$tDyp?2l|I_0NT2_XqTbRR$7`Yo{TY%?y(PX2mlfD>iiK__ zDzIGmdSW#7^=M>$UlpOxp4n{5_Ptc2;0(I-VFTJW#OTx2Zx}VrHg+(p zfW|KHV*>fFq5V@#w<}aL({BAocNa9#e*rg`pU*8^w;G>7`d1>EkhhBHQ`QkC?D=ly z>8&=z_fkP3M%^^j4zagtz3GI>J@k^SBr;hT$T&ZkhvwXvjeZ7SWlA4&^GQn!c6pW= z@Vs4%G-@#4YyC&mpns1){U~7;GQVvPp!}|X%!ksa;BbQ~+rtKOxk*JZsCY3Zenix1Ecvo=Qaq62#8uFHIQ z%6V-w7oqSCo6xVXcUhbNGwptwaxQFHYEW)WFQ3;J(&P0gA$pJvSNBAwx69a^KnIjy zxB^WVy+Bc7vhdehb=oDTiUMo3nRc@!D2G*TUgbZ9eLA&-t~+;>aUbD)e~o3Fd;J*O zW|K*$Di_m*;%Rg|vmHJBdWj8AT1sOSC%FzTQANsm&gj{{24;okCS&H443&HWeUUBl+?q=Qd4pzBhX z*j2M0vsLT)bgAZV_HHDXS?Cl{tE^^*iaOBMxueY9nDc01gcCEhdOd|@>L}i2lvyR8 zfxa<`^zklLn%${PbFMbie2WXrcP>k*)Sr${tf)qIYqm4vzK2nEvo+mzV;7bG_nW`J zcE0e>r5g0{xdH#*r-@Y4@CUM9Ye(f3ck-X_Yoj5?f7vBJtX#Kj*@dJ^yHVuQR8+P4 zHu);jOlSU^hPLc%L<_ri(5!VXoKKgBEdTtX`+t;#egP3&lUu)^+?f3zcd2udkzSu=gY8TMT;%Tn;v~AI~ zE3))zoDY&Qa-tQ9->AkamTA1p@d5XYpoHbq7}xd5Oc=)#KAPx2r51GX{WAB_pLh-1 z7CIX(tCDv0_~u2Q62X`pJS^O!-QfX9>~tZlIk37 zWG6NtHb3=-Fs)X^HR{AObZzf)wB&xgaAr4;wb#sNPn@dbdu98RM8()G$zi(lsBmrV>J$Qxt^^zs^BwPS%_&^Ezuc%rV)NH?c8eoc~>@gjV$H zHwQj0VKo;t&yO|+VX@C(;Mm83W3mVaW{Ap zaE@9=Vb^6T3((Bo=1^l|<;zvcZEMDKf`JEtX^pXAr^t6Z&_ zK0_Doo=E{a=|L9?m|aXCt?@wDtxRay5*62g`+LyJ;r(>^)^57SZvZW5?qwD^uA&jy zS#%_g@rcaasdf?6eFqyA)adX^v>^ zq#baJ$}FaOtO(JdTuFGoCqWinp+Y5`) zsqif@{%jO|XwOH3iHg)xv4QdY8;=Z{WRUlxbLd0jX?|&69vaKw(dy)T%pJi z=HoH?Ax51_D&0U4WHF=Idx|Q~y(gT_@$TJ2PN9CbhRWQljCSl4)H8NqdABvN{&Evv+$9IXUKtTtIU_i~$&WaH zokG5!*N3<4B_Z#$DE$282ncJj=T$$~A)SxHfwl89P|-xN!$1e_)=-Bjr8@-4Ll>|| zd5yrg!U;4jF2oTh^?`e9Ca&s~hHqtNfejr0NOYYZ89D}e^{X=R{8CGhyY>bsYkG>M zd^h04JpitD(S-b$E~NOc1}{az9tJOvCyzI~K=Zim;9+bCSc!L2G%gbdVz{t0R1;MSdZP^L|=<|o(J zH1&X+4jbcHCl2w9kLnTLEJsM>Zh(P}MkGJ>DLx(Pjl+E7z}e5Y@T8b9yf!dZpm)!N zY+k90t!-4X>01f1W6>ILb;?qbaxw!rES3YWlcdSMZ(H$rg(~*jD@RPdPU4i==5WW2 zN`aQr+onAOI>gv`I&dl)1zle^kl-hGux9HB&^T_&)7i2U|97Pqynl2auiktE=Oup@ zXnGrfp_98@o^<>0g6jQv3R@22aUT;Pa^V}EG(`4Lf)3oWBK1`a51g7%JWMNG&PF^pEGgg`rr$teC zc999JFD>Ufx|U?YKtH(mEtlt|l*@C-HYX=~^|9-v8tmZo8uS!9fTxRJ@N&IGpvTG- zP>BEG;?xiLy^ICd@7RO`Zic~)hh~$eKp`-zz__nm4r=Qj<30ZS7)9W=W-Cps$70!_WE^{NN;&$WD=VRg0WoF>YRX_Ob zn*uB}RRcNt+U}CaA4Hw+~2#U{4fL;GdlCukL2|6QZlZILcctelVnZ=yN4^8Gl zmpU0T=Zq$dj5maXx%rK&;0Tb-)*_=_TZjnOCaTvD^F;fsNFH(^?{!T`V4)GLT{n|V zY*|4f0(S^jKAj7Lg3jR|;Q{1e{#z`*U>g4RvlKg0BO<|bB?Ig0fmTB+zN{Wdj@{m=F18Z-BoR%KJ!@8KJPd%C>-zl7! z_|;*yB?wDt{=nVJGs*K?A_6TtNmA3K0Yx-6!5Oxi@TQF#{9;*yf2@)uBPaDq+NX3u z$ekKsU+>HHlBU3})L6lba}L0AToP1G9TH^r_u+i23B<+Bh$QqV;jaoOF{2^@U&Kek znQ9B6-w?%RCyxTvCRNgZ)RRNxMF<8wjfhO|B9c^+if2v~f=uN*z(7|F8dP`zRpd;T zIeEd|x~pNGk~QoPGJ_h*&g9GLa8mkd8cA&3DDV(qd}B#4i9P5=;>TL?nRmy)J3oK8 z(9Q;SPvh}6#qGqqj1FOsM6RzFR>D(t&?av(y~uw#ONg3;pNqbxI#jyz4y^n(oiyOd z(7eb9))>pfv$>UkcU}wr98$uUQg#E~faIpl^%&&Vf5j`0dEspd9iWJN7g|beA$<}G zZRSsg&xBmRS|tk`i&P3qQmXJiP36Xs=IM|GTfy8P&Tye`i@@Ws4r!ANCBC*dz{iyW zToPbFroEZeG^hfEJ~bCi;?3SX5by;deGKk1best z#7majkoX8GsFt`J6g{2ba(^gE|h%Ai&`qh)~oaA9xGNgcrKxQ;ZrETyrLGI@b~VyRF#nl@&A^;E|+WOA_Dq z2X8J?BORywLCc|fP=09=OjyfcGv)zr=1+MTW3nFqZ7~E-M-5<`LAk(l(3d>B{26E6 z{{%Q25?FZikf6L_8kwK02=`10gMY6ify|~heCVDGa7)Vq-gQTDsJJiw9$*7pHo8Fn zbOms2w>7l5MDYrFJEC@~7-+3%#p3hJvHZyj;96k`Ud3Mr@~$TM$hr(rQ<9A>Cg{S! z+YfoVr)Lw=I|6RhrZ!nWbHmGqKY|0AWjMr@>xjzD!fOR8u)N|O&bjOa99HDw{rBXd z<@1h~ode|LyXg!|99VdL;p-oqvj;&c26xU@nSE+5Y42^Cbx&%zUe7g>6EmyH6+ zC|1V&cg9$5r3kS|C>IQGG$jAc9|SuZE`pPW?Ks9sx{KHG%&JL!uO>;oFbY0NdVsn*S@@uO5V!1q*!0!ywqQ|HH276ihh^fVNNsc* zk1bY&V!JFs!CyyWv{)J(lT#sWp+7)zn=Y{oD*^5Kh1l9z4t|!GA_XU8$s)P)f`?~{ zu=K^>xUQoDSI5{umn0+d*oph@5ex3^{0Uk+HOS2m;Q*g#7UT_BkVwVrIQFYQIBxe1 z&oIS!Oa2%*9ljPkwAswFt5b&SM}7f$p%FPEDMgC5IN_+7mawbPgA5se#}y%|_}>~S z`1O`M7+5eDTuGDSrJsBN3i9mW@bX)N!F2$?eA@>+hP~iM5k0UhT#CFq=>#jDZv>|O z9Q&g>3-~l`1&@Oq@%&k}xMhbc=$V<!yb0Wj=XgH=%R(pP<4Tk#5&a+U zLro7>{zwFNlXrmK%kBf=d&+jQ_HsvH~&6UWzo*MW8a=NL{vsY1o!3&k}Xo?Vz>P{w$CucRy>C18vt(r@y)gZTiUIoUd&EWo%!9@G_ZQ%JJ z8SikJ;WE)&0&Z@d1O{$TCY~;m@YHcLa%jSSfgnH+M-K7e$+#%7I zCYXTyTaSQ^R9iz*=~CQ2)tMYS80WGvGZK8?t^+l~MjM9{p7Q1|(IIao0>JO0AD~|U z3bv5bC3_o}1BH~e_`l*&d|Uhv{(ANUHh=IB=j9+=q!S3E_5+f4%O4i$%D~ARcY;0t zeFyXM-T}Wf5gabr#dGnu1Y2gE!;>m4pjqW0_R$#;bW9t@1wa3SiFy-&g6C7b$Zs-v z9P5UET_D)cW+yi0SQhRxb>ZhIBY1V&VqDqfMFw~tBp?L>M%Y?4I3=n{++k z`UBjBv9zhMhp`|ZRCURp)vL(eCAm%VOC?}#s6IaMP#YEmsgfm2OK@MQEZnnR96A>` z!1)!cz}8QppmQ2Sbl)@NN5*@eauk5S)~ zg4PUc=%u31b6oTjzxh5329(7?tu9aUKGv1Q*iMGB6}i~o{Q_>ylPB3f9Ka`@GdZ|+ z7G&SNY$}})Ova}rk*2|X+&>bGx4SRrIR+O2sb4O`j9vKZHV!4U!w6ff;>%6;*~4TgT4cNm>#{5tUs*_ z2P!?`ATt-XmwW{YQGsA`kui}^_9Ax6Ea0~#a?mD0i>yeROIj94!r|XNKzFM#c_isY z;;P00v(yA;26>Q=!>_QGpB$08a*fB=trD327fl|_tOM@-Qbc^t2e9I%8Pt2hCqrA; zfmv(v1pgLuY*;k`khw9F^mcB*kBTSb<{Asg3*9X!ka8hcdL4+#15udfa}Q%?HgWIv zBkDIypuLMXiIn1=9lwPb+FBD&u|GIjZxx(!=nQaMIu{P#hy2IiCJdxBtz{Yo(CZU>(=e8tLYvq_^@ zDhZ17Ahl;N0)MmqMtL3|_B{0AQhx4msO~EG+o6MByUinF_8iA=RV2>(>PQN= zC&IDC640IN?}v!%f(v>sz;fH2rfUYHc=8e}kjSw%udjB5GRgr!!fqUUewzTr83F#9 z56Sz!9B^6uxq z%WYuvHaD2HR8LTF<{Tz-e`3F_kFev`MI@|yF7!{egyJfvakO6(W^ILn>+<#_!yyDB zMN?eD^&Jya%}8-?KUm0d^<<_#6!`c#!54)r&wGbI`Dkj)b^PyOy9WkP@8>x@>s>GK zt?B~9+DahICW4gsNW!dwa=dzOouHz?g*0ywfws{Sq}1ORTo=8DI|tUmMI8U7ydcRX zEQ{miL`(+X-@SC%aUhO&_1O#geELpR*scCtw~C14A>W|PkLqS zVZfZb;L+VH*nReiCdDV(FmlZZsB4sh-=|K2_f`Y|3;j*_dsrn1IC=`}CNF??D<6Sc z*B;PNX$Ech=45t`2R?gUlH^WvBjHxNK=rv@;9RUH86IEMs4uGp!o#%5klZ<3Ak>1H z;4MBEQV0Gk-43V9Q~>6}FFa@86#(j1VP)^{c+1o2@IuO6!3CZ+9By9+4*SWFDT(Dk zSlNJ=CVLV;kGt4yeJzfh83iS`9_H=m81R>+^x@}jcX&TyF*w1!%Q3aGg4Gp7khnEQ zU^FEcSBBrmYc~ahqYvewSeY(S)P97E&bEQ{X^zlnqYjj-bAhY1op6hOJMed%N!Az3 zk!sh~@Txy})ifjG`bipInC^rNjv}&L z#+5V|d%#uJzJhU&SNNk)3f`Msiao;>$?U!6(DVvJK5u#gI%Qr9em^M#)?0i*{7qfb zHN6zC{FDStRqf#)k7n_7Dp9_=~OTeB?J2L0B7q^~> zlFNBB$S#{l0^2qd;vK#N=FYZ(Z$piU>b-WC2gY9n)XInV{o4b3Li1dB+tfDL2>3Do;mU)i~W;8=Ylg}F5ihL zON&9fa$T^h&YBbzIl}sUa9E4JZ>U)^y>%R_L#N`^Nmwp*Yy;ssS-rtmh_%Mj7?2GH`!MEvCaL=w8Y zg6G*l_XOGsZ$dTAZiTs0!Y0g7-uv;*w9B0_5X?11CKa3C$)h>uYxtTsiJ=!ScP zY^802wR`(rLU#Yd%K2(gdD$S2j-Sulcd3Z?%6SQKQC22;<|nXSas;q%7zWa_%$*L1 zegQ{^A_PmOPlAt`QqVW?D$xD22NX4=;=m09!01qzV9hxLP;L4NoI17;ENsZ)eRx(0 zZtYYfi-TVB+|_U5fr5PkbV>&Ah|-0!UIgc;_F?drW3e9Ldf+W7&I|Wp9C1JfntvbT z9TgVhlO?ts+j<$}iw! z&;!hjv)Fnmw+6et!!9A&yXP#q+Wn7h7Z=- zCkm2c>OpwXJ22Tp7QT4t2l@k~$$u|6qRY1~Y|}jl?5S3RAC+9el7Uwse_kgR2^s)) zQtI$gn>tCHa%=;Y6i!-j4Huhq0hfU5z|W@?FIRYqb%Q35cR7l1OJ5(Z4xCQL4;qlaLtAk1byYGe zCmL4nxe0;_8F*p=4|?7j2VvWiiFoL7Tt876uT^Wsg?dY1^WYoc`<>#J8*y;ij11h^ z8UgoSQ74z85{d7f>p<@=zy^+xRd=B42jPoUOPp& ze1XeMN*0k6y?Nxpu6y{7_9>pnhb|x=mIoU`(ukwm1{WqIlIW~Thp9iO!sqEWu(VDQ zK9}4^B5I%Drdb(e*fEftc)tz$*q?RDR?dN%wy(kAz5<-OIh3R<*C0<;)q^iKZjkHt zkQ76qZtI2VZJA&h72XXUm z2mEe@Dv7yo4j&%TgO4I#gSSp0@XIb1tTf(?#aI3UtJE{FRrpMviN7hw9li*}E`9-L zOkH5cyT8EXaw&MWRgB0*E0Zoa9bz$~Kyc)O8TlJn3f#^V;K?<%Joa7~`I7eutX5n{ zWK&)M!5U*|8dZz?D#fuk&ysY|cmi{@;Fd5Y(tC{Cr;qBB7t39N|AP%MCus=Bs>BiR zACA!ZgDknXs03%}s=-k6P$FGzPQGcG%BDUY!G# zJu`q&?haT}G!2*g_W*CvLhz~}8qB;sgIID-9QR#ySl>K|wESle=LBn!-96ewPiYta zZ%_w%O;7>#&pgRC3mxK)Cd01SrEsLbAHNIqfKH1t!8XMN_}yaye7CuiSK0HJTgw1> zjc?)X@^lgt^AM=@J`q^GPzA|b%>h*5xOrndY$>M*eIL6+pT1F0{dp@4FI`A(s#?R2 z-nORsFT-J*({A3CPb{8($qd>LZX$IR$FOB^G;w|62jkDHlXd&dbH8P6e>7`|G%az*#l|eVS_`fEc>z67xWqlaW|8^N1_~r&biHMSe(`x}b zs0m;1x5BRy+=%(xSHK-Dfe{M-K;j%PD0@(Z)C!it%>HoLcfYcq3tO_{bK;a)uKV)QYrE*NSZ9YFoP(l zr;_WAvcT{k$Dev73#1p=;OT!BkyHC3V4nuRdzvm&Vxn@W1yJ>a2ZMU5M>=i^{q zJ0NBpgDV3jz^@@eL|M%o*21|&e#v2h#R)4gZAXPmY~=)(@Js;q4R0gOQ(y2#Jj`KC zbTS!Q!h_M`ExbNCQCMB!PG)8MLa#wUmbTplbDn8KPtCnNrI4IZuS1jnl{;5O|fU<}mB{F9NeN+Xy|#Mi*D zh7nM+)PrnooC2Ecm2l1Wb3A{(9l5kQoV=cC4NpD~hK_DsSS=nw<&$#6;V8$$fl=go z!zbK!wG?-5UJN%F*pU=DO*r>kKYkzJLU8JGf=8_3=Fe`#&_CUiDAudPeTuV5#FYv#aBvfRlgxtlH->P%K!nU! z%ENs32%!DnCivo>2WdL9oM`Tqg6@*)aO-kI;=Zh1Fn2_YOgj!?qK^d$&X*;fTh9R3 z)=yyjWd+i1DGKjP!x(f%_hRRWjsmwk6_*QA6V{E08FJk!CwU};;8Km|0pWsZzC(f zU`_xe0VX6iCXqBB^@CgXhL9VJoAEO)4@`gTOUx9NpugRET%Pg|yPn<$JTyk}+7<`s zJ0?MHpEQPVTf2E%j&$MPl^fu`;Bs)Pz=rtDF^1v4ok-@h4d7+1B}xC}3}ZHwgRLh| zG`a2z;qm}A%*0pYVej?OasBqD%|#O+SAHTNwnf6ipSq+WSq5tFeFC;-xx)9ym%!at z$#`o>EZN|w0uQfF1TS_sgVO2{T)IOHY8E@gl-stXVFn^yJ+TfO-a3=RhFPHd;b-7= zz!moYjDRbylw(tg*55wj(rN9tp+KWwLS$)_tg z&S@e%bKa5kWb43fiOHbd!W4Wcj}GCsQh4e4FlZ>@Kw9T4gi)Jb@IG~jl1B|Z zsJ%8GNX`0=o92gs^a_rZZV#d3Yh~DHz5{;pDhC!vV@bX33~)u(3|@?11ePA%1ZwY@ zz#RjjFtkjH++XH}z49VJ+O;;p%{E)I%xETLMEihU#X~$*p{nUo@;|}Wt{gD$#ymKc znnL$8c>?QsKY&RK$0%Pgfz&fuz@0nG1#wLCf7}_TLcx_MGqp|dXR#r1AUfnRgWxHD zLn3u9o#220ppqetKbU+0zsFa=u%bS2a?};)*6aWaLoLYU^liY|HWMc#*g=sP0KrQK zGHYEh7^+f$=L;=>_z^CfDYYZdYV=9Dwh4JP?R10aIt_BzR0|$Ez-3?pc{nHewIJh? zF|_3C!p)CONa&L{_#DR))9Ag2e=H{^-Cj?Bs-5}nT~?T4QF^F zo_}~+7nNX^_b8}|Zxu{fZv@@-7vm$7xEyUlkD$)E51T$FIQQTnPKxq{(I)1=qGblz ztULt19DL664|Ile9=7wSMFEdpv{X=1eF+%W%aiuV8-lnMgf}ewDDX_Y1x~wu2OCQc z($ z-eM0DW4(bK*ddC0{u+`|>xpDM>@qL7-je*up23rP<3n1q7n1aaIY3$x5=XOO++MN< z{x|Wopdcx|X}!H7NkgvWaf=5iN?8rW9{KQk3qzsFlR^OJ2SW9OfABUR4YEdp0M=p_Ht4M}kv;Jxl`!~aP|;}h3k0^xpLob*|eEYO^T ze;=L1x$QV!-IXY4HRTR?^}!Xs@-!i%-8%sLS_)h{qDm%yZNO1~B{=8JOp+fHN1huS zLx~t4io9ZMTeyp75h?{uH~PW*Cq{wC zzbNwb-(0YwM*~Ji?gf%7HOLoj9{hfP5lGOPLmVwEVQrNzi3~O(p5xx+%cZBhu8v#S z$zK-ggXNsN=Lj&Blg2A~zNF%a8<}Yn4eK|vI7)Xq(S9UDRvvkb&wDFF$rF6|<<4th zJg6&Dmz?@UHyi3|Gpl|y`^4mKAYqF>C#Z#&z zuge$)M^D0HQ~W`r;&PaE%pU#`6(a>_^l^NUq2S*c1v2g@#pRVr*jGsf)?O1O&H+6* z%O}@mcFzFz`OMv~`g5GiJiLi`_89(QSS673*oo)db0+qN!T8u2BQj^H7*3VeB&zZM zaQBT{;08Aj=od)^pUrG|2WN7#TOScPI%Wtpw9H}OTm$l|nBYr`2f@pWt_0s$fZO}~ zn<|b?!_;RE$qqe;3vS3j9n(1SIAP`yG2ybXPe z|HS-ky7}xAkm5Lu-(oU&WxQU{(HBT=-h2$E)}@f0$2ve{z8A+mHHRs267V4F2_u`| z0-4WJK%NnYKX%T>vzNR9eq%;Z(u>O_Y|Y^1mt|nMQxX0Z<%3c2W^lIG0$OM)Ldln= z-|;O=oQ0k%vphgy$+yP4cDlN z??rZDdIX)3V@!EFT+p#OCz(Xwd#v-my=+kj5Q<)qWWxUXF;nk|AhEGRbYE;5%J8pa zE(-qfV?s@7?fa-XGu2T)tOEvk&!Ct?_5bG36$6`R;Z4H`);G zD^(QE*3CdNar@bKCruf%?M>_(B|a^hep~2wQv!)jj6?@-0NTMhoexfV$~bntrJ-;? z8rvk(JU-_PwYGdmOYStFDEJb+y4TEzwpXHw?WyR#yqs%7o+`R_{{#Cc{Dts-Ycl`) zq%`5VTwT7aRU}fJazyCaXvO#ll8_D7MXSeD==`h8_-KX%s+skM`79fZPFTJ|>RHO@ zgGe|sR$0m@YVg>ALIrl)`)iDaWCaqP)JBVM=`ekwVd%fAG?e9JgO(}nV+8ze)UI}h zdZ2X9JcW^wpAn68I?I$Fk3~@e9;$bojc)Dr7xp`Lp&7kT8T6`In6^N;zM+9F%Xo(ViJQ~a#k zyKL98#sN)ib%_)j+IX4Df3QT6M|RR-RjzY7Zw*~^(+us`319*Xj?(+lr_s6>FBtpP z|KsSs7)Ba*aL z8rp@1_V`LvexJwhUyu8b&*R?D=X~Dhyq>SUs3cUx${$#PelG|{mn<6SvBPr6djpsWdbMo3*agd*&&mbtDx zNpStwRaU7d32mDeMdvLHV85S^V=kKZp!Sk9baBxdc33Nv@zOUz8qbW8=5cp{e)wd8 z%?fqF`*(+ty-Nbk4=YBWt9R3O7hROI-HkPRyAO#PwWv>P3_t9YG=2EbiQ%OsKRUP5oKP{5ajmthsuV4Yr0%zE>JM#j=-99W!7bYK*6${1}SA2cq`L zC*jwmWJW8ypHW!%fUzpwO&_n{fSf;O(_zgl_Uwf;RutbrnIH14Z`DSaSyBqEPkc`B zH*^)E+!an}+SVns$A2Art#}dw{7zA#)>1m0K8E5aTC;xV!jRgUTB=h4P&Sw0zxlzF z_4H|gX)5cHchM%An{EbQ-xyB^cATQUQd`l*H51TBgUPgd=nlKK?i~9q^9-u^Z#r#{ z(nQdq0R7%6WM=>Dr;8@r(8UvzS)*THm>X@WRL9tgURJtC4U`@+E3YyXgso>=er==9 zswt?(ub7FLs7&pn+)<|f8+P(tTRJSaou1jE%1(G|Mai5;boA#WBxo~6zruW|;NBEE z>Aiwr;Hp1*s#b^|`>oKB8GFT1lh%%-A>o^rZeqKsBr zSu>9Py4003TlWndVJuVZkW6tm`^a-9n>W*e7LHZZi%cDw^>~QAje9t+(2SiQS&QeozHe;Wq%pTRg5Cr zWw~_a#Ay0&elmT3QkzNGM?^}&vUFL|5tMnZ4((f`j$ZFgqdgY8L`R24QF`eByir() zKD}9v6t)5jNi zD4`{U{WQ1})z8jjhStqOdM=}^Se|_Ab^S$b@%=&OS4}j9Z={*Dh-BJe(TvXYeP=R# zKcbp@)6u@nr8IQ%AiMqKE_%ZF62lArM+ZLX(}$DSv)^L%(M^|bW^UXx_T$!AwC-3A z65Za0;)V_~os158A5x9x=T4`~96IUDTZ+t?<4x@PCn{*u;74YLbfwu)c^r>`c5ug#~P?#(pi>LvDlpvpDEEyZ?79>w`Sm>5twZb>0-g#&&g7q@BQ?J1g6&oozs07*?=@ zgX!q-MiKh4eJ_(d%a7JI*rCH`Gng5h3TdEQGD<&aNwd8D*n_c4s4jmS{r9bfc^|r; z&Yb;_@sHocmdmTb5M^T;;^oR1EI&z$67%S|H^1oEQ7LNEw2MwJZek@h^O=m(0a819nEkPLJUzQ+4S!YXFx}YM!T4+vrv~ORv_$ltmNdDw&SWl%4i8(Q?RgC{#Z~&*BYGW-wxvZ&025zuJs)|TW|t}h0LPQr@11= ztAicwYC|)2c+v*WlG(83EOH65V^=Dd5UUkC=y4}2=6CiTcAuvnJ#|!)dVJ|-o;Vuw zcjuO{XH@)A7+c1=mB}OLmm~ZyshL>q;~_dWWt3eaQIGOa7kk{inXxtMWr12Fb2x81 zI;QuEUOZWew$#^(a>Ko7ilr@T3OUK&e?FJ4xzUHVO!$u4oJJca}Zof!$~2KL_H zoow@&wMf0-KI8m&9epq=$pj3@(ALQZk<6iG^uol8XwtWFt^BT3Bqn*7i3&Q$9*)eS zMty$lxZFY{c$Y}Kj!mO|;?8uJc{2J>cRT-q^DWkCh%-?tV^G6z9eZ9)6AimoQDOE> zw6l3JWzA=zD~qkr7SG9a@#k*X-x(;HQ{IB&&)L&?KT6n^E>HH*9VPZ=*%Q?^JJEmN+gH?O(Sllw!U z&ZlqedX=d({!uD_&v72LKbA+|`dHHK{7Fp9N_muIzL8eVZ)UHgnxNcE9W1`00PgIL zqMM|yF{N=8tU_D`l_`x8S-5tf6+_c#yipW-9bZc8c;YDNr9Csp*^%a*cH-a4`itb> z{uNy`I;(^Tur#b0VRS`8>P~?MhBU!}$Sx#mpgQ zU)3yhd22Ot?!QNl1Qt{AtL38gd(ShAWcBGA`57XckQwOog>BS6K9eR6RI{hjtf`xh zCFL|}_Fb(zmE-Kgxwn!TS&QRz?5&DGHeaoE%X=y2dcgr2+NVsX49YPc@=Ah&;v*c3*P|Qe8==W_5ltw*#rh^GprP+)*@F|7pgRhlG|c)dihD9dH}3qxUT>+T zho`#p#wSms-R7&%dvn4`K6>cx<8$=NKLb?t!-u{q-i-e07}EAqJ9=y13%1}$5aYUM zfK@pYhC;W0<$E4BqJMrJr!VaeQ!5n#eROb;F~EI{rS$k#hd1e*N1Td6{)V#~hYlc{ zw$Jp*jCxwuwj6!9XVkjJ1f%r3<1?PXJrFJ^4LpVC`%cA;hC%c+Pc z(V{ z2P+`w%@f(y%A3eHKwiKzFJc#JO%>Gda7NP)rc;ZBMnCiy($~5hfgjL9Mf1v;M+IeQXR12tLw=q>b^@8(AEqB!4Q^!R-#RHV;kV>Kp;;-VK(>7{p=tS9HHEgm3uJml@~P#eQ(N zr*#W5n3G&>leo1v4W8acKc(+s)ZtF%@SDp>_C`H2dp84Bbi85CaQczXB#HQFg@ z!tN-yK_1h!xbx+A_Ftf;?xE@RX&LJ9z4os9r?h1H`G8$3r|4p`;l~mSRz_)JrjM-^+k4UG;LZG zMfb}*qFai0(;3E!NVk<~)sZL^?K!iVL7}@*(f3xmdzCJ$HFk=#9`7Rwx7R4>aS!vQ zyc<25u^&b4jYK7;55Uf`zs#A9O047l&1j(ZJ$>V8OVa|*(}yAE)OWR1YyZcMC`2wB zec+t=D^D3luIVa`7_|^3P2&UJZw?JAhJoMyz7_|;kZZ%z0 zfzHhuVAaC1k>7!OWUaD_hC1d@#}i@n!iOt#_MkfI_-aSrMW3N3JocjWXAQ{F-x%3% zu3$ATU#Fvy2J~`K6I#D$jDFs1f@VFNg347A=+VjdnU>U(^ls2aM*H4gR249fp7EZ~ z=1t2$$J!leRj(^*i0$B?`7@b@tFj{IjtL!~v6(&n@+@;MyN305$)N6rOVF;yCML#6 zM(`_4jCS%1Xz}trv|}Me(>|o5+O8%vxH}8|m>Y*0&)i1WC+%ZM<$iQ#kt{P!@*uU@ zc9H+3!U-AEiK3C&=~T6>m@b(!msZXaAlnt2k(+rD(=jSd*DNYw`aHatx9^^z=_*pB zAm%axCnmHyb+qumC(q_*zVoCD3JPfd&s9uurHJb7y~1|JB{2S*>X^A2KN#s7igb2f z9=rXzHu5cc!4&jFQU7uE^pxZY)@RLbCidGq{!>Fq!R$7@*62?i=(X?~8&kyj9G*Sw zyLuaHcw;iNNp}^~Q<+75R~%&J4sdg)ym3N=ELi^oX#D5j^njT<6MI95bk&FXk~Z3?ku*@xH8YsmoaIb@RWl_y zyHFw1K+K;Op&Jutv0ca;JyXg>FUvcafqj&&)ZRo5A__z!AD#K%C#*!KIs2*8#7Xqp ztaz^XqY;IltYBXrJw)sOo~5Y=e}8GSIBW!J_HP>C12 zX~yej^xVt0h zDYPonS~Se$(1YjZp?lUsQYR0gZr&=?>3sryWvZBUkK!rsjt<*AIThupJYd4y&#_Um zQ_=gh9I7aGjyb$pzSXNVoNfHhqi)^D(0o-_RPl8`4K0x-Jyd_g|7 z7B@y3W#8EEv0IP{M;@=Jx2D1`8Pri4vAzLQTc_y#VoVeiSeNnXOhl$Xs=Ip#G3mpM z!j!8>?1u|-cdDmJvkTFIIwg8HZU^&QFaue1{$^UYsG)^NTWPEKBBap!f;DorMz@Xk zpiRM-kx$2JX33I;G=Dr_AX{!t&t)LWdnzUPoBW!2m@$Pa=tv6KXUepC@jNtP{S^9o zx;*_|Hm>#ir2&49m<@ZXqX-4x=6Y=g4=XSU zM({%)^-Cnumw$~!xntdEe*Flw@-9a|ek7t(tE!mzOF?M;;7d{dd>Lxve;(Z^>_*Y) z7U*AZ7E7d7zi zFB>YYDp(d?NE?@0iJa5-^Vg*8V`4(fkjegdYWE|T$vPyCzFwGsk|HNDVu#vjqgxr1 zA{s9miK#N`=P<8j*YZ#7a@W6-Bxf0-n`!^~LMBr4HuFG_QFLJJp* z3ua$a;;!{Ln)2rbb5zWqmQ*Dn_k~Z`6{jt!u>2%TN3XJd+J)5ah8q=~a}d4j7BV9; zAJ`1-T}WK&5IS(Wg6}r&0vgcC<7e!6%6Bizp~oWnsoU`pc4MV{>)jWVX-nHoy3}c6 zYYAsbTy>yS63##>=8d%N;Q}-v_o+xe%@^74>R}3o)95dm_snJ~bJQ_gf^jE@n17OI z*xYJqW*J*UXU9yUPO~|S$?g=@{bE2Z)E04uNC`Eewy5le2b*u1i+;&*Ynp{EXjR5m zs@RxK3;r!bl2ranMptFoyreZ+4;6FOSiGygFpwD%-kl%SKd8DI3$LN9MjVJdWz z>7IglXuE6)y;Pfm8bc&nYaL9G&Z<+$=-VQs3vO| zdq^}T)rM|o%IM{q+4N~v8oeKED;nyrM^`wjx{-6KV33=FED!gY(c_HW0Ke ze28zv^@a3B8_{v*r9~ux|7QH&2+u!p<0! zaOfzFd2)qKn5>Vcx5@~P-ax40k19X^(m3=!T_2Tq#;^~|OlXpiFJdm$qd9SgH1=`` zb7}Wn6x34Bd})7xX3Cl}{0En)`*4%!gt-B2`?&(`Zty^(A5zgj6Bp_heSs{Ik>-m8 z>d5TqPv*Ul+XMNkM%zzM7JRRoMh|7QBO6;SbYMp}vnl%|egFCy>D|_c@NdAp)a|49 zYzZ>>Kt%PPF0@W}Cg%%YL~hle**}XbP)>6++8!@SUDLQ{1MOs~!wAD(F=L|#Z;19V zB}n*2gBHF`r|ut%*z9W$n73c5QRdB3R&AIwg!#%eBjga(=j}tEi^tJxwYM0rKOyMq z{Vp_q>KOB_sDfFW_YC=prXl?X=~kR|pZVw9Ko{1U^Gg&r(^d7$=(GjWR4e)}de*>o zhD?>EciNF?mhEb~H5s;+jLoHzi5I9*NDaEVGMDa8Uck=SyoZfl;(>O&Y8G9Oe=GFW zGNWH{4O-)O1@)XurRUqD8IQ?VP*H(CG7K`OM|LG6#f?kQQit)(%q|uGc=y42q|t zq8U{v#{3nw8O~+BCaa+G%7rN6d9lbnG>_dhrbSnd8Y0Wlb|y`7E$eWm4IMJ@VVo0V z*tJ{sAoUB<^lNz^ni^t_c1RzkXS8lnlc^M~?R28yi_+2d3=?XXGZEdN)l4m?w!#J3 z2hi*{Czz^5e~@c(EZvoy!94A9LZ4<9(40j=)PBo{-cU40E`rI-&w0UUn3u=?nsSTv zK7WetT|0-}s`$F4j% zjqm57&mKGRjCuR*AW|4O$R6IQKm$7V(d(ZMGNRA}biR&0-EpZ06?CQ1VH0nZv%CN` zoap_(7t}jjNOQQklR*D2d#$~Jt_&Wfa}JuLXQE?lm8l(PEsU{=x6P2=oleov>?6zq zonz=u0V>K9ZQ0hc4!n(_m>Qv~n<;c14}V2_R;Tgv?WdrCeT|%VriQ*5a^C#k6g0;$huL;! z7B#KTp-vQ|bzUKC*BqE|k>!#)~-#jw;f5*4b_d!Rf@r6xjq1twO zM=^t*LEGq`_ZR7k*mAZd4xpghE719pT=aaa4f1O?ppnP8f%{k%nz~YkSz%I*F1uII zuT$?aM~+6Jnj&3+Pqivir}s5ydVg?2cAFdaK|zvwLhA3cSA-NgY_iDu?JJ z?+hWet`^XSnmaVB$&J3h_LK4Ze4Yk@ToiHJfgX+XM-gQ!s48bD4}Vsq&MOOPj?a5! z_cEM+xKf2(VvLabAQ1Jb6ceUd2boV6u|M;cu{yO!7&(=x$S;>=-5wo6NzF^xpqEyt z{*eqVqc_-PhX6e(n@zn_2iT$5N@Ut^z}6a_W|!Z7NL6PL=F4tfbT4{@)+lo`_}I_P z({9Ai&r_sJBY&gd)CKhFun0-jJD{`AxOcRiEFw@dpaRPS?9y$fs6MX&odTPwa#{ik zj_g2_m!Cmyu_tI~Q~`}qlW+A4mumg;UR`jfb3B7KUPAr`;%ukV19ndYw?n^uPQ+u=~sry>Fe%PZ@DV&I=Z!0I^=a z>{=1sf1`?uYg}OOot(>T@U3EnOO(-y1rmai3$M}VsoZ?#^c2CLP!E)Q+loG2Z$h13 z97g%^8_@RRN9-R)i>|pAg{1EsM6x)Wn)uCQPc@!JAxPk%@fN~n79|7-2a(?7BII(TfuSGp}UrEYV42rNb`vzd*1a1@)RqepU(%NPo}YK zrPp)jbHY@Vo3V^myArlRQ=QsroTVpnPa(2(BE2^z zhN^z|unG}f%r-l7Ml59&lGuHnCVBJeTMrc)9Mgj=(&X6kGC$Nc4+&Z;)=~9heJ1?O zERM5zH({MM+VHm11 zAj4WHSE9$?lZqz76HB5 zUM0H152pTYmF$A73#{qJP_*^LL-tRs6cwCqpl~RkK0ft`%}o%AmN1J%JJ#AG^N?C* z&C@GnSM+fjdn|~4c1TANf;-Fy;|`>js7H(2S5SRn59^YqNfVVh5t18SECv`-tOhBBeTq@|heUeePaQk= zcAse9`D9V{bb_KDx*_Sb3v|za++k(T*H3w9$zSqBQ}8fYkvre<9Gwo8Vtk5{&^uKx z=H!(T_V-9MMeGNvJN^~w-^fzy%InCY*9pxDM@)))3R>B^j8T?S6IlK}Lz^C?&{WM8 zXy&ptsG0YVZ!0-TbpFk8(I4ae$hUtzy_|WJIsFB*ht8kJSBIOK_a`O@{J%e9**#_G za-uR*(s_%2{-K5_F>E*O$$G$U%k*G-_Ha((-3(^@F9RC)b}hZ7ID$4EJwUB~r%^`c zpGd%U0liaD;{P5yk6!nlr!kds=wuVGwK2?<$yN0gH4QFCjmce%gPAiL37kY%-1x<2 zmENYA=fjXrpb0%TQ;d$ZsL>^De7Z$m5;g3}rE#}bp#2t6C~W&>cD(){R&IEdLHQq; z+~YD-{D}=+)Uu6j>b0fcrVq1yJN~fYg{DaN=x$QY`OhZO%@DH^t(u*B0(( zY`3}41^P-9D;;B(&w_NL*Hnh*UB#Le=QCo%jkMrFBj2y?Hf!tGOCN1NNepo1$4 zY2n5Pw0rdt>YR~HLpc{*VkCzjopYAXO17jb`+qU=ww3T^)mjuKbf+0dVre?pUzF%6 z(K_j8C(=*RrG}1a*z8jZcg9SCDkm&rT;;vcuq3zd7Z;DNDTN}{d*`W_#!hBvY}$dBJ>z-n9VM0EC2I(ESVncVxt=3ewfwP*IDSIa^XbN{5M z#Y>fX&5q?z%o%7fC7DK>XVAv(y(qSHJIz`7l3DlZH~W|RGqU*;TG#4XAk$mH%-*g! z)VBI0GuFov-Gm(a{Id$0KeQetby(9nxlp?GfHT^n&N2#bFR^0#_c9-DsY98S zxCAd0^MI(wp2+ZJL84+fIlRjoIJ()uk4o!=p`_eF0Br48S;Urwj3AQ zA5{l3V^KIEVl9004?#A66*=FgO}d}wBK1q|l8m}N%Sz+Zxy@P*+DGHbUz>2R6=P1pF7 z|2*~K@kVDdI$Q@DJv2!8@4HS^6B1#B7Fk-1;M*2YA~PSrOY_%~!O8CA-c%p>V^SVo zV$7Y*?(q<6Hu{oB0xJ-!S`R)V5pZJ6i4*^au)yUGxcsR?XfMafq@kB^(FG53t^WhA zKbXw>sa=lSf5tlRDwTwH_d7ym+vzZ-Bak>ed4SVKCIWczKhRP7AKvnXXug(Yd-^F@l>EI2$d8~p5cEd? zW{!Hmta0_YMcti@dfAW#XJ*3V|CIx~E-h%js2psV`yVc?+Js$=MEIKY4P1ToGR`;5 z1NE^@Jn1btc-H11A+kJzADHYX4@(3P+*BcrUJt>GCGoa20+D#Q2mDx?0kY1`gimUW;Qrfu9RI}ze7ZD&q~{dl;b>dVLHLI! z2C2fl1|F=qs*m@X-fU4c77)$RQE*Bj6bd<4r`&n~j7Bd7M{OsPn)0>W@7Rj;bH>)c zeo4~A+ti#=k&Y+Z+y=TzQaBe`5XqmRm>+!zd@&>-(Zd-m&eVmu4O76b?caH?3s-@K z%dUaf1(UF+&jnoh(Vx%>m$AX%Zcw;W1@1ED@xZ4m$7{w=a-M)J z>qy6q>*o-$+ONFUY+I7tc?nOHv4+)kQ^>8^6)oC{W<*@=3ime|axCL9Fp)YTyvgO# z?^(yeWv zpZ^I2OxG9jY)iG-U_kPT6;&^gQ&Zg3EX&qg+Zf~}u$+0Fzo>7P4! zcU+EyzO;bvw77HWRTJU*xfobyIS2*!pX27$wO|%!XqfWc;XmpQ%NJ=wuWBtaAw`JK zCc4ALDFk?p?ZT-483+TUnc zn7kD)o@fU3Tmr~;r(Upms~l+(8Iu;niNsJzspUz~Z{WMt);a3YL~wtC5U&`gjgu9g z;Bj0A_`kzPfX=2E=rhrq1SB@{(gK$dlVei&&-KZ0ThmhF!=33L8wkW+pCM=8I*|5K zXGnbkS#KN-6_p`Le76BUf*DZx_X4tX-Wagy=S-9vh4>(sy&9K&saebNDyV**LR>l% z;LeKe;7VUGEZM0F&wkW~AHB>$z%D6hVpI=SH9rm9mb8X19^Vu?YxqH*N^LUlf-k%}Q=e4DSVQ6AKzQlO zb>M5~PyE)~5X~hz@J><;`F%%+=op5Qbn}KCTUy~0~`AX!C8y-u&Hw=(OK~h>{+;!_u`li45|0R zva?n}vkRy2w(e9od};(w^WKX;J(&f5OB@wSbR^^U3rZjy>cRsr?*aRRH9#lGj#U4- zg{SpfksIflK-K*>AlGdW*!V;_JLbM{+LrwlpyJ)a6HipYiQ#cXf$0Z){U{QFHi6O( zX?Qxa8b@;X_Y=K+*i@ncJpU9y%#-c|x0@Lx{3HYC{+dgk*!tjd{ynU!=mr*astTWp zZt)~*^@yFH6Nyu40M^s}u}OaymaEwTJE}h8p><98(M&z^+w?IAc6tVW@t2TCo8v%9 zw-=n|bQ1Jw{=m5x+GP zPkg@!c7Icc;#O|(wa*B?avR|khjm1|@C>#XSp=f5KN0o%_D;^lfje51=wVFDv;!|v8!_Ua7rGZ>>b+* zw*T~om*(n_cTrrH*={CD=?W*&+M&?ivlOV_jlriMUjyBc1zIUQ_;a=-T#yw2dw&xA z=fW4f|EmmX&*h$z0l8T0))q!u6Tc zp+~#}%-^R8ov|?)mmY@~G|weu{Wzk8B!T=?Q|R*PH&9f+1g?FXL#~<*3xnpaB`wCX zL~~Ow@8sbPIQ`XO@KNsz_`NQuWuoO_EK4rpi4F}QJ1ZEzRN9YICEP)&KkCe zwr*aZ=P3~Qb}8v{kb$0c1V@>50_LtVlzrp^tNQ=p4KFW%{eLs@;BFpS>S$u7kU z(`10*e`i4D?EBb!f-iP6yoWvHB4Od5d0ZCP1jf~mBSkkafi16cu>E3Hc=X%`n15-&pi+LPwRyvnb{z^_$&C!IcJ?ZZqO-y4T)S` zB%FTN25w#URp|V?7K0s9(+-drO`I8Yx*yffvXqThRfKOAg{EMLd1 zqa=xM*BLC{AqO{W`QaZ~pMe9Px)4)hWPhi3H5Xi$Gzxt?;Q$0?3Wb z2NDq!^hR#tnQgLw0d1yay0y4acm4>LvH5|Oc0@rvr9L1$;0zti6bPN>MO6CjNR0kX zFfPm(mYoU)qDOA<()eh2{P-ChXO@fWv#p5UDL1%vs0{D@u1e%edvMk*bK*C}14giq zaA5LU_$lNXc-!Vh?iy^tyE^R2-+kPMWY+`SH*^+EpDhP@errka+BvvfdOk?D5`!v+ znox6*DijrH6Or*%JnrltV68V79x)PNFRyeEb*hs$;PDDfJ7q~yO737S90SeWHxbv% zs$}W;Q1bk20IYlO1Ji8M@RU`rfvf2yprH$(j^|@+8=3?pEcSzvOC~V*{d}@$rvh&D zV_=nN0NX#)C$|?BVhd*%8255HF<;(=6Y^7us<8z*=%)!Q#TF3<-+3@|Vi%sN8%eY| zPw&?XBa-58Ec__fihYvqVaL+(WZs=kV6e>#u4o9yx&WBfL zk@Cp9;CFi}d>*Vuloz*weJ_*Yk#CN0S#&O*vEVlL`nZ&&4tkKH%Q|o@MH22w%*O-y zn_+V5Ih7TW z!(s8&sWAA50d9ZcKtlGlTr{#A^V`LFdt=n0-#Jg>K0Ja$7AIokl3DQZ zEq@sG1d{J$IZRKqf)3&get2;T;W@~XeL68DqW33WdvQ5jeq}Fcm=#UN%;t~-1CB7q zLIytV+{n2u8qlQvBk*3T0e3rukz-|69P+VV`Twni9nD#G@v;{7Ohc;H= z_=ycTE0RyHr3>K870a>9@({AS?>KJG_k#Gi4CG?{;Q2@qezepEZ@k@$&%V}#|E2_x z7Gy|%BxVzv0W+8qy$l)(1myD$O^D>zzVB(8Km>DTjv`@?O(ls9SQV4NZO z5~&EU@i*YzL&juu`8r&z*oFh!65y{bYl!;pCHTO}#e|<6N~}yh;2N1;EH?5V=*bI( zs=2?hxBPc}=&><0Dw`&JE^5bHXH@b&Oi9DG(W{6~>>TJg=>g7t@enNI%&YOgwaNCq zPULEuIZ?6Z!A*zD@U$76RizL?+Krw$ZC!5wH}9GWJBDuHs>@RNlbr+!m}!gC2Cwku z9h*Z`SgqNm@rIiw3OW*{!!>oqqG^h%7QU`I* zc~j{6^&y^Q9R)g6jfv4!WitIi053Rg0{K$Pf|rxlg2i%f$7|-2 zPtScp!*4IJ$LI!-oFYlA%EEYw0hxHo;#5##cL#jfU`a|gWP{Jzk8t^2AL2TF9X#^b zhlm&D!hoaWNleNY;jQ-*V5H+w5T>j^qGiIN+V40};XV->?KB0`yu%^5s!CRxW`W_K zX0WGm1=$z zTN7UHew>saMke+8lZYkpFy*r@40MZyJ4dWY44?@K{j4mrK8ZNM%w zrNEL^h+G9y&3A2LVD-B+;_r@N|FeZKrgkS;S2^OmGiWiHJOct#p7mZq2EJO5 ztUKdi$+>S>9PI|~y~#xT$P9R}ei;e55<^ZoCzHF-8mdP<2cKv+mbe!UOV^r{IUa#f zJu`sRT<*ZWY{0!vf*n%o=C341$Ta!KtA&T!AxbW>ZJg_<(l2-`a zzO4m$Q5dL~n!?)lHSk4FG>nLvM#dGchSP{HaT#`jc`yWa`uV|iskg92);qj?a zg~!ct3`t}O12yW^p!C8pxFup6d3Ag?apNh1?eSqm-r5S1`z;{L;S4YpvjpCbW%$SY zJe+$u0-ktTjjMKj!jl49c$&&)7|cm_u1nellR0}k?{pd&%UnQ?D=&w`-hlZ1dkton zO(hxmIpoilUhKLu6kbTn1t}E?BxU7&Jalh1@J>`E%%&COzSK0{y11D*`^z@CV4gK> z`l|xNo@l|=(JFAe#tiPYk%hPDE>Iy{iK7oKg=LE^xa*e>d`za3&>$Jm^~V%ut{YE+ zI!}Xrjk2(3f`F8@9>UPuflO^0Pd={q#A3%Car4hw@Jgixe}CqIFIyz>)W!u0YZkl1 z%!f^Y$JmjyUAiPNQH5;mybVIvJ3^tQC9Ius6-cK^fM*e-z{X00*n9Ng^ZM^N$8!(h zJ*fdj3uCd%qocyoDA%T2kA{S=VpftWMF4M@cZ18{?BUE;n@D6qG_=q3An|o~fOPgo zC_M89G>7VtS55WKH?>1~*H1){ic3F*a!)*kSDL?ofz{S9?Vbej^L7Rk=r{s%V|nX+ z7m`fQ42?Wn38M1-p~$!n2OYnGg(sh5#qHK`NN7f`OwfQ2e9cK@lqak%!eHEsg-|G! z#q-3cz>up7_`16Q+-Gd^_PCPBW%EG1q9tsd zVn8--cY+h-C80vET1)oT*}!z}4IHm)N)|Yb#~I8X3Zt7n)F^+Z6UZSO|hwMB{rdE1{ph3H<2Bkmjd0b2zPixv*(&&*uL;L_XhDJcGD5nZ38SU;;H)pw!tDG2;@V?NEap$ciA%%D z>G^xHTVfP8MJ3>R!gzAEJPNHuRdye*I>>hoP- z#7$k8b>}~x)N2Eh5YHngp6|sz_3rTgT1OJ1=>hK)7?HaADP-ey3vSNc)x6{P4Dv$e z1fHm(iq{6-#>p~!I17;56UoWM{ca!d;HV=!;;aA#MW=+~2h>PkNhAdI3S{_*439nQ zO`b>11ba19h~(3=n7FxPHN*FybT`0PRAph*ctB=)E`%RHLUQn$B)Jh|M=C{gKpuY! zsm#0vevTrt^U7QB16mqdc3=QeoagU%b3vKM;g^zyYqeXQ#weviQ^u+e%_ymbF0QFeN=>*b@Hv>) zsl`(}IRtcS_qFH;y#sYIw$A@}_i*2ow;Yr?{pUn=j$P#}SC9DoV!ta^hgs>{HnMvL2A?0K}!v2tMbWJv}#9QZK=f z8xtVeATQ5-UX#p1(vbO5jHRvoVWGPT2t2F-w>@@*(MhuKzcwWz**+c~S~rnIZvWb{ zwc|PFof5-i|D)(m+@bpZIDlKSW*3ocMN~p5%sq3?C`1%lQ)%D#^{Z9cBeaPUDM^&2 zQey6z^BGDiQkIfZN&BuKQPZcGta&E+|N1h*E>Kq2O6xz$%Ux979V`ZH*6lw zoBg(-5dn_0b#5>J)w-15E#JcJo0G?<>XvgEdIg-}>}WK6rCxBV zM;+(yrsHJm+0ExCaYu|bsn(NG^vaAsf*O|~6uwDFX9Qay)4mDFr@@N4MR?JXvDUKZ z<1l)?Hh_AWkLM;zbgB5kjmn>!HlnI&`CM?k3u2`|xRXtrsJguyN~ts9lv@nw&ognZ z6MhXqdHb$$A)3MBXKXB`9uK($A5E^bJPO4O{9bYN`bpkwTQfgQIF?`hRt=e?p5_vE znxbv%W%hB~7hdUpy*O@S2_J4$Ee;>NoSMB?=T(<5{)uD@|7i=NqoOLrJK0d`3PVu% zQbjs)rW$`)VoGP5JBhzqIEfo))C%627V{G-b?Bf2OHto@Bwq7&5<2y02yLv7K;I5* z=4~&iA*q4`-Eyp(k8fRrPPvUlD=SlJQClFY6=d)ad}Gm=@qy_3^+8C?fM^WkQ*aAXYty zs&fG}#c3wmw`Mk%$Cq#yR%h{6G1s`ko)RwoQX)DwZUD_K^boJlnMTLqA@pDGGWyqe z4FAKRf}d({iDcOu&R~KTwbT^xieH}d%Xu?&`X!?8e<`5+m3!&A&HH%&)@R%dPJn(M zspQ3dv9wDXh}yOW(!CE8c{z`@Xvk0%nz`E+4Xmr+;#A87pR$4kFOTfuJ_$l-#QS(U zHq9G(s(bL`T95I9am#78!yCS2^>o^C(ttV|p5~IvK5%Q@bhu^rWBA*BAGj1 zd74PYR*5Fw>*ZH(bwG>N6U3MEFY^CJJmQVJoX|I!iAr2~Q=k1@bvu4}r%o!H6(8b0xK5;)aJ#3!7%bAT&U4Hc|fKa!6szQI4<*}`9(Fo$}GjnK*I34*Ry zS$v9%0YBNdh#URKiNDgYoj)zV7A*{1L^V!!axQ9F+@{x&bY|2l?rmNIo%l#cT(t5A zXER;GM;QIbRow}uzV#!iR>~M86JNN4J{rh#Xc})UzmR^_3Z)e*h9Y0*0`Ay=o7@Qn zEkRGR1^-z#%TUi$M#&{3(NwcY!Q<15>Co*p+=qr`bn^{kWZ1qGO(%Ex1a%eudh!Z- z@AECryeJHPZJvtU_b=mJtS~BSJIOb;PDEx;BGK^CZru8k+u~Ne2Yhvq2L0X?=V~eI z$C|O;Nx*l16LgBUqp>N~f=WYQR6A%eeQ-J&MW-8}m$J|KBeNc_$;S!KJE@|t`nlYM z%~PpX_(odMZi5sZdbrHF+o_Z0c3$27K zKG3{!7JW2M7e!qcaf%&ID9|{STO_)`=cgUzQ<7VGRk0mC^?5lB(+x+DTMrAy1?RYS z-`hyVou2gg12>u^>z2ys_oi!8^td$@wfrZ|fz+(mi?)@kqx^xF`2i=?QI=={EqbC& z6^1XTdmBD+&fT8q>S-U^esVZkTECRO?69E@n-6mr1Kx70Di_LrKU?IU79a>bDMZJU zJo)M7d(b#S=)qUjoW&tT1#4VUglq%0sc$UuztZ?Tb|kI)}Y33r4GIahl*w^Cpf!ug=~ToHXQ6$EhG* zvLc)=dOTY8-pLEdwy)yu!j*Im|BD;3#F_@GZ%5ab3`GM=wJGY(020~F&uqFCcn=RIL74Z#|Ux@cidd^K9x`}RoyasW{7t-36 zC$5`E>2PZ8i1Ya%;@*|r6^yG{NDGX*_~kZnG-_Tf*Z#=?g&Zm7t5vXU?mB}WoHLUS z$Ta2Vp7Ns0XD*=5f%EwTcf8Q4$1-zp{3%{};U`}1;|am``B?(xVRKy-H*Q1{+06ax z9Y^Gajp)9@HPme3aqd=ji&!v31-(O2bfb|udZ)XQu6y!AV0y?BU0VBt%aPfx$}*$a zV3#)9qHjz69*1z*$~-@NFhYUPWZjywYi7&?+2_48=Z$5)sMC9Eu64~S8uKDr5El7K zY~gl?dnzzJz7_DxDwL^j<#X9y(;%d9JA-R$ zvO!X{7o4$I179NRMk+Z0Vo5xvvBm~y`=Mj5t#v24T+3j@XXo<9udO+6#S-qf={&mj zxfZv0-5R>uRvl%)EcCK|CS6%D8l`7$L95^7^Vdo;I8P-)V^@CVf(6FteC=ad0na(U zK;r}V*?coCpTD2)yFHk0f2&PbZI7jCuSRh4drtGeyqEH+VT$NuVLWG&)xoKHuj6{w zO-EsF+B7Qb6{kRbXr%Ew4qeFMi@)dd{r8k;tBVM|^cgF@Qs#&XO)qZP`d=$*6|++~(m$$YG+usTotda)O3TBweCB>}YVa~Pf5R?9C@ z^`~+s>D;4&JG|ZAy{LY$CZAnUCpJvIB(_6~P(p6B>x`@S1^>R<(kRac?Z2^VNHsx$ch}IDFd+DHd7Nd$Z1RKNEqB`?Un_ zKf5AaJEl{zHpfJ&Kj0P?xMtzNG-q=C=S$*sZy&5GODEbz=Fr&uf=!msgRdmCc=4Cax0Fjv@;Y$ek2wi3-bpCw#kcN%B>6lHN(P02G$mSy7M_5Ala|2D z3FmOp>UL-hd&Zvr6q2^D&xM{*i^;NxQa0=B7d+oaNff#XL(`4RL~gzo@GcQB_FWS$ zHui<7lL4&WR1t*%{p`oMJ}f9HXM;cXV*BGT`i z4qw&;l0SW#WNXtw_Tys)`Mf+2-~6d2`i_ngBd&=Vgcp-nT^YpklC5a*twNa4BPNs9 z8cBAq2!gi@o)MqG*^pAFEUo>fM64g(#7>VrF=wGCx@WooQbps*?aKo|l)Q|k^;HX{ z7ZgN)OX4B&m5$_LQ51SV^NnQAtM#Pf%TpHfBMt<+o8gQ88`3e%119;@KtZiCQSDHG z)d5bRE}$S+b&^zN$B^3^qe#PPU+9CkY=OrSa$?kN`1V3uwC!yb{yeG;5BiXSqjsmt za=|ZUhR;IyX^{pu|6Y*vjmu%Zqcx>ZM`%HxfjTK1dW3vg{ZQg^G#chR)I!a=R3`L~ zfiMdr>A!s#<{l4#v2TjuLe3^wx@$S!F|7jodoClc+f4AA79klhdo}!5kOD6k{A249 z)3D}lSq&E3Ni<8<;KKF$a8Kk-kasPVwH$yurQcYSr+_#FkCS{0b%uqzU*d+H`K+gI zAt}}K6*)B*;Q>b%u|SP@Nphrw-0NBn2g2oKXWq}lsO}$Z^FnuWdU_$O$@XQx8v2D1 z1(gue7zZ)p9k_CcDKRdU61$yW!Dsmka#7kt`s0o9OQ*s3zyF#*YoTm?{C+<+cp}F( zK3fFViaPAwMqSZ>2`5>2sFbV<4<_ANp)mS*8eIF61nuYQu;rq=II!o*^j1rgd2hHa?1+vPo^V|W zL!15)&CQ2lvz8%C;Rsn*_JBN+6*=DR%3*kxJRX!L%coWAu=hsep(~>f4pyH4Me|U& zbSR5R`X%reLZJA_EONSW(4>Sr}r_c?;z zmAy>0NrmBA10YrTDmH0Mf}L;L(SX0j?3mwSuo;jF2P>`EtwpmTE&eF@{MMHq zsm~;eY4+@?Z61qgnE)lqCt=hAEz%QrSF*GM!-L>nwtxN}NpSdD&}`caGfK9w1+fvh zCz`Q{N8T{gNfUC%7?LIRZ<$0?4~Gsef{~rC$)xCFvb$#%e)r@*P)DWYZ2DPvx9cOD zIYC_%d1wy4JRpfZ6b%98sLObe(sq`9^B@VD8ZP{-*n^W2?+TkOqo6p{hqz`{lSus+ z_U3aP^*3TvqHs6ALX#0j6XC*;j z+A*^9=5E#^zK=g7e_`vm6mmT!fo)utLDc5;5wY5SQeahxpGEjccMhLKwrp(0aj_ok z{IL<_s$Mfu>&U@ojho^2tz@|G??7%Iy-HNk0PsxLH7vglH zg#2qdO4~@3 zP*^#Sfv;yQ8){-8`urr(cLRI;ay$$ftm9i|8qoYO+MYlZxabaK^y4<2D&j?DES^ zIm3^Hw?D*=^2WsY!w39rTpknd_W>oBSZI`UV^jVZNrG4+z32P zj%;5C)ke_}zIrPBefkO8uNSfnz9!PpkB`87)=&u28z7QT3n4W(Z-T~-UqVqYPrg67 z3HlYMaP@E_aMHIEdO4|*^b%VfFA zHx}DA4II1*z&7?1i%K_wRc-`UjE{x%i;1M>QjlbD{Z6cFdVrk%pg?Aw7$KapQ|5`b zyOROsGYGkNl<0X^5;DOBjy`-Ke0{(V&K~u`A@2UnBiE5A-%5g&hr@`!e;iJJkq_S2 zO{A)cpRkprmA&4W!i>(0gc;6qtoLyZWIiw^>c~)(;9pKsb`%j4%_Yz_u$HLmdr1-> z7vjSw;+WmkG7|Oe53EYL4P`p9SR5Bke8OJC?z0=&1f`E8TK_S8U6{#cJi87P=_%4# zwGJz^kqxVRQRZahe4xL+pfW8zur>;)iw+kx%UyM*_QT*$D+A@Fs~UD%X*8nS$@ zNaT|`!t{=rtdxyb*aP_~9Lk$Q1`?6NHPYPfMY7@WDzRGX(0ds)6p>G^x%n_>e= z$YwL*XS@~`eK3*QKm04p6z_vO7cH=T)DckY^o78Od&ud=W|mNM3C|xr0n|L@@DopY zSY2!lx4&tUkbwb^{h|(kzr->BA#-qRz-OWEv3hu`-;b}ZQY6Ex*1+$|VtiZhh_GfU z?r4c8XhuHyEZIvUCO8s@Dak~~WsNYRcnupBa1~;ceBtevOR%D3yu|rv95l`y2+jR6 zhu~7Vu&W)gjq4*)lM{oNAN;~jZu@|3WNY0Q1N=x+Ydbj=$b`hlR&=+(oJ5`YL*$Y^ zF`q>y@I3rC_V-^1*GD~OclVAL&CYosS$~M&_nC+B1yeI&r+z%)hbdv3T|?OKvqQ#c?(Buczgd#4 zdmrP&p%>}pL4!fwJf4L8jtAm+8*WUUNrn^#kVu6MJT}N_cW+yKX%=`j3^s~kbYARd-M&04{sEt@28Bzr_K7<$TRuuX5T&L zI&q~$p?NF>G__0g%lD80=lXGm{~7$jBLM6+>*CMe@@)(M%5UQSeg+=iIcY;gm$_`DHBY8S?l2=XbSkT#xh2RC}-hTpj*;e4?bJOsZTPs$w^y9zN(yNrqj+cufMVE-x5rY^hxl#B>|+v`i5}(AWbmqtHwTl>&gA| zci6QeEvZwXid0)9b4dzi+05fw;6LmiTdW-eMR5a|=$0A?RIfs5O+Ir9j>D@ZzuDW` zMWo+iA<11COddZ;AXm4wvXs!1)Kn#b7(8Bz{a)F?_MEaK*k*-GQs%{o3wHw6d1G+YxNQ2>()t7cJ3h?xkCfb zANmeIOdbL6yk!hc#&6;J+QX6+<&Naggg3%v(=zx)+1J5$!D4u)DP*f7)4^2d02z5L zjO6Xt$FT4;u?n(Zt*f#IIIYDlGRSn@^sSST-cF3ESnQ{%e!jxXOV{ z!yo{6Rg|#qOk*56NLkACW|G;xZp`6aHLmNKK|YwzCne@v*`QZBEcvagRO87R^72_U z?pbI9b|;hZKe>FE|6NVAV^JMm==X!XV#COr@0KidZ4`VP@f}xvF((6kro-1sC-Hw? zEZ!NB=nvEY;4jdqf0mf3TN4iONO{@ax1%8R)Am59!4hKVdO*Ca#9qe z!{+XP36{oz@KRy~dWmPD(^(ffKE%Q+i)h%AwikwLMapchCs+>Gl2LDK*oX6aY)V@v z9&Z;p)nRP;>toj0`uFS{T>Rr7}NZ#oEu3C1)DQ#n-#=$(tY9 zMdvhO;|hJ^d})yA`i@EPd5(~@Y)pgl%V*(b?^U+NJP^D?z6xKqxRHyZKAA6=PmV2Q zY}_Yh={D^c_AcNONJ|5Ve> z(#e!bBS5J12hVslSajB`6AGP2vyN$xU|nkr6zo0^b%_tz0ncD?zBQ1!K0QdTzM78x zYh=%YFGJu6a)%|K{;`8i2JmOjM3|iVns6#W+)D4062eqaF+M8uQNNQluwP1dz` z4^C1l17~@CwqyPsGQ8|NS+nUX3B*P?g~zfEhC@)8yPL#Rb&w`>ocNzV!eVbZf!@9< za^v(6I6Tx7dmQ~ExiIaGCx?b9yHyxTTjDssz&UHi8ae?d194qpm84}ONbSEI;9_sJj^ z5-oeD90B(73*g-(PqHDSgRL(+NnRIhU=^~pX~Xm+@U^QIKA67;Jni$zzzN4l%qVw~ z*22O1-Zq@56acC=LK66hkQIZTGv9UpkvwlRSR7f%K9L)enuCv+`B*;<{Qb(&bnzF)y49tjB@uxT*!~zv_Z&1F+xr*NKKS z4t#VL5Q|aYgmnW};x_}|v4wvpvg5huVe{v8lc(uOL#Nze zdv@zWchDFz_3LqRN>v9mC)-2b_6;QLXdW?`W(51g?BV2-U+`o2IZ(2(W{*$af@L~3 zqKm&aK-Qd%n4K>arWqcEY_^MB-8qp|_ooZ>`8hOrY(4Xg_`x^@byAI$;MAU6_V0}X z=-d0TgfV7h$Iv1IrD@n%wE$jzn+0u)>`3OVYIt&5MRa!3Ibw2UJ2QBj0TW|ZLj8SH zGM>eecl$oDcQ^hq)w{(^RXi1pn@^G;ofwGQ{g zb^T=0-I79vwf^A`eqRlfudgQu_!@?7zT@$3^6=%JCQ(WpNm870VVhYJScN7-+G96- z=C4WZ8BLrCZF9Yy2%!63dp279gx?r7q*w~AhVl! z(7Tk*sMn1{Ky`7UO$c)g-sV;bK6bSO5=#Z$VD(j-iZAf*GbnVWZ}yfteK!{ zBJ8Z03%M;j$;{43IBwGiyRMaxtRWZKr`gAWcq@zMxEYY`%PCPE)hX#4I!eM^QdmKw zKq7tHic6AT3Ii8?!9rU-7<_s&r0zUJW?oSsMQcxxiix|(0kAy^zPVz8EnZ)~jVa{d2Z0Sy4aLWAz)9)uq#=~s3qk0Z&Uw8@sJFtTt zavLZbn!bcR{WlNiYH;L>jge3|xf+hDy%yGQoqU8lwn=P)BYH7pBT&*h

    < z$g-N~p>Cn~z@y~F4Kp%5R#`f`{3u+kzexV+Hxs1~XC=KZ^MyELDWndJW7jhjp)PN* zG%2H*HEfgzwk71=$h6S~(WTukHaflszsDBs!U44hwdECUG1|-AQ-b19p*mIG;9={PT<-wd zy^@TLu`o9^lhxSE9JZfF+2S>ah#0+Q6Ky=9>WDvy9IQYTg<0^}d@PJ=NnwFgu`(gvF)r4cwfygyzcuEGJo4{Nb?I~V*_I)zlN1d`elaaJLw;?;8QN#el`JK z#x;^3v*X}t?J(y0LKDz84bl5nPn>nR0`g3wnd!S5*g2(v&E0U0b+i}Yn&Tz-@v)1f zN5dRO^-d)Q=Uicl-C>fhaRdxLZGe$Gtw{^MOKe(Z;4^j$i0i12P&~s z9qKB!g}No9;EYK)`LBN`S!88M)RuU_CHq@s;k=FTGs2Qxx&43$wx<$TR0gj1I8ar2 zMYJ#OWubqyMO@!)*k`!~{|*fz{m*wm_`*h*{!JM__?g2Rd~-=~mdw8qw2%d|J&Icj zIS{rk1{QBtBvrOoLD9w>e*YKC4B|3ny)8#!xQi=fu_aKg^9?>89*18ApCd2Jrb3_k^xH-C^Ax7XxG z-w&8J>lvAQS%lXeDJ3CRe^_YzHyG^7!{>`*@Sv|7NvroAq5ly9F-!YH9FFagS&i*@ zteFN#@}fz5r4KD0q9E;EK0w43XoA75YvkHFUD%+I1QGXS9Z_2~z!iOjs1z;o>x68s zJTx3Wr$&*9{uOX#|83GGH=OAo(}K`5aM(%SWcN zEBtUWKTj2xKTab{zMX?0)d+ZY{ulZ0^EKvkIUDKe%53mbJECgyn+)1p3lW;f;n)6X z_$B{>+4R-ntsh4U=buprkJvA8_56F>=b|YERbMiywTWFFc7Y8`)q}`oeq^cQXl%79 zk0fd-vs$M!!nQO`mUeCu+p>2bWVMZ8_9dV2to-RjY4K}j6c$5lOC~Xm?b&SBmlH&D ziXlwY&=bzrt|Pf+FYvR?bA_jOeIe&H2Z(>hGueFh3yxXp4%?e*g&GqNOJ z`sbsF=#aAX^UyuWYEcXrcVl9>@c;?@^Gqf+nNId@83`3F zdr5g-Fv;E;bK*j~pjM-W(?(PHHge)&N*; zw-;ZlR^zVC?7>pAo3QX@2;6;kgSouAEt$7cAG{xFk?+Czq~g|n;1v?E-M$Pmcy=p3 zztl?fK36Ci^9)00+ZmXnWX~qnzJtx{kB|q7t>E`a0D{YD5O8-g_DxIz3o{9^)c;1- zxO`>1j<1JG?lh3%L3m{FWtNp?2!5#JKBpFJUd1I$y@Stg=a|n+QHCt$@CbIndue zL=0WvZeVlUX+)ON1svzgMSOiaNS!-&4VLjv^l8BzRe`EPM+^W+;H;3 z0!SXBKtvJiL)Vea=w?t`dJ1~aO(1hWIb!9dfK6M{$?e~hNY4_G z?B6{{w5_)Y)`t%zn@t{(N#;ppyHzr5S*I(ptj~w`qeGI{`PZJr9L{!GCWu?DlJrPGen5D!QG}AefIq?%- zz440LD@U2r3vwbh<9W zrLX1iYRN}P>^?&FjGfLN#GZnngO~B7qlei`<2m@`b`8mn)O4s|3!nVyg+JLViH?MC1bRyo76;sr41XJoy)CA|mNhzT+|3n0 zCeI)lu?&1>PKJ5EU*UfZCFFq36Uow?0m7XB^rWu`&LcbL+A@B+1Y129kz~7Q;`%~S z`gfqADo1`pnp~gD=?Z0IWQUQ;$-Ito_r>s-G@PHdmgDB8xM9fe3_no2JkNqlaDdR z#8+Nbx+3x^duL)uR?lk1M4%?x?eL1|)b4`S7V(5=-zTqjsfq%Z&S6K^7qdeh8LTi! zL6k1@&0^9%k)&8TV*mOv{Fvs33un)P{={99S$@Zu!&x1Xzs%CzJL{xy_8$`wxQ~Hm z_f?=(ev_5IdqEbW3M_TVhMtNjJb1%7QgG>sWM6T!m3gp6QywBZ*F)zRb=>T|63)!BAu~R{W=qar0_994vMy^pz9$L9 zgU9#cDBgqADawObWh}`q8Z3>kGm{P+6M|3mETTWgFDLP9nxH6spTtG41@23nL^oqh z@YB#J9K2^S-u3!9GEc9<>$U2cw?wx8o_m`N7%!Wt_qq_DJ{74DiQ(79WSBYpAgMIH zCY&7c6VH4ngBwarie^6tx_w=(w z*pL9XH2h&sl59_5aRi={y%{X|Ox$lW4;F36z{gWn@P&&B#F?bAKW@>?;Hwin9aTuC zXNAMCV+{Y!^JGn4)$j^Q*v(1jn0L~AP`TqEbPT>EkqmZ$-y;u`;;nVe)*=0SBONdAJrBzZe4yRGsI_far z)^n`zz+$-EKA(N?HwCYK6np;FCMR?9Kqo^(6z`RfmtDCGZ}vEVFK;8g(4a)-FVTZ8 z+iKV{(^hiDIT7?-Rq>w*TGE)ad9rg38^}k;leqfSCUEXJ1*Lzw;IaK7crF=9E*`1G zBOWD?W8)@6^q{K{G-fZHy(lCFOUFyk4zU7z7ax39stHSzvgv!FisJ~HCxCYbWW8um=>#Y37e!uCtr(p;Bu(#(Ad z@M7RjM&wpv-EC-EK7-g<<~b zY`dN++3Ru&a*eAO?zQ1bQye%3ox!6nYoPol4Vm}Ura)D89_Kqh6!7w^b9v_(SNUT`vU9gTR-j~cBJP-AgeIGq@l#&( ziS3HTf_*B=G(e%7_xda0*Q`0lJvgY$Pv}&p(o=+YSUCW7(^1Ia<1jkwu?yn)p=f-3 z0l#;=3H1@pe?sn;Hw`zoiz!?cqWQ>Mtbo9Mc=vC5pwkYdKJ{TajIZ>96?X} zMEu@ic4+bd3;N2vneR$GSb1WRHfrpM=ANr9K?m-ft~B$RE!ejB4L?hvhx1yH%8l!m zqh*#F=GW zgdVEwI>sA(_oNG6{1D95Q{a9+8pXW{7=g^a{5enCWG>ZLfEpazxVC@^f)hnO+{Xz4 zGTY=Q*Rk;c|1NtNvY1oPEA6*MR$JB4(9;tIf{;M8LEj$bt=AEl?lnVZzrXTtrhVr( zC0o0~9;?R+hp2Tvo$ZS-K;^&q6N$;vrtrqJU;>f5W-P{Nd&ol*uwrhq%{?&PZ97 zhraXV03UZe5b3qlaC>S-AT9a7+>;}=^u)h8=!MD@!NC>Ig1NV@3dX2i5G$ky^3y(V zM6YhX<%Ki0i*@uTaS4;fg8l?qrr@C?=i%PRuPHV_Z~w~E7?#9;=yF4KO9JVulKH4X zY{j2>Tv7Q`Z8WvMW{leI%tA%S9t!lUe(^(z7BU#>%d5S7FIYAEHm`Ml2R*KQg$p)a zz(-r2;j5?B2&$x&e9!15=$2R=b;^xLf&&)(m;S$;+3RrrpY}z5Q(X&Jpa5L!(^kHz zp^m>8Eke<+aFrBN146DZR zFZ&Ru;ip9($XukDNge#a0p0v(#U$?X#aK?$(TJ)j-{#*r_|fl2?Wr3R(V4qKxJP;y zczkdcf9TdP{)MLnFS-_nRxTWlT>cKCuJXsle=FYdYh@OPZTxOr(~vmw3rAdxqZ8iLkad(oQ@7Ex8Tk@Qu?3*IoB@d2L|(etY}c#nH=T)Dgw zN^K5Di+6jVC0@U|#Ve=rh0|r3=wXZLoO$c$-fLA{3(*(TI3xaGe~h>+>8zNE!udHD zzVHKNnfED9zXYvE?(q8~eZ?m767he_EG2fY2O6s-o<2b_5C87R=vRZBENFleqT} zhSO%OL}!0*^9$?|OQI`jJ+f~o>UsU0E0MXHy^&+-tG_q6u8CfB z#Bc?gY-2{xp83JgzdwR=FQ3iNS6MEw-;B__8K$(izFT~^$%c=26wpIzp7fHX8BHN? zxV(joKkzu59@8+Q#KXl^%2}fW7nIPYl19E?XpX!?WSI|TE!uTr8jUYg=5+-jC{o}m z_!Ml;9}L1s!P*Ed4Ko+-_%ejm2Bjit9I_|lb; zO6Z#)SnRFzj|=)8C-^Hm%<1c|;4c^L<#Kjf${d;dyi0hY>!|*V{GLT-boz}6Xx$+t zv_#Dtocz(l+^M%$xT1+U{1I73?tp;Ba^b1U1Lq}t{ibQCxWf>wyOqi(tv98O zb{^F9Q4jz7KM|L>Z6M7&8AjI+SEP0C<*8M}27=lLWuI30ywt^%GCOOJ0Ekvj+%TQ$u+or z;{%KtyZ>v|uZ@;oqw(1Jx&JU1BX2UB4sopevS;gt zGv53QyE-0iwWpVO3z}Ioif(v%jd#|p;}v$TS{jp(Ct)?8&tyQL}u|dseiZyzn=4oCeOK1vwrY> zzfW^d+-1-CONO+R{o(CvMKrf|ELAG&=FbE=is!Bu(VhvtT;`!yoR)Spr+?xaH}i!K zz5dgk26!Cjy9{*Ex3>mpW}zu62zu!nbNm^%F|Lq*CB4h-2$?`bi;QSmP#b?w)(1J| zhX*Q}?SxY7ET~nwkT18gr?(0Qq4+6wG(hJHS1Ou7cSxf7H^OuLukE)4OJ}U%CcfV% z2=08sdEA(WO3DvZYz!Yp6YG7@m+89bQ?VVokF?O4*^S(zu^Pz#)n_gu&KSK}SI8~N zn1B>4+W7V*{rt#`AKZSi0-t@u0>$|K<^Nom&I#7_@N;&npprc{G*6aK39AvJokJdR zaXXw*wQv}{x#1_jG3RQf^3GuH+O~2&=-PgMp3X&XVdOXQk}se64GYBLfE6R?)+-k1 z&d7H0IJOw=dXP8qzZTc37iwc=qi6-G&O3aILh#eKrmh; zj`sh>p%Igj+4NYkX09v3FZT17h81$*Ax3m~T(zKgnL6rrlVxJdMjW;I!jpdG+*Wo*JoAd)eBl@azblO+U!3-r|j=wy_U++#(_Qj-4b_{QHM~e zzz3Orb*0YJfU`**O^5vW%}pKA!*?4z;`82wpkdL|X|LfD6#3o-87|wx8QB=2w!MB7 z2g#!qY12`{1s!DA+rT^aYM>cjFDf1G8gPEIUhp4gr*dvaV^Otoqifl00Xix)Kr2@6 zM4cJBG&rT3-%u5f#`HYpcg_u^A7%N0DQrKdaqc%4EiXcoW2e#$Cd~r5C+6tzIe#>I zn+eU0*w2lX3#KWq27Jjf6Ey6t9>47PSQIdIEt=WX$xrS~;}0x~M#Ax4oPv`GslU*s z`6~m_g(Vg=d|3%^&}NOs=&z%X9x9+w``_>q z&hkS$Gnb&h4^)xO@a1UBd{-2v5JIzBWd9plK&viIqv%41c;>Zz+^l8;&P-;%&dzzm zf6e9v{q;@U(MA6B%=|C>{=q@$K~xy$_<1r-AWmqS%mS15oZ3Ty6-=f2kDaO2@qyfEU6~E*KaTENF^OKS z@uf-c8@T>@O`50aLr+-iA$g4samgBGDp8c6V*&BMZ;I3-wWqa-Pq}9xA^`4 z7&`N~nw~I>m-dY|MH{UO?UwGDJMSf05YeWT$i5RILMf$9o0Li=TV%~v_spFsBw30m zWQm_Fkv$^9?|kmx=a2h2XJ+2_dFS~)obw4pPPdED^l=KD3bG*UqaJW>zlRXxp@Y$w zN;h;=WiC0`?n!<%j6;i;QK^mYNzPnjEV=l6nl!y92;CmF6nU=mBLCHT@wKbN(7`=1 zXnRi(8AKhq!k$T}r&J4FR3D4_jsf~+>p)K5o`f2;ElJ}R2eQ|C5?VNR82|Nmh198O zI8iE?$^F_B#php|KpHQ8;7_jcLjV1nj`)fCWXAdH{Pbla)LPjjTGA{}76_cAy`BwR z!>aFG@>PZECN~e%Fy9{C`GL{BR-xVsk3fHqj3x~;)cNo)p{UhoAo=TQL^7vNC650V z@^!Z@q!1{uN1Kj`X5Gx;1CuACWezhrlg-DZ9lOjq%?=-8^6ewv=dgqO$=$6UpWuWF z)ROo;Qyclbk4HI`&Vj@su7LME@J^H{nM<5bk3z?n3Aa!@{JEX+9RKg9A^)^&Iqzy* z#XYL4;(~>`0fmW(V3R|3_Rc8&OCzU|G#E{>$m1fn>XHzVCHj0uLWZ3l&HdPej8 z>HcK7w2%9+NsZ)YXLG4vq@2Q$5rjK%$?f}JT@vl-gXWqq;5rmCMR#5%@t=Pm=Hk<* zlKebnWYaGA>5sJVYai5eqXgdU_~IB|-=V*><$OB7P~dGpI6sx!()*3qw(=xdB5U4Z z$9>N7LZ>L2c}N3NHOSAvr@X;%6_noVhD=XYaJvfolNjAQeDpaBWGfPpm>D8|(f)R> z+D{KvUC83^XB_7ajGTxbRGM)gM+^SjJME&szXUe+sMWAJmYuW*uxJK zSBb`{#296w`6@o9X39{=;!b8h$d zBIy=l&NbXUFB(YYNDex|o%1(FKTDhVg!WlvQEW56^{|)dbd5T?eoPfT+9`)ZzjSbd z7ns|!QIRy9-ayKJ>Z4p+d2%$_k|1t0zbR}ADe!FKOw9{LYc~6H(XqDVYHb$x;dLj!fr_$S=KjT>gKR)h4z1)$g8ma6j6P^XZun zs+I0O;XI7u`9w7%6m;GOxtzGqZI4sqZhPnR*9rxe_98=464xWTn)!x%vmi!PYM459 z?Cv;zq4&R2Cb+FLr07u~Pp?#J{<1EfLxn|C7gZ{Q?SJbDnBe`&w8 zbWjbC%Pi2l(j#2y@gLG#HWD;tdZS2|@5+Vl`^MdkRS`HROSuu8K1voH=S82qI7!(Z zw|m=$OREp=;BTg!;`zV2Nb%r5-tCz@dQv@z*uEY?{Qu15!t)OEn@+S>RnJM38XBo0 zgL=ffPWtSYbkUZ`+l}NV4IPR+qJ`AV)eZ>O@8z8GHIT_!Uu2Q3#d~Upp+m91rT2p; z^0`L)dB+d~E+|QvTna!$@}PrH>~4{6n>mQ22NdwjVr#fHic-G8p@YY-`jM;qw79tX znbL(3`lQQL54HaJ%dHdGcBaqD_-Oxr==70IT&~m<&3)XD#0cDljN&+cwEj*0PL~Vu z3$-Wv_H^?fKjuqCjfh{Fcaz`$ljC2SZj(;hW6A%~bK?G-ZR4ZEwNca-Q)$>$H?;7m zA-XpH2*2xm4Ig!0k1X0HCL=ojadRtExDk?1+?uB=c`q+za^7wa7yM4pyzazH<5mqM zJJe>PgKG7Bq~dSs(p)=q;H)FrzovwfjF)mJQ*8*c{KLPB?BzOMnV`qRSMyJXeB{aC ziCkLb2=d@qIoJNLPkO3Q5k=fw#LX9;jS=*e7AGK_iY@t7sYvuj&gC>$3?Q=x_S~eUVc2+{GiwKuH&VrlwnnPEQ3_y`*9Ej(cQ2xsN9zMiS(D|H$ z(M!oN^2yql|Iglr*#0cx<)$g}ex;|SX~Nl_>vc$!{LYM@c}AY}j!{CcXSC2_Rhj4` zGemv>LsmzbeZXb_4nj=U{ff{o3{J?3O z_;S0ng*sYa70Grz=4$g3xtzCeq^lEExcgs+AuVr9zQEZTeO+fx&ir$ByR%W9Kb5zi zuW$As|Lps7CM7e_+8r0AE*?u!jF}F~|LIQ7W!>kZv-3oUzc*9`{pUc+X3rNL>hmMY zpLO^wi*C`aSP{9dbeliyu$|ldC5`($z=s6I{^JI{m*bC&b|8v>`f(5NApUrmH?f~0 zk5;~U#(h_f;v#(qkiV(kBCS|XrDh(d)vC^K zJSeblYI*)b?QX8QWE(%%uZH^|XuM1*f8zZda7g}s7K97oQL&Eb|`wI;P+7D-PGxh4uQ3MUo!_0aSBPENL< zgWKf&g!3yeb+h|9gUn0q&tIsi5p^YDbn{m^@9@MEF_kH3f<_FAOIS)K2Sg#`EI;(} zp)Ts{wLuDj>v$bI8>G`@gYIA7%#R;60mVdwq3s{8@iNO5XubM?suk1nMdfj0P}zv# zB&B{fntgGM+dx-4l<0GuySHH;vN`me^R*7)aykS)Y|dv+q&pX7?;1v?-|FM#HH}Dj zqc3^A_9W-sIfhppbC26IEfm@Bb|D+0U-3a8 zLs8}G2&Annj;}Vw_FR%i~ zClZ7M(9r>r=%&#B_NYAPQx18NbA8YFG0#m&O0Et$J;aeuX(N1rml+q^ah1=iut!6# z1|#F|H~5mmZKz-B403-)JSlmxn7h+87KyhRNke{B^R5;5`QVM#B68A1iJM~3w>NKhvXBTk?YCxoTX$1XZ+rPD48urlWYT#fiMs0{2qvgcC96i z0yjP{T#<}&Pe4y1ED1dlM(!?;<@APGlYgTONX5yeXxlt1bhr5j_c&|_ns-|nS(Q#9 z8bTh(wMXMf+RjAs_4Hvr#rGX|dF)`4>ob9jOS2~?ABLlGLS6!Iyoh9-nToQ%_VQK3 zY|yb)Zk(^c!`#^Wo!jOWfL;c*x~)8Gh)AL@3J_S6(=#;?x7UXJzVbu*W8(o{f37px zK^}0iEB(mY10Kjm6o6Jww?NUZV^QaZ3Xx6wG8D78hkvrlkpx$0lRRH@!dW)rhl*uQdr)`n@D%xdWNP0Y1qi;ej(dZaW z-VGa1e6q%qr6I$~58Gb8JYXQQTX;&UJ71{1^_HV^6JPW0hs{V;uAu)^^ztb|6&-`V(~U{NwIQTA<9PM+ zv2)1iueE|+v5DVm-ydBZv>N5_v?0On!XDZFhacrXoDVs+lzY2zB7!wm#8u7$74APK zUHnpn1oIJj6?0GY+tg7cS>izspRD9cdPV5$<)J)IQ6)PiC%NczTT$Y(O=w%!eDY{S z2DkplPVSG}2~L_DhD6&82q|mgMz&}0GvnM*Tt@*v_ofXoQJ>2-l;7a0E|~C}T)*)O zM*1W$r<5E0B#L`~Nd-kzE28&{Bgvxk+UUzKfAqYsoU3&M@?Yd2^lIK3WE``cJ3c0e zoRHRWF`7|sk?8@XuQ3As>#!x=O9l{?#G_oi?QAsmmzYm7vLH+DhjGrmL1@O}*U~(* z&E(Ed5nAs=cmZ#UoEIga#L3;F_{dpkdsG|my3COj$Ec#+?lZ~8vTxj!AWxJVZ;he` zVU+eYg1cbb#mkhNxX*801-9}!Vtis8dcG-$b5V99jg^{+MYy0NaR)iqEq3I>;iiVF> zMSb4(=-jpQoMO%_w8K&fwOm<9G{}(;eRU4Xn%pTxnj=wH^7V^cw@NbVK2Alq$IKxE{+mSdrz}T3j}^$*DfVRQdo7Z(Sq)uEDdG2*Ch)ng9(;W5 zGIZC{g}9%)#gCjl2-W&)2)P3094`9^%;a;`zY;tM zyQGiu)x61^ut=n7WPIkN2fYkIdM^uNO4)1MB)k?RUo^ zr;LK?4}W}#Pny6hdvKTgvR|LH%pc0NWp;70Tm1>7S)#bdbCAYGAl~bJdGD`IWc%!y z}PhQPhUyucBY*ZPo2dD-y4EL#ct^25Nqi%BNgt3Mhbs2aU5D| z{gS&>yh3`lxL4%;$(g_C$GG}vLo)EKF*&kc0a@qvCu?Ti;BQSm#(NvCA;-=h=OZ#g z$#mDF{MN~KXl^f9ZYhlAIcuDCFA|s;%L-g=(Yl)|H%Mxh{IWUdoNz z+QS*f-Qdn&+RTNe5q^xa5(zB-#hXaVM8&QMJ@EX?U$EK5qn+U*5wA@8uKwbrMONIh zLjv!+XALix+sx0*KgADHOBOA)4<$mJHJ8?HhBmNVu3xADe_F#3jaZMl>>yRN@4^Ev z{EPt^HEneDzHgl(y|C8m6T#PnZAwxQ&!EbY-$iac`xMem2*Uv zqRNf+yeDn6wd0ic+M>M*%q{7CKfd{JxM*OE7h0z~n#ijyL)$EpQPa#;Uf0(PU7K%8 zF6NBn9th{f*kPG`o3@2$YqnT=Y+o@~_Ee9%wO=0%5821ROgtb;FMPz)=VIjF94s)N zz0e`Dh?`ZjR>(A4$<@Wy@R4tIkW$hd(pTh2@*b|0>P_j73}G!FbZ8bge$`E(7VQ>I zDYHZR%M1`XJDxL_7nXl+uNW4Z_kzsGnh^ig6h{ab?^_&9&TX#Pqb<0Pgm*a16 z`sRbt$93h>sVsT)*7}o*B^Njgi;vvZ2SMD8f3LYk@-Mk98>*|1 zC2i-_o#MC^OU8?MM0o3^a%4)hh~JnIhR&2La}Rf3=Q3LxrDx(V^6rt88+3LwIkMG) zQ+0^r=FF(CPMc*!7JDD%ZjSOs$MCWTP(>eM`dH8T{(DzCF^PJtHK-+oT z=XdW#|4mm!k>6KygYKsDRhLYN?!=|l!3V7f$Sg_b+`rNt3A0JWyPugYG!j4G`&0_d~&LDK{w;76QdB=?!;7op0)NzlZy^-~5by56p4-#p) z8NL2D4DB@-M#^5fav5!1d>OCHeXU>58~xKJs)wq$wDh;!ipvX7d19foB3=P~G#1YC zz8xIX^CgpR_;H&?jzDH#6@=M3BIm;r_{y7;&}4}QIlnJh$UjQvl|D^Dljhd*Tbpya z6Xxao9=U7$m#qp!Vbu_FyfcuGIkXUM>HWoJZXL+Y)5pmCY zzbk*VD;|xS8zkHV5|j9XjeK?KbH2i(R;oC83;OB)pHy2imT#G&f`)iV$UL_KsW>f& ze6_Vhv3~7=pBeWY+Xn~?f~R^z!FuKpXPGVGk##9 zIV!j7<0gN%3ec;bGIXCfWwg+;|kSC8;+LLGT$N4oIU-L~J zZs=Ivdg35kkNn5!qpkWX z%pSeny_0KwEcB`O=a4ghx;c|aGtu>>8@X+H21IRj7T0nsRkSKQTdFlm8QqT>hTarR zBg0Hr@p2x0e2?*VZrzG;=&||)baauBAL4M1d!T7b_I|6YPB9drT_LWdqF)>-zAxi? zo?CKO9%fw1DpzSdCNI*FS*0#D|!9j{m9A>v&g3}w?rZ7YN-421z~orhvfRs zbBFd%CK@>bNNemO&SqaEO5fTbl6+WAPT|K~OZ$6%d5j*JQtrjy8r3QNIO;z>Cc1|^ zc~FbY{x%M+YmY-nVxoVugcVI3&L)>IyuRONJooZrw!ZTyPVFos4S(fo2HR8S zxmT4L^E>bm8*i{nI>hd^e}J{&KJ4$n4UoE{6+gVNna*8gDz2!Mlhj}Kp(8il#y&eV zYw9mIv6;r%GKZ7eaP9OFI{xlB_Vbwu)9pHp>jEA_i9tnjlLI37R89;g}Os*YQ<>qaDGpy> zdl&b1on*vFUb6b90Sr_BDZV&wJFxR65;{DbD$V$g=daVov-7RNE`7H+g*wBB8GGo7 zXRB%+E@}d$9~8cI-=iyI4B^8^YuH|8P@|$c5N}DnMQ-Fe;nU_ytn2DZSxcmY;FlW! z*7Kd&^rPSKUR?tgyFL-$i;rXT=G|o8k%@TWq&wJ7X}FMQ9>nZEx#RgRQL^Zmb1V|} zz!#UvV!PrF>=e5V&x+2KCHy)FEv*V{*6ho$`Mo22YYCiaD5d)i z1fQelYfvodrjP6sSoTCG@f0!-#_TZ#B`bT}yYHDecK;~f&J~AU zH<3-%ivxMZMA?Gk4Q$~k8N60CgEpSz^jA&&z-tTpBaJssQCG1yt0vpG{5=fV&P^;vrKLL3c<2 zBMnbjAGcnf(oiPD-Zvx5u;ApMTk)pNch$GH$br+;I5z zESy%MHa2R(64-n95|qr=k@($LVGA7&$*NX5fZ8U(n|=NjZp(ZG5hmfVYVJ*0v2gEv z@?ItQJ5!g{4?c!3r|kwgwRka||4JOHQ^B6to8Z`kPVAbs4V&VZ2$JDZaIX!R+;u&A zFtCR%4qXaHKK<~5iZRUg?s_~eT_ia@vji5#mQdyWs_cfA7M^rH52A*y1n9~D)r4-g zt9cX*2P07XcaEi;QmeUs&AMi4(phme`vvOhgYk(QZJ<=>4c7PnVuu;4S=OSf_}$?l z?9KLM`szzPD+wC`>vi&IO==D_O@4=w*XR2wN3gM?XfqpjF+;w6ys*J^IC)_Uuh(bGi zQ%|s2ZiNSyd&2YUV_3WZe-bMkhpoe&;@I=npqg*TR72$@DIVdpEuf!djlivw{5(cK z742lNw-(c$5cQg-33=??n*vz){5ox|Phc)~Y54r~vuxL}uk>yK2bR08!ly?^pk+f6 zo)sr}pSND7TFLuZs)tcVRBBuF#FD~1OAfI2w zcAsyjk_id4VctV*;TOiGbeM>DejNxlyD#Cz)uuQvJD2@uF&z4}7eVLN0%q=U9>*kS z0KM@I8(90prDdM%{eS`}SNo?J20HPYg342Y%C==6yJ8)L9&(K8&5;pHG>3J^bKoQK#m8Cdx`>htrGU zsJ|=p6FlOBQ^b<(>(ucsEiKkKejdcdN3a`iVRX~%40yKe1@#`33hx8^vx;>C*rXH5 zY^`@c$&c*+WGRlL*w+1~Oge6dxH#ggY*_Xj=BoV;i%XA#X5w3Dy)~CL|H~Gyp0^yj z2f4Eof8>~K&_%j)!4$f0wl1^^j8|vlK2WW#5xyWSu9TL!`=iI$Cj>?B<(C zvU9K0uxZG4IJ0FBThiCS8e2Y6yX^Y(=<~jQXtsnB9j^8vAmKnw|y)iOsd3Y*QJHW8T=0$Koele|z83WI~ z6eMe6tl;*q!>qN`m|Z%N4&Pc%GynHa;HG_t>70y%760v%S@`Y(*Mpih-#g~A_fj2Z z9T)N}d1Tf}<5>M{NHvv7WoyyRU$4xBf%$EKx?SXZuq{j%!M*2gr-R@m!- z>FHfKZo+C@D}NLpZ+;IeGH1cCWXwjTTm{LB*L2C<0G!`?p59t|3r6orX08rC^wN@F ztm~2+tndp5jpvyxcGo;uQ@>MoSFZ>3%$!> z>eZ!C4cYix;eQN8tYL}^1m38x8&h`uDQk>ACANv#%hrz%U@k92G&)jAvM4kW&Xzr< zwcqpDov(Rpz-bdX$f|45b#F%!8%UPH>NRS+lVg4ZnEPM04)kJYTRv8s-GO}CZ? zEGwAIDrfeWblD5*-~SN4T|5mdb4{tIL#jW6~6-pZ}-2~IpqxAQ&3;3$18}mK5 z7Cfe&W$Jc5%rNmDTbH(x?Z2u(m6hj+r7g*L<=P@hZ8$3%w6%Z^Jzq<$N8E?!M%SoB zvkQB@O~!w=I5EvVa@4rwGnA}z!jI)R@YxVV$7C3@waeAvud4-<&peCIDmcOICmjA- zF$2CdFnISp0sdC+g8Am}@b^y@^!7&|>Z3CPziXHXH~;iwZ(R~-pOq%A_S*#BuQ#yb z3HD+Y-(DOm@ZFB-nL|=bo%j<`qnn2o(RHi(gL76jdl=wO2c4Hl&{Ti6=u9Hi4B7+Z z4|Fni@2GfC!bF&5>BHXt>!2Z_Vpiqi%BFnRhn%%D#Fj_GnDOmYR&*tl9#PxO{0=*@ z(^+Tnnfhe$u#5@Jw5w8Ve&-HkhwWzp9+R=SCl)l^8liQc3KYz*pgHI^JRE$54(ggg z_s9}Kqd5jv>o1_wE+#OuZKgFVybAELNt@_@a!qWXz~??~sRf(1g=0`4G~$9CE1DZi zwdbs1FJIQc@C0AHSi2e9KN`sP?zu##83cg!k}^@=#GZzJp4ei9zYd5I4zENA~# z$Y9#gG`3Yvm+A+EvHRNjtk2I}a%FcKvyW7Tje(3l{anZD&}sHvu}nPdm@C!|EQIhi z8>#Pxavbh8lu?}%Fi+hau3;VKv|IzO%)Z2=f34a6-dH-i<{@*>zQY1%slZti88z@1 zvu(u(A$svm)@xmkYhs>@ohx?HHKYo3JWN399s%_YwRqP{ZG66WBWnt3q~Cldiz9}} z;7Yv%4UKq*2m9QlabsF33q46EcB-(&ZD(-&WO>PNdn?JLfhkaPb|Uq4-UZFQJl;6e zij{Sj3SMq6=FSb4Sok*K$+nZ2S>kgr4!J%LpCefRPv3Dtj|%Io zDHZ?RQvnMWk77|InXP-R5AspjP?kCn(i$!Bq37ZF_!uR&XK(_2JJBB(DRi(gE0po1 zMG6o<{sc5lngzB?8{oounYb&_k=;G;6x+B9_AgZl6jz+VcK8td*c3_cU(&C63*9U) zYBEbw6~o=%+d=>RW&GGF9v#o;*p2~5*|FATWd80+mxynzpC(8ml z3^gQ%YU`oWB^}a?9IwUzWp9?fd-D=i6A~sCjJvtVgWcU@fi zGQI08%Tv^b<>#%LcdwjeQh>JXpeCb%xdUZuw(Mthri@vuyGT^mt!GbW1cE`Vz#@-K zL+9?VWGUyau#Z}mxJWUD&5-3%_s>c6jByc;5Ng>^x^eI$`xvMcr?aXd-&yXb=d#&N z4l+$I5jbSa(|ogDYCT1n_6V%x%+ze`5fX;C>zUU)tlmwH)$QQF@NPCw?gQ4FKct3( zNqB!`@Wz&}%t6;h@^a~W8r5Wo=XyueJ*wom)Ai*y^O;bDikDlsv8+U>w;|0*Rf1#a{+ECx&baf5$^ty!7dER zz}?$y*u2`s7^Tc+!z$*p$&FbotJo42igrOnjysdq7t@w>6ZXYBPVADM2CmOsp+zl6 zte!VS@Cu7z)#f>L(8pW!M9Vd{^zcdUM!jJbLC@I%r%bY#KQczsZVcnt6e&a{6_+m!=K(TQ!)O zLMl^v62gKO9fhutrR-aV5`KJ74}0kMidFaJz?PGyu(+iZ%$CM5*NycsR;_)W$s&6V}EonzOx z-(b`0N}(?JFifZ&A!sQRSjV}g;J3IK^5>3ZMX#+GnXyRLMq!uIVlhd-UB;5^WX86AtnX!JwKOG{((t(9uNn0;lf zx*Xfs+(?Z#qneprcbNIi>+mz$8g8B(Mfa+=(G3F*u^Wqav*gdKm}7Jv9d|eeyR|H! z?UV1(&HORE);<|3R_>Lxm|S2rn=|!{0Ij7PdA`pQ9fRzvlqvwzX!ug zC#pFz8R9RVgf*u3*=yT{lECoUxZJIj^{o#>?cp0hsjLMqW|!g#|K#WjzhLlJxQUlr z8^G~V9|8YO$F;A=;_b6uF!`@B)O^uHJkASYUbP?C%Dm`?X<_hk%}bnhq?Nh9jAK|M z9ri0!LiK`JHuuvNre-38cR`n7dXPU;n=yjTYS{yop9a!#0&Bcz?k2eRvIU~X-lU7O z&A|V18eQsA0x=Qtl05allXU2Y_#Js{sDBY#EZr!RTb)cFm}^MVR(Zk9*EezP z>~6Ly`X(N>v<_50-Czgr%@ymd+s+zeF0sB*@->y2x%jot30bdN3jF7h$W|1UgY^A; znmas|B~Bj>b#48?`jww7;@vn_r?Ep|3>>1eCkNoSk0H*9i~{8|+RR{ZH)KRufST1K zym0eA_T)vOz!;AK%Z0I+7T%#lji2D!<(#Db-CnpbYYGm}RE6FH``PH606KA%9pu}- zrCut-VM^;sKx7_MtF6bLzH?~IJ1JZFEEU!ao=fLg7h$jNd+-Z2W3g2_+ua$?tV7?^ z%sy-O<*fu>0-=vHm$HgHIZ1M`8{YRR07|^o;ZEB;uzGtKHnwk+X-68tzgySn)Tn$m zb9Et*Wf$R`Yc{LY?O=arzkrB2JHdO?BS^m&0vc-$($L?};m`Z8Xiw%mW<4|)ewQDH1N>&g1bS+vJe0DXLn(g%y;ffIySlt67z4%mgNPa^&b@_r&f)wd6kn0udKI- zJ#tI2u#7o)DyHyDqzv52K?%Yh!vWLFopF;DA6jR;gxlG(GWv6rY8tK z6~%$*fC30#GCnlR7QXv+(_5dtp)vD_xIX+Ad+IU}YV``4+o5xGLVP*%oUs_5xW~g$ zi-GX>X&`J8-pSf<`I`POUy1{sj)TnU(aiK&8dKWi$ok~Y;kefe>9EyWl5J{3*oh&( zXp+-1nB>?)L8Oo6)5>hNMzHytbR;r@p0rgl-W;GjPY3e--p zTU%PN`xJfVXg!iXe=!~TmkE8{psCD%{cia5-V+`^+zB%l)`Ml*Njy0~nf6Xy zjNK%HRx_rKR{5CWB@tb8diNui+BKQYGpdk1E=i|V9|FWHe9B-A48T+7HP8(^ zz`534{30=%PFQgfS8cD8Rq3QKHQm1~VeCp;U{p^(jC}&%#>C0I|D0#bRIAvbVJqRW zz&ZI+_z0WHf1~A3UQm*i4uy^T*uDBY)NIRo@N3kiM+WQR`OXTG(Hhg3rCd3LdR^lF zjpo>?C&J!Ol_WO=_F&P(Ic)NRc=$8q0UIv0#N%2|i9bj(;oD8Un)Kg-rttMAeR*gQ zm|skQbrM3AJikCqSgA~|#79P*p0ZOXO(l=JOj)B*6BgrBG~}?gc;{ks_K_m{5!#mKoO!JZUIteJU?09_PvTMZJXGss8Nj?gnf*JD7c3`T#$^CiG+X z?&6wv1RUIZWW5g;vgO`y*sD+n{O#x-RX;4YN3E;jYFyoL$+HP#L~Zy+&A{ zp^Pow%Z58-v#9}z*itqN z!nx%?ps^>HmL{xYuA%u*+p?XFuf2u8>DtAQT4bK7u#<=aS$z zli+plWw>)-1e0lMfx7G;-59f)#oqcz1J=L5Tl!ncM!mTWKPTj{F%@CVbo~=?W_^D) zb^)-{E&2Fu?0@w1u0qIFByeG{Ar474V8;H{kSAzzx{=C~3jxk7Z9zUeGi?ce+!@HM zMkSG{KIJ%Mrmm#azF*CxmBIA9*K=B;tHxGn<()p#9iRn0)t^EZQI%md$H`gy)xNY~DJY`*I{SLJDkDJ;;i~tLU?> z{UnWeAQ&Il75LXSGJQu6<~nZy9KJe0R(e^1c7C72j+bkY*(Yt;xRQ8yq*BHn7hK2R z|GLu~EAG(|{}r*VX=mwXJdAb21)RfgXJ#{N(4ejp^iP+TSe!nG`dzk!luadYW$k#H zZM_3xGY3OKQ5W4b%b#W(9>_{RZ(?#o%|Lr+A#M_#gR;tRH0xmz&B^h@Z>+Gy_<}z) z=4^ocf8VLZGYqQwEEp^i&bVqo*wsT}zt#)7Q}Y)# z+or=7(JfUC7!8uTfF^b0Ua!lpfdNha2CI&%Xh_sii$4;>HNbk|D}P; zkV5u`o&$@l88Bsu0=}mA4t{>OVB_S>LH(36%N)B$%rbS@7b_#`-eV4HTe89Ix-a(K zA{BQEc{)0mLg?>xyQNDxRs3iiMbi>?vSd`w`BHMNl|M*mZeKKO0%Q7)* zHqW5GldiM=#$MFHfrGMIEf`)d&$u%Zmi;V=o;A5h-5frFmbbq+S75+DSUMk`EV@c( z;svxJeE>8pPl694qwxUw{dDrxQ+Vp*W|-t$hl9-=>HJXnnn6}K;rk;SHvh&O$V^VB zVUfwKH*Eu(bAX@~qB%mYN*`UEW?VC(au6;awT_vc9tmw%W3gAP78|sEK65B2fj^gC zgZ3SJM!V-j`pOkBc-=8!?wEiBo%T~tElYaoiFVDtXT~*se|J-*5^e19JQw%;d&WI5 zN@B(T?yx!qKiJ%p3nOMZ(zv0Q!St@6drkZUi=7(T%Iz9ZHuR*RMZCZ&eo^@KV@2eW za)KJp8V+NgC$pq{fs-DciUk-sHe8a~W8utKEtn4;%t<<(h7hBOnFeq60&7n9)NH2a#o(?$c&cPfWdRvp( ziQ=6!IY$CJW36DNPbR!*GNZbpg*3u`8Ld=$h&Q(-(b^eFfZsRZlH>90MsY8jRx=1c zy}wU#u(j z4=~rD?GwCN!`|yqbHG=qfh(x<42}uGH&kQAb@2nqMb`fGKCJOcz_Yt%usa>=cH;4m53n10cJOQOarm3} zixmgCvnqU$7Oj2?pWB1!uNpaK{9`RvbLys_V+vVH8lm~|liABF6Pa$eA)WEC05=`k z4Ts+7P^-B!;g!oG5?z&!{l4YU0M~uEuxmKf&YB9JqgrvRx;=$dOP@@hT_BhoSrr)m+YKctfI1$f2-U?v5JjC$Ievl+iT*6i_NyQIhld$@u z)3Ohi+BG%d3k8WH4c{zE!d|1T*uh;L`0|fQH1?MUNTvhy!#T9z&PwqfW+wLQd&JV` zD$>hM2kD8N{sIGU0rh+3!`3;?W{y9!7`~eX?WBxir@g|QcoYkqa!}@Udl?Td3@=bIQX_+u}b5xId)F({_xsw&X3;1v5h z%bqReht^E05wjmL_n)5PYt`qnAk!n|`T?ZJUqwCh zyepVQHVZ$706Ox|b?D1bgH@d$Swch{I6a=pZqktu=%FcTc$f*&h=;V*!w~v!N~9+G zrS$ioCCt^0V>6>aLY~S-ykPiIW|J95kF~9btv=ddFPF-ygn32C$r+IB{TM0(2QXEu zKiG6iCdl=sWB$q%yyds5WSB!Sdz)I#9;rJ^u1uUw^&GFjt#{Qdn^%xz?vG@3Y%qL| zNnpKBYBe#}OKFb44$k!UVEI)=GQXYI;3O9QTW}RF-S8iEJ>|+W!n|>kaWT`j$)Rl&kPSH>db*w91^kQwb`+XL&FtHv%^hAeARRuIsMe1zNQTCtN` zcGGzee#73E|LCbWTS%Dr7B{Zk!*(qSVOJvSA@6fKzxKvYJ~c8%l)7A-kW<6?bnm^@ z;hN1{jIAa4I2Di?3PsW?5AyWQQdB80>mm<);GX`SO*SzXG&hM8oswUH&h#4aMfPE6 zi)}n@$_rBccrxYP0%Jn@cDcQy(SX1-z`b|IYW$`GaUWu)uSb*6vq z3GCK2AET4kz?NA$P*S}VEYuL<5g&Kv=Nf%D%j7-S!{hv(*bun7Q@2PLaZ0I3>HVlL!~`(WRYmt!E;9i=-fSqOoyG=E8d2DDVu=n_Wi&I z-{#@ShpmooS~5cRMJIL;>5vPn5mYqBK=c9NF|{GMG&UQLJAVrJ=Eae_vwMX1r!Iuw z@4v^L^{vA5vn~k_D;^ZSj-Ev3+>;^?3e7>D8%t#WV&TgJQ{XcF2cWv*Q|tKe2GAq! zS8M!M6ZkbtgV|Y3gsL?!@t0tG=wem_4vTq$9w~q5aC9aS?&K|q4*^)Ka4{&&(1DfT zny~fP2&gsa2qPCyf%}f?k#%++@X=EVa7M$9l#aXN*m~t0R($rW*``<$cKmS0`!e1+ zw%wRQX4iHK-#C8+WTq9lp&9@aI~?Gd#BVsJEgDMRmWRoE4IvnuOpFqApz~@aVmNLm z@G91ZSz1~`u|fxOSxkcj@_wIIojYi>@xy!n?E(ACAA(w$2Cypi38U>c8Om<50;Q5O zVYu~JkPtqR-14r(o(lv-aRVY|I&Z+5raIgj(gD6jo@TV-Rp71RWW0OjIWQ+}BD}j< z9!kwPA~g5f3_>c*pySCz_+8VD%y*v!HwI@4&Bo{8f!*go`o`mUi@=fO2AvZI{)>V1 z-(KdP-%kw1wS{gzGUV$QJ7VG~gu<5Wi`MY@~Z1;T$WYwI2`rEl=?(XL}tjq~G z$jLyb0COn$5x^Inc5v22Ibt?=U3he{C-|UYNtQE?#OlW~@RlT#xFgDN-0}O49io{; z@N25D#V`i1>0d-b8XpRc%Qg{f`>F7xxG!wGD`MQo-^Fu$7sDUjQ=mhj51CP|M^vv) zg!5zGgZ^W@ced~!=uc_^oA-ra->MBj+)fg9u2q1{og5~A{wZPHH4W&y>K#^3Xu}QB zs?cYj8P+J#CJOIr@GtwTz+f z76Df-nMSJWJV|%V5s>ZE!N{gg1b@QE;-0c-(AE0~%Uu)+XU#f~A89gVFH|M2H@1)y z-A>RV)s84%vV+U7X9C;pa^z3cS>S#59{4bN5N~^*1b?ly2Zv5lA?$D^X%)TysK29E9{g}-}OLybx9lZJTocBHdb>q0Pszg0LgSa2iAzQ?+ z;hJbep@R1YGI;nHK7HZ|2)BrWM-Rn9S@&v(+g1yJnDQ&EvUfK5w_z8FKjVTuW&NPj z4S?SWZQ;K<7baL;-v7;{hnntdvIWSEZR7mY;I=OY%j!X}m!gxQ{ zBjt9Jpt_|r-uP!4@XMP4ex=nh66?f>V$gkjFF~6u{^$XVed9x~O8mGXVd5hrvF#x`nv?fDUL*R|j2Oc>76EtiP z2-~AHNYAJtU7bxngl%FX#h=lyXkOtJUKb% zBo0dAvjTf0h|JhQ-2Q4T=&XGWKIesih=KXg&MF_g-^iDYrFsiXJhy|5AqAjL^%$_p zTnX>0d;;6#4}*J(ck%75;_xiLKmEFS3Gr%*B>N97#$ zEX&%09Y@q*{03>VsKp*ez1a-q;UOG;a5Z#ZyBa@Ovz&bTPC(u+ITB}@gEy3Vz*!5X zG7o-A!}m*gV-j}@P2b4F^%j<-!Tzt%E^h)kY8-__J!OI0RtgelTn6b598PXAgr-X; z!9@)>aRfJnpI(%M_eM9t_;K68(OhTp=)@f8T4oGwc3KMMXCmz4Xao1Jn?W}24a1iz zS3<0yKq9uqU=67`u&!1MeDWU$E6byqy9Y~zI)_@AZ_fu?`=BShIr}aU_A#WxY&v{& z${5B?FM;K)lW_B-VCc1W5!o2qg~u$|32js4Nt@{eoINT_%3Um>iuuD<<%_wvT&+X+ z;&IAnYCz3s<-!b!15Lvw9rlVy8Ag{z6ppIcX-u;x%q$GJW z4|xB$bY{75kFW_?)y;w{EYwJ#ssS9s&(kCY>&X87OJKf;&&|Ga!`r^EVzxW<3-3hS zz$>*qVB+z1?BrI&B$%Z`4r)T7pB2>AbQRVcEQ1Pe1!M{Pp3lr|haN`j$(`$^*mKD} z+;__n@^z8SI4wX@y?n^Um>sRICaVZ2?7~r>2XLuTkMP}(tJoO6!Sie{g9*6@anAQb zu)=Hryckg;_a4}iy#=8-z`4osZmAoL@$JPk%){ZG*C%lI zl2!QjA|KKxnoh#^EhV*WkAzxRv|zxdapbC*Jlqh$yPuDZ$%1#fFz4hKtWen{x>LjZkapD^CD!|{jQ zW*jT_9gG25u*O9Kr+nr8ra2r|w@` zSA%zP8u0AVA9(jpW%zZYD;(C6B->kKh4xz{;i8^FEJ%_8)uSK4_UDRt(zV$zJZurf z1HbVUBMYdMXaMh!Ab6=~I}R#Y3=2Zcq3byY8ic5l#X2)#!)6<@WG>|1fcv@__NkOt_-N1*Wd$$At4SI7!Tbgg5U8 z-RsSX=T7zXB)c&X=enza+a~5D@c9kUbelIZCX9nm56>p?1Nx+KQVN;Tr4Gkj z-$wkcOkvrB$z;O`C35Y-lh&W-?g6!>%KW;;75;bn4G^S75$!SM%-AJvFwUw1%UZYc zE&(8G+b@GZpJo%En)!HekuouN6vwxQ{^Hy6r?JU4XHehh1$Q`n1&VqG!tuG=fa;#9 zq~ObBB3cy*H@#X3H%ZMSU0g6+Sm;h7r>rKvE3Duyzc8{n$qugZ9ZRGv&ttvkqu|2b zXSkmn1(C_Gz)03SXxM+%Q9Dc%H%KJhO8uiA?& zvONsmHtCVsr)I<95lJHMsDR@}5!`j9Md-8bG|=km!vW?pux^7J%rMn}hWPYgdo$@-2T9Rx7pi5oB&SpfYmDZ_x?`M@Da9fE)L%z&48 z`Q&2_|D1k>zxkw)rEfGqu7oF?Bcn`A4~`S={r(XwR^NsXUykK-+0%&XiSG`CehWmsb3#|&Iq#E&0u1J4p7HPf5)}=i_ zu3`spt64}4%=3j;Q3f`zbRcIgJ`G7+EAm{L~ZS zw{OY9`|k{i{?k-g9=#SmabE#Da`nixi*a!EJ59Vt{|0E;x&^2^-oS^T9Bka(2(GDH z6IDhVN?J67jppNoC;Xm*j9o|Z#tjqT`Fuk##WMgXU)&8Qo^T@@-<}5TUS_a=(Fc%t zC=A+f{}22WJi!(J=E2+#3Zz_bFYq}m3tx@w6$;&YKqgoNMg?O?-KG)jQ&z`p{hACj ze2mE80Uh|XqaGCK?*X};dzrq2c`UALSHfb zZ?z-5)|S^gx!)GQTHOoeA9<0C)2gIzq5(KBxq;lMngV-eUJHkY(umxTGq@Kt^6xkW z_%<#8zF7%i@KOs>qj3aJST_%<2F2l1GkwW6MOEUtUInhb??fiV=iq>mIC$034KxP2 zGqxG3z}tEd%-^mEpR}sLfeqcDqv0Q@UHuZ@aCd|a{M3(1E%?x znciU?_&Qb<41Frc$sg6>1EZ~=UEhcau9PBn;tKe%d>`}jnlUlsJ(At)4}eKCF%Zl+ z&iqvoz=V;r_}MyF;yC0dEc0QpM~ys*&|Zvd6XJxg)phWW?N4xV#t_)nJpw-EEhm3O zX@m>(fv7EnobMV2ulvq|!EY+$i|cCe;JX5OD{T*M%AE$0w|npr^;*1ZU017<-K>TaGZNX)*rtLoPobedz3NpdL z3zFn)r3SpZUKeI|Z^na(((qU1T=Fhu8^QHc;b;CGv{m{a?w38v$R1b&TY_bv$Z8hY z>FEu}tTGXrRBFSd`I;~@UKeiKo{v3K^r7LIt2oZ49@}gG?lw?8GKGY`9p!V%+dg1 zu^&IaDh)MYx1Zg@>X$V@EpH;u(Hp|rS`Ij_vkKhx3dJ*Dj={@Hx8agw5^#G%GZ4O= zOy2VzT+{AM*l_a>?snQj{^RY1Gpk00W4Xt|;#CO9RHp+NDo0ko>BN%Di$U1Z0c>eB z5ni*K1&gFcaAdRwJXGrkCD+Y_8q|>pBZA>y?KI}nOby7NhY4M+#tKKdD_~p8LKv}5 zgJ?yjgDMYCvb4EDXkKjy-GdZJ8JbE2yC7j}kAkQiYqIpAnDG7C8HB6;#2FNWc*4Uc;l8mQJW_NkZvmca%T$;voMFg>`iM5Jy!sMs7oUgs zy%zz4vApHqG>PQyJPwr2rjmURmymibEqK{aj&w`9k|H}BuuaDtb~-%+V*_+>@x(4L z6UhK~qbkO3zZ1+3{EN4%t%S9&W#EfduZ4M6XOrkX$FZyFTGCo>L2ha+6LyU*hSmQ* z;(s0SfM$k6i!cLNA8QT6ERN%ClHu4(V8&ccKg7JUnGKSY>%hca7jZ=LWN_lmCbH0h zH!)linGD|ZdUE1DR*quH`;ph+_uXkk@sS!ZT6PIPzo+RqX+tZHyWfwG+Da4cU*e$n zq!`&=7(wzHq=>$_CLHbXBcIb%!QEA%L@`ql*oQA8RbMs<|0A5-Ps z1?1rw9b#Nq0%B@x$eK@s!l18X;T6>e{Hfj(tW{kEC-}|)iLweL|K41 zCRYh{Y$pOMS!am0OoF%cjmguW6Jb@3JeDYkA~L0_@Fw3O@TKP%rpG?uVVXI@QV4z?Yz9}XSs{J40E&KAYp(G#RSk4;-y}NO__d2jd zG>xduG>2}cx^RqPH&)o{!wmk>Cf4OG!tRAHz=-))vhjyD%$z3&7tRkOk`m(_B{Wun z=?|PpUqT@C{F}{)_o#zE-;{}q<4VYl90mV1+yeuTEr|4*9I#Qx9yZ%dfXz<^`Tfkp z7>gFe%BG3rm;Q86G9m_3|3H#f7DC?s*e&eluHjdf*~IevG+Zz$#ker{n5*nKVz6sH znb+2cR~?CjONw@p>+cPr0{Z8;s?LDeNiT&>u6rHBw#+50?@E|^YcBbavm8E$@JujI7jYgY5{k{c={SJRTrd1^pT z+_ZyvIp>*~#)zcy`yQ5S-ATXSZO4kh{vo^{5G+d>AUPlZd+=?Yu{NC z^Jq2{#nlLF3ar6A!|9;R?=p_9Pb4|ghe1*DBp}VWlhWHJLi=VX5@+&<8Ts}X%vk9_ zet%v{cH2!L{(Gmw1N=Jwu%#Wb^;agDeo0t2MnERdvmz&|EAcx`A6P6z@Y=zd&|b6> z>;BP(PmGpA`Hu;Q~@05wsTB6u0y&lm&0*?)XB7ouFRvZ zwpMfH6xisn8^1mCo7wV44aTef6W&?44&I-=i=?d4BX<)v!r7NQm}@U{93=eRVA$a( zc(dp;?n`wiJ_=t!UgR7SCZ-63C-C0ZW+QS$uo7)w6_w6#DU5^;GIt$UMAizK8^v&Q#7UvP?sew6%gxr(akj** zQlI=P>g91AS2$^{v2duoN;t(T6_^Gz;^QaB!pJ6su=`jW5uBO|3TDLJM7)y0JU%E;S*D=$fC%lX*(H2wcO0KAr{MSnX>jtE8*JR@<~4!By`T z!;CNf(8H_=EO9L+sT)_p%$=rqY7yTZB)yw@}a@n8VDs{||@m zQ~?sr5b6p%Li9IB2zi|S*5%uDG0}J7g-o>!Jz8}odvcN+ljkq)OkT7ZHM_`?t1;h6F z;U6t|Fm=yOd@tk@K*G6jGrxbj$a@@#6}Kh{`O@&lieETYmGAnhxPx2h8c?;^j4bT* z#QITF@h7i2aJq#&j8f3z_aXj(SF5&zg2z_y2fwBn6V!s|>W7jtdlliblQUpdry7xW zVc?Gc*5i{8#L1z{8^Ot6^YK9NbmCqA6Q_hp600@_mQ*dloepcr9e%I=MMD}yH=@ZW zw|4yb2Wb^xHCU}V1K#>C6u&6h2xAY9C(fIWgE2XVkWhKJDQ+tBJJ=oGW(IA?cK1iQ|0+mTs~mvrQ4S(_hR~q?wSqpHe_!MHYt5#-wOk5R5p{ExhWW z$wc(oKqG5qc;=1?w2S=>%7<1$E#AxwzI+Gxzc7Or-iU!Jmp*(JsKfP7^q_2-G7*y! zgNx>I4;cULU_oPOmya{ z!ya}T*_O0Ts4Q}W%IUr&%|sn0Hm(3kD-nD;u$A~VOoo4jZFou0LT1|BcVMpFZjj{g zMW`1ifmv@|{`aW|OX+HOp!+Kj^+^#2Zaoy#>jDs)&LoZN0e`li29tZ{lHXs<`SVF* zxah|M(x#`$OqWxF1+DG4=}tJVXMBI91 z;FecTu>PGg@haAYi6P#k=HU*~y*m|J&krR%Zx#{#ZVk|xJqW%Hc){lEv*1i-FC(|X zh|F-gi=W!$3rX<=_~^A4`5Ku5I{cJj*li2=!g@9s3fF~YnT8}=XCg5h*a+6-D-dq- zFRZsVkVLQ91*)?2$;2(+7`JO%iPP07K;FccTs%7#BLf{ce!3ko3ABRGzxzRdc7wsv{Z%nmTQqPYXx{4>Jwy;gY8Qk zVR-mj@OQQ=4BkAA%rTfm2Bxc#+p!*-;(!hviEoKJ@B z7sAR!7truU2}HY`15*?)gUv%H!QsP*3_nnUMFwI-bYBHZd@_LcuP=fw`%Xsh`$fjH zU@kGA{t-9db%*AIdHA2tL;S9K2Fal_L91x2i#2;fqa_7O8^E=@F1@oXuRxIgz zybhL&EQo!K3;snGLDAl)po8ziC@MVKdd1;wYvbK~M(5=ze2zae^Uoc0d_I38v|Z2- z1c@R>uiX<#pLb}3=fLWS=i^ z0htsR$Y=+Vm*c#_k4QWCZQKd$FzyTX`g6wd$?9>$Q^pJW@B9mpk}OzVb{1c3x&Ulf z9Kzjlv*7UuHbT30L%4A9AkaR)5RRq^vFt7@C^zjKZm?H_R^xN3b1zc}-P@5~DaHK_r_uXrx`{AsH2u)rU;lV|uwtvSiv6$=wgI)u_z&vCxk zL{g%z1Kpex!8KPg=n-`S2QUAEZ&xV-nZQNx;!9)rIaU+jV_gbCAqGYJmD@hu@ zw1K0^17LykJ0>O82t$c=mm?pHCCJQS=RPdHR z=W%A=7wj#s35$C!;kOT`k)cfwLCULsP~^Otl-?V{E{{x!o0tXkm@t7PzvPn(_&B~Q z|4w+}ECh}p1th)Fh!h6K^W8}HWZ$(ZY z5M*z-66GI?aE^gHJfEy3JQ=PCj~K3GTE5i->wb4A_^1F|eg%`S9UFz0#@a%nl19F1 z*x}ZT!+ht947r!sjf+D+nd%bW*Te(Wmps<2Y%Jt-PW zJ($lM!kVqOz!(w|P$LxIVMIh7HUw)u!D*Uad>`I>(6Ik1KKAH1FuO1XzB*%#^Se60 zfvff~dr!GADOH7tjidtIhe`y>DU7FnmfU|-3y3^xG5f#QXy^pI6CPPBu6B&@Zhp| za?U~)roA6a(r3p)+eA4yOVyC1xG#ZAK6o?INv1F`C=G}m^o19@r^Eb)I4EnXOMI93 z!_GH)uwT5>OL0BXk{lvn#!Bu+Q!pqCb};sM-Nn)GwYx-&~x*1{;iTD^SZv7Y+p? z@7j~-ck)X#eB4oF^VuKW{BoVmFK=Qaa?c|6!8vN2A=V~+Ok9v8i| zpA+ubx3hvj%gfnkqjgBM?h3myv67Qpy8?l@`RG!2Dw5rti@F$DYN{TFZo8ajk6d~r zdP`R#t$IxqQWi$P!x405-dU=CwuEgnz9;~jGLSQ|K#FE^PV+(s*|Q}oZR*lLL`}&; zR;gtPO+3C1c`gyMx0cT1h}02Q(XJ8wn{*bHg;};KHcX^fe`ImytFF>&)pAtgwg)Zl zl5nbU%c0hTYfz?y6uRF^*Z{pDdM5fTRg|$sqpSEF$QozvNof{)O#c*n-fkcIUMlT$ zYwm1xJM1XEKGH_JGMCVO^5@vky9z|7w`)4>rAJwZy6f!0hnKj3&&%kM54Bw8sr9Jf zkUZUJs)5kd5^7za#ib|iqFWwo@OSQcpX})^6g{X!U)mc|)1FLPb@Vrz@2idO9$F~4 zP$9?W^bR8P*N-@mE>Cq$S0nfFE2!UBA9}QG3VJOPq0F|soPtUnXDdC74letQYF#|( zT2nRpEv}n;@H+@i%U?#fJr1B}Cl?5&TAQJa8{ur=<5|evR*hb}W7Bq2VLDO_J1)2$ zS48!i4sq`?uA!RKZEWeoNMtF+`wdIZb3soj0>T31|Me(M3LTFYT$Z8to~LnNmptd@ zSI42O1yg8i6r^Vkg;RZr8fv~q3w`QAob}HE)@xEBTQffw&3(QGDK_PzIr>+*XYn@$ zc%L(B@ll~K?kRGyRhH=9sd6sGe`add0Q(?-1RpP@}qWzSQ+y19v}; z7p!7x*|K4A^k3w5wtuQxTaTftQ$F9hJ!_f?9aY@P9@g53>_U9G=&VukArhVy%~fmSY;rZ0R9 zoGwSUp{2SMwJYVLm!)T@q16TM)}~w}fijRp_Dc3yM>+lPyj0umTQgV(7fq^b)60I} zmWqCj++j7}I?}5r_n^dx9D4rOJ`{GLh&%9W9qVY|LRVgBXT1*uu$e-fMNJMkiF{eBmkG_H*+(X!w+#%NP@rv+Tf&?B^Vr8+0YTQf2N^Xc*R zx#*SiTy%TFR&J|jAjgGoL!s*=d2g?gR(aJ?!&I3zfy7T%VaFAi3l7js-U0oZ*F(<* zO{BUvO9cZ%&uHS@wQO{*8LN9s0nO18QR(_I+&7O)>~44ly;`&gonG;SK4LJ-S?=e; z)+-^=G&!fk5`VbH(9@#EP9xTHehzgHzR3CDY&zv=Cwdrg0XP{4YK!WLc?`bbe7K@4a?E?+&VP*(?!brI?-rbE*EofC%QfugofW3AR9+HRG~GG1}vP+8k?7K zmsk6cq}6ZPlOZQ)h;=bt@vH#Z9M2JCIcq@Y4J**Xshim9JypE>pvp=5tYuFKJsgG3JDxw8w{keCdYbCXTXW2X9VN^#pj9M-(L=!HRQAzD?(fNXFoVL9* z+EOcvRHG(39Xj7jQ>R}RJxS>o9a}h`bL|tsM9V95-150p5MRhj9GB+?)#lM!luthu zFQ!r-{}WtIpN>AaUPDbIee@CM%03^ILn}^rvtuPEIxW8QiT$B5j}GBM;Zoq`qt1nqL*%c!+7lZ)@7pC&tDd6L(@-CQ;jF0UHtHAPR_XM^_E^ zb85OLQS$CTTta#=wJ96sIu6`G_f2jKLIwHg#f|NBDB~}?ofxpfuHCd^XPUq_O^f@M ze1@|M;GL?#W%Tovbo68GQNf;rvUKc12`8U+FItr4g-U-o(q!Lu(Mxl=HhHP{XvKSl zw%JF|(Fei$Xo8C+Dtjl(PCKj#Xa5bRt`kzwkiU>>o@Qy}v4!;SeplLKEG@X2)WB|6 zE)i_-&thZ#wQ)11en9>W<|y)eK3mwho33*;<5Hx)bH9zB5s%WftSIb-pp?DFO|UGW z|Glyn9eDEwMSl!KZ?={oPv0xFz~u~kYr}}>@{F12Y4JR#l?}RlcE+8KUz$Z1>R+VU z{Qtn#s1aS zMwfz}==N(ZXmZ^t`l+42Fdc1AcNK++Dux%M*`+JdV2Lc(Ds>t?AC;uy5l*6&E$0y* zoMQ`|_i}}j*@8>z4_UWJY4mbz9jYD;pfe*qXnETscK(Bde17ZzvV41sKHIf~o?cx- zt-k$c{qyzcKmL1fhKaYygsV}XiVRkLrvtimeLq?sm4KGRZqdmt`tlx zNb~qHcHj2D?B6d_=#00ibP>OU8N522u1oPl;l~NN=pf;wvV1m*9r(^|R&_xST1}{1 z#RB%in|YjH<2CAfej@ts&=?f)@dg)Yc2Ka4_v+N`ltinTT3TgaPs^sT?5MDs{^YZ! zAA8NIO5io%*L$A5|NbKSc}J4%@Xe(C6E?FOqMvf+=?uHY+mK$GybZ?kd1;?<>jcI= zjdX9&6DU7*I`7imq~qrvr#pmesExi8>Oh~^F6IBY99fPN8$C=nWYvnazTcwebHtGP zxh!E51Y)5G&+_OQdLtLPj1RAdxuiBH_x4=?HIpe(MB&owSZ z#%ABBl3y|{S@syc5`5zHE7u|C8K=>#O%3SghvldubCl-&IEv;>Y2l2%l+cywh18|} zDBIPli_CitqL%Wrbj6RV~4EOUVVcukQ%Tkl2r&tL1U!T~Rdf7DH=9 z#}Qg%kE*t4pt5hAZ_h#sFbm1g z%BHzzhe1U(?}Fu>M{&!K(B&zYxOo=C+*+RxbhJPbWorhpKD+m$3};(p->6BS$a&J5 zWqQ;NOh(hAhmgT-DK51NP_Vj|y&l~v7^s_!(3B}uRJ#}1nkv&7p=;TAuLV^7n-n@X z&ye2C+Cj5g3t7jn$5EJC9cw>hFV$Wq%egq~pgOS*_TTDu_GtY>LBY87G|y48Eq=`r z?n+b=n|862b<`@P9|y+rM)E&edLsgPZ?B>sY%7ud;3*V;x67nWH9Zr3lwXUUzkEf{cbd|#>e6kb@&VUp zxRU;m)Dn4CrqfBvBKC`dmXk-^HR@x33q`6G!M6c!GzFdFMq+Ad(dliX1f}&zy3&{S zuBu}vOtWRruDgSL1D$C7Va!#Wxx?x4{!`8Oc-|qfLeZ9G+@uau?%#x0tlP0Vq*7Ca zmcKkG*x+YNAJzS3JH=}#eSd)NR=tBBNZsJ#-NdgH5}fK_9Bx(XZPpIM*kt?DZAaZC@TY(ZG4)PA&UXS<^KwqKA*B zq0#I`9Cpvj{>-}!*Q}yli_ir6^VW)sQHh2bQsRg6{*B97Q&&zb)fGhfU zKNhL#RIzPtAzb@sO-_}yL61-SbAM&mQHj_E?Asl-bYGVuN;w&eZX6uvbZhM&f$_;v zq^?ByIlKybXY`znwC<#rFV~=LA7YT(pHJN4p>^oMq84i8n#*y8$3*r{YPwivZxkWm@A=c*Y1nrbSEM4D;tpFgnstyf0PaC&ZE^e z#ey#bW=?NctJ33qZqZdXi)*>EnZBwlr6q4aqKie7(WTN+)_y}dyD2MyuHJhcHCv{m z8N1$aPEy6(?0O-29`cnPaGyq1_iaQ{yQi=RZg40F4Uoj=1ZCS~BGJ-^NNz5Fs675X z?HgQ-j10D*807@?=FnZ5R4eNgc>XiJ7krwIei~xsJ}1*Pi?`5A*_Et%0AQu|4swYn zGr4VnTFAiAoXV&qvZ1GAxdFve?&qr@wCZU&{G0fUsxFm5MqOg)$iWKadAbM%%ax;$ z{N?nx8B3p9^6SWj4#>vh0kT-bTg1UR)C0~I*!Yj7f|3e)HfsXf-JF09%s)@ZJFGZpBy?Z1AI-c#tHZ$6z9NJX^JaY4ptoP#v=KURveMrVn73mMi-d$+LCy;UG{a3bu= zlc6a`y6DdRA6dt$7CK?=c{De_o*lYU&qn#Ev)0M)xJ|y}S>>EURBYOZMmrQxqDY^f z99K@`pI8aZBJ$AB&9+Vo-{sNtF*5A3DV-clut6PnAE2@yV)W^`{WLqHp2mba(4Q|4 zaDD72T9vQSrgYk=jjsE{wR|1dc4wy&g`b4z@#=KC<9H~wP|#-eTLL(VM&8JoyMvRh z?nBoYp@90|6tuUh2=;&TN5huqS#8r>>}>v^aA@azS~;A`9UNbc*wcjUF1XL8d0t@U zG?yW@fn1c-G?8wpa-kL*%q;G8%cPL~DKGxOUq>Hh0rl z)SxsOnMas9CB>zPV0EbIqT32oiVRU(nFHN8HfSyv?f$R&8kWaO`XzR~oRQ+HcN{{kJ_2%~6&axq@eZ`iVFR7$X z!@E%JKQSi-ecl<}_=VN`w1B(aJHn-&jb-g;enC6Ef0O&VLP2wN2hFHFjHGtt(Kd_U zoY}Q%+O7YGTa_k__7)@|2F;+6hbn2VX(5{b>@rubvLCHVFQs>Q(?NG!B^`Wm2Cdb9 zjYe|4Sw<@t?9#M$N?j*Sqf4-0)nGo2-&nwI859{>vDG4lB?{;r~0g4 zSqUw_I);0C^f(nilS#!(tmu?6ZuH$tME7sH#+??|aN4MyL<0^Ou={jp(dSQ*Z{BQ0 zek3{)8o(Z2HpEV_uYl!QYUuXvr)=`cNgf}uQ zy@`Czg`y);Z$*F4xY1!(WmKVhjHXA3Ih~h1!2WyCD`-?*f>tL6BabmxkpHMU^4mPF z&Ee8mHvi^W`t+ccD5d6vAlT8D)zEA~%eB{26$PQ_qs=+=)x4cO(pQW||7N3pnZ;Dj zT((VNsSPV$couD+^b!TfB#F+LXCkR>i%{g(U!3IS>D-~6pSdlgGtu^=p-AXa#f{D# zi)6gKSa-24Y*Fni_E?x7-R}jFp3OQGdN-aMifP#mZG&izq!p7T{Pob8i{;gfdY)I zSu5F6+8Ap{pM6}2rex1WpN4%zKYjA3Oq&VK9!{Ybe3x?m9{l{ySf57SxyHTM(*wVx ztXP>hCG2FiK@M~-M+*#hi{6D7q9G3z+IL|c@)whBJF@#eO^B{V|Hf+2apAgXI;x^` z-e+*m|2C8H3rs}GX}a`jYz=Zte8Z-Htrf_abdp1=&rm_aLA3naD=O9{K<+u2D0;Ih z4R>L=&0E{K+J!*$m#K%v6%Sea8=0uvC5QGp7>FdAx1r$Y>yd@>R=R(c2AW=eLKJ0X zD)8PPO63-HQ(eUvw6yd%RbgZ4Z~nKuZmNy!rrOf3-0!UZq(SaXzZETVK7kf93iO|f z7-~+c=8pXrg!BW(qV@!PI(NfeG(#emeRX!K=u~V5eG=|TyKZ?%8pQ`7LNli&J6aLaX9oX!6YdmIT|d)|;lwa=MJuXA|gMZi@K@WU1c z$j7n^<%0lr6daA{r$WIDp(i$ZbO5^x!Ab=}DL)vSW(?-}^8NDuE<>|zR>lzElq zy|8c2D{_a11FY>Tf!(TN0*5I_xH}_*2~|?VYt^?f^KH`L)5dnV>UTPRA<%*2$3x)< zMQzgMYC1PGWDhK~It@R&7{EWZ70_|`7u;)N%k&-mPG*+8!Aef`@OkGcJZ8HVZe!;F zcm9iquT-{>tD1Fq&w53nbzdoI@z{}hdGkG2uU7?ES_Z?K>@CcZB^)ft;lLHHqJncv ziW&R+@i<`r4R~0hmN8LF2fFniv%gVmeYO44a`cDL;S*r6f-UE342#c!u;|$tn)$% z_Xif?H!-oy%K7JD$nY%QKCT-4Zj}h-5?DQdG6&xJb{6*G;o#CKh6wI~kjPWti%L*n>oMF#UC*_~W4jp6&V z{qWwRP*VGNI5c?b#iY$-YnLPi_;m6*X~g-?H1_SH1ELGbg9XW?)a@p|Vl@d9C+^}w zHmh^;P5^nde;w>In`ZEZCY+P`f_tJ-1#TYR03FpAlj;vs$d5i<{K|l*I5>O_xi-py zYxK&E5$Hve;%8ZwPJJ@k_a+_|cN9X6vB%5@>v#MU-aqkNp=)bP-bJ8)^Q9efm=itW;`z-2cr;H|-UZ12dQoMbEyYSk%YnUHzUy`O(40CVm7~>hY9(rZ5mFJV=(Dehw zR5Rn~#4j`KzL3BfFI=!&|9sp;XJ zrs3TCQaqnV0X*Te9`|q*U~5eiPBeUh4>{PtJ4RoyOJ5QH=~pvG?(KDkI#^4p!IxZ= z?hgGj+L)Po=NXS61t@bjnH)6PFBp)1OLl7IK!eCqetftXK2oP3sMzEQOPkB_@mgiv zdS?c1P?6?!&yR&S1~0%|bxRm@ZYFa=hK%i}nN3%x zvGTe|@~DFlCfeBqGI8-hV8{F|^z(gc|bgY)SmVW{MbAHA* z)!WQ;Es?e z>6k36V;&9p)9H9=_IzfW>JI*{yP`N!XgmJ5<|SFT+<+M)&f{=ND-cq3&-!Oe==Sc2jM#>d0y$d7E*ZEb$C-&nOxjAf@j6Q!&zKEeB$sN zoc-VgR+Gww(wuqt`M-TQfn7I+PWQ;C@H}k$bdtZr-Vi<Z5!j-Ok?{a*Tz?WvaFXu~}0 zr`HD^clAKQ2~k05WIHJ?{{u@}&Eie;EN4XXUXz(EqJp_b!^DFuGkDX6)m7UsGk;m8 zl=HItWFc=J-ux<+al4g(+h5A^lAN~S2Qf}i=geR7&k6Q_KkMgDdn)pLY=iL6CKT3;K7j^vw+o`!nO@?NWzbh{0zZ)R zB7MKukgfG%ygB>wvAs$qWBGOppK&pP!*ey@ROJX)qA8yojGYCuU1vdMYe-&hc7T%# zJNPH|7?KCypM3b?c4QGeE2f#?=irOHYT$f&FbW#ClSoN9s&7fjUA3= zYnAeSYjL`n8Jtj)!AtkYlh#4e|#p5 zohFMQf8z|;RdQ;LNj* zf(07SA)S)S92UBTpZP8%qr=yd_uKEo2ah_q!z)ynFJ=+~)uZ0HAW?^C{&S2Go171$ zH0*G&!92nAxqFa%C>v*`*kFU?TZ{>RId{PIDZYHB7oL)D#Rft-xJE-A4is>>yCtWf zi$^zZkI{v^{kQR_x=OA%uNAM&%z`yx0LG2qAp@O#p#H@WoO^8_^eP$PA7A#3aSb12 z{7lkdTACR=8=Q;_rkC<23XU)jHdnz?Cw)k&*}(W?G@u=%p?URty!`hy@?Me=KCb1= zOl#f1ogul*-PMPn{`Z6MB(sdX1GYoQ3VEopAdK;d=w-rk9Jo7gu7}>4RS*v3;p+n; zq|zfPcz=5`9?)rIBt4cu&#jhRwG1v}u;@IlSQ`M#|4ys)t%M{_(pWme903t7OQK6(fj$_K&SE-j?}VSlE+;skWm^TfZ`RxzpV z;qc-)4<>4CDYjFYB{+532HHIsWK^6xxF#zj;rsO(xHESO{@T-u8&!+2wr4!sYuzI% z(?+1M!*Sw?^CGCS|16xG5eQAkUz7e(D;OP?-$<;zP2N?}#HxqEkN)}FT{KK>qRgk&Qu0ortTvE#S6|T?@;9pJL ziGyz1Foyffu~xc1ALe<&Z@&u2SI`v>70N^H+iA>~0&%QZu@Or9-6Jz=a&hLP92~Ta zW+u*^gB~1KPpkO{qvX~zs7_3vH<3viJi3S7yg!j=*0jQ_`j7dC#7E$jPeuY}^8_@_ zd`zZQ*}{oy72M?BLpa+>3!k*OiEXosp#{s5>owHEe>fKKXXzmGe@;hJvl6_0=sI4o z?GZk!*UWwNqKf>m;yg)AUxG8HJh9`8V^GX`HC(;&D@<}N!vpC@vEs`grv z^lFB9X9tsHe{k<`J~Az5v&m!2+n8-dWzdUdmDZwqexa)|oajuWHM8Q#;(q$kClXCl5?RT!H1DIk%b5FZyZz_f`19SGuwmvpwsJ}Fh`h2V!RQ}F542e{#=E>G&q5Thuo zfzje+FvxujhyJ<*jhdqQw+m8mdS4DBJ(t7#E8Rk#JAV%sUXEq5PjO(pFNROlNpho7 zDETVB3`&hALfSehk^YtrBRa%zpjrvIP9F0<(Q?`&mlc$ zuHy!uTn62O=P~<6D`1REB3XR*9vHKa!&+XAc=Uxj+(#_Np2x&6U$YRp4P0Y3o7vzK zK@z0Kj2YbEv$ELuzB1HPs~{hz{>2yNSYOOcH(WZyiLbePJ3KzWiuDqD!J{+x!=&B; zeDLxE^1I{_?y;T}>}q9BI($3_LvQ;)i4$VDlNS%w+^n#b3npuNyZL@XqF88wF-(3K zfQ!>fn0KQCw&WJU%*V&@rggXQ^_@52Uhfy&7IuwA__h zrOe0Ox|p>2!5wR181)AV*vEX7%TyWh-W0pzV|{C3FWAY5ZYdzO)RJLx`C{I24?psE znj0x=-GYbAC*g*RW;kt!jKFTeUEH(T9VX_n9IwqKaHehoUDue3pLDgsHt7m{#9x*t zv`?7#-w6SE*S>&sTVhOF-zj1mHW$NRI-)$^&TIVNnxA0tPE#@@Pyj=pe_HRC81 z9cX;+Gr9R2hNCS5WbHLI{Cv;`4i?>HoAeFjndmThm=L4yUEhs|grZ=YhYvjP&kwF# zNnk=?2_3G8;-$}4;SaT%I5_7$_g36h@{s2=&=s|V+uyx{%fIw6OWW(=4YMUsa{~{% z#1;@2uZH6kixN`HIF3>DQp3cRL-0nu5wkKl4}N*D1g{L)&*T)6I8tT_p8G_CH@Z0z zJI@j3`SbrlDNPBu@s<+aeBT3Ro+lVys}h!Zzm&|<(SyAerf}Zy1t_&pAK%T-#CpG~ zVd$a)V#SCi8JYZo50*TF>OD(wRxX40%F*y!k&-}bnLWN|Gm42Nih^mff*-N4hh&s3 z$^4nO7^|+L< zH{h5fpYY4T9}$?9q1R@NiMApWyYp2kORXM{(irM^jRH&FN&RD z);Qacp#mY^%TL)@Kq_Ojv5q-YdxMm2Nydj|wlQJRws5!4cILf_IxkcEBDOzq4c}1H zCx`Nmz=b|uS6Hc1*0PFalCg(0H zC&@z}nYE$G@Q98BzWioA`LH|}D;R}fh4rW4gM+b*RdzB{oAVo|ET|!8(G8G`s^Ef^ z|3OP(F+pj+5x)NJ6U_7ytNNNI=JxpHuB{iR?pLRz$2_$Ms@^39dbFop7fp!%aF!?C9mPO$1AYXg^<2cTf`X8h~gfXf6LUdbK09P<#9s93S9P@&mD^_#SMOgjPlH_@Jsl7 zxZqVBn;kzNmX1U06c)>zSp0_(5pHG(r&VypoLVwxy$e1JL`j_lVVoS*Mhd-(V)S>c zg3(|tsla-n;<;NGKjGQfLt0X>=2t)0vnY;C&?DjfsZw&W_X);ak!8w$@?m^KKk?r; zaG}}0Go=4iA$Q;%jdj!uq2d-PQg^cq{wm)<3g2tPuZCF<@18|C`GP3#gZ2r2o%s{E z%>4qsRQ`=W`TGrC-rvof-%P{Bm+kSG#U1Q)M-eW2(L$ur<72 zzJM3uKMCtas`0yWPgs*QmytjBi}CmQM4XW7U=;byeCL((Fy~SaDY)dr945KUvRYTZ zZ*&0;bUqLNIhEiJ)uS+&{jY03>BB2gGvRmZ-Q@Fcv!TA;eBQvIEB0dhf2%2Re2*^S zy9}h04+}2g9rwl<&MXJKOnVKN=a|Tyt5XE`yE@_LI}sCpzl!vmie$RjnT?*p3~uR# z0}!bKJZ93voXuhPpLAV3<$o1k9QA{1UGkvK{(q$Yl2>rzCgQ4?oZ~+YWA~svyP4B7 z(&56oZLs&_0>P$rXPB{PA&eYMAT;v7lmE1Z;kD_vq@_qF6ZEE>ba;FN+f|UHqYry8 z0C?l7DPzI;dbW?-5dp2P_2Bn@)%+5FHt+h{E7DfWh&yN3OuW&O;{H)_g>Mr}naKGa zOkGnr^O^ge_P^NA%*a}b=U+6y<7bcK=C2>e$ftdrml4>%)a%7d%!0d-!N?=&pi&uVJl|ijD#K-GS8Mx#QFwr zd6d#We5V+NC)b$rCoC&{>qEF-!UQ|VmXLIQ5MyoILCQwYz{;=O@i%T3%uN=9-pLsZ zHCux5+9gWD%})o>$5*m4eRQ$9=n5S4&6 zESJYqe;Gcz--KM6pN>Pe3FE_ybm za9%c44l{8MxB-T-FP_^t=-IMnWgf7*cM^WDK66H4_7RF2oEO73JPAG#Qf_X(4}sKd>yX{cZ;9Fvhi1N@@@rzP1ApH zW|KD5sSUu1Bj=#bPzx6FFM`IuKVZIVC}Z7vjj_Mk&it`vxrT=a$U}PvaMOR<{8hV@ z@Tz^0*svFJf1e71CH0t$cgur9D`lW^QUd1KF|hdHA}H_agbO0dna*EMa8q6>%n3*2 ztzUu6!uMBTfJy|bH*ST&gC+P!%PIWu&NiqoUWuEZ4)OCp=aasdjG=LWHH_|G1f_0W z$1e-J;5r!@JnxGFbNuTE@<5F~_GH;&g=I70h8uPG*VGTvf5SDV-6$Cr8LuMK{kD<| zF4qx)$r0$V`5Ti~CB#(vJiu4Tqqx|32TWyacf)u&!QVd`u=VC;a!e?nvAyksx5WjK z3m^UAsx-|b1I8klX2V)$!|X6-D<#Ks`ca6(x65GZ@p_V<`IPG}InG~oc7z-ex5Nj| z&cMgD*JD0V!TWRL8N$jH)&@!OZpeASaM5+x*8T=#I9!gYs~Y?VvaM(3FGti0C_%IQl4fW8O!pV;TNR6~Qo~^GFB@{L?QuL` zRv4$uR}|cg!KAnEIC@s>lueJ)LE$AHU9jX>f>~eVJu-zgEM6C91H0?qjIXbb>jl_keu%u#?~IcmQ{u zhfwr!3_09)0ZWRMk!BN@$VA-&##deg7Jc_)dhc9-_GM|z$!os&Rr?6eKly+gHSZvP z|Du7sb|DfU{&WlL%RPin!zuWVekJtrdI_hu{Dneu(#a%x661p=c&>~XRCF4GSz?(` z@q8J6CbxlUU4Ijrxpea@-g`mw&sK0pO+H-s{xyy*E5gn*otf{R&!B<9ZG6yO5<8!k z6Lj@PGyU-gq3L3FO;2c$)7~ynEB-q?Tw{y>wpB9fCBD2Ks0zo|zJq&YNbL0Q4cF~q zAdI?|3cpn?#nC6@xqp5BlB4D?$z_eA*yz>hS&hDB@nkZrC0mLz)?$!<_BmtgXZp4xM>It~s2* zw+_9-gg(84yS%T!cA<|nyyAh!*PO%Nlc!;lR3d)hv>b-N{fLj}PBP^yONbw(J>bA6 zFEWw1f!!+%1X~9*puT4ooPU1d018r+YoAtmi)OLiqy6d39lphoBSWG%)wUE*y!r0?zKg8w@e z>LZmD5I=$41Kc>>7ZZ^TB!)r=5tzHejwNLD#>j!qxhrcb^OR#9dA%EXTE-q#kO&4U>lU>W#?SQ0n0RTf^Q%EtAm*j zqB^{DHT_I@Hp`}(yoeo^mcqcD`$)e=5^wo&i*LDbE3|aj2fKv38BM_`(-^FZmx+`3 zWWg|IobK|~HwX#-1Xkb!{a^99mhdO%jb#?8E5a=gQ*oa3GpwQO2`kjKh;9)8 zW!A8K$H6%4dDRSerK;h|JN_I!!Ckmt#+vEWx8{4!lOu0@EX5i{yI4PU9_umqN`C)& z2iLE$fI|nKleNXAq{xOi?iu#JR<8XPa!+6xoW08%kBanTjSq^vTqa;6Tm3VOZ|Ei#(=qms#4= zO>P(;f(4_|uwC*7ocFjCI>8>Ulez(t=;-3SswWJ=nVI8L6YT4U^+V3>LeOoxVswlgpLN zZg)3Wc=-(8ktoK#ODx>g%4+pl1^m59C!t39HJGOziG5!4@u!KqFsn=n?lO5o?h=!O zvpfRHJDXb}rT839^j(3azY?IZL=-=tQGyW(=g6;S#n9ys%Y#2&g1@%v;GE7#sFzrQ zcR%=w=f2&KWfoE}Z;2x|6uQjLS>*&yObbldFUDAZna>NWk%cxdVwuS&)wunP82O_+ z8aho-a6I@Jp6T2L|G3>Gty9gJwukSTuPeEXRJuQt?#7U(b?NYaLIiAR5QEcP z75I*+AU7=jP1d}Da6?Tu{F2cI*M!xO_Y(d=rSVF9woC&qOy|PZo@i2SSvYCA#SoTi z&0>~addTeb>17fIuH)lQ0pxU#CfQS)06U)+VbM(q=1{*}2Sp zyhz}0dy^b1i-q(BRe{&dW_<2>5)9nZNG|t&MZO8x&ol#f*uRd)YcJe`qq|!mR&XJG z#(e~uoeyD)|9dhrjSpYxcH`21A3Tulh-LC^VA;kba=up?f9a(mmUX!mn*eq@K5Z0{8jBAU#-uPPM6YzB4NI8#oiG96}5iGHdzGjGDoDPDXXuoAH+th za&Tbm4{|2!p=^I+1@G28h1VwY$jHn3Fd$_ed|6S%Y-?CVCTT5$lP|TPW_c1+5T6Hm zGsJn0Rf$-nd^KG6_$iiN%y3Vo9L1CM9A@1WLqSfc2Xu?eWEuzRuyBzK44bi^gc82c zy}}Ca+@DSEOj|BkSS%0I!e>LNiO1yqq&!$_(n@@d8Nq=nq4-nsLuS$M5EwMxjOWkW z2u(f0`I#?k@wB!qZ+2@YW0mCr4fn*7JD(RZA9uAd&()9N7*>Z2+uBc-4=M>X8V#78 z3azB}DKqS8SOf*^H~Hgj0sq~I!_z;ene4q@jL_FMn0TlVf00`X^HnPG-IOkr(U(Np z4(`M$f~)vQ^egg#%Nf|~5dym=Ebz!J5uDj|A4Z2JGu?Bf@c#A3@h*c;q`|Eg+<3b&1eX@uc*m9KUiwPjUaRTYe_7y-F9z;i!+Bw6-Yw+cH zIQqHE5g3;)Clb}|>E-DQIM=R)gNP1!f)1}i11=;{*K0-iyT}k-|G4x`ZYbjP)C04n ze?ij)5&C_0DpHqCyZ7g{FLmvRKIiIVQTpw(K>Dx$A^L;R5u){tDShR{E_%RvD?I_c zh##N-qt)Ry+BVdZZmYK=j(_U_jT=3Qw(I`HnOT8!ThAx7r*;)>6z@&1S-p?w7i&Ok z9i-^3J=(PYWpjde6M!E11w<5^=lS)B9^KIoKE+w$F1`mnEbL5-O9&Gy+<3&XtPO;{<_3DvnRZG!RRO41TY<^#kErnl zMrcY^jGyN&aiWX+H31zccgvEFuAw(7zK`&@0;oDCyD& zNSI!WI#$Th(-kGC(Q^%W_PU(2v0?+Ooo!c9qj7u3JlkPxUXsl#DpBSK$KU#a~3Ep$Wa{ z(q%NuaumEexRlN>UrqaNCu!GqV8A$PSbqiWH(kcAJy{?#C`((s;h@u> zF;Gs)1J}NHaN>8K;Y2#~Xz9Xn6vHhA;c>wr?2Hs)b3cUARR4~um1=xs^$4;+_TS=ywm2srH<0xtDGIcgdDbceSK5;wU8JfCbuMTYa}8=voi zDtaf{wCOey&TK^DJN>}5lb6um;vy8XrjO$_&DZ5Yph30@T)I2~^tPpPst*@~H#a)bLp5O#9asyDTxIC1>jmhkN*~&)bp>31 zWkN~jLZWjHuR1D+Lw`!=QKQaZEOh;Z>5XR?^g2cuDcHMmZaz%}M}}t;vx_c+-c>)5 z!mQYF^E`~Z|* z97U#LXV8D?YV_u)P-Hw00F{^_xOaD}%Xd56zN>)zZ&@~(GL>v%Hy<2qkFo6 zz{yyhcw>MNc)!g;-z4As;p76!(mEA9Avh?KT7z^O-f{FKAF??&>xrnY63`X`=&@ZH z90%7*O6puEIKovTTEoXs`9+f6+WED5G@=NE9oh>#RGQICG8FAm&qs#;jH&J|#Jiuqe_uz=TB8|c5WB27L_M1 zt`q}FWqM!%G(ovJU(nCZ6cW}6<*W(bit?!f%2}xkB{q&U&rK# zL>C3B_okcq^Osvt^Z{8EeEl}NRy;x9i~`QCbQ_D$IhUzg$(7W^)g6>Zuv+!LPlX&^ z7g>TP^a+n_dsKBh1KG5z)2r14sCUskqP~@nj;Fssd;ZO!H}pH9d|7d{Fj4`H1g%4h z{SfM`A3#5SG-=h}`gF2G8t6KH7RkKSrkyKRfSG%Bi19ZmNM6Q}ewV(2Ugq`#ymzeU zY_gQ5JtM{FXmMfUz~sI50DA8=*?1O-S@L8V>uz>+T2+*@OGjj9ywK8-|7Ho z3PM5jmI-876Nwrgyacnf3agFR7lQ(?DL@{gu+oP$N!gyAdDlH=!ePt7uQ&e2XEua8&;GHDwr; zL3xg> zvonYTm(>U_rKiZ^gc$vFjw$VQOpD&Q*NBkUTM2SrJ8{A{%%IyvxoDZrC1C5@LACr9 zAXWcc)Y0Y@oa(M(5Y#F}IJ|j74VWK83PG#r{~Q%5l2N5HcRgTvOGZRciaz-JP=Zb! zx(m__m53Q3k62KvfV#hxvQV?UgLZEV=Y0Md0M>gup`>a(+Ip@g?fNbce0}E&%B;CW zL~b<_HWs7z?vNlFq)mt>;a4E$KQ(%m%6uYbxf3yW!(H?@Uy;r&L?G+YDk5E$kMwVw z&`ny}bbI(_`tDn`uRyM~w$JB zZ;;BJGNkPM?p9AIKeq7paYeJ;ZvcLLC)g})fM_)dTFGrU{mSw`qEcIyIA@xO`gRA< zukIKFCB6*NQzuO;N$jDstUB-22&oeVe=>pD2M*DE&w%JLGi2+*0`zP7I{Lp5X?m{o z2BOZVj?=vTDN-^MrjKiC0J%Xwlvyc4AKmc)MIJW?dx*10^PCLry;=@627d!O2Nr`% zszc~#?{x}9M}jDN1i2mep!t_#fpdWaA*1VuMi*?NcZ+I*;FIQ{L5gM0?O8&X-{?Vf zw-T{?hYymNQm3o$d_h`!=Ma}|{!!&Wz5#`IBE+2fT(q}nE+<@b6eZ|3qXT+rKsn$U zC;7+_*qyl=#NIWh?mRtgepmkqx^|97d5kOo_u^gw2Y(^LI7giLmfV4Uh3KQT6&5H{ zeGfYOOaV;BUgX%D=@LFm)rj@dbLf%r1L$DO9D4qC5qd$01hHItE*h&DMu$J@()Vm- zh;S$l{@b+$bgQd^*U=2Rcuy7W-I9(-`T=^HQ3i@;CZae0ZKCHEH>0sY148+uB9-jf1mWsmo-+u=s#cU7_#6Dyegqm1_)`xm z8<5S$&B!yCWsNChQdFA|;UT;PeOz41iL0~#*NKsmv#u8w^J_x;&Oy>yQ>jt`|t59sT0qWLOr(Lcp(|+*_iE!hcl+EFVwB=@b zI$_5*P*96Fn;nNazqh-ARlbcV=Sd>+335ewA`!^lGLBQ7DFA!;FF@PyYmV!E8M;ro zpYs|gb6zb7M^Ban8;-{y?fQqv>y<6?FbJo5+lJ5rQ6+j? zk31bOsYIN+t&W;|o6!B#DQY0=I5_1|K)p)8O!3bcagJWn1heWUD7W;V)TUVp)Ftge zpwS^}G2(s^nDk}>8L>ae>6!%RT& ztrfiGdT>6Yujp-n3>bjI#FdF$@Z`c$x@k!sl^>}NN~`KY);nYLLgOpa&D>5OYZECox> zs(}+5<{LNXW7Ij?+BC8G|0-RcFGN7sR|@(NT` zmwXpv`dA89BVBEPczpbT+<8KVR?5{(ErD$~EXJn*A;Hh`kfEt+mN zQiGSrIp^-@BadIxsEhv9!+ulv+Ap*O(VezwMLFcY82}6t)VKM(vbH4 zGQ?c*2JS*vz}J!raCL4DT1dS{JN_}KIG;v?!jWkE-Zenyr#TT~*#$}oUyel4c0%fy zIkBF76C=ZRq%IduTzoRjo>fmlknTG$YfcQuuHG1Y9@8NHvH8rq>!b-^OEcn~VkNk~ za}{Tvt~v2WR0?=}`j60aa3jnTOzFp)-hg)O0yJl`b*Zrvp&!m81nsj3Zbbt)zGDs% zyZ1VXb8;cJB%CC86-J;fU=dAn{i+XVFQLEZr-B_jbcob_ZJ^1|jB>YfAX<2Npct;F zZz#yqiiTQ5#^F4mKHHIQlAc0KwbM|iC!%gVl_Wgs6sz5%ctq>HQlM%tOq z;O_CqXjZ2Nab8xQP`(>P3oaU1koj7~Ay;i8_~H`qU5w3j9!u0nFCAxj4Ke+!^h4?vhBPTR_H45C~K&;VL zCnhq}z$qI`ATbIFMK4GsibaF$Gx9`1OAJuk^b4ev@Q8_5k_4OeMKrVQvlEBJiQQ}- z|Me0>`n>BqF!(l*7`hih=w*CEP(p>C)KR9VT6>Ugo*r$%e-9e>s#QmJWr6aop=j;{ zOJdf=PO#_bGGgt45URyqj2O5oMv2%wN6y3Bi8cB8pd`_Z&iY#qjEck2$FCj0cgaJr zD&Ra2vn>HyQ(US(m#y25Y7i%jq=*sy)$~ZgACz3yhE}R>rtO|t6EEq7D78d{9^dLg zDCybJZZrNMw=zImJ1wMtt4y$W!-VOB!K>)Qnh9uspgtYuEJ75Oout+@4uWL4&mg~s zN6T)jLiU|0oPCeOk#?aQ-D<;Ta?VKP^q#*@t#AGao_$#WL?8$4Zp;U>TG@Qi-P;L( zXAqZ`I?x+ev--qOPom+!BXs+qG)?-hr=z!W=@E_!efESBtqtc9{=wb=CTyY|^<)T} zoLN+gwl*Q8_80BkmqxLd6Nvy_HDb|?OJGudf-~2GOFU9_Ck}atqy0A5sWY2uQA=qf zdQkZntaamn-X<+{sCpC_?$)B)URn^(bt{0OT2ggI^l5@@2_!xU@21~dou&(KpC#s1 z9-!X^PJ^SP9g+f`I46Hy&& zXa*ebih|A+GQ^Pb9%}7|7t~bhUyA`pL!gpMaKdAIsTzl06m=qw+Ee(Biqn!tyRPUV zZBHZ4YAuAkewCusBfB_hi}Jx%?KtrJ;taywxa^+k;RJC1Wj)nl7lT%47E_AkNAnM{ zPbqi97BqD9I{NRSCL#7+3jE^<5d()ZsBvC6a!vF=`#e6Oc^kq|pkgh!nsoxiU2Op= zspr8g)4k?p`s-15a~nAOaTOR?v>!2TufhGhvwpuQ-bPj2v;+6ItO4{X6V4W;0`T^?XZ8BoZ_xCDD2|_` zFd$7WLAX;Pl~c18MMy71k{TzF=Jv;6cx4<1`VD}(8L`0dpdr<9q>?&+lZWPwM1k>- z1E_oWiNzn2tteJy4vMC_M^iklR4}nI7{$rvqkffnw13D<>i%wBDtE_qVB|3lX6jX2 z)NSpsuyBb-7X;1dU!x6&UloQ<&XGX{7Pct7^BgC<29c_aUoCrIfy0H*i>* z46Gy|@LVMW47+6MQ?GV|_-jiDeYr-eBhCuF66XV*Rh6Kt!IhJ9D;kL|*Qa`4%W+Px zin36Vb3y℞Y)!a{$k~3XBb9BfFA$z-ap)0EZude6t0}6tsZj>A9eVGN*cWRDo*? zj6v?mIOub6qE??g2>vRqMP9zGz+Ju+=sih7$yp8Hz={a4;>{(BANGcl=oUxG38iT1 z%%>L1Z1XrDxtoD@!6jj1osqTC5#g+@~?A$ml`=Y5nxdybpaXXdM z7K9EhiAL^ILYxS#xxiaZ7R=Whq)c><0hcdJkoU(2D7saIl8RRb2DLMRS8E~QWdEqj zeVs-fyE+H0K3L8%Ny+3GTcjeX-@_=sS(I*?^B;0;*bTPaV%fVwq9A^UFRDCv44C!^ zQ4?P-S$vxpL#^EW8azynv1m=v<-C8?LWRnRfEUFXATFyMI5$Xwk7^Ytde2diseXzR zKdc6X|Fl#e-c!VB_rzYITvLpZ8p{c;V#+%J|N}dZ+;GJ7P}_wX2|K zYE5Xz;R05RY~ZNM9!Jg{c}ObnF1oPY49s5g7nH{e5g(g_k=N!0)g<-F`L@vZ$k;NM)eD(s&Sbewx$eDMUyb$5r3fAGI(%&!^nO3PEsCG&ob+&!Ov* z!CF~8g2)13aKzu@dGAqb;cJZct(2o;&4kcZH3`Z#_7m{Usskqja*@Mf8}KA?3$^s? zLGZJM9!)@{Pid5Gm5fXy8W0XTPlZ@5q<_v{fRr_2 z!1z{kRDGZi^~A2^*k1j@F_Q5I^dq+4s#ip%W4n?5U^v*HY=r7&h@#TFsbE|&nRD~` zcd)2f9bEKQrA=b1M>rnIY3UpNX0FZ0`4+&?qf-r3WJ}&7%Np@18X!TY!9$QEq z{}hQFg5oG0gC_JRl4CLI^qzB2I3NAJIe}(|ou$GWFMvaY7TPg)04X1krPbnAqs|Zh z$X4+(*t+L~g)uoyy&7||urie-TvDolL469!T_{#A&s zc?k)~NPH=(O_)zT4UqtwDzd=hvO!AaM;;3JDMwTVJOHiouh5r2z36H^4gSZ_nTFNa z#Bo^CqLh>hm1q-@Qq-9_&m3BWNFkKaOZF%!OQC(yN=1dVXqS}MGjnD_k`_xLWi3l0 zl6~iWzIB}sU3ISKdS>SLpWl6JR<(l-1w;78j$Rz|r&*k^!5_R$YrEikQ^NIq|WCx|!72ldNiL5h48 zIIdACu5s@X?@d&LWcF52r@tB)l_Z0QHx=L}<0-Iaj3r!kLYd^Y$w8NPBUrinD_CPP zg8bVsf?RPKi&xm3!kZG7s6|sf0D|ZUWUwy&+H&xcNN>+%6 zW<=q$Khr_1?@h4aZyzwZZV2xKE4VFs60x~wO2+-7*yPe7d}Gyvx}ne$Smu6)wa?j5 zyd&HV2bgL?EWH~X89yYb&~O5i?naB(XCA>LwWi}Y7sISGw^o3~pMT-k_pS=2tTl!9 zsfKVuV>p(c9FFHpc>f^p8~;9zCmw#b!M~0sg4zQb;GD#aEGf|g#pC(fLcJmAWsAXx zuoK{(dp5A1E(Sgc#_&?xUr_qxnP5FoAfB(Y@yF0O-2d%~_+Oh2u~Rc364$W=J+uIB z5sUEz4Ly?T)`1gL$H6TZ6$Q_~)B?A}ARx@m1^PDE!PV5eK$QIwWEh9zMK&XO*YR+2 zZTJcC)9!vSPZ|k|)9dg-uM6U*=1pKniXo~0Do0FAf8m7+u|V<5DgO143}&S70{<#| zYX=?8$$Fp7q#=F^any^zpUz(aDV9o5@!JIGIDZ?Enj#|@^Fo10y`eYf92>6UW0%Sd&;BUh+zc*cSR(CA?fo&3hbn|n&}Suz)2 z4RFJ=wEy7(wE?k3xg5B+<_(TGcM)HjDKDtj)g1zp9^fbwxPA$@VZ8+J~qX>nr6JP-M#X2|H8MN1>3udljf_n+$Fc*}CC;4ha zv3v`5>og)F*$+5M>a9TG>j&{Ozbt{>uA{)HM}aIf$p(vk$BUm^M}VNA-vT-LZ`dcx z0k;|75hrg7z?Bm}3Zn9@z?pP8U>m#|{QGB5Qfw8-3sV*H3|^~SqSOV1^2z*ro`T{L zBY?t~;iNxo8cF>rN7M%!>UKEf;h-m>fLmAv+U<(*nb%>U<(@og3_aXq*= zLj@kP&jsF|53G~BElI#;E4*vQeX+~XZv6YBHZXg753HA;0_HA^2WOgR3m)_SnHyGv zfAI)vbPchZqLE4fzg7*5dhZc4};ws=8{yS zqk?aOBZ4W)iGps`3*t-7xnjrv6v?H+i+I}faqyeXZ~Wu(J7&*XHjOZ$9uvkoT0|1ASb;`?I9M=;7Ppj27O1$@~9;3Qk}fV_jfB7K1qF zKjOw!b8BmI5Zqxo8G1&l6HkL-*5ABG!{Pn1FvYw`oU}S$JoBSAoF&r_bRFeL<-!IK za;ymV@!k%v!gBCHX*ifQ;~-vExgBr4&;jPY>ZtoJSc2uYEyCgc2|%Rv#Jba9CivWt z2>h033EmV6YWu&=6?aeiUYC`WDjwJX!NJKJ#Fb6=1?E5GAWrymH!R$-VfF2G%~3IeAl8~l83K5+W78r}KtSs|)0zUVJ%@_P@v9Nm-1K zmX?ddBdJw*W|Cms)8Eyy83s7O?|R)K#TC%uyfs|BUW<^oAvkTxM0kFa4z!u9MC=kA z@YS&C@b~o!@Wy^JxpT<@=kM?W4R`l|2dawXTJuG)Zk{&GdH4*3H7gKdh?MyJriXY` z!7W_+PX(qp-4eKygE*pQG_DD#22XF*;039&q}8MeH`n9=$EOAOcJ_Vn=<5}5zt@5^ zYflDuPBr5)>vCMR?KM#7-3a`Cg<(lvB=)tv0=iyRfcq2dF}b-D&!76G*7oZb!H+_T zU?xit%(}f4Pq*y>)6{r}$cKkGeq~zqe|3S@i{r-QDVzU_KV_^C)1Q@KFljLmwdw)K z+irqw!}rzsYhMRT1k%vs@SxZt2o&^;CNU>J zVw;s?!6fI!ptQzV9KYxR&a+GqH})LB){322b)+G1f3^po`v`D+{$sGkYc$+oJ{~8G zc`V*({!#E)XRElhbiCkZN0s22fC1wT)fFGAR!NJeGgK38% z>HPpe^p8;Sn{)YM)4=IqUGyY8d9Mkc598{bT?T-{7BgIv^dB?|pAT2P3)mfS)^$H#s&N*(v|q)l;v%sg+Xjk{cZyXGuEa)K zHiFq>d~xfhYdE8y;fo`(z;O-4#AgCg<3(d4*Bfd zh{qi60MKl-xZ;Thnfd!Yer<6ZY&d*h5PD#q;Cs`2a3E+D$c%~*C%wtU*YiWMXnhAR z>^uX;q(%#FN<9;ZUd4fXjn6?;@LXUJtBc=lTLkt`E32ES`UWi8M8VTQ z84kDDjlVB_3W7eTh^-?Pi1Oz!@a_9bqO^UY_}L~a=rr$^!11&l{4}`>cz8@B22RSP zQSS1xC!w0qB*cM z=P7>GI|59$)&V`Uck{WY7+;c=f$kfVz{QRPJYaYmxF4QKqOM`w&gU%)yH0~ytpo@C zjKE8mhk{R+UIPhM15eT&>q-dLy*|3c8RT$;G3>f#sgOyN4dQF(ELHrYDk1fVnQ$&KmiWZQNMW?s8pSC}J9 zV*8D-RWko~-K)W>Dj&d_YZ|p6q#hTqs{+@4Par}5U-8y+TR_RK_W-A6;Q8jW0oSPl zGh*ina-BxO;Y&NQbk0=r_CX(b_azTsPDS4xMlFik-6A% z*Lff{Di#xipIF_om#b8XLh8D6xQ5S>Xu?blI$I`;EzaT@+~J8RK;aSXoX{ira^ohq zJzR~wSw0G7*G3>egDj5DACBhxRwU$Qke23QK>$#B%SCFw)1@qo+LESc1(y;k^*s9wu zY_z{3dw!sa)9g~1HeV(5nihr`)#jZijb+Fz9L?>YJ5r-5qWMrwXAl&v%I zW7>mi{0x2#`h3q8=`tf^_qvo@`{WC~q;(330)p9lwfX3;V+GHm{mKSt*az7F#gXW2E z(7#%q6&|g^;!U5ig4;*XlPL?)HLt7O=hjGete{z9KIsg1fWM1Bg-UZTkq+y#>SJa`htbL0P&()9SNgu~0=3_lgk1K!u|GK*QO?s9Y*DbN z{@cPcLjSr;^x2a>NpbfEWTJHfmC5ALncibq#v)_%aGVCZaWjH-uM1^C24m3((Jqs21W%&~tSOEH|pM%ZXEx$!z&iAM=q z-L?qjeptqxp~JaG-30XC*D^LV%e?;n&M@Cv zfhE(v5G`zb;J^-COlLJ;O1QaeXJNUOd%5XTP5UWfkb%Fy_Idr@*-9L;|=p1v%YkDm7JKwq0j)t8t( z;ljILvXQ&zvYDkoVs)#6RzF&d#BD|_EZ3BNKc>gMeAbaKozckb zvo>4vlc6#Fe%$a9OSX24J2y4qqHxKMyOOjF6Z$V_Gm|a!Vc$Ge>8F*?*ob8!wxlZ& z-JPx~d|lSS5;tYB6}KbN*{%qBzp_XebY&!WTU8l3*-DAB4%}kTH^i}2x*nPS(qXxL zUpqebGJTojN%OwRa}y@}BImgEoPOs?mU(6h8&O=zO)FDin>Tx)q}*6GKRE$~KjN7x zZ3djm1`8JE#XBTEs<4ZXWYJSI-T^_=sn58p!h5UIk^l2}q?_v})JwBQjXJT&Pwpuz z-7U=?o|9$=CRZSMy_{t?WuT*714#DzX|8Kh47!mdM(Pth=sTSVVe#kdLa(2rB<1}g z?)JuDuJ!jYk*!uTJN(TK9Z?(33U*nbap(jx`+1CuFqRb+WRy|0zXV;C9>ud;8TEhQ z#%7)MVdWu*cm~{isvzhMB=bdmbbExI($>`7Xa&}(vESs3-%A)O?xki&=%whCzI<9jedPygsC%Q>= z^`K1s&?`?-KUZf#{Zzj;U&dYxxxcV^NvbJN&> z%z3JnVvl;1l(-L3C5-bp$@dk#*yr7O^zpS2^run>{fgJ*8Ylln|Ai>A*Bfus`R&J0 z_G%sUPxy}0^LCbOLICAlQKVgM?o6fZBaN1mMbmy{pj8j`8P(gxj!Bszxlnr{@JT`r zCIOsT;}kCJyFAZptYxcv6F8^e4qV%cd0c~)95?fBKX*IzIcI6vjYxGfbu4LQIWMKr zQ28!awaiGkIWCt@9Xd(X?y94yUz3qtEYC`8N#}HzA7b(P>TE;UQhIsSdD_FB;l4R1 z(usFMx%9?Cu70*JTXiFn!qI=Z&Hw446YJE;C;J%I>2eTVJ3RrJA5cI&r*r7FKR+ca zCKhoyM=uIjJ4T@k9-hqGzlaTM^5zPOhGc2{J)C>gH?-HQhI2Zn#LgU%LUzJ4EW|&9O&!Q);ZIW0tI{*v#r=+) zR**YQxHbp*q)nH!%v#5i4Q+WwBF_z1Imcb|)1{o(1(Xm;+2m`L+~Tn*RHU;6_5I?z z>Q~RxQ?IgwPN8jRRZ9scI=YRvURI~jTa`r~`pK;eu%L!6@pMn=W%OqC24w&458LKw z%(Rs>n8~LPboYqqOy$%#&aIN?~&U2pRA^xuGwC?A`XQ=*Ig`Tw$5HaEiATGDsQC#x$R#$$xd3f1M+(4hrXc zT0(lgFNeBcP!Nrsc$w=Nx`Cu^c`whFSExF`2d!Crg}W6plFIy26J1nMsDJi#HZvSj zM2g_!=+UujtrW@==vhBCIRdIbZVE1Y6%aQL~!f2`b__SM+dxm^xPW>KC z=)OhY<$pl;4$ooL*idxYZ!dg4TbWgttYgm#jw1736BImLkvq2kB&z!8!=kQkmHZgn zf#w+*urA?bPCR0d@S$okY8)0O`L99~HA)9@QmQrs#X(Rjj zH4c51*JM@m-_q#in~|0We~w!FP2%&}M0oVyLhhy*(;bihK(mOktbW{Ddi(HW(&;k; z{f#vw=T_*VpigQdEuNDs{M3r1a;#8P@D;kGnRf@MeqgcZdT2<9KD+I5mT}uZaPlgf zkwL8qny&PoPJB3**&W=1{>KOgywg-=33V}SE>ryl`y8DG25TqjMRb6z>jZjq3 zYD9l@grHHE^hAGF*s#n`fyl;lH8(VCFTJ3(h5d6dr~PIo=)kk1oYluN)?vNL1& zg2ffy<#C%hoWh**bsgbYJw#trXt4H7>3Y>i%Y`RBqEUHR6f=F?j*3hy+19^`s53iXOFNY)>bXVYdcV_%ler|w{o_3=*Q!wQVuOJzv= ztuomPXgGZm<|w*@WTAHlpeCZJ{cIsCe~5d}P(D|&Ea2Re~98CHC7McYF4(e}hY zba&5T>c2+5e)yXI*xad7^s>fTwmxGjdhb1^8UFN4)<84Ek+Cb8~UElB+Q5Sx3jkEM?u!@e(Wp>F4H2!(Tck(|yI6hWItlq%--DfbcczpfC>R%GYCU5i~X+R5< z?Aa!VG;Tz)Guu@biFOqya{Z%fSt0gCii;TcRev&fe9>PzuQeUn=!7#q9p(8L3Y@O< zXLL#uz$}W(P_f>e`XRWDKFr8tw%fD0ifKO(x|fL5#xgWTWi#EI`ipAjC7_X2n>nLl z2bhCHgbe7H1or9c$OxdNREVeRBh^`D>r^XW@YklyVa;Zx6^ugmuQ%@5;nLM3I4X9D4 z^$SpOmn-Ut&=nEo_cZBq8>hSOKgqP^dy(eAV{YB((cCkoZm#WG6g&N{5_!$2VO-N} zHldw&*P&KsKNQRfk4DkgSs4;M(w)mp3}%t#?(}!^U37u34JM1_xZ>MY%x%|9+Ryum zPi}KVX*#DV*bu|sv>oIAlaZ~b7yMb?5lx=wGn;E(b&jgo_i$yEBUr`i1T@eb$o7dk zgi4tu=+2dRPUnjp4O}>m)7|os12)o>$xLHOx1!mmEO*p3tB*QNeaac8Tti1rVs^lJ zfcC!|Uau~wWI^V?x$cKaY}wMWXo6k>t3Kt-Zsp8HX8$Q7Tq(z19=^l^7VKu<(w5Nt zMq}o^U#1?77{kJUK0{kJF6D+TujKp_$D#7RXm)7j6gt+nlir3V*ywaGv;JAdc5N!= zLgZ}Ns-y|1$M6R-77ozG^S+{E`!w0dV8D(kOOS{C8}2aK7ECpO}T&zQK!gF zK1A|Iv!3ZF9%n&%pCk?0^&E7WimG~|$rz&;&RCV&B-X91NOyM?GF$E?yzyrhdc|Mew=tZEeyx&Xw`+!R29>tx<}7z6UblzI%+`QQ zFV&*g*K(L<%y;U|=YRWVq|u_P0<>|eoJjPW_ipdgKp6!dlG!&dGUJL)sz3Y%^0bX% zTRZ}}x2?NSkybl8BhjpX$GaV_LOagPEQxBk7BbbR1Dsw=2A#G?3Qp@bW3EQK5ZTqo z^~<=TnN!CimGytAuVMrbPMgR56V!4SO(WQpGkro+n^VFbl~-KQ8zHN>u^;`)*P#-F zMQn@jLEfQM&H81WxC1@0oWU4nc7O9Jc5u;qiQe!o7I{;Q#@a4J4|QG(%TG)}*WM4( z!Sn;1>AEUbw`B{m2~*})Pl!Y3`FVTdv*~PQQ7uwD?Tp@6263$~y3yG!mZ-@mlP;`1 zjeaj};h)z`w*IO`;ry_dZ zjalsUV#312x9eNH6%=OkohhD)h;)IoIRlrjxD zG>g-EyPgey!MMGTpK|5>R>)`R1higljl}iQa9Wry!_}UX6Om#5^uAXSEf)qd<7XPI z#o;Bptr^DoMyI1-olY8PQX*au$MYCnXV?2ZDQ3^aJJBt83N2c(4f)S>=R(^ap~deO zviTD%P+?g=E!pt~)w32dJ2FATCp|{ho?^ zZl!adrF10IZA@tRd@odTXAVdFb9)s^w#-O?q+T}=jV1B8QYHJIbk)DYIVlhzIsU)bxf!K z$#-+J^5W6QhaB6Q#%C$w3+&<>jy+E3JaU9W#u|T_U>mXD9E#5y)Eu&AD_$3JmSqh{asJquXI5z=U=1MlXkOfYNhPnr8%fM z@j4soHejL+%1B$Kirc4t1GTslnBvetixpFe+EW!YXa7W=5}sM z^Aj|q;kj@tA1WEyItU4N9Z@El$D@r>!_Rq{4Nr3=#Gk-DIjKVvq;#)3h>@ut;-) zw5$jk>H2{Faqvg{r5w&wR+V|)-hu8N+lNAQr=m}z<|2A}3OZeJn%mHp$z_}`;n|IT zDC0ysdSv*5Yfb0+V6+{LMH zl4jACi8Sr*ajw=H(?Z=h# z)@Tte9Eap??LyZ}Bav=KCL0}y@a8 zBFd%fAJobs(aBtT^Kcj!I^Ldn*gCR-eOWB!t)O0du`*hiw~jUS@ql5id1!gVLbPu4 z$oeB%vZCz1KpWf1?X*ss%l{9fv=g|R2Z5~I@j0ul+KTiaY(?jIHmQUE zZ*KW7gl4WWWA4L`Q*rNUreA4}23zh*s?6VVn~FE{{$CxYjI`>VN1kBH%6aVQsHf=a zy<8M1<;SW-A;@@-ESnUrjebE*Vm-j2>{JCbv#d$@y1D}0-_5c4zy5J5^K97c(nW|J z*o*G0Es?k@DsqeNK1A18E?R$WEL);Hi*c%osC>z58lP#(9SpvVBEM~8>V_YXD3`xp3ho1@XrW1!lJLJ2!;F(GpfnZ=dMkk`0y6p)^bMFgKYy zsoKHSwlyHB<}UViRuN13{)7e3pMi9Tj>@7VBKTn{c$c>SY3pcE{kCm$LgTZml2|ur)RRvL_0RjaYTI>Or~ox z`na(fmcrW;D$qi8gL=10D%tTV3x0{;iM&4NbNr@*VVNt`Q5T?5@#?H!EsjQh--`Aw z)McicvFO*`bF|ZC3_9645{Vy0vmq0``WEkTNaX6JeZ` zx(0pZW69d}P1&!ld5oU_EbKhsg@XP3(Bk)xk=23=c-b2zQSQ}gROqvr1zU_pCexn_ z zWYNS0U#OSf8dPwyLKyt062;AQM(qzHxO<-0C8J|x*#05@{Ic2=WqbZ)KkiquZEuxW zo8>}IS1pW%2E3%}oA0?DA1HW%SB_m|N@wEO-)ct`?8VW)UJ3Azl8ElP z?8Yh+6i_vmN15)O%)ZrtF54l^YQ}Fv%hnj9uA>(0V3q-uE4E|towFndeGE~i$_38h zYZZ5Se3bBOLKM|}dzlTTX`+kA^wI9o#$08e5$XyzMH+c#+(PRDcBcFYy<=ZSn~mRc zFN%y=M70?Gw;HhPGw!47w!K`HX+FzzT!FmrETcI!vsvuf5LAK>qN~eWX`6{o{n#J; z_u%!3o71=n9j1<)&cUms`k4-Y4j|}N$ zD4RD@`0tw?=O5%Fu`J4xyuFc)wk9CN4Hj{&_M_`JXn&!R0Ws)F(oUo`bcx=|{LB9I zE7ouB{!0sXwQ%Q74rf<=HgMr?1& zQIy1z=Ng{>;f>6>AKa2%SLlWHdfeBrU92VXtmMa5cZpw{9=Z+?eK{i14cVBF49_B=qJ!3NlWzJ&gPxT4fyyZB$`O1gAt1@KA9(_O$8=~po?e?ffb~akXuP2w9 zH={%T<9G*UA@|YZ21<~hgKk9Gio8o7a3_PbXi@!Z&XT{E@iW;*QfPRBYX##(EuULZ z(Vt-`;9CrrvmlW>@j4WlJ)F;~b(eBRzTc70fIC}I@5WXgdV}tss^HHQiuDUz<2f^V zZ{$097rPzrOSe@YL52|vxg85NSz2}jEB76aI>9lP4Vci@s}3Fb9EaYy8c^r8i|Ep{ zRwjGt3O8Q$K9V~V!#)`8rFlQ@a;*ndQE~bbTK*=LIlZ)I6RW4AJG#E;(lTexQZ^ld zjKic`U#-4o^mZ20Hb<0n5U}URtkGN-HB??{$UeJX@{ z%yPFh?~pmmcEmae-&xo)p$?{xc8ov~b|dQ>#p~c4H8`oQ{rKHqtclIktbR zDO%*=!sSeIWBX5+v3)yUPIpx8(qrpNPP5F;W89MKnk>U=IiJ0R(w{Xy*uX$CYWI6b zkF<|RdwX0sS6gM4I;EW5YjNf0G@sDST>{Q7rNc3qXDPN4TlBrKtUZp6KJF z6Ug~e0sZ>;5;ag)MV`*D(ZDAwbjtTLyLH-!-Y6Z#IxmHAEeqBl6{RwikB-yf?>k{c zwlVKEE@gk57qH?@;}PAURKEc0v0S`|={=hr9-lM=-(Dwf!pDDZs$uUwZ*HJyIBkS;PE%fN&JK}Li!>y@`> z?K2gS;ovDr{B29>e$_!zx4{$DPH988y8kloJ75 zxxktgXl}m(TWeb<+?eWzj3-ypg*i&B$z?saW7IcF=J0tMe#tGobe&zcT)=Xyjgj1* ztL))Ng}c$>EbP8oA;~#6#O}V`ixiAz(+fJ6QG47G^y2#u?zv7i@5gH3 zMoTHz$E}!0SMYP&-4jlug^iI+f7&7bF424>yu6vxiXF5`tBY<)x0QsM?bdQ$xsO0SU?%SoBkH#$bMiQ|%)p88s}E+iZI z3q82oc8&DC*F(0rPmG3rD`Q>0#%SwVeoYh&P>I@k`fFP@8@gT0?G31+9(?Y0)=nJ4E|lE_h+ooC+ilS{IiRt zaMEo0q$)J=fkr+4;7z+9*PyW{*JM~;v7UU9D*zMDBtx_+o-90Y6r$x$By(>xnYbVVit4tLb$5f{GEa3< z^-u#R+9U^OJth=1(nYzM=rbE`N^s8~J%Rw;t%UYtC@dU&O*N_TR4LD68N5Jpnf_?9- zV6+t=DJk2ab%`-K6T`FMY&OE3ranY4rW@?34r*V?;n9%75vF+?^LnVu?s-vsu3NZ!VCwE5N6c6M!_x z1u@1a@vIN01Szt7&wOY$DP8Vqy?)ww9JbK{I=7aA7*l0pzx^c`_>h6`HtfQOUUqzyvN z?T{fhA}zSUasZE&H-hU{To#{yQvymK%oY#TAQEmBK_1*c2(!!Pl1&r6pxKjSc$}vt zxH?r0i&Law{C}CSC*l|}aF2m6Po@%68BMZZZZ+P0E(?3@fpAhs208j|4%~7+ob)y% z!O22pn7?NVnef>FzJ9j@^Fj^ikiVH2v>bq0XBLwY9V>9ik~**?p%1SL7)Jh^YXeOs z+sK}&Zcy)FH%OdRK>X@X!`(KgiTB4WXx-T(a17TL@AzE?;#^IMc)UG4w$GLvT0RQ) zAB)3diZ{VZseMF4{~&aXRUw%w(QvWrVG_0Z0NiolAc;BbN#aBSP~~7W@pByl9XAun z*vunv(ZUE~u`2}%jw^#5@51psbvYQk)e{DQVZ^b?5VkH?B_D2PgTaD6FwSo_S+!FI zw#CeZagv{SdBiw4xN{Ux`G7q= z+zRN_{>3l6eZ^kN;o#KLTeyGF5Jud{6dV=;SoOx9EV~&D-P0b6qdF|e-PaHXy+g!h z>Tz-67%5^7EMR-)Oo9`l!MTZfVxLL!r0$YDj;h-WH;vUHK`(6}`tTQvcKhR{o4rZj zYfsqxbqAr>ond3ZGV(fqBi#SVlNfGQhhygzlF;H@m|*BjWS(b3wW+OEt#9X&S3v^s z!yr#ER<;-4=}?9fH@D$&^Zl@PpAFp9uz_5>xe~r}%ody~^2Yl$K7#Gq{Q|MtIyh~5 zEy&3p!YV8J!Q|X97_o7QV0z6T@v`UAFgw%)-0<8l9`jgVe601NV1fL6IJ{Sxd{deM zl@zDp5vx4m;@N$8?*(|XT?|Nc82Rv)>5o38DFXRqxdON!m#l=sP`Tr~k6 zvWz2-Kg2=paMPOeOV;z5;1SKkQ~~4p)AhM9Q@t;jFxL?C{tO zE~j>6sf`(Q>YhRT-G1RodJAE6-$J5oXGo?`oCW)yE+N?kOW=h=E@bvu43-_VB;?^d z*xD}*TP`HQE73>Dq zCx*j~kK>5Eem{8AJcPq7{{7|<81-c>K9-U!Fj10-GqM(ovknCSUn2pW(0+!5%S6BpWoe|sF$q?^)h0EW zrtn#!4q;YGq{%%I3%dRRKkw1x`mHojrZAF>SUe1VDIZ2|P1q0b|Jg!<=yLeA!kv8f z+YVEmb`t01Q{d#!qx+z;`5pWM-l&tc)K)q$^UOQ${q2 z^*RdAm;&N?Dgs7m#DfaLPnj*BEAnN`LqeFzSDqLdDi2* z!Zjd$stLUL)0TWbX9Pjie3Cyg4#t{AlZM7f7&Wkw^xh7KhldW3Pn!-vcg;b8QP6X6 zcI7PG_4grA+9<}}FISP)h}Cdq`zkVQQ~c*5BVJ4y03Z5Z+2cd3J=~G zCJ;3I!B)QNP}j(p$iDZSQ%L z)45EXi&B7Dx-a$+*#T3&SNtrO0qeP6F`bl-ZCthxm6?I?-0MK{f|G}b6GjQ#K76lj zc$^^SJSJGb)eM3DPWwo>=3$7{mk_rtab$vBI2^MtjEs1a0oUBQ0)BgpC+GGLgS&pq zL;1)0@bS!cymE#c{7=q=sQ9Xr7)5h9-Nlx4#y!AWa_4}%Yxd%uyc{pfc_R58wvDV5 zhJ#lxCf8ZkheFN0GoYh_EqVXvH1Hj>jr85HA`cuEL#?k?Bw1}b{1o&S*AA5erDr@_ zFiZx%O`ivsT%1T=>~ev9ic#eAu_kck#tWQPs0Fp>*MfzMO|fk72Qdt^0W~#Q*!h}( zY^bz@ZyHw;wb|1^Vc-GiZ0k?L_8x+pvk#E8%z1EGml2tDD+j1Qk|);}rGa_nUU>WX z{UC<_ygc(g_+Ib}E9_H-5AEJztyjwhDqd@Wj7y#%%k2+NSr%-yg4nJaRxeu-@ zTLK2(RpYSC5YV@#80?#tKsq%d;V{!UqWs|~%+zhFy?nM@>}^^FbUIoD7FH+Vy@|=> zz|&-yvhxJF6}6SXo&NC4p-n{fjWb*lehSvdC6KB2W8sxK$4Myoit*i%uqFE?4)X2= zuFlcq;I{}kDlC$u>^lZ~_l&Oln0`lKcd8D1hWr-y9Pa_WcTWO5?*lHZvL^rBV@SLC zVc1i+k2Lg#z$`6Uyxhkc<`q_AeQPyRc_ox2wnsyQb9;zr`7clzorX0-N5Mzi9$u%y#**8JMil>5crcxv{ zP6SKF1mij5j^Nub_LHMK4?x+CA!KIXJD!2DUC^{%1vst03ThOk$xc-vaVy&hyF`fC ztdHXPbVVSj^C>o&|4STnTp2p3Cc*n$JW(}>fmUNyx-cJX0nUMtZVJ2MUXWE>%J198wqlti{abBCksOyKle_P9xvfZEzH zl618Ki*5`8PlNr~$wmA>MfV+7^Y_OAyh5q8mzE|<8SU=p+;eUkRHUV}Ns9^%qmVSU z7b}5dN*%mrvMG5C~RSqyTOToG=yTF!D8MytH6|gHGpnak=fD3j%U#&Mr zanV97@_GgExT%jBp?t9McpeTE&ta{mM0_Bw8jpXh05X5dv8Hkk@UL#5Rn{FopIdQ` z*0>-<{d7$RyN_k!iiJ6#ZP_LqrgoG*@qP*Qx4D{MZKH{u=JG%s`@if`HU-Y3V!&B; zF8z2&gSudyNAIu|0+;k&&?sIFC|B!Zg-`>^$7KQjFrf+_*J|L(uFI57X&{|DwvN+m zXhBcYkKkt?%jAcdGM%}9H&9!2M>+BeZ|Q}Wx2gFiMV-G)>^OYWC{Pu$6(8E*1qy!Z za;ELxgfpmc;8(OB$FCP~yo4O+^NOr_To^{{&Yr_MpK_q;{2Y8-Xg1L5J5BFpwqd<} zi6G&%4-T$f20nf8qwuK?s#{tTcvoi8n}p9%BVBE@fLlXt+M|QT3`D@CT~FvPEmNT0 zeu9&H+yQq_Xo0u17q*QG1M<-mR67-pg>MQ|8f7auBUhcli>nH_ewH!#MTg;E#@Qg< zEd_gR+74DFrQr2-kbi%J3UzQ>2)(T>pTAC33z+$+;Od$0sU-od+sm`Z>jKbgu=ZYl@F$;IO!$uzL+P&&Rc^qltiX$B-8L0mH<5Dd7m z_NCb}%3}L7DTwHv3Blx<}9hB_`a?fU*6}3_tuAi@xVx&Zsq`Pw`hRmU#uhf%M)05>*Ean5F9j96`a^I z2X7~7z+WpG%#qlOrEAl`_DlEaTUD-Lm-{C=XciZI0g-s>{cGwczn4~X|41DP8mBpj zedv-5anL@0f>ZYCq}7V>HpSEnHE9{}e1-}#+I>+KFM}Be-9+21tYDc!=kX(E4(`P3( zc^L#&FE+>Vp8y#3t>etU`Hny5vL3bc*`dywFDW?UP73hQO~TriNx&pu2h2S%6Tgzs z0Q8KT^v`dlv_aZ1)$MkimcQ_Xom1k$ju{N1P#eB&vtEX0=hj=Csj& zC02ror-69D!wzg&)KBkS=#L}agFsu9KVH@l1#H*XQyUte^D{lQK=Jny`qyr8I-$#% z-V-p&`IvWrA8%%ko8K)3`h&)J_EmZCNiUuHaa4%zFLcMMVsezHhcf7U!o!N1^MKYs z1+9`w;jw?wlxj>CwRH^uW&0Ye^J}uJgdM8DjXaPz{~)XdCb;O2sF}Vu0JyIk;H-FMW7-9regu z6F+aU=sfkuo3-Prs0Q1Ge4GAb^pdpcAoa5})=-@ZJl!R+ZHqNt>uCuTDtXu}xQ>HA z{p2WqE~TB6bE$d5D%8HJXB>OKc>1n~E0y|Nj$e7q7R$WL0L3;g_+_*=jsIlh4D$%k zny?Me{xnGWJ}#uRlr*s4CUx*YTndy|&I75d-0_?J>p}9tAX?=XrdOC9$%d0=`8+SFhBey?c{Zvk`{VE@7*^-cT6==qCZk;i&O=0|NKGV zup}RM4;O;s$O2sZ{j=4g4kIdN?MKS+&|SLgo(avbc}96ZXrjS9H;|O+c; zzRUVN$yvIC2WG{qR0OmI~9eTgR!m zoBed#SrXP8xEc+2ji|8%J_+O z3+1WYM%%Jueu4B9ZM9-HDCmx-dzb!4cg{7%@6MZoFP`DJtaBZ;XGJ;KIB|fTR}O-r z{3CehOMR{Z0KIrN71a?XT`Ctpy&W@-G?06og{u)zss zkjsBcS1A=&iOfPtz!5e-#&FZcyxM_KVB`4HKTt~V~dwyqk&W! zoY4XUJ%0F>=o&CqwVTGp`LwixB^B|0J}&yb6|}!K1+i%{xPF}~=>KGo13kad5{}MP zg`)~~bePN0?bZjP_KrB}nG2}e-9g33HiAWWs&L_-O0Z0~8GjbG0YA3ipwFgxb>`Ts z;f*(r<6jp_fr-g6Y#G!D-mR(QuRAcrxpC(#eW}xwnod{J8|AgZntCN1H7Y`jbQpot zAEdD8pgjOn3G{og3Cx1*I!a?JSo+%;BkKa--j<0ozGMO0_6&SQY%SRJA_nh{UJke; z&ba+e31z)~KWD`xz#SH9_>i^{M6d-4!NaVX7}(b?q-z&cx6L!Ya6^b|0AK zw-tN)?*LEAV)1!O4*%RJ&A#t5@m&XRDv_;~#uZEB^?F8N8S5bJ%AbZWE*__DX0Wd9 zOL^+NZ?^T{ZLGuQ*2dx6rgEN7%b<#@*xx(4g?ir{3`{)Z!IJu|_?u50NcZIPRgXZh zF3kwvz{^1AR*EW1&!CA|EOkw;guY>Fh=ZFDsHg;3Apn4{4IVf!#uLcsx#RWpd>}6} zL8)ujP;<|n<$Eg2^Isfqqa&ni0NW*onG+?zVJeruMm?QQLy44g@O-@I(Pw&fn-Lvr z@{wP^wuXB0v5#6Xu!1h=R-x=fis{#`r#YR?wP1JCemog!1LmB{#@qg^24&VBSZ&G+ z)DNu0X6E`}?O+^T-8=)VYSO{h)vNJ;kuD%4vXpia{NX63v+wO-0eIk*g#`oCz_l%5 zpr*D0*PDfat7>ud#4I!LhBF6m`YFTzr6-K5RrNrMPAA>U+dK8b1E&v~PQuw|1 z8tU`chqV1w5wL1J46mxn1Dh%~;AO?xK&D*^8`hM9v5^WKpk5BLlXLNc1&8Ram1n5p z8IR~X5p^(c+yclKO5owbm(&u*2(OuUnlg}MeMXllYJH?WE;(3Bo$iUJnbv>Q%@S|E zkJ4+ZcKj$^WT*_>eCE(G$BOC72}6*@onk^XmT>`=Cuh*cYFzLxHUfLxiR7QXo6f1q%cnT8>UienG_W!|89#j# z0V3^!@%_`=@nZXMV7EL0pC0l7b<^u8Uz2?5`cxig%drwVFj^h#T4aeo{dEDh>HGM0 zt%q@SL^){WmEpRGY9P#&!M7f$0t+P-{Kj)S1|S-1N~HjGxxM(pFGVoETnI!uUZR~& ziv!P3Gx71iv+%{SDe6L^ICklNPhqzupg#5o{bGw3aIP=p)P2bVo0k{hLz-E@>BSsu z&^3?0Qn!G!P;)MI^~o#F$)lU_ycde#vSB=4@uU>oyR!l)`5mF{J@{0^avqk^yG?)I zxEMUuKzK`C5};@Y;2|?eXKFp8S|?MnhJH6|iD{;4X_zUrvUpm>%f2dX1Gfx z8dy|r!BypV`4UGc@NRe+UX{TEra!~6Z8BTO&3;D5S2}>*Q7-sFPA-^vHXXmQN(821 zo%F?0&-3*&vZ%&2N%Vd&9gGCc28oKsIQNkUc=&Uep5v_wLT;L1kYfdWOfJ#>z2f+V z`U@ zAJ^spo^%lix}J~sCl!E|vkqXlflUBh-Go2y-2hBBx6$5Tt%1=KNvteq3r4Htae|FG zC|%)<3xD_m_3tKlinkg}`g!2~fN#{f^&(=1 z1A(bk6Mz1}<+!w#-A}Fd!wO%<>ANC2c*6k)@QR~@mjpF~-SoW0@;JOH3&bAUi<|Dpfo$!qctdIfJ(^KZDJEUB zR(4!P56%n%!R-3+?5Y4DzGE$Rc;Lket9ZjtGF0q5o$ZS63PkXO#JQj}PzK-8n+c9D z zJIMcTQcQKdIY7l;KEsbKJIdc&T}XW?T25v8G_hy7y1@R_dph~+aeB8@E}qmg1s_~f z@UIPJ;G&j4ICyRi)-4VJ_wg#Mbn*{%@7WptqgRy_^?N`6K@giOQL>)C)AgEird;rF zEDx0acaKJrgenf6j+4?h13otr`(0lPzFjiI%koq4N6}qCa;GO2(~1L2es@zx~+a?Pt_6pNuBTuN_&1tmY z>qW|7_io(punlaJNW|NYr-EdqJZeWw2wnIPP$T6B@Z7RMkT9smX1Fv^YuW69nr#R0 zk;+2gHFX3R%gNuOUDrMa0qzDf9EyFYS z!~;`b4ZNTId@a+u>88c(xl`F@DpAiHpkYINFEa&LY9oB0c^!WPYaLIQ^#gHlvv8cw z-Ok1h+I+2L_xTgyr>#{V9LDea3V`h25}Yxf0-}2IaLr^M;NHu{;_Eko=A@fc@Ax%N zwMQ_gSN1Pm6#Aad5Pd}TMMz+PnrV zQ#Hi*ohTslF$^!?UBd~QvZ0QL2Gh5n%m*FP@o zlx;>huWuKak?4V6uFU`~8JYOhvT(pFa>dRnG2qc2C;Ys!5~STfh_6c@05Jg-czj40 z#}_XG3F2Dp-oPCzZO!FuTNF!2*dC;RE&oCN3X-Sd#B-=<(f_FS=~ws-qUR{fktg&) zEm5HN%AQ{M+7*XO*@3sV?AXm%fk)7n-jOTCSIjD>^>@w!&L4`wkGX}|Y;i95tD1u? zzO++S4&Uh8S7(5Z`8l*`S_AMdJc?5~tHDO^BRDiK7%YKv@SNTaATVGXwrDTHF~cdK zhaE%aELjh#&jwNPjUhBy`IN75TnGF1ou#g@*(&IvH4WZ%QZ!!@yxeP!-_(7i*2d`5 zA;Xf`cb5{l1vK#WwV$cVr=@sozzfKy?Za*QJHg&3=P0xNUU=c9DQe`&722{l8w{Sv z#Ip=`1LMDvIL3^osPx~efV}cHSO_E4%#Dv9StX` zhCi>Y@7l?NscF*KqNbb*v)hfYtI1%OJPpv?zX&JG1o3~%S5i;4RjEFgB1$n&92-mN z0>#4sYtJ+UPp$dFVTD9Dw%+W7)n#c46wbga&%dW?TpB5dA6vnI?QR^!O$CAjO`M6)PjoNUN-dH5 zMtj|^rFJXWV1e07>e7C9y!uc8yJwA|ftoz1@m9zEcI)^H*j&`=mU#Sm`3-7c*=kBu zWr#1-Ce5i=GO^mkcu;0bU4X`g`S^lwBzXFLCmxj!q|Alt`A06R(?+wqDdUehK<8dE zHr-nVmIfWeNwels9-e)C$+JrI_hDBcWRuBSt5Kj}Dh(TuOiK8?0mc2Xgl^`Xpmg2Y zIM!FXAcei#CaUT{39aE!_kWpTyPbg?wkT>Hzj`fbnApG{o*JZ5^+ssL=tKOw$c?1_ zswZ>JGU1-Ea<=C+4Jw!W6Rk{R*4?-OcZ7SQm4ag=WojB(n3F<6PY&Z-(PhYTSsAhX zNYJf}mmnr;1N{A(W_C-)F-Z9i~IavVdAxDv)nPLisw zSfo%VM*3!2kuBOqaJ!2uIrjZEvEzoJmD9$#Pki^mmm5ovKz5YZbyf~!k?*MZpMhm^sU&4$*Z?;cK3Z2zlNmRPn{@%__C3z+zaK zt9?|5JRMVHDnf-3`|*e?cLh3ap8_|USd$NKw~1S19sH-WgtaFaVre7H6Wx?W)=DX( zg7lpNv6J(W<`COoly@E4ghIkGQ|z*zGr|=8JHWg=A_iUl#36jJgUk6tqdO0dLUTVO zVzhE5dD>u)RCV7l(+;VSr(K^(M1&#Id0L57SX0mbjZ>!F_&Qvhe0M61TrbAPlNdxuFy>32BDr9^+8St`D8?R3IN=G)Y&x4&`}&q0QPk zs6KZs8ax(-l$A7iN~YIPdYdA1KiLbF%a#J;VLP;d&9475)j)E=Yr$VtEplw&AX+VP z6iMryYBDJq0(cO7mcwajn z@i(<|e@WJ$){Pt3zS$yTs{EPlIUOKx-+PiS`Yu{O8HL>HQc>63WoUhM46HcNoMeOCMD9FxTl>%!QcD2 zdW|W>wk4l*?`>d0=m*>xgBwWhfl9%!(mWKix{2Ab;s!i+J(TG=+5%mL>rioHDilfe zBZDD3NT9bA>A9WDl%N1&di^xg(A6fv4^EN4nLAN3r-Fph)hOfQd8qJemtgu|%#9Ed zLbk7+QP75mOmnCLk^0GcQGGm^p)rQCFU&#i0~^Wfw}+VKgfL=bbB#2-Uq~9xy@2yP zcf#q1ERfHOYuwZpXL2>|7vp3kN*sO|qN}^N!wy~`>05pR{hR%QOk?v=UtQBiEoHJ@ zUyggD4@suXktR9xJ=2`rytt0+ddej$F8LrQ({JQ+y8vy}H%I^dsV2%nm>AvkB@I!} z(FdL;lQi&@i{u7Ljf*BPc_C6!KoTLwT!t1Be@W-8iAEhXE6zrf04f0#ky z1m})WGq2>D#x7=&`QWxV9D9^y(to{__{4-x`ye7eC>#aB+0-V>YUN@ZJR^v-&u6S!>)?r*@qr2o*49 zTE%FKVihrbugvRv8NsN$Oh(4mity47h_BQn!l_OOKOQM1tKTd{pB5c}=PYHBXmcGY z`<#n{TsEM`9t~(iQ6*FA_6bGMSCEZT6-wQ*mIUf+F-Fg_*_(l{;k(HuMnh4(YrNho`q9o%A!?n|p`u7Q-l^@1@dA^Ep zW4^%jZ%d(e@_VvMdo|NN#|6q~L~xBw#CZ<-yU1tlUBrK(39%05k`(_oWR4YO}akYN7h!|-BbCMk}e*QIjrI6U6&j{J5zlUD}G;NesdR3ARe z#Pn8jpL{4riOcIy-_eKg%=|>Ac~%KDJAIePjGu?2u|H9&P%yIfgz(vtBj`}77g9TA z23sw-(CT9;DsN@`Q?6AZ>*Q6W{+K=bt1Tipe)1Bj>(JrWuFHq|wI4{lRy3iOoDsCR z)e_Zr+ev}ZN2X%>3AhQJV=m~gLqeewOx?34F!|1b`>IBtCoqiTp*`;!qugevzg%`=FUI6;6VM`vZh(P|gmm&RYl}v!y6?Elj95Umd6!cB6 zL_sojXx+sW^es$-d`|b}PW)6NPwkGuE&XF;$*LFJZJNIYHc54)oE+wQhZh3vWvs!I z$To}}<}vq%rtwZviv@SZZZR%?#t`?v6r?R~gd(#8P@qx~x*q09er!6=bj*GQ7etN0 zc_Ed|n}LhWm&r+Z|9d4m^SBf~3SxUJjF-U)-d863(P|hns0XWk<`PGpCFsUFKT>Kt zL|%-a6GZ&?8eX4Pik!~dkhQv4utLL)WULb>>62PWV6~7P7X>iMHVsWOio~WijVLWm zMh`2z$#y*sL9;!`pmRNwu>;kgJ%uOTl}LcG9I17F zN$zcZ%3QNDCpwWn%*Fl?-rnv#bTu0t(v)@Y0%|W@eQ8Z zU;Fj31xxbg^a#xH+6(X8yvPVvtf+Uj?QA>)36 zw%=ylmz^fepTF5?dH8A664yv#9JWEv=}#G-3{g_=@QI1=YbI9jT*<5-7I^atQ}UyG zKiLr@#M@Gxz^rK*BGFby1tY&xk@N9pWWOCSdiRad13!PHyvviAFYzO_9cLIHmKkq} zyg-zFx|x&L&CxUWJ?PG`1M|x)k${1GGR}?{8HZ1kM^&tuq&J8BVV&Q0yHiAW;Xe4v zSrk4C&Be+{p4_ffCK3h;ysAyNh`iJ^^jGP=pyqEhvY7A4UH@f7u-fM=`aM(#-vybG zpK_Ow)>s82tb|B#>Khn6uNL`<8X|qgV!>POAjFMGhDHsFWN(N(@q9iE2Ww`d8D$Fa zOH&6Ki5-C+D@0&JgAx38u$*j6O-B!PFq&2H7-=o_01f?~q`9)03(T`g>FaiMj+2Y- zMeRdPa@U!SH~NHPPg1nMtObKhvDGSzGPV64Lt zBzNi)^X+OVGj(~0;bv7anVwZl#?mKHZ+0p&KWt4Kj7XtO@e2ZtF?Ky&T#PbqH8U}B z?L+ow7?|#Y^Meb~zYzsevCg>b=*|=rZCQ>A zOBXQ;?`8_-99)K0GC#RD{(sT|4lr3)ZwL-uUdAXV7!Z?D6(rQWob09YpwF=-#8Gb} zeE)Y67OOqP^Xv=Yvzl$_7(1^%3D74c@Gj%?(v_UZm}1=W4UqT3Y!qL}nrB^|$ey#B zTnWpAJCcr*Y@a&RU!IH(yqm_Wk6#V#drqLDth4ad7feREe+6O2+9;r0p=<3ng7yqh zj9qd#qhIm~&Rtqf1fj)@cOXS}9?OJkd2VEpM-!a8IE@Kib&xrk(ZUF{Tv7Rd|KQ74 za?~vyGtyXHOFEj?Bh8JyC`6fy*1UPhwTZ|>qe>?TaL&g)2U1D;qIGDaw>ZyfxP!6z z(TBb~9O8B^jYT_B44CkbsW38R6FIpwg?qLjRA5CUyAl)1&}KmfirummDf#avYZXF> zvseK0SFM?x@_vp2Ee*-O>?|~F{gjOMB$Ga^N>We@n3-W3+`+s1Nu8B1imqvf=f?Tu zN>q{{E2D`qkxxR)EY-S1tQw$Qq6+EUqeen%qPb&M4rH-N6}dmzL4vBck%fk=!CqkrGHw;}f9+6KJ1RF_32+rS<3r>_422&KCR;7~C!* z2am2LtFkrW@tRs#^JpPT*zO3m0CHR{O|~fmGCXeHO`?)Z;3hjYNjE5sV`W zWoB;PMO01nSWl>w#2!pT=I`>5^Pg1mL_P>{$Ya51K^mi~)IuVxMoH*78RS%4i?VtG zNmau~D5dF3WE9fi8s~E`JKzjaknun->7{$r$^%*ci8MW}c58A|>5 zA1waj2xFY)@j^FSqlEd@Xuh8cIv1kJi@$pd5z=Dz0#rnJ0P5VT?!F(~{5DPAzVDHXt6liiN`#Xq0{`%q@wSq-)}6rkb- zA7Q|$Ayn?*#2k6F3T?Y}AK>sd82c!l7>7ine!X7CK>8%Hk1c~9nafGgt8(JKb_3Y( zBMJ47o<>LBZy~(sgUsY91M>3EYuIKJ&z$&eg+4drGMYs-=)O@A*-{vdtoayC6#PWe z(Pu$vyBE>5_he?fitsjnJVPeCg^9pD0B(CBO4NQYCDJ>~h{*C8yq{APvOTVh#-2(- z-TYqC!JAF2^Dm>vp3#hO*BT@qB!p&I-ecGGam@VwKK4%h9%8KemRw1f;l+29vZiDq zOleyPKc!7De;QN;eVa^~jK`7W-kxs4o27}Agk{jyr#pz=zvs-mvrBky%*Dx8>nf-i zbP8QL8$r_a7ZD{pf=tC^c)mxBnQ=o6!BC(me3H)AB9cnrl|uw2%^V^hR9Wvd#1M_m zmO}?Z+|U9?d8nxTh#Qi`)?^4Gi?ipDX_cvPL31ON_OOP^n|Cl>tCh&O&Nb%MvkWLV z7(+Zdelzl~Kg03AGf3<2pRil~J7+vf5*?ZGR**B&&PW^QkU%H+GL7QGCl< z{&IWCIms6c{rwDDcjY^*-%il7{6S)&HOh?28S>`NwMC{kYsu3aS)`MBz`dw`lKJc4 zjeb}5qDaG36dGv*7kbu^+gBfu1fCD_oG;1)_$M@17lTdlaj43*MnK=4fo70+I5m4G zR^KrX;=4ttR_h0x)g?k!!wba0&=6^TEJTyb_A`-_ZYXeW7c^hK0p2;dm#LNgNCa!< z5PNAi-fIQR}>AE`Kp0qx$BlFf8!{7&c z^|guAf6DNDlDt4pnuB64=#X}o5oG_CCY#^sqX!Z)NHM7we%mtwS7+}hmAm(IuNbDG zb*WWEM1B)Xy0V?4ws;3!tO|6vL7l2=aT%eqs*uQ>r8ti?iV7dFx3<_Yl)~xdq)DT1;~P@(EViLQ0*}(1aOA#y-#B zkw@n6>-AWu9VW($?5Ibxs@h2Bd(o~N>xD_axF@pLl<8`by#_PYBhkMr7G$CCH|~3{ z570l8$Iw-_keflsvW-``0kaKA|HU#gK6#V5W|GE})VxR3^CXb*#3$x^hZr<%`T|e9 zX-66~4;9}yh9*v_GXLr`m_1*+kb;yX6Z=AlgeSUj7yVgJF0F4vj*pk31Gb)BYm82! zg{wQ5yl?wS$Gs@-R^czija_Hc4u7}{7Uz>?lcGep>OP~qe-RUm?{WwK{zFj#^Sczsj zw!&M@mtbHaflYjACdTL#cZ+8!Q+~M>4f{VuPND}<*eq+Pb@MBuX)}kMesPak@FAIT zy`0WX_%6?Tyiy7olx3h4PMF|!aJ`^gFbkcS!_J3)i^=F&5ma(T3`RGZG3|d^1%gFO zQMzs=QLJraUX^&kLcL0|IpHzaqn3T{g=&$8vH<<-mLXNP5{$a&J#_KhLgKdfGt-#8 z7};8g@f<%pv&KRXY#CDH>OIjRy4$vrUoj;}XS+3G&pJ`?e;FijR0CVjZ$R#1dgNQa zAw046J)Aannz?*ZgLtkv%(zeOAzvcQ(ddmNZs~veB)g}T@e~?mgs$I$gYP_0+2%h? zRZJe7baO>77ft}T<;J|aU3Z~YPn!Uou|x-)mJ>PW(?m9I4ztv2Jx~5zGZ!8JC^#gO zcJQbpCMzsX6+HN#G?D zT@fEr*4)6g+}Df-?+L z>|cS_ok~dC1~UJh`$FBbmF1>8WHa-3{bB4DOed{-EYaq$F}NecSFm}N0$IP&5e_Of zLGY~{y-Kt~i|22H52X0;xZxJ`a^nO<(%D?yHdy6qm7);kMT^4g7S$o5o`D&9H*$KXZ?n%_x&= zbEXq-txUAji$Oo!$GL|@ZVKl5EAS322q7Wu?r?17HaNr60G~g5gNe4^f+-xNA$%8q@6p%{%{?J0}P^w6#w_Ek8uqY$&wo`UiBzBnR+# zmr>l6ex$b)lTHs)wERmoT3cs=4t2GI*+zG`p~45rtOY^j?QL_ecy~S#{qUL_dFTzK zStrWn;bq*p?LX4ovw*j?j`fO+T!~oEFbPM+eoBA+8-pK z9=9B5tA$AQKPTqoxly2+UB&E*E+!f*pR4snnVcw^(N!Lq1&xPBQP}r%cD~mpyF(5# zp?k`KQ%nKV7j&F4`7j3=yw>PyPTGfdNDh*TQPzx~j7CP^%mreON4VEk50QjyEx3E7 zEm2;YL(Yq9vitW3r1W_Wv~)JY!jDbh$`JJ~lP6|qL9Z0!thkOeoZ3&i1>I0)>mw#~ zp)px%;n>yf)5Lh=*8C~2<76RM)gJQcn+RU{)0Npdfk_E30G&EDj7%JxNU!Y@A|7xUDV;ft+Wg(QR|aM? zD|^kk!kZ&We%@>5QoSbQMpFez?;Zz`8D+AQx@{8dkqpI+4afdeDtH}u;5?W z8`9MnLo{?M5q(7(z31IP7k6hONz)ei-Ov^pOthn6UtLCKa0xOTa7H0#lwsMKb%I$v zi($3&eZs{35PXk*MLb4R&{(UDpepJ!%tLR6BlGC8q`!)!}l*r~s zY2^0hAQE$V7!B2`ko*j7c*;#&V88hy+?#)ffcYt;Vc82o$FM)~TqZ%{)?Y+hStFRn z!3cl*L1wc?is{~!$Sye&<$PB`r0pfM)Jlq}78wAAQ#Tl$ecMRH4iB!yn6jBWNIm=fg~)GZNTdo1YtZi+tM~z17b(E6 z#&O8}E1Q22+JI8R3J~NaB4CN)8M|MwJ|-?_SPyPA)|8 zH^ouHv?$`Xv6tM~;ffT$6_Ux-`SADMDJHaCmiMK*{+Z*`rV2iIWH!gR2kNX%|VVQZjuXwTJY1fz3||G1p2XUiZO3ZVb&hVBZvLP z1-ss%#+S#^xKLp1bS zRZIS@oke_Il}X0=kAlH^E)lBu1CvC0h+5MDq;t#|`QF?@PTx5|gu5$HSfMA$-anI= zYw3~~%^j$BD35Gc(cyjeJc#(_T3x(UU-;bf3wkGJ4DV#^LW^FfA?+>sg7xdqlW~nf z$aCsKDUX)087(4Q!_GhfC8WkX=kLV4?OaEeOnE@tlq?k7e+6wB&OvvK%TZADR-$mE z8^s$gK&f#l=*z7+F~JBln#ayNq#nzQ?@}3j@Crkvk6JuBdr*!&U&tgD zpHj%Y)<>wYVlK0D*>=?SC>aT>Ng@%d8eyjpV&k6N^+s$Q*}c^yBA1#NH;7St_Hy!2 zGz*S>N++2!mC(eYzv%Z9dl=%`kCbPbpyHZR^fpKglHG$`|6(o)eN{p}2lL5S<-1OXksbM^F1xm~|+#uCGWVYxzV%(@jL_=+uW{F%BI&U9Zp5;SU_*;{K;nU=- z^CDujVgYeFAWBX>oriKoDw(@iyWr);FA3_q59Np@T2qHetNtAHt}T!hYdnDadJYIG zUlUZhI|k|hJ&J-hi^5MoX(qY#8*?V~B{_ok1Nr&)7&vu8fSzeHH5V{daTC0LL7cQ6;=|hxt*Go( z2{Wr-iCFYqV$IjN{5kuaf3U))9*-Wv$R?>Hj) zf-~sH_D~e|atCVFl#*C-ZQpZ>Et}M5Lw@oC(}P`F)#i-Br`wha(A}WLhG6n+_>v| ziErpX#+`*eXW?7-wGmM?}%`H9%iv#Tmzk^CdJd-_yg)?&qQu1&18bz z`?oiG!>~pEWa<6oB-QFZ2`n67l-m^<{bp1O@%%xLdl!OoL|K?3jxb(p)^h<)a4yotNBFAFpsE!qKsuJ%~e3=iwd@5m1cp=9PTaREkwjM)#^Q=P`P!(?3wShv&`mw~ zT6hM}m)+0LZoMP8$(moWY}Qqx!Xd$K7D88<6oy9?l#y^zA=$~?M`3#wk`*(KaqN8S z!9giw*08(8EYi0`+Mo9l&&YVxlVFV+3)!;(Y86S#X@foza%A?CbaePEhIx$&Br8XX z=NQjGKWZhq7hA&2_|!v^u1a)O)jVeYYs-Y2@|F{u>>K32A1tHw;|X%OsS77t=P@nQ zufsJ`ka#_?BNxxeqTy+N?74I?y7S^G*=%-<`El2SB%BLGZYoh^m;W`AUt9u1?9RYN z)A|_!)*{MNlB~-&3#BzGqn`JHg0Hf@DBxNny4Gk5y%QdxG0ACN-uL<#p?!w1eMbZ; z3%tbKws{~p_Pd^WI=6)chJIxh?7fNp>v;t$62C(~{RU(`BZu@nQ6Sk%`_TT!Ritaj zadPN-FPi3Ng}~}c^s49-99Y0X4P0&JcqUt8G>k^tx@BnHr%R~t6JS*1?xL5y+2};+ ze$@V2hgj@Fs6*!-3eUAi8zZL^(aW~zr>-Y)no1+GTe6s!L7$;}Qaq}Dd5L+DU5sRW zs)_H?H*k-9Ck&LaWsI$9G{e*f`5(Q+eD2Of4(D}P7VMz|D;OkD~K%#OnLwxIM}qS&@}dR5U#I+~*z%Dl|2k1EKIFpcZQOFeE z4KJE+Wp1Xe=iHn=@uNyoAzwy?pSV_*Xl*$_u5P>o&-aN{(RYfN<611^q&AnF3h0I# zEUTf+b9a)nekD=gdzCyGUm!^KErkCazDhRa6vKUo&Jy}}JS;U@$c&tbW=&jltBpT;FqU7jzK3~Tc@?&Oe@EW>xWGRR!OX>8 z!d<<2m9sl%1V=9lvkDrWm|Y1s%`DYzq#qWRAODB^jw!Yd~Wt-8Lptq0N6x$HzWqvogb0z|F#q z9^XBi>b*RK4Evv;L(d)YAI)~;5NS#u{}=^=zA573%NyuhTsn9+?M5a1&BMb>%xIXP zk1xzv&er+q0E_TUv`^O*wVnEmv|8?fc^4Pq{m&HXhv&j6vE(nPeARPsFuSE9ZO9g7 zRX+o><7Z%r;DvOEng}?Sw*tsI+0maDj+PrI#{rX~cJO*Y=zqve1l5aM=`@uuWWYVzLnTOalp5cTcH5_b60;8QdKdeG)YlcP zd+&MhMfN#r`w~%^=mO}I;!WuKdQ&!b?|;+_p>Mx(*Ei(~=*OkDn&sPWyS}Ir%bMlC4DB)T-krJ*jBch9F>~ zZ%I!d_6EyWvgoeoDQZc|JX-hLV>HS89V_b|1124`p`U(u&hDSIm(tP6ro3Lvrh|VwooRq?pUWA;>wIX$(C!N!l=0li-i#U)a>+AfXyu~rmY zEd0QmZ$At&8fT)XDn%4R^8oxbjP_k@14h*nEwN75unkjplq$ z;eGg%%gz+OBX4^2giIJd{%*0GJ<)80U4{Jeo@px7oyDHGvC0LXwOx&-SM>>h1BKIj zme6j8w}I@R7HEc;0&UHluPEPQ*MroN!gKOCZ9 zh&IllJ|MGyso+O;0LZDHPET>Gr2+(s7;RjMiq`Z|^2_d5GTL3>#{+jXC8>!$7r!0Z zo+juAdyZGQWF^pXAt-5|8os+&97lR4qxoy?!6PjK)-@gi=5vKC7sc1?-Ya9^*AsDC zYmfr4Q<-+~+7JHN6@lba_Fz^qgOnRJ@a1(5$S00PgU9%&O?ZpDslf_G#*Toa{Zql; zjB}v(&UIk3MxJgLehbadd4GT zPm7CCx$`jZR8l_3Hd8{<9ljvJvJKRRX@ZfTB6N8AOxpae3OzC6Ol?V-L|Y9mVU=4n zkjDuz%B3l(=9Kn<8HivZn7rs{*6grouO9zV~f(yyqkkS@RQ|`XNsn6d3~6 z{St`0%0q!ZT_|yGCeQ5Cc{XVH51Jw5EIL(Q19rQLC?Dq#V0mdjI$f-Z&rcRdGU7&H zVVD#h^s=9Iz*4kUoG}Q>;scqTkHNA|O?v-)FR(|D3j#yGf}NHORYB=s->vW2uX&XV z40Xq-)!EWm@%>X|aytM#ESn9sNMXvubI@YMAZYsD@ zAT7LGRtg%9%is@h;t+J~MlnZEpf$;Ws?X(`zzZ;@;g>v{0Erqk})}W_j zThNwhUHszA36wmo0y&*|$2#A=3o1&=!O78dAf3Ak&GR#>yjUC#+I$?TFeypU^L!(a z+a!Yy?fT2RFu4>=j{`iVQYkFIT^_9xzk~MlR~8wOGAs583R(~JTevsnF@9xz!!2JPtXK)>cr z!w-Il<9hKDWVeJy!`c~WcoB_0$Ua5q@}#K0Z`+W_M-J>i@Pdl3-OtLahojFw48Vz> z*D1g5bL_WA-#}pUGo&@;BYJY-Ec;qTzVgIBp;zapB^9}J9C+tz(5W|6X)mc}u+7>Y zeM$L(M0RJR`o+?yJ+=fKv>s)Dk8}xzh^e##UjsF{JV${$wXw%^UG%zl3gRhigJZ?b}wkR%13+wGl^htDJD!=O4gcph-&!eSyxeMnJgw0CJQXK;K&z zqa<0U< zVK!(!vKS;KJh!>rcNBG}>fqg%Ww3Vj0(RgZgPuCIqP-4lL4UL|IMSVftZ)qqT+xcE zii<#Jr3r95kq4f4o<|M}S-`?!CjB+W8Yy|%Qq<9W;L{g|)W)pnSw}R59wjf}m&T|6 zJ}?Ir+VP-u)dFE3H%GdkmeOCU=FqAAVgP(uPK#*k;Wjy8Hc9^lYI4^!yn0D0Rh^s# z4z^E0=%NUoebO1l2U`1J6B5%@ z67p$<%&e_*@U`@v)Q71qSVL8bF7lJ5yJcL!>96|q+o?{p#O&$xP0dIuc$q8h${Q$v%ne|ih= zKR10Ws-a15sW7KI9{aJ?R@3Q*_T}`?W&8)@GwMnu#b}=;M3ZRDnX8t|RcG z4B>|TNMiRmGCo{Lc|3?kUFy2{gwQX4%G!Wd2&<>Wa^}!};u3W5k8hx-DH$BT_>q_I zuT8I&*;3j5ek&-dUPK?!*QULWYt!HNou_u>B4DTY5G?!&srl~=z@NM$pqZnI+-87;%XDiStE=$mQjT!Xjp?vT=z@FV$s6da$H31%3j5Ql( z;fR-7V3m-C_C#hL;>}A!(Qhj%FJJEm&*#ShtEu+^Z`Iw(bi@5<$ix!&&XUDa=jCzc z#}ybJGQuOGQ*nogkQdZ434hVPi6%We1+-jTQB1fA{uv-cPaq!sv!+B&Qe5M*w@zgWTr)cZHKCsPF$dJxD4eGd~X!+dhl-@`;yHxQY zy8UYi9a$+r5@F?N(K9U+ziBhNt)mEbKEDReHQ%6)xt9X%bPYNor4IaAV*(b5zeR8Q zov6G_Mb^>k5Zjl18_79{;|;a3sN&;EbaAUa`hG73{WqG$D#*P;yTm8q8QM9>M)o^LKRi*&CF)X%e0oyTTe;`~_Ye@4$2uOG+-d zg0eX^ht{fS0B$|n^qw>F^xV)opr%m*KD@_3#&jWSku{@vmnVRdwF9`_vL19@H>F>h zsnKiN?CJSmOlV)Tw;*Tm3;4cP6_|amVSl@IvZ0smRP4+vLjQJjBb^_uZ0vy=-ijZm zkd@h7>hQ9B^kR4xN(9GIhu#`=GjRaTo_rWJC!IhSZelbmAfGLC%t94Mmw+c3NkC~V zgB?iJMXfP8DEQB095+jmN}d)74lI?Y&V71E5{8=C^vmkdMi@|{xI6K=j;g} z>B<9|m>x`2O8Kir$^RNkv46Ort~qBr=amyA1HtQ z2~6_O2dZ`Vz_6V%9duNZHf=wHoMvx9ZoaEP;O1rQ#Zz^>GM8q`G3gwz9r+K5i1#B9 zbO4#>)g!Ra7`dB#q;^X`X7zdn)TdcbLA;`j&28T}%5d8Yv|(>LQd!vvjE}ql@f9i5 z94#9V-(SUc?wN-EJA4TBpN#|N_E&(lcM#}5x)H6nP{wN2rf9iV7z*0=1#M5iSD7WO zd#kU#0x$fm!S_NPKyID@Usi_#e$!i!b;Y&vxW-I;Y;?1CXC8hx}}$(0Gq6(%SVI73T_g zYP=0>@E-w(wpf6-83tfY#8WW%%ZwIzl*UFbuLJX=74dMxbbRSqBHClL0=%h}2L?Xh z!JQ+GJioR-;P~e{)bMl$F3xnrKSVrm)!!ks`*t9@TW}EfbuYt(M@E40Lv31YtqQGV zV@#VZ|AhX?8(=%Lb7&}h4&E#GlQKF`3#QyUL5s&7q66oy0QF%K^cvT0(O_>VYNJ*bAOg?*~DVmVe-JBI71%|eDuFSYFPa?r4C7G{oLshq|0z*{ZW;`)W_ z@D8b2`1-gJy)Zg~3h&$xj+?8}b6kI;THYvfK0k?G8>~n>tf;A6o1lTwqoYW5*+P88 zRg&KMBn&)pv!K)4*viA=w|N46;SRKQ5Tqn6Lk34|vDTmy)_<@P|D@W{Cp+QVfIH2diVQ9 z!vA9v4)oT-J5>Gg6W_&{l=1L?U^!N)q_KmL-LfO*5%TUEqYi{F2J_OT=@^5{?D(@W zAwSF)oA~U)nbRFl@!vT%UqsluEbgP_@rA&2wJR0uu$a~|ybC(bh_EPG&?S*6;k*C_sj8_us?g3YEM!4-3MB3a|tV8h}*uuhRr+X;8M zR(3sjq`=Y@=QJU%w?f{*~4kFLOdf@&{AG{n5L-WJlAOnRT=yHw}es_EV zH8=5b{(o&iN6QRUx|rjshZf?=zh0xp`xMGurHGGhUWQMUegwCe+u-WaD&VL5n3bdY zz^KtI~4>8y356sL@U)+;IDvwn}v@G+pj@g>#faS$6$^1|I6Z@|}&u9W3% zRgl}z57xOTQbE&H(La@8G~}98zPPrWb!fD#^bDv%sd*E~OU)InN`cgqquO|^C!7Te zzt~BQqip}Zi)i!8G&DF(3U}`zC}d_8^2imkR@SKCJ=0T=;omaUv>_dBdnQZ&vdsl0 zW))~~jyOI4TMD&8U$incEe;i&zY0FiN}^bkn96>?LeQirjV2P*@PmonXe_6j^_eDz z|IHpipP%nxrwu+wj{9R-y+9-3T<%6imj$7Sn`gkl^j6;FYHi9swibC@PDbth#w@at zVy$h@RqT3LPX*4t04}HpQROcu3+KK2lxEZ$_QjGKo>S~&q}aKGF?z}kPq0>dvEjrPGSRFc|N zVE+CzFzV33zoewVJ&Q~1oxo>o(S3?G^W}ju)nj1oJ$vxfpb2PYh|y2wG=ar37qEKb zJi4%w55k?J!F2SQati4J-$^=%lQBT|AKqdg{~8A?MXXUfNTDKIDtRL{GwAzz6nOGW zgO(o60`rCMc(2$;@ZrKRpZW%j*?eNtsT4{)^%M0DhEtz2HIbR6 z4!wMxHnJbe0M62Wly}`la98a;m3u4-*y>~f*8mmDZM+#QdZ7cH<^&0K5)-t3LW|uK zQ2?UsuY*~q=A#=2KZCvZE5Y?S_o$;d3rv~LQVB1kfXDn4&@pnKjgy`ZwBHqg;cMb_ zm$+Bu-%)#z@vT!B!y-q=&=~l2WrH7?vN&VjZ)A8i5UG%9INy5$^}h>blVS(Z*MHX$ z|HmcJupu36jrj(?{>lY588eWx8bV(jD8T7FqqbSufsSZ7G^6}3OP4jVs{*c3ITyAd zsR0Mpcg`x*akB;h*$%KmBb{~mXiV>A8qvwUhNwRK0HV@jkac7bSU^c2kC_EXEm6V7 zhBcwT`Bngj9syYSu>lC2PuXl=w~faTN+h4^bBP2fg~KL$4ga58hAj2G0A( z!0nEBHhkM@*1gXG9eWrL^w);4Z7Bg%lTc6U6LP?ISyJe)aTa=&coAvd&q3AG!>I10 z-KeRj9UU!5KpXv8z)laLgsw?+|6DwkfP7iZqq2;>8ZD-az+5E}(>(zo;Rs8r3$cIc(n3-54pvGoJ&Lk=s?Y|PTC_Y&1k62I1~R`W&|b@GfUn3_ zQ2R<3*=;*jK|j(&eq9|r`QCK!?2G}ZjW(t&nnuAH1s^u1*%KA~ilF8`752iJ7*wB7 zr$5O20md4+s37o@a2}V(FSMm_UV<(1r^C?d(FpV)Dh&k(GR0N}zt+`-- z+(smCeh!&0M-)Pd`ZIWSf4D*#I?(eb)sG-x7*!v<8a=*j>zue=IP z6=m_Qo=$M?0^1CLelNYl=)!ZFa-@J5T?$A5R zR*(dC8*QqT*n0r^<)t8KQ;P&b)`!DAA*DsPa;M|l6Fwv_*pD~uD z?3D!7@ERf+cPdWYV0HY3|< zS9v^{<6yAuDRRC{(6WeX)O@cO)ubum#F9X2{CN;)*jWNrZk7OcY28$}RuAYbv&N6F zzCk{pHUPCPX}oJ{ePGQ#OL}>jhma8&gLd+zaPgN%)K1+wc-6}ewC#&9=O9=GJrI?^ zIGk&o`3 zu%LJMUj+xcZvk(kyFe>~pf}rYqOg)KpzEFtCa?5h6dZSMG`yhSX3; zoYN`4Yz34YY)7B6uLiMe=CUSU_fgBa>9o@BZ9v_V12y(i^tZ<**#Bi38dpC9Y8uUf zU~~^SDRvXR_#=xnv!ua;maQl#Yc6s}V@>1?#_>nQM>twZ;pybYYp zdw@~6BKRx4RLC_)VB1wCx>WQ4I-eU0hFwdLtWPf*U8hAq`H=*+3OP2Vl2Ul-u0~{` z8Hpauie#4^C}5w4NCN4r!m}O|X<%rX0HUrc;x0uVa-H%Om@Y^FBfl)^p*@%2Cw zb*K<%WPh(bA_sw^Oesn-HwXNMKImS?S}*{6fS3PQP9w!Avg7d(O1w|-i&2cmxo&hnj!@zmm23yuEfuh_7)H+WdFPvS4o}{&-l=ZFbr1+%w*_n-o1o5VZKlrZMT6QuH<8n&@90JDK}zzb0%pC9>BeqJdOR=z z@!y|d^-oIUZS$_ODt}%g>xpT2Ox7K(S(yb&8#myjwE5U$@eO2aQb{fI5OT9#O5ocw z(`=q^JjxC)%>zH^dT{IKE0nOj3&m!t($Cg?Wj8XFRQ{7Yz}i6<*LUThrD>wrgL#iG zDIEe8hk6ku{SnPlHDn!q$I$w0FTD2PDe91g1@$_ILtb8m;Dd__@;rPM6dt{g&Xwk( zvrREz@rOEaQMQ~-%+x_~Fb}*MnGLLFJ*IvI79mCU3aj@(27Mefqpx2d0;)_0veD(x zv)LV#*M$UNJ?B2(;v6pU2%mi zyKttm;f*zR{-J^ev0(ttKMYY@euqGB+ zC0DVtcu&D1gV|v7qXvMdxuR-e&+}Z6fo!kogKHy;fFDzWeivqfHL_(tGbFNJj5FCgvV?WiG(kHv@1po4$20foL&E%cPiHiPphG`AjXOuYbh-CRY- z*))So89NcMX<;i8x=@8oC#87z(Vc9qQ@pgEa`Y@VwX()w3fi%4DjJqH!-rdTvqM3p zXn$EO`qs1+Yq)o!k0*+O;n@qoIB+Zd!SDyrk$po2ODeNAh0^rJYn?VvgjzalcN)pY zo(H8re}ZSHD}fGu6{xt_(Ia(_co*wh!QWk(K;jD{)Fxd)TlY2a-QAkHE}}=P-J1eZ z8zX^}e*)Vhdy5J_fmn&e$KbTGIQSmoLc~7xD6^I^y&T$uXyY3kD@)o zbFNkqDQr9x37#gTp*89fXxB?2zmaQ2y^)D%INA|?_-u|UjL(8eURzOiOFq)xZ~(n_ z4FMhb2`D(e6KOln0N+C1v1f%XWoJPNsPopNV?+(pYLRS}6~uU!Jm zkCXvPziIUPjnd%fyn{eG#R`nHIMW&IR`9pA9>{;`qTWS)110;S*!{h+$bQ5J4aLNu zJ?`tk_h}K8ul$aovIA?8@7XsTUWxI7S~ zmfZ&X+&m}? zV4q77${j62^3SfKkYlqzaL_>%b<-0?oi@c=QtZ+2g$z&_k`Br`H_<;7TY=BEx9IvD zT~sJBiOvw_9lA#v(W$>j!3=0E)CX>W{o7{H?UekwZr4!4H4!)c)kNt!D9Gt$S|@V4n)r3`uM+V-n?AK zynHZJj`q?$3qBVAL6gLE(V36)X_qBud5hm_3G)a~0dL_wnZS?$itnz7*9o;e_R~yy z!(c5se9w?(qQ}6abH}MXdI>#Em~F_U&!unG+Tfx_O*(w>JfOUd2ksSrMSIT8NAkit z+i_78*DV|c+k%zw6mcy|QJC;6$15_`Zg_}YXzy?qad~KDH#h!~G?wu|jl&eb{xmuyM(Wdmf!vyU=HmhRO z)2R{1Oq?6f)@`&>5z?8fcn|SYGw2T zsT*?uSapRJ9vrwN?nFQCN}3jo__M3;rxSK>{c;8XZZ)O@p+>N_Wk&G&mE zs4vX+%gtfUM*bknYwy`B-7!GaS(n{7bv1or{~Vf0n1aEVne>NfHTp{WIOzHP9!*)S zNk5BwODQKFMNeB(kn55w=#IYvU2?hx%xaRy>tE@Et`0lgbZ;T%DNUx$93Wo1iUQ9= zy4X^k5!7i{facl)x>Z|`P7)79({AzUCa*K#(BTsBZ&(P!E)#J2 z#CrVuEJaKAY0@Xpe?vd(3V`!=1$@DgpleWswp;s^%@bxFZhP?#DgV@`)k8T{Mt=Yg z-s#b0ecj-j6-Dc{&c~lPJ>2e#!S$amSkz}0zP(EX3;cz=ZF}K)?>`2=xTb@5y=|v1 zt(t*7dNk?P4!^)C&jkN%6Qg~zB!GiUE?7JN14xaKqi0O+LG@<_(I!>`AF{E;6{^a3 zUV8=-rbq(StRnPBONnOOmFX!iLap!M3cA$@(iKUw>2m|0kbAQs{cZ1W;60cJpaBo~ zKO*48I}7~I&j9c9n@z3Cb;9y~_AstQ056Zqz;V@S;724>Ay)}LJXU^yH6eX>r+>d-ptQ1 zW%ds4bb>Kd`+W|!x_S}^^(nUF>od9bKU(nclvsva`3PF~Jt5_re?z>>1zulO09Vd9 zP8`J$t-JdGkyg$SsBdcLr0={V?e;WdE}urm%norHE~)Tlj}sIxYb6f-r-}a!JLu)U z4+@lmnW1&%h5&@w)=&benKH>z5Fn;b^#L z(Q$6*{1qrRe*o6)ohcYStPNiqtR*H~B@_K?9z3XtwU!t{zOWa_n6#wIZq%10%V zo$of2BdUOG`s+-(2PZi0swvsV4l;_iG34w0`DD29Hgs7~%Q(ufh0WHdxJQkJut8;n z-(MmI4f{CKu^^m(#6pyKiqw(*lqUGr^)XJ2RD#oLl;~I73vQ#>Ip#*sH754C1KI6% z30}!Af%vD;zx_LcYkMCdsIglIZ#m^~ayidP$U|lLs5Bg^p9mycz7o*#?hQ_%FB>`v zc~Wv~{&L+FbNN1>2(x8tnt-#OLe|lQF3dSWo=cnqWB;Xz&<*CDh2bdXY=ebD=iE1_Li!CyM< z8- zuE4@i?Zjs4ZB8%$0eow@f!sT`n)qw`GKTK5Fjw7~NX6eGn;YlAJ2eW7n$jeuxxpNU z&W(b#CHI+a;k%i8bK79r<%4AJzya<^zOW6gq?jj12Dye=ZA=??oq3ra4(~gzBi8ow z;3FTFn3%LNzDs@)g$2vW_vjmNVX72?L6=C#u^HSk*JI>@jYgHPl^fhJ97=wfH!!EH z`#1;gJU{NmBf+Xwd}jD4Bv0zz)0zhcn4~33xM`+$Av-gK(cXCmeu}h%`#(;#y%Czv zHLdq2AEtZ4;8`=s^&vMXX`ezQw^fP2UrdcFh6!= zl2=pjkoZeU+=7W5;v^jd!?sGpQl}*1Rlfl8d}(sJyq%lxdWJijKeZ~-O2oE#<9m8U z;SqDH%8^Jf%z-ZtOL3M30^*e@2Wu}RGRa9Xu(C@}aA9c%IiYu+|2=m%bR>z;(U36D z6j%Z?~ST6IfC>kz2&I;79Wf=Pje4EI~GZi5I9=`oqb4lZd}jIb0)im+Sbx3w}SK zLAw3~FmKs%PHtf_y52g%waaqwY^5)$x_*O%s_$nCBwGYNrTv`31)<;dat*gHHiqv! zX%pGm5l265%4S+D{^O*Wai;nABwP8wXy%G@04X19=X?CFfxGhWbJt$9f={L?{0hHk z{J!3BF8FCZBjhZR6zL%@Vc#k8p6P;X7CvTvl?M=BNDy>ZtRdSEbqTUt>iA{ari6f_DeiQ2J3zArE2KnOSTxNeEb9T~g5>Rm;zE4jh#!h`q zbdWFnqwh|Hs|EM;!x6}YR&$rU6`0PKYRuc^8{w6sSGbzRpazEpl~Q3=?_W8m2tGOY{$v z5zUfVX5hOR9JskcP;8sYUr=1cZ~yNLytgKg&^F4<$3u?%V_|(vfOiIQYk34EnL{LG zN;9|Y#8NmhM-2X*ahG|)-s8`|*2tt}9wl3yWEm@!9Z)Ajl9=-hpzNTupg3w2Zc|Q$ z-t~!mRrMTbzA%mW2@<(`XXi4Lecy69e*~P*_!zf;yDz-f6-hc8#HwV!nsKH&wp@_7 z(1pQ0phJj3l@b3l9Q?C_IjgS)uc)*$*1u+RjY|$N$%}Tvy-m{a@_juhKT-#^p_|}m zjz7$N-o&gC`YQ9!?T3$zQn~uor-}X7100_AnyIPMs4_b0&HtU~05`otn2`r8(fs*n%N_AfE;-QzvRp~ntmoXk2cZE?#VFg-YFA@o^`}<+9l$hP{co@ z8AbbFo)1-guQ6g56Zi*?B@nf;Bjo#RYr&(x^~~C1m!X)b9`qK=f$`_HNQq`G92?4n z8}=;bJPu1#`Bpd*dQ&2K`YN1QP~{{ps~XB7E5_m80Cy?rC2Y=lLd5S$ke?zET(yoC zb7EqcyXx!2&xO*kaaINwmvDp0T9GOwNZOK-55e3Cc!B<2Kik%Sy$)0y6|t>YZcSVc z^GV_{S$OE(5O=tAfYBIim^!UewUu}k6VJ8GC z>sK;MCois=>m$d>m^~s_3R{>E6c0ydVKQCE9&V1!AmUbSncmbn{3tKu%W?wBQPyX4NTGc|@~7E$nR_(oV#{});wYUO9#J4H%NbD(XU7E|U> z2{%qzLK?|ohJRhLYR^C$J+NO18a(NTpTAmgXtO!Vdo9mZjMp=t?r=mkokw0Ts9sDyH|D|0+3L2pmd&h+vp46rT))B950^043KHR? zvpYGF78%=9Fc(f3e-$WSy1?B1$3mXaxBC0+4`%C+*~IrkB1}vlVKxMXkgEn>On0cL zt!Y#qDL;6glj@viE0(buCR6=HNiLRb?`$GF3V)EU2Xai0)J6WvJvy*ffre){2)*dL zAX)L%1Qvv@BE`6ZoIdQs85XW%e)F#}OKB6bySIk>v@Vu2tW_rqzwPJWNe*OwJbXuX zZx?bCtuB*dO@F*M%8*E_rZcu!kBpZq!0)E_u$}Z>IOet&##!7TdtLJ&S~eBx`!~Ty zTc?v5Zk~*jkm;GGe1eqo7F3<-nFVhjm`WluqUhsSMHg=+hcZ$i zw%Ndx6~_wFXHMY)%q6Q1UwOzp6tZmcmuA9RlXB7*oWgl7$tC(X)T)#$gNVKAAon?E z2UK_}S+%Qalc0uwhkLRqpI_rUiS!J(pZLr&_hKPz%=Wv=@ZT*~N5Pyo5HU7FG2p zXH=O+J!Y&dqsf!f+r+TCf;n_qhFqzg!w7IbJW(zM_t}17RP+d$@RlK+VM~}zx2`cG z>9PEqs(p;gR;{Xj^n^L}YJ!Bm9p&ObYI126dhkntDT51w1h;A~a^?QEkT%NJadfhl>rC6ZCyuV%}wUP0D%X%L-hvQX*c zHgMcxpYW!VIP6oAfDdl7?rlnJs zE3BDHo_+9#B*j>8SHg(d zS_IR_-*EPO{Yhn-18K93BxSa(@RIK$qHwa5@3K4=K6)kO(|S(l_LrKF7yZ^%2Rbjp z?8_8+2(~Z<+mg6|)?!ZPyP^QUxJ`z~zmW-@8H}fO4B0re3{v8%@x{YPXj`mGTpCB1n&94TeG8(-)W<{y(9Eg_vODS>dkjc|@@5B4&cBUXow)k_B%{ zT43(qc&NTOlSsK^?!1i>Y^d2nvaDu8%l3`1-^qouc&$dBEcSz6ZCeT67Rh-WJOuZ| z?&b9DH#3(v`Ep9aD`D}QpW~gM6u9H<3*pPJZuHkF(;;hhk%^q^NY>pi;aiwlR2j4P znWwj`h?de}!6Cg;@?MvN2a`34Lb55`Qer~14sIp(-z8yo-b2AVffQ-nB2iV%HzrkI zbf9`!88Mr5nlq3Vtztr5g?`43oNnO*dQ;@Is;VLj_$>A;9co@moGN#a9f}QbhTTVa zftN{MI%L5q9elXcdns%$JOa^HFwFM{!#ki_%oOszWsvD=)gsecbZjH< zE+j+UGPY$wt6;PeAlZ2#{PM0S60Z~sH$4)sY8tKP|Bg_q(veN!dw(tF8qBonwvD-&iiF?)B{DkW25f#I&v$>z3Fi1)5c6}EWOC$V2+CH%nt%ZK zrZSlUuY`QCRifnc#0!|^=?|T#P2@_CY}NbCV_ZeZH0-1ZyiNk>Tm7oxTWrnSkCrM&yh#?@gkuGJczSa+7&y6Ox=%J0Cx zHY=b-OAF`cnn^y+uq59kmceUfH@V;>R~Ub=9j;V)&y{Wyg{6_ERS~AM;K7b6=>0>7 z{610!m-mZtt}8@riGL|G$5D!Nyr=@}Zz#YCuNCkfoledsZ-9r4NB9Nn=R)ZWhD=q} z0sRpG|ZgXeJcbGT$E{y$%4z%r%q^uX66E=XQ@QI# zj%_L796x098;UEqLki~vNK}HnJ#>yMap;7PdEW^1dXME)2kD>uTK=H?G+219g}nZ? zj%=KBj(CW_V*E#Bn+|)1P$xzQ{~^I0GhR*26sW93lHW0}k$QAZ+t2k|t0fwilb> zi(9*yZAV#wm!Uh{B>j==T+eYkX1Xznla#sbf+$XNY%yoMSCOP;nA)yyUO_Id7it0B z=ZQ|Brme)?L!`)IPE|(tTynGNHK#YdkXumt0m|gB=8pABSEGFGvQl;m$@&MQqJE=S&zm|xeKcV*`ZF32ag0T-E&8nvoTu#wxk)r=_*cj8P& z{m7$5@#Gu58}dz_GDq)EfusAM^WUPI+-G)B;Bc=2x}At5H_nEUSiM%FQhbV=DRY*I zX)Gn551oT%qINLaWrEAJnp_o}y_DoUoCM}S*^_Bvx>I*zk|3EyE?8r^bM(ru^|?Zw=gRP z`~}q^He}!ufhn7ZxXgVwm>*AfaQF1}W^QL(Hr#@tv9lPP{Z(-IZ92STd5;Tq6k}Sh2{pDEJp^fdBj%1rxC+jeJn_+I z9M>I&l8UyR%vcUHM_6C8jSM*BrYg#)&Wt=-U`{4H5=o}#Yvzy8i!c6^gI4oI;DL3H zL|vkoTg_#Vgnf58#ZF#TQ0WIjrGtJ|rkDr+F`@}9$Z!=O880=OsDr;?r%z(K=x$-;ru*E zZvIbbv|kVIm>mjDlNve2?B%w%W;!v&bQPoSHHA|=qd+d~GU66g3*h5p+ey6ESFYo( zk-*J&547tjBUy$jaG5e-7jIj@4fym!S-UoFjomB0>gF`^Z2LY!FL}pgJ>sCRF7rQ% z&O9#0uZ_d)(MC#}gvb_2QO$GaIT2FIR;b8QT2xY`B+|a`8)-+jR77Z=Gv}U2i=|MO zEEV}#LSzrAciunxclylCGv~g~eZJ?qEYr#%;h-&a!eAEqSsuatcmd7 zmdjwii-WhSEagQ-QmMuc`nr5Jv_-DPdG1x{Ad`W$6cllZ+c$b@X(3F|%cUF5>*+a_ zM*2-6gSM|x#7YU9Antms$kbujWE)_8qOaN zh8pEZSmT-}eD6~cb??*0g1MLY_CGW&OY=_P9SvG+U5%5NA>PbxUkTX7+Jt8C4C(Nx zIsAmXcUiw9s1^0!KyFW7fFk8{wwi5 z!X*ln-ShEawnnxzZFR9&3!uAOkBJ2vPtBI*)UnNS7^Gt6zkpA6-AAE!bZv1 zz?ge;AVqr^H^eW*t`E(iL(IX=JQT}6{#_e_rDQDs&Z(t-()w_xZWpUsIY8fz+DGqS zOoH9(w+KxOPxGV0M3{Ye3eNOB!uMtuVuM-cu-7#VhZvscU;TUvpV`<3_x}{b%M-V# zyT>PN>T{NEE^mP6CdRORhovfouZS< zG5)!=41d|0f>)lirBTP6snNd}U~78d%biQ;VMi+7x2#z>kaQ2E>s?^wo+yZVECVKg z^ei>)eejUSY0&lL@D?UkI9`c@wL=m+>HaRq?KK97&Non)77Ruz?cv-C@Y2Oi%g4fU%PF(ei7Yf^P_`=`IGv+6It z-P1mH!Lre8W6Avv-9D&UEsf#14jpGpF&Alf`(U!$qM({tzww9t^Q!PAfTj zTsYcYNdM)ZfvQn+Y3`$Ytl{CA*tJ9j+KNU2sS3gKhPCnJvFfz`u$)L@sS1AOGaCB_ zihI@gXvp4toArvmO0R#Ij3dli*oa?qVUOf3S{l*LhR;r74Y!m~U&)I&F)9|nJ(&h? ze~(a?ee#wTF;Vn+)qkM1Adr6|OU&!@e+=E59N315Qt)XS!p<)q2=%@lr8{p}L!9&> zzL!ljj2)XN{C(XTKz%h#ZIi(}{a>>mPawLXv(3Bt)7OWws}z^v7X#<8!}uTcw|Kp5aQMZ_u6O|-U889E zF?C^%lRTvUnF&Up%Aj}DI7{K(cwF?h2`275#2MDg!jY%*k>09Rpgi~-E18|a-xdwg zfXQWSu<}%zx$7e+`Q8>z?fSu5?id3TWc$GT-vn6nCx*&wSkJOqqajCPBKxcGDF*Wr z_$6P0j~!A2WGwDwwngK+`P;!QVk`br7K(KX&I3IC0hfrG==R3v>~Ll`{hAq1H&>{VrBzWh3hqc-s4a;}_A;S2HxL|xG``jJl7SHSa1twqdx%YW+_T)2mj*lgc z6tz%|#rvtjkz5citf2q(HDj^%m(C&fSTtt`|F56{YX=(Pud7w@v#VZs-Vea5ttB9> zq8s1-&;h~u7Fa=A2D0yKW1o+c+1dT6LRP%rnAmFyoPG~G(M29#dFg`H#S9re@m%iq z)T!V${xQ`Rt!6{k{lkCk#oXiBvCu5DpFOU*ldtej0`LEF5+2kB!Ige9cviI%PM*!8 z^|msi%|mVM#ES{EY2sMgdqvjrwyiS#T5kx)=3WLX_X$|@nu6t$&OdZ%nJb)9x{cL& z(V%#L9Su_XOs{``$Zn66p@vJu|EseQZ={WA+3T@5ergD8-ggqd8=Pl1>>b5Elbj*i z<8c@xSqYKi3GwV}`5im~Kc(sa771Iv?WcDH=4`H*H5IjBl*n?f1AGlNhC5eu*wp-a zuq|5#Hr8v1-FnT~*=x7>y;;F&tVz#DB!T&(iCECvNN0Jb`#fe{PJe`;&!4cutMOQQ%M`HTSF??YmBPJWV&K7v3;5~!Sgh*a0gK89 z+4-v-=*d|T?9B0p*?02k)Zg+L^eS({)3gL^)dFk&Z`(SuCn$!W7GVL&){*#pQWL)@ z%?ekqJP&)%@v%&wBt1}f2Le4W(jDsSz)~j#cO9G!sU`?R+>Na*AykzYHsyD&aYcq;SaiTI``Q1pUht=va+u_-c%{sP^XweR5G2 zdh4}tNkAw)RN%uexG9eh7v5yGm+*0{*GHk+q5&}z&WJ8GIt+_f#e-H~JC(CaVo%Lz zgZRTEIObLvE2^9)e*Z$Yd4drR4L#2v^*0#11#W`!LrJ(qIa@rVb;pNdc0j<4F_2tY zh1ax8iDdrCz-!$V;F4H^Z|<7K2ELyOJnemOy72_i@ZA)GgP?C!G%mmleBzRX2CRTC zU8aXGSPip&Q@)cTTT5Z%j4?R6{}ENz*?_&SdcvE;r)+6lB|ddBjXv3Xif{8vUpzis z0h1=3!Nm9&mE3uNUe3y8)$V1}YBwSD`CY&uCjh1YbiqdUC?xPQ`OCWv@T7t;&}y=T z%XP}wRp}t~m&o9N(Fxdi*)15n=#39Nn=AGb{b1RnF8HLFrIxxRN9aF45^nCS0iB7D z=yGx#PjfL4X1jgD9{mfceUp`*Ufj%Ov_q7&fj zN_NtjOx%3-1G{Ut2KaW)$02q{=z%TDAT+oPYm$cG(LfoNbRGkj1FQIl!=mv!hwCuK zNFa<6I$@O;qj2?y$#l^BjS$_O3_91xzmcURKQ96r|T#tX(IGA{Vfz3yalHMNmf@=4nLff1P_rKV z&-nsVvQ6+bbci*Jse&x$A(}eThW-q=gbT&b)TXBkX@@aV&ZwC$u1q;+UvymK|k_PrL; z^WjX(wR|?@L>`!R3ZO=59G;`s$nST##-1O4gI-ivj2pxZ$=O%8;?BsuaNx~U>?4(i zr~mC>@7z~`@Q-4bboo)>e{W!;qUXbcA~9VlA{YOh7Xy!aPQpX)NBp2y$JyQV2|b49 z;fbGOu;-s*y2o&q5EUvw{^O;rMaX$bm$YURrmw@(Qyj3xN+Y~_h9moN-d^mbagRMw zH(K;o#zTQ=aAH1k#jRmto zX&ggK231*$mZ{XJq?|vzmxr6}kI`e#Tkyu}P6)ad3IY1TSaix2R3+W%k5D0Wm|*tQ zayu+OxQE+r@?cnanBBi#Ml>^IGV_~fjR`9OPIQc)Bqm6)q`^V)ti9;=Y7c{G}J= z@J{<{aF%;TmJ1`nom~xCoj0)By0vt-y90dq8~~Fmmb2Es_rQd93vk|)fYZIc-~$z_ z+00wDxOihatAA_{D1S}DUek-fWrsdH%3>O;YxoW)k3Y}O&F!StdM_zkeHWft&Bd!w zJJ{tJ;GRk0_zzE#wwDu`w~eKNWBXy(FAaF-I2ujY;Im`Q z`{;vh>+lypSG+BEJ(F$n6nlH5DY@cI%_@iHp`yQ{Tg%n4^QkSKh}5 z_88%Y<4yc@DY0MBT?04YYsL{WUJy0v7CwH_0NX7*PYsui;JOh9+*_Q)KAMmUcJ4Fr zV-_jp({=mTAJWQfjGiUU;BO&7h{$6isM%syY6 z#uhDk$Xp)eQx(n$Zk0%5=T2(~_?`zbKUs*&oPr-}9gf4dV$036f_>rbQs86;g z@8%xGdUhv3ohX36M=bS|(6T&5ULX&%GHO!vjJBAq;jOhzh3{g2?d!iCd}SFm*sOdE z)Y8Y{bNjy0Q9}$2w4cDo``0J6&o?8}x4_~^+*2&MsOI98LLHJp!r~Dkxb(FCbyiB8a|FeVh zG#d6Ar^3Lqf3#QcvCw2>KK{398Q9G(f&TpyM3*1`qy9WeQBOxbotd40Q$yzB46{G{ zuJCerCOm<6elUYrcU$~UI}1|ItCP9UBgA#g7rQ>w0r#u+*yjFDnh5_<*`tebTzxIg zSTfo&EP`VX{N4@s$v*bBhAp1oq>ANhX9_a|JFu~%G)`#k0-b+3}8!eOCl>I;UX0w%FD8(;F*}Jx7nLD6+e!yrlg* zLqKn08kp3X!}8u5-hK_tU)4Sj|FYf*2|+vXQ14-QwQUq#@-`QyIq&4BTU?<_8?Ljm z>FjGev!w_;r%wivC#Y$hJD2+>m+0oBHcrq5$#+R^9o_68P@)(%i z@DS8AV?ZxyJGfTJz<;i;^lG0uD77Q3=9Yq&-i`vfwfTH5`XNsKIT5h8Gk@oehwReo zG^qYy2aI+Gn?F?sUpjS%eX@5u>=C^@#m-8}Vs&j(;M5@v>h)(i`_6O>-1D)3 z&OIp*CEJX5?g*yE{0jEhU?Ok~-e&Fvjrumk#boChyB+HkGlGpuY zSsPg>3Q&WpEg@{ChCIIO)J}%WwP9t60Dhb?gSC5OK)Sb-+jw&W8~!l`GVd=Gtw~L$ z5+5+$F!mmvIja;$mL-YZwpQfJmjoPg`w+Ogt)kZtTH>-BPV7GCgRrfv4ps-O#w!z# zA~PJxHddR#{c1n>!AXK%&UNp8o}b0183p#oK!aX;J*`=1rkel!vTXfljaZD6WZZ)KLzBR&i<4?lt*J5e8i8g#FE5Vz-9D|X) zpYWaEzIda>KAf0u$M(%SPm0R7VPz|^_cK@v4z*jeeq$6wkBXf5hgRp~jdfFS(;~#L zNZgCxEGq}$sx%nV?!~uL28H%gJ=Fg6c-XXIm`!j}v3#Uj26oHRF(W4ey$QRZqj)1M zYpAEiyc~>OQ{ZQ{qG&`@TDbn+XBd~Q%0|zy$9}~eRBuGsd+7??=oyK-&MHCworP>+ zI>4VfRqXA>3-R67Aw2tTA$#X3!UOev&}lGQ1TnGr!h$4r@wrg`M@4zjo@>wWq1Fk) z4Y3@5gE)WrdJ@KgogAFb|G?(yMq;bgB8XK9065}_U!BwE&kGF4^5Qvg(2m4?;F$AX3AKQSLU3IB4d z#Hz10Sl+l?jdNt&SafX_TM6!2DT zgJn${@twC$@W9Xtyiat}oG48wD3E7MT-L*x2@?2pTQ|EL`C-{(R@i%@9;7UA10w#- zt-s(5GWyjR=a`5V1lrTvy5icb_Kg1g-bUxly9V1CCvbIsNU>feYqfBUNUFgMZ@utF zIB3*_e{7NkhwD$l!Z(sKEB)~yUn%M-w-tYod@Pod{oxbEt9<8pUHD1PQjK>@_)WIr zbL^@tT*3aOjoK#o{gXQS#c?XQcs5g04+W9UhG&SMHibRat%V=DkFbB+YT1~dPWXVS z6BOlVV5LA0wo%6p4=FsO@82)5oRVpV53bUMqWEbr&-4JBc+`>2{ai%-Ul`IW|1G8R ztCNA(eI1)$9}QrQZ+VUnFGaNIwL} zt&7Cj&dt_iLyRSj_~Z#C-jj1YRpdk1c`bbbcatUttmeu zEVyz3hxjqaM)J{JS$jPBszCEwYT&9Zo`b1mG;4F~lBS@uKDEZVrp1MVz$2JwiE zy@)$u_?iSXE#*Odo)!M@Z5V#iJYDQpGUeBsCE)O08C>5HDzr4bjMv?95nf3>Lm%8! zr>P1tkeI0h`c1O1IHUzeorz_4TQ$oK{E=Tk+vZIQ))lz)ZZOtGK^4K4Wb4g=hK@F3sLFAFPuxr+6*)p3u+S!jFz znttw!WXE`^vnTf%;<&82uyaQxdl+@o{>pjSL!zD?)R;<}oKMjkjz8hk^2xYp{1;fi zGl|{gWlak|nBya|YsF`IZ`iNe`@yef1?Ws0M_zf(!XC_8c6+Nik#-yf&!^r;OBgKr za_hLT(UZlumbVLg-hM_?FI}9C8=Lu1sGb6AvA75M7K?Wl*04q`ovdwZ3|4wI z2N!QPgq3wc!X^6Saom9kpz$;p1ibH1UObG>JRXgi+uyjkhju`D@EEL0PEZa16lj&1 zjwPc5*afeQEU!+MV7=A%;44E)xJz8C{=G)+|(*+|8bn5C(Z2Z(zIIw>k4Kvcgtk9VMYW`I`&}apD z$=LFqsgM;WIbdh(%?7&NXK(J+5Gh}}1k=}P;!Hbh7|@gw9Vu+W5mZVPve5)bUUh_Q z`69e9WF9^%-rsKQJ&d1bH?g-}6h#h;im@Y;hex|~VBVz5n6Hq{+Do}ooOce6%u_}O zrKUrT@ojM~SKRA1&B7%^WAWivOQ5389e*n0*uf7+Xu=_Va4V^#!)b+hw`vfc@hz48 z>am$hOc&>IMo+=64x@3Le2sPi^z$}%mSg9otR`1io zV5*EebK<$rPZz=h#uf+B^+NfTQ=r-JGXI#mAH+nxpo8HxsPuI`4oRq>(w~f=P(B}q z#a=|KJWI>B`Rbx^H`3U&SShDSIPQ2*s9KW&ex7QU^^&74{ZrlG@4_{d z`L_?gs#UYQuEfHrda)yIXTH$*<5K90OvPh1xiBMaW<90zo+;y`adAS6 z>L_TMr-!4=Zqkv|a{TatvSm%fe*A6O3*qly18}g;3EECSVn?z!!i|@pJl%Z7D# z={~Ddunak4J4O;me4B*Bw>$)&-S=>u_)dvk&QXwCqJS^>{$bxF_pohkHTb^66}-FO z!1Ch9c41@ePf!kjghW@QK!$f2W-MvI8-}YOD5!wt%hbWDhC;UEydz{LcC%8dUxk-; zzk~G&z{+-C7vj$t9bF`x-6E_YJrTVk*Kc}a-0 z34^=bC-Cp?rsl@a=#^AI=%Z~^y?7efRM-QGR$-<47sCVD+w?lwhEGV3!DHSwz;VPE zuRX`u&0^nE)tYK(oEZdx2h7>|V+Pq#=kAI7?i0A%@hz@d*e$%cRL$~C*HySTT@z+r z9*6bDY0%PN8>rvt1>iDeAHQf#F`hYh8>}6fC3ZceW5*}!!S2grw(DgE-#5S-YBSD3 zq8Wqt9Dau;4L>mg;8dr_N~hJj?W*gF_MqpH!4_8{4pE<(+Pnw zo~LnIO)_@;^$6$Bc+S3;q4+|{bCB_NV`n7p!?{7PSx{HN?Wd&J8wXU#mm4AcF#);6 z`*twjer`0r|F9PO%Rc5u$W$KaG87n(5C2oEM@!PDUH>=Z#M zJNM*9oLkaL&+gm?lbmK_b(!OI>boEO3D=Jc&u`cRdld}X({*KVEIj~!Y&--1er|%B zrsvtKiiE#pTLx|pE5yH&SFw%G32-rb5~N-H1ja8l*ifgR?6eQga63%~h7~Uf$9Jz{ z_2xEG{}ngz+44*nUipI#Noe5|<#Es&v=$$=mBdws(om?Frw z1Kql*fScbf#s2pDOtX}okjIe zjkgR(vHYKd2jTO4ArxdQ;pm0pSwvhttF<#%7_GJfYB$B+0@Bc1|9U*Q& z_qWB5f!F#oW3Bu+P-$gb$T!@fURfk=)RK1u^jhXqu=`2;AhRTWugDuRP<6f3{>I<$|mhT!WD zaNoN-co$b7th1SlXZB1KXEb84^O|*dcliVy+c*U;KXHj2<>MeM6S>2ZBNe#c>MrOT zE`it=Wi+_j2|kJExbMZYFb}(EDt+V*J5Rh1iY^YuJFb5f{cj4;@Q8oDJm0vDeKSUYP|(8 zqwW#bp03E=4r-v=AF%MMVL8q_=?T{&FX7>DKWKl?8=UR(m2P|B3_0JWKxMFsUdM$j zJ?em$&Fd4s*& z;L|BrJabYVJULA0kjxvQO@M`ENsu{qJ@%UI`_YFD2WH@NV+(N4kH>Uj*aRH!GJ)Nr z`-h$IaV;D))`aki-w>Hr= zGk}##E5%&NE>g}-#Q(LNBE`Ds|O zDV>JeR^m#rCo(3d5V~Wd@YaDQNDQ5c`xD0Ur%12I!`IehowSAgy5$Zq_pBy3^k*Pj zibQFI+{$T!{AFg`?Y5(Ml zRN8Lglj8dPZ%7hvs{X{KXtlDFw>=T|&rZk6??dr_)6CdTHV^Z()9A5P(?y+nuc@K> zRdH6k1goJ*c&^xa(;&>o{8{QO9G-;B^nc=%S?8#eQ3Z}_>0&$IH$kAcH5T4@Ku@ph zr4Rb^!MRckSH+&EIrXcsU78PEOOC{mhuWY>--o^AbBE0=0o3?5!%z{4gPPNX>p1S4lVTuso@e~ zWoj;enp`~$>Qq422^Gsqj{)JR%1YtYOX85sIvi3690qQrMD?5m*% zGG!6U`*d))_0<4>$?#;OL??q%kECxasEA?eETR}Z8t+$eYFU$`aZ1$ zwNRhl+w+Q5DJ@`cZ*+tk&zJDW*>}>d88`8J%Sce^-A~_UZG^VOt8{tPBpRh912Y{u zu-x5jP+R_t?l^Z2$M5>ehDy%FgtL`_}Ry!6apPPaGMjaO}u|ERGfBN7L$@kc< z4HMwE-&;E8h&^?X@Wqi*3UF`55l9@xz^;m8V6dkTTV*@*iuKnsNn9eQ`YxGkE4#&0 z2U#XSel&Nt)r324U(We1XyZQN$IP=6kD27(p>@HYyDf&)R0K7bqIrvYV~GCzNOE{j zB))HkGw7trsoHptS`_+ii}5UYVow`UjXR{}LJg$6W3~PC3^Ww+r1K+=?z*Zy`5t zdy$>@($JTZBvh!FOezY}$nQ;exyy_1bFasBG6jaOn2UvBf1E~{r&WG}Os1*c78#D!uNP@Z-hrm|?m`!8 z3sGCe5#;^k2(g%1M6zC8Vag9EBGZ}bGUhS@IgnkuXY=9!`n$;zAx#TkW0q9rxTeA=_o!u2ZbwbARz@Ch@y=XYM8kd z!O=O$OU%J{dcKlO-sMOv2lo4{1g?L>F4mUGRP0bF*#Mn*m|o>9D$N1mL? zB*nRzNW&=)?SD0eXS7@(NQ+Wr)>f%A-*op7$$9=HynQE{Rk{~NZd5`g#}v@q(JI79 z%mcq_?1sj+2BSQ$gXEH879FIY3r%r-N^jzAQZvjJTm zT1UiHjx;qfNNFM;ZLVWUZ0ch2MY>Y(%F#)%WLSph=5A@h734CrLkQ2p;!mCb$}4rF zVslY_aR!o|bC`rTXOXvLl&{|JsveZ+gd ztb+Gv+Cp^3!xOzu@+VnS1IP~}8#LkCVkBv8MGR|g$TaumM1GG4nQF2FrKRmc*mgPb zP{pKnHH!+(tx%!lQ_ke)9qxPR9p=J~dIrz7=4GD_U~Vy~T(NXIm#Z&D7HQ2Q$)To5 zW@J9vm+#E&)7i}Zow%Mk5V3>_AE!Z9KAcRV-De|TJ!5oiMm*zc7RwCS#Bfd837k^A zhuE*M4VCI`BiGy9Nr3@IYPlB3!PkPQX|QB{*hSvg2ZOwb(Vc?j7or4_wk||JWF?XD zutOxz5&1pZhVr$YQC0SO^5FDF^1h9SqU%h=44VbS-G?C^Q3@nuk1Q#gAcq=a6p_b} zh)YbU6g#NO8HpD!7=idc(ynM*Jeh_u#IG zUYakxby*t zi4H2ep-L3SNt4kA@~CA*15JC8%8U}GGtnk#+zdf1xBFNonHiNv`W(~H(d-P=?bgq< zjQnF>K9wVz&;Jom3)UdtuME2MgCQSIEg@#D+sQ4bb>!Hj4d{2M_;YizM>BeCP^^tD zx%|kUIQc|gHEUffaDCdyo95)qo4z9&S?>u$>n`jkCH9fz`I>HSMC_OQHR~F)_Ifk( z^llc(zMMkdeN9FYD>Kktei-w>Vk1)>>&vxAcyb3%grbEuAqcMRCE4!bB>%P#a^be1 z38tG#{8SI(Hd&KQZB`_e2UJkbdpR`r?h>-(l{q;&Y=(Sa@lpBgRz`Jy3v)R9CRab^ z5qEP$4PC93M)mvU$b)1>;u#-97CqTVl=%CQWJnY`{a-3NIXzD7GK(SkV%apu-KOgPUw?IdpZbEmrjYm7IK5(myrx3Zf%H-{^BC%p+P^;Axw7wz> zDP?6KtLhArpPEfZON5ZYo(SU9v=2E;#31LMIFzFkkB0Zf5!#qYj9&LLia#f#5!;z0 zAW)mM9R17vGugz*@9W@-tW>$E@A*i~=RgB)nWX1eF1fZdl2Ef$ay}sf=?$czHP1L^ z$y*~Nd1C=dbeK=1KHlV(rrzMzcwJ*^imoyGWxnXZ^fdHsTN?T0oZTZUh5?I;_5o}- z5Ex#U7KAO^C+P4B6g>K`j2mva#*FqTVhXG?i5-b0pYx*8)KiIQ=w>-L_vsu?^3gC) zy4!(yw@3UvTlyfK4c_Edq7U)ZKgpHXW^!pibC_#Oa+vf69h4w51+_k(MsAE_8T97`b)l5Yhk8%FK4Z!wf!X;YNMF!R0KN%r);Q=Sgnu63BfU z%RA;8N%m-@kV1`gBq~Tkzi=NjVY418vCtEH^e-+dWJr=0m@)x|Dbx&Ye`IDz(Hkvu=9>YwGw_$X|V%twWJWgi0 z6I0wXgV8-VoqPS=j_dbNCPzQUl56>~=yzoz5+QFAa%2l}=5Izv6g<%jvl&cP_yp#! zXe_tOX;AP`Em@$Uc1GYFS;|vfw1BrzwuI48sbbU)oa0h!ia8TM52AI^pA=T^LeIwR zMuX>{Fy31qFe%z?+`51kuEkc&ELBeD!Y;%zkdecbiT$JZuc?xaB30yFJ`QcE8Rm32 z2_j)3iM+D^G0)}yadzY<*KPKNnS1FQV>ZVI{a(BjNf$07A8*@}G>>yM53Qe)z>_-dqWlp^~ypk>hjQt(t1=Gwh9%` zT}4`**OBa+qr8*fLwK6)NHBH93c=iaE71;-DXI|8A#chp$nLC_XxY&vXe+TIe`IXP z+H-}((>!n$o{RH5&S~iV^91x*yl9`9pF)b&zHyZ*Z@K7g?-?aw7vpR2O;E5? zl^gqBiSc!N!cz*~%RJi6XXKCbxa-44+^6WxD02En)aJjDJY2DvyqKPgIDQto-jG4= zir1E&(r1i>n7{nuX&x>xyEc@zUenJr!69px_vPEs})RIf)0=k^$eMy%93^=Ml{I^-Ix}E%2w=0t5=4S z#$VxN#v?hhtU!WX6i6U%J4saJK8@7!W|2K%hG`i5jw z_{jhn+F_=CyO1j#!?&;=G$slgHOcAFHlFdaBu1P5W>R?ZoN2~vl&N)?37ce2zUxW} zjz7rc9{#;4_#q+wJnd(3%XABw=NA0BD02h!;SftsG_a^*qXBCDY)o#C%0P!4Ldd@@ z$>i|5SB#OZJ@P2nLOz<Y8#n=$Cwvk;^@W-JNtVNjLKMlzw7Pwv4yuC3=zNDk#k?! zA!tl5XSUmZ5r2PrNbA}!X86nmlDs~avoSf$wbzQc1=B{Dw8l)X$191^Pk+WZAKXG> z3dDPzG=_ZNGL!Va4ksQW6SS~qEvabQOxo|ol2h9qQB-&Y`6CP@8Cm0zocaKF{nA7f zb#OMiR_K93?X1zat-m<8GBxtr&jbz1TcH;!J>2RiGsusB?r5sO5ou!{(M?@SekX+^ zt?8$dR2`9nPE`No zavVdr0esgYQWR!Uopzgdbnq4PI8L2_huF`_ws66<7HEh53X-6(iY$7YPNshv#hYxi zfPDNoiu8Mj6475}q*nu|8h{LJ*a z$0Dig(}`2YRup?T5G9Wae3Gb#pdfATrP1iwh#Jn8Y_lwDtnU>^PJB!|mYu7_f6SBt1oP6`k=f)VE?CSpa;obXv`KbBBAa}axZU2Z}hhy zyS>}VhB4a-@8N)8!2gUucgGu^v%+IuL8l~Bd-ON2PU@GyW%#u~SoVf_vb&vm{i~JJ zdG(fatQK-2=bK#mqh?0JwTbC|s>i+jrOyqeT8QTw=3>`l0$HyUO@f=mHLyM&d7p4X z?(VA)9A8bmK06Vq<}6g{mx}!R#lPLSOj6Y!gRYhwM3=(D$d-<1a`>e!St{-zet&jA zypT0WNiLd5ygEb<6i1*Kvlz6kEshuVbBOo-?hnB^y_w)q!$CoollBLm(3P~g(`OL7k<5$AcvWO(mv@?XqMbYt>7^u2k6t8yRW=xaH&JzO5?9}Ga7)x6M+pB|*P z&!3E4r9j?Qjc_U&vPjB*G?Lx)f?MJDk&Bu3iJ2kshuQQWgI14UfR1}vkf2I)vE$rd zyl3zrbJIM~rawE;sy+uqe3v3>Xhq^bt{@q?E!?=CR_??5Ta13jE#}k2>FDejT_iZA zNscv6CIOm9Q015$l=>-~9RHU`YQ?UaSGNuelp9ldw%1qj@^j*mBP-rlvXNwAdn}QB zHJ#Mk&L`W@LUdsg59OT>NAhz+#lG86;-nQpf`9yAt{eVhVxot+ryYN}_ufa4|Kn_A z=ax+ZjvOW@_gWKPpA~UFZ;iV0Y|!jSJ5YVI8#3_SPHe~QB!5#D35EwB3a**F;f-HW zDdzFVk~3SwiH~VGYH*K1a))-3;Q!o6&S*EZsL2QQC=`%9>pbEcorlzah{Rks*D28zoGvXBsGr2o186=}Onb`kJMiztOXLa^= zZs+K|+$ZTkh920(c8vh>daJ3i(;ID z4sf4$J95_}Be*U3>D+`lsZ38*ER#Prf-HT$hdfxg2gT-uA!Q>8ri3FrO{;c6c#a&$ zm&xb$4n}j!Lk}@MSEHEH?;FUk0ef=H(H`YCIw0>PTdw<72v=7X%*+W~BlfuLN4v^{ zQI>oNL6i5Brxsg@>^5g&d(IWn*Y4=~y+|~CI}~Xi3ni1{BFLHDl4!B_PlgTu%$YV1 za%-)Xh|O(Taz#}Nl@3UuowFjjDKjHEt~rv?NRDF4{aD`P#Wof}yc>0E3-ttk>2EE( z9jF=q^s>6K6Mxq2tNkBE=i!g#_r`HsWQAlkMY1vy&wZYAo=RjTTatuCg;LR?l9iE7 z$tpw|ltk|Hob!~>(ALn@H#D@hRMPMM{Q+LQZs&F1*L9ui^M2oAPQT?m+wp{*Rr7&u zEt~-Y4X1$Znf{W@wpkFHnZ$idvWab|FXa-U2azNClzOn zWZ`YuGgP3@g@+xpF+V3&nINWbNe8x=$FU(H5F zUZ$CG(Q9TU+M3u2=iIUKRR?T)(;l23aRJ^1kPZ4Bz)q_7VT9`!GRGd|xT^#&D$<$3HZS zTx8jgUCa&X>&%?NTkKy|59aIaZg!VPH5(AT9lz&l=Lx!jV7M*<)Gj~6`w_pBsZ)t# zb&fc(SFZVEY&4L-G@_e&0^e|0x_8W=G(d;ApI@8Uuv(t3mdx5UlrgEq)d( zjTL@8W7~bNG8+%yWagK=X5%(b!2jh;1S%0apqgI-q=SpW@~?&X%!xyI`FtDjGY5f* zT70g4MtFnaTKs0`db}iM11Qnk2*%2=PPl1T`rl*PhNc6jmKuy!MK`nS=`Xfsb1%DBe+6#MOT%wfGQm;v43H?l zk)3m&uwFNAFx}-WH7qwQ5Dob&wbX)j?$E>^J9CnU2{`lp!)?teA6 z=muM|?+g2_GX)55CIZd1WtYeN(;o#+J4?zYGG9X#;mY46z$4ZqnL84}z~^AFR~mdsn7 zSb0ACf2c{hG`F&2-_-vYL5O#rWi2SL!RV(>=m05<dWx-OgEgf&K*2@;0yM=U4lI{R^a9PR)S-tYeCi5A>OJFhk5Z9!EK2u zWy*F6EoQ%t3$fI&*{)wH;%#dB|?)v@gvIc7` z^qY%6ms^0CL|b4~v6+dqNny_~! z0;4zp=1;W7ORqu9Z66>$IABK*dakDnal1JPeQFlaHGRScNM2CrDa z3`{j*>L)~Ct^a~>OWsQGR51(;d9KEn=dZ*AuLD8I)L?L-H4PU}Pr`eaBmo+d20T^v zV|nd;_;W-W_;EZFoV&CF5Bt004GuoQOVbNH-U)%S^jvU$@m$Qcme^NUlU+Z-iaqcS zF)Ih>F|`4mjFIvcX8cslTE1^*SI}rYJ9{k-+`SguIu-*GepIrr+w)nsRfm{k9}h9c zgNdMJYbHZ& z)XPSBn~fjvzQu}zy9*`3TW?t$vsnzA2s>EC@m;KfaR)P}w3Dd{O<{!w9;_Y1XZ%+% zjN*S`V9`}?5b@j}k5>-F^ImZ~94UK%<KJ>c4lU3iaGGIlr_$9{XXlHECd6{9-AnNfJu)TX%dRvW2p z;T?}W!Ta1jj;R5r%)vH8woOrqRaRSxsuZm(K-OM{8xm%*A!qPoezHODh7A0 z{J=-f_4MrKDsJE02M@|vg0yP};OSjGyqj-?KdhL^K6f)=8xGB7GL}0qlE0IHr_4_9 zzB3L-E=s`D>(s$au78i-KOWNq3fOP8ANab~4YZze!6C=>1#i++1X22^AZy_{^vBtE-sK;E|`O* z)>?A&W(!cAU;#{nRoHX`J$6HwHuLYN2D9P*Y9=n(lL>ko$X*=h!rGnOflUQl@t*U0 zL49&ExYE28I|r}Fm(E9lpO<66;l?>&^22H1z||v(;oiuJw?PVh( z2{XK96ZSeC2IL$zf%XOO+O$=Y*lS~fj4JmXAK<1S{Vv(-;4)~ovYY$Z^p82n2nM-x71c#qqYan6gskRjF(`$_ltqWr{%!3ObonhARR_u)v(Oq{h-7T6xw!FFUeSZOg0h?(wZ!#));iIK&+Kr)O0$;YD^L|Aje3HJU4x84j~pJCt^#BZhy$B$z_dM{3C^{e zVU_Ksn1AUcTPd9dp5M;E!x{TIC$=8$(3^~{v!sDysXDkLv9HZa=nS65JK@P+ys+oO zwcze8S9XtuJP>zs<2kEN!dEpcv4Ha(+B|Rt&AspFeST^dFS3VvqPNR7o=H%z19fOElB_xXMw)A8Tf~I4*umC$9ON$#Yj~TNR8J6 z|BQ7&_k}aIhSz@czWLv8{W9FohQ8W`*U}xpKXMP4Gh+^pYLo(R?zrH3+&H>V0dB+6P!MAxpa9aTkpMS|l z7pmdm>8&q~G=ng!Bxw;_xFmKJZy*O3>0I=ouB5YF6@>)m5u?*({j@@vEf$!|V zMh{tFe(WI|xZoFi(9aUfM$E&9zK#b~T@Vz%pMvEj3K^|HAsg{bksZ#K0e638g12fJ z`0R-^?0317SP$>#Zq9pWiA*f4#13pG+tTqlc_m85j35f zf>cX9kimsxbadx2{CRRPYWC;694d;iKyEQQ_v@>$VY?h%@$NZ)(*-%fgk9d0vn~k@ z?IY-(X$~+@U``FaMu;#vom8JmBagW}rlG-SxU(K1vv;;Ma`|{7&@`eZS{$G6-wt?6 z+!$G`e$THi--zy(XuiD;@gU#qS|^vZ$FoN zQI@6$D`Ux#J=yfmrd(mHYXhp__B8*zae&7su0?vi3^JTHh<2RQN9)c`pu0Ous63|~ zyy0hxCOl{(g<_n9*R^Fzt@lUI@ZjteMQltmJ6 zTB8eB1L4KV3A9zr70r@5i{f|OLLUzJQ19AISZ?5l_LQHZU-UC*-X#@OpOHy*S`*Qp z+Ay+b)=?BAqb=%plBT=-&eHv-#YDf&ddT&U<-*>(1$5c=R`O$_Dt$Ohg{&J~L6g?p zBBy6&z@gS&+R#%9eZ9kBCch0{kZhs$es(DMc|9t$(xtEB6HrLUVK`}E0?aCqq|>L> z&>n|TqLRtb8|HdwNpla%4M;*I_mh#g@ga1|L>B4kg$Yk|Rtb;H(-UpZia=^>?xM7o zZ1nPNsc>8W0J-jvK+3sXvBIv;DDG+=x)ZPwMgEo%*gHKZ8JC0Tq_AVKKIkVsC8ZaSXPen@i3{6rrAwEp(9MQGFg>it>*==I;tvO-o)G!2VrRQRS0r_1po{w{{Y7tB$DYXuBlD?&$|57PA9$+Wgh7k!emq?(dea3cId zu1CzEzsd~g?yqam`tQ{!;Nctc-o6RZhkJ?60Xb3reI5EuYYa9G-+{9u^2jHXQ)u-L zTiTx>ixhhv5@oR`ZDI56=-O~Gq{|;9-#;3mgAXsl*T8|+xT>OOUXrwR)(aAIYnteU z)DKo-rU-s~G@UlAmZDZ`o2iPjB?`QJp1;w296jQcO835>i)8Z5>9-RUgR>ORj>%(G zHq-?lu z{nlLgT9HL!>@evtx1%23iSVqN6^cu8Mjy82(w~)l8q(WAK5f+z5E*GSb4(c_c%EWO zIg&l;4g?!9b%{-*?> zP2_%(He}Z8(=|5wuqJvf6%XA@CcAbDjrta%zdAc<)y4^^GIR~?nJOvTysL=*n3s&6 z?VW-q7%QNZUyq2^%2jkINVtaRzNl0H~l%iXP~^Pd;l~kt8nTmO4dB zP?s)?s(Sttc5eL$RZo-<3F|3TIf3)?jN2ug>QF#u$i=~&bJvNfV4bkmaUrbUAc8}x zNz9>!XLO+A28>u|iPn5b=KMm1w93*69yL5ddM)6(-YAJoBb{N`=G$Pt(M>W@g5#CT2EQ%>nB9++cqHRa~equU4S} zGw4g#jb!zwQ8GG`3I8qaLB!_+tj&}YW&Ed4ue~fFi|1(3D#3Z$zH3z2nJ^Pg3UxwB z91FkO_XheL>p|t*w!z2)`>2P3B`IBQO+BBNP+i*)RIhv)xhKy@0aNxNq2@_4Z#Gx& zT#u$QfXhgpI0ldHY~^oVzZx0oBf81uG`e)=0_RXlN1J{Qp)oEynA+QjKEAq)`a@KgqJLN4MiH5cg^~e%6Z_ zG?3$IIFE^=n(fZiGTIGB{V5O%XEf1Xl`_7`kUd%0tVQ4ZKct2g0+P1Dfx^Oes`ly} zwSSq;@y^^}uJlnllG{yF?|-4OGC@~qHW@1!MXq&HXu^i$@c4-+sx;Y*ER36g zy0aD`Y0DjuQ9MqLtWrYcR{{AC#84&0qx8sL4S~@ES$^ERi|EV3+eCeSJ*@~1funQx zq6ky)Y`g~Vd#Q+QS05(N zMw0p6R-qKWJxpU}2B0YwdFW`D6|LRh0q5OQqge{>5Kulj@oqQgW~`^jE-RtBuW@ME z*6q*?jPi9$7o(ZMC3L!1I6YuxieAdCq}G3e=_b)T)O0YAel_moJF1N*;dW8TeE2x( znV!ghDK%GAw#SqN?^H&A{!p@}L59Zj3y|?y+1Y#lyYLKQYcxt>CJyP1uvEW|9V~_m#?XvGVrX^S3|8pZ-e2$|> z1ub;0^#C26Sc1-b?T5>n187wzp^~C#ICLVNHqPBfGnOcc_TMdn9~MtVycltjc2gxX zsjh_2@+;|7-hNd6f)y63XhG{F8ziy+AiWxo`17uok_GRT(Af_rG;E3ulAD=8g@R{7 z&zZ;Q{kva~#FrWv7s+Ku4cuwX?jH1?O(Pr>m!$U{7jQe@_vp!0rYQB-8Ki0~&A&78 zlF;U23+Yz15DI3EkbxzQWa{z`^w-XS{538Dey3}Romc@HxpbMOAdBUI9_MvqVbLrypF1R4RSg2o`f>u7F z@Vr?X>3kVRd%h*nyb5!n{9AYGJYyCrGX-?_=tq(*X+V3<%}2Rq>LTwOEy&uUoQ7_@ z3cbHn!sjcjXlq;n8lT+{KWnh`l;kYxJH8F091mK^;miGBO-2X0CJU7suhLyt&cOOA zH@L^r7G3J>gVSvk(4W{n{A4E;;fxce2PA8~v+GupHA2H)Q#otEgg(lx_z=;-n7^rC?rO41Ic4dJTj)}cA{$A}a9 zZ%ZmY5MD$z;<`9)WG&e;=!F^vny6z{5pwnzh8I4SqhlIJsPFa%aNco4(YOO0XvG-~ z4v96NKC8ch?v))ti52_M_K%-ok*gVH2IJ7J-$J;qtQL9wT7`}&Erv+z6I^VlB=8xE zLkaWMsEKJJZNGa8O}<<}y*w+3-qIZOYHGRgX0(AQI4A?F>P+VN8a4dBYevGO?lrV{ z=O2z2qfCQh&FPJ}`*8N`7`po74z$c~G7|R6(<^(WkXZFhdW_?zRfQM9^3#*)q4UOQ z*|@zl@_Zr8`nQxAYwm>yhN|fO8)9fmQUVG9XHipZ4>WiPX!Mgo;_aD5kF_u7xU#nq z@g5>k8!ro`JbRISr8v3dp8z%YnWLwQxnyC0wqWX-q&Y{>`^;S^?skqwsY;RIty$#hnq(w7Ydlru*n%-WzEmtTiDce7 zg93NFC1W@GggU1u3C7HyQlaigzDtKQO|Rjey|y(>XYqC%S1s4sK$~@!yR<_px7gdgh&ojy(~s}>!mh6(?BuH`ntk3BMHnTb+u#^{;{63?#r>e) zR$0kxUfp_%?@8iiS=i{Rx&6NslZ_wAQVLh08+=%r5; z)W(mY5-H>0y!vdqj_b?3pX;GIug_F_yC#$xQ%7~4t+d>u2c1z;5vc5JfKL?=v06I{ zqN2))xb;tJ+42p_OtmGsk+YC{-*mLy(u{ui-h>jS+oR^PUGP{kiz;+Cx74ileKep zpyMTx$m~xRsWYh(7N2$@Ok@c?^5YX6(VK(}Z>kEmHBJ$PT=YX3(JAco$ty^w>ln=I z3_^Q?F4HW1EtGR<2gwc}B2pPzXn2n?(hpq*=}8d{_*qB#i{%98mPDgTrzB~V+;e`e z{RebJ$&9p=&7^rcLU_FUBfNO~H^0P+kPMl2a`%`HP4L9%&HzObcm5L&?8~Jm#S4i- z@lE*Lr3=EzvDCO{2%flIFLd4&Ic5E)9_ue=WGv8tOT>mVbyJ&xcJfs=eOn>($(}B%pXixG*YA`RKww&geJw;n+ zhd;+QKAuJu4pkt@u}qrtxRrk2orNl12GP4Y-{G>S^P#Y4fE2B}hTL{&QzJGR9;}!r ze7NE{NpKqr~Y27Yb`*{eMzuAgO(v_P_hP%&G-u&sgbbd z?-@!??ne&wOXQk-X;8aWGcNa?+(j^(e!3lK70bRLD{)Ur~=QV8-F{JAiX$J zCwxir9uAPSKU+}Bt6n-;xKCKgW%zaarilb8y68uHDsh#Fqq8!WAcKes>f4=xt^|Fh z^8%CTQ%gs(Vdn;TyibAdGJXWNbIh8uyb|=N^E|?umV$+{#&9y)Nqc`<5wP5ycurD8 zbiE0|HP!rs34ckpmLyHz*G8u7%%LYki)oYde{gPM1N3cs3*YA*g|o!Q(;)N@Dvg&$ zBfFJF&s0=UrsY$T=rbEhd{7mwRaKx*8?*$^8WgB=mJ@aF<{UX8xya&29vZs27yccM zLvopyN$_7=`nXI+;KHOY$)HatXk9gE?NmUQ%@cu2S_F^;Y} zpob0>aGv)>ZPYE3KmzVol1il%;!>#s9Y6C#J>uO|Kt7OM$A`p1#ax6={zdsELO6Kv z1-XA#9ldS+3@6^4Nq&f|$x`Jpx~INXcu{jHYO3}{n_p$ni?Op2t6ze?%k4oMV>v%z z)PQifW-&^wO-J_w6VaXR2u*;obne5?wD&QK&2E^YP}OuQYKG{9kF(HWwlm%KH-|<= z#iMc@i_Tn%p=j3_nsO?FsyLWY;ly!>zf>OO*z`g%O%1A+w;6?0E+OJ?I#FoQX?h@f z6uMm7gDkYCqw%rx(VQ}fZX_=zg^CVv-;eoJ=v9G|cXQmPw-&_Bh7sPB9%7JKDVz{+ zo9@1=PCj}~N4-6Bkeh!r#4&Yfif=zzXDcn()oLWr3;#@i zM4^0xsJ3b5Dp>8Z3=AyWOOP)c+_s3n-8S4ig%^q_N1N5} zK8ljI&8DZhck~tmhA#Q9osL_A(1#T|q~%lr`gX1qwdCs%hnUOcc*h;&xYv~?WXHk> zzS+p?`v^(B@|i4@&w$%cEkxS_TjA^FDNs6jAspI19^KPp&;V}`PVtXG(PpxOxm`iX z6AYp&OHWbV-M9IHdE9I?azJ=Dxf6NFC<%r{{iOTYWol?~h~8XlEn4#HDBWnc91SEy zqA~Rh6tyg!A2Z8`{M@>Xz8{^7HaT$I;+@;j-Tj69sjd6rPM!??S5|-mdaY=&T@59? zv*gGZC2HR-$9cIe;M>)9RQKB+q8d8{i)`rZK^taLrUEamgYgut<%MEX$#8;;V+-!7*={-7n zR1A&3r6Rf$l_3nittd#mQ%=twHWfY@4u>Z`&7$UC{z4lMS%LGD614eAD%sT&L>Giu zAz^113T-+=rtXxXf$tO0`NT!2XWKI{=ScR$X4vltg-P%F>A&9*@aJb8GE{d)NU9c(?_N@`XN!ClHlpJQ)v0sN+f<|HHyEE$fA=g$u8~rXmEljGLNr7?2pIr znN1;y9L^%cX%jK|y@@Vc`G`DijYh??v1p6uHhxXA1RZ_hLnY@{A#MAkNM89D-+Z2- zsP;+@a*H`c*AyIrtE2^JW`GMSyj0Gy?k*7*aVaGFbeS$RkQE#bJ1?}&|3nUO^|IRt zg$JFvrL3V2QnD-%&hb174_tHL&vuj{=iw%P@WLDXHQVCQUBOwDuDgpC9)5_#1BK9U zu%D#g@Su}t^U!e9dEwKxjr4l?Jm^}!3vCYRq9D%@4yRm(4THVV&)kPzGnNt*mqk%+ z?wS4Pn~KyM&FNmPZhq&5csjLX6;fEf7ZygX5=9JOMi)(A(S~~yAT=H&Yaf}>zj_ja z$RuC-@IWYezi5z_-gyTXJ~~7d5qEB^&!EFixn#@a(`2t&0V!A4MK!vMsppo-=;80* z{E_M$;{Uy$?;wAgmab1j2l^Ap!{l)KJ z$Nij5ILZ}nqcWFLY0V&}!QM}WI{6{AixmoEAFoE=>MT)A%v{l#b}d2B)^?iDe4&D? zZgk%Z6L@^_B!RwTEQN%DC;?{#PA1-i zN`ga?hoP!YGgh_bJc#)faP&q3vKhWWjPq5wvwQ@;+-gC^+;;M-Ry5NEOSEXqh9h+5 z=W5iXVMN8Zeix3bX(w;KarfvIjWqE4Ns_bLAKf`H9#(AzRQ=-^oygCGXG^ruD`5@F zE>d-SiC5UI^OwF%{A?2;!Xx*k#&z)exq?*A6@w;ZKb&})VhiMY;Dw1(^roY7?6nx1+aU^`=6v5IP^M$p^snlTI zH!6E>G5YMTfO=QC)6d%@=5;78KRYJSQ}I*-tP3`N*%%+=3QbEI^Bt-@=_;=`{042PpV3UNBSs zp>XAZl4y3%IpP#Cjpi*aga$EMRN(cDpzYJ)jXHZ8cBP01dtxQg7|ei(&w?8@W2 zr^CfM5gcX60=YEpq`d8_Bs?sU=CX#geS$r*G_fIlS1!V-5;;`y*G?39Djg1N`atI7 z?jwteIS)^QJjA}2C~q2spH`nD$z`^1f><8a?q7hWl0)$O(aXXn>1W7c>;@t6PvCd) zM0#f=mKe>frw4rZ!enp=m2+oehb331zCR2L<~>12C(cGSgHobC(Hpo#Zh=s`=`cBM zdI-I(Y=%qwo(rY&-wQP>_9HO;AQwVtCt{_>LWRAvVNy*KIx=WL_vMtK-B~^Sl|Kzc z`?Gw>)3J>-_1Yz>HHW(=@zh4ornSIF8)T^7`fQ>wrwAEzr_<;67o*pDwWQ+BbK3d4 z0I42YhQ?f8!9AwBsBBvx)#5*c;VI>uk4s&6vXYxyrOrT!r>#(-yB&<*c#e*bnm`YE z*`Xf`ZFx&)%@!5>F-GadVF(#Ia@^ufq_=t#O}iUI6`})?cL1SZzdvKN>rT)`I*O?B zm#VOI!~-3Y*+bF9Dpbhr>YROegNCL88BJ=`{&I6KR2Y$H+-S{7i>c^Fc^(GjF96EFXvItd+d3J=FbC+irfl@*g(-}h#d*-J z9v}y4XO8!-$baV;K(#JP(_+&BC@MWhfL9|~b$SS%ecTTN{`{irixkn-rxA3O-CH>2 z*gL+bO{Gv-+now4)2W486GBiVUCyU58%ODq>wIY8l?zYicfjT&inLG84vidp!8bBDr`{F-wa@LOYl=Jh z^S3Rcca+YPtFD!Z+iOJ<(+*OFV$MAkF`arTxS_LYmuU0nSW2g+!sk}1lsBu7${jj_ z9*;37=2aHjxvzqr2(%?e=f9w)5eYi>vl~2!yM`L9%aH#$C#1S8iJE{HNY!Tr5_`kv z_7+-bUwtBV%2|Qj;!4PdnnZqOa2=X%{uV`^-2iW18Acg_JeqJokA5?hKphHVf(Vx} zvW>gCTl{rAJ-GY`QByNU15>^TE2bVnYtwR(U``53Td)~g+sdG(LLEBz`y{%qmW~=c z8|Vv{bYv4!M__I!YWJw)|McrXKX#WP4es5tto{OxRp=1jYBYgQkBp*A#XV@@8it;9 zxgy-~O9vG^+Yj>}2XS*iNu>WQ7b+@MBNq=tVd^hOs$0_n$AWz6qP$!>eeO{>O?Nx> z$i7QrY9;82{C;F%h0u}*9m3Jyl#I15hN(9B=w;guq^A~6@+Wq~is3qP!GX}-8+gEd zVNn}d;l#EXonsGbsQ*N8icWG?@&H*nM=c1N{ zA&zjc}MoHq^33RVSI&aMH4PJh}W*XaN~RugQBnhS2G#It%gShk`?3~%*Y zjDL15#93a~*uOjN@kegGVbJS6(`+^!RCVrScDvdFu@P1F=;Wo$XPf!Je$GuM|MN{I z)XfwG<@PbWOLibtT!J~*KgxQx$FOZ*4zPKlp^U?grJ!k}KX}rMnN?Gzz^}$tV4a~G zPWZB$d7*KWQCt6wRVaGN+Lk%t0mT%?>TM~zNS(2*V{JjbvmyAVLYQT%WWlG3U?3ye z!l-9y0s}Q!ysPdrbGjxNONyMp=FX(Hc-gD$+1jCbHdb?RcVH)Ll;Q;r9Bg3*`hCHd z?o(~DMpMD7-Sfa^v5SmKwFy|^qz)S2Pvf4^8ZdRwMo{+36_{n6Zu_!Dh2wN60sH>9 zY>B31Tgd`teBMG2l!m7;xs58!zAR4wG^N3o_^n`l#7E}tt@QIMw~WB0caC7%BuN~U zB#u{^2jkx#J#g6b1}2irJ@nVE2Y-f^gBFglAYsMrGcfX4vD+NqjB96?*3Q8$A2ONX zXSdi*9Wy|JWCvTRDbMYY6GkrY9M4bM1HX*Z0<&+d#c|9-=H-|;*nB7$pi&<|SBJ7@ z{l6H@v&Jzk5={Nr2KL*oFsvE18q2Tt$49#eyJ(3Vcz8z+T&&GzKKPZjQA25!v)_vB{TLV87PhlLj>li=2Be2jA@D9nF;1yD-)jm95x^5Z1V>*bcKxm{Q<^FO^>;~%o3>CD1#?KOBo^+ zz~#adfY0~i?3^LC?bW|VX3gkf))6JKSzCO->o#*hzs$n(GJi5F`KipA%!%N&)*5hR z|6@ivzJwV+-VQ|gxr0IeLacgo685^wnMjv!z{b-c+x}A*f9i0>W#yhY`*jbKQW%6m zX4m<>y-Li!cuCycFc+`mauY?@*5KtYa@fMlciOg;-ejt@t}z2mQuxmtbsSm9^?l6& zm{nH7>ppq_k15OX`4wyMYq48x4Il0Cj(ArrOj(Ji-&@A|m-Vv6?a$lFSLm>^NC|9P zv4j`peXTyPpon zU~yvPviC5roi^iF&yrb%U6`%@qQ~6yImVV;nuV2OTi6@2Ynj?=IRH(3!AIK@OwOUD z_*07;_)x6RBy4iTMO8Jd(fLcvvnFwDwA&X~PSWJKF|jyKGnow;)MhiM-eYcLih;R) zqixq4ba8_P#2;d^7>n`t!1ug7Yv_>8yX2b8o^F?9wyDbE@Qz$wl*k>Bn|IhBU!SqU zdVSsr*(;3H#}~FIBEIpqplr6N#~myRQ3HRDi19MNzGpT{PvdDdIbR$uyfQI-=J3J`;k@52JRXE z;B0<7uDxQkir=$KB(woIIT1X?<}5hqjYrI+cyEp+G0zNUV%a+#tdsCBV>GFcUDCFu z?Nq^eMgsJ)cIgI8{On4`pjiNx-!lT%e_VmdmKk6w$8fqgrUq<^#X*ttG2ZJyaWH34 z6%=O{G3F<3@f4e%@(}a|FO-$RujFZ9OLRAz?K2%ag;z0m)J`)Ri`&_?oATHyM!|OZ zok-SpT|b*ME$#dwum6~~mFfWcY{P4}6J{VP2-wRsFe~qAfPw!YFJ7dK_rJZz>|@&5 z?p>3BZ?hSDcwDS)*t>S7YguJmL5nni3cXD3LQCAPD#02?Vb*_j1yhq6gXc79fo-CL z>`2#gEO%rRCS%>~q_rB@wxNi5xc)cG+%shK!XCF(A2!7L*$csHV8&&Nx$*bCf6S!i z-*~59O~Dg&mBE5FW$dBSAB?TlKQ`FAlyTzDn!$jft=@|pZCCj`y!6yT)>>8!>pgP- zSNs+5lL?7T=$snnL&ksX8DmfO)D967{Anq8TBU)POPJ#l7BH{BNwk^#-p}etq_oAE z7+}ReSxk0+3iFnq%3Jrv7if+|fqOrcm`~wPnYW{sp!tacJH^h+c2k%<&ina*DV<)& z$nUOXdux@j>FzL$)vvG|>q&VY&%A-<^)7AY?ceg96-4}C9b~##zVSuY#!bEL z{wanW|uso&^5%WCvcnTLGL-FvpHRTX}1w-0&lv7<|yyh`no<$6VEi?1U2%SnQH@ zTfL7iaELVp7ghw|n&{b}%)Ond@1BeU>2DUa6tm@z%>Z?>16Jd0cnK4uz{*;Ae71Hw zyC#ppzZ{?Q+@6X7gOEFIg-)xPBdIEF?y@Tv@dtb12{vXElv<72S;>je%6xjaV z?t{zCv_QdEX?%aGI=09hU=DtL&MtPCi5sgHwt3!m1fRBNup0^){O`Xsrhl#= zCre*pi^MK7kK_edJpLuq`jfjK$O*uTpI@_z77LiGbrbQcCR=Q}#*;UT%L6%VKgXP2 z?}x`-wFVXK-`UZ-EIajJ6qpa*+qNA&&3eCk$CxK-fxBmXK}NIyzlH)lP~^;>RoVy! zS~udbFdzK;YAAS6Rm}?dCak>i3#PA4mtC=221NPK#%&#ZEJ8N8d!ZM;1*YQRpMJpX z;4t&AeLgNOywAL8cLINcCIQ{iOUy2D4-oIj$JLJEc-ew^Jk?1HS|7dYWSSN!g8J?| zZRG4})=|?Nm}J>83)>ukbVw&p=S?VHYU_aCv)`G1MHMiybq{ajZwo*~;dsbZ5m=Vp zW9o_?u*ZTzL0yy#*6j+yLuabbFaNXz-{G>_&kOpPx||i*v%n3X=g!<1ngF({B(lOs z`ru^sc&0Co%fuM3!@s4yz-;k1Y?}9H_R30k@OksIwz>L4jMoJ*ET{XOO^kN}savnJ z+N-^>XxJM}-{=8C^6Ocrr*4=%qKj>z&MGztvbc=NR*O%37`lLHN=E3oc*V$v9r!4W8OL;lHc< znelFxK(gb16rG7%jn5m#TeWCcQi-%DDoS-`&N~M!L`93O71^?6sgSi@r6lc9p#?4Y z+Um@lnUf_%QbY+Y_Prvz_?_P$aL)B!?=|nt{XFyBpL;wgUr_}*9vi@IJ8Fgc5cky=Zk@%qR$HG~^XX1`KW1!2yhj^wn2W1wTfJMHMWP{%l5^&vw9GkX+s1*gm z_u(e+#k@(Ri_L-|Kif*V(@E6y|c5L1{99}CVUIF z@=~vj$494>@POM&qMLUM=*k7=L{e=hqXz0le6$q;3qK6e2g9KX65mgge#n z0-vvchusDxy0A4*IDR&Aj_@O3nfFBe-{=(ZW9Cs_j7C3r(<%eu#!B$&*fp@Jr53kI zc|x;RU1Iog8t!*2!*(j6a9xWz;g*(yw73OOd-Wh_^)rQSp)y2zoCz`h;|F`4B)&GU zn)<)#4MP9TTOogV0r}CEg`Z}d!pn|MFezs-e%g4GcOb}}*FPdgm=H1+?0uU8<{W$v z#=M$R|KQbkkac`4=?FX{oY65F94wwk?(gfssy*()#yAD|`cefpJ=+L=YlyMS76j|k zeF@b<`1?7@?BeXXu)*4Z$ch#@k87U^6|&UH@YpFNNYjB#bW$c!I_Gi7-c97r!I3cM z)NS6jO}?Z^$q0(CAHZ|nSCctEMmT3gyTkdHYr)hiEqJKqI^f+ga?Yym0eU+S{1;LM z<_*Mv?zH8^+B_Xt=i9&(g>QKFz+>#cO^)mzDgpnxrojT4X;AUD8jQYH14JfPa81Tb zFw5lyR*+I6=X0Dv&o3=u<+FScqB(+`c%Tp0J<@cpKbFrM_3At?@7;J_@{kHF-q9J7teo=iSu!LT}_`;W}`s7#40&M*DB`i#)Qy+$(_ed$krjq8KdTjZNR~!oiZbu6?E-S=fGni{Cd08y20+J! zR{VL|NANPq4-EQR!;)2gxbhFgfAD>*G`$KJw~Qs8k8{E0`F(iZst%C!YAiIpbccsk z2^f}<@VN~$aDZ}F{kpf-(C6A~UKUU%GhZ9xwhfYcd*UGeBi>CEmIAQi{yyBb=?Zx8 z`7?Neg2}_LN@S_yJaV9W3wi&+3(uJO0B3%x#L-XnpwW+eLQSzW`EGO?KTeqff3sD* zy$7Zce%DyiEA0=jD6Iw?)fN-wwMp<+^b24g9VgtqbQ-kJ*$CDhHX+OMN0E2AJ4tZa zFo?~6iWTMMq0bbFUo>DQ&uNtFNnH(NjYksns2Beo7{DtxPk^JBjwe$L%t*Fi5bM5J0v9?df%u>L#35xe44xkk zHprWUoRp(@>dA6YDAASDSSVb)Zvm;yAF6-pXiaE=F}av*LC)ph<#~q1;I?alP?LP; z?YC4TpLdFYz0G!^d}gt*zvLq5Xj(#6kBq?y(yqj%vJb>a>gxi_SZJNm4L*lQkxw>b ziACoY;eT%&iFTw7tnT+GW^;CeuIf10fAKQ#4%v>+)~v=p1!6o0EOvgXs{wmDF93@f zBVpFu`QVzNIyp3Q2ClGCB|R!Vc=ShKoXUIW?2wdU_trPzUABKMgY@Q`a z`sqVb|MUVYyji$k`=YR0&z9&1+mdN00$Muq;qDYySe!Hlj%ghWHZP6C!RI9FYrP+w zUc>Q7&m-Wmpa}cP1d&oDX|fC1kt=8XLFn24gu@MCKqgK`=p$j**lp5>mY%ubA@4sh zPQoc1=O~MH?v5iNngc*>!&oBT90pf!fFx>gD^b~c7(XsI#fLBY2(=f^Cu1|0!DSOO zvE9j8nR601oo4n`)KK#IPM4ol2(G8;xD+6 z>XH}h{s8OIvT(vxS^T>VssN$(^Y~2x^F?z*$CWlmq${=>%igBYOo@DBm6ux76+z0;7waKniR?fgUHbl zFulf*eB1U1*O|_S2p~990}Gc5ia?ijsx`%|M8l(Si)Nptr^Sr0Bn{PDGo~q_cqGHZ`XH_h9Gn2 z-+C@^dck$@_*aea@q2CJ^5+_E_~?x*4DR#3q=sUR$ShDXI1Exu^df01#!CGn3e|DnqzIzTi{Yw_SU$Gg!+2%=-{2`gG z>LUCiNXDjLH%oFKbV$qCkz{J|LinSwkvHZB52y`Kfomlr4vR!n$<~h>VfaW0FF#BN z&ToxKl)?cp{%0cCdu=?4Su9P~-~^c17$%&vXfmX|9T>Tt!?%vR!QhNXJY9W7;eRUx zFj+1ZY*R2I$y2SN)YLc5uU=>1HDmqZoSkKAF9R_-Ge`%}*0R>)-LD z&|L_MRj1=Xi&OaJz7Ih8d=g%}CL6du+DMGI{K9TO=MrPt`Clic8(!0i|4@ifosT5m8aOE$PBI=Tt?=(?8K!u zlAfs41tDE;Nm7T1P%%g!?z}V~@~YB>`6}9Ek={h&>S;g%w46!HlP+*Pp&9!tDG@BX zgxh4@>O*%3!MAQ%;^-Vej^!zmfVxe1_@@w$H{yXULBBxM4F_^^(MHGC0R^GvLkoN| z%95O#?m}*^7yt^Y)8U%AIynA?EeToq4ix8I0+wPW=y4*qt`Iu&E)6MxaLIqZU%8xj z^`|F1ELR8es_MbqG#zq9uK|w`FM|%78^C6NO;}keAmy%4gn2KglYef67pp&xWESX= z@QDWG*n@O%b!!*4Yz~9)g%+9dEDCInyvXyMFI7WC%Nc+kzmiJ{;9|1DBtg zN$o;niNhWXdu!>2n5{J}?1`{)6OB_$@5g zp^D{${$T6!5cp`H9TYZ3gMq`%INDGdn@G6OCFbXW!GIi0T@wJ;rOhJ0nn%EAWx38z ztCoUo)fch#CttYW^LkrO^68@NO}9By7Kh2I9Q1>Pz-!j?8w66$M4 zM(ogoO=}wQHuI%eL)`*=Jg^YFYpw^=x2=I;kEKY)Vo86FPkNl3L6C%Ku@^gs2vf{ z{Rciehr*hvh4^B37~ZEfm)sbAwZ2m@n&e(v4D+K$5bK`FB&~=e=UisNqt`{w5&fFP z;foTUU9JWXImtOsbT7uYXUoFeQ!@BA$A%d66l3XuL|_u$10u5K0IV~S*T2z- z)Mv~jX$78SP3b*gY_yE@Pg3B8`XhMea5ZmBtb}KyBOudXZ6Mxv<;dL4eBtl1W}sX# z3T|t$CVPK!&`jqS&YEvs-{^b`m?f;i4hoS#aibpWU2F%xq!r+$xuL*0VFoySFa}mW z)h1#S0ogslm-ODq5nkj?B#!a1c=O`(!Zamw*q(kzD7rBns-H80UUv`(wVMhq78rul zC#0d)B_8ir%{Gv-`x0TtHzA=Q~Z?6=Z2RMPlBP~eqj#*&Stta5j={S5;YZ=HMx!Y-}@>8Lr zrV4y}ClOEJhdEJMA@~rYNh+&XkQvg_@XOXByyB7_T$-E!Kdw_Enyo`%mW0z9SF?;v zX?ud_wMxONYnAxt{AEO_?gpa1#$bJ01NiHpG0*DJ1z~pj0gz{+%KKHOf$d)Tkten5 zVW3$eIoF;{W;Y)Jo54Q(O7ReWe@hEARY};Oquc;*SOd7$!**R*(Sn_DGSwT@|>iAspzBJdPhci^uBT0upX; z86PiMLEg%4Ao~{vfNNjQ0UtC1ss$LrYhpR342;v!QwmVcrCLhn(bdf4|`U3O7e`4=OegSV8=5RKF8S;0UnZ_OX7CJlaJTFNd;y%m%tHQ=D@v)u{`%>%kZB)WALM?C0I@=5nOUic7D}R z2~Ibh0^PZ9g&z*Q25Te7K`Z04cxrY9c-c3dJf!=85BUN{?pTBGozlUPf%8a0j3i@- zIl`?a8e|o=AlLMa$pORa`g_m(;M)z#Q26qzv)29t!1$0HS=EyVdB$2B68Sj(ngvXqIi7^ri9w)L9=H_t9!OwExO#~dj5zNLUw?`P_5Pc%iBQ6kJ8X!B zfmR^%ge*2x8ckFj$B?}y|MBE(>p*(!4S;v`0zvkWg#SGR8sPx=wRVs5_7yL2G5?^X zhYCScls1{f;k1zVRcB@-`nf|_0oTrD2r z+AEDbrS+Z8p>;Y$zGO66q4*R$F>Y{P$Ug>l?o)z|+0O9zw#nF_bOoWIZ2kK!98|mEiWSA@K%V0RoK=#KA6y#&{ZwN? zWZYujkokQa6)p$8jx_U*M0DY%|HcWwesHZHQU6%@vCj}RTt5S%%^m=CcMX_kQ!mM6 zYQb{RmZX=rkc9Y+hkAP-fY5)5LJjqKWcSkj^{(~%L@$Gki?%rF*maP-e+88OXh=%e=z=So+lA+%%XsxZ z7SQ8VCFqkCwwJH z9#%Y#z)LP%2QLER@Ie!ybFQpB`FTAbJoZzDUz>CA)u?WqG)fmp^{Bz374zWtR}*2U z;ZdF_mV%w;BcM-%5s_M80rTGK!Q=m#OZd23c$;i?f*Zcmh;(}*aM1R_J0<+|$G^vr zO1u3yJ<im@s^B5)v`Df?`0|MM*|R{kcO{gIYyscYr^0YsL*W~#Wbis{J&s>) z4~Wbc>{(lm6+g$3uNV69>)y+_`FRHJf4sl`o^2FP`jUooBs#tHrW#lwVVp@d72)+I zlVRj=AhbTQ8>dC&fG){?;6c^lGuO+w|3Vfh9LxrDCz=uY`&r<$uOb;*97k4d-A!We zy26^ckz~q9OXxT+49?0N1DMZ!pc^^{T$xpm(?cc`r|hXv_ftIVKWPEeRBi}mdzW#skA$z(WM9Zv0>#IaRUa9lnjAreECIT}$7dgO;(JziKG0pIkECMGhMuwnmRDCIK@ z0)K7Cq7Ci5jxG)I^J^pwHuHg2pErQY))81J9)-_k4&b)=uj<`jZ@^;`BS0zFf)m0t zL7V++a3@-Y=xq%oJ`3ODSBbW;`N|#KT>Tgfi5Ek~_wul2gfYqVG$e0cw&H73i|P%J zjfZW^E@PYNvLIULo>1+{WzgtbA#@rW47?sE5~(d4;Nv%Oyv6Zr$XCaiAZqdtp8S$B z;D32G_PX&1tL|;!Mf{P65o_c?-Kr8W?Uf_$mdrPp^M~-~lgYTQ@HFsEO9h#({v>2k zF^Dmo03De-TvC?^cNM9^r{>1^xq&*-Y}CT9j5kVHh?79R&0#S1#$n;7z}5JrSv@ve zaRC4QyOJz+@dLr%-Lb%DG#vFb6Zn~rhaQ}v@bj)S;3{4Tt*-tPj{RgsvaVd_85t!w zPnu@}jm~_>Pdul>tgTis_oF=d5~T(|-EqKm1`33xy~FR~tv+jiQ4%7tM<6J05os^da7 ztv-ZLUH%DHZ=6VC_J)y>_G`#?+j;n=&Il;qya3v!kA!dLYs09`fOM)A;h5zqfVYPL z*G5@#dqFEOn4r@-p{lKpd02#nkSp)lD~Xz^+}`PaAx{&s&O zY=HAg(Qqt|)|TW5Z2XL0yN82)e*1Ch?s@R&tljX-I6oNh{V)Df=M95~4e*_bJaW7|V!mvR=79l?L_>h{|J&MC#Eit#Y;(M~Adpiic9#$@t0 zNIb_c!$t>d!H=p8Tx{I|ik5m3k5p-RShC)PhaCyM+X$TgdoSE-+a_FaAPY2TYr(rq zG=Yv`JCHp!o$MIp2i29;VcQ=Ic-m3}KhuCjR-pyFbI1dp2d)a={Tc`N?vf@+yGnS! zZ5+v&2FBZ%2#7QJ#dBVDNO)uQY4AlY6Z}~v={K&sjt6h&WBc28fsWcjl7G$xZfGwA zt<(2|oN8OF80^K1_-F67h@d(f7{9bZpvI^OhmyDGP+Ic797huhC?Bw>8NhOvO{TC1R$%6fSo?&WnuV$j*|pz`D}_zSu13Eg8-wv!du=f_2_@4Z37?!;~1w2})gBw33fSUP?=e;OL*!T1SxF>1`$D|qVKQ7s` z<&B`b)DjS~Hv;=bo#!pvtBu?CHGt*_BXF7JVfVMLgno#}-|lI_jW64T^;hJG|87U} z>Z&2!*MAh8{5%ZIUQ?lb-w^P8qC>)jr+KNuUVQzP08SoN2ip6-<2kb)<1M90!Vl51 zP}^q_aSc-^U%smn-f1_ivdjoB^pPitJJ-Sk;qrKWhYtAT*MOUfJm9!dlJDWG2W)vQ z!n5Slh1}p(-u7x!Xp-m-Cwj@lXJ;n>QIIOhU!hClSEqt&mmh=A#&%@I(p>QFjW_rb zAcg1798GRy+QN--gLq+WJa~M%6uTRa#nz_%ys|BwxK#EX?p-b6JiGN^m1r4aapN%0 zCQ=1jWm?t$2zrZ6yJr&9J_T~}u!bbp$PhmBF(Mz9>(qai+_nGy^8zO&=#wsQL)b5W z3=CRmk>Fx2XQwKCxC_`2ZJSNly>&9FUB|G$Z6b~_vVcDxy{~&8+d}1coMf^$YuLJ@ z0TSNO7b+8Xh(DWEq3Z5Dl&w+5&O0{KCF&*IO{G!X(C@R9uC}9Y>s}$-V?|V5<0cDQ z*TX)l=b`vKBh;wcM!y>wupG_fT(K#nF%@o1qr*UOrsM+O<8LCW-8Nc$L9vi+K6H-> z>Nc`jDlbG46T48%od>LK*(H`fqn~TnPhjWCf85qx>yYm$H}P=f41U@_BmVXAe(dtH zetyV-V)mpef%(LLrg@cfnd|h&tnbtXs=47mD4VdI4ax07Kkwb3I#-`_58fSMyG;$5 z@zs6w0hRcf;>R;}Q87CDvr%;G>3Q~I@EEFfXb}litQgc81v|do5qZvg$lGGHmF1

    H(%ht|4+$b){|w!QmBo4p^h*q-P7*`CT&rd=N;E?6vHk&sB&c-EpTQX^5y z!$zd*lgzD)hg|hKd$cs{f#`F$2XlpA`B}f!nNOrVw`6MpH}9%smiyOsr1R_*-ElaM z$**!^A6DI_Z}!~b-?sLmqsv~3E~hyP{AN8sVH3{N@e5bcyaQUu0Xv8uAKAi<7ya{z@btb&$obC`371iaB+zna@}ns^*N*=Ca8m@O?4s)flAO zC9$l3@q2E^f3wlEov|!*?NT=W!C88wqK5UXkz|q&f8zG3j7H^7A*`gw9XN~X*$clE zWIphXJDOe3qCd{1Q_ppwvh~T_+JkfG`!B_`F2yjtLe;_k)nc2!`!R5k8ED99Wsm>%XYPgp|%gIEKqho z`upS#KPuf(G_JQ8HGEmle4hE!)uG9-vd5bxXv|>s?kCW^u9H-0+D&dn`8t-XEltNB z3P(02^U;@0OC_wwJ@oFOZ|rgOc`j4JHr#kvO3-lU2KQ&6hxr{pgJPCVM?pSssm}Qs z^xYw2aeTZcDsh>L9BeYEafeWJcU%?PxBQ53)5GbKS^a%Xy5%R8E|nMOtX_wJO%Ipm z*NBYTZc65_&ryRN#mLLKm&Sz;bKPDO#r=y7na07z+}TOVwC;QnchIDXb^U%x|8>K2i$8Pwu?LbJN@82K>(T3V|5$9Vn)t^2M*iD37m#2{Ity2bXIwxo zH_q)g|MsfW=#}dbuqWlrFh-3lyFZ@4AP{g%bv&tg&UL1d&7(8r3TW(leY9O|wMa5r z!YPLBK^r?1X=Q8zU*(n`60ZEo2CrP81)&b;*+|AO9kr2-He1fDdv8M^vl1mdN@vL# zR!FgJt7zM$OuGJQ6!$h(3;EBMVO!-#qjO_R*f{xq=I`l=#_m1N<~B1LvGxdhw{e7E z<+*CqTey|>JMBjWPoL5)S);hu_WCq^sDjJ*wN_-%QH9o=?`CZHDk_n1{{w9x^XyaCU);S!+ja@s2<&HSPa)+x#Z`$2ivZ}OLPOXN z{b$&s#7P{!?7$CSHVNH~1n93{GOE8)MQwgHifC{je?{#xrE#E^((iawwMOnq@fw3F0ePpc$~fXCt5DWXCF)oxP((*M4!&D zV5jZdQCe9ut#lTkQ+bB$)9B zqj60>OwYWMJz4dZcB?F>6V`8FNx=^MJvqNQ|D^-8RelnhT`Td$tlL3z9+}fOVn|=? znt|R-s*rG;s?n>;bo64r7c+)DzS5FR^s(w8=gM`m{eE`*M|*ndIu&VjH?xiSB2Bnb!Z&n^V_R3xWeyR+#8a*W#iiUsC)cP@ zH?0Kfcdr}YHk!tIjF-?drQb}){s2n7(a7aCZl{kBzhroKAY=`;L}%qj2C%> z?OJz^Z8$#%j_Tv!ybn_v%X!tpIO5*GK$Zuz-)qFU+ zdP9MJc@wpZWAv!pZ1jlSp^k%Y+~nkcXx;My?)qIz_Hr3F+key7yW`O1^OfA~6T6uG$_SMGIhD2l z{l$W(yhBDAUuo4#W%i9{gG&A1uqT?moX?0+5(Z@}x>^03bFrAgB@X5yf%ze{B=e+b z`k^{>16m+iRe7#8%7EHDbz@4hd35$BccwFVkt^8zgq}`%&*dFngeJRPYec_Hq?x$vjYw<|z;qw#3!2ZY6p<~tNY2s?N%uu@KPsP#Hg-#K;+$cQt$smg z|5?Z;6m-)cfg@2t+h246E-XR}IP98vC{v2XUN43#oGgq@;<#x#6b`LF3&_yZ-i@1yr zrpWpIefBZ04$TY9K&=hGIlT#`Y^ZlSO;;%8Ow_C?*{i`m@t32S0Wa8`$hGKBup#o( zkzw1@LM3d)&&btTQLsqsESj!z7(M!UocpZQ!YX_YvXSvE^ky)hsYpC|Ye!WhsWxYn zF~^);_W#b>n@X79kfZ2kyEfBX30Y_MQM6U9hg0;wgZ|ykV?!Do3i5hMlUt-Yt)(YW z;kyr_v6}bL{87{BN0U#eivJQVlrR{_O^#rh(NWx}G&fZA)&mKTOR*abk6A|EY{qv! z%K~25pl}lvk$lZUHbt_pZ4BsS*E`cuNUbMl{{9PdiE-oS1?=OdTHZy^%@7je}x4G659jx>u9`#z=|Dn$N7I=b<69f;*@x8A3+8mVkr=WMj}OD}s| zXNP8e$mY7qe`v6)5-qH|gnn)tgZdu4gj}8mH@EIG%U!bowb#9&2RFJ3)bvxh!k&@T z=?g+(;R4aC#uB>HO2U7b(#5=vXR_XwEQ3!7Y! zef$};T&@}&Q*5PsW~hm|J=2(cZaj;5?8(v|9pc8tBr&%nMwb@8WNw){%p6>0i%*)zzOMIRpAP2?O3tjYb3Qgg{?5iMQ0Z1AzN1s?nAJQ zICBhSCK=V-3JEVU%h!}@OY9jHo7~a&2VGM74?eKMWdb}l5P1R&iM_Dnqxt}?yusdk& zzQw3aqlX@jy2xp`sfdTnb6L?WWpVTsCzRu@CBA6?i>%A@rwIc#>P1N!Hqeb1V>iT5;FlIRRv)5b$Fg@tTG4N~(WS zkDWYlflVC#h-8}2(vV|+SdqCdt6EdSE%F_KqMbIfd)01ii~TeKxU?NvRSwgk&*xaZ zpC5A9az~MP9tw@@ryh%+GyUUMq${@q`6c%vZrov}=`Y1fT?^13!yELe*)!_-&Vzq7 z*qrWd%cFI;i0$~>KymzQ&Nw8WZI#O5PP7@bGkbc`LuF6YX8n;ZxO<1bE6`D?@(_|1r+NOk0L{x>E$E2NY?@& z=N>C2JwKjxjx}UU+RNC<1>UHOJB})+O+gPs57D`KjhwDHm-9>iC(-7YxY`SExRL)X z(C1lUNL2e$G-J#<7J5rgeDT8^^lgzV^}IEk9eZ$;s~(mW{eC(L9sLt5IoB_Unh(Yy zw~kiE*_aC&{vD-*{e8&eP6PER%%p+GIOe)MmvzWQvV}%(k;Y&>b*j6-O_rm zKj>ZH9;H)yb!ib-r`wLSR#hNR?_voSqY3#IRkMTl6*;djd)V8+FWeM7z<9xY&UsrO z=^tWyse(-Ia`Dl76?c)AcpwB1L0c1oBjZy$>W zqV}_+mQHAy=Ldd7_yRWn(RJ?jgeRO+^;asWE8&cGPh(Ms{LzZ+N_H$%51FYHv5KV{ z;tlx&=wn(AC*vE9_SK%Dc_*hZZG*%7F&}GL!NRe~+J8E_S80QO*#4k@m&Gzqi9dP6 zoj8=V(}r%0RYzU=jr6?98?NfjB5I?4ol`U`XZ%^}+=JiCS^L7P-0n%1Xst{d^4S|r zx0SYXZ_>xHHBd&dw_6^CxfqBC_y#buk3-#84%1DAhxo59yka+RN3+D6t#Eht4RkN) zCe`;&Vcva<(7}d1=+O9esFs(*%`Lvq?UC?ks&9EQk;ftiD(7%vH!h*ux05*a`8g8) z*aa#SNU{owu5jjY)l3*ZK&tO$qHUrJobskE>_}`I9P+=1&Yqiq#@v=-2X?tj{QHE< z>T72!vMQ0Bj2jm_ZwI&2;XiI_N(z;_cbwfiGK+oq;Dx@~S)!H8A5*L63Mj@^37dMH z;M&fpp}(8%Ap2)y_IwHKx`^EG;Ot4Ol0#RCiGL0+x!P(u6r%5IE zxHFt517RW-uiVHv{#wk`$7zZ~3bWYN-}RjQt3IU6A4?*qL@*nVCKx=amQB&jVzc;j z+3%88w8i8W@?Y03>aEyM)4uIvX(=-7^UOK?VE;e#n06_v|8xOWD|sUG=eleb*N;-q zSg;AV53+ZGYnWMSKJCkhLsk8vwma*)*vpn-QzKl-u39{>`5nVEKLZ1wtq43YrtaDdC^^eKp-}D=Y zX1?3Ur|#P59Lr#jzAQzjRTr~tVHNWp9D{CkPC_KD5!>DrN*MJy>{a_Tw%zR!o$51+ z&2A;sW3;CDdiE&PeDM_j`!7W{yG~B-WqiO z+-|hJ^Q?r8T}m@r>NtAh9@`YL3;lXGNH_0OXJv|7Xx=etlzZ_olT%7%uk&}(0qI?A zk8wFIk1%9u1>0FpSSA|R5sDV*%|)h_hgr({cywi~j9`WFEUxUsc64~#ZKipyn}VWO z)ICZ}r@05xqiJinlvj6&OqK;wjkv>3nfySBBP`J}mxENr=@XY~oXTxjzZ6ZhWGMJS zC%t0c#6`O-M=J%txVzIQbCcZr(b0LiT>Y9TWNy~Wnsj&3^Bwb$f>|H)QW)T(Y<45{ z<`&fW{3GRGIKz5-5pvp7&Z4E&ke75jIuuw*9X_o_pSSusC&r&Z3Px#ckszBpx!6$9 zBjGVyXUM{lLZf2i1$)OyDV3t#IN923WfLo?2;CQ7>^y)Q2 zs9#DTbH4!1k+Ai=6a|zwTk<_DHf70G9_*F#U4HBu4Z7yqWO{#Ly(n_?Iht~=jQS?T zphqLmBB%5(=*Y7-+^1_YOjRz7Z5YaFf-9LjUj6ur4xPa5{0Vs;`$@$T|k>LMP&uFSd0&hMPV%^#PF z9Fk`sspoq9jQiu*HAyX?xpXx;adJF85Zx%M5AEk-ljGUP0|(K}l0g=jp@2GX9z`=l zy148sa)Q)~T`1$pV)nx%i`^Nih4ySaK&+M=pjG4Qn9J@H)JrxF{aKJlc|%87?<>MB zYIr5eGr7)61DoiyxHtN4#kRH6do6)ZSuBr4uf z$}j9)fXsz0Y{({G((^B4sx!WF!z+(*0h{w#gT{P@FLsKym|jCqf1c+S4#hFY?3Zlc zMQh|xwi5lhF^Lte+s1zTKcl~I+p)ON1oowRmnd`*<{yy#&ArohM&Cc||~4(n7mWbWJy=HYG3)E{fI>3{3dqvx0Dlg^i{W$X{szPAxcjZ$Dv;jt+8m?^uU z?#BP}-Ip7d9i+Jt&qbCxV+H&ByjaIU12$uN1Yb~f7X@GHlCbD)*ox9h=9N>*{w=+M z{wN*gvhqBb-m|f!npP8adJ*u=`6wi72$D+mh z9Vp6KN9@#on+qE8jDdqe$Lj(XVW*^zVYWBIfSykcOe-`)(CX?z*Y4Z(7)l^(fM~$Oi&U*4JJ+FpUJnE ztdo7*vbGv(l=7E~b6Z%{787K1MGXahY^OV$cConS$)dJ7=4f>EMKYmzHp-vrgS_WB zqQHJrrcfcFs-%X}3qh0kU$}$lcHu>gESQ>GcxYcp$*c5Eiv z6FNYbYo;O>@7Kur{90s@5lGdZ{9)%DB%EcLLbm$gey+RX73!{zXSVZI(HY5|(rcEE z7Wo;n!DIjLGhJfzwK-cThDdj*saUGVklDYMr?HNYIbCTP{>=2JoRr%=T9>cJxhmG< zk9*?Sc5+A3d(Gr#X+EU3E-vU}VVuOb=q|c>&wb?1EeK&w zvz1xT&Xvr4kB9iqYFQ-m*F+5m5H~SqSahlH4PE%~IsJ5F2Y0o*0C_)u$#%A0rGLf~ zw#_Dv&b&E7{4OYxZNEK>MthxNL9rHSpWAU1b|8_R%1}YIM?zS{{86lQMlV03HIlWP z*f6zg5>9&38%{jfMx|oaL@)gV=oZTZC{N8mV7ywy`uF@mZkYr0tJ^Kq>^F*O(78&HTQH7Z{%bPPsY&bWanG7C4c;a2=_62kA zzX{Nf&=d4}N*=RxtzddPCa~_|y<8if%04B>iy}2L*sLXKm@>EY+bLT^PNE zKi#GoWqtk0ofeFRt&&`UGJ|TiQpbTM?zUlP3$8Hx?H|E*KU-F%QpH`qv6lU`(qOV5 z?xCAkE>h1lKwrF=!Gv{%`0H56Z0Ba6s|K~)(9n7A+}jkoHHBjv!kW2cJqbs5tv}UW zNm$q%V^re6V^=~PSd4us%R6xhm6i4}7@vXaHovENYF{{o;62=-&p(i201utNEy3*h zUf?u#8&I9xDs=48VdjWDalrSx@my7PYX6ejgFbrnaA? z_xg4q)1$v>*_DYBWc@{S?M6F#u85hB?kUbp`veNya|Cq^NQvFbO6d#d1Cn~bnm$$z zV+Bs*L{*zZ(0?wcMO$|B*xk8uY=m+dy7Ew!iCi1q7T$c1n3{>X}>Y24s;Oi*JB zW^P3@Kf3WNW)v|cV;hv@lFeje7z;UCf|fj+gZ9o}LibC01)}MbQKhj5y72NBm+Fzp zev-vvG)aJJHl&D7J+Vi6C4q>iSV!fi1+&k)2>150B>ie~5=(IyqPuM4IK7_-XvLx^ zmeH|~{XFZ$luV4pCt@N*<<6RHNx*g%R48UzTer|_qs>vl+B;M#e^q2sb6*>VqzCO-A!bmq&jeGigECzeXyJ)4Tc7ZkGAXpy9snZnP&suJy8$!tG535*QfX!XDneuHl#ov5gX)+ID@kBcfv#by;_UzJ*R@o7Tu@O(o3l#e=f>h=!ibNJkP!h57N8VT};jI zAJRWEOwD0g$=KZ(ti?h`Fv zIhox~vl1bbKeQpujNOTrWcrm?a_&4ubZ${PLpPrSE{reu@-2q#9dhGmL?1_~!)_?4 z>I>_3%j4;PtfLy`^)zNv6tgrk<*a(HBdtGPY_obY0>c^Xk4qNnmHC1)en{%VQCGMT zp276}q)BLERvb$_>`6D|YKts7 z6X)6G?K13s)LZWKYgLi*O+E4Nn`bDUSBFO2or3PW*s{Cz25iUZOrfZJqbW$(Qs zA@`j7IaXFy$WAmUDJpFZA&Dpr6;h;>Qd#Am<8w$_G(<(I)K?J=rJ+Rq?(aXikIy~# zb3gC*`}2Ojo-kwG!i@3)W2d8wraNWo9bo*^=Fr(UR8gm99O_NE#VErTn%yLhT+H|x zEMn>m&v$_Pqd0jyy@q&C{?f?z)3~SgbC~nL6X@-jO|-t_ zIQ3jIO2v;Vql154`LmU0R5jfSJ$_com71AEo8`3`x+tw_{M#L7PD={14OC#zzsqpu z8bKOlt&i?+yvKy*y0DAlW9Zog0fhNkLb*9t=;yoZP^ecr`@&Ix7Q9uWTZE_6{TD9L zn|rL$4l@h7Z#0&R71pA=%q7Y`T!0R*C_$T2*3w0uQPkmLJag?n%d}rhM5m{hu_?xL(fM8v3fAhN zXWI2lj7~P4zT+qrwA?_qDaxVQef*3?UE$`dbF3&lDU5s`ap=W6qUcW*ho6V`nIN&2e0+q>LV)6rMfKp)ua;Teb~W1&dp=BBQk00%p_(ej-d}kpEmud z9%SwQ{bep)yhM9L@I6MLhXIBnz~sPdNkISyEFejcZF{P`+HVAwM`piE_s_&13 z(@c3tStfXQ47Y&Z!PawgID1c7UV5abkD!hSy+e6rDxEL9%DeM`?ZJ_-pY z9h;YsgAaN@da(sKU8_p|X0>y`ho8LW`URxkA`+`_Y~+Mh#t|F44?yJA7Vz-LVj||a z4O}!*hXuc0;n01}VA63F2`?8X4U&}{{-_w{?zJO@D+Ga-P&Myo*c1F^PX*YdtU!!o zyE!kO?F6TW!(f!oYjE(6D@@bSfa2@L$xW;elM?H|E>(T}{lrUP6+45NeU!%cktq

    P=)C-r(s9`>@#x2e@^SH+*?-I#{(G^8V>dLl^69%*Br2f?#|2L{P}*k zW6TMTzbgi!gCh7vQvz}9jDpx{H`sa41y;O}g-hpK0yymzFe>He%RienM@vlzS_*#z zmcsH-LD3lUDI~Zg1CrCJ&TyrQ7;)Mr361p%!4GyAXg!Q3FAaw975ig2Xh9Ou^|FDo zIWojv!-#Ha=FPP7AJx7-WcExC@bZraaN_-hT_CpUvX3tc%^S4aYSQ;keZkKhUF zU*#3r?1mo?*W;0vd< ziW>xfclbf}z5wP9)DNniQDYcCFbz=Ido$sVddfgSRWKkOl%JTc>(_Z@{UrF<-7nc*DC{$E#k0D zNfxKANRdoEl_kmC4$QG%hHpRGNn9m1fsFU&WVwd{x$hEA4BP^sQKT)t9KHo4oG1d* ze;71wef%45{ov|4(}la07L!UJANV}60RQ1E#%(q3K>1Yz?lo5< zwHrU+@HbCzatVVs7kR?l)2v|iNEkdVQ4UUo>Jmrv4j)~72H0KT`!v3NIPG`@@s3gA zl-6s(yTp^kXv~LKzPQ0}P1iwLc{xwdB?6~R&VkB&L-C))YM%6?8Bnpsiky0X5wGZu zCliyxlkT^($^L($B!QNJ$t6xixW9`x%RvzMRM+9pk453mUs2@WoMGTy zSp%wDo??Y6X{fBI4Ruw!v3-~%d=h__mwFb#^pU&x&Wk8wCRPM)6)D4VP7!eN3j$I{ z@?gM*0FX1K1rOc=pDk;Dj4b%xxp)@ zI^@;0?Tw;^Kk%awF(Pzg60mk2uHv>);)Po8m zZLp>PB?ukZ4~(bDkOC|Y`j1b7-OmNd5xYRBt7ryqn5)Ah;s4?I$P{RM)R7EazRYW^ zslys;=*8Sv1)a{Q-d7qrU1f$iRSz{P=%u)rGO--g(^U!yE%U-fF>p-Lptu)dTF3I0LHIDG{MeQDWEY zN9sT+sLEP~mvNIw2rwWOJq|FyE&{%M5RCOgB#DK;GUVK9!)JG@liyDkz%-E%@@~$0 za!Xd2=!$0Gq!-5UO{gtAP_qM0{hI{Zp$2f4p$?2IT@By!cT8UKorI(lXK=KO8n*Sb zfO=s$;Z2xKx#Qs%t)xshmd^*>`~h+i&1+)0-U|Gv+`GlUNwB z`7};70Awn&9EeOL;m^mtVSS=3S+yz~XzXA}(i|0--y};^_N^w1FU%teN3Md|BTCSx z!v~IUS0ZuKyoqj$2DHszMv_MT;f%^3!05aT?8!X<+Mj<1n%O%6^UxD6t~MhRRuQB+ z`H_RUmXeWU-7RW{IhNm4~2S~c}MpN!y8&B zu=~g}HeCI+i@NJ*gzfv4?BOm!ACJQ&tk{BRmP|JIQlb)N@UG_U2fkE|lDT~BfJp;S<- z;7lS-)9_tyQI23mBVg7gW1recIPKON(EThHYAhK7?|Q9a;;L5s(|;LJyeLSf?k55N zG;MOWc@P&4{o{o2GI^6Vme8uEh{O4`5+?M8kfY5eub)& z;l)lIJ$?zF<}Y4qxybMwjvwTW*?Zup;r5(_MFKqYLPPkNH;Wi4DnpsiZ}FD99AbP@ z2_6hofHBRjVC;wl)Ua;lfJ9>wnNhRI0%7-(j4%h;Kj- zp=td#=y7m?8LyB0h%m_-~e zheGL+70|ouE%xw!53tw;$H-A-a${HpRwX~jN+$&1Kf(D>d6gk3l-fmZCGN$ljdEn! zOL?L#mukuIU&nFM3UkTS4 z+~8>ZWS~e;Jm1n#aeQB(0B;``!s9E|cpifr!NEK(87UriG&P-Y9Q9s6!uG_%6-x-1 zHZp~~`*S&NYR>Sl;5L#m+aJoArsBxs@zAMz3Wz4oCU3nlIOp*TzY8@YA}gCg+K;>7 zQB)XIh&TnV|6E0aK17m#4|K3Z|6=08*+qV9@SmO22rlJ6m!FmMg};VP$%b`@uv(-z zTweTvx1+!rD!=^&W-dI7CAZjfKCCk$uI8`75~J7Pc8?ErTg!m!god@gR>myPyD7gYni2;dN~rxFNnD7o_pMGF?&FJZcKnyf5P*{UY$LG6(!k zO@dbrSK+OBqqrX%5M{ncCixYB>hXnGpwkMD@Ll8*KOZvRF@$)SE+iLzc|vZ7BxK() z@Iapjkv^=6UDo=-meVfSF;W;t&s_*lec#l`-ZtQyZ-bZyDZ^>s!k}Y|3BL~51fM7U z#?ey)IC7#MC!4l&;+R0#xFiB-JZR(Pw7msIdJZJ&r308J)(5@@$G~svIb=uXUyeXj zE$|$VA->ZwCrUGbOdl)8>1KLxzfS-W@(_f#PVwEuT~9!`2j5eh`LfY+XgMh|m`^&M zJCWLlibUP!82&5~MEdFz;rzA|TpoA}IJxxTpko%0>G;mkOnAWAb!aPI=W7E|lNHRW zx(LE6B*_$xfXlO9V1<9v;j!uYz$woMs+ud2>JSAuy|f4`?GA&661%|r{2|V}w@+|g zv@$O=HUmhUIEP;)gcDd6Mm%IjvBnljQX2Ffbcfl%R|B%-@;(gIw|D_j7k;fWXNtF` zNFQ$GU*9J&x+E@o0lX#Ffi+!*K-%TyJlyApPWb;VIUD7M2-My*pOZY9)DO$64g{V*)tzt{gJisd2uR!|ECV? zi#b54@Nm*`PKyjbQiSzwp=8Y$E!YsQK*9_id9~7B@bZ=kaQ~_!M{IWhEIVLDq9a0y zwvaIiGxE40XGG0$@Amk(7Jjq zUb4{#ED^8*8ka)g8odIXJEaWC7kzTcND2nNF6PuIs}N5qAE>d?nK(98;Ah4VDiS69 zv$-2Y&0?@~Oe9&#k>EsL9>o27E&`dl7Q9PageaEq_l}oI!@2L5K)I6;o`~OuWlzo~ z#QrS^m#~DnU;J=Sk0i|23xFY4_dxHp&3JW#I~=?j;g~IK3mp%0V9U0bc$MxBm|*n} z1d6oc(xs}v%ibR5S4`rhcN}64@_N zaM=1TzR|8ljJq76#F!v?Yau|i!?UopVjfOt&Em;Dv?a;cV_{RCJ$a^OMCyVCq4C;M z+#QgQH6IW2Rxw&cW7}$YL|`@U7FQq=W?wk9X}08N*&$Hyz=6CQA~@f{iWH^y0B*20 zPJ7+JncBjTE$u<%&6yiu(^M##_P!ZxGO#B?ZT-As1BbA9a5O0HQiFx@7kMYnSi^Tm z1mTl`0J7m4e`h*t7{pWr!aM5%$ejhV8bADzf*B5;@U6-bVDx1%iIT4akqhJq_nj5t z{2S-&%sK~{y)QT)FMGpen|pY3g#^41F%POga1xf@fhS9@ak9=A z;AsJo@LT-|5a4@FZHMPUla+q3pgjVtS(ZfVPQ>!A9Ec(J`1O8lNEz;3x&Vl6l_OW~ zw1R;_dy>_$3ka9Uk7LV9s>3+g~^_gE#&c>Ht=smj)Xld2HU*}(COACzTc;V%|169O~qe> zp-KaMtEK~g>`*2bB{$+Dqqi~M;sbnvH>sWoB(16aynRcLfEzPUa-K|E4jwh@frFRA zU@&xpidnv}uS|&8h$)coOCN%%!j0gXm^tUnadk3}?=ak6&^$@^R@8tdsH*jF%$PHNpU=)ia^YmcQT+-$4COb_;lYY8veC6NPv%h};X* z1$X&pv9(YVZryhUk6e%i?LY$hUJ-)RPD|njW)uuT8&VwT$V*LJgDp=9!=2wO;Ty&R zwkd{^A3jcvoe{?PQ0NOVom}N<|FMVWwnFe6-#0q5d+xyhk-l}@BDu7>-q)!u&U`=)qH)}INyT2`jG-Rr_G1!!dAhEPx3h6 zB_!AQpKC|B3~AJz0gKK661=n*dtR===K|)#W1DRVyI~2Cv{?(x`29IuWgB?5M~Ihc z5<;ZDOE_NO_jt_ZKVWxR25u8m@GgCNb+}fC5j5q+MA`gmzd!;V%Erf?R-ve%tCHnpXe^E8uNnZxO5L0sZu9*6T3N)CuHFg z5(n?S$R*u3^oeZYHo}f-z(&LV) zoJ!8!{%jy87DOhbwD~>7Jg_q7D9(0#0XhfxHUK|Aq2|w5PM?!A%=mL0%swUuuU&5B z2rt$pHaY^tSZ5P>UaL%^cK&jF&?5qi|LBv=%V&~-AwxKLUJ*+2dwW&K4&ZcaE)>Zd z$A|p9LDZrrJi)q2+}a!jmvk0lTHVPDEnGmli}Qh>w=5j%5rxy9Hi32J-q0@o7wD?i zCr$O)Sj$e79NgxJ#e5^l+9VhJEPEqfc4-Zf_2*BWRs!33>7s;M1?0z~PV->`ZL}Jh^;cX;v7JN%)NO>P5(<^jsj|z8>o$Vw-UtKQq8HJQ-Iy7vu61 zO8DC*Nc@FT$Qcw0I4{Lug{&IX6wAPto4SGSo3nU4D+iaw>XQ{4(!js9R>ZS#7Uw65 zg5I0#$g|UOu=c-To{QZTM^!c!{%yGe($)l%{91Roxy}^p_*Y?_52o-fEa5~unKj0# zU*+82W(@B>V?kz#J?Rn=!PWB)1Jzbv2s`q4&NUn?CG89MO;NBt{5MCVXq>~ud&8Bp zj`5E9FT*K8nDROv`oV+O0*Up(<)rP} z0#aM!fxnDA2PwWHjiqJcFifHozb&&SdkO=wV9yVny;}mG^ooaFUlqyf!0WtwcQZ&$ zXDKgtZz=vQUeVGT3`zVpk3tr-j zmuJE6RmRXu;sbs^s6kA-hQRlrI8rdy!gsu!$Tx9A5~<}vdgY_=??3IhXn{Scy(tbq ze-k4$i6?+{q%n-V?F|i1t|OyOIpAI4Eb_yzV%x^pd0ZRS)`Pnc&W!DX@UdCadJs;4XL{ zX#SW6^JMt0)8-fl4_HgUS z2$(!B4;$j-$!g2@IC6~vn76nEqZ+<5nGg%pYm$lg9c9o_wS=fku7icjAyBE@3CiSe z22F=Th{;JFX8$ZE&nM)dkboG+&|C@2nEk^3dzx`m*fHMnH;N>r_%Ux0{{_+3x0k_C zrZgBWe2%9~oydJVE-dxgPDEUIT${;l+SZ& zDFv2q=Mve-VEEzG6lbC90>E+9C#Q_#u)Ksi$R~So4Z9O62TlON^KS6JjV9o{bu>9Y zo8MPBxNDAUO&fm06!CLR9&kL;vOw3(t$3sQX}tORVz6h9G3TvV43SzhI%o0j7mZ3e z44H}Iq3kJV&>I*^?ie$0z|f2gtL+3ri?sMj7jZm)H4P$annW5{D>67giQIGvhUF{t z$czFN(ChLITTiZlw+jPd?lJy#$uNUTx0T?v&0;XK)(e*oJHfoUW?<18zSZ}w9qbKK z<1n%@u+QKsF4SIzw{Bv9S9btO>K7(5M*BH_`X2!ZG{#G29BX86{Q*pSIJCJhLN4!D z;oZAX%n2JD0AVH!kE}Sdiz>BOd1Gk=Al5{lh~zs6vC}yvqTrv~zH6`V74Gu?)`pyPkw4857q` z6(Wx(c=qOVV9*I!;1Rh1uB=jlJ^q&Xo8UMe){ugQtW{;cZI_F1wN}m6@=m;4Wyo z?|{dE9_cb#1%n$llbMekNnyMTzBtN(1(#od0lyh=kzNfZesRG4yFV#b+`&7g^aRXN zGlw~kp5mRW1)c-*BO)_(9>g$tFkjY$e12JlU5r&gY@P;r+~9&uLN#&EAy3fI zeHiS!&cGHoZDL?sfDIp{!y~0pWGL)!qXktXPd(DGCI4RS7<>`i7#aeSEDvuVdj+EZ ze!^k7z4*a_)%>}uF^~)y!T(u$^4z1VIc@%1$t~jwUUZTkxN~U?Tp9j`H{XAS%}=-y zV+j$`*dGjYGm>%fi)?tpUodK~`PO#Asy*-1Ab0eEHM`;u>a=z$eQ9 z??e~xVH1bgXUIT}?DycxK5^pMZwa<&3&N{&myiv?a#(rW04^MOiW_g;0#-urvF0_t zZ5=;``!=vZ-Xw;YFUsX5TrX^_96OFrRdHd?Tto8r*K;sZD+~2r=)iYRH6WU;L0a9? z@cU05IL^%(uCvl6h_`_^7H$EgzZ8MGmigqd@@X8GXh@a_90b2dz2K8_e-iAy1x9Zl z0W~YbaP96;690D?jK2*5#ehQ|tq#Ne3A(t+X+#=+Ay~X<0hm0m1AmtJk%W87 zaNg`6cr0;0xLP4bJPwJI1A}eA|BftKbI%!GT0^pSha$4?_=Mm1vAffP4;RmdrzefbJuOSp z?R6IKlr<+iTjpY+7%7-|D3fz?lz`#i+HfyT<~&v~gaewtz`0xB!PL2Q9Ev`IZwnJ} z>gaNOTeBMiJ;VLJxX1*&l;aG6;qPz(2LMvxr!!ypn?1khUG6ZLLTmk2;6v&rabyDUV z3D<1c3ic`L;*CB_aa{tzzHSRhb?0*UXl^myr(j5yeYS$Wl{?AS8S|jK=0j}u-wmDt zKl_XZ@GbL4xj2OOgnvl8ulR*B4A~b90zzb?;li_j+{P?2~obyT$zOQY;9=-SR zr_L|Hjb}-U-ulDN?A?v(hH1cBJ%Th|_QIp)_HeqLCpj?Oj4d|Y2hu$epxR!TeE*Y$ zPZ*{`y`lvqddo@2r^-gesh||xJItSBv7_Mik)NFM)?eU3^il$riIcqnVR!?5jO}jo zEzdunaPG}uVlp@aa%)6MZq|P|@jxtT_Hrlj9+71DI76lh8rx^^2Z}ai(<%$N_tsaicyK29{m}s~4Cv(f+!f}j z)m;MNccbB%w`n}G5y92wHk^>DU{Y5$0z|@dfQ@W64s)^wPWL}Jdd~xpI}runMqh7K zW}<=o<4<@=hbCAvOO3n`Q*@u!@2dGvn#a7o?Qn)8a^x*#SPy+eu z2MZ?kfNJAEJoe}pxGh(Qqn8EY9}m*-JjaY{Ozx)qRS(F9SD+ zE@hO6!I9rsNFWzDcq_p6<5%&V{58-$cRKXjCJi{{8o2$@7x2En2Mg`(!lUl_SZZYx ze*cvVvpemG(SK!l2rY%R+qdB2|198#JYBfW;SNZb@+8#?yKrrSCX86xj#E{=iMZV) zh`8_sBxvrzCAXqT&PE@yl5g1-Z`L4fnOA^1*8zS=4uBFdnIPQa23S2X!E5Qu$I@3` z;SGBfNy@53^0_-6|4qkuOvRnV@MpKXPv-++Z+;ElKLD%?gNS7lKL?N;!Jih410}wb ztaqG0_i_}0QhuorHXh)}-V`97J9fYT{XmjEX-bY8^Jh5UojDf8u3%$`8@!xi2gB>b zVAi^~_yJFgm~0b*ZYdHtQ*@XYS}8>~M@i%7D);e~(=x=s#~YeGpANRqEVGXeyD}V0s zB9<${h;|pyeCi`MoX_9`GrO>cMh*Nv5w86F75wAp^01d2;LoFy#BSm{U@|O7#;^h@ zl{$~DuG_*(x6H}lCSNk|Ul{)QNR!v890R!gxw-F%HRwNNNLq8ffLCNV5aK`oKcn|S zP@o!#5z&MK{QmfE>;d1{O2FU$wvn=AQ8*dA3|^OZBsbbr$<_@Q@T`R!q5pFoa_*=g zXL-LPNt~KSTvhy`ZH_yU_BMz0RSJ9uegH6DFYugX#M#}u930Iv;!S@RLQczn#Jhw~ za+E7Fd8==RVv~3+yxrjwxC6Y%>SY#Stw=v!TXYVrIp2?ae#VfGc7W{H?*JRVF>s^n z5suf)*FbDpIO&?<0w>+|$y#-OztqkbhQ50bTwZ;_;i~<(!I_8OJl=&BXS?!b4>|G_ z<^FTL(SHliwA98wzQ`b@y&M#%RE4w^hgebm%wxdcgk8Jc0Wmr@O-9?&sL!+SblI^$ zmN{sSQu7`%@^AK_=#!~P=h+rC`AwAxakrz6RqZr%%ME6kpBNK?#aPXi2kB_9G9BtT zf!Kag#C5xf>b!o^l7HpQS@T!SZ3|zTnb5;06;GjUhxRZx?RHZ|>$ymN%OvVsEswI= zZm{f49ukmhL}p9)p5D_FD05yOlDmGLt&pEU?Yml0rL6#RKD3&3J94fmXr~67`8pll znq30dxDB#xUrgD6`zh$8%?{)qiKutZJGQy!J<>bym6^~uirRmvqZRuf(eE$X&^J`X zNPH+pCvUpagf|Q-`rAOoQ-)CfnPxWSP&MtaWB8L|yS8-i7u6Vo5l6{26cFE6^G zANxBIllFiXq*-#Ec`o$xNj;|}5m`+={Ta-xw>y!ibQrbyE9~@eLn-B2zhD^6PfY5x z{V2un6RA1HMWd2)kgw?gtF_sUzn{N^KFMumJBHHf@N$>tbLMVz)_*PNxS3B=5G<@-Ox=E2ivjacURc%+;l>=L8f?6H@Jnb@T=yV=g)ZC<7 zG$rV+V^>kElqiyIk6?bkJKrSoYCY{cvjGL!m>@p^F_hvS#qJNxq*l|uqr?c|=J5ZX za&4m0>3ro}TJvBo}q|20W`n}dSyM6z4`Ezyu>4n-fwnOEO>sNu~c zZ0?^5nlkhYd4(QedR;T9zpkNENsk*kbwq=XhuvdFBcs{mVh@zy7}Na7A)j@8yPw`N z+1C_&^DUdvxR=@`d_eo2Y@teLwo!YDVfgt#09{wELVs(mW6rNtWb6m;Gaga?%)F4b ztjFM8>M)+qX7l%3o(mkJAPITC9&m=fA#2Wm1Jq&AB(Mnw})}o7Rq^z6Db)m47h& z<5#9e~0XppAhGv=WYC3Fj87FiMk>V0Q?W2I5Q9lvUPz8rN;6rtzPjb2+Qk6QayFrgddn?&QDqC+bW zA-}>{=7#HjD(sz&HmsGSdj}5C_dbHi{Gl(ZGq6WO1$)p%_ZoDIQ%k+Y70{Mtv*>g; zJ*S^lMN~3DlM4CuGfD!dXheSz4e1hYj{3v@9bbFQO%rHB zzk0ejQQP+SctyOYWDjLIch;ecx6(!-0o1D#V%xUTIJdFKMR9 z`ZDdCH_G1eT0=Fq{$qE~bE6H)ZyBkxzUaTYoycX|8Se7>4eUUbpwrmntu)L)fd06{ z_bPUBQ0AWbXx8m$6#4Kf`(AJfnqgFgf@Mz9ohyf!HBDh?U!EYiM*8WX#3las)U7C1W7B$UC5-H<^a)*Yog#XRWG{5ehK zr>8sZ5xqv$eVu5+P&gG5vv&#!Z)5GyNw(im1Ui`uGEQl0>ChyXPBr_|x0f)jUALN% zD1XWfwX%%glRc<5$AgNz(PS@u3S#`GJFvU?JBxx=DQMM;8gyf28P(z2L=8XF&{0!e zD);zPQ`deQW_`yVcKl&F9qb-xYEbxt4?R~w3h$@06CeGVDRm8QMoAVkuPXqZx*1N7 z9Q==!kC9-`uiVdkEtN!SO?O!1_r|DOma^KOf4Bi-zngArOtD68huD_}!p*}_h>cKx z)udzZ#RRY4jF$dQV{yX_w8rTqdOqY&wet1pD_gU;SMJ%c`xi=~bK82DtBKXLC`Q~# zZo479X&}oS{I8p3o6^wkYzwEJ-NQ8dx|-7o%%Of7$LRN^b@bGz5#44ajT9X3pv`}7 zkf+idx+&cOyC zuq0x##RzJ5vHMz1Fh{-#pbcUbG+@Uj#%NV44J@xli}ycg2Neg=+{2PiufrReoHZ`A zUfG&@%;wP#s|~5UpgWz4tY9vjm}0(P+0G7~NTWBhZ9)FHxYNX^?QEMCVW+w)xeEdx zQvZfLw6S%H9qgH;)18k~V>E*r*{U@U$JDT04jd%yAk0KoDDWw$Kq@#ZA2k=0GZCSf zc9xyRH8=PcTUH&-J@=m*0zw9WQ2Yv5K% zy@u;Z$ix#WIb}eFt6I_F0AD(H)nB%O@0<;xV{A!gBlwVwxC$#M6BMsblh@xua*Or& zXDgBOK-rACsarIME6$;|AHSm7hX-l*U!msuT8ri-=|%KC*8>H$>QK%1ge~T2FiRsV zQF7N48eM#aweFqQ@2n3GhZboE=&`_t@`=V~l0xCi+3{9Q#gk0>x;jvFZGK%Q5RjZl3cZ+V#$i zKC~WW>8~ty_~t^|H*$g1a9bZ^CB%jCPeD zlO}0R2b~0(W#v0)&&C{LyYn8h`nD0B)h=L%N6TqpC!u%m573gr$K1_fqRnSZ522mU zvKiCi9JIjU4?6e8k-B`$K--HYp{Az;-P3H!y6@7a(KTlnx1|idb>TYgV=vR;CudQ} zM^UHw58k0^E1DUd4X=>=jNOPM^bzeEKZac9E@z^uM3Mg;BNS*9h|2#MAiCo@ns1uK zy&*M|S${$miRTO8J~iY5EqF zk$)1&{Fy;-{a40a7udp@y>zDKjgRTx<4;l0Em3;kauj7;PNsEJ*O;J~EY?xJi-dgn z%u1G@XL8=lqLUU{bcvKGEn0xsI?K&;>AD5z)Lbd%Am1QbGhsjvPO9(|sJ9`rrzJGB zR+n}y9iy#77zqd{I)yf7)8myTs5MiN@l860PDo18*Eb$BW6wHKtYZN(I}lIx?p;Br zI?RzkcOE@*j>nZWKST{QT$%p$Rp@rJaC2FpkQ28gow3-JLk|%X=KTPobG6S?)9cZU zXq`U2VJFbs`11(couxP&c|EdKHp5JrhM3Iv}4Z5vnb`g-wW! zplh5|X#b!$8oTlq?R^(Toy(F*aNBNX1>b94`aO{TS9uKSJ)UF|c-i!6<4jbyEt@$N z{F>d;Rz=%R9l$GN5|GP zU*|qz6^|c8ub0lE7aX3ltXCF2QP9Uc=&)!$n#-XB#(UWU5lqLP-lB;s?=kahZZKEE z)tK91!f5Sy43*a@f`JETvG2__S)){>MrScA%aGnP~1NKNiVFp*sglSRJOEY1y}gmgN*uIMt0hS2fUc zDUX>u>7Gb?{}8Qh89{z~qnRU@=FkI%$xidXn=%vno6(f3ywfXdF{Zk*kY+ZT)B0-) zY=7xqI?4b3jLfF9uXgk>%-vncTIMj6)2?Fb()OSeLj_cu)kA%z8YpU=A?g`uVK!FX zg_|CZap&HR;rqfZ^i=ITW;Av=lfVBd75{gtsS{T*>%XMYq8b${F4sbZG%aZ5re|nR zP#B{a$3v5*tI_t~Gmu1g4VBh)VNyTysH-5~3>G`g+KwbMop}a)yKEy0`=O3(Pj>Tb zorAQU@8)p->Wt`S33jmKIY!N(Lsg$*U|JP!f40mDtdyyH+?ut57ns3(DKn!>`{kh=%->ReN~o; zWM7FnwW!*mn5egmy;lpmXr)fc>0ETL${OuEI>KtW8=>#hd(j$iFWPRlj+z)KQd6G> zcG9Sr`JQ`%$*A{dav$$v-v_Qn2A28kq~~I!qBDb*hVbo|uhY@@vY+g)pM&fwnId%V zvlzV-q{D0|u|{1=$tX+L0_DlxX$p+m&Gz-X(Ty)oAd8|T^sJ$?>CtOJ_HUGxQ;*O- zn%q*Vbz0tmlEzI&G*0eBO&*|FV ztMt#T-xP)i(TZ1Pxc}TA=IUwzl&jo~b{tuRR(&@|IoYvve{M1|@4myD=qyBeLJi1e zWik>tx0tH$xsD#Dav68$r)b@KV|u{Vo3^W*MR(*U*rDfg&4cxy&{TLl-BLNqe%5QF zdv#OU+IOm~bY}yqmmi?7!kptYIe=g<#7 z``Ih{e8*VdgYWEQBJTq_w06N+B<^{fg~4jH?LZ-V(`3*T`?ea*i<3k*g?wn@$i*fn zi$eNa@&z&zHbk0BgP7B91vFF31pO5srn4jEn*&_*Xm9mlv~Tf+rseboJ*FOxN?m`@ zrFwIl^OGL4URIab-q*Iwt=M_Uxv7th+;f3%;EB+9sUv88&Lw(LC7A|!YoY$WPpJR+ zI>ha}NzM1Wvzx~>sA7sS`*lGk-;~)%H?mJ?U=_>6wZtM@aWC||Ag(z(+Mk~KRYGg8 zQFKuIPgCVDdwNzg7r~(w^!O!Hwn&{o#S};O{Q`q#=qX0`TvVc0r8ZMyJ`FYLn$XPU zKjCYSc}?0}X*%7#oVLEbPc^+(qBFvWXoavSvw}EKz9PqHT3<&Wg|(1rT>-mp<~UQ9 z!MEpgexSI&@=mJtFIcVd`&7uiLJJbfn1O*CksF?ZEbL;{BfZLMZ@ zq!_W!UV5Utb>G?Gk%Xp9>0^xB6;pbDpFAttbqx)*zF^|bq}dh66r5ID_cB94dl4_{ zK6C5AZ`!$E4aJ&j(w8rE*sm{+Qb(T|be}XAy`CyzoaA(n?=NGnQn4)EEA)c0i&L0O zMV0W-&ny(1vW>2_dBpuO*O&IIzh`XvhS|fzJJEwUOPZ{!&&)oRj=EMBGM)C6-p!G5 zQoO!|ee-Kx)4_N9nGfN9Xwzy(bU}6z`s;I<-G62bweFaO_JriKhc{nlCi$N6Rl!Sa zdhjkLCu#+E;af{)?!SXRALiOpzZv=qwKaizY;Hz2cj4E?%iH|<4Ev@FC&cD%CC3|#NFOluDk#?%? zdd`FlsxmR=x~Tc;8m7y5gf%{Kon}8v=BA}gcFi#7Q>MmJ>^R-tRlc}RWOjILTC z%!KsCvO4u!xto7JL_G&zpz(Yo`teK#{cpah)3npynZu&DsIu!i>g89#IuBY>JMx7o znO%V1>OUm8^N!M!Dfig2!&O0v0?~UW_QnUw!ik6mk zJ?A|4Y44;$w6uhbj6$*s6>U)@qmqmgD)gN5+^1}X$k!-T6j8`1Bc$K``~Ugtd7b-Q z*Y&y1=lxkqMPp@XeaJ=rotq`}uh(3pw(*6Ke|G_sCYr`}n5LogGmUBGUllsXk<$f+ zzq1xO>g?#~MzpO)n{_e`p#$Qs{FnRc&>o*jsH#Qy#gW${IJZPIILgX17Mu z7dJCzzSBP1DJMe9>^|||nIjrox`1gtei_-^Po+mai>cTa1JtzoC6nFl!dKwfiUD6G zIXdqtW>wVz^!EHDRN5LvPcs(uo0_KMHE{vj`(p*`Qv4N#x;D^<_u81-9@5m>v7VN` z7h`-)beKJ2GVJooZf3ZUo2!?}@v(^%N=-6h{`zy zQ5Y5?i6kZV&X14GN*60Mwc;1OqIrrss??x$ha*_wnIihLU!rMi>>ld7D2D5=F46mJ zI)B|ccsEc~WVlbJHq_|uiSvCH_ zlq{5ca}tYwy=6irB$!G57vb+aajdx~pdYy!sg|C>Eb*~oPQ|3Mek1BMBBzvUXHP?y zTHZ3&?RkvG=go{wZw#HVDUe=Rpp3E{5rUt2bX`Q5FnysowVYE$?|tuJ>n>%pyZml4 zd$)(7M>0N?Ggk`x9!63rqaqZX){RX2?@+m8%h_BzWeWUMXnXKXTC!q{Njh|a7H^GY zMel3U?U%|?!hbK=-}+n8(1IhVBYZcS`20QlY|R~{FliSG{UGA_NR?wfX{4i~6BX=R z(?&MC#{s#WxJ&C~kFc!iCUn$6hKA=zGhK_~sFRC)(-SC0b+sSR@H@49|8sJ56@Q$) zd-M#`Cl`haL;gcIl|t!)pbJ!cdKFqWkc9FEw5fE?Cw8r$8{Y8f585|hr0M6RN9?@| zBQz^<9}3Rgjx^d>w&HUr^Qc&gF^-dAcT~^gxOiCjNkJRMa2Bv#@dlJ%Z$tMCSu$3F zO7tmXGqwD!!D$=TX!fnIZ1wmHcKkvJI%+TE=dIk2danFsWsAO~<5c+@xtaS#^CC7OoMY6~Y@pxPD>^nFvPQmL3xw@&YS7GIzl0ONm$52O zPcd=F(&(2pB@Ck`&A+tj2HUNd%EUjeK;QM0(K${lH?bICjZ&>>#CJ!=LH#XjrZ>PG zefyCE8fNjj{rS`0lZ7ED=63)6%6(?yFX|)6Om7icvYIramH8ChD z)|Zx-STc$AON7%T44Nu0?_nO87*f$aeyF=FnyL*=M5bHSQNWEk%%i~lC|r9MO?sWh z8hS5A+Z_6t<2P1O1C>_hV7oP?XWqaihsT&_ebrQJ|8FKU;SRIxmL_$4&!^6=$te8D zW#O9@OSySG4$W1ML>g}Yn8x$@XyMFKelw~>qvPXf(YtZ*YJ|I6=XfF!BJDU-Gy$bC zZOE2}vv%DF`010TQm4-f%!ZUh^xcK!sQXL_%D7g+96T;XMV1_7(w)z-<`25qE$cO# zPGA1Z&R+kV_VpZ~v#v?fub;EfVJk&;@g`Z+k(W**)C1X;GkwgT>I?iiuO6b9eTPt~ zK*I6mcWy_u+K=wNWr(gJ3CEUslaZvx1y+505h5eb?8q5e$B9iB=-UNF^v_$9rpvlQ zX2P~eM)F{+@R(qnKAIYXwp$D`p_7X!dHDrBu9}P{8_q{3-gdJo6RQ}-C;!;3X9&BZ zks|Jq2)esTl-bx+%WOZA&-(1%#B91&P8AeFsl!cw)R3QyBHgr*P*b!i@q!fs5=)tz zZ+zIr9*MMX{%?NPp(^@?yAL`I2k3wGWr*?mja1!&=@;ia43Ws7I;UG`-|02BS!>6Za5`b7tpj~5cNE>+kV1v>JCNO= z9kqG9o67zzVG~8T|NogR8~J$+Js5Y85l{2fu` zFrXo3>eOcIK}PO`1N+8qBC_K&#Jv0hTIFVm#?CHB4)PU3r;d8|zw^)O_|IW>S(J#- z`5;@EMYr!FAHsbJ2}nuqwt>dNiWc(+m*H{7nX8pYns*8_r?=4Rt}Moe;iInG z>zEI3CZb-RE3>k)o*yfHQTQ`zudw^<9p+261oC{+kG$pl_?MZ*O-}J~L^fK4Nh+%l zMs|7fj}Q1#M=A-*EPW+loId<|BDfI8OH@c?$g8HYG zpfjz#%ve)CvP}6%&%7$bd-lttD}m1u`7O)%evxEKHqLB%P$){*CTmeOeOEd!-jPvT zB8HY!s?Z~QTIhd&?lH{ePwb%k3p8V{8EvY{M=u{+QmL;Thw`uk((9b)IC)4DrM?)T zZ8{2c;?W95W5!{2B^yUCX$X(Ny^eX29{EPXs~K#MLbH`Vjfk%&Vnn(CcHRU#j=x{Wrdwbqq+ z`R$bOdCn2Kw-D0~yNk$leJ(W+JwxZD#-hO_9{PG$g}qz7o^GURR9zI%eH-?mCfLZ# zNloR?a+h)ZvEdk+bG)6+Fsr4qYhTjk98TIZC+=@ZhY;?_6Zb`$4CIp-^4@f*7VZa>U3&1T!|`N zw3x;BGEl?aXTmLK)7f7OA2H^~GElWo5|TaEOv@Lzu)LI`bP@zYxdJQt?RzkN_`{pE zso#n&HP|q1F1mDRWDTy8F|nYg z4^iSxh7!K?(=>@DkE|Xh)_Vm5)*qp+a>K%ls|8$7ZBF^sTj<5p95AZH0jbO`5@xnd zq%jBTD9+r9Hr-TYZ9+t|;lT-d=+I~=^7H9olGLs;;qo)-z@~0$(NoU0P7h&% zeD(Mf`x025uO}7hKS)hDw(|DmmCWJHt5ot_1*=%%PG>*rL6!^QRnSd+&2vkMUXL{m~U2?~`LQOP*1gx2w?AC2?rQ`C)R?Cll$NG2`_1H0D}m zEp6}Tax)8~(X$XSD!%(VQpFR{BdK9}wtEu_9+E^;*2&TIPc@87=w7;ITn*JsDMiy$ zz917RaisO|I?{e=FFfcMC0ym-36F0$in_{TSg(1hRPd>qzDY4+Rn%C$Hr4 zEwpZ-#69Zh)}nSA_0^5F|F;eO99e+;w1%j+Z!x_P@sr&ZZo}rjsHERE&qiJzQuLAf z5p=0JkCpmUj!cr`P{7Wotd7M@S|yqTx0>Xl{P(5k4Bv%maV+A`xSWVCEiI>I8LMcZ zOEFhJRD^lQ@3EB!4^w_Z8v7|}En{rH3*1q^$L77*%XU2LWtQAELu(7FXnvUkx>2J| zef26?+rAU@OS2vS%lBGz?$I}5x_$y(6EK6-nKu<>#!EOR%&kT$EwiXqSq6GvaR}KZ z7^3Q_k<2Ni$6olGN!1!f9ea|Gqwe$V?5DPk^e}u#MZfPvbN78kN3ABJgcnC?-LQVs zTr$zIeRTo-Q^O(-SI==-8)$9q8l+;hfSNB6r}GDn(utESn9Cx5Xw&0D(yr!Y;QqeS+-CvCWp*OEC@dqo}cIJp+}&Nzq6 z&9WICnN!T+&j*-;st4G$9tv#P=oH7M!pY41wY@Z_K*+ovV~`@3dA4EoBBo~om#6g4 z1M%)uA=NF%sq`aJ^tb;LzfD<(d&8kgOUINPo92OVmW?lD=Oj3=`{ge)>&!U@cfGjd6+d&~uGKy0 zN&a{COM(Qm@rDu8@3IK3OU*?UXV;MK#VcqwFMu&h=CS>mLiWDse(tW{CHR&-Gp;pUL%!ZMh0~Nq1Odwz5)i;Z>HOVTc83?K*N=op8sCCZ zZDY{?ER)w?W(c>PZUK@H3`mvdPkc5YpBEGPM^Lyw04iIS!(*;Mn!(dxlF36pxO=K$^V4ve! zLA%E+_#UgkKZE9=WP&blgXBzD*jWjRW=g=i(W#{D+i5J@JW1doZcHNDEXlHUUT}hp z7P&tL$(mOh*rDMPxOFHOkMkh;=MYJRBT6tu@esC?t`RKR#ChM%HOc)@FEYt_9QTy_ zLMwlcXHsttB|D=*j+i?|5BSl=qZK_q{EjLPjmIkf}eR)WX(eFL_&X3!&kk;AffL%4MOGq2Ik z5DFL0kc3n1;5VyE!sLEq;i3V&*E5Hw`feBuZ2ASf2P}E1H74X=fG=?rb%qI0@f66kFB)X-*$v<&4S;`A z+HuSSV?kt{FU}0^1--{wz>$TzZiz|bc6F~|c73|g*CZ22N6>-{8{nlu?2 zv*OTwpDj6m*P4{?H-O7KH^P7Qy?}|9BF%|5AjhMhck!MKdH;GIk(W`y3OeU`ub02T zMfq{$!UsON^YJ<;nzsPJ~8l0*55yx!h{LQ-;lRqm$Kx>;l zu~@A^`gF{R|FaO{x=@_(`jT*$<_VtM-*gNO)JS&v7INmNHCeu88oAJ347`q<#!i21 zh+29W46R-b?34pZG(P}#wH64>>h*zsZZK5LUJX)=!l4J}u^3+e&cU|%|Ngywyrt@e z;AI$qV|K2*c8=k6FyJ5VP?RH`BbC_K^B_33-d`|8-337_hIuL75=3!YA69g=g17Re ziQCy0d|WsMK3Hc(ZuwjWJu`OVgKLlAdp`|H!om>Vx8;_^$MXnJevvncE4_jfA|l{; zrv;4p{F=8)W;Tgl*2bHCV-ne2I)a~rC=!=B4i=2n^Ja8w;8p`o;_zk$q%arrltZv# zygm7-w+ELN*Mj|BVPu_>98}eD6qX$7qZUkwA;ye|P8Bla2n6P4pf%SJ!=(Twcl)Q13=awY{ zizPx~^P?cR?vfR0{JQ~u+4cmydbb!}8FL4^M;5^0sN?oFe*&N-dmYIBodc)+^us6b zR^V0Xx!{e*G&nWV54u@j0DBf)ahN2oO6JCjz(Yr3p|O=U*}Tk`m#*6jZYnN?;!`9D z4mbyUtCNvOe2 zlN!O}y{q7~{7%6fdubTBK7f>oeFyeGY@p22JRtp30`kHT96xMG8e{Vsb6y9Lv5hid z_H7q9UN256c85R-vI6_Rh$Mxx9^)gsBT0UNA(p!|hNXL^6CYI_(it_E=l4*O#76Lm zn$%h%_*8{AsV^t#SEs<%%yT^3Fag$Tj)x~FWC}*aY&h2NZS}b_11?lHhx6?uQn9Z z(S;{|8N-_a%fOvuA>OzG=ba zeBLlv<7N-T?GVi5SRgZ6A((i{h)C?T#`_jKky~%`!H(UFAbgRIJv-BI;gfKDeGkX8 zI@|&ze$FNJGj-rp*^h$j2{Cx~Hzj!bjs)C&W-2*9a36n*k%8sGMkM5wCA{rfCI}vy zLXv7^q5Q6GP?X1q${GqVY|I+2;JB<#*_!Z!tqHsXzGK&A3^@_G6%UuIj*vS#@U7k|6297$^qDBb>INrLG!RIZBn1(%rH}BrQ5&-I=xJV9 zOCs$38BgA?*COc(55YQHKBR3Yap8^8V;U(7HFCtZJwP zsh7ThGkI$8(AZ@lePSD!V0I0BO%oySvK3&t;$i%N??L9AoC{k|coF}+h2Y1(6!4#e z13BmC1vZ_D?*x>rxA-cto*}yEHs+q(`!vE`l#_B%q316s(M> z$89^-6Jdr>aOhnl_@$)>53ZCZR#T<1Qq2hd*7z65ZTO3~nf2rF-co@<-VZ!lBL~G~ z%kg-jGs&3t27IJXv3bK&d?4fjj+*a{51b-ELDPv07Wom|_j>U4rM>u?91kziTmiq` zx(BXrJc@5#Ea$CD`U`e7#|sRuoyKL`|6%^DY<#1GyLP6wfiH)(fS~jz{_~qpK(u>%@VF3oexed@8f+xb6GIaW)b;)R2G`pMv}H*#(})JCWyZ!1>=J| zf!XUZ(C2XmpNJREYc$y3pVy?!Eu8Nz*(M0PQUYkQD2;4ehMUAZ|zB*ECpRYlc0Z38ukgC z4PQ^NfWC{L3H&nD$m}Dt;kP3S@RRRN{9mUo`J=cOi~li8uI3rMo82*lG?EWgS-;&5X~8s6FB0OS&5VW8W8nyd!7L1OTH1;;F`>jw90=8zK3-#NUoR-ifdSYX=(;ozd3c=a7e zxOirXV6o;C!MCju4iRX7-pGohLnRd`(&+uV-jXcU|tDg^^Y z{lN~7%kwtlZDaVZFhS?VWGv|30}7YL!F?+L(WsdVSMStrT(iLr{4rZd8saCB&P`l> zY`6x_IZlJ3Wu0KN>wj3x_GJ;fVJBbplMGJFpsf;paVg*jsM}YEKl;=sSisH zg@eANrSRV)f5G0}1w7Ah`5^432U)~@9G}-cfE~T>gW+#6#J!A1CM8Ki|JhMsEN2TYwo4&%`s_)=lVV&s zUxDPiDMGcCoxtM)$101f7yR0>kz8IqgXCo=5V@7%WV+>gQk}Aa?6D}rMf;6$+htGE z(cg~$bqRTfTN*$_-C{B`W)B#;6Ch~Vp#ZDqI}zI*JYpo7Nz%(}c{g-7!(F9Q;Fjr2 zfo1<>81&{F?^)tmf#D5bc&&H}bYD7CFp1-RS1(C|??6+dR-p=!Q@o31)Z>6ujypUS zT@0>1nhnqV>H~Y)xNkpf2XUSxM%o(;iH{8=;yH7%Ol}m2*8)VT=^F2M=t_83!5up8 z(#1_n6-Y)s0@BAff$+vaUhSw14ECQzuAdrftesrTdA^?lF$YOfur!rdZ=(jq%M!1Y z-Yo+c%q#_>G8q^@(;_#0?Q!{6S&}069t0`blA*a4WM_Um-f!^UVV&zjBB+%C0||Y2 zw?_pYY*r!Y&p&*_$enalti{lpM-I4HlXbBs0=Mc|e1GXf9@!g8q!mr~;R7i>wt!(@=WT?)!(6#;G4SD?@HF|XFj2IjVjlPvo{awO>}u=1LLUzJWG zXoH8tbk$LiUJ*&e0vw1{o(vc*Fa!el0%!!U<%Mq;1bHf|0zDryl9lWWEzRAC+~e)g z*mx0aF;fDKN6cWhMhD(gI+e7}_=)X`26%}zeq?aRba)_fKwx&%1{SL>fcNzmk$W}@ zq+CQDhFlkifqpwllzu7+50NH=7q{at?|tFKb1Cpz)m`wWa|To$-bljkC6Hk{mE3&y zT5$4A4NjOn6TVT&5zMmlB{NjF;&b!o!s3%0ll_w|@u<)O4XZf5LZu^Ybvcd=&P?FV zNem|iUlllwhRd#~nMJC)L`d-Wm!QS%EGW?yBj&5B!1;(rV0pY9aky;=P1+-%mz+A} zFR&yx=U)SLE!H4@*GrIFRS16Z7m^!S=aFXp8lcfG1?Ty_#}DF6!0!Z0xbOE6_7KK8 z9QzamH_S99=Vxk@`woWK+qnc=Soy-K*N!*ph@=vOKa*kl0TFm6?;E~iBnmerJppTY z#;`Fg4P1Vp4V}xw;ZYeUxW}gkByU)32o89u;cd_7H#X`zfh$Xu$u+svyrzV2;HjnoymvvL`0zH8 zsOma!@U#`Yp(Y8MO5@1#&_Yo9)*Dn!GsT))-GGg;Jn?8U6KE}|;oY4eOC+oe1)nV@ zL8bKdpt(zfJTcl#UYko1|M3|3wrwWKO#R_7<)JNcN$ zNC#46zXYy&;|%}I8{>_Hz7xc9{coFm66ax`2Jd-kka?4L;?e>icsaEd@A&)*)1Gb+ z)T0cS%uNTb^Nk>FHh{bDt|2S5>TsIy3D5K52>7HTPF}|7k>&Fj!hDAj;ILVmL~Nf* zjz5$lmE|LX)rUs~r#J2f8go1G`dwGBP#i#yWX`wbp@E|gC}6+CrqC&c#Ze+oWa_U_ z;8Au8JpH2p{cGz%(eoi}!}a4|9TdRAnsiV}UNu_P{=v&elJO?7T#$9F3PkD0lC|0w zz;CA+cwU7WJaOYWX!VK4Y3)_GEzywhtiOWE?>Mg5k14!F5f-S7I+IwHPOwe)F_1bt z1O8oJ3mU&%1HTP87RJj?aPzGx@ay`F_w{MQ{8cf6&Hg?3r2k~HYlSCF*rfp9+$ba? z5_bSTy$JC-L&%&%urV5N2XSFy>~N?1l`yVv`i<_8t-FYex6G#v80ULvk0Eu~(c!Aa*0i5IyA9aU2y#@q0<3*i2fjtm0g4``@SXb(-oVRe zSlFNfce*!#CDC1t`7vVHGxIX1d#J&4$Q|DJGL>9b8{<76K8B@PcarU`4bKPvZJblC z2~s~K!$S^jylWvfxXW0d%#L;;ji)mJ);KH}n>7kr4(h}A3B5q~ZZ^1M;0Yr-ra`@n z2%ZWifE-+pYi*oKcZ($NWb83Kq1J}%bi9J^9ytqkola>q{nv~4tbGW;<6dmJQy*6M zRO8_~EgbtdUvN=Q8LFFU!JJWZ82wP1h`$Ji?o0r*U)#lNZEV4Rw?#tUK2KQoVIJ9Z z<0F2xB^fF!Z-L4iP^nBiy@`^)yedFPFtwW#3A#EV2x!7*z{clua1=` zzT#?d$?FUpEn-f<2}^jNt10>YqGa7JRdV859PIff0ITlZ25veHz_vUJ2VIdQ?NMKO zZ}z-stoA7bIyt7W%drVe;^xrY#2oxOZ3)~zKb2Qj(uB#rSQun;5=3m413u%fSkNL* zES5m>B3J@GO1=ZWN@Qb0c_-LC<34CRsY(_OPJntJBH?AZW4L1AG=BQ!0$9f7Bi%XR zK?bs#LH1V(foGExsd=W2k2c=NZsrehW#9}Vy0;e_*X_gNIU2BYY$oigpGvf1e2B;j z40v-KVYB~45H@KxQJJht1pbg{zya`E=Pu|FzmIS3=XjeBMDWSqGr^{P#{|jo6NquZ zR|kH00BpKvNe*4N=WXhq1)P;N;m?!5z@If5=xa)`X>Z+{lk5@^ESSr^cI`oC*Cp6<*ypC~y+)0>wYIcnS)p zq(NhWV9gwV^4C5Ti{8J0PtNinTSC);my#x2)T%}vh2~+)H$G&m$U`jdJcB&{mj>S@ z1jEAbcCc#P99ld|CV$El$>%H~A-DIt)!+~&b4vx9+L%Q>LJ<^aF8QhcIV z4PMV6_#ifx9Dd-X-+Zbi?@1!RAr#{)?X6PG`%b!3(mW04WF;O^k z*$WWH?UEFF*TH>2g4~eUjeRVvIGyq^{ygI^P95pMF6VZ0fP5LaHs%6P$F37n~Dlddk9D zo72GEbOiuegJ9w{O|syt4H3U*0XY&oZ>AuQocf(gK2`eQWS3od-1Zg@FL{S`HmVCe zV%>RFS54r$PMt)=pu-FsX{g!K97}79R~&%r^7FfT#fG+Ba#~;pwev(SSDD9jjy`F%k(o| z@Y;r$rt||l+ZOzG>3kTXVF{zps*%FvL0-VB$>h=}Ns_fUwXvz!l{i`K1H~M}_{H35 zME=tkQ0kC_uMZr*WbXJHPjXYlfyr}z;YizTF$egIpA5;!m? zf`qO9CwSVdfSX1S@DfUf@NZjVa=|+Ze}OMS&E+Dz{OT@z9ypSy6;sI{FBu{sa|!R* zyb&Bwekzcwe}%P7)roJfF2o~@OuAg&RkbVik*uhU&#3!$l72@-tt*cUpj7_c2KlyF zkIZiF;Ket*1&fYm0%4vsT+lxN_|ZJP*RKZmzcYjLe4{|7iU`>!atcdxjQbtm4r4jV z*LZTM1Kcid0N+gUgY(_uaPCt_f+Ba5eC2kmzIiLW^m;GOwY><$>lTu(^OH&IStZi1 zWCqebV#t%rdZ0h34CHk4NkQRalCwP)Px0@>-8*N4f_)0mX;}%FxL*qP$_;|pgc>lj z)D}vJcH3_{p$1KViNc7z6TpHAU-7oqxiE4y5ghOfAm6#nrr-G$yrK9jAb7}=H|-Q6 zr%aQH*2|07;F~T_=~x}U_U9C@z^nvF*;av7eQ{SNienIeRuW#?bRMh7i@`^%KD@s; z6Wr->BHKhy^ZZ#~`1E2I-n~eXm>L?ue`|Vqi_}d8l9l^`uDC6kWvxU8&bt$nc0ckt zl+#fO#O3F#dvL!9?7_o z40QW0f`hyN;VIiSq0ZFf0{yt@#7XuTcscJWFTGp}`j&G!L@mwukZvlkQsEicl=WCJ zc1er8{x6W^B*&7B?d$Kb*taWb~~4ES3p247{mkXzZESlRX`Z$`-j z!P`g!qHL}M|8$gI>6{ybUyxetVrB$CZpz0iw>pp;lJao;s4NgmJO_Mp<>0b~;c!N; zJsjy#2Js=@K=sOY?7L_Nu{guP^G`f*Oj0Hu&~YP&2Ae@g{Y}6el7&8Rxjm&^>HqoJ zu;4!xuw@L92@jo!xK=x8cw-D-cSyoa5mh{6Yd2oJL(E``ir-*)ZtW_814x(QU1Szj}3&J_BYK8j&{KE1IczAm*Ib>)8M;s1-AS(m-O}QCQ$t%Q}niHX%LIba5 z(G=*zvw~}iTLE*j8t+$_1fAIhK-qOajuY1;9Vg;}yqYFaWDWqy zZ$`Yv(+Rjv8VCB z{p(?7{bSy4`?mt;>S(Y++lhGh%_ENtdIYF6536QRChw#QcvmGS0(ZZASY@$>K;+9_ zARgucD`RKE=UkswB&|tqnM>pGWG+9$P?~UrX|UPHnOqu`hNU{zaB_$|oXKSxI;xnF zN`>2vs%EpH=D=dSb>BhW%5Br({=^vkpjS!o)w)LTGjR}qEh)p1-ynIqJ%Plnc#oN} zNaCn;3MUol0_Wo<B=yGlBNqd2rU|4T6!)Gx5d1!@$AGABS{^ z5EY9~UgMJqqug7CzZBkM!pv5}xEwjuIby<<a z%u>LeY`jI!j{I2?MW#qiC&%xfz{)ePf*ldxu*7zMQdY;YV0TP~deYn*39oLjOO)7u z`szl+o7BnRoOUppTZTcmCozHRxg0ck_~f`O)ZsXW>vYZWv#dgg@RKnPy?-Y_6U1W0 zvNUjQKfsZP&v4lT*SU<@#iX-16uw@Sh6lG@#ba?1(C~I1y1BFfNr~oDyKo;i<>oAU z#y*l}O<0M_%J)-6jXm^Ly%z$%o2aYFIrL8AGPOM?Pn|gLoRbbAz7dhgFF=--+*~cx zvYAeAC6%$$V|euP{Yj3^r?1kH_xsQsIuUvAeZ+5&U&Kg1jYH9Gqr|{Q4`sbAqAUL1 zVqXMwpbrIANXkQk3I0CFWUKvTw*TEvLw0;;>?1Q6n~W@4_WmFW7&wP6xW%FAJF9CKK&VpQ%lPr%Bj?MAT2LXn7wgVsTdT6$ zcUL37uHC|d?!!#7L^c2Sf9G-Wi9RG+ew!cBHYl7KBGObisz@~+hB!v_$0BvhmFzau z#!L)*&nU#bV80nOpfJ&iDF4wlsx~bfUESk>-mi+GwHnK){D3Vp-ZU=UiOiV~B9crb zJ0f(}a6%#HYpHmHg5&oa9mp?z9+f;UB=svhXvp=Q?6Mhy=;bees>m@iOb=>MQBMuz zwBb8n-TS?8>VwDZu;WeiFw6i&XjjwKKR3_=*Int+kJk9mSTtMc_Y`GpJwkU+Zz4k+ z=V;(54~bm4g)$O)sMqjYI@-b63@z9xii+`gY=e?JL)Tl;ut`oG+=ri(_>pm3;Z-ukzWqFbXFhr zzxlxk@&}j;Usln*kvg<6^A}1m7HK*-dz8I@Y7I*Ix|Wqy-G&54$*4klfW2}uj|ux* zKuyn>(iq!9G(WNg>0Feft;>H3U;OuriQDyy?ftMBJy^*@lF^x{_=^kuZ5P05R$rr4 zNz<4kYx!*0a(}kUr-DAboy$s1xqucZzGX9o%8olz#hc*T2~AUOl%j=cNi;86%yDVq zI=VK+pV8#Jp4kVVGwhRFtejgc0`a0vvd6NiK5C#l7yF~m1BUGKeYw=*P#S-O;4WJ; zTg;Ky+C;A;?w}SLi>bn&IwY-_#6%CQVehHbGTmZ5l*`4YSFC3_F6NkKlYTE@yW^xC zH4Vj)MGm1|U&ZO|r^kd6uA)p%r3LDrFe1#F6Tv)4D`FRB5*m8w6x#P=4kK0D$ewuK z#yliaO=Ty(2rYNsq}wx#>1<@s^oirGyg0U)THbxa@6v8UN*S*i(H?smdFT>K_m@J} zU*Z`**Yotpo*q_f;$C{VubJn+ql#|q{me9u>e7;c9QK0F5S54?q}rZhj@IVSsZr$t zy6$%>-5zI7FG~2(JW)r-6*CJ_NNNl-v^t(1GP+4mD9G@K>gQ9r*Y;@2^~KadQH|Q9 zZ9<=DD6-{?I~HpkXU4a$K(wolIrg)OFI63aj7ln)#o>0W)toqVu;#T;m~)J;?x{k% zOzn~6U^{*9e1u=I{W0_JND*yZK1|C;qNtHr9aS8ef(HLJ(sngWqqa+vlhOYtj*0u?4#K(Oxs{JU0mwNe>#?il=oxwz1Wtu5mqqze%-=e-a3@T?xu5( zePE>Hj__}EpF}=}k?6JjC-heS29tm1J!3g;hHTEwW$`LEdLXC`xh7(yu~LTSRI?PV z%n}B@F{6bqcGFL{b~1CfWz+C~$7u16KK}0y4aohzIXkhK<*Nm*LVudCvMWN1gq4*5|^sZfRo=-glv~iyau;@RE_$w?P}$TBE@S+H{%OLzH^S z3*EZ5j}3iXjGTrL-#%TMRa98OE?U&ZerlRPGa8B!FQ%5NssCdoYqQuA<73q3pbj0b z+J{V(Al%B-Et-XDA@j1RQo%Pgs4qczg$ zXTAtC|3xtKaJZPB{Fu(Jm^{t|>O4mF4@#+6L_9l`tBKs3&q9UZ1RB1~o_6(lv6jzw zp-(}dn4_yp(4W6POvsI+Xj$HLWV_-nUEw@~#`^~%a7UicyBx$06p0GmpGr1~ch5xf zN7p0Q@pd*bW;v>C;ry+-8Pt8p7~5&aNBb@gQK4ueI;ZW*9DZ2KM73Q;ho=;=t)m6p z*(zi0kJ!<+2ORrv{0%Y*6+@C%|M=EF?y$d7_p^G>RwAXrU+fm{zuf26!(7gELI)EK znaD%eY468u;uX7`m6@dm7o*YtF?1(>H8o)zz}wTJT_xK0lC-Gq%sq3{D($NjrKqSV zBqGs9Q7S2wtq_qSMEA@+lM+HI63J552)&48iTC^o=X39wGtV>ge1DW&Vl8E8I){?# zeua)l-X=p~t0}R4XBd~Yl4RL%9{R12g??}3lc3BvGRLhD8TNz{$qxsq+^*vcZ|*2M zwWwI&-+hBR`FE13!-}Y^b|bTg79#xU3UZ4-L3}q$BhUUww4_U#QCxk1DRO3OPwEz=<9Cl_ZSP09 z`qtFXKe9Z<)AGn{Yb-i=Y6iMVVJ{tM@gP)CF-j9#q_XV z;$=%e^0H}yku$a?&FjU{kf|1_nUzCvUG-X539Uv?R|FxI2PaXva3wRb@H|PZm`(Jo zj*%-LP7&!)8KihIN$_jvA@$9y3R%`95)E!G)&2Sc!Pi$&lX+K|aK|uazMc7rM{dnMG(cwt`#`c_27!alt-m*-n%gVt_`U3^U7Q z+^D#z+st@SGD$qPo!Ey@Cu3dij8g_*P(50W0z-2JI!8^3tc3|P@VyLGUbs!|);UgH zO>tx7YgUqLT35*P&8Ea;e<-m@F(n@}?-S=D-`3y40%oSs1~kTcN{LrjGwM<2(dd~6 z#B`4nGwu_?WQfahd0iXG==Od};JcYb$jMQY`$QS*y)Olw66Io2)W!L9bjh(Q`9f>QToZ=66*e^;7aLV}5%#nuF|7l;vMi zcd>wK_f{iO$8@MoZN_*-ggN@}$0T|B_Xt{ZeK%RYy_j4`Ge?|!uMj+?V zUBprP0a6)kri6anCQ)Ds(ahCg8qQxq5lfB}i5D}GweCCS!?ZQjw<=X;-@_c_zO;qf z`^S~sIiE%1SC>*-?^7hAq#gy&Y^0`jwNqE#-=!okHg2Nkl=Sv0w{lr8V{mFMvi>DUh9~QZ?dDp<)c=Rx?am?ZisqmV|D~cs@w+L7C*5Rm zR|EAqVJY>obsq_AlVQA(IZ_OfVrS+|kgu;T8FEn|Z3}hC*Yh)ZYK81OLsP9qeRbsc zObN7hWQ5T^pM}Oe-ckqKl*kA>7d{dDf*kFahQH zAi)zU(r>?j(sNHB{#r7~WAG#89>mUyZ52hgVt10OIpQS3Z3}aM)+>S7lX|qNwVs_P zyF(VsRFT4$6(snUCQ?{&fp8jKP<6&~6!hvoqW}0aCyVb;TNRHG+k&D4y}8pw{t?_~PK@Ffacex2QMT_R@{KVhU1QhT zx+^(VGfJCUka6A8DqBUkS%Cl!vfiS?};B)erw z@UmZ~qA(;N&I}4cvLlvU-Ti&R^7-6K-U^_aQVYn@kQ>l~C)Y=1~3) z1w`lB0wOuO8a-;rB~!_+RDehcv-N~o>&fy96z{1iI@fj>&H2zseq`?E%IO|J=}l`H zhn5obC22XcE*}yH+jWe8*)*Y&$j*%=>_>;{K9@CPzreUdh%K8~V{;jtvRWkRbMgSS0Ljk4`U0;@)%nE4ZF0 zfwo-1Vl`-p9l67SUH73xQmA&LlL1a;@UNYh>?N=m_w7ZO-Ry-ZFZUo+*= zg8rwZ=4ZPDrmbFhotqx z9zn{*7_yS5()ur2LU8ZfY*b`c#+t3qsFOn}WTL5JlmA<1DBNBwT@1VpVt|Bp+p(vkeM($nB#5EnDibsD0L!NI*OE}FH zS}cjQKA%F`o01V37(fTsn75joDj=Tqzfsj#1hMCKQwL9;Bq=(qWwa`XJgiAU(Q}i~ z$#>V#h|E5&O}QNNWtlAl&g&xu>92y->)83=EoTJ{LM9|LwURM*PiJV^2IfJmG0Em< zakCrF6XR+5BzW^y!bqecwa4d)InRlRClyj|2Xd+J%d|-1SP8n#e~yYrrV~-t8#{K; zf%utl$&ZR91TVXc^hAH5ufv6C$#^O9Y+OdJJxoH=4XTmRQ;PXuCBfX8RZRk2N0^$@ zi_DF@Urc#sH411gM;E?Z5ncTy{A*^Bf)YUhN@nv1+`c)HRzD$Py!JZtDt`+(Ev|}O zri#%1s{&;Fs18|57NI`YOBO^hgVG-A^cRQn^S;o5evgEO^BK1t-KKZC|jTH4tqvenHp|^+a z$(U6ZV}8DaT5eOw?cQ+$EqRkkR(;h#Ys>4XTX7{+bFdsECz6bcCn_o7?aWa3B|4Xh_{3w><66#q*!~F_-l}7svPRDD{{)nx zNg+D$JR3dq-NWp)TuLpAvPT{Y@93xVZd2*9!mLFzNnb0xM46t6C!1&0GyA@aGdIE( za`A{HiFwt<{7zU){C-#=Ro*+~vi_K0r=LIiv_}EGezSo%chS_pqkqYZ8Zjbz;RbUu z*o_)eN+S~%-IV&qSxo&@9VvTx9qsP$Br&jtlstPcFt%GHpyC}!XQ>#nPlw2#m6?w3 z_aSM+cGQWYk3_@GgtQh)A&&?eofAtU3z`mb16^{MYwW!G0-F~k^F<6QQLd+M$1Oy@ zLK4K|S0NR)@EY}J*qC_E)Mg6T(MX{rANg8-Cazx^n0cfBA(d6<1z$q4h{G1)R)g2g zl=?OkB-Xx-dC|X@av&SF@(Q7$9ZaT;!>+j1`TNnCK@=HP~9n~`knkf;59ofWHdj`3g z?2d|O=u^LxbC{i?2hgAro5MNq9jW~ICUDitW0nPkBTANSlznVK$5==7I=V_q9Ysjn z(thT1-CAtn*NuLK{v(af&87@B)2W@a%*kV`U(6CiY1E-2)Y|j2kE_z2#%+)e zqNoHNk|CDOXgQr^ik$;cO~4&?6vz^tSXn}Tq_a64I}Om=rkhmd>vN3c%NtNA$^hAw zvsit4bXurfUG-eQi=ASqzSd~|a@=v6ojWaE& zwy&b(Oj<6wex#Ynd83Z@Xeko)_7T*Zlu!J;RnV2}apq(~J1lD2L#9nLM?`Lr$j7(| zT=w`124n-t(}*hS?fb7}Z`1&#_{IrQwe0%(cpX}W3P_X47w*upTypneA$bceP+3$h zau;1jnMzrqbBTS-$_tXjivB^mBo^}Cq$UFQ6$dDL-E-trcRA50vT9Wp2_~_7a>&bj zW=zQ5LT=cO*}R@b|4?&l2eagd4XRszkh$l27!mPwKK*7+HEE|qOCy& zw$!%1>-5JB-$~oCQS{2xjEs+8rK&|fQZo+?FtFkc*UW2(beG&kmko{)KVj|GXTLSk z#?2oEmaoFdG@twA=UgweSksV<+YLi2r`^U4iKFWyR z`ba7Dc9G8W^1QU=yNIZTBe|-{+N~9jkX~9YwJazGHAknB-6!3-KAM&2OxarWZCbcM zj;f+kcY89|<+dQQ+=k4(XU;pC%^{APmZQ?p2`b{;5Rt3=%oItzqSj=5;-(~KsrUzxBwUYA+*!Y5*Z6eQZc#}tExbcr zi{v5^`3zM5O^ZAz`ON+1P>M#~LQ%m0%>?~RAg-@Opvc^zBSnZv~~$lS3U`0ZDz* zMyACUqMo*RiYJwcO5%H%=|UoaamQ!yJoZSFv8*~2WL-f9=IlW4wkx)VDQ1!E$y4Z|6YI1a9wbkS<&eaaE7WAR zJN0gH9|=}qznf*&C|Hlpj!E-Fm4}BIg^@w@)G`mLx5yHQ(GV0rEstE7mqWt6v`G8z zmB_`$9)$*mBW|W4dj0n}bxT7SwWRJt3h#nZi2qjV+nZFx`L>IU@^3QLs|=94b2N!E zI!XP|sU+O-Vx$rqk78VHnbj}Gm;}2#vOvX`{I1jyJl>B4$0fE91(<}K1xt~()*ypc zETXP+)rikQKuRkxGv}ik`uX%P5(_OQ^QU`LpGGGs#U;;)aiS+#l3~hI^>{@M+8I#8 z|MsAC!v_N2uhk^I-h#2gPpDY^9O}l^O6Hn$JQ0pCL{*z#pjo@-A+NzC+1R`)lo|H+ve13UDHZNjVJFd4pNMAqL^-Z0?8e;%M35-6+NT6Pg%M z6c{O8LPh20kemKFA~7LO>Lc>VtwUC=*Amww`|nB!T$Ux`L>YNBvK9`LiaIM5sh_Gz zWOJ1rS6(?X*4InC&@%H#BVY5Mg>>%$=t4VCdO>W{59TafY5N!{VK-x~pMC#BJ?hh=%yDw5jjC8xG zk%AJ052&F-9%~4Y-6SZR{gX-FYK7#^w=nhA;-uF%8`T_@B(rlTao?rW0@Fh`(Z@D_ zCbu<@m^}Z*eYSErq=S2y$k|TZD0Op`zv(o*%8XL{Q~4-^endIg%%a|u+#t@sy2)VQ zFRo^$Dg|!FFmlt65;cDLO5&j%4F4Oem^c^!FdypU@9>`Al(xJ-kR z66rV{j?!&SnKVT{g> zesrFZiESeB`CHJZw-=~Yhi%b`n?-1yPzX7(gp2m2JYuT5mC(_rB}jI&5s}Nm zRK2`0(St71nE6?0QsD7An&Bwh*9bSDzLJZdhOE81bsM8UHH!0`bWG;fb>1a{h2#8 zPjyhB)_a(2#MQ(iU@oEBT2a1?9TUH#h%9E5c>l>CWNhkA2&9HyK+~PNM7hBjnf9GQ z{VzA8BbBl|*N`vFRJ>U$lkGqv%ZkX|NI5j`xHoC=$s@E+61mjiM3#rVrLNu);-&2v zW@>r)?4I}@b5HjJx?HdV4RnYi_jBu!va<)FO!QjQULT}xzG+0M4$^4v_)Bt^Ze=10 z++btxPU`o=BiwhY56K5ImwB*r8;N#Ugd$d{l25~%$)_`o=+wDn5}-fK?ar%V$}$wl z-!D=meV_*oh{vIj<;G;5&ULiFYbUb(r9%qMXQSwKA(Sv{0MFnF^EkhH1*cdet?6w) zYqV`f6E4YQEb%yc7THB!OeGN;^(|!P(Mhgu-6eA4w{YuJ;w4n^@HU0K`?*KaH;%XXG3#yOfkv?^zd;JFax7iYHCkz^&9!AbI zCZq7RZp31=k9d@GQKR%h>Ue>GIoK6T?2k_{L+Dq?u8 zh)i6bPIR{v5ZB&9D!EOKaWL41#51F)vwjS+PI4eJ%dSzcld7nnCm#uJeKAFydvgWR zK^|zLp^7Z*Z>RorMWG5G)|lg*#W$kti14j)a;epptbFK4Sr%6kVMTdnfs`3p__z>h z_4|?5vWw)<$2ZKae+uY4o8y+%a2(xC+kL5YN&MlN&5 z$ijRNBG=SSdUdqWzX%4sFji?*&NU~wHD0KruOI1H&mc-&+7vBji*_YkC*wJ@$dZ3+ zsWPiFlwf0tlolos9o7Rpuw^+49OAJ#koU=|%$3wRnmzLyI)eh`rjrBlIYdPE7}Dj9 zp{$c!l9Y3Vxi+3ja&C=L4*ma8d82?@=I=^2`ouCD=6yt7fB97OnKMXwNdX{8GeXM(UmRq;xKGJourNd<;$*)E!vg^(~Wcum^U%A|dGI&}~ z-M^fMWFAYjE?AI*svbEo`0@z#xZn$=7SlrT<9VcQ=mxo_xfN;Mk!PT$1!L0}RNWDVPUqvTMRokBVdg?K~#pN*iuGof(`g4h^w;^#K zIxd*EP>dLEQA0uX%B_Cs7uc(056H4Z>|RE=jHGuCQsu$N(9$C{)QmkkKt0c$n&2Hk zRy|56zi}~f*7?KTrP4?49yyH;l%HTO?K(x`&zxmMO^nfMwWWx+=_z`s-^PvXK1Dx2 zeHMvM&J;`^$wMNh+X&2jE%>82$vyMcmir_?8R^!*R)>&_q;aJ;5-yvME;Ni%`_WTo z-Q<0E*I$bX_V7bvvOfh*+te6%=Mgh?y;xB3q7a2HX{S!1T9V4z-Cdk8Qo-gRPVC)H z1m{mt<*cK&`-M7MKfa6Wb21WXTcn|{GlEcG)(z(AsUwK*dm4S*AdI^6PobK{=dsFx zljQurY&0LFpw@Y_m>)o!T%p4S4-Vy0X-Qjf;kKDfL7+9+7k7mcQ_n&QLaYlIdJk6R%$m1LkRbbl zi-Vd34tMvEBLgp~!w1@_6ZJ1Bf5$zH`MpBov7-Uq%GPPUJl_`mdlyPLxp}0<+m;Bq zPcjylI#HzyOfPDC^Nbf*C)RiNVe50ykMD=5|ORsXCkp4N~4P zYkFgtNj9so>un=-dDCAczkG~|?wMj*vN|Y{92<1OU?!uV98GLi-sFbfkt9_s9y7La z516hmmB4$0ExMR&O8k79$f=f3toY+c0%J;v+QuZ(wA7ya@3T3VK(}AphDPQ0qEo z5J~q5qSg3?5%V=9Ek+xdc*Avs8D#TYud{3KPz%+xBAhxBJt1IzYOwv=F_gNe4Ry^L zB@?e-3w{l+AYXs&M|0m*Q7?1biHgr(fy_O71}E-kMl7{?)^I2=s;uht^d zm_XvRE}r6u>_9U_Yq{5y#ZlGLxhPa|DOxh^0;6|zGdHtnf@++*44oTyAU47-M1Pwd z+7kJOS-@t$9*UUBJD}9fwW;!CJ|20_M7_U7HF;VQ>!rs@)_?BkS9&?R-M5=N>sAsP zJS0UzN>>oq%Nc~1@}5~U=O}tTuF9LU<2ZH9_!g6~PzJqYds#NN;;jWgE6^sUh02p} zAp5TWM?};qawt@j^m@@I(1^Kl0qtzHCjnVgGqoME^ z4{i+>rd6L9;6&D!2~SL=Mf2w1vd6KQGo%MKXUf4zr~(abztO%I*4tL7iQ($kg?39b zVsP{kap20ygA%!Ouq$ZOJpH_AidP<%Z8el8B1UXF+9Lkpqrr66FMyPlpcl!E0h zJg3nG2N*J$H~K8hHWLFo1E4`}T<3udj)#t*wI z!Ad=Q*ikkIN}bsW3;(!5aAqg=cG-Zx=LF)BfiBLmg%tcQ9}hztd@<*O48E5<7d}r4 zfN{nifa*ke2atn7>4U)Yv@QP6%aN}2^n~+fvGYQK4*12V^>C-%KXAr6m>=UEONVtx z;cr)RXt`>Nt~;ZLZ&(88(%=k_GiLB^cMZRN`%n7F5g}UtS_fzsrto685>ENZ!fZJc z`dl0rHkZ!BBiA4C@84|WTX#y~yQ}NKsj+`xK=K{GWxX93KC_m+gMW)&w74}GBGv)M4$qyyYAR%P$93t{J$3Glhc5K>n%X&e0!I`4}XJ^62^gTbBk zIQiOU`pU33yj#5t+E*P18)W_A`cVt zSp@d0>H*g;FXZa`yD`rI-4XM8ejXi=wSR6&q~3PU9Z58 zWNA3)4d|8oMS-$l1zv55@Ob-l804M-Tf{@4Dm&L@n=(#E0creR-5Gc9api>ispGF( z-*I&BT7eo(F5Ga%2r@BoaKJwlZ+kt&&zTVi`#;*kKjRAgPc;q&Z&b z;)~Bcx&_jgt-vYAAilOC6dN|%!`x-WIq4jX9Srq(o!5T|d@vyg@KmN&>Lgh0r!RDDOXm7dW z9P@S+d~=f&&N-`x!})PojEcoC8WZ3g+7o8hWn$5z1@zl2Z8)RY0cxzC1YyNrfa>LO zI)`ecGjxhMH;q?e?R8Q(WS1|FpWzQJ;uT=jrTY$^kCcEwL=oqIb;11xvUGEf3^o*> zgJ;@2rWL$xApV^I*Ju~gei6F(YuO4s-z*dFW_`;e@fI+1ED?KKCS%Ib0ltp3flf7! zxKG9!hfC$b$EDeDtTP=q+hpLVAV1iDKpz4jf83-Oi;o|_O-IfC1`=h99ApI-s>hlaHA z%f)o>GY&Qz+CdM14BB*_4{rK=iSu-1ob%Sh2v+;3!Ox^n9KJoZfbG8T z`KB}VvEli6%uk-8?>=v)9hVtlEkJ$1}MSt$w_Mr7gMnSDTNxQ({`E&;ct zN`ekYKkT2LiC3+1!fi2**laieYTXWm;dUDE{AX4AZkG@ISpOaP7q7tG18(?Eh99gh zuwA?t0(-osW3~2fnyH=!nG2De;O+P6w*G_k<=7&y z>w-DV0~EbbE)yRKafV*)KCo?u6-=*-b>KX_PiJ5F1G35{>C%M?IP9ed9Ld(@QGV>`ecWOlWv-ABgP_!#`y+abAZAR_0#>w$HLT z^%~#6#rO^Iy?O+^{ofqi9#c#^ZJnau=IjERhpb>=)*u+QcE{_F%%%C6Ve@Z2Mk00f%bfHC?5Qr z&K#aa;|B<5m`LKF^O>-IiyKT!@Pcp~7as0fgmw4(;CG8V46plI?j6bjS#B?Bw z<65kMozdg8>p3M{FT{f#k9BcHgaRzDSpyAKop7%B!In<*ROs3H9nAPI3u{Pi!qJvV zP^IKApt4nIz3MEya9IcV5~c)&T^#T}RTp5EwF;j;BSB|9_J=b^E-aZD#mzv=N=Q~0BnpAl5*H$>7&B4tVEbszZ2IxG9=iHv-f(tH~;hQO%aE$Q; z1C{o;_N_Voqk4#67Td`2lAev#kH2>Sc`|T*b`Ln==zyu`GW5sqd0=Y!5_tMdAa3M_ z(Debc`Dce$LccZ7K=zjt;K=-Hx;mzTp87Eitlr4O-{U;^^G6`O=plt?D}SMzumx5L zc}btqRE8rdvtg&-W1t@I3^sfy1-(CC(x&r^@a`2}oTFehjI`Sg<6kAh&+`_-%MUT# zyPWmPax>UVmoLoX@?+b);TzU%-h0y-K2kG;xWfRe zDtY7LVShZ|UYUN+lsTNN35ESTcEQY?I{u)}3OsXLEVir(0a*$9_9t5vp_JSbXuIPW za5*rWzSK91e!t5Nr{p^0+7NxLnSr3w3o|&HQo$)Gi{y+xFv12GXW*-?VVt)Ddu*}Z z2ny94=bQAd#lNqArLzOq!K_0H9gQg4&op&0Z?`s#ZvMm{^e?7W3U}ha(=|c-jyjs@nS&da zHPa&jxwPlGxiE4n6!+&C;gvxXV0EAgJSxw_m0wTIU#iQ6bwkVP<5O=qXWw6@+ZFE7 za{?CPji=y3~Vd- zUf#OcjNe8d%umK+v%=uLt-et4tUPX*tah-oe+EWAhGPHDWx#Q40OWe>!lYMm;6_pg zeT_36x3_l#*_Pd$4+;`k(?c8@`I=!3e@FVTp&9II5QUVZ8}5kZbMCS>nNj>Q)}Csj z7u5sIm%9w?#*TqGyaGIs`Jersk4123xh4!S~dgF{?djCyj zaQ0~%fBLE~z{sot{JG|frS^K`P)QR^J2-;x>!qNTP&e?d*iApWwgc;|4+o0nA@H4^ zCyXxJ3rvK?v2L9fzPio~d;IF9`}!1M4Erl(c+bP#)@?1zWqqJ0>;7cUk%u2*`@qOX zL+qDQOuzEV!?(n_xJ70se1B>iyimc#PXZw9c@zfQkBtJA<`@{VXA6BH*Aah4+4%b9 zX&`u=H;cdx(?WgLP~oBu@U6W8LJW?Axg#`vY^yttzTpIYAH+eisVEpk>B5JGfv|uz zkhn*~@SoV#pts!x4*ToiY}yPC99a!S*qOIQMnw*%Ub@0uZ9VK)*ah6%UeZgQH0dVx zIf`^c`B5tsaN~do&QE2%xo7^M_@pXaR@g#27hL3Tu1kQ54l$tp-U3dfyE=AkzeSfv z*+ajl{!ml%Kl;*jMSPbp3>^zz0ajkaC&Jrz3mJJvELr?2PcH_`Ds=h83Sjw zhRNn zb`5pm&q@-4g7P^qC?$y&W$V#R+ZW?~`|r_((SLyR_1!RVaSF_s;fm#>mf_Viv*Gz? z?ev>v+VJ-eU0ASnHJrKG!eP#xL12}rgilyS!aj2sXt>A?>n1J7NlLfDqg*?@{^DD( zCu#=Rl4=jlqXS{pDn}f$Vme;tu%8~D=Hg&#>w)9U^1-2sb#S};DF+97`2j5?$56`hzpHh@LAn)2?ddAI-G(PGKqgBRe_&f#6 z=5TOgW-#vj<_i-Kwu7(b=jg@r{()3ILwc639{k(47E6VtV14fb5cfVDDsUXIu%i@~ zvCzfFX5#SJl6f%Vpe0WBYXsXCoa3yt)}>{;B5?g_0A;Jz!F9tXbg0^EdSWV{&v`c; zz6msk67U&VK0kpz-@Xk0j*Nuo+FshPsg1*3BRpE6?H)Zi=Osw?p9}hU%`G|_AL#7+ zI{^<2Q{~_}q<1sFdIepKOItx#t;>lU{-k z9`wZ4``7iMVocEUxEB zLZ$nAIkVU-hoZWC+OT_w&er${hC0{cI;m3Jelv%)p|-*2D=6q0U=N!Eobku>1z2)F zyFO*i!o7S;kh4Y$B#xKRRWY++S*Zt*nMi`cdiog66UP;wE#TUc$KXK*dkXlokk;Lt z4X$r;2M!yH9P%G<@uI;J`uuqS*Ch9VM_XD!{t_*m5Hm@8SqS0jS6AaveI-!YEuihC z%5nP|H+=JUE({A_17}m?^tYwKcu>CsM29VQ7|_tbk1KlU_eqDr#b`a)ut67|w0jBM zzM9kh1&=wSzpbI4hXovdCP~--Rm3KaYax6R5A=nVvBZvLc=na8P}}(h*r6(hWzXl} zQ5#PvF*h9EXL~PvtwR3iiyrVw&2o70iyKzW_rtHFByp?(5lT4c@^e}Z!Q$K{ z80lKVA77Tz#hbR_J`o;{qBg-yk@fKM4-I^KU^+I)u!7o$En!rcEBsRbm)T?62bU16uL~v;HQ8k#WoJg;5nvQj@b05oz!0n?P81Tvg-kv27mW*5CEAj@ofvtO`la=wRsW>qp@bs5e zbcIAT9!+0{9g^c<-6sQxmLn`k-Nr;L1!mDxsMhrp+&kUI@BePv za>vF1hTQR{iyBuryxB?N`NF2~w*-atM6|)S<5}R(8bkV6s4PDDc@B2aNdsRVbLh!1 z9o+8Xh`l#xgSiuP;7dJYILN+_7HW;0Q#Ofs-G(oqB_jow-Fym!GM7VpiAJCoKT3ZO zyhE>y-w$3D$-yT_g<;zvd3MJ9E$C)#r|vubKm=%E_Y6gR>0toY?{>lAXVq|S3lFB4 zC_>8$wr3kJ0Abz}_AOUd!edX>@s9o+te;o~ew!j#qJ!zRtT9pcvXiZq{IRH%FB~ZL zf+t)pSYMg-16Iu7YjOvG2KOo*kT4A&^s>YiXDso8ReeBz>#P%FjanqU&K;xG%Y^dr3AGhy@ z>!SBT621Ya@%-`lhy;{*ssRl>o`Zd#chFhw;gIJ%4F`L!#xDK~aH#Tbz}TeQ`{JyPSzNf!5v_c$7hp zLH+(H*t~6C9~ z@WNhcXk+e$Z!AxSXnruv&aJ#ct=nti`Vcv$980K7Ko4Luf=(aU7?p&Aqc3+1XgZ83Iu_R2f-S=X7I z_B$Im_bgY#*Y{UK(L!Uq>RAB(Wnqi=hJ9*zycrVLUW zH^RDXL)f)K5sUwh!&jFar3cdn_|?xoagGhLYw_(JSoVz^);t#vN4%y(?ZIVGXXh$= zSNU#Q;`17;b?gFYpZk&TuvI_@-iu(LxqyCZ@&vs4^@gsVkwhnHror8#j6<$%0zQzl z6yLlgg9A2pf~TxyJfo!-X!If&Q0fK$xZ7aAlU)3Qa)2VI)$x+`URcXZ7PQ7a1V7hk z;rlOU;GlS4I5m+5KTpkLb0Q+KdeUmVC0HHHDZZv%8_ehvyCGcKr;2APX~Q2nbK$1z z^Wi;JJ$g{clascsk$xDH2OiFIgnCb>)9cU91?ggbxbCn9n+xHHEvj|!=~)brNZ5tb z4k4@;O=uJ8UBF%>h2A;u2|adR2}-`a%Ml<~9P?)f&^3^TGr#xIPcrOqMA-<~g#uvU zT1zNg!oj+xv$3x#&9Phc3sk+FL%)kN#ENmZS}5ZKob{3>@Yk6L{>qmjc=8du$KU=0 zBuhr&vacOWU#lopq!6T@uAXw;P`VuSd7xMXcj@;I1q@ZD@?8oqVw^ zzZt9*QG}ce{=ntF1>QJpo{8jkl(uq9`gtTp5l)9gN+W3-kt(gHjM(~ zmRKD3Ck$Io_kx2rt?&+a6&%W1Fk3Fk<6m2|ImT>Gla%#ipyhJcK@aD`B~d0&wm5-L zUEdGVY-YjaSjbml&yMthhiHk8`C#DPOnB~j1U6k2iPd6saE*TwOc9}=8OY|YLH}oF=7#wkDJ8rwe&}(B^gZga`KX^wqeU&w9 z{?xY9siOllJz@(N@6p6H*?RbgD;GB^1>hy4^)y*8N^5nha%QG%#lKx^IQ@2rT^BmQ z3+3PR{$*&JP;1A zJ(YzkX%8$b>51iY89I7f@RjH_7r|jA8R+M_4VTxakw^TKt}FAcJ&D{T~pNzL55;HNqK6F8ITa zd^qpwCQc7~d)T==7aPhd!gVS#Fh4Me{~@W7Kbmd^o&~0W^G*G8mJnd4k;R$+y#OexhH+Lzw$Wb}h_D$#FZpS`-cWKr56B#R4!&hn0E5g@&QAAX zdeyKE=D8Zd&PAtzllFa1&iqXn4%oA24`J|ftq#1xp5un@dd>N;;yJ(FG8(tZ76S36 z>vSVy3vYG+U^0INywVRK^WP$TRB0M~`-THs^Hkuj%{F+7lf}urcLDGpT0`$VZJc9x z4{Y;*Fw}{HbBelX?csBre%mL3lVe+K6ELbq-W~aD`U+y9Yas&A};Kd06uOD)@f=I#?f+gco&W z(ehO;F!6{MwzSv80}pP4WcEBP<8M8EwLBA#_j}_D3Bz>kxnf#zcP%(>$HRRdrf|(} zGrCsoCQa8QVG*VsI5?Do89sC9Dt2!kGgieHu-5j;`;+t@9S5wKB#w72XLGoYM}cV8 zn}57J3sy;cLIT+Qh-cc^VMzr&Kg0=#t4qMeC(nZT1v20r48mKpR&lzn&c(q);ka#@ zJ6>2E0*5CJVXbE{T>fh(6SO@`=4Gb6gGjc<#VKXSQK$Ur!h`sR6}B-qFTx zF0fW*7e6mW9*YAFT)Srh?%Es*XMYKW;`UCsca;zRr5S>ij(Xw+S;4S$)(;T)I~~qR zy$!5C#o>czl(FR&SD5-k1nmEpfOipT9Ne9WvmS21krA%2>Y_gQH;uJA!BH?@(gN)c&2v5&d(OqgEP= zugx>Zx%Ejfv?mrGT+Tj=aU*#C?thBT!=J1Fi{r?inaP$>Dh(t)pZh+aZ=td^HMMt| zN;{jZlo6#UB3Xr!&%N*Sp&~SBNhQ(H($b(vzx(?KJaF%QkJmZp`5ejqoNvT>T8|N) z*76iSbR5T?VSQPn&l}n2GGke}8z7Xi%&gkNA6-W4=?R_EhqG2A-PolURfUgU`3NhI z%w$#0F)XPNv9DiU5C^+%6y|@`7q;&45h@yc3MVgjX6qrE{m9i*Jk& z9?O&!YF0N0J~uxUTz{eFWIOe^KzZ>Zp_|XssuywMPaKcr0!3337amT^&!ds1uKl}&9U-U0R#@}(Xfti$&cm;rzhqa7Ek9TaCm$Ox{?_nMoI6P<%&&d)d?Wdgx_i$CI_ zJrY5G-fH3B`#Xg}_5$JM{B>;dz6I=@Kko!LrpgQT`CNQe^l;X8&vLdYZ8qzfyH@C_ zv|hOTp9|aXFv+Q7pjw=w!`IVmHnMsq8`(3ib>f(LHwD%^7Yn1d`U&r+&0;tIb7$u| zR*BQTs<98|-f>>Dwr}!9l>@@!c1E~yWgvTR`x-W_?Qs=1VxDjcZ`2#_7$_X?6)Bw2 z>MLB?;mY>ByD3QcpewxCa7)m#)t7yv7Q$XRXClmet02rOR}o&z4j1@^?G-vr4i`?l z%CK5qdaSMX65*|r0m7p2>w*zqzY0d}QeZXsJQY8diC;(t#cI1s1iRg52}Zv$6@H30 z6+XLSBdG7z5h}{Jh&Ma`6L?9X8-wkl^Zf!Y*7BA+8xenQa_y$nG;9&tBpChYT8>#UBqmu980ew`xb` zP(eY>WFh1firuDd56<;oL653Nw;b8R(@lpdX zv5KX&@M6ji;hR=%ex&NPAeOOX4_y(kLr<9tUAMOjj%s^|?UyfSzwE3Llx|!js5NtE z`(G=u8e@HhuScyFtn5}5&U2O$WFC4i_QbNXoU5#{vdY=vn z_c|-E>63MZWiPDRnvahKa+|IS$VXXWu2qwm`PJdP;ZB|)I}F>w_7Ep-zn#L2S1%4FGP z)R&#O%S%{9SBd8xR~DbuonQ66IayqiF`MP8oY?s2Q+%)11i|CRF=DN9BjJ*iF7ZdJ z72?QCd&SP~#o}MPR)}98C>BfnTAYJYH}X9U&CX9u1j6+*R9L^|M&c2MnF2QhKFjI< zEAS_ZRj*twiWlDJ-Lflvg@qHg2`vu{I*ZH&!Yj|li<^t1gcGfd#f6t{3XTjNBK$X2 z!#Q<@ukb~Rw{R4WU{6U@*`#qiqxPi{Teaz*K+|y|JF4%R*h{P^tdTHmkBYPN?m|~~ zo6Z{claaiz`|56%ObQpyQsi@B%_Zz5KKpaKHC$*>Tp{Rwugjj9Gnais?Zod*2dfH} zZWBJ|U3(3ZEo@2jSMiVd5aE@h2ZU_!I@VHP%WCp#4(n$pW?x6Le}o%YqlT$M)%7cc zQ~R~qraT9sV_AbZH$#T~o+@OGrvAsye7a0n=#(co_1=hOM-5{WszwX#x5o)0hEjpa zVQb!{isIAbFN@EuG7w&#v`FZvK8LNB6|tY&{K4Gi9x=KVh`TdJO498PLQ>gInCE6s zAUYPc4jIt_t-q*nc^vH5KTa;bno4cuI)pLPTkxODbcin;3y#J`+@VVjBwu8TZwqhJ z=f|HB`|f4<%|TNXsxqAF1(bkEAOj=Cu_#)%2E0DL2XC`NQXQGW?0ho<-dwT8acE3F z*0zvw`KM`SQ;fh^`WeYN-9i&5exO72L*PMU0Tii^MY^#W;vT4zPddeL+3O6o2)Dvv zX7VDhTg_obJuT2mCDbh@u4i6R!Zi z=M6})=RHnl%oCip{2lXo;xZEaLxd*&EGIf!iD#x0$(fVptd3eW9Tp@H&2rH=dA=)L zag>Jk2V=nL<#Ooc*%PbB9EF9iL&>!K0!S|9VBY+4>`cEzcI>^2kFM;3EU9wFD7S^_ zoOP5ogB0$0{Ei+ucNAWq%faBi`uJggHrF}(BqUA`L%sEp;8othn4BqvVy`x#!iO~Y z!_T#)FO`R_+zD8tnMc#jBk<@H19JDP7Z}%ivrA3nM9$Ye@L-q~f7_10{w=By^G8~8 z!{9tU9}$atZ-2(xj8)+BzL92$WZ{(J5qP(K6-W8Gnwb@S6>YM=gUR zmIAcDQON##^oVSUmBkqjrTi@N4w%vd>=N=66!)rv*E0oB%Daxvr}-8;QwO}iHyR3m zZh^ml?YK{Rd{?>o6gueKi|aT!%q-Z87H0ym@5ObpeWVaSI}N~teiaBC^^2KrI2>;r z{!RhC$blV6IMT5SeXg|-^^D(8Z%^=@sU<#RF5sfyLul6v2_2bzmQ$47i3?R;^Rwsu z_-k7n@{6}sVj(_`xIg`_c#XI{-R@cEW?b| zjkGUE7hirk0k3}y;=is$)ZMupT36qLw&LS3#!m${uXzSct&Hf?^;hh|g+?G3I0Swz zR>SLC9AUyLJrcLJoPU>Y)5a)Mobo-LruzuUjrS8#GoNP_EZs@%ZtrBL-_xT1p9;bwFM!sc5E{HQ39BzE!iRTO5~T)!&3{wjR*Q((3q){n>u`}w)?ens zlni+JqZTG!c!S@j{vp2moUy>Cl+0VQ4_daVq1DPzbji9-=0&){p6{KE@ukDG>`*Jp z%$)|~r6SQOJqC|oVem=r0lb!^OK)b|;h5Sr^po5WXe~H_d)6Cbf2%RM0P)08>M_>d z+ebFm|D=Jn@97#t4|;j)MT~EH%+DaqfLGfc`EM}+UVk}6tTp%34ACR9H%1L6w%p|| zxh=$k4~cLvRRFIFlu@Qd2}Kw)t8O9JqY?Jl?29UWm@BX{R}m()j6%5pE#fp)9px|Ig19UXGAK0`3Z(qe ziMqj%H8wEiTQYS0D}|7p;cF($Qq= zzeW;#qzRY0)Y5As5zlYhMsCdw=WOp3p=iTzx{}!ruRkus?tjn7-z~D>@KgFDv z)tFKDAAozq88~Zu9-bJMQNNEf;fG-``SY_Iw&i+5K}ja;@}rE|J7uy1ECka)bpTyq|?=Yq<6cq=wu_W>}9e%Kf z$PaLIw7MJdxnY1?!lbaSz?E)p>?9k0?87VjJz%{>0^@Q_AEP5)(h)BVFz!k?eCC$o zlNW*593oAxxF_R7^`kf`u>>-Izr)2$wvB zUzVc(-Ra_6$Axq(&)pfXIvEYiY*6J$4@CXjfE#}vMgNJHah~W1czr8?ciQQYU}*x4 z6C*(R;Zhpsdy?Cg--*wjCW7;SnfTWH6OCn^@zs!Bq~uQ-4cV=Mz9;L+tQmW8=h=Dq zVpAz6Xu3$JjA;gw0T28b{FX$Tq(EMe6P08tz~sbaT>YzrJb2&4Jgc6H8ZbswTww!q zqjr)-s%hjR7lC`;fq{X9$J zvhNxvW0Z{T1y`vBcTymh=J)S2vf#e2hCbEN=4bJ9kT*NR?Fp&4^koO!XdS?)=dZ{I z>nHTw#ZqedYCW8|en4R4?E>@f2GaT3+nB$r4Y@6^<>}!%K&v`8+B!V}Y`Y`j_Z<(Y zY1=`C51V+lSr9&G&R~A66?6aHxmsOe*@Nj}GNkOKiKus<3p{+9jWb&f@p`eEIK3q-HFl@93`Mt+SP566H0 zAjaH0`tnR3hMXOZs~nc$(lf1~J985LcH0Ws_p%}9^f7MO?0Ll1M^<9mbDyN&6BA+_ z$msT3;?kP)0KOR@6P*Ez-yOr|oXw#5(i@c{^ytIRdEBU*Ga)Zfo$-AsVKlZ`k(ZkB z*k7t!<(;^Zn36==w&6ZwSo(wXXqv%>2fCcr=)>snT#vY^Rzm$%#O1y_An^H72r~GF zvlkd(S8)fYe0LC2^1i|8pJu2!e*$~bvXFZ;zKUkuo{hE}zhd%NX~~@J+VDdBf$I1b zV^ip5Cg}G_td%)WTBD_5_Cf8dJ`+ZWVx9j3+Z9%D`LiDMv<`!X^(LZUdykRriN~ql zjh}Q_z(Ewg8!K6FtAkg^pJ!G!aAfX14RlQxefITMys^<_UW%xNQpE!D#@J7(f&pUvR!p^vUlZ6L%fj2=&jfynwe82vIA zgUbg9Q+iPIRk7AnLcppE_O2cnYe=$!o8px|H$sm`OizjU?U`AyL@rpJUZF#Z^ z?DyuP%2ivU>sCTb%hv;=)=g`t2XJRT9ib7jrbO=9cbKX0ls>t?1x?9G`c#Y1j>Gx5 z*wh^!JM?k*r3oe-=^=6*P0SCKx5C$%9%N$31$4401xlvVedki?o?i)&vViBTdLGA3 z{yI1`dVn0%D}=z}8!$!w9jMhGr|bV{h;%2U(+lSGS^iG-FN|9CdOr+ooqq)lwdh>PQ z$iLw<^L!0iliNib_a0}SyJgeG{QtBwJC0iKosD*1FM)552(Ao~frTM^NR9Ri>f;=X z6>m3!xrc~^xS8$B%&z9wjoC7| z=wZ4Aa6$)KR^7Fzp7b0b#YWd3swA?DYXF^*O#Ttqx2&TC4@Eo)!g z%hZE|)K%yS4uzQy&$16o!nrL~cd_B>ax(QWMH^#PxN4F|6id6&$is~MeeoFE-MlcT zb3gX|{Q-ZJ+%YcpF?+mb3|VGwgIlwC77)CmKDE`b=%5G9Zs;ODniA^W!}An>s=|^W zYna;ffsS^oAQ#tW(-SKuQaArJ8XYxVl9HQ7dS?i{6>kBZfq2-VSx4lQ z7UKC^Yw3l?*C=;x3MYPi4O+z0@$0${JSblz^c&EGbWLg09$^9;R#W#2qqrH1J5aS! zi0>|ZBVse2ja97MSC zGmI`R0jsx~(0N7CdAH6p{IPopB>cIGc1JQmAyij%<8?m?Thc|NY;91ZEA&qte! zF9qROS3(Ii3qtj_Cw~kf zeF&mt+pbr2chA{Py=_xyHP+fhdv%Uoc>nY(z? zV4jN%o4c;o(Q2d|fYA ztk})FA$ZKR2WomPP~o?o9?U%m&E>{);7T#>bJ7;+I4j}ZH;ZU$;$Don zyanH_Q^E}6MyhOglRmle2|Nnd!#SO|#E#Ds9;U3u<-esRVWY;QkSrqai)3)hhWRi= z%>oNTN8rxEbSP7IhRq{Rk*;}BM9_2qm)f3%PFTVO-%%9R?G7Z5_lMHd@N8PS6*NoTk5<3bk98^ z+UE=;O6&<@6b7`rGYX?az3EVylbGI?0H2OT(eC_lq#$xWEM9UE#y%?omvuwvn$h#g zKyNvI5Lio?Dn&d#e=T>#U?VubSOLlH8|aZs`TYA<4f`uaqH``r*k!@n{p$4k%?MH&yP3SvNvGW+4%EsU&@#vjU6w@AUB(1{JXggx)6H?c zjttLAcnAv64m&euLCC2tl45O(ixb!2u-Dp>a?@ov`=Jl{m)U{qJ2a@q(Fb^AqbFIr za}@3#UrH}TKcsx1OLg?Mpvd78y!>{T*~Z^PCPQh4(iEOmFI z9-~%}?^hzQeXj`Lybgn;=O>8Y+*M>%VKKjirNG!PYK%{HDn5NS886hzi-z__Lc07l zh*u0pZ=R8`>7yB%EKWn)a|CyfI*(aC29k~SrR2Udzm}14IQuVU(_g(pv)y$hMm7~~ zKHZ_hjezCjPD8`r2n@JgKn=Fk(B!p+Y?qdmsQgS78tM(_%(vczB{ILceNA;_ldQ7j z{!MFGxippZ7Dv#g%|YDf{rc4KSOQYnS@`SIl%49CA%XvuD z^p2tVe^+5}-#qvnq=Y4M>(F3k2~4-&1v%KlIMnW=&zNHOxOwfvgd?tC9~R4XZpnftX>sse^bnm{irS$@*qde~ z(HF!)H_u`Tc`8JG{usL()v-@_j=`xg@x8V z^x}qh@E|{q(ngKHt-=z1Z-|O& zBx;vZpij_=+uv-0Z|gE4s=Wa8wTDY$rvBqZr{)n(!x{~W-+;j05j~o$A@-Xs3|3Zf za_c@aFA^$X)$nAZs~rpX9BUz{-W80OSm5-!5bzk7f~Rhz(I9pywQPL^#W_8+S-GDh z`>KiQw>Knjj}rduW8w8$9n$f52aGeZz=#KX5&mY9iMmOc-w*|hs^yTiSPs_AK2kd` zlpIo}0q#f~{92cXu2)as z$+ik6dh|4C@0pDH9~Yu!moAL5`p$$H58%|!JUVt_13uU;f`f%hR3Pn0K7S~I?%V0K z?YI>L%@i*E>`$Z%ZHM5cw|Agex*OI{ zoeSQD6XEW*N~mOPCDXo}K)ZJg=eB1JH!lmpvN{g-ZE1qh3qO$L_2F>n@FhC^NgYUT zyn~`+hB*GuI%>gaF!9QoIQK{%8I(z5C1*+*@*$h9F;AhbYmcLGOdQ^cK10Wxe*+7S znPF^i6|sAGi0t~N2y*34Fh4$v+*5dA(c4{mInOr~}9f_Uc%Nb}{hkLyp#a1$4pnLZzMrai*0Of$^>ybG&+-qo#3vSC`di^S#u~`Pp14lDWYs5IRypzaDreJeL9a!ki z2NT5|xVriX+?x>!x_a5=owytS};4KNNl${N{Qe{w7g>%W+oM zciz467RIEc6V1)}#GobuzVYRRB>g9P!1i$x8F_UbJ@t7UY|uwx<;0UP@VSQk=NgLFQ>&v5yHDnDyHw`T+cwKEZNnCNKz1xSn{gI=a&q7aHxxUvPs7ycP*6TU3hs(i z>5)nwm~x{YQ=cn|4g~OQ*3c^4bEX217VP8hg*D=uH?tvxuZJ{l2|=l!-MH)hb(~dQ z39qFhVdly+uz7e8buoEQnk?VJcXbjJM$FTM;Cj%!Q0LpfN{BGUsj=3K+R4c=BZfN#bjU z6}&IxWsN(y7NpWBO$DA2cnyA~4ieHGgr8oWfT+h3>Ts}LIPz3Eo^ESJ&2{!9gK)q6WfD3>B*I5^u30;$Z>)%Je__KTxA{b z;r#+GsllFK_btB!MCH{;B!a~$0=`x^AmrP zwUt^h>!vCWj*Y;_a^rAc$8@5}vxl5zIPzs!I_}DTLv`;M z`1TWWW2#8e<2iKqH-3z_WHc_eY~`4To>(V)9jmN8@WuUOpuDdRwC9;a+yREa0e%A8 zB5T(;i8#?m$<;aMiM=b817v{995`vCNZr$sGof%s?CTZK|a0Y z*C&>C?A}U0w))aLe=5M(fkJ>y6UY}dVR={swTtF?|MMznVX_zOne&bbic11V!x?CJ zSqm3G5@O)PQhWs&_&N0`j&YcS%STGXwn;bOUE*CBd{qvc*;ejlLN(dxu0W=?u4itF z$Kt=ehwx|K5&Y`9mFadeAhJKYXv&O4+;HO_`O3H8k9|Im^W~Mj^NrWyW2HtFJllPC4wRqbT}A->sf~W4%NeHh5Nwj;c#U73AsGY z3F4X_4#-;R$95+KV}M{+qk z4P|1Uz=a2;aN_)5vaaJ9v7L4pqkW^uj=;mTYG@klj!%M`cjltvo@sbpIRbP)&BA({ zJa7%XfrdYvU~pC)iSAZ|;nrX1lDI6C@Gh)A*Fo~vs05ro`QU%n!>O8wvS`TomE`5Q zulV`ES$sEGMke0h!quzB!SauhIO%Q?33Cf)e%&s?Wrybo!{>%T<1pU2^f&=6ukYXv z|1uM8?iLbLS6k9mlnj=`JV~cU9?XsS&Ix5L@U+DV?DisLgP}G&==n&cHAJwwR}o&T z)}#F0vuKyw$e8F{1X=SzdeJBX#mZ0M(4`BQA$Bz7tpM2_ye)-GJ(E zbf8~-1NpaiIbCX70+)7+p~6^}Uf{;!QrAux7UPM78u1|UyNJ<}EV85WD|dd48f-n4 zP3X=N#^~-myuB?ATV{{K=%iAtXs*N6pKcKC6&ge{c08_*T+JB1nkZRVTm}xd;b?aE z5bdz<;>Oe*CeACrGlmC~Nqy)WD#J-r3)3j_EBHB`vSuZ*RSn_q`I)fuSP{(dtS4tI z&yYgHL9${N&!lVg#FroJ(IKpn9@>{pWtM1);-8--y~ep1Wn#Ra_J!6h)NwE+6lIZQdhA`3hIpzgBL)Om?Wab?)6?;0Oo}m-&l`f$=&pOD< zo+`2+a5av-Y$9?y+edeJw7^B1UAXW)|IlK9yzsro$nMXBjE;kN!?Y6REj2{}((6eP z+$IwjYT~i*!(cEj9v8%K1Nn_MVEZf_)Xpx1fYM?JzYs_6-L53%U2P<0zb)@1T#ky} zli^OhKAkbr2xc`CNKX@^mdhaBpz#4T)813VyD11ZDp2)06c(4Qg^(}LAZ;l_=i9NM zw|oo?(b@^4tqYj}r4@Mp-BFCJjzQ46jAPv*Fyj0Nx_PZ1%^6jX$;~{|?Za{!>!~7Y zxp9Y+I@*g*p4=pJq?>Vi?tS*vC@XxtA{4T|c!NXrEu!KWOx+YOVNTyx^5qL zv@iDfAmbN|^v_1&$U&U;LjrxXIsDJj8Z8v>VBK5-;-iGhhNwzz_OF7VH|L>o)e%gx z`b!18&&{Q6DX$t_0hWjMVQR@P8X5f!-Z>=VDc_HzJvfCvWe?Iv^RB|qmPoSFEQaJo znLuG#9>H&gXjvsBhPzr}*hhWQl6@g~+~+Lue6SyylI0-L+6eN}qLAU;JW9ik!LSlJ z=y-`_OxkJukQfBFsyBh5#Q;{HpF{p6#z412D3|J}BT1`@1najhjKC-f0x}!1I_Lzv zY?E-mmrrDrGUb_nQCnek@h<2;drF{|6$4jntt4jeNxicdLZ&3 zDZZJ3X>tiP`%Dw83H63d*KDXd`WPQRNnx%D+(7NcS$3VuR=j4rkSaG`=SL%#H(zjP&^d=cTUwTXee!J5*tPEWIC8lGln}usnN>$AeT_lR$*$^b3i)%wu z!Row)NU7umq}Z&5=aZb_=PDVA-uR=K{+oZsFGtY7Dw!CQXO2X}A9%F`CUW(_bWb{UK`|G@1%Tm&urnzVj#IngaU#=jfxWYNK3 zaxWo(?7r`Sv)q@!`?#T4H0=jdd2S2WJmMstSTPp7zD@&q9R}z;UMr_LRJ54)n=#7X z#6RyA2DLrKz%|o|y2e^qs@sUKguH`lC5L7O$?$xytR$gLfaPiv@boH{Jo% zs@hmd?oY~Y@lb^px2weQm<1>#dE%+zxp2fq8n!H04qnyL_=|~zVNPRV#EfF}q|+eH zT7lXq<Y+jofrJ8JiCRexaoke%`# zq)*99I$ATxvAb)rZ7ovYIh0D-%whHR-+)c_7xC!jY`htwfZ7o}+b?GqHFytr)T3HB zE$}%sd}*fJ%Dr)X&j8shzJ(JM!iW{`4xF{xhmbfc@JnqbU+ZmY{rduZh=$B3gPHgx zVm}0WYD3S}%`o?RJRKOR3u}YR(VfrovOlOx>ZckD#@)-p6TM;Nj&?t1#XDEi>MmmG ztiQr#(Jh$$Boy_A#F4ve)lssrDAi3J*ur+le2HvfKQClxK{r>)p&i)C<nO|=kC2AGrUx?#>tBe-{Z8^(0mW9^+=uwgb|{~?W> z)RATM$$UEzxNe~v9j@WqGv+uy^#pVMX9D-c*NZ%mUV#}F9rWa=YE~n&4_59f!smlL zZ!zWo`@J`bI%}&*{Hj^htcc)hqUF&~y%evT$lzq&_ckZ_6BP6dIBS|irmTO?v-Wj~ z-J-X6$F%QQXwnF?x8IiI>CMV(){;O44F|^P=@!I(vVcr>LJ9xZ;#>BYm0EEZZ+Oe zSO%X8OF&B78-i~hr$-Fy$RXwv`Igp7U-mQH&6BUNF|`#w1o}gS+#9|IlMFrzx_DxG z8|X#kgHJ;^+0U9u)~rw?Q>>kd@@Y$nIHX^w-f4=EFdynZrV{@`A87xjj4oZ@N#ovf zoM-C`gTHi0vFRpJ65a>RUww?dK|L{4E`^YgedNjhN+>a1Pj`H3Br2Cz;d#{}5 z3K!e!z{6x3-szGeqUTN``$NWLEbpk?dvz7`{is0sVP}cnB2~$MGp<1Wfz`0cOkI@N zG7?*QUehU``PyF%-$P`64do3O{+u2KZ~xrwsg&qaZ>~}c*8e;Yv zb(HMp*B-q&WZ;e`UK?`*7W|jM|L%tP&Oaaj);*+(PgO-3DO+)VyDt{kt4f;m#dOrE zDR|aVOm3fRqRU^e!N~H1)?<&$Q|8x0YpD6C3pPJW z;i-itrmvZdK1(WTePWz&`n4qL&U;M%dcEd|tP0o{kAOMGvZ&jCitN|7LRfo>XAg{{ zQ3X5c>f%t6_q&L@e8wBH{a@o2?^s*_i#rl% zZ2vMiq4<^t`ZVJnr>z(p>jrUUS@hNn4Jdo+iJi7ju)U}boYtzto>6Jo9{G?&t?0(% z$8XTO*qe04U=op$--MBt!4$Jz;Fye!1Tu}daa2sr9iPMFOUO{>G-R0x&|qp9%62vr z{i~a4o!%DMwNe4Q zc3y{DJ160qj?eIAj;^HS313$ne}ewi^rc^tjbZZQD@=ZD9(*|C0NpP#VAlOi;_}=G z{DX%>;@bmsN|rwMB#wenpQpg;lx$l1@f<1Hah&f{*2f_VR zN~^^ZpRwEVi!)0!4&LA_t2SfQYX#ge+z&05>oPvyy6|738T5X>ND{OMK}w{>=ScEk z6MPRgb4PjCWfPgIeUd&)DMy3DU09s6618nB(Bw@xOn&qX=7spemEj8b&{~8MKIh>~ zs~a|;kZVBT)ad2^G z4eSnB3ATerNc;*BIyc(D$)0*@NX<}1cM}!6@1fsPmk{;P9#|P@2mW0%V7aqC+8<1$ zlV-X>{{0o4ozH%#eBDKhDsSVwZMQkk9$#2iUxtg1jR&WSwPAt5Zqs%caCGUrMw-tpnbC?uX9w0A~2q5G-t5jbNcDn%Yfqo%0oOgQcO!*inPN z*c8GzThxK^@p<5pRgEU4m+;2KCc-^Z$CKaB!t`}@WW*pxzC|{1{XIwUt=A^*Q}{SA z>2@LQiB(Lz$5Udyu^Nn41(3oA3Z%(l2EHo4f!6ah@Z*$YFlE9vj2JT>lDKHPcHtBp zKR+7u*#^{84T8|F$zb$o2riNrQu-$k!1)jn#m$DV2OfaUf_{1{{0fnmA1Rz=@Q|MB z%c2)|>}6`(u95a_DMW3-KM3&H!tFO};MOJ0f{C-u@Z()o2u;0@!CDu{OO?;GGC`WN z+k_zh;5d5MUE}*}??Yh0D%iO4BzhkV;A;ReAbl*2@0+TC@X5zfep?$!b&J7dyZJD~ zKLJPBUc;!>KSAw|0F@qVQ|HVWjL3KomU6P9S#nwM{W1gJ+izh>>L7hr=#Os?4qs1zQ80sc&2d{mHxB^e@|C(5#!}(0C#2W4BNrK5w0sJ_lB7Ibg#(HF&(P z4Qv8kNau7bkye+sFUM60`Wia$pF1)pP%Z`io?1&&3B_e{prEAMFnKjxW|7=JdB;7wlWL9xrSjBN6a-q?3kC)KQhsTIPOa zEgh3l-5`Km{qThP#f4zby#~_Xt0;Ns)J%hp%V6N4c)VM(RA?-)fUlBlyc9K;)3VKk zn71P^%V8)^o_-#m98#ug&1lh_9BgpSAy^eRT;vprGRZkIzH#?OHhiSszqpsUp2$HlugOcq#L-P zc{aC1T9&Clx)|4P(npW!D%5@3I_L|u1gX;7%$Ue6aP6;x=--?XBym^)w0{P!Z|`e3 zvUC)Fm>~}$DRs1^ra1WUENE01imbiH;Jv6)*!)BRZujPs#e>DP+_$b%Fnuw*ajJv&Jj&YBFG zrN5Zrmk*-*2^DNE^J1#rD#LQm%}}%B1U3d-hETb5>_}XQNvo33F7qSZ7SIm+f_z|G zObSuR;XMG&lfbf!p>aoV!lICVdVb$o%+861O)@*6e47h2+%5;XRd2y9R30iDACPdH z)x>D?RxD__LW~D4gR%U8+6UR`rJ*TS7r2lZCw6GmJ~E zGVJ^Dl0=+wq8g?Ju=&(Kbe5Wk;&B0>m2g5hUhy#LkaW>AejCyLSp$eBDB_cw7wBZ` zi8#_Vj!YOnfPq=VaNi0=682dY%mmt!KjjRr*n9^UCzz9ARXya9b`FGvl`^V^)}l2v zX4n)dCz`Wx0!e$+iQ5tc^b6&CHf6%7^Lkgf&E&v*n{!lMF_HVquS+@qCBZjY1IbjY z0?fI74O}wzFqk{1{Wd;j@@PW=ahqz4`fRzap#FqEgJGu8#m_`kUs?E(&ydvyZ% z5_%Q>>{7&45XQM|m4}~3dEf|Qm^O1ej5a^YO71SeTjrUhx_v&*WgnnR%ssAw?*~kG zpxB`?ncP)!#&0nU`h;I$J{yOLt`*6^YUU*g?Ca&;xVgtBOpNrbfO8DTi6%Bp( z4?gvEGY1q>8TV@qi%rWT>u8Z${x-zZ#uXaZHAuo~n} zR*38`EMTaP=LcDm834b+VKLG{@wV$A1hE5*DMMEWA!HfiJcw1VikV{^UL$(`3`p`CdaL+1sWXnn>s$HGFiqe!AhsSaA=TI2qLb{g?cB55T)XR4vAw9bNP#wJKpG!Wh1YE4(i?WWSrqcI}&C6lf- zQZjsQ8V(sX0KF0e$t;yIe!OUcm*cd&4ZGrkO-N2Yw4 z@;{2sJS@iV3*fZxd)iAyS|~}&+&k}`L?ubK5Gs<2R8*EMUnwoLP)Q_2A}vH(=H7X4 zmLy4(lqDpJk|hyBe(&$^d7hbh=FXfs=iGBXAGT8^k@JCzsKVA&m;BHOF7 zbou(L^yV9FB(Xpg_JXRol*@8v$nQ9*u3xy;^SfP^_8V z6j8=#WsIvEnl+J5oc4`9?sAQklW0#;~_Q}k~e=pG$#Ta_mq@HQ% z7iU!^<*Br^F};!XK_u_c%AU7>%)0u^iawOSVWutTcC$t_S>AUkLCNrW6s+%o{3G)C z(eWd6)Z!yJz`4FUpU0v*dzwj<-7I#wekk3Yx0fkXw#ycwqMFlU)~KaNzi+EEnu zIx;QF)YhkpZQ)$LZ|stpgC_scWAntRQIRS#-tn6~Y;l(VEMJ}Tf2UIaP6ffVF9nS1 zUSryh6$BBAfwZAO9<}<#GAC+wF`7@~ndiNobj4@_^Ri_r3ZMeCuU~~KrB7#G-FSjR zHyvU|?!Kp`;jhrdbqzE@Qh`x*Ji@w!HH+jFPSVmH(sbP4Ojc~eLVDBq7IhI!u8VUH zVwUFxBIA09`0qEe>txc=?(}hWgA(ufm7EXg{%(6();}gH2njDrIj=9ZR2~TLIHq71Ky`cc=)ZXs5GR?+4QSGAY4>a784w(}IE?Nx|Wgc3M!? zi0ZFL+=(dz9j$h%X-lx-U2i;;40IT+R3-S zv4x6VOhZcdcC&83)Y*q~xxJs)$#g`Fo68)2g$xFp*rA0k?EVGY;gUnm+&I}S_T`(C z=-`ZWX4gV#W@>H%3X_L*K}m|#r_F~hsnS75cRS!%Dn%>rJ)j zLUWS0elu3yl>&%Y4t8GF5B7BF<5kUj!I+UOp0ZLLjy-P`9>@`MdU`7nckf8SjSp9H zj3ypfO0)2K^&Xz-w?yDybQHf^+rmQ$6NzB*x7ywtKHN+E6VX=hI zVEY~oQv6^T@T<*im;UalIa+LZwD5Gx{bx5wu%UdF+9)PxU34Or{F zDOtSH8g4DoAYTuzfU~`O@$_FK;N61>w)Pz z!?5$Bm9?*Ge}ef4X@$zfd|c)cKqT)uMwXbY9Fsvy4hv86Vl z9HIEdzUz2gi!3l&VS!~UTJf$O55dg8k8mN!?<+iW&nYc&i&N`~7N-}%M?j0>co?SO zi+#nX0msU}Sk3YpHrU$<{Cwi@-ZKI?Va;MlJExMQS3a)7$Vr|5-qz*ptLu9TpnC{R9jS#^Ez<(O{BhJvMQD z1@2{5V$m7|W%GQo*zd#O)08jRRxKR6soVkeZM}FbJ{ydFb|5>F&f?QMH-S5SL@0h% z2fEWRp{X(i&6k^mK`GtfR&%%Tut5)K9}(euPtSncUzd`k#AQ%6DVU^f^n@oy)|pw8N& z`d;AtFo4XgodG{iG$G9EaHuh8M*KW9pg|tzx>%_J-C|0x*}l1vXeE4ioMukuOhvf}I0eDp9!^hygLN*Y3O#)WaCBU}F*&Nf{ z3arHoNV~-4pe4GH3=m$JuM<p6$!~_pe1o7y-W6hy!QKjR@>U|5eu;R2LYDZ zmWeljJG>vy=ff7i5o|YYK1|O9WcC4PsNgq_C?vbW5?^g{DsT)ud6((bHIzEBO>lc9r%T;($NDr9L%9DU?JkX*uin0GQU~gecT&IWuoio$njg(Z}n6Crb zNY;lF?$mv<(+;%+%N13f6N;WVt5jAn}ydD=fRizr;y03SdQg!zU~I{tbZn{ z;Ml)XJ}#u*!Wu5u7{hb=+&GVdEz#}fK_)*MU!A(&=|uGf+|;}d7&L3ZGwZ9IPK2?b z-d7rL>FUCf)pH=(G@Ep4d%~#$^TS zq2=Y1!mYd2p&;)eo^9yG+Y!=?AIfoR1^#M$QJ4XY{u+?s*zr(TPm(0P)+Q@#7XYz? z8swDE0C0Nq2dAkn=AByo6zBEq1Fxo-!(DAnINiV&&RwEOIu^gcl8ufqI@y%$;kfF< z8Zu<$=6B(ol1^-{5D!c@D8Q!m)5tmDRJaT46T_*-WYgzqF!OdWc`@PxxA0`hftj5D zqt=K-XnY0NI>exAtqL)_0AX{L23dB|7cUX~i!xE7C+lT}llZUbqwYWe>0gT1g;F_Q^ zZkzHy;1(|jrx8;pp2umR*fNP6{JRvkJX=oQ-U)!ZQf7E4)C{r_?{U~)2UsVa04)zp zB!`A$;c>?qT&UVe>Wjz2MLX7!($8ja?~@^+o{t_*^X2?RQUy3~yB2t@updmDtb;f1 z(gm;H_}~@m^=p%TvTJ}z%_{p(BsQ3+*PFx zXUlft9p^mZh5asMn|}aYF#^fU(e323J_0u<%pv-QwO~m~EZNJ-lKy=&K=$l>Jf(Ui zG*=!5hVKsGn4e`Jz;hpV_Q?ls+rQ!HRn9=E>p$|JgE8<+nnVJ%3qbY?SYxx)6SAfO zo@qx^$uKYD*;jPk- zIBhi_{%%zyd6m&nD_;)oc07eAeHa4r1N-4MUp{=$p9i-2^#aLTl2BT=2A8KD2JDO{ z_}=whq)`MQDGnmdmeb&we`{gjrNu;C%@^j%xDd0FR!~*)40c~2wAJkc^ZHY*-}`Jn=*R@@Uh?0F0N-S?A_3Ex0ka1i;d_8ojWECIbg z9|jjfe88o=e4uqD6UYSunD)eigbY~2`?BN7o`y9zCT1mWs4W!!n_>WqIG%l;z6?RN zI?%rQEmnK04bM%rAsv0{&~HKvJT3bH4EWx_7j=q3{(~@RGvP8;Uo!=k|LwpjiF?U^ z*F!*rof`S`{3Q4g?Esxa_JcUDMxNQ{IpC-I7Q!TL$9pv(=h3Sc&ZwGA2E>2kH7enF z<`2#}cj^)_*y&9Ey>x){!y(xbG7e7YxD6x+v#^be5ahcg!R;T+;nBzU@a$G)82@Ps zte%zzEYhukxMB+Ucs&L_v6P1QTpTe!^CkE^=O8JGm&Ywjt;oNERB(E$ANgnD!W*j} z#)X~%;AX`d^3G``Pjr1PcIVvyYLz#@ilPW`Dk&cr3wl9M+z;NJl1%(5><%!J_=B_c zPl6NTyLrv@sZep3C2sm<0?#LD3onTj!K0&_K;xI4JddriSbT3JFW9q9SSharDR4}u=-Hv^DwmrbGbnIIB6+XimB$DN7GB;m}rbs#k4hR{l325Eaz2ALkdo>=NX;UB(J$Cev4I_L}r6HY)I(F zO2ef98z08({tOV?H3RCO-hsPjS;1)!i?E}NIaVI~gWo9r0}8us$*O&qfnWMKB5YQK zXd1?KxnIF6wK>>h-T~e%#SP%wvbXrph%8jE=8;f)1K9DNfx6iV!exh2L33mUcs{5m z{82s?ht(C8|_<0d6zr~3zmmTd~S{dNFeSUMyeN|A(ypYLF~qSv4| zd74w~B}uOCYQ<59q@i8%WLT`blob1JfJ-i|A#R6`g17IBc$;os1WI;}`0vjS;Nfr` z%uyQQ9e!j4Bpp;?qH+~(=iu$iW9BKA|G|w8~7V|oSTn*i`1d}js(oe-4hybsRJ&*)bTu=#JkfGL}s zhuyAq&(Gp{-}nP`v=oTeDJ}TLuLv-SV(`i{C#?FG<5^NO5TT-NP0zCZo zPzoXjE<~}nkPrT;8H~8Xrakx-QjJ#?r<|&tDgQ?GTz+$r}z)BaA zEBeFui}h7(bXJT=o{tpHyX`{e*xUh)FT+Wa$vj{mrAJOXd&9Xw{-o#qTkx;vnUg}s zIC%ZH8x}SDz&$=^c@LU(A-`|Auu4e+UM-WrS}W9mv9&gD@`VN9#oq-m^5Fp1SsVt{ zBpu+ENPC|5+EHLFP{W^0GHPGF;Ss$YGq@wjiOdyhlA*iSF#4A}vHD~S2fa@Ujp|-F z&2zkn8#Zj@ov@Q3OU5wHS62t7oVUry!V!MBKzNg?yTLE{Zl1@?Ht@-TfuqhU|YmYX=H)OBj%6x5De)m5-JN7*|nd6U7 zjB1mGKV{+6SVa<5ybMS0zYcEwn~Mz_euMeusz84$#}Iy~LeABAz@?I%pesw2xc%^i zKl4n;@%guSp0+Y1Z><^ZO4TP@eV*XP?Mh_FEsnt}!I0*jKA<5QM@o(lfqTmGMD$;j z@KuI1bW|}X16Me=?W;YY&_WBI+IkMx2YA4^6SAZ$wfk!+(Wq2))X%+AH?dj zM|qZF2wFbYCi=|*P?RMDyMB(=PM(_x2Crz=+EgC~P4dq`2dn9LG>!8R=9Gaaqq9J8 zuPmqxs{qgZqnvn_EAeupaZvooKcVT50ijX*0MAPI1fFp3AXxKn1bc{ylheoq7s)D; z*b+-KU?_AdQ95!29Mxlp)cBuxLcn`r-Lg@~f5WXn`L`E*7%Wc41WVwON3GcI&0>;W z#D|3qYe}Aw7c6{f4re9L6-N5s2E1ln>}i;feV_D#$w#e-#pI7*>qS4ZceDl!zw#s> z++#RcVh`>!mIn#14`Kd~0xSeFFY7Qh2%f?(p2H=io$*CF%B=ja{!IvNZ4m-t|x)w)3Wvb>=G2 z`K>GQdKt|%b{arfqq$?$%T^GwN&qvz%_JIiHt=?@GkM&sN059MxV>D73{}d)+uz=U z4LwFgB`g=LO1C4GmZ@O4y&a@xWdf);2K1g9lS!Mtg0%74WO3F{pg7|WIDSlwd_AcN z%L{VxpC}i&welrCczPk1d)4DNZI)2&_agGR#u)Bc;73jcn-ia>pFzfT1Cr;Y4KpH3 zz@rm8cy*sRA53jOm~zyCyxuYyYCm=&zOO9c{k7k5&^RT|yM7sGnwY}1DvB^@l{>EJ zeFBX1f8gg=ETHIvfV5gV!2vy2V)@yTY?il!&-S?!^A1PoeCs`M+}47puD=d6ZVloq z1Jg*D&pdeT{X*h->Hl??EnKmF7U>?gf+Z1i$=lNkP&#P=p-qDzeBuUDu=go&xVr*7 z{i^_Z^)2{A@f7%?A7gJb&Pi3d5I6j-!rr@`fO%kjZF6Id}8j}0T^c!z!*0R9QZJe$xa@czS6GWvcT z_;kUZ{0SHpN}g9DCUQz}ETIGo=gxqabxoj^#0NYgvVeygB#A`MOmH2<^BN*$V0iE? z;9;YPUs-c`RfI0C;qohC-DLR3JOlhmvV&`kRAA#@O(JWx9H?yW!IM5_gQ81r@XT># zGC$D+PM9uBKGr$F?OBjSuH{4ZbB<&%G8vq)@*uA^UIULK{fXOX2Y8`V4VrI$1U`q& zVAgJZcq+`R*7&Cmyw4`{nqwMq!Wji8rJZ7=af1c+;rvH;4(NcP?Kknled@5MQJ=V3 znBdfn>G;(e*V@CIZ-c8!Pq|vB1MI%gj>l*9f@-~E_!=4k<;rXDw2lQ}0G{fGxpT$A85IK>U5JBjeSZWn0CIkd+mTSHyxOuRa3DcsyL zgP6^8g$Xuxu<*nYJa;z7!`8lly~B*j(e2_;XM;7dnJEpk&Q|dXMn2YjRaC$lo@RK> z&0%aP$Pnr`#p1Eq@13^FNW-8w&L8?+1+HmVA(b<_fY#61LOtO&AoF?y4$V6QY#lww zIDsB?oI9KRb-WDvYGjG$e|JIJ^?P`%%7VNTbAYq>IEZ|HGEzpMc!;4luq$ABy@<;_9BeKubysE+lWT zgb~N8FYm!0hdn{God$`2(g}1qXKgCvyoEO|@FotYVeyGe!8ncsFLl;|gy!2oXlP12 zZyn-E#P#BQmuujv*qvL)&E&h~1DJ?45Z99MY5JywGjn`k*v!0+Vaf>3r!r9LSsiZN^&L+sF@!5#{=k3E=)vFa(s(PEgZiGoi60r~0O52~ zTyxBY7;9F6ik%L`aq=|itu99%?VSP@-$|1N+lKJwdKupKWJTQK+r%r3spAzr%f?+t z!@wCi7yRHe=Z+5Oz{_4+aP|EiZ1cMjte<-c1dSHd_SKsT`5ybhg?eht_$LcE0Y;EGRFfmhvUl47TcFkJWyS9gB`vDa+LkeLo#l%Yn}2$h6K%uB(C zZTs-~EDhk@5iIPogCrwb8h&>%B))Gy@)jmffvJz4;nChvkRR8D?HkX7D(;MvB^h9H z>noV%uS+VWhCoN?crxB`JZaIkhSsNzh}J<{n6l&%PEBkA>!%jr3!xW)$B{Up<5e?= zIJv_rEem+-;C`Tb(u_!^4if)JTRZ9h8Mw0yIG%J%c(;(mC!ZG1%=x%_W3LUjanqh8&gMMb?lFD&6{YzKc0)7WqWJJ@16#;GLo@Wn-y#dECMm? z_iFGq6WqRD#_KvV0Qy{qYpoyWf+@-^wM{{WFlVMAC}`>fqu<(j8?xesZaC4saZ9vu z&hJya-hHW2NtK)j=7gGccnz+%(+!eZ&q!1$FETvt0=xW%mjbf+(< ztsWSM9mBp0JtiKmeQoW78|qXboIZj>Id@Wc$2**}R2R?bIR(ZWq~l$=7^n{_kPc6C z@VHTeoImBo%e*KH##uFB|NUz~{b(jOajeH{Z^*(^Piydr3UTPJzZhR?D8xsF&v}Vm z7qL>jHuzo;0|xr+K<(Bg;LRo(D4MB48jB#TpI}bHm;T0G3nmMfiA~4-ZOOuI)2HG$ z{ImFbe1Gj*_ar=t$pi)&?=aXr1vX3+C)HOUVKd)Z;9CD*93iF&T=tqm_sV&=rdSQG z30a2un&Zjr`)$1T*SBzTNWasGqblU!)$rP{3E!{^v#aKOj4JO&xC*@3oPu??wSz_b zc7gP1&7ff5tx*2Kc`(27AeMCq!PT3+aN*@SIQpSB>6Q+HO?&2&?dw)RjU)|dF_0nb z?Mehk4!aBG>wbY>dPzL?fDnj;F1&(srnsbmtIIYgVCx^Zf%?azAk6FrFf7$_YQKLI zh-H2TQ36LUhz^0F?i}Hf(>BChMiQoN&?O~XrQxBX%i#BALoi#)32cm-2=*$c3CHKH z1^J{;_;QXXu>RErKE2Jt&IND5Qj=VKChi}IH$I9}&-Q_UA6fX`uNLsRu!@(Kl?@7P z^&M?pegT64Y0&X>KZwws2!3(169vy#VE=MC2$)h_T$cfW^gA5i+z)2SzsByV;$)h) zDK2|SaD~i$;ZV#Na9E-aqSO*V;&Bb|7L^HYN40s6WGCaROAqtDD{GKaE>9_a(1M>P zhJpNH8^CvYgwLAi0;c#O?w6lW=I1g{br~Qu#2HQvJpr;Nnv&vA6g(~0AOoiRg`cDQ zuyOeb-i3-^ctWfUad&6Is>ZiiVOBe+(6MruFzG7qjiWrT8hG<=e-8(TFKz*!pR55Z zuFL`F6;6X*?RLji+ZJ*?aSf+&VFy5o@mJx+w6h>VBNkt8?8Zg?=D@b66#qEp26oEV zfXGu}IK#OaJj^_T2PgJnZNWtNZjL%BJ>dYi&d&yhC%bsI6Q+V=-eTZCBQc?7YXwlL z)WaTr8K9PPe_dCPgm#BwiPhr}I9Z~La|&AmhONVMCmVpr!wcc!vJXO+dlIl>L9NjH zr#Jqu+X>ijJdaE7rGOcJYH(8bU94eZ4%4v?|<4AZf)Q!8ZTmnrN-2^ARBH?q{wInJv0&?>#HSqQn^6{qx>~v8f z<4#}1qK56T=G_j`qZJDiT0_X75WrU&PGpA24Sqe)hy%2DK%HrO2(pNQTAK1C{beXT zEWUxHsYJm0ssb!&u!GD4L2%E95VFKO96v0HfO093M2KQwdqp^0d?%22-d_!eWY&=s zaaq!OXc?UH(1L8znGIcK?8uS3@$k~Uc;b9H5#IEQtua^K23sF*CaSHG&^}NK+K#h< zv;Icl8D&-=I^LUj%v%S0F8GtpKi9(<%L2*jkj1bi$ek2UxjT^TXt?Y0S1@}L6sEg|cmS8*6=?$98HGJ!D0*_X^4zXB$Iy926<{fYhb&CqH>I1w&ppqj%nGGyrm zU(R1f4sKfrlR0IMq$iQ=@r0X2 zZVS&?X(q5ei?Y8V7o_et`sY02Y8Quxa@p_V476CWr z-NB7Eo1wk)MslWjBShdFm}(eJ0+Uz4*Yko%-l#4KI1verM%I(eTkBzT>~-7}xeGS0 z-b9KgMsxSTM}cg{O0qJ{2d0DskaNHNNpjpO_-6cK(iyn~Zr-~brkQLc%}dt8vqNFz zzT0=~CK(BX?{hxqEz$6LX)AB@g3aW#egvGkaWfg(CP!3D*Fc49bBNy-FUa062gl8( zk#IwG=mw0)AEgt(o9zYG-=wjKUCldLJb{RP+5)X^FC$lL*23q4NzmD0HOX7x3H5i( zC*u+Zd3hSnFm(N7QdT$vhVrMu%XX2(|FIubmkuV|x0_)p$+d8P*-T<8YY$hJ=b$8= zKGLVHAlOwQBd~OqVD{a-z}{OqgWghm2^B9Y(wB3-(`8c?>h`Doqn3jqNcXcQ`@p=4 zP5TVk*~P|uLyJyQzb6~Te4fMDbzP-1jvPij3%R-vS?MZt$Jv243YVi5 zhYzzVL*`^#a1l5>OHq(~M^bRGJc)hVrYCqMyuu#}*Jq{S9s2mqZ~FP~LQ&aRKEFTn zFgj;_P;|COqVC(hV`xXi0F$&_Q=sh_!oKJB9mFSVuoH4NvJo{tXzD&W!3KHA#HUg; zK`xd5#czZ-m8_%+KaU~lnyF~fL5^jM8|mqklgKqqfv$XK!fx1lh<2}TqKC|OA@PnP zrpLjBmLHbl-wb?>Hk2+!KW_xH8COS{>9Sjp>x!prK!hps9^6dT8g$W))Fpynw~diB z$AW0MY)K| z=JTe4dD_pAy5<@7>m(%X7vgrGk0vL694qflAB)B|F2PLLE&<2A#D%BrMCsyRqyHDh(^r_wK<{jmz zwxplk@zVemzkER7ylG&4wELO%|4y<`mL{O0e=>sck0hCU%|H0(4KmoN^PlmR`VFZH zTp>E%YY$b6t7+8oqs&0@YwEl8Fms~cg*hdujh;=P$G!{A+AVy=2@)@CuA7?XGLW)oC@vVaU=mD=VdaiUHG8bNf=ceScw&&HDT95k3tx za7<2+;M&IAeANVfjW-~TbFcaQH`XYqrIhkDmoWjF)A_*Rj{D>ENj_c{~M@C-SG4=BNkReji{pFILADgzq6Ru(`zDc%Vl(7eFr}>;Wo2*;!5-^P={lznCE$p0W@(M9+M zeTysLAIxp!AHO}o|MtrP?R&i*nb)MVyV}~>TZP3;-Hu=y|E>u=%xPlBHi`@I-X_k8 z)ry9LRZzo%E3|E7Fa7q%f!V!Y8ucEzjb3O5P=AeKBrEV{>cyPtn`}bgx~mGdO>SeC zwEy8NDFxHN0ZGgkM_ZKgei;>;A4x+?-m^;>3y$IFkJ8kR(hJLS7}=e(QQfamzLv5i znp?Pv76)h2z`|?jWc+XDF2_VPb8`}1G>T!QCX7dGUnJ4IR53xoWJwfjGZ9@0(m=wn z!>nn|S2|NV39PxAONY1X&{3xrrmVFe83uIIJk>CYW^){a#B+|49M7v94kZImIk-SkEyYJk36xChGG$+Z^nA$gtk6S(afnpT7 zOP+CTWKgc9JldB0jG5XFklf58=o2?WGy9OEsCI!Gdw%r;y6H~?o0}jf*lRQoEqxKe zET~MNb-m}=!obNi_x20^yBaY;wjhc6TN6g*W+xjGHX9$WKgZ84`a^ZpFsm|UDqVW% z7|oOFVRYhdpajlsq%udX&ao()Znq$4ldL84e%&!v-uI6v@oydFf5}H$(lTiM!u4$R zhO@}lVFrzBkgBr_-$ctUDIgOcHM*wyt7w^K4lwDI5ERL&2~NJB%#yk1X*X|#8QAoS z&igPz8~qNTbNQxpB9f(3pYTvVTSaBJB#9mvuR*&H-z4_SW}xJ*F*fy66F+^wJ57@H zMdr_iIK?Sn)KzYSB0otXrdzwNApZ=q-LK8X%3mTuo+Y|o;YAP5YMlsd!&D9o(&ff+QtTMVc%!ZOlZcEo0H^Z`I7sH}+`G z$re!vdy3w34yN<|YexU|yyY+a(n{4E9njU`WTwhhj)6LXD9iF}60YF&DkH{hgk&7!;R zr_!@sE{xo#Xe!jmLqiRR&@~HJls&!_+1%_wqOMGqH#w4iG2s{tTlcX8jVBprDIOX; z7|6Vm)IxW^e?~7}O3-oUnEIf2R`1$0LFl-}^z5V}hPfc(YfVa}a^G3B33kzhD!V%Q zH_?3Z{EA3n4rT5vFs4I`N<|aJk{Q;?kJ{vJrhl_4*cm+==|6u=&5Y)waqsl#F~vM) z|Ho?4N8Ltx<>*FuH&~`_(F?=6_sIct_t)2q|I$M!IOZeu)4z)vk}uNO(ne-S_Jx^`pQ?2Gh+&Weqete*i6MDwUO2QDa`Lzk#xc(Fc@WM-HBqR;-l zW;c68Fh`FKumhihkp?pfl{-jM`)}&3&2crs)|q#>6hgi(bB77~v(=c!mK>-4^}cBN z+#uSQv!Bn?rF8LffHIZ$Gw$8;b>8obXvdXGM$YUsYwGNbUV|(qzeFC*P@9bevv#t{ z4{wt7pFc9u^fAr&Paox8c|+3+^r-#La=NkE1Lb#$80+ghMSHkBF@L%j>rpkx^7~fv z!*#q_`vuV`dE0EHz+Ol0@5a+Eeg;|@7Rsux8lsu?R!m9DHqpOCXlm?JX3kJA{nKDegE@Cl(mf?sWLSsH{;p>S zX)IEJ$gu4cOL&G@jyqkrqi`;*{t6AF*eHeBcp4WPrEOT z@E1P0fQz9fu2PPbalKrdi?$X`!Q%4 zDv{1Y!rY7W_764D-vjRK21zF-yi5{J%(%};eAE+3+ulcmSwnOkx5JpX)1E4C`p9S` zy=Ck-RneZQ<5A+1=}b3PNA6n3QN^4*^!T(Vs{VG3wVmt7SblV&Z~qdcbk38$+kFWQ zIZQ`!%6Hh&h*xarWG8g`R1oJOn_3sEw2(fZd4;ihWR1QWb#Pv_t8AlQGgEy&4uP09 zdiIu{!1+rB-OYLY3d|4E-?^tz(#9F0gj7Acw@{sqKd#0I*Evx`?Hm*=m%*GqehfVp z=8G);u|A}$|Mu3--Q`>-_CK3<0w zWeb?=9~RQ5;69pt@G|pVberR@ifC;7DrV}N>F5~uf7v=PMzg*JAiryune3{Ubir|q za(+)>{_|C1lMC0N8NPbxbKew-;&-7r#lDmpo?wAyKb!Gai^d%LC$ebWi`3^@(YLwH z%%zcbHgZRZ@T~6*YT>CTxET1FHHx0bar8atfh13ZxwXD?kH+o zU&y{rNoCUYr?I`$7t#B#IJd>8+tmNm4_47In@r=*c&oua`gHCUidV=8Do&NsMGv!) z9aW=!m4t~tkj(zq=1&)vdD2^r)#zC8WcuexA{~f1h#b6<*^b4SY20Fj%vM}wQK$`) zaFG?X%=j+yF}lq(F;VDly9Mi5oQcB!Ss|m5Z=wT=tI_L`(5L}oQj1mR& zfRc_bl^w|D{JTf!$s;^b_W1kkhR9OZZIrJ45zeMI zSs{Cazf3^49DS3s2aS9=it4Xs(p4`vAib%sR2Rgvf(eq;uWKGxFGZLe5H``Yh|RM#98?X`a` ziWtn{uN%8cZ)>WEYA2PTs&|;*zcc}?Q{ITY=t*Wh$J+0APhj3u+0cZz#mpKn6VB0@ zh?WM(3F^O|V&7fLrRGs0)Yn!8rDoeBjbH_~db>WIQ1+Z2NX=yStsX=FbzEe>EG*#r zY20Azo71TRQe(xHAJNJZ4H|jXpW|meW;MMdsNnV{RCS2syG^L4ACK1{-s=iPF6XlT zftQe&;3*8c>`tF6DWjjJ+vz0}Cq{hNbhOl{j2KG>)8b@jq}-82f43&k@W6|-aVUUI zPaQ^ntvR$;Q3v@9y3=IYQ|SGnLTdAEHuEztpV7N=9JzkHOx32kvb7EV==1yt^r%ml zR;e#Q?L+#4pQ|@ex8W`5%u6pMo^YFE(oPV3^$9^|^;+2CW-&JNqa*bt`-(UN{+*b?Bs3~BTeNv-f&2mVJ5m;Pb$nqC-8;q3tx}|2 zN;0%Bz)d8z@H(5nS4f?1&0$X8!zgsAltAO)eP&6kFWPM&j}{I@ic}w1(y;s)C^z;5 z8-8*-?Mfa`<(9ZH!8y~Jj5-5$ORWrz6jJ2(dkZ>yOoe7Xwqw^NRZxlO_e|dJCVu1X z7W8R-I$bjohKR&zYOZ#R8m_p7jJ6!%n;lA~HA_R0bG~?;`wDY*x!if$a+@;iqR&w| zHyz|L-bmnRu#A2AFcGcTG0J#|Hj6@j<{_Z;h-r6lM9YllpaZ=1jGd<*b5+fdjr;3P z?<9U>OjHZe#1?SobO9#;&aU9m}d5?)MB*!vIjaE zG>I82u%M$03el+hK`MWz4Xx(K)9-n!>D~NP^e(fO`OY~N73$m3oNO;TRb>bbSn1Z; z=TtJsgvpGgmW_Zf+JpMf=kfhII9_PCHBH{~k-b{I2KAX_@+0sZ_TP%9)K)Q-Zk2n+ z0(p)V6=TXAzH5QfH107x^JUaSvx#}>--F~|O0k^^VINr^c_d`)b78`{n;aAt5_=Y;bJ@AO7mB;hz#qLC>9Hp8n4K^}GK^(Rnyx_5E=?DMCUtlvGL?AuFEqdF~;k5*i}O zDngT#$jaV3Bb$nO;2G!M&pGGwdB0wh*^z`QzAGq! zcRSm~Sab#6@mX59x(#J1s@>So)ea}0lEPnX@?D7^cvDF{yD;}TKWy-5wkG6@_;>PA zZlC&n-l;2{y{mu7eL6G(N8F2JUMp2u?LIvgbLt82A*W=Oema4lax##&sXIomuQg(2 zyBfqvYvh^gIy-!%;~1~ANiXk%vJKx*O z4L1MF=Hk;VW?4OVKrf3eaUNpz)IzJJHIgxd4(S$y9W~5&q9N;FG7+CQmTNU%S1sJt zG~%q2XE=v8FLo^7jfHfdV2{nD_}*FzZqscK)@hc&ww^M_uf>+|b-@unc|-=2cQ9uU zYgJmsudeg5Y4u_?!P8rHIiKzAP+&GI^Vywi2eE<19Q;+w7xx@#<5zSC;e>ORT+*l0 zSWV?#i=DuqepvF1TQKVe>%YU;x@(nO;ge15xXD<&uktQF?0k>6)Jog;l;Crvnv#fx$7Tr>LY#P$V z2K~M#uAUdjer+xkPg&*7{hVxox850SRVlcl5(?AtoZ+W&d59KvdvSwZy(`b#DV%TN zy0Y*VeP!IzZO2qELW^?Dem1)2CD(4Og7JZixT@!!m*mzTj?-+zgeGI-mvEK|K{P%gR6LhSWoj0$_2J~wnauzGPd>~= zwKc}vn$1^Hv1)HU$IN=hvI%~TOmAp2*You$FSq>wAOGqp7kn;(_xZAt$p+-|fmYXW zu6`xEcu9^KDShYO>5pPp`?8rtV4`h_Jcu3s#qp6j=3Mi-BW&+(7cR^oj4RmlfVqq- z!fyALvUJfjR(MVp_xwxavQ*CU_J6k1m9g&l+1-WoxN8E-tSgi5zQ-^M@8>vrS`Ra`<*e^d5d35uKjrIn|G8uk@%DwPXB>Wg`#l#d_7`MqG`dnB7?SiFV9)Wyg<)umdqo{MAcw?BUmm;q zL;DUt^36Bwv@w}w6#277riN_9fggOE^BC->Z_B&9%HhZFk+)hrwuYZK`!AOkf22h} zK?%p#eiS@9r7ZEDCnFjk`CT*S;ahA8Z$2zuaPXywtGuij(%|`*Hk0s%K^=Tgj-nN4 z-NAQ~X0!AAhclftNz02RBL0tfKcCYylF6G$wYL4vV@u?|b28EAxTJCS`RExnjQ3J( z<#ZQ`RRwPPhn%at>%0G0jGF1Q!KWxqD+@g41Q zyw3bu?u&uoaqn+nzBgCmANK04a%h!!_$zt*XOmF#Hk@WfQNb);CYN7ufa9|ztvsaq zGnc*;tZJ!<_YD8ZzqCb6;bAY=+<%IDuR4``@XQP!NmJxgPM9*|f4?}F0mLFd=fT1W zmF%sYz`wWk;p*k?i%+cYz<2e>S)Ki7gw+S%VeegMvm_%T+(XF-d+kr0#=O1U>2M>V zf6~d*qiq6TMW}(_H?SKO>RkK0T-LD6g-yxa!$oSE^4^jlzUkCQdQNxkp+)d!BU-t;tl+HVkB_W2hldaG--_SGHk zl(H||P!Z3L=>Fh$85rZ!4jJr_?sGhO_C$QlI3D*M8^Qty>0%S#Am(4v*U~3fg#G-o z*`y+K&h~Q_E9uU~)8%s5nsoCPk9tjJvAB^d34h0l&+p-M5B%Vsw|;IpdvpY+oukeU zbbn?$8&0s}A2V=@b3J>nKM3Er^qeLBwZ>P>j^fy$2e|6cIp&{I$nGwvW^$o7*inUb z%+aBVTY3I64!Y`qXT6`#AFrK^jdVt_54!0%{lpS>()dG*x@`%&eM<&^wLFUxAM6$n zmYdQNReyoY`jNsa=8xtydl#|6D%za0XE^gH%_6qGQ-wPCmRnJAmUgxO;y!LqCbxPH zx`S&qJ+;p8Z?0k@;t*fPR)ATep=b$D(=(GgatP$3ZOf~+uE|$p@yK!>UQ`qol zLaq5RvGwMH?>PCyUcNcnn=fe7;^v#^@E8A-;paMaToSjPX<1~pc;yR@f;9_yZ%<9e z|2vF*olN_@ME@K<8BTY z90J}T~Gwxse0KMZNKmihu*&}1SOy1( zO|RFnPh$^5`)q>O7#Xs$W{NB#Na$0T?8h6L^;xY~s`&7&8~A2=DwYrTV7e)t(ABY& zPk4BoKW3qWBj>cWsLBe7?Bz4qE~6BdV`^X(o$-v@pDSZ^u<0%*`3fvuV3z7gU*;|f zF6Pk{r^MUWx3DexjpA&<7k~WK0r9_~x-4(_1#xBeE;g-IfnU)mxXuMnPHk5YHjXUe z?`Vv}FMkWZg2wy~_t~{?yG%$&}%D=U-mD)o_k zmTEIov{l2(+QEFc=p=vJgM0z zZaIO3ZNXJN(p?!3(WCRUKTjx92qf!*Cb@Z<6`%=1pS z`0Bmiyz8mM9MufK&I3!t%Jw~M@hn4@UzW>Hs(Hx=m;qB67{+paC-XPDlv`JnrQ#)1 zFJQUWM_3dSiN_qe&!Zeq99C`054=ia58dV1lh}CPVw@MQ^=;)JP4#Cc)(yO$fi_o) z$5<_~lx|&95Wv#%s`yd17dc1YMZ8n2Dt_Nq&hdJAyhpi))dK0$%%=1eC#yS!J+hyQ z9ao38IK|3go4ZZilp7-P?&9~jd1E3@?K$7Fy$x`|FBQDBaUVZz`*Y@83w-+XDdP6j z{rtjECsz9523~evmHpoPTii3C$qtk|a)lnD*e+VV)s2a`fI)FMXR#GFU*EtAEYq03 zLM+oUScY+}KHj)>8fQDM93M}*E}r1Ng^MeGiYxxvvb0fsc<;o8c#ZVU7Kwtu96grA z=I@+`7r*+0C(TV}3N`uIZmB7@Zt!AnO+v9&S0CSVdI4*TUC5LaikR|v6KuabjxPvT z!A4pjmN(sp(-&E?3#aB{qc3m7gI3G1yumK)(8&_qSf_}ynpC+fIx}%ZUM%-ygp$=7 z&!Oz|sh6~F>Kd%Ob`)C`R>y{3|HQ{k&tbPm-e&5;J~K=99NSRpi1$r-%3o{DW^axy z;kN3Iz?Dx!@Q_Wtti;P4XHFMkGy5JssdtU={JvzJhW<%kVNyDyB;@Dd$ zEo>gY0zZ9Wi}?7TKS%`T~>!jNH&%ilQNbY=W` zP$Ap%@daKTFqIn}Hd{P*awBu!D)fF2KNkPUPUP<|+KbO?{3pI=T`Mr!r{Mhw$*kHJ zxYbsp*gvmH_)ndY_=N2WcC(^_yD2=kK3hv#hHgKMt(6F-PCIbcZk3iWnMTa-o{0}M z=HSn(<+ujfH0Cn7p4s{*GJ|O?Z1kJwxJ`8_-f%pdWeVQRZ&if9y8jmLlX&7Rfmt?T zyCQ#c`V=h6I?ED5Z*!Rj$xQP2Id^iw2^_NhB8DYN*zw|b@y{<>IO=-@yRWwgCr7Qr z58kI?mqmeC#?p(^cYVzjR>q0DqH^))oG85DS~S-X+{+&8@4))y!T8Opx46P673cB4 zxCzW9bkQF(m1hfsMV{3k?eh43Lje<$Nf>f$wy9Ji5FQHh`%(XGII~X@6a-v z2_t0q>0V7db>cgGxqph_dYs1l&ojgq=8j=qvtqFSM@M#O#S=DMZxVmu*?ul5$%n1c zn#4`Q*TfsLrEp`Hu>TA{%NM$oi+x@h;EcL?g8wXqjVq|dtE$V{2<3FV&-W271}xB9Wb zs*Yo;-0ula%use^Po3B(^&_j+-ox21J;HaT#d8nJ4zX3yH`vmdzbSusUW;(-#_U%z zcjNpwrgY@9ShZ_Eo2@yUHC>Jp$2Pn1kKK>rsF`{=aP$Psc1Vk5%bGZJq!LFu_KSZm z8-yoCKW(`&JP{Xay5YNOi`gQ{M5ebXnY$p|sdlF9=JG$S!@Kf3m;q|wlaikbvslZy zd9PQpRX1hWm$kar00pWo~jKUmtL@Bam_GI+wr+NtuFXDkB$PoMy>G=7Z7;(l+j-{#OFk#x1v-JPRooTqhP(~1&zRVO~ z`g4Z=@lu-EC%@+8&ud%Vzo><|SK7>M{Ymij?i2rfR)cqXWV4&o7Pr*D@#o$JOEGi4 zl3NjUjDPh&iA~r(iJ1i0V#lvbTK4{X&S_1W%6~d@opbK4=Ykv#vcF*!>?d5Af%=zu^yD8N~uu z8M4u~QdW1&t@wdkl=JTO-~+}_#g)OA@E$|K?-(zFtBM0eeXtZ==w~SDTymcLw{a2D z9r6fOd&Uu;=wYCtJ%&p44WmP1Pl;sLxsdUD&FGO8iHOP{MIQoZ(N867=yi25d}ilR zeuW|&tuUh1Lbj!4f3~FpE|gr|7ze9wz81~!-bfFeD}hBn@}RL~2c0NYNn3n+(7mxE zNWzul0#ju^dGZ9%l`dPV`!9~Db5f#M>MZelE)UPlVvrSV5#=tKNT+_gEV;RG667p( zhM+0l(DLJA^QCeJioI0m*9o_pp2>YjLFGRseCRD=@>2u8?Yl+(lNRP!b);yUb3M6z zUjza6uh5qVqD4E-_j^@TZMExn6kY#ELS9%L9#}}B=VRlAzYi|L)cgT|N`<)3L zmujHOwj3^~=RzM}LbC?#rBBQk!i9Eka4NH+1uFysb?O1K%_8* z?Ra`{=mhW(^5~bnvdN?=b(Wgqi%9K525H#si5&8WP&bvGq}j;>y4qyE_gQQ$jAXdJ_jjmyMit)oC*1s z6(se&776gtlH4=bp|7(QpwV?TJdB6|-h2tl$ec)}>(Z%zh>=8h!C3UK$S0lD;l;XzqxS%)C64ns-{!`As%t(h+I+aMA~PELEqnPkxcY2?87O z+g@~d#(-#e$6Zk=D--QAwj#a{? zoiimB>L#?TR)apccZZz0t_=6JX28dRP0&a;K-a33RBdY(HS19X1*!L_QC*p4O*EyI zt=VYpb|Kf`bVzb1*8zz%HR*4UdDQZe2AG~fFhBDJ%GekIMeffDDo~`SD|Bgnvo@V^ zEdd^#F96wKTbRrbCGY>*(?er}>7VAM=*AHsVs7=YvFZ&d=s0 zs^8I?kkJC)syHpuy(l@UndmED0-sO+%w0VOb_k zur8y4YZT%8Cr^;kGN5Pe&XB|bDPkl25iQ*rk7hglme{Fogq7Rl;m5-udiq@&wY0FI zw8n>?4VeKR)hl38b2#-DX47>X)-UTSp>BogmC<5L5;OO!E9o=5Z^iXWM94RviaBmEGW* zy%)?JF`w?5zLv`C*izZkx*!ZyLT&SPNx_u0#981m1j=7TzkbYy$BSEtT<}{`JIMoW zt9yc^W&P=-Q)_7OVJ!&Qp$MX>Bj})%vCzVgg1RB8W!WcW2@ znEP)MgzR#Ik;*ScKTr^@_?Jvmqu-HNuf;SHJ|bC-TFVM0dupb>hIYTPh7ILQA?)y6 z8uM%hwcqWFM(>t`J~an&Q~eLB)>uSti*>2Q=@HVIen=8GtB)wII*fGHGGOZF#W441 zGQD)VijJDQfPO6ZKvL!UkfwbC>0UIZJ;bHi*3$(R_X^qUCnXkD`B%skHEm*}q=-D# zu9IEgABp^1TG7K)pH6)?F#hghh;<#1L@t>Lt1AMK!`^h!CBrpj{MD0W z!uv?{ChrltoC@^zwsx}p;VjVV@rB70Y(n=3|apdZ%!MFoK2_BU^I zOXsR+@t+yw_q180KWLTc<)|E_8x2&(PeL4f_7detMX(&Of)gjyY3Xu#`oh)+(*H|> z+KC=?{ccD4R%$u&ODzHYamz`HSf8E>Oo!u1`Y>Tt63u^KM%&A76S{0Oy`fizLi8S^ zGQ&qCL!?FRtwunuxeXkb8&1cK9W40}PSKrt0*6r9kQS5k-QXNgzJ`jZBsLGN6xgtp z4TC^?-2{l*U_>{abf=MTw5eC!1n|2Vgr4tmX?k&R9BOyphr9|VkPm0iNxr3~!<&_9 zFjzm1I{mGo+2xHyKPHV1@peK7evbpYhl}8oavsqU@{YUTjFbGX_oUHP?UFlkzOeOv z6r5z zZL#RaV>3F}Mwgm~529shza$FRbs%<4k>?0%s}b@>8PD(c?}H2=HERf5SNvi5>D@cY(0t|Q*FgcKEgwTejj&g{XGxz&xly}V zM0f16L@%b8!4lnjXrFyPTuDuYS!HQ-g=sB4(NPX3t7E`?dl79ktf2+(cS(L37(%tV zH*wt>N?$*pDll~v1b}BKlFk~3K23c`%$HL1*;fZD!_vvn#3Y(~X*ro@oBML6|DCx ztQhH?JuJFs{~szqlfiDxXEHnTGI4ufL;R*MfS8TvkwZiUlF3kkH8baerG^IGP*p-l z-G5C!jT6(wllW6lhi^qiCW9csW)<9hs7zJ7P4S|yZ>7_79+72e>>Cwt=De(KF2w{x{#KgFonBI+`m91nWa~kRN8V!2fG3+V)?eWZJr|;I_&U zc6vlm*+pseL^0691fU&qbKym66lmYvhB9U}5xLCik`ucQin6a%SRQGfhO9M?HtRPu zirQPZ(0+GJw%rH?mbC+>S?z#d&vN0UVHO>JzLp-s`PBKsMj8-S49hZ@oo=wQ0?g~-J{Y8S$To9drEo7kSp@ci=hT;qf zGW7QrX(}m*!e^L>6b5Z1nPWG>#5*%Arhf~k*Yy)=*Odrpatnib@7B`q3!~@TXV03>!O7ENpzVeUAUo|<}nF&<1QuqyPTwuh< zHDJ0f4rDeYLG_s+8lsy;XZ~FUlXF}l%wi2quSlnU{r-^q&IgjJ+$lM{ntshOCj)H` z^pb29`4O=NC8a!R+E;gvoVY?oooTTWkK2Q3zQI+bd|eGRyj0L1?Fbq;&XJt--2fNo zWPwKI3VQzMOuDz<1)6?%fYS}3CJI^Gk)!*_hxTz)vo#P(5B8E9k-D^DRKWB?XN^lKl!1OS+X#y zEJ~7kBb$Vj?na@@W~0_Iov06H!=1B%kgTFk8~!2s`)wwzZY-kjzbAlMcr{#T9SN03 zri16TA#}UtAA&auS@D}bFt$^cnwOp+{&fY=wG)F?ZZ^%&D5OU(+!jTtropp+@0uSR zyiDFCDiA}GhDw*O6(t1s5~HK(bU{xZbxeu}>4;*eYjdS;N2BRQ&K(?EH^WxhM5r>4 zggFBnY4W{%>fxGBmfz}=SbQl#g>}Z%>Xr%(c(a<$N|+7~5x(FX5{M2xJb=#r9ZS>} z%_p4^*>v!W2wJ){0lLT5!l9RQz(2PKx%$ke9gnxubf2m86wa+pcgMNd8c{P70KbWK*1PoIv6iHI(!y! zLn=h=?iQ%p9}6Q@z3JrZ@pP_pkEpNdBiZ<^hD5e>6OZV{)cv0qtr+VF)#D=}MKlEc z?hYU`=bez~+7V*uZ$)he>k0K&2R>Y!3vpowPV+wg>7e1&0t5` zKRkkdpHp|L!lh00&C?QH8MzS67@7*Frsk4m#}eqxkGts0iN&=2V>6kgn+FBjOCdbf zn(jcdBXIiGYS_wiNH1J9?Ty4!ImR4SU(>c!I;_ifP1`oHL`=TJDa$Cz$9 zaff_UJBU*LdcgXW$7n~-5=usgP*c;}q(|`%dGPEtI=avmJg?M{J0=@RWaBz??D++> za+yEu)3pScBAiiFgtNfH9MCxR9a%`_(8KX%^d4IQYFhooJWHRxsXsvEgCpo*#T4pr zIt(g&cEBF(Xj+=@K%cs9h9kpvg04@UNIs*VOegkaY5H$TTcbK?hG?M6{Uhiq@=`Q; zmk(UHtPHBXuJnBRDjNM@E>+rKh#JdopdGW7!EkW~9pla<$MaI)Z__R?*&GM+BIHF= z{-66@o<|Ku0gxNI6+)*kqx~!I6O$jR^icgE@@M-rSTXeyDjjg4jcfOlsB|rOyEYuE z4(@=(>IHB;U14x-Z! zZKZ+sXHoR32q>Lr3Qx7KiKgv*MIvQO$g2(@N82Q1So|+Cr}ZA{Ra}dXOt_35{qu$; z<|*WwbTf%D8HDEQl@aw8IZ?~rQ>aw&Ju!=lwEQ{zC0bx|6>XZWB-woSERm#+CGl#} zqQKLWNvc1QOgTiGvh?MNoN%rj-Pnfu{Q~F+>0r9ryapAXW|D1L?<65jhLWGBN7L@V z(_ox|3K*N_k=m5CkS3E28-vzRuYr|xe)xXT`p~!N(cW*8*`>vzyzS*Q-bf3l$|dN{)OJL%LTWOpwUX&AH{s=(9XXBza`^V`(^F zw3h7uF@nlns3-1jI@DviBN7Yj@$qo}pCyHAF`=pktHCS6!t zZ3u2XTPef=^@S?o>9(xf&hkjTLhsVc(b$|!FvVKJ-sv1yd zLrrpK&`os2(|{x@JtH?iyd@S*Gr+0hI@zcjNP@*zqBqx^nkW878F!Unee!e~5_Mi=6t@)KZGDacw~v5Jqf!X<9YnRdY9+rl zuaOd45%h<>K#;r*jo-e5Dz3_=6QXnA*_7SzPjZ5Qw=TWvWCr10+rZ@YG4i#*3u2}R zAj_M=z0xuj8v8bbKJ=Lao9$v@$Y2g$Dwsp)cOk#u<4eCJCO)ZEa9F3qZpM{d>jFm6Ets8&896OZapG;uI&ikS!N{VUPRTos5Ah@=e;!usX) z>77tsa#d>?lo$#u&}Y7|?$A|aBG*73@3f=C=C1~oYzu*%=u0miaHi+8JBg8yQ-2|_ z6w}P+A(e|>AlGk5L)HY*5=tPG+KWgtgc@GYmORHnVkT;%#UrdPJr-u`zC_pC4XIXTI(d4vK$0N6j;0yi zK({aH!{!O;Xnsd3lK!}h$o`v5E1T6I{K`JE|D7BiT%{qg2zZa)e$E2L_g_dzdj>^~ zNz|vZi#SM!K=A=z*cF%y`_`|6;j`1J0r}cv^Huf@qD=p!sZvi`CJle_Mr8bheU;JT|NgWyNDsbo&Rlsdv5ol^9(kEAQsH`X)#=Xb_k5$`1W^O79 z42z%|^_!{h8Gkr><}oqe5J-2J+EBl$=~Ta93B7@ZlHf52Cf|Spv_II z=;{lqv)=7jq%Kn1XBljsJgeCQnvF8RK8FE!v6qzJx2H2FpO^S-d4bFiNJ-u= zOA*;*hfBtPRs-fd2JKz1L+W);plZ`}l5asJ0~@<7#Ton2_I;hmYi$8J)t@dYdaOpj zMOjczM;lJI1Lzd4ge}#-n-@oU(Z_j-bWdg~)qQD16`K>m^}qGd_`Qi75p+WK`Q>TDlEnomlJ_S6|ziiYn)&mE{J7yXb}y)1z(SIWUBqktah-b2^4 z<${urD@69^(wN3V+8wYR^&D7_RHvOG?_z$DNi2=3e$FCu>N7#l-3NwV{w#@#PNhY! z)*`R=Ww7~p8kD@+4i~Sb(3?%AbX=1xu{t}3cE;PHNWuTI{lXBGk~b9H?vD|E2W$FH zOCGFkO+oJKNNQJuY1rsEdc`4|4u84@OeU6rs*yTypA~8A_p$W5j6Gdvv;ce$eMVc( z`cXwE8~U+YpI#aL1ii90ge{h9;Onaa$>Q2%@K;qt4pnRE7!5TT^yDkKRWpp1{uv>8 z60ZZ*Lkx)j?iqAZhnSSgpCRGGcgrHJs383yN(fX&Pc&DPqWmnoSI^UWE$GQ&@GvVB_OlP;mJ|~~t zv3exodrpa_caEdBg@N=TuMNw+y=BFdW%HM5`* zPnUtt#4zE%xQ(Qaj)rqW&&g%45Ph!S`cbWHLon8 zPXkk6v?v}p!QS4;<7{8f`4CqJ{gb26v$+ zl7oUZ^&$A{DRQf^o^+e#iTXxP6`0fVWI$p@JFF9R?qR!_jpDdw-<^rCTad%AN>F{Hg@g<+suflU~y6U@n_c-T^@4Zi0&x>n}Ar3P6GA0%0D_iZ|@u1u!mHYCFQqiIl}@Qsw(E7IW1XXp$1 zg`~e)1M3h$gNy;4lV?R|#f_z1)tvAy{f#CZRzj0^+SAmaIQpNRH>B8@gYh(TxOBHk z6dgaEUJ2((xxy+)Tp)1E+@0y1v`w_o#2l{AGk`36dpdQ?4sxjDvgDZH$>aW1Tb^>W zgD>t&pjz+(-OyZ2CqCRnN68%%`SbE+kflom}SgY`MO``&T%vU4!}Q`4niGIeQJp*H-R z6a%hAu7>R7qcgz{ukVZIn_8k}J9sTIU>0jkd5ZjgjpEGmlGPOjy%Qg{4 z&Wi=#WpQ+kw>NRwvK~}iJwa9CP4`#^(n6F2+S_-4_BkVZbxSjPxJVuTJwHP9kBoyC&P*cv{&(m$o^#? z$$w!<^>2-XW0ftaa8y?FuCfLqt210Og&z)Yl=NZmw?O!rVM6y*JJ64{7HC-K@8%I( z93@fNJUOr@ng*q+fwB5(=!>|GF78l+gngm#pX@mL*MBX&lE0gn;{{~ftTgn>TtOa1fCLl$69K%C^UiUWnUzN!e5i6llqDHO&hYjXbKT> zQ;DaxHnmkaB8nQoXzIr6cb6Le92LFiBJ{Dw*LA4^~bj-e>+0-33nctIbL(eqsxY zSFMH0lp*kB-FD;^e2pBc?;$!aMG*ET6TW#B()djkH2LER^zBt9l27guy*adlba{u0 zwzW)z>QC9EOW21g6ajs|;sRO48NsyfcL>4+F7y{SNQ_gap1reZ^7L=yuxS7-lHZ5) z?FylOkQaTGmPel)_kpa64Dc}=NiBk@EQm02WLqx zxSv2f>r+H-N9vmoJ55JMOUA?2e{;x^n78D9p_gRme?hcroLF=sP7fUY1QmcvBBdI$ z(CE_nWTDR^VzlEnne2WAwO%PlS57Xa;b#QC_Aa59c3R;1{r3mm^A9E6gMSc{>z_%- z*bovbw+vN;?~*hPQXtdEhEaZO5`84>og+V{fOm5yoqH>u4!V#A(^iy%ny5rF!>bov zzuYW|>PxfCqAB#OP|qEPC4pUQ8Ca~GK)n|&fI)W!`Q$ErlCV=BPM))eHFvFrd$J<> zxoR%D+As9^4fm7Whw((OL72z+v53xlWCz{W6CnN9UD1k^)6Ly84@>e+juGBh4d`jb zF!)&@O-GiFrF%ci2|bySw9kAfoKKw!$NW-Z%=}=ei`!1sn{uf6{&af1F_ESkCBO%- z3YaxNRr2GcH?eoqLSrAFK)$ysB{tVFv)Wj^Mg9c1QQqWDO5GM_rNnRnD5W_5eFK8#-VoPS+*4?mjh!D^^E zJL#rkrE$QB-Cb6}hWcJ=nO&sbLR;svv9`x>ghvAF&|1nK?>Wn@Q{TnAjb6-aJ^sTF za#v>RkygyGSP$Pm?Zp(zOPPw4BTI;U$j}H;0 zr!o5v8Q8AHj7|T1MBpzxvTSDNj+Q5yN|SFISyaAMjvhVT`rzA>m;|F3Qxv1vO(C}+yNdROy!!A z+nCwmD1K#S4t};?llT9R=T0n-#dBv4vidw`QmaP$Va~@ShZ{cQE$^q+$OeuBdz}!$ zu8ik+qU|p(dR-x=CY|_)yTE3;=gl|o;h4TsCckL+1blnpVfOi>Fwd@O$bQh>TxL}r zc6QFk9nl}TsG`@LwU!;9|7a}EDIU?Hw)!obJ*$oNv`*&o3NdF<*~+c97>{FC*|Y50 zd#p2PCmuHMDEs^(jqMG;#I7A$jX(WHSTxTHPa3fjUsG7X4*e=;#Z~dR^V>G@;|M#} z`&|y#Puhj|ZW6KOSB?vr^%30P$&XlAojm@jk;`mVfc>iNiO?SY)i>;9lTm)5zg&;&HA@0^V30-4eC&0Pu=(9kpsh98z-M<@0bg| zwmT8qM(yQ&lS{Z^bdK0`O*!6wcQ^}Pn2O&zG~xqm2H3Ca8ZO7(z-nF9R(yQ3y1;7t z$sFDP@{x^OcvCmQ*M4^%x9|FB_H5Q0JX&iE9v*#?OTE5}KM~M@S4~sIpEcICbX;y@ z?xP*q+XA`PuKn}bvn6tv=q0hKK5zIP$@AE=ymjn=-aBsWq;o9D5aDkdHsib-0t0B6 z19sPZ%YW~vz=7@3*jPacXM6W^evbxW=@mop%R439=$&`5#hD#^<5OXrZ^S%y`d1da zon6X`H{QcWg}S(mUxiKGRe9+-E?ByC5}q9JgSCBp!#U0p;VV&L{8jUcmf{mL@GJEY zoHS=U?)ad?j8lay+~Z89FzXoJk+l?C|6YbQ19&)EAI+bti=D^EEG9bW9`Jr{|hhpXbeIr+Q7e%^z=Ic@r}%d@cO1N|p7T6LQF}t60OgMOZDf zj@`Ss8yNqVUw);pC2u4!hjG2kDeVNyI@^H@BQ~|f$jNcHT-UKZ0=H%TjyPPoav7`K zJ)3ombHNQ>cR00WkFcqeEj#$|68AOiz1X7260gm;DV}I?ttHvMpP#9*hi}MT!9t!4 zFp-K1Z#p^;Z!582Z-?B#F)PR7>3w_o0*GpPa$rj9<{xKpi2gs$x9$QrX@#)vM=RlF zea%dLk{T|^tYC-x2wV32IQ}&64j(Bk$Lm|I!d1W0_?8**xJqE>gcn}KH*6nr9tT44 zcW#u}xor2esCj5G;v9<5 zszUZq9>wO|V`i2PG9TRQ(aYo0YwFqcbi90oja+t^j`92jEoOaaZ^%(r=WYs}yHva8 zxl%QCSt&**+Kw_!D#97Z-AB>bz&#|{ES!&j*?^MdCZM~oHhMHH&vvCNvHq2bG^Qzr zj1F9;_S@spdv89SB}f$P<rE;2s7(=UlV+)8ls?iPv_mT*FQHl2)X;PUk^=o>8ZCxI;h< z71QXP*!#$EY6Nl^IgflYe$j1Rp;$_89y*m2OA#jo#i`ZMZCmA;px;6xgf@hP+@$pf)vMJOvGg=?{$XJZ`zN?K9x9fRAq}P`X}#&9+_uR(~SzK!g&GOn70=lm6(aTCwrm3M+zup z<4yGKy*YiVHpV(1kwY@dioA(FZyG;4igB_2!OYq&iO{1Su1~Q(x9z4H6VRE)u+r9) z!@htz>vU;*)(qHV!et^V3y^EdINdBLXZ+HmkP7Fg;A;0(uG5k)NHVSu-A+|Nuf|Im z!+;;8sG^3}4e*h}s3982ub~Tt9_B2CDB-nrn(6+Ksz|0U+4A4eS7w>U`XHlCwx?{C70HQ~&GcoyxQ@|nG@D@}h$@TkZcghtBi zP*$Tjtz8m@)~~H&|68>HInD~EH(suw7b|X3H}gBlKm^ghFTXQZVVX#4g&dFnRGH=k z)UyrF{}{(LlhNH?3pPSrie~g%&@t0&Mz3oPbG_&UGOb{ljy>~ea{Lt3Zsbpc-MmoN zMk{)F$^v#oSBje5NMrSW#iFz} z)J4mwZDAapp>NA8TlABWdYFiEAO2xW(!{t^y_d05u4tk3^?Vew!-L!PcLS5zwwU(4 zkf~Y!@guA4>d6kXnl;;ge_}UHo`*W#iJ-3?1?Y8!3@X3sEp%A#GclH-)Wmm``P*2{ z+VZUEPMIk7!i_HW;V3}a);rPk6MLu%yu>a(FGtU=RA;QVbus(9!r6jldj!el{p_Oq zW6bM-O8T`>wC16k6+4_ThH&8t_P2&B8rn6$if;MB^75zC?yxA9D{G9@D`Swyv{L%} z>NQ5qT@@Ms7fI*1tfKkRwY2JcJZf;`kSeR&0*mqEwAJk!IoX`c`mcLSlMmH1I7*3q z^)C`#4M?MTzZRh3!;w_(<`A>^TLqo8=q36b_F7M0Y=QZkdv%bRX*Kub~hU@xI#@j-?$ph%43T9 zSjMw%63^;I2BSWE4Lb*lG9|Q@%1key>-#4&P6KMlNsLF+M=vvJ3LhZ1M4BhDwFrsn zsW0o#nAeBajkwb=<&D(#(=|5O%LC0bKEsw@ z)91-rbTK;0Dy+_RUsTv7OTz~M_jp4%y4{|`e0yY#E-Z*++be|i(||K-p*?F@0d00G88z0nB*Zl`sA4px_mE`+0=cJ z_L|RU!17Y$eAt;DEwtcCEEr&P6G!N~dNt}1zEg01Vt^8xm%=q^J!(*f+^YF6>8&mE zsbP*Yy142rH+f8_=4(GAw3zRrMQM+zRqIctmZWeyqjsYuqZKqwM4LBI62|@TM+#~E z)kp7gn&>)y43m+@q2VHQ*^yM&dUe~X&MA7nRN>7lv)2;HAK2s7R9vd0d*fU95mA;af6Y*&~W z`>CcJ&9Kd80)_{f(u<{3HGBo*H`qv9nqvfkCd$-6ES|nfm?iXn7Bedy4>B915*U-5 zYHr%JtLVQw?r5SL((FH;=vAm8vbd2XNPYf-4i`q#9YbwY`Ntia>GzUbGcT8ksR(B) zM5|~U(4tX0PN8>2?cCuyEvCe~kTzvb792fmNzeEf(eo*p)aQ5)iw=p^Jc8qbwW|kd zeug96DJw6056Wd!X1*8jdDDdF@GaY*QAktsf3qzufz(bs5lz@DqTY=qO!nUvrXk0b zw?`$8IT;j%ZjRq%tX^NCxx#h5sP>~Ex%4%RUfjnui(O0GK4E6(157t+PePyNP3Z%l zx#-zM9Q%6zG$vH}TlW;c|BW$}_Zn^bPyuc_*DUoN-kcuV_+UQ!)#aVwcFYP!N2Gv0$}>hkBfqd? zM`t37CC_O=dlq}UEF0a9PC=_eJlILM?9fS`2%5Y@jvAZSGSh`Mo^Ml{dCi zGY@&Rcj^`De)Ox3B7UriQob^kG}1XrMjZFC`8v1ZH|^hUt8ysBYCdat@|%` z?!sc~uH8r7hkDrfgR7X1Mai^E%$9Ebb{#ERze_RX$cJK9aPt_We0h-7 zcvZ->;{7ylWjM1`Pn8|ao{19VQkeBc&b0B4(8vxhVVeq;(AT}HyqQ%c?B8ZgFIcwI zE)OYGE0sodo_(bUvnuF{@z;V???0mzPZW@=cLbdu$ES;B_M+8$1t?@hpJ(HE7tQii zW3GIbVly8m(H&hOY+l+srqjR-Jv>Tj%=0kT@We8@B6~jyT$;jOOiM?rz+EPy5iv}= zJFWRtf)d7!kzr^s(-*9V9OAo~sbzB9@FFipO=tn$U6+Pd%f3ZZv)7|OuVi|$b`HIO zzS5Oer_ppfTdHzels))9mzity42iAJLj0RH^oROhCi!9_@|&cO0CRwi`)~-Y>ha`F zvX6%nr#_%7rCDrlna~6gwqZ{7K4Ujth(f`W9&uAmg3ziFO<@m2otAqf(L{A=sy4EN z7dZbBdVOOT$|P}QRBtuCBcj9FrIaxhb1u-UN$zOavXgtSu#Q%13oXNImh|XiarEBp zCC$I(Ps7f)BA0e?_U+C0+!(17$k1UiYnRStzs*%cI<~i{)Y?YY`q)9__3s2@Uiy-m zWBia^bhn(V^O4Uyn|vQ#u{=z5-dHl_pZ-J7?^OzWK2O-AcK5K^hz_q%EC`w9H?zHm zKeE+Y9jNFgZD;=c`N0m@ccO2H7{({M(sjW$<> zGsl@5$oP64d$Ay&9+W;oGvDn*)%QebxS}mJAOArQ7R4jO;MeRcB8`Sz=dg286m@g| zk0;V-Qqxr=^o^qLa6K|-Gt!xU)U<60@)>x?_Q=K}J=YIhrw(!SYnBsZ*0O>f7*{}9 z`%Br~`kL&%d>!8BX-@QcT8iMUzyulF6(aT3an$y;D6f83q1_oX=z+LIX3tLnJxuGEzE}tD`VY-0)#EDd*&t50rN5-l zSZ(x7yNlT_3k5eu=FrJ!`nf%##x?4eO6b=ffPSXrpr`d&^!;^7TKq?Ycj1;18unRB z)5=__S#upK4!DedAFiaG22Yq0y9JEBo)-1dsGvBpNf0N@rTKaIFzXP`VLgOi#{HBa zq1iReI+-7$Jlznc{+0n7V<3-KhWK#ng!vuEIuxkS-mBop4*6Yts)=Ok5jRnwE8KC>wF;^jT=(KY|kO*_-6 z%A1QQBetFOFzcabBG=J{v)?cxv#!x;0&}{1KBg*;I?O!dR(dpVI$Q7T%SQVDr5@rT zsD7`h!1Lf*X5^SIZPvfURqnWmb}cOse0RFSuC8vNTEET-J*AV>sQf1J${uM$mMHo`94;YvUJETy-adl(0~GU64TLzP;L=}dD+-mc(3R8034vv8v? zy3ndX?`?dAe)&FTH9CXQ^qjwJl;UF=@Hw37ZZ@KCHDXzr;$i0HlTsRG+Q53upUH$m zOpoWNGBHl-Xpi+xWMxpnY|;3D9!BL0L?Y^t*`^65RJ~l7i?NnrZI;mmEye-7FV<603Eh0u z%#PT^QhB~4GIi>u3#Xo8ch0pze|j^~wrN|D;|3LachXbV#f{Q4zmgb>+8NX|VKVQ= zt9_`ZKAlNk)kRGz@~ElMd&qq6NfkCcK!5&IN4sjZYvSIg(+ZiDRNOY2z9?>`O5sNs(Z>3xpPga>^cd0^GPc@xuK68>FH$M9!DU%zeY7%e-{h9(#&c|VJtl-v4UOt zy8?x`FGSY%6y+OR@V@mKB0<<3YIIB-b!~NJ<3E|#G@nsKgPWy!@jGp3qu(Fy>4aZ& zPsM(2CYoN8Xy!q)-AfsRtv9LOpG|@_D*g1}1})ZR+ehmBOdP$c)Z>c z0+Imj5$8R_N-~9Mf=LQuiqR;En-T@+Ru+M_N}HG;t=6UGf>nfNAw~#k9~U86zMjV zBbnlAI$N+F-FUl$Y6tYOWbQCKqiYr8eR~^CIv2?3$F

    Nz>^kVgGpam@(a6X^O7y zOXrUK)j)549!0x0x$;^nlNr&2(bVq9M%pH^m#Y5!h`!3?AhCLLnrSA?8}sg>2WyQ{ zmD3sS{C9E4a`qhh_;4lK6g3yEmAWF}=%{ctnsU&LnYU4z_8)egj0&%bW}@GrxwL8J zZK`=foY`(vMB%-MR6@o9iDqQcfXPBHv`v{t-k>z_O&|B&&Mr2&Y9sm?S4+nt6ww`S zBs%!^24iR!Ne$+%XS3?s*;c1yG}I+Z+2ko~iI+L|yOJC`Jv0T~xu41qZC`pLv5HA8 z6xP;v*3x;};WXga7q;L{6w;`QX1rug=0HN^=@>Ol}%$*D8tpxMsYynuF}xtr_U@g6C+loeXcm;(B&noHq@3T2AkN{=>`? zI^V+mtMrcU3Z%+N(*5aKj9AzOROwzx-R+4WZ(9lXr_BLW#Evo=#a`6>h48rksijx1 zDAnj!3$t)?Y?w)HS!h9PJnQ_Wo1&0tdfeWFy&890V9W`opY=X6vAt{2&2vHM*RPX| z$7u;RCealoQHLS_&BCaJ_~xPnm}13iG^L z10`xk_Mc?x?_Q&Rq4Us$v@VLjt4J3`__ERsWoYU>dFuP@I9kamK>L>pt=!znw6paE zBh|WyIx~xyT;=ujYu^l{=DuB^FRZnuG{w@7F6n|cjVda3#*~^?ZJ}GP&SEwSvm>6W z7g4F@7r0v=8`6M3tEl_WYt&Y89F-}@(*zPlb*)z-@u^v?tKBEIe8nF@vSudtitY+} z?QRA)-vE=DUt`%#>`i9p4wjyMV#DSL?_u3Jix^eUBmvkQ$9yI}f@u9fcB8~&)bVc_ z&AWGkomKXXTN#^;@}&%FzFRi1e)3M#`S<{{d50W(=t%>+Y~@U}WyNX4z&iR=aT_a` zIVO-=9!z65Hj#IWb!uwPI-%3gEvS#*7IfKD3&kQ$q_X@Fo!fAY)?eR97f*CEc^YCo zT=NDkZc3m}-@DVYRxvvFn-n@eC6BF>osZ_NI>ZEzm!K^1)f73rLf>^yGdIqZlgvw$ znfd%Il56ip76}9N%a(FxeDzusHg_S7596>(xA}A<&xO5izZK~vokhE@In!~sN?NmN zlyThhjLFi`L3b+;u}clr=_}p8>a2bCRA)7x+ig)!!pgu9vVMc^Gp(J?Qw@z zKA4LhdexygYK6=_Z=jE%s%*N?9CQ<0r5`mS*~jtosZ?hbGj)_lrSD7B2nse+y_vGS z?G}b8`}rE79U8~vEjFUt?~c%gGhZ+c4mM2P$2M-~N)FYwPoyKs?)2mJD)dcz5--4U zC;NHfX4K=+&hY)SX~)q^bdRu)e0iJ4>0~-Tr_vQl0cg8?9u2iUg{rRw(Nm$}OhzP+)i~O~DECCu*gQp^snB8kyXY#7 zoHCb@M{aR zW&dyLw$+!l@~=mMqfe>2`gfEu-vUW;X3_j4QB?752Q%;KZbs&x3-vizN3+E3xFVu} z?(*=ZW`$d*cfBJ|!=a2h=vRVvUO0kmQgZ2!7YyAMI+I#AQuNd%l()iA;jD}4v6B@OP`+H^vTb6oL@a^Utc6Ns{(`05!H_Mqb zno+aS$|zAPo}SIBW`-e=JEcs-?lsi2KO5!Vxr~mkRimFxyt(#oPY5EkW>be_#f(h& z9k#$V3OP-15pP5k71eJ|CdWo4;)29-`}z(HSshs;vCo6Vk6?d zsYPq_1JOUP7A8z2hG-ToX9k`Sc4258d&F)?cyAq{WfE~HeDpYVe5t~w=qOSeR>mle zJx2x;>sa-!rH&4aCIdU!AT5S-4+ubR!(*LzGwrN!8!m(BG41rBSxd4!cXpg`5_ zmeXDMM@0QkC?z{=EFi$Cj&7BbCRUAh2WHm+m{!p4C%+b@BGC_Yg zQg(MiHJi}hM15ySFt4ktpv>1mYCi84&5({_&bIck$pNA@3idmZdf+Z3z2P{D+vLT& zGT6!7%vpeBM=#Rq_*K-?D3#TB;US4*9aJFIfi!njqW%?H=yQVyx9IH?Zuwm^xUZ*| z-Jo}a_Ds2o3RH{Pos#v)V9b)bZ%9JVwJd4g=S6h0Llk@I2Sl@5WqEg>U}p7Eit0p) zSc|#C#Pr%Ku3Uy04RTdQxngILbULAo@df7EY=U083VRgV^HKbNuh_T2EmS#c36j+8 zVB+gL$TM&r1)pj`+fr{(e~n0_UR%OVOEa$V53@%@aYxWc>)j|^{Sh0mCxEKGX`}VM zVJLLkCssiY@$#fjGS6lf39aN9r2W_!m2Y^?ya_tS96K$+v=1F)J{^vrS4x&r`RWI> z?^YQ#dwy9EJ!dy#(5;2^KRw_E2L+?;;#M^ASuA_D=p0-1OrG+4PM`tbMs)MPIPfI% z5Los60TxS=$1kY_Y5a1GXoMbyxh0~o|Jr%*9hpM$0y`4Bv>(sg(1hdrm*Wk`n}Oiw zb|B~;ytSrInXF$v1di>thjL!q`CIm$#{oTcpjKxo-$C*UIQ5eQSGpLJ>iXBXs3oaN zI&3dlH@pSToTLNeLKMiXuhQh*eI>ZbV;%T-O#-jicz}DIO2HO=KOhz%gZ*-*!n*U% zF}Wc|61r1B?F>J-=kjh+mf%c8W&7|qdt0JXWd>LL?gJ|>Ea4=r(SRO7{dkvSJNA4m zN31>ek+e(upk4}u+RL>0)m=QkFTCUG%x$Rt*H{6lssVia+LQEqKf(>WClS{RuW-xt zG4R&s4QJz^3ID5}2~-_dA*OHd;f9}Q@y_6Bc-<>|;A6|fw%#73NA@J?I~xnHrY(l| zR_Tx<(!ON8VI}?B+j6Hnqo{D=-o#@jCNT+i>5qj)XnjxH>byR?h)h9Tm_Y{FjdIPXxkMkYR$b#i-XTXfsAadaAR$ODzhSypf z5X}Hr_@iqWC{MEnO_RQWyXjx?e!Bp?^nn57uKo#TE*83cLFaHspC-{r*-R3I2iZ z{|6c2u~XUp4`U6Cs15w8zH>yy=Yt1>Kk(JQnOLk+3-+Wfg9Ga~k-o|gc-kcs zk{L9cI7rWdvzTtcJ!ZvzSecCRjbZBCZ-5{38ZRE6M~;iQ z!2iyR!p|YH#NIxIXuVB^p+a}hvZn-GSfT|ruKJPH(KR^jG6tJAI71(^Nkr$M2Nr!Z z1FM-Tkd`MWL4wc2%IoEdu+zW=Uft(ItOB*jo(*EeC2;{>a#8?h9k2zn-VhL=p8_Ne zg>&;x%781W2Uo>83g?9V$iO0o%o?}Hv-DE&$l7>s0c){;zJS3?F znwB5upWZpf*EigTw_N51=VZ1K)rn> zZXPp+oy3+*q$uJarzC|YXDfCfVz6v&C76)gh1(W8V5692uK9L{u*2CTEHbMY92tMg z|2jB~!`fuwHfiA;d-)VH2NmI`siwsDQYrTBJq2F%9Kenz4^{gMXRu5@n?S3x|KXV; zUEpIuGI(p4z%ens0eW?7z@8&{oRgz(z@kVIykh=(?CyOQKP@z`)=}CCR^1E$&!;4C zT-SW%=yzCgytYmvMIR@_BbzC{)9Q%(2~Swm>jAhbN|XJLtZGSK$P7^uzCBR4{g z`MLLhgDP2NIIo_?fd>q+^6oz1a4sLHe3T)9=aV^?Q*FuTpd1i&elzH^=;L_K7Wys& zSFq)95f-()gdbeI4*Z{)fcfXo@q;dB;Uvr5xYt^nL~gZ&7itSZVt6XPdwVLU>dcjsCa8JBYO520+7G~#QXX^v} zc|@GQ_|7$Ou&n}f=7GNt!d&_6FEDPzY-0>;ZpYL}DugDQG*tv-;3+JNy7> z5hmgn_+*>UY2(O~e-EdUj?LEasJ%8=U@8fflq)$8HP3NU!86Wor3cl!7tO(2x2wW= zg`Hr<=~&<$ABpjV1&sUM#y=fZ1pchO2aG&5N!k3Hc{9%`-yT|z#F9dV0ZJmrqHi?kz zydmHvErXZbl82{12Eq*+M2LB}9@(X&&0jZY3ZH^k{Drk2@#n=$v7Gx)yl(G9u>bff z*MvJpkk-Cb|%(uZ{ag-+kp2{ zO=7k001iEL2}~YV#q)Ef;Os-Uf!E11px1pXo+>E|H`vc5(nZg))xV#>$No5uZ}@=g zza8S7UT#P3BZnomkJbdiXe}6dcYHkD)57i zCRumQ5#F_S0|$~<;R4=d^7`BX_}X?Ut`2X;A3P1=$GyhzgR={H)vQKta1@F6wE$?T z?Lw|CG9&RX)rjl4Z@9f_1%EH!9<2E?nY_Gx4BQoNd5de@ik*D+@V$sPG2$4(Tytx9 zN;p53?yV0wz2&&-Fpn>J>LfUA@!zB_{9_n zus~IkYF$|uVM~eFYV5fhkU{3g6a$-3bK3J^)6{kK2dmj~o+}JRFz=5mysI(it zcHtOFO=|(?%j$rJsVZqJdj~cz(;?oQ9^*}3AMik*u$G-7i^2JDeEOahoM!7mNF|rb&|NMLB1^1>u*G;Z z@X*)*%abqT`omTvs$>I>w!Om9V-S&Aww8ac>;`9f`W#ZI>j8&kZ6PX`gxmaDaD<#4 z9J^x!z_T%&ynF>ob6f~lZMBDyV*OzHmtAmp?F@2cd>RQ|s{p@$$O94jU+|`r9eCdN z2YANXqxjwQivWz@2O{4JanZ*9>V!>Y9IF#mpf2?%K;gWNQ&0l@h#U zy)XHANEL?h{eecI@Eyg?26C0P30~|-R(8mP5cSAgGb5+-_ZG;5FQZFA`R8xI-d)>u z@sszr<~+QQoxD15`Fm0L_}4U8d@~&<+r}MD6;q-IE{qfqN6q`&5G${LBDDs~Ui) z`8j_0Oba+TEJ;>`1dznbBUSrX>vJ5DI5%r!IeM#U6-DEj{h4I z#pVwi!JoyooY~(_gAwgEev{2(&LV?gFi&3=-U{`G!P$$*b;6U3$4a^vK!u+oT-3M>FAy~$zK3mr z=9yb~h7$*9KO)##b1$%*_zpxBO0mTP4hb3Q08Vc1RmmV?cLU(}UdG!mj<{?XZNSfFy~9&e2Y`S4Y`C)Q z3O~&K0_f+=bIlz=c*)93SZ8Yu{;k*xYMzdRSC^WB!(cTy(56PrzPk|pAIaeIxs^Ed z9-q_do(x>W)M4P2g;2aTknCMHo5b(x1rP6(b3_l$#x}Ln$d%|ZKptENP5XxMPmvBB z6#ARe(H{EmdzlL`RfIOyzhc}Q&q^OY%MI@q=vV}Vb`QdIh@54%DBe%367St zBXVh(;F^3aST`dZEc*E!DBVycTSceg_hwQ+%4vvGIM*1r(kNUj{Q=)=9KqtAqA=ml z6)?|D9ln3<0y!VdNbsJ=czb#suz!CZJlWig_gZOUzHJ`n89lA$nIdBjRWrP61tawn7yae2t zeF+@aeGe*<_&_p6hB#RYGYHhxiE)z-sS36c&fvBK?X9Xro|#7m4*S5y!;e65dLStF z5?b_AF5|ZFM0_Y~A3rnxBHj{e3*h@HV4-R(xE1ja{2mq~O)ID4>&IL9$Htd|ZAL%A zm4nqlP+89z^7&i^U!4HOn-sy)`@#4Z$A)ys8pHX)=Rt{^4oFkg#d9y^1ET9icn_5U*OYN3Hn9e0CM0lKDTU?-}*KJ*OyEor zpcxCM^T#=o`$WjKt-6GIxcf;K^!dY8U5(@&b z(J%2^Z9xHG$x;mv$@qXhQg`v0FM6ae>Nfc3gu&+4P2jJGJTx45iK&7=Uw)-6oFNdp zB9uqWKFO27yhx6-=v;F5#eLvMyxb(5eJGW6If=1MHamTwU_@2 z73*>kUU-6kpA&;udPeb5tQ|Qy|5YOh}9Vdwl69B;T~eV1-RF_%VgWqWyi< z_u}eZFU%>&yw$(?ol{%E%#NmNJ@WK8(|CQl)Vf5YIohn#;Oq^t%y~3fY z_dt%HJost<7zhIUIsY<3`BwWBpoy3(JaBXdnIUhEOO;Z=Y_A@^$8Bpov4-%)XF9{| z$hC0S;~?VmcrsaaObV(rNivsxAOnHdpXW+r^lA`LHAT>-iJjo|mRxny>sEva0uNJ@7q zk{6PVmd)4(IN5TPqz3v0Z zfHUUJ{)pel&Vjti8t`$?2!HeTX>hU8Qti$>{2pQ1swupmOReh`DP? zZe&&C|K3F7pF%TLOZOSDu6c+9WO^{(QVxtB3fKCr?oeh~H;}n1NA3nL;jiEpW5-Qr zfof$l|52_7+&f@bT^;ZaUl^PKxzo*wNz*O-plTQXYJUx%NKhryDDG&2-;S%m<=PC$jom`l z7S145j{rWO=S(i-Rsy#LIbg-}^FVUJa`HW!|Lzcy^iHV$VArs(+G3JLL0j}*C$9GoDh67D%aP-)5JZ!lF7CguN zB|$bgzn9BTF7PJBU$;Sp4_)Bwy&QbfQh@n$_hFf-6~JMJA!%~pksUksKqD&~Xj)e2 zT34n=mKR5W?b6La^7LLDvR-JdF$3Uc#uPXjPykXqgGuxW2iTu*25<&rtN&cB=D$ui zBxa5=Sj9*Us}=TuuDyORwbPF?OF5NuB|?I93g_nExq0G3wL)B-u@GvV9>njMG>&!X z27c0sW_)VP04KGo5lC#f!Jl+68&4Z@CmY^u#9?pd;MAB*m%+HV{HNt&VB*vmr?A3_ zEc5q;_wQ9Ye=Uh1Nr3EZkQNoh$4ulgtpRd-} z4Q87i05RXKi2n*Lasr&hA#wI3;#?(F6&!k%1tRQvaXMcEzOS9l(fwG>S34U7TKaZFu1pbdPBJI8|3rzqs|nn# zZ3G=++Blb)T8{8RN#lrJRMpAYB1J)A8o~`8gRL zTAU5G9a17TPqawqu|oWi>A~qwAMl%hmtf85lEl+H08CyJ1xnU@sn(tmgS)2daCQVg z#^g#2b~O?s3;)h0dFTAVW?>FPL~{oo3o;;n@~t3sB^N$=IRo4??Z%@H<<2vv)POth zr3q_ch>!RG0n>kc2DUo(L~7K4;PwJeu)$}SUl*<63t18PbbkV7xN0QJVyxQsg$AyR zjKexD>+zc-lc2M88rWr^NGwIqkx}V_E>+1YvPzF~g zCE~Lk=kQL?uOMw-o@?8yAY3SwkCUe=kB61s^~jD~Q8;g@A?%-M!*zc)V@~8;pq;!GS5CD8j;o%6xS(O7+bzdG zFC_<4BSc~Lp;5rU`kfOTE{1RJ)OE=#yp6p+3G3^_Klq>jOW+%p_JN!c6&!qafBV0m||+3l;O};Y54L?CSK__gzr{%;Hh6j_%ALf0nySI_-x)(@^~AMB*!NJ zG5gyf{hA7#^1U3Sl)T0}uf4-ji;T$Xp>sHcUkYS=pMl1GPrzHD%lUS*HW=Pyh|yUJ z$`0QFt&jd;e+45l;MRwAwr7A(x>rHS!6DG-+Jggge07RTL^E)q^`>B2Z=i5Pq&CwDXV= zR9DF2JPEmtCl*}epSi0HQ-i(nosP-GVI~hB2wlg|(KCRJolAuI9S+q29@Ai}_8nnf zOe3xh=*IBtG`O8j1J3i_Oif=Bp~nKg-AAnaA@%fp`ynsBzG zJ!!bmfaSmM0WW{2bG|Gc<6l<^04G^Fz#k~VHh*7n9(-PjSBY)GIkOk>qa?iHoS|m$ zI{6d-U&=?23IoAnlWefXvyS63A_dlQF9_@T4p10tOkAEWC5v`tfM*Lo<011FESGu& zyBt1_TYR^Jm6dLI_0h>>c%2J*pP@t6zE+0mS=AtWEN4rX<@G` zkAb`IS6uXWCY;=<2(2EBWAjO+Am;IVuxMf@ze+?7Yt{^beTt6(C*d5Z0uKOR5 ztkA{F?DRpwjZXgl^h$8iMS~Mir$TmpdWQS3D*trKcTUBD2z=_T3(`Tlwgl@&SZV~8vYGe z9u)s&K|<18h@iq7FC59l8*-z-*^L{pM4%xjA*l+e{u{)EN|C?I+JK*YGxj<54sUdG z!*8xjfQB$ z;vbZ|4lIfQwD5EG>y?9p2{8Av!?fdiq8BI%CC*XvPB`3r6{Cqk+fLmnR5?9 zA{0sq?b0TZQravjd$J~Lib6>eW}b6T5sECOlu)Aeo6?^4ciul>W`3G;KIdMpi&N%w zkRPt`laaMp}*tNz@r$G1yg!4z&?#Wzi`=SNA}aBP(lIq8PTU-W)56JaA= z*KMvKdKbl5b!c4YjCDIuJ7W8mzxAcC&e3@vXZCedPDTM&fa_23SJ}98E_^MwI;=Ul zI^=~6=KyNe#To_F?Vb?9#ICx^nHwZ6SYTzx86L{Bl97q7O$@K(+?yC=HRt?b-O9Ne zME8O@$3i=aBe~ax3CqyrSEgAD2LBq@KJU^NoZZmHxzR6HcYDk>zE(^h|C?$Bry<9c zuWoURKgs$BM`ybli6IZTVBDExU)mb5a}Uv7`q7yJaq4=BOg$I;OhL zZ~aw^y_w;h39BCQJ5o8Ex9G+>roGN;^zw1OIXlY1=Sl*)Wu&g6N=|UltCFu)I+e+l z;c@CqLoL6%%Phog zlAq2Pa2xnb_iy6o{yArr7=D_wmve?ce@w8IM*0;FIrox3(?-a-^!+%;Yr2nMezhKl zgv?-+_VTTyg7gHY3!^xdyk}OzH)?#|HZGI;xtRa0MO)A`WW>pO`^@sq3#_XdE8zdG zcjYLVFXQ)*eao*{JeN^VtE&C8zmBu^GO3GJ?&62hn%dDVmW;c9J?G#qKHpYsGDi^D zZFQ;R13&SA8}ngQiZefboWTB+JY!;%$an2XepTlfL6f~d|84t1#w3(-`j<}?q}+Dn4BpsP_aZ-)f4VD^b9!Hd)smI= zf_s0G>pps4sZE|BCa{Mp&a={AHSyUT!EVX5oQG`&9F4pDx&Rw7=9krOzU695{vC^z zR%_0=3YvJ|`J+wx%<&H;{IYFaf!%}@e%N7a#=pGO${_9=KSqzSGF2PPjBB3BSvPH% zGuf$^FZbyJ=c9ZzXTd~eK|zEn$NK#~&da8Px@MlOAj?-maMEiNlX$$$w-={k1*WQAVd&dMS4m^I(Eab${UA0&D{;oB-oy)(fV9U2Gxx@*3k;Cs% zm@P>E`^;+lZ9~D{Pb&n&COn3)B2SmUJrfX^Z?)M^#9L!Etq%MB zry#<6vcPa=CI7>jeWEvJ05kRBDbA54lbPf}duF1gl>lbU5?EZk!EgOOS1_Cz&#`Aj zCZuZ<1>ND}1P6K+F>9vRbBe4k@QZUqtWkS6ereq%{=BzlO!`)Trdh&Lupz=(U@=~U ziTLBlNIh2&R0@p+3A<&O6jOQ5D;e&F%7htoKLo1bweC^!i|F4h(N zInl{U`(-Ftk?kxvU+l`&IE;VN|UDq>>%S{Bmw;cuU{>Fk-5#Qa=(S->#S7*}C zOEGcBefUxW8Ns6@E6y4}8$rwYU_p}WY^K59nQ`0N!%4aAA^6d;NHF^8F2^9-L|}X% zNU%@H#`pjvZaj z4CN!AZ^H5?UE*`PODZ|H-kUSBw^I21P58sZ?a%bY$V^+AdIv6rUzeEs3&lejuPZgA)Jul-E~?2`7q|6BbkK71(pL3 z^aM}s)fk7vKKzAwCW1fv@;PL=AOFhwlY$VvNaA z45md3Hi^CBs5mPy!>uCzXIC|UegA64vTY`F;I@y|g0Uh__-F(_C}kVx*9MkMT4M6{_j2KAh7;%W6_dP-jxqq&|qF+oz;XIT#aKDI-& z@gywOiU60u`=EO^n#@s)2EQE#K(==;>GxHH$&QvdlHNyDM!O;9hChC9zC_(pS3phV zX5v~sNYpI?f%l{mRcgXeJ&eJnor6?6;3wBXsKQ%+*%8;>twY6jTfAnGiER@vvP;&! zfwR&QwDqtdSre5GRY#Smx%32_D98psD+|0|7YM83&$8ZM3n9#Jyzpn=2f{f1BPLOW zV6!w91hvz#ccCvmQ}`X*PNn1*PD^GyV~ipeFRq=F{D0nnyj8)EtWibMhs8J;_!P8gpA}- z!?p-!+UaSaBoqJXOJG{$3O0&AS#H{Nmw`G1TJ>IS&e7@^Fq#8~o3$)~*p;lU0 zvq4VC|5wQdB=6;VbtK@ou0vQk*A!&N-GB|X)1X8=nL39iqw|^F_;k`G=7GEva=-H- zf5-~9KNroyn`wjI`UbkrwUO-e+66Pu{KPMl^x@306iAw_hY@S0Vdk?|8mxDRNZBjm z?OCPNY0Cn7V=#@R7Xp}Dd4p?Y4}>0HN*unQhOXDWh{Q>YPufU0gX>{yX102_5hvjP5!8ke|lZ;m)?{GExD;3i=9WR)jIaX*h;VbvG zf*v(Hp-Wx`Ft{Kv8P*v`(O8LV_=ncRp*wZZ!A}SI0!?b1Ylo@tj??UlA}sjMBMPy# z*nX`XS8fj^-!5fvHNT5?0otAb6JvrO4#)@>{Fn^3^8!iAkpW=tJS5}w(#d`wAJW)b z3f$S^u-;0A-l_|L?p;w-v`~gF_ZHE5rxY?-at2Pl{ga(#R7HQ%LBa151^9jTF5PaM z2ll@_DBF>S$%FMYqe>dh{I-*e;agzpez}aKgZG*nT0ql?J~Z$h=v7=Yst0A7335@0uAOJ#uFbCp*N?Sl*fLg7j;db z;Qd>2@>dyJ2Py&Q#R6<@vj-XGGrZ~&G1fFk$nc8{Zt2veAR~DfMtQ!pFIojpW{CC@ z#p^&O!Uz--T}ei<==~6$E)zc z_@&TZz|!rdw{g$OH27hhjGmDb;jG&LsS~j5r?c%KGtCm8RfuEEd0G6E)`Z()dbqbF zo{=8Yi%`5F2I`hYP&>Ij$nNUK%>^u;e>oGksm`ZL{$bpoXG-zk@mN?ly8-QItHbUU z$57xci_2y9leLY5^tWp#6^m9Dc05jp%j43qKcE4WyiKUtf|s;lT|V(BvLf>~-e;F9 z-Xy_4OW=L5JAE+!GRZw(&8pnb!+g2fXgxL`9tCC6gd-~C@J2Tb(3;6zl{JL#7N@~f z>(^A~R3Lu#oQ8Le&k*y52RJ!ENoZj$FZ5KYLFF+U(9l*9=U4fm|AEriFc$CS-GZBAqc& z6Yu(Lg{mw?V3Vg}tdk41e|VXm_2%K6XXmkF;4`uIDQv)`%?f{emR8gxR2R}(tI-D*}&pT3wwS1COurf(~8%iW1&)`Tqh5t0t$$DP3G zm$X1-^?CMQ!ACf^O9;dH<8k&L86q8^0>KH_!25m^vHZM@H|;?`tO#^x&#Z1CUyh6e z$Dwk%MpRLFZ72Z?YE@xRKsXxMY=@xl@f71tFh8^tCYSWUsCNnpT*-k|$Ns}9Dvj(S zza~tW8Vfv?6JQ*x3^r*&_~F6;*?mPGiR~v zcA=8ri|FNnFQBQz$J=jL125JYg08KlNjKn=$Cy8YWmFR>}RB{DU;Kx`?txoi+#|ZD*9i(@@oq+l|Q+Vt1vmj4b4malt>7o3ubmAf%2>Vt_^>d^k?Yskh zQN9+h#Wm1yt3u+`9*!seXp$FhdZ2tZ4cZUQgwqEu!|K=bFX092S4<%afq&T(M+?c| z&ShMmWf@3Lmcyo~cig9~kC~QCI68&EJp+mCDWht|^^( z-qaMOR2QS&5-X58nNKr4)v3#;!?bFBF@zr3OfI_Sa}8(Cf^tbdoHtEC_h?Ie+x#89 zf9^z)^^dNpuEo5HK>kSJaZo>g9THxgBR$!xnNf>K=(A$LFrb0)TA$BuzNo~D{c#bU z*OcOy(GM`@Q#{%%S^=grBH>iHGll0qwzz~~2{Pk-+eu_K@xs%55vX>v@Dr7## zW=kEqv63NAd51-JhBU-DuoxA)0*fQv@RdwA@E2Z%Abt`sOZKyu$_wGDYdI+ySP1If zli|p?qp)m?A}G1k^A*(RK#XHESM9YmN{V*PCw8rb)xR?^Ps~BMs=ym2-Tx0}?x`e_ zjwKj3-Hq!0vfu@bX@oJ&Ca`hSZcHz<0#DTm_)fG}>wIP^x`~PIv9k|w)8BVspS%<% zY9GN%9*Xc#rdmoP%uE;E-zD}VqJogK?!VZo1T;yUC=7DahCcl=_I~48ywh_EQY1&_LuC~ZJHIT2tMQXD-~pdVyLPeHV(YDI_By zKJcV7m)LJGp)Ve!(((4WkmVQ;*P)#@I+v5BHb~SvJ0N4fD$h-IJ=HqsPx^2a7l$pR z6wh%Rl#!kvNWhbUEWVaJOH9_E^q+oP?uXWqh^PnD)xI(QSn@(6HeI`6YQ8HB=KpiU{Gu_C3&C z^Nh>9Am}^hKC3Wf2dUF9uy32k5~&k&;e9G{=W5CcrRGXtbMSWDXk!giEhl0_P9zwP zB*D{SUEY6(BUy1-6|nfHg%5Vw(s(0^!ENcZ%RxfeC@$g&n4iGz&!=E+i8c5?Sq%G4 zx$qChL4H)U;E0qwZ~Xg(C~ntnCB0t)G|%nEFhLioX)0!{?!6@%;z`i;Fb9)jg&5j> zgqDlvgKzeD_~*G9Uc4U9TRYhWZ@j++)khyP%ATEok)g15$8l;W%2(XUFKDJ}FG+H# zV;6;;g15~AY&KsF9o=(ziVPbp~@Z5UbTa|_;e8;!CW}fp9N*X z#q_?79nPAWPqxjB0pqCC5Vk&+o{<$Nu?zjc?|wZ_b|`?D3`^*%tfpgj?7`xfgCyFb z7UK-fU_zw?#J|!7;V}^-^+gn!WLQG}?Gej{iu3h&4~Bj5OUXM7DXK-2m%T(a>;gW$sv@jUsl&Xr zGjPbs5^gP1hBH2Xym#{tkW zg{Wg_k_rUI?c;i#dP5)gza#;NhtbdKHysdbAqRc)h=#)zSZmcnN1ejihhq=IrVerP z+D1;eDt?GtmbidAxG`|=s5I|ZQ3la(?#88ijiK;FFBl$>&H>u_(~dQ=>yEa2Xt2E2XZuPGjnI-Tij^bheuk^GgHg;@!IYR5$C#;RzxV# zp_*8fITMc`Q;)*+Bl{pHHyMmNbEx?wPf{72L01pwVA=9ZpdNgVy|DT@$yxe^=sY}u z?Y(XASwwunis^qWEq{3wg{6ix4gFc;y&(Yy=7WsDVJSI7Yg4v>@a4h^H>?%tKF~61YIMNk5L|J~d zcskV&-vM^lx{1#6BkZ@mEo9|`UQCEO$Xa>NM2YA*+$r8&^o#c)+@GY4+CS%@qSRTU zJaIZ0&szv4>C<5P`)AC)fEjq|(GYHZu7iuhT1hheN9;|rh`^EyR?QyVEqUhDnQ@>t z;TKp{@f`9wRZN)J{gX!OZGwrBLNd^x1oGc6aQU8h;C16Lcspd$b7UJ?s2hfXWiMd$ zS`Ca*Q{);ZoD(q|&(Yu8)v3436>{Bj7adG3W6$)gV?Uk?C;Q$_07rH%xZIjYpLL6J z0`DLBd841b9C9DEPK$E)y%HRMO8BaB{g6>SuS!En=adewG2gtZI7ahCzy zJlRW8ZVoAWtOX}YE*z>jPVTB+!7F=LaVt7kU|M4%>aQ6GnYoihzLE}@t-KL$^n1|z z?q%f9T5TNh>0y1n55mjmmH3}vx^NM7hk9`dTEzBqtFE1dpqWxwcPa*GR||dXAA$|+ zd3wD_bmpFDgFwgSko-9rzs%Pos)jew)24x|cuNY(GdE)P!!g45mHXk%h%B}m9-}|+ zs(^0PN9<@SLPhlhG<)7`+!uHi3$hcSL#YB%pXQ*7%UTfA2*$ntR8gUC4v%}vm{^7>Ipe@_~P z6-5H#7QRq$b)<|$FJT3d*3WR0yAr$>KM0;>d2qUBHXhg)1sShQ=58j^gp6 zJF1(mo-V5I%z1_fkc_RCMN7!@2I?~U7>YU#U>_u$hn zTH*s#h z1G~Kl@K(Axw_{fd_@!3kt_&UMa4y2S8D=y{Ntx*UyM?mH*P+b)w;1$r1O6RbLp}fA z1%8JKh|(grT=6p==V!bi&oB7m z^Lqv~{_92d(d)M0c4eXY#Yi$F{vtIb#gA z-Atfx<`g*{7lX3~3~6k}1>#!K2tOYw66-&f7!sz#TQ^Wn{Eg1jwhU<=6MO-dd5io@ zHZQ>To*qnn@d!7*dV=y_9;Y zaL{H?6qGe5k@5j&5)~dnnx0pJ#K>)G?zI5E^-mJ6@W{s8dv$c~qQ|&(t{qOf9zj+8 zwvqTJnh-ruz_fcNkT@kSX8stULm6cxm??xcfw5$|SP|_xvJNXm>^3)rVPY!sILWS< z2F1l`v_|yamj5oro8BLXAq$>y_bWVQFBzYr=Wd9x%sUCzL!Ll6D~slN7a^Ek1#TCA zQLR~{)arLBF;|QN!>f)UuP`1LzMKtJf==9i_%xNV^+dJ0emp&<9h{#;6Q*}MF3cH+ zuYOEIB6%Eq#$>S;5^k8M5sNuG@42qVJMk=cJlS=A4~ft9U58wg-Ns zA!P&FqsF9aowV2PIg!*0#fJ;qv25O9Qa`+bZC`Q;8Y-29N8h@E`;{+j?&>=FXh#~i=Z^_i zuGb=O_eJB-Pe0UTE$JOO2^hCriLP{(=2e~ugna)D>Jd2|XYngVY&s2^q&!WSXPE;! zOe;dv`|$D4dbvl};0&?D#tD_x(YS+E&stLF$+| zX$`7PJ_g-pnPk`TO%QRDi@c0<^f@wKSmAdS$IdwdXT+ZoosF|F^+PsJ?7JYF?U)D zM2*wLZIx!c?k+L(m7D`XAPW8+8^#!F$JN;qh@0HV;MZV(TcxjgrIAu8+!W{;enfa;C@tG7<@C7 zaEpzE(??$eXJkB_n|BZmRRs9U^c!JC_XOTdBf$nQ@$ZKlSe{eK+2oLgNtTm&j`)|X zG?jst&nIAb=m8pjJO=-HO~T8wPc!2GhDBWAt=RG~lth|%qQNIoPj!qC?kUVcRT0Pf z&B`wD-LK12*>6HMM8Egokp}80Duf@QUI%NiuR0y3Toaxt-363y02izF_(^o}^q9@xJHi z6JFA3u({KKkgG+yF!FORdOhu-Ib+0lzxs6WKKaW^zOI4ROFr1p zssJjv)zm;<7e8(=gin9G;2G~8v=_RI{4nyoXa^o0ee;Z%2^st}?-osd9*o@6Jv2K} z8BP|DgA%tB{J=Fzuyso-ls+n=xOfJU@lb%LyY$grGm15yIKu7pngZ(u-cT_i76-hZ z(r5!4{8BSY$5b6aGnmCaV*4A;R-8lWvV5XVchWVF+u*Q+BjHb$5x)QW5}9yOzjfbp zSo5|M{zz(4iN*UtG%yZVOO}&agL_4s$Vv#5SA;J*$8jiSDkOZ~3_IK1QD>wU4~u%- zK^J2&Yw-^fwr>MH_sLu;{2RtscwTUj^mIw+8=z|EoSuvvPN zaM>(b>@V6tf+RnZ@#lqDIj#^MepSc1=N@SP{3%{q+(#QX)#KdN{ald{38KE9V@otk zsOE%Zc6viU?W)-hdfhkJe}Ny!2k}JY{*ppTi9ne7DhOwN8BgC`Is$VZSCCn9a+o?# zOC}9{W6G zeLXgzP0$RQS2zjxip~el_<3}RZw^VQwO}*iV(X%&M?mSW(~zTf8Rt3{-y&y)a&0K!{eUg*vMl~0GA{8~-Jh^k{AkHQpHD4gT zG?dFIh2BL8@fb9zlOx?Tv>~$7j9k5Ag0>E4;P#)d!T(a^P7ait)5>Z)Clc_pmxmID$R4?!n z)@AHL+mpv=ai%GZShdpaPf8&;G7A*GQKGB)isbndP;Pw$UDaz~^-?ZZ!(~1AP6(pg z^oOAA$tF0tI2u2?&cs7kcvxY1g3bS34ldp8?0vr$YIn^Sm5e{a*%2jFooazr6As|$ z+zsgc?>b@9Phs||Iq+HZHeD~8jmdermgH}n375FLaUb5LpAQ&f#-|QUDSOR^XgQLe zHP+}{Bt`e(5`5;}2Ga5|+%%;$h>kYFiyAkePt>;&PqlpOeU zMpd}vwU+Rh{4v};6iSvDTqH$~u^7-CNo0~@n0xDQad)pxr#d}6xD#uNrm+j~$=!T* z__{ABrgmYLZ#sGK<21-^m4Jgjvf#cW9JGt-$@=`qf+P{wUPo#%(KQz%%d`UV>M9fR z#GFIg_M{9B_ljw!87xi}`eoVh9?S-UR2?|A^)S3-~-ClI}dY7FHAvlIy3{c>B(-U>AIh zAkt69aWAe{+f4r^fX|#m%SmILDXtn!Ts~eX6w!t%V&#-Pi@Y`rGJ# zsd1R@u#BYoWZ;%i4Om^MhL0|0fzpRZ^inPlA5NA>-&I2D(;yBR>q0SwSflN^OiHES zplo3hH|P9V?DBL4%$kF*H;Pt%YTdc@Q!Ls4m>3$DHD0)BgA3?uUWZRpe$uKg5tFte zgm_oQ(E{ZZII&U#WJ(u+-@Vz0I@Yl3St4TOQR=b65`Q~JpkE%Bn|_`L@+Sk(%XAry zwAcX+qI&cweADRd+H@+pux}u0$ z@;P$ZuNI}=RZIQfzVX1Q!&hZof0BTpIF=b$_WE2z@9I!Jrg0{bBcEKFX|B{vqr z_vjAVJ?e!nN9*xSK{Yl$JC2t39MCHIB0X3gL%;FWQSEdoRA-cf)w{9y+0+nL>Z=JC zRdT^{v;r19)Sy#fhERH*H%Pm$1#_LL!p#PmG_!YwXpK7zJBqtN(PS~M68XtwJ_S&V zk0H>XF#*)(H^BT!FWLV}@4(4v_lbCeKCwH{NSc~G;M4;{_Qty%5L&K-!BKN)S9=>- z?XyA{yr_fCSc|xlV~!Wp@F@O@r%Sjsi!z9%MKEAf6lJz~OR_h({@hUc<#)7fnm}GOmE@ zk*k9RTDIim?hEwr%GuEB%i_aHlCYRF4PL*uknFIXj$upQ!`#4h_`2-}yxX2mFV&WFZ99~q-F+o-IQjrtx_;nb=tn7S>BHZ93ThiNzIw7lc+C~gkt=0hj)`tNLb zb0LnHNr{HX{hwiN;82(DJ?iGoQDTmHe=YdZ^WQ~EsQQY##oMtgdD91v|AicKPJ5=L_Go*Hx)ph zT@B9IeoQcL*bl0uJi-5PCsYiMYy_2P zJ^H1u5?%dc@xXIu=JyjHw0!;+cbzrj5t*^f=8G={Ghe!q&~iDfPxXXls-2LZuRwRC zrJ(w1H(2?|2lwns<2gRa5#5+0u$+GwnvY8$W>27I2gN}waUT9^AaKjF18xS- z#G*%OBVu(3cD)2a)d^^PFV4H8v>py0yiA^) zUk3X&aiDMhMGzmfrPAN?Kvl*Kl79CSqmu??@s??Dg}7l|ix9#W6w<}3_|?a5qFUdU(99{!4MZV}j`r6ycz-T*ayG0e}yio*H;2|RcsoiIgS zH0*~p9@=OL+Se2Dgq1zM2}%HUonAV2{7gK!sSg$zeHG189mDY=$Hy7jIDF{!nbDRl zWmYjC$n#HzxTEY3NziUa@ko&$GRhc}mOi67$=zX(k7|v^@VI@yR$dY4NBGo{HevT6^Y!)LIrFY<=&|bW5cN>;{*$k7{hd@zB zIc<=6MCZP%$9*bqY3xRSF#fm@iWe&5q!Z8a*7!giihD<|%rO*hq7zXm$_nfK*3jsT zdAK~qgJfKN$X>})1^KppaI`mrTR&0ckS{s`1>ToLzVj+#@N_<0DODuzG*3h8&9$US z^C>s>Hix;gY9;u1JR#!;uaGM|0G00=)bP7D?9qzmmZ(j^j3e@3U`O!AxEf6Ve;$fm zhu;<_lG&@fLH@5f^^=&4otp}9lB~Qix-1AnLVshUjtmwW%%D>Pioh;Mm87(V;rUoA z{FHGHY#(gILMEMP>ZF1QPeJ~!ya_Hp({W&*J>2fx2p?-|DI;}=41}B~H|tk|@+(g& zHNyu&F2~Ro5p1mY#sV1d*@rSoci7)0EFI#;;=`R|@K*OtqW=c4+q4Xp%o+p9-4Sr) zxi-wM*Q5obGD4oF7uLLDNoTw0|5CmTj+Gw*=GXmj$+hNvKQBAw-I9X?_!Jf9x z!w-iHFxO@u41G9*X;+uy-nnPMqc0As3oc&^J)=D(>}bdbP+yIn1x2aH=xFD3*vq9geb7Qq6y1i z!G}(fYdb?7?rhM8ca6TNxM@FG{nG|mJuA>3SjgL^%fh1?F&thvgGv_eCDp}TxWzw$ z4R37F**XGN-?D(M31wh9Qw`R6d?At(9zb4~D~$>3;mTUsk($iyA_rd{TrJiT<WC`irul(WI0rvyT?^Y>jCGbv9#OR7F0LgB_=GkZku6lxzUqeA&i*tT#ynf>Aw zdivfaZOXp*?o1Ud@vR2sqn9A!S^@@5?gRgROHnRN0L$CO;Img6x9`a%M*j}sbna8~ z=pq;Uwlu+urwtG$-Ao=$ECiKHqvXK%8==#G_IMhn;_y_ zA3ul}!p-4MXDz+y8;Q4nP67E>R?wGli%w7Q7TV`JiQMSJsN2M1jqx>n=+uQ0r3WFw zt`gr)R)L%xk;`sb9?IS1i01ku;hOP3mL{L#_HO=2cBa@v*FZTmT`q+psTcIv=4_~G ze~2!5MmTQB28NO=pzzsQ*sgpZk2Iyjxq*eaUbc#SERMj{BF_Sh9A|yz+@RD*2&(Sc zbaFuqCW>;O;V)012~l)JC57zVTFo`$Jf@+}lE83{V7X)%tafW->qf^@y#*%ZvxJaV z{CtMLPHOVjSM9{)Cm)GypA}VQR>ATf(VZW38PD4o5V2qVj5ucBs1|FiBu4nrSK{maG3=fI+A7Z|epvwIytF;cZ zANY{p{rjMT2BkaE7KP6 zgm>3a9jhYRJAEZQ@i)Zb#Sck#Y(DvQa0{$lv>Cd1i9{#89dl&kVfJE4;q^JU32!8c zy%M|+RQpa)3&U=KY<~l*YIBti4%ZUbTOlO-O9A2ETYxvBv!UKc7Zz!xP(8Ju*yLn} zO@9ty&l_>xzxK@sUit;-{`8lkUjXb@m518EH}vSew_s?bgv$Lk!kN8d!UgM< zg}!1lc_cXz8@gxlUiEN5$AS{)p+GuLo09!sTKU@fi4eMlL)IKDC$)#q!sRtpP<{0R zSvNxi6zYVmO?wj5TbV(1WCXX;Mi1Q|US*1wHE_o*`3^(rs^C<+8=KWD$=2(Op?`KY zd9Sqx^YArDn>`{;=8ve%ykBI>%-teyl7m1oRSqp!4dKsiW9hxH0X%0f&8j8#(9`-^ zxIc2Tux(2KDhJLaPqH$=Xx%=v{nkXi&MrpJZ5n95OALB@q=ngT7a{hbBII0}OU_(2`A1Pq2 z(JT9Oz^((ShI$AX+_}bm7F|emUhf9aTqE>#?577(3ux-1DQNwBn1qYw8|FL+qaKeF zXpp-GeVI^z#e1!B^T2$}N=b${6@MWhsS>ve23f=6%OWpu5ZX>x#vAvOv9cnV-b~~| z>C-8|FUiIO_L-u-*c{}k=<|lMbjht>)5w1H-PFyt7cTrcjw>wWd3Kk?FgfKk`F=za z1m4Z+#+|UXp`vKFkuGA)?QFGW4Fm5?LA^&24c$fdkbNX#J~$ zzHQt>&MV|nuR%#VFI9mTsnCG&o0_pn+8aL2lofUypGrO%27`-~9&cFBlxR#Ur@aYB zA*gv3as9bO=+SW&6Hm-%v;Pzk_iHtD=a>;@!If~ltRTznyWr0{?bL_w_QydBf3Rtn z?vcipK2)5bf&=$UK}CNo&F@WwQa5ohd-NT?ihft~Ry3xH?}n@T^|Z(HD;}NIPMc-) zd53rCA>53_?ctY*;qfG5SoWU0vo)gDN}^d9qe$2!a=qNS{e#$-{*R&a4y5u6<2Z!| zMMcRdMKlme-1oWfv67Oar4&k0N{Pq_*?aF*Mp^At;=a#)kM=PaSff-&bp{`DQC>XtfjK=nZ4w{d@J+xRR!Tjh`}m}L?W~2{X}Y#! z4=;aNpFF(9E1PIUFDI|E_mLr^%Gm#IWfIk|v#8`-J>OKYkNh^ALaJ5Al8O@H zeqhRcax5)_ecL^hI4z$=b=qTS#Pmt6IXzQYtCc&%)Y<0bq!ktN z{I|R3rJ5txl7~AIscuIF8{3gh=B^DShs@e(rcN%=rKjnr`ZjtiriN%$*%0>!$HiqD z@${?9czW=)AD!ppFHLQWB_{ck$%1QTWaQT}`gW29$*vj2YSvGqqPJT7ng_SQCOVcr z3N0ZQg?{F}8(R67rbTSqr#<{ydR*|D3mN7j89s909qvK&6{6oUnm>PIJJB_}%GXX_ zApWQ&rdv15k|NK?tV8i2tILjiq#Dv@{%5=31ydOzHCo(D?SKDAuHQdHJNKB7qig%H zp!SL2npw>{nklhxv6H>w7fJ2HN4EwXKS;kM$XNOJjBTy>zbBy5p2~>yzyVsn;A^vUgM-UBHKs zv9Euz52l#0&xG~*dmKZS%dB8We3PTKCk@%VvMWf^&k}NwzbSSxI!e}u@1-F&S2C(nAV?y7W5^pdk zpXx0h!bfCNzAUto9x^y8^?FxM+7=|zjz?3hZmud~jSnAYtLI#&@6`v;(b%4 z+ahH2iyg=f1%K9Z%_m+=_LA)jrxHbbL)Pz%8pPRX3t5*L(T`hB_RH$g7QN5pvsDb4 z8nr-}k66Orkr-OdS$ChV+j53{U%7|(tCX?oKO@`PeMts%pBc6uRyoMeqrL3>?E~l# ziwZu^?;Jhj(aYOKrPI)&2HsVwWtCLM@lB?;#VVWQNY$4U)XroPb*K{Duhp9Ds=-%@ z-H12DDc+4=Wx0!AopYTo`co`jICZyl`IJK9^C6D5d#ls3v&%`(#7Q(Xbc=Mf<|g(C z{-%-cd+5RZR+2ZSm==4Dr$a|-(cq~`q{{CkIo54Mteq5EPxwusi=N*nV-MY9>-!gw z>mQLlo$o+~%vEHUjVYks<}y~L?`{eH-4PVRe8^OxS9;m~YPPFRos>lJovMYa(t-e+4 z?ATb^pE;{_L;F(d;_5|n)$Qoku|w(L_n)O2{wl4Hn#Pfl!ran{m{c<5W3*V)*@j)V zJOFT6G-}<6z^al&{1^P8B%gaTu^a1|2LT&~RfS^B}1~Qk4}U*`JdtTAnov!#~e}5w>RVD1HX0 zjfny6UpugU-wwR>E)}m9&%q03#&B}Obg*gBfjHk*PDRwl_&uY{xtga;tla|<=Up|c zJGjPD=J)xQ%~uO?)3`!Ve_H@aFY-b6yb^Q=jK!lX)p2f#F}n6ez()fw1PzRi+K8R0o8aB9P0%zT z798hf!urWcSkRD&ebrgm`fx02e3%Hz?mDnz$z*6UlgDA-hXLsr3_neW!?>(u`!yk8#Uk8!YYVtk`NK;-6-L@JnDZB4 zf3_H|ys(EwX9t6C$zbdpIt1$rr($&668KzWi*o~KqhZ%1{OR4tOmW@KCtA~dw+z>kK{-2Uf1%!&ISnLLl% z%$4`@&~bIQq-Op7mWs8nnV5rZEw?;hTfS+2X>rFbUsNiwh4D$c=x#R+ZGB8ILGXmg z>iVLj&JX2QrQ@Exad7%iI7Svn<5K%&c&oLSCT#F$A8_@7*EkwoF;$)dRe0*U) z9<*aHXow43+Pn;UD*QpXBggQOCa~t+R8Sk{1nfa;2vyoo&XPyTi|M$nTMz8^ z>O=63N#J-@1m`@bqG8cgY`8iLA6hb){Kpu+Ul*9}RR%CWu#LG~ufl21I4^PWR$%OX zhC@j1BD`DciB`wb@cs8Z@EekYCiU595?qS;75eBf&;YC+8o=5Yrtnw&0mB^Lz)9AY zbJIM7Bo~fOfdvs$@OX(n8fi_%ZRLgFva<*`pD09?q&>KAfE=f3UMw;!s1Y4WerUNe zJ^{G%vDoewi^3f)Zt04{63uAX*0U3)YNf#F3vSRnW+jeyaK}?J+wp5nDvX((f^%La zWB08B9MPSQ(^e&c!p=lEdN&=O=Rf31%-fl^*;knmg};ojhA{U{-MHZ)0o*Qi56*gZ zENl*rL`z{k8~-2$tDlBK+2LCNJ!~Ql)c=OT> z_FK&XGfo$;t(pd>BaA?{*AdRWGR5+Lrf}(-87O|60nLhl2Syn~ewz_gNvFay&+Hb@ z7pazKE)|NFc6xK)KhB2AMnk-`c`6nQOdPf0_K2}87&nRm)?2~!S%t{5d0_A;7rJeC z!ObhTub>c)7($5rGXd_`0=?yX2mJCxaU(VCfYViK3~SBE>b zERUPsJO;E(RM6${FpQou4XdN$(fm~`=xmP#t;rcs-jdX!Ip0V!UDr_ZLH&p2fZiYu zwq-CK5gE*h<6TS|zXiI3HlcXZCbUxx#Fy(=gTKvMyfR=dh8af*j;UHnwX$@!as=3Z0udtri?T^O7je2Lkj+Rgof+nmpmzufGD@hFoO z1s$f*uuLr(B(K8I_rz+L*}D~7)HVTKvkLZFx}xpjWjNrPJBHdNV9J&RC=E@7G0`cI z`70B&KBl51G!>(Q3vf(8yCgd7jcDum9?`kmi)l(B4u6Z1Qq^VzwA6A1#3&_P%gq zpgwH2pMZ`>CgO~Z#^@lw3(O)*(Kfvlty`)v_Q+iLY&sX$?RUf-MvGCQ#FRPq?x#ex zdI(piq%ClV(z%L^oy?#Y;Y{oHL(G1`*%G8>3r*jzGC!|{K||?NCQO~+zy0&kZ2nw~ z7Zrk?TqYV+X5x$8#dtqB7UK`d2%d#DCj4hD)6=sOsv}loa{MZcw{piLrb!sHHWfk? zvLL`X6Jl=0w3JWShDw(larm!5)KIve0^>B#j4RQ}z&)#WqT1sk)JZObFYPk;cD)itYS(i* zx*xd^G7#>bdd*B7{hYB9+%HR%lc8sEGN|aKfy@>I)QT@=dbmAI$)J=G2w?zRdF>#fmK(*@0^4n@ax+322A3im&h0ROKX60Rk{r)vQi ze`7Vye97YSd>xLPq6ve#>PRG%IXW)R5hQO}!$04u*bt*&9cz76ws{4U< zsUO^ay$=4yOa=AVG5GFFEb^=3@V<@;eA)MdJ2vke=eqnP_dah0PDu5Ful1`TyUz#0 z9_`>l{$z9K2WD}W5AJh=6_a52m$~ThU>?@q6SM=#li*mBCc4~G$Mt*EF{d>J3(uv) ztaTX>FV2JZyCS85DUF%T#q$$-UwmlKG*( zEXE8^fXD$^XjYboz3t@~F)tCEWi#;I-*ikqor|u!Qn10Z02BiXpwT=DM&~`Zz<;|W z&!44;Ud|gS@=l9K*-se|Qj-O5Gqb_nH4VN6W?}P>Ec~yq09CbiFxgfXOs6HwG*qZU z(O3s`k!*s6_t!&R_BybMTncr&?sEp+A2=wL;VKftplWn9USArE-@BvHdT}`JYMBlj zmzjg>MLn3Gw-#m|D8MCOO7Vc@E}SrSC5+37!GhozB)y3^O5Fe!ejYD*vwy!tVTU3! zL%IZu2ksDD6=9&!9}Dfiw#(T8)#otb$o{Rzgzxc4z@VNJ!d%2Oh7-dfgDL{v&vI zv_6RDw*C}N5iMj|F3rM}9W!C|;~8L^G7mN+rsC2Eo$4+oBF#g78!sx%MN zS9{@w$HVZWj}mb{9Sv{g%9sdyfw6yXFE@056L;+{y0V{3HRPe#Fs1M;rG)xkYAGv-!kk#0j%(AKZ_Qt z7ogYqIq3LpHr!f18+1HfLD#gGD~kQZSbhJ-^gnyg+?0yB?vYcNB+zC|{c;(vrv=>E zIR#A8hdjn-sXtTSwG66S7vWdEMYxcx#$AdjlFJ_wMKLqiioRRC63GlNvL=XPzFvo=^P|a(lxMpW5);!pQA)BvC1}@-5 zLmuUeP78clpG{Wi5G8`DD1b8}0&7dW@SdZh5Lms}Bi{?7#hcUB{ zL^0dIp4r)Cjt-hLVRMWbpcet@=3;2CFT{cRg*ba#6@Ivu)qE?aXqJQVK+B{Tw2SEF1i}xbp+i?XsC%SA_5Dg1w>1$bb|!*SMLc}?oe#(O7`!4dclX6bf_qmMpwK$A zeyJEH<%!|w04FFfcESVkF0hMtf|KpRU{UV`Dfg_fQFQ@k&sW4t>C>^e%@7VL8^Y#i zg45ugDi&*vffWKUqTQC8;cY_@UJBnV@FlbGV3iJ3w5wu9;#jOr z7=*PO8SJevhm5FM0$0@mw3UXVlEEPOwQ~rF9ebIeHi3nF9)DXq*wxJUyApO$*Q9(=MVo?({9Kx>2qQT@rsMY$I+p?jByCc*qe9KF3^QL>84D! zmkIaxnmT7OCXBP2_k@}K`W1Kb{ButA?H4ZMt>EK)_*XR7Nu5z$@JMoWVjB7erhscg z3b=dbLjKI>%#HzfxsH`@xzR7Wxni3mjEC87&hAJZx9WT~x5PuB?=h;(H&&Ms?U^RB z9gqk?8L9ZQB@rcK%dq?HS053Gu%c)jERs&dt^E_R^RpQ5yzS-Q)U`5S z=RRTfb(mP}ioVVDJPTn?*<4}5of5co*THy7`ioJ#sQ}v-HZmWDqe}Y(jtRs@rfSy@ z=7@L$IDB_OzftbkX)p`z8mt+pb>{Y%&*lDWSK<=;c3@m%82B6u1tOOUgU2^AW8%(p zGlx*_i{*9Bw{bW#b*(X{{$wgQVwWK&+r_{JR)h(oW?^oh9V$Ot14E-7(XneewyRA= z=Yt<4DPLAGm--EvGTB2+Y6An~1mBSIdP_VZc!dLgyyhD2zhR1eUNawd4h8o}WB8?L zhDq1V(Kb_r(UI}6(Krq-wZ!71HBtD}@U$qPszmZ8#iYK0w$;SgOh0x`n0dEY8AlhHdsgAW0*;@RRObmC+&Nk>hbLP%P#%TF>=E_7PY``>fZ&! zh9yFQP8<%(jz{;pL|h|N!#Fn{;WiyIx za%vfuyF?CF{YZcdZv*kfn{7Czc_jw&dN{{zJd}YEv>Lo+Iu_ny>})eRkJ%Tw!`mI1 zeA5#6*;I~8=at}+*ed*Q@^$9$;}O`9`8}(aVC5<>e|| zvCIMeF2rL0j=#)e!E08l-OL?5se}$=9*7i=hcVuVGbKKE=7HCmecWKb6U;-i2F5zQ zoq3~~1G7(O;K&o1IBG-&nx4GI9Jza*32(9CW*jwzkd4J$e$^hPIA#~qerGiUj0cuj z_zJ!&H&9u*2CjRK!e5n&;QeC&47zIovU4_Yzi&?z{rfJs$V06}7oCUU9s_;on{EP` z-6pUgrwuZbICQO}K^pTpe<{=>{)ItkYc1FAi@C!sK(gb&pIG6uEs zLR-^+%*+VdVG&+X88R2% zPYHxAyZX3oawB2kQ#nYWV}*Py9~+xTskRi{tg{YRt+{vOb=(O&p=}oF>)eXxW4EDo!ve5unGb0_ z7I6G_AtR^c#pE}db7f~#fEyGKYMT=AjBhN8NG_fgGJD_FBv>3B2RYj_;P-T0l$TKe zS;1#6`jE`j&Yy?aA%eU@5p35nf=kZk_)yUW)XE4XFZ71#_e?Nt=X)l2(EteXT*#~% zB)C=wSi{*3Ht_lJQmD}%0;`$_qJ!iwcgFf1x8_l2b8vi)NJl46lsab_lWJ&+v%Eav zOu#yrXOIBfo{y9)Q*;r{>ae!_wMV1H?NuwMJ+^^KFyGCTj^E4_9sVQ1F`gp&_`Rj# zq*Tewq|KP8uoNbEyMXypV;K6imNPBZg|FL9AfwttsB^c%u;5KtBHoC>qlA2La2Otq ziHGZU@zCy{2HUpg!#}4KJT)=}lbe$9>cAMBJtzUluXyk{FXXq8p&-_F#QqKkoc2l& zFR#o%z0;Yn=xHYG@Gb&t`}xdZ^M%b>PYW#7-d`15t41(&rz0n;vzl}Mw~4W>AAlbc zWZ;yqEChY)WM0cOOJW1AaJn_0I8EQFLM@mE>oSt@g>ozwRgfU?iM>V4l;z+7e-L;FAGYqzH;~Xzhi3f6Z7!r zMaF9NDEw6Pi>d7U$LRZvnTIv4V{V=q&gdwQWn=_4eqeJ5BRZtO z<$Zt69IN@s_(rP2tM31pze1jVYN;`PRkK8evuWJ+*Ott~OJkX-F|y3g&$GC3{c?=0 z=#|JRbee@o-v->5IN;-qxwz|)D}EEQ+;L}}V2A1gIFL3MnlAOU{4@_|THb^(Ydv=} ze=AROLtF+juRpF31$o3tqVL*bs;wP73AKj59WF5S;z%5^_Zbs3zl(8q&fx%)G<9;}{!xN7V zV{mMbDzkiKDW~`#lWWWxiF#+snXc=doECk`McNDXsmx>UPw_`4cjy;pOYU=Ko5cpn zC94d^JNYuB<1`pPN|kZICwZv(IsiUJ{bh^{gSbtn-di?Xy=_ka*VYm>-3lbtQ_;0` zGTI;g#w9=kr_#_bvigi7DkHW0B}rOHF#&H0Q<+j3~GWpKb67MHx9fQ4CAsP(G?7Ei5! zD>tjac!c1PT00KjRSG%s4n3IcrpwtpIl+w3MrO-uU0A(&Icz*{kDKEyku)0Pz%3cj zwk`ud>SkcXk$m)>-^zWTafK-p&iW?|9xxs+T<|L{20xEQFeYFN+$-qe`Wm|#M~i37 z-TfDs{uW;xspt)&d>`o54uk3^nT)!&8@GB~23PyQNtC-|qDbc60q)$_q1@ajHfXrN zh+F2Hv;HLdKzy!{?%_NChM0#U-nbfdRSl#rC(_1nJ$B9?rT#Mzf zSiuz+1>p@&yJ7jh?%anahvoGjhgT(~6?vStXG@qs9uw6X}D7DU1hE)2gN z4?)N5RJ5L543CQna8P;yj^DKlUE_1{%BC#1?2rw=F6F{~ixNnlQi>TH%24A)5~?`} z=cLA1SXdYV1v671U$PnFCkouGWqvUAfZ(#&;{lEDR-$aL557WAT)H9}Ub;k}t!f0G z*pq;Y6GUj8E(Y0d5lE&uLG=+EIQiQSFNE7-%D^SK>2N-%?ajfblXFn-dMSzpFQvJ~ zWA4#lAycH=xW3X1aLY@<8U1N^>`NxnYYF&R$SC(tjfaUECGfH~7GjfP@jyy6DhYK* zA*Tr=axAdp?pR#t5`%Y*H$cq{UvzEv!q)RKI4F1yt{86zk?*abrEm#|G8MVo!6~9) z34Nk-hU2-mFYA~kQ`NX@bEb2V`n8gVdwpC~XbIIp&9hL$H2B*(Rlmu4pi32#xLUEoRj4RCbi%{CU3|%=6PH`oO_gsb1vneTUIvy z_Sb~Y5puXZXEd(=zJ#;x(}q`xqtR{87))0ii*CZYwL#byPdUdxZF?%TC0Ic03s+n_ zW-(rrY{20@UNGjnmbkW=n>L@W&|}9Y=3RtFlz=H1vly<$Emjp9zGP;X&wlBM|e?CSj~p22~BeF(L8anK|ii7>k@E%^G)4S;U;u zk(?e@Em={miWf9gp-;$-AK)kuE9ppb7oU|l{*mRr>;09qU3CN1nkBgJ=NeS)^}sc0 zUf}5Cj!O1x(0Zi}{%DLxn~r!$GmV9}_md&n$4K<5XMv<&@QHJZA1jnGK z25zc5&AB~oxhPlCiP_7dk7l3j4Juroju z>>Y<4f8t>5xHy=Smku+JOaRTXGck^sV`!x#HrnqN+_R++zPk*Loi7L1pF^>1vmV&o z)rQG8X2N-467aRhnHDvbGooD*H<8S@9tKx;aSxBb;jU$!=U!y1;Gd&o;ntNA5LK@W z-XbBJ2=|9A+3R7XTqYb@tB+e2kAtv98X!4127I!TnD?p+xFvDc+^+}alCK#vaQ!zu zcznVV#>(r#$*am{*pA_vlcFcZ|S&>mt!eEd`s| z24?e`RPOt=BJRqarCgS>9w>KAL$if5F^IRs?5}qu($;iQVa#-qXNMZoQIjv4XQ#z= z)bvWO_7`*OYV%uMb1$`=HyGUV(qSyOeItwA(N^%a)&^t`F9HF~$GH59=dSx^a7F!9 z+<)sg;n?}hq3`@!crAEdU6P7nyh90!?Ml)4b~WxPEkHCYf*Hj{uwJ+?^eoYYewmS2 zFiQm^-jBsUh9Ve|B)IM3r(!~oDZbFW%!CiS%azDorJ?_rJLaLldbr4QXrPg?L^g&u0pNj2{MNk!?KyHz&4|vd;Tlj zVpv?E#HznTG~whh5W^wnXUbo}i?Rq(^vB}d3r=8vV-Y-E>j4|8Z%ghB8H+3Ggc|d$ z8-6zPL$8NgVE)kn8kCzEPGCHC2(Y8BFaZNv~!+qBVm%|8sj$4A47H?i>R{Clos z-zwJho*A+6JSQ6bh&vo#$%)9B7VoI* zmP>YOTbg{T6g9~$m!wS3QOyVF(C&|M5I*dL0fW57SWHPS{hVnJD+YM`V2F zy2L83O5$cYgNr(81q;`2MqR}KY}zS&?j>o=v-KyK7iGeG>>UN2Cw##6pf@(~8*p0w zWE?d32=_pz+>jFyEpl!5TRbEF;}mX-xayt@qS@14i{_2=w~WYm&4o|f z0KcDmL;r?Akp3+e74O#Od|gwyV{3$5@sv5n9ybDV!3-{I?-la%H0UW5zEj~@rqL{1 zw1$NbpO&GF$Qk3D{qg8QGjN>cjggnvq0*vl$h@uNu7=1lM}6Lk>L;pmX&+9A{=Qwr z43;W0;e*o{`^Pnm;xPkB=qnxWS5=niOU^Css!}U+_0oCf!-`GJy5wx;;z1|$4w;W9 z&-kPLvH{#{?cvO>tc#)65 zW#nbXx7cSdX|dE7oV~syh(^s`^0P9WkbU#1>+G@gaOz^V_4XM$D)%J0GW!=>?DCVh zo~uefJM(OA=rB?x8%l3{%^^+0uh6?GmaNZWS28DKD=i8MAk(gvv-a{bG=0bj_E5|# zwjfiM%1l^6L}`ynj#nKqd=o@c^RkJUl#qu{-T6ms9lxdNC12eh$rqQsVPA*L<6Sn! z5-!eyO0P8W)vwD)i`ygd7UwR0lUXDk=zLV#H-8K5QkrJ9_0c_c38qL@bIi!;-xJvV z>GCwwVGuoQkwX=JCD9wEg8L#ihppyLlIV9A$&rD>i4Pc(8_gPY{o`sfuOytkp>Il6 z$}aJ08V9Mm<45UY{dT^0S|b5FF`?_~*fcmr7O4_4v#*kf^9g_J^lRSQU5oH>-RzNi zd6KaH8h`7%Ce`ukW=~oPJ@so>ldUml*_BuF*vczP^!kZDvN2>HnSP3GZA?lde|se~bn88GW&JcV zs34C_6gZseffD}K=m+$>jR7fQ!^A&g9+16JmDr+f%+{~fA#$b_?2e8ys%$MG1DhU^ zbhixp`~3`hU|=%8#@>ct)OogcY5{*c(VW!eEg`W<6>M0aCLP(bg7i&EmF~DdgxF7X zCX$+;(#_gW`KJ?h)1qPDsp-AnBx~z7vL<*AHF%IF^*z%`*v_5wo^B)k_x7#$lI2x; zB<(z#S60rSOh3!^EU}~g@g_98D~}WhtMC~EGsxDgQLX)5ihTL;Z1#jNW!LUBp?}u6 zlEqD@sH(c}8;(myd|VcL44tE|Cqe_hPSUH(oVt51}E z_HW={Y#G;jHQR_>QQpfp`2*>_A4II4j3IZNbjaojLtB%A57Ct;jJS_}gNTF8A-;Kv z4coht(u99$Y~ks6_Lr=v)$u3C5B#K$Uk@m@%EpVxDgS)heeD!=w%f_N*{hLug$Fd^ z&Kq{b{;71Ct~DLkCLw`!XIND}gD#l*Snwu1W(~AW$l~mHS~9Mc*u2<7bDmbyEOmKy z>U+3*rLXdkc^-&yR+Eni;bk`ULdK_R3(|?S#jvfRCIc+M#FVn zX#M$tSMNESzau`xI^9ho zHUb;U_obdVEnR{c-H~jp^*DpX*6P@iF|bXC!&P z?LX>NS4FLpGI*b+L9L%{C1l9XY1B_)B5k}jhsbosl0!aqRI*c;``sHy$Gu4wJI$19 zHSsX9I_qvRjWHB~{ky^!7}7Dd_)Z zy?7Vt#cu*T?@47BVG5+*?L52g$XBxLKq~9@NtG|`Tubm0VV6ejqJOUhk>%DPHoEQ7KHjNvsMQ8vU2-L?fO@VK z`s~eih%YGWvPSi)t;50y+p1~67O#pWI!lz<$^X>EEk9G}kM3|X_^t~*JaRXgrVv2> z?6PMKHzkv8%Nj6GV-_iyRYBF;d`YY}kaN}p>7~kBtogNSc4EB>)!QFSJ{pauwh>Ki z{+*rlky<qRRy3D`%DPeQR|@+4kgJdv&+4h;2hE z867^F&pf<>d@UWo?~-(}Z*Mx$s|wMiOum(6IumF`>rJWgrZ@D#stxq^oq_BKlfhPs z6StEp)`%#VY^H2cFiCnki2OWwlpglqi~Ex@`7Or<-hAC}_RWHNdVlbDvEB52^k;w{ zO{ps(%QVgC$Pr!`tJ5QOcR5Ov7GEM^-v+XZH!IlkRr$0(=qa73UQS!;*3r-JyUF$@ z3-V`449R=V^)aEoQRYqlMY!)d#2oKZPp19ZA=l2a}g8|5AB} z8PW;0HB>iZ6#3hfM(hh$(9t8p*an$JtbSrEai3ksyU2BuzxC1Ng{hiVZP5%;Cbx{% zU5uly5&yA&?>=En;;hMT_B6{a%AhXSyU4^Tdaccq`^lY{OH}u~~Z`cUz zKFb5rXZ2giZO3PP=ZiL0;o?}*ud$CzTQ1C2RxDu;+5RAtt=;If^aB2cb_{z%;8T_I z2gvEy3T(;LtNb;UP`=)9J!wdgC1<+p$&<^&sN;xH#47VVdp@Os=N>&^7jEfe=YCQq zYKDf?Q{eI_mkuNHm(*#kV?6oM(ZjBJa-P~w4IxbWaawrXf>i>-<1<&WwsWd^U7K;_%#>6b7_gsq`0nIq)L7H-ml1R^ z_ly5{T&eZi4KL~JoeA{UvLd>_<{#hJ_nuu5KEUew+wt^W!K zU!+LXHF@^Ns7lsk+F_y}%#VtoJ9@JDo0gyXA4U6ltD;NnfU>WwR@Vyh@Mn?K?3Iqyk*Gn^#aqtt zqjnw>tGy7`|J_#rnoQ}7Fg5nk!dcWuIh2l!w zdQ0;ZzxqTRDH`KPlQr$xIF&1`ny((~9GT9VrFoOB0f&f9y&pf~s}lQifIex5=X6(w zE?v1`Hr-a1LhVNh+%!W|vR80M>{ZbtebYYh&E9YMW6tMjwQ>$Ae<>pO7vHBzcVE!C zy1U4enJ2{?_g)ka>pv;*J+w&E%V@e)t%I*tR;Hsz53$;}OrFgi7D0ZzIm#zpRHf?z zdr8b#H$KF!NN{!)N$<~*p{k$N*=&d>^GFf@b$S{V&rYExR^waUyG}#mwMrWB=L;`> zG?+dSW~%Kow~=ve!f)|v68TrCPo0+jAWN4YBrh)d^N}B}OS9CIY00&N>^y^ww7$HP zb{mwj-8v;?+AiexsFcy8#YNIgyG$yL-b1%pR`MfOYw=Q}L-hSIUGnI$A35`HD}|+@ zH0^@|y;!!9B!tVc(@RWRPY)YLi=OWjW`C2&$Xq#6WRXW^mRgWs;XdTIizVGI*Dqb@ zQ9?)d2JzFvj*u(ow~_gSAMwZxpaEC*@s=Evr`fD!oC^whTl9gN6WZ1S6+|kmY49o>`{7B{Wy;oguj>R;TPY%C^dg>VHL0s z==QhXe!LRRoLe)FNS*4LxWOCoI)~9o#$T`&$G`H>tKjU^g8OCOiM@0?dTA4C- zewi!Xv%QetY9>SPeJGdukJ`kx`MVQmnIyqg*hoi4ZliPb_e#ga9HQsytHm|vw5<{Y zjo4T68HAlr**n@wG>AJXop7aLo~($$?_tO)bSc!NT5Xz4nk9!uimf30FCADBXq)TPoH2WkDikKEFymE_!kW31QHH1^c1wS4!w zKl~uM@4U*T(`^6xJ;V|u^u*Rq={7TWmenw%dTz>ACO&2CXt{61c2E+3X!B+ID@vnP z^i`Ix^DiI|2d$)CZ|?AVqqVGp2CpJ5eFJIG%zN4Kzy2Xwy@4rG%?l-3z z>sFA)y)ssVKj-n*2g}Hc_m)&;s)Tm^y&zua|BkQ)ZQ?J=PwCh&OqyeEy>E z8~V9>DqSov+MCBGvAg6RvWwS7(rU$V|5J1x{#<>39Jht+y|VXcn4fb#_t>Q(MT*jp zww9u;%*bp=M5)juE#q^}=N^134M|GcDikG@3Q76h-#_5N&b3s+_Y`X9`e8~3?=x{){}`_WbI_Ys%=$kpqp#jK@bjw` z&`psdRwB~cEk1ZBcvc}!|8g@bFFg-={yC0*foy7xN)agKlb)5!*dawe{a3n#@lB0K z$1=kBk7p$#Lw!Xwu-uU~`2lER+(LHrgC`pOoq#O*fAK{dk1;;)ZP8d?6>5#Uf@ZAF zXO$~b*m*NgGE>V;kVyPhxJF7!I7#@3lI169rFa?7ug#V`o~H*pKYqn`BOLI%>S2(5 z)Eho>NW}}k{sA9U`tUTfAgEux7YxJ&!JA5!#O!Yya2q%aZkZVp-?Y!Tyj2zsN~@67 zqDef7Z#qQd%mCP3r9zHX#o&)45Am&%KR|Aa3>m-XP4-A#61=vTBilduk+cX2cwOxw z_-_q_GUtp)e7XT_^3@?XpA3Nh@f?B7`%UEifi$2sI*0sc@f4(Uce(GKCc&|t|L~%p zVdR9d4YX8Kha=0a;KdIam`g&C=xQ-C__vx@GoZ`cX>9=>95f=CZr^#uE9K#i36{jl zqzhbKAWAgd?}CgGSF*~o5d?mdB%4moBL}p?;1?TTnAN)&27kT-f~s8L)=+(zRP#Zg>JzHmJes^1&MEp%Gx&YYZ=>VK6VR4tH=sjnO+TtZO>X`?K-^;O`bCrw!HM z(l6pL@QFULU2X{qEWU%3W#aJCL?7s|!WI7izB**f1hUmj3{Us-AlV@^2xX^&SQkB4 zuWVl;f@Oj3m2_;}cn^O|{EKB~=#jUrhr#p1pD|xV8hSNDxY5)YHupIbr44+%tY9UP zu+PG}SH`fHl@IxJ3z6-KGX)PWGsM&TDfoL-7VZl^4HQ>b;JH`cIiJ{y0ls}nBCeNA7WoBH+l6U`rhGnzB@lGFaXK^)HbT%GWSYU8-z=Q;xHiC*> z#;|bhGO{%1FwXv(3EaYTiKmz}jM7?wb)$?(PxN&`!;k}<*C+;)o6PWc|4HQEJ2A4c z-2ra*=oOq(T|*8OYT~QTKE!r`Ixwug&XWx40GG1=;C)CDitV+9bYv3Qrn?!$mAb?D z0&9G5f;$mJaG8qIv%qb+DEVqE3N6BJgKIwTaNmHaU{=&Dd^lwf*y@3B_- zV-OYa2&)vQ;>4CN@ZQr9ioGEqO5c? zloCq<8=d}uoRKDc@Ie~ZStbXwJqrW|H&#Fo*WbX){3vhtVn#4lB~2EWT7icTV|j;{ zhY;^Y{k(*8OYr^|X1t)0Q{Z~pWw7k;b^Khi0z|YN!sB6LP=Dz@f#~Y%*t73G{v6fp zdj8*ioELwy#-nN&?Ade)uQ5L-2tU&Ru0L0Wo?$~^L^Xt$vqK#uOB@1>vjGn19|B)m zas)$F49VXZLB_+q$PO6+97tfJ48F*u z0-Jr>@J%|8oHJWUQq%w9=0YQ)^}ztnEY&A>uI1tP+fRefT<(tFZ4ZNY{17~g%EKkg z?Qlb>HWZ{A5fi1A@UKq-7+L%QZyZX&XIuZ^mkoAgo-DnwQhAVBefK7)laeO064oV2{`QBC7Z-NGi9Fu^-?Q-x*mj(F! z;wg4$Q-n%S=7FYj^5nTs75Fw=7P51%;2#pp;o;J99R1`SK6}&nj;l%)tjbddd%|x45Gw&nljqL0^cWm5EPYf=k+TI0dqtf7MDH8c3ves zt2aL2L0lr9vO))huGt9s>U+S5nh?y?cj1g@&%u;b9bj=F8{7Q!g;xtriEYqC5_#e( z*uH5lkj^n7akgvlvK2%4cl->{rkITnp8AGYeNZG@pNhdqvkKh&W{~&xv>3i(path= z$rA}TU$8*88~B7?=1JU}f_bJ!@Ss2*4i&q>UQt-%Rp||tf~E11D0lMiOdz(4%fdN0 z6VIA1N!r(_3x22G#eb*=zSa1tCMmtbm2WwTEWj${{_j)RMOTbya9Iy;_jLTU&mBuF zoliuDxNJp7KK_`x0siaNA#48AAyco4lEteLJmh`@FZ@pd$_o+fzM@}q=#duTg|>lH zDbs8E<$JOHaUn1}a1XQwD8ah%er)A36)e)#Bf&Ev0sDN&rbVJe#4Z^mKH>(u6cM7> zKgzrF`3BH9zZECXOBFm%euJN1jKW}TAMOBFP>4N%{qZSKYpXs{*F>=C$5eP{e-Uu9 z`yx;Z8V3n6H?hH=S6~H~15v$V?+Rrm0IwrE@P(_LxI#4=e3rTfW~)h&+wWE2pV%|t z?d-j{s3;h&dT9yYuO0ve2~)}YO_#v69%VSybvF6Ws~+spZWZ9yXD~@}fXn~B!#(a3 zz*qe%Ku*5{YYe>wLH@0{GtrEMCvv&@W;qfjE(?{_6$$)ulXpkcg?Ow=#5dCH$(F13 za9{Xk{A)-VR7*@EjU8I}S3^2>m}3BsjAenJ=F?zB9Z~SghCOVm? zWVQip`KbY`MvUOc{d{M6-GbK1YsQl;6B}jcWhz<`1Z+) z_q^v;^_-j1WRlfIeB;CjUcGn?h?BkrdNQ;jd?rt_)^Q1#_*a7C=fC5g3Q3|AI1BrT zbIzC1iEu{eL%e%#A8@#-1;{NeSYg{#(=kyTj$gWthxK^mY=Jz4o(gce z-2fSHoN;7MGd?$AI!tQ!#+$!KVq2Rop4=62^8MQfJnAL`4X1wqIVlSOdSD85(sRJi z5=l7yRk5q8+Yqppkb^w$^T5(-7QA=(4}MzdU2}A42#&h?1)I<41S*G%!F)po%2n9H zR(~sE{;Ccn%;e^jT_!~BW?_wq!*?8b<(D9Y%h~+f{vA)bYX!oSO`ydda}u~;2~J5~ zi}SS~;`#_tXx;Ts(CA-*yCtbK;BGQ z!qr*;Yi25mm(0gjTqb;6qXo~NFeW&A^%qb%ZV1v2XMstK3ei{p1SYWBfM2GDZ`YW> z*BKgw-}M9(&dA~^K9nJz$0cg+PdpF!`r^=Nd=Bign*_<-#+q9}zrjj*X_&PnrzW=N z4{$Pfg<{j?IkxU8?vi+cO}>qQ3-9W|<#;jT1OTo%6hOXqUjPRxEOFzRWkhD46KwmL zhP9Wf!s;gm&}mf>D404C{x-WMDOiab|=y{iP^nK$IMA2fjG_4Z^}a|bB?a)qZ@ z%sHDbOd{@Mmx10Q$m?%Dj0b%UNo+U<#>wN@@q{P#Nay?wYm&jo4{C%13`p`~S)$UN zgsZP#2mBo(a85!YSatn3-s=~Mlcua7fqt`L?7u8rA!>-vtTZR;`>q3(7e{fGkq1$* zY{P#hzOLRGaRnE;N`ab|b0E#60WaqqKVCl*u>ac;urquVXEz&?0|_g@XiEzy{Cg1J znkWk6H|+!h;T)`9B1H;%4B*=6F9PElH}X|c23ROq!P~mW!G?o+IIGYV@Apv`+)q#d z38ht_VY@vp*HwoRUln;FpIF@TCW#n7SV)#gsRApF&gzl94Cj#$@;2|}vTRq(aP=KW zBKj&ta9raAzUiZk2lmSog8_AtS||>3=bx)Co_QGmIW{EN<30?;qBP(f%V3!K(-L;> z@ggMIQqXVOf-iiOCZnein6yj;zBu^^^E_S(78;0=d`A&t=a-KC-BJZ9(OS6kZ4*A< zWxzAq=tA1s48hvz+Q7v}wPsi8WD=&{0p#|oK(MP4yq)a8ogdf1y{7r_Y>yQPo!|j? z&VK_mkF^OhB9$ProegLGU4{Rdzr-3cxAE%osbtRqchcMPSfG4c9Nyim1JjbFh{&pm zVBswjGW1LzPGfiRL=0VdyYF+e^-p>DxLQ>p_b3?8Do@5Y`6ocanS($vb|*NfEDQf$ zH-ZahB;iHRkAO@CU-ELB9l22U7CYD|k;@61Q1##$d`FFf^65cu&enItZH}wD`ag06~~*g zHW~oom*0T>-{={U{$?1 z{OzhwhR<#%M~@EU;?=vr`zTfN*6}k=a6W{2pKWYPq}#p2gm3^UoK-^wQ3@%J|%`%_fXLOlm}m5 zJ%Q7{4S`4370DfE9;xl;-Zyc+aDq(~c=g_jL^lS3RL=+CPU1vJpNW%TKUFBmmWO!U z09F^P!O-R( zUSOm-ncRQl00)yyp{?~=upsdh?}BwTPR!=MkyP6O3@QU-8yf)S?gTzb{=grN+rS-J zWjz1yR^Ti4L{Kix<;F8T0prIbs)}B`EnUr6%Jn?x{-Z+HM~`Buv`8nRc5TLXIy+(Ee*L(8)QWtiI=Y#%#k*bWiF!-52tcuHavwP)W45 zhP}bGbxq*7Eo0UzPK0}{N5dfknn)$OZjVTm%kJMI-*q2!2udWJAd4R15cYGaWf*P-(8ay;XCJC+GI zILY`}mN0V+?a&6zMC32k%$Dw5KwakwSS_18{u{0@RO)af(oVfjxi>XiwI+(LeR_)a zFX#HC9(OVtgKg;FIX$;JrEEHo|DBoNWR8+%6d>EAK)Qyzje2b&$MgoTMYe%OblWXy z=C+jsGTI7xR$rT&7U_ zZCx3wrtJX>)5P3tbfs~r(FJzEqo0u+--HbPotVMJlbN0SKG3v}2(9z-LXq2VaereK zE!njL!H6AnM%q3$DRnwpwmugr4gFx_PhDn@<$KXYD;Ij*e<9!Hcpd-A@zr$Fx+64O zUJvH296%wDm4uVqlKEA4N>S*WB5prX#rhddXOA6Z^BMVc$V|B7G*) zF{c=nO|qmmy-sx7dvU(V2?2F@V2`xEJZ6@cKc!MJ3(%GD80OXDTWsk5y=b^U2TfxR zurCX53d=IzAc7gR`A!Z!_72en9%@u$wI(|^Y9A8tMc4^<%4mb%Co<=aByImPApH6< zMc7)K#_kENX1AB=qN#2@Y)$<;;XSFt?9lJC6mj3~Ru7EaZgG3zLat*~Ps+fp$RLs* zI&nYR9k`C#W+*VFt9|(Yy%DFEpIg&A>#XQU6$^B#xCd=7Zf6d+@KJ)X8>M0=nFn6K z8LKN{X!esZ>XWdHJ#*(Ws#yM#W~{$Vy^m*7o%#c8Qs-fGYIGG!aS||+M(=3L`8?Li z)q!rl`3YsFWTOZ3+u0ZAPN0{KX2Kta2PpB_#7bqW{=A+_tb@y_*Ns4pM23jNE9v(9w>BenzgzI*$(5bW+{6>lA$|U;u>8bX(k9RId`p zSGpiz_(ra@Is7<1IAtb!)ozZij?YEEl3%fkQ*&vws2F`zKAD-*s*3Ka$kJI^3Cxkk zHe@oV6e*VMWT*9vvLQVw{2-+Y`buAmStW6f(U-c4-WzY3|+@FrMu$Z zFrLn}%#WC@$eQcd`#}n5+m2l7SYD3GZwhFPOfCIzb4)n#bQE$3v}3YUFjKgGh#3@! z(TCMlNJ;V=bqn-GSBoQwnB)QeiMhMbFA7lXh21niS4sHz?hp27Z2`q@CA9B|6}`EI zhc0`)WY%SGLwlnVn9tpX$e8P1-<7>w$jkFVvtNtT+iS~Evttdv8LfwfcK#gyD9xWS zSBb6~GeDjmmstPe7wG%WYLw&BL&y5xvuWyb$Y~@Eed+E5=cRnnQPxjbUcZKptG+?E zvK)AywhaI+UVx^o?81qXh zG|onYddfS~8{%zrQH2C5yKK%5n{fM7*E9Uk7do`jd^0_AOO|G;d}8aSN}^|7ci6(| z$52-NHq@TqNv|b4GDg)SH1(4xEejr{Hiug%D9}N?t2uo4g?&t1(+zqx!i@Q(Encg= zH;hBCkX~w<#{O#k+ zly1ESs~nIbDoZ=i@UW*$yw{ie4hY-2z1fb=HE2uBJ$A}bgv_`O`oUe{G{S#3`?^Ea?Zfw5 zw94NTooxxC*3(34=Wi54?siSA9vDEcdIYSGt3A~!KgM=v`?H(%&ZAeSn}u28E7|0w z`t)^hC$sil9b3Ma>x6%3L{*Km5LkDX`S;&AviO$8)LjjtrYmZMck@HoqRId?K7IxX z?j;bdsVRKZCu^w2f_2o}Ck7?5{`8333i{^lNm{L*Nfn%rvL)@m_@BWu+H}9BFxK`YMGo~O%gEVG;ST8#?J5G2^TMxxd&|)`my-FUNo}rd&hHT%=SoF-&o7VQM zVm+UU($u2@Hl^LJ*5x0UhWsao%zu1lBD2)pWR7k|BOPVvQHcPblw|wAZc$vMZU@ zx>U{1AI(Hz+99+M5UOrQNvxp|cMe-pfSB*fCBB;+NA^#@WbZ?L*o)$Bi!d?8`1I zsYGi!o7jB8B{Xkw5xKgn5QYAMmcQKY_-4A0U^+A9J&lH_cFdN>#`zGv7R$wyZTm>NiiJr4`j|>1SCq9-oi=l#ik>oD92o zkrxwXsKDw*9YxP$MQf4fEhaVi3HGd2qG3v&c--e2P41SiRge8hVix~qj5pk41MJe# zDG_g)-g^eQTemRtrnI42>k8B{3)9@8Ka4}F89kV{f*C*mhn^Z`(a|=2Hc(uG9=~Re zj-1(qa-?$5?7bpxKJ|XWJ!%>V)UHN4a=(z=n>@5=LIr&;Q^JbASVFb#Mle4NF0z#m zBan_v0@btNqvzqvk%ZfQrqg{1%J7Y#f}gJ#p@|xDTrnWr*P==V1#elybL!~M(RK#k zThAh6*=EG~=IEBn$t)YR2YowULXS+drzg^m(!%d~ z=#p45HLci3b!uM|wP77pG)512VyJSf_)iQ(+B-9v7`X8%r@eSqm zZ=%H)$JrC(XBnNtm-t|dvfC$)MP8Pk!p4~Q5zXWwqL2&dV8UZY#_b=wc+NYHgHfi1 zrBBiG7dgl!o25STf01Q%o=_y~C29p}!fQ42XwZt+$fm=H?KsxYINzSb=3dHVeoj_H z?+kCUCN0%Sx!<0yU;?Ru@?qrVbP|2KF~Af#ePIqbd}RM*1=jwz+N_!oZ=#gO~BriC|q^Kv-wLvXt?!W@{)g+XwKBGu^ z6`~slN+^HDJ$lG4i{50ysrbw$HgAs;y&+MKF87Eb*PedbBT`4TxQ-Fe=}L^`-~cU) zJB}Vqcc&uV32gXNUv_?-6*Y96!K~>kp_=`-_^HlX=%;1_N;i_AvmfT5zHjMh(8)|X{7XpwJz#4IKvb~2cVqBoy@UkU96u=F+15yl3f*k0R=xOMrk8=__^-Y zXy$k^QoOYk6^BUHo}9k~9qw2_`|IBiCL`$_N3DjqzHjSh^nGbC!{g=-Uv?qo z)oHV4hb3w~pH*`#R}y;#7PFf(DEoqA+AbX*CQkJaSjBmxj9J`GmVezDUD01k1EOST z?6QyS_UK+3Fy}WFX*WVa{grfWq$l&1W6o`rN@>-%>vZOQLU$f4Va9Tj`G}h{f7UtB z!7z8aB_oNwI{XPOXC>Vx*p{JT&ztDe+Q&?Jvo|8u-K_JIkBp+JtXtKCWoV$Xj4DZ% zAjtJW@LZmtJ#ydJ6!{VO<=0D^b5N1$F5N~uzy+qkUYlvl>}J^OCG4l0CuvSh3xAd{ z5$#i(hl1CnvK}s&4PZ@au4@Uiw{8vl!DKg`!MtErD!gGE^f3SM_WhuG>jZXJU?NgF z=FC1Lrqp!pQra%fm4>jV>4J-$tZ-CFjRt=4x z6pzfad5m)K2%Z0m$IQyCV^2?&roNm9@oTa)FaM1anCzDh+{34V$NtyxrHfgDbwV+C z6_`M`!`tw!o&#XVs%pGzT$gO{IUxXEVkA7h5^U=9Cp|xw3l=74xOGl~ zx>+NzF-4q29I!lzS_z}sQIA=0sS14j#+AP50>ZBl3ngrEjeB+AKEEdPP+U#tX0hf6_7Q6TYaHiVMQJkqKj2z|cX;JKM4VwaS) zoR`uWrggc)N}JQzH%y7Qw{;vC*8A231>MILyDiDP4O&EWt~Xiqf!iDBIDxFKVq{(l zj}(YD<4Ht^mr(W;)Zg}j_hokQq~t5{28BP^u&D_*M)m=LT04k6V?nGh_>!no5x}kN zES}mV3U>x><*DkHgZV8Sf7z%6bL^3&r`sf@!Y>U(+4{kE~ei8ra8^jTppR=IaxanDbym_Zm{5 z*az%2)ySLUJ8Ck_li_K^Qcyv39t`+>8+$~_kq-VRoPJ;hp4RrQ z=HSXGTr=!Rq_3O+`^|?07nj@?NNqx3^2QmYne*YvzMKs`Kbet5ClpD>sVqG7h~m%h zJV_-GCoXs9L84QE&63-&*PW%RL>cD@3g?(A>Z+Ycs|J=FMQoc=C!714Suh_Zcx(?5IqJky7w?Spqo7acX(1 zg&=&(G1nq82cF!#14?cT10Vbv@igoEJPpes&|fAAA6amY6Cej-L^Ppe%OsLht%X}e z)k(+{DI)734^5>3Nf+$KQqE@tSCbHsJf}qh2bbaG(2dY}Ru&LSQ@m-|2Ckbug5z!y zkaI&5c7DM?Vzdr_p6o`7s~6%ow@z}pLxy|~S0R-vc<_tA09>CLMh32j!Qa!f@ujz- zP)7nlmwgjSO4?rFuY3|KXmh;Ejt}^w{sBB&`x)4M%>-_n*Nul=|Kc2_jj(sqJ#co+ z53Vo};#V`QptPeg`1`j6jJmJHAJ^Z+i3ukJGmLKv=5y{&QQs=u@XdkjY^n!V3fzu8 zxd!ZhKO5w`R$`a(3Ls*7Tj1KPM~*)4z;dcz!NuoiFv~i^UPB0PBmmcr^afDBZ48K* zxe_I#B#>*L41zb`u92A~12fDt@rQ{Afm6bOAZ&UNsrobvj%Mq@t7Ex%?jHufFfGBQ zJq=hQ(Z;p>)H)z*r$FAEHYW$9B*~Qn#k|~y`8>s}7CialC9pnAia2{u2l`T?`ZDWeI(Alwm;5>zdOaI5yWwk{EFOWvF92 zI8_=BzJ(R@a$NW}n-=Y+s)B2W4%y-9>~YfaiZ0Z5sy2@R8tuw@;O zJbINx)~=li{;S9bd9SO%*0s*??xqkZ6Fv{;_w;}QMQ*3o<3|i$O@tP6wqc#8E5P`T zUBoDDF{~UI$2PC^h<}VFG4>zBxh|`T_Sr(98K6eSiv|R%{~wRpbz$32U&>)(4s;hQRX?fw*_pD(9z7!#MAzJA4|U z4fkEI;7PYB;1Q>AV6rKSxGXx2^N9$Vd3`o96`cep4}<};_)usV@Y&^1qXis@eZdRv zcnYqjTfm_8GfCeTJGgI}F;2EoCu1%EhKpSW-}XbueBTS-8x7-Y##S{>4e0`pyH4b1=Pw+z z#gA-XwvZ^;d2sW>HekH`3ocfbB85CRsM&7~IVJ(lNv-EaNKb%CyL&+0g%v<2@(H+i zDnp>XWfI)nZv&Sl?tpg-?tl}#CzTh*qpRapIk%*VEbI&Cyj97XWdBb?(z;8G9P8JI zRQ^q&UnP2|ZAeI+to@NE9tM-nDPpffN5S8y zjRHA%1C*fWIOxDTUeQbmXlURMpL-aPE4S2O+y{Hu`qPZ$DoT-%Hw8E_zg-|L_Y`mV zY)V`dci=t2W-vJ-=4qJ)jA{)UqYJ*8B&U!JRl-Fp>OPW=`6|Qovh#CqaY+ zpVa-j0vav`k${d@pw@Uh%#ih2 z7fY;Ec^ZgF{}$v;SPb6{HDQ^l0Z^$$ne27;BWH>?kdUDnFiwUCS|3lxm$G$WQ+A7Zv-0*ssx#5rQnUTj=bR|0pNtpdYp9SCwLU?2VRo1f@i^! z8eR!Ntn2~sHE;*_9A#kgtS~|55e4FBH-Ynn3vlzV4PdY6Ecm`51az&Q0?*_r zz(s4h=Y2JI?tAUV?Hk+iy?P^3X=g-IMPFgf)INcuwk|pC@B;67(%^c?;1*a}Z4d9S z4Fh-`g~Z_@EUuR!zZL+v9ojgiGN4fDpn4RNxf@Gf8a4SAj@p zG}JPjM_|n(5I@=u-o4g=?)QITqoZ;V_XWTQn%_V{xHb8;`!Z%lro)o=GqC81^`PeA zL9BJ8L!h*DrJ!1G3b}iZ4<~Qf4(BcZfQyeh!q669Y(z0I{iY7R@*m(DKYeK5avnTV zH78#!C*$H`N%HZf2wZ=&5~!3Ifn#OsVTi2^Y&_N|@GTAl)ospXv(^w^JO+sD-$1-( zk`0+U=!-Ll#GvW?iDVboCH3uO95;jA$En}*YEE63BiWNR@hQ(4L`*9X-~D0-WXv`S z?hPrzBe#v4N)Wi@sEIv8$AM_m>{5N?gti0bF6%0FnnuhMGEtufaCYCfHz9=#8_h~ zOb+<~TJE04_J^d&TmyGfdQS?fpa03T4O1X*{o{a>`cdG1pbYEDX_NlRv!Iq-6MoV% z0s5)`0x!7dq^zI|tVmKNC5ygrXJsbu#1a;ieIBft^imb9U%iEBd;SCcJx_5^csEu| zE3EpTY=P}^_eHMsO3nmTwV}RRi3AlYU0M_9=pr`Q}EZ=(&7vGg8WpZbz_5 zjQ9okuJ}V>s}G@u`Zv7xcpFAk6lSMH13xQua`IpvPqCmLTrg?E5oxMqj@c=JV%QF_ z_~t5daDygXn%xNA`0IdG(kl~M~Z+3X|)^% z3Ih&;nsugxdtC8qmG?Y{4K3KUI}+6Q0rH60z`32z1WqD7*lwCX{BPR}>}p^_GV4{y zEA!1{vBx$rTr3LMa|UGIni?KMpMoc9g?RdNY4X6~tn1?KRbWJR30OVQ3GV(>;q zUl^WX56>-`P6TS?DDYr^6-F#mLF)%Lz>u;;h9nK))uEGl5xw)6Nzo-7pRaR&@y8 zOljaPoy35l{7Ix-8iA?5{mI465#WeX8c4AFh>NvcfK$PLVDia1WJ;z2xpBgrWG5lm zlMzq)ZKTMn#oGnC=CY(Cyb>2$SdyO-isT#TR@P18oS65&gn`)e7*^Se5c$Qmbs%sPxy%HG$6i!OqP4%gwy zcG6_zyAAwOFyyj+I2AlR>qOdvEy>u*)$oHO%S$32@a!m~{q$qln z_ebX>F7KRQiZ`E1@E|u*8B@M&)>f}G%S>Or(o512;zh%V6GKgH?;RLmZ*MfI#(fEhoW|)|2 zPyBDVL!ITtf{cHnB;#8F*tlgnEcV|4Q-806#@yYI%4chEBVr@WdXk5C1W$y|eb0kS zDaz!ByFJ;qe+Mw#uL%#&m4Qp_r$Q}}n>fR~9slc$!l{GyaQJ*9_K#W&;lUjsOk)c4 zbp3)o#zuM3(b{DBqcNZ&yAhabdBQToow&4pCD?n^6;>W92Um4Z@uI$-fak4CP(pqh zqkNzXMHdrVj=a$+lNGGS+$2^mX(Kv${{!0Mr9>5DchD%M`84U`*qC==_xl?iad z%vZUy%qG2DI(N1LBRaGI`KjrktAiD+&RGrA*7gG}hz~_Nr&LhuCKr@3p_Tn<)J@;E zu0?lkqUp~VXWG5bi?(dO!bY_uu&eeNA@gMy*?{~F{J_{KWc^|S`^f7mJ&4XR&fQ$j z8>FDv3RSnK{gHIWvT=T>!cjEpk;phb-i<1joI{yEx^QyeVg7?JF-(8qB&uTdk%%0X zppS+FQPZq9bj>|+DpXIS!}-dz*v^b8ciT?g)+*Q9CS@^FGJQg_Ooxdz4`j>#9c5-b zWKpW21U=k>Q0j~d>bdiW5PhA+cFq`Jg)1PkD89gk@UJivuM8lOMlm#&{utK$JSTi9 z)x%CVKEwX%yNrUPw5a(#Q~Kvx6uVj`1NEoTp7rJUE z%Re^vIxC&qOHF&5nQKe!*bU18s`5r^Vc~QUQE;cDdiGI(JMiGydL? zR;Ej!g5hdLG4+V>-YB0=TuTrHRmjwy<4WW#nM-*PRUO@p7V55{E7!KtBjyv?Al)AJ zSi%}KO~j4{_H99%j25x;pCzL9tB$lFM~&v+sAV5El(NAiQ>oSkQCfI^4>I65A>PRi z$h^3W9~x;ziogD2b$_j*4mYH5iq>y-f9GpGlfy=GN9TkIF>U^nFjyuXP+caW2n+cG}d&F|8MU- z)-=VFuP<|((HK>zm2r|{lI)|Iv+Yqx$|8?$J$8nk_?t`TC!R!*-eaZLpP)Wp^Z8BA z1?c0aN=Cio4#g7`Y9IJDpqfMv8Wvm4v?2{gZL0;lT1yS72zR+v8a!j4Uc5-{P$HB2 z;TIcZb(+<_3~G0Ki?ZGx-gKjfHMMxz!~TmDP}V1enG^O4op~quKZ?%8ucqgX8aY_?_Q> za9{V#ne)s%@8|tNR}&A@ClyujM`Es^8TB31OE(Nn@|P0j z+$|BV3m4J)!|IIUYtGTBT#bqkO+-H`GElR0EQ=S(@b{hm$X1xT()x^ZDAzU>xlg); z1nf5EVUH)9FtC^MFSwu$LOV2BYaX3gXT*%ehoWki8LS=0aFzIo(5t#^^lG1gUsUeF zY`ZHZnp^Z9@r;`3#E~gzUa1sSX`IZSKblBSS-wS|)|{i0J`|zgKRNVEP!kgdTiF$^ zom78vE!EDwN%wZBpnnyEbiXG+>-%0)rT;9^l4ddVG43c$V>OZa=5!Qmxqwl=H-|pB z6Gugwe`#q;5qeh>jVwH4>Aj2X%(p);gzm}zu~Viz=1*@HV}AQOGY&_NF&CoBQLcLg zP3}8}q(^)C1v&}T>uxCfw<1CKnUO$;N@LIkz4t8Eji(DvmZ48os;EnMh!IvVLZO#s zY3ihzC@b1aX#cRD4ViU|8a;fBQsrMmosIT<$K(_$*1v?_f2@K$p8iIWB}rlE06m#$!E{r&WnOb(&Gc z$M4JyJ$X?PkwRa#uBBoc9k8?9mgYxzi~6!0Sb<_CbFeTJ-Ticx+5sb!?7xEMZ>Sck z+4-=e2h$lP`;+L^hN(0xy%0rLC!Of!s{^R8pqF|T zoM%*5)bZPXy`#JZzo^ABITW~R5KV5nPfJdGWa~1OY5b=f)IKr++0T5*>{b#86K4KJ zn=Kd9RgF1}n#p&zo`wqB8mFRHi#ITvK0QGm?-$YQW@@OtPYdn2a~+*CtVAvHQdInR z0;iAHVh$^>p{~m|@uQRG(3l5%+3_0Jn1R~&`~}dT`lT6)dS>?se|^|Wlb06=>#lR( zm4tjYjF-TynY^EV5xqq(gNjg%tg%aSlL4cD&W%cec2?`eL==(5#o8_WN0qZ=@qgLg zY;((J)@G)dD7F0wmhgh;Fwc%w6navViv@HG(}l*CN-~QEDOyrukFJp!Xw5TAw5|3% zyUIa9)X&?(eyn_jUY#wVJ3Accj&qhs?M9(cDK4MC12j_h-ZDDWTMM&&HQ80~qG-s~ zGBmQrf$O>Nr4kq1sG(i~iocyiSK3~s`GqE&myvU-6}zxEH!0DRJb?0>RneQfigf8x z?$f`Zj4pItgD$M-V-mm0vegR@Fu@^9C``Af>+>YhE9iqP7mAR`E<33IxOQJ zvJ8DFp2%4L>|^Yv*P!=SYuL>GERfUbaiojd>pAM9&vg&{5tf zd;MN3-G4}l@}`K`iWwYTW3Eo`9+4MmPitTlUSaAllgfO#AfhXoQuO|!vdagLm;Azs z(@{a1flGPU37WF4n0odZp@@cX`u2W5-TH@R!z&Wef2JwyFR>U#WZTW0a$G}IY}ANQ zHi3zVNcdw{R|f-&fBb6MKs8UutI^ z*KnxdS#vZwoC+mf|DfrGIY_#32YsYa&R#uTf|f^Jq8pkQQK>Oy+Pu^UtzD-^ZJ#8d zjozJ%PL4ENxJFGhmVKH{ykJ1(1)JEZf>8d($cr?4iI_{()pD#Iyqr$!IY>9?{9q&R zU1X#kTWRK#Qby>wlzIJ5lj8vNT~f4aP*jf%QzaI~eo$GBK5ooGXS_4%(A@zxYG5)u zJg$lET2{wbN}NhnE_JfX(YNWav%JV{{YO@(-3yt^MbfUnr_i*AHOOgdI5q0qOk-B< zMcO7RNYqN15p9e+-LH&fw-Rx4C5Xs0hH9iWq^uV&gm8KHHub!Zk^$`mY(75=oZ zqv*RV3K@TrD!5N%EuW>LZtF(sXLJ|MHkilc%Kl(~ep^Jp%&kQ#z5AG{UsKV}WqRC6 z-x&q<&892$4G{1b(vdTV(ZtwfdPC_b4fOpe98sUeJQM9^BfrTqH@Z@gO8A&C{m&n? z@U{z+=vj|uJhi7X@G-J3NJ0NR0%={>Wj1!Ol9BGe$x6PSh_sK4Qm4{Xw&Troq;4e1 zSkIbF)t+~ut;ZK|JXSJWBHhg|#Y3ELEs1ptTSiy6>|@3IhuHM*aqPdRf7x*TE|wm+ zhx$F$*$tl;p!0(fQ@j1UX@#JZ9GBKbtkoSfU8{xp=K7m2G4CV|2--`r_E&W9^KrU- zdj`7n@B+=7_JgUA?ql8eWFvpoS@f#8tjM%R1`%&J>a)I?7H&|chl^9ttSKxKX_eB< z)GPF4|8J)LT??u(NT*(`Clh-+kqNhQqywr4(Ir1iRKd-Vgql8QulF1jI%XWEt_o6g zo9hAOyX8F9Q;|WD+CS*N&qd5_XMoyF$LJ@e9ELgXh?ZPp_#M+k=yuOhD%{Q02PZXM z{4}@IVU0|ta{OW1brRC1Z}?xOwo@B)fr;x#K_fe^BgJxev`h9r`=wza<2#Uv zjy7b|d_OgGL#L3QFqn?4j~$@F@gJCDztT~0&L70fpC}5R@R;4BJRo$4zKN`cbNQ}$ zSLqtSd1OOpGY?MIlP7U6p~uxe<_iB7y)i43m7VHJ>va6l?!pH4jMjhjWX^w7OK^&v z{P!U>>ggxxCX9TS3tdq#az=Cd=YR+Cxl7(Id-x+bKt@L4}Jxv&qqF~Jf zv`MuTjm$Gf%#Uie_3Co;?yDLz-D)N~sY{L~+nz#T^HOw~Txa$kNMzu2TeN!DM*cth za2oL1f|2b3ls4Fk81@tmce;n1K6TN7n5!IX z9gCI(B(YbwZU!Y6_R+hshIH>ee>D5Bi0R%iiSd-kL!*`>=*!|pblzCN8c#opv~)aJ zHEqn_Q1uk$H325PK7mQPdLGH8Y-D3pjAHMHmTOFy zCtD8kH~*=m!4;Nt?)4^mHf|Xc$2US1r#U}%(Ju7e{uXN6Z%s2uI?IM=iZZ|U(}hQF zP~j&rWLDVXQ`SdNr-LDJxUT~)OU8E86v8NiVw^Hww zcI+doqukKKasE1w5biv!#m?P>`Inj_ICgvv@;G>x-PiDd2DCY|o%6a8xj73xm8)QP zt>vTcgdjAqdMAHub`m>rELzwi9!vY1N~wtRyowzl$kN}M!XOd4leY^gdE_%wjaSli z#TzJXDtEcC#ux>AikQB3U0U+Eg}JaR2R*2mBue~;-{()=I^k_Z}yICu2vNb}Ru7uKh$4IKaDx6vU@gB0ecOM1J ztU~R|>zME=4V2-R&u++wM_&?W)9Pu9(4XhQtmL~&ni?a+&C_A3_ehSdt87Av)!&#? zJEpi)oUvwqUkPFle>z4}uL+R9*8*-Ye#VwtYoa5L|Hzwh-3axJp&d&jXy)=T^m_Fg zS|@e~&B<+Geo0uPnXAt+ZO>Ctb6OPtNkJR)XW=}Qp8t#nHmst(7UC!_=LK>#KR_*{ z-=nqc6zcM>fLW7R#(b#DVa`Q2aJ7S#NcZ>`M(=hvE4NS#JxMBHFZs>m?mtclZ)BvS zC7tDH;f*t>zjHhdD?Nl>D(g^>#Q&I&(RVEz}$Rrx^wuPU3{sN;Nz5|t4 zF*N2|E_-0;D(V^H(@ytvtwPo^TR-FEc% zZ%8L|zEKI3#LT&Rf*G8r%iOVbMc@DQ@pty#p}US6a;kbW^5d--F-`u8Q+F)IHS&{5 z*o9bdGXEiO-n7R&3sVJ9{mcgTbl=AJ=Ww1au8$aG_y`X>%EOd9Kbyl8f4(z-0 zGC()Gu;glhE5{k&C6D}}KI2NJT{a@(HD7Syo1NIwwH|+e{}D8lcn}4*M?k`{sAl6~ zCldZa6Q1s!08zqJ;`(w4FiX~exo!)2E92^bKgU?42Up|3I}rNc2KY~uIUcI`g!jj~ z@aE*c5o|uIK;EA>#oap}<34VxX1#I>&-7*~Xv^FQr0Y30M(w*Zvs-~I>2!eW=S_r~ zjxuoRo(xbPC{O-wPsfQ_T(sbIU+iI1!b@s22UpT!1Z(=$p~~1dyl_V&KC}1~-o5p# zbLzWUWPU_G(C2zUAx0DM7=X99skrE#%Hg@lR@=fa8~~mE>Bqi!p{&qJhX^-O{>8owZ}lu zgn?^6YQtB{D}loG9#G5ure=jsa3e+)>Uu5_^u)f!OYcjNFR3P+7tssGPks$H9djf- zG7Vs>m@*vsGLfih=@DbuG^{&22}@V+!rj^~f^zPsj zk<;L$7d5=`A6MaLkqcnu&?#_vkt^|>;f#|w2ij;?I}o3B9cQQ*ld0*Zq;;9KTF~ z*_2Xz-fa=-;&DE z1n{fq2X-Bd0?&6Sle3q&%=@F;v8%PXV5Q(Q);Xy~;+FT}{}%oboGmi-=&ZIC(ZUAEeFwgGG2M z`4_hgXb+8pqwOs`$+@Y{u_qv^GI$bW6K9SI zOP@r14P{}yfbyOcE0aSp%fNc~Y9JHz5eLu^0qV~es9hC@b{rGf{7Qptvd{q874Lx7 zTSpvadmU`tC=N^C=|B(VX;8Yt5cWHF;NENbc#B;uxLhX-7keKE9qP*P(rR=3Z)G5s z^8}=$a2JR+GsRz%PVrv;F@kJv5XidB?V;c1gTK+qVC>j_p!w33gx)oQBRaFmjHWv9 z_=gs8ubF}Q?&46ojUijJO7XEsLz0zHfsP{XbW-^|n^F;$Xq;m(twuFeoP zKDos+nY0!Ugz1GwB4<``?k z`m?uyy>$(EyL&NIIJp7WCGHXAsy@f|%cqc&vbr$r%L9Cf^VwK({k;YHF1TfFmO!yB z0l$Bk$D5mc3OxPi4A1M$Aj?*50m;VlP)lByeEB_st8g;s5ppI+y7{=TToEXxzYsJV z&LWzl`S`i@f51NI5cu_T99;fs4vFy|Ptqn%A&0j+!m-_mOg1bQynAT{=c=la$d6)R zV-e-JT?evARh*(cPe zH=5+T*ji_At7?42tpy~VISdBBHVXP&459DpSHQquiRW_L7;26g!K(e{aPjhuIGXdR z#q6I-uKPN}wzwg1#_WUO*U$0f$k9JFTLjPX9fxmtN{0xKwJc_ zPbdMkyF#&~X&5=I^a6LwoWqB-rjh3n%dzOj2#!gL!Y3YlN=_&<#y=w#?XFLa@@5{i!(<<<>NE2VlzXaB3uI1f&_a9C=Qwn;y{X6aIGpyGa zj;*e`LC;m1P)b>V=gyA;KdZd(g4@=hO;r}JidiQZIn|4;^lV9DcL2H2JDzN4Yyq#X zU*Yjj#MKo2tpXx#afscMarUr0m*ck(mcDg^wP!6zc-;lf{d}13pg*19r0mGVLfv0gGo~yqEmf49zVTuLGaUMss z(>(GoE`X|$5N&^vPOx2;&v(!H>en;MSN5Y3b#W2CY8KN)Cai+PP0-XIe7{PE&7f^2H3p4DD$o%jwY~XK3 zBJ64fQ9G{ysU&-NF}xK#9vi~)#|n-1-#n#0-1{+@ycj#xIxWXr3i>?nGZ|?vL*XQ&8 z%S^z+bR&XW)JfU9zxc`I~66wf-+`h3`&n2cW-jAFfW zK_uZM=TCk#igOe@u-?omv!*4j#`}XPaE*3~~ndSAk-cFt< z-<$|*3l4&k0~f&i@yg^03wEISS! zSaZB>aXF59V@nEhZGjq3m-uTLIMM)Dq3-}+!$>3 zLlruBi~{RUSt2)M5uDAl0D8_gaq(#yRx1jJ$O8b_Z|;U8u8#cb^yCw z-hf}}SwSsVO{n6dM^^jTkYv$4tnjM>1gENEW6@mVraBv+*8YU;3_9@lwa-EF8!Omu z{YMb3l_^M{$z_JVbiidxqj*xurhMBiB}Opb_dh|u)pX$ST!A#7_=KHGx!QNclY|Z5#A+I5u&rGReke}BlF{Q}dw(GB z!ZuBu(zgbB*@{CVFXX+dRwRnKgW!I2F241dJA`e-K*`{PEI+*O4IO3KhfV?DN?^AfbE zyv0j4O(1XgIB}awh2Sdd0~-g$;Oo!~48oM5T+1kLgXws|r9wH5k9^HjUp5JXiL=OK z^&Y%fPYO0TiQ(yL8X&Sr6_ys|VVQnIA}eM{+DRB*dqNM6(b&hB1G^TWx>AxU#n(@V5!jUhRPIPrnT$%`3sH(M(_$v6u`sm_p%KN%)=BB8&G| zVvQFCcvVUV&ZxeH*SBY2Gm|Miw}72BIw!@+ln)%cM*ai87pmZ2mMa7^)=EQ-X9qz0 z&UhUEK^I4MyagKHo`bo{6NtiAL>!W31>+86)JS`A>tfz+oG`|~(7Dn0O}#d$SSk%? zY?KD8)+plX+l*lUo>h3f>jLPOu?yr#Xp*&3iQtnd$9xsu#5*47!AGZj!MF4Z-d*cC zVC^9T;_awLRu2k!Ef)_6v@)FGiBu7oY}|+|J_~tw_8EZ4kr~A6LNR`1y+bg`Egu}L z)q}!QM9|QCTu?i&i1%&1gp(+Qo5#`+18z^1vH7lkfk}udzGYqLY?=NDq>CGpiN6>e zI^2Q%%N$|q-Xi>#YQs4zyz%!ZI=r8EXX6`rRv@R=7Tgxs2iw*+0nN5t%%pq=7U||7 z=36qbnrsQbW<-Mct^K&(YB6Dd-{w4mj_`ozX*|fEPSQLl61Spp(DC3JaPK(B3m#hr zY>cjeAj4}ou0?``k2K(-95L{kD1mVwRPn_lI&fyM9gx$xjMLU~j?DNj!DsaYV3&?K zyd9bcc1w7J&I=m=Ij{sUWeTMC_jleJx%=SPw-v;;;}CfG`XHDcdH|0W9tE32Zwabu zjtKs)Edb|!DC5^Q_naphb$}=vUBR5-e7yO4hrnoE7))t!0@}|aK(v((+%#hujOJ$Q zz%0+3yEls8@CA3pM8 zKAv3J2Tnao!!os5U{KT!?6@Ep+O{n_(KE)8|I46X%79WkCQx(V*hGX;Gfo!G+7 z1_vf3;gRd&Fs)o)wU8}DQguB?JWhi{rhojk}Og9 zFBaEyTS23w9J*C-7yW19Mi<`LHJ%!uxD_RHQ0)ay(+68}_Bw`-}p*eAbd zveV1haZ@hPoLP71l0PZv>xMsQmRt!{cQd2D4}Fo(hG=>r`VOtyXOF&DHlnkfzhZ93 z0pzC~Ko<|mpy600KYv9qR~=EHD`8kbJihf+V+0o_Gn$9 zfStuZT(m^!yKgUDbifCF{-;R8HeN>C#x}4kA{U{wIf*Fyg)U8=myL!(W!TJ3rf5j! zpzwHW5c^Ww4$bmRLGORNV?%WXmvxiw(T$45?4mp^S~992bi2}kqF+$H-^GWFtfMvx z(V9wERr?@A#Yd>gegYaSkU`R~m(r$$a`feaq|3{TbND4a2T&wf$e!7_5M^cN&=)($ zi!O=1Vt6_3=$l|@Ki~huqj5-x(ex4EM&)<)b((+XlJYwnAqkMEG^DOIsKbcKR zPi0x(Y_!8J4xP(r;!8NkGw;&O(56k}xfyM)hgUO;o?jnCyN*p@%{M;d7=x#b#|LTp zVYV{cHuX4C?^AM__b-@Ywv?H)v6M;FPvXyvuplFAvl-h{FVWNcXDFM}$JUOkWk0;O zrf;R@)3t(lCXH>S^E4kJnbVLQ;1J`J=)VV|)2hcY`@*Ge~t!l~ZL zCe}D#+~s=iO<|;C7scClAuMHvc0Wy{GnWFF|JJzDXF$d!=iFQLIOG_8e`p)hyewoR zngfL?>A7@PmJ^N6?4(OQdf2&ZZZkT5Nv!+(QW)}Kf@oQ_JDny|MZA4S&@A~g$c(*) zbTgJS=RUNRet{ZG0IIrlfw9ky~PH%t;bX$8Lo%&-E)3akby|QIC zefFV?$+!^DC=F}!RYD%KqIZwkR`XQaW<}^7aFF?wRE5N@q@cAcJ~O&s7E&!2Q`SE( z3vIDEO+!0e>8Lelh8dShr6SJ|&1)id_?<0{;u#Bl-d|&eE!R@-Nz>U;t~V4YRQOb~6iE0uv)B5(Q9$)AM)$iO?SBjCv6%&E z{Lv(oo#I1xMeU})RgzfQHRWtthAYipoy|_`@DOFSPh&J7w3Ds^qt<3RYT2=d06AzAZyxooXe42kIJb7ZNA}* zeC8C=uU^$czw;ZA@B2w8eP^{0+4eHOC-|bTYK`=4^u^V0E+(v91%%n8OLH{^qyV`QqML+iz1fM%i%W|!!PRKG_#sI z<=NA_;{#B>?=Vt&Awbf{DyfU;HFJ99PxcL8M|ivVKC|;zBn^9Xj5+@KBh!`DjG`46iq+<;hj5f^*U7 zOSR}jL=mzdIfvpS57DRMWz?o;6a9X4KXb^(0WFanB*792H2&5yl)p5Rb4Nj%I8T=S z#bG6T77ruSg&bEKra^V2*CG2U)A^6_W_D=yHMFT94MixgV)TC>M0xC7^h9h6s$i2* z2HnVT3XrT1e?$gVpq^{9H!&-Y~NJ@tAG+#JHG$C}w}G znn^?5FdI^AhRVtk=mm!w#%+BsG7&98QbRFlV#-^lh0CGa-CfL1=6Kop+vhOi{XdXq z+&A`aYo~C1r!~FJIl?|GX42jD(P(?d3)X(aMK-Z+JBs&sz=-pYvEpVDXhZr06ycT4 zeA|`IOio?QuSyT6Ga7!dzYf}<%&dkXXO{4LCz0`5iU>0x*z43>8Dw0p;>_n-QUyE z;zm^9evzS$C)k4@B3VAxav3L?k0z?gqtXk^D{cOn-hb3_NCVxNGsr)@&nee+PcP6^Fi`i;GJzlJHQJ%=i*-!m>D z#WZNW7M))hLoc0+Ln>KwkXuM8RVy?=TW|5`>NIs)bR>Y)nl_bM`0hX%M+(uiFTv;u z*Fz7zZ|Ji0@KqGLJ*1`^YNG|gTj2!SDRbj)32xZIo zGX;BNY4O@eOm75(_P?m04(~?za!|;~x9mpMEqPqd-bVDvgX@LfSEjl66&sd(k!>4z z$%W~;qxM*Dy8Ci36Y6C}-Oh~(=h(C}v&t&a+X6{;4VZ_VE!$Ydzfr7c;cC(OLbwXh*uFDB*>IXgJ0~Bp+5!?`_$HF4SKT26D3+-4~u=W)Eg_okQqmC3n_H z?Jj$+Qb2v9D;f22NqR6qo*kKgljOWyMTfO-AyaWr_T$thauH^uJZ?@w{COOs?k-}k zOWL9C`%f8#T6Yw%(ud7GoJ{Yhh|sm7xhR#L$i^gZr$O8QptdRPjEA=&`s1_=i>=?p zlI5w$AUBy#UR=p+`4URY-w#m9^X8(@D_-)aPEvN++2h697EPh=f{GYdyPxRmsy<=t zn>$R_`!~X6xhXwb_)u`&>=UScbhMx=78nx6uVIV@zY~cYg8aQ$ln~#%1;Ne0J65UG%BR3naS6 zqwYL8#B(@?zCMXTv+bwTIGs45x1kfW;ORTWw=Wik9!aBun1ys#&VS6k_MOP3>Ik}Z zMqT)3+%_sQlSEq%S2MF~p0NMUB0ACVG|irx%%lck>cr(72Ybyzs_s>2Q_exQi<6l3 zUKFGIf=^Sg(ig($6l?wz8zOYzYESR}SOhdKqImL_FP`&(PMGe1d{|YN>d^aauFN!M z*^>ikyH*-K!Lc}Z_uDWD@8f7;eGV=Bw*#5Iea!58mx;E99YSRi`p8yESCqx5BdKmX z=4y8u3Tus^+5J}Rt3BhGeNVZ1^!Wmmb9gq&@QOq+7ClrnBOM)Y@k85FEU2%PBC0qw z1t~P!GG+OX_?te;(|&sum$qwJXrY%D3UqcrdBsy$(S{uMGC9HccrIe6CPcGL=w!6w z7^eA$3|ZN^yV=uIrlFO$N}xV(9P@s-ke*yOn+Y;MPMs>m(2;YQ%ndVX7wcU;%vcgnQ)t_wq&m{iN^cO_JLq6CLRu{e%%!MI#;Wx_D~C#GAPk|p@j^7U_8%Il^<>8q z4PA=Egy^Pp1?~F0it>XFp`v+~$fw~NGrtgtyd+++tmg@8@U@)t%vMpOhBN54Nfo+q zAParn`%h8XK^eWCHp z3GDq33h3T(MVhA+ORvh5ppwtC=nE|#bv*V5&D7LpQe8Y)g_A-m;MnQI-;R+(`E?(qU?*ApV*jIZ?o$YMQi5S(H0=UwHaJB08BV zhi z#G=fU?-X5>V^)qvqiBN(Oth;iC7xW4spBx!XuZVzV#P&@!7<3pmq-0-qiN%rBMenf zr0#+wM#c9Ko$t<_(|2Oj)W3*^2lLtcKE;$em!qvq2_Hyi(tBIa(9oU=v@Ir;{;P>W z>(5uQDC{adyZ;gse3fB!s`ZhpnGsc7pp2YtBhlf}L^dbtCY$gin?_Y~?CZBEVf?y2 zR5AAkBmHJ0=f|%^dF>L&P&$mQn4>K889B}mdOMy8y}kwIaID;y*#k`KlnZEG(!c}=(z_^$z5_4Vh?T0$ zi4)%RY|Kn1ZTxH6a&;HWYj{eX_x?nQZ5GrhGaJ?9HVQqTEkTZX<3vl-huQ5lHOwZv z1@vrq2|HlfiAJ=CgqMXX=+E^>)bZ|0YHP8F@_tt!D_;#{qTq#Q^v@90wm2{b2NTeb z${ZRWuoL;kYq(tT=H@u>?xN)?t#pyzT-0S{hz8%Sqb&(F$d4Z<9DiUtGqR|eZeA}> zH~zeh)-e~@?*|)cMTiNyoZQU>yj{S2j@ynFeC|SJgT4IYvv<=pw1#cq6b?R~K8)9p z0ScZ(DBsJ7`BB%$z&&r-r{*`2Z~Yj3ymUPpx+KZ?eLc!N(>_N3e(z)?q_n8vgbCC~ zcMj{Pz^8>7h5X~PuW0YD88m5*#c5=<_gD{EOme3s`0rVk2ir(#&Mk(jFA@TS?)HwSpI+t^Vzj{YB+m`kX>3%%NEcrZyJ~m)_ z6NWOA`XJ}9DmF}<)ThHV5Ifg{pLOh<4rL`}u&7s;<9n$qk z7nc2N0;-;XbIgfD4Y52>Q8$OQ|8@dJ4>~MC|7}ko+FV zxje$~H=E<&_&O=9yoLuAO{e1ZR}KM#{kG(M#!cW?b`6YDSK{z08~n?gO$ri@V3U+P z;6l#W3=c1FaO!9^kbK`(eQQ7hOs*Zq3oC^9=>uiBVo?;Rc+duL={KHp-D{v+1L3Ol zsbtrdK=3{`2h+?vu;yGMuiwd%=!Tj>;Tc;Pt9k;Tp8Xs**Ez$fi*?BQtRgVppSyX9 zJ1y`_IE1Y%W#BmNI`HztM|@uA8BW37p(%>TFpRxSC`-_*OS{K=?T+K6L68`930Q9 z1S@WuW5t#ZEIz#s7z-q!*~wnKbaOeL)chB}J7_?{=68Wixnc0+eXu~RZXA5GLkb7p z90#qsy$LbVAg06L!Gvcy_`f>~;Ps|>yno;xFL0|RjNd6M@Z0VV4}3s)xBXeLeT5ht zKjjoI4!?|FZ>|Fg^LF6#`}By#;3IJF^-jTQ8Et$e#g3F7=*2hvtYGG#HCb>+9xJ_2 z#HU_Pg+DWI%+PWbf>km@5N?r&ahRx2gmByhGOvfK-+Q{JgOR4@8T>?)L9s z$puZ4BS@*esg#EMPId$3&8r1Bia&sXZ41DL-3Q4LiEB8p<^XoTZb=Tu7{F&Pn{b8Y z3-IHF0jMl7C;KASLc1mJuxWt=(G5ES%6+CmOl9G`+Fl%J^$EzBuLDmexxo3EzwnE{ z>tGQt6Hl?ThhO$y!{?0xARIdaG`wsDjjG%l8a~K7wQ&Mj{<#BuJ=~2Kn>@p(s`ulm z6)j+y#Z2;Jmj@2~%mbA?9b#Od4*S+N0S~b(9IG*mKOa?|@yCJJ^-62TI85!&!f?V3z@1;`zZ8h&IWSzRn2Hd43rz|J{RI zZa5HE%@{cLqX##hcmn?0JyG!Jz8RF=V~y8kxsz#Ka`2P71Ux~$;I20hdF4?Id6X6o zO_TJTbFUWTuoNqDRNDa7j-Lnyo>uZ+XqN&LrzzNNZ5Y?P?ge;30MPhX14fs{;?4kd z=>F9LPMxemEI40Fl=u@I_OuLi?$E%Ebs@xd&IfQvK^Ln&m`LWX41#}?2XL-Qvh$d( z70?W{7PzS-fFtwy(CSqp7-Cl8&xwb?{YSFIq%RMgt?d>Z4d8fLyBg=3WAOsbQ-Q>7 zoC*nD#AO+_`;!FO#{#8`BY1FnC(t|5UAyTFkLU_IaLrCY0$28P<69okZDgIZ)Mq3S_>ugG+_&JUjo_ zynbOnP~2OMe>s(cS`CE-)ju#?6i_M9gyd~V$ z`Ty+aVl4UkG%!~F3A#QX2fOxu!mc@M;Q8t2aq*5xF!I!FuKAu5V)bxQ3N*Mx$XA{Y~Rz4_pcISEx z;n?k<8r)*_thUVlG}wAb4Zi-PMs7LCf|qK=xZ;#8%S+L-CC`EDnGvAN(FUd;&H{BmU*VMxjbY`Z`@l&wiW!+9@G52k93l=-##G$~s4x|i#rpYqoZ(|Og$33g0g4^J!njHv@a3?D5LhS7}o{aV6i>AQ;l%^8W+z%G*kPsZ$5(ZDmE#Z}Y+=EHs z5ialUykK9?dhn=hJnU?Z;t<_O1{wPH-^zt}w zV|BZrd^dzA^A=*Qjj{wj>co5ezk#?jr*P)X8DRHLS-3h*0oKYc!z=zr(V6&D^>uL= znKh9H6pX4jP-hP`jkmJ7QVlg^s54r^+;2fydDC&^i6MR=OsyPrnVhM31p zf2(KQ3WuTWAJ-giz&~rL{(zxuTy0g!M71KO>(2o8bLD!ebN(ByS#SVW_*ID{2VB^z z`vk7(%xcjj!JR8p6u8x|?WJ=B=9BlFfzky1Ma<#<>X|;(Pu$r@5^4XXCTtorifbf7vhKo+D=6>~iE3!WvR5Ie+ z2sU4m%Dg)!Vp%$Yv-MJCZBJ|ya_Z-pZ|}6ERmVoL8)uzhf{SFF$NGMpNz6yidW$+! zK4}e;#Wab+15>zTm(`^|=ZujK&RJKoUS`h}8)=EOWlu|bUUf2CJnBR{ypJ-C!X8uf zm!q8R?6K_cvKq$!;6ujCSVwyJ)NmGsJjRokMr@$P4$-Z>DvZoy7+aXpPkMM-1-DGm ztE5`U8OEt5amD{Rv!V=TX-v{$Y5H|N=CrG;^zD5G>8Dw<+2>|P(kGIfK3fo}qzsNbwGC%znaIZ@tV7@QC2VAIESH zH3mt0Gy|D8Kk_)=mD{-ez9n2o$}uK}&a1qUWxMOHgpa*UeUyHW}%%)$997rwjS;%Ezm?Xs48^Yx(|C z#ouvEQiHzKZ+zANU(MVkw#_bqHvd4x`0bGYi3NwvC{3s z78KuzAIlyI{mGdpW-tp{qL}^_L#0JN512jbHQXHI6w#!3MfUWGUF@$-95?i^9;^7F zoSFXmFf-!zBqnY|7W44HkP_7$R_yRy(ac2g4MxE~QIyK>6>@Lqxa3E_M79eLb6K|K zOx^Sp?$*{K=J#cW-Lc7zz1L&MSU-ElOfa9qZ6=%9%_nwBzlPbfLyL5{*Pq%%Ycz$L zBk3SB&s)j`^$z6he~MU_Pv^J=zfG7m4&S-zg&N$yO)t1(V|Gh_6}a2=`;pAmbOuSq ziuugcU%W`=?iAKgQ;GepF|@dP&GC}0R8y)^`H;D=Z7j3X-=}!y_lMlP-8VVqP3{aU zu$J|G^|%u4Xr^nM8lz!#RLC^liTE3P-aE)8*Nx>dFG= zcLLkn-&*=wq9AqNZZ35l`Gs4kmcf0jnJG>E+fSPF{EgkXh94Y>9>N}#591~=<3yXX zIPMHHLpoAdk&VsPVvRxvN&hgsNLB3#*PD8l+dAeb=RRf{Ybr2T%Su(Hr)>30emy$H zu~|d7a|dM1CQ+^Ey8SdR>A8s8{&;rDQmd!jDHB!J;Q1Ai-@fNu_pGCgid7eLe!@dW z$F+pRr%Ehm+h3a4@Q$mwR?9ru`;PNXh!GBDMJaS=5l#$XK@Pvz%odv%q$W^j+5=w*K82?(b#=R=M{#GkEchk`GlwrQYvv zaoS4VT&s|!&JNa)Zhf+w$$vbZoK^5pw$XGk zBTM+i$mx9--CrwWTBZ(T{WqLoY;I(hoYnZpwBKrDwl+%+j~fx{khVgGZLH-_p~ge$DbQ>4GvH2+@yn~Ma&F=Lwbif{nkQS;iD;yIe(qG z@3NkajPRDuzj270o^V_gdUPH4XUQQ>D@&7IyTFcn_uiV#`?INpk$=r~3mjROhbOt; zMajh#l}_xAJtL(GOZu@p3^O^_PsrqNVL08q(Ncf@0aI2ba8@fF+4z(i?&*`kQsr7j zsl1ydtMnzRjszI;Y?&LhS*@|vjCetPjwwGg)8ud3Aiya9Xn&iUdua!(m6B?Wftt3!-i z@IS^PxrAxGXeRwL#+419bb{+p|I3ZDChV%roeU9u7nSHLu#u))N-DmMm+HUOV4c=Y zlRoS3!@SoUBwguoj(c?Y3zt6jDVN@LRV3*-%|+M@l}1+lDS0C6C!Kl2n*H2=1T%YB zs$E3`!+3-#v1KDT=E$sJc7X#FguRqbW@+17^goz@My7VF!9h9zJmw6i_tx&k=k7Of&YLJ0$j^}VpR53DCl6y2Dhwp_uQ%N5 zzZb%uj`-x2FHXGU3|a@ap#RU07_z=UR=nQ>llnDcj{i{{c3TDfs?9+z`30eO-q4x{ zF8E&WD2WSN3OPaZP?jAJBA;`F*_1ARkhTebvQweScN&g2Xa-L280tOkq+iHhc0ag2&!>*-J?pc&qL;e(hN= z4Rpw*3M$63y>X1}kmNn-+w%)(T^!j{{uZAachG{o3pnI_7MXLtkp9~`M(XHpL#One zfI0WHL2jZBk=|93nU59|m7^x4$2$_WZ)ZW)G&d+Wn+=wx6)-tL2}^{`;yKe|%>BBL zZuuGoE45#cm*4%-y=MVF=$?(*lK!&e!sk4DKY(s?Jqj5+i|P0XecAHB9x}{jA?2Sr z(j7~-f}!tOd{f0k*OUM7@ZWBXJM{wZt{8=VRqMpjHaGY~J0!SgEe~r0o|3|VKlIs) zArk+710-m%4rZ3krZ%=;aqM&xRJ{HL+|ANRG;@>e^f9KNQg=Yit91C$Pw={S3BF)g zUAr|kGYEu7U|4$?HUyoat`%_sIe$)6NThQHSJp?v|;HF#aW%EzA5vP!HOz$*=juv_BV+TnhQcNUnypH5r z_EKCo{{wx~G67#{dSGkdaGByn4}ROnRBYM)L_CUH1}|F6=%XFq=*ztY&@rnK=8>o3 zHB)O*PiPgxhQPl>bJlm_EE-@&Rhgr_3)@X{KhDXC4yB^x`Jp$q@N5jSY5yT*bQsd_g zEVO(B+Qk&Ie9u7J20fg%)f@LX?8Z+g)gf)2GqKxQgF8n=L0|SMaC{+wHwq(|M$dA# zt?Hk6o987wZ`DkytqSNzhw-2c6?E*y3#2DEiLMH>#&WkBNKbF&H?O@z+H_SQW!`d{ zs`Q48-9M9ZBRA5M*-yx#Q~9udSRVRr8z^p5GDY3lCgkqAC9soHnD}KEYMf8RuN`^t z#7F}dnHOQM6;EvZKaqKZ3}s68wRCdrAj$q;@nEVk4!*jlpo5V8tJ&ni2Kt7h+U_vU zuk1a&>Q+YbO8h|WrY|h}beFEZ^q5Tl)xdj$5}7&b18NOCC9W_kM8lX0ey`mb+Ipm1 zY|RTkxw}v4^Tn#{_sRFz<6H{d63+R9UNNLn#}G%`9ww_tj}tol z9FnT*+oX#3BSqkD(WGgO2)WvLdSXq5uWOp^;@#l zj)!~Qg7>yy0OS}A!1c*<<+!-|Za%b% zgT*mLhvD=0-EiZ}KuL};``5X-6kZ7nNbZ3Nr0vLR!Kbb$3GWLP4PNWWeY7|ycq=_& zdx)*fqirNL&)5Y96Bp8Z!_5WnZz5H-84bF@Y51T_ndHpRBcYpZiQ3)qsI$TtqF0#m z?yBF&B!O!-L3AEx{P8AZHeM39hfgGDc1L6R$#+=V_ZXOwhLSU?>#_Rja(dhF8vN~X zLASmTSmm!TcDuO{8m*(mFP>cy>seibyB;g?YgI6gQQFG3a+~nhEi2eLau#L`=^za` zQQ#7Gj2<+YOI>wxakG~#_8CUAHwIkhi|vlH>Xm<~akdSPFVw)}PyVB~k5~(2(qm8_ z+)nQ&f8;N(8i8{sWk7jq81A3_g&Hc(2ER3$sQjOd4zN1`pFI?T#KzE>NjHe{O%p8P z5!;6Eqz&VR%%`^|7W{OjYOAy8L2H4O%AJ*V_@0}KOFOqppixe-4;2BwEm2x zX@B}rt@^LxlqA4Gk1yi&=MW+O~t$wA4uGS4D#O-7w{r&l5TKk@dnv1D?wU#w{Zn=+-RbwG-(FoZ9 z?|9PaIFA0bqe3j@HhdqO4r$G!WwIVC2>+V^dz6LjW5rJ1ecD|7a&ZhKq62E2$q*5JdsNI=S5uS zEY!b0R*NHx<`IpBsbuxKc%rSc6gMa}(OG9Xv`QBfzlBHOPStCA>3b-QxR8PkuXx(p zeH0sC9-=zavoO=wg|vs5z;25qD9{L@7d!^jC|h+nVD$hFk4qCcd5+X-%TSo;8^I5{ z>4}zW+i`}e;NfgB#yYhox;Quws@Ke+`bXYhvGMX22qb@OQH_K6IZz znt3ry9;ihYj!+a|pH)vM8bnmpeTe4#zr@mi7#PU3_%+j; za3t47ZQbuPFD|UXj$KEoy^;p}mW2~8aT*yoRTC7OYpGe>Zdj*z7uSpGap{2?42ib} zX|W0({!d^=SA|iR>shkmh+FgJg+{#m{kj>7SB+(S^dI24q;Y0FI^4Dwt zI?PO=GaScDrakLs>tZs=pb?L#MZh%Na6Fn$JUSE_9Y#ZL;%gXkZ7VLCa)`K0Z<7A( zy-2m5^ANF47dPgmfOW5>z>GY}C+AqvQA^Ljln7t(t@;#laVdpl`4c!#$er!JG!0`L zUeWc|D(*hl3_M;->RSpxQGRvIXbw$8{DEzJ3$RO*;+cUt>w@9c@X& zEI&w|MlkT@7T7ZEA)0%6K~n8aaLow7rtm08oIVQ{s8L$`?jQc}+XKh#hodwx2s5Iy zi5LF|O7$t&4g5{rPccw7_z?^lw}VDE=?M(9JXm)sksNQ>4$J&3ATcrppUhhWEwv#u zMBs+$r`)0QbwkC|`_F=s(^Z%pbQmMzw~(NuH%PVTL!HYBP<+S@4}>J+fnUBL***(R&dz}i z?ttz=Y0{;Rt@Kyzbhyccknb-xu}|LEvwYNe2%dA3WVh6jogX)2=;N*YxhoaiCj*hp zalbyM%e4}=@Cw;7IafP4Oq=r!;(nLN)Q$)nZu&#!KpWqld9 zx1PrGvG!oPvJg|QrLy{0-%uA>2x;5%5La)w5Asij$-E|hAR;S{{&GG5-*Z=^PDvHo zYK_B`g)T6nFP4nEJwVp&n2H{~-ehfq1oNIOgm0f((6zLSOc~jPn_sq*YXbLd(wl`) zcE}ht>|IE!)@f{8`4c?qSHOp~JUl3;A=we1Anu!*2D_~ai2b7N^mRWqEWbS+jqm!w z!Xf%t^Hmx9;=l0KBNmWzk0NP|YPWcbaG!2&mdC4mFVT^~;bctrF_7Q359aPVg|SM` znCEi}XH^K_*EI@)`sUEEgc^F|JkOh@7Ld3lH(=z6{dkLyf@Ftt^wYE+I6xi(^M!+= zSP7av1FUkbqi^QOVbQg#P(J%GI`G%XJ54Y0=7KrCD|t`7#xIw?P0s=)ZFMX)xC!-D zkFn*=MsypWO8TZ5$^4ePk;X0cP|N1?na(9*RbgLbhIj~F?(&fs<;}v|Dd*7X*9Gi* zy$-*P-9~HsmI*^5dvedrhCDTN$CcSTk(qTL*RQGNKh0_-1_p=ukoID{FYGEdSF8q) z_YB!{E({Egex_-0+Ax6cB||sQl(f!i!x-}KD#{jC?7;XLKWMg1BZ;3JPc8c? z5}DTlYG+eKL2d(j=AWb0ar^1}h9KI}ss!`4ZAD-3=Km}-qW0btn&)W~C%8&h#SEqg ze%%MnloFc$@D8;v(TDdf6$l4S!PN2!xXJqx%gA(0*p`HTMRB0AIZ)Q}HlI384uYQB z5dq?A5}4$A>}hfkWVu9f?0RdO>dTVH#W!1~}p%8JoBhgREH4 z-S&^pS&&Ur--g2YDhlx=$cF3>Gw`l4Us;Nn=C|L=|O#%Z7L|VRI#0t&j)4A1o!KctzVxha~E_Ua!gDH?|3jTl@xelkQTC*bI5 zcKH71OVYSH1kdT3!gW1ipP@1glLR;Q?bB=EnWh3R6D`9#QFmz4$Vm_`orlxf^}(j# z39WbD2pM{oWOUXYoL}vXFK!-z33qe|H9IEi`R;^sM!cm;|K-pde;C*``W4x^&=7hK z6%u;mB95zHLA#<~P@|4|c(}X)FC4rK5%q?6?ZXWG<;sXtyV^*9!#sFb<0I}jO$$F- z9;D_&v$3ClDU?dW;kUyLSm2)lKAv-+`qv3^AL3!*#CbU9?FG>F`+DAp-=~cM!~3cAFZn-w*vFjPZNSF$i8W0%9Cg zAwNEfKh<^!Ca8_V+W!VfBbbXMjqxNYnK}|@$xT={#h5BJJrhq2Fo5fe#*+%-PptJr z=nrRaxG*ajb=v>ZtkNz_UGN@Sj0VB)L=);a81YO;BW<58FZ-Nv8y+8d4$Xs-$)}|{ z=xLycYdkYiQZx>>UtW$SXV=5`;2Aj9qX)K!eiduD9>=A6$-L(W3Jo{M6a9dG5~CHL zDg8MQ4I3ZR)F(sOK&KHHr8yc}oDv~Iqy+sy2P!@*;fC-OFnzZQcSrcsX1#HeB0WpV z*Okeb=N<}e={fY$97}xaSk7yE#zFObYxve@!+ZCv!5K%3@TkUMGGc!U{;3JaMLI#) z6Ffv#Zr&;#)cO(N%Y zVR$}K^LIxLlN0=?!qYgmwUu1^=8H=;&qLWVLu%>ZL#NChEB=`oDb{bhBHq6)3+Fx^ ziYuD7fX^i$FFFGte)T4BzS%~b@_vy1;mg3J{|TC&JR24FG(e&1Fv+d~(QstlFDB8n<~+fIGC!hbeMriGv%u~Se@fTYK*E)MIMOQ^etff$osiE! zc^gWW{a3+kQ1Hb8&n$@Zy=I);lt5nzPJ-r&FT85XN$4|9CW~~=k@fzB?&Cc%UH%Q- z?`0-_BJ4j5XJ6pNPAlRq>>IUTT@22(t1;ql8t9LoPgK6G0@sK{kbAp>i1+K_8COqS zS}l)rCfuf@R!;}1rVQ@y&jf?e&$RE*9}-KPaKMj1{&?G1#SMse0(G{Cu>!;S0eKbMP0MqVNYUe?WPIdl`q99jU+Z_V*x=hxEWKHceQI(r zZmFGke%dAR%@I{_aAOLPn zI}STc<_x%ud$c&*OI2m7Z61j)PDrKe4}?MJbax_uyp(^dqJTZpuYwZIm@jeF$9|U| z;{XqH99iy#=4&rwuTvg;XN7vknhntr!6TJiL#qI$$c(Zjo zaNA8~*1BJyEbR#VS({9+8qdT|EiFlNRRR`YY$nH+&W3h@qI&p>0=~~!*0z|{P%brlr7yu7P=SX ztq)oB+dm6jAneAqEAqHtbsbT7RfQ8vx5Ba;rgZ$(o4DHPF~48fKaRb1oYeeULLH1; zq2!t&$!YGTKG%1lQS3bULSKXJEGe;Z{)+N;lc8hW0`OU7gio9wV^3%X?kOGx?`}NC zMJMCMD|}KxCynPH%NJsR%$3^jIgEEKLcztLfYfyQV*2J@>VIY@3{1Q(9vB;lFD?b( z6xE5qk9vu-9*m^n6E=d=!X4D>r5rpP(n8}_?ZN#!(($LV4_;Z$(XmP@P^D9hN^w_U zOw0y|H<=4qQw^gQv_PKGYuqMh3x~x=sKtU|&~WP(nSSgY;^N`h6ncw@wuQqakr!Ma zYzuQXZ^Ge=M}dmZFH{RD!MZ#BpuPJ)5~$%Xt$010B8Ht9Qm2cCsp5y$u3pq?|#FthLv;mWgM?bHzx_nDEX*gOZWte%U#S5HCzk0)V| z`UT7x9g72xcf$Tz)5YHpUZ)8PexR-To<1qMh&{IrWmjCM z+54#7e^tU6n}CPDPJno~0VH=*7I`~0kE~Kkhbq6B`1`JyHyhY4H7SvYs>&twVXz3w z6e{7@&lh&b(hh77L(Lq64l?Y#5&>X#LyKFlKb67yhc%uYJ@ST)GXW;x3_1EY+Y}LJZ;9kY7+)^&3~V>59jz zWZ>hsiZYLWhsC|J2Xyb$4dT{;UgVK;u)q=!7!5mzi*<(9Vo1Sa`tNiR{pp;?*O|4G zZCz?`q+$TxF*`(7ZBv#g3_d}1EoMkoJ1O(WT~AZTvw~B?-VgS8i#{>G5)8wWzuvJ_|MH7=)vjE2@v}GIkI($s8RTosD6;) zjuD4Q?6fg3aAg_IaybKuBPYOxliOj|MJt^9r&t^BgNFeO3W~@6KhglB>!@$jH=zjkao*c3Sk_B%4>(v3UexfcI zzR0H+kG>#X+iM`QaUdBtNf$5w*$M>_J~%Wg6Hi?(p;L1mXhdWZaVzAp!{Ig@iPp#8 zxs7DZI%jwncNER*V^DkLC9>9Q7DO2D2Q|(KU#NuRz6ri0E^0NO5|9XHjo-+7i*D*z zFc=@}cEhN-m&m0`17+Tz;;J=pFMr=30BWfq1=0Bc(`a7HK{*wavWu8#1r-Zq~l5NTNnM{Fs6ON+Wav;3jO=8nt zLUMF$Fr^?3)?eL?eE(LNrcaKcdOqEB+Y(_$UoC|(%BC{& z;#;uAVk-JNm6KJoreTqgFUb_<3KMh(kWtR{!ZW^J8Z<5vx4905qLm{-e*R$cs5TbH z9X`gj8Q(>P`CQE&^2!_T)8sL7X; z@G>(CoBlZBAVo8J`T27?JusOK@4o~?7OKILHHlQaaU{L1_#8`xc*>Y5JRSbRhFrVp zkHgNj(x0BA@%#L6lCI?;{vB1qM;X}QvC(J9PhS-{l<-UZV@NKU@yZC#I{JYXvrurI zoPw>n5nyW>M{X+ux(tja-&%yd%f2Xn{=c;}e)wW!_vXOyygIlm3V|ejp-+5WUZNy; zOWA*x^M_PQz z-_tyGzY;;dod_Uw~YR37T=2$?#-*8g7M@d!mM;8 z`v1NL7ovM<2$2KMe=gYWcY?_-6Y*PI0@eja5Z8&JIEOdIphY2|*QL%Etv*VOx2M3y zY1PPg$KuBPJ-E}p0U{Ni@;2}G;`s%?N%G)qC_0}{qvk*1_wF&5MA?>+sT-xZcu@}Q z+Lny{>YK>!HT!YHpdG}$FAr^<6H#7q6v$?-hKb2*A?55=y!FLEvgUXW81muJeL5L( zdVkO%zomTF5y3-dtq$Ac7m?64r$FPwZE@NPQ~Geu6)HLd6@a4dM zxS*Ltvu7=&l^c{X$Ot_(7-s19{zEUmR}X_W!^qwO?v`ghnC@@ zo&8~Q!2yyXRIA&41ZQTZ4Sp|>(g(jsNV*F$Al>L9d^$XXuMv1Hit=+|tz|#V{-Fpj z(hJB>&vrU&$7`xnP;RWKE{eZHQ{qTEuK1pRu#2+SK7Ek=}nfA`x0M|~n zqo1b<(f@ml-1I+-FTK9vJr5<|zig)WADfSg z4ogOEASK`1h;C0fnOt|68a8S`c%~!SveKCD=ya!YyAt8!Z)IL3E}IBkRE*ih@yF{^ zaC_2KvY^%hRRspjm>hSiP&FLqj?h^CK?nJCAR!pP<3+6R_jC z9m4o)RMdMBk}Av4x-AF4524_%wolBJhGR?e5KiLVM0LX?u-M=dcG-qv>V&(@;MP)_ zJMt@0YdeO;T?N!#7EZ@z*x;AVZ^8U<4Gjz`CntnCu8@u;i&L({dG0kCa3}&=My5hh zqQKOF<8*7gx-23;2Kpz4ll+GxV8WS2aKHNky7I@!t*NUpGbS7-b!~UgWNJtQ_TL1~}I3rI170$)EO*!tw|W@_K?)JmKO8K442KshCuQ^Uvl} z_1(TG^PNGgbCo5t4B~L+foQl+Uf^3jb9CEv1P`q~4G$Wv_~F}I=<|YL8fkV3ips+$JOlHM=nUA5lQ#7ueFP3!Ztwk!;lgKcrwe?)?6zxl3STa4bMF7vrC zKsL8_y13VSI|Y`@?BmobdmJv`Q^}7Q z`HZeuTu79L2a_!B5?o7O0!tZVer})x(LZ}xJaq3?;=<(7F?DWq;-g+F_7Ys2zmsWm zX)~Tw%puLo5994+9qgUmJRXKAh?9>%*)#_j6(Elmf8DW0-vWp94y0B3$8gN>FQjE? zGj{e;sI>b=Cl=b%emn2suknPgDOZyX>=c+?`|hFlrZ;T%kN!Ad_d99lwhu74k0;mK zqRA-pF?=`kgdo(Q)b}*n`}9&}#|LPM&vD%nPW;4KA95YnK+t)?Tlh7Yc#WR`@77tv zf%{UpYFPpcv=d>~LNCy{_eiSIw@YjoJOU%W2NT}x2qfr7P@R=_{Fu~(R8pTx#0OpA zicV0)P>M6M8+8060rpILBL5kWh?XY4-Kiqib3q~VLyGcUN9q2YW}%N^g3m-)Ntm8p)?GM^Nqp zAr;BmAkoX^k2^SEx7q}({p*CZ&JJJwmq#LdQ}}-m?(s*v`bp})o(CIJ58uSmAp2rK z58SK3?*WEz{(2+vSY|>OmK}z=02iFyHC}3DW{E)u2TA>RyP!qwVBGScfvPq>r^Do$ z$*nvijC4H-Pk>oNJHjOezyB zsJ5~Pm7IJm9&2a_8M^{uT_#6QR}DeE1_>)931m7;Kjmocw(I& z-d4OzU#o>ex>gq-*l-cu>(g-7hAG%P{W;zI`Z+GTit8h>LVA3Vj_cv(U*&DpMz~~$pe4G${VU+LnQgD6w!KStm@Nsb= z?Kg6rz%lwuyHi`Kq~IhC7yRmTe~p#x{X7VN)>+`-AD@V|Y&s6u8-XcD%@Nit0_pNF z{_9q4m~s7>_-^GH7_uQB6RKBH#U4|teQuB}d~XwRGHWG<8v_AK_JF0XGZY>_ghiY& zhFw2~c|;u?f2E0CS1FL%b4$>>^d+C*tpqn(9H@Ks3mUa6o&I_ElYV}AL~OTy4z#pX zue-#uPu>1A42%8gXzJ9nfPB-J{_5xg6AK-#C3-M z$W`Nd>MU6b7RJI|rK1)^4@$|zVT$Mw^n$?Pm-y+xOzg?C0w@xCD%-QkHD)Dzd9#o7 zx3Hzl3~b@0Xe;$n8;Vxz^XVd!W8}e*GHCecDGl2n3=_1%@m|D4__wl`Y6dOA)#t;| z;6fGKJ!=k_p9^7U2wvj^#n8s z_oCs|0!>URi^Jd<#`xJ*3wCC{$IGTVnDE_7_N6@urmS~_&IhyU1p6jhv}_`((ADq#=lUo zy!M<9_idtA6S7hFzZu9>tb*5*qd~Ml4qtpzL1pn!+$rm%^=~}smZL2w>;6LB>KNQP zBN~;z-Q;<#op^JW5jIGwq4OI9z^$cMx2MAM!il&erU-ULheJl|ACfp_I9Rss;CkA! z=^(um*!684y)b$`34Lq|^Zcx2YNZkQ{dEd3Md9MH^awrh+zi!To`MJEkvQMa7(`{K zppWe5-^rFkabTv9pAUwEQXRC-$;Hfv{Yd=4xAcMMnkbLtM~y(!6U5KAJZJ zA}{?VGcH!crcV=L_$(AStNHLz-54}uX5zs`CrNX1JUMP-35`dxVf{%ZS(MK`JTyC= zj`C^3N4}>>zfHc7(YcAYd{zZ5{7~7t7emMqlWr1lKL{H#9%7F9PEZZ|Oyc{_60ImL zd@-V&^tsQ(2^Iu93{_=k?!=&HkBGcJ5l(#epCM2BuflB_p*X!@CUsGHChj*j7Iduq z`9l?_QR74@-E*u2XD_oBoA8PB_1kdz``AS~`RfXJwl10M)wX~aTC34nJ(;f;xP1jl z(XevU3K}p$UzXV*Cy8s%0)v?6X!zI%7rryXb7v`3hepGLwJ+)U9YfIm@-C8k>lFGp zuSB`aA89}mFEBGN;D@*`Qs-qskULUEroJc|w(S`MbL7@T@0<5jckynFiSI=fiM(WQ z`*3I*or>1S{!woeeLn1aAI#bpP0!y|mAv%$j7-&c`o6Z9$~P^Mc7EFlD`Ub?vEmYH zt++|=)79*^wjj)`uf=y+gYdr56{?tTD*jZfK}xMT8gu?KT&wA!Wfy;n{ijBuhEp!E z>*iy?-<@Q^bs?9X-$;tn+R5W4Ejn#pDttCr1Uq}w;A->|xSo6y?rG|iJCk+@8J&H2 zxA8P+*9Q}uja8VUTt-9pHlUJREax11gq^5!1b^#3phsHgO7$w+dADT^)bvEA5NMu6 zI^&e-wZTVd(1Cn%FDnFJ?kpus8-I}ZMq^c3h+Dnr7rJ0y9t2OfAopE^!; zN2eXD;AZh@@O^2H$-aJuxWWYx#%P>Td=+mD;{k?&7!+xw$d3oEc_9M%=Ur1d%fUnN0d0z zJxJ)?l~HT`Sa>lk8Acx;A_@Mpg#VH~9{LVX1k&rrZe1S^U-Z)WL)I?Pu;m1<{5Bk~ zE?WgT>gKp?SRoA+nPSnReHfNfC)`8yq3&!5q-?1#4!7{ z>(+p6P%ag#QTV*WAD0vt((&D2>Di&(IB5QTY?aHU=|3%D^`A;&wZvK$X5$(IFk`Bqg?hNXkN`Q{uOQ_oZg~$~jj&_vAV3XJ!ov z1y-mv_`SHPZw}nmD#9O14&-fYJ2BrD0n??kA$8nCT#?s97YcoxAg+|(I(ayno;3z7 zn+IgAa}x32wUFG2X(r3(odn6uzr4l%8PqY86V6v9+0I|~YWB}dU3kx# z9ZjWE1I=U+(>uiu{ob>8PTZwKryqxu@Y(cIkT3RC&VVC-GOr;7GJey~6BBKY}^C9t#-(J8{OJZ_vH_BR%uW8`IN_ zWT`@Zs`aBk9bwu6HP;NtxUG70>$K_M9eM(iK8WCSof~S;3&%qXOmOtF71QeQosL|Gg>$uOZCxhsRrC%>`cb;u*je1WqLj4woFeb~4q*CsKPWt`06w9? z;At?1ZaMOv>{h))s{Te1i^UuX`yWN;9Z&W5$8oaBrlHKN$e#C{`#JW=NT`fLrLrnb zBTbZu5Lt;rn2$Jd$XC8n4%X;25wQb12f?4LtF;hRhV&@-u4>T#9CNG*ip#>*&Dy!|eKG z1!TGNCo|AnU*Z z`QA&c@Ov1|ep$~XT()Y9`Wc5f83g~PWgap=mQ9PdsnT5Q`P{LRW4v;A*86`i;Bu*s zRMW=UDQf9Sm|e1+nb-CWm0Y%By=J^estp66_1k<_Y5iWB89A2*voXx=qv3SHU>{YH z>ZeAzXVLmv6Z-0Urm$#u9a=c?94hAA1g?Umtim}1ZmND0?I{&;x?_~XkQZV6o|dc3 z{oNVp7RPN~pdChU?l{B-)Rwa=WY^H@TT=AH$aQL%C)*}h6zQ})@ep$^={35S`UHDqE;rAE1_t8f*qsWcTdsfOcy zRW;hq*rn3g-6eF|@B}(nTAk|U9Y*t89x^BTMTNU+=TnzLKP3FyMwi}=p*ts6pfJ54 zB*K5jJm+REZ%$1|$-B-8Pbj}dWrmb0E#>a>p3S5i8nWnyuS1Mi$UFMdr5>Goa)DJJ zvvvZj|FDPd{$!roK4MlsNI<{7^fT3k#=sH<^bYgu-sWdn~`%*y*vtU#9&^ z30sRx(T9oIeD819*d5CUXt&-~wy*dFJ++%#L%cM8J>L%Hx*4HJ->dXglOFvxOB#6v zG^4cL!)QBu8CAURVfSyi#cVMzWoGpnvbVlpW)`gGvLBPTBi;r(Myu^Ay0Wc`F=?%$ zS%NRDv!j5Xf47bOu9t~wrsSag)!od|`_R0KG1g#KcW= zqAxxkLB|)~K?CmN%)yge&_xYDG#E6D-p<@a<5pIYksdp`mV_aPp)6+3nYaA7nieWm zGleGhO>_cD|DhRTA~dJ-9MY92V_2)}%<<59UlW=#HHcmisb=!Zv*@AFUGy8reQ)@-LwFp@qD7s_ zXt*bc-kI=)ahtoHKI_@Wx;J+)x~uY;h}Tv{r(3u;%x;IT;|SvUy>P?n^B}6 zYev}dFiOyjOSPsobW4%ulA|nLJXF>0*^u zZDxg6?;Cf&NGoqch6wX!VP!to7qu)_2l5mJSvn*TywyC~hqqS|x|HC0-() z)NwS&UI*FFH)6Ls-Djnp>yX|^7h`3zpPoptq0i^+N0*N0vmsJ@*lllju%WI;+0A~T zXz{E)h^Km<`B&D!INLX)>6d;mdROkDgt@mF*WwQ-N%AOi{42}$pFvuu( z#WM1@;O3Y%Zp3jU@g5nemERA?rH+j4qM&rqAXapr-$wpyj8W(Bbk{+Eo1?+qqyl zt<^b-9yErtOT+In)AgP)iAaYP8`?mt-yWy3qujHDzYwX9W+K7dD0ZkQg<0@0i&}q^ za*AJ~M^!c+K;v7anCrR^=z@Grw6tCXrQMco6Q>P`v&SN88QfHKu#w6)zhF=TcTfwWLYVd!G{Lt}6tKq&?syi7C*$ z{y%)&)D--2oCx>xq~OFlC9+QT22bqn9AdrPk*Kw}K}nZsJ~)M3@9aAvPo)U(bX0)%oCl zZ$1tW&E!S(P9jf%9pRX3aP9*hk(AvHg%clwD5It1MN0veb(A0{V#YXksWdFGmn8Bl zuRGjVR>OB*#qy3C#tY=GXJB)VW3(dq2{@4`O3&+L1NOPeq z99hQkR!%jw`o!zwbADg2#h<-`+Y)C4g-{A|s(d^! z=c?f7`r|nDy)`*|W)}FqM-pDTXG<2@HtFllaK&QTv3K2U-^;PvE9L z55Wv_S5Qi_frH^8hjUw}3P9aLy!>D==e;|EbN^xxo@WDBb}5jd*eJL%;~nqVwJ(B$ z??l0A?}vg~4}YTL>r6)ectT|dR}wdBO-A&+h|O;|IA9k*j&iSU&I05Q=giLWawP+= zZAoBh2)^Y!5qh{i1&XaYwzh7^@7hY#{X ziB7dMTrVq5Y<$#aD?4=Y0m1fn!)BXeMML44}_;ZSvaI z0h<1)0v@53K&i+clLyWCPw5YUOakHaxfY~wHUr;Zy$af!B;Zz~zt}<=0R2=8a(`zN z?mKS{2ijdA;8>@#GVkKxSIW@vgCYz&GZ)^!^br*74JAucmyn?u)!6r>I`Fmp0hA`0 zLL;>$AgSqntGDDUuy465Z2oKne-wIx)9YrEqzq3ohY1FvKURPno@v0+vJ3Q|7lUoK zJFtzrEIH6-K_cfaCa7`|jy#+WLr<-QM?ble+}u{M-!KN+im!*omJ@k{kzQ~)-xRKJ zSP0WPoQdYAI6=b#Png{k1YIsq70`e**xFq;i99ZhB}K|!(D}VFl=HM8<{5hU{EGni+;llSqH0B6 z_XG<1>+PUkx;r!$n8F1!o?zj3AM&hVDd}r4gvIMUfX8eTB6iZ7#CXhcFgfJ{Gul6Z z@z0h}f94Oc!aM}delr!u?wSu@c_+fOsliaaDwg!_O(2nne8{N}X}mQ?n_+_M7U*94 z9mEYhzIqXa_k*=( zJxOBL9P*vNiNwW)lhS)>(9m-+T$DMVY`GJSAD?xBBZp_huv}R(t(#*+va_H=asaGe zF%@1`xQNYER*|_uYl-2xBq9*;!2HF#;eD758=Nz-c$O7(9@B>D>hAF1{z1GfGtBY+ zbO4VzM8k$KecUC+Cx3QPytyQS?3`HzOphy*&@wSHa>bMQf1iz2&(wj=s+YiF`vz#F zbQ`C(a=gE1b6}>EH@qP$1E)|$UZ{g6nSO#H5)QU7-Q0}qFG(hgFYhMS2^O$QkOSs! zhotMyJQ9XY2~u7o=rN3fv->0A?I=;OD1fJXbdla>&e&q$;ZuGea}{ z=AjmBf8+*VMH>_QnRBqcoC&-r@PIErCX%9A<{Uq54K$8i38y_&A(s83pm#(U9(?Nz zqaNDAR(%zCz`7f+^Q{6^sR3}n$OrC}oKIe><$+AI27L6Nji6k7GpLwz68L{Ih1g32 zUYKS{+S=#fqM$&MdRm`M+TaXF`%7^4<^aLzXg&;C4@l?g3*bilEO-u^z;Ojgvi4G5 z4*ZOhl|SOrrs;6+m;#=>Gn^NA^0go-hMq-%Jxe~~ad!!U?{O8f&u1dx-<$zon2kD4kFX{c zUp&aoeGqz0Z^qZYFM2&6j)optz8CCmA0&}=tY!2BeI~QcD-oP?lW;k*0DsVYW5udpiM5g|GjnDoFfZ-p)fZYRs z@MESE*u3L7u3f(y52lF1CijWZH((mcwK)i?_vW`cFOnf2rR2Cz&Wzmd4#TwvRY26I zMy%zdD|mQR1ItLXfV-(8WYOC?AQdPF!+Cm8sYHy#mwpAAUgBis<|$*6}wO}Al1OB^a3jNk<9;ObrKa*o;URh@#<* z*Rdr2uON&rh%Y=3o}M{@C(Ys5b@2>oU1UQ{C)hz}$BAUw zJ44c3%B}g`wV-m+Q=Sn10C858U?k>)po_1?n^-PJQqOQ}yk!{3=sf48j1Ggj>VE^qVtlfMT)YVMk1$B4vR*x^{ z)wVd15Rr%Y*fUZ1VQwq9Q9Bi0^PCGegDBEbwF*qa^T^@V_9Xo|$Jg3@1pmg~ur}Wh zs+%=R7_M3+Kp#6=Wz&6$(>r%^Q;~t%TH<6&{%rg$ zLKmNYBMTo81vq2v2)OMc4$YMeN%9t5to33TB=R{%rn5D%ZZ6|(Uvy8ThKI*nuzblU#~n8SoD}X2`8~G8GCmkyykG%7 ztbYkCyR=AR*!Y$>+~RIPwhX$H z@_90_#?zdfaI6CpIUlNU>w`1;F{wJjn|^2>#@YKxh3~ zu;%YEym!I~$nRQ8ly9sdgD3);B+eq~Itp+fPYq7lE<%#_>EeU!nYiJ|L)_H)2H0MH z2Nvr_l3L+f!fa1~=`wPF7FdH^sb@gs$~Z6O(Je6a#)~AJOe3G(8^KFcRf(InEGd2? zz;mX_!29C0z25RWQL;Iv6cc^Ur=3Yx^?!Iv=#&bIu)mk*u9zip<1ihdF7*&vD2dL_unr8sh0 zJC-2lt*}Vc1H7LQ17s|_LCza%tgv{8V`-QR`LWJ{Jo)zn1dAw>g=;2~^p%R_>!)6j zxBe+^wKae{_o_peuu0_ejd;PL2c`J1*DHbCtT3?bcNG}EwFO&0slvXN6i8lEf=ydK z0d4mYaBY<=dAvIk$MJNb{fY>vKUb10vaSGwLpxyO-!y2vcsp5D)d4t37J0{=2Sk0$ z;VNlqV2W46d+UX4UMxNjx+R_inRj1#yJoLy-E(#gK4SC^4=wi~TelgLs)Ld+RsJXrD0_<4nSFMCP9c_kCKiW9=3^W!AUW5ZxPsGp|l_&hDwCXHp``=v_j#i3W4~ zpEX%wu^Y_qA9LKa^}FED$2R=NxCX@82LhMGb{w&LJs1&e!P6Cv;qAHEKzYSYoTR^o zb5cZNH?7@Z-n@K#uqacIw?qv;pPJ`5^FuH8Z#RaXRwnR5i7}BYSAi#TF5r!|Lagpv z58haF-j3-ZU@jQr#SLg+YI0TZeEB)>_DwEmkm$y*zP}U{5qJ1D3&D_5Gcwo07Cw5l z3VIoHdz!xm0biWKDC!5FUg!aXFBkA&E$0jO%yEn^suk1~>=BqPyN-YH zI8xS1;QwD7*fhZ(Y}wR=vnOUbM*p$}ZYP#FChc5@dtQiZb=9N(q2P&C}>OT)vi9l zmbsDw1KN*4aXJV*hk21=hq3mZN7!{F9~^VI18zUb!C&5LK^3{Ppf7!ZCobEKr5xVj zZOx6ih2M>%UF*R*s6c9&Tzql$OT0ke3T&OKME)%*1OeC>3h%PWc^I?xHW?mHSL0RGm@|k z+0NT}e=_JW*xh1@>$BG5zxIC=^zt1ZLYJFWeNy8C_@aDptCy>mZ;Oh8c)Ec^-vxgV;^+*)A zzMS4}g#|sU;02Eu@_e;8bUXSAcNC6dn~iNC_N^*e9nO%j0VR@JGzF$_{DL*+-2;C6 z`oPB0J6JtM0xBiW=tFJoV_w`U8c|bBi4)W>KS156^Z!QpXuOi z&Ii1{To*p}y#aRDn35y+9$@t|D@kvbKT$GW3xm&h2&PGME`HY&E0RXrmi>KRBcB3XI;a34`x7v9jf5K zzn5U{K3%X%@)f=k(u5b8X~L909bo19Z0}8%-S3?XX}%N4XvR2*e?(-HiSPl#_(S0ijqgoAF$lsKcFF^ z7Ap-%!G(&N@W3ZoGSUIzy#OU5w)QjbeIg14%AKvEvcG_xS|!jj5+i*k7+)03rvK?{ zJ9WEhFm4su%(+;9=I(OkHchaXzK*|%n&J_oH>s2LR9i{kv{lgFK`ACOZwHl~IEz{O zypA2+>x@ilKB1SMeZp6z6)xn)efEx69<)L8oO?F_WX1EP#Bk+uk%?KG-e(&@V6dFm$h6kU&8CtMksjZ&ZT zsdM&8%K!G7?MU@Q!229|8&rbJuTJK=swU|5hit0dlFV#r&_KrNZNj&|PB8Iv8u)+R zTR0zC2-==;npv~{5p()&J@T#WW|AGcnVZ?|Oygo9Dp>lSZFu#8F<$+M)p_@r@#OB; zeIo;ejyXF~{qiC9bD^1&Z1V+Xh2bIopyWQ*@T!2hd+uJ51dU6{&q_CY?bHniP3tpS;yJ2LFD z6jgS%eJiq*lt+7AAEM)2=E0A3qhH3J(0l)tphL8RU6h@Q?tDsSJ1>v2-g$j=VR|=n zSiXQ=VcA3{dk?V&kLA#U1<%-k6>3iY9@X?D=Lvkwklm2b@paZMFGR|fPn2I4~M(2DP`aB#>j}5)yd#g&L{d)@71wRzq zVs2Hl2deIH@Ao)W{p^M^mvm9d6NJ4~cnN(Q-h;-Uy`+^Z>ZndDpn9gJZ0)jh{3Qj} z=x@C(b8Uk<%}eE&a=!ayiyRixv3XuGOE1kZ-gWjH$ z#ncCnGm3XL__xYy(Q@rvw5}%=JqSKbul32&J_kQG(q=Yd`%mO}$~MV*dk9i%J-6%;I|qhvL^IW zunW~M+0GPfJx=egIEo6BYf;FU9K~GkP$v2{y}GRv8MLG`QYesLBqNC?z3pem#3E@4 zPm*b0U5r?_t zBsZ4ot zTvs}Cu6xc+&;H<_JQ|A@$3@U6tsSUi!8nn;?v7+lFVT^jckH=7u6HF6X}dT&f%9iu zAjkK!;PHcYh-a!p%f+kd^WCEKX~avRYkWI$Ojblu7c`j-k2uyCH$VTgKpd6lU7|)8 z_RvKEG3eELZ4@%2hJNsw&p*8+mhO-~MaR`W=;d>s^!ZVDRL8xKny53ZP9LB}a;4~m zPb->!qzqX_6(WP_#>joT344v3H76}?LCcg|Sz%=<+I6av6>N;Boy{^dRoocqyh-9R z-Deq1Z&S2l=_=H9oa_|vF=3-hg~e~*BTd^#x+yM?|Dt9a-{C|S zO*rtB>$tyVrc7Lj);6w3^9py;mPpR;ej|&u-Q$I3OdFvSXZh1Z;Ue_esxcEy&`|+)7jL|Z;=0Z-D6r`-9~?05~um= zjM!2xJ3kgOLN`3frglFIkc?K6%A;ljLLXF5Li^OsY2DMgd`-I#6lFCdcD6ppeVE8D z`)!I0QYO#`C0kKO-e>f#IFM>YRM189DpAo~L3SU26je^gUnDmZRbc)O4 zrOawzzMOx>Y|OpJ9uP04*9PuUxBoI&|FCrG)~iE{dl1s{&qrUkwlMvg@@>5rS8@K* zsZLv_1)+Ur#mt)B+UV+eGsdbtgAHl=f;zt*K(*W7GHbR>MVo@MP-JQfda$WQ80-HO zNz96(carv@HT}QY;wi(__I(*+d430?&+GW>45irW0#Dj?*b^-((5C)D7E~v9KRvhe zFcYAdiTHOk*q*^*dT0WmqhEdbSq19M3zv83I@i;Ac)f-RlTx8}7etv^G>D4-254hn z3-u3GrabPsVg39(lC0Rxqyqu_AxRVE>Iu-PN6XQKEE{?!G?4o7^O}yX$YfbCTk>=s>hZbHy0oUVW6yT;$G1t*U7u=^kN;FAE94s0 z)4b2*o}Le*-MUcy=WurRYJawD-9%<=q8Jr>@t4KAT*q8inyIX~JwuYmqlGnxN<}5?bqUvZ-L;-plW5Z^?szQ!&59z6-cl`9$Gw9?)dq%?i5kEAs zlU*KrkvVKDppV8B(O;K-K8dU2hvnp<*(wX@);^<0y=YT7p-ny%*V|en8fbW^lG0xBdjPwSy>loplUnvh7Bm`+f`I};tj2x zSi?H5x<}uANo5cGdyj5S+0R}bIE~i4eayU&nUAb*6rkS`rx;Z|ae7qoC1cV%!dO4m zr8jm>=i3DivD4g)oE-Bl*=sL1(DDtXRQ~2iCUtW(-Fa2n>36~&v^stQ_1UnSrtpnu zn&KyBmeqPxyzwF)=vG13X9l6eVJ7rs(m8xOU5jZG-@*o+TuUupo@Cp-SD+o;!;H54 zY?QYxj@_GH#>BWlHs~Y6)Xl4BH(Iv|Uua%nFDt6FZGDnYe~NP%23L2gAjV_W%iG!O zqh%aVshX+{Eu+4lwj<+LBD74sk=<5S!k4M}#WA>4kz9o*TF!Md%~i25R-e!6eG_TB zk}v9X{qSDkqdJ{-Y_3JUx=u_5*H^Dub%#wIJVrN7EobhHMPl`qGX)I9Ng7 ziEl=e%eGOYt``3KuEWsvU@98gBtoY(H!{F)FMHK)H#M~>5Zc_{g9=Vfak~91iN;>K z%WjIV;zyqpYpcBfA2K-IiH6S4pj$Jf+H$9qAd9W5(S(hZk?t3w=3XUwedIX(Jhl(z zuA5FL#U-NUzIW-gm>DR@ww#8%9z@;Ov{3Ij#~y1xPak9lvadG9vMu?OomL-fMEgzx zWXEx>lkMKnUKKY~-+ztHukWGn`rT>JY*AEq!k)@l=F&fXCd|*`S`>E2p0YE}P&=t^ zHdrT!pYk=B-WuOa{ip4r+P}B5%4@%~F}k9t+n@z4uFPZ4Z=Owq4)~(_k$ifUa|>9@ zMzWh$i?qpX_`teyoq*q;^I|96`NWAftdQd0-#fN`MF3r z)p%u4--1olZ*c|k=&WYDrYo~tF&7 zsv4DX%C(L_H8*~vwhaVH{F_PDURxoRClj0=9WkRjD=bi5?L1+j(MRTunih?``-y!? zs`-P%`{;|+Qf(`>cCf>i5+Jrr0g3cEG4*q_nS4H>0l1I*{mIrQ^T z8{^e`k@R*anC|jTAe}0re0^x`|hAk7W3$n zm6@nvsw1ksm(OU}PiGW*CE7Z49x?YO?q=AWHAqi!9{N7!id?@$py#%FaBPz<`Zez< znYz3gxn~Q|vHJ5UqB4M1%qyc26ROeFf>h?izc*}ASp#}8-Ytw7xW$HDd%zyKQ-pjQ zpRqE2JJ8F<-E^@R*UirNLxw(*bmQ<2>in+-*z@DOX3`#& zZ`r3C_341!cN$v3(74X`tXz%&DXhsr|3Z9_SyDHeN0DBe zJfplYm|i@QiK1$%QJ%>Ml6E_o@}r`K_pg}J*W(tf#ZHwrBW)kL`ld1Ccar;mqB4uE zfw5?d$9h(M_bc?xJeiG?ilZxg57VW#+ZeL_9~$&ohaAQ-*{_fFor(|aq{Yuh=;x(1 zLg{VVBg)ZMikWq6sQj5rB|Gkl*d3%>}Yd)JMiETiuCS_8YzRPU+ zJH@t{Hl67CdqZTTe1qAMU4b5y=26EB+o{4;%0|lFr*(g#(3UkmOhd9dQ&P2g3iLOUZ zSLBfoY^5D00=DqwIkeB980|7|Wqlf_)3^B=Xml^fDBcr|)H^j$e$@zbvwAsQR-?&$ zxfsP<;_|Os?j|7J?o*8OyqCzTZ2^8U-vHcru0!soX5#-A8WM(c-)+vy#jN8Oywbh^ zEMK+;JYFJ)Eo~4 z=5trwF|a#ePmZ2;A*WU@B}>LR|J>E@*zK7H`Fy1pUvuExfrsqji<~9!$*pBTn{qSq zTU>5xiY-iVmV{RFdhqXn$ z;Nz-HY~Bo%9&0w zf$Vd+*e`u2UK-iNb80SXeKau#pU^JBHNKhHm=vTsI0}a8y4e)RX6Y&9h=s(DRW>%7lJ#Sxg6$m zOBfS;3mazhh}RT`sLZt|qh;#&#Z?KAux1MBaqxH4c~e00%yyA}JwNjJdp4O|(~WN| zx{v>8I^h?Ao!DmaDCobZ3@xLj;OSg(_*d&2=xUw?eUm3Z^JGJ4>UyjtZj*bw16<}Yq|Ag| zFdxDCS-$v&k|f?A=z;f7el0i=AR`F3+lYB@>cNfhOThj9eefu;1w88>;CcS}Bw$J^ zcykZc2>yPR0tx4L32sk5g>5YSaG;h5EIY<|;w@kEH1d_9(nM`?Lt2X{Rhg6GCtAdl z`+fwMjR=s@Bdy7DB(i@7wlX_|`vzx%WX(@FCEOH0`}qV**--58@fP;Kl?C$Mw}WW8 zP0#Xiyvjqz z-M)6D@vjl-Uv5XNPV1193K=-Ay$h>6e~B+_AHdyXa%A$U*LY>_eLQ>pbxb>#k}m5= z;#?O>vXnweN|_Gake>|{hwUJ%x)^NJz6rAMOYm{pbKYJH&M$w&5<7V<#xqB%1qM>v zarg09vgi9Eax!=oG1CkoIp?EsopOm_zJC+`vO*O{50!vr@2>Ga?my*NZ61!(;w_*B zm+1M4WNWo9Wb=)no2m}X z&6R-)PWpJ@wpcM%=ADh|1ZS@TZOx%=l{# z&%f7#b_HUN!N;tCd9yzrzP6DU!_886e3Bp%$?jzN`yY7TFAu1g$b*Rm<(8>AMvwB~yDOh~ zerDxh)0&;YIY$E0i7&y?PiKI|!mq9Ot6X@)B8~VwEdurjrQnN1eP}zE%he6AfLwhF z*6Y>)x6hA(^v&GKwKAQ3zU#d|O+}X?hev2oN5p`)&=un3DYfT_!Pv+y;*Z1+JfBInGo@_keD@(p6 zI+7C-@9>PZ-lX3t5H6|pgTV_D;rE-~aB6%2Xw6xPWB#hZIRAr=OpO!tK=WX2*%CO> z+Z%>yX%an)cUUE;7)UjhV5KLCKy2b~Uc}=^c=qj`*wZl)=MRK|zNfVSKfD8Y7rxsKoT7>w#7R~Hd!|B@8CM+7?8$^us?jhdDjDV$t$`uir;)^rN*ujgo@4|a$B8k#kAsBCNb>U`Vy;D*+YxKKFv2Y??R@ZjG%W2kDE1`>L2*l7hxgGUP{ka_xs z_)%^HcKKBZ4)}7pznNyd7k<-#!xD3RS3e%e$tb`@d#1t6>A!*H4QaUV_YM-UfXmn) z+fCLtWs$3^Zv*@LcR_vLaUc_42OL~^#G%!M#D6|5@NAk&##X!nHl~nEKk>nO89mq_ zHxs@-W&vC8Si@&#mN4?oJdC_VK=TrHvUgn`?(YEb*FFRIx+w}?I5r)=i2Mbv-5&?! zNGjMEL4ZMq6!~>rpLlY)nx1wpmtJX%6Mwj14}C2`jCO@UW@Qi#ypW4UZ))J?C;7Mv z#}nbBwPe)KlXyQ)AVy!MNhs$)wknh+?bQmz`te36qZkicKK=p$@!l{&dMQv)`XqQ@ z=nn?RwegMF5MGL4;PDuHIIV;S3s-YK=;gNotMnL{PJRSreFwmUo2S49#Dif&3UD14 z+PNO^5v&+>2e}Fl0Nrm%Br5|*kgY6nIyaZRyYdX2INl62=DxyOPt(AEb$Prm7hbh~ z4>=A(f8>IQ-xu&4i&5-ss85PFwBW?YKX_6bhk)U`ZA4mXCcu-t$>dH`A&7nvCvQ0 zeW(^Ju37}nY~{Ss{0Ly2zTE-LsN>x{Ylvlxn|YGSU-6jLu56mV`ihn?o?>n&C?I>9Bx&vgpPK4s}lc89I8Z6^{ zn>Ovy6Xrn^!$2ssvx1vGz66^cM!?jYQ{m3SaUehX1UP9% zflqDgv4+(l{6sbe=SN)S9l63M$mA=pCB(S(!>hfx$V>u^dN%Op=()iCTOnMh>H=@* z+ry&|AA!~7O7M8L~4GUlhh#-@|YJ$&SqJVGFAoL?tRMpSYZvaH$LWx$ZY~0 zXMWm`@7fLOxoT;S+yX~M`Cc65S_>v!YQ+UCWS15&t#-Z0w-L&H+FT9SsKCS?o-ZS9FdzPfdRRKtgYLTl!oDW!Dj3n59 z!NX!YB=w~YaelE2zZldfAWMh*DAy%tET@p)+_O$!QilXAH6gQ=uV6SymYBP1k(2;K zB9d=OW(DdJb$>^E{Cg4pXe)wkk4WRy^3A-2;5ED$*J{oOEREOwMDTZq5vP+RfwOO+1$!pDqqlt%D#vtpcs#f81|mJQ%Rv_?o}GxpQYB#L zB?YL@b)@!Akbr|!j%?~1#uCqUiC>Wf8U1NT;s+*?D>C+E#eQufTL6g?b|Rt^xcs&b zL*lDrNtiH#)MQ=89ju176}H?M*HwejF^zdZ(sM&7UgZwYCMv?4 ziyYt;=mjTpYjDzEYZCgO4aqKYBV&Wc+b`&zVoZ#wvi^;Z#0q2OXl+9aSG(zUr~6O^LLMHsKE>KrovOFv|(zJBs895 z4sZJW1=D+XfUNn0yhBQkaG2zQ`tgH!hFYiKrr`w~@KhfMTyh7ZlVrf*E_-mmbO-P2 z^aJ>S$$I>o{e(wnEyHu^aoRL z4ll23~snE8-KT%fsKD# zVGDmX96!1SpHbArDUvJMeQ+{sfv&!Tt=S6GV@Rp6SLM|XlQAUHB}6;v;ITjpNOCA z<*YoWn{+Z4KFWa7mS>E;X$RZ5*&0{m4X~5t46*94Dqdxzgr9QnF)`-@J3DGT{&j_i z_iwbv<>uDdd)NrH`r3dpBm?Y+ErH$mG}djwH}>LLHCEl4h`3*KLz2h2>*2%AkT8O8hK zS;eD6JWY)sOqoClh<&6$Y?T^#`~D8QYu^s`B(TTOJ%V*zuoEn~766hy#DQ`Y2u>`q z1N&9Rf!?WS88G(_6XO1t!S10o4q71-mf1{#93!HG#5z=4tZ;OyibKsl8Ura%3`j2RkbtmjSz{lP=b zyV)zj=lT_3LqQODxPCRTKCJ-U0wsYZl>kw9dYDrhHc;mBP4w)f64_FJ5{1OIlB{bQ zG{0~S&EKU?qNhAUmrs<5s%GWWN(BZ*CuG5nNsEw1;#By?CXm)TZJ^rKF0?pYl}bfg zkd_U)sBnQSY}oe??kF5frMJI;`|Izb6~U@>w`dQMzMz2WIUdD?KNPL@P^Qv!GPZg<9dBer{wjVVEo_=7NUalHC4tDrbs0+ghUvRc_u$%bQ4}U@ zq!ovP>6`DG5IRU8FPn)XQIJ2G6z7V513j_J%X6e>Zv~t=Pm_G=(t{F_8%e*XBHHp- zmR`Gfjjox>WecSaQsu{z^iQc9%3Iw}45h{h3^lZn?2$P%Y-bp9>dq7CbU{>qY#+aK zS`V7Lo51vHDN)qk@0_!D=Q;*$xDE-eT%a)OWgSSmk$o>&rl&`AXoDeaU- zbFPQO)^T6?dHuHNm1iQ|xpSE8KV=SoYb1-b_N?Um3oRnWJ7v@^_6{v`(?L_Z-;=UG zZm2BTg(5RK^rzr6-CA2i_s!3NCuf!;jgF7-rdt!5tvMbpdvgX^s2ZW-gn4M%U?&JM zzeTRhb*H2A52FpTWytHMthkXk8BJ{OMsh1;km_|a;=TJlJ;bWR*pPF4p+_>6^(duT zfdaa4-6GmI`x;Vo2_!$|7ZIs*o6)M~f8^;5UHT#nMzfMRS7#BjdFo0J z{kI7{=(vH>l7GYYy}d}4Oo4-`_0-&3mAa=5k+U;Y1P`3!MZUdCqJn#wR5SgPD0Ybh zVorU)8EZU53QzBoZ%Z?g0q3xJwfq2FLoSmSpJ&qtibqJ!tfNHP<{)kSR0nPRHQ{v2 zal|gei0oV82g@De=z?8|bV=J(^v2o<-ReGxl3%-{j$=}EVEZFc$s0YY5!3~5-0((C zPN8(kK}$ioMhKbG?2Y3&@2dRkWNJI+CtpYYGJO(}fSAW+)I~51*DZ*r+NVoN@zXQN zp8gZ*mP(?Yi0LpXeiMINfIZ1r%W>Xqq{xNcHDsopF-`wILhk-nBTR-J(Q2-y-_qYf zGQz?l#c4F#Dg|w6=A6w}pA#t>K%<3*Vpqp*%KrUHvc?=m4P{}-&8CB2vG6qgEX_G- zIp&j^&VAGuD?~2Ce^K6!0l4~RBHGiVDhMvo6c`?^MY}(yiE;(;XwRW7L_`LV&b&t` z=R+d0KHf_oyxs~-obyEANj-WnE{*CB-9gtg?dVKvOmEB0L3J)^s>-$&OVHsg3lmtTQ9oMNex};ji;*p$7pw-5rt1Z=la z*L8@vtBGc$FXQJ)>k4uQbx~Mx3YxxV5*=_{fr8Cr=;?tCc&vl?=$JL$>YVr`D5vNqvdGDqcdntT>>;+5QC1IND17&w4vO* zJbsCave+`|GHtvbgL*Rdp#T2F)3>h&VET&Vd@YOdw11f~4XyN}6+7q9EqlJ8VZJ`{ zljtUXv$D}O!&fkSYAxv+Ng`JQtnk`t&r!0BHjKEHK~Cqnh+OZCr8zR0Xm3aoH1%xY zXY&~H#Zf}MYppR2GLD8nBz8f~`RyW&!#0A;u~Bq4Dur%MZxDa}RdmZ>89m`;K@vCz zeZBsAq%qck=4b}P#NCtV8Gm~y9g;(bIYvuhudLvuu!RPE+(O$Qe1JE2f6%+q71a80 z3JT=>TGDsqZfZV%oNR#p(lm3E*Xzt(;k}>#6jcm$b`mi%nCL|= zR7C#UbIGx{U*OYB0Q+uc()wqAsc&~8DcNO4k6ajmV|RNQ%>2SzSTDxULl=b;H3R_-= z2A@=m-2bQ`Nu^UTuGNd*FzGS$k$#3&_g+Urr&476DF}U>8-iwstDepLM!Hw_ejbuC2v#t=C8)?H!;T#LU?lV1` z=8L@ltVa=P@_6#vT9NU}`{>{$buxLKBhjYae8`VQMxsQNu=4;t;S8xsVJUr{zgw{5 zhrM9=iScM4!;Dlt6;OS8pYOFWjMm^dq?mV_+8EA*-^RJ1^2}54%b{|lsz+(@unjez zTPxZ%?=A6}DTkJ)RnU-tXK-NORJ52Y;2!?+p2n>_3}w0NVo|%vaoBxD;n%(N*A2_g)~Jk=!Y%!NbO1{375A< zYdSU}ecz*Kjnpj^W!nNTk9~}?BfoPw!ALaJR)PL=Zl&Ft8%5tYeMQ#u?-TI)EkAYB zMymN%fhM{bil1JwfUz&;p~Qx4RQ|UTg*kT-jmO&PKW`D@o3x>I{b}U5j~{ftbC($8 zC)2qcBj2d(3o2S*AZ`l%N!n*jL1t6mqmjMi#j>8~aj!`R4*6mU`GxT?BboCU)F+ao zcf3V;!gDm|xDMUXuf#9!Q>8h7*CHpaPTK#MlAWU~h_1B+NwqEEvYnU7e*I|Vg|?zI zYHG*^T!GqtduaZae4_8wi4_A?(azdrbbV_PoV4Qx+`Qr{+Lv{ezW92WZqrMrpv#Ev zA55h20YVhCO_SD7w4yG5O;M=lCHiuQDm-s96HXk-qY@X=Xt&ZEvdQ-c|H|G>lKv)# z&Mj~x!{@e>_~_lVecLE(IFnA5uc*O`oo%3dUlqJ}FM%{akLEJ1d1#ti1JW*$6o2O} zhf%*+w0`^RyjdTE+POU38lG{F=4vAqN}o0LP-W0yEbTMxR}s!s2lokW$n z8=<}LTXA9}k%$%Nkd7=UF1nbumJVqjTuy z4fgQ!O(XHGSN!3I){gpgUQ)HjS!e=A-HVIl+;Vuc8m<`eE9fD0t+AJ`$f#qh~C#=noxF zv}}hlIxDL|K2etzmtCDoy{5QPo%=_T`|ZPU_Oq$zuv#|Vb-o3Nq`nt z<3$=$l0=2S8|bf@PxB+|k-GO<${p%N{g?ewj!7AMQfER`?N`v0r*BEV!ymLkznIt- z?4%!K4aMUs>&TYx+)B7ulTQDEVc*I*bV=n+=%rtc#=JJBw^UA|mVGJokQ#y6x(A8G zgnY7TNSR8w&xHNYi>ZN6is-~0+5hv|P~+lO&{{AJ8IEwAZZ6+5yNjV}QYxtY(JERR zYARmgS0GAEeMVP@=-SCpsWhT(IZUs=ibQ$ty)#=s6zsQm1smMWZF~_7GC(i#{ z%wH+D7(EpY!1sz*QRcP)q#u2SIz5<$WUG_}Dl`?oUMb?3L(AYnR0{UVRC!(tDC)agvmjtpWTJcJ|1-Ix>|a}u#LDM zkfC=AyJ&3EAGqY&X7rc4w&QO!!!oOL#H;W$G|z28{%bVplY6SPv}-j?9~_4=t_`5V zbuHvsMmSxi@Bw<6%tHQ=li)wT2{QXsN0Yg(K+Mi`ikC*C6OF~>>Vv)Hv9t?v8sL0_ z>Y;RGYz*fy+d^YoE}_&2X~HYMhSD9X$n+g8s5j~u)Q#sMx#eGw#*I#LbL?C4YE~0k zw|@mn)p7^t4k!v5AI%h07M(%wua6N&lr+#2?~Fu?dvqidmB#!H$$ z@eO@*y9!O?a+=}Rj_5)CbTt0V2p_)Vx?ARpsT7yRcCl%K?Jp~cc)}}K*&dG`y;x4o zwx-kEpszHObECjYHL_}ACNe7659=hBh+Kub=-%AZNMd#bT>Hm}pWvc`w(p!G_?dKy zUexoY#$(r@-218m8Hr;eMJ8ADS^6_H9xx%x%YPBk^0oBMnNsv-oi&+|V<6g;VIauA zcay);vXO+_7}1O4J!$iR98EVDkyT6@eX!aEe|7LqX|57Cv_Amlc75_+wrB!zb$u2qE#T-ZsADHcOOG< zZdjqNh$hsf{JrhuGsUR8`$ETfhtLV&6$7$ZPBFaCrnOLC=Di}>h zQF@_7&0I~~&67nJ(~D@>_qpPE0h;9c#L0Adf1ju{W)V3gcb6V{hS5#Ax9Bdn_m1v% zB;q+34*zcEo8M6uP$cNU59Z7XKDN7uaN&dlU`&?>G1aFaLxD`v@B3VaQ%50(t#RcwZ29Y>iPlc zOev?a8Zjv7i7~pr@CpSN7`P{U25Ry@47G)S>DvGY8nbl_irdi%S9Dm=N80A}zm&0J z&%)J6Q|wMG4y;DZ&~fDYL52F(XTpKuY?4z^j%Q3OL5KTw&}wl!9COGH{dc8+hE5X_ zndf~->0AUoEvrhr}SuB>B#EX8Pw@>9c{j|kdn`J z(DSgaApY1Dns!(jjc(hBJO(4lJJkvFSt-{m%s7cUXa9%%3lib|Cyq3)v;}?7Y9a?t zNs?~O+ak%YvT)%mDY0H=9<`|0L+m~K=-Kz0G^^Ji9n{gF)e9D(-yfZ+{6;C{#H|vaBq1vF&ZO}L@$BkEmhj+Q--gJTbl5^;b& z)xCC(RI1JqDawh+f{uD>Y@9`g{;Xr&Zn-g2?`H8FPc*RmLM(yr>Ii1#?>=Gc&0Z#` ztWQWwrZcWhf$R}s2^+n_mr1bK1Vlq5oK(@v%A|W?og?k6Uq~i%K7T4!%`(NaBMtDQ zQ?qgHd23+THwJw7UI)6Pbio1Qf`6rc;thBvv%Zru*oaS87&*;ZU?V4wpU!ggh&W|@ zIkTHxYg*4t-E9j}+%2%z+1sq(xD*eFrLby$EjwISz&=b-Vt!vK<-L>nz*>htU_&RG z=uIPP&+jwLSZfR_e3 zc-hlFwkqQYFEQXTZ-t!}yY|yZ)}gP64fyN_eqEHq&inqb`o4p_>-&BR^CmD%Yu|k~ zZmuFW3-JOaLXLSRhghL~Dl5^nTv(lBh7XJ`!4*}kt#9pUWQQt!5$L6BwxZF?Fir!4313Y=?1bO_p^N zqx8%IH16@i+q}bgOM?o8?@gtduWF<0#4B8Oa&3$-sArf}lTK&0Fbr7t-UK-R(+ABT z1$cCqGhSNiiu-#b*p(hZAkAR|YrDgocT4{tZ;aesruk$K`=ZZ>)ngChH>2iYwaW{3 zv9>K%aDB#9UeO019E+IZS8>$`S12>D1;?0MQ-AXO!z-8vQ5K-%lQ&ij|HNEfxwm?} zb}{2}buV-DnjRbSY9-UUdn10_KMN>!`hr^?w!m#+0;^Y8!b^$+xG+8(e_E(1_Ba78)X7X93}qzW7z-5)9`cgZ7BI3KE(=$*l`}sk1~K}cfxMQp zW0|$$wE$Cq_ui7iGv7I2JNs-#!RrR|zDOHX2Oj2`nMvXNz0%;)RSVp;cr)I**%EXl zZRWjPy$CO7-EjXwZPxyp1oLp#OJJF^r|v9bYbLD}nmRmSHCER$C9h(H zK8Lq6^QTAxo285JghC4}X)PmM`F<}W@|lVG|HfdcUJd5_y!VXV98X5-)mmng!C2uy zgFdsSC5~O-smbo^ZL1k_@@3R-ZexFkMl)~kaJ)MuCH(Dx1kj5V3zt_afI~LJj!MbC zxaq%Y78{f>d#9GLPh32(hP(q#mHf&C=W75}^Mhd3k{~c}(g*W5hvWa;JJ|~z!>qzk z43G>eWnIQ>1ID$X;N8WkSaB!_BYih;fz!XIjKkm> zd?0)qs5}z_B+tyo>TBF_lgf4=r?Lq=*0RP$^ABM!HUNLzu^Kn1xPx62yutP5{@gs( z102!Tz$b3F;-L&z?Bwo*)$Epl&C(M=SD3Q$K6==>T+;@ zBpSG-9>y|jeehSj76jUH+_A*1c-FEY+^PDJF_B#dWd0lilZ&>2s_ea(Hu_+J!hg8= zmKRQn9LH9_Mqq{bAa;MZ4+l0xgH!(gz{hSEK02}kI|&zo{Pn(|H7p9e+7<%B&HeCX zw5{4-~>Zc${bxc2u@w zCRO+VNB>>eZ1GmSXy5@XpnEY20wBM7E6`C6!O1_vu}z`^_}X$9Oe)y{ z&S(XL-#)YP(R-Wl66t^Jv7Za^eMup^xz-zmUgTb1>kvM1a5|_f*a~Dy_hQ+`J-D`a z29Vmk9b6d*2kjhV@v?X+P84p#yNkkc&&ePh?zkBkw>bl)_1uhEZ!>mL_5%ub^MHm; zB;In-7b9;MFk_D=Sh#2xFm?+DYA-*sZ}+alr&kB!X?B75(JKQ`{oD=6`YZuU-faXk zekx)adxm3&h!LU<69ZCj}G|hpB*6Y+ct1ShGplUT#F~I@&H?otOVy@oMu{=&cY9G?ZI;v z1mTWBV_+iM5AJtF;7rwE?7;DMWu*K-)SxsQF=&au@K!KEOHF`TO9GoD(gqPJgxMe7 z%_}gT$im}PcmUmHdJft#rWPg4@c;n4YK)mhkuGfcOD$lVbf(63>f)T(=GI$qxes!sG1v z$Q<708y$?5t1{q?TH!}8F7xEyU1XG01KB~G#n#TqV_ds-;JaG^4qnX%HDZ9%|LNlP zPuJPugaBq+{|7d*;<_*~>?)J3K9=2L8Nxi){=lwG&vkUH)dFd+MeIr=U0_$ivFR>q zVe3X&pt@ZX@7$t|wPze<=5L;Y|2t%chaT_118GIf+FU-K`+6DJY&rqddi-RMYuN%} zS-jBc+e3!0hM1jPR;7kF1y3x}U;~PZm}6#5tjWdaj7-rzrn(#Aq3yH4i@h?8Tl;l( z!tUp6LgE9SfwUdAkh(719b*Q9=YM0ndSZp^2iur$Z4207xpLkilNC7Y&skP-vjCio zO=k2j`4FMg zhfZdlM<&yJ;wC$(DhL?*b4;^7CA{{xIq0d%5`Oz-#9mx3!_$d=!e-~q#d(=(_~Qjh zAYIP!f>)T~h#ym!leg!9&nG1?<3)J2ztzFrxEDP6@ngZ!@6O=+`BFxD5VPyvOaeuL zTX6UVJ-nkx7c|||#nCs098DL+Rrh85V(2L;c6EX?Hq*7i8LX%kkLTQjcpawRStFvcpC4eC0IIrhiiZHsEGEvtRaQ!=be6L8cD!XAKcsefmoaY{3%{j2>A6Rl3mOLa;Lv}VSNMjPKNQEh>-k{! z(Hd4gWIoeA|0!enZ3?(~*H9R&ERXxP-QgL3y~7^ao5XCmvxf0^^%g$0)#%^Dx9W~6S#F-oW9+1#OF#`VKi z&@e?F&#f~8wX3DD+MhYXFAEEqSADA(!x($!isLo5VR9gQ@}E6+-Jc?KpQM7XcqP@S ze>%z@V1pDzGb_d}V(H9v(e-mSd#A3gB;;BmM! z-xhS78et=bW`bEK48ej3Cw#1}jopx9inqg^%tDX*%!TrF_GhjZD9;LGSS@IpE?eFyol>O#Iqw|!CW4n<7m%3z?!5v z@H}U&2iKn)gJrVKEM32d*Tpe@rx46|j9VlOb5{m0Z71VDuU-l3eU7pnCm*sgbGUrT z6pn#LW7&mwcClwCNPx{s(?F6dm;Y|J2B&tI;Mx<#tcPa2(9muI_V?4m`#pT{v-+7p z|5q*3QecE1+?EF1{EWL#xx*V6n2ZBU4>G?W9cS(yn1_2go@shYJ~Mn)7O$LO1Ol{8 znfk69Jd62?`0tOKY~ACfxHw=Bmth}aK%ym9UQo`|1j&K_uBEVM{O7{1X|CY?MLn=^ zxdr~9cAv4nw+X;E512M(IV^tP#Td3v1=M>nc(#8j?$|K~?-U5YZWDRnq9cvJe7VW& z8$~$jv>DDsbHKiBm)To~ZnC+Xr(T!%V*s|K9b%g01>j?IF7L>Obhf6~7>qKK_(Q>Y zcFE2j$A$gsSniWKuDH_2=*nRx!{|J_KN4YjVFO-YHxF#(qAL2&A=tHt`~K4A*TJVu(CX(*>Qz}h#{LHDLzl4&71Xeo#2~x9N*-(!Tw|LzrLm%p*EPW> zRl%A`NsN)A29_$m#0fY`$w&_P7qxMY?-1>T&o%nA(v+Sn{-g2^$`E#j+ zxjgLzv+wJjZVP#vO}3WpVQf;_rusPIqtxHlN#6_ zt&jZ=%L2*rUUs{iGmi3n$_QgGviDjHz<~V^;h54VY-Mei5Pypjas?lj-rCC;v~Oom z=`vupLK0KCOU#;k$bA08$CVDm^gRr;BSs^8n!a661Mo2nnSv)gPy(gT@3Zn;mm+sTJL}^s2Xbe=W4|W3v%?`j*qGVzjwgD&nbhVK zUQv|<{u$B6C>g(DmtT1%oL(8nocl4vYx}-})!X6Ad=2Sj1VL%+lukD`Xa5mKU|YiAwd?hBK*TxWf{7cm=0!i5(6 z%e?+&?|J)PX$!yD&trmm9C({DRdB9Q2Xwc}v97&8*$um;!L^A`*!wz_EY>l_Mm#0( zxA76XI5moW#N`A(TTTEefxp<*Kb}?J`0L4>jh4idT4h1`T><-JbTYm(M+b!cP{y~u z88NZFy3Fn6W5D?>GT2W)gXQhO?CXOL;BDazcE-q2;fEY8U~2!C4GF7dNF?F1n}#@` zW{^>=c*2HfdJ0eHa{2z{m=TJ1uv(T=*$D-o+5JYIOoMwG^KV_L@chDQOs=aH7EGK4 zV%y3c`zJOt3fVtd$K6W6^0_n{w)iqznmfn_j*$hom#DH2F1%uI*Hkg%^t5=twE~!} zM-Q{#e52SIk1z4I_IWa{=#en@#s_vyS*OtPn;dJ$@!5aNjlsFE#(|_M`RwsfC9E7H z3A7Tk*nROa%-O@rKsihggY5}SWmpcAC~XX?R*l7{Rf?IQo$-jz#+z;kvRd&ZbobIy1#JLmTV z+#e+k)(QsLjFSDV%wb88TV%saJAavZo2bk_&a7gNZ~w#Ish0++TlF2$jzx^$f4A5p zw`4$kv^MWq^(?%_SR3SwRlxeEFEa~dEkISgGTvMt{TwLbW zE5dG3$z>j>=mptScSH>dVN`wof-`*jPKx zC$EfN>2}eFgT1t5`DdD2szXkEYev4|FQ`F0g!EZ3?JT&948Gi@-4mL~$e%K}D^rGQ zPX3R6sS}`=W9Oq#lZS9v$Cgf-CqtQfj;CZ`B(8a_iFUs~i=Hmdq)X*y(uTpAqS@7j z)HEUm{`A{`u5p~Na(NrF&Gih;{m_JNowlMkrIlgjtx_sIb{EGJ%7CLU39))6rCrya&{~0b5*LtX1cO%IRzjt%3W5RGMn1_KgS_|66r4M@P~_ASd zYiW$tFh3_YA0~bMPILX@(34FZV@;Lw$a9}ffUzoNZoMK2v(Jg{%-o9H_iduC=rk$N zlOyNP#`3>il^68vOhgOc{-N_GZ05h;AAnwG{38#IWym7uNmTB~5~%BzN&oYnj6Pnz zOW#lWNdy~~QfGTDv@CipdfPsq9(jKX{N22+M+h zj1B0eunbaQb%}n~dJ98eFGkS+I*Qz;O=BO)P~XX1rh=6uV_kYccDjfjm=#6`J1?Ox z_grBh=LUZ~OHy1YXGXTDucfnE)KE2Ne+Q#QRPv#i`ajbqu^}($g7w<)sG>2=@_K<} zOZTG{w-ez44M%vW@iH8Ed>-obULvclQ&Iil6yon{MIY8D)AdXx+4H6oxzF)}x5Uq2 zApMp30=J9+r27$%KLh81NH zsn4+zDT^eKrKt__Ts1`;v`>ZlyfC0U`0L;bNh@fn+X2r^>LyJpaeTWO5KL<;gWZE1 zH==0-O1|efFT1|MY12#KQ|CeI78yo0lRlwjuOC#|=`VC*3#o1G3Hrr7l3ewF4V`c^#{$7(v%bA2jwG(L@P_l=NY|2}xta5l_8mPO~k z$%WGw=Tc)CX+h?Y2wr=nOP9YdBgv_hzaX&}8M;ToX>rBK*o<>^x2RLogGIEVV?7O> zHv?L?u120b zwYMuJ_a>+yRmG#^M~MKL>b`(i_H&K*GFd1t58`;2t|%oWmmdM+(D%Av2z5DEvwAT~ z8EK#`-sfnoYYUBd_XBpCtBAjM1;O{MC9w!$>8j}O(60OtVUaBLyxm9>kDjML!wm(J z(qY8>eirqpoO5$r#PM*&9rj>RR=>9xEQ0rp{>%Q%yUNVc(b&qBgb9}sD(j|G) zJE;l|w<-&ol?FlZtS@8%I*UHfiG$|Pn?=q4Jg5gXr|hPKDDu}LVt3GiK3=^X*;H;q zA=PPQN8~l+Zyt{_T=b|{N;XvjvJ|N4Ql;)QqSjn3vei?GUZ$Lz@6Spanw3Q}+Y)J; zv9s7>`AL#ta0CU$&g3^84yApp2Ic)qq|RGh(4LHSsAPeZc==pFA6~qK8Xcl(d$u~< zJ7*F9;b9`0DDR7o9>++hCJqJ+ouadC9icK{(Wmtn(A3Fi>B5#9Byay)vC-u7@kEG*@>E-LU zVShnB(&T)2t7PlRwc>I#S0UF!BT;kYMO3vt z7P;(k=C`i#A-{%VMK+%)4Vh#`pXU@ntK{|kAn#99q&@=~AFxIhAu~~F-Y>d7Yy<^R zOL}&S22IsCkM1?zL4DSzkl%^P0=oxYBsy#kJ*=W5F4uld+pW_0k=KIJDNji%TRs&I zCSoF*6-{E()zL#cC;Gwv3O%|o6Y*g#f2PfKbT@k&wR=5Xyl>sd{t!#E9E44aFaF1OQJZcFL5^wV_LqJ`*suM7QY zqk`NXUK1soJSDd@4aJYGx`=Avd+1csjE-OSp;r87@S_bRQ|eFBMP(DjG*b`7Y#~&C zsvK4Oy_C*d6;7KU`=g+0dGRs#z0`A7HhC?jLh}|JL_^Ju{H+Zah@ajDa`c^;Uwe*0 zIgzFCz}#54XL2fvlfX2w3PV@7Bswq7Sp4nsaoV|g5%sToiMGs(C!P<}smrVwl&o#S z|6n8|a1IS4inZRfX<90@-K2or<=&Ee*#+dHwgbH#9!r)*oQ7}i%cGtI3{E}qg!e1+ zfdbD|F#bR)TIBEoCC`t5?%oZ+_edM;4XT7+mR&^49_JvN1#9Vlv7B?ZdLMlLO$wDt zucY*EG8NfAB34}9;b74U;^t-y*UeZ%uR6w1+xE$H$cNis+MkmhF~xAQ?=|X}?T8NU z4nap8521d#0)2N;Li%_a%+%XVMxWj!&R=elM2B3uu0IQ%l}e@wug;*d7)fer+y(C~ zjwO(@l&KKCFh^Gu-|8(ZP1g&K5Uy@w2S zzaSmAuTr;@E6GdWR%$eyNwijKkn1^)NEx)yH3AAC+u z+j~HdO9J#eSs6a{oFpzA!})FfKZ!y%E#k|su|ae8l_B{(Hlh!vPhl^-NH*6bpf2m( z{DL2zv~1`t>X3U+KD{s$zdT(GcP|Wv;fB#vjmu7LR8$wOQa+CEotR2*uJk7>l;@+l z!|(ZZb6&yiwx^-L6y;i2(JJ+LwC18LS}E~`e|{Z> zZ*eDl%Khd>9;#A<)2oQbqH2y|ejYu}Il+HwP%oN)^&;AS^C$5%n}cpy{6s;Ur_o}y zKxA%0N$ZryNYir=Mz;-;dxeF}*~e$llIu5-goYO3?R-tE=ZToLD!K6J^Hs3zsv+8z zBZS+A$D-VI)mV>fzv`xRp!u&qL1lUFvn^OpYqdS-micB-)#x!TyOn}Wmp?|2o&-`0 z`IS_A^Jf}h-j6qgYol5RV-#cOgSvzL(eyS%zuVN%lYSELp|U%gq$f$|+{i#lojK^- z!Bmv;&wwW0Hl=62X2ZOUbhK(=A}mmF0RGc1p}Cg}L>yU5e7_-)Jl16R*b-Hb5 z5qz8R24qK9pl3C4sP*?^%KUo8ubBKBz1(j_4sIPl>Y_2|?>!Z=-ztj!894=mJ$cZy zqL92=Q-;P*HKXs>ou;Q1Gf;rvSUNxKJ8AFUhV;q;sI8#5R^y#o6A|D=6^_uBu`-$x+dCv^gzf4`wI*?+8wiraC)(A?Pq^|OPW^zT{)d+o=7zMv=%79Nz3*oalqkRAB-AM4& z6!RDy%^mVc(byafsk>7DD0>)~>>&O={U+^vq9FKSUO<1v zEr$++OHoHuB+MQ?id+>235x!YKFi-h4k<3BZ?G4&KG{s)?Rrfo955o^x)-9q-_lWQ zLnc&Msfy-jrx4z{pTy^~o5(kShdy2zL`z2<(TY?Cem}Gn-uwAPq@%_Y^ckmG+@_Q(Hzrll^$=rt~Nl&4=5>tu8QAw0EoJ^*Nrop!DDuT76C+Whk zIrL3%k7zJg9W@J5Mbb$=BJEgPgw0OCb1_XM!gn)z{b(BMonH=bO#ch>IyQ)=I?4!S zUp+>J=}Dr|{&*C-!kyf6%7@D=5|ONg75#YiB|3O=GW@);8;LAF)3-MQVAqN?bj(Vb zKEwT@WxD~bR^E<|-DyX8Q^t!d6%=s9%Tk(uD1+{*Or%C--1V=rhia61qsQ`PXyB^? z`g|}RC1|%JVZ=@{l z&O49^)IhuJBSn*<0$}LuL6m-PK56asLAhHb#Dxxv$!u6iM~1okm^Hfe_3SZJka3-? zvq&a~g3_Vctl2cHXDoDkBce5hskAI+5_NdImOfjiEY=Lwqp)}as%|<(8~11;znQ1d zv7atTcGfF=*KsyX?-5hC00a8*v@`03iKzN@2Y=pO1i7yxGGb+`pIszOkxQB>3dEFM$(X_?Xc^I0xAbZLzEpm+ zYb#uMk)d+F&&ka>bE!Akh3d4OXk4lsx$D)<*FAU}g;2g`gXsBdWUo~#G$$lBmIVk#U+ zT2ZMe#!8Ncgh1k{%tI4vI*5a#D?M>xJ+U4)iTgii3O#P$h>G(MlPc3uq8$5+|8;LZ z>J01W7*VmL?$H==#&-wQ_3ILS@xUGhw65aMv;2&_JddEDr8?Adzbx|l)amX?PZBCRSkC-+Vsh`BwRZ ztp#xZ(Ji>*P82iO!*SY-Sa1P*MW)=QuCqEGn|5cQ`npW8oH7pk^j3p;U;7Fw;(LB( z#wnVg-3K(HQ*grceA<$e37TW7<+@>~LHS7({4+~}ar5`{#iA=bZrlJv_e6sAt7Y<* zZ7#HJ&J*}Ib|3mIx=A+POB z%mnTFz48ySPwC)}TR3HEmpU)qFx04iNI#QRaPpM9-0@vM3{_bGlIOb86Pp0u)*Po( zw)BA88?51WXBs>oRmW=JE%dFoGWyOi#=(z)hVr}EK53`?$+$8;(r{N2ClC;%R7V3FxBakyzd|{{5tZde8~3xbl|-& z@VYn`hUgrjLrS;7YO_l4TgAEla!XF<)bJx8RPdu|l{|Vev2~gBf@MBt?0$+Xr<q02qJyK=XAu%(OoTVRb6l?UXw2vQrB#46ub$ z>1(J0uEc^g?4D*xwtCO9Fah&h%os|70#Kox!>7;Y2xHk+SuWo90OeOEgK7r(((O!ZP{ZVe+(Qv19fe>)!Ca}mv%e1l?)6ygZ5dqIOeB6 zYV9lrNndyVVzD`G4(PzHzHU&Lehk;m?uG_O#^CZ$Woe&27wEXK{&>ScpD)bI26pHf zT{Q74oNY$4|gO5ShyW8_OC2Q*;$JO{X=cL&F>9}JN{ z0IY@{vMWtCYI|RX>Br9L_?;)nRb7 z)*nNhUGVg1Cn)%L9*ylju$A+#K}wHW+F_H8I@#LNvQ&{qW-E4^r3%lAg$}0AREA^X zOX<=Psp$TFAS^ezERUN#0xjK+(7lZ*JjBokC-%PwyOI>O*;#fN>URNOu1};V6n=!W zvMqFtUz7B)iYq;(@e!g<#q$a8Eg`4CA7UmCfJ>upu+8Ux^SpH)@Yv26W?#6ksPV)= zrdAm@jfz6e>9e4EQ7Xvp#X(d54KT9%MC|wbIhRI6f`9BV>TyU5DwXW0$B*k$ujXXz zuU7&CGBzl%Q2eu>ztvv|yUz86U8~Hn#z?{HIM@qS)@LbZ+VrGNk)9~sl&4@X+?R(vD3P}H zxsBeLwh-wMgJE|S^J?2O&{b^Wul@C9p%GhQmhoWNV)ua>57{+Yo>eHP3m4+? zU7omz-{b@PhSPy@AEkW`6Dy;SbBF$$VVPM^@CiRD_bsr6 zy#tkDz$iy(JJ}1@Ru9GQTJN~GVNKj5s{ z`S`ZuI&~bq45|X1=`#hn&FmNhp;)u7p9z=usnth&h-Z0LKj<1G2N)1k&Q)i`(_`J6 z`Tp^zVB5+q(x--VF=Jx@WNvbS(7j`D(v}EZRWS>)V+-KGK8`KFE$I4#p>(B_oQ`tc z51#c;>CeNbr6=WkV8WneYW-px{`LrhOK%82XZKk?->{UP{guRDdvBF*ESo2-H+TrM z7Y0K3pnWuG_66w}Wd9?=c&_$8B`dKyCJu#Kj9dGHt zrP)x#uG3wY%&^n-KmI;_3IAT)PAfN6O3TO2mrW^i!5dFA@u5W~f|ViAN!@TuavELH zxDAHRoCJaAj_`p4)3N={L^!mtD~vgl1m?O==-HhqnDBfa{jxEKuG*7|Y{VLR+)i=c zFu2C2M0>!r_j^E7c80IK+6~UnQLp=6bA@j3_<<*mZ(=8J`(k8s6R7t7ft@3}@v}Rw zLEC@LnBTq-)}E=sJWHdx?5s{0zR(*q>avNiktuz<>NCXbxrGC6H&bWHGJbbzJodf1 zonO{h+&^;C)#%jrq)KU=Dfre6P0n< zxg9X@SrW}^RmR1-Pvy1~G-dDe8qiSEfJ>Jg;-2;g@%s+Sh0LC^yf{@v_Dcl3i z3FG)|t6A_NH5ApqD1p0eHaL~H!P&m;xMp1v{AtYP`$uktAF=xK6T(A0TUn@B`}NW$ zCmp`)T9D9m55j9=S45}yU6ap8*%Z7tBiz@d3KB>x1R(tZGK zH#g81RV_G2t(E%Sal?RXBjEW%6`9FnQF>CR8XP-U!?QLS7DVP^?XWnwX=e$pR|)o* zvI3LBEa;|^2aLXxaPL)}IAWX?Q(S22hspFuf+cP6y)cx)VyUaqM&oCpn|2mbAOf!a?k%MG!YjWXJYBfFYca^?d zxC7e;1@g692bhT<8Xgyc#{Xm6*a+*0q)%5O`()eUqQ9r?o+ri zPtk>alJKCz1YXV;NUOf3VygBHoE}kvO~Xn-I@%ix*7m{1kdUuq`1Tltv_=JbDz`#Od+(3Pg%oXc7IS`kWf46iRg>U*f$rFGxwz0F>01K+hUq z{QGb$mP~2@H+?Hq9+?80@6P2PpN*7NZM*?*w`Aa?i>7?&(aYSs??hP_&u+4SHHX0Q z;0r#(U0oVH!Ab^Zt6}rSQa(W3N9WrN=bm07v(QwqZ0e=_Zs%xN^ydq=GgLUOF8N{H zdVk38TaQbAnb(=x7|CKBgJD8oKfXSA2xK201(BbSPir}^aEWe#CpQwQhKn5iLO;k) zX{*%jd69r$?WXbGzk_(pSe?4a@g2C$GMY|b94|j%wwc;*ZpPhhDs+GSO}?io7)yfI z%CiqD*m0`I(dkbk_qwWD*S!2H{rxf*W7;3c_mkUjy7wtg9^2!ip(8L{a!|TZZ#O@* z&Vnvks04lbt&^*x8Vpd+hU;}|br#*MFz>#xEVDcYyRI3Fj-K=2hO31D7nH{JA(H}R-w~o?8NS-1-!lb3$3tjrq5arz|)X}IH<#% z4xU(#vRA<%u5FZG&@;d%qnqfjJq6Tg#x-<0I|dvFD|iJDc3@1#Tr>)Q4KaJq$#-mQ zq0tvl@`jZ@G%_O*Olvm5@E%8Kzq{}0@bXleqv#stIdv@aZ#cY`_dZII+H z!L$!I*y`sw(n&d|sEOeg`OfLNxb&(nZ9CP-zm@(1YmxKViVk$`Wk647OQ`MLFVg$J z)#S0)enaD{e=tEB43l3((m>-k?Ccv$H0*sCiL(fwqRL?1o*Lu>-LTcMg!fjj=3DI& zq|D(4K3h5*oqlemea3&ppfooyZumi!RGj5y;|uwXt&5?kqZa4a&7@E5x?pR>US1Ng z14+PUESPKrV@~&hmzF_wMedVf#6r5}el57k)REqwmnI zbb!_|KDE^hQoWU-*M|nURX(4lE;HcU+f8wQpais@yu;@U-%Ab7X=C(yBk+7&$JhFM zvVKSq68KY83UE$q+yH0l0j3z>H^Ad~|Lxub0Sp{;~I1 z-)}BIQ{)c442HtR@6B?k{7f%QGvc?m)pGYUw=is$73|9G%Vxs}zU#0A|E7MU(Mb$K zk51(DU^D7(%L3I&DR^8V_;_V&$gADDGNTeBdA{opD1ZNohxE|^Rg*XLheI|VDJw)v z^<@6Rvj;v5GlSh1%)oxMD1Z8R8Bb5vhDC=CaFgMBFhO+(pP;r3?~>zal>LRb?CXi& ze&y1Z4PV5+(;BGhVi~^hQ*cVgsN>*ANig+-2mNK(%uP0GQD3Se3t6)f$Lpp*j^sRb z*j2{2uDOZhu2j$;o)Ro=2!z|Ihj`83iQtntSem}N57?}0!qbA{Y;kBcMqZc8i#w-7 z+sj`xLH8Hj`Wp{k(owMUg)J(WaI&hnFuW8yl%BSi!`++*I8{}~-Q4T(Yc+w#-IvwL zo@!(6e}?ENrXlNruzY+t4t;P6Te@`e-I*p}GpU-lq;%3xqaHy&ZEF~?*#|ZqpD*2z zl>@JWt1-du6McHcue$P_%*PLg5}dm zeJ2*PDBpC9>EFR8eCrMG`dY%~&hvEBwnqL+CzTr6-Irg?@I>G3DG*j=5 zYOMH1bw3bIw@0OdbJhcS$?|gffxn5c>|`H^XncYP7OBcY&M;}H+D^2ykHXS3fTedA zu$qHi@$r;txa3V0t$&fukL)X_CgrJ$?^Y{M@@kO&{c=?9yiNsuQV+=A-QNwX)1xu1 z*9mM>E~Q)ZD)E%39?UfE1KOK%LH<4hXKmk&N5i&bA=Brr5z}B_6qAnKZ;SgB`8~Vo zpQw3zJ+)t=Ec^1V41S$7!!l=6`srI7faM4KDMWBE3y=9 znRc!dWCupOuk!kycjTi!)bZ5^27}+7Y0?3kPC#ISmNdDnFZwm##BP&w`ISO_wE1RE8!6_ zQk?%c0&KreCubeG!qI!3?zb?9-QF5#{9iYSjXwnLjXR-F;3Rt7|2-XR;|@K7Vx)cg z-{adB7Sct}BA{b6gzy_>9k zUkn{Ofk26?QeEK%^SXeL=|HM%;Fy6kWc_>zhVl17az>=<+6O_OQ7dTtI!zj|`V=aK z%mS@rYGC?yGaS2gh@YLF3wmkUcu}newHq)a=1I zMNjzZE^=zJZn#X_c)$F|uP)G`FJNHJSFXE%9_D^d#hZUZ&|USZB4=pM1Cl=QS0|o> zyHO=qFn_uJMh6V+G^Vo*TgWEw-B`D#5Ci|KA>k9cNUM*f94b?i zMcsP{4nNYNU{@`~-Hw*_mAs*T^U7d(cm=n5dlG&-%!6U;_G4z}VRX2*g?1dwl0Hi6 zkG9=HsJCS&f7|jJP2x9F{b{ps$BaohWT+kJTYcv4SGJ=t#$0x1P%Q#V7nax#DZ?;jsu)kWrHn}xoWmGbwiPtnC6A7J&JDZHcB2!C7k!O&lc za82VPd^E3tCmxn?SH&83xc`zas8PrDlb`YZ*0tbqcRBXaT!}|7r{lDxY3Lr?gI>2R zhnAmzFk%06_+b2ROyl4ZhXx)Jyt)0+d`fw?qbOkQiOvY*bpTOXG z&vB)(HvR98Iv!F_=NW6y$z@gX@WbUOK!7PEKl@6zB&pPij*8x^!6W*8!W}yCcpNWs zxz9^44u_5V(_pdNW{6MgS~s)oCLSnU4l^V5U`A6u-F&{DmaecxZ9RoAyf%S4Y`TuK z3hk(B{BL}7_KNh;?}J#StxB){vxe@E>bP-6Gx0Pcy!RImnTq~a$Q#x`6U%|e4tkCD zr-#Y*=On^YF@^dCAI88&zPy_nH$S~gx+~e24)+@i zZ+~jYLgVH_@XJEJXh9ZURd_7<6(jUp7Dr5zYq-T>ee7&EfSHFy_ybGjLAhOM?gz$l zK6Q~z%{<2?pUlwvu{jR1G6mE5HMHor1W#7qL!YPz{M*M8*rNWEe@=c$%jS-dwuSeX zULVj7z5X4=o!zG>n9xpqW$a0;i%-D2`Up$^Y{sj0Md&%%1P%@B0{7Az;n5OBzG-9+ ztQcDXOU$l9&lUT?|Ku2Gx-$?q9;ik;mp^FzL_^m1Yz7)-T;+#7iuhxzX1c*m#Qf&v zU4)CRlmgB6@o9dHt3@qTdt{7~NjaFH=ss*0=?QfMrM9=9lS~1%R90>Wr0N!Xm-xR;>Ty8 zwD&bWFH7MN#V1^M>@v{%)dqw1`C<9lsjxot0$qPA2NyIvpqcjbL1lXo_;0GGwu@T0 z2YM>9>*;t@c^MQ+=73}0{W$4DBwA!1K!dtPxI#>myGN+V#tqV`o9OTmHofOC&p!(X zB_HG8OwYo5pYMFtfgDsj-Np;YCqv}TefZ)(ZF)0%7}~T&;YIfiob7Gr*DlNG&@&Y{ zz9@>%yS)z9KhVU&-rI2B3~OdS=alsJmF@K0!FFlbxN`ogV)te8KJXMC9ASZPQwr(N7m9mRz733j^a|r{#9{S~JbFK`26g6~ z#`d`P)PM3Sv>Pm^qYvwVAu4(w<9kEw%~`nm%Wi&s{!Lj#k-VVU5bwFD%7UjT z)iw561m871rEA?Cv29NsKQZqxACvbJ`JHE2m}5<1RSvVY3m2ni=`CqxQ69dnS}$Gh znT%m`44}1Y4=laD6D?aSu#d}l&rANu8K!p-(qa9o6gmG?ZXdW62Yu# z2F^N@1&xQ)XlCI6YLz_&8r;Kqo>2@8x*^h+&v&7=a}ZxLsty8XhAXl>1~|WHJbZOK z40fl7W0U?N^k*%_66pSyd zf>}9-(V*oowAnS`($Hnxqtu1QmW1%{8lUjvzDcN*eU%Ne^Q3)Vl|aRhOL%6Kk$lS0 z{k-?E^`Nh}g_X`)M8{VI%JVJ5;lHFa&hlQBkaRwoc6{#(+vl8y=nsnPrud59j;+Sq zZ^|L0yBFWNJ@~tqnZv*(Qw9 zzt6w@ONMKT|7)}BiZ{dJ@xu8^y3=d}8n{@3dF5E>e(4#_pZft{``<$G@h>QeB6=^kabr^*eA8lw_ zU}07hTuZCL#_lI@-S$-|bI!zS@BJ`p*&)6*=?#ATJQ~ibdVp)0f;S$LgD>WP<6-|P zW4B}?OEF1;nYp&O-Tompe^Elq?`Ghh`e8I+#s#UtiUpt+?t;hm&Xpfu{2FZiu7Pr- z6k7xtJk?i~4c6W&JvqOhO#k{AaKBSbr#iosUhO%!&SXJvTvPJ~HfqW+>T5EFKO6?b zGOp6Z7Clh+Kk+hfu`L z3cAVUj}(2cIAz#p_nzKAq*!;V6XA;0TV8G)f#0;Ykg_v2bbfdn9CSH?bHi&`56h2m z^3yx&H#v^-F1@heCYR=J>Ot$*G{Jx?HvB=k37tIVihNJ{eEfIRm>SMe^dVjhrjPzT z=2~7`FrxOFqQ`g~=c*Lorr0#dIrbbidkn#a-{11Vk2k?MMQ_h+a03M0$;Q;!=Qz5v znQT230P^}cAd`cnTXn4Iq*@jE(kd^hd0rbaY+nh9eeGb3PbL3T@P%Jk8O&P?Q|T?k z3^?az3paWlgR>)-DjW>6V8nC9opNYz4BxgK+tU{C0Y|^fmwMOmPhbVL zk%?Dv+{#NEFXHmeJ9z5mC?50Sr~K#Gzf`yOA}@I43>7+mc+X2te50Bc{tJ0P56zRq zKOdz!rE@1?rP3=B6&i=rP9DJOjyKT#+Gngdoyzrk2GM)7Byj52Pk8)Cxvs1%3)0nB z;kmynrTyQR@Yt{0VN_cR?pflB_j(?sw=FZeNDR@KXhHOPYF0yy28kHj`z1a2%?B(yD(CG1*@Wp;j+zJs;zqr9931R{T^_9)fh)2`>KQ!k4 zVHjpS2Q+%5QnOc2`Q?CO)b-p7-)$qIVEqnEO^L#aG6Kfxk8rT^2ma}NDz?cc^VHd8 zu&?qL7dk_@&F{@<99qJEjxV9f6VB1R=li8o*BPO$?rn&jughzv9D_1N&1_z91>*HE zeA0RtX3%4h6cI+#n{w&$_N8Ebpp_A$Gi*2|oGhD7F zc+zUsEAnk0(`h%IZt(PRZ*)}}Cw*~VjcyEo49Y#~`6~stQ}6L^dVYgIpI4|dX}$pe zbsFF?{ef8gEDL5otb-Fc-i8kg0b@Z>?DkN;E4rI1HtiJ7_RW$11T>EKDD<89lc*P)oTzeKmDB> z*c^wXC>iMODu$k~Z=(9S0Bmg(_}T>st!C5FPklHhKD2-QY95%e+Zh=%R<-Tn(X7aQqex) zvN%1>l*o&LrM0=Styjh{`);k`^z1jnrmwc--A8qzwsVTGLYPagvtV-M%Tkh2HJD9Y z`&&HOuJB+v8M5vZuL=Ej$Fs~&acrZ4^VU1lhFDGYBo~!e5ci;&q-O0<(Xh~m-558X zt;`(5ZiY1rwx)#5XmnwVvxl%3cQR@Uk^Zf2nY3s2a zt;2=g-N%vQO~z#WRU6VcaSj{P)Q9~2-jDSBp+{b>-&||2EMp)1E!garz*4PNkc%N2 z#6!Vio2sEt27Y-jzFFUwmF|~|!hC(U+be*WCfE@_MMiR}mNFUHD@{DEx0v<+>cM{a z&S1r9gbey<$~ym?7vGgW7Dqp^C8kG{$fA4EWXXT4iQyP+7I<^0xXzQ;E?t)*UK_JZ zIM8bj^LAd%W{#c1>N^i999s3_kcI&)e!K}Q$j+8*tq2yphr|dsl;Z^V)$7FDjmwy4 zXcTLBI-BhrnkeqF9KePZd9mA({g}hp262?KBK%+_B}&h=h+nxbnesJD5KI>d2XC$v zRMu<|st%2khd&lqyKC&Np~(OwDo0Gfi5hdVHlfU=EE8u zpA*riT-c3b%VV;n#rSB*egmOnKRrq>LFrHruI-2QL+gUAyfVReNlL`0ckdQ$M2M z(F~JGiduULZw?mrC^Ex+j@7+RHG-YAU>cla^nf#>M(tVLk^`nG+Dj&-R zO`OX1msSWnw;Hmo_q18h6K1TSwMjUmc;3TegILPXc`WCNKrWRVGqX3NSo*1+Oe1t1 zF+Q5kin^sT?XXk@3o4oHou=g+&YFa%$VW9jdYUl3D_aEF30KMWk{sdDsQ_Vdn=Mmb z)htFmQ)RzvMhPK*HOcjB^@5ASD_3J^ENZxqCNuq9N!U~=xp;I1>y&3no>(dg2^YHw z+Zub2Yg=@Qk-;&+_~LItX}gwW^QBbLYx^GYj?F=_&%HashMR;aJ zs&qa(?B&gjBnd>`JDDwsO<~)=Y+%ukSCQB4gUD9}8@qYHAfo)fLs&j`v*7h=i}1(g zvM{JlS$t`_LgBlQWPfDKSj^UVg#)!4TN?OK{P16Ic4^v{TBpPFm|4S2_V+(8*41#X zq9>EW3LWFw7#hzyU86~^k0}{4#fg0LFehu~`LG$g>{z%{ftY#JnHk+4BL1uBM^rx@ z5xzx#66QYME(|-oUcoPoW>cNkvTIIr$ivbo@-k>W>Go?OX}Ba|Z3{+`(n}uX{(CdB zq2;o~^^P~G@dzNc$2`c1vO(;j*F<)uO~H7)<;e7k&kISn49K8#JEDBND~U`yC$^$L zi@XuY2K^kbuzeN@+X6)A;$!oLTUAe-3RgG@hq82uXE$RK+O-?`W!ND$z8O&SW=EXh z*e^}8_GP{}AgrUdrkjSaa=nU}S$SP}+BQ}sUkb(9u1?~sg@`w3$mVaq1 zE72KBVvAphH`=a>heQ=-y{{K}Havlxevv@byqA+ab!&E``aryxH`)M0Pwx!ZO}R2xC1bvjmNa%=n1|b6&DX=>5lnrJoqi zE_j)Rd8f~xUvA>Da>IU5Elm@R$lJS)Z*OP{Xh$Go6RV39 ze}t9dJMr%1Awp=M5~0{2r&hYcP&iq$)p^6@0DuBsdTgF=Jb9R!K$a>kCz?Z= z#9B6!rMiZ()MvAqQK^hb{G_aJiG*2=7FfpGYr-cD6=Bt^v4X}cbs^7bwz#0ei9}2n zNbCl6()szG_LxnwTpOmji=L% zn);_@BJI&CjC=Y}Xz!**DlZzbe+|AQ=|MR8`oo2cnKGJ9ZCcNMG|Sk_7*AGzB!EmS z^b}hC5+shlE(%FgH;KhQ`^0%qFNnO4D0bZVEOdztB^v&#$-!(t;&ji94LI^sP=7Uq z}^+UDWmf{&@fHT>~>ZIywi__@SEJT=dl+!**(=<>4*S$+7fpn5oxt?n4i z=7c-3BTm)|m-Ps8MPs;VJ3K_}drn&{QtHd%w@+p#lczElQ(!+AEF-;4lbG+z6gInR zHS0JTO-lBJ3y=JLtKU!VB?K$hW)GW6(PCG&2*#(x?+H$V%ina?{yv=j$T&XvqLvS$ohV7#qJ}O*!_N9VvKndDLU#aD35wB7)6c| zqPB!Gt$Sf)qWuDLClN?o$nmmLk z>~>^ko&kB<{7$UbzaoxpAHo7$kBj5&)L45D6Y}Gd16efep0F!Yi``5zW7#HMnfrVb zmhZowq+MD~){L7)mXq;B>8K+sf0;op2c#3@k@4iyu92*z+L+vNFeBh-Kx~%{5gyIn zNHizJ6R|Cc*woHri8}}xlDn7;Gq)i#o<9}uIES+R$7i#LHgnmQ;!na*wg|ZZ3C9NZ2B38kLB435jfJgCUFguP=)|nLmtF*%kLK!UG@k=Gd}Y`vi_`Ilrua&_H=3ynS4>$GL!_tXTjU`(ud zv44ipH@7EQV&9t>s|+A5qoy%(MN61|@s?yA6C@`u$4JVpR}qKBv1HmcAL5oikJVZ! zvA8aSnAHSTrtw;tq|I&>3jDOmr*}^T5A|*0<6>KKgPD^Jg@ee-)nCLzhLefLJuihn zcr+>8rMUYhI1;TZ`lRBoEm?o_m9xX50G3$l%U<*kVQu?fh`HZW*}b*N?BVm3?9o>@ zwsgzO+BuJs#Pr#7Y7D#277N@wg$&pyI89qEs7&lZCSQtSl8E(e;p3&us%D3n{7A~8 z0~IXFsg5kl(}4sl+~Lda+!TK{=&@z*a>Smc7VOrpxy;2!!J(aTQ(U;V3;FuJNqFmK zOd16fX8L+Ma}8a_MzsgBhc}mqcMp$Z4moaYn%hv;vNKrNFAZX$VZrRbaXw5su})Z1 zG?(nq5{UNxC8WXls;C+JP1x7lmBhMU62dpB5gD1y46b>wKQ}1r^!g)ODSk=`uf@^P zU&Wzo?Ah@&Gd4w-$w5Kd&VJcE^*09dRV)+X|91d8?SZYbvpp zO(&;*xRZ_WTu3U{Wmhtmh%u`tGUu;qWa}3{Vm)^TIT<{eOxU0)lA=oS#I3htdUJpz zbjcxcfGkMtQ<5c`pC}hyym}MM1W!^sKt}x7ETZdV!2Ft56RW*}q=Nep?bu7A+0Gvl z= z)5XH8xqd7{WfJiYm`b)uMiZsBbG20?SFzBs5iCnLh_z_+VWWP|67FUn7PBVY73^(n z*o-m|CXC-H>P&nk%(v5K1yz|f>VXNu`+56BHIpu+kMbmTDO<{(DNSXn->(R-R-F+I zwFB9Hhx^XI@7Xc6AHyV0JMKzO*B_AN_9?2lJN>*!;;d>;TlHXxr))_tpEDAj%kUmjgrc9oy=#gVgi}e_2&%~XMj6q+8L(rU6+@H);n`v~MeQ?l80+W5lqUEy=R2BY`Pju|$gLovowbx~a=RqdTn-|?yu8V}Y+s@# z8OXYZEoAb4f$RngVws-}$V%sQmODI+MLQ+2{i^1q?}yn8&0Lvb{ah9^w!bJAdl7?a z{^Y&3gSxb%@XY$mZD!z${rcpuoJ5#OzY4P@@=#O8zvpky7sVT zIy%pUfN8T?{x1cyTx&e@zoE=VG)I%q*@;AB!9w!G>{QLPUSrwhxicAtyRq?dWnppN zIbLel4Py*0gUQw;JW{m*PrZ7~-#9;{*FKv=l7$lt@*WA953_i9*WKKte-*7Ayq_1W zQNhuNl0Zknv>17NE{rKu_@KX9P&XdLm7C+BP{Cv@Jbi)xTeJ(uK9qx6&|ZADqDnH| zHX2{vjG`}G+T|P9KEcdvUs#Y;gTro^ao#UF2aQ_~JU?`V-41b5+~JJQkp=1OGcS-k4I zE!t#TK#NK<+{_w>Cf;2kFFFH;&(gz{*Dm8NMNWUnw*9!d$^@s@xxqZWW}dF#B1UX} zf_po?ag?(fxTo#p!sROZ+_IF*I;~+y`8xP-LJ8m1{}?s7sts|8pYYPO$24N(E4pRY z1DZ0<4o8`l^RwCCr3D)XNGGpVa3$YW@fOvAIQ;lLMSn{K`<6GDEz4BdZ*vf0~+sXk=aZrjX%Znjq`Ax zdID@7vY+QfsEN%J&nae*2h zeyQNt>}hh>8yPUm+7?$kCb86}cXZ&^BA(S{E4(wGMDInp(crhsAvAOg+)^T zVeCU#pV%8dHu^%j*2`aq87UCzfn<(0x#zWji8UE2Lh`5N8_ySR6Q-`{WZ2<^A9*RqC2 z^&8D4-L2`6TR-?nw;$BDj~=~QyaU6fP2!J^Q(JYHuQgA?)79zlOttZTyEmTQu|Pik(*$tezec)n=K|<^GZzE0 z&p^!g?lM)6V*W?T6n$eVsFwT-f+nXc6q)}K@71s{U=WY2JqoS$xm>U)h7r^hR7ZF~ z)S~?`anwLQq}?37%=@Cv*Oj#Ek3al>iq6E5s;7-D7WY*0W`70p0T;hgQc9A!)-E6u#QTbto~Xz2pI{TlNNh4A{(O zaBgtxwM41m;sE^X!#eu%eFD3!gvV5Ohoe_YooLjV(D!E-HTglFCCNOA;CMAr^hKucBFgNsP0?b<`(t8PAp-Wc30v(C^#x`R4`wF3z2K zNY+q_t|%(y-&-v((LNJpPcV16TNlaMZThUhw(A0VdX_MKHMx%skxyb5Eo!C@JC)FT z=WrUgT!_B1Q(*TTUWB>>=F-gh|CokRcXZtVpq7q0bXU-0z3RqBl%ZKkM+y^(b>$*7 zlKUN550namx=fKtH)h2YU!pC?7T6hwY-S!F+Ddd=2L-+IN%Z~l1mv{z7b;kU8O1XB z8jrm?=)P?+yIGL2y!rz~3$CUkdvmE8{X8Xh_2x!)bFmjwRrQ2DEhWL42=f^2uanH_ zQY%_F{V5%IILoeQxqb}^--2GrUq(ur1@xKU7;5rSwrkrRPlwk%WZmmc>AtN+{InBd zOqt^eDl;;RdBpXh+DCViapf*XWbY7LXxB_Fd-*70Os?iNu1CKef3b~PNSbuR0gO6HAfP#VhQLK1f4mI#5HK zC)|OjlC*lydA?9U6Mel!7s*#Sp`nBtuJq!~Y>&=ts;MP}0JD!nxcezsZ9pTJCDXN% zIZRdTK6Y)yJGN??8MPy&^c**y{pSBFuFC@`D^#9c&ar2nX0Br|T6xf* zi{-T57M=tJ$DvD=Y+1)sgctd!&lpAkXsts8`95C zZOdgpZx6l zDI;WmhTHw)9KF$8%1`g`LmB(7@Ppix?D{WUV_QpwYR;@SW?hfJW$TX#*4>Ab>|<#+ z6jc8KDfH!{DH(fdd-*uZ+gF8tG@YUwi+C)PR>Br%I-vgb=IrO+rl@DmTIT(h>$Ej` z9Wt-H!Ax1Hg=`+aV5297*wHdVH8Y%Xw>V*!KGf&??ES;;6F)|+l0<5fyz9}%Ro~#X zj%VD&GnUlT#0|ZWT}&rO^6B?;sWgdqh>Cj~+Bq-VOl?fQ(ci+0?V8S;)=aF}NQKp= zp%Z0?X>@`hXU+Q?O_&`?SB82roBmr(eY8i}w9C8r+-(W8{$V!y(~`$JoPR`TR_#M7 zM>3IfsDM$L8jEtzb<%T*b;xq3q}}OOS^D7Z2sLvLL?&Zvq5r!Llqn>%ajF7yHoJgs za4@7Fwr*zz<0Po4%Q_^s-jrT6+k_5YoyFdqBSJ5TT_GQR1wQ&$d^E97fwp`YVLt9_ zqgtYyX;89AO}Pf3^sYDck@-N6zix(b{wGFyN^)X&$n)NkmS| zybx+0qP8*!S*$SPhlqDD)+W=~OZ=1cnC>QIs40Wo4j!VqGE3;Phc9SI&Rb?){$b|a zWDU1*mWrL~4jF#&!9}#sa6b(bBBedz}6-rHW+5YlUcDYD9 z$1%Q{?TzEm(E~q`f}dPX%i9>#FOo_%_R2F`v`!^^z*@Bcnv?!=d|!S7Gd z+UZGX(LFnqEwhNLu3Zkpcbc&`?>s>-MyuJxpGN3k$!Fa#V_BU%OtvdXG3N+Ag_HN(XYA&=DL8p zFnoNP4z|5x_O$-tiq{)3fh8v>$2F3^Q=Gw+Oz%Q5u>)}TxhGuv3pVVlU68I5WWA`5 z@259sbu&oYP0%Me#02lHVS`3S(BQWvlrt+Cy-Hx{>pRMJ)p6O>F(rl`t6fc+&9rG(hJ1cfGXjKM! z8Dwdv*))?oK11+bFS?y^D``Zx7G@#^P{od{okd?3N7H9N;+P*>79&HQ1T9 zLkGko=>e@}Ty>wzC`LkwX`OeLo-1J4(Wm*`Xpd89O`|S48+sCX4_riFgCn?}X&cZ_ zr2wklDnk3lv>3(p$>`6l6LeQbIWzC08C&!;k!q|qMPyPKjkKGw-GaP=0E22Ob+?J# zrx1xGcj?s#mJIYaGxKL>Q-jyjsQ<})_S-->irB6|_bSMvgS!Hd zL!2}H_-_fiGUQD6`z14~mmH{a_za2~hluclX|y!wJ~|K)iE?I3Fr7E1)GT@a4(X*X zrmv!V(7MCbRQ`l1YVFdcF(+m+i@TKgkxn&?Y#&5dvsBomG&#Fz^6kXz-Xx6@^zHkJ z)$+dyzALAGJx)g&4N#it5$3NTFRy6FMizViW{xSOqxaU?to_Z6+(%8dXnsx!tB>~3 z7g4S#zDcm3SZs+Fms&EQ+km=8Dci{>ZbsD`Por*gE7nZv3VpwMoWE}C1^!O^9AxEk znQ93*GY!KcG#Lx=6kzc ziq8~n*-UlH9qBeUi!rco=l|_qf;5mAsti&_M&naaQ1&|JpG zC&qMOX$^PXbuk(>sDKRe6o}&|4dh|iijF?`ib&FLB*gbbeuYn1i3B6GG!j(x`e3?0O@eQEXB~Pb5kQ4Ivk~0v$C&zw*VNtWL~0unXpW&IJ-ng_ zEy`PhKCUi6$98y8aC8MKTHVSl^DLmb^{NbR-9ql*qs zd`b;>FnSUCeqHByvCV?x6kZF^v74|LEY- z73|PMh`PiJShTyEs!hGmimdRZH{BAM-p^Wg8bM*mv-lQ$CE(m+KB>@x=jlw4m>|FB zwS=8>dnM~_|CaCM{+8KbI7%n7cAyZEV1y&n(WHhw@*YN1zE;A{g=0^ZC$7T`AtCNv z<3jdFpBU{(yn&+ILfA3SWadwU5pzS>k-=wQAeZsW$lNrEJ^K3(-R>)i9#~&x_kTTt z+}8?NMYUmOgU>vg8)L)1DQH5sD(vVkdX>9&lPLS~YARZLq=43+Q0B z5dWgIP)&+d6RQ>62)`ul1DfAlDy+|>h*m@>(6u6p~ad03yxvTU5+yMU+GYTE%~gC znG@Y@!=aAafbQ#;MnhrC=*-*!zUap$X5FYLcJtFGuZLKC}Ozr>cXDsMDN(=1*~EmIbs{G$sy) z)5wcYNu2003-CK%1|A7M0gg)msC4ERCw`U!Tyo?-ZaV(}jGz9+dE=)BQy(XRH~R!< z-%Iav#Dk>BpMm3?&n~M-xpyFt$r=NyqAkGS)oS?8ToL*n6NP@PA~Dm{0>cL~fi=gG z^lsIH9|kPR^$p%2e2yAC=+1(awU+RaWDi*CpAFO^O2OpcpTMeQ75FzI3)@?~z?HYw z@OqqxZKQ`b=@bzrGd8clWnu9kVhE6)LsB^7^I~#hem(48)D zsFq=VWEqHCfxx@-l{na;35W%4UGQ&2lyqer0ToNFz|n9;GVhKGan~BaR&TZvMa!GO z_qj6MtY}0w@M)6xo(>WN3YBC%w|i&_|0!{jffLIruY`5bn|iIynnoHeo|oC#3@eIvK?U6k`Dao zC&h;{ZOC|>rtO+HnfRhl5f&G{1&r$!;U{)-xY$LJQGWS7nDsA)Gdl$1GA1PV95tb0ng@xw{H=Gd)QxKT&@?~6>yLVtEzycxiSg= zmw;7N^YEGv5eqaMIY8$}B1nHGMR3wnJXbw;bIHmt3Sk%0gIt*8W+D7{t8-_r*R&BHiZ&L--1VTWuPB!$2x6)!B}N5CUStR ztbK+RC%2QcZ{~Ayqe?;awRiaKpXu=Fry`&cUtTr7<2*3`Buj>04r60O9xl^Wf;sWe z!Dk1t>Oj|3)9vPzM3gxc`_{9FYRWPjlObO9Lq ziAZK>7Vmg%1?D~lIHPVMSw8j%@7!fhCYeGIsj`;De>;tjKU9Sk(q5bbZZa_O)PaeC zLf`=7NO(WCz@PPUc;TOU80wgk$(?40c|RUviR>oG)6}A{&AlLA(y5FEQvIC?WxWK&j6znCZMii#K0sm4p;pvzUVBXiCSbtQ@ z)<@Bti0|wd@Vg^G$6tgz+_D#zebV8z%KQNtYbpfu$|K)4^J9Cmp;#R{fBytF?BP|HEw}+zok<4LSy4c)VT>~^P@m*`tpxd# zAArGh6%uFr6^A(+W330)n5MkL9=#X9ytEnQcik=ga;q-cEQ5%iqB5y7E8`T4e#A;& zO*uO%{J>FozIvMAT7BgnbFRd;V!5%az((NNJ$^!o)Ot$5xm`vaFMBzdt5^eg#j~M8 zwZE`>1Asdnwa_&5CKT zC}R+NTs0*RA_nlp4Fh7Iqyt8N>jI&B&v<=huR-vqd1OdX+f|lnuC7p$C2Cfdq$8SO zy_`s_toIe)sB#APZ;QgiKZiL5|70Qmf;iN$osOkl&w_}G+3?=LW3W~1Hy{(9V5`I+ zn4>)k<_s^x(NprPRsLgm!=ak+)AgU zsX!V^|ME;ud<0uh0tKnZz+zb9mbK0I^^j=EqoPNlKGyZWJ|Rfs5ccS8CiF2 z^Q5id(u2E*N~bdXP%8rK*B!_DZ{=+d6CNf<#zE0uCvfap8jzPh19;AQBtfJD05T8W z-9Lj2*tcW;f8OA4!#cP%w-5pwe4MX9p1)iU3+^ew0Af!<=C|Ob55sw5&AZ8J>I!9* z3&2W^ek@t>3m+S-!lB0dacfX>wfW<4z@*K@ibrH2m$v~detiq1Hv~CO+ zZiXji+(_eu4tDm%Sb`Styo)TrlO38wJk*Pzk2~?SLK%`RVApmz+ym2ccHkM(Pw?Q# zd)~PJD5qAy2fPwE%b!o1OZ=s*$b@SKo>2b|TcjjX)@ewBgq^F4`u<&-K(zX34o zZXg!T(uDP0DkSZYB@Em`d3KTEWY%~KwmUZ;a*jU5(sK-X;h-ICpAb0EkEY@~M?1mu zOTJ|JPf@r%_BWP!DGE!n4v^IwgaXc1T<3g@q7~t43DrEM~ERISv25muK@cc0^d<*X)ZTqkB?nmoEA9Dd8 zp6x-DYaQ^`Q-48(tUNp>-iiN7)Z_JVCFB;ElYas8@d4N09HY97>hi6fV0daB_^U7h zN(~OcBd?~CIR6*;LU{lYDFOIP(+aXeDXV($>rEhthrwPFGVr&E4g6iZ6SmlHB36f; zpgmUvr)SzhNw3=+p3-|@?RSr7qx144{qMk z51yn;1C`~8c%5(*2b+Ea-anklkxXg$vP7LYWtl>QTEIKD<~_E~l7-sM+9bKx0p?Y5 zN$}hfAQd=)!@m6hoNH(CTlFCvais$otUC@bfBb~wAI*c~KYOuMln?Ql^9#&ys|5Qt zOW>Z>Zx%$WnUn6b;;`p)3}}3^g0$S42EYHi!1*lTsP0Pcg?`7}fr`>eUf_W@z_~*M zt`e4oRt>kY!}@vfxQq#Lcd#U~0#9=4feqx`$3XDHWH#I!Wy-V3wg%|(Fc230gjE(O zf(Nz(*e6aI95hJ;``b=p?m0Ku^*$ZEYL5aYtp>y}nFD>#w}N-G?*W^l6kh1KFqrr> z1HRa^fu}d~7x>y4&YLq|gs5D(jhFey@gjtiIGdh2k9P%!S1!jD`+eZrULCkN_6g{CE1-Fn=VE@B9(la(40zBh2Ic%#61~I^U^HwySna$9 zPBZQS(_c!GqJek#C?3o3 zz|Uiqfp(Ke^$bsz_ta?J^cVQJrUi&cYmy+xa%`0p1vVNbV2$`YxK8{N-dRa_wrC?6u6T#* zlJvmwDh@${ahlqu1+N8k;lj?}`296gqIygZ7} zbx@O(i!UQ*pE;7L1DA1YiVV?Yd%(;Kg}m#RzvIuh%}8g+LXPwAMzGCNg*UxWaDPu5 zSD(xs2by|XBxH6UewaRm@O0g9!qJPIB_s^2^{nEAh!$XnZ;N4>`vrVbLlbJe_aGY| zDii9pgrL_#xb%7-?p3Yl3AM%H+PGVs@qYqtb^TI2nqy1shnDh|pGpRI__9zUTpdJe zMF9P&uRzapDZzhP4CWo(j;*SN$d3-eoG&Ryvd&`g`dKV~nIQ~Mau{Ma=nTA%XppS9 zPu2e2UZlIK($+s@0$1sH=$n@uW?s*rc`x}>73wtC;0^I)HO1$eAvOJ0pWz*ZiBoL%@8Z<)Akv1$((6cq~cmv_4YDdqNEa!L-o#p zyIS(l(u9E9XZXB_J0)OaY!&ZbwJ|n+ArE9Y`f&e22G-5dCHEd%fWq*DybbpeG+8i{ zXe_zONp~|K#in}XQ@AqRHy8%i9W=s6wq@g)j@LnszZ55_@hONeF2(kLnnBxo4m|oB zz$gu6GTP+`M++?k-Cc;}ZyCblcclRlHHQo3_wuf^*pnG=_26uiSD?xV!q@hW*xAe# z{;JxF3rAPs5#jRO{K(T2Av=JU(&B2Z&fHliKVZU^qja^WQgNa(#&+?tilxKONG8 z%dAw%?ras_*z zlV2QQ`)KJo&MjF@*o-7e(xM&LH5 zXW)832bf&Dl2hAz9e)^z#@X+U$k6PmaQEh7Jh*BM*A*zhArp6e^3O3S)7pSN>whymCwxZn}IH98B2^emZ{z za|)mERH+9J+VKdaftgS|S&ldrsu0)mmpF6JS$y8}8}HzQ4L~Y+J@EdQgZDXoHz(ymk3OY*IK27HmquP0u*hKCUM4LyHMHV{bq{JUa%Gmdu1- zl+*|gBY4i^L0B13AsoZ+bXr)rBH zab#~GiO!eEF-x^0*vC`Cd1B&T%&+b{bi{0~9Uix~d-d}URr{O8J{o_|$WSTN>t~HZ z;!9Dv`WEC%JyBpl7qeI75OeQgH!AXf!CIX0rSeky(7j4kBxbjZd#2ur-SccEQv0=$ z{wEYerBW?v>eEX|bn|A$9avHy7el7K&xTQg4^Yy>GE^odicEKYtJV|Lh@tCgI(XaLCF`WQbos+#+;)G28AXaDmK@<%jp2 zVm99QMbCQMnf-P3tQ0gx%Li-V&CW0Ac7+YwHk^$1$LXVX_i5;0dlt3Yp@hPAjWHeD zESS}+CFrKrBK%DYeCQ+LBwGE*k3Rd|LcjE%Mh&r-kn6J=uHTn7+AFBxNF~YFXnT5~ z{jVa}lF+j#XHh$nyT2V}Z0e#BzJlzXL-SBab1Xl3-!U}hyL!zzh41w4&$m>GD~VE+ zc+8~@mGnii2I@;)Ku-tqnOl_%yDYL7MY1Z)pT{|j#3yOAKkGJk{d^%5x_&iELzZaj z-8htR@e)#zt*0-%GiZ$FKAH{%1y==k8oKcd6Wic~?mSse-=@7p3)-rY&*e)rs=Nw4 z&0T6Yrz4zY+{0-RT~;Fjr_-nNG|}GvjZDtw&#ca#DNKpiPX3_67`iboL1UMwA^U_* z`g)f=9r7)pm3O}|Q`OI*4S!zOw`}D$WaJ|C*j)5wxRMc4 zDxz{~Qj~{2Fis|;j6;wLO?^OV+TeM(vXx^ZTw^B1Nk@&RdV>t}BnA4h@q z_O$75Iajsx0$Tqo6Fs*&hLZ1TuuG#H>@we6V@10rDc9x_vbL&Vuv#Eu&%I$#U*iaW&aVHr61C zToOM(i@Rc(JD{6Q)f708=e0q9$yl~RGm=}rAPE&+v_NZ@J5r-xy3`?Bo{Z0TfUh_= zxkhVN@IzJYYgEE6ala@(MVCi5q5{u_OxDn2ddD~$eLkgz2A2)6PFMeJ_HV|16C$ilWiFmypEg7@9BC&LkeWiWaUdW<|eFLvucULlFh} z%%i?E*85-tqjcghlcXo8RVnte)4lTfXTvWu7TO=#smV3ed4rPOdZTu3^x~z+?bk7C2;H)G%%{p3weuf&1{3EmPV*t>!;9>6-g< z<7rT<0bQo=$oxFG8{PVMl^^W5m`C7t@mpAoIZY~(h=Y!?ne0his; zloQJ6bOT|m1&$h~VI93R*^OS_*~2_8abfoUcqw@IR-t6?LCXJaNu8c$qdjw~kZ$jF zcI%c}z8TM&$-P{{I31IuXLC|f+LRk8|4$0F=&z-#WfRbp)ggtQ zPh7di$NU=;Jxzlx9xCO|e(1~0Qq@HpWzQn%#BuhK$SNu?C*X2-WbwTRE2*@*GbPdh zm0B!iKKAe9E2YVyP_KS=v!)p9Sbf2Dqj&x0tYR#OG zIn*NNBrAnU{R&reivwd;_vvTGKI_t_J+H!iPU02|C+IKL4 z1)EdoIp;p)w)zx8VcKlUyvO93V=UXzR>Sq27yP!Z+ z!Y+9766Ib!gvz9f)!P&3TJSKMU3w3_VeRRi&Ukcuwxr zPNR`SAxxY}6}NdhWtz@>z7l=QxuFLwSP)83ti7NtFcWWO}J zvPP7h`=Ok5{aM0D9ZIEgPfC#eI%8%{U@>|$XhYw*U7@hggf`5*h~7=HL#;MtsEfo2 zJk2Mmu~;%)_^6&U}u3K6@Pjfpf1>eA`!)oV^%X{A;V zi|H2ucN^aH1Dz}Nr#((z*xmoC*v=&`bZ>GucNba5Sno=vkvWhhA+;smic}g!( zntL9t?Yu@{Pj z8ioiQGaS~Fca*+%IL@B?3b;?ETcE9@9JHZx0n+o*MI*c0*v=b>dXKiTgC__h^InE> ze+qIY&CRH&P9RrxK`KSRrI18!8rrm>n|<|sJzG$ik6!gj*7#(sW1E$9P+0j-#-vmk z$v+=rs#g`Wb*DVgx#wN9{?J}>d-Y3pZ{8DjP5uB%-JpT?tlNV=`Ol{HpRCZyed*MB zdOaH?GuzJLKm_8R0qlo0w)AH8BzyP5bFQOT7#&%8k$a~z6jgq8X2s@kY1x%2C@SF* z|MTHKM)gfQyZ!M|&wZbrJX|{8?kaaytDrMIvK`?6esn{88U*{_V$ek?RRrbZqe=yIcL?T)Wguv^3nh zhTk?pCM89f96`oedR!)&B7K#Sd!~cdZwX_9W@)o-hW%{E&nEP~=n(B&ABV9_rBP}HVq=515~`eXbMZh)kh{$Vve@6hpRK`*?XBgzjtihPen(?2BF=MGhUo7my^f(=Z_9kyxoGHrAarwuzA=~N_D^FkyQE1eXlzsXS|JmDSre{bLN!o3uLkkbl(nU{c@0sbS4QVi!OS_S& zv`HT5L)mi1eA~jvvZuFs@Emgh2($pLS%1gURS6LX)=j(5?2YOf}*(vCIkQrsS zegDdCXkN(piWk$qRVPuSS2h!VKMd`wFlO$In4^(d#>~#!+32^qZcU!>OnN$e9&1ys z!mOI!$8Ht4LrNl(k$PS<(rD0QU;cT9xKFfd5-tfiQq?Gd|56my1WMDJs~=JIUc!8y zoWlUUE9ls{*VOmMX?Czn2mNX8KwEd!GiQP`>G_E+G^6_f9kHxJUGfa0FguE!JEli3 zd_6@6_Nb!tP3^R;!xc65K4wKx0lN3Nky#w>#y9aR5tK;u&>1gh!PTqT^_M@Te}8^Q zM|A3$=ey4%g(=pkGG+^FmoR~jeG%dIv>swq-W8xY<3RMW%AcNIc!`a7Q(;m4*f(fw`JkXzb8?o;QQm*b~PjsGMmP;f?kW62I#cc6UM!NBWvN6 zz|`#CjamZFGAj#m=(LKPbRTTuUQr&W=0CnNOG6?VGLp>hKlh37TP#qnU|oD3nha|W zc_aIt=g3SoigEhZ%04_~#ty!`z?-QyzeHNAVsh}i(5!8P18oRu`oY}c1mAjv}jLG8U(dSg)!9LVRSC&6zR0O&F zJsraIrL7LD%r9f(&h|3D#0WR^fInldD2XoLKSJ;9&Su{gRde^}{bofr@#j(-+Tz!3G1`1 z?}yoj_DDfSbv3`IHjoZ28>M~S@o2}0Ed6tK3>|)=gEBAMpj}S8(NM3jT|>`zzKiP= zWE(!p_G}eL@;xR<<#q(Sb6y~QBgn2ka{4NIuYQ2lT@yj~w^dS&9q#D!%iriulPOhj z{|{LON+I5qD@ft`5Sz#3vYePn_Tl{nteDaS68U)=UA(*r4ZTmKxA$<7{DpAp){)Kr zDGWzHUx_2##aUdRg~K#Le=&8p--<+U6vFcJcj50J@zhXh8XXmQwp)QQJMiT!3Royi zjUt4Zm9>`W>&$g@mP8)2`V|KmO_QbUpMNy8f?;ac`5=`VNqYb2dE`6$0k~ptjQT&^ z&6;a`r<&tdl>f~Th zsh7kWCbH!Wy{jR{jAZ7cxpza+p?F2R?-gyV=j?Xw=1pEm@3xrTRl`S&wn+_&E;@rO zyYf&#+HT72O<|m;`*2^Iyg_s2iczB=Yn)e-DB#hg5zpm23O)0Sx!*R*-Uyh&Tz$C{ z=@3PFL8O>=r-VBzl|8p*a}RZ{>DrU1vdn{CeAR;{=TAivuMRVd0=n2z zmm&6hLmBmcevLbRK$LF&IFtU>K8z;vSD@t_6Qq3IT7bAbLt3uG%#90+YsMFEW8Vk8 zMt_=S^5+;9BQR+~rKQ@?wttV9@9{pg_EtX|7&67q>uoKXS7|CZJ5gXK?p~u`k6F+L zW(5+@G-f7bIOIgKVvVmhWv?fCppclu>?=<{svTHQkH#T7<-{1%9GrlLzz4)%D{y-4 zh(*uq<{%HVr_|$aD($tv=;2Fk?$tF_sB?0Fl@a=bic9X(%KN#3aU#mTvwb7L7W~=D zjwaye^jr$m$`KH3sX@NhW^G_*%?_#5tUmv@&6e<6gPy|u6+yi zt~Z0-e%2(xW;R^$TTuH8H6j%uT{veu;QiCJC%f8hh>hNHAiVA*INN0coTh&T*=x0N z=X`6DC2Wi<>g+jXs~-UULP_%C{7ewEg99%N7lZn#(fIhb{XpTn1k`>$5B}YwNR(a0 z@#Vi~@ZSYX;Ld}?U=ceVN}I_Li{u3S;`bYCmB%(DJi24S<%#Y1&d+k3w@M6D<<)_g zE*?D21_!L+yM$!A$b;OGubB72n!LYo26&OFP;Kr?xcGP&$=H3Qy7mo2{0d#5o4*H{ zxachS!+C*v00r)?-3IZEf;qZzSu(s_zE>N2VINN3blZUl5MJtJP^ADU@I?Rcf7tJ&OZ3v8JO(m`dik#2w&cy3MCIG^H;9`6mZf2xO%4G%k zM|c1v^;`s#H-f-Fmj|4h%2)V9{V&d-$t^s2OabTrF$V>^&4BaquT_JGFMyAU^WpMH zedzQmj8k{@7eK9rU}AYN&$qo#05lGQn7G9xbeSPGUA7JXe)J9Rf4m;T3|AO<%#n2L zm`PMNIKt?P%i!a^8a&(^2X^Si;v!WA*fn$oCkIZ2PiD@6PMD9)nkMkOy+3f}-&(+Y zyUJ1gHU~)dJ;9Quk|d#d4vGC;%d-p8hXG9#k8NEE4!xRAtY@yp&mRS0lk8n2EL)O% zfgC8eTLwVPsZmjH0IRGz4x zyag8epXA-!bquZvRwTB=3ecv^9{${325yKc5%XAilCIAo3io4p<4dl9?3jfxZM7bZ z+OChqb3Ay5$^+r1HfdPX;D!Sx?{l8p8^LX&gQ>2xN3ji1H~lQZ%n0q}LDN z_(~D7t(<4u+9(b63f-{#q#O=_ZMewp3*M$aACfIIasBxiP;zTGe!VylyY2nRsb8%K zLx-(lp4WMtnY0Kdrl`RFEz9u9$_v0~ z+^f51>uLQTNR5?&6SBr+w6eWgp+Xt^iK)VOZtk`PPHOmcYz!w%p9c)x#<6SCY8V?N z3g(_rC$rPTFdesqN9=wG@&*&|%Z6Ip4sLT5$GQY~E;z~4f2$3TZ!5tkwSVHuQ5P^A zzz6dlrGb?nDaY);Ssd3*I>b`<9tV$?;%mY>B>Z)~ZP%)1kh<_8$j!?GN{1nM)}zcZ zSQ5#z%Dm3|#}H5vAA{B0T*wpMXuNFWcIcV*7lgfgg-_kOfi+(kLB+djV3YnwER_C| zw=&oOo8KM7`|l^=sVA=Tw0;yWNXvSSm-LvE!5=U1j?NO!%r!H3aeHSG{aroSJ~ zKyRx3BE~oe&s9{r%R0hOVjr;6;=Q=DYl3rW(MxQ3*PWEixPUe0Jpz3zhO7GLI>M70 zOK?rf6yl%L3=E{#!MyjsfrDi>xL>xOCtPMidLP2e&MT+ePKb`8K8DyDwsDWPNv98z~|-L!>C`fUQ(mklp=Ar;{O8}mUolN`rSB%#pk5DTJMVIS-2Q}PnsYf1f?GjI z-+p2f=}zd>JG^q6E|8Skjmyqxf@=W*cto*>x2dv*S8_kt_Pod^5PDORq|T5c2iBzH zUagrBFPH<$)H%dBawYcj_lH-%xR8>-o4DLe8P?;A)xpm$@E#ov0uArnf$OH*AT=Zy z|K06Dwxn+a-&B=BZ*wZ{zR=Ta{RSaN9 zML$?1D+3H;>u~fXc_O##DhDn52(-r)NO%VWFa5wEQ!N^BCimew9XXIU!v-YG>jV$x zZowMISHl$j`EcspqvZ60x$uWE#p2!sH%d&kRv7Cfzv{Vu)1v&<_A6l$3>@*l+IuHtol?~7_15lY>dc!oy(xPHPUv5&rh5+ zZwNn6D+lWYzqdYY3OsW>n5@3B3d>>1EGsSQh%gTd?7gg0L) z6Lc?)1|lj~anc1o@9m=z3^(xb_uE&1;p-$U^x_>S$MPEnXEk7RZ2{*E=OXqhOTrt? zC2*82vSsUEj_}Sk8ImD^f{)EC_UkX5~GhCi6*;)f$s&4ASLXlxg+CunN- z0tyEeNoJEAm_5x2jExQA*MfR!=%PVzPAmnF*Vu#DwTM{ldjkL{!Rp@vK$Kw`SpUZw z5A3A)^W!8O@UtGfdsYBx!Q5SIJi~mBW{+zJs2R;dtXSOETw~5%jbv#Y$F> zfH~g)4l5P#3}R;BiT^g?=i{dEcgrQ9yU7!KP8-A~A&bbD6*BPj%P7coU0lsIe8I`x zs$1Rh(OJM=&xcEgg0TOdVSH$%4LPhJ3!8;xU@u_f#N-9cJQl#imQs~5Q|AgJo&O5X3Jm2Sm zhx>1GevXXeGX8#`?%wbI$5zC+49k&pRXzA&t|jPxs7k7CF9EZ*_W*0zji5Tsj%evs zf;&S;ad)2qtQwI8lMj8vk}qB0!H)~!;YdgD`qClbEYO8XF-`7H+H(9My*FS+m_5lJ zIfAphoWMwr74h@f2;R(ghA+Jp$l_h?SZ}Q-**4#n1b+pfe&KhB%u z5*b}!d}KN$Ps$D1CRYV!0%cH2)KNj6TjYy?3J#-Q)aKkrDj!}BUcRg;x1}@ z^+#)P>5>l|C-p=;dVC_xcq)Y>c9mc~1t(zd*@+G1+xU}D3&B-IFSvVm5;(bHm-`AO zS1|9vIGC`v0hG;K2U>zuaC?0^{@y1;W@=1_cC)ixyF~z2gx2sEO>E{X zA3K15KGPsGCH*j#as*9LalrJe6F#YH4mYi!Ak|H`kd?gkxo#klOUJ8?}} z#IH-xfl4tsz)P^7FD1bEqk}uVsJsfC?GlGhBGKKmt;4wbnj;nqo&=vhSVey8P9>d# zdhqDGXF%d~3RpAa7eA}R8XM+%k$h7H@{LDC4ytU<3@b?bGAqGdoO)d`^oK`ct1ebU zANe!aF2Es=qVSK7wea=k$^JJcncbG0~@@9%PW#!h50&wym^?7&StPm()a&f!#70^`yZc;{<5 z_;qL^$%rq)-+jJ;IlI@8J~1;WpIXN;VTZ8y(M~Y$$xkpII~Q9dhjWZYer5B3ULZWt z1U@~VMwBNiz%}d3@U3&}`E#Fpf=N;bu_1}aMxSM&f88PQqG}kgm$xF4#s}++UypnI za?HV}XV?=(*(u~t*GpiYP|tr>b z^E^+9EQ$FIUJI`Q*ISAxK>je6j|HC@+ zJQ%jm4w`NG0e<`vG2EUd9MzADIFa06{G$sb;CSdEpr_~!E^NBOr@y2@>CGPS?$j5~ zcBBCPL*v0Ojx%T!?V?M~<`8p9b(my5k^CNwz|A^Bym#_LPE^Y%SoEt2m{*#@RV117 zb@Eh@<~*HDY3lop!CP5E3tv-xvPTA@p?faetRGQ zAHe}Y>xCshGtUl({P~V&0Uk^#UJG73{;Abi1ZK(b*nKJ#}mr^HMPG9`M%&%hhb$+v=@5n99ni=1|*%kl4HNggKi z=7D)G%Qy}%)jV3K+vD4THe`+eYV5mW25FWxBW{S{8|^iPLF%e7u1+)u_*4^b&|HR_ zzkLRId7ePxr842~yTlJXZcn)k-%2EX`m-x2>y^sWJv1{UujGVD*1Yms`oWGJ3p0k^%o!C)tCdh zTPN}dnk3*@U>aVRvJJl()4*54dpO?i4{`I!0~}{JX}IjqRH*M>13uLL!&zDYCk~wF zNYQf~BfbRQ{%#_1ez`FPk+(QHNNDx6fM z2e)eKVx&@oFKOFy_Ki&>Biv>1@a?T|p`{jSD@+7Q;ykjw^*iU%^!wOi<0R->5F_&a z8Gs3z9{BlB4Pu*VN>=ZR!uj$kxPG<}yD}_}v^&KgUr@qdorE#Aw1>ZM730+lH1YND zAz;4N9W<&3f#hS)@Tm1CP?T$oo#ak}M$-`fEjdH7s7#e(y}2C7xLcB;12V8=1tR(X z{l<5Se`3kDOiroVFdm2;#bE1gyydP0jA8y_g+xyx`R+77)@U2{aZkhHI+Cz{<}Z&! zKmYO-&qk8+i|%;9{1aZZKM$V`4+P)K#L467<{)Xj8-(6ogFDV;V3j+sFmH+*@!$Fp zcXjrH+j=UbV7n=Fvf#n!+gJFRd@J(L+7kxfRe_QrZ#Z3P0>BMXhqlI}c)w~d2>N=M zUsI(7JnI~xewHaQH=PNlhNR-#wp(#>@IwIfeSpda3Ak~QH5tDo0|Q<9LEwH4c=UH4 zl;~Um@4A^1vECPeb40@9cY6xB7B2%;#VYZiRD04lH4B6WOeNFapX6^@Z$>QhszKYn zIDYq1X{c-G4p#e_LEk7x55ay>Uh2POzQHm;E^aUq`GEU9zH;8)6r# zGtefT_>;;95NEc6c=Y#z;*_h{uxcVsnxu-Cf2jk1x;>#yk0~^WH6q#*mO$N;N0{`> zy27FhMQpPE6yDJ81gh1;r8?o;QE=)w^El@Nv+MUxn(KImJsTm#omnu*9q!elb+c^P zk}(~0@cABgx6?JcKV63%3YyLioLtW2G&!Mj6>HJh=2A3gK_}caL!KI)N@H!r@=%KV zTY5NX5fzeQdg@Puh`;Gzg0Be$lYKAJKf*uk+TW@|gZW39ma99FiOA_O>!vuOBG*F2 z9v5Lu<57Cy`vr#gMH?M;HbuT|lbBuPJ7PXQqpL^N1d*HTQFDS5P}Cvt5;ko>1{x*+fT4YTvfroo2`uOMI~h0(uJ;sB+*8zcvk-W zV#=XHwt0p()jqqKO6R?%&mEct&hw6<_wjcb75#_k;(grL^6Ho>pHFdVfbilRm6Ol9>V@cyCq1s{3*B)D9Nq|Pm#z_fG%>7(B(cK zwF)=VT?dxa+p|)Tb65phYFk4Us(Mjkmo2(q|DBtwd4%#`K4)i{MKgi(?1Ve2MNOr$ zNsQ5xWZHN#0tL@VV+QsOF!Ib+TE6rjo2)pUQI&0?Ln8n0QTu9k>SHf79wS8JPwQ#3 z<21Bsw*w70e*i79UQCVpO?W%f3W1_u4)>L4Zr=4)7V;5)MOUXyrqUamsZoWf_j&6# zTJG&MWej`D_`1+950v?u%lvml3ue$2ZH9S1YmFH6s%2RGv=SLPqLF3=B!s>Rbg z?~RaHLnVqX*Pu(Yzpp7-Ls62}#HnBC#+_;lBHA%=pkfZn6Pj6H9!kQ^ihH z+o?hC?CeK&^J6^aJTq7{mpT38tzgY`bF}Z{ z2e2vN10v<>jKP1sD0}oa^`wAKjd{jqdupPyCzc_dTUo6At~lnq9r{-|_qE1OgCSm55;h@@02n3rJ_g)!!Dkb=mCVYW!L&Vmv9 zuX%)PRou(!OUMeHmL6imo<3(?jwm61`FiSQXpD--XQ9GreQ?&>aMT*oLRaiFqNnfG zfp0B+u!dj7R&};hO|fERT%nEDAIxP74_xNf+vTwv|H=u@s0z3T7>JC!>Y2(<$EdAa z5mmmHOdpT=po$}o)OhGCBRTmr{Tq@;^N(ys0rk3cUtt%`S$tF=d{ECQs?FxDxRHWx zd7q<+BIn7$SV{$=^}i_Z0DXJR8j&U4jLq~^R9!D7tbaFwCy{!A{qbBEtvsiUywt>K zVbWA2$UnjgRJ3^^RV}EMX3`0VezSApqG?lQ1^Zuk3o06$CwTMc0(&mu6?bVB1?R(ehgv-Exi z-JTYX{+pSEc5J?lj_AIn=VQZ=tn@+d*S>Vrl5H#78gV>XC2MDjM{g2u2Q^qA#xBKsUHabd$p&ZqrW@8}#cjlAK470tG(Uo&y*w3rR@e$0|rPSoV) z2Ii8i9i6-<9$DOMW8c|bW{pOCY4|ftkN(IN5Qdt_zrYwjDv zJLsbFN-8<}hBC6ltX=Xi22HR>jlH4NrfnPAU!lUDw|dRw!s$r+(`wq%vWllIxkxzw z;Azs$wo^B09<3`XMX$GqBLCMpNcwISt!>U{LEJ^O^6q5xMnj#qcttDioO2bKj~gJ1 z)*QB}I)i&oT$b)uf5VPE)~0)uG--+ri};4B$ZL}pitM>4xPMHU8a{b|&U;3%-YdvDLb`5j3%wz$O=UK_fAVp7d(1OgY6{H`*8(cpTue8{cSOt=(~qK zT=RzAQsR&7GV+*=01ejLRSntt@1?mb*CU4#3Fgzg7@B5cjwZ-^QJU<8&ivp}Hv?Uv zLVXtZvT&R&^LIk46MC7^6{&3H*L?Oxs0#`#dxefFgfQM$+gYz~rfA(iO@#Cec}I9L zbZVRq_u%zAbY_q}5^rzjx@-iD>y;~_`|4XX;KXV=CvYcybm1}+7F~&=z!p??Fdf}V z_(q2pf1y70>saFr>lp39ZhGzCRkU$uDQ%R@MotIE*!7+Q)Dc#T?q`Lg*D}Y@ir>jJ zCEzrdTiZy-WY4jTc^uvTZBSq`CM}q{QJ<#7FQ%`C2f(AwRVcqYkor9k`CkXx&@(WM zn3Bb)=UWfb%~;My>>#vwFzkC`9Jz!%R*{2=QykUONH*Y-Q_t8jv%WmW%kz(_XrC$gf`DrW?e($`qh9(@13K_*QUy>wTv2dK_K2 zb``5U=uF=p&!7i77P0UB6PRR=*X#=ofAq_1I`_f;C1{h_0dAwQHc$NF6=vGxd~|Z7 z5zF2)!K~g9x~FR%>YrPS+P77roFx^ESc5ORrtZqlOhMGd><=q*a}IU;DookL1izlFrFK;1#NqC(VDy$ly~vFVEUo~u0gjobNRL@o%fB)y4313 zM>pIPEEI3%s_TiqlS3QPT$7D-&9P2e=S0~^?o0OR)GWF;@G5KL&_>gblu;SsHTL>$ zJKCosi%N9WQImxcEr=V!KW9mxFD~6|xU3&FOqAyNm;B&LzOG^JBsVhOpD3c!;w{Wz zwjGV_R$_jyEJU7jyy)cWG9=Gjzk!M%Z zt`~E-hvmPZUB#E#%+7r7m0VxC?tG)j7xs>M_l&Y@`wiHUjx-wZ{sAjJ|2(tpP7;$A zkjoC&Jz;kqj}cr_|1Q{Cc9d$4G@<%{O(@?Yn^E^XN|iS!Q9pNk)LGHTtofkGh8S)F~IO5 zN4fsK5v;+DCET_1GZ?GSZ|Ub&ALe+`ANEpJDS5AWl2LS4MxR#p((Cfmss9916uR>~ zBd*mzlN;@k+ZJskW)gwgi}H}4`+lnGVt~qSTGL^-R3>+YEpJZ343R7O9aC~99ewUO z$M||lQ-|>BC_v*Jv$fw6?c4tyz1$&WQ#Pz*hNQMI=j4vlt%|11m$+Mk4L4fY$0KK{ zv&(7Lq+ktI^0-FT?$x2&9d^PLoir-$6!01kE;6|+(qZ0NFNd4szD#7H?4Repuze`0d*uXM+spkS!n%+undyi52 zBb#2guR`Iu-qi4~I{D*dxcg?nL@ z0}8sTN55qML9^W?c?MTUm;-+wqbd4{w8-N!P43KLWGvm7sHZh-^?ftpCl70S+NuOS zc~ZtCeaS@ukG?YDCaR2IwJH1ZY_?!i>v?XTWgv}MY)T)Tsbqm(4C>a?W^>z0M2S_>VA^@~dfsOe6Q1t}@yn z;%FC^Q55=YBfX;j8}*&iqaRwn3Eb|s(=988n8Alx)Jf#>T~YK64P;o*J%N6 z$2YT2v<1w)F*7=iNoGzJ#iIwJxs=*Maf&7uvN?ScGzF+riJ@3p23F9SQ9krg-DI9j z-3yd@+JSp=vjoU#}cue3$poL%dGvr?Q(q$p~ilu8im3LC8HGp3IcSta?)`Zsfl`~Dx6zEHi17lt6NyXHpg@x8_Y-yVSy;{AM zuFgA2{q__x-DIVjGLFO%Itl4oq8Wv6b@U@LDO1Qd82dIppXJc9p{_y zEc7gyA{}$YahT88n)=cg_@yAz?F331IWIbuYp^dPrRn>H;#BHJ8S>S ztLV*l(o}KYQAX==B|Wm&fCg-X!UYc!P%Ezn{cath66L1!yUtlOE^(Eee4+puJm)hD zW;dau3xnxgZ%Ot~`%OXo>?-6t`y12KIR~Z4%t!s_osl$qlA1f@)3^T3s5{D#W_L=` zg6T>~ajiJ&)JkG=ZBx-RkS=U{uG@l8-`ybl=AsZF6|6%#@ zpO~7BFVJ3FQ$3jL&WT}sM8DxLcV5t;d&yiG z7Y$luRKcF%m9c_8S88TZOjV~$MtjTp;JSzp>^awMwDox(b4Xl_SN+``#hsjpa%%FZ zWI3OHzZi{{uhgWsathFrjb~~3P$j#t{}|2x#Al4!E9m}^r4&PT=b?cOJ6Y6CXNEHwtaMe8iN$K%OL&arObZ*5&HVC z9MxVgCzQ>zV6q;EvWh#VpnQ zy@Xknb3pKk?PVsm+W zh<4B3NYC#cWoM;#AkiMi`VII}jg4m+&-p$`zfc{ya;$l_?+R&J)-!gtr55dJ=wxo_ zZKt)9%b5HqS>d3636p1TkL>bA?|XcgGEyju`gzYn;rDvMYr|S5!5|q}wH>947p!Nk z!yTwu$TajQr;>SQ)W|-k9%%6jYcfH>0oSH@p~+ZhMazM;#AY=}hqJ zZYMr8wwf0?C@tp!A(iFZQ@03*`pQg-*zh5YG z4nE^^icdSj$gB>0+42i`Q1cF`wzOi!V+y2Op@1_Zmw~Rud7#!J2N;Q*Wv+HM_(|i- zy06pR$p`f4LS=xzA+=!=YE5lIWnZ&BapoB%*55y zhLaPq0KT%V$8#q?;&gf#fuk--{J4V2B;7F;Q|}odua?ihv)-DFdbMGDzd$(F*o1vn zXhE~MwYdA+FsS%AfKUJC;44&!cO1IG`NBHGcbA2Ds=yV`D82@yqCK&TcLUZRT?}M3 zLOCW831ohu1sNQx!|2Zk&ikM5ac+wj8GQN@^ozN`p@|JxLoyFHIubnN(m&9*Z3g*- zW{_o{6ZvK`GfC~Z6Fg~Y2aO&*1L@;RP>e{!!^8hPIJp{7)v_2!cOM3xOAO(q9Z$h8 zRy5s`-i1?J85nQ07|dTC#~EGTgx93aCBLrOG{A^qxGCP<2n2j4^lna6f^nzg8BLgYuLd7CkbyIbLm}C)0;Vyh zq~yzae7Nx@m|uPs1k7{9l4c@jl&b<+u!l<)%{Rh%GwVJ4TU5XXo;Tb*-4rTUijfH+ zCxFqO@A##1G;z+`RpW+9Cgk-o8S*I13VPSL!@0!{c<)hjX!myku@-y)^`ib) zlw~h|s^dhjl!2{+IQ55*4R z#TgUGL7!WkSw|sp(Ol`VYt$G%U(9h}Vp?YIu zo5&tE50}k<19X06Ve^xn;8ea7>DZ#dUwHfx$Vz(&I%m(vgJSM5f94~P(7sZTB&QEw zjCy)>3*O2LzSjjp5F^-t!rvjDy5paTaBD2F+LY2{f;E~;1pc(T3 zi*0v+gEbi6P0Hg3^jrgfJZ^!`{lBoKk~z?Ds{nhRt-%tL$MA(^QpDRpg}5iWK~pVt zc(M5n=q_%-!R7m*QR*`6?4ezEI$PAqRC>Wtd8Y*Huf4i%xK~4T1|P*iqQ-CcUp-hW z#U(kxmOwus2~7vIJ34_F zp7tUEJ7vktYD2!v=24(urNEbKG=~%3aEWp6Kagwl9`ADyBNmr`;ViI3;Npmd zV3*(qP&~7bm=3kzUnUO#_;Llu?y>O=eCzbrrKg_@7 zsf(S)<;l5gJQ(b{65gu4g6V{G59wP`@ZpJToN1O~q#*k_UK@}A=UvaiPxeeD2~!?{ zoGI>P#f#1O-5E0&T7C_~qnDPL$3vXk;)6w$_=E3CdS$Z(19{ z3FB>;`*A+8TjxTcz6UwJwG#g={*Dv3E`?b|y&muMR${$Z6~HSw9r&J{MGCkzcumAd z;J%iFAH-~g#{$Ih;a{EjZsIh^{h~+u4X=Uni##ZEMg?l*q>#o<(s1M1IG~q)7Mnai zfCoc4{IDK|%#L^lWIDEUT849gagR2fcR~rGNJ-K*pa(BGPJ+2=nxrO90XE&Tf*$gG zkQ-(UtJFm7+52YTx62Xw-H?PT*XnWomD%Lsx*_~yRu&e_tizKlUU}r*mE&(-t49u= zmxUMgmO)uw54L%H83>OnL!H^3K+NhYo?7WbPCs>kvMu{S_`wHQLc}DhhxA}I+h`cH z{vMusbqY=@35{qg~~x0~35+fcLsiR0U z-?j()j^6+QpSp44mwKR&eBk|%WQ^r4XK>{|00Pn<0Ai2f&h&7)nSbyWuWx%GZ3&`0?#fc5BgQHwVxM1 zUbNTVOVTHA^Zl^O`o-YU%6+i;vL^gcX+zxqq;cZqUvVnF_(D!wIp~WvBIPxbWI7W{ z&KNn9mh9_brkgR5DP00I4$Oh2!%CvobO@LfR|jUqEWnpfU&Vgeao}OlWb!;;5o;iB}ftnFqZDmWC_4ZiR_d35RecS;@Qg94F@Yo5|6 z9G+`RB-a_kk}y3u!-9dfNxyOQWNR|z#w~nn&j7wNs}M8op0HXW7w{C?@QXwlnDa+; z?{3?O*HqsH_<<8_&(k7yCE8@AGMXx(td5YJ$^=A*enaUI9M6}Q*$Mf zDP}OXwg8+_n+i+cMFQc6NYXHcf|TP7Iq}^U=LN0DEscR#Msp2(WS|JshtJ^Mif;JS zH$zY}*BnS(I1QGFJjR&UjlIO2iJg}gxuKhbk6E;WK3zlDw%LU2say*BheP168;|hB zR+n1)#7sQ;p9;L2q6Ko+?17UWiIYXu8%bG~GgLR94lX3Cz`jpz;Ovv*c+P*KzW5JG zGXGII*nB`9YFX5HST(%BYKsiuNUjmIxw{6wYa)2^IZJ#$x`8iwa1U&V6`gAj1;WNj z)1lF!dwlq%7&P1ngR8_ZaD)lv*zl(;d?|xDB`4;PYt0S#uigZ*cV#9hFp=dizc`i5 zxYfbA-68EL=1kDrcN`Ch98(6y1>lCz2eQwLI3}aEWK?^SXrAOMm|-sB^1LO< z>II5KS$#iIZMw^GEZ`FV57l)u-Il=bP8m3Gsws?qtpJs8p67@MzvZkOlOzi72R;0j z<^lQoaZq7-8(2(o=j_&;#Q*#7HLwgbhfTZeiQ0<@vOU6w7~FjYa_*VK_u{|s$I97t z&j;d2fv_DUj)cO5_D*nXY$Nbqd;XX^)NN#~Sg&~BLS`kj!dXoP#9gp^f5@I9bBbSO1AG3ehL;3)@1PP~M z&0EgFV6kdnkuzt~!3EDEu;r^NX%n<#GfQdccv~5I zKADEgRCUPTXmyyqO2FyZx&=O913~hJY)+y1AFOl+c`R6=QCGb+oxj!LHW-$8iWRgZ zz^M5*Fi$!PiwDZ!BFXbO@7-+BXn6$BnRSehj(EYSniUY7PXOUnB9`3f8$P8x%wL*+ z25Za;BII2L=VU_z_AvhejMl9uiT-cu*xQ;g<_v;9cL<2}5wU7i#38pF2j4d-^WRB1 z0Uwh|VCKc`kO?~gmObDRp7JAH6FikzejNsnUg^WJ{|v}rXBRkqg@IkO4tjjd8UX)p zRf22fdpUoPWOBBidxh0|N`e2+6+~LPkE6WPmq^au3Qzob%g==txC4N^dyyg6F5Gyu0-C-ngm~UB&$D)dajj! z!2UVjjmY!?eU9h4 zK++kqjtD35NU&Nr);jA!`Y-43leHe<>~b^muG+P3&m=qerp}6djyQ*XE$74OJq4ib zjmVFg=LJRz!XYbS-K&fHl-)+Jm5=B;};gQw<_Z#^oc zPwg|H0|u~Z<}N&X^%qY5sZSD)7{aQT^WmnbNU-jiI-XVQhIjIh0y8Zih?d5}tI6(o zk;NhYINzQ8>rEr2jutSjO_hiR+{EWa4#`?8WfHk!F7epj3!PoyDEOjepK0{{1+%9Sei0;@7~$2PR~aggnlf8iPl(?qexsF?d09 z8V1wsNq_f5@_h9e4$*c8C;Q^b+>)8_oZB(*$f}&Pq6{!*B^Z=q| z{|W!qAHnEB4z^C4LK3@v04SOT2-~DYt~)rvf&7Jd*0{Rp+U;xMOa4p=vRCiLCZkEhL0hva)Fe?jJKa^$TI|47+FSe=`XPyIZD zn}m|&X|NVq&Dn!3rYMrzQraZ!?<#ysaRQSAI zft=O&3yQ~lL9df0DHY`6u{#{7`(rNgO1le;Us^%V@^U=GRh?A5jm6t*yFp1~B=SgV z7kEa+B1TUS6^2zIkI))gCT}d*CgVmcVLhWvifDGBGOxW{N5oc9IzwL65>kvsSZNo! zcWfOyLnM-!={ZWdWyg@}ty##*G!8AhA&pEd5n9_c4TbHNV#Y--x5(fegr26c#_}%Q z`W9=n>F-40$Gjf8kefgwA01#fPOW6FW&&=jlNj&H(mQC-;ww;V|9~P#;^-Iual!1C z4an)wC1&$H#4I^+k5Q`s%&xu}kBThiC~Vj(*ty+@xhpPWt{M!4_npJ3;=$AGyP6!- zw=@oQ-T2MK+#O=Tj}GM1u0}Jq*9pETQmVc5nuw`7&S=UH(+Mw>*=Wt>i2OT%LJE!P z^k_GB_vZH$+;Bm|uS$?dCuI-GeWtdx$*j^lD6q&@q~Cm}qeBB0sAumYn*GZavDFJH z@Zr+AXGA=K7WZ{x6mC&{bx{aUp$TJdZcdwdSpU zT7aH-r*n6|(nELbC(^E?^~_ug6?D=2Hrmk5MGq3E3ud)n0aqL&5bwMPHMEVU7os1r z8^+9phj)AuJahWTq`eQNyc4U?>%*$T-`IzZy7?EyeHdnTC(b}y?>y!Dl;30jjuKSV z-HN)Rss-ZHG}s$fo%EbeFV|mg7dl$H4E+c_ghq~p(gSXb>2vv`%=aQ=_JTt`^0D!! z4JPu!vll->pfHCOJ7R*ir0i!xoJ8-qFG=u5#D6kf1*wek88haib^}dZR!8I9Wq1Yu z1<|}@A(Nx+gu?a}vRCC6({nY8ketF4_T`iZC_-)vn|{KJ-RFIp(Q97B`t%)R(vGIm z=Bx4Sm24ws@Z=?Q^^Q2oP2w@n6UNcN-~sA2M}l4dUYG4}4rF3ZHgL_Kiac(oljy!_ zV#0+<(oEXcQIW@@5|Q{Zv~%-MR%*E$yEgMT^QESc3GZlT9v8^-`ew~WzGyxwl1Zd% z6mQUX4dcwGjGx?FaU)FDG+jC=C7-<}x;N4lyNAlhf|h<~>@TWrzZ!SJT+C7wmeuTDIs} zHoHQqo_jplj@~--O|aYWBfaw=g4toA$a^^V9_mcCpyJ`m^mODUR-H)k_U4OG^AK-T zy^6&@W?ZFn&Mbq=hvJZ=;5vGh@s@q7PT8Z=Bv5$ECVK7JWyTw;Q~XDRP9y&z*RPiJ z`yxN~mG2>LX@n1Ym8T4068ya^M*5&W0XyQ)-p`y;8i+1%p7$n zo~ExPrGzu@FJs<3&7l=n3X!SR3ske~9-0!U!n3@#3oRS4 zNB2TkA#2Tff{qta0-eT9g3=fju1A$B%9)=<%Z+lFQuCQKy7VG_-l<4$eE!6WmpIe2 za%S|hL^}0fp@%m0mNMTAve-lB(rl}0Hqu*dPcKdQNjGnjryxrnvC3zt2J?-{9u!9( zKgFV8PX%=49v3-|y`X=5^_dZi(@0~k2hEw9MCa~0i{^aqWHj8jp^qXz#IUq1B9$+h z3+o;WH1&g7bH6M~HUF@FG8XIu<4iWz5u&8nAgWSfgVubVgceB^p}1W(^zpiUba2`m zro%RhmhKKkyFP5EOPD``o8Nx0ZR76f=}}XfaPTno=}JQ9QtmJgKE-%W@p1Nvmor+% z>!+8i0_eT1b67uykC^&XRITMaJqN#V-6m`DCS8?aHm2OB-E-csy+4g9*(HH`B_i39 z9WUwrPi@@FsM|=h{{%I8X+Tq7Jwyvtr_gUbB43!&YbIV$OH-zr(${%A(d%o+81s<| zMi3LkO7yrfy+*UC)O0Jf_&viU%e|#5PmH0lbCj~?Stx@u7ww#8#d1onGVcy2(7E3` zX~&8zB=IU36@h7t=G7WTDZHLe*e76BewhgF?AwK8Rk+-FFFvC9TW++-?k9Wu`!&Ja zun1}_VZsbmPNCsW4m9X+0d*`qOn-P7p~v~{Z0S^g`aSYBx^tz6`E@@7X%{R<7ysGN zR`<)OqFa)=9cN1dTQaDWQz+VY{s_IDiRdJBnEw1KPp7JinCOq+S@E%68awp@bMj9O zs&wvRyh1XlxgY}V+k1$+TVRTGs*6z8&{C@CD=l=GaRkjdT*|b$x=`au(}nH&RqXlm zL-gqQH8x(vXMhhMxkqd=+2N;cf?azZkwgANcI}2%CUxT?dO_5M1pCw(m(law?mAx@ zAxb04`rbkh3|f#+XaalFtcZrqdP{pV526d5pP7b77ijwHC)BfT0^RhfiHW{`0tISc z<^BrJ75O1|qxo&hNb~9!_VC~ddd6=zczB_W^3NTZLJe5JqqNv>`)%2et~tbLER)GVO(M0sLBnB8SeRwjc3= zQkO91{e^hs$mpSAzBw}LuBWcSnbiHB(E5dK{mC-lhq z4DD6gjyi=N)azq0HM6jxAD1a}_4<#Ap5;9HB4rUW7V+W6MX5|s*)F=z;V&cJHNY&n zS;*uY$#&ZeO=-=ddtJo@@e1{43o07b35z-*jwgv$3hGyjaYQtokCVQ{G& zd-lczVer^bYHOcM>#W^r(99vaXU9F1BrHKGyKXUid-R0XVQq9n3E}=pv0%^7^I{IA z<)aiHp$qeW?L-btuXKv9;F7?~nLAguf*#+WxB6lX2eLJ5)XMiWKbHsyt zGjtuDY^2QFRR4)eJ#|OEpVL@pDaHPjFY}tM{Q+n|7JMNaZPiXwzLgvo? z4HR3Rp-S5{5l7Kn&?xeH1$rx^5r=(9pyo=)Ljsv3=C`2#q%PXCMPC?mqM6#ZRiXt^ z9ux|*ShUxOJ=P$Op8xlTeR)EXe)*7t`VATcfxREdNR9?;|8l!vhe0+s;O9ct4_su^ zKJ?R9ai%nbvzM;bX=F#1?V(QlBauVG5VLNtI$d9XRWPJ~n6{)df|TNIw7Um0u`Z6J zJl&l&z0fZ(F1BUbdt>POO$KP7MV;~gc_1}QRV(JdUK(u-@Tn> zWN%HNipL)#5ARr%A!0HUUq42Tce0Vr5k+CERWij(Wsp2)9hHtv1KM*>GI}{{*}tk2 zY0LIz)?t)Trn!9G^DAO#GycI38W> z=T8NKWu;4)bzgd-38?@)b zY37Hz4i!rlEwC*-=B>J^h*#vH;74w#3!Oy4N7ZOop*MP>sl-e;caQnKNx&@lZvnj% z9L(yp%hPv?o6zm~d~|fn8|I-&E8Ej~1Syv%QrCc4l-pHJ-dLESq_#|^eO{j6Lryh) z+ueoIeoSO1?TtWMM$;Joi%XH2qA`=FT#kPK8|3CLT8CDj`Ob!zG^5ycYILkf5{UyL z5&PYXp66X}E#y=xlaCA^@E4DPi|uKCvsE&jzz$^sAKxkF|58rFAW6!dnxx zI_4xJwd54F=vhKfe3C%f-WyO#`$XhzbA*vnXk__QFEh-!nY64ObEl|kFg{OqGIe89 z(ejCj6dV@NbFZ|y9nJ0Rub$07F}ZQz!>&_ ziq6BIs{fDUSs58wWo0GFRte|a&-*rMC>kp5(vYaMzG+J+g%GkUltRnMJ?DPjw-OZ% zlBQ@#QnX1Lzw`SG&b^QG_gT; z`}3K;v#kR3&nBh%dr)D)3*xy&1Nu+Tp^Gxh$gWUAouo_^Vqgd;*)_^!{r_)PyiuKe8s zwYl1$TXh;dMjgS<)t^xl7UG(u4r-~X1G~)X>F^5&=)*umagWk*+H&I#35(AG+qqUW za@`Qz^2k`+v;H!i8DK@a7peKN2QDvIN6#6(CwSABb^XFo^#%3dZd!`> z_g;bi8ZI#T+)`-1a9cXGa1kl3T94T)|6r@}JM!=7QQqo74D`SM3O+m^LZ|k7O%n=Z zKvSs8gG76A?)~Esr{^IHx&MY-dj6K~Y1qOaR8hiN%BdttISp>E8X$NA2SBEi4*7AY zk(|j&1Z=PbmB$fSy6-FoxhbN>&i>ea>M|Vvbr4rh+%53$|_%VUF7#rh?5XOR;Wd61Kklf$w$pfWw@J zu*JI<*4!2Rdg_aDo$^nTwM2y(_!V0(;f?cD&x&?j)ay)qpso& z^qi)F7wvt;1q%*f;k0XX_O!M5b$$=sv{*&Fcc>GUw-WkJ-WA=I8^Csm0i3=08bZ!p zB$<11N!ZnOq*TC-Hx!qnenCIjcyWcd}=9x0oLQbxgx1#hJE|O z%xv#uR%&NT#OiA7uheXA%}Qg*9oO%S$Zr(uF*bxh*`mO9|7XG4JKbWQKAXhqca7nP zuIOZ-UZ0Q2C}oml%ld4O;r8+d95e3!9K1^A_asAp{x&^! zlbN;0TFb*8ibcU}$;v^@mBtE2?)){TOS6Dk{7Qu%_LpT18(i2gnnCP3moZF~kt!e7 zmdd5|DDVcUOL^6HTJcuU_WT@d!EVz{;yhMO9O_TIU{tv)o5 ze{{)#&HG@R=DhAdvXakd*NNCDXoE7w`DT#v+l3tUQjaQ8|TXo+BbqN z=+%5P;v)+5S>@nnfH^}kZ#w)X%Cg(CKzfXGTR~WGxQRCQt z340~?Z|bvp2QF|w0uC@9TUhp*z};A?p}?EB4&iIpQz0*@ykg4mQLNp0U@xi(nRYE# zn1v(WF;yml|9o{5Q@GrX_qjWp8#ln2J>NNzz46}yHZt}M2T4lo%Q{BlH{>ak_r#e^ zscsb*bRL|=^m?w*!J1WZwW>J(tcm;f=@xUNWgp}G$d&&Qpvha+pOu*B|6~SDQ)ko4 zO;``p6)Zb?3a@)~JAduqSI+)lEH~z+KRbKIYUWs#H9zoUgGZXoo>x0Fg;85l#yr2f zn^WJigR^?8#M|eUG8020IQayIUp4e3S1YQP%ziVCYa2D0Kk-MAy}nYBeYWK{v%9X5 z(~g?LyR7%;y~>vJ!4ebpOrSkG@ct1=$#h#rr9+7yrCBAJm%f>c8)MG(oeE+H zJMVKwrS5!fk`EhUW600@ae%wvEyq7kZ{eKxw=<%zOW30yC$h`_$~Y5iPhM8baYf7Y zd8JD#e8Qy>Y)*&x`}NicWQy%*Q)BSdl-d}U%2zR z^I3Iw4gQmOFyBy>$~DPR$(17l1LM;eR{lud75OJpZePAX7a-fpsAWBNmpi1!nRX0e zW*NR^nD3{#$d=EXce-4~$i7G1RSiG>jk+TfQ9OnBUn|F*zV?q1?3avX%v{b`$awAW zbmOx}O$5{#TA1yGVa7Kjc#%w{Wo=-}caz9s4_k-=VI+hW%FI z6I%wb%~PI8Y%Lygi?>~uDD~O0(T>Njqz<>>b9jLxx_S^_5%^V-_bZU!nC`=eDn>AI z8P_?NTrK|GS~*^KT|a)a&o^#xj59ODcL;we`x7T8a4M8z&oi1T5fW+IS5EEiBSrz8 zC5b!onD?)2*`GqbmhmVeX+2l%Q5L8xgl~-Cy9QhHJBP*aI_J~e!wg#`j>4YzYt}=? z-9?9Y3974j8oG>GdRUYF$we^H+s{AjprAWJZ?zxM#koBFxP^b?5;)) z{@0)*5<4d_^`1AHo2@*78LY45(TMR(LpE|(-z)Q5 z|Ecr)g?;`TWrqLmH<`^3e9Sc3FJQIRT9|`>Mlnw6(>!RnC%aBMlXJQz&*xtaV^%!h z!Uw;;$_+i<$py`x$h$Ur^OKx)*?zZN*{Lf>vC4Z**xk>IIo)C#UjD!czQ#O&54|*s z{q|`%%Z{~XJ1Qbrc~b{=RZyU$T1S_4o~Fl#{X(XqYylhhNsIlG-@q6;?qsvl%b7u` zo@|QKXpiD#MfO%jH#a#lhgo;mhCyu=HpJP14GA3~+2m`(?rKuvT_1%=LUX>jkG!J7 zD4Vu0jsb6&*}?ajE!(vj|Hh%Lo_-`__gIa+{dY0ndv7%xJ$n|fzdf8yIeSAgc^>61 z4j98e3w_7zdmv#{l!h`2Ng=$g`B!cgSI;$z_4%YH&n17n6S#WQSWe@r1FN{?8q?jq zo4w9CvIkxsVbqV#;oX9tGv1;`W~AS0{^tIfJn5ao8|ZCjG~^#M`>}yJH^7s9JkFH; z`K5#zzHBv}BMml4$i02zv6t0r ze8cox>dzW&9KdH(4`b|fPkCIFda+CYpKqt8%*RiD%JuG&=Ntvi-UTfm{!jO7ZUgb+ zZ_Vpv%4bbu*ZoBgg>9gcQ})2_<>m;*~7(j-j}R58p*Gjwa`QFgur2%B*zcDa)X=G zZpD5zDdW;+NV%S&R{>jt!1+3cyXe${duQ7qa=GadGgIRcKoy% zGx>2g7JTrq;avWr;oPMG8+fPV?u>J971N?Ri+Mi%H?uEb9ls&@ie!UYG~=n=9azC zV%G+kvhDTrxUJTgm=z^cnF&!^tkW!4c7%x=8(6)7QG6}W>0NlvCAcP4r2X5bKRsIcO69EAj-Vvv0RYq`9EUXk5z3A5 zP?*f@Q@hQ59b?7Xp4!J;KcvTI%Q^Dnzf@J6X!T(~?^`T6TqBZHz1-*?)_0WoI5&dH z>gUV*rYvADC%aXZqxiL(|*uDJGbJ}c5e1+td#$-lCL&{yP z)Mhd*o^$L8Q~uWP7>~o;YsT-~E3U{*>KWlld z%m26~pSw97ZDolVu5b#5u8b&n6XVDAx<_>1krXKT@N;rm_Tj8u%$wb}nPG#RxMa%- z{BeOHHaSg)x3^u*j?fF?*6Zr<`ro#2J#!3MjV}Y)+5K&pC1?IHf#AuTJUhdM4fy76 z_M()F+j^3TeKFZRVn8nUq286R@fWj?mnpMq(qwLUw;3xNoy9cSbCTHu>X=s+19&sL zcuD1kd?9mI*r{F{C-I>(`2@{@eD=uE{II4#UdN4ct>1uuWTnby_t^0gJ7YdhG~T15 z(~7MbTg~_z9OZfst`-~lFNTQ`M;;F_-n5Y zaGgr=T-)UWiErIL#w$$QgW6f}^22JmirRCW-^dqSzkr7x{f?fHXj(~`Cr^j+KR+um z1~)!2!6Wx`%_VBQ>|82m-3byG=Z1Mmpp9ugkKT zQ$fid+`jqTi%)gj-JRiVZOSnArb`m@7A@I-n8Cblk76dI+pt~dtC<%w%PJyDRKJP?O31Yd`7isGV^q>1ADNb-6J)`jz7}iTT!=n5tCp&i;uY4#zp)3@~i9KGY%nj z0xM9=Pjb@WR}EjwmhmvF0Xz!-Xm?(m9=5qBV#)w0I2{ylOFD z+%t~-kvD?vpKZd9{UWfycRBG@(MG(p;S@e;QysVZ+9Ez6>kYSfR3tZ3`#1BgU;?xB z(+K|Lf&Sdh(90ejw~Hj{;voKR<3#RiOg`gUR>>Tfj%5~=Nj=7|8qUAnTF984EtVvx zi20X!VqRUt$*psS0>9=&BIhz@GMB9B!Mi!E;z#Dq;m@8P%jJylVQn1xJe03^uy=kr zveiBJxh->6uy4b?IOnW5?%>HLk6rw2$$1-PezKJt=U+95U2A{B<7$@~^Y>^h*SmNl zyI#DX@#;63iSgBEyYDO!GL4%!>4#S4&EQ1tmjP55FX{ET2sx4~1I&2)CDYmDzjqmZ z!*@(^Yc_Xgyb*h^^pVHsNrr65ilzL23)4AM`P-6PW9@nW?ILzgv<+WwaFMzA?J_5C z8R~^6> z`Eja%dA)l+tF<_f88qV(^GJM@yQ%SvF^!zXZk#L6J}GHsc5!m7RlrDg`klYriXpLF zUzD)vZW0)7_q}_xjmhX zhFveW`t5n<))EbN(h)i4OtURtoMyu_9*bG8*(|TLPL9`pG?OoXHi9!^NAjOSX7Sl4 zHt=ia4`coc^@Eww7p|n>88=JNm~a2u$vi2g%=ZVS5?AjErmxtDtshv+TzRO-mY@I0 zwVM8LTk*L+^Zao!bF{^aQ9LNm&Utr%YcVVEIQyUb zs#)xh+m77qnkVR3BhdIyA7;3d$ELTILIZ%j$|_vwFDOZd4?_PWc=lZvUB!Y zu&Li;nCg*dI4gsEkEG5!Om!~HUkcM?O>DE6yP&LlOw)srU$4sUmQI*HX3e3lW{_On}ecsr|i$4);%inhOmBd(Cu>73; zOat~XrLsom;j>Zvps72!13wLT*Ih|W*O%R#H1GuX+|Gp`=^4VWKcK;%&&uOA1sn04 zQ!bNxiDi>UGwhw!BiVmD*7L+bk3FO8$=0ms&w7Mlv&%Eb%FDFl^Jau zBlz~zngb;+i%>Who{~qVtVdnjVW#-$0KU>uf#0#hkase=!zC7NW+O6YGJCTPS*Or( zZ1tUC?9o6gmfW^vE1s?7BR>Xlub1Cre0n8Jgq0=}USzx^b%JEB=JYhcWvt!ao4fE!15T}yc#rgV8;-5N< z<=? zFD*VXLl?A2Y_Dc;!v}iu<7a;3OjUh3yRY&*AGwinoimyZ*y+Vq&J|eSl|MLx@P2}b zdI(?MEbs$W%-L0y-t6^dwgO6SKD%I|9PfDQ0`pdN5VP0eGgEkdo20j;o|$__O(MU0 zH}iP#GG@l55xi6MdoJgp55HfH@GadCmlw z!i3xIWww7j%)fYY)qlQo<;Peyri^meO)0ZO(A-5j_Az&!N+k}=P^NB^F2B~(m0R;} zBon$6JhuKK;4b?LB0?K1A;g28N**;qbBeKGT9yC!ciGM>>i zJix7dIEcG{xRYb=^>S~Q_Hkk0B(=JQ|N<7(r^@w^7Y53bH7iTdE}-J z;SEn4^1I)<(97?eaQJdFaeZh#yqxL6cW#^|9TRY#6dXK?Q$GnAH~rthx4=j&oH8G# zrDfp91V!nkObhBla`4q%BiV+2@nrw98ZcTq2yWT#N5jPn!D5Xg{ObKdifU}(Zrud% z-I@Z0<#Qq7tAw=vZIxDESqVyp&%_-o;L* zP1D{dpsc;_LCf!cg5Hqzg!P$; z&!)_V)WHwHy|ElJPLIaN!rZ@h*axB(D`XuS+hNbm6dHD2UCgA~$X=>%hZqejw3qgf zsM$C0QpRQa_0mA$dFu4zV-Jk#OU0(BIFhNq50?(QCpBJUhU@_eO_)>(%IQzYr(in_ zS{zSC@0n$oB)d$>3w1MKC4K+dQN&2L7jtlne86qBfM_v}#5lV9e&WvmLK6-Jd1x$^9KyP-0gQQ7O z{A0*IWH$bx0oBD+dW?ldudmTjyS}6EE>oEQRF(R~nGvty%8={OL2c*AgX#|?Ur!g~ zFUbh;fYM?TFkD||xNI%RwT9v2#v7R3Z#-r84#k9jx9H3j%TQ(RE$WzZ3i{hkqzg>t zF#B0LvctCG*dLjAsHT@`t@uymeWE|MY7WPpNmJ=o_am@XdnTNDwTyiCx{Hgaj6w4T zbudb)hPnyb;xo6d3XB-R?o;pvw>+N#oA+D7e@?L&=I4TS7yn3gM{K2H)i!w1m&flf z&q2ek4%~h^1tZlAaN_oQ`X^h3SQZyT=ZPI?VZV!HY_!06j;BOjcLmLg&k^=;j~?!n zX5q`fJv3f>5#F9Rm^#ux$m`Y>mz>iF>uYh+{|akFd*A?V`t%d`GzW;rZ*3t~o95Fs z{?AFT(;Nsk>7b$N0+`3s5-JuR#zQ|A&=<8qc;8<`JUAl^ElyQ3nil`a-CaD2w*H4z zqrx%DC4(%yI}FBHOruIq4#T3w{bA9bcsLe$jQ+MCfYr)LWTE*9A$u?z-=tFfc<&=R zPwpohA&|~Sk@R9KHN;s|yiH7&z^zp1liRjJTD)8z!ja|)0 z$h>lWbT9HJ2BJhZ^KE~ut)B#&gEHwfvArxcavRwc-Ai(egR!V-g!nc6M#FcN5;$f= zRDw^#P`w>sU#1N=8-LQBwe8Yv-jy`GvFzU~Eb%#oC>rFt$4x=4LI$ zgfnU~BSoZ>&*b6Q_I=QmbA)tgn8I!Ih2+LJFF5izP^9hCB%(I`z;pO$GRiy~oz6d? z0sp>{)m>HawtX__jr<9NOB_MV-3>h78HkQKK12(?8O^@jqXYf-!hh^DIJGGXUhm$G z8wNZlR(+eP!Qp6p-!K)`0|#P(%5(DI%xchzs`rRh&chwH+wgxqS+361PEf)LBqk;eh*evPng?C;S$ic{5JAVZEvk=B3=HT0D!1 z>qRIZIT?pcUyLIH^Y}kYE`#=oi(p&$hn-hiF1`ED5>vxnu+wxm+7F2bgHwM=>G2Gx zzvzY`FRF3Y;hXs7{bgKv_qo)t%N7gj3(3fpm9So+43;?m5#8KtEc^SlM|9nAg(%Hx zHJr$OgYV|VNS|tr#wlk$)6tO}beR{C3iWtuR&f}=G_EAq#$N(Ag(m5%wxBHT%cBd$gQ7i--(nA5RHN(T%RtK7OJHDzBAjfmMWd1oXwt+9kvRs>$O zFM@vyw4`hO-ARUXI9)Pb0@>%gsdGXQsO~Yw+~oP-RhJg5SmsaHIJLaj>q2_SOaB*9KSNqSkR~+p^1GO4pLS zy}xnMrG@xAYK8Q;?r?F%gnIhxLoaRJ5CEm^z2F{t1iC&NkTj>;wDG?v9H2BncJ+c4 z&Dg6eo?Tx7E^`^FT7Mza{N?}{^Y1+bZH&SGYclAAcaMmuyFV^p{D2PlOT;Eh#`IwE zMyPn#fp5OY2z;I>oTQ^F+w&?5`n052S7j^vz2_M25q787YNiYN+&gquXaK|qMuFMc zBY3v4Q`CW0XkPf2=#F0s7Y8mE?buyP7vwvjSYSXde&QthFf|%CCcYy#&xXURq)@6g zB^Q3})`phIz5y(}4vWkD>3rN}M}z6=i-~LDqv=c-ieVrWjs;8~=Twdy1xuk9(@gRu0{V zL(RsDz3<&6%N=K;gX=21da%E^_D&aX`qB?=mp{R>c{iXlXdU?c834!c4a0rckHe`( zIoT%VgK)rZpl~04fnTC?q*^lsmkAuCVOJDo$8H?NuFnVzeGkhFtf05*J*jw=0ndI4 zOtniK%sXQ&^AvkKMybjCWCw8IaC0cU=LuyNG5F})4%CcZ0Vjs1kd{|- zp>F$EoDrD;37cc!t$7qC&b9>`nY*-nQ2{M|FbYdXZxtn-wt*_RPd_-Vf(if9p#RZ0 zw9C8?+@T&ieP|}})#jxqk0)Sk;2%7aU5?8n(V!D*k1u98$gZ1xLhVn7Ao!G;bcf*< z+%Y?cK;?FPwNlQWgIll0a6(D#{G}O~tdE7}=QBY3OFX z8%{hM0Rv3*aqg)YxUu3MMxGDHzJW>b=F4QVbKfP%ewR=B#zX*-mqOR+dWcy&ADR`8 z;=L9FneEGNT+p-zUf!NWEUolF-FY)<5%_y}#8!5y{TlfaU`GGwgkt-lKhW>hYI1da z8rBY29|f6fQhn}`97bGzCz|_SelHk8M2$~Dk_7~IRbxG zEfmwV4uHmjjiCK654=M+z#b(-Y!9;#nXSJ70i|-#=@v=LOyp#bs(Dhddp*qWz6bA) zjfPnQf7Z#`60!n#(VYBwu(K{1mipSjf$@TN%=sowsTe7-JY8Al z-A&kUf{-bFtqfvrAjsawpy9uVMC_a`{pPEJ)xX!%%|nf$^X?&B{^z!|T18FVx@-%G zql4I)R}|>WwJNev%T8fT;AP%W(Dkp#IEu9oj08UKVwm>m6E|33Zo114r@9r(Y29sq zHr>b-qMjpe_4x-km-eHPBPWun57z;gaF8B;y&FfYnaQ@q+JU&?6FoLx21{ojhIMBN z{K|~>NW8;}9~^hbq0DJf$ItPyyUQ*3N6`_m!0HdWOxyvA1}v;!^HCaic^o<~83Xm_ zI4p6?BJ+*zLY$5qEJYX0DZVAjnsbaKp4dvK>1C30J~`5Rf{re1!gKOV4ru;HMRC?K zIl6GHfp}s+PqbgLi@slSo-ZkGlmL4({6n&K4U!Kh+*Zk7TsR>k49{i#s! zCV&qASueW3=glzDKZ^l$ zA$JcYORR`*x0CGHn8Van@BmqD7yw!W3_$MG4sv(>JhJwR8l(aOn2y6NeNzv zu0u{fhz@S@#4)OYB0u#`aHqz&cZEAP6(6Qye+P+MJIA6^mlYj4D+(6oZI-V79t-cg zSAgcoY2d%O9UiwYK6+o|VqtY#VgWqu0> zZ3xHfU;jZh?+?E%2a$C{?-27>=c(s`zfif*0bxc2^u!IPiP};$A7X^F(=Jgs{1j)} z<`U3Q3kT;YwywV~MhHIOg;J`@7DrT8QH z1>RFEla|ihK!z5^lWF7fz*;R7TrRtkw!?7%UoOJfE_;+3cj2eXX{da3FDjO2lag^$ zalXlS)Ghxp z2K6q&r>ULv$ifWtd6NZxlk4$ko4`N3br3Z=@4<7wc(Q)%K`8k*9@%Y%*k>&#d+H+w zNv;v>uGeO(&px5IPYi>G4XWbz+p~b}{sLODgD5*7AKSK$78{2#qI{DCsM6682lhQB z{O9X9%GQU@Am60^8_Yqd?x@dhKk1V*T4IByKrmiYf@|W2qTZ}H!K<{7PWDd+my_*M zsPqvH+*O1(G8*A*K`aT5P$X zqu?1`A_Gv%orN3UX}~zG#kA&Zf6BeP2>QagEjjs?-8V>09A*&(a8HNg$m3w1TqpW; z+Xj;N2>v5|TWbDJ1&3-qA$G5vP-1S4KfC+U;Em-^kvd>#J4^por|oX74FN(!IecWbQG70AJ;viFK7Lu3tP9)&VEDLywJn= zFmVDJ93f)EhgtZ#;X2l<=!*9)uEqPVKdDoorg)C2f-GVEYEkUA8)#$iDHUg@lCUoy zA=@{FHr<)X7hdy2{kQ{AB24HX-(G{_<3aSqi~vX;JrAdL??#252V~(A!4tEn6f{?3q3rJQpiNv*5>==eP+}Q(s_=e+eyL*hn^cx&Sj{BR{2RDsJuyL4!u@t{-taGS5Nz4vazf%|i5w_gWsR(%33^gZ><4~F{W$FOZxH1=3+r7OI%N&E#n z*~R0R;d8MroNB2gqk|RTu}!LI#vehqyRnz7RZS&#{>Z3lr6rtc4#xcf1^8>=H8>3%}bTLg(33bJLl zL-6wDF1poF9&bw9i6Y*{t$_^gTJ==a={g zO_oK%rQ82#*OFB7UmAe^x&`dLNsp*~r&`G7ivtUUl_zMiB|Tb}-ov8B`U-%{n3S$KY}HD>Abx=(I# zhSs7Y2wM^XVJ|j-^X43^n5s*^emsR|9vp#rrdA-cR|T8G2>ScOSqPSmqRZ9~$H)9} zFg$&l>g-L$r#Oi^+FYThF9}%`S_8zp4g^57$6{(&zX_se$CI7y@kI0NAkbVQoV^$Y zu<*P`Gsh*tfi;z&>t#jft(}d^x8mpmf8l+MRwg^1RKn~ZB1)LyvWT4oZ|w4*g?STT z_ppsv)fj;r_kE|j$NFQ)pt(3p&kwIe#FCf7_p{wOOZ0X21=`hin{G7uPI~)>kuUuR zGW6DY5+ZnBpH7}dBX%xDA7?=u8=)$`a-;`O28Gc{v5_!yN`bUbb1rsxD$^m)Bf(;6 zCajLqpfwh%Fof%b{+0S-+c%-)=*ltl(&7k@KFxoxjp|pA;t-wbkau+d|NX+Ex--H%bJQkvIh#Z$POrvI^)@imZ$EremBBlq zZP*_d#9BXl4EFsip_>^=uHC3W&+74_-tP;@Y|r(mSD7ze>0U$n{k%^$zM4nEqs?S} z+PcvEZyb*Qxf4nz9RhxwnlKYch4-gKQj^PjMEW`M_%<(?J~KHc)uD3q0yMd?hPKv7Vd63;45fm%F(MxQvrI*|mTO?= zO$VHHR1;U!C)0If5)gFek==K{k-9BvxISQq^hy6cFgC}O)D;NckPt^qXw#r|3wTC+ zI-2x#zaaB{+lh`_47DEikc{t~k0Xmz#kn6w)2&nUrIWVWN(;^yLtNJt^q%|yo{8k4 zL2?xT%4+adogCU`kD?b%OkjOqJ#o0g(r3afFMpmI^py&~vC$D;el^EzUqk8}mqyP0 z8-yXK1=Eg2!tsi7m}cEb7k)cL6%I>9xzG0CE;)ZPbZIy++f2zL4?FRIK6&Z+nCV0i zUDC^8wczFDPF80p;ELJl=*OmE10RiVglp!hqaAHtRYT2;3(3LacKUN`H0ZV^i^}em z;>cIIIMNF7?|5rCySfcmyQktIQVdRbNLq6Afh0L41l&DO!G`@&H0fVBHS1?UCQn}v z5DnmgHh*$_DcJgJV!42f-)+qUYVHKN`0ruU(hX`Tlr;<9b&#VGyNrH`Id5#a|%E zYGI1WOS(s3b$8r8NBULH$BHe>$-xs+Xo;Q*OMUjksoIVB^2SvXIaE%(U-X@F!8$1G z-^lqY9)-ly&a&Ib8i-TA7iJpj(v*@2xU$$Cl=}4}8p0gf#VQ7>21up)89u1Ldm5;l z&!pz&DbT5_0f(v-#Y@$v5c7^CVtDGGsD6(bPO~w@f^AEnK0XTiX$M2suUjH{)o|i8 zX(>#T%Yt1kfmq{u4;xg!QX9ML@G0yn+!?hWPYqVWV`BoqWOM|TlO7@bwHv@zf2S9{ zD4L&0p=Q@zh-djBlppeoUK&#b1$CEc>y8>4GXJFp5 zGt$fNhJm-YA#O=OK%8O&XrjG8q@0i=#_cabv1hbM>*yJDdQeH?9|WLTj~l*?9sv92 zM2g0LK8y<%&Lt(A;~^%mn^^C<4I`8CN#V5@P`20-mL)BR@1f&ip@uu`KUPWeC7R?z z{z8cD{7Vn?S%K*DHLS`X1@icXUi)s1fgi@e{Gdwmx%3`g*eIO+5y`mwt(G{@p@Z9| za0*;@-p8zD28H4awRx~-iFoc$V zeN3j%c+@!ZnOY2wV83;r#4KZT!ApD%dQ}Hw*V-IfvDs4Q{Bkhvow}FU4K9S`Pds6o z=npxbr;Opm5N-`m#lJQ0XydMWxG6^H7#D$eT>8rrN7LGWx; zIs~4&Pd$pB!0S*2@#$(kJZsItnQk@Q@bCmxR?CK}Wyj>QZ5F~ljM64={* zP`PnGIcf3&o97AMRRvw#(RCeDmiz~XE^V~#L^FAt_?Z6Q{))yk=zpgYB?>c%UG;Uc z$!i=9yLAZ;UL6jb7K|o8s%^>n_S=yAnMDbvJ7~&HxipMD>fpBs-G{pOOi2MR*Y!&SarrvcS0ts(k;0=!U*!-pRd;6ua=>6FK+ zGF6o{kbN75i(CD{=|TZ4wNZxRbumylxe<3SHim^;&2dNQFp<*V1nH=I(OmMxMW7f} z2$c;6#7A#68v0lHpdl%9(cp#ZEE7uwMsO}{v$PfACJ1KrSwVr3mQM#j~LjV#mY^IsI_Q2 z*v=0JTXS!)zcX3pwR$Sm_N_!0lXx;R>N{CzXo1fsTgv>k`pB<-PNFa68)5!78Lcln zChbruCeyt3W7fiSBExCqlJy7}7@UbmOI2vqv|ec3a~_I>x~=HST{3^?Mc5uD2!=jI z3BAolQf9Ybnp+--U)-v(r{O-yt*8T(zJgW<20t3EmkwJQj6Zy4;6lYhoMj=8MZqh` z_t+y4;Wd{`30)4_<*hJ&@E(%7&JPnp5WOdLf>AsrdN`1NZQKQp(r~!XwA01yBcQQ_ zhhW=PIBA;&aewav%WWs4q;-j?X8kqxwZSC(a9jOcCc;@fE|M0#4eNaBt zh2A6Upz8WH2>mh=sy*|_ha>UOyyUg?M8pdG_r8ScZ22nvbiDyK$r)kcRbBeRyq>Nu z5E$k=N~xc}X4XA6C8xm46y zCS9q1jr1vg!Y)5+)cBc;)>+?Z`GaU7$1DZvUKmr$uLH@ek#nG0 z@*qmvOySO>d7=kjzDYH!hQO}_EKCR;g39U>uxREnoVV!@s-K90xvREe`vgBM&Dp~4 z+j*PZj8Kv}^w!XOKB2H<^F|aP7bL2n0J~ZQx)B}MvIOlrk6~$Y18usY2c}2cMTTjgq4P}~JRX+@%{j?r ziamgGgD%XhScN6(>!|g9EgYh{ALN41&|9KOGWzSF>W;7U*O)UnUb7Xx zDQL)U9o!60f=l&LP-!WgQMVITIA$TSLh5b9g=f z5Dc6?lRfXp60>|?dV8O$z=Trd2JhF!w_d&=e_%Y>xom{Y{#us6waLZ%OJ7Q9_AGGI z(Sn#^>f$mjvI~UKd$cA6z z$KXU`L#lqciiT*;!G;6&xa`e%i2SJ}i>zxDeER9cwMQP7Pkx6Js}O^_+bGK?;u70u zAnBQi?Z0>74x2&+_SfmuIY0EgJ}neo!2e=DC{Ef>ILfV*G>&H%TTj25bnJ6A)^KE(y5bn zn19d%wyrG%lZV5=cvB?g{57H7DvFr0X%?=VHy3tZX%kI}G=!-M_vz7N-FW}FDypnM zf+HL}!2k;2s8K0DJWO89erOPJd+t(Qt#Zokt7g<;knB;&A*uG*W_UYFSv(_bEd01( z2Oqb50+Wnw(kBkWjz}#DX8m)dtI8+AmUMHPrz7NZJQ$5T!fQzA_y$~W7mgFhIMeta zeKezcEU00#=s@FBR4$W{-XV_2UrU3DKT^@o{sm;-+y|*Y3n}qfiIsjeRLE__Cq}tU zUD-$S^z437=LAoYUh_k2^iCt{5vJm^)iEd|D)befinb4@khg(bVbSw>P?xQMnT`68 z)i#}JoUoIGEPpQ29W)X@%7;RK`$rfT_yk^+%JE-pu0eP8K~|)o26?LbvY_&0cz(Q@ zX86X!m8EA$-+}Wu(yj=Ud=fF#R0rF4O%)NPJXC#^L;4#HgdAdwh4ER$ceFh2cFqGy zr6*Q>eI=^+p^2sam&3--FJWD=0`}J%BC{%eNmrU~rcQZdNXtb7e3AP=>g*tVM{MsA zsr!8#y!bdaL@CirHXHE7t#}b*_mCFZH_`R4_hFLj5Ab91q_3uGiRB$P3p}y_oGbY0 zSE)QBe}y|uUG+L8y$4`q+Z*`1-d~vcPNcUE-@=X$+pvCZHg$F!PbZYQ!|b+gu;4`t zYRHD-HjzGArL~<-GJ1_co}O^{Od`$S;Yl`Z`U3xzoyE`-H8B1Abv$mff%I3q4C@nK zO6x4t@IOU8&ZN^iNwW~2LAOWnLkcr%g6wFewPw36JFtp|NLMWqfS>bNzixiKjhEbl223hNP}IP z=+BQ6loVwEj-CTY(>g^vwB4z)>s_3g-%eX|=fIuSp0K1~8IGyEC>oHM1<%{Q(^nz- zpyRxX|CVxwG?*PC!ENhsW&2R-x1tR<#TCPE-7I{z&!5a68>$ z$|tqOg|F^Q>(>0Gak7PEiPIgYDR8a_H%Ls(aN8??fDzHsr4-3+iWMhSEQJf96VV=wLVSUK>gGFKWkF)&P=R z`jdVumr$pJi>T=CZNlzWk$vbI0g65*G9d$nZ0qbVi>diU4uAi{x1P|)%`z|eB$bD( zncDcFAeroK^AP$NS9mMh0_`69vRwB*WDFPK(L?t!{)?IDW^OwjwJ-&wH;svE`Y@Qo z?VwwqT*Eq(ZQxJzspn8d>=86PsYg$f!U6KK|50?_fmD8f95*U^l&lD$LMSWldG0wE zl_Y6MsZ>f!QQDecLR3a(rN|B?(Q=>Xo)4lWN=hXSTC&P$YW<$yzxTg;?-`%-`Mlq+ z_gUv|nlPc7cwJq>jXcZu5B^0#7ymoS5o_|r)&hI~+(SMp%_iIbT_G9A^CSZ|b@A`I zI^hbLi7?*qC>ZiRBx|0p)~M7a0kteP&S15mFq>Dp8wiOHKvwLeB46zSxC7Y|7=>8812w) z4D7otq1^l>Sjc{-PIuacHSeY9LJKe05i*9TRr!;KVv2*4;^5K^XVPi067&sj!1ce{ z@Oi2oZjK}aqdUn!h7zt0DZxjxc-B@T-#djtu#Zxe6lPt5v0zH(FUV$0(zIalT!+fF z`jq@gvLng=Wzy?ovMIi~o|ed&60dc-PGW5xTK#bb{n@Ttd45bLEp`#W!HcGN?6s#t zg;EvFUDl&(?_9#!5_Md-<0?MF^Q%0+q%oeMu960380_JB0QQGJL%vQMnYJMd9*wBL zGafzXE^G;gEPIv8d8FS!J>LXVKgrxMBf5M_An z;X;IR0dQ1PMbtd008W{WAS3dIOHy^e!#3MCqQuwo7jT_Wb>CR3bU%|k&s`0h*PMcP ziMPnUfD;fZbV%PJVpx;&Dc zZ;zL(dp;Z#YG%>dHLIC#Bj>^2O_ymC&xJ63H;5DZ)JdJQl*oLOkQ`g#iroT_!uld< z(KbVW80is4EQaObt`$1MHIFpm{0I}WJVcRrlMHfVeH`deC;0XHFy6oXJW&r(hl8^6 z@YaApPN31#^lG>f6#}n%3UKcHY9cp`B_BKc=@K&&a>T(C z%$^R1Pli|E?LYx_=4)Tc{U&nu<9TrZ{sXE@tU#r|RHEvA5$pomU}vluY^f_Cx9{*! zY?T&bdpb#IbVr&F{rpL;&n$B-n6+XGxa-ow7qNt6%17jB#JjRqOS&{Y@Tz=81> znY$m;;8EZ;vi-L#crTj`i;r62fDig2hkRSG_#O<=#dbICZCT_Sh z0dDf|q@SS?@Mys@$=X9V@VGaTAeecHtJ+eE=LOf2p)c7`{56712pI`kYO%nW7?QHr z6LI0FaiBHXk|gAW6PtTVVDsbzespFgZO;2fKDO$@A099ebt;c(JG!03KY9mR@m21jDsmDxz6>(PX9-Fpko#lG?vDIQy|Z%{tr(0b#YcD*iX! zdd?7UyITl@lSdM_lp#riQyKj1V=8;nG^V;ab4O-%2}I@12&7=9j>1bWq*=!lvklGAtrzKeTt)q`_fR(S?- zMGqxwvJ$}FcoP(|+3Dd@gcGd2|MY|J7=kId(dQ=g7&zXVB-HGJY#9H#QoQ2q)pO}f2gSnzxQ0ObDET51C z-A~NOw)sW4o{N@z53nPZ0b%rwvpo)fsShKMXwvOuE?DUlkiSui_~xG;a(2I=WdEFF zU}boeoxOe;nWFv>y#G@tX+g^L)HG$%DBv|Nk=A&p%m-{Jxe9+a9wSSI7I5!WJvrVy z1xFhuP@T{iQhzgrepkw%%n1jWm2^R(*PthA>>JOsT+*T5Y@#SGJQn}7C3GtD3ZEk{ z!1K5|{pR$NcFq4pqDE(fPrnBpuAM@1)8fg{ntrV6=>wq~ZAqV36)cOX5lZ#M69vu9 zlC#1W8b{>DRGl1)n@sL$>hS;{7kUu|Gkh6wU@lSnC`cif&%!%?Lzn&O^@J$)c zHt>KDt!J<%`#8C*94tK5vX1;*Ab~KY74Uef8!SrNMsmvLV_SY6P#1EUImRp{*Sr7W z5zB7jMJtXF6`uyOr0@wg7%!w-mj46U`AL$LVsD;dxkrM&W|7UBNYcAjK-RsqqEsAvXfawq?|kOna!FYNKT@oHSI*BoDyMNutH%clHtL5F`bDQGK*=`$cl2 zaW~z2elBe7%qE{{HD!1`!jcpRto6wcTn(NPKYSDRPOKHW@_>|mk&B3c*9-_mdEgUw zA8Nm9h&ErN_+wx)-r8d<@!#Ep@ATJ`vG>11$cbu5Sihev9I64SrY>C0YcuOirU2eJ zTeNrIHMm%?U09;M5!PXX(ro#bijl25yFOL-+?$sL6Cx+q$hG1ow2%|rb^P`;o+mive>C|#mA|*C3>0U z!Ogk&ecMQh(V!BsZtkXa+26?B=d*;idC9QD?JO)W0CHiuj^yH_lW42sUAW2j#Wvhb zBYAvHOV-^b!Lb*K=3F;WajQjEn>}INKOOKHKZAN+s--jT3Sbk@>Hgd?g|6oBR^KQ& zFt&<-!e{xAxa~MGXp*a3a`-sL;(Um6@q!8F>&TfEqeVN;=E8iJWSG6<0uG+Em_XeY zVy3zcjITD38Oyelr{@Q$)TBjtAgcqHEm7ppV|(()(x!6V2E5M_SlMs{~PxVnf zxaG#ekEg+8^xD06IL|@!jZ+{Bhipj*dj(&ubAh9VZ8-EwnZ!?eDLnN~7hZfThUtIS z3x92%NqRIKNU(4_&ePW>17@mlf0KabWYxl|FL@;VU_Du66$?tKm~IH%2~~Dq@jIS1 zJZ@biS=6!e|_6*nRV8d?R z9YJf#&cN7b#UxnH4(5$a2e; zTkz_~+ey$|KhnDC25vz+A#CJzW@-LU9GW?t6wf!RtUEQIZ1`hHcD~`cwJSqOaLQUz zX*QX9TCae%=`-krIJrt$4NH)Hca2`1uLv{7ohP|GM>^kT3*X9oES&U6lcHirnA~58 zCtf7b`$|mCuN#S_l-7ags#iFFxfB>5`2vAg-QfI@Ik1pPp!>TG$(DyBM6y=;c;c>S z!oHW&anH3}X!!0;ijJ4TxkaC_Id=1-TIYNo?DXL+scXdM<+NZoJw*p z-=nVz?n$TTxF*e{yOD7wS62@{b_v>lji ztQD^0&q8H&{!B=6Bg+$)lWZF!`1oM7NVRz^+177P#3MU|X-%r2=%Y=2TH|0sZ8q7^ ze}m`wx)NNVsLT_t5Ak=FNtM@rksfuQr`k$)fdkMn{*ybkax@Z32` zIaxf3XG6z_(f9Yg$u;{7qI-8Saq3?Rb8qB8?}t0&^)Gc%l(!*?Ir9{&9a+b(iwY`U z`5Tt_)sPJg$M>VRfhfF0^7@1mJc_V^ccrC5)kXcxq@Nw6Y#|3-57}8I@f=6Mk zj;E5l&j)dPu}o#$f-G|TNfFsYok)<|Lei9+2gxfA!J=iKnCi)Opf&6YnVnZB*rHc=PjoLku&^Crdy_*S*{!fKk*b+)Jc97(;VI*?p6;gVB1&NUh2CaBqQJ9a9 zXu>>M@aw6;@e(9FEN4_{Uh|ZUKlK9omnV?)jMrrTs4Q4t(goTBV@ciX0FY?Lk;HT* z(zd^qw(6IVJC@&|>WTqhQsV{JVi_@<|d{C3{)Zk0fID8h7S}~8j?OKUt>|f#Y>yF?L zkNMhVtqxtLS}qyZKT+IN9D!|&=YnwTbrSH~S7^8#F>r<3N#Vc_6zAlmh39XLg} z(!Jt%JfUYLS-ovL95r1aIdI`HEO7~ks7Vb_;=UN4+#7*Q{zj2A{O9&MgLsR~F4%Hn zFRc9?OV(Euz~kh6i0w!C@%2eU4HqTSD!Y|5?yG>OA=e~XUQsaQeL>PHe*@2c`UxlA z&RH-h zwIAdsSAl7ezp$|T6R5;nLfZycqPsg2@9Q5Yd9OgB(PoGii{go+O9Yu*lt7+*+(TvV z{2*KM4)HxSfkeK@1iBA)F=TKIEF6AEvX<9|$^RDuyZ`JU+BN~C;Z`c-CawhK_jBQp zLk>tXvtUJQA;cHFByLZ1XugZOP+fBzIkPsMWDcF9lQ)e3fmtD3{^txvF)zVA$4PR2 zbp`2sAq%mji9Fl4g1%VVOMRp!kPENgQPrauulQWa3_8EYZt@@KuCP*&^C+bsSJ&bT zSH?l=Uw3ZiDQTh6r2V9D1d+U1Ye0H^V}vHFQppDW88qb)}LIVFThcm{yM0mFp z@?|VU8}@wxD?C7ghUUTM8v;-(;fTS&Ajznl22E{Oh>LVJ7_*%`M|nAUDD1^Oo#$bp z^mur^BU_?tcb6uwP9b0GIeI2%IbD7BDBS2KmFBr~aO}+k#On#4!O`!@&0KSf@N96CKx6smKOG;AjLie*INZou}qUjg&e;te@ zbTA*EQAr`upCdt!oFLxAF5=VsbBWTTN6_*lmOQl|PE2*}!0l2aKVRR)bK;xeidX|) z>Skd(gT=7Y%ML<5oTS|!BS`Gj$3(!+Ase-4&|AyW$kwkpnIzn;cMhI7g~7j<#|YB0uKc+q0FRnJjqd$sO0Ip?6dl_8ioN)AH%aF8 z7jm&#*mmC$P$-*0lxiRHYr~9=9n(i{-Yg?IFBZ^+CfcxE*d%oP>`s1nEr4Sduke&_ z7wEB0ZIWYKj`P~npmOasD!VR;6b)Za{>2`kQ5GkOpG$+p_<^E`1O*fCQh%&Iq8IBZ zCPKGlKaSq53}$k7$$vS&!EQu0>0g;dTJao6&C`-dFNp{F-2xhH{0`r*+aX!w#=#S_ zSJo%sl91S}P@MJZu(|?Djup_{ASs3Z`3nwY3I`Ab; z8>pyCfj7j5_-suft8bfy z%yGr#OvJ+Z+-CK~+}QudGB1CvV2qQzxm?{QLBkDCPWPWWr&zbT!d@9i zr}lOhb64&;tN1*eZ5aL#HNPEM(NwvLt60^K=2p2ezkR&80T;rJDtwQmEEE`@GCMB$ zwjn29H=1p^+JdZS+i+_JEI5;;m8hcp2z&d15%=eVBj)IrbDfp%~1A6)9r-ZfMo-FYcNoJ`C9=BE9wy2l@3Rdi}xUZ3lrAq!f~{c9Ae-9 zX%m~w@@EDt-mooq{Wx>ZgQ;3_8ZB*CMU}&6um^Vxp~`?i?7F6Q_Lc5mL5O1oJJHjE z8~)=K+A62dOsV)P$kANIjJ(#!RxVz|#TR%mE1Fj^$2y<0tF#t!zvg)|i~fz_=IK|l zle`+Q@3=Rfn|XE}C;wIvT@5g07K~lNZM3ywo=i62Jmjr8EuL$UKK>GtJw2C2N3T1+ z9pS({mepn^%#&g-y!yb(4cBHkKDXV?OkZr-BFEhoen7v*$UA-AD?v>gKB1)s9%7RL z!WDh$MLic!BE#f~T-4wbv^Z6Y=^86w`y*eW32Uvn)B1|c`w^qioSkdACS{%(^Q;La z?0O`YY;@wvTKt$hbr;!b*$&+MtFFv6-r9J@yFxJeK?f@<#H_9SEUw|(EG9n9oAGb( z=NuZIvm510#e3VE(Z|VrZhYtwvD#}r)?iK@I?`o{ER7Vn7u6R~=+FnG)cKPQ(6r-T zy40e13Vq1_yBU{u+L=jsW5P|j=)?W_&z_lEV$ZEt??Jy8$}^)DOy(v||AwS=-m_QV z-)F<8_n?PmNvu++HoMnr61p?ojqUY|V1NBqXVU*uWTeIzp#7d7S@T1$(ThcY#V@p# zx%6vGxP$hOP&=QwI5c`1x6syyv1-g>t)DO8?kfp+7Tamo_J6%GL>21C1SF8PUXOI3+~-dALfgKBh$0An+@AN zS#UGmfpLF0f?4xOmfI}bC5~!|K#zTAGm0j^*cDYvxj)YxnA#mvxYO|}%+X8z2r8Fw z*Vb6xSqTDW;Yr# zOcbn{DvhqMKPf0Y=Zv&}O=GwJ;2DCWAENu^W4RlLS+q*$n4m*fm)keWi!n;o<1}0s za7*i5m@W@JCf2o>g=cEq<#FoFu)mktoUvVkdW{w6`w=U4QeG~ylWJujV;!!dr3`Vm zTi99GBGDG56r^Z=lXb4qWj?G|jj%w0 z{tn`R$EU?h?=z@+lOfwQCrP|;T3*EqnF;9X{t&T}WVzrY4@FJ*t&ZGZeiU2!7qJ16 zKIn%>8WP@AMsX@51#{X)RGj!viuHDWSRvK11Nki1WNT^@*}4~|sHt``AJuP1GRhgYyGKysk&x@n)_=&&m zU&p=;JjQNLl|||*!??!9N$BIVVsz;J5c^y@7af<}X9u(%vd3-}B0norHs zf<*hG(G4qKcD}}3Hfczed$Vd7w=XCGDP}2g;aj@V3VAu^=Tq~ z*R{Et6I#q7X-8(4i$3>Gr;3FeMo2Vq7MdPz%|2C$MGJHk+5EmFHU=A`!0U_Ixd$8s zg}>sF!ESxlzup&xw-VO#Knid>c2_DO|nrbl?_eOGY?VH&$%Cb;-IeGS?su=>7B zP#w69)ydz;4(8j6KXq;t@Ar>DqZxBHy|9EuCq|?6f@SQl;2{C0y905v7h&ScBnR7Z$GhuMRwiP`{+-0_zJI;q>4|kHA*UC4 z@6lq5)xM*(yKk~@>g2hUe=>~Z&RbUTu_KaqwPbb%I&tCd^O=lrJND@ARyMawl^Hv6 z4>S7xWcJ|w72HUxAf__1M1193Ke{{=E#A_(i|r=S%uzI$8r1C`ZD(8 zqPXu3uhGXmPo{5^KNpnI%R2l{Lw@V(#Of(6ENM38PWV4WQ)(Wgz^JqA)2cJ9E@$It za!Qlg$`0o)`aeKBzkU#B)Jb#k$x6&DnIv>F@h%dtD?-!4--+!{&18aNE7*J$q)U4wF06nae)y%o&YTWj<{VC_OZr!G1GX6o4+a!op=K-?hgmsTt!`q!+`w7t~iM%#r1;6n#Nu z7BWoS_9@)u_^n8h+rxbtbDaHt{~D^;c$2-PugdKEbP#}ATF zo9I5zuyzGGNa@qb-(oP?)SuvA#1#;>m|DXXgcjnr+*_@`Zg`Ih#3H>@ag;_nu zfm7*;Wh)lf3r-zWWDI{U_(=sxjm4{hhkj%7N=V?9SZXt;ZbO*3E`{&fuu~F-v`hUfir=51sB~gNstp^ZDf{qbib>u9IW_jhn&! zu&6~BePy_N{n>0=fG0XQWY5`5H)6z@BbYRW61LH}O5o9^ftI^ZXU-J>n>Nc4_4o%f zb24?g)F*jt$#6sN-hy~Bn)V9?E&9y9dD6}RwpAR4XAP>&FSFd!W{1Z!}58HZ6<%@1}>)=TzM=n{6p+H96_ zapqiPe;{w(Wznzi$8AcWEXdW;Ev* zCdD95#J%KskT+b+xj*wWQ9&{043m1=4YLMnqq%iX| zvyioFwqcZWO*wI*-%4~b+ z$er^TBc8m(#%YuHRkXSII~y^12@0GwDEQYZVovrIiK{HD(876^)lhLOE`iMo*5VYM-yo~<#i%DQ zhvfuTsBQR1BtEhm`T5$Qx;GTc10p;(5eL!OV#L zJ4})J&&-%Ruu78|wy&$gW7Rn}1`W3+*~a7GBiTh=W7eT z&UjVv>fZwtu`hy+?t8)}=M85 z)GqTp<)_-riGV@&>zsD<;k*n}A32OGiArHbEv{(o2TgQ#OAjm6Y{@m1>M)7>#xi@w z2ApKPkUiU$hOT+fK<7(`*!_C4?8LZz?8Rjp#l337(a6q9f!nC#Vyl}zXya`?@uk&f zoc0x2rs0J=6JBi0-QHa#_%U@MyHdSTJXht5llY`GQ>~!JiC#%@ra5F^!}SHqgAEP z?Pq_8<(Dr;M`U{iNAG?TCkMH+pN}6Fq$xZ@##v+7t^=!??I9bvmIWT%$krvy{az2QegH8D^N+{2u@u%m-*`ckv)`c z#qN3K%h-mdp|acaSt}(E@e#w#C?wR2wU3Gy_c}KV42N%FeYX5!oq}gD)O#6Ir89{; zbty&Ao9?(uQfBRU5cC9tIVbS;P3gz_H3;9 zjUYp^LSQ!2h+Vovg~?u}$(_3e?3?W!=&?<`c#eM&THC(_9Vt1&+MTIKj(gv+?=9=u zq0&OsTazw+FixAD^u?aF&GtoO6#Cc^!5!#cj0UrXXIGluie-D&s51HU)fvz12W(hz zfFLMqqhq|+Gt`lNykckCZPxYPQslV&95T4`nyt9`7(F28*(bH<*eKOFv|*MKqwuYh zea!318$1S4&~S$HnCQS%v`=MhEbKUil5>LR1;^3emj}^Mz&!MM`wY&HSTL52tC(te zKFfVUJDPt>nUnB&+#8k|AUXcN^WN8l=GK2h?$7l(`Sb{+-0i`+Iy-Tn4bCAA_mg7T zXO<{($}_=~C%vqwW&%@@;>In~n8nGNn=`+Is@SK?mZBY3I#7AK9`{*oCEKN!g}A>L zQK3l`T07{AmL8nKO07lgz?p7Abrfb}UmKx+_q36wNe-KRQd|6Aw;bB4D!BgIvDzuj z+m)FwYg8flX22SMW0;KNXHeRCj4{qZBNe z4=3c9C;evJv;FsB!d2cwe8`pe8%+hKR}Vk$9#5v+Tc^&HR=2G+e!{nPfdW2eHo(w5j~H{ z#OemPzH2kCKG`Q(@%uPY8u%%R-IxIjmA*^PO^GA6O9shVUxmu)!2w+B^j7+DV<=q7 zc?J)aHj}(tIxu8cE_B%?Tls72XZ)y8K~#Ku9%OtgB2WIhkb503=(n9QlB6ZBM3!eR z{$o2OeLbfHehyM7q5CXpA3sHS^Zar8+UOW8JNO8vyr4v>v5&Tn3kGTIhC|BL;7G=K zT=#b+(Xx%ePn$YfUTK1F^^}t`_oGCmOofcRJGJtF)@In3^O;<#O~kv64ohSjJ%yt} z;)oZYdCw+TlT|wlnZ60%h3=2%f#r`(nz2b<6s+Y6R+Ao4sU?-5*xd?tek-XeNyGap zM@tS?3MwQ2=F*l)uO(K$;-I~lBRX|8Bzm1ZeVQvxzvm=E4q`;@W+h~!VhNSD6~Ow- zF_L$qbE(F1F>!vowo>2EoVW-i#IucO2nji8i9ZAD!Vi$4>-O}Sf@>-%2G(|Sd4h*Oe;Udn=JKV5-y zsf0Ue88Ed#t|DQVbmdYf1ETEH$?KeXK8Y>wJ(-e@J1(f;bDcAZ%for_-sLXrIDQ&f zgYz&LkdI4bqljUvuIOZy6zDu%1Jmqg0QbpLXb`eal)OQWpZ~0pBo8hmizUjH+hYv@ zho6Mog9Y&2K0w&Gbsp}Vw*%(Bx+>i8$cI$x&4cN;{)3*t2)f3pQ^I(r!{n%zHu4yF`=A<4@sDtLH=Ym10o0j3gEsk;Hvm z51iQ?4Sru`d3~`TaKiPR>CqZG_3tQZVzZw~G>;^2nzr!y^QW<1n=M|aCLl+->EvU+ z1coi&0f`reiFYMM(rC5kIP`HQbjyu^)u{?R_e&QKUD!fCjV^}SB`x?u>@?wl0~4@= z)pjCvza3X~U8gFGr;|4^kEoXN5HYnc5k|D`A`=^SV?C4A_>$**a?oH8Z7?;0c_;cI zZ&Z%3|7||a{o6u1ZkrLix}D%pjPO%D8*$Mv&{0C(S{dPvkGIL@nLmVv>$OB@#r?uO-SMP2e7m1|LOkOzafGr~jdAx>E7_(JADv1|lzt6~X%F5-=KVNPaiHk;LR@ zljzIy$$x>?1n>SptOaL4A@mx}Kb4Bjb2H)3qFG>bx*u1rvL*{(c~hLT4LZMH0)^m> zbWW)+S(Ii$T2%^(g1sqp3ui zm9Ep1BaC4`@2d+_m_{m!3+c8lE!t+9PH&%{AX;{}8&_>V1|yp~NS|SkMC-&hD$_NC zPR<^I?+qm5uDB7<*OCmWo|#}U2f?%4CD13QNREzJ084tl(A)QUw$%zXVwHZF)F$^s zoS_x%;<>fUJtASjA{SVuw*(^X9^+o!Xlh=-dq&GDL3)vhEEv+jl9ZXaV}gX<6W%7J zn&HrP>j)lw%#-Y?P=|nW*BmCKV zS#rNn08^qe0Sm==&6|%@S!WntJS2bui7wQuyeEva4C(r`m_Gb!2osEKVYQSYHG1)j zT7Oo6#9MB-{PbD!ao`8-a9 z9aoNVfJbt}$g+)V$>0pL%6)H-fX;*Ixcg)b`D^V&@8n@n)}+GoDVfCi@>kq&PmwsT zYlpOEy~;T)srYrl72w|9AoPe9Z0TP_w(JUnlb3H1hn=Q$(a>#d>*Yyj6lc>6r)(?; zSxQ`09k=IHkvXf^z zmFVrF-()#@{d5hi%D&0uERd%4-EV2n-&dsNn{?&l@mWI8!}HOdn-uJE+<>( zdJ)i$0NpKZWO2qToF%oM#MRjn)xYJ~hZB%bxn=m3{7d4zY7O~T$7{9hR+4CKFI=-i zmMpE+C4Z+p!8geqc>THp_b-15%N8CcOXpvv4{aTJW~MIoH?+axpf&JV*hJQldn8wsY}vg zA7Rh4+9J8>*(6_gHFewDLKTANLZyv1-JCj~e0X~ZM);>L3NyRjZ?59Si*KvQ}zX(K*teGQL1ZAS__GT_3tnQ+ux z2Nw#nDB3VYN;EFh-L_+KjBF40CSeck>ysyIO0~)SyI*MR-r>Y6QWw|$84gYo1Xs61VOu?OAONvXWtNIm-p_4|_u*l7jKIxALT-^Cex@t5P{1G34Mhc~Q>& zS6J}tJBiijz39rfX|{tYJwJL4nIeCT^xYmtqH_;{R04lalXpT{<3&^_(2tDD8zj?D zb<=x)&XCIS8)1)=pz`R{22kx(RGBa<6JgulG?>;k5fo>?!Jp6Yp5XSMH2IDz7z-?^<^xPDrH6q}$#IbB ztspvuhpFxMF_M{LH9Y#{JtEY+gGEOafxYmAZt9;%mbTxAl)g+7(@MZl>5_1k#u5DB zU=LMW>5hG6!pMskb08rwpE&lO!?#OJ@Y94SIQO#-LURu@H|CrsnSF&|G3^m~WvvZQ zEwqVsksZ8dXAoUj#WUt($-~+6$dx1(CTh4!@-CZ`e_qddEz5M`ac4Boe9xg1*L;9Y z_2bEl&*5a6?`>L>sD|CN8)5m+CTK~Q0kWbO{J_s^e%l|@pbsF`PZ-BO$IlG#t3TmOm_jUuqmOH*>hUO&4Kng&|&cQ;R1 zvt+(dMp~Y%4V;H3d|QA|2gbnsGd6g8x+)#H`!U4}bwow{(m7P^fgOMAlO%;<X%_&_f%34|Qx7_L|FL-6R+8@GB3YMdCR{jWHCa@G=%{2R(M8iQ zbgIR9;VnKdV2$;3BDxa=m;D?d@xBGY`RS4eTB(?rFCyo9L!hTM1Cz>&< zL2m0V_~m?pv^~BB3uj#r)=bf$l6fZNL_`$NR@(tfA(O}?df<#-Tfx$PF1Wk4z;!#` zA8)pZ#A;8%XWr()opC#0|9>}0iBCEC;2#Xr6gBB=jY8OdLyqkEIEhx&OvUZH2e5mi zlc-8ZMif?blz*FD2AvmANav9sgpo}lv#Kt=e=96`QVV~U zCBiKq-nrr?Cu&-Mp4`}BO>CnCkSMzg+Gb|Lx#2fxET8k?Tzrn1-75#jmHmmo8NgAdzZir^AL_}Zw9FCC^e)2c^Ah3e({*I_ zdVff?)~Vdib4SYW`NPGpXR)XIG0wh!GcG6<;c>G!OKfjlq^xBTbeGjazP=3Er0GZo zf?8-k&rcqFAOlAq3Tg1!Dy&`pkzUxn0J>)%l)SPzMRdOm;k{kIY16RzMB~|TB6rP) z6g-dQl=im^_u72HvimyW>HyC)DcMM#@fyPMn0vdPAK?^ zY4VCJDl>fro*RtdNt+Wa+TIS&FAm|>&s*?=1^m0aC77fhL&Vzv;f>uJiQqyWL|)=K z%9=Ugw$~OXteJ;R6CEl^;TG&RbX%Bm&qRo?m*eYWX2YEH20$Yi(ta&oGQV^&wUgUJ zk8;aM{>GR1YIi;UI>{gJoKZ$L=CXJo^a}YFkpWi>E|6xp&R+TC5XKA_=_7?%8>^H()2`V3{g|}rFS;mqKA8&MbD?H zh&+1=>CRUSY4!KuGS_zqoc;Nk`G4x1^Nt-%q`w1Nzh*fq9a3g0mG`lU`WBq3nHN*4 zaTbl%%VqT@Y+?+Yw{p1fpV(D$4xMas;k+wcnfsbg*k;)o;>d*--1}NzW}0X;XFoWT zd2IL(#fZ$gc~7Ld!HwEX;S_c5L7F;qCcB)y+3U_tGnvf%*e%7qyY!fCNnOO5R82;B zh!b1iJc+LOuI4UOIdbEiB2a*|wz&Sl6Ex%GGx1n)#q_g?MzOw6*?jf6) ziH!C}2PSesB4=}S8*^t~IOo*i#ZI_7iCbLo6*V=RvYMj8YkjXy3O4O8K^heuD9vg% zGQmw~@yR;&v40uczTX3_eYB04)4ra2@o_Jcm*~x9E$U-^oqtp$EWC;4XRl`Z(kSaU zhZRImGT;tMWD^h zVkci4&3UB%LTf_yFk_8YaZd3@%z{fwY`8%?>TE7%^$w~rTkLnE;v{pfVwW8=kJpxk zdU|mRuNQJ-VwN(SMpXz3GSs*mHVy2CkXHiZt1D2rxjv(uFoQ8ZUd39k4`-*8U9XTt z7;>f_)0xl_YHZrVC=@@}fpIe&!(Ci0&q!y?6<>^2XZnf<+0$D$ReTTh<)qa`Y_|Db z!DOKu<9(0UCN-&Yfu@6KjKTjXIuC!WzCVE5Gka4i5g}ROIp?`YW=a~A2JMn>sgyL7 zjLeXXN=w72%#z}s^W5`J`n08Nkn%W{P;!+h<&>dKb??_vt~$>UpK??$qmln;fz{b zFQ3etnV}04-Yg{3euThpY4>oQs8xRB-V$WWK<(v3}U%cZj*JbDiFU8+r%8>zyCfjQ*S3mL2*@(BbTeT*{; zL$ITaB`I|q2j70zB;6M7SS(nb{E@qb@rrn`DHVY7BvCK4(3dcC{)4i!Z-bRB<*eKE zYHSg>ij7y{uoHhKky@L9$Jg1xoNE|-b9V*cl^!$;V)3C}xgfo64mQ5H5}r!)CljDB&PAZM@PakVJ)=!Uwl!R-f8D42Jq9!`Z=WiRT#u951jT zM)_s9%S0J^#c9H4bG^x!;(K5oBVzcv&jI&HO|U>>Icasg4zz3HNlcL;JXdG~l8Sg( z)xh4VGKwc+tKG@vp5-ucfvA`GT^f#l)`0elYw_H;N)b!uKrZ<4!S2$F08di{p{oMn zb5&KMWM%>5H1?8zlc&OL9X?!`twqMQ?R7ekESfhSF@mcM6^ zw)-#dkKO~!gxlj9t85-?`U;<(n+Y_xxqxJy1Q5D36eit@Ap9SHdD)`Q_~ZKTypq`S z_^hWa`8jBhcR$VpuKWrt(~!j5y6hN6(~I$%4!6$9Mmqs&V3|A3P|C-M0qwtC}< zM)1OaFE+Ox2ZH&Z@S3ghyuC@C*dc2+xGP_S54F|+*Yo@Fr)>kgqZ1c_5(iygm~M|ee><`F4_KzLN!$DwDtiIcPq9uYr+FNEM5@Ax*EQjP4ezjaPXN9%kcd0% z-HAlz3OK%B9d2VrS%;|$iI289bRKsDr0f)ft*ah_?8qBi|fw!(s?6mV@@Qlt! z%=ac>jhi1nCvyl21H_2p%qGwq(}M3-X5cfwTJYZ}D|l*xIw#+%|+I?a8UhED~! zfSiE&p!B#Yk=U;Xlg_yi(cfIWEH$51DUkvD%P;b*RHa4yq@Y{a3Ztl0h|VCr*EAeiTY`?GHXp_v|$oVyZ8zYhgv`)nZp zl|H%hZw$;5UIwi%m551lH5hgC;jNsf0^j+32lGo*@!=Q8*+mw%ApDB~e)6lIC$Z=$ z79U#;tu1AM`tR;)EkP={Qld({KVATZX;!$QB?%fiO@k|)dO`T9_juv#*?30IY~a(v z=lNHtu{Ecr5cSi=_|EEyaJhIV*7S_28XA`Y!uW2ym(wTXoRB+s_ijDNZgqz_$bc9o zEQCg#XR&;S60uY3bz0ef44WMiv1{Xh@CHs);JAtt_~OzFU<)S>yGLKLR|Lu+{I_Ur z<>qnxKjWk07edL6ZbWQ*t<&mNwfI5i6e6Fk z1b3?XJMm1mkO?hUalqza-eGDF&!}vKBZ-P+ouwTt&v)gGUl9d|&IRB_oA-g~J?8u8x9CsnN1*7Rw2A| zQ$!4RgDG*3(TA`2i-_!iI82dpCuK{#;L>mhSXD8Ovz4s7LOxg(ZrCsr}_G4^8s~ObP_aU5OH&`hh#cnQG2ui|_;&QpWI7iQm^A6W)shzbC`n4M{}9xgR{6yd9g2=74jHQ?b7DIS}PyL3ZpO z#H+axuwdj7kZW{+pO}e2=j|`{P_DdV+mMmtqLBm=Uj7Btr0pl4J9@xzpYc$GtRmOy zRbklDm*A4wE8x8)1-}~a2ftV7yEI|9 z7oYL$B++?b(8U7A9BRp~#;0DUf<84r;$vCCYFc>W%j;~+4c^+Vh}U&h;?%7>NcF>?AmDjEXj;;QW#&2KsQE=8b8j*5)@}eg zH&fv5L&;=tk}4dty9Xj(*W=*E&*1m7NxVWA4XBisi_gy(tKMlniMWk3fS24Wcx=!$ zJnHHK=3DZ}MMqDfx=|Jm84QCvA&yJM5U`7Q}kCzwynqvA2o zxTC6fb}r91@DJF~a2XqJ4FqaOR^lSQISj7!hpjg5|z zdwoMF!jcP|C%<|bs&{sgH0XGkWRlwsxPh9F$R0p^SL9-42P;#O7#x+#u> z-$k79qEY|qr!B1@-|+^}S2xG0wyogrj2x`9JOQ5x{RGOVtt6Amcfz#kzj2D!5UVnI zF-{CkVC~#EoGj@@(q4E#+qJ4hdF^T_E;Ap0Tg@ZA1JmI13kzVpnH01);**Iw25`W< z8qb&A!WQj02CCu2P+ftVK))4 zZv%dw`G-Aa)xp^>zA(2S14v(R<6T}&an&k5IWn^a@ZxiEvuNKa(mRybE5#!fzI?dq z(G3t1v6gflH;1kfGsq;7xL{SNHH=^9PyAkpn4#}6B=@-~*=HF>bjBya4`t!(u!j=a zie7^|{&Vp4&(q;jIajh}+6EYOX$?6gg~2#^d1CzB9Cmch!GBGB!Plv;fz#^=aQnLz z#9GRP_}Pm?j}I~+L=ekHJQX8$21-y#)|X^ht%I)WR~<+0UI0S>x$x8`J}H0y4_K@S zgiURYc%A7So=ds}ap1Gy{331qecSu$@IZIs@$G8WqAX7+Gm6N2JzLBlbc6@bi<5!g z<6v5=GQR%g8<5wm1m;;9_;1>J>>K|K=!I**72AjLSl{dC}HvJ1&Bn+SZi zNP-y&P54XM3ihMTR1z&S4u)J6v2f*U;BqY?5Up>)eNqO{3iRX7bH{Lw?lOEuBwuxj z*hNfZ>)10ZEn&vz3m{GR5S}nt&g+|NMAlLZxLIF~OxY3$dllQ+VAYkNp&<@Ck33^d z+A~;R{Xf;nKk0$q{1WV*EuT$(eF+-lp5&0HdQ7HtJM zyMp5#I+czauBk)Y;%2r0o~U82Mv`W1 z1D#j@!_z(mfLS+Z!b?ZjkX1|O!H5I4fLmn4ZIYWSBE%YI^ zcO-IFzreK8C!k~1D{SYGjz7-YMjWmgLhEh^QvX^F{)nl;#-Ta9MMwU_tFmm!<{}BF zflsSJ(m8qdNvSmS2JX<*$&jcl90dzLTar~q>SWiS`RvzQ8la}j0Uq%9#yccc1on^S zVvXBXK<2G8*4a6YoR^&pKS`b79lkvdylvXk)>>uO6-Cuc_dM+eqZ#lzFR%LkAf9`Piu`sZ{*#R!!xB*x#Hi3a7RqWK0%HZ(R z=eT5YA+KYZUUgWjn20f6!D~Lx5SgE!L9E0UQIpyQde+~>1KBPRGrpvam4ss5=fQE| z8aTmmF?sG-12Rn*s9GrogOWz^-GgG_M>NG9(@(Sc?cb$UJ9wRPpD`KSMsg!I!MfLS;7jlY(75*%n4x-8^gf30(4GJP zW0?iF9QP#u^55fguOQqtwHLIQ&ma;q(&WoULG|FtT6R$UDoY*Jcq0=RvzhfBIQMV? zh*MG_J}u$ZR{0;mg;{YpW{(w+^&Q0HRkh%_dyscc&d{U*khke2 zyy3iS_-~{Ww2yd?BV0v{>^FP-E6D^q*hvy)X97I#tbmd_>14CXT_9FsNxnPSz)9o( z;1Tn9&?qs5&3)O6AM~|>L3wKuUhhQ$GYo+Ai?M3A=*wWcZv%Eavw>I+Cqn5CFOs2- zLCOA95Iv3Jp)M1=Pf`Jrz+9HjHpE4H8b$p4O>EY(1s>QFNQ48*V5ZGnsJM73&e-e) znvGWT%KZnhbLih{eCG_7PV@k(E?%HrZ2{5fpA1`k{E5bX0ob9+2ZMZ7T;x}XC-2`( z=JagEYu)O2&kU8}kCd-~byEe^*PF2E?@(}c<_^ZvFBv_bvI99i6FJT|USKM&zoh9U zmzck&0@0tfp{OJ@jC;Sbo0Bp8%M5PXhD?jR=naE-IyR+S5R&Un>3&DyAGs<v?ngIS<`0=`}3j;`MK zrYFS|=>fL|$iDd)(w!ybsxww_Ue^{2#p@cmt+SsB!d2AKtRt?BZE`kuc==m_nX?Dy zWu+-p>y=6|MG%}uL(n349h>qyE(-V!_!mSE<)JS8NK+>d)=I`x5nUG`d^3sv~-DK{< zh%)YD8_-%F<$o{TfWF+{hiro;3ZLE`WEA#_JAZrqj7yn252>bNHd2?lInj9)@xPT~@(akdueB za({3?R>=sr+Z|?pOzcBzZd~T3L@q$fZ6gIY#reppv6^aU@1toad}+(GNp$8W%s7wT zLw;(t^oMdhGuN^ct=*nS4bB=P)s+^UmgyWiX-yL&-;#uehUKYfDC@j`*?j7-aTkl#u>w8trij%|-+wzF+? z()tRfazYaqcvy*A4YV*{F7BqCZr-RW_Yqg;KFPVx+=fQWo42_yvAe^yH?HI&-^EhI!C1Say2{h! z<(Fq1t0!s#q+dc0lFI0tUxmnCr;TR0pJ7TO447d53{-Q*ny+^#iYgAtQ2&-hLDGUY zR1qpbxzAr1_sFt8I z*_XPHtVDwG4>(PqM4A@1j#=jL0FAsFruM~}^uD8rbq&&TR{H&qF^ln|nerSXo4uYw zVIT9TU_SF<`+XYr?<;rW*lSeQ-bFW@UX2RYa;PVLr?0^zH?Q~ zy&aF4gFnjAV}su)LN5_{`1jJ!k>lu^l%uH3R@BRpT!!v0nnH_1=b#(44RqYb6Lbz| z!XNl@iaMl}pjGA0$blYWYVR*&oK4oF7!O2^JPuMLI!43Zir8XvGe#EQWNrp!3#RC= zMPFme(DD-*^wah$%p!wmZpkYLbZ+n`cWlgt)5;MOMt_=!Za!8*a+5C7MG1M_kgLcC zn`ufvzzuNaWk(w2oyPQgj8Ki%UV&4FGrA^v7QM-P#!2YOb4fMVId1-Px-Te=X?u{# z%^auB_&pWl)E7PFM+l2()aXM-vD=bb@A$<}aZuo%JdZ`6%zNpaTd5RJ%VXRYy=L04 zpJYO%*HgVoDO^LNi4bOAL_Q(UIaBki{Hq=vbk8?OF5>+Q)M4fM{+SIni<^PERnA{vZWWg_ouiCVvk%uox<$bM9C z9-I_|Hr(stT3vF`H%D{!*e4B62=%BUvzp4?f}r)QwoXD`w1*RoKl zR18zlyqQXT)27wCHwb=tT|ilrN>CMNDcE*5mx><3gh ze<_kbB5DVk#|loUOrwsA3&~}TPELncgmg3Os2zHbdbYaq-B!(_U0ZJPe?+ArwK)fn zz_JJFwZ+p9(_N6uRzCA@qYvZNwwM-_&tTI2J{EOhFY*n8BhY2rV5UGhnHlQ%!F~R? zgjQ(ZB>_atIqv*rZu~?s0VD z1#J{MFi8KFxT5mu4;ZJla>!#?(s@xx6|#{rq(#zFv@umfT@0o%6K4iuukSs~ zm*1P|?es$AxmgS0ah)jj*+Ty70#`w&qzW4PIvKSzxuL>adyuDO5LYweK*j$haAbih zQ}VJD4Q~xcXZBCw+AlY8*S^m~U*!W(xBW|WV!>OKm2jUsytNsPq~@SqMvc^KrM}QQ zy@UC$(ul?jL>>3egcJ@3f+Zo5y#@L{<9*)fKC*|BHwJa`i;5GW58iN*<7ci@7|nGaI8NvD-yvg> z|MO3@4!8A#1$A-ZBi)*ZC{43lz&uG7#EUw&>dljBLkou-w(8Iof;j5x5ht4GE1<{x zEbfWM3hHTafVyl;W{NT{(NA1DnuhD1v+MEX8+3y$P6_st38Tm3{(*DNL6Af3eNs~ixhnjWEd%(GEJil{lVA((Gl zC}7&~Ep!DJBQ|;%T@y83lX$`OO2-oVY{dy$t9YIJZ{9Wf>(^JNP4bpN=TIlIw(Mjy zQ5jl2tB>)1zn*#QI>g<2lEard zs-eS$|KHhhM3*o9WeS`QQ5`=o`l8jG+n^+dKDpduzA|}ep|}NO($&iSeiTieWO7ik zlfJXr2oKe6&S$J7_2|YEZ-)9;<0G^1;r z@XEk?imtfRk8dW?tw(iHidGIkIv@oNUG3mU-ZMZhH5{i>Fi#*-A8?UM3?8tK<--4R z{Obo~(OBdM{^&oEd(rZ%;Q1tRq&<9?mbx}^jm{@g>!lz@Pp^tj$vBQ`l&nyV-V_@B zy@y|VPlCFoGH80Z5&F5_4Ov=SqN#2rNWD0TdK=5rzDI7Ty}XRp8=9gmM!TuL;|vO* zG*SpdD6Art>Q-r@G0&5nd;JA;zqOAE*)|FJSIeV;3thCtd%7^wtQ}uy^rgQ>`%ply zt?)wS4^B6}mAUgGr3*yw7`A*#YNXP~T#HN*wVgufQLP5<;KG;8#YQRAG__$P@$}#B4D|C<2D4;H53N=+q=znu`m>hbz{tB=^l4BJ zxnU>a)-UrIpWs`_(_ccksNa_!>O6~vGmQi%^gU4GY9nfK-~waYn83x4d=oT(dCp(Z zU5ezy<%H@ulS;OjqbA>4ZnAU;dfR)2nbn+%JxyD=8|RIjfAFIOGLQZF<6f*3>=QXh zVc}ks`Q`$DOYZ|lU-T@w<9n12zUV>cZis|Tq7GkRgSK#N{|aavcwDpz`c?i2k>UUiKw=q#n&>vlBiiaBmwF3Ql*r;SG2P`#)}%LQlCKXVn)j+~Du_|_BjNp&sS z@+clL=fu#_g|2j(WHjL@9T!8*r=Q>$%WhhDcn)eDyviuZ?4+)8x6t8OW#{#qtdN=B5y4+# zEogc$hk0BhA>pbe&HV=WHiq!sAIzz_R@SPpYxypF}J*?0ZD0-DJpX%=NKZlBr*W-FR z!GD9O!Lox9AHC0QT;q?Pc5SQ%Ys+smwdH^>-?)pCh% zh3Mmg$*5l443$J`(@%p2$gzs!vTf=aMdc5S;D8nqBp4(4oC3PRl+vZ(ZIAxlYnG}L_qojh)e!XqGkzVjzDWzz~ew^CX#e`+&xBTX8yReR9ke*wsLZUk+p z-bHO1EzqdTZRUxU9@R*XkD)Mo^_*H2&5h&VIU z<5h&7*?tiD>))XLms>c)j5vn<9uHSGo<=i_9-s%x8)@^d3H(tvY3C=}MvSAH7LEGa zO}+093d$ziaQExZF&mHEVWy_OV*=xL3tsLNqJcFx(WBT9K_u5mpBcPkB5zCa54lak z(ZiEK4Y>sFT+qU6ZMK3DgZiWj8=S;j!u;{dJ=)~_x5a?(Q;rLi60y^C6Bzf{5N@>D4|a?r z;I@hv{D|^BkV#U+mb%JjEY%FXPDVeQe_U*N(ckrvtsrW)QQ( z4v!9eW?!704z~n;2lp&40TbIwHap7-UXpkZ#WD+QkL$zn2fy*!Gd?*<=^U>< zFYmmqA`3qWiARHrj1{(fM#c#s-_|&n$>daCLc2|WjMln0tB=z}VxrZ2@ z@=YA-Z#xI>HUHtg%^AkqH$MUm-L5e3pCfo3Z$t`>egL;LZFqKaJgE0M<8Zvi9EPio zf!G)Sf%1P6U`z5eGUj4PF6$n_*Jow2!9fZ*?B`70r$aWpve>7*4&`i--|i{y&c6bV z?X%{^^iF~6JjTP+npU3r&kOj!$2_Pi{tcvbJOp6}B0-U>J_)GNCd0~W@nk1}+oK#w zkZdA2drE?+teFPvE;*6TZR1Jnvr62rR{$oxQ76-9dXQH~8t|Ob{cOQ5bLhl;10up6 z_#86@vC|FU^gV$fR^bT9{oR2#B({NvTgF4U?io*I!4e=gWJ>gdOvw89U|f3K3Wxu$ z0zVeq1KA4I?B3UtfmnSgTMB1m<8vx7wY-hz;?HNx!hFbB`)r`EB?YY(J;xr4$CHkK zjo7WMfZgVOw`%X>g>c;N2)w@d4S4_lyi+8af~EhQz=y}pW_3&J?ThZuhiB$p;kCW_ zgD2aZ!D~9U!$Fawp#JQAo?9rNcn64Pfy(c(!*B_>xlsG?}U^nkHe)=(#H{q8l5kFANYu_$O)}EDzvy5wSowX&IG5HU> zo3{ZB4O)`m;92Bdj5|(Aeu|CLJHaPYZMb`NoX9UgS&r>zH{5$&{eFUg7xDKUFba5w zD`qTV56&3Dt%GhrCb1ERNta*?*FL=PFT%V3-UqslY2Z_GKe)E!0eB^miWA=?;134( zu}k%a>UyOS5D=sU`sYZ&H+ysOVmks%ig$z7H!r}-mO^cNs@wdM^|gcq{OJIv%PjN<0dV^~@w+@+6!tUu8(2rVKpt;5jZIO2Lg`!`Slt z7WVTY8DhEg3U>OSK-TQN4}5n1z^Sv2;Zz_?rbc64`GXar5m+Ij6;4B`E%d@#_f2um#QgGb#;@H$AruGiE+ zM-c~lozi#@3U$HST?){r;RKN4RH3bl8kEXXgQ>yhkX#gl>$)G{X*I3j@51Wxwz#Lk6doXpTcfoWdBy= z{EPup4p-x@WdYcfdx=AfXTVQ(ciECaBV2sd6N9A_N!t4~(7ZGlsN~jwKh0^ZM7s<$ zPxc2Ld1`PYw+$~B@<{hUHvCm&BM!u)Bkarw<{xU1}+h*=ci(cWwvxwRY&s_C-z zAH&#RQfjc=^%^MWt3jV8A-?tT0SFoRR=qG(9R}s3gR>ti@%W+~Y%JA-U#^uTKW@6> zKZWMJYOPzmzdxqJ1lKY4-6BuiIsPy%^sy!TV`bs6^m`s2QHL?AC7|`-QQr0U0eIcE zGfvXm_2Aa~J$R4qDc;Jegz9W1H}(ks0=tKQ2|MY!;<6+UVD#uYZoSOno8o;~=fN55 zuTa7J9(5Hg?1%>GP#+`&YLW$97T$ABmspA%byr$7iA+lgUT|BJywR-#)0T*?E7AGq zcdi#-`T7FvU-AOmESW;4ikzzQZxUH?smnM&SDm00TG(*9G*NZe0)cr0KJ~dFoYtFD>ZjLsu-gO$TzMjHHm?q(p+QaPqd=8ilufh`>?&H(O%c{o< zzTp~ueJA~=dgO0II+(v8683+$hRU+@fwgHqj=Y*mPHle7t3JB0YLbXmRjkq^{-1ML zx1C&zx1X8>@2v|Q$ z#6R7u@ZZ>7ctoz8HxFF}c89Nm(2u`>qKJdOSK5pdG=qqVjRy(WPUIcrS1Ca)fOZ=aui0gKM|u@iyIIvAxb{wNY#aFpGxOZw3Usp$Qj&`PE7AXX!W)GGi&; zHr@meOjWAkyGHNgbtJ!}Sb_95`Z=U8Aob2-kFoB&jYzW{@s1;F~Y zB$57^#WQXX0Qc6?D$@hq_*JPTFEH{UNKCruq*CBXj>%2s@wJ^;hdUMkEK-BU?Yr>R zty74!h7#HKI~<(M{l==<-NMfGrK~BNjL$wNxOLzdUi|w$rn5}Jmy{`Zf9)T;~!2?db!ozs)kI(q~^%U@2$%vPD`Vm_(PY&q%CV;hzwt?niNjS*GgMugJI8JpA zTRAX#M86YNAFIDut^a+c7U`dQ7mF%{;2zHlj;%?@Yu<^$<3Ir_1dOo{oXm)? zR4UuN?mE^N%psOdqwH<9+t@BiiKo0Y5Uw~o4kqr?fWHNn@VMS`=)Zu)VlvK9s!QYr z70R*divY3lvjTi8b@qdWA}O-T#P>6b@eZ9(e8{T|tYPwar`xvxeCHlEye&cgG595OA5t)&#y zk*j7s-WZaOJ=MHfG1B;0gExVt*|&-v$eoNXzZIwfy>Fjf`(7+s; zq+SCBEyT%w?kV2CeFpGk`LL5sZZ|%pF+W#Q>x@;%3=1-eId>ld8psp*5WG( ze0-5_j5!mAMfFOg#xn(9cpV86@=VC5cL7A(djc3NbtJzpA8_h^VM_d8WZ^0Hc0}aC zW^Z0u0&fSH!0T%#!iAX?>=nI7V9`Z^qqx2@w6~KWDFQA{M)BSN&mI4L-6^^5*5%Ffe2DG@$e)9hd(M z29J*LV0?}i{P9+gnCmIS`H*vp#hH^9t2yBBXHra z0P@UsDf#iW2hY;gf{UMRfGyu0c=vC5!|{)7@uefnNVR-1ELL4&JiDZ!+-tkUb zuwHf@l)m>1#N}DT@6Hc!ysaWRcuE{T)@;PeGEsP7a{xJ*X33lDt4v%c`U8EpdC>Uj z4)*tL4cHs>4bMLz1~Zdg;4GaDzVnCeX4w1r^(A}e@Px0JpA zcmgXhR>ErP3S_p9kaZRvS1P@dWEE@TJa==l+VKQlX+4cheHDf$E4tuonIdK%1;CK! z@_2);9URen1?-C^LC2({*sJ9kxc#CL2&O86SHsq1;Z0d~;gotHHDV5z;BMeIaXM6R zxQ;cQmyzq=REcYoEwMP2fCnEsz&nMj9QxXHz^qt(*csFe?3XKn=!8ljl`9Wlj{L!H z(l^-NQTWm(;IktGOBOVyzgNuv##6)`;uQ@f& z>GG_ryp@VkI1>ipk%kJV`HksVv?mB6jAw!wlhSxcm+F9nbqd7%V33pY8YOUMpB?_O z+6F9i%XM0;Sb|Oengjo9>!J(t=qKV7lqvF=@!aZ=n*b*EKY@9##Us8uK;w-KI7EfcpV$rWPq z0OXuKisyZBX4Q(_!1OI2@Vu_?V0u?3uO(Rp*H@H+XBVG>Aq_cpU$hP!7sKdl?K8OGZ#qw4>gS}p@gVpx(umiLuV<6Cp5w)ec=Dh1 zs(7uRE32$8PS(CF!X4ZCfNB)sbr08wA#mDO_46KN^oX1PQ3(?dFI5XsT=pr z|B54~t9t|%AUxXew@nAYu^(*F0n`i)@3>o08-x2uWX9du(FALvXTmdq~i+OUdE3m|! zewOB0@t#t3-Z>GBG*dbU`22Puf0?hmnT}TYa)}3k8V&eZT_KjtCHU~PTzM*DuC zyG4NIY=*%OlPlHs#xL>s$Y5}{PKY<%S-`r_$^`Fjijm1O+sPrj0q0N_{-0Iek%)u^5<8RMJs)uBpPiQ2QeV`ciSPpUz->T44 zlS;W0=6?mhU?VlV-NjA%ze9sAqo*IKI6v0B%y^EfQM=a*(eo2q`8wy`aKAFMkg@d* zewNpEl;Br|u7Nn3yk?pJMYwUl_nt*JX2fyl#nRC`6wB$Fj~DeFQ&IoMeyS*aq^2!KHCKx18_~buEj3FSgSjTCBU=WQEVrY-oG&v+uf7nJ-B)q0 zS`@{|eEcIwEzV(H1g@YvUY1g}BWSTw{hrZUnM%D9uk=l+nl(5we`F~*1Y0t;>`QUix)BGD~ z%3~dFs_GLuZJfCNOnnt?xKs`EqW-h<4%#^n8 zauA9>FzBEZPjJcQAsYW!%vs7L1Km`*&fP1waX$4ZmL8iPi(Y(wM(z57(A4dI$lsv> zZIIPxhDKv)OxOlGCqfOmyEfA5Ut>&MRvMRmtq6SJ;;3|ACs!=Ho|`di3n!I%gC0Mg z!#vQaqf*=Cm}zo3=+})L8XMfk*y+omJ&QZ(y@^lhr!jq6n<0Vha`#Zd+S{CIoE2Si zxE-YwdeKZrJ9K`kqVsOUbh@JZ9^)Tznmf$zVEDBnkJBs!cy)8&P{zzsO+_ zO-oPkn4;1SX44ZzI#F#CIyz=VHy`h%7S|_oHiZRry~AW=>zhuUtJS!k;#ictzLfr$ z5=V_cc%#WvW(iIe2Gh9#=?EQCpfNTR>9nBJjPoyVW(~9yEs33$Kvnz&PZq-oO=Q`7o||?dZ|r?|i*^yODzQaay(b4ATuH(B|BOOkmA5 zWGr`Fkn%VdsWv}FdFk$G;YyY(uGL2sg~3QhsTRp@ibfV(G-@+{1|3^Pt!0&J`Yw=1 zU71W~RedY(7IzoiWqQZCoQO7~A!o|I@IM{@EMN>Aj3Vf6K&#TT{iHSM;t% zHVM5166$*L|j|tWlisqhATp2$DQ+gL#pxNIB zY1*j1v-Pe|A{W&yCh~0-Ip(^Tj>=@Br{jKdRqySP=E!QUM%R<(565$jU!;++#E~NT zO?1tMQs#O74+cLep#|9uoY(S=^zx6jOyB(HOyePI=5uW+V`wFI)1&dRsLi^JhVT5v zMZHrHYD!}I8mOSRZ?13~S$}$W>S5&C^ap;5D?lrf{WwGUJUZ>H62mLgKxbC! zbH7aztzWi|Zi-D52qh_hzVmgi?6o!Rv`uARv01daYc~CqmCX#_>=v{w@e)p0G7%lv zZ9-2)oMT!pnbZGn9%Fob4bVSZYsSky1)VQ0=h|i%lOldN*K96o@~wEoCC~lJ&AXGu zk%}Z#DJ$xfEuW17ma*JJO+C7*>o0mEyh!~NSWfN0Wv2afE@xdcpV1m(sOzguH2C-^ z1LvKi*<%{Cq-Z>~t?uR)$vB}4wWs+}&&}wZ#kU#vO=lSSj||Q4|H1vt@uWwmOr*v- zi|M|hOw`cjAxzitq65`4nZQ@6XrFR17e475y1l_c@S*t-b-nl-sSk;IqV0;*dv^oc zvuhD5JU4_M?OMU9ZZ<;35fjkQPfwVzG8QTwnCLwCJB4Qd?BH8BJQ95QZ?a&2ehCWK z5~m%i5)@mDQ9-T^opMl4(6m(vrJzcb$G4*5YmxIo|6?>)CW(~$`!Mc_;==qFx!k{n z4+XCewV?^CjH#c!KP`UH#r$}tg6!5$rlRLBYWaAGzLuCzpJ!U3pS~&7tUrj;y41^` zmvMuPAFu5!Zn=^B9NNi53>u@fjtF|)Y%TJ!@ut^8?r>Q;+Gy9!yNq%UKvHV&IUiSP zS{)@R^bYv|4{h#24R`oROFN0%PeiW75FIMHRvYDu1A6#(4)?Kf2TD1uK;I!5jx8zU zmd&=~7Wq!0GCNW^^(BANzrS11l5uj(@PVUT)T30oG|Z1_X>g=RWdBFedH>b?zHz*g zq@|%0q9jeyr1L)abx-m(#(v)#xR=W<82dJoaxsEtyega1BI}g>K^>qBA@i^&OBN5l$ zA<{>zusu&6MjIOnF1$^_Zf{kzJ&=SlI}I_VESCDcazV?{b)pQ_1=w^g8G>#W)3QA_ zfN=-$RNF>cIH8+%N@U{NcO3I2{y0wF&2dtjH`8xg7r8TS8IkJr;^nodilg-;x!-*? zsx2;nOBhWDrXVQ3yF#TSQn&^21QcpT;S;hAKVCi$_6`x)b5x#;)V0E%5^4DNuY*Q~ zrocygDtdU^7N1IpL|e4g#d?p=LvDUM1m0?cx|B%NH84b

    Xu|eT>EL+%ezH5GBUG zrhoEN>FS%gxGa1re0;GFV;DIw5Y8sNCS}ri=^)0{onhAMjU}snZR`{yu?uN(=Q$DF~%>@X(G$-r+HKGP|U zW`g81^86){oICY$D%taN7^_-G_Dw&23 z-CsoNokv0LQzHMpS{gMuvl6U4j+04ey+xWtRve|64RU`~;pN3F`oO)HcKNN~Kk+Vw zzd6@LWAbZ6n(|vQYxpD<)ZZa%o2J1(xoVCV8I7`!bzxY_6u+EU3@snC(DT@b z$?(w?{Irip@1%P)bn*o7tkuTw1OojT({b{?*&w%R1$@x&<~K_#h_aWRfVof3@B^OD zCR6)VK(t*K51)}io#Q$T)r#gg)|@kymO{3k~0;69&UD0cv4aT*L|-xH0WW`i^TBivqb45_hvkH6=3Q=hIg zXxsMe~QS`9OD*V-^0oiX{Ks#I> zY=>3FOONj-d*5CMo#F^$DXoW(ZpWixOBTdeB=Vap7NP1-bqpOECSw<7!j_~O`rbAj zjc>ezQ$rcB54j%Df?kw*wuMH`^9Ao&iD+=;9UL#$6;H1Ff}vwqflJc}I(WX9tk`CN zJC_}XEdjM;>Hxy;^TxavPWu^p@F9L(ok@Cks=~q>W^`unXX0EVg}2Qw;s=Q+Q znp()27V5(7R(#{lGasapp#-0?rL$vkIt)c;;)t16Z@KaxYfgg zM_sUYRTEt5vBLWHbUK2R(pLF=D2?KyZr2_h9co7{eD2`qz)+}fbby)v;t|TsP$%dY zc^y5LwElLW%*R%8R(K3d7G1{?i*Cb~5k5qEiw1wflXjdUwuX}&N8IgzC+rQf5Qlb5 zqMh5+#dljT!Hi9L#G`XJbf1_{#$QuqGD6JY@#t~le?iAlTd9q%)_b!%%+ZJDA_^!=k!wT>IpvsI}i2+M6`E^)(yV<1B^k+oS2zpmCz6<#J;E z{?%aa$hllE8A;t4nA71(Elt+KwIV<8 z+B69d1~rrZIm5KV`W?R~BL+AX78-8fhT|6nBKOyW?Rk6&r44V>C&TMt_RVe_s`*1q zioc`Xp-E!%pcLGy(Sm=9?-A+K@z|TmMH z-`Ah=tt+O{^`9d|`FB$BzxH-uKQDsDuN;%RaUzWlivqy{CH!(E88yBbK&IOYy1^m_ zD=bT3fw2qJjrAoT%XRo#s>)PlaTk2uY(Wq1DWv}LLsa0Qi*8#gaQ>qQJc;CJ&HNd3if=U61Xy}=T9&fVSTIv!LF zxO@7XGN_AKfyv806T3KLOn9IOUQ(;+@UIz=V|^V;Cid~Km}-jGoY+r#UvI$R*IXCj zTPxnMJ3zWs%0>U2`q3u8if`f5g3Ap4AKlUZOnLAZ5G<_SG$gPB*uZmH1 zL<&fsz0ZG8^%R`djd0C>RahWn!*QW6)1$xFK*IYx;P)P)<8+sk4-S%$wAqft9+iWd zOwP6agW-W`j&uE*W>TfjZ4y2oGJ);oHu|+zwd``h1eWV^^faW85wi zkJ<1vE<$HyuTv{nIULFLsWgqZ}DGwr3&w=4lN;JFg5?B9wz^ck@+)B9yP_`;<6I@ahn8wR}F_XT95Is#xlr|4d8DM z7>lyYPjk2JCBiucA-HO~J68oM!}duFa7jWKIxLgHlUpQF(MAWH{(DW;lT2{mwF12A zBPIB{*$`*ElqBby7;)iVNA!F?k1u)kIv#uM4B5F!*ji`~IyIL?&wIDS0RwNc{bvkZ z-T4c?w;jS2uYZw#>9e?Dbvg|&c>yY`La?Y%8!XoLksZ&k!PPul(p;}dV+<@XXM;QJ z9~Q7p{= zq6s6-C!m8%BYeNPiv~D60T=ZR~ZhxSR9&Zi_?N+#GSSBct-sg{ae2m4S83JbnmVxGBcs>RX4w}Q6K+3%)&7i zlDPeOHM&e{!Fa19*eEF>aJiaAkME79j>;Wab6*~2OS}=TDapde!DDdu)e|sEB?+JF zzehKB5rof)h0jkelIu?4klr&5>WekRZ-$#uCY+65n;(MN)|)s>wi%4oYel62 z6LGHF7BDkd3MY5BVR_#$3EJMuA+@HGPpXA@_US&v>>#>gX+7?f+z;&UTI5G&Q&_Do z*g7SW)X$TorgMx1QL%1VYV$_8?E58pWL^mFi~2xyMjXRumG)>bIf%ra|AmSEddM3y zS-2{nV{U(KhfjOt#l@37s2MLCmvvmGyLj!QkB8dH*Iq@z{#zoDu?mN4i!X~Rt?ol` z5$BGmI7D_F)&R~KLVn#egr)DNi%X0n;Y^!~_?x>idheVIU1TkeDp-O|en{QZ-l12B z4^+E$5!L%mL^jX{TR2C?;A3g9Pt6CSe3o_ajN&I+?WbQtK9Kl^lkhFT4lM=`fXUZ- zTr_4TTFp^`ev>TFH?zY1?~93#*DJc!Xo5J?SC-ht8{qCF2I?K_sEtxK?r{jjh=eyN zrywtQ+~Gkx&l{og8)?|Ws0)Tmj)Q^k30f0)3uB~GFn+zUAf$CBQT&+>;(-Tr^0^>9 zX+9d*s&2ZB+n+w}J{w)a;z3y7NatxNka02-!SmV&GHtf3_-RiyL|m!{wU?!2u9rE? zaLRxYJKcg9yVYtLS4pTF=1baORet()nOKRT`cROG7yu6((aDU6O zXwT!uv8{OO*d!>|iv-2A99X9xOLT6H!^!jYaKWO3bdh-q3c|g0J48qcUvU^_}jK4KNWp_r>)oW)0??W7{HUCILM(!jb!*Zz8 zxea!OWuoRCDd>{p!XA}lsP{-#{AOSjvDsQri(G0+dhA(}vw079X5R+&`TDTQ{VZ?R zzLo3*PXf#D7ofaL968xzk9_N3$UVt@*UMf4Gu}ZY*3*Un^9&Ngdktp|90LtCRU#+Afi@Tod@^vK`j6*V57e zYwUe9M853Uhbz+pAu9S1c{QXZ4qbW~(r+F{Ew0Nee|iJCX}1X{8!CYMs}`y*Jr@hk zxuNRF3V1wr867v21*1J z+680FKG`;&al`%V-owXFrdVQ=K-zvOQ;C*h$a?e$YMfuPaYMFLf8$ono}$EL+BVSx z8{_yxYa`%IVGd8VX%qII`%6mwD~W{dW{f(gNN04P!y&hNvTvauI6oak+lB@jzw`vN zeV!O|oOhzVxhF0*z5$$Ol!^?!$ah{fG>f-m_oeB`xW6KH!v|n6ZX*_3|0NRNv#F|c z9JOBY5amrb<5C(4$LE*;>68-u>F2=$Q(yd5AVjB~W;pKaZA_JlWn}uF@oOHu#LSXi z{AF$npsD(+=;%RxsCdTtJ4BoL>8DbO+Yfgf+rtCH#UDXtM5Jg|o+%8zPNPxlFCc4e zPW)UNiAshno>7y<50%rn&Q62y@9=9HaOodi-BJzpe)iU!b9PlF7dJ zU8r%)1oHlfvGnm7+}ACS)85^qFLxB8;$|7~+UN!n6_`qOZ>7Omt5%vbEdWgvJZ=451o2>cSz7phC@Mq%dkVTZ0+n{)0 zF0Ar7M%y$SMa~JGWYWhZ;xtMf{nR$0T%Af0``mel{zg^u8U{!20sK2Mgyb^jIP z%lXQv`5~L^EYuVeY{ zv_gEE9@cmO$9d|YHh(1j=qg2RW=z84&$g4Snha1;cE>=wR(AiLyY#VQ995eq3ntz* z#6$Wz32Pro+Gd>+eIC6JMk=0!WvyBi0)invheu6gEU;IWV>sXOgN(Aj=($x3de!HW zTT=r;p>;N{+p7cHrRlNoQM38#oWxtyZ{YOa2e=kRkC~JJkp>&i}JXCGfMt79Tz_Zj_f-M zCi?qf3<}7#DfUo(I~w!-fAJ5P@1eGhJ#hbxDs)#SU}er!5NwP^w(2oSQBy^qMvmJt zJ(_+BpM_)TXmL`~RuC_|1ATQW0$t%1QRnt)cqQEqhC&N*!xCvhwz~y<^LR)dr=-Hw znd9-Z%dAITB^x*GMQTP{9K?ub`{;Of+z_gKH^ScyW3mZVKNo$~aPq);mXH zk-`$BzOAUA{+FNsO&|37G=M(3M>cyeg~|7lF?N3jNqrbZtfHbx@s>OMNd+Zjv&mm_ z_updJAU_$>TOv``u>gY3juO01z6+tj(?k#GO0H)tK}h3Ty2xFGmQveMbCx>o`j`W& zB_@)9gQX&`T~=@=xeD}{XE}0&C6;3h>?9oGRGDy_DhUT(n+CA|q3_9e2a_uldZsr19 z-d!xY4Tz5&Y$bZAXZZ~j!zRKr&b2cUy3t40l}v(6Saj?lo?B*&E6sSMQ|}nIJ*p(@ z3~!4@jdH^`zZ?WMU$StXN+65n$X!^1wp zgSIN-`mG9zx+Y^z`AKR%Qw8;RjHjwS(YWBm8}vD73=b3YaL!YI(dL19Sh8}aSS7ER zpRJ{f-*QI?Y!##gb>TwnUu!{coUj*8{Fn}AQ_th+8abMNae#i%6yW}YN#xw&S#+qO z13C;AW35CE;+_D0+5{muJvRf7#uPFq{01Htr$WZ4u{bhN9uJhpLiKxddWyThMknjy zhdMcN^4A?;UT%V|#qIQ?b`tWhNs0Ay%kcXRWtzQ^&RrM|`H!+epg98`4b}4lzUhlS9nPTL z5`At)8_8^(Tu-e0XJPQ<KdU7;<;WAAI;a6^Z)0F|_c}~DX9C}(g#_cdPM2I6`ENrY46HZ7-q3hZ zn&U}+b~exmJw^OaV+kawH)CTxQGf-8*)%BoNP)X952{_9-0`31L|S;#-j@Re`mn0j3jKFz=MBUpZS*szE~^1&dPs3 zgPZplikzjk;<*bQFy>SW4P1JO)W5Z*Ek+5nDM(-Nd3Po~-OvKRqj`96@=3B+CY0JI zr_!Sjro%SV@vv3(rKrc^A^uVGATIskaMo)p+T;|nIpd=7-SAg(;9HSMEyfiEg<}N) zYC0gfKw1n%32><57&&t^3tx?Of-PhVqz7CSxm*c?W{abc>M$LfnoY$|_gv(k?H0Eq}x;q%ets2)m%Za-^Ef8n_LUt7LHWQs}glG|L~cPyZCBr zD81XW4*Qp0h4%N{|Nn|~&S}($ljoeImrux`;<{EaQ@YOkxJ^$KmS~C7OOj!)?{1E3 zYs?yrPZsqEuk%t8Rf*E^Y`DDhG-jtQBytng;r!OKWOJW=dDyc*wsti`#K z4%B$Z3$iw$oIF_Li?8O*5c>@d(4G89I30SK7P&`|sYRXSy)wbd;Po)=U>mu==Qi2) zbs8=D-p_esDv-A^ovgj14IkEh5OsA4VTqO=Y@YWAg;lpeNn#|1>!st5)iIE~r-AMo z3ZwVvQYx#aDu};UOe2hgh>Gc;sJhh%{5IXA7W-Ec|D_YK*x()i*yriEL0~GDsJ_S# z>p4Tn6;IK$hH>Ji%Mny*_yoUd}RlIUU397V!u&w+XbAnBoYtDr$K6# zC;Cksi_Yik(D{ZM3{8we4}L8P&?IGgJayeY`xtGo}G|}%7;QcV?b&3cn^$a#-YTr5_s19MdVfJN!oHg!4}h4 zIQYh!#QoY0yO(SxKO)uOXnX<9| z9HGMp_dvl+LJ9{L;lr07$)~|N*wx-cV@LlW|An4J%{({Qt6@UzCS5^2rvccPQiQ{Q ze_`C8Lh!Q)qSD-~^-}&L(Bha$0WpfOsZgFg;Lh$hW?i(vYO-JoRVEn{Zs@Mf&Bi0I zGsjjHfgqB_5`%?!Sj7_?bX3J^E0hGmuX{x&A93#zb|U`mpA1Wv*Fap}8aT4O5EZnR zlh1*OHzP`6+r{Vna?2HHyRCy(c6}kPD;n`hl+Q2*559wE;boA17eT+L0v&tL5_!E#U?A%U^@E>qgZV6cIyDP5 zM?J%mdy`;%^E>Lu%SB`J6f94X1QTXGG;X8(?@ihu82yPV1xygPO{=A`T=v|wt^<3R z<0z5)oax?LM)e;a$DoDNKuT0d)zr?Cm5zRB5VL@vm+g(#g+3_tD+$W-#)`9p{=@43 zI7ej^pW1zDC%>is(N=}g@JoIv^gjDRCZFjixpOvS#oA=-yS|IOc&UL^YHEVhKSCkb z>@vLlJQv(uFO%Fu*WqeV7+$HF8r$0X8KiK&se*Isc#>eCAK0xo&r8)>Thh+WZ7<<|T=KuU8;( zna{|cyd!jxjStQ5aHbbN#&hl|U3ARsqA@HHA0N?+PyI*Zrg#alM8SJ{R(=|sx)lNw zKE;4nz6k`M4iI^^NsGVNRuW4Eju8+Tk5#EX{A)8V!%%r6xTsweSr2amk0rW9?$T}Q zdg(vx-og^ShF_%0I)xN0izQ1Qv=GVPQ)rF163pu4<^gr>H23in9Oah{svFwLuc9-M zUsM8LgRS{jB@7{7MnQbWDIRUEy`*ouV_^j6-jRaiv?^o+)X$oXZe%3hidTef5t%rr zZV^?L31gfE>)_0sAY!!YDy+>BQs(byIC|tMtJ-uYGzZm$*Slal#RQCv(0NHSGgWh~Nx7#yc1hov$L z1Re%Ef$n{R51)<@6wYa-<=ShZrKuipjTKHa+X?0WTKOS1QV{g6fSVzEq0G^%SU7%| zERoQKXGiT}_g!W2ZEGo5<*Y8=|6@FiX;mVE@aNEWPzwIZNKo618}xJMMQ}@G=%p?F z{EI6C_%2>L)Ns5p$?%Whuif~8s!8$i&2Sdd56N)CRFY1bn}&(sj)Cdv?;LnXM!fy< zDp(v>MzVgUQS}|t7*_F|zGq@+7{@caBJBv#yQARaN;MRUBt)ILN9pbLPefwfC|s@O z3a!arBGcEaF|p+UsVd_5R43O_?LKFepS}tAT{;2*nO}~AR>vt)YyHjkVHQL9SGT~eQ(uU@TrYWe`Uou!l@grI8zH_P-%et7ou@Cf z4p8}RQw0s1O7THNlj!MRbC8~`BFvn>J2HUHOfA=f`#q_`I zjIqXGT=ks^WA$aezHtZ1I;0MdwT|+=s+vGS zdM2I|-G#RXel&Q)EDT+=jT*hMAejSZsJ|=%@_OEp5#>Me%j3y7w5XZvT|P(`-C2np z?;_B(wuAV3N~8RaEPA=(N^@k z&oR{$w^HVZE*`I#C35yu25U=qv9cML^XB^T6@lYjeN97on*?nBw*cORCqhYl6gFQp z0RGLB>ql&hPMkAeykY7INh!$Sl^0Z@=<6%=a3NyXwMGd1o8P9J)?E{PsfK=K1s$k>lT( za|oPbL!gv*2sHh@AW=>OgP$qzPu*KUB;On3-@Y&`e|(JoH|j3Un7#yFt4D*T`wp<% zdkv3oI7M=L%wVm>b+n!{63bhH@RGF)Y4OT|E#QDN7pa4%!z)t%VmFNuYGT{^-|!>W z3H(mWalA8s)W$VTp`E0lhBt{dKRQT?)T9Lpt5m^yQ#RRK_MTL``{Lmm85$hJ!_a?@ zaH@SAwW&IZx)}?wdh>T|;vJ)nmVTh?ssbm;2hqmVsaU;&yN`ZTAuC6g;F$0k;&p~q zAeT#s_UtQs*V25FyIP%pdTJ+3kFJ43E$y&n?^b%=&mE3)_ZX4!S4bTmqQ#Gw;Gr%Z z!H+xtfk)L@h+|KK({>l+=UBjD?H8hj#y?>=I}x}24uj(RH_7XmFuc4shsw;!#-EuF z=|gXSxV2>z@jX}$;udRuSHL`6`#T5=tMc$&TpQJl%|g?ieRQjuh}QMZ$9YyK!1?}D z+A#MkIr&HtKJM8z1}#sRqy1Dp)c8FLr`7+a3%}2U6)~%^ z=EP%ulAkTSN`Wax_@D|DZVYG^o#O$By1aF#AtYoD#|H z`v~sgyixLa@%?GKGbW2#jf;lU4*8h=rv`U?F#@}CXL^~-Q|1Jo!p>Lwu}yU!3MWY6 zn)peWlqVx_F1dk!FHv}0GKF6HHHIqnje-65&S6|!xoG+4GjMo8Dj7(R#8ub2$$gC$ z(qcG)?E5el?^qfOmToyq>@SQ1>5kb%Cub9s|LGJt963*Og6m;YR|v{&x4>$;g`&eZ z8r7x%tjQga>KF0r@4kmsZ@HZ3MFCmpki$6y^zdQxaWqSv2rpdb!4f4N&YqwMTQ6J^ zrR}RPYxITc?d9&6-I+K?h1ps-TGg=d^&o(KB;?eLY1s+$mlogsmtOQq$W^6E%P}q z@P#nYudIRhnwOyVRuGnkR+C1{`8cY=9rcP&i4Hh+f!y>jP#v-pN=x7X82CZ-O z(TNOE_xdSt+wcq>`Hu6Bca0O@8r^{{m;REcSV_Nc_oUCcthLQ#F<#$)NthpF&2}_L z;_^d|4YA#Vd$_1XSv+rW zlvu+2HMygC9=4aJ5ZnCQn5#U6S?H(?wLTr>$e%1a{g4IZt*E1cK3mD2Tbm#_UlO%H z6o95DpA1K;lK-lu;ZN3n{vio>a=mu}-|M9;U42eb5TafLqFNvLomY(m;|}5ILkmd6 zvp|eB{Yn2QE{07NpW&wRGuZil0Hu=saAK_no4WELtcZR>`072_G)c%R{CP?BHqIn> zcypmF^)BgBctp-k*@h1M6fFFbh-!IPKu_?4=?+pN@4xM4pSipQ<;pvB&YqjB%wZXk zNF*tkZY9D4%L^d*L6PX&@M*GU#09kax(;JkE71Gaw=ltQ1OEQiM3=3zBZKZWMB{G^ z7~G1jt@>vS&&pM}`-L5U`rlr9b(10g`QQ(n(fblfe-9}r=_lJH)3|)p5Ildk;g;_a zc-!cUNa7&()|+Jn>D!K>>!nLwwuSdebPS<&a) zd=gw6PU@0->9t2=1>IjqgMsd0)Lk=H;P2*uFUu~#0;%Wt$VLJ0TH2w<=;y@o*>+4= zE+x2N6ADVtk3gC{$H&UDr!vQup+vhgk{9+gZFnzqmoMO$Ta}n@ah5b^oG1T|-6o5Y zn#k6W)pX&9cX(GU4?OoeBJE}kTNHE25}Q6O`*oE@jl6_cu5KVx$^ya6GZN=1da%yE zia5#27;EQqU85~RoNEK1^Ygs0^?MVY_p=?&y7bZZJ$qoyFt9O?JDQC92h%VB7rT`1a%> z{Oo3gkJA-UnB>7m_U?efwDE!v-3b`F<^j=Oe~G+o@TI}^qwr#OH#OGt!Hox&!y~10 z5M-!JJD5Q9XlTJP?J_uinkz{5it+32wJ2F#faFmgJZXIctF~^VHam7v`-OZ6N#=H2 zs;-H|jpM0@Lk(Q-iNcCc^O+x7!HoYtH(cIw0;{Gz7rB1xWnxC%pwf=^0$CGD($HCq zre)Ez&gBEn`lgHC=0@V4LNmJL>n3z%8sWvR2;4fn0MeGo2=?%vaqsI%WaO_92sNkl z&t`RKundMd4!`*)PF5Bd>$R*BQZxX4~w(7&R4B2XfE#tb)6ez$=D%gcFjpRojngn zD}AI>edXx$8=+hl6ayQ+Yhd(^R#D!SQ&69)3k?p>_~SAg$$XB`Q(JTB4#>4c33y!<~sgv3#q#c;&;V zbe&={wv;+S*SiRMs=N`4FJ$7g_3DCU^@|`a(?FD%S&0d9MudDf#piqK=|e8Zl4Vjw z7HztMH^JB8&eONV@wFVLT=v2D-hV_>yHx}iFZqcwoi@Q0iChdRjDRK82jF3~5{f@q z5Gj`u*mbJ}#{L}(^FLam)7fM2Oq`6P7Pj*fm;Zp`(!V&>bszaF;RrniEWFmw<`|WM z;CMNcgvmYT8&%5T2BkS9O!%AsKJ6%EoiBr)qf61Y!V>l_c#muJ!=TCk5zLWFr8{?f zQqf2+cslVIR^H^gDmIsxz#%O`_~$awvZq?$Cz}EyA16ww6Tdg@EPtBWHPN@J6}WQw zW(fba2bHmf+Ad7Tu_eoiYKsx)epkZbL6NBRKoWhTbc4Q~b06N~E&T2{MzChaZ?Z}$ zk}jHF$`6bi4Nuw#WE}0~-&XPBJB!|MyPh#1Sf4{Gc6<`;{$q<`i#GJ{v4eA=qs4)_{F9bmKl!VroFH#>XIhLi7&n zr?NTm^kx1hlHIZ!8jR1;-ukzAud19B?i~*%DpI06Z(sOpb_Ttj{a{CyD!%@f$q$XF z6wQpgPDiS41p8;#DTEf_2xTp7^qh%3K3@2vd<1TpT*w#rM}TTcHb~?diM?eeV$`Y# z*!?C3jx~6p`9dz||Ko;*wjOZgLJV41hv6oUsk-C(Q7o?>AX{=LL8QG89QYgym9bGo z`r$(OVYd)FzGs17%ptt9-2+0Vm=Pe7%F6P%cr$h~RBWRBGn>M5}ks`o5~ zY}0CO&rJDG< zya$&?YIF0N50EivMJsDeNYA?>V5dfrPFqQw|E!znu3P|%Z3and^C8-~ry37`jinP) zM&Od8Zg5vM4xfLKN1LWsa69xgnhEmgLVYK&i>t?k?~-7+=001$|18y-9ZU1e^HKWG z4gBD}oeKK%@SOiByx?dC#V1n8!&EDJ+K%&06b=%Lyj{4yPzfgVyK?jI2E6N*!e4z& zNOrY9C+i;}Nq3SJBzE~jUUxkH{q&7w9NkaTzWU<=i%DD;jO(qemBHnTO9Vsb&WQHE zBKY4pj-~t70yKsufvt8m4H5F_16fDN>FI?p@7Cakbu7KdWt$C2#b{S5NplwK3)&p^ z(*rXj!9p??(yqo*jn#(mrJu_|Z_S0#Y9(x&T{P~M(84bz$MIXtO)?8R`Odqxp=(Gx zZ5z0V>*vR&`fPj@qUF3B-U1k_j2I} znqE>)WZ(W4h1|F2H+ek-t+xo%EXPBe=`DJ4!gW|u8;*<4h(I=R8O}2*$G>m6eHzY9 z8TsKYeoj4vrh8lZAz zRw9YFRV09HC++|Bfkc5CTJ#@8-DxfSfBCO)5BIJ2hr2+&ZL~kC8<-Vbo*qZZ=& z_6zO{OeYF83sI=wN_Tln3Kq6K=H7;uzY~HaDzjxFd=gR)8ueR`0uL@bEIGw?1Ip`T7TtZ%wmNAcB<&3&_Zh~D;}G~ z=8QkWuydC)g-j9?6{W)zzh6~5Uu_l}u~3=)ygHUW+OwDOyWYq1nitM|?P=wGxS}C! z;+$`<6Fpf+j#ZMxIj|IbXR=8*7O}OPQiKw<9YT{sCd?S4c%E8PJ#VF)3+vT7#2d5K zLYVMqBYXbzQD){Bj=QF~l{w#O!;F5Y!CrPWVqqDu>kdV*n~x_l&KJjXC5~ujxA|t_ zq@~ArA(NI0Go~J`b?Ms7E|>9U`m_dZpRMFG({H@69m+k=+ZklYz8mZp?i^EFJ4w}? zefMNFv)VJa_Q2w3cKes346jm!5tkle>TY>5a(k3mnWM_A`Hwa1{ADrBXpODxXrVP5 z#(A7iy^mvLWsb6+$LZEUT`YS`aST)1X2N!UFBdAk)n!eF5_tv<^V#E&$lK1bH@B51 zvwGICjF)N{Z+ zmc#3r&IJ>gt+zG^<0sq|o~cRTDQu4wDn3nRUp_p_SdR%54sJfmT=k4*pY01`auqy< zk(|5trH=vQFSCR#+g-(*;DSQ$xPxqp)j{UvL>K1Knn-4DoE!UJ&j?ocvn$g_TzO>@ zM}&gxN48al$+nxW)>i8$Ckk(>&g8YHJMj$WPPSFstj1csJ!LDecGLEwt1j@0iaH zj$O&tR5{r8nXF~*-&Ept6|G=AThxSsd*`s%^nHcXt{!1cVJ-8#-k^5(4l!@0vK&v{ zH;?ycOq=koY6h?6%?j2A;uzboF5!88B6C`z$u|4!WG2JHl07OkspISOn`;|lh2 zZ2(Vi;3==HJdSt$?n-9PdwSXft8M?4PZxg%5H&c$8-FR<~wW!_!dPJx&tBsQxv!;e}C5 zeu6n0*A&nD^TC1L_a}*YG7{t4@X${7qrMu)N{M2B3pAK5`+p0= z&J&(&tsLuCVp(g)d5M?ZE93P!CfBA`o7hTgc?!Mjm+`D0s|uUhLrlP^y=?8uMxpep z4J=P$Gkd<_AaiK74tvC41!H?{H}lteH`@Rc*-tO_v1g+FnTZxQ>+T@?JUDSp1p-t?cc`SY|>+Kz?G5B zna9pI)EBN^_LHZuj$zJ)CNoMM-fZSE8OA(&KYRI=H=8!Xn(;R%6b`O)WM1th!soY# zYgN2_c#lt~)U?u5fY3KK9&^!wj@<6PnN1$=Hvsu%48>p*HzzNo{zq zwD6l=w{W&)8Sh+3tx!JEkC!+$sa9@^o-j-G6wf30U+tC5gX}5T#fS}LSf6KWnZKvf znewz4wne~W+%G1xlYh-(40<03=h!?DPCB8=z79=ig36NEB{fHQ0kw(j9j{ zM|u%&oyRuTS#1~7Y#hNhOfqDeQ|7Us?nW??;gRgQ*kPe%z+QGwqyyW#@+$A&6w0O5 zEw+m+jtZlUeVP76_U!hRwyamhux;|sUZMWC1I+sOw|O&fPGt%wZe^FY9b$SX9E`%9=;L3qL~_BC&vX0rD0y_lY!NG3vSHCyLj%##)k@#fyrX0+S=*jIjh zrhh{$6XF}i#I&~Zrb$Gx1Hu)|6P(6=9XwUtwyT6E{ZEqhn>oTZPi8l>kiVNPU!ct_ z-#?jMVz0miTT8GZt4)O)hA#-0Of+KWPyCNDjj|SADF9}Kpzf{*V**kVH zZ)Zg@@oWq`EjWyMpg)7Twf27$op(Hy-vh^!y+>B|9vPW&&$-XJB%@Mk(G(e_sjVq{ zhe)W<(xOs)x#!&HTv8O3c3LzkDV0iz`aQot@5}3T-RF5dXFRX-{=7dau;%a#c(`O8 zxjpy-w08OvAI@QAWAz)$+9ksolXKwhH9v$G&dNa?5t$W8w0J0^qZ&3OFn4k|n#HVeaH> z*t=&-gL+gDkf~E4iPBO;?QR;$%1?)fC%)mu;bgc+%?th+*-NOQBHU3LNaADj;kJGG zM7;JHh{#<+e7~)Q*5z*G;^IV7bSH<5Umgs*<$_4_W+PCW!1<6*+u&zzA=tamAMTvh z41887;FOysxWFX@e)m%clhk$)_Q(cUFtZxej1PfRLc?I(<`8)GSp|_$t{@*wmB{lm zVUT~M6neA!d*qtQusxS_!Cp*IXT(76c zxy{5~971;Ra!HBSuxGQ`K48*t&U5dLW5D*C5j?ZZo~VT?Ld|#?+%30(h<>V)6qS{* z&R-QC__&+Iugru|+&*G?VjJLj`$MUmStMQ~6I%b4BJcKl64O1(;4<$hXbe_^TZWzh zDK6%%)Y?6TEVP?b+>_4#>?tPI;4qaOWYdjx=zw7eg@qJ>QtVxOrqHI3%?VL|(iI3YvDqn6~Al&b`eu$}NeQbUqV?uAK~gj3dCVnYVb}%VTlX zxhSDV)Gy$5=@Xa~6^d0JCwlswuHto;Oy*Sw?jurN8=zudv#`~_05*?KCU$q`!{U8% z;2Ky6wNN=;AQu7^6Xl805hG8P#2+AtN|1P^iTLjccaq-XjK9_KfQ{r&aM!90D7Z|8 zAKinAU$`zvS*SufES~cweJlownq$Z@pL87mv<=YYW8LlTBdn0z>kM+m{V~Vucw+!C*)90uth> z5BHDT47S{^!WKzSK-uIHJW=&0nA*lY$Ltc)Q==&~?@cBbw&%f%p;_c=;|K74?g&07 zN`#lyZTBG2E~!iW`~MEu(pz{A&P0j~p#;Q7PZ99sDqoX_^-nZ~7qhxb0?oES@( zbobz-3V|%#6BtaW@=W+_g9Ol(b%oa-%;N5UuQ0K%8b{_o!J5h0utpf4xd4JI@5caUbUqow2IW+vg5WI_5#>9X?EZ>LN@&6b%!jS3$3u9HKnGkmoYE zo@|`033URdOa*n4|a}FHZ@EmJ*xDxq$Idb)pw3_nt-8 zBX8*YCI}BYT>;HnLqN#?f`hsOajfSrthH%5*)rc3#+7w}+shr{E^cg0&fN)D&$1wG z*a!}DJ<{^eGs#0!6Il1uolN^Ro_LL00KYXaCfBMHg@cn#;i$wsG7i*%nTyZkiEYbp zvfOVRS^f^FggOJ$Ksl1R%@ES?L*Vb6v2a<8H2G0p3_f}2!2Jq_@ZDDjlBu8pmHG^! zuY-|sr~Pd z(Coq#yk|xQxcAnpA$KGi{`s3plG6kvrEV(eiAf@rXeDg=vK&6$8xJ+OHMdbR54`GFT$1Q1@M8%f!c0Cx^&lQ)JD#9cChtaJ>6{WH^vd6x{y+!9XWQ{MuO*ON(r z)mj{os6j3RRWf_SSa@tu6SCh_;N=q;uzT-*vTNr=Qc=B}7><)BlLWT#Z+0daJfsXK zuZ?Wzysb=3y>-FT78$(MZz6H#-q($qJRo(S4BzSPBy+~^f#0+A;o}!eVVRQ;7&YL| zu5^9ym;eSJuQq|f*1_c2(gv(I-+~+t3MARv{Nb6_U}#5|!kF7B@a2{?vN?D=acW5; zM-#)~^Mk8}?c5m5z9k^Wf5+nCN#%I{qELvmED9Cuml#7nn41QP8jz{`IK*R=M5zZK1Z8FvSqw(^6c zYHLWN{d2sYaUe6alVSX>J>>X~5YqExGue7H6?$( z=-KI)4mMmE#$T^T;Z{FFf~D8P4=jhs)fQ$%l%oxFFS@c(kqsI#&B%65TX^i0Z#Fl2*13Y1_wT1l=dnSt(=g+oT>~%XSg%DG2i8BwrvS|z;i>c z?Oja$5-oJ)XBIog-bk=lMZ{{zuVG`=LQ#CIH|uwGD_`}K7Dr$-vc=zBP=Qh-E7&=i zDrQO}C*S8(XY3c2d8*I0o*gf$T|SF*W$GjGZVN%5+EwP4#agbjPz@!|NMY`N8Bd4$ z0@#h~)x}f0ZZXwPUdVn|0kuxBLP~M^SX6=&%(SM;g+0acX=ydi@WHsY3)!Az#kp52*8MR%Z;npMU;R}f@j@K1;&HBbX z6wIZS$0yT|FQ&5H|0DzH&Z8l71%ZV=L3>V@DloQ;e#AJ8BB|C7rOOz2Tgir!8-3e!^9@YqKs;cvcBq|K+db8 z=9@&nNzS0Hg*LQB$%wgDt;0sE7N9xP-5Be9j-8E-rJ7S$v%Fd3__69|xSqFhXl_n3 z(%bqRz2;qEN)iL;)~PaRb3+O1JZgw|7rwG*vxXSqrNd0bgOkjk!}bxUGP9%aZRh-Hd+WydUL-`;h!G6=c1H!&MI#(R0h=(YJ#pbXom$ddjz!?YyXp z;>NsUAD4b&KOeWIn}q$0e)9@+@^Jnhwc>h4cdTF%6^}C6U-PJyT@vz1&19^k z^g=%K3GJ5aURpF33Z~Q zkGuGrZh6D2pYv$!#7eYqOx5`>{PFDv87rkP zsJ-wg`{~y_22M+$qtowGkAPQ&I zPHPV~FzeowGDT-sQFqII=yBb8^fAmq{P=`CopgQ+(@}1UI{$Lck6T)7#=9JP?UMps zZ(l+WwjG5xiaVI!lWZ7^@)z`r+*bz9@kQ>8JzAEbFSrp_Ae?Z)l@?zzK$h;g9G~-? zvA>#z7VBxCiw-&Ti|ZNw!86A=&ZUHXE^1)s#2cVc1qO*7KCs_vTbc2p6X>|ZiPZX% zCViWFkd^;aKySC-VF%PkMK>3pp}Rw~+0xiT6u(B6+44$&3MVP?2mLc?CmxT)3ylTq zbDF6AV@-j!XD(fHK)@ck9mO-X=CUd_!8Brz6qBV`#?Iws($VdU z&?eC%##XWuaS0o=_tH+9>TE2Qh}=kXXG|4HoJnFTHD)2pee!~F?JOhiG7w*1v7323 zu#ZVNuS%6pG@xZRt<>fGFz=J*=(V?>J$SF1sLBK_I=>LF&w zcJ6wei=~l6NgEm0 zxk=2Cu3D;Rt|Jyp`qTBcK~$-~mW}kbrK2v-(H@s?s5CW$Hk$RZ%s?I@{2=B~o~2;1 zf)lbc>ZKndHPFdPiO9$7sc7t@06Jw<0u9Icq5U5=)49vy=x6tObkF)GU3k)hmV6IH zFU~nKt)c?DZc!|IRO=k7&}-&v#O;M`riSR#oSn2FeI5OKxu30iB};dnu4by_Y|;2* zzlr*~(ol?VOJ}fZj?-Y8_dLkm5_&%AY<}Pk*8cWh znw0+ol}pY;-sEzGZ+8h-5*V>-O+0&C>x zi8A$^5%277Cb-9rJvT9sZ5o_|E|woAg^m)b< zM3sh_#M4XZ?zQ=hvt||R-Q$YZwOrx9b}z%DVPmMT`F?trxxv1Qc+KCnBZaNGyB8f= zREUn2=_0|ra(ZH;AN|;Uo66NXq2uc+;Nj`3QOWF2{4OLfu&Iw>25ANgdA^?}alS93 zuZeW8%5^RWZbkR}cY;lwv4@5k`=eFK_H=QiuJ}JGUBn;B6SY=kpgt*C#DplY)l<_% zlDD3qiqZ!3ZD=mrv?GwcKVO2j9a~EsUZ14N;S%EOWAc#l3R5(W%j{F##k6M`6!U!Q9xTA{br}?zNyNJC8+8KQQINGMhvEgnX(ENHK zYxrn64T|buZVMh#wR#!U+U`pm=PDyPxhw3(!ANG=n<;2eqJlBC9Yoib?xYFda~L`Q z9E9G?XD0~UL>qVApb|0}=&ecx9jKBWz`Ky$4t4r-E{JET*fK zpE2`-kyO+<*8J^v(Q;~t?kpN) z30s2hAAZcPK9rBxbq|^RMF&_zKMSfH+{^io$Iw+t<#cevQC4|X2K{Gjgr=%br32Z6 z{LYXq^xZ8Rx^~ekw6XsUn%KT7G@$`)pwLVahhvpYU+M?dd|v(>V$)T^z8dTMGTpKpgmATXKk zyjjfJPMko8_jR$i4;Y~T?k1rq&-pa)XA1Jz-pK9|B`{7?)mg32N7)#=lk9^*F+SvS znI>$WO?x`duuRk!e%zgO?ri-X$KmFo8B6@A)FDYRb|a`@?-Y7_#SY|Z`;S@W%bkr| zucajB2*38kW47?|H70-kRyJ$SZ`wR#7%AK-U}wF3$n*p>Q|oG+0E?my4RxJ zzN_fE3VrHg#6!~yGEhwAJT!359DRF_sN4Qd=G-zW%JMKoS6uW^$w5Q0^YbHc_|G)j z61anQ1f;QcBev*~u@OD+Vwkypq6od8e;S&)q*2F5xlH9VCu*?#7pvfNjrkKcpHUm% z%wE_#1(i=L6Rlmu809_rF}<=dI}9*HO{@w7+_ z8q?gbap?5+Y&0+UHFMv@7R@`DOry`Gu~T#M(3j|3dTW`2SZ)tbu&X$o&UBfCZmrwH z`eSK+)&(oE<|Tc!VaZg+^Ti4Den}vmaONZ`%|bZ579y8%d}g{;0`l6ji`f|zg2sFs zWiw-Ys8Y~5dTySQKuWlqnxC-YKhQN~4@cH8KbNKfO;H&Q_@qp0-;F~*4)m~(=A2*! z`){K2r&8z^w*t|=?;F@K*L|!r15o9Obo5zQR_rxWLZ8jsPcQl|XOA>&pi!a@R*JgP zANLl~g*tv44QKLF4{*oJ~O|6$gyWW>tTU$f9u^~qf^t@LR7b3RN1j$l7|;h7m!ap}9!|ZWRU1o0x?QRiT;hGvUBTrNKY_^&ezz_ zUOBUeX@nQ(68%n*hM6Zn;@TlpGbxbX>f*A#cNeg~O^%_Bkws`(+)5hyd^+7(Iz(@q zCNckMNTJ5-a_G!cGc@i(JeBSI&HM_~p;l{I*0gpJ6Q{RF)b4wO-Vd!q^M(T07<7-x zaZ*An$W|IQa+J0OyRym6XPH%p$D=vDD#*a+1si&A6K%Bpz)tv{!JONAh57YPjwTH| zpiSI9!)!@B)d(!2ff9+R@#{%s6o#mu?QGiC>_z*t9q8Rx`S=!5X?&~4<++MMdKIhVs8@Cue^)Tddu#;XfX`ptU z0rYsJ9&!(Hq?=<15lL@C8buMPZ=akgOEBIW-v=u#-W`WtaK;qv7l{uFqi;)vd4+@AJz?AXXc7hz-T| zoLfTHv>sVoiOHs-6O8pYQ@UBzmUBCu6b&xSXEY|5vbPKL7?-89k%WT~efYDU?&7US zpUbW=+uHbOp3EV1wtJVzp!X88@fI-YQ=T*biY-vs{bE#iV8+oyw$i&Xg;gHe4Qw9CX3>86VaxS z&ur>~G?b|Ro4CH5N27xti0+MKq7Qd+5b@Fwe?RLg{*t?qHm;MQZ$V#-Zr6Auv%mulW_r{AvOERJ>8@zD#VAdAP)<9x=Fq!Wdzl>{4d~R4JWiJEf!dXI z(bFSq=>g=&&idp|U7H$E;n<_JnA}A55phgtec-OavtryRY+4!EYVbV&Z(7n zm2;#V5dEE=g_D7s!Xm0KlCfuQgDLx}Z<=H}Z`?RO@Am^a&i5z8mYkBC) z^>AvQ=)pI+m4@=SCemy8MVU3keB6P?I-%*}gyvL4K$v`qGe!ieCum{2S|#(Mf6MOI$MI#O3J4 z=l_^zvx@1?J7dvz*}sfNc?61^qk*(v6d=u>Qe@azM^%Q+gcBm=_#3H>IQ+^|1rl^XSx|x5_Lh^6GN(GT8f(fMxxR`E%df>ISn;^ zLd|_X@DJyfBG+qYxh{+%*6Ls(yF@x4tvE7~|3gk*Y{bnueq$xX&y`kEy`J|pXMUEb zK*E>-ni0q(_6xeL+{IS@ zMcd-GvD0=AG7_9)V&R&FOh8&A`&K3!Mc>M$mFpwf+-paXXP!UZF-MsR*6U|3nVh8s z3*C`=g$)z-Y9cDjvqWc=_R>2?odkAzNpxj?Kf8<@gC8zlr21dBP@AZ^@ZIeSVfi6b zQvB{YFbIHP!X`;l?qg1**6P99ofa_duOC@mmV!U$`~+DOZON6nOGwN$e`xvCl1TmC z@1f+HDHLSb;px7A@eR#cWZkBlo_~M$VXd}$@MEtnoDii7zhO0!m8(TcZk2*Co)7eP zI*c0@w&J$e+1S}Mgm-YZ8%(OUBz3B~q+h5EUyr&%wS6;)wAwg$DES}Y`wR&`WI2!* zQ4u(I_6t0Bp(I>qV@%XT&7i%GK0bHkIhH-8LTp$8{P)El?siy7^0K9gYK^t$+?olb z^Ym6r<9z`;st_bP;GZdn3wf+ka?mdM2CwoCbiv^J|V@Ss>XV`x<5I#FHp8WIphrON!lMl+9oS{WSrcEj@*#ifqXF(EYfq*py`MvLlsqb>P)Y$-tW(#YP2cq)$2%f6n|W zlyAAiQ~%e=n{(|Ic8tsw9#Ec0Hl&mTjjvAd*iR{J_i-7hY?6lc9KELVdbd!m<||gs zb%sY)OTe=ye&T6O6R_tkBUn~<3M{yh+%UdVlYHoK1y`@y!w-arXUa9AUk0zu`GXzpJcz@dUzj)l2(Rk4IX>ny=()nj4E|hG z3YGjQ)ay`M(&iO}5XOOuc17Qy)|wBWP77BDYu8q{ypCOX;_SH+va7XDM*^nC$%9MCHqZ4HDi zX5+~6(zl>Cd=RIf91A)_Odl8Up*PBZ?C|XW1LB}g#)|)tIVI=6OS0K`jW1!&4VeGy5xGjJao$OgsBeS@$TQFU};J#5Js7klD-L0@AqWl?#qz8 z|J0$f?IhR|{FY}h(hpuMm*BDvRqlU6=o@Pe`TlP3SL$Ufr{E4Zj+Y}_^UUGExnA%o zd?wj*{R21@?Mg=6my)5X@lflA9rS4L!!?7mAWifkm%A0oHYH?`zUnd-Rgbd-oY89=vWQt^$s8{ zZ5(5zZV5f=e2KSU7L;GjCr7%gvD8Wvs54*+)8JYDj!oFUO22GHot9}sr_Jm7C|hYwO4@%f0^#5L4{ls=e9c1Q%0;i~=k zXHGHD)mjDp5A+Llbfn?;d*Nh~x-lGGa1WoYNy5r^XF$~@GQi{UHBfl_ILNbk3tYVQ zadgEuu=+1I&ulKkdO_~wMEw@L+Iu#sZjpwmmjD#!lw!x#`M~^l6)?Jc65ILp)%$oh z^9E)V@S@%%<8A3m@Zk&>hpc^p7uA4^@XFx=BFq z6~WrJ$AwP!Vh&F5B$(Ovr&r9;eV^bB5M4BK@Jp^W$zX8YZ7x1o`Y7@8g z2pD)n5Bge1LwDQpu$p7;8isfzylMd)>{ryM627*0MqX8@8u9a=V5 z0`)tR#As<5sQK6lV#hV%tob6|xG+T`y&#lWy6odU4!Z$#8{goj2OqIYwh^rIOy&hw zOT)CxpZM1XfAY4u5hz7S;mn;zWchbVxVX-Qw|=Mx^affI+a6pE!qXqQm_$u6PORCgk?Yo_PA`Nw&SPHU3fFZ;kyv?5UzX~UL@iqI;0 z1K7j;#wDTZ4P$fF$*hf|_*CvsT)R{UwyR2!q)-Vs@?;XOnh*sn>Lf6KrYdRN$0OFZ zNnodsER@zA@6o6>7FvmnphBG$skx*FXH_f13&Czgq~;8_j++9{PcbDhLKeKTM|i5q zSs))QfXvD_Siz*9WAqK+)7$#cYn?gS=vNGaw+V%5`)7b2jUn*Y_$TgKyAa%Hx`nk@ zDv;?;cBH%Afs8nt6MwUr@cMySWVO~DI2s>}H?=#!N1++M{v zOBv4J(A%IqXh;^X>8tPFYEAAKy1_aPD;TM|nizguNnUhc6!y~={O^zqc{14y+E#AB z+QlD0sh;K;!Y9Bz1PC3GKekacTHi@Xkx8Q*l(qzTm>sT?N3STvm!z(YiK!uC9@p!41 zKzwfu%s-++Jn2mkKTR7>pUlVJEuLicb|+7-G>(T(>H?3;&VeCTPVAhL)1*!+`uQodr>@AFqARYuqOL9y|)t z;_d75A!halWdEKw;O%v3T+GcsRaq~=m80fF=ksh3+`xiv>tZl({bGD1E(DB`PQeFs zJn@+Ri$HYB92Qj8fzU0nV4z!%bbT2^Y^Sljvsv6aCh-@r*@?nBQVMidZ-Ido-VNoi z9fV)!Oasb&>Ow!${k$C|KY0qCG9*Stj;J0s7Ajm{2n17l@Q_0go*(ZFpC5H0Q`1Gj zq%#Bz{gEP~r$b;6p8_HcchFS&LYRqV@B!sK;mjKyV4;iy%+ysR-^)A+UttWnEWHF; z-PkIOJY`C5mI}bEaf=||>5s z?nNBI^@a2Y+q+A`>)z75N%IS^n3V-ZEY>MJ0k%GU z4>r!P5~fw20UmM-gs#@>adO8c{8zOI7+LFhzHa@=8)G#UR1a%F=@fZraHj+33Y1~g zvT4w0nH0GlwT{T9TacB*I^gs8D_~`n9n9xtf~g9L!YPw|fHc>Et2Y z`K}5>zJI{|+`HttnhQYD@ea1<%fWRLm%)EOhdu26YJ!0w33%SC9-nW?$0|pLz|*Uf zc!!Rz0Twg<0tcW+Bo;3LUn)dcHT$ebROv5#WRQVAie~UqH#g=Fo@y}rQz!iV`?PS^ zv6J9T*+tM6qYu^HKH(+LJMh8H8F(~lKVE0C3irf5FDr z#MZ9By_JqI{D(ij6FVLH+3S;2(|pPDfjp4+p&GxwIRvJR`~bsm!tpt_3w%i+La$%` zGatk;+H9`acjUn&klH;>3Y&Iu8+%&{HW$x>*>Qw^YcMs(-8LC!*Nkl zQt*aE06hX+$Whz|v_CR1VeNF%o$gCQYt7-g?nT_1xrtX|We1<^Oym`&J_AhOO`g;6 zFEAO~K#Zi}=V`hy#7>5cl)8O1<1GjE*A!&bSk<(UskQZPAW7evZvv>Yt&-+cl#B>>)mMhKE*B=l@lt_b# z9IKYBst&f;jR1eP82r~zhBq~z!QqMa&?~YMd+M2y-+y9(W|t9Jvfqx|%PYarL=`A! zq=8R0kH?FwYCz#~2k6-0M;2#TLwz%Q`2FS?e5AsO%pGS!3j2jZlX?agce)ejI%(KB z_Ys(OSj_90Z`)9FVGQ(*;E}C!6roR7C9d^1A;CRHu&Fl{SWWK7y%EWvd)zx=d2grZ zV3jsu&0g?c#yfx^jIrA!O(<3I16&JNCVI_#@q#0|q{Uc+yw~XhGqhyz>+rEKDxAwo zwj99e{(r#bidm%U7!UTmGl5K}G#QChCMW*76H8SmqSihE(z0oI`HV?WhINDL)*ht8 zX*J-T%>xm4;^3Fx!*~rl3&u}=3AQYnLON1ah*HN=a_hM@+#}ZleqC$9j`Est!CDEh z_kK0o&@K#^0@$8x|59Sys!?a@_aQL4K8NaX}d_4adS8iDX zH-8HNORF`>-dP5qNA?vk3yb0Xdm%#}jEsd}GStbhlLEMCXbjX2`v*GI=5qH*S7>{v z2rp1}g^RDJfJ;77WKVW5%*ua>MaF zO## z(`qx^+x{L`wF5G?`yY5`=SWlzD#7PJr$8&6!+6fKMKEDYD)w+1L*|wbfXwaw`4^g9=!HxU?xqJ6jv}LXS)$3O9oRgZtMt``X!`E00+Ko3kKSsYVxz0Zh}&eF z>FAyFu*Fgn*(8polk#ei!o{8R-T;fTrzW82dB!LtV=eWm+s_mxeWg`hrHs6;gupo2 zRs3p$KV9uzOAGW5B8j&)w4~?|dt6B*dU0qDvM$-fs+c&_M_wUll2Ia?_TU^dS@I^r z@=fd#O9Mes#~q~dyn;53zlYranF%)a6^ZzH`{{6)p+HUz(BtNu2k}ZOzfr9WZJLtH zmer;si8Zavnxa;^WpO8SU02N9J_eZiukWMVQ@5(mh(^(Zhy?R6={Y19iAfjNpRXmeX=>%kXsflpyTM1>6aMAxfSfni67DJ!7Pz# zt1o@K^*v+!w~3!Tm`9y*Lukw`dAiZnl|FAzr8mFIBDHD>u8)uF;IK=gcayW30L4!H$EIGDsMy9 z&-(dS&+E|CcRkGHtW(%4Q<7FpP((rHb7)z|C;p0?Zm7dVg4y)OhkZR_P9sk5rg}@q zvUptr`gQUHv-DIk>lGYCPp9tZ`z2pO-_|c;EPTuPEB%b=`4~&;9FJ&@p(=gD@ogbR?FN#C+0h^GN2tN`q7Fv!L+2&mEP$MhNmA7vdhcD znApJ;%;3Wk6jp2@+A-Y3KAU-tinb;rGxUpj8h(hMI0e!XBaT5!*v5oP6wwn`V&k?w>CRD*wu`M)Fjk)|3tmO9KH&~!(?^u98a*cvp)FA1fED9Z;2jr9W)O; zO_7S>Dq1I`>@M~dS-2&M&h_}gURZw&HMkyU{kGgfzR|PMjvXFcl^I#Pc5inWsQN3%;)(6Xrm z%>9|iM6a5A(e%liQM65CLs_scLLwzUd33&xU|1rQW-ReyPhAUa+GZ?K|M;Mho z;dJT|&J}wsk^bD5jJ!4HbWXB$GjrIy z%su?Fr?1%Dg0-mcNEBN6ED?RoeaspyEktgqCdl%`6aMHu1vC&o0ZvtvK)rfN?1G(k z^mLN4*g7hYUe{7VmAp9gDcce0bnw~GkP&*NA&@=l5kzm#xdCOf%&F%d7iPJZ1nsGO zNfTN{bkuPSW#^5jkI#0anX8}jPxc40>)OYOt%s~=Pu3|~xU5DLuAIbZZHuCMN~f5Z z@ns}u*%{>J$T@RLO*m&fWtLEdqMc z-WF#H`Q@~eV}qVGMv->|4JcDN5v{)5gR~1?vTnwDv~yAux1MOCC7(3e4VO&O+=NJa z^|3dfG5EzSOs-+mkBdbM86W0uL>Rgx&hh>(*T-@lSpZ zK^njHnF6qwDY(}~7erQ~X>UDQmuZU-Q=dp7or=dP44MddwKp{w{d!?2l zm*ISN>6DWq<>?2wF5qgW$07+uXK7OPDd}wQI|sp8uWa^|X$Cs;mg|8wEkcEb^U;Nf zjZ}HH7HTZZMW>%Sqs8k~L^gl#(0T3_G)8+lRpl!S9&XO3h23&!rFRnQ%HGY6I_I%{ z_t#Q&^YKjJptiWrv4Yjuw-$}&rl6Pvz&G1<5RL1YLERo|(8S_%{N=N*FiGNK6jQ5? znq&6S2vrR#^K2#)>vXad^YT8p?S+IBwQ1JWTz0;ah@H&aMgxv4ULH<^kz9Z&TTG{25k|CgzCN9~HlMDpo`?ExZK1!mrlWo>S$gNZ9aZ+#=UkQVs6Xdpe|IDs zwWy?^*D73}^|u^4uttiN$l@W{UGMo5)peM9!)U(VqfF%Zb`CW;;g4K`G1|XO#6-@1 zj_zI@L@xpZ`1{nF*bNWznf`cl+Oc3iDxEk?rDaNxo!u+O`DO;P^3J8&rZKc2ERl-j zQ&HQ-05lg*5(m1cvZS~cU76s+CggHWj!%xEyVJMP?!pje;5ne_M>3Fc=tOFe5Q5Y; zoTSPssm!i93%Kr21-kdu7_o)KaT=67k?!O25Ld<&ikka%P&nV4=_J?CkC;tV9B`gi zb||r2e=xm#;Xaf8#+02oJW=G+AYzVdmLmt=G-gp;E=Y)t5-^+7eF(@#qI+Pq)v+2t%PU3U`QIyaZE6=aHX6}6GeUY7l%`2_u_IY>WrJz>RR zd)VPe+O$>hl)b&Iku5TsLf0O!6d0WS!5%rqImc@z(}GRU8QIoeDo)ox#!*>J!x<|y z1{bnb7uArd$~nYt`AOC8*U~QVXse&!-WC-T0~h|X-;Mqdr>VvhS~ zQS}#&f*`9U{DqM((PH0sc+!X6qF#ewWD~!bsutVOpQpE>>@`;GhnY9g%7}YZF60Q~ zCVz?ML`tG1f!nAxx3Nz+^Nl&;G6o%6oy6Yw=ZfNVb?EvCPxqQrv^SF4#;B7nakK&b#!m z&mhxvz>0k@szB9iZ0XrMsZ8u8LsUWhnTkts%$4d?^t71s-+xpPPl>pMeo02twSSK= zD)=?)_`HA$Bvw(+XhlSf!kD7|TGYd_Q(&bXs$}bF_RT7h?V}nbEv1NV|LI{uZZ)9? zA5)l1ueLMtvpI*{#3*vWNE#X5c!YMuXX3D$TgZEz9ZlYNl!cL{EYlQ=aQ<|5Q{QD; zEu1NGeYl&>26>F@V>ep8>kk^3S4ZFRPNUj@G^%}LI<<7T!urY?v0lxM=;({t|D)(U z{JHx6IBv^~WJ^Z&C@bzcpL@tG8GVtKB9#W(OWGttiY6LLMnp0{=X~x#G8#%lQc95` z8Z@L*zt8U<_(DOMalzlP4s;}p`u-#{=dFvIV-93Xl zes~1c)fCd0{av&@vz-mO?=F~9TSEUl4WWJxgY5ZtlW3vzNhE%4kexnYhmQ4doafOn zUikqv!My8Q+=JY&Y;#)#ee&%pXE<#h)$9JuK2JD+(x!=U7hky`udt7F)|Pe@xv!Ax z;eVlvZDQHno#}M)F9G{p-;uLj8iy`;>2m+3g(Hjg84wB`koW8kaIz|HR0+yc5g`W_ zup$ku>lZ=V2H_NHN!5l5+^OJNBDL3fj#ddWQD}oGTBAL}9bBZwmV~6Bkw*(r;VK2n ze9>puHJm~XX%ED`6EL(9?4j651P3I)X%bZPK{R?fxNn{Ig(iD0ZWC86Tz5>Ky| ze|iU--n*8|5zg#hsrP7BKSSe^<5-W&8EB@3k@G=s9eU*ARs@Q#p(B+A==kgX==g* z@}nn(yj#4%%v$%F>uBreW$ZPX^+=E1OY?+2Q>)fGnh+Gt`S~E!I5V5xc%jDrzuS8j zJ00~ljd8Et&e1cmGuaR$i`psDrs%s^9#^@3fGz)}j&8^nQ>Pul+&j*Q&RVL&9#Qvj zetRpFwk@uq75bYg{yBktwpEUeZqPW-gCm!RGikkd7cE?>hqTv+ zqo4Ed3e4OdBA2jd+$xm~Xweo!#GN}v-(QTN{xgnH2eDJ=sqbOBV{QyBk#l#pKg7|C zlc%!ok3|GqOdV+6qb7Ps<_{O%U60C)%IV()vh=U41*h=*v9Npo5Bq1e7}+s!h2;h8 zr6N^46cA#9!h~7G+;B-cV~HfY(l-sYoZdmd{|iDd^tI@?`v-x_^+ejIZcC4UHKD1^ zE_7+wC;E1NE9E7;U?VgpqVL1+IF~RZsYlWL=;ZKHEDo&>kkK(xL z$qe0C^aL5DCZp2VM!eTAFgrLUj+GSV9TtsAvZL?;=NB---YQT)R~L@b{U%FT*L!1R zPNf3(=eZ%8_O}|ixXxqdmR`btFWKUr$U$HMJ8-UlGuT|A3S~8wp~P%U(rS1QcdCoS z70-q5GZ!?0=yOS6L7EKN;yE2eEFQsLPn?0A=VYf-iI4fOYn{ntiM3AOgk0C85h)TI z{1aEDo?>|ZnE;=NCJvlD?3oz{?iQFb{8xuDt)EDytWpQ!bt*XTdrO%t$+~<*xuf;I?%*EbcDGr!I)XXORZPZ5M;v>y(H|=oe6)^ua0Y)fccx z-W;ARoIS?#c&8ltl5j1O;%e;We&X(T9KT8s;yP&Bsl{ zbv*~0W}M~gY?%WR#V3&8HCKSL!)5SB#GaVt7h;W7Z}=H&>Ok{XmUy;T z0KdPVfxe_2nJUcAe~UYfm;9MVn%#Uz`FUrk<&)08)>Q=#*?a~{mtDy4R~eGB$eNg@ zC<4s^W0CD80o*_$Onpl&R3rFdr+u zeT+A?_Jfso8^HIYkrt2ZDSlNngV`OF03`CdaNSw}zv}vevHiDk-|#`K*60YXhIx{}#!BFGJ&lYS{{&~3 z$1&+=eBu26F5#>n*O^x92)>t?ArA2z!@Nn?oo+G1jHD|f*}E@+vdlvKeqt(^=By7D zZztfce+IDbcnrz9w~SFC8F=nzMedV zOw6=}2bUdUe(W$J-=vh_!ln)U!QpVSNof?s6>I}8!a<;^SkZZMIS!^l(;PeH^nH>h-wQpe^@d;4vV-rqp zj{q7PJNcE1tzgK+V*ZuUxv*0AvG5o$5ZU<;XITv3%sch?y_q$>31rBEOZS-dZc#wu z(=m`$s!d8uB;ls=4Dd(jcB5ws@Q)02vNzQM{|k{Iy(2cT)bS`@Z?X*2Rr7G*YDAW6 zZ6PhDr}_P=mT>8%aj;8f80`9V1!n}w@?WGcArFG&FyHn!ZccZBm7!|nTW|++yx*Nf zRbK)vsY1p#{soTG--bP!OYqO%TDU9vCqI7~BrP)6K*^xKkV7!VKtc{0{W^&)?Tw(o z&%}veRg8yPCc-x+Rv=qP2fpz71%`Tb$PHtE*s47MHk}#==Xw3WcwB`XKR6A}+kGD# zfbrnUvLt-&nkQWMSA{H8YzJ?p&B!$~V_2a&8K?fZ4P3T3fz-!3^ktPb?FF0N5COyGM(^Z=q0EIfy=g1}-axWjHDmUFxSt}VNPQ=9(* z(O>tmuEbmZK*l++S@tzXj{=>X<`&iTT`}cb8`WWV*)(wd-DdEACdvdBfOC^LyaZjtqmTLQIzS_NWx;QW&#Q*CB<{NVO<};Vp6d~S<3+;NWwa#x9vS}7rOH+w$W^{kAac5o21EEESN z)ekWilYj>V=%!{AAd>A#hZdFab}7R%-rD!i-tAfDYs^f&t=y%eUg9%2mkUP zzSn^ABH{djYq7w`Wf|68GMR*Lo(?w|ZpCX3wmMxnt%j%QO@KZXGhlq37E$XQ1Uc^V zWZuh(&}ok&$*SFg!&a68XA>Ec*_8nlW)y)ZuOEQ}>~f;^RFSmxoX1X+^YE6C%{Y=t zz!z6EqpPZz?5a%AE}x61U6#Q&z3qi=EKd@3BnamYSmVso8Q6QO53ty!2@~Y@ z0|Td5I27duBhzCx7VG+dP!HXCNBtf{5R?(H9nEpC{84M0`CF){yW%Q`3_*m zErAO0#117A&?9abaBJ`5!zBb0S5f$)?l4eu^n|;23h#%x0VH_t0c>C}9aY>*c+CC<`g|x9=97mTrB}P9|QQyKbEPKlP2Xh3E02)4)Cv)CJ~0>q?3=7WY^VaMbmI8WMgA~qv8F`#WFJ_*Z;sPWel2tGoF|;`l~hxvR|XpN z=P`v=qfEt}KCEwd5^T?o!Fs~)at{(Ezdxtq>b;quyz4cdyUiI+dZkLNTRpMQe>36f zOMI}XXNk~V_7GHfJ3ubN2t>Xgz_)Z0m_MV|FfgGIw5*N=_bxQ!$%l?H;o+FCn7af& zzWD;I`mqUUnpI+*RcEndk_6D6@(??9rsC-fmV@XPacm*|26Ocj$l#_Hu%*p{U$$8U z>iPd>x)*=I?j@QqxUCly{PqG$AnN$;`Xsz3hlJ`tqH z-vG)lyKtAvB`4ne7XF#QWhBS>9kBcoOe~i!17A-FnX{=vW>EJjA2k*O?<2*)H?{;D zMQnt1X%lhdDM>i5@DSb@WC4?sw=ld~A!|N82e1CAhxuc7ux8pHejns;t>Sjv)}};e z**M{!lJTI@#|du{&jG*e&tbC%3xOa)l9(x#18-+D{`=lze5=pGtcFG{FnZ}q=zV{< z#<&BQKUaZ2clwcr?31AVnlKYont(%R7BGGJm$B*l?_`uW3RY!2z?YV{GG%Vn zV1B+XR6jDOMp4g{DbOqd3;lh;?57K$x2&#^1qOKjs+U;LF9*#3Btb6CsKKubOyON> zgm;7=2jTuAxF}ak=ss@)@*aWkN9Z@c!7Ej$dv_5o^}Y;JDy)EX!)-@q@dQ{ft_>gf zFJx>~-(aE32c(IN0p{C9tm}LLZ`zUr_A1+gWF2KNf1(qJx7mk-+9yMgYjQAQiw#u# zCPm6ZAL8W;T0oKZdeHB783#QTGJUre;ij;8Sbf87>?3iVex{3j9up;#ci_r)yWm zVPNVe{PDR6xC>Kpc#}1-7tWW3-;VMZUabKFVGgP4`)=~C&KAraSq^tEn@>b{&LBol zjbNkSFaAyEgBa@yKjVQX{1Uwh%l0;d(7>&D#zF;be9{e*;`OlkPAN{Sxd%4dUk3wa zhwxf)X~Gj`3Lfg|!x&XFFe85e=S1~_1>Txi#6_7uXm`LVe@P22Ev&Vp3;RN(G6xp+pC3USzAgC){^ z$jC+?sOszj&m5aYyjP0Diw(N)rR-Z=lV1d~GhGNRp9yyDktPA$7JRNz1-3>N;)n<{ z=4-hwT>RahJhgewpR4^0eEF;hCGSR)^jtf9(anGiopk{-rS`(c78N*k8`cLbHXO$PX$^w^ zB6Yy|v$rwux{beguZ17hiQ*=qi!Xa^Ae`^A2I`IS$a1A8jLIG*pz|UDU!OIJ**`U+P**$|ey?ZZFOCG@sCQ89WhgG4KIRL^Jz%W;LKL5192iU6^ z;Yk~%$*|TVV6a`5tm!a^&;Dx$mcsScd8z_Th%<)fT7Fokz?7`+{e+Wc{J{+K*fT`*7ypvT^a^GQyC&K z5Qkf;4&fq?;>u+U%s|P?3VhnW4g^eD1u|}^k&NDGe0jMRELy^1g}E2-<*8rsJ^f~2 z?9+$)vaW(jm7RFmatD|$Udqf<_>K2ij$z3-A?v;~5>)!_#+?hr@r!Y5yiU6oOfL20 z&&~72N}aw?Hu^ev&(k4#6OUqxwhqVNA5-x2-twCHX`evWx^LK{rJD(Bzm09RTEOfY zX*jUD5jWh*2S*cLVu#c2pfJx9_QuJADQ1bdb7wNB<45zc=~0jyGU%kW!J7QLZNg|K zoyXPm1o)RC1Cz^cfDZXOY*M_t=FyN0Irdxvt4-d{UvywDu-$VXCq(MwfW{usCcB5p z@Be|L&)jkPLVf|?COLTLvOP?UnL+fGQov|Z7WVx03Czv&!*{#{{4DSHph3)-lr!VF zH|h}H^<9%6{k$2o8lQ2zq8E-ieGpH#*apOHQ>q<8WXbl1FZhSl1)(E$1BvY?b+1oRZc*d#1^lNT#`LSZm;H6HBK2~Ea!{?vOc#mJd zbcIHDmf$$A22iq74!-D;!;g!m!H<4{cQLSQA-9uRHqzJNy)}} zNqd01eLwcg)?mg^1Ku;|AWn*tfmat#B;P*EkRz$~@Zxm`klz%KXK?`_cLY`TH%V2WK!6Gcxeu&FlGJ>Qo@dG=reWgV;5g9#|?(!CVz36XBln4fHj0 zDq1LL6p*1K^uTfn`tIflbPHRd^HRp_+^kZhzd_OYgMlwlt$oQR6hEX_%YG8gO_(i; z`Gr357okMeUNqorgA}eM(08eoT%OQtu(}O%IZ+xk&}$jG@Q2R@GbiaGnH)AYU?ysB z3uXf^WYTY!R@3$ir`aX`|Is~FKX^;|mj&|!IQHMoVbotA!{gE(Q@iXq-|5u{43=G?TCj>vq5H9GfwCO2qhMAwLxqX|z1 zsJvVjRY_!{IrAEki_vQCkyIIbZ0;CW2x5@7!zf$*=QvkoXO7CfO6kY?Bk0fXH|Wi$ zVCpcVg>`;Jc%4y~QOwjK)VOsTZTz!_KIW;>3-1sdG zUM)HzAx@2g`l;<^vD$?twn#8@O6}HHZM;wHR&GSN*BN+@LHH-6d2ERwQXuSI9b3qG z4-ax_|4O)W{WoZ6V-jVTYtlVgakSSc0EtGQ;2IAqpq5cH!PqZDmKQsP7A}%@9*EIk z`-URn`0w|qfTqmoEu^#o`7b^>~mDCYe4>t61d zs3?^!Ye63?+XB( z8m(vOuXPrRls|(i7G6S@F&DZ1OIkE2e6sVytBdIS&8OJdC!Q$2zm(p|QQ;=%JJCZ@ zH`$W=7dTldXD)y85UQ||LvPhvImrZx+ND>zxRGcB)+8{R-qc-8KOBFE76wP6l*@+P zF{u(dJ-(6FJ-@*fMHSG(ql-D4iY|IS_$PPLEtc-8KE`ELPDZK|DV@+~z*{4=5jnnk ziawH5T5?y8eS2KoxvT1$kO4f*%8d)?_TAQ~eyczrZfL>!_{t;cW45)!?X8^SSS!0b zu8wZboy`?VRneZo3a)=mEgL1`z_Y1tL7xj>Ah|AADrCu55Ek|~tn{Fmt=aH-I>?|{Z&i*shH(Lv>5FZh!oc_rI z1rB}4Ql>$-El|#HBFK=QN&D}dVTCR3=!~5{`iHv(@llUyoUJRmpFPdli`25~+gjAW`&x%ud%PAKJczw6#9aqU!cgKB=p>^iSzVJL~rl%>HLT`p?l&2 z+IQhLd%gZT`lc{WU)IN?$EEpPwa^Xe$Yyge;uq zI#?|e1@vQL3w`;li;I1yLU#u&rpH=lPz$ScWcMnGUH?hB*5CpXup;LKe=YjBpM%=Y zt}!ySt~i{#8*mME+X&C4rzV{0+P6r;)m2dTat)Q4ZSB0}eIkl|-bZiow$bHJ4B5M~ zqR#O<=dc%l4YKmd18kp70{h*m1-*&9A-MR%uGZZAC2w%ODfP;}#cq<^$1a{XnOeM+ zcEcebr#Wj5Jy zuWt3SI~F{q&ie{@-!FPmWsOE8H@X0wP=7|JD6K*jL%%pxmkjE0bP+9HG|ZDdtV*r& z^{CsiP`XCur{G8I4pfjW#Z@azr*U^ybLV1m(5Vkav_H<7y|DK?DQZz;g~JogvFThJ_PfT>Z#Pb`=cjE!DewE)+Z`gc>$S^K#CA7yVdNptNU@t8Tp>;Ul^T)F z(O1a0HHr5j{x&QPxs*A3ZcXlik!~h2-{rWZSAWYR6CIqUQHs z(b=2<)(M$niS$YdeaMP)Q-n?e|TNUH|t1J@bQ4GaQUi#oHwQMNcJaP&OurUOt=N z{NTkYdmiIn9=cB_)y$}6-?Xt|%mv=+k{ooSo1v2hhtV22@!E^+@9DE=UA(&sF3^MH zv)PA!LSEo~9O_T6WM4N2P}_Nq!mV%CvkHm(j6EZS;*E>AMM;e8j zOs3^AUF;W!rRbNN11dNbMh~x5be6rpUa)JgJYDgj47I6Nb0VK_2>vd!LB|UxqCFNX zsmbM&bY`SC`g|xKbu8sMXE#itSx=gII>*nkSGrxeVWH#GuVFq?s;y@krV)ugEJ3CR zuCPx_K2x=r4xV;(D_u5dh4!w0&!!5UU*K^N4LWduo)}UnV*hH<7VuW%QS71OlA*MdL$vZ!R_Rw~bIL^{g7g36{b z_#c{yOwWX)WBzk!)7orw;HfJ;nBz|+>`l;htqHY}eUqI>|HhyPE1hW8jGOGbo3SWs zlOEdCd5@+hnmV&>>jWNJ9qfnMEL{{@MCj=y4t4E|n&svuA+w zcHLh*!x2#=yM7nD$16a%j-R3;AClR4@z*q}u#-KYc^XN~*WyM$Ek++lcCaPs$#j41 z30iS#2iIDp?3_5wlsyx0kzJT@p8Iw`0F62I3D&HuLqF4IBBNgd-k`4yednvg`m8FT zQvZoilQto9?8!r6x3_cB(mbk*@@RQv5e;`eMze$0vRCZ^&4Xj8!*T{)`Fs&Fe6^K& z#J}h6CH<_FNvTOk5^HF45aq}7xtU+@b3iwb&dEJTt>P&gvnLZh zI(n13>)q$3&dK0-zqDwR;&&96Z-C}){>Zu*N;zk)_Y=B%*K=McRjAji1GI3B1AWwC z!*gsr&S@)-q7&DxIrnUaE~Ycsd`U+nd)J?~M@*w0cehgezTd3t@E9wmvxt4{_LME+ zb<&et4v=giN3-Y00(9l)I1dk%qjK^6Ec|zzR%Enu4!S?tgg=v=Q&jzsv~>Ae-r zKGce`!+)V1|0{H&0iR3kB&^H2OzwJ)9*s6krcc}j=*ITzD9k>d9_#strtdD{{cMwE z?K*AICC^3Nqc}~}xv`A%nOg;)w^-3)u~_uD;4o4ts$kzobg%_)YS_bx7Bo(6C7rTT z*LjKfa<(s^}wyeDCVgX$=D09kGe~kJCZt9^9m-)fciN zWwC-4-aSY)N*ZlHcZ0j-kI~b{UgYBk>EDn=sLa?rN9=eNY?f%E<&Q%g|pa}M;kmsP|9C-?#AZF>=w;t_L6We zDHr>X-CjSz`Rer)c;EakZzFr0uF_Jc>TmASMb+u3UaOF90%E8)WjkH$l8I(t@}SLa zQ)y+#bync?iQAhZMnhyf*<}GCJom}-9xbXI;+%Z_xem=- z3eJeo`uGjp$hHb0r+$OhEeb}n)m>0}Lo>?vOQH*NPtaFi^HBei8eYBHJj!3<;;iN{ zfie-UC}>Nlz^nHK8*+Lkid}J^^&+=v&$Inp#*&XHF6R?fm~sN05jjhJ8V;hIp5FqI z$#v+;YBQ9xdp(N%+Q`M-s7KN=@3`iget~_)2cF)lR&-aX>Gyt9$Sq5J5;yAJx)$bhchF5!%Rm{G4{Njfj;9WVD=9_^_;&TG7C zg(~jsV-MZqxCKWhp*_ot=*B{Sy6oXIy5H9c9dbR16g#JJy%wIRx;um2+1bTz-}xLV ziJI4vmyg-f8Mj#XgQ#|Wq>%3n&__FD55v^n{nRt_9M!DS5twmf+{#~pDCkXa&CMCg zbo_TIYxu~Ut&KQK{Y=$`p7Rsvi@OCq(qxaWpM6Xc?2gdp@)S<`swVoj#u*tLc)*3J z6w|FMD^$fE0?uUzmKXA1Q7QAPQb*zSFB2Dc!p}&gr(21#%&OXy9r{I;rZ5;7IXO1r- zdR2efr|Q}CPwP`!y{(4(POhRK6S7g%0|n|Cxte;sI>mhn7O!1+bq;d3ilS6%KWlui zk4veRMCzJa?ET{jtiGNS{nA@RJzBGQQBN+R4gJSZoM|O{WNRQgVDOWx9w*2n=QO%1 z{g$)xc0k0?SJ+`Z1$|qmN&gr*&^K9?u<+bsl=WpR+t=Sl7erKYC42I@(Z*z?JhPIQ zytamm+!%t?B0Q;do~X0K%`#*sdlLCtwsRYkjA+BM-RS5IdHACHG|k;qN%wU=Vv|b~ zP=9V5yFKD6*T$rx=%VwqXjvgs=VLXGKx;0x+V5wT`kx<5o%q*tbQL|`< zY&Yx4oaBnXUcvkO^-*MO2uk~vOg#@s)jEu=pmx$#bo}Wl8s+>34fGtRzn2R+X}?$` z>DPy>Ry{@0yL`|a_qS~0-gn&EBk{<}?lyIJxDd_UQAt&0YG=yo1uE9G-EKTF4aAupwja@VD*U6a5Qx>z6q>{ci^`-hnQ@v zN8TT}ilyHB!Cc=tbvOLhQ&jb?DcEDNv z4d3;68GiWuCHSX12KL%}!?NS@BunVlk#%mx3xjXtFGmZ3m0h2s|Jnh(I_4Ni5~kiX zUf9E;RS-n)o=iCO2bksXu-Nws{Cj#94q3Gt^yXUdk5nGRlY$TOH~zDRBely()7O4X zHs9cjO)tlH7Egwol$3Db!v*Bw%;oT8@>{%p;W)nK83t_^Nt0F!b)pn!18il|`NJ;X z!HKxJL`&@)&eQb9W4aHSj9CtNyLlwIs6J8X!cl-D$CB~%k^mA_*pGXcG=ZXH3$RPj zZ2bM%Z_J;y01kbA!&i{F$2^UZAkD8F$o2hJ{KChY*yjujmh2aHUvB=&m$%R%xpb=2 zfVv9Vlqzhf@l}E{GewCm{Uz+-5warNCXg&k3G#7H8xCz!B&|lKQ=cxKcNx?b-CIu*&_yfPrk!;=}&u}^r_(%1M^lQjv_*&*ELjRAGSw_84KiNIZ) z;rj&}5y??Y=yQGriG4W=%p?53qjYna<6}W$9|S@aFvL^^uVu`{PJ#_9gUBR#DN>VX z4u6+CV@|HpC$gtSVb{lGAaYWM+$@NM%^IrkQ(`Dy%NqtoFCECU_lK~;Jw9W1e4iua z$H0WyI#B0(HmF>ehi{fDf;Bgy!N1MZNvOOmIAq2`5BW2Au0s`QPn9IOiprq>v?;uL z^%3|vzY>?;+5%lpCcvu-?KSZSZD5~R0%TDj~ zcyfTh?dTe^?1eJ?QldoGr%xq2?TyI%HG_DGoG)DCc!*57mrM+{T7hPrci?B5CDC}k zh`fK452Stw_awv<@vm2acvOdjvrjbOw}e=tV4X@v1IB>+0#S0xJ_)N{G{s9QKF>VV zBMyV!Y$U$jJIOP?3M^`zD(rJJhf#y~@ZvdQ@KLNM33FcwRaPH>ueL4%u2J8e3g2xb zCwhwbht6LD+Q#vqOj^L`dYtCJ`_uvM<^+=N#zg2s;-N^+Ydl(z4>pX};=}+)a?HI1 zv?e9vk;(F8PDCTFcXY!pr79$t^MnD11L2CMV4zTw%+x67@->|-pt;ajmt8AQ(q-0> zBQ{IO$8G8S{97~Oe6dZWHXxo5(LV6^?rI?atcRI3X$`)9I~%0L%96#!DnPcyS-Ah6 z4n{H!$oO7EaPiA>$n7WqzYM!@x3DXCW!ZGtb4wo|cfENJQqIpUkjEY0C(5A zk-9-$GWnx7d6V%0Yr!qh_V5aL49_DPfghM!$p9!_uLaL-ZNVxApKz$C8|>}(2e&Po zg?V-yd9?)Wr-d8t^mQI33--9jELQ-W*OpU0~YEw2gtmV!M@jK~{DXZTF< zFcH=llKs{HVV!?l!JBuMMAqDrgzl@v8+^N%7x77OQC5cIW&Sm+y0C_?R(1$Z`yMOw z@T|o3Q{BkE(~rQ=m3|=EYeb?t!=Q%9VWKuHM+OIF;E@gh2V8=Q?^l1K7XYyMh6&`+ z_es#+P8+uDf6VAC4u(M(lE)9CF@J$9TxoL>@9oqkbF{A3EUL{1Yu(%N1EZYkRb@TE z&-flNVob>F`VbguX9oz85|+1l=1_8EfGJE-CGsB=p#7^f@S|cPX64tB z{hx%%+7zLCuptk>vebabces)3o3%*783U*{T>*YJUq*gzTS=<2A^;r)i$ zFsJm&$sb#E~3{tnQ}A(Tv) zh$J_Di<52Zr$7+&9OUH|gYPS4h-GPusQV|DFta`~@jO~un5d^%?V z5p(?wUSutR{2D6~DM$csoFs_Lu^u34%JHq**TIZ@K2ZH$f(u_Oz?sek;B>$U6EZ`F zxSf8&&#syRcts{K{KHGE^L`k}<_dYDzBy2%Z3o_~Gys}SRs)g*U_)cHu>Wia7@D{m zP73F0>J(Q(16u?5tsDmfwn;mIEx zh?@C5{2)XU+V<;0-QRk|-7^$UIwVdOdzp~)@k(U!_5UwdJSI!}~-}sZd?>E4^kYFaz zCXUhiTaTOK77{q*0%?LQS(uX!p6V+?aiNdXWZe0LVq$ZrK%-l?F_N&@~n^rU*~ zo-2HdMg`ctRteq`-k(=W5b!5rE_vP1!I(z5K<&Q^iTSSm!mL9fIQ>Bb-oY%%%<+UG zUJUt_DodWM3M3!ZOo_HYkz@-Qy%^7RBz)ouSbcaE8TtU>cj5Ju{_h?Bkt0d!=W4_tZovsry#waFr(XKR2j%z?D2%Z%uw6 zHCT2`oUCkhhxKRH;MF3T{I-a{j7_5#&`P!;{kNT=+Dz>lIiCw4vUM2_-r+%7zb}QY z&6=b<^Ag@+a0N%(PbG&}JmYKS2ZNO7N5MH^Z#_DDkYV0>5rYgncsj)s^r8%4?Q4vO z#}%P#S_{4_Hy@hG3Uv*VmvO^sjLnkP0%g7d9PhCu=_)@M_0c;FH=_gGoUa4f-W)uh z{{ml_x|-DOnhBMjXp&B0U%f?QFOD4zhLV>UV%B>O+$fYHigT)P%=b>P^@J+^^6D=m zA95c&sV@i4n?7SfzZLm9CPP+bI*=y=3&6L#9&nM?W4z^*FNs#k#0z2?0p7k12k9wb z?d!InBWou9QswM4HR=by^KLrG6(X;KR9$lC&1?Krb^z1~S>BJy^T66W6sQkA!gUQY zWUcoDytMQgV-w;48-57q^xOs@Fmoc+zm(y>D18#NQw1I@cOyr7rC`Up77K>(d6j%!H^ueH14{j26z3!{1Wo);IfWP96xT7)&OD8+R&Hzo){9raXDsZU{!pI03y0wDv@oz?;w~7&`~(kAooCv64l~9IJ$T`lf%AhUiTi3P$VmOgY4;pR>6L7J?NAgj*1UpeMefJ>QT_nz*af_e zMsev-0=Dp}#DUZ75eOE4mI3+PqBmD>$ z7aqW+4^LxFVJ>Bggd-_DT8z(VD3dj!hhWO)1ZXRC32uAc0$z4Gz$>0tf%DvF*gq>1 z#4T4PKlZ4TxmgCVKI$~yDgO{;E*635V|zif&lbEtN0pd<$pSggcH)l*?VN^$Zq8LJ zjo?Kq7clDw#m2ce@TC(3zi(6|jn)@{;$AWGI(U?6+<6wC4d~)`lv|PE>QKC;&knW) z9mNhESHap#>A-tLo51*0YjY;Tcoi#{A<_i42xp4?=m%ivbptqm ze=Yx#{WzX}#tcq+wg@J^kj1Yr-vC`Io}{qh6K*bf0|wuvf*!mLbo_eA@QfA6IbC^t zcG(w@nAQsFJ!P=|NC_Sp`~;kpq_8xH!ODPG(BHd>EbCK+iC;yC=i@_8t2Q13kRJ- z3z$E58Xj5S1^#?Afi*&Ylc?{8Q`z6(n~+ytc(fI~HXP^ge$bB7H{Jtu!6Ljj?iRQ| zc{lhZ@ew1#|FBQ^!5U>M33rW)!P)U8_}iIy{2(xwAKq7mt^aw#RMlmS)g~=cx7`rS zJlqSyQI}JmSTHCanT2oF`2j-{QP2{v3jLJZaetaH;%?N9JHwvvz0K|ekWq)dcb*6G zy^7GlU=kV3?f?sqAH*ts447@>PYzT&kkt~=&^TKLPG~rZHwjs_4gPI>+eJbSZ=nKI zIA(+IO%8zTch7-7mEy3XdH{FLk{~IQ>hbvDFd%o_2`;yuO5Qn5BOmjILHVCC{>Bwv%Yhw&I8~?@yVoTxdTT0~GIt-@Tg~4f!U6_9%15e&D0Umv& zMM`4z;U<9!IbWTJ_eY!u`}7Xusw=AafV9+(LqhCOG>%q}pk+g5>^OR4zm zjGZ9tfh?SmHVDQ|)XDmq?J(5M0hVxnM3}yTn{5g})tj@po9_g-FP}|LMDFH0P5lmL zR~kaORC%~)g$v$M%_Fl+<)Ckb2+_`U1YU2>0WY6V_~BJ$sP3vxqJ%Tq9JinN#qE#G z!}-Sm7rX~QS|$oJgucNY#_QmOikrZ!qZLcck%li;STL782SI5<1D+^!Elv;5a{5(l zOS-fV!kMB%C%J(z7no2DWRG12A{spGZTZ4!PV!OkAa(+@nVE*mZ$|^I4QGbCu!`k2XuYpawJ!#9p8l*gR7J4#u7fq`>DY%d0*kaE+RB?O|Io)aIzG?mB+9!SI zF}y_dd-o$MczIVaukQi(?Wza&?bJ_hoMgifO{dwsR$H``j3cQEeZmh6Lv!EvbJuha zqkaFSB70$8|MUkF`ag!w#GQ&S48ST|h-8Um3y~<=xHI>BH!4Y_g_QPRd)h?%Mnv`{ zg(wwKvXl~c=AQ3{Qd)$PHrljErA0+izxf01^W107%$+-P-tT>Bw+~sEok=pShmnw? zeA(RVy_i~|hmCK>;Ad_xINaVKw$hKM%WKzSL?2&VYEvmYyYCbUy&~o}u`GJo%!ZSU zE-u~BMHd(u!G~@?cvLzNws}fWYuXjUGI#W4G`Nx>q)u?*@Er-4mxA!8v0;P9A>V{fS)_v(8@50 zG|bhW*P-|+sEG?=$WXs($Dz~HHYwD6lfNzqvZ=T07hWs9X`_aFsa z)OiqdHO;9Qa-jtI)Yug_1xgv3KwIZzwj zj(3Vnx-6k^N(a$*_={T|bcyb>erTinQl?bi&aZi$M51;-r}r}^!t9RKM5*PMY}S-Y z{E?c4cb(m287C+t?#~ogFvGyrtRBp}ZlZL?O?Y*$8+Qz>BhSvIi2s@?Nq!Zqfma^) zpf$pndhb3e?tj6EPrpA5irQ|%W5p=p*`=?%lh(qI)rIusl{F|oc7*o1WK8x+bwI)W z3ctHwA8|CZgUosIhNuN4((tVZ@lT(zPrGeD<=sj|0{uK zhPxm*(g1pmtKoDv#jCtJhW8snRnmTw_>VTI{nCb3Zw{hfLaTad*D<1gUds8*4uR;7 zE?jxQ3ioAxrc3t@#}CIVg!lTl$UDnU7^M|Sbqn*Txhxbax}U(rfzODVXD!qi>4%P^=fXqa_(q#t3!D$Wvf(6m7loK`6$}Wp zgT8N{Vz5F4t`(ZE{hk!UGG{+rSgeWT2YjHv9-R?u>)nuz4sS;A*~j%vQGvI{iWsPM z3S|49Kz)loxNl6v@ehPO$I0&aJUta(=A1_N^j=y!74c``d7j4k7gIZg34aO{h{x(<|7EV_zgwTi(q@eTTI8jV{*4w0}ETX5w; z2_%1aBKKN<&?ipr_;|uZ%orL^s_%TjQe`Q7B5o7}g>=#ScBIR1RS=CbYq-4Jfb2bc zpA-f<$sK-=MW)~mniUO2pD#HiN96>5SXu)n!|tN-AHgemU^+}w^nx;WA6BTWMP>i5 zBsy80bk{1P>B_li7gdMJCza$j#@A%^qfdxkO}2}zcT;wd{t4ohx&|!SGV1KT7~Mxr zgHwy%(&?j9a8p1!+1mF#nVI{6hRiMI6<@z06>7)@P==$o4xZ7Eu*ll0J&y1UiYBE!7pB_OZhNFp2%RV@$(oNp1`ACxwtc7jA z7NK6mariL7RveJNj$T-J1{#u%fNbg!aFx6u4?}11SzdWO`~IDHe~pr)s9~IhN5Te!XkrFP zmHJ6Kz9^uTp9##^lT7_0{c(TvK>2lEL<8dw5brbd<*d#KX#bl9i*&zJtu4*?u-yXZ zvZ;8$Q6!T!B%}zS0r9coUjKw=!TFIEBj`9t? zxwy9aAT$mvB=eUmfXN^;V#FMTmU2gG{rMoiZFo)TBR>*f^ql-W=nBzU5)A4)TKq(> zm+Twkh`VcA;cs-MSiMpaRO}{T+w@1`we9(2AZBhW3zgN=IQ|b(5@igA&eu znT(Q?jr7aBG&I6fu=JxQ`rPu8jl5KhALcotx{VIx{;~mMgE&0oYb5uJzKgF&KiT_n zgCt9sWLmGUjbAD_jM#b~8v1R4J|Deh3G6pnVnQec+~5$pSHQay3G}qKz0eh}A>`Z$ z*`Z^7X!b*GdjCNM9er&FHVb{yi{GZqKwrwYB4+qhe+77Z?L8nk2UMo{;*adj= zIj-7cO4Lq8$b46oVQGOQw0ABOc-9x>fPxu3`EeOGy61??TxwAN&Tl$lS^^$)2tprw zZOL&ABLCF>l2@~rP?wQ0xLn^KmK^N`MO!6!IbjuDes>%i&Pbwx?-P+dv0B!({~~TZ z^%yp)MB&@#7I6R13TWA14_%FgM8&`h`+vJjPQwq`vnv;2!r4n;5Ly+ zkuExav^k_F{G_+uPQ=QTe zX`gi@xk=yz=Wl_&lPAKDrg0M9f#CZ@Wl4<3fB0#k1zs?k2_NleqEYl%=u>1)wbX4O zGkPGNzGhDTR8$GaVF<_N9s?G{UJD(1^QArbL zsmy~5mf3hqc^+N4$%|eL>?40un1DSh@%V9%rrdnZ1khDA1+~H|+0KD+cw4>&58WA! z**<5WbhDCt*4#1jpD*HZ=L{&kRxC##ExWE6zkNc@fOe(`DPd*T}&S-EI9|dg_1&rKNCrqbK zqsQz!$6z@B&3)@!2K+muF za9T@6lG&R@$9I_0wNpB<`OOiqS$~;Io-DwVT?+E{y;ER@-DC(c-b?KMegb7hW%>F~ z+mX9775Z&lEAUN&Xlo;-j;ST&N6Hy2J01#dADdC{PXKD^mw`%{18Dw=KyAqaIPU%j zQlj2d3GAk!R+aegPqnyqz&ZMk9Rjvxam1gQj%Cl>;AN*3x(+WSC;VC3&>BJB_NbF* zVQDZ&_`T25?!t9yJXzXWE}p1iN*sm^11(V!oHZRR*;s7~7N^Y7$ml=ZD)5#M4;#Sb z$*1A&2TyFuaGUh-Z9eJu^M3yF>#zG@wT(#4Rs6}p-#Rwir*~2ln1;ST0+ zoC1%hPKKL5H08g-#h@8*3Cdj!;M~Gm)C_wEV6O-3eC6c7E_3?PqXw2nRMX=cV~In+ zeq2!9A5BXOV7cE*LJydW7a325=*SN8>iS6>RFngA{tW|vlfNWrw~l1Y*i$rr(-HB# z89vaXlOeluaS$c?lXxY&FgP;snz(KD6Bw>BA2y{#!rmVr=xwD`Y!R5aLv3K&mw!IrT8F2 zOYS%JE8C~~hHTf#1~{63f@-BXkag?s3T;|T$f{GIzq45=I`;yDXeJ~+%K%5oQe4xR zi?uET=xVdIvY+Z3;Ab!k=VE<@mi-K}Uv&+}6y=a|&KA4dW8uN5N~LR$dLp4Wt}6Bba-q+D=UdN12vnFohEGw`H3hd=thAj7qZBdvPc(dbwTOMvl7)Y_*VBPJCW7ghEcS)nb-1|k2y$;H!m#9O z809-evOyezWwQchB{B)>z1>Fl)=-&f!A|%uEfs7I72uMm$HD5PCfSb>u<&z5+h#L`3_Q+AFwH`2whGvP$v^)KoDy#S>1vgp4_Q>Y^Ojt;i-=$1bM z<8~wpZeKbA)2#-RE17=cV-jT?TANGnon1whC-#@0=~0#k>@5MkWvA%B9wn62CBf#U zXK4DUXJqKk^W^253f!`CknBgyGE|*@9BpE&ak^=z?3S<457<5cCdY<>hah*mD@<){9wSrQy=ju*&#e4o0**5}#U4fnu{cNvXWo-w<1JtM@Api)?~*O~ zHrx&e|Mtb^#%0*NR$p@N=0}VT9#4nrRZ!()JaLV?I{2 zJ|?(UCHb%}#tUl;j*AzmnM;m+9*Ad`ej&HV6^TzaO#nl$6Lhkcg5>t1KztCk8uolq zLxVq`NPYioS`lu9(@#Dnrm^MtBGd`@j!Ph^{a&!v5pzhvZFLBGZARv%caeLk`=M>z zAzaq?73X(Jolp6a!?_;L;Jn}OW=`aob1Iu%nO&X0&u}po4Xl{R7ri>h9iHXQ7Fgsm zmr~T3rQ!cc%Yt2KqV|km~W0)kN6Fcd}M{b?A z6Pww1k8!Kf ze>^m0_9pgc$LJJ_PAr?ow0eHz=4h~d$A>-KBORfgGAl`RChRnm@!pS(zkZ2%y7V_w zaqtzhajmiRSe7BH`~5zrWT?y5y7!1yPWEGBmU}Rpg*NM?zxkKzHoj$oEE`1BbFrw@ zY=*Pp%Dvo1`#X$RZmx7p?=h~XuNiB^nDc*2Dn*|0{dlir6EGYEq2FlsiqN8+vfW15 z^Zw0jx1P&Q8+b$1vAUdTO)Fz+94feZGmAOiY!kormV)5Dy3J`17{;dT7{!8n5pyO~ z%!GYe%BiM}WcPo2z+72e$eg=T!HwPhi&3b{XPB9#%mlY@%$AI`(x7k!w#UQAX~nBM z+?WJi_Ls7R&FO08#(G-w$GwNK=0E*;$6XD~kZ>h_{nca4n)FDnP^njR>sga?+%g@u zaAszCkKhHUxnRK5778rZ-Vm;!V-&Br?U~f}&1N=Q`5%+kIDbOMbqC(%;dkzL(?Gt= zGR67JrFiE#TpgqIrjc`;puo@16waxAhJ1o!i1g%x9?`{l>O3EwEFz6I?C1W&`1?!< z=l6aTn{MR5iB^aihq$>sqc@X(ubIP)-CD#<7CNYxw1=?$M

    UpMP<7cRt5quE6+Y zEAVzTXGNxM;Y>_ZBe!fsq3CpnIHJ3dNhJ^#F49W(Dw1oLj?RQ^}0DX+0I zmmAP$IMZT!gn85amxDv_ppW8&zP{OC%5pE9Al(M z|1oT4X$GV9ct<~dh|$)fAV_qjTa9#OO6_bV@})0oiHdhG3X z7cTkHb>`+#4{qz6D5swLJSWoX&tB7OVD|LR;p^PS@-ffTxIQFKaCt^@8(ZcxNq!fZ zuiDo+<9j1nE#+Oz2J^R!t@|@3;jIVnmzc)YJv8OMx~`T+W=!M%#S&g=#Fw(D9U6Ra zbRySu(3xwOp5qSOJtf`Jks_TosGU1`riSzSTgn`}w2nLb=aBSQtb|)9`^vC8A97zJ zSYe-MF5_P^m@jiV@BE}noj3H-<`iE3VZN?c=5+<%4|c5NT%Dv$Vs#JGU*!O!*byOO zqHZ#euWIoZ!_9d~w1sH#>t})v`XytRT`DqEspPt9o-zGbnegrpM{#Np$L0CUMdDU7 zXFgw%-|h2?d0DK_Uyk|9V~Q4QnqtPf-C7{M_{W(^s4nBSd9`xuPc?7> zBlWp+XI60bLw&iDZoyBPGD51Nr^ByaKa|h%_uzC|J^1?vteBJ9wxYM%H>CUghH%b< za=GLEjxqr*x48ipip)2ID_rKWU@kPXl4&?@%$Y_#;M~$~aYc*W`B`sAvd=s!xC+1D z+}h&OE6P8gGB=Np=a+j;<(B<-hdH&li^~u0%RV$RW;c6$mKtcE6s=nFnECel3A3Q} z6VrKb$^>t2EEAtm$&{EHvA=Q$b6=kiTM_saj1-dA33-w?F%+;8(k`K(n!xy>b z`&+oiyfEhF?Of)rfeRnDiQzv)<#R8u?%^hd%S6ud!=lTtHcID27)#6JO?ma%z1-7M z18!-^WIi_4j#YJg$y^-K!PWcOGhstdbNiS7;2u3!6V(oGWK8$2W3Eq(X728><&G?W z&E$U#<1X=^oc9S1AeF%?>=sW&e#QH{+`emPnY)Ue9D8CJqx0u2Q=q(rTj?oyu!Nb1 zt;391FN;=Y(;{nruh5JC^}>+xuC(Qz1fF0HY4vg&o<87i7w_XLf^Tzk=MP{XCiKZCrzKhQxUlRVZoOX@^M1KId%Vm=G;_%^#-8Z%&5xc-*E`-~ zdhXpRf9(;?SdA&>Ud;T}?ZLB(D@wb=x@jOl{xoORpe;CX+)!Rs&_}yH} z=+jKI)p2fayCv+z8`DM0`r5F=e*R`km-#VwOuHE6E0nuE;ed!g z9ml+0evs?FQp=p$D!9zXjABph?jySMKA-CxZp51@xG{4#ns5?FU+G-`x%@zJy6Erj zhtfLdf6{^gU+#l@3U}u3Rn9Cgf-73k#2sFc$R&h$OS9|s7{&7onA8$&E~KoOi$1E) zF8_hd*4lnd&A%tyPV>i%LrX7Xx9>A&*l~-=KJ3Mt2p!Q=-}W;vVvR+yOR`0|cfN90 zjg5HaTvN7Dbv2vdzLaYg<`q7@Rb&+X6xptX*^JNm9L{aIHM?J)Abw&&)6KV2!`Wu+>>{w>V2_;_TfeQcR-v+K-K2vHS*I~`RUEr`llQ$?` z!cBc?&*bk`lqM|YM23f6OFv#bz@^qvj&$+d@9Xv4{=gsHs`6FL>^p}1%6{pbJ6poV zCo+n(&vzl=+@#%x0fYXA(x7<~Hwl5>)7e z_!6J_Qk8uVq(dh4W8)P&MDfa(IVSDom2e9=*TJsi1cMrL&wDU?b5^X>b%P=|YPLT2 z!AZhgsRjPM*Gm4sW(8i0w_&UYEa0|>3T&v^4d&MIzI<7fA?u$?*j>%5x#i)Gyb)6? zYUGspDig}t6e9C|&_S-;;Xh7s@Hkel=^f{O;Su+tNROX&_bIoorbaZ)Z6=puroxZ> zDwldbj^}ho8?c`J4cI@*AGyPpf;*DA#l@^q;$y>hNgMLU2tB~43Dbh_ORoeN@u>ke zY)-{IrnF}aUsXShQ&!PrKLqwl`zeiJ+$Da}{h=4=hQQ(&dz1^v|pO!DSy3Lu*4wmL;=&}x`bSt-NDlc?J{e~BF z!H3M5CoB7~GJQS1pMxL_r>e*^yMqC2!3a^>J?II_hsSYmOI*j`jRza=h*{J@?gkil=8E#HB~ot6`fkqYuaq13){z~OX~{c1u;fQsjbvXu zxy)U6(O_4vS;3J}R{Z-<;i6TdS|)aN+m*bEN#%=@W0^glpK?w*J|dgi1xzO%WXhDR z*_z0a?AmHo?seE%Chn3ppXaQ@XO3gJ3RZB=PK@JzMp*J%MkP$ioKmUxi)vS+Qyw|UBFsoQRe9Sa%Zn;KFpO= zIb%CPS+u-%z0`5rc8>F3E*+DgFS@wYo1d}oJagq(5!ZaSFTdMKmrva>#JTs42h)%I z;6BeSUVMTtA^pPqZ4OQBB?4rz7!<-uG?Hzu~&lgo1%wpupL8<_*8XE=0t&R89>;b(=la^u!`aCJFhT+0w0R@bhDSu^k?vwO!pZmXUN z-^i=-ImO0YYwBDs>b3{-dGT6q#gj3-&#M8fmCY5AwzI%G$_)9CCKKl7qaNWua+hdN z@nAmuUORW`@KNUC*n#Z!ZDYAlN)MQeBQ2OA3(hft-7~pPsgx1xTJsCIDg4?-Lw2Sx zyOWyOmz_R&5c}$Q0C&m6k{d4WV$$LS4_U1NTl7GiU!ppaS1=I#cB)mPNaud6qShQ< zYwj%msH#4FADsZ^sew*T+1x7q0AN)6)qsYo3mWAhD}~E+Iijs zPBj0Bz=gSg=bkoL^Z)6$a_6%;w($~!v^P=vA68mruQ4U z<#ThHw6?{}Qe4f6?fdXQeV%fiS6?umQfv0Sz>cT7TJwI4EmIoufE$!x!>=rSE|O15 zV7C66BKkad9Cv{n;&#nBE=~2R;O6WxVn=VXV3#~T%tbYvm+rkVNi=$EEHnKPFRJZq zW{O;|JI{(0xYi#@+z`8Z>C#=6to}J2_D<_dro_;(e9)T~(O%{ABD)RtqF1jQICHnV z%(iid8S7)A6Wr4*M0VSp*wF`_M8C4LIjgpCUcH|Kf59S|Sth>2ZJ89x^sy8%TCX}8 zg(?fdxf{Z)zM;Uxmd@pG9#!BQcVsf#g;~Fk^e%VUHH-7B@?nZhP1##a8k6Spf-Bs5 z(^=hO7_)De0n5KCXAHlK`L}zF+4?EY>?wPHX55!b|9m22Ue%ZX zoQ0xrH!_I@OL zv$@h$XxQrPa^i0Z_3oVP+gW0)rqxsDbpL0*X z)=A&AH;Zag^!Psdkz9+fh#7i*CckS*HDg{fmOUyBV(kPKW zfBIx6^Y&mIcW~iY#8RY z=RHN-p>MB5Kj%#5e~;4RzHhzCeJ%IprUXCZOb#SS$8jH-acivD_m*b-ytcJWso_k< zA=Z;$yt5xW*Hl=8d%}1BGhI}YKa<<;;>fRC?ZP-5)8xAR4{`=gb97DPT}4xQEF z_qR#7f}2HL?4APVxYb2$Si+{*d$Vrl z^AJCuhuxJddcS@GJM^8zVUL8S%WqxKnjQl8O9#)(E;v~C z99>YPB)PfeIL>jlz`t|z$N^A+G_OL)d8bO8=4j#3$IjGa(-Lve-JkHg)QU#uoCWUm z0Awy#%T~#{pgVpjeS736Z8*J2ma6-mc#EAS1#9P#@3z)-39IW8MvhGCd9i*#dklxz_a5EWu-eWqS+gv-}z)Nsd$u+8||mzza<{1I`h~8W6Wnb)ae+&r-+n4`7<$uQ}$j z7$&P1fRE5+Gg>(cJiC`bkz*=2<4H_g_>ufv@Dch=Zxe6c9YPnSt)Tm=ztRD%wYc@8 zItc$Y(Q#EdX>1x#ZU%~|WA#&*^u!sZ?WV9Kbp&*ERYTt>hW^ami$E1|-k&UZ*OLk+ zuAD6Bum_ouX+tJPc|rE4FLZ6Y9K|JRq-u>3)k)h&KIVGJK2NEo9|V@|r29J3Fw~d$ z%k|~z0eYk>WD6O-KMqz!c;l11f53gfC*UhX1@D~#YWy~ZgJMg`_=ZT-n0J$ZlWU0N zNhH|eFCx5)raR?(airx@@Lm2Hj@d*|+eE~DCnHfdI-IFA_%5S5-Co@oO?RAnqUjcd&j$lhzGTD2zhg^@8qo>_-Jan;)F43_dIXMZS z;%0`?1~l4YC97~;s4U!iME=AQq4mJH^i2m?dr2cswX&My>>k|s-(YcX$ zMDHtBSj7>m&z5xBTvr^hsG6K8{s~Kh-BDS!3>bwFIA`W8R}7iJOXp6Z;+2PC%D(+* z7}ZB!wyuOuK70Tc?98He^AceBD1@?5p)0#90k;-vK-|_)Y#4Bc`L$~U`jUHCIv#10 zgD+0saRKKW9V55KA0nBLql9bdG_2cl30Ew3p?{iFVC;|vbRX^nCTU}-{CN%)9Yr$v zfgf43JQ4is&yhi6w&JXZtwg2GOk5SR0?yUPaICHxB)oOO#xpAL{I;SzQl%6f z7iCeyy@zGL8mu6AtTjH7NjaN8U|K2Lbvw@2ro_+|Jb`37oYIP zS(}vQM}N8Cv#IVdvveZv`dUCM<_yHSby6y3R#21DO`vqt8WpA~@_iC-lQRu&psJQ7 zt5UIr{Nn3`5Bp0#FZ2O5+gD($*+z)K9-mHLLAJ~=ro2-g)b~F{XJ>fBq(S8{ctR$u zoddFgS_i;$_Y~-x*Gv5;w8Gj{D@@Hcz&!%TIBm9sRxdCm`$m?~<}I#7M__SxR9>X) zi0SYqr~>tT+GQJC6tNGiCuQ#|Wa=9eVfk6vb`agMx(&GAr*FM+dxUL9=uEbhz%E_aHgLJ2?Ryl z_k5R(a~UY{JSoDNo=&h}NfGa^#Us-ZNL7y{(<{bffQ*}i?a|X|)A$fDKNW!nN%L^y zg;;z)Bpe-;N6Cx7D}&vdL-;Z4HLVuB8mW$c_`)bs7CUnqjyJzU#*Z2W&X@k7t%n;# z9nYtS#!E1GjT3s_b-?d;iovF`0AjWmz@fR-q{-=r_`{{=IC)l-gxYa_YxSia-tQP2!k(~q2Z<(Q1-qOf4|iuMp3V5le;5y zdECJ1g?nKRb;h4tc7fZx7uY}H5KP!K4ib(1(K&4d%$#ToHFcNC{pH4T&%#%-#owyo zU;8%xg@-eC?pP1e)~XV(C;R!{my^LDzL_q*TnOQ^8rtXjBL*<%YKo5xUF6;q^qavwh`XSI)ou0Q?8XRv_!Ro3{Vra=agXjPp9k;5CgC5ZFJ03v26exIlB;8; zqS4QO^7gjA=sO`AUn}(B^eG8~`>cc#=WTHRmI|$Inv1DrhVpyg0`Zk+F6fJIqKn%w z7^pQsJ}ypT0* zH~M7j!=CZ4gx2+XE>US?+{|CMH<*eh02T^&0foD@!(aBNl9aLFVfP!H-EC z85`(;#u3xNXvh&97k>@LIi2G3T?RwdluoLZb)MdCO`zjL;vw^l1)LQb(g@oGocnDs zKC$(}2~u6+u#Kg&vh8rm{PE<*_;U1ntOC&kzZ3cO!8oKm8yj}#$`*&*MD?-&&{Wir zAD!Bc>jk4oNaeFZt4@Eot+v2Mrx(R)4s$r14!kbUxOn_WUDqsxg(m-CIVj zMigKoYQvfoF?=CQ#oz8s!PeJpRQ;_vj&4W<1HXeeFA+u zEn@5Oh5noW7nTN(K#lIT;u;TqaEzM?GK)~Sf58@iG%TZT!9s)6aJ|f>>LoMlX|z2i0i_|wz`_U;_^$VZrM%mkTGaAeGMF~ zTaV2?Zg^OG8F_u_3?%IiM*Wl`ar?hF+7A}uPuW)dx6c+Y4ef)!ezl96R=glv zgY)phy)AGgES3~58;1$zljU+*f5~v)cCllV(1w0=0Gb^S!&xweKj}L1mCSeA0ls)x z4Gycw45`YNW{dHBMOBUR>WsjXB^R#oih8LDG15+OieW#!ZIxsYdwn z@^zAFxlA0o$OVR*XArObq3HVWlq@_o9S$d577tL20EH7`R3Ct_%JCvPRU6RtaVz1Y z!gn&+DiquPjlr^myTLR3JNbGg7M#RG;eLQGsXbwe);<3qd~GWoxj7Z?yIAA91qm>C zUNTIcc$13Huf`n@9?~63*<$_XbWl$k3tw6R3-=y`Ph;~?;O!*^S7dN-x-LvLUr7tP zpOY_liimyWc$AJhPj;88!h*vYFtKQmY_z8tefK~Y=UpimcW3KL-o1{6h+-Q_*zLVU zq8|pu&jx`;LodC1+y|GXDZzU+Ey?-uQ(&oc17-(?;29Pv3|~#={LZ5n-c7@V4WGz{ zxEdOs9s_B6)?zc?O&JdlSlF3^{kJ)PVuzZ{)RgA4=BK( z(O+aq|6GVgQwmmBsiBs#54N_8|g_|79y+xDYc z`Z?I4DtM>+B2A_rWjFgA;E!i?i6{QD1^=}S&iLqxS{;|^kF3qO>Bw>%v340QUHKFp zwW8p`SY-&@5C@ZH+u?4LTs$H!qyOa{Ax29lz`my+AXKkI7936BY}9Csaq>dvy&fbA#W;N{t*#Ks)@5o>V&$OO<8np>r>FT;5YMX+0E3qUigAl%$oPL8l? z)H*i~*X4Yt)(H>bk=|drSE++!{OcwT5rz^6p*8Dvpoz9leo2ZS)stLhj#PxIOJrj& z)7+3)G`YEplqhzH@2h{o+U1%gV$4Hm<__WA+4VI$o>2_N zdVaN_*)rn zrsRObdyJrCQ94x3@`RG9W60Q~{v_Z`96YzM$NP!lbk^3BHGaiGNrmCSZR0u=6GiaO83~!8ijk4_1m>#}`L?~5=6HjcXhg9!V+3>ee z{74;BdUe5{u4ZGVM9IFd-irB3N%ZomdBK7+$R@4$m7N!~I{fxSx7^(0ytu{Twh^?lkL{tbN!RYz&Npl@DwYSMQ^RHvOUD z^(OiFi}_&I(HAC*v&4_YeI%ik_Hw1rOboHfCmpLzB$2)C=rBDV_4f#~l3!EsO7uM% z|6e8y&YB`l|0@!2C<-87!@V)MRf12$n(4B0pQw(bktBbvwtTlsHnuk_Kv`suNsDMno{Rj?hx1^cb_pdu zryw=tOEiLn{@FREuR;yU_9)eWW3jbY~gucWu&GKA<(BTH-J z@Y0CBIO6d<`p`#NXtAFr-xp1Tew8tp-*}%^;83{6`Cy259U7`$CO(di(jZ-12wJBi zXCrlJMtB9hxnGWv?YsR3ip%HOAln9;+tTFfFcoZFSMYX<7)O^8fHsuzNO)3MS^-v5xzjPUf-^`Zn z`>l!ZFQ}m&zY-h1&whcVVA7WW@% zfVY;~*iof`maA>V-M21~_v$yqeTD_2{k?tEz&BRhRU?Ob(`;1KHo|zp_kAbzDjmA| z9|_m!ixoNtXk&R_N!Hs-MEB(kJmao`Ird}dGUpZ$?a9O6F#*`uCYOBwahv=;E(VQd zci`%xG%#}Ci*xim!O@`vpN1@wy-J;i_lJgHz@KsWp~e_fWs#&w9EH$!oN5F-q8pUv z(Z?8qk1U^)%U@gRQ!gWlcK>SBm>NoMt~!fP&6kjTvA|%zjG)&CCxenoF!ld%pEcSR z4;x4Hk?X3Zz$$|oeCY8&RR0)GE8kB<{h!YGeCS&^y5K5={yGY2mUZ;Q;dJu)qd$5_ z=s?KQX>j+yxe#c_%UWm}NwQf-9tLQ_c!8_^z#XKS?=NG8m|D=PVE#pFXm0=Q?G?A(K{YQ31AiG>j~d#LJ}$PjZ(TkiUVU&mc=}Jn9eRBw zvSXX5pT!fh>EAb)Bl89ayQ8R~U?um~9|Z+jR;X>OPIHDg;DQn<9qwO;mt*gtNpmvx zPrk<-$d*G@-crH2=?I}K0?;Ki3Z}k|BZmqXLQ}n+Eb3YaHVE_5x%a}s7aH((qM_W! z<~7#t)&bY{T9`1bjO0x7z{h&3Sh?}FEPDBMoPT{Gjc={PVzoWQbh$S9d2bxKx9%&s zbtQ)cOgjd%^Ws2jr4NWQovD%Q98}hA$DNNkeonzb>WT^U{Qv_=!H8}2z^kJqcy=@9 z?$DJCt8Brzb4vJjiv=iIj6<~{%RoCoO>(yQEKGhG38S(+>8q6lebT>^#U?2@!8#Cg z=P7`)?Q^o@Kz}r|{g2#gGr$37M&qY%_ekaAIGTOnBX+4~frtK0aBH|oRd+JPqE{xe zFh=k!IVn5gfwVJ54=l2rKs7uVjy^v@d-~-=&&qgsT{@os)3N~71CPnZ-$|i^nw{i| z6Fs1Cw>mwTCh+KOgeG0C#2GUlP~R0|Fu%ph&2g02LFXMw&A3R@Gc{r8$i7fB{5NKb zO~kD-ccShdK=Mx6QjOrHaP5a1Di56`IJS#%irp1*>$EHR*8eM`?*-}a)of8ZNr4vV5pqA7j3egAb`7zKWo>c zx>6)&$}C|(b}>9{e@-kO>EHnU?YMyDFk$>#xMB1HIYDLzR@71uS!GFP?XXj?e)9Azu?}-y=Q#h@7H)LWze?5We`0qnT}E> zlG)!tOXOClLHlnLm{Rlum-bE@g5R3HeUE)o-CU-Vj8^EWaypVH{|mCBzSBnAfHd} zCJ$XSQ3i>jZ!bK-mnZ7}KTDA>rQJgR5%Wmp0C`wyH5|L@Tj7wZ0nEv3rJ}K|L~dar zbR4dNpRN+2e^fd63~ZF#xu_>Rw!H+jugTumSC$}easm%NULx8gw-I!`m7wzGV~)RO zfN2K~V^*>u+=)7bBjshUweU62mU~>L*ZAObb2m(kh-AwmMvKhND$w+sHpU0J!ccw| zauYq#;6SMK)q&&G%1u#Jdqzui{n2sSmnX}nhC0EL1~1{lpjDFBO^VWTLpw^6&*II; zBlsVI3&CjRS=qcQV>B2T z>Yj3M$&GA75MTk!B&0_Ixo!niy;(J3zlsJMTP?$CQrs)p{BG>o1{ zcKlrj(%y%ASep`Fuf+5?U&c$5#K-aAWQytSi*OY@2_? zB%ujn6WfJP-p>JAz^C>|99D+K9EwbVU8%9}9Il^I+844B+{ zn$q753gd3k_l3dW>U|isWg1E6Lu;%IDyQ9{o}?q}sPOsEe`LPxI;i)aL56f3!2?Vy zEP0tu1Li1^X>Hf=LhNxITl$mg`yHn01v{wS#!*rqBiYWI)rP~WhLAP55S@Y;*umhP=75EeeVPp-Bsg2-7% z_>q6@;dJXt682a2z4s2nI|Vy%gxp%%60``f&1j}7{}q9X;!>#2QIRSvI)tviR+!LI zEL0du+{f3j2pp|*H&A|wAiDRjXVN}6uwX}T}z+kl+*e`ZSWTtL*5E6 zDB3Q9Sq}v8>!E-gy7rn3bdZq9AOBH7fCldIc|e*jh;V035!JhO0%~8~f=$h5;OYHq z!doNP;IuLZ%{x}$i~;HB{C7Utuy~EcHSq!d^bW?=y~%h+&jvz+l_A`^k`zCfPiH1N zKxNfgDA?CRHD+7j*bj+d{WcMk+#+eNXBzAY-X_srJ{#IcO(crfZVI!Swb9c-j0TPY z7;QBg107d`>Apv3eYFy8!wd1@#-p%yX*68_xD)8rP@?c(6%HSjMRRgRfZP0Vik=?M z-oFKxkIbM^=X2=SDKT`5lZ=n@^##4AH6Cxf(}I@M z^y0G7upL%nRH738w@3{uROiqvPk-SxeKFmur;9r`$oNQi6X1hFHr~{eWw~~{!-HHw3wW_+&FxJ9rgO#@V9JVG(Gq4yO-%O~{vm23VeXU$TAg6?BZ>MpOhY zF!7YB=wq=a)crV$Hm)bBx1*k9?Cj~}*c+Y{-L{5z2Gvy8@-12Eu?C#icfq?TIcQ}o zB_FyP@XPXK{O_m&E-$?S$-{Txj{CW=sdx`Ge=H`cM*@Y4Yk1IG&sZ zKU%!9B_k_yB`trF;L%5^P`v*oxcbcji{xkYUup&vO^C(RiZ~pYW)JGI2;Yl!A@W)e z`7pkcN-l4O-R(!PeU27{HQ9(9w#@<8wXs-vG?BKhbEKEskAvQ-Lhy)cgsav1xckdE z$nG|w6U;|I*^NQS>eo{Z5OQZHYHBSa zGxv`c1qVKs$mfj2UW?=SW#?ctotjIAR%?h1S}gJ4th@N|qX^P(#*_7za`2$tG5pmL zL67(;fR2ZZ6IYvq(+=%LyT4KNZt8Fn{l1HGj|;JO>_5_m(y9Fo zbd=>5VjRCyxQy)phn7Nk$iJ6ZXxc*RoRQMwzo&p;ls>GzZ9p2^4M8Q_7~{I`0=_s1 zG?|k0@mI*<>w&PYBbiJen@I<*h=$3>Gr%k10=YgRMKXW16H(W+0?+i}B*yO&D7;sI z$8|2W;hZD7sL0Xy+(Wc-@pMprstCGQ$KdkY_xYc}6L7{@EgXF|4Hw@XN+eUJ;Fov# zu(;S5cb>dMnvQS4ui;AYZ(JiaR-7Q|KOjY|BkEWm-blCiji-l}EJS1P6Ywa#7`7$Y zlJL8&Ol8n5I(Ms#g)-?QCWvxDIO2k&O8yo0**cMggEGC*v6%F5IpAudD536Fl3rg6 zT+{CdVVdWlvHKjVJ}4!=VRPtfNgM`Uv%`%FgTQi~oA8R$PFN9n6x$`|;koxg$XIog(vfyxY?ZPYtX1 z+A%F8b<8>v;c*^|ZJf~{su1sJpT)=Z+4R)ZS~6j|D`~v_o~rM2r?^xT6J94_zxE&E z(QO4Ot7P7kx$Web=o!h`cMmr-#-X>|Q-U`~;uiPq)Wf5Vx?Ihmtp}e;7RmfZ_v@}p zhV3mu{5ua8r3j&UD?pK|}Hw(_dK0Xx` z_w|z_S5Cv+xan{vA{HzZ4hSzU7SO!%<4kXqM#rIuj3Pmmg|Mh5pJZ4q6G}GgKxBmkiUKxa!GQvd zQdmX497zF#2NkGi^bu2b?Z7nYC3HMrhB22K$x(3~9^0%TGP+hor&{kPqpWjj_01;X zh1*e>y)zh3o!&xb_h``&btlQd>r3f@Cx7Ug8)al<2?H;WjUr;%ZZ-1#MDS1@gFgG` z30LVQQfuLDux)8UwIZp|WMmz>-+e%`VoV`n?+fa8coc4%oGO{pc#67Bo&nm+B7_4r zTY^bwBrPc#ic3p1V0fUKXypb)oTOMnon|IO)j1=|U5kb<(Nkd1+Xhq=x00#hoaC&n z8f3OFMh}6A?Atey&PrQ~x13dlYYyMQqVSJc_3pl8&IKpfD%(lco%AF@ujNHKPu*d( z*MCCGKPlKDI0(;P4;LMOvKCU3hk@DJWO&+^FG&}+fm{A4NSQCd>8?%WzVSp18N~?q ztH!~;;G3{u-%5CSah&i&uQA47K1FQap22^X4aC&a289~RQlmhEcg9{M8ZLU08pAQ>THV97!@wi3i6)J;b60$K34!k%5 z(!LnFVR#1dSaA*wL)Hk#ANfVSFD)iy&m8jM#B!Y3H3O_REvL03ePrEu4|GhCeB z02^9AVPujv>3Jm6A7i6%hWrqcws8qaJ}F9Unx^9bi&yyLy%(yLzQ$5>ckp^!MYc3s z(rcZ*H0XOXFF&J)JUZ%vvhE_7TDqD3*}50BgVw^Jb|JQ^o5I-sBRDZ>6TZ2cjq?Nd zL3E$JDC+eraCVz1V&`ARic~ecG4CE079WSl%oVU1Iu-H+Z-m?LFCbG4uR-pfEkv>P zEt%+71;5S|K-qtKQvW%(>C-zg;I;k~vHVj^2Q^VpkXMj|Y^`vRS#KZj!;QH*R;)A&682)L**S@C&3bgcI~!vTA0xrHTQM)al2p7~iN^NcSY7=FpEV@X zm}ZdZZ5smDnv+md_prj(}moT%V-{ckF?0{lc0fp)GGHD4nF*n z9#yR*vD>so{8)63f^D_#u0Wj?jfmu6A>oD|Gcu*CH5(HP>r9eZ~_ z#9@-Lc(`m3j?X%QTqrMNs_E3x1~Fk`Tw=c(zT2v<@jIw{o^am9K`#hT&<%Fp4wp z-jmgN7LXC?6QuuDo1^GMFf@_Lv@tuCPFm%H6JI`nZ9c8Y1l53ClomGF)qvJUiRAgX zuW&4G8>p`tD|I_P1BHu&aBE;El&ZZY4VM9~El`&(8F`2<@F~KDqfOz}L7 z=;Zxk+N@xL2U2S#U-M;*K|?M0e8-+u2|X)GOHGAW<-^g=_8@-kFosEmB5*o2T>A9a z3e2521iv)h6|VgqK@yiPAetT;bi@1r41PNZN3KzWAMF>(Sl$ebX2{ru4KGP||7fsz z)k}KzpQYU%e`tee44iuIirPLspzv!dnww4#23U=vp_UWTzI!QXudKn3Tg_nbPg_xy z;b>U1b_T{LvaILb!7wEu5;CBSXx%*^^I7kKH!i)x-VJRM`&asKWk3t@eP}|VP#^#7 zuM{4sy(A33JxEk%o<%-IWuV|t2D>hEF^-n)SskYxfK)yKU6v+;OQsJg_fo_0-SznB z>}in6iJ<#*HB5+oPUf#=(a|7J_}Tz*(m5$P`lbm6E2@(1mZo%Msy%F+cMB~{W1!lt z49aePCAZVdQD=OXWW_LT^7+zDcs%Zau=z-;u#3*3gYSQ*-;19}UOtsWb*mVdY14{B zUc3`-db1v;(cPp{)^{?t$O?B1)y8DZfu%Bb@u~JAST#=*=W4FSz?ykv#XfmlbVa5M z{*0t`b(^qp;%yjoN>`d**hy+8Xo2*`8S2T#fQ54-Y23C9?1T3}U~)DToD1SZQ^i#M z(0`P7cf`F+8A+L*C;20q`dmTU_)m30?QAPZ}A=o}@X zc+y{2-tfaQ4O_PbT-K}NKy4>*v5tos^qjdSj8yFB%2MczdAqA2hlKF{d z@Y?k%2Ju!8Abz)KLGuPh=^p=a zlIf`&Y1;6YoQbOdmT!QILycfN;-x~Z%%iCC8zR9=BGv+@}41!^9I!zT9ZItNp$PWAd%*b0nz{mU2yBkffHxb=!>O}SRc_!#eowLW(C5Jh!jx! zHVQ{L946=Hj=}Nt54;{@O(yHQ;N>7^UNp4??#TZCU?XLzR{9)pG+qnwE4}dM#8@&W z`4RE;jzDf>4yrfqCVj_V;En$*a8`gOdGHq(NjtJLZ!j)$m4Js$5rVtZ7hpIANwE(^YFtLB~OHJ3cAsLiW}YY zObHAc4`cR@hj94qP?2iTT5_WQ3SAN}>$Lk~3kF6Ta7NZ8G{2n(YEOoX5`ARC)Wjv^ z##8~izW2fSvGO?aXeGuP*CI_z0||3TSofoj=x2)P;R;ZapZ72?HoQ3PPIgJg+YD*L4X^Upqp&wn4$zK9@b^t=py3hN*)d@{Zk3&C~(s%~KY9nF0ssl_@y-b=9 zB%{t$dC`vhL!{>pTEV9*6QS_+NiyQ|cTzX@Grf0Q6~=oM(z55{$#q9DRIF>Immleh zE>8g1W9^2Elr>1*wx76P48-JHHr}aIkbdwPjcd6GnHRzy<74u#i+y6|uII_l2-LaW*kc#v-lw|=cBeRq6$J!c6H{B42W+uSJE zc@Xaf%!S|QeDGbl4w{{hplYCjE}Kq5Xp)!km*#MoS+g55^8BGAq6z{Iz68~RWkAH4 zw9Lwwn5mC}81hz9Iz&!-ZnPCVTGUAo9!da{);pwR?qvAh^NKWV9F9r9e9=WoAnSVa zl+KkgK6~{R<7TC4px}^5XU88W12dLkY<(9^vt5oMzUuf`nhP1IPt1NDp~0^0Y_hg7 z>|FGUj6MHB;wbBMtC3rV4}J#HmFq6x+E?Ci^yxcdtm-Jd>>7#e(lFfE@tZEcA;8k7 ze{p%aH;#Uu$zBMopsRJXi)Az({ggSpM1)JwA>I=YZ8=Qm_^uIlF1&;j9xbDF zGVk!Zme;J)P6q6btBX`8&gDyNKhSgh8L-t+0;g>maF2Ya!`r01^TQBO&9^1T@_9P* zi#NIkzo2HuCtyj+V)*+^6T^#BQLDQJb2V+?`ClJM7^;TO^S0r*Q3<$Dr;@%q7(y0p z|3Jr-ucZ|iO2BVQEndvHfy<2#kNVE@Zysy z*?-#yevTUi{NM<9@na?qm-)~xpEx3H|ND;`pL|HBhb|*u>z|Vu-dSiV^D=c0U5C!% zQmisO2eSv9qrIMn5O5@vs_m|V?XsTtSMQsJ#py@j+vGAyx2GC(%#?XAF9y&7=@*3m zg$Cll>6_8y0@9y4(_t>tis?CJ5;j;770%3&^~u(w+T7JtD>)Xn1l=T$t+l{zhE%fR z@d&t*GaEd@mXPEXiTLUIB+$6yC5hxtQlq1NKowS>zHXQBsE{r3YTLSf?iSSA{hIk*q>6>KPwTp)( zlLtBp^;+eDU0*_ef9fJD>b}FiY564hMi4HO&Hn2}*U6pCGbA`XR^rfQf;y6kRO)pL zP5l~amG3=SS9v1x9riHrY6qCr3nj%X#*4lg41_xoGPe6^0{T;vD(c77yGu+qb2_7}jsH4O&WWXLji3M5CYulM;i$r4 zuvt19!MK|k>b%FB`>iP#;ffjZ2T`-UpI*A)foERqHGBwFf*8$0oW0xt-K+{|!>kz4TI0+7P}0V; zX*$xCky79ee5Ld4V@Y=5DlA!W1XoV&p&Pe-qncN`F-+qV&2~t{6~Fe-{xd2hGp2??B}K)57Bic-o6TO=isQ##N&?FjB0y@q0gA^7}WE9@(+mUPPcMk@xTKvzsIuKv)5TH1{? zdFFp4`PyLV$s|wG`7KgXBVR1jyUXd)_5u>)_FU*Sb}XpAYZ1OXb_Di3+)o}1xq*T! zM`SZrAUSp5Iw*?LNOkQD{?g3T7#rdbd#5_!vk!yOcF!Tmcxr_13#Q@yUuR+Bxc6|s zvbtZRpByPB0Y5Ei;V+>uLdH2BE}f0DqZfeLDjjKp`F>d* zYc^@pbih8r5Aseci)baD#DGWM_tsPr|0tsf0{#7xMoAb{vc%;(tqJ*IBnDHo{P(8WxVP20W!N ziuMaNvUH^bODbd;4_!bf**vgpB(Nh63fZuev^n!QRd2VZQ=MaI_~P;8pY}Fk_1g<% zyXd}9>%|z@Ir|`hX%sujHUM9@T?WH9*#tv>kr87?lc(x6c>eJ)k|n2=aoP!Z{vb$_c`F*5vivdY+7n#6N?E#W&<>&c-j8I%7)9bvreNB|9XRrCq>ROJ z5Y}~0Ak(BhB*s|vFBjik9+*p{9R<`^{x2da@lOYD++wB7ty))r{K?tRA|3+ zNZ4_6HALP!#~W?xLbmuYmOkE31M(H5)*8XE>&{kWOLd@LYy)?VlTg`f5`?%PhRL~V z_+<1z+%)3~+D~xDig+91k%Uy?hckwK_W|aGJF4$7Cn;T~&~1E@Oz;(v^i>C7Lf~)m z*UUqBY5idQpqWR0tZyRhNC&+4IuewA=hEB*_uyQKFOJK!!)t{jF=Wgx%<_o>tp@=3 zD1yEmdylqEr~{=-4CEYIM)IRN=%CADsBa$x7fK$ZZD%eRs3y}c^+Xt1KOZ+gIRynj zmf)Br!=cI249w|JVfKUz^kMrJP}2yYs)J?P*F;rmzT6Vw`Fb6ERcnDc2Ts$5KS6ZX zt^uH)GaQDU_Q$JJSXg)L8=W-Y0yjlHlbA(C@^w?{=(56H2q%ikdCOtM$y1)5*3P2` zbT{CyCQGsVja;0brw9p|Y2#$wE5d-}uLu*SOir)XmWwP`5)2{BX0V^a3M}b z8Z;~&!LXV<)cZ-a$sc0m&R}bm9~5jkfYxqrC3_x~pxUMyq1M_pVBI?uKeWl7qYrFl zyUpz|(yJJEepVNS$udmqmy7YolUF!5{2Q`HWmsXgA9proLT+p@A;rS8k(M;^ zqdi*RU4;w2+ktI@jB_&go6w|B_L<-0>5U1mz`!w!^xsm%7thRa{u*sz*t+@n@38?4 zj(Nv7-#D#oy#v2G7bvO z%xJ@E?v?*Wrtql-KVeydxUg;sch@1CdpPU}Gx?H`D>#+Tn5&%Prp%ke$hP2uX8a~z zy~&PS61I*xSES7BHgynt^cZkI<*PXjr{kQ~-k*Z@ic2n7{ECU6DC=oc?Bmk&y#$qm zR&fUxr3(~_O_<4HNF zL%0(?5zHPVd#*hAsGzytfVp;dD6`>_5BHK9yDYi1gK>P+%V_R3;|u+_igOZWR6Dd1 zE4VIVZh4RA<~jEZ0tcLOR{s*l?8-b*J$}v1>ya6Wm_O z&0XQjJTcnMeV*UONY2O_ZhRk$b4ELg2OhY`&03qsOj#W$E;x2lOypF!mR&lG;M#g_ z--{J4i~hagQX{@H^-tskkG7|a-`_G|CLB!_|DAJ?i8!drjhyk48P{pae_yzoi9N8D zvm5kIa4vd|;OqP_#*gK>6WhuKBX;cABwASLH}`~>0y?16BS2%P3 zH2w(Y%nReZoewiA_J+)ir)tcwzmLQ&6TgZr=PC#)`?AGD%+$pLRy`9edL1oL_>#r+ zJku3?7~;fjl<0H+uE;U6<_U5CR$pfB8_Ko4Y-0>gjo?yh<4Ri1 z*!p&cDU|Qv?j5b*AgGNgvaMhgXX`$w+!R48+6dzGMj z#s#sFD1ecEnp_jqFq_kwl_IVUtmX{5>X`?AR@~;d+05l*He$nU4?)x_Ul*hMJ>sIs zTER)<$AZ6Zi^Vtg%yH?OUdUpmnTHd6<=lUu6CJpS{lrgFw5X8Lk-m)5!qg375!#g;bZj-$f-xks^+ z8K;j@?viIIV|>n>v(34{jR`B_eyeJ;wKmf@N%$^i(>LIbCf}}^($&O$ck5wv4T88n z&3f_u=NGsd1!ouEs8>wGpY?(%Gt~LW6PC>9S#g4gO$%AGAWdE|`aRQjW&v~j)m5fS zu9!QQ_L6a4Xw4lc59E&hP3M+PR^-Nwa1{Hs%dzv+T=WA#C*y56gehKK!ntlKWO8;)7r&SNpBI&-oaO^p@sqWG1i7M7?1R>e+@d|X%o)cl z!MymzHGgNF5$m~Yx%}Mqka_axli;`C22Q>}c5QSE#oC2p<^)|PP%Z8d%-g5Rg&M9G z->9i_9zG9D&y3R?-n=RP>-~vaBd^EGL6?7S6*+g`K@whRDY?)F!SGl5}Tm1bS z=kjR~ze8D<9buR$)-jdigNs`^%l^UaH_KQK4YqJEpWE{HCq*+B+kF_*qQ$Iu$PMmf zx>-$_x3h~NWG@q-=*W8>wPxQ(9Tpd;W-=*Ht7_b)M>1x$j?8DLfi8^u9%kZ!Q`~<0 zk4)b0@8XyS75=`ZCVSv|Cdc>AXBs@7Gi5t$c+u4qX4xuFM(+1BZgmO6oYZb$=3KkS zY^VtpqDsEqO}@r+_wYX^cJx2S zZn`<={CuwX2vL&mY{m%nn1+j&8rSe2i=HMd4D$CCeI zXTT~6niwt90xrMyEVH3v1oQmZS*EvgCu6m2tILHs9?Z>xy`0_Ev5d=XbMc6ARW?iTne*s_V=P5I}W54n`ED!fwy%b!}D%rFf}Vy#=tnUP7O z1P^xZcTwGOlG`$3i+EbxJbsGO0Cp?J2{yd5W6d=k*_A$nxy|L*xIgh8+{p!>IkN&i zws?Czw=m>76F)|uX&yK3`eV6Yt z`4<(r3CAp$DF+Agvz*^EQ~Ev#zGeD4mgTxLGq3*RG~Xq2t9$=2v7Tw%=x;a0`BSHh z147!F&a89}?Y}V!(RGYORfjL+UNaLa_4!N@;YlxLLdTBf-aZUxtWWtdH;3^|#-sqw zHtdMl{=Admo__%68|25iE>L6lm|kIK$qnXj-QpSdl?HsWn;P%3WFg~W*~rbFZNx_L z8f?hj0&d`#O^i8caK0B4Ijbp!+~IB&UQK-ht4S=_CiabBs8ol&qb1`79v;D8>C$9_ z?aWxaFGhTOyDh(c=XWMKO`E-wJDeY!r_XPg_M2Iq*UXI2HRAi%Yx1?KRm|^g{_LEU zzC3TTjNf`%n>Et5;^U8+vd6)I9p(Ot%N=jT$9=V6M@}|km+zM2pIQuI`*gMW0I4?b z;ik%_tSMtcOnSK2q4&9l9VJZ8sLxE$o{9W87{hCIX9z6*MsN${Co=-uU}n(kP%b32 z-znvK4;Mf4Bo`j_llflY$6GW4JN3wXHqk_bKX5FiCfw`}WBydayqG$h^AV}?b+uAP z>4yWWJ*h*GD@0~W^$?!?R^tOY4%Sp!)p3!B^x4ndYOIUL9qx0xtjj!p9GfZc&L*@x z%t6~^G z%}kp;m_3l4B3j2CH}>vpS1;Q!-az8#kE05~RzfOdiVa{Pj!5Mc5)xb4PjAI&RjTa>JXK{n7w{u?-wAtfPot)%}J}Xq#V8@O4 z%e_Bwj;YAd;UyZsxqnxc*=R-G8rMF1c1B7wQ#Hek8+_<4clWCiTd-Y=ZCKOEE#LN( zJ68OT+1>br37px(IjUb^_6<_!cOHMqiStf!doSp)YRLonq*!HMG3^19lJ=G>S!u|A z`J=|t0m}S>_m*6SuM+3-%(-S?>{#*HU)9W_WqSPSKXUxy#W_sTg(|_k`fH5h`5{bZ z%v~;(`Xe#>$P_TQO1T<3>Tf;S5WiLXs=7QFG5Feb0uxbRRdwr*WNqfuKRevWP8I80^c z{G7xb4WG`GwaIvUA@5D@8mkYT|@OHms%YIPUs(D?UiJ5B+Ac zo6VHQa&G&89cbsqzEhvap9`;NvSeK6(-j8%t11yY&H6c`EkBQ0?|qVav|UBKIcsmt zS04@5v2-Y#GCq)BlxD^!$~f|$hO%tU0vG<`f!oYK9ZUY#v&mezf(`qmK3m{pXTm0Y z9nD^x&hSnf&N)kmy7DO}7w~RlXR)IU*D+o(jocucU~W^<7e>L&lr2jg!~fmn#DBl( z#O{7}oAYs%WBZ@pVNyIt@b@b&abMPtX9q2CVJ8|-=O5)R=3Fu-@(ZR+<6ZLo*s-O^ z?8q9(XIhx>eO>0PPRtUlu44caySRBVMEihurea>>PvI-gdu^G(0dyPzZKV=j% zRM;HOlMmQ8l&$I=&BuhB@s|(HV28(5aGO2K1>a0N#92NG;;aC3cKv)a{&veyUj3{d zdt!zfe`9S?b*9}pvBq{0b4u|oGxeS`|2LiGFOLzjk9W=Go|U-rt)rCr5g9`E#KPCy zyQvGslUL^pO3t=3rBV9qg{z7@4$K@0_d2tyzcB2e3IX50uZ@xMhIn4( z6Q|#?gq@*S#x(C)%NY%hsM$p#1>drcGX^7u@h5Kgaa$fburIUK`L?CQ*uXno%=JTB z{QZ*un9#T{+*e0sJ~r8e{Ywn^8B6q8`D-1{=gUS4HoQ&X=FC3Dgt&W&&&uE82EUDE zx`zmld+M49bSAH~)zb>RP-{fyaSb$|=g z-6t-ereD+cM1j}vb!8LRmNH&y>jmuiZm!POnT`C(vSa#%eD#fDE@TA5{xWrBf9`PQ z?^KNA2XAv=BA=VMY~Ae4>WVe_wx1U4lia^dPQy$dR#bDp`zRv zr86J2j^*FoVA$M@QOt~&6Zz0xBHnv*DBE#gt!B^44b0Q3V@#WuEI1&0$sH}wEY*UOq-+xP!o%BeLH;^$iGUIL7S<@6)Q+-2T^I!_M!eAl0 z&Z3h$+NsL2707(>8P11zYq5sQRrte0PYYs3$gyPCJ!XuTEq?_w>&&AN$OCNu3oxV#E}d3wSI(?B&82>=5t`z8>r< z6GhhG*KclhG3Az~%Cq01M~Ypdyg1#Y0slsr^#HTi6*-subkUea*6Y-tdn%gfNz;@_y z+@J8NT%EZpzkRe+FmpmHCrsYLoM;JShFYv)uFKmnuMhQ!Lrsggxt%W=n^Xb6@VRnz z=jQ*Iha*pkeK&pJJSI0X#50+>7HPx&3!ln=DNJW_F39t%L;f+MH3RtT>-RE;Pk$Fo zDURWG{?{XJNef{}?`@{zu?M3$K96~NDUEx5zJOcuz=l1#eUz2Tq5zASTGu#~GcxKHk+1!`;i7}Ei3X~&s*k%9za$m}Z zGl%NBYsNVYWg^4_`CmmBnd9ml!>+i(JulVcoc1o`?zz`8>GdBNjjpfUVQ-!pe_|Jt z@puAnsc6QIX^rFT^GrG8_aDUjgH)MVOFu5F*T1SVbPVHgP?;^N4&Zi;tCMy0KVY70 zi)W^UTX6q8K5_#e269KvbTNLxPsM!KO0GS9gG<{befB{X$8puIT<#HheskIY_Hdh( zi$`5!&CBJ$M6OS-&e*<%Gl@GTo|hRZs1#K(7j{%~7dxuBU8y#_P01f7FIJE9EE~cO znJs4A=e*z+kIG;!?CBPmtb5LU>`Z0)eiN?a-*YZGienxYx$q?yotTed@0e2e93lQnla$5p-zsi}x(d3#l5_HiuXV@5v_ zv>rIi>(GFR;rm?g|@X6Ls7d|>@=W`f-&#^1V#(Fl_WI0AIr}W< z{Db52#CI7ZRhIvB{H|ba{tjlbPXyR0O5-pa`J0+HSYqFg(bIFyTRh+o- zID;E2xDR>jxZaQz;@+<(n1wGcaJRmu3cxx{&{rET)@{gT`b1`&&bpmkp6z8OE zzi+_?9noh`JYUCU9=*Y=J#~$nni|L5scwd0j=QL?*H7BDV?Xq#>EhmhM!4j&He>`# zA>@&kD9O^FI$bverDq$^KKlsX^F2ws_hyk(bbzqJCXc-Jj}rD_|}HQl?= zA8xKYh+fmj&_&q_qS^s|!jbQDp}%A+(YR@X3yd9L_w@yMB(gzr%eWu2N}_3i=|_agUFZqJ+s8qp%4MRpJ(tAY^u+Ih zcO(->9ma~!cVX`Fl_Y8J7tGp~MXDk%f?Dw?(fX&2C~SR7(maDD1O63~2V0VGWVsPu zI&mLk#;uZEe;!4e!()V%6(8xg8#%C|{1{y>)5yKOM&qq%N}{xXZS>{jT=@HCh}3=G zBQifJMk01^km)s+q-+00QsUJGxk)*=(5eUIKWTtCz7W>feI+A97C|OG1oC?4$j#@a zq;!iVH2H7F?I9}C!uCaw)|`h~xSx2=R1kfh+Xu>hw&18X1_qY{z!<$+I~aP4ntYOdzK>_6&Zn^z7DO*GsfPjLqtgvf^mXZ z2YM_TDSVbwA*_1+nrLr7hL#;)$nDDekRwULUBfCNZ?V0kYt%K+dKgCzu9QRipOq+m zcRrkd6A$jGH^Db=4UUm@irSBukC5;ZuT38Yp*5p0`}YarXON6V*3R^TjT7x$762YmDT`Pd&`h@xj~`weU826RuFB5V`aM^%`%9&39!z z1dJv$tGtEUt6|^}Fqo=Mxk>bAU#1pm;kZS$3wPe{B}GM}L=kZ#r2E5aq3?PJ8Loa3 zSDS8>brM~r*@Jb!|KTEV`f&nBotj3TuDC>u78cRqb-H9jBck^o7f^IBg3tQbN!>|J zQAlqV3?|3e0dD38IBWv27=02Zx z11J2esOpVCSmo)C`t$ChkH;}uo}3PEcIx1Is|R$_oEMUa9Y^qMYaO%@Gn%#Akd>&|p6IJ{0r$Y>kk#VeLku~ZgQ{q%4E9C)oSG04k;@#AfAfvXLKG>pJO`@M)+-Vo{J z7_sEpykTJfB#|ts&V)Sq-O%`U7^V*T0w==U=#91*Y#n-z{k;7q-rZsbY4`{&0y~NP zeicYMnJUql|CpGLD}da%W0b#QjY~exgfr=*F?eb|uDrWl>U!42PY>YP`WcXavWvSwHat+cx=>CJVe zK9|zLrNa#l`sLEGwijS$;Ux^{%El+9$G~r&9|jJ6DH)I+h2^t(0uD7YPtpLge#s`V zQgIDuis1>YV@eohe4;4Aui>MU+AWhVQqQe|?XY%&k*My{jxrVCnQWg9j+dDL8C8)pq*w6l(r?|MBljTfGBsTWaFDX|6}OP!(#g4IGk4PX;EoU+C++$d+yvb zmQ*B3MTzW$to^dK@B1Q4i&P4Q60P^#xo2bvC5m=!5=De8A=PjGn`fS9{+MU(oOAE_ zem?I@l}0K+uv-LP4>N&fh9q@AZZo{~pa5%VnP5}XgD{5?bk_g!*{ShF%*5}2GWKdv z>*8&!G_i=k{l*n4D zNH?eA6B7(i^j!yR5OkW11$+FWouAp>8`p7)vkKpR^dj^*EsFp4uEB1N@=)WD5YCU{ zQU|!}aI%x2cep5)6XoH7OYINf;0al*;Hb-WusTm&EMcI7=XP8nSoe~D%Ca-2wee8a zJ*xGl6HZxin_@!#!glu&cA>Kk+2(VJRnmD2`?9-W%)(T*fSV@h*m5{&yPWWxSL=9p z{*_Y&lLvY6NRGSdwF?XqT7f^KNk(V3HMTzRANAg4KXkhvh-o_bF zc}6KUc7ego6Q#Ud`4X7l^ng8gI~XGWTX5kQCA{Qi9ft}o=2iUkg#O8L{IW+1Q1nDO zRu<&MpD0xEypNjU4V%>YMjg#?cFbi~E3cS3{NpU8nWqI$lrG2LHfP`wp}lbPnrvur z+a4OD#eytu3!C{zl(PT)gfq|L3T|=hgb~K0ti(M-+>$SI84Mb_4;`ZwvFq@CYN+W1)g;&tOAABbo#Z?C zzZ13irP+3@e@hb%vje=%C2!#`r4sn|<~5G^T0t%=C>NfzX<+Ai%*OSqS*+3Z8T^87 zWxm1C6>_q}pIZ3c2XESB%ygWL=6w^mh=WVBu#3z|>X+VHD#^qezu)fxAIQp4bGC=E zuGIt3`xk)EW?aOB>$KqS^_Fn(f(>PO@ITmcHiHfEJHbA3ZD(S&Z1~$fNPIQT0_(ea z;+9nc9^UqYSj#t`dM;EUn7^-N(%N3&*9JF}OtBI|YF<=9$G{^($*dHQ76i@DSy)%Eb@ zwJ@lEU6Ptn=MHtuC#ih}cc6P`0?xH6hWk1iD6jQ9`Gb3};oQDcQ0G}AOfMnuzSSF< z`GTH@h|@{Dj^tO7Mqtur>u`u>;KJ1ZA zfiG@cf{6#UVPIK0Z0s(;p7Z0O$d1?e)`?sA{gxaoV(P>`>&%AQb?w;il@hgXWjTDH zl!Fw7c2TsT@4ct8gcT6r;Gfz(IAQ51CAP8wmakOe$|yd>W_L|tl|~Jxf8HASujd-} z-YJhaU6=z`aAf$~60&gI%p}|){RNh9HpK0dQ=IiB<&<%2tblJ-NjWZ1~JbNG7@AHS&!x!=wtkU%P>StPsZD9-bI*3n{msx4xp!_xXJz80Ln-+EjC1(kTWg%y@R1|;ql~R` z?8j0%k11NRlIreQ2s@uo!)ilAxM|T@j!5ECSZnkUFZo@GHzX})w_K3MQ&BE(>t7)p zaB`eg7UYm_@%&(sUJ^WLoDZKBx3T5ws{A#Xe7K@n4K^ow;LPqhuu04xFK7~Eq>ImE z)8tuPTCk7!+`Ix`|LEmJjNjsX^RI#(?YgidtN5`iDo-Fc8Uw}!4^^eh=woJ z4_yT3ahBkXKG|%xL>=V5zm9KJD)8-PXJa~2jw^Mx1+F_O$mIySc%oY@`DO3^Qgg1{ z#9trJ<-2Tc#CCgT!#VYWuGJrJ7%%9Q$~C7@&xCfsWWhbYaC>;`_Negx ztSZ8*+h=f_vj!=TAyWE0n_Zd^YV&5l&BXhuRX7+|*YyxB*;GY<9sH z*KN(9j83WJ@~yHwE$aeGF4YYeZ~Q_1RyYIc1G{-K%htl{b=K6Ytk~O zV}q~gIj}XGlX$&RHTXF-1Pkl-Q?4OVkXbIzl~Adqs#oRXxaoRojjIv#dA6IWHkb`e1M9Y;&4 zYH~Jq{TGUTIym_3e}-_gP8f9EISBvq=2CxQ9jlvejze?H@TJ4~@ao~6u&Xf|4-ajJ zX`9O-b-5mo{3~Vy$}-^bNgs;LqHy6-75+wxbX>W-f0n?|qz~E_jx1+Dt7!Z%E~Yyy6%i7sU_MBdMhk(%ccZGAetw0d8yhC%7Iv zaLA1wK{qg(T^M+o)!60(hu@jOg)I*3HI+D+HyQ)$Q)6N18#ikE1wPAcyd-!%$cnAj zhpT!c@hk@`Zh=G!zS8%FGe>I%y!t*44=bzS(x;*LtdR>gc^QSzW#q%=PqVSz#x_d1 z%^8ojQ*hrESIRT~2~0Q;1rK+GLx*=}oUDD1SbmZRdo`qjJa!G?fk#(4Szj+>vy(S* zMu{=yziOHZdu@QX&^0jj%@}L(CYgFI`it#FX?SQr@VvKY5tdX)$I1=$>}eTkDu$6~ z`F6u>PwxWu^P?Rw+N1(k51O!VY((LV1Q|;FiUd63Hiql6^{GAE2uun3PMRDYN|~39 zcb!>bvImZY2+lLq1WxfxT;-|*s|A_w{pB_=eCRpbx;uqRbo8a3 zJO0LumH^Ajyk%`P!b#z{EGp=w6&u!P&6b*T@t1YJc)$J}=#};g>YSc{{47yu_)Z+B zs!?!hlr%f{pb6wYEMm70-QwuJZex=Ur*pE!HTXr#3s~!QVfd&*8@@fxqXredaf@LN z&$ud!GTD5AC-Trhz}XPSw_B6&oo5!9FeY%NT@;=_1~_)-N;t3lo7tmtmccm+9N4XW z47Z!gK=Q8K z2fgv6JmY&Q7`W(;q35N-;6}4Lb8XF1P(S^c@UAdp@YhTv)RzP1`|n3FDqEQLe|wph zuN%lyzciVM^ZS6|Egw`KrMa*J^EORcK%id13!!eS)JLWP1jm_{Gc%tpt6H-OZ^E> z1fQbWk4qUb6&f9!Jp#;Pq{!ZHVMuMKF7qLz5xFmOWjtfgk`EliQPBYpMs5Hwvh|^) zMDRo6#E1vEbYm#_?x-%auIeSiMMC7x!;R?PYEdTs-cz8mUlu(bH6UfYa)>Vqs-XMY zUwTEH5M3>S!D%}KX6sR5X0`KG)Sl!?F4%L&hK;QROOpveIa;EMRDU#(x1IS{n}t%0 z_mC+_Ta5>CRk=4X#1{gDOiT<(UCi8wl2wr#F1V0K(eC+z^Q&P^ezG^>q!$u zdTW7Ix&kRLx0K;5)+Il6h0&@mqri8%fNtvVL~3?xn4SG5jN^kq^7@1#n!>-(BZK)& z=C;M;AsG=Sse=U(bvr?nss=bN#RC6HZD#vdz=+2$A=jF%LdDck@I_Mx<;LCsA3u+v z-|B{p<#PoleBnAWNcAicQ}+@8wh3GpQbd+3D}k8199pIK5;Sh`Mzta9L2&X3W|My$ zb8X4sh8>LLKOyp=UQ2bZxWG>*^aPz=JY{=LeDpr+s$;{clJlg?v6Aub5m z-t=I$#fg)(IkI5We`m-i;pU8uh$ixF=|u)b#YkoJ8?f#(U@C>@p#COvCSG?FsbjLA zcK@x$sC}MKoOV+t(^uJ(>N6LTw|=c74N`0vr;0)#*2(~(71Q8K^nB)hpBjjpKFwsk z-^%FsS&cBm{7=Z_HFZrz3nCY^(4Geo3Feq(9KFbtl zMqW$-XLJl`QaWU4RX;kkZWC%Rssw?x+)6og8bv2vLtnqSBY#C}1~$4A1}m~`9pqhT z@m;#4{}e#%&_=>hLx_}W+6?vxX4>biJ8K@Mi!(d3@@cI~Bl4(;5XowEBcE&r-2C|- zJk)AO?-ZYb=QaMeeHO zq)mgXi$s`W^997A^((8((|>?rw-wBVNNdJsTL@$OE0x?VDa?4+8Zp1T%o#EJ&mdG$ z47`2jK|YBk$;tBvneV1g(fEu(a0}i65$fmBnU*6o|Dy`oayJh}n>aAhk2W*sLT8d6 zrxq~ygLn;Jhh%!MhcX(+(wW^q&yoM8bjZU`+EJM7Hq=zRgXr~6L&%yWuV0#^vx!MG z{+|*PBN0)vv`CrJ?2{k^=|*CwjvNSERR}^ZrJzrpfYh%~CmOH30YBC@BO3{9J1skp ztg!PUg|u~9(DlW(WP{+dIm$--P}nRVJ<|6{VezrAyvP zf|0AM&{{1uX5La)=4q2W6Ln)BGc$V^nZx@97A|*WQYH5xlOhK)T0fF#E2$=oE(7f6Y1m~VR^SIsIw%b9E z^o5U+RP-{ziGCyqiL4{EUq%ow%n2~7>lV7|eFyAL{DF$eVt{5&f(St-Pp0P{lIhF^ zr!tK}!}L{Ru=y}r(`gIdog4;EF>8@fnKt=tahR>{tGB54o2zZ}t>w(P+HiFS(j?z| zxd2*mCi9z!Ko^gjk)Q5Q5h-L2ZT_zaWVAF9`Zkx*yS?|pg+W)K)VBm3JZubH;CG~a zauE{(P04e|`apo0KR7&fhR(};f`axM&^c@7kWST6Ad9z~sYw~B=|SP-(Nn8HoAU)` ze|8fX>S?Bzn*JhEgN~vP|5e-WsQ3(q(q}Sb$sf`0E=i{Mzh!9W*EPgk>KeF-KG@X6 zZANLSrl9+>KT==aid=Csn4H5SMDY!Dx84V|G-@-O4z{6rTb=+%xnLld)(Y0UDw01h zr=kl7F9I2ZM)WTs3cRmK2OW=^Kytq%nN%=Iv~X83+dkbVE{pvp!u-nUHrM4qSo9$= zFv|ix-K~e-M_m9fctgb12wO%(;S@@gGbT6tK1G((!Dy*z43f*9kMad)M+<*Np;bYP zQRq+tIF=zziug-`+My`Y@`NW6w$25fiTeTXYN8uy@{LF!@P_jL)t|N+u2H@t0Y^y-1VE&G`vD z<^>^xvcuq(MG(SS$3YAuPA3eUt$9H3Km-jDY_0{3O2%Zmf)VIj_Lv@8D){YB6cC1E zUNuo+yO52^BjojKjCjK=VwTEf(w|NQA;Sg}viPhic=*|hDF|^Vh>E@7tV(0ekjfI! zrqD%br_5m<>aAi#H7v*_M#q8MbTz%fd=owAg_Es?g9dV4`3$K)Sk4$($br^}p=fKe z7D=mY1J})R(cHRb5DD98KfOl4ZX2UTN~XcP2`74w+!ZjnWENw3aXGk@IYmEq(k0%h z>;>GKm%t#8kB+N{0^x5mXk7~=jo-Wml}^hDCBr`;p{b0%(7J`` zkyyaY`f0-$PMedr)veIhodOT=1081G4tbLFNJRf#w*|5fM~P0agTxfih~UFg5hE_t0x8C;+uzh8Vs(h7?Aw)!D!urZs3$E0!Ch$k-y`# zK}KaIC|zdAZ1Mx3=B5I9G2jJoFIHsy%%7l-gZZFPs1BHXYC}yc^Jw8x1v<6J4uuu? zfCI92X+c_oDWhLAg677!leEo7;0Aqr=uFtXLmO`v*sExvb303 z=$Qi65WZmA)`M|SucCcBPLj{}jMK61m0-W`X3%C{On+MV8KkG{kn6Kok=6>wnS!w` zHcsO+$Wqx@x_P5Bv-)2vf^RYS&K&`39KF!Dv~Z*>$SVmobfCiVY_Rua9X&3%22xyJ zpgtzvHa{bWK5^e11iKca5Q$K5<>x<8JWmD1zs~><<;<9Wdw$qfFMEiFt1rWX>xeY1Zs7AQy7bw3!j6B8GGbTGU8Ge5lS-PWw*uLpW zwY(PxnR|LLZlccx{fULlhl?`Iw>*-(|2GJDc|QfOWwgjY+y0=74g0Edtjj@Xq8!=p zH!3t$TJzpcD}?a<>l;*rv9jv*V)7CWTADc>Z%_!A(c`ekV|)v;w)+$C6yOZAZ29 ziBb@1FU`DS)fk(qKR{kbiFBUUVYXbrO(=j1t;|ghVrpYqKX?h_^ z+)VoTEIg+HvM9;S-L#2(#Z9>Z#?=?Nd7MmoH7$rtNm0QjPJIw(Lw?0AZ&j>Q7 z8kfO2(~BtZ#dVOp<{v1l*93{qYQSvGWsvz^3Z06_!Y&jDvT)(HI*#+0 zuMHbWpSXQweHtI_f8J~Rvt^u+kuL^8pCuVF?1g3*6@%icE;Lgl2V9AH0Iol!<<<5BbxB(>_0YTb6^)b0jU9~VeB?8QX$p+f-dBbnt#?*aE$YeABi6nWwL9OlD- zBq_7H1PvLiLu(8=Ym@{TNAnlGh%)U(_2(0Tp+zGR?b3mAOlKh(IW@BLp&n6VZ3N`P z&FEFC9HKlR0_3$yGx2TjYhIOTkTZTEaJ6;{2)$ZH3XPgF*TiNpk`Et(7MVD-Rdf}| zpDGqy?;Dse@3PVH6C~L_UP1TvYyp#D7s15A!0NFlv(T>3isZLL7UV;%Xr%Du7|`KW zf&6?ea(bmQ^XUv^-Y3=P(~v^neda^(Zw^kAWNFfWD6kQ}B8WD6imy9Qy(A zS6h?Z^HiDiZI)$9^n!>&nO7j{&O2bO?1uUeYM__;;$$<=nOyl+mf4#q3;5Ya%&YWK z^yINBdF7Bed9*(o^}m)PQxZez));YS`&tEtdBY-o&Oad9K4P10ph7ETzeP^D4hZhI z0Oa!ji0RlZD8?d;9#fDZJ-tiOzJIOY@SPzPX#W;$t`%eMW;7sW|06)jOM#h)S0F!y zTa)?jN0HwAOi;e%DO$E(6J0#xiTrw%0X;1txUY~V748+)OlrSGCtZiYLMskI_sKDT z4;Rt(?)Siz({{v+z&3Qko{utz&B=Y|ZONh&4$PHt8`NYciTd@wA`Rmd^dwN$Hf*OP z<99o*M&hF)*->aj&YhQvibDs1k3>6RLksq@pPE3Sj3O5;)~ zVI0=>(XQf?s4XTKW${}zrj@2wI&LFF8g?-fzfFlF6N`w?%bdwWGYMqy@;*AS z!-jzdGa25We?Y#)3oJbrMZUO|N$&a`#SDFtBTXL{qhoH;q)Au|?VeR2DrFino8T?Sdh1_JH3AOaYprM5VbPx1}o1z4> z+;t69@%rD?-yJ9MYJ=B!mvI&rDH)`y5*l#LWExYx{U&u4+TeOCSA50eAWT>k#R^Ua zD1MJDtl6-H-JBK<{b_}AC)S5zjRa?OMNZ@)7gbyHC6tR>zo?grJr znUP663HV=f3O<|5hjt!5Y*Mg*FLFf#cjjHdo#zYiiewjj$V-rk2pGei0#@(Woe6B` zmfLVW*1()?XYt`>v#913e74$D4aYPOQ9g5`DcL$7ytZ!-+*i(035o*Wbf+N0a4P~D zTTAlK3>)ErL}C1ANgQ6jq$u*vl%mg3&PKQr#&y5i0zw3?5eH1*$rP@*&d!OT#~MdkBz0136@2;;EfR8^k)EWt$P9u zyOzWJOJCU9ci}wlhX7pLl#P*?J$^Z9hW9;NiaVrCc|VeeSfvvy;68)Hu+%>VzqC7r zXHC3f3#Pi^r}{7$rDTk+7Kq}GNMXL;i7@uL{Vq7xlmew>4shlPJdZCt)3K-@pA~)w zp;xRp1YI6j}kF%{hkCC#9ATJak3Y#Pcpiw2D@~-w%ap9iux#?M!m6e9l zBHB>obv``vOIwfyY+}V~AFzq(+K_+i67Ia@&I*+^z;qRX^CDIco^%-FZR(f}k%T<| z$$UNP>)~1WgD@X<=yX6Y>Bp4T#92J#XY?gKqqIP9BedvVpVaOmNiLY_Uf6z;q^ zN-Y`E!e=^{!Ua?|tWvJyOiH;>PV-nc>G>lZZoL^gw^`s?wO5?wwTXDhQ%=2iEJ+HO@obU9jM}9s5T38|Ob!6?SQZCuiNC{m{(C5}%u;058mLV}I{8gU{rb zu=aKz;ZPTW=X=<*OFNh0-^6}uRmCYdu{9GLPi5hw_VsY>2VK^C)(_6;s(xtj%9!e$ zvjCndOTf)J!xtBuqBi8tVQzV`{CdP)_Wl)!$4LG_ZO>F9X9eC`;ID30%Gn+g3hSRsJ zha(*)OjXTDhMUaw@xmiAJZe?|=9G#G{3Q(4uY8@e<+6={8@U!+ou#SK&>qTYk1$8< z#XjDQ-T(1w@7{r5EHklR@O;>|AOatJDd5F!+z&@BrdegLDcB{qnQ%B#3MDHYxszqX zg3OmEthg`1njX3cGh5qm{Hxvg(z7t!7jd6l^`(IQN?d?@Ue>ccPaCo4*>60BoY(BI z{4CZ~KcCm?eVe@>D36aNxp0mJT!PEf262i1fB0_5Pu}2#C3x+~V(!{mqm;D&GMF1( zj%TkCgS_2{JzrH0Pt7*N$@k8&+Qz@wjX_s&U~ez-XmDd?RyMJ7Rc)xpMidS{UVzzbgx$!2ta~c;`u3_@JwR84UPMHGDP0 zOFQe~rx{yd%fkrjOyM~G`8%3zJN%SAk?>fse}r?Ai|SbCXBl|+ZY@e=o+xEzaRmml zX?SYxTvj3L8KoH#z|J^p1>16zSpSI`e387v6eVpB7jM8sSoaC)cj*mW5NC%=GIqh0 z?F#r_Q5a0P5rJnVM&MUQ!SJoab@(@&#Kk-8@b+$7{CjCSoS*WNlQ?<9oLPb{d*?aZaH=MROtdfIC~A-6>|z+DL4mrtXzUL`L)N8S=g4}+v zeh8bgj^%N#Mq;8s75X^k;EK)8*k^Yh{At;SpQUYwdADP6z2-ei+ix3nv-KFPEJ(u3 zBzv*gA}jLhVjsv!D9698f>|N2^RVb<3j4+-5^KJjOLg-l1ioJ$jx0CB^np&c&RU)Q z=kpvN2zyTvSqPS@TSM19Zq$u3BRsge4nOIr#122>I3iBYoa^&mV+_)H#k#rdJ(+O$ zFfs#w+7pRuje4Q^jXiAS;f1jD-&RTp3sW5{XY!{$gu^rH0yk=@0e*hB7^WS#OG)Vr zv0WiiY)#-oXz=bg{&Djy%+9+-X)eiuule?HzK8@j3-{n$2Sx0;)CBggpJ3&W4dXeI z`M4)lz({lr!Tb>}=A8)RT^QBn86U{N3Bnz$@kK4{ZBPZRy`pfAo(!Duy28^j?ZpO4 zl5kVOd8{ID3{OU{;86NyRH{k>n0beiC@+u23qE|oX4gf(D>jtTLVKC{!lz%wf9ItS6x+@`_ch=uMflEuOj?zI-fPMGJ?OJ zTCl5kY-hopo79lqMb>=9MdqS)7|vJqp}0y}cw{tK;LY=fXLt3ozBOk^XG=>Mz6rB? zMz69fNKt_=)`(IL6l4zKd|+H`6jYonfZVj(lvPOt3|J&szn@9KT<#9uvw;a}<4+O? zR+X^g;5yrq*~S|?KEkp7^8%Wj-^whMOM@q~>Y=b-BK0pX8ZMbs!?~s{6p_@#9{+QZ zS`%;*{!C56M~`p7gZI_BFQ(0K>ux9fS-hN0@E7L3l9&lUC3aH_9qz&#ihSN2))iX1 zpT~Yn+}Lf_-zh>n0&51}W``x-vBiPW_-V5N7JEJ)j+RHj;0IZ3L;3{ulWk?oR4aL_ z8}>svMFZ~obrSIHcoy`ux`uB)v7&Oe=Cl4YdvO&w1cTm+3p^T`)Y>yLP_J|erJa9~ zXRDnI=eKQ#dav~PYGXTbak&Ve_m%@Ay_Hxd_agg3T9$u_`a_+ny^dE1JobU3S(G5V zNj>={%AfBNi@jGAV)Kk{$lXIG51TL!b+jlr(e;&eMZFO!t$BnvBBGiUO>93EgQDrh~v(%ru#C!9NWi+kwMTPe+8Uj@(~U?ZHCKz zIWRgTl=Epoj9K^Y7(OSoO2F!lr)Cw};ri#XSp7{De~D%-e)485wJCTn%&+>19qS_? zBNc{&mljbiY2}pb#9Q{<zly=}Uv1#TB;b((zgS!O41UJ;T282EB&A<* znOD$x5o&dq<1YKZIQjQHSZQ(`Cv<*b6*W%5A{#9n)Vcyck=Dd#zTKcqJOP#}6XQCj zs&U8PhgT&kI$(GyjB1e?}1sA4h`_mvAgk)^xf z20bgdecMAeOtly$nXJcKWHhmBV+o#o!^Io?UD^Hhp6rmzLio;n8Ta|HI8+xOWk2;2 zcyjH3a6^SP3<=#0FMZv~9$6&Lm6$KslewmBpK~;&*x?2pTlV2~r%NdwbssyKY=myt z6xj>kec*!NBlwGaCZxqAuxO(lbWIcGr`}qH?Wa@V?-yxo-Wp|i|Hc{EE#&~K^{ z+J*3zZUP5uNaBUh6L?=o8=#9KLp9lo@qb8cXa8Hg8}fT}sQ1g{@#LZs%vxW9x^{jX zlN>+x*ZO=aGJiF$*4YnR-pk{lH7WSq9C5DMKoi{Tg7HqRSTb~=8QzH9LtVN56pr1j zhJ|-@NVdI%+Oo?Fe(p|($@@InBIN@1(f)o2wo;-gJ zz9v-gMx|bIM_eLX5p)&)xBeWysd)+Ce&q}OwsEMd_gCP#2S?$mZ!@`LrZmo1D#sxr zLwLK|Ik-*YCg*H{1N?PV2H#6Q%5T>-&_&Z+*@Zf$K%7xf0$zqrkm?GndyRtH$^Ha27jfJ;5n^ zI&r}AYFj;zA9U+peQ;lSc?}xwLRI(jP|?yYNdEFaVrjr7!93(FTCFO~_*oCvjC4Ap zQ?@2_*wjn3cGoIYe{P)qcxsSV+>(y8iZX!A+I2wl#}(pyxE^?yGzY0~I}9RRq*026 zA~81QjikB;i1ZRJcz*8{&=3Y6EAw0EpHVrg@f;?WoXY_Lk{D6C6?ELBH=VX_ zGukfe3l{XqppT&{pf3<2yX0K7a}E!EGJj9Me4k3|-wX%zmo_kN?u;HEI%?Y^yAcfw z*P>nXHHZ}RE#PSLH#)lZHri=p1@eAI1JRSGi2$Q4Fe1n;J~t~xZ&a$0UDy%AV^<}e zq}V_p)h%fM*LQ^D_Iv2h^B06+&=J(OBL|uKh$3>L5ajLjr#lzOfLVi+bnB4}wDW}{ z3K%H^2`vGrw73zZbloP5j-3K)FF!|-W&kDIg#+S{I)lmShA9!?wm(kJPnTTwS1a_~2YF4s-DA(dTsCn&1`(=D3)CVfSER95TGAI{t zuN5H2jW>{#e=0p^q#n)7+>9Jj4x`=H`M`a*3oVk|Nwj%1(Yg=#v{3MOWK}&wz&uJp z=2x#1cW)84$Ml22hvr47Yw{9M?k`8|V?X3vq07X^-J|VQPotxCw)E@BV&t*sGO;7# zE%FFXMYGFS6Vvw>*k&G$p@qH8!L01-Xw@MOxWB9pl}1#c$L6ZcLy`Sp;*elAvEzd6 z7vFT-qy@_WBc=)xX4u+_YV;xJmHA+C!*MjId>wtyyei;_y{HNI--=%9ZUp^>=WO=; z^0sXj4y?JoAcfG)EChe2yHSJ81+Y&t6LebS(v=5hfVHn5A=+Vtc%5&B;=Bfki3h8} zyqY!O(M-=8%g>Ya**gR1Vs!%YrtJum_iN~VK95nVTP<3y#6`cf@6t|hAAx_NS7?!E zgY>#{nYJI;7PNHyD)2wOy+(3mDP6Om3|t@BPF$F}jXVo)5J$8lk=@BA!mZhymg+S_ z#YvZlB?$`5l3%C64c~iUu&)X@^%yZAO$c~JnUQ`@U%~w&^BA$WrvYapBux}enAVF9 z;M2`0^lWh|IyT_U{CW|FW=nvYs)fVAaYTq|xf@pVXSfOFT*|dA#KScS-xWx4HEnb$ z_!oLMIFrnr8G$}NeGBmWXTa4DA!eNf^XiB;Vw*d_;l$<4GQZ=5$)^JJUyU#Aa$S@( zebbCGlkJ%W_KCnhf*|FiR-mtoYeCh|Y;Z#EF*tPD63qL34QQyI0CSAJ$d94NX?fKs z8pPC~w3-4?UeO0GnqLH9!yP*KPdRGy^+HO|qG)?nX4#uPdbIBRSJ3O!ETRFeI2;iya5ATANYV z$9+{>2oVrmQb(w{w}Lob0q5zRC-Rc3L0NNkY3A(!VhisOx5Tr7U8Dk&y6F?WUgs9L zy4M95Yf3QLO&Cw3I{J{%60*2&94)<^i+V%&sAm2X8!z9zOq$Fe zu;Jfpbjo2+z%xri{tt(UycM&^AD&V`&!Q1T#?K_TWmthXVLizArWToISPtUNeL(xM za`a*S8`_~-ip-0Y1qT8TqOcxk5N$t)@l%?l|2_N#BE)`xWj1#~n13jEYTJaK$}B+t zdRfq!s?D74737u0#=&C4Y2do;Cio%MiAK~VkPCkcQVEaP#?^^H zw_p#7U%8z0l)YnfSGEK2lv9Y&Uz+5~9V#IAU^TeKO(hnHbP@!&1$|B|0DYc$#MYf> zK(h8L(AK>l{Ac}zzAw8L{V5h>gyP(hNb)!kt{(%EMQV#8@M;W4S64wWu9J0_Wd+e+PY5xOHrbM2 z4HuK8uLi-e10p`H?*&q0^J)Ty@&LJ8l~G%l1ca<|fuYKJU@zeOe)-x-w_g<}%de<` zXM6+j;Pa1~5u1f7|i){l=wIU%u z3FPofkPqKoP;*+k1H3#h4$fb;WQrVXP^Qr{L@!;9?xrHpt9pkR))i)+4(p(SHA-Z; z>c#4X2i;KUm@%5ZYe*K}T1F_2BD%ZK6}^^B1r7O5HT=`p(ZlDe$gsgOR=>}&?KB7NWG2p*|zhM8;Hrpg4goYlvg349J zz;XT)8eVOoU+&w876j{)qwRk{+l*{b_}v@ietv{p9+;thq8Z?-{RyDGTA69d%LQX5 zZ)#55ogya6=7H~juhqPZY)1+w^-->(71^4z5A?LWM0snA!1v>AKx?>!==6;RsiI|s z3M=5YnWv-CG;4D1kyNz*+D}A*A@HWcfZnJjMb!FjLPAfcYs>?if#E(;@L;DUQ?hz1 za<4UD+K(xrpmsfS$2D(odSE$oWcOJ@GqMs4T>cNa`jvy4r#ygD#Py=2 zy{<%!w{|5+#WLWIloey!5LYu0wFTAXwbK3^NmBjRB)!+<8t8v=2R&>!hjcoB)qL@I z4rc8V1yLFnj9BMM`nZ5o71mouY?Y8_`X7xF+jpG<;-nnNy{yGJ|CFKi;+jEUm?)Uh zV9AKymnXcq_P~3VIp|-a$^7&60_(r;K(cb%f!*t2n@`gB(fuA}(s}3>y0KP?k$s?S zdniTlze!XC*XnJUmVal6`iXJWymA%N^Mhp6|}nLdLc zV(>3m`_h%+Zx@^}4m?BywXTfe9~DMY%#3{Y=r&Rk<1*VGttH=`2F#UALvk$8iJbP* zV)|pu$YQ01Oz7iP%)wncpw_Jfc=2|ECr|oO#`R^4{^JECx5Aux(zSq5FIQ!XUdxeV zdru>~*7?lw_kna>0i-K-%>=jO7Xd?d97X$DkVmt;5fpF-C0p$oF{{nwp7+z}uk|*@ znLn5GxW0(dOo#;DcokXg&~58!Ygx1Vxjggj;~!LaRfC+vu4MM)n3MUT-lS;7d3q{A zf+_TUgcR!(Nf$vDMzdL;eEIKxiq6Czs`rcIS+a#nmJmvmq(~Gq^PEYPN)jbYn~JoE z&?4WGeJ_$VvP23cEyT=y&V)8aiPB=rR*|B8uiyRs0p|5`=gz(7Ip=-eA1&d(=Szic z&XQbXF-d`Wbvfs~VjzdFm?(Jm?<=S8p{-z_!$e{8tkuG;uFJU&pI-`!mOBgUe#mnd zYkwAu7&~yemUFo_2j>YxLML($@6{7N7n%#r{?6jgVVURrihRzmk8)hMmm+l0%fiP&q$bfXRa{G z?o*BQy#mg@@UH9b&FaFUan@XyLL1@TsS@16#ml%-1H+v1Tsz_5o$VHFlXi1mpMK^1 zNwyTuWTsjEyttCPV{|;Xa{W8O0rRh%fj#U_l-(lXR4qPh5Z@%ou(uL^`NwcpNvR5F zoHOK9#^>d&P-cc+f@o zB*jkXXq_c+cF_iBdK-O?eM;CH$dR-N4 z&HTcto}eexdEp{lV{K3q^KlF}!B<>xa=oSSp}`7HUtX)=WbibL+JrgW-!GSO+oD-+ zENeP<&}0?Y=uN2LP`VwbV96U!R#hWs%8jp_N+~hnf}k{kOXhU0cIHgs(I^LQhFL1d ztZ=DtRoaMPWbFgajU{a?%XVCFcAF;m+*xyBf7=9Z?%LI?|4)T`d&q+8 z_gO=zbxEHa;ni8w$hw;4O2xU4MXQAE%m+@?rOn){t2bMwvvu-@sDGSB^B;n3ua0sh zwg(BnO>yNKglyu5n@ByPaB7-{1IH; zXC|<>o6DVFFV0;ZJ6S0HS(5wX=WOmYcNcEP$yLG|3njUkPh7Z%r|1i3Z2|79clyGg z=hTGXTAT&lJ*_obEF<~w&+VEiT?2x|*Am>E$g-M6e**;KCDkJJ*Uh$zI(8hOLu4c`^dc`KnyIlOBSDb`^qVFGJ45oH4=z_P)OF zn#l=Q8pk=}^GC4$PZVb`x`J~sWn4{A^>Rx;rdFW*tX`0koLaM`irqyUEa$up-M~q` zpKUppIW36LTT~O#njr{(AhJk)b)IutUnuapp&+c(-o%mi@(}2!u--NMdmMu{Kg+r) zNrJEPahwPB*EnzFC)6mnw{QZjBL!|nNrEMY4K>HVCs;0hRmQpYB}h=6x|SoqZ8^tz zQL><1XQjaZYow*SgQXx-V-3g8Z9V6(xhA*cqn@zlzbZl6<|K~7by?1`DRG>^acLZ( zb|%NOUsPjNV#1kJuuI@#7A**=)#g00_OHn@G2m$K2;xjs=(Tt*d0xQT8pRp2E{Suq z#IwezI*0RU#W8^a8`Ar5N1Na_dU96gW?OVuw${YDCR%>{*2R&?jpN82OyJDOsuIM9 zN^yDqvV!D)M+6zKo>~T|Y~lPn+8}r;9wa!HH&CSWxXikUpevWUrtl-Fo51cc-YXl#%LItVbCu;J^2ae*RLQa=qc#Y`6 z`s;14zX;YuhH>Uer*QVDjjj1HbA%&arzI#*UMBciz^^f$m@2TXNa0M}>dx72o@SYw z_fgRE_z`DMeh#NyAxf~Lll2b!rU@jTmkOHac?t%^$8&psZ?<4l0BZK_d&WsoEE8N1 zh;w9>6FDA}a#=R}ho#hY>FUJw4>&uQUa?$v)SHu=cgu2juz5{{d1m$B#S^$Y?sEih zgG~jVE-N`MkFz%9eQVZ=GGN~>cSO*9kZqj#(fe9g7}u4i#vyE zdUI}ChWE@DwD<*ZEFSl@wZdKX|h9ryyy19*( zZyo&vxx?24`3d8yVvbzk$T>2CxQYUnTQ{`a+MCT8pCJ-0^FxJc45pJAKA-EF4{>8nAPw4Or55RQkAt=#dnVU<+IL zKKhW3n7oB;GE3=yE%`(yI~CMk*pS8gKY2Uft;f24hN&3Y4mG9U>AlxQyj?LbG4mc9 zGjr%D5-Es0N?JDk|V*l8BQ+7tl?-yjOLgW^sHAnA=2 zGJy{{%f=S-=B35Lr2oos>x4?YUoeh;bX5eLog!ge{rbzZ+NRj}=J?Lm;qo z0N$%-lbb>T?eE?|zLj!N>v0E}wvr({y9C@jPl10R$pz9(9)jWRNEpgYCqo~rnGOEE zGeFDLrJ*b{5iDC1{kTJ=b zD0iumtTPhfqI(_WZj3Soeux=^q5o(wWy*(mQS~-!FPL_q&errWhkxq0RyQkm# z$_Gw7x(sdNi(xOz>G)F>5i$Bm6f?SctJ5N>-o-+=KJ`D`lz5B`M1A58JyjAt50T+n zwk8lSmUEjtAcjS358Y*lA%v=KkEs8l{y!F=?;%^V)x{2jT)$v? z_iQw(Ov0v}JYPN1Vs8q(``^jSPL{pfqk?Rx4s9};%J|BKLM0w!cI2eOn@cgMn(>XCsZ8KC zj;7L|akE9r3tPa_R1AVwt-! z(3}YyD#d_N-p)v;&4&p_Z%}SVEUdhv3*74`7_kmEPqbPMtRAaUk7gN&U$`8~)FtWN z!`ARK>@8h#(Ewe2FO#L$0Q8*xQX4NZY@DLTH#^b+ir-R!WB!ZyfBr?E2AARaurM%H zk0G|Zrr->XX~h4J6n;)$hcGJ_X9gUBfX5+py4Y$=OOK%k>g4&0{!_wD4qEI?Fb4XD zreKE+>&a_O!prSDnVp*g@xUEjvU^_|D(yW8mlh_no=J0}qV$ADBn{Kpb;lvXO%uM< zXz&M$uM*62z&*1IaptcXV3x86r$=q3|0#!|m1+`_(Ih%m;yC<^SqMvxN`r0XW8SH* zQjaB;q04t#29_>>)A$VGN_{8btIuI%bMIQRf5um1yGkl&@?s*Yt>}oXRa=e z(%JmG?RB`N(1@N|nn@fgCK1m?<3#Sq67bkXIS8#&M8_W&sIAlq?3NIa!O<6R@XmBR z8lVfB#!5KdbR|qHc*)HWDS>gi^g*P&Z+0*SXNK$tfd*skN@C&aNKK*l9#Znr6d# zHFI9DJc#tzdwuxZVo;Z}2mRZTM0I%r-E(UK<*l{AW41y@b4&z&Nq$Lx#a%^-lF5+M z#b;!{vK|eCx71Z{I-TSE2+5W=G&*+z+?Mx;D;^$<*_Sftu8_u-<+FIvn)7JKs1}rN zoP-y1rI@IyzvQU(E@(EqiUX@J;r2_mSVtN7<2wv}-Sc7o?%?$VhGC?MQowvZsX~q-G8Z4X&R+L zh7cS)gKHZ32B*qsK^m3E^9GCg<*(+#xX~u2yfhB)Gz7xQrP`<+rV4h~!myR6NcR*H z-o%Qx@GUTuHm3)GZukw-Q+@{SC>Ns6PX%yYbCS&3xsO?Pc{~}MItMjHQ{mG7Zju@0 zL*3?%5$&>ikKaFA(Um^aXfNv|j6T%{Wg5K{HeE!gNEv?PR|%ZH?L9r=br-FB@)te9F(kKUgKYHOou`0*ODZq{9p3ttPn%2mAJ3Y;JeO4!<9~{o8Btjb%_7wCVR&L5#{A<$%6vH)CEV8PifM+d6K8=Jn!O0et8TtIP zzYMQz=3>%m)_A3I4^Dj8%h>F&MU$E<7?hn4&NDtk#;imvDZMCsQN4gyT`z;B>n4lJ zSQc%_Wi?eww}hdcCNS{gEX(AvGjrY&I$mQtZfD1#dOwNX z#uy^MQ-SX)-U&jc^pwFQ$^W4GNI%a~U_Cd_fzJ zHWILuhwU-JV5g-?EOHBJ*~K*Cbnqme)ml#rAG{%Fi~ob(E7NiDjRVZHEf+9rKnDXW z3wdKc@S*;;h;DBF1@U7?$Q$2Vv}wU8N!oaeh*rtt>6|^31pH+Njee7DN6ukWWjUN< zGqDZVtU;Y*OH7HD;}`n>rb7`%Ao4Sy1y7T};OsAA#t%`hdKpBYO*rX3VfJ$+ zP~pctnBd<@haX$gT{c&V&Hj4UWn7Q`GK*17bCk~Ehrl$+*I>HD0dBI~x81I*%uBf; zh?}+>13rAD_jl9?^{egC|Is6upcH~%f~7H_*c}A|L#X2^l3%lRu)(*OyW5$~n!h#& zPr5!P!*8ZS<$|%`oSnvI1;)TTRXuv%KA3r9tw=Ik+o85dyxy=)Q2_H=}gXi3M41Pa>A1AMZzyBVD^+%*dw_I%LjzgU&nWKy=S-+O{^DAH) zb_$Y0?m+yjc66V&6_bDXzz<_N_&P5R`p=vvzsIu}T<~vtv)Knu8rEXc@E?>A-J!bj z%KZDw^+=HBbl9)FlxKdh4EnPyLx zuo3lVDeqZTA?jav&J3JS#g~4Uu%u`!j{AaCVS^*?o*x1F9@}8>fHck8`3z*nuY)y< zHNjDlWzILxfG5&faMloEAXy0>NIZoe(R&_$MFRTmRN~9n6k%bz7WvKXB<^>E(A4e{ zc=jbSswdmw_`FuupWBW1kDo-}MO>_HaYvb&c;3ogVz7E@E|Jo_L(7cj)9h2RpuX|| zb81Km+(y(uebF(JWvd5k)a*ffM-RhS@&(Izc>)Kiha_Cw}6{%?HpPCx*3!LF`3g<0~pn>sX@Oh3g z^LC>(_idj&c<|k@ZQUF=sjmzUbED{B(lPRjWs7F2@j)tFO!O}%gdY4m3-`w5p>~5g zEWA^MH^y7R`L}+wFQN^C=7lx=_l{X4Ej1whaEdagYDsU2nko~ez_`!`~^J8b@qooO;Se{BPJT}t0 zBh{pJ=0PgAr~{|4wdCE;Pw-(#C*$kA7J3XcsOsfN7;k6??O(2g{h~~$yQE4EvFv6| z3j1E(T@2HmXW~)LIN^uQt59-Km0#w39>=_22P2s|__#QO994fnbx)tfDi2ARSHeRx zc1I-R(=j;hTmVnH4r7v*8+M1kfS^5>!0HuH8Sif5*&KoU*o^ead_HNtBLuA*@hG%8 zjb+tkxY(|XOj_klHFKuGjf-3Hys{zY+9aWmp9HMGpovGl`6ykpkKI&ThZo}>;<(=T zR3T>_)Ay6|mTZhA;e!S=v3x5teC{&u%%|scS7#Q?7?T4<#l~2(B@K03-qQW~#&ke! zEO<($lCXFyTC}E(m^Hs*QWK=en=wc%cCpV{zctvhzeHf|9Sv5=3wgfjP4vjsKjhbc zgVfkcK-Lwm!i%#2{kX9h=)9l2HnD<`$2ruRO$@fD~=Wv?6sZ%2Nu=Oio`Ye>i(F6iSM_6!zT+6T4)dqLYP9k*}>$-Ll6pmOYo@PI)a{q>&-E`Jh6!X+n=eGfcv zq00+FaJe2Yh_{vZ9={4la4FVI83Ep08#tye4J$<&d~L7W#4cbx=h3edIBU3?$g_3v zgxnZ3=I#Z1RWEYj!bx;|XGZSNHGvlj8%gErJ)kq~19QB3FW&nz!2GKX!LET{cq}oM z7^^QKxBWLTIW`5*Pj2Au>6yeQY7azhQRUf2`x17i3ixij*q(u|=s;`~EjiguO`D3* z-n|8TvZJ`Q^Tsl(K4f$B;+0_jA4hn5PmfMncZyt_A51Ptg<<`o+4P)V8Zj6#fcViA z)_Ly7i-{@2HBRkxq;?WES5Cv*i*K_2e{pim_Z!LmDFxZX@)&z_8FTpNCRn}5hy2P{ zglO>?{O-yjzbsg9{Q#dFzHl5})?S7&1-n@GzZ6_v-5`x+-!WpOjB0tWC)-%g-8s7v zgOqxxNW}+a-&ND4cfB#=kRf;TBV)$*g&(@hHdFcg=I}515R@y#kf?>GFfmvX=5{&> zzUS0qcEm;;TJn-vVtfH#G`vBHn~^wQ;~-wJPJ)SMQCO58f$Edupq{eF5%YRF<>qC) z{(C2S|NTu3OGnAy%U78k^|i3zE5&$^zqJ0&VM@M?L*}pr{3{KkfxJQ(kjR2(&Z;0* z+RckSAO%&S3ScXHlNhaxe^b!Iwl_d87%HHKp2*F=1< zZ-iUyegVug5~29XPPn&|1C{Ov!TH%Wde}7|CD6odY)}o;|b0fLhoV`tjXGTpe=_BzK)-dNlfA^jZbk*x-te_wPae94WqG<8JDa zJVM{8t%L39@vvaac^K?k2bcKQhW~!(tgwf7%L#^S%=^?fqCZD-OOESflsP zSun9Q1$Nl|gX1+bV8P8ByzQb`Ec~yN*K=n*wEil^i+3D}(upw`J&=M*&A)jP@o%tV z^?Do*>w<@2nh-A3K&dlwJd=e~Saqfq_Z!_LUAMPkbj>H${k;cO(tIIcsw`h`_j{(` zuNx|kKHu{PD^sm6OMN2R? z>I}|Saz{y-dS<|GGwzg$hl@)CXtZDh(6bL{+ZZ{>UhV+mKa+7~|1K=*xkp!brcs5r zf2fv1AsNt#2I;=D5Iv#?J=Ui9qD}!gMjJ8zs~+rfm4#gw9np!+QHTwdhqspJA;-Lw zp6R^?<7#Swd%T>STk1(ICp;5u{;fnGnTg>_lz!g!6__B|zWbDW~Nm9+f6T?>*A-|qSbK~kj zDtQ{TTv!7on$xhQTMNe;)R58Gc_i9z7dp0Rpi7@ENUUE5QJM>}e)Do%H_MWK44X+7 z)@I|~4|#%T3gx7wDhZxt2(fupD~_M80}^)!7`cW_MtnjhPiw_Q{)nUvf9{zPP#%;N z8G9SRRS#>DyQBj))BtoJm`R)Z*TAfVjl`PQgEGe#!sl)!xNfiywhmO$|JD!C7jcH7W?9+35|9=>rLXHEaie24xzN&ro2!(_^<)Vkl~SN3{thcu z1JLi`H?n5OerQsbg=YOrWKCfL=D)g!7ecFX{E!TPqRe{MuX_!LllplvbS>Jj94uD= zG^S2jv|(u|*?j#j9q~L(6#GL-^vNgGprjfV=f~nZWCE%UqrA!X@pNLXD_nZl3B5-r(3_QyL9K8n{jVT{*Za2!Tz^W5 z@^^2;gCnW%%Q&0f%DhSSX(Hq6EDd(ImH3gTQ_1kvO#1y^57}z;g@)>v!1#9u$yJ|v zXrHA6&pHq{>Kno13IVy+{t#ns1=Hq@fuw!xDUfHKzH35N8G-0E$i;5Kp@Levd%|Rp zNLoxEr}fbNn+i!_$`cw_-i5#GQo+yM8#9N3K(#dyHypc3X0#y*%5kKAdV##hkHg@> zQ6U)`X#w+75>%<#4eyB6g60W*P#RMXwI&EYURTlXzpZq2)pq!_cLQ8nvlV|{b|m{1 z?YYlIm$0JXJH4o-2tKPX!@Zf8Nx9A)ynf&&EC~sOZoM_IX#*ckBYSX-l{7+~1b=b3 z1p4RPhtL5};oQylNztlOVm4SuAM=LD>76oQWwQ~pEkBUKiFLUCRvu|EHpL5f*P>hF zIL0=I0mD~Q_<1(!ygt+=vwxn2m2Z5AUrRdNRd9u%MF$|*rklxgJO;B){iJm%1@zJk zCm2t((e88v>A(1l6e)y2g49$PrxnY4ZITA_uDybufKUidbiv{CB3w}+4i~nZfcq%} za5+T|p6@?_CefaF)wqvbRH}gZQd0<-bBzXF*@w04J926Md9Z$QkLq>Jg=<%C($cgu z*w!|PL64R|hxs_D_b4OYMNh!$$XM!~e4B)=D+KrV%dqg-N!*cBPV;}<#TBLkX3SVq zsI)U++*@Q(Ow1of4%|Q}QG+eT>F{%BCDc890J$yIbY$x^JbL6Z^Pg82uX_D`-rVRk zQnuzK2pd;1J=#Bc&y-wn^7=s{^}qr;*JKJ#47cLb%PP2CMj46^ze9_AC#bxLK%eRq zqE%l7I(P0dS7ld&%iWjw#KMY*L<)E-flEy7#6$;=ALV`w5@Y7Cx(LjkTDnd>iJl4g z%{cAL=PflV#lcISkj^V3fscZiE|0NrZr%h^*AP#fI2)nFTLDEmny}A9O(Z$86LgoY zgh|HInAcuUN~c(&WLTi!^MBXL;C(&>@Fhfd&AAwTvKe120K9)b22~CX@mh`U!Nm|K z-sZ2{;L(S%usSCI{@Cb%yVzT%Zc`D)o|_Kad)GjOVi~MCFc#-;V)wmwFjzevCWWetMlAmEHs78kl1QJ6mmI=z!{VpF z_}zonK_PE((Gt`(bcd&IGqL)|1u{YRCLN~7d4?V-Y`^ggKDj)d|Iob&5>&fMH5!ry zVv^&KV{y@yAJ@w_)-aNzo0R1f26Dn&)%kIz6_^1k75-!q&UdFxcyg8mh0z zN*6Vzlhh*#pIPG>1*6Zf8vmr^*u~ad}Xkrvuef1GJGc|)`&+&m-w{tPax(2tD z+Ctzh1=M*UK}0_o-ibsNc&`x&y*UR-mi!7)d(ujroKs8J?yKYV=$uErst`*0r9?X> zHBhmNK{V*m5xpyIB)0iVP@D3ao|=Imc;iNEi6`+c{lT37)5eMHod|QkDu`Y!+7D*= z2KcSxJsx&_&TNm_MPNJ@lFDzi2u$%zF-h zcb9|bBn5s;hYUXV9N_Js$?X2+Kcb<#hj>S?Byv)6{1KCxI4w4U7P|Wr*Wc#k(es%o zcIp(Q#KOc*kW-!m6 z!K3?=x$<2D^h)~~IwYA0PMKH9s-=3ka%nK0bX&>``*a({>p5`0OauI*cEGL^QM9M& z7zCC4;-%{4g4akq%BYu7{o)E3{dR!1OIE?Zf>=^y$M&r|qhO_X9R2h=2ZNIxsNWHD zI47~6$sBLEz-AG~J>bvUPuvizQEBdtDL!K642z zS|dr=xG|!`jlJ}Z?+`8GG|^4<=J-@02D3FssDtV)Y#ekX-YQ8@cI*kvEbvN(V*GHJSVI4%P8^QAHviMAL zJiletH_CrBPSg@xL^qn};y!aT!Wa~S&y!`aeXs)eJlc+qE-hHK?IyT)JJAbU-jhpz z<{)MLnrlZ7;#|)jx+rfvxSjVxnYcgX{I09`VWgNSooyiRgQnA8FKb9dn>=lLERILE z?d4U3v%Z)o4IohdL$}Y4;>PZfCuM&+&?4|4S`7lk{cyw-%jaaz`HRfhZUI!!bEM`D z4p{!X5uVJjh5e6Zcn9BaVh#+3fmeqC2A#YJ+|LIPU@>N9?Bz9D3F*`tQ z5rl!v<ljJUT-n9oA>7N_nkI58t4)X+t zJ$t#AG^0-5b1+-|hwPYofDT-6fMw5iAU|vg7**}2X8){6_U-Y!^cA{jG^EO`ukAxo zb10@=(INqx_hZ*j4fwm%5@d_3;58G+825>ZMm7!6y;ZhYa48Qv+;=i(W@f>S4tL(y zfH8ETZx%kB*GpfTWa7H{eD3ZEhGg~+4sX#M0_mUM!pYP)nqL@1oL2XdDn$qIQyZi{hez$>0n-dKp+bQ9Bt63}bYBEExAT=emqD@2Bfaj!*G@D`r1;62Pz!G8q{ zF+^RJYW9zkx2DtS{N5rAlK4)>9X`d*rjjA`skx|sDuo}7S6TN>J+Zax|;8_w*z2-~tP23_Op)Lo$9-w4zP6hMvXDTJ`4P?XmH8gAyPsBUl zPK}qpU=sGUQySF*N}M>D?t6mmuTG*C;#cT&G^R#7FQ8VuMD?muo^a>XAkiI4ft|uo zT=HWHT$Kr?S8BODM@M_y_+A>H$m*bbi4;a3Fox1Anxd|l`=r+kh}hmZ+-0~5EWQ4J zjU~|IVJagOtfJn!fblowMWcxevG0WmvEv-aSECv9C|jFec`A!{TCPC+B^~tUr?GRW z`>6TV9~;l?Ar3EOMF%xY$np{=GQ%kr68mP;b>W%J_7)A67r07RM^4AirbyVbD+*u7 zq{30(KyYwdf!|&OF?_y*(Bw_%nSOx$+_MPWmK?^e?g_}>YJl%m_kgDBA^4{?j2cVN zqKw@ts;mDJgBuNC&faO%ds`iR+oA%KC6lnuo9*q`2ce!%9NFsU4~-vRu-=;#&H$Uc z_h{`g`f!CJRPfo4W3;SjGxVQI z#Mv`@>8hnK$$*>%Em@q7WX3z*eW?M`zo~)Vc_%6AY8sEH|9OHD1fs$VPns3Efp-3> zqcZ!-mmNi^+Pa1|Otj@cOj8wEe-Xh##bZRHQ%LNRmC2H$ z@*+JpUrSfl0(_@gVf;xhblMcbvh~BLd2}^B#yVDvcSu6#2?6bDyFz@QY{bPb2S9lJ z8eH4e0Jjb(lXgJ^Gchh0nhST5Ee`h36}thqWE=vuUHXJ8{Slfo1?2BaO}yp*5zIR^ zghzc1XqNs6*K$uWD&D(9^YuQFEAq;8mdgauymPrY^;a)Z)ZGln1Lok~lJR)mi2V&S zJ8*Tk1sbKzMyIM)Vjn5S&sWpp@qJf8nNtIr1umy08;0m2|7_^|I~!kLbR%2GD`TTx z0j8uDGOL@E>ED!akp6rEE`4zzLlqxD-M*b}zj%+@`P)!e>O-Q|q~nQLBT@Rp$#`Z$ z4DJ}rq(60?&_Ux5&wk(`?dw;9+wOeqUG0fKmFDsWVkX^PKvuYMcJGv3Emp@!bHJyt0E9`l&-g%oVuX_(`~Ze>|AIj{^7c%b}km##j0m2f6%d zbltM&SWwS;c7knTsJj$~@3Z&wSrJxf+LNtcFAxKEPMXKMXx^>N!|{42$dgbV-scrd zh-CLXwAYS?D%G`+jm2=)dLeE86@!FzD=63fqFzd?=|Afb{J=i1t@}EtF3$O^G8eDXmL#FJWPh^|w=sycB(8#oBVy^wh>J^oEDf$?0x9g_cy?*iJ ziW>01wGbTKtIuZ5%Hnx{Kh%?Qp^f*r{8_XEId<;2b3h(mo|_2$cYLugIRW~6u939k z70kqqp($g3kRrYe*$c*T29(jVxi2`9?T+N z(R|(gyyf4n()W|Qsdnu?C_c^pHkfX@RWqLs?tf1c&Y$He|5AZ{m*yik$_QStP6ice zHL~`d1^;=Q6wW_!hN{v?crj@%zqN(=`~S}0wr9)&%2gjTlv{!cgBLKh zMu}{_VnN5H%_I`B=U|<}8T@wZ09LQAX1eY-zL^ETyHsfQqf3-UQ&}EP!vfxo z8^99&Z}5`&58l7*Be$9yq3+%p^0XyNc(CRY{c<4%$DXsn%JEJtgzd#${jbvt7sbFi)d1(sc>~gZ z@nrtyWmsUc7*3RgL-V2eu<(cy^d4M_+oTrY^gGAU=)o??W!+ygXh8qPNQ>sJFr)dh zPatcI8{)Wp*6EjnGLG6HtC|fy6H8Id@*2ua5RkifhN&x86Mlaxp-wALfW?z%P*N@f zy?bZCsv~`4<6_iJBVcwotqDita%Q%u} z|FZ-Vzr7^^Iz{y0KbHUdiaci09P-zmkkvcb?8+)bNKc!F>ORMbWBx@r{)T}~yja|8 zoB_&4!DJuHRz){gkRtyqnB5+Pw<0Gnp-Qh9YojDEoIaHlo$&_!^NPrwX9S-kouT%6 zsBo9P3#=^tj-oLIhe+gFpU0Y?=TSYRj7|zYftoXN z*BW9zB8HHTp7cE);-E|Tzwwjq&{U#wyYB9?-kJ> z!gDC5d4>co4#cQBS^Ss(mNKzDaLkL+Tl>=qoQ$NF#m+E;?PcvV6_JSkr(|}$KQY=~ z1;1B>qh$4IN)so*)>C$D-k2LkB(@1T^>^U#;3VwUKY-80I`K^89Zpx`O?;BlhBG`R z@ZI$XV8~A<#gq0y*G4vfPfA+k@PH>;0DG`V%MpJp=&w1Fbpu1Le4}pVrf~k;WZXB$ z3I1Mhz}6r6tkZTf#r!OoIqxSrlpm*#-!(*ixqT3?pFmAxX5pI0W^|Bcb1QG$!(C~f z@b7RR6My~|mG)T%9f=O;dU_qcsbsxwn{LC%#5UM8K%;eI=fx;GQLb)SMpZyt{71`)$KO^p3ODJ<0}q7k7X(9_AfWwONi4svV2ZGV!` z)GQ1wx2$DcrfkQ|(ob}}+Bd9Y=Mw?{9fw6Lw9%^nJ6JybLYEt`-l_GGl&P11*|!y7 zvF{A5(p;C3<@udg?8%u6vY_ZIjZ>d1(9zCp>guH=`n)0(_bKhh9@`ku80C<< z>K}B^_#?QeaVam;+5jJDhB8_bAt0%L4zp9ggBd8pOAReF)5{?q~$%#}Q_{$jczgzm@yiHm7K_Ztn``EyOkx)D&twPL~&gG{D z4+y8R?x9+xQc0Ug;Q4oEj9(U*DX{hDFtQ@d?v?VoF)fLe=uEW#O<3Q(j4{D;-Qdh`Sc5XRT@Rj*mql4LPLgyV?G)>GD4^+*7_dDjmhm<4q+eZLFHk)0#_%O~LjYmf-1yX(J zC2ha_2!82jg6)(*m?N1>+Kw>9aM?e)wlW&l9e09O-(sBSsew+49=OMOE?q5r3=+Me zM6u#Cu74_zOv48HM9xU`xbm$vx)ay6amXmorm2W^Wl=`Ib3&ITU080iJsgqE~;pahtN0D z5dG#XJY4>ihIaZ3H(!r|+#|aDS-u}&qsnsSWRWlM;TxqQXDO)FuvYb$S9E`m0T z_F$@YEPC$fL{0lT+WRyHJqI1}n4tsQZ99Nxrfd;@TpLSDuIfWlYZ7jr7mBN8ETJ1{X@aHDc`@9l|k{qE>+!?>8+2Fy~_8>bv1uupgk$Edq zP<_TeB<&l~dB7gBK4znFTL5+cl0tY*F}#}33K*snML5?|=pDCulD*K7lxMNM{coo+ z>|_&(%bNtRBot76)qApLwiO0XmlScuqp^F}PR!{)jaAnD%-O?wpi}UKo?W2NJ8hFi z{I2)%&g&AGbHoP^`gMcrf3d{W))h-8YeU4BXV4iIPqKXC&~`%tE{-O|*3puv2v5*; zyYGTUc0QW>Hi7h_IAQ6fGG20E9K06G!>!xp`DY!A7>~tcaYAoCk+{X~zQ{dAyPg9~ zQxb>V+WHI2v(!;0GXpo?Atce`C`3FR!%yDlj0oY(o0h|j;LK#W+OEksM|o3e+DNpy zk>pG73yeROMgM8NV(vPgz&{pmC}aB;mg5s>U)l;M_SaL7H(D@#y$=-lMMBZCZu(}{ z7mS>lLcguJ3~_68aCzu^nk(r63i}V?IBf}XDXEEEiPS~iCL{V|c^mU~_zExN;}>GP z<{mxm<%c0Gzm?YO0nMiBq^08|w#o4D__bl`KK%n2)a#-~QUEdiYYgcc2SIA_RwkpZ zmM++6h-W$58HL@=&^l!&TnI>j@Qn&sbT$W%9M6TF>NnZ*moq-n(!zcnD|pp)hHw^H zLt$YZzKSZN18bk~;zPLb`BMqJSa*O1WL$;W>`ecKcq&=n{g;}w3SnZ?70|CyA)izW zsP~r=%5Tlb#Yd(=>=S+HyJSPBrar;H6_0UMa1~yuI!E4+b0ksNKwi(Ejl$JYq$*CE z7^uji-SStwG1oK5Cb_f3qw*~Eep^efIj~JoV-@~>xiaRcdH`_PUA)~^hoJuB6OcYN z1$FF?qLsyAxUq2n3Q}DN=fiPwyHb~_S`wrD?OEi;8Yi#` zoCJMhi!rp+o3w0SOigW+@cYC`^g&q}bT+x+KtEd-Opy?M`JG1ZeoKcepM0AAwgdxr zmVkHgEZiOCj2ET~7Jl+H#_SeFc3yr13KX9rKi>!p4miO3b#gd!;eEU`;0`Nl%%LOg z8qXnY2zo7!z_oxA7+-FU7I%G+tNyRXYVJ%p*>Va@14rV`P;?-X-nvV9Ti1>kZQk>pmP`ntcGHJpn&>624eDlEZ|osuH{?mKdjS3G z>`T6?OoBNJ_4pSR4KX`voXB*UJo<-bpiPfFE`9L=?ntxWnf*Cxw&W{4uxuah9sNUoTIClLqG>7L9^(nW>pOjn@=$8Y& z^A+&uoku9M`5Z}$oJA7tC{EE~_c?UO|0G-=SO@>B=**+B`nmvIG*E<6AtEK2LS?vTzwae987e~wr6?sz zq=5{1%^^jiG^c?kX>`x_UP@_HseVlw4N0LwrA*)b{yP8Owa&VG@3qg~&-335!T*-bkI{%4`MBS3=>UOJZL*~$C{J5DrUf4kVh=dRF)dC4~1 zZem^1kFy4=F0gewQrPTIZtUZ*SK_jI1+kHLtti0qvPfhrylczeW9{$7v!%jL5_Jpa z<1anu2bc&uvXH>P_%W3qbn?CQ{l5%xreZ%oB!3;d#o2=`S~r~?vpn5_Z zTFy#aL63bb^tYXbT3KO19l!9GNqO_F6jtfj1#$K<5!)AI$$z^1UhKc;WSLE|2`gQ# z$ZobauzCCWI)B`$M*2qfJfHimly506Vat?fvDJ%L3VSK8g_UZDVa-qbZ49e|SXsG)aY6e(E0n&7UdK zr)!g0Vs|!oi>v%juwK7X`R%0wQ+@1yzRg?4X0>?=`#MvOy(Y|Fsb@_SeT!Sb z`?3Q~TGIgbX>lhzSf!|J^0EQtaS!A~zc@yG-OEelnKqp_TRc&E z%C1J*ZGD$5{rimH+J8=DEa{VWUI}2Wy}Q|=^GkX6`V-<7{dVc@)>G1%l6vXR&S7le z$jwqmo%4K`FmJ?gcAs5wN|;Mv zorQkc1+m}cT-7=LO^>4IzU+8^a?a4mgpi!9}IP-`2-jLJ0gJHh->NRzn<;wXy zjoro9RG;Ni`mB_iIw^t9gcf1nrAcJ^q(IsPXGO|2TGFxwbJz+@-iku8Z<)MJJ?2WsHXBRVrahK6o+x9zG9|q4wj1J#W9P*^nxky0)IW>8cG$3U zyHWhG={rBxs!ZGycAjMzbJjNICEuFdE**3woIg8mu&Cj|1GdL+vUsl%EA}W@&wd!c zNvhf3$A1ytDWpAWY|eO*^yfBR{!RaJe&CNy?1e+EB74V){IeZm@%@`w>6#>VmVO#LLS2C!7FW*uMO>0v6!!B6wtQe61SW)H+fx>-1K9886Uyzz z<%@nq_5EnZJN#tDyX0%d8D0WYSedz_OoAmt-Sv0D?N!$4lDC=Gh z-~L%)AgqFA^1H#ycrSQuiKi_q_kgADOfY#Pf#sfg)-%>c!LoL1__fm#CWNi0C!B4d zEqDstQFDMcOFjC1n+r^FPeK{>B$N|9A3(V79T_tC znLGSmvXwCy=#9>7DAo-qC6#4skzW&!BS$3QcD-wiig6-3-b}<1ff-ogYD%p9b_gy| zRcyU%h0O2=1Q(j)TOA1~%EdrVeLctKMMLDm6dbr971R6jh}0|vo6oh;8eDAIzwQmh1O_6 zwf0bIbTE-v7QPpH3f@Dx9b5RhE*Tf=BdckT(gk%OYFvAfvu)7s*WVL!-i24F7zF z6gq9f2aVQv!^{A+bE6ndeGB}iXF#=F)9G3Jx3sZ1jvjZ)K(n)12O)1`yDJThqoka% zQf#yqUsnL56QiJViUjjh%jAUOO6WwHvWqErL7r5~rQ;U+lfr89mx;|K;}PIyE=9nQQ-gL_H+bnUn_ zxMgOJ-#&X`(q`4tsZV^cx-XU}RPSbdbDz?_etj}a!yer3`9kNH96DlU0LU*POoOu} z=@}{^PD6e&)6HX;ZY>d0HZqZ^lrP{O>{*8g%7RcwaHJ{RT8oJl!{KLxr$Am7(J{R) zV7w$4&pe952_^n?@uonO`>TUbdR=h*rmKunAHn6< z9*hq*j)w`V*09A|1V>Iep{1n-3~ouqZDaOgXreB(KhMH#%zNv%+B(o+@RehOEa0n- z9hvE41ZrJccyF09oHJ;lzwXV%1=0iLOWPEDll+!3{JWo~JDFL%kX)rxuFZq(x`v>* zECh8H4uOt;t7yd*4YD9FCn%?ql|G1=FgMBvZzEfnyiW& z<~h;#W4g)fc?alaE&`PXXXEctY2e(JiBmgk$iSu3;r~9#++YH2*B;Q!jvy3IsU-=k z)?=jLZ~vDy8n7XYYMW|`k$nMMz1$|mhXUd zTr!+2*atqFmg6z)eURch3=J}u(z>QYBtM(C7H`gE7OhjlIVHE5-79&zG&q4=iuy@% zgIehA>hGjO`2-DAj=-}UDS3Wq3$AIF2VE6wIPI^1G3Oajr<(9}UMTER&=ulB7*tt# z!^pHac#v&|pC`scgv&psc}F7MZ2zM4fWF{t3}Nw!&vtC@b_O3~2`W97gKM)1q;0rC z8;Tj|Wq&a{k9b4dw|s6wnIklH8k1*F)G%_EGs(U$hc(xYsS^pdiao+J4YvfovD+~^ z`s;kO61sXT%N8JW4awg+Q{esyD~$(D;PBinWbV~nq@7mKsHQwJAY?fD$@(Do{uW6o zUV+T03>zJLe2F1{tvAvIRcx+yOy`#w1?F zjl}3`a&y-=G8TSANo!v$ryHC=`E5C+;WNIGs`u`&_2z48@oyQZ3Ou$4(rnbb;sATD z?nW=sZ|YRL7@a5fk~7o|6~4w1S+$%14swM0d+hU-15Xe7XMY?(o z@m!fX>^f_VS%w$rCFgGLy`vQ!r=?Bua+h<%e8mu6>x=i)!yz}!0X6PAU|*01swS#J zP2LYWCP7-dQ!)dK(=6dzn}iw)ygmt^NFP@2(Mg#6 zYbeedph{0)iJ%JO92v)n3G{JK9VyC;!`O~gFgj+7H;>K+vN;W1E~LP@5M|oKA0ZE= z?-=)LFS7q=7x5XQLJnHb1xffM1hWC9vGv=j?LMBHpBqhF)=B96g|hJQ-(8}W9>`T4 zK*9U@)moQNp+|$ZcLm7d1>K8u>pt^?f#A=xvDEjp={}(OA$wiNcMO%&rbg zl0U%$Q}>zEV;;kC`jt6!Vb9Of(Cf3fhYZW<9!sNhv<{Mi28l$)d%Af|rv|~tWo@);fii4clZ0=I{NS=-D9V}&`7l5oo*z3# zyLFm~sl!BEQfCL!5fg;k&JC{Hug1HwD?vLZ13&*w1C?2WLF;lNnl!k9y_^?n99~14 zTc25FX)eb@kMqd3wN;dT8iK`#5SmD}OT2N^VpA04e!N7l7i@i`4Ey5WLI` zLR)tQM&a90=9QWwdR`GhS8V|~Kj18l^!i;oe_uNdR2ob?PHq7AkZa`S+Y~zFsKB@M zbI18!si1cz2}5e`(Nnr!FxNH~MUUcO#yDHB*44n86bXdshTsU!76N(!)&DI5-~8pc z-$w>Z-0aDTuNr9HY6)Xw9Z_$*J^UPQkF{GC!tW1p^sGb=v~&}3SNmFUPH@GKWwJE) zz2H!_w}mCYO<{V>5WMiw5Hs^_VVB8bxSD5)`PC^9H7f~sZo9{M4%$uY4`$QG-aW+6 z@fbI^I*^{09A$hDPUa4{OvAO0#^SQ%EA;2VfzT;?nLbsV4V@DXQ7^Fq7_UFX>Dz6@ z!)~fXy<;Nc{w~OqXcAfm%GmH%C$s@I?vhYlI zDNOjHjQwZ*z%Eyp{2nTT1{q~MlH&+kJ(@V|&}wvl)J!I?^TNEz;jsMGK5!G*wHH-$ z;Kz9#sLPIq8OuVk`rH<RNz(B@8Op0Wj<6CGUVmuX<{eV3SPo}kj)C8TC< z1wEkASo+#|44MC;fj%C3l{`MV8RmZohf3YAG*vqk#>|c-_ln$L?_eG7%eMufE}nq_ za(l_-_?y>(151 z$wAEIOVmM08Ajw4QRU?0q)qn_vDlkUTK$&5p_jA4bkZXFyVxGaS6Jb`kAb-OtPzyB zuftY>9VjyNLaJi|JDt2S>e+1?c9~%5D-|I}n&X&1njj+MZIk{Z zfBUCl?F4NI9Q=@YEzg6;1txf4MGo}o=U98~O~X41-K5ep8CT!F#8i@*Ag3`E3o>TI z?CwuQJJSfxgamU_HV%ih`#xBqun*B=3`|$e!4Z>2Kuz{ExNfY7Bh`!`H`9S)NvjDrOe6Ik#zehBPRJ@3uD)qg1*fqWSMOWULjAZ zxvY>EstsYPz~Q{8t&fk5-fC&EN8CQgCY*B`koWX4zwB6mxYAy24J zHY|#V+nZ)ty?bt9{k-BIQy*?fO;Uf;Pc0evpwJxjeRp7ztOYmCw2pKfm!mQsGYERD z0-xqYP~5i$M+PQBQTJ*pvwsYBeb~f=HJW0_3V+C(-b|V#iQFK8Et@xFGu@oJ8k=UU z0hiQOnE!GB{MnaA-a2lfUEz`B!VVKOGghWUBLCwe{C3dpmx@H^yATsqhFhDdDKYP2 z_cK%F-;sayZ#eU&H)N=FHht=GfE#O3M55-%(WKtjR^`)5$)?1w+>1#{pslI_^J@XV z@7G0XC4fTjEIi_xggt`KxFtFcHEiQygl0O7?+?dS+6k~|P$u+-=fax0G^7T*;H+B| zs5b8e3^K*>-{ZjhqZzDww;J3IkHN#+=R>6IQ>y7uLl>QVMpVSl=$s*HSgBx&!Jif2 zV~-)`{hk1$w-*t88-46OzKhJ|6Y=Uy9~>IB20HSbu{8TQ{qySzGww|R>B{4+R|mET zE{SpYuPuSh7_Nk4+2y5aMh028r$X|ku#0>V7`oGTR|(v22k`zH2k&c>VIx># z&hVOx%d0|cJOsMG#z^^$R%|6_~<-e&R^oG#$sCS3Xh$M*1oq-1x6JW?hd**Ig zHAyc~rJvS5BDardpq_O-o!;9`+{edJ^FRmu>%!puIr?DRFb6kp8Un?)%;3(_-_-k| zF?i^Oq0O=+T)m6HNrMp##)rbf@A6nTW*z7!d?oI;=7G8AO*%lRwLdMyE&EH&qP20O;Vbg@vjHv*_&}$_!JGudPQ@mN=a$hA3AMPDzv6% zL2+~np8k{pxvJr~qkT8}PWOWR%^B#l-v>Q%Vo+oFY>+(JgjKWj$@egGa=dXm_17~a zO9cyKaqLJ)%)P<&E6V_{W(%3c8zFR{IR@2kfnSaR%t*^U+=uSH+|3t8v`Td@Zc15# ztw&7ZVxlKrk$k4!pJgy_;Q^vyqJV*y+i6jw9rmS~kPkt#QT%5b`C8A>*z^rF^8PuR x?B<3)8vJqIx!Ev@ti;2Uj??-lgRT8iJDIG=VRU|-B}CP4gLS+je$tMF{{hM@-0lDX diff --git a/diffusers-0.27.0/examples/text_to_image/run-4.sh b/diffusers-0.27.0/examples/text_to_image/run-4.sh index bfa0d92..1adb4f7 100755 --- a/diffusers-0.27.0/examples/text_to_image/run-4.sh +++ b/diffusers-0.27.0/examples/text_to_image/run-4.sh @@ -1,11 +1,11 @@ -export MODEL_NAME="/path/to/stable-diffusion-2-1-base/" +# source /public/home/chenxi/hopt/dtk24042-miopen.sh +export MODEL_NAME="/path/to/stable-diffusion-2-1-base" export OUTPUT_DIR="./output/sd2.1" -export DATASET_NAME="/path/to/data/train-00000-of-00001-566cc9b19d7203f8.parquet" +export DATASET_NAME="/path/to/train-00000-of-00001-566cc9b19d7203f8.parquet" #export DATASET_NAME="./pokemon-blip-captions" -#export HIP_VISIBLE_DEVICES=7 +export HIP_VISIBLE_DEVICES=4,5,6,7 #export LD_LIBRARY_PATH=/opt/rocblas-install/lib/:$LD_LIBRARY_PATH #export PYTORCH_HIP_ALLOC_CONF=max_split_size_mb:25314 -export HIP_VISIBLE_DEVICES=4,5,6,7 torchrun --nproc_per_node=4 train_text_to_image_lora.py \ --pretrained_model_name_or_path=$MODEL_NAME \ @@ -14,7 +14,7 @@ torchrun --nproc_per_node=4 train_text_to_image_lora.py \ --resolution=960 \ --center_crop \ --random_flip \ - --train_batch_size=8 \ + --train_batch_size=2 \ --gradient_accumulation_steps=4 \ --max_train_steps=101 \ --learning_rate=1e-04 \ diff --git a/diffusers-0.27.0/examples/text_to_image/run.sh b/diffusers-0.27.0/examples/text_to_image/run.sh index 4a3d69d..1f7c298 100755 --- a/diffusers-0.27.0/examples/text_to_image/run.sh +++ b/diffusers-0.27.0/examples/text_to_image/run.sh @@ -1,20 +1,20 @@ # source /public/home/chenxi/hopt/dtk24042-miopen.sh -export MODEL_NAME="/path/to/stable-diffusion-2-1-base/" +export MODEL_NAME="/path/to/stable-diffusion-2-1-base" export OUTPUT_DIR="./output/sd2.1" -export DATASET_NAME="/path/to/data/train-00000-of-00001-566cc9b19d7203f8.parquet" +export DATASET_NAME="path/to/train-00000-of-00001-566cc9b19d7203f8.parquet" #export DATASET_NAME="./pokemon-blip-captions" -export HIP_VISIBLE_DEVICES=4,5,6,7 +export HIP_VISIBLE_DEVICES=7 #export LD_LIBRARY_PATH=/opt/rocblas-install/lib/:$LD_LIBRARY_PATH #export PYTORCH_HIP_ALLOC_CONF=max_split_size_mb:25314 -python train_text_to_image_lora.py \ +python train_text_to_image_lora.py \ --pretrained_model_name_or_path=$MODEL_NAME \ --dataset_name=$DATASET_NAME \ --dataloader_num_workers=8 \ --resolution=960 \ --center_crop \ --random_flip \ - --train_batch_size=2 \ + --train_batch_size=8 \ --gradient_accumulation_steps=4 \ --max_train_steps=101 \ --learning_rate=1e-04 \ diff --git a/diffusers-0.27.0/examples/text_to_image/train_text_to_image_lora.py b/diffusers-0.27.0/examples/text_to_image/train_text_to_image_lora.py index adeef7b..54cd0a6 100755 --- a/diffusers-0.27.0/examples/text_to_image/train_text_to_image_lora.py +++ b/diffusers-0.27.0/examples/text_to_image/train_text_to_image_lora.py @@ -766,7 +766,7 @@ def main(): for epoch in range(first_epoch, args.num_train_epochs): unet.train() train_loss = 0.0 - # from layer_check_pt import acc_check_hook, register_hook + from layer_check_pt import acc_check_hook, register_hook for step, batch in enumerate(train_dataloader): with accelerator.accumulate(unet): # Convert images to latent space diff --git a/diffusers-0.27.0/examples/textual_inversion/README.md b/diffusers-0.27.0/examples/textual_inversion/README.md new file mode 100755 index 0000000..9e3a622 --- /dev/null +++ b/diffusers-0.27.0/examples/textual_inversion/README.md @@ -0,0 +1,150 @@ +## Textual Inversion fine-tuning example + +[Textual inversion](https://arxiv.org/abs/2208.01618) is a method to personalize text2image models like stable diffusion on your own images using just 3-5 examples. +The `textual_inversion.py` script shows how to implement the training procedure and adapt it for stable diffusion. + +## Running on Colab + +Colab for training +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/sd_textual_inversion_training.ipynb) + +Colab for inference +[![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/stable_conceptualizer_inference.ipynb) + +## Running locally with PyTorch +### Installing the dependencies + +Before running the scripts, make sure to install the library's training dependencies: + +**Important** + +To make sure you can successfully run the latest versions of the example scripts, we highly recommend **installing from source** and keeping the install up to date as we update the example scripts frequently and install some example-specific requirements. To do this, execute the following steps in a new virtual environment: +```bash +git clone https://github.com/huggingface/diffusers +cd diffusers +pip install . +``` + +Then cd in the example folder and run: +```bash +pip install -r requirements.txt +``` + +And initialize an [🤗 Accelerate](https://github.com/huggingface/accelerate/) environment with: + +```bash +accelerate config +``` + +### Cat toy example + +First, let's login so that we can upload the checkpoint to the Hub during training: + +```bash +huggingface-cli login +``` + +Now let's get our dataset. For this example we will use some cat images: https://huggingface.co/datasets/diffusers/cat_toy_example . + +Let's first download it locally: + +```py +from huggingface_hub import snapshot_download + +local_dir = "./cat" +snapshot_download("diffusers/cat_toy_example", local_dir=local_dir, repo_type="dataset", ignore_patterns=".gitattributes") +``` + +This will be our training data. +Now we can launch the training using: + +**___Note: Change the `resolution` to 768 if you are using the [stable-diffusion-2](https://huggingface.co/stabilityai/stable-diffusion-2) 768x768 model.___** + +**___Note: Please follow the [README_sdxl.md](./README_sdxl.md) if you are using the [stable-diffusion-xl](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0).___** + +```bash +export MODEL_NAME="runwayml/stable-diffusion-v1-5" +export DATA_DIR="./cat" + +accelerate launch textual_inversion.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATA_DIR \ + --learnable_property="object" \ + --placeholder_token="" \ + --initializer_token="toy" \ + --resolution=512 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=4 \ + --max_train_steps=3000 \ + --learning_rate=5.0e-04 \ + --scale_lr \ + --lr_scheduler="constant" \ + --lr_warmup_steps=0 \ + --push_to_hub \ + --output_dir="textual_inversion_cat" +``` + +A full training run takes ~1 hour on one V100 GPU. + +**Note**: As described in [the official paper](https://arxiv.org/abs/2208.01618) +only one embedding vector is used for the placeholder token, *e.g.* `""`. +However, one can also add multiple embedding vectors for the placeholder token +to increase the number of fine-tuneable parameters. This can help the model to learn +more complex details. To use multiple embedding vectors, you should define `--num_vectors` +to a number larger than one, *e.g.*: +```bash +--num_vectors 5 +``` + +The saved textual inversion vectors will then be larger in size compared to the default case. + +### Inference + +Once you have trained a model using above command, the inference can be done simply using the `StableDiffusionPipeline`. Make sure to include the `placeholder_token` in your prompt. + +```python +from diffusers import StableDiffusionPipeline +import torch + +model_id = "path-to-your-trained-model" +pipe = StableDiffusionPipeline.from_pretrained(model_id,torch_dtype=torch.float16).to("cuda") + +prompt = "A backpack" + +image = pipe(prompt, num_inference_steps=50, guidance_scale=7.5).images[0] + +image.save("cat-backpack.png") +``` + + +## Training with Flax/JAX + +For faster training on TPUs and GPUs you can leverage the flax training example. Follow the instructions above to get the model and dataset before running the script. + +Before running the scripts, make sure to install the library's training dependencies: + +```bash +pip install -U -r requirements_flax.txt +``` + +```bash +export MODEL_NAME="duongna/stable-diffusion-v1-4-flax" +export DATA_DIR="path-to-dir-containing-images" + +python textual_inversion_flax.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATA_DIR \ + --learnable_property="object" \ + --placeholder_token="" \ + --initializer_token="toy" \ + --resolution=512 \ + --train_batch_size=1 \ + --max_train_steps=3000 \ + --learning_rate=5.0e-04 \ + --scale_lr \ + --output_dir="textual_inversion_cat" +``` +It should be at least 70% faster than the PyTorch script with the same configuration. + +### Training with xformers: +You can enable memory efficient attention by [installing xFormers](https://github.com/facebookresearch/xformers#installing-xformers) and padding the `--enable_xformers_memory_efficient_attention` argument to the script. This is not available with the Flax/JAX implementation. diff --git a/diffusers-0.27.0/examples/textual_inversion/README_sdxl.md b/diffusers-0.27.0/examples/textual_inversion/README_sdxl.md new file mode 100755 index 0000000..2c1c80f --- /dev/null +++ b/diffusers-0.27.0/examples/textual_inversion/README_sdxl.md @@ -0,0 +1,26 @@ +## Textual Inversion fine-tuning example for SDXL + +``` +export MODEL_NAME="stabilityai/stable-diffusion-xl-base-1.0" +export DATA_DIR="./cat" + +accelerate launch textual_inversion_sdxl.py \ + --pretrained_model_name_or_path=$MODEL_NAME \ + --train_data_dir=$DATA_DIR \ + --learnable_property="object" \ + --placeholder_token="" \ + --initializer_token="toy" \ + --mixed_precision="bf16" \ + --resolution=768 \ + --train_batch_size=1 \ + --gradient_accumulation_steps=4 \ + --max_train_steps=500 \ + --learning_rate=5.0e-04 \ + --scale_lr \ + --lr_scheduler="constant" \ + --lr_warmup_steps=0 \ + --save_as_full_pipeline \ + --output_dir="./textual_inversion_cat_sdxl" +``` + +For now, only training of the first text encoder is supported. \ No newline at end of file diff --git a/diffusers-0.27.0/examples/textual_inversion/requirements.txt b/diffusers-0.27.0/examples/textual_inversion/requirements.txt new file mode 100755 index 0000000..7a61298 --- /dev/null +++ b/diffusers-0.27.0/examples/textual_inversion/requirements.txt @@ -0,0 +1,6 @@ +accelerate>=0.16.0 +torchvision +transformers>=4.25.1 +ftfy +tensorboard +Jinja2 diff --git a/diffusers-0.27.0/examples/textual_inversion/requirements_flax.txt b/diffusers-0.27.0/examples/textual_inversion/requirements_flax.txt new file mode 100755 index 0000000..8f85ad5 --- /dev/null +++ b/diffusers-0.27.0/examples/textual_inversion/requirements_flax.txt @@ -0,0 +1,8 @@ +transformers>=4.25.1 +flax +optax +torch +torchvision +ftfy +tensorboard +Jinja2 diff --git a/diffusers-0.27.0/examples/textual_inversion/test_textual_inversion.py b/diffusers-0.27.0/examples/textual_inversion/test_textual_inversion.py new file mode 100755 index 0000000..fa0b2c2 --- /dev/null +++ b/diffusers-0.27.0/examples/textual_inversion/test_textual_inversion.py @@ -0,0 +1,152 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import sys +import tempfile + + +sys.path.append("..") +from test_examples_utils import ExamplesTestsAccelerate, run_command # noqa: E402 + + +logging.basicConfig(level=logging.DEBUG) + +logger = logging.getLogger() +stream_handler = logging.StreamHandler(sys.stdout) +logger.addHandler(stream_handler) + + +class TextualInversion(ExamplesTestsAccelerate): + def test_textual_inversion(self): + with tempfile.TemporaryDirectory() as tmpdir: + test_args = f""" + examples/textual_inversion/textual_inversion.py + --pretrained_model_name_or_path hf-internal-testing/tiny-stable-diffusion-pipe + --train_data_dir docs/source/en/imgs + --learnable_property object + --placeholder_token + --initializer_token a + --save_steps 1 + --num_vectors 2 + --resolution 64 + --train_batch_size 1 + --gradient_accumulation_steps 1 + --max_train_steps 2 + --learning_rate 5.0e-04 + --scale_lr + --lr_scheduler constant + --lr_warmup_steps 0 + --output_dir {tmpdir} + """.split() + + run_command(self._launch_args + test_args) + # save_pretrained smoke test + self.assertTrue(os.path.isfile(os.path.join(tmpdir, "learned_embeds.safetensors"))) + + def test_textual_inversion_checkpointing(self): + with tempfile.TemporaryDirectory() as tmpdir: + test_args = f""" + examples/textual_inversion/textual_inversion.py + --pretrained_model_name_or_path hf-internal-testing/tiny-stable-diffusion-pipe + --train_data_dir docs/source/en/imgs + --learnable_property object + --placeholder_token + --initializer_token a + --save_steps 1 + --num_vectors 2 + --resolution 64 + --train_batch_size 1 + --gradient_accumulation_steps 1 + --max_train_steps 3 + --learning_rate 5.0e-04 + --scale_lr + --lr_scheduler constant + --lr_warmup_steps 0 + --output_dir {tmpdir} + --checkpointing_steps=1 + --checkpoints_total_limit=2 + """.split() + + run_command(self._launch_args + test_args) + + # check checkpoint directories exist + self.assertEqual( + {x for x in os.listdir(tmpdir) if "checkpoint" in x}, + {"checkpoint-2", "checkpoint-3"}, + ) + + def test_textual_inversion_checkpointing_checkpoints_total_limit_removes_multiple_checkpoints(self): + with tempfile.TemporaryDirectory() as tmpdir: + test_args = f""" + examples/textual_inversion/textual_inversion.py + --pretrained_model_name_or_path hf-internal-testing/tiny-stable-diffusion-pipe + --train_data_dir docs/source/en/imgs + --learnable_property object + --placeholder_token + --initializer_token a + --save_steps 1 + --num_vectors 2 + --resolution 64 + --train_batch_size 1 + --gradient_accumulation_steps 1 + --max_train_steps 2 + --learning_rate 5.0e-04 + --scale_lr + --lr_scheduler constant + --lr_warmup_steps 0 + --output_dir {tmpdir} + --checkpointing_steps=1 + """.split() + + run_command(self._launch_args + test_args) + + # check checkpoint directories exist + self.assertEqual( + {x for x in os.listdir(tmpdir) if "checkpoint" in x}, + {"checkpoint-1", "checkpoint-2"}, + ) + + resume_run_args = f""" + examples/textual_inversion/textual_inversion.py + --pretrained_model_name_or_path hf-internal-testing/tiny-stable-diffusion-pipe + --train_data_dir docs/source/en/imgs + --learnable_property object + --placeholder_token + --initializer_token a + --save_steps 1 + --num_vectors 2 + --resolution 64 + --train_batch_size 1 + --gradient_accumulation_steps 1 + --max_train_steps 2 + --learning_rate 5.0e-04 + --scale_lr + --lr_scheduler constant + --lr_warmup_steps 0 + --output_dir {tmpdir} + --checkpointing_steps=1 + --resume_from_checkpoint=checkpoint-2 + --checkpoints_total_limit=2 + """.split() + + run_command(self._launch_args + resume_run_args) + + # check checkpoint directories exist + self.assertEqual( + {x for x in os.listdir(tmpdir) if "checkpoint" in x}, + {"checkpoint-2", "checkpoint-3"}, + ) diff --git a/diffusers-0.27.0/examples/textual_inversion/test_textual_inversion_sdxl.py b/diffusers-0.27.0/examples/textual_inversion/test_textual_inversion_sdxl.py new file mode 100755 index 0000000..a861708 --- /dev/null +++ b/diffusers-0.27.0/examples/textual_inversion/test_textual_inversion_sdxl.py @@ -0,0 +1,152 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import sys +import tempfile + + +sys.path.append("..") +from test_examples_utils import ExamplesTestsAccelerate, run_command # noqa: E402 + + +logging.basicConfig(level=logging.DEBUG) + +logger = logging.getLogger() +stream_handler = logging.StreamHandler(sys.stdout) +logger.addHandler(stream_handler) + + +class TextualInversionSdxl(ExamplesTestsAccelerate): + def test_textual_inversion_sdxl(self): + with tempfile.TemporaryDirectory() as tmpdir: + test_args = f""" + examples/textual_inversion/textual_inversion_sdxl.py + --pretrained_model_name_or_path hf-internal-testing/tiny-sdxl-pipe + --train_data_dir docs/source/en/imgs + --learnable_property object + --placeholder_token + --initializer_token a + --save_steps 1 + --num_vectors 2 + --resolution 64 + --train_batch_size 1 + --gradient_accumulation_steps 1 + --max_train_steps 2 + --learning_rate 5.0e-04 + --scale_lr + --lr_scheduler constant + --lr_warmup_steps 0 + --output_dir {tmpdir} + """.split() + + run_command(self._launch_args + test_args) + # save_pretrained smoke test + self.assertTrue(os.path.isfile(os.path.join(tmpdir, "learned_embeds.safetensors"))) + + def test_textual_inversion_sdxl_checkpointing(self): + with tempfile.TemporaryDirectory() as tmpdir: + test_args = f""" + examples/textual_inversion/textual_inversion_sdxl.py + --pretrained_model_name_or_path hf-internal-testing/tiny-sdxl-pipe + --train_data_dir docs/source/en/imgs + --learnable_property object + --placeholder_token + --initializer_token a + --save_steps 1 + --num_vectors 2 + --resolution 64 + --train_batch_size 1 + --gradient_accumulation_steps 1 + --max_train_steps 3 + --learning_rate 5.0e-04 + --scale_lr + --lr_scheduler constant + --lr_warmup_steps 0 + --output_dir {tmpdir} + --checkpointing_steps=1 + --checkpoints_total_limit=2 + """.split() + + run_command(self._launch_args + test_args) + + # check checkpoint directories exist + self.assertEqual( + {x for x in os.listdir(tmpdir) if "checkpoint" in x}, + {"checkpoint-2", "checkpoint-3"}, + ) + + def test_textual_inversion_sdxl_checkpointing_checkpoints_total_limit_removes_multiple_checkpoints(self): + with tempfile.TemporaryDirectory() as tmpdir: + test_args = f""" + examples/textual_inversion/textual_inversion_sdxl.py + --pretrained_model_name_or_path hf-internal-testing/tiny-sdxl-pipe + --train_data_dir docs/source/en/imgs + --learnable_property object + --placeholder_token + --initializer_token a + --save_steps 1 + --num_vectors 2 + --resolution 64 + --train_batch_size 1 + --gradient_accumulation_steps 1 + --max_train_steps 2 + --learning_rate 5.0e-04 + --scale_lr + --lr_scheduler constant + --lr_warmup_steps 0 + --output_dir {tmpdir} + --checkpointing_steps=1 + """.split() + + run_command(self._launch_args + test_args) + + # check checkpoint directories exist + self.assertEqual( + {x for x in os.listdir(tmpdir) if "checkpoint" in x}, + {"checkpoint-1", "checkpoint-2"}, + ) + + resume_run_args = f""" + examples/textual_inversion/textual_inversion_sdxl.py + --pretrained_model_name_or_path hf-internal-testing/tiny-sdxl-pipe + --train_data_dir docs/source/en/imgs + --learnable_property object + --placeholder_token + --initializer_token a + --save_steps 1 + --num_vectors 2 + --resolution 64 + --train_batch_size 1 + --gradient_accumulation_steps 1 + --max_train_steps 2 + --learning_rate 5.0e-04 + --scale_lr + --lr_scheduler constant + --lr_warmup_steps 0 + --output_dir {tmpdir} + --checkpointing_steps=1 + --resume_from_checkpoint=checkpoint-2 + --checkpoints_total_limit=2 + """.split() + + run_command(self._launch_args + resume_run_args) + + # check checkpoint directories exist + self.assertEqual( + {x for x in os.listdir(tmpdir) if "checkpoint" in x}, + {"checkpoint-2", "checkpoint-3"}, + ) diff --git a/diffusers-0.27.0/examples/textual_inversion/textual_inversion.py b/diffusers-0.27.0/examples/textual_inversion/textual_inversion.py new file mode 100755 index 0000000..460c0de --- /dev/null +++ b/diffusers-0.27.0/examples/textual_inversion/textual_inversion.py @@ -0,0 +1,1012 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import logging +import math +import os +import random +import shutil +import warnings +from pathlib import Path + +import numpy as np +import PIL +import safetensors +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import ProjectConfiguration, set_seed +from huggingface_hub import create_repo, upload_folder + +# TODO: remove and import from diffusers.utils when the new version of diffusers is released +from packaging import version +from PIL import Image +from torch.utils.data import Dataset +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import CLIPTextModel, CLIPTokenizer + +import diffusers +from diffusers import ( + AutoencoderKL, + DDPMScheduler, + DiffusionPipeline, + DPMSolverMultistepScheduler, + StableDiffusionPipeline, + UNet2DConditionModel, +) +from diffusers.optimization import get_scheduler +from diffusers.utils import check_min_version, is_wandb_available +from diffusers.utils.hub_utils import load_or_create_model_card, populate_model_card +from diffusers.utils.import_utils import is_xformers_available + + +if is_wandb_available(): + import wandb + +if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"): + PIL_INTERPOLATION = { + "linear": PIL.Image.Resampling.BILINEAR, + "bilinear": PIL.Image.Resampling.BILINEAR, + "bicubic": PIL.Image.Resampling.BICUBIC, + "lanczos": PIL.Image.Resampling.LANCZOS, + "nearest": PIL.Image.Resampling.NEAREST, + } +else: + PIL_INTERPOLATION = { + "linear": PIL.Image.LINEAR, + "bilinear": PIL.Image.BILINEAR, + "bicubic": PIL.Image.BICUBIC, + "lanczos": PIL.Image.LANCZOS, + "nearest": PIL.Image.NEAREST, + } +# ------------------------------------------------------------------------------ + + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.27.0") + +logger = get_logger(__name__) + + +def save_model_card(repo_id: str, images: list = None, base_model: str = None, repo_folder: str = None): + img_str = "" + if images is not None: + for i, image in enumerate(images): + image.save(os.path.join(repo_folder, f"image_{i}.png")) + img_str += f"![img_{i}](./image_{i}.png)\n" + model_description = f""" +# Textual inversion text2image fine-tuning - {repo_id} +These are textual inversion adaption weights for {base_model}. You can find some example images in the following. \n +{img_str} +""" + model_card = load_or_create_model_card( + repo_id_or_path=repo_id, + from_training=True, + license="creativeml-openrail-m", + base_model=base_model, + model_description=model_description, + inference=True, + ) + + tags = [ + "stable-diffusion", + "stable-diffusion-diffusers", + "text-to-image", + "diffusers", + "textual_inversion", + "diffusers-training", + ] + model_card = populate_model_card(model_card, tags=tags) + + model_card.save(os.path.join(repo_folder, "README.md")) + + +def log_validation(text_encoder, tokenizer, unet, vae, args, accelerator, weight_dtype, epoch): + logger.info( + f"Running validation... \n Generating {args.num_validation_images} images with prompt:" + f" {args.validation_prompt}." + ) + # create pipeline (note: unet and vae are loaded again in float32) + pipeline = DiffusionPipeline.from_pretrained( + args.pretrained_model_name_or_path, + text_encoder=accelerator.unwrap_model(text_encoder), + tokenizer=tokenizer, + unet=unet, + vae=vae, + safety_checker=None, + revision=args.revision, + variant=args.variant, + torch_dtype=weight_dtype, + ) + pipeline.scheduler = DPMSolverMultistepScheduler.from_config(pipeline.scheduler.config) + pipeline = pipeline.to(accelerator.device) + pipeline.set_progress_bar_config(disable=True) + + # run inference + generator = None if args.seed is None else torch.Generator(device=accelerator.device).manual_seed(args.seed) + images = [] + for _ in range(args.num_validation_images): + with torch.autocast("cuda"): + image = pipeline(args.validation_prompt, num_inference_steps=25, generator=generator).images[0] + images.append(image) + + for tracker in accelerator.trackers: + if tracker.name == "tensorboard": + np_images = np.stack([np.asarray(img) for img in images]) + tracker.writer.add_images("validation", np_images, epoch, dataformats="NHWC") + if tracker.name == "wandb": + tracker.log( + { + "validation": [ + wandb.Image(image, caption=f"{i}: {args.validation_prompt}") for i, image in enumerate(images) + ] + } + ) + + del pipeline + torch.cuda.empty_cache() + return images + + +def save_progress(text_encoder, placeholder_token_ids, accelerator, args, save_path, safe_serialization=True): + logger.info("Saving embeddings") + learned_embeds = ( + accelerator.unwrap_model(text_encoder) + .get_input_embeddings() + .weight[min(placeholder_token_ids) : max(placeholder_token_ids) + 1] + ) + learned_embeds_dict = {args.placeholder_token: learned_embeds.detach().cpu()} + + if safe_serialization: + safetensors.torch.save_file(learned_embeds_dict, save_path, metadata={"format": "pt"}) + else: + torch.save(learned_embeds_dict, save_path) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--save_steps", + type=int, + default=500, + help="Save learned_embeds.bin every X updates steps.", + ) + parser.add_argument( + "--save_as_full_pipeline", + action="store_true", + help="Save the complete stable diffusion pipeline.", + ) + parser.add_argument( + "--num_vectors", + type=int, + default=1, + help="How many textual inversion vectors shall be used to learn the concept.", + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--tokenizer_name", + type=str, + default=None, + help="Pretrained tokenizer name or path if not the same as model_name", + ) + parser.add_argument( + "--train_data_dir", type=str, default=None, required=True, help="A folder containing the training data." + ) + parser.add_argument( + "--placeholder_token", + type=str, + default=None, + required=True, + help="A token to use as a placeholder for the concept.", + ) + parser.add_argument( + "--initializer_token", type=str, default=None, required=True, help="A token to use as initializer word." + ) + parser.add_argument("--learnable_property", type=str, default="object", help="Choose between 'object' and 'style'") + parser.add_argument("--repeats", type=int, default=100, help="How many times to repeat the training data.") + parser.add_argument( + "--output_dir", + type=str, + default="text-inversion-model", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--resolution", + type=int, + default=512, + help=( + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" + ), + ) + parser.add_argument( + "--center_crop", action="store_true", help="Whether to center crop images before resizing to resolution." + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=5000, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--lr_num_cycles", + type=int, + default=1, + help="Number of hard resets of the lr in cosine_with_restarts scheduler.", + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default="no", + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose" + "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10." + "and Nvidia Ampere GPU or Intel Gen 4 Xeon (and later) ." + ), + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument( + "--validation_prompt", + type=str, + default=None, + help="A prompt that is used during validation to verify that the model is learning.", + ) + parser.add_argument( + "--num_validation_images", + type=int, + default=4, + help="Number of images that should be generated during validation with `validation_prompt`.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=100, + help=( + "Run validation every X steps. Validation consists of running the prompt" + " `args.validation_prompt` multiple times: `args.num_validation_images`" + " and logging the images." + ), + ) + parser.add_argument( + "--validation_epochs", + type=int, + default=None, + help=( + "Deprecated in favor of validation_steps. Run validation every X epochs. Validation consists of running the prompt" + " `args.validation_prompt` multiple times: `args.num_validation_images`" + " and logging the images." + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--enable_xformers_memory_efficient_attention", action="store_true", help="Whether or not to use xformers." + ) + parser.add_argument( + "--no_safe_serialization", + action="store_true", + help="If specified save the checkpoint not in `safetensors` format, but in original PyTorch format instead.", + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + if args.train_data_dir is None: + raise ValueError("You must specify a train data directory.") + + return args + + +imagenet_templates_small = [ + "a photo of a {}", + "a rendering of a {}", + "a cropped photo of the {}", + "the photo of a {}", + "a photo of a clean {}", + "a photo of a dirty {}", + "a dark photo of the {}", + "a photo of my {}", + "a photo of the cool {}", + "a close-up photo of a {}", + "a bright photo of the {}", + "a cropped photo of a {}", + "a photo of the {}", + "a good photo of the {}", + "a photo of one {}", + "a close-up photo of the {}", + "a rendition of the {}", + "a photo of the clean {}", + "a rendition of a {}", + "a photo of a nice {}", + "a good photo of a {}", + "a photo of the nice {}", + "a photo of the small {}", + "a photo of the weird {}", + "a photo of the large {}", + "a photo of a cool {}", + "a photo of a small {}", +] + +imagenet_style_templates_small = [ + "a painting in the style of {}", + "a rendering in the style of {}", + "a cropped painting in the style of {}", + "the painting in the style of {}", + "a clean painting in the style of {}", + "a dirty painting in the style of {}", + "a dark painting in the style of {}", + "a picture in the style of {}", + "a cool painting in the style of {}", + "a close-up painting in the style of {}", + "a bright painting in the style of {}", + "a cropped painting in the style of {}", + "a good painting in the style of {}", + "a close-up painting in the style of {}", + "a rendition in the style of {}", + "a nice painting in the style of {}", + "a small painting in the style of {}", + "a weird painting in the style of {}", + "a large painting in the style of {}", +] + + +class TextualInversionDataset(Dataset): + def __init__( + self, + data_root, + tokenizer, + learnable_property="object", # [object, style] + size=512, + repeats=100, + interpolation="bicubic", + flip_p=0.5, + set="train", + placeholder_token="*", + center_crop=False, + ): + self.data_root = data_root + self.tokenizer = tokenizer + self.learnable_property = learnable_property + self.size = size + self.placeholder_token = placeholder_token + self.center_crop = center_crop + self.flip_p = flip_p + + self.image_paths = [os.path.join(self.data_root, file_path) for file_path in os.listdir(self.data_root)] + + self.num_images = len(self.image_paths) + self._length = self.num_images + + if set == "train": + self._length = self.num_images * repeats + + self.interpolation = { + "linear": PIL_INTERPOLATION["linear"], + "bilinear": PIL_INTERPOLATION["bilinear"], + "bicubic": PIL_INTERPOLATION["bicubic"], + "lanczos": PIL_INTERPOLATION["lanczos"], + }[interpolation] + + self.templates = imagenet_style_templates_small if learnable_property == "style" else imagenet_templates_small + self.flip_transform = transforms.RandomHorizontalFlip(p=self.flip_p) + + def __len__(self): + return self._length + + def __getitem__(self, i): + example = {} + image = Image.open(self.image_paths[i % self.num_images]) + + if not image.mode == "RGB": + image = image.convert("RGB") + + placeholder_string = self.placeholder_token + text = random.choice(self.templates).format(placeholder_string) + + example["input_ids"] = self.tokenizer( + text, + padding="max_length", + truncation=True, + max_length=self.tokenizer.model_max_length, + return_tensors="pt", + ).input_ids[0] + + # default to score-sde preprocessing + img = np.array(image).astype(np.uint8) + + if self.center_crop: + crop = min(img.shape[0], img.shape[1]) + ( + h, + w, + ) = ( + img.shape[0], + img.shape[1], + ) + img = img[(h - crop) // 2 : (h + crop) // 2, (w - crop) // 2 : (w + crop) // 2] + + image = Image.fromarray(img) + image = image.resize((self.size, self.size), resample=self.interpolation) + + image = self.flip_transform(image) + image = np.array(image).astype(np.uint8) + image = (image / 127.5 - 1.0).astype(np.float32) + + example["pixel_values"] = torch.from_numpy(image).permute(2, 0, 1) + return example + + +def main(): + args = parse_args() + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + logging_dir = os.path.join(args.output_dir, args.logging_dir) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + if args.report_to == "wandb": + if not is_wandb_available(): + raise ImportError("Make sure to install wandb if you want to use it for logging during training.") + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + + # Load tokenizer + if args.tokenizer_name: + tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_name) + elif args.pretrained_model_name_or_path: + tokenizer = CLIPTokenizer.from_pretrained(args.pretrained_model_name_or_path, subfolder="tokenizer") + + # Load scheduler and models + noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler") + text_encoder = CLIPTextModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", revision=args.revision + ) + vae = AutoencoderKL.from_pretrained( + args.pretrained_model_name_or_path, subfolder="vae", revision=args.revision, variant=args.variant + ) + unet = UNet2DConditionModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="unet", revision=args.revision, variant=args.variant + ) + + # Add the placeholder token in tokenizer + placeholder_tokens = [args.placeholder_token] + + if args.num_vectors < 1: + raise ValueError(f"--num_vectors has to be larger or equal to 1, but is {args.num_vectors}") + + # add dummy tokens for multi-vector + additional_tokens = [] + for i in range(1, args.num_vectors): + additional_tokens.append(f"{args.placeholder_token}_{i}") + placeholder_tokens += additional_tokens + + num_added_tokens = tokenizer.add_tokens(placeholder_tokens) + if num_added_tokens != args.num_vectors: + raise ValueError( + f"The tokenizer already contains the token {args.placeholder_token}. Please pass a different" + " `placeholder_token` that is not already in the tokenizer." + ) + + # Convert the initializer_token, placeholder_token to ids + token_ids = tokenizer.encode(args.initializer_token, add_special_tokens=False) + # Check if initializer_token is a single token or a sequence of tokens + if len(token_ids) > 1: + raise ValueError("The initializer token must be a single token.") + + initializer_token_id = token_ids[0] + placeholder_token_ids = tokenizer.convert_tokens_to_ids(placeholder_tokens) + + # Resize the token embeddings as we are adding new special tokens to the tokenizer + text_encoder.resize_token_embeddings(len(tokenizer)) + + # Initialise the newly added placeholder token with the embeddings of the initializer token + token_embeds = text_encoder.get_input_embeddings().weight.data + with torch.no_grad(): + for token_id in placeholder_token_ids: + token_embeds[token_id] = token_embeds[initializer_token_id].clone() + + # Freeze vae and unet + vae.requires_grad_(False) + unet.requires_grad_(False) + # Freeze all parameters except for the token embeddings in text encoder + text_encoder.text_model.encoder.requires_grad_(False) + text_encoder.text_model.final_layer_norm.requires_grad_(False) + text_encoder.text_model.embeddings.position_embedding.requires_grad_(False) + + if args.gradient_checkpointing: + # Keep unet in train mode if we are using gradient checkpointing to save memory. + # The dropout cannot be != 0 so it doesn't matter if we are in eval or train mode. + unet.train() + text_encoder.gradient_checkpointing_enable() + unet.enable_gradient_checkpointing() + + if args.enable_xformers_memory_efficient_attention: + if is_xformers_available(): + import xformers + + xformers_version = version.parse(xformers.__version__) + if xformers_version == version.parse("0.0.16"): + logger.warning( + "xFormers 0.0.16 cannot be used for training in some GPUs. If you observe problems during training, please update xFormers to at least 0.0.17. See https://huggingface.co/docs/diffusers/main/en/optimization/xformers for more details." + ) + unet.enable_xformers_memory_efficient_attention() + else: + raise ValueError("xformers is not available. Make sure it is installed correctly") + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + optimizer = torch.optim.AdamW( + text_encoder.get_input_embeddings().parameters(), # only optimize the embeddings + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Dataset and DataLoaders creation: + train_dataset = TextualInversionDataset( + data_root=args.train_data_dir, + tokenizer=tokenizer, + size=args.resolution, + placeholder_token=(" ".join(tokenizer.convert_ids_to_tokens(placeholder_token_ids))), + repeats=args.repeats, + learnable_property=args.learnable_property, + center_crop=args.center_crop, + set="train", + ) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.train_batch_size, shuffle=True, num_workers=args.dataloader_num_workers + ) + if args.validation_epochs is not None: + warnings.warn( + f"FutureWarning: You are doing logging with validation_epochs={args.validation_epochs}." + " Deprecated validation_epochs in favor of `validation_steps`" + f"Setting `args.validation_steps` to {args.validation_epochs * len(train_dataset)}", + FutureWarning, + stacklevel=2, + ) + args.validation_steps = args.validation_epochs * len(train_dataset) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + num_cycles=args.lr_num_cycles, + ) + + text_encoder.train() + # Prepare everything with our `accelerator`. + text_encoder, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + text_encoder, optimizer, train_dataloader, lr_scheduler + ) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora unet) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + + # Move vae and unet to device and cast to weight_dtype + unet.to(accelerator.device, dtype=weight_dtype) + vae.to(accelerator.device, dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + accelerator.init_trackers("textual_inversion", config=vars(args)) + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + first_epoch = global_step // num_update_steps_per_epoch + + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + # keep original embeddings as reference + orig_embeds_params = accelerator.unwrap_model(text_encoder).get_input_embeddings().weight.data.clone() + + for epoch in range(first_epoch, args.num_train_epochs): + text_encoder.train() + for step, batch in enumerate(train_dataloader): + with accelerator.accumulate(text_encoder): + # Convert images to latent space + latents = vae.encode(batch["pixel_values"].to(dtype=weight_dtype)).latent_dist.sample().detach() + latents = latents * vae.config.scaling_factor + + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=latents.device) + timesteps = timesteps.long() + + # Add noise to the latents according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + + # Get the text embedding for conditioning + encoder_hidden_states = text_encoder(batch["input_ids"])[0].to(dtype=weight_dtype) + + # Predict the noise residual + model_pred = unet(noisy_latents, timesteps, encoder_hidden_states).sample + + # Get the target for loss depending on the prediction type + if noise_scheduler.config.prediction_type == "epsilon": + target = noise + elif noise_scheduler.config.prediction_type == "v_prediction": + target = noise_scheduler.get_velocity(latents, noise, timesteps) + else: + raise ValueError(f"Unknown prediction type {noise_scheduler.config.prediction_type}") + + loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean") + + accelerator.backward(loss) + + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Let's make sure we don't update any embedding weights besides the newly added token + index_no_updates = torch.ones((len(tokenizer),), dtype=torch.bool) + index_no_updates[min(placeholder_token_ids) : max(placeholder_token_ids) + 1] = False + + with torch.no_grad(): + accelerator.unwrap_model(text_encoder).get_input_embeddings().weight[ + index_no_updates + ] = orig_embeds_params[index_no_updates] + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + images = [] + progress_bar.update(1) + global_step += 1 + if global_step % args.save_steps == 0: + weight_name = ( + f"learned_embeds-steps-{global_step}.bin" + if args.no_safe_serialization + else f"learned_embeds-steps-{global_step}.safetensors" + ) + save_path = os.path.join(args.output_dir, weight_name) + save_progress( + text_encoder, + placeholder_token_ids, + accelerator, + args, + save_path, + safe_serialization=not args.no_safe_serialization, + ) + + if accelerator.is_main_process: + if global_step % args.checkpointing_steps == 0: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if args.validation_prompt is not None and global_step % args.validation_steps == 0: + images = log_validation( + text_encoder, tokenizer, unet, vae, args, accelerator, weight_dtype, epoch + ) + + logs = {"loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + accelerator.log(logs, step=global_step) + + if global_step >= args.max_train_steps: + break + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + if args.push_to_hub and not args.save_as_full_pipeline: + logger.warning("Enabling full model saving because --push_to_hub=True was specified.") + save_full_model = True + else: + save_full_model = args.save_as_full_pipeline + if save_full_model: + pipeline = StableDiffusionPipeline.from_pretrained( + args.pretrained_model_name_or_path, + text_encoder=accelerator.unwrap_model(text_encoder), + vae=vae, + unet=unet, + tokenizer=tokenizer, + ) + pipeline.save_pretrained(args.output_dir) + # Save the newly trained embeddings + weight_name = "learned_embeds.bin" if args.no_safe_serialization else "learned_embeds.safetensors" + save_path = os.path.join(args.output_dir, weight_name) + save_progress( + text_encoder, + placeholder_token_ids, + accelerator, + args, + save_path, + safe_serialization=not args.no_safe_serialization, + ) + + if args.push_to_hub: + save_model_card( + repo_id, + images=images, + base_model=args.pretrained_model_name_or_path, + repo_folder=args.output_dir, + ) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["step_*", "epoch_*"], + ) + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/diffusers-0.27.0/examples/textual_inversion/textual_inversion_flax.py b/diffusers-0.27.0/examples/textual_inversion/textual_inversion_flax.py new file mode 100755 index 0000000..d5973da --- /dev/null +++ b/diffusers-0.27.0/examples/textual_inversion/textual_inversion_flax.py @@ -0,0 +1,681 @@ +import argparse +import logging +import math +import os +import random +from pathlib import Path + +import jax +import jax.numpy as jnp +import numpy as np +import optax +import PIL +import torch +import torch.utils.checkpoint +import transformers +from flax import jax_utils +from flax.training import train_state +from flax.training.common_utils import shard +from huggingface_hub import create_repo, upload_folder + +# TODO: remove and import from diffusers.utils when the new version of diffusers is released +from packaging import version +from PIL import Image +from torch.utils.data import Dataset +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import CLIPImageProcessor, CLIPTokenizer, FlaxCLIPTextModel, set_seed + +from diffusers import ( + FlaxAutoencoderKL, + FlaxDDPMScheduler, + FlaxPNDMScheduler, + FlaxStableDiffusionPipeline, + FlaxUNet2DConditionModel, +) +from diffusers.pipelines.stable_diffusion import FlaxStableDiffusionSafetyChecker +from diffusers.utils import check_min_version + + +if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"): + PIL_INTERPOLATION = { + "linear": PIL.Image.Resampling.BILINEAR, + "bilinear": PIL.Image.Resampling.BILINEAR, + "bicubic": PIL.Image.Resampling.BICUBIC, + "lanczos": PIL.Image.Resampling.LANCZOS, + "nearest": PIL.Image.Resampling.NEAREST, + } +else: + PIL_INTERPOLATION = { + "linear": PIL.Image.LINEAR, + "bilinear": PIL.Image.BILINEAR, + "bicubic": PIL.Image.BICUBIC, + "lanczos": PIL.Image.LANCZOS, + "nearest": PIL.Image.NEAREST, + } +# ------------------------------------------------------------------------------ + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.27.0") + +logger = logging.getLogger(__name__) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--tokenizer_name", + type=str, + default=None, + help="Pretrained tokenizer name or path if not the same as model_name", + ) + parser.add_argument( + "--train_data_dir", type=str, default=None, required=True, help="A folder containing the training data." + ) + parser.add_argument( + "--placeholder_token", + type=str, + default=None, + required=True, + help="A token to use as a placeholder for the concept.", + ) + parser.add_argument( + "--initializer_token", type=str, default=None, required=True, help="A token to use as initializer word." + ) + parser.add_argument("--learnable_property", type=str, default="object", help="Choose between 'object' and 'style'") + parser.add_argument("--repeats", type=int, default=100, help="How many times to repeat the training data.") + parser.add_argument( + "--output_dir", + type=str, + default="text-inversion-model", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument("--seed", type=int, default=42, help="A seed for reproducible training.") + parser.add_argument( + "--resolution", + type=int, + default=512, + help=( + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" + ), + ) + parser.add_argument( + "--center_crop", action="store_true", help="Whether to center crop images before resizing to resolution." + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=5000, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--save_steps", + type=int, + default=500, + help="Save learned_embeds.bin every X updates steps.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=True, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument( + "--use_auth_token", + action="store_true", + help=( + "Will use the token generated when running `huggingface-cli login` (necessary to use this script with" + " private models)." + ), + ) + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + if args.train_data_dir is None: + raise ValueError("You must specify a train data directory.") + + return args + + +imagenet_templates_small = [ + "a photo of a {}", + "a rendering of a {}", + "a cropped photo of the {}", + "the photo of a {}", + "a photo of a clean {}", + "a photo of a dirty {}", + "a dark photo of the {}", + "a photo of my {}", + "a photo of the cool {}", + "a close-up photo of a {}", + "a bright photo of the {}", + "a cropped photo of a {}", + "a photo of the {}", + "a good photo of the {}", + "a photo of one {}", + "a close-up photo of the {}", + "a rendition of the {}", + "a photo of the clean {}", + "a rendition of a {}", + "a photo of a nice {}", + "a good photo of a {}", + "a photo of the nice {}", + "a photo of the small {}", + "a photo of the weird {}", + "a photo of the large {}", + "a photo of a cool {}", + "a photo of a small {}", +] + +imagenet_style_templates_small = [ + "a painting in the style of {}", + "a rendering in the style of {}", + "a cropped painting in the style of {}", + "the painting in the style of {}", + "a clean painting in the style of {}", + "a dirty painting in the style of {}", + "a dark painting in the style of {}", + "a picture in the style of {}", + "a cool painting in the style of {}", + "a close-up painting in the style of {}", + "a bright painting in the style of {}", + "a cropped painting in the style of {}", + "a good painting in the style of {}", + "a close-up painting in the style of {}", + "a rendition in the style of {}", + "a nice painting in the style of {}", + "a small painting in the style of {}", + "a weird painting in the style of {}", + "a large painting in the style of {}", +] + + +class TextualInversionDataset(Dataset): + def __init__( + self, + data_root, + tokenizer, + learnable_property="object", # [object, style] + size=512, + repeats=100, + interpolation="bicubic", + flip_p=0.5, + set="train", + placeholder_token="*", + center_crop=False, + ): + self.data_root = data_root + self.tokenizer = tokenizer + self.learnable_property = learnable_property + self.size = size + self.placeholder_token = placeholder_token + self.center_crop = center_crop + self.flip_p = flip_p + + self.image_paths = [os.path.join(self.data_root, file_path) for file_path in os.listdir(self.data_root)] + + self.num_images = len(self.image_paths) + self._length = self.num_images + + if set == "train": + self._length = self.num_images * repeats + + self.interpolation = { + "linear": PIL_INTERPOLATION["linear"], + "bilinear": PIL_INTERPOLATION["bilinear"], + "bicubic": PIL_INTERPOLATION["bicubic"], + "lanczos": PIL_INTERPOLATION["lanczos"], + }[interpolation] + + self.templates = imagenet_style_templates_small if learnable_property == "style" else imagenet_templates_small + self.flip_transform = transforms.RandomHorizontalFlip(p=self.flip_p) + + def __len__(self): + return self._length + + def __getitem__(self, i): + example = {} + image = Image.open(self.image_paths[i % self.num_images]) + + if not image.mode == "RGB": + image = image.convert("RGB") + + placeholder_string = self.placeholder_token + text = random.choice(self.templates).format(placeholder_string) + + example["input_ids"] = self.tokenizer( + text, + padding="max_length", + truncation=True, + max_length=self.tokenizer.model_max_length, + return_tensors="pt", + ).input_ids[0] + + # default to score-sde preprocessing + img = np.array(image).astype(np.uint8) + + if self.center_crop: + crop = min(img.shape[0], img.shape[1]) + ( + h, + w, + ) = ( + img.shape[0], + img.shape[1], + ) + img = img[(h - crop) // 2 : (h + crop) // 2, (w - crop) // 2 : (w + crop) // 2] + + image = Image.fromarray(img) + image = image.resize((self.size, self.size), resample=self.interpolation) + + image = self.flip_transform(image) + image = np.array(image).astype(np.uint8) + image = (image / 127.5 - 1.0).astype(np.float32) + + example["pixel_values"] = torch.from_numpy(image).permute(2, 0, 1) + return example + + +def resize_token_embeddings(model, new_num_tokens, initializer_token_id, placeholder_token_id, rng): + if model.config.vocab_size == new_num_tokens or new_num_tokens is None: + return + model.config.vocab_size = new_num_tokens + + params = model.params + old_embeddings = params["text_model"]["embeddings"]["token_embedding"]["embedding"] + old_num_tokens, emb_dim = old_embeddings.shape + + initializer = jax.nn.initializers.normal() + + new_embeddings = initializer(rng, (new_num_tokens, emb_dim)) + new_embeddings = new_embeddings.at[:old_num_tokens].set(old_embeddings) + new_embeddings = new_embeddings.at[placeholder_token_id].set(new_embeddings[initializer_token_id]) + params["text_model"]["embeddings"]["token_embedding"]["embedding"] = new_embeddings + + model.params = params + return model + + +def get_params_to_save(params): + return jax.device_get(jax.tree_util.tree_map(lambda x: x[0], params)) + + +def main(): + args = parse_args() + + if args.seed is not None: + set_seed(args.seed) + + if jax.process_index() == 0: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + # Setup logging, we only want one process per machine to log things on the screen. + logger.setLevel(logging.INFO if jax.process_index() == 0 else logging.ERROR) + if jax.process_index() == 0: + transformers.utils.logging.set_verbosity_info() + else: + transformers.utils.logging.set_verbosity_error() + + # Load the tokenizer and add the placeholder token as a additional special token + if args.tokenizer_name: + tokenizer = CLIPTokenizer.from_pretrained(args.tokenizer_name) + elif args.pretrained_model_name_or_path: + tokenizer = CLIPTokenizer.from_pretrained(args.pretrained_model_name_or_path, subfolder="tokenizer") + + # Add the placeholder token in tokenizer + num_added_tokens = tokenizer.add_tokens(args.placeholder_token) + if num_added_tokens == 0: + raise ValueError( + f"The tokenizer already contains the token {args.placeholder_token}. Please pass a different" + " `placeholder_token` that is not already in the tokenizer." + ) + + # Convert the initializer_token, placeholder_token to ids + token_ids = tokenizer.encode(args.initializer_token, add_special_tokens=False) + # Check if initializer_token is a single token or a sequence of tokens + if len(token_ids) > 1: + raise ValueError("The initializer token must be a single token.") + + initializer_token_id = token_ids[0] + placeholder_token_id = tokenizer.convert_tokens_to_ids(args.placeholder_token) + + # Load models and create wrapper for stable diffusion + text_encoder = FlaxCLIPTextModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", revision=args.revision + ) + vae, vae_params = FlaxAutoencoderKL.from_pretrained( + args.pretrained_model_name_or_path, subfolder="vae", revision=args.revision + ) + unet, unet_params = FlaxUNet2DConditionModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="unet", revision=args.revision + ) + + # Create sampling rng + rng = jax.random.PRNGKey(args.seed) + rng, _ = jax.random.split(rng) + # Resize the token embeddings as we are adding new special tokens to the tokenizer + text_encoder = resize_token_embeddings( + text_encoder, len(tokenizer), initializer_token_id, placeholder_token_id, rng + ) + original_token_embeds = text_encoder.params["text_model"]["embeddings"]["token_embedding"]["embedding"] + + train_dataset = TextualInversionDataset( + data_root=args.train_data_dir, + tokenizer=tokenizer, + size=args.resolution, + placeholder_token=args.placeholder_token, + repeats=args.repeats, + learnable_property=args.learnable_property, + center_crop=args.center_crop, + set="train", + ) + + def collate_fn(examples): + pixel_values = torch.stack([example["pixel_values"] for example in examples]) + input_ids = torch.stack([example["input_ids"] for example in examples]) + + batch = {"pixel_values": pixel_values, "input_ids": input_ids} + batch = {k: v.numpy() for k, v in batch.items()} + + return batch + + total_train_batch_size = args.train_batch_size * jax.local_device_count() + train_dataloader = torch.utils.data.DataLoader( + train_dataset, batch_size=total_train_batch_size, shuffle=True, drop_last=True, collate_fn=collate_fn + ) + + # Optimization + if args.scale_lr: + args.learning_rate = args.learning_rate * total_train_batch_size + + constant_scheduler = optax.constant_schedule(args.learning_rate) + + optimizer = optax.adamw( + learning_rate=constant_scheduler, + b1=args.adam_beta1, + b2=args.adam_beta2, + eps=args.adam_epsilon, + weight_decay=args.adam_weight_decay, + ) + + def create_mask(params, label_fn): + def _map(params, mask, label_fn): + for k in params: + if label_fn(k): + mask[k] = "token_embedding" + else: + if isinstance(params[k], dict): + mask[k] = {} + _map(params[k], mask[k], label_fn) + else: + mask[k] = "zero" + + mask = {} + _map(params, mask, label_fn) + return mask + + def zero_grads(): + # from https://github.com/deepmind/optax/issues/159#issuecomment-896459491 + def init_fn(_): + return () + + def update_fn(updates, state, params=None): + return jax.tree_util.tree_map(jnp.zeros_like, updates), () + + return optax.GradientTransformation(init_fn, update_fn) + + # Zero out gradients of layers other than the token embedding layer + tx = optax.multi_transform( + {"token_embedding": optimizer, "zero": zero_grads()}, + create_mask(text_encoder.params, lambda s: s == "token_embedding"), + ) + + state = train_state.TrainState.create(apply_fn=text_encoder.__call__, params=text_encoder.params, tx=tx) + + noise_scheduler = FlaxDDPMScheduler( + beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", num_train_timesteps=1000 + ) + noise_scheduler_state = noise_scheduler.create_state() + + # Initialize our training + train_rngs = jax.random.split(rng, jax.local_device_count()) + + # Define gradient train step fn + def train_step(state, vae_params, unet_params, batch, train_rng): + dropout_rng, sample_rng, new_train_rng = jax.random.split(train_rng, 3) + + def compute_loss(params): + vae_outputs = vae.apply( + {"params": vae_params}, batch["pixel_values"], deterministic=True, method=vae.encode + ) + latents = vae_outputs.latent_dist.sample(sample_rng) + # (NHWC) -> (NCHW) + latents = jnp.transpose(latents, (0, 3, 1, 2)) + latents = latents * vae.config.scaling_factor + + noise_rng, timestep_rng = jax.random.split(sample_rng) + noise = jax.random.normal(noise_rng, latents.shape) + bsz = latents.shape[0] + timesteps = jax.random.randint( + timestep_rng, + (bsz,), + 0, + noise_scheduler.config.num_train_timesteps, + ) + noisy_latents = noise_scheduler.add_noise(noise_scheduler_state, latents, noise, timesteps) + encoder_hidden_states = state.apply_fn( + batch["input_ids"], params=params, dropout_rng=dropout_rng, train=True + )[0] + # Predict the noise residual and compute loss + model_pred = unet.apply( + {"params": unet_params}, noisy_latents, timesteps, encoder_hidden_states, train=False + ).sample + + # Get the target for loss depending on the prediction type + if noise_scheduler.config.prediction_type == "epsilon": + target = noise + elif noise_scheduler.config.prediction_type == "v_prediction": + target = noise_scheduler.get_velocity(noise_scheduler_state, latents, noise, timesteps) + else: + raise ValueError(f"Unknown prediction type {noise_scheduler.config.prediction_type}") + + loss = (target - model_pred) ** 2 + loss = loss.mean() + + return loss + + grad_fn = jax.value_and_grad(compute_loss) + loss, grad = grad_fn(state.params) + grad = jax.lax.pmean(grad, "batch") + new_state = state.apply_gradients(grads=grad) + + # Keep the token embeddings fixed except the newly added embeddings for the concept, + # as we only want to optimize the concept embeddings + token_embeds = original_token_embeds.at[placeholder_token_id].set( + new_state.params["text_model"]["embeddings"]["token_embedding"]["embedding"][placeholder_token_id] + ) + new_state.params["text_model"]["embeddings"]["token_embedding"]["embedding"] = token_embeds + + metrics = {"loss": loss} + metrics = jax.lax.pmean(metrics, axis_name="batch") + return new_state, metrics, new_train_rng + + # Create parallel version of the train and eval step + p_train_step = jax.pmap(train_step, "batch", donate_argnums=(0,)) + + # Replicate the train state on each device + state = jax_utils.replicate(state) + vae_params = jax_utils.replicate(vae_params) + unet_params = jax_utils.replicate(unet_params) + + # Train! + num_update_steps_per_epoch = math.ceil(len(train_dataloader)) + + # Scheduler and math around the number of training steps. + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel & distributed) = {total_train_batch_size}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + + global_step = 0 + + epochs = tqdm(range(args.num_train_epochs), desc=f"Epoch ... (1/{args.num_train_epochs})", position=0) + for epoch in epochs: + # ======================== Training ================================ + + train_metrics = [] + + steps_per_epoch = len(train_dataset) // total_train_batch_size + train_step_progress_bar = tqdm(total=steps_per_epoch, desc="Training...", position=1, leave=False) + # train + for batch in train_dataloader: + batch = shard(batch) + state, train_metric, train_rngs = p_train_step(state, vae_params, unet_params, batch, train_rngs) + train_metrics.append(train_metric) + + train_step_progress_bar.update(1) + global_step += 1 + + if global_step >= args.max_train_steps: + break + if global_step % args.save_steps == 0: + learned_embeds = get_params_to_save(state.params)["text_model"]["embeddings"]["token_embedding"][ + "embedding" + ][placeholder_token_id] + learned_embeds_dict = {args.placeholder_token: learned_embeds} + jnp.save( + os.path.join(args.output_dir, "learned_embeds-" + str(global_step) + ".npy"), learned_embeds_dict + ) + + train_metric = jax_utils.unreplicate(train_metric) + + train_step_progress_bar.close() + epochs.write(f"Epoch... ({epoch + 1}/{args.num_train_epochs} | Loss: {train_metric['loss']})") + + # Create the pipeline using using the trained modules and save it. + if jax.process_index() == 0: + scheduler = FlaxPNDMScheduler( + beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", skip_prk_steps=True + ) + safety_checker = FlaxStableDiffusionSafetyChecker.from_pretrained( + "CompVis/stable-diffusion-safety-checker", from_pt=True + ) + pipeline = FlaxStableDiffusionPipeline( + text_encoder=text_encoder, + vae=vae, + unet=unet, + tokenizer=tokenizer, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=CLIPImageProcessor.from_pretrained("openai/clip-vit-base-patch32"), + ) + + pipeline.save_pretrained( + args.output_dir, + params={ + "text_encoder": get_params_to_save(state.params), + "vae": get_params_to_save(vae_params), + "unet": get_params_to_save(unet_params), + "safety_checker": safety_checker.params, + }, + ) + + # Also save the newly trained embeddings + learned_embeds = get_params_to_save(state.params)["text_model"]["embeddings"]["token_embedding"]["embedding"][ + placeholder_token_id + ] + learned_embeds_dict = {args.placeholder_token: learned_embeds} + jnp.save(os.path.join(args.output_dir, "learned_embeds.npy"), learned_embeds_dict) + + if args.push_to_hub: + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["step_*", "epoch_*"], + ) + + +if __name__ == "__main__": + main() diff --git a/diffusers-0.27.0/examples/textual_inversion/textual_inversion_sdxl.py b/diffusers-0.27.0/examples/textual_inversion/textual_inversion_sdxl.py new file mode 100755 index 0000000..980b255 --- /dev/null +++ b/diffusers-0.27.0/examples/textual_inversion/textual_inversion_sdxl.py @@ -0,0 +1,1070 @@ +#!/usr/bin/env python +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import logging +import math +import os +import random +import shutil +from pathlib import Path + +import numpy as np +import PIL +import safetensors +import torch +import torch.nn.functional as F +import torch.utils.checkpoint +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.utils import ProjectConfiguration, set_seed +from huggingface_hub import create_repo, upload_folder +from packaging import version +from PIL import Image +from torch.utils.data import Dataset +from torchvision import transforms +from tqdm.auto import tqdm +from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer + +import diffusers +from diffusers import ( + AutoencoderKL, + DDPMScheduler, + DiffusionPipeline, + DPMSolverMultistepScheduler, + UNet2DConditionModel, +) +from diffusers.optimization import get_scheduler +from diffusers.utils import check_min_version, is_wandb_available +from diffusers.utils.hub_utils import load_or_create_model_card, populate_model_card +from diffusers.utils.import_utils import is_xformers_available + + +if is_wandb_available(): + import wandb + +if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"): + PIL_INTERPOLATION = { + "linear": PIL.Image.Resampling.BILINEAR, + "bilinear": PIL.Image.Resampling.BILINEAR, + "bicubic": PIL.Image.Resampling.BICUBIC, + "lanczos": PIL.Image.Resampling.LANCZOS, + "nearest": PIL.Image.Resampling.NEAREST, + } +else: + PIL_INTERPOLATION = { + "linear": PIL.Image.LINEAR, + "bilinear": PIL.Image.BILINEAR, + "bicubic": PIL.Image.BICUBIC, + "lanczos": PIL.Image.LANCZOS, + "nearest": PIL.Image.NEAREST, + } +# ------------------------------------------------------------------------------ + + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.27.0") + +logger = get_logger(__name__) + + +def save_model_card(repo_id: str, images=None, base_model=str, repo_folder=None): + img_str = "" + for i, image in enumerate(images): + image.save(os.path.join(repo_folder, f"image_{i}.png")) + img_str += f"![img_{i}](./image_{i}.png)\n" + + model_description = f""" +# Textual inversion text2image fine-tuning - {repo_id} +These are textual inversion adaption weights for {base_model}. You can find some example images in the following. \n +{img_str} +""" + model_card = load_or_create_model_card( + repo_id_or_path=repo_id, + from_training=True, + license="creativeml-openrail-m", + base_model=base_model, + model_description=model_description, + inference=True, + ) + + tags = [ + "stable-diffusion-xl", + "stable-diffusion-xl-diffusers", + "text-to-image", + "diffusers", + "diffusers-training", + "textual_inversion", + ] + + model_card = populate_model_card(model_card, tags=tags) + + model_card.save(os.path.join(repo_folder, "README.md")) + + +def log_validation( + text_encoder_1, + text_encoder_2, + tokenizer_1, + tokenizer_2, + unet, + vae, + args, + accelerator, + weight_dtype, + epoch, + is_final_validation=False, +): + logger.info( + f"Running validation... \n Generating {args.num_validation_images} images with prompt:" + f" {args.validation_prompt}." + ) + pipeline = DiffusionPipeline.from_pretrained( + args.pretrained_model_name_or_path, + text_encoder=accelerator.unwrap_model(text_encoder_1), + text_encoder_2=text_encoder_2, + tokenizer=tokenizer_1, + tokenizer_2=tokenizer_2, + unet=unet, + vae=vae, + safety_checker=None, + revision=args.revision, + variant=args.variant, + torch_dtype=weight_dtype, + ) + pipeline.scheduler = DPMSolverMultistepScheduler.from_config(pipeline.scheduler.config) + pipeline = pipeline.to(accelerator.device) + pipeline.set_progress_bar_config(disable=True) + + # run inference + generator = None if args.seed is None else torch.Generator(device=accelerator.device).manual_seed(args.seed) + images = [] + for _ in range(args.num_validation_images): + image = pipeline(args.validation_prompt, num_inference_steps=25, generator=generator).images[0] + images.append(image) + + tracker_key = "test" if is_final_validation else "validation" + for tracker in accelerator.trackers: + if tracker.name == "tensorboard": + np_images = np.stack([np.asarray(img) for img in images]) + tracker.writer.add_images(tracker_key, np_images, epoch, dataformats="NHWC") + if tracker.name == "wandb": + tracker.log( + { + tracker_key: [ + wandb.Image(image, caption=f"{i}: {args.validation_prompt}") for i, image in enumerate(images) + ] + } + ) + + del pipeline + torch.cuda.empty_cache() + return images + + +def save_progress(text_encoder, placeholder_token_ids, accelerator, args, save_path, safe_serialization=True): + logger.info("Saving embeddings") + learned_embeds = ( + accelerator.unwrap_model(text_encoder) + .get_input_embeddings() + .weight[min(placeholder_token_ids) : max(placeholder_token_ids) + 1] + ) + learned_embeds_dict = {args.placeholder_token: learned_embeds.detach().cpu()} + + if safe_serialization: + safetensors.torch.save_file(learned_embeds_dict, save_path, metadata={"format": "pt"}) + else: + torch.save(learned_embeds_dict, save_path) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--save_steps", + type=int, + default=500, + help="Save learned_embeds.bin every X updates steps.", + ) + parser.add_argument( + "--save_as_full_pipeline", + action="store_true", + help="Save the complete stable diffusion pipeline.", + ) + parser.add_argument( + "--num_vectors", + type=int, + default=1, + help="How many textual inversion vectors shall be used to learn the concept.", + ) + parser.add_argument( + "--pretrained_model_name_or_path", + type=str, + default=None, + required=True, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--revision", + type=str, + default=None, + required=False, + help="Revision of pretrained model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--variant", + type=str, + default=None, + help="Variant of the model files of the pretrained model identifier from huggingface.co/models, 'e.g.' fp16", + ) + parser.add_argument( + "--train_data_dir", type=str, default=None, required=True, help="A folder containing the training data." + ) + parser.add_argument( + "--placeholder_token", + type=str, + default=None, + required=True, + help="A token to use as a placeholder for the concept.", + ) + parser.add_argument( + "--initializer_token", type=str, default=None, required=True, help="A token to use as initializer word." + ) + parser.add_argument("--learnable_property", type=str, default="object", help="Choose between 'object' and 'style'") + parser.add_argument("--repeats", type=int, default=100, help="How many times to repeat the training data.") + parser.add_argument( + "--output_dir", + type=str, + default="text-inversion-model", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--resolution", + type=int, + default=512, + help=( + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" + ), + ) + parser.add_argument( + "--center_crop", action="store_true", help="Whether to center crop images before resizing to resolution." + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=5000, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--scale_lr", + action="store_true", + default=False, + help="Scale the learning rate by the number of GPUs, gradient accumulation steps, and batch size.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--lr_num_cycles", + type=int, + default=1, + help="Number of hard resets of the lr in cosine_with_restarts scheduler.", + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument("--adam_weight_decay", type=float, default=1e-2, help="Weight decay to use.") + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default="no", + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose" + "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10." + "and an Nvidia Ampere GPU." + ), + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument( + "--validation_prompt", + type=str, + default=None, + help="A prompt that is used during validation to verify that the model is learning.", + ) + parser.add_argument( + "--num_validation_images", + type=int, + default=4, + help="Number of images that should be generated during validation with `validation_prompt`.", + ) + parser.add_argument( + "--validation_steps", + type=int, + default=100, + help=( + "Run validation every X steps. Validation consists of running the prompt" + " `args.validation_prompt` multiple times: `args.num_validation_images`" + " and logging the images." + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--enable_xformers_memory_efficient_attention", action="store_true", help="Whether or not to use xformers." + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + if args.train_data_dir is None: + raise ValueError("You must specify a train data directory.") + + return args + + +imagenet_templates_small = [ + "a photo of a {}", + "a rendering of a {}", + "a cropped photo of the {}", + "the photo of a {}", + "a photo of a clean {}", + "a photo of a dirty {}", + "a dark photo of the {}", + "a photo of my {}", + "a photo of the cool {}", + "a close-up photo of a {}", + "a bright photo of the {}", + "a cropped photo of a {}", + "a photo of the {}", + "a good photo of the {}", + "a photo of one {}", + "a close-up photo of the {}", + "a rendition of the {}", + "a photo of the clean {}", + "a rendition of a {}", + "a photo of a nice {}", + "a good photo of a {}", + "a photo of the nice {}", + "a photo of the small {}", + "a photo of the weird {}", + "a photo of the large {}", + "a photo of a cool {}", + "a photo of a small {}", +] + +imagenet_style_templates_small = [ + "a painting in the style of {}", + "a rendering in the style of {}", + "a cropped painting in the style of {}", + "the painting in the style of {}", + "a clean painting in the style of {}", + "a dirty painting in the style of {}", + "a dark painting in the style of {}", + "a picture in the style of {}", + "a cool painting in the style of {}", + "a close-up painting in the style of {}", + "a bright painting in the style of {}", + "a cropped painting in the style of {}", + "a good painting in the style of {}", + "a close-up painting in the style of {}", + "a rendition in the style of {}", + "a nice painting in the style of {}", + "a small painting in the style of {}", + "a weird painting in the style of {}", + "a large painting in the style of {}", +] + + +class TextualInversionDataset(Dataset): + def __init__( + self, + data_root, + tokenizer_1, + tokenizer_2, + learnable_property="object", # [object, style] + size=512, + repeats=100, + interpolation="bicubic", + flip_p=0.5, + set="train", + placeholder_token="*", + center_crop=False, + ): + self.data_root = data_root + self.tokenizer_1 = tokenizer_1 + self.tokenizer_2 = tokenizer_2 + self.learnable_property = learnable_property + self.size = size + self.placeholder_token = placeholder_token + self.center_crop = center_crop + self.flip_p = flip_p + + self.image_paths = [os.path.join(self.data_root, file_path) for file_path in os.listdir(self.data_root)] + + self.num_images = len(self.image_paths) + self._length = self.num_images + + if set == "train": + self._length = self.num_images * repeats + + self.interpolation = { + "linear": PIL_INTERPOLATION["linear"], + "bilinear": PIL_INTERPOLATION["bilinear"], + "bicubic": PIL_INTERPOLATION["bicubic"], + "lanczos": PIL_INTERPOLATION["lanczos"], + }[interpolation] + + self.templates = imagenet_style_templates_small if learnable_property == "style" else imagenet_templates_small + self.flip_transform = transforms.RandomHorizontalFlip(p=self.flip_p) + self.crop = transforms.CenterCrop(size) if center_crop else transforms.RandomCrop(size) + + def __len__(self): + return self._length + + def __getitem__(self, i): + example = {} + image = Image.open(self.image_paths[i % self.num_images]) + + if not image.mode == "RGB": + image = image.convert("RGB") + + placeholder_string = self.placeholder_token + text = random.choice(self.templates).format(placeholder_string) + + example["original_size"] = (image.height, image.width) + + image = image.resize((self.size, self.size), resample=self.interpolation) + + if self.center_crop: + y1 = max(0, int(round((image.height - self.size) / 2.0))) + x1 = max(0, int(round((image.width - self.size) / 2.0))) + image = self.crop(image) + else: + y1, x1, h, w = self.crop.get_params(image, (self.size, self.size)) + image = transforms.functional.crop(image, y1, x1, h, w) + + example["crop_top_left"] = (y1, x1) + + example["input_ids_1"] = self.tokenizer_1( + text, + padding="max_length", + truncation=True, + max_length=self.tokenizer_1.model_max_length, + return_tensors="pt", + ).input_ids[0] + + example["input_ids_2"] = self.tokenizer_2( + text, + padding="max_length", + truncation=True, + max_length=self.tokenizer_2.model_max_length, + return_tensors="pt", + ).input_ids[0] + + # default to score-sde preprocessing + img = np.array(image).astype(np.uint8) + + image = Image.fromarray(img) + + image = self.flip_transform(image) + image = np.array(image).astype(np.uint8) + image = (image / 127.5 - 1.0).astype(np.float32) + + example["pixel_values"] = torch.from_numpy(image).permute(2, 0, 1) + return example + + +def main(): + args = parse_args() + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + logging_dir = os.path.join(args.output_dir, args.logging_dir) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + if args.report_to == "wandb": + if not is_wandb_available(): + raise ImportError("Make sure to install wandb if you want to use it for logging during training.") + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + transformers.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + transformers.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + + # Load tokenizer + tokenizer_1 = CLIPTokenizer.from_pretrained(args.pretrained_model_name_or_path, subfolder="tokenizer") + tokenizer_2 = CLIPTokenizer.from_pretrained(args.pretrained_model_name_or_path, subfolder="tokenizer_2") + + # Load scheduler and models + noise_scheduler = DDPMScheduler.from_pretrained(args.pretrained_model_name_or_path, subfolder="scheduler") + text_encoder_1 = CLIPTextModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder", revision=args.revision + ) + text_encoder_2 = CLIPTextModelWithProjection.from_pretrained( + args.pretrained_model_name_or_path, subfolder="text_encoder_2", revision=args.revision + ) + vae = AutoencoderKL.from_pretrained( + args.pretrained_model_name_or_path, subfolder="vae", revision=args.revision, variant=args.variant + ) + unet = UNet2DConditionModel.from_pretrained( + args.pretrained_model_name_or_path, subfolder="unet", revision=args.revision, variant=args.variant + ) + + # Add the placeholder token in tokenizer_1 + placeholder_tokens = [args.placeholder_token] + + if args.num_vectors < 1: + raise ValueError(f"--num_vectors has to be larger or equal to 1, but is {args.num_vectors}") + + # add dummy tokens for multi-vector + additional_tokens = [] + for i in range(1, args.num_vectors): + additional_tokens.append(f"{args.placeholder_token}_{i}") + placeholder_tokens += additional_tokens + + num_added_tokens = tokenizer_1.add_tokens(placeholder_tokens) + if num_added_tokens != args.num_vectors: + raise ValueError( + f"The tokenizer already contains the token {args.placeholder_token}. Please pass a different" + " `placeholder_token` that is not already in the tokenizer." + ) + + # Convert the initializer_token, placeholder_token to ids + token_ids = tokenizer_1.encode(args.initializer_token, add_special_tokens=False) + # Check if initializer_token is a single token or a sequence of tokens + if len(token_ids) > 1: + raise ValueError("The initializer token must be a single token.") + + initializer_token_id = token_ids[0] + placeholder_token_ids = tokenizer_1.convert_tokens_to_ids(placeholder_tokens) + + # Resize the token embeddings as we are adding new special tokens to the tokenizer + text_encoder_1.resize_token_embeddings(len(tokenizer_1)) + + # Initialise the newly added placeholder token with the embeddings of the initializer token + token_embeds = text_encoder_1.get_input_embeddings().weight.data + with torch.no_grad(): + for token_id in placeholder_token_ids: + token_embeds[token_id] = token_embeds[initializer_token_id].clone() + + # Freeze vae and unet + vae.requires_grad_(False) + unet.requires_grad_(False) + text_encoder_2.requires_grad_(False) + # Freeze all parameters except for the token embeddings in text encoder + text_encoder_1.text_model.encoder.requires_grad_(False) + text_encoder_1.text_model.final_layer_norm.requires_grad_(False) + text_encoder_1.text_model.embeddings.position_embedding.requires_grad_(False) + + if args.gradient_checkpointing: + text_encoder_1.gradient_checkpointing_enable() + + if args.enable_xformers_memory_efficient_attention: + if is_xformers_available(): + import xformers + + xformers_version = version.parse(xformers.__version__) + if xformers_version == version.parse("0.0.16"): + logger.warning( + "xFormers 0.0.16 cannot be used for training in some GPUs. If you observe problems during training, please update xFormers to at least 0.0.17. See https://huggingface.co/docs/diffusers/main/en/optimization/xformers for more details." + ) + unet.enable_xformers_memory_efficient_attention() + else: + raise ValueError("xformers is not available. Make sure it is installed correctly") + + # Enable TF32 for faster training on Ampere GPUs, + # cf https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.scale_lr: + args.learning_rate = ( + args.learning_rate * args.gradient_accumulation_steps * args.train_batch_size * accelerator.num_processes + ) + + # Initialize the optimizer + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "To use 8-bit Adam, please install the bitsandbytes library: `pip install bitsandbytes`." + ) + + optimizer_class = bnb.optim.AdamW8bit + else: + optimizer_class = torch.optim.AdamW + + optimizer = optimizer_class( + text_encoder_1.get_input_embeddings().parameters(), # only optimize the embeddings + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + placeholder_token = " ".join(tokenizer_1.convert_ids_to_tokens(placeholder_token_ids)) + # Dataset and DataLoaders creation: + train_dataset = TextualInversionDataset( + data_root=args.train_data_dir, + tokenizer_1=tokenizer_1, + tokenizer_2=tokenizer_2, + size=args.resolution, + placeholder_token=placeholder_token, + repeats=args.repeats, + learnable_property=args.learnable_property, + center_crop=args.center_crop, + set="train", + ) + train_dataloader = torch.utils.data.DataLoader( + train_dataset, batch_size=args.train_batch_size, shuffle=True, num_workers=args.dataloader_num_workers + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * accelerator.num_processes, + num_training_steps=args.max_train_steps * accelerator.num_processes, + num_cycles=args.lr_num_cycles, + ) + + text_encoder_1.train() + # Prepare everything with our `accelerator`. + text_encoder_1, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + text_encoder_1, optimizer, train_dataloader, lr_scheduler + ) + + # For mixed precision training we cast all non-trainable weigths (vae, non-lora text_encoder and non-lora unet) to half-precision + # as these weights are only used for inference, keeping weights in full precision is not required. + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + + # Move vae and unet and text_encoder_2 to device and cast to weight_dtype + unet.to(accelerator.device, dtype=weight_dtype) + vae.to(accelerator.device, dtype=weight_dtype) + text_encoder_2.to(accelerator.device, dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + accelerator.init_trackers("textual_inversion", config=vars(args)) + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + initial_global_step = 0 + else: + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + initial_global_step = global_step + first_epoch = global_step // num_update_steps_per_epoch + + else: + initial_global_step = 0 + + progress_bar = tqdm( + range(0, args.max_train_steps), + initial=initial_global_step, + desc="Steps", + # Only show the progress bar once on each machine. + disable=not accelerator.is_local_main_process, + ) + + # keep original embeddings as reference + orig_embeds_params = accelerator.unwrap_model(text_encoder_1).get_input_embeddings().weight.data.clone() + + for epoch in range(first_epoch, args.num_train_epochs): + text_encoder_1.train() + for step, batch in enumerate(train_dataloader): + with accelerator.accumulate(text_encoder_1): + # Convert images to latent space + latents = vae.encode(batch["pixel_values"].to(dtype=weight_dtype)).latent_dist.sample().detach() + latents = latents * vae.config.scaling_factor + + # Sample noise that we'll add to the latents + noise = torch.randn_like(latents) + bsz = latents.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint(0, noise_scheduler.config.num_train_timesteps, (bsz,), device=latents.device) + timesteps = timesteps.long() + + # Add noise to the latents according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_latents = noise_scheduler.add_noise(latents, noise, timesteps) + + # Get the text embedding for conditioning + encoder_hidden_states_1 = ( + text_encoder_1(batch["input_ids_1"], output_hidden_states=True) + .hidden_states[-2] + .to(dtype=weight_dtype) + ) + encoder_output_2 = text_encoder_2( + batch["input_ids_2"].reshape(batch["input_ids_1"].shape[0], -1), output_hidden_states=True + ) + encoder_hidden_states_2 = encoder_output_2.hidden_states[-2].to(dtype=weight_dtype) + original_size = [ + (batch["original_size"][0][i].item(), batch["original_size"][1][i].item()) + for i in range(args.train_batch_size) + ] + crop_top_left = [ + (batch["crop_top_left"][0][i].item(), batch["crop_top_left"][1][i].item()) + for i in range(args.train_batch_size) + ] + target_size = (args.resolution, args.resolution) + add_time_ids = torch.cat( + [ + torch.tensor(original_size[i] + crop_top_left[i] + target_size) + for i in range(args.train_batch_size) + ] + ).to(accelerator.device, dtype=weight_dtype) + added_cond_kwargs = {"text_embeds": encoder_output_2[0], "time_ids": add_time_ids} + encoder_hidden_states = torch.cat([encoder_hidden_states_1, encoder_hidden_states_2], dim=-1) + + # Predict the noise residual + model_pred = unet( + noisy_latents, timesteps, encoder_hidden_states, added_cond_kwargs=added_cond_kwargs + ).sample + + # Get the target for loss depending on the prediction type + if noise_scheduler.config.prediction_type == "epsilon": + target = noise + elif noise_scheduler.config.prediction_type == "v_prediction": + target = noise_scheduler.get_velocity(latents, noise, timesteps) + else: + raise ValueError(f"Unknown prediction type {noise_scheduler.config.prediction_type}") + + loss = F.mse_loss(model_pred.float(), target.float(), reduction="mean") + + accelerator.backward(loss) + + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Let's make sure we don't update any embedding weights besides the newly added token + index_no_updates = torch.ones((len(tokenizer_1),), dtype=torch.bool) + index_no_updates[min(placeholder_token_ids) : max(placeholder_token_ids) + 1] = False + + with torch.no_grad(): + accelerator.unwrap_model(text_encoder_1).get_input_embeddings().weight[ + index_no_updates + ] = orig_embeds_params[index_no_updates] + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + images = [] + progress_bar.update(1) + global_step += 1 + if global_step % args.save_steps == 0: + weight_name = f"learned_embeds-steps-{global_step}.safetensors" + save_path = os.path.join(args.output_dir, weight_name) + save_progress( + text_encoder_1, + placeholder_token_ids, + accelerator, + args, + save_path, + safe_serialization=True, + ) + + if accelerator.is_main_process: + if global_step % args.checkpointing_steps == 0: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + if args.validation_prompt is not None and global_step % args.validation_steps == 0: + images = log_validation( + text_encoder_1, + text_encoder_2, + tokenizer_1, + tokenizer_2, + unet, + vae, + args, + accelerator, + weight_dtype, + epoch, + ) + + logs = {"loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + accelerator.log(logs, step=global_step) + + if global_step >= args.max_train_steps: + break + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + if args.validation_prompt: + images = log_validation( + text_encoder_1, + text_encoder_2, + tokenizer_1, + tokenizer_2, + unet, + vae, + args, + accelerator, + weight_dtype, + epoch, + is_final_validation=True, + ) + + if args.push_to_hub and not args.save_as_full_pipeline: + logger.warning("Enabling full model saving because --push_to_hub=True was specified.") + save_full_model = True + else: + save_full_model = args.save_as_full_pipeline + if save_full_model: + pipeline = DiffusionPipeline.from_pretrained( + args.pretrained_model_name_or_path, + text_encoder=accelerator.unwrap_model(text_encoder_1), + text_encoder_2=text_encoder_2, + vae=vae, + unet=unet, + tokenizer=tokenizer_1, + tokenizer_2=tokenizer_2, + ) + pipeline.save_pretrained(args.output_dir) + # Save the newly trained embeddings + weight_name = "learned_embeds.safetensors" + save_path = os.path.join(args.output_dir, weight_name) + save_progress( + text_encoder_1, + placeholder_token_ids, + accelerator, + args, + save_path, + safe_serialization=True, + ) + + if args.push_to_hub: + save_model_card( + repo_id, + images=images, + base_model=args.pretrained_model_name_or_path, + repo_folder=args.output_dir, + ) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["step_*", "epoch_*"], + ) + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/diffusers-0.27.0/examples/unconditional_image_generation/README.md b/diffusers-0.27.0/examples/unconditional_image_generation/README.md new file mode 100755 index 0000000..2990b3a --- /dev/null +++ b/diffusers-0.27.0/examples/unconditional_image_generation/README.md @@ -0,0 +1,163 @@ +## Training an unconditional diffusion model + +Creating a training image set is [described in a different document](https://huggingface.co/docs/datasets/image_process#image-datasets). + +### Installing the dependencies + +Before running the scripts, make sure to install the library's training dependencies: + +**Important** + +To make sure you can successfully run the latest versions of the example scripts, we highly recommend **installing from source** and keeping the install up to date as we update the example scripts frequently and install some example-specific requirements. To do this, execute the following steps in a new virtual environment: +```bash +git clone https://github.com/huggingface/diffusers +cd diffusers +pip install . +``` + +Then cd in the example folder and run +```bash +pip install -r requirements.txt +``` + + +And initialize an [🤗Accelerate](https://github.com/huggingface/accelerate/) environment with: + +```bash +accelerate config +``` + +### Unconditional Flowers + +The command to train a DDPM UNet model on the Oxford Flowers dataset: + +```bash +accelerate launch train_unconditional.py \ + --dataset_name="huggan/flowers-102-categories" \ + --resolution=64 --center_crop --random_flip \ + --output_dir="ddpm-ema-flowers-64" \ + --train_batch_size=16 \ + --num_epochs=100 \ + --gradient_accumulation_steps=1 \ + --use_ema \ + --learning_rate=1e-4 \ + --lr_warmup_steps=500 \ + --mixed_precision=no \ + --push_to_hub +``` +An example trained model: https://huggingface.co/anton-l/ddpm-ema-flowers-64 + +A full training run takes 2 hours on 4xV100 GPUs. + + + + +### Unconditional Pokemon + +The command to train a DDPM UNet model on the Pokemon dataset: + +```bash +accelerate launch train_unconditional.py \ + --dataset_name="huggan/pokemon" \ + --resolution=64 --center_crop --random_flip \ + --output_dir="ddpm-ema-pokemon-64" \ + --train_batch_size=16 \ + --num_epochs=100 \ + --gradient_accumulation_steps=1 \ + --use_ema \ + --learning_rate=1e-4 \ + --lr_warmup_steps=500 \ + --mixed_precision=no \ + --push_to_hub +``` +An example trained model: https://huggingface.co/anton-l/ddpm-ema-pokemon-64 + +A full training run takes 2 hours on 4xV100 GPUs. + + + +### Training with multiple GPUs + +`accelerate` allows for seamless multi-GPU training. Follow the instructions [here](https://huggingface.co/docs/accelerate/basic_tutorials/launch) +for running distributed training with `accelerate`. Here is an example command: + +```bash +accelerate launch --mixed_precision="fp16" --multi_gpu train_unconditional.py \ + --dataset_name="huggan/pokemon" \ + --resolution=64 --center_crop --random_flip \ + --output_dir="ddpm-ema-pokemon-64" \ + --train_batch_size=16 \ + --num_epochs=100 \ + --gradient_accumulation_steps=1 \ + --use_ema \ + --learning_rate=1e-4 \ + --lr_warmup_steps=500 \ + --mixed_precision="fp16" \ + --logger="wandb" +``` + +To be able to use Weights and Biases (`wandb`) as a logger you need to install the library: `pip install wandb`. + +### Using your own data + +To use your own dataset, there are 2 ways: +- you can either provide your own folder as `--train_data_dir` +- or you can upload your dataset to the hub (possibly as a private repo, if you prefer so), and simply pass the `--dataset_name` argument. + +Below, we explain both in more detail. + +#### Provide the dataset as a folder + +If you provide your own folders with images, the script expects the following directory structure: + +```bash +data_dir/xxx.png +data_dir/xxy.png +data_dir/[...]/xxz.png +``` + +In other words, the script will take care of gathering all images inside the folder. You can then run the script like this: + +```bash +accelerate launch train_unconditional.py \ + --train_data_dir \ + +``` + +Internally, the script will use the [`ImageFolder`](https://huggingface.co/docs/datasets/v2.0.0/en/image_process#imagefolder) feature which will automatically turn the folders into 🤗 Dataset objects. + +#### Upload your data to the hub, as a (possibly private) repo + +It's very easy (and convenient) to upload your image dataset to the hub using the [`ImageFolder`](https://huggingface.co/docs/datasets/v2.0.0/en/image_process#imagefolder) feature available in 🤗 Datasets. Simply do the following: + +```python +from datasets import load_dataset + +# example 1: local folder +dataset = load_dataset("imagefolder", data_dir="path_to_your_folder") + +# example 2: local files (supported formats are tar, gzip, zip, xz, rar, zstd) +dataset = load_dataset("imagefolder", data_files="path_to_zip_file") + +# example 3: remote files (supported formats are tar, gzip, zip, xz, rar, zstd) +dataset = load_dataset("imagefolder", data_files="https://download.microsoft.com/download/3/E/1/3E1C3F21-ECDB-4869-8368-6DEBA77B919F/kagglecatsanddogs_3367a.zip") + +# example 4: providing several splits +dataset = load_dataset("imagefolder", data_files={"train": ["path/to/file1", "path/to/file2"], "test": ["path/to/file3", "path/to/file4"]}) +``` + +`ImageFolder` will create an `image` column containing the PIL-encoded images. + +Next, push it to the hub! + +```python +# assuming you have ran the huggingface-cli login command in a terminal +dataset.push_to_hub("name_of_your_dataset") + +# if you want to push to a private repo, simply pass private=True: +dataset.push_to_hub("name_of_your_dataset", private=True) +``` + +and that's it! You can now train your model by simply setting the `--dataset_name` argument to the name of your dataset on the hub. + +More on this can also be found in [this blog post](https://huggingface.co/blog/image-search-datasets). diff --git a/diffusers-0.27.0/examples/unconditional_image_generation/requirements.txt b/diffusers-0.27.0/examples/unconditional_image_generation/requirements.txt new file mode 100755 index 0000000..f366720 --- /dev/null +++ b/diffusers-0.27.0/examples/unconditional_image_generation/requirements.txt @@ -0,0 +1,3 @@ +accelerate>=0.16.0 +torchvision +datasets diff --git a/diffusers-0.27.0/examples/unconditional_image_generation/test_unconditional.py b/diffusers-0.27.0/examples/unconditional_image_generation/test_unconditional.py new file mode 100755 index 0000000..ef71da0 --- /dev/null +++ b/diffusers-0.27.0/examples/unconditional_image_generation/test_unconditional.py @@ -0,0 +1,130 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import sys +import tempfile + + +sys.path.append("..") +from test_examples_utils import ExamplesTestsAccelerate, run_command # noqa: E402 + + +logging.basicConfig(level=logging.DEBUG) + +logger = logging.getLogger() +stream_handler = logging.StreamHandler(sys.stdout) +logger.addHandler(stream_handler) + + +class Unconditional(ExamplesTestsAccelerate): + def test_train_unconditional(self): + with tempfile.TemporaryDirectory() as tmpdir: + test_args = f""" + examples/unconditional_image_generation/train_unconditional.py + --dataset_name hf-internal-testing/dummy_image_class_data + --model_config_name_or_path diffusers/ddpm_dummy + --resolution 64 + --output_dir {tmpdir} + --train_batch_size 2 + --num_epochs 1 + --gradient_accumulation_steps 1 + --ddpm_num_inference_steps 2 + --learning_rate 1e-3 + --lr_warmup_steps 5 + """.split() + + run_command(self._launch_args + test_args, return_stdout=True) + # save_pretrained smoke test + self.assertTrue(os.path.isfile(os.path.join(tmpdir, "unet", "diffusion_pytorch_model.safetensors"))) + self.assertTrue(os.path.isfile(os.path.join(tmpdir, "scheduler", "scheduler_config.json"))) + + def test_unconditional_checkpointing_checkpoints_total_limit(self): + with tempfile.TemporaryDirectory() as tmpdir: + initial_run_args = f""" + examples/unconditional_image_generation/train_unconditional.py + --dataset_name hf-internal-testing/dummy_image_class_data + --model_config_name_or_path diffusers/ddpm_dummy + --resolution 64 + --output_dir {tmpdir} + --train_batch_size 1 + --num_epochs 1 + --gradient_accumulation_steps 1 + --ddpm_num_inference_steps 2 + --learning_rate 1e-3 + --lr_warmup_steps 5 + --checkpointing_steps=2 + --checkpoints_total_limit=2 + """.split() + + run_command(self._launch_args + initial_run_args) + + # check checkpoint directories exist + self.assertEqual( + {x for x in os.listdir(tmpdir) if "checkpoint" in x}, + # checkpoint-2 should have been deleted + {"checkpoint-4", "checkpoint-6"}, + ) + + def test_unconditional_checkpointing_checkpoints_total_limit_removes_multiple_checkpoints(self): + with tempfile.TemporaryDirectory() as tmpdir: + initial_run_args = f""" + examples/unconditional_image_generation/train_unconditional.py + --dataset_name hf-internal-testing/dummy_image_class_data + --model_config_name_or_path diffusers/ddpm_dummy + --resolution 64 + --output_dir {tmpdir} + --train_batch_size 1 + --num_epochs 1 + --gradient_accumulation_steps 1 + --ddpm_num_inference_steps 1 + --learning_rate 1e-3 + --lr_warmup_steps 5 + --checkpointing_steps=2 + """.split() + + run_command(self._launch_args + initial_run_args) + + # check checkpoint directories exist + self.assertEqual( + {x for x in os.listdir(tmpdir) if "checkpoint" in x}, + {"checkpoint-2", "checkpoint-4", "checkpoint-6"}, + ) + + resume_run_args = f""" + examples/unconditional_image_generation/train_unconditional.py + --dataset_name hf-internal-testing/dummy_image_class_data + --model_config_name_or_path diffusers/ddpm_dummy + --resolution 64 + --output_dir {tmpdir} + --train_batch_size 1 + --num_epochs 2 + --gradient_accumulation_steps 1 + --ddpm_num_inference_steps 1 + --learning_rate 1e-3 + --lr_warmup_steps 5 + --resume_from_checkpoint=checkpoint-6 + --checkpointing_steps=2 + --checkpoints_total_limit=2 + """.split() + + run_command(self._launch_args + resume_run_args) + + # check checkpoint directories exist + self.assertEqual( + {x for x in os.listdir(tmpdir) if "checkpoint" in x}, + {"checkpoint-10", "checkpoint-12"}, + ) diff --git a/diffusers-0.27.0/examples/unconditional_image_generation/train_unconditional.py b/diffusers-0.27.0/examples/unconditional_image_generation/train_unconditional.py new file mode 100755 index 0000000..315401e --- /dev/null +++ b/diffusers-0.27.0/examples/unconditional_image_generation/train_unconditional.py @@ -0,0 +1,704 @@ +import argparse +import inspect +import logging +import math +import os +import shutil +from datetime import timedelta +from pathlib import Path + +import accelerate +import datasets +import torch +import torch.nn.functional as F +from accelerate import Accelerator, InitProcessGroupKwargs +from accelerate.logging import get_logger +from accelerate.utils import ProjectConfiguration +from datasets import load_dataset +from huggingface_hub import create_repo, upload_folder +from packaging import version +from torchvision import transforms +from tqdm.auto import tqdm + +import diffusers +from diffusers import DDPMPipeline, DDPMScheduler, UNet2DModel +from diffusers.optimization import get_scheduler +from diffusers.training_utils import EMAModel +from diffusers.utils import check_min_version, is_accelerate_version, is_tensorboard_available, is_wandb_available +from diffusers.utils.import_utils import is_xformers_available + + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.27.0") + +logger = get_logger(__name__, log_level="INFO") + + +def _extract_into_tensor(arr, timesteps, broadcast_shape): + """ + Extract values from a 1-D numpy array for a batch of indices. + + :param arr: the 1-D numpy array. + :param timesteps: a tensor of indices into the array to extract. + :param broadcast_shape: a larger shape of K dimensions with the batch + dimension equal to the length of timesteps. + :return: a tensor of shape [batch_size, 1, ...] where the shape has K dims. + """ + if not isinstance(arr, torch.Tensor): + arr = torch.from_numpy(arr) + res = arr[timesteps].float().to(timesteps.device) + while len(res.shape) < len(broadcast_shape): + res = res[..., None] + return res.expand(broadcast_shape) + + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of a training script.") + parser.add_argument( + "--dataset_name", + type=str, + default=None, + help=( + "The name of the Dataset (from the HuggingFace hub) to train on (could be your own, possibly private," + " dataset). It can also be a path pointing to a local copy of a dataset in your filesystem," + " or to a folder containing files that HF Datasets can understand." + ), + ) + parser.add_argument( + "--dataset_config_name", + type=str, + default=None, + help="The config of the Dataset, leave as None if there's only one config.", + ) + parser.add_argument( + "--model_config_name_or_path", + type=str, + default=None, + help="The config of the UNet model to train, leave as None to use standard DDPM configuration.", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. Folder contents must follow the structure described in" + " https://huggingface.co/docs/datasets/image_dataset#imagefolder. In particular, a `metadata.jsonl` file" + " must exist to provide the captions for the images. Ignored if `dataset_name` is specified." + ), + ) + parser.add_argument( + "--output_dir", + type=str, + default="ddpm-model-64", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument("--overwrite_output_dir", action="store_true") + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument( + "--resolution", + type=int, + default=64, + help=( + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" + ), + ) + parser.add_argument( + "--center_crop", + default=False, + action="store_true", + help=( + "Whether to center crop the input images to the resolution. If not set, the images will be randomly" + " cropped. The images will be resized to the resolution first before cropping." + ), + ) + parser.add_argument( + "--random_flip", + default=False, + action="store_true", + help="whether to randomly flip images horizontally", + ) + parser.add_argument( + "--train_batch_size", type=int, default=16, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument( + "--eval_batch_size", type=int, default=16, help="The number of images to generate for evaluation." + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "The number of subprocesses to use for data loading. 0 means that the data will be loaded in the main" + " process." + ), + ) + parser.add_argument("--num_epochs", type=int, default=100) + parser.add_argument("--save_images_epochs", type=int, default=10, help="How often to save images during training.") + parser.add_argument( + "--save_model_epochs", type=int, default=10, help="How often to save the model during training." + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="Initial learning rate (after the potential warmup period) to use.", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="cosine", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument("--adam_beta1", type=float, default=0.95, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument( + "--adam_weight_decay", type=float, default=1e-6, help="Weight decay magnitude for the Adam optimizer." + ) + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer.") + parser.add_argument( + "--use_ema", + action="store_true", + help="Whether to use Exponential Moving Average for the final model weights.", + ) + parser.add_argument("--ema_inv_gamma", type=float, default=1.0, help="The inverse gamma value for the EMA decay.") + parser.add_argument("--ema_power", type=float, default=3 / 4, help="The power value for the EMA decay.") + parser.add_argument("--ema_max_decay", type=float, default=0.9999, help="The maximum decay magnitude for EMA.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--hub_private_repo", action="store_true", help="Whether or not to create a private repository." + ) + parser.add_argument( + "--logger", + type=str, + default="tensorboard", + choices=["tensorboard", "wandb"], + help=( + "Whether to use [tensorboard](https://www.tensorflow.org/tensorboard) or [wandb](https://www.wandb.ai)" + " for experiment tracking and logging of model metrics and model checkpoints" + ), + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--mixed_precision", + type=str, + default="no", + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose" + "between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >= 1.10." + "and an Nvidia Ampere GPU." + ), + ) + parser.add_argument( + "--prediction_type", + type=str, + default="epsilon", + choices=["epsilon", "sample"], + help="Whether the model should predict the 'epsilon'/noise error or directly the reconstructed image 'x0'.", + ) + parser.add_argument("--ddpm_num_steps", type=int, default=1000) + parser.add_argument("--ddpm_num_inference_steps", type=int, default=1000) + parser.add_argument("--ddpm_beta_schedule", type=str, default="linear") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--enable_xformers_memory_efficient_attention", action="store_true", help="Whether or not to use xformers." + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + if args.dataset_name is None and args.train_data_dir is None: + raise ValueError("You must specify either a dataset name from the hub or a train data directory.") + + return args + + +def main(args): + logging_dir = os.path.join(args.output_dir, args.logging_dir) + accelerator_project_config = ProjectConfiguration(project_dir=args.output_dir, logging_dir=logging_dir) + + kwargs = InitProcessGroupKwargs(timeout=timedelta(seconds=7200)) # a big number for high resolution or big dataset + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.logger, + project_config=accelerator_project_config, + kwargs_handlers=[kwargs], + ) + + if args.logger == "tensorboard": + if not is_tensorboard_available(): + raise ImportError("Make sure to install tensorboard if you want to use it for logging during training.") + + elif args.logger == "wandb": + if not is_wandb_available(): + raise ImportError("Make sure to install wandb if you want to use it for logging during training.") + import wandb + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + if args.use_ema: + ema_model.save_pretrained(os.path.join(output_dir, "unet_ema")) + + for i, model in enumerate(models): + model.save_pretrained(os.path.join(output_dir, "unet")) + + # make sure to pop weight so that corresponding model is not saved again + weights.pop() + + def load_model_hook(models, input_dir): + if args.use_ema: + load_model = EMAModel.from_pretrained(os.path.join(input_dir, "unet_ema"), UNet2DModel) + ema_model.load_state_dict(load_model.state_dict()) + ema_model.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = UNet2DModel.from_pretrained(input_dir, subfolder="unet") + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + diffusers.utils.logging.set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + diffusers.utils.logging.set_verbosity_error() + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + + # Initialize the model + if args.model_config_name_or_path is None: + model = UNet2DModel( + sample_size=args.resolution, + in_channels=3, + out_channels=3, + layers_per_block=2, + block_out_channels=(128, 128, 256, 256, 512, 512), + down_block_types=( + "DownBlock2D", + "DownBlock2D", + "DownBlock2D", + "DownBlock2D", + "AttnDownBlock2D", + "DownBlock2D", + ), + up_block_types=( + "UpBlock2D", + "AttnUpBlock2D", + "UpBlock2D", + "UpBlock2D", + "UpBlock2D", + "UpBlock2D", + ), + ) + else: + config = UNet2DModel.load_config(args.model_config_name_or_path) + model = UNet2DModel.from_config(config) + + # Create EMA for the model. + if args.use_ema: + ema_model = EMAModel( + model.parameters(), + decay=args.ema_max_decay, + use_ema_warmup=True, + inv_gamma=args.ema_inv_gamma, + power=args.ema_power, + model_cls=UNet2DModel, + model_config=model.config, + ) + + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + args.mixed_precision = accelerator.mixed_precision + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + args.mixed_precision = accelerator.mixed_precision + + if args.enable_xformers_memory_efficient_attention: + if is_xformers_available(): + import xformers + + xformers_version = version.parse(xformers.__version__) + if xformers_version == version.parse("0.0.16"): + logger.warning( + "xFormers 0.0.16 cannot be used for training in some GPUs. If you observe problems during training, please update xFormers to at least 0.0.17. See https://huggingface.co/docs/diffusers/main/en/optimization/xformers for more details." + ) + model.enable_xformers_memory_efficient_attention() + else: + raise ValueError("xformers is not available. Make sure it is installed correctly") + + # Initialize the scheduler + accepts_prediction_type = "prediction_type" in set(inspect.signature(DDPMScheduler.__init__).parameters.keys()) + if accepts_prediction_type: + noise_scheduler = DDPMScheduler( + num_train_timesteps=args.ddpm_num_steps, + beta_schedule=args.ddpm_beta_schedule, + prediction_type=args.prediction_type, + ) + else: + noise_scheduler = DDPMScheduler(num_train_timesteps=args.ddpm_num_steps, beta_schedule=args.ddpm_beta_schedule) + + # Initialize the optimizer + optimizer = torch.optim.AdamW( + model.parameters(), + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the datasets: you can either provide your own training and evaluation files (see below) + # or specify a Dataset from the hub (the dataset will be downloaded automatically from the datasets Hub). + + # In distributed training, the load_dataset function guarantees that only one local process can concurrently + # download the dataset. + if args.dataset_name is not None: + dataset = load_dataset( + args.dataset_name, + args.dataset_config_name, + cache_dir=args.cache_dir, + split="train", + ) + else: + dataset = load_dataset("imagefolder", data_dir=args.train_data_dir, cache_dir=args.cache_dir, split="train") + # See more about loading custom images at + # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder + + # Preprocessing the datasets and DataLoaders creation. + augmentations = transforms.Compose( + [ + transforms.Resize(args.resolution, interpolation=transforms.InterpolationMode.BILINEAR), + transforms.CenterCrop(args.resolution) if args.center_crop else transforms.RandomCrop(args.resolution), + transforms.RandomHorizontalFlip() if args.random_flip else transforms.Lambda(lambda x: x), + transforms.ToTensor(), + transforms.Normalize([0.5], [0.5]), + ] + ) + + def transform_images(examples): + images = [augmentations(image.convert("RGB")) for image in examples["image"]] + return {"input": images} + + logger.info(f"Dataset size: {len(dataset)}") + + dataset.set_transform(transform_images) + train_dataloader = torch.utils.data.DataLoader( + dataset, batch_size=args.train_batch_size, shuffle=True, num_workers=args.dataloader_num_workers + ) + + # Initialize the learning rate scheduler + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, + num_training_steps=(len(train_dataloader) * args.num_epochs), + ) + + # Prepare everything with our `accelerator`. + model, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + model, optimizer, train_dataloader, lr_scheduler + ) + + if args.use_ema: + ema_model.to(accelerator.device) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + run = os.path.split(__file__)[-1].split(".")[0] + accelerator.init_trackers(run) + + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + max_train_steps = args.num_epochs * num_update_steps_per_epoch + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(dataset)}") + logger.info(f" Num Epochs = {args.num_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {max_train_steps}") + + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + else: + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + resume_global_step = global_step * args.gradient_accumulation_steps + first_epoch = global_step // num_update_steps_per_epoch + resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) + + # Train! + for epoch in range(first_epoch, args.num_epochs): + model.train() + progress_bar = tqdm(total=num_update_steps_per_epoch, disable=not accelerator.is_local_main_process) + progress_bar.set_description(f"Epoch {epoch}") + for step, batch in enumerate(train_dataloader): + # Skip steps until we reach the resumed step + if args.resume_from_checkpoint and epoch == first_epoch and step < resume_step: + if step % args.gradient_accumulation_steps == 0: + progress_bar.update(1) + continue + + clean_images = batch["input"].to(weight_dtype) + # Sample noise that we'll add to the images + noise = torch.randn(clean_images.shape, dtype=weight_dtype, device=clean_images.device) + bsz = clean_images.shape[0] + # Sample a random timestep for each image + timesteps = torch.randint( + 0, noise_scheduler.config.num_train_timesteps, (bsz,), device=clean_images.device + ).long() + + # Add noise to the clean images according to the noise magnitude at each timestep + # (this is the forward diffusion process) + noisy_images = noise_scheduler.add_noise(clean_images, noise, timesteps) + + with accelerator.accumulate(model): + # Predict the noise residual + model_output = model(noisy_images, timesteps).sample + + if args.prediction_type == "epsilon": + loss = F.mse_loss(model_output.float(), noise.float()) # this could have different weights! + elif args.prediction_type == "sample": + alpha_t = _extract_into_tensor( + noise_scheduler.alphas_cumprod, timesteps, (clean_images.shape[0], 1, 1, 1) + ) + snr_weights = alpha_t / (1 - alpha_t) + # use SNR weighting from distillation paper + loss = snr_weights * F.mse_loss(model_output.float(), clean_images.float(), reduction="none") + loss = loss.mean() + else: + raise ValueError(f"Unsupported prediction type: {args.prediction_type}") + + accelerator.backward(loss) + + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(model.parameters(), 1.0) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + if args.use_ema: + ema_model.step(model.parameters()) + progress_bar.update(1) + global_step += 1 + + if accelerator.is_main_process: + if global_step % args.checkpointing_steps == 0: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + logs = {"loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0], "step": global_step} + if args.use_ema: + logs["ema_decay"] = ema_model.cur_decay_value + progress_bar.set_postfix(**logs) + accelerator.log(logs, step=global_step) + progress_bar.close() + + accelerator.wait_for_everyone() + + # Generate sample images for visual inspection + if accelerator.is_main_process: + if epoch % args.save_images_epochs == 0 or epoch == args.num_epochs - 1: + unet = accelerator.unwrap_model(model) + + if args.use_ema: + ema_model.store(unet.parameters()) + ema_model.copy_to(unet.parameters()) + + pipeline = DDPMPipeline( + unet=unet, + scheduler=noise_scheduler, + ) + + generator = torch.Generator(device=pipeline.device).manual_seed(0) + # run pipeline in inference (sample random noise and denoise) + images = pipeline( + generator=generator, + batch_size=args.eval_batch_size, + num_inference_steps=args.ddpm_num_inference_steps, + output_type="numpy", + ).images + + if args.use_ema: + ema_model.restore(unet.parameters()) + + # denormalize the images and save to tensorboard + images_processed = (images * 255).round().astype("uint8") + + if args.logger == "tensorboard": + if is_accelerate_version(">=", "0.17.0.dev0"): + tracker = accelerator.get_tracker("tensorboard", unwrap=True) + else: + tracker = accelerator.get_tracker("tensorboard") + tracker.add_images("test_samples", images_processed.transpose(0, 3, 1, 2), epoch) + elif args.logger == "wandb": + # Upcoming `log_images` helper coming in https://github.com/huggingface/accelerate/pull/962/files + accelerator.get_tracker("wandb").log( + {"test_samples": [wandb.Image(img) for img in images_processed], "epoch": epoch}, + step=global_step, + ) + + if epoch % args.save_model_epochs == 0 or epoch == args.num_epochs - 1: + # save the model + unet = accelerator.unwrap_model(model) + + if args.use_ema: + ema_model.store(unet.parameters()) + ema_model.copy_to(unet.parameters()) + + pipeline = DDPMPipeline( + unet=unet, + scheduler=noise_scheduler, + ) + + pipeline.save_pretrained(args.output_dir) + + if args.use_ema: + ema_model.restore(unet.parameters()) + + if args.push_to_hub: + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message=f"Epoch {epoch}", + ignore_patterns=["step_*", "epoch_*"], + ) + + accelerator.end_training() + + +if __name__ == "__main__": + args = parse_args() + main(args) diff --git a/diffusers-0.27.0/examples/wuerstchen/text_to_image/README.md b/diffusers-0.27.0/examples/wuerstchen/text_to_image/README.md new file mode 100755 index 0000000..d655259 --- /dev/null +++ b/diffusers-0.27.0/examples/wuerstchen/text_to_image/README.md @@ -0,0 +1,93 @@ +# Würstchen text-to-image fine-tuning + +## Running locally with PyTorch + +Before running the scripts, make sure to install the library's training dependencies: + +**Important** + +To make sure you can successfully run the latest versions of the example scripts, we highly recommend **installing from source** and keeping the install up to date. To do this, execute the following steps in a new virtual environment: +```bash +git clone https://github.com/huggingface/diffusers +cd diffusers +pip install . +``` + +Then cd into the example folder and run +```bash +cd examples/wuerstchen/text_to_image +pip install -r requirements.txt +``` + +And initialize an [🤗Accelerate](https://github.com/huggingface/accelerate/) environment with: + +```bash +accelerate config +``` +For this example we want to directly store the trained LoRA embeddings on the Hub, so we need to be logged in and add the `--push_to_hub` flag to the training script. To log in, run: +```bash +huggingface-cli login +``` + +## Prior training + +You can fine-tune the Würstchen prior model with the `train_text_to_image_prior.py` script. Note that we currently support `--gradient_checkpointing` for prior model fine-tuning so you can use it for more GPU memory constrained setups. + +
    + + +```bash +export DATASET_NAME="lambdalabs/pokemon-blip-captions" + +accelerate launch train_text_to_image_prior.py \ + --mixed_precision="fp16" \ + --dataset_name=$DATASET_NAME \ + --resolution=768 \ + --train_batch_size=4 \ + --gradient_accumulation_steps=4 \ + --gradient_checkpointing \ + --dataloader_num_workers=4 \ + --max_train_steps=15000 \ + --learning_rate=1e-05 \ + --max_grad_norm=1 \ + --checkpoints_total_limit=3 \ + --lr_scheduler="constant" --lr_warmup_steps=0 \ + --validation_prompts="A robot pokemon, 4k photo" \ + --report_to="wandb" \ + --push_to_hub \ + --output_dir="wuerstchen-prior-pokemon-model" +``` + + +## Training with LoRA + +Low-Rank Adaption of Large Language Models (or LoRA) was first introduced by Microsoft in [LoRA: Low-Rank Adaptation of Large Language Models](https://arxiv.org/abs/2106.09685) by *Edward J. Hu, Yelong Shen, Phillip Wallis, Zeyuan Allen-Zhu, Yuanzhi Li, Shean Wang, Lu Wang, Weizhu Chen*. + +In a nutshell, LoRA allows adapting pretrained models by adding pairs of rank-decomposition matrices to existing weights and **only** training those newly added weights. This has a couple of advantages: + +- Previous pretrained weights are kept frozen so that the model is not prone to [catastrophic forgetting](https://www.pnas.org/doi/10.1073/pnas.1611835114). +- Rank-decomposition matrices have significantly fewer parameters than original model, which means that trained LoRA weights are easily portable. +- LoRA attention layers allow to control to which extent the model is adapted toward new training images via a `scale` parameter. + + +### Prior Training + +First, you need to set up your development environment as explained in the [installation](#Running-locally-with-PyTorch) section. Make sure to set the `DATASET_NAME` environment variable. Here, we will use the [Pokemon captions dataset](https://huggingface.co/datasets/lambdalabs/pokemon-blip-captions). + +```bash +export DATASET_NAME="lambdalabs/pokemon-blip-captions" + +accelerate launch train_text_to_image_lora_prior.py \ + --mixed_precision="fp16" \ + --dataset_name=$DATASET_NAME --caption_column="text" \ + --resolution=768 \ + --train_batch_size=8 \ + --num_train_epochs=100 --checkpointing_steps=5000 \ + --learning_rate=1e-04 --lr_scheduler="constant" --lr_warmup_steps=0 \ + --seed=42 \ + --rank=4 \ + --validation_prompt="cute dragon creature" \ + --report_to="wandb" \ + --push_to_hub \ + --output_dir="wuerstchen-prior-pokemon-lora" +``` diff --git a/diffusers-0.27.0/examples/wuerstchen/text_to_image/__init__.py b/diffusers-0.27.0/examples/wuerstchen/text_to_image/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/examples/wuerstchen/text_to_image/modeling_efficient_net_encoder.py b/diffusers-0.27.0/examples/wuerstchen/text_to_image/modeling_efficient_net_encoder.py new file mode 100755 index 0000000..bd551eb --- /dev/null +++ b/diffusers-0.27.0/examples/wuerstchen/text_to_image/modeling_efficient_net_encoder.py @@ -0,0 +1,23 @@ +import torch.nn as nn +from torchvision.models import efficientnet_v2_l, efficientnet_v2_s + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.models.modeling_utils import ModelMixin + + +class EfficientNetEncoder(ModelMixin, ConfigMixin): + @register_to_config + def __init__(self, c_latent=16, c_cond=1280, effnet="efficientnet_v2_s"): + super().__init__() + + if effnet == "efficientnet_v2_s": + self.backbone = efficientnet_v2_s(weights="DEFAULT").features + else: + self.backbone = efficientnet_v2_l(weights="DEFAULT").features + self.mapper = nn.Sequential( + nn.Conv2d(c_cond, c_latent, kernel_size=1, bias=False), + nn.BatchNorm2d(c_latent), # then normalize them to have mean 0 and std 1 + ) + + def forward(self, x): + return self.mapper(self.backbone(x)) diff --git a/diffusers-0.27.0/examples/wuerstchen/text_to_image/requirements.txt b/diffusers-0.27.0/examples/wuerstchen/text_to_image/requirements.txt new file mode 100755 index 0000000..dfe8f3d --- /dev/null +++ b/diffusers-0.27.0/examples/wuerstchen/text_to_image/requirements.txt @@ -0,0 +1,7 @@ +accelerate>=0.16.0 +torchvision +transformers>=4.25.1 +wandb +bitsandbytes +deepspeed +peft>=0.6.0 diff --git a/diffusers-0.27.0/examples/wuerstchen/text_to_image/train_text_to_image_lora_prior.py b/diffusers-0.27.0/examples/wuerstchen/text_to_image/train_text_to_image_lora_prior.py new file mode 100755 index 0000000..b25e67d --- /dev/null +++ b/diffusers-0.27.0/examples/wuerstchen/text_to_image/train_text_to_image_lora_prior.py @@ -0,0 +1,945 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import logging +import math +import os +import random +import shutil +from pathlib import Path + +import datasets +import numpy as np +import torch +import torch.nn.functional as F +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState, is_initialized +from accelerate.utils import ProjectConfiguration, set_seed +from datasets import load_dataset +from huggingface_hub import create_repo, hf_hub_download, upload_folder +from modeling_efficient_net_encoder import EfficientNetEncoder +from peft import LoraConfig +from peft.utils import get_peft_model_state_dict +from torchvision import transforms +from tqdm import tqdm +from transformers import CLIPTextModel, PreTrainedTokenizerFast +from transformers.utils import ContextManagers + +from diffusers import AutoPipelineForText2Image, DDPMWuerstchenScheduler, WuerstchenPriorPipeline +from diffusers.optimization import get_scheduler +from diffusers.pipelines.wuerstchen import DEFAULT_STAGE_C_TIMESTEPS, WuerstchenPrior +from diffusers.utils import check_min_version, is_wandb_available, make_image_grid +from diffusers.utils.logging import set_verbosity_error, set_verbosity_info + + +if is_wandb_available(): + import wandb + + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.27.0") + +logger = get_logger(__name__, log_level="INFO") + +DATASET_NAME_MAPPING = { + "lambdalabs/pokemon-blip-captions": ("image", "text"), +} + + +def save_model_card( + args, + repo_id: str, + images=None, + repo_folder=None, +): + img_str = "" + if len(images) > 0: + image_grid = make_image_grid(images, 1, len(args.validation_prompts)) + image_grid.save(os.path.join(repo_folder, "val_imgs_grid.png")) + img_str += "![val_imgs_grid](./val_imgs_grid.png)\n" + + yaml = f""" +--- +license: mit +base_model: {args.pretrained_prior_model_name_or_path} +datasets: +- {args.dataset_name} +tags: +- wuerstchen +- text-to-image +- diffusers +- diffusers-training +- lora +inference: true +--- + """ + model_card = f""" +# LoRA Finetuning - {repo_id} + +This pipeline was finetuned from **{args.pretrained_prior_model_name_or_path}** on the **{args.dataset_name}** dataset. Below are some example images generated with the finetuned pipeline using the following prompts: {args.validation_prompts}: \n +{img_str} + +## Pipeline usage + +You can use the pipeline like so: + +```python +from diffusers import DiffusionPipeline +import torch + +pipeline = AutoPipelineForText2Image.from_pretrained( + "{args.pretrained_decoder_model_name_or_path}", torch_dtype={args.weight_dtype} + ) +# load lora weights from folder: +pipeline.prior_pipe.load_lora_weights("{repo_id}", torch_dtype={args.weight_dtype}) + +image = pipeline(prompt=prompt).images[0] +image.save("my_image.png") +``` + +## Training info + +These are the key hyperparameters used during training: + +* LoRA rank: {args.rank} +* Epochs: {args.num_train_epochs} +* Learning rate: {args.learning_rate} +* Batch size: {args.train_batch_size} +* Gradient accumulation steps: {args.gradient_accumulation_steps} +* Image resolution: {args.resolution} +* Mixed-precision: {args.mixed_precision} + +""" + wandb_info = "" + if is_wandb_available(): + wandb_run_url = None + if wandb.run is not None: + wandb_run_url = wandb.run.url + + if wandb_run_url is not None: + wandb_info = f""" +More information on all the CLI arguments and the environment are available on your [`wandb` run page]({wandb_run_url}). +""" + + model_card += wandb_info + + with open(os.path.join(repo_folder, "README.md"), "w") as f: + f.write(yaml + model_card) + + +def log_validation(text_encoder, tokenizer, prior, args, accelerator, weight_dtype, epoch): + logger.info("Running validation... ") + + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, + prior=accelerator.unwrap_model(prior), + prior_text_encoder=accelerator.unwrap_model(text_encoder), + prior_tokenizer=tokenizer, + torch_dtype=weight_dtype, + ) + pipeline = pipeline.to(accelerator.device) + pipeline.set_progress_bar_config(disable=True) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.cuda.amp.autocast(): + image = pipeline( + args.validation_prompts[i], + prior_timesteps=DEFAULT_STAGE_C_TIMESTEPS, + generator=generator, + height=args.resolution, + width=args.resolution, + ).images[0] + images.append(image) + + for tracker in accelerator.trackers: + if tracker.name == "tensorboard": + np_images = np.stack([np.asarray(img) for img in images]) + tracker.writer.add_images("validation", np_images, epoch, dataformats="NHWC") + elif tracker.name == "wandb": + tracker.log( + { + "validation": [ + wandb.Image(image, caption=f"{i}: {args.validation_prompts[i]}") + for i, image in enumerate(images) + ] + } + ) + else: + logger.warning(f"image logging not implemented for {tracker.name}") + + del pipeline + torch.cuda.empty_cache() + + return images + + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of finetuning Würstchen Prior.") + parser.add_argument( + "--rank", + type=int, + default=4, + help=("The dimension of the LoRA update matrices."), + ) + parser.add_argument( + "--pretrained_decoder_model_name_or_path", + type=str, + default="warp-ai/wuerstchen", + required=False, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--pretrained_prior_model_name_or_path", + type=str, + default="warp-ai/wuerstchen-prior", + required=False, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--dataset_name", + type=str, + default=None, + help=( + "The name of the Dataset (from the HuggingFace hub) to train on (could be your own, possibly private," + " dataset). It can also be a path pointing to a local copy of a dataset in your filesystem," + " or to a folder containing files that 🤗 Datasets can understand." + ), + ) + parser.add_argument( + "--dataset_config_name", + type=str, + default=None, + help="The config of the Dataset, leave as None if there's only one config.", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. Folder contents must follow the structure described in" + " https://huggingface.co/docs/datasets/image_dataset#imagefolder. In particular, a `metadata.jsonl` file" + " must exist to provide the captions for the images. Ignored if `dataset_name` is specified." + ), + ) + parser.add_argument( + "--image_column", type=str, default="image", help="The column of the dataset containing an image." + ) + parser.add_argument( + "--caption_column", + type=str, + default="text", + help="The column of the dataset containing a caption or a list of captions.", + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="wuerstchen-model-finetuned-lora", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--resolution", + type=int, + default=512, + help=( + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" + ), + ) + parser.add_argument( + "--train_batch_size", type=int, default=1, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="learning rate", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument( + "--adam_weight_decay", + type=float, + default=0.0, + required=False, + help="weight decay_to_use", + ) + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # Sanity checks + if args.dataset_name is None and args.train_data_dir is None: + raise ValueError("Need either a dataset name or a training folder.") + + return args + + +def main(): + args = parse_args() + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + logging_dir = os.path.join(args.output_dir, args.logging_dir) + accelerator_project_config = ProjectConfiguration( + total_limit=args.checkpoints_total_limit, project_dir=args.output_dir, logging_dir=logging_dir + ) + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + + # Load scheduler, effnet, tokenizer, clip_model + noise_scheduler = DDPMWuerstchenScheduler() + tokenizer = PreTrainedTokenizerFast.from_pretrained( + args.pretrained_prior_model_name_or_path, subfolder="tokenizer" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + pretrained_checkpoint_file = hf_hub_download("dome272/wuerstchen", filename="model_v2_stage_b.pt") + state_dict = torch.load(pretrained_checkpoint_file, map_location="cpu") + image_encoder = EfficientNetEncoder() + image_encoder.load_state_dict(state_dict["effnet_state_dict"]) + image_encoder.eval() + + text_encoder = CLIPTextModel.from_pretrained( + args.pretrained_prior_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ).eval() + + # Freeze text_encoder, cast to weight_dtype and image_encoder and move to device + text_encoder.requires_grad_(False) + image_encoder.requires_grad_(False) + image_encoder.to(accelerator.device, dtype=weight_dtype) + text_encoder.to(accelerator.device, dtype=weight_dtype) + + # load prior model, cast to weight_dtype and move to device + prior = WuerstchenPrior.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder="prior") + prior.to(accelerator.device, dtype=weight_dtype) + + # lora attn processor + prior_lora_config = LoraConfig( + r=args.rank, + lora_alpha=args.rank, + target_modules=["to_k", "to_q", "to_v", "to_out.0", "add_k_proj", "add_v_proj"], + ) + # Add adapter and make sure the trainable params are in float32. + prior.add_adapter(prior_lora_config) + if args.mixed_precision == "fp16": + for param in prior.parameters(): + # only upcast trainable parameters (LoRA) into fp32 + if param.requires_grad: + param.data = param.to(torch.float32) + + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if accelerator.is_main_process: + prior_lora_layers_to_save = None + + for model in models: + if isinstance(model, type(accelerator.unwrap_model(prior))): + prior_lora_layers_to_save = get_peft_model_state_dict(model) + else: + raise ValueError(f"unexpected save model: {model.__class__}") + + # make sure to pop weight so that corresponding model is not saved again + weights.pop() + + WuerstchenPriorPipeline.save_lora_weights( + output_dir, + unet_lora_layers=prior_lora_layers_to_save, + ) + + def load_model_hook(models, input_dir): + prior_ = None + + while len(models) > 0: + model = models.pop() + + if isinstance(model, type(accelerator.unwrap_model(prior))): + prior_ = model + else: + raise ValueError(f"unexpected save model: {model.__class__}") + + lora_state_dict, network_alphas = WuerstchenPriorPipeline.lora_state_dict(input_dir) + WuerstchenPriorPipeline.load_lora_into_unet(lora_state_dict, network_alphas=network_alphas, unet=prior_) + WuerstchenPriorPipeline.load_lora_into_text_encoder( + lora_state_dict, + network_alphas=network_alphas, + ) + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + else: + optimizer_cls = torch.optim.AdamW + params_to_optimize = list(filter(lambda p: p.requires_grad, prior.parameters())) + optimizer = optimizer_cls( + params_to_optimize, + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the datasets: you can either provide your own training and evaluation files (see below) + # or specify a Dataset from the hub (the dataset will be downloaded automatically from the datasets Hub). + + # In distributed training, the load_dataset function guarantees that only one local process can concurrently + # download the dataset. + if args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + dataset = load_dataset( + args.dataset_name, + args.dataset_config_name, + cache_dir=args.cache_dir, + ) + else: + data_files = {} + if args.train_data_dir is not None: + data_files["train"] = os.path.join(args.train_data_dir, "**") + dataset = load_dataset( + "imagefolder", + data_files=data_files, + cache_dir=args.cache_dir, + ) + # See more about loading custom images at + # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder + + # Preprocessing the datasets. + # We need to tokenize inputs and targets. + column_names = dataset["train"].column_names + + # Get the column names for input/target. + dataset_columns = DATASET_NAME_MAPPING.get(args.dataset_name, None) + if args.image_column is None: + image_column = dataset_columns[0] if dataset_columns is not None else column_names[0] + else: + image_column = args.image_column + if image_column not in column_names: + raise ValueError( + f"--image_column' value '{args.image_column}' needs to be one of: {', '.join(column_names)}" + ) + if args.caption_column is None: + caption_column = dataset_columns[1] if dataset_columns is not None else column_names[1] + else: + caption_column = args.caption_column + if caption_column not in column_names: + raise ValueError( + f"--caption_column' value '{args.caption_column}' needs to be one of: {', '.join(column_names)}" + ) + + # Preprocessing the datasets. + # We need to tokenize input captions and transform the images + def tokenize_captions(examples, is_train=True): + captions = [] + for caption in examples[caption_column]: + if isinstance(caption, str): + captions.append(caption) + elif isinstance(caption, (list, np.ndarray)): + # take a random caption if there are multiple + captions.append(random.choice(caption) if is_train else caption[0]) + else: + raise ValueError( + f"Caption column `{caption_column}` should contain either strings or lists of strings." + ) + inputs = tokenizer( + captions, max_length=tokenizer.model_max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + text_input_ids = inputs.input_ids + text_mask = inputs.attention_mask.bool() + return text_input_ids, text_mask + + effnet_transforms = transforms.Compose( + [ + transforms.Resize(args.resolution, interpolation=transforms.InterpolationMode.BILINEAR, antialias=True), + transforms.CenterCrop(args.resolution), + transforms.ToTensor(), + transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), + ] + ) + + def preprocess_train(examples): + images = [image.convert("RGB") for image in examples[image_column]] + examples["effnet_pixel_values"] = [effnet_transforms(image) for image in images] + examples["text_input_ids"], examples["text_mask"] = tokenize_captions(examples) + return examples + + with accelerator.main_process_first(): + if args.max_train_samples is not None: + dataset["train"] = dataset["train"].shuffle(seed=args.seed).select(range(args.max_train_samples)) + # Set the training transforms + train_dataset = dataset["train"].with_transform(preprocess_train) + + def collate_fn(examples): + effnet_pixel_values = torch.stack([example["effnet_pixel_values"] for example in examples]) + effnet_pixel_values = effnet_pixel_values.to(memory_format=torch.contiguous_format).float() + text_input_ids = torch.stack([example["text_input_ids"] for example in examples]) + text_mask = torch.stack([example["text_mask"] for example in examples]) + return {"effnet_pixel_values": effnet_pixel_values, "text_input_ids": text_input_ids, "text_mask": text_mask} + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + shuffle=True, + collate_fn=collate_fn, + batch_size=args.train_batch_size, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, + num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, + ) + + prior, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + prior, optimizer, train_dataloader, lr_scheduler + ) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + tracker_config.pop("validation_prompts") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + else: + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + resume_global_step = global_step * args.gradient_accumulation_steps + first_epoch = global_step // num_update_steps_per_epoch + resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) + + # Only show the progress bar once on each machine. + progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) + progress_bar.set_description("Steps") + + for epoch in range(first_epoch, args.num_train_epochs): + prior.train() + train_loss = 0.0 + for step, batch in enumerate(train_dataloader): + # Skip steps until we reach the resumed step + if args.resume_from_checkpoint and epoch == first_epoch and step < resume_step: + if step % args.gradient_accumulation_steps == 0: + progress_bar.update(1) + continue + + with accelerator.accumulate(prior): + # Convert images to latent space + text_input_ids, text_mask, effnet_images = ( + batch["text_input_ids"], + batch["text_mask"], + batch["effnet_pixel_values"].to(weight_dtype), + ) + + with torch.no_grad(): + text_encoder_output = text_encoder(text_input_ids, attention_mask=text_mask) + prompt_embeds = text_encoder_output.last_hidden_state + image_embeds = image_encoder(effnet_images) + # scale + image_embeds = image_embeds.add(1.0).div(42.0) + + # Sample noise that we'll add to the image_embeds + noise = torch.randn_like(image_embeds) + bsz = image_embeds.shape[0] + + # Sample a random timestep for each image + timesteps = torch.rand((bsz,), device=image_embeds.device, dtype=weight_dtype) + + # add noise to latent + noisy_latents = noise_scheduler.add_noise(image_embeds, noise, timesteps) + + # Predict the noise residual and compute losscd + pred_noise = prior(noisy_latents, timesteps, prompt_embeds) + + # vanilla loss + loss = F.mse_loss(pred_noise.float(), noise.float(), reduction="mean") + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(params_to_optimize, args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + log_validation(text_encoder, tokenizer, prior, args, accelerator, weight_dtype, global_step) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + prior = accelerator.unwrap_model(prior) + prior = prior.to(torch.float32) + + prior_lora_state_dict = get_peft_model_state_dict(prior) + + WuerstchenPriorPipeline.save_lora_weights( + save_directory=args.output_dir, + unet_lora_layers=prior_lora_state_dict, + ) + + # Run a final round of inference. + images = [] + if args.validation_prompts is not None: + logger.info("Running inference for collecting generated images...") + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, + prior_text_encoder=accelerator.unwrap_model(text_encoder), + prior_tokenizer=tokenizer, + torch_dtype=weight_dtype, + ) + pipeline = pipeline.to(accelerator.device) + + # load lora weights + pipeline.prior_pipe.load_lora_weights(args.output_dir, weight_name="pytorch_lora_weights.safetensors") + pipeline.set_progress_bar_config(disable=True) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.cuda.amp.autocast(): + image = pipeline( + args.validation_prompts[i], + prior_timesteps=DEFAULT_STAGE_C_TIMESTEPS, + generator=generator, + width=args.resolution, + height=args.resolution, + ).images[0] + images.append(image) + + if args.push_to_hub: + save_model_card(args, repo_id, images, repo_folder=args.output_dir) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["step_*", "epoch_*"], + ) + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/diffusers-0.27.0/examples/wuerstchen/text_to_image/train_text_to_image_prior.py b/diffusers-0.27.0/examples/wuerstchen/text_to_image/train_text_to_image_prior.py new file mode 100755 index 0000000..a30b6d6 --- /dev/null +++ b/diffusers-0.27.0/examples/wuerstchen/text_to_image/train_text_to_image_prior.py @@ -0,0 +1,932 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +import argparse +import logging +import math +import os +import random +import shutil +from pathlib import Path + +import accelerate +import datasets +import numpy as np +import torch +import torch.nn.functional as F +import transformers +from accelerate import Accelerator +from accelerate.logging import get_logger +from accelerate.state import AcceleratorState, is_initialized +from accelerate.utils import ProjectConfiguration, set_seed +from datasets import load_dataset +from huggingface_hub import create_repo, hf_hub_download, upload_folder +from modeling_efficient_net_encoder import EfficientNetEncoder +from packaging import version +from torchvision import transforms +from tqdm import tqdm +from transformers import CLIPTextModel, PreTrainedTokenizerFast +from transformers.utils import ContextManagers + +from diffusers import AutoPipelineForText2Image, DDPMWuerstchenScheduler +from diffusers.optimization import get_scheduler +from diffusers.pipelines.wuerstchen import DEFAULT_STAGE_C_TIMESTEPS, WuerstchenPrior +from diffusers.training_utils import EMAModel +from diffusers.utils import check_min_version, is_wandb_available, make_image_grid +from diffusers.utils.logging import set_verbosity_error, set_verbosity_info + + +if is_wandb_available(): + import wandb + + +# Will error if the minimal version of diffusers is not installed. Remove at your own risks. +check_min_version("0.27.0") + +logger = get_logger(__name__, log_level="INFO") + +DATASET_NAME_MAPPING = { + "lambdalabs/pokemon-blip-captions": ("image", "text"), +} + + +def save_model_card( + args, + repo_id: str, + images=None, + repo_folder=None, +): + img_str = "" + if len(images) > 0: + image_grid = make_image_grid(images, 1, len(args.validation_prompts)) + image_grid.save(os.path.join(repo_folder, "val_imgs_grid.png")) + img_str += "![val_imgs_grid](./val_imgs_grid.png)\n" + + yaml = f""" +--- +license: mit +base_model: {args.pretrained_prior_model_name_or_path} +datasets: +- {args.dataset_name} +tags: +- wuerstchen +- text-to-image +- diffusers +- diffusers-training +inference: true +--- + """ + model_card = f""" +# Finetuning - {repo_id} + +This pipeline was finetuned from **{args.pretrained_prior_model_name_or_path}** on the **{args.dataset_name}** dataset. Below are some example images generated with the finetuned pipeline using the following prompts: {args.validation_prompts}: \n +{img_str} + +## Pipeline usage + +You can use the pipeline like so: + +```python +from diffusers import DiffusionPipeline +import torch + +pipe_prior = DiffusionPipeline.from_pretrained("{repo_id}", torch_dtype={args.weight_dtype}) +pipe_t2i = DiffusionPipeline.from_pretrained("{args.pretrained_decoder_model_name_or_path}", torch_dtype={args.weight_dtype}) +prompt = "{args.validation_prompts[0]}" +(image_embeds,) = pipe_prior(prompt).to_tuple() +image = pipe_t2i(image_embeddings=image_embeds, prompt=prompt).images[0] +image.save("my_image.png") +``` + +## Training info + +These are the key hyperparameters used during training: + +* Epochs: {args.num_train_epochs} +* Learning rate: {args.learning_rate} +* Batch size: {args.train_batch_size} +* Gradient accumulation steps: {args.gradient_accumulation_steps} +* Image resolution: {args.resolution} +* Mixed-precision: {args.mixed_precision} + +""" + wandb_info = "" + if is_wandb_available(): + wandb_run_url = None + if wandb.run is not None: + wandb_run_url = wandb.run.url + + if wandb_run_url is not None: + wandb_info = f""" +More information on all the CLI arguments and the environment are available on your [`wandb` run page]({wandb_run_url}). +""" + + model_card += wandb_info + + with open(os.path.join(repo_folder, "README.md"), "w") as f: + f.write(yaml + model_card) + + +def log_validation(text_encoder, tokenizer, prior, args, accelerator, weight_dtype, epoch): + logger.info("Running validation... ") + + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, + prior_prior=accelerator.unwrap_model(prior), + prior_text_encoder=accelerator.unwrap_model(text_encoder), + prior_tokenizer=tokenizer, + torch_dtype=weight_dtype, + ) + pipeline = pipeline.to(accelerator.device) + pipeline.set_progress_bar_config(disable=True) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + images = [] + for i in range(len(args.validation_prompts)): + with torch.autocast("cuda"): + image = pipeline( + args.validation_prompts[i], + prior_timesteps=DEFAULT_STAGE_C_TIMESTEPS, + generator=generator, + height=args.resolution, + width=args.resolution, + ).images[0] + + images.append(image) + + for tracker in accelerator.trackers: + if tracker.name == "tensorboard": + np_images = np.stack([np.asarray(img) for img in images]) + tracker.writer.add_images("validation", np_images, epoch, dataformats="NHWC") + elif tracker.name == "wandb": + tracker.log( + { + "validation": [ + wandb.Image(image, caption=f"{i}: {args.validation_prompts[i]}") + for i, image in enumerate(images) + ] + } + ) + else: + logger.warning(f"image logging not implemented for {tracker.name}") + + del pipeline + torch.cuda.empty_cache() + + return images + + +def parse_args(): + parser = argparse.ArgumentParser(description="Simple example of finetuning Würstchen Prior.") + parser.add_argument( + "--pretrained_decoder_model_name_or_path", + type=str, + default="warp-ai/wuerstchen", + required=False, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--pretrained_prior_model_name_or_path", + type=str, + default="warp-ai/wuerstchen-prior", + required=False, + help="Path to pretrained model or model identifier from huggingface.co/models.", + ) + parser.add_argument( + "--dataset_name", + type=str, + default=None, + help=( + "The name of the Dataset (from the HuggingFace hub) to train on (could be your own, possibly private," + " dataset). It can also be a path pointing to a local copy of a dataset in your filesystem," + " or to a folder containing files that 🤗 Datasets can understand." + ), + ) + parser.add_argument( + "--dataset_config_name", + type=str, + default=None, + help="The config of the Dataset, leave as None if there's only one config.", + ) + parser.add_argument( + "--train_data_dir", + type=str, + default=None, + help=( + "A folder containing the training data. Folder contents must follow the structure described in" + " https://huggingface.co/docs/datasets/image_dataset#imagefolder. In particular, a `metadata.jsonl` file" + " must exist to provide the captions for the images. Ignored if `dataset_name` is specified." + ), + ) + parser.add_argument( + "--image_column", type=str, default="image", help="The column of the dataset containing an image." + ) + parser.add_argument( + "--caption_column", + type=str, + default="text", + help="The column of the dataset containing a caption or a list of captions.", + ) + parser.add_argument( + "--max_train_samples", + type=int, + default=None, + help=( + "For debugging purposes or quicker training, truncate the number of training examples to this " + "value if set." + ), + ) + parser.add_argument( + "--validation_prompts", + type=str, + default=None, + nargs="+", + help=("A set of prompts evaluated every `--validation_epochs` and logged to `--report_to`."), + ) + parser.add_argument( + "--output_dir", + type=str, + default="wuerstchen-model-finetuned", + help="The output directory where the model predictions and checkpoints will be written.", + ) + parser.add_argument( + "--cache_dir", + type=str, + default=None, + help="The directory where the downloaded models and datasets will be stored.", + ) + parser.add_argument("--seed", type=int, default=None, help="A seed for reproducible training.") + parser.add_argument( + "--resolution", + type=int, + default=512, + help=( + "The resolution for input images, all the images in the train/validation dataset will be resized to this" + " resolution" + ), + ) + parser.add_argument( + "--train_batch_size", type=int, default=1, help="Batch size (per device) for the training dataloader." + ) + parser.add_argument("--num_train_epochs", type=int, default=100) + parser.add_argument( + "--max_train_steps", + type=int, + default=None, + help="Total number of training steps to perform. If provided, overrides num_train_epochs.", + ) + parser.add_argument( + "--gradient_accumulation_steps", + type=int, + default=1, + help="Number of updates steps to accumulate before performing a backward/update pass.", + ) + parser.add_argument( + "--gradient_checkpointing", + action="store_true", + help="Whether or not to use gradient checkpointing to save memory at the expense of slower backward pass.", + ) + parser.add_argument( + "--learning_rate", + type=float, + default=1e-4, + help="learning rate", + ) + parser.add_argument( + "--lr_scheduler", + type=str, + default="constant", + help=( + 'The scheduler type to use. Choose between ["linear", "cosine", "cosine_with_restarts", "polynomial",' + ' "constant", "constant_with_warmup"]' + ), + ) + parser.add_argument( + "--lr_warmup_steps", type=int, default=500, help="Number of steps for the warmup in the lr scheduler." + ) + parser.add_argument( + "--use_8bit_adam", action="store_true", help="Whether or not to use 8-bit Adam from bitsandbytes." + ) + parser.add_argument( + "--allow_tf32", + action="store_true", + help=( + "Whether or not to allow TF32 on Ampere GPUs. Can be used to speed up training. For more information, see" + " https://pytorch.org/docs/stable/notes/cuda.html#tensorfloat-32-tf32-on-ampere-devices" + ), + ) + parser.add_argument("--use_ema", action="store_true", help="Whether to use EMA model.") + parser.add_argument( + "--dataloader_num_workers", + type=int, + default=0, + help=( + "Number of subprocesses to use for data loading. 0 means that the data will be loaded in the main process." + ), + ) + parser.add_argument("--adam_beta1", type=float, default=0.9, help="The beta1 parameter for the Adam optimizer.") + parser.add_argument("--adam_beta2", type=float, default=0.999, help="The beta2 parameter for the Adam optimizer.") + parser.add_argument( + "--adam_weight_decay", + type=float, + default=0.0, + required=False, + help="weight decay_to_use", + ) + parser.add_argument("--adam_epsilon", type=float, default=1e-08, help="Epsilon value for the Adam optimizer") + parser.add_argument("--max_grad_norm", default=1.0, type=float, help="Max gradient norm.") + parser.add_argument("--push_to_hub", action="store_true", help="Whether or not to push the model to the Hub.") + parser.add_argument("--hub_token", type=str, default=None, help="The token to use to push to the Model Hub.") + parser.add_argument( + "--hub_model_id", + type=str, + default=None, + help="The name of the repository to keep in sync with the local `output_dir`.", + ) + parser.add_argument( + "--logging_dir", + type=str, + default="logs", + help=( + "[TensorBoard](https://www.tensorflow.org/tensorboard) log directory. Will default to" + " *output_dir/runs/**CURRENT_DATETIME_HOSTNAME***." + ), + ) + parser.add_argument( + "--mixed_precision", + type=str, + default=None, + choices=["no", "fp16", "bf16"], + help=( + "Whether to use mixed precision. Choose between fp16 and bf16 (bfloat16). Bf16 requires PyTorch >=" + " 1.10.and an Nvidia Ampere GPU. Default to the value of accelerate config of the current system or the" + " flag passed with the `accelerate.launch` command. Use this argument to override the accelerate config." + ), + ) + parser.add_argument( + "--report_to", + type=str, + default="tensorboard", + help=( + 'The integration to report the results and logs to. Supported platforms are `"tensorboard"`' + ' (default), `"wandb"` and `"comet_ml"`. Use `"all"` to report to all integrations.' + ), + ) + parser.add_argument("--local_rank", type=int, default=-1, help="For distributed training: local_rank") + parser.add_argument( + "--checkpointing_steps", + type=int, + default=500, + help=( + "Save a checkpoint of the training state every X updates. These checkpoints are only suitable for resuming" + " training using `--resume_from_checkpoint`." + ), + ) + parser.add_argument( + "--checkpoints_total_limit", + type=int, + default=None, + help=("Max number of checkpoints to store."), + ) + parser.add_argument( + "--resume_from_checkpoint", + type=str, + default=None, + help=( + "Whether training should be resumed from a previous checkpoint. Use a path saved by" + ' `--checkpointing_steps`, or `"latest"` to automatically select the last available checkpoint.' + ), + ) + parser.add_argument( + "--validation_epochs", + type=int, + default=5, + help="Run validation every X epochs.", + ) + parser.add_argument( + "--tracker_project_name", + type=str, + default="text2image-fine-tune", + help=( + "The `project_name` argument passed to Accelerator.init_trackers for" + " more information see https://huggingface.co/docs/accelerate/v0.17.0/en/package_reference/accelerator#accelerate.Accelerator" + ), + ) + + args = parser.parse_args() + env_local_rank = int(os.environ.get("LOCAL_RANK", -1)) + if env_local_rank != -1 and env_local_rank != args.local_rank: + args.local_rank = env_local_rank + + # Sanity checks + if args.dataset_name is None and args.train_data_dir is None: + raise ValueError("Need either a dataset name or a training folder.") + + return args + + +def main(): + args = parse_args() + if args.report_to == "wandb" and args.hub_token is not None: + raise ValueError( + "You cannot use both --report_to=wandb and --hub_token due to a security risk of exposing your token." + " Please use `huggingface-cli login` to authenticate with the Hub." + ) + + logging_dir = os.path.join(args.output_dir, args.logging_dir) + accelerator_project_config = ProjectConfiguration( + total_limit=args.checkpoints_total_limit, project_dir=args.output_dir, logging_dir=logging_dir + ) + accelerator = Accelerator( + gradient_accumulation_steps=args.gradient_accumulation_steps, + mixed_precision=args.mixed_precision, + log_with=args.report_to, + project_config=accelerator_project_config, + ) + + # Make one log on every process with the configuration for debugging. + logging.basicConfig( + format="%(asctime)s - %(levelname)s - %(name)s - %(message)s", + datefmt="%m/%d/%Y %H:%M:%S", + level=logging.INFO, + ) + logger.info(accelerator.state, main_process_only=False) + if accelerator.is_local_main_process: + datasets.utils.logging.set_verbosity_warning() + transformers.utils.logging.set_verbosity_warning() + set_verbosity_info() + else: + datasets.utils.logging.set_verbosity_error() + transformers.utils.logging.set_verbosity_error() + set_verbosity_error() + + # If passed along, set the training seed now. + if args.seed is not None: + set_seed(args.seed) + + # Handle the repository creation + if accelerator.is_main_process: + if args.output_dir is not None: + os.makedirs(args.output_dir, exist_ok=True) + + if args.push_to_hub: + repo_id = create_repo( + repo_id=args.hub_model_id or Path(args.output_dir).name, exist_ok=True, token=args.hub_token + ).repo_id + + # Load scheduler, effnet, tokenizer, clip_model + noise_scheduler = DDPMWuerstchenScheduler() + tokenizer = PreTrainedTokenizerFast.from_pretrained( + args.pretrained_prior_model_name_or_path, subfolder="tokenizer" + ) + + def deepspeed_zero_init_disabled_context_manager(): + """ + returns either a context list that includes one that will disable zero.Init or an empty context list + """ + deepspeed_plugin = AcceleratorState().deepspeed_plugin if is_initialized() else None + if deepspeed_plugin is None: + return [] + + return [deepspeed_plugin.zero3_init_context_manager(enable=False)] + + weight_dtype = torch.float32 + if accelerator.mixed_precision == "fp16": + weight_dtype = torch.float16 + elif accelerator.mixed_precision == "bf16": + weight_dtype = torch.bfloat16 + with ContextManagers(deepspeed_zero_init_disabled_context_manager()): + pretrained_checkpoint_file = hf_hub_download("dome272/wuerstchen", filename="model_v2_stage_b.pt") + state_dict = torch.load(pretrained_checkpoint_file, map_location="cpu") + image_encoder = EfficientNetEncoder() + image_encoder.load_state_dict(state_dict["effnet_state_dict"]) + image_encoder.eval() + + text_encoder = CLIPTextModel.from_pretrained( + args.pretrained_prior_model_name_or_path, subfolder="text_encoder", torch_dtype=weight_dtype + ).eval() + + # Freeze text_encoder and image_encoder + text_encoder.requires_grad_(False) + image_encoder.requires_grad_(False) + + # load prior model + prior = WuerstchenPrior.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder="prior") + + # Create EMA for the prior + if args.use_ema: + ema_prior = WuerstchenPrior.from_pretrained(args.pretrained_prior_model_name_or_path, subfolder="prior") + ema_prior = EMAModel(ema_prior.parameters(), model_cls=WuerstchenPrior, model_config=ema_prior.config) + ema_prior.to(accelerator.device) + + # `accelerate` 0.16.0 will have better support for customized saving + if version.parse(accelerate.__version__) >= version.parse("0.16.0"): + # create custom saving & loading hooks so that `accelerator.save_state(...)` serializes in a nice format + def save_model_hook(models, weights, output_dir): + if args.use_ema: + ema_prior.save_pretrained(os.path.join(output_dir, "prior_ema")) + + for i, model in enumerate(models): + model.save_pretrained(os.path.join(output_dir, "prior")) + + # make sure to pop weight so that corresponding model is not saved again + weights.pop() + + def load_model_hook(models, input_dir): + if args.use_ema: + load_model = EMAModel.from_pretrained(os.path.join(input_dir, "prior_ema"), WuerstchenPrior) + ema_prior.load_state_dict(load_model.state_dict()) + ema_prior.to(accelerator.device) + del load_model + + for i in range(len(models)): + # pop models so that they are not loaded again + model = models.pop() + + # load diffusers style into model + load_model = WuerstchenPrior.from_pretrained(input_dir, subfolder="prior") + model.register_to_config(**load_model.config) + + model.load_state_dict(load_model.state_dict()) + del load_model + + accelerator.register_save_state_pre_hook(save_model_hook) + accelerator.register_load_state_pre_hook(load_model_hook) + + if args.gradient_checkpointing: + prior.enable_gradient_checkpointing() + + if args.allow_tf32: + torch.backends.cuda.matmul.allow_tf32 = True + + if args.use_8bit_adam: + try: + import bitsandbytes as bnb + except ImportError: + raise ImportError( + "Please install bitsandbytes to use 8-bit Adam. You can do so by running `pip install bitsandbytes`" + ) + + optimizer_cls = bnb.optim.AdamW8bit + else: + optimizer_cls = torch.optim.AdamW + optimizer = optimizer_cls( + prior.parameters(), + lr=args.learning_rate, + betas=(args.adam_beta1, args.adam_beta2), + weight_decay=args.adam_weight_decay, + eps=args.adam_epsilon, + ) + + # Get the datasets: you can either provide your own training and evaluation files (see below) + # or specify a Dataset from the hub (the dataset will be downloaded automatically from the datasets Hub). + + # In distributed training, the load_dataset function guarantees that only one local process can concurrently + # download the dataset. + if args.dataset_name is not None: + # Downloading and loading a dataset from the hub. + dataset = load_dataset( + args.dataset_name, + args.dataset_config_name, + cache_dir=args.cache_dir, + ) + else: + data_files = {} + if args.train_data_dir is not None: + data_files["train"] = os.path.join(args.train_data_dir, "**") + dataset = load_dataset( + "imagefolder", + data_files=data_files, + cache_dir=args.cache_dir, + ) + # See more about loading custom images at + # https://huggingface.co/docs/datasets/v2.4.0/en/image_load#imagefolder + + # Preprocessing the datasets. + # We need to tokenize inputs and targets. + column_names = dataset["train"].column_names + + # Get the column names for input/target. + dataset_columns = DATASET_NAME_MAPPING.get(args.dataset_name, None) + if args.image_column is None: + image_column = dataset_columns[0] if dataset_columns is not None else column_names[0] + else: + image_column = args.image_column + if image_column not in column_names: + raise ValueError( + f"--image_column' value '{args.image_column}' needs to be one of: {', '.join(column_names)}" + ) + if args.caption_column is None: + caption_column = dataset_columns[1] if dataset_columns is not None else column_names[1] + else: + caption_column = args.caption_column + if caption_column not in column_names: + raise ValueError( + f"--caption_column' value '{args.caption_column}' needs to be one of: {', '.join(column_names)}" + ) + + # Preprocessing the datasets. + # We need to tokenize input captions and transform the images + def tokenize_captions(examples, is_train=True): + captions = [] + for caption in examples[caption_column]: + if isinstance(caption, str): + captions.append(caption) + elif isinstance(caption, (list, np.ndarray)): + # take a random caption if there are multiple + captions.append(random.choice(caption) if is_train else caption[0]) + else: + raise ValueError( + f"Caption column `{caption_column}` should contain either strings or lists of strings." + ) + inputs = tokenizer( + captions, max_length=tokenizer.model_max_length, padding="max_length", truncation=True, return_tensors="pt" + ) + text_input_ids = inputs.input_ids + text_mask = inputs.attention_mask.bool() + return text_input_ids, text_mask + + effnet_transforms = transforms.Compose( + [ + transforms.Resize(args.resolution, interpolation=transforms.InterpolationMode.BILINEAR, antialias=True), + transforms.CenterCrop(args.resolution), + transforms.ToTensor(), + transforms.Normalize(mean=(0.485, 0.456, 0.406), std=(0.229, 0.224, 0.225)), + ] + ) + + def preprocess_train(examples): + images = [image.convert("RGB") for image in examples[image_column]] + examples["effnet_pixel_values"] = [effnet_transforms(image) for image in images] + examples["text_input_ids"], examples["text_mask"] = tokenize_captions(examples) + return examples + + with accelerator.main_process_first(): + if args.max_train_samples is not None: + dataset["train"] = dataset["train"].shuffle(seed=args.seed).select(range(args.max_train_samples)) + # Set the training transforms + train_dataset = dataset["train"].with_transform(preprocess_train) + + def collate_fn(examples): + effnet_pixel_values = torch.stack([example["effnet_pixel_values"] for example in examples]) + effnet_pixel_values = effnet_pixel_values.to(memory_format=torch.contiguous_format).float() + text_input_ids = torch.stack([example["text_input_ids"] for example in examples]) + text_mask = torch.stack([example["text_mask"] for example in examples]) + return {"effnet_pixel_values": effnet_pixel_values, "text_input_ids": text_input_ids, "text_mask": text_mask} + + # DataLoaders creation: + train_dataloader = torch.utils.data.DataLoader( + train_dataset, + shuffle=True, + collate_fn=collate_fn, + batch_size=args.train_batch_size, + num_workers=args.dataloader_num_workers, + ) + + # Scheduler and math around the number of training steps. + overrode_max_train_steps = False + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if args.max_train_steps is None: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + overrode_max_train_steps = True + + lr_scheduler = get_scheduler( + args.lr_scheduler, + optimizer=optimizer, + num_warmup_steps=args.lr_warmup_steps * args.gradient_accumulation_steps, + num_training_steps=args.max_train_steps * args.gradient_accumulation_steps, + ) + + prior, optimizer, train_dataloader, lr_scheduler = accelerator.prepare( + prior, optimizer, train_dataloader, lr_scheduler + ) + image_encoder.to(accelerator.device, dtype=weight_dtype) + text_encoder.to(accelerator.device, dtype=weight_dtype) + + # We need to recalculate our total training steps as the size of the training dataloader may have changed. + num_update_steps_per_epoch = math.ceil(len(train_dataloader) / args.gradient_accumulation_steps) + if overrode_max_train_steps: + args.max_train_steps = args.num_train_epochs * num_update_steps_per_epoch + # Afterwards we recalculate our number of training epochs + args.num_train_epochs = math.ceil(args.max_train_steps / num_update_steps_per_epoch) + + # We need to initialize the trackers we use, and also store our configuration. + # The trackers initializes automatically on the main process. + if accelerator.is_main_process: + tracker_config = dict(vars(args)) + tracker_config.pop("validation_prompts") + accelerator.init_trackers(args.tracker_project_name, tracker_config) + + # Train! + total_batch_size = args.train_batch_size * accelerator.num_processes * args.gradient_accumulation_steps + + logger.info("***** Running training *****") + logger.info(f" Num examples = {len(train_dataset)}") + logger.info(f" Num Epochs = {args.num_train_epochs}") + logger.info(f" Instantaneous batch size per device = {args.train_batch_size}") + logger.info(f" Total train batch size (w. parallel, distributed & accumulation) = {total_batch_size}") + logger.info(f" Gradient Accumulation steps = {args.gradient_accumulation_steps}") + logger.info(f" Total optimization steps = {args.max_train_steps}") + global_step = 0 + first_epoch = 0 + + # Potentially load in the weights and states from a previous save + if args.resume_from_checkpoint: + if args.resume_from_checkpoint != "latest": + path = os.path.basename(args.resume_from_checkpoint) + else: + # Get the most recent checkpoint + dirs = os.listdir(args.output_dir) + dirs = [d for d in dirs if d.startswith("checkpoint")] + dirs = sorted(dirs, key=lambda x: int(x.split("-")[1])) + path = dirs[-1] if len(dirs) > 0 else None + + if path is None: + accelerator.print( + f"Checkpoint '{args.resume_from_checkpoint}' does not exist. Starting a new training run." + ) + args.resume_from_checkpoint = None + else: + accelerator.print(f"Resuming from checkpoint {path}") + accelerator.load_state(os.path.join(args.output_dir, path)) + global_step = int(path.split("-")[1]) + + resume_global_step = global_step * args.gradient_accumulation_steps + first_epoch = global_step // num_update_steps_per_epoch + resume_step = resume_global_step % (num_update_steps_per_epoch * args.gradient_accumulation_steps) + + # Only show the progress bar once on each machine. + progress_bar = tqdm(range(global_step, args.max_train_steps), disable=not accelerator.is_local_main_process) + progress_bar.set_description("Steps") + + for epoch in range(first_epoch, args.num_train_epochs): + prior.train() + train_loss = 0.0 + for step, batch in enumerate(train_dataloader): + # Skip steps until we reach the resumed step + if args.resume_from_checkpoint and epoch == first_epoch and step < resume_step: + if step % args.gradient_accumulation_steps == 0: + progress_bar.update(1) + continue + + with accelerator.accumulate(prior): + # Convert images to latent space + text_input_ids, text_mask, effnet_images = ( + batch["text_input_ids"], + batch["text_mask"], + batch["effnet_pixel_values"].to(weight_dtype), + ) + + with torch.no_grad(): + text_encoder_output = text_encoder(text_input_ids, attention_mask=text_mask) + prompt_embeds = text_encoder_output.last_hidden_state + image_embeds = image_encoder(effnet_images) + # scale + image_embeds = image_embeds.add(1.0).div(42.0) + + # Sample noise that we'll add to the image_embeds + noise = torch.randn_like(image_embeds) + bsz = image_embeds.shape[0] + + # Sample a random timestep for each image + timesteps = torch.rand((bsz,), device=image_embeds.device, dtype=weight_dtype) + + # add noise to latent + noisy_latents = noise_scheduler.add_noise(image_embeds, noise, timesteps) + + # Predict the noise residual and compute losscd + pred_noise = prior(noisy_latents, timesteps, prompt_embeds) + + # vanilla loss + loss = F.mse_loss(pred_noise.float(), noise.float(), reduction="mean") + + # Gather the losses across all processes for logging (if we use distributed training). + avg_loss = accelerator.gather(loss.repeat(args.train_batch_size)).mean() + train_loss += avg_loss.item() / args.gradient_accumulation_steps + + # Backpropagate + accelerator.backward(loss) + if accelerator.sync_gradients: + accelerator.clip_grad_norm_(prior.parameters(), args.max_grad_norm) + optimizer.step() + lr_scheduler.step() + optimizer.zero_grad() + + # Checks if the accelerator has performed an optimization step behind the scenes + if accelerator.sync_gradients: + if args.use_ema: + ema_prior.step(prior.parameters()) + progress_bar.update(1) + global_step += 1 + accelerator.log({"train_loss": train_loss}, step=global_step) + train_loss = 0.0 + + if global_step % args.checkpointing_steps == 0: + if accelerator.is_main_process: + # _before_ saving state, check if this save would set us over the `checkpoints_total_limit` + if args.checkpoints_total_limit is not None: + checkpoints = os.listdir(args.output_dir) + checkpoints = [d for d in checkpoints if d.startswith("checkpoint")] + checkpoints = sorted(checkpoints, key=lambda x: int(x.split("-")[1])) + + # before we save the new checkpoint, we need to have at _most_ `checkpoints_total_limit - 1` checkpoints + if len(checkpoints) >= args.checkpoints_total_limit: + num_to_remove = len(checkpoints) - args.checkpoints_total_limit + 1 + removing_checkpoints = checkpoints[0:num_to_remove] + + logger.info( + f"{len(checkpoints)} checkpoints already exist, removing {len(removing_checkpoints)} checkpoints" + ) + logger.info(f"removing checkpoints: {', '.join(removing_checkpoints)}") + + for removing_checkpoint in removing_checkpoints: + removing_checkpoint = os.path.join(args.output_dir, removing_checkpoint) + shutil.rmtree(removing_checkpoint) + + save_path = os.path.join(args.output_dir, f"checkpoint-{global_step}") + accelerator.save_state(save_path) + logger.info(f"Saved state to {save_path}") + + logs = {"step_loss": loss.detach().item(), "lr": lr_scheduler.get_last_lr()[0]} + progress_bar.set_postfix(**logs) + + if global_step >= args.max_train_steps: + break + + if accelerator.is_main_process: + if args.validation_prompts is not None and epoch % args.validation_epochs == 0: + if args.use_ema: + # Store the UNet parameters temporarily and load the EMA parameters to perform inference. + ema_prior.store(prior.parameters()) + ema_prior.copy_to(prior.parameters()) + log_validation(text_encoder, tokenizer, prior, args, accelerator, weight_dtype, global_step) + if args.use_ema: + # Switch back to the original UNet parameters. + ema_prior.restore(prior.parameters()) + + # Create the pipeline using the trained modules and save it. + accelerator.wait_for_everyone() + if accelerator.is_main_process: + prior = accelerator.unwrap_model(prior) + if args.use_ema: + ema_prior.copy_to(prior.parameters()) + + pipeline = AutoPipelineForText2Image.from_pretrained( + args.pretrained_decoder_model_name_or_path, + prior_prior=prior, + prior_text_encoder=accelerator.unwrap_model(text_encoder), + prior_tokenizer=tokenizer, + ) + pipeline.prior_pipe.save_pretrained(os.path.join(args.output_dir, "prior_pipeline")) + + # Run a final round of inference. + images = [] + if args.validation_prompts is not None: + logger.info("Running inference for collecting generated images...") + pipeline = pipeline.to(accelerator.device, torch_dtype=weight_dtype) + pipeline.set_progress_bar_config(disable=True) + + if args.seed is None: + generator = None + else: + generator = torch.Generator(device=accelerator.device).manual_seed(args.seed) + + for i in range(len(args.validation_prompts)): + with torch.autocast("cuda"): + image = pipeline( + args.validation_prompts[i], + prior_timesteps=DEFAULT_STAGE_C_TIMESTEPS, + generator=generator, + width=args.resolution, + height=args.resolution, + ).images[0] + images.append(image) + + if args.push_to_hub: + save_model_card(args, repo_id, images, repo_folder=args.output_dir) + upload_folder( + repo_id=repo_id, + folder_path=args.output_dir, + commit_message="End of training", + ignore_patterns=["step_*", "epoch_*"], + ) + + accelerator.end_training() + + +if __name__ == "__main__": + main() diff --git a/diffusers-0.27.0/pyproject.toml b/diffusers-0.27.0/pyproject.toml new file mode 100755 index 0000000..0612f2f --- /dev/null +++ b/diffusers-0.27.0/pyproject.toml @@ -0,0 +1,27 @@ +[tool.ruff] +# Never enforce `E501` (line length violations). +ignore = ["C901", "E501", "E741", "F402", "F823"] +select = ["C", "E", "F", "I", "W"] +line-length = 119 + +# Ignore import violations in all `__init__.py` files. +[tool.ruff.per-file-ignores] +"__init__.py" = ["E402", "F401", "F403", "F811"] +"src/diffusers/utils/dummy_*.py" = ["F401"] + +[tool.ruff.isort] +lines-after-imports = 2 +known-first-party = ["diffusers"] + +[tool.ruff.format] +# Like Black, use double quotes for strings. +quote-style = "double" + +# Like Black, indent with spaces, rather than tabs. +indent-style = "space" + +# Like Black, respect magic trailing commas. +skip-magic-trailing-comma = false + +# Like Black, automatically detect the appropriate line ending. +line-ending = "auto" diff --git a/diffusers-0.27.0/scripts/__init__.py b/diffusers-0.27.0/scripts/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/scripts/change_naming_configs_and_checkpoints.py b/diffusers-0.27.0/scripts/change_naming_configs_and_checkpoints.py new file mode 100755 index 0000000..6dc8098 --- /dev/null +++ b/diffusers-0.27.0/scripts/change_naming_configs_and_checkpoints.py @@ -0,0 +1,113 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the LDM checkpoints. """ + +import argparse +import json +import os + +import torch +from transformers.file_utils import has_file + +from diffusers import UNet2DConditionModel, UNet2DModel + + +do_only_config = False +do_only_weights = True +do_only_renaming = False + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--repo_path", + default=None, + type=str, + required=True, + help="The config json file corresponding to the architecture.", + ) + + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + + args = parser.parse_args() + + config_parameters_to_change = { + "image_size": "sample_size", + "num_res_blocks": "layers_per_block", + "block_channels": "block_out_channels", + "down_blocks": "down_block_types", + "up_blocks": "up_block_types", + "downscale_freq_shift": "freq_shift", + "resnet_num_groups": "norm_num_groups", + "resnet_act_fn": "act_fn", + "resnet_eps": "norm_eps", + "num_head_channels": "attention_head_dim", + } + + key_parameters_to_change = { + "time_steps": "time_proj", + "mid": "mid_block", + "downsample_blocks": "down_blocks", + "upsample_blocks": "up_blocks", + } + + subfolder = "" if has_file(args.repo_path, "config.json") else "unet" + + with open(os.path.join(args.repo_path, subfolder, "config.json"), "r", encoding="utf-8") as reader: + text = reader.read() + config = json.loads(text) + + if do_only_config: + for key in config_parameters_to_change.keys(): + config.pop(key, None) + + if has_file(args.repo_path, "config.json"): + model = UNet2DModel(**config) + else: + class_name = UNet2DConditionModel if "ldm-text2im-large-256" in args.repo_path else UNet2DModel + model = class_name(**config) + + if do_only_config: + model.save_config(os.path.join(args.repo_path, subfolder)) + + config = dict(model.config) + + if do_only_renaming: + for key, value in config_parameters_to_change.items(): + if key in config: + config[value] = config[key] + del config[key] + + config["down_block_types"] = [k.replace("UNetRes", "") for k in config["down_block_types"]] + config["up_block_types"] = [k.replace("UNetRes", "") for k in config["up_block_types"]] + + if do_only_weights: + state_dict = torch.load(os.path.join(args.repo_path, subfolder, "diffusion_pytorch_model.bin")) + + new_state_dict = {} + for param_key, param_value in state_dict.items(): + if param_key.endswith(".op.bias") or param_key.endswith(".op.weight"): + continue + has_changed = False + for key, new_key in key_parameters_to_change.items(): + if not has_changed and param_key.split(".")[0] == key: + new_state_dict[".".join([new_key] + param_key.split(".")[1:])] = param_value + has_changed = True + if not has_changed: + new_state_dict[param_key] = param_value + + model.load_state_dict(new_state_dict) + model.save_pretrained(os.path.join(args.repo_path, subfolder)) diff --git a/diffusers-0.27.0/scripts/conversion_ldm_uncond.py b/diffusers-0.27.0/scripts/conversion_ldm_uncond.py new file mode 100755 index 0000000..8c22cc1 --- /dev/null +++ b/diffusers-0.27.0/scripts/conversion_ldm_uncond.py @@ -0,0 +1,56 @@ +import argparse + +import torch +import yaml + +from diffusers import DDIMScheduler, LDMPipeline, UNetLDMModel, VQModel + + +def convert_ldm_original(checkpoint_path, config_path, output_path): + config = yaml.safe_load(config_path) + state_dict = torch.load(checkpoint_path, map_location="cpu")["model"] + keys = list(state_dict.keys()) + + # extract state_dict for VQVAE + first_stage_dict = {} + first_stage_key = "first_stage_model." + for key in keys: + if key.startswith(first_stage_key): + first_stage_dict[key.replace(first_stage_key, "")] = state_dict[key] + + # extract state_dict for UNetLDM + unet_state_dict = {} + unet_key = "model.diffusion_model." + for key in keys: + if key.startswith(unet_key): + unet_state_dict[key.replace(unet_key, "")] = state_dict[key] + + vqvae_init_args = config["model"]["params"]["first_stage_config"]["params"] + unet_init_args = config["model"]["params"]["unet_config"]["params"] + + vqvae = VQModel(**vqvae_init_args).eval() + vqvae.load_state_dict(first_stage_dict) + + unet = UNetLDMModel(**unet_init_args).eval() + unet.load_state_dict(unet_state_dict) + + noise_scheduler = DDIMScheduler( + timesteps=config["model"]["params"]["timesteps"], + beta_schedule="scaled_linear", + beta_start=config["model"]["params"]["linear_start"], + beta_end=config["model"]["params"]["linear_end"], + clip_sample=False, + ) + + pipeline = LDMPipeline(vqvae, unet, noise_scheduler) + pipeline.save_pretrained(output_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--checkpoint_path", type=str, required=True) + parser.add_argument("--config_path", type=str, required=True) + parser.add_argument("--output_path", type=str, required=True) + args = parser.parse_args() + + convert_ldm_original(args.checkpoint_path, args.config_path, args.output_path) diff --git a/diffusers-0.27.0/scripts/convert_amused.py b/diffusers-0.27.0/scripts/convert_amused.py new file mode 100755 index 0000000..21be29d --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_amused.py @@ -0,0 +1,523 @@ +import inspect +import os +from argparse import ArgumentParser + +import numpy as np +import torch +from muse import MaskGiTUViT, VQGANModel +from muse import PipelineMuse as OldPipelineMuse +from transformers import CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import VQModel +from diffusers.models.attention_processor import AttnProcessor +from diffusers.models.unets.uvit_2d import UVit2DModel +from diffusers.pipelines.amused.pipeline_amused import AmusedPipeline +from diffusers.schedulers import AmusedScheduler + + +torch.backends.cuda.enable_flash_sdp(False) +torch.backends.cuda.enable_mem_efficient_sdp(False) +torch.backends.cuda.enable_math_sdp(True) + +os.environ["CUDA_LAUNCH_BLOCKING"] = "1" +os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8" +torch.use_deterministic_algorithms(True) + +# Enable CUDNN deterministic mode +torch.backends.cudnn.deterministic = True +torch.backends.cudnn.benchmark = False +torch.backends.cuda.matmul.allow_tf32 = False + +device = "cuda" + + +def main(): + args = ArgumentParser() + args.add_argument("--model_256", action="store_true") + args.add_argument("--write_to", type=str, required=False, default=None) + args.add_argument("--transformer_path", type=str, required=False, default=None) + args = args.parse_args() + + transformer_path = args.transformer_path + subfolder = "transformer" + + if transformer_path is None: + if args.model_256: + transformer_path = "openMUSE/muse-256" + else: + transformer_path = ( + "../research-run-512-checkpoints/research-run-512-with-downsample-checkpoint-554000/unwrapped_model/" + ) + subfolder = None + + old_transformer = MaskGiTUViT.from_pretrained(transformer_path, subfolder=subfolder) + + old_transformer.to(device) + + old_vae = VQGANModel.from_pretrained("openMUSE/muse-512", subfolder="vae") + old_vae.to(device) + + vqvae = make_vqvae(old_vae) + + tokenizer = CLIPTokenizer.from_pretrained("openMUSE/muse-512", subfolder="text_encoder") + + text_encoder = CLIPTextModelWithProjection.from_pretrained("openMUSE/muse-512", subfolder="text_encoder") + text_encoder.to(device) + + transformer = make_transformer(old_transformer, args.model_256) + + scheduler = AmusedScheduler(mask_token_id=old_transformer.config.mask_token_id) + + new_pipe = AmusedPipeline( + vqvae=vqvae, tokenizer=tokenizer, text_encoder=text_encoder, transformer=transformer, scheduler=scheduler + ) + + old_pipe = OldPipelineMuse( + vae=old_vae, transformer=old_transformer, text_encoder=text_encoder, tokenizer=tokenizer + ) + old_pipe.to(device) + + if args.model_256: + transformer_seq_len = 256 + orig_size = (256, 256) + else: + transformer_seq_len = 1024 + orig_size = (512, 512) + + old_out = old_pipe( + "dog", + generator=torch.Generator(device).manual_seed(0), + transformer_seq_len=transformer_seq_len, + orig_size=orig_size, + timesteps=12, + )[0] + + new_out = new_pipe("dog", generator=torch.Generator(device).manual_seed(0)).images[0] + + old_out = np.array(old_out) + new_out = np.array(new_out) + + diff = np.abs(old_out.astype(np.float64) - new_out.astype(np.float64)) + + # assert diff diff.sum() == 0 + print("skipping pipeline full equivalence check") + + print(f"max diff: {diff.max()}, diff.sum() / diff.size {diff.sum() / diff.size}") + + if args.model_256: + assert diff.max() <= 3 + assert diff.sum() / diff.size < 0.7 + else: + assert diff.max() <= 1 + assert diff.sum() / diff.size < 0.4 + + if args.write_to is not None: + new_pipe.save_pretrained(args.write_to) + + +def make_transformer(old_transformer, model_256): + args = dict(old_transformer.config) + force_down_up_sample = args["force_down_up_sample"] + + signature = inspect.signature(UVit2DModel.__init__) + + args_ = { + "downsample": force_down_up_sample, + "upsample": force_down_up_sample, + "block_out_channels": args["block_out_channels"][0], + "sample_size": 16 if model_256 else 32, + } + + for s in list(signature.parameters.keys()): + if s in ["self", "downsample", "upsample", "sample_size", "block_out_channels"]: + continue + + args_[s] = args[s] + + new_transformer = UVit2DModel(**args_) + new_transformer.to(device) + + new_transformer.set_attn_processor(AttnProcessor()) + + state_dict = old_transformer.state_dict() + + state_dict["cond_embed.linear_1.weight"] = state_dict.pop("cond_embed.0.weight") + state_dict["cond_embed.linear_2.weight"] = state_dict.pop("cond_embed.2.weight") + + for i in range(22): + state_dict[f"transformer_layers.{i}.norm1.norm.weight"] = state_dict.pop( + f"transformer_layers.{i}.attn_layer_norm.weight" + ) + state_dict[f"transformer_layers.{i}.norm1.linear.weight"] = state_dict.pop( + f"transformer_layers.{i}.self_attn_adaLN_modulation.mapper.weight" + ) + + state_dict[f"transformer_layers.{i}.attn1.to_q.weight"] = state_dict.pop( + f"transformer_layers.{i}.attention.query.weight" + ) + state_dict[f"transformer_layers.{i}.attn1.to_k.weight"] = state_dict.pop( + f"transformer_layers.{i}.attention.key.weight" + ) + state_dict[f"transformer_layers.{i}.attn1.to_v.weight"] = state_dict.pop( + f"transformer_layers.{i}.attention.value.weight" + ) + state_dict[f"transformer_layers.{i}.attn1.to_out.0.weight"] = state_dict.pop( + f"transformer_layers.{i}.attention.out.weight" + ) + + state_dict[f"transformer_layers.{i}.norm2.norm.weight"] = state_dict.pop( + f"transformer_layers.{i}.crossattn_layer_norm.weight" + ) + state_dict[f"transformer_layers.{i}.norm2.linear.weight"] = state_dict.pop( + f"transformer_layers.{i}.cross_attn_adaLN_modulation.mapper.weight" + ) + + state_dict[f"transformer_layers.{i}.attn2.to_q.weight"] = state_dict.pop( + f"transformer_layers.{i}.crossattention.query.weight" + ) + state_dict[f"transformer_layers.{i}.attn2.to_k.weight"] = state_dict.pop( + f"transformer_layers.{i}.crossattention.key.weight" + ) + state_dict[f"transformer_layers.{i}.attn2.to_v.weight"] = state_dict.pop( + f"transformer_layers.{i}.crossattention.value.weight" + ) + state_dict[f"transformer_layers.{i}.attn2.to_out.0.weight"] = state_dict.pop( + f"transformer_layers.{i}.crossattention.out.weight" + ) + + state_dict[f"transformer_layers.{i}.norm3.norm.weight"] = state_dict.pop( + f"transformer_layers.{i}.ffn.pre_mlp_layer_norm.weight" + ) + state_dict[f"transformer_layers.{i}.norm3.linear.weight"] = state_dict.pop( + f"transformer_layers.{i}.ffn.adaLN_modulation.mapper.weight" + ) + + wi_0_weight = state_dict.pop(f"transformer_layers.{i}.ffn.wi_0.weight") + wi_1_weight = state_dict.pop(f"transformer_layers.{i}.ffn.wi_1.weight") + proj_weight = torch.concat([wi_1_weight, wi_0_weight], dim=0) + state_dict[f"transformer_layers.{i}.ff.net.0.proj.weight"] = proj_weight + + state_dict[f"transformer_layers.{i}.ff.net.2.weight"] = state_dict.pop(f"transformer_layers.{i}.ffn.wo.weight") + + if force_down_up_sample: + state_dict["down_block.downsample.norm.weight"] = state_dict.pop("down_blocks.0.downsample.0.norm.weight") + state_dict["down_block.downsample.conv.weight"] = state_dict.pop("down_blocks.0.downsample.1.weight") + + state_dict["up_block.upsample.norm.weight"] = state_dict.pop("up_blocks.0.upsample.0.norm.weight") + state_dict["up_block.upsample.conv.weight"] = state_dict.pop("up_blocks.0.upsample.1.weight") + + state_dict["mlm_layer.layer_norm.weight"] = state_dict.pop("mlm_layer.layer_norm.norm.weight") + + for i in range(3): + state_dict[f"down_block.res_blocks.{i}.norm.weight"] = state_dict.pop( + f"down_blocks.0.res_blocks.{i}.norm.norm.weight" + ) + state_dict[f"down_block.res_blocks.{i}.channelwise_linear_1.weight"] = state_dict.pop( + f"down_blocks.0.res_blocks.{i}.channelwise.0.weight" + ) + state_dict[f"down_block.res_blocks.{i}.channelwise_norm.gamma"] = state_dict.pop( + f"down_blocks.0.res_blocks.{i}.channelwise.2.gamma" + ) + state_dict[f"down_block.res_blocks.{i}.channelwise_norm.beta"] = state_dict.pop( + f"down_blocks.0.res_blocks.{i}.channelwise.2.beta" + ) + state_dict[f"down_block.res_blocks.{i}.channelwise_linear_2.weight"] = state_dict.pop( + f"down_blocks.0.res_blocks.{i}.channelwise.4.weight" + ) + state_dict[f"down_block.res_blocks.{i}.cond_embeds_mapper.weight"] = state_dict.pop( + f"down_blocks.0.res_blocks.{i}.adaLN_modulation.mapper.weight" + ) + + state_dict[f"down_block.attention_blocks.{i}.norm1.weight"] = state_dict.pop( + f"down_blocks.0.attention_blocks.{i}.attn_layer_norm.weight" + ) + state_dict[f"down_block.attention_blocks.{i}.attn1.to_q.weight"] = state_dict.pop( + f"down_blocks.0.attention_blocks.{i}.attention.query.weight" + ) + state_dict[f"down_block.attention_blocks.{i}.attn1.to_k.weight"] = state_dict.pop( + f"down_blocks.0.attention_blocks.{i}.attention.key.weight" + ) + state_dict[f"down_block.attention_blocks.{i}.attn1.to_v.weight"] = state_dict.pop( + f"down_blocks.0.attention_blocks.{i}.attention.value.weight" + ) + state_dict[f"down_block.attention_blocks.{i}.attn1.to_out.0.weight"] = state_dict.pop( + f"down_blocks.0.attention_blocks.{i}.attention.out.weight" + ) + + state_dict[f"down_block.attention_blocks.{i}.norm2.weight"] = state_dict.pop( + f"down_blocks.0.attention_blocks.{i}.crossattn_layer_norm.weight" + ) + state_dict[f"down_block.attention_blocks.{i}.attn2.to_q.weight"] = state_dict.pop( + f"down_blocks.0.attention_blocks.{i}.crossattention.query.weight" + ) + state_dict[f"down_block.attention_blocks.{i}.attn2.to_k.weight"] = state_dict.pop( + f"down_blocks.0.attention_blocks.{i}.crossattention.key.weight" + ) + state_dict[f"down_block.attention_blocks.{i}.attn2.to_v.weight"] = state_dict.pop( + f"down_blocks.0.attention_blocks.{i}.crossattention.value.weight" + ) + state_dict[f"down_block.attention_blocks.{i}.attn2.to_out.0.weight"] = state_dict.pop( + f"down_blocks.0.attention_blocks.{i}.crossattention.out.weight" + ) + + state_dict[f"up_block.res_blocks.{i}.norm.weight"] = state_dict.pop( + f"up_blocks.0.res_blocks.{i}.norm.norm.weight" + ) + state_dict[f"up_block.res_blocks.{i}.channelwise_linear_1.weight"] = state_dict.pop( + f"up_blocks.0.res_blocks.{i}.channelwise.0.weight" + ) + state_dict[f"up_block.res_blocks.{i}.channelwise_norm.gamma"] = state_dict.pop( + f"up_blocks.0.res_blocks.{i}.channelwise.2.gamma" + ) + state_dict[f"up_block.res_blocks.{i}.channelwise_norm.beta"] = state_dict.pop( + f"up_blocks.0.res_blocks.{i}.channelwise.2.beta" + ) + state_dict[f"up_block.res_blocks.{i}.channelwise_linear_2.weight"] = state_dict.pop( + f"up_blocks.0.res_blocks.{i}.channelwise.4.weight" + ) + state_dict[f"up_block.res_blocks.{i}.cond_embeds_mapper.weight"] = state_dict.pop( + f"up_blocks.0.res_blocks.{i}.adaLN_modulation.mapper.weight" + ) + + state_dict[f"up_block.attention_blocks.{i}.norm1.weight"] = state_dict.pop( + f"up_blocks.0.attention_blocks.{i}.attn_layer_norm.weight" + ) + state_dict[f"up_block.attention_blocks.{i}.attn1.to_q.weight"] = state_dict.pop( + f"up_blocks.0.attention_blocks.{i}.attention.query.weight" + ) + state_dict[f"up_block.attention_blocks.{i}.attn1.to_k.weight"] = state_dict.pop( + f"up_blocks.0.attention_blocks.{i}.attention.key.weight" + ) + state_dict[f"up_block.attention_blocks.{i}.attn1.to_v.weight"] = state_dict.pop( + f"up_blocks.0.attention_blocks.{i}.attention.value.weight" + ) + state_dict[f"up_block.attention_blocks.{i}.attn1.to_out.0.weight"] = state_dict.pop( + f"up_blocks.0.attention_blocks.{i}.attention.out.weight" + ) + + state_dict[f"up_block.attention_blocks.{i}.norm2.weight"] = state_dict.pop( + f"up_blocks.0.attention_blocks.{i}.crossattn_layer_norm.weight" + ) + state_dict[f"up_block.attention_blocks.{i}.attn2.to_q.weight"] = state_dict.pop( + f"up_blocks.0.attention_blocks.{i}.crossattention.query.weight" + ) + state_dict[f"up_block.attention_blocks.{i}.attn2.to_k.weight"] = state_dict.pop( + f"up_blocks.0.attention_blocks.{i}.crossattention.key.weight" + ) + state_dict[f"up_block.attention_blocks.{i}.attn2.to_v.weight"] = state_dict.pop( + f"up_blocks.0.attention_blocks.{i}.crossattention.value.weight" + ) + state_dict[f"up_block.attention_blocks.{i}.attn2.to_out.0.weight"] = state_dict.pop( + f"up_blocks.0.attention_blocks.{i}.crossattention.out.weight" + ) + + for key in list(state_dict.keys()): + if key.startswith("up_blocks.0"): + key_ = "up_block." + ".".join(key.split(".")[2:]) + state_dict[key_] = state_dict.pop(key) + + if key.startswith("down_blocks.0"): + key_ = "down_block." + ".".join(key.split(".")[2:]) + state_dict[key_] = state_dict.pop(key) + + new_transformer.load_state_dict(state_dict) + + input_ids = torch.randint(0, 10, (1, 32, 32), device=old_transformer.device) + encoder_hidden_states = torch.randn((1, 77, 768), device=old_transformer.device) + cond_embeds = torch.randn((1, 768), device=old_transformer.device) + micro_conds = torch.tensor([[512, 512, 0, 0, 6]], dtype=torch.float32, device=old_transformer.device) + + old_out = old_transformer(input_ids.reshape(1, -1), encoder_hidden_states, cond_embeds, micro_conds) + old_out = old_out.reshape(1, 32, 32, 8192).permute(0, 3, 1, 2) + + new_out = new_transformer(input_ids, encoder_hidden_states, cond_embeds, micro_conds) + + # NOTE: these differences are solely due to using the geglu block that has a single linear layer of + # double output dimension instead of two different linear layers + max_diff = (old_out - new_out).abs().max() + total_diff = (old_out - new_out).abs().sum() + print(f"Transformer max_diff: {max_diff} total_diff: {total_diff}") + assert max_diff < 0.01 + assert total_diff < 1500 + + return new_transformer + + +def make_vqvae(old_vae): + new_vae = VQModel( + act_fn="silu", + block_out_channels=[128, 256, 256, 512, 768], + down_block_types=[ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + ], + in_channels=3, + latent_channels=64, + layers_per_block=2, + norm_num_groups=32, + num_vq_embeddings=8192, + out_channels=3, + sample_size=32, + up_block_types=[ + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + ], + mid_block_add_attention=False, + lookup_from_codebook=True, + ) + new_vae.to(device) + + # fmt: off + + new_state_dict = {} + + old_state_dict = old_vae.state_dict() + + new_state_dict["encoder.conv_in.weight"] = old_state_dict.pop("encoder.conv_in.weight") + new_state_dict["encoder.conv_in.bias"] = old_state_dict.pop("encoder.conv_in.bias") + + convert_vae_block_state_dict(old_state_dict, "encoder.down.0", new_state_dict, "encoder.down_blocks.0") + convert_vae_block_state_dict(old_state_dict, "encoder.down.1", new_state_dict, "encoder.down_blocks.1") + convert_vae_block_state_dict(old_state_dict, "encoder.down.2", new_state_dict, "encoder.down_blocks.2") + convert_vae_block_state_dict(old_state_dict, "encoder.down.3", new_state_dict, "encoder.down_blocks.3") + convert_vae_block_state_dict(old_state_dict, "encoder.down.4", new_state_dict, "encoder.down_blocks.4") + + new_state_dict["encoder.mid_block.resnets.0.norm1.weight"] = old_state_dict.pop("encoder.mid.block_1.norm1.weight") + new_state_dict["encoder.mid_block.resnets.0.norm1.bias"] = old_state_dict.pop("encoder.mid.block_1.norm1.bias") + new_state_dict["encoder.mid_block.resnets.0.conv1.weight"] = old_state_dict.pop("encoder.mid.block_1.conv1.weight") + new_state_dict["encoder.mid_block.resnets.0.conv1.bias"] = old_state_dict.pop("encoder.mid.block_1.conv1.bias") + new_state_dict["encoder.mid_block.resnets.0.norm2.weight"] = old_state_dict.pop("encoder.mid.block_1.norm2.weight") + new_state_dict["encoder.mid_block.resnets.0.norm2.bias"] = old_state_dict.pop("encoder.mid.block_1.norm2.bias") + new_state_dict["encoder.mid_block.resnets.0.conv2.weight"] = old_state_dict.pop("encoder.mid.block_1.conv2.weight") + new_state_dict["encoder.mid_block.resnets.0.conv2.bias"] = old_state_dict.pop("encoder.mid.block_1.conv2.bias") + new_state_dict["encoder.mid_block.resnets.1.norm1.weight"] = old_state_dict.pop("encoder.mid.block_2.norm1.weight") + new_state_dict["encoder.mid_block.resnets.1.norm1.bias"] = old_state_dict.pop("encoder.mid.block_2.norm1.bias") + new_state_dict["encoder.mid_block.resnets.1.conv1.weight"] = old_state_dict.pop("encoder.mid.block_2.conv1.weight") + new_state_dict["encoder.mid_block.resnets.1.conv1.bias"] = old_state_dict.pop("encoder.mid.block_2.conv1.bias") + new_state_dict["encoder.mid_block.resnets.1.norm2.weight"] = old_state_dict.pop("encoder.mid.block_2.norm2.weight") + new_state_dict["encoder.mid_block.resnets.1.norm2.bias"] = old_state_dict.pop("encoder.mid.block_2.norm2.bias") + new_state_dict["encoder.mid_block.resnets.1.conv2.weight"] = old_state_dict.pop("encoder.mid.block_2.conv2.weight") + new_state_dict["encoder.mid_block.resnets.1.conv2.bias"] = old_state_dict.pop("encoder.mid.block_2.conv2.bias") + new_state_dict["encoder.conv_norm_out.weight"] = old_state_dict.pop("encoder.norm_out.weight") + new_state_dict["encoder.conv_norm_out.bias"] = old_state_dict.pop("encoder.norm_out.bias") + new_state_dict["encoder.conv_out.weight"] = old_state_dict.pop("encoder.conv_out.weight") + new_state_dict["encoder.conv_out.bias"] = old_state_dict.pop("encoder.conv_out.bias") + new_state_dict["quant_conv.weight"] = old_state_dict.pop("quant_conv.weight") + new_state_dict["quant_conv.bias"] = old_state_dict.pop("quant_conv.bias") + new_state_dict["quantize.embedding.weight"] = old_state_dict.pop("quantize.embedding.weight") + new_state_dict["post_quant_conv.weight"] = old_state_dict.pop("post_quant_conv.weight") + new_state_dict["post_quant_conv.bias"] = old_state_dict.pop("post_quant_conv.bias") + new_state_dict["decoder.conv_in.weight"] = old_state_dict.pop("decoder.conv_in.weight") + new_state_dict["decoder.conv_in.bias"] = old_state_dict.pop("decoder.conv_in.bias") + new_state_dict["decoder.mid_block.resnets.0.norm1.weight"] = old_state_dict.pop("decoder.mid.block_1.norm1.weight") + new_state_dict["decoder.mid_block.resnets.0.norm1.bias"] = old_state_dict.pop("decoder.mid.block_1.norm1.bias") + new_state_dict["decoder.mid_block.resnets.0.conv1.weight"] = old_state_dict.pop("decoder.mid.block_1.conv1.weight") + new_state_dict["decoder.mid_block.resnets.0.conv1.bias"] = old_state_dict.pop("decoder.mid.block_1.conv1.bias") + new_state_dict["decoder.mid_block.resnets.0.norm2.weight"] = old_state_dict.pop("decoder.mid.block_1.norm2.weight") + new_state_dict["decoder.mid_block.resnets.0.norm2.bias"] = old_state_dict.pop("decoder.mid.block_1.norm2.bias") + new_state_dict["decoder.mid_block.resnets.0.conv2.weight"] = old_state_dict.pop("decoder.mid.block_1.conv2.weight") + new_state_dict["decoder.mid_block.resnets.0.conv2.bias"] = old_state_dict.pop("decoder.mid.block_1.conv2.bias") + new_state_dict["decoder.mid_block.resnets.1.norm1.weight"] = old_state_dict.pop("decoder.mid.block_2.norm1.weight") + new_state_dict["decoder.mid_block.resnets.1.norm1.bias"] = old_state_dict.pop("decoder.mid.block_2.norm1.bias") + new_state_dict["decoder.mid_block.resnets.1.conv1.weight"] = old_state_dict.pop("decoder.mid.block_2.conv1.weight") + new_state_dict["decoder.mid_block.resnets.1.conv1.bias"] = old_state_dict.pop("decoder.mid.block_2.conv1.bias") + new_state_dict["decoder.mid_block.resnets.1.norm2.weight"] = old_state_dict.pop("decoder.mid.block_2.norm2.weight") + new_state_dict["decoder.mid_block.resnets.1.norm2.bias"] = old_state_dict.pop("decoder.mid.block_2.norm2.bias") + new_state_dict["decoder.mid_block.resnets.1.conv2.weight"] = old_state_dict.pop("decoder.mid.block_2.conv2.weight") + new_state_dict["decoder.mid_block.resnets.1.conv2.bias"] = old_state_dict.pop("decoder.mid.block_2.conv2.bias") + + convert_vae_block_state_dict(old_state_dict, "decoder.up.0", new_state_dict, "decoder.up_blocks.4") + convert_vae_block_state_dict(old_state_dict, "decoder.up.1", new_state_dict, "decoder.up_blocks.3") + convert_vae_block_state_dict(old_state_dict, "decoder.up.2", new_state_dict, "decoder.up_blocks.2") + convert_vae_block_state_dict(old_state_dict, "decoder.up.3", new_state_dict, "decoder.up_blocks.1") + convert_vae_block_state_dict(old_state_dict, "decoder.up.4", new_state_dict, "decoder.up_blocks.0") + + new_state_dict["decoder.conv_norm_out.weight"] = old_state_dict.pop("decoder.norm_out.weight") + new_state_dict["decoder.conv_norm_out.bias"] = old_state_dict.pop("decoder.norm_out.bias") + new_state_dict["decoder.conv_out.weight"] = old_state_dict.pop("decoder.conv_out.weight") + new_state_dict["decoder.conv_out.bias"] = old_state_dict.pop("decoder.conv_out.bias") + + # fmt: on + + assert len(old_state_dict.keys()) == 0 + + new_vae.load_state_dict(new_state_dict) + + input = torch.randn((1, 3, 512, 512), device=device) + input = input.clamp(-1, 1) + + old_encoder_output = old_vae.quant_conv(old_vae.encoder(input)) + new_encoder_output = new_vae.quant_conv(new_vae.encoder(input)) + assert (old_encoder_output == new_encoder_output).all() + + old_decoder_output = old_vae.decoder(old_vae.post_quant_conv(old_encoder_output)) + new_decoder_output = new_vae.decoder(new_vae.post_quant_conv(new_encoder_output)) + + # assert (old_decoder_output == new_decoder_output).all() + print("kipping vae decoder equivalence check") + print(f"vae decoder diff {(old_decoder_output - new_decoder_output).float().abs().sum()}") + + old_output = old_vae(input)[0] + new_output = new_vae(input)[0] + + # assert (old_output == new_output).all() + print("skipping full vae equivalence check") + print(f"vae full diff { (old_output - new_output).float().abs().sum()}") + + return new_vae + + +def convert_vae_block_state_dict(old_state_dict, prefix_from, new_state_dict, prefix_to): + # fmt: off + + new_state_dict[f"{prefix_to}.resnets.0.norm1.weight"] = old_state_dict.pop(f"{prefix_from}.block.0.norm1.weight") + new_state_dict[f"{prefix_to}.resnets.0.norm1.bias"] = old_state_dict.pop(f"{prefix_from}.block.0.norm1.bias") + new_state_dict[f"{prefix_to}.resnets.0.conv1.weight"] = old_state_dict.pop(f"{prefix_from}.block.0.conv1.weight") + new_state_dict[f"{prefix_to}.resnets.0.conv1.bias"] = old_state_dict.pop(f"{prefix_from}.block.0.conv1.bias") + new_state_dict[f"{prefix_to}.resnets.0.norm2.weight"] = old_state_dict.pop(f"{prefix_from}.block.0.norm2.weight") + new_state_dict[f"{prefix_to}.resnets.0.norm2.bias"] = old_state_dict.pop(f"{prefix_from}.block.0.norm2.bias") + new_state_dict[f"{prefix_to}.resnets.0.conv2.weight"] = old_state_dict.pop(f"{prefix_from}.block.0.conv2.weight") + new_state_dict[f"{prefix_to}.resnets.0.conv2.bias"] = old_state_dict.pop(f"{prefix_from}.block.0.conv2.bias") + + if f"{prefix_from}.block.0.nin_shortcut.weight" in old_state_dict: + new_state_dict[f"{prefix_to}.resnets.0.conv_shortcut.weight"] = old_state_dict.pop(f"{prefix_from}.block.0.nin_shortcut.weight") + new_state_dict[f"{prefix_to}.resnets.0.conv_shortcut.bias"] = old_state_dict.pop(f"{prefix_from}.block.0.nin_shortcut.bias") + + new_state_dict[f"{prefix_to}.resnets.1.norm1.weight"] = old_state_dict.pop(f"{prefix_from}.block.1.norm1.weight") + new_state_dict[f"{prefix_to}.resnets.1.norm1.bias"] = old_state_dict.pop(f"{prefix_from}.block.1.norm1.bias") + new_state_dict[f"{prefix_to}.resnets.1.conv1.weight"] = old_state_dict.pop(f"{prefix_from}.block.1.conv1.weight") + new_state_dict[f"{prefix_to}.resnets.1.conv1.bias"] = old_state_dict.pop(f"{prefix_from}.block.1.conv1.bias") + new_state_dict[f"{prefix_to}.resnets.1.norm2.weight"] = old_state_dict.pop(f"{prefix_from}.block.1.norm2.weight") + new_state_dict[f"{prefix_to}.resnets.1.norm2.bias"] = old_state_dict.pop(f"{prefix_from}.block.1.norm2.bias") + new_state_dict[f"{prefix_to}.resnets.1.conv2.weight"] = old_state_dict.pop(f"{prefix_from}.block.1.conv2.weight") + new_state_dict[f"{prefix_to}.resnets.1.conv2.bias"] = old_state_dict.pop(f"{prefix_from}.block.1.conv2.bias") + + if f"{prefix_from}.downsample.conv.weight" in old_state_dict: + new_state_dict[f"{prefix_to}.downsamplers.0.conv.weight"] = old_state_dict.pop(f"{prefix_from}.downsample.conv.weight") + new_state_dict[f"{prefix_to}.downsamplers.0.conv.bias"] = old_state_dict.pop(f"{prefix_from}.downsample.conv.bias") + + if f"{prefix_from}.upsample.conv.weight" in old_state_dict: + new_state_dict[f"{prefix_to}.upsamplers.0.conv.weight"] = old_state_dict.pop(f"{prefix_from}.upsample.conv.weight") + new_state_dict[f"{prefix_to}.upsamplers.0.conv.bias"] = old_state_dict.pop(f"{prefix_from}.upsample.conv.bias") + + if f"{prefix_from}.block.2.norm1.weight" in old_state_dict: + new_state_dict[f"{prefix_to}.resnets.2.norm1.weight"] = old_state_dict.pop(f"{prefix_from}.block.2.norm1.weight") + new_state_dict[f"{prefix_to}.resnets.2.norm1.bias"] = old_state_dict.pop(f"{prefix_from}.block.2.norm1.bias") + new_state_dict[f"{prefix_to}.resnets.2.conv1.weight"] = old_state_dict.pop(f"{prefix_from}.block.2.conv1.weight") + new_state_dict[f"{prefix_to}.resnets.2.conv1.bias"] = old_state_dict.pop(f"{prefix_from}.block.2.conv1.bias") + new_state_dict[f"{prefix_to}.resnets.2.norm2.weight"] = old_state_dict.pop(f"{prefix_from}.block.2.norm2.weight") + new_state_dict[f"{prefix_to}.resnets.2.norm2.bias"] = old_state_dict.pop(f"{prefix_from}.block.2.norm2.bias") + new_state_dict[f"{prefix_to}.resnets.2.conv2.weight"] = old_state_dict.pop(f"{prefix_from}.block.2.conv2.weight") + new_state_dict[f"{prefix_to}.resnets.2.conv2.bias"] = old_state_dict.pop(f"{prefix_from}.block.2.conv2.bias") + + # fmt: on + + +if __name__ == "__main__": + main() diff --git a/diffusers-0.27.0/scripts/convert_animatediff_motion_lora_to_diffusers.py b/diffusers-0.27.0/scripts/convert_animatediff_motion_lora_to_diffusers.py new file mode 100755 index 0000000..509a734 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_animatediff_motion_lora_to_diffusers.py @@ -0,0 +1,51 @@ +import argparse + +import torch +from safetensors.torch import save_file + + +def convert_motion_module(original_state_dict): + converted_state_dict = {} + for k, v in original_state_dict.items(): + if "pos_encoder" in k: + continue + + else: + converted_state_dict[ + k.replace(".norms.0", ".norm1") + .replace(".norms.1", ".norm2") + .replace(".ff_norm", ".norm3") + .replace(".attention_blocks.0", ".attn1") + .replace(".attention_blocks.1", ".attn2") + .replace(".temporal_transformer", "") + ] = v + + return converted_state_dict + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--ckpt_path", type=str, required=True) + parser.add_argument("--output_path", type=str, required=True) + + return parser.parse_args() + + +if __name__ == "__main__": + args = get_args() + + state_dict = torch.load(args.ckpt_path, map_location="cpu") + + if "state_dict" in state_dict.keys(): + state_dict = state_dict["state_dict"] + + conv_state_dict = convert_motion_module(state_dict) + + # convert to new format + output_dict = {} + for module_name, params in conv_state_dict.items(): + if type(params) is not torch.Tensor: + continue + output_dict.update({f"unet.{module_name}": params}) + + save_file(output_dict, f"{args.output_path}/diffusion_pytorch_model.safetensors") diff --git a/diffusers-0.27.0/scripts/convert_animatediff_motion_module_to_diffusers.py b/diffusers-0.27.0/scripts/convert_animatediff_motion_module_to_diffusers.py new file mode 100755 index 0000000..ceb967a --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_animatediff_motion_module_to_diffusers.py @@ -0,0 +1,54 @@ +import argparse + +import torch + +from diffusers import MotionAdapter + + +def convert_motion_module(original_state_dict): + converted_state_dict = {} + for k, v in original_state_dict.items(): + if "pos_encoder" in k: + continue + + else: + converted_state_dict[ + k.replace(".norms.0", ".norm1") + .replace(".norms.1", ".norm2") + .replace(".ff_norm", ".norm3") + .replace(".attention_blocks.0", ".attn1") + .replace(".attention_blocks.1", ".attn2") + .replace(".temporal_transformer", "") + ] = v + + return converted_state_dict + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--ckpt_path", type=str, required=True) + parser.add_argument("--output_path", type=str, required=True) + parser.add_argument("--use_motion_mid_block", action="store_true") + parser.add_argument("--motion_max_seq_length", type=int, default=32) + parser.add_argument("--save_fp16", action="store_true") + + return parser.parse_args() + + +if __name__ == "__main__": + args = get_args() + + state_dict = torch.load(args.ckpt_path, map_location="cpu") + if "state_dict" in state_dict.keys(): + state_dict = state_dict["state_dict"] + + conv_state_dict = convert_motion_module(state_dict) + adapter = MotionAdapter( + use_motion_mid_block=args.use_motion_mid_block, motion_max_seq_length=args.motion_max_seq_length + ) + # skip loading position embeddings + adapter.load_state_dict(conv_state_dict, strict=False) + adapter.save_pretrained(args.output_path) + + if args.save_fp16: + adapter.to(torch.float16).save_pretrained(args.output_path, variant="fp16") diff --git a/diffusers-0.27.0/scripts/convert_asymmetric_vqgan_to_diffusers.py b/diffusers-0.27.0/scripts/convert_asymmetric_vqgan_to_diffusers.py new file mode 100755 index 0000000..ffb735e --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_asymmetric_vqgan_to_diffusers.py @@ -0,0 +1,184 @@ +import argparse +import time +from pathlib import Path +from typing import Any, Dict, Literal + +import torch + +from diffusers import AsymmetricAutoencoderKL + + +ASYMMETRIC_AUTOENCODER_KL_x_1_5_CONFIG = { + "in_channels": 3, + "out_channels": 3, + "down_block_types": [ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + ], + "down_block_out_channels": [128, 256, 512, 512], + "layers_per_down_block": 2, + "up_block_types": [ + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + ], + "up_block_out_channels": [192, 384, 768, 768], + "layers_per_up_block": 3, + "act_fn": "silu", + "latent_channels": 4, + "norm_num_groups": 32, + "sample_size": 256, + "scaling_factor": 0.18215, +} + +ASYMMETRIC_AUTOENCODER_KL_x_2_CONFIG = { + "in_channels": 3, + "out_channels": 3, + "down_block_types": [ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + ], + "down_block_out_channels": [128, 256, 512, 512], + "layers_per_down_block": 2, + "up_block_types": [ + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + "UpDecoderBlock2D", + ], + "up_block_out_channels": [256, 512, 1024, 1024], + "layers_per_up_block": 5, + "act_fn": "silu", + "latent_channels": 4, + "norm_num_groups": 32, + "sample_size": 256, + "scaling_factor": 0.18215, +} + + +def convert_asymmetric_autoencoder_kl_state_dict(original_state_dict: Dict[str, Any]) -> Dict[str, Any]: + converted_state_dict = {} + for k, v in original_state_dict.items(): + if k.startswith("encoder."): + converted_state_dict[ + k.replace("encoder.down.", "encoder.down_blocks.") + .replace("encoder.mid.", "encoder.mid_block.") + .replace("encoder.norm_out.", "encoder.conv_norm_out.") + .replace(".downsample.", ".downsamplers.0.") + .replace(".nin_shortcut.", ".conv_shortcut.") + .replace(".block.", ".resnets.") + .replace(".block_1.", ".resnets.0.") + .replace(".block_2.", ".resnets.1.") + .replace(".attn_1.k.", ".attentions.0.to_k.") + .replace(".attn_1.q.", ".attentions.0.to_q.") + .replace(".attn_1.v.", ".attentions.0.to_v.") + .replace(".attn_1.proj_out.", ".attentions.0.to_out.0.") + .replace(".attn_1.norm.", ".attentions.0.group_norm.") + ] = v + elif k.startswith("decoder.") and "up_layers" not in k: + converted_state_dict[ + k.replace("decoder.encoder.", "decoder.condition_encoder.") + .replace(".norm_out.", ".conv_norm_out.") + .replace(".up.0.", ".up_blocks.3.") + .replace(".up.1.", ".up_blocks.2.") + .replace(".up.2.", ".up_blocks.1.") + .replace(".up.3.", ".up_blocks.0.") + .replace(".block.", ".resnets.") + .replace("mid", "mid_block") + .replace(".0.upsample.", ".0.upsamplers.0.") + .replace(".1.upsample.", ".1.upsamplers.0.") + .replace(".2.upsample.", ".2.upsamplers.0.") + .replace(".nin_shortcut.", ".conv_shortcut.") + .replace(".block_1.", ".resnets.0.") + .replace(".block_2.", ".resnets.1.") + .replace(".attn_1.k.", ".attentions.0.to_k.") + .replace(".attn_1.q.", ".attentions.0.to_q.") + .replace(".attn_1.v.", ".attentions.0.to_v.") + .replace(".attn_1.proj_out.", ".attentions.0.to_out.0.") + .replace(".attn_1.norm.", ".attentions.0.group_norm.") + ] = v + elif k.startswith("quant_conv."): + converted_state_dict[k] = v + elif k.startswith("post_quant_conv."): + converted_state_dict[k] = v + else: + print(f" skipping key `{k}`") + # fix weights shape + for k, v in converted_state_dict.items(): + if ( + (k.startswith("encoder.mid_block.attentions.0") or k.startswith("decoder.mid_block.attentions.0")) + and k.endswith("weight") + and ("to_q" in k or "to_k" in k or "to_v" in k or "to_out" in k) + ): + converted_state_dict[k] = converted_state_dict[k][:, :, 0, 0] + + return converted_state_dict + + +def get_asymmetric_autoencoder_kl_from_original_checkpoint( + scale: Literal["1.5", "2"], original_checkpoint_path: str, map_location: torch.device +) -> AsymmetricAutoencoderKL: + print("Loading original state_dict") + original_state_dict = torch.load(original_checkpoint_path, map_location=map_location) + original_state_dict = original_state_dict["state_dict"] + print("Converting state_dict") + converted_state_dict = convert_asymmetric_autoencoder_kl_state_dict(original_state_dict) + kwargs = ASYMMETRIC_AUTOENCODER_KL_x_1_5_CONFIG if scale == "1.5" else ASYMMETRIC_AUTOENCODER_KL_x_2_CONFIG + print("Initializing AsymmetricAutoencoderKL model") + asymmetric_autoencoder_kl = AsymmetricAutoencoderKL(**kwargs) + print("Loading weight from converted state_dict") + asymmetric_autoencoder_kl.load_state_dict(converted_state_dict) + asymmetric_autoencoder_kl.eval() + print("AsymmetricAutoencoderKL successfully initialized") + return asymmetric_autoencoder_kl + + +if __name__ == "__main__": + start = time.time() + parser = argparse.ArgumentParser() + parser.add_argument( + "--scale", + default=None, + type=str, + required=True, + help="Asymmetric VQGAN scale: `1.5` or `2`", + ) + parser.add_argument( + "--original_checkpoint_path", + default=None, + type=str, + required=True, + help="Path to the original Asymmetric VQGAN checkpoint", + ) + parser.add_argument( + "--output_path", + default=None, + type=str, + required=True, + help="Path to save pretrained AsymmetricAutoencoderKL model", + ) + parser.add_argument( + "--map_location", + default="cpu", + type=str, + required=False, + help="The device passed to `map_location` when loading the checkpoint", + ) + args = parser.parse_args() + + assert args.scale in ["1.5", "2"], f"{args.scale} should be `1.5` of `2`" + assert Path(args.original_checkpoint_path).is_file() + + asymmetric_autoencoder_kl = get_asymmetric_autoencoder_kl_from_original_checkpoint( + scale=args.scale, + original_checkpoint_path=args.original_checkpoint_path, + map_location=torch.device(args.map_location), + ) + print("Saving pretrained AsymmetricAutoencoderKL") + asymmetric_autoencoder_kl.save_pretrained(args.output_path) + print(f"Done in {time.time() - start:.2f} seconds") diff --git a/diffusers-0.27.0/scripts/convert_blipdiffusion_to_diffusers.py b/diffusers-0.27.0/scripts/convert_blipdiffusion_to_diffusers.py new file mode 100755 index 0000000..03cf67e --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_blipdiffusion_to_diffusers.py @@ -0,0 +1,343 @@ +""" +This script requires you to build `LAVIS` from source, since the pip version doesn't have BLIP Diffusion. Follow instructions here: https://github.com/salesforce/LAVIS/tree/main. +""" + +import argparse +import os +import tempfile + +import torch +from lavis.models import load_model_and_preprocess +from transformers import CLIPTokenizer +from transformers.models.blip_2.configuration_blip_2 import Blip2Config + +from diffusers import ( + AutoencoderKL, + PNDMScheduler, + UNet2DConditionModel, +) +from diffusers.pipelines import BlipDiffusionPipeline +from diffusers.pipelines.blip_diffusion.blip_image_processing import BlipImageProcessor +from diffusers.pipelines.blip_diffusion.modeling_blip2 import Blip2QFormerModel +from diffusers.pipelines.blip_diffusion.modeling_ctx_clip import ContextCLIPTextModel + + +BLIP2_CONFIG = { + "vision_config": { + "hidden_size": 1024, + "num_hidden_layers": 23, + "num_attention_heads": 16, + "image_size": 224, + "patch_size": 14, + "intermediate_size": 4096, + "hidden_act": "quick_gelu", + }, + "qformer_config": { + "cross_attention_frequency": 1, + "encoder_hidden_size": 1024, + "vocab_size": 30523, + }, + "num_query_tokens": 16, +} +blip2config = Blip2Config(**BLIP2_CONFIG) + + +def qformer_model_from_original_config(): + qformer = Blip2QFormerModel(blip2config) + return qformer + + +def embeddings_from_original_checkpoint(model, diffuser_embeddings_prefix, original_embeddings_prefix): + embeddings = {} + embeddings.update( + { + f"{diffuser_embeddings_prefix}.word_embeddings.weight": model[ + f"{original_embeddings_prefix}.word_embeddings.weight" + ] + } + ) + embeddings.update( + { + f"{diffuser_embeddings_prefix}.position_embeddings.weight": model[ + f"{original_embeddings_prefix}.position_embeddings.weight" + ] + } + ) + embeddings.update( + {f"{diffuser_embeddings_prefix}.LayerNorm.weight": model[f"{original_embeddings_prefix}.LayerNorm.weight"]} + ) + embeddings.update( + {f"{diffuser_embeddings_prefix}.LayerNorm.bias": model[f"{original_embeddings_prefix}.LayerNorm.bias"]} + ) + return embeddings + + +def proj_layer_from_original_checkpoint(model, diffuser_proj_prefix, original_proj_prefix): + proj_layer = {} + proj_layer.update({f"{diffuser_proj_prefix}.dense1.weight": model[f"{original_proj_prefix}.dense1.weight"]}) + proj_layer.update({f"{diffuser_proj_prefix}.dense1.bias": model[f"{original_proj_prefix}.dense1.bias"]}) + proj_layer.update({f"{diffuser_proj_prefix}.dense2.weight": model[f"{original_proj_prefix}.dense2.weight"]}) + proj_layer.update({f"{diffuser_proj_prefix}.dense2.bias": model[f"{original_proj_prefix}.dense2.bias"]}) + proj_layer.update({f"{diffuser_proj_prefix}.LayerNorm.weight": model[f"{original_proj_prefix}.LayerNorm.weight"]}) + proj_layer.update({f"{diffuser_proj_prefix}.LayerNorm.bias": model[f"{original_proj_prefix}.LayerNorm.bias"]}) + return proj_layer + + +def attention_from_original_checkpoint(model, diffuser_attention_prefix, original_attention_prefix): + attention = {} + attention.update( + { + f"{diffuser_attention_prefix}.attention.query.weight": model[ + f"{original_attention_prefix}.self.query.weight" + ] + } + ) + attention.update( + {f"{diffuser_attention_prefix}.attention.query.bias": model[f"{original_attention_prefix}.self.query.bias"]} + ) + attention.update( + {f"{diffuser_attention_prefix}.attention.key.weight": model[f"{original_attention_prefix}.self.key.weight"]} + ) + attention.update( + {f"{diffuser_attention_prefix}.attention.key.bias": model[f"{original_attention_prefix}.self.key.bias"]} + ) + attention.update( + { + f"{diffuser_attention_prefix}.attention.value.weight": model[ + f"{original_attention_prefix}.self.value.weight" + ] + } + ) + attention.update( + {f"{diffuser_attention_prefix}.attention.value.bias": model[f"{original_attention_prefix}.self.value.bias"]} + ) + attention.update( + {f"{diffuser_attention_prefix}.output.dense.weight": model[f"{original_attention_prefix}.output.dense.weight"]} + ) + attention.update( + {f"{diffuser_attention_prefix}.output.dense.bias": model[f"{original_attention_prefix}.output.dense.bias"]} + ) + attention.update( + { + f"{diffuser_attention_prefix}.output.LayerNorm.weight": model[ + f"{original_attention_prefix}.output.LayerNorm.weight" + ] + } + ) + attention.update( + { + f"{diffuser_attention_prefix}.output.LayerNorm.bias": model[ + f"{original_attention_prefix}.output.LayerNorm.bias" + ] + } + ) + return attention + + +def output_layers_from_original_checkpoint(model, diffuser_output_prefix, original_output_prefix): + output_layers = {} + output_layers.update({f"{diffuser_output_prefix}.dense.weight": model[f"{original_output_prefix}.dense.weight"]}) + output_layers.update({f"{diffuser_output_prefix}.dense.bias": model[f"{original_output_prefix}.dense.bias"]}) + output_layers.update( + {f"{diffuser_output_prefix}.LayerNorm.weight": model[f"{original_output_prefix}.LayerNorm.weight"]} + ) + output_layers.update( + {f"{diffuser_output_prefix}.LayerNorm.bias": model[f"{original_output_prefix}.LayerNorm.bias"]} + ) + return output_layers + + +def encoder_from_original_checkpoint(model, diffuser_encoder_prefix, original_encoder_prefix): + encoder = {} + for i in range(blip2config.qformer_config.num_hidden_layers): + encoder.update( + attention_from_original_checkpoint( + model, f"{diffuser_encoder_prefix}.{i}.attention", f"{original_encoder_prefix}.{i}.attention" + ) + ) + encoder.update( + attention_from_original_checkpoint( + model, f"{diffuser_encoder_prefix}.{i}.crossattention", f"{original_encoder_prefix}.{i}.crossattention" + ) + ) + + encoder.update( + { + f"{diffuser_encoder_prefix}.{i}.intermediate.dense.weight": model[ + f"{original_encoder_prefix}.{i}.intermediate.dense.weight" + ] + } + ) + encoder.update( + { + f"{diffuser_encoder_prefix}.{i}.intermediate.dense.bias": model[ + f"{original_encoder_prefix}.{i}.intermediate.dense.bias" + ] + } + ) + encoder.update( + { + f"{diffuser_encoder_prefix}.{i}.intermediate_query.dense.weight": model[ + f"{original_encoder_prefix}.{i}.intermediate_query.dense.weight" + ] + } + ) + encoder.update( + { + f"{diffuser_encoder_prefix}.{i}.intermediate_query.dense.bias": model[ + f"{original_encoder_prefix}.{i}.intermediate_query.dense.bias" + ] + } + ) + + encoder.update( + output_layers_from_original_checkpoint( + model, f"{diffuser_encoder_prefix}.{i}.output", f"{original_encoder_prefix}.{i}.output" + ) + ) + encoder.update( + output_layers_from_original_checkpoint( + model, f"{diffuser_encoder_prefix}.{i}.output_query", f"{original_encoder_prefix}.{i}.output_query" + ) + ) + return encoder + + +def visual_encoder_layer_from_original_checkpoint(model, diffuser_prefix, original_prefix): + visual_encoder_layer = {} + + visual_encoder_layer.update({f"{diffuser_prefix}.layer_norm1.weight": model[f"{original_prefix}.ln_1.weight"]}) + visual_encoder_layer.update({f"{diffuser_prefix}.layer_norm1.bias": model[f"{original_prefix}.ln_1.bias"]}) + visual_encoder_layer.update({f"{diffuser_prefix}.layer_norm2.weight": model[f"{original_prefix}.ln_2.weight"]}) + visual_encoder_layer.update({f"{diffuser_prefix}.layer_norm2.bias": model[f"{original_prefix}.ln_2.bias"]}) + visual_encoder_layer.update( + {f"{diffuser_prefix}.self_attn.qkv.weight": model[f"{original_prefix}.attn.in_proj_weight"]} + ) + visual_encoder_layer.update( + {f"{diffuser_prefix}.self_attn.qkv.bias": model[f"{original_prefix}.attn.in_proj_bias"]} + ) + visual_encoder_layer.update( + {f"{diffuser_prefix}.self_attn.projection.weight": model[f"{original_prefix}.attn.out_proj.weight"]} + ) + visual_encoder_layer.update( + {f"{diffuser_prefix}.self_attn.projection.bias": model[f"{original_prefix}.attn.out_proj.bias"]} + ) + visual_encoder_layer.update({f"{diffuser_prefix}.mlp.fc1.weight": model[f"{original_prefix}.mlp.c_fc.weight"]}) + visual_encoder_layer.update({f"{diffuser_prefix}.mlp.fc1.bias": model[f"{original_prefix}.mlp.c_fc.bias"]}) + visual_encoder_layer.update({f"{diffuser_prefix}.mlp.fc2.weight": model[f"{original_prefix}.mlp.c_proj.weight"]}) + visual_encoder_layer.update({f"{diffuser_prefix}.mlp.fc2.bias": model[f"{original_prefix}.mlp.c_proj.bias"]}) + + return visual_encoder_layer + + +def visual_encoder_from_original_checkpoint(model, diffuser_prefix, original_prefix): + visual_encoder = {} + + visual_encoder.update( + { + f"{diffuser_prefix}.embeddings.class_embedding": model[f"{original_prefix}.class_embedding"] + .unsqueeze(0) + .unsqueeze(0) + } + ) + visual_encoder.update( + { + f"{diffuser_prefix}.embeddings.position_embedding": model[ + f"{original_prefix}.positional_embedding" + ].unsqueeze(0) + } + ) + visual_encoder.update( + {f"{diffuser_prefix}.embeddings.patch_embedding.weight": model[f"{original_prefix}.conv1.weight"]} + ) + visual_encoder.update({f"{diffuser_prefix}.pre_layernorm.weight": model[f"{original_prefix}.ln_pre.weight"]}) + visual_encoder.update({f"{diffuser_prefix}.pre_layernorm.bias": model[f"{original_prefix}.ln_pre.bias"]}) + + for i in range(blip2config.vision_config.num_hidden_layers): + visual_encoder.update( + visual_encoder_layer_from_original_checkpoint( + model, f"{diffuser_prefix}.encoder.layers.{i}", f"{original_prefix}.transformer.resblocks.{i}" + ) + ) + + visual_encoder.update({f"{diffuser_prefix}.post_layernorm.weight": model["blip.ln_vision.weight"]}) + visual_encoder.update({f"{diffuser_prefix}.post_layernorm.bias": model["blip.ln_vision.bias"]}) + + return visual_encoder + + +def qformer_original_checkpoint_to_diffusers_checkpoint(model): + qformer_checkpoint = {} + qformer_checkpoint.update(embeddings_from_original_checkpoint(model, "embeddings", "blip.Qformer.bert.embeddings")) + qformer_checkpoint.update({"query_tokens": model["blip.query_tokens"]}) + qformer_checkpoint.update(proj_layer_from_original_checkpoint(model, "proj_layer", "proj_layer")) + qformer_checkpoint.update( + encoder_from_original_checkpoint(model, "encoder.layer", "blip.Qformer.bert.encoder.layer") + ) + qformer_checkpoint.update(visual_encoder_from_original_checkpoint(model, "visual_encoder", "blip.visual_encoder")) + return qformer_checkpoint + + +def get_qformer(model): + print("loading qformer") + + qformer = qformer_model_from_original_config() + qformer_diffusers_checkpoint = qformer_original_checkpoint_to_diffusers_checkpoint(model) + + load_checkpoint_to_model(qformer_diffusers_checkpoint, qformer) + + print("done loading qformer") + return qformer + + +def load_checkpoint_to_model(checkpoint, model): + with tempfile.NamedTemporaryFile(delete=False) as file: + torch.save(checkpoint, file.name) + del checkpoint + model.load_state_dict(torch.load(file.name), strict=False) + + os.remove(file.name) + + +def save_blip_diffusion_model(model, args): + qformer = get_qformer(model) + qformer.eval() + + text_encoder = ContextCLIPTextModel.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="text_encoder") + vae = AutoencoderKL.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="vae") + + unet = UNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="unet") + vae.eval() + text_encoder.eval() + scheduler = PNDMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + set_alpha_to_one=False, + skip_prk_steps=True, + ) + tokenizer = CLIPTokenizer.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="tokenizer") + image_processor = BlipImageProcessor() + blip_diffusion = BlipDiffusionPipeline( + tokenizer=tokenizer, + text_encoder=text_encoder, + vae=vae, + unet=unet, + scheduler=scheduler, + qformer=qformer, + image_processor=image_processor, + ) + blip_diffusion.save_pretrained(args.checkpoint_path) + + +def main(args): + model, _, _ = load_model_and_preprocess("blip_diffusion", "base", device="cpu", is_eval=True) + save_blip_diffusion_model(model.state_dict(), args) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--checkpoint_path", default=None, type=str, required=True, help="Path to the output model.") + args = parser.parse_args() + + main(args) diff --git a/diffusers-0.27.0/scripts/convert_consistency_decoder.py b/diffusers-0.27.0/scripts/convert_consistency_decoder.py new file mode 100755 index 0000000..0cb5fc5 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_consistency_decoder.py @@ -0,0 +1,1128 @@ +import math +import os +import urllib +import warnings +from argparse import ArgumentParser + +import torch +import torch.nn as nn +import torch.nn.functional as F +from huggingface_hub.utils import insecure_hashlib +from safetensors.torch import load_file as stl +from tqdm import tqdm + +from diffusers import AutoencoderKL, ConsistencyDecoderVAE, DiffusionPipeline, StableDiffusionPipeline, UNet2DModel +from diffusers.models.autoencoders.vae import Encoder +from diffusers.models.embeddings import TimestepEmbedding +from diffusers.models.unets.unet_2d_blocks import ResnetDownsampleBlock2D, ResnetUpsampleBlock2D, UNetMidBlock2D + + +args = ArgumentParser() +args.add_argument("--save_pretrained", required=False, default=None, type=str) +args.add_argument("--test_image", required=True, type=str) +args = args.parse_args() + + +def _extract_into_tensor(arr, timesteps, broadcast_shape): + # from: https://github.com/openai/guided-diffusion/blob/22e0df8183507e13a7813f8d38d51b072ca1e67c/guided_diffusion/gaussian_diffusion.py#L895 """ + res = arr[timesteps].float() + dims_to_append = len(broadcast_shape) - len(res.shape) + return res[(...,) + (None,) * dims_to_append] + + +def betas_for_alpha_bar(num_diffusion_timesteps, alpha_bar, max_beta=0.999): + # from: https://github.com/openai/guided-diffusion/blob/22e0df8183507e13a7813f8d38d51b072ca1e67c/guided_diffusion/gaussian_diffusion.py#L45 + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta)) + return torch.tensor(betas) + + +def _download(url: str, root: str): + os.makedirs(root, exist_ok=True) + filename = os.path.basename(url) + + expected_sha256 = url.split("/")[-2] + download_target = os.path.join(root, filename) + + if os.path.exists(download_target) and not os.path.isfile(download_target): + raise RuntimeError(f"{download_target} exists and is not a regular file") + + if os.path.isfile(download_target): + if insecure_hashlib.sha256(open(download_target, "rb").read()).hexdigest() == expected_sha256: + return download_target + else: + warnings.warn(f"{download_target} exists, but the SHA256 checksum does not match; re-downloading the file") + + with urllib.request.urlopen(url) as source, open(download_target, "wb") as output: + with tqdm( + total=int(source.info().get("Content-Length")), + ncols=80, + unit="iB", + unit_scale=True, + unit_divisor=1024, + ) as loop: + while True: + buffer = source.read(8192) + if not buffer: + break + + output.write(buffer) + loop.update(len(buffer)) + + if insecure_hashlib.sha256(open(download_target, "rb").read()).hexdigest() != expected_sha256: + raise RuntimeError("Model has been downloaded but the SHA256 checksum does not not match") + + return download_target + + +class ConsistencyDecoder: + def __init__(self, device="cuda:0", download_root=os.path.expanduser("~/.cache/clip")): + self.n_distilled_steps = 64 + download_target = _download( + "https://openaipublic.azureedge.net/diff-vae/c9cebd3132dd9c42936d803e33424145a748843c8f716c0814838bdc8a2fe7cb/decoder.pt", + download_root, + ) + self.ckpt = torch.jit.load(download_target).to(device) + self.device = device + sigma_data = 0.5 + betas = betas_for_alpha_bar(1024, lambda t: math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2).to(device) + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + self.sqrt_alphas_cumprod = torch.sqrt(alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 - alphas_cumprod) + sqrt_recip_alphas_cumprod = torch.sqrt(1.0 / alphas_cumprod) + sigmas = torch.sqrt(1.0 / alphas_cumprod - 1) + self.c_skip = sqrt_recip_alphas_cumprod * sigma_data**2 / (sigmas**2 + sigma_data**2) + self.c_out = sigmas * sigma_data / (sigmas**2 + sigma_data**2) ** 0.5 + self.c_in = sqrt_recip_alphas_cumprod / (sigmas**2 + sigma_data**2) ** 0.5 + + @staticmethod + def round_timesteps(timesteps, total_timesteps, n_distilled_steps, truncate_start=True): + with torch.no_grad(): + space = torch.div(total_timesteps, n_distilled_steps, rounding_mode="floor") + rounded_timesteps = (torch.div(timesteps, space, rounding_mode="floor") + 1) * space + if truncate_start: + rounded_timesteps[rounded_timesteps == total_timesteps] -= space + else: + rounded_timesteps[rounded_timesteps == total_timesteps] -= space + rounded_timesteps[rounded_timesteps == 0] += space + return rounded_timesteps + + @staticmethod + def ldm_transform_latent(z, extra_scale_factor=1): + channel_means = [0.38862467, 0.02253063, 0.07381133, -0.0171294] + channel_stds = [0.9654121, 1.0440036, 0.76147926, 0.77022034] + + if len(z.shape) != 4: + raise ValueError() + + z = z * 0.18215 + channels = [z[:, i] for i in range(z.shape[1])] + + channels = [extra_scale_factor * (c - channel_means[i]) / channel_stds[i] for i, c in enumerate(channels)] + return torch.stack(channels, dim=1) + + @torch.no_grad() + def __call__( + self, + features: torch.Tensor, + schedule=[1.0, 0.5], + generator=None, + ): + features = self.ldm_transform_latent(features) + ts = self.round_timesteps( + torch.arange(0, 1024), + 1024, + self.n_distilled_steps, + truncate_start=False, + ) + shape = ( + features.size(0), + 3, + 8 * features.size(2), + 8 * features.size(3), + ) + x_start = torch.zeros(shape, device=features.device, dtype=features.dtype) + schedule_timesteps = [int((1024 - 1) * s) for s in schedule] + for i in schedule_timesteps: + t = ts[i].item() + t_ = torch.tensor([t] * features.shape[0]).to(self.device) + # noise = torch.randn_like(x_start) + noise = torch.randn(x_start.shape, dtype=x_start.dtype, generator=generator).to(device=x_start.device) + x_start = ( + _extract_into_tensor(self.sqrt_alphas_cumprod, t_, x_start.shape) * x_start + + _extract_into_tensor(self.sqrt_one_minus_alphas_cumprod, t_, x_start.shape) * noise + ) + c_in = _extract_into_tensor(self.c_in, t_, x_start.shape) + + import torch.nn.functional as F + + from diffusers import UNet2DModel + + if isinstance(self.ckpt, UNet2DModel): + input = torch.concat([c_in * x_start, F.upsample_nearest(features, scale_factor=8)], dim=1) + model_output = self.ckpt(input, t_).sample + else: + model_output = self.ckpt(c_in * x_start, t_, features=features) + + B, C = x_start.shape[:2] + model_output, _ = torch.split(model_output, C, dim=1) + pred_xstart = ( + _extract_into_tensor(self.c_out, t_, x_start.shape) * model_output + + _extract_into_tensor(self.c_skip, t_, x_start.shape) * x_start + ).clamp(-1, 1) + x_start = pred_xstart + return x_start + + +def save_image(image, name): + import numpy as np + from PIL import Image + + image = image[0].cpu().numpy() + image = (image + 1.0) * 127.5 + image = image.clip(0, 255).astype(np.uint8) + image = Image.fromarray(image.transpose(1, 2, 0)) + image.save(name) + + +def load_image(uri, size=None, center_crop=False): + import numpy as np + from PIL import Image + + image = Image.open(uri) + if center_crop: + image = image.crop( + ( + (image.width - min(image.width, image.height)) // 2, + (image.height - min(image.width, image.height)) // 2, + (image.width + min(image.width, image.height)) // 2, + (image.height + min(image.width, image.height)) // 2, + ) + ) + if size is not None: + image = image.resize(size) + image = torch.tensor(np.array(image).transpose(2, 0, 1)).unsqueeze(0).float() + image = image / 127.5 - 1.0 + return image + + +class TimestepEmbedding_(nn.Module): + def __init__(self, n_time=1024, n_emb=320, n_out=1280) -> None: + super().__init__() + self.emb = nn.Embedding(n_time, n_emb) + self.f_1 = nn.Linear(n_emb, n_out) + self.f_2 = nn.Linear(n_out, n_out) + + def forward(self, x) -> torch.Tensor: + x = self.emb(x) + x = self.f_1(x) + x = F.silu(x) + return self.f_2(x) + + +class ImageEmbedding(nn.Module): + def __init__(self, in_channels=7, out_channels=320) -> None: + super().__init__() + self.f = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1) + + def forward(self, x) -> torch.Tensor: + return self.f(x) + + +class ImageUnembedding(nn.Module): + def __init__(self, in_channels=320, out_channels=6) -> None: + super().__init__() + self.gn = nn.GroupNorm(32, in_channels) + self.f = nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1) + + def forward(self, x) -> torch.Tensor: + return self.f(F.silu(self.gn(x))) + + +class ConvResblock(nn.Module): + def __init__(self, in_features=320, out_features=320) -> None: + super().__init__() + self.f_t = nn.Linear(1280, out_features * 2) + + self.gn_1 = nn.GroupNorm(32, in_features) + self.f_1 = nn.Conv2d(in_features, out_features, kernel_size=3, padding=1) + + self.gn_2 = nn.GroupNorm(32, out_features) + self.f_2 = nn.Conv2d(out_features, out_features, kernel_size=3, padding=1) + + skip_conv = in_features != out_features + self.f_s = nn.Conv2d(in_features, out_features, kernel_size=1, padding=0) if skip_conv else nn.Identity() + + def forward(self, x, t): + x_skip = x + t = self.f_t(F.silu(t)) + t = t.chunk(2, dim=1) + t_1 = t[0].unsqueeze(dim=2).unsqueeze(dim=3) + 1 + t_2 = t[1].unsqueeze(dim=2).unsqueeze(dim=3) + + gn_1 = F.silu(self.gn_1(x)) + f_1 = self.f_1(gn_1) + + gn_2 = self.gn_2(f_1) + + return self.f_s(x_skip) + self.f_2(F.silu(gn_2 * t_1 + t_2)) + + +# Also ConvResblock +class Downsample(nn.Module): + def __init__(self, in_channels=320) -> None: + super().__init__() + self.f_t = nn.Linear(1280, in_channels * 2) + + self.gn_1 = nn.GroupNorm(32, in_channels) + self.f_1 = nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1) + self.gn_2 = nn.GroupNorm(32, in_channels) + + self.f_2 = nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1) + + def forward(self, x, t) -> torch.Tensor: + x_skip = x + + t = self.f_t(F.silu(t)) + t_1, t_2 = t.chunk(2, dim=1) + t_1 = t_1.unsqueeze(2).unsqueeze(3) + 1 + t_2 = t_2.unsqueeze(2).unsqueeze(3) + + gn_1 = F.silu(self.gn_1(x)) + avg_pool2d = F.avg_pool2d(gn_1, kernel_size=(2, 2), stride=None) + + f_1 = self.f_1(avg_pool2d) + gn_2 = self.gn_2(f_1) + + f_2 = self.f_2(F.silu(t_2 + (t_1 * gn_2))) + + return f_2 + F.avg_pool2d(x_skip, kernel_size=(2, 2), stride=None) + + +# Also ConvResblock +class Upsample(nn.Module): + def __init__(self, in_channels=1024) -> None: + super().__init__() + self.f_t = nn.Linear(1280, in_channels * 2) + + self.gn_1 = nn.GroupNorm(32, in_channels) + self.f_1 = nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1) + self.gn_2 = nn.GroupNorm(32, in_channels) + + self.f_2 = nn.Conv2d(in_channels, in_channels, kernel_size=3, padding=1) + + def forward(self, x, t) -> torch.Tensor: + x_skip = x + + t = self.f_t(F.silu(t)) + t_1, t_2 = t.chunk(2, dim=1) + t_1 = t_1.unsqueeze(2).unsqueeze(3) + 1 + t_2 = t_2.unsqueeze(2).unsqueeze(3) + + gn_1 = F.silu(self.gn_1(x)) + upsample = F.upsample_nearest(gn_1, scale_factor=2) + f_1 = self.f_1(upsample) + gn_2 = self.gn_2(f_1) + + f_2 = self.f_2(F.silu(t_2 + (t_1 * gn_2))) + + return f_2 + F.upsample_nearest(x_skip, scale_factor=2) + + +class ConvUNetVAE(nn.Module): + def __init__(self) -> None: + super().__init__() + self.embed_image = ImageEmbedding() + self.embed_time = TimestepEmbedding_() + + down_0 = nn.ModuleList( + [ + ConvResblock(320, 320), + ConvResblock(320, 320), + ConvResblock(320, 320), + Downsample(320), + ] + ) + down_1 = nn.ModuleList( + [ + ConvResblock(320, 640), + ConvResblock(640, 640), + ConvResblock(640, 640), + Downsample(640), + ] + ) + down_2 = nn.ModuleList( + [ + ConvResblock(640, 1024), + ConvResblock(1024, 1024), + ConvResblock(1024, 1024), + Downsample(1024), + ] + ) + down_3 = nn.ModuleList( + [ + ConvResblock(1024, 1024), + ConvResblock(1024, 1024), + ConvResblock(1024, 1024), + ] + ) + self.down = nn.ModuleList( + [ + down_0, + down_1, + down_2, + down_3, + ] + ) + + self.mid = nn.ModuleList( + [ + ConvResblock(1024, 1024), + ConvResblock(1024, 1024), + ] + ) + + up_3 = nn.ModuleList( + [ + ConvResblock(1024 * 2, 1024), + ConvResblock(1024 * 2, 1024), + ConvResblock(1024 * 2, 1024), + ConvResblock(1024 * 2, 1024), + Upsample(1024), + ] + ) + up_2 = nn.ModuleList( + [ + ConvResblock(1024 * 2, 1024), + ConvResblock(1024 * 2, 1024), + ConvResblock(1024 * 2, 1024), + ConvResblock(1024 + 640, 1024), + Upsample(1024), + ] + ) + up_1 = nn.ModuleList( + [ + ConvResblock(1024 + 640, 640), + ConvResblock(640 * 2, 640), + ConvResblock(640 * 2, 640), + ConvResblock(320 + 640, 640), + Upsample(640), + ] + ) + up_0 = nn.ModuleList( + [ + ConvResblock(320 + 640, 320), + ConvResblock(320 * 2, 320), + ConvResblock(320 * 2, 320), + ConvResblock(320 * 2, 320), + ] + ) + self.up = nn.ModuleList( + [ + up_0, + up_1, + up_2, + up_3, + ] + ) + + self.output = ImageUnembedding() + + def forward(self, x, t, features) -> torch.Tensor: + converted = hasattr(self, "converted") and self.converted + + x = torch.cat([x, F.upsample_nearest(features, scale_factor=8)], dim=1) + + if converted: + t = self.time_embedding(self.time_proj(t)) + else: + t = self.embed_time(t) + + x = self.embed_image(x) + + skips = [x] + for i, down in enumerate(self.down): + if converted and i in [0, 1, 2, 3]: + x, skips_ = down(x, t) + for skip in skips_: + skips.append(skip) + else: + for block in down: + x = block(x, t) + skips.append(x) + print(x.float().abs().sum()) + + if converted: + x = self.mid(x, t) + else: + for i in range(2): + x = self.mid[i](x, t) + print(x.float().abs().sum()) + + for i, up in enumerate(self.up[::-1]): + if converted and i in [0, 1, 2, 3]: + skip_4 = skips.pop() + skip_3 = skips.pop() + skip_2 = skips.pop() + skip_1 = skips.pop() + skips_ = (skip_1, skip_2, skip_3, skip_4) + x = up(x, skips_, t) + else: + for block in up: + if isinstance(block, ConvResblock): + x = torch.concat([x, skips.pop()], dim=1) + x = block(x, t) + + return self.output(x) + + +def rename_state_dict_key(k): + k = k.replace("blocks.", "") + for i in range(5): + k = k.replace(f"down_{i}_", f"down.{i}.") + k = k.replace(f"conv_{i}.", f"{i}.") + k = k.replace(f"up_{i}_", f"up.{i}.") + k = k.replace(f"mid_{i}", f"mid.{i}") + k = k.replace("upsamp.", "4.") + k = k.replace("downsamp.", "3.") + k = k.replace("f_t.w", "f_t.weight").replace("f_t.b", "f_t.bias") + k = k.replace("f_1.w", "f_1.weight").replace("f_1.b", "f_1.bias") + k = k.replace("f_2.w", "f_2.weight").replace("f_2.b", "f_2.bias") + k = k.replace("f_s.w", "f_s.weight").replace("f_s.b", "f_s.bias") + k = k.replace("f.w", "f.weight").replace("f.b", "f.bias") + k = k.replace("gn_1.g", "gn_1.weight").replace("gn_1.b", "gn_1.bias") + k = k.replace("gn_2.g", "gn_2.weight").replace("gn_2.b", "gn_2.bias") + k = k.replace("gn.g", "gn.weight").replace("gn.b", "gn.bias") + return k + + +def rename_state_dict(sd, embedding): + sd = {rename_state_dict_key(k): v for k, v in sd.items()} + sd["embed_time.emb.weight"] = embedding["weight"] + return sd + + +# encode with stable diffusion vae +pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) +pipe.vae.cuda() + +# construct original decoder with jitted model +decoder_consistency = ConsistencyDecoder(device="cuda:0") + +# construct UNet code, overwrite the decoder with conv_unet_vae +model = ConvUNetVAE() +model.load_state_dict( + rename_state_dict( + stl("consistency_decoder.safetensors"), + stl("embedding.safetensors"), + ) +) +model = model.cuda() + +decoder_consistency.ckpt = model + +image = load_image(args.test_image, size=(256, 256), center_crop=True) +latent = pipe.vae.encode(image.half().cuda()).latent_dist.sample() + +# decode with gan +sample_gan = pipe.vae.decode(latent).sample.detach() +save_image(sample_gan, "gan.png") + +# decode with conv_unet_vae +sample_consistency_orig = decoder_consistency(latent, generator=torch.Generator("cpu").manual_seed(0)) +save_image(sample_consistency_orig, "con_orig.png") + + +########### conversion + +print("CONVERSION") + +print("DOWN BLOCK ONE") + +block_one_sd_orig = model.down[0].state_dict() +block_one_sd_new = {} + +for i in range(3): + block_one_sd_new[f"resnets.{i}.norm1.weight"] = block_one_sd_orig.pop(f"{i}.gn_1.weight") + block_one_sd_new[f"resnets.{i}.norm1.bias"] = block_one_sd_orig.pop(f"{i}.gn_1.bias") + block_one_sd_new[f"resnets.{i}.conv1.weight"] = block_one_sd_orig.pop(f"{i}.f_1.weight") + block_one_sd_new[f"resnets.{i}.conv1.bias"] = block_one_sd_orig.pop(f"{i}.f_1.bias") + block_one_sd_new[f"resnets.{i}.time_emb_proj.weight"] = block_one_sd_orig.pop(f"{i}.f_t.weight") + block_one_sd_new[f"resnets.{i}.time_emb_proj.bias"] = block_one_sd_orig.pop(f"{i}.f_t.bias") + block_one_sd_new[f"resnets.{i}.norm2.weight"] = block_one_sd_orig.pop(f"{i}.gn_2.weight") + block_one_sd_new[f"resnets.{i}.norm2.bias"] = block_one_sd_orig.pop(f"{i}.gn_2.bias") + block_one_sd_new[f"resnets.{i}.conv2.weight"] = block_one_sd_orig.pop(f"{i}.f_2.weight") + block_one_sd_new[f"resnets.{i}.conv2.bias"] = block_one_sd_orig.pop(f"{i}.f_2.bias") + +block_one_sd_new["downsamplers.0.norm1.weight"] = block_one_sd_orig.pop("3.gn_1.weight") +block_one_sd_new["downsamplers.0.norm1.bias"] = block_one_sd_orig.pop("3.gn_1.bias") +block_one_sd_new["downsamplers.0.conv1.weight"] = block_one_sd_orig.pop("3.f_1.weight") +block_one_sd_new["downsamplers.0.conv1.bias"] = block_one_sd_orig.pop("3.f_1.bias") +block_one_sd_new["downsamplers.0.time_emb_proj.weight"] = block_one_sd_orig.pop("3.f_t.weight") +block_one_sd_new["downsamplers.0.time_emb_proj.bias"] = block_one_sd_orig.pop("3.f_t.bias") +block_one_sd_new["downsamplers.0.norm2.weight"] = block_one_sd_orig.pop("3.gn_2.weight") +block_one_sd_new["downsamplers.0.norm2.bias"] = block_one_sd_orig.pop("3.gn_2.bias") +block_one_sd_new["downsamplers.0.conv2.weight"] = block_one_sd_orig.pop("3.f_2.weight") +block_one_sd_new["downsamplers.0.conv2.bias"] = block_one_sd_orig.pop("3.f_2.bias") + +assert len(block_one_sd_orig) == 0 + +block_one = ResnetDownsampleBlock2D( + in_channels=320, + out_channels=320, + temb_channels=1280, + num_layers=3, + add_downsample=True, + resnet_time_scale_shift="scale_shift", + resnet_eps=1e-5, +) + +block_one.load_state_dict(block_one_sd_new) + +print("DOWN BLOCK TWO") + +block_two_sd_orig = model.down[1].state_dict() +block_two_sd_new = {} + +for i in range(3): + block_two_sd_new[f"resnets.{i}.norm1.weight"] = block_two_sd_orig.pop(f"{i}.gn_1.weight") + block_two_sd_new[f"resnets.{i}.norm1.bias"] = block_two_sd_orig.pop(f"{i}.gn_1.bias") + block_two_sd_new[f"resnets.{i}.conv1.weight"] = block_two_sd_orig.pop(f"{i}.f_1.weight") + block_two_sd_new[f"resnets.{i}.conv1.bias"] = block_two_sd_orig.pop(f"{i}.f_1.bias") + block_two_sd_new[f"resnets.{i}.time_emb_proj.weight"] = block_two_sd_orig.pop(f"{i}.f_t.weight") + block_two_sd_new[f"resnets.{i}.time_emb_proj.bias"] = block_two_sd_orig.pop(f"{i}.f_t.bias") + block_two_sd_new[f"resnets.{i}.norm2.weight"] = block_two_sd_orig.pop(f"{i}.gn_2.weight") + block_two_sd_new[f"resnets.{i}.norm2.bias"] = block_two_sd_orig.pop(f"{i}.gn_2.bias") + block_two_sd_new[f"resnets.{i}.conv2.weight"] = block_two_sd_orig.pop(f"{i}.f_2.weight") + block_two_sd_new[f"resnets.{i}.conv2.bias"] = block_two_sd_orig.pop(f"{i}.f_2.bias") + + if i == 0: + block_two_sd_new[f"resnets.{i}.conv_shortcut.weight"] = block_two_sd_orig.pop(f"{i}.f_s.weight") + block_two_sd_new[f"resnets.{i}.conv_shortcut.bias"] = block_two_sd_orig.pop(f"{i}.f_s.bias") + +block_two_sd_new["downsamplers.0.norm1.weight"] = block_two_sd_orig.pop("3.gn_1.weight") +block_two_sd_new["downsamplers.0.norm1.bias"] = block_two_sd_orig.pop("3.gn_1.bias") +block_two_sd_new["downsamplers.0.conv1.weight"] = block_two_sd_orig.pop("3.f_1.weight") +block_two_sd_new["downsamplers.0.conv1.bias"] = block_two_sd_orig.pop("3.f_1.bias") +block_two_sd_new["downsamplers.0.time_emb_proj.weight"] = block_two_sd_orig.pop("3.f_t.weight") +block_two_sd_new["downsamplers.0.time_emb_proj.bias"] = block_two_sd_orig.pop("3.f_t.bias") +block_two_sd_new["downsamplers.0.norm2.weight"] = block_two_sd_orig.pop("3.gn_2.weight") +block_two_sd_new["downsamplers.0.norm2.bias"] = block_two_sd_orig.pop("3.gn_2.bias") +block_two_sd_new["downsamplers.0.conv2.weight"] = block_two_sd_orig.pop("3.f_2.weight") +block_two_sd_new["downsamplers.0.conv2.bias"] = block_two_sd_orig.pop("3.f_2.bias") + +assert len(block_two_sd_orig) == 0 + +block_two = ResnetDownsampleBlock2D( + in_channels=320, + out_channels=640, + temb_channels=1280, + num_layers=3, + add_downsample=True, + resnet_time_scale_shift="scale_shift", + resnet_eps=1e-5, +) + +block_two.load_state_dict(block_two_sd_new) + +print("DOWN BLOCK THREE") + +block_three_sd_orig = model.down[2].state_dict() +block_three_sd_new = {} + +for i in range(3): + block_three_sd_new[f"resnets.{i}.norm1.weight"] = block_three_sd_orig.pop(f"{i}.gn_1.weight") + block_three_sd_new[f"resnets.{i}.norm1.bias"] = block_three_sd_orig.pop(f"{i}.gn_1.bias") + block_three_sd_new[f"resnets.{i}.conv1.weight"] = block_three_sd_orig.pop(f"{i}.f_1.weight") + block_three_sd_new[f"resnets.{i}.conv1.bias"] = block_three_sd_orig.pop(f"{i}.f_1.bias") + block_three_sd_new[f"resnets.{i}.time_emb_proj.weight"] = block_three_sd_orig.pop(f"{i}.f_t.weight") + block_three_sd_new[f"resnets.{i}.time_emb_proj.bias"] = block_three_sd_orig.pop(f"{i}.f_t.bias") + block_three_sd_new[f"resnets.{i}.norm2.weight"] = block_three_sd_orig.pop(f"{i}.gn_2.weight") + block_three_sd_new[f"resnets.{i}.norm2.bias"] = block_three_sd_orig.pop(f"{i}.gn_2.bias") + block_three_sd_new[f"resnets.{i}.conv2.weight"] = block_three_sd_orig.pop(f"{i}.f_2.weight") + block_three_sd_new[f"resnets.{i}.conv2.bias"] = block_three_sd_orig.pop(f"{i}.f_2.bias") + + if i == 0: + block_three_sd_new[f"resnets.{i}.conv_shortcut.weight"] = block_three_sd_orig.pop(f"{i}.f_s.weight") + block_three_sd_new[f"resnets.{i}.conv_shortcut.bias"] = block_three_sd_orig.pop(f"{i}.f_s.bias") + +block_three_sd_new["downsamplers.0.norm1.weight"] = block_three_sd_orig.pop("3.gn_1.weight") +block_three_sd_new["downsamplers.0.norm1.bias"] = block_three_sd_orig.pop("3.gn_1.bias") +block_three_sd_new["downsamplers.0.conv1.weight"] = block_three_sd_orig.pop("3.f_1.weight") +block_three_sd_new["downsamplers.0.conv1.bias"] = block_three_sd_orig.pop("3.f_1.bias") +block_three_sd_new["downsamplers.0.time_emb_proj.weight"] = block_three_sd_orig.pop("3.f_t.weight") +block_three_sd_new["downsamplers.0.time_emb_proj.bias"] = block_three_sd_orig.pop("3.f_t.bias") +block_three_sd_new["downsamplers.0.norm2.weight"] = block_three_sd_orig.pop("3.gn_2.weight") +block_three_sd_new["downsamplers.0.norm2.bias"] = block_three_sd_orig.pop("3.gn_2.bias") +block_three_sd_new["downsamplers.0.conv2.weight"] = block_three_sd_orig.pop("3.f_2.weight") +block_three_sd_new["downsamplers.0.conv2.bias"] = block_three_sd_orig.pop("3.f_2.bias") + +assert len(block_three_sd_orig) == 0 + +block_three = ResnetDownsampleBlock2D( + in_channels=640, + out_channels=1024, + temb_channels=1280, + num_layers=3, + add_downsample=True, + resnet_time_scale_shift="scale_shift", + resnet_eps=1e-5, +) + +block_three.load_state_dict(block_three_sd_new) + +print("DOWN BLOCK FOUR") + +block_four_sd_orig = model.down[3].state_dict() +block_four_sd_new = {} + +for i in range(3): + block_four_sd_new[f"resnets.{i}.norm1.weight"] = block_four_sd_orig.pop(f"{i}.gn_1.weight") + block_four_sd_new[f"resnets.{i}.norm1.bias"] = block_four_sd_orig.pop(f"{i}.gn_1.bias") + block_four_sd_new[f"resnets.{i}.conv1.weight"] = block_four_sd_orig.pop(f"{i}.f_1.weight") + block_four_sd_new[f"resnets.{i}.conv1.bias"] = block_four_sd_orig.pop(f"{i}.f_1.bias") + block_four_sd_new[f"resnets.{i}.time_emb_proj.weight"] = block_four_sd_orig.pop(f"{i}.f_t.weight") + block_four_sd_new[f"resnets.{i}.time_emb_proj.bias"] = block_four_sd_orig.pop(f"{i}.f_t.bias") + block_four_sd_new[f"resnets.{i}.norm2.weight"] = block_four_sd_orig.pop(f"{i}.gn_2.weight") + block_four_sd_new[f"resnets.{i}.norm2.bias"] = block_four_sd_orig.pop(f"{i}.gn_2.bias") + block_four_sd_new[f"resnets.{i}.conv2.weight"] = block_four_sd_orig.pop(f"{i}.f_2.weight") + block_four_sd_new[f"resnets.{i}.conv2.bias"] = block_four_sd_orig.pop(f"{i}.f_2.bias") + +assert len(block_four_sd_orig) == 0 + +block_four = ResnetDownsampleBlock2D( + in_channels=1024, + out_channels=1024, + temb_channels=1280, + num_layers=3, + add_downsample=False, + resnet_time_scale_shift="scale_shift", + resnet_eps=1e-5, +) + +block_four.load_state_dict(block_four_sd_new) + + +print("MID BLOCK 1") + +mid_block_one_sd_orig = model.mid.state_dict() +mid_block_one_sd_new = {} + +for i in range(2): + mid_block_one_sd_new[f"resnets.{i}.norm1.weight"] = mid_block_one_sd_orig.pop(f"{i}.gn_1.weight") + mid_block_one_sd_new[f"resnets.{i}.norm1.bias"] = mid_block_one_sd_orig.pop(f"{i}.gn_1.bias") + mid_block_one_sd_new[f"resnets.{i}.conv1.weight"] = mid_block_one_sd_orig.pop(f"{i}.f_1.weight") + mid_block_one_sd_new[f"resnets.{i}.conv1.bias"] = mid_block_one_sd_orig.pop(f"{i}.f_1.bias") + mid_block_one_sd_new[f"resnets.{i}.time_emb_proj.weight"] = mid_block_one_sd_orig.pop(f"{i}.f_t.weight") + mid_block_one_sd_new[f"resnets.{i}.time_emb_proj.bias"] = mid_block_one_sd_orig.pop(f"{i}.f_t.bias") + mid_block_one_sd_new[f"resnets.{i}.norm2.weight"] = mid_block_one_sd_orig.pop(f"{i}.gn_2.weight") + mid_block_one_sd_new[f"resnets.{i}.norm2.bias"] = mid_block_one_sd_orig.pop(f"{i}.gn_2.bias") + mid_block_one_sd_new[f"resnets.{i}.conv2.weight"] = mid_block_one_sd_orig.pop(f"{i}.f_2.weight") + mid_block_one_sd_new[f"resnets.{i}.conv2.bias"] = mid_block_one_sd_orig.pop(f"{i}.f_2.bias") + +assert len(mid_block_one_sd_orig) == 0 + +mid_block_one = UNetMidBlock2D( + in_channels=1024, + temb_channels=1280, + num_layers=1, + resnet_time_scale_shift="scale_shift", + resnet_eps=1e-5, + add_attention=False, +) + +mid_block_one.load_state_dict(mid_block_one_sd_new) + +print("UP BLOCK ONE") + +up_block_one_sd_orig = model.up[-1].state_dict() +up_block_one_sd_new = {} + +for i in range(4): + up_block_one_sd_new[f"resnets.{i}.norm1.weight"] = up_block_one_sd_orig.pop(f"{i}.gn_1.weight") + up_block_one_sd_new[f"resnets.{i}.norm1.bias"] = up_block_one_sd_orig.pop(f"{i}.gn_1.bias") + up_block_one_sd_new[f"resnets.{i}.conv1.weight"] = up_block_one_sd_orig.pop(f"{i}.f_1.weight") + up_block_one_sd_new[f"resnets.{i}.conv1.bias"] = up_block_one_sd_orig.pop(f"{i}.f_1.bias") + up_block_one_sd_new[f"resnets.{i}.time_emb_proj.weight"] = up_block_one_sd_orig.pop(f"{i}.f_t.weight") + up_block_one_sd_new[f"resnets.{i}.time_emb_proj.bias"] = up_block_one_sd_orig.pop(f"{i}.f_t.bias") + up_block_one_sd_new[f"resnets.{i}.norm2.weight"] = up_block_one_sd_orig.pop(f"{i}.gn_2.weight") + up_block_one_sd_new[f"resnets.{i}.norm2.bias"] = up_block_one_sd_orig.pop(f"{i}.gn_2.bias") + up_block_one_sd_new[f"resnets.{i}.conv2.weight"] = up_block_one_sd_orig.pop(f"{i}.f_2.weight") + up_block_one_sd_new[f"resnets.{i}.conv2.bias"] = up_block_one_sd_orig.pop(f"{i}.f_2.bias") + up_block_one_sd_new[f"resnets.{i}.conv_shortcut.weight"] = up_block_one_sd_orig.pop(f"{i}.f_s.weight") + up_block_one_sd_new[f"resnets.{i}.conv_shortcut.bias"] = up_block_one_sd_orig.pop(f"{i}.f_s.bias") + +up_block_one_sd_new["upsamplers.0.norm1.weight"] = up_block_one_sd_orig.pop("4.gn_1.weight") +up_block_one_sd_new["upsamplers.0.norm1.bias"] = up_block_one_sd_orig.pop("4.gn_1.bias") +up_block_one_sd_new["upsamplers.0.conv1.weight"] = up_block_one_sd_orig.pop("4.f_1.weight") +up_block_one_sd_new["upsamplers.0.conv1.bias"] = up_block_one_sd_orig.pop("4.f_1.bias") +up_block_one_sd_new["upsamplers.0.time_emb_proj.weight"] = up_block_one_sd_orig.pop("4.f_t.weight") +up_block_one_sd_new["upsamplers.0.time_emb_proj.bias"] = up_block_one_sd_orig.pop("4.f_t.bias") +up_block_one_sd_new["upsamplers.0.norm2.weight"] = up_block_one_sd_orig.pop("4.gn_2.weight") +up_block_one_sd_new["upsamplers.0.norm2.bias"] = up_block_one_sd_orig.pop("4.gn_2.bias") +up_block_one_sd_new["upsamplers.0.conv2.weight"] = up_block_one_sd_orig.pop("4.f_2.weight") +up_block_one_sd_new["upsamplers.0.conv2.bias"] = up_block_one_sd_orig.pop("4.f_2.bias") + +assert len(up_block_one_sd_orig) == 0 + +up_block_one = ResnetUpsampleBlock2D( + in_channels=1024, + prev_output_channel=1024, + out_channels=1024, + temb_channels=1280, + num_layers=4, + add_upsample=True, + resnet_time_scale_shift="scale_shift", + resnet_eps=1e-5, +) + +up_block_one.load_state_dict(up_block_one_sd_new) + +print("UP BLOCK TWO") + +up_block_two_sd_orig = model.up[-2].state_dict() +up_block_two_sd_new = {} + +for i in range(4): + up_block_two_sd_new[f"resnets.{i}.norm1.weight"] = up_block_two_sd_orig.pop(f"{i}.gn_1.weight") + up_block_two_sd_new[f"resnets.{i}.norm1.bias"] = up_block_two_sd_orig.pop(f"{i}.gn_1.bias") + up_block_two_sd_new[f"resnets.{i}.conv1.weight"] = up_block_two_sd_orig.pop(f"{i}.f_1.weight") + up_block_two_sd_new[f"resnets.{i}.conv1.bias"] = up_block_two_sd_orig.pop(f"{i}.f_1.bias") + up_block_two_sd_new[f"resnets.{i}.time_emb_proj.weight"] = up_block_two_sd_orig.pop(f"{i}.f_t.weight") + up_block_two_sd_new[f"resnets.{i}.time_emb_proj.bias"] = up_block_two_sd_orig.pop(f"{i}.f_t.bias") + up_block_two_sd_new[f"resnets.{i}.norm2.weight"] = up_block_two_sd_orig.pop(f"{i}.gn_2.weight") + up_block_two_sd_new[f"resnets.{i}.norm2.bias"] = up_block_two_sd_orig.pop(f"{i}.gn_2.bias") + up_block_two_sd_new[f"resnets.{i}.conv2.weight"] = up_block_two_sd_orig.pop(f"{i}.f_2.weight") + up_block_two_sd_new[f"resnets.{i}.conv2.bias"] = up_block_two_sd_orig.pop(f"{i}.f_2.bias") + up_block_two_sd_new[f"resnets.{i}.conv_shortcut.weight"] = up_block_two_sd_orig.pop(f"{i}.f_s.weight") + up_block_two_sd_new[f"resnets.{i}.conv_shortcut.bias"] = up_block_two_sd_orig.pop(f"{i}.f_s.bias") + +up_block_two_sd_new["upsamplers.0.norm1.weight"] = up_block_two_sd_orig.pop("4.gn_1.weight") +up_block_two_sd_new["upsamplers.0.norm1.bias"] = up_block_two_sd_orig.pop("4.gn_1.bias") +up_block_two_sd_new["upsamplers.0.conv1.weight"] = up_block_two_sd_orig.pop("4.f_1.weight") +up_block_two_sd_new["upsamplers.0.conv1.bias"] = up_block_two_sd_orig.pop("4.f_1.bias") +up_block_two_sd_new["upsamplers.0.time_emb_proj.weight"] = up_block_two_sd_orig.pop("4.f_t.weight") +up_block_two_sd_new["upsamplers.0.time_emb_proj.bias"] = up_block_two_sd_orig.pop("4.f_t.bias") +up_block_two_sd_new["upsamplers.0.norm2.weight"] = up_block_two_sd_orig.pop("4.gn_2.weight") +up_block_two_sd_new["upsamplers.0.norm2.bias"] = up_block_two_sd_orig.pop("4.gn_2.bias") +up_block_two_sd_new["upsamplers.0.conv2.weight"] = up_block_two_sd_orig.pop("4.f_2.weight") +up_block_two_sd_new["upsamplers.0.conv2.bias"] = up_block_two_sd_orig.pop("4.f_2.bias") + +assert len(up_block_two_sd_orig) == 0 + +up_block_two = ResnetUpsampleBlock2D( + in_channels=640, + prev_output_channel=1024, + out_channels=1024, + temb_channels=1280, + num_layers=4, + add_upsample=True, + resnet_time_scale_shift="scale_shift", + resnet_eps=1e-5, +) + +up_block_two.load_state_dict(up_block_two_sd_new) + +print("UP BLOCK THREE") + +up_block_three_sd_orig = model.up[-3].state_dict() +up_block_three_sd_new = {} + +for i in range(4): + up_block_three_sd_new[f"resnets.{i}.norm1.weight"] = up_block_three_sd_orig.pop(f"{i}.gn_1.weight") + up_block_three_sd_new[f"resnets.{i}.norm1.bias"] = up_block_three_sd_orig.pop(f"{i}.gn_1.bias") + up_block_three_sd_new[f"resnets.{i}.conv1.weight"] = up_block_three_sd_orig.pop(f"{i}.f_1.weight") + up_block_three_sd_new[f"resnets.{i}.conv1.bias"] = up_block_three_sd_orig.pop(f"{i}.f_1.bias") + up_block_three_sd_new[f"resnets.{i}.time_emb_proj.weight"] = up_block_three_sd_orig.pop(f"{i}.f_t.weight") + up_block_three_sd_new[f"resnets.{i}.time_emb_proj.bias"] = up_block_three_sd_orig.pop(f"{i}.f_t.bias") + up_block_three_sd_new[f"resnets.{i}.norm2.weight"] = up_block_three_sd_orig.pop(f"{i}.gn_2.weight") + up_block_three_sd_new[f"resnets.{i}.norm2.bias"] = up_block_three_sd_orig.pop(f"{i}.gn_2.bias") + up_block_three_sd_new[f"resnets.{i}.conv2.weight"] = up_block_three_sd_orig.pop(f"{i}.f_2.weight") + up_block_three_sd_new[f"resnets.{i}.conv2.bias"] = up_block_three_sd_orig.pop(f"{i}.f_2.bias") + up_block_three_sd_new[f"resnets.{i}.conv_shortcut.weight"] = up_block_three_sd_orig.pop(f"{i}.f_s.weight") + up_block_three_sd_new[f"resnets.{i}.conv_shortcut.bias"] = up_block_three_sd_orig.pop(f"{i}.f_s.bias") + +up_block_three_sd_new["upsamplers.0.norm1.weight"] = up_block_three_sd_orig.pop("4.gn_1.weight") +up_block_three_sd_new["upsamplers.0.norm1.bias"] = up_block_three_sd_orig.pop("4.gn_1.bias") +up_block_three_sd_new["upsamplers.0.conv1.weight"] = up_block_three_sd_orig.pop("4.f_1.weight") +up_block_three_sd_new["upsamplers.0.conv1.bias"] = up_block_three_sd_orig.pop("4.f_1.bias") +up_block_three_sd_new["upsamplers.0.time_emb_proj.weight"] = up_block_three_sd_orig.pop("4.f_t.weight") +up_block_three_sd_new["upsamplers.0.time_emb_proj.bias"] = up_block_three_sd_orig.pop("4.f_t.bias") +up_block_three_sd_new["upsamplers.0.norm2.weight"] = up_block_three_sd_orig.pop("4.gn_2.weight") +up_block_three_sd_new["upsamplers.0.norm2.bias"] = up_block_three_sd_orig.pop("4.gn_2.bias") +up_block_three_sd_new["upsamplers.0.conv2.weight"] = up_block_three_sd_orig.pop("4.f_2.weight") +up_block_three_sd_new["upsamplers.0.conv2.bias"] = up_block_three_sd_orig.pop("4.f_2.bias") + +assert len(up_block_three_sd_orig) == 0 + +up_block_three = ResnetUpsampleBlock2D( + in_channels=320, + prev_output_channel=1024, + out_channels=640, + temb_channels=1280, + num_layers=4, + add_upsample=True, + resnet_time_scale_shift="scale_shift", + resnet_eps=1e-5, +) + +up_block_three.load_state_dict(up_block_three_sd_new) + +print("UP BLOCK FOUR") + +up_block_four_sd_orig = model.up[-4].state_dict() +up_block_four_sd_new = {} + +for i in range(4): + up_block_four_sd_new[f"resnets.{i}.norm1.weight"] = up_block_four_sd_orig.pop(f"{i}.gn_1.weight") + up_block_four_sd_new[f"resnets.{i}.norm1.bias"] = up_block_four_sd_orig.pop(f"{i}.gn_1.bias") + up_block_four_sd_new[f"resnets.{i}.conv1.weight"] = up_block_four_sd_orig.pop(f"{i}.f_1.weight") + up_block_four_sd_new[f"resnets.{i}.conv1.bias"] = up_block_four_sd_orig.pop(f"{i}.f_1.bias") + up_block_four_sd_new[f"resnets.{i}.time_emb_proj.weight"] = up_block_four_sd_orig.pop(f"{i}.f_t.weight") + up_block_four_sd_new[f"resnets.{i}.time_emb_proj.bias"] = up_block_four_sd_orig.pop(f"{i}.f_t.bias") + up_block_four_sd_new[f"resnets.{i}.norm2.weight"] = up_block_four_sd_orig.pop(f"{i}.gn_2.weight") + up_block_four_sd_new[f"resnets.{i}.norm2.bias"] = up_block_four_sd_orig.pop(f"{i}.gn_2.bias") + up_block_four_sd_new[f"resnets.{i}.conv2.weight"] = up_block_four_sd_orig.pop(f"{i}.f_2.weight") + up_block_four_sd_new[f"resnets.{i}.conv2.bias"] = up_block_four_sd_orig.pop(f"{i}.f_2.bias") + up_block_four_sd_new[f"resnets.{i}.conv_shortcut.weight"] = up_block_four_sd_orig.pop(f"{i}.f_s.weight") + up_block_four_sd_new[f"resnets.{i}.conv_shortcut.bias"] = up_block_four_sd_orig.pop(f"{i}.f_s.bias") + +assert len(up_block_four_sd_orig) == 0 + +up_block_four = ResnetUpsampleBlock2D( + in_channels=320, + prev_output_channel=640, + out_channels=320, + temb_channels=1280, + num_layers=4, + add_upsample=False, + resnet_time_scale_shift="scale_shift", + resnet_eps=1e-5, +) + +up_block_four.load_state_dict(up_block_four_sd_new) + +print("initial projection (conv_in)") + +conv_in_sd_orig = model.embed_image.state_dict() +conv_in_sd_new = {} + +conv_in_sd_new["weight"] = conv_in_sd_orig.pop("f.weight") +conv_in_sd_new["bias"] = conv_in_sd_orig.pop("f.bias") + +assert len(conv_in_sd_orig) == 0 + +block_out_channels = [320, 640, 1024, 1024] + +in_channels = 7 +conv_in_kernel = 3 +conv_in_padding = (conv_in_kernel - 1) // 2 +conv_in = nn.Conv2d(in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding) + +conv_in.load_state_dict(conv_in_sd_new) + +print("out projection (conv_out) (conv_norm_out)") +out_channels = 6 +norm_num_groups = 32 +norm_eps = 1e-5 +act_fn = "silu" +conv_out_kernel = 3 +conv_out_padding = (conv_out_kernel - 1) // 2 +conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps) +# uses torch.functional in orig +# conv_act = get_activation(act_fn) +conv_out = nn.Conv2d(block_out_channels[0], out_channels, kernel_size=conv_out_kernel, padding=conv_out_padding) + +conv_norm_out.load_state_dict(model.output.gn.state_dict()) +conv_out.load_state_dict(model.output.f.state_dict()) + +print("timestep projection (time_proj) (time_embedding)") + +f1_sd = model.embed_time.f_1.state_dict() +f2_sd = model.embed_time.f_2.state_dict() + +time_embedding_sd = { + "linear_1.weight": f1_sd.pop("weight"), + "linear_1.bias": f1_sd.pop("bias"), + "linear_2.weight": f2_sd.pop("weight"), + "linear_2.bias": f2_sd.pop("bias"), +} + +assert len(f1_sd) == 0 +assert len(f2_sd) == 0 + +time_embedding_type = "learned" +num_train_timesteps = 1024 +time_embedding_dim = 1280 + +time_proj = nn.Embedding(num_train_timesteps, block_out_channels[0]) +timestep_input_dim = block_out_channels[0] + +time_embedding = TimestepEmbedding(timestep_input_dim, time_embedding_dim) + +time_proj.load_state_dict(model.embed_time.emb.state_dict()) +time_embedding.load_state_dict(time_embedding_sd) + +print("CONVERT") + +time_embedding.to("cuda") +time_proj.to("cuda") +conv_in.to("cuda") + +block_one.to("cuda") +block_two.to("cuda") +block_three.to("cuda") +block_four.to("cuda") + +mid_block_one.to("cuda") + +up_block_one.to("cuda") +up_block_two.to("cuda") +up_block_three.to("cuda") +up_block_four.to("cuda") + +conv_norm_out.to("cuda") +conv_out.to("cuda") + +model.time_proj = time_proj +model.time_embedding = time_embedding +model.embed_image = conv_in + +model.down[0] = block_one +model.down[1] = block_two +model.down[2] = block_three +model.down[3] = block_four + +model.mid = mid_block_one + +model.up[-1] = up_block_one +model.up[-2] = up_block_two +model.up[-3] = up_block_three +model.up[-4] = up_block_four + +model.output.gn = conv_norm_out +model.output.f = conv_out + +model.converted = True + +sample_consistency_new = decoder_consistency(latent, generator=torch.Generator("cpu").manual_seed(0)) +save_image(sample_consistency_new, "con_new.png") + +assert (sample_consistency_orig == sample_consistency_new).all() + +print("making unet") + +unet = UNet2DModel( + in_channels=in_channels, + out_channels=out_channels, + down_block_types=( + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + ), + up_block_types=( + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + ), + block_out_channels=block_out_channels, + layers_per_block=3, + norm_num_groups=norm_num_groups, + norm_eps=norm_eps, + resnet_time_scale_shift="scale_shift", + time_embedding_type="learned", + num_train_timesteps=num_train_timesteps, + add_attention=False, +) + +unet_state_dict = {} + + +def add_state_dict(prefix, mod): + for k, v in mod.state_dict().items(): + unet_state_dict[f"{prefix}.{k}"] = v + + +add_state_dict("conv_in", conv_in) +add_state_dict("time_proj", time_proj) +add_state_dict("time_embedding", time_embedding) +add_state_dict("down_blocks.0", block_one) +add_state_dict("down_blocks.1", block_two) +add_state_dict("down_blocks.2", block_three) +add_state_dict("down_blocks.3", block_four) +add_state_dict("mid_block", mid_block_one) +add_state_dict("up_blocks.0", up_block_one) +add_state_dict("up_blocks.1", up_block_two) +add_state_dict("up_blocks.2", up_block_three) +add_state_dict("up_blocks.3", up_block_four) +add_state_dict("conv_norm_out", conv_norm_out) +add_state_dict("conv_out", conv_out) + +unet.load_state_dict(unet_state_dict) + +print("running with diffusers unet") + +unet.to("cuda") + +decoder_consistency.ckpt = unet + +sample_consistency_new_2 = decoder_consistency(latent, generator=torch.Generator("cpu").manual_seed(0)) +save_image(sample_consistency_new_2, "con_new_2.png") + +assert (sample_consistency_orig == sample_consistency_new_2).all() + +print("running with diffusers model") + +Encoder.old_constructor = Encoder.__init__ + + +def new_constructor(self, **kwargs): + self.old_constructor(**kwargs) + self.constructor_arguments = kwargs + + +Encoder.__init__ = new_constructor + + +vae = AutoencoderKL.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="vae") +consistency_vae = ConsistencyDecoderVAE( + encoder_args=vae.encoder.constructor_arguments, + decoder_args=unet.config, + scaling_factor=vae.config.scaling_factor, + block_out_channels=vae.config.block_out_channels, + latent_channels=vae.config.latent_channels, +) +consistency_vae.encoder.load_state_dict(vae.encoder.state_dict()) +consistency_vae.quant_conv.load_state_dict(vae.quant_conv.state_dict()) +consistency_vae.decoder_unet.load_state_dict(unet.state_dict()) + +consistency_vae.to(dtype=torch.float16, device="cuda") + +sample_consistency_new_3 = consistency_vae.decode( + 0.18215 * latent, generator=torch.Generator("cpu").manual_seed(0) +).sample + +print("max difference") +print((sample_consistency_orig - sample_consistency_new_3).abs().max()) +print("total difference") +print((sample_consistency_orig - sample_consistency_new_3).abs().sum()) +# assert (sample_consistency_orig == sample_consistency_new_3).all() + +print("running with diffusers pipeline") + +pipe = DiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", vae=consistency_vae, torch_dtype=torch.float16 +) +pipe.to("cuda") + +pipe("horse", generator=torch.Generator("cpu").manual_seed(0)).images[0].save("horse.png") + + +if args.save_pretrained is not None: + consistency_vae.save_pretrained(args.save_pretrained) diff --git a/diffusers-0.27.0/scripts/convert_consistency_to_diffusers.py b/diffusers-0.27.0/scripts/convert_consistency_to_diffusers.py new file mode 100755 index 0000000..0f8b4dd --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_consistency_to_diffusers.py @@ -0,0 +1,315 @@ +import argparse +import os + +import torch + +from diffusers import ( + CMStochasticIterativeScheduler, + ConsistencyModelPipeline, + UNet2DModel, +) + + +TEST_UNET_CONFIG = { + "sample_size": 32, + "in_channels": 3, + "out_channels": 3, + "layers_per_block": 2, + "num_class_embeds": 1000, + "block_out_channels": [32, 64], + "attention_head_dim": 8, + "down_block_types": [ + "ResnetDownsampleBlock2D", + "AttnDownBlock2D", + ], + "up_block_types": [ + "AttnUpBlock2D", + "ResnetUpsampleBlock2D", + ], + "resnet_time_scale_shift": "scale_shift", + "attn_norm_num_groups": 32, + "upsample_type": "resnet", + "downsample_type": "resnet", +} + +IMAGENET_64_UNET_CONFIG = { + "sample_size": 64, + "in_channels": 3, + "out_channels": 3, + "layers_per_block": 3, + "num_class_embeds": 1000, + "block_out_channels": [192, 192 * 2, 192 * 3, 192 * 4], + "attention_head_dim": 64, + "down_block_types": [ + "ResnetDownsampleBlock2D", + "AttnDownBlock2D", + "AttnDownBlock2D", + "AttnDownBlock2D", + ], + "up_block_types": [ + "AttnUpBlock2D", + "AttnUpBlock2D", + "AttnUpBlock2D", + "ResnetUpsampleBlock2D", + ], + "resnet_time_scale_shift": "scale_shift", + "attn_norm_num_groups": 32, + "upsample_type": "resnet", + "downsample_type": "resnet", +} + +LSUN_256_UNET_CONFIG = { + "sample_size": 256, + "in_channels": 3, + "out_channels": 3, + "layers_per_block": 2, + "num_class_embeds": None, + "block_out_channels": [256, 256, 256 * 2, 256 * 2, 256 * 4, 256 * 4], + "attention_head_dim": 64, + "down_block_types": [ + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + "AttnDownBlock2D", + "AttnDownBlock2D", + "AttnDownBlock2D", + ], + "up_block_types": [ + "AttnUpBlock2D", + "AttnUpBlock2D", + "AttnUpBlock2D", + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + ], + "resnet_time_scale_shift": "default", + "upsample_type": "resnet", + "downsample_type": "resnet", +} + +CD_SCHEDULER_CONFIG = { + "num_train_timesteps": 40, + "sigma_min": 0.002, + "sigma_max": 80.0, +} + +CT_IMAGENET_64_SCHEDULER_CONFIG = { + "num_train_timesteps": 201, + "sigma_min": 0.002, + "sigma_max": 80.0, +} + +CT_LSUN_256_SCHEDULER_CONFIG = { + "num_train_timesteps": 151, + "sigma_min": 0.002, + "sigma_max": 80.0, +} + + +def str2bool(v): + """ + https://stackoverflow.com/questions/15008758/parsing-boolean-values-with-argparse + """ + if isinstance(v, bool): + return v + if v.lower() in ("yes", "true", "t", "y", "1"): + return True + elif v.lower() in ("no", "false", "f", "n", "0"): + return False + else: + raise argparse.ArgumentTypeError("boolean value expected") + + +def convert_resnet(checkpoint, new_checkpoint, old_prefix, new_prefix, has_skip=False): + new_checkpoint[f"{new_prefix}.norm1.weight"] = checkpoint[f"{old_prefix}.in_layers.0.weight"] + new_checkpoint[f"{new_prefix}.norm1.bias"] = checkpoint[f"{old_prefix}.in_layers.0.bias"] + new_checkpoint[f"{new_prefix}.conv1.weight"] = checkpoint[f"{old_prefix}.in_layers.2.weight"] + new_checkpoint[f"{new_prefix}.conv1.bias"] = checkpoint[f"{old_prefix}.in_layers.2.bias"] + new_checkpoint[f"{new_prefix}.time_emb_proj.weight"] = checkpoint[f"{old_prefix}.emb_layers.1.weight"] + new_checkpoint[f"{new_prefix}.time_emb_proj.bias"] = checkpoint[f"{old_prefix}.emb_layers.1.bias"] + new_checkpoint[f"{new_prefix}.norm2.weight"] = checkpoint[f"{old_prefix}.out_layers.0.weight"] + new_checkpoint[f"{new_prefix}.norm2.bias"] = checkpoint[f"{old_prefix}.out_layers.0.bias"] + new_checkpoint[f"{new_prefix}.conv2.weight"] = checkpoint[f"{old_prefix}.out_layers.3.weight"] + new_checkpoint[f"{new_prefix}.conv2.bias"] = checkpoint[f"{old_prefix}.out_layers.3.bias"] + + if has_skip: + new_checkpoint[f"{new_prefix}.conv_shortcut.weight"] = checkpoint[f"{old_prefix}.skip_connection.weight"] + new_checkpoint[f"{new_prefix}.conv_shortcut.bias"] = checkpoint[f"{old_prefix}.skip_connection.bias"] + + return new_checkpoint + + +def convert_attention(checkpoint, new_checkpoint, old_prefix, new_prefix, attention_dim=None): + weight_q, weight_k, weight_v = checkpoint[f"{old_prefix}.qkv.weight"].chunk(3, dim=0) + bias_q, bias_k, bias_v = checkpoint[f"{old_prefix}.qkv.bias"].chunk(3, dim=0) + + new_checkpoint[f"{new_prefix}.group_norm.weight"] = checkpoint[f"{old_prefix}.norm.weight"] + new_checkpoint[f"{new_prefix}.group_norm.bias"] = checkpoint[f"{old_prefix}.norm.bias"] + + new_checkpoint[f"{new_prefix}.to_q.weight"] = weight_q.squeeze(-1).squeeze(-1) + new_checkpoint[f"{new_prefix}.to_q.bias"] = bias_q.squeeze(-1).squeeze(-1) + new_checkpoint[f"{new_prefix}.to_k.weight"] = weight_k.squeeze(-1).squeeze(-1) + new_checkpoint[f"{new_prefix}.to_k.bias"] = bias_k.squeeze(-1).squeeze(-1) + new_checkpoint[f"{new_prefix}.to_v.weight"] = weight_v.squeeze(-1).squeeze(-1) + new_checkpoint[f"{new_prefix}.to_v.bias"] = bias_v.squeeze(-1).squeeze(-1) + + new_checkpoint[f"{new_prefix}.to_out.0.weight"] = ( + checkpoint[f"{old_prefix}.proj_out.weight"].squeeze(-1).squeeze(-1) + ) + new_checkpoint[f"{new_prefix}.to_out.0.bias"] = checkpoint[f"{old_prefix}.proj_out.bias"].squeeze(-1).squeeze(-1) + + return new_checkpoint + + +def con_pt_to_diffuser(checkpoint_path: str, unet_config): + checkpoint = torch.load(checkpoint_path, map_location="cpu") + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = checkpoint["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = checkpoint["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = checkpoint["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = checkpoint["time_embed.2.bias"] + + if unet_config["num_class_embeds"] is not None: + new_checkpoint["class_embedding.weight"] = checkpoint["label_emb.weight"] + + new_checkpoint["conv_in.weight"] = checkpoint["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = checkpoint["input_blocks.0.0.bias"] + + down_block_types = unet_config["down_block_types"] + layers_per_block = unet_config["layers_per_block"] + attention_head_dim = unet_config["attention_head_dim"] + channels_list = unet_config["block_out_channels"] + current_layer = 1 + prev_channels = channels_list[0] + + for i, layer_type in enumerate(down_block_types): + current_channels = channels_list[i] + downsample_block_has_skip = current_channels != prev_channels + if layer_type == "ResnetDownsampleBlock2D": + for j in range(layers_per_block): + new_prefix = f"down_blocks.{i}.resnets.{j}" + old_prefix = f"input_blocks.{current_layer}.0" + has_skip = True if j == 0 and downsample_block_has_skip else False + new_checkpoint = convert_resnet(checkpoint, new_checkpoint, old_prefix, new_prefix, has_skip=has_skip) + current_layer += 1 + + elif layer_type == "AttnDownBlock2D": + for j in range(layers_per_block): + new_prefix = f"down_blocks.{i}.resnets.{j}" + old_prefix = f"input_blocks.{current_layer}.0" + has_skip = True if j == 0 and downsample_block_has_skip else False + new_checkpoint = convert_resnet(checkpoint, new_checkpoint, old_prefix, new_prefix, has_skip=has_skip) + new_prefix = f"down_blocks.{i}.attentions.{j}" + old_prefix = f"input_blocks.{current_layer}.1" + new_checkpoint = convert_attention( + checkpoint, new_checkpoint, old_prefix, new_prefix, attention_head_dim + ) + current_layer += 1 + + if i != len(down_block_types) - 1: + new_prefix = f"down_blocks.{i}.downsamplers.0" + old_prefix = f"input_blocks.{current_layer}.0" + new_checkpoint = convert_resnet(checkpoint, new_checkpoint, old_prefix, new_prefix) + current_layer += 1 + + prev_channels = current_channels + + # hardcoded the mid-block for now + new_prefix = "mid_block.resnets.0" + old_prefix = "middle_block.0" + new_checkpoint = convert_resnet(checkpoint, new_checkpoint, old_prefix, new_prefix) + new_prefix = "mid_block.attentions.0" + old_prefix = "middle_block.1" + new_checkpoint = convert_attention(checkpoint, new_checkpoint, old_prefix, new_prefix, attention_head_dim) + new_prefix = "mid_block.resnets.1" + old_prefix = "middle_block.2" + new_checkpoint = convert_resnet(checkpoint, new_checkpoint, old_prefix, new_prefix) + + current_layer = 0 + up_block_types = unet_config["up_block_types"] + + for i, layer_type in enumerate(up_block_types): + if layer_type == "ResnetUpsampleBlock2D": + for j in range(layers_per_block + 1): + new_prefix = f"up_blocks.{i}.resnets.{j}" + old_prefix = f"output_blocks.{current_layer}.0" + new_checkpoint = convert_resnet(checkpoint, new_checkpoint, old_prefix, new_prefix, has_skip=True) + current_layer += 1 + + if i != len(up_block_types) - 1: + new_prefix = f"up_blocks.{i}.upsamplers.0" + old_prefix = f"output_blocks.{current_layer-1}.1" + new_checkpoint = convert_resnet(checkpoint, new_checkpoint, old_prefix, new_prefix) + elif layer_type == "AttnUpBlock2D": + for j in range(layers_per_block + 1): + new_prefix = f"up_blocks.{i}.resnets.{j}" + old_prefix = f"output_blocks.{current_layer}.0" + new_checkpoint = convert_resnet(checkpoint, new_checkpoint, old_prefix, new_prefix, has_skip=True) + new_prefix = f"up_blocks.{i}.attentions.{j}" + old_prefix = f"output_blocks.{current_layer}.1" + new_checkpoint = convert_attention( + checkpoint, new_checkpoint, old_prefix, new_prefix, attention_head_dim + ) + current_layer += 1 + + if i != len(up_block_types) - 1: + new_prefix = f"up_blocks.{i}.upsamplers.0" + old_prefix = f"output_blocks.{current_layer-1}.2" + new_checkpoint = convert_resnet(checkpoint, new_checkpoint, old_prefix, new_prefix) + + new_checkpoint["conv_norm_out.weight"] = checkpoint["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = checkpoint["out.0.bias"] + new_checkpoint["conv_out.weight"] = checkpoint["out.2.weight"] + new_checkpoint["conv_out.bias"] = checkpoint["out.2.bias"] + + return new_checkpoint + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--unet_path", default=None, type=str, required=True, help="Path to the unet.pt to convert.") + parser.add_argument( + "--dump_path", default=None, type=str, required=True, help="Path to output the converted UNet model." + ) + parser.add_argument("--class_cond", default=True, type=str, help="Whether the model is class-conditional.") + + args = parser.parse_args() + args.class_cond = str2bool(args.class_cond) + + ckpt_name = os.path.basename(args.unet_path) + print(f"Checkpoint: {ckpt_name}") + + # Get U-Net config + if "imagenet64" in ckpt_name: + unet_config = IMAGENET_64_UNET_CONFIG + elif "256" in ckpt_name and (("bedroom" in ckpt_name) or ("cat" in ckpt_name)): + unet_config = LSUN_256_UNET_CONFIG + elif "test" in ckpt_name: + unet_config = TEST_UNET_CONFIG + else: + raise ValueError(f"Checkpoint type {ckpt_name} is not currently supported.") + + if not args.class_cond: + unet_config["num_class_embeds"] = None + + converted_unet_ckpt = con_pt_to_diffuser(args.unet_path, unet_config) + + image_unet = UNet2DModel(**unet_config) + image_unet.load_state_dict(converted_unet_ckpt) + + # Get scheduler config + if "cd" in ckpt_name or "test" in ckpt_name: + scheduler_config = CD_SCHEDULER_CONFIG + elif "ct" in ckpt_name and "imagenet64" in ckpt_name: + scheduler_config = CT_IMAGENET_64_SCHEDULER_CONFIG + elif "ct" in ckpt_name and "256" in ckpt_name and (("bedroom" in ckpt_name) or ("cat" in ckpt_name)): + scheduler_config = CT_LSUN_256_SCHEDULER_CONFIG + else: + raise ValueError(f"Checkpoint type {ckpt_name} is not currently supported.") + + cm_scheduler = CMStochasticIterativeScheduler(**scheduler_config) + + consistency_model = ConsistencyModelPipeline(unet=image_unet, scheduler=cm_scheduler) + consistency_model.save_pretrained(args.dump_path) diff --git a/diffusers-0.27.0/scripts/convert_dance_diffusion_to_diffusers.py b/diffusers-0.27.0/scripts/convert_dance_diffusion_to_diffusers.py new file mode 100755 index 0000000..ce69bfe --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_dance_diffusion_to_diffusers.py @@ -0,0 +1,345 @@ +#!/usr/bin/env python3 +import argparse +import math +import os +from copy import deepcopy + +import requests +import torch +from audio_diffusion.models import DiffusionAttnUnet1D +from diffusion import sampling +from torch import nn + +from diffusers import DanceDiffusionPipeline, IPNDMScheduler, UNet1DModel + + +MODELS_MAP = { + "gwf-440k": { + "url": "https://model-server.zqevans2.workers.dev/gwf-440k.ckpt", + "sample_rate": 48000, + "sample_size": 65536, + }, + "jmann-small-190k": { + "url": "https://model-server.zqevans2.workers.dev/jmann-small-190k.ckpt", + "sample_rate": 48000, + "sample_size": 65536, + }, + "jmann-large-580k": { + "url": "https://model-server.zqevans2.workers.dev/jmann-large-580k.ckpt", + "sample_rate": 48000, + "sample_size": 131072, + }, + "maestro-uncond-150k": { + "url": "https://model-server.zqevans2.workers.dev/maestro-uncond-150k.ckpt", + "sample_rate": 16000, + "sample_size": 65536, + }, + "unlocked-uncond-250k": { + "url": "https://model-server.zqevans2.workers.dev/unlocked-uncond-250k.ckpt", + "sample_rate": 16000, + "sample_size": 65536, + }, + "honk-140k": { + "url": "https://model-server.zqevans2.workers.dev/honk-140k.ckpt", + "sample_rate": 16000, + "sample_size": 65536, + }, +} + + +def alpha_sigma_to_t(alpha, sigma): + """Returns a timestep, given the scaling factors for the clean image and for + the noise.""" + return torch.atan2(sigma, alpha) / math.pi * 2 + + +def get_crash_schedule(t): + sigma = torch.sin(t * math.pi / 2) ** 2 + alpha = (1 - sigma**2) ** 0.5 + return alpha_sigma_to_t(alpha, sigma) + + +class Object(object): + pass + + +class DiffusionUncond(nn.Module): + def __init__(self, global_args): + super().__init__() + + self.diffusion = DiffusionAttnUnet1D(global_args, n_attn_layers=4) + self.diffusion_ema = deepcopy(self.diffusion) + self.rng = torch.quasirandom.SobolEngine(1, scramble=True) + + +def download(model_name): + url = MODELS_MAP[model_name]["url"] + r = requests.get(url, stream=True) + + local_filename = f"./{model_name}.ckpt" + with open(local_filename, "wb") as fp: + for chunk in r.iter_content(chunk_size=8192): + fp.write(chunk) + + return local_filename + + +DOWN_NUM_TO_LAYER = { + "1": "resnets.0", + "2": "attentions.0", + "3": "resnets.1", + "4": "attentions.1", + "5": "resnets.2", + "6": "attentions.2", +} +UP_NUM_TO_LAYER = { + "8": "resnets.0", + "9": "attentions.0", + "10": "resnets.1", + "11": "attentions.1", + "12": "resnets.2", + "13": "attentions.2", +} +MID_NUM_TO_LAYER = { + "1": "resnets.0", + "2": "attentions.0", + "3": "resnets.1", + "4": "attentions.1", + "5": "resnets.2", + "6": "attentions.2", + "8": "resnets.3", + "9": "attentions.3", + "10": "resnets.4", + "11": "attentions.4", + "12": "resnets.5", + "13": "attentions.5", +} +DEPTH_0_TO_LAYER = { + "0": "resnets.0", + "1": "resnets.1", + "2": "resnets.2", + "4": "resnets.0", + "5": "resnets.1", + "6": "resnets.2", +} + +RES_CONV_MAP = { + "skip": "conv_skip", + "main.0": "conv_1", + "main.1": "group_norm_1", + "main.3": "conv_2", + "main.4": "group_norm_2", +} + +ATTN_MAP = { + "norm": "group_norm", + "qkv_proj": ["query", "key", "value"], + "out_proj": ["proj_attn"], +} + + +def convert_resconv_naming(name): + if name.startswith("skip"): + return name.replace("skip", RES_CONV_MAP["skip"]) + + # name has to be of format main.{digit} + if not name.startswith("main."): + raise ValueError(f"ResConvBlock error with {name}") + + return name.replace(name[:6], RES_CONV_MAP[name[:6]]) + + +def convert_attn_naming(name): + for key, value in ATTN_MAP.items(): + if name.startswith(key) and not isinstance(value, list): + return name.replace(key, value) + elif name.startswith(key): + return [name.replace(key, v) for v in value] + raise ValueError(f"Attn error with {name}") + + +def rename(input_string, max_depth=13): + string = input_string + + if string.split(".")[0] == "timestep_embed": + return string.replace("timestep_embed", "time_proj") + + depth = 0 + if string.startswith("net.3."): + depth += 1 + string = string[6:] + elif string.startswith("net."): + string = string[4:] + + while string.startswith("main.7."): + depth += 1 + string = string[7:] + + if string.startswith("main."): + string = string[5:] + + # mid block + if string[:2].isdigit(): + layer_num = string[:2] + string_left = string[2:] + else: + layer_num = string[0] + string_left = string[1:] + + if depth == max_depth: + new_layer = MID_NUM_TO_LAYER[layer_num] + prefix = "mid_block" + elif depth > 0 and int(layer_num) < 7: + new_layer = DOWN_NUM_TO_LAYER[layer_num] + prefix = f"down_blocks.{depth}" + elif depth > 0 and int(layer_num) > 7: + new_layer = UP_NUM_TO_LAYER[layer_num] + prefix = f"up_blocks.{max_depth - depth - 1}" + elif depth == 0: + new_layer = DEPTH_0_TO_LAYER[layer_num] + prefix = f"up_blocks.{max_depth - 1}" if int(layer_num) > 3 else "down_blocks.0" + + if not string_left.startswith("."): + raise ValueError(f"Naming error with {input_string} and string_left: {string_left}.") + + string_left = string_left[1:] + + if "resnets" in new_layer: + string_left = convert_resconv_naming(string_left) + elif "attentions" in new_layer: + new_string_left = convert_attn_naming(string_left) + string_left = new_string_left + + if not isinstance(string_left, list): + new_string = prefix + "." + new_layer + "." + string_left + else: + new_string = [prefix + "." + new_layer + "." + s for s in string_left] + return new_string + + +def rename_orig_weights(state_dict): + new_state_dict = {} + for k, v in state_dict.items(): + if k.endswith("kernel"): + # up- and downsample layers, don't have trainable weights + continue + + new_k = rename(k) + + # check if we need to transform from Conv => Linear for attention + if isinstance(new_k, list): + new_state_dict = transform_conv_attns(new_state_dict, new_k, v) + else: + new_state_dict[new_k] = v + + return new_state_dict + + +def transform_conv_attns(new_state_dict, new_k, v): + if len(new_k) == 1: + if len(v.shape) == 3: + # weight + new_state_dict[new_k[0]] = v[:, :, 0] + else: + # bias + new_state_dict[new_k[0]] = v + else: + # qkv matrices + trippled_shape = v.shape[0] + single_shape = trippled_shape // 3 + for i in range(3): + if len(v.shape) == 3: + new_state_dict[new_k[i]] = v[i * single_shape : (i + 1) * single_shape, :, 0] + else: + new_state_dict[new_k[i]] = v[i * single_shape : (i + 1) * single_shape] + return new_state_dict + + +def main(args): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + model_name = args.model_path.split("/")[-1].split(".")[0] + if not os.path.isfile(args.model_path): + assert ( + model_name == args.model_path + ), f"Make sure to provide one of the official model names {MODELS_MAP.keys()}" + args.model_path = download(model_name) + + sample_rate = MODELS_MAP[model_name]["sample_rate"] + sample_size = MODELS_MAP[model_name]["sample_size"] + + config = Object() + config.sample_size = sample_size + config.sample_rate = sample_rate + config.latent_dim = 0 + + diffusers_model = UNet1DModel(sample_size=sample_size, sample_rate=sample_rate) + diffusers_state_dict = diffusers_model.state_dict() + + orig_model = DiffusionUncond(config) + orig_model.load_state_dict(torch.load(args.model_path, map_location=device)["state_dict"]) + orig_model = orig_model.diffusion_ema.eval() + orig_model_state_dict = orig_model.state_dict() + renamed_state_dict = rename_orig_weights(orig_model_state_dict) + + renamed_minus_diffusers = set(renamed_state_dict.keys()) - set(diffusers_state_dict.keys()) + diffusers_minus_renamed = set(diffusers_state_dict.keys()) - set(renamed_state_dict.keys()) + + assert len(renamed_minus_diffusers) == 0, f"Problem with {renamed_minus_diffusers}" + assert all(k.endswith("kernel") for k in list(diffusers_minus_renamed)), f"Problem with {diffusers_minus_renamed}" + + for key, value in renamed_state_dict.items(): + assert ( + diffusers_state_dict[key].squeeze().shape == value.squeeze().shape + ), f"Shape for {key} doesn't match. Diffusers: {diffusers_state_dict[key].shape} vs. {value.shape}" + if key == "time_proj.weight": + value = value.squeeze() + + diffusers_state_dict[key] = value + + diffusers_model.load_state_dict(diffusers_state_dict) + + steps = 100 + seed = 33 + + diffusers_scheduler = IPNDMScheduler(num_train_timesteps=steps) + + generator = torch.manual_seed(seed) + noise = torch.randn([1, 2, config.sample_size], generator=generator).to(device) + + t = torch.linspace(1, 0, steps + 1, device=device)[:-1] + step_list = get_crash_schedule(t) + + pipe = DanceDiffusionPipeline(unet=diffusers_model, scheduler=diffusers_scheduler) + + generator = torch.manual_seed(33) + audio = pipe(num_inference_steps=steps, generator=generator).audios + + generated = sampling.iplms_sample(orig_model, noise, step_list, {}) + generated = generated.clamp(-1, 1) + + diff_sum = (generated - audio).abs().sum() + diff_max = (generated - audio).abs().max() + + if args.save: + pipe.save_pretrained(args.checkpoint_path) + + print("Diff sum", diff_sum) + print("Diff max", diff_max) + + assert diff_max < 1e-3, f"Diff max: {diff_max} is too much :-/" + + print(f"Conversion for {model_name} successful!") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", default=None, type=str, required=True, help="Path to the model to convert.") + parser.add_argument( + "--save", default=True, type=bool, required=False, help="Whether to save the converted model or not." + ) + parser.add_argument("--checkpoint_path", default=None, type=str, required=True, help="Path to the output model.") + args = parser.parse_args() + + main(args) diff --git a/diffusers-0.27.0/scripts/convert_ddpm_original_checkpoint_to_diffusers.py b/diffusers-0.27.0/scripts/convert_ddpm_original_checkpoint_to_diffusers.py new file mode 100755 index 0000000..4659578 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_ddpm_original_checkpoint_to_diffusers.py @@ -0,0 +1,431 @@ +import argparse +import json + +import torch + +from diffusers import AutoencoderKL, DDPMPipeline, DDPMScheduler, UNet2DModel, VQModel + + +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + mapping = [] + for old_item in old_list: + new_item = old_item + new_item = new_item.replace("block.", "resnets.") + new_item = new_item.replace("conv_shorcut", "conv1") + new_item = new_item.replace("in_shortcut", "conv_shortcut") + new_item = new_item.replace("temb_proj", "time_emb_proj") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_attention_paths(old_list, n_shave_prefix_segments=0, in_mid=False): + mapping = [] + for old_item in old_list: + new_item = old_item + + # In `model.mid`, the layer is called `attn`. + if not in_mid: + new_item = new_item.replace("attn", "attentions") + new_item = new_item.replace(".k.", ".key.") + new_item = new_item.replace(".v.", ".value.") + new_item = new_item.replace(".q.", ".query.") + + new_item = new_item.replace("proj_out", "proj_attn") + new_item = new_item.replace("norm", "group_norm") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def assign_to_checkpoint( + paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None +): + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + if attention_paths_to_split is not None: + if config is None: + raise ValueError("Please specify the config if setting 'attention_paths_to_split' to 'True'.") + + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // config.get("num_head_channels", 1) // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape).squeeze() + checkpoint[path_map["key"]] = key.reshape(target_shape).squeeze() + checkpoint[path_map["value"]] = value.reshape(target_shape).squeeze() + + for path in paths: + new_path = path["new"] + + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + new_path = new_path.replace("down.", "down_blocks.") + new_path = new_path.replace("up.", "up_blocks.") + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + if "attentions" in new_path: + checkpoint[new_path] = old_checkpoint[path["old"]].squeeze() + else: + checkpoint[new_path] = old_checkpoint[path["old"]] + + +def convert_ddpm_checkpoint(checkpoint, config): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = checkpoint["temb.dense.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = checkpoint["temb.dense.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = checkpoint["temb.dense.1.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = checkpoint["temb.dense.1.bias"] + + new_checkpoint["conv_norm_out.weight"] = checkpoint["norm_out.weight"] + new_checkpoint["conv_norm_out.bias"] = checkpoint["norm_out.bias"] + + new_checkpoint["conv_in.weight"] = checkpoint["conv_in.weight"] + new_checkpoint["conv_in.bias"] = checkpoint["conv_in.bias"] + new_checkpoint["conv_out.weight"] = checkpoint["conv_out.weight"] + new_checkpoint["conv_out.bias"] = checkpoint["conv_out.bias"] + + num_down_blocks = len({".".join(layer.split(".")[:2]) for layer in checkpoint if "down" in layer}) + down_blocks = { + layer_id: [key for key in checkpoint if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + num_up_blocks = len({".".join(layer.split(".")[:2]) for layer in checkpoint if "up" in layer}) + up_blocks = {layer_id: [key for key in checkpoint if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks)} + + for i in range(num_down_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + + if any("downsample" in layer for layer in down_blocks[i]): + new_checkpoint[f"down_blocks.{i}.downsamplers.0.conv.weight"] = checkpoint[ + f"down.{i}.downsample.op.weight" + ] + new_checkpoint[f"down_blocks.{i}.downsamplers.0.conv.bias"] = checkpoint[f"down.{i}.downsample.op.bias"] + # new_checkpoint[f'down_blocks.{i}.downsamplers.0.op.weight'] = checkpoint[f'down.{i}.downsample.conv.weight'] + # new_checkpoint[f'down_blocks.{i}.downsamplers.0.op.bias'] = checkpoint[f'down.{i}.downsample.conv.bias'] + + if any("block" in layer for layer in down_blocks[i]): + num_blocks = len( + {".".join(shave_segments(layer, 2).split(".")[:2]) for layer in down_blocks[i] if "block" in layer} + ) + blocks = { + layer_id: [key for key in down_blocks[i] if f"block.{layer_id}" in key] + for layer_id in range(num_blocks) + } + + if num_blocks > 0: + for j in range(config["layers_per_block"]): + paths = renew_resnet_paths(blocks[j]) + assign_to_checkpoint(paths, new_checkpoint, checkpoint) + + if any("attn" in layer for layer in down_blocks[i]): + num_attn = len( + {".".join(shave_segments(layer, 2).split(".")[:2]) for layer in down_blocks[i] if "attn" in layer} + ) + attns = { + layer_id: [key for key in down_blocks[i] if f"attn.{layer_id}" in key] + for layer_id in range(num_blocks) + } + + if num_attn > 0: + for j in range(config["layers_per_block"]): + paths = renew_attention_paths(attns[j]) + assign_to_checkpoint(paths, new_checkpoint, checkpoint, config=config) + + mid_block_1_layers = [key for key in checkpoint if "mid.block_1" in key] + mid_block_2_layers = [key for key in checkpoint if "mid.block_2" in key] + mid_attn_1_layers = [key for key in checkpoint if "mid.attn_1" in key] + + # Mid new 2 + paths = renew_resnet_paths(mid_block_1_layers) + assign_to_checkpoint( + paths, + new_checkpoint, + checkpoint, + additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "block_1", "new": "resnets.0"}], + ) + + paths = renew_resnet_paths(mid_block_2_layers) + assign_to_checkpoint( + paths, + new_checkpoint, + checkpoint, + additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "block_2", "new": "resnets.1"}], + ) + + paths = renew_attention_paths(mid_attn_1_layers, in_mid=True) + assign_to_checkpoint( + paths, + new_checkpoint, + checkpoint, + additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "attn_1", "new": "attentions.0"}], + ) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + + if any("upsample" in layer for layer in up_blocks[i]): + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = checkpoint[ + f"up.{i}.upsample.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = checkpoint[f"up.{i}.upsample.conv.bias"] + + if any("block" in layer for layer in up_blocks[i]): + num_blocks = len( + {".".join(shave_segments(layer, 2).split(".")[:2]) for layer in up_blocks[i] if "block" in layer} + ) + blocks = { + layer_id: [key for key in up_blocks[i] if f"block.{layer_id}" in key] for layer_id in range(num_blocks) + } + + if num_blocks > 0: + for j in range(config["layers_per_block"] + 1): + replace_indices = {"old": f"up_blocks.{i}", "new": f"up_blocks.{block_id}"} + paths = renew_resnet_paths(blocks[j]) + assign_to_checkpoint(paths, new_checkpoint, checkpoint, additional_replacements=[replace_indices]) + + if any("attn" in layer for layer in up_blocks[i]): + num_attn = len( + {".".join(shave_segments(layer, 2).split(".")[:2]) for layer in up_blocks[i] if "attn" in layer} + ) + attns = { + layer_id: [key for key in up_blocks[i] if f"attn.{layer_id}" in key] for layer_id in range(num_blocks) + } + + if num_attn > 0: + for j in range(config["layers_per_block"] + 1): + replace_indices = {"old": f"up_blocks.{i}", "new": f"up_blocks.{block_id}"} + paths = renew_attention_paths(attns[j]) + assign_to_checkpoint(paths, new_checkpoint, checkpoint, additional_replacements=[replace_indices]) + + new_checkpoint = {k.replace("mid_new_2", "mid_block"): v for k, v in new_checkpoint.items()} + return new_checkpoint + + +def convert_vq_autoenc_checkpoint(checkpoint, config): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + new_checkpoint = {} + + new_checkpoint["encoder.conv_norm_out.weight"] = checkpoint["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = checkpoint["encoder.norm_out.bias"] + + new_checkpoint["encoder.conv_in.weight"] = checkpoint["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = checkpoint["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = checkpoint["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = checkpoint["encoder.conv_out.bias"] + + new_checkpoint["decoder.conv_norm_out.weight"] = checkpoint["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = checkpoint["decoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = checkpoint["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = checkpoint["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = checkpoint["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = checkpoint["decoder.conv_out.bias"] + + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in checkpoint if "down" in layer}) + down_blocks = { + layer_id: [key for key in checkpoint if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in checkpoint if "up" in layer}) + up_blocks = {layer_id: [key for key in checkpoint if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks)} + + for i in range(num_down_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + + if any("downsample" in layer for layer in down_blocks[i]): + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = checkpoint[ + f"encoder.down.{i}.downsample.conv.weight" + ] + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = checkpoint[ + f"encoder.down.{i}.downsample.conv.bias" + ] + + if any("block" in layer for layer in down_blocks[i]): + num_blocks = len( + {".".join(shave_segments(layer, 3).split(".")[:3]) for layer in down_blocks[i] if "block" in layer} + ) + blocks = { + layer_id: [key for key in down_blocks[i] if f"block.{layer_id}" in key] + for layer_id in range(num_blocks) + } + + if num_blocks > 0: + for j in range(config["layers_per_block"]): + paths = renew_resnet_paths(blocks[j]) + assign_to_checkpoint(paths, new_checkpoint, checkpoint) + + if any("attn" in layer for layer in down_blocks[i]): + num_attn = len( + {".".join(shave_segments(layer, 3).split(".")[:3]) for layer in down_blocks[i] if "attn" in layer} + ) + attns = { + layer_id: [key for key in down_blocks[i] if f"attn.{layer_id}" in key] + for layer_id in range(num_blocks) + } + + if num_attn > 0: + for j in range(config["layers_per_block"]): + paths = renew_attention_paths(attns[j]) + assign_to_checkpoint(paths, new_checkpoint, checkpoint, config=config) + + mid_block_1_layers = [key for key in checkpoint if "mid.block_1" in key] + mid_block_2_layers = [key for key in checkpoint if "mid.block_2" in key] + mid_attn_1_layers = [key for key in checkpoint if "mid.attn_1" in key] + + # Mid new 2 + paths = renew_resnet_paths(mid_block_1_layers) + assign_to_checkpoint( + paths, + new_checkpoint, + checkpoint, + additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "block_1", "new": "resnets.0"}], + ) + + paths = renew_resnet_paths(mid_block_2_layers) + assign_to_checkpoint( + paths, + new_checkpoint, + checkpoint, + additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "block_2", "new": "resnets.1"}], + ) + + paths = renew_attention_paths(mid_attn_1_layers, in_mid=True) + assign_to_checkpoint( + paths, + new_checkpoint, + checkpoint, + additional_replacements=[{"old": "mid.", "new": "mid_new_2."}, {"old": "attn_1", "new": "attentions.0"}], + ) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + + if any("upsample" in layer for layer in up_blocks[i]): + new_checkpoint[f"decoder.up_blocks.{block_id}.upsamplers.0.conv.weight"] = checkpoint[ + f"decoder.up.{i}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{block_id}.upsamplers.0.conv.bias"] = checkpoint[ + f"decoder.up.{i}.upsample.conv.bias" + ] + + if any("block" in layer for layer in up_blocks[i]): + num_blocks = len( + {".".join(shave_segments(layer, 3).split(".")[:3]) for layer in up_blocks[i] if "block" in layer} + ) + blocks = { + layer_id: [key for key in up_blocks[i] if f"block.{layer_id}" in key] for layer_id in range(num_blocks) + } + + if num_blocks > 0: + for j in range(config["layers_per_block"] + 1): + replace_indices = {"old": f"up_blocks.{i}", "new": f"up_blocks.{block_id}"} + paths = renew_resnet_paths(blocks[j]) + assign_to_checkpoint(paths, new_checkpoint, checkpoint, additional_replacements=[replace_indices]) + + if any("attn" in layer for layer in up_blocks[i]): + num_attn = len( + {".".join(shave_segments(layer, 3).split(".")[:3]) for layer in up_blocks[i] if "attn" in layer} + ) + attns = { + layer_id: [key for key in up_blocks[i] if f"attn.{layer_id}" in key] for layer_id in range(num_blocks) + } + + if num_attn > 0: + for j in range(config["layers_per_block"] + 1): + replace_indices = {"old": f"up_blocks.{i}", "new": f"up_blocks.{block_id}"} + paths = renew_attention_paths(attns[j]) + assign_to_checkpoint(paths, new_checkpoint, checkpoint, additional_replacements=[replace_indices]) + + new_checkpoint = {k.replace("mid_new_2", "mid_block"): v for k, v in new_checkpoint.items()} + new_checkpoint["quant_conv.weight"] = checkpoint["quant_conv.weight"] + new_checkpoint["quant_conv.bias"] = checkpoint["quant_conv.bias"] + if "quantize.embedding.weight" in checkpoint: + new_checkpoint["quantize.embedding.weight"] = checkpoint["quantize.embedding.weight"] + new_checkpoint["post_quant_conv.weight"] = checkpoint["post_quant_conv.weight"] + new_checkpoint["post_quant_conv.bias"] = checkpoint["post_quant_conv.bias"] + + return new_checkpoint + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + + parser.add_argument( + "--config_file", + default=None, + type=str, + required=True, + help="The config json file corresponding to the architecture.", + ) + + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + + args = parser.parse_args() + checkpoint = torch.load(args.checkpoint_path) + + with open(args.config_file) as f: + config = json.loads(f.read()) + + # unet case + key_prefix_set = {key.split(".")[0] for key in checkpoint.keys()} + if "encoder" in key_prefix_set and "decoder" in key_prefix_set: + converted_checkpoint = convert_vq_autoenc_checkpoint(checkpoint, config) + else: + converted_checkpoint = convert_ddpm_checkpoint(checkpoint, config) + + if "ddpm" in config: + del config["ddpm"] + + if config["_class_name"] == "VQModel": + model = VQModel(**config) + model.load_state_dict(converted_checkpoint) + model.save_pretrained(args.dump_path) + elif config["_class_name"] == "AutoencoderKL": + model = AutoencoderKL(**config) + model.load_state_dict(converted_checkpoint) + model.save_pretrained(args.dump_path) + else: + model = UNet2DModel(**config) + model.load_state_dict(converted_checkpoint) + + scheduler = DDPMScheduler.from_config("/".join(args.checkpoint_path.split("/")[:-1])) + + pipe = DDPMPipeline(unet=model, scheduler=scheduler) + pipe.save_pretrained(args.dump_path) diff --git a/diffusers-0.27.0/scripts/convert_diffusers_sdxl_lora_to_webui.py b/diffusers-0.27.0/scripts/convert_diffusers_sdxl_lora_to_webui.py new file mode 100755 index 0000000..dfb3871 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_diffusers_sdxl_lora_to_webui.py @@ -0,0 +1,56 @@ +# Script for converting a Hugging Face Diffusers trained SDXL LoRAs to Kohya format +# This means that you can input your diffusers-trained LoRAs and +# Get the output to work with WebUIs such as AUTOMATIC1111, ComfyUI, SD.Next and others. + +# To get started you can find some cool `diffusers` trained LoRAs such as this cute Corgy +# https://huggingface.co/ignasbud/corgy_dog_LoRA/, download its `pytorch_lora_weights.safetensors` file +# and run the script: +# python convert_diffusers_sdxl_lora_to_webui.py --input_lora pytorch_lora_weights.safetensors --output_lora corgy.safetensors +# now you can use corgy.safetensors in your WebUI of choice! + +# To train your own, here are some diffusers training scripts and utils that you can use and then convert: +# LoRA Ease - no code SDXL Dreambooth LoRA trainer: https://huggingface.co/spaces/multimodalart/lora-ease +# Dreambooth Advanced Training Script - state of the art techniques such as pivotal tuning and prodigy optimizer: +# - Script: https://github.com/huggingface/diffusers/blob/main/examples/advanced_diffusion_training/train_dreambooth_lora_sdxl_advanced.py +# - Colab (only on Pro): https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/SDXL_Dreambooth_LoRA_advanced_example.ipynb +# Canonical diffusers training scripts: +# - Script: https://github.com/huggingface/diffusers/blob/main/examples/dreambooth/train_dreambooth_lora_sdxl.py +# - Colab (runs on free tier): https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/SDXL_DreamBooth_LoRA_.ipynb + +import argparse +import os + +from safetensors.torch import load_file, save_file + +from diffusers.utils import convert_all_state_dict_to_peft, convert_state_dict_to_kohya + + +def convert_and_save(input_lora, output_lora=None): + if output_lora is None: + base_name = os.path.splitext(input_lora)[0] + output_lora = f"{base_name}_webui.safetensors" + + diffusers_state_dict = load_file(input_lora) + peft_state_dict = convert_all_state_dict_to_peft(diffusers_state_dict) + kohya_state_dict = convert_state_dict_to_kohya(peft_state_dict) + save_file(kohya_state_dict, output_lora) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Convert LoRA model to PEFT and then to Kohya format.") + parser.add_argument( + "--input_lora", + type=str, + required=True, + help="Path to the input LoRA model file in the diffusers format.", + ) + parser.add_argument( + "--output_lora", + type=str, + required=False, + help="Path for the converted LoRA (safetensors format for AUTOMATIC1111, ComfyUI, etc.). Optional, defaults to input name with a _webui suffix.", + ) + + args = parser.parse_args() + + convert_and_save(args.input_lora, args.output_lora) diff --git a/diffusers-0.27.0/scripts/convert_diffusers_to_original_sdxl.py b/diffusers-0.27.0/scripts/convert_diffusers_to_original_sdxl.py new file mode 100755 index 0000000..648d037 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_diffusers_to_original_sdxl.py @@ -0,0 +1,350 @@ +# Script for converting a HF Diffusers saved pipeline to a Stable Diffusion checkpoint. +# *Only* converts the UNet, VAE, and Text Encoder. +# Does not convert optimizer state or any other thing. + +import argparse +import os.path as osp +import re + +import torch +from safetensors.torch import load_file, save_file + + +# =================# +# UNet Conversion # +# =================# + +unet_conversion_map = [ + # (stable-diffusion, HF Diffusers) + ("time_embed.0.weight", "time_embedding.linear_1.weight"), + ("time_embed.0.bias", "time_embedding.linear_1.bias"), + ("time_embed.2.weight", "time_embedding.linear_2.weight"), + ("time_embed.2.bias", "time_embedding.linear_2.bias"), + ("input_blocks.0.0.weight", "conv_in.weight"), + ("input_blocks.0.0.bias", "conv_in.bias"), + ("out.0.weight", "conv_norm_out.weight"), + ("out.0.bias", "conv_norm_out.bias"), + ("out.2.weight", "conv_out.weight"), + ("out.2.bias", "conv_out.bias"), + # the following are for sdxl + ("label_emb.0.0.weight", "add_embedding.linear_1.weight"), + ("label_emb.0.0.bias", "add_embedding.linear_1.bias"), + ("label_emb.0.2.weight", "add_embedding.linear_2.weight"), + ("label_emb.0.2.bias", "add_embedding.linear_2.bias"), +] + +unet_conversion_map_resnet = [ + # (stable-diffusion, HF Diffusers) + ("in_layers.0", "norm1"), + ("in_layers.2", "conv1"), + ("out_layers.0", "norm2"), + ("out_layers.3", "conv2"), + ("emb_layers.1", "time_emb_proj"), + ("skip_connection", "conv_shortcut"), +] + +unet_conversion_map_layer = [] +# hardcoded number of downblocks and resnets/attentions... +# would need smarter logic for other networks. +for i in range(3): + # loop over downblocks/upblocks + + for j in range(2): + # loop over resnets/attentions for downblocks + hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}." + sd_down_res_prefix = f"input_blocks.{3*i + j + 1}.0." + unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix)) + + if i > 0: + hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}." + sd_down_atn_prefix = f"input_blocks.{3*i + j + 1}.1." + unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix)) + + for j in range(4): + # loop over resnets/attentions for upblocks + hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}." + sd_up_res_prefix = f"output_blocks.{3*i + j}.0." + unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix)) + + if i < 2: + # no attention layers in up_blocks.0 + hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}." + sd_up_atn_prefix = f"output_blocks.{3 * i + j}.1." + unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix)) + + if i < 3: + # no downsample in down_blocks.3 + hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv." + sd_downsample_prefix = f"input_blocks.{3*(i+1)}.0.op." + unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix)) + + # no upsample in up_blocks.3 + hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0." + sd_upsample_prefix = f"output_blocks.{3*i + 2}.{1 if i == 0 else 2}." + unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix)) +unet_conversion_map_layer.append(("output_blocks.2.2.conv.", "output_blocks.2.1.conv.")) + +hf_mid_atn_prefix = "mid_block.attentions.0." +sd_mid_atn_prefix = "middle_block.1." +unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix)) +for j in range(2): + hf_mid_res_prefix = f"mid_block.resnets.{j}." + sd_mid_res_prefix = f"middle_block.{2*j}." + unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix)) + + +def convert_unet_state_dict(unet_state_dict): + # buyer beware: this is a *brittle* function, + # and correct output requires that all of these pieces interact in + # the exact order in which I have arranged them. + mapping = {k: k for k in unet_state_dict.keys()} + for sd_name, hf_name in unet_conversion_map: + mapping[hf_name] = sd_name + for k, v in mapping.items(): + if "resnets" in k: + for sd_part, hf_part in unet_conversion_map_resnet: + v = v.replace(hf_part, sd_part) + mapping[k] = v + for k, v in mapping.items(): + for sd_part, hf_part in unet_conversion_map_layer: + v = v.replace(hf_part, sd_part) + mapping[k] = v + new_state_dict = {sd_name: unet_state_dict[hf_name] for hf_name, sd_name in mapping.items()} + return new_state_dict + + +# ================# +# VAE Conversion # +# ================# + +vae_conversion_map = [ + # (stable-diffusion, HF Diffusers) + ("nin_shortcut", "conv_shortcut"), + ("norm_out", "conv_norm_out"), + ("mid.attn_1.", "mid_block.attentions.0."), +] + +for i in range(4): + # down_blocks have two resnets + for j in range(2): + hf_down_prefix = f"encoder.down_blocks.{i}.resnets.{j}." + sd_down_prefix = f"encoder.down.{i}.block.{j}." + vae_conversion_map.append((sd_down_prefix, hf_down_prefix)) + + if i < 3: + hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0." + sd_downsample_prefix = f"down.{i}.downsample." + vae_conversion_map.append((sd_downsample_prefix, hf_downsample_prefix)) + + hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0." + sd_upsample_prefix = f"up.{3-i}.upsample." + vae_conversion_map.append((sd_upsample_prefix, hf_upsample_prefix)) + + # up_blocks have three resnets + # also, up blocks in hf are numbered in reverse from sd + for j in range(3): + hf_up_prefix = f"decoder.up_blocks.{i}.resnets.{j}." + sd_up_prefix = f"decoder.up.{3-i}.block.{j}." + vae_conversion_map.append((sd_up_prefix, hf_up_prefix)) + +# this part accounts for mid blocks in both the encoder and the decoder +for i in range(2): + hf_mid_res_prefix = f"mid_block.resnets.{i}." + sd_mid_res_prefix = f"mid.block_{i+1}." + vae_conversion_map.append((sd_mid_res_prefix, hf_mid_res_prefix)) + + +vae_conversion_map_attn = [ + # (stable-diffusion, HF Diffusers) + ("norm.", "group_norm."), + # the following are for SDXL + ("q.", "to_q."), + ("k.", "to_k."), + ("v.", "to_v."), + ("proj_out.", "to_out.0."), +] + + +def reshape_weight_for_sd(w): + # convert HF linear weights to SD conv2d weights + if not w.ndim == 1: + return w.reshape(*w.shape, 1, 1) + else: + return w + + +def convert_vae_state_dict(vae_state_dict): + mapping = {k: k for k in vae_state_dict.keys()} + for k, v in mapping.items(): + for sd_part, hf_part in vae_conversion_map: + v = v.replace(hf_part, sd_part) + mapping[k] = v + for k, v in mapping.items(): + if "attentions" in k: + for sd_part, hf_part in vae_conversion_map_attn: + v = v.replace(hf_part, sd_part) + mapping[k] = v + new_state_dict = {v: vae_state_dict[k] for k, v in mapping.items()} + weights_to_convert = ["q", "k", "v", "proj_out"] + for k, v in new_state_dict.items(): + for weight_name in weights_to_convert: + if f"mid.attn_1.{weight_name}.weight" in k: + print(f"Reshaping {k} for SD format") + new_state_dict[k] = reshape_weight_for_sd(v) + return new_state_dict + + +# =========================# +# Text Encoder Conversion # +# =========================# + + +textenc_conversion_lst = [ + # (stable-diffusion, HF Diffusers) + ("transformer.resblocks.", "text_model.encoder.layers."), + ("ln_1", "layer_norm1"), + ("ln_2", "layer_norm2"), + (".c_fc.", ".fc1."), + (".c_proj.", ".fc2."), + (".attn", ".self_attn"), + ("ln_final.", "text_model.final_layer_norm."), + ("token_embedding.weight", "text_model.embeddings.token_embedding.weight"), + ("positional_embedding", "text_model.embeddings.position_embedding.weight"), +] +protected = {re.escape(x[1]): x[0] for x in textenc_conversion_lst} +textenc_pattern = re.compile("|".join(protected.keys())) + +# Ordering is from https://github.com/pytorch/pytorch/blob/master/test/cpp/api/modules.cpp +code2idx = {"q": 0, "k": 1, "v": 2} + + +def convert_openclip_text_enc_state_dict(text_enc_dict): + new_state_dict = {} + capture_qkv_weight = {} + capture_qkv_bias = {} + for k, v in text_enc_dict.items(): + if ( + k.endswith(".self_attn.q_proj.weight") + or k.endswith(".self_attn.k_proj.weight") + or k.endswith(".self_attn.v_proj.weight") + ): + k_pre = k[: -len(".q_proj.weight")] + k_code = k[-len("q_proj.weight")] + if k_pre not in capture_qkv_weight: + capture_qkv_weight[k_pre] = [None, None, None] + capture_qkv_weight[k_pre][code2idx[k_code]] = v + continue + + if ( + k.endswith(".self_attn.q_proj.bias") + or k.endswith(".self_attn.k_proj.bias") + or k.endswith(".self_attn.v_proj.bias") + ): + k_pre = k[: -len(".q_proj.bias")] + k_code = k[-len("q_proj.bias")] + if k_pre not in capture_qkv_bias: + capture_qkv_bias[k_pre] = [None, None, None] + capture_qkv_bias[k_pre][code2idx[k_code]] = v + continue + + relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k) + new_state_dict[relabelled_key] = v + + for k_pre, tensors in capture_qkv_weight.items(): + if None in tensors: + raise Exception("CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing") + relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k_pre) + new_state_dict[relabelled_key + ".in_proj_weight"] = torch.cat(tensors) + + for k_pre, tensors in capture_qkv_bias.items(): + if None in tensors: + raise Exception("CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing") + relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k_pre) + new_state_dict[relabelled_key + ".in_proj_bias"] = torch.cat(tensors) + + return new_state_dict + + +def convert_openai_text_enc_state_dict(text_enc_dict): + return text_enc_dict + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", default=None, type=str, required=True, help="Path to the model to convert.") + parser.add_argument("--checkpoint_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument("--half", action="store_true", help="Save weights in half precision.") + parser.add_argument( + "--use_safetensors", action="store_true", help="Save weights use safetensors, default is ckpt." + ) + + args = parser.parse_args() + + assert args.model_path is not None, "Must provide a model path!" + + assert args.checkpoint_path is not None, "Must provide a checkpoint path!" + + # Path for safetensors + unet_path = osp.join(args.model_path, "unet", "diffusion_pytorch_model.safetensors") + vae_path = osp.join(args.model_path, "vae", "diffusion_pytorch_model.safetensors") + text_enc_path = osp.join(args.model_path, "text_encoder", "model.safetensors") + text_enc_2_path = osp.join(args.model_path, "text_encoder_2", "model.safetensors") + + # Load models from safetensors if it exists, if it doesn't pytorch + if osp.exists(unet_path): + unet_state_dict = load_file(unet_path, device="cpu") + else: + unet_path = osp.join(args.model_path, "unet", "diffusion_pytorch_model.bin") + unet_state_dict = torch.load(unet_path, map_location="cpu") + + if osp.exists(vae_path): + vae_state_dict = load_file(vae_path, device="cpu") + else: + vae_path = osp.join(args.model_path, "vae", "diffusion_pytorch_model.bin") + vae_state_dict = torch.load(vae_path, map_location="cpu") + + if osp.exists(text_enc_path): + text_enc_dict = load_file(text_enc_path, device="cpu") + else: + text_enc_path = osp.join(args.model_path, "text_encoder", "pytorch_model.bin") + text_enc_dict = torch.load(text_enc_path, map_location="cpu") + + if osp.exists(text_enc_2_path): + text_enc_2_dict = load_file(text_enc_2_path, device="cpu") + else: + text_enc_2_path = osp.join(args.model_path, "text_encoder_2", "pytorch_model.bin") + text_enc_2_dict = torch.load(text_enc_2_path, map_location="cpu") + + # Convert the UNet model + unet_state_dict = convert_unet_state_dict(unet_state_dict) + unet_state_dict = {"model.diffusion_model." + k: v for k, v in unet_state_dict.items()} + + # Convert the VAE model + vae_state_dict = convert_vae_state_dict(vae_state_dict) + vae_state_dict = {"first_stage_model." + k: v for k, v in vae_state_dict.items()} + + # Convert text encoder 1 + text_enc_dict = convert_openai_text_enc_state_dict(text_enc_dict) + text_enc_dict = {"conditioner.embedders.0.transformer." + k: v for k, v in text_enc_dict.items()} + + # Convert text encoder 2 + text_enc_2_dict = convert_openclip_text_enc_state_dict(text_enc_2_dict) + text_enc_2_dict = {"conditioner.embedders.1.model." + k: v for k, v in text_enc_2_dict.items()} + # We call the `.T.contiguous()` to match what's done in + # https://github.com/huggingface/diffusers/blob/84905ca7287876b925b6bf8e9bb92fec21c78764/src/diffusers/loaders/single_file_utils.py#L1085 + text_enc_2_dict["conditioner.embedders.1.model.text_projection"] = text_enc_2_dict.pop( + "conditioner.embedders.1.model.text_projection.weight" + ).T.contiguous() + + # Put together new checkpoint + state_dict = {**unet_state_dict, **vae_state_dict, **text_enc_dict, **text_enc_2_dict} + + if args.half: + state_dict = {k: v.half() for k, v in state_dict.items()} + + if args.use_safetensors: + save_file(state_dict, args.checkpoint_path) + else: + state_dict = {"state_dict": state_dict} + torch.save(state_dict, args.checkpoint_path) diff --git a/diffusers-0.27.0/scripts/convert_diffusers_to_original_stable_diffusion.py b/diffusers-0.27.0/scripts/convert_diffusers_to_original_stable_diffusion.py new file mode 100755 index 0000000..d1b7df0 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_diffusers_to_original_stable_diffusion.py @@ -0,0 +1,353 @@ +# Script for converting a HF Diffusers saved pipeline to a Stable Diffusion checkpoint. +# *Only* converts the UNet, VAE, and Text Encoder. +# Does not convert optimizer state or any other thing. + +import argparse +import os.path as osp +import re + +import torch +from safetensors.torch import load_file, save_file + + +# =================# +# UNet Conversion # +# =================# + +unet_conversion_map = [ + # (stable-diffusion, HF Diffusers) + ("time_embed.0.weight", "time_embedding.linear_1.weight"), + ("time_embed.0.bias", "time_embedding.linear_1.bias"), + ("time_embed.2.weight", "time_embedding.linear_2.weight"), + ("time_embed.2.bias", "time_embedding.linear_2.bias"), + ("input_blocks.0.0.weight", "conv_in.weight"), + ("input_blocks.0.0.bias", "conv_in.bias"), + ("out.0.weight", "conv_norm_out.weight"), + ("out.0.bias", "conv_norm_out.bias"), + ("out.2.weight", "conv_out.weight"), + ("out.2.bias", "conv_out.bias"), +] + +unet_conversion_map_resnet = [ + # (stable-diffusion, HF Diffusers) + ("in_layers.0", "norm1"), + ("in_layers.2", "conv1"), + ("out_layers.0", "norm2"), + ("out_layers.3", "conv2"), + ("emb_layers.1", "time_emb_proj"), + ("skip_connection", "conv_shortcut"), +] + +unet_conversion_map_layer = [] +# hardcoded number of downblocks and resnets/attentions... +# would need smarter logic for other networks. +for i in range(4): + # loop over downblocks/upblocks + + for j in range(2): + # loop over resnets/attentions for downblocks + hf_down_res_prefix = f"down_blocks.{i}.resnets.{j}." + sd_down_res_prefix = f"input_blocks.{3*i + j + 1}.0." + unet_conversion_map_layer.append((sd_down_res_prefix, hf_down_res_prefix)) + + if i < 3: + # no attention layers in down_blocks.3 + hf_down_atn_prefix = f"down_blocks.{i}.attentions.{j}." + sd_down_atn_prefix = f"input_blocks.{3*i + j + 1}.1." + unet_conversion_map_layer.append((sd_down_atn_prefix, hf_down_atn_prefix)) + + for j in range(3): + # loop over resnets/attentions for upblocks + hf_up_res_prefix = f"up_blocks.{i}.resnets.{j}." + sd_up_res_prefix = f"output_blocks.{3*i + j}.0." + unet_conversion_map_layer.append((sd_up_res_prefix, hf_up_res_prefix)) + + if i > 0: + # no attention layers in up_blocks.0 + hf_up_atn_prefix = f"up_blocks.{i}.attentions.{j}." + sd_up_atn_prefix = f"output_blocks.{3*i + j}.1." + unet_conversion_map_layer.append((sd_up_atn_prefix, hf_up_atn_prefix)) + + if i < 3: + # no downsample in down_blocks.3 + hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0.conv." + sd_downsample_prefix = f"input_blocks.{3*(i+1)}.0.op." + unet_conversion_map_layer.append((sd_downsample_prefix, hf_downsample_prefix)) + + # no upsample in up_blocks.3 + hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0." + sd_upsample_prefix = f"output_blocks.{3*i + 2}.{1 if i == 0 else 2}." + unet_conversion_map_layer.append((sd_upsample_prefix, hf_upsample_prefix)) + +hf_mid_atn_prefix = "mid_block.attentions.0." +sd_mid_atn_prefix = "middle_block.1." +unet_conversion_map_layer.append((sd_mid_atn_prefix, hf_mid_atn_prefix)) + +for j in range(2): + hf_mid_res_prefix = f"mid_block.resnets.{j}." + sd_mid_res_prefix = f"middle_block.{2*j}." + unet_conversion_map_layer.append((sd_mid_res_prefix, hf_mid_res_prefix)) + + +def convert_unet_state_dict(unet_state_dict): + # buyer beware: this is a *brittle* function, + # and correct output requires that all of these pieces interact in + # the exact order in which I have arranged them. + mapping = {k: k for k in unet_state_dict.keys()} + for sd_name, hf_name in unet_conversion_map: + mapping[hf_name] = sd_name + for k, v in mapping.items(): + if "resnets" in k: + for sd_part, hf_part in unet_conversion_map_resnet: + v = v.replace(hf_part, sd_part) + mapping[k] = v + for k, v in mapping.items(): + for sd_part, hf_part in unet_conversion_map_layer: + v = v.replace(hf_part, sd_part) + mapping[k] = v + new_state_dict = {v: unet_state_dict[k] for k, v in mapping.items()} + return new_state_dict + + +# ================# +# VAE Conversion # +# ================# + +vae_conversion_map = [ + # (stable-diffusion, HF Diffusers) + ("nin_shortcut", "conv_shortcut"), + ("norm_out", "conv_norm_out"), + ("mid.attn_1.", "mid_block.attentions.0."), +] + +for i in range(4): + # down_blocks have two resnets + for j in range(2): + hf_down_prefix = f"encoder.down_blocks.{i}.resnets.{j}." + sd_down_prefix = f"encoder.down.{i}.block.{j}." + vae_conversion_map.append((sd_down_prefix, hf_down_prefix)) + + if i < 3: + hf_downsample_prefix = f"down_blocks.{i}.downsamplers.0." + sd_downsample_prefix = f"down.{i}.downsample." + vae_conversion_map.append((sd_downsample_prefix, hf_downsample_prefix)) + + hf_upsample_prefix = f"up_blocks.{i}.upsamplers.0." + sd_upsample_prefix = f"up.{3-i}.upsample." + vae_conversion_map.append((sd_upsample_prefix, hf_upsample_prefix)) + + # up_blocks have three resnets + # also, up blocks in hf are numbered in reverse from sd + for j in range(3): + hf_up_prefix = f"decoder.up_blocks.{i}.resnets.{j}." + sd_up_prefix = f"decoder.up.{3-i}.block.{j}." + vae_conversion_map.append((sd_up_prefix, hf_up_prefix)) + +# this part accounts for mid blocks in both the encoder and the decoder +for i in range(2): + hf_mid_res_prefix = f"mid_block.resnets.{i}." + sd_mid_res_prefix = f"mid.block_{i+1}." + vae_conversion_map.append((sd_mid_res_prefix, hf_mid_res_prefix)) + + +vae_conversion_map_attn = [ + # (stable-diffusion, HF Diffusers) + ("norm.", "group_norm."), + ("q.", "query."), + ("k.", "key."), + ("v.", "value."), + ("proj_out.", "proj_attn."), +] + +# This is probably not the most ideal solution, but it does work. +vae_extra_conversion_map = [ + ("to_q", "q"), + ("to_k", "k"), + ("to_v", "v"), + ("to_out.0", "proj_out"), +] + + +def reshape_weight_for_sd(w): + # convert HF linear weights to SD conv2d weights + if not w.ndim == 1: + return w.reshape(*w.shape, 1, 1) + else: + return w + + +def convert_vae_state_dict(vae_state_dict): + mapping = {k: k for k in vae_state_dict.keys()} + for k, v in mapping.items(): + for sd_part, hf_part in vae_conversion_map: + v = v.replace(hf_part, sd_part) + mapping[k] = v + for k, v in mapping.items(): + if "attentions" in k: + for sd_part, hf_part in vae_conversion_map_attn: + v = v.replace(hf_part, sd_part) + mapping[k] = v + new_state_dict = {v: vae_state_dict[k] for k, v in mapping.items()} + weights_to_convert = ["q", "k", "v", "proj_out"] + keys_to_rename = {} + for k, v in new_state_dict.items(): + for weight_name in weights_to_convert: + if f"mid.attn_1.{weight_name}.weight" in k: + print(f"Reshaping {k} for SD format") + new_state_dict[k] = reshape_weight_for_sd(v) + for weight_name, real_weight_name in vae_extra_conversion_map: + if f"mid.attn_1.{weight_name}.weight" in k or f"mid.attn_1.{weight_name}.bias" in k: + keys_to_rename[k] = k.replace(weight_name, real_weight_name) + for k, v in keys_to_rename.items(): + if k in new_state_dict: + print(f"Renaming {k} to {v}") + new_state_dict[v] = reshape_weight_for_sd(new_state_dict[k]) + del new_state_dict[k] + return new_state_dict + + +# =========================# +# Text Encoder Conversion # +# =========================# + + +textenc_conversion_lst = [ + # (stable-diffusion, HF Diffusers) + ("resblocks.", "text_model.encoder.layers."), + ("ln_1", "layer_norm1"), + ("ln_2", "layer_norm2"), + (".c_fc.", ".fc1."), + (".c_proj.", ".fc2."), + (".attn", ".self_attn"), + ("ln_final.", "transformer.text_model.final_layer_norm."), + ("token_embedding.weight", "transformer.text_model.embeddings.token_embedding.weight"), + ("positional_embedding", "transformer.text_model.embeddings.position_embedding.weight"), +] +protected = {re.escape(x[1]): x[0] for x in textenc_conversion_lst} +textenc_pattern = re.compile("|".join(protected.keys())) + +# Ordering is from https://github.com/pytorch/pytorch/blob/master/test/cpp/api/modules.cpp +code2idx = {"q": 0, "k": 1, "v": 2} + + +def convert_text_enc_state_dict_v20(text_enc_dict): + new_state_dict = {} + capture_qkv_weight = {} + capture_qkv_bias = {} + for k, v in text_enc_dict.items(): + if ( + k.endswith(".self_attn.q_proj.weight") + or k.endswith(".self_attn.k_proj.weight") + or k.endswith(".self_attn.v_proj.weight") + ): + k_pre = k[: -len(".q_proj.weight")] + k_code = k[-len("q_proj.weight")] + if k_pre not in capture_qkv_weight: + capture_qkv_weight[k_pre] = [None, None, None] + capture_qkv_weight[k_pre][code2idx[k_code]] = v + continue + + if ( + k.endswith(".self_attn.q_proj.bias") + or k.endswith(".self_attn.k_proj.bias") + or k.endswith(".self_attn.v_proj.bias") + ): + k_pre = k[: -len(".q_proj.bias")] + k_code = k[-len("q_proj.bias")] + if k_pre not in capture_qkv_bias: + capture_qkv_bias[k_pre] = [None, None, None] + capture_qkv_bias[k_pre][code2idx[k_code]] = v + continue + + relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k) + new_state_dict[relabelled_key] = v + + for k_pre, tensors in capture_qkv_weight.items(): + if None in tensors: + raise Exception("CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing") + relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k_pre) + new_state_dict[relabelled_key + ".in_proj_weight"] = torch.cat(tensors) + + for k_pre, tensors in capture_qkv_bias.items(): + if None in tensors: + raise Exception("CORRUPTED MODEL: one of the q-k-v values for the text encoder was missing") + relabelled_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], k_pre) + new_state_dict[relabelled_key + ".in_proj_bias"] = torch.cat(tensors) + + return new_state_dict + + +def convert_text_enc_state_dict(text_enc_dict): + return text_enc_dict + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--model_path", default=None, type=str, required=True, help="Path to the model to convert.") + parser.add_argument("--checkpoint_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument("--half", action="store_true", help="Save weights in half precision.") + parser.add_argument( + "--use_safetensors", action="store_true", help="Save weights use safetensors, default is ckpt." + ) + + args = parser.parse_args() + + assert args.model_path is not None, "Must provide a model path!" + + assert args.checkpoint_path is not None, "Must provide a checkpoint path!" + + # Path for safetensors + unet_path = osp.join(args.model_path, "unet", "diffusion_pytorch_model.safetensors") + vae_path = osp.join(args.model_path, "vae", "diffusion_pytorch_model.safetensors") + text_enc_path = osp.join(args.model_path, "text_encoder", "model.safetensors") + + # Load models from safetensors if it exists, if it doesn't pytorch + if osp.exists(unet_path): + unet_state_dict = load_file(unet_path, device="cpu") + else: + unet_path = osp.join(args.model_path, "unet", "diffusion_pytorch_model.bin") + unet_state_dict = torch.load(unet_path, map_location="cpu") + + if osp.exists(vae_path): + vae_state_dict = load_file(vae_path, device="cpu") + else: + vae_path = osp.join(args.model_path, "vae", "diffusion_pytorch_model.bin") + vae_state_dict = torch.load(vae_path, map_location="cpu") + + if osp.exists(text_enc_path): + text_enc_dict = load_file(text_enc_path, device="cpu") + else: + text_enc_path = osp.join(args.model_path, "text_encoder", "pytorch_model.bin") + text_enc_dict = torch.load(text_enc_path, map_location="cpu") + + # Convert the UNet model + unet_state_dict = convert_unet_state_dict(unet_state_dict) + unet_state_dict = {"model.diffusion_model." + k: v for k, v in unet_state_dict.items()} + + # Convert the VAE model + vae_state_dict = convert_vae_state_dict(vae_state_dict) + vae_state_dict = {"first_stage_model." + k: v for k, v in vae_state_dict.items()} + + # Easiest way to identify v2.0 model seems to be that the text encoder (OpenCLIP) is deeper + is_v20_model = "text_model.encoder.layers.22.layer_norm2.bias" in text_enc_dict + + if is_v20_model: + # Need to add the tag 'transformer' in advance so we can knock it out from the final layer-norm + text_enc_dict = {"transformer." + k: v for k, v in text_enc_dict.items()} + text_enc_dict = convert_text_enc_state_dict_v20(text_enc_dict) + text_enc_dict = {"cond_stage_model.model." + k: v for k, v in text_enc_dict.items()} + else: + text_enc_dict = convert_text_enc_state_dict(text_enc_dict) + text_enc_dict = {"cond_stage_model.transformer." + k: v for k, v in text_enc_dict.items()} + + # Put together new checkpoint + state_dict = {**unet_state_dict, **vae_state_dict, **text_enc_dict} + if args.half: + state_dict = {k: v.half() for k, v in state_dict.items()} + + if args.use_safetensors: + save_file(state_dict, args.checkpoint_path) + else: + state_dict = {"state_dict": state_dict} + torch.save(state_dict, args.checkpoint_path) diff --git a/diffusers-0.27.0/scripts/convert_dit_to_diffusers.py b/diffusers-0.27.0/scripts/convert_dit_to_diffusers.py new file mode 100755 index 0000000..dc127f6 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_dit_to_diffusers.py @@ -0,0 +1,162 @@ +import argparse +import os + +import torch +from torchvision.datasets.utils import download_url + +from diffusers import AutoencoderKL, DDIMScheduler, DiTPipeline, Transformer2DModel + + +pretrained_models = {512: "DiT-XL-2-512x512.pt", 256: "DiT-XL-2-256x256.pt"} + + +def download_model(model_name): + """ + Downloads a pre-trained DiT model from the web. + """ + local_path = f"pretrained_models/{model_name}" + if not os.path.isfile(local_path): + os.makedirs("pretrained_models", exist_ok=True) + web_path = f"https://dl.fbaipublicfiles.com/DiT/models/{model_name}" + download_url(web_path, "pretrained_models") + model = torch.load(local_path, map_location=lambda storage, loc: storage) + return model + + +def main(args): + state_dict = download_model(pretrained_models[args.image_size]) + + state_dict["pos_embed.proj.weight"] = state_dict["x_embedder.proj.weight"] + state_dict["pos_embed.proj.bias"] = state_dict["x_embedder.proj.bias"] + state_dict.pop("x_embedder.proj.weight") + state_dict.pop("x_embedder.proj.bias") + + for depth in range(28): + state_dict[f"transformer_blocks.{depth}.norm1.emb.timestep_embedder.linear_1.weight"] = state_dict[ + "t_embedder.mlp.0.weight" + ] + state_dict[f"transformer_blocks.{depth}.norm1.emb.timestep_embedder.linear_1.bias"] = state_dict[ + "t_embedder.mlp.0.bias" + ] + state_dict[f"transformer_blocks.{depth}.norm1.emb.timestep_embedder.linear_2.weight"] = state_dict[ + "t_embedder.mlp.2.weight" + ] + state_dict[f"transformer_blocks.{depth}.norm1.emb.timestep_embedder.linear_2.bias"] = state_dict[ + "t_embedder.mlp.2.bias" + ] + state_dict[f"transformer_blocks.{depth}.norm1.emb.class_embedder.embedding_table.weight"] = state_dict[ + "y_embedder.embedding_table.weight" + ] + + state_dict[f"transformer_blocks.{depth}.norm1.linear.weight"] = state_dict[ + f"blocks.{depth}.adaLN_modulation.1.weight" + ] + state_dict[f"transformer_blocks.{depth}.norm1.linear.bias"] = state_dict[ + f"blocks.{depth}.adaLN_modulation.1.bias" + ] + + q, k, v = torch.chunk(state_dict[f"blocks.{depth}.attn.qkv.weight"], 3, dim=0) + q_bias, k_bias, v_bias = torch.chunk(state_dict[f"blocks.{depth}.attn.qkv.bias"], 3, dim=0) + + state_dict[f"transformer_blocks.{depth}.attn1.to_q.weight"] = q + state_dict[f"transformer_blocks.{depth}.attn1.to_q.bias"] = q_bias + state_dict[f"transformer_blocks.{depth}.attn1.to_k.weight"] = k + state_dict[f"transformer_blocks.{depth}.attn1.to_k.bias"] = k_bias + state_dict[f"transformer_blocks.{depth}.attn1.to_v.weight"] = v + state_dict[f"transformer_blocks.{depth}.attn1.to_v.bias"] = v_bias + + state_dict[f"transformer_blocks.{depth}.attn1.to_out.0.weight"] = state_dict[ + f"blocks.{depth}.attn.proj.weight" + ] + state_dict[f"transformer_blocks.{depth}.attn1.to_out.0.bias"] = state_dict[f"blocks.{depth}.attn.proj.bias"] + + state_dict[f"transformer_blocks.{depth}.ff.net.0.proj.weight"] = state_dict[f"blocks.{depth}.mlp.fc1.weight"] + state_dict[f"transformer_blocks.{depth}.ff.net.0.proj.bias"] = state_dict[f"blocks.{depth}.mlp.fc1.bias"] + state_dict[f"transformer_blocks.{depth}.ff.net.2.weight"] = state_dict[f"blocks.{depth}.mlp.fc2.weight"] + state_dict[f"transformer_blocks.{depth}.ff.net.2.bias"] = state_dict[f"blocks.{depth}.mlp.fc2.bias"] + + state_dict.pop(f"blocks.{depth}.attn.qkv.weight") + state_dict.pop(f"blocks.{depth}.attn.qkv.bias") + state_dict.pop(f"blocks.{depth}.attn.proj.weight") + state_dict.pop(f"blocks.{depth}.attn.proj.bias") + state_dict.pop(f"blocks.{depth}.mlp.fc1.weight") + state_dict.pop(f"blocks.{depth}.mlp.fc1.bias") + state_dict.pop(f"blocks.{depth}.mlp.fc2.weight") + state_dict.pop(f"blocks.{depth}.mlp.fc2.bias") + state_dict.pop(f"blocks.{depth}.adaLN_modulation.1.weight") + state_dict.pop(f"blocks.{depth}.adaLN_modulation.1.bias") + + state_dict.pop("t_embedder.mlp.0.weight") + state_dict.pop("t_embedder.mlp.0.bias") + state_dict.pop("t_embedder.mlp.2.weight") + state_dict.pop("t_embedder.mlp.2.bias") + state_dict.pop("y_embedder.embedding_table.weight") + + state_dict["proj_out_1.weight"] = state_dict["final_layer.adaLN_modulation.1.weight"] + state_dict["proj_out_1.bias"] = state_dict["final_layer.adaLN_modulation.1.bias"] + state_dict["proj_out_2.weight"] = state_dict["final_layer.linear.weight"] + state_dict["proj_out_2.bias"] = state_dict["final_layer.linear.bias"] + + state_dict.pop("final_layer.linear.weight") + state_dict.pop("final_layer.linear.bias") + state_dict.pop("final_layer.adaLN_modulation.1.weight") + state_dict.pop("final_layer.adaLN_modulation.1.bias") + + # DiT XL/2 + transformer = Transformer2DModel( + sample_size=args.image_size // 8, + num_layers=28, + attention_head_dim=72, + in_channels=4, + out_channels=8, + patch_size=2, + attention_bias=True, + num_attention_heads=16, + activation_fn="gelu-approximate", + num_embeds_ada_norm=1000, + norm_type="ada_norm_zero", + norm_elementwise_affine=False, + ) + transformer.load_state_dict(state_dict, strict=True) + + scheduler = DDIMScheduler( + num_train_timesteps=1000, + beta_schedule="linear", + prediction_type="epsilon", + clip_sample=False, + ) + + vae = AutoencoderKL.from_pretrained(args.vae_model) + + pipeline = DiTPipeline(transformer=transformer, vae=vae, scheduler=scheduler) + + if args.save: + pipeline.save_pretrained(args.checkpoint_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--image_size", + default=256, + type=int, + required=False, + help="Image size of pretrained model, either 256 or 512.", + ) + parser.add_argument( + "--vae_model", + default="stabilityai/sd-vae-ft-ema", + type=str, + required=False, + help="Path to pretrained VAE model, either stabilityai/sd-vae-ft-mse or stabilityai/sd-vae-ft-ema.", + ) + parser.add_argument( + "--save", default=True, type=bool, required=False, help="Whether to save the converted pipeline or not." + ) + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the output pipeline." + ) + + args = parser.parse_args() + main(args) diff --git a/diffusers-0.27.0/scripts/convert_gligen_to_diffusers.py b/diffusers-0.27.0/scripts/convert_gligen_to_diffusers.py new file mode 100755 index 0000000..83c1f92 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_gligen_to_diffusers.py @@ -0,0 +1,581 @@ +import argparse +import re + +import torch +import yaml +from transformers import ( + CLIPProcessor, + CLIPTextModel, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + StableDiffusionGLIGENPipeline, + StableDiffusionGLIGENTextImagePipeline, + UNet2DConditionModel, +) +from diffusers.pipelines.stable_diffusion.convert_from_ckpt import ( + assign_to_checkpoint, + conv_attn_to_linear, + protected, + renew_attention_paths, + renew_resnet_paths, + renew_vae_attention_paths, + renew_vae_resnet_paths, + shave_segments, + textenc_conversion_map, + textenc_pattern, +) + + +def convert_open_clip_checkpoint(checkpoint): + checkpoint = checkpoint["text_encoder"] + text_model = CLIPTextModel.from_pretrained("openai/clip-vit-large-patch14") + + keys = list(checkpoint.keys()) + + text_model_dict = {} + + if "cond_stage_model.model.text_projection" in checkpoint: + d_model = int(checkpoint["cond_stage_model.model.text_projection"].shape[0]) + else: + d_model = 1024 + + for key in keys: + if "resblocks.23" in key: # Diffusers drops the final layer and only uses the penultimate layer + continue + if key in textenc_conversion_map: + text_model_dict[textenc_conversion_map[key]] = checkpoint[key] + # if key.startswith("cond_stage_model.model.transformer."): + new_key = key[len("transformer.") :] + if new_key.endswith(".in_proj_weight"): + new_key = new_key[: -len(".in_proj_weight")] + new_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], new_key) + text_model_dict[new_key + ".q_proj.weight"] = checkpoint[key][:d_model, :] + text_model_dict[new_key + ".k_proj.weight"] = checkpoint[key][d_model : d_model * 2, :] + text_model_dict[new_key + ".v_proj.weight"] = checkpoint[key][d_model * 2 :, :] + elif new_key.endswith(".in_proj_bias"): + new_key = new_key[: -len(".in_proj_bias")] + new_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], new_key) + text_model_dict[new_key + ".q_proj.bias"] = checkpoint[key][:d_model] + text_model_dict[new_key + ".k_proj.bias"] = checkpoint[key][d_model : d_model * 2] + text_model_dict[new_key + ".v_proj.bias"] = checkpoint[key][d_model * 2 :] + else: + if key != "transformer.text_model.embeddings.position_ids": + new_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], new_key) + + text_model_dict[new_key] = checkpoint[key] + + if key == "transformer.text_model.embeddings.token_embedding.weight": + text_model_dict["text_model.embeddings.token_embedding.weight"] = checkpoint[key] + + text_model_dict.pop("text_model.embeddings.transformer.text_model.embeddings.token_embedding.weight") + + text_model.load_state_dict(text_model_dict) + + return text_model + + +def convert_gligen_vae_checkpoint(checkpoint, config): + checkpoint = checkpoint["autoencoder"] + vae_state_dict = {} + vae_key = "first_stage_model." + keys = list(checkpoint.keys()) + for key in keys: + vae_state_dict[key.replace(vae_key, "")] = checkpoint.get(key) + + new_checkpoint = {} + + new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"] + new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"] + new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"] + + new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"] + new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"] + new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"] + new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer}) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer}) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + + for key in new_checkpoint.keys(): + if "encoder.mid_block.attentions.0" in key or "decoder.mid_block.attentions.0" in key: + if "query" in key: + new_checkpoint[key.replace("query", "to_q")] = new_checkpoint.pop(key) + if "value" in key: + new_checkpoint[key.replace("value", "to_v")] = new_checkpoint.pop(key) + if "key" in key: + new_checkpoint[key.replace("key", "to_k")] = new_checkpoint.pop(key) + if "proj_attn" in key: + new_checkpoint[key.replace("proj_attn", "to_out.0")] = new_checkpoint.pop(key) + + return new_checkpoint + + +def convert_gligen_unet_checkpoint(checkpoint, config, path=None, extract_ema=False): + unet_state_dict = {} + checkpoint = checkpoint["model"] + keys = list(checkpoint.keys()) + + unet_key = "model.diffusion_model." + + if sum(k.startswith("model_ema") for k in keys) > 100 and extract_ema: + print(f"Checkpoint {path} has bot EMA and non-EMA weights.") + print( + "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA" + " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag." + ) + for key in keys: + if key.startswith("model.diffusion_model"): + flat_ema_key = "model_ema." + "".join(key.split(".")[1:]) + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(flat_ema_key) + else: + if sum(k.startswith("model_ema") for k in keys) > 100: + print( + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA" + " weights (usually better for inference), please make sure to add the `--extract_ema` flag." + ) + for key in keys: + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key) + + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"] + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key] + for layer_id in range(num_output_blocks) + } + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + + if f"input_blocks.{i}.0.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + resnet_0_paths = renew_resnet_paths(resnet_0) + assign_to_checkpoint(resnet_0_paths, new_checkpoint, unet_state_dict, config=config) + + resnet_1_paths = renew_resnet_paths(resnet_1) + assign_to_checkpoint(resnet_1_paths, new_checkpoint, unet_state_dict, config=config) + + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key] + + resnet_0_paths = renew_resnet_paths(resnets) + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + output_block_list = {k: sorted(v) for k, v in output_block_list.items()} + if ["conv.bias", "conv.weight"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.bias", "conv.weight"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = { + "old": f"output_blocks.{i}.1", + "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + + new_checkpoint[new_path] = unet_state_dict[old_path] + + for key in keys: + if "position_net" in key: + new_checkpoint[key] = unet_state_dict[key] + + return new_checkpoint + + +def create_vae_config(original_config, image_size: int): + vae_params = original_config["autoencoder"]["params"]["ddconfig"] + _ = original_config["autoencoder"]["params"]["embed_dim"] + + block_out_channels = [vae_params["ch"] * mult for mult in vae_params["ch_mult"]] + down_block_types = ["DownEncoderBlock2D"] * len(block_out_channels) + up_block_types = ["UpDecoderBlock2D"] * len(block_out_channels) + + config = { + "sample_size": image_size, + "in_channels": vae_params["in_channels"], + "out_channels": vae_params["out_ch"], + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "latent_channels": vae_params["z_channels"], + "layers_per_block": vae_params["num_res_blocks"], + } + + return config + + +def create_unet_config(original_config, image_size: int, attention_type): + unet_params = original_config["model"]["params"] + vae_params = original_config["autoencoder"]["params"]["ddconfig"] + + block_out_channels = [unet_params["model_channels"] * mult for mult in unet_params["channel_mult"]] + + down_block_types = [] + resolution = 1 + for i in range(len(block_out_channels)): + block_type = "CrossAttnDownBlock2D" if resolution in unet_params["attention_resolutions"] else "DownBlock2D" + down_block_types.append(block_type) + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + block_type = "CrossAttnUpBlock2D" if resolution in unet_params["attention_resolutions"] else "UpBlock2D" + up_block_types.append(block_type) + resolution //= 2 + + vae_scale_factor = 2 ** (len(vae_params["ch_mult"]) - 1) + + head_dim = unet_params["num_heads"] if "num_heads" in unet_params else None + use_linear_projection = ( + unet_params["use_linear_in_transformer"] if "use_linear_in_transformer" in unet_params else False + ) + if use_linear_projection: + if head_dim is None: + head_dim = [5, 10, 20, 20] + + config = { + "sample_size": image_size // vae_scale_factor, + "in_channels": unet_params["in_channels"], + "down_block_types": tuple(down_block_types), + "block_out_channels": tuple(block_out_channels), + "layers_per_block": unet_params["num_res_blocks"], + "cross_attention_dim": unet_params["context_dim"], + "attention_head_dim": head_dim, + "use_linear_projection": use_linear_projection, + "attention_type": attention_type, + } + + return config + + +def convert_gligen_to_diffusers( + checkpoint_path: str, + original_config_file: str, + attention_type: str, + image_size: int = 512, + extract_ema: bool = False, + num_in_channels: int = None, + device: str = None, +): + if device is None: + device = "cuda" if torch.cuda.is_available() else "cpu" + checkpoint = torch.load(checkpoint_path, map_location=device) + else: + checkpoint = torch.load(checkpoint_path, map_location=device) + + if "global_step" in checkpoint: + checkpoint["global_step"] + else: + print("global_step key not found in model") + + original_config = yaml.safe_load(original_config_file) + + if num_in_channels is not None: + original_config["model"]["params"]["in_channels"] = num_in_channels + + num_train_timesteps = original_config["diffusion"]["params"]["timesteps"] + beta_start = original_config["diffusion"]["params"]["linear_start"] + beta_end = original_config["diffusion"]["params"]["linear_end"] + + scheduler = DDIMScheduler( + beta_end=beta_end, + beta_schedule="scaled_linear", + beta_start=beta_start, + num_train_timesteps=num_train_timesteps, + steps_offset=1, + clip_sample=False, + set_alpha_to_one=False, + prediction_type="epsilon", + ) + + # Convert the UNet2DConditionalModel model + unet_config = create_unet_config(original_config, image_size, attention_type) + unet = UNet2DConditionModel(**unet_config) + + converted_unet_checkpoint = convert_gligen_unet_checkpoint( + checkpoint, unet_config, path=checkpoint_path, extract_ema=extract_ema + ) + + unet.load_state_dict(converted_unet_checkpoint) + + # Convert the VAE model + vae_config = create_vae_config(original_config, image_size) + converted_vae_checkpoint = convert_gligen_vae_checkpoint(checkpoint, vae_config) + + vae = AutoencoderKL(**vae_config) + vae.load_state_dict(converted_vae_checkpoint) + + # Convert the text model + text_encoder = convert_open_clip_checkpoint(checkpoint) + tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14") + + if attention_type == "gated-text-image": + image_encoder = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-large-patch14") + processor = CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14") + + pipe = StableDiffusionGLIGENTextImagePipeline( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + image_encoder=image_encoder, + processor=processor, + unet=unet, + scheduler=scheduler, + safety_checker=None, + feature_extractor=None, + ) + elif attention_type == "gated": + pipe = StableDiffusionGLIGENPipeline( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=None, + feature_extractor=None, + ) + + return pipe + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + parser.add_argument( + "--original_config_file", + default=None, + type=str, + required=True, + help="The YAML config file corresponding to the gligen architecture.", + ) + parser.add_argument( + "--num_in_channels", + default=None, + type=int, + help="The number of input channels. If `None` number of input channels will be automatically inferred.", + ) + parser.add_argument( + "--extract_ema", + action="store_true", + help=( + "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights" + " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield" + " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning." + ), + ) + parser.add_argument( + "--attention_type", + default=None, + type=str, + required=True, + help="Type of attention ex: gated or gated-text-image", + ) + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument("--device", type=str, help="Device to use.") + parser.add_argument("--half", action="store_true", help="Save weights in half precision.") + + args = parser.parse_args() + + pipe = convert_gligen_to_diffusers( + checkpoint_path=args.checkpoint_path, + original_config_file=args.original_config_file, + attention_type=args.attention_type, + extract_ema=args.extract_ema, + num_in_channels=args.num_in_channels, + device=args.device, + ) + + if args.half: + pipe.to(dtype=torch.float16) + + pipe.save_pretrained(args.dump_path) diff --git a/diffusers-0.27.0/scripts/convert_i2vgen_to_diffusers.py b/diffusers-0.27.0/scripts/convert_i2vgen_to_diffusers.py new file mode 100755 index 0000000..daf30ad --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_i2vgen_to_diffusers.py @@ -0,0 +1,510 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the LDM checkpoints. """ + +import argparse + +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from diffusers import DDIMScheduler, I2VGenXLPipeline, I2VGenXLUNet, StableDiffusionPipeline + + +CLIP_ID = "laion/CLIP-ViT-H-14-laion2B-s32B-b79K" + + +def assign_to_checkpoint( + paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None +): + """ + This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits + attention layers, and takes into account additional replacements that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + # Splits the attention layers into three variables. + if attention_paths_to_split is not None: + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape) + checkpoint[path_map["key"]] = key.reshape(target_shape) + checkpoint[path_map["value"]] = value.reshape(target_shape) + + for path in paths: + new_path = path["new"] + + # These have already been assigned + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + # proj_attn.weight has to be converted from conv 1D to linear + weight = old_checkpoint[path["old"]] + names = ["proj_attn.weight"] + names_2 = ["proj_out.weight", "proj_in.weight"] + if any(k in new_path for k in names): + checkpoint[new_path] = weight[:, :, 0] + elif any(k in new_path for k in names_2) and len(weight.shape) > 2 and ".attentions." not in new_path: + checkpoint[new_path] = weight[:, :, 0] + else: + checkpoint[new_path] = weight + + +def renew_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +def renew_temp_conv_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + mapping.append({"old": old_item, "new": old_item}) + + return mapping + + +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("emb_layers.1", "time_emb_proj") + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + if "temopral_conv" not in old_item: + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def convert_ldm_unet_checkpoint(checkpoint, config, path=None, extract_ema=False): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + + # extract state_dict for UNet + unet_state_dict = {} + keys = list(checkpoint.keys()) + + unet_key = "model.diffusion_model." + + # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA + if sum(k.startswith("model_ema") for k in keys) > 100 and extract_ema: + print(f"Checkpoint {path} has both EMA and non-EMA weights.") + print( + "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA" + " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag." + ) + for key in keys: + if key.startswith("model.diffusion_model"): + flat_ema_key = "model_ema." + "".join(key.split(".")[1:]) + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(flat_ema_key) + else: + if sum(k.startswith("model_ema") for k in keys) > 100: + print( + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA" + " weights (usually better for inference), please make sure to add the `--extract_ema` flag." + ) + + for key in keys: + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key) + + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"] + + additional_embedding_substrings = [ + "local_image_concat", + "context_embedding", + "local_image_embedding", + "fps_embedding", + ] + for k in unet_state_dict: + if any(substring in k for substring in additional_embedding_substrings): + diffusers_key = k.replace("local_image_concat", "image_latents_proj_in").replace( + "local_image_embedding", "image_latents_context_embedding" + ) + new_checkpoint[diffusers_key] = unet_state_dict[k] + + # temporal encoder. + new_checkpoint["image_latents_temporal_encoder.norm1.weight"] = unet_state_dict[ + "local_temporal_encoder.layers.0.0.norm.weight" + ] + new_checkpoint["image_latents_temporal_encoder.norm1.bias"] = unet_state_dict[ + "local_temporal_encoder.layers.0.0.norm.bias" + ] + + # attention + qkv = unet_state_dict["local_temporal_encoder.layers.0.0.fn.to_qkv.weight"] + q, k, v = torch.chunk(qkv, 3, dim=0) + new_checkpoint["image_latents_temporal_encoder.attn1.to_q.weight"] = q + new_checkpoint["image_latents_temporal_encoder.attn1.to_k.weight"] = k + new_checkpoint["image_latents_temporal_encoder.attn1.to_v.weight"] = v + new_checkpoint["image_latents_temporal_encoder.attn1.to_out.0.weight"] = unet_state_dict[ + "local_temporal_encoder.layers.0.0.fn.to_out.0.weight" + ] + new_checkpoint["image_latents_temporal_encoder.attn1.to_out.0.bias"] = unet_state_dict[ + "local_temporal_encoder.layers.0.0.fn.to_out.0.bias" + ] + + # feedforward + new_checkpoint["image_latents_temporal_encoder.ff.net.0.proj.weight"] = unet_state_dict[ + "local_temporal_encoder.layers.0.1.net.0.0.weight" + ] + new_checkpoint["image_latents_temporal_encoder.ff.net.0.proj.bias"] = unet_state_dict[ + "local_temporal_encoder.layers.0.1.net.0.0.bias" + ] + new_checkpoint["image_latents_temporal_encoder.ff.net.2.weight"] = unet_state_dict[ + "local_temporal_encoder.layers.0.1.net.2.weight" + ] + new_checkpoint["image_latents_temporal_encoder.ff.net.2.bias"] = unet_state_dict[ + "local_temporal_encoder.layers.0.1.net.2.bias" + ] + + if "class_embed_type" in config: + if config["class_embed_type"] is None: + # No parameters to port + ... + elif config["class_embed_type"] == "timestep" or config["class_embed_type"] == "projection": + new_checkpoint["class_embedding.linear_1.weight"] = unet_state_dict["label_emb.0.0.weight"] + new_checkpoint["class_embedding.linear_1.bias"] = unet_state_dict["label_emb.0.0.bias"] + new_checkpoint["class_embedding.linear_2.weight"] = unet_state_dict["label_emb.0.2.weight"] + new_checkpoint["class_embedding.linear_2.bias"] = unet_state_dict["label_emb.0.2.bias"] + else: + raise NotImplementedError(f"Not implemented `class_embed_type`: {config['class_embed_type']}") + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + first_temp_attention = [v for v in unet_state_dict if v.startswith("input_blocks.0.1")] + paths = renew_attention_paths(first_temp_attention) + meta_path = {"old": "input_blocks.0.1", "new": "transformer_in"} + assign_to_checkpoint(paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config) + + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key] + for layer_id in range(num_output_blocks) + } + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + temp_attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.2" in key] + + if f"input_blocks.{i}.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.op.bias" + ) + + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + temporal_convs = [key for key in resnets if "temopral_conv" in key] + paths = renew_temp_conv_paths(temporal_convs) + meta_path = { + "old": f"input_blocks.{i}.0.temopral_conv", + "new": f"down_blocks.{block_id}.temp_convs.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(temp_attentions): + paths = renew_attention_paths(temp_attentions) + meta_path = { + "old": f"input_blocks.{i}.2", + "new": f"down_blocks.{block_id}.temp_attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + resnet_0 = middle_blocks[0] + temporal_convs_0 = [key for key in resnet_0 if "temopral_conv" in key] + attentions = middle_blocks[1] + temp_attentions = middle_blocks[2] + resnet_1 = middle_blocks[3] + temporal_convs_1 = [key for key in resnet_1 if "temopral_conv" in key] + + resnet_0_paths = renew_resnet_paths(resnet_0) + meta_path = {"old": "middle_block.0", "new": "mid_block.resnets.0"} + assign_to_checkpoint( + resnet_0_paths, new_checkpoint, unet_state_dict, config=config, additional_replacements=[meta_path] + ) + + temp_conv_0_paths = renew_temp_conv_paths(temporal_convs_0) + meta_path = {"old": "middle_block.0.temopral_conv", "new": "mid_block.temp_convs.0"} + assign_to_checkpoint( + temp_conv_0_paths, new_checkpoint, unet_state_dict, config=config, additional_replacements=[meta_path] + ) + + resnet_1_paths = renew_resnet_paths(resnet_1) + meta_path = {"old": "middle_block.3", "new": "mid_block.resnets.1"} + assign_to_checkpoint( + resnet_1_paths, new_checkpoint, unet_state_dict, config=config, additional_replacements=[meta_path] + ) + + temp_conv_1_paths = renew_temp_conv_paths(temporal_convs_1) + meta_path = {"old": "middle_block.3.temopral_conv", "new": "mid_block.temp_convs.1"} + assign_to_checkpoint( + temp_conv_1_paths, new_checkpoint, unet_state_dict, config=config, additional_replacements=[meta_path] + ) + + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + temp_attentions_paths = renew_attention_paths(temp_attentions) + meta_path = {"old": "middle_block.2", "new": "mid_block.temp_attentions.0"} + assign_to_checkpoint( + temp_attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key] + temp_attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.2" in key] + + resnet_0_paths = renew_resnet_paths(resnets) + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + temporal_convs = [key for key in resnets if "temopral_conv" in key] + paths = renew_temp_conv_paths(temporal_convs) + meta_path = { + "old": f"output_blocks.{i}.0.temopral_conv", + "new": f"up_blocks.{block_id}.temp_convs.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + output_block_list = {k: sorted(v) for k, v in output_block_list.items()} + if ["conv.bias", "conv.weight"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.bias", "conv.weight"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = { + "old": f"output_blocks.{i}.1", + "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(temp_attentions): + paths = renew_attention_paths(temp_attentions) + meta_path = { + "old": f"output_blocks.{i}.2", + "new": f"up_blocks.{block_id}.temp_attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + new_checkpoint[new_path] = unet_state_dict[old_path] + + temopral_conv_paths = [l for l in output_block_layers if "temopral_conv" in l] + for path in temopral_conv_paths: + pruned_path = path.split("temopral_conv.")[-1] + old_path = ".".join(["output_blocks", str(i), str(block_id), "temopral_conv", pruned_path]) + new_path = ".".join(["up_blocks", str(block_id), "temp_convs", str(layer_in_block_id), pruned_path]) + new_checkpoint[new_path] = unet_state_dict[old_path] + + return new_checkpoint + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--unet_checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument("--push_to_hub", action="store_true") + args = parser.parse_args() + + # UNet + unet_checkpoint = torch.load(args.unet_checkpoint_path, map_location="cpu") + unet_checkpoint = unet_checkpoint["state_dict"] + unet = I2VGenXLUNet(sample_size=32) + + converted_ckpt = convert_ldm_unet_checkpoint(unet_checkpoint, unet.config) + + diff_0 = set(unet.state_dict().keys()) - set(converted_ckpt.keys()) + diff_1 = set(converted_ckpt.keys()) - set(unet.state_dict().keys()) + + assert len(diff_0) == len(diff_1) == 0, "Converted weights don't match" + + unet.load_state_dict(converted_ckpt, strict=True) + + # vae + temp_pipe = StableDiffusionPipeline.from_single_file( + "https://huggingface.co/ali-vilab/i2vgen-xl/blob/main/models/v2-1_512-ema-pruned.ckpt" + ) + vae = temp_pipe.vae + del temp_pipe + + # text encoder and tokenizer + text_encoder = CLIPTextModel.from_pretrained(CLIP_ID) + tokenizer = CLIPTokenizer.from_pretrained(CLIP_ID) + + # image encoder and feature extractor + image_encoder = CLIPVisionModelWithProjection.from_pretrained(CLIP_ID) + feature_extractor = CLIPImageProcessor.from_pretrained(CLIP_ID) + + # scheduler + # https://github.com/ali-vilab/i2vgen-xl/blob/main/configs/i2vgen_xl_train.yaml + scheduler = DDIMScheduler( + beta_schedule="squaredcos_cap_v2", + rescale_betas_zero_snr=True, + set_alpha_to_one=True, + clip_sample=False, + steps_offset=1, + timestep_spacing="leading", + prediction_type="v_prediction", + ) + + # final + pipeline = I2VGenXLPipeline( + unet=unet, + vae=vae, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + text_encoder=text_encoder, + tokenizer=tokenizer, + scheduler=scheduler, + ) + + pipeline.save_pretrained(args.dump_path, push_to_hub=args.push_to_hub) diff --git a/diffusers-0.27.0/scripts/convert_if.py b/diffusers-0.27.0/scripts/convert_if.py new file mode 100755 index 0000000..c4588f4 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_if.py @@ -0,0 +1,1250 @@ +import argparse +import inspect +import os + +import numpy as np +import torch +import yaml +from torch.nn import functional as F +from transformers import CLIPConfig, CLIPImageProcessor, CLIPVisionModelWithProjection, T5EncoderModel, T5Tokenizer + +from diffusers import DDPMScheduler, IFPipeline, IFSuperResolutionPipeline, UNet2DConditionModel +from diffusers.pipelines.deepfloyd_if.safety_checker import IFSafetyChecker + + +def parse_args(): + parser = argparse.ArgumentParser() + + parser.add_argument("--dump_path", required=False, default=None, type=str) + + parser.add_argument("--dump_path_stage_2", required=False, default=None, type=str) + + parser.add_argument("--dump_path_stage_3", required=False, default=None, type=str) + + parser.add_argument("--unet_config", required=False, default=None, type=str, help="Path to unet config file") + + parser.add_argument( + "--unet_checkpoint_path", required=False, default=None, type=str, help="Path to unet checkpoint file" + ) + + parser.add_argument( + "--unet_checkpoint_path_stage_2", + required=False, + default=None, + type=str, + help="Path to stage 2 unet checkpoint file", + ) + + parser.add_argument( + "--unet_checkpoint_path_stage_3", + required=False, + default=None, + type=str, + help="Path to stage 3 unet checkpoint file", + ) + + parser.add_argument("--p_head_path", type=str, required=True) + + parser.add_argument("--w_head_path", type=str, required=True) + + args = parser.parse_args() + + return args + + +def main(args): + tokenizer = T5Tokenizer.from_pretrained("google/t5-v1_1-xxl") + text_encoder = T5EncoderModel.from_pretrained("google/t5-v1_1-xxl") + + feature_extractor = CLIPImageProcessor.from_pretrained("openai/clip-vit-large-patch14") + safety_checker = convert_safety_checker(p_head_path=args.p_head_path, w_head_path=args.w_head_path) + + if args.unet_config is not None and args.unet_checkpoint_path is not None and args.dump_path is not None: + convert_stage_1_pipeline(tokenizer, text_encoder, feature_extractor, safety_checker, args) + + if args.unet_checkpoint_path_stage_2 is not None and args.dump_path_stage_2 is not None: + convert_super_res_pipeline(tokenizer, text_encoder, feature_extractor, safety_checker, args, stage=2) + + if args.unet_checkpoint_path_stage_3 is not None and args.dump_path_stage_3 is not None: + convert_super_res_pipeline(tokenizer, text_encoder, feature_extractor, safety_checker, args, stage=3) + + +def convert_stage_1_pipeline(tokenizer, text_encoder, feature_extractor, safety_checker, args): + unet = get_stage_1_unet(args.unet_config, args.unet_checkpoint_path) + + scheduler = DDPMScheduler( + variance_type="learned_range", + beta_schedule="squaredcos_cap_v2", + prediction_type="epsilon", + thresholding=True, + dynamic_thresholding_ratio=0.95, + sample_max_value=1.5, + ) + + pipe = IFPipeline( + tokenizer=tokenizer, + text_encoder=text_encoder, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + requires_safety_checker=True, + ) + + pipe.save_pretrained(args.dump_path) + + +def convert_super_res_pipeline(tokenizer, text_encoder, feature_extractor, safety_checker, args, stage): + if stage == 2: + unet_checkpoint_path = args.unet_checkpoint_path_stage_2 + sample_size = None + dump_path = args.dump_path_stage_2 + elif stage == 3: + unet_checkpoint_path = args.unet_checkpoint_path_stage_3 + sample_size = 1024 + dump_path = args.dump_path_stage_3 + else: + assert False + + unet = get_super_res_unet(unet_checkpoint_path, verify_param_count=False, sample_size=sample_size) + + image_noising_scheduler = DDPMScheduler( + beta_schedule="squaredcos_cap_v2", + ) + + scheduler = DDPMScheduler( + variance_type="learned_range", + beta_schedule="squaredcos_cap_v2", + prediction_type="epsilon", + thresholding=True, + dynamic_thresholding_ratio=0.95, + sample_max_value=1.0, + ) + + pipe = IFSuperResolutionPipeline( + tokenizer=tokenizer, + text_encoder=text_encoder, + unet=unet, + scheduler=scheduler, + image_noising_scheduler=image_noising_scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + requires_safety_checker=True, + ) + + pipe.save_pretrained(dump_path) + + +def get_stage_1_unet(unet_config, unet_checkpoint_path): + original_unet_config = yaml.safe_load(unet_config) + original_unet_config = original_unet_config["params"] + + unet_diffusers_config = create_unet_diffusers_config(original_unet_config) + + unet = UNet2DConditionModel(**unet_diffusers_config) + + device = "cuda" if torch.cuda.is_available() else "cpu" + unet_checkpoint = torch.load(unet_checkpoint_path, map_location=device) + + converted_unet_checkpoint = convert_ldm_unet_checkpoint( + unet_checkpoint, unet_diffusers_config, path=unet_checkpoint_path + ) + + unet.load_state_dict(converted_unet_checkpoint) + + return unet + + +def convert_safety_checker(p_head_path, w_head_path): + state_dict = {} + + # p head + + p_head = np.load(p_head_path) + + p_head_weights = p_head["weights"] + p_head_weights = torch.from_numpy(p_head_weights) + p_head_weights = p_head_weights.unsqueeze(0) + + p_head_biases = p_head["biases"] + p_head_biases = torch.from_numpy(p_head_biases) + p_head_biases = p_head_biases.unsqueeze(0) + + state_dict["p_head.weight"] = p_head_weights + state_dict["p_head.bias"] = p_head_biases + + # w head + + w_head = np.load(w_head_path) + + w_head_weights = w_head["weights"] + w_head_weights = torch.from_numpy(w_head_weights) + w_head_weights = w_head_weights.unsqueeze(0) + + w_head_biases = w_head["biases"] + w_head_biases = torch.from_numpy(w_head_biases) + w_head_biases = w_head_biases.unsqueeze(0) + + state_dict["w_head.weight"] = w_head_weights + state_dict["w_head.bias"] = w_head_biases + + # vision model + + vision_model = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-large-patch14") + vision_model_state_dict = vision_model.state_dict() + + for key, value in vision_model_state_dict.items(): + key = f"vision_model.{key}" + state_dict[key] = value + + # full model + + config = CLIPConfig.from_pretrained("openai/clip-vit-large-patch14") + safety_checker = IFSafetyChecker(config) + + safety_checker.load_state_dict(state_dict) + + return safety_checker + + +def create_unet_diffusers_config(original_unet_config, class_embed_type=None): + attention_resolutions = parse_list(original_unet_config["attention_resolutions"]) + attention_resolutions = [original_unet_config["image_size"] // int(res) for res in attention_resolutions] + + channel_mult = parse_list(original_unet_config["channel_mult"]) + block_out_channels = [original_unet_config["model_channels"] * mult for mult in channel_mult] + + down_block_types = [] + resolution = 1 + + for i in range(len(block_out_channels)): + if resolution in attention_resolutions: + block_type = "SimpleCrossAttnDownBlock2D" + elif original_unet_config["resblock_updown"]: + block_type = "ResnetDownsampleBlock2D" + else: + block_type = "DownBlock2D" + + down_block_types.append(block_type) + + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + if resolution in attention_resolutions: + block_type = "SimpleCrossAttnUpBlock2D" + elif original_unet_config["resblock_updown"]: + block_type = "ResnetUpsampleBlock2D" + else: + block_type = "UpBlock2D" + up_block_types.append(block_type) + resolution //= 2 + + head_dim = original_unet_config["num_head_channels"] + + use_linear_projection = ( + original_unet_config["use_linear_in_transformer"] + if "use_linear_in_transformer" in original_unet_config + else False + ) + if use_linear_projection: + # stable diffusion 2-base-512 and 2-768 + if head_dim is None: + head_dim = [5, 10, 20, 20] + + projection_class_embeddings_input_dim = None + + if class_embed_type is None: + if "num_classes" in original_unet_config: + if original_unet_config["num_classes"] == "sequential": + class_embed_type = "projection" + assert "adm_in_channels" in original_unet_config + projection_class_embeddings_input_dim = original_unet_config["adm_in_channels"] + else: + raise NotImplementedError( + f"Unknown conditional unet num_classes config: {original_unet_config['num_classes']}" + ) + + config = { + "sample_size": original_unet_config["image_size"], + "in_channels": original_unet_config["in_channels"], + "down_block_types": tuple(down_block_types), + "block_out_channels": tuple(block_out_channels), + "layers_per_block": original_unet_config["num_res_blocks"], + "cross_attention_dim": original_unet_config["encoder_channels"], + "attention_head_dim": head_dim, + "use_linear_projection": use_linear_projection, + "class_embed_type": class_embed_type, + "projection_class_embeddings_input_dim": projection_class_embeddings_input_dim, + "out_channels": original_unet_config["out_channels"], + "up_block_types": tuple(up_block_types), + "upcast_attention": False, # TODO: guessing + "cross_attention_norm": "group_norm", + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "addition_embed_type": "text", + "act_fn": "gelu", + } + + if original_unet_config["use_scale_shift_norm"]: + config["resnet_time_scale_shift"] = "scale_shift" + + if "encoder_dim" in original_unet_config: + config["encoder_hid_dim"] = original_unet_config["encoder_dim"] + + return config + + +def convert_ldm_unet_checkpoint(unet_state_dict, config, path=None): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"] + + if config["class_embed_type"] in [None, "identity"]: + # No parameters to port + ... + elif config["class_embed_type"] == "timestep" or config["class_embed_type"] == "projection": + new_checkpoint["class_embedding.linear_1.weight"] = unet_state_dict["label_emb.0.0.weight"] + new_checkpoint["class_embedding.linear_1.bias"] = unet_state_dict["label_emb.0.0.bias"] + new_checkpoint["class_embedding.linear_2.weight"] = unet_state_dict["label_emb.0.2.weight"] + new_checkpoint["class_embedding.linear_2.bias"] = unet_state_dict["label_emb.0.2.bias"] + else: + raise NotImplementedError(f"Not implemented `class_embed_type`: {config['class_embed_type']}") + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}." in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}." in key] + for layer_id in range(num_output_blocks) + } + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + + if f"input_blocks.{i}.0.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + + paths = renew_resnet_paths(resnets) + + # TODO need better check than i in [4, 8, 12, 16] + block_type = config["down_block_types"][block_id] + if (block_type == "ResnetDownsampleBlock2D" or block_type == "SimpleCrossAttnDownBlock2D") and i in [ + 4, + 8, + 12, + 16, + ]: + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.downsamplers.0"} + else: + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(attentions): + old_path = f"input_blocks.{i}.1" + new_path = f"down_blocks.{block_id}.attentions.{layer_in_block_id}" + + assign_attention_to_checkpoint( + new_checkpoint=new_checkpoint, + unet_state_dict=unet_state_dict, + old_path=old_path, + new_path=new_path, + config=config, + ) + + paths = renew_attention_paths(attentions) + meta_path = {"old": old_path, "new": new_path} + assign_to_checkpoint( + paths, + new_checkpoint, + unet_state_dict, + additional_replacements=[meta_path], + config=config, + ) + + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + resnet_0_paths = renew_resnet_paths(resnet_0) + assign_to_checkpoint(resnet_0_paths, new_checkpoint, unet_state_dict, config=config) + + resnet_1_paths = renew_resnet_paths(resnet_1) + assign_to_checkpoint(resnet_1_paths, new_checkpoint, unet_state_dict, config=config) + + old_path = "middle_block.1" + new_path = "mid_block.attentions.0" + + assign_attention_to_checkpoint( + new_checkpoint=new_checkpoint, + unet_state_dict=unet_state_dict, + old_path=old_path, + new_path=new_path, + config=config, + ) + + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + # len(output_block_list) == 1 -> resnet + # len(output_block_list) == 2 -> resnet, attention + # len(output_block_list) == 3 -> resnet, attention, upscale resnet + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key] + + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + output_block_list = {k: sorted(v) for k, v in output_block_list.items()} + if ["conv.bias", "conv.weight"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.bias", "conv.weight"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + + if len(attentions): + old_path = f"output_blocks.{i}.1" + new_path = f"up_blocks.{block_id}.attentions.{layer_in_block_id}" + + assign_attention_to_checkpoint( + new_checkpoint=new_checkpoint, + unet_state_dict=unet_state_dict, + old_path=old_path, + new_path=new_path, + config=config, + ) + + paths = renew_attention_paths(attentions) + meta_path = { + "old": old_path, + "new": new_path, + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(output_block_list) == 3: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.2" in key] + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"output_blocks.{i}.2", "new": f"up_blocks.{block_id}.upsamplers.0"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + + new_checkpoint[new_path] = unet_state_dict[old_path] + + if "encoder_proj.weight" in unet_state_dict: + new_checkpoint["encoder_hid_proj.weight"] = unet_state_dict.pop("encoder_proj.weight") + new_checkpoint["encoder_hid_proj.bias"] = unet_state_dict.pop("encoder_proj.bias") + + if "encoder_pooling.0.weight" in unet_state_dict: + new_checkpoint["add_embedding.norm1.weight"] = unet_state_dict.pop("encoder_pooling.0.weight") + new_checkpoint["add_embedding.norm1.bias"] = unet_state_dict.pop("encoder_pooling.0.bias") + + new_checkpoint["add_embedding.pool.positional_embedding"] = unet_state_dict.pop( + "encoder_pooling.1.positional_embedding" + ) + new_checkpoint["add_embedding.pool.k_proj.weight"] = unet_state_dict.pop("encoder_pooling.1.k_proj.weight") + new_checkpoint["add_embedding.pool.k_proj.bias"] = unet_state_dict.pop("encoder_pooling.1.k_proj.bias") + new_checkpoint["add_embedding.pool.q_proj.weight"] = unet_state_dict.pop("encoder_pooling.1.q_proj.weight") + new_checkpoint["add_embedding.pool.q_proj.bias"] = unet_state_dict.pop("encoder_pooling.1.q_proj.bias") + new_checkpoint["add_embedding.pool.v_proj.weight"] = unet_state_dict.pop("encoder_pooling.1.v_proj.weight") + new_checkpoint["add_embedding.pool.v_proj.bias"] = unet_state_dict.pop("encoder_pooling.1.v_proj.bias") + + new_checkpoint["add_embedding.proj.weight"] = unet_state_dict.pop("encoder_pooling.2.weight") + new_checkpoint["add_embedding.proj.bias"] = unet_state_dict.pop("encoder_pooling.2.bias") + + new_checkpoint["add_embedding.norm2.weight"] = unet_state_dict.pop("encoder_pooling.3.weight") + new_checkpoint["add_embedding.norm2.bias"] = unet_state_dict.pop("encoder_pooling.3.bias") + + return new_checkpoint + + +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("emb_layers.1", "time_emb_proj") + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + if "qkv" in new_item: + continue + + if "encoder_kv" in new_item: + continue + + new_item = new_item.replace("norm.weight", "group_norm.weight") + new_item = new_item.replace("norm.bias", "group_norm.bias") + + new_item = new_item.replace("proj_out.weight", "to_out.0.weight") + new_item = new_item.replace("proj_out.bias", "to_out.0.bias") + + new_item = new_item.replace("norm_encoder.weight", "norm_cross.weight") + new_item = new_item.replace("norm_encoder.bias", "norm_cross.bias") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def assign_attention_to_checkpoint(new_checkpoint, unet_state_dict, old_path, new_path, config): + qkv_weight = unet_state_dict.pop(f"{old_path}.qkv.weight") + qkv_weight = qkv_weight[:, :, 0] + + qkv_bias = unet_state_dict.pop(f"{old_path}.qkv.bias") + + is_cross_attn_only = "only_cross_attention" in config and config["only_cross_attention"] + + split = 1 if is_cross_attn_only else 3 + + weights, bias = split_attentions( + weight=qkv_weight, + bias=qkv_bias, + split=split, + chunk_size=config["attention_head_dim"], + ) + + if is_cross_attn_only: + query_weight, q_bias = weights, bias + new_checkpoint[f"{new_path}.to_q.weight"] = query_weight[0] + new_checkpoint[f"{new_path}.to_q.bias"] = q_bias[0] + else: + [query_weight, key_weight, value_weight], [q_bias, k_bias, v_bias] = weights, bias + new_checkpoint[f"{new_path}.to_q.weight"] = query_weight + new_checkpoint[f"{new_path}.to_q.bias"] = q_bias + new_checkpoint[f"{new_path}.to_k.weight"] = key_weight + new_checkpoint[f"{new_path}.to_k.bias"] = k_bias + new_checkpoint[f"{new_path}.to_v.weight"] = value_weight + new_checkpoint[f"{new_path}.to_v.bias"] = v_bias + + encoder_kv_weight = unet_state_dict.pop(f"{old_path}.encoder_kv.weight") + encoder_kv_weight = encoder_kv_weight[:, :, 0] + + encoder_kv_bias = unet_state_dict.pop(f"{old_path}.encoder_kv.bias") + + [encoder_k_weight, encoder_v_weight], [encoder_k_bias, encoder_v_bias] = split_attentions( + weight=encoder_kv_weight, + bias=encoder_kv_bias, + split=2, + chunk_size=config["attention_head_dim"], + ) + + new_checkpoint[f"{new_path}.add_k_proj.weight"] = encoder_k_weight + new_checkpoint[f"{new_path}.add_k_proj.bias"] = encoder_k_bias + new_checkpoint[f"{new_path}.add_v_proj.weight"] = encoder_v_weight + new_checkpoint[f"{new_path}.add_v_proj.bias"] = encoder_v_bias + + +def assign_to_checkpoint(paths, checkpoint, old_checkpoint, additional_replacements=None, config=None): + """ + This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits + attention layers, and takes into account additional replacements that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + for path in paths: + new_path = path["new"] + + # Global renaming happens here + new_path = new_path.replace("middle_block.0", "mid_block.resnets.0") + new_path = new_path.replace("middle_block.1", "mid_block.attentions.0") + new_path = new_path.replace("middle_block.2", "mid_block.resnets.1") + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + # proj_attn.weight has to be converted from conv 1D to linear + if "proj_attn.weight" in new_path or "to_out.0.weight" in new_path: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0] + else: + checkpoint[new_path] = old_checkpoint[path["old"]] + + +# TODO maybe document and/or can do more efficiently (build indices in for loop and extract once for each split?) +def split_attentions(*, weight, bias, split, chunk_size): + weights = [None] * split + biases = [None] * split + + weights_biases_idx = 0 + + for starting_row_index in range(0, weight.shape[0], chunk_size): + row_indices = torch.arange(starting_row_index, starting_row_index + chunk_size) + + weight_rows = weight[row_indices, :] + bias_rows = bias[row_indices] + + if weights[weights_biases_idx] is None: + weights[weights_biases_idx] = weight_rows + biases[weights_biases_idx] = bias_rows + else: + assert weights[weights_biases_idx] is not None + weights[weights_biases_idx] = torch.concat([weights[weights_biases_idx], weight_rows]) + biases[weights_biases_idx] = torch.concat([biases[weights_biases_idx], bias_rows]) + + weights_biases_idx = (weights_biases_idx + 1) % split + + return weights, biases + + +def parse_list(value): + if isinstance(value, str): + value = value.split(",") + value = [int(v) for v in value] + elif isinstance(value, list): + pass + else: + raise ValueError(f"Can't parse list for type: {type(value)}") + + return value + + +# below is copy and pasted from original convert_if_stage_2.py script + + +def get_super_res_unet(unet_checkpoint_path, verify_param_count=True, sample_size=None): + orig_path = unet_checkpoint_path + + original_unet_config = yaml.safe_load(os.path.join(orig_path, "config.yml")) + original_unet_config = original_unet_config["params"] + + unet_diffusers_config = superres_create_unet_diffusers_config(original_unet_config) + unet_diffusers_config["time_embedding_dim"] = original_unet_config["model_channels"] * int( + original_unet_config["channel_mult"].split(",")[-1] + ) + if original_unet_config["encoder_dim"] != original_unet_config["encoder_channels"]: + unet_diffusers_config["encoder_hid_dim"] = original_unet_config["encoder_dim"] + unet_diffusers_config["class_embed_type"] = "timestep" + unet_diffusers_config["addition_embed_type"] = "text" + + unet_diffusers_config["time_embedding_act_fn"] = "gelu" + unet_diffusers_config["resnet_skip_time_act"] = True + unet_diffusers_config["resnet_out_scale_factor"] = 1 / 0.7071 + unet_diffusers_config["mid_block_scale_factor"] = 1 / 0.7071 + unet_diffusers_config["only_cross_attention"] = ( + bool(original_unet_config["disable_self_attentions"]) + if ( + "disable_self_attentions" in original_unet_config + and isinstance(original_unet_config["disable_self_attentions"], int) + ) + else True + ) + + if sample_size is None: + unet_diffusers_config["sample_size"] = original_unet_config["image_size"] + else: + # The second upscaler unet's sample size is incorrectly specified + # in the config and is instead hardcoded in source + unet_diffusers_config["sample_size"] = sample_size + + unet_checkpoint = torch.load(os.path.join(unet_checkpoint_path, "pytorch_model.bin"), map_location="cpu") + + if verify_param_count: + # check that architecture matches - is a bit slow + verify_param_count(orig_path, unet_diffusers_config) + + converted_unet_checkpoint = superres_convert_ldm_unet_checkpoint( + unet_checkpoint, unet_diffusers_config, path=unet_checkpoint_path + ) + converted_keys = converted_unet_checkpoint.keys() + + model = UNet2DConditionModel(**unet_diffusers_config) + expected_weights = model.state_dict().keys() + + diff_c_e = set(converted_keys) - set(expected_weights) + diff_e_c = set(expected_weights) - set(converted_keys) + + assert len(diff_e_c) == 0, f"Expected, but not converted: {diff_e_c}" + assert len(diff_c_e) == 0, f"Converted, but not expected: {diff_c_e}" + + model.load_state_dict(converted_unet_checkpoint) + + return model + + +def superres_create_unet_diffusers_config(original_unet_config): + attention_resolutions = parse_list(original_unet_config["attention_resolutions"]) + attention_resolutions = [original_unet_config["image_size"] // int(res) for res in attention_resolutions] + + channel_mult = parse_list(original_unet_config["channel_mult"]) + block_out_channels = [original_unet_config["model_channels"] * mult for mult in channel_mult] + + down_block_types = [] + resolution = 1 + + for i in range(len(block_out_channels)): + if resolution in attention_resolutions: + block_type = "SimpleCrossAttnDownBlock2D" + elif original_unet_config["resblock_updown"]: + block_type = "ResnetDownsampleBlock2D" + else: + block_type = "DownBlock2D" + + down_block_types.append(block_type) + + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + if resolution in attention_resolutions: + block_type = "SimpleCrossAttnUpBlock2D" + elif original_unet_config["resblock_updown"]: + block_type = "ResnetUpsampleBlock2D" + else: + block_type = "UpBlock2D" + up_block_types.append(block_type) + resolution //= 2 + + head_dim = original_unet_config["num_head_channels"] + use_linear_projection = ( + original_unet_config["use_linear_in_transformer"] + if "use_linear_in_transformer" in original_unet_config + else False + ) + if use_linear_projection: + # stable diffusion 2-base-512 and 2-768 + if head_dim is None: + head_dim = [5, 10, 20, 20] + + class_embed_type = None + projection_class_embeddings_input_dim = None + + if "num_classes" in original_unet_config: + if original_unet_config["num_classes"] == "sequential": + class_embed_type = "projection" + assert "adm_in_channels" in original_unet_config + projection_class_embeddings_input_dim = original_unet_config["adm_in_channels"] + else: + raise NotImplementedError( + f"Unknown conditional unet num_classes config: {original_unet_config['num_classes']}" + ) + + config = { + "in_channels": original_unet_config["in_channels"], + "down_block_types": tuple(down_block_types), + "block_out_channels": tuple(block_out_channels), + "layers_per_block": tuple(original_unet_config["num_res_blocks"]), + "cross_attention_dim": original_unet_config["encoder_channels"], + "attention_head_dim": head_dim, + "use_linear_projection": use_linear_projection, + "class_embed_type": class_embed_type, + "projection_class_embeddings_input_dim": projection_class_embeddings_input_dim, + "out_channels": original_unet_config["out_channels"], + "up_block_types": tuple(up_block_types), + "upcast_attention": False, # TODO: guessing + "cross_attention_norm": "group_norm", + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "act_fn": "gelu", + } + + if original_unet_config["use_scale_shift_norm"]: + config["resnet_time_scale_shift"] = "scale_shift" + + return config + + +def superres_convert_ldm_unet_checkpoint(unet_state_dict, config, path=None, extract_ema=False): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"] + + if config["class_embed_type"] is None: + # No parameters to port + ... + elif config["class_embed_type"] == "timestep" or config["class_embed_type"] == "projection": + new_checkpoint["class_embedding.linear_1.weight"] = unet_state_dict["aug_proj.0.weight"] + new_checkpoint["class_embedding.linear_1.bias"] = unet_state_dict["aug_proj.0.bias"] + new_checkpoint["class_embedding.linear_2.weight"] = unet_state_dict["aug_proj.2.weight"] + new_checkpoint["class_embedding.linear_2.bias"] = unet_state_dict["aug_proj.2.bias"] + else: + raise NotImplementedError(f"Not implemented `class_embed_type`: {config['class_embed_type']}") + + if "encoder_proj.weight" in unet_state_dict: + new_checkpoint["encoder_hid_proj.weight"] = unet_state_dict["encoder_proj.weight"] + new_checkpoint["encoder_hid_proj.bias"] = unet_state_dict["encoder_proj.bias"] + + if "encoder_pooling.0.weight" in unet_state_dict: + mapping = { + "encoder_pooling.0": "add_embedding.norm1", + "encoder_pooling.1": "add_embedding.pool", + "encoder_pooling.2": "add_embedding.proj", + "encoder_pooling.3": "add_embedding.norm2", + } + for key in unet_state_dict.keys(): + if key.startswith("encoder_pooling"): + prefix = key[: len("encoder_pooling.0")] + new_key = key.replace(prefix, mapping[prefix]) + new_checkpoint[new_key] = unet_state_dict[key] + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}." in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}." in key] + for layer_id in range(num_output_blocks) + } + if not isinstance(config["layers_per_block"], int): + layers_per_block_list = [e + 1 for e in config["layers_per_block"]] + layers_per_block_cumsum = list(np.cumsum(layers_per_block_list)) + downsampler_ids = layers_per_block_cumsum + else: + # TODO need better check than i in [4, 8, 12, 16] + downsampler_ids = [4, 8, 12, 16] + + for i in range(1, num_input_blocks): + if isinstance(config["layers_per_block"], int): + layers_per_block = config["layers_per_block"] + block_id = (i - 1) // (layers_per_block + 1) + layer_in_block_id = (i - 1) % (layers_per_block + 1) + else: + block_id = next(k for k, n in enumerate(layers_per_block_cumsum) if (i - 1) < n) + passed_blocks = layers_per_block_cumsum[block_id - 1] if block_id > 0 else 0 + layer_in_block_id = (i - 1) - passed_blocks + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + + if f"input_blocks.{i}.0.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + + paths = renew_resnet_paths(resnets) + + block_type = config["down_block_types"][block_id] + if ( + block_type == "ResnetDownsampleBlock2D" or block_type == "SimpleCrossAttnDownBlock2D" + ) and i in downsampler_ids: + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.downsamplers.0"} + else: + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(attentions): + old_path = f"input_blocks.{i}.1" + new_path = f"down_blocks.{block_id}.attentions.{layer_in_block_id}" + + assign_attention_to_checkpoint( + new_checkpoint=new_checkpoint, + unet_state_dict=unet_state_dict, + old_path=old_path, + new_path=new_path, + config=config, + ) + + paths = renew_attention_paths(attentions) + meta_path = {"old": old_path, "new": new_path} + assign_to_checkpoint( + paths, + new_checkpoint, + unet_state_dict, + additional_replacements=[meta_path], + config=config, + ) + + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + resnet_0_paths = renew_resnet_paths(resnet_0) + assign_to_checkpoint(resnet_0_paths, new_checkpoint, unet_state_dict, config=config) + + resnet_1_paths = renew_resnet_paths(resnet_1) + assign_to_checkpoint(resnet_1_paths, new_checkpoint, unet_state_dict, config=config) + + old_path = "middle_block.1" + new_path = "mid_block.attentions.0" + + assign_attention_to_checkpoint( + new_checkpoint=new_checkpoint, + unet_state_dict=unet_state_dict, + old_path=old_path, + new_path=new_path, + config=config, + ) + + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + if not isinstance(config["layers_per_block"], int): + layers_per_block_list = list(reversed([e + 1 for e in config["layers_per_block"]])) + layers_per_block_cumsum = list(np.cumsum(layers_per_block_list)) + + for i in range(num_output_blocks): + if isinstance(config["layers_per_block"], int): + layers_per_block = config["layers_per_block"] + block_id = i // (layers_per_block + 1) + layer_in_block_id = i % (layers_per_block + 1) + else: + block_id = next(k for k, n in enumerate(layers_per_block_cumsum) if i < n) + passed_blocks = layers_per_block_cumsum[block_id - 1] if block_id > 0 else 0 + layer_in_block_id = i - passed_blocks + + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + # len(output_block_list) == 1 -> resnet + # len(output_block_list) == 2 -> resnet, attention or resnet, upscale resnet + # len(output_block_list) == 3 -> resnet, attention, upscale resnet + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + + has_attention = True + if len(output_block_list) == 2 and any("in_layers" in k for k in output_block_list["1"]): + has_attention = False + + maybe_attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key] + + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + output_block_list = {k: sorted(v) for k, v in output_block_list.items()} + if ["conv.bias", "conv.weight"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.bias", "conv.weight"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + # this layer was no attention + has_attention = False + maybe_attentions = [] + + if has_attention: + old_path = f"output_blocks.{i}.1" + new_path = f"up_blocks.{block_id}.attentions.{layer_in_block_id}" + + assign_attention_to_checkpoint( + new_checkpoint=new_checkpoint, + unet_state_dict=unet_state_dict, + old_path=old_path, + new_path=new_path, + config=config, + ) + + paths = renew_attention_paths(maybe_attentions) + meta_path = { + "old": old_path, + "new": new_path, + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(output_block_list) == 3 or (not has_attention and len(maybe_attentions) > 0): + layer_id = len(output_block_list) - 1 + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.{layer_id}" in key] + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"output_blocks.{i}.{layer_id}", "new": f"up_blocks.{block_id}.upsamplers.0"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + + new_checkpoint[new_path] = unet_state_dict[old_path] + + return new_checkpoint + + +def verify_param_count(orig_path, unet_diffusers_config): + if "-II-" in orig_path: + from deepfloyd_if.modules import IFStageII + + if_II = IFStageII(device="cpu", dir_or_name=orig_path) + elif "-III-" in orig_path: + from deepfloyd_if.modules import IFStageIII + + if_II = IFStageIII(device="cpu", dir_or_name=orig_path) + else: + assert f"Weird name. Should have -II- or -III- in path: {orig_path}" + + unet = UNet2DConditionModel(**unet_diffusers_config) + + # in params + assert_param_count(unet.time_embedding, if_II.model.time_embed) + assert_param_count(unet.conv_in, if_II.model.input_blocks[:1]) + + # downblocks + assert_param_count(unet.down_blocks[0], if_II.model.input_blocks[1:4]) + assert_param_count(unet.down_blocks[1], if_II.model.input_blocks[4:7]) + assert_param_count(unet.down_blocks[2], if_II.model.input_blocks[7:11]) + + if "-II-" in orig_path: + assert_param_count(unet.down_blocks[3], if_II.model.input_blocks[11:17]) + assert_param_count(unet.down_blocks[4], if_II.model.input_blocks[17:]) + if "-III-" in orig_path: + assert_param_count(unet.down_blocks[3], if_II.model.input_blocks[11:15]) + assert_param_count(unet.down_blocks[4], if_II.model.input_blocks[15:20]) + assert_param_count(unet.down_blocks[5], if_II.model.input_blocks[20:]) + + # mid block + assert_param_count(unet.mid_block, if_II.model.middle_block) + + # up block + if "-II-" in orig_path: + assert_param_count(unet.up_blocks[0], if_II.model.output_blocks[:6]) + assert_param_count(unet.up_blocks[1], if_II.model.output_blocks[6:12]) + assert_param_count(unet.up_blocks[2], if_II.model.output_blocks[12:16]) + assert_param_count(unet.up_blocks[3], if_II.model.output_blocks[16:19]) + assert_param_count(unet.up_blocks[4], if_II.model.output_blocks[19:]) + if "-III-" in orig_path: + assert_param_count(unet.up_blocks[0], if_II.model.output_blocks[:5]) + assert_param_count(unet.up_blocks[1], if_II.model.output_blocks[5:10]) + assert_param_count(unet.up_blocks[2], if_II.model.output_blocks[10:14]) + assert_param_count(unet.up_blocks[3], if_II.model.output_blocks[14:18]) + assert_param_count(unet.up_blocks[4], if_II.model.output_blocks[18:21]) + assert_param_count(unet.up_blocks[5], if_II.model.output_blocks[21:24]) + + # out params + assert_param_count(unet.conv_norm_out, if_II.model.out[0]) + assert_param_count(unet.conv_out, if_II.model.out[2]) + + # make sure all model architecture has same param count + assert_param_count(unet, if_II.model) + + +def assert_param_count(model_1, model_2): + count_1 = sum(p.numel() for p in model_1.parameters()) + count_2 = sum(p.numel() for p in model_2.parameters()) + assert count_1 == count_2, f"{model_1.__class__}: {count_1} != {model_2.__class__}: {count_2}" + + +def superres_check_against_original(dump_path, unet_checkpoint_path): + model_path = dump_path + model = UNet2DConditionModel.from_pretrained(model_path) + model.to("cuda") + orig_path = unet_checkpoint_path + + if "-II-" in orig_path: + from deepfloyd_if.modules import IFStageII + + if_II_model = IFStageII(device="cuda", dir_or_name=orig_path, model_kwargs={"precision": "fp32"}).model + elif "-III-" in orig_path: + from deepfloyd_if.modules import IFStageIII + + if_II_model = IFStageIII(device="cuda", dir_or_name=orig_path, model_kwargs={"precision": "fp32"}).model + + batch_size = 1 + channels = model.in_channels // 2 + height = model.sample_size + width = model.sample_size + height = 1024 + width = 1024 + + torch.manual_seed(0) + + latents = torch.randn((batch_size, channels, height, width), device=model.device) + image_small = torch.randn((batch_size, channels, height // 4, width // 4), device=model.device) + + interpolate_antialias = {} + if "antialias" in inspect.signature(F.interpolate).parameters: + interpolate_antialias["antialias"] = True + image_upscaled = F.interpolate( + image_small, size=[height, width], mode="bicubic", align_corners=False, **interpolate_antialias + ) + + latent_model_input = torch.cat([latents, image_upscaled], dim=1).to(model.dtype) + t = torch.tensor([5], device=model.device).to(model.dtype) + + seq_len = 64 + encoder_hidden_states = torch.randn((batch_size, seq_len, model.config.encoder_hid_dim), device=model.device).to( + model.dtype + ) + + fake_class_labels = torch.tensor([t], device=model.device).to(model.dtype) + + with torch.no_grad(): + out = if_II_model(latent_model_input, t, aug_steps=fake_class_labels, text_emb=encoder_hidden_states) + + if_II_model.to("cpu") + del if_II_model + import gc + + torch.cuda.empty_cache() + gc.collect() + print(50 * "=") + + with torch.no_grad(): + noise_pred = model( + sample=latent_model_input, + encoder_hidden_states=encoder_hidden_states, + class_labels=fake_class_labels, + timestep=t, + ).sample + + print("Out shape", noise_pred.shape) + print("Diff", (out - noise_pred).abs().sum()) + + +if __name__ == "__main__": + main(parse_args()) diff --git a/diffusers-0.27.0/scripts/convert_k_upscaler_to_diffusers.py b/diffusers-0.27.0/scripts/convert_k_upscaler_to_diffusers.py new file mode 100755 index 0000000..62abedd --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_k_upscaler_to_diffusers.py @@ -0,0 +1,297 @@ +import argparse + +import huggingface_hub +import k_diffusion as K +import torch + +from diffusers import UNet2DConditionModel + + +UPSCALER_REPO = "pcuenq/k-upscaler" + + +def resnet_to_diffusers_checkpoint(resnet, checkpoint, *, diffusers_resnet_prefix, resnet_prefix): + rv = { + # norm1 + f"{diffusers_resnet_prefix}.norm1.linear.weight": checkpoint[f"{resnet_prefix}.main.0.mapper.weight"], + f"{diffusers_resnet_prefix}.norm1.linear.bias": checkpoint[f"{resnet_prefix}.main.0.mapper.bias"], + # conv1 + f"{diffusers_resnet_prefix}.conv1.weight": checkpoint[f"{resnet_prefix}.main.2.weight"], + f"{diffusers_resnet_prefix}.conv1.bias": checkpoint[f"{resnet_prefix}.main.2.bias"], + # norm2 + f"{diffusers_resnet_prefix}.norm2.linear.weight": checkpoint[f"{resnet_prefix}.main.4.mapper.weight"], + f"{diffusers_resnet_prefix}.norm2.linear.bias": checkpoint[f"{resnet_prefix}.main.4.mapper.bias"], + # conv2 + f"{diffusers_resnet_prefix}.conv2.weight": checkpoint[f"{resnet_prefix}.main.6.weight"], + f"{diffusers_resnet_prefix}.conv2.bias": checkpoint[f"{resnet_prefix}.main.6.bias"], + } + + if resnet.conv_shortcut is not None: + rv.update( + { + f"{diffusers_resnet_prefix}.conv_shortcut.weight": checkpoint[f"{resnet_prefix}.skip.weight"], + } + ) + + return rv + + +def self_attn_to_diffusers_checkpoint(checkpoint, *, diffusers_attention_prefix, attention_prefix): + weight_q, weight_k, weight_v = checkpoint[f"{attention_prefix}.qkv_proj.weight"].chunk(3, dim=0) + bias_q, bias_k, bias_v = checkpoint[f"{attention_prefix}.qkv_proj.bias"].chunk(3, dim=0) + rv = { + # norm + f"{diffusers_attention_prefix}.norm1.linear.weight": checkpoint[f"{attention_prefix}.norm_in.mapper.weight"], + f"{diffusers_attention_prefix}.norm1.linear.bias": checkpoint[f"{attention_prefix}.norm_in.mapper.bias"], + # to_q + f"{diffusers_attention_prefix}.attn1.to_q.weight": weight_q.squeeze(-1).squeeze(-1), + f"{diffusers_attention_prefix}.attn1.to_q.bias": bias_q, + # to_k + f"{diffusers_attention_prefix}.attn1.to_k.weight": weight_k.squeeze(-1).squeeze(-1), + f"{diffusers_attention_prefix}.attn1.to_k.bias": bias_k, + # to_v + f"{diffusers_attention_prefix}.attn1.to_v.weight": weight_v.squeeze(-1).squeeze(-1), + f"{diffusers_attention_prefix}.attn1.to_v.bias": bias_v, + # to_out + f"{diffusers_attention_prefix}.attn1.to_out.0.weight": checkpoint[f"{attention_prefix}.out_proj.weight"] + .squeeze(-1) + .squeeze(-1), + f"{diffusers_attention_prefix}.attn1.to_out.0.bias": checkpoint[f"{attention_prefix}.out_proj.bias"], + } + + return rv + + +def cross_attn_to_diffusers_checkpoint( + checkpoint, *, diffusers_attention_prefix, diffusers_attention_index, attention_prefix +): + weight_k, weight_v = checkpoint[f"{attention_prefix}.kv_proj.weight"].chunk(2, dim=0) + bias_k, bias_v = checkpoint[f"{attention_prefix}.kv_proj.bias"].chunk(2, dim=0) + + rv = { + # norm2 (ada groupnorm) + f"{diffusers_attention_prefix}.norm{diffusers_attention_index}.linear.weight": checkpoint[ + f"{attention_prefix}.norm_dec.mapper.weight" + ], + f"{diffusers_attention_prefix}.norm{diffusers_attention_index}.linear.bias": checkpoint[ + f"{attention_prefix}.norm_dec.mapper.bias" + ], + # layernorm on encoder_hidden_state + f"{diffusers_attention_prefix}.attn{diffusers_attention_index}.norm_cross.weight": checkpoint[ + f"{attention_prefix}.norm_enc.weight" + ], + f"{diffusers_attention_prefix}.attn{diffusers_attention_index}.norm_cross.bias": checkpoint[ + f"{attention_prefix}.norm_enc.bias" + ], + # to_q + f"{diffusers_attention_prefix}.attn{diffusers_attention_index}.to_q.weight": checkpoint[ + f"{attention_prefix}.q_proj.weight" + ] + .squeeze(-1) + .squeeze(-1), + f"{diffusers_attention_prefix}.attn{diffusers_attention_index}.to_q.bias": checkpoint[ + f"{attention_prefix}.q_proj.bias" + ], + # to_k + f"{diffusers_attention_prefix}.attn{diffusers_attention_index}.to_k.weight": weight_k.squeeze(-1).squeeze(-1), + f"{diffusers_attention_prefix}.attn{diffusers_attention_index}.to_k.bias": bias_k, + # to_v + f"{diffusers_attention_prefix}.attn{diffusers_attention_index}.to_v.weight": weight_v.squeeze(-1).squeeze(-1), + f"{diffusers_attention_prefix}.attn{diffusers_attention_index}.to_v.bias": bias_v, + # to_out + f"{diffusers_attention_prefix}.attn{diffusers_attention_index}.to_out.0.weight": checkpoint[ + f"{attention_prefix}.out_proj.weight" + ] + .squeeze(-1) + .squeeze(-1), + f"{diffusers_attention_prefix}.attn{diffusers_attention_index}.to_out.0.bias": checkpoint[ + f"{attention_prefix}.out_proj.bias" + ], + } + + return rv + + +def block_to_diffusers_checkpoint(block, checkpoint, block_idx, block_type): + block_prefix = "inner_model.u_net.u_blocks" if block_type == "up" else "inner_model.u_net.d_blocks" + block_prefix = f"{block_prefix}.{block_idx}" + + diffusers_checkpoint = {} + + if not hasattr(block, "attentions"): + n = 1 # resnet only + elif not block.attentions[0].add_self_attention: + n = 2 # resnet -> cross-attention + else: + n = 3 # resnet -> self-attention -> cross-attention) + + for resnet_idx, resnet in enumerate(block.resnets): + # diffusers_resnet_prefix = f"{diffusers_up_block_prefix}.resnets.{resnet_idx}" + diffusers_resnet_prefix = f"{block_type}_blocks.{block_idx}.resnets.{resnet_idx}" + idx = n * resnet_idx if block_type == "up" else n * resnet_idx + 1 + resnet_prefix = f"{block_prefix}.{idx}" if block_type == "up" else f"{block_prefix}.{idx}" + + diffusers_checkpoint.update( + resnet_to_diffusers_checkpoint( + resnet, checkpoint, diffusers_resnet_prefix=diffusers_resnet_prefix, resnet_prefix=resnet_prefix + ) + ) + + if hasattr(block, "attentions"): + for attention_idx, attention in enumerate(block.attentions): + diffusers_attention_prefix = f"{block_type}_blocks.{block_idx}.attentions.{attention_idx}" + idx = n * attention_idx + 1 if block_type == "up" else n * attention_idx + 2 + self_attention_prefix = f"{block_prefix}.{idx}" + cross_attention_prefix = f"{block_prefix}.{idx }" + cross_attention_index = 1 if not attention.add_self_attention else 2 + idx = ( + n * attention_idx + cross_attention_index + if block_type == "up" + else n * attention_idx + cross_attention_index + 1 + ) + cross_attention_prefix = f"{block_prefix}.{idx }" + + diffusers_checkpoint.update( + cross_attn_to_diffusers_checkpoint( + checkpoint, + diffusers_attention_prefix=diffusers_attention_prefix, + diffusers_attention_index=2, + attention_prefix=cross_attention_prefix, + ) + ) + + if attention.add_self_attention is True: + diffusers_checkpoint.update( + self_attn_to_diffusers_checkpoint( + checkpoint, + diffusers_attention_prefix=diffusers_attention_prefix, + attention_prefix=self_attention_prefix, + ) + ) + + return diffusers_checkpoint + + +def unet_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + # pre-processing + diffusers_checkpoint.update( + { + "conv_in.weight": checkpoint["inner_model.proj_in.weight"], + "conv_in.bias": checkpoint["inner_model.proj_in.bias"], + } + ) + + # timestep and class embedding + diffusers_checkpoint.update( + { + "time_proj.weight": checkpoint["inner_model.timestep_embed.weight"].squeeze(-1), + "time_embedding.linear_1.weight": checkpoint["inner_model.mapping.0.weight"], + "time_embedding.linear_1.bias": checkpoint["inner_model.mapping.0.bias"], + "time_embedding.linear_2.weight": checkpoint["inner_model.mapping.2.weight"], + "time_embedding.linear_2.bias": checkpoint["inner_model.mapping.2.bias"], + "time_embedding.cond_proj.weight": checkpoint["inner_model.mapping_cond.weight"], + } + ) + + # down_blocks + for down_block_idx, down_block in enumerate(model.down_blocks): + diffusers_checkpoint.update(block_to_diffusers_checkpoint(down_block, checkpoint, down_block_idx, "down")) + + # up_blocks + for up_block_idx, up_block in enumerate(model.up_blocks): + diffusers_checkpoint.update(block_to_diffusers_checkpoint(up_block, checkpoint, up_block_idx, "up")) + + # post-processing + diffusers_checkpoint.update( + { + "conv_out.weight": checkpoint["inner_model.proj_out.weight"], + "conv_out.bias": checkpoint["inner_model.proj_out.bias"], + } + ) + + return diffusers_checkpoint + + +def unet_model_from_original_config(original_config): + in_channels = original_config["input_channels"] + original_config["unet_cond_dim"] + out_channels = original_config["input_channels"] + (1 if original_config["has_variance"] else 0) + + block_out_channels = original_config["channels"] + + assert ( + len(set(original_config["depths"])) == 1 + ), "UNet2DConditionModel currently do not support blocks with different number of layers" + layers_per_block = original_config["depths"][0] + + class_labels_dim = original_config["mapping_cond_dim"] + cross_attention_dim = original_config["cross_cond_dim"] + + attn1_types = [] + attn2_types = [] + for s, c in zip(original_config["self_attn_depths"], original_config["cross_attn_depths"]): + if s: + a1 = "self" + a2 = "cross" if c else None + elif c: + a1 = "cross" + a2 = None + else: + a1 = None + a2 = None + attn1_types.append(a1) + attn2_types.append(a2) + + unet = UNet2DConditionModel( + in_channels=in_channels, + out_channels=out_channels, + down_block_types=("KDownBlock2D", "KCrossAttnDownBlock2D", "KCrossAttnDownBlock2D", "KCrossAttnDownBlock2D"), + mid_block_type=None, + up_block_types=("KCrossAttnUpBlock2D", "KCrossAttnUpBlock2D", "KCrossAttnUpBlock2D", "KUpBlock2D"), + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn="gelu", + norm_num_groups=None, + cross_attention_dim=cross_attention_dim, + attention_head_dim=64, + time_cond_proj_dim=class_labels_dim, + resnet_time_scale_shift="scale_shift", + time_embedding_type="fourier", + timestep_post_act="gelu", + conv_in_kernel=1, + conv_out_kernel=1, + ) + + return unet + + +def main(args): + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + orig_config_path = huggingface_hub.hf_hub_download(UPSCALER_REPO, "config_laion_text_cond_latent_upscaler_2.json") + orig_weights_path = huggingface_hub.hf_hub_download( + UPSCALER_REPO, "laion_text_cond_latent_upscaler_2_1_00470000_slim.pth" + ) + print(f"loading original model configuration from {orig_config_path}") + print(f"loading original model checkpoint from {orig_weights_path}") + + print("converting to diffusers unet") + orig_config = K.config.load_config(open(orig_config_path))["model"] + model = unet_model_from_original_config(orig_config) + + orig_checkpoint = torch.load(orig_weights_path, map_location=device)["model_ema"] + converted_checkpoint = unet_to_diffusers_checkpoint(model, orig_checkpoint) + + model.load_state_dict(converted_checkpoint, strict=True) + model.save_pretrained(args.dump_path) + print(f"saving converted unet model in {args.dump_path}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + args = parser.parse_args() + + main(args) diff --git a/diffusers-0.27.0/scripts/convert_kakao_brain_unclip_to_diffusers.py b/diffusers-0.27.0/scripts/convert_kakao_brain_unclip_to_diffusers.py new file mode 100755 index 0000000..5135eae --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_kakao_brain_unclip_to_diffusers.py @@ -0,0 +1,1159 @@ +import argparse +import tempfile + +import torch +from accelerate import load_checkpoint_and_dispatch +from transformers import CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import UnCLIPPipeline, UNet2DConditionModel, UNet2DModel +from diffusers.models.transformers.prior_transformer import PriorTransformer +from diffusers.pipelines.unclip.text_proj import UnCLIPTextProjModel +from diffusers.schedulers.scheduling_unclip import UnCLIPScheduler + + +r""" +Example - From the diffusers root directory: + +Download weights: +```sh +$ wget https://arena.kakaocdn.net/brainrepo/models/karlo-public/v1.0.0.alpha/efdf6206d8ed593961593dc029a8affa/decoder-ckpt-step%3D01000000-of-01000000.ckpt +$ wget https://arena.kakaocdn.net/brainrepo/models/karlo-public/v1.0.0.alpha/4226b831ae0279020d134281f3c31590/improved-sr-ckpt-step%3D1.2M.ckpt +$ wget https://arena.kakaocdn.net/brainrepo/models/karlo-public/v1.0.0.alpha/85626483eaca9f581e2a78d31ff905ca/prior-ckpt-step%3D01000000-of-01000000.ckpt +$ wget https://arena.kakaocdn.net/brainrepo/models/karlo-public/v1.0.0.alpha/0b62380a75e56f073e2844ab5199153d/ViT-L-14_stats.th +``` + +Convert the model: +```sh +$ python scripts/convert_kakao_brain_unclip_to_diffusers.py \ + --decoder_checkpoint_path ./decoder-ckpt-step\=01000000-of-01000000.ckpt \ + --super_res_unet_checkpoint_path ./improved-sr-ckpt-step\=1.2M.ckpt \ + --prior_checkpoint_path ./prior-ckpt-step\=01000000-of-01000000.ckpt \ + --clip_stat_path ./ViT-L-14_stats.th \ + --dump_path +``` +""" + + +# prior + +PRIOR_ORIGINAL_PREFIX = "model" + +# Uses default arguments +PRIOR_CONFIG = {} + + +def prior_model_from_original_config(): + model = PriorTransformer(**PRIOR_CONFIG) + + return model + + +def prior_original_checkpoint_to_diffusers_checkpoint(model, checkpoint, clip_stats_checkpoint): + diffusers_checkpoint = {} + + # .time_embed.0 -> .time_embedding.linear_1 + diffusers_checkpoint.update( + { + "time_embedding.linear_1.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.0.weight"], + "time_embedding.linear_1.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.0.bias"], + } + ) + + # .clip_img_proj -> .proj_in + diffusers_checkpoint.update( + { + "proj_in.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.clip_img_proj.weight"], + "proj_in.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.clip_img_proj.bias"], + } + ) + + # .text_emb_proj -> .embedding_proj + diffusers_checkpoint.update( + { + "embedding_proj.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.text_emb_proj.weight"], + "embedding_proj.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.text_emb_proj.bias"], + } + ) + + # .text_enc_proj -> .encoder_hidden_states_proj + diffusers_checkpoint.update( + { + "encoder_hidden_states_proj.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.text_enc_proj.weight"], + "encoder_hidden_states_proj.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.text_enc_proj.bias"], + } + ) + + # .positional_embedding -> .positional_embedding + diffusers_checkpoint.update({"positional_embedding": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.positional_embedding"]}) + + # .prd_emb -> .prd_embedding + diffusers_checkpoint.update({"prd_embedding": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.prd_emb"]}) + + # .time_embed.2 -> .time_embedding.linear_2 + diffusers_checkpoint.update( + { + "time_embedding.linear_2.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.2.weight"], + "time_embedding.linear_2.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.2.bias"], + } + ) + + # .resblocks. -> .transformer_blocks. + for idx in range(len(model.transformer_blocks)): + diffusers_transformer_prefix = f"transformer_blocks.{idx}" + original_transformer_prefix = f"{PRIOR_ORIGINAL_PREFIX}.transformer.resblocks.{idx}" + + # .attn -> .attn1 + diffusers_attention_prefix = f"{diffusers_transformer_prefix}.attn1" + original_attention_prefix = f"{original_transformer_prefix}.attn" + diffusers_checkpoint.update( + prior_attention_to_diffusers( + checkpoint, + diffusers_attention_prefix=diffusers_attention_prefix, + original_attention_prefix=original_attention_prefix, + attention_head_dim=model.attention_head_dim, + ) + ) + + # .mlp -> .ff + diffusers_ff_prefix = f"{diffusers_transformer_prefix}.ff" + original_ff_prefix = f"{original_transformer_prefix}.mlp" + diffusers_checkpoint.update( + prior_ff_to_diffusers( + checkpoint, diffusers_ff_prefix=diffusers_ff_prefix, original_ff_prefix=original_ff_prefix + ) + ) + + # .ln_1 -> .norm1 + diffusers_checkpoint.update( + { + f"{diffusers_transformer_prefix}.norm1.weight": checkpoint[ + f"{original_transformer_prefix}.ln_1.weight" + ], + f"{diffusers_transformer_prefix}.norm1.bias": checkpoint[f"{original_transformer_prefix}.ln_1.bias"], + } + ) + + # .ln_2 -> .norm3 + diffusers_checkpoint.update( + { + f"{diffusers_transformer_prefix}.norm3.weight": checkpoint[ + f"{original_transformer_prefix}.ln_2.weight" + ], + f"{diffusers_transformer_prefix}.norm3.bias": checkpoint[f"{original_transformer_prefix}.ln_2.bias"], + } + ) + + # .final_ln -> .norm_out + diffusers_checkpoint.update( + { + "norm_out.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.final_ln.weight"], + "norm_out.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.final_ln.bias"], + } + ) + + # .out_proj -> .proj_to_clip_embeddings + diffusers_checkpoint.update( + { + "proj_to_clip_embeddings.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.out_proj.weight"], + "proj_to_clip_embeddings.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.out_proj.bias"], + } + ) + + # clip stats + clip_mean, clip_std = clip_stats_checkpoint + clip_mean = clip_mean[None, :] + clip_std = clip_std[None, :] + + diffusers_checkpoint.update({"clip_mean": clip_mean, "clip_std": clip_std}) + + return diffusers_checkpoint + + +def prior_attention_to_diffusers( + checkpoint, *, diffusers_attention_prefix, original_attention_prefix, attention_head_dim +): + diffusers_checkpoint = {} + + # .c_qkv -> .{to_q, to_k, to_v} + [q_weight, k_weight, v_weight], [q_bias, k_bias, v_bias] = split_attentions( + weight=checkpoint[f"{original_attention_prefix}.c_qkv.weight"], + bias=checkpoint[f"{original_attention_prefix}.c_qkv.bias"], + split=3, + chunk_size=attention_head_dim, + ) + + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.to_q.weight": q_weight, + f"{diffusers_attention_prefix}.to_q.bias": q_bias, + f"{diffusers_attention_prefix}.to_k.weight": k_weight, + f"{diffusers_attention_prefix}.to_k.bias": k_bias, + f"{diffusers_attention_prefix}.to_v.weight": v_weight, + f"{diffusers_attention_prefix}.to_v.bias": v_bias, + } + ) + + # .c_proj -> .to_out.0 + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.to_out.0.weight": checkpoint[f"{original_attention_prefix}.c_proj.weight"], + f"{diffusers_attention_prefix}.to_out.0.bias": checkpoint[f"{original_attention_prefix}.c_proj.bias"], + } + ) + + return diffusers_checkpoint + + +def prior_ff_to_diffusers(checkpoint, *, diffusers_ff_prefix, original_ff_prefix): + diffusers_checkpoint = { + # .c_fc -> .net.0.proj + f"{diffusers_ff_prefix}.net.{0}.proj.weight": checkpoint[f"{original_ff_prefix}.c_fc.weight"], + f"{diffusers_ff_prefix}.net.{0}.proj.bias": checkpoint[f"{original_ff_prefix}.c_fc.bias"], + # .c_proj -> .net.2 + f"{diffusers_ff_prefix}.net.{2}.weight": checkpoint[f"{original_ff_prefix}.c_proj.weight"], + f"{diffusers_ff_prefix}.net.{2}.bias": checkpoint[f"{original_ff_prefix}.c_proj.bias"], + } + + return diffusers_checkpoint + + +# done prior + + +# decoder + +DECODER_ORIGINAL_PREFIX = "model" + +# We are hardcoding the model configuration for now. If we need to generalize to more model configurations, we can +# update then. +DECODER_CONFIG = { + "sample_size": 64, + "layers_per_block": 3, + "down_block_types": ( + "ResnetDownsampleBlock2D", + "SimpleCrossAttnDownBlock2D", + "SimpleCrossAttnDownBlock2D", + "SimpleCrossAttnDownBlock2D", + ), + "up_block_types": ( + "SimpleCrossAttnUpBlock2D", + "SimpleCrossAttnUpBlock2D", + "SimpleCrossAttnUpBlock2D", + "ResnetUpsampleBlock2D", + ), + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "block_out_channels": (320, 640, 960, 1280), + "in_channels": 3, + "out_channels": 6, + "cross_attention_dim": 1536, + "class_embed_type": "identity", + "attention_head_dim": 64, + "resnet_time_scale_shift": "scale_shift", +} + + +def decoder_model_from_original_config(): + model = UNet2DConditionModel(**DECODER_CONFIG) + + return model + + +def decoder_original_checkpoint_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + original_unet_prefix = DECODER_ORIGINAL_PREFIX + num_head_channels = DECODER_CONFIG["attention_head_dim"] + + diffusers_checkpoint.update(unet_time_embeddings(checkpoint, original_unet_prefix)) + diffusers_checkpoint.update(unet_conv_in(checkpoint, original_unet_prefix)) + + # .input_blocks -> .down_blocks + + original_down_block_idx = 1 + + for diffusers_down_block_idx in range(len(model.down_blocks)): + checkpoint_update, num_original_down_blocks = unet_downblock_to_diffusers_checkpoint( + model, + checkpoint, + diffusers_down_block_idx=diffusers_down_block_idx, + original_down_block_idx=original_down_block_idx, + original_unet_prefix=original_unet_prefix, + num_head_channels=num_head_channels, + ) + + original_down_block_idx += num_original_down_blocks + + diffusers_checkpoint.update(checkpoint_update) + + # done .input_blocks -> .down_blocks + + diffusers_checkpoint.update( + unet_midblock_to_diffusers_checkpoint( + model, + checkpoint, + original_unet_prefix=original_unet_prefix, + num_head_channels=num_head_channels, + ) + ) + + # .output_blocks -> .up_blocks + + original_up_block_idx = 0 + + for diffusers_up_block_idx in range(len(model.up_blocks)): + checkpoint_update, num_original_up_blocks = unet_upblock_to_diffusers_checkpoint( + model, + checkpoint, + diffusers_up_block_idx=diffusers_up_block_idx, + original_up_block_idx=original_up_block_idx, + original_unet_prefix=original_unet_prefix, + num_head_channels=num_head_channels, + ) + + original_up_block_idx += num_original_up_blocks + + diffusers_checkpoint.update(checkpoint_update) + + # done .output_blocks -> .up_blocks + + diffusers_checkpoint.update(unet_conv_norm_out(checkpoint, original_unet_prefix)) + diffusers_checkpoint.update(unet_conv_out(checkpoint, original_unet_prefix)) + + return diffusers_checkpoint + + +# done decoder + +# text proj + + +def text_proj_from_original_config(): + # From the conditional unet constructor where the dimension of the projected time embeddings is + # constructed + time_embed_dim = DECODER_CONFIG["block_out_channels"][0] * 4 + + cross_attention_dim = DECODER_CONFIG["cross_attention_dim"] + + model = UnCLIPTextProjModel(time_embed_dim=time_embed_dim, cross_attention_dim=cross_attention_dim) + + return model + + +# Note that the input checkpoint is the original decoder checkpoint +def text_proj_original_checkpoint_to_diffusers_checkpoint(checkpoint): + diffusers_checkpoint = { + # .text_seq_proj.0 -> .encoder_hidden_states_proj + "encoder_hidden_states_proj.weight": checkpoint[f"{DECODER_ORIGINAL_PREFIX}.text_seq_proj.0.weight"], + "encoder_hidden_states_proj.bias": checkpoint[f"{DECODER_ORIGINAL_PREFIX}.text_seq_proj.0.bias"], + # .text_seq_proj.1 -> .text_encoder_hidden_states_norm + "text_encoder_hidden_states_norm.weight": checkpoint[f"{DECODER_ORIGINAL_PREFIX}.text_seq_proj.1.weight"], + "text_encoder_hidden_states_norm.bias": checkpoint[f"{DECODER_ORIGINAL_PREFIX}.text_seq_proj.1.bias"], + # .clip_tok_proj -> .clip_extra_context_tokens_proj + "clip_extra_context_tokens_proj.weight": checkpoint[f"{DECODER_ORIGINAL_PREFIX}.clip_tok_proj.weight"], + "clip_extra_context_tokens_proj.bias": checkpoint[f"{DECODER_ORIGINAL_PREFIX}.clip_tok_proj.bias"], + # .text_feat_proj -> .embedding_proj + "embedding_proj.weight": checkpoint[f"{DECODER_ORIGINAL_PREFIX}.text_feat_proj.weight"], + "embedding_proj.bias": checkpoint[f"{DECODER_ORIGINAL_PREFIX}.text_feat_proj.bias"], + # .cf_param -> .learned_classifier_free_guidance_embeddings + "learned_classifier_free_guidance_embeddings": checkpoint[f"{DECODER_ORIGINAL_PREFIX}.cf_param"], + # .clip_emb -> .clip_image_embeddings_project_to_time_embeddings + "clip_image_embeddings_project_to_time_embeddings.weight": checkpoint[ + f"{DECODER_ORIGINAL_PREFIX}.clip_emb.weight" + ], + "clip_image_embeddings_project_to_time_embeddings.bias": checkpoint[ + f"{DECODER_ORIGINAL_PREFIX}.clip_emb.bias" + ], + } + + return diffusers_checkpoint + + +# done text proj + +# super res unet first steps + +SUPER_RES_UNET_FIRST_STEPS_PREFIX = "model_first_steps" + +SUPER_RES_UNET_FIRST_STEPS_CONFIG = { + "sample_size": 256, + "layers_per_block": 3, + "down_block_types": ( + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + ), + "up_block_types": ( + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + ), + "block_out_channels": (320, 640, 960, 1280), + "in_channels": 6, + "out_channels": 3, + "add_attention": False, +} + + +def super_res_unet_first_steps_model_from_original_config(): + model = UNet2DModel(**SUPER_RES_UNET_FIRST_STEPS_CONFIG) + + return model + + +def super_res_unet_first_steps_original_checkpoint_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + original_unet_prefix = SUPER_RES_UNET_FIRST_STEPS_PREFIX + + diffusers_checkpoint.update(unet_time_embeddings(checkpoint, original_unet_prefix)) + diffusers_checkpoint.update(unet_conv_in(checkpoint, original_unet_prefix)) + + # .input_blocks -> .down_blocks + + original_down_block_idx = 1 + + for diffusers_down_block_idx in range(len(model.down_blocks)): + checkpoint_update, num_original_down_blocks = unet_downblock_to_diffusers_checkpoint( + model, + checkpoint, + diffusers_down_block_idx=diffusers_down_block_idx, + original_down_block_idx=original_down_block_idx, + original_unet_prefix=original_unet_prefix, + num_head_channels=None, + ) + + original_down_block_idx += num_original_down_blocks + + diffusers_checkpoint.update(checkpoint_update) + + diffusers_checkpoint.update( + unet_midblock_to_diffusers_checkpoint( + model, + checkpoint, + original_unet_prefix=original_unet_prefix, + num_head_channels=None, + ) + ) + + # .output_blocks -> .up_blocks + + original_up_block_idx = 0 + + for diffusers_up_block_idx in range(len(model.up_blocks)): + checkpoint_update, num_original_up_blocks = unet_upblock_to_diffusers_checkpoint( + model, + checkpoint, + diffusers_up_block_idx=diffusers_up_block_idx, + original_up_block_idx=original_up_block_idx, + original_unet_prefix=original_unet_prefix, + num_head_channels=None, + ) + + original_up_block_idx += num_original_up_blocks + + diffusers_checkpoint.update(checkpoint_update) + + # done .output_blocks -> .up_blocks + + diffusers_checkpoint.update(unet_conv_norm_out(checkpoint, original_unet_prefix)) + diffusers_checkpoint.update(unet_conv_out(checkpoint, original_unet_prefix)) + + return diffusers_checkpoint + + +# done super res unet first steps + +# super res unet last step + +SUPER_RES_UNET_LAST_STEP_PREFIX = "model_last_step" + +SUPER_RES_UNET_LAST_STEP_CONFIG = { + "sample_size": 256, + "layers_per_block": 3, + "down_block_types": ( + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + ), + "up_block_types": ( + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + ), + "block_out_channels": (320, 640, 960, 1280), + "in_channels": 6, + "out_channels": 3, + "add_attention": False, +} + + +def super_res_unet_last_step_model_from_original_config(): + model = UNet2DModel(**SUPER_RES_UNET_LAST_STEP_CONFIG) + + return model + + +def super_res_unet_last_step_original_checkpoint_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + original_unet_prefix = SUPER_RES_UNET_LAST_STEP_PREFIX + + diffusers_checkpoint.update(unet_time_embeddings(checkpoint, original_unet_prefix)) + diffusers_checkpoint.update(unet_conv_in(checkpoint, original_unet_prefix)) + + # .input_blocks -> .down_blocks + + original_down_block_idx = 1 + + for diffusers_down_block_idx in range(len(model.down_blocks)): + checkpoint_update, num_original_down_blocks = unet_downblock_to_diffusers_checkpoint( + model, + checkpoint, + diffusers_down_block_idx=diffusers_down_block_idx, + original_down_block_idx=original_down_block_idx, + original_unet_prefix=original_unet_prefix, + num_head_channels=None, + ) + + original_down_block_idx += num_original_down_blocks + + diffusers_checkpoint.update(checkpoint_update) + + diffusers_checkpoint.update( + unet_midblock_to_diffusers_checkpoint( + model, + checkpoint, + original_unet_prefix=original_unet_prefix, + num_head_channels=None, + ) + ) + + # .output_blocks -> .up_blocks + + original_up_block_idx = 0 + + for diffusers_up_block_idx in range(len(model.up_blocks)): + checkpoint_update, num_original_up_blocks = unet_upblock_to_diffusers_checkpoint( + model, + checkpoint, + diffusers_up_block_idx=diffusers_up_block_idx, + original_up_block_idx=original_up_block_idx, + original_unet_prefix=original_unet_prefix, + num_head_channels=None, + ) + + original_up_block_idx += num_original_up_blocks + + diffusers_checkpoint.update(checkpoint_update) + + # done .output_blocks -> .up_blocks + + diffusers_checkpoint.update(unet_conv_norm_out(checkpoint, original_unet_prefix)) + diffusers_checkpoint.update(unet_conv_out(checkpoint, original_unet_prefix)) + + return diffusers_checkpoint + + +# done super res unet last step + + +# unet utils + + +# .time_embed -> .time_embedding +def unet_time_embeddings(checkpoint, original_unet_prefix): + diffusers_checkpoint = {} + + diffusers_checkpoint.update( + { + "time_embedding.linear_1.weight": checkpoint[f"{original_unet_prefix}.time_embed.0.weight"], + "time_embedding.linear_1.bias": checkpoint[f"{original_unet_prefix}.time_embed.0.bias"], + "time_embedding.linear_2.weight": checkpoint[f"{original_unet_prefix}.time_embed.2.weight"], + "time_embedding.linear_2.bias": checkpoint[f"{original_unet_prefix}.time_embed.2.bias"], + } + ) + + return diffusers_checkpoint + + +# .input_blocks.0 -> .conv_in +def unet_conv_in(checkpoint, original_unet_prefix): + diffusers_checkpoint = {} + + diffusers_checkpoint.update( + { + "conv_in.weight": checkpoint[f"{original_unet_prefix}.input_blocks.0.0.weight"], + "conv_in.bias": checkpoint[f"{original_unet_prefix}.input_blocks.0.0.bias"], + } + ) + + return diffusers_checkpoint + + +# .out.0 -> .conv_norm_out +def unet_conv_norm_out(checkpoint, original_unet_prefix): + diffusers_checkpoint = {} + + diffusers_checkpoint.update( + { + "conv_norm_out.weight": checkpoint[f"{original_unet_prefix}.out.0.weight"], + "conv_norm_out.bias": checkpoint[f"{original_unet_prefix}.out.0.bias"], + } + ) + + return diffusers_checkpoint + + +# .out.2 -> .conv_out +def unet_conv_out(checkpoint, original_unet_prefix): + diffusers_checkpoint = {} + + diffusers_checkpoint.update( + { + "conv_out.weight": checkpoint[f"{original_unet_prefix}.out.2.weight"], + "conv_out.bias": checkpoint[f"{original_unet_prefix}.out.2.bias"], + } + ) + + return diffusers_checkpoint + + +# .input_blocks -> .down_blocks +def unet_downblock_to_diffusers_checkpoint( + model, checkpoint, *, diffusers_down_block_idx, original_down_block_idx, original_unet_prefix, num_head_channels +): + diffusers_checkpoint = {} + + diffusers_resnet_prefix = f"down_blocks.{diffusers_down_block_idx}.resnets" + original_down_block_prefix = f"{original_unet_prefix}.input_blocks" + + down_block = model.down_blocks[diffusers_down_block_idx] + + num_resnets = len(down_block.resnets) + + if down_block.downsamplers is None: + downsampler = False + else: + assert len(down_block.downsamplers) == 1 + downsampler = True + # The downsample block is also a resnet + num_resnets += 1 + + for resnet_idx_inc in range(num_resnets): + full_resnet_prefix = f"{original_down_block_prefix}.{original_down_block_idx + resnet_idx_inc}.0" + + if downsampler and resnet_idx_inc == num_resnets - 1: + # this is a downsample block + full_diffusers_resnet_prefix = f"down_blocks.{diffusers_down_block_idx}.downsamplers.0" + else: + # this is a regular resnet block + full_diffusers_resnet_prefix = f"{diffusers_resnet_prefix}.{resnet_idx_inc}" + + diffusers_checkpoint.update( + resnet_to_diffusers_checkpoint( + checkpoint, resnet_prefix=full_resnet_prefix, diffusers_resnet_prefix=full_diffusers_resnet_prefix + ) + ) + + if hasattr(down_block, "attentions"): + num_attentions = len(down_block.attentions) + diffusers_attention_prefix = f"down_blocks.{diffusers_down_block_idx}.attentions" + + for attention_idx_inc in range(num_attentions): + full_attention_prefix = f"{original_down_block_prefix}.{original_down_block_idx + attention_idx_inc}.1" + full_diffusers_attention_prefix = f"{diffusers_attention_prefix}.{attention_idx_inc}" + + diffusers_checkpoint.update( + attention_to_diffusers_checkpoint( + checkpoint, + attention_prefix=full_attention_prefix, + diffusers_attention_prefix=full_diffusers_attention_prefix, + num_head_channels=num_head_channels, + ) + ) + + num_original_down_blocks = num_resnets + + return diffusers_checkpoint, num_original_down_blocks + + +# .middle_block -> .mid_block +def unet_midblock_to_diffusers_checkpoint(model, checkpoint, *, original_unet_prefix, num_head_channels): + diffusers_checkpoint = {} + + # block 0 + + original_block_idx = 0 + + diffusers_checkpoint.update( + resnet_to_diffusers_checkpoint( + checkpoint, + diffusers_resnet_prefix="mid_block.resnets.0", + resnet_prefix=f"{original_unet_prefix}.middle_block.{original_block_idx}", + ) + ) + + original_block_idx += 1 + + # optional block 1 + + if hasattr(model.mid_block, "attentions") and model.mid_block.attentions[0] is not None: + diffusers_checkpoint.update( + attention_to_diffusers_checkpoint( + checkpoint, + diffusers_attention_prefix="mid_block.attentions.0", + attention_prefix=f"{original_unet_prefix}.middle_block.{original_block_idx}", + num_head_channels=num_head_channels, + ) + ) + original_block_idx += 1 + + # block 1 or block 2 + + diffusers_checkpoint.update( + resnet_to_diffusers_checkpoint( + checkpoint, + diffusers_resnet_prefix="mid_block.resnets.1", + resnet_prefix=f"{original_unet_prefix}.middle_block.{original_block_idx}", + ) + ) + + return diffusers_checkpoint + + +# .output_blocks -> .up_blocks +def unet_upblock_to_diffusers_checkpoint( + model, checkpoint, *, diffusers_up_block_idx, original_up_block_idx, original_unet_prefix, num_head_channels +): + diffusers_checkpoint = {} + + diffusers_resnet_prefix = f"up_blocks.{diffusers_up_block_idx}.resnets" + original_up_block_prefix = f"{original_unet_prefix}.output_blocks" + + up_block = model.up_blocks[diffusers_up_block_idx] + + num_resnets = len(up_block.resnets) + + if up_block.upsamplers is None: + upsampler = False + else: + assert len(up_block.upsamplers) == 1 + upsampler = True + # The upsample block is also a resnet + num_resnets += 1 + + has_attentions = hasattr(up_block, "attentions") + + for resnet_idx_inc in range(num_resnets): + if upsampler and resnet_idx_inc == num_resnets - 1: + # this is an upsample block + if has_attentions: + # There is a middle attention block that we skip + original_resnet_block_idx = 2 + else: + original_resnet_block_idx = 1 + + # we add the `minus 1` because the last two resnets are stuck together in the same output block + full_resnet_prefix = ( + f"{original_up_block_prefix}.{original_up_block_idx + resnet_idx_inc - 1}.{original_resnet_block_idx}" + ) + + full_diffusers_resnet_prefix = f"up_blocks.{diffusers_up_block_idx}.upsamplers.0" + else: + # this is a regular resnet block + full_resnet_prefix = f"{original_up_block_prefix}.{original_up_block_idx + resnet_idx_inc}.0" + full_diffusers_resnet_prefix = f"{diffusers_resnet_prefix}.{resnet_idx_inc}" + + diffusers_checkpoint.update( + resnet_to_diffusers_checkpoint( + checkpoint, resnet_prefix=full_resnet_prefix, diffusers_resnet_prefix=full_diffusers_resnet_prefix + ) + ) + + if has_attentions: + num_attentions = len(up_block.attentions) + diffusers_attention_prefix = f"up_blocks.{diffusers_up_block_idx}.attentions" + + for attention_idx_inc in range(num_attentions): + full_attention_prefix = f"{original_up_block_prefix}.{original_up_block_idx + attention_idx_inc}.1" + full_diffusers_attention_prefix = f"{diffusers_attention_prefix}.{attention_idx_inc}" + + diffusers_checkpoint.update( + attention_to_diffusers_checkpoint( + checkpoint, + attention_prefix=full_attention_prefix, + diffusers_attention_prefix=full_diffusers_attention_prefix, + num_head_channels=num_head_channels, + ) + ) + + num_original_down_blocks = num_resnets - 1 if upsampler else num_resnets + + return diffusers_checkpoint, num_original_down_blocks + + +def resnet_to_diffusers_checkpoint(checkpoint, *, diffusers_resnet_prefix, resnet_prefix): + diffusers_checkpoint = { + f"{diffusers_resnet_prefix}.norm1.weight": checkpoint[f"{resnet_prefix}.in_layers.0.weight"], + f"{diffusers_resnet_prefix}.norm1.bias": checkpoint[f"{resnet_prefix}.in_layers.0.bias"], + f"{diffusers_resnet_prefix}.conv1.weight": checkpoint[f"{resnet_prefix}.in_layers.2.weight"], + f"{diffusers_resnet_prefix}.conv1.bias": checkpoint[f"{resnet_prefix}.in_layers.2.bias"], + f"{diffusers_resnet_prefix}.time_emb_proj.weight": checkpoint[f"{resnet_prefix}.emb_layers.1.weight"], + f"{diffusers_resnet_prefix}.time_emb_proj.bias": checkpoint[f"{resnet_prefix}.emb_layers.1.bias"], + f"{diffusers_resnet_prefix}.norm2.weight": checkpoint[f"{resnet_prefix}.out_layers.0.weight"], + f"{diffusers_resnet_prefix}.norm2.bias": checkpoint[f"{resnet_prefix}.out_layers.0.bias"], + f"{diffusers_resnet_prefix}.conv2.weight": checkpoint[f"{resnet_prefix}.out_layers.3.weight"], + f"{diffusers_resnet_prefix}.conv2.bias": checkpoint[f"{resnet_prefix}.out_layers.3.bias"], + } + + skip_connection_prefix = f"{resnet_prefix}.skip_connection" + + if f"{skip_connection_prefix}.weight" in checkpoint: + diffusers_checkpoint.update( + { + f"{diffusers_resnet_prefix}.conv_shortcut.weight": checkpoint[f"{skip_connection_prefix}.weight"], + f"{diffusers_resnet_prefix}.conv_shortcut.bias": checkpoint[f"{skip_connection_prefix}.bias"], + } + ) + + return diffusers_checkpoint + + +def attention_to_diffusers_checkpoint(checkpoint, *, diffusers_attention_prefix, attention_prefix, num_head_channels): + diffusers_checkpoint = {} + + # .norm -> .group_norm + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.group_norm.weight": checkpoint[f"{attention_prefix}.norm.weight"], + f"{diffusers_attention_prefix}.group_norm.bias": checkpoint[f"{attention_prefix}.norm.bias"], + } + ) + + # .qkv -> .{query, key, value} + [q_weight, k_weight, v_weight], [q_bias, k_bias, v_bias] = split_attentions( + weight=checkpoint[f"{attention_prefix}.qkv.weight"][:, :, 0], + bias=checkpoint[f"{attention_prefix}.qkv.bias"], + split=3, + chunk_size=num_head_channels, + ) + + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.to_q.weight": q_weight, + f"{diffusers_attention_prefix}.to_q.bias": q_bias, + f"{diffusers_attention_prefix}.to_k.weight": k_weight, + f"{diffusers_attention_prefix}.to_k.bias": k_bias, + f"{diffusers_attention_prefix}.to_v.weight": v_weight, + f"{diffusers_attention_prefix}.to_v.bias": v_bias, + } + ) + + # .encoder_kv -> .{context_key, context_value} + [encoder_k_weight, encoder_v_weight], [encoder_k_bias, encoder_v_bias] = split_attentions( + weight=checkpoint[f"{attention_prefix}.encoder_kv.weight"][:, :, 0], + bias=checkpoint[f"{attention_prefix}.encoder_kv.bias"], + split=2, + chunk_size=num_head_channels, + ) + + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.add_k_proj.weight": encoder_k_weight, + f"{diffusers_attention_prefix}.add_k_proj.bias": encoder_k_bias, + f"{diffusers_attention_prefix}.add_v_proj.weight": encoder_v_weight, + f"{diffusers_attention_prefix}.add_v_proj.bias": encoder_v_bias, + } + ) + + # .proj_out (1d conv) -> .proj_attn (linear) + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.to_out.0.weight": checkpoint[f"{attention_prefix}.proj_out.weight"][ + :, :, 0 + ], + f"{diffusers_attention_prefix}.to_out.0.bias": checkpoint[f"{attention_prefix}.proj_out.bias"], + } + ) + + return diffusers_checkpoint + + +# TODO maybe document and/or can do more efficiently (build indices in for loop and extract once for each split?) +def split_attentions(*, weight, bias, split, chunk_size): + weights = [None] * split + biases = [None] * split + + weights_biases_idx = 0 + + for starting_row_index in range(0, weight.shape[0], chunk_size): + row_indices = torch.arange(starting_row_index, starting_row_index + chunk_size) + + weight_rows = weight[row_indices, :] + bias_rows = bias[row_indices] + + if weights[weights_biases_idx] is None: + assert weights[weights_biases_idx] is None + weights[weights_biases_idx] = weight_rows + biases[weights_biases_idx] = bias_rows + else: + assert weights[weights_biases_idx] is not None + weights[weights_biases_idx] = torch.concat([weights[weights_biases_idx], weight_rows]) + biases[weights_biases_idx] = torch.concat([biases[weights_biases_idx], bias_rows]) + + weights_biases_idx = (weights_biases_idx + 1) % split + + return weights, biases + + +# done unet utils + + +# Driver functions + + +def text_encoder(): + print("loading CLIP text encoder") + + clip_name = "openai/clip-vit-large-patch14" + + # sets pad_value to 0 + pad_token = "!" + + tokenizer_model = CLIPTokenizer.from_pretrained(clip_name, pad_token=pad_token, device_map="auto") + + assert tokenizer_model.convert_tokens_to_ids(pad_token) == 0 + + text_encoder_model = CLIPTextModelWithProjection.from_pretrained( + clip_name, + # `CLIPTextModel` does not support device_map="auto" + # device_map="auto" + ) + + print("done loading CLIP text encoder") + + return text_encoder_model, tokenizer_model + + +def prior(*, args, checkpoint_map_location): + print("loading prior") + + prior_checkpoint = torch.load(args.prior_checkpoint_path, map_location=checkpoint_map_location) + prior_checkpoint = prior_checkpoint["state_dict"] + + clip_stats_checkpoint = torch.load(args.clip_stat_path, map_location=checkpoint_map_location) + + prior_model = prior_model_from_original_config() + + prior_diffusers_checkpoint = prior_original_checkpoint_to_diffusers_checkpoint( + prior_model, prior_checkpoint, clip_stats_checkpoint + ) + + del prior_checkpoint + del clip_stats_checkpoint + + load_checkpoint_to_model(prior_diffusers_checkpoint, prior_model, strict=True) + + print("done loading prior") + + return prior_model + + +def decoder(*, args, checkpoint_map_location): + print("loading decoder") + + decoder_checkpoint = torch.load(args.decoder_checkpoint_path, map_location=checkpoint_map_location) + decoder_checkpoint = decoder_checkpoint["state_dict"] + + decoder_model = decoder_model_from_original_config() + + decoder_diffusers_checkpoint = decoder_original_checkpoint_to_diffusers_checkpoint( + decoder_model, decoder_checkpoint + ) + + # text proj interlude + + # The original decoder implementation includes a set of parameters that are used + # for creating the `encoder_hidden_states` which are what the U-net is conditioned + # on. The diffusers conditional unet directly takes the encoder_hidden_states. We pull + # the parameters into the UnCLIPTextProjModel class + text_proj_model = text_proj_from_original_config() + + text_proj_checkpoint = text_proj_original_checkpoint_to_diffusers_checkpoint(decoder_checkpoint) + + load_checkpoint_to_model(text_proj_checkpoint, text_proj_model, strict=True) + + # done text proj interlude + + del decoder_checkpoint + + load_checkpoint_to_model(decoder_diffusers_checkpoint, decoder_model, strict=True) + + print("done loading decoder") + + return decoder_model, text_proj_model + + +def super_res_unet(*, args, checkpoint_map_location): + print("loading super resolution unet") + + super_res_checkpoint = torch.load(args.super_res_unet_checkpoint_path, map_location=checkpoint_map_location) + super_res_checkpoint = super_res_checkpoint["state_dict"] + + # model_first_steps + + super_res_first_model = super_res_unet_first_steps_model_from_original_config() + + super_res_first_steps_checkpoint = super_res_unet_first_steps_original_checkpoint_to_diffusers_checkpoint( + super_res_first_model, super_res_checkpoint + ) + + # model_last_step + super_res_last_model = super_res_unet_last_step_model_from_original_config() + + super_res_last_step_checkpoint = super_res_unet_last_step_original_checkpoint_to_diffusers_checkpoint( + super_res_last_model, super_res_checkpoint + ) + + del super_res_checkpoint + + load_checkpoint_to_model(super_res_first_steps_checkpoint, super_res_first_model, strict=True) + + load_checkpoint_to_model(super_res_last_step_checkpoint, super_res_last_model, strict=True) + + print("done loading super resolution unet") + + return super_res_first_model, super_res_last_model + + +def load_checkpoint_to_model(checkpoint, model, strict=False): + with tempfile.NamedTemporaryFile() as file: + torch.save(checkpoint, file.name) + del checkpoint + if strict: + model.load_state_dict(torch.load(file.name), strict=True) + else: + load_checkpoint_and_dispatch(model, file.name, device_map="auto") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + + parser.add_argument( + "--prior_checkpoint_path", + default=None, + type=str, + required=True, + help="Path to the prior checkpoint to convert.", + ) + + parser.add_argument( + "--decoder_checkpoint_path", + default=None, + type=str, + required=True, + help="Path to the decoder checkpoint to convert.", + ) + + parser.add_argument( + "--super_res_unet_checkpoint_path", + default=None, + type=str, + required=True, + help="Path to the super resolution checkpoint to convert.", + ) + + parser.add_argument( + "--clip_stat_path", default=None, type=str, required=True, help="Path to the clip stats checkpoint to convert." + ) + + parser.add_argument( + "--checkpoint_load_device", + default="cpu", + type=str, + required=False, + help="The device passed to `map_location` when loading checkpoints.", + ) + + parser.add_argument( + "--debug", + default=None, + type=str, + required=False, + help="Only run a specific stage of the convert script. Used for debugging", + ) + + args = parser.parse_args() + + print(f"loading checkpoints to {args.checkpoint_load_device}") + + checkpoint_map_location = torch.device(args.checkpoint_load_device) + + if args.debug is not None: + print(f"debug: only executing {args.debug}") + + if args.debug is None: + text_encoder_model, tokenizer_model = text_encoder() + + prior_model = prior(args=args, checkpoint_map_location=checkpoint_map_location) + + decoder_model, text_proj_model = decoder(args=args, checkpoint_map_location=checkpoint_map_location) + + super_res_first_model, super_res_last_model = super_res_unet( + args=args, checkpoint_map_location=checkpoint_map_location + ) + + prior_scheduler = UnCLIPScheduler( + variance_type="fixed_small_log", + prediction_type="sample", + num_train_timesteps=1000, + clip_sample_range=5.0, + ) + + decoder_scheduler = UnCLIPScheduler( + variance_type="learned_range", + prediction_type="epsilon", + num_train_timesteps=1000, + ) + + super_res_scheduler = UnCLIPScheduler( + variance_type="fixed_small_log", + prediction_type="epsilon", + num_train_timesteps=1000, + ) + + print(f"saving Kakao Brain unCLIP to {args.dump_path}") + + pipe = UnCLIPPipeline( + prior=prior_model, + decoder=decoder_model, + text_proj=text_proj_model, + tokenizer=tokenizer_model, + text_encoder=text_encoder_model, + super_res_first=super_res_first_model, + super_res_last=super_res_last_model, + prior_scheduler=prior_scheduler, + decoder_scheduler=decoder_scheduler, + super_res_scheduler=super_res_scheduler, + ) + pipe.save_pretrained(args.dump_path) + + print("done writing Kakao Brain unCLIP") + elif args.debug == "text_encoder": + text_encoder_model, tokenizer_model = text_encoder() + elif args.debug == "prior": + prior_model = prior(args=args, checkpoint_map_location=checkpoint_map_location) + elif args.debug == "decoder": + decoder_model, text_proj_model = decoder(args=args, checkpoint_map_location=checkpoint_map_location) + elif args.debug == "super_res_unet": + super_res_first_model, super_res_last_model = super_res_unet( + args=args, checkpoint_map_location=checkpoint_map_location + ) + else: + raise ValueError(f"unknown debug value : {args.debug}") diff --git a/diffusers-0.27.0/scripts/convert_kandinsky3_unet.py b/diffusers-0.27.0/scripts/convert_kandinsky3_unet.py new file mode 100755 index 0000000..4fe8c54 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_kandinsky3_unet.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +import argparse +import fnmatch + +from safetensors.torch import load_file + +from diffusers import Kandinsky3UNet + + +MAPPING = { + "to_time_embed.1": "time_embedding.linear_1", + "to_time_embed.3": "time_embedding.linear_2", + "in_layer": "conv_in", + "out_layer.0": "conv_norm_out", + "out_layer.2": "conv_out", + "down_samples": "down_blocks", + "up_samples": "up_blocks", + "projection_lin": "encoder_hid_proj.projection_linear", + "projection_ln": "encoder_hid_proj.projection_norm", + "feature_pooling": "add_time_condition", + "to_query": "to_q", + "to_key": "to_k", + "to_value": "to_v", + "output_layer": "to_out.0", + "self_attention_block": "attentions.0", +} + +DYNAMIC_MAP = { + "resnet_attn_blocks.*.0": "resnets_in.*", + "resnet_attn_blocks.*.1": ("attentions.*", 1), + "resnet_attn_blocks.*.2": "resnets_out.*", +} +# MAPPING = {} + + +def convert_state_dict(unet_state_dict): + """ + Convert the state dict of a U-Net model to match the key format expected by Kandinsky3UNet model. + Args: + unet_model (torch.nn.Module): The original U-Net model. + unet_kandi3_model (torch.nn.Module): The Kandinsky3UNet model to match keys with. + + Returns: + OrderedDict: The converted state dictionary. + """ + # Example of renaming logic (this will vary based on your model's architecture) + converted_state_dict = {} + for key in unet_state_dict: + new_key = key + for pattern, new_pattern in MAPPING.items(): + new_key = new_key.replace(pattern, new_pattern) + + for dyn_pattern, dyn_new_pattern in DYNAMIC_MAP.items(): + has_matched = False + if fnmatch.fnmatch(new_key, f"*.{dyn_pattern}.*") and not has_matched: + star = int(new_key.split(dyn_pattern.split(".")[0])[-1].split(".")[1]) + + if isinstance(dyn_new_pattern, tuple): + new_star = star + dyn_new_pattern[-1] + dyn_new_pattern = dyn_new_pattern[0] + else: + new_star = star + + pattern = dyn_pattern.replace("*", str(star)) + new_pattern = dyn_new_pattern.replace("*", str(new_star)) + + new_key = new_key.replace(pattern, new_pattern) + has_matched = True + + converted_state_dict[new_key] = unet_state_dict[key] + + return converted_state_dict + + +def main(model_path, output_path): + # Load your original U-Net model + unet_state_dict = load_file(model_path) + + # Initialize your Kandinsky3UNet model + config = {} + + # Convert the state dict + converted_state_dict = convert_state_dict(unet_state_dict) + + unet = Kandinsky3UNet(config) + unet.load_state_dict(converted_state_dict) + + unet.save_pretrained(output_path) + print(f"Converted model saved to {output_path}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Convert U-Net PyTorch model to Kandinsky3UNet format") + parser.add_argument("--model_path", type=str, required=True, help="Path to the original U-Net PyTorch model") + parser.add_argument("--output_path", type=str, required=True, help="Path to save the converted model") + + args = parser.parse_args() + main(args.model_path, args.output_path) diff --git a/diffusers-0.27.0/scripts/convert_kandinsky_to_diffusers.py b/diffusers-0.27.0/scripts/convert_kandinsky_to_diffusers.py new file mode 100755 index 0000000..8d3f7b6 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_kandinsky_to_diffusers.py @@ -0,0 +1,1411 @@ +import argparse +import os +import tempfile + +import torch +from accelerate import load_checkpoint_and_dispatch + +from diffusers import UNet2DConditionModel +from diffusers.models.transformers.prior_transformer import PriorTransformer +from diffusers.models.vq_model import VQModel + + +""" +Example - From the diffusers root directory: + +Download weights: +```sh +$ wget https://huggingface.co/ai-forever/Kandinsky_2.1/blob/main/prior_fp16.ckpt +``` + +Convert the model: +```sh +python scripts/convert_kandinsky_to_diffusers.py \ + --prior_checkpoint_path /home/yiyi_huggingface_co/Kandinsky-2/checkpoints_Kandinsky_2.1/prior_fp16.ckpt \ + --clip_stat_path /home/yiyi_huggingface_co/Kandinsky-2/checkpoints_Kandinsky_2.1/ViT-L-14_stats.th \ + --text2img_checkpoint_path /home/yiyi_huggingface_co/Kandinsky-2/checkpoints_Kandinsky_2.1/decoder_fp16.ckpt \ + --inpaint_text2img_checkpoint_path /home/yiyi_huggingface_co/Kandinsky-2/checkpoints_Kandinsky_2.1/inpainting_fp16.ckpt \ + --movq_checkpoint_path /home/yiyi_huggingface_co/Kandinsky-2/checkpoints_Kandinsky_2.1/movq_final.ckpt \ + --dump_path /home/yiyi_huggingface_co/dump \ + --debug decoder +``` +""" + + +# prior + +PRIOR_ORIGINAL_PREFIX = "model" + +# Uses default arguments +PRIOR_CONFIG = {} + + +def prior_model_from_original_config(): + model = PriorTransformer(**PRIOR_CONFIG) + + return model + + +def prior_original_checkpoint_to_diffusers_checkpoint(model, checkpoint, clip_stats_checkpoint): + diffusers_checkpoint = {} + + # .time_embed.0 -> .time_embedding.linear_1 + diffusers_checkpoint.update( + { + "time_embedding.linear_1.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.0.weight"], + "time_embedding.linear_1.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.0.bias"], + } + ) + + # .clip_img_proj -> .proj_in + diffusers_checkpoint.update( + { + "proj_in.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.clip_img_proj.weight"], + "proj_in.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.clip_img_proj.bias"], + } + ) + + # .text_emb_proj -> .embedding_proj + diffusers_checkpoint.update( + { + "embedding_proj.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.text_emb_proj.weight"], + "embedding_proj.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.text_emb_proj.bias"], + } + ) + + # .text_enc_proj -> .encoder_hidden_states_proj + diffusers_checkpoint.update( + { + "encoder_hidden_states_proj.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.text_enc_proj.weight"], + "encoder_hidden_states_proj.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.text_enc_proj.bias"], + } + ) + + # .positional_embedding -> .positional_embedding + diffusers_checkpoint.update({"positional_embedding": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.positional_embedding"]}) + + # .prd_emb -> .prd_embedding + diffusers_checkpoint.update({"prd_embedding": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.prd_emb"]}) + + # .time_embed.2 -> .time_embedding.linear_2 + diffusers_checkpoint.update( + { + "time_embedding.linear_2.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.2.weight"], + "time_embedding.linear_2.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.2.bias"], + } + ) + + # .resblocks. -> .transformer_blocks. + for idx in range(len(model.transformer_blocks)): + diffusers_transformer_prefix = f"transformer_blocks.{idx}" + original_transformer_prefix = f"{PRIOR_ORIGINAL_PREFIX}.transformer.resblocks.{idx}" + + # .attn -> .attn1 + diffusers_attention_prefix = f"{diffusers_transformer_prefix}.attn1" + original_attention_prefix = f"{original_transformer_prefix}.attn" + diffusers_checkpoint.update( + prior_attention_to_diffusers( + checkpoint, + diffusers_attention_prefix=diffusers_attention_prefix, + original_attention_prefix=original_attention_prefix, + attention_head_dim=model.attention_head_dim, + ) + ) + + # .mlp -> .ff + diffusers_ff_prefix = f"{diffusers_transformer_prefix}.ff" + original_ff_prefix = f"{original_transformer_prefix}.mlp" + diffusers_checkpoint.update( + prior_ff_to_diffusers( + checkpoint, diffusers_ff_prefix=diffusers_ff_prefix, original_ff_prefix=original_ff_prefix + ) + ) + + # .ln_1 -> .norm1 + diffusers_checkpoint.update( + { + f"{diffusers_transformer_prefix}.norm1.weight": checkpoint[ + f"{original_transformer_prefix}.ln_1.weight" + ], + f"{diffusers_transformer_prefix}.norm1.bias": checkpoint[f"{original_transformer_prefix}.ln_1.bias"], + } + ) + + # .ln_2 -> .norm3 + diffusers_checkpoint.update( + { + f"{diffusers_transformer_prefix}.norm3.weight": checkpoint[ + f"{original_transformer_prefix}.ln_2.weight" + ], + f"{diffusers_transformer_prefix}.norm3.bias": checkpoint[f"{original_transformer_prefix}.ln_2.bias"], + } + ) + + # .final_ln -> .norm_out + diffusers_checkpoint.update( + { + "norm_out.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.final_ln.weight"], + "norm_out.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.final_ln.bias"], + } + ) + + # .out_proj -> .proj_to_clip_embeddings + diffusers_checkpoint.update( + { + "proj_to_clip_embeddings.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.out_proj.weight"], + "proj_to_clip_embeddings.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.out_proj.bias"], + } + ) + + # clip stats + clip_mean, clip_std = clip_stats_checkpoint + clip_mean = clip_mean[None, :] + clip_std = clip_std[None, :] + + diffusers_checkpoint.update({"clip_mean": clip_mean, "clip_std": clip_std}) + + return diffusers_checkpoint + + +def prior_attention_to_diffusers( + checkpoint, *, diffusers_attention_prefix, original_attention_prefix, attention_head_dim +): + diffusers_checkpoint = {} + + # .c_qkv -> .{to_q, to_k, to_v} + [q_weight, k_weight, v_weight], [q_bias, k_bias, v_bias] = split_attentions( + weight=checkpoint[f"{original_attention_prefix}.c_qkv.weight"], + bias=checkpoint[f"{original_attention_prefix}.c_qkv.bias"], + split=3, + chunk_size=attention_head_dim, + ) + + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.to_q.weight": q_weight, + f"{diffusers_attention_prefix}.to_q.bias": q_bias, + f"{diffusers_attention_prefix}.to_k.weight": k_weight, + f"{diffusers_attention_prefix}.to_k.bias": k_bias, + f"{diffusers_attention_prefix}.to_v.weight": v_weight, + f"{diffusers_attention_prefix}.to_v.bias": v_bias, + } + ) + + # .c_proj -> .to_out.0 + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.to_out.0.weight": checkpoint[f"{original_attention_prefix}.c_proj.weight"], + f"{diffusers_attention_prefix}.to_out.0.bias": checkpoint[f"{original_attention_prefix}.c_proj.bias"], + } + ) + + return diffusers_checkpoint + + +def prior_ff_to_diffusers(checkpoint, *, diffusers_ff_prefix, original_ff_prefix): + diffusers_checkpoint = { + # .c_fc -> .net.0.proj + f"{diffusers_ff_prefix}.net.{0}.proj.weight": checkpoint[f"{original_ff_prefix}.c_fc.weight"], + f"{diffusers_ff_prefix}.net.{0}.proj.bias": checkpoint[f"{original_ff_prefix}.c_fc.bias"], + # .c_proj -> .net.2 + f"{diffusers_ff_prefix}.net.{2}.weight": checkpoint[f"{original_ff_prefix}.c_proj.weight"], + f"{diffusers_ff_prefix}.net.{2}.bias": checkpoint[f"{original_ff_prefix}.c_proj.bias"], + } + + return diffusers_checkpoint + + +# done prior + +# unet + +# We are hardcoding the model configuration for now. If we need to generalize to more model configurations, we can +# update then. + +UNET_CONFIG = { + "act_fn": "silu", + "addition_embed_type": "text_image", + "addition_embed_type_num_heads": 64, + "attention_head_dim": 64, + "block_out_channels": [384, 768, 1152, 1536], + "center_input_sample": False, + "class_embed_type": None, + "class_embeddings_concat": False, + "conv_in_kernel": 3, + "conv_out_kernel": 3, + "cross_attention_dim": 768, + "cross_attention_norm": None, + "down_block_types": [ + "ResnetDownsampleBlock2D", + "SimpleCrossAttnDownBlock2D", + "SimpleCrossAttnDownBlock2D", + "SimpleCrossAttnDownBlock2D", + ], + "downsample_padding": 1, + "dual_cross_attention": False, + "encoder_hid_dim": 1024, + "encoder_hid_dim_type": "text_image_proj", + "flip_sin_to_cos": True, + "freq_shift": 0, + "in_channels": 4, + "layers_per_block": 3, + "mid_block_only_cross_attention": None, + "mid_block_scale_factor": 1, + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "norm_eps": 1e-05, + "norm_num_groups": 32, + "num_class_embeds": None, + "only_cross_attention": False, + "out_channels": 8, + "projection_class_embeddings_input_dim": None, + "resnet_out_scale_factor": 1.0, + "resnet_skip_time_act": False, + "resnet_time_scale_shift": "scale_shift", + "sample_size": 64, + "time_cond_proj_dim": None, + "time_embedding_act_fn": None, + "time_embedding_dim": None, + "time_embedding_type": "positional", + "timestep_post_act": None, + "up_block_types": [ + "SimpleCrossAttnUpBlock2D", + "SimpleCrossAttnUpBlock2D", + "SimpleCrossAttnUpBlock2D", + "ResnetUpsampleBlock2D", + ], + "upcast_attention": False, + "use_linear_projection": False, +} + + +def unet_model_from_original_config(): + model = UNet2DConditionModel(**UNET_CONFIG) + + return model + + +def unet_original_checkpoint_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + num_head_channels = UNET_CONFIG["attention_head_dim"] + + diffusers_checkpoint.update(unet_time_embeddings(checkpoint)) + diffusers_checkpoint.update(unet_conv_in(checkpoint)) + diffusers_checkpoint.update(unet_add_embedding(checkpoint)) + diffusers_checkpoint.update(unet_encoder_hid_proj(checkpoint)) + + # .input_blocks -> .down_blocks + + original_down_block_idx = 1 + + for diffusers_down_block_idx in range(len(model.down_blocks)): + checkpoint_update, num_original_down_blocks = unet_downblock_to_diffusers_checkpoint( + model, + checkpoint, + diffusers_down_block_idx=diffusers_down_block_idx, + original_down_block_idx=original_down_block_idx, + num_head_channels=num_head_channels, + ) + + original_down_block_idx += num_original_down_blocks + + diffusers_checkpoint.update(checkpoint_update) + + # done .input_blocks -> .down_blocks + + diffusers_checkpoint.update( + unet_midblock_to_diffusers_checkpoint( + model, + checkpoint, + num_head_channels=num_head_channels, + ) + ) + + # .output_blocks -> .up_blocks + + original_up_block_idx = 0 + + for diffusers_up_block_idx in range(len(model.up_blocks)): + checkpoint_update, num_original_up_blocks = unet_upblock_to_diffusers_checkpoint( + model, + checkpoint, + diffusers_up_block_idx=diffusers_up_block_idx, + original_up_block_idx=original_up_block_idx, + num_head_channels=num_head_channels, + ) + + original_up_block_idx += num_original_up_blocks + + diffusers_checkpoint.update(checkpoint_update) + + # done .output_blocks -> .up_blocks + + diffusers_checkpoint.update(unet_conv_norm_out(checkpoint)) + diffusers_checkpoint.update(unet_conv_out(checkpoint)) + + return diffusers_checkpoint + + +# done unet + +# inpaint unet + +# We are hardcoding the model configuration for now. If we need to generalize to more model configurations, we can +# update then. + +INPAINT_UNET_CONFIG = { + "act_fn": "silu", + "addition_embed_type": "text_image", + "addition_embed_type_num_heads": 64, + "attention_head_dim": 64, + "block_out_channels": [384, 768, 1152, 1536], + "center_input_sample": False, + "class_embed_type": None, + "class_embeddings_concat": None, + "conv_in_kernel": 3, + "conv_out_kernel": 3, + "cross_attention_dim": 768, + "cross_attention_norm": None, + "down_block_types": [ + "ResnetDownsampleBlock2D", + "SimpleCrossAttnDownBlock2D", + "SimpleCrossAttnDownBlock2D", + "SimpleCrossAttnDownBlock2D", + ], + "downsample_padding": 1, + "dual_cross_attention": False, + "encoder_hid_dim": 1024, + "encoder_hid_dim_type": "text_image_proj", + "flip_sin_to_cos": True, + "freq_shift": 0, + "in_channels": 9, + "layers_per_block": 3, + "mid_block_only_cross_attention": None, + "mid_block_scale_factor": 1, + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "norm_eps": 1e-05, + "norm_num_groups": 32, + "num_class_embeds": None, + "only_cross_attention": False, + "out_channels": 8, + "projection_class_embeddings_input_dim": None, + "resnet_out_scale_factor": 1.0, + "resnet_skip_time_act": False, + "resnet_time_scale_shift": "scale_shift", + "sample_size": 64, + "time_cond_proj_dim": None, + "time_embedding_act_fn": None, + "time_embedding_dim": None, + "time_embedding_type": "positional", + "timestep_post_act": None, + "up_block_types": [ + "SimpleCrossAttnUpBlock2D", + "SimpleCrossAttnUpBlock2D", + "SimpleCrossAttnUpBlock2D", + "ResnetUpsampleBlock2D", + ], + "upcast_attention": False, + "use_linear_projection": False, +} + + +def inpaint_unet_model_from_original_config(): + model = UNet2DConditionModel(**INPAINT_UNET_CONFIG) + + return model + + +def inpaint_unet_original_checkpoint_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + num_head_channels = INPAINT_UNET_CONFIG["attention_head_dim"] + + diffusers_checkpoint.update(unet_time_embeddings(checkpoint)) + diffusers_checkpoint.update(unet_conv_in(checkpoint)) + diffusers_checkpoint.update(unet_add_embedding(checkpoint)) + diffusers_checkpoint.update(unet_encoder_hid_proj(checkpoint)) + + # .input_blocks -> .down_blocks + + original_down_block_idx = 1 + + for diffusers_down_block_idx in range(len(model.down_blocks)): + checkpoint_update, num_original_down_blocks = unet_downblock_to_diffusers_checkpoint( + model, + checkpoint, + diffusers_down_block_idx=diffusers_down_block_idx, + original_down_block_idx=original_down_block_idx, + num_head_channels=num_head_channels, + ) + + original_down_block_idx += num_original_down_blocks + + diffusers_checkpoint.update(checkpoint_update) + + # done .input_blocks -> .down_blocks + + diffusers_checkpoint.update( + unet_midblock_to_diffusers_checkpoint( + model, + checkpoint, + num_head_channels=num_head_channels, + ) + ) + + # .output_blocks -> .up_blocks + + original_up_block_idx = 0 + + for diffusers_up_block_idx in range(len(model.up_blocks)): + checkpoint_update, num_original_up_blocks = unet_upblock_to_diffusers_checkpoint( + model, + checkpoint, + diffusers_up_block_idx=diffusers_up_block_idx, + original_up_block_idx=original_up_block_idx, + num_head_channels=num_head_channels, + ) + + original_up_block_idx += num_original_up_blocks + + diffusers_checkpoint.update(checkpoint_update) + + # done .output_blocks -> .up_blocks + + diffusers_checkpoint.update(unet_conv_norm_out(checkpoint)) + diffusers_checkpoint.update(unet_conv_out(checkpoint)) + + return diffusers_checkpoint + + +# done inpaint unet + + +# unet utils + + +# .time_embed -> .time_embedding +def unet_time_embeddings(checkpoint): + diffusers_checkpoint = {} + + diffusers_checkpoint.update( + { + "time_embedding.linear_1.weight": checkpoint["time_embed.0.weight"], + "time_embedding.linear_1.bias": checkpoint["time_embed.0.bias"], + "time_embedding.linear_2.weight": checkpoint["time_embed.2.weight"], + "time_embedding.linear_2.bias": checkpoint["time_embed.2.bias"], + } + ) + + return diffusers_checkpoint + + +# .input_blocks.0 -> .conv_in +def unet_conv_in(checkpoint): + diffusers_checkpoint = {} + + diffusers_checkpoint.update( + { + "conv_in.weight": checkpoint["input_blocks.0.0.weight"], + "conv_in.bias": checkpoint["input_blocks.0.0.bias"], + } + ) + + return diffusers_checkpoint + + +def unet_add_embedding(checkpoint): + diffusers_checkpoint = {} + + diffusers_checkpoint.update( + { + "add_embedding.text_norm.weight": checkpoint["ln_model_n.weight"], + "add_embedding.text_norm.bias": checkpoint["ln_model_n.bias"], + "add_embedding.text_proj.weight": checkpoint["proj_n.weight"], + "add_embedding.text_proj.bias": checkpoint["proj_n.bias"], + "add_embedding.image_proj.weight": checkpoint["img_layer.weight"], + "add_embedding.image_proj.bias": checkpoint["img_layer.bias"], + } + ) + + return diffusers_checkpoint + + +def unet_encoder_hid_proj(checkpoint): + diffusers_checkpoint = {} + + diffusers_checkpoint.update( + { + "encoder_hid_proj.image_embeds.weight": checkpoint["clip_to_seq.weight"], + "encoder_hid_proj.image_embeds.bias": checkpoint["clip_to_seq.bias"], + "encoder_hid_proj.text_proj.weight": checkpoint["to_model_dim_n.weight"], + "encoder_hid_proj.text_proj.bias": checkpoint["to_model_dim_n.bias"], + } + ) + + return diffusers_checkpoint + + +# .out.0 -> .conv_norm_out +def unet_conv_norm_out(checkpoint): + diffusers_checkpoint = {} + + diffusers_checkpoint.update( + { + "conv_norm_out.weight": checkpoint["out.0.weight"], + "conv_norm_out.bias": checkpoint["out.0.bias"], + } + ) + + return diffusers_checkpoint + + +# .out.2 -> .conv_out +def unet_conv_out(checkpoint): + diffusers_checkpoint = {} + + diffusers_checkpoint.update( + { + "conv_out.weight": checkpoint["out.2.weight"], + "conv_out.bias": checkpoint["out.2.bias"], + } + ) + + return diffusers_checkpoint + + +# .input_blocks -> .down_blocks +def unet_downblock_to_diffusers_checkpoint( + model, checkpoint, *, diffusers_down_block_idx, original_down_block_idx, num_head_channels +): + diffusers_checkpoint = {} + + diffusers_resnet_prefix = f"down_blocks.{diffusers_down_block_idx}.resnets" + original_down_block_prefix = "input_blocks" + + down_block = model.down_blocks[diffusers_down_block_idx] + + num_resnets = len(down_block.resnets) + + if down_block.downsamplers is None: + downsampler = False + else: + assert len(down_block.downsamplers) == 1 + downsampler = True + # The downsample block is also a resnet + num_resnets += 1 + + for resnet_idx_inc in range(num_resnets): + full_resnet_prefix = f"{original_down_block_prefix}.{original_down_block_idx + resnet_idx_inc}.0" + + if downsampler and resnet_idx_inc == num_resnets - 1: + # this is a downsample block + full_diffusers_resnet_prefix = f"down_blocks.{diffusers_down_block_idx}.downsamplers.0" + else: + # this is a regular resnet block + full_diffusers_resnet_prefix = f"{diffusers_resnet_prefix}.{resnet_idx_inc}" + + diffusers_checkpoint.update( + resnet_to_diffusers_checkpoint( + checkpoint, resnet_prefix=full_resnet_prefix, diffusers_resnet_prefix=full_diffusers_resnet_prefix + ) + ) + + if hasattr(down_block, "attentions"): + num_attentions = len(down_block.attentions) + diffusers_attention_prefix = f"down_blocks.{diffusers_down_block_idx}.attentions" + + for attention_idx_inc in range(num_attentions): + full_attention_prefix = f"{original_down_block_prefix}.{original_down_block_idx + attention_idx_inc}.1" + full_diffusers_attention_prefix = f"{diffusers_attention_prefix}.{attention_idx_inc}" + + diffusers_checkpoint.update( + attention_to_diffusers_checkpoint( + checkpoint, + attention_prefix=full_attention_prefix, + diffusers_attention_prefix=full_diffusers_attention_prefix, + num_head_channels=num_head_channels, + ) + ) + + num_original_down_blocks = num_resnets + + return diffusers_checkpoint, num_original_down_blocks + + +# .middle_block -> .mid_block +def unet_midblock_to_diffusers_checkpoint(model, checkpoint, *, num_head_channels): + diffusers_checkpoint = {} + + # block 0 + + original_block_idx = 0 + + diffusers_checkpoint.update( + resnet_to_diffusers_checkpoint( + checkpoint, + diffusers_resnet_prefix="mid_block.resnets.0", + resnet_prefix=f"middle_block.{original_block_idx}", + ) + ) + + original_block_idx += 1 + + # optional block 1 + + if hasattr(model.mid_block, "attentions") and model.mid_block.attentions[0] is not None: + diffusers_checkpoint.update( + attention_to_diffusers_checkpoint( + checkpoint, + diffusers_attention_prefix="mid_block.attentions.0", + attention_prefix=f"middle_block.{original_block_idx}", + num_head_channels=num_head_channels, + ) + ) + original_block_idx += 1 + + # block 1 or block 2 + + diffusers_checkpoint.update( + resnet_to_diffusers_checkpoint( + checkpoint, + diffusers_resnet_prefix="mid_block.resnets.1", + resnet_prefix=f"middle_block.{original_block_idx}", + ) + ) + + return diffusers_checkpoint + + +# .output_blocks -> .up_blocks +def unet_upblock_to_diffusers_checkpoint( + model, checkpoint, *, diffusers_up_block_idx, original_up_block_idx, num_head_channels +): + diffusers_checkpoint = {} + + diffusers_resnet_prefix = f"up_blocks.{diffusers_up_block_idx}.resnets" + original_up_block_prefix = "output_blocks" + + up_block = model.up_blocks[diffusers_up_block_idx] + + num_resnets = len(up_block.resnets) + + if up_block.upsamplers is None: + upsampler = False + else: + assert len(up_block.upsamplers) == 1 + upsampler = True + # The upsample block is also a resnet + num_resnets += 1 + + has_attentions = hasattr(up_block, "attentions") + + for resnet_idx_inc in range(num_resnets): + if upsampler and resnet_idx_inc == num_resnets - 1: + # this is an upsample block + if has_attentions: + # There is a middle attention block that we skip + original_resnet_block_idx = 2 + else: + original_resnet_block_idx = 1 + + # we add the `minus 1` because the last two resnets are stuck together in the same output block + full_resnet_prefix = ( + f"{original_up_block_prefix}.{original_up_block_idx + resnet_idx_inc - 1}.{original_resnet_block_idx}" + ) + + full_diffusers_resnet_prefix = f"up_blocks.{diffusers_up_block_idx}.upsamplers.0" + else: + # this is a regular resnet block + full_resnet_prefix = f"{original_up_block_prefix}.{original_up_block_idx + resnet_idx_inc}.0" + full_diffusers_resnet_prefix = f"{diffusers_resnet_prefix}.{resnet_idx_inc}" + + diffusers_checkpoint.update( + resnet_to_diffusers_checkpoint( + checkpoint, resnet_prefix=full_resnet_prefix, diffusers_resnet_prefix=full_diffusers_resnet_prefix + ) + ) + + if has_attentions: + num_attentions = len(up_block.attentions) + diffusers_attention_prefix = f"up_blocks.{diffusers_up_block_idx}.attentions" + + for attention_idx_inc in range(num_attentions): + full_attention_prefix = f"{original_up_block_prefix}.{original_up_block_idx + attention_idx_inc}.1" + full_diffusers_attention_prefix = f"{diffusers_attention_prefix}.{attention_idx_inc}" + + diffusers_checkpoint.update( + attention_to_diffusers_checkpoint( + checkpoint, + attention_prefix=full_attention_prefix, + diffusers_attention_prefix=full_diffusers_attention_prefix, + num_head_channels=num_head_channels, + ) + ) + + num_original_down_blocks = num_resnets - 1 if upsampler else num_resnets + + return diffusers_checkpoint, num_original_down_blocks + + +def resnet_to_diffusers_checkpoint(checkpoint, *, diffusers_resnet_prefix, resnet_prefix): + diffusers_checkpoint = { + f"{diffusers_resnet_prefix}.norm1.weight": checkpoint[f"{resnet_prefix}.in_layers.0.weight"], + f"{diffusers_resnet_prefix}.norm1.bias": checkpoint[f"{resnet_prefix}.in_layers.0.bias"], + f"{diffusers_resnet_prefix}.conv1.weight": checkpoint[f"{resnet_prefix}.in_layers.2.weight"], + f"{diffusers_resnet_prefix}.conv1.bias": checkpoint[f"{resnet_prefix}.in_layers.2.bias"], + f"{diffusers_resnet_prefix}.time_emb_proj.weight": checkpoint[f"{resnet_prefix}.emb_layers.1.weight"], + f"{diffusers_resnet_prefix}.time_emb_proj.bias": checkpoint[f"{resnet_prefix}.emb_layers.1.bias"], + f"{diffusers_resnet_prefix}.norm2.weight": checkpoint[f"{resnet_prefix}.out_layers.0.weight"], + f"{diffusers_resnet_prefix}.norm2.bias": checkpoint[f"{resnet_prefix}.out_layers.0.bias"], + f"{diffusers_resnet_prefix}.conv2.weight": checkpoint[f"{resnet_prefix}.out_layers.3.weight"], + f"{diffusers_resnet_prefix}.conv2.bias": checkpoint[f"{resnet_prefix}.out_layers.3.bias"], + } + + skip_connection_prefix = f"{resnet_prefix}.skip_connection" + + if f"{skip_connection_prefix}.weight" in checkpoint: + diffusers_checkpoint.update( + { + f"{diffusers_resnet_prefix}.conv_shortcut.weight": checkpoint[f"{skip_connection_prefix}.weight"], + f"{diffusers_resnet_prefix}.conv_shortcut.bias": checkpoint[f"{skip_connection_prefix}.bias"], + } + ) + + return diffusers_checkpoint + + +def attention_to_diffusers_checkpoint(checkpoint, *, diffusers_attention_prefix, attention_prefix, num_head_channels): + diffusers_checkpoint = {} + + # .norm -> .group_norm + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.group_norm.weight": checkpoint[f"{attention_prefix}.norm.weight"], + f"{diffusers_attention_prefix}.group_norm.bias": checkpoint[f"{attention_prefix}.norm.bias"], + } + ) + + # .qkv -> .{query, key, value} + [q_weight, k_weight, v_weight], [q_bias, k_bias, v_bias] = split_attentions( + weight=checkpoint[f"{attention_prefix}.qkv.weight"][:, :, 0], + bias=checkpoint[f"{attention_prefix}.qkv.bias"], + split=3, + chunk_size=num_head_channels, + ) + + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.to_q.weight": q_weight, + f"{diffusers_attention_prefix}.to_q.bias": q_bias, + f"{diffusers_attention_prefix}.to_k.weight": k_weight, + f"{diffusers_attention_prefix}.to_k.bias": k_bias, + f"{diffusers_attention_prefix}.to_v.weight": v_weight, + f"{diffusers_attention_prefix}.to_v.bias": v_bias, + } + ) + + # .encoder_kv -> .{context_key, context_value} + [encoder_k_weight, encoder_v_weight], [encoder_k_bias, encoder_v_bias] = split_attentions( + weight=checkpoint[f"{attention_prefix}.encoder_kv.weight"][:, :, 0], + bias=checkpoint[f"{attention_prefix}.encoder_kv.bias"], + split=2, + chunk_size=num_head_channels, + ) + + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.add_k_proj.weight": encoder_k_weight, + f"{diffusers_attention_prefix}.add_k_proj.bias": encoder_k_bias, + f"{diffusers_attention_prefix}.add_v_proj.weight": encoder_v_weight, + f"{diffusers_attention_prefix}.add_v_proj.bias": encoder_v_bias, + } + ) + + # .proj_out (1d conv) -> .proj_attn (linear) + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.to_out.0.weight": checkpoint[f"{attention_prefix}.proj_out.weight"][ + :, :, 0 + ], + f"{diffusers_attention_prefix}.to_out.0.bias": checkpoint[f"{attention_prefix}.proj_out.bias"], + } + ) + + return diffusers_checkpoint + + +# TODO maybe document and/or can do more efficiently (build indices in for loop and extract once for each split?) +def split_attentions(*, weight, bias, split, chunk_size): + weights = [None] * split + biases = [None] * split + + weights_biases_idx = 0 + + for starting_row_index in range(0, weight.shape[0], chunk_size): + row_indices = torch.arange(starting_row_index, starting_row_index + chunk_size) + + weight_rows = weight[row_indices, :] + bias_rows = bias[row_indices] + + if weights[weights_biases_idx] is None: + assert weights[weights_biases_idx] is None + weights[weights_biases_idx] = weight_rows + biases[weights_biases_idx] = bias_rows + else: + assert weights[weights_biases_idx] is not None + weights[weights_biases_idx] = torch.concat([weights[weights_biases_idx], weight_rows]) + biases[weights_biases_idx] = torch.concat([biases[weights_biases_idx], bias_rows]) + + weights_biases_idx = (weights_biases_idx + 1) % split + + return weights, biases + + +# done unet utils + + +def prior(*, args, checkpoint_map_location): + print("loading prior") + + prior_checkpoint = torch.load(args.prior_checkpoint_path, map_location=checkpoint_map_location) + + clip_stats_checkpoint = torch.load(args.clip_stat_path, map_location=checkpoint_map_location) + + prior_model = prior_model_from_original_config() + + prior_diffusers_checkpoint = prior_original_checkpoint_to_diffusers_checkpoint( + prior_model, prior_checkpoint, clip_stats_checkpoint + ) + + del prior_checkpoint + del clip_stats_checkpoint + + load_checkpoint_to_model(prior_diffusers_checkpoint, prior_model, strict=True) + + print("done loading prior") + + return prior_model + + +def text2img(*, args, checkpoint_map_location): + print("loading text2img") + + text2img_checkpoint = torch.load(args.text2img_checkpoint_path, map_location=checkpoint_map_location) + + unet_model = unet_model_from_original_config() + + unet_diffusers_checkpoint = unet_original_checkpoint_to_diffusers_checkpoint(unet_model, text2img_checkpoint) + + del text2img_checkpoint + + load_checkpoint_to_model(unet_diffusers_checkpoint, unet_model, strict=True) + + print("done loading text2img") + + return unet_model + + +def inpaint_text2img(*, args, checkpoint_map_location): + print("loading inpaint text2img") + + inpaint_text2img_checkpoint = torch.load( + args.inpaint_text2img_checkpoint_path, map_location=checkpoint_map_location + ) + + inpaint_unet_model = inpaint_unet_model_from_original_config() + + inpaint_unet_diffusers_checkpoint = inpaint_unet_original_checkpoint_to_diffusers_checkpoint( + inpaint_unet_model, inpaint_text2img_checkpoint + ) + + del inpaint_text2img_checkpoint + + load_checkpoint_to_model(inpaint_unet_diffusers_checkpoint, inpaint_unet_model, strict=True) + + print("done loading inpaint text2img") + + return inpaint_unet_model + + +# movq + +MOVQ_CONFIG = { + "in_channels": 3, + "out_channels": 3, + "latent_channels": 4, + "down_block_types": ("DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D", "AttnDownEncoderBlock2D"), + "up_block_types": ("AttnUpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"), + "num_vq_embeddings": 16384, + "block_out_channels": (128, 256, 256, 512), + "vq_embed_dim": 4, + "layers_per_block": 2, + "norm_type": "spatial", +} + + +def movq_model_from_original_config(): + movq = VQModel(**MOVQ_CONFIG) + return movq + + +def movq_encoder_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + # conv_in + diffusers_checkpoint.update( + { + "encoder.conv_in.weight": checkpoint["encoder.conv_in.weight"], + "encoder.conv_in.bias": checkpoint["encoder.conv_in.bias"], + } + ) + + # down_blocks + for down_block_idx, down_block in enumerate(model.encoder.down_blocks): + diffusers_down_block_prefix = f"encoder.down_blocks.{down_block_idx}" + down_block_prefix = f"encoder.down.{down_block_idx}" + + # resnets + for resnet_idx, resnet in enumerate(down_block.resnets): + diffusers_resnet_prefix = f"{diffusers_down_block_prefix}.resnets.{resnet_idx}" + resnet_prefix = f"{down_block_prefix}.block.{resnet_idx}" + + diffusers_checkpoint.update( + movq_resnet_to_diffusers_checkpoint( + resnet, checkpoint, diffusers_resnet_prefix=diffusers_resnet_prefix, resnet_prefix=resnet_prefix + ) + ) + + # downsample + + # do not include the downsample when on the last down block + # There is no downsample on the last down block + if down_block_idx != len(model.encoder.down_blocks) - 1: + # There's a single downsample in the original checkpoint but a list of downsamples + # in the diffusers model. + diffusers_downsample_prefix = f"{diffusers_down_block_prefix}.downsamplers.0.conv" + downsample_prefix = f"{down_block_prefix}.downsample.conv" + diffusers_checkpoint.update( + { + f"{diffusers_downsample_prefix}.weight": checkpoint[f"{downsample_prefix}.weight"], + f"{diffusers_downsample_prefix}.bias": checkpoint[f"{downsample_prefix}.bias"], + } + ) + + # attentions + + if hasattr(down_block, "attentions"): + for attention_idx, _ in enumerate(down_block.attentions): + diffusers_attention_prefix = f"{diffusers_down_block_prefix}.attentions.{attention_idx}" + attention_prefix = f"{down_block_prefix}.attn.{attention_idx}" + diffusers_checkpoint.update( + movq_attention_to_diffusers_checkpoint( + checkpoint, + diffusers_attention_prefix=diffusers_attention_prefix, + attention_prefix=attention_prefix, + ) + ) + + # mid block + + # mid block attentions + + # There is a single hardcoded attention block in the middle of the VQ-diffusion encoder + diffusers_attention_prefix = "encoder.mid_block.attentions.0" + attention_prefix = "encoder.mid.attn_1" + diffusers_checkpoint.update( + movq_attention_to_diffusers_checkpoint( + checkpoint, diffusers_attention_prefix=diffusers_attention_prefix, attention_prefix=attention_prefix + ) + ) + + # mid block resnets + + for diffusers_resnet_idx, resnet in enumerate(model.encoder.mid_block.resnets): + diffusers_resnet_prefix = f"encoder.mid_block.resnets.{diffusers_resnet_idx}" + + # the hardcoded prefixes to `block_` are 1 and 2 + orig_resnet_idx = diffusers_resnet_idx + 1 + # There are two hardcoded resnets in the middle of the VQ-diffusion encoder + resnet_prefix = f"encoder.mid.block_{orig_resnet_idx}" + + diffusers_checkpoint.update( + movq_resnet_to_diffusers_checkpoint( + resnet, checkpoint, diffusers_resnet_prefix=diffusers_resnet_prefix, resnet_prefix=resnet_prefix + ) + ) + + diffusers_checkpoint.update( + { + # conv_norm_out + "encoder.conv_norm_out.weight": checkpoint["encoder.norm_out.weight"], + "encoder.conv_norm_out.bias": checkpoint["encoder.norm_out.bias"], + # conv_out + "encoder.conv_out.weight": checkpoint["encoder.conv_out.weight"], + "encoder.conv_out.bias": checkpoint["encoder.conv_out.bias"], + } + ) + + return diffusers_checkpoint + + +def movq_decoder_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + # conv in + diffusers_checkpoint.update( + { + "decoder.conv_in.weight": checkpoint["decoder.conv_in.weight"], + "decoder.conv_in.bias": checkpoint["decoder.conv_in.bias"], + } + ) + + # up_blocks + + for diffusers_up_block_idx, up_block in enumerate(model.decoder.up_blocks): + # up_blocks are stored in reverse order in the VQ-diffusion checkpoint + orig_up_block_idx = len(model.decoder.up_blocks) - 1 - diffusers_up_block_idx + + diffusers_up_block_prefix = f"decoder.up_blocks.{diffusers_up_block_idx}" + up_block_prefix = f"decoder.up.{orig_up_block_idx}" + + # resnets + for resnet_idx, resnet in enumerate(up_block.resnets): + diffusers_resnet_prefix = f"{diffusers_up_block_prefix}.resnets.{resnet_idx}" + resnet_prefix = f"{up_block_prefix}.block.{resnet_idx}" + + diffusers_checkpoint.update( + movq_resnet_to_diffusers_checkpoint_spatial_norm( + resnet, checkpoint, diffusers_resnet_prefix=diffusers_resnet_prefix, resnet_prefix=resnet_prefix + ) + ) + + # upsample + + # there is no up sample on the last up block + if diffusers_up_block_idx != len(model.decoder.up_blocks) - 1: + # There's a single upsample in the VQ-diffusion checkpoint but a list of downsamples + # in the diffusers model. + diffusers_downsample_prefix = f"{diffusers_up_block_prefix}.upsamplers.0.conv" + downsample_prefix = f"{up_block_prefix}.upsample.conv" + diffusers_checkpoint.update( + { + f"{diffusers_downsample_prefix}.weight": checkpoint[f"{downsample_prefix}.weight"], + f"{diffusers_downsample_prefix}.bias": checkpoint[f"{downsample_prefix}.bias"], + } + ) + + # attentions + + if hasattr(up_block, "attentions"): + for attention_idx, _ in enumerate(up_block.attentions): + diffusers_attention_prefix = f"{diffusers_up_block_prefix}.attentions.{attention_idx}" + attention_prefix = f"{up_block_prefix}.attn.{attention_idx}" + diffusers_checkpoint.update( + movq_attention_to_diffusers_checkpoint_spatial_norm( + checkpoint, + diffusers_attention_prefix=diffusers_attention_prefix, + attention_prefix=attention_prefix, + ) + ) + + # mid block + + # mid block attentions + + # There is a single hardcoded attention block in the middle of the VQ-diffusion decoder + diffusers_attention_prefix = "decoder.mid_block.attentions.0" + attention_prefix = "decoder.mid.attn_1" + diffusers_checkpoint.update( + movq_attention_to_diffusers_checkpoint_spatial_norm( + checkpoint, diffusers_attention_prefix=diffusers_attention_prefix, attention_prefix=attention_prefix + ) + ) + + # mid block resnets + + for diffusers_resnet_idx, resnet in enumerate(model.encoder.mid_block.resnets): + diffusers_resnet_prefix = f"decoder.mid_block.resnets.{diffusers_resnet_idx}" + + # the hardcoded prefixes to `block_` are 1 and 2 + orig_resnet_idx = diffusers_resnet_idx + 1 + # There are two hardcoded resnets in the middle of the VQ-diffusion decoder + resnet_prefix = f"decoder.mid.block_{orig_resnet_idx}" + + diffusers_checkpoint.update( + movq_resnet_to_diffusers_checkpoint_spatial_norm( + resnet, checkpoint, diffusers_resnet_prefix=diffusers_resnet_prefix, resnet_prefix=resnet_prefix + ) + ) + + diffusers_checkpoint.update( + { + # conv_norm_out + "decoder.conv_norm_out.norm_layer.weight": checkpoint["decoder.norm_out.norm_layer.weight"], + "decoder.conv_norm_out.norm_layer.bias": checkpoint["decoder.norm_out.norm_layer.bias"], + "decoder.conv_norm_out.conv_y.weight": checkpoint["decoder.norm_out.conv_y.weight"], + "decoder.conv_norm_out.conv_y.bias": checkpoint["decoder.norm_out.conv_y.bias"], + "decoder.conv_norm_out.conv_b.weight": checkpoint["decoder.norm_out.conv_b.weight"], + "decoder.conv_norm_out.conv_b.bias": checkpoint["decoder.norm_out.conv_b.bias"], + # conv_out + "decoder.conv_out.weight": checkpoint["decoder.conv_out.weight"], + "decoder.conv_out.bias": checkpoint["decoder.conv_out.bias"], + } + ) + + return diffusers_checkpoint + + +def movq_resnet_to_diffusers_checkpoint(resnet, checkpoint, *, diffusers_resnet_prefix, resnet_prefix): + rv = { + # norm1 + f"{diffusers_resnet_prefix}.norm1.weight": checkpoint[f"{resnet_prefix}.norm1.weight"], + f"{diffusers_resnet_prefix}.norm1.bias": checkpoint[f"{resnet_prefix}.norm1.bias"], + # conv1 + f"{diffusers_resnet_prefix}.conv1.weight": checkpoint[f"{resnet_prefix}.conv1.weight"], + f"{diffusers_resnet_prefix}.conv1.bias": checkpoint[f"{resnet_prefix}.conv1.bias"], + # norm2 + f"{diffusers_resnet_prefix}.norm2.weight": checkpoint[f"{resnet_prefix}.norm2.weight"], + f"{diffusers_resnet_prefix}.norm2.bias": checkpoint[f"{resnet_prefix}.norm2.bias"], + # conv2 + f"{diffusers_resnet_prefix}.conv2.weight": checkpoint[f"{resnet_prefix}.conv2.weight"], + f"{diffusers_resnet_prefix}.conv2.bias": checkpoint[f"{resnet_prefix}.conv2.bias"], + } + + if resnet.conv_shortcut is not None: + rv.update( + { + f"{diffusers_resnet_prefix}.conv_shortcut.weight": checkpoint[f"{resnet_prefix}.nin_shortcut.weight"], + f"{diffusers_resnet_prefix}.conv_shortcut.bias": checkpoint[f"{resnet_prefix}.nin_shortcut.bias"], + } + ) + + return rv + + +def movq_resnet_to_diffusers_checkpoint_spatial_norm(resnet, checkpoint, *, diffusers_resnet_prefix, resnet_prefix): + rv = { + # norm1 + f"{diffusers_resnet_prefix}.norm1.norm_layer.weight": checkpoint[f"{resnet_prefix}.norm1.norm_layer.weight"], + f"{diffusers_resnet_prefix}.norm1.norm_layer.bias": checkpoint[f"{resnet_prefix}.norm1.norm_layer.bias"], + f"{diffusers_resnet_prefix}.norm1.conv_y.weight": checkpoint[f"{resnet_prefix}.norm1.conv_y.weight"], + f"{diffusers_resnet_prefix}.norm1.conv_y.bias": checkpoint[f"{resnet_prefix}.norm1.conv_y.bias"], + f"{diffusers_resnet_prefix}.norm1.conv_b.weight": checkpoint[f"{resnet_prefix}.norm1.conv_b.weight"], + f"{diffusers_resnet_prefix}.norm1.conv_b.bias": checkpoint[f"{resnet_prefix}.norm1.conv_b.bias"], + # conv1 + f"{diffusers_resnet_prefix}.conv1.weight": checkpoint[f"{resnet_prefix}.conv1.weight"], + f"{diffusers_resnet_prefix}.conv1.bias": checkpoint[f"{resnet_prefix}.conv1.bias"], + # norm2 + f"{diffusers_resnet_prefix}.norm2.norm_layer.weight": checkpoint[f"{resnet_prefix}.norm2.norm_layer.weight"], + f"{diffusers_resnet_prefix}.norm2.norm_layer.bias": checkpoint[f"{resnet_prefix}.norm2.norm_layer.bias"], + f"{diffusers_resnet_prefix}.norm2.conv_y.weight": checkpoint[f"{resnet_prefix}.norm2.conv_y.weight"], + f"{diffusers_resnet_prefix}.norm2.conv_y.bias": checkpoint[f"{resnet_prefix}.norm2.conv_y.bias"], + f"{diffusers_resnet_prefix}.norm2.conv_b.weight": checkpoint[f"{resnet_prefix}.norm2.conv_b.weight"], + f"{diffusers_resnet_prefix}.norm2.conv_b.bias": checkpoint[f"{resnet_prefix}.norm2.conv_b.bias"], + # conv2 + f"{diffusers_resnet_prefix}.conv2.weight": checkpoint[f"{resnet_prefix}.conv2.weight"], + f"{diffusers_resnet_prefix}.conv2.bias": checkpoint[f"{resnet_prefix}.conv2.bias"], + } + + if resnet.conv_shortcut is not None: + rv.update( + { + f"{diffusers_resnet_prefix}.conv_shortcut.weight": checkpoint[f"{resnet_prefix}.nin_shortcut.weight"], + f"{diffusers_resnet_prefix}.conv_shortcut.bias": checkpoint[f"{resnet_prefix}.nin_shortcut.bias"], + } + ) + + return rv + + +def movq_attention_to_diffusers_checkpoint(checkpoint, *, diffusers_attention_prefix, attention_prefix): + return { + # norm + f"{diffusers_attention_prefix}.group_norm.weight": checkpoint[f"{attention_prefix}.norm.weight"], + f"{diffusers_attention_prefix}.group_norm.bias": checkpoint[f"{attention_prefix}.norm.bias"], + # query + f"{diffusers_attention_prefix}.to_q.weight": checkpoint[f"{attention_prefix}.q.weight"][:, :, 0, 0], + f"{diffusers_attention_prefix}.to_q.bias": checkpoint[f"{attention_prefix}.q.bias"], + # key + f"{diffusers_attention_prefix}.to_k.weight": checkpoint[f"{attention_prefix}.k.weight"][:, :, 0, 0], + f"{diffusers_attention_prefix}.to_k.bias": checkpoint[f"{attention_prefix}.k.bias"], + # value + f"{diffusers_attention_prefix}.to_v.weight": checkpoint[f"{attention_prefix}.v.weight"][:, :, 0, 0], + f"{diffusers_attention_prefix}.to_v.bias": checkpoint[f"{attention_prefix}.v.bias"], + # proj_attn + f"{diffusers_attention_prefix}.to_out.0.weight": checkpoint[f"{attention_prefix}.proj_out.weight"][:, :, 0, 0], + f"{diffusers_attention_prefix}.to_out.0.bias": checkpoint[f"{attention_prefix}.proj_out.bias"], + } + + +def movq_attention_to_diffusers_checkpoint_spatial_norm(checkpoint, *, diffusers_attention_prefix, attention_prefix): + return { + # norm + f"{diffusers_attention_prefix}.spatial_norm.norm_layer.weight": checkpoint[ + f"{attention_prefix}.norm.norm_layer.weight" + ], + f"{diffusers_attention_prefix}.spatial_norm.norm_layer.bias": checkpoint[ + f"{attention_prefix}.norm.norm_layer.bias" + ], + f"{diffusers_attention_prefix}.spatial_norm.conv_y.weight": checkpoint[ + f"{attention_prefix}.norm.conv_y.weight" + ], + f"{diffusers_attention_prefix}.spatial_norm.conv_y.bias": checkpoint[f"{attention_prefix}.norm.conv_y.bias"], + f"{diffusers_attention_prefix}.spatial_norm.conv_b.weight": checkpoint[ + f"{attention_prefix}.norm.conv_b.weight" + ], + f"{diffusers_attention_prefix}.spatial_norm.conv_b.bias": checkpoint[f"{attention_prefix}.norm.conv_b.bias"], + # query + f"{diffusers_attention_prefix}.to_q.weight": checkpoint[f"{attention_prefix}.q.weight"][:, :, 0, 0], + f"{diffusers_attention_prefix}.to_q.bias": checkpoint[f"{attention_prefix}.q.bias"], + # key + f"{diffusers_attention_prefix}.to_k.weight": checkpoint[f"{attention_prefix}.k.weight"][:, :, 0, 0], + f"{diffusers_attention_prefix}.to_k.bias": checkpoint[f"{attention_prefix}.k.bias"], + # value + f"{diffusers_attention_prefix}.to_v.weight": checkpoint[f"{attention_prefix}.v.weight"][:, :, 0, 0], + f"{diffusers_attention_prefix}.to_v.bias": checkpoint[f"{attention_prefix}.v.bias"], + # proj_attn + f"{diffusers_attention_prefix}.to_out.0.weight": checkpoint[f"{attention_prefix}.proj_out.weight"][:, :, 0, 0], + f"{diffusers_attention_prefix}.to_out.0.bias": checkpoint[f"{attention_prefix}.proj_out.bias"], + } + + +def movq_original_checkpoint_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + diffusers_checkpoint.update(movq_encoder_to_diffusers_checkpoint(model, checkpoint)) + + # quant_conv + + diffusers_checkpoint.update( + { + "quant_conv.weight": checkpoint["quant_conv.weight"], + "quant_conv.bias": checkpoint["quant_conv.bias"], + } + ) + + # quantize + diffusers_checkpoint.update({"quantize.embedding.weight": checkpoint["quantize.embedding.weight"]}) + + # post_quant_conv + diffusers_checkpoint.update( + { + "post_quant_conv.weight": checkpoint["post_quant_conv.weight"], + "post_quant_conv.bias": checkpoint["post_quant_conv.bias"], + } + ) + + # decoder + diffusers_checkpoint.update(movq_decoder_to_diffusers_checkpoint(model, checkpoint)) + + return diffusers_checkpoint + + +def movq(*, args, checkpoint_map_location): + print("loading movq") + + movq_checkpoint = torch.load(args.movq_checkpoint_path, map_location=checkpoint_map_location) + + movq_model = movq_model_from_original_config() + + movq_diffusers_checkpoint = movq_original_checkpoint_to_diffusers_checkpoint(movq_model, movq_checkpoint) + + del movq_checkpoint + + load_checkpoint_to_model(movq_diffusers_checkpoint, movq_model, strict=True) + + print("done loading movq") + + return movq_model + + +def load_checkpoint_to_model(checkpoint, model, strict=False): + with tempfile.NamedTemporaryFile(delete=False) as file: + torch.save(checkpoint, file.name) + del checkpoint + if strict: + model.load_state_dict(torch.load(file.name), strict=True) + else: + load_checkpoint_and_dispatch(model, file.name, device_map="auto") + os.remove(file.name) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + + parser.add_argument( + "--prior_checkpoint_path", + default=None, + type=str, + required=False, + help="Path to the prior checkpoint to convert.", + ) + parser.add_argument( + "--clip_stat_path", + default=None, + type=str, + required=False, + help="Path to the clip stats checkpoint to convert.", + ) + parser.add_argument( + "--text2img_checkpoint_path", + default=None, + type=str, + required=False, + help="Path to the text2img checkpoint to convert.", + ) + parser.add_argument( + "--movq_checkpoint_path", + default=None, + type=str, + required=False, + help="Path to the text2img checkpoint to convert.", + ) + parser.add_argument( + "--inpaint_text2img_checkpoint_path", + default=None, + type=str, + required=False, + help="Path to the inpaint text2img checkpoint to convert.", + ) + parser.add_argument( + "--checkpoint_load_device", + default="cpu", + type=str, + required=False, + help="The device passed to `map_location` when loading checkpoints.", + ) + + parser.add_argument( + "--debug", + default=None, + type=str, + required=False, + help="Only run a specific stage of the convert script. Used for debugging", + ) + + args = parser.parse_args() + + print(f"loading checkpoints to {args.checkpoint_load_device}") + + checkpoint_map_location = torch.device(args.checkpoint_load_device) + + if args.debug is not None: + print(f"debug: only executing {args.debug}") + + if args.debug is None: + print("to-do") + elif args.debug == "prior": + prior_model = prior(args=args, checkpoint_map_location=checkpoint_map_location) + prior_model.save_pretrained(args.dump_path) + elif args.debug == "text2img": + unet_model = text2img(args=args, checkpoint_map_location=checkpoint_map_location) + unet_model.save_pretrained(f"{args.dump_path}/unet") + elif args.debug == "inpaint_text2img": + inpaint_unet_model = inpaint_text2img(args=args, checkpoint_map_location=checkpoint_map_location) + inpaint_unet_model.save_pretrained(f"{args.dump_path}/inpaint_unet") + elif args.debug == "decoder": + decoder = movq(args=args, checkpoint_map_location=checkpoint_map_location) + decoder.save_pretrained(f"{args.dump_path}/decoder") + else: + raise ValueError(f"unknown debug value : {args.debug}") diff --git a/diffusers-0.27.0/scripts/convert_ldm_original_checkpoint_to_diffusers.py b/diffusers-0.27.0/scripts/convert_ldm_original_checkpoint_to_diffusers.py new file mode 100755 index 0000000..05f0e1e --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_ldm_original_checkpoint_to_diffusers.py @@ -0,0 +1,359 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the LDM checkpoints. """ + +import argparse +import json + +import torch + +from diffusers import DDPMScheduler, LDMPipeline, UNet2DModel, VQModel + + +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("emb_layers.1", "time_emb_proj") + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("norm.weight", "group_norm.weight") + new_item = new_item.replace("norm.bias", "group_norm.bias") + + new_item = new_item.replace("proj_out.weight", "proj_attn.weight") + new_item = new_item.replace("proj_out.bias", "proj_attn.bias") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def assign_to_checkpoint( + paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None +): + """ + This does the final conversion step: take locally converted weights and apply a global renaming + to them. It splits attention layers, and takes into account additional replacements + that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + # Splits the attention layers into three variables. + if attention_paths_to_split is not None: + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape) + checkpoint[path_map["key"]] = key.reshape(target_shape) + checkpoint[path_map["value"]] = value.reshape(target_shape) + + for path in paths: + new_path = path["new"] + + # These have already been assigned + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + # Global renaming happens here + new_path = new_path.replace("middle_block.0", "mid_block.resnets.0") + new_path = new_path.replace("middle_block.1", "mid_block.attentions.0") + new_path = new_path.replace("middle_block.2", "mid_block.resnets.1") + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + # proj_attn.weight has to be converted from conv 1D to linear + if "proj_attn.weight" in new_path: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0] + else: + checkpoint[new_path] = old_checkpoint[path["old"]] + + +def convert_ldm_checkpoint(checkpoint, config): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = checkpoint["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = checkpoint["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = checkpoint["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = checkpoint["time_embed.2.bias"] + + new_checkpoint["conv_in.weight"] = checkpoint["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = checkpoint["input_blocks.0.0.bias"] + + new_checkpoint["conv_norm_out.weight"] = checkpoint["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = checkpoint["out.0.bias"] + new_checkpoint["conv_out.weight"] = checkpoint["out.2.weight"] + new_checkpoint["conv_out.bias"] = checkpoint["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in checkpoint if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in checkpoint if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in checkpoint if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in checkpoint if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in checkpoint if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in checkpoint if f"output_blocks.{layer_id}" in key] + for layer_id in range(num_output_blocks) + } + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["num_res_blocks"] + 1) + layer_in_block_id = (i - 1) % (config["num_res_blocks"] + 1) + + resnets = [key for key in input_blocks[i] if f"input_blocks.{i}.0" in key] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + + if f"input_blocks.{i}.0.op.weight" in checkpoint: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = checkpoint[ + f"input_blocks.{i}.0.op.weight" + ] + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = checkpoint[ + f"input_blocks.{i}.0.op.bias" + ] + continue + + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + resnet_op = {"old": "resnets.2.op", "new": "downsamplers.0.op"} + assign_to_checkpoint( + paths, new_checkpoint, checkpoint, additional_replacements=[meta_path, resnet_op], config=config + ) + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = { + "old": f"input_blocks.{i}.1", + "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}", + } + to_split = { + f"input_blocks.{i}.1.qkv.bias": { + "key": f"down_blocks.{block_id}.attentions.{layer_in_block_id}.key.bias", + "query": f"down_blocks.{block_id}.attentions.{layer_in_block_id}.query.bias", + "value": f"down_blocks.{block_id}.attentions.{layer_in_block_id}.value.bias", + }, + f"input_blocks.{i}.1.qkv.weight": { + "key": f"down_blocks.{block_id}.attentions.{layer_in_block_id}.key.weight", + "query": f"down_blocks.{block_id}.attentions.{layer_in_block_id}.query.weight", + "value": f"down_blocks.{block_id}.attentions.{layer_in_block_id}.value.weight", + }, + } + assign_to_checkpoint( + paths, + new_checkpoint, + checkpoint, + additional_replacements=[meta_path], + attention_paths_to_split=to_split, + config=config, + ) + + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + resnet_0_paths = renew_resnet_paths(resnet_0) + assign_to_checkpoint(resnet_0_paths, new_checkpoint, checkpoint, config=config) + + resnet_1_paths = renew_resnet_paths(resnet_1) + assign_to_checkpoint(resnet_1_paths, new_checkpoint, checkpoint, config=config) + + attentions_paths = renew_attention_paths(attentions) + to_split = { + "middle_block.1.qkv.bias": { + "key": "mid_block.attentions.0.key.bias", + "query": "mid_block.attentions.0.query.bias", + "value": "mid_block.attentions.0.value.bias", + }, + "middle_block.1.qkv.weight": { + "key": "mid_block.attentions.0.key.weight", + "query": "mid_block.attentions.0.query.weight", + "value": "mid_block.attentions.0.value.weight", + }, + } + assign_to_checkpoint( + attentions_paths, new_checkpoint, checkpoint, attention_paths_to_split=to_split, config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["num_res_blocks"] + 1) + layer_in_block_id = i % (config["num_res_blocks"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key] + + resnet_0_paths = renew_resnet_paths(resnets) + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint(paths, new_checkpoint, checkpoint, additional_replacements=[meta_path], config=config) + + if ["conv.weight", "conv.bias"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.weight", "conv.bias"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = checkpoint[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = checkpoint[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = { + "old": f"output_blocks.{i}.1", + "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}", + } + to_split = { + f"output_blocks.{i}.1.qkv.bias": { + "key": f"up_blocks.{block_id}.attentions.{layer_in_block_id}.key.bias", + "query": f"up_blocks.{block_id}.attentions.{layer_in_block_id}.query.bias", + "value": f"up_blocks.{block_id}.attentions.{layer_in_block_id}.value.bias", + }, + f"output_blocks.{i}.1.qkv.weight": { + "key": f"up_blocks.{block_id}.attentions.{layer_in_block_id}.key.weight", + "query": f"up_blocks.{block_id}.attentions.{layer_in_block_id}.query.weight", + "value": f"up_blocks.{block_id}.attentions.{layer_in_block_id}.value.weight", + }, + } + assign_to_checkpoint( + paths, + new_checkpoint, + checkpoint, + additional_replacements=[meta_path], + attention_paths_to_split=to_split if any("qkv" in key for key in attentions) else None, + config=config, + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + + new_checkpoint[new_path] = checkpoint[old_path] + + return new_checkpoint + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + + parser.add_argument( + "--config_file", + default=None, + type=str, + required=True, + help="The config json file corresponding to the architecture.", + ) + + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + + args = parser.parse_args() + + checkpoint = torch.load(args.checkpoint_path) + + with open(args.config_file) as f: + config = json.loads(f.read()) + + converted_checkpoint = convert_ldm_checkpoint(checkpoint, config) + + if "ldm" in config: + del config["ldm"] + + model = UNet2DModel(**config) + model.load_state_dict(converted_checkpoint) + + try: + scheduler = DDPMScheduler.from_config("/".join(args.checkpoint_path.split("/")[:-1])) + vqvae = VQModel.from_pretrained("/".join(args.checkpoint_path.split("/")[:-1])) + + pipe = LDMPipeline(unet=model, scheduler=scheduler, vae=vqvae) + pipe.save_pretrained(args.dump_path) + except: # noqa: E722 + model.save_pretrained(args.dump_path) diff --git a/diffusers-0.27.0/scripts/convert_lora_safetensor_to_diffusers.py b/diffusers-0.27.0/scripts/convert_lora_safetensor_to_diffusers.py new file mode 100755 index 0000000..3f05c5a --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_lora_safetensor_to_diffusers.py @@ -0,0 +1,128 @@ +# coding=utf-8 +# Copyright 2024, Haofan Wang, Qixun Wang, All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" Conversion script for the LoRA's safetensors checkpoints. """ + +import argparse + +import torch +from safetensors.torch import load_file + +from diffusers import StableDiffusionPipeline + + +def convert(base_model_path, checkpoint_path, LORA_PREFIX_UNET, LORA_PREFIX_TEXT_ENCODER, alpha): + # load base model + pipeline = StableDiffusionPipeline.from_pretrained(base_model_path, torch_dtype=torch.float32) + + # load LoRA weight from .safetensors + state_dict = load_file(checkpoint_path) + + visited = [] + + # directly update weight in diffusers model + for key in state_dict: + # it is suggested to print out the key, it usually will be something like below + # "lora_te_text_model_encoder_layers_0_self_attn_k_proj.lora_down.weight" + + # as we have set the alpha beforehand, so just skip + if ".alpha" in key or key in visited: + continue + + if "text" in key: + layer_infos = key.split(".")[0].split(LORA_PREFIX_TEXT_ENCODER + "_")[-1].split("_") + curr_layer = pipeline.text_encoder + else: + layer_infos = key.split(".")[0].split(LORA_PREFIX_UNET + "_")[-1].split("_") + curr_layer = pipeline.unet + + # find the target layer + temp_name = layer_infos.pop(0) + while len(layer_infos) > -1: + try: + curr_layer = curr_layer.__getattr__(temp_name) + if len(layer_infos) > 0: + temp_name = layer_infos.pop(0) + elif len(layer_infos) == 0: + break + except Exception: + if len(temp_name) > 0: + temp_name += "_" + layer_infos.pop(0) + else: + temp_name = layer_infos.pop(0) + + pair_keys = [] + if "lora_down" in key: + pair_keys.append(key.replace("lora_down", "lora_up")) + pair_keys.append(key) + else: + pair_keys.append(key) + pair_keys.append(key.replace("lora_up", "lora_down")) + + # update weight + if len(state_dict[pair_keys[0]].shape) == 4: + weight_up = state_dict[pair_keys[0]].squeeze(3).squeeze(2).to(torch.float32) + weight_down = state_dict[pair_keys[1]].squeeze(3).squeeze(2).to(torch.float32) + curr_layer.weight.data += alpha * torch.mm(weight_up, weight_down).unsqueeze(2).unsqueeze(3) + else: + weight_up = state_dict[pair_keys[0]].to(torch.float32) + weight_down = state_dict[pair_keys[1]].to(torch.float32) + curr_layer.weight.data += alpha * torch.mm(weight_up, weight_down) + + # update visited list + for item in pair_keys: + visited.append(item) + + return pipeline + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--base_model_path", default=None, type=str, required=True, help="Path to the base model in diffusers format." + ) + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument( + "--lora_prefix_unet", default="lora_unet", type=str, help="The prefix of UNet weight in safetensors" + ) + parser.add_argument( + "--lora_prefix_text_encoder", + default="lora_te", + type=str, + help="The prefix of text encoder weight in safetensors", + ) + parser.add_argument("--alpha", default=0.75, type=float, help="The merging ratio in W = W0 + alpha * deltaW") + parser.add_argument( + "--to_safetensors", action="store_true", help="Whether to store pipeline in safetensors format or not." + ) + parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)") + + args = parser.parse_args() + + base_model_path = args.base_model_path + checkpoint_path = args.checkpoint_path + dump_path = args.dump_path + lora_prefix_unet = args.lora_prefix_unet + lora_prefix_text_encoder = args.lora_prefix_text_encoder + alpha = args.alpha + + pipe = convert(base_model_path, checkpoint_path, lora_prefix_unet, lora_prefix_text_encoder, alpha) + + pipe = pipe.to(args.device) + pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors) diff --git a/diffusers-0.27.0/scripts/convert_models_diffuser_to_diffusers.py b/diffusers-0.27.0/scripts/convert_models_diffuser_to_diffusers.py new file mode 100755 index 0000000..cc5321e --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_models_diffuser_to_diffusers.py @@ -0,0 +1,100 @@ +import json +import os + +import torch + +from diffusers import UNet1DModel + + +os.makedirs("hub/hopper-medium-v2/unet/hor32", exist_ok=True) +os.makedirs("hub/hopper-medium-v2/unet/hor128", exist_ok=True) + +os.makedirs("hub/hopper-medium-v2/value_function", exist_ok=True) + + +def unet(hor): + if hor == 128: + down_block_types = ("DownResnetBlock1D", "DownResnetBlock1D", "DownResnetBlock1D") + block_out_channels = (32, 128, 256) + up_block_types = ("UpResnetBlock1D", "UpResnetBlock1D") + + elif hor == 32: + down_block_types = ("DownResnetBlock1D", "DownResnetBlock1D", "DownResnetBlock1D", "DownResnetBlock1D") + block_out_channels = (32, 64, 128, 256) + up_block_types = ("UpResnetBlock1D", "UpResnetBlock1D", "UpResnetBlock1D") + model = torch.load(f"/Users/bglickenhaus/Documents/diffuser/temporal_unet-hopper-mediumv2-hor{hor}.torch") + state_dict = model.state_dict() + config = { + "down_block_types": down_block_types, + "block_out_channels": block_out_channels, + "up_block_types": up_block_types, + "layers_per_block": 1, + "use_timestep_embedding": True, + "out_block_type": "OutConv1DBlock", + "norm_num_groups": 8, + "downsample_each_block": False, + "in_channels": 14, + "out_channels": 14, + "extra_in_channels": 0, + "time_embedding_type": "positional", + "flip_sin_to_cos": False, + "freq_shift": 1, + "sample_size": 65536, + "mid_block_type": "MidResTemporalBlock1D", + "act_fn": "mish", + } + hf_value_function = UNet1DModel(**config) + print(f"length of state dict: {len(state_dict.keys())}") + print(f"length of value function dict: {len(hf_value_function.state_dict().keys())}") + mapping = dict(zip(model.state_dict().keys(), hf_value_function.state_dict().keys())) + for k, v in mapping.items(): + state_dict[v] = state_dict.pop(k) + hf_value_function.load_state_dict(state_dict) + + torch.save(hf_value_function.state_dict(), f"hub/hopper-medium-v2/unet/hor{hor}/diffusion_pytorch_model.bin") + with open(f"hub/hopper-medium-v2/unet/hor{hor}/config.json", "w") as f: + json.dump(config, f) + + +def value_function(): + config = { + "in_channels": 14, + "down_block_types": ("DownResnetBlock1D", "DownResnetBlock1D", "DownResnetBlock1D", "DownResnetBlock1D"), + "up_block_types": (), + "out_block_type": "ValueFunction", + "mid_block_type": "ValueFunctionMidBlock1D", + "block_out_channels": (32, 64, 128, 256), + "layers_per_block": 1, + "downsample_each_block": True, + "sample_size": 65536, + "out_channels": 14, + "extra_in_channels": 0, + "time_embedding_type": "positional", + "use_timestep_embedding": True, + "flip_sin_to_cos": False, + "freq_shift": 1, + "norm_num_groups": 8, + "act_fn": "mish", + } + + model = torch.load("/Users/bglickenhaus/Documents/diffuser/value_function-hopper-mediumv2-hor32.torch") + state_dict = model + hf_value_function = UNet1DModel(**config) + print(f"length of state dict: {len(state_dict.keys())}") + print(f"length of value function dict: {len(hf_value_function.state_dict().keys())}") + + mapping = dict(zip(state_dict.keys(), hf_value_function.state_dict().keys())) + for k, v in mapping.items(): + state_dict[v] = state_dict.pop(k) + + hf_value_function.load_state_dict(state_dict) + + torch.save(hf_value_function.state_dict(), "hub/hopper-medium-v2/value_function/diffusion_pytorch_model.bin") + with open("hub/hopper-medium-v2/value_function/config.json", "w") as f: + json.dump(config, f) + + +if __name__ == "__main__": + unet(32) + # unet(128) + value_function() diff --git a/diffusers-0.27.0/scripts/convert_ms_text_to_video_to_diffusers.py b/diffusers-0.27.0/scripts/convert_ms_text_to_video_to_diffusers.py new file mode 100755 index 0000000..528f662 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_ms_text_to_video_to_diffusers.py @@ -0,0 +1,428 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the LDM checkpoints. """ + +import argparse + +import torch + +from diffusers import UNet3DConditionModel + + +def assign_to_checkpoint( + paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None +): + """ + This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits + attention layers, and takes into account additional replacements that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + # Splits the attention layers into three variables. + if attention_paths_to_split is not None: + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape) + checkpoint[path_map["key"]] = key.reshape(target_shape) + checkpoint[path_map["value"]] = value.reshape(target_shape) + + for path in paths: + new_path = path["new"] + + # These have already been assigned + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + # proj_attn.weight has to be converted from conv 1D to linear + weight = old_checkpoint[path["old"]] + names = ["proj_attn.weight"] + names_2 = ["proj_out.weight", "proj_in.weight"] + if any(k in new_path for k in names): + checkpoint[new_path] = weight[:, :, 0] + elif any(k in new_path for k in names_2) and len(weight.shape) > 2 and ".attentions." not in new_path: + checkpoint[new_path] = weight[:, :, 0] + else: + checkpoint[new_path] = weight + + +def renew_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + # new_item = new_item.replace('norm.weight', 'group_norm.weight') + # new_item = new_item.replace('norm.bias', 'group_norm.bias') + + # new_item = new_item.replace('proj_out.weight', 'proj_attn.weight') + # new_item = new_item.replace('proj_out.bias', 'proj_attn.bias') + + # new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +def renew_temp_conv_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + mapping.append({"old": old_item, "new": old_item}) + + return mapping + + +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("emb_layers.1", "time_emb_proj") + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + if "temopral_conv" not in old_item: + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def convert_ldm_unet_checkpoint(checkpoint, config, path=None, extract_ema=False): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + + # extract state_dict for UNet + unet_state_dict = {} + keys = list(checkpoint.keys()) + + unet_key = "model.diffusion_model." + + # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA + if sum(k.startswith("model_ema") for k in keys) > 100 and extract_ema: + print(f"Checkpoint {path} has both EMA and non-EMA weights.") + print( + "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA" + " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag." + ) + for key in keys: + if key.startswith("model.diffusion_model"): + flat_ema_key = "model_ema." + "".join(key.split(".")[1:]) + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(flat_ema_key) + else: + if sum(k.startswith("model_ema") for k in keys) > 100: + print( + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA" + " weights (usually better for inference), please make sure to add the `--extract_ema` flag." + ) + + for key in keys: + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key) + + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"] + + if config["class_embed_type"] is None: + # No parameters to port + ... + elif config["class_embed_type"] == "timestep" or config["class_embed_type"] == "projection": + new_checkpoint["class_embedding.linear_1.weight"] = unet_state_dict["label_emb.0.0.weight"] + new_checkpoint["class_embedding.linear_1.bias"] = unet_state_dict["label_emb.0.0.bias"] + new_checkpoint["class_embedding.linear_2.weight"] = unet_state_dict["label_emb.0.2.weight"] + new_checkpoint["class_embedding.linear_2.bias"] = unet_state_dict["label_emb.0.2.bias"] + else: + raise NotImplementedError(f"Not implemented `class_embed_type`: {config['class_embed_type']}") + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + first_temp_attention = [v for v in unet_state_dict if v.startswith("input_blocks.0.1")] + paths = renew_attention_paths(first_temp_attention) + meta_path = {"old": "input_blocks.0.1", "new": "transformer_in"} + assign_to_checkpoint(paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config) + + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key] + for layer_id in range(num_output_blocks) + } + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + temp_attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.2" in key] + + if f"input_blocks.{i}.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.op.bias" + ) + + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + temporal_convs = [key for key in resnets if "temopral_conv" in key] + paths = renew_temp_conv_paths(temporal_convs) + meta_path = { + "old": f"input_blocks.{i}.0.temopral_conv", + "new": f"down_blocks.{block_id}.temp_convs.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(temp_attentions): + paths = renew_attention_paths(temp_attentions) + meta_path = { + "old": f"input_blocks.{i}.2", + "new": f"down_blocks.{block_id}.temp_attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + resnet_0 = middle_blocks[0] + temporal_convs_0 = [key for key in resnet_0 if "temopral_conv" in key] + attentions = middle_blocks[1] + temp_attentions = middle_blocks[2] + resnet_1 = middle_blocks[3] + temporal_convs_1 = [key for key in resnet_1 if "temopral_conv" in key] + + resnet_0_paths = renew_resnet_paths(resnet_0) + meta_path = {"old": "middle_block.0", "new": "mid_block.resnets.0"} + assign_to_checkpoint( + resnet_0_paths, new_checkpoint, unet_state_dict, config=config, additional_replacements=[meta_path] + ) + + temp_conv_0_paths = renew_temp_conv_paths(temporal_convs_0) + meta_path = {"old": "middle_block.0.temopral_conv", "new": "mid_block.temp_convs.0"} + assign_to_checkpoint( + temp_conv_0_paths, new_checkpoint, unet_state_dict, config=config, additional_replacements=[meta_path] + ) + + resnet_1_paths = renew_resnet_paths(resnet_1) + meta_path = {"old": "middle_block.3", "new": "mid_block.resnets.1"} + assign_to_checkpoint( + resnet_1_paths, new_checkpoint, unet_state_dict, config=config, additional_replacements=[meta_path] + ) + + temp_conv_1_paths = renew_temp_conv_paths(temporal_convs_1) + meta_path = {"old": "middle_block.3.temopral_conv", "new": "mid_block.temp_convs.1"} + assign_to_checkpoint( + temp_conv_1_paths, new_checkpoint, unet_state_dict, config=config, additional_replacements=[meta_path] + ) + + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + temp_attentions_paths = renew_attention_paths(temp_attentions) + meta_path = {"old": "middle_block.2", "new": "mid_block.temp_attentions.0"} + assign_to_checkpoint( + temp_attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key] + temp_attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.2" in key] + + resnet_0_paths = renew_resnet_paths(resnets) + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + temporal_convs = [key for key in resnets if "temopral_conv" in key] + paths = renew_temp_conv_paths(temporal_convs) + meta_path = { + "old": f"output_blocks.{i}.0.temopral_conv", + "new": f"up_blocks.{block_id}.temp_convs.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + output_block_list = {k: sorted(v) for k, v in output_block_list.items()} + if ["conv.bias", "conv.weight"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.bias", "conv.weight"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = { + "old": f"output_blocks.{i}.1", + "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(temp_attentions): + paths = renew_attention_paths(temp_attentions) + meta_path = { + "old": f"output_blocks.{i}.2", + "new": f"up_blocks.{block_id}.temp_attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + new_checkpoint[new_path] = unet_state_dict[old_path] + + temopral_conv_paths = [l for l in output_block_layers if "temopral_conv" in l] + for path in temopral_conv_paths: + pruned_path = path.split("temopral_conv.")[-1] + old_path = ".".join(["output_blocks", str(i), str(block_id), "temopral_conv", pruned_path]) + new_path = ".".join(["up_blocks", str(block_id), "temp_convs", str(layer_in_block_id), pruned_path]) + new_checkpoint[new_path] = unet_state_dict[old_path] + + return new_checkpoint + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + args = parser.parse_args() + + unet_checkpoint = torch.load(args.checkpoint_path, map_location="cpu") + unet = UNet3DConditionModel() + + converted_ckpt = convert_ldm_unet_checkpoint(unet_checkpoint, unet.config) + + diff_0 = set(unet.state_dict().keys()) - set(converted_ckpt.keys()) + diff_1 = set(converted_ckpt.keys()) - set(unet.state_dict().keys()) + + assert len(diff_0) == len(diff_1) == 0, "Converted weights don't match" + + # load state_dict + unet.load_state_dict(converted_ckpt) + + unet.save_pretrained(args.dump_path) + + # -- finish converting the unet -- diff --git a/diffusers-0.27.0/scripts/convert_music_spectrogram_to_diffusers.py b/diffusers-0.27.0/scripts/convert_music_spectrogram_to_diffusers.py new file mode 100755 index 0000000..41ee8b9 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_music_spectrogram_to_diffusers.py @@ -0,0 +1,213 @@ +#!/usr/bin/env python3 +import argparse +import os + +import jax as jnp +import numpy as onp +import torch +import torch.nn as nn +from music_spectrogram_diffusion import inference +from t5x import checkpoints + +from diffusers import DDPMScheduler, OnnxRuntimeModel, SpectrogramDiffusionPipeline +from diffusers.pipelines.spectrogram_diffusion import SpectrogramContEncoder, SpectrogramNotesEncoder, T5FilmDecoder + + +MODEL = "base_with_context" + + +def load_notes_encoder(weights, model): + model.token_embedder.weight = nn.Parameter(torch.FloatTensor(weights["token_embedder"]["embedding"])) + model.position_encoding.weight = nn.Parameter( + torch.FloatTensor(weights["Embed_0"]["embedding"]), requires_grad=False + ) + for lyr_num, lyr in enumerate(model.encoders): + ly_weight = weights[f"layers_{lyr_num}"] + lyr.layer[0].layer_norm.weight = nn.Parameter( + torch.FloatTensor(ly_weight["pre_attention_layer_norm"]["scale"]) + ) + + attention_weights = ly_weight["attention"] + lyr.layer[0].SelfAttention.q.weight = nn.Parameter(torch.FloatTensor(attention_weights["query"]["kernel"].T)) + lyr.layer[0].SelfAttention.k.weight = nn.Parameter(torch.FloatTensor(attention_weights["key"]["kernel"].T)) + lyr.layer[0].SelfAttention.v.weight = nn.Parameter(torch.FloatTensor(attention_weights["value"]["kernel"].T)) + lyr.layer[0].SelfAttention.o.weight = nn.Parameter(torch.FloatTensor(attention_weights["out"]["kernel"].T)) + + lyr.layer[1].layer_norm.weight = nn.Parameter(torch.FloatTensor(ly_weight["pre_mlp_layer_norm"]["scale"])) + + lyr.layer[1].DenseReluDense.wi_0.weight = nn.Parameter(torch.FloatTensor(ly_weight["mlp"]["wi_0"]["kernel"].T)) + lyr.layer[1].DenseReluDense.wi_1.weight = nn.Parameter(torch.FloatTensor(ly_weight["mlp"]["wi_1"]["kernel"].T)) + lyr.layer[1].DenseReluDense.wo.weight = nn.Parameter(torch.FloatTensor(ly_weight["mlp"]["wo"]["kernel"].T)) + + model.layer_norm.weight = nn.Parameter(torch.FloatTensor(weights["encoder_norm"]["scale"])) + return model + + +def load_continuous_encoder(weights, model): + model.input_proj.weight = nn.Parameter(torch.FloatTensor(weights["input_proj"]["kernel"].T)) + + model.position_encoding.weight = nn.Parameter( + torch.FloatTensor(weights["Embed_0"]["embedding"]), requires_grad=False + ) + + for lyr_num, lyr in enumerate(model.encoders): + ly_weight = weights[f"layers_{lyr_num}"] + attention_weights = ly_weight["attention"] + + lyr.layer[0].SelfAttention.q.weight = nn.Parameter(torch.FloatTensor(attention_weights["query"]["kernel"].T)) + lyr.layer[0].SelfAttention.k.weight = nn.Parameter(torch.FloatTensor(attention_weights["key"]["kernel"].T)) + lyr.layer[0].SelfAttention.v.weight = nn.Parameter(torch.FloatTensor(attention_weights["value"]["kernel"].T)) + lyr.layer[0].SelfAttention.o.weight = nn.Parameter(torch.FloatTensor(attention_weights["out"]["kernel"].T)) + lyr.layer[0].layer_norm.weight = nn.Parameter( + torch.FloatTensor(ly_weight["pre_attention_layer_norm"]["scale"]) + ) + + lyr.layer[1].DenseReluDense.wi_0.weight = nn.Parameter(torch.FloatTensor(ly_weight["mlp"]["wi_0"]["kernel"].T)) + lyr.layer[1].DenseReluDense.wi_1.weight = nn.Parameter(torch.FloatTensor(ly_weight["mlp"]["wi_1"]["kernel"].T)) + lyr.layer[1].DenseReluDense.wo.weight = nn.Parameter(torch.FloatTensor(ly_weight["mlp"]["wo"]["kernel"].T)) + lyr.layer[1].layer_norm.weight = nn.Parameter(torch.FloatTensor(ly_weight["pre_mlp_layer_norm"]["scale"])) + + model.layer_norm.weight = nn.Parameter(torch.FloatTensor(weights["encoder_norm"]["scale"])) + + return model + + +def load_decoder(weights, model): + model.conditioning_emb[0].weight = nn.Parameter(torch.FloatTensor(weights["time_emb_dense0"]["kernel"].T)) + model.conditioning_emb[2].weight = nn.Parameter(torch.FloatTensor(weights["time_emb_dense1"]["kernel"].T)) + + model.position_encoding.weight = nn.Parameter( + torch.FloatTensor(weights["Embed_0"]["embedding"]), requires_grad=False + ) + + model.continuous_inputs_projection.weight = nn.Parameter( + torch.FloatTensor(weights["continuous_inputs_projection"]["kernel"].T) + ) + + for lyr_num, lyr in enumerate(model.decoders): + ly_weight = weights[f"layers_{lyr_num}"] + lyr.layer[0].layer_norm.weight = nn.Parameter( + torch.FloatTensor(ly_weight["pre_self_attention_layer_norm"]["scale"]) + ) + + lyr.layer[0].FiLMLayer.scale_bias.weight = nn.Parameter( + torch.FloatTensor(ly_weight["FiLMLayer_0"]["DenseGeneral_0"]["kernel"].T) + ) + + attention_weights = ly_weight["self_attention"] + lyr.layer[0].attention.to_q.weight = nn.Parameter(torch.FloatTensor(attention_weights["query"]["kernel"].T)) + lyr.layer[0].attention.to_k.weight = nn.Parameter(torch.FloatTensor(attention_weights["key"]["kernel"].T)) + lyr.layer[0].attention.to_v.weight = nn.Parameter(torch.FloatTensor(attention_weights["value"]["kernel"].T)) + lyr.layer[0].attention.to_out[0].weight = nn.Parameter(torch.FloatTensor(attention_weights["out"]["kernel"].T)) + + attention_weights = ly_weight["MultiHeadDotProductAttention_0"] + lyr.layer[1].attention.to_q.weight = nn.Parameter(torch.FloatTensor(attention_weights["query"]["kernel"].T)) + lyr.layer[1].attention.to_k.weight = nn.Parameter(torch.FloatTensor(attention_weights["key"]["kernel"].T)) + lyr.layer[1].attention.to_v.weight = nn.Parameter(torch.FloatTensor(attention_weights["value"]["kernel"].T)) + lyr.layer[1].attention.to_out[0].weight = nn.Parameter(torch.FloatTensor(attention_weights["out"]["kernel"].T)) + lyr.layer[1].layer_norm.weight = nn.Parameter( + torch.FloatTensor(ly_weight["pre_cross_attention_layer_norm"]["scale"]) + ) + + lyr.layer[2].layer_norm.weight = nn.Parameter(torch.FloatTensor(ly_weight["pre_mlp_layer_norm"]["scale"])) + lyr.layer[2].film.scale_bias.weight = nn.Parameter( + torch.FloatTensor(ly_weight["FiLMLayer_1"]["DenseGeneral_0"]["kernel"].T) + ) + lyr.layer[2].DenseReluDense.wi_0.weight = nn.Parameter(torch.FloatTensor(ly_weight["mlp"]["wi_0"]["kernel"].T)) + lyr.layer[2].DenseReluDense.wi_1.weight = nn.Parameter(torch.FloatTensor(ly_weight["mlp"]["wi_1"]["kernel"].T)) + lyr.layer[2].DenseReluDense.wo.weight = nn.Parameter(torch.FloatTensor(ly_weight["mlp"]["wo"]["kernel"].T)) + + model.decoder_norm.weight = nn.Parameter(torch.FloatTensor(weights["decoder_norm"]["scale"])) + + model.spec_out.weight = nn.Parameter(torch.FloatTensor(weights["spec_out_dense"]["kernel"].T)) + + return model + + +def main(args): + t5_checkpoint = checkpoints.load_t5x_checkpoint(args.checkpoint_path) + t5_checkpoint = jnp.tree_util.tree_map(onp.array, t5_checkpoint) + + gin_overrides = [ + "from __gin__ import dynamic_registration", + "from music_spectrogram_diffusion.models.diffusion import diffusion_utils", + "diffusion_utils.ClassifierFreeGuidanceConfig.eval_condition_weight = 2.0", + "diffusion_utils.DiffusionConfig.classifier_free_guidance = @diffusion_utils.ClassifierFreeGuidanceConfig()", + ] + + gin_file = os.path.join(args.checkpoint_path, "..", "config.gin") + gin_config = inference.parse_training_gin_file(gin_file, gin_overrides) + synth_model = inference.InferenceModel(args.checkpoint_path, gin_config) + + scheduler = DDPMScheduler(beta_schedule="squaredcos_cap_v2", variance_type="fixed_large") + + notes_encoder = SpectrogramNotesEncoder( + max_length=synth_model.sequence_length["inputs"], + vocab_size=synth_model.model.module.config.vocab_size, + d_model=synth_model.model.module.config.emb_dim, + dropout_rate=synth_model.model.module.config.dropout_rate, + num_layers=synth_model.model.module.config.num_encoder_layers, + num_heads=synth_model.model.module.config.num_heads, + d_kv=synth_model.model.module.config.head_dim, + d_ff=synth_model.model.module.config.mlp_dim, + feed_forward_proj="gated-gelu", + ) + + continuous_encoder = SpectrogramContEncoder( + input_dims=synth_model.audio_codec.n_dims, + targets_context_length=synth_model.sequence_length["targets_context"], + d_model=synth_model.model.module.config.emb_dim, + dropout_rate=synth_model.model.module.config.dropout_rate, + num_layers=synth_model.model.module.config.num_encoder_layers, + num_heads=synth_model.model.module.config.num_heads, + d_kv=synth_model.model.module.config.head_dim, + d_ff=synth_model.model.module.config.mlp_dim, + feed_forward_proj="gated-gelu", + ) + + decoder = T5FilmDecoder( + input_dims=synth_model.audio_codec.n_dims, + targets_length=synth_model.sequence_length["targets_context"], + max_decoder_noise_time=synth_model.model.module.config.max_decoder_noise_time, + d_model=synth_model.model.module.config.emb_dim, + num_layers=synth_model.model.module.config.num_decoder_layers, + num_heads=synth_model.model.module.config.num_heads, + d_kv=synth_model.model.module.config.head_dim, + d_ff=synth_model.model.module.config.mlp_dim, + dropout_rate=synth_model.model.module.config.dropout_rate, + ) + + notes_encoder = load_notes_encoder(t5_checkpoint["target"]["token_encoder"], notes_encoder) + continuous_encoder = load_continuous_encoder(t5_checkpoint["target"]["continuous_encoder"], continuous_encoder) + decoder = load_decoder(t5_checkpoint["target"]["decoder"], decoder) + + melgan = OnnxRuntimeModel.from_pretrained("kashif/soundstream_mel_decoder") + + pipe = SpectrogramDiffusionPipeline( + notes_encoder=notes_encoder, + continuous_encoder=continuous_encoder, + decoder=decoder, + scheduler=scheduler, + melgan=melgan, + ) + if args.save: + pipe.save_pretrained(args.output_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--output_path", default=None, type=str, required=True, help="Path to the converted model.") + parser.add_argument( + "--save", default=True, type=bool, required=False, help="Whether to save the converted model or not." + ) + parser.add_argument( + "--checkpoint_path", + default=f"{MODEL}/checkpoint_500000", + type=str, + required=False, + help="Path to the original jax model checkpoint.", + ) + args = parser.parse_args() + + main(args) diff --git a/diffusers-0.27.0/scripts/convert_ncsnpp_original_checkpoint_to_diffusers.py b/diffusers-0.27.0/scripts/convert_ncsnpp_original_checkpoint_to_diffusers.py new file mode 100755 index 0000000..56e866d --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_ncsnpp_original_checkpoint_to_diffusers.py @@ -0,0 +1,185 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the NCSNPP checkpoints. """ + +import argparse +import json + +import torch + +from diffusers import ScoreSdeVePipeline, ScoreSdeVeScheduler, UNet2DModel + + +def convert_ncsnpp_checkpoint(checkpoint, config): + """ + Takes a state dict and the path to + """ + new_model_architecture = UNet2DModel(**config) + new_model_architecture.time_proj.W.data = checkpoint["all_modules.0.W"].data + new_model_architecture.time_proj.weight.data = checkpoint["all_modules.0.W"].data + new_model_architecture.time_embedding.linear_1.weight.data = checkpoint["all_modules.1.weight"].data + new_model_architecture.time_embedding.linear_1.bias.data = checkpoint["all_modules.1.bias"].data + + new_model_architecture.time_embedding.linear_2.weight.data = checkpoint["all_modules.2.weight"].data + new_model_architecture.time_embedding.linear_2.bias.data = checkpoint["all_modules.2.bias"].data + + new_model_architecture.conv_in.weight.data = checkpoint["all_modules.3.weight"].data + new_model_architecture.conv_in.bias.data = checkpoint["all_modules.3.bias"].data + + new_model_architecture.conv_norm_out.weight.data = checkpoint[list(checkpoint.keys())[-4]].data + new_model_architecture.conv_norm_out.bias.data = checkpoint[list(checkpoint.keys())[-3]].data + new_model_architecture.conv_out.weight.data = checkpoint[list(checkpoint.keys())[-2]].data + new_model_architecture.conv_out.bias.data = checkpoint[list(checkpoint.keys())[-1]].data + + module_index = 4 + + def set_attention_weights(new_layer, old_checkpoint, index): + new_layer.query.weight.data = old_checkpoint[f"all_modules.{index}.NIN_0.W"].data.T + new_layer.key.weight.data = old_checkpoint[f"all_modules.{index}.NIN_1.W"].data.T + new_layer.value.weight.data = old_checkpoint[f"all_modules.{index}.NIN_2.W"].data.T + + new_layer.query.bias.data = old_checkpoint[f"all_modules.{index}.NIN_0.b"].data + new_layer.key.bias.data = old_checkpoint[f"all_modules.{index}.NIN_1.b"].data + new_layer.value.bias.data = old_checkpoint[f"all_modules.{index}.NIN_2.b"].data + + new_layer.proj_attn.weight.data = old_checkpoint[f"all_modules.{index}.NIN_3.W"].data.T + new_layer.proj_attn.bias.data = old_checkpoint[f"all_modules.{index}.NIN_3.b"].data + + new_layer.group_norm.weight.data = old_checkpoint[f"all_modules.{index}.GroupNorm_0.weight"].data + new_layer.group_norm.bias.data = old_checkpoint[f"all_modules.{index}.GroupNorm_0.bias"].data + + def set_resnet_weights(new_layer, old_checkpoint, index): + new_layer.conv1.weight.data = old_checkpoint[f"all_modules.{index}.Conv_0.weight"].data + new_layer.conv1.bias.data = old_checkpoint[f"all_modules.{index}.Conv_0.bias"].data + new_layer.norm1.weight.data = old_checkpoint[f"all_modules.{index}.GroupNorm_0.weight"].data + new_layer.norm1.bias.data = old_checkpoint[f"all_modules.{index}.GroupNorm_0.bias"].data + + new_layer.conv2.weight.data = old_checkpoint[f"all_modules.{index}.Conv_1.weight"].data + new_layer.conv2.bias.data = old_checkpoint[f"all_modules.{index}.Conv_1.bias"].data + new_layer.norm2.weight.data = old_checkpoint[f"all_modules.{index}.GroupNorm_1.weight"].data + new_layer.norm2.bias.data = old_checkpoint[f"all_modules.{index}.GroupNorm_1.bias"].data + + new_layer.time_emb_proj.weight.data = old_checkpoint[f"all_modules.{index}.Dense_0.weight"].data + new_layer.time_emb_proj.bias.data = old_checkpoint[f"all_modules.{index}.Dense_0.bias"].data + + if new_layer.in_channels != new_layer.out_channels or new_layer.up or new_layer.down: + new_layer.conv_shortcut.weight.data = old_checkpoint[f"all_modules.{index}.Conv_2.weight"].data + new_layer.conv_shortcut.bias.data = old_checkpoint[f"all_modules.{index}.Conv_2.bias"].data + + for i, block in enumerate(new_model_architecture.downsample_blocks): + has_attentions = hasattr(block, "attentions") + for j in range(len(block.resnets)): + set_resnet_weights(block.resnets[j], checkpoint, module_index) + module_index += 1 + if has_attentions: + set_attention_weights(block.attentions[j], checkpoint, module_index) + module_index += 1 + + if hasattr(block, "downsamplers") and block.downsamplers is not None: + set_resnet_weights(block.resnet_down, checkpoint, module_index) + module_index += 1 + block.skip_conv.weight.data = checkpoint[f"all_modules.{module_index}.Conv_0.weight"].data + block.skip_conv.bias.data = checkpoint[f"all_modules.{module_index}.Conv_0.bias"].data + module_index += 1 + + set_resnet_weights(new_model_architecture.mid_block.resnets[0], checkpoint, module_index) + module_index += 1 + set_attention_weights(new_model_architecture.mid_block.attentions[0], checkpoint, module_index) + module_index += 1 + set_resnet_weights(new_model_architecture.mid_block.resnets[1], checkpoint, module_index) + module_index += 1 + + for i, block in enumerate(new_model_architecture.up_blocks): + has_attentions = hasattr(block, "attentions") + for j in range(len(block.resnets)): + set_resnet_weights(block.resnets[j], checkpoint, module_index) + module_index += 1 + if has_attentions: + set_attention_weights( + block.attentions[0], checkpoint, module_index + ) # why can there only be a single attention layer for up? + module_index += 1 + + if hasattr(block, "resnet_up") and block.resnet_up is not None: + block.skip_norm.weight.data = checkpoint[f"all_modules.{module_index}.weight"].data + block.skip_norm.bias.data = checkpoint[f"all_modules.{module_index}.bias"].data + module_index += 1 + block.skip_conv.weight.data = checkpoint[f"all_modules.{module_index}.weight"].data + block.skip_conv.bias.data = checkpoint[f"all_modules.{module_index}.bias"].data + module_index += 1 + set_resnet_weights(block.resnet_up, checkpoint, module_index) + module_index += 1 + + new_model_architecture.conv_norm_out.weight.data = checkpoint[f"all_modules.{module_index}.weight"].data + new_model_architecture.conv_norm_out.bias.data = checkpoint[f"all_modules.{module_index}.bias"].data + module_index += 1 + new_model_architecture.conv_out.weight.data = checkpoint[f"all_modules.{module_index}.weight"].data + new_model_architecture.conv_out.bias.data = checkpoint[f"all_modules.{module_index}.bias"].data + + return new_model_architecture.state_dict() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", + default="/Users/arthurzucker/Work/diffusers/ArthurZ/diffusion_pytorch_model.bin", + type=str, + required=False, + help="Path to the checkpoint to convert.", + ) + + parser.add_argument( + "--config_file", + default="/Users/arthurzucker/Work/diffusers/ArthurZ/config.json", + type=str, + required=False, + help="The config json file corresponding to the architecture.", + ) + + parser.add_argument( + "--dump_path", + default="/Users/arthurzucker/Work/diffusers/ArthurZ/diffusion_model_new.pt", + type=str, + required=False, + help="Path to the output model.", + ) + + args = parser.parse_args() + + checkpoint = torch.load(args.checkpoint_path, map_location="cpu") + + with open(args.config_file) as f: + config = json.loads(f.read()) + + converted_checkpoint = convert_ncsnpp_checkpoint( + checkpoint, + config, + ) + + if "sde" in config: + del config["sde"] + + model = UNet2DModel(**config) + model.load_state_dict(converted_checkpoint) + + try: + scheduler = ScoreSdeVeScheduler.from_config("/".join(args.checkpoint_path.split("/")[:-1])) + + pipe = ScoreSdeVePipeline(unet=model, scheduler=scheduler) + pipe.save_pretrained(args.dump_path) + except: # noqa: E722 + model.save_pretrained(args.dump_path) diff --git a/diffusers-0.27.0/scripts/convert_original_audioldm2_to_diffusers.py b/diffusers-0.27.0/scripts/convert_original_audioldm2_to_diffusers.py new file mode 100755 index 0000000..72064d4 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_original_audioldm2_to_diffusers.py @@ -0,0 +1,1135 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the AudioLDM2 checkpoints.""" + +import argparse +import re +from typing import List, Union + +import torch +import yaml +from transformers import ( + AutoFeatureExtractor, + AutoTokenizer, + ClapConfig, + ClapModel, + GPT2Config, + GPT2Model, + SpeechT5HifiGan, + SpeechT5HifiGanConfig, + T5Config, + T5EncoderModel, +) + +from diffusers import ( + AudioLDM2Pipeline, + AudioLDM2ProjectionModel, + AudioLDM2UNet2DConditionModel, + AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, +) +from diffusers.utils import is_safetensors_available +from diffusers.utils.import_utils import BACKENDS_MAPPING + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.shave_segments +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_resnet_paths +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("emb_layers.1", "time_emb_proj") + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_vae_resnet_paths +def renew_vae_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("nin_shortcut", "conv_shortcut") + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_attention_paths +def renew_attention_paths(old_list): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + # new_item = new_item.replace('norm.weight', 'group_norm.weight') + # new_item = new_item.replace('norm.bias', 'group_norm.bias') + + # new_item = new_item.replace('proj_out.weight', 'proj_attn.weight') + # new_item = new_item.replace('proj_out.bias', 'proj_attn.bias') + + # new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_vae_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("norm.weight", "group_norm.weight") + new_item = new_item.replace("norm.bias", "group_norm.bias") + + new_item = new_item.replace("q.weight", "to_q.weight") + new_item = new_item.replace("q.bias", "to_q.bias") + + new_item = new_item.replace("k.weight", "to_k.weight") + new_item = new_item.replace("k.bias", "to_k.bias") + + new_item = new_item.replace("v.weight", "to_v.weight") + new_item = new_item.replace("v.bias", "to_v.bias") + + new_item = new_item.replace("proj_out.weight", "to_out.0.weight") + new_item = new_item.replace("proj_out.bias", "to_out.0.bias") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def assign_to_checkpoint( + paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None +): + """ + This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits + attention layers, and takes into account additional replacements that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + # Splits the attention layers into three variables. + if attention_paths_to_split is not None: + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape) + checkpoint[path_map["key"]] = key.reshape(target_shape) + checkpoint[path_map["value"]] = value.reshape(target_shape) + + for path in paths: + new_path = path["new"] + + # These have already been assigned + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + # proj_attn.weight has to be converted from conv 1D to linear + if "proj_attn.weight" in new_path: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0] + else: + checkpoint[new_path] = old_checkpoint[path["old"]] + + +def conv_attn_to_linear(checkpoint): + keys = list(checkpoint.keys()) + attn_keys = ["to_q.weight", "to_k.weight", "to_v.weight"] + proj_key = "to_out.0.weight" + for key in keys: + if ".".join(key.split(".")[-2:]) in attn_keys or ".".join(key.split(".")[-3:]) == proj_key: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key].squeeze() + + +def create_unet_diffusers_config(original_config, image_size: int): + """ + Creates a UNet config for diffusers based on the config of the original AudioLDM2 model. + """ + unet_params = original_config["model"]["params"]["unet_config"]["params"] + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + + block_out_channels = [unet_params["model_channels"] * mult for mult in unet_params["channel_mult"]] + + down_block_types = [] + resolution = 1 + for i in range(len(block_out_channels)): + block_type = "CrossAttnDownBlock2D" if resolution in unet_params["attention_resolutions"] else "DownBlock2D" + down_block_types.append(block_type) + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + block_type = "CrossAttnUpBlock2D" if resolution in unet_params["attention_resolutions"] else "UpBlock2D" + up_block_types.append(block_type) + resolution //= 2 + + vae_scale_factor = 2 ** (len(vae_params["ch_mult"]) - 1) + + cross_attention_dim = list(unet_params["context_dim"]) if "context_dim" in unet_params else block_out_channels + if len(cross_attention_dim) > 1: + # require two or more cross-attention layers per-block, each of different dimension + cross_attention_dim = [cross_attention_dim for _ in range(len(block_out_channels))] + + config = { + "sample_size": image_size // vae_scale_factor, + "in_channels": unet_params["in_channels"], + "out_channels": unet_params["out_channels"], + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "layers_per_block": unet_params["num_res_blocks"], + "transformer_layers_per_block": unet_params["transformer_depth"], + "cross_attention_dim": tuple(cross_attention_dim), + } + + return config + + +# Adapted from diffusers.pipelines.stable_diffusion.convert_from_ckpt.create_vae_diffusers_config +def create_vae_diffusers_config(original_config, checkpoint, image_size: int): + """ + Creates a VAE config for diffusers based on the config of the original AudioLDM2 model. Compared to the original + Stable Diffusion conversion, this function passes a *learnt* VAE scaling factor to the diffusers VAE. + """ + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + _ = original_config["model"]["params"]["first_stage_config"]["params"]["embed_dim"] + + block_out_channels = [vae_params["ch"] * mult for mult in vae_params["ch_mult"]] + down_block_types = ["DownEncoderBlock2D"] * len(block_out_channels) + up_block_types = ["UpDecoderBlock2D"] * len(block_out_channels) + + scaling_factor = checkpoint["scale_factor"] if "scale_by_std" in original_config["model"]["params"] else 0.18215 + + config = { + "sample_size": image_size, + "in_channels": vae_params["in_channels"], + "out_channels": vae_params["out_ch"], + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "latent_channels": vae_params["z_channels"], + "layers_per_block": vae_params["num_res_blocks"], + "scaling_factor": float(scaling_factor), + } + return config + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.create_diffusers_schedular +def create_diffusers_schedular(original_config): + schedular = DDIMScheduler( + num_train_timesteps=original_config["model"]["params"]["timesteps"], + beta_start=original_config["model"]["params"]["linear_start"], + beta_end=original_config["model"]["params"]["linear_end"], + beta_schedule="scaled_linear", + ) + return schedular + + +def convert_ldm_unet_checkpoint(checkpoint, config, path=None, extract_ema=False): + """ + Takes a state dict and a config, and returns a converted UNet checkpoint. + """ + + # extract state_dict for UNet + unet_state_dict = {} + keys = list(checkpoint.keys()) + + unet_key = "model.diffusion_model." + # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA + if sum(k.startswith("model_ema") for k in keys) > 100 and extract_ema: + print(f"Checkpoint {path} has both EMA and non-EMA weights.") + print( + "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA" + " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag." + ) + for key in keys: + if key.startswith("model.diffusion_model"): + flat_ema_key = "model_ema." + "".join(key.split(".")[1:]) + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(flat_ema_key) + else: + if sum(k.startswith("model_ema") for k in keys) > 100: + print( + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA" + " weights (usually better for inference), please make sure to add the `--extract_ema` flag." + ) + + # strip the unet prefix from the weight names + for key in keys: + if key.startswith(unet_key): + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key) + + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"] + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}." in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}." in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}." in key] + for layer_id in range(num_output_blocks) + } + + # Check how many Transformer blocks we have per layer + if isinstance(config.get("cross_attention_dim"), (list, tuple)): + if isinstance(config["cross_attention_dim"][0], (list, tuple)): + # in this case we have multiple cross-attention layers per-block + num_attention_layers = len(config.get("cross_attention_dim")[0]) + else: + num_attention_layers = 1 + + if config.get("extra_self_attn_layer"): + num_attention_layers += 1 + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.0" not in key] + + if f"input_blocks.{i}.0.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = [ + { + "old": f"input_blocks.{i}.{1 + layer_id}", + "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id * num_attention_layers + layer_id}", + } + for layer_id in range(num_attention_layers) + ] + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=meta_path, config=config + ) + + resnet_0 = middle_blocks[0] + resnet_1 = middle_blocks[num_middle_blocks - 1] + + resnet_0_paths = renew_resnet_paths(resnet_0) + meta_path = {"old": "middle_block.0", "new": "mid_block.resnets.0"} + assign_to_checkpoint( + resnet_0_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + resnet_1_paths = renew_resnet_paths(resnet_1) + meta_path = {"old": f"middle_block.{len(middle_blocks) - 1}", "new": "mid_block.resnets.1"} + assign_to_checkpoint( + resnet_1_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(1, num_middle_blocks - 1): + attentions = middle_blocks[i] + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": f"middle_block.{i}", "new": f"mid_block.attentions.{i - 1}"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.0" not in key] + + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + output_block_list = {k: sorted(v) for k, v in output_block_list.items()} + if ["conv.bias", "conv.weight"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.bias", "conv.weight"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + attentions.remove(f"output_blocks.{i}.{index}.conv.bias") + attentions.remove(f"output_blocks.{i}.{index}.conv.weight") + + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = [ + { + "old": f"output_blocks.{i}.{1 + layer_id}", + "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id * num_attention_layers + layer_id}", + } + for layer_id in range(num_attention_layers) + ] + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=meta_path, config=config + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + + new_checkpoint[new_path] = unet_state_dict[old_path] + + return new_checkpoint + + +def convert_ldm_vae_checkpoint(checkpoint, config): + # extract state dict for VAE + vae_state_dict = {} + vae_key = "first_stage_model." + keys = list(checkpoint.keys()) + for key in keys: + if key.startswith(vae_key): + vae_state_dict[key.replace(vae_key, "")] = checkpoint.get(key) + + new_checkpoint = {} + + new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"] + new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"] + new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"] + + new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"] + new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"] + new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"] + new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer}) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer}) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + return new_checkpoint + + +CLAP_KEYS_TO_MODIFY_MAPPING = { + "text_branch": "text_model", + "audio_branch": "audio_model.audio_encoder", + "attn": "attention.self", + "self.proj": "output.dense", + "attention.self_mask": "attn_mask", + "mlp.fc1": "intermediate.dense", + "mlp.fc2": "output.dense", + "norm1": "layernorm_before", + "norm2": "layernorm_after", + "bn0": "batch_norm", +} + +CLAP_KEYS_TO_IGNORE = [ + "text_transform", + "audio_transform", + "stft", + "logmel_extractor", + "tscam_conv", + "head", + "attn_mask", +] + +CLAP_EXPECTED_MISSING_KEYS = ["text_model.embeddings.token_type_ids"] + + +def convert_open_clap_checkpoint(checkpoint): + """ + Takes a state dict and returns a converted CLAP checkpoint. + """ + # extract state dict for CLAP text embedding model, discarding the audio component + model_state_dict = {} + model_key = "clap.model." + keys = list(checkpoint.keys()) + for key in keys: + if key.startswith(model_key): + model_state_dict[key.replace(model_key, "")] = checkpoint.get(key) + + new_checkpoint = {} + + sequential_layers_pattern = r".*sequential.(\d+).*" + text_projection_pattern = r".*_projection.(\d+).*" + + for key, value in model_state_dict.items(): + # check if key should be ignored in mapping - if so map it to a key name that we'll filter out at the end + for key_to_ignore in CLAP_KEYS_TO_IGNORE: + if key_to_ignore in key: + key = "spectrogram" + + # check if any key needs to be modified + for key_to_modify, new_key in CLAP_KEYS_TO_MODIFY_MAPPING.items(): + if key_to_modify in key: + key = key.replace(key_to_modify, new_key) + + if re.match(sequential_layers_pattern, key): + # replace sequential layers with list + sequential_layer = re.match(sequential_layers_pattern, key).group(1) + + key = key.replace(f"sequential.{sequential_layer}.", f"layers.{int(sequential_layer)//3}.linear.") + elif re.match(text_projection_pattern, key): + projecton_layer = int(re.match(text_projection_pattern, key).group(1)) + + # Because in CLAP they use `nn.Sequential`... + transformers_projection_layer = 1 if projecton_layer == 0 else 2 + + key = key.replace(f"_projection.{projecton_layer}.", f"_projection.linear{transformers_projection_layer}.") + + if "audio" and "qkv" in key: + # split qkv into query key and value + mixed_qkv = value + qkv_dim = mixed_qkv.size(0) // 3 + + query_layer = mixed_qkv[:qkv_dim] + key_layer = mixed_qkv[qkv_dim : qkv_dim * 2] + value_layer = mixed_qkv[qkv_dim * 2 :] + + new_checkpoint[key.replace("qkv", "query")] = query_layer + new_checkpoint[key.replace("qkv", "key")] = key_layer + new_checkpoint[key.replace("qkv", "value")] = value_layer + elif key != "spectrogram": + new_checkpoint[key] = value + + return new_checkpoint + + +def create_transformers_vocoder_config(original_config): + """ + Creates a config for transformers SpeechT5HifiGan based on the config of the vocoder model. + """ + vocoder_params = original_config["model"]["params"]["vocoder_config"]["params"] + + config = { + "model_in_dim": vocoder_params["num_mels"], + "sampling_rate": vocoder_params["sampling_rate"], + "upsample_initial_channel": vocoder_params["upsample_initial_channel"], + "upsample_rates": list(vocoder_params["upsample_rates"]), + "upsample_kernel_sizes": list(vocoder_params["upsample_kernel_sizes"]), + "resblock_kernel_sizes": list(vocoder_params["resblock_kernel_sizes"]), + "resblock_dilation_sizes": [ + list(resblock_dilation) for resblock_dilation in vocoder_params["resblock_dilation_sizes"] + ], + "normalize_before": False, + } + + return config + + +def extract_sub_model(checkpoint, key_prefix): + """ + Takes a state dict and returns the state dict for a particular sub-model. + """ + + sub_model_state_dict = {} + keys = list(checkpoint.keys()) + for key in keys: + if key.startswith(key_prefix): + sub_model_state_dict[key.replace(key_prefix, "")] = checkpoint.get(key) + + return sub_model_state_dict + + +def convert_hifigan_checkpoint(checkpoint, config): + """ + Takes a state dict and config, and returns a converted HiFiGAN vocoder checkpoint. + """ + # extract state dict for vocoder + vocoder_state_dict = extract_sub_model(checkpoint, key_prefix="first_stage_model.vocoder.") + + # fix upsampler keys, everything else is correct already + for i in range(len(config.upsample_rates)): + vocoder_state_dict[f"upsampler.{i}.weight"] = vocoder_state_dict.pop(f"ups.{i}.weight") + vocoder_state_dict[f"upsampler.{i}.bias"] = vocoder_state_dict.pop(f"ups.{i}.bias") + + if not config.normalize_before: + # if we don't set normalize_before then these variables are unused, so we set them to their initialised values + vocoder_state_dict["mean"] = torch.zeros(config.model_in_dim) + vocoder_state_dict["scale"] = torch.ones(config.model_in_dim) + + return vocoder_state_dict + + +def convert_projection_checkpoint(checkpoint): + projection_state_dict = {} + conditioner_state_dict = extract_sub_model(checkpoint, key_prefix="cond_stage_models.0.") + + projection_state_dict["sos_embed"] = conditioner_state_dict["start_of_sequence_tokens.weight"][0] + projection_state_dict["sos_embed_1"] = conditioner_state_dict["start_of_sequence_tokens.weight"][1] + + projection_state_dict["eos_embed"] = conditioner_state_dict["end_of_sequence_tokens.weight"][0] + projection_state_dict["eos_embed_1"] = conditioner_state_dict["end_of_sequence_tokens.weight"][1] + + projection_state_dict["projection.weight"] = conditioner_state_dict["input_sequence_embed_linear.0.weight"] + projection_state_dict["projection.bias"] = conditioner_state_dict["input_sequence_embed_linear.0.bias"] + + projection_state_dict["projection_1.weight"] = conditioner_state_dict["input_sequence_embed_linear.1.weight"] + projection_state_dict["projection_1.bias"] = conditioner_state_dict["input_sequence_embed_linear.1.bias"] + + return projection_state_dict + + +# Adapted from https://github.com/haoheliu/AudioLDM2/blob/81ad2c6ce015c1310387695e2dae975a7d2ed6fd/audioldm2/utils.py#L143 +DEFAULT_CONFIG = { + "model": { + "params": { + "linear_start": 0.0015, + "linear_end": 0.0195, + "timesteps": 1000, + "channels": 8, + "scale_by_std": True, + "unet_config": { + "target": "audioldm2.latent_diffusion.openaimodel.UNetModel", + "params": { + "context_dim": [None, 768, 1024], + "in_channels": 8, + "out_channels": 8, + "model_channels": 128, + "attention_resolutions": [8, 4, 2], + "num_res_blocks": 2, + "channel_mult": [1, 2, 3, 5], + "num_head_channels": 32, + "transformer_depth": 1, + }, + }, + "first_stage_config": { + "target": "audioldm2.variational_autoencoder.autoencoder.AutoencoderKL", + "params": { + "embed_dim": 8, + "ddconfig": { + "z_channels": 8, + "resolution": 256, + "in_channels": 1, + "out_ch": 1, + "ch": 128, + "ch_mult": [1, 2, 4], + "num_res_blocks": 2, + }, + }, + }, + "cond_stage_config": { + "crossattn_audiomae_generated": { + "target": "audioldm2.latent_diffusion.modules.encoders.modules.SequenceGenAudioMAECond", + "params": { + "sequence_gen_length": 8, + "sequence_input_embed_dim": [512, 1024], + }, + } + }, + "vocoder_config": { + "target": "audioldm2.first_stage_model.vocoder", + "params": { + "upsample_rates": [5, 4, 2, 2, 2], + "upsample_kernel_sizes": [16, 16, 8, 4, 4], + "upsample_initial_channel": 1024, + "resblock_kernel_sizes": [3, 7, 11], + "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]], + "num_mels": 64, + "sampling_rate": 16000, + }, + }, + }, + }, +} + + +def load_pipeline_from_original_AudioLDM2_ckpt( + checkpoint_path: str, + original_config_file: str = None, + image_size: int = 1024, + prediction_type: str = None, + extract_ema: bool = False, + scheduler_type: str = "ddim", + cross_attention_dim: Union[List, List[List]] = None, + transformer_layers_per_block: int = None, + device: str = None, + from_safetensors: bool = False, +) -> AudioLDM2Pipeline: + """ + Load an AudioLDM2 pipeline object from a `.ckpt`/`.safetensors` file and (ideally) a `.yaml` config file. + + Although many of the arguments can be automatically inferred, some of these rely on brittle checks against the + global step count, which will likely fail for models that have undergone further fine-tuning. Therefore, it is + recommended that you override the default values and/or supply an `original_config_file` wherever possible. + + Args: + checkpoint_path (`str`): Path to `.ckpt` file. + original_config_file (`str`): + Path to `.yaml` config file corresponding to the original architecture. If `None`, will be automatically + set to the AudioLDM2 base config. + image_size (`int`, *optional*, defaults to 1024): + The image size that the model was trained on. + prediction_type (`str`, *optional*): + The prediction type that the model was trained on. If `None`, will be automatically + inferred by looking for a key in the config. For the default config, the prediction type is `'epsilon'`. + scheduler_type (`str`, *optional*, defaults to 'ddim'): + Type of scheduler to use. Should be one of `["pndm", "lms", "heun", "euler", "euler-ancestral", "dpm", + "ddim"]`. + cross_attention_dim (`list`, *optional*, defaults to `None`): + The dimension of the cross-attention layers. If `None`, the cross-attention dimension will be + automatically inferred. Set to `[768, 1024]` for the base model, or `[768, 1024, None]` for the large model. + transformer_layers_per_block (`int`, *optional*, defaults to `None`): + The number of transformer layers in each transformer block. If `None`, number of layers will be " + "automatically inferred. Set to `1` for the base model, or `2` for the large model. + extract_ema (`bool`, *optional*, defaults to `False`): Only relevant for + checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights or not. Defaults to + `False`. Pass `True` to extract the EMA weights. EMA weights usually yield higher quality images for + inference. Non-EMA weights are usually better to continue fine-tuning. + device (`str`, *optional*, defaults to `None`): + The device to use. Pass `None` to determine automatically. + from_safetensors (`str`, *optional*, defaults to `False`): + If `checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch. + return: An AudioLDM2Pipeline object representing the passed-in `.ckpt`/`.safetensors` file. + """ + + if from_safetensors: + if not is_safetensors_available(): + raise ValueError(BACKENDS_MAPPING["safetensors"][1]) + + from safetensors import safe_open + + checkpoint = {} + with safe_open(checkpoint_path, framework="pt", device="cpu") as f: + for key in f.keys(): + checkpoint[key] = f.get_tensor(key) + else: + if device is None: + device = "cuda" if torch.cuda.is_available() else "cpu" + checkpoint = torch.load(checkpoint_path, map_location=device) + else: + checkpoint = torch.load(checkpoint_path, map_location=device) + + if "state_dict" in checkpoint: + checkpoint = checkpoint["state_dict"] + + if original_config_file is None: + original_config = DEFAULT_CONFIG + else: + original_config = yaml.safe_load(original_config_file) + + if image_size is not None: + original_config["model"]["params"]["unet_config"]["params"]["image_size"] = image_size + + if cross_attention_dim is not None: + original_config["model"]["params"]["unet_config"]["params"]["context_dim"] = cross_attention_dim + + if transformer_layers_per_block is not None: + original_config["model"]["params"]["unet_config"]["params"]["transformer_depth"] = transformer_layers_per_block + + if ( + "parameterization" in original_config["model"]["params"] + and original_config["model"]["params"]["parameterization"] == "v" + ): + if prediction_type is None: + prediction_type = "v_prediction" + else: + if prediction_type is None: + prediction_type = "epsilon" + + num_train_timesteps = original_config["model"]["params"]["timesteps"] + beta_start = original_config["model"]["params"]["linear_start"] + beta_end = original_config["model"]["params"]["linear_end"] + + scheduler = DDIMScheduler( + beta_end=beta_end, + beta_schedule="scaled_linear", + beta_start=beta_start, + num_train_timesteps=num_train_timesteps, + steps_offset=1, + clip_sample=False, + set_alpha_to_one=False, + prediction_type=prediction_type, + ) + # make sure scheduler works correctly with DDIM + scheduler.register_to_config(clip_sample=False) + + if scheduler_type == "pndm": + config = dict(scheduler.config) + config["skip_prk_steps"] = True + scheduler = PNDMScheduler.from_config(config) + elif scheduler_type == "lms": + scheduler = LMSDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "heun": + scheduler = HeunDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "euler": + scheduler = EulerDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "euler-ancestral": + scheduler = EulerAncestralDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "dpm": + scheduler = DPMSolverMultistepScheduler.from_config(scheduler.config) + elif scheduler_type == "ddim": + scheduler = scheduler + else: + raise ValueError(f"Scheduler of type {scheduler_type} doesn't exist!") + + # Convert the UNet2DModel + unet_config = create_unet_diffusers_config(original_config, image_size=image_size) + unet = AudioLDM2UNet2DConditionModel(**unet_config) + + converted_unet_checkpoint = convert_ldm_unet_checkpoint( + checkpoint, unet_config, path=checkpoint_path, extract_ema=extract_ema + ) + + unet.load_state_dict(converted_unet_checkpoint) + + # Convert the VAE model + vae_config = create_vae_diffusers_config(original_config, checkpoint=checkpoint, image_size=image_size) + converted_vae_checkpoint = convert_ldm_vae_checkpoint(checkpoint, vae_config) + + vae = AutoencoderKL(**vae_config) + vae.load_state_dict(converted_vae_checkpoint) + + # Convert the joint audio-text encoding model + clap_config = ClapConfig.from_pretrained("laion/clap-htsat-unfused") + clap_config.audio_config.update( + { + "patch_embeds_hidden_size": 128, + "hidden_size": 1024, + "depths": [2, 2, 12, 2], + } + ) + # AudioLDM2 uses the same tokenizer and feature extractor as the original CLAP model + clap_tokenizer = AutoTokenizer.from_pretrained("laion/clap-htsat-unfused") + clap_feature_extractor = AutoFeatureExtractor.from_pretrained("laion/clap-htsat-unfused") + + converted_clap_model = convert_open_clap_checkpoint(checkpoint) + clap_model = ClapModel(clap_config) + + missing_keys, unexpected_keys = clap_model.load_state_dict(converted_clap_model, strict=False) + # we expect not to have token_type_ids in our original state dict so let's ignore them + missing_keys = list(set(missing_keys) - set(CLAP_EXPECTED_MISSING_KEYS)) + + if len(unexpected_keys) > 0: + raise ValueError(f"Unexpected keys when loading CLAP model: {unexpected_keys}") + + if len(missing_keys) > 0: + raise ValueError(f"Missing keys when loading CLAP model: {missing_keys}") + + # Convert the vocoder model + vocoder_config = create_transformers_vocoder_config(original_config) + vocoder_config = SpeechT5HifiGanConfig(**vocoder_config) + converted_vocoder_checkpoint = convert_hifigan_checkpoint(checkpoint, vocoder_config) + + vocoder = SpeechT5HifiGan(vocoder_config) + vocoder.load_state_dict(converted_vocoder_checkpoint) + + # Convert the Flan-T5 encoder model: AudioLDM2 uses the same configuration and tokenizer as the original Flan-T5 large model + t5_config = T5Config.from_pretrained("google/flan-t5-large") + converted_t5_checkpoint = extract_sub_model(checkpoint, key_prefix="cond_stage_models.1.model.") + + t5_tokenizer = AutoTokenizer.from_pretrained("google/flan-t5-large") + # hard-coded in the original implementation (i.e. not retrievable from the config) + t5_tokenizer.model_max_length = 128 + t5_model = T5EncoderModel(t5_config) + t5_model.load_state_dict(converted_t5_checkpoint) + + # Convert the GPT2 encoder model: AudioLDM2 uses the same configuration as the original GPT2 base model + gpt2_config = GPT2Config.from_pretrained("gpt2") + gpt2_model = GPT2Model(gpt2_config) + gpt2_model.config.max_new_tokens = original_config["model"]["params"]["cond_stage_config"][ + "crossattn_audiomae_generated" + ]["params"]["sequence_gen_length"] + + converted_gpt2_checkpoint = extract_sub_model(checkpoint, key_prefix="cond_stage_models.0.model.") + gpt2_model.load_state_dict(converted_gpt2_checkpoint) + + # Convert the extra embedding / projection layers + projection_model = AudioLDM2ProjectionModel(clap_config.projection_dim, t5_config.d_model, gpt2_config.n_embd) + + converted_projection_checkpoint = convert_projection_checkpoint(checkpoint) + projection_model.load_state_dict(converted_projection_checkpoint) + + # Instantiate the diffusers pipeline + pipe = AudioLDM2Pipeline( + vae=vae, + text_encoder=clap_model, + text_encoder_2=t5_model, + projection_model=projection_model, + language_model=gpt2_model, + tokenizer=clap_tokenizer, + tokenizer_2=t5_tokenizer, + feature_extractor=clap_feature_extractor, + unet=unet, + scheduler=scheduler, + vocoder=vocoder, + ) + + return pipe + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + parser.add_argument( + "--original_config_file", + default=None, + type=str, + help="The YAML config file corresponding to the original architecture.", + ) + parser.add_argument( + "--cross_attention_dim", + default=None, + type=int, + nargs="+", + help="The dimension of the cross-attention layers. If `None`, the cross-attention dimension will be " + "automatically inferred. Set to `768+1024` for the base model, or `768+1024+640` for the large model", + ) + parser.add_argument( + "--transformer_layers_per_block", + default=None, + type=int, + help="The number of transformer layers in each transformer block. If `None`, number of layers will be " + "automatically inferred. Set to `1` for the base model, or `2` for the large model.", + ) + parser.add_argument( + "--scheduler_type", + default="ddim", + type=str, + help="Type of scheduler to use. Should be one of ['pndm', 'lms', 'ddim', 'euler', 'euler-ancestral', 'dpm']", + ) + parser.add_argument( + "--image_size", + default=1048, + type=int, + help="The image size that the model was trained on.", + ) + parser.add_argument( + "--prediction_type", + default=None, + type=str, + help=("The prediction type that the model was trained on."), + ) + parser.add_argument( + "--extract_ema", + action="store_true", + help=( + "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights" + " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield" + " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning." + ), + ) + parser.add_argument( + "--from_safetensors", + action="store_true", + help="If `--checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch.", + ) + parser.add_argument( + "--to_safetensors", + action="store_true", + help="Whether to store pipeline in safetensors format or not.", + ) + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)") + args = parser.parse_args() + + pipe = load_pipeline_from_original_AudioLDM2_ckpt( + checkpoint_path=args.checkpoint_path, + original_config_file=args.original_config_file, + image_size=args.image_size, + prediction_type=args.prediction_type, + extract_ema=args.extract_ema, + scheduler_type=args.scheduler_type, + cross_attention_dim=args.cross_attention_dim, + transformer_layers_per_block=args.transformer_layers_per_block, + from_safetensors=args.from_safetensors, + device=args.device, + ) + pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors) diff --git a/diffusers-0.27.0/scripts/convert_original_audioldm_to_diffusers.py b/diffusers-0.27.0/scripts/convert_original_audioldm_to_diffusers.py new file mode 100755 index 0000000..fbb6b08 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_original_audioldm_to_diffusers.py @@ -0,0 +1,1042 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the AudioLDM checkpoints.""" + +import argparse +import re + +import torch +import yaml +from transformers import ( + AutoTokenizer, + ClapTextConfig, + ClapTextModelWithProjection, + SpeechT5HifiGan, + SpeechT5HifiGanConfig, +) + +from diffusers import ( + AudioLDMPipeline, + AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + UNet2DConditionModel, +) + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.shave_segments +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_resnet_paths +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("emb_layers.1", "time_emb_proj") + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_vae_resnet_paths +def renew_vae_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("nin_shortcut", "conv_shortcut") + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_attention_paths +def renew_attention_paths(old_list): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + # new_item = new_item.replace('norm.weight', 'group_norm.weight') + # new_item = new_item.replace('norm.bias', 'group_norm.bias') + + # new_item = new_item.replace('proj_out.weight', 'proj_attn.weight') + # new_item = new_item.replace('proj_out.bias', 'proj_attn.bias') + + # new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_vae_attention_paths +def renew_vae_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("norm.weight", "group_norm.weight") + new_item = new_item.replace("norm.bias", "group_norm.bias") + + new_item = new_item.replace("q.weight", "query.weight") + new_item = new_item.replace("q.bias", "query.bias") + + new_item = new_item.replace("k.weight", "key.weight") + new_item = new_item.replace("k.bias", "key.bias") + + new_item = new_item.replace("v.weight", "value.weight") + new_item = new_item.replace("v.bias", "value.bias") + + new_item = new_item.replace("proj_out.weight", "proj_attn.weight") + new_item = new_item.replace("proj_out.bias", "proj_attn.bias") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.assign_to_checkpoint +def assign_to_checkpoint( + paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None +): + """ + This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits + attention layers, and takes into account additional replacements that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + # Splits the attention layers into three variables. + if attention_paths_to_split is not None: + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape) + checkpoint[path_map["key"]] = key.reshape(target_shape) + checkpoint[path_map["value"]] = value.reshape(target_shape) + + for path in paths: + new_path = path["new"] + + # These have already been assigned + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + # Global renaming happens here + new_path = new_path.replace("middle_block.0", "mid_block.resnets.0") + new_path = new_path.replace("middle_block.1", "mid_block.attentions.0") + new_path = new_path.replace("middle_block.2", "mid_block.resnets.1") + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + # proj_attn.weight has to be converted from conv 1D to linear + if "proj_attn.weight" in new_path: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0] + else: + checkpoint[new_path] = old_checkpoint[path["old"]] + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.conv_attn_to_linear +def conv_attn_to_linear(checkpoint): + keys = list(checkpoint.keys()) + attn_keys = ["query.weight", "key.weight", "value.weight"] + for key in keys: + if ".".join(key.split(".")[-2:]) in attn_keys: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0, 0] + elif "proj_attn.weight" in key: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0] + + +def create_unet_diffusers_config(original_config, image_size: int): + """ + Creates a UNet config for diffusers based on the config of the original AudioLDM model. + """ + unet_params = original_config["model"]["params"]["unet_config"]["params"] + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + + block_out_channels = [unet_params["model_channels"] * mult for mult in unet_params["channel_mult"]] + + down_block_types = [] + resolution = 1 + for i in range(len(block_out_channels)): + block_type = "CrossAttnDownBlock2D" if resolution in unet_params["attention_resolutions"] else "DownBlock2D" + down_block_types.append(block_type) + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + block_type = "CrossAttnUpBlock2D" if resolution in unet_params["attention_resolutions"] else "UpBlock2D" + up_block_types.append(block_type) + resolution //= 2 + + vae_scale_factor = 2 ** (len(vae_params["ch_mult"]) - 1) + + cross_attention_dim = ( + unet_params["cross_attention_dim"] if "cross_attention_dim" in unet_params else block_out_channels + ) + + class_embed_type = "simple_projection" if "extra_film_condition_dim" in unet_params else None + projection_class_embeddings_input_dim = ( + unet_params["extra_film_condition_dim"] if "extra_film_condition_dim" in unet_params else None + ) + class_embeddings_concat = unet_params["extra_film_use_concat"] if "extra_film_use_concat" in unet_params else None + + config = { + "sample_size": image_size // vae_scale_factor, + "in_channels": unet_params["in_channels"], + "out_channels": unet_params["out_channels"], + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "layers_per_block": unet_params["num_res_blocks"], + "cross_attention_dim": cross_attention_dim, + "class_embed_type": class_embed_type, + "projection_class_embeddings_input_dim": projection_class_embeddings_input_dim, + "class_embeddings_concat": class_embeddings_concat, + } + + return config + + +# Adapted from diffusers.pipelines.stable_diffusion.convert_from_ckpt.create_vae_diffusers_config +def create_vae_diffusers_config(original_config, checkpoint, image_size: int): + """ + Creates a VAE config for diffusers based on the config of the original AudioLDM model. Compared to the original + Stable Diffusion conversion, this function passes a *learnt* VAE scaling factor to the diffusers VAE. + """ + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + _ = original_config["model"]["params"]["first_stage_config"]["params"]["embed_dim"] + + block_out_channels = [vae_params["ch"] * mult for mult in vae_params["ch_mult"]] + down_block_types = ["DownEncoderBlock2D"] * len(block_out_channels) + up_block_types = ["UpDecoderBlock2D"] * len(block_out_channels) + + scaling_factor = checkpoint["scale_factor"] if "scale_by_std" in original_config["model"]["params"] else 0.18215 + + config = { + "sample_size": image_size, + "in_channels": vae_params["in_channels"], + "out_channels": vae_params["out_ch"], + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "latent_channels": vae_params["z_channels"], + "layers_per_block": vae_params["num_res_blocks"], + "scaling_factor": float(scaling_factor), + } + return config + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.create_diffusers_schedular +def create_diffusers_schedular(original_config): + schedular = DDIMScheduler( + num_train_timesteps=original_config["model"]["params"]["timesteps"], + beta_start=original_config["model"]["params"]["linear_start"], + beta_end=original_config["model"]["params"]["linear_end"], + beta_schedule="scaled_linear", + ) + return schedular + + +# Adapted from diffusers.pipelines.stable_diffusion.convert_from_ckpt.convert_ldm_unet_checkpoint +def convert_ldm_unet_checkpoint(checkpoint, config, path=None, extract_ema=False): + """ + Takes a state dict and a config, and returns a converted checkpoint. Compared to the original Stable Diffusion + conversion, this function additionally converts the learnt film embedding linear layer. + """ + + # extract state_dict for UNet + unet_state_dict = {} + keys = list(checkpoint.keys()) + + unet_key = "model.diffusion_model." + # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA + if sum(k.startswith("model_ema") for k in keys) > 100 and extract_ema: + print(f"Checkpoint {path} has both EMA and non-EMA weights.") + print( + "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA" + " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag." + ) + for key in keys: + if key.startswith("model.diffusion_model"): + flat_ema_key = "model_ema." + "".join(key.split(".")[1:]) + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(flat_ema_key) + else: + if sum(k.startswith("model_ema") for k in keys) > 100: + print( + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA" + " weights (usually better for inference), please make sure to add the `--extract_ema` flag." + ) + + for key in keys: + if key.startswith(unet_key): + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key) + + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"] + + new_checkpoint["class_embedding.weight"] = unet_state_dict["film_emb.weight"] + new_checkpoint["class_embedding.bias"] = unet_state_dict["film_emb.bias"] + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key] + for layer_id in range(num_output_blocks) + } + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + + if f"input_blocks.{i}.0.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + resnet_0_paths = renew_resnet_paths(resnet_0) + assign_to_checkpoint(resnet_0_paths, new_checkpoint, unet_state_dict, config=config) + + resnet_1_paths = renew_resnet_paths(resnet_1) + assign_to_checkpoint(resnet_1_paths, new_checkpoint, unet_state_dict, config=config) + + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key] + + resnet_0_paths = renew_resnet_paths(resnets) + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + output_block_list = {k: sorted(v) for k, v in output_block_list.items()} + if ["conv.bias", "conv.weight"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.bias", "conv.weight"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = { + "old": f"output_blocks.{i}.1", + "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + + new_checkpoint[new_path] = unet_state_dict[old_path] + + return new_checkpoint + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.convert_ldm_vae_checkpoint +def convert_ldm_vae_checkpoint(checkpoint, config): + # extract state dict for VAE + vae_state_dict = {} + vae_key = "first_stage_model." + keys = list(checkpoint.keys()) + for key in keys: + if key.startswith(vae_key): + vae_state_dict[key.replace(vae_key, "")] = checkpoint.get(key) + + new_checkpoint = {} + + new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"] + new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"] + new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"] + + new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"] + new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"] + new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"] + new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer}) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer}) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + return new_checkpoint + + +CLAP_KEYS_TO_MODIFY_MAPPING = { + "text_branch": "text_model", + "attn": "attention.self", + "self.proj": "output.dense", + "attention.self_mask": "attn_mask", + "mlp.fc1": "intermediate.dense", + "mlp.fc2": "output.dense", + "norm1": "layernorm_before", + "norm2": "layernorm_after", + "bn0": "batch_norm", +} + +CLAP_KEYS_TO_IGNORE = ["text_transform"] + +CLAP_EXPECTED_MISSING_KEYS = ["text_model.embeddings.token_type_ids"] + + +def convert_open_clap_checkpoint(checkpoint): + """ + Takes a state dict and returns a converted CLAP checkpoint. + """ + # extract state dict for CLAP text embedding model, discarding the audio component + model_state_dict = {} + model_key = "cond_stage_model.model.text_" + keys = list(checkpoint.keys()) + for key in keys: + if key.startswith(model_key): + model_state_dict[key.replace(model_key, "text_")] = checkpoint.get(key) + + new_checkpoint = {} + + sequential_layers_pattern = r".*sequential.(\d+).*" + text_projection_pattern = r".*_projection.(\d+).*" + + for key, value in model_state_dict.items(): + # check if key should be ignored in mapping + if key.split(".")[0] in CLAP_KEYS_TO_IGNORE: + continue + + # check if any key needs to be modified + for key_to_modify, new_key in CLAP_KEYS_TO_MODIFY_MAPPING.items(): + if key_to_modify in key: + key = key.replace(key_to_modify, new_key) + + if re.match(sequential_layers_pattern, key): + # replace sequential layers with list + sequential_layer = re.match(sequential_layers_pattern, key).group(1) + + key = key.replace(f"sequential.{sequential_layer}.", f"layers.{int(sequential_layer)//3}.linear.") + elif re.match(text_projection_pattern, key): + projecton_layer = int(re.match(text_projection_pattern, key).group(1)) + + # Because in CLAP they use `nn.Sequential`... + transformers_projection_layer = 1 if projecton_layer == 0 else 2 + + key = key.replace(f"_projection.{projecton_layer}.", f"_projection.linear{transformers_projection_layer}.") + + if "audio" and "qkv" in key: + # split qkv into query key and value + mixed_qkv = value + qkv_dim = mixed_qkv.size(0) // 3 + + query_layer = mixed_qkv[:qkv_dim] + key_layer = mixed_qkv[qkv_dim : qkv_dim * 2] + value_layer = mixed_qkv[qkv_dim * 2 :] + + new_checkpoint[key.replace("qkv", "query")] = query_layer + new_checkpoint[key.replace("qkv", "key")] = key_layer + new_checkpoint[key.replace("qkv", "value")] = value_layer + else: + new_checkpoint[key] = value + + return new_checkpoint + + +def create_transformers_vocoder_config(original_config): + """ + Creates a config for transformers SpeechT5HifiGan based on the config of the vocoder model. + """ + vocoder_params = original_config["model"]["params"]["vocoder_config"]["params"] + + config = { + "model_in_dim": vocoder_params["num_mels"], + "sampling_rate": vocoder_params["sampling_rate"], + "upsample_initial_channel": vocoder_params["upsample_initial_channel"], + "upsample_rates": list(vocoder_params["upsample_rates"]), + "upsample_kernel_sizes": list(vocoder_params["upsample_kernel_sizes"]), + "resblock_kernel_sizes": list(vocoder_params["resblock_kernel_sizes"]), + "resblock_dilation_sizes": [ + list(resblock_dilation) for resblock_dilation in vocoder_params["resblock_dilation_sizes"] + ], + "normalize_before": False, + } + + return config + + +def convert_hifigan_checkpoint(checkpoint, config): + """ + Takes a state dict and config, and returns a converted HiFiGAN vocoder checkpoint. + """ + # extract state dict for vocoder + vocoder_state_dict = {} + vocoder_key = "first_stage_model.vocoder." + keys = list(checkpoint.keys()) + for key in keys: + if key.startswith(vocoder_key): + vocoder_state_dict[key.replace(vocoder_key, "")] = checkpoint.get(key) + + # fix upsampler keys, everything else is correct already + for i in range(len(config.upsample_rates)): + vocoder_state_dict[f"upsampler.{i}.weight"] = vocoder_state_dict.pop(f"ups.{i}.weight") + vocoder_state_dict[f"upsampler.{i}.bias"] = vocoder_state_dict.pop(f"ups.{i}.bias") + + if not config.normalize_before: + # if we don't set normalize_before then these variables are unused, so we set them to their initialised values + vocoder_state_dict["mean"] = torch.zeros(config.model_in_dim) + vocoder_state_dict["scale"] = torch.ones(config.model_in_dim) + + return vocoder_state_dict + + +# Adapted from https://huggingface.co/spaces/haoheliu/audioldm-text-to-audio-generation/blob/84a0384742a22bd80c44e903e241f0623e874f1d/audioldm/utils.py#L72-L73 +DEFAULT_CONFIG = { + "model": { + "params": { + "linear_start": 0.0015, + "linear_end": 0.0195, + "timesteps": 1000, + "channels": 8, + "scale_by_std": True, + "unet_config": { + "target": "audioldm.latent_diffusion.openaimodel.UNetModel", + "params": { + "extra_film_condition_dim": 512, + "extra_film_use_concat": True, + "in_channels": 8, + "out_channels": 8, + "model_channels": 128, + "attention_resolutions": [8, 4, 2], + "num_res_blocks": 2, + "channel_mult": [1, 2, 3, 5], + "num_head_channels": 32, + }, + }, + "first_stage_config": { + "target": "audioldm.variational_autoencoder.autoencoder.AutoencoderKL", + "params": { + "embed_dim": 8, + "ddconfig": { + "z_channels": 8, + "resolution": 256, + "in_channels": 1, + "out_ch": 1, + "ch": 128, + "ch_mult": [1, 2, 4], + "num_res_blocks": 2, + }, + }, + }, + "vocoder_config": { + "target": "audioldm.first_stage_model.vocoder", + "params": { + "upsample_rates": [5, 4, 2, 2, 2], + "upsample_kernel_sizes": [16, 16, 8, 4, 4], + "upsample_initial_channel": 1024, + "resblock_kernel_sizes": [3, 7, 11], + "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]], + "num_mels": 64, + "sampling_rate": 16000, + }, + }, + }, + }, +} + + +def load_pipeline_from_original_audioldm_ckpt( + checkpoint_path: str, + original_config_file: str = None, + image_size: int = 512, + prediction_type: str = None, + extract_ema: bool = False, + scheduler_type: str = "ddim", + num_in_channels: int = None, + model_channels: int = None, + num_head_channels: int = None, + device: str = None, + from_safetensors: bool = False, +) -> AudioLDMPipeline: + """ + Load an AudioLDM pipeline object from a `.ckpt`/`.safetensors` file and (ideally) a `.yaml` config file. + + Although many of the arguments can be automatically inferred, some of these rely on brittle checks against the + global step count, which will likely fail for models that have undergone further fine-tuning. Therefore, it is + recommended that you override the default values and/or supply an `original_config_file` wherever possible. + + Args: + checkpoint_path (`str`): Path to `.ckpt` file. + original_config_file (`str`): + Path to `.yaml` config file corresponding to the original architecture. If `None`, will be automatically + set to the audioldm-s-full-v2 config. + image_size (`int`, *optional*, defaults to 512): + The image size that the model was trained on. + prediction_type (`str`, *optional*): + The prediction type that the model was trained on. If `None`, will be automatically + inferred by looking for a key in the config. For the default config, the prediction type is `'epsilon'`. + num_in_channels (`int`, *optional*, defaults to None): + The number of UNet input channels. If `None`, it will be automatically inferred from the config. + model_channels (`int`, *optional*, defaults to None): + The number of UNet model channels. If `None`, it will be automatically inferred from the config. Override + to 128 for the small checkpoints, 192 for the medium checkpoints and 256 for the large. + num_head_channels (`int`, *optional*, defaults to None): + The number of UNet head channels. If `None`, it will be automatically inferred from the config. Override + to 32 for the small and medium checkpoints, and 64 for the large. + scheduler_type (`str`, *optional*, defaults to 'pndm'): + Type of scheduler to use. Should be one of `["pndm", "lms", "heun", "euler", "euler-ancestral", "dpm", + "ddim"]`. + extract_ema (`bool`, *optional*, defaults to `False`): Only relevant for + checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights or not. Defaults to + `False`. Pass `True` to extract the EMA weights. EMA weights usually yield higher quality images for + inference. Non-EMA weights are usually better to continue fine-tuning. + device (`str`, *optional*, defaults to `None`): + The device to use. Pass `None` to determine automatically. + from_safetensors (`str`, *optional*, defaults to `False`): + If `checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch. + return: An AudioLDMPipeline object representing the passed-in `.ckpt`/`.safetensors` file. + """ + + if from_safetensors: + from safetensors import safe_open + + checkpoint = {} + with safe_open(checkpoint_path, framework="pt", device="cpu") as f: + for key in f.keys(): + checkpoint[key] = f.get_tensor(key) + else: + if device is None: + device = "cuda" if torch.cuda.is_available() else "cpu" + checkpoint = torch.load(checkpoint_path, map_location=device) + else: + checkpoint = torch.load(checkpoint_path, map_location=device) + + if "state_dict" in checkpoint: + checkpoint = checkpoint["state_dict"] + + if original_config_file is None: + original_config = DEFAULT_CONFIG + else: + original_config = yaml.safe_load(original_config_file) + + if num_in_channels is not None: + original_config["model"]["params"]["unet_config"]["params"]["in_channels"] = num_in_channels + + if model_channels is not None: + original_config["model"]["params"]["unet_config"]["params"]["model_channels"] = model_channels + + if num_head_channels is not None: + original_config["model"]["params"]["unet_config"]["params"]["num_head_channels"] = num_head_channels + + if ( + "parameterization" in original_config["model"]["params"] + and original_config["model"]["params"]["parameterization"] == "v" + ): + if prediction_type is None: + prediction_type = "v_prediction" + else: + if prediction_type is None: + prediction_type = "epsilon" + + if image_size is None: + image_size = 512 + + num_train_timesteps = original_config["model"]["params"]["timesteps"] + beta_start = original_config["model"]["params"]["linear_start"] + beta_end = original_config["model"]["params"]["linear_end"] + + scheduler = DDIMScheduler( + beta_end=beta_end, + beta_schedule="scaled_linear", + beta_start=beta_start, + num_train_timesteps=num_train_timesteps, + steps_offset=1, + clip_sample=False, + set_alpha_to_one=False, + prediction_type=prediction_type, + ) + # make sure scheduler works correctly with DDIM + scheduler.register_to_config(clip_sample=False) + + if scheduler_type == "pndm": + config = dict(scheduler.config) + config["skip_prk_steps"] = True + scheduler = PNDMScheduler.from_config(config) + elif scheduler_type == "lms": + scheduler = LMSDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "heun": + scheduler = HeunDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "euler": + scheduler = EulerDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "euler-ancestral": + scheduler = EulerAncestralDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "dpm": + scheduler = DPMSolverMultistepScheduler.from_config(scheduler.config) + elif scheduler_type == "ddim": + scheduler = scheduler + else: + raise ValueError(f"Scheduler of type {scheduler_type} doesn't exist!") + + # Convert the UNet2DModel + unet_config = create_unet_diffusers_config(original_config, image_size=image_size) + unet = UNet2DConditionModel(**unet_config) + + converted_unet_checkpoint = convert_ldm_unet_checkpoint( + checkpoint, unet_config, path=checkpoint_path, extract_ema=extract_ema + ) + + unet.load_state_dict(converted_unet_checkpoint) + + # Convert the VAE model + vae_config = create_vae_diffusers_config(original_config, checkpoint=checkpoint, image_size=image_size) + converted_vae_checkpoint = convert_ldm_vae_checkpoint(checkpoint, vae_config) + + vae = AutoencoderKL(**vae_config) + vae.load_state_dict(converted_vae_checkpoint) + + # Convert the text model + # AudioLDM uses the same configuration and tokenizer as the original CLAP model + config = ClapTextConfig.from_pretrained("laion/clap-htsat-unfused") + tokenizer = AutoTokenizer.from_pretrained("laion/clap-htsat-unfused") + + converted_text_model = convert_open_clap_checkpoint(checkpoint) + text_model = ClapTextModelWithProjection(config) + + missing_keys, unexpected_keys = text_model.load_state_dict(converted_text_model, strict=False) + # we expect not to have token_type_ids in our original state dict so let's ignore them + missing_keys = list(set(missing_keys) - set(CLAP_EXPECTED_MISSING_KEYS)) + + if len(unexpected_keys) > 0: + raise ValueError(f"Unexpected keys when loading CLAP model: {unexpected_keys}") + + if len(missing_keys) > 0: + raise ValueError(f"Missing keys when loading CLAP model: {missing_keys}") + + # Convert the vocoder model + vocoder_config = create_transformers_vocoder_config(original_config) + vocoder_config = SpeechT5HifiGanConfig(**vocoder_config) + converted_vocoder_checkpoint = convert_hifigan_checkpoint(checkpoint, vocoder_config) + + vocoder = SpeechT5HifiGan(vocoder_config) + vocoder.load_state_dict(converted_vocoder_checkpoint) + + # Instantiate the diffusers pipeline + pipe = AudioLDMPipeline( + vae=vae, + text_encoder=text_model, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + vocoder=vocoder, + ) + + return pipe + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + parser.add_argument( + "--original_config_file", + default=None, + type=str, + help="The YAML config file corresponding to the original architecture.", + ) + parser.add_argument( + "--num_in_channels", + default=None, + type=int, + help="The number of input channels. If `None` number of input channels will be automatically inferred.", + ) + parser.add_argument( + "--model_channels", + default=None, + type=int, + help="The number of UNet model channels. If `None`, it will be automatically inferred from the config. Override" + " to 128 for the small checkpoints, 192 for the medium checkpoints and 256 for the large.", + ) + parser.add_argument( + "--num_head_channels", + default=None, + type=int, + help="The number of UNet head channels. If `None`, it will be automatically inferred from the config. Override" + " to 32 for the small and medium checkpoints, and 64 for the large.", + ) + parser.add_argument( + "--scheduler_type", + default="ddim", + type=str, + help="Type of scheduler to use. Should be one of ['pndm', 'lms', 'ddim', 'euler', 'euler-ancestral', 'dpm']", + ) + parser.add_argument( + "--image_size", + default=None, + type=int, + help=("The image size that the model was trained on."), + ) + parser.add_argument( + "--prediction_type", + default=None, + type=str, + help=("The prediction type that the model was trained on."), + ) + parser.add_argument( + "--extract_ema", + action="store_true", + help=( + "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights" + " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield" + " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning." + ), + ) + parser.add_argument( + "--from_safetensors", + action="store_true", + help="If `--checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch.", + ) + parser.add_argument( + "--to_safetensors", + action="store_true", + help="Whether to store pipeline in safetensors format or not.", + ) + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)") + args = parser.parse_args() + + pipe = load_pipeline_from_original_audioldm_ckpt( + checkpoint_path=args.checkpoint_path, + original_config_file=args.original_config_file, + image_size=args.image_size, + prediction_type=args.prediction_type, + extract_ema=args.extract_ema, + scheduler_type=args.scheduler_type, + num_in_channels=args.num_in_channels, + model_channels=args.model_channels, + num_head_channels=args.num_head_channels, + from_safetensors=args.from_safetensors, + device=args.device, + ) + pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors) diff --git a/diffusers-0.27.0/scripts/convert_original_controlnet_to_diffusers.py b/diffusers-0.27.0/scripts/convert_original_controlnet_to_diffusers.py new file mode 100755 index 0000000..4a8e2d6 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_original_controlnet_to_diffusers.py @@ -0,0 +1,109 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for stable diffusion checkpoints which _only_ contain a controlnet. """ + +import argparse + +from diffusers.pipelines.stable_diffusion.convert_from_ckpt import download_controlnet_from_original_ckpt + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + parser.add_argument( + "--original_config_file", + type=str, + required=True, + help="The YAML config file corresponding to the original architecture.", + ) + parser.add_argument( + "--num_in_channels", + default=None, + type=int, + help="The number of input channels. If `None` number of input channels will be automatically inferred.", + ) + parser.add_argument( + "--image_size", + default=512, + type=int, + help=( + "The image size that the model was trained on. Use 512 for Stable Diffusion v1.X and Stable Siffusion v2" + " Base. Use 768 for Stable Diffusion v2." + ), + ) + parser.add_argument( + "--extract_ema", + action="store_true", + help=( + "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights" + " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield" + " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning." + ), + ) + parser.add_argument( + "--upcast_attention", + action="store_true", + help=( + "Whether the attention computation should always be upcasted. This is necessary when running stable" + " diffusion 2.1." + ), + ) + parser.add_argument( + "--from_safetensors", + action="store_true", + help="If `--checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch.", + ) + parser.add_argument( + "--to_safetensors", + action="store_true", + help="Whether to store pipeline in safetensors format or not.", + ) + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)") + + # small workaround to get argparser to parse a boolean input as either true _or_ false + def parse_bool(string): + if string == "True": + return True + elif string == "False": + return False + else: + raise ValueError(f"could not parse string as bool {string}") + + parser.add_argument( + "--use_linear_projection", help="Override for use linear projection", required=False, type=parse_bool + ) + + parser.add_argument("--cross_attention_dim", help="Override for cross attention_dim", required=False, type=int) + + args = parser.parse_args() + + controlnet = download_controlnet_from_original_ckpt( + checkpoint_path=args.checkpoint_path, + original_config_file=args.original_config_file, + image_size=args.image_size, + extract_ema=args.extract_ema, + num_in_channels=args.num_in_channels, + upcast_attention=args.upcast_attention, + from_safetensors=args.from_safetensors, + device=args.device, + use_linear_projection=args.use_linear_projection, + cross_attention_dim=args.cross_attention_dim, + ) + + controlnet.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors) diff --git a/diffusers-0.27.0/scripts/convert_original_musicldm_to_diffusers.py b/diffusers-0.27.0/scripts/convert_original_musicldm_to_diffusers.py new file mode 100755 index 0000000..3de91b4 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_original_musicldm_to_diffusers.py @@ -0,0 +1,1056 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the MusicLDM checkpoints.""" + +import argparse +import re + +import torch +import yaml +from transformers import ( + AutoFeatureExtractor, + AutoTokenizer, + ClapConfig, + ClapModel, + SpeechT5HifiGan, + SpeechT5HifiGanConfig, +) + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + LMSDiscreteScheduler, + MusicLDMPipeline, + PNDMScheduler, + UNet2DConditionModel, +) + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.shave_segments +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_resnet_paths +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("emb_layers.1", "time_emb_proj") + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_vae_resnet_paths +def renew_vae_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("nin_shortcut", "conv_shortcut") + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_attention_paths +def renew_attention_paths(old_list): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + # new_item = new_item.replace('norm.weight', 'group_norm.weight') + # new_item = new_item.replace('norm.bias', 'group_norm.bias') + + # new_item = new_item.replace('proj_out.weight', 'proj_attn.weight') + # new_item = new_item.replace('proj_out.bias', 'proj_attn.bias') + + # new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_vae_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("norm.weight", "group_norm.weight") + new_item = new_item.replace("norm.bias", "group_norm.bias") + + new_item = new_item.replace("q.weight", "to_q.weight") + new_item = new_item.replace("q.bias", "to_q.bias") + + new_item = new_item.replace("k.weight", "to_k.weight") + new_item = new_item.replace("k.bias", "to_k.bias") + + new_item = new_item.replace("v.weight", "to_v.weight") + new_item = new_item.replace("v.bias", "to_v.bias") + + new_item = new_item.replace("proj_out.weight", "to_out.0.weight") + new_item = new_item.replace("proj_out.bias", "to_out.0.bias") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.assign_to_checkpoint +def assign_to_checkpoint( + paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None +): + """ + This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits + attention layers, and takes into account additional replacements that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + # Splits the attention layers into three variables. + if attention_paths_to_split is not None: + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape) + checkpoint[path_map["key"]] = key.reshape(target_shape) + checkpoint[path_map["value"]] = value.reshape(target_shape) + + for path in paths: + new_path = path["new"] + + # These have already been assigned + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + # Global renaming happens here + new_path = new_path.replace("middle_block.0", "mid_block.resnets.0") + new_path = new_path.replace("middle_block.1", "mid_block.attentions.0") + new_path = new_path.replace("middle_block.2", "mid_block.resnets.1") + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + # proj_attn.weight has to be converted from conv 1D to linear + if "proj_attn.weight" in new_path: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0] + else: + checkpoint[new_path] = old_checkpoint[path["old"]] + + +def conv_attn_to_linear(checkpoint): + keys = list(checkpoint.keys()) + attn_keys = ["to_q.weight", "to_k.weight", "to_v.weight"] + proj_key = "to_out.0.weight" + for key in keys: + if ".".join(key.split(".")[-2:]) in attn_keys or ".".join(key.split(".")[-3:]) == proj_key: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key].squeeze() + + +def create_unet_diffusers_config(original_config, image_size: int): + """ + Creates a UNet config for diffusers based on the config of the original MusicLDM model. + """ + unet_params = original_config["model"]["params"]["unet_config"]["params"] + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + + block_out_channels = [unet_params["model_channels"] * mult for mult in unet_params["channel_mult"]] + + down_block_types = [] + resolution = 1 + for i in range(len(block_out_channels)): + block_type = "CrossAttnDownBlock2D" if resolution in unet_params["attention_resolutions"] else "DownBlock2D" + down_block_types.append(block_type) + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + block_type = "CrossAttnUpBlock2D" if resolution in unet_params["attention_resolutions"] else "UpBlock2D" + up_block_types.append(block_type) + resolution //= 2 + + vae_scale_factor = 2 ** (len(vae_params["ch_mult"]) - 1) + + cross_attention_dim = ( + unet_params["cross_attention_dim"] if "cross_attention_dim" in unet_params else block_out_channels + ) + + class_embed_type = "simple_projection" if "extra_film_condition_dim" in unet_params else None + projection_class_embeddings_input_dim = ( + unet_params["extra_film_condition_dim"] if "extra_film_condition_dim" in unet_params else None + ) + class_embeddings_concat = unet_params["extra_film_use_concat"] if "extra_film_use_concat" in unet_params else None + + config = { + "sample_size": image_size // vae_scale_factor, + "in_channels": unet_params["in_channels"], + "out_channels": unet_params["out_channels"], + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "layers_per_block": unet_params["num_res_blocks"], + "cross_attention_dim": cross_attention_dim, + "class_embed_type": class_embed_type, + "projection_class_embeddings_input_dim": projection_class_embeddings_input_dim, + "class_embeddings_concat": class_embeddings_concat, + } + + return config + + +# Adapted from diffusers.pipelines.stable_diffusion.convert_from_ckpt.create_vae_diffusers_config +def create_vae_diffusers_config(original_config, checkpoint, image_size: int): + """ + Creates a VAE config for diffusers based on the config of the original MusicLDM model. Compared to the original + Stable Diffusion conversion, this function passes a *learnt* VAE scaling factor to the diffusers VAE. + """ + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + _ = original_config["model"]["params"]["first_stage_config"]["params"]["embed_dim"] + + block_out_channels = [vae_params["ch"] * mult for mult in vae_params["ch_mult"]] + down_block_types = ["DownEncoderBlock2D"] * len(block_out_channels) + up_block_types = ["UpDecoderBlock2D"] * len(block_out_channels) + + scaling_factor = checkpoint["scale_factor"] if "scale_by_std" in original_config["model"]["params"] else 0.18215 + + config = { + "sample_size": image_size, + "in_channels": vae_params["in_channels"], + "out_channels": vae_params["out_ch"], + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "latent_channels": vae_params["z_channels"], + "layers_per_block": vae_params["num_res_blocks"], + "scaling_factor": float(scaling_factor), + } + return config + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.create_diffusers_schedular +def create_diffusers_schedular(original_config): + schedular = DDIMScheduler( + num_train_timesteps=original_config["model"]["params"]["timesteps"], + beta_start=original_config["model"]["params"]["linear_start"], + beta_end=original_config["model"]["params"]["linear_end"], + beta_schedule="scaled_linear", + ) + return schedular + + +def convert_ldm_unet_checkpoint(checkpoint, config, path=None, extract_ema=False): + """ + Takes a state dict and a config, and returns a converted checkpoint. Compared to the original Stable Diffusion + conversion, this function additionally converts the learnt film embedding linear layer. + """ + + # extract state_dict for UNet + unet_state_dict = {} + keys = list(checkpoint.keys()) + + unet_key = "model.diffusion_model." + # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA + if sum(k.startswith("model_ema") for k in keys) > 100 and extract_ema: + print(f"Checkpoint {path} has both EMA and non-EMA weights.") + print( + "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA" + " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag." + ) + for key in keys: + if key.startswith("model.diffusion_model"): + flat_ema_key = "model_ema." + "".join(key.split(".")[1:]) + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(flat_ema_key) + else: + if sum(k.startswith("model_ema") for k in keys) > 100: + print( + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA" + " weights (usually better for inference), please make sure to add the `--extract_ema` flag." + ) + + for key in keys: + if key.startswith(unet_key): + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key) + + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"] + + new_checkpoint["class_embedding.weight"] = unet_state_dict["film_emb.weight"] + new_checkpoint["class_embedding.bias"] = unet_state_dict["film_emb.bias"] + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key] + for layer_id in range(num_output_blocks) + } + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + + if f"input_blocks.{i}.0.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + resnet_0_paths = renew_resnet_paths(resnet_0) + assign_to_checkpoint(resnet_0_paths, new_checkpoint, unet_state_dict, config=config) + + resnet_1_paths = renew_resnet_paths(resnet_1) + assign_to_checkpoint(resnet_1_paths, new_checkpoint, unet_state_dict, config=config) + + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key] + + resnet_0_paths = renew_resnet_paths(resnets) + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + output_block_list = {k: sorted(v) for k, v in output_block_list.items()} + if ["conv.bias", "conv.weight"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.bias", "conv.weight"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = { + "old": f"output_blocks.{i}.1", + "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + + new_checkpoint[new_path] = unet_state_dict[old_path] + + return new_checkpoint + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.convert_ldm_vae_checkpoint +def convert_ldm_vae_checkpoint(checkpoint, config): + # extract state dict for VAE + vae_state_dict = {} + vae_key = "first_stage_model." + keys = list(checkpoint.keys()) + for key in keys: + if key.startswith(vae_key): + vae_state_dict[key.replace(vae_key, "")] = checkpoint.get(key) + + new_checkpoint = {} + + new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"] + new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"] + new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"] + + new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"] + new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"] + new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"] + new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer}) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer}) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + return new_checkpoint + + +CLAP_KEYS_TO_MODIFY_MAPPING = { + "text_branch": "text_model", + "audio_branch": "audio_model.audio_encoder", + "attn": "attention.self", + "self.proj": "output.dense", + "attention.self_mask": "attn_mask", + "mlp.fc1": "intermediate.dense", + "mlp.fc2": "output.dense", + "norm1": "layernorm_before", + "norm2": "layernorm_after", + "bn0": "batch_norm", +} + +CLAP_KEYS_TO_IGNORE = [ + "text_transform", + "audio_transform", + "stft", + "logmel_extractor", + "tscam_conv", + "head", + "attn_mask", +] + +CLAP_EXPECTED_MISSING_KEYS = ["text_model.embeddings.token_type_ids"] + + +def convert_open_clap_checkpoint(checkpoint): + """ + Takes a state dict and returns a converted CLAP checkpoint. + """ + # extract state dict for CLAP text embedding model, discarding the audio component + model_state_dict = {} + model_key = "cond_stage_model.model." + keys = list(checkpoint.keys()) + for key in keys: + if key.startswith(model_key): + model_state_dict[key.replace(model_key, "")] = checkpoint.get(key) + + new_checkpoint = {} + + sequential_layers_pattern = r".*sequential.(\d+).*" + text_projection_pattern = r".*_projection.(\d+).*" + + for key, value in model_state_dict.items(): + # check if key should be ignored in mapping - if so map it to a key name that we'll filter out at the end + for key_to_ignore in CLAP_KEYS_TO_IGNORE: + if key_to_ignore in key: + key = "spectrogram" + + # check if any key needs to be modified + for key_to_modify, new_key in CLAP_KEYS_TO_MODIFY_MAPPING.items(): + if key_to_modify in key: + key = key.replace(key_to_modify, new_key) + + if re.match(sequential_layers_pattern, key): + # replace sequential layers with list + sequential_layer = re.match(sequential_layers_pattern, key).group(1) + + key = key.replace(f"sequential.{sequential_layer}.", f"layers.{int(sequential_layer)//3}.linear.") + elif re.match(text_projection_pattern, key): + projecton_layer = int(re.match(text_projection_pattern, key).group(1)) + + # Because in CLAP they use `nn.Sequential`... + transformers_projection_layer = 1 if projecton_layer == 0 else 2 + + key = key.replace(f"_projection.{projecton_layer}.", f"_projection.linear{transformers_projection_layer}.") + + if "audio" and "qkv" in key: + # split qkv into query key and value + mixed_qkv = value + qkv_dim = mixed_qkv.size(0) // 3 + + query_layer = mixed_qkv[:qkv_dim] + key_layer = mixed_qkv[qkv_dim : qkv_dim * 2] + value_layer = mixed_qkv[qkv_dim * 2 :] + + new_checkpoint[key.replace("qkv", "query")] = query_layer + new_checkpoint[key.replace("qkv", "key")] = key_layer + new_checkpoint[key.replace("qkv", "value")] = value_layer + elif key != "spectrogram": + new_checkpoint[key] = value + + return new_checkpoint + + +def create_transformers_vocoder_config(original_config): + """ + Creates a config for transformers SpeechT5HifiGan based on the config of the vocoder model. + """ + vocoder_params = original_config["model"]["params"]["vocoder_config"]["params"] + + config = { + "model_in_dim": vocoder_params["num_mels"], + "sampling_rate": vocoder_params["sampling_rate"], + "upsample_initial_channel": vocoder_params["upsample_initial_channel"], + "upsample_rates": list(vocoder_params["upsample_rates"]), + "upsample_kernel_sizes": list(vocoder_params["upsample_kernel_sizes"]), + "resblock_kernel_sizes": list(vocoder_params["resblock_kernel_sizes"]), + "resblock_dilation_sizes": [ + list(resblock_dilation) for resblock_dilation in vocoder_params["resblock_dilation_sizes"] + ], + "normalize_before": False, + } + + return config + + +def convert_hifigan_checkpoint(checkpoint, config): + """ + Takes a state dict and config, and returns a converted HiFiGAN vocoder checkpoint. + """ + # extract state dict for vocoder + vocoder_state_dict = {} + vocoder_key = "first_stage_model.vocoder." + keys = list(checkpoint.keys()) + for key in keys: + if key.startswith(vocoder_key): + vocoder_state_dict[key.replace(vocoder_key, "")] = checkpoint.get(key) + + # fix upsampler keys, everything else is correct already + for i in range(len(config.upsample_rates)): + vocoder_state_dict[f"upsampler.{i}.weight"] = vocoder_state_dict.pop(f"ups.{i}.weight") + vocoder_state_dict[f"upsampler.{i}.bias"] = vocoder_state_dict.pop(f"ups.{i}.bias") + + if not config.normalize_before: + # if we don't set normalize_before then these variables are unused, so we set them to their initialised values + vocoder_state_dict["mean"] = torch.zeros(config.model_in_dim) + vocoder_state_dict["scale"] = torch.ones(config.model_in_dim) + + return vocoder_state_dict + + +# Adapted from https://huggingface.co/spaces/haoheliu/MusicLDM-text-to-audio-generation/blob/84a0384742a22bd80c44e903e241f0623e874f1d/MusicLDM/utils.py#L72-L73 +DEFAULT_CONFIG = { + "model": { + "params": { + "linear_start": 0.0015, + "linear_end": 0.0195, + "timesteps": 1000, + "channels": 8, + "scale_by_std": True, + "unet_config": { + "target": "MusicLDM.latent_diffusion.openaimodel.UNetModel", + "params": { + "extra_film_condition_dim": 512, + "extra_film_use_concat": True, + "in_channels": 8, + "out_channels": 8, + "model_channels": 128, + "attention_resolutions": [8, 4, 2], + "num_res_blocks": 2, + "channel_mult": [1, 2, 3, 5], + "num_head_channels": 32, + }, + }, + "first_stage_config": { + "target": "MusicLDM.variational_autoencoder.autoencoder.AutoencoderKL", + "params": { + "embed_dim": 8, + "ddconfig": { + "z_channels": 8, + "resolution": 256, + "in_channels": 1, + "out_ch": 1, + "ch": 128, + "ch_mult": [1, 2, 4], + "num_res_blocks": 2, + }, + }, + }, + "vocoder_config": { + "target": "MusicLDM.first_stage_model.vocoder", + "params": { + "upsample_rates": [5, 4, 2, 2, 2], + "upsample_kernel_sizes": [16, 16, 8, 4, 4], + "upsample_initial_channel": 1024, + "resblock_kernel_sizes": [3, 7, 11], + "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]], + "num_mels": 64, + "sampling_rate": 16000, + }, + }, + }, + }, +} + + +def load_pipeline_from_original_MusicLDM_ckpt( + checkpoint_path: str, + original_config_file: str = None, + image_size: int = 1024, + prediction_type: str = None, + extract_ema: bool = False, + scheduler_type: str = "ddim", + num_in_channels: int = None, + model_channels: int = None, + num_head_channels: int = None, + device: str = None, + from_safetensors: bool = False, +) -> MusicLDMPipeline: + """ + Load an MusicLDM pipeline object from a `.ckpt`/`.safetensors` file and (ideally) a `.yaml` config file. + + Although many of the arguments can be automatically inferred, some of these rely on brittle checks against the + global step count, which will likely fail for models that have undergone further fine-tuning. Therefore, it is + recommended that you override the default values and/or supply an `original_config_file` wherever possible. + + Args: + checkpoint_path (`str`): Path to `.ckpt` file. + original_config_file (`str`): + Path to `.yaml` config file corresponding to the original architecture. If `None`, will be automatically + set to the MusicLDM-s-full-v2 config. + image_size (`int`, *optional*, defaults to 1024): + The image size that the model was trained on. + prediction_type (`str`, *optional*): + The prediction type that the model was trained on. If `None`, will be automatically + inferred by looking for a key in the config. For the default config, the prediction type is `'epsilon'`. + num_in_channels (`int`, *optional*, defaults to None): + The number of UNet input channels. If `None`, it will be automatically inferred from the config. + model_channels (`int`, *optional*, defaults to None): + The number of UNet model channels. If `None`, it will be automatically inferred from the config. Override + to 128 for the small checkpoints, 192 for the medium checkpoints and 256 for the large. + num_head_channels (`int`, *optional*, defaults to None): + The number of UNet head channels. If `None`, it will be automatically inferred from the config. Override + to 32 for the small and medium checkpoints, and 64 for the large. + scheduler_type (`str`, *optional*, defaults to 'pndm'): + Type of scheduler to use. Should be one of `["pndm", "lms", "heun", "euler", "euler-ancestral", "dpm", + "ddim"]`. + extract_ema (`bool`, *optional*, defaults to `False`): Only relevant for + checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights or not. Defaults to + `False`. Pass `True` to extract the EMA weights. EMA weights usually yield higher quality images for + inference. Non-EMA weights are usually better to continue fine-tuning. + device (`str`, *optional*, defaults to `None`): + The device to use. Pass `None` to determine automatically. + from_safetensors (`str`, *optional*, defaults to `False`): + If `checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch. + return: An MusicLDMPipeline object representing the passed-in `.ckpt`/`.safetensors` file. + """ + if from_safetensors: + from safetensors import safe_open + + checkpoint = {} + with safe_open(checkpoint_path, framework="pt", device="cpu") as f: + for key in f.keys(): + checkpoint[key] = f.get_tensor(key) + else: + if device is None: + device = "cuda" if torch.cuda.is_available() else "cpu" + checkpoint = torch.load(checkpoint_path, map_location=device) + else: + checkpoint = torch.load(checkpoint_path, map_location=device) + + if "state_dict" in checkpoint: + checkpoint = checkpoint["state_dict"] + + if original_config_file is None: + original_config = DEFAULT_CONFIG + else: + original_config = yaml.safe_load(original_config_file) + + if num_in_channels is not None: + original_config["model"]["params"]["unet_config"]["params"]["in_channels"] = num_in_channels + + if model_channels is not None: + original_config["model"]["params"]["unet_config"]["params"]["model_channels"] = model_channels + + if num_head_channels is not None: + original_config["model"]["params"]["unet_config"]["params"]["num_head_channels"] = num_head_channels + + if ( + "parameterization" in original_config["model"]["params"] + and original_config["model"]["params"]["parameterization"] == "v" + ): + if prediction_type is None: + prediction_type = "v_prediction" + else: + if prediction_type is None: + prediction_type = "epsilon" + + if image_size is None: + image_size = 512 + + num_train_timesteps = original_config["model"]["params"]["timesteps"] + beta_start = original_config["model"]["params"]["linear_start"] + beta_end = original_config["model"]["params"]["linear_end"] + + scheduler = DDIMScheduler( + beta_end=beta_end, + beta_schedule="scaled_linear", + beta_start=beta_start, + num_train_timesteps=num_train_timesteps, + steps_offset=1, + clip_sample=False, + set_alpha_to_one=False, + prediction_type=prediction_type, + ) + # make sure scheduler works correctly with DDIM + scheduler.register_to_config(clip_sample=False) + + if scheduler_type == "pndm": + config = dict(scheduler.config) + config["skip_prk_steps"] = True + scheduler = PNDMScheduler.from_config(config) + elif scheduler_type == "lms": + scheduler = LMSDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "heun": + scheduler = HeunDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "euler": + scheduler = EulerDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "euler-ancestral": + scheduler = EulerAncestralDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "dpm": + scheduler = DPMSolverMultistepScheduler.from_config(scheduler.config) + elif scheduler_type == "ddim": + scheduler = scheduler + else: + raise ValueError(f"Scheduler of type {scheduler_type} doesn't exist!") + + # Convert the UNet2DModel + unet_config = create_unet_diffusers_config(original_config, image_size=image_size) + unet = UNet2DConditionModel(**unet_config) + + converted_unet_checkpoint = convert_ldm_unet_checkpoint( + checkpoint, unet_config, path=checkpoint_path, extract_ema=extract_ema + ) + + unet.load_state_dict(converted_unet_checkpoint) + + # Convert the VAE model + vae_config = create_vae_diffusers_config(original_config, checkpoint=checkpoint, image_size=image_size) + converted_vae_checkpoint = convert_ldm_vae_checkpoint(checkpoint, vae_config) + + vae = AutoencoderKL(**vae_config) + vae.load_state_dict(converted_vae_checkpoint) + + # Convert the text model + # MusicLDM uses the same tokenizer as the original CLAP model, but a slightly different configuration + config = ClapConfig.from_pretrained("laion/clap-htsat-unfused") + config.audio_config.update( + { + "patch_embeds_hidden_size": 128, + "hidden_size": 1024, + "depths": [2, 2, 12, 2], + } + ) + tokenizer = AutoTokenizer.from_pretrained("laion/clap-htsat-unfused") + feature_extractor = AutoFeatureExtractor.from_pretrained("laion/clap-htsat-unfused") + + converted_text_model = convert_open_clap_checkpoint(checkpoint) + text_model = ClapModel(config) + + missing_keys, unexpected_keys = text_model.load_state_dict(converted_text_model, strict=False) + # we expect not to have token_type_ids in our original state dict so let's ignore them + missing_keys = list(set(missing_keys) - set(CLAP_EXPECTED_MISSING_KEYS)) + + if len(unexpected_keys) > 0: + raise ValueError(f"Unexpected keys when loading CLAP model: {unexpected_keys}") + + if len(missing_keys) > 0: + raise ValueError(f"Missing keys when loading CLAP model: {missing_keys}") + + # Convert the vocoder model + vocoder_config = create_transformers_vocoder_config(original_config) + vocoder_config = SpeechT5HifiGanConfig(**vocoder_config) + converted_vocoder_checkpoint = convert_hifigan_checkpoint(checkpoint, vocoder_config) + + vocoder = SpeechT5HifiGan(vocoder_config) + vocoder.load_state_dict(converted_vocoder_checkpoint) + + # Instantiate the diffusers pipeline + pipe = MusicLDMPipeline( + vae=vae, + text_encoder=text_model, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + vocoder=vocoder, + feature_extractor=feature_extractor, + ) + + return pipe + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + parser.add_argument( + "--original_config_file", + default=None, + type=str, + help="The YAML config file corresponding to the original architecture.", + ) + parser.add_argument( + "--num_in_channels", + default=None, + type=int, + help="The number of input channels. If `None` number of input channels will be automatically inferred.", + ) + parser.add_argument( + "--model_channels", + default=None, + type=int, + help="The number of UNet model channels. If `None`, it will be automatically inferred from the config. Override" + " to 128 for the small checkpoints, 192 for the medium checkpoints and 256 for the large.", + ) + parser.add_argument( + "--num_head_channels", + default=None, + type=int, + help="The number of UNet head channels. If `None`, it will be automatically inferred from the config. Override" + " to 32 for the small and medium checkpoints, and 64 for the large.", + ) + parser.add_argument( + "--scheduler_type", + default="ddim", + type=str, + help="Type of scheduler to use. Should be one of ['pndm', 'lms', 'ddim', 'euler', 'euler-ancestral', 'dpm']", + ) + parser.add_argument( + "--image_size", + default=None, + type=int, + help=("The image size that the model was trained on."), + ) + parser.add_argument( + "--prediction_type", + default=None, + type=str, + help=("The prediction type that the model was trained on."), + ) + parser.add_argument( + "--extract_ema", + action="store_true", + help=( + "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights" + " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield" + " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning." + ), + ) + parser.add_argument( + "--from_safetensors", + action="store_true", + help="If `--checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch.", + ) + parser.add_argument( + "--to_safetensors", + action="store_true", + help="Whether to store pipeline in safetensors format or not.", + ) + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)") + args = parser.parse_args() + + pipe = load_pipeline_from_original_MusicLDM_ckpt( + checkpoint_path=args.checkpoint_path, + original_config_file=args.original_config_file, + image_size=args.image_size, + prediction_type=args.prediction_type, + extract_ema=args.extract_ema, + scheduler_type=args.scheduler_type, + num_in_channels=args.num_in_channels, + model_channels=args.model_channels, + num_head_channels=args.num_head_channels, + from_safetensors=args.from_safetensors, + device=args.device, + ) + pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors) diff --git a/diffusers-0.27.0/scripts/convert_original_stable_diffusion_to_diffusers.py b/diffusers-0.27.0/scripts/convert_original_stable_diffusion_to_diffusers.py new file mode 100755 index 0000000..914f7d1 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_original_stable_diffusion_to_diffusers.py @@ -0,0 +1,188 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the LDM checkpoints. """ + +import argparse +import importlib + +import torch + +from diffusers.pipelines.stable_diffusion.convert_from_ckpt import download_from_original_stable_diffusion_ckpt + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + # !wget https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml + parser.add_argument( + "--original_config_file", + default=None, + type=str, + help="The YAML config file corresponding to the original architecture.", + ) + parser.add_argument( + "--config_files", + default=None, + type=str, + help="The YAML config file corresponding to the architecture.", + ) + parser.add_argument( + "--num_in_channels", + default=None, + type=int, + help="The number of input channels. If `None` number of input channels will be automatically inferred.", + ) + parser.add_argument( + "--scheduler_type", + default="pndm", + type=str, + help="Type of scheduler to use. Should be one of ['pndm', 'lms', 'ddim', 'euler', 'euler-ancestral', 'dpm']", + ) + parser.add_argument( + "--pipeline_type", + default=None, + type=str, + help=( + "The pipeline type. One of 'FrozenOpenCLIPEmbedder', 'FrozenCLIPEmbedder', 'PaintByExample'" + ". If `None` pipeline will be automatically inferred." + ), + ) + parser.add_argument( + "--image_size", + default=None, + type=int, + help=( + "The image size that the model was trained on. Use 512 for Stable Diffusion v1.X and Stable Siffusion v2" + " Base. Use 768 for Stable Diffusion v2." + ), + ) + parser.add_argument( + "--prediction_type", + default=None, + type=str, + help=( + "The prediction type that the model was trained on. Use 'epsilon' for Stable Diffusion v1.X and Stable" + " Diffusion v2 Base. Use 'v_prediction' for Stable Diffusion v2." + ), + ) + parser.add_argument( + "--extract_ema", + action="store_true", + help=( + "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights" + " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield" + " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning." + ), + ) + parser.add_argument( + "--upcast_attention", + action="store_true", + help=( + "Whether the attention computation should always be upcasted. This is necessary when running stable" + " diffusion 2.1." + ), + ) + parser.add_argument( + "--from_safetensors", + action="store_true", + help="If `--checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch.", + ) + parser.add_argument( + "--to_safetensors", + action="store_true", + help="Whether to store pipeline in safetensors format or not.", + ) + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)") + parser.add_argument( + "--stable_unclip", + type=str, + default=None, + required=False, + help="Set if this is a stable unCLIP model. One of 'txt2img' or 'img2img'.", + ) + parser.add_argument( + "--stable_unclip_prior", + type=str, + default=None, + required=False, + help="Set if this is a stable unCLIP txt2img model. Selects which prior to use. If `--stable_unclip` is set to `txt2img`, the karlo prior (https://huggingface.co/kakaobrain/karlo-v1-alpha/tree/main/prior) is selected by default.", + ) + parser.add_argument( + "--clip_stats_path", + type=str, + help="Path to the clip stats file. Only required if the stable unclip model's config specifies `model.params.noise_aug_config.params.clip_stats_path`.", + required=False, + ) + parser.add_argument( + "--controlnet", action="store_true", default=None, help="Set flag if this is a controlnet checkpoint." + ) + parser.add_argument("--half", action="store_true", help="Save weights in half precision.") + parser.add_argument( + "--vae_path", + type=str, + default=None, + required=False, + help="Set to a path, hub id to an already converted vae to not convert it again.", + ) + parser.add_argument( + "--pipeline_class_name", + type=str, + default=None, + required=False, + help="Specify the pipeline class name", + ) + + args = parser.parse_args() + + if args.pipeline_class_name is not None: + library = importlib.import_module("diffusers") + class_obj = getattr(library, args.pipeline_class_name) + pipeline_class = class_obj + else: + pipeline_class = None + + pipe = download_from_original_stable_diffusion_ckpt( + checkpoint_path_or_dict=args.checkpoint_path, + original_config_file=args.original_config_file, + config_files=args.config_files, + image_size=args.image_size, + prediction_type=args.prediction_type, + model_type=args.pipeline_type, + extract_ema=args.extract_ema, + scheduler_type=args.scheduler_type, + num_in_channels=args.num_in_channels, + upcast_attention=args.upcast_attention, + from_safetensors=args.from_safetensors, + device=args.device, + stable_unclip=args.stable_unclip, + stable_unclip_prior=args.stable_unclip_prior, + clip_stats_path=args.clip_stats_path, + controlnet=args.controlnet, + vae_path=args.vae_path, + pipeline_class=pipeline_class, + ) + + if args.half: + pipe.to(dtype=torch.float16) + + if args.controlnet: + # only save the controlnet model + pipe.controlnet.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors) + else: + pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors) diff --git a/diffusers-0.27.0/scripts/convert_original_t2i_adapter.py b/diffusers-0.27.0/scripts/convert_original_t2i_adapter.py new file mode 100755 index 0000000..95c8817 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_original_t2i_adapter.py @@ -0,0 +1,250 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Conversion script for the T2I-Adapter checkpoints. +""" + +import argparse + +import torch + +from diffusers import T2IAdapter + + +def convert_adapter(src_state, in_channels): + original_body_length = max([int(x.split(".")[1]) for x in src_state.keys() if "body." in x]) + 1 + + assert original_body_length == 8 + + # (0, 1) -> channels 1 + assert src_state["body.0.block1.weight"].shape == (320, 320, 3, 3) + + # (2, 3) -> channels 2 + assert src_state["body.2.in_conv.weight"].shape == (640, 320, 1, 1) + + # (4, 5) -> channels 3 + assert src_state["body.4.in_conv.weight"].shape == (1280, 640, 1, 1) + + # (6, 7) -> channels 4 + assert src_state["body.6.block1.weight"].shape == (1280, 1280, 3, 3) + + res_state = { + "adapter.conv_in.weight": src_state.pop("conv_in.weight"), + "adapter.conv_in.bias": src_state.pop("conv_in.bias"), + # 0.resnets.0 + "adapter.body.0.resnets.0.block1.weight": src_state.pop("body.0.block1.weight"), + "adapter.body.0.resnets.0.block1.bias": src_state.pop("body.0.block1.bias"), + "adapter.body.0.resnets.0.block2.weight": src_state.pop("body.0.block2.weight"), + "adapter.body.0.resnets.0.block2.bias": src_state.pop("body.0.block2.bias"), + # 0.resnets.1 + "adapter.body.0.resnets.1.block1.weight": src_state.pop("body.1.block1.weight"), + "adapter.body.0.resnets.1.block1.bias": src_state.pop("body.1.block1.bias"), + "adapter.body.0.resnets.1.block2.weight": src_state.pop("body.1.block2.weight"), + "adapter.body.0.resnets.1.block2.bias": src_state.pop("body.1.block2.bias"), + # 1 + "adapter.body.1.in_conv.weight": src_state.pop("body.2.in_conv.weight"), + "adapter.body.1.in_conv.bias": src_state.pop("body.2.in_conv.bias"), + # 1.resnets.0 + "adapter.body.1.resnets.0.block1.weight": src_state.pop("body.2.block1.weight"), + "adapter.body.1.resnets.0.block1.bias": src_state.pop("body.2.block1.bias"), + "adapter.body.1.resnets.0.block2.weight": src_state.pop("body.2.block2.weight"), + "adapter.body.1.resnets.0.block2.bias": src_state.pop("body.2.block2.bias"), + # 1.resnets.1 + "adapter.body.1.resnets.1.block1.weight": src_state.pop("body.3.block1.weight"), + "adapter.body.1.resnets.1.block1.bias": src_state.pop("body.3.block1.bias"), + "adapter.body.1.resnets.1.block2.weight": src_state.pop("body.3.block2.weight"), + "adapter.body.1.resnets.1.block2.bias": src_state.pop("body.3.block2.bias"), + # 2 + "adapter.body.2.in_conv.weight": src_state.pop("body.4.in_conv.weight"), + "adapter.body.2.in_conv.bias": src_state.pop("body.4.in_conv.bias"), + # 2.resnets.0 + "adapter.body.2.resnets.0.block1.weight": src_state.pop("body.4.block1.weight"), + "adapter.body.2.resnets.0.block1.bias": src_state.pop("body.4.block1.bias"), + "adapter.body.2.resnets.0.block2.weight": src_state.pop("body.4.block2.weight"), + "adapter.body.2.resnets.0.block2.bias": src_state.pop("body.4.block2.bias"), + # 2.resnets.1 + "adapter.body.2.resnets.1.block1.weight": src_state.pop("body.5.block1.weight"), + "adapter.body.2.resnets.1.block1.bias": src_state.pop("body.5.block1.bias"), + "adapter.body.2.resnets.1.block2.weight": src_state.pop("body.5.block2.weight"), + "adapter.body.2.resnets.1.block2.bias": src_state.pop("body.5.block2.bias"), + # 3.resnets.0 + "adapter.body.3.resnets.0.block1.weight": src_state.pop("body.6.block1.weight"), + "adapter.body.3.resnets.0.block1.bias": src_state.pop("body.6.block1.bias"), + "adapter.body.3.resnets.0.block2.weight": src_state.pop("body.6.block2.weight"), + "adapter.body.3.resnets.0.block2.bias": src_state.pop("body.6.block2.bias"), + # 3.resnets.1 + "adapter.body.3.resnets.1.block1.weight": src_state.pop("body.7.block1.weight"), + "adapter.body.3.resnets.1.block1.bias": src_state.pop("body.7.block1.bias"), + "adapter.body.3.resnets.1.block2.weight": src_state.pop("body.7.block2.weight"), + "adapter.body.3.resnets.1.block2.bias": src_state.pop("body.7.block2.bias"), + } + + assert len(src_state) == 0 + + adapter = T2IAdapter(in_channels=in_channels, adapter_type="full_adapter") + + adapter.load_state_dict(res_state) + + return adapter + + +def convert_light_adapter(src_state): + original_body_length = max([int(x.split(".")[1]) for x in src_state.keys() if "body." in x]) + 1 + + assert original_body_length == 4 + + res_state = { + # body.0.in_conv + "adapter.body.0.in_conv.weight": src_state.pop("body.0.in_conv.weight"), + "adapter.body.0.in_conv.bias": src_state.pop("body.0.in_conv.bias"), + # body.0.resnets.0 + "adapter.body.0.resnets.0.block1.weight": src_state.pop("body.0.body.0.block1.weight"), + "adapter.body.0.resnets.0.block1.bias": src_state.pop("body.0.body.0.block1.bias"), + "adapter.body.0.resnets.0.block2.weight": src_state.pop("body.0.body.0.block2.weight"), + "adapter.body.0.resnets.0.block2.bias": src_state.pop("body.0.body.0.block2.bias"), + # body.0.resnets.1 + "adapter.body.0.resnets.1.block1.weight": src_state.pop("body.0.body.1.block1.weight"), + "adapter.body.0.resnets.1.block1.bias": src_state.pop("body.0.body.1.block1.bias"), + "adapter.body.0.resnets.1.block2.weight": src_state.pop("body.0.body.1.block2.weight"), + "adapter.body.0.resnets.1.block2.bias": src_state.pop("body.0.body.1.block2.bias"), + # body.0.resnets.2 + "adapter.body.0.resnets.2.block1.weight": src_state.pop("body.0.body.2.block1.weight"), + "adapter.body.0.resnets.2.block1.bias": src_state.pop("body.0.body.2.block1.bias"), + "adapter.body.0.resnets.2.block2.weight": src_state.pop("body.0.body.2.block2.weight"), + "adapter.body.0.resnets.2.block2.bias": src_state.pop("body.0.body.2.block2.bias"), + # body.0.resnets.3 + "adapter.body.0.resnets.3.block1.weight": src_state.pop("body.0.body.3.block1.weight"), + "adapter.body.0.resnets.3.block1.bias": src_state.pop("body.0.body.3.block1.bias"), + "adapter.body.0.resnets.3.block2.weight": src_state.pop("body.0.body.3.block2.weight"), + "adapter.body.0.resnets.3.block2.bias": src_state.pop("body.0.body.3.block2.bias"), + # body.0.out_conv + "adapter.body.0.out_conv.weight": src_state.pop("body.0.out_conv.weight"), + "adapter.body.0.out_conv.bias": src_state.pop("body.0.out_conv.bias"), + # body.1.in_conv + "adapter.body.1.in_conv.weight": src_state.pop("body.1.in_conv.weight"), + "adapter.body.1.in_conv.bias": src_state.pop("body.1.in_conv.bias"), + # body.1.resnets.0 + "adapter.body.1.resnets.0.block1.weight": src_state.pop("body.1.body.0.block1.weight"), + "adapter.body.1.resnets.0.block1.bias": src_state.pop("body.1.body.0.block1.bias"), + "adapter.body.1.resnets.0.block2.weight": src_state.pop("body.1.body.0.block2.weight"), + "adapter.body.1.resnets.0.block2.bias": src_state.pop("body.1.body.0.block2.bias"), + # body.1.resnets.1 + "adapter.body.1.resnets.1.block1.weight": src_state.pop("body.1.body.1.block1.weight"), + "adapter.body.1.resnets.1.block1.bias": src_state.pop("body.1.body.1.block1.bias"), + "adapter.body.1.resnets.1.block2.weight": src_state.pop("body.1.body.1.block2.weight"), + "adapter.body.1.resnets.1.block2.bias": src_state.pop("body.1.body.1.block2.bias"), + # body.1.body.2 + "adapter.body.1.resnets.2.block1.weight": src_state.pop("body.1.body.2.block1.weight"), + "adapter.body.1.resnets.2.block1.bias": src_state.pop("body.1.body.2.block1.bias"), + "adapter.body.1.resnets.2.block2.weight": src_state.pop("body.1.body.2.block2.weight"), + "adapter.body.1.resnets.2.block2.bias": src_state.pop("body.1.body.2.block2.bias"), + # body.1.body.3 + "adapter.body.1.resnets.3.block1.weight": src_state.pop("body.1.body.3.block1.weight"), + "adapter.body.1.resnets.3.block1.bias": src_state.pop("body.1.body.3.block1.bias"), + "adapter.body.1.resnets.3.block2.weight": src_state.pop("body.1.body.3.block2.weight"), + "adapter.body.1.resnets.3.block2.bias": src_state.pop("body.1.body.3.block2.bias"), + # body.1.out_conv + "adapter.body.1.out_conv.weight": src_state.pop("body.1.out_conv.weight"), + "adapter.body.1.out_conv.bias": src_state.pop("body.1.out_conv.bias"), + # body.2.in_conv + "adapter.body.2.in_conv.weight": src_state.pop("body.2.in_conv.weight"), + "adapter.body.2.in_conv.bias": src_state.pop("body.2.in_conv.bias"), + # body.2.body.0 + "adapter.body.2.resnets.0.block1.weight": src_state.pop("body.2.body.0.block1.weight"), + "adapter.body.2.resnets.0.block1.bias": src_state.pop("body.2.body.0.block1.bias"), + "adapter.body.2.resnets.0.block2.weight": src_state.pop("body.2.body.0.block2.weight"), + "adapter.body.2.resnets.0.block2.bias": src_state.pop("body.2.body.0.block2.bias"), + # body.2.body.1 + "adapter.body.2.resnets.1.block1.weight": src_state.pop("body.2.body.1.block1.weight"), + "adapter.body.2.resnets.1.block1.bias": src_state.pop("body.2.body.1.block1.bias"), + "adapter.body.2.resnets.1.block2.weight": src_state.pop("body.2.body.1.block2.weight"), + "adapter.body.2.resnets.1.block2.bias": src_state.pop("body.2.body.1.block2.bias"), + # body.2.body.2 + "adapter.body.2.resnets.2.block1.weight": src_state.pop("body.2.body.2.block1.weight"), + "adapter.body.2.resnets.2.block1.bias": src_state.pop("body.2.body.2.block1.bias"), + "adapter.body.2.resnets.2.block2.weight": src_state.pop("body.2.body.2.block2.weight"), + "adapter.body.2.resnets.2.block2.bias": src_state.pop("body.2.body.2.block2.bias"), + # body.2.body.3 + "adapter.body.2.resnets.3.block1.weight": src_state.pop("body.2.body.3.block1.weight"), + "adapter.body.2.resnets.3.block1.bias": src_state.pop("body.2.body.3.block1.bias"), + "adapter.body.2.resnets.3.block2.weight": src_state.pop("body.2.body.3.block2.weight"), + "adapter.body.2.resnets.3.block2.bias": src_state.pop("body.2.body.3.block2.bias"), + # body.2.out_conv + "adapter.body.2.out_conv.weight": src_state.pop("body.2.out_conv.weight"), + "adapter.body.2.out_conv.bias": src_state.pop("body.2.out_conv.bias"), + # body.3.in_conv + "adapter.body.3.in_conv.weight": src_state.pop("body.3.in_conv.weight"), + "adapter.body.3.in_conv.bias": src_state.pop("body.3.in_conv.bias"), + # body.3.body.0 + "adapter.body.3.resnets.0.block1.weight": src_state.pop("body.3.body.0.block1.weight"), + "adapter.body.3.resnets.0.block1.bias": src_state.pop("body.3.body.0.block1.bias"), + "adapter.body.3.resnets.0.block2.weight": src_state.pop("body.3.body.0.block2.weight"), + "adapter.body.3.resnets.0.block2.bias": src_state.pop("body.3.body.0.block2.bias"), + # body.3.body.1 + "adapter.body.3.resnets.1.block1.weight": src_state.pop("body.3.body.1.block1.weight"), + "adapter.body.3.resnets.1.block1.bias": src_state.pop("body.3.body.1.block1.bias"), + "adapter.body.3.resnets.1.block2.weight": src_state.pop("body.3.body.1.block2.weight"), + "adapter.body.3.resnets.1.block2.bias": src_state.pop("body.3.body.1.block2.bias"), + # body.3.body.2 + "adapter.body.3.resnets.2.block1.weight": src_state.pop("body.3.body.2.block1.weight"), + "adapter.body.3.resnets.2.block1.bias": src_state.pop("body.3.body.2.block1.bias"), + "adapter.body.3.resnets.2.block2.weight": src_state.pop("body.3.body.2.block2.weight"), + "adapter.body.3.resnets.2.block2.bias": src_state.pop("body.3.body.2.block2.bias"), + # body.3.body.3 + "adapter.body.3.resnets.3.block1.weight": src_state.pop("body.3.body.3.block1.weight"), + "adapter.body.3.resnets.3.block1.bias": src_state.pop("body.3.body.3.block1.bias"), + "adapter.body.3.resnets.3.block2.weight": src_state.pop("body.3.body.3.block2.weight"), + "adapter.body.3.resnets.3.block2.bias": src_state.pop("body.3.body.3.block2.bias"), + # body.3.out_conv + "adapter.body.3.out_conv.weight": src_state.pop("body.3.out_conv.weight"), + "adapter.body.3.out_conv.bias": src_state.pop("body.3.out_conv.bias"), + } + + assert len(src_state) == 0 + + adapter = T2IAdapter(in_channels=3, channels=[320, 640, 1280], num_res_blocks=4, adapter_type="light_adapter") + + adapter.load_state_dict(res_state) + + return adapter + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + parser.add_argument( + "--output_path", default=None, type=str, required=True, help="Path to the store the result checkpoint." + ) + parser.add_argument( + "--is_adapter_light", + action="store_true", + help="Is checkpoint come from Adapter-Light architecture. ex: color-adapter", + ) + parser.add_argument("--in_channels", required=False, type=int, help="Input channels for non-light adapter") + + args = parser.parse_args() + src_state = torch.load(args.checkpoint_path) + + if args.is_adapter_light: + adapter = convert_light_adapter(src_state) + else: + if args.in_channels is None: + raise ValueError("set `--in_channels=`") + adapter = convert_adapter(src_state, args.in_channels) + + adapter.save_pretrained(args.output_path) diff --git a/diffusers-0.27.0/scripts/convert_pixart_alpha_to_diffusers.py b/diffusers-0.27.0/scripts/convert_pixart_alpha_to_diffusers.py new file mode 100755 index 0000000..228b479 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_pixart_alpha_to_diffusers.py @@ -0,0 +1,198 @@ +import argparse +import os + +import torch +from transformers import T5EncoderModel, T5Tokenizer + +from diffusers import AutoencoderKL, DPMSolverMultistepScheduler, PixArtAlphaPipeline, Transformer2DModel + + +ckpt_id = "PixArt-alpha/PixArt-alpha" +# https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/scripts/inference.py#L125 +interpolation_scale = {256: 0.5, 512: 1, 1024: 2} + + +def main(args): + all_state_dict = torch.load(args.orig_ckpt_path, map_location="cpu") + state_dict = all_state_dict.pop("state_dict") + converted_state_dict = {} + + # Patch embeddings. + converted_state_dict["pos_embed.proj.weight"] = state_dict.pop("x_embedder.proj.weight") + converted_state_dict["pos_embed.proj.bias"] = state_dict.pop("x_embedder.proj.bias") + + # Caption projection. + converted_state_dict["caption_projection.linear_1.weight"] = state_dict.pop("y_embedder.y_proj.fc1.weight") + converted_state_dict["caption_projection.linear_1.bias"] = state_dict.pop("y_embedder.y_proj.fc1.bias") + converted_state_dict["caption_projection.linear_2.weight"] = state_dict.pop("y_embedder.y_proj.fc2.weight") + converted_state_dict["caption_projection.linear_2.bias"] = state_dict.pop("y_embedder.y_proj.fc2.bias") + + # AdaLN-single LN + converted_state_dict["adaln_single.emb.timestep_embedder.linear_1.weight"] = state_dict.pop( + "t_embedder.mlp.0.weight" + ) + converted_state_dict["adaln_single.emb.timestep_embedder.linear_1.bias"] = state_dict.pop("t_embedder.mlp.0.bias") + converted_state_dict["adaln_single.emb.timestep_embedder.linear_2.weight"] = state_dict.pop( + "t_embedder.mlp.2.weight" + ) + converted_state_dict["adaln_single.emb.timestep_embedder.linear_2.bias"] = state_dict.pop("t_embedder.mlp.2.bias") + + if args.image_size == 1024: + # Resolution. + converted_state_dict["adaln_single.emb.resolution_embedder.linear_1.weight"] = state_dict.pop( + "csize_embedder.mlp.0.weight" + ) + converted_state_dict["adaln_single.emb.resolution_embedder.linear_1.bias"] = state_dict.pop( + "csize_embedder.mlp.0.bias" + ) + converted_state_dict["adaln_single.emb.resolution_embedder.linear_2.weight"] = state_dict.pop( + "csize_embedder.mlp.2.weight" + ) + converted_state_dict["adaln_single.emb.resolution_embedder.linear_2.bias"] = state_dict.pop( + "csize_embedder.mlp.2.bias" + ) + # Aspect ratio. + converted_state_dict["adaln_single.emb.aspect_ratio_embedder.linear_1.weight"] = state_dict.pop( + "ar_embedder.mlp.0.weight" + ) + converted_state_dict["adaln_single.emb.aspect_ratio_embedder.linear_1.bias"] = state_dict.pop( + "ar_embedder.mlp.0.bias" + ) + converted_state_dict["adaln_single.emb.aspect_ratio_embedder.linear_2.weight"] = state_dict.pop( + "ar_embedder.mlp.2.weight" + ) + converted_state_dict["adaln_single.emb.aspect_ratio_embedder.linear_2.bias"] = state_dict.pop( + "ar_embedder.mlp.2.bias" + ) + # Shared norm. + converted_state_dict["adaln_single.linear.weight"] = state_dict.pop("t_block.1.weight") + converted_state_dict["adaln_single.linear.bias"] = state_dict.pop("t_block.1.bias") + + for depth in range(28): + # Transformer blocks. + converted_state_dict[f"transformer_blocks.{depth}.scale_shift_table"] = state_dict.pop( + f"blocks.{depth}.scale_shift_table" + ) + + # Attention is all you need 🤘 + + # Self attention. + q, k, v = torch.chunk(state_dict.pop(f"blocks.{depth}.attn.qkv.weight"), 3, dim=0) + q_bias, k_bias, v_bias = torch.chunk(state_dict.pop(f"blocks.{depth}.attn.qkv.bias"), 3, dim=0) + converted_state_dict[f"transformer_blocks.{depth}.attn1.to_q.weight"] = q + converted_state_dict[f"transformer_blocks.{depth}.attn1.to_q.bias"] = q_bias + converted_state_dict[f"transformer_blocks.{depth}.attn1.to_k.weight"] = k + converted_state_dict[f"transformer_blocks.{depth}.attn1.to_k.bias"] = k_bias + converted_state_dict[f"transformer_blocks.{depth}.attn1.to_v.weight"] = v + converted_state_dict[f"transformer_blocks.{depth}.attn1.to_v.bias"] = v_bias + # Projection. + converted_state_dict[f"transformer_blocks.{depth}.attn1.to_out.0.weight"] = state_dict.pop( + f"blocks.{depth}.attn.proj.weight" + ) + converted_state_dict[f"transformer_blocks.{depth}.attn1.to_out.0.bias"] = state_dict.pop( + f"blocks.{depth}.attn.proj.bias" + ) + + # Feed-forward. + converted_state_dict[f"transformer_blocks.{depth}.ff.net.0.proj.weight"] = state_dict.pop( + f"blocks.{depth}.mlp.fc1.weight" + ) + converted_state_dict[f"transformer_blocks.{depth}.ff.net.0.proj.bias"] = state_dict.pop( + f"blocks.{depth}.mlp.fc1.bias" + ) + converted_state_dict[f"transformer_blocks.{depth}.ff.net.2.weight"] = state_dict.pop( + f"blocks.{depth}.mlp.fc2.weight" + ) + converted_state_dict[f"transformer_blocks.{depth}.ff.net.2.bias"] = state_dict.pop( + f"blocks.{depth}.mlp.fc2.bias" + ) + + # Cross-attention. + q = state_dict.pop(f"blocks.{depth}.cross_attn.q_linear.weight") + q_bias = state_dict.pop(f"blocks.{depth}.cross_attn.q_linear.bias") + k, v = torch.chunk(state_dict.pop(f"blocks.{depth}.cross_attn.kv_linear.weight"), 2, dim=0) + k_bias, v_bias = torch.chunk(state_dict.pop(f"blocks.{depth}.cross_attn.kv_linear.bias"), 2, dim=0) + + converted_state_dict[f"transformer_blocks.{depth}.attn2.to_q.weight"] = q + converted_state_dict[f"transformer_blocks.{depth}.attn2.to_q.bias"] = q_bias + converted_state_dict[f"transformer_blocks.{depth}.attn2.to_k.weight"] = k + converted_state_dict[f"transformer_blocks.{depth}.attn2.to_k.bias"] = k_bias + converted_state_dict[f"transformer_blocks.{depth}.attn2.to_v.weight"] = v + converted_state_dict[f"transformer_blocks.{depth}.attn2.to_v.bias"] = v_bias + + converted_state_dict[f"transformer_blocks.{depth}.attn2.to_out.0.weight"] = state_dict.pop( + f"blocks.{depth}.cross_attn.proj.weight" + ) + converted_state_dict[f"transformer_blocks.{depth}.attn2.to_out.0.bias"] = state_dict.pop( + f"blocks.{depth}.cross_attn.proj.bias" + ) + + # Final block. + converted_state_dict["proj_out.weight"] = state_dict.pop("final_layer.linear.weight") + converted_state_dict["proj_out.bias"] = state_dict.pop("final_layer.linear.bias") + converted_state_dict["scale_shift_table"] = state_dict.pop("final_layer.scale_shift_table") + + # DiT XL/2 + transformer = Transformer2DModel( + sample_size=args.image_size // 8, + num_layers=28, + attention_head_dim=72, + in_channels=4, + out_channels=8, + patch_size=2, + attention_bias=True, + num_attention_heads=16, + cross_attention_dim=1152, + activation_fn="gelu-approximate", + num_embeds_ada_norm=1000, + norm_type="ada_norm_single", + norm_elementwise_affine=False, + norm_eps=1e-6, + caption_channels=4096, + ) + transformer.load_state_dict(converted_state_dict, strict=True) + + assert transformer.pos_embed.pos_embed is not None + state_dict.pop("pos_embed") + state_dict.pop("y_embedder.y_embedding") + assert len(state_dict) == 0, f"State dict is not empty, {state_dict.keys()}" + + num_model_params = sum(p.numel() for p in transformer.parameters()) + print(f"Total number of transformer parameters: {num_model_params}") + + if args.only_transformer: + transformer.save_pretrained(os.path.join(args.dump_path, "transformer")) + else: + scheduler = DPMSolverMultistepScheduler() + + vae = AutoencoderKL.from_pretrained(ckpt_id, subfolder="sd-vae-ft-ema") + + tokenizer = T5Tokenizer.from_pretrained(ckpt_id, subfolder="t5-v1_1-xxl") + text_encoder = T5EncoderModel.from_pretrained(ckpt_id, subfolder="t5-v1_1-xxl") + + pipeline = PixArtAlphaPipeline( + tokenizer=tokenizer, text_encoder=text_encoder, transformer=transformer, vae=vae, scheduler=scheduler + ) + + pipeline.save_pretrained(args.dump_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--orig_ckpt_path", default=None, type=str, required=False, help="Path to the checkpoint to convert." + ) + parser.add_argument( + "--image_size", + default=1024, + type=int, + choices=[256, 512, 1024], + required=False, + help="Image size of pretrained model, either 512 or 1024.", + ) + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output pipeline.") + parser.add_argument("--only_transformer", default=True, type=bool, required=True) + + args = parser.parse_args() + main(args) diff --git a/diffusers-0.27.0/scripts/convert_shap_e_to_diffusers.py b/diffusers-0.27.0/scripts/convert_shap_e_to_diffusers.py new file mode 100755 index 0000000..b903b4e --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_shap_e_to_diffusers.py @@ -0,0 +1,1080 @@ +import argparse +import tempfile + +import torch +from accelerate import load_checkpoint_and_dispatch + +from diffusers.models.transformers.prior_transformer import PriorTransformer +from diffusers.pipelines.shap_e import ShapERenderer + + +""" +Example - From the diffusers root directory: + +Download weights: +```sh +$ wget "https://openaipublic.azureedge.net/main/shap-e/text_cond.pt" +``` + +Convert the model: +```sh +$ python scripts/convert_shap_e_to_diffusers.py \ + --prior_checkpoint_path /home/yiyi_huggingface_co/shap-e/shap_e_model_cache/text_cond.pt \ + --prior_image_checkpoint_path /home/yiyi_huggingface_co/shap-e/shap_e_model_cache/image_cond.pt \ + --transmitter_checkpoint_path /home/yiyi_huggingface_co/shap-e/shap_e_model_cache/transmitter.pt\ + --dump_path /home/yiyi_huggingface_co/model_repo/shap-e-img2img/shap_e_renderer\ + --debug renderer +``` +""" + + +# prior + +PRIOR_ORIGINAL_PREFIX = "wrapped" + +PRIOR_CONFIG = { + "num_attention_heads": 16, + "attention_head_dim": 1024 // 16, + "num_layers": 24, + "embedding_dim": 1024, + "num_embeddings": 1024, + "additional_embeddings": 0, + "time_embed_act_fn": "gelu", + "norm_in_type": "layer", + "encoder_hid_proj_type": None, + "added_emb_type": None, + "time_embed_dim": 1024 * 4, + "embedding_proj_dim": 768, + "clip_embed_dim": 1024 * 2, +} + + +def prior_model_from_original_config(): + model = PriorTransformer(**PRIOR_CONFIG) + + return model + + +def prior_original_checkpoint_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + # .time_embed.c_fc -> .time_embedding.linear_1 + diffusers_checkpoint.update( + { + "time_embedding.linear_1.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.c_fc.weight"], + "time_embedding.linear_1.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.c_fc.bias"], + } + ) + + # .time_embed.c_proj -> .time_embedding.linear_2 + diffusers_checkpoint.update( + { + "time_embedding.linear_2.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.c_proj.weight"], + "time_embedding.linear_2.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.time_embed.c_proj.bias"], + } + ) + + # .input_proj -> .proj_in + diffusers_checkpoint.update( + { + "proj_in.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.input_proj.weight"], + "proj_in.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.input_proj.bias"], + } + ) + + # .clip_emb -> .embedding_proj + diffusers_checkpoint.update( + { + "embedding_proj.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.clip_embed.weight"], + "embedding_proj.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.clip_embed.bias"], + } + ) + + # .pos_emb -> .positional_embedding + diffusers_checkpoint.update({"positional_embedding": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.pos_emb"][None, :]}) + + # .ln_pre -> .norm_in + diffusers_checkpoint.update( + { + "norm_in.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.ln_pre.weight"], + "norm_in.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.ln_pre.bias"], + } + ) + + # .backbone.resblocks. -> .transformer_blocks. + for idx in range(len(model.transformer_blocks)): + diffusers_transformer_prefix = f"transformer_blocks.{idx}" + original_transformer_prefix = f"{PRIOR_ORIGINAL_PREFIX}.backbone.resblocks.{idx}" + + # .attn -> .attn1 + diffusers_attention_prefix = f"{diffusers_transformer_prefix}.attn1" + original_attention_prefix = f"{original_transformer_prefix}.attn" + diffusers_checkpoint.update( + prior_attention_to_diffusers( + checkpoint, + diffusers_attention_prefix=diffusers_attention_prefix, + original_attention_prefix=original_attention_prefix, + attention_head_dim=model.attention_head_dim, + ) + ) + + # .mlp -> .ff + diffusers_ff_prefix = f"{diffusers_transformer_prefix}.ff" + original_ff_prefix = f"{original_transformer_prefix}.mlp" + diffusers_checkpoint.update( + prior_ff_to_diffusers( + checkpoint, diffusers_ff_prefix=diffusers_ff_prefix, original_ff_prefix=original_ff_prefix + ) + ) + + # .ln_1 -> .norm1 + diffusers_checkpoint.update( + { + f"{diffusers_transformer_prefix}.norm1.weight": checkpoint[ + f"{original_transformer_prefix}.ln_1.weight" + ], + f"{diffusers_transformer_prefix}.norm1.bias": checkpoint[f"{original_transformer_prefix}.ln_1.bias"], + } + ) + + # .ln_2 -> .norm3 + diffusers_checkpoint.update( + { + f"{diffusers_transformer_prefix}.norm3.weight": checkpoint[ + f"{original_transformer_prefix}.ln_2.weight" + ], + f"{diffusers_transformer_prefix}.norm3.bias": checkpoint[f"{original_transformer_prefix}.ln_2.bias"], + } + ) + + # .ln_post -> .norm_out + diffusers_checkpoint.update( + { + "norm_out.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.ln_post.weight"], + "norm_out.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.ln_post.bias"], + } + ) + + # .output_proj -> .proj_to_clip_embeddings + diffusers_checkpoint.update( + { + "proj_to_clip_embeddings.weight": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.output_proj.weight"], + "proj_to_clip_embeddings.bias": checkpoint[f"{PRIOR_ORIGINAL_PREFIX}.output_proj.bias"], + } + ) + + return diffusers_checkpoint + + +def prior_attention_to_diffusers( + checkpoint, *, diffusers_attention_prefix, original_attention_prefix, attention_head_dim +): + diffusers_checkpoint = {} + + # .c_qkv -> .{to_q, to_k, to_v} + [q_weight, k_weight, v_weight], [q_bias, k_bias, v_bias] = split_attentions( + weight=checkpoint[f"{original_attention_prefix}.c_qkv.weight"], + bias=checkpoint[f"{original_attention_prefix}.c_qkv.bias"], + split=3, + chunk_size=attention_head_dim, + ) + + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.to_q.weight": q_weight, + f"{diffusers_attention_prefix}.to_q.bias": q_bias, + f"{diffusers_attention_prefix}.to_k.weight": k_weight, + f"{diffusers_attention_prefix}.to_k.bias": k_bias, + f"{diffusers_attention_prefix}.to_v.weight": v_weight, + f"{diffusers_attention_prefix}.to_v.bias": v_bias, + } + ) + + # .c_proj -> .to_out.0 + diffusers_checkpoint.update( + { + f"{diffusers_attention_prefix}.to_out.0.weight": checkpoint[f"{original_attention_prefix}.c_proj.weight"], + f"{diffusers_attention_prefix}.to_out.0.bias": checkpoint[f"{original_attention_prefix}.c_proj.bias"], + } + ) + + return diffusers_checkpoint + + +def prior_ff_to_diffusers(checkpoint, *, diffusers_ff_prefix, original_ff_prefix): + diffusers_checkpoint = { + # .c_fc -> .net.0.proj + f"{diffusers_ff_prefix}.net.{0}.proj.weight": checkpoint[f"{original_ff_prefix}.c_fc.weight"], + f"{diffusers_ff_prefix}.net.{0}.proj.bias": checkpoint[f"{original_ff_prefix}.c_fc.bias"], + # .c_proj -> .net.2 + f"{diffusers_ff_prefix}.net.{2}.weight": checkpoint[f"{original_ff_prefix}.c_proj.weight"], + f"{diffusers_ff_prefix}.net.{2}.bias": checkpoint[f"{original_ff_prefix}.c_proj.bias"], + } + + return diffusers_checkpoint + + +# done prior + + +# prior_image (only slightly different from prior) + + +PRIOR_IMAGE_ORIGINAL_PREFIX = "wrapped" + +# Uses default arguments +PRIOR_IMAGE_CONFIG = { + "num_attention_heads": 8, + "attention_head_dim": 1024 // 8, + "num_layers": 24, + "embedding_dim": 1024, + "num_embeddings": 1024, + "additional_embeddings": 0, + "time_embed_act_fn": "gelu", + "norm_in_type": "layer", + "embedding_proj_norm_type": "layer", + "encoder_hid_proj_type": None, + "added_emb_type": None, + "time_embed_dim": 1024 * 4, + "embedding_proj_dim": 1024, + "clip_embed_dim": 1024 * 2, +} + + +def prior_image_model_from_original_config(): + model = PriorTransformer(**PRIOR_IMAGE_CONFIG) + + return model + + +def prior_image_original_checkpoint_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + # .time_embed.c_fc -> .time_embedding.linear_1 + diffusers_checkpoint.update( + { + "time_embedding.linear_1.weight": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.time_embed.c_fc.weight"], + "time_embedding.linear_1.bias": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.time_embed.c_fc.bias"], + } + ) + + # .time_embed.c_proj -> .time_embedding.linear_2 + diffusers_checkpoint.update( + { + "time_embedding.linear_2.weight": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.time_embed.c_proj.weight"], + "time_embedding.linear_2.bias": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.time_embed.c_proj.bias"], + } + ) + + # .input_proj -> .proj_in + diffusers_checkpoint.update( + { + "proj_in.weight": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.input_proj.weight"], + "proj_in.bias": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.input_proj.bias"], + } + ) + + # .clip_embed.0 -> .embedding_proj_norm + diffusers_checkpoint.update( + { + "embedding_proj_norm.weight": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.clip_embed.0.weight"], + "embedding_proj_norm.bias": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.clip_embed.0.bias"], + } + ) + + # ..clip_embed.1 -> .embedding_proj + diffusers_checkpoint.update( + { + "embedding_proj.weight": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.clip_embed.1.weight"], + "embedding_proj.bias": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.clip_embed.1.bias"], + } + ) + + # .pos_emb -> .positional_embedding + diffusers_checkpoint.update( + {"positional_embedding": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.pos_emb"][None, :]} + ) + + # .ln_pre -> .norm_in + diffusers_checkpoint.update( + { + "norm_in.weight": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.ln_pre.weight"], + "norm_in.bias": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.ln_pre.bias"], + } + ) + + # .backbone.resblocks. -> .transformer_blocks. + for idx in range(len(model.transformer_blocks)): + diffusers_transformer_prefix = f"transformer_blocks.{idx}" + original_transformer_prefix = f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.backbone.resblocks.{idx}" + + # .attn -> .attn1 + diffusers_attention_prefix = f"{diffusers_transformer_prefix}.attn1" + original_attention_prefix = f"{original_transformer_prefix}.attn" + diffusers_checkpoint.update( + prior_attention_to_diffusers( + checkpoint, + diffusers_attention_prefix=diffusers_attention_prefix, + original_attention_prefix=original_attention_prefix, + attention_head_dim=model.attention_head_dim, + ) + ) + + # .mlp -> .ff + diffusers_ff_prefix = f"{diffusers_transformer_prefix}.ff" + original_ff_prefix = f"{original_transformer_prefix}.mlp" + diffusers_checkpoint.update( + prior_ff_to_diffusers( + checkpoint, diffusers_ff_prefix=diffusers_ff_prefix, original_ff_prefix=original_ff_prefix + ) + ) + + # .ln_1 -> .norm1 + diffusers_checkpoint.update( + { + f"{diffusers_transformer_prefix}.norm1.weight": checkpoint[ + f"{original_transformer_prefix}.ln_1.weight" + ], + f"{diffusers_transformer_prefix}.norm1.bias": checkpoint[f"{original_transformer_prefix}.ln_1.bias"], + } + ) + + # .ln_2 -> .norm3 + diffusers_checkpoint.update( + { + f"{diffusers_transformer_prefix}.norm3.weight": checkpoint[ + f"{original_transformer_prefix}.ln_2.weight" + ], + f"{diffusers_transformer_prefix}.norm3.bias": checkpoint[f"{original_transformer_prefix}.ln_2.bias"], + } + ) + + # .ln_post -> .norm_out + diffusers_checkpoint.update( + { + "norm_out.weight": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.ln_post.weight"], + "norm_out.bias": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.ln_post.bias"], + } + ) + + # .output_proj -> .proj_to_clip_embeddings + diffusers_checkpoint.update( + { + "proj_to_clip_embeddings.weight": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.output_proj.weight"], + "proj_to_clip_embeddings.bias": checkpoint[f"{PRIOR_IMAGE_ORIGINAL_PREFIX}.output_proj.bias"], + } + ) + + return diffusers_checkpoint + + +# done prior_image + + +# renderer + +## create the lookup table for marching cubes method used in MeshDecoder + +MC_TABLE = [ + [], + [[0, 1, 0, 2, 0, 4]], + [[1, 0, 1, 5, 1, 3]], + [[0, 4, 1, 5, 0, 2], [1, 5, 1, 3, 0, 2]], + [[2, 0, 2, 3, 2, 6]], + [[0, 1, 2, 3, 0, 4], [2, 3, 2, 6, 0, 4]], + [[1, 0, 1, 5, 1, 3], [2, 6, 0, 2, 3, 2]], + [[3, 2, 2, 6, 3, 1], [3, 1, 2, 6, 1, 5], [1, 5, 2, 6, 0, 4]], + [[3, 1, 3, 7, 3, 2]], + [[0, 2, 0, 4, 0, 1], [3, 7, 2, 3, 1, 3]], + [[1, 5, 3, 7, 1, 0], [3, 7, 3, 2, 1, 0]], + [[2, 0, 0, 4, 2, 3], [2, 3, 0, 4, 3, 7], [3, 7, 0, 4, 1, 5]], + [[2, 0, 3, 1, 2, 6], [3, 1, 3, 7, 2, 6]], + [[1, 3, 3, 7, 1, 0], [1, 0, 3, 7, 0, 4], [0, 4, 3, 7, 2, 6]], + [[0, 1, 1, 5, 0, 2], [0, 2, 1, 5, 2, 6], [2, 6, 1, 5, 3, 7]], + [[0, 4, 1, 5, 3, 7], [0, 4, 3, 7, 2, 6]], + [[4, 0, 4, 6, 4, 5]], + [[0, 2, 4, 6, 0, 1], [4, 6, 4, 5, 0, 1]], + [[1, 5, 1, 3, 1, 0], [4, 6, 5, 4, 0, 4]], + [[5, 1, 1, 3, 5, 4], [5, 4, 1, 3, 4, 6], [4, 6, 1, 3, 0, 2]], + [[2, 0, 2, 3, 2, 6], [4, 5, 0, 4, 6, 4]], + [[6, 4, 4, 5, 6, 2], [6, 2, 4, 5, 2, 3], [2, 3, 4, 5, 0, 1]], + [[2, 6, 2, 0, 3, 2], [1, 0, 1, 5, 3, 1], [6, 4, 5, 4, 0, 4]], + [[1, 3, 5, 4, 1, 5], [1, 3, 4, 6, 5, 4], [1, 3, 3, 2, 4, 6], [3, 2, 2, 6, 4, 6]], + [[3, 1, 3, 7, 3, 2], [6, 4, 5, 4, 0, 4]], + [[4, 5, 0, 1, 4, 6], [0, 1, 0, 2, 4, 6], [7, 3, 2, 3, 1, 3]], + [[3, 2, 1, 0, 3, 7], [1, 0, 1, 5, 3, 7], [6, 4, 5, 4, 0, 4]], + [[3, 7, 3, 2, 1, 5], [3, 2, 6, 4, 1, 5], [1, 5, 6, 4, 5, 4], [3, 2, 2, 0, 6, 4]], + [[3, 7, 2, 6, 3, 1], [2, 6, 2, 0, 3, 1], [5, 4, 0, 4, 6, 4]], + [[1, 0, 1, 3, 5, 4], [1, 3, 2, 6, 5, 4], [1, 3, 3, 7, 2, 6], [5, 4, 2, 6, 4, 6]], + [[0, 1, 1, 5, 0, 2], [0, 2, 1, 5, 2, 6], [2, 6, 1, 5, 3, 7], [4, 5, 0, 4, 4, 6]], + [[6, 2, 4, 6, 4, 5], [4, 5, 5, 1, 6, 2], [6, 2, 5, 1, 7, 3]], + [[5, 1, 5, 4, 5, 7]], + [[0, 1, 0, 2, 0, 4], [5, 7, 1, 5, 4, 5]], + [[1, 0, 5, 4, 1, 3], [5, 4, 5, 7, 1, 3]], + [[4, 5, 5, 7, 4, 0], [4, 0, 5, 7, 0, 2], [0, 2, 5, 7, 1, 3]], + [[2, 0, 2, 3, 2, 6], [7, 5, 1, 5, 4, 5]], + [[2, 6, 0, 4, 2, 3], [0, 4, 0, 1, 2, 3], [7, 5, 1, 5, 4, 5]], + [[5, 7, 1, 3, 5, 4], [1, 3, 1, 0, 5, 4], [6, 2, 0, 2, 3, 2]], + [[3, 1, 3, 2, 7, 5], [3, 2, 0, 4, 7, 5], [3, 2, 2, 6, 0, 4], [7, 5, 0, 4, 5, 4]], + [[3, 7, 3, 2, 3, 1], [5, 4, 7, 5, 1, 5]], + [[0, 4, 0, 1, 2, 0], [3, 1, 3, 7, 2, 3], [4, 5, 7, 5, 1, 5]], + [[7, 3, 3, 2, 7, 5], [7, 5, 3, 2, 5, 4], [5, 4, 3, 2, 1, 0]], + [[0, 4, 2, 3, 0, 2], [0, 4, 3, 7, 2, 3], [0, 4, 4, 5, 3, 7], [4, 5, 5, 7, 3, 7]], + [[2, 0, 3, 1, 2, 6], [3, 1, 3, 7, 2, 6], [4, 5, 7, 5, 1, 5]], + [[1, 3, 3, 7, 1, 0], [1, 0, 3, 7, 0, 4], [0, 4, 3, 7, 2, 6], [5, 7, 1, 5, 5, 4]], + [[2, 6, 2, 0, 3, 7], [2, 0, 4, 5, 3, 7], [3, 7, 4, 5, 7, 5], [2, 0, 0, 1, 4, 5]], + [[4, 0, 5, 4, 5, 7], [5, 7, 7, 3, 4, 0], [4, 0, 7, 3, 6, 2]], + [[4, 6, 5, 7, 4, 0], [5, 7, 5, 1, 4, 0]], + [[1, 0, 0, 2, 1, 5], [1, 5, 0, 2, 5, 7], [5, 7, 0, 2, 4, 6]], + [[0, 4, 4, 6, 0, 1], [0, 1, 4, 6, 1, 3], [1, 3, 4, 6, 5, 7]], + [[0, 2, 4, 6, 5, 7], [0, 2, 5, 7, 1, 3]], + [[5, 1, 4, 0, 5, 7], [4, 0, 4, 6, 5, 7], [3, 2, 6, 2, 0, 2]], + [[2, 3, 2, 6, 0, 1], [2, 6, 7, 5, 0, 1], [0, 1, 7, 5, 1, 5], [2, 6, 6, 4, 7, 5]], + [[0, 4, 4, 6, 0, 1], [0, 1, 4, 6, 1, 3], [1, 3, 4, 6, 5, 7], [2, 6, 0, 2, 2, 3]], + [[3, 1, 2, 3, 2, 6], [2, 6, 6, 4, 3, 1], [3, 1, 6, 4, 7, 5]], + [[4, 6, 5, 7, 4, 0], [5, 7, 5, 1, 4, 0], [2, 3, 1, 3, 7, 3]], + [[1, 0, 0, 2, 1, 5], [1, 5, 0, 2, 5, 7], [5, 7, 0, 2, 4, 6], [3, 2, 1, 3, 3, 7]], + [[0, 1, 0, 4, 2, 3], [0, 4, 5, 7, 2, 3], [0, 4, 4, 6, 5, 7], [2, 3, 5, 7, 3, 7]], + [[7, 5, 3, 7, 3, 2], [3, 2, 2, 0, 7, 5], [7, 5, 2, 0, 6, 4]], + [[0, 4, 4, 6, 5, 7], [0, 4, 5, 7, 1, 5], [0, 2, 1, 3, 3, 7], [3, 7, 2, 6, 0, 2]], + [ + [3, 1, 7, 3, 6, 2], + [6, 2, 0, 1, 3, 1], + [6, 4, 0, 1, 6, 2], + [6, 4, 5, 1, 0, 1], + [6, 4, 7, 5, 5, 1], + ], + [ + [4, 0, 6, 4, 7, 5], + [7, 5, 1, 0, 4, 0], + [7, 3, 1, 0, 7, 5], + [7, 3, 2, 0, 1, 0], + [7, 3, 6, 2, 2, 0], + ], + [[7, 3, 6, 2, 6, 4], [7, 5, 7, 3, 6, 4]], + [[6, 2, 6, 7, 6, 4]], + [[0, 4, 0, 1, 0, 2], [6, 7, 4, 6, 2, 6]], + [[1, 0, 1, 5, 1, 3], [7, 6, 4, 6, 2, 6]], + [[1, 3, 0, 2, 1, 5], [0, 2, 0, 4, 1, 5], [7, 6, 4, 6, 2, 6]], + [[2, 3, 6, 7, 2, 0], [6, 7, 6, 4, 2, 0]], + [[4, 0, 0, 1, 4, 6], [4, 6, 0, 1, 6, 7], [6, 7, 0, 1, 2, 3]], + [[6, 4, 2, 0, 6, 7], [2, 0, 2, 3, 6, 7], [5, 1, 3, 1, 0, 1]], + [[1, 5, 1, 3, 0, 4], [1, 3, 7, 6, 0, 4], [0, 4, 7, 6, 4, 6], [1, 3, 3, 2, 7, 6]], + [[3, 2, 3, 1, 3, 7], [6, 4, 2, 6, 7, 6]], + [[3, 7, 3, 2, 1, 3], [0, 2, 0, 4, 1, 0], [7, 6, 4, 6, 2, 6]], + [[1, 5, 3, 7, 1, 0], [3, 7, 3, 2, 1, 0], [4, 6, 2, 6, 7, 6]], + [[2, 0, 0, 4, 2, 3], [2, 3, 0, 4, 3, 7], [3, 7, 0, 4, 1, 5], [6, 4, 2, 6, 6, 7]], + [[7, 6, 6, 4, 7, 3], [7, 3, 6, 4, 3, 1], [3, 1, 6, 4, 2, 0]], + [[0, 1, 4, 6, 0, 4], [0, 1, 6, 7, 4, 6], [0, 1, 1, 3, 6, 7], [1, 3, 3, 7, 6, 7]], + [[0, 2, 0, 1, 4, 6], [0, 1, 3, 7, 4, 6], [0, 1, 1, 5, 3, 7], [4, 6, 3, 7, 6, 7]], + [[7, 3, 6, 7, 6, 4], [6, 4, 4, 0, 7, 3], [7, 3, 4, 0, 5, 1]], + [[4, 0, 6, 2, 4, 5], [6, 2, 6, 7, 4, 5]], + [[2, 6, 6, 7, 2, 0], [2, 0, 6, 7, 0, 1], [0, 1, 6, 7, 4, 5]], + [[6, 7, 4, 5, 6, 2], [4, 5, 4, 0, 6, 2], [3, 1, 0, 1, 5, 1]], + [[2, 0, 2, 6, 3, 1], [2, 6, 4, 5, 3, 1], [2, 6, 6, 7, 4, 5], [3, 1, 4, 5, 1, 5]], + [[0, 2, 2, 3, 0, 4], [0, 4, 2, 3, 4, 5], [4, 5, 2, 3, 6, 7]], + [[0, 1, 2, 3, 6, 7], [0, 1, 6, 7, 4, 5]], + [[0, 2, 2, 3, 0, 4], [0, 4, 2, 3, 4, 5], [4, 5, 2, 3, 6, 7], [1, 3, 0, 1, 1, 5]], + [[5, 4, 1, 5, 1, 3], [1, 3, 3, 2, 5, 4], [5, 4, 3, 2, 7, 6]], + [[4, 0, 6, 2, 4, 5], [6, 2, 6, 7, 4, 5], [1, 3, 7, 3, 2, 3]], + [[2, 6, 6, 7, 2, 0], [2, 0, 6, 7, 0, 1], [0, 1, 6, 7, 4, 5], [3, 7, 2, 3, 3, 1]], + [[0, 1, 1, 5, 3, 7], [0, 1, 3, 7, 2, 3], [0, 4, 2, 6, 6, 7], [6, 7, 4, 5, 0, 4]], + [ + [6, 2, 7, 6, 5, 4], + [5, 4, 0, 2, 6, 2], + [5, 1, 0, 2, 5, 4], + [5, 1, 3, 2, 0, 2], + [5, 1, 7, 3, 3, 2], + ], + [[3, 1, 3, 7, 2, 0], [3, 7, 5, 4, 2, 0], [2, 0, 5, 4, 0, 4], [3, 7, 7, 6, 5, 4]], + [[1, 0, 3, 1, 3, 7], [3, 7, 7, 6, 1, 0], [1, 0, 7, 6, 5, 4]], + [ + [1, 0, 5, 1, 7, 3], + [7, 3, 2, 0, 1, 0], + [7, 6, 2, 0, 7, 3], + [7, 6, 4, 0, 2, 0], + [7, 6, 5, 4, 4, 0], + ], + [[7, 6, 5, 4, 5, 1], [7, 3, 7, 6, 5, 1]], + [[5, 7, 5, 1, 5, 4], [6, 2, 7, 6, 4, 6]], + [[0, 2, 0, 4, 1, 0], [5, 4, 5, 7, 1, 5], [2, 6, 7, 6, 4, 6]], + [[1, 0, 5, 4, 1, 3], [5, 4, 5, 7, 1, 3], [2, 6, 7, 6, 4, 6]], + [[4, 5, 5, 7, 4, 0], [4, 0, 5, 7, 0, 2], [0, 2, 5, 7, 1, 3], [6, 7, 4, 6, 6, 2]], + [[2, 3, 6, 7, 2, 0], [6, 7, 6, 4, 2, 0], [1, 5, 4, 5, 7, 5]], + [[4, 0, 0, 1, 4, 6], [4, 6, 0, 1, 6, 7], [6, 7, 0, 1, 2, 3], [5, 1, 4, 5, 5, 7]], + [[0, 2, 2, 3, 6, 7], [0, 2, 6, 7, 4, 6], [0, 1, 4, 5, 5, 7], [5, 7, 1, 3, 0, 1]], + [ + [5, 4, 7, 5, 3, 1], + [3, 1, 0, 4, 5, 4], + [3, 2, 0, 4, 3, 1], + [3, 2, 6, 4, 0, 4], + [3, 2, 7, 6, 6, 4], + ], + [[5, 4, 5, 7, 1, 5], [3, 7, 3, 2, 1, 3], [4, 6, 2, 6, 7, 6]], + [[1, 0, 0, 2, 0, 4], [1, 5, 5, 4, 5, 7], [3, 2, 1, 3, 3, 7], [2, 6, 7, 6, 4, 6]], + [[7, 3, 3, 2, 7, 5], [7, 5, 3, 2, 5, 4], [5, 4, 3, 2, 1, 0], [6, 2, 7, 6, 6, 4]], + [ + [0, 4, 2, 3, 0, 2], + [0, 4, 3, 7, 2, 3], + [0, 4, 4, 5, 3, 7], + [4, 5, 5, 7, 3, 7], + [6, 7, 4, 6, 2, 6], + ], + [[7, 6, 6, 4, 7, 3], [7, 3, 6, 4, 3, 1], [3, 1, 6, 4, 2, 0], [5, 4, 7, 5, 5, 1]], + [ + [0, 1, 4, 6, 0, 4], + [0, 1, 6, 7, 4, 6], + [0, 1, 1, 3, 6, 7], + [1, 3, 3, 7, 6, 7], + [5, 7, 1, 5, 4, 5], + ], + [ + [6, 7, 4, 6, 0, 2], + [0, 2, 3, 7, 6, 7], + [0, 1, 3, 7, 0, 2], + [0, 1, 5, 7, 3, 7], + [0, 1, 4, 5, 5, 7], + ], + [[4, 0, 6, 7, 4, 6], [4, 0, 7, 3, 6, 7], [4, 0, 5, 7, 7, 3], [4, 5, 5, 7, 4, 0]], + [[7, 5, 5, 1, 7, 6], [7, 6, 5, 1, 6, 2], [6, 2, 5, 1, 4, 0]], + [[0, 2, 1, 5, 0, 1], [0, 2, 5, 7, 1, 5], [0, 2, 2, 6, 5, 7], [2, 6, 6, 7, 5, 7]], + [[1, 3, 1, 0, 5, 7], [1, 0, 2, 6, 5, 7], [5, 7, 2, 6, 7, 6], [1, 0, 0, 4, 2, 6]], + [[2, 0, 6, 2, 6, 7], [6, 7, 7, 5, 2, 0], [2, 0, 7, 5, 3, 1]], + [[0, 4, 0, 2, 1, 5], [0, 2, 6, 7, 1, 5], [0, 2, 2, 3, 6, 7], [1, 5, 6, 7, 5, 7]], + [[7, 6, 5, 7, 5, 1], [5, 1, 1, 0, 7, 6], [7, 6, 1, 0, 3, 2]], + [ + [2, 0, 3, 2, 7, 6], + [7, 6, 4, 0, 2, 0], + [7, 5, 4, 0, 7, 6], + [7, 5, 1, 0, 4, 0], + [7, 5, 3, 1, 1, 0], + ], + [[7, 5, 3, 1, 3, 2], [7, 6, 7, 5, 3, 2]], + [[7, 5, 5, 1, 7, 6], [7, 6, 5, 1, 6, 2], [6, 2, 5, 1, 4, 0], [3, 1, 7, 3, 3, 2]], + [ + [0, 2, 1, 5, 0, 1], + [0, 2, 5, 7, 1, 5], + [0, 2, 2, 6, 5, 7], + [2, 6, 6, 7, 5, 7], + [3, 7, 2, 3, 1, 3], + ], + [ + [3, 7, 2, 3, 0, 1], + [0, 1, 5, 7, 3, 7], + [0, 4, 5, 7, 0, 1], + [0, 4, 6, 7, 5, 7], + [0, 4, 2, 6, 6, 7], + ], + [[2, 0, 3, 7, 2, 3], [2, 0, 7, 5, 3, 7], [2, 0, 6, 7, 7, 5], [2, 6, 6, 7, 2, 0]], + [ + [5, 7, 1, 5, 0, 4], + [0, 4, 6, 7, 5, 7], + [0, 2, 6, 7, 0, 4], + [0, 2, 3, 7, 6, 7], + [0, 2, 1, 3, 3, 7], + ], + [[1, 0, 5, 7, 1, 5], [1, 0, 7, 6, 5, 7], [1, 0, 3, 7, 7, 6], [1, 3, 3, 7, 1, 0]], + [[0, 2, 0, 1, 0, 4], [3, 7, 6, 7, 5, 7]], + [[7, 5, 7, 3, 7, 6]], + [[7, 3, 7, 5, 7, 6]], + [[0, 1, 0, 2, 0, 4], [6, 7, 3, 7, 5, 7]], + [[1, 3, 1, 0, 1, 5], [7, 6, 3, 7, 5, 7]], + [[0, 4, 1, 5, 0, 2], [1, 5, 1, 3, 0, 2], [6, 7, 3, 7, 5, 7]], + [[2, 6, 2, 0, 2, 3], [7, 5, 6, 7, 3, 7]], + [[0, 1, 2, 3, 0, 4], [2, 3, 2, 6, 0, 4], [5, 7, 6, 7, 3, 7]], + [[1, 5, 1, 3, 0, 1], [2, 3, 2, 6, 0, 2], [5, 7, 6, 7, 3, 7]], + [[3, 2, 2, 6, 3, 1], [3, 1, 2, 6, 1, 5], [1, 5, 2, 6, 0, 4], [7, 6, 3, 7, 7, 5]], + [[3, 1, 7, 5, 3, 2], [7, 5, 7, 6, 3, 2]], + [[7, 6, 3, 2, 7, 5], [3, 2, 3, 1, 7, 5], [4, 0, 1, 0, 2, 0]], + [[5, 7, 7, 6, 5, 1], [5, 1, 7, 6, 1, 0], [1, 0, 7, 6, 3, 2]], + [[2, 3, 2, 0, 6, 7], [2, 0, 1, 5, 6, 7], [2, 0, 0, 4, 1, 5], [6, 7, 1, 5, 7, 5]], + [[6, 2, 2, 0, 6, 7], [6, 7, 2, 0, 7, 5], [7, 5, 2, 0, 3, 1]], + [[0, 4, 0, 1, 2, 6], [0, 1, 5, 7, 2, 6], [2, 6, 5, 7, 6, 7], [0, 1, 1, 3, 5, 7]], + [[1, 5, 0, 2, 1, 0], [1, 5, 2, 6, 0, 2], [1, 5, 5, 7, 2, 6], [5, 7, 7, 6, 2, 6]], + [[5, 1, 7, 5, 7, 6], [7, 6, 6, 2, 5, 1], [5, 1, 6, 2, 4, 0]], + [[4, 5, 4, 0, 4, 6], [7, 3, 5, 7, 6, 7]], + [[0, 2, 4, 6, 0, 1], [4, 6, 4, 5, 0, 1], [3, 7, 5, 7, 6, 7]], + [[4, 6, 4, 5, 0, 4], [1, 5, 1, 3, 0, 1], [6, 7, 3, 7, 5, 7]], + [[5, 1, 1, 3, 5, 4], [5, 4, 1, 3, 4, 6], [4, 6, 1, 3, 0, 2], [7, 3, 5, 7, 7, 6]], + [[2, 3, 2, 6, 0, 2], [4, 6, 4, 5, 0, 4], [3, 7, 5, 7, 6, 7]], + [[6, 4, 4, 5, 6, 2], [6, 2, 4, 5, 2, 3], [2, 3, 4, 5, 0, 1], [7, 5, 6, 7, 7, 3]], + [[0, 1, 1, 5, 1, 3], [0, 2, 2, 3, 2, 6], [4, 5, 0, 4, 4, 6], [5, 7, 6, 7, 3, 7]], + [ + [1, 3, 5, 4, 1, 5], + [1, 3, 4, 6, 5, 4], + [1, 3, 3, 2, 4, 6], + [3, 2, 2, 6, 4, 6], + [7, 6, 3, 7, 5, 7], + ], + [[3, 1, 7, 5, 3, 2], [7, 5, 7, 6, 3, 2], [0, 4, 6, 4, 5, 4]], + [[1, 0, 0, 2, 4, 6], [1, 0, 4, 6, 5, 4], [1, 3, 5, 7, 7, 6], [7, 6, 3, 2, 1, 3]], + [[5, 7, 7, 6, 5, 1], [5, 1, 7, 6, 1, 0], [1, 0, 7, 6, 3, 2], [4, 6, 5, 4, 4, 0]], + [ + [7, 5, 6, 7, 2, 3], + [2, 3, 1, 5, 7, 5], + [2, 0, 1, 5, 2, 3], + [2, 0, 4, 5, 1, 5], + [2, 0, 6, 4, 4, 5], + ], + [[6, 2, 2, 0, 6, 7], [6, 7, 2, 0, 7, 5], [7, 5, 2, 0, 3, 1], [4, 0, 6, 4, 4, 5]], + [ + [4, 6, 5, 4, 1, 0], + [1, 0, 2, 6, 4, 6], + [1, 3, 2, 6, 1, 0], + [1, 3, 7, 6, 2, 6], + [1, 3, 5, 7, 7, 6], + ], + [ + [1, 5, 0, 2, 1, 0], + [1, 5, 2, 6, 0, 2], + [1, 5, 5, 7, 2, 6], + [5, 7, 7, 6, 2, 6], + [4, 6, 5, 4, 0, 4], + ], + [[5, 1, 4, 6, 5, 4], [5, 1, 6, 2, 4, 6], [5, 1, 7, 6, 6, 2], [5, 7, 7, 6, 5, 1]], + [[5, 4, 7, 6, 5, 1], [7, 6, 7, 3, 5, 1]], + [[7, 3, 5, 1, 7, 6], [5, 1, 5, 4, 7, 6], [2, 0, 4, 0, 1, 0]], + [[3, 1, 1, 0, 3, 7], [3, 7, 1, 0, 7, 6], [7, 6, 1, 0, 5, 4]], + [[0, 2, 0, 4, 1, 3], [0, 4, 6, 7, 1, 3], [1, 3, 6, 7, 3, 7], [0, 4, 4, 5, 6, 7]], + [[5, 4, 7, 6, 5, 1], [7, 6, 7, 3, 5, 1], [0, 2, 3, 2, 6, 2]], + [[1, 5, 5, 4, 7, 6], [1, 5, 7, 6, 3, 7], [1, 0, 3, 2, 2, 6], [2, 6, 0, 4, 1, 0]], + [[3, 1, 1, 0, 3, 7], [3, 7, 1, 0, 7, 6], [7, 6, 1, 0, 5, 4], [2, 0, 3, 2, 2, 6]], + [ + [2, 3, 6, 2, 4, 0], + [4, 0, 1, 3, 2, 3], + [4, 5, 1, 3, 4, 0], + [4, 5, 7, 3, 1, 3], + [4, 5, 6, 7, 7, 3], + ], + [[1, 5, 5, 4, 1, 3], [1, 3, 5, 4, 3, 2], [3, 2, 5, 4, 7, 6]], + [[1, 5, 5, 4, 1, 3], [1, 3, 5, 4, 3, 2], [3, 2, 5, 4, 7, 6], [0, 4, 1, 0, 0, 2]], + [[1, 0, 5, 4, 7, 6], [1, 0, 7, 6, 3, 2]], + [[2, 3, 0, 2, 0, 4], [0, 4, 4, 5, 2, 3], [2, 3, 4, 5, 6, 7]], + [[1, 3, 1, 5, 0, 2], [1, 5, 7, 6, 0, 2], [1, 5, 5, 4, 7, 6], [0, 2, 7, 6, 2, 6]], + [ + [5, 1, 4, 5, 6, 7], + [6, 7, 3, 1, 5, 1], + [6, 2, 3, 1, 6, 7], + [6, 2, 0, 1, 3, 1], + [6, 2, 4, 0, 0, 1], + ], + [[6, 7, 2, 6, 2, 0], [2, 0, 0, 1, 6, 7], [6, 7, 0, 1, 4, 5]], + [[6, 2, 4, 0, 4, 5], [6, 7, 6, 2, 4, 5]], + [[6, 7, 7, 3, 6, 4], [6, 4, 7, 3, 4, 0], [4, 0, 7, 3, 5, 1]], + [[1, 5, 1, 0, 3, 7], [1, 0, 4, 6, 3, 7], [1, 0, 0, 2, 4, 6], [3, 7, 4, 6, 7, 6]], + [[1, 0, 3, 7, 1, 3], [1, 0, 7, 6, 3, 7], [1, 0, 0, 4, 7, 6], [0, 4, 4, 6, 7, 6]], + [[6, 4, 7, 6, 7, 3], [7, 3, 3, 1, 6, 4], [6, 4, 3, 1, 2, 0]], + [[6, 7, 7, 3, 6, 4], [6, 4, 7, 3, 4, 0], [4, 0, 7, 3, 5, 1], [2, 3, 6, 2, 2, 0]], + [ + [7, 6, 3, 7, 1, 5], + [1, 5, 4, 6, 7, 6], + [1, 0, 4, 6, 1, 5], + [1, 0, 2, 6, 4, 6], + [1, 0, 3, 2, 2, 6], + ], + [ + [1, 0, 3, 7, 1, 3], + [1, 0, 7, 6, 3, 7], + [1, 0, 0, 4, 7, 6], + [0, 4, 4, 6, 7, 6], + [2, 6, 0, 2, 3, 2], + ], + [[3, 1, 7, 6, 3, 7], [3, 1, 6, 4, 7, 6], [3, 1, 2, 6, 6, 4], [3, 2, 2, 6, 3, 1]], + [[3, 2, 3, 1, 7, 6], [3, 1, 0, 4, 7, 6], [7, 6, 0, 4, 6, 4], [3, 1, 1, 5, 0, 4]], + [ + [0, 1, 2, 0, 6, 4], + [6, 4, 5, 1, 0, 1], + [6, 7, 5, 1, 6, 4], + [6, 7, 3, 1, 5, 1], + [6, 7, 2, 3, 3, 1], + ], + [[0, 1, 4, 0, 4, 6], [4, 6, 6, 7, 0, 1], [0, 1, 6, 7, 2, 3]], + [[6, 7, 2, 3, 2, 0], [6, 4, 6, 7, 2, 0]], + [ + [2, 6, 0, 2, 1, 3], + [1, 3, 7, 6, 2, 6], + [1, 5, 7, 6, 1, 3], + [1, 5, 4, 6, 7, 6], + [1, 5, 0, 4, 4, 6], + ], + [[1, 5, 1, 0, 1, 3], [4, 6, 7, 6, 2, 6]], + [[0, 1, 2, 6, 0, 2], [0, 1, 6, 7, 2, 6], [0, 1, 4, 6, 6, 7], [0, 4, 4, 6, 0, 1]], + [[6, 7, 6, 2, 6, 4]], + [[6, 2, 7, 3, 6, 4], [7, 3, 7, 5, 6, 4]], + [[7, 5, 6, 4, 7, 3], [6, 4, 6, 2, 7, 3], [1, 0, 2, 0, 4, 0]], + [[6, 2, 7, 3, 6, 4], [7, 3, 7, 5, 6, 4], [0, 1, 5, 1, 3, 1]], + [[2, 0, 0, 4, 1, 5], [2, 0, 1, 5, 3, 1], [2, 6, 3, 7, 7, 5], [7, 5, 6, 4, 2, 6]], + [[3, 7, 7, 5, 3, 2], [3, 2, 7, 5, 2, 0], [2, 0, 7, 5, 6, 4]], + [[3, 2, 3, 7, 1, 0], [3, 7, 6, 4, 1, 0], [3, 7, 7, 5, 6, 4], [1, 0, 6, 4, 0, 4]], + [[3, 7, 7, 5, 3, 2], [3, 2, 7, 5, 2, 0], [2, 0, 7, 5, 6, 4], [1, 5, 3, 1, 1, 0]], + [ + [7, 3, 5, 7, 4, 6], + [4, 6, 2, 3, 7, 3], + [4, 0, 2, 3, 4, 6], + [4, 0, 1, 3, 2, 3], + [4, 0, 5, 1, 1, 3], + ], + [[2, 3, 3, 1, 2, 6], [2, 6, 3, 1, 6, 4], [6, 4, 3, 1, 7, 5]], + [[2, 3, 3, 1, 2, 6], [2, 6, 3, 1, 6, 4], [6, 4, 3, 1, 7, 5], [0, 1, 2, 0, 0, 4]], + [[1, 0, 1, 5, 3, 2], [1, 5, 4, 6, 3, 2], [3, 2, 4, 6, 2, 6], [1, 5, 5, 7, 4, 6]], + [ + [0, 2, 4, 0, 5, 1], + [5, 1, 3, 2, 0, 2], + [5, 7, 3, 2, 5, 1], + [5, 7, 6, 2, 3, 2], + [5, 7, 4, 6, 6, 2], + ], + [[2, 0, 3, 1, 7, 5], [2, 0, 7, 5, 6, 4]], + [[4, 6, 0, 4, 0, 1], [0, 1, 1, 3, 4, 6], [4, 6, 1, 3, 5, 7]], + [[0, 2, 1, 0, 1, 5], [1, 5, 5, 7, 0, 2], [0, 2, 5, 7, 4, 6]], + [[5, 7, 4, 6, 4, 0], [5, 1, 5, 7, 4, 0]], + [[5, 4, 4, 0, 5, 7], [5, 7, 4, 0, 7, 3], [7, 3, 4, 0, 6, 2]], + [[0, 1, 0, 2, 4, 5], [0, 2, 3, 7, 4, 5], [4, 5, 3, 7, 5, 7], [0, 2, 2, 6, 3, 7]], + [[5, 4, 4, 0, 5, 7], [5, 7, 4, 0, 7, 3], [7, 3, 4, 0, 6, 2], [1, 0, 5, 1, 1, 3]], + [ + [1, 5, 3, 1, 2, 0], + [2, 0, 4, 5, 1, 5], + [2, 6, 4, 5, 2, 0], + [2, 6, 7, 5, 4, 5], + [2, 6, 3, 7, 7, 5], + ], + [[2, 3, 0, 4, 2, 0], [2, 3, 4, 5, 0, 4], [2, 3, 3, 7, 4, 5], [3, 7, 7, 5, 4, 5]], + [[3, 2, 7, 3, 7, 5], [7, 5, 5, 4, 3, 2], [3, 2, 5, 4, 1, 0]], + [ + [2, 3, 0, 4, 2, 0], + [2, 3, 4, 5, 0, 4], + [2, 3, 3, 7, 4, 5], + [3, 7, 7, 5, 4, 5], + [1, 5, 3, 1, 0, 1], + ], + [[3, 2, 1, 5, 3, 1], [3, 2, 5, 4, 1, 5], [3, 2, 7, 5, 5, 4], [3, 7, 7, 5, 3, 2]], + [[2, 6, 2, 3, 0, 4], [2, 3, 7, 5, 0, 4], [2, 3, 3, 1, 7, 5], [0, 4, 7, 5, 4, 5]], + [ + [3, 2, 1, 3, 5, 7], + [5, 7, 6, 2, 3, 2], + [5, 4, 6, 2, 5, 7], + [5, 4, 0, 2, 6, 2], + [5, 4, 1, 0, 0, 2], + ], + [ + [4, 5, 0, 4, 2, 6], + [2, 6, 7, 5, 4, 5], + [2, 3, 7, 5, 2, 6], + [2, 3, 1, 5, 7, 5], + [2, 3, 0, 1, 1, 5], + ], + [[2, 3, 2, 0, 2, 6], [1, 5, 7, 5, 4, 5]], + [[5, 7, 4, 5, 4, 0], [4, 0, 0, 2, 5, 7], [5, 7, 0, 2, 1, 3]], + [[5, 4, 1, 0, 1, 3], [5, 7, 5, 4, 1, 3]], + [[0, 2, 4, 5, 0, 4], [0, 2, 5, 7, 4, 5], [0, 2, 1, 5, 5, 7], [0, 1, 1, 5, 0, 2]], + [[5, 4, 5, 1, 5, 7]], + [[4, 6, 6, 2, 4, 5], [4, 5, 6, 2, 5, 1], [5, 1, 6, 2, 7, 3]], + [[4, 6, 6, 2, 4, 5], [4, 5, 6, 2, 5, 1], [5, 1, 6, 2, 7, 3], [0, 2, 4, 0, 0, 1]], + [[3, 7, 3, 1, 2, 6], [3, 1, 5, 4, 2, 6], [3, 1, 1, 0, 5, 4], [2, 6, 5, 4, 6, 4]], + [ + [6, 4, 2, 6, 3, 7], + [3, 7, 5, 4, 6, 4], + [3, 1, 5, 4, 3, 7], + [3, 1, 0, 4, 5, 4], + [3, 1, 2, 0, 0, 4], + ], + [[2, 0, 2, 3, 6, 4], [2, 3, 1, 5, 6, 4], [6, 4, 1, 5, 4, 5], [2, 3, 3, 7, 1, 5]], + [ + [0, 4, 1, 0, 3, 2], + [3, 2, 6, 4, 0, 4], + [3, 7, 6, 4, 3, 2], + [3, 7, 5, 4, 6, 4], + [3, 7, 1, 5, 5, 4], + ], + [ + [1, 3, 0, 1, 4, 5], + [4, 5, 7, 3, 1, 3], + [4, 6, 7, 3, 4, 5], + [4, 6, 2, 3, 7, 3], + [4, 6, 0, 2, 2, 3], + ], + [[3, 7, 3, 1, 3, 2], [5, 4, 6, 4, 0, 4]], + [[3, 1, 2, 6, 3, 2], [3, 1, 6, 4, 2, 6], [3, 1, 1, 5, 6, 4], [1, 5, 5, 4, 6, 4]], + [ + [3, 1, 2, 6, 3, 2], + [3, 1, 6, 4, 2, 6], + [3, 1, 1, 5, 6, 4], + [1, 5, 5, 4, 6, 4], + [0, 4, 1, 0, 2, 0], + ], + [[4, 5, 6, 4, 6, 2], [6, 2, 2, 3, 4, 5], [4, 5, 2, 3, 0, 1]], + [[2, 3, 6, 4, 2, 6], [2, 3, 4, 5, 6, 4], [2, 3, 0, 4, 4, 5], [2, 0, 0, 4, 2, 3]], + [[1, 3, 5, 1, 5, 4], [5, 4, 4, 6, 1, 3], [1, 3, 4, 6, 0, 2]], + [[1, 3, 0, 4, 1, 0], [1, 3, 4, 6, 0, 4], [1, 3, 5, 4, 4, 6], [1, 5, 5, 4, 1, 3]], + [[4, 6, 0, 2, 0, 1], [4, 5, 4, 6, 0, 1]], + [[4, 6, 4, 0, 4, 5]], + [[4, 0, 6, 2, 7, 3], [4, 0, 7, 3, 5, 1]], + [[1, 5, 0, 1, 0, 2], [0, 2, 2, 6, 1, 5], [1, 5, 2, 6, 3, 7]], + [[3, 7, 1, 3, 1, 0], [1, 0, 0, 4, 3, 7], [3, 7, 0, 4, 2, 6]], + [[3, 1, 2, 0, 2, 6], [3, 7, 3, 1, 2, 6]], + [[0, 4, 2, 0, 2, 3], [2, 3, 3, 7, 0, 4], [0, 4, 3, 7, 1, 5]], + [[3, 7, 1, 5, 1, 0], [3, 2, 3, 7, 1, 0]], + [[0, 4, 1, 3, 0, 1], [0, 4, 3, 7, 1, 3], [0, 4, 2, 3, 3, 7], [0, 2, 2, 3, 0, 4]], + [[3, 7, 3, 1, 3, 2]], + [[2, 6, 3, 2, 3, 1], [3, 1, 1, 5, 2, 6], [2, 6, 1, 5, 0, 4]], + [[1, 5, 3, 2, 1, 3], [1, 5, 2, 6, 3, 2], [1, 5, 0, 2, 2, 6], [1, 0, 0, 2, 1, 5]], + [[2, 3, 0, 1, 0, 4], [2, 6, 2, 3, 0, 4]], + [[2, 3, 2, 0, 2, 6]], + [[1, 5, 0, 4, 0, 2], [1, 3, 1, 5, 0, 2]], + [[1, 5, 1, 0, 1, 3]], + [[0, 2, 0, 1, 0, 4]], + [], +] + + +def create_mc_lookup_table(): + cases = torch.zeros(256, 5, 3, dtype=torch.long) + masks = torch.zeros(256, 5, dtype=torch.bool) + + edge_to_index = { + (0, 1): 0, + (2, 3): 1, + (4, 5): 2, + (6, 7): 3, + (0, 2): 4, + (1, 3): 5, + (4, 6): 6, + (5, 7): 7, + (0, 4): 8, + (1, 5): 9, + (2, 6): 10, + (3, 7): 11, + } + + for i, case in enumerate(MC_TABLE): + for j, tri in enumerate(case): + for k, (c1, c2) in enumerate(zip(tri[::2], tri[1::2])): + cases[i, j, k] = edge_to_index[(c1, c2) if c1 < c2 else (c2, c1)] + masks[i, j] = True + return cases, masks + + +RENDERER_CONFIG = {} + + +def renderer_model_from_original_config(): + model = ShapERenderer(**RENDERER_CONFIG) + + return model + + +RENDERER_MLP_ORIGINAL_PREFIX = "renderer.nerstf" + +RENDERER_PARAMS_PROJ_ORIGINAL_PREFIX = "encoder.params_proj" + + +def renderer_model_original_checkpoint_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + diffusers_checkpoint.update( + {f"mlp.{k}": checkpoint[f"{RENDERER_MLP_ORIGINAL_PREFIX}.{k}"] for k in model.mlp.state_dict().keys()} + ) + + diffusers_checkpoint.update( + { + f"params_proj.{k}": checkpoint[f"{RENDERER_PARAMS_PROJ_ORIGINAL_PREFIX}.{k}"] + for k in model.params_proj.state_dict().keys() + } + ) + + diffusers_checkpoint.update({"void.background": model.state_dict()["void.background"]}) + + cases, masks = create_mc_lookup_table() + + diffusers_checkpoint.update({"mesh_decoder.cases": cases}) + diffusers_checkpoint.update({"mesh_decoder.masks": masks}) + + return diffusers_checkpoint + + +# done renderer + + +# TODO maybe document and/or can do more efficiently (build indices in for loop and extract once for each split?) +def split_attentions(*, weight, bias, split, chunk_size): + weights = [None] * split + biases = [None] * split + + weights_biases_idx = 0 + + for starting_row_index in range(0, weight.shape[0], chunk_size): + row_indices = torch.arange(starting_row_index, starting_row_index + chunk_size) + + weight_rows = weight[row_indices, :] + bias_rows = bias[row_indices] + + if weights[weights_biases_idx] is None: + assert weights[weights_biases_idx] is None + weights[weights_biases_idx] = weight_rows + biases[weights_biases_idx] = bias_rows + else: + assert weights[weights_biases_idx] is not None + weights[weights_biases_idx] = torch.concat([weights[weights_biases_idx], weight_rows]) + biases[weights_biases_idx] = torch.concat([biases[weights_biases_idx], bias_rows]) + + weights_biases_idx = (weights_biases_idx + 1) % split + + return weights, biases + + +# done unet utils + + +# Driver functions + + +def prior(*, args, checkpoint_map_location): + print("loading prior") + + prior_checkpoint = torch.load(args.prior_checkpoint_path, map_location=checkpoint_map_location) + + prior_model = prior_model_from_original_config() + + prior_diffusers_checkpoint = prior_original_checkpoint_to_diffusers_checkpoint(prior_model, prior_checkpoint) + + del prior_checkpoint + + load_prior_checkpoint_to_model(prior_diffusers_checkpoint, prior_model) + + print("done loading prior") + + return prior_model + + +def prior_image(*, args, checkpoint_map_location): + print("loading prior_image") + + print(f"load checkpoint from {args.prior_image_checkpoint_path}") + prior_checkpoint = torch.load(args.prior_image_checkpoint_path, map_location=checkpoint_map_location) + + prior_model = prior_image_model_from_original_config() + + prior_diffusers_checkpoint = prior_image_original_checkpoint_to_diffusers_checkpoint(prior_model, prior_checkpoint) + + del prior_checkpoint + + load_prior_checkpoint_to_model(prior_diffusers_checkpoint, prior_model) + + print("done loading prior_image") + + return prior_model + + +def renderer(*, args, checkpoint_map_location): + print(" loading renderer") + + renderer_checkpoint = torch.load(args.transmitter_checkpoint_path, map_location=checkpoint_map_location) + + renderer_model = renderer_model_from_original_config() + + renderer_diffusers_checkpoint = renderer_model_original_checkpoint_to_diffusers_checkpoint( + renderer_model, renderer_checkpoint + ) + + del renderer_checkpoint + + load_checkpoint_to_model(renderer_diffusers_checkpoint, renderer_model, strict=True) + + print("done loading renderer") + + return renderer_model + + +# prior model will expect clip_mean and clip_std, whic are missing from the state_dict +PRIOR_EXPECTED_MISSING_KEYS = ["clip_mean", "clip_std"] + + +def load_prior_checkpoint_to_model(checkpoint, model): + with tempfile.NamedTemporaryFile() as file: + torch.save(checkpoint, file.name) + del checkpoint + missing_keys, unexpected_keys = model.load_state_dict(torch.load(file.name), strict=False) + missing_keys = list(set(missing_keys) - set(PRIOR_EXPECTED_MISSING_KEYS)) + + if len(unexpected_keys) > 0: + raise ValueError(f"Unexpected keys when loading prior model: {unexpected_keys}") + if len(missing_keys) > 0: + raise ValueError(f"Missing keys when loading prior model: {missing_keys}") + + +def load_checkpoint_to_model(checkpoint, model, strict=False): + with tempfile.NamedTemporaryFile() as file: + torch.save(checkpoint, file.name) + del checkpoint + if strict: + model.load_state_dict(torch.load(file.name), strict=True) + else: + load_checkpoint_and_dispatch(model, file.name, device_map="auto") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + + parser.add_argument( + "--prior_checkpoint_path", + default=None, + type=str, + required=False, + help="Path to the prior checkpoint to convert.", + ) + + parser.add_argument( + "--prior_image_checkpoint_path", + default=None, + type=str, + required=False, + help="Path to the prior_image checkpoint to convert.", + ) + + parser.add_argument( + "--transmitter_checkpoint_path", + default=None, + type=str, + required=False, + help="Path to the transmitter checkpoint to convert.", + ) + + parser.add_argument( + "--checkpoint_load_device", + default="cpu", + type=str, + required=False, + help="The device passed to `map_location` when loading checkpoints.", + ) + + parser.add_argument( + "--debug", + default=None, + type=str, + required=False, + help="Only run a specific stage of the convert script. Used for debugging", + ) + + args = parser.parse_args() + + print(f"loading checkpoints to {args.checkpoint_load_device}") + + checkpoint_map_location = torch.device(args.checkpoint_load_device) + + if args.debug is not None: + print(f"debug: only executing {args.debug}") + + if args.debug is None: + print("YiYi TO-DO") + elif args.debug == "prior": + prior_model = prior(args=args, checkpoint_map_location=checkpoint_map_location) + prior_model.save_pretrained(args.dump_path) + elif args.debug == "prior_image": + prior_model = prior_image(args=args, checkpoint_map_location=checkpoint_map_location) + prior_model.save_pretrained(args.dump_path) + elif args.debug == "renderer": + renderer_model = renderer(args=args, checkpoint_map_location=checkpoint_map_location) + renderer_model.save_pretrained(args.dump_path) + else: + raise ValueError(f"unknown debug value : {args.debug}") diff --git a/diffusers-0.27.0/scripts/convert_stable_cascade.py b/diffusers-0.27.0/scripts/convert_stable_cascade.py new file mode 100755 index 0000000..ce10970 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_stable_cascade.py @@ -0,0 +1,218 @@ +# Run this script to convert the Stable Cascade model weights to a diffusers pipeline. +import argparse +from contextlib import nullcontext + +import torch +from safetensors.torch import load_file +from transformers import ( + AutoTokenizer, + CLIPConfig, + CLIPImageProcessor, + CLIPTextModelWithProjection, + CLIPVisionModelWithProjection, +) + +from diffusers import ( + DDPMWuerstchenScheduler, + StableCascadeCombinedPipeline, + StableCascadeDecoderPipeline, + StableCascadePriorPipeline, +) +from diffusers.loaders.single_file_utils import convert_stable_cascade_unet_single_file_to_diffusers +from diffusers.models import StableCascadeUNet +from diffusers.models.modeling_utils import load_model_dict_into_meta +from diffusers.pipelines.wuerstchen import PaellaVQModel +from diffusers.utils import is_accelerate_available + + +if is_accelerate_available(): + from accelerate import init_empty_weights + +parser = argparse.ArgumentParser(description="Convert Stable Cascade model weights to a diffusers pipeline") +parser.add_argument("--model_path", type=str, help="Location of Stable Cascade weights") +parser.add_argument("--stage_c_name", type=str, default="stage_c.safetensors", help="Name of stage c checkpoint file") +parser.add_argument("--stage_b_name", type=str, default="stage_b.safetensors", help="Name of stage b checkpoint file") +parser.add_argument("--skip_stage_c", action="store_true", help="Skip converting stage c") +parser.add_argument("--skip_stage_b", action="store_true", help="Skip converting stage b") +parser.add_argument("--use_safetensors", action="store_true", help="Use SafeTensors for conversion") +parser.add_argument( + "--prior_output_path", default="stable-cascade-prior", type=str, help="Hub organization to save the pipelines to" +) +parser.add_argument( + "--decoder_output_path", + type=str, + default="stable-cascade-decoder", + help="Hub organization to save the pipelines to", +) +parser.add_argument( + "--combined_output_path", + type=str, + default="stable-cascade-combined", + help="Hub organization to save the pipelines to", +) +parser.add_argument("--save_combined", action="store_true") +parser.add_argument("--push_to_hub", action="store_true", help="Push to hub") +parser.add_argument("--variant", type=str, help="Set to bf16 to save bfloat16 weights") + +args = parser.parse_args() + +if args.skip_stage_b and args.skip_stage_c: + raise ValueError("At least one stage should be converted") +if (args.skip_stage_b or args.skip_stage_c) and args.save_combined: + raise ValueError("Cannot skip stages when creating a combined pipeline") + +model_path = args.model_path + +device = "cpu" +if args.variant == "bf16": + dtype = torch.bfloat16 +else: + dtype = torch.float32 + +# set paths to model weights +prior_checkpoint_path = f"{model_path}/{args.stage_c_name}" +decoder_checkpoint_path = f"{model_path}/{args.stage_b_name}" + +# Clip Text encoder and tokenizer +config = CLIPConfig.from_pretrained("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k") +config.text_config.projection_dim = config.projection_dim +text_encoder = CLIPTextModelWithProjection.from_pretrained( + "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", config=config.text_config +) +tokenizer = AutoTokenizer.from_pretrained("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k") + +# image processor +feature_extractor = CLIPImageProcessor() +image_encoder = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-large-patch14") + +# scheduler for prior and decoder +scheduler = DDPMWuerstchenScheduler() +ctx = init_empty_weights if is_accelerate_available() else nullcontext + +if not args.skip_stage_c: + # Prior + if args.use_safetensors: + prior_orig_state_dict = load_file(prior_checkpoint_path, device=device) + else: + prior_orig_state_dict = torch.load(prior_checkpoint_path, map_location=device) + + prior_state_dict = convert_stable_cascade_unet_single_file_to_diffusers(prior_orig_state_dict) + + with ctx(): + prior_model = StableCascadeUNet( + in_channels=16, + out_channels=16, + timestep_ratio_embedding_dim=64, + patch_size=1, + conditioning_dim=2048, + block_out_channels=[2048, 2048], + num_attention_heads=[32, 32], + down_num_layers_per_block=[8, 24], + up_num_layers_per_block=[24, 8], + down_blocks_repeat_mappers=[1, 1], + up_blocks_repeat_mappers=[1, 1], + block_types_per_layer=[ + ["SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"], + ["SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"], + ], + clip_text_in_channels=1280, + clip_text_pooled_in_channels=1280, + clip_image_in_channels=768, + clip_seq=4, + kernel_size=3, + dropout=[0.1, 0.1], + self_attn=True, + timestep_conditioning_type=["sca", "crp"], + switch_level=[False], + ) + if is_accelerate_available(): + load_model_dict_into_meta(prior_model, prior_state_dict) + else: + prior_model.load_state_dict(prior_state_dict) + + # Prior pipeline + prior_pipeline = StableCascadePriorPipeline( + prior=prior_model, + tokenizer=tokenizer, + text_encoder=text_encoder, + image_encoder=image_encoder, + scheduler=scheduler, + feature_extractor=feature_extractor, + ) + prior_pipeline.to(dtype).save_pretrained( + args.prior_output_path, push_to_hub=args.push_to_hub, variant=args.variant + ) + +if not args.skip_stage_b: + # Decoder + if args.use_safetensors: + decoder_orig_state_dict = load_file(decoder_checkpoint_path, device=device) + else: + decoder_orig_state_dict = torch.load(decoder_checkpoint_path, map_location=device) + + decoder_state_dict = convert_stable_cascade_unet_single_file_to_diffusers(decoder_orig_state_dict) + with ctx(): + decoder = StableCascadeUNet( + in_channels=4, + out_channels=4, + timestep_ratio_embedding_dim=64, + patch_size=2, + conditioning_dim=1280, + block_out_channels=[320, 640, 1280, 1280], + down_num_layers_per_block=[2, 6, 28, 6], + up_num_layers_per_block=[6, 28, 6, 2], + down_blocks_repeat_mappers=[1, 1, 1, 1], + up_blocks_repeat_mappers=[3, 3, 2, 2], + num_attention_heads=[0, 0, 20, 20], + block_types_per_layer=[ + ["SDCascadeResBlock", "SDCascadeTimestepBlock"], + ["SDCascadeResBlock", "SDCascadeTimestepBlock"], + ["SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"], + ["SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"], + ], + clip_text_pooled_in_channels=1280, + clip_seq=4, + effnet_in_channels=16, + pixel_mapper_in_channels=3, + kernel_size=3, + dropout=[0, 0, 0.1, 0.1], + self_attn=True, + timestep_conditioning_type=["sca"], + ) + + if is_accelerate_available(): + load_model_dict_into_meta(decoder, decoder_state_dict) + else: + decoder.load_state_dict(decoder_state_dict) + + # VQGAN from Wuerstchen-V2 + vqmodel = PaellaVQModel.from_pretrained("warp-ai/wuerstchen", subfolder="vqgan") + + # Decoder pipeline + decoder_pipeline = StableCascadeDecoderPipeline( + decoder=decoder, text_encoder=text_encoder, tokenizer=tokenizer, vqgan=vqmodel, scheduler=scheduler + ) + decoder_pipeline.to(dtype).save_pretrained( + args.decoder_output_path, push_to_hub=args.push_to_hub, variant=args.variant + ) + +if args.save_combined: + # Stable Cascade combined pipeline + stable_cascade_pipeline = StableCascadeCombinedPipeline( + # Decoder + text_encoder=text_encoder, + tokenizer=tokenizer, + decoder=decoder, + scheduler=scheduler, + vqgan=vqmodel, + # Prior + prior_text_encoder=text_encoder, + prior_tokenizer=tokenizer, + prior_prior=prior_model, + prior_scheduler=scheduler, + prior_image_encoder=image_encoder, + prior_feature_extractor=feature_extractor, + ) + stable_cascade_pipeline.to(dtype).save_pretrained( + args.combined_output_path, push_to_hub=args.push_to_hub, variant=args.variant + ) diff --git a/diffusers-0.27.0/scripts/convert_stable_cascade_lite.py b/diffusers-0.27.0/scripts/convert_stable_cascade_lite.py new file mode 100755 index 0000000..ddccaa3 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_stable_cascade_lite.py @@ -0,0 +1,226 @@ +# Run this script to convert the Stable Cascade model weights to a diffusers pipeline. +import argparse +from contextlib import nullcontext + +import torch +from safetensors.torch import load_file +from transformers import ( + AutoTokenizer, + CLIPConfig, + CLIPImageProcessor, + CLIPTextModelWithProjection, + CLIPVisionModelWithProjection, +) + +from diffusers import ( + DDPMWuerstchenScheduler, + StableCascadeCombinedPipeline, + StableCascadeDecoderPipeline, + StableCascadePriorPipeline, +) +from diffusers.loaders.single_file_utils import convert_stable_cascade_unet_single_file_to_diffusers +from diffusers.models import StableCascadeUNet +from diffusers.models.modeling_utils import load_model_dict_into_meta +from diffusers.pipelines.wuerstchen import PaellaVQModel +from diffusers.utils import is_accelerate_available + + +if is_accelerate_available(): + from accelerate import init_empty_weights + +parser = argparse.ArgumentParser(description="Convert Stable Cascade model weights to a diffusers pipeline") +parser.add_argument("--model_path", type=str, help="Location of Stable Cascade weights") +parser.add_argument( + "--stage_c_name", type=str, default="stage_c_lite.safetensors", help="Name of stage c checkpoint file" +) +parser.add_argument( + "--stage_b_name", type=str, default="stage_b_lite.safetensors", help="Name of stage b checkpoint file" +) +parser.add_argument("--skip_stage_c", action="store_true", help="Skip converting stage c") +parser.add_argument("--skip_stage_b", action="store_true", help="Skip converting stage b") +parser.add_argument("--use_safetensors", action="store_true", help="Use SafeTensors for conversion") +parser.add_argument( + "--prior_output_path", + default="stable-cascade-prior-lite", + type=str, + help="Hub organization to save the pipelines to", +) +parser.add_argument( + "--decoder_output_path", + type=str, + default="stable-cascade-decoder-lite", + help="Hub organization to save the pipelines to", +) +parser.add_argument( + "--combined_output_path", + type=str, + default="stable-cascade-combined-lite", + help="Hub organization to save the pipelines to", +) +parser.add_argument("--save_combined", action="store_true") +parser.add_argument("--push_to_hub", action="store_true", help="Push to hub") +parser.add_argument("--variant", type=str, help="Set to bf16 to save bfloat16 weights") + +args = parser.parse_args() + +if args.skip_stage_b and args.skip_stage_c: + raise ValueError("At least one stage should be converted") +if (args.skip_stage_b or args.skip_stage_c) and args.save_combined: + raise ValueError("Cannot skip stages when creating a combined pipeline") + +model_path = args.model_path + +device = "cpu" +if args.variant == "bf16": + dtype = torch.bfloat16 +else: + dtype = torch.float32 + +# set paths to model weights +prior_checkpoint_path = f"{model_path}/{args.stage_c_name}" +decoder_checkpoint_path = f"{model_path}/{args.stage_b_name}" + +# Clip Text encoder and tokenizer +config = CLIPConfig.from_pretrained("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k") +config.text_config.projection_dim = config.projection_dim +text_encoder = CLIPTextModelWithProjection.from_pretrained( + "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", config=config.text_config +) +tokenizer = AutoTokenizer.from_pretrained("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k") + +# image processor +feature_extractor = CLIPImageProcessor() +image_encoder = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-large-patch14") +# scheduler for prior and decoder +scheduler = DDPMWuerstchenScheduler() + +ctx = init_empty_weights if is_accelerate_available() else nullcontext + +if not args.skip_stage_c: + # Prior + if args.use_safetensors: + prior_orig_state_dict = load_file(prior_checkpoint_path, device=device) + else: + prior_orig_state_dict = torch.load(prior_checkpoint_path, map_location=device) + + prior_state_dict = convert_stable_cascade_unet_single_file_to_diffusers(prior_orig_state_dict) + with ctx(): + prior_model = StableCascadeUNet( + in_channels=16, + out_channels=16, + timestep_ratio_embedding_dim=64, + patch_size=1, + conditioning_dim=1536, + block_out_channels=[1536, 1536], + num_attention_heads=[24, 24], + down_num_layers_per_block=[4, 12], + up_num_layers_per_block=[12, 4], + down_blocks_repeat_mappers=[1, 1], + up_blocks_repeat_mappers=[1, 1], + block_types_per_layer=[ + ["SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"], + ["SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"], + ], + clip_text_in_channels=1280, + clip_text_pooled_in_channels=1280, + clip_image_in_channels=768, + clip_seq=4, + kernel_size=3, + dropout=[0.1, 0.1], + self_attn=True, + timestep_conditioning_type=["sca", "crp"], + switch_level=[False], + ) + + if is_accelerate_available(): + load_model_dict_into_meta(prior_model, prior_state_dict) + else: + prior_model.load_state_dict(prior_state_dict) + + # Prior pipeline + prior_pipeline = StableCascadePriorPipeline( + prior=prior_model, + tokenizer=tokenizer, + text_encoder=text_encoder, + image_encoder=image_encoder, + scheduler=scheduler, + feature_extractor=feature_extractor, + ) + prior_pipeline.to(dtype).save_pretrained( + args.prior_output_path, push_to_hub=args.push_to_hub, variant=args.variant + ) + +if not args.skip_stage_b: + # Decoder + if args.use_safetensors: + decoder_orig_state_dict = load_file(decoder_checkpoint_path, device=device) + else: + decoder_orig_state_dict = torch.load(decoder_checkpoint_path, map_location=device) + + decoder_state_dict = convert_stable_cascade_unet_single_file_to_diffusers(decoder_orig_state_dict) + + with ctx(): + decoder = StableCascadeUNet( + in_channels=4, + out_channels=4, + timestep_ratio_embedding_dim=64, + patch_size=2, + conditioning_dim=1280, + block_out_channels=[320, 576, 1152, 1152], + down_num_layers_per_block=[2, 4, 14, 4], + up_num_layers_per_block=[4, 14, 4, 2], + down_blocks_repeat_mappers=[1, 1, 1, 1], + up_blocks_repeat_mappers=[2, 2, 2, 2], + num_attention_heads=[0, 9, 18, 18], + block_types_per_layer=[ + ["SDCascadeResBlock", "SDCascadeTimestepBlock"], + ["SDCascadeResBlock", "SDCascadeTimestepBlock"], + ["SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"], + ["SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"], + ], + clip_text_pooled_in_channels=1280, + clip_seq=4, + effnet_in_channels=16, + pixel_mapper_in_channels=3, + kernel_size=3, + dropout=[0, 0, 0.1, 0.1], + self_attn=True, + timestep_conditioning_type=["sca"], + ) + + if is_accelerate_available(): + load_model_dict_into_meta(decoder, decoder_state_dict) + else: + decoder.load_state_dict(decoder_state_dict) + + # VQGAN from Wuerstchen-V2 + vqmodel = PaellaVQModel.from_pretrained("warp-ai/wuerstchen", subfolder="vqgan") + + # Decoder pipeline + decoder_pipeline = StableCascadeDecoderPipeline( + decoder=decoder, text_encoder=text_encoder, tokenizer=tokenizer, vqgan=vqmodel, scheduler=scheduler + ) + decoder_pipeline.to(dtype).save_pretrained( + args.decoder_output_path, push_to_hub=args.push_to_hub, variant=args.variant + ) + +if args.save_combined: + # Stable Cascade combined pipeline + stable_cascade_pipeline = StableCascadeCombinedPipeline( + # Decoder + text_encoder=text_encoder, + tokenizer=tokenizer, + decoder=decoder, + scheduler=scheduler, + vqgan=vqmodel, + # Prior + prior_text_encoder=text_encoder, + prior_tokenizer=tokenizer, + prior_prior=prior_model, + prior_scheduler=scheduler, + prior_image_encoder=image_encoder, + prior_feature_extractor=feature_extractor, + ) + stable_cascade_pipeline.to(dtype).save_pretrained( + args.combined_output_path, push_to_hub=args.push_to_hub, variant=args.variant + ) diff --git a/diffusers-0.27.0/scripts/convert_stable_diffusion_checkpoint_to_onnx.py b/diffusers-0.27.0/scripts/convert_stable_diffusion_checkpoint_to_onnx.py new file mode 100755 index 0000000..96546b6 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_stable_diffusion_checkpoint_to_onnx.py @@ -0,0 +1,265 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import shutil +from pathlib import Path + +import onnx +import torch +from packaging import version +from torch.onnx import export + +from diffusers import OnnxRuntimeModel, OnnxStableDiffusionPipeline, StableDiffusionPipeline + + +is_torch_less_than_1_11 = version.parse(version.parse(torch.__version__).base_version) < version.parse("1.11") + + +def onnx_export( + model, + model_args: tuple, + output_path: Path, + ordered_input_names, + output_names, + dynamic_axes, + opset, + use_external_data_format=False, +): + output_path.parent.mkdir(parents=True, exist_ok=True) + # PyTorch deprecated the `enable_onnx_checker` and `use_external_data_format` arguments in v1.11, + # so we check the torch version for backwards compatibility + if is_torch_less_than_1_11: + export( + model, + model_args, + f=output_path.as_posix(), + input_names=ordered_input_names, + output_names=output_names, + dynamic_axes=dynamic_axes, + do_constant_folding=True, + use_external_data_format=use_external_data_format, + enable_onnx_checker=True, + opset_version=opset, + ) + else: + export( + model, + model_args, + f=output_path.as_posix(), + input_names=ordered_input_names, + output_names=output_names, + dynamic_axes=dynamic_axes, + do_constant_folding=True, + opset_version=opset, + ) + + +@torch.no_grad() +def convert_models(model_path: str, output_path: str, opset: int, fp16: bool = False): + dtype = torch.float16 if fp16 else torch.float32 + if fp16 and torch.cuda.is_available(): + device = "cuda" + elif fp16 and not torch.cuda.is_available(): + raise ValueError("`float16` model export is only supported on GPUs with CUDA") + else: + device = "cpu" + pipeline = StableDiffusionPipeline.from_pretrained(model_path, torch_dtype=dtype).to(device) + output_path = Path(output_path) + + # TEXT ENCODER + num_tokens = pipeline.text_encoder.config.max_position_embeddings + text_hidden_size = pipeline.text_encoder.config.hidden_size + text_input = pipeline.tokenizer( + "A sample prompt", + padding="max_length", + max_length=pipeline.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + onnx_export( + pipeline.text_encoder, + # casting to torch.int32 until the CLIP fix is released: https://github.com/huggingface/transformers/pull/18515/files + model_args=(text_input.input_ids.to(device=device, dtype=torch.int32)), + output_path=output_path / "text_encoder" / "model.onnx", + ordered_input_names=["input_ids"], + output_names=["last_hidden_state", "pooler_output"], + dynamic_axes={ + "input_ids": {0: "batch", 1: "sequence"}, + }, + opset=opset, + ) + del pipeline.text_encoder + + # UNET + unet_in_channels = pipeline.unet.config.in_channels + unet_sample_size = pipeline.unet.config.sample_size + unet_path = output_path / "unet" / "model.onnx" + onnx_export( + pipeline.unet, + model_args=( + torch.randn(2, unet_in_channels, unet_sample_size, unet_sample_size).to(device=device, dtype=dtype), + torch.randn(2).to(device=device, dtype=dtype), + torch.randn(2, num_tokens, text_hidden_size).to(device=device, dtype=dtype), + False, + ), + output_path=unet_path, + ordered_input_names=["sample", "timestep", "encoder_hidden_states", "return_dict"], + output_names=["out_sample"], # has to be different from "sample" for correct tracing + dynamic_axes={ + "sample": {0: "batch", 1: "channels", 2: "height", 3: "width"}, + "timestep": {0: "batch"}, + "encoder_hidden_states": {0: "batch", 1: "sequence"}, + }, + opset=opset, + use_external_data_format=True, # UNet is > 2GB, so the weights need to be split + ) + unet_model_path = str(unet_path.absolute().as_posix()) + unet_dir = os.path.dirname(unet_model_path) + unet = onnx.load(unet_model_path) + # clean up existing tensor files + shutil.rmtree(unet_dir) + os.mkdir(unet_dir) + # collate external tensor files into one + onnx.save_model( + unet, + unet_model_path, + save_as_external_data=True, + all_tensors_to_one_file=True, + location="weights.pb", + convert_attribute=False, + ) + del pipeline.unet + + # VAE ENCODER + vae_encoder = pipeline.vae + vae_in_channels = vae_encoder.config.in_channels + vae_sample_size = vae_encoder.config.sample_size + # need to get the raw tensor output (sample) from the encoder + vae_encoder.forward = lambda sample, return_dict: vae_encoder.encode(sample, return_dict)[0].sample() + onnx_export( + vae_encoder, + model_args=( + torch.randn(1, vae_in_channels, vae_sample_size, vae_sample_size).to(device=device, dtype=dtype), + False, + ), + output_path=output_path / "vae_encoder" / "model.onnx", + ordered_input_names=["sample", "return_dict"], + output_names=["latent_sample"], + dynamic_axes={ + "sample": {0: "batch", 1: "channels", 2: "height", 3: "width"}, + }, + opset=opset, + ) + + # VAE DECODER + vae_decoder = pipeline.vae + vae_latent_channels = vae_decoder.config.latent_channels + vae_out_channels = vae_decoder.config.out_channels + # forward only through the decoder part + vae_decoder.forward = vae_encoder.decode + onnx_export( + vae_decoder, + model_args=( + torch.randn(1, vae_latent_channels, unet_sample_size, unet_sample_size).to(device=device, dtype=dtype), + False, + ), + output_path=output_path / "vae_decoder" / "model.onnx", + ordered_input_names=["latent_sample", "return_dict"], + output_names=["sample"], + dynamic_axes={ + "latent_sample": {0: "batch", 1: "channels", 2: "height", 3: "width"}, + }, + opset=opset, + ) + del pipeline.vae + + # SAFETY CHECKER + if pipeline.safety_checker is not None: + safety_checker = pipeline.safety_checker + clip_num_channels = safety_checker.config.vision_config.num_channels + clip_image_size = safety_checker.config.vision_config.image_size + safety_checker.forward = safety_checker.forward_onnx + onnx_export( + pipeline.safety_checker, + model_args=( + torch.randn( + 1, + clip_num_channels, + clip_image_size, + clip_image_size, + ).to(device=device, dtype=dtype), + torch.randn(1, vae_sample_size, vae_sample_size, vae_out_channels).to(device=device, dtype=dtype), + ), + output_path=output_path / "safety_checker" / "model.onnx", + ordered_input_names=["clip_input", "images"], + output_names=["out_images", "has_nsfw_concepts"], + dynamic_axes={ + "clip_input": {0: "batch", 1: "channels", 2: "height", 3: "width"}, + "images": {0: "batch", 1: "height", 2: "width", 3: "channels"}, + }, + opset=opset, + ) + del pipeline.safety_checker + safety_checker = OnnxRuntimeModel.from_pretrained(output_path / "safety_checker") + feature_extractor = pipeline.feature_extractor + else: + safety_checker = None + feature_extractor = None + + onnx_pipeline = OnnxStableDiffusionPipeline( + vae_encoder=OnnxRuntimeModel.from_pretrained(output_path / "vae_encoder"), + vae_decoder=OnnxRuntimeModel.from_pretrained(output_path / "vae_decoder"), + text_encoder=OnnxRuntimeModel.from_pretrained(output_path / "text_encoder"), + tokenizer=pipeline.tokenizer, + unet=OnnxRuntimeModel.from_pretrained(output_path / "unet"), + scheduler=pipeline.scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + requires_safety_checker=safety_checker is not None, + ) + + onnx_pipeline.save_pretrained(output_path) + print("ONNX pipeline saved to", output_path) + + del pipeline + del onnx_pipeline + _ = OnnxStableDiffusionPipeline.from_pretrained(output_path, provider="CPUExecutionProvider") + print("ONNX pipeline is loadable") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--model_path", + type=str, + required=True, + help="Path to the `diffusers` checkpoint to convert (either a local directory or on the Hub).", + ) + + parser.add_argument("--output_path", type=str, required=True, help="Path to the output model.") + + parser.add_argument( + "--opset", + default=14, + type=int, + help="The version of the ONNX operator set to use.", + ) + parser.add_argument("--fp16", action="store_true", default=False, help="Export the models in `float16` mode") + + args = parser.parse_args() + + convert_models(args.model_path, args.output_path, args.opset, args.fp16) diff --git a/diffusers-0.27.0/scripts/convert_stable_diffusion_controlnet_to_onnx.py b/diffusers-0.27.0/scripts/convert_stable_diffusion_controlnet_to_onnx.py new file mode 100755 index 0000000..4af39b2 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_stable_diffusion_controlnet_to_onnx.py @@ -0,0 +1,505 @@ +import argparse +import os +import shutil +from pathlib import Path + +import onnx +import onnx_graphsurgeon as gs +import torch +from onnx import shape_inference +from packaging import version +from polygraphy.backend.onnx.loader import fold_constants +from torch.onnx import export + +from diffusers import ( + ControlNetModel, + StableDiffusionControlNetImg2ImgPipeline, +) +from diffusers.models.attention_processor import AttnProcessor +from diffusers.pipelines.controlnet.pipeline_controlnet_sd_xl import StableDiffusionXLControlNetPipeline + + +is_torch_less_than_1_11 = version.parse(version.parse(torch.__version__).base_version) < version.parse("1.11") +is_torch_2_0_1 = version.parse(version.parse(torch.__version__).base_version) == version.parse("2.0.1") + + +class Optimizer: + def __init__(self, onnx_graph, verbose=False): + self.graph = gs.import_onnx(onnx_graph) + self.verbose = verbose + + def info(self, prefix): + if self.verbose: + print( + f"{prefix} .. {len(self.graph.nodes)} nodes, {len(self.graph.tensors().keys())} tensors, {len(self.graph.inputs)} inputs, {len(self.graph.outputs)} outputs" + ) + + def cleanup(self, return_onnx=False): + self.graph.cleanup().toposort() + if return_onnx: + return gs.export_onnx(self.graph) + + def select_outputs(self, keep, names=None): + self.graph.outputs = [self.graph.outputs[o] for o in keep] + if names: + for i, name in enumerate(names): + self.graph.outputs[i].name = name + + def fold_constants(self, return_onnx=False): + onnx_graph = fold_constants(gs.export_onnx(self.graph), allow_onnxruntime_shape_inference=True) + self.graph = gs.import_onnx(onnx_graph) + if return_onnx: + return onnx_graph + + def infer_shapes(self, return_onnx=False): + onnx_graph = gs.export_onnx(self.graph) + if onnx_graph.ByteSize() > 2147483648: + raise TypeError("ERROR: model size exceeds supported 2GB limit") + else: + onnx_graph = shape_inference.infer_shapes(onnx_graph) + + self.graph = gs.import_onnx(onnx_graph) + if return_onnx: + return onnx_graph + + +def optimize(onnx_graph, name, verbose): + opt = Optimizer(onnx_graph, verbose=verbose) + opt.info(name + ": original") + opt.cleanup() + opt.info(name + ": cleanup") + opt.fold_constants() + opt.info(name + ": fold constants") + # opt.infer_shapes() + # opt.info(name + ': shape inference') + onnx_opt_graph = opt.cleanup(return_onnx=True) + opt.info(name + ": finished") + return onnx_opt_graph + + +class UNet2DConditionControlNetModel(torch.nn.Module): + def __init__( + self, + unet, + controlnets: ControlNetModel, + ): + super().__init__() + self.unet = unet + self.controlnets = controlnets + + def forward( + self, + sample, + timestep, + encoder_hidden_states, + controlnet_conds, + controlnet_scales, + ): + for i, (controlnet_cond, conditioning_scale, controlnet) in enumerate( + zip(controlnet_conds, controlnet_scales, self.controlnets) + ): + down_samples, mid_sample = controlnet( + sample, + timestep, + encoder_hidden_states=encoder_hidden_states, + controlnet_cond=controlnet_cond, + conditioning_scale=conditioning_scale, + return_dict=False, + ) + + # merge samples + if i == 0: + down_block_res_samples, mid_block_res_sample = down_samples, mid_sample + else: + down_block_res_samples = [ + samples_prev + samples_curr + for samples_prev, samples_curr in zip(down_block_res_samples, down_samples) + ] + mid_block_res_sample += mid_sample + + noise_pred = self.unet( + sample, + timestep, + encoder_hidden_states=encoder_hidden_states, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + return_dict=False, + )[0] + return noise_pred + + +class UNet2DConditionXLControlNetModel(torch.nn.Module): + def __init__( + self, + unet, + controlnets: ControlNetModel, + ): + super().__init__() + self.unet = unet + self.controlnets = controlnets + + def forward( + self, + sample, + timestep, + encoder_hidden_states, + controlnet_conds, + controlnet_scales, + text_embeds, + time_ids, + ): + added_cond_kwargs = {"text_embeds": text_embeds, "time_ids": time_ids} + for i, (controlnet_cond, conditioning_scale, controlnet) in enumerate( + zip(controlnet_conds, controlnet_scales, self.controlnets) + ): + down_samples, mid_sample = controlnet( + sample, + timestep, + encoder_hidden_states=encoder_hidden_states, + controlnet_cond=controlnet_cond, + conditioning_scale=conditioning_scale, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + ) + + # merge samples + if i == 0: + down_block_res_samples, mid_block_res_sample = down_samples, mid_sample + else: + down_block_res_samples = [ + samples_prev + samples_curr + for samples_prev, samples_curr in zip(down_block_res_samples, down_samples) + ] + mid_block_res_sample += mid_sample + + noise_pred = self.unet( + sample, + timestep, + encoder_hidden_states=encoder_hidden_states, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + return noise_pred + + +def onnx_export( + model, + model_args: tuple, + output_path: Path, + ordered_input_names, + output_names, + dynamic_axes, + opset, + use_external_data_format=False, +): + output_path.parent.mkdir(parents=True, exist_ok=True) + # PyTorch deprecated the `enable_onnx_checker` and `use_external_data_format` arguments in v1.11, + # so we check the torch version for backwards compatibility + with torch.inference_mode(), torch.autocast("cuda"): + if is_torch_less_than_1_11: + export( + model, + model_args, + f=output_path.as_posix(), + input_names=ordered_input_names, + output_names=output_names, + dynamic_axes=dynamic_axes, + do_constant_folding=True, + use_external_data_format=use_external_data_format, + enable_onnx_checker=True, + opset_version=opset, + ) + else: + export( + model, + model_args, + f=output_path.as_posix(), + input_names=ordered_input_names, + output_names=output_names, + dynamic_axes=dynamic_axes, + do_constant_folding=True, + opset_version=opset, + ) + + +@torch.no_grad() +def convert_models( + model_path: str, controlnet_path: list, output_path: str, opset: int, fp16: bool = False, sd_xl: bool = False +): + """ + Function to convert models in stable diffusion controlnet pipeline into ONNX format + + Example: + python convert_stable_diffusion_controlnet_to_onnx.py + --model_path danbrown/RevAnimated-v1-2-2 + --controlnet_path lllyasviel/control_v11f1e_sd15_tile ioclab/brightness-controlnet + --output_path path-to-models-stable_diffusion/RevAnimated-v1-2-2 + --fp16 + + Example for SD XL: + python convert_stable_diffusion_controlnet_to_onnx.py + --model_path stabilityai/stable-diffusion-xl-base-1.0 + --controlnet_path SargeZT/sdxl-controlnet-seg + --output_path path-to-models-stable_diffusion/stable-diffusion-xl-base-1.0 + --fp16 + --sd_xl + + Returns: + create 4 onnx models in output path + text_encoder/model.onnx + unet/model.onnx + unet/weights.pb + vae_encoder/model.onnx + vae_decoder/model.onnx + + run test script in diffusers/examples/community + python test_onnx_controlnet.py + --sd_model danbrown/RevAnimated-v1-2-2 + --onnx_model_dir path-to-models-stable_diffusion/RevAnimated-v1-2-2 + --qr_img_path path-to-qr-code-image + """ + dtype = torch.float16 if fp16 else torch.float32 + if fp16 and torch.cuda.is_available(): + device = "cuda" + elif fp16 and not torch.cuda.is_available(): + raise ValueError("`float16` model export is only supported on GPUs with CUDA") + else: + device = "cpu" + + # init controlnet + controlnets = [] + for path in controlnet_path: + controlnet = ControlNetModel.from_pretrained(path, torch_dtype=dtype).to(device) + if is_torch_2_0_1: + controlnet.set_attn_processor(AttnProcessor()) + controlnets.append(controlnet) + + if sd_xl: + if len(controlnets) == 1: + controlnet = controlnets[0] + else: + raise ValueError("MultiControlNet is not yet supported.") + pipeline = StableDiffusionXLControlNetPipeline.from_pretrained( + model_path, controlnet=controlnet, torch_dtype=dtype, variant="fp16", use_safetensors=True + ).to(device) + else: + pipeline = StableDiffusionControlNetImg2ImgPipeline.from_pretrained( + model_path, controlnet=controlnets, torch_dtype=dtype + ).to(device) + + output_path = Path(output_path) + if is_torch_2_0_1: + pipeline.unet.set_attn_processor(AttnProcessor()) + pipeline.vae.set_attn_processor(AttnProcessor()) + + # # TEXT ENCODER + num_tokens = pipeline.text_encoder.config.max_position_embeddings + text_hidden_size = pipeline.text_encoder.config.hidden_size + text_input = pipeline.tokenizer( + "A sample prompt", + padding="max_length", + max_length=pipeline.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + onnx_export( + pipeline.text_encoder, + # casting to torch.int32 until the CLIP fix is released: https://github.com/huggingface/transformers/pull/18515/files + model_args=(text_input.input_ids.to(device=device, dtype=torch.int32)), + output_path=output_path / "text_encoder" / "model.onnx", + ordered_input_names=["input_ids"], + output_names=["last_hidden_state", "pooler_output"], + dynamic_axes={ + "input_ids": {0: "batch", 1: "sequence"}, + }, + opset=opset, + ) + del pipeline.text_encoder + + # # UNET + if sd_xl: + controlnets = torch.nn.ModuleList(controlnets) + unet_controlnet = UNet2DConditionXLControlNetModel(pipeline.unet, controlnets) + unet_in_channels = pipeline.unet.config.in_channels + unet_sample_size = pipeline.unet.config.sample_size + text_hidden_size = 2048 + img_size = 8 * unet_sample_size + unet_path = output_path / "unet" / "model.onnx" + + onnx_export( + unet_controlnet, + model_args=( + torch.randn(2, unet_in_channels, unet_sample_size, unet_sample_size).to(device=device, dtype=dtype), + torch.tensor([1.0]).to(device=device, dtype=dtype), + torch.randn(2, num_tokens, text_hidden_size).to(device=device, dtype=dtype), + torch.randn(len(controlnets), 2, 3, img_size, img_size).to(device=device, dtype=dtype), + torch.randn(len(controlnets), 1).to(device=device, dtype=dtype), + torch.randn(2, 1280).to(device=device, dtype=dtype), + torch.rand(2, 6).to(device=device, dtype=dtype), + ), + output_path=unet_path, + ordered_input_names=[ + "sample", + "timestep", + "encoder_hidden_states", + "controlnet_conds", + "conditioning_scales", + "text_embeds", + "time_ids", + ], + output_names=["noise_pred"], # has to be different from "sample" for correct tracing + dynamic_axes={ + "sample": {0: "2B", 2: "H", 3: "W"}, + "encoder_hidden_states": {0: "2B"}, + "controlnet_conds": {1: "2B", 3: "8H", 4: "8W"}, + "text_embeds": {0: "2B"}, + "time_ids": {0: "2B"}, + }, + opset=opset, + use_external_data_format=True, # UNet is > 2GB, so the weights need to be split + ) + unet_model_path = str(unet_path.absolute().as_posix()) + unet_dir = os.path.dirname(unet_model_path) + # optimize onnx + shape_inference.infer_shapes_path(unet_model_path, unet_model_path) + unet_opt_graph = optimize(onnx.load(unet_model_path), name="Unet", verbose=True) + # clean up existing tensor files + shutil.rmtree(unet_dir) + os.mkdir(unet_dir) + # collate external tensor files into one + onnx.save_model( + unet_opt_graph, + unet_model_path, + save_as_external_data=True, + all_tensors_to_one_file=True, + location="weights.pb", + convert_attribute=False, + ) + del pipeline.unet + else: + controlnets = torch.nn.ModuleList(controlnets) + unet_controlnet = UNet2DConditionControlNetModel(pipeline.unet, controlnets) + unet_in_channels = pipeline.unet.config.in_channels + unet_sample_size = pipeline.unet.config.sample_size + img_size = 8 * unet_sample_size + unet_path = output_path / "unet" / "model.onnx" + + onnx_export( + unet_controlnet, + model_args=( + torch.randn(2, unet_in_channels, unet_sample_size, unet_sample_size).to(device=device, dtype=dtype), + torch.tensor([1.0]).to(device=device, dtype=dtype), + torch.randn(2, num_tokens, text_hidden_size).to(device=device, dtype=dtype), + torch.randn(len(controlnets), 2, 3, img_size, img_size).to(device=device, dtype=dtype), + torch.randn(len(controlnets), 1).to(device=device, dtype=dtype), + ), + output_path=unet_path, + ordered_input_names=[ + "sample", + "timestep", + "encoder_hidden_states", + "controlnet_conds", + "conditioning_scales", + ], + output_names=["noise_pred"], # has to be different from "sample" for correct tracing + dynamic_axes={ + "sample": {0: "2B", 2: "H", 3: "W"}, + "encoder_hidden_states": {0: "2B"}, + "controlnet_conds": {1: "2B", 3: "8H", 4: "8W"}, + }, + opset=opset, + use_external_data_format=True, # UNet is > 2GB, so the weights need to be split + ) + unet_model_path = str(unet_path.absolute().as_posix()) + unet_dir = os.path.dirname(unet_model_path) + # optimize onnx + shape_inference.infer_shapes_path(unet_model_path, unet_model_path) + unet_opt_graph = optimize(onnx.load(unet_model_path), name="Unet", verbose=True) + # clean up existing tensor files + shutil.rmtree(unet_dir) + os.mkdir(unet_dir) + # collate external tensor files into one + onnx.save_model( + unet_opt_graph, + unet_model_path, + save_as_external_data=True, + all_tensors_to_one_file=True, + location="weights.pb", + convert_attribute=False, + ) + del pipeline.unet + + # VAE ENCODER + vae_encoder = pipeline.vae + vae_in_channels = vae_encoder.config.in_channels + vae_sample_size = vae_encoder.config.sample_size + # need to get the raw tensor output (sample) from the encoder + vae_encoder.forward = lambda sample: vae_encoder.encode(sample).latent_dist.sample() + onnx_export( + vae_encoder, + model_args=(torch.randn(1, vae_in_channels, vae_sample_size, vae_sample_size).to(device=device, dtype=dtype),), + output_path=output_path / "vae_encoder" / "model.onnx", + ordered_input_names=["sample"], + output_names=["latent_sample"], + dynamic_axes={ + "sample": {0: "batch", 1: "channels", 2: "height", 3: "width"}, + }, + opset=opset, + ) + + # VAE DECODER + vae_decoder = pipeline.vae + vae_latent_channels = vae_decoder.config.latent_channels + # forward only through the decoder part + vae_decoder.forward = vae_encoder.decode + onnx_export( + vae_decoder, + model_args=( + torch.randn(1, vae_latent_channels, unet_sample_size, unet_sample_size).to(device=device, dtype=dtype), + ), + output_path=output_path / "vae_decoder" / "model.onnx", + ordered_input_names=["latent_sample"], + output_names=["sample"], + dynamic_axes={ + "latent_sample": {0: "batch", 1: "channels", 2: "height", 3: "width"}, + }, + opset=opset, + ) + del pipeline.vae + + del pipeline + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--sd_xl", action="store_true", default=False, help="SD XL pipeline") + + parser.add_argument( + "--model_path", + type=str, + required=True, + help="Path to the `diffusers` checkpoint to convert (either a local directory or on the Hub).", + ) + + parser.add_argument( + "--controlnet_path", + nargs="+", + required=True, + help="Path to the `controlnet` checkpoint to convert (either a local directory or on the Hub).", + ) + + parser.add_argument("--output_path", type=str, required=True, help="Path to the output model.") + + parser.add_argument( + "--opset", + default=14, + type=int, + help="The version of the ONNX operator set to use.", + ) + parser.add_argument("--fp16", action="store_true", default=False, help="Export the models in `float16` mode") + + args = parser.parse_args() + + convert_models(args.model_path, args.controlnet_path, args.output_path, args.opset, args.fp16, args.sd_xl) diff --git a/diffusers-0.27.0/scripts/convert_stable_diffusion_controlnet_to_tensorrt.py b/diffusers-0.27.0/scripts/convert_stable_diffusion_controlnet_to_tensorrt.py new file mode 100755 index 0000000..52ab02c --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_stable_diffusion_controlnet_to_tensorrt.py @@ -0,0 +1,121 @@ +import argparse +import sys + +import tensorrt as trt + + +def convert_models(onnx_path: str, num_controlnet: int, output_path: str, fp16: bool = False, sd_xl: bool = False): + """ + Function to convert models in stable diffusion controlnet pipeline into TensorRT format + + Example: + python convert_stable_diffusion_controlnet_to_tensorrt.py + --onnx_path path-to-models-stable_diffusion/RevAnimated-v1-2-2/unet/model.onnx + --output_path path-to-models-stable_diffusion/RevAnimated-v1-2-2/unet/model.engine + --fp16 + --num_controlnet 2 + + Example for SD XL: + python convert_stable_diffusion_controlnet_to_tensorrt.py + --onnx_path path-to-models-stable_diffusion/stable-diffusion-xl-base-1.0/unet/model.onnx + --output_path path-to-models-stable_diffusion/stable-diffusion-xl-base-1.0/unet/model.engine + --fp16 + --num_controlnet 1 + --sd_xl + + Returns: + unet/model.engine + + run test script in diffusers/examples/community + python test_onnx_controlnet.py + --sd_model danbrown/RevAnimated-v1-2-2 + --onnx_model_dir path-to-models-stable_diffusion/RevAnimated-v1-2-2 + --unet_engine_path path-to-models-stable_diffusion/stable-diffusion-xl-base-1.0/unet/model.engine + --qr_img_path path-to-qr-code-image + """ + # UNET + if sd_xl: + batch_size = 1 + unet_in_channels = 4 + unet_sample_size = 64 + num_tokens = 77 + text_hidden_size = 2048 + img_size = 512 + + text_embeds_shape = (2 * batch_size, 1280) + time_ids_shape = (2 * batch_size, 6) + else: + batch_size = 1 + unet_in_channels = 4 + unet_sample_size = 64 + num_tokens = 77 + text_hidden_size = 768 + img_size = 512 + batch_size = 1 + + latents_shape = (2 * batch_size, unet_in_channels, unet_sample_size, unet_sample_size) + embed_shape = (2 * batch_size, num_tokens, text_hidden_size) + controlnet_conds_shape = (num_controlnet, 2 * batch_size, 3, img_size, img_size) + + TRT_LOGGER = trt.Logger(trt.Logger.VERBOSE) + TRT_BUILDER = trt.Builder(TRT_LOGGER) + TRT_RUNTIME = trt.Runtime(TRT_LOGGER) + + network = TRT_BUILDER.create_network(1 << int(trt.NetworkDefinitionCreationFlag.EXPLICIT_BATCH)) + onnx_parser = trt.OnnxParser(network, TRT_LOGGER) + + parse_success = onnx_parser.parse_from_file(onnx_path) + for idx in range(onnx_parser.num_errors): + print(onnx_parser.get_error(idx)) + if not parse_success: + sys.exit("ONNX model parsing failed") + print("Load Onnx model done") + + profile = TRT_BUILDER.create_optimization_profile() + + profile.set_shape("sample", latents_shape, latents_shape, latents_shape) + profile.set_shape("encoder_hidden_states", embed_shape, embed_shape, embed_shape) + profile.set_shape("controlnet_conds", controlnet_conds_shape, controlnet_conds_shape, controlnet_conds_shape) + if sd_xl: + profile.set_shape("text_embeds", text_embeds_shape, text_embeds_shape, text_embeds_shape) + profile.set_shape("time_ids", time_ids_shape, time_ids_shape, time_ids_shape) + + config = TRT_BUILDER.create_builder_config() + config.add_optimization_profile(profile) + config.set_preview_feature(trt.PreviewFeature.DISABLE_EXTERNAL_TACTIC_SOURCES_FOR_CORE_0805, True) + if fp16: + config.set_flag(trt.BuilderFlag.FP16) + + plan = TRT_BUILDER.build_serialized_network(network, config) + if plan is None: + sys.exit("Failed building engine") + print("Succeeded building engine") + + engine = TRT_RUNTIME.deserialize_cuda_engine(plan) + + ## save TRT engine + with open(output_path, "wb") as f: + f.write(engine.serialize()) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--sd_xl", action="store_true", default=False, help="SD XL pipeline") + + parser.add_argument( + "--onnx_path", + type=str, + required=True, + help="Path to the onnx checkpoint to convert", + ) + + parser.add_argument("--num_controlnet", type=int) + + parser.add_argument("--output_path", type=str, required=True, help="Path to the output model.") + + parser.add_argument("--fp16", action="store_true", default=False, help="Export the models in `float16` mode") + + args = parser.parse_args() + + convert_models(args.onnx_path, args.num_controlnet, args.output_path, args.fp16, args.sd_xl) diff --git a/diffusers-0.27.0/scripts/convert_svd_to_diffusers.py b/diffusers-0.27.0/scripts/convert_svd_to_diffusers.py new file mode 100755 index 0000000..3243ce2 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_svd_to_diffusers.py @@ -0,0 +1,730 @@ +from diffusers.utils import is_accelerate_available, logging + + +if is_accelerate_available(): + pass + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def create_unet_diffusers_config(original_config, image_size: int, controlnet=False): + """ + Creates a config for the diffusers based on the config of the LDM model. + """ + if controlnet: + unet_params = original_config.model.params.control_stage_config.params + else: + if "unet_config" in original_config.model.params and original_config.model.params.unet_config is not None: + unet_params = original_config.model.params.unet_config.params + else: + unet_params = original_config.model.params.network_config.params + + vae_params = original_config.model.params.first_stage_config.params.encoder_config.params + + block_out_channels = [unet_params.model_channels * mult for mult in unet_params.channel_mult] + + down_block_types = [] + resolution = 1 + for i in range(len(block_out_channels)): + block_type = ( + "CrossAttnDownBlockSpatioTemporal" + if resolution in unet_params.attention_resolutions + else "DownBlockSpatioTemporal" + ) + down_block_types.append(block_type) + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + block_type = ( + "CrossAttnUpBlockSpatioTemporal" + if resolution in unet_params.attention_resolutions + else "UpBlockSpatioTemporal" + ) + up_block_types.append(block_type) + resolution //= 2 + + if unet_params.transformer_depth is not None: + transformer_layers_per_block = ( + unet_params.transformer_depth + if isinstance(unet_params.transformer_depth, int) + else list(unet_params.transformer_depth) + ) + else: + transformer_layers_per_block = 1 + + vae_scale_factor = 2 ** (len(vae_params.ch_mult) - 1) + + head_dim = unet_params.num_heads if "num_heads" in unet_params else None + use_linear_projection = ( + unet_params.use_linear_in_transformer if "use_linear_in_transformer" in unet_params else False + ) + if use_linear_projection: + # stable diffusion 2-base-512 and 2-768 + if head_dim is None: + head_dim_mult = unet_params.model_channels // unet_params.num_head_channels + head_dim = [head_dim_mult * c for c in list(unet_params.channel_mult)] + + class_embed_type = None + addition_embed_type = None + addition_time_embed_dim = None + projection_class_embeddings_input_dim = None + context_dim = None + + if unet_params.context_dim is not None: + context_dim = ( + unet_params.context_dim if isinstance(unet_params.context_dim, int) else unet_params.context_dim[0] + ) + + if "num_classes" in unet_params: + if unet_params.num_classes == "sequential": + addition_time_embed_dim = 256 + assert "adm_in_channels" in unet_params + projection_class_embeddings_input_dim = unet_params.adm_in_channels + + config = { + "sample_size": image_size // vae_scale_factor, + "in_channels": unet_params.in_channels, + "down_block_types": tuple(down_block_types), + "block_out_channels": tuple(block_out_channels), + "layers_per_block": unet_params.num_res_blocks, + "cross_attention_dim": context_dim, + "attention_head_dim": head_dim, + "use_linear_projection": use_linear_projection, + "class_embed_type": class_embed_type, + "addition_embed_type": addition_embed_type, + "addition_time_embed_dim": addition_time_embed_dim, + "projection_class_embeddings_input_dim": projection_class_embeddings_input_dim, + "transformer_layers_per_block": transformer_layers_per_block, + } + + if "disable_self_attentions" in unet_params: + config["only_cross_attention"] = unet_params.disable_self_attentions + + if "num_classes" in unet_params and isinstance(unet_params.num_classes, int): + config["num_class_embeds"] = unet_params.num_classes + + if controlnet: + config["conditioning_channels"] = unet_params.hint_channels + else: + config["out_channels"] = unet_params.out_channels + config["up_block_types"] = tuple(up_block_types) + + return config + + +def assign_to_checkpoint( + paths, + checkpoint, + old_checkpoint, + attention_paths_to_split=None, + additional_replacements=None, + config=None, + mid_block_suffix="", +): + """ + This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits + attention layers, and takes into account additional replacements that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + # Splits the attention layers into three variables. + if attention_paths_to_split is not None: + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape) + checkpoint[path_map["key"]] = key.reshape(target_shape) + checkpoint[path_map["value"]] = value.reshape(target_shape) + + if mid_block_suffix is not None: + mid_block_suffix = f".{mid_block_suffix}" + else: + mid_block_suffix = "" + + for path in paths: + new_path = path["new"] + + # These have already been assigned + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + # Global renaming happens here + new_path = new_path.replace("middle_block.0", f"mid_block.resnets.0{mid_block_suffix}") + new_path = new_path.replace("middle_block.1", "mid_block.attentions.0") + new_path = new_path.replace("middle_block.2", f"mid_block.resnets.1{mid_block_suffix}") + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + if new_path == "mid_block.resnets.0.spatial_res_block.norm1.weight": + print("yeyy") + + # proj_attn.weight has to be converted from conv 1D to linear + is_attn_weight = "proj_attn.weight" in new_path or ("attentions" in new_path and "to_" in new_path) + shape = old_checkpoint[path["old"]].shape + if is_attn_weight and len(shape) == 3: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0] + elif is_attn_weight and len(shape) == 4: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0, 0] + else: + checkpoint[new_path] = old_checkpoint[path["old"]] + + +def renew_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + # new_item = new_item.replace('norm.weight', 'group_norm.weight') + # new_item = new_item.replace('norm.bias', 'group_norm.bias') + + # new_item = new_item.replace('proj_out.weight', 'proj_attn.weight') + # new_item = new_item.replace('proj_out.bias', 'proj_attn.bias') + + # new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + new_item = new_item.replace("time_stack", "temporal_transformer_blocks") + + new_item = new_item.replace("time_pos_embed.0.bias", "time_pos_embed.linear_1.bias") + new_item = new_item.replace("time_pos_embed.0.weight", "time_pos_embed.linear_1.weight") + new_item = new_item.replace("time_pos_embed.2.bias", "time_pos_embed.linear_2.bias") + new_item = new_item.replace("time_pos_embed.2.weight", "time_pos_embed.linear_2.weight") + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("emb_layers.1", "time_emb_proj") + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = new_item.replace("time_stack.", "") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def convert_ldm_unet_checkpoint( + checkpoint, config, path=None, extract_ema=False, controlnet=False, skip_extract_state_dict=False +): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + + if skip_extract_state_dict: + unet_state_dict = checkpoint + else: + # extract state_dict for UNet + unet_state_dict = {} + keys = list(checkpoint.keys()) + + unet_key = "model.diffusion_model." + + # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA + if sum(k.startswith("model_ema") for k in keys) > 100 and extract_ema: + logger.warning(f"Checkpoint {path} has both EMA and non-EMA weights.") + logger.warning( + "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA" + " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag." + ) + for key in keys: + if key.startswith("model.diffusion_model"): + flat_ema_key = "model_ema." + "".join(key.split(".")[1:]) + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(flat_ema_key) + else: + if sum(k.startswith("model_ema") for k in keys) > 100: + logger.warning( + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA" + " weights (usually better for inference), please make sure to add the `--extract_ema` flag." + ) + + for key in keys: + if key.startswith(unet_key): + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key) + + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"] + + if config["class_embed_type"] is None: + # No parameters to port + ... + elif config["class_embed_type"] == "timestep" or config["class_embed_type"] == "projection": + new_checkpoint["class_embedding.linear_1.weight"] = unet_state_dict["label_emb.0.0.weight"] + new_checkpoint["class_embedding.linear_1.bias"] = unet_state_dict["label_emb.0.0.bias"] + new_checkpoint["class_embedding.linear_2.weight"] = unet_state_dict["label_emb.0.2.weight"] + new_checkpoint["class_embedding.linear_2.bias"] = unet_state_dict["label_emb.0.2.bias"] + else: + raise NotImplementedError(f"Not implemented `class_embed_type`: {config['class_embed_type']}") + + # if config["addition_embed_type"] == "text_time": + new_checkpoint["add_embedding.linear_1.weight"] = unet_state_dict["label_emb.0.0.weight"] + new_checkpoint["add_embedding.linear_1.bias"] = unet_state_dict["label_emb.0.0.bias"] + new_checkpoint["add_embedding.linear_2.weight"] = unet_state_dict["label_emb.0.2.weight"] + new_checkpoint["add_embedding.linear_2.bias"] = unet_state_dict["label_emb.0.2.bias"] + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key] + for layer_id in range(num_output_blocks) + } + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + spatial_resnets = [ + key + for key in input_blocks[i] + if f"input_blocks.{i}.0" in key + and ( + f"input_blocks.{i}.0.op" not in key + and f"input_blocks.{i}.0.time_stack" not in key + and f"input_blocks.{i}.0.time_mixer" not in key + ) + ] + temporal_resnets = [key for key in input_blocks[i] if f"input_blocks.{i}.0.time_stack" in key] + # import ipdb; ipdb.set_trace() + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + + if f"input_blocks.{i}.0.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + + paths = renew_resnet_paths(spatial_resnets) + meta_path = { + "old": f"input_blocks.{i}.0", + "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}.spatial_res_block", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + paths = renew_resnet_paths(temporal_resnets) + meta_path = { + "old": f"input_blocks.{i}.0", + "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}.temporal_res_block", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + # TODO resnet time_mixer.mix_factor + if f"input_blocks.{i}.0.time_mixer.mix_factor" in unet_state_dict: + new_checkpoint[ + f"down_blocks.{block_id}.resnets.{layer_in_block_id}.time_mixer.mix_factor" + ] = unet_state_dict[f"input_blocks.{i}.0.time_mixer.mix_factor"] + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"} + # import ipdb; ipdb.set_trace() + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + resnet_0_spatial = [key for key in resnet_0 if "time_stack" not in key and "time_mixer" not in key] + resnet_0_paths = renew_resnet_paths(resnet_0_spatial) + # import ipdb; ipdb.set_trace() + assign_to_checkpoint( + resnet_0_paths, new_checkpoint, unet_state_dict, config=config, mid_block_suffix="spatial_res_block" + ) + + resnet_0_temporal = [key for key in resnet_0 if "time_stack" in key and "time_mixer" not in key] + resnet_0_paths = renew_resnet_paths(resnet_0_temporal) + assign_to_checkpoint( + resnet_0_paths, new_checkpoint, unet_state_dict, config=config, mid_block_suffix="temporal_res_block" + ) + + resnet_1_spatial = [key for key in resnet_1 if "time_stack" not in key and "time_mixer" not in key] + resnet_1_paths = renew_resnet_paths(resnet_1_spatial) + assign_to_checkpoint( + resnet_1_paths, new_checkpoint, unet_state_dict, config=config, mid_block_suffix="spatial_res_block" + ) + + resnet_1_temporal = [key for key in resnet_1 if "time_stack" in key and "time_mixer" not in key] + resnet_1_paths = renew_resnet_paths(resnet_1_temporal) + assign_to_checkpoint( + resnet_1_paths, new_checkpoint, unet_state_dict, config=config, mid_block_suffix="temporal_res_block" + ) + + new_checkpoint["mid_block.resnets.0.time_mixer.mix_factor"] = unet_state_dict[ + "middle_block.0.time_mixer.mix_factor" + ] + new_checkpoint["mid_block.resnets.1.time_mixer.mix_factor"] = unet_state_dict[ + "middle_block.2.time_mixer.mix_factor" + ] + + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + if len(output_block_list) > 1: + spatial_resnets = [ + key + for key in output_blocks[i] + if f"output_blocks.{i}.0" in key + and (f"output_blocks.{i}.0.time_stack" not in key and "time_mixer" not in key) + ] + # import ipdb; ipdb.set_trace() + + temporal_resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0.time_stack" in key] + + paths = renew_resnet_paths(spatial_resnets) + meta_path = { + "old": f"output_blocks.{i}.0", + "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}.spatial_res_block", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + paths = renew_resnet_paths(temporal_resnets) + meta_path = { + "old": f"output_blocks.{i}.0", + "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}.temporal_res_block", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if f"output_blocks.{i}.0.time_mixer.mix_factor" in unet_state_dict: + new_checkpoint[ + f"up_blocks.{block_id}.resnets.{layer_in_block_id}.time_mixer.mix_factor" + ] = unet_state_dict[f"output_blocks.{i}.0.time_mixer.mix_factor"] + + output_block_list = {k: sorted(v) for k, v in output_block_list.items()} + if ["conv.bias", "conv.weight"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.bias", "conv.weight"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key and "conv" not in key] + if len(attentions): + paths = renew_attention_paths(attentions) + # import ipdb; ipdb.set_trace() + meta_path = { + "old": f"output_blocks.{i}.1", + "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + else: + spatial_layers = [ + layer for layer in output_block_layers if "time_stack" not in layer and "time_mixer" not in layer + ] + resnet_0_paths = renew_resnet_paths(spatial_layers, n_shave_prefix_segments=1) + # import ipdb; ipdb.set_trace() + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join( + ["up_blocks", str(block_id), "resnets", str(layer_in_block_id), "spatial_res_block", path["new"]] + ) + + new_checkpoint[new_path] = unet_state_dict[old_path] + + temporal_layers = [ + layer for layer in output_block_layers if "time_stack" in layer and "time_mixer" not in key + ] + resnet_0_paths = renew_resnet_paths(temporal_layers, n_shave_prefix_segments=1) + # import ipdb; ipdb.set_trace() + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join( + ["up_blocks", str(block_id), "resnets", str(layer_in_block_id), "temporal_res_block", path["new"]] + ) + + new_checkpoint[new_path] = unet_state_dict[old_path] + + new_checkpoint["up_blocks.0.resnets.0.time_mixer.mix_factor"] = unet_state_dict[ + f"output_blocks.{str(i)}.0.time_mixer.mix_factor" + ] + + return new_checkpoint + + +def conv_attn_to_linear(checkpoint): + keys = list(checkpoint.keys()) + attn_keys = ["to_q.weight", "to_k.weight", "to_v.weight"] + for key in keys: + if ".".join(key.split(".")[-2:]) in attn_keys: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0, 0] + elif "proj_attn.weight" in key: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0] + + +def renew_vae_resnet_paths(old_list, n_shave_prefix_segments=0, is_temporal=False): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + # Temporal resnet + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = new_item.replace("time_stack.", "temporal_res_block.") + + # Spatial resnet + new_item = new_item.replace("conv1", "spatial_res_block.conv1") + new_item = new_item.replace("norm1", "spatial_res_block.norm1") + + new_item = new_item.replace("conv2", "spatial_res_block.conv2") + new_item = new_item.replace("norm2", "spatial_res_block.norm2") + + new_item = new_item.replace("nin_shortcut", "spatial_res_block.conv_shortcut") + + new_item = new_item.replace("mix_factor", "spatial_res_block.time_mixer.mix_factor") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_vae_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("norm.weight", "group_norm.weight") + new_item = new_item.replace("norm.bias", "group_norm.bias") + + new_item = new_item.replace("q.weight", "to_q.weight") + new_item = new_item.replace("q.bias", "to_q.bias") + + new_item = new_item.replace("k.weight", "to_k.weight") + new_item = new_item.replace("k.bias", "to_k.bias") + + new_item = new_item.replace("v.weight", "to_v.weight") + new_item = new_item.replace("v.bias", "to_v.bias") + + new_item = new_item.replace("proj_out.weight", "to_out.0.weight") + new_item = new_item.replace("proj_out.bias", "to_out.0.bias") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def convert_ldm_vae_checkpoint(checkpoint, config): + # extract state dict for VAE + vae_state_dict = {} + keys = list(checkpoint.keys()) + vae_key = "first_stage_model." if any(k.startswith("first_stage_model.") for k in keys) else "" + for key in keys: + if key.startswith(vae_key): + vae_state_dict[key.replace(vae_key, "")] = checkpoint.get(key) + + new_checkpoint = {} + + new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"] + new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"] + new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"] + new_checkpoint["decoder.time_conv_out.weight"] = vae_state_dict["decoder.time_mix_conv.weight"] + new_checkpoint["decoder.time_conv_out.bias"] = vae_state_dict["decoder.time_mix_conv.bias"] + + # new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"] + # new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"] + # new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"] + # new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer}) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer}) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + return new_checkpoint diff --git a/diffusers-0.27.0/scripts/convert_tiny_autoencoder_to_diffusers.py b/diffusers-0.27.0/scripts/convert_tiny_autoencoder_to_diffusers.py new file mode 100755 index 0000000..9bb2df9 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_tiny_autoencoder_to_diffusers.py @@ -0,0 +1,71 @@ +import argparse + +import safetensors.torch + +from diffusers import AutoencoderTiny + + +""" +Example - From the diffusers root directory: + +Download the weights: +```sh +$ wget -q https://huggingface.co/madebyollin/taesd/resolve/main/taesd_encoder.safetensors +$ wget -q https://huggingface.co/madebyollin/taesd/resolve/main/taesd_decoder.safetensors +``` + +Convert the model: +```sh +$ python scripts/convert_tiny_autoencoder_to_diffusers.py \ + --encoder_ckpt_path taesd_encoder.safetensors \ + --decoder_ckpt_path taesd_decoder.safetensors \ + --dump_path taesd-diffusers +``` +""" + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument( + "--encoder_ckpt_path", + default=None, + type=str, + required=True, + help="Path to the encoder ckpt.", + ) + parser.add_argument( + "--decoder_ckpt_path", + default=None, + type=str, + required=True, + help="Path to the decoder ckpt.", + ) + parser.add_argument( + "--use_safetensors", action="store_true", help="Whether to serialize in the safetensors format." + ) + args = parser.parse_args() + + print("Loading the original state_dicts of the encoder and the decoder...") + encoder_state_dict = safetensors.torch.load_file(args.encoder_ckpt_path) + decoder_state_dict = safetensors.torch.load_file(args.decoder_ckpt_path) + + print("Populating the state_dicts in the diffusers format...") + tiny_autoencoder = AutoencoderTiny() + new_state_dict = {} + + # Modify the encoder state dict. + for k in encoder_state_dict: + new_state_dict.update({f"encoder.layers.{k}": encoder_state_dict[k]}) + + # Modify the decoder state dict. + for k in decoder_state_dict: + layer_id = int(k.split(".")[0]) - 1 + new_k = str(layer_id) + "." + ".".join(k.split(".")[1:]) + new_state_dict.update({f"decoder.layers.{new_k}": decoder_state_dict[k]}) + + # Assertion tests with the original implementation can be found here: + # https://gist.github.com/sayakpaul/337b0988f08bd2cf2b248206f760e28f + tiny_autoencoder.load_state_dict(new_state_dict) + print("Population successful, serializing...") + tiny_autoencoder.save_pretrained(args.dump_path, safe_serialization=args.use_safetensors) diff --git a/diffusers-0.27.0/scripts/convert_unclip_txt2img_to_image_variation.py b/diffusers-0.27.0/scripts/convert_unclip_txt2img_to_image_variation.py new file mode 100755 index 0000000..07f8ebf --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_unclip_txt2img_to_image_variation.py @@ -0,0 +1,41 @@ +import argparse + +from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection + +from diffusers import UnCLIPImageVariationPipeline, UnCLIPPipeline + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + + parser.add_argument( + "--txt2img_unclip", + default="kakaobrain/karlo-v1-alpha", + type=str, + required=False, + help="The pretrained txt2img unclip.", + ) + + args = parser.parse_args() + + txt2img = UnCLIPPipeline.from_pretrained(args.txt2img_unclip) + + feature_extractor = CLIPImageProcessor() + image_encoder = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-large-patch14") + + img2img = UnCLIPImageVariationPipeline( + decoder=txt2img.decoder, + text_encoder=txt2img.text_encoder, + tokenizer=txt2img.tokenizer, + text_proj=txt2img.text_proj, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + super_res_first=txt2img.super_res_first, + super_res_last=txt2img.super_res_last, + decoder_scheduler=txt2img.decoder_scheduler, + super_res_scheduler=txt2img.super_res_scheduler, + ) + + img2img.save_pretrained(args.dump_path) diff --git a/diffusers-0.27.0/scripts/convert_unidiffuser_to_diffusers.py b/diffusers-0.27.0/scripts/convert_unidiffuser_to_diffusers.py new file mode 100755 index 0000000..4c38172 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_unidiffuser_to_diffusers.py @@ -0,0 +1,786 @@ +# Convert the original UniDiffuser checkpoints into diffusers equivalents. + +import argparse +from argparse import Namespace + +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextConfig, + CLIPTextModel, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, + GPT2Tokenizer, +) + +from diffusers import ( + AutoencoderKL, + DPMSolverMultistepScheduler, + UniDiffuserModel, + UniDiffuserPipeline, + UniDiffuserTextDecoder, +) + + +SCHEDULER_CONFIG = Namespace( + **{ + "beta_start": 0.00085, + "beta_end": 0.012, + "beta_schedule": "scaled_linear", + "solver_order": 3, + } +) + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.shave_segments +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_vae_resnet_paths +def renew_vae_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("nin_shortcut", "conv_shortcut") + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.renew_vae_attention_paths +def renew_vae_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("norm.weight", "group_norm.weight") + new_item = new_item.replace("norm.bias", "group_norm.bias") + + new_item = new_item.replace("q.weight", "to_q.weight") + new_item = new_item.replace("q.bias", "to_q.bias") + + new_item = new_item.replace("k.weight", "to_k.weight") + new_item = new_item.replace("k.bias", "to_k.bias") + + new_item = new_item.replace("v.weight", "to_v.weight") + new_item = new_item.replace("v.bias", "to_v.bias") + + new_item = new_item.replace("proj_out.weight", "to_out.0.weight") + new_item = new_item.replace("proj_out.bias", "to_out.0.bias") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.conv_attn_to_linear +def conv_attn_to_linear(checkpoint): + keys = list(checkpoint.keys()) + attn_keys = ["query.weight", "key.weight", "value.weight"] + for key in keys: + if ".".join(key.split(".")[-2:]) in attn_keys: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0, 0] + elif "proj_attn.weight" in key: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0] + + +# Modified from diffusers.pipelines.stable_diffusion.convert_from_ckpt.assign_to_checkpoint +# config.num_head_channels => num_head_channels +def assign_to_checkpoint( + paths, + checkpoint, + old_checkpoint, + attention_paths_to_split=None, + additional_replacements=None, + num_head_channels=1, +): + """ + This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits + attention layers, and takes into account additional replacements that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + # Splits the attention layers into three variables. + if attention_paths_to_split is not None: + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // num_head_channels // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape) + checkpoint[path_map["key"]] = key.reshape(target_shape) + checkpoint[path_map["value"]] = value.reshape(target_shape) + + for path in paths: + new_path = path["new"] + + # These have already been assigned + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + # Global renaming happens here + new_path = new_path.replace("middle_block.0", "mid_block.resnets.0") + new_path = new_path.replace("middle_block.1", "mid_block.attentions.0") + new_path = new_path.replace("middle_block.2", "mid_block.resnets.1") + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + # proj_attn.weight has to be converted from conv 1D to linear + is_attn_weight = "proj_attn.weight" in new_path or ("attentions" in new_path and "to_" in new_path) + shape = old_checkpoint[path["old"]].shape + if is_attn_weight and len(shape) == 3: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0] + elif is_attn_weight and len(shape) == 4: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0, 0] + else: + checkpoint[new_path] = old_checkpoint[path["old"]] + + +def create_vae_diffusers_config(config_type): + # Hardcoded for now + if args.config_type == "test": + vae_config = create_vae_diffusers_config_test() + elif args.config_type == "big": + vae_config = create_vae_diffusers_config_big() + else: + raise NotImplementedError( + f"Config type {config_type} is not implemented, currently only config types" + " 'test' and 'big' are available." + ) + return vae_config + + +def create_unidiffuser_unet_config(config_type, version): + # Hardcoded for now + if args.config_type == "test": + unet_config = create_unidiffuser_unet_config_test() + elif args.config_type == "big": + unet_config = create_unidiffuser_unet_config_big() + else: + raise NotImplementedError( + f"Config type {config_type} is not implemented, currently only config types" + " 'test' and 'big' are available." + ) + # Unidiffuser-v1 uses data type embeddings + if version == 1: + unet_config["use_data_type_embedding"] = True + return unet_config + + +def create_text_decoder_config(config_type): + # Hardcoded for now + if args.config_type == "test": + text_decoder_config = create_text_decoder_config_test() + elif args.config_type == "big": + text_decoder_config = create_text_decoder_config_big() + else: + raise NotImplementedError( + f"Config type {config_type} is not implemented, currently only config types" + " 'test' and 'big' are available." + ) + return text_decoder_config + + +# Hardcoded configs for test versions of the UniDiffuser models, corresponding to those in the fast default tests. +def create_vae_diffusers_config_test(): + vae_config = { + "sample_size": 32, + "in_channels": 3, + "out_channels": 3, + "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"], + "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D"], + "block_out_channels": [32, 64], + "latent_channels": 4, + "layers_per_block": 1, + } + return vae_config + + +def create_unidiffuser_unet_config_test(): + unet_config = { + "text_dim": 32, + "clip_img_dim": 32, + "num_text_tokens": 77, + "num_attention_heads": 2, + "attention_head_dim": 8, + "in_channels": 4, + "out_channels": 4, + "num_layers": 2, + "dropout": 0.0, + "norm_num_groups": 32, + "attention_bias": False, + "sample_size": 16, + "patch_size": 2, + "activation_fn": "gelu", + "num_embeds_ada_norm": 1000, + "norm_type": "layer_norm", + "block_type": "unidiffuser", + "pre_layer_norm": False, + "use_timestep_embedding": False, + "norm_elementwise_affine": True, + "use_patch_pos_embed": False, + "ff_final_dropout": True, + "use_data_type_embedding": False, + } + return unet_config + + +def create_text_decoder_config_test(): + text_decoder_config = { + "prefix_length": 77, + "prefix_inner_dim": 32, + "prefix_hidden_dim": 32, + "vocab_size": 1025, # 1024 + 1 for new EOS token + "n_positions": 1024, + "n_embd": 32, + "n_layer": 5, + "n_head": 4, + "n_inner": 37, + "activation_function": "gelu", + "resid_pdrop": 0.1, + "embd_pdrop": 0.1, + "attn_pdrop": 0.1, + "layer_norm_epsilon": 1e-5, + "initializer_range": 0.02, + } + return text_decoder_config + + +# Hardcoded configs for the UniDiffuser V1 model at https://huggingface.co/thu-ml/unidiffuser-v1 +# See also https://github.com/thu-ml/unidiffuser/blob/main/configs/sample_unidiffuser_v1.py +def create_vae_diffusers_config_big(): + vae_config = { + "sample_size": 256, + "in_channels": 3, + "out_channels": 3, + "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D"], + "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"], + "block_out_channels": [128, 256, 512, 512], + "latent_channels": 4, + "layers_per_block": 2, + } + return vae_config + + +def create_unidiffuser_unet_config_big(): + unet_config = { + "text_dim": 64, + "clip_img_dim": 512, + "num_text_tokens": 77, + "num_attention_heads": 24, + "attention_head_dim": 64, + "in_channels": 4, + "out_channels": 4, + "num_layers": 30, + "dropout": 0.0, + "norm_num_groups": 32, + "attention_bias": False, + "sample_size": 64, + "patch_size": 2, + "activation_fn": "gelu", + "num_embeds_ada_norm": 1000, + "norm_type": "layer_norm", + "block_type": "unidiffuser", + "pre_layer_norm": False, + "use_timestep_embedding": False, + "norm_elementwise_affine": True, + "use_patch_pos_embed": False, + "ff_final_dropout": True, + "use_data_type_embedding": False, + } + return unet_config + + +# From https://huggingface.co/gpt2/blob/main/config.json, the GPT2 checkpoint used by UniDiffuser +def create_text_decoder_config_big(): + text_decoder_config = { + "prefix_length": 77, + "prefix_inner_dim": 768, + "prefix_hidden_dim": 64, + "vocab_size": 50258, # 50257 + 1 for new EOS token + "n_positions": 1024, + "n_embd": 768, + "n_layer": 12, + "n_head": 12, + "n_inner": 3072, + "activation_function": "gelu", + "resid_pdrop": 0.1, + "embd_pdrop": 0.1, + "attn_pdrop": 0.1, + "layer_norm_epsilon": 1e-5, + "initializer_range": 0.02, + } + return text_decoder_config + + +# Based on diffusers.pipelines.stable_diffusion.convert_from_ckpt.convert_ldm_vae_checkpoint +def convert_vae_to_diffusers(ckpt, diffusers_model, num_head_channels=1): + """ + Converts a UniDiffuser autoencoder_kl.pth checkpoint to a diffusers AutoencoderKL. + """ + # autoencoder_kl.pth ckpt is a torch state dict + vae_state_dict = torch.load(ckpt, map_location="cpu") + + new_checkpoint = {} + + new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"] + new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"] + new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"] + + new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"] + new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"] + new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"] + new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer}) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer}) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"} + assign_to_checkpoint( + paths, + new_checkpoint, + vae_state_dict, + additional_replacements=[meta_path], + num_head_channels=num_head_channels, # not used in vae + ) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint( + paths, + new_checkpoint, + vae_state_dict, + additional_replacements=[meta_path], + num_head_channels=num_head_channels, # not used in vae + ) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + paths, + new_checkpoint, + vae_state_dict, + additional_replacements=[meta_path], + num_head_channels=num_head_channels, # not used in vae + ) + conv_attn_to_linear(new_checkpoint) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"} + assign_to_checkpoint( + paths, + new_checkpoint, + vae_state_dict, + additional_replacements=[meta_path], + num_head_channels=num_head_channels, # not used in vae + ) + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint( + paths, + new_checkpoint, + vae_state_dict, + additional_replacements=[meta_path], + num_head_channels=num_head_channels, # not used in vae + ) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + paths, + new_checkpoint, + vae_state_dict, + additional_replacements=[meta_path], + num_head_channels=num_head_channels, # not used in vae + ) + conv_attn_to_linear(new_checkpoint) + + missing_keys, unexpected_keys = diffusers_model.load_state_dict(new_checkpoint) + for missing_key in missing_keys: + print(f"Missing key: {missing_key}") + for unexpected_key in unexpected_keys: + print(f"Unexpected key: {unexpected_key}") + + return diffusers_model + + +def convert_uvit_block_to_diffusers_block( + uvit_state_dict, + new_state_dict, + block_prefix, + new_prefix="transformer.transformer_", + skip_connection=False, +): + """ + Maps the keys in a UniDiffuser transformer block (`Block`) to the keys in a diffusers transformer block + (`UTransformerBlock`/`UniDiffuserBlock`). + """ + prefix = new_prefix + block_prefix + if skip_connection: + new_state_dict[prefix + ".skip.skip_linear.weight"] = uvit_state_dict[block_prefix + ".skip_linear.weight"] + new_state_dict[prefix + ".skip.skip_linear.bias"] = uvit_state_dict[block_prefix + ".skip_linear.bias"] + new_state_dict[prefix + ".skip.norm.weight"] = uvit_state_dict[block_prefix + ".norm1.weight"] + new_state_dict[prefix + ".skip.norm.bias"] = uvit_state_dict[block_prefix + ".norm1.bias"] + + # Create the prefix string for out_blocks. + prefix += ".block" + + # Split up attention qkv.weight into to_q.weight, to_k.weight, to_v.weight + qkv = uvit_state_dict[block_prefix + ".attn.qkv.weight"] + new_attn_keys = [".attn1.to_q.weight", ".attn1.to_k.weight", ".attn1.to_v.weight"] + new_attn_keys = [prefix + key for key in new_attn_keys] + shape = qkv.shape[0] // len(new_attn_keys) + for i, attn_key in enumerate(new_attn_keys): + new_state_dict[attn_key] = qkv[i * shape : (i + 1) * shape] + + new_state_dict[prefix + ".attn1.to_out.0.weight"] = uvit_state_dict[block_prefix + ".attn.proj.weight"] + new_state_dict[prefix + ".attn1.to_out.0.bias"] = uvit_state_dict[block_prefix + ".attn.proj.bias"] + new_state_dict[prefix + ".norm1.weight"] = uvit_state_dict[block_prefix + ".norm2.weight"] + new_state_dict[prefix + ".norm1.bias"] = uvit_state_dict[block_prefix + ".norm2.bias"] + new_state_dict[prefix + ".ff.net.0.proj.weight"] = uvit_state_dict[block_prefix + ".mlp.fc1.weight"] + new_state_dict[prefix + ".ff.net.0.proj.bias"] = uvit_state_dict[block_prefix + ".mlp.fc1.bias"] + new_state_dict[prefix + ".ff.net.2.weight"] = uvit_state_dict[block_prefix + ".mlp.fc2.weight"] + new_state_dict[prefix + ".ff.net.2.bias"] = uvit_state_dict[block_prefix + ".mlp.fc2.bias"] + new_state_dict[prefix + ".norm3.weight"] = uvit_state_dict[block_prefix + ".norm3.weight"] + new_state_dict[prefix + ".norm3.bias"] = uvit_state_dict[block_prefix + ".norm3.bias"] + + return uvit_state_dict, new_state_dict + + +def convert_uvit_to_diffusers(ckpt, diffusers_model): + """ + Converts a UniDiffuser uvit_v*.pth checkpoint to a diffusers UniDiffusersModel. + """ + # uvit_v*.pth ckpt is a torch state dict + uvit_state_dict = torch.load(ckpt, map_location="cpu") + + new_state_dict = {} + + # Input layers + new_state_dict["vae_img_in.proj.weight"] = uvit_state_dict["patch_embed.proj.weight"] + new_state_dict["vae_img_in.proj.bias"] = uvit_state_dict["patch_embed.proj.bias"] + new_state_dict["clip_img_in.weight"] = uvit_state_dict["clip_img_embed.weight"] + new_state_dict["clip_img_in.bias"] = uvit_state_dict["clip_img_embed.bias"] + new_state_dict["text_in.weight"] = uvit_state_dict["text_embed.weight"] + new_state_dict["text_in.bias"] = uvit_state_dict["text_embed.bias"] + + new_state_dict["pos_embed"] = uvit_state_dict["pos_embed"] + + # Handle data type token embeddings for UniDiffuser-v1 + if "token_embedding.weight" in uvit_state_dict and diffusers_model.use_data_type_embedding: + new_state_dict["data_type_pos_embed_token"] = uvit_state_dict["pos_embed_token"] + new_state_dict["data_type_token_embedding.weight"] = uvit_state_dict["token_embedding.weight"] + + # Also initialize the PatchEmbedding in UTransformer2DModel with the PatchEmbedding from the checkpoint. + # This isn't used in the current implementation, so might want to remove. + new_state_dict["transformer.pos_embed.proj.weight"] = uvit_state_dict["patch_embed.proj.weight"] + new_state_dict["transformer.pos_embed.proj.bias"] = uvit_state_dict["patch_embed.proj.bias"] + + # Output layers + new_state_dict["transformer.norm_out.weight"] = uvit_state_dict["norm.weight"] + new_state_dict["transformer.norm_out.bias"] = uvit_state_dict["norm.bias"] + + new_state_dict["vae_img_out.weight"] = uvit_state_dict["decoder_pred.weight"] + new_state_dict["vae_img_out.bias"] = uvit_state_dict["decoder_pred.bias"] + new_state_dict["clip_img_out.weight"] = uvit_state_dict["clip_img_out.weight"] + new_state_dict["clip_img_out.bias"] = uvit_state_dict["clip_img_out.bias"] + new_state_dict["text_out.weight"] = uvit_state_dict["text_out.weight"] + new_state_dict["text_out.bias"] = uvit_state_dict["text_out.bias"] + + # in_blocks + in_blocks_prefixes = {".".join(layer.split(".")[:2]) for layer in uvit_state_dict if "in_blocks" in layer} + for in_block_prefix in list(in_blocks_prefixes): + convert_uvit_block_to_diffusers_block(uvit_state_dict, new_state_dict, in_block_prefix) + + # mid_block + # Assume there's only one mid block + convert_uvit_block_to_diffusers_block(uvit_state_dict, new_state_dict, "mid_block") + + # out_blocks + out_blocks_prefixes = {".".join(layer.split(".")[:2]) for layer in uvit_state_dict if "out_blocks" in layer} + for out_block_prefix in list(out_blocks_prefixes): + convert_uvit_block_to_diffusers_block(uvit_state_dict, new_state_dict, out_block_prefix, skip_connection=True) + + missing_keys, unexpected_keys = diffusers_model.load_state_dict(new_state_dict) + for missing_key in missing_keys: + print(f"Missing key: {missing_key}") + for unexpected_key in unexpected_keys: + print(f"Unexpected key: {unexpected_key}") + + return diffusers_model + + +def convert_caption_decoder_to_diffusers(ckpt, diffusers_model): + """ + Converts a UniDiffuser caption_decoder.pth checkpoint to a diffusers UniDiffuserTextDecoder. + """ + # caption_decoder.pth ckpt is a torch state dict + checkpoint_state_dict = torch.load(ckpt, map_location="cpu") + decoder_state_dict = {} + # Remove the "module." prefix, if necessary + caption_decoder_key = "module." + for key in checkpoint_state_dict: + if key.startswith(caption_decoder_key): + decoder_state_dict[key.replace(caption_decoder_key, "")] = checkpoint_state_dict.get(key) + else: + decoder_state_dict[key] = checkpoint_state_dict.get(key) + + new_state_dict = {} + + # Encoder and Decoder + new_state_dict["encode_prefix.weight"] = decoder_state_dict["encode_prefix.weight"] + new_state_dict["encode_prefix.bias"] = decoder_state_dict["encode_prefix.bias"] + new_state_dict["decode_prefix.weight"] = decoder_state_dict["decode_prefix.weight"] + new_state_dict["decode_prefix.bias"] = decoder_state_dict["decode_prefix.bias"] + + # Internal GPT2LMHeadModel transformer model + for key, val in decoder_state_dict.items(): + if key.startswith("gpt"): + suffix = key[len("gpt") :] + new_state_dict["transformer" + suffix] = val + + missing_keys, unexpected_keys = diffusers_model.load_state_dict(new_state_dict) + for missing_key in missing_keys: + print(f"Missing key: {missing_key}") + for unexpected_key in unexpected_keys: + print(f"Unexpected key: {unexpected_key}") + + return diffusers_model + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--caption_decoder_checkpoint_path", + default=None, + type=str, + required=False, + help="Path to caption decoder checkpoint to convert.", + ) + parser.add_argument( + "--uvit_checkpoint_path", default=None, type=str, required=False, help="Path to U-ViT checkpoint to convert." + ) + parser.add_argument( + "--vae_checkpoint_path", + default=None, + type=str, + required=False, + help="Path to VAE checkpoint to convert.", + ) + parser.add_argument( + "--pipeline_output_path", + default=None, + type=str, + required=True, + help="Path to save the output pipeline to.", + ) + parser.add_argument( + "--config_type", + default="test", + type=str, + help=( + "Config type to use. Should be 'test' to create small models for testing or 'big' to convert a full" + " checkpoint." + ), + ) + parser.add_argument( + "--version", + default=0, + type=int, + help="The UniDiffuser model type to convert to. Should be 0 for UniDiffuser-v0 and 1 for UniDiffuser-v1.", + ) + parser.add_argument( + "--safe_serialization", + action="store_true", + help="Whether to use safetensors/safe seialization when saving the pipeline.", + ) + + args = parser.parse_args() + + # Convert the VAE model. + if args.vae_checkpoint_path is not None: + vae_config = create_vae_diffusers_config(args.config_type) + vae = AutoencoderKL(**vae_config) + vae = convert_vae_to_diffusers(args.vae_checkpoint_path, vae) + + # Convert the U-ViT ("unet") model. + if args.uvit_checkpoint_path is not None: + unet_config = create_unidiffuser_unet_config(args.config_type, args.version) + unet = UniDiffuserModel(**unet_config) + unet = convert_uvit_to_diffusers(args.uvit_checkpoint_path, unet) + + # Convert the caption decoder ("text_decoder") model. + if args.caption_decoder_checkpoint_path is not None: + text_decoder_config = create_text_decoder_config(args.config_type) + text_decoder = UniDiffuserTextDecoder(**text_decoder_config) + text_decoder = convert_caption_decoder_to_diffusers(args.caption_decoder_checkpoint_path, text_decoder) + + # Scheduler is the same for both the test and big models. + scheduler_config = SCHEDULER_CONFIG + scheduler = DPMSolverMultistepScheduler( + beta_start=scheduler_config.beta_start, + beta_end=scheduler_config.beta_end, + beta_schedule=scheduler_config.beta_schedule, + solver_order=scheduler_config.solver_order, + ) + + if args.config_type == "test": + # Make a small random CLIPTextModel + torch.manual_seed(0) + clip_text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(clip_text_encoder_config) + clip_tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + # Make a small random CLIPVisionModel and accompanying CLIPImageProcessor + torch.manual_seed(0) + clip_image_encoder_config = CLIPVisionConfig( + image_size=32, + patch_size=2, + num_channels=3, + hidden_size=32, + projection_dim=32, + num_hidden_layers=5, + num_attention_heads=4, + intermediate_size=37, + dropout=0.1, + attention_dropout=0.1, + initializer_range=0.02, + ) + image_encoder = CLIPVisionModelWithProjection(clip_image_encoder_config) + image_processor = CLIPImageProcessor(crop_size=32, size=32) + + # Note that the text_decoder should already have its token embeddings resized. + text_tokenizer = GPT2Tokenizer.from_pretrained("hf-internal-testing/tiny-random-GPT2Model") + eos = "<|EOS|>" + special_tokens_dict = {"eos_token": eos} + text_tokenizer.add_special_tokens(special_tokens_dict) + elif args.config_type == "big": + text_encoder = CLIPTextModel.from_pretrained("openai/clip-vit-large-patch14") + clip_tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14") + + image_encoder = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-base-patch32") + image_processor = CLIPImageProcessor.from_pretrained("openai/clip-vit-base-patch32") + + # Note that the text_decoder should already have its token embeddings resized. + text_tokenizer = GPT2Tokenizer.from_pretrained("gpt2") + eos = "<|EOS|>" + special_tokens_dict = {"eos_token": eos} + text_tokenizer.add_special_tokens(special_tokens_dict) + else: + raise NotImplementedError( + f"Config type {args.config_type} is not implemented, currently only config types" + " 'test' and 'big' are available." + ) + + pipeline = UniDiffuserPipeline( + vae=vae, + text_encoder=text_encoder, + image_encoder=image_encoder, + clip_image_processor=image_processor, + clip_tokenizer=clip_tokenizer, + text_decoder=text_decoder, + text_tokenizer=text_tokenizer, + unet=unet, + scheduler=scheduler, + ) + pipeline.save_pretrained(args.pipeline_output_path, safe_serialization=args.safe_serialization) diff --git a/diffusers-0.27.0/scripts/convert_vae_diff_to_onnx.py b/diffusers-0.27.0/scripts/convert_vae_diff_to_onnx.py new file mode 100755 index 0000000..e023e04 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_vae_diff_to_onnx.py @@ -0,0 +1,122 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from pathlib import Path + +import torch +from packaging import version +from torch.onnx import export + +from diffusers import AutoencoderKL + + +is_torch_less_than_1_11 = version.parse(version.parse(torch.__version__).base_version) < version.parse("1.11") + + +def onnx_export( + model, + model_args: tuple, + output_path: Path, + ordered_input_names, + output_names, + dynamic_axes, + opset, + use_external_data_format=False, +): + output_path.parent.mkdir(parents=True, exist_ok=True) + # PyTorch deprecated the `enable_onnx_checker` and `use_external_data_format` arguments in v1.11, + # so we check the torch version for backwards compatibility + if is_torch_less_than_1_11: + export( + model, + model_args, + f=output_path.as_posix(), + input_names=ordered_input_names, + output_names=output_names, + dynamic_axes=dynamic_axes, + do_constant_folding=True, + use_external_data_format=use_external_data_format, + enable_onnx_checker=True, + opset_version=opset, + ) + else: + export( + model, + model_args, + f=output_path.as_posix(), + input_names=ordered_input_names, + output_names=output_names, + dynamic_axes=dynamic_axes, + do_constant_folding=True, + opset_version=opset, + ) + + +@torch.no_grad() +def convert_models(model_path: str, output_path: str, opset: int, fp16: bool = False): + dtype = torch.float16 if fp16 else torch.float32 + if fp16 and torch.cuda.is_available(): + device = "cuda" + elif fp16 and not torch.cuda.is_available(): + raise ValueError("`float16` model export is only supported on GPUs with CUDA") + else: + device = "cpu" + output_path = Path(output_path) + + # VAE DECODER + vae_decoder = AutoencoderKL.from_pretrained(model_path + "/vae") + vae_latent_channels = vae_decoder.config.latent_channels + # forward only through the decoder part + vae_decoder.forward = vae_decoder.decode + onnx_export( + vae_decoder, + model_args=( + torch.randn(1, vae_latent_channels, 25, 25).to(device=device, dtype=dtype), + False, + ), + output_path=output_path / "vae_decoder" / "model.onnx", + ordered_input_names=["latent_sample", "return_dict"], + output_names=["sample"], + dynamic_axes={ + "latent_sample": {0: "batch", 1: "channels", 2: "height", 3: "width"}, + }, + opset=opset, + ) + del vae_decoder + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--model_path", + type=str, + required=True, + help="Path to the `diffusers` checkpoint to convert (either a local directory or on the Hub).", + ) + + parser.add_argument("--output_path", type=str, required=True, help="Path to the output model.") + parser.add_argument( + "--opset", + default=14, + type=int, + help="The version of the ONNX operator set to use.", + ) + parser.add_argument("--fp16", action="store_true", default=False, help="Export the models in `float16` mode") + + args = parser.parse_args() + print(args.output_path) + convert_models(args.model_path, args.output_path, args.opset, args.fp16) + print("SD: Done: ONNX") diff --git a/diffusers-0.27.0/scripts/convert_vae_pt_to_diffusers.py b/diffusers-0.27.0/scripts/convert_vae_pt_to_diffusers.py new file mode 100755 index 0000000..a4f967c --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_vae_pt_to_diffusers.py @@ -0,0 +1,159 @@ +import argparse +import io + +import requests +import torch +import yaml + +from diffusers import AutoencoderKL +from diffusers.pipelines.stable_diffusion.convert_from_ckpt import ( + assign_to_checkpoint, + conv_attn_to_linear, + create_vae_diffusers_config, + renew_vae_attention_paths, + renew_vae_resnet_paths, +) + + +def custom_convert_ldm_vae_checkpoint(checkpoint, config): + vae_state_dict = checkpoint + + new_checkpoint = {} + + new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"] + new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"] + new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"] + + new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"] + new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"] + new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"] + new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer}) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer}) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + return new_checkpoint + + +def vae_pt_to_vae_diffuser( + checkpoint_path: str, + output_path: str, +): + # Only support V1 + r = requests.get( + " https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml" + ) + io_obj = io.BytesIO(r.content) + + original_config = yaml.safe_load(io_obj) + image_size = 512 + device = "cuda" if torch.cuda.is_available() else "cpu" + if checkpoint_path.endswith("safetensors"): + from safetensors import safe_open + + checkpoint = {} + with safe_open(checkpoint_path, framework="pt", device="cpu") as f: + for key in f.keys(): + checkpoint[key] = f.get_tensor(key) + else: + checkpoint = torch.load(checkpoint_path, map_location=device)["state_dict"] + + # Convert the VAE model. + vae_config = create_vae_diffusers_config(original_config, image_size=image_size) + converted_vae_checkpoint = custom_convert_ldm_vae_checkpoint(checkpoint, vae_config) + + vae = AutoencoderKL(**vae_config) + vae.load_state_dict(converted_vae_checkpoint) + vae.save_pretrained(output_path) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument("--vae_pt_path", default=None, type=str, required=True, help="Path to the VAE.pt to convert.") + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the VAE.pt to convert.") + + args = parser.parse_args() + + vae_pt_to_vae_diffuser(args.vae_pt_path, args.dump_path) diff --git a/diffusers-0.27.0/scripts/convert_versatile_diffusion_to_diffusers.py b/diffusers-0.27.0/scripts/convert_versatile_diffusion_to_diffusers.py new file mode 100755 index 0000000..f8eed04 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_versatile_diffusion_to_diffusers.py @@ -0,0 +1,791 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the Versatile Stable Diffusion checkpoints. """ + +import argparse +from argparse import Namespace + +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + UNet2DConditionModel, + VersatileDiffusionPipeline, +) +from diffusers.pipelines.versatile_diffusion.modeling_text_unet import UNetFlatConditionModel + + +SCHEDULER_CONFIG = Namespace( + **{ + "beta_linear_start": 0.00085, + "beta_linear_end": 0.012, + "timesteps": 1000, + "scale_factor": 0.18215, + } +) + +IMAGE_UNET_CONFIG = Namespace( + **{ + "input_channels": 4, + "model_channels": 320, + "output_channels": 4, + "num_noattn_blocks": [2, 2, 2, 2], + "channel_mult": [1, 2, 4, 4], + "with_attn": [True, True, True, False], + "num_heads": 8, + "context_dim": 768, + "use_checkpoint": True, + } +) + +TEXT_UNET_CONFIG = Namespace( + **{ + "input_channels": 768, + "model_channels": 320, + "output_channels": 768, + "num_noattn_blocks": [2, 2, 2, 2], + "channel_mult": [1, 2, 4, 4], + "second_dim": [4, 4, 4, 4], + "with_attn": [True, True, True, False], + "num_heads": 8, + "context_dim": 768, + "use_checkpoint": True, + } +) + +AUTOENCODER_CONFIG = Namespace( + **{ + "double_z": True, + "z_channels": 4, + "resolution": 256, + "in_channels": 3, + "out_ch": 3, + "ch": 128, + "ch_mult": [1, 2, 4, 4], + "num_res_blocks": 2, + "attn_resolutions": [], + "dropout": 0.0, + } +) + + +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("emb_layers.1", "time_emb_proj") + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_vae_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("nin_shortcut", "conv_shortcut") + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + # new_item = new_item.replace('norm.weight', 'group_norm.weight') + # new_item = new_item.replace('norm.bias', 'group_norm.bias') + + # new_item = new_item.replace('proj_out.weight', 'proj_attn.weight') + # new_item = new_item.replace('proj_out.bias', 'proj_attn.bias') + + # new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_vae_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("norm.weight", "group_norm.weight") + new_item = new_item.replace("norm.bias", "group_norm.bias") + + new_item = new_item.replace("q.weight", "query.weight") + new_item = new_item.replace("q.bias", "query.bias") + + new_item = new_item.replace("k.weight", "key.weight") + new_item = new_item.replace("k.bias", "key.bias") + + new_item = new_item.replace("v.weight", "value.weight") + new_item = new_item.replace("v.bias", "value.bias") + + new_item = new_item.replace("proj_out.weight", "proj_attn.weight") + new_item = new_item.replace("proj_out.bias", "proj_attn.bias") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def assign_to_checkpoint( + paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None +): + """ + This does the final conversion step: take locally converted weights and apply a global renaming + to them. It splits attention layers, and takes into account additional replacements + that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + # Splits the attention layers into three variables. + if attention_paths_to_split is not None: + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape) + checkpoint[path_map["key"]] = key.reshape(target_shape) + checkpoint[path_map["value"]] = value.reshape(target_shape) + + for path in paths: + new_path = path["new"] + + # These have already been assigned + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + # Global renaming happens here + new_path = new_path.replace("middle_block.0", "mid_block.resnets.0") + new_path = new_path.replace("middle_block.1", "mid_block.attentions.0") + new_path = new_path.replace("middle_block.2", "mid_block.resnets.1") + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + # proj_attn.weight has to be converted from conv 1D to linear + if "proj_attn.weight" in new_path: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0] + elif path["old"] in old_checkpoint: + checkpoint[new_path] = old_checkpoint[path["old"]] + + +def conv_attn_to_linear(checkpoint): + keys = list(checkpoint.keys()) + attn_keys = ["query.weight", "key.weight", "value.weight"] + for key in keys: + if ".".join(key.split(".")[-2:]) in attn_keys: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0, 0] + elif "proj_attn.weight" in key: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0] + + +def create_image_unet_diffusers_config(unet_params): + """ + Creates a config for the diffusers based on the config of the VD model. + """ + + block_out_channels = [unet_params.model_channels * mult for mult in unet_params.channel_mult] + + down_block_types = [] + resolution = 1 + for i in range(len(block_out_channels)): + block_type = "CrossAttnDownBlock2D" if unet_params.with_attn[i] else "DownBlock2D" + down_block_types.append(block_type) + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + block_type = "CrossAttnUpBlock2D" if unet_params.with_attn[-i - 1] else "UpBlock2D" + up_block_types.append(block_type) + resolution //= 2 + + if not all(n == unet_params.num_noattn_blocks[0] for n in unet_params.num_noattn_blocks): + raise ValueError("Not all num_res_blocks are equal, which is not supported in this script.") + + config = { + "sample_size": None, + "in_channels": unet_params.input_channels, + "out_channels": unet_params.output_channels, + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "layers_per_block": unet_params.num_noattn_blocks[0], + "cross_attention_dim": unet_params.context_dim, + "attention_head_dim": unet_params.num_heads, + } + + return config + + +def create_text_unet_diffusers_config(unet_params): + """ + Creates a config for the diffusers based on the config of the VD model. + """ + + block_out_channels = [unet_params.model_channels * mult for mult in unet_params.channel_mult] + + down_block_types = [] + resolution = 1 + for i in range(len(block_out_channels)): + block_type = "CrossAttnDownBlockFlat" if unet_params.with_attn[i] else "DownBlockFlat" + down_block_types.append(block_type) + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + block_type = "CrossAttnUpBlockFlat" if unet_params.with_attn[-i - 1] else "UpBlockFlat" + up_block_types.append(block_type) + resolution //= 2 + + if not all(n == unet_params.num_noattn_blocks[0] for n in unet_params.num_noattn_blocks): + raise ValueError("Not all num_res_blocks are equal, which is not supported in this script.") + + config = { + "sample_size": None, + "in_channels": (unet_params.input_channels, 1, 1), + "out_channels": (unet_params.output_channels, 1, 1), + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "layers_per_block": unet_params.num_noattn_blocks[0], + "cross_attention_dim": unet_params.context_dim, + "attention_head_dim": unet_params.num_heads, + } + + return config + + +def create_vae_diffusers_config(vae_params): + """ + Creates a config for the diffusers based on the config of the VD model. + """ + + block_out_channels = [vae_params.ch * mult for mult in vae_params.ch_mult] + down_block_types = ["DownEncoderBlock2D"] * len(block_out_channels) + up_block_types = ["UpDecoderBlock2D"] * len(block_out_channels) + + config = { + "sample_size": vae_params.resolution, + "in_channels": vae_params.in_channels, + "out_channels": vae_params.out_ch, + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "latent_channels": vae_params.z_channels, + "layers_per_block": vae_params.num_res_blocks, + } + return config + + +def create_diffusers_scheduler(original_config): + schedular = DDIMScheduler( + num_train_timesteps=original_config.model.params.timesteps, + beta_start=original_config.model.params.linear_start, + beta_end=original_config.model.params.linear_end, + beta_schedule="scaled_linear", + ) + return schedular + + +def convert_vd_unet_checkpoint(checkpoint, config, unet_key, extract_ema=False): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + + # extract state_dict for UNet + unet_state_dict = {} + keys = list(checkpoint.keys()) + + # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA + if sum(k.startswith("model_ema") for k in keys) > 100: + print("Checkpoint has both EMA and non-EMA weights.") + if extract_ema: + print( + "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA" + " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag." + ) + for key in keys: + if key.startswith("model.diffusion_model"): + flat_ema_key = "model_ema." + "".join(key.split(".")[1:]) + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(flat_ema_key) + else: + print( + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA" + " weights (usually better for inference), please make sure to add the `--extract_ema` flag." + ) + + for key in keys: + if key.startswith(unet_key): + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key) + + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = checkpoint["model.diffusion_model.time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = checkpoint["model.diffusion_model.time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = checkpoint["model.diffusion_model.time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = checkpoint["model.diffusion_model.time_embed.2.bias"] + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key] + for layer_id in range(num_output_blocks) + } + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + + if f"input_blocks.{i}.0.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + elif f"input_blocks.{i}.0.weight" in unet_state_dict: + # text_unet uses linear layers in place of downsamplers + shape = unet_state_dict[f"input_blocks.{i}.0.weight"].shape + if shape[0] != shape[1]: + continue + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.bias" + ) + + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + resnet_0_paths = renew_resnet_paths(resnet_0) + assign_to_checkpoint(resnet_0_paths, new_checkpoint, unet_state_dict, config=config) + + resnet_1_paths = renew_resnet_paths(resnet_1) + assign_to_checkpoint(resnet_1_paths, new_checkpoint, unet_state_dict, config=config) + + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key] + + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if ["conv.weight", "conv.bias"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.weight", "conv.bias"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + elif f"output_blocks.{i}.1.weight" in unet_state_dict: + # text_unet uses linear layers in place of upsamplers + shape = unet_state_dict[f"output_blocks.{i}.1.weight"].shape + if shape[0] != shape[1]: + continue + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.weight"] = unet_state_dict.pop( + f"output_blocks.{i}.1.weight" + ) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.bias"] = unet_state_dict.pop( + f"output_blocks.{i}.1.bias" + ) + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + elif f"output_blocks.{i}.2.weight" in unet_state_dict: + # text_unet uses linear layers in place of upsamplers + shape = unet_state_dict[f"output_blocks.{i}.2.weight"].shape + if shape[0] != shape[1]: + continue + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.weight"] = unet_state_dict.pop( + f"output_blocks.{i}.2.weight" + ) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.bias"] = unet_state_dict.pop( + f"output_blocks.{i}.2.bias" + ) + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = { + "old": f"output_blocks.{i}.1", + "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + + new_checkpoint[new_path] = unet_state_dict[old_path] + + return new_checkpoint + + +def convert_vd_vae_checkpoint(checkpoint, config): + # extract state dict for VAE + vae_state_dict = {} + keys = list(checkpoint.keys()) + for key in keys: + vae_state_dict[key] = checkpoint.get(key) + + new_checkpoint = {} + + new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"] + new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"] + new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"] + + new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"] + new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"] + new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"] + new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer}) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer}) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + return new_checkpoint + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--unet_checkpoint_path", default=None, type=str, required=False, help="Path to the checkpoint to convert." + ) + parser.add_argument( + "--vae_checkpoint_path", default=None, type=str, required=False, help="Path to the checkpoint to convert." + ) + parser.add_argument( + "--optimus_checkpoint_path", default=None, type=str, required=False, help="Path to the checkpoint to convert." + ) + parser.add_argument( + "--scheduler_type", + default="pndm", + type=str, + help="Type of scheduler to use. Should be one of ['pndm', 'lms', 'ddim', 'euler', 'euler-ancestral', 'dpm']", + ) + parser.add_argument( + "--extract_ema", + action="store_true", + help=( + "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights" + " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield" + " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning." + ), + ) + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + + args = parser.parse_args() + + scheduler_config = SCHEDULER_CONFIG + + num_train_timesteps = scheduler_config.timesteps + beta_start = scheduler_config.beta_linear_start + beta_end = scheduler_config.beta_linear_end + if args.scheduler_type == "pndm": + scheduler = PNDMScheduler( + beta_end=beta_end, + beta_schedule="scaled_linear", + beta_start=beta_start, + num_train_timesteps=num_train_timesteps, + skip_prk_steps=True, + steps_offset=1, + ) + elif args.scheduler_type == "lms": + scheduler = LMSDiscreteScheduler(beta_start=beta_start, beta_end=beta_end, beta_schedule="scaled_linear") + elif args.scheduler_type == "euler": + scheduler = EulerDiscreteScheduler(beta_start=beta_start, beta_end=beta_end, beta_schedule="scaled_linear") + elif args.scheduler_type == "euler-ancestral": + scheduler = EulerAncestralDiscreteScheduler( + beta_start=beta_start, beta_end=beta_end, beta_schedule="scaled_linear" + ) + elif args.scheduler_type == "dpm": + scheduler = DPMSolverMultistepScheduler( + beta_start=beta_start, beta_end=beta_end, beta_schedule="scaled_linear" + ) + elif args.scheduler_type == "ddim": + scheduler = DDIMScheduler( + beta_start=beta_start, + beta_end=beta_end, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + steps_offset=1, + ) + else: + raise ValueError(f"Scheduler of type {args.scheduler_type} doesn't exist!") + + # Convert the UNet2DConditionModel models. + if args.unet_checkpoint_path is not None: + # image UNet + image_unet_config = create_image_unet_diffusers_config(IMAGE_UNET_CONFIG) + checkpoint = torch.load(args.unet_checkpoint_path) + converted_image_unet_checkpoint = convert_vd_unet_checkpoint( + checkpoint, image_unet_config, unet_key="model.diffusion_model.unet_image.", extract_ema=args.extract_ema + ) + image_unet = UNet2DConditionModel(**image_unet_config) + image_unet.load_state_dict(converted_image_unet_checkpoint) + + # text UNet + text_unet_config = create_text_unet_diffusers_config(TEXT_UNET_CONFIG) + converted_text_unet_checkpoint = convert_vd_unet_checkpoint( + checkpoint, text_unet_config, unet_key="model.diffusion_model.unet_text.", extract_ema=args.extract_ema + ) + text_unet = UNetFlatConditionModel(**text_unet_config) + text_unet.load_state_dict(converted_text_unet_checkpoint) + + # Convert the VAE model. + if args.vae_checkpoint_path is not None: + vae_config = create_vae_diffusers_config(AUTOENCODER_CONFIG) + checkpoint = torch.load(args.vae_checkpoint_path) + converted_vae_checkpoint = convert_vd_vae_checkpoint(checkpoint, vae_config) + + vae = AutoencoderKL(**vae_config) + vae.load_state_dict(converted_vae_checkpoint) + + tokenizer = CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14") + image_feature_extractor = CLIPImageProcessor.from_pretrained("openai/clip-vit-large-patch14") + text_encoder = CLIPTextModelWithProjection.from_pretrained("openai/clip-vit-large-patch14") + image_encoder = CLIPVisionModelWithProjection.from_pretrained("openai/clip-vit-large-patch14") + + pipe = VersatileDiffusionPipeline( + scheduler=scheduler, + tokenizer=tokenizer, + image_feature_extractor=image_feature_extractor, + text_encoder=text_encoder, + image_encoder=image_encoder, + image_unet=image_unet, + text_unet=text_unet, + vae=vae, + ) + pipe.save_pretrained(args.dump_path) diff --git a/diffusers-0.27.0/scripts/convert_vq_diffusion_to_diffusers.py b/diffusers-0.27.0/scripts/convert_vq_diffusion_to_diffusers.py new file mode 100755 index 0000000..7da6b40 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_vq_diffusion_to_diffusers.py @@ -0,0 +1,916 @@ +""" +This script ports models from VQ-diffusion (https://github.com/microsoft/VQ-Diffusion) to diffusers. + +It currently only supports porting the ITHQ dataset. + +ITHQ dataset: +```sh +# From the root directory of diffusers. + +# Download the VQVAE checkpoint +$ wget https://facevcstandard.blob.core.windows.net/v-zhictang/Improved-VQ-Diffusion_model_release/ithq_vqvae.pth?sv=2020-10-02&st=2022-05-30T15%3A17%3A18Z&se=2030-05-31T15%3A17%3A00Z&sr=b&sp=r&sig=1jVavHFPpUjDs%2FTO1V3PTezaNbPp2Nx8MxiWI7y6fEY%3D -O ithq_vqvae.pth + +# Download the VQVAE config +# NOTE that in VQ-diffusion the documented file is `configs/ithq.yaml` but the target class +# `image_synthesis.modeling.codecs.image_codec.ema_vqvae.PatchVQVAE` +# loads `OUTPUT/pretrained_model/taming_dvae/config.yaml` +$ wget https://raw.githubusercontent.com/microsoft/VQ-Diffusion/main/OUTPUT/pretrained_model/taming_dvae/config.yaml -O ithq_vqvae.yaml + +# Download the main model checkpoint +$ wget https://facevcstandard.blob.core.windows.net/v-zhictang/Improved-VQ-Diffusion_model_release/ithq_learnable.pth?sv=2020-10-02&st=2022-05-30T10%3A22%3A06Z&se=2030-05-31T10%3A22%3A00Z&sr=b&sp=r&sig=GOE%2Bza02%2FPnGxYVOOPtwrTR4RA3%2F5NVgMxdW4kjaEZ8%3D -O ithq_learnable.pth + +# Download the main model config +$ wget https://raw.githubusercontent.com/microsoft/VQ-Diffusion/main/configs/ithq.yaml -O ithq.yaml + +# run the convert script +$ python ./scripts/convert_vq_diffusion_to_diffusers.py \ + --checkpoint_path ./ithq_learnable.pth \ + --original_config_file ./ithq.yaml \ + --vqvae_checkpoint_path ./ithq_vqvae.pth \ + --vqvae_original_config_file ./ithq_vqvae.yaml \ + --dump_path +``` +""" + +import argparse +import tempfile + +import torch +import yaml +from accelerate import init_empty_weights, load_checkpoint_and_dispatch +from transformers import CLIPTextModel, CLIPTokenizer +from yaml.loader import FullLoader + +from diffusers import Transformer2DModel, VQDiffusionPipeline, VQDiffusionScheduler, VQModel +from diffusers.pipelines.vq_diffusion.pipeline_vq_diffusion import LearnedClassifierFreeSamplingEmbeddings + + +# vqvae model + +PORTED_VQVAES = ["image_synthesis.modeling.codecs.image_codec.patch_vqgan.PatchVQGAN"] + + +def vqvae_model_from_original_config(original_config): + assert ( + original_config["target"] in PORTED_VQVAES + ), f"{original_config['target']} has not yet been ported to diffusers." + + original_config = original_config["params"] + + original_encoder_config = original_config["encoder_config"]["params"] + original_decoder_config = original_config["decoder_config"]["params"] + + in_channels = original_encoder_config["in_channels"] + out_channels = original_decoder_config["out_ch"] + + down_block_types = get_down_block_types(original_encoder_config) + up_block_types = get_up_block_types(original_decoder_config) + + assert original_encoder_config["ch"] == original_decoder_config["ch"] + assert original_encoder_config["ch_mult"] == original_decoder_config["ch_mult"] + block_out_channels = tuple( + [original_encoder_config["ch"] * a_ch_mult for a_ch_mult in original_encoder_config["ch_mult"]] + ) + + assert original_encoder_config["num_res_blocks"] == original_decoder_config["num_res_blocks"] + layers_per_block = original_encoder_config["num_res_blocks"] + + assert original_encoder_config["z_channels"] == original_decoder_config["z_channels"] + latent_channels = original_encoder_config["z_channels"] + + num_vq_embeddings = original_config["n_embed"] + + # Hard coded value for ResnetBlock.GoupNorm(num_groups) in VQ-diffusion + norm_num_groups = 32 + + e_dim = original_config["embed_dim"] + + model = VQModel( + in_channels=in_channels, + out_channels=out_channels, + down_block_types=down_block_types, + up_block_types=up_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + latent_channels=latent_channels, + num_vq_embeddings=num_vq_embeddings, + norm_num_groups=norm_num_groups, + vq_embed_dim=e_dim, + ) + + return model + + +def get_down_block_types(original_encoder_config): + attn_resolutions = coerce_attn_resolutions(original_encoder_config["attn_resolutions"]) + num_resolutions = len(original_encoder_config["ch_mult"]) + resolution = coerce_resolution(original_encoder_config["resolution"]) + + curr_res = resolution + down_block_types = [] + + for _ in range(num_resolutions): + if curr_res in attn_resolutions: + down_block_type = "AttnDownEncoderBlock2D" + else: + down_block_type = "DownEncoderBlock2D" + + down_block_types.append(down_block_type) + + curr_res = [r // 2 for r in curr_res] + + return down_block_types + + +def get_up_block_types(original_decoder_config): + attn_resolutions = coerce_attn_resolutions(original_decoder_config["attn_resolutions"]) + num_resolutions = len(original_decoder_config["ch_mult"]) + resolution = coerce_resolution(original_decoder_config["resolution"]) + + curr_res = [r // 2 ** (num_resolutions - 1) for r in resolution] + up_block_types = [] + + for _ in reversed(range(num_resolutions)): + if curr_res in attn_resolutions: + up_block_type = "AttnUpDecoderBlock2D" + else: + up_block_type = "UpDecoderBlock2D" + + up_block_types.append(up_block_type) + + curr_res = [r * 2 for r in curr_res] + + return up_block_types + + +def coerce_attn_resolutions(attn_resolutions): + attn_resolutions = list(attn_resolutions) + attn_resolutions_ = [] + for ar in attn_resolutions: + if isinstance(ar, (list, tuple)): + attn_resolutions_.append(list(ar)) + else: + attn_resolutions_.append([ar, ar]) + return attn_resolutions_ + + +def coerce_resolution(resolution): + if isinstance(resolution, int): + resolution = [resolution, resolution] # H, W + elif isinstance(resolution, (tuple, list)): + resolution = list(resolution) + else: + raise ValueError("Unknown type of resolution:", resolution) + return resolution + + +# done vqvae model + +# vqvae checkpoint + + +def vqvae_original_checkpoint_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + diffusers_checkpoint.update(vqvae_encoder_to_diffusers_checkpoint(model, checkpoint)) + + # quant_conv + + diffusers_checkpoint.update( + { + "quant_conv.weight": checkpoint["quant_conv.weight"], + "quant_conv.bias": checkpoint["quant_conv.bias"], + } + ) + + # quantize + diffusers_checkpoint.update({"quantize.embedding.weight": checkpoint["quantize.embedding"]}) + + # post_quant_conv + diffusers_checkpoint.update( + { + "post_quant_conv.weight": checkpoint["post_quant_conv.weight"], + "post_quant_conv.bias": checkpoint["post_quant_conv.bias"], + } + ) + + # decoder + diffusers_checkpoint.update(vqvae_decoder_to_diffusers_checkpoint(model, checkpoint)) + + return diffusers_checkpoint + + +def vqvae_encoder_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + # conv_in + diffusers_checkpoint.update( + { + "encoder.conv_in.weight": checkpoint["encoder.conv_in.weight"], + "encoder.conv_in.bias": checkpoint["encoder.conv_in.bias"], + } + ) + + # down_blocks + for down_block_idx, down_block in enumerate(model.encoder.down_blocks): + diffusers_down_block_prefix = f"encoder.down_blocks.{down_block_idx}" + down_block_prefix = f"encoder.down.{down_block_idx}" + + # resnets + for resnet_idx, resnet in enumerate(down_block.resnets): + diffusers_resnet_prefix = f"{diffusers_down_block_prefix}.resnets.{resnet_idx}" + resnet_prefix = f"{down_block_prefix}.block.{resnet_idx}" + + diffusers_checkpoint.update( + vqvae_resnet_to_diffusers_checkpoint( + resnet, checkpoint, diffusers_resnet_prefix=diffusers_resnet_prefix, resnet_prefix=resnet_prefix + ) + ) + + # downsample + + # do not include the downsample when on the last down block + # There is no downsample on the last down block + if down_block_idx != len(model.encoder.down_blocks) - 1: + # There's a single downsample in the original checkpoint but a list of downsamples + # in the diffusers model. + diffusers_downsample_prefix = f"{diffusers_down_block_prefix}.downsamplers.0.conv" + downsample_prefix = f"{down_block_prefix}.downsample.conv" + diffusers_checkpoint.update( + { + f"{diffusers_downsample_prefix}.weight": checkpoint[f"{downsample_prefix}.weight"], + f"{diffusers_downsample_prefix}.bias": checkpoint[f"{downsample_prefix}.bias"], + } + ) + + # attentions + + if hasattr(down_block, "attentions"): + for attention_idx, _ in enumerate(down_block.attentions): + diffusers_attention_prefix = f"{diffusers_down_block_prefix}.attentions.{attention_idx}" + attention_prefix = f"{down_block_prefix}.attn.{attention_idx}" + diffusers_checkpoint.update( + vqvae_attention_to_diffusers_checkpoint( + checkpoint, + diffusers_attention_prefix=diffusers_attention_prefix, + attention_prefix=attention_prefix, + ) + ) + + # mid block + + # mid block attentions + + # There is a single hardcoded attention block in the middle of the VQ-diffusion encoder + diffusers_attention_prefix = "encoder.mid_block.attentions.0" + attention_prefix = "encoder.mid.attn_1" + diffusers_checkpoint.update( + vqvae_attention_to_diffusers_checkpoint( + checkpoint, diffusers_attention_prefix=diffusers_attention_prefix, attention_prefix=attention_prefix + ) + ) + + # mid block resnets + + for diffusers_resnet_idx, resnet in enumerate(model.encoder.mid_block.resnets): + diffusers_resnet_prefix = f"encoder.mid_block.resnets.{diffusers_resnet_idx}" + + # the hardcoded prefixes to `block_` are 1 and 2 + orig_resnet_idx = diffusers_resnet_idx + 1 + # There are two hardcoded resnets in the middle of the VQ-diffusion encoder + resnet_prefix = f"encoder.mid.block_{orig_resnet_idx}" + + diffusers_checkpoint.update( + vqvae_resnet_to_diffusers_checkpoint( + resnet, checkpoint, diffusers_resnet_prefix=diffusers_resnet_prefix, resnet_prefix=resnet_prefix + ) + ) + + diffusers_checkpoint.update( + { + # conv_norm_out + "encoder.conv_norm_out.weight": checkpoint["encoder.norm_out.weight"], + "encoder.conv_norm_out.bias": checkpoint["encoder.norm_out.bias"], + # conv_out + "encoder.conv_out.weight": checkpoint["encoder.conv_out.weight"], + "encoder.conv_out.bias": checkpoint["encoder.conv_out.bias"], + } + ) + + return diffusers_checkpoint + + +def vqvae_decoder_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + # conv in + diffusers_checkpoint.update( + { + "decoder.conv_in.weight": checkpoint["decoder.conv_in.weight"], + "decoder.conv_in.bias": checkpoint["decoder.conv_in.bias"], + } + ) + + # up_blocks + + for diffusers_up_block_idx, up_block in enumerate(model.decoder.up_blocks): + # up_blocks are stored in reverse order in the VQ-diffusion checkpoint + orig_up_block_idx = len(model.decoder.up_blocks) - 1 - diffusers_up_block_idx + + diffusers_up_block_prefix = f"decoder.up_blocks.{diffusers_up_block_idx}" + up_block_prefix = f"decoder.up.{orig_up_block_idx}" + + # resnets + for resnet_idx, resnet in enumerate(up_block.resnets): + diffusers_resnet_prefix = f"{diffusers_up_block_prefix}.resnets.{resnet_idx}" + resnet_prefix = f"{up_block_prefix}.block.{resnet_idx}" + + diffusers_checkpoint.update( + vqvae_resnet_to_diffusers_checkpoint( + resnet, checkpoint, diffusers_resnet_prefix=diffusers_resnet_prefix, resnet_prefix=resnet_prefix + ) + ) + + # upsample + + # there is no up sample on the last up block + if diffusers_up_block_idx != len(model.decoder.up_blocks) - 1: + # There's a single upsample in the VQ-diffusion checkpoint but a list of downsamples + # in the diffusers model. + diffusers_downsample_prefix = f"{diffusers_up_block_prefix}.upsamplers.0.conv" + downsample_prefix = f"{up_block_prefix}.upsample.conv" + diffusers_checkpoint.update( + { + f"{diffusers_downsample_prefix}.weight": checkpoint[f"{downsample_prefix}.weight"], + f"{diffusers_downsample_prefix}.bias": checkpoint[f"{downsample_prefix}.bias"], + } + ) + + # attentions + + if hasattr(up_block, "attentions"): + for attention_idx, _ in enumerate(up_block.attentions): + diffusers_attention_prefix = f"{diffusers_up_block_prefix}.attentions.{attention_idx}" + attention_prefix = f"{up_block_prefix}.attn.{attention_idx}" + diffusers_checkpoint.update( + vqvae_attention_to_diffusers_checkpoint( + checkpoint, + diffusers_attention_prefix=diffusers_attention_prefix, + attention_prefix=attention_prefix, + ) + ) + + # mid block + + # mid block attentions + + # There is a single hardcoded attention block in the middle of the VQ-diffusion decoder + diffusers_attention_prefix = "decoder.mid_block.attentions.0" + attention_prefix = "decoder.mid.attn_1" + diffusers_checkpoint.update( + vqvae_attention_to_diffusers_checkpoint( + checkpoint, diffusers_attention_prefix=diffusers_attention_prefix, attention_prefix=attention_prefix + ) + ) + + # mid block resnets + + for diffusers_resnet_idx, resnet in enumerate(model.encoder.mid_block.resnets): + diffusers_resnet_prefix = f"decoder.mid_block.resnets.{diffusers_resnet_idx}" + + # the hardcoded prefixes to `block_` are 1 and 2 + orig_resnet_idx = diffusers_resnet_idx + 1 + # There are two hardcoded resnets in the middle of the VQ-diffusion decoder + resnet_prefix = f"decoder.mid.block_{orig_resnet_idx}" + + diffusers_checkpoint.update( + vqvae_resnet_to_diffusers_checkpoint( + resnet, checkpoint, diffusers_resnet_prefix=diffusers_resnet_prefix, resnet_prefix=resnet_prefix + ) + ) + + diffusers_checkpoint.update( + { + # conv_norm_out + "decoder.conv_norm_out.weight": checkpoint["decoder.norm_out.weight"], + "decoder.conv_norm_out.bias": checkpoint["decoder.norm_out.bias"], + # conv_out + "decoder.conv_out.weight": checkpoint["decoder.conv_out.weight"], + "decoder.conv_out.bias": checkpoint["decoder.conv_out.bias"], + } + ) + + return diffusers_checkpoint + + +def vqvae_resnet_to_diffusers_checkpoint(resnet, checkpoint, *, diffusers_resnet_prefix, resnet_prefix): + rv = { + # norm1 + f"{diffusers_resnet_prefix}.norm1.weight": checkpoint[f"{resnet_prefix}.norm1.weight"], + f"{diffusers_resnet_prefix}.norm1.bias": checkpoint[f"{resnet_prefix}.norm1.bias"], + # conv1 + f"{diffusers_resnet_prefix}.conv1.weight": checkpoint[f"{resnet_prefix}.conv1.weight"], + f"{diffusers_resnet_prefix}.conv1.bias": checkpoint[f"{resnet_prefix}.conv1.bias"], + # norm2 + f"{diffusers_resnet_prefix}.norm2.weight": checkpoint[f"{resnet_prefix}.norm2.weight"], + f"{diffusers_resnet_prefix}.norm2.bias": checkpoint[f"{resnet_prefix}.norm2.bias"], + # conv2 + f"{diffusers_resnet_prefix}.conv2.weight": checkpoint[f"{resnet_prefix}.conv2.weight"], + f"{diffusers_resnet_prefix}.conv2.bias": checkpoint[f"{resnet_prefix}.conv2.bias"], + } + + if resnet.conv_shortcut is not None: + rv.update( + { + f"{diffusers_resnet_prefix}.conv_shortcut.weight": checkpoint[f"{resnet_prefix}.nin_shortcut.weight"], + f"{diffusers_resnet_prefix}.conv_shortcut.bias": checkpoint[f"{resnet_prefix}.nin_shortcut.bias"], + } + ) + + return rv + + +def vqvae_attention_to_diffusers_checkpoint(checkpoint, *, diffusers_attention_prefix, attention_prefix): + return { + # group_norm + f"{diffusers_attention_prefix}.group_norm.weight": checkpoint[f"{attention_prefix}.norm.weight"], + f"{diffusers_attention_prefix}.group_norm.bias": checkpoint[f"{attention_prefix}.norm.bias"], + # query + f"{diffusers_attention_prefix}.query.weight": checkpoint[f"{attention_prefix}.q.weight"][:, :, 0, 0], + f"{diffusers_attention_prefix}.query.bias": checkpoint[f"{attention_prefix}.q.bias"], + # key + f"{diffusers_attention_prefix}.key.weight": checkpoint[f"{attention_prefix}.k.weight"][:, :, 0, 0], + f"{diffusers_attention_prefix}.key.bias": checkpoint[f"{attention_prefix}.k.bias"], + # value + f"{diffusers_attention_prefix}.value.weight": checkpoint[f"{attention_prefix}.v.weight"][:, :, 0, 0], + f"{diffusers_attention_prefix}.value.bias": checkpoint[f"{attention_prefix}.v.bias"], + # proj_attn + f"{diffusers_attention_prefix}.proj_attn.weight": checkpoint[f"{attention_prefix}.proj_out.weight"][ + :, :, 0, 0 + ], + f"{diffusers_attention_prefix}.proj_attn.bias": checkpoint[f"{attention_prefix}.proj_out.bias"], + } + + +# done vqvae checkpoint + +# transformer model + +PORTED_DIFFUSIONS = ["image_synthesis.modeling.transformers.diffusion_transformer.DiffusionTransformer"] +PORTED_TRANSFORMERS = ["image_synthesis.modeling.transformers.transformer_utils.Text2ImageTransformer"] +PORTED_CONTENT_EMBEDDINGS = ["image_synthesis.modeling.embeddings.dalle_mask_image_embedding.DalleMaskImageEmbedding"] + + +def transformer_model_from_original_config( + original_diffusion_config, original_transformer_config, original_content_embedding_config +): + assert ( + original_diffusion_config["target"] in PORTED_DIFFUSIONS + ), f"{original_diffusion_config['target']} has not yet been ported to diffusers." + assert ( + original_transformer_config["target"] in PORTED_TRANSFORMERS + ), f"{original_transformer_config['target']} has not yet been ported to diffusers." + assert ( + original_content_embedding_config["target"] in PORTED_CONTENT_EMBEDDINGS + ), f"{original_content_embedding_config['target']} has not yet been ported to diffusers." + + original_diffusion_config = original_diffusion_config["params"] + original_transformer_config = original_transformer_config["params"] + original_content_embedding_config = original_content_embedding_config["params"] + + inner_dim = original_transformer_config["n_embd"] + + n_heads = original_transformer_config["n_head"] + + # VQ-Diffusion gives dimension of the multi-headed attention layers as the + # number of attention heads times the sequence length (the dimension) of a + # single head. We want to specify our attention blocks with those values + # specified separately + assert inner_dim % n_heads == 0 + d_head = inner_dim // n_heads + + depth = original_transformer_config["n_layer"] + context_dim = original_transformer_config["condition_dim"] + + num_embed = original_content_embedding_config["num_embed"] + # the number of embeddings in the transformer includes the mask embedding. + # the content embedding (the vqvae) does not include the mask embedding. + num_embed = num_embed + 1 + + height = original_transformer_config["content_spatial_size"][0] + width = original_transformer_config["content_spatial_size"][1] + + assert width == height, "width has to be equal to height" + dropout = original_transformer_config["resid_pdrop"] + num_embeds_ada_norm = original_diffusion_config["diffusion_step"] + + model_kwargs = { + "attention_bias": True, + "cross_attention_dim": context_dim, + "attention_head_dim": d_head, + "num_layers": depth, + "dropout": dropout, + "num_attention_heads": n_heads, + "num_vector_embeds": num_embed, + "num_embeds_ada_norm": num_embeds_ada_norm, + "norm_num_groups": 32, + "sample_size": width, + "activation_fn": "geglu-approximate", + } + + model = Transformer2DModel(**model_kwargs) + return model + + +# done transformer model + +# transformer checkpoint + + +def transformer_original_checkpoint_to_diffusers_checkpoint(model, checkpoint): + diffusers_checkpoint = {} + + transformer_prefix = "transformer.transformer" + + diffusers_latent_image_embedding_prefix = "latent_image_embedding" + latent_image_embedding_prefix = f"{transformer_prefix}.content_emb" + + # DalleMaskImageEmbedding + diffusers_checkpoint.update( + { + f"{diffusers_latent_image_embedding_prefix}.emb.weight": checkpoint[ + f"{latent_image_embedding_prefix}.emb.weight" + ], + f"{diffusers_latent_image_embedding_prefix}.height_emb.weight": checkpoint[ + f"{latent_image_embedding_prefix}.height_emb.weight" + ], + f"{diffusers_latent_image_embedding_prefix}.width_emb.weight": checkpoint[ + f"{latent_image_embedding_prefix}.width_emb.weight" + ], + } + ) + + # transformer blocks + for transformer_block_idx, transformer_block in enumerate(model.transformer_blocks): + diffusers_transformer_block_prefix = f"transformer_blocks.{transformer_block_idx}" + transformer_block_prefix = f"{transformer_prefix}.blocks.{transformer_block_idx}" + + # ada norm block + diffusers_ada_norm_prefix = f"{diffusers_transformer_block_prefix}.norm1" + ada_norm_prefix = f"{transformer_block_prefix}.ln1" + + diffusers_checkpoint.update( + transformer_ada_norm_to_diffusers_checkpoint( + checkpoint, diffusers_ada_norm_prefix=diffusers_ada_norm_prefix, ada_norm_prefix=ada_norm_prefix + ) + ) + + # attention block + diffusers_attention_prefix = f"{diffusers_transformer_block_prefix}.attn1" + attention_prefix = f"{transformer_block_prefix}.attn1" + + diffusers_checkpoint.update( + transformer_attention_to_diffusers_checkpoint( + checkpoint, diffusers_attention_prefix=diffusers_attention_prefix, attention_prefix=attention_prefix + ) + ) + + # ada norm block + diffusers_ada_norm_prefix = f"{diffusers_transformer_block_prefix}.norm2" + ada_norm_prefix = f"{transformer_block_prefix}.ln1_1" + + diffusers_checkpoint.update( + transformer_ada_norm_to_diffusers_checkpoint( + checkpoint, diffusers_ada_norm_prefix=diffusers_ada_norm_prefix, ada_norm_prefix=ada_norm_prefix + ) + ) + + # attention block + diffusers_attention_prefix = f"{diffusers_transformer_block_prefix}.attn2" + attention_prefix = f"{transformer_block_prefix}.attn2" + + diffusers_checkpoint.update( + transformer_attention_to_diffusers_checkpoint( + checkpoint, diffusers_attention_prefix=diffusers_attention_prefix, attention_prefix=attention_prefix + ) + ) + + # norm block + diffusers_norm_block_prefix = f"{diffusers_transformer_block_prefix}.norm3" + norm_block_prefix = f"{transformer_block_prefix}.ln2" + + diffusers_checkpoint.update( + { + f"{diffusers_norm_block_prefix}.weight": checkpoint[f"{norm_block_prefix}.weight"], + f"{diffusers_norm_block_prefix}.bias": checkpoint[f"{norm_block_prefix}.bias"], + } + ) + + # feedforward block + diffusers_feedforward_prefix = f"{diffusers_transformer_block_prefix}.ff" + feedforward_prefix = f"{transformer_block_prefix}.mlp" + + diffusers_checkpoint.update( + transformer_feedforward_to_diffusers_checkpoint( + checkpoint, + diffusers_feedforward_prefix=diffusers_feedforward_prefix, + feedforward_prefix=feedforward_prefix, + ) + ) + + # to logits + + diffusers_norm_out_prefix = "norm_out" + norm_out_prefix = f"{transformer_prefix}.to_logits.0" + + diffusers_checkpoint.update( + { + f"{diffusers_norm_out_prefix}.weight": checkpoint[f"{norm_out_prefix}.weight"], + f"{diffusers_norm_out_prefix}.bias": checkpoint[f"{norm_out_prefix}.bias"], + } + ) + + diffusers_out_prefix = "out" + out_prefix = f"{transformer_prefix}.to_logits.1" + + diffusers_checkpoint.update( + { + f"{diffusers_out_prefix}.weight": checkpoint[f"{out_prefix}.weight"], + f"{diffusers_out_prefix}.bias": checkpoint[f"{out_prefix}.bias"], + } + ) + + return diffusers_checkpoint + + +def transformer_ada_norm_to_diffusers_checkpoint(checkpoint, *, diffusers_ada_norm_prefix, ada_norm_prefix): + return { + f"{diffusers_ada_norm_prefix}.emb.weight": checkpoint[f"{ada_norm_prefix}.emb.weight"], + f"{diffusers_ada_norm_prefix}.linear.weight": checkpoint[f"{ada_norm_prefix}.linear.weight"], + f"{diffusers_ada_norm_prefix}.linear.bias": checkpoint[f"{ada_norm_prefix}.linear.bias"], + } + + +def transformer_attention_to_diffusers_checkpoint(checkpoint, *, diffusers_attention_prefix, attention_prefix): + return { + # key + f"{diffusers_attention_prefix}.to_k.weight": checkpoint[f"{attention_prefix}.key.weight"], + f"{diffusers_attention_prefix}.to_k.bias": checkpoint[f"{attention_prefix}.key.bias"], + # query + f"{diffusers_attention_prefix}.to_q.weight": checkpoint[f"{attention_prefix}.query.weight"], + f"{diffusers_attention_prefix}.to_q.bias": checkpoint[f"{attention_prefix}.query.bias"], + # value + f"{diffusers_attention_prefix}.to_v.weight": checkpoint[f"{attention_prefix}.value.weight"], + f"{diffusers_attention_prefix}.to_v.bias": checkpoint[f"{attention_prefix}.value.bias"], + # linear out + f"{diffusers_attention_prefix}.to_out.0.weight": checkpoint[f"{attention_prefix}.proj.weight"], + f"{diffusers_attention_prefix}.to_out.0.bias": checkpoint[f"{attention_prefix}.proj.bias"], + } + + +def transformer_feedforward_to_diffusers_checkpoint(checkpoint, *, diffusers_feedforward_prefix, feedforward_prefix): + return { + f"{diffusers_feedforward_prefix}.net.0.proj.weight": checkpoint[f"{feedforward_prefix}.0.weight"], + f"{diffusers_feedforward_prefix}.net.0.proj.bias": checkpoint[f"{feedforward_prefix}.0.bias"], + f"{diffusers_feedforward_prefix}.net.2.weight": checkpoint[f"{feedforward_prefix}.2.weight"], + f"{diffusers_feedforward_prefix}.net.2.bias": checkpoint[f"{feedforward_prefix}.2.bias"], + } + + +# done transformer checkpoint + + +def read_config_file(filename): + # The yaml file contains annotations that certain values should + # loaded as tuples. + with open(filename) as f: + original_config = yaml.load(f, FullLoader) + + return original_config + + +# We take separate arguments for the vqvae because the ITHQ vqvae config file +# is separate from the config file for the rest of the model. +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--vqvae_checkpoint_path", + default=None, + type=str, + required=True, + help="Path to the vqvae checkpoint to convert.", + ) + + parser.add_argument( + "--vqvae_original_config_file", + default=None, + type=str, + required=True, + help="The YAML config file corresponding to the original architecture for the vqvae.", + ) + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + + parser.add_argument( + "--original_config_file", + default=None, + type=str, + required=True, + help="The YAML config file corresponding to the original architecture.", + ) + + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + + parser.add_argument( + "--checkpoint_load_device", + default="cpu", + type=str, + required=False, + help="The device passed to `map_location` when loading checkpoints.", + ) + + # See link for how ema weights are always selected + # https://github.com/microsoft/VQ-Diffusion/blob/3c98e77f721db7c787b76304fa2c96a36c7b00af/inference_VQ_Diffusion.py#L65 + parser.add_argument( + "--no_use_ema", + action="store_true", + required=False, + help=( + "Set to not use the ema weights from the original VQ-Diffusion checkpoint. You probably do not want to set" + " it as the original VQ-Diffusion always uses the ema weights when loading models." + ), + ) + + args = parser.parse_args() + + use_ema = not args.no_use_ema + + print(f"loading checkpoints to {args.checkpoint_load_device}") + + checkpoint_map_location = torch.device(args.checkpoint_load_device) + + # vqvae_model + + print(f"loading vqvae, config: {args.vqvae_original_config_file}, checkpoint: {args.vqvae_checkpoint_path}") + + vqvae_original_config = read_config_file(args.vqvae_original_config_file).model + vqvae_checkpoint = torch.load(args.vqvae_checkpoint_path, map_location=checkpoint_map_location)["model"] + + with init_empty_weights(): + vqvae_model = vqvae_model_from_original_config(vqvae_original_config) + + vqvae_diffusers_checkpoint = vqvae_original_checkpoint_to_diffusers_checkpoint(vqvae_model, vqvae_checkpoint) + + with tempfile.NamedTemporaryFile() as vqvae_diffusers_checkpoint_file: + torch.save(vqvae_diffusers_checkpoint, vqvae_diffusers_checkpoint_file.name) + del vqvae_diffusers_checkpoint + del vqvae_checkpoint + load_checkpoint_and_dispatch(vqvae_model, vqvae_diffusers_checkpoint_file.name, device_map="auto") + + print("done loading vqvae") + + # done vqvae_model + + # transformer_model + + print( + f"loading transformer, config: {args.original_config_file}, checkpoint: {args.checkpoint_path}, use ema:" + f" {use_ema}" + ) + + original_config = read_config_file(args.original_config_file).model + + diffusion_config = original_config["params"]["diffusion_config"] + transformer_config = original_config["params"]["diffusion_config"]["params"]["transformer_config"] + content_embedding_config = original_config["params"]["diffusion_config"]["params"]["content_emb_config"] + + pre_checkpoint = torch.load(args.checkpoint_path, map_location=checkpoint_map_location) + + if use_ema: + if "ema" in pre_checkpoint: + checkpoint = {} + for k, v in pre_checkpoint["model"].items(): + checkpoint[k] = v + + for k, v in pre_checkpoint["ema"].items(): + # The ema weights are only used on the transformer. To mimic their key as if they came + # from the state_dict for the top level model, we prefix with an additional "transformer." + # See the source linked in the args.use_ema config for more information. + checkpoint[f"transformer.{k}"] = v + else: + print("attempted to load ema weights but no ema weights are specified in the loaded checkpoint.") + checkpoint = pre_checkpoint["model"] + else: + checkpoint = pre_checkpoint["model"] + + del pre_checkpoint + + with init_empty_weights(): + transformer_model = transformer_model_from_original_config( + diffusion_config, transformer_config, content_embedding_config + ) + + diffusers_transformer_checkpoint = transformer_original_checkpoint_to_diffusers_checkpoint( + transformer_model, checkpoint + ) + + # classifier free sampling embeddings interlude + + # The learned embeddings are stored on the transformer in the original VQ-diffusion. We store them on a separate + # model, so we pull them off the checkpoint before the checkpoint is deleted. + + learnable_classifier_free_sampling_embeddings = diffusion_config["params"].learnable_cf + + if learnable_classifier_free_sampling_embeddings: + learned_classifier_free_sampling_embeddings_embeddings = checkpoint["transformer.empty_text_embed"] + else: + learned_classifier_free_sampling_embeddings_embeddings = None + + # done classifier free sampling embeddings interlude + + with tempfile.NamedTemporaryFile() as diffusers_transformer_checkpoint_file: + torch.save(diffusers_transformer_checkpoint, diffusers_transformer_checkpoint_file.name) + del diffusers_transformer_checkpoint + del checkpoint + load_checkpoint_and_dispatch(transformer_model, diffusers_transformer_checkpoint_file.name, device_map="auto") + + print("done loading transformer") + + # done transformer_model + + # text encoder + + print("loading CLIP text encoder") + + clip_name = "openai/clip-vit-base-patch32" + + # The original VQ-Diffusion specifies the pad value by the int used in the + # returned tokens. Each model uses `0` as the pad value. The transformers clip api + # specifies the pad value via the token before it has been tokenized. The `!` pad + # token is the same as padding with the `0` pad value. + pad_token = "!" + + tokenizer_model = CLIPTokenizer.from_pretrained(clip_name, pad_token=pad_token, device_map="auto") + + assert tokenizer_model.convert_tokens_to_ids(pad_token) == 0 + + text_encoder_model = CLIPTextModel.from_pretrained( + clip_name, + # `CLIPTextModel` does not support device_map="auto" + # device_map="auto" + ) + + print("done loading CLIP text encoder") + + # done text encoder + + # scheduler + + scheduler_model = VQDiffusionScheduler( + # the scheduler has the same number of embeddings as the transformer + num_vec_classes=transformer_model.num_vector_embeds + ) + + # done scheduler + + # learned classifier free sampling embeddings + + with init_empty_weights(): + learned_classifier_free_sampling_embeddings_model = LearnedClassifierFreeSamplingEmbeddings( + learnable_classifier_free_sampling_embeddings, + hidden_size=text_encoder_model.config.hidden_size, + length=tokenizer_model.model_max_length, + ) + + learned_classifier_free_sampling_checkpoint = { + "embeddings": learned_classifier_free_sampling_embeddings_embeddings.float() + } + + with tempfile.NamedTemporaryFile() as learned_classifier_free_sampling_checkpoint_file: + torch.save(learned_classifier_free_sampling_checkpoint, learned_classifier_free_sampling_checkpoint_file.name) + del learned_classifier_free_sampling_checkpoint + del learned_classifier_free_sampling_embeddings_embeddings + load_checkpoint_and_dispatch( + learned_classifier_free_sampling_embeddings_model, + learned_classifier_free_sampling_checkpoint_file.name, + device_map="auto", + ) + + # done learned classifier free sampling embeddings + + print(f"saving VQ diffusion model, path: {args.dump_path}") + + pipe = VQDiffusionPipeline( + vqvae=vqvae_model, + transformer=transformer_model, + tokenizer=tokenizer_model, + text_encoder=text_encoder_model, + learned_classifier_free_sampling_embeddings=learned_classifier_free_sampling_embeddings_model, + scheduler=scheduler_model, + ) + pipe.save_pretrained(args.dump_path) + + print("done writing VQ diffusion model") diff --git a/diffusers-0.27.0/scripts/convert_wuerstchen.py b/diffusers-0.27.0/scripts/convert_wuerstchen.py new file mode 100755 index 0000000..23d45d3 --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_wuerstchen.py @@ -0,0 +1,115 @@ +# Run inside root directory of official source code: https://github.com/dome272/wuerstchen/ +import os + +import torch +from transformers import AutoTokenizer, CLIPTextModel +from vqgan import VQModel + +from diffusers import ( + DDPMWuerstchenScheduler, + WuerstchenCombinedPipeline, + WuerstchenDecoderPipeline, + WuerstchenPriorPipeline, +) +from diffusers.pipelines.wuerstchen import PaellaVQModel, WuerstchenDiffNeXt, WuerstchenPrior + + +model_path = "models/" +device = "cpu" + +paella_vqmodel = VQModel() +state_dict = torch.load(os.path.join(model_path, "vqgan_f4_v1_500k.pt"), map_location=device)["state_dict"] +paella_vqmodel.load_state_dict(state_dict) + +state_dict["vquantizer.embedding.weight"] = state_dict["vquantizer.codebook.weight"] +state_dict.pop("vquantizer.codebook.weight") +vqmodel = PaellaVQModel(num_vq_embeddings=paella_vqmodel.codebook_size, latent_channels=paella_vqmodel.c_latent) +vqmodel.load_state_dict(state_dict) + +# Clip Text encoder and tokenizer +text_encoder = CLIPTextModel.from_pretrained("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k") +tokenizer = AutoTokenizer.from_pretrained("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k") + +# Generator +gen_text_encoder = CLIPTextModel.from_pretrained("laion/CLIP-ViT-H-14-laion2B-s32B-b79K").to("cpu") +gen_tokenizer = AutoTokenizer.from_pretrained("laion/CLIP-ViT-H-14-laion2B-s32B-b79K") + +orig_state_dict = torch.load(os.path.join(model_path, "model_v2_stage_b.pt"), map_location=device)["state_dict"] +state_dict = {} +for key in orig_state_dict.keys(): + if key.endswith("in_proj_weight"): + weights = orig_state_dict[key].chunk(3, 0) + state_dict[key.replace("attn.in_proj_weight", "to_q.weight")] = weights[0] + state_dict[key.replace("attn.in_proj_weight", "to_k.weight")] = weights[1] + state_dict[key.replace("attn.in_proj_weight", "to_v.weight")] = weights[2] + elif key.endswith("in_proj_bias"): + weights = orig_state_dict[key].chunk(3, 0) + state_dict[key.replace("attn.in_proj_bias", "to_q.bias")] = weights[0] + state_dict[key.replace("attn.in_proj_bias", "to_k.bias")] = weights[1] + state_dict[key.replace("attn.in_proj_bias", "to_v.bias")] = weights[2] + elif key.endswith("out_proj.weight"): + weights = orig_state_dict[key] + state_dict[key.replace("attn.out_proj.weight", "to_out.0.weight")] = weights + elif key.endswith("out_proj.bias"): + weights = orig_state_dict[key] + state_dict[key.replace("attn.out_proj.bias", "to_out.0.bias")] = weights + else: + state_dict[key] = orig_state_dict[key] +deocder = WuerstchenDiffNeXt() +deocder.load_state_dict(state_dict) + +# Prior +orig_state_dict = torch.load(os.path.join(model_path, "model_v3_stage_c.pt"), map_location=device)["ema_state_dict"] +state_dict = {} +for key in orig_state_dict.keys(): + if key.endswith("in_proj_weight"): + weights = orig_state_dict[key].chunk(3, 0) + state_dict[key.replace("attn.in_proj_weight", "to_q.weight")] = weights[0] + state_dict[key.replace("attn.in_proj_weight", "to_k.weight")] = weights[1] + state_dict[key.replace("attn.in_proj_weight", "to_v.weight")] = weights[2] + elif key.endswith("in_proj_bias"): + weights = orig_state_dict[key].chunk(3, 0) + state_dict[key.replace("attn.in_proj_bias", "to_q.bias")] = weights[0] + state_dict[key.replace("attn.in_proj_bias", "to_k.bias")] = weights[1] + state_dict[key.replace("attn.in_proj_bias", "to_v.bias")] = weights[2] + elif key.endswith("out_proj.weight"): + weights = orig_state_dict[key] + state_dict[key.replace("attn.out_proj.weight", "to_out.0.weight")] = weights + elif key.endswith("out_proj.bias"): + weights = orig_state_dict[key] + state_dict[key.replace("attn.out_proj.bias", "to_out.0.bias")] = weights + else: + state_dict[key] = orig_state_dict[key] +prior_model = WuerstchenPrior(c_in=16, c=1536, c_cond=1280, c_r=64, depth=32, nhead=24).to(device) +prior_model.load_state_dict(state_dict) + +# scheduler +scheduler = DDPMWuerstchenScheduler() + +# Prior pipeline +prior_pipeline = WuerstchenPriorPipeline( + prior=prior_model, text_encoder=text_encoder, tokenizer=tokenizer, scheduler=scheduler +) + +prior_pipeline.save_pretrained("warp-ai/wuerstchen-prior") + +decoder_pipeline = WuerstchenDecoderPipeline( + text_encoder=gen_text_encoder, tokenizer=gen_tokenizer, vqgan=vqmodel, decoder=deocder, scheduler=scheduler +) +decoder_pipeline.save_pretrained("warp-ai/wuerstchen") + +# Wuerstchen pipeline +wuerstchen_pipeline = WuerstchenCombinedPipeline( + # Decoder + text_encoder=gen_text_encoder, + tokenizer=gen_tokenizer, + decoder=deocder, + scheduler=scheduler, + vqgan=vqmodel, + # Prior + prior_tokenizer=tokenizer, + prior_text_encoder=text_encoder, + prior=prior_model, + prior_scheduler=scheduler, +) +wuerstchen_pipeline.save_pretrained("warp-ai/WuerstchenCombinedPipeline") diff --git a/diffusers-0.27.0/scripts/convert_zero123_to_diffusers.py b/diffusers-0.27.0/scripts/convert_zero123_to_diffusers.py new file mode 100755 index 0000000..3bb6f6c --- /dev/null +++ b/diffusers-0.27.0/scripts/convert_zero123_to_diffusers.py @@ -0,0 +1,806 @@ +""" +This script modified from +https://github.com/huggingface/diffusers/blob/bc691231360a4cbc7d19a58742ebb8ed0f05e027/scripts/convert_original_stable_diffusion_to_diffusers.py + +Convert original Zero1to3 checkpoint to diffusers checkpoint. + +# run the convert script +$ python convert_zero123_to_diffusers.py \ + --checkpoint_path /path/zero123/105000.ckpt \ + --dump_path ./zero1to3 \ + --original_config_file /path/zero123/configs/sd-objaverse-finetune-c_concat-256.yaml +``` +""" +import argparse + +import torch +import yaml +from accelerate import init_empty_weights +from accelerate.utils import set_module_tensor_to_device +from pipeline_zero1to3 import CCProjection, Zero1to3StableDiffusionPipeline +from transformers import ( + CLIPImageProcessor, + CLIPVisionModelWithProjection, +) + +from diffusers.models import ( + AutoencoderKL, + UNet2DConditionModel, +) +from diffusers.schedulers import DDIMScheduler +from diffusers.utils import logging + + +logger = logging.get_logger(__name__) + + +def create_unet_diffusers_config(original_config, image_size: int, controlnet=False): + """ + Creates a config for the diffusers based on the config of the LDM model. + """ + if controlnet: + unet_params = original_config["model"]["params"]["control_stage_config"]["params"] + else: + if ( + "unet_config" in original_config["model"]["params"] + and original_config["model"]["params"]["unet_config"] is not None + ): + unet_params = original_config["model"]["params"]["unet_config"]["params"] + else: + unet_params = original_config["model"]["params"]["network_config"]["params"] + + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + + block_out_channels = [unet_params["model_channels"] * mult for mult in unet_params["channel_mult"]] + + down_block_types = [] + resolution = 1 + for i in range(len(block_out_channels)): + block_type = "CrossAttnDownBlock2D" if resolution in unet_params["attention_resolutions"] else "DownBlock2D" + down_block_types.append(block_type) + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + block_type = "CrossAttnUpBlock2D" if resolution in unet_params["attention_resolutions"] else "UpBlock2D" + up_block_types.append(block_type) + resolution //= 2 + + if unet_params["transformer_depth"] is not None: + transformer_layers_per_block = ( + unet_params["transformer_depth"] + if isinstance(unet_params["transformer_depth"], int) + else list(unet_params["transformer_depth"]) + ) + else: + transformer_layers_per_block = 1 + + vae_scale_factor = 2 ** (len(vae_params["ch_mult"]) - 1) + + head_dim = unet_params["num_heads"] if "num_heads" in unet_params else None + use_linear_projection = ( + unet_params["use_linear_in_transformer"] if "use_linear_in_transformer" in unet_params else False + ) + if use_linear_projection: + # stable diffusion 2-base-512 and 2-768 + if head_dim is None: + head_dim_mult = unet_params["model_channels"] // unet_params["num_head_channels"] + head_dim = [head_dim_mult * c for c in list(unet_params["channel_mult"])] + + class_embed_type = None + addition_embed_type = None + addition_time_embed_dim = None + projection_class_embeddings_input_dim = None + context_dim = None + + if unet_params["context_dim"] is not None: + context_dim = ( + unet_params["context_dim"] + if isinstance(unet_params["context_dim"], int) + else unet_params["context_dim"][0] + ) + + if "num_classes" in unet_params: + if unet_params["num_classes"] == "sequential": + if context_dim in [2048, 1280]: + # SDXL + addition_embed_type = "text_time" + addition_time_embed_dim = 256 + else: + class_embed_type = "projection" + assert "adm_in_channels" in unet_params + projection_class_embeddings_input_dim = unet_params["adm_in_channels"] + else: + raise NotImplementedError(f"Unknown conditional unet num_classes config: {unet_params["num_classes"]}") + + config = { + "sample_size": image_size // vae_scale_factor, + "in_channels": unet_params["in_channels"], + "down_block_types": tuple(down_block_types), + "block_out_channels": tuple(block_out_channels), + "layers_per_block": unet_params["num_res_blocks"], + "cross_attention_dim": context_dim, + "attention_head_dim": head_dim, + "use_linear_projection": use_linear_projection, + "class_embed_type": class_embed_type, + "addition_embed_type": addition_embed_type, + "addition_time_embed_dim": addition_time_embed_dim, + "projection_class_embeddings_input_dim": projection_class_embeddings_input_dim, + "transformer_layers_per_block": transformer_layers_per_block, + } + + if controlnet: + config["conditioning_channels"] = unet_params["hint_channels"] + else: + config["out_channels"] = unet_params["out_channels"] + config["up_block_types"] = tuple(up_block_types) + + return config + + +def assign_to_checkpoint( + paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None +): + """ + This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits + attention layers, and takes into account additional replacements that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + # Splits the attention layers into three variables. + if attention_paths_to_split is not None: + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape) + checkpoint[path_map["key"]] = key.reshape(target_shape) + checkpoint[path_map["value"]] = value.reshape(target_shape) + + for path in paths: + new_path = path["new"] + + # These have already been assigned + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + # Global renaming happens here + new_path = new_path.replace("middle_block.0", "mid_block.resnets.0") + new_path = new_path.replace("middle_block.1", "mid_block.attentions.0") + new_path = new_path.replace("middle_block.2", "mid_block.resnets.1") + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + # proj_attn.weight has to be converted from conv 1D to linear + is_attn_weight = "proj_attn.weight" in new_path or ("attentions" in new_path and "to_" in new_path) + shape = old_checkpoint[path["old"]].shape + if is_attn_weight and len(shape) == 3: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0] + elif is_attn_weight and len(shape) == 4: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0, 0] + else: + checkpoint[new_path] = old_checkpoint[path["old"]] + + +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("emb_layers.1", "time_emb_proj") + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + # new_item = new_item.replace('norm.weight', 'group_norm.weight') + # new_item = new_item.replace('norm.bias', 'group_norm.bias') + + # new_item = new_item.replace('proj_out.weight', 'proj_attn.weight') + # new_item = new_item.replace('proj_out.bias', 'proj_attn.bias') + + # new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def convert_ldm_unet_checkpoint( + checkpoint, config, path=None, extract_ema=False, controlnet=False, skip_extract_state_dict=False +): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + + if skip_extract_state_dict: + unet_state_dict = checkpoint + else: + # extract state_dict for UNet + unet_state_dict = {} + keys = list(checkpoint.keys()) + + if controlnet: + unet_key = "control_model." + else: + unet_key = "model.diffusion_model." + + # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA + if sum(k.startswith("model_ema") for k in keys) > 100 and extract_ema: + logger.warning(f"Checkpoint {path} has both EMA and non-EMA weights.") + logger.warning( + "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA" + " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag." + ) + for key in keys: + if key.startswith("model.diffusion_model"): + flat_ema_key = "model_ema." + "".join(key.split(".")[1:]) + unet_state_dict[key.replace(unet_key, "")] = checkpoint[flat_ema_key] + else: + if sum(k.startswith("model_ema") for k in keys) > 100: + logger.warning( + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA" + " weights (usually better for inference), please make sure to add the `--extract_ema` flag." + ) + + for key in keys: + if key.startswith(unet_key): + unet_state_dict[key.replace(unet_key, "")] = checkpoint[key] + + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"] + + if config["class_embed_type"] is None: + # No parameters to port + ... + elif config["class_embed_type"] == "timestep" or config["class_embed_type"] == "projection": + new_checkpoint["class_embedding.linear_1.weight"] = unet_state_dict["label_emb.0.0.weight"] + new_checkpoint["class_embedding.linear_1.bias"] = unet_state_dict["label_emb.0.0.bias"] + new_checkpoint["class_embedding.linear_2.weight"] = unet_state_dict["label_emb.0.2.weight"] + new_checkpoint["class_embedding.linear_2.bias"] = unet_state_dict["label_emb.0.2.bias"] + else: + raise NotImplementedError(f"Not implemented `class_embed_type`: {config['class_embed_type']}") + + if config["addition_embed_type"] == "text_time": + new_checkpoint["add_embedding.linear_1.weight"] = unet_state_dict["label_emb.0.0.weight"] + new_checkpoint["add_embedding.linear_1.bias"] = unet_state_dict["label_emb.0.0.bias"] + new_checkpoint["add_embedding.linear_2.weight"] = unet_state_dict["label_emb.0.2.weight"] + new_checkpoint["add_embedding.linear_2.bias"] = unet_state_dict["label_emb.0.2.bias"] + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + if not controlnet: + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key] + for layer_id in range(num_output_blocks) + } + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + + if f"input_blocks.{i}.0.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + resnet_0_paths = renew_resnet_paths(resnet_0) + assign_to_checkpoint(resnet_0_paths, new_checkpoint, unet_state_dict, config=config) + + resnet_1_paths = renew_resnet_paths(resnet_1) + assign_to_checkpoint(resnet_1_paths, new_checkpoint, unet_state_dict, config=config) + + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key] + + resnet_0_paths = renew_resnet_paths(resnets) + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + output_block_list = {k: sorted(v) for k, v in output_block_list.items()} + if ["conv.bias", "conv.weight"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.bias", "conv.weight"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = { + "old": f"output_blocks.{i}.1", + "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + + new_checkpoint[new_path] = unet_state_dict[old_path] + + if controlnet: + # conditioning embedding + + orig_index = 0 + + new_checkpoint["controlnet_cond_embedding.conv_in.weight"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.weight" + ) + new_checkpoint["controlnet_cond_embedding.conv_in.bias"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.bias" + ) + + orig_index += 2 + + diffusers_index = 0 + + while diffusers_index < 6: + new_checkpoint[f"controlnet_cond_embedding.blocks.{diffusers_index}.weight"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.weight" + ) + new_checkpoint[f"controlnet_cond_embedding.blocks.{diffusers_index}.bias"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.bias" + ) + diffusers_index += 1 + orig_index += 2 + + new_checkpoint["controlnet_cond_embedding.conv_out.weight"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.weight" + ) + new_checkpoint["controlnet_cond_embedding.conv_out.bias"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.bias" + ) + + # down blocks + for i in range(num_input_blocks): + new_checkpoint[f"controlnet_down_blocks.{i}.weight"] = unet_state_dict.pop(f"zero_convs.{i}.0.weight") + new_checkpoint[f"controlnet_down_blocks.{i}.bias"] = unet_state_dict.pop(f"zero_convs.{i}.0.bias") + + # mid block + new_checkpoint["controlnet_mid_block.weight"] = unet_state_dict.pop("middle_block_out.0.weight") + new_checkpoint["controlnet_mid_block.bias"] = unet_state_dict.pop("middle_block_out.0.bias") + + return new_checkpoint + + +def create_vae_diffusers_config(original_config, image_size: int): + """ + Creates a config for the diffusers based on the config of the LDM model. + """ + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + _ = original_config["model"]["params"]["first_stage_config"]["params"]["embed_dim"] + + block_out_channels = [vae_params["ch"] * mult for mult in vae_params["ch_mult"]] + down_block_types = ["DownEncoderBlock2D"] * len(block_out_channels) + up_block_types = ["UpDecoderBlock2D"] * len(block_out_channels) + + config = { + "sample_size": image_size, + "in_channels": vae_params["in_channels"], + "out_channels": vae_params["out_ch"], + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "latent_channels": vae_params["z_channels"], + "layers_per_block": vae_params["num_res_blocks"], + } + return config + + +def convert_ldm_vae_checkpoint(checkpoint, config): + # extract state dict for VAE + vae_state_dict = {} + vae_key = "first_stage_model." + keys = list(checkpoint.keys()) + for key in keys: + if key.startswith(vae_key): + vae_state_dict[key.replace(vae_key, "")] = checkpoint.get(key) + + new_checkpoint = {} + + new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"] + new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"] + new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"] + + new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"] + new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"] + new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"] + new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer}) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer}) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + return new_checkpoint + + +def renew_vae_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("nin_shortcut", "conv_shortcut") + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_vae_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("norm.weight", "group_norm.weight") + new_item = new_item.replace("norm.bias", "group_norm.bias") + + new_item = new_item.replace("q.weight", "to_q.weight") + new_item = new_item.replace("q.bias", "to_q.bias") + + new_item = new_item.replace("k.weight", "to_k.weight") + new_item = new_item.replace("k.bias", "to_k.bias") + + new_item = new_item.replace("v.weight", "to_v.weight") + new_item = new_item.replace("v.bias", "to_v.bias") + + new_item = new_item.replace("proj_out.weight", "to_out.0.weight") + new_item = new_item.replace("proj_out.bias", "to_out.0.bias") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def conv_attn_to_linear(checkpoint): + keys = list(checkpoint.keys()) + attn_keys = ["query.weight", "key.weight", "value.weight"] + for key in keys: + if ".".join(key.split(".")[-2:]) in attn_keys: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0, 0] + elif "proj_attn.weight" in key: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0] + + +def convert_from_original_zero123_ckpt(checkpoint_path, original_config_file, extract_ema, device): + ckpt = torch.load(checkpoint_path, map_location=device) + ckpt["global_step"] + checkpoint = ckpt["state_dict"] + del ckpt + torch.cuda.empty_cache() + + original_config = yaml.safe_load(original_config_file) + original_config["model"]["params"]["cond_stage_config"]["target"].split(".")[-1] + num_in_channels = 8 + original_config["model"]["params"]["unet_config"]["params"]["in_channels"] = num_in_channels + prediction_type = "epsilon" + image_size = 256 + num_train_timesteps = getattr(original_config["model"]["params"], "timesteps", None) or 1000 + + beta_start = getattr(original_config["model"]["params"], "linear_start", None) or 0.02 + beta_end = getattr(original_config["model"]["params"], "linear_end", None) or 0.085 + scheduler = DDIMScheduler( + beta_end=beta_end, + beta_schedule="scaled_linear", + beta_start=beta_start, + num_train_timesteps=num_train_timesteps, + steps_offset=1, + clip_sample=False, + set_alpha_to_one=False, + prediction_type=prediction_type, + ) + scheduler.register_to_config(clip_sample=False) + + # Convert the UNet2DConditionModel model. + upcast_attention = None + unet_config = create_unet_diffusers_config(original_config, image_size=image_size) + unet_config["upcast_attention"] = upcast_attention + with init_empty_weights(): + unet = UNet2DConditionModel(**unet_config) + converted_unet_checkpoint = convert_ldm_unet_checkpoint( + checkpoint, unet_config, path=None, extract_ema=extract_ema + ) + for param_name, param in converted_unet_checkpoint.items(): + set_module_tensor_to_device(unet, param_name, "cpu", value=param) + + # Convert the VAE model. + vae_config = create_vae_diffusers_config(original_config, image_size=image_size) + converted_vae_checkpoint = convert_ldm_vae_checkpoint(checkpoint, vae_config) + + if ( + "model" in original_config + and "params" in original_config["model"] + and "scale_factor" in original_config["model"]["params"] + ): + vae_scaling_factor = original_config["model"]["params"]["scale_factor"] + else: + vae_scaling_factor = 0.18215 # default SD scaling factor + + vae_config["scaling_factor"] = vae_scaling_factor + + with init_empty_weights(): + vae = AutoencoderKL(**vae_config) + + for param_name, param in converted_vae_checkpoint.items(): + set_module_tensor_to_device(vae, param_name, "cpu", value=param) + + feature_extractor = CLIPImageProcessor.from_pretrained( + "lambdalabs/sd-image-variations-diffusers", subfolder="feature_extractor" + ) + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + "lambdalabs/sd-image-variations-diffusers", subfolder="image_encoder" + ) + + cc_projection = CCProjection() + cc_projection.load_state_dict( + { + "projection.weight": checkpoint["cc_projection.weight"].cpu(), + "projection.bias": checkpoint["cc_projection.bias"].cpu(), + } + ) + + pipe = Zero1to3StableDiffusionPipeline( + vae, image_encoder, unet, scheduler, None, feature_extractor, cc_projection, requires_safety_checker=False + ) + + return pipe + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + + parser.add_argument( + "--checkpoint_path", default=None, type=str, required=True, help="Path to the checkpoint to convert." + ) + parser.add_argument( + "--original_config_file", + default=None, + type=str, + help="The YAML config file corresponding to the original architecture.", + ) + parser.add_argument( + "--extract_ema", + action="store_true", + help=( + "Only relevant for checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights" + " or not. Defaults to `False`. Add `--extract_ema` to extract the EMA weights. EMA weights usually yield" + " higher quality images for inference. Non-EMA weights are usually better to continue fine-tuning." + ), + ) + parser.add_argument( + "--to_safetensors", + action="store_true", + help="Whether to store pipeline in safetensors format or not.", + ) + parser.add_argument("--half", action="store_true", help="Save weights in half precision.") + parser.add_argument("--dump_path", default=None, type=str, required=True, help="Path to the output model.") + parser.add_argument("--device", type=str, help="Device to use (e.g. cpu, cuda:0, cuda:1, etc.)") + args = parser.parse_args() + + pipe = convert_from_original_zero123_ckpt( + checkpoint_path=args.checkpoint_path, + original_config_file=args.original_config_file, + extract_ema=args.extract_ema, + device=args.device, + ) + + if args.half: + pipe.to(dtype=torch.float16) + + pipe.save_pretrained(args.dump_path, safe_serialization=args.to_safetensors) diff --git a/diffusers-0.27.0/scripts/generate_logits.py b/diffusers-0.27.0/scripts/generate_logits.py new file mode 100755 index 0000000..89dce0e --- /dev/null +++ b/diffusers-0.27.0/scripts/generate_logits.py @@ -0,0 +1,127 @@ +import random + +import torch +from huggingface_hub import HfApi + +from diffusers import UNet2DModel + + +api = HfApi() + +results = {} +# fmt: off +results["google_ddpm_cifar10_32"] = torch.tensor([ + -0.7515, -1.6883, 0.2420, 0.0300, 0.6347, 1.3433, -1.1743, -3.7467, + 1.2342, -2.2485, 0.4636, 0.8076, -0.7991, 0.3969, 0.8498, 0.9189, + -1.8887, -3.3522, 0.7639, 0.2040, 0.6271, -2.7148, -1.6316, 3.0839, + 0.3186, 0.2721, -0.9759, -1.2461, 2.6257, 1.3557 +]) +results["google_ddpm_ema_bedroom_256"] = torch.tensor([ + -2.3639, -2.5344, 0.0054, -0.6674, 1.5990, 1.0158, 0.3124, -2.1436, + 1.8795, -2.5429, -0.1566, -0.3973, 1.2490, 2.6447, 1.2283, -0.5208, + -2.8154, -3.5119, 2.3838, 1.2033, 1.7201, -2.1256, -1.4576, 2.7948, + 2.4204, -0.9752, -1.2546, 0.8027, 3.2758, 3.1365 +]) +results["CompVis_ldm_celebahq_256"] = torch.tensor([ + -0.6531, -0.6891, -0.3172, -0.5375, -0.9140, -0.5367, -0.1175, -0.7869, + -0.3808, -0.4513, -0.2098, -0.0083, 0.3183, 0.5140, 0.2247, -0.1304, + -0.1302, -0.2802, -0.2084, -0.2025, -0.4967, -0.4873, -0.0861, 0.6925, + 0.0250, 0.1290, -0.1543, 0.6316, 1.0460, 1.4943 +]) +results["google_ncsnpp_ffhq_1024"] = torch.tensor([ + 0.0911, 0.1107, 0.0182, 0.0435, -0.0805, -0.0608, 0.0381, 0.2172, + -0.0280, 0.1327, -0.0299, -0.0255, -0.0050, -0.1170, -0.1046, 0.0309, + 0.1367, 0.1728, -0.0533, -0.0748, -0.0534, 0.1624, 0.0384, -0.1805, + -0.0707, 0.0642, 0.0220, -0.0134, -0.1333, -0.1505 +]) +results["google_ncsnpp_bedroom_256"] = torch.tensor([ + 0.1321, 0.1337, 0.0440, 0.0622, -0.0591, -0.0370, 0.0503, 0.2133, + -0.0177, 0.1415, -0.0116, -0.0112, 0.0044, -0.0980, -0.0789, 0.0395, + 0.1502, 0.1785, -0.0488, -0.0514, -0.0404, 0.1539, 0.0454, -0.1559, + -0.0665, 0.0659, 0.0383, -0.0005, -0.1266, -0.1386 +]) +results["google_ncsnpp_celebahq_256"] = torch.tensor([ + 0.1154, 0.1218, 0.0307, 0.0526, -0.0711, -0.0541, 0.0366, 0.2078, + -0.0267, 0.1317, -0.0226, -0.0193, -0.0014, -0.1055, -0.0902, 0.0330, + 0.1391, 0.1709, -0.0562, -0.0693, -0.0560, 0.1482, 0.0381, -0.1683, + -0.0681, 0.0661, 0.0331, -0.0046, -0.1268, -0.1431 +]) +results["google_ncsnpp_church_256"] = torch.tensor([ + 0.1192, 0.1240, 0.0414, 0.0606, -0.0557, -0.0412, 0.0430, 0.2042, + -0.0200, 0.1385, -0.0115, -0.0132, 0.0017, -0.0965, -0.0802, 0.0398, + 0.1433, 0.1747, -0.0458, -0.0533, -0.0407, 0.1545, 0.0419, -0.1574, + -0.0645, 0.0626, 0.0341, -0.0010, -0.1199, -0.1390 +]) +results["google_ncsnpp_ffhq_256"] = torch.tensor([ + 0.1075, 0.1074, 0.0205, 0.0431, -0.0774, -0.0607, 0.0298, 0.2042, + -0.0320, 0.1267, -0.0281, -0.0250, -0.0064, -0.1091, -0.0946, 0.0290, + 0.1328, 0.1650, -0.0580, -0.0738, -0.0586, 0.1440, 0.0337, -0.1746, + -0.0712, 0.0605, 0.0250, -0.0099, -0.1316, -0.1473 +]) +results["google_ddpm_cat_256"] = torch.tensor([ + -1.4572, -2.0481, -0.0414, -0.6005, 1.4136, 0.5848, 0.4028, -2.7330, + 1.2212, -2.1228, 0.2155, 0.4039, 0.7662, 2.0535, 0.7477, -0.3243, + -2.1758, -2.7648, 1.6947, 0.7026, 1.2338, -1.6078, -0.8682, 2.2810, + 1.8574, -0.5718, -0.5586, -0.0186, 2.3415, 2.1251]) +results["google_ddpm_celebahq_256"] = torch.tensor([ + -1.3690, -1.9720, -0.4090, -0.6966, 1.4660, 0.9938, -0.1385, -2.7324, + 0.7736, -1.8917, 0.2923, 0.4293, 0.1693, 1.4112, 1.1887, -0.3181, + -2.2160, -2.6381, 1.3170, 0.8163, 0.9240, -1.6544, -0.6099, 2.5259, + 1.6430, -0.9090, -0.9392, -0.0126, 2.4268, 2.3266 +]) +results["google_ddpm_ema_celebahq_256"] = torch.tensor([ + -1.3525, -1.9628, -0.3956, -0.6860, 1.4664, 1.0014, -0.1259, -2.7212, + 0.7772, -1.8811, 0.2996, 0.4388, 0.1704, 1.4029, 1.1701, -0.3027, + -2.2053, -2.6287, 1.3350, 0.8131, 0.9274, -1.6292, -0.6098, 2.5131, + 1.6505, -0.8958, -0.9298, -0.0151, 2.4257, 2.3355 +]) +results["google_ddpm_church_256"] = torch.tensor([ + -2.0585, -2.7897, -0.2850, -0.8940, 1.9052, 0.5702, 0.6345, -3.8959, + 1.5932, -3.2319, 0.1974, 0.0287, 1.7566, 2.6543, 0.8387, -0.5351, + -3.2736, -4.3375, 2.9029, 1.6390, 1.4640, -2.1701, -1.9013, 2.9341, + 3.4981, -0.6255, -1.1644, -0.1591, 3.7097, 3.2066 +]) +results["google_ddpm_bedroom_256"] = torch.tensor([ + -2.3139, -2.5594, -0.0197, -0.6785, 1.7001, 1.1606, 0.3075, -2.1740, + 1.8071, -2.5630, -0.0926, -0.3811, 1.2116, 2.6246, 1.2731, -0.5398, + -2.8153, -3.6140, 2.3893, 1.3262, 1.6258, -2.1856, -1.3267, 2.8395, + 2.3779, -1.0623, -1.2468, 0.8959, 3.3367, 3.2243 +]) +results["google_ddpm_ema_church_256"] = torch.tensor([ + -2.0628, -2.7667, -0.2089, -0.8263, 2.0539, 0.5992, 0.6495, -3.8336, + 1.6025, -3.2817, 0.1721, -0.0633, 1.7516, 2.7039, 0.8100, -0.5908, + -3.2113, -4.4343, 2.9257, 1.3632, 1.5562, -2.1489, -1.9894, 3.0560, + 3.3396, -0.7328, -1.0417, 0.0383, 3.7093, 3.2343 +]) +results["google_ddpm_ema_cat_256"] = torch.tensor([ + -1.4574, -2.0569, -0.0473, -0.6117, 1.4018, 0.5769, 0.4129, -2.7344, + 1.2241, -2.1397, 0.2000, 0.3937, 0.7616, 2.0453, 0.7324, -0.3391, + -2.1746, -2.7744, 1.6963, 0.6921, 1.2187, -1.6172, -0.8877, 2.2439, + 1.8471, -0.5839, -0.5605, -0.0464, 2.3250, 2.1219 +]) +# fmt: on + +models = api.list_models(filter="diffusers") +for mod in models: + if "google" in mod.author or mod.modelId == "CompVis/ldm-celebahq-256": + local_checkpoint = "/home/patrick/google_checkpoints/" + mod.modelId.split("/")[-1] + + print(f"Started running {mod.modelId}!!!") + + if mod.modelId.startswith("CompVis"): + model = UNet2DModel.from_pretrained(local_checkpoint, subfolder="unet") + else: + model = UNet2DModel.from_pretrained(local_checkpoint) + + torch.manual_seed(0) + random.seed(0) + + noise = torch.randn(1, model.config.in_channels, model.config.sample_size, model.config.sample_size) + time_step = torch.tensor([10] * noise.shape[0]) + with torch.no_grad(): + logits = model(noise, time_step).sample + + assert torch.allclose( + logits[0, 0, 0, :30], results["_".join("_".join(mod.modelId.split("/")).split("-"))], atol=1e-3 + ) + print(f"{mod.modelId} has passed successfully!!!") diff --git a/diffusers-0.27.0/scripts/log_reports.py b/diffusers-0.27.0/scripts/log_reports.py new file mode 100755 index 0000000..dd1b258 --- /dev/null +++ b/diffusers-0.27.0/scripts/log_reports.py @@ -0,0 +1,139 @@ +import argparse +import json +import os +from datetime import date +from pathlib import Path + +from slack_sdk import WebClient +from tabulate import tabulate + + +MAX_LEN_MESSAGE = 2900 # slack endpoint has a limit of 3001 characters + +parser = argparse.ArgumentParser() +parser.add_argument("--slack_channel_name", default="diffusers-ci-nightly") + + +def main(slack_channel_name=None): + failed = [] + passed = [] + + group_info = [] + + total_num_failed = 0 + empty_file = False or len(list(Path().glob("*.log"))) == 0 + + total_empty_files = [] + + for log in Path().glob("*.log"): + section_num_failed = 0 + i = 0 + with open(log) as f: + for line in f: + line = json.loads(line) + i += 1 + if line.get("nodeid", "") != "": + test = line["nodeid"] + if line.get("duration", None) is not None: + duration = f'{line["duration"]:.4f}' + if line.get("outcome", "") == "failed": + section_num_failed += 1 + failed.append([test, duration, log.name.split("_")[0]]) + total_num_failed += 1 + else: + passed.append([test, duration, log.name.split("_")[0]]) + empty_file = i == 0 + group_info.append([str(log), section_num_failed, failed]) + total_empty_files.append(empty_file) + os.remove(log) + failed = [] + text = ( + "🌞 There were no failures!" + if not any(total_empty_files) + else "Something went wrong there is at least one empty file - please check GH action results." + ) + no_error_payload = { + "type": "section", + "text": { + "type": "plain_text", + "text": text, + "emoji": True, + }, + } + + message = "" + payload = [ + { + "type": "header", + "text": { + "type": "plain_text", + "text": "🤗 Results of the Diffusers scheduled nightly tests.", + }, + }, + ] + if total_num_failed > 0: + for i, (name, num_failed, failed_tests) in enumerate(group_info): + if num_failed > 0: + if num_failed == 1: + message += f"*{name}: {num_failed} failed test*\n" + else: + message += f"*{name}: {num_failed} failed tests*\n" + failed_table = [] + for test in failed_tests: + failed_table.append(test[0].split("::")) + failed_table = tabulate( + failed_table, + headers=["Test Location", "Test Case", "Test Name"], + showindex="always", + tablefmt="grid", + maxcolwidths=[12, 12, 12], + ) + message += "\n```\n" + failed_table + "\n```" + + if total_empty_files[i]: + message += f"\n*{name}: Warning! Empty file - please check the GitHub action job *\n" + print(f"### {message}") + else: + payload.append(no_error_payload) + + if len(message) > MAX_LEN_MESSAGE: + print(f"Truncating long message from {len(message)} to {MAX_LEN_MESSAGE}") + message = message[:MAX_LEN_MESSAGE] + "..." + + if len(message) != 0: + md_report = { + "type": "section", + "text": {"type": "mrkdwn", "text": message}, + } + payload.append(md_report) + action_button = { + "type": "section", + "text": {"type": "mrkdwn", "text": "*For more details:*"}, + "accessory": { + "type": "button", + "text": {"type": "plain_text", "text": "Check Action results", "emoji": True}, + "url": f"https://github.com/huggingface/diffusers/actions/runs/{os.environ['GITHUB_RUN_ID']}", + }, + } + payload.append(action_button) + + date_report = { + "type": "context", + "elements": [ + { + "type": "plain_text", + "text": f"Nightly test results for {date.today()}", + }, + ], + } + payload.append(date_report) + + print(payload) + + client = WebClient(token=os.environ.get("SLACK_API_TOKEN")) + client.chat_postMessage(channel=f"#{slack_channel_name}", text=message, blocks=payload) + + +if __name__ == "__main__": + args = parser.parse_args() + main(args.slack_channel_name) diff --git a/diffusers-0.27.0/src/diffusers/__init__.py b/diffusers-0.27.0/src/diffusers/__init__.py new file mode 100755 index 0000000..2f258e9 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/__init__.py @@ -0,0 +1,787 @@ +__version__ = "0.27.0" + +from typing import TYPE_CHECKING + +from .utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + is_flax_available, + is_k_diffusion_available, + is_librosa_available, + is_note_seq_available, + is_onnx_available, + is_scipy_available, + is_torch_available, + is_torchsde_available, + is_transformers_available, +) + + +# Lazy Import based on +# https://github.com/huggingface/transformers/blob/main/src/transformers/__init__.py + +# When adding a new object to this init, please add it to `_import_structure`. The `_import_structure` is a dictionary submodule to list of object names, +# and is used to defer the actual importing for when the objects are requested. +# This way `import diffusers` provides the names in the namespace without actually importing anything (and especially none of the backends). + +_import_structure = { + "configuration_utils": ["ConfigMixin"], + "models": [], + "pipelines": [], + "schedulers": [], + "utils": [ + "OptionalDependencyNotAvailable", + "is_flax_available", + "is_inflect_available", + "is_invisible_watermark_available", + "is_k_diffusion_available", + "is_k_diffusion_version", + "is_librosa_available", + "is_note_seq_available", + "is_onnx_available", + "is_scipy_available", + "is_torch_available", + "is_torchsde_available", + "is_transformers_available", + "is_transformers_version", + "is_unidecode_available", + "logging", + ], +} + +try: + if not is_onnx_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_onnx_objects # noqa F403 + + _import_structure["utils.dummy_onnx_objects"] = [ + name for name in dir(dummy_onnx_objects) if not name.startswith("_") + ] + +else: + _import_structure["pipelines"].extend(["OnnxRuntimeModel"]) + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_pt_objects # noqa F403 + + _import_structure["utils.dummy_pt_objects"] = [name for name in dir(dummy_pt_objects) if not name.startswith("_")] + +else: + _import_structure["models"].extend( + [ + "AsymmetricAutoencoderKL", + "AutoencoderKL", + "AutoencoderKLTemporalDecoder", + "AutoencoderTiny", + "ConsistencyDecoderVAE", + "ControlNetModel", + "I2VGenXLUNet", + "Kandinsky3UNet", + "ModelMixin", + "MotionAdapter", + "MultiAdapter", + "PriorTransformer", + "StableCascadeUNet", + "T2IAdapter", + "T5FilmDecoder", + "Transformer2DModel", + "UNet1DModel", + "UNet2DConditionModel", + "UNet2DModel", + "UNet3DConditionModel", + "UNetMotionModel", + "UNetSpatioTemporalConditionModel", + "UVit2DModel", + "VQModel", + ] + ) + + _import_structure["optimization"] = [ + "get_constant_schedule", + "get_constant_schedule_with_warmup", + "get_cosine_schedule_with_warmup", + "get_cosine_with_hard_restarts_schedule_with_warmup", + "get_linear_schedule_with_warmup", + "get_polynomial_decay_schedule_with_warmup", + "get_scheduler", + ] + _import_structure["pipelines"].extend( + [ + "AudioPipelineOutput", + "AutoPipelineForImage2Image", + "AutoPipelineForInpainting", + "AutoPipelineForText2Image", + "ConsistencyModelPipeline", + "DanceDiffusionPipeline", + "DDIMPipeline", + "DDPMPipeline", + "DiffusionPipeline", + "DiTPipeline", + "ImagePipelineOutput", + "KarrasVePipeline", + "LDMPipeline", + "LDMSuperResolutionPipeline", + "PNDMPipeline", + "RePaintPipeline", + "ScoreSdeVePipeline", + "StableDiffusionMixin", + ] + ) + _import_structure["schedulers"].extend( + [ + "AmusedScheduler", + "CMStochasticIterativeScheduler", + "DDIMInverseScheduler", + "DDIMParallelScheduler", + "DDIMScheduler", + "DDPMParallelScheduler", + "DDPMScheduler", + "DDPMWuerstchenScheduler", + "DEISMultistepScheduler", + "DPMSolverMultistepInverseScheduler", + "DPMSolverMultistepScheduler", + "DPMSolverSinglestepScheduler", + "EDMDPMSolverMultistepScheduler", + "EDMEulerScheduler", + "EulerAncestralDiscreteScheduler", + "EulerDiscreteScheduler", + "HeunDiscreteScheduler", + "IPNDMScheduler", + "KarrasVeScheduler", + "KDPM2AncestralDiscreteScheduler", + "KDPM2DiscreteScheduler", + "LCMScheduler", + "PNDMScheduler", + "RePaintScheduler", + "SASolverScheduler", + "SchedulerMixin", + "ScoreSdeVeScheduler", + "TCDScheduler", + "UnCLIPScheduler", + "UniPCMultistepScheduler", + "VQDiffusionScheduler", + ] + ) + _import_structure["training_utils"] = ["EMAModel"] + +try: + if not (is_torch_available() and is_scipy_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_torch_and_scipy_objects # noqa F403 + + _import_structure["utils.dummy_torch_and_scipy_objects"] = [ + name for name in dir(dummy_torch_and_scipy_objects) if not name.startswith("_") + ] + +else: + _import_structure["schedulers"].extend(["LMSDiscreteScheduler"]) + +try: + if not (is_torch_available() and is_torchsde_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_torch_and_torchsde_objects # noqa F403 + + _import_structure["utils.dummy_torch_and_torchsde_objects"] = [ + name for name in dir(dummy_torch_and_torchsde_objects) if not name.startswith("_") + ] + +else: + _import_structure["schedulers"].extend(["DPMSolverSDEScheduler"]) + +try: + if not (is_torch_available() and is_transformers_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_torch_and_transformers_objects # noqa F403 + + _import_structure["utils.dummy_torch_and_transformers_objects"] = [ + name for name in dir(dummy_torch_and_transformers_objects) if not name.startswith("_") + ] + +else: + _import_structure["pipelines"].extend( + [ + "AltDiffusionImg2ImgPipeline", + "AltDiffusionPipeline", + "AmusedImg2ImgPipeline", + "AmusedInpaintPipeline", + "AmusedPipeline", + "AnimateDiffPipeline", + "AnimateDiffVideoToVideoPipeline", + "AudioLDM2Pipeline", + "AudioLDM2ProjectionModel", + "AudioLDM2UNet2DConditionModel", + "AudioLDMPipeline", + "BlipDiffusionControlNetPipeline", + "BlipDiffusionPipeline", + "CLIPImageProjection", + "CycleDiffusionPipeline", + "I2VGenXLPipeline", + "IFImg2ImgPipeline", + "IFImg2ImgSuperResolutionPipeline", + "IFInpaintingPipeline", + "IFInpaintingSuperResolutionPipeline", + "IFPipeline", + "IFSuperResolutionPipeline", + "ImageTextPipelineOutput", + "Kandinsky3Img2ImgPipeline", + "Kandinsky3Pipeline", + "KandinskyCombinedPipeline", + "KandinskyImg2ImgCombinedPipeline", + "KandinskyImg2ImgPipeline", + "KandinskyInpaintCombinedPipeline", + "KandinskyInpaintPipeline", + "KandinskyPipeline", + "KandinskyPriorPipeline", + "KandinskyV22CombinedPipeline", + "KandinskyV22ControlnetImg2ImgPipeline", + "KandinskyV22ControlnetPipeline", + "KandinskyV22Img2ImgCombinedPipeline", + "KandinskyV22Img2ImgPipeline", + "KandinskyV22InpaintCombinedPipeline", + "KandinskyV22InpaintPipeline", + "KandinskyV22Pipeline", + "KandinskyV22PriorEmb2EmbPipeline", + "KandinskyV22PriorPipeline", + "LatentConsistencyModelImg2ImgPipeline", + "LatentConsistencyModelPipeline", + "LDMTextToImagePipeline", + "LEditsPPPipelineStableDiffusion", + "LEditsPPPipelineStableDiffusionXL", + "MusicLDMPipeline", + "PaintByExamplePipeline", + "PIAPipeline", + "PixArtAlphaPipeline", + "SemanticStableDiffusionPipeline", + "ShapEImg2ImgPipeline", + "ShapEPipeline", + "StableCascadeCombinedPipeline", + "StableCascadeDecoderPipeline", + "StableCascadePriorPipeline", + "StableDiffusionAdapterPipeline", + "StableDiffusionAttendAndExcitePipeline", + "StableDiffusionControlNetImg2ImgPipeline", + "StableDiffusionControlNetInpaintPipeline", + "StableDiffusionControlNetPipeline", + "StableDiffusionDepth2ImgPipeline", + "StableDiffusionDiffEditPipeline", + "StableDiffusionGLIGENPipeline", + "StableDiffusionGLIGENTextImagePipeline", + "StableDiffusionImageVariationPipeline", + "StableDiffusionImg2ImgPipeline", + "StableDiffusionInpaintPipeline", + "StableDiffusionInpaintPipelineLegacy", + "StableDiffusionInstructPix2PixPipeline", + "StableDiffusionLatentUpscalePipeline", + "StableDiffusionLDM3DPipeline", + "StableDiffusionModelEditingPipeline", + "StableDiffusionPanoramaPipeline", + "StableDiffusionParadigmsPipeline", + "StableDiffusionPipeline", + "StableDiffusionPipelineSafe", + "StableDiffusionPix2PixZeroPipeline", + "StableDiffusionSAGPipeline", + "StableDiffusionUpscalePipeline", + "StableDiffusionXLAdapterPipeline", + "StableDiffusionXLControlNetImg2ImgPipeline", + "StableDiffusionXLControlNetInpaintPipeline", + "StableDiffusionXLControlNetPipeline", + "StableDiffusionXLImg2ImgPipeline", + "StableDiffusionXLInpaintPipeline", + "StableDiffusionXLInstructPix2PixPipeline", + "StableDiffusionXLPipeline", + "StableUnCLIPImg2ImgPipeline", + "StableUnCLIPPipeline", + "StableVideoDiffusionPipeline", + "TextToVideoSDPipeline", + "TextToVideoZeroPipeline", + "TextToVideoZeroSDXLPipeline", + "UnCLIPImageVariationPipeline", + "UnCLIPPipeline", + "UniDiffuserModel", + "UniDiffuserPipeline", + "UniDiffuserTextDecoder", + "VersatileDiffusionDualGuidedPipeline", + "VersatileDiffusionImageVariationPipeline", + "VersatileDiffusionPipeline", + "VersatileDiffusionTextToImagePipeline", + "VideoToVideoSDPipeline", + "VQDiffusionPipeline", + "WuerstchenCombinedPipeline", + "WuerstchenDecoderPipeline", + "WuerstchenPriorPipeline", + ] + ) + +try: + if not (is_torch_available() and is_transformers_available() and is_k_diffusion_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_torch_and_transformers_and_k_diffusion_objects # noqa F403 + + _import_structure["utils.dummy_torch_and_transformers_and_k_diffusion_objects"] = [ + name for name in dir(dummy_torch_and_transformers_and_k_diffusion_objects) if not name.startswith("_") + ] + +else: + _import_structure["pipelines"].extend(["StableDiffusionKDiffusionPipeline", "StableDiffusionXLKDiffusionPipeline"]) + +try: + if not (is_torch_available() and is_transformers_available() and is_onnx_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_torch_and_transformers_and_onnx_objects # noqa F403 + + _import_structure["utils.dummy_torch_and_transformers_and_onnx_objects"] = [ + name for name in dir(dummy_torch_and_transformers_and_onnx_objects) if not name.startswith("_") + ] + +else: + _import_structure["pipelines"].extend( + [ + "OnnxStableDiffusionImg2ImgPipeline", + "OnnxStableDiffusionInpaintPipeline", + "OnnxStableDiffusionInpaintPipelineLegacy", + "OnnxStableDiffusionPipeline", + "OnnxStableDiffusionUpscalePipeline", + "StableDiffusionOnnxPipeline", + ] + ) + +try: + if not (is_torch_available() and is_librosa_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_torch_and_librosa_objects # noqa F403 + + _import_structure["utils.dummy_torch_and_librosa_objects"] = [ + name for name in dir(dummy_torch_and_librosa_objects) if not name.startswith("_") + ] + +else: + _import_structure["pipelines"].extend(["AudioDiffusionPipeline", "Mel"]) + +try: + if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_transformers_and_torch_and_note_seq_objects # noqa F403 + + _import_structure["utils.dummy_transformers_and_torch_and_note_seq_objects"] = [ + name for name in dir(dummy_transformers_and_torch_and_note_seq_objects) if not name.startswith("_") + ] + + +else: + _import_structure["pipelines"].extend(["SpectrogramDiffusionPipeline"]) + +try: + if not is_flax_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_flax_objects # noqa F403 + + _import_structure["utils.dummy_flax_objects"] = [ + name for name in dir(dummy_flax_objects) if not name.startswith("_") + ] + + +else: + _import_structure["models.controlnet_flax"] = ["FlaxControlNetModel"] + _import_structure["models.modeling_flax_utils"] = ["FlaxModelMixin"] + _import_structure["models.unets.unet_2d_condition_flax"] = ["FlaxUNet2DConditionModel"] + _import_structure["models.vae_flax"] = ["FlaxAutoencoderKL"] + _import_structure["pipelines"].extend(["FlaxDiffusionPipeline"]) + _import_structure["schedulers"].extend( + [ + "FlaxDDIMScheduler", + "FlaxDDPMScheduler", + "FlaxDPMSolverMultistepScheduler", + "FlaxEulerDiscreteScheduler", + "FlaxKarrasVeScheduler", + "FlaxLMSDiscreteScheduler", + "FlaxPNDMScheduler", + "FlaxSchedulerMixin", + "FlaxScoreSdeVeScheduler", + ] + ) + + +try: + if not (is_flax_available() and is_transformers_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_flax_and_transformers_objects # noqa F403 + + _import_structure["utils.dummy_flax_and_transformers_objects"] = [ + name for name in dir(dummy_flax_and_transformers_objects) if not name.startswith("_") + ] + + +else: + _import_structure["pipelines"].extend( + [ + "FlaxStableDiffusionControlNetPipeline", + "FlaxStableDiffusionImg2ImgPipeline", + "FlaxStableDiffusionInpaintPipeline", + "FlaxStableDiffusionPipeline", + "FlaxStableDiffusionXLPipeline", + ] + ) + +try: + if not (is_note_seq_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from .utils import dummy_note_seq_objects # noqa F403 + + _import_structure["utils.dummy_note_seq_objects"] = [ + name for name in dir(dummy_note_seq_objects) if not name.startswith("_") + ] + + +else: + _import_structure["pipelines"].extend(["MidiProcessor"]) + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .configuration_utils import ConfigMixin + + try: + if not is_onnx_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_onnx_objects import * # noqa F403 + else: + from .pipelines import OnnxRuntimeModel + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_pt_objects import * # noqa F403 + else: + from .models import ( + AsymmetricAutoencoderKL, + AutoencoderKL, + AutoencoderKLTemporalDecoder, + AutoencoderTiny, + ConsistencyDecoderVAE, + ControlNetModel, + I2VGenXLUNet, + Kandinsky3UNet, + ModelMixin, + MotionAdapter, + MultiAdapter, + PriorTransformer, + T2IAdapter, + T5FilmDecoder, + Transformer2DModel, + UNet1DModel, + UNet2DConditionModel, + UNet2DModel, + UNet3DConditionModel, + UNetMotionModel, + UNetSpatioTemporalConditionModel, + UVit2DModel, + VQModel, + ) + from .optimization import ( + get_constant_schedule, + get_constant_schedule_with_warmup, + get_cosine_schedule_with_warmup, + get_cosine_with_hard_restarts_schedule_with_warmup, + get_linear_schedule_with_warmup, + get_polynomial_decay_schedule_with_warmup, + get_scheduler, + ) + from .pipelines import ( + AudioPipelineOutput, + AutoPipelineForImage2Image, + AutoPipelineForInpainting, + AutoPipelineForText2Image, + BlipDiffusionControlNetPipeline, + BlipDiffusionPipeline, + CLIPImageProjection, + ConsistencyModelPipeline, + DanceDiffusionPipeline, + DDIMPipeline, + DDPMPipeline, + DiffusionPipeline, + DiTPipeline, + ImagePipelineOutput, + KarrasVePipeline, + LDMPipeline, + LDMSuperResolutionPipeline, + PNDMPipeline, + RePaintPipeline, + ScoreSdeVePipeline, + StableDiffusionMixin, + ) + from .schedulers import ( + AmusedScheduler, + CMStochasticIterativeScheduler, + DDIMInverseScheduler, + DDIMParallelScheduler, + DDIMScheduler, + DDPMParallelScheduler, + DDPMScheduler, + DDPMWuerstchenScheduler, + DEISMultistepScheduler, + DPMSolverMultistepInverseScheduler, + DPMSolverMultistepScheduler, + DPMSolverSinglestepScheduler, + EDMDPMSolverMultistepScheduler, + EDMEulerScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + IPNDMScheduler, + KarrasVeScheduler, + KDPM2AncestralDiscreteScheduler, + KDPM2DiscreteScheduler, + LCMScheduler, + PNDMScheduler, + RePaintScheduler, + SASolverScheduler, + SchedulerMixin, + ScoreSdeVeScheduler, + TCDScheduler, + UnCLIPScheduler, + UniPCMultistepScheduler, + VQDiffusionScheduler, + ) + from .training_utils import EMAModel + + try: + if not (is_torch_available() and is_scipy_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_torch_and_scipy_objects import * # noqa F403 + else: + from .schedulers import LMSDiscreteScheduler + + try: + if not (is_torch_available() and is_torchsde_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_torch_and_torchsde_objects import * # noqa F403 + else: + from .schedulers import DPMSolverSDEScheduler + + try: + if not (is_torch_available() and is_transformers_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_torch_and_transformers_objects import * # noqa F403 + else: + from .pipelines import ( + AltDiffusionImg2ImgPipeline, + AltDiffusionPipeline, + AmusedImg2ImgPipeline, + AmusedInpaintPipeline, + AmusedPipeline, + AnimateDiffPipeline, + AnimateDiffVideoToVideoPipeline, + AudioLDM2Pipeline, + AudioLDM2ProjectionModel, + AudioLDM2UNet2DConditionModel, + AudioLDMPipeline, + CLIPImageProjection, + CycleDiffusionPipeline, + I2VGenXLPipeline, + IFImg2ImgPipeline, + IFImg2ImgSuperResolutionPipeline, + IFInpaintingPipeline, + IFInpaintingSuperResolutionPipeline, + IFPipeline, + IFSuperResolutionPipeline, + ImageTextPipelineOutput, + Kandinsky3Img2ImgPipeline, + Kandinsky3Pipeline, + KandinskyCombinedPipeline, + KandinskyImg2ImgCombinedPipeline, + KandinskyImg2ImgPipeline, + KandinskyInpaintCombinedPipeline, + KandinskyInpaintPipeline, + KandinskyPipeline, + KandinskyPriorPipeline, + KandinskyV22CombinedPipeline, + KandinskyV22ControlnetImg2ImgPipeline, + KandinskyV22ControlnetPipeline, + KandinskyV22Img2ImgCombinedPipeline, + KandinskyV22Img2ImgPipeline, + KandinskyV22InpaintCombinedPipeline, + KandinskyV22InpaintPipeline, + KandinskyV22Pipeline, + KandinskyV22PriorEmb2EmbPipeline, + KandinskyV22PriorPipeline, + LatentConsistencyModelImg2ImgPipeline, + LatentConsistencyModelPipeline, + LDMTextToImagePipeline, + LEditsPPPipelineStableDiffusion, + LEditsPPPipelineStableDiffusionXL, + MusicLDMPipeline, + PaintByExamplePipeline, + PIAPipeline, + PixArtAlphaPipeline, + SemanticStableDiffusionPipeline, + ShapEImg2ImgPipeline, + ShapEPipeline, + StableCascadeCombinedPipeline, + StableCascadeDecoderPipeline, + StableCascadePriorPipeline, + StableDiffusionAdapterPipeline, + StableDiffusionAttendAndExcitePipeline, + StableDiffusionControlNetImg2ImgPipeline, + StableDiffusionControlNetInpaintPipeline, + StableDiffusionControlNetPipeline, + StableDiffusionDepth2ImgPipeline, + StableDiffusionDiffEditPipeline, + StableDiffusionGLIGENPipeline, + StableDiffusionGLIGENTextImagePipeline, + StableDiffusionImageVariationPipeline, + StableDiffusionImg2ImgPipeline, + StableDiffusionInpaintPipeline, + StableDiffusionInpaintPipelineLegacy, + StableDiffusionInstructPix2PixPipeline, + StableDiffusionLatentUpscalePipeline, + StableDiffusionLDM3DPipeline, + StableDiffusionModelEditingPipeline, + StableDiffusionPanoramaPipeline, + StableDiffusionParadigmsPipeline, + StableDiffusionPipeline, + StableDiffusionPipelineSafe, + StableDiffusionPix2PixZeroPipeline, + StableDiffusionSAGPipeline, + StableDiffusionUpscalePipeline, + StableDiffusionXLAdapterPipeline, + StableDiffusionXLControlNetImg2ImgPipeline, + StableDiffusionXLControlNetInpaintPipeline, + StableDiffusionXLControlNetPipeline, + StableDiffusionXLImg2ImgPipeline, + StableDiffusionXLInpaintPipeline, + StableDiffusionXLInstructPix2PixPipeline, + StableDiffusionXLPipeline, + StableUnCLIPImg2ImgPipeline, + StableUnCLIPPipeline, + StableVideoDiffusionPipeline, + TextToVideoSDPipeline, + TextToVideoZeroPipeline, + TextToVideoZeroSDXLPipeline, + UnCLIPImageVariationPipeline, + UnCLIPPipeline, + UniDiffuserModel, + UniDiffuserPipeline, + UniDiffuserTextDecoder, + VersatileDiffusionDualGuidedPipeline, + VersatileDiffusionImageVariationPipeline, + VersatileDiffusionPipeline, + VersatileDiffusionTextToImagePipeline, + VideoToVideoSDPipeline, + VQDiffusionPipeline, + WuerstchenCombinedPipeline, + WuerstchenDecoderPipeline, + WuerstchenPriorPipeline, + ) + + try: + if not (is_torch_available() and is_transformers_available() and is_k_diffusion_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_torch_and_transformers_and_k_diffusion_objects import * # noqa F403 + else: + from .pipelines import StableDiffusionKDiffusionPipeline, StableDiffusionXLKDiffusionPipeline + + try: + if not (is_torch_available() and is_transformers_available() and is_onnx_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_torch_and_transformers_and_onnx_objects import * # noqa F403 + else: + from .pipelines import ( + OnnxStableDiffusionImg2ImgPipeline, + OnnxStableDiffusionInpaintPipeline, + OnnxStableDiffusionInpaintPipelineLegacy, + OnnxStableDiffusionPipeline, + OnnxStableDiffusionUpscalePipeline, + StableDiffusionOnnxPipeline, + ) + + try: + if not (is_torch_available() and is_librosa_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_torch_and_librosa_objects import * # noqa F403 + else: + from .pipelines import AudioDiffusionPipeline, Mel + + try: + if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_transformers_and_torch_and_note_seq_objects import * # noqa F403 + else: + from .pipelines import SpectrogramDiffusionPipeline + + try: + if not is_flax_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_flax_objects import * # noqa F403 + else: + from .models.controlnet_flax import FlaxControlNetModel + from .models.modeling_flax_utils import FlaxModelMixin + from .models.unets.unet_2d_condition_flax import FlaxUNet2DConditionModel + from .models.vae_flax import FlaxAutoencoderKL + from .pipelines import FlaxDiffusionPipeline + from .schedulers import ( + FlaxDDIMScheduler, + FlaxDDPMScheduler, + FlaxDPMSolverMultistepScheduler, + FlaxEulerDiscreteScheduler, + FlaxKarrasVeScheduler, + FlaxLMSDiscreteScheduler, + FlaxPNDMScheduler, + FlaxSchedulerMixin, + FlaxScoreSdeVeScheduler, + ) + + try: + if not (is_flax_available() and is_transformers_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_flax_and_transformers_objects import * # noqa F403 + else: + from .pipelines import ( + FlaxStableDiffusionControlNetPipeline, + FlaxStableDiffusionImg2ImgPipeline, + FlaxStableDiffusionInpaintPipeline, + FlaxStableDiffusionPipeline, + FlaxStableDiffusionXLPipeline, + ) + + try: + if not (is_note_seq_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from .utils.dummy_note_seq_objects import * # noqa F403 + else: + from .pipelines import MidiProcessor + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + extra_objects={"__version__": __version__}, + ) diff --git a/diffusers-0.27.0/src/diffusers/commands/__init__.py b/diffusers-0.27.0/src/diffusers/commands/__init__.py new file mode 100755 index 0000000..8208283 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/commands/__init__.py @@ -0,0 +1,27 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from abc import ABC, abstractmethod +from argparse import ArgumentParser + + +class BaseDiffusersCLICommand(ABC): + @staticmethod + @abstractmethod + def register_subcommand(parser: ArgumentParser): + raise NotImplementedError() + + @abstractmethod + def run(self): + raise NotImplementedError() diff --git a/diffusers-0.27.0/src/diffusers/commands/diffusers_cli.py b/diffusers-0.27.0/src/diffusers/commands/diffusers_cli.py new file mode 100755 index 0000000..f582c3b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/commands/diffusers_cli.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from argparse import ArgumentParser + +from .env import EnvironmentCommand +from .fp16_safetensors import FP16SafetensorsCommand + + +def main(): + parser = ArgumentParser("Diffusers CLI tool", usage="diffusers-cli []") + commands_parser = parser.add_subparsers(help="diffusers-cli command helpers") + + # Register commands + EnvironmentCommand.register_subcommand(commands_parser) + FP16SafetensorsCommand.register_subcommand(commands_parser) + + # Let's go + args = parser.parse_args() + + if not hasattr(args, "func"): + parser.print_help() + exit(1) + + # Run + service = args.func(args) + service.run() + + +if __name__ == "__main__": + main() diff --git a/diffusers-0.27.0/src/diffusers/commands/env.py b/diffusers-0.27.0/src/diffusers/commands/env.py new file mode 100755 index 0000000..baa69b3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/commands/env.py @@ -0,0 +1,84 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import platform +from argparse import ArgumentParser + +import huggingface_hub + +from .. import __version__ as version +from ..utils import is_accelerate_available, is_torch_available, is_transformers_available, is_xformers_available +from . import BaseDiffusersCLICommand + + +def info_command_factory(_): + return EnvironmentCommand() + + +class EnvironmentCommand(BaseDiffusersCLICommand): + @staticmethod + def register_subcommand(parser: ArgumentParser): + download_parser = parser.add_parser("env") + download_parser.set_defaults(func=info_command_factory) + + def run(self): + hub_version = huggingface_hub.__version__ + + pt_version = "not installed" + pt_cuda_available = "NA" + if is_torch_available(): + import torch + + pt_version = torch.__version__ + pt_cuda_available = torch.cuda.is_available() + + transformers_version = "not installed" + if is_transformers_available(): + import transformers + + transformers_version = transformers.__version__ + + accelerate_version = "not installed" + if is_accelerate_available(): + import accelerate + + accelerate_version = accelerate.__version__ + + xformers_version = "not installed" + if is_xformers_available(): + import xformers + + xformers_version = xformers.__version__ + + info = { + "`diffusers` version": version, + "Platform": platform.platform(), + "Python version": platform.python_version(), + "PyTorch version (GPU?)": f"{pt_version} ({pt_cuda_available})", + "Huggingface_hub version": hub_version, + "Transformers version": transformers_version, + "Accelerate version": accelerate_version, + "xFormers version": xformers_version, + "Using GPU in script?": "", + "Using distributed or parallel set-up in script?": "", + } + + print("\nCopy-and-paste the text below in your GitHub issue and FILL OUT the two last points.\n") + print(self.format_dict(info)) + + return info + + @staticmethod + def format_dict(d): + return "\n".join([f"- {prop}: {val}" for prop, val in d.items()]) + "\n" diff --git a/diffusers-0.27.0/src/diffusers/commands/fp16_safetensors.py b/diffusers-0.27.0/src/diffusers/commands/fp16_safetensors.py new file mode 100755 index 0000000..b26b881 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/commands/fp16_safetensors.py @@ -0,0 +1,132 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Usage example: + diffusers-cli fp16_safetensors --ckpt_id=openai/shap-e --fp16 --use_safetensors +""" + +import glob +import json +import warnings +from argparse import ArgumentParser, Namespace +from importlib import import_module + +import huggingface_hub +import torch +from huggingface_hub import hf_hub_download +from packaging import version + +from ..utils import logging +from . import BaseDiffusersCLICommand + + +def conversion_command_factory(args: Namespace): + if args.use_auth_token: + warnings.warn( + "The `--use_auth_token` flag is deprecated and will be removed in a future version. Authentication is now" + " handled automatically if user is logged in." + ) + return FP16SafetensorsCommand(args.ckpt_id, args.fp16, args.use_safetensors) + + +class FP16SafetensorsCommand(BaseDiffusersCLICommand): + @staticmethod + def register_subcommand(parser: ArgumentParser): + conversion_parser = parser.add_parser("fp16_safetensors") + conversion_parser.add_argument( + "--ckpt_id", + type=str, + help="Repo id of the checkpoints on which to run the conversion. Example: 'openai/shap-e'.", + ) + conversion_parser.add_argument( + "--fp16", action="store_true", help="If serializing the variables in FP16 precision." + ) + conversion_parser.add_argument( + "--use_safetensors", action="store_true", help="If serializing in the safetensors format." + ) + conversion_parser.add_argument( + "--use_auth_token", + action="store_true", + help="When working with checkpoints having private visibility. When used `huggingface-cli login` needs to be run beforehand.", + ) + conversion_parser.set_defaults(func=conversion_command_factory) + + def __init__(self, ckpt_id: str, fp16: bool, use_safetensors: bool): + self.logger = logging.get_logger("diffusers-cli/fp16_safetensors") + self.ckpt_id = ckpt_id + self.local_ckpt_dir = f"/tmp/{ckpt_id}" + self.fp16 = fp16 + + self.use_safetensors = use_safetensors + + if not self.use_safetensors and not self.fp16: + raise NotImplementedError( + "When `use_safetensors` and `fp16` both are False, then this command is of no use." + ) + + def run(self): + if version.parse(huggingface_hub.__version__) < version.parse("0.9.0"): + raise ImportError( + "The huggingface_hub version must be >= 0.9.0 to use this command. Please update your huggingface_hub" + " installation." + ) + else: + from huggingface_hub import create_commit + from huggingface_hub._commit_api import CommitOperationAdd + + model_index = hf_hub_download(repo_id=self.ckpt_id, filename="model_index.json") + with open(model_index, "r") as f: + pipeline_class_name = json.load(f)["_class_name"] + pipeline_class = getattr(import_module("diffusers"), pipeline_class_name) + self.logger.info(f"Pipeline class imported: {pipeline_class_name}.") + + # Load the appropriate pipeline. We could have use `DiffusionPipeline` + # here, but just to avoid any rough edge cases. + pipeline = pipeline_class.from_pretrained( + self.ckpt_id, torch_dtype=torch.float16 if self.fp16 else torch.float32 + ) + pipeline.save_pretrained( + self.local_ckpt_dir, + safe_serialization=True if self.use_safetensors else False, + variant="fp16" if self.fp16 else None, + ) + self.logger.info(f"Pipeline locally saved to {self.local_ckpt_dir}.") + + # Fetch all the paths. + if self.fp16: + modified_paths = glob.glob(f"{self.local_ckpt_dir}/*/*.fp16.*") + elif self.use_safetensors: + modified_paths = glob.glob(f"{self.local_ckpt_dir}/*/*.safetensors") + + # Prepare for the PR. + commit_message = f"Serialize variables with FP16: {self.fp16} and safetensors: {self.use_safetensors}." + operations = [] + for path in modified_paths: + operations.append(CommitOperationAdd(path_in_repo="/".join(path.split("/")[4:]), path_or_fileobj=path)) + + # Open the PR. + commit_description = ( + "Variables converted by the [`diffusers`' `fp16_safetensors`" + " CLI](https://github.com/huggingface/diffusers/blob/main/src/diffusers/commands/fp16_safetensors.py)." + ) + hub_pr_url = create_commit( + repo_id=self.ckpt_id, + operations=operations, + commit_message=commit_message, + commit_description=commit_description, + repo_type="model", + create_pr=True, + ).pr_url + self.logger.info(f"PR created here: {hub_pr_url}.") diff --git a/diffusers-0.27.0/src/diffusers/configuration_utils.py b/diffusers-0.27.0/src/diffusers/configuration_utils.py new file mode 100755 index 0000000..189ef43 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/configuration_utils.py @@ -0,0 +1,703 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" ConfigMixin base class and utilities.""" +import dataclasses +import functools +import importlib +import inspect +import json +import os +import re +from collections import OrderedDict +from pathlib import PosixPath +from typing import Any, Dict, Tuple, Union + +import numpy as np +from huggingface_hub import create_repo, hf_hub_download +from huggingface_hub.utils import ( + EntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, + validate_hf_hub_args, +) +from requests import HTTPError + +from . import __version__ +from .utils import ( + HUGGINGFACE_CO_RESOLVE_ENDPOINT, + DummyObject, + deprecate, + extract_commit_hash, + http_user_agent, + logging, +) + + +logger = logging.get_logger(__name__) + +_re_configuration_file = re.compile(r"config\.(.*)\.json") + + +class FrozenDict(OrderedDict): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + for key, value in self.items(): + setattr(self, key, value) + + self.__frozen = True + + def __delitem__(self, *args, **kwargs): + raise Exception(f"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.") + + def setdefault(self, *args, **kwargs): + raise Exception(f"You cannot use ``setdefault`` on a {self.__class__.__name__} instance.") + + def pop(self, *args, **kwargs): + raise Exception(f"You cannot use ``pop`` on a {self.__class__.__name__} instance.") + + def update(self, *args, **kwargs): + raise Exception(f"You cannot use ``update`` on a {self.__class__.__name__} instance.") + + def __setattr__(self, name, value): + if hasattr(self, "__frozen") and self.__frozen: + raise Exception(f"You cannot use ``__setattr__`` on a {self.__class__.__name__} instance.") + super().__setattr__(name, value) + + def __setitem__(self, name, value): + if hasattr(self, "__frozen") and self.__frozen: + raise Exception(f"You cannot use ``__setattr__`` on a {self.__class__.__name__} instance.") + super().__setitem__(name, value) + + +class ConfigMixin: + r""" + Base class for all configuration classes. All configuration parameters are stored under `self.config`. Also + provides the [`~ConfigMixin.from_config`] and [`~ConfigMixin.save_config`] methods for loading, downloading, and + saving classes that inherit from [`ConfigMixin`]. + + Class attributes: + - **config_name** (`str`) -- A filename under which the config should stored when calling + [`~ConfigMixin.save_config`] (should be overridden by parent class). + - **ignore_for_config** (`List[str]`) -- A list of attributes that should not be saved in the config (should be + overridden by subclass). + - **has_compatibles** (`bool`) -- Whether the class has compatible classes (should be overridden by subclass). + - **_deprecated_kwargs** (`List[str]`) -- Keyword arguments that are deprecated. Note that the `init` function + should only have a `kwargs` argument if at least one argument is deprecated (should be overridden by + subclass). + """ + + config_name = None + ignore_for_config = [] + has_compatibles = False + + _deprecated_kwargs = [] + + def register_to_config(self, **kwargs): + if self.config_name is None: + raise NotImplementedError(f"Make sure that {self.__class__} has defined a class name `config_name`") + # Special case for `kwargs` used in deprecation warning added to schedulers + # TODO: remove this when we remove the deprecation warning, and the `kwargs` argument, + # or solve in a more general way. + kwargs.pop("kwargs", None) + + if not hasattr(self, "_internal_dict"): + internal_dict = kwargs + else: + previous_dict = dict(self._internal_dict) + internal_dict = {**self._internal_dict, **kwargs} + logger.debug(f"Updating config from {previous_dict} to {internal_dict}") + + self._internal_dict = FrozenDict(internal_dict) + + def __getattr__(self, name: str) -> Any: + """The only reason we overwrite `getattr` here is to gracefully deprecate accessing + config attributes directly. See https://github.com/huggingface/diffusers/pull/3129 + + This function is mostly copied from PyTorch's __getattr__ overwrite: + https://pytorch.org/docs/stable/_modules/torch/nn/modules/module.html#Module + """ + + is_in_config = "_internal_dict" in self.__dict__ and hasattr(self.__dict__["_internal_dict"], name) + is_attribute = name in self.__dict__ + + if is_in_config and not is_attribute: + deprecation_message = f"Accessing config attribute `{name}` directly via '{type(self).__name__}' object attribute is deprecated. Please access '{name}' over '{type(self).__name__}'s config object instead, e.g. 'scheduler.config.{name}'." + deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False) + return self._internal_dict[name] + + raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'") + + def save_config(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + """ + Save a configuration object to the directory specified in `save_directory` so that it can be reloaded using the + [`~ConfigMixin.from_config`] class method. + + Args: + save_directory (`str` or `os.PathLike`): + Directory where the configuration JSON file is saved (will be created if it does not exist). + push_to_hub (`bool`, *optional*, defaults to `False`): + Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the + repository you want to push to with `repo_id` (will default to the name of `save_directory` in your + namespace). + kwargs (`Dict[str, Any]`, *optional*): + Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. + """ + if os.path.isfile(save_directory): + raise AssertionError(f"Provided path ({save_directory}) should be a directory, not a file") + + os.makedirs(save_directory, exist_ok=True) + + # If we save using the predefined names, we can load using `from_config` + output_config_file = os.path.join(save_directory, self.config_name) + + self.to_json_file(output_config_file) + logger.info(f"Configuration saved in {output_config_file}") + + if push_to_hub: + commit_message = kwargs.pop("commit_message", None) + private = kwargs.pop("private", False) + create_pr = kwargs.pop("create_pr", False) + token = kwargs.pop("token", None) + repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1]) + repo_id = create_repo(repo_id, exist_ok=True, private=private, token=token).repo_id + + self._upload_folder( + save_directory, + repo_id, + token=token, + commit_message=commit_message, + create_pr=create_pr, + ) + + @classmethod + def from_config(cls, config: Union[FrozenDict, Dict[str, Any]] = None, return_unused_kwargs=False, **kwargs): + r""" + Instantiate a Python class from a config dictionary. + + Parameters: + config (`Dict[str, Any]`): + A config dictionary from which the Python class is instantiated. Make sure to only load configuration + files of compatible classes. + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + Whether kwargs that are not consumed by the Python class should be returned or not. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to update the configuration object (after it is loaded) and initiate the Python class. + `**kwargs` are passed directly to the underlying scheduler/model's `__init__` method and eventually + overwrite the same named arguments in `config`. + + Returns: + [`ModelMixin`] or [`SchedulerMixin`]: + A model or scheduler object instantiated from a config dictionary. + + Examples: + + ```python + >>> from diffusers import DDPMScheduler, DDIMScheduler, PNDMScheduler + + >>> # Download scheduler from huggingface.co and cache. + >>> scheduler = DDPMScheduler.from_pretrained("google/ddpm-cifar10-32") + + >>> # Instantiate DDIM scheduler class with same config as DDPM + >>> scheduler = DDIMScheduler.from_config(scheduler.config) + + >>> # Instantiate PNDM scheduler class with same config as DDPM + >>> scheduler = PNDMScheduler.from_config(scheduler.config) + ``` + """ + # <===== TO BE REMOVED WITH DEPRECATION + # TODO(Patrick) - make sure to remove the following lines when config=="model_path" is deprecated + if "pretrained_model_name_or_path" in kwargs: + config = kwargs.pop("pretrained_model_name_or_path") + + if config is None: + raise ValueError("Please make sure to provide a config as the first positional argument.") + # ======> + + if not isinstance(config, dict): + deprecation_message = "It is deprecated to pass a pretrained model name or path to `from_config`." + if "Scheduler" in cls.__name__: + deprecation_message += ( + f"If you were trying to load a scheduler, please use {cls}.from_pretrained(...) instead." + " Otherwise, please make sure to pass a configuration dictionary instead. This functionality will" + " be removed in v1.0.0." + ) + elif "Model" in cls.__name__: + deprecation_message += ( + f"If you were trying to load a model, please use {cls}.load_config(...) followed by" + f" {cls}.from_config(...) instead. Otherwise, please make sure to pass a configuration dictionary" + " instead. This functionality will be removed in v1.0.0." + ) + deprecate("config-passed-as-path", "1.0.0", deprecation_message, standard_warn=False) + config, kwargs = cls.load_config(pretrained_model_name_or_path=config, return_unused_kwargs=True, **kwargs) + + init_dict, unused_kwargs, hidden_dict = cls.extract_init_dict(config, **kwargs) + + # Allow dtype to be specified on initialization + if "dtype" in unused_kwargs: + init_dict["dtype"] = unused_kwargs.pop("dtype") + + # add possible deprecated kwargs + for deprecated_kwarg in cls._deprecated_kwargs: + if deprecated_kwarg in unused_kwargs: + init_dict[deprecated_kwarg] = unused_kwargs.pop(deprecated_kwarg) + + # Return model and optionally state and/or unused_kwargs + model = cls(**init_dict) + + # make sure to also save config parameters that might be used for compatible classes + # update _class_name + if "_class_name" in hidden_dict: + hidden_dict["_class_name"] = cls.__name__ + + model.register_to_config(**hidden_dict) + + # add hidden kwargs of compatible classes to unused_kwargs + unused_kwargs = {**unused_kwargs, **hidden_dict} + + if return_unused_kwargs: + return (model, unused_kwargs) + else: + return model + + @classmethod + def get_config_dict(cls, *args, **kwargs): + deprecation_message = ( + f" The function get_config_dict is deprecated. Please use {cls}.load_config instead. This function will be" + " removed in version v1.0.0" + ) + deprecate("get_config_dict", "1.0.0", deprecation_message, standard_warn=False) + return cls.load_config(*args, **kwargs) + + @classmethod + @validate_hf_hub_args + def load_config( + cls, + pretrained_model_name_or_path: Union[str, os.PathLike], + return_unused_kwargs=False, + return_commit_hash=False, + **kwargs, + ) -> Tuple[Dict[str, Any], Dict[str, Any]]: + r""" + Load a model or scheduler configuration. + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on + the Hub. + - A path to a *directory* (for example `./my_model_directory`) containing model weights saved with + [`~ConfigMixin.save_config`]. + + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + subfolder (`str`, *optional*, defaults to `""`): + The subfolder location of a model file within a larger model repository on the Hub or locally. + return_unused_kwargs (`bool`, *optional*, defaults to `False): + Whether unused keyword arguments of the config are returned. + return_commit_hash (`bool`, *optional*, defaults to `False): + Whether the `commit_hash` of the loaded configuration are returned. + + Returns: + `dict`: + A dictionary of all the parameters stored in a JSON configuration file. + + """ + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + _ = kwargs.pop("mirror", None) + subfolder = kwargs.pop("subfolder", None) + user_agent = kwargs.pop("user_agent", {}) + + user_agent = {**user_agent, "file_type": "config"} + user_agent = http_user_agent(user_agent) + + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + + if cls.config_name is None: + raise ValueError( + "`self.config_name` is not defined. Note that one should not load a config from " + "`ConfigMixin`. Please make sure to define `config_name` in a class inheriting from `ConfigMixin`" + ) + + if os.path.isfile(pretrained_model_name_or_path): + config_file = pretrained_model_name_or_path + elif os.path.isdir(pretrained_model_name_or_path): + if os.path.isfile(os.path.join(pretrained_model_name_or_path, cls.config_name)): + # Load from a PyTorch checkpoint + config_file = os.path.join(pretrained_model_name_or_path, cls.config_name) + elif subfolder is not None and os.path.isfile( + os.path.join(pretrained_model_name_or_path, subfolder, cls.config_name) + ): + config_file = os.path.join(pretrained_model_name_or_path, subfolder, cls.config_name) + else: + raise EnvironmentError( + f"Error no file named {cls.config_name} found in directory {pretrained_model_name_or_path}." + ) + else: + try: + # Load from URL or cache if already cached + config_file = hf_hub_download( + pretrained_model_name_or_path, + filename=cls.config_name, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + token=token, + user_agent=user_agent, + subfolder=subfolder, + revision=revision, + ) + except RepositoryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier" + " listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a" + " token having permission to this repo with `token` or log in with `huggingface-cli login`." + ) + except RevisionNotFoundError: + raise EnvironmentError( + f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for" + " this model name. Check the model page at" + f" 'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions." + ) + except EntryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} does not appear to have a file named {cls.config_name}." + ) + except HTTPError as err: + raise EnvironmentError( + "There was a specific connection error when trying to load" + f" {pretrained_model_name_or_path}:\n{err}" + ) + except ValueError: + raise EnvironmentError( + f"We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it" + f" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a" + f" directory containing a {cls.config_name} file.\nCheckout your internet connection or see how to" + " run the library in offline mode at" + " 'https://huggingface.co/docs/diffusers/installation#offline-mode'." + ) + except EnvironmentError: + raise EnvironmentError( + f"Can't load config for '{pretrained_model_name_or_path}'. If you were trying to load it from " + "'https://huggingface.co/models', make sure you don't have a local directory with the same name. " + f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory " + f"containing a {cls.config_name} file" + ) + + try: + # Load config dict + config_dict = cls._dict_from_json_file(config_file) + + commit_hash = extract_commit_hash(config_file) + except (json.JSONDecodeError, UnicodeDecodeError): + raise EnvironmentError(f"It looks like the config file at '{config_file}' is not a valid JSON file.") + + if not (return_unused_kwargs or return_commit_hash): + return config_dict + + outputs = (config_dict,) + + if return_unused_kwargs: + outputs += (kwargs,) + + if return_commit_hash: + outputs += (commit_hash,) + + return outputs + + @staticmethod + def _get_init_keys(cls): + return set(dict(inspect.signature(cls.__init__).parameters).keys()) + + @classmethod + def extract_init_dict(cls, config_dict, **kwargs): + # Skip keys that were not present in the original config, so default __init__ values were used + used_defaults = config_dict.get("_use_default_values", []) + config_dict = {k: v for k, v in config_dict.items() if k not in used_defaults and k != "_use_default_values"} + + # 0. Copy origin config dict + original_dict = dict(config_dict.items()) + + # 1. Retrieve expected config attributes from __init__ signature + expected_keys = cls._get_init_keys(cls) + expected_keys.remove("self") + # remove general kwargs if present in dict + if "kwargs" in expected_keys: + expected_keys.remove("kwargs") + # remove flax internal keys + if hasattr(cls, "_flax_internal_args"): + for arg in cls._flax_internal_args: + expected_keys.remove(arg) + + # 2. Remove attributes that cannot be expected from expected config attributes + # remove keys to be ignored + if len(cls.ignore_for_config) > 0: + expected_keys = expected_keys - set(cls.ignore_for_config) + + # load diffusers library to import compatible and original scheduler + diffusers_library = importlib.import_module(__name__.split(".")[0]) + + if cls.has_compatibles: + compatible_classes = [c for c in cls._get_compatibles() if not isinstance(c, DummyObject)] + else: + compatible_classes = [] + + expected_keys_comp_cls = set() + for c in compatible_classes: + expected_keys_c = cls._get_init_keys(c) + expected_keys_comp_cls = expected_keys_comp_cls.union(expected_keys_c) + expected_keys_comp_cls = expected_keys_comp_cls - cls._get_init_keys(cls) + config_dict = {k: v for k, v in config_dict.items() if k not in expected_keys_comp_cls} + + # remove attributes from orig class that cannot be expected + orig_cls_name = config_dict.pop("_class_name", cls.__name__) + if ( + isinstance(orig_cls_name, str) + and orig_cls_name != cls.__name__ + and hasattr(diffusers_library, orig_cls_name) + ): + orig_cls = getattr(diffusers_library, orig_cls_name) + unexpected_keys_from_orig = cls._get_init_keys(orig_cls) - expected_keys + config_dict = {k: v for k, v in config_dict.items() if k not in unexpected_keys_from_orig} + elif not isinstance(orig_cls_name, str) and not isinstance(orig_cls_name, (list, tuple)): + raise ValueError( + "Make sure that the `_class_name` is of type string or list of string (for custom pipelines)." + ) + + # remove private attributes + config_dict = {k: v for k, v in config_dict.items() if not k.startswith("_")} + + # 3. Create keyword arguments that will be passed to __init__ from expected keyword arguments + init_dict = {} + for key in expected_keys: + # if config param is passed to kwarg and is present in config dict + # it should overwrite existing config dict key + if key in kwargs and key in config_dict: + config_dict[key] = kwargs.pop(key) + + if key in kwargs: + # overwrite key + init_dict[key] = kwargs.pop(key) + elif key in config_dict: + # use value from config dict + init_dict[key] = config_dict.pop(key) + + # 4. Give nice warning if unexpected values have been passed + if len(config_dict) > 0: + logger.warning( + f"The config attributes {config_dict} were passed to {cls.__name__}, " + "but are not expected and will be ignored. Please verify your " + f"{cls.config_name} configuration file." + ) + + # 5. Give nice info if config attributes are initialized to default because they have not been passed + passed_keys = set(init_dict.keys()) + if len(expected_keys - passed_keys) > 0: + logger.info( + f"{expected_keys - passed_keys} was not found in config. Values will be initialized to default values." + ) + + # 6. Define unused keyword arguments + unused_kwargs = {**config_dict, **kwargs} + + # 7. Define "hidden" config parameters that were saved for compatible classes + hidden_config_dict = {k: v for k, v in original_dict.items() if k not in init_dict} + + return init_dict, unused_kwargs, hidden_config_dict + + @classmethod + def _dict_from_json_file(cls, json_file: Union[str, os.PathLike]): + with open(json_file, "r", encoding="utf-8") as reader: + text = reader.read() + return json.loads(text) + + def __repr__(self): + return f"{self.__class__.__name__} {self.to_json_string()}" + + @property + def config(self) -> Dict[str, Any]: + """ + Returns the config of the class as a frozen dictionary + + Returns: + `Dict[str, Any]`: Config of the class. + """ + return self._internal_dict + + def to_json_string(self) -> str: + """ + Serializes the configuration instance to a JSON string. + + Returns: + `str`: + String containing all the attributes that make up the configuration instance in JSON format. + """ + config_dict = self._internal_dict if hasattr(self, "_internal_dict") else {} + config_dict["_class_name"] = self.__class__.__name__ + config_dict["_diffusers_version"] = __version__ + + def to_json_saveable(value): + if isinstance(value, np.ndarray): + value = value.tolist() + elif isinstance(value, PosixPath): + value = str(value) + return value + + config_dict = {k: to_json_saveable(v) for k, v in config_dict.items()} + # Don't save "_ignore_files" or "_use_default_values" + config_dict.pop("_ignore_files", None) + config_dict.pop("_use_default_values", None) + + return json.dumps(config_dict, indent=2, sort_keys=True) + "\n" + + def to_json_file(self, json_file_path: Union[str, os.PathLike]): + """ + Save the configuration instance's parameters to a JSON file. + + Args: + json_file_path (`str` or `os.PathLike`): + Path to the JSON file to save a configuration instance's parameters. + """ + with open(json_file_path, "w", encoding="utf-8") as writer: + writer.write(self.to_json_string()) + + +def register_to_config(init): + r""" + Decorator to apply on the init of classes inheriting from [`ConfigMixin`] so that all the arguments are + automatically sent to `self.register_for_config`. To ignore a specific argument accepted by the init but that + shouldn't be registered in the config, use the `ignore_for_config` class variable + + Warning: Once decorated, all private arguments (beginning with an underscore) are trashed and not sent to the init! + """ + + @functools.wraps(init) + def inner_init(self, *args, **kwargs): + # Ignore private kwargs in the init. + init_kwargs = {k: v for k, v in kwargs.items() if not k.startswith("_")} + config_init_kwargs = {k: v for k, v in kwargs.items() if k.startswith("_")} + if not isinstance(self, ConfigMixin): + raise RuntimeError( + f"`@register_for_config` was applied to {self.__class__.__name__} init method, but this class does " + "not inherit from `ConfigMixin`." + ) + + ignore = getattr(self, "ignore_for_config", []) + # Get positional arguments aligned with kwargs + new_kwargs = {} + signature = inspect.signature(init) + parameters = { + name: p.default for i, (name, p) in enumerate(signature.parameters.items()) if i > 0 and name not in ignore + } + for arg, name in zip(args, parameters.keys()): + new_kwargs[name] = arg + + # Then add all kwargs + new_kwargs.update( + { + k: init_kwargs.get(k, default) + for k, default in parameters.items() + if k not in ignore and k not in new_kwargs + } + ) + + # Take note of the parameters that were not present in the loaded config + if len(set(new_kwargs.keys()) - set(init_kwargs)) > 0: + new_kwargs["_use_default_values"] = list(set(new_kwargs.keys()) - set(init_kwargs)) + + new_kwargs = {**config_init_kwargs, **new_kwargs} + getattr(self, "register_to_config")(**new_kwargs) + init(self, *args, **init_kwargs) + + return inner_init + + +def flax_register_to_config(cls): + original_init = cls.__init__ + + @functools.wraps(original_init) + def init(self, *args, **kwargs): + if not isinstance(self, ConfigMixin): + raise RuntimeError( + f"`@register_for_config` was applied to {self.__class__.__name__} init method, but this class does " + "not inherit from `ConfigMixin`." + ) + + # Ignore private kwargs in the init. Retrieve all passed attributes + init_kwargs = dict(kwargs.items()) + + # Retrieve default values + fields = dataclasses.fields(self) + default_kwargs = {} + for field in fields: + # ignore flax specific attributes + if field.name in self._flax_internal_args: + continue + if type(field.default) == dataclasses._MISSING_TYPE: + default_kwargs[field.name] = None + else: + default_kwargs[field.name] = getattr(self, field.name) + + # Make sure init_kwargs override default kwargs + new_kwargs = {**default_kwargs, **init_kwargs} + # dtype should be part of `init_kwargs`, but not `new_kwargs` + if "dtype" in new_kwargs: + new_kwargs.pop("dtype") + + # Get positional arguments aligned with kwargs + for i, arg in enumerate(args): + name = fields[i].name + new_kwargs[name] = arg + + # Take note of the parameters that were not present in the loaded config + if len(set(new_kwargs.keys()) - set(init_kwargs)) > 0: + new_kwargs["_use_default_values"] = list(set(new_kwargs.keys()) - set(init_kwargs)) + + getattr(self, "register_to_config")(**new_kwargs) + original_init(self, *args, **kwargs) + + cls.__init__ = init + return cls diff --git a/diffusers-0.27.0/src/diffusers/dependency_versions_check.py b/diffusers-0.27.0/src/diffusers/dependency_versions_check.py new file mode 100755 index 0000000..0728b3a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/dependency_versions_check.py @@ -0,0 +1,34 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from .dependency_versions_table import deps +from .utils.versions import require_version, require_version_core + + +# define which module versions we always want to check at run time +# (usually the ones defined in `install_requires` in setup.py) +# +# order specific notes: +# - tqdm must be checked before tokenizers + +pkgs_to_check_at_runtime = "python requests filelock numpy".split() +for pkg in pkgs_to_check_at_runtime: + if pkg in deps: + require_version_core(deps[pkg]) + else: + raise ValueError(f"can't find {pkg} in {deps.keys()}, check dependency_versions_table.py") + + +def dep_version_check(pkg, hint=None): + require_version(deps[pkg], hint) diff --git a/diffusers-0.27.0/src/diffusers/dependency_versions_table.py b/diffusers-0.27.0/src/diffusers/dependency_versions_table.py new file mode 100755 index 0000000..e92a486 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/dependency_versions_table.py @@ -0,0 +1,45 @@ +# THIS FILE HAS BEEN AUTOGENERATED. To update: +# 1. modify the `_deps` dict in setup.py +# 2. run `make deps_table_update` +deps = { + "Pillow": "Pillow", + "accelerate": "accelerate>=0.11.0", + "compel": "compel==0.1.8", + "datasets": "datasets", + "filelock": "filelock", + "flax": "flax>=0.4.1", + "hf-doc-builder": "hf-doc-builder>=0.3.0", + "huggingface-hub": "huggingface-hub>=0.20.2", + "requests-mock": "requests-mock==1.10.0", + "importlib_metadata": "importlib_metadata", + "invisible-watermark": "invisible-watermark>=0.2.0", + "isort": "isort>=5.5.4", + "jax": "jax>=0.4.1", + "jaxlib": "jaxlib>=0.4.1", + "Jinja2": "Jinja2", + "k-diffusion": "k-diffusion>=0.0.12", + "torchsde": "torchsde", + "note_seq": "note_seq", + "librosa": "librosa", + "numpy": "numpy", + "parameterized": "parameterized", + "peft": "peft>=0.6.0", + "protobuf": "protobuf>=3.20.3,<4", + "pytest": "pytest", + "pytest-timeout": "pytest-timeout", + "pytest-xdist": "pytest-xdist", + "python": "python>=3.8.0", + "ruff": "ruff==0.1.5", + "safetensors": "safetensors>=0.3.1", + "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92", + "GitPython": "GitPython<3.1.19", + "scipy": "scipy", + "onnx": "onnx", + "regex": "regex!=2019.12.17", + "requests": "requests", + "tensorboard": "tensorboard", + "torch": "torch>=1.4", + "torchvision": "torchvision", + "transformers": "transformers>=4.25.1", + "urllib3": "urllib3<=2.0.0", +} diff --git a/diffusers-0.27.0/src/diffusers/experimental/README.md b/diffusers-0.27.0/src/diffusers/experimental/README.md new file mode 100755 index 0000000..81a9de8 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/experimental/README.md @@ -0,0 +1,5 @@ +# 🧨 Diffusers Experimental + +We are adding experimental code to support novel applications and usages of the Diffusers library. +Currently, the following experiments are supported: +* Reinforcement learning via an implementation of the [Diffuser](https://arxiv.org/abs/2205.09991) model. \ No newline at end of file diff --git a/diffusers-0.27.0/src/diffusers/experimental/__init__.py b/diffusers-0.27.0/src/diffusers/experimental/__init__.py new file mode 100755 index 0000000..ebc8155 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/experimental/__init__.py @@ -0,0 +1 @@ +from .rl import ValueGuidedRLPipeline diff --git a/diffusers-0.27.0/src/diffusers/experimental/rl/__init__.py b/diffusers-0.27.0/src/diffusers/experimental/rl/__init__.py new file mode 100755 index 0000000..7b338d3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/experimental/rl/__init__.py @@ -0,0 +1 @@ +from .value_guided_sampling import ValueGuidedRLPipeline diff --git a/diffusers-0.27.0/src/diffusers/experimental/rl/value_guided_sampling.py b/diffusers-0.27.0/src/diffusers/experimental/rl/value_guided_sampling.py new file mode 100755 index 0000000..2f9de85 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/experimental/rl/value_guided_sampling.py @@ -0,0 +1,153 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import torch +import tqdm + +from ...models.unets.unet_1d import UNet1DModel +from ...pipelines import DiffusionPipeline +from ...utils.dummy_pt_objects import DDPMScheduler +from ...utils.torch_utils import randn_tensor + + +class ValueGuidedRLPipeline(DiffusionPipeline): + r""" + Pipeline for value-guided sampling from a diffusion model trained to predict sequences of states. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + value_function ([`UNet1DModel`]): + A specialized UNet for fine-tuning trajectories base on reward. + unet ([`UNet1DModel`]): + UNet architecture to denoise the encoded trajectories. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded trajectories. Default for this + application is [`DDPMScheduler`]. + env (): + An environment following the OpenAI gym API to act in. For now only Hopper has pretrained models. + """ + + def __init__( + self, + value_function: UNet1DModel, + unet: UNet1DModel, + scheduler: DDPMScheduler, + env, + ): + super().__init__() + + self.register_modules(value_function=value_function, unet=unet, scheduler=scheduler, env=env) + + self.data = env.get_dataset() + self.means = {} + for key in self.data.keys(): + try: + self.means[key] = self.data[key].mean() + except: # noqa: E722 + pass + self.stds = {} + for key in self.data.keys(): + try: + self.stds[key] = self.data[key].std() + except: # noqa: E722 + pass + self.state_dim = env.observation_space.shape[0] + self.action_dim = env.action_space.shape[0] + + def normalize(self, x_in, key): + return (x_in - self.means[key]) / self.stds[key] + + def de_normalize(self, x_in, key): + return x_in * self.stds[key] + self.means[key] + + def to_torch(self, x_in): + if isinstance(x_in, dict): + return {k: self.to_torch(v) for k, v in x_in.items()} + elif torch.is_tensor(x_in): + return x_in.to(self.unet.device) + return torch.tensor(x_in, device=self.unet.device) + + def reset_x0(self, x_in, cond, act_dim): + for key, val in cond.items(): + x_in[:, key, act_dim:] = val.clone() + return x_in + + def run_diffusion(self, x, conditions, n_guide_steps, scale): + batch_size = x.shape[0] + y = None + for i in tqdm.tqdm(self.scheduler.timesteps): + # create batch of timesteps to pass into model + timesteps = torch.full((batch_size,), i, device=self.unet.device, dtype=torch.long) + for _ in range(n_guide_steps): + with torch.enable_grad(): + x.requires_grad_() + + # permute to match dimension for pre-trained models + y = self.value_function(x.permute(0, 2, 1), timesteps).sample + grad = torch.autograd.grad([y.sum()], [x])[0] + + posterior_variance = self.scheduler._get_variance(i) + model_std = torch.exp(0.5 * posterior_variance) + grad = model_std * grad + + grad[timesteps < 2] = 0 + x = x.detach() + x = x + scale * grad + x = self.reset_x0(x, conditions, self.action_dim) + + prev_x = self.unet(x.permute(0, 2, 1), timesteps).sample.permute(0, 2, 1) + + # TODO: verify deprecation of this kwarg + x = self.scheduler.step(prev_x, i, x)["prev_sample"] + + # apply conditions to the trajectory (set the initial state) + x = self.reset_x0(x, conditions, self.action_dim) + x = self.to_torch(x) + return x, y + + def __call__(self, obs, batch_size=64, planning_horizon=32, n_guide_steps=2, scale=0.1): + # normalize the observations and create batch dimension + obs = self.normalize(obs, "observations") + obs = obs[None].repeat(batch_size, axis=0) + + conditions = {0: self.to_torch(obs)} + shape = (batch_size, planning_horizon, self.state_dim + self.action_dim) + + # generate initial noise and apply our conditions (to make the trajectories start at current state) + x1 = randn_tensor(shape, device=self.unet.device) + x = self.reset_x0(x1, conditions, self.action_dim) + x = self.to_torch(x) + + # run the diffusion process + x, y = self.run_diffusion(x, conditions, n_guide_steps, scale) + + # sort output trajectories by value + sorted_idx = y.argsort(0, descending=True).squeeze() + sorted_values = x[sorted_idx] + actions = sorted_values[:, :, : self.action_dim] + actions = actions.detach().cpu().numpy() + denorm_actions = self.de_normalize(actions, key="actions") + + # select the action with the highest value + if y is not None: + selected_index = 0 + else: + # if we didn't run value guiding, select a random action + selected_index = np.random.randint(0, batch_size) + + denorm_actions = denorm_actions[selected_index, 0] + return denorm_actions diff --git a/diffusers-0.27.0/src/diffusers/image_processor.py b/diffusers-0.27.0/src/diffusers/image_processor.py new file mode 100755 index 0000000..daeb8fd --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/image_processor.py @@ -0,0 +1,990 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +import warnings +from typing import List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from PIL import Image, ImageFilter, ImageOps + +from .configuration_utils import ConfigMixin, register_to_config +from .utils import CONFIG_NAME, PIL_INTERPOLATION, deprecate + + +PipelineImageInput = Union[ + PIL.Image.Image, + np.ndarray, + torch.FloatTensor, + List[PIL.Image.Image], + List[np.ndarray], + List[torch.FloatTensor], +] + +PipelineDepthInput = PipelineImageInput + + +class VaeImageProcessor(ConfigMixin): + """ + Image processor for VAE. + + Args: + do_resize (`bool`, *optional*, defaults to `True`): + Whether to downscale the image's (height, width) dimensions to multiples of `vae_scale_factor`. Can accept + `height` and `width` arguments from [`image_processor.VaeImageProcessor.preprocess`] method. + vae_scale_factor (`int`, *optional*, defaults to `8`): + VAE scale factor. If `do_resize` is `True`, the image is automatically resized to multiples of this factor. + resample (`str`, *optional*, defaults to `lanczos`): + Resampling filter to use when resizing the image. + do_normalize (`bool`, *optional*, defaults to `True`): + Whether to normalize the image to [-1,1]. + do_binarize (`bool`, *optional*, defaults to `False`): + Whether to binarize the image to 0/1. + do_convert_rgb (`bool`, *optional*, defaults to be `False`): + Whether to convert the images to RGB format. + do_convert_grayscale (`bool`, *optional*, defaults to be `False`): + Whether to convert the images to grayscale format. + """ + + config_name = CONFIG_NAME + + @register_to_config + def __init__( + self, + do_resize: bool = True, + vae_scale_factor: int = 8, + resample: str = "lanczos", + do_normalize: bool = True, + do_binarize: bool = False, + do_convert_rgb: bool = False, + do_convert_grayscale: bool = False, + ): + super().__init__() + if do_convert_rgb and do_convert_grayscale: + raise ValueError( + "`do_convert_rgb` and `do_convert_grayscale` can not both be set to `True`," + " if you intended to convert the image into RGB format, please set `do_convert_grayscale = False`.", + " if you intended to convert the image into grayscale format, please set `do_convert_rgb = False`", + ) + self.config.do_convert_rgb = False + + @staticmethod + def numpy_to_pil(images: np.ndarray) -> List[PIL.Image.Image]: + """ + Convert a numpy image or a batch of images to a PIL image. + """ + if images.ndim == 3: + images = images[None, ...] + images = (images * 255).round().astype("uint8") + if images.shape[-1] == 1: + # special case for grayscale (single channel) images + pil_images = [Image.fromarray(image.squeeze(), mode="L") for image in images] + else: + pil_images = [Image.fromarray(image) for image in images] + + return pil_images + + @staticmethod + def pil_to_numpy(images: Union[List[PIL.Image.Image], PIL.Image.Image]) -> np.ndarray: + """ + Convert a PIL image or a list of PIL images to NumPy arrays. + """ + if not isinstance(images, list): + images = [images] + images = [np.array(image).astype(np.float32) / 255.0 for image in images] + images = np.stack(images, axis=0) + + return images + + @staticmethod + def numpy_to_pt(images: np.ndarray) -> torch.FloatTensor: + """ + Convert a NumPy image to a PyTorch tensor. + """ + if images.ndim == 3: + images = images[..., None] + + images = torch.from_numpy(images.transpose(0, 3, 1, 2)) + return images + + @staticmethod + def pt_to_numpy(images: torch.FloatTensor) -> np.ndarray: + """ + Convert a PyTorch tensor to a NumPy image. + """ + images = images.cpu().permute(0, 2, 3, 1).float().numpy() + return images + + @staticmethod + def normalize(images: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]: + """ + Normalize an image array to [-1,1]. + """ + return 2.0 * images - 1.0 + + @staticmethod + def denormalize(images: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]: + """ + Denormalize an image array to [0,1]. + """ + return (images / 2 + 0.5).clamp(0, 1) + + @staticmethod + def convert_to_rgb(image: PIL.Image.Image) -> PIL.Image.Image: + """ + Converts a PIL image to RGB format. + """ + image = image.convert("RGB") + + return image + + @staticmethod + def convert_to_grayscale(image: PIL.Image.Image) -> PIL.Image.Image: + """ + Converts a PIL image to grayscale format. + """ + image = image.convert("L") + + return image + + @staticmethod + def blur(image: PIL.Image.Image, blur_factor: int = 4) -> PIL.Image.Image: + """ + Applies Gaussian blur to an image. + """ + image = image.filter(ImageFilter.GaussianBlur(blur_factor)) + + return image + + @staticmethod + def get_crop_region(mask_image: PIL.Image.Image, width: int, height: int, pad=0): + """ + Finds a rectangular region that contains all masked ares in an image, and expands region to match the aspect ratio of the original image; + for example, if user drew mask in a 128x32 region, and the dimensions for processing are 512x512, the region will be expanded to 128x128. + + Args: + mask_image (PIL.Image.Image): Mask image. + width (int): Width of the image to be processed. + height (int): Height of the image to be processed. + pad (int, optional): Padding to be added to the crop region. Defaults to 0. + + Returns: + tuple: (x1, y1, x2, y2) represent a rectangular region that contains all masked ares in an image and matches the original aspect ratio. + """ + + mask_image = mask_image.convert("L") + mask = np.array(mask_image) + + # 1. find a rectangular region that contains all masked ares in an image + h, w = mask.shape + crop_left = 0 + for i in range(w): + if not (mask[:, i] == 0).all(): + break + crop_left += 1 + + crop_right = 0 + for i in reversed(range(w)): + if not (mask[:, i] == 0).all(): + break + crop_right += 1 + + crop_top = 0 + for i in range(h): + if not (mask[i] == 0).all(): + break + crop_top += 1 + + crop_bottom = 0 + for i in reversed(range(h)): + if not (mask[i] == 0).all(): + break + crop_bottom += 1 + + # 2. add padding to the crop region + x1, y1, x2, y2 = ( + int(max(crop_left - pad, 0)), + int(max(crop_top - pad, 0)), + int(min(w - crop_right + pad, w)), + int(min(h - crop_bottom + pad, h)), + ) + + # 3. expands crop region to match the aspect ratio of the image to be processed + ratio_crop_region = (x2 - x1) / (y2 - y1) + ratio_processing = width / height + + if ratio_crop_region > ratio_processing: + desired_height = (x2 - x1) / ratio_processing + desired_height_diff = int(desired_height - (y2 - y1)) + y1 -= desired_height_diff // 2 + y2 += desired_height_diff - desired_height_diff // 2 + if y2 >= mask_image.height: + diff = y2 - mask_image.height + y2 -= diff + y1 -= diff + if y1 < 0: + y2 -= y1 + y1 -= y1 + if y2 >= mask_image.height: + y2 = mask_image.height + else: + desired_width = (y2 - y1) * ratio_processing + desired_width_diff = int(desired_width - (x2 - x1)) + x1 -= desired_width_diff // 2 + x2 += desired_width_diff - desired_width_diff // 2 + if x2 >= mask_image.width: + diff = x2 - mask_image.width + x2 -= diff + x1 -= diff + if x1 < 0: + x2 -= x1 + x1 -= x1 + if x2 >= mask_image.width: + x2 = mask_image.width + + return x1, y1, x2, y2 + + def _resize_and_fill( + self, + image: PIL.Image.Image, + width: int, + height: int, + ) -> PIL.Image.Image: + """ + Resize the image to fit within the specified width and height, maintaining the aspect ratio, and then center the image within the dimensions, filling empty with data from image. + + Args: + image: The image to resize. + width: The width to resize the image to. + height: The height to resize the image to. + """ + + ratio = width / height + src_ratio = image.width / image.height + + src_w = width if ratio < src_ratio else image.width * height // image.height + src_h = height if ratio >= src_ratio else image.height * width // image.width + + resized = image.resize((src_w, src_h), resample=PIL_INTERPOLATION["lanczos"]) + res = Image.new("RGB", (width, height)) + res.paste(resized, box=(width // 2 - src_w // 2, height // 2 - src_h // 2)) + + if ratio < src_ratio: + fill_height = height // 2 - src_h // 2 + if fill_height > 0: + res.paste(resized.resize((width, fill_height), box=(0, 0, width, 0)), box=(0, 0)) + res.paste( + resized.resize((width, fill_height), box=(0, resized.height, width, resized.height)), + box=(0, fill_height + src_h), + ) + elif ratio > src_ratio: + fill_width = width // 2 - src_w // 2 + if fill_width > 0: + res.paste(resized.resize((fill_width, height), box=(0, 0, 0, height)), box=(0, 0)) + res.paste( + resized.resize((fill_width, height), box=(resized.width, 0, resized.width, height)), + box=(fill_width + src_w, 0), + ) + + return res + + def _resize_and_crop( + self, + image: PIL.Image.Image, + width: int, + height: int, + ) -> PIL.Image.Image: + """ + Resize the image to fit within the specified width and height, maintaining the aspect ratio, and then center the image within the dimensions, cropping the excess. + + Args: + image: The image to resize. + width: The width to resize the image to. + height: The height to resize the image to. + """ + ratio = width / height + src_ratio = image.width / image.height + + src_w = width if ratio > src_ratio else image.width * height // image.height + src_h = height if ratio <= src_ratio else image.height * width // image.width + + resized = image.resize((src_w, src_h), resample=PIL_INTERPOLATION["lanczos"]) + res = Image.new("RGB", (width, height)) + res.paste(resized, box=(width // 2 - src_w // 2, height // 2 - src_h // 2)) + return res + + def resize( + self, + image: Union[PIL.Image.Image, np.ndarray, torch.Tensor], + height: int, + width: int, + resize_mode: str = "default", # "default", "fill", "crop" + ) -> Union[PIL.Image.Image, np.ndarray, torch.Tensor]: + """ + Resize image. + + Args: + image (`PIL.Image.Image`, `np.ndarray` or `torch.Tensor`): + The image input, can be a PIL image, numpy array or pytorch tensor. + height (`int`): + The height to resize to. + width (`int`): + The width to resize to. + resize_mode (`str`, *optional*, defaults to `default`): + The resize mode to use, can be one of `default` or `fill`. If `default`, will resize the image to fit + within the specified width and height, and it may not maintaining the original aspect ratio. + If `fill`, will resize the image to fit within the specified width and height, maintaining the aspect ratio, and then center the image + within the dimensions, filling empty with data from image. + If `crop`, will resize the image to fit within the specified width and height, maintaining the aspect ratio, and then center the image + within the dimensions, cropping the excess. + Note that resize_mode `fill` and `crop` are only supported for PIL image input. + + Returns: + `PIL.Image.Image`, `np.ndarray` or `torch.Tensor`: + The resized image. + """ + if resize_mode != "default" and not isinstance(image, PIL.Image.Image): + raise ValueError(f"Only PIL image input is supported for resize_mode {resize_mode}") + if isinstance(image, PIL.Image.Image): + if resize_mode == "default": + image = image.resize((width, height), resample=PIL_INTERPOLATION[self.config.resample]) + elif resize_mode == "fill": + image = self._resize_and_fill(image, width, height) + elif resize_mode == "crop": + image = self._resize_and_crop(image, width, height) + else: + raise ValueError(f"resize_mode {resize_mode} is not supported") + + elif isinstance(image, torch.Tensor): + image = torch.nn.functional.interpolate( + image, + size=(height, width), + ) + elif isinstance(image, np.ndarray): + image = self.numpy_to_pt(image) + image = torch.nn.functional.interpolate( + image, + size=(height, width), + ) + image = self.pt_to_numpy(image) + return image + + def binarize(self, image: PIL.Image.Image) -> PIL.Image.Image: + """ + Create a mask. + + Args: + image (`PIL.Image.Image`): + The image input, should be a PIL image. + + Returns: + `PIL.Image.Image`: + The binarized image. Values less than 0.5 are set to 0, values greater than 0.5 are set to 1. + """ + image[image < 0.5] = 0 + image[image >= 0.5] = 1 + + return image + + def get_default_height_width( + self, + image: Union[PIL.Image.Image, np.ndarray, torch.Tensor], + height: Optional[int] = None, + width: Optional[int] = None, + ) -> Tuple[int, int]: + """ + This function return the height and width that are downscaled to the next integer multiple of + `vae_scale_factor`. + + Args: + image(`PIL.Image.Image`, `np.ndarray` or `torch.Tensor`): + The image input, can be a PIL image, numpy array or pytorch tensor. if it is a numpy array, should have + shape `[batch, height, width]` or `[batch, height, width, channel]` if it is a pytorch tensor, should + have shape `[batch, channel, height, width]`. + height (`int`, *optional*, defaults to `None`): + The height in preprocessed image. If `None`, will use the height of `image` input. + width (`int`, *optional*`, defaults to `None`): + The width in preprocessed. If `None`, will use the width of the `image` input. + """ + + if height is None: + if isinstance(image, PIL.Image.Image): + height = image.height + elif isinstance(image, torch.Tensor): + height = image.shape[2] + else: + height = image.shape[1] + + if width is None: + if isinstance(image, PIL.Image.Image): + width = image.width + elif isinstance(image, torch.Tensor): + width = image.shape[3] + else: + width = image.shape[2] + + width, height = ( + x - x % self.config.vae_scale_factor for x in (width, height) + ) # resize to integer multiple of vae_scale_factor + + return height, width + + def preprocess( + self, + image: PipelineImageInput, + height: Optional[int] = None, + width: Optional[int] = None, + resize_mode: str = "default", # "default", "fill", "crop" + crops_coords: Optional[Tuple[int, int, int, int]] = None, + ) -> torch.Tensor: + """ + Preprocess the image input. + + Args: + image (`pipeline_image_input`): + The image input, accepted formats are PIL images, NumPy arrays, PyTorch tensors; Also accept list of supported formats. + height (`int`, *optional*, defaults to `None`): + The height in preprocessed image. If `None`, will use the `get_default_height_width()` to get default height. + width (`int`, *optional*`, defaults to `None`): + The width in preprocessed. If `None`, will use get_default_height_width()` to get the default width. + resize_mode (`str`, *optional*, defaults to `default`): + The resize mode, can be one of `default` or `fill`. If `default`, will resize the image to fit + within the specified width and height, and it may not maintaining the original aspect ratio. + If `fill`, will resize the image to fit within the specified width and height, maintaining the aspect ratio, and then center the image + within the dimensions, filling empty with data from image. + If `crop`, will resize the image to fit within the specified width and height, maintaining the aspect ratio, and then center the image + within the dimensions, cropping the excess. + Note that resize_mode `fill` and `crop` are only supported for PIL image input. + crops_coords (`List[Tuple[int, int, int, int]]`, *optional*, defaults to `None`): + The crop coordinates for each image in the batch. If `None`, will not crop the image. + """ + supported_formats = (PIL.Image.Image, np.ndarray, torch.Tensor) + + # Expand the missing dimension for 3-dimensional pytorch tensor or numpy array that represents grayscale image + if self.config.do_convert_grayscale and isinstance(image, (torch.Tensor, np.ndarray)) and image.ndim == 3: + if isinstance(image, torch.Tensor): + # if image is a pytorch tensor could have 2 possible shapes: + # 1. batch x height x width: we should insert the channel dimension at position 1 + # 2. channel x height x width: we should insert batch dimension at position 0, + # however, since both channel and batch dimension has same size 1, it is same to insert at position 1 + # for simplicity, we insert a dimension of size 1 at position 1 for both cases + image = image.unsqueeze(1) + else: + # if it is a numpy array, it could have 2 possible shapes: + # 1. batch x height x width: insert channel dimension on last position + # 2. height x width x channel: insert batch dimension on first position + if image.shape[-1] == 1: + image = np.expand_dims(image, axis=0) + else: + image = np.expand_dims(image, axis=-1) + + if isinstance(image, supported_formats): + image = [image] + elif not (isinstance(image, list) and all(isinstance(i, supported_formats) for i in image)): + raise ValueError( + f"Input is in incorrect format: {[type(i) for i in image]}. Currently, we only support {', '.join(supported_formats)}" + ) + + if isinstance(image[0], PIL.Image.Image): + if crops_coords is not None: + image = [i.crop(crops_coords) for i in image] + if self.config.do_resize: + height, width = self.get_default_height_width(image[0], height, width) + image = [self.resize(i, height, width, resize_mode=resize_mode) for i in image] + if self.config.do_convert_rgb: + image = [self.convert_to_rgb(i) for i in image] + elif self.config.do_convert_grayscale: + image = [self.convert_to_grayscale(i) for i in image] + image = self.pil_to_numpy(image) # to np + image = self.numpy_to_pt(image) # to pt + + elif isinstance(image[0], np.ndarray): + image = np.concatenate(image, axis=0) if image[0].ndim == 4 else np.stack(image, axis=0) + + image = self.numpy_to_pt(image) + + height, width = self.get_default_height_width(image, height, width) + if self.config.do_resize: + image = self.resize(image, height, width) + + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, axis=0) if image[0].ndim == 4 else torch.stack(image, axis=0) + + if self.config.do_convert_grayscale and image.ndim == 3: + image = image.unsqueeze(1) + + channel = image.shape[1] + # don't need any preprocess if the image is latents + if channel == 4: + return image + + height, width = self.get_default_height_width(image, height, width) + if self.config.do_resize: + image = self.resize(image, height, width) + + # expected range [0,1], normalize to [-1,1] + do_normalize = self.config.do_normalize + if do_normalize and image.min() < 0: + warnings.warn( + "Passing `image` as torch tensor with value range in [-1,1] is deprecated. The expected value range for image tensor is [0,1] " + f"when passing as pytorch tensor or numpy Array. You passed `image` with value range [{image.min()},{image.max()}]", + FutureWarning, + ) + do_normalize = False + + if do_normalize: + image = self.normalize(image) + + if self.config.do_binarize: + image = self.binarize(image) + + return image + + def postprocess( + self, + image: torch.FloatTensor, + output_type: str = "pil", + do_denormalize: Optional[List[bool]] = None, + ) -> Union[PIL.Image.Image, np.ndarray, torch.FloatTensor]: + """ + Postprocess the image output from tensor to `output_type`. + + Args: + image (`torch.FloatTensor`): + The image input, should be a pytorch tensor with shape `B x C x H x W`. + output_type (`str`, *optional*, defaults to `pil`): + The output type of the image, can be one of `pil`, `np`, `pt`, `latent`. + do_denormalize (`List[bool]`, *optional*, defaults to `None`): + Whether to denormalize the image to [0,1]. If `None`, will use the value of `do_normalize` in the + `VaeImageProcessor` config. + + Returns: + `PIL.Image.Image`, `np.ndarray` or `torch.FloatTensor`: + The postprocessed image. + """ + if not isinstance(image, torch.Tensor): + raise ValueError( + f"Input for postprocessing is in incorrect format: {type(image)}. We only support pytorch tensor" + ) + if output_type not in ["latent", "pt", "np", "pil"]: + deprecation_message = ( + f"the output_type {output_type} is outdated and has been set to `np`. Please make sure to set it to one of these instead: " + "`pil`, `np`, `pt`, `latent`" + ) + deprecate("Unsupported output_type", "1.0.0", deprecation_message, standard_warn=False) + output_type = "np" + + if output_type == "latent": + return image + + if do_denormalize is None: + do_denormalize = [self.config.do_normalize] * image.shape[0] + + image = torch.stack( + [self.denormalize(image[i]) if do_denormalize[i] else image[i] for i in range(image.shape[0])] + ) + + if output_type == "pt": + return image + + image = self.pt_to_numpy(image) + + if output_type == "np": + return image + + if output_type == "pil": + return self.numpy_to_pil(image) + + def apply_overlay( + self, + mask: PIL.Image.Image, + init_image: PIL.Image.Image, + image: PIL.Image.Image, + crop_coords: Optional[Tuple[int, int, int, int]] = None, + ) -> PIL.Image.Image: + """ + overlay the inpaint output to the original image + """ + + width, height = image.width, image.height + + init_image = self.resize(init_image, width=width, height=height) + mask = self.resize(mask, width=width, height=height) + + init_image_masked = PIL.Image.new("RGBa", (width, height)) + init_image_masked.paste(init_image.convert("RGBA").convert("RGBa"), mask=ImageOps.invert(mask.convert("L"))) + init_image_masked = init_image_masked.convert("RGBA") + + if crop_coords is not None: + x, y, x2, y2 = crop_coords + w = x2 - x + h = y2 - y + base_image = PIL.Image.new("RGBA", (width, height)) + image = self.resize(image, height=h, width=w, resize_mode="crop") + base_image.paste(image, (x, y)) + image = base_image.convert("RGB") + + image = image.convert("RGBA") + image.alpha_composite(init_image_masked) + image = image.convert("RGB") + + return image + + +class VaeImageProcessorLDM3D(VaeImageProcessor): + """ + Image processor for VAE LDM3D. + + Args: + do_resize (`bool`, *optional*, defaults to `True`): + Whether to downscale the image's (height, width) dimensions to multiples of `vae_scale_factor`. + vae_scale_factor (`int`, *optional*, defaults to `8`): + VAE scale factor. If `do_resize` is `True`, the image is automatically resized to multiples of this factor. + resample (`str`, *optional*, defaults to `lanczos`): + Resampling filter to use when resizing the image. + do_normalize (`bool`, *optional*, defaults to `True`): + Whether to normalize the image to [-1,1]. + """ + + config_name = CONFIG_NAME + + @register_to_config + def __init__( + self, + do_resize: bool = True, + vae_scale_factor: int = 8, + resample: str = "lanczos", + do_normalize: bool = True, + ): + super().__init__() + + @staticmethod + def numpy_to_pil(images: np.ndarray) -> List[PIL.Image.Image]: + """ + Convert a NumPy image or a batch of images to a PIL image. + """ + if images.ndim == 3: + images = images[None, ...] + images = (images * 255).round().astype("uint8") + if images.shape[-1] == 1: + # special case for grayscale (single channel) images + pil_images = [Image.fromarray(image.squeeze(), mode="L") for image in images] + else: + pil_images = [Image.fromarray(image[:, :, :3]) for image in images] + + return pil_images + + @staticmethod + def depth_pil_to_numpy(images: Union[List[PIL.Image.Image], PIL.Image.Image]) -> np.ndarray: + """ + Convert a PIL image or a list of PIL images to NumPy arrays. + """ + if not isinstance(images, list): + images = [images] + + images = [np.array(image).astype(np.float32) / (2**16 - 1) for image in images] + images = np.stack(images, axis=0) + return images + + @staticmethod + def rgblike_to_depthmap(image: Union[np.ndarray, torch.Tensor]) -> Union[np.ndarray, torch.Tensor]: + """ + Args: + image: RGB-like depth image + + Returns: depth map + + """ + return image[:, :, 1] * 2**8 + image[:, :, 2] + + def numpy_to_depth(self, images: np.ndarray) -> List[PIL.Image.Image]: + """ + Convert a NumPy depth image or a batch of images to a PIL image. + """ + if images.ndim == 3: + images = images[None, ...] + images_depth = images[:, :, :, 3:] + if images.shape[-1] == 6: + images_depth = (images_depth * 255).round().astype("uint8") + pil_images = [ + Image.fromarray(self.rgblike_to_depthmap(image_depth), mode="I;16") for image_depth in images_depth + ] + elif images.shape[-1] == 4: + images_depth = (images_depth * 65535.0).astype(np.uint16) + pil_images = [Image.fromarray(image_depth, mode="I;16") for image_depth in images_depth] + else: + raise Exception("Not supported") + + return pil_images + + def postprocess( + self, + image: torch.FloatTensor, + output_type: str = "pil", + do_denormalize: Optional[List[bool]] = None, + ) -> Union[PIL.Image.Image, np.ndarray, torch.FloatTensor]: + """ + Postprocess the image output from tensor to `output_type`. + + Args: + image (`torch.FloatTensor`): + The image input, should be a pytorch tensor with shape `B x C x H x W`. + output_type (`str`, *optional*, defaults to `pil`): + The output type of the image, can be one of `pil`, `np`, `pt`, `latent`. + do_denormalize (`List[bool]`, *optional*, defaults to `None`): + Whether to denormalize the image to [0,1]. If `None`, will use the value of `do_normalize` in the + `VaeImageProcessor` config. + + Returns: + `PIL.Image.Image`, `np.ndarray` or `torch.FloatTensor`: + The postprocessed image. + """ + if not isinstance(image, torch.Tensor): + raise ValueError( + f"Input for postprocessing is in incorrect format: {type(image)}. We only support pytorch tensor" + ) + if output_type not in ["latent", "pt", "np", "pil"]: + deprecation_message = ( + f"the output_type {output_type} is outdated and has been set to `np`. Please make sure to set it to one of these instead: " + "`pil`, `np`, `pt`, `latent`" + ) + deprecate("Unsupported output_type", "1.0.0", deprecation_message, standard_warn=False) + output_type = "np" + + if do_denormalize is None: + do_denormalize = [self.config.do_normalize] * image.shape[0] + + image = torch.stack( + [self.denormalize(image[i]) if do_denormalize[i] else image[i] for i in range(image.shape[0])] + ) + + image = self.pt_to_numpy(image) + + if output_type == "np": + if image.shape[-1] == 6: + image_depth = np.stack([self.rgblike_to_depthmap(im[:, :, 3:]) for im in image], axis=0) + else: + image_depth = image[:, :, :, 3:] + return image[:, :, :, :3], image_depth + + if output_type == "pil": + return self.numpy_to_pil(image), self.numpy_to_depth(image) + else: + raise Exception(f"This type {output_type} is not supported") + + def preprocess( + self, + rgb: Union[torch.FloatTensor, PIL.Image.Image, np.ndarray], + depth: Union[torch.FloatTensor, PIL.Image.Image, np.ndarray], + height: Optional[int] = None, + width: Optional[int] = None, + target_res: Optional[int] = None, + ) -> torch.Tensor: + """ + Preprocess the image input. Accepted formats are PIL images, NumPy arrays or PyTorch tensors. + """ + supported_formats = (PIL.Image.Image, np.ndarray, torch.Tensor) + + # Expand the missing dimension for 3-dimensional pytorch tensor or numpy array that represents grayscale image + if self.config.do_convert_grayscale and isinstance(rgb, (torch.Tensor, np.ndarray)) and rgb.ndim == 3: + raise Exception("This is not yet supported") + + if isinstance(rgb, supported_formats): + rgb = [rgb] + depth = [depth] + elif not (isinstance(rgb, list) and all(isinstance(i, supported_formats) for i in rgb)): + raise ValueError( + f"Input is in incorrect format: {[type(i) for i in rgb]}. Currently, we only support {', '.join(supported_formats)}" + ) + + if isinstance(rgb[0], PIL.Image.Image): + if self.config.do_convert_rgb: + raise Exception("This is not yet supported") + # rgb = [self.convert_to_rgb(i) for i in rgb] + # depth = [self.convert_to_depth(i) for i in depth] #TODO define convert_to_depth + if self.config.do_resize or target_res: + height, width = self.get_default_height_width(rgb[0], height, width) if not target_res else target_res + rgb = [self.resize(i, height, width) for i in rgb] + depth = [self.resize(i, height, width) for i in depth] + rgb = self.pil_to_numpy(rgb) # to np + rgb = self.numpy_to_pt(rgb) # to pt + + depth = self.depth_pil_to_numpy(depth) # to np + depth = self.numpy_to_pt(depth) # to pt + + elif isinstance(rgb[0], np.ndarray): + rgb = np.concatenate(rgb, axis=0) if rgb[0].ndim == 4 else np.stack(rgb, axis=0) + rgb = self.numpy_to_pt(rgb) + height, width = self.get_default_height_width(rgb, height, width) + if self.config.do_resize: + rgb = self.resize(rgb, height, width) + + depth = np.concatenate(depth, axis=0) if rgb[0].ndim == 4 else np.stack(depth, axis=0) + depth = self.numpy_to_pt(depth) + height, width = self.get_default_height_width(depth, height, width) + if self.config.do_resize: + depth = self.resize(depth, height, width) + + elif isinstance(rgb[0], torch.Tensor): + raise Exception("This is not yet supported") + # rgb = torch.cat(rgb, axis=0) if rgb[0].ndim == 4 else torch.stack(rgb, axis=0) + + # if self.config.do_convert_grayscale and rgb.ndim == 3: + # rgb = rgb.unsqueeze(1) + + # channel = rgb.shape[1] + + # height, width = self.get_default_height_width(rgb, height, width) + # if self.config.do_resize: + # rgb = self.resize(rgb, height, width) + + # depth = torch.cat(depth, axis=0) if depth[0].ndim == 4 else torch.stack(depth, axis=0) + + # if self.config.do_convert_grayscale and depth.ndim == 3: + # depth = depth.unsqueeze(1) + + # channel = depth.shape[1] + # # don't need any preprocess if the image is latents + # if depth == 4: + # return rgb, depth + + # height, width = self.get_default_height_width(depth, height, width) + # if self.config.do_resize: + # depth = self.resize(depth, height, width) + # expected range [0,1], normalize to [-1,1] + do_normalize = self.config.do_normalize + if rgb.min() < 0 and do_normalize: + warnings.warn( + "Passing `image` as torch tensor with value range in [-1,1] is deprecated. The expected value range for image tensor is [0,1] " + f"when passing as pytorch tensor or numpy Array. You passed `image` with value range [{rgb.min()},{rgb.max()}]", + FutureWarning, + ) + do_normalize = False + + if do_normalize: + rgb = self.normalize(rgb) + depth = self.normalize(depth) + + if self.config.do_binarize: + rgb = self.binarize(rgb) + depth = self.binarize(depth) + + return rgb, depth + + +class IPAdapterMaskProcessor(VaeImageProcessor): + """ + Image processor for IP Adapter image masks. + + Args: + do_resize (`bool`, *optional*, defaults to `True`): + Whether to downscale the image's (height, width) dimensions to multiples of `vae_scale_factor`. + vae_scale_factor (`int`, *optional*, defaults to `8`): + VAE scale factor. If `do_resize` is `True`, the image is automatically resized to multiples of this factor. + resample (`str`, *optional*, defaults to `lanczos`): + Resampling filter to use when resizing the image. + do_normalize (`bool`, *optional*, defaults to `False`): + Whether to normalize the image to [-1,1]. + do_binarize (`bool`, *optional*, defaults to `True`): + Whether to binarize the image to 0/1. + do_convert_grayscale (`bool`, *optional*, defaults to be `True`): + Whether to convert the images to grayscale format. + + """ + + config_name = CONFIG_NAME + + @register_to_config + def __init__( + self, + do_resize: bool = True, + vae_scale_factor: int = 8, + resample: str = "lanczos", + do_normalize: bool = False, + do_binarize: bool = True, + do_convert_grayscale: bool = True, + ): + super().__init__( + do_resize=do_resize, + vae_scale_factor=vae_scale_factor, + resample=resample, + do_normalize=do_normalize, + do_binarize=do_binarize, + do_convert_grayscale=do_convert_grayscale, + ) + + @staticmethod + def downsample(mask: torch.FloatTensor, batch_size: int, num_queries: int, value_embed_dim: int): + """ + Downsamples the provided mask tensor to match the expected dimensions for scaled dot-product attention. + If the aspect ratio of the mask does not match the aspect ratio of the output image, a warning is issued. + + Args: + mask (`torch.FloatTensor`): + The input mask tensor generated with `IPAdapterMaskProcessor.preprocess()`. + batch_size (`int`): + The batch size. + num_queries (`int`): + The number of queries. + value_embed_dim (`int`): + The dimensionality of the value embeddings. + + Returns: + `torch.FloatTensor`: + The downsampled mask tensor. + + """ + o_h = mask.shape[1] + o_w = mask.shape[2] + ratio = o_w / o_h + mask_h = int(math.sqrt(num_queries / ratio)) + mask_h = int(mask_h) + int((num_queries % int(mask_h)) != 0) + mask_w = num_queries // mask_h + + mask_downsample = F.interpolate(mask.unsqueeze(0), size=(mask_h, mask_w), mode="bicubic").squeeze(0) + + # Repeat batch_size times + if mask_downsample.shape[0] < batch_size: + mask_downsample = mask_downsample.repeat(batch_size, 1, 1) + + mask_downsample = mask_downsample.view(mask_downsample.shape[0], -1) + + downsampled_area = mask_h * mask_w + # If the output image and the mask do not have the same aspect ratio, tensor shapes will not match + # Pad tensor if downsampled_mask.shape[1] is smaller than num_queries + if downsampled_area < num_queries: + warnings.warn( + "The aspect ratio of the mask does not match the aspect ratio of the output image. " + "Please update your masks or adjust the output size for optimal performance.", + UserWarning, + ) + mask_downsample = F.pad(mask_downsample, (0, num_queries - mask_downsample.shape[1]), value=0.0) + # Discard last embeddings if downsampled_mask.shape[1] is bigger than num_queries + if downsampled_area > num_queries: + warnings.warn( + "The aspect ratio of the mask does not match the aspect ratio of the output image. " + "Please update your masks or adjust the output size for optimal performance.", + UserWarning, + ) + mask_downsample = mask_downsample[:, :num_queries] + + # Repeat last dimension to match SDPA output shape + mask_downsample = mask_downsample.view(mask_downsample.shape[0], mask_downsample.shape[1], 1).repeat( + 1, 1, value_embed_dim + ) + + return mask_downsample diff --git a/diffusers-0.27.0/src/diffusers/loaders/__init__.py b/diffusers-0.27.0/src/diffusers/loaders/__init__.py new file mode 100755 index 0000000..4da0474 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/__init__.py @@ -0,0 +1,88 @@ +from typing import TYPE_CHECKING + +from ..utils import DIFFUSERS_SLOW_IMPORT, _LazyModule, deprecate +from ..utils.import_utils import is_peft_available, is_torch_available, is_transformers_available + + +def text_encoder_lora_state_dict(text_encoder): + deprecate( + "text_encoder_load_state_dict in `models`", + "0.27.0", + "`text_encoder_lora_state_dict` is deprecated and will be removed in 0.27.0. Make sure to retrieve the weights using `get_peft_model`. See https://huggingface.co/docs/peft/v0.6.2/en/quicktour#peftmodel for more information.", + ) + state_dict = {} + + for name, module in text_encoder_attn_modules(text_encoder): + for k, v in module.q_proj.lora_linear_layer.state_dict().items(): + state_dict[f"{name}.q_proj.lora_linear_layer.{k}"] = v + + for k, v in module.k_proj.lora_linear_layer.state_dict().items(): + state_dict[f"{name}.k_proj.lora_linear_layer.{k}"] = v + + for k, v in module.v_proj.lora_linear_layer.state_dict().items(): + state_dict[f"{name}.v_proj.lora_linear_layer.{k}"] = v + + for k, v in module.out_proj.lora_linear_layer.state_dict().items(): + state_dict[f"{name}.out_proj.lora_linear_layer.{k}"] = v + + return state_dict + + +if is_transformers_available(): + + def text_encoder_attn_modules(text_encoder): + deprecate( + "text_encoder_attn_modules in `models`", + "0.27.0", + "`text_encoder_lora_state_dict` is deprecated and will be removed in 0.27.0. Make sure to retrieve the weights using `get_peft_model`. See https://huggingface.co/docs/peft/v0.6.2/en/quicktour#peftmodel for more information.", + ) + from transformers import CLIPTextModel, CLIPTextModelWithProjection + + attn_modules = [] + + if isinstance(text_encoder, (CLIPTextModel, CLIPTextModelWithProjection)): + for i, layer in enumerate(text_encoder.text_model.encoder.layers): + name = f"text_model.encoder.layers.{i}.self_attn" + mod = layer.self_attn + attn_modules.append((name, mod)) + else: + raise ValueError(f"do not know how to get attention modules for: {text_encoder.__class__.__name__}") + + return attn_modules + + +_import_structure = {} + +if is_torch_available(): + _import_structure["autoencoder"] = ["FromOriginalVAEMixin"] + + _import_structure["controlnet"] = ["FromOriginalControlNetMixin"] + _import_structure["unet"] = ["UNet2DConditionLoadersMixin"] + _import_structure["utils"] = ["AttnProcsLayers"] + if is_transformers_available(): + _import_structure["single_file"] = ["FromSingleFileMixin"] + _import_structure["lora"] = ["LoraLoaderMixin", "StableDiffusionXLLoraLoaderMixin"] + _import_structure["textual_inversion"] = ["TextualInversionLoaderMixin"] + _import_structure["ip_adapter"] = ["IPAdapterMixin"] + +_import_structure["peft"] = ["PeftAdapterMixin"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + if is_torch_available(): + from .autoencoder import FromOriginalVAEMixin + from .controlnet import FromOriginalControlNetMixin + from .unet import UNet2DConditionLoadersMixin + from .utils import AttnProcsLayers + + if is_transformers_available(): + from .ip_adapter import IPAdapterMixin + from .lora import LoraLoaderMixin, StableDiffusionXLLoraLoaderMixin + from .single_file import FromSingleFileMixin + from .textual_inversion import TextualInversionLoaderMixin + + from .peft import PeftAdapterMixin +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) diff --git a/diffusers-0.27.0/src/diffusers/loaders/autoencoder.py b/diffusers-0.27.0/src/diffusers/loaders/autoencoder.py new file mode 100755 index 0000000..b91d27f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/autoencoder.py @@ -0,0 +1,146 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from huggingface_hub.utils import validate_hf_hub_args + +from .single_file_utils import ( + create_diffusers_vae_model_from_ldm, + fetch_ldm_config_and_checkpoint, +) + + +class FromOriginalVAEMixin: + """ + Load pretrained AutoencoderKL weights saved in the `.ckpt` or `.safetensors` format into a [`AutoencoderKL`]. + """ + + @classmethod + @validate_hf_hub_args + def from_single_file(cls, pretrained_model_link_or_path, **kwargs): + r""" + Instantiate a [`AutoencoderKL`] from pretrained ControlNet weights saved in the original `.ckpt` or + `.safetensors` format. The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pretrained_model_link_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + - A link to the `.ckpt` file (for example + `"https://huggingface.co//blob/main/.ckpt"`) on the Hub. + - A path to a *file* containing all pipeline weights. + config_file (`str`, *optional*): + Filepath to the configuration YAML file associated with the model. If not provided it will default to: + https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If `"auto"` is passed, the + dtype is automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to True, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + image_size (`int`, *optional*, defaults to 512): + The image size the model was trained on. Use 512 for all Stable Diffusion v1 models and the Stable + Diffusion v2 base model. Use 768 for Stable Diffusion v2. + scaling_factor (`float`, *optional*, defaults to 0.18215): + The component-wise standard deviation of the trained latent space computed using the first batch of the + training set. This is used to scale the latent space to have unit variance when training the diffusion + model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the + diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z + = 1 / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution + Image Synthesis with Latent Diffusion Models](https://arxiv.org/abs/2112.10752) paper. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (for example the pipeline components of the + specific pipeline class). The overwritten components are directly passed to the pipelines `__init__` + method. See example below for more information. + + + + Make sure to pass both `image_size` and `scaling_factor` to `from_single_file()` if you're loading + a VAE from SDXL or a Stable Diffusion v2 model or higher. + + + + Examples: + + ```py + from diffusers import AutoencoderKL + + url = "https://huggingface.co/stabilityai/sd-vae-ft-mse-original/blob/main/vae-ft-mse-840000-ema-pruned.safetensors" # can also be local file + model = AutoencoderKL.from_single_file(url) + ``` + """ + + original_config_file = kwargs.pop("original_config_file", None) + config_file = kwargs.pop("config_file", None) + resume_download = kwargs.pop("resume_download", False) + force_download = kwargs.pop("force_download", False) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + cache_dir = kwargs.pop("cache_dir", None) + local_files_only = kwargs.pop("local_files_only", None) + revision = kwargs.pop("revision", None) + torch_dtype = kwargs.pop("torch_dtype", None) + + class_name = cls.__name__ + + if (config_file is not None) and (original_config_file is not None): + raise ValueError( + "You cannot pass both `config_file` and `original_config_file` to `from_single_file`. Please use only one of these arguments." + ) + + original_config_file = original_config_file or config_file + original_config, checkpoint = fetch_ldm_config_and_checkpoint( + pretrained_model_link_or_path=pretrained_model_link_or_path, + class_name=class_name, + original_config_file=original_config_file, + resume_download=resume_download, + force_download=force_download, + proxies=proxies, + token=token, + revision=revision, + local_files_only=local_files_only, + cache_dir=cache_dir, + ) + + image_size = kwargs.pop("image_size", None) + scaling_factor = kwargs.pop("scaling_factor", None) + component = create_diffusers_vae_model_from_ldm( + class_name, + original_config, + checkpoint, + image_size=image_size, + scaling_factor=scaling_factor, + torch_dtype=torch_dtype, + ) + vae = component["vae"] + if torch_dtype is not None: + vae = vae.to(torch_dtype) + + return vae diff --git a/diffusers-0.27.0/src/diffusers/loaders/controlnet.py b/diffusers-0.27.0/src/diffusers/loaders/controlnet.py new file mode 100755 index 0000000..d323f60 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/controlnet.py @@ -0,0 +1,136 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from huggingface_hub.utils import validate_hf_hub_args + +from .single_file_utils import ( + create_diffusers_controlnet_model_from_ldm, + fetch_ldm_config_and_checkpoint, +) + + +class FromOriginalControlNetMixin: + """ + Load pretrained ControlNet weights saved in the `.ckpt` or `.safetensors` format into a [`ControlNetModel`]. + """ + + @classmethod + @validate_hf_hub_args + def from_single_file(cls, pretrained_model_link_or_path, **kwargs): + r""" + Instantiate a [`ControlNetModel`] from pretrained ControlNet weights saved in the original `.ckpt` or + `.safetensors` format. The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pretrained_model_link_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + - A link to the `.ckpt` file (for example + `"https://huggingface.co//blob/main/.ckpt"`) on the Hub. + - A path to a *file* containing all pipeline weights. + config_file (`str`, *optional*): + Filepath to the configuration YAML file associated with the model. If not provided it will default to: + https://raw.githubusercontent.com/lllyasviel/ControlNet/main/models/cldm_v15.yaml + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If `"auto"` is passed, the + dtype is automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to True, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + image_size (`int`, *optional*, defaults to 512): + The image size the model was trained on. Use 512 for all Stable Diffusion v1 models and the Stable + Diffusion v2 base model. Use 768 for Stable Diffusion v2. + upcast_attention (`bool`, *optional*, defaults to `None`): + Whether the attention computation should always be upcasted. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (for example the pipeline components of the + specific pipeline class). The overwritten components are directly passed to the pipelines `__init__` + method. See example below for more information. + + Examples: + + ```py + from diffusers import StableDiffusionControlNetPipeline, ControlNetModel + + url = "https://huggingface.co/lllyasviel/ControlNet-v1-1/blob/main/control_v11p_sd15_canny.pth" # can also be a local path + model = ControlNetModel.from_single_file(url) + + url = "https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned.safetensors" # can also be a local path + pipe = StableDiffusionControlNetPipeline.from_single_file(url, controlnet=controlnet) + ``` + """ + original_config_file = kwargs.pop("original_config_file", None) + config_file = kwargs.pop("config_file", None) + resume_download = kwargs.pop("resume_download", False) + force_download = kwargs.pop("force_download", False) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + cache_dir = kwargs.pop("cache_dir", None) + local_files_only = kwargs.pop("local_files_only", None) + revision = kwargs.pop("revision", None) + torch_dtype = kwargs.pop("torch_dtype", None) + + class_name = cls.__name__ + if (config_file is not None) and (original_config_file is not None): + raise ValueError( + "You cannot pass both `config_file` and `original_config_file` to `from_single_file`. Please use only one of these arguments." + ) + + original_config_file = config_file or original_config_file + original_config, checkpoint = fetch_ldm_config_and_checkpoint( + pretrained_model_link_or_path=pretrained_model_link_or_path, + class_name=class_name, + original_config_file=original_config_file, + resume_download=resume_download, + force_download=force_download, + proxies=proxies, + token=token, + revision=revision, + local_files_only=local_files_only, + cache_dir=cache_dir, + ) + + upcast_attention = kwargs.pop("upcast_attention", False) + image_size = kwargs.pop("image_size", None) + + component = create_diffusers_controlnet_model_from_ldm( + class_name, + original_config, + checkpoint, + upcast_attention=upcast_attention, + image_size=image_size, + torch_dtype=torch_dtype, + ) + controlnet = component["controlnet"] + if torch_dtype is not None: + controlnet = controlnet.to(torch_dtype) + + return controlnet diff --git a/diffusers-0.27.0/src/diffusers/loaders/ip_adapter.py b/diffusers-0.27.0/src/diffusers/loaders/ip_adapter.py new file mode 100755 index 0000000..93959b9 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/ip_adapter.py @@ -0,0 +1,281 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from pathlib import Path +from typing import Dict, List, Optional, Union + +import torch +from huggingface_hub.utils import validate_hf_hub_args +from safetensors import safe_open + +from ..models.modeling_utils import _LOW_CPU_MEM_USAGE_DEFAULT +from ..utils import ( + _get_model_file, + is_accelerate_available, + is_torch_version, + is_transformers_available, + logging, +) + + +if is_transformers_available(): + from transformers import ( + CLIPImageProcessor, + CLIPVisionModelWithProjection, + ) + + from ..models.attention_processor import ( + IPAdapterAttnProcessor, + IPAdapterAttnProcessor2_0, + ) + +logger = logging.get_logger(__name__) + + +class IPAdapterMixin: + """Mixin for handling IP Adapters.""" + + @validate_hf_hub_args + def load_ip_adapter( + self, + pretrained_model_name_or_path_or_dict: Union[str, List[str], Dict[str, torch.Tensor]], + subfolder: Union[str, List[str]], + weight_name: Union[str, List[str]], + image_encoder_folder: Optional[str] = "image_encoder", + **kwargs, + ): + """ + Parameters: + pretrained_model_name_or_path_or_dict (`str` or `List[str]` or `os.PathLike` or `List[os.PathLike]` or `dict` or `List[dict]`): + Can be either: + + - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on + the Hub. + - A path to a *directory* (for example `./my_model_directory`) containing the model weights saved + with [`ModelMixin.save_pretrained`]. + - A [torch state + dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict). + subfolder (`str` or `List[str]`): + The subfolder location of a model file within a larger model repository on the Hub or locally. + If a list is passed, it should have the same length as `weight_name`. + weight_name (`str` or `List[str]`): + The name of the weight file to load. If a list is passed, it should have the same length as + `weight_name`. + image_encoder_folder (`str`, *optional*, defaults to `image_encoder`): + The subfolder location of the image encoder within a larger model repository on the Hub or locally. + Pass `None` to not load the image encoder. If the image encoder is located in a folder inside `subfolder`, + you only need to pass the name of the folder that contains image encoder weights, e.g. `image_encoder_folder="image_encoder"`. + If the image encoder is located in a folder other than `subfolder`, you should pass the path to the folder that contains image encoder weights, + for example, `image_encoder_folder="different_subfolder/image_encoder"`. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + """ + + # handle the list inputs for multiple IP Adapters + if not isinstance(weight_name, list): + weight_name = [weight_name] + + if not isinstance(pretrained_model_name_or_path_or_dict, list): + pretrained_model_name_or_path_or_dict = [pretrained_model_name_or_path_or_dict] + if len(pretrained_model_name_or_path_or_dict) == 1: + pretrained_model_name_or_path_or_dict = pretrained_model_name_or_path_or_dict * len(weight_name) + + if not isinstance(subfolder, list): + subfolder = [subfolder] + if len(subfolder) == 1: + subfolder = subfolder * len(weight_name) + + if len(weight_name) != len(pretrained_model_name_or_path_or_dict): + raise ValueError("`weight_name` and `pretrained_model_name_or_path_or_dict` must have the same length.") + + if len(weight_name) != len(subfolder): + raise ValueError("`weight_name` and `subfolder` must have the same length.") + + # Load the main state dict first. + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) + revision = kwargs.pop("revision", None) + low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", _LOW_CPU_MEM_USAGE_DEFAULT) + + if low_cpu_mem_usage and not is_accelerate_available(): + low_cpu_mem_usage = False + logger.warning( + "Cannot initialize model with low cpu memory usage because `accelerate` was not found in the" + " environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install" + " `accelerate` for faster and less memory-intense model loading. You can do so with: \n```\npip" + " install accelerate\n```\n." + ) + + if low_cpu_mem_usage is True and not is_torch_version(">=", "1.9.0"): + raise NotImplementedError( + "Low memory initialization requires torch >= 1.9.0. Please either update your PyTorch version or set" + " `low_cpu_mem_usage=False`." + ) + + user_agent = { + "file_type": "attn_procs_weights", + "framework": "pytorch", + } + state_dicts = [] + for pretrained_model_name_or_path_or_dict, weight_name, subfolder in zip( + pretrained_model_name_or_path_or_dict, weight_name, subfolder + ): + if not isinstance(pretrained_model_name_or_path_or_dict, dict): + model_file = _get_model_file( + pretrained_model_name_or_path_or_dict, + weights_name=weight_name, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + if weight_name.endswith(".safetensors"): + state_dict = {"image_proj": {}, "ip_adapter": {}} + with safe_open(model_file, framework="pt", device="cpu") as f: + for key in f.keys(): + if key.startswith("image_proj."): + state_dict["image_proj"][key.replace("image_proj.", "")] = f.get_tensor(key) + elif key.startswith("ip_adapter."): + state_dict["ip_adapter"][key.replace("ip_adapter.", "")] = f.get_tensor(key) + else: + state_dict = torch.load(model_file, map_location="cpu") + else: + state_dict = pretrained_model_name_or_path_or_dict + + keys = list(state_dict.keys()) + if keys != ["image_proj", "ip_adapter"]: + raise ValueError("Required keys are (`image_proj` and `ip_adapter`) missing from the state dict.") + + state_dicts.append(state_dict) + + # load CLIP image encoder here if it has not been registered to the pipeline yet + if hasattr(self, "image_encoder") and getattr(self, "image_encoder", None) is None: + if image_encoder_folder is not None: + if not isinstance(pretrained_model_name_or_path_or_dict, dict): + logger.info(f"loading image_encoder from {pretrained_model_name_or_path_or_dict}") + if image_encoder_folder.count("/") == 0: + image_encoder_subfolder = Path(subfolder, image_encoder_folder).as_posix() + else: + image_encoder_subfolder = Path(image_encoder_folder).as_posix() + + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + pretrained_model_name_or_path_or_dict, + subfolder=image_encoder_subfolder, + low_cpu_mem_usage=low_cpu_mem_usage, + ).to(self.device, dtype=self.dtype) + self.register_modules(image_encoder=image_encoder) + else: + raise ValueError( + "`image_encoder` cannot be loaded because `pretrained_model_name_or_path_or_dict` is a state dict." + ) + else: + logger.warning( + "image_encoder is not loaded since `image_encoder_folder=None` passed. You will not be able to use `ip_adapter_image` when calling the pipeline with IP-Adapter." + "Use `ip_adapter_image_embeds` to pass pre-generated image embedding instead." + ) + + # create feature extractor if it has not been registered to the pipeline yet + if hasattr(self, "feature_extractor") and getattr(self, "feature_extractor", None) is None: + feature_extractor = CLIPImageProcessor() + self.register_modules(feature_extractor=feature_extractor) + + # load ip-adapter into unet + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + unet._load_ip_adapter_weights(state_dicts, low_cpu_mem_usage=low_cpu_mem_usage) + + def set_ip_adapter_scale(self, scale): + """ + Sets the conditioning scale between text and image. + + Example: + + ```py + pipeline.set_ip_adapter_scale(0.5) + ``` + """ + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + for attn_processor in unet.attn_processors.values(): + if isinstance(attn_processor, (IPAdapterAttnProcessor, IPAdapterAttnProcessor2_0)): + if not isinstance(scale, list): + scale = [scale] * len(attn_processor.scale) + if len(attn_processor.scale) != len(scale): + raise ValueError( + f"`scale` should be a list of same length as the number if ip-adapters " + f"Expected {len(attn_processor.scale)} but got {len(scale)}." + ) + attn_processor.scale = scale + + def unload_ip_adapter(self): + """ + Unloads the IP Adapter weights + + Examples: + + ```python + >>> # Assuming `pipeline` is already loaded with the IP Adapter weights. + >>> pipeline.unload_ip_adapter() + >>> ... + ``` + """ + # remove CLIP image encoder + if hasattr(self, "image_encoder") and getattr(self, "image_encoder", None) is not None: + self.image_encoder = None + self.register_to_config(image_encoder=[None, None]) + + # remove feature extractor only when safety_checker is None as safety_checker uses + # the feature_extractor later + if not hasattr(self, "safety_checker"): + if hasattr(self, "feature_extractor") and getattr(self, "feature_extractor", None) is not None: + self.feature_extractor = None + self.register_to_config(feature_extractor=[None, None]) + + # remove hidden encoder + self.unet.encoder_hid_proj = None + self.config.encoder_hid_dim_type = None + + # restore original Unet attention processors layers + self.unet.set_default_attn_processor() diff --git a/diffusers-0.27.0/src/diffusers/loaders/lora.py b/diffusers-0.27.0/src/diffusers/loaders/lora.py new file mode 100755 index 0000000..c6077f3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/lora.py @@ -0,0 +1,1349 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import os +from pathlib import Path +from typing import Callable, Dict, List, Optional, Union + +import safetensors +import torch +from huggingface_hub import model_info +from huggingface_hub.constants import HF_HUB_OFFLINE +from huggingface_hub.utils import validate_hf_hub_args +from packaging import version +from torch import nn + +from .. import __version__ +from ..models.modeling_utils import _LOW_CPU_MEM_USAGE_DEFAULT +from ..utils import ( + USE_PEFT_BACKEND, + _get_model_file, + convert_state_dict_to_diffusers, + convert_state_dict_to_peft, + convert_unet_state_dict_to_peft, + delete_adapter_layers, + get_adapter_name, + get_peft_kwargs, + is_accelerate_available, + is_transformers_available, + logging, + recurse_remove_peft_layers, + scale_lora_layers, + set_adapter_layers, + set_weights_and_activate_adapters, +) +from .lora_conversion_utils import _convert_kohya_lora_to_diffusers, _maybe_map_sgm_blocks_to_diffusers + + +if is_transformers_available(): + from transformers import PreTrainedModel + + from ..models.lora import text_encoder_attn_modules, text_encoder_mlp_modules + +if is_accelerate_available(): + from accelerate.hooks import AlignDevicesHook, CpuOffload, remove_hook_from_module + +logger = logging.get_logger(__name__) + +TEXT_ENCODER_NAME = "text_encoder" +UNET_NAME = "unet" +TRANSFORMER_NAME = "transformer" + +LORA_WEIGHT_NAME = "pytorch_lora_weights.bin" +LORA_WEIGHT_NAME_SAFE = "pytorch_lora_weights.safetensors" + +LORA_DEPRECATION_MESSAGE = "You are using an old version of LoRA backend. This will be deprecated in the next releases in favor of PEFT make sure to install the latest PEFT and transformers packages in the future." + + +class LoraLoaderMixin: + r""" + Load LoRA layers into [`UNet2DConditionModel`] and + [`CLIPTextModel`](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel). + """ + + text_encoder_name = TEXT_ENCODER_NAME + unet_name = UNET_NAME + transformer_name = TRANSFORMER_NAME + num_fused_loras = 0 + + def load_lora_weights( + self, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], adapter_name=None, **kwargs + ): + """ + Load LoRA weights specified in `pretrained_model_name_or_path_or_dict` into `self.unet` and + `self.text_encoder`. + + All kwargs are forwarded to `self.lora_state_dict`. + + See [`~loaders.LoraLoaderMixin.lora_state_dict`] for more details on how the state dict is loaded. + + See [`~loaders.LoraLoaderMixin.load_lora_into_unet`] for more details on how the state dict is loaded into + `self.unet`. + + See [`~loaders.LoraLoaderMixin.load_lora_into_text_encoder`] for more details on how the state dict is loaded + into `self.text_encoder`. + + Parameters: + pretrained_model_name_or_path_or_dict (`str` or `os.PathLike` or `dict`): + See [`~loaders.LoraLoaderMixin.lora_state_dict`]. + kwargs (`dict`, *optional*): + See [`~loaders.LoraLoaderMixin.lora_state_dict`]. + adapter_name (`str`, *optional*): + Adapter name to be used for referencing the loaded adapter model. If not specified, it will use + `default_{i}` where i is the total number of adapters being loaded. + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + + # if a dict is passed, copy it instead of modifying it inplace + if isinstance(pretrained_model_name_or_path_or_dict, dict): + pretrained_model_name_or_path_or_dict = pretrained_model_name_or_path_or_dict.copy() + + # First, ensure that the checkpoint is a compatible one and can be successfully loaded. + state_dict, network_alphas = self.lora_state_dict(pretrained_model_name_or_path_or_dict, **kwargs) + + is_correct_format = all("lora" in key for key in state_dict.keys()) + if not is_correct_format: + raise ValueError("Invalid LoRA checkpoint.") + + low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", _LOW_CPU_MEM_USAGE_DEFAULT) + + self.load_lora_into_unet( + state_dict, + network_alphas=network_alphas, + unet=getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet, + low_cpu_mem_usage=low_cpu_mem_usage, + adapter_name=adapter_name, + _pipeline=self, + ) + self.load_lora_into_text_encoder( + state_dict, + network_alphas=network_alphas, + text_encoder=getattr(self, self.text_encoder_name) + if not hasattr(self, "text_encoder") + else self.text_encoder, + lora_scale=self.lora_scale, + low_cpu_mem_usage=low_cpu_mem_usage, + adapter_name=adapter_name, + _pipeline=self, + ) + + @classmethod + @validate_hf_hub_args + def lora_state_dict( + cls, + pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + **kwargs, + ): + r""" + Return state dict for lora weights and the network alphas. + + + + We support loading A1111 formatted LoRA checkpoints in a limited capacity. + + This function is experimental and might change in the future. + + + + Parameters: + pretrained_model_name_or_path_or_dict (`str` or `os.PathLike` or `dict`): + Can be either: + + - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on + the Hub. + - A path to a *directory* (for example `./my_model_directory`) containing the model weights saved + with [`ModelMixin.save_pretrained`]. + - A [torch state + dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict). + + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + subfolder (`str`, *optional*, defaults to `""`): + The subfolder location of a model file within a larger model repository on the Hub or locally. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you're downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + + """ + # Load the main state dict first which has the LoRA layers for either of + # UNet and text encoder or both. + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) + revision = kwargs.pop("revision", None) + subfolder = kwargs.pop("subfolder", None) + weight_name = kwargs.pop("weight_name", None) + unet_config = kwargs.pop("unet_config", None) + use_safetensors = kwargs.pop("use_safetensors", None) + + allow_pickle = False + if use_safetensors is None: + use_safetensors = True + allow_pickle = True + + user_agent = { + "file_type": "attn_procs_weights", + "framework": "pytorch", + } + + model_file = None + if not isinstance(pretrained_model_name_or_path_or_dict, dict): + # Let's first try to load .safetensors weights + if (use_safetensors and weight_name is None) or ( + weight_name is not None and weight_name.endswith(".safetensors") + ): + try: + # Here we're relaxing the loading check to enable more Inference API + # friendliness where sometimes, it's not at all possible to automatically + # determine `weight_name`. + if weight_name is None: + weight_name = cls._best_guess_weight_name( + pretrained_model_name_or_path_or_dict, + file_extension=".safetensors", + local_files_only=local_files_only, + ) + model_file = _get_model_file( + pretrained_model_name_or_path_or_dict, + weights_name=weight_name or LORA_WEIGHT_NAME_SAFE, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + state_dict = safetensors.torch.load_file(model_file, device="cpu") + except (IOError, safetensors.SafetensorError) as e: + if not allow_pickle: + raise e + # try loading non-safetensors weights + model_file = None + pass + + if model_file is None: + if weight_name is None: + weight_name = cls._best_guess_weight_name( + pretrained_model_name_or_path_or_dict, file_extension=".bin", local_files_only=local_files_only + ) + model_file = _get_model_file( + pretrained_model_name_or_path_or_dict, + weights_name=weight_name or LORA_WEIGHT_NAME, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + state_dict = torch.load(model_file, map_location="cpu") + else: + state_dict = pretrained_model_name_or_path_or_dict + + network_alphas = None + # TODO: replace it with a method from `state_dict_utils` + if all( + ( + k.startswith("lora_te_") + or k.startswith("lora_unet_") + or k.startswith("lora_te1_") + or k.startswith("lora_te2_") + ) + for k in state_dict.keys() + ): + # Map SDXL blocks correctly. + if unet_config is not None: + # use unet config to remap block numbers + state_dict = _maybe_map_sgm_blocks_to_diffusers(state_dict, unet_config) + state_dict, network_alphas = _convert_kohya_lora_to_diffusers(state_dict) + + return state_dict, network_alphas + + @classmethod + def _best_guess_weight_name( + cls, pretrained_model_name_or_path_or_dict, file_extension=".safetensors", local_files_only=False + ): + if local_files_only or HF_HUB_OFFLINE: + raise ValueError("When using the offline mode, you must specify a `weight_name`.") + + targeted_files = [] + + if os.path.isfile(pretrained_model_name_or_path_or_dict): + return + elif os.path.isdir(pretrained_model_name_or_path_or_dict): + targeted_files = [ + f for f in os.listdir(pretrained_model_name_or_path_or_dict) if f.endswith(file_extension) + ] + else: + files_in_repo = model_info(pretrained_model_name_or_path_or_dict).siblings + targeted_files = [f.rfilename for f in files_in_repo if f.rfilename.endswith(file_extension)] + if len(targeted_files) == 0: + return + + # "scheduler" does not correspond to a LoRA checkpoint. + # "optimizer" does not correspond to a LoRA checkpoint + # only top-level checkpoints are considered and not the other ones, hence "checkpoint". + unallowed_substrings = {"scheduler", "optimizer", "checkpoint"} + targeted_files = list( + filter(lambda x: all(substring not in x for substring in unallowed_substrings), targeted_files) + ) + + if any(f.endswith(LORA_WEIGHT_NAME) for f in targeted_files): + targeted_files = list(filter(lambda x: x.endswith(LORA_WEIGHT_NAME), targeted_files)) + elif any(f.endswith(LORA_WEIGHT_NAME_SAFE) for f in targeted_files): + targeted_files = list(filter(lambda x: x.endswith(LORA_WEIGHT_NAME_SAFE), targeted_files)) + + if len(targeted_files) > 1: + raise ValueError( + f"Provided path contains more than one weights file in the {file_extension} format. Either specify `weight_name` in `load_lora_weights` or make sure there's only one `.safetensors` or `.bin` file in {pretrained_model_name_or_path_or_dict}." + ) + weight_name = targeted_files[0] + return weight_name + + @classmethod + def _optionally_disable_offloading(cls, _pipeline): + """ + Optionally removes offloading in case the pipeline has been already sequentially offloaded to CPU. + + Args: + _pipeline (`DiffusionPipeline`): + The pipeline to disable offloading for. + + Returns: + tuple: + A tuple indicating if `is_model_cpu_offload` or `is_sequential_cpu_offload` is True. + """ + is_model_cpu_offload = False + is_sequential_cpu_offload = False + + if _pipeline is not None: + for _, component in _pipeline.components.items(): + if isinstance(component, nn.Module) and hasattr(component, "_hf_hook"): + if not is_model_cpu_offload: + is_model_cpu_offload = isinstance(component._hf_hook, CpuOffload) + if not is_sequential_cpu_offload: + is_sequential_cpu_offload = isinstance(component._hf_hook, AlignDevicesHook) + + logger.info( + "Accelerate hooks detected. Since you have called `load_lora_weights()`, the previous hooks will be first removed. Then the LoRA parameters will be loaded and the hooks will be applied again." + ) + remove_hook_from_module(component, recurse=is_sequential_cpu_offload) + + return (is_model_cpu_offload, is_sequential_cpu_offload) + + @classmethod + def load_lora_into_unet( + cls, state_dict, network_alphas, unet, low_cpu_mem_usage=None, adapter_name=None, _pipeline=None + ): + """ + This will load the LoRA layers specified in `state_dict` into `unet`. + + Parameters: + state_dict (`dict`): + A standard state dict containing the lora layer parameters. The keys can either be indexed directly + into the unet or prefixed with an additional `unet` which can be used to distinguish between text + encoder lora layers. + network_alphas (`Dict[str, float]`): + See `LoRALinearLayer` for more details. + unet (`UNet2DConditionModel`): + The UNet model to load the LoRA layers into. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + adapter_name (`str`, *optional*): + Adapter name to be used for referencing the loaded adapter model. If not specified, it will use + `default_{i}` where i is the total number of adapters being loaded. + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + + from peft import LoraConfig, inject_adapter_in_model, set_peft_model_state_dict + + low_cpu_mem_usage = low_cpu_mem_usage if low_cpu_mem_usage is not None else _LOW_CPU_MEM_USAGE_DEFAULT + # If the serialization format is new (introduced in https://github.com/huggingface/diffusers/pull/2918), + # then the `state_dict` keys should have `cls.unet_name` and/or `cls.text_encoder_name` as + # their prefixes. + keys = list(state_dict.keys()) + + if all(key.startswith(cls.unet_name) or key.startswith(cls.text_encoder_name) for key in keys): + # Load the layers corresponding to UNet. + logger.info(f"Loading {cls.unet_name}.") + + unet_keys = [k for k in keys if k.startswith(cls.unet_name)] + state_dict = {k.replace(f"{cls.unet_name}.", ""): v for k, v in state_dict.items() if k in unet_keys} + + if network_alphas is not None: + alpha_keys = [k for k in network_alphas.keys() if k.startswith(cls.unet_name)] + network_alphas = { + k.replace(f"{cls.unet_name}.", ""): v for k, v in network_alphas.items() if k in alpha_keys + } + + else: + # Otherwise, we're dealing with the old format. This means the `state_dict` should only + # contain the module names of the `unet` as its keys WITHOUT any prefix. + if not USE_PEFT_BACKEND: + warn_message = "You have saved the LoRA weights using the old format. To convert the old LoRA weights to the new format, you can first load them in a dictionary and then create a new dictionary like the following: `new_state_dict = {f'unet.{module_name}': params for module_name, params in old_state_dict.items()}`." + logger.warning(warn_message) + + if len(state_dict.keys()) > 0: + if adapter_name in getattr(unet, "peft_config", {}): + raise ValueError( + f"Adapter name {adapter_name} already in use in the Unet - please select a new adapter name." + ) + + state_dict = convert_unet_state_dict_to_peft(state_dict) + + if network_alphas is not None: + # The alphas state dict have the same structure as Unet, thus we convert it to peft format using + # `convert_unet_state_dict_to_peft` method. + network_alphas = convert_unet_state_dict_to_peft(network_alphas) + + rank = {} + for key, val in state_dict.items(): + if "lora_B" in key: + rank[key] = val.shape[1] + + lora_config_kwargs = get_peft_kwargs(rank, network_alphas, state_dict, is_unet=True) + lora_config = LoraConfig(**lora_config_kwargs) + + # adapter_name + if adapter_name is None: + adapter_name = get_adapter_name(unet) + + # In case the pipeline has been already offloaded to CPU - temporarily remove the hooks + # otherwise loading LoRA weights will lead to an error + is_model_cpu_offload, is_sequential_cpu_offload = cls._optionally_disable_offloading(_pipeline) + + inject_adapter_in_model(lora_config, unet, adapter_name=adapter_name) + incompatible_keys = set_peft_model_state_dict(unet, state_dict, adapter_name) + + if incompatible_keys is not None: + # check only for unexpected keys + unexpected_keys = getattr(incompatible_keys, "unexpected_keys", None) + if unexpected_keys: + logger.warning( + f"Loading adapter weights from state_dict led to unexpected keys not found in the model: " + f" {unexpected_keys}. " + ) + + # Offload back. + if is_model_cpu_offload: + _pipeline.enable_model_cpu_offload() + elif is_sequential_cpu_offload: + _pipeline.enable_sequential_cpu_offload() + # Unsafe code /> + + unet.load_attn_procs( + state_dict, network_alphas=network_alphas, low_cpu_mem_usage=low_cpu_mem_usage, _pipeline=_pipeline + ) + + @classmethod + def load_lora_into_text_encoder( + cls, + state_dict, + network_alphas, + text_encoder, + prefix=None, + lora_scale=1.0, + low_cpu_mem_usage=None, + adapter_name=None, + _pipeline=None, + ): + """ + This will load the LoRA layers specified in `state_dict` into `text_encoder` + + Parameters: + state_dict (`dict`): + A standard state dict containing the lora layer parameters. The key should be prefixed with an + additional `text_encoder` to distinguish between unet lora layers. + network_alphas (`Dict[str, float]`): + See `LoRALinearLayer` for more details. + text_encoder (`CLIPTextModel`): + The text encoder model to load the LoRA layers into. + prefix (`str`): + Expected prefix of the `text_encoder` in the `state_dict`. + lora_scale (`float`): + How much to scale the output of the lora linear layer before it is added with the output of the regular + lora layer. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + adapter_name (`str`, *optional*): + Adapter name to be used for referencing the loaded adapter model. If not specified, it will use + `default_{i}` where i is the total number of adapters being loaded. + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + + from peft import LoraConfig + + low_cpu_mem_usage = low_cpu_mem_usage if low_cpu_mem_usage is not None else _LOW_CPU_MEM_USAGE_DEFAULT + + # If the serialization format is new (introduced in https://github.com/huggingface/diffusers/pull/2918), + # then the `state_dict` keys should have `self.unet_name` and/or `self.text_encoder_name` as + # their prefixes. + keys = list(state_dict.keys()) + prefix = cls.text_encoder_name if prefix is None else prefix + + # Safe prefix to check with. + if any(cls.text_encoder_name in key for key in keys): + # Load the layers corresponding to text encoder and make necessary adjustments. + text_encoder_keys = [k for k in keys if k.startswith(prefix) and k.split(".")[0] == prefix] + text_encoder_lora_state_dict = { + k.replace(f"{prefix}.", ""): v for k, v in state_dict.items() if k in text_encoder_keys + } + + if len(text_encoder_lora_state_dict) > 0: + logger.info(f"Loading {prefix}.") + rank = {} + text_encoder_lora_state_dict = convert_state_dict_to_diffusers(text_encoder_lora_state_dict) + + # convert state dict + text_encoder_lora_state_dict = convert_state_dict_to_peft(text_encoder_lora_state_dict) + + for name, _ in text_encoder_attn_modules(text_encoder): + rank_key = f"{name}.out_proj.lora_B.weight" + rank[rank_key] = text_encoder_lora_state_dict[rank_key].shape[1] + + patch_mlp = any(".mlp." in key for key in text_encoder_lora_state_dict.keys()) + if patch_mlp: + for name, _ in text_encoder_mlp_modules(text_encoder): + rank_key_fc1 = f"{name}.fc1.lora_B.weight" + rank_key_fc2 = f"{name}.fc2.lora_B.weight" + + rank[rank_key_fc1] = text_encoder_lora_state_dict[rank_key_fc1].shape[1] + rank[rank_key_fc2] = text_encoder_lora_state_dict[rank_key_fc2].shape[1] + + if network_alphas is not None: + alpha_keys = [ + k for k in network_alphas.keys() if k.startswith(prefix) and k.split(".")[0] == prefix + ] + network_alphas = { + k.replace(f"{prefix}.", ""): v for k, v in network_alphas.items() if k in alpha_keys + } + + lora_config_kwargs = get_peft_kwargs(rank, network_alphas, text_encoder_lora_state_dict, is_unet=False) + lora_config = LoraConfig(**lora_config_kwargs) + + # adapter_name + if adapter_name is None: + adapter_name = get_adapter_name(text_encoder) + + is_model_cpu_offload, is_sequential_cpu_offload = cls._optionally_disable_offloading(_pipeline) + + # inject LoRA layers and load the state dict + # in transformers we automatically check whether the adapter name is already in use or not + text_encoder.load_adapter( + adapter_name=adapter_name, + adapter_state_dict=text_encoder_lora_state_dict, + peft_config=lora_config, + ) + + # scale LoRA layers with `lora_scale` + scale_lora_layers(text_encoder, weight=lora_scale) + + text_encoder.to(device=text_encoder.device, dtype=text_encoder.dtype) + + # Offload back. + if is_model_cpu_offload: + _pipeline.enable_model_cpu_offload() + elif is_sequential_cpu_offload: + _pipeline.enable_sequential_cpu_offload() + # Unsafe code /> + + @classmethod + def load_lora_into_transformer( + cls, state_dict, network_alphas, transformer, low_cpu_mem_usage=None, adapter_name=None, _pipeline=None + ): + """ + This will load the LoRA layers specified in `state_dict` into `transformer`. + + Parameters: + state_dict (`dict`): + A standard state dict containing the lora layer parameters. The keys can either be indexed directly + into the unet or prefixed with an additional `unet` which can be used to distinguish between text + encoder lora layers. + network_alphas (`Dict[str, float]`): + See `LoRALinearLayer` for more details. + unet (`UNet2DConditionModel`): + The UNet model to load the LoRA layers into. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + adapter_name (`str`, *optional*): + Adapter name to be used for referencing the loaded adapter model. If not specified, it will use + `default_{i}` where i is the total number of adapters being loaded. + """ + from peft import LoraConfig, inject_adapter_in_model, set_peft_model_state_dict + + low_cpu_mem_usage = low_cpu_mem_usage if low_cpu_mem_usage is not None else _LOW_CPU_MEM_USAGE_DEFAULT + + keys = list(state_dict.keys()) + + transformer_keys = [k for k in keys if k.startswith(cls.transformer_name)] + state_dict = { + k.replace(f"{cls.transformer_name}.", ""): v for k, v in state_dict.items() if k in transformer_keys + } + + if network_alphas is not None: + alpha_keys = [k for k in network_alphas.keys() if k.startswith(cls.transformer_name)] + network_alphas = { + k.replace(f"{cls.transformer_name}.", ""): v for k, v in network_alphas.items() if k in alpha_keys + } + + if len(state_dict.keys()) > 0: + if adapter_name in getattr(transformer, "peft_config", {}): + raise ValueError( + f"Adapter name {adapter_name} already in use in the transformer - please select a new adapter name." + ) + + rank = {} + for key, val in state_dict.items(): + if "lora_B" in key: + rank[key] = val.shape[1] + + lora_config_kwargs = get_peft_kwargs(rank, network_alphas, state_dict) + lora_config = LoraConfig(**lora_config_kwargs) + + # adapter_name + if adapter_name is None: + adapter_name = get_adapter_name(transformer) + + # In case the pipeline has been already offloaded to CPU - temporarily remove the hooks + # otherwise loading LoRA weights will lead to an error + is_model_cpu_offload, is_sequential_cpu_offload = cls._optionally_disable_offloading(_pipeline) + + inject_adapter_in_model(lora_config, transformer, adapter_name=adapter_name) + incompatible_keys = set_peft_model_state_dict(transformer, state_dict, adapter_name) + + if incompatible_keys is not None: + # check only for unexpected keys + unexpected_keys = getattr(incompatible_keys, "unexpected_keys", None) + if unexpected_keys: + logger.warning( + f"Loading adapter weights from state_dict led to unexpected keys not found in the model: " + f" {unexpected_keys}. " + ) + + # Offload back. + if is_model_cpu_offload: + _pipeline.enable_model_cpu_offload() + elif is_sequential_cpu_offload: + _pipeline.enable_sequential_cpu_offload() + # Unsafe code /> + + @property + def lora_scale(self) -> float: + # property function that returns the lora scale which can be set at run time by the pipeline. + # if _lora_scale has not been set, return 1 + return self._lora_scale if hasattr(self, "_lora_scale") else 1.0 + + def _remove_text_encoder_monkey_patch(self): + remove_method = recurse_remove_peft_layers + if hasattr(self, "text_encoder"): + remove_method(self.text_encoder) + # In case text encoder have no Lora attached + if getattr(self.text_encoder, "peft_config", None) is not None: + del self.text_encoder.peft_config + self.text_encoder._hf_peft_config_loaded = None + + if hasattr(self, "text_encoder_2"): + remove_method(self.text_encoder_2) + if getattr(self.text_encoder_2, "peft_config", None) is not None: + del self.text_encoder_2.peft_config + self.text_encoder_2._hf_peft_config_loaded = None + + @classmethod + def save_lora_weights( + cls, + save_directory: Union[str, os.PathLike], + unet_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + text_encoder_lora_layers: Dict[str, torch.nn.Module] = None, + transformer_lora_layers: Dict[str, torch.nn.Module] = None, + is_main_process: bool = True, + weight_name: str = None, + save_function: Callable = None, + safe_serialization: bool = True, + ): + r""" + Save the LoRA parameters corresponding to the UNet and text encoder. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to save LoRA parameters to. Will be created if it doesn't exist. + unet_lora_layers (`Dict[str, torch.nn.Module]` or `Dict[str, torch.Tensor]`): + State dict of the LoRA layers corresponding to the `unet`. + text_encoder_lora_layers (`Dict[str, torch.nn.Module]` or `Dict[str, torch.Tensor]`): + State dict of the LoRA layers corresponding to the `text_encoder`. Must explicitly pass the text + encoder LoRA state dict because it comes from 🤗 Transformers. + is_main_process (`bool`, *optional*, defaults to `True`): + Whether the process calling this is the main process or not. Useful during distributed training and you + need to call this function on all processes. In this case, set `is_main_process=True` only on the main + process to avoid race conditions. + save_function (`Callable`): + The function to use to save the state dictionary. Useful during distributed training when you need to + replace `torch.save` with another method. Can be configured with the environment variable + `DIFFUSERS_SAVE_MODE`. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether to save the model using `safetensors` or the traditional PyTorch way with `pickle`. + """ + state_dict = {} + + def pack_weights(layers, prefix): + layers_weights = layers.state_dict() if isinstance(layers, torch.nn.Module) else layers + layers_state_dict = {f"{prefix}.{module_name}": param for module_name, param in layers_weights.items()} + return layers_state_dict + + if not (unet_lora_layers or text_encoder_lora_layers or transformer_lora_layers): + raise ValueError( + "You must pass at least one of `unet_lora_layers`, `text_encoder_lora_layers`, or `transformer_lora_layers`." + ) + + if unet_lora_layers: + state_dict.update(pack_weights(unet_lora_layers, cls.unet_name)) + + if text_encoder_lora_layers: + state_dict.update(pack_weights(text_encoder_lora_layers, cls.text_encoder_name)) + + if transformer_lora_layers: + state_dict.update(pack_weights(transformer_lora_layers, "transformer")) + + # Save the model + cls.write_lora_layers( + state_dict=state_dict, + save_directory=save_directory, + is_main_process=is_main_process, + weight_name=weight_name, + save_function=save_function, + safe_serialization=safe_serialization, + ) + + @staticmethod + def write_lora_layers( + state_dict: Dict[str, torch.Tensor], + save_directory: str, + is_main_process: bool, + weight_name: str, + save_function: Callable, + safe_serialization: bool, + ): + if os.path.isfile(save_directory): + logger.error(f"Provided path ({save_directory}) should be a directory, not a file") + return + + if save_function is None: + if safe_serialization: + + def save_function(weights, filename): + return safetensors.torch.save_file(weights, filename, metadata={"format": "pt"}) + + else: + save_function = torch.save + + os.makedirs(save_directory, exist_ok=True) + + if weight_name is None: + if safe_serialization: + weight_name = LORA_WEIGHT_NAME_SAFE + else: + weight_name = LORA_WEIGHT_NAME + + save_path = Path(save_directory, weight_name).as_posix() + save_function(state_dict, save_path) + logger.info(f"Model weights saved in {save_path}") + + def unload_lora_weights(self): + """ + Unloads the LoRA parameters. + + Examples: + + ```python + >>> # Assuming `pipeline` is already loaded with the LoRA parameters. + >>> pipeline.unload_lora_weights() + >>> ... + ``` + """ + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + + if not USE_PEFT_BACKEND: + if version.parse(__version__) > version.parse("0.23"): + logger.warning( + "You are using `unload_lora_weights` to disable and unload lora weights. If you want to iteratively enable and disable adapter weights," + "you can use `pipe.enable_lora()` or `pipe.disable_lora()`. After installing the latest version of PEFT." + ) + + for _, module in unet.named_modules(): + if hasattr(module, "set_lora_layer"): + module.set_lora_layer(None) + else: + recurse_remove_peft_layers(unet) + if hasattr(unet, "peft_config"): + del unet.peft_config + + # Safe to call the following regardless of LoRA. + self._remove_text_encoder_monkey_patch() + + def fuse_lora( + self, + fuse_unet: bool = True, + fuse_text_encoder: bool = True, + lora_scale: float = 1.0, + safe_fusing: bool = False, + adapter_names: Optional[List[str]] = None, + ): + r""" + Fuses the LoRA parameters into the original parameters of the corresponding blocks. + + + + This is an experimental API. + + + + Args: + fuse_unet (`bool`, defaults to `True`): Whether to fuse the UNet LoRA parameters. + fuse_text_encoder (`bool`, defaults to `True`): + Whether to fuse the text encoder LoRA parameters. If the text encoder wasn't monkey-patched with the + LoRA parameters then it won't have any effect. + lora_scale (`float`, defaults to 1.0): + Controls how much to influence the outputs with the LoRA parameters. + safe_fusing (`bool`, defaults to `False`): + Whether to check fused weights for NaN values before fusing and if values are NaN not fusing them. + adapter_names (`List[str]`, *optional*): + Adapter names to be used for fusing. If nothing is passed, all active adapters will be fused. + + Example: + + ```py + from diffusers import DiffusionPipeline + import torch + + pipeline = DiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ).to("cuda") + pipeline.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors", adapter_name="pixel") + pipeline.fuse_lora(lora_scale=0.7) + ``` + """ + from peft.tuners.tuners_utils import BaseTunerLayer + + if fuse_unet or fuse_text_encoder: + self.num_fused_loras += 1 + if self.num_fused_loras > 1: + logger.warning( + "The current API is supported for operating with a single LoRA file. You are trying to load and fuse more than one LoRA which is not well-supported.", + ) + + if fuse_unet: + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + unet.fuse_lora(lora_scale, safe_fusing=safe_fusing, adapter_names=adapter_names) + + def fuse_text_encoder_lora(text_encoder, lora_scale=1.0, safe_fusing=False, adapter_names=None): + merge_kwargs = {"safe_merge": safe_fusing} + + for module in text_encoder.modules(): + if isinstance(module, BaseTunerLayer): + if lora_scale != 1.0: + module.scale_layer(lora_scale) + + # For BC with previous PEFT versions, we need to check the signature + # of the `merge` method to see if it supports the `adapter_names` argument. + supported_merge_kwargs = list(inspect.signature(module.merge).parameters) + if "adapter_names" in supported_merge_kwargs: + merge_kwargs["adapter_names"] = adapter_names + elif "adapter_names" not in supported_merge_kwargs and adapter_names is not None: + raise ValueError( + "The `adapter_names` argument is not supported with your PEFT version. " + "Please upgrade to the latest version of PEFT. `pip install -U peft`" + ) + + module.merge(**merge_kwargs) + + if fuse_text_encoder: + if hasattr(self, "text_encoder"): + fuse_text_encoder_lora(self.text_encoder, lora_scale, safe_fusing, adapter_names=adapter_names) + if hasattr(self, "text_encoder_2"): + fuse_text_encoder_lora(self.text_encoder_2, lora_scale, safe_fusing, adapter_names=adapter_names) + + def unfuse_lora(self, unfuse_unet: bool = True, unfuse_text_encoder: bool = True): + r""" + Reverses the effect of + [`pipe.fuse_lora()`](https://huggingface.co/docs/diffusers/main/en/api/loaders#diffusers.loaders.LoraLoaderMixin.fuse_lora). + + + + This is an experimental API. + + + + Args: + unfuse_unet (`bool`, defaults to `True`): Whether to unfuse the UNet LoRA parameters. + unfuse_text_encoder (`bool`, defaults to `True`): + Whether to unfuse the text encoder LoRA parameters. If the text encoder wasn't monkey-patched with the + LoRA parameters then it won't have any effect. + """ + from peft.tuners.tuners_utils import BaseTunerLayer + + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + if unfuse_unet: + for module in unet.modules(): + if isinstance(module, BaseTunerLayer): + module.unmerge() + + def unfuse_text_encoder_lora(text_encoder): + for module in text_encoder.modules(): + if isinstance(module, BaseTunerLayer): + module.unmerge() + + if unfuse_text_encoder: + if hasattr(self, "text_encoder"): + unfuse_text_encoder_lora(self.text_encoder) + if hasattr(self, "text_encoder_2"): + unfuse_text_encoder_lora(self.text_encoder_2) + + self.num_fused_loras -= 1 + + def set_adapters_for_text_encoder( + self, + adapter_names: Union[List[str], str], + text_encoder: Optional["PreTrainedModel"] = None, # noqa: F821 + text_encoder_weights: List[float] = None, + ): + """ + Sets the adapter layers for the text encoder. + + Args: + adapter_names (`List[str]` or `str`): + The names of the adapters to use. + text_encoder (`torch.nn.Module`, *optional*): + The text encoder module to set the adapter layers for. If `None`, it will try to get the `text_encoder` + attribute. + text_encoder_weights (`List[float]`, *optional*): + The weights to use for the text encoder. If `None`, the weights are set to `1.0` for all the adapters. + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + + def process_weights(adapter_names, weights): + if weights is None: + weights = [1.0] * len(adapter_names) + elif isinstance(weights, float): + weights = [weights] + + if len(adapter_names) != len(weights): + raise ValueError( + f"Length of adapter names {len(adapter_names)} is not equal to the length of the weights {len(weights)}" + ) + return weights + + adapter_names = [adapter_names] if isinstance(adapter_names, str) else adapter_names + text_encoder_weights = process_weights(adapter_names, text_encoder_weights) + text_encoder = text_encoder or getattr(self, "text_encoder", None) + if text_encoder is None: + raise ValueError( + "The pipeline does not have a default `pipe.text_encoder` class. Please make sure to pass a `text_encoder` instead." + ) + set_weights_and_activate_adapters(text_encoder, adapter_names, text_encoder_weights) + + def disable_lora_for_text_encoder(self, text_encoder: Optional["PreTrainedModel"] = None): + """ + Disables the LoRA layers for the text encoder. + + Args: + text_encoder (`torch.nn.Module`, *optional*): + The text encoder module to disable the LoRA layers for. If `None`, it will try to get the + `text_encoder` attribute. + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + + text_encoder = text_encoder or getattr(self, "text_encoder", None) + if text_encoder is None: + raise ValueError("Text Encoder not found.") + set_adapter_layers(text_encoder, enabled=False) + + def enable_lora_for_text_encoder(self, text_encoder: Optional["PreTrainedModel"] = None): + """ + Enables the LoRA layers for the text encoder. + + Args: + text_encoder (`torch.nn.Module`, *optional*): + The text encoder module to enable the LoRA layers for. If `None`, it will try to get the `text_encoder` + attribute. + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + text_encoder = text_encoder or getattr(self, "text_encoder", None) + if text_encoder is None: + raise ValueError("Text Encoder not found.") + set_adapter_layers(self.text_encoder, enabled=True) + + def set_adapters( + self, + adapter_names: Union[List[str], str], + adapter_weights: Optional[List[float]] = None, + ): + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + # Handle the UNET + unet.set_adapters(adapter_names, adapter_weights) + + # Handle the Text Encoder + if hasattr(self, "text_encoder"): + self.set_adapters_for_text_encoder(adapter_names, self.text_encoder, adapter_weights) + if hasattr(self, "text_encoder_2"): + self.set_adapters_for_text_encoder(adapter_names, self.text_encoder_2, adapter_weights) + + def disable_lora(self): + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + + # Disable unet adapters + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + unet.disable_lora() + + # Disable text encoder adapters + if hasattr(self, "text_encoder"): + self.disable_lora_for_text_encoder(self.text_encoder) + if hasattr(self, "text_encoder_2"): + self.disable_lora_for_text_encoder(self.text_encoder_2) + + def enable_lora(self): + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + + # Enable unet adapters + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + unet.enable_lora() + + # Enable text encoder adapters + if hasattr(self, "text_encoder"): + self.enable_lora_for_text_encoder(self.text_encoder) + if hasattr(self, "text_encoder_2"): + self.enable_lora_for_text_encoder(self.text_encoder_2) + + def delete_adapters(self, adapter_names: Union[List[str], str]): + """ + Args: + Deletes the LoRA layers of `adapter_name` for the unet and text-encoder(s). + adapter_names (`Union[List[str], str]`): + The names of the adapter to delete. Can be a single string or a list of strings + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + + if isinstance(adapter_names, str): + adapter_names = [adapter_names] + + # Delete unet adapters + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + unet.delete_adapters(adapter_names) + + for adapter_name in adapter_names: + # Delete text encoder adapters + if hasattr(self, "text_encoder"): + delete_adapter_layers(self.text_encoder, adapter_name) + if hasattr(self, "text_encoder_2"): + delete_adapter_layers(self.text_encoder_2, adapter_name) + + def get_active_adapters(self) -> List[str]: + """ + Gets the list of the current active adapters. + + Example: + + ```python + from diffusers import DiffusionPipeline + + pipeline = DiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + ).to("cuda") + pipeline.load_lora_weights("CiroN2022/toy-face", weight_name="toy_face_sdxl.safetensors", adapter_name="toy") + pipeline.get_active_adapters() + ``` + """ + if not USE_PEFT_BACKEND: + raise ValueError( + "PEFT backend is required for this method. Please install the latest version of PEFT `pip install -U peft`" + ) + + from peft.tuners.tuners_utils import BaseTunerLayer + + active_adapters = [] + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + for module in unet.modules(): + if isinstance(module, BaseTunerLayer): + active_adapters = module.active_adapters + break + + return active_adapters + + def get_list_adapters(self) -> Dict[str, List[str]]: + """ + Gets the current list of all available adapters in the pipeline. + """ + if not USE_PEFT_BACKEND: + raise ValueError( + "PEFT backend is required for this method. Please install the latest version of PEFT `pip install -U peft`" + ) + + set_adapters = {} + + if hasattr(self, "text_encoder") and hasattr(self.text_encoder, "peft_config"): + set_adapters["text_encoder"] = list(self.text_encoder.peft_config.keys()) + + if hasattr(self, "text_encoder_2") and hasattr(self.text_encoder_2, "peft_config"): + set_adapters["text_encoder_2"] = list(self.text_encoder_2.peft_config.keys()) + + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + if hasattr(self, self.unet_name) and hasattr(unet, "peft_config"): + set_adapters[self.unet_name] = list(self.unet.peft_config.keys()) + + return set_adapters + + def set_lora_device(self, adapter_names: List[str], device: Union[torch.device, str, int]) -> None: + """ + Moves the LoRAs listed in `adapter_names` to a target device. Useful for offloading the LoRA to the CPU in case + you want to load multiple adapters and free some GPU memory. + + Args: + adapter_names (`List[str]`): + List of adapters to send device to. + device (`Union[torch.device, str, int]`): + Device to send the adapters to. Can be either a torch device, a str or an integer. + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + + from peft.tuners.tuners_utils import BaseTunerLayer + + # Handle the UNET + unet = getattr(self, self.unet_name) if not hasattr(self, "unet") else self.unet + for unet_module in unet.modules(): + if isinstance(unet_module, BaseTunerLayer): + for adapter_name in adapter_names: + unet_module.lora_A[adapter_name].to(device) + unet_module.lora_B[adapter_name].to(device) + + # Handle the text encoder + modules_to_process = [] + if hasattr(self, "text_encoder"): + modules_to_process.append(self.text_encoder) + + if hasattr(self, "text_encoder_2"): + modules_to_process.append(self.text_encoder_2) + + for text_encoder in modules_to_process: + # loop over submodules + for text_encoder_module in text_encoder.modules(): + if isinstance(text_encoder_module, BaseTunerLayer): + for adapter_name in adapter_names: + text_encoder_module.lora_A[adapter_name].to(device) + text_encoder_module.lora_B[adapter_name].to(device) + + +class StableDiffusionXLLoraLoaderMixin(LoraLoaderMixin): + """This class overrides `LoraLoaderMixin` with LoRA loading/saving code that's specific to SDXL""" + + # Override to properly handle the loading and unloading of the additional text encoder. + def load_lora_weights( + self, + pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], + adapter_name: Optional[str] = None, + **kwargs, + ): + """ + Load LoRA weights specified in `pretrained_model_name_or_path_or_dict` into `self.unet` and + `self.text_encoder`. + + All kwargs are forwarded to `self.lora_state_dict`. + + See [`~loaders.LoraLoaderMixin.lora_state_dict`] for more details on how the state dict is loaded. + + See [`~loaders.LoraLoaderMixin.load_lora_into_unet`] for more details on how the state dict is loaded into + `self.unet`. + + See [`~loaders.LoraLoaderMixin.load_lora_into_text_encoder`] for more details on how the state dict is loaded + into `self.text_encoder`. + + Parameters: + pretrained_model_name_or_path_or_dict (`str` or `os.PathLike` or `dict`): + See [`~loaders.LoraLoaderMixin.lora_state_dict`]. + adapter_name (`str`, *optional*): + Adapter name to be used for referencing the loaded adapter model. If not specified, it will use + `default_{i}` where i is the total number of adapters being loaded. + kwargs (`dict`, *optional*): + See [`~loaders.LoraLoaderMixin.lora_state_dict`]. + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + + # We could have accessed the unet config from `lora_state_dict()` too. We pass + # it here explicitly to be able to tell that it's coming from an SDXL + # pipeline. + + # if a dict is passed, copy it instead of modifying it inplace + if isinstance(pretrained_model_name_or_path_or_dict, dict): + pretrained_model_name_or_path_or_dict = pretrained_model_name_or_path_or_dict.copy() + + # First, ensure that the checkpoint is a compatible one and can be successfully loaded. + state_dict, network_alphas = self.lora_state_dict( + pretrained_model_name_or_path_or_dict, + unet_config=self.unet.config, + **kwargs, + ) + is_correct_format = all("lora" in key for key in state_dict.keys()) + if not is_correct_format: + raise ValueError("Invalid LoRA checkpoint.") + + self.load_lora_into_unet( + state_dict, network_alphas=network_alphas, unet=self.unet, adapter_name=adapter_name, _pipeline=self + ) + text_encoder_state_dict = {k: v for k, v in state_dict.items() if "text_encoder." in k} + if len(text_encoder_state_dict) > 0: + self.load_lora_into_text_encoder( + text_encoder_state_dict, + network_alphas=network_alphas, + text_encoder=self.text_encoder, + prefix="text_encoder", + lora_scale=self.lora_scale, + adapter_name=adapter_name, + _pipeline=self, + ) + + text_encoder_2_state_dict = {k: v for k, v in state_dict.items() if "text_encoder_2." in k} + if len(text_encoder_2_state_dict) > 0: + self.load_lora_into_text_encoder( + text_encoder_2_state_dict, + network_alphas=network_alphas, + text_encoder=self.text_encoder_2, + prefix="text_encoder_2", + lora_scale=self.lora_scale, + adapter_name=adapter_name, + _pipeline=self, + ) + + @classmethod + def save_lora_weights( + cls, + save_directory: Union[str, os.PathLike], + unet_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + text_encoder_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + text_encoder_2_lora_layers: Dict[str, Union[torch.nn.Module, torch.Tensor]] = None, + is_main_process: bool = True, + weight_name: str = None, + save_function: Callable = None, + safe_serialization: bool = True, + ): + r""" + Save the LoRA parameters corresponding to the UNet and text encoder. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to save LoRA parameters to. Will be created if it doesn't exist. + unet_lora_layers (`Dict[str, torch.nn.Module]` or `Dict[str, torch.Tensor]`): + State dict of the LoRA layers corresponding to the `unet`. + text_encoder_lora_layers (`Dict[str, torch.nn.Module]` or `Dict[str, torch.Tensor]`): + State dict of the LoRA layers corresponding to the `text_encoder`. Must explicitly pass the text + encoder LoRA state dict because it comes from 🤗 Transformers. + is_main_process (`bool`, *optional*, defaults to `True`): + Whether the process calling this is the main process or not. Useful during distributed training and you + need to call this function on all processes. In this case, set `is_main_process=True` only on the main + process to avoid race conditions. + save_function (`Callable`): + The function to use to save the state dictionary. Useful during distributed training when you need to + replace `torch.save` with another method. Can be configured with the environment variable + `DIFFUSERS_SAVE_MODE`. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether to save the model using `safetensors` or the traditional PyTorch way with `pickle`. + """ + state_dict = {} + + def pack_weights(layers, prefix): + layers_weights = layers.state_dict() if isinstance(layers, torch.nn.Module) else layers + layers_state_dict = {f"{prefix}.{module_name}": param for module_name, param in layers_weights.items()} + return layers_state_dict + + if not (unet_lora_layers or text_encoder_lora_layers or text_encoder_2_lora_layers): + raise ValueError( + "You must pass at least one of `unet_lora_layers`, `text_encoder_lora_layers` or `text_encoder_2_lora_layers`." + ) + + if unet_lora_layers: + state_dict.update(pack_weights(unet_lora_layers, "unet")) + + if text_encoder_lora_layers and text_encoder_2_lora_layers: + state_dict.update(pack_weights(text_encoder_lora_layers, "text_encoder")) + state_dict.update(pack_weights(text_encoder_2_lora_layers, "text_encoder_2")) + + cls.write_lora_layers( + state_dict=state_dict, + save_directory=save_directory, + is_main_process=is_main_process, + weight_name=weight_name, + save_function=save_function, + safe_serialization=safe_serialization, + ) + + def _remove_text_encoder_monkey_patch(self): + recurse_remove_peft_layers(self.text_encoder) + # TODO: @younesbelkada handle this in transformers side + if getattr(self.text_encoder, "peft_config", None) is not None: + del self.text_encoder.peft_config + self.text_encoder._hf_peft_config_loaded = None + + recurse_remove_peft_layers(self.text_encoder_2) + if getattr(self.text_encoder_2, "peft_config", None) is not None: + del self.text_encoder_2.peft_config + self.text_encoder_2._hf_peft_config_loaded = None diff --git a/diffusers-0.27.0/src/diffusers/loaders/lora_conversion_utils.py b/diffusers-0.27.0/src/diffusers/loaders/lora_conversion_utils.py new file mode 100755 index 0000000..e968ef9 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/lora_conversion_utils.py @@ -0,0 +1,284 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import re + +from ..utils import logging + + +logger = logging.get_logger(__name__) + + +def _maybe_map_sgm_blocks_to_diffusers(state_dict, unet_config, delimiter="_", block_slice_pos=5): + # 1. get all state_dict_keys + all_keys = list(state_dict.keys()) + sgm_patterns = ["input_blocks", "middle_block", "output_blocks"] + + # 2. check if needs remapping, if not return original dict + is_in_sgm_format = False + for key in all_keys: + if any(p in key for p in sgm_patterns): + is_in_sgm_format = True + break + + if not is_in_sgm_format: + return state_dict + + # 3. Else remap from SGM patterns + new_state_dict = {} + inner_block_map = ["resnets", "attentions", "upsamplers"] + + # Retrieves # of down, mid and up blocks + input_block_ids, middle_block_ids, output_block_ids = set(), set(), set() + + for layer in all_keys: + if "text" in layer: + new_state_dict[layer] = state_dict.pop(layer) + else: + layer_id = int(layer.split(delimiter)[:block_slice_pos][-1]) + if sgm_patterns[0] in layer: + input_block_ids.add(layer_id) + elif sgm_patterns[1] in layer: + middle_block_ids.add(layer_id) + elif sgm_patterns[2] in layer: + output_block_ids.add(layer_id) + else: + raise ValueError(f"Checkpoint not supported because layer {layer} not supported.") + + input_blocks = { + layer_id: [key for key in state_dict if f"input_blocks{delimiter}{layer_id}" in key] + for layer_id in input_block_ids + } + middle_blocks = { + layer_id: [key for key in state_dict if f"middle_block{delimiter}{layer_id}" in key] + for layer_id in middle_block_ids + } + output_blocks = { + layer_id: [key for key in state_dict if f"output_blocks{delimiter}{layer_id}" in key] + for layer_id in output_block_ids + } + + # Rename keys accordingly + for i in input_block_ids: + block_id = (i - 1) // (unet_config.layers_per_block + 1) + layer_in_block_id = (i - 1) % (unet_config.layers_per_block + 1) + + for key in input_blocks[i]: + inner_block_id = int(key.split(delimiter)[block_slice_pos]) + inner_block_key = inner_block_map[inner_block_id] if "op" not in key else "downsamplers" + inner_layers_in_block = str(layer_in_block_id) if "op" not in key else "0" + new_key = delimiter.join( + key.split(delimiter)[: block_slice_pos - 1] + + [str(block_id), inner_block_key, inner_layers_in_block] + + key.split(delimiter)[block_slice_pos + 1 :] + ) + new_state_dict[new_key] = state_dict.pop(key) + + for i in middle_block_ids: + key_part = None + if i == 0: + key_part = [inner_block_map[0], "0"] + elif i == 1: + key_part = [inner_block_map[1], "0"] + elif i == 2: + key_part = [inner_block_map[0], "1"] + else: + raise ValueError(f"Invalid middle block id {i}.") + + for key in middle_blocks[i]: + new_key = delimiter.join( + key.split(delimiter)[: block_slice_pos - 1] + key_part + key.split(delimiter)[block_slice_pos:] + ) + new_state_dict[new_key] = state_dict.pop(key) + + for i in output_block_ids: + block_id = i // (unet_config.layers_per_block + 1) + layer_in_block_id = i % (unet_config.layers_per_block + 1) + + for key in output_blocks[i]: + inner_block_id = int(key.split(delimiter)[block_slice_pos]) + inner_block_key = inner_block_map[inner_block_id] + inner_layers_in_block = str(layer_in_block_id) if inner_block_id < 2 else "0" + new_key = delimiter.join( + key.split(delimiter)[: block_slice_pos - 1] + + [str(block_id), inner_block_key, inner_layers_in_block] + + key.split(delimiter)[block_slice_pos + 1 :] + ) + new_state_dict[new_key] = state_dict.pop(key) + + if len(state_dict) > 0: + raise ValueError("At this point all state dict entries have to be converted.") + + return new_state_dict + + +def _convert_kohya_lora_to_diffusers(state_dict, unet_name="unet", text_encoder_name="text_encoder"): + unet_state_dict = {} + te_state_dict = {} + te2_state_dict = {} + network_alphas = {} + + # every down weight has a corresponding up weight and potentially an alpha weight + lora_keys = [k for k in state_dict.keys() if k.endswith("lora_down.weight")] + for key in lora_keys: + lora_name = key.split(".")[0] + lora_name_up = lora_name + ".lora_up.weight" + lora_name_alpha = lora_name + ".alpha" + + if lora_name.startswith("lora_unet_"): + diffusers_name = key.replace("lora_unet_", "").replace("_", ".") + + if "input.blocks" in diffusers_name: + diffusers_name = diffusers_name.replace("input.blocks", "down_blocks") + else: + diffusers_name = diffusers_name.replace("down.blocks", "down_blocks") + + if "middle.block" in diffusers_name: + diffusers_name = diffusers_name.replace("middle.block", "mid_block") + else: + diffusers_name = diffusers_name.replace("mid.block", "mid_block") + if "output.blocks" in diffusers_name: + diffusers_name = diffusers_name.replace("output.blocks", "up_blocks") + else: + diffusers_name = diffusers_name.replace("up.blocks", "up_blocks") + + diffusers_name = diffusers_name.replace("transformer.blocks", "transformer_blocks") + diffusers_name = diffusers_name.replace("to.q.lora", "to_q_lora") + diffusers_name = diffusers_name.replace("to.k.lora", "to_k_lora") + diffusers_name = diffusers_name.replace("to.v.lora", "to_v_lora") + diffusers_name = diffusers_name.replace("to.out.0.lora", "to_out_lora") + diffusers_name = diffusers_name.replace("proj.in", "proj_in") + diffusers_name = diffusers_name.replace("proj.out", "proj_out") + diffusers_name = diffusers_name.replace("emb.layers", "time_emb_proj") + + # SDXL specificity. + if "emb" in diffusers_name and "time.emb.proj" not in diffusers_name: + pattern = r"\.\d+(?=\D*$)" + diffusers_name = re.sub(pattern, "", diffusers_name, count=1) + if ".in." in diffusers_name: + diffusers_name = diffusers_name.replace("in.layers.2", "conv1") + if ".out." in diffusers_name: + diffusers_name = diffusers_name.replace("out.layers.3", "conv2") + if "downsamplers" in diffusers_name or "upsamplers" in diffusers_name: + diffusers_name = diffusers_name.replace("op", "conv") + if "skip" in diffusers_name: + diffusers_name = diffusers_name.replace("skip.connection", "conv_shortcut") + + # LyCORIS specificity. + if "time.emb.proj" in diffusers_name: + diffusers_name = diffusers_name.replace("time.emb.proj", "time_emb_proj") + if "conv.shortcut" in diffusers_name: + diffusers_name = diffusers_name.replace("conv.shortcut", "conv_shortcut") + + # General coverage. + if "transformer_blocks" in diffusers_name: + if "attn1" in diffusers_name or "attn2" in diffusers_name: + diffusers_name = diffusers_name.replace("attn1", "attn1.processor") + diffusers_name = diffusers_name.replace("attn2", "attn2.processor") + unet_state_dict[diffusers_name] = state_dict.pop(key) + unet_state_dict[diffusers_name.replace(".down.", ".up.")] = state_dict.pop(lora_name_up) + elif "ff" in diffusers_name: + unet_state_dict[diffusers_name] = state_dict.pop(key) + unet_state_dict[diffusers_name.replace(".down.", ".up.")] = state_dict.pop(lora_name_up) + elif any(key in diffusers_name for key in ("proj_in", "proj_out")): + unet_state_dict[diffusers_name] = state_dict.pop(key) + unet_state_dict[diffusers_name.replace(".down.", ".up.")] = state_dict.pop(lora_name_up) + else: + unet_state_dict[diffusers_name] = state_dict.pop(key) + unet_state_dict[diffusers_name.replace(".down.", ".up.")] = state_dict.pop(lora_name_up) + + elif lora_name.startswith("lora_te_"): + diffusers_name = key.replace("lora_te_", "").replace("_", ".") + diffusers_name = diffusers_name.replace("text.model", "text_model") + diffusers_name = diffusers_name.replace("self.attn", "self_attn") + diffusers_name = diffusers_name.replace("q.proj.lora", "to_q_lora") + diffusers_name = diffusers_name.replace("k.proj.lora", "to_k_lora") + diffusers_name = diffusers_name.replace("v.proj.lora", "to_v_lora") + diffusers_name = diffusers_name.replace("out.proj.lora", "to_out_lora") + if "self_attn" in diffusers_name: + te_state_dict[diffusers_name] = state_dict.pop(key) + te_state_dict[diffusers_name.replace(".down.", ".up.")] = state_dict.pop(lora_name_up) + elif "mlp" in diffusers_name: + # Be aware that this is the new diffusers convention and the rest of the code might + # not utilize it yet. + diffusers_name = diffusers_name.replace(".lora.", ".lora_linear_layer.") + te_state_dict[diffusers_name] = state_dict.pop(key) + te_state_dict[diffusers_name.replace(".down.", ".up.")] = state_dict.pop(lora_name_up) + + # (sayakpaul): Duplicate code. Needs to be cleaned. + elif lora_name.startswith("lora_te1_"): + diffusers_name = key.replace("lora_te1_", "").replace("_", ".") + diffusers_name = diffusers_name.replace("text.model", "text_model") + diffusers_name = diffusers_name.replace("self.attn", "self_attn") + diffusers_name = diffusers_name.replace("q.proj.lora", "to_q_lora") + diffusers_name = diffusers_name.replace("k.proj.lora", "to_k_lora") + diffusers_name = diffusers_name.replace("v.proj.lora", "to_v_lora") + diffusers_name = diffusers_name.replace("out.proj.lora", "to_out_lora") + if "self_attn" in diffusers_name: + te_state_dict[diffusers_name] = state_dict.pop(key) + te_state_dict[diffusers_name.replace(".down.", ".up.")] = state_dict.pop(lora_name_up) + elif "mlp" in diffusers_name: + # Be aware that this is the new diffusers convention and the rest of the code might + # not utilize it yet. + diffusers_name = diffusers_name.replace(".lora.", ".lora_linear_layer.") + te_state_dict[diffusers_name] = state_dict.pop(key) + te_state_dict[diffusers_name.replace(".down.", ".up.")] = state_dict.pop(lora_name_up) + + # (sayakpaul): Duplicate code. Needs to be cleaned. + elif lora_name.startswith("lora_te2_"): + diffusers_name = key.replace("lora_te2_", "").replace("_", ".") + diffusers_name = diffusers_name.replace("text.model", "text_model") + diffusers_name = diffusers_name.replace("self.attn", "self_attn") + diffusers_name = diffusers_name.replace("q.proj.lora", "to_q_lora") + diffusers_name = diffusers_name.replace("k.proj.lora", "to_k_lora") + diffusers_name = diffusers_name.replace("v.proj.lora", "to_v_lora") + diffusers_name = diffusers_name.replace("out.proj.lora", "to_out_lora") + if "self_attn" in diffusers_name: + te2_state_dict[diffusers_name] = state_dict.pop(key) + te2_state_dict[diffusers_name.replace(".down.", ".up.")] = state_dict.pop(lora_name_up) + elif "mlp" in diffusers_name: + # Be aware that this is the new diffusers convention and the rest of the code might + # not utilize it yet. + diffusers_name = diffusers_name.replace(".lora.", ".lora_linear_layer.") + te2_state_dict[diffusers_name] = state_dict.pop(key) + te2_state_dict[diffusers_name.replace(".down.", ".up.")] = state_dict.pop(lora_name_up) + + # Rename the alphas so that they can be mapped appropriately. + if lora_name_alpha in state_dict: + alpha = state_dict.pop(lora_name_alpha).item() + if lora_name_alpha.startswith("lora_unet_"): + prefix = "unet." + elif lora_name_alpha.startswith(("lora_te_", "lora_te1_")): + prefix = "text_encoder." + else: + prefix = "text_encoder_2." + new_name = prefix + diffusers_name.split(".lora.")[0] + ".alpha" + network_alphas.update({new_name: alpha}) + + if len(state_dict) > 0: + raise ValueError(f"The following keys have not been correctly be renamed: \n\n {', '.join(state_dict.keys())}") + + logger.info("Kohya-style checkpoint detected.") + unet_state_dict = {f"{unet_name}.{module_name}": params for module_name, params in unet_state_dict.items()} + te_state_dict = {f"{text_encoder_name}.{module_name}": params for module_name, params in te_state_dict.items()} + te2_state_dict = ( + {f"text_encoder_2.{module_name}": params for module_name, params in te2_state_dict.items()} + if len(te2_state_dict) > 0 + else None + ) + if te2_state_dict is not None: + te_state_dict.update(te2_state_dict) + + new_state_dict = {**unet_state_dict, **te_state_dict} + return new_state_dict, network_alphas diff --git a/diffusers-0.27.0/src/diffusers/loaders/peft.py b/diffusers-0.27.0/src/diffusers/loaders/peft.py new file mode 100755 index 0000000..01dbd34 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/peft.py @@ -0,0 +1,186 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List, Union + +from ..utils import MIN_PEFT_VERSION, check_peft_version, is_peft_available + + +class PeftAdapterMixin: + """ + A class containing all functions for loading and using adapters weights that are supported in PEFT library. For + more details about adapters and injecting them in a transformer-based model, check out the PEFT [documentation](https://huggingface.co/docs/peft/index). + + Install the latest version of PEFT, and use this mixin to: + + - Attach new adapters in the model. + - Attach multiple adapters and iteratively activate/deactivate them. + - Activate/deactivate all adapters from the model. + - Get a list of the active adapters. + """ + + _hf_peft_config_loaded = False + + def add_adapter(self, adapter_config, adapter_name: str = "default") -> None: + r""" + Adds a new adapter to the current model for training. If no adapter name is passed, a default name is assigned + to the adapter to follow the convention of the PEFT library. + + If you are not familiar with adapters and PEFT methods, we invite you to read more about them in the PEFT + [documentation](https://huggingface.co/docs/peft). + + Args: + adapter_config (`[~peft.PeftConfig]`): + The configuration of the adapter to add; supported adapters are non-prefix tuning and adaption prompt + methods. + adapter_name (`str`, *optional*, defaults to `"default"`): + The name of the adapter to add. If no name is passed, a default name is assigned to the adapter. + """ + check_peft_version(min_version=MIN_PEFT_VERSION) + + if not is_peft_available(): + raise ImportError("PEFT is not available. Please install PEFT to use this function: `pip install peft`.") + + from peft import PeftConfig, inject_adapter_in_model + + if not self._hf_peft_config_loaded: + self._hf_peft_config_loaded = True + elif adapter_name in self.peft_config: + raise ValueError(f"Adapter with name {adapter_name} already exists. Please use a different name.") + + if not isinstance(adapter_config, PeftConfig): + raise ValueError( + f"adapter_config should be an instance of PeftConfig. Got {type(adapter_config)} instead." + ) + + # Unlike transformers, here we don't need to retrieve the name_or_path of the unet as the loading logic is + # handled by the `load_lora_layers` or `LoraLoaderMixin`. Therefore we set it to `None` here. + adapter_config.base_model_name_or_path = None + inject_adapter_in_model(adapter_config, self, adapter_name) + self.set_adapter(adapter_name) + + def set_adapter(self, adapter_name: Union[str, List[str]]) -> None: + """ + Sets a specific adapter by forcing the model to only use that adapter and disables the other adapters. + + If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT + [documentation](https://huggingface.co/docs/peft). + + Args: + adapter_name (Union[str, List[str]])): + The list of adapters to set or the adapter name in the case of a single adapter. + """ + check_peft_version(min_version=MIN_PEFT_VERSION) + + if not self._hf_peft_config_loaded: + raise ValueError("No adapter loaded. Please load an adapter first.") + + if isinstance(adapter_name, str): + adapter_name = [adapter_name] + + missing = set(adapter_name) - set(self.peft_config) + if len(missing) > 0: + raise ValueError( + f"Following adapter(s) could not be found: {', '.join(missing)}. Make sure you are passing the correct adapter name(s)." + f" current loaded adapters are: {list(self.peft_config.keys())}" + ) + + from peft.tuners.tuners_utils import BaseTunerLayer + + _adapters_has_been_set = False + + for _, module in self.named_modules(): + if isinstance(module, BaseTunerLayer): + if hasattr(module, "set_adapter"): + module.set_adapter(adapter_name) + # Previous versions of PEFT does not support multi-adapter inference + elif not hasattr(module, "set_adapter") and len(adapter_name) != 1: + raise ValueError( + "You are trying to set multiple adapters and you have a PEFT version that does not support multi-adapter inference. Please upgrade to the latest version of PEFT." + " `pip install -U peft` or `pip install -U git+https://github.com/huggingface/peft.git`" + ) + else: + module.active_adapter = adapter_name + _adapters_has_been_set = True + + if not _adapters_has_been_set: + raise ValueError( + "Did not succeeded in setting the adapter. Please make sure you are using a model that supports adapters." + ) + + def disable_adapters(self) -> None: + r""" + Disable all adapters attached to the model and fallback to inference with the base model only. + + If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT + [documentation](https://huggingface.co/docs/peft). + """ + check_peft_version(min_version=MIN_PEFT_VERSION) + + if not self._hf_peft_config_loaded: + raise ValueError("No adapter loaded. Please load an adapter first.") + + from peft.tuners.tuners_utils import BaseTunerLayer + + for _, module in self.named_modules(): + if isinstance(module, BaseTunerLayer): + if hasattr(module, "enable_adapters"): + module.enable_adapters(enabled=False) + else: + # support for older PEFT versions + module.disable_adapters = True + + def enable_adapters(self) -> None: + """ + Enable adapters that are attached to the model. The model uses `self.active_adapters()` to retrieve the + list of adapters to enable. + + If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT + [documentation](https://huggingface.co/docs/peft). + """ + check_peft_version(min_version=MIN_PEFT_VERSION) + + if not self._hf_peft_config_loaded: + raise ValueError("No adapter loaded. Please load an adapter first.") + + from peft.tuners.tuners_utils import BaseTunerLayer + + for _, module in self.named_modules(): + if isinstance(module, BaseTunerLayer): + if hasattr(module, "enable_adapters"): + module.enable_adapters(enabled=True) + else: + # support for older PEFT versions + module.disable_adapters = False + + def active_adapters(self) -> List[str]: + """ + Gets the current list of active adapters of the model. + + If you are not familiar with adapters and PEFT methods, we invite you to read more about them on the PEFT + [documentation](https://huggingface.co/docs/peft). + """ + check_peft_version(min_version=MIN_PEFT_VERSION) + + if not is_peft_available(): + raise ImportError("PEFT is not available. Please install PEFT to use this function: `pip install peft`.") + + if not self._hf_peft_config_loaded: + raise ValueError("No adapter loaded. Please load an adapter first.") + + from peft.tuners.tuners_utils import BaseTunerLayer + + for _, module in self.named_modules(): + if isinstance(module, BaseTunerLayer): + return module.active_adapter diff --git a/diffusers-0.27.0/src/diffusers/loaders/single_file.py b/diffusers-0.27.0/src/diffusers/loaders/single_file.py new file mode 100755 index 0000000..0d384b1 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/single_file.py @@ -0,0 +1,318 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from huggingface_hub.utils import validate_hf_hub_args + +from ..utils import is_transformers_available, logging +from .single_file_utils import ( + create_diffusers_unet_model_from_ldm, + create_diffusers_vae_model_from_ldm, + create_scheduler_from_ldm, + create_text_encoders_and_tokenizers_from_ldm, + fetch_ldm_config_and_checkpoint, + infer_model_type, +) + + +logger = logging.get_logger(__name__) + +# Pipelines that support the SDXL Refiner checkpoint +REFINER_PIPELINES = [ + "StableDiffusionXLImg2ImgPipeline", + "StableDiffusionXLInpaintPipeline", + "StableDiffusionXLControlNetImg2ImgPipeline", +] + +if is_transformers_available(): + from transformers import AutoFeatureExtractor + + +def build_sub_model_components( + pipeline_components, + pipeline_class_name, + component_name, + original_config, + checkpoint, + local_files_only=False, + load_safety_checker=False, + model_type=None, + image_size=None, + torch_dtype=None, + **kwargs, +): + if component_name in pipeline_components: + return {} + + if component_name == "unet": + num_in_channels = kwargs.pop("num_in_channels", None) + upcast_attention = kwargs.pop("upcast_attention", None) + + unet_components = create_diffusers_unet_model_from_ldm( + pipeline_class_name, + original_config, + checkpoint, + num_in_channels=num_in_channels, + image_size=image_size, + torch_dtype=torch_dtype, + model_type=model_type, + upcast_attention=upcast_attention, + ) + return unet_components + + if component_name == "vae": + scaling_factor = kwargs.get("scaling_factor", None) + vae_components = create_diffusers_vae_model_from_ldm( + pipeline_class_name, + original_config, + checkpoint, + image_size, + scaling_factor, + torch_dtype, + model_type=model_type, + ) + return vae_components + + if component_name == "scheduler": + scheduler_type = kwargs.get("scheduler_type", "ddim") + prediction_type = kwargs.get("prediction_type", None) + + scheduler_components = create_scheduler_from_ldm( + pipeline_class_name, + original_config, + checkpoint, + scheduler_type=scheduler_type, + prediction_type=prediction_type, + model_type=model_type, + ) + + return scheduler_components + + if component_name in ["text_encoder", "text_encoder_2", "tokenizer", "tokenizer_2"]: + text_encoder_components = create_text_encoders_and_tokenizers_from_ldm( + original_config, + checkpoint, + model_type=model_type, + local_files_only=local_files_only, + torch_dtype=torch_dtype, + ) + return text_encoder_components + + if component_name == "safety_checker": + if load_safety_checker: + from ..pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + safety_checker = StableDiffusionSafetyChecker.from_pretrained( + "CompVis/stable-diffusion-safety-checker", local_files_only=local_files_only, torch_dtype=torch_dtype + ) + else: + safety_checker = None + return {"safety_checker": safety_checker} + + if component_name == "feature_extractor": + if load_safety_checker: + feature_extractor = AutoFeatureExtractor.from_pretrained( + "CompVis/stable-diffusion-safety-checker", local_files_only=local_files_only + ) + else: + feature_extractor = None + return {"feature_extractor": feature_extractor} + + return + + +def set_additional_components( + pipeline_class_name, + original_config, + checkpoint=None, + model_type=None, +): + components = {} + if pipeline_class_name in REFINER_PIPELINES: + model_type = infer_model_type(original_config, checkpoint=checkpoint, model_type=model_type) + is_refiner = model_type == "SDXL-Refiner" + components.update( + { + "requires_aesthetics_score": is_refiner, + "force_zeros_for_empty_prompt": False if is_refiner else True, + } + ) + + return components + + +class FromSingleFileMixin: + """ + Load model weights saved in the `.ckpt` format into a [`DiffusionPipeline`]. + """ + + @classmethod + @validate_hf_hub_args + def from_single_file(cls, pretrained_model_link_or_path, **kwargs): + r""" + Instantiate a [`DiffusionPipeline`] from pretrained pipeline weights saved in the `.ckpt` or `.safetensors` + format. The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pretrained_model_link_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + - A link to the `.ckpt` file (for example + `"https://huggingface.co//blob/main/.ckpt"`) on the Hub. + - A path to a *file* containing all pipeline weights. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + original_config_file (`str`, *optional*): + The path to the original config file that was used to train the model. If not provided, the config file + will be inferred from the checkpoint file. + model_type (`str`, *optional*): + The type of model to load. If not provided, the model type will be inferred from the checkpoint file. + image_size (`int`, *optional*): + The size of the image output. It's used to configure the `sample_size` parameter of the UNet and VAE model. + load_safety_checker (`bool`, *optional*, defaults to `False`): + Whether to load the safety checker model or not. By default, the safety checker is not loaded unless a `safety_checker` component is passed to the `kwargs`. + num_in_channels (`int`, *optional*): + Specify the number of input channels for the UNet model. Read more about how to configure UNet model with this parameter + [here](https://huggingface.co/docs/diffusers/training/adapt_a_model#configure-unet2dconditionmodel-parameters). + scaling_factor (`float`, *optional*): + The scaling factor to use for the VAE model. If not provided, it is inferred from the config file first. + If the scaling factor is not found in the config file, the default value 0.18215 is used. + scheduler_type (`str`, *optional*): + The type of scheduler to load. If not provided, the scheduler type will be inferred from the checkpoint file. + prediction_type (`str`, *optional*): + The type of prediction to load. If not provided, the prediction type will be inferred from the checkpoint file. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline + class). The overwritten components are passed directly to the pipelines `__init__` method. See example + below for more information. + + Examples: + + ```py + >>> from diffusers import StableDiffusionPipeline + + >>> # Download pipeline from huggingface.co and cache. + >>> pipeline = StableDiffusionPipeline.from_single_file( + ... "https://huggingface.co/WarriorMama777/OrangeMixs/blob/main/Models/AbyssOrangeMix/AbyssOrangeMix.safetensors" + ... ) + + >>> # Download pipeline from local file + >>> # file is downloaded under ./v1-5-pruned-emaonly.ckpt + >>> pipeline = StableDiffusionPipeline.from_single_file("./v1-5-pruned-emaonly") + + >>> # Enable float16 and move to GPU + >>> pipeline = StableDiffusionPipeline.from_single_file( + ... "https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned-emaonly.ckpt", + ... torch_dtype=torch.float16, + ... ) + >>> pipeline.to("cuda") + ``` + """ + original_config_file = kwargs.pop("original_config_file", None) + resume_download = kwargs.pop("resume_download", False) + force_download = kwargs.pop("force_download", False) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + cache_dir = kwargs.pop("cache_dir", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + torch_dtype = kwargs.pop("torch_dtype", None) + + class_name = cls.__name__ + + original_config, checkpoint = fetch_ldm_config_and_checkpoint( + pretrained_model_link_or_path=pretrained_model_link_or_path, + class_name=class_name, + original_config_file=original_config_file, + resume_download=resume_download, + force_download=force_download, + proxies=proxies, + token=token, + revision=revision, + local_files_only=local_files_only, + cache_dir=cache_dir, + ) + + from ..pipelines.pipeline_utils import _get_pipeline_class + + pipeline_class = _get_pipeline_class( + cls, + config=None, + cache_dir=cache_dir, + ) + + expected_modules, optional_kwargs = cls._get_signature_keys(pipeline_class) + passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} + passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} + + model_type = kwargs.pop("model_type", None) + image_size = kwargs.pop("image_size", None) + load_safety_checker = (kwargs.pop("load_safety_checker", False)) or ( + passed_class_obj.get("safety_checker", None) is not None + ) + + init_kwargs = {} + for name in expected_modules: + if name in passed_class_obj: + init_kwargs[name] = passed_class_obj[name] + else: + components = build_sub_model_components( + init_kwargs, + class_name, + name, + original_config, + checkpoint, + model_type=model_type, + image_size=image_size, + load_safety_checker=load_safety_checker, + local_files_only=local_files_only, + torch_dtype=torch_dtype, + **kwargs, + ) + if not components: + continue + init_kwargs.update(components) + + additional_components = set_additional_components( + class_name, original_config, checkpoint=checkpoint, model_type=model_type + ) + if additional_components: + init_kwargs.update(additional_components) + + init_kwargs.update(passed_pipe_kwargs) + pipe = pipeline_class(**init_kwargs) + + if torch_dtype is not None: + pipe.to(dtype=torch_dtype) + + return pipe diff --git a/diffusers-0.27.0/src/diffusers/loaders/single_file_utils.py b/diffusers-0.27.0/src/diffusers/loaders/single_file_utils.py new file mode 100755 index 0000000..cdaa080 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/single_file_utils.py @@ -0,0 +1,1617 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the Stable Diffusion checkpoints.""" + +import os +import re +from contextlib import nullcontext +from io import BytesIO +from urllib.parse import urlparse + +import requests +import yaml + +from ..models.modeling_utils import load_state_dict +from ..schedulers import ( + DDIMScheduler, + DDPMScheduler, + DPMSolverMultistepScheduler, + EDMDPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, +) +from ..utils import is_accelerate_available, is_transformers_available, logging +from ..utils.hub_utils import _get_model_file + + +if is_transformers_available(): + from transformers import ( + CLIPTextConfig, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + ) + +if is_accelerate_available(): + from accelerate import init_empty_weights + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +CONFIG_URLS = { + "v1": "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml", + "v2": "https://raw.githubusercontent.com/Stability-AI/stablediffusion/main/configs/stable-diffusion/v2-inference-v.yaml", + "xl": "https://raw.githubusercontent.com/Stability-AI/generative-models/main/configs/inference/sd_xl_base.yaml", + "xl_refiner": "https://raw.githubusercontent.com/Stability-AI/generative-models/main/configs/inference/sd_xl_refiner.yaml", + "upscale": "https://raw.githubusercontent.com/Stability-AI/stablediffusion/main/configs/stable-diffusion/x4-upscaling.yaml", + "controlnet": "https://raw.githubusercontent.com/lllyasviel/ControlNet/main/models/cldm_v15.yaml", +} + +CHECKPOINT_KEY_NAMES = { + "v2": "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_k.weight", + "xl_base": "conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_proj.bias", + "xl_refiner": "conditioner.embedders.0.model.transformer.resblocks.9.mlp.c_proj.bias", +} + +SCHEDULER_DEFAULT_CONFIG = { + "beta_schedule": "scaled_linear", + "beta_start": 0.00085, + "beta_end": 0.012, + "interpolation_type": "linear", + "num_train_timesteps": 1000, + "prediction_type": "epsilon", + "sample_max_value": 1.0, + "set_alpha_to_one": False, + "skip_prk_steps": True, + "steps_offset": 1, + "timestep_spacing": "leading", +} + + +STABLE_CASCADE_DEFAULT_CONFIGS = { + "stage_c": {"pretrained_model_name_or_path": "diffusers/stable-cascade-configs", "subfolder": "prior"}, + "stage_c_lite": {"pretrained_model_name_or_path": "diffusers/stable-cascade-configs", "subfolder": "prior_lite"}, + "stage_b": {"pretrained_model_name_or_path": "diffusers/stable-cascade-configs", "subfolder": "decoder"}, + "stage_b_lite": {"pretrained_model_name_or_path": "diffusers/stable-cascade-configs", "subfolder": "decoder_lite"}, +} + + +def convert_stable_cascade_unet_single_file_to_diffusers(original_state_dict): + is_stage_c = "clip_txt_mapper.weight" in original_state_dict + + if is_stage_c: + state_dict = {} + for key in original_state_dict.keys(): + if key.endswith("in_proj_weight"): + weights = original_state_dict[key].chunk(3, 0) + state_dict[key.replace("attn.in_proj_weight", "to_q.weight")] = weights[0] + state_dict[key.replace("attn.in_proj_weight", "to_k.weight")] = weights[1] + state_dict[key.replace("attn.in_proj_weight", "to_v.weight")] = weights[2] + elif key.endswith("in_proj_bias"): + weights = original_state_dict[key].chunk(3, 0) + state_dict[key.replace("attn.in_proj_bias", "to_q.bias")] = weights[0] + state_dict[key.replace("attn.in_proj_bias", "to_k.bias")] = weights[1] + state_dict[key.replace("attn.in_proj_bias", "to_v.bias")] = weights[2] + elif key.endswith("out_proj.weight"): + weights = original_state_dict[key] + state_dict[key.replace("attn.out_proj.weight", "to_out.0.weight")] = weights + elif key.endswith("out_proj.bias"): + weights = original_state_dict[key] + state_dict[key.replace("attn.out_proj.bias", "to_out.0.bias")] = weights + else: + state_dict[key] = original_state_dict[key] + else: + state_dict = {} + for key in original_state_dict.keys(): + if key.endswith("in_proj_weight"): + weights = original_state_dict[key].chunk(3, 0) + state_dict[key.replace("attn.in_proj_weight", "to_q.weight")] = weights[0] + state_dict[key.replace("attn.in_proj_weight", "to_k.weight")] = weights[1] + state_dict[key.replace("attn.in_proj_weight", "to_v.weight")] = weights[2] + elif key.endswith("in_proj_bias"): + weights = original_state_dict[key].chunk(3, 0) + state_dict[key.replace("attn.in_proj_bias", "to_q.bias")] = weights[0] + state_dict[key.replace("attn.in_proj_bias", "to_k.bias")] = weights[1] + state_dict[key.replace("attn.in_proj_bias", "to_v.bias")] = weights[2] + elif key.endswith("out_proj.weight"): + weights = original_state_dict[key] + state_dict[key.replace("attn.out_proj.weight", "to_out.0.weight")] = weights + elif key.endswith("out_proj.bias"): + weights = original_state_dict[key] + state_dict[key.replace("attn.out_proj.bias", "to_out.0.bias")] = weights + # rename clip_mapper to clip_txt_pooled_mapper + elif key.endswith("clip_mapper.weight"): + weights = original_state_dict[key] + state_dict[key.replace("clip_mapper.weight", "clip_txt_pooled_mapper.weight")] = weights + elif key.endswith("clip_mapper.bias"): + weights = original_state_dict[key] + state_dict[key.replace("clip_mapper.bias", "clip_txt_pooled_mapper.bias")] = weights + else: + state_dict[key] = original_state_dict[key] + + return state_dict + + +def infer_stable_cascade_single_file_config(checkpoint): + is_stage_c = "clip_txt_mapper.weight" in checkpoint + is_stage_b = "down_blocks.1.0.channelwise.0.weight" in checkpoint + + if is_stage_c and (checkpoint["clip_txt_mapper.weight"].shape[0] == 1536): + config_type = "stage_c_lite" + elif is_stage_c and (checkpoint["clip_txt_mapper.weight"].shape[0] == 2048): + config_type = "stage_c" + elif is_stage_b and checkpoint["down_blocks.1.0.channelwise.0.weight"].shape[-1] == 576: + config_type = "stage_b_lite" + elif is_stage_b and checkpoint["down_blocks.1.0.channelwise.0.weight"].shape[-1] == 640: + config_type = "stage_b" + + return STABLE_CASCADE_DEFAULT_CONFIGS[config_type] + + +DIFFUSERS_TO_LDM_MAPPING = { + "unet": { + "layers": { + "time_embedding.linear_1.weight": "time_embed.0.weight", + "time_embedding.linear_1.bias": "time_embed.0.bias", + "time_embedding.linear_2.weight": "time_embed.2.weight", + "time_embedding.linear_2.bias": "time_embed.2.bias", + "conv_in.weight": "input_blocks.0.0.weight", + "conv_in.bias": "input_blocks.0.0.bias", + "conv_norm_out.weight": "out.0.weight", + "conv_norm_out.bias": "out.0.bias", + "conv_out.weight": "out.2.weight", + "conv_out.bias": "out.2.bias", + }, + "class_embed_type": { + "class_embedding.linear_1.weight": "label_emb.0.0.weight", + "class_embedding.linear_1.bias": "label_emb.0.0.bias", + "class_embedding.linear_2.weight": "label_emb.0.2.weight", + "class_embedding.linear_2.bias": "label_emb.0.2.bias", + }, + "addition_embed_type": { + "add_embedding.linear_1.weight": "label_emb.0.0.weight", + "add_embedding.linear_1.bias": "label_emb.0.0.bias", + "add_embedding.linear_2.weight": "label_emb.0.2.weight", + "add_embedding.linear_2.bias": "label_emb.0.2.bias", + }, + }, + "controlnet": { + "layers": { + "time_embedding.linear_1.weight": "time_embed.0.weight", + "time_embedding.linear_1.bias": "time_embed.0.bias", + "time_embedding.linear_2.weight": "time_embed.2.weight", + "time_embedding.linear_2.bias": "time_embed.2.bias", + "conv_in.weight": "input_blocks.0.0.weight", + "conv_in.bias": "input_blocks.0.0.bias", + "controlnet_cond_embedding.conv_in.weight": "input_hint_block.0.weight", + "controlnet_cond_embedding.conv_in.bias": "input_hint_block.0.bias", + "controlnet_cond_embedding.conv_out.weight": "input_hint_block.14.weight", + "controlnet_cond_embedding.conv_out.bias": "input_hint_block.14.bias", + }, + "class_embed_type": { + "class_embedding.linear_1.weight": "label_emb.0.0.weight", + "class_embedding.linear_1.bias": "label_emb.0.0.bias", + "class_embedding.linear_2.weight": "label_emb.0.2.weight", + "class_embedding.linear_2.bias": "label_emb.0.2.bias", + }, + "addition_embed_type": { + "add_embedding.linear_1.weight": "label_emb.0.0.weight", + "add_embedding.linear_1.bias": "label_emb.0.0.bias", + "add_embedding.linear_2.weight": "label_emb.0.2.weight", + "add_embedding.linear_2.bias": "label_emb.0.2.bias", + }, + }, + "vae": { + "encoder.conv_in.weight": "encoder.conv_in.weight", + "encoder.conv_in.bias": "encoder.conv_in.bias", + "encoder.conv_out.weight": "encoder.conv_out.weight", + "encoder.conv_out.bias": "encoder.conv_out.bias", + "encoder.conv_norm_out.weight": "encoder.norm_out.weight", + "encoder.conv_norm_out.bias": "encoder.norm_out.bias", + "decoder.conv_in.weight": "decoder.conv_in.weight", + "decoder.conv_in.bias": "decoder.conv_in.bias", + "decoder.conv_out.weight": "decoder.conv_out.weight", + "decoder.conv_out.bias": "decoder.conv_out.bias", + "decoder.conv_norm_out.weight": "decoder.norm_out.weight", + "decoder.conv_norm_out.bias": "decoder.norm_out.bias", + "quant_conv.weight": "quant_conv.weight", + "quant_conv.bias": "quant_conv.bias", + "post_quant_conv.weight": "post_quant_conv.weight", + "post_quant_conv.bias": "post_quant_conv.bias", + }, + "openclip": { + "layers": { + "text_model.embeddings.position_embedding.weight": "positional_embedding", + "text_model.embeddings.token_embedding.weight": "token_embedding.weight", + "text_model.final_layer_norm.weight": "ln_final.weight", + "text_model.final_layer_norm.bias": "ln_final.bias", + "text_projection.weight": "text_projection", + }, + "transformer": { + "text_model.encoder.layers.": "resblocks.", + "layer_norm1": "ln_1", + "layer_norm2": "ln_2", + ".fc1.": ".c_fc.", + ".fc2.": ".c_proj.", + ".self_attn": ".attn", + "transformer.text_model.final_layer_norm.": "ln_final.", + "transformer.text_model.embeddings.token_embedding.weight": "token_embedding.weight", + "transformer.text_model.embeddings.position_embedding.weight": "positional_embedding", + }, + }, +} + +LDM_VAE_KEY = "first_stage_model." +LDM_VAE_DEFAULT_SCALING_FACTOR = 0.18215 +PLAYGROUND_VAE_SCALING_FACTOR = 0.5 +LDM_UNET_KEY = "model.diffusion_model." +LDM_CONTROLNET_KEY = "control_model." +LDM_CLIP_PREFIX_TO_REMOVE = ["cond_stage_model.transformer.", "conditioner.embedders.0.transformer."] +LDM_OPEN_CLIP_TEXT_PROJECTION_DIM = 1024 + +SD_2_TEXT_ENCODER_KEYS_TO_IGNORE = [ + "cond_stage_model.model.transformer.resblocks.23.attn.in_proj_bias", + "cond_stage_model.model.transformer.resblocks.23.attn.in_proj_weight", + "cond_stage_model.model.transformer.resblocks.23.attn.out_proj.bias", + "cond_stage_model.model.transformer.resblocks.23.attn.out_proj.weight", + "cond_stage_model.model.transformer.resblocks.23.ln_1.bias", + "cond_stage_model.model.transformer.resblocks.23.ln_1.weight", + "cond_stage_model.model.transformer.resblocks.23.ln_2.bias", + "cond_stage_model.model.transformer.resblocks.23.ln_2.weight", + "cond_stage_model.model.transformer.resblocks.23.mlp.c_fc.bias", + "cond_stage_model.model.transformer.resblocks.23.mlp.c_fc.weight", + "cond_stage_model.model.transformer.resblocks.23.mlp.c_proj.bias", + "cond_stage_model.model.transformer.resblocks.23.mlp.c_proj.weight", + "cond_stage_model.model.text_projection", +] + + +VALID_URL_PREFIXES = ["https://huggingface.co/", "huggingface.co/", "hf.co/", "https://hf.co/"] + + +def _extract_repo_id_and_weights_name(pretrained_model_name_or_path): + pattern = r"([^/]+)/([^/]+)/(?:blob/main/)?(.+)" + weights_name = None + repo_id = (None,) + for prefix in VALID_URL_PREFIXES: + pretrained_model_name_or_path = pretrained_model_name_or_path.replace(prefix, "") + match = re.match(pattern, pretrained_model_name_or_path) + if not match: + return repo_id, weights_name + + repo_id = f"{match.group(1)}/{match.group(2)}" + weights_name = match.group(3) + + return repo_id, weights_name + + +def fetch_ldm_config_and_checkpoint( + pretrained_model_link_or_path, + class_name, + original_config_file=None, + resume_download=False, + force_download=False, + proxies=None, + token=None, + cache_dir=None, + local_files_only=None, + revision=None, +): + checkpoint = load_single_file_model_checkpoint( + pretrained_model_link_or_path, + resume_download=resume_download, + force_download=force_download, + proxies=proxies, + token=token, + cache_dir=cache_dir, + local_files_only=local_files_only, + revision=revision, + ) + original_config = fetch_original_config(class_name, checkpoint, original_config_file) + + return original_config, checkpoint + + +def load_single_file_model_checkpoint( + pretrained_model_link_or_path, + resume_download=False, + force_download=False, + proxies=None, + token=None, + cache_dir=None, + local_files_only=None, + revision=None, +): + if os.path.isfile(pretrained_model_link_or_path): + checkpoint = load_state_dict(pretrained_model_link_or_path) + else: + repo_id, weights_name = _extract_repo_id_and_weights_name(pretrained_model_link_or_path) + checkpoint_path = _get_model_file( + repo_id, + weights_name=weights_name, + force_download=force_download, + cache_dir=cache_dir, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + ) + checkpoint = load_state_dict(checkpoint_path) + + # some checkpoints contain the model state dict under a "state_dict" key + while "state_dict" in checkpoint: + checkpoint = checkpoint["state_dict"] + + return checkpoint + + +def infer_original_config_file(class_name, checkpoint): + if CHECKPOINT_KEY_NAMES["v2"] in checkpoint and checkpoint[CHECKPOINT_KEY_NAMES["v2"]].shape[-1] == 1024: + config_url = CONFIG_URLS["v2"] + + elif CHECKPOINT_KEY_NAMES["xl_base"] in checkpoint: + config_url = CONFIG_URLS["xl"] + + elif CHECKPOINT_KEY_NAMES["xl_refiner"] in checkpoint: + config_url = CONFIG_URLS["xl_refiner"] + + elif class_name == "StableDiffusionUpscalePipeline": + config_url = CONFIG_URLS["upscale"] + + elif class_name == "ControlNetModel": + config_url = CONFIG_URLS["controlnet"] + + else: + config_url = CONFIG_URLS["v1"] + + original_config_file = BytesIO(requests.get(config_url).content) + + return original_config_file + + +def fetch_original_config(pipeline_class_name, checkpoint, original_config_file=None): + def is_valid_url(url): + result = urlparse(url) + if result.scheme and result.netloc: + return True + + return False + + if original_config_file is None: + original_config_file = infer_original_config_file(pipeline_class_name, checkpoint) + + elif os.path.isfile(original_config_file): + with open(original_config_file, "r") as fp: + original_config_file = fp.read() + + elif is_valid_url(original_config_file): + original_config_file = BytesIO(requests.get(original_config_file).content) + + else: + raise ValueError("Invalid `original_config_file` provided. Please set it to a valid file path or URL.") + + original_config = yaml.safe_load(original_config_file) + + return original_config + + +def infer_model_type(original_config, checkpoint, model_type=None): + if model_type is not None: + return model_type + + has_cond_stage_config = ( + "cond_stage_config" in original_config["model"]["params"] + and original_config["model"]["params"]["cond_stage_config"] is not None + ) + has_network_config = ( + "network_config" in original_config["model"]["params"] + and original_config["model"]["params"]["network_config"] is not None + ) + + if has_cond_stage_config: + model_type = original_config["model"]["params"]["cond_stage_config"]["target"].split(".")[-1] + + elif has_network_config: + context_dim = original_config["model"]["params"]["network_config"]["params"]["context_dim"] + if "edm_mean" in checkpoint and "edm_std" in checkpoint: + model_type = "Playground" + elif context_dim == 2048: + model_type = "SDXL" + else: + model_type = "SDXL-Refiner" + else: + raise ValueError("Unable to infer model type from config") + + logger.debug(f"No `model_type` given, `model_type` inferred as: {model_type}") + + return model_type + + +def get_default_scheduler_config(): + return SCHEDULER_DEFAULT_CONFIG + + +def set_image_size(pipeline_class_name, original_config, checkpoint, image_size=None, model_type=None): + if image_size: + return image_size + + global_step = checkpoint["global_step"] if "global_step" in checkpoint else None + model_type = infer_model_type(original_config, checkpoint, model_type) + + if pipeline_class_name == "StableDiffusionUpscalePipeline": + image_size = original_config["model"]["params"]["unet_config"]["params"]["image_size"] + return image_size + + elif model_type in ["SDXL", "SDXL-Refiner", "Playground"]: + image_size = 1024 + return image_size + + elif ( + "parameterization" in original_config["model"]["params"] + and original_config["model"]["params"]["parameterization"] == "v" + ): + # NOTE: For stable diffusion 2 base one has to pass `image_size==512` + # as it relies on a brittle global step parameter here + image_size = 512 if global_step == 875000 else 768 + return image_size + + else: + image_size = 512 + return image_size + + +# Copied from diffusers.pipelines.stable_diffusion.convert_from_ckpt.conv_attn_to_linear +def conv_attn_to_linear(checkpoint): + keys = list(checkpoint.keys()) + attn_keys = ["query.weight", "key.weight", "value.weight"] + for key in keys: + if ".".join(key.split(".")[-2:]) in attn_keys: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0, 0] + elif "proj_attn.weight" in key: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0] + + +def create_unet_diffusers_config(original_config, image_size: int): + """ + Creates a config for the diffusers based on the config of the LDM model. + """ + if ( + "unet_config" in original_config["model"]["params"] + and original_config["model"]["params"]["unet_config"] is not None + ): + unet_params = original_config["model"]["params"]["unet_config"]["params"] + else: + unet_params = original_config["model"]["params"]["network_config"]["params"] + + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + block_out_channels = [unet_params["model_channels"] * mult for mult in unet_params["channel_mult"]] + + down_block_types = [] + resolution = 1 + for i in range(len(block_out_channels)): + block_type = "CrossAttnDownBlock2D" if resolution in unet_params["attention_resolutions"] else "DownBlock2D" + down_block_types.append(block_type) + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + block_type = "CrossAttnUpBlock2D" if resolution in unet_params["attention_resolutions"] else "UpBlock2D" + up_block_types.append(block_type) + resolution //= 2 + + if unet_params["transformer_depth"] is not None: + transformer_layers_per_block = ( + unet_params["transformer_depth"] + if isinstance(unet_params["transformer_depth"], int) + else list(unet_params["transformer_depth"]) + ) + else: + transformer_layers_per_block = 1 + + vae_scale_factor = 2 ** (len(vae_params["ch_mult"]) - 1) + + head_dim = unet_params["num_heads"] if "num_heads" in unet_params else None + use_linear_projection = ( + unet_params["use_linear_in_transformer"] if "use_linear_in_transformer" in unet_params else False + ) + if use_linear_projection: + # stable diffusion 2-base-512 and 2-768 + if head_dim is None: + head_dim_mult = unet_params["model_channels"] // unet_params["num_head_channels"] + head_dim = [head_dim_mult * c for c in list(unet_params["channel_mult"])] + + class_embed_type = None + addition_embed_type = None + addition_time_embed_dim = None + projection_class_embeddings_input_dim = None + context_dim = None + + if unet_params["context_dim"] is not None: + context_dim = ( + unet_params["context_dim"] + if isinstance(unet_params["context_dim"], int) + else unet_params["context_dim"][0] + ) + + if "num_classes" in unet_params: + if unet_params["num_classes"] == "sequential": + if context_dim in [2048, 1280]: + # SDXL + addition_embed_type = "text_time" + addition_time_embed_dim = 256 + else: + class_embed_type = "projection" + assert "adm_in_channels" in unet_params + projection_class_embeddings_input_dim = unet_params["adm_in_channels"] + + config = { + "sample_size": image_size // vae_scale_factor, + "in_channels": unet_params["in_channels"], + "down_block_types": down_block_types, + "block_out_channels": block_out_channels, + "layers_per_block": unet_params["num_res_blocks"], + "cross_attention_dim": context_dim, + "attention_head_dim": head_dim, + "use_linear_projection": use_linear_projection, + "class_embed_type": class_embed_type, + "addition_embed_type": addition_embed_type, + "addition_time_embed_dim": addition_time_embed_dim, + "projection_class_embeddings_input_dim": projection_class_embeddings_input_dim, + "transformer_layers_per_block": transformer_layers_per_block, + } + + if "disable_self_attentions" in unet_params: + config["only_cross_attention"] = unet_params["disable_self_attentions"] + + if "num_classes" in unet_params and isinstance(unet_params["num_classes"], int): + config["num_class_embeds"] = unet_params["num_classes"] + + config["out_channels"] = unet_params["out_channels"] + config["up_block_types"] = up_block_types + + return config + + +def create_controlnet_diffusers_config(original_config, image_size: int): + unet_params = original_config["model"]["params"]["control_stage_config"]["params"] + diffusers_unet_config = create_unet_diffusers_config(original_config, image_size=image_size) + + controlnet_config = { + "conditioning_channels": unet_params["hint_channels"], + "in_channels": diffusers_unet_config["in_channels"], + "down_block_types": diffusers_unet_config["down_block_types"], + "block_out_channels": diffusers_unet_config["block_out_channels"], + "layers_per_block": diffusers_unet_config["layers_per_block"], + "cross_attention_dim": diffusers_unet_config["cross_attention_dim"], + "attention_head_dim": diffusers_unet_config["attention_head_dim"], + "use_linear_projection": diffusers_unet_config["use_linear_projection"], + "class_embed_type": diffusers_unet_config["class_embed_type"], + "addition_embed_type": diffusers_unet_config["addition_embed_type"], + "addition_time_embed_dim": diffusers_unet_config["addition_time_embed_dim"], + "projection_class_embeddings_input_dim": diffusers_unet_config["projection_class_embeddings_input_dim"], + "transformer_layers_per_block": diffusers_unet_config["transformer_layers_per_block"], + } + + return controlnet_config + + +def create_vae_diffusers_config(original_config, image_size, scaling_factor=None, latents_mean=None, latents_std=None): + """ + Creates a config for the diffusers based on the config of the LDM model. + """ + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + if (scaling_factor is None) and (latents_mean is not None) and (latents_std is not None): + scaling_factor = PLAYGROUND_VAE_SCALING_FACTOR + elif (scaling_factor is None) and ("scale_factor" in original_config["model"]["params"]): + scaling_factor = original_config["model"]["params"]["scale_factor"] + elif scaling_factor is None: + scaling_factor = LDM_VAE_DEFAULT_SCALING_FACTOR + + block_out_channels = [vae_params["ch"] * mult for mult in vae_params["ch_mult"]] + down_block_types = ["DownEncoderBlock2D"] * len(block_out_channels) + up_block_types = ["UpDecoderBlock2D"] * len(block_out_channels) + + config = { + "sample_size": image_size, + "in_channels": vae_params["in_channels"], + "out_channels": vae_params["out_ch"], + "down_block_types": down_block_types, + "up_block_types": up_block_types, + "block_out_channels": block_out_channels, + "latent_channels": vae_params["z_channels"], + "layers_per_block": vae_params["num_res_blocks"], + "scaling_factor": scaling_factor, + } + if latents_mean is not None and latents_std is not None: + config.update({"latents_mean": latents_mean, "latents_std": latents_std}) + + return config + + +def update_unet_resnet_ldm_to_diffusers(ldm_keys, new_checkpoint, checkpoint, mapping=None): + for ldm_key in ldm_keys: + diffusers_key = ( + ldm_key.replace("in_layers.0", "norm1") + .replace("in_layers.2", "conv1") + .replace("out_layers.0", "norm2") + .replace("out_layers.3", "conv2") + .replace("emb_layers.1", "time_emb_proj") + .replace("skip_connection", "conv_shortcut") + ) + if mapping: + diffusers_key = diffusers_key.replace(mapping["old"], mapping["new"]) + new_checkpoint[diffusers_key] = checkpoint.pop(ldm_key) + + +def update_unet_attention_ldm_to_diffusers(ldm_keys, new_checkpoint, checkpoint, mapping): + for ldm_key in ldm_keys: + diffusers_key = ldm_key.replace(mapping["old"], mapping["new"]) + new_checkpoint[diffusers_key] = checkpoint.pop(ldm_key) + + +def convert_ldm_unet_checkpoint(checkpoint, config, extract_ema=False): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + # extract state_dict for UNet + unet_state_dict = {} + keys = list(checkpoint.keys()) + unet_key = LDM_UNET_KEY + + # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA + if sum(k.startswith("model_ema") for k in keys) > 100 and extract_ema: + logger.warning("Checkpoint has both EMA and non-EMA weights.") + logger.warning( + "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA" + " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag." + ) + for key in keys: + if key.startswith("model.diffusion_model"): + flat_ema_key = "model_ema." + "".join(key.split(".")[1:]) + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(flat_ema_key) + else: + if sum(k.startswith("model_ema") for k in keys) > 100: + logger.warning( + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA" + " weights (usually better for inference), please make sure to add the `--extract_ema` flag." + ) + for key in keys: + if key.startswith(unet_key): + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key) + + new_checkpoint = {} + ldm_unet_keys = DIFFUSERS_TO_LDM_MAPPING["unet"]["layers"] + for diffusers_key, ldm_key in ldm_unet_keys.items(): + if ldm_key not in unet_state_dict: + continue + new_checkpoint[diffusers_key] = unet_state_dict[ldm_key] + + if ("class_embed_type" in config) and (config["class_embed_type"] in ["timestep", "projection"]): + class_embed_keys = DIFFUSERS_TO_LDM_MAPPING["unet"]["class_embed_type"] + for diffusers_key, ldm_key in class_embed_keys.items(): + new_checkpoint[diffusers_key] = unet_state_dict[ldm_key] + + if ("addition_embed_type" in config) and (config["addition_embed_type"] == "text_time"): + addition_embed_keys = DIFFUSERS_TO_LDM_MAPPING["unet"]["addition_embed_type"] + for diffusers_key, ldm_key in addition_embed_keys.items(): + new_checkpoint[diffusers_key] = unet_state_dict[ldm_key] + + # Relevant to StableDiffusionUpscalePipeline + if "num_class_embeds" in config: + if (config["num_class_embeds"] is not None) and ("label_emb.weight" in unet_state_dict): + new_checkpoint["class_embedding.weight"] = unet_state_dict["label_emb.weight"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key] + for layer_id in range(num_output_blocks) + } + + # Down blocks + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + update_unet_resnet_ldm_to_diffusers( + resnets, + new_checkpoint, + unet_state_dict, + {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"}, + ) + + if f"input_blocks.{i}.0.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + if attentions: + update_unet_attention_ldm_to_diffusers( + attentions, + new_checkpoint, + unet_state_dict, + {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"}, + ) + + # Mid blocks + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + update_unet_resnet_ldm_to_diffusers( + resnet_0, new_checkpoint, unet_state_dict, mapping={"old": "middle_block.0", "new": "mid_block.resnets.0"} + ) + update_unet_resnet_ldm_to_diffusers( + resnet_1, new_checkpoint, unet_state_dict, mapping={"old": "middle_block.2", "new": "mid_block.resnets.1"} + ) + update_unet_attention_ldm_to_diffusers( + attentions, new_checkpoint, unet_state_dict, mapping={"old": "middle_block.1", "new": "mid_block.attentions.0"} + ) + + # Up Blocks + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + + resnets = [ + key for key in output_blocks[i] if f"output_blocks.{i}.0" in key and f"output_blocks.{i}.0.op" not in key + ] + update_unet_resnet_ldm_to_diffusers( + resnets, + new_checkpoint, + unet_state_dict, + {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"}, + ) + + attentions = [ + key for key in output_blocks[i] if f"output_blocks.{i}.1" in key and f"output_blocks.{i}.1.conv" not in key + ] + if attentions: + update_unet_attention_ldm_to_diffusers( + attentions, + new_checkpoint, + unet_state_dict, + {"old": f"output_blocks.{i}.1", "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}"}, + ) + + if f"output_blocks.{i}.1.conv.weight" in unet_state_dict: + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.1.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.1.conv.bias" + ] + if f"output_blocks.{i}.2.conv.weight" in unet_state_dict: + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.2.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.2.conv.bias" + ] + + return new_checkpoint + + +def convert_controlnet_checkpoint( + checkpoint, + config, +): + # Some controlnet ckpt files are distributed independently from the rest of the + # model components i.e. https://huggingface.co/thibaud/controlnet-sd21/ + if "time_embed.0.weight" in checkpoint: + controlnet_state_dict = checkpoint + + else: + controlnet_state_dict = {} + keys = list(checkpoint.keys()) + controlnet_key = LDM_CONTROLNET_KEY + for key in keys: + if key.startswith(controlnet_key): + controlnet_state_dict[key.replace(controlnet_key, "")] = checkpoint.pop(key) + + new_checkpoint = {} + ldm_controlnet_keys = DIFFUSERS_TO_LDM_MAPPING["controlnet"]["layers"] + for diffusers_key, ldm_key in ldm_controlnet_keys.items(): + if ldm_key not in controlnet_state_dict: + continue + new_checkpoint[diffusers_key] = controlnet_state_dict[ldm_key] + + # Retrieves the keys for the input blocks only + num_input_blocks = len( + {".".join(layer.split(".")[:2]) for layer in controlnet_state_dict if "input_blocks" in layer} + ) + input_blocks = { + layer_id: [key for key in controlnet_state_dict if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Down blocks + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + update_unet_resnet_ldm_to_diffusers( + resnets, + new_checkpoint, + controlnet_state_dict, + {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"}, + ) + + if f"input_blocks.{i}.0.op.weight" in controlnet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = controlnet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = controlnet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + if attentions: + update_unet_attention_ldm_to_diffusers( + attentions, + new_checkpoint, + controlnet_state_dict, + {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"}, + ) + + # controlnet down blocks + for i in range(num_input_blocks): + new_checkpoint[f"controlnet_down_blocks.{i}.weight"] = controlnet_state_dict.pop(f"zero_convs.{i}.0.weight") + new_checkpoint[f"controlnet_down_blocks.{i}.bias"] = controlnet_state_dict.pop(f"zero_convs.{i}.0.bias") + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len( + {".".join(layer.split(".")[:2]) for layer in controlnet_state_dict if "middle_block" in layer} + ) + middle_blocks = { + layer_id: [key for key in controlnet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + if middle_blocks: + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + update_unet_resnet_ldm_to_diffusers( + resnet_0, + new_checkpoint, + controlnet_state_dict, + mapping={"old": "middle_block.0", "new": "mid_block.resnets.0"}, + ) + update_unet_resnet_ldm_to_diffusers( + resnet_1, + new_checkpoint, + controlnet_state_dict, + mapping={"old": "middle_block.2", "new": "mid_block.resnets.1"}, + ) + update_unet_attention_ldm_to_diffusers( + attentions, + new_checkpoint, + controlnet_state_dict, + mapping={"old": "middle_block.1", "new": "mid_block.attentions.0"}, + ) + + # mid block + new_checkpoint["controlnet_mid_block.weight"] = controlnet_state_dict.pop("middle_block_out.0.weight") + new_checkpoint["controlnet_mid_block.bias"] = controlnet_state_dict.pop("middle_block_out.0.bias") + + # controlnet cond embedding blocks + cond_embedding_blocks = { + ".".join(layer.split(".")[:2]) + for layer in controlnet_state_dict + if "input_hint_block" in layer and ("input_hint_block.0" not in layer) and ("input_hint_block.14" not in layer) + } + num_cond_embedding_blocks = len(cond_embedding_blocks) + + for idx in range(1, num_cond_embedding_blocks + 1): + diffusers_idx = idx - 1 + cond_block_id = 2 * idx + + new_checkpoint[f"controlnet_cond_embedding.blocks.{diffusers_idx}.weight"] = controlnet_state_dict.pop( + f"input_hint_block.{cond_block_id}.weight" + ) + new_checkpoint[f"controlnet_cond_embedding.blocks.{diffusers_idx}.bias"] = controlnet_state_dict.pop( + f"input_hint_block.{cond_block_id}.bias" + ) + + return new_checkpoint + + +def create_diffusers_controlnet_model_from_ldm( + pipeline_class_name, original_config, checkpoint, upcast_attention=False, image_size=None, torch_dtype=None +): + # import here to avoid circular imports + from ..models import ControlNetModel + + image_size = set_image_size(pipeline_class_name, original_config, checkpoint, image_size=image_size) + + diffusers_config = create_controlnet_diffusers_config(original_config, image_size=image_size) + diffusers_config["upcast_attention"] = upcast_attention + + diffusers_format_controlnet_checkpoint = convert_controlnet_checkpoint(checkpoint, diffusers_config) + + ctx = init_empty_weights if is_accelerate_available() else nullcontext + with ctx(): + controlnet = ControlNetModel(**diffusers_config) + + if is_accelerate_available(): + from ..models.modeling_utils import load_model_dict_into_meta + + unexpected_keys = load_model_dict_into_meta( + controlnet, diffusers_format_controlnet_checkpoint, dtype=torch_dtype + ) + if controlnet._keys_to_ignore_on_load_unexpected is not None: + for pat in controlnet._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint were not used when initializing {controlnet.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + else: + controlnet.load_state_dict(diffusers_format_controlnet_checkpoint) + + if torch_dtype is not None: + controlnet = controlnet.to(torch_dtype) + + return {"controlnet": controlnet} + + +def update_vae_resnet_ldm_to_diffusers(keys, new_checkpoint, checkpoint, mapping): + for ldm_key in keys: + diffusers_key = ldm_key.replace(mapping["old"], mapping["new"]).replace("nin_shortcut", "conv_shortcut") + new_checkpoint[diffusers_key] = checkpoint.pop(ldm_key) + + +def update_vae_attentions_ldm_to_diffusers(keys, new_checkpoint, checkpoint, mapping): + for ldm_key in keys: + diffusers_key = ( + ldm_key.replace(mapping["old"], mapping["new"]) + .replace("norm.weight", "group_norm.weight") + .replace("norm.bias", "group_norm.bias") + .replace("q.weight", "to_q.weight") + .replace("q.bias", "to_q.bias") + .replace("k.weight", "to_k.weight") + .replace("k.bias", "to_k.bias") + .replace("v.weight", "to_v.weight") + .replace("v.bias", "to_v.bias") + .replace("proj_out.weight", "to_out.0.weight") + .replace("proj_out.bias", "to_out.0.bias") + ) + new_checkpoint[diffusers_key] = checkpoint.pop(ldm_key) + + # proj_attn.weight has to be converted from conv 1D to linear + shape = new_checkpoint[diffusers_key].shape + + if len(shape) == 3: + new_checkpoint[diffusers_key] = new_checkpoint[diffusers_key][:, :, 0] + elif len(shape) == 4: + new_checkpoint[diffusers_key] = new_checkpoint[diffusers_key][:, :, 0, 0] + + +def convert_ldm_vae_checkpoint(checkpoint, config): + # extract state dict for VAE + # remove the LDM_VAE_KEY prefix from the ldm checkpoint keys so that it is easier to map them to diffusers keys + vae_state_dict = {} + keys = list(checkpoint.keys()) + vae_key = LDM_VAE_KEY if any(k.startswith(LDM_VAE_KEY) for k in keys) else "" + for key in keys: + if key.startswith(vae_key): + vae_state_dict[key.replace(vae_key, "")] = checkpoint.get(key) + + new_checkpoint = {} + vae_diffusers_ldm_map = DIFFUSERS_TO_LDM_MAPPING["vae"] + for diffusers_key, ldm_key in vae_diffusers_ldm_map.items(): + if ldm_key not in vae_state_dict: + continue + new_checkpoint[diffusers_key] = vae_state_dict[ldm_key] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len(config["down_block_types"]) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + update_vae_resnet_ldm_to_diffusers( + resnets, + new_checkpoint, + vae_state_dict, + mapping={"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"}, + ) + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + update_vae_resnet_ldm_to_diffusers( + resnets, + new_checkpoint, + vae_state_dict, + mapping={"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"}, + ) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + update_vae_attentions_ldm_to_diffusers( + mid_attentions, new_checkpoint, vae_state_dict, mapping={"old": "mid.attn_1", "new": "mid_block.attentions.0"} + ) + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len(config["up_block_types"]) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + update_vae_resnet_ldm_to_diffusers( + resnets, + new_checkpoint, + vae_state_dict, + mapping={"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"}, + ) + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + update_vae_resnet_ldm_to_diffusers( + resnets, + new_checkpoint, + vae_state_dict, + mapping={"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"}, + ) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + update_vae_attentions_ldm_to_diffusers( + mid_attentions, new_checkpoint, vae_state_dict, mapping={"old": "mid.attn_1", "new": "mid_block.attentions.0"} + ) + conv_attn_to_linear(new_checkpoint) + + return new_checkpoint + + +def create_text_encoder_from_ldm_clip_checkpoint(config_name, checkpoint, local_files_only=False, torch_dtype=None): + try: + config = CLIPTextConfig.from_pretrained(config_name, local_files_only=local_files_only) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the configuration in the following path: 'openai/clip-vit-large-patch14'." + ) + + ctx = init_empty_weights if is_accelerate_available() else nullcontext + with ctx(): + text_model = CLIPTextModel(config) + + keys = list(checkpoint.keys()) + text_model_dict = {} + + remove_prefixes = LDM_CLIP_PREFIX_TO_REMOVE + + for key in keys: + for prefix in remove_prefixes: + if key.startswith(prefix): + diffusers_key = key.replace(prefix, "") + text_model_dict[diffusers_key] = checkpoint[key] + + if is_accelerate_available(): + from ..models.modeling_utils import load_model_dict_into_meta + + unexpected_keys = load_model_dict_into_meta(text_model, text_model_dict, dtype=torch_dtype) + if text_model._keys_to_ignore_on_load_unexpected is not None: + for pat in text_model._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint were not used when initializing {text_model.__class__.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + else: + if not (hasattr(text_model, "embeddings") and hasattr(text_model.embeddings.position_ids)): + text_model_dict.pop("text_model.embeddings.position_ids", None) + + text_model.load_state_dict(text_model_dict) + + if torch_dtype is not None: + text_model = text_model.to(torch_dtype) + + return text_model + + +def create_text_encoder_from_open_clip_checkpoint( + config_name, + checkpoint, + prefix="cond_stage_model.model.", + has_projection=False, + local_files_only=False, + torch_dtype=None, + **config_kwargs, +): + try: + config = CLIPTextConfig.from_pretrained(config_name, **config_kwargs, local_files_only=local_files_only) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the configuration in the following path: '{config_name}'." + ) + + ctx = init_empty_weights if is_accelerate_available() else nullcontext + with ctx(): + text_model = CLIPTextModelWithProjection(config) if has_projection else CLIPTextModel(config) + + text_model_dict = {} + text_proj_key = prefix + "text_projection" + text_proj_dim = ( + int(checkpoint[text_proj_key].shape[0]) if text_proj_key in checkpoint else LDM_OPEN_CLIP_TEXT_PROJECTION_DIM + ) + text_model_dict["text_model.embeddings.position_ids"] = text_model.text_model.embeddings.get_buffer("position_ids") + + keys = list(checkpoint.keys()) + keys_to_ignore = SD_2_TEXT_ENCODER_KEYS_TO_IGNORE + + openclip_diffusers_ldm_map = DIFFUSERS_TO_LDM_MAPPING["openclip"]["layers"] + for diffusers_key, ldm_key in openclip_diffusers_ldm_map.items(): + ldm_key = prefix + ldm_key + if ldm_key not in checkpoint: + continue + if ldm_key in keys_to_ignore: + continue + if ldm_key.endswith("text_projection"): + text_model_dict[diffusers_key] = checkpoint[ldm_key].T.contiguous() + else: + text_model_dict[diffusers_key] = checkpoint[ldm_key] + + for key in keys: + if key in keys_to_ignore: + continue + + if not key.startswith(prefix + "transformer."): + continue + + diffusers_key = key.replace(prefix + "transformer.", "") + transformer_diffusers_to_ldm_map = DIFFUSERS_TO_LDM_MAPPING["openclip"]["transformer"] + for new_key, old_key in transformer_diffusers_to_ldm_map.items(): + diffusers_key = ( + diffusers_key.replace(old_key, new_key).replace(".in_proj_weight", "").replace(".in_proj_bias", "") + ) + + if key.endswith(".in_proj_weight"): + weight_value = checkpoint[key] + + text_model_dict[diffusers_key + ".q_proj.weight"] = weight_value[:text_proj_dim, :] + text_model_dict[diffusers_key + ".k_proj.weight"] = weight_value[text_proj_dim : text_proj_dim * 2, :] + text_model_dict[diffusers_key + ".v_proj.weight"] = weight_value[text_proj_dim * 2 :, :] + + elif key.endswith(".in_proj_bias"): + weight_value = checkpoint[key] + text_model_dict[diffusers_key + ".q_proj.bias"] = weight_value[:text_proj_dim] + text_model_dict[diffusers_key + ".k_proj.bias"] = weight_value[text_proj_dim : text_proj_dim * 2] + text_model_dict[diffusers_key + ".v_proj.bias"] = weight_value[text_proj_dim * 2 :] + else: + text_model_dict[diffusers_key] = checkpoint[key] + + if is_accelerate_available(): + from ..models.modeling_utils import load_model_dict_into_meta + + unexpected_keys = load_model_dict_into_meta(text_model, text_model_dict, dtype=torch_dtype) + if text_model._keys_to_ignore_on_load_unexpected is not None: + for pat in text_model._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint were not used when initializing {text_model.__class__.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + + else: + if not (hasattr(text_model, "embeddings") and hasattr(text_model.embeddings.position_ids)): + text_model_dict.pop("text_model.embeddings.position_ids", None) + + text_model.load_state_dict(text_model_dict) + + if torch_dtype is not None: + text_model = text_model.to(torch_dtype) + + return text_model + + +def create_diffusers_unet_model_from_ldm( + pipeline_class_name, + original_config, + checkpoint, + num_in_channels=None, + upcast_attention=None, + extract_ema=False, + image_size=None, + torch_dtype=None, + model_type=None, +): + from ..models import UNet2DConditionModel + + if num_in_channels is None: + if pipeline_class_name in [ + "StableDiffusionInpaintPipeline", + "StableDiffusionControlNetInpaintPipeline", + "StableDiffusionXLInpaintPipeline", + "StableDiffusionXLControlNetInpaintPipeline", + ]: + num_in_channels = 9 + + elif pipeline_class_name == "StableDiffusionUpscalePipeline": + num_in_channels = 7 + + else: + num_in_channels = 4 + + image_size = set_image_size( + pipeline_class_name, original_config, checkpoint, image_size=image_size, model_type=model_type + ) + unet_config = create_unet_diffusers_config(original_config, image_size=image_size) + unet_config["in_channels"] = num_in_channels + if upcast_attention is not None: + unet_config["upcast_attention"] = upcast_attention + + diffusers_format_unet_checkpoint = convert_ldm_unet_checkpoint(checkpoint, unet_config, extract_ema=extract_ema) + ctx = init_empty_weights if is_accelerate_available() else nullcontext + + with ctx(): + unet = UNet2DConditionModel(**unet_config) + + if is_accelerate_available(): + from ..models.modeling_utils import load_model_dict_into_meta + + unexpected_keys = load_model_dict_into_meta(unet, diffusers_format_unet_checkpoint, dtype=torch_dtype) + if unet._keys_to_ignore_on_load_unexpected is not None: + for pat in unet._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint were not used when initializing {unet.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + else: + unet.load_state_dict(diffusers_format_unet_checkpoint) + + if torch_dtype is not None: + unet = unet.to(torch_dtype) + + return {"unet": unet} + + +def create_diffusers_vae_model_from_ldm( + pipeline_class_name, + original_config, + checkpoint, + image_size=None, + scaling_factor=None, + torch_dtype=None, + model_type=None, +): + # import here to avoid circular imports + from ..models import AutoencoderKL + + image_size = set_image_size( + pipeline_class_name, original_config, checkpoint, image_size=image_size, model_type=model_type + ) + model_type = infer_model_type(original_config, checkpoint, model_type) + + if model_type == "Playground": + edm_mean = ( + checkpoint["edm_mean"].to(dtype=torch_dtype).tolist() if torch_dtype else checkpoint["edm_mean"].tolist() + ) + edm_std = ( + checkpoint["edm_std"].to(dtype=torch_dtype).tolist() if torch_dtype else checkpoint["edm_std"].tolist() + ) + else: + edm_mean = None + edm_std = None + + vae_config = create_vae_diffusers_config( + original_config, + image_size=image_size, + scaling_factor=scaling_factor, + latents_mean=edm_mean, + latents_std=edm_std, + ) + diffusers_format_vae_checkpoint = convert_ldm_vae_checkpoint(checkpoint, vae_config) + ctx = init_empty_weights if is_accelerate_available() else nullcontext + + with ctx(): + vae = AutoencoderKL(**vae_config) + + if is_accelerate_available(): + from ..models.modeling_utils import load_model_dict_into_meta + + unexpected_keys = load_model_dict_into_meta(vae, diffusers_format_vae_checkpoint, dtype=torch_dtype) + if vae._keys_to_ignore_on_load_unexpected is not None: + for pat in vae._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint were not used when initializing {vae.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + else: + vae.load_state_dict(diffusers_format_vae_checkpoint) + + if torch_dtype is not None: + vae = vae.to(torch_dtype) + + return {"vae": vae} + + +def create_text_encoders_and_tokenizers_from_ldm( + original_config, + checkpoint, + model_type=None, + local_files_only=False, + torch_dtype=None, +): + model_type = infer_model_type(original_config, checkpoint=checkpoint, model_type=model_type) + + if model_type == "FrozenOpenCLIPEmbedder": + config_name = "stabilityai/stable-diffusion-2" + config_kwargs = {"subfolder": "text_encoder"} + + try: + text_encoder = create_text_encoder_from_open_clip_checkpoint( + config_name, checkpoint, local_files_only=local_files_only, torch_dtype=torch_dtype, **config_kwargs + ) + tokenizer = CLIPTokenizer.from_pretrained( + config_name, subfolder="tokenizer", local_files_only=local_files_only + ) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the text_encoder in the following path: '{config_name}'." + ) + else: + return {"text_encoder": text_encoder, "tokenizer": tokenizer} + + elif model_type == "FrozenCLIPEmbedder": + try: + config_name = "openai/clip-vit-large-patch14" + text_encoder = create_text_encoder_from_ldm_clip_checkpoint( + config_name, + checkpoint, + local_files_only=local_files_only, + torch_dtype=torch_dtype, + ) + tokenizer = CLIPTokenizer.from_pretrained(config_name, local_files_only=local_files_only) + + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the tokenizer in the following path: '{config_name}'." + ) + else: + return {"text_encoder": text_encoder, "tokenizer": tokenizer} + + elif model_type == "SDXL-Refiner": + config_name = "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k" + config_kwargs = {"projection_dim": 1280} + prefix = "conditioner.embedders.0.model." + + try: + tokenizer_2 = CLIPTokenizer.from_pretrained(config_name, pad_token="!", local_files_only=local_files_only) + text_encoder_2 = create_text_encoder_from_open_clip_checkpoint( + config_name, + checkpoint, + prefix=prefix, + has_projection=True, + local_files_only=local_files_only, + torch_dtype=torch_dtype, + **config_kwargs, + ) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the text_encoder_2 and tokenizer_2 in the following path: {config_name} with `pad_token` set to '!'." + ) + + else: + return { + "text_encoder": None, + "tokenizer": None, + "tokenizer_2": tokenizer_2, + "text_encoder_2": text_encoder_2, + } + + elif model_type in ["SDXL", "Playground"]: + try: + config_name = "openai/clip-vit-large-patch14" + tokenizer = CLIPTokenizer.from_pretrained(config_name, local_files_only=local_files_only) + text_encoder = create_text_encoder_from_ldm_clip_checkpoint( + config_name, checkpoint, local_files_only=local_files_only, torch_dtype=torch_dtype + ) + + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the text_encoder and tokenizer in the following path: 'openai/clip-vit-large-patch14'." + ) + + try: + config_name = "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k" + config_kwargs = {"projection_dim": 1280} + prefix = "conditioner.embedders.1.model." + tokenizer_2 = CLIPTokenizer.from_pretrained(config_name, pad_token="!", local_files_only=local_files_only) + text_encoder_2 = create_text_encoder_from_open_clip_checkpoint( + config_name, + checkpoint, + prefix=prefix, + has_projection=True, + local_files_only=local_files_only, + torch_dtype=torch_dtype, + **config_kwargs, + ) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the text_encoder_2 and tokenizer_2 in the following path: {config_name} with `pad_token` set to '!'." + ) + + return { + "tokenizer": tokenizer, + "text_encoder": text_encoder, + "tokenizer_2": tokenizer_2, + "text_encoder_2": text_encoder_2, + } + + return + + +def create_scheduler_from_ldm( + pipeline_class_name, + original_config, + checkpoint, + prediction_type=None, + scheduler_type="ddim", + model_type=None, +): + scheduler_config = get_default_scheduler_config() + model_type = infer_model_type(original_config, checkpoint=checkpoint, model_type=model_type) + + global_step = checkpoint["global_step"] if "global_step" in checkpoint else None + + num_train_timesteps = getattr(original_config["model"]["params"], "timesteps", None) or 1000 + scheduler_config["num_train_timesteps"] = num_train_timesteps + + if ( + "parameterization" in original_config["model"]["params"] + and original_config["model"]["params"]["parameterization"] == "v" + ): + if prediction_type is None: + # NOTE: For stable diffusion 2 base it is recommended to pass `prediction_type=="epsilon"` + # as it relies on a brittle global step parameter here + prediction_type = "epsilon" if global_step == 875000 else "v_prediction" + + else: + prediction_type = prediction_type or "epsilon" + + scheduler_config["prediction_type"] = prediction_type + + if model_type in ["SDXL", "SDXL-Refiner"]: + scheduler_type = "euler" + elif model_type == "Playground": + scheduler_type = "edm_dpm_solver_multistep" + else: + beta_start = original_config["model"]["params"].get("linear_start", 0.02) + beta_end = original_config["model"]["params"].get("linear_end", 0.085) + scheduler_config["beta_start"] = beta_start + scheduler_config["beta_end"] = beta_end + scheduler_config["beta_schedule"] = "scaled_linear" + scheduler_config["clip_sample"] = False + scheduler_config["set_alpha_to_one"] = False + + if scheduler_type == "pndm": + scheduler_config["skip_prk_steps"] = True + scheduler = PNDMScheduler.from_config(scheduler_config) + + elif scheduler_type == "lms": + scheduler = LMSDiscreteScheduler.from_config(scheduler_config) + + elif scheduler_type == "heun": + scheduler = HeunDiscreteScheduler.from_config(scheduler_config) + + elif scheduler_type == "euler": + scheduler = EulerDiscreteScheduler.from_config(scheduler_config) + + elif scheduler_type == "euler-ancestral": + scheduler = EulerAncestralDiscreteScheduler.from_config(scheduler_config) + + elif scheduler_type == "dpm": + scheduler = DPMSolverMultistepScheduler.from_config(scheduler_config) + + elif scheduler_type == "ddim": + scheduler = DDIMScheduler.from_config(scheduler_config) + + elif scheduler_type == "edm_dpm_solver_multistep": + scheduler_config = { + "algorithm_type": "dpmsolver++", + "dynamic_thresholding_ratio": 0.995, + "euler_at_final": False, + "final_sigmas_type": "zero", + "lower_order_final": True, + "num_train_timesteps": 1000, + "prediction_type": "epsilon", + "rho": 7.0, + "sample_max_value": 1.0, + "sigma_data": 0.5, + "sigma_max": 80.0, + "sigma_min": 0.002, + "solver_order": 2, + "solver_type": "midpoint", + "thresholding": False, + } + scheduler = EDMDPMSolverMultistepScheduler(**scheduler_config) + + else: + raise ValueError(f"Scheduler of type {scheduler_type} doesn't exist!") + + if pipeline_class_name == "StableDiffusionUpscalePipeline": + scheduler = DDIMScheduler.from_pretrained("stabilityai/stable-diffusion-x4-upscaler", subfolder="scheduler") + low_res_scheduler = DDPMScheduler.from_pretrained( + "stabilityai/stable-diffusion-x4-upscaler", subfolder="low_res_scheduler" + ) + + return { + "scheduler": scheduler, + "low_res_scheduler": low_res_scheduler, + } + + return {"scheduler": scheduler} diff --git a/diffusers-0.27.0/src/diffusers/loaders/textual_inversion.py b/diffusers-0.27.0/src/diffusers/loaders/textual_inversion.py new file mode 100755 index 0000000..aaaf4b6 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/textual_inversion.py @@ -0,0 +1,562 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, List, Optional, Union + +import safetensors +import torch +from huggingface_hub.utils import validate_hf_hub_args +from torch import nn + +from ..utils import _get_model_file, is_accelerate_available, is_transformers_available, logging + + +if is_transformers_available(): + from transformers import PreTrainedModel, PreTrainedTokenizer + +if is_accelerate_available(): + from accelerate.hooks import AlignDevicesHook, CpuOffload, remove_hook_from_module + +logger = logging.get_logger(__name__) + +TEXT_INVERSION_NAME = "learned_embeds.bin" +TEXT_INVERSION_NAME_SAFE = "learned_embeds.safetensors" + + +@validate_hf_hub_args +def load_textual_inversion_state_dicts(pretrained_model_name_or_paths, **kwargs): + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) + revision = kwargs.pop("revision", None) + subfolder = kwargs.pop("subfolder", None) + weight_name = kwargs.pop("weight_name", None) + use_safetensors = kwargs.pop("use_safetensors", None) + + allow_pickle = False + if use_safetensors is None: + use_safetensors = True + allow_pickle = True + + user_agent = { + "file_type": "text_inversion", + "framework": "pytorch", + } + state_dicts = [] + for pretrained_model_name_or_path in pretrained_model_name_or_paths: + if not isinstance(pretrained_model_name_or_path, (dict, torch.Tensor)): + # 3.1. Load textual inversion file + model_file = None + + # Let's first try to load .safetensors weights + if (use_safetensors and weight_name is None) or ( + weight_name is not None and weight_name.endswith(".safetensors") + ): + try: + model_file = _get_model_file( + pretrained_model_name_or_path, + weights_name=weight_name or TEXT_INVERSION_NAME_SAFE, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + state_dict = safetensors.torch.load_file(model_file, device="cpu") + except Exception as e: + if not allow_pickle: + raise e + + model_file = None + + if model_file is None: + model_file = _get_model_file( + pretrained_model_name_or_path, + weights_name=weight_name or TEXT_INVERSION_NAME, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + state_dict = torch.load(model_file, map_location="cpu") + else: + state_dict = pretrained_model_name_or_path + + state_dicts.append(state_dict) + + return state_dicts + + +class TextualInversionLoaderMixin: + r""" + Load Textual Inversion tokens and embeddings to the tokenizer and text encoder. + """ + + def maybe_convert_prompt(self, prompt: Union[str, List[str]], tokenizer: "PreTrainedTokenizer"): # noqa: F821 + r""" + Processes prompts that include a special token corresponding to a multi-vector textual inversion embedding to + be replaced with multiple special tokens each corresponding to one of the vectors. If the prompt has no textual + inversion token or if the textual inversion token is a single vector, the input prompt is returned. + + Parameters: + prompt (`str` or list of `str`): + The prompt or prompts to guide the image generation. + tokenizer (`PreTrainedTokenizer`): + The tokenizer responsible for encoding the prompt into input tokens. + + Returns: + `str` or list of `str`: The converted prompt + """ + if not isinstance(prompt, List): + prompts = [prompt] + else: + prompts = prompt + + prompts = [self._maybe_convert_prompt(p, tokenizer) for p in prompts] + + if not isinstance(prompt, List): + return prompts[0] + + return prompts + + def _maybe_convert_prompt(self, prompt: str, tokenizer: "PreTrainedTokenizer"): # noqa: F821 + r""" + Maybe convert a prompt into a "multi vector"-compatible prompt. If the prompt includes a token that corresponds + to a multi-vector textual inversion embedding, this function will process the prompt so that the special token + is replaced with multiple special tokens each corresponding to one of the vectors. If the prompt has no textual + inversion token or a textual inversion token that is a single vector, the input prompt is simply returned. + + Parameters: + prompt (`str`): + The prompt to guide the image generation. + tokenizer (`PreTrainedTokenizer`): + The tokenizer responsible for encoding the prompt into input tokens. + + Returns: + `str`: The converted prompt + """ + tokens = tokenizer.tokenize(prompt) + unique_tokens = set(tokens) + for token in unique_tokens: + if token in tokenizer.added_tokens_encoder: + replacement = token + i = 1 + while f"{token}_{i}" in tokenizer.added_tokens_encoder: + replacement += f" {token}_{i}" + i += 1 + + prompt = prompt.replace(token, replacement) + + return prompt + + def _check_text_inv_inputs(self, tokenizer, text_encoder, pretrained_model_name_or_paths, tokens): + if tokenizer is None: + raise ValueError( + f"{self.__class__.__name__} requires `self.tokenizer` or passing a `tokenizer` of type `PreTrainedTokenizer` for calling" + f" `{self.load_textual_inversion.__name__}`" + ) + + if text_encoder is None: + raise ValueError( + f"{self.__class__.__name__} requires `self.text_encoder` or passing a `text_encoder` of type `PreTrainedModel` for calling" + f" `{self.load_textual_inversion.__name__}`" + ) + + if len(pretrained_model_name_or_paths) > 1 and len(pretrained_model_name_or_paths) != len(tokens): + raise ValueError( + f"You have passed a list of models of length {len(pretrained_model_name_or_paths)}, and list of tokens of length {len(tokens)} " + f"Make sure both lists have the same length." + ) + + valid_tokens = [t for t in tokens if t is not None] + if len(set(valid_tokens)) < len(valid_tokens): + raise ValueError(f"You have passed a list of tokens that contains duplicates: {tokens}") + + @staticmethod + def _retrieve_tokens_and_embeddings(tokens, state_dicts, tokenizer): + all_tokens = [] + all_embeddings = [] + for state_dict, token in zip(state_dicts, tokens): + if isinstance(state_dict, torch.Tensor): + if token is None: + raise ValueError( + "You are trying to load a textual inversion embedding that has been saved as a PyTorch tensor. Make sure to pass the name of the corresponding token in this case: `token=...`." + ) + loaded_token = token + embedding = state_dict + elif len(state_dict) == 1: + # diffusers + loaded_token, embedding = next(iter(state_dict.items())) + elif "string_to_param" in state_dict: + # A1111 + loaded_token = state_dict["name"] + embedding = state_dict["string_to_param"]["*"] + else: + raise ValueError( + f"Loaded state dictionary is incorrect: {state_dict}. \n\n" + "Please verify that the loaded state dictionary of the textual embedding either only has a single key or includes the `string_to_param`" + " input key." + ) + + if token is not None and loaded_token != token: + logger.info(f"The loaded token: {loaded_token} is overwritten by the passed token {token}.") + else: + token = loaded_token + + if token in tokenizer.get_vocab(): + raise ValueError( + f"Token {token} already in tokenizer vocabulary. Please choose a different token name or remove {token} and embedding from the tokenizer and text encoder." + ) + + all_tokens.append(token) + all_embeddings.append(embedding) + + return all_tokens, all_embeddings + + @staticmethod + def _extend_tokens_and_embeddings(tokens, embeddings, tokenizer): + all_tokens = [] + all_embeddings = [] + + for embedding, token in zip(embeddings, tokens): + if f"{token}_1" in tokenizer.get_vocab(): + multi_vector_tokens = [token] + i = 1 + while f"{token}_{i}" in tokenizer.added_tokens_encoder: + multi_vector_tokens.append(f"{token}_{i}") + i += 1 + + raise ValueError( + f"Multi-vector Token {multi_vector_tokens} already in tokenizer vocabulary. Please choose a different token name or remove the {multi_vector_tokens} and embedding from the tokenizer and text encoder." + ) + + is_multi_vector = len(embedding.shape) > 1 and embedding.shape[0] > 1 + if is_multi_vector: + all_tokens += [token] + [f"{token}_{i}" for i in range(1, embedding.shape[0])] + all_embeddings += [e for e in embedding] # noqa: C416 + else: + all_tokens += [token] + all_embeddings += [embedding[0]] if len(embedding.shape) > 1 else [embedding] + + return all_tokens, all_embeddings + + @validate_hf_hub_args + def load_textual_inversion( + self, + pretrained_model_name_or_path: Union[str, List[str], Dict[str, torch.Tensor], List[Dict[str, torch.Tensor]]], + token: Optional[Union[str, List[str]]] = None, + tokenizer: Optional["PreTrainedTokenizer"] = None, # noqa: F821 + text_encoder: Optional["PreTrainedModel"] = None, # noqa: F821 + **kwargs, + ): + r""" + Load Textual Inversion embeddings into the text encoder of [`StableDiffusionPipeline`] (both 🤗 Diffusers and + Automatic1111 formats are supported). + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike` or `List[str or os.PathLike]` or `Dict` or `List[Dict]`): + Can be either one of the following or a list of them: + + - A string, the *model id* (for example `sd-concepts-library/low-poly-hd-logos-icons`) of a + pretrained model hosted on the Hub. + - A path to a *directory* (for example `./my_text_inversion_directory/`) containing the textual + inversion weights. + - A path to a *file* (for example `./my_text_inversions.pt`) containing textual inversion weights. + - A [torch state + dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict). + + token (`str` or `List[str]`, *optional*): + Override the token to use for the textual inversion weights. If `pretrained_model_name_or_path` is a + list, then `token` must also be a list of equal length. + text_encoder ([`~transformers.CLIPTextModel`], *optional*): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + If not specified, function will take self.tokenizer. + tokenizer ([`~transformers.CLIPTokenizer`], *optional*): + A `CLIPTokenizer` to tokenize text. If not specified, function will take self.tokenizer. + weight_name (`str`, *optional*): + Name of a custom weight file. This should be used when: + + - The saved textual inversion file is in 🤗 Diffusers format, but was saved under a specific weight + name such as `text_inv.bin`. + - The saved textual inversion file is in the Automatic1111 format. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + subfolder (`str`, *optional*, defaults to `""`): + The subfolder location of a model file within a larger model repository on the Hub or locally. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you're downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + + Example: + + To load a Textual Inversion embedding vector in 🤗 Diffusers format: + + ```py + from diffusers import StableDiffusionPipeline + import torch + + model_id = "runwayml/stable-diffusion-v1-5" + pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda") + + pipe.load_textual_inversion("sd-concepts-library/cat-toy") + + prompt = "A backpack" + + image = pipe(prompt, num_inference_steps=50).images[0] + image.save("cat-backpack.png") + ``` + + To load a Textual Inversion embedding vector in Automatic1111 format, make sure to download the vector first + (for example from [civitAI](https://civitai.com/models/3036?modelVersionId=9857)) and then load the vector + locally: + + ```py + from diffusers import StableDiffusionPipeline + import torch + + model_id = "runwayml/stable-diffusion-v1-5" + pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda") + + pipe.load_textual_inversion("./charturnerv2.pt", token="charturnerv2") + + prompt = "charturnerv2, multiple views of the same character in the same outfit, a character turnaround of a woman wearing a black jacket and red shirt, best quality, intricate details." + + image = pipe(prompt, num_inference_steps=50).images[0] + image.save("character.png") + ``` + + """ + # 1. Set correct tokenizer and text encoder + tokenizer = tokenizer or getattr(self, "tokenizer", None) + text_encoder = text_encoder or getattr(self, "text_encoder", None) + + # 2. Normalize inputs + pretrained_model_name_or_paths = ( + [pretrained_model_name_or_path] + if not isinstance(pretrained_model_name_or_path, list) + else pretrained_model_name_or_path + ) + tokens = [token] if not isinstance(token, list) else token + if tokens[0] is None: + tokens = tokens * len(pretrained_model_name_or_paths) + + # 3. Check inputs + self._check_text_inv_inputs(tokenizer, text_encoder, pretrained_model_name_or_paths, tokens) + + # 4. Load state dicts of textual embeddings + state_dicts = load_textual_inversion_state_dicts(pretrained_model_name_or_paths, **kwargs) + + # 4.1 Handle the special case when state_dict is a tensor that contains n embeddings for n tokens + if len(tokens) > 1 and len(state_dicts) == 1: + if isinstance(state_dicts[0], torch.Tensor): + state_dicts = list(state_dicts[0]) + if len(tokens) != len(state_dicts): + raise ValueError( + f"You have passed a state_dict contains {len(state_dicts)} embeddings, and list of tokens of length {len(tokens)} " + f"Make sure both have the same length." + ) + + # 4. Retrieve tokens and embeddings + tokens, embeddings = self._retrieve_tokens_and_embeddings(tokens, state_dicts, tokenizer) + + # 5. Extend tokens and embeddings for multi vector + tokens, embeddings = self._extend_tokens_and_embeddings(tokens, embeddings, tokenizer) + + # 6. Make sure all embeddings have the correct size + expected_emb_dim = text_encoder.get_input_embeddings().weight.shape[-1] + if any(expected_emb_dim != emb.shape[-1] for emb in embeddings): + raise ValueError( + "Loaded embeddings are of incorrect shape. Expected each textual inversion embedding " + "to be of shape {input_embeddings.shape[-1]}, but are {embeddings.shape[-1]} " + ) + + # 7. Now we can be sure that loading the embedding matrix works + # < Unsafe code: + + # 7.1 Offload all hooks in case the pipeline was cpu offloaded before make sure, we offload and onload again + is_model_cpu_offload = False + is_sequential_cpu_offload = False + for _, component in self.components.items(): + if isinstance(component, nn.Module): + if hasattr(component, "_hf_hook"): + is_model_cpu_offload = isinstance(getattr(component, "_hf_hook"), CpuOffload) + is_sequential_cpu_offload = isinstance(getattr(component, "_hf_hook"), AlignDevicesHook) + logger.info( + "Accelerate hooks detected. Since you have called `load_textual_inversion()`, the previous hooks will be first removed. Then the textual inversion parameters will be loaded and the hooks will be applied again." + ) + remove_hook_from_module(component, recurse=is_sequential_cpu_offload) + + # 7.2 save expected device and dtype + device = text_encoder.device + dtype = text_encoder.dtype + + # 7.3 Increase token embedding matrix + text_encoder.resize_token_embeddings(len(tokenizer) + len(tokens)) + input_embeddings = text_encoder.get_input_embeddings().weight + + # 7.4 Load token and embedding + for token, embedding in zip(tokens, embeddings): + # add tokens and get ids + tokenizer.add_tokens(token) + token_id = tokenizer.convert_tokens_to_ids(token) + input_embeddings.data[token_id] = embedding + logger.info(f"Loaded textual inversion embedding for {token}.") + + input_embeddings.to(dtype=dtype, device=device) + + # 7.5 Offload the model again + if is_model_cpu_offload: + self.enable_model_cpu_offload() + elif is_sequential_cpu_offload: + self.enable_sequential_cpu_offload() + + # / Unsafe Code > + + def unload_textual_inversion( + self, + tokens: Optional[Union[str, List[str]]] = None, + tokenizer: Optional["PreTrainedTokenizer"] = None, + text_encoder: Optional["PreTrainedModel"] = None, + ): + r""" + Unload Textual Inversion embeddings from the text encoder of [`StableDiffusionPipeline`] + + Example: + ```py + from diffusers import AutoPipelineForText2Image + import torch + + pipeline = AutoPipelineForText2Image.from_pretrained("runwayml/stable-diffusion-v1-5") + + # Example 1 + pipeline.load_textual_inversion("sd-concepts-library/gta5-artwork") + pipeline.load_textual_inversion("sd-concepts-library/moeb-style") + + # Remove all token embeddings + pipeline.unload_textual_inversion() + + # Example 2 + pipeline.load_textual_inversion("sd-concepts-library/moeb-style") + pipeline.load_textual_inversion("sd-concepts-library/gta5-artwork") + + # Remove just one token + pipeline.unload_textual_inversion("") + + # Example 3: unload from SDXL + pipeline = AutoPipelineForText2Image.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") + embedding_path = hf_hub_download(repo_id="linoyts/web_y2k", filename="web_y2k_emb.safetensors", repo_type="model") + + # load embeddings to the text encoders + state_dict = load_file(embedding_path) + + # load embeddings of text_encoder 1 (CLIP ViT-L/14) + pipeline.load_textual_inversion(state_dict["clip_l"], token=["", ""], text_encoder=pipeline.text_encoder, tokenizer=pipeline.tokenizer) + # load embeddings of text_encoder 2 (CLIP ViT-G/14) + pipeline.load_textual_inversion(state_dict["clip_g"], token=["", ""], text_encoder=pipeline.text_encoder_2, tokenizer=pipeline.tokenizer_2) + + # Unload explicitly from both text encoders abd tokenizers + pipeline.unload_textual_inversion(tokens=["", ""], text_encoder=pipeline.text_encoder, tokenizer=pipeline.tokenizer) + pipeline.unload_textual_inversion(tokens=["", ""], text_encoder=pipeline.text_encoder_2, tokenizer=pipeline.tokenizer_2) + + ``` + """ + + tokenizer = tokenizer or getattr(self, "tokenizer", None) + text_encoder = text_encoder or getattr(self, "text_encoder", None) + + # Get textual inversion tokens and ids + token_ids = [] + last_special_token_id = None + + if tokens: + if isinstance(tokens, str): + tokens = [tokens] + for added_token_id, added_token in tokenizer.added_tokens_decoder.items(): + if not added_token.special: + if added_token.content in tokens: + token_ids.append(added_token_id) + else: + last_special_token_id = added_token_id + if len(token_ids) == 0: + raise ValueError("No tokens to remove found") + else: + tokens = [] + for added_token_id, added_token in tokenizer.added_tokens_decoder.items(): + if not added_token.special: + token_ids.append(added_token_id) + tokens.append(added_token.content) + else: + last_special_token_id = added_token_id + + # Delete from tokenizer + for token_id, token_to_remove in zip(token_ids, tokens): + del tokenizer._added_tokens_decoder[token_id] + del tokenizer._added_tokens_encoder[token_to_remove] + + # Make all token ids sequential in tokenizer + key_id = 1 + for token_id in tokenizer.added_tokens_decoder: + if token_id > last_special_token_id and token_id > last_special_token_id + key_id: + token = tokenizer._added_tokens_decoder[token_id] + tokenizer._added_tokens_decoder[last_special_token_id + key_id] = token + del tokenizer._added_tokens_decoder[token_id] + tokenizer._added_tokens_encoder[token.content] = last_special_token_id + key_id + key_id += 1 + tokenizer._update_trie() + + # Delete from text encoder + text_embedding_dim = text_encoder.get_input_embeddings().embedding_dim + temp_text_embedding_weights = text_encoder.get_input_embeddings().weight + text_embedding_weights = temp_text_embedding_weights[: last_special_token_id + 1] + to_append = [] + for i in range(last_special_token_id + 1, temp_text_embedding_weights.shape[0]): + if i not in token_ids: + to_append.append(temp_text_embedding_weights[i].unsqueeze(0)) + if len(to_append) > 0: + to_append = torch.cat(to_append, dim=0) + text_embedding_weights = torch.cat([text_embedding_weights, to_append], dim=0) + text_embeddings_filtered = nn.Embedding(text_embedding_weights.shape[0], text_embedding_dim) + text_embeddings_filtered.weight.data = text_embedding_weights + text_encoder.set_input_embeddings(text_embeddings_filtered) diff --git a/diffusers-0.27.0/src/diffusers/loaders/unet.py b/diffusers-0.27.0/src/diffusers/loaders/unet.py new file mode 100755 index 0000000..0a9544d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/unet.py @@ -0,0 +1,1003 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import os +from collections import defaultdict +from contextlib import nullcontext +from functools import partial +from pathlib import Path +from typing import Callable, Dict, List, Optional, Union + +import safetensors +import torch +import torch.nn.functional as F +from huggingface_hub.utils import validate_hf_hub_args +from torch import nn + +from ..models.embeddings import ( + ImageProjection, + IPAdapterFullImageProjection, + IPAdapterPlusImageProjection, + MultiIPAdapterImageProjection, +) +from ..models.modeling_utils import _LOW_CPU_MEM_USAGE_DEFAULT, load_model_dict_into_meta +from ..utils import ( + USE_PEFT_BACKEND, + _get_model_file, + delete_adapter_layers, + is_accelerate_available, + is_torch_version, + logging, + set_adapter_layers, + set_weights_and_activate_adapters, +) +from .single_file_utils import ( + convert_stable_cascade_unet_single_file_to_diffusers, + infer_stable_cascade_single_file_config, + load_single_file_model_checkpoint, +) +from .utils import AttnProcsLayers + + +if is_accelerate_available(): + from accelerate import init_empty_weights + from accelerate.hooks import AlignDevicesHook, CpuOffload, remove_hook_from_module + +logger = logging.get_logger(__name__) + + +TEXT_ENCODER_NAME = "text_encoder" +UNET_NAME = "unet" + +LORA_WEIGHT_NAME = "pytorch_lora_weights.bin" +LORA_WEIGHT_NAME_SAFE = "pytorch_lora_weights.safetensors" + +CUSTOM_DIFFUSION_WEIGHT_NAME = "pytorch_custom_diffusion_weights.bin" +CUSTOM_DIFFUSION_WEIGHT_NAME_SAFE = "pytorch_custom_diffusion_weights.safetensors" + + +class UNet2DConditionLoadersMixin: + """ + Load LoRA layers into a [`UNet2DCondtionModel`]. + """ + + text_encoder_name = TEXT_ENCODER_NAME + unet_name = UNET_NAME + + @validate_hf_hub_args + def load_attn_procs(self, pretrained_model_name_or_path_or_dict: Union[str, Dict[str, torch.Tensor]], **kwargs): + r""" + Load pretrained attention processor layers into [`UNet2DConditionModel`]. Attention processor layers have to be + defined in + [`attention_processor.py`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py) + and be a `torch.nn.Module` class. + + Parameters: + pretrained_model_name_or_path_or_dict (`str` or `os.PathLike` or `dict`): + Can be either: + + - A string, the model id (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on + the Hub. + - A path to a directory (for example `./my_model_directory`) containing the model weights saved + with [`ModelMixin.save_pretrained`]. + - A [torch state + dict](https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict). + + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + subfolder (`str`, *optional*, defaults to `""`): + The subfolder location of a model file within a larger model repository on the Hub or locally. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + + Example: + + ```py + from diffusers import AutoPipelineForText2Image + import torch + + pipeline = AutoPipelineForText2Image.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ).to("cuda") + pipeline.unet.load_attn_procs( + "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic" + ) + ``` + """ + from ..models.attention_processor import CustomDiffusionAttnProcessor + from ..models.lora import LoRACompatibleConv, LoRACompatibleLinear, LoRAConv2dLayer, LoRALinearLayer + + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) + revision = kwargs.pop("revision", None) + subfolder = kwargs.pop("subfolder", None) + weight_name = kwargs.pop("weight_name", None) + use_safetensors = kwargs.pop("use_safetensors", None) + low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", _LOW_CPU_MEM_USAGE_DEFAULT) + # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. + # See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning + network_alphas = kwargs.pop("network_alphas", None) + + _pipeline = kwargs.pop("_pipeline", None) + + is_network_alphas_none = network_alphas is None + + allow_pickle = False + + if use_safetensors is None: + use_safetensors = True + allow_pickle = True + + user_agent = { + "file_type": "attn_procs_weights", + "framework": "pytorch", + } + + model_file = None + if not isinstance(pretrained_model_name_or_path_or_dict, dict): + # Let's first try to load .safetensors weights + if (use_safetensors and weight_name is None) or ( + weight_name is not None and weight_name.endswith(".safetensors") + ): + try: + model_file = _get_model_file( + pretrained_model_name_or_path_or_dict, + weights_name=weight_name or LORA_WEIGHT_NAME_SAFE, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + state_dict = safetensors.torch.load_file(model_file, device="cpu") + except IOError as e: + if not allow_pickle: + raise e + # try loading non-safetensors weights + pass + if model_file is None: + model_file = _get_model_file( + pretrained_model_name_or_path_or_dict, + weights_name=weight_name or LORA_WEIGHT_NAME, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + ) + state_dict = torch.load(model_file, map_location="cpu") + else: + state_dict = pretrained_model_name_or_path_or_dict + + # fill attn processors + lora_layers_list = [] + + is_lora = all(("lora" in k or k.endswith(".alpha")) for k in state_dict.keys()) and not USE_PEFT_BACKEND + is_custom_diffusion = any("custom_diffusion" in k for k in state_dict.keys()) + + if is_lora: + # correct keys + state_dict, network_alphas = self.convert_state_dict_legacy_attn_format(state_dict, network_alphas) + + if network_alphas is not None: + network_alphas_keys = list(network_alphas.keys()) + used_network_alphas_keys = set() + + lora_grouped_dict = defaultdict(dict) + mapped_network_alphas = {} + + all_keys = list(state_dict.keys()) + for key in all_keys: + value = state_dict.pop(key) + attn_processor_key, sub_key = ".".join(key.split(".")[:-3]), ".".join(key.split(".")[-3:]) + lora_grouped_dict[attn_processor_key][sub_key] = value + + # Create another `mapped_network_alphas` dictionary so that we can properly map them. + if network_alphas is not None: + for k in network_alphas_keys: + if k.replace(".alpha", "") in key: + mapped_network_alphas.update({attn_processor_key: network_alphas.get(k)}) + used_network_alphas_keys.add(k) + + if not is_network_alphas_none: + if len(set(network_alphas_keys) - used_network_alphas_keys) > 0: + raise ValueError( + f"The `network_alphas` has to be empty at this point but has the following keys \n\n {', '.join(network_alphas.keys())}" + ) + + if len(state_dict) > 0: + raise ValueError( + f"The `state_dict` has to be empty at this point but has the following keys \n\n {', '.join(state_dict.keys())}" + ) + + for key, value_dict in lora_grouped_dict.items(): + attn_processor = self + for sub_key in key.split("."): + attn_processor = getattr(attn_processor, sub_key) + + # Process non-attention layers, which don't have to_{k,v,q,out_proj}_lora layers + # or add_{k,v,q,out_proj}_proj_lora layers. + rank = value_dict["lora.down.weight"].shape[0] + + if isinstance(attn_processor, LoRACompatibleConv): + in_features = attn_processor.in_channels + out_features = attn_processor.out_channels + kernel_size = attn_processor.kernel_size + + ctx = init_empty_weights if low_cpu_mem_usage else nullcontext + with ctx(): + lora = LoRAConv2dLayer( + in_features=in_features, + out_features=out_features, + rank=rank, + kernel_size=kernel_size, + stride=attn_processor.stride, + padding=attn_processor.padding, + network_alpha=mapped_network_alphas.get(key), + ) + elif isinstance(attn_processor, LoRACompatibleLinear): + ctx = init_empty_weights if low_cpu_mem_usage else nullcontext + with ctx(): + lora = LoRALinearLayer( + attn_processor.in_features, + attn_processor.out_features, + rank, + mapped_network_alphas.get(key), + ) + else: + raise ValueError(f"Module {key} is not a LoRACompatibleConv or LoRACompatibleLinear module.") + + value_dict = {k.replace("lora.", ""): v for k, v in value_dict.items()} + lora_layers_list.append((attn_processor, lora)) + + if low_cpu_mem_usage: + device = next(iter(value_dict.values())).device + dtype = next(iter(value_dict.values())).dtype + load_model_dict_into_meta(lora, value_dict, device=device, dtype=dtype) + else: + lora.load_state_dict(value_dict) + + elif is_custom_diffusion: + attn_processors = {} + custom_diffusion_grouped_dict = defaultdict(dict) + for key, value in state_dict.items(): + if len(value) == 0: + custom_diffusion_grouped_dict[key] = {} + else: + if "to_out" in key: + attn_processor_key, sub_key = ".".join(key.split(".")[:-3]), ".".join(key.split(".")[-3:]) + else: + attn_processor_key, sub_key = ".".join(key.split(".")[:-2]), ".".join(key.split(".")[-2:]) + custom_diffusion_grouped_dict[attn_processor_key][sub_key] = value + + for key, value_dict in custom_diffusion_grouped_dict.items(): + if len(value_dict) == 0: + attn_processors[key] = CustomDiffusionAttnProcessor( + train_kv=False, train_q_out=False, hidden_size=None, cross_attention_dim=None + ) + else: + cross_attention_dim = value_dict["to_k_custom_diffusion.weight"].shape[1] + hidden_size = value_dict["to_k_custom_diffusion.weight"].shape[0] + train_q_out = True if "to_q_custom_diffusion.weight" in value_dict else False + attn_processors[key] = CustomDiffusionAttnProcessor( + train_kv=True, + train_q_out=train_q_out, + hidden_size=hidden_size, + cross_attention_dim=cross_attention_dim, + ) + attn_processors[key].load_state_dict(value_dict) + elif USE_PEFT_BACKEND: + # In that case we have nothing to do as loading the adapter weights is already handled above by `set_peft_model_state_dict` + # on the Unet + pass + else: + raise ValueError( + f"{model_file} does not seem to be in the correct format expected by LoRA or Custom Diffusion training." + ) + + # + + def convert_state_dict_legacy_attn_format(self, state_dict, network_alphas): + is_new_lora_format = all( + key.startswith(self.unet_name) or key.startswith(self.text_encoder_name) for key in state_dict.keys() + ) + if is_new_lora_format: + # Strip the `"unet"` prefix. + is_text_encoder_present = any(key.startswith(self.text_encoder_name) for key in state_dict.keys()) + if is_text_encoder_present: + warn_message = "The state_dict contains LoRA params corresponding to the text encoder which are not being used here. To use both UNet and text encoder related LoRA params, use [`pipe.load_lora_weights()`](https://huggingface.co/docs/diffusers/main/en/api/loaders#diffusers.loaders.LoraLoaderMixin.load_lora_weights)." + logger.warning(warn_message) + unet_keys = [k for k in state_dict.keys() if k.startswith(self.unet_name)] + state_dict = {k.replace(f"{self.unet_name}.", ""): v for k, v in state_dict.items() if k in unet_keys} + + # change processor format to 'pure' LoRACompatibleLinear format + if any("processor" in k.split(".") for k in state_dict.keys()): + + def format_to_lora_compatible(key): + if "processor" not in key.split("."): + return key + return key.replace(".processor", "").replace("to_out_lora", "to_out.0.lora").replace("_lora", ".lora") + + state_dict = {format_to_lora_compatible(k): v for k, v in state_dict.items()} + + if network_alphas is not None: + network_alphas = {format_to_lora_compatible(k): v for k, v in network_alphas.items()} + return state_dict, network_alphas + + def save_attn_procs( + self, + save_directory: Union[str, os.PathLike], + is_main_process: bool = True, + weight_name: str = None, + save_function: Callable = None, + safe_serialization: bool = True, + **kwargs, + ): + r""" + Save attention processor layers to a directory so that it can be reloaded with the + [`~loaders.UNet2DConditionLoadersMixin.load_attn_procs`] method. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to save an attention processor to (will be created if it doesn't exist). + is_main_process (`bool`, *optional*, defaults to `True`): + Whether the process calling this is the main process or not. Useful during distributed training and you + need to call this function on all processes. In this case, set `is_main_process=True` only on the main + process to avoid race conditions. + save_function (`Callable`): + The function to use to save the state dictionary. Useful during distributed training when you need to + replace `torch.save` with another method. Can be configured with the environment variable + `DIFFUSERS_SAVE_MODE`. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether to save the model using `safetensors` or with `pickle`. + + Example: + + ```py + import torch + from diffusers import DiffusionPipeline + + pipeline = DiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", + torch_dtype=torch.float16, + ).to("cuda") + pipeline.unet.load_attn_procs("path-to-save-model", weight_name="pytorch_custom_diffusion_weights.bin") + pipeline.unet.save_attn_procs("path-to-save-model", weight_name="pytorch_custom_diffusion_weights.bin") + ``` + """ + from ..models.attention_processor import ( + CustomDiffusionAttnProcessor, + CustomDiffusionAttnProcessor2_0, + CustomDiffusionXFormersAttnProcessor, + ) + + if os.path.isfile(save_directory): + logger.error(f"Provided path ({save_directory}) should be a directory, not a file") + return + + if save_function is None: + if safe_serialization: + + def save_function(weights, filename): + return safetensors.torch.save_file(weights, filename, metadata={"format": "pt"}) + + else: + save_function = torch.save + + os.makedirs(save_directory, exist_ok=True) + + is_custom_diffusion = any( + isinstance( + x, + (CustomDiffusionAttnProcessor, CustomDiffusionAttnProcessor2_0, CustomDiffusionXFormersAttnProcessor), + ) + for (_, x) in self.attn_processors.items() + ) + if is_custom_diffusion: + model_to_save = AttnProcsLayers( + { + y: x + for (y, x) in self.attn_processors.items() + if isinstance( + x, + ( + CustomDiffusionAttnProcessor, + CustomDiffusionAttnProcessor2_0, + CustomDiffusionXFormersAttnProcessor, + ), + ) + } + ) + state_dict = model_to_save.state_dict() + for name, attn in self.attn_processors.items(): + if len(attn.state_dict()) == 0: + state_dict[name] = {} + else: + model_to_save = AttnProcsLayers(self.attn_processors) + state_dict = model_to_save.state_dict() + + if weight_name is None: + if safe_serialization: + weight_name = CUSTOM_DIFFUSION_WEIGHT_NAME_SAFE if is_custom_diffusion else LORA_WEIGHT_NAME_SAFE + else: + weight_name = CUSTOM_DIFFUSION_WEIGHT_NAME if is_custom_diffusion else LORA_WEIGHT_NAME + + # Save the model + save_path = Path(save_directory, weight_name).as_posix() + save_function(state_dict, save_path) + logger.info(f"Model weights saved in {save_path}") + + def fuse_lora(self, lora_scale=1.0, safe_fusing=False, adapter_names=None): + self.lora_scale = lora_scale + self._safe_fusing = safe_fusing + self.apply(partial(self._fuse_lora_apply, adapter_names=adapter_names)) + + def _fuse_lora_apply(self, module, adapter_names=None): + if not USE_PEFT_BACKEND: + if hasattr(module, "_fuse_lora"): + module._fuse_lora(self.lora_scale, self._safe_fusing) + + if adapter_names is not None: + raise ValueError( + "The `adapter_names` argument is not supported in your environment. Please switch" + " to PEFT backend to use this argument by installing latest PEFT and transformers." + " `pip install -U peft transformers`" + ) + else: + from peft.tuners.tuners_utils import BaseTunerLayer + + merge_kwargs = {"safe_merge": self._safe_fusing} + + if isinstance(module, BaseTunerLayer): + if self.lora_scale != 1.0: + module.scale_layer(self.lora_scale) + + # For BC with prevous PEFT versions, we need to check the signature + # of the `merge` method to see if it supports the `adapter_names` argument. + supported_merge_kwargs = list(inspect.signature(module.merge).parameters) + if "adapter_names" in supported_merge_kwargs: + merge_kwargs["adapter_names"] = adapter_names + elif "adapter_names" not in supported_merge_kwargs and adapter_names is not None: + raise ValueError( + "The `adapter_names` argument is not supported with your PEFT version. Please upgrade" + " to the latest version of PEFT. `pip install -U peft`" + ) + + module.merge(**merge_kwargs) + + def unfuse_lora(self): + self.apply(self._unfuse_lora_apply) + + def _unfuse_lora_apply(self, module): + if not USE_PEFT_BACKEND: + if hasattr(module, "_unfuse_lora"): + module._unfuse_lora() + else: + from peft.tuners.tuners_utils import BaseTunerLayer + + if isinstance(module, BaseTunerLayer): + module.unmerge() + + def set_adapters( + self, + adapter_names: Union[List[str], str], + weights: Optional[Union[List[float], float]] = None, + ): + """ + Set the currently active adapters for use in the UNet. + + Args: + adapter_names (`List[str]` or `str`): + The names of the adapters to use. + adapter_weights (`Union[List[float], float]`, *optional*): + The adapter(s) weights to use with the UNet. If `None`, the weights are set to `1.0` for all the + adapters. + + Example: + + ```py + from diffusers import AutoPipelineForText2Image + import torch + + pipeline = AutoPipelineForText2Image.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ).to("cuda") + pipeline.load_lora_weights( + "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic" + ) + pipeline.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors", adapter_name="pixel") + pipeline.set_adapters(["cinematic", "pixel"], adapter_weights=[0.5, 0.5]) + ``` + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for `set_adapters()`.") + + adapter_names = [adapter_names] if isinstance(adapter_names, str) else adapter_names + + if weights is None: + weights = [1.0] * len(adapter_names) + elif isinstance(weights, float): + weights = [weights] * len(adapter_names) + + if len(adapter_names) != len(weights): + raise ValueError( + f"Length of adapter names {len(adapter_names)} is not equal to the length of their weights {len(weights)}." + ) + + set_weights_and_activate_adapters(self, adapter_names, weights) + + def disable_lora(self): + """ + Disable the UNet's active LoRA layers. + + Example: + + ```py + from diffusers import AutoPipelineForText2Image + import torch + + pipeline = AutoPipelineForText2Image.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ).to("cuda") + pipeline.load_lora_weights( + "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic" + ) + pipeline.disable_lora() + ``` + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + set_adapter_layers(self, enabled=False) + + def enable_lora(self): + """ + Enable the UNet's active LoRA layers. + + Example: + + ```py + from diffusers import AutoPipelineForText2Image + import torch + + pipeline = AutoPipelineForText2Image.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ).to("cuda") + pipeline.load_lora_weights( + "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_name="cinematic" + ) + pipeline.enable_lora() + ``` + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + set_adapter_layers(self, enabled=True) + + def delete_adapters(self, adapter_names: Union[List[str], str]): + """ + Delete an adapter's LoRA layers from the UNet. + + Args: + adapter_names (`Union[List[str], str]`): + The names (single string or list of strings) of the adapter to delete. + + Example: + + ```py + from diffusers import AutoPipelineForText2Image + import torch + + pipeline = AutoPipelineForText2Image.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ).to("cuda") + pipeline.load_lora_weights( + "jbilcke-hf/sdxl-cinematic-1", weight_name="pytorch_lora_weights.safetensors", adapter_names="cinematic" + ) + pipeline.delete_adapters("cinematic") + ``` + """ + if not USE_PEFT_BACKEND: + raise ValueError("PEFT backend is required for this method.") + + if isinstance(adapter_names, str): + adapter_names = [adapter_names] + + for adapter_name in adapter_names: + delete_adapter_layers(self, adapter_name) + + # Pop also the corresponding adapter from the config + if hasattr(self, "peft_config"): + self.peft_config.pop(adapter_name, None) + + def _convert_ip_adapter_image_proj_to_diffusers(self, state_dict, low_cpu_mem_usage=False): + if low_cpu_mem_usage: + if is_accelerate_available(): + from accelerate import init_empty_weights + + else: + low_cpu_mem_usage = False + logger.warning( + "Cannot initialize model with low cpu memory usage because `accelerate` was not found in the" + " environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install" + " `accelerate` for faster and less memory-intense model loading. You can do so with: \n```\npip" + " install accelerate\n```\n." + ) + + if low_cpu_mem_usage is True and not is_torch_version(">=", "1.9.0"): + raise NotImplementedError( + "Low memory initialization requires torch >= 1.9.0. Please either update your PyTorch version or set" + " `low_cpu_mem_usage=False`." + ) + + updated_state_dict = {} + image_projection = None + init_context = init_empty_weights if low_cpu_mem_usage else nullcontext + + if "proj.weight" in state_dict: + # IP-Adapter + num_image_text_embeds = 4 + clip_embeddings_dim = state_dict["proj.weight"].shape[-1] + cross_attention_dim = state_dict["proj.weight"].shape[0] // 4 + + with init_context(): + image_projection = ImageProjection( + cross_attention_dim=cross_attention_dim, + image_embed_dim=clip_embeddings_dim, + num_image_text_embeds=num_image_text_embeds, + ) + + for key, value in state_dict.items(): + diffusers_name = key.replace("proj", "image_embeds") + updated_state_dict[diffusers_name] = value + + elif "proj.3.weight" in state_dict: + # IP-Adapter Full + clip_embeddings_dim = state_dict["proj.0.weight"].shape[0] + cross_attention_dim = state_dict["proj.3.weight"].shape[0] + + with init_context(): + image_projection = IPAdapterFullImageProjection( + cross_attention_dim=cross_attention_dim, image_embed_dim=clip_embeddings_dim + ) + + for key, value in state_dict.items(): + diffusers_name = key.replace("proj.0", "ff.net.0.proj") + diffusers_name = diffusers_name.replace("proj.2", "ff.net.2") + diffusers_name = diffusers_name.replace("proj.3", "norm") + updated_state_dict[diffusers_name] = value + + else: + # IP-Adapter Plus + num_image_text_embeds = state_dict["latents"].shape[1] + embed_dims = state_dict["proj_in.weight"].shape[1] + output_dims = state_dict["proj_out.weight"].shape[0] + hidden_dims = state_dict["latents"].shape[2] + heads = state_dict["layers.0.0.to_q.weight"].shape[0] // 64 + + with init_context(): + image_projection = IPAdapterPlusImageProjection( + embed_dims=embed_dims, + output_dims=output_dims, + hidden_dims=hidden_dims, + heads=heads, + num_queries=num_image_text_embeds, + ) + + for key, value in state_dict.items(): + diffusers_name = key.replace("0.to", "2.to") + diffusers_name = diffusers_name.replace("1.0.weight", "3.0.weight") + diffusers_name = diffusers_name.replace("1.0.bias", "3.0.bias") + diffusers_name = diffusers_name.replace("1.1.weight", "3.1.net.0.proj.weight") + diffusers_name = diffusers_name.replace("1.3.weight", "3.1.net.2.weight") + + if "norm1" in diffusers_name: + updated_state_dict[diffusers_name.replace("0.norm1", "0")] = value + elif "norm2" in diffusers_name: + updated_state_dict[diffusers_name.replace("0.norm2", "1")] = value + elif "to_kv" in diffusers_name: + v_chunk = value.chunk(2, dim=0) + updated_state_dict[diffusers_name.replace("to_kv", "to_k")] = v_chunk[0] + updated_state_dict[diffusers_name.replace("to_kv", "to_v")] = v_chunk[1] + elif "to_out" in diffusers_name: + updated_state_dict[diffusers_name.replace("to_out", "to_out.0")] = value + else: + updated_state_dict[diffusers_name] = value + + if not low_cpu_mem_usage: + image_projection.load_state_dict(updated_state_dict) + else: + load_model_dict_into_meta(image_projection, updated_state_dict, device=self.device, dtype=self.dtype) + + return image_projection + + def _convert_ip_adapter_attn_to_diffusers(self, state_dicts, low_cpu_mem_usage=False): + from ..models.attention_processor import ( + AttnProcessor, + AttnProcessor2_0, + IPAdapterAttnProcessor, + IPAdapterAttnProcessor2_0, + ) + + if low_cpu_mem_usage: + if is_accelerate_available(): + from accelerate import init_empty_weights + + else: + low_cpu_mem_usage = False + logger.warning( + "Cannot initialize model with low cpu memory usage because `accelerate` was not found in the" + " environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install" + " `accelerate` for faster and less memory-intense model loading. You can do so with: \n```\npip" + " install accelerate\n```\n." + ) + + if low_cpu_mem_usage is True and not is_torch_version(">=", "1.9.0"): + raise NotImplementedError( + "Low memory initialization requires torch >= 1.9.0. Please either update your PyTorch version or set" + " `low_cpu_mem_usage=False`." + ) + + # set ip-adapter cross-attention processors & load state_dict + attn_procs = {} + key_id = 1 + init_context = init_empty_weights if low_cpu_mem_usage else nullcontext + for name in self.attn_processors.keys(): + cross_attention_dim = None if name.endswith("attn1.processor") else self.config.cross_attention_dim + if name.startswith("mid_block"): + hidden_size = self.config.block_out_channels[-1] + elif name.startswith("up_blocks"): + block_id = int(name[len("up_blocks.")]) + hidden_size = list(reversed(self.config.block_out_channels))[block_id] + elif name.startswith("down_blocks"): + block_id = int(name[len("down_blocks.")]) + hidden_size = self.config.block_out_channels[block_id] + + if cross_attention_dim is None or "motion_modules" in name: + attn_processor_class = ( + AttnProcessor2_0 if hasattr(F, "scaled_dot_product_attention") else AttnProcessor + ) + attn_procs[name] = attn_processor_class() + else: + attn_processor_class = ( + IPAdapterAttnProcessor2_0 if hasattr(F, "scaled_dot_product_attention") else IPAdapterAttnProcessor + ) + num_image_text_embeds = [] + for state_dict in state_dicts: + if "proj.weight" in state_dict["image_proj"]: + # IP-Adapter + num_image_text_embeds += [4] + elif "proj.3.weight" in state_dict["image_proj"]: + # IP-Adapter Full Face + num_image_text_embeds += [257] # 256 CLIP tokens + 1 CLS token + else: + # IP-Adapter Plus + num_image_text_embeds += [state_dict["image_proj"]["latents"].shape[1]] + + with init_context(): + attn_procs[name] = attn_processor_class( + hidden_size=hidden_size, + cross_attention_dim=cross_attention_dim, + scale=1.0, + num_tokens=num_image_text_embeds, + ) + + value_dict = {} + for i, state_dict in enumerate(state_dicts): + value_dict.update({f"to_k_ip.{i}.weight": state_dict["ip_adapter"][f"{key_id}.to_k_ip.weight"]}) + value_dict.update({f"to_v_ip.{i}.weight": state_dict["ip_adapter"][f"{key_id}.to_v_ip.weight"]}) + + if not low_cpu_mem_usage: + attn_procs[name].load_state_dict(value_dict) + else: + device = next(iter(value_dict.values())).device + dtype = next(iter(value_dict.values())).dtype + load_model_dict_into_meta(attn_procs[name], value_dict, device=device, dtype=dtype) + + key_id += 2 + + return attn_procs + + def _load_ip_adapter_weights(self, state_dicts, low_cpu_mem_usage=False): + if not isinstance(state_dicts, list): + state_dicts = [state_dicts] + # Set encoder_hid_proj after loading ip_adapter weights, + # because `IPAdapterPlusImageProjection` also has `attn_processors`. + self.encoder_hid_proj = None + + attn_procs = self._convert_ip_adapter_attn_to_diffusers(state_dicts, low_cpu_mem_usage=low_cpu_mem_usage) + self.set_attn_processor(attn_procs) + + # convert IP-Adapter Image Projection layers to diffusers + image_projection_layers = [] + for state_dict in state_dicts: + image_projection_layer = self._convert_ip_adapter_image_proj_to_diffusers( + state_dict["image_proj"], low_cpu_mem_usage=low_cpu_mem_usage + ) + image_projection_layers.append(image_projection_layer) + + self.encoder_hid_proj = MultiIPAdapterImageProjection(image_projection_layers) + self.config.encoder_hid_dim_type = "ip_image_proj" + + self.to(dtype=self.dtype, device=self.device) + + +class FromOriginalUNetMixin: + """ + Load pretrained UNet model weights saved in the `.ckpt` or `.safetensors` format into a [`StableCascadeUNet`]. + """ + + @classmethod + @validate_hf_hub_args + def from_single_file(cls, pretrained_model_link_or_path, **kwargs): + r""" + Instantiate a [`StableCascadeUNet`] from pretrained StableCascadeUNet weights saved in the original `.ckpt` or + `.safetensors` format. The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pretrained_model_link_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + - A link to the `.ckpt` file (for example + `"https://huggingface.co//blob/main/.ckpt"`) on the Hub. + - A path to a *file* containing all pipeline weights. + config: (`dict`, *optional*): + Dictionary containing the configuration of the model: + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If `"auto"` is passed, the + dtype is automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to True, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables of the model. + + """ + class_name = cls.__name__ + if class_name != "StableCascadeUNet": + raise ValueError("FromOriginalUNetMixin is currently only compatible with StableCascadeUNet") + + config = kwargs.pop("config", None) + resume_download = kwargs.pop("resume_download", False) + force_download = kwargs.pop("force_download", False) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + cache_dir = kwargs.pop("cache_dir", None) + local_files_only = kwargs.pop("local_files_only", None) + revision = kwargs.pop("revision", None) + torch_dtype = kwargs.pop("torch_dtype", None) + + checkpoint = load_single_file_model_checkpoint( + pretrained_model_link_or_path, + resume_download=resume_download, + force_download=force_download, + proxies=proxies, + token=token, + cache_dir=cache_dir, + local_files_only=local_files_only, + revision=revision, + ) + + if config is None: + config = infer_stable_cascade_single_file_config(checkpoint) + model_config = cls.load_config(**config, **kwargs) + else: + model_config = config + + ctx = init_empty_weights if is_accelerate_available() else nullcontext + with ctx(): + model = cls.from_config(model_config, **kwargs) + + diffusers_format_checkpoint = convert_stable_cascade_unet_single_file_to_diffusers(checkpoint) + if is_accelerate_available(): + unexpected_keys = load_model_dict_into_meta(model, diffusers_format_checkpoint, dtype=torch_dtype) + if len(unexpected_keys) > 0: + logger.warn( + f"Some weights of the model checkpoint were not used when initializing {cls.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + + else: + model.load_state_dict(diffusers_format_checkpoint) + + if torch_dtype is not None: + model.to(torch_dtype) + + return model diff --git a/diffusers-0.27.0/src/diffusers/loaders/utils.py b/diffusers-0.27.0/src/diffusers/loaders/utils.py new file mode 100755 index 0000000..142d72b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/loaders/utils.py @@ -0,0 +1,59 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict + +import torch + + +class AttnProcsLayers(torch.nn.Module): + def __init__(self, state_dict: Dict[str, torch.Tensor]): + super().__init__() + self.layers = torch.nn.ModuleList(state_dict.values()) + self.mapping = dict(enumerate(state_dict.keys())) + self.rev_mapping = {v: k for k, v in enumerate(state_dict.keys())} + + # .processor for unet, .self_attn for text encoder + self.split_keys = [".processor", ".self_attn"] + + # we add a hook to state_dict() and load_state_dict() so that the + # naming fits with `unet.attn_processors` + def map_to(module, state_dict, *args, **kwargs): + new_state_dict = {} + for key, value in state_dict.items(): + num = int(key.split(".")[1]) # 0 is always "layers" + new_key = key.replace(f"layers.{num}", module.mapping[num]) + new_state_dict[new_key] = value + + return new_state_dict + + def remap_key(key, state_dict): + for k in self.split_keys: + if k in key: + return key.split(k)[0] + k + + raise ValueError( + f"There seems to be a problem with the state_dict: {set(state_dict.keys())}. {key} has to have one of {self.split_keys}." + ) + + def map_from(module, state_dict, *args, **kwargs): + all_keys = list(state_dict.keys()) + for key in all_keys: + replace_key = remap_key(key, state_dict) + new_key = key.replace(replace_key, f"layers.{module.rev_mapping[replace_key]}") + state_dict[new_key] = state_dict[key] + del state_dict[key] + + self._register_state_dict_hook(map_to) + self._register_load_state_dict_pre_hook(map_from, with_module=True) diff --git a/diffusers-0.27.0/src/diffusers/models/README.md b/diffusers-0.27.0/src/diffusers/models/README.md new file mode 100755 index 0000000..fb91f59 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/README.md @@ -0,0 +1,3 @@ +# Models + +For more detail on the models, please refer to the [docs](https://huggingface.co/docs/diffusers/api/models/overview). \ No newline at end of file diff --git a/diffusers-0.27.0/src/diffusers/models/__init__.py b/diffusers-0.27.0/src/diffusers/models/__init__.py new file mode 100755 index 0000000..da77e44 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/__init__.py @@ -0,0 +1,103 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from ..utils import ( + DIFFUSERS_SLOW_IMPORT, + _LazyModule, + is_flax_available, + is_torch_available, +) + + +_import_structure = {} + +if is_torch_available(): + _import_structure["adapter"] = ["MultiAdapter", "T2IAdapter"] + _import_structure["autoencoders.autoencoder_asym_kl"] = ["AsymmetricAutoencoderKL"] + _import_structure["autoencoders.autoencoder_kl"] = ["AutoencoderKL"] + _import_structure["autoencoders.autoencoder_kl_temporal_decoder"] = ["AutoencoderKLTemporalDecoder"] + _import_structure["autoencoders.autoencoder_tiny"] = ["AutoencoderTiny"] + _import_structure["autoencoders.consistency_decoder_vae"] = ["ConsistencyDecoderVAE"] + _import_structure["controlnet"] = ["ControlNetModel"] + _import_structure["dual_transformer_2d"] = ["DualTransformer2DModel"] + _import_structure["embeddings"] = ["ImageProjection"] + _import_structure["modeling_utils"] = ["ModelMixin"] + _import_structure["transformers.prior_transformer"] = ["PriorTransformer"] + _import_structure["transformers.t5_film_transformer"] = ["T5FilmDecoder"] + _import_structure["transformers.transformer_2d"] = ["Transformer2DModel"] + _import_structure["transformers.transformer_temporal"] = ["TransformerTemporalModel"] + _import_structure["unets.unet_1d"] = ["UNet1DModel"] + _import_structure["unets.unet_2d"] = ["UNet2DModel"] + _import_structure["unets.unet_2d_condition"] = ["UNet2DConditionModel"] + _import_structure["unets.unet_3d_condition"] = ["UNet3DConditionModel"] + _import_structure["unets.unet_i2vgen_xl"] = ["I2VGenXLUNet"] + _import_structure["unets.unet_kandinsky3"] = ["Kandinsky3UNet"] + _import_structure["unets.unet_motion_model"] = ["MotionAdapter", "UNetMotionModel"] + _import_structure["unets.unet_spatio_temporal_condition"] = ["UNetSpatioTemporalConditionModel"] + _import_structure["unets.unet_stable_cascade"] = ["StableCascadeUNet"] + _import_structure["unets.uvit_2d"] = ["UVit2DModel"] + _import_structure["vq_model"] = ["VQModel"] + +if is_flax_available(): + _import_structure["controlnet_flax"] = ["FlaxControlNetModel"] + _import_structure["unets.unet_2d_condition_flax"] = ["FlaxUNet2DConditionModel"] + _import_structure["vae_flax"] = ["FlaxAutoencoderKL"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + if is_torch_available(): + from .adapter import MultiAdapter, T2IAdapter + from .autoencoders import ( + AsymmetricAutoencoderKL, + AutoencoderKL, + AutoencoderKLTemporalDecoder, + AutoencoderTiny, + ConsistencyDecoderVAE, + ) + from .controlnet import ControlNetModel + from .embeddings import ImageProjection + from .modeling_utils import ModelMixin + from .transformers import ( + DualTransformer2DModel, + PriorTransformer, + T5FilmDecoder, + Transformer2DModel, + TransformerTemporalModel, + ) + from .unets import ( + I2VGenXLUNet, + Kandinsky3UNet, + MotionAdapter, + StableCascadeUNet, + UNet1DModel, + UNet2DConditionModel, + UNet2DModel, + UNet3DConditionModel, + UNetMotionModel, + UNetSpatioTemporalConditionModel, + UVit2DModel, + ) + from .vq_model import VQModel + + if is_flax_available(): + from .controlnet_flax import FlaxControlNetModel + from .unets import FlaxUNet2DConditionModel + from .vae_flax import FlaxAutoencoderKL + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) diff --git a/diffusers-0.27.0/src/diffusers/models/activations.py b/diffusers-0.27.0/src/diffusers/models/activations.py new file mode 100755 index 0000000..cec83bd --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/activations.py @@ -0,0 +1,123 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn.functional as F +from torch import nn + +from ..utils import deprecate + + +ACTIVATION_FUNCTIONS = { + "swish": nn.SiLU(), + "silu": nn.SiLU(), + "mish": nn.Mish(), + "gelu": nn.GELU(), + "relu": nn.ReLU(), +} + + +def get_activation(act_fn: str) -> nn.Module: + """Helper function to get activation function from string. + + Args: + act_fn (str): Name of activation function. + + Returns: + nn.Module: Activation function. + """ + + act_fn = act_fn.lower() + if act_fn in ACTIVATION_FUNCTIONS: + return ACTIVATION_FUNCTIONS[act_fn] + else: + raise ValueError(f"Unsupported activation function: {act_fn}") + + +class GELU(nn.Module): + r""" + GELU activation function with tanh approximation support with `approximate="tanh"`. + + Parameters: + dim_in (`int`): The number of channels in the input. + dim_out (`int`): The number of channels in the output. + approximate (`str`, *optional*, defaults to `"none"`): If `"tanh"`, use tanh approximation. + bias (`bool`, defaults to True): Whether to use a bias in the linear layer. + """ + + def __init__(self, dim_in: int, dim_out: int, approximate: str = "none", bias: bool = True): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out, bias=bias) + self.approximate = approximate + + def gelu(self, gate: torch.Tensor) -> torch.Tensor: + if gate.device.type != "mps": + return F.gelu(gate, approximate=self.approximate) + # mps: gelu is not implemented for float16 + return F.gelu(gate.to(dtype=torch.float32), approximate=self.approximate).to(dtype=gate.dtype) + + def forward(self, hidden_states): + hidden_states = self.proj(hidden_states) + hidden_states = self.gelu(hidden_states) + return hidden_states + + +class GEGLU(nn.Module): + r""" + A [variant](https://arxiv.org/abs/2002.05202) of the gated linear unit activation function. + + Parameters: + dim_in (`int`): The number of channels in the input. + dim_out (`int`): The number of channels in the output. + bias (`bool`, defaults to True): Whether to use a bias in the linear layer. + """ + + def __init__(self, dim_in: int, dim_out: int, bias: bool = True): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out * 2, bias=bias) + + def gelu(self, gate: torch.Tensor) -> torch.Tensor: + if gate.device.type != "mps": + return F.gelu(gate) + # mps: gelu is not implemented for float16 + return F.gelu(gate.to(dtype=torch.float32)).to(dtype=gate.dtype) + + def forward(self, hidden_states, *args, **kwargs): + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + hidden_states, gate = self.proj(hidden_states).chunk(2, dim=-1) + return hidden_states * self.gelu(gate) + + +class ApproximateGELU(nn.Module): + r""" + The approximate form of the Gaussian Error Linear Unit (GELU). For more details, see section 2 of this + [paper](https://arxiv.org/abs/1606.08415). + + Parameters: + dim_in (`int`): The number of channels in the input. + dim_out (`int`): The number of channels in the output. + bias (`bool`, defaults to True): Whether to use a bias in the linear layer. + """ + + def __init__(self, dim_in: int, dim_out: int, bias: bool = True): + super().__init__() + self.proj = nn.Linear(dim_in, dim_out, bias=bias) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = self.proj(x) + return x * torch.sigmoid(1.702 * x) diff --git a/diffusers-0.27.0/src/diffusers/models/adapter.py b/diffusers-0.27.0/src/diffusers/models/adapter.py new file mode 100755 index 0000000..0f4b2ec --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/adapter.py @@ -0,0 +1,584 @@ +# Copyright 2022 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from typing import Callable, List, Optional, Union + +import torch +import torch.nn as nn + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import logging +from .modeling_utils import ModelMixin + + +logger = logging.get_logger(__name__) + + +class MultiAdapter(ModelMixin): + r""" + MultiAdapter is a wrapper model that contains multiple adapter models and merges their outputs according to + user-assigned weighting. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for the generic methods the library + implements for all the model (such as downloading or saving, etc.) + + Parameters: + adapters (`List[T2IAdapter]`, *optional*, defaults to None): + A list of `T2IAdapter` model instances. + """ + + def __init__(self, adapters: List["T2IAdapter"]): + super(MultiAdapter, self).__init__() + + self.num_adapter = len(adapters) + self.adapters = nn.ModuleList(adapters) + + if len(adapters) == 0: + raise ValueError("Expecting at least one adapter") + + if len(adapters) == 1: + raise ValueError("For a single adapter, please use the `T2IAdapter` class instead of `MultiAdapter`") + + # The outputs from each adapter are added together with a weight. + # This means that the change in dimensions from downsampling must + # be the same for all adapters. Inductively, it also means the + # downscale_factor and total_downscale_factor must be the same for all + # adapters. + first_adapter_total_downscale_factor = adapters[0].total_downscale_factor + first_adapter_downscale_factor = adapters[0].downscale_factor + for idx in range(1, len(adapters)): + if ( + adapters[idx].total_downscale_factor != first_adapter_total_downscale_factor + or adapters[idx].downscale_factor != first_adapter_downscale_factor + ): + raise ValueError( + f"Expecting all adapters to have the same downscaling behavior, but got:\n" + f"adapters[0].total_downscale_factor={first_adapter_total_downscale_factor}\n" + f"adapters[0].downscale_factor={first_adapter_downscale_factor}\n" + f"adapter[`{idx}`].total_downscale_factor={adapters[idx].total_downscale_factor}\n" + f"adapter[`{idx}`].downscale_factor={adapters[idx].downscale_factor}" + ) + + self.total_downscale_factor = first_adapter_total_downscale_factor + self.downscale_factor = first_adapter_downscale_factor + + def forward(self, xs: torch.Tensor, adapter_weights: Optional[List[float]] = None) -> List[torch.Tensor]: + r""" + Args: + xs (`torch.Tensor`): + (batch, channel, height, width) input images for multiple adapter models concated along dimension 1, + `channel` should equal to `num_adapter` * "number of channel of image". + adapter_weights (`List[float]`, *optional*, defaults to None): + List of floats representing the weight which will be multiply to each adapter's output before adding + them together. + """ + if adapter_weights is None: + adapter_weights = torch.tensor([1 / self.num_adapter] * self.num_adapter) + else: + adapter_weights = torch.tensor(adapter_weights) + + accume_state = None + for x, w, adapter in zip(xs, adapter_weights, self.adapters): + features = adapter(x) + if accume_state is None: + accume_state = features + for i in range(len(accume_state)): + accume_state[i] = w * accume_state[i] + else: + for i in range(len(features)): + accume_state[i] += w * features[i] + return accume_state + + def save_pretrained( + self, + save_directory: Union[str, os.PathLike], + is_main_process: bool = True, + save_function: Callable = None, + safe_serialization: bool = True, + variant: Optional[str] = None, + ): + """ + Save a model and its configuration file to a directory, so that it can be re-loaded using the + `[`~models.adapter.MultiAdapter.from_pretrained`]` class method. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to which to save. Will be created if it doesn't exist. + is_main_process (`bool`, *optional*, defaults to `True`): + Whether the process calling this is the main process or not. Useful when in distributed training like + TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on + the main process to avoid race conditions. + save_function (`Callable`): + The function to use to save the state dictionary. Useful on distributed training like TPUs when one + need to replace `torch.save` by another method. Can be configured with the environment variable + `DIFFUSERS_SAVE_MODE`. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + variant (`str`, *optional*): + If specified, weights are saved in the format pytorch_model..bin. + """ + idx = 0 + model_path_to_save = save_directory + for adapter in self.adapters: + adapter.save_pretrained( + model_path_to_save, + is_main_process=is_main_process, + save_function=save_function, + safe_serialization=safe_serialization, + variant=variant, + ) + + idx += 1 + model_path_to_save = model_path_to_save + f"_{idx}" + + @classmethod + def from_pretrained(cls, pretrained_model_path: Optional[Union[str, os.PathLike]], **kwargs): + r""" + Instantiate a pretrained MultiAdapter model from multiple pre-trained adapter models. + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train + the model, you should first set it back in training mode with `model.train()`. + + The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come + pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning + task. + + The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those + weights are discarded. + + Parameters: + pretrained_model_path (`os.PathLike`): + A path to a *directory* containing model weights saved using + [`~diffusers.models.adapter.MultiAdapter.save_pretrained`], e.g., `./my_model_directory/adapter`. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model under this dtype. If `"auto"` is passed the dtype + will be automatically derived from the model's weights. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn't need to be refined to each + parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the + same device. + + To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier to maximum memory. Will default to the maximum memory available for each + GPU and the available CPU RAM if unset. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading by not initializing the weights and only loading the pre-trained weights. This + also tries to not use more than 1x model size in CPU memory (including peak memory) while loading the + model. This is only supported when torch version >= 1.9.0. If you are using an older version of torch, + setting this argument to `True` will raise an error. + variant (`str`, *optional*): + If specified load weights from `variant` filename, *e.g.* pytorch_model..bin. `variant` is + ignored when using `from_flax`. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the `safetensors` weights will be downloaded if they're available **and** if the + `safetensors` library is installed. If set to `True`, the model will be forcibly loaded from + `safetensors` weights. If set to `False`, loading will *not* use `safetensors`. + """ + idx = 0 + adapters = [] + + # load adapter and append to list until no adapter directory exists anymore + # first adapter has to be saved under `./mydirectory/adapter` to be compliant with `DiffusionPipeline.from_pretrained` + # second, third, ... adapters have to be saved under `./mydirectory/adapter_1`, `./mydirectory/adapter_2`, ... + model_path_to_load = pretrained_model_path + while os.path.isdir(model_path_to_load): + adapter = T2IAdapter.from_pretrained(model_path_to_load, **kwargs) + adapters.append(adapter) + + idx += 1 + model_path_to_load = pretrained_model_path + f"_{idx}" + + logger.info(f"{len(adapters)} adapters loaded from {pretrained_model_path}.") + + if len(adapters) == 0: + raise ValueError( + f"No T2IAdapters found under {os.path.dirname(pretrained_model_path)}. Expected at least {pretrained_model_path + '_0'}." + ) + + return cls(adapters) + + +class T2IAdapter(ModelMixin, ConfigMixin): + r""" + A simple ResNet-like model that accepts images containing control signals such as keyposes and depth. The model + generates multiple feature maps that are used as additional conditioning in [`UNet2DConditionModel`]. The model's + architecture follows the original implementation of + [Adapter](https://github.com/TencentARC/T2I-Adapter/blob/686de4681515662c0ac2ffa07bf5dda83af1038a/ldm/modules/encoders/adapter.py#L97) + and + [AdapterLight](https://github.com/TencentARC/T2I-Adapter/blob/686de4681515662c0ac2ffa07bf5dda83af1038a/ldm/modules/encoders/adapter.py#L235). + + This model inherits from [`ModelMixin`]. Check the superclass documentation for the generic methods the library + implements for all the model (such as downloading or saving, etc.) + + Parameters: + in_channels (`int`, *optional*, defaults to 3): + Number of channels of Aapter's input(*control image*). Set this parameter to 1 if you're using gray scale + image as *control image*. + channels (`List[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The number of channel of each downsample block's output hidden state. The `len(block_out_channels)` will + also determine the number of downsample blocks in the Adapter. + num_res_blocks (`int`, *optional*, defaults to 2): + Number of ResNet blocks in each downsample block. + downscale_factor (`int`, *optional*, defaults to 8): + A factor that determines the total downscale factor of the Adapter. + adapter_type (`str`, *optional*, defaults to `full_adapter`): + The type of Adapter to use. Choose either `full_adapter` or `full_adapter_xl` or `light_adapter`. + """ + + @register_to_config + def __init__( + self, + in_channels: int = 3, + channels: List[int] = [320, 640, 1280, 1280], + num_res_blocks: int = 2, + downscale_factor: int = 8, + adapter_type: str = "full_adapter", + ): + super().__init__() + + if adapter_type == "full_adapter": + self.adapter = FullAdapter(in_channels, channels, num_res_blocks, downscale_factor) + elif adapter_type == "full_adapter_xl": + self.adapter = FullAdapterXL(in_channels, channels, num_res_blocks, downscale_factor) + elif adapter_type == "light_adapter": + self.adapter = LightAdapter(in_channels, channels, num_res_blocks, downscale_factor) + else: + raise ValueError( + f"Unsupported adapter_type: '{adapter_type}'. Choose either 'full_adapter' or " + "'full_adapter_xl' or 'light_adapter'." + ) + + def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + r""" + This function processes the input tensor `x` through the adapter model and returns a list of feature tensors, + each representing information extracted at a different scale from the input. The length of the list is + determined by the number of downsample blocks in the Adapter, as specified by the `channels` and + `num_res_blocks` parameters during initialization. + """ + return self.adapter(x) + + @property + def total_downscale_factor(self): + return self.adapter.total_downscale_factor + + @property + def downscale_factor(self): + """The downscale factor applied in the T2I-Adapter's initial pixel unshuffle operation. If an input image's dimensions are + not evenly divisible by the downscale_factor then an exception will be raised. + """ + return self.adapter.unshuffle.downscale_factor + + +# full adapter + + +class FullAdapter(nn.Module): + r""" + See [`T2IAdapter`] for more information. + """ + + def __init__( + self, + in_channels: int = 3, + channels: List[int] = [320, 640, 1280, 1280], + num_res_blocks: int = 2, + downscale_factor: int = 8, + ): + super().__init__() + + in_channels = in_channels * downscale_factor**2 + + self.unshuffle = nn.PixelUnshuffle(downscale_factor) + self.conv_in = nn.Conv2d(in_channels, channels[0], kernel_size=3, padding=1) + + self.body = nn.ModuleList( + [ + AdapterBlock(channels[0], channels[0], num_res_blocks), + *[ + AdapterBlock(channels[i - 1], channels[i], num_res_blocks, down=True) + for i in range(1, len(channels)) + ], + ] + ) + + self.total_downscale_factor = downscale_factor * 2 ** (len(channels) - 1) + + def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + r""" + This method processes the input tensor `x` through the FullAdapter model and performs operations including + pixel unshuffling, convolution, and a stack of AdapterBlocks. It returns a list of feature tensors, each + capturing information at a different stage of processing within the FullAdapter model. The number of feature + tensors in the list is determined by the number of downsample blocks specified during initialization. + """ + x = self.unshuffle(x) + x = self.conv_in(x) + + features = [] + + for block in self.body: + x = block(x) + features.append(x) + + return features + + +class FullAdapterXL(nn.Module): + r""" + See [`T2IAdapter`] for more information. + """ + + def __init__( + self, + in_channels: int = 3, + channels: List[int] = [320, 640, 1280, 1280], + num_res_blocks: int = 2, + downscale_factor: int = 16, + ): + super().__init__() + + in_channels = in_channels * downscale_factor**2 + + self.unshuffle = nn.PixelUnshuffle(downscale_factor) + self.conv_in = nn.Conv2d(in_channels, channels[0], kernel_size=3, padding=1) + + self.body = [] + # blocks to extract XL features with dimensions of [320, 64, 64], [640, 64, 64], [1280, 32, 32], [1280, 32, 32] + for i in range(len(channels)): + if i == 1: + self.body.append(AdapterBlock(channels[i - 1], channels[i], num_res_blocks)) + elif i == 2: + self.body.append(AdapterBlock(channels[i - 1], channels[i], num_res_blocks, down=True)) + else: + self.body.append(AdapterBlock(channels[i], channels[i], num_res_blocks)) + + self.body = nn.ModuleList(self.body) + # XL has only one downsampling AdapterBlock. + self.total_downscale_factor = downscale_factor * 2 + + def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + r""" + This method takes the tensor x as input and processes it through FullAdapterXL model. It consists of operations + including unshuffling pixels, applying convolution layer and appending each block into list of feature tensors. + """ + x = self.unshuffle(x) + x = self.conv_in(x) + + features = [] + + for block in self.body: + x = block(x) + features.append(x) + + return features + + +class AdapterBlock(nn.Module): + r""" + An AdapterBlock is a helper model that contains multiple ResNet-like blocks. It is used in the `FullAdapter` and + `FullAdapterXL` models. + + Parameters: + in_channels (`int`): + Number of channels of AdapterBlock's input. + out_channels (`int`): + Number of channels of AdapterBlock's output. + num_res_blocks (`int`): + Number of ResNet blocks in the AdapterBlock. + down (`bool`, *optional*, defaults to `False`): + Whether to perform downsampling on AdapterBlock's input. + """ + + def __init__(self, in_channels: int, out_channels: int, num_res_blocks: int, down: bool = False): + super().__init__() + + self.downsample = None + if down: + self.downsample = nn.AvgPool2d(kernel_size=2, stride=2, ceil_mode=True) + + self.in_conv = None + if in_channels != out_channels: + self.in_conv = nn.Conv2d(in_channels, out_channels, kernel_size=1) + + self.resnets = nn.Sequential( + *[AdapterResnetBlock(out_channels) for _ in range(num_res_blocks)], + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + r""" + This method takes tensor x as input and performs operations downsampling and convolutional layers if the + self.downsample and self.in_conv properties of AdapterBlock model are specified. Then it applies a series of + residual blocks to the input tensor. + """ + if self.downsample is not None: + x = self.downsample(x) + + if self.in_conv is not None: + x = self.in_conv(x) + + x = self.resnets(x) + + return x + + +class AdapterResnetBlock(nn.Module): + r""" + An `AdapterResnetBlock` is a helper model that implements a ResNet-like block. + + Parameters: + channels (`int`): + Number of channels of AdapterResnetBlock's input and output. + """ + + def __init__(self, channels: int): + super().__init__() + self.block1 = nn.Conv2d(channels, channels, kernel_size=3, padding=1) + self.act = nn.ReLU() + self.block2 = nn.Conv2d(channels, channels, kernel_size=1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + r""" + This method takes input tensor x and applies a convolutional layer, ReLU activation, and another convolutional + layer on the input tensor. It returns addition with the input tensor. + """ + + h = self.act(self.block1(x)) + h = self.block2(h) + + return h + x + + +# light adapter + + +class LightAdapter(nn.Module): + r""" + See [`T2IAdapter`] for more information. + """ + + def __init__( + self, + in_channels: int = 3, + channels: List[int] = [320, 640, 1280], + num_res_blocks: int = 4, + downscale_factor: int = 8, + ): + super().__init__() + + in_channels = in_channels * downscale_factor**2 + + self.unshuffle = nn.PixelUnshuffle(downscale_factor) + + self.body = nn.ModuleList( + [ + LightAdapterBlock(in_channels, channels[0], num_res_blocks), + *[ + LightAdapterBlock(channels[i], channels[i + 1], num_res_blocks, down=True) + for i in range(len(channels) - 1) + ], + LightAdapterBlock(channels[-1], channels[-1], num_res_blocks, down=True), + ] + ) + + self.total_downscale_factor = downscale_factor * (2 ** len(channels)) + + def forward(self, x: torch.Tensor) -> List[torch.Tensor]: + r""" + This method takes the input tensor x and performs downscaling and appends it in list of feature tensors. Each + feature tensor corresponds to a different level of processing within the LightAdapter. + """ + x = self.unshuffle(x) + + features = [] + + for block in self.body: + x = block(x) + features.append(x) + + return features + + +class LightAdapterBlock(nn.Module): + r""" + A `LightAdapterBlock` is a helper model that contains multiple `LightAdapterResnetBlocks`. It is used in the + `LightAdapter` model. + + Parameters: + in_channels (`int`): + Number of channels of LightAdapterBlock's input. + out_channels (`int`): + Number of channels of LightAdapterBlock's output. + num_res_blocks (`int`): + Number of LightAdapterResnetBlocks in the LightAdapterBlock. + down (`bool`, *optional*, defaults to `False`): + Whether to perform downsampling on LightAdapterBlock's input. + """ + + def __init__(self, in_channels: int, out_channels: int, num_res_blocks: int, down: bool = False): + super().__init__() + mid_channels = out_channels // 4 + + self.downsample = None + if down: + self.downsample = nn.AvgPool2d(kernel_size=2, stride=2, ceil_mode=True) + + self.in_conv = nn.Conv2d(in_channels, mid_channels, kernel_size=1) + self.resnets = nn.Sequential(*[LightAdapterResnetBlock(mid_channels) for _ in range(num_res_blocks)]) + self.out_conv = nn.Conv2d(mid_channels, out_channels, kernel_size=1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + r""" + This method takes tensor x as input and performs downsampling if required. Then it applies in convolution + layer, a sequence of residual blocks, and out convolutional layer. + """ + if self.downsample is not None: + x = self.downsample(x) + + x = self.in_conv(x) + x = self.resnets(x) + x = self.out_conv(x) + + return x + + +class LightAdapterResnetBlock(nn.Module): + """ + A `LightAdapterResnetBlock` is a helper model that implements a ResNet-like block with a slightly different + architecture than `AdapterResnetBlock`. + + Parameters: + channels (`int`): + Number of channels of LightAdapterResnetBlock's input and output. + """ + + def __init__(self, channels: int): + super().__init__() + self.block1 = nn.Conv2d(channels, channels, kernel_size=3, padding=1) + self.act = nn.ReLU() + self.block2 = nn.Conv2d(channels, channels, kernel_size=3, padding=1) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + r""" + This function takes input tensor x and processes it through one convolutional layer, ReLU activation, and + another convolutional layer and adds it to input tensor. + """ + + h = self.act(self.block1(x)) + h = self.block2(h) + + return h + x diff --git a/diffusers-0.27.0/src/diffusers/models/attention.py b/diffusers-0.27.0/src/diffusers/models/attention.py new file mode 100755 index 0000000..3d45cfa --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/attention.py @@ -0,0 +1,665 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Dict, Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from ..utils import deprecate, logging +from ..utils.torch_utils import maybe_allow_in_graph +from .activations import GEGLU, GELU, ApproximateGELU +from .attention_processor import Attention +from .embeddings import SinusoidalPositionalEmbedding +from .normalization import AdaLayerNorm, AdaLayerNormContinuous, AdaLayerNormZero, RMSNorm + + +logger = logging.get_logger(__name__) + + +def _chunked_feed_forward(ff: nn.Module, hidden_states: torch.Tensor, chunk_dim: int, chunk_size: int): + # "feed_forward_chunk_size" can be used to save memory + if hidden_states.shape[chunk_dim] % chunk_size != 0: + raise ValueError( + f"`hidden_states` dimension to be chunked: {hidden_states.shape[chunk_dim]} has to be divisible by chunk size: {chunk_size}. Make sure to set an appropriate `chunk_size` when calling `unet.enable_forward_chunking`." + ) + + num_chunks = hidden_states.shape[chunk_dim] // chunk_size + ff_output = torch.cat( + [ff(hid_slice) for hid_slice in hidden_states.chunk(num_chunks, dim=chunk_dim)], + dim=chunk_dim, + ) + return ff_output + + +@maybe_allow_in_graph +class GatedSelfAttentionDense(nn.Module): + r""" + A gated self-attention dense layer that combines visual features and object features. + + Parameters: + query_dim (`int`): The number of channels in the query. + context_dim (`int`): The number of channels in the context. + n_heads (`int`): The number of heads to use for attention. + d_head (`int`): The number of channels in each head. + """ + + def __init__(self, query_dim: int, context_dim: int, n_heads: int, d_head: int): + super().__init__() + + # we need a linear projection since we need cat visual feature and obj feature + self.linear = nn.Linear(context_dim, query_dim) + + self.attn = Attention(query_dim=query_dim, heads=n_heads, dim_head=d_head) + self.ff = FeedForward(query_dim, activation_fn="geglu") + + self.norm1 = nn.LayerNorm(query_dim) + self.norm2 = nn.LayerNorm(query_dim) + + self.register_parameter("alpha_attn", nn.Parameter(torch.tensor(0.0))) + self.register_parameter("alpha_dense", nn.Parameter(torch.tensor(0.0))) + + self.enabled = True + + def forward(self, x: torch.Tensor, objs: torch.Tensor) -> torch.Tensor: + if not self.enabled: + return x + + n_visual = x.shape[1] + objs = self.linear(objs) + + x = x + self.alpha_attn.tanh() * self.attn(self.norm1(torch.cat([x, objs], dim=1)))[:, :n_visual, :] + x = x + self.alpha_dense.tanh() * self.ff(self.norm2(x)) + + return x + + +@maybe_allow_in_graph +class BasicTransformerBlock(nn.Module): + r""" + A basic Transformer block. + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + num_embeds_ada_norm (: + obj: `int`, *optional*): The number of diffusion steps used during training. See `Transformer2DModel`. + attention_bias (: + obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter. + only_cross_attention (`bool`, *optional*): + Whether to use only cross-attention layers. In this case two cross attention layers are used. + double_self_attention (`bool`, *optional*): + Whether to use two self-attention layers. In this case no cross attention layers are used. + upcast_attention (`bool`, *optional*): + Whether to upcast the attention computation to float32. This is useful for mixed precision training. + norm_elementwise_affine (`bool`, *optional*, defaults to `True`): + Whether to use learnable elementwise affine parameters for normalization. + norm_type (`str`, *optional*, defaults to `"layer_norm"`): + The normalization layer to use. Can be `"layer_norm"`, `"ada_norm"` or `"ada_norm_zero"`. + final_dropout (`bool` *optional*, defaults to False): + Whether to apply a final dropout after the last feed-forward layer. + attention_type (`str`, *optional*, defaults to `"default"`): + The type of attention to use. Can be `"default"` or `"gated"` or `"gated-text-image"`. + positional_embeddings (`str`, *optional*, defaults to `None`): + The type of positional embeddings to apply to. + num_positional_embeddings (`int`, *optional*, defaults to `None`): + The maximum number of positional embeddings to apply. + """ + + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + dropout=0.0, + cross_attention_dim: Optional[int] = None, + activation_fn: str = "geglu", + num_embeds_ada_norm: Optional[int] = None, + attention_bias: bool = False, + only_cross_attention: bool = False, + double_self_attention: bool = False, + upcast_attention: bool = False, + norm_elementwise_affine: bool = True, + norm_type: str = "layer_norm", # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single', 'ada_norm_continuous', 'layer_norm_i2vgen' + norm_eps: float = 1e-5, + final_dropout: bool = False, + attention_type: str = "default", + positional_embeddings: Optional[str] = None, + num_positional_embeddings: Optional[int] = None, + ada_norm_continous_conditioning_embedding_dim: Optional[int] = None, + ada_norm_bias: Optional[int] = None, + ff_inner_dim: Optional[int] = None, + ff_bias: bool = True, + attention_out_bias: bool = True, + ): + super().__init__() + self.only_cross_attention = only_cross_attention + + # We keep these boolean flags for backward-compatibility. + self.use_ada_layer_norm_zero = (num_embeds_ada_norm is not None) and norm_type == "ada_norm_zero" + self.use_ada_layer_norm = (num_embeds_ada_norm is not None) and norm_type == "ada_norm" + self.use_ada_layer_norm_single = norm_type == "ada_norm_single" + self.use_layer_norm = norm_type == "layer_norm" + self.use_ada_layer_norm_continuous = norm_type == "ada_norm_continuous" + + if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None: + raise ValueError( + f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to" + f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}." + ) + + self.norm_type = norm_type + self.num_embeds_ada_norm = num_embeds_ada_norm + + if positional_embeddings and (num_positional_embeddings is None): + raise ValueError( + "If `positional_embedding` type is defined, `num_positition_embeddings` must also be defined." + ) + + if positional_embeddings == "sinusoidal": + self.pos_embed = SinusoidalPositionalEmbedding(dim, max_seq_length=num_positional_embeddings) + else: + self.pos_embed = None + + # Define 3 blocks. Each block has its own normalization layer. + # 1. Self-Attn + if norm_type == "ada_norm": + self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm) + elif norm_type == "ada_norm_zero": + self.norm1 = AdaLayerNormZero(dim, num_embeds_ada_norm) + elif norm_type == "ada_norm_continuous": + self.norm1 = AdaLayerNormContinuous( + dim, + ada_norm_continous_conditioning_embedding_dim, + norm_elementwise_affine, + norm_eps, + ada_norm_bias, + "rms_norm", + ) + else: + self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps) + + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=cross_attention_dim if only_cross_attention else None, + upcast_attention=upcast_attention, + out_bias=attention_out_bias, + ) + + # 2. Cross-Attn + if cross_attention_dim is not None or double_self_attention: + # We currently only use AdaLayerNormZero for self attention where there will only be one attention block. + # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during + # the second cross attention block. + if norm_type == "ada_norm": + self.norm2 = AdaLayerNorm(dim, num_embeds_ada_norm) + elif norm_type == "ada_norm_continuous": + self.norm2 = AdaLayerNormContinuous( + dim, + ada_norm_continous_conditioning_embedding_dim, + norm_elementwise_affine, + norm_eps, + ada_norm_bias, + "rms_norm", + ) + else: + self.norm2 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine) + + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim if not double_self_attention else None, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + upcast_attention=upcast_attention, + out_bias=attention_out_bias, + ) # is self-attn if encoder_hidden_states is none + else: + self.norm2 = None + self.attn2 = None + + # 3. Feed-forward + if norm_type == "ada_norm_continuous": + self.norm3 = AdaLayerNormContinuous( + dim, + ada_norm_continous_conditioning_embedding_dim, + norm_elementwise_affine, + norm_eps, + ada_norm_bias, + "layer_norm", + ) + + elif norm_type in ["ada_norm_zero", "ada_norm", "layer_norm", "ada_norm_continuous"]: + self.norm3 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine) + elif norm_type == "layer_norm_i2vgen": + self.norm3 = None + + self.ff = FeedForward( + dim, + dropout=dropout, + activation_fn=activation_fn, + final_dropout=final_dropout, + inner_dim=ff_inner_dim, + bias=ff_bias, + ) + + # 4. Fuser + if attention_type == "gated" or attention_type == "gated-text-image": + self.fuser = GatedSelfAttentionDense(dim, cross_attention_dim, num_attention_heads, attention_head_dim) + + # 5. Scale-shift for PixArt-Alpha. + if norm_type == "ada_norm_single": + self.scale_shift_table = nn.Parameter(torch.randn(6, dim) / dim**0.5) + + # let chunk size default to None + self._chunk_size = None + self._chunk_dim = 0 + + def set_chunk_feed_forward(self, chunk_size: Optional[int], dim: int = 0): + # Sets chunk feed-forward + self._chunk_size = chunk_size + self._chunk_dim = dim + + def forward( + self, + hidden_states: torch.FloatTensor, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + timestep: Optional[torch.LongTensor] = None, + cross_attention_kwargs: Dict[str, Any] = None, + class_labels: Optional[torch.LongTensor] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + ) -> torch.FloatTensor: + if cross_attention_kwargs is not None: + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + # Notice that normalization is always applied before the real computation in the following blocks. + # 0. Self-Attention + batch_size = hidden_states.shape[0] + + if self.norm_type == "ada_norm": + norm_hidden_states = self.norm1(hidden_states, timestep) + elif self.norm_type == "ada_norm_zero": + norm_hidden_states, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.norm1( + hidden_states, timestep, class_labels, hidden_dtype=hidden_states.dtype + ) + elif self.norm_type in ["layer_norm", "layer_norm_i2vgen"]: + norm_hidden_states = self.norm1(hidden_states) + elif self.norm_type == "ada_norm_continuous": + norm_hidden_states = self.norm1(hidden_states, added_cond_kwargs["pooled_text_emb"]) + elif self.norm_type == "ada_norm_single": + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = ( + self.scale_shift_table[None] + timestep.reshape(batch_size, 6, -1) + ).chunk(6, dim=1) + norm_hidden_states = self.norm1(hidden_states) + norm_hidden_states = norm_hidden_states * (1 + scale_msa) + shift_msa + norm_hidden_states = norm_hidden_states.squeeze(1) + else: + raise ValueError("Incorrect norm used") + + if self.pos_embed is not None: + norm_hidden_states = self.pos_embed(norm_hidden_states) + + # 1. Prepare GLIGEN inputs + cross_attention_kwargs = cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {} + gligen_kwargs = cross_attention_kwargs.pop("gligen", None) + + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + if self.norm_type == "ada_norm_zero": + attn_output = gate_msa.unsqueeze(1) * attn_output + elif self.norm_type == "ada_norm_single": + attn_output = gate_msa * attn_output + + hidden_states = attn_output + hidden_states + if hidden_states.ndim == 4: + hidden_states = hidden_states.squeeze(1) + + # 1.2 GLIGEN Control + if gligen_kwargs is not None: + hidden_states = self.fuser(hidden_states, gligen_kwargs["objs"]) + + # 3. Cross-Attention + if self.attn2 is not None: + if self.norm_type == "ada_norm": + norm_hidden_states = self.norm2(hidden_states, timestep) + elif self.norm_type in ["ada_norm_zero", "layer_norm", "layer_norm_i2vgen"]: + norm_hidden_states = self.norm2(hidden_states) + elif self.norm_type == "ada_norm_single": + # For PixArt norm2 isn't applied here: + # https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L70C1-L76C103 + norm_hidden_states = hidden_states + elif self.norm_type == "ada_norm_continuous": + norm_hidden_states = self.norm2(hidden_states, added_cond_kwargs["pooled_text_emb"]) + else: + raise ValueError("Incorrect norm") + + if self.pos_embed is not None and self.norm_type != "ada_norm_single": + norm_hidden_states = self.pos_embed(norm_hidden_states) + + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + **cross_attention_kwargs, + ) + hidden_states = attn_output + hidden_states + + # 4. Feed-forward + # i2vgen doesn't have this norm 🤷‍♂️ + if self.norm_type == "ada_norm_continuous": + norm_hidden_states = self.norm3(hidden_states, added_cond_kwargs["pooled_text_emb"]) + elif not self.norm_type == "ada_norm_single": + norm_hidden_states = self.norm3(hidden_states) + + if self.norm_type == "ada_norm_zero": + norm_hidden_states = norm_hidden_states * (1 + scale_mlp[:, None]) + shift_mlp[:, None] + + if self.norm_type == "ada_norm_single": + norm_hidden_states = self.norm2(hidden_states) + norm_hidden_states = norm_hidden_states * (1 + scale_mlp) + shift_mlp + + if self._chunk_size is not None: + # "feed_forward_chunk_size" can be used to save memory + ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size) + else: + ff_output = self.ff(norm_hidden_states) + + if self.norm_type == "ada_norm_zero": + ff_output = gate_mlp.unsqueeze(1) * ff_output + elif self.norm_type == "ada_norm_single": + ff_output = gate_mlp * ff_output + + hidden_states = ff_output + hidden_states + if hidden_states.ndim == 4: + hidden_states = hidden_states.squeeze(1) + + return hidden_states + + +@maybe_allow_in_graph +class TemporalBasicTransformerBlock(nn.Module): + r""" + A basic Transformer block for video like data. + + Parameters: + dim (`int`): The number of channels in the input and output. + time_mix_inner_dim (`int`): The number of channels for temporal attention. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + """ + + def __init__( + self, + dim: int, + time_mix_inner_dim: int, + num_attention_heads: int, + attention_head_dim: int, + cross_attention_dim: Optional[int] = None, + ): + super().__init__() + self.is_res = dim == time_mix_inner_dim + + self.norm_in = nn.LayerNorm(dim) + + # Define 3 blocks. Each block has its own normalization layer. + # 1. Self-Attn + self.ff_in = FeedForward( + dim, + dim_out=time_mix_inner_dim, + activation_fn="geglu", + ) + + self.norm1 = nn.LayerNorm(time_mix_inner_dim) + self.attn1 = Attention( + query_dim=time_mix_inner_dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + cross_attention_dim=None, + ) + + # 2. Cross-Attn + if cross_attention_dim is not None: + # We currently only use AdaLayerNormZero for self attention where there will only be one attention block. + # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during + # the second cross attention block. + self.norm2 = nn.LayerNorm(time_mix_inner_dim) + self.attn2 = Attention( + query_dim=time_mix_inner_dim, + cross_attention_dim=cross_attention_dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + ) # is self-attn if encoder_hidden_states is none + else: + self.norm2 = None + self.attn2 = None + + # 3. Feed-forward + self.norm3 = nn.LayerNorm(time_mix_inner_dim) + self.ff = FeedForward(time_mix_inner_dim, activation_fn="geglu") + + # let chunk size default to None + self._chunk_size = None + self._chunk_dim = None + + def set_chunk_feed_forward(self, chunk_size: Optional[int], **kwargs): + # Sets chunk feed-forward + self._chunk_size = chunk_size + # chunk dim should be hardcoded to 1 to have better speed vs. memory trade-off + self._chunk_dim = 1 + + def forward( + self, + hidden_states: torch.FloatTensor, + num_frames: int, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + # Notice that normalization is always applied before the real computation in the following blocks. + # 0. Self-Attention + batch_size = hidden_states.shape[0] + + batch_frames, seq_length, channels = hidden_states.shape + batch_size = batch_frames // num_frames + + hidden_states = hidden_states[None, :].reshape(batch_size, num_frames, seq_length, channels) + hidden_states = hidden_states.permute(0, 2, 1, 3) + hidden_states = hidden_states.reshape(batch_size * seq_length, num_frames, channels) + + residual = hidden_states + hidden_states = self.norm_in(hidden_states) + + if self._chunk_size is not None: + hidden_states = _chunked_feed_forward(self.ff_in, hidden_states, self._chunk_dim, self._chunk_size) + else: + hidden_states = self.ff_in(hidden_states) + + if self.is_res: + hidden_states = hidden_states + residual + + norm_hidden_states = self.norm1(hidden_states) + attn_output = self.attn1(norm_hidden_states, encoder_hidden_states=None) + hidden_states = attn_output + hidden_states + + # 3. Cross-Attention + if self.attn2 is not None: + norm_hidden_states = self.norm2(hidden_states) + attn_output = self.attn2(norm_hidden_states, encoder_hidden_states=encoder_hidden_states) + hidden_states = attn_output + hidden_states + + # 4. Feed-forward + norm_hidden_states = self.norm3(hidden_states) + + if self._chunk_size is not None: + ff_output = _chunked_feed_forward(self.ff, norm_hidden_states, self._chunk_dim, self._chunk_size) + else: + ff_output = self.ff(norm_hidden_states) + + if self.is_res: + hidden_states = ff_output + hidden_states + else: + hidden_states = ff_output + + hidden_states = hidden_states[None, :].reshape(batch_size, seq_length, num_frames, channels) + hidden_states = hidden_states.permute(0, 2, 1, 3) + hidden_states = hidden_states.reshape(batch_size * num_frames, seq_length, channels) + + return hidden_states + + +class SkipFFTransformerBlock(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + kv_input_dim: int, + kv_input_dim_proj_use_bias: bool, + dropout=0.0, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + attention_out_bias: bool = True, + ): + super().__init__() + if kv_input_dim != dim: + self.kv_mapper = nn.Linear(kv_input_dim, dim, kv_input_dim_proj_use_bias) + else: + self.kv_mapper = None + + self.norm1 = RMSNorm(dim, 1e-06) + + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=cross_attention_dim, + out_bias=attention_out_bias, + ) + + self.norm2 = RMSNorm(dim, 1e-06) + + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + out_bias=attention_out_bias, + ) + + def forward(self, hidden_states, encoder_hidden_states, cross_attention_kwargs): + cross_attention_kwargs = cross_attention_kwargs.copy() if cross_attention_kwargs is not None else {} + + if self.kv_mapper is not None: + encoder_hidden_states = self.kv_mapper(F.silu(encoder_hidden_states)) + + norm_hidden_states = self.norm1(hidden_states) + + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + **cross_attention_kwargs, + ) + + hidden_states = attn_output + hidden_states + + norm_hidden_states = self.norm2(hidden_states) + + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + **cross_attention_kwargs, + ) + + hidden_states = attn_output + hidden_states + + return hidden_states + + +class FeedForward(nn.Module): + r""" + A feed-forward layer. + + Parameters: + dim (`int`): The number of channels in the input. + dim_out (`int`, *optional*): The number of channels in the output. If not given, defaults to `dim`. + mult (`int`, *optional*, defaults to 4): The multiplier to use for the hidden dimension. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + final_dropout (`bool` *optional*, defaults to False): Apply a final dropout. + bias (`bool`, defaults to True): Whether to use a bias in the linear layer. + """ + + def __init__( + self, + dim: int, + dim_out: Optional[int] = None, + mult: int = 4, + dropout: float = 0.0, + activation_fn: str = "geglu", + final_dropout: bool = False, + inner_dim=None, + bias: bool = True, + ): + super().__init__() + if inner_dim is None: + inner_dim = int(dim * mult) + dim_out = dim_out if dim_out is not None else dim + linear_cls = nn.Linear + + if activation_fn == "gelu": + act_fn = GELU(dim, inner_dim, bias=bias) + if activation_fn == "gelu-approximate": + act_fn = GELU(dim, inner_dim, approximate="tanh", bias=bias) + elif activation_fn == "geglu": + act_fn = GEGLU(dim, inner_dim, bias=bias) + elif activation_fn == "geglu-approximate": + act_fn = ApproximateGELU(dim, inner_dim, bias=bias) + + self.net = nn.ModuleList([]) + # project in + self.net.append(act_fn) + # project dropout + self.net.append(nn.Dropout(dropout)) + # project out + self.net.append(linear_cls(inner_dim, dim_out, bias=bias)) + # FF as used in Vision Transformer, MLP-Mixer, etc. have a final dropout + if final_dropout: + self.net.append(nn.Dropout(dropout)) + + def forward(self, hidden_states: torch.Tensor, *args, **kwargs) -> torch.Tensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + for module in self.net: + hidden_states = module(hidden_states) + return hidden_states diff --git a/diffusers-0.27.0/src/diffusers/models/attention_flax.py b/diffusers-0.27.0/src/diffusers/models/attention_flax.py new file mode 100755 index 0000000..25ae5d0 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/attention_flax.py @@ -0,0 +1,494 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import functools +import math + +import flax.linen as nn +import jax +import jax.numpy as jnp + + +def _query_chunk_attention(query, key, value, precision, key_chunk_size: int = 4096): + """Multi-head dot product attention with a limited number of queries.""" + num_kv, num_heads, k_features = key.shape[-3:] + v_features = value.shape[-1] + key_chunk_size = min(key_chunk_size, num_kv) + query = query / jnp.sqrt(k_features) + + @functools.partial(jax.checkpoint, prevent_cse=False) + def summarize_chunk(query, key, value): + attn_weights = jnp.einsum("...qhd,...khd->...qhk", query, key, precision=precision) + + max_score = jnp.max(attn_weights, axis=-1, keepdims=True) + max_score = jax.lax.stop_gradient(max_score) + exp_weights = jnp.exp(attn_weights - max_score) + + exp_values = jnp.einsum("...vhf,...qhv->...qhf", value, exp_weights, precision=precision) + max_score = jnp.einsum("...qhk->...qh", max_score) + + return (exp_values, exp_weights.sum(axis=-1), max_score) + + def chunk_scanner(chunk_idx): + # julienne key array + key_chunk = jax.lax.dynamic_slice( + operand=key, + start_indices=[0] * (key.ndim - 3) + [chunk_idx, 0, 0], # [...,k,h,d] + slice_sizes=list(key.shape[:-3]) + [key_chunk_size, num_heads, k_features], # [...,k,h,d] + ) + + # julienne value array + value_chunk = jax.lax.dynamic_slice( + operand=value, + start_indices=[0] * (value.ndim - 3) + [chunk_idx, 0, 0], # [...,v,h,d] + slice_sizes=list(value.shape[:-3]) + [key_chunk_size, num_heads, v_features], # [...,v,h,d] + ) + + return summarize_chunk(query, key_chunk, value_chunk) + + chunk_values, chunk_weights, chunk_max = jax.lax.map(f=chunk_scanner, xs=jnp.arange(0, num_kv, key_chunk_size)) + + global_max = jnp.max(chunk_max, axis=0, keepdims=True) + max_diffs = jnp.exp(chunk_max - global_max) + + chunk_values *= jnp.expand_dims(max_diffs, axis=-1) + chunk_weights *= max_diffs + + all_values = chunk_values.sum(axis=0) + all_weights = jnp.expand_dims(chunk_weights, -1).sum(axis=0) + + return all_values / all_weights + + +def jax_memory_efficient_attention( + query, key, value, precision=jax.lax.Precision.HIGHEST, query_chunk_size: int = 1024, key_chunk_size: int = 4096 +): + r""" + Flax Memory-efficient multi-head dot product attention. https://arxiv.org/abs/2112.05682v2 + https://github.com/AminRezaei0x443/memory-efficient-attention + + Args: + query (`jnp.ndarray`): (batch..., query_length, head, query_key_depth_per_head) + key (`jnp.ndarray`): (batch..., key_value_length, head, query_key_depth_per_head) + value (`jnp.ndarray`): (batch..., key_value_length, head, value_depth_per_head) + precision (`jax.lax.Precision`, *optional*, defaults to `jax.lax.Precision.HIGHEST`): + numerical precision for computation + query_chunk_size (`int`, *optional*, defaults to 1024): + chunk size to divide query array value must divide query_length equally without remainder + key_chunk_size (`int`, *optional*, defaults to 4096): + chunk size to divide key and value array value must divide key_value_length equally without remainder + + Returns: + (`jnp.ndarray`) with shape of (batch..., query_length, head, value_depth_per_head) + """ + num_q, num_heads, q_features = query.shape[-3:] + + def chunk_scanner(chunk_idx, _): + # julienne query array + query_chunk = jax.lax.dynamic_slice( + operand=query, + start_indices=([0] * (query.ndim - 3)) + [chunk_idx, 0, 0], # [...,q,h,d] + slice_sizes=list(query.shape[:-3]) + [min(query_chunk_size, num_q), num_heads, q_features], # [...,q,h,d] + ) + + return ( + chunk_idx + query_chunk_size, # unused ignore it + _query_chunk_attention( + query=query_chunk, key=key, value=value, precision=precision, key_chunk_size=key_chunk_size + ), + ) + + _, res = jax.lax.scan( + f=chunk_scanner, + init=0, + xs=None, + length=math.ceil(num_q / query_chunk_size), # start counter # stop counter + ) + + return jnp.concatenate(res, axis=-3) # fuse the chunked result back + + +class FlaxAttention(nn.Module): + r""" + A Flax multi-head attention module as described in: https://arxiv.org/abs/1706.03762 + + Parameters: + query_dim (:obj:`int`): + Input hidden states dimension + heads (:obj:`int`, *optional*, defaults to 8): + Number of heads + dim_head (:obj:`int`, *optional*, defaults to 64): + Hidden states dimension inside each head + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + use_memory_efficient_attention (`bool`, *optional*, defaults to `False`): + enable memory efficient attention https://arxiv.org/abs/2112.05682 + split_head_dim (`bool`, *optional*, defaults to `False`): + Whether to split the head dimension into a new axis for the self-attention computation. In most cases, + enabling this flag should speed up the computation for Stable Diffusion 2.x and Stable Diffusion XL. + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + + """ + + query_dim: int + heads: int = 8 + dim_head: int = 64 + dropout: float = 0.0 + use_memory_efficient_attention: bool = False + split_head_dim: bool = False + dtype: jnp.dtype = jnp.float32 + + def setup(self): + inner_dim = self.dim_head * self.heads + self.scale = self.dim_head**-0.5 + + # Weights were exported with old names {to_q, to_k, to_v, to_out} + self.query = nn.Dense(inner_dim, use_bias=False, dtype=self.dtype, name="to_q") + self.key = nn.Dense(inner_dim, use_bias=False, dtype=self.dtype, name="to_k") + self.value = nn.Dense(inner_dim, use_bias=False, dtype=self.dtype, name="to_v") + + self.proj_attn = nn.Dense(self.query_dim, dtype=self.dtype, name="to_out_0") + self.dropout_layer = nn.Dropout(rate=self.dropout) + + def reshape_heads_to_batch_dim(self, tensor): + batch_size, seq_len, dim = tensor.shape + head_size = self.heads + tensor = tensor.reshape(batch_size, seq_len, head_size, dim // head_size) + tensor = jnp.transpose(tensor, (0, 2, 1, 3)) + tensor = tensor.reshape(batch_size * head_size, seq_len, dim // head_size) + return tensor + + def reshape_batch_dim_to_heads(self, tensor): + batch_size, seq_len, dim = tensor.shape + head_size = self.heads + tensor = tensor.reshape(batch_size // head_size, head_size, seq_len, dim) + tensor = jnp.transpose(tensor, (0, 2, 1, 3)) + tensor = tensor.reshape(batch_size // head_size, seq_len, dim * head_size) + return tensor + + def __call__(self, hidden_states, context=None, deterministic=True): + context = hidden_states if context is None else context + + query_proj = self.query(hidden_states) + key_proj = self.key(context) + value_proj = self.value(context) + + if self.split_head_dim: + b = hidden_states.shape[0] + query_states = jnp.reshape(query_proj, (b, -1, self.heads, self.dim_head)) + key_states = jnp.reshape(key_proj, (b, -1, self.heads, self.dim_head)) + value_states = jnp.reshape(value_proj, (b, -1, self.heads, self.dim_head)) + else: + query_states = self.reshape_heads_to_batch_dim(query_proj) + key_states = self.reshape_heads_to_batch_dim(key_proj) + value_states = self.reshape_heads_to_batch_dim(value_proj) + + if self.use_memory_efficient_attention: + query_states = query_states.transpose(1, 0, 2) + key_states = key_states.transpose(1, 0, 2) + value_states = value_states.transpose(1, 0, 2) + + # this if statement create a chunk size for each layer of the unet + # the chunk size is equal to the query_length dimension of the deepest layer of the unet + + flatten_latent_dim = query_states.shape[-3] + if flatten_latent_dim % 64 == 0: + query_chunk_size = int(flatten_latent_dim / 64) + elif flatten_latent_dim % 16 == 0: + query_chunk_size = int(flatten_latent_dim / 16) + elif flatten_latent_dim % 4 == 0: + query_chunk_size = int(flatten_latent_dim / 4) + else: + query_chunk_size = int(flatten_latent_dim) + + hidden_states = jax_memory_efficient_attention( + query_states, key_states, value_states, query_chunk_size=query_chunk_size, key_chunk_size=4096 * 4 + ) + + hidden_states = hidden_states.transpose(1, 0, 2) + else: + # compute attentions + if self.split_head_dim: + attention_scores = jnp.einsum("b t n h, b f n h -> b n f t", key_states, query_states) + else: + attention_scores = jnp.einsum("b i d, b j d->b i j", query_states, key_states) + + attention_scores = attention_scores * self.scale + attention_probs = nn.softmax(attention_scores, axis=-1 if self.split_head_dim else 2) + + # attend to values + if self.split_head_dim: + hidden_states = jnp.einsum("b n f t, b t n h -> b f n h", attention_probs, value_states) + b = hidden_states.shape[0] + hidden_states = jnp.reshape(hidden_states, (b, -1, self.heads * self.dim_head)) + else: + hidden_states = jnp.einsum("b i j, b j d -> b i d", attention_probs, value_states) + hidden_states = self.reshape_batch_dim_to_heads(hidden_states) + + hidden_states = self.proj_attn(hidden_states) + return self.dropout_layer(hidden_states, deterministic=deterministic) + + +class FlaxBasicTransformerBlock(nn.Module): + r""" + A Flax transformer block layer with `GLU` (Gated Linear Unit) activation function as described in: + https://arxiv.org/abs/1706.03762 + + + Parameters: + dim (:obj:`int`): + Inner hidden states dimension + n_heads (:obj:`int`): + Number of heads + d_head (:obj:`int`): + Hidden states dimension inside each head + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + only_cross_attention (`bool`, defaults to `False`): + Whether to only apply cross attention. + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + use_memory_efficient_attention (`bool`, *optional*, defaults to `False`): + enable memory efficient attention https://arxiv.org/abs/2112.05682 + split_head_dim (`bool`, *optional*, defaults to `False`): + Whether to split the head dimension into a new axis for the self-attention computation. In most cases, + enabling this flag should speed up the computation for Stable Diffusion 2.x and Stable Diffusion XL. + """ + + dim: int + n_heads: int + d_head: int + dropout: float = 0.0 + only_cross_attention: bool = False + dtype: jnp.dtype = jnp.float32 + use_memory_efficient_attention: bool = False + split_head_dim: bool = False + + def setup(self): + # self attention (or cross_attention if only_cross_attention is True) + self.attn1 = FlaxAttention( + self.dim, + self.n_heads, + self.d_head, + self.dropout, + self.use_memory_efficient_attention, + self.split_head_dim, + dtype=self.dtype, + ) + # cross attention + self.attn2 = FlaxAttention( + self.dim, + self.n_heads, + self.d_head, + self.dropout, + self.use_memory_efficient_attention, + self.split_head_dim, + dtype=self.dtype, + ) + self.ff = FlaxFeedForward(dim=self.dim, dropout=self.dropout, dtype=self.dtype) + self.norm1 = nn.LayerNorm(epsilon=1e-5, dtype=self.dtype) + self.norm2 = nn.LayerNorm(epsilon=1e-5, dtype=self.dtype) + self.norm3 = nn.LayerNorm(epsilon=1e-5, dtype=self.dtype) + self.dropout_layer = nn.Dropout(rate=self.dropout) + + def __call__(self, hidden_states, context, deterministic=True): + # self attention + residual = hidden_states + if self.only_cross_attention: + hidden_states = self.attn1(self.norm1(hidden_states), context, deterministic=deterministic) + else: + hidden_states = self.attn1(self.norm1(hidden_states), deterministic=deterministic) + hidden_states = hidden_states + residual + + # cross attention + residual = hidden_states + hidden_states = self.attn2(self.norm2(hidden_states), context, deterministic=deterministic) + hidden_states = hidden_states + residual + + # feed forward + residual = hidden_states + hidden_states = self.ff(self.norm3(hidden_states), deterministic=deterministic) + hidden_states = hidden_states + residual + + return self.dropout_layer(hidden_states, deterministic=deterministic) + + +class FlaxTransformer2DModel(nn.Module): + r""" + A Spatial Transformer layer with Gated Linear Unit (GLU) activation function as described in: + https://arxiv.org/pdf/1506.02025.pdf + + + Parameters: + in_channels (:obj:`int`): + Input number of channels + n_heads (:obj:`int`): + Number of heads + d_head (:obj:`int`): + Hidden states dimension inside each head + depth (:obj:`int`, *optional*, defaults to 1): + Number of transformers block + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + use_linear_projection (`bool`, defaults to `False`): tbd + only_cross_attention (`bool`, defaults to `False`): tbd + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + use_memory_efficient_attention (`bool`, *optional*, defaults to `False`): + enable memory efficient attention https://arxiv.org/abs/2112.05682 + split_head_dim (`bool`, *optional*, defaults to `False`): + Whether to split the head dimension into a new axis for the self-attention computation. In most cases, + enabling this flag should speed up the computation for Stable Diffusion 2.x and Stable Diffusion XL. + """ + + in_channels: int + n_heads: int + d_head: int + depth: int = 1 + dropout: float = 0.0 + use_linear_projection: bool = False + only_cross_attention: bool = False + dtype: jnp.dtype = jnp.float32 + use_memory_efficient_attention: bool = False + split_head_dim: bool = False + + def setup(self): + self.norm = nn.GroupNorm(num_groups=32, epsilon=1e-5) + + inner_dim = self.n_heads * self.d_head + if self.use_linear_projection: + self.proj_in = nn.Dense(inner_dim, dtype=self.dtype) + else: + self.proj_in = nn.Conv( + inner_dim, + kernel_size=(1, 1), + strides=(1, 1), + padding="VALID", + dtype=self.dtype, + ) + + self.transformer_blocks = [ + FlaxBasicTransformerBlock( + inner_dim, + self.n_heads, + self.d_head, + dropout=self.dropout, + only_cross_attention=self.only_cross_attention, + dtype=self.dtype, + use_memory_efficient_attention=self.use_memory_efficient_attention, + split_head_dim=self.split_head_dim, + ) + for _ in range(self.depth) + ] + + if self.use_linear_projection: + self.proj_out = nn.Dense(inner_dim, dtype=self.dtype) + else: + self.proj_out = nn.Conv( + inner_dim, + kernel_size=(1, 1), + strides=(1, 1), + padding="VALID", + dtype=self.dtype, + ) + + self.dropout_layer = nn.Dropout(rate=self.dropout) + + def __call__(self, hidden_states, context, deterministic=True): + batch, height, width, channels = hidden_states.shape + residual = hidden_states + hidden_states = self.norm(hidden_states) + if self.use_linear_projection: + hidden_states = hidden_states.reshape(batch, height * width, channels) + hidden_states = self.proj_in(hidden_states) + else: + hidden_states = self.proj_in(hidden_states) + hidden_states = hidden_states.reshape(batch, height * width, channels) + + for transformer_block in self.transformer_blocks: + hidden_states = transformer_block(hidden_states, context, deterministic=deterministic) + + if self.use_linear_projection: + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states.reshape(batch, height, width, channels) + else: + hidden_states = hidden_states.reshape(batch, height, width, channels) + hidden_states = self.proj_out(hidden_states) + + hidden_states = hidden_states + residual + return self.dropout_layer(hidden_states, deterministic=deterministic) + + +class FlaxFeedForward(nn.Module): + r""" + Flax module that encapsulates two Linear layers separated by a non-linearity. It is the counterpart of PyTorch's + [`FeedForward`] class, with the following simplifications: + - The activation function is currently hardcoded to a gated linear unit from: + https://arxiv.org/abs/2002.05202 + - `dim_out` is equal to `dim`. + - The number of hidden dimensions is hardcoded to `dim * 4` in [`FlaxGELU`]. + + Parameters: + dim (:obj:`int`): + Inner hidden states dimension + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + dim: int + dropout: float = 0.0 + dtype: jnp.dtype = jnp.float32 + + def setup(self): + # The second linear layer needs to be called + # net_2 for now to match the index of the Sequential layer + self.net_0 = FlaxGEGLU(self.dim, self.dropout, self.dtype) + self.net_2 = nn.Dense(self.dim, dtype=self.dtype) + + def __call__(self, hidden_states, deterministic=True): + hidden_states = self.net_0(hidden_states, deterministic=deterministic) + hidden_states = self.net_2(hidden_states) + return hidden_states + + +class FlaxGEGLU(nn.Module): + r""" + Flax implementation of a Linear layer followed by the variant of the gated linear unit activation function from + https://arxiv.org/abs/2002.05202. + + Parameters: + dim (:obj:`int`): + Input hidden states dimension + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + dim: int + dropout: float = 0.0 + dtype: jnp.dtype = jnp.float32 + + def setup(self): + inner_dim = self.dim * 4 + self.proj = nn.Dense(inner_dim * 2, dtype=self.dtype) + self.dropout_layer = nn.Dropout(rate=self.dropout) + + def __call__(self, hidden_states, deterministic=True): + hidden_states = self.proj(hidden_states) + hidden_linear, hidden_gelu = jnp.split(hidden_states, 2, axis=2) + return self.dropout_layer(hidden_linear * nn.gelu(hidden_gelu), deterministic=deterministic) diff --git a/diffusers-0.27.0/src/diffusers/models/attention_processor.py b/diffusers-0.27.0/src/diffusers/models/attention_processor.py new file mode 100755 index 0000000..44fbd58 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/attention_processor.py @@ -0,0 +1,2480 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +from importlib import import_module +from typing import Callable, Optional, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from ..image_processor import IPAdapterMaskProcessor +from ..utils import deprecate, logging +from ..utils.import_utils import is_xformers_available +from ..utils.torch_utils import maybe_allow_in_graph +from .lora import LoRALinearLayer + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +if is_xformers_available(): + import xformers + import xformers.ops +else: + xformers = None + + +@maybe_allow_in_graph +class Attention(nn.Module): + r""" + A cross attention layer. + + Parameters: + query_dim (`int`): + The number of channels in the query. + cross_attention_dim (`int`, *optional*): + The number of channels in the encoder_hidden_states. If not given, defaults to `query_dim`. + heads (`int`, *optional*, defaults to 8): + The number of heads to use for multi-head attention. + dim_head (`int`, *optional*, defaults to 64): + The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): + The dropout probability to use. + bias (`bool`, *optional*, defaults to False): + Set to `True` for the query, key, and value linear layers to contain a bias parameter. + upcast_attention (`bool`, *optional*, defaults to False): + Set to `True` to upcast the attention computation to `float32`. + upcast_softmax (`bool`, *optional*, defaults to False): + Set to `True` to upcast the softmax computation to `float32`. + cross_attention_norm (`str`, *optional*, defaults to `None`): + The type of normalization to use for the cross attention. Can be `None`, `layer_norm`, or `group_norm`. + cross_attention_norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups to use for the group norm in the cross attention. + added_kv_proj_dim (`int`, *optional*, defaults to `None`): + The number of channels to use for the added key and value projections. If `None`, no projection is used. + norm_num_groups (`int`, *optional*, defaults to `None`): + The number of groups to use for the group norm in the attention. + spatial_norm_dim (`int`, *optional*, defaults to `None`): + The number of channels to use for the spatial normalization. + out_bias (`bool`, *optional*, defaults to `True`): + Set to `True` to use a bias in the output linear layer. + scale_qk (`bool`, *optional*, defaults to `True`): + Set to `True` to scale the query and key by `1 / sqrt(dim_head)`. + only_cross_attention (`bool`, *optional*, defaults to `False`): + Set to `True` to only use cross attention and not added_kv_proj_dim. Can only be set to `True` if + `added_kv_proj_dim` is not `None`. + eps (`float`, *optional*, defaults to 1e-5): + An additional value added to the denominator in group normalization that is used for numerical stability. + rescale_output_factor (`float`, *optional*, defaults to 1.0): + A factor to rescale the output by dividing it with this value. + residual_connection (`bool`, *optional*, defaults to `False`): + Set to `True` to add the residual connection to the output. + _from_deprecated_attn_block (`bool`, *optional*, defaults to `False`): + Set to `True` if the attention block is loaded from a deprecated state dict. + processor (`AttnProcessor`, *optional*, defaults to `None`): + The attention processor to use. If `None`, defaults to `AttnProcessor2_0` if `torch 2.x` is used and + `AttnProcessor` otherwise. + """ + + def __init__( + self, + query_dim: int, + cross_attention_dim: Optional[int] = None, + heads: int = 8, + dim_head: int = 64, + dropout: float = 0.0, + bias: bool = False, + upcast_attention: bool = False, + upcast_softmax: bool = False, + cross_attention_norm: Optional[str] = None, + cross_attention_norm_num_groups: int = 32, + added_kv_proj_dim: Optional[int] = None, + norm_num_groups: Optional[int] = None, + spatial_norm_dim: Optional[int] = None, + out_bias: bool = True, + scale_qk: bool = True, + only_cross_attention: bool = False, + eps: float = 1e-5, + rescale_output_factor: float = 1.0, + residual_connection: bool = False, + _from_deprecated_attn_block: bool = False, + processor: Optional["AttnProcessor"] = None, + out_dim: int = None, + ): + super().__init__() + self.inner_dim = out_dim if out_dim is not None else dim_head * heads + self.query_dim = query_dim + self.use_bias = bias + self.is_cross_attention = cross_attention_dim is not None + self.cross_attention_dim = cross_attention_dim if cross_attention_dim is not None else query_dim + self.upcast_attention = upcast_attention + self.upcast_softmax = upcast_softmax + self.rescale_output_factor = rescale_output_factor + self.residual_connection = residual_connection + self.dropout = dropout + self.fused_projections = False + self.out_dim = out_dim if out_dim is not None else query_dim + + # we make use of this private variable to know whether this class is loaded + # with an deprecated state dict so that we can convert it on the fly + self._from_deprecated_attn_block = _from_deprecated_attn_block + + self.scale_qk = scale_qk + self.scale = dim_head**-0.5 if self.scale_qk else 1.0 + + self.heads = out_dim // dim_head if out_dim is not None else heads + # for slice_size > 0 the attention score computation + # is split across the batch axis to save memory + # You can set slice_size with `set_attention_slice` + self.sliceable_head_dim = heads + + self.added_kv_proj_dim = added_kv_proj_dim + self.only_cross_attention = only_cross_attention + + if self.added_kv_proj_dim is None and self.only_cross_attention: + raise ValueError( + "`only_cross_attention` can only be set to True if `added_kv_proj_dim` is not None. Make sure to set either `only_cross_attention=False` or define `added_kv_proj_dim`." + ) + + if norm_num_groups is not None: + self.group_norm = nn.GroupNorm(num_channels=query_dim, num_groups=norm_num_groups, eps=eps, affine=True) + else: + self.group_norm = None + + if spatial_norm_dim is not None: + self.spatial_norm = SpatialNorm(f_channels=query_dim, zq_channels=spatial_norm_dim) + else: + self.spatial_norm = None + + if cross_attention_norm is None: + self.norm_cross = None + elif cross_attention_norm == "layer_norm": + self.norm_cross = nn.LayerNorm(self.cross_attention_dim) + elif cross_attention_norm == "group_norm": + if self.added_kv_proj_dim is not None: + # The given `encoder_hidden_states` are initially of shape + # (batch_size, seq_len, added_kv_proj_dim) before being projected + # to (batch_size, seq_len, cross_attention_dim). The norm is applied + # before the projection, so we need to use `added_kv_proj_dim` as + # the number of channels for the group norm. + norm_cross_num_channels = added_kv_proj_dim + else: + norm_cross_num_channels = self.cross_attention_dim + + self.norm_cross = nn.GroupNorm( + num_channels=norm_cross_num_channels, num_groups=cross_attention_norm_num_groups, eps=1e-5, affine=True + ) + else: + raise ValueError( + f"unknown cross_attention_norm: {cross_attention_norm}. Should be None, 'layer_norm' or 'group_norm'" + ) + + linear_cls = nn.Linear + + self.linear_cls = linear_cls + self.to_q = linear_cls(query_dim, self.inner_dim, bias=bias) + + if not self.only_cross_attention: + # only relevant for the `AddedKVProcessor` classes + self.to_k = linear_cls(self.cross_attention_dim, self.inner_dim, bias=bias) + self.to_v = linear_cls(self.cross_attention_dim, self.inner_dim, bias=bias) + else: + self.to_k = None + self.to_v = None + + if self.added_kv_proj_dim is not None: + self.add_k_proj = linear_cls(added_kv_proj_dim, self.inner_dim) + self.add_v_proj = linear_cls(added_kv_proj_dim, self.inner_dim) + + self.to_out = nn.ModuleList([]) + self.to_out.append(linear_cls(self.inner_dim, self.out_dim, bias=out_bias)) + self.to_out.append(nn.Dropout(dropout)) + + # set attention processor + # We use the AttnProcessor2_0 by default when torch 2.x is used which uses + # torch.nn.functional.scaled_dot_product_attention for native Flash/memory_efficient_attention + # but only if it has the default `scale` argument. TODO remove scale_qk check when we move to torch 2.1 + if processor is None: + processor = ( + AttnProcessor2_0() if hasattr(F, "scaled_dot_product_attention") and self.scale_qk else AttnProcessor() + ) + self.set_processor(processor) + + def set_use_memory_efficient_attention_xformers( + self, use_memory_efficient_attention_xformers: bool, attention_op: Optional[Callable] = None + ) -> None: + r""" + Set whether to use memory efficient attention from `xformers` or not. + + Args: + use_memory_efficient_attention_xformers (`bool`): + Whether to use memory efficient attention from `xformers` or not. + attention_op (`Callable`, *optional*): + The attention operation to use. Defaults to `None` which uses the default attention operation from + `xformers`. + """ + is_lora = hasattr(self, "processor") and isinstance( + self.processor, + LORA_ATTENTION_PROCESSORS, + ) + is_custom_diffusion = hasattr(self, "processor") and isinstance( + self.processor, + (CustomDiffusionAttnProcessor, CustomDiffusionXFormersAttnProcessor, CustomDiffusionAttnProcessor2_0), + ) + is_added_kv_processor = hasattr(self, "processor") and isinstance( + self.processor, + ( + AttnAddedKVProcessor, + AttnAddedKVProcessor2_0, + SlicedAttnAddedKVProcessor, + XFormersAttnAddedKVProcessor, + LoRAAttnAddedKVProcessor, + ), + ) + + if use_memory_efficient_attention_xformers: + if is_added_kv_processor and (is_lora or is_custom_diffusion): + raise NotImplementedError( + f"Memory efficient attention is currently not supported for LoRA or custom diffusion for attention processor type {self.processor}" + ) + if not is_xformers_available(): + raise ModuleNotFoundError( + ( + "Refer to https://github.com/facebookresearch/xformers for more information on how to install" + " xformers" + ), + name="xformers", + ) + elif not torch.cuda.is_available(): + raise ValueError( + "torch.cuda.is_available() should be True but is False. xformers' memory efficient attention is" + " only available for GPU " + ) + else: + try: + # Make sure we can run the memory efficient attention + _ = xformers.ops.memory_efficient_attention( + torch.randn((1, 2, 40), device="cuda"), + torch.randn((1, 2, 40), device="cuda"), + torch.randn((1, 2, 40), device="cuda"), + ) + except Exception as e: + raise e + + if is_lora: + # TODO (sayakpaul): should we throw a warning if someone wants to use the xformers + # variant when using PT 2.0 now that we have LoRAAttnProcessor2_0? + processor = LoRAXFormersAttnProcessor( + hidden_size=self.processor.hidden_size, + cross_attention_dim=self.processor.cross_attention_dim, + rank=self.processor.rank, + attention_op=attention_op, + ) + processor.load_state_dict(self.processor.state_dict()) + processor.to(self.processor.to_q_lora.up.weight.device) + elif is_custom_diffusion: + processor = CustomDiffusionXFormersAttnProcessor( + train_kv=self.processor.train_kv, + train_q_out=self.processor.train_q_out, + hidden_size=self.processor.hidden_size, + cross_attention_dim=self.processor.cross_attention_dim, + attention_op=attention_op, + ) + processor.load_state_dict(self.processor.state_dict()) + if hasattr(self.processor, "to_k_custom_diffusion"): + processor.to(self.processor.to_k_custom_diffusion.weight.device) + elif is_added_kv_processor: + # TODO(Patrick, Suraj, William) - currently xformers doesn't work for UnCLIP + # which uses this type of cross attention ONLY because the attention mask of format + # [0, ..., -10.000, ..., 0, ...,] is not supported + # throw warning + logger.info( + "Memory efficient attention with `xformers` might currently not work correctly if an attention mask is required for the attention operation." + ) + processor = XFormersAttnAddedKVProcessor(attention_op=attention_op) + else: + processor = XFormersAttnProcessor(attention_op=attention_op) + else: + if is_lora: + attn_processor_class = ( + LoRAAttnProcessor2_0 if hasattr(F, "scaled_dot_product_attention") else LoRAAttnProcessor + ) + processor = attn_processor_class( + hidden_size=self.processor.hidden_size, + cross_attention_dim=self.processor.cross_attention_dim, + rank=self.processor.rank, + ) + processor.load_state_dict(self.processor.state_dict()) + processor.to(self.processor.to_q_lora.up.weight.device) + elif is_custom_diffusion: + attn_processor_class = ( + CustomDiffusionAttnProcessor2_0 + if hasattr(F, "scaled_dot_product_attention") + else CustomDiffusionAttnProcessor + ) + processor = attn_processor_class( + train_kv=self.processor.train_kv, + train_q_out=self.processor.train_q_out, + hidden_size=self.processor.hidden_size, + cross_attention_dim=self.processor.cross_attention_dim, + ) + processor.load_state_dict(self.processor.state_dict()) + if hasattr(self.processor, "to_k_custom_diffusion"): + processor.to(self.processor.to_k_custom_diffusion.weight.device) + else: + # set attention processor + # We use the AttnProcessor2_0 by default when torch 2.x is used which uses + # torch.nn.functional.scaled_dot_product_attention for native Flash/memory_efficient_attention + # but only if it has the default `scale` argument. TODO remove scale_qk check when we move to torch 2.1 + processor = ( + AttnProcessor2_0() + if hasattr(F, "scaled_dot_product_attention") and self.scale_qk + else AttnProcessor() + ) + + self.set_processor(processor) + + def set_attention_slice(self, slice_size: int) -> None: + r""" + Set the slice size for attention computation. + + Args: + slice_size (`int`): + The slice size for attention computation. + """ + if slice_size is not None and slice_size > self.sliceable_head_dim: + raise ValueError(f"slice_size {slice_size} has to be smaller or equal to {self.sliceable_head_dim}.") + + if slice_size is not None and self.added_kv_proj_dim is not None: + processor = SlicedAttnAddedKVProcessor(slice_size) + elif slice_size is not None: + processor = SlicedAttnProcessor(slice_size) + elif self.added_kv_proj_dim is not None: + processor = AttnAddedKVProcessor() + else: + # set attention processor + # We use the AttnProcessor2_0 by default when torch 2.x is used which uses + # torch.nn.functional.scaled_dot_product_attention for native Flash/memory_efficient_attention + # but only if it has the default `scale` argument. TODO remove scale_qk check when we move to torch 2.1 + processor = ( + AttnProcessor2_0() if hasattr(F, "scaled_dot_product_attention") and self.scale_qk else AttnProcessor() + ) + + self.set_processor(processor) + + def set_processor(self, processor: "AttnProcessor") -> None: + r""" + Set the attention processor to use. + + Args: + processor (`AttnProcessor`): + The attention processor to use. + """ + # if current processor is in `self._modules` and if passed `processor` is not, we need to + # pop `processor` from `self._modules` + if ( + hasattr(self, "processor") + and isinstance(self.processor, torch.nn.Module) + and not isinstance(processor, torch.nn.Module) + ): + logger.info(f"You are removing possibly trained weights of {self.processor} with {processor}") + self._modules.pop("processor") + + self.processor = processor + + def get_processor(self, return_deprecated_lora: bool = False) -> "AttentionProcessor": + r""" + Get the attention processor in use. + + Args: + return_deprecated_lora (`bool`, *optional*, defaults to `False`): + Set to `True` to return the deprecated LoRA attention processor. + + Returns: + "AttentionProcessor": The attention processor in use. + """ + if not return_deprecated_lora: + return self.processor + + # TODO(Sayak, Patrick). The rest of the function is needed to ensure backwards compatible + # serialization format for LoRA Attention Processors. It should be deleted once the integration + # with PEFT is completed. + is_lora_activated = { + name: module.lora_layer is not None + for name, module in self.named_modules() + if hasattr(module, "lora_layer") + } + + # 1. if no layer has a LoRA activated we can return the processor as usual + if not any(is_lora_activated.values()): + return self.processor + + # If doesn't apply LoRA do `add_k_proj` or `add_v_proj` + is_lora_activated.pop("add_k_proj", None) + is_lora_activated.pop("add_v_proj", None) + # 2. else it is not posssible that only some layers have LoRA activated + if not all(is_lora_activated.values()): + raise ValueError( + f"Make sure that either all layers or no layers have LoRA activated, but have {is_lora_activated}" + ) + + # 3. And we need to merge the current LoRA layers into the corresponding LoRA attention processor + non_lora_processor_cls_name = self.processor.__class__.__name__ + lora_processor_cls = getattr(import_module(__name__), "LoRA" + non_lora_processor_cls_name) + + hidden_size = self.inner_dim + + # now create a LoRA attention processor from the LoRA layers + if lora_processor_cls in [LoRAAttnProcessor, LoRAAttnProcessor2_0, LoRAXFormersAttnProcessor]: + kwargs = { + "cross_attention_dim": self.cross_attention_dim, + "rank": self.to_q.lora_layer.rank, + "network_alpha": self.to_q.lora_layer.network_alpha, + "q_rank": self.to_q.lora_layer.rank, + "q_hidden_size": self.to_q.lora_layer.out_features, + "k_rank": self.to_k.lora_layer.rank, + "k_hidden_size": self.to_k.lora_layer.out_features, + "v_rank": self.to_v.lora_layer.rank, + "v_hidden_size": self.to_v.lora_layer.out_features, + "out_rank": self.to_out[0].lora_layer.rank, + "out_hidden_size": self.to_out[0].lora_layer.out_features, + } + + if hasattr(self.processor, "attention_op"): + kwargs["attention_op"] = self.processor.attention_op + + lora_processor = lora_processor_cls(hidden_size, **kwargs) + lora_processor.to_q_lora.load_state_dict(self.to_q.lora_layer.state_dict()) + lora_processor.to_k_lora.load_state_dict(self.to_k.lora_layer.state_dict()) + lora_processor.to_v_lora.load_state_dict(self.to_v.lora_layer.state_dict()) + lora_processor.to_out_lora.load_state_dict(self.to_out[0].lora_layer.state_dict()) + elif lora_processor_cls == LoRAAttnAddedKVProcessor: + lora_processor = lora_processor_cls( + hidden_size, + cross_attention_dim=self.add_k_proj.weight.shape[0], + rank=self.to_q.lora_layer.rank, + network_alpha=self.to_q.lora_layer.network_alpha, + ) + lora_processor.to_q_lora.load_state_dict(self.to_q.lora_layer.state_dict()) + lora_processor.to_k_lora.load_state_dict(self.to_k.lora_layer.state_dict()) + lora_processor.to_v_lora.load_state_dict(self.to_v.lora_layer.state_dict()) + lora_processor.to_out_lora.load_state_dict(self.to_out[0].lora_layer.state_dict()) + + # only save if used + if self.add_k_proj.lora_layer is not None: + lora_processor.add_k_proj_lora.load_state_dict(self.add_k_proj.lora_layer.state_dict()) + lora_processor.add_v_proj_lora.load_state_dict(self.add_v_proj.lora_layer.state_dict()) + else: + lora_processor.add_k_proj_lora = None + lora_processor.add_v_proj_lora = None + else: + raise ValueError(f"{lora_processor_cls} does not exist.") + + return lora_processor + + def forward( + self, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + **cross_attention_kwargs, + ) -> torch.Tensor: + r""" + The forward method of the `Attention` class. + + Args: + hidden_states (`torch.Tensor`): + The hidden states of the query. + encoder_hidden_states (`torch.Tensor`, *optional*): + The hidden states of the encoder. + attention_mask (`torch.Tensor`, *optional*): + The attention mask to use. If `None`, no mask is applied. + **cross_attention_kwargs: + Additional keyword arguments to pass along to the cross attention. + + Returns: + `torch.Tensor`: The output of the attention layer. + """ + # The `Attention` class can call different attention processors / attention functions + # here we simply pass along all tensors to the selected processor class + # For standard processors that are defined here, `**cross_attention_kwargs` is empty + + attn_parameters = set(inspect.signature(self.processor.__call__).parameters.keys()) + unused_kwargs = [k for k, _ in cross_attention_kwargs.items() if k not in attn_parameters] + if len(unused_kwargs) > 0: + logger.warning( + f"cross_attention_kwargs {unused_kwargs} are not expected by {self.processor.__class__.__name__} and will be ignored." + ) + cross_attention_kwargs = {k: w for k, w in cross_attention_kwargs.items() if k in attn_parameters} + + return self.processor( + self, + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + + def batch_to_head_dim(self, tensor: torch.Tensor) -> torch.Tensor: + r""" + Reshape the tensor from `[batch_size, seq_len, dim]` to `[batch_size // heads, seq_len, dim * heads]`. `heads` + is the number of heads initialized while constructing the `Attention` class. + + Args: + tensor (`torch.Tensor`): The tensor to reshape. + + Returns: + `torch.Tensor`: The reshaped tensor. + """ + head_size = self.heads + batch_size, seq_len, dim = tensor.shape + tensor = tensor.reshape(batch_size // head_size, head_size, seq_len, dim) + tensor = tensor.permute(0, 2, 1, 3).reshape(batch_size // head_size, seq_len, dim * head_size) + return tensor + + def head_to_batch_dim(self, tensor: torch.Tensor, out_dim: int = 3) -> torch.Tensor: + r""" + Reshape the tensor from `[batch_size, seq_len, dim]` to `[batch_size, seq_len, heads, dim // heads]` `heads` is + the number of heads initialized while constructing the `Attention` class. + + Args: + tensor (`torch.Tensor`): The tensor to reshape. + out_dim (`int`, *optional*, defaults to `3`): The output dimension of the tensor. If `3`, the tensor is + reshaped to `[batch_size * heads, seq_len, dim // heads]`. + + Returns: + `torch.Tensor`: The reshaped tensor. + """ + head_size = self.heads + if tensor.ndim == 3: + batch_size, seq_len, dim = tensor.shape + extra_dim = 1 + else: + batch_size, extra_dim, seq_len, dim = tensor.shape + tensor = tensor.reshape(batch_size, seq_len * extra_dim, head_size, dim // head_size) + tensor = tensor.permute(0, 2, 1, 3) + + if out_dim == 3: + tensor = tensor.reshape(batch_size * head_size, seq_len * extra_dim, dim // head_size) + + return tensor + + def get_attention_scores( + self, query: torch.Tensor, key: torch.Tensor, attention_mask: torch.Tensor = None + ) -> torch.Tensor: + r""" + Compute the attention scores. + + Args: + query (`torch.Tensor`): The query tensor. + key (`torch.Tensor`): The key tensor. + attention_mask (`torch.Tensor`, *optional*): The attention mask to use. If `None`, no mask is applied. + + Returns: + `torch.Tensor`: The attention probabilities/scores. + """ + dtype = query.dtype + if self.upcast_attention: + query = query.float() + key = key.float() + + if attention_mask is None: + baddbmm_input = torch.empty( + query.shape[0], query.shape[1], key.shape[1], dtype=query.dtype, device=query.device + ) + beta = 0 + else: + baddbmm_input = attention_mask + beta = 1 + + attention_scores = torch.baddbmm( + baddbmm_input, + query, + key.transpose(-1, -2), + beta=beta, + alpha=self.scale, + ) + del baddbmm_input + + if self.upcast_softmax: + attention_scores = attention_scores.float() + + attention_probs = attention_scores.softmax(dim=-1) + del attention_scores + + attention_probs = attention_probs.to(dtype) + + return attention_probs + + def prepare_attention_mask( + self, attention_mask: torch.Tensor, target_length: int, batch_size: int, out_dim: int = 3 + ) -> torch.Tensor: + r""" + Prepare the attention mask for the attention computation. + + Args: + attention_mask (`torch.Tensor`): + The attention mask to prepare. + target_length (`int`): + The target length of the attention mask. This is the length of the attention mask after padding. + batch_size (`int`): + The batch size, which is used to repeat the attention mask. + out_dim (`int`, *optional*, defaults to `3`): + The output dimension of the attention mask. Can be either `3` or `4`. + + Returns: + `torch.Tensor`: The prepared attention mask. + """ + head_size = self.heads + if attention_mask is None: + return attention_mask + + current_length: int = attention_mask.shape[-1] + if current_length != target_length: + if attention_mask.device.type == "mps": + # HACK: MPS: Does not support padding by greater than dimension of input tensor. + # Instead, we can manually construct the padding tensor. + padding_shape = (attention_mask.shape[0], attention_mask.shape[1], target_length) + padding = torch.zeros(padding_shape, dtype=attention_mask.dtype, device=attention_mask.device) + attention_mask = torch.cat([attention_mask, padding], dim=2) + else: + # TODO: for pipelines such as stable-diffusion, padding cross-attn mask: + # we want to instead pad by (0, remaining_length), where remaining_length is: + # remaining_length: int = target_length - current_length + # TODO: re-enable tests/models/test_models_unet_2d_condition.py#test_model_xattn_padding + attention_mask = F.pad(attention_mask, (0, target_length), value=0.0) + + if out_dim == 3: + if attention_mask.shape[0] < batch_size * head_size: + attention_mask = attention_mask.repeat_interleave(head_size, dim=0) + elif out_dim == 4: + attention_mask = attention_mask.unsqueeze(1) + attention_mask = attention_mask.repeat_interleave(head_size, dim=1) + + return attention_mask + + def norm_encoder_hidden_states(self, encoder_hidden_states: torch.Tensor) -> torch.Tensor: + r""" + Normalize the encoder hidden states. Requires `self.norm_cross` to be specified when constructing the + `Attention` class. + + Args: + encoder_hidden_states (`torch.Tensor`): Hidden states of the encoder. + + Returns: + `torch.Tensor`: The normalized encoder hidden states. + """ + assert self.norm_cross is not None, "self.norm_cross must be defined to call self.norm_encoder_hidden_states" + + if isinstance(self.norm_cross, nn.LayerNorm): + encoder_hidden_states = self.norm_cross(encoder_hidden_states) + elif isinstance(self.norm_cross, nn.GroupNorm): + # Group norm norms along the channels dimension and expects + # input to be in the shape of (N, C, *). In this case, we want + # to norm along the hidden dimension, so we need to move + # (batch_size, sequence_length, hidden_size) -> + # (batch_size, hidden_size, sequence_length) + encoder_hidden_states = encoder_hidden_states.transpose(1, 2) + encoder_hidden_states = self.norm_cross(encoder_hidden_states) + encoder_hidden_states = encoder_hidden_states.transpose(1, 2) + else: + assert False + + return encoder_hidden_states + + @torch.no_grad() + def fuse_projections(self, fuse=True): + device = self.to_q.weight.data.device + dtype = self.to_q.weight.data.dtype + + if not self.is_cross_attention: + # fetch weight matrices. + concatenated_weights = torch.cat([self.to_q.weight.data, self.to_k.weight.data, self.to_v.weight.data]) + in_features = concatenated_weights.shape[1] + out_features = concatenated_weights.shape[0] + + # create a new single projection layer and copy over the weights. + self.to_qkv = self.linear_cls(in_features, out_features, bias=self.use_bias, device=device, dtype=dtype) + self.to_qkv.weight.copy_(concatenated_weights) + if self.use_bias: + concatenated_bias = torch.cat([self.to_q.bias.data, self.to_k.bias.data, self.to_v.bias.data]) + self.to_qkv.bias.copy_(concatenated_bias) + + else: + concatenated_weights = torch.cat([self.to_k.weight.data, self.to_v.weight.data]) + in_features = concatenated_weights.shape[1] + out_features = concatenated_weights.shape[0] + + self.to_kv = self.linear_cls(in_features, out_features, bias=self.use_bias, device=device, dtype=dtype) + self.to_kv.weight.copy_(concatenated_weights) + if self.use_bias: + concatenated_bias = torch.cat([self.to_k.bias.data, self.to_v.bias.data]) + self.to_kv.bias.copy_(concatenated_bias) + + self.fused_projections = fuse + + +class AttnProcessor: + r""" + Default processor for performing attention-related computations. + """ + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + temb: Optional[torch.FloatTensor] = None, + *args, + **kwargs, + ) -> torch.Tensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + residual = hidden_states + + if attn.spatial_norm is not None: + hidden_states = attn.spatial_norm(hidden_states, temb) + + input_ndim = hidden_states.ndim + + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + if attn.group_norm is not None: + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + if input_ndim == 4: + hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) + + if attn.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / attn.rescale_output_factor + + return hidden_states + + +class CustomDiffusionAttnProcessor(nn.Module): + r""" + Processor for implementing attention for the Custom Diffusion method. + + Args: + train_kv (`bool`, defaults to `True`): + Whether to newly train the key and value matrices corresponding to the text features. + train_q_out (`bool`, defaults to `True`): + Whether to newly train query matrices corresponding to the latent image features. + hidden_size (`int`, *optional*, defaults to `None`): + The hidden size of the attention layer. + cross_attention_dim (`int`, *optional*, defaults to `None`): + The number of channels in the `encoder_hidden_states`. + out_bias (`bool`, defaults to `True`): + Whether to include the bias parameter in `train_q_out`. + dropout (`float`, *optional*, defaults to 0.0): + The dropout probability to use. + """ + + def __init__( + self, + train_kv: bool = True, + train_q_out: bool = True, + hidden_size: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + out_bias: bool = True, + dropout: float = 0.0, + ): + super().__init__() + self.train_kv = train_kv + self.train_q_out = train_q_out + + self.hidden_size = hidden_size + self.cross_attention_dim = cross_attention_dim + + # `_custom_diffusion` id for easy serialization and loading. + if self.train_kv: + self.to_k_custom_diffusion = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False) + self.to_v_custom_diffusion = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False) + if self.train_q_out: + self.to_q_custom_diffusion = nn.Linear(hidden_size, hidden_size, bias=False) + self.to_out_custom_diffusion = nn.ModuleList([]) + self.to_out_custom_diffusion.append(nn.Linear(hidden_size, hidden_size, bias=out_bias)) + self.to_out_custom_diffusion.append(nn.Dropout(dropout)) + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.Tensor: + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + if self.train_q_out: + query = self.to_q_custom_diffusion(hidden_states).to(attn.to_q.weight.dtype) + else: + query = attn.to_q(hidden_states.to(attn.to_q.weight.dtype)) + + if encoder_hidden_states is None: + crossattn = False + encoder_hidden_states = hidden_states + else: + crossattn = True + if attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + if self.train_kv: + key = self.to_k_custom_diffusion(encoder_hidden_states.to(self.to_k_custom_diffusion.weight.dtype)) + value = self.to_v_custom_diffusion(encoder_hidden_states.to(self.to_v_custom_diffusion.weight.dtype)) + key = key.to(attn.to_q.weight.dtype) + value = value.to(attn.to_q.weight.dtype) + else: + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + if crossattn: + detach = torch.ones_like(key) + detach[:, :1, :] = detach[:, :1, :] * 0.0 + key = detach * key + (1 - detach) * key.detach() + value = detach * value + (1 - detach) * value.detach() + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + if self.train_q_out: + # linear proj + hidden_states = self.to_out_custom_diffusion[0](hidden_states) + # dropout + hidden_states = self.to_out_custom_diffusion[1](hidden_states) + else: + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +class AttnAddedKVProcessor: + r""" + Processor for performing attention-related computations with extra learnable key and value matrices for the text + encoder. + """ + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + *args, + **kwargs, + ) -> torch.Tensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + residual = hidden_states + + hidden_states = hidden_states.view(hidden_states.shape[0], hidden_states.shape[1], -1).transpose(1, 2) + batch_size, sequence_length, _ = hidden_states.shape + + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + query = attn.head_to_batch_dim(query) + + encoder_hidden_states_key_proj = attn.add_k_proj(encoder_hidden_states) + encoder_hidden_states_value_proj = attn.add_v_proj(encoder_hidden_states) + encoder_hidden_states_key_proj = attn.head_to_batch_dim(encoder_hidden_states_key_proj) + encoder_hidden_states_value_proj = attn.head_to_batch_dim(encoder_hidden_states_value_proj) + + if not attn.only_cross_attention: + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + key = torch.cat([encoder_hidden_states_key_proj, key], dim=1) + value = torch.cat([encoder_hidden_states_value_proj, value], dim=1) + else: + key = encoder_hidden_states_key_proj + value = encoder_hidden_states_value_proj + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + hidden_states = hidden_states.transpose(-1, -2).reshape(residual.shape) + hidden_states = hidden_states + residual + + return hidden_states + + +class AttnAddedKVProcessor2_0: + r""" + Processor for performing scaled dot-product attention (enabled by default if you're using PyTorch 2.0), with extra + learnable key and value matrices for the text encoder. + """ + + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError( + "AttnAddedKVProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0." + ) + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + *args, + **kwargs, + ) -> torch.Tensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + residual = hidden_states + + hidden_states = hidden_states.view(hidden_states.shape[0], hidden_states.shape[1], -1).transpose(1, 2) + batch_size, sequence_length, _ = hidden_states.shape + + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size, out_dim=4) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + query = attn.head_to_batch_dim(query, out_dim=4) + + encoder_hidden_states_key_proj = attn.add_k_proj(encoder_hidden_states) + encoder_hidden_states_value_proj = attn.add_v_proj(encoder_hidden_states) + encoder_hidden_states_key_proj = attn.head_to_batch_dim(encoder_hidden_states_key_proj, out_dim=4) + encoder_hidden_states_value_proj = attn.head_to_batch_dim(encoder_hidden_states_value_proj, out_dim=4) + + if not attn.only_cross_attention: + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + key = attn.head_to_batch_dim(key, out_dim=4) + value = attn.head_to_batch_dim(value, out_dim=4) + key = torch.cat([encoder_hidden_states_key_proj, key], dim=2) + value = torch.cat([encoder_hidden_states_value_proj, value], dim=2) + else: + key = encoder_hidden_states_key_proj + value = encoder_hidden_states_value_proj + + # the output of sdp = (batch, num_heads, seq_len, head_dim) + # TODO: add support for attn.scale when we move to Torch 2.1 + hidden_states = F.scaled_dot_product_attention( + query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False + ) + hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, residual.shape[1]) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + hidden_states = hidden_states.transpose(-1, -2).reshape(residual.shape) + hidden_states = hidden_states + residual + + return hidden_states + + +class XFormersAttnAddedKVProcessor: + r""" + Processor for implementing memory efficient attention using xFormers. + + Args: + attention_op (`Callable`, *optional*, defaults to `None`): + The base + [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to + use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best + operator. + """ + + def __init__(self, attention_op: Optional[Callable] = None): + self.attention_op = attention_op + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.Tensor: + residual = hidden_states + hidden_states = hidden_states.view(hidden_states.shape[0], hidden_states.shape[1], -1).transpose(1, 2) + batch_size, sequence_length, _ = hidden_states.shape + + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + query = attn.head_to_batch_dim(query) + + encoder_hidden_states_key_proj = attn.add_k_proj(encoder_hidden_states) + encoder_hidden_states_value_proj = attn.add_v_proj(encoder_hidden_states) + encoder_hidden_states_key_proj = attn.head_to_batch_dim(encoder_hidden_states_key_proj) + encoder_hidden_states_value_proj = attn.head_to_batch_dim(encoder_hidden_states_value_proj) + + if not attn.only_cross_attention: + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + key = torch.cat([encoder_hidden_states_key_proj, key], dim=1) + value = torch.cat([encoder_hidden_states_value_proj, value], dim=1) + else: + key = encoder_hidden_states_key_proj + value = encoder_hidden_states_value_proj + + hidden_states = xformers.ops.memory_efficient_attention( + query, key, value, attn_bias=attention_mask, op=self.attention_op, scale=attn.scale + ) + hidden_states = hidden_states.to(query.dtype) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + hidden_states = hidden_states.transpose(-1, -2).reshape(residual.shape) + hidden_states = hidden_states + residual + + return hidden_states + + +class XFormersAttnProcessor: + r""" + Processor for implementing memory efficient attention using xFormers. + + Args: + attention_op (`Callable`, *optional*, defaults to `None`): + The base + [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to + use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best + operator. + """ + + def __init__(self, attention_op: Optional[Callable] = None): + self.attention_op = attention_op + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + temb: Optional[torch.FloatTensor] = None, + *args, + **kwargs, + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + residual = hidden_states + + if attn.spatial_norm is not None: + hidden_states = attn.spatial_norm(hidden_states, temb) + + input_ndim = hidden_states.ndim + + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + batch_size, key_tokens, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + + attention_mask = attn.prepare_attention_mask(attention_mask, key_tokens, batch_size) + if attention_mask is not None: + # expand our mask's singleton query_tokens dimension: + # [batch*heads, 1, key_tokens] -> + # [batch*heads, query_tokens, key_tokens] + # so that it can be added as a bias onto the attention scores that xformers computes: + # [batch*heads, query_tokens, key_tokens] + # we do this explicitly because xformers doesn't broadcast the singleton dimension for us. + _, query_tokens, _ = hidden_states.shape + attention_mask = attention_mask.expand(-1, query_tokens, -1) + + if attn.group_norm is not None: + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + query = attn.head_to_batch_dim(query).contiguous() + key = attn.head_to_batch_dim(key).contiguous() + value = attn.head_to_batch_dim(value).contiguous() + + hidden_states = xformers.ops.memory_efficient_attention( + query, key, value, attn_bias=attention_mask, op=self.attention_op, scale=attn.scale + ) + hidden_states = hidden_states.to(query.dtype) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + if input_ndim == 4: + hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) + + if attn.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / attn.rescale_output_factor + + return hidden_states + + +class AttnProcessor2_0: + r""" + Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). + """ + + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.") + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + temb: Optional[torch.FloatTensor] = None, + *args, + **kwargs, + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + residual = hidden_states + if attn.spatial_norm is not None: + hidden_states = attn.spatial_norm(hidden_states, temb) + + input_ndim = hidden_states.ndim + + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + + if attention_mask is not None: + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + # scaled_dot_product_attention expects attention_mask shape to be + # (batch, heads, source_length, target_length) + attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) + + if attn.group_norm is not None: + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + inner_dim = key.shape[-1] + head_dim = inner_dim // attn.heads + + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # the output of sdp = (batch, num_heads, seq_len, head_dim) + # TODO: add support for attn.scale when we move to Torch 2.1 + hidden_states = F.scaled_dot_product_attention( + query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False + ) + + hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + hidden_states = hidden_states.to(query.dtype) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + if input_ndim == 4: + hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) + + if attn.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / attn.rescale_output_factor + + return hidden_states + + +class FusedAttnProcessor2_0: + r""" + Processor for implementing scaled dot-product attention (enabled by default if you're using PyTorch 2.0). + It uses fused projection layers. For self-attention modules, all projection matrices (i.e., query, + key, value) are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is currently 🧪 experimental in nature and can change in future. + + + """ + + def __init__(self): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError( + "FusedAttnProcessor2_0 requires at least PyTorch 2.0, to use it. Please upgrade PyTorch to > 2.0." + ) + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + temb: Optional[torch.FloatTensor] = None, + *args, + **kwargs, + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + residual = hidden_states + if attn.spatial_norm is not None: + hidden_states = attn.spatial_norm(hidden_states, temb) + + input_ndim = hidden_states.ndim + + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + + if attention_mask is not None: + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + # scaled_dot_product_attention expects attention_mask shape to be + # (batch, heads, source_length, target_length) + attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) + + if attn.group_norm is not None: + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + if encoder_hidden_states is None: + qkv = attn.to_qkv(hidden_states) + split_size = qkv.shape[-1] // 3 + query, key, value = torch.split(qkv, split_size, dim=-1) + else: + if attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + query = attn.to_q(hidden_states) + + kv = attn.to_kv(encoder_hidden_states) + split_size = kv.shape[-1] // 2 + key, value = torch.split(kv, split_size, dim=-1) + + inner_dim = key.shape[-1] + head_dim = inner_dim // attn.heads + + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # the output of sdp = (batch, num_heads, seq_len, head_dim) + # TODO: add support for attn.scale when we move to Torch 2.1 + hidden_states = F.scaled_dot_product_attention( + query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False + ) + + hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + hidden_states = hidden_states.to(query.dtype) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + if input_ndim == 4: + hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) + + if attn.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / attn.rescale_output_factor + + return hidden_states + + +class CustomDiffusionXFormersAttnProcessor(nn.Module): + r""" + Processor for implementing memory efficient attention using xFormers for the Custom Diffusion method. + + Args: + train_kv (`bool`, defaults to `True`): + Whether to newly train the key and value matrices corresponding to the text features. + train_q_out (`bool`, defaults to `True`): + Whether to newly train query matrices corresponding to the latent image features. + hidden_size (`int`, *optional*, defaults to `None`): + The hidden size of the attention layer. + cross_attention_dim (`int`, *optional*, defaults to `None`): + The number of channels in the `encoder_hidden_states`. + out_bias (`bool`, defaults to `True`): + Whether to include the bias parameter in `train_q_out`. + dropout (`float`, *optional*, defaults to 0.0): + The dropout probability to use. + attention_op (`Callable`, *optional*, defaults to `None`): + The base + [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to use + as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best operator. + """ + + def __init__( + self, + train_kv: bool = True, + train_q_out: bool = False, + hidden_size: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + out_bias: bool = True, + dropout: float = 0.0, + attention_op: Optional[Callable] = None, + ): + super().__init__() + self.train_kv = train_kv + self.train_q_out = train_q_out + + self.hidden_size = hidden_size + self.cross_attention_dim = cross_attention_dim + self.attention_op = attention_op + + # `_custom_diffusion` id for easy serialization and loading. + if self.train_kv: + self.to_k_custom_diffusion = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False) + self.to_v_custom_diffusion = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False) + if self.train_q_out: + self.to_q_custom_diffusion = nn.Linear(hidden_size, hidden_size, bias=False) + self.to_out_custom_diffusion = nn.ModuleList([]) + self.to_out_custom_diffusion.append(nn.Linear(hidden_size, hidden_size, bias=out_bias)) + self.to_out_custom_diffusion.append(nn.Dropout(dropout)) + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + if self.train_q_out: + query = self.to_q_custom_diffusion(hidden_states).to(attn.to_q.weight.dtype) + else: + query = attn.to_q(hidden_states.to(attn.to_q.weight.dtype)) + + if encoder_hidden_states is None: + crossattn = False + encoder_hidden_states = hidden_states + else: + crossattn = True + if attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + if self.train_kv: + key = self.to_k_custom_diffusion(encoder_hidden_states.to(self.to_k_custom_diffusion.weight.dtype)) + value = self.to_v_custom_diffusion(encoder_hidden_states.to(self.to_v_custom_diffusion.weight.dtype)) + key = key.to(attn.to_q.weight.dtype) + value = value.to(attn.to_q.weight.dtype) + else: + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + if crossattn: + detach = torch.ones_like(key) + detach[:, :1, :] = detach[:, :1, :] * 0.0 + key = detach * key + (1 - detach) * key.detach() + value = detach * value + (1 - detach) * value.detach() + + query = attn.head_to_batch_dim(query).contiguous() + key = attn.head_to_batch_dim(key).contiguous() + value = attn.head_to_batch_dim(value).contiguous() + + hidden_states = xformers.ops.memory_efficient_attention( + query, key, value, attn_bias=attention_mask, op=self.attention_op, scale=attn.scale + ) + hidden_states = hidden_states.to(query.dtype) + hidden_states = attn.batch_to_head_dim(hidden_states) + + if self.train_q_out: + # linear proj + hidden_states = self.to_out_custom_diffusion[0](hidden_states) + # dropout + hidden_states = self.to_out_custom_diffusion[1](hidden_states) + else: + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +class CustomDiffusionAttnProcessor2_0(nn.Module): + r""" + Processor for implementing attention for the Custom Diffusion method using PyTorch 2.0’s memory-efficient scaled + dot-product attention. + + Args: + train_kv (`bool`, defaults to `True`): + Whether to newly train the key and value matrices corresponding to the text features. + train_q_out (`bool`, defaults to `True`): + Whether to newly train query matrices corresponding to the latent image features. + hidden_size (`int`, *optional*, defaults to `None`): + The hidden size of the attention layer. + cross_attention_dim (`int`, *optional*, defaults to `None`): + The number of channels in the `encoder_hidden_states`. + out_bias (`bool`, defaults to `True`): + Whether to include the bias parameter in `train_q_out`. + dropout (`float`, *optional*, defaults to 0.0): + The dropout probability to use. + """ + + def __init__( + self, + train_kv: bool = True, + train_q_out: bool = True, + hidden_size: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + out_bias: bool = True, + dropout: float = 0.0, + ): + super().__init__() + self.train_kv = train_kv + self.train_q_out = train_q_out + + self.hidden_size = hidden_size + self.cross_attention_dim = cross_attention_dim + + # `_custom_diffusion` id for easy serialization and loading. + if self.train_kv: + self.to_k_custom_diffusion = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False) + self.to_v_custom_diffusion = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=False) + if self.train_q_out: + self.to_q_custom_diffusion = nn.Linear(hidden_size, hidden_size, bias=False) + self.to_out_custom_diffusion = nn.ModuleList([]) + self.to_out_custom_diffusion.append(nn.Linear(hidden_size, hidden_size, bias=out_bias)) + self.to_out_custom_diffusion.append(nn.Dropout(dropout)) + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + if self.train_q_out: + query = self.to_q_custom_diffusion(hidden_states) + else: + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + crossattn = False + encoder_hidden_states = hidden_states + else: + crossattn = True + if attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + if self.train_kv: + key = self.to_k_custom_diffusion(encoder_hidden_states.to(self.to_k_custom_diffusion.weight.dtype)) + value = self.to_v_custom_diffusion(encoder_hidden_states.to(self.to_v_custom_diffusion.weight.dtype)) + key = key.to(attn.to_q.weight.dtype) + value = value.to(attn.to_q.weight.dtype) + + else: + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + if crossattn: + detach = torch.ones_like(key) + detach[:, :1, :] = detach[:, :1, :] * 0.0 + key = detach * key + (1 - detach) * key.detach() + value = detach * value + (1 - detach) * value.detach() + + inner_dim = hidden_states.shape[-1] + + head_dim = inner_dim // attn.heads + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # the output of sdp = (batch, num_heads, seq_len, head_dim) + # TODO: add support for attn.scale when we move to Torch 2.1 + hidden_states = F.scaled_dot_product_attention( + query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False + ) + + hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + hidden_states = hidden_states.to(query.dtype) + + if self.train_q_out: + # linear proj + hidden_states = self.to_out_custom_diffusion[0](hidden_states) + # dropout + hidden_states = self.to_out_custom_diffusion[1](hidden_states) + else: + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +class SlicedAttnProcessor: + r""" + Processor for implementing sliced attention. + + Args: + slice_size (`int`, *optional*): + The number of steps to compute attention. Uses as many slices as `attention_head_dim // slice_size`, and + `attention_head_dim` must be a multiple of the `slice_size`. + """ + + def __init__(self, slice_size: int): + self.slice_size = slice_size + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + residual = hidden_states + + input_ndim = hidden_states.ndim + + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + if attn.group_norm is not None: + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + dim = query.shape[-1] + query = attn.head_to_batch_dim(query) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + batch_size_attention, query_tokens, _ = query.shape + hidden_states = torch.zeros( + (batch_size_attention, query_tokens, dim // attn.heads), device=query.device, dtype=query.dtype + ) + + for i in range(batch_size_attention // self.slice_size): + start_idx = i * self.slice_size + end_idx = (i + 1) * self.slice_size + + query_slice = query[start_idx:end_idx] + key_slice = key[start_idx:end_idx] + attn_mask_slice = attention_mask[start_idx:end_idx] if attention_mask is not None else None + + attn_slice = attn.get_attention_scores(query_slice, key_slice, attn_mask_slice) + + attn_slice = torch.bmm(attn_slice, value[start_idx:end_idx]) + + hidden_states[start_idx:end_idx] = attn_slice + + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + if input_ndim == 4: + hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) + + if attn.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / attn.rescale_output_factor + + return hidden_states + + +class SlicedAttnAddedKVProcessor: + r""" + Processor for implementing sliced attention with extra learnable key and value matrices for the text encoder. + + Args: + slice_size (`int`, *optional*): + The number of steps to compute attention. Uses as many slices as `attention_head_dim // slice_size`, and + `attention_head_dim` must be a multiple of the `slice_size`. + """ + + def __init__(self, slice_size): + self.slice_size = slice_size + + def __call__( + self, + attn: "Attention", + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + temb: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + residual = hidden_states + + if attn.spatial_norm is not None: + hidden_states = attn.spatial_norm(hidden_states, temb) + + hidden_states = hidden_states.view(hidden_states.shape[0], hidden_states.shape[1], -1).transpose(1, 2) + + batch_size, sequence_length, _ = hidden_states.shape + + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + dim = query.shape[-1] + query = attn.head_to_batch_dim(query) + + encoder_hidden_states_key_proj = attn.add_k_proj(encoder_hidden_states) + encoder_hidden_states_value_proj = attn.add_v_proj(encoder_hidden_states) + + encoder_hidden_states_key_proj = attn.head_to_batch_dim(encoder_hidden_states_key_proj) + encoder_hidden_states_value_proj = attn.head_to_batch_dim(encoder_hidden_states_value_proj) + + if not attn.only_cross_attention: + key = attn.to_k(hidden_states) + value = attn.to_v(hidden_states) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + key = torch.cat([encoder_hidden_states_key_proj, key], dim=1) + value = torch.cat([encoder_hidden_states_value_proj, value], dim=1) + else: + key = encoder_hidden_states_key_proj + value = encoder_hidden_states_value_proj + + batch_size_attention, query_tokens, _ = query.shape + hidden_states = torch.zeros( + (batch_size_attention, query_tokens, dim // attn.heads), device=query.device, dtype=query.dtype + ) + + for i in range(batch_size_attention // self.slice_size): + start_idx = i * self.slice_size + end_idx = (i + 1) * self.slice_size + + query_slice = query[start_idx:end_idx] + key_slice = key[start_idx:end_idx] + attn_mask_slice = attention_mask[start_idx:end_idx] if attention_mask is not None else None + + attn_slice = attn.get_attention_scores(query_slice, key_slice, attn_mask_slice) + + attn_slice = torch.bmm(attn_slice, value[start_idx:end_idx]) + + hidden_states[start_idx:end_idx] = attn_slice + + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + hidden_states = hidden_states.transpose(-1, -2).reshape(residual.shape) + hidden_states = hidden_states + residual + + return hidden_states + + +class SpatialNorm(nn.Module): + """ + Spatially conditioned normalization as defined in https://arxiv.org/abs/2209.09002. + + Args: + f_channels (`int`): + The number of channels for input to group normalization layer, and output of the spatial norm layer. + zq_channels (`int`): + The number of channels for the quantized vector as described in the paper. + """ + + def __init__( + self, + f_channels: int, + zq_channels: int, + ): + super().__init__() + self.norm_layer = nn.GroupNorm(num_channels=f_channels, num_groups=32, eps=1e-6, affine=True) + self.conv_y = nn.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0) + self.conv_b = nn.Conv2d(zq_channels, f_channels, kernel_size=1, stride=1, padding=0) + + def forward(self, f: torch.FloatTensor, zq: torch.FloatTensor) -> torch.FloatTensor: + f_size = f.shape[-2:] + zq = F.interpolate(zq, size=f_size, mode="nearest") + norm_f = self.norm_layer(f) + new_f = norm_f * self.conv_y(zq) + self.conv_b(zq) + return new_f + + +class LoRAAttnProcessor(nn.Module): + def __init__( + self, + hidden_size: int, + cross_attention_dim: Optional[int] = None, + rank: int = 4, + network_alpha: Optional[int] = None, + **kwargs, + ): + deprecation_message = "Using LoRAAttnProcessor is deprecated. Please use the PEFT backend for all things LoRA. You can install PEFT by running `pip install peft`." + deprecate("LoRAAttnProcessor", "0.30.0", deprecation_message, standard_warn=False) + + super().__init__() + + self.hidden_size = hidden_size + self.cross_attention_dim = cross_attention_dim + self.rank = rank + + q_rank = kwargs.pop("q_rank", None) + q_hidden_size = kwargs.pop("q_hidden_size", None) + q_rank = q_rank if q_rank is not None else rank + q_hidden_size = q_hidden_size if q_hidden_size is not None else hidden_size + + v_rank = kwargs.pop("v_rank", None) + v_hidden_size = kwargs.pop("v_hidden_size", None) + v_rank = v_rank if v_rank is not None else rank + v_hidden_size = v_hidden_size if v_hidden_size is not None else hidden_size + + out_rank = kwargs.pop("out_rank", None) + out_hidden_size = kwargs.pop("out_hidden_size", None) + out_rank = out_rank if out_rank is not None else rank + out_hidden_size = out_hidden_size if out_hidden_size is not None else hidden_size + + self.to_q_lora = LoRALinearLayer(q_hidden_size, q_hidden_size, q_rank, network_alpha) + self.to_k_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha) + self.to_v_lora = LoRALinearLayer(cross_attention_dim or v_hidden_size, v_hidden_size, v_rank, network_alpha) + self.to_out_lora = LoRALinearLayer(out_hidden_size, out_hidden_size, out_rank, network_alpha) + + def __call__(self, attn: Attention, hidden_states: torch.FloatTensor, **kwargs) -> torch.FloatTensor: + self_cls_name = self.__class__.__name__ + deprecate( + self_cls_name, + "0.26.0", + ( + f"Make sure use {self_cls_name[4:]} instead by setting" + "LoRA layers to `self.{to_q,to_k,to_v,to_out[0]}.lora_layer` respectively. This will be done automatically when using" + " `LoraLoaderMixin.load_lora_weights`" + ), + ) + attn.to_q.lora_layer = self.to_q_lora.to(hidden_states.device) + attn.to_k.lora_layer = self.to_k_lora.to(hidden_states.device) + attn.to_v.lora_layer = self.to_v_lora.to(hidden_states.device) + attn.to_out[0].lora_layer = self.to_out_lora.to(hidden_states.device) + + attn._modules.pop("processor") + attn.processor = AttnProcessor() + return attn.processor(attn, hidden_states, **kwargs) + + +class LoRAAttnProcessor2_0(nn.Module): + def __init__( + self, + hidden_size: int, + cross_attention_dim: Optional[int] = None, + rank: int = 4, + network_alpha: Optional[int] = None, + **kwargs, + ): + deprecation_message = "Using LoRAAttnProcessor is deprecated. Please use the PEFT backend for all things LoRA. You can install PEFT by running `pip install peft`." + deprecate("LoRAAttnProcessor2_0", "0.30.0", deprecation_message, standard_warn=False) + + super().__init__() + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.") + + self.hidden_size = hidden_size + self.cross_attention_dim = cross_attention_dim + self.rank = rank + + q_rank = kwargs.pop("q_rank", None) + q_hidden_size = kwargs.pop("q_hidden_size", None) + q_rank = q_rank if q_rank is not None else rank + q_hidden_size = q_hidden_size if q_hidden_size is not None else hidden_size + + v_rank = kwargs.pop("v_rank", None) + v_hidden_size = kwargs.pop("v_hidden_size", None) + v_rank = v_rank if v_rank is not None else rank + v_hidden_size = v_hidden_size if v_hidden_size is not None else hidden_size + + out_rank = kwargs.pop("out_rank", None) + out_hidden_size = kwargs.pop("out_hidden_size", None) + out_rank = out_rank if out_rank is not None else rank + out_hidden_size = out_hidden_size if out_hidden_size is not None else hidden_size + + self.to_q_lora = LoRALinearLayer(q_hidden_size, q_hidden_size, q_rank, network_alpha) + self.to_k_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha) + self.to_v_lora = LoRALinearLayer(cross_attention_dim or v_hidden_size, v_hidden_size, v_rank, network_alpha) + self.to_out_lora = LoRALinearLayer(out_hidden_size, out_hidden_size, out_rank, network_alpha) + + def __call__(self, attn: Attention, hidden_states: torch.FloatTensor, **kwargs) -> torch.FloatTensor: + self_cls_name = self.__class__.__name__ + deprecate( + self_cls_name, + "0.26.0", + ( + f"Make sure use {self_cls_name[4:]} instead by setting" + "LoRA layers to `self.{to_q,to_k,to_v,to_out[0]}.lora_layer` respectively. This will be done automatically when using" + " `LoraLoaderMixin.load_lora_weights`" + ), + ) + attn.to_q.lora_layer = self.to_q_lora.to(hidden_states.device) + attn.to_k.lora_layer = self.to_k_lora.to(hidden_states.device) + attn.to_v.lora_layer = self.to_v_lora.to(hidden_states.device) + attn.to_out[0].lora_layer = self.to_out_lora.to(hidden_states.device) + + attn._modules.pop("processor") + attn.processor = AttnProcessor2_0() + return attn.processor(attn, hidden_states, **kwargs) + + +class LoRAXFormersAttnProcessor(nn.Module): + r""" + Processor for implementing the LoRA attention mechanism with memory efficient attention using xFormers. + + Args: + hidden_size (`int`, *optional*): + The hidden size of the attention layer. + cross_attention_dim (`int`, *optional*): + The number of channels in the `encoder_hidden_states`. + rank (`int`, defaults to 4): + The dimension of the LoRA update matrices. + attention_op (`Callable`, *optional*, defaults to `None`): + The base + [operator](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.AttentionOpBase) to + use as the attention operator. It is recommended to set to `None`, and allow xFormers to choose the best + operator. + network_alpha (`int`, *optional*): + Equivalent to `alpha` but it's usage is specific to Kohya (A1111) style LoRAs. + kwargs (`dict`): + Additional keyword arguments to pass to the `LoRALinearLayer` layers. + """ + + def __init__( + self, + hidden_size: int, + cross_attention_dim: int, + rank: int = 4, + attention_op: Optional[Callable] = None, + network_alpha: Optional[int] = None, + **kwargs, + ): + super().__init__() + + self.hidden_size = hidden_size + self.cross_attention_dim = cross_attention_dim + self.rank = rank + self.attention_op = attention_op + + q_rank = kwargs.pop("q_rank", None) + q_hidden_size = kwargs.pop("q_hidden_size", None) + q_rank = q_rank if q_rank is not None else rank + q_hidden_size = q_hidden_size if q_hidden_size is not None else hidden_size + + v_rank = kwargs.pop("v_rank", None) + v_hidden_size = kwargs.pop("v_hidden_size", None) + v_rank = v_rank if v_rank is not None else rank + v_hidden_size = v_hidden_size if v_hidden_size is not None else hidden_size + + out_rank = kwargs.pop("out_rank", None) + out_hidden_size = kwargs.pop("out_hidden_size", None) + out_rank = out_rank if out_rank is not None else rank + out_hidden_size = out_hidden_size if out_hidden_size is not None else hidden_size + + self.to_q_lora = LoRALinearLayer(q_hidden_size, q_hidden_size, q_rank, network_alpha) + self.to_k_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha) + self.to_v_lora = LoRALinearLayer(cross_attention_dim or v_hidden_size, v_hidden_size, v_rank, network_alpha) + self.to_out_lora = LoRALinearLayer(out_hidden_size, out_hidden_size, out_rank, network_alpha) + + def __call__(self, attn: Attention, hidden_states: torch.FloatTensor, **kwargs) -> torch.FloatTensor: + self_cls_name = self.__class__.__name__ + deprecate( + self_cls_name, + "0.26.0", + ( + f"Make sure use {self_cls_name[4:]} instead by setting" + "LoRA layers to `self.{to_q,to_k,to_v,add_k_proj,add_v_proj,to_out[0]}.lora_layer` respectively. This will be done automatically when using" + " `LoraLoaderMixin.load_lora_weights`" + ), + ) + attn.to_q.lora_layer = self.to_q_lora.to(hidden_states.device) + attn.to_k.lora_layer = self.to_k_lora.to(hidden_states.device) + attn.to_v.lora_layer = self.to_v_lora.to(hidden_states.device) + attn.to_out[0].lora_layer = self.to_out_lora.to(hidden_states.device) + + attn._modules.pop("processor") + attn.processor = XFormersAttnProcessor() + return attn.processor(attn, hidden_states, **kwargs) + + +class LoRAAttnAddedKVProcessor(nn.Module): + r""" + Processor for implementing the LoRA attention mechanism with extra learnable key and value matrices for the text + encoder. + + Args: + hidden_size (`int`, *optional*): + The hidden size of the attention layer. + cross_attention_dim (`int`, *optional*, defaults to `None`): + The number of channels in the `encoder_hidden_states`. + rank (`int`, defaults to 4): + The dimension of the LoRA update matrices. + network_alpha (`int`, *optional*): + Equivalent to `alpha` but it's usage is specific to Kohya (A1111) style LoRAs. + kwargs (`dict`): + Additional keyword arguments to pass to the `LoRALinearLayer` layers. + """ + + def __init__( + self, + hidden_size: int, + cross_attention_dim: Optional[int] = None, + rank: int = 4, + network_alpha: Optional[int] = None, + ): + super().__init__() + + self.hidden_size = hidden_size + self.cross_attention_dim = cross_attention_dim + self.rank = rank + + self.to_q_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha) + self.add_k_proj_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha) + self.add_v_proj_lora = LoRALinearLayer(cross_attention_dim or hidden_size, hidden_size, rank, network_alpha) + self.to_k_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha) + self.to_v_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha) + self.to_out_lora = LoRALinearLayer(hidden_size, hidden_size, rank, network_alpha) + + def __call__(self, attn: Attention, hidden_states: torch.FloatTensor, **kwargs) -> torch.FloatTensor: + self_cls_name = self.__class__.__name__ + deprecate( + self_cls_name, + "0.26.0", + ( + f"Make sure use {self_cls_name[4:]} instead by setting" + "LoRA layers to `self.{to_q,to_k,to_v,add_k_proj,add_v_proj,to_out[0]}.lora_layer` respectively. This will be done automatically when using" + " `LoraLoaderMixin.load_lora_weights`" + ), + ) + attn.to_q.lora_layer = self.to_q_lora.to(hidden_states.device) + attn.to_k.lora_layer = self.to_k_lora.to(hidden_states.device) + attn.to_v.lora_layer = self.to_v_lora.to(hidden_states.device) + attn.to_out[0].lora_layer = self.to_out_lora.to(hidden_states.device) + + attn._modules.pop("processor") + attn.processor = AttnAddedKVProcessor() + return attn.processor(attn, hidden_states, **kwargs) + + +class IPAdapterAttnProcessor(nn.Module): + r""" + Attention processor for Multiple IP-Adapater. + + Args: + hidden_size (`int`): + The hidden size of the attention layer. + cross_attention_dim (`int`): + The number of channels in the `encoder_hidden_states`. + num_tokens (`int`, `Tuple[int]` or `List[int]`, defaults to `(4,)`): + The context length of the image features. + scale (`float` or List[`float`], defaults to 1.0): + the weight scale of image prompt. + """ + + def __init__(self, hidden_size, cross_attention_dim=None, num_tokens=(4,), scale=1.0): + super().__init__() + + self.hidden_size = hidden_size + self.cross_attention_dim = cross_attention_dim + + if not isinstance(num_tokens, (tuple, list)): + num_tokens = [num_tokens] + self.num_tokens = num_tokens + + if not isinstance(scale, list): + scale = [scale] * len(num_tokens) + if len(scale) != len(num_tokens): + raise ValueError("`scale` should be a list of integers with the same length as `num_tokens`.") + self.scale = scale + + self.to_k_ip = nn.ModuleList( + [nn.Linear(cross_attention_dim, hidden_size, bias=False) for _ in range(len(num_tokens))] + ) + self.to_v_ip = nn.ModuleList( + [nn.Linear(cross_attention_dim, hidden_size, bias=False) for _ in range(len(num_tokens))] + ) + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + temb: Optional[torch.FloatTensor] = None, + scale: float = 1.0, + ip_adapter_masks: Optional[torch.FloatTensor] = None, + ): + residual = hidden_states + + # separate ip_hidden_states from encoder_hidden_states + if encoder_hidden_states is not None: + if isinstance(encoder_hidden_states, tuple): + encoder_hidden_states, ip_hidden_states = encoder_hidden_states + else: + deprecation_message = ( + "You have passed a tensor as `encoder_hidden_states`.This is deprecated and will be removed in a future release." + " Please make sure to update your script to pass `encoder_hidden_states` as a tuple to supress this warning." + ) + deprecate("encoder_hidden_states not a tuple", "1.0.0", deprecation_message, standard_warn=False) + end_pos = encoder_hidden_states.shape[1] - self.num_tokens[0] + encoder_hidden_states, ip_hidden_states = ( + encoder_hidden_states[:, :end_pos, :], + [encoder_hidden_states[:, end_pos:, :]], + ) + + if attn.spatial_norm is not None: + hidden_states = attn.spatial_norm(hidden_states, temb) + + input_ndim = hidden_states.ndim + + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + if attn.group_norm is not None: + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + if ip_adapter_masks is not None: + if not isinstance(ip_adapter_masks, torch.Tensor) or ip_adapter_masks.ndim != 4: + raise ValueError( + " ip_adapter_mask should be a tensor with shape [num_ip_adapter, 1, height, width]." + " Please use `IPAdapterMaskProcessor` to preprocess your mask" + ) + if len(ip_adapter_masks) != len(self.scale): + raise ValueError( + f"Number of ip_adapter_masks ({len(ip_adapter_masks)}) must match number of IP-Adapters ({len(self.scale)})" + ) + else: + ip_adapter_masks = [None] * len(self.scale) + + # for ip-adapter + for current_ip_hidden_states, scale, to_k_ip, to_v_ip, mask in zip( + ip_hidden_states, self.scale, self.to_k_ip, self.to_v_ip, ip_adapter_masks + ): + ip_key = to_k_ip(current_ip_hidden_states) + ip_value = to_v_ip(current_ip_hidden_states) + + ip_key = attn.head_to_batch_dim(ip_key) + ip_value = attn.head_to_batch_dim(ip_value) + + ip_attention_probs = attn.get_attention_scores(query, ip_key, None) + current_ip_hidden_states = torch.bmm(ip_attention_probs, ip_value) + current_ip_hidden_states = attn.batch_to_head_dim(current_ip_hidden_states) + + if mask is not None: + mask_downsample = IPAdapterMaskProcessor.downsample( + mask, batch_size, current_ip_hidden_states.shape[1], current_ip_hidden_states.shape[2] + ) + + mask_downsample = mask_downsample.to(dtype=query.dtype, device=query.device) + + current_ip_hidden_states = current_ip_hidden_states * mask_downsample + + hidden_states = hidden_states + scale * current_ip_hidden_states + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + if input_ndim == 4: + hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) + + if attn.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / attn.rescale_output_factor + + return hidden_states + + +class IPAdapterAttnProcessor2_0(torch.nn.Module): + r""" + Attention processor for IP-Adapater for PyTorch 2.0. + + Args: + hidden_size (`int`): + The hidden size of the attention layer. + cross_attention_dim (`int`): + The number of channels in the `encoder_hidden_states`. + num_tokens (`int`, `Tuple[int]` or `List[int]`, defaults to `(4,)`): + The context length of the image features. + scale (`float` or `List[float]`, defaults to 1.0): + the weight scale of image prompt. + """ + + def __init__(self, hidden_size, cross_attention_dim=None, num_tokens=(4,), scale=1.0): + super().__init__() + + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError( + f"{self.__class__.__name__} requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0." + ) + + self.hidden_size = hidden_size + self.cross_attention_dim = cross_attention_dim + + if not isinstance(num_tokens, (tuple, list)): + num_tokens = [num_tokens] + self.num_tokens = num_tokens + + if not isinstance(scale, list): + scale = [scale] * len(num_tokens) + if len(scale) != len(num_tokens): + raise ValueError("`scale` should be a list of integers with the same length as `num_tokens`.") + self.scale = scale + + self.to_k_ip = nn.ModuleList( + [nn.Linear(cross_attention_dim, hidden_size, bias=False) for _ in range(len(num_tokens))] + ) + self.to_v_ip = nn.ModuleList( + [nn.Linear(cross_attention_dim, hidden_size, bias=False) for _ in range(len(num_tokens))] + ) + + def __call__( + self, + attn: Attention, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + temb: Optional[torch.FloatTensor] = None, + scale: float = 1.0, + ip_adapter_masks: Optional[torch.FloatTensor] = None, + ): + residual = hidden_states + + # separate ip_hidden_states from encoder_hidden_states + if encoder_hidden_states is not None: + if isinstance(encoder_hidden_states, tuple): + encoder_hidden_states, ip_hidden_states = encoder_hidden_states + else: + deprecation_message = ( + "You have passed a tensor as `encoder_hidden_states`.This is deprecated and will be removed in a future release." + " Please make sure to update your script to pass `encoder_hidden_states` as a tuple to supress this warning." + ) + deprecate("encoder_hidden_states not a tuple", "1.0.0", deprecation_message, standard_warn=False) + end_pos = encoder_hidden_states.shape[1] - self.num_tokens[0] + encoder_hidden_states, ip_hidden_states = ( + encoder_hidden_states[:, :end_pos, :], + [encoder_hidden_states[:, end_pos:, :]], + ) + + if attn.spatial_norm is not None: + hidden_states = attn.spatial_norm(hidden_states, temb) + + input_ndim = hidden_states.ndim + + if input_ndim == 4: + batch_size, channel, height, width = hidden_states.shape + hidden_states = hidden_states.view(batch_size, channel, height * width).transpose(1, 2) + + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + + if attention_mask is not None: + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + # scaled_dot_product_attention expects attention_mask shape to be + # (batch, heads, source_length, target_length) + attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) + + if attn.group_norm is not None: + hidden_states = attn.group_norm(hidden_states.transpose(1, 2)).transpose(1, 2) + + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + inner_dim = key.shape[-1] + head_dim = inner_dim // attn.heads + + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # the output of sdp = (batch, num_heads, seq_len, head_dim) + # TODO: add support for attn.scale when we move to Torch 2.1 + hidden_states = F.scaled_dot_product_attention( + query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False + ) + + hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + hidden_states = hidden_states.to(query.dtype) + + if ip_adapter_masks is not None: + if not isinstance(ip_adapter_masks, torch.Tensor) or ip_adapter_masks.ndim != 4: + raise ValueError( + " ip_adapter_mask should be a tensor with shape [num_ip_adapter, 1, height, width]." + " Please use `IPAdapterMaskProcessor` to preprocess your mask" + ) + if len(ip_adapter_masks) != len(self.scale): + raise ValueError( + f"Number of ip_adapter_masks ({len(ip_adapter_masks)}) must match number of IP-Adapters ({len(self.scale)})" + ) + else: + ip_adapter_masks = [None] * len(self.scale) + + # for ip-adapter + for current_ip_hidden_states, scale, to_k_ip, to_v_ip, mask in zip( + ip_hidden_states, self.scale, self.to_k_ip, self.to_v_ip, ip_adapter_masks + ): + ip_key = to_k_ip(current_ip_hidden_states) + ip_value = to_v_ip(current_ip_hidden_states) + + ip_key = ip_key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + ip_value = ip_value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # the output of sdp = (batch, num_heads, seq_len, head_dim) + # TODO: add support for attn.scale when we move to Torch 2.1 + current_ip_hidden_states = F.scaled_dot_product_attention( + query, ip_key, ip_value, attn_mask=None, dropout_p=0.0, is_causal=False + ) + + current_ip_hidden_states = current_ip_hidden_states.transpose(1, 2).reshape( + batch_size, -1, attn.heads * head_dim + ) + current_ip_hidden_states = current_ip_hidden_states.to(query.dtype) + + if mask is not None: + mask_downsample = IPAdapterMaskProcessor.downsample( + mask, batch_size, current_ip_hidden_states.shape[1], current_ip_hidden_states.shape[2] + ) + + mask_downsample = mask_downsample.to(dtype=query.dtype, device=query.device) + + current_ip_hidden_states = current_ip_hidden_states * mask_downsample + + hidden_states = hidden_states + scale * current_ip_hidden_states + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + if input_ndim == 4: + hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width) + + if attn.residual_connection: + hidden_states = hidden_states + residual + + hidden_states = hidden_states / attn.rescale_output_factor + + return hidden_states + + +LORA_ATTENTION_PROCESSORS = ( + LoRAAttnProcessor, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + LoRAAttnAddedKVProcessor, +) + +ADDED_KV_ATTENTION_PROCESSORS = ( + AttnAddedKVProcessor, + SlicedAttnAddedKVProcessor, + AttnAddedKVProcessor2_0, + XFormersAttnAddedKVProcessor, + LoRAAttnAddedKVProcessor, +) + +CROSS_ATTENTION_PROCESSORS = ( + AttnProcessor, + AttnProcessor2_0, + XFormersAttnProcessor, + SlicedAttnProcessor, + LoRAAttnProcessor, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + IPAdapterAttnProcessor, + IPAdapterAttnProcessor2_0, +) + +AttentionProcessor = Union[ + AttnProcessor, + AttnProcessor2_0, + FusedAttnProcessor2_0, + XFormersAttnProcessor, + SlicedAttnProcessor, + AttnAddedKVProcessor, + SlicedAttnAddedKVProcessor, + AttnAddedKVProcessor2_0, + XFormersAttnAddedKVProcessor, + CustomDiffusionAttnProcessor, + CustomDiffusionXFormersAttnProcessor, + CustomDiffusionAttnProcessor2_0, + # deprecated + LoRAAttnProcessor, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + LoRAAttnAddedKVProcessor, +] diff --git a/diffusers-0.27.0/src/diffusers/models/autoencoders/__init__.py b/diffusers-0.27.0/src/diffusers/models/autoencoders/__init__.py new file mode 100755 index 0000000..201a40f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/autoencoders/__init__.py @@ -0,0 +1,5 @@ +from .autoencoder_asym_kl import AsymmetricAutoencoderKL +from .autoencoder_kl import AutoencoderKL +from .autoencoder_kl_temporal_decoder import AutoencoderKLTemporalDecoder +from .autoencoder_tiny import AutoencoderTiny +from .consistency_decoder_vae import ConsistencyDecoderVAE diff --git a/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_asym_kl.py b/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_asym_kl.py new file mode 100755 index 0000000..fc2041d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_asym_kl.py @@ -0,0 +1,186 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...utils.accelerate_utils import apply_forward_hook +from ..modeling_outputs import AutoencoderKLOutput +from ..modeling_utils import ModelMixin +from .vae import DecoderOutput, DiagonalGaussianDistribution, Encoder, MaskConditionDecoder + + +class AsymmetricAutoencoderKL(ModelMixin, ConfigMixin): + r""" + Designing a Better Asymmetric VQGAN for StableDiffusion https://arxiv.org/abs/2306.04632 . A VAE model with KL loss + for encoding images into latents and decoding latent representations into images. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + in_channels (int, *optional*, defaults to 3): Number of channels in the input image. + out_channels (int, *optional*, defaults to 3): Number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + Tuple of downsample block types. + down_block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): + Tuple of down block output channels. + layers_per_down_block (`int`, *optional*, defaults to `1`): + Number layers for down block. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + Tuple of upsample block types. + up_block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): + Tuple of up block output channels. + layers_per_up_block (`int`, *optional*, defaults to `1`): + Number layers for up block. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + latent_channels (`int`, *optional*, defaults to 4): Number of channels in the latent space. + sample_size (`int`, *optional*, defaults to `32`): Sample input size. + norm_num_groups (`int`, *optional*, defaults to `32`): + Number of groups to use for the first normalization layer in ResNet blocks. + scaling_factor (`float`, *optional*, defaults to 0.18215): + The component-wise standard deviation of the trained latent space computed using the first batch of the + training set. This is used to scale the latent space to have unit variance when training the diffusion + model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the + diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1 + / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image + Synthesis with Latent Diffusion Models](https://arxiv.org/abs/2112.10752) paper. + """ + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str, ...] = ("DownEncoderBlock2D",), + down_block_out_channels: Tuple[int, ...] = (64,), + layers_per_down_block: int = 1, + up_block_types: Tuple[str, ...] = ("UpDecoderBlock2D",), + up_block_out_channels: Tuple[int, ...] = (64,), + layers_per_up_block: int = 1, + act_fn: str = "silu", + latent_channels: int = 4, + norm_num_groups: int = 32, + sample_size: int = 32, + scaling_factor: float = 0.18215, + ) -> None: + super().__init__() + + # pass init params to Encoder + self.encoder = Encoder( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=down_block_out_channels, + layers_per_block=layers_per_down_block, + act_fn=act_fn, + norm_num_groups=norm_num_groups, + double_z=True, + ) + + # pass init params to Decoder + self.decoder = MaskConditionDecoder( + in_channels=latent_channels, + out_channels=out_channels, + up_block_types=up_block_types, + block_out_channels=up_block_out_channels, + layers_per_block=layers_per_up_block, + act_fn=act_fn, + norm_num_groups=norm_num_groups, + ) + + self.quant_conv = nn.Conv2d(2 * latent_channels, 2 * latent_channels, 1) + self.post_quant_conv = nn.Conv2d(latent_channels, latent_channels, 1) + + self.use_slicing = False + self.use_tiling = False + + self.register_to_config(block_out_channels=up_block_out_channels) + self.register_to_config(force_upcast=False) + + @apply_forward_hook + def encode( + self, x: torch.FloatTensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[torch.FloatTensor]]: + h = self.encoder(x) + moments = self.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + def _decode( + self, + z: torch.FloatTensor, + image: Optional[torch.FloatTensor] = None, + mask: Optional[torch.FloatTensor] = None, + return_dict: bool = True, + ) -> Union[DecoderOutput, Tuple[torch.FloatTensor]]: + z = self.post_quant_conv(z) + dec = self.decoder(z, image, mask) + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + @apply_forward_hook + def decode( + self, + z: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + image: Optional[torch.FloatTensor] = None, + mask: Optional[torch.FloatTensor] = None, + return_dict: bool = True, + ) -> Union[DecoderOutput, Tuple[torch.FloatTensor]]: + decoded = self._decode(z, image, mask).sample + + if not return_dict: + return (decoded,) + + return DecoderOutput(sample=decoded) + + def forward( + self, + sample: torch.FloatTensor, + mask: Optional[torch.FloatTensor] = None, + sample_posterior: bool = False, + return_dict: bool = True, + generator: Optional[torch.Generator] = None, + ) -> Union[DecoderOutput, Tuple[torch.FloatTensor]]: + r""" + Args: + sample (`torch.FloatTensor`): Input sample. + mask (`torch.FloatTensor`, *optional*, defaults to `None`): Optional inpainting mask. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample from the posterior. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z, sample, mask).sample + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) diff --git a/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_kl.py b/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_kl.py new file mode 100755 index 0000000..9bbf202 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_kl.py @@ -0,0 +1,489 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...loaders import FromOriginalVAEMixin +from ...utils.accelerate_utils import apply_forward_hook +from ..attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + Attention, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from ..modeling_outputs import AutoencoderKLOutput +from ..modeling_utils import ModelMixin +from .vae import Decoder, DecoderOutput, DiagonalGaussianDistribution, Encoder + + +class AutoencoderKL(ModelMixin, ConfigMixin, FromOriginalVAEMixin): + r""" + A VAE model with KL loss for encoding images into latents and decoding latent representations into images. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + in_channels (int, *optional*, defaults to 3): Number of channels in the input image. + out_channels (int, *optional*, defaults to 3): Number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + Tuple of downsample block types. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + Tuple of upsample block types. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): + Tuple of block output channels. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + latent_channels (`int`, *optional*, defaults to 4): Number of channels in the latent space. + sample_size (`int`, *optional*, defaults to `32`): Sample input size. + scaling_factor (`float`, *optional*, defaults to 0.18215): + The component-wise standard deviation of the trained latent space computed using the first batch of the + training set. This is used to scale the latent space to have unit variance when training the diffusion + model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the + diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1 + / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image + Synthesis with Latent Diffusion Models](https://arxiv.org/abs/2112.10752) paper. + force_upcast (`bool`, *optional*, default to `True`): + If enabled it will force the VAE to run in float32 for high image resolution pipelines, such as SD-XL. VAE + can be fine-tuned / trained to a lower range without loosing too much precision in which case + `force_upcast` can be set to `False` - see: https://huggingface.co/madebyollin/sdxl-vae-fp16-fix + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str] = ("DownEncoderBlock2D",), + up_block_types: Tuple[str] = ("UpDecoderBlock2D",), + block_out_channels: Tuple[int] = (64,), + layers_per_block: int = 1, + act_fn: str = "silu", + latent_channels: int = 4, + norm_num_groups: int = 32, + sample_size: int = 32, + scaling_factor: float = 0.18215, + latents_mean: Optional[Tuple[float]] = None, + latents_std: Optional[Tuple[float]] = None, + force_upcast: float = True, + ): + super().__init__() + + # pass init params to Encoder + self.encoder = Encoder( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn=act_fn, + norm_num_groups=norm_num_groups, + double_z=True, + ) + + # pass init params to Decoder + self.decoder = Decoder( + in_channels=latent_channels, + out_channels=out_channels, + up_block_types=up_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + norm_num_groups=norm_num_groups, + act_fn=act_fn, + ) + + self.quant_conv = nn.Conv2d(2 * latent_channels, 2 * latent_channels, 1) + self.post_quant_conv = nn.Conv2d(latent_channels, latent_channels, 1) + + self.use_slicing = False + self.use_tiling = False + + # only relevant if vae tiling is enabled + self.tile_sample_min_size = self.config.sample_size + sample_size = ( + self.config.sample_size[0] + if isinstance(self.config.sample_size, (list, tuple)) + else self.config.sample_size + ) + self.tile_latent_min_size = int(sample_size / (2 ** (len(self.config.block_out_channels) - 1))) + self.tile_overlap_factor = 0.25 + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (Encoder, Decoder)): + module.gradient_checkpointing = value + + def enable_tiling(self, use_tiling: bool = True): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.use_tiling = use_tiling + + def disable_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_tiling` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.enable_tiling(False) + + def enable_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.use_slicing = True + + def disable_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_slicing` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_slicing = False + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + @apply_forward_hook + def encode( + self, x: torch.FloatTensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + """ + Encode a batch of images into latents. + + Args: + x (`torch.FloatTensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + The latent representations of the encoded images. If `return_dict` is True, a + [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. + """ + if self.use_tiling and (x.shape[-1] > self.tile_sample_min_size or x.shape[-2] > self.tile_sample_min_size): + return self.tiled_encode(x, return_dict=return_dict) + + if self.use_slicing and x.shape[0] > 1: + encoded_slices = [self.encoder(x_slice) for x_slice in x.split(1)] + h = torch.cat(encoded_slices) + else: + h = self.encoder(x) + + moments = self.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + def _decode(self, z: torch.FloatTensor, return_dict: bool = True) -> Union[DecoderOutput, torch.FloatTensor]: + if self.use_tiling and (z.shape[-1] > self.tile_latent_min_size or z.shape[-2] > self.tile_latent_min_size): + return self.tiled_decode(z, return_dict=return_dict) + + z = self.post_quant_conv(z) + dec = self.decoder(z) + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + @apply_forward_hook + def decode( + self, z: torch.FloatTensor, return_dict: bool = True, generator=None + ) -> Union[DecoderOutput, torch.FloatTensor]: + """ + Decode a batch of images. + + Args: + z (`torch.FloatTensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + + """ + if self.use_slicing and z.shape[0] > 1: + decoded_slices = [self._decode(z_slice).sample for z_slice in z.split(1)] + decoded = torch.cat(decoded_slices) + else: + decoded = self._decode(z).sample + + if not return_dict: + return (decoded,) + + return DecoderOutput(sample=decoded) + + def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[2], b.shape[2], blend_extent) + for y in range(blend_extent): + b[:, :, y, :] = a[:, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, y, :] * (y / blend_extent) + return b + + def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[3], b.shape[3], blend_extent) + for x in range(blend_extent): + b[:, :, :, x] = a[:, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, x] * (x / blend_extent) + return b + + def tiled_encode(self, x: torch.FloatTensor, return_dict: bool = True) -> AutoencoderKLOutput: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. The end result of tiled encoding is + different from non-tiled encoding because each tile uses a different encoder. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. You may still see tile-sized changes in the + output, but they should be much less noticeable. + + Args: + x (`torch.FloatTensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + [`~models.autoencoder_kl.AutoencoderKLOutput`] or `tuple`: + If return_dict is True, a [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain + `tuple` is returned. + """ + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + + # Split the image into 512x512 tiles and encode them separately. + rows = [] + for i in range(0, x.shape[2], overlap_size): + row = [] + for j in range(0, x.shape[3], overlap_size): + tile = x[:, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size] + tile = self.encoder(tile) + tile = self.quant_conv(tile) + row.append(tile) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + moments = torch.cat(result_rows, dim=2) + posterior = DiagonalGaussianDistribution(moments) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + def tiled_decode(self, z: torch.FloatTensor, return_dict: bool = True) -> Union[DecoderOutput, torch.FloatTensor]: + r""" + Decode a batch of images using a tiled decoder. + + Args: + z (`torch.FloatTensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + """ + overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor) + row_limit = self.tile_sample_min_size - blend_extent + + # Split z into overlapping 64x64 tiles and decode them separately. + # The tiles have an overlap to avoid seams between tiles. + rows = [] + for i in range(0, z.shape[2], overlap_size): + row = [] + for j in range(0, z.shape[3], overlap_size): + tile = z[:, :, i : i + self.tile_latent_min_size, j : j + self.tile_latent_min_size] + tile = self.post_quant_conv(tile) + decoded = self.decoder(tile) + row.append(decoded) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + dec = torch.cat(result_rows, dim=2) + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + def forward( + self, + sample: torch.FloatTensor, + sample_posterior: bool = False, + return_dict: bool = True, + generator: Optional[torch.Generator] = None, + ) -> Union[DecoderOutput, torch.FloatTensor]: + r""" + Args: + sample (`torch.FloatTensor`): Input sample. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample from the posterior. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z).sample + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.fuse_qkv_projections + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, + key, value) are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is 🧪 experimental. + + + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.") + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.unfuse_qkv_projections + def unfuse_qkv_projections(self): + """Disables the fused QKV projection if enabled. + + + + This API is 🧪 experimental. + + + + """ + if self.original_attn_processors is not None: + self.set_attn_processor(self.original_attn_processors) diff --git a/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py b/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py new file mode 100755 index 0000000..b12226f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_kl_temporal_decoder.py @@ -0,0 +1,399 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...utils import is_torch_version +from ...utils.accelerate_utils import apply_forward_hook +from ..attention_processor import CROSS_ATTENTION_PROCESSORS, AttentionProcessor, AttnProcessor +from ..modeling_outputs import AutoencoderKLOutput +from ..modeling_utils import ModelMixin +from ..unets.unet_3d_blocks import MidBlockTemporalDecoder, UpBlockTemporalDecoder +from .vae import DecoderOutput, DiagonalGaussianDistribution, Encoder + + +class TemporalDecoder(nn.Module): + def __init__( + self, + in_channels: int = 4, + out_channels: int = 3, + block_out_channels: Tuple[int] = (128, 256, 512, 512), + layers_per_block: int = 2, + ): + super().__init__() + self.layers_per_block = layers_per_block + + self.conv_in = nn.Conv2d(in_channels, block_out_channels[-1], kernel_size=3, stride=1, padding=1) + self.mid_block = MidBlockTemporalDecoder( + num_layers=self.layers_per_block, + in_channels=block_out_channels[-1], + out_channels=block_out_channels[-1], + attention_head_dim=block_out_channels[-1], + ) + + # up + self.up_blocks = nn.ModuleList([]) + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i in range(len(block_out_channels)): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + + is_final_block = i == len(block_out_channels) - 1 + up_block = UpBlockTemporalDecoder( + num_layers=self.layers_per_block + 1, + in_channels=prev_output_channel, + out_channels=output_channel, + add_upsample=not is_final_block, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=32, eps=1e-6) + + self.conv_act = nn.SiLU() + self.conv_out = torch.nn.Conv2d( + in_channels=block_out_channels[0], + out_channels=out_channels, + kernel_size=3, + padding=1, + ) + + conv_out_kernel_size = (3, 1, 1) + padding = [int(k // 2) for k in conv_out_kernel_size] + self.time_conv_out = torch.nn.Conv3d( + in_channels=out_channels, + out_channels=out_channels, + kernel_size=conv_out_kernel_size, + padding=padding, + ) + + self.gradient_checkpointing = False + + def forward( + self, + sample: torch.FloatTensor, + image_only_indicator: torch.FloatTensor, + num_frames: int = 1, + ) -> torch.FloatTensor: + r"""The forward method of the `Decoder` class.""" + + sample = self.conv_in(sample) + + upscale_dtype = next(iter(self.up_blocks.parameters())).dtype + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), + sample, + image_only_indicator, + use_reentrant=False, + ) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(up_block), + sample, + image_only_indicator, + use_reentrant=False, + ) + else: + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), + sample, + image_only_indicator, + ) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(up_block), + sample, + image_only_indicator, + ) + else: + # middle + sample = self.mid_block(sample, image_only_indicator=image_only_indicator) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = up_block(sample, image_only_indicator=image_only_indicator) + + # post-process + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + batch_frames, channels, height, width = sample.shape + batch_size = batch_frames // num_frames + sample = sample[None, :].reshape(batch_size, num_frames, channels, height, width).permute(0, 2, 1, 3, 4) + sample = self.time_conv_out(sample) + + sample = sample.permute(0, 2, 1, 3, 4).reshape(batch_frames, channels, height, width) + + return sample + + +class AutoencoderKLTemporalDecoder(ModelMixin, ConfigMixin): + r""" + A VAE model with KL loss for encoding images into latents and decoding latent representations into images. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + in_channels (int, *optional*, defaults to 3): Number of channels in the input image. + out_channels (int, *optional*, defaults to 3): Number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + Tuple of downsample block types. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): + Tuple of block output channels. + layers_per_block: (`int`, *optional*, defaults to 1): Number of layers per block. + latent_channels (`int`, *optional*, defaults to 4): Number of channels in the latent space. + sample_size (`int`, *optional*, defaults to `32`): Sample input size. + scaling_factor (`float`, *optional*, defaults to 0.18215): + The component-wise standard deviation of the trained latent space computed using the first batch of the + training set. This is used to scale the latent space to have unit variance when training the diffusion + model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the + diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1 + / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image + Synthesis with Latent Diffusion Models](https://arxiv.org/abs/2112.10752) paper. + force_upcast (`bool`, *optional*, default to `True`): + If enabled it will force the VAE to run in float32 for high image resolution pipelines, such as SD-XL. VAE + can be fine-tuned / trained to a lower range without loosing too much precision in which case + `force_upcast` can be set to `False` - see: https://huggingface.co/madebyollin/sdxl-vae-fp16-fix + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str] = ("DownEncoderBlock2D",), + block_out_channels: Tuple[int] = (64,), + layers_per_block: int = 1, + latent_channels: int = 4, + sample_size: int = 32, + scaling_factor: float = 0.18215, + force_upcast: float = True, + ): + super().__init__() + + # pass init params to Encoder + self.encoder = Encoder( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + double_z=True, + ) + + # pass init params to Decoder + self.decoder = TemporalDecoder( + in_channels=latent_channels, + out_channels=out_channels, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + ) + + self.quant_conv = nn.Conv2d(2 * latent_channels, 2 * latent_channels, 1) + + sample_size = ( + self.config.sample_size[0] + if isinstance(self.config.sample_size, (list, tuple)) + else self.config.sample_size + ) + self.tile_latent_min_size = int(sample_size / (2 ** (len(self.config.block_out_channels) - 1))) + self.tile_overlap_factor = 0.25 + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (Encoder, TemporalDecoder)): + module.gradient_checkpointing = value + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + @apply_forward_hook + def encode( + self, x: torch.FloatTensor, return_dict: bool = True + ) -> Union[AutoencoderKLOutput, Tuple[DiagonalGaussianDistribution]]: + """ + Encode a batch of images into latents. + + Args: + x (`torch.FloatTensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. + + Returns: + The latent representations of the encoded images. If `return_dict` is True, a + [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. + """ + h = self.encoder(x) + moments = self.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + + if not return_dict: + return (posterior,) + + return AutoencoderKLOutput(latent_dist=posterior) + + @apply_forward_hook + def decode( + self, + z: torch.FloatTensor, + num_frames: int, + return_dict: bool = True, + ) -> Union[DecoderOutput, torch.FloatTensor]: + """ + Decode a batch of images. + + Args: + z (`torch.FloatTensor`): Input batch of latent vectors. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vae.DecoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is + returned. + + """ + batch_size = z.shape[0] // num_frames + image_only_indicator = torch.zeros(batch_size, num_frames, dtype=z.dtype, device=z.device) + decoded = self.decoder(z, num_frames=num_frames, image_only_indicator=image_only_indicator) + + if not return_dict: + return (decoded,) + + return DecoderOutput(sample=decoded) + + def forward( + self, + sample: torch.FloatTensor, + sample_posterior: bool = False, + return_dict: bool = True, + generator: Optional[torch.Generator] = None, + num_frames: int = 1, + ) -> Union[DecoderOutput, torch.FloatTensor]: + r""" + Args: + sample (`torch.FloatTensor`): Input sample. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample from the posterior. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + + dec = self.decode(z, num_frames=num_frames).sample + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) diff --git a/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_tiny.py b/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_tiny.py new file mode 100755 index 0000000..ef43526 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/autoencoders/autoencoder_tiny.py @@ -0,0 +1,347 @@ +# Copyright 2024 Ollin Boer Bohan and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import torch + +from ...configuration_utils import ConfigMixin, register_to_config +from ...utils import BaseOutput +from ...utils.accelerate_utils import apply_forward_hook +from ..modeling_utils import ModelMixin +from .vae import DecoderOutput, DecoderTiny, EncoderTiny + + +@dataclass +class AutoencoderTinyOutput(BaseOutput): + """ + Output of AutoencoderTiny encoding method. + + Args: + latents (`torch.Tensor`): Encoded outputs of the `Encoder`. + + """ + + latents: torch.Tensor + + +class AutoencoderTiny(ModelMixin, ConfigMixin): + r""" + A tiny distilled VAE model for encoding images into latents and decoding latent representations into images. + + [`AutoencoderTiny`] is a wrapper around the original implementation of `TAESD`. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for its generic methods implemented for + all models (such as downloading or saving). + + Parameters: + in_channels (`int`, *optional*, defaults to 3): Number of channels in the input image. + out_channels (`int`, *optional*, defaults to 3): Number of channels in the output. + encoder_block_out_channels (`Tuple[int]`, *optional*, defaults to `(64, 64, 64, 64)`): + Tuple of integers representing the number of output channels for each encoder block. The length of the + tuple should be equal to the number of encoder blocks. + decoder_block_out_channels (`Tuple[int]`, *optional*, defaults to `(64, 64, 64, 64)`): + Tuple of integers representing the number of output channels for each decoder block. The length of the + tuple should be equal to the number of decoder blocks. + act_fn (`str`, *optional*, defaults to `"relu"`): + Activation function to be used throughout the model. + latent_channels (`int`, *optional*, defaults to 4): + Number of channels in the latent representation. The latent space acts as a compressed representation of + the input image. + upsampling_scaling_factor (`int`, *optional*, defaults to 2): + Scaling factor for upsampling in the decoder. It determines the size of the output image during the + upsampling process. + num_encoder_blocks (`Tuple[int]`, *optional*, defaults to `(1, 3, 3, 3)`): + Tuple of integers representing the number of encoder blocks at each stage of the encoding process. The + length of the tuple should be equal to the number of stages in the encoder. Each stage has a different + number of encoder blocks. + num_decoder_blocks (`Tuple[int]`, *optional*, defaults to `(3, 3, 3, 1)`): + Tuple of integers representing the number of decoder blocks at each stage of the decoding process. The + length of the tuple should be equal to the number of stages in the decoder. Each stage has a different + number of decoder blocks. + latent_magnitude (`float`, *optional*, defaults to 3.0): + Magnitude of the latent representation. This parameter scales the latent representation values to control + the extent of information preservation. + latent_shift (float, *optional*, defaults to 0.5): + Shift applied to the latent representation. This parameter controls the center of the latent space. + scaling_factor (`float`, *optional*, defaults to 1.0): + The component-wise standard deviation of the trained latent space computed using the first batch of the + training set. This is used to scale the latent space to have unit variance when training the diffusion + model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the + diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1 + / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image + Synthesis with Latent Diffusion Models](https://arxiv.org/abs/2112.10752) paper. For this Autoencoder, + however, no such scaling factor was used, hence the value of 1.0 as the default. + force_upcast (`bool`, *optional*, default to `False`): + If enabled it will force the VAE to run in float32 for high image resolution pipelines, such as SD-XL. VAE + can be fine-tuned / trained to a lower range without losing too much precision, in which case + `force_upcast` can be set to `False` (see this fp16-friendly + [AutoEncoder](https://huggingface.co/madebyollin/sdxl-vae-fp16-fix)). + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + encoder_block_out_channels: Tuple[int, ...] = (64, 64, 64, 64), + decoder_block_out_channels: Tuple[int, ...] = (64, 64, 64, 64), + act_fn: str = "relu", + latent_channels: int = 4, + upsampling_scaling_factor: int = 2, + num_encoder_blocks: Tuple[int, ...] = (1, 3, 3, 3), + num_decoder_blocks: Tuple[int, ...] = (3, 3, 3, 1), + latent_magnitude: int = 3, + latent_shift: float = 0.5, + force_upcast: bool = False, + scaling_factor: float = 1.0, + ): + super().__init__() + + if len(encoder_block_out_channels) != len(num_encoder_blocks): + raise ValueError("`encoder_block_out_channels` should have the same length as `num_encoder_blocks`.") + if len(decoder_block_out_channels) != len(num_decoder_blocks): + raise ValueError("`decoder_block_out_channels` should have the same length as `num_decoder_blocks`.") + + self.encoder = EncoderTiny( + in_channels=in_channels, + out_channels=latent_channels, + num_blocks=num_encoder_blocks, + block_out_channels=encoder_block_out_channels, + act_fn=act_fn, + ) + + self.decoder = DecoderTiny( + in_channels=latent_channels, + out_channels=out_channels, + num_blocks=num_decoder_blocks, + block_out_channels=decoder_block_out_channels, + upsampling_scaling_factor=upsampling_scaling_factor, + act_fn=act_fn, + ) + + self.latent_magnitude = latent_magnitude + self.latent_shift = latent_shift + self.scaling_factor = scaling_factor + + self.use_slicing = False + self.use_tiling = False + + # only relevant if vae tiling is enabled + self.spatial_scale_factor = 2**out_channels + self.tile_overlap_factor = 0.125 + self.tile_sample_min_size = 512 + self.tile_latent_min_size = self.tile_sample_min_size // self.spatial_scale_factor + + self.register_to_config(block_out_channels=decoder_block_out_channels) + self.register_to_config(force_upcast=False) + + def _set_gradient_checkpointing(self, module, value: bool = False) -> None: + if isinstance(module, (EncoderTiny, DecoderTiny)): + module.gradient_checkpointing = value + + def scale_latents(self, x: torch.FloatTensor) -> torch.FloatTensor: + """raw latents -> [0, 1]""" + return x.div(2 * self.latent_magnitude).add(self.latent_shift).clamp(0, 1) + + def unscale_latents(self, x: torch.FloatTensor) -> torch.FloatTensor: + """[0, 1] -> raw latents""" + return x.sub(self.latent_shift).mul(2 * self.latent_magnitude) + + def enable_slicing(self) -> None: + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.use_slicing = True + + def disable_slicing(self) -> None: + r""" + Disable sliced VAE decoding. If `enable_slicing` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_slicing = False + + def enable_tiling(self, use_tiling: bool = True) -> None: + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.use_tiling = use_tiling + + def disable_tiling(self) -> None: + r""" + Disable tiled VAE decoding. If `enable_tiling` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.enable_tiling(False) + + def _tiled_encode(self, x: torch.FloatTensor) -> torch.FloatTensor: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. + + Args: + x (`torch.FloatTensor`): Input batch of images. + + Returns: + `torch.FloatTensor`: Encoded batch of images. + """ + # scale of encoder output relative to input + sf = self.spatial_scale_factor + tile_size = self.tile_sample_min_size + + # number of pixels to blend and to traverse between tile + blend_size = int(tile_size * self.tile_overlap_factor) + traverse_size = tile_size - blend_size + + # tiles index (up/left) + ti = range(0, x.shape[-2], traverse_size) + tj = range(0, x.shape[-1], traverse_size) + + # mask for blending + blend_masks = torch.stack( + torch.meshgrid([torch.arange(tile_size / sf) / (blend_size / sf - 1)] * 2, indexing="ij") + ) + blend_masks = blend_masks.clamp(0, 1).to(x.device) + + # output array + out = torch.zeros(x.shape[0], 4, x.shape[-2] // sf, x.shape[-1] // sf, device=x.device) + for i in ti: + for j in tj: + tile_in = x[..., i : i + tile_size, j : j + tile_size] + # tile result + tile_out = out[..., i // sf : (i + tile_size) // sf, j // sf : (j + tile_size) // sf] + tile = self.encoder(tile_in) + h, w = tile.shape[-2], tile.shape[-1] + # blend tile result into output + blend_mask_i = torch.ones_like(blend_masks[0]) if i == 0 else blend_masks[0] + blend_mask_j = torch.ones_like(blend_masks[1]) if j == 0 else blend_masks[1] + blend_mask = blend_mask_i * blend_mask_j + tile, blend_mask = tile[..., :h, :w], blend_mask[..., :h, :w] + tile_out.copy_(blend_mask * tile + (1 - blend_mask) * tile_out) + return out + + def _tiled_decode(self, x: torch.FloatTensor) -> torch.FloatTensor: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. + + Args: + x (`torch.FloatTensor`): Input batch of images. + + Returns: + `torch.FloatTensor`: Encoded batch of images. + """ + # scale of decoder output relative to input + sf = self.spatial_scale_factor + tile_size = self.tile_latent_min_size + + # number of pixels to blend and to traverse between tiles + blend_size = int(tile_size * self.tile_overlap_factor) + traverse_size = tile_size - blend_size + + # tiles index (up/left) + ti = range(0, x.shape[-2], traverse_size) + tj = range(0, x.shape[-1], traverse_size) + + # mask for blending + blend_masks = torch.stack( + torch.meshgrid([torch.arange(tile_size * sf) / (blend_size * sf - 1)] * 2, indexing="ij") + ) + blend_masks = blend_masks.clamp(0, 1).to(x.device) + + # output array + out = torch.zeros(x.shape[0], 3, x.shape[-2] * sf, x.shape[-1] * sf, device=x.device) + for i in ti: + for j in tj: + tile_in = x[..., i : i + tile_size, j : j + tile_size] + # tile result + tile_out = out[..., i * sf : (i + tile_size) * sf, j * sf : (j + tile_size) * sf] + tile = self.decoder(tile_in) + h, w = tile.shape[-2], tile.shape[-1] + # blend tile result into output + blend_mask_i = torch.ones_like(blend_masks[0]) if i == 0 else blend_masks[0] + blend_mask_j = torch.ones_like(blend_masks[1]) if j == 0 else blend_masks[1] + blend_mask = (blend_mask_i * blend_mask_j)[..., :h, :w] + tile_out.copy_(blend_mask * tile + (1 - blend_mask) * tile_out) + return out + + @apply_forward_hook + def encode( + self, x: torch.FloatTensor, return_dict: bool = True + ) -> Union[AutoencoderTinyOutput, Tuple[torch.FloatTensor]]: + if self.use_slicing and x.shape[0] > 1: + output = [ + self._tiled_encode(x_slice) if self.use_tiling else self.encoder(x_slice) for x_slice in x.split(1) + ] + output = torch.cat(output) + else: + output = self._tiled_encode(x) if self.use_tiling else self.encoder(x) + + if not return_dict: + return (output,) + + return AutoencoderTinyOutput(latents=output) + + @apply_forward_hook + def decode( + self, x: torch.FloatTensor, generator: Optional[torch.Generator] = None, return_dict: bool = True + ) -> Union[DecoderOutput, Tuple[torch.FloatTensor]]: + if self.use_slicing and x.shape[0] > 1: + output = [self._tiled_decode(x_slice) if self.use_tiling else self.decoder(x) for x_slice in x.split(1)] + output = torch.cat(output) + else: + output = self._tiled_decode(x) if self.use_tiling else self.decoder(x) + + if not return_dict: + return (output,) + + return DecoderOutput(sample=output) + + def forward( + self, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[DecoderOutput, Tuple[torch.FloatTensor]]: + r""" + Args: + sample (`torch.FloatTensor`): Input sample. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + enc = self.encode(sample).latents + + # scale latents to be in [0, 1], then quantize latents to a byte tensor, + # as if we were storing the latents in an RGBA uint8 image. + scaled_enc = self.scale_latents(enc).mul_(255).round_().byte() + + # unquantize latents back into [0, 1], then unscale latents back to their original range, + # as if we were loading the latents from an RGBA uint8 image. + unscaled_enc = self.unscale_latents(scaled_enc / 255.0) + + dec = self.decode(unscaled_enc) + + if not return_dict: + return (dec,) + return DecoderOutput(sample=dec) diff --git a/diffusers-0.27.0/src/diffusers/models/autoencoders/consistency_decoder_vae.py b/diffusers-0.27.0/src/diffusers/models/autoencoders/consistency_decoder_vae.py new file mode 100755 index 0000000..72c512d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/autoencoders/consistency_decoder_vae.py @@ -0,0 +1,435 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Dict, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...schedulers import ConsistencyDecoderScheduler +from ...utils import BaseOutput +from ...utils.accelerate_utils import apply_forward_hook +from ...utils.torch_utils import randn_tensor +from ..attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from ..modeling_utils import ModelMixin +from ..unets.unet_2d import UNet2DModel +from .vae import DecoderOutput, DiagonalGaussianDistribution, Encoder + + +@dataclass +class ConsistencyDecoderVAEOutput(BaseOutput): + """ + Output of encoding method. + + Args: + latent_dist (`DiagonalGaussianDistribution`): + Encoded outputs of `Encoder` represented as the mean and logvar of `DiagonalGaussianDistribution`. + `DiagonalGaussianDistribution` allows for sampling latents from the distribution. + """ + + latent_dist: "DiagonalGaussianDistribution" + + +class ConsistencyDecoderVAE(ModelMixin, ConfigMixin): + r""" + The consistency decoder used with DALL-E 3. + + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionPipeline, ConsistencyDecoderVAE + + >>> vae = ConsistencyDecoderVAE.from_pretrained("openai/consistency-decoder", torch_dtype=torch.float16) + >>> pipe = StableDiffusionPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", vae=vae, torch_dtype=torch.float16 + ... ).to("cuda") + + >>> pipe("horse", generator=torch.manual_seed(0)).images + ``` + """ + + @register_to_config + def __init__( + self, + scaling_factor: float = 0.18215, + latent_channels: int = 4, + encoder_act_fn: str = "silu", + encoder_block_out_channels: Tuple[int, ...] = (128, 256, 512, 512), + encoder_double_z: bool = True, + encoder_down_block_types: Tuple[str, ...] = ( + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + ), + encoder_in_channels: int = 3, + encoder_layers_per_block: int = 2, + encoder_norm_num_groups: int = 32, + encoder_out_channels: int = 4, + decoder_add_attention: bool = False, + decoder_block_out_channels: Tuple[int, ...] = (320, 640, 1024, 1024), + decoder_down_block_types: Tuple[str, ...] = ( + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + "ResnetDownsampleBlock2D", + ), + decoder_downsample_padding: int = 1, + decoder_in_channels: int = 7, + decoder_layers_per_block: int = 3, + decoder_norm_eps: float = 1e-05, + decoder_norm_num_groups: int = 32, + decoder_num_train_timesteps: int = 1024, + decoder_out_channels: int = 6, + decoder_resnet_time_scale_shift: str = "scale_shift", + decoder_time_embedding_type: str = "learned", + decoder_up_block_types: Tuple[str, ...] = ( + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + "ResnetUpsampleBlock2D", + ), + ): + super().__init__() + self.encoder = Encoder( + act_fn=encoder_act_fn, + block_out_channels=encoder_block_out_channels, + double_z=encoder_double_z, + down_block_types=encoder_down_block_types, + in_channels=encoder_in_channels, + layers_per_block=encoder_layers_per_block, + norm_num_groups=encoder_norm_num_groups, + out_channels=encoder_out_channels, + ) + + self.decoder_unet = UNet2DModel( + add_attention=decoder_add_attention, + block_out_channels=decoder_block_out_channels, + down_block_types=decoder_down_block_types, + downsample_padding=decoder_downsample_padding, + in_channels=decoder_in_channels, + layers_per_block=decoder_layers_per_block, + norm_eps=decoder_norm_eps, + norm_num_groups=decoder_norm_num_groups, + num_train_timesteps=decoder_num_train_timesteps, + out_channels=decoder_out_channels, + resnet_time_scale_shift=decoder_resnet_time_scale_shift, + time_embedding_type=decoder_time_embedding_type, + up_block_types=decoder_up_block_types, + ) + self.decoder_scheduler = ConsistencyDecoderScheduler() + self.register_to_config(block_out_channels=encoder_block_out_channels) + self.register_to_config(force_upcast=False) + self.register_buffer( + "means", + torch.tensor([0.38862467, 0.02253063, 0.07381133, -0.0171294])[None, :, None, None], + persistent=False, + ) + self.register_buffer( + "stds", torch.tensor([0.9654121, 1.0440036, 0.76147926, 0.77022034])[None, :, None, None], persistent=False + ) + + self.quant_conv = nn.Conv2d(2 * latent_channels, 2 * latent_channels, 1) + + self.use_slicing = False + self.use_tiling = False + + # Copied from diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL.enable_tiling + def enable_tiling(self, use_tiling: bool = True): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.use_tiling = use_tiling + + # Copied from diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL.disable_tiling + def disable_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_tiling` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.enable_tiling(False) + + # Copied from diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL.enable_slicing + def enable_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.use_slicing = True + + # Copied from diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL.disable_slicing + def disable_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_slicing` was previously enabled, this method will go back to computing + decoding in one step. + """ + self.use_slicing = False + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + @apply_forward_hook + def encode( + self, x: torch.FloatTensor, return_dict: bool = True + ) -> Union[ConsistencyDecoderVAEOutput, Tuple[DiagonalGaussianDistribution]]: + """ + Encode a batch of images into latents. + + Args: + x (`torch.FloatTensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether to return a [`~models.consistecy_decoder_vae.ConsistencyDecoderOoutput`] instead of a plain + tuple. + + Returns: + The latent representations of the encoded images. If `return_dict` is True, a + [`~models.consistency_decoder_vae.ConsistencyDecoderVAEOutput`] is returned, otherwise a plain `tuple` + is returned. + """ + if self.use_tiling and (x.shape[-1] > self.tile_sample_min_size or x.shape[-2] > self.tile_sample_min_size): + return self.tiled_encode(x, return_dict=return_dict) + + if self.use_slicing and x.shape[0] > 1: + encoded_slices = [self.encoder(x_slice) for x_slice in x.split(1)] + h = torch.cat(encoded_slices) + else: + h = self.encoder(x) + + moments = self.quant_conv(h) + posterior = DiagonalGaussianDistribution(moments) + + if not return_dict: + return (posterior,) + + return ConsistencyDecoderVAEOutput(latent_dist=posterior) + + @apply_forward_hook + def decode( + self, + z: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + num_inference_steps: int = 2, + ) -> Union[DecoderOutput, Tuple[torch.FloatTensor]]: + z = (z * self.config.scaling_factor - self.means) / self.stds + + scale_factor = 2 ** (len(self.config.block_out_channels) - 1) + z = F.interpolate(z, mode="nearest", scale_factor=scale_factor) + + batch_size, _, height, width = z.shape + + self.decoder_scheduler.set_timesteps(num_inference_steps, device=self.device) + + x_t = self.decoder_scheduler.init_noise_sigma * randn_tensor( + (batch_size, 3, height, width), generator=generator, dtype=z.dtype, device=z.device + ) + + for t in self.decoder_scheduler.timesteps: + model_input = torch.concat([self.decoder_scheduler.scale_model_input(x_t, t), z], dim=1) + model_output = self.decoder_unet(model_input, t).sample[:, :3, :, :] + prev_sample = self.decoder_scheduler.step(model_output, t, x_t, generator).prev_sample + x_t = prev_sample + + x_0 = x_t + + if not return_dict: + return (x_0,) + + return DecoderOutput(sample=x_0) + + # Copied from diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL.blend_v + def blend_v(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[2], b.shape[2], blend_extent) + for y in range(blend_extent): + b[:, :, y, :] = a[:, :, -blend_extent + y, :] * (1 - y / blend_extent) + b[:, :, y, :] * (y / blend_extent) + return b + + # Copied from diffusers.models.autoencoders.autoencoder_kl.AutoencoderKL.blend_h + def blend_h(self, a: torch.Tensor, b: torch.Tensor, blend_extent: int) -> torch.Tensor: + blend_extent = min(a.shape[3], b.shape[3], blend_extent) + for x in range(blend_extent): + b[:, :, :, x] = a[:, :, :, -blend_extent + x] * (1 - x / blend_extent) + b[:, :, :, x] * (x / blend_extent) + return b + + def tiled_encode(self, x: torch.FloatTensor, return_dict: bool = True) -> ConsistencyDecoderVAEOutput: + r"""Encode a batch of images using a tiled encoder. + + When this option is enabled, the VAE will split the input tensor into tiles to compute encoding in several + steps. This is useful to keep memory use constant regardless of image size. The end result of tiled encoding is + different from non-tiled encoding because each tile uses a different encoder. To avoid tiling artifacts, the + tiles overlap and are blended together to form a smooth output. You may still see tile-sized changes in the + output, but they should be much less noticeable. + + Args: + x (`torch.FloatTensor`): Input batch of images. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.consistency_decoder_vae.ConsistencyDecoderVAEOutput`] instead of a + plain tuple. + + Returns: + [`~models.consistency_decoder_vae.ConsistencyDecoderVAEOutput`] or `tuple`: + If return_dict is True, a [`~models.consistency_decoder_vae.ConsistencyDecoderVAEOutput`] is returned, + otherwise a plain `tuple` is returned. + """ + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = self.tile_latent_min_size - blend_extent + + # Split the image into 512x512 tiles and encode them separately. + rows = [] + for i in range(0, x.shape[2], overlap_size): + row = [] + for j in range(0, x.shape[3], overlap_size): + tile = x[:, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size] + tile = self.encoder(tile) + tile = self.quant_conv(tile) + row.append(tile) + rows.append(row) + result_rows = [] + for i, row in enumerate(rows): + result_row = [] + for j, tile in enumerate(row): + # blend the above tile and the left tile + # to the current tile and add the current tile to the result row + if i > 0: + tile = self.blend_v(rows[i - 1][j], tile, blend_extent) + if j > 0: + tile = self.blend_h(row[j - 1], tile, blend_extent) + result_row.append(tile[:, :, :row_limit, :row_limit]) + result_rows.append(torch.cat(result_row, dim=3)) + + moments = torch.cat(result_rows, dim=2) + posterior = DiagonalGaussianDistribution(moments) + + if not return_dict: + return (posterior,) + + return ConsistencyDecoderVAEOutput(latent_dist=posterior) + + def forward( + self, + sample: torch.FloatTensor, + sample_posterior: bool = False, + return_dict: bool = True, + generator: Optional[torch.Generator] = None, + ) -> Union[DecoderOutput, Tuple[torch.FloatTensor]]: + r""" + Args: + sample (`torch.FloatTensor`): Input sample. + sample_posterior (`bool`, *optional*, defaults to `False`): + Whether to sample from the posterior. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + generator (`torch.Generator`, *optional*, defaults to `None`): + Generator to use for sampling. + + Returns: + [`DecoderOutput`] or `tuple`: + If return_dict is True, a [`DecoderOutput`] is returned, otherwise a plain `tuple` is returned. + """ + x = sample + posterior = self.encode(x).latent_dist + if sample_posterior: + z = posterior.sample(generator=generator) + else: + z = posterior.mode() + dec = self.decode(z, generator=generator).sample + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) diff --git a/diffusers-0.27.0/src/diffusers/models/autoencoders/vae.py b/diffusers-0.27.0/src/diffusers/models/autoencoders/vae.py new file mode 100755 index 0000000..cf2e373 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/autoencoders/vae.py @@ -0,0 +1,983 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Optional, Tuple + +import numpy as np +import torch +import torch.nn as nn + +from ...utils import BaseOutput, is_torch_version +from ...utils.torch_utils import randn_tensor +from ..activations import get_activation +from ..attention_processor import SpatialNorm +from ..unets.unet_2d_blocks import ( + AutoencoderTinyBlock, + UNetMidBlock2D, + get_down_block, + get_up_block, +) + + +@dataclass +class DecoderOutput(BaseOutput): + r""" + Output of decoding method. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + The decoded output sample from the last layer of the model. + """ + + sample: torch.FloatTensor + + +class Encoder(nn.Module): + r""" + The `Encoder` layer of a variational autoencoder that encodes its input into a latent representation. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + down_block_types (`Tuple[str, ...]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + The types of down blocks to use. See `~diffusers.models.unet_2d_blocks.get_down_block` for available + options. + block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + The number of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups for normalization. + act_fn (`str`, *optional*, defaults to `"silu"`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + double_z (`bool`, *optional*, defaults to `True`): + Whether to double the number of output channels for the last block. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str, ...] = ("DownEncoderBlock2D",), + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + double_z: bool = True, + mid_block_add_attention=True, + ): + super().__init__() + self.layers_per_block = layers_per_block + + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[0], + kernel_size=3, + stride=1, + padding=1, + ) + + self.mid_block = None + self.down_blocks = nn.ModuleList([]) + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=self.layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + add_downsample=not is_final_block, + resnet_eps=1e-6, + downsample_padding=0, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attention_head_dim=output_channel, + temb_channels=None, + ) + self.down_blocks.append(down_block) + + # mid + self.mid_block = UNetMidBlock2D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + output_scale_factor=1, + resnet_time_scale_shift="default", + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + temb_channels=None, + add_attention=mid_block_add_attention, + ) + + # out + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[-1], num_groups=norm_num_groups, eps=1e-6) + self.conv_act = nn.SiLU() + + conv_out_channels = 2 * out_channels if double_z else out_channels + self.conv_out = nn.Conv2d(block_out_channels[-1], conv_out_channels, 3, padding=1) + + self.gradient_checkpointing = False + + def forward(self, sample: torch.FloatTensor) -> torch.FloatTensor: + r"""The forward method of the `Encoder` class.""" + + sample = self.conv_in(sample) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + # down + if is_torch_version(">=", "1.11.0"): + for down_block in self.down_blocks: + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(down_block), sample, use_reentrant=False + ) + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), sample, use_reentrant=False + ) + else: + for down_block in self.down_blocks: + sample = torch.utils.checkpoint.checkpoint(create_custom_forward(down_block), sample) + # middle + sample = torch.utils.checkpoint.checkpoint(create_custom_forward(self.mid_block), sample) + + else: + # down + for down_block in self.down_blocks: + sample = down_block(sample) + + # middle + sample = self.mid_block(sample) + + # post-process + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + return sample + + +class Decoder(nn.Module): + r""" + The `Decoder` layer of a variational autoencoder that decodes its latent representation into an output sample. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + up_block_types (`Tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + The types of up blocks to use. See `~diffusers.models.unet_2d_blocks.get_up_block` for available options. + block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + The number of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups for normalization. + act_fn (`str`, *optional*, defaults to `"silu"`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + norm_type (`str`, *optional*, defaults to `"group"`): + The normalization type to use. Can be either `"group"` or `"spatial"`. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + up_block_types: Tuple[str, ...] = ("UpDecoderBlock2D",), + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + norm_type: str = "group", # group, spatial + mid_block_add_attention=True, + ): + super().__init__() + self.layers_per_block = layers_per_block + + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[-1], + kernel_size=3, + stride=1, + padding=1, + ) + + self.mid_block = None + self.up_blocks = nn.ModuleList([]) + + temb_channels = in_channels if norm_type == "spatial" else None + + # mid + self.mid_block = UNetMidBlock2D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + output_scale_factor=1, + resnet_time_scale_shift="default" if norm_type == "group" else norm_type, + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + temb_channels=temb_channels, + add_attention=mid_block_add_attention, + ) + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + + is_final_block = i == len(block_out_channels) - 1 + + up_block = get_up_block( + up_block_type, + num_layers=self.layers_per_block + 1, + in_channels=prev_output_channel, + out_channels=output_channel, + prev_output_channel=None, + add_upsample=not is_final_block, + resnet_eps=1e-6, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attention_head_dim=output_channel, + temb_channels=temb_channels, + resnet_time_scale_shift=norm_type, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_type == "spatial": + self.conv_norm_out = SpatialNorm(block_out_channels[0], temb_channels) + else: + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6) + self.conv_act = nn.SiLU() + self.conv_out = nn.Conv2d(block_out_channels[0], out_channels, 3, padding=1) + + self.gradient_checkpointing = False + + def forward( + self, + sample: torch.FloatTensor, + latent_embeds: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + r"""The forward method of the `Decoder` class.""" + + sample = self.conv_in(sample) + + upscale_dtype = next(iter(self.up_blocks.parameters())).dtype + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), + sample, + latent_embeds, + use_reentrant=False, + ) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(up_block), + sample, + latent_embeds, + use_reentrant=False, + ) + else: + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), sample, latent_embeds + ) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = torch.utils.checkpoint.checkpoint(create_custom_forward(up_block), sample, latent_embeds) + else: + # middle + sample = self.mid_block(sample, latent_embeds) + sample = sample.to(upscale_dtype) + + # up + for up_block in self.up_blocks: + sample = up_block(sample, latent_embeds) + + # post-process + if latent_embeds is None: + sample = self.conv_norm_out(sample) + else: + sample = self.conv_norm_out(sample, latent_embeds) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + return sample + + +class UpSample(nn.Module): + r""" + The `UpSample` layer of a variational autoencoder that upsamples its input. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + ) -> None: + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.deconv = nn.ConvTranspose2d(in_channels, out_channels, kernel_size=4, stride=2, padding=1) + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + r"""The forward method of the `UpSample` class.""" + x = torch.relu(x) + x = self.deconv(x) + return x + + +class MaskConditionEncoder(nn.Module): + """ + used in AsymmetricAutoencoderKL + """ + + def __init__( + self, + in_ch: int, + out_ch: int = 192, + res_ch: int = 768, + stride: int = 16, + ) -> None: + super().__init__() + + channels = [] + while stride > 1: + stride = stride // 2 + in_ch_ = out_ch * 2 + if out_ch > res_ch: + out_ch = res_ch + if stride == 1: + in_ch_ = res_ch + channels.append((in_ch_, out_ch)) + out_ch *= 2 + + out_channels = [] + for _in_ch, _out_ch in channels: + out_channels.append(_out_ch) + out_channels.append(channels[-1][0]) + + layers = [] + in_ch_ = in_ch + for l in range(len(out_channels)): + out_ch_ = out_channels[l] + if l == 0 or l == 1: + layers.append(nn.Conv2d(in_ch_, out_ch_, kernel_size=3, stride=1, padding=1)) + else: + layers.append(nn.Conv2d(in_ch_, out_ch_, kernel_size=4, stride=2, padding=1)) + in_ch_ = out_ch_ + + self.layers = nn.Sequential(*layers) + + def forward(self, x: torch.FloatTensor, mask=None) -> torch.FloatTensor: + r"""The forward method of the `MaskConditionEncoder` class.""" + out = {} + for l in range(len(self.layers)): + layer = self.layers[l] + x = layer(x) + out[str(tuple(x.shape))] = x + x = torch.relu(x) + return out + + +class MaskConditionDecoder(nn.Module): + r"""The `MaskConditionDecoder` should be used in combination with [`AsymmetricAutoencoderKL`] to enhance the model's + decoder with a conditioner on the mask and masked image. + + Args: + in_channels (`int`, *optional*, defaults to 3): + The number of input channels. + out_channels (`int`, *optional*, defaults to 3): + The number of output channels. + up_block_types (`Tuple[str, ...]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + The types of up blocks to use. See `~diffusers.models.unet_2d_blocks.get_up_block` for available options. + block_out_channels (`Tuple[int, ...]`, *optional*, defaults to `(64,)`): + The number of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups for normalization. + act_fn (`str`, *optional*, defaults to `"silu"`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + norm_type (`str`, *optional*, defaults to `"group"`): + The normalization type to use. Can be either `"group"` or `"spatial"`. + """ + + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + up_block_types: Tuple[str, ...] = ("UpDecoderBlock2D",), + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 2, + norm_num_groups: int = 32, + act_fn: str = "silu", + norm_type: str = "group", # group, spatial + ): + super().__init__() + self.layers_per_block = layers_per_block + + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[-1], + kernel_size=3, + stride=1, + padding=1, + ) + + self.mid_block = None + self.up_blocks = nn.ModuleList([]) + + temb_channels = in_channels if norm_type == "spatial" else None + + # mid + self.mid_block = UNetMidBlock2D( + in_channels=block_out_channels[-1], + resnet_eps=1e-6, + resnet_act_fn=act_fn, + output_scale_factor=1, + resnet_time_scale_shift="default" if norm_type == "group" else norm_type, + attention_head_dim=block_out_channels[-1], + resnet_groups=norm_num_groups, + temb_channels=temb_channels, + ) + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + + is_final_block = i == len(block_out_channels) - 1 + + up_block = get_up_block( + up_block_type, + num_layers=self.layers_per_block + 1, + in_channels=prev_output_channel, + out_channels=output_channel, + prev_output_channel=None, + add_upsample=not is_final_block, + resnet_eps=1e-6, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attention_head_dim=output_channel, + temb_channels=temb_channels, + resnet_time_scale_shift=norm_type, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # condition encoder + self.condition_encoder = MaskConditionEncoder( + in_ch=out_channels, + out_ch=block_out_channels[0], + res_ch=block_out_channels[-1], + ) + + # out + if norm_type == "spatial": + self.conv_norm_out = SpatialNorm(block_out_channels[0], temb_channels) + else: + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-6) + self.conv_act = nn.SiLU() + self.conv_out = nn.Conv2d(block_out_channels[0], out_channels, 3, padding=1) + + self.gradient_checkpointing = False + + def forward( + self, + z: torch.FloatTensor, + image: Optional[torch.FloatTensor] = None, + mask: Optional[torch.FloatTensor] = None, + latent_embeds: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + r"""The forward method of the `MaskConditionDecoder` class.""" + sample = z + sample = self.conv_in(sample) + + upscale_dtype = next(iter(self.up_blocks.parameters())).dtype + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), + sample, + latent_embeds, + use_reentrant=False, + ) + sample = sample.to(upscale_dtype) + + # condition encoder + if image is not None and mask is not None: + masked_image = (1 - mask) * image + im_x = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.condition_encoder), + masked_image, + mask, + use_reentrant=False, + ) + + # up + for up_block in self.up_blocks: + if image is not None and mask is not None: + sample_ = im_x[str(tuple(sample.shape))] + mask_ = nn.functional.interpolate(mask, size=sample.shape[-2:], mode="nearest") + sample = sample * mask_ + sample_ * (1 - mask_) + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(up_block), + sample, + latent_embeds, + use_reentrant=False, + ) + if image is not None and mask is not None: + sample = sample * mask + im_x[str(tuple(sample.shape))] * (1 - mask) + else: + # middle + sample = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.mid_block), sample, latent_embeds + ) + sample = sample.to(upscale_dtype) + + # condition encoder + if image is not None and mask is not None: + masked_image = (1 - mask) * image + im_x = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.condition_encoder), + masked_image, + mask, + ) + + # up + for up_block in self.up_blocks: + if image is not None and mask is not None: + sample_ = im_x[str(tuple(sample.shape))] + mask_ = nn.functional.interpolate(mask, size=sample.shape[-2:], mode="nearest") + sample = sample * mask_ + sample_ * (1 - mask_) + sample = torch.utils.checkpoint.checkpoint(create_custom_forward(up_block), sample, latent_embeds) + if image is not None and mask is not None: + sample = sample * mask + im_x[str(tuple(sample.shape))] * (1 - mask) + else: + # middle + sample = self.mid_block(sample, latent_embeds) + sample = sample.to(upscale_dtype) + + # condition encoder + if image is not None and mask is not None: + masked_image = (1 - mask) * image + im_x = self.condition_encoder(masked_image, mask) + + # up + for up_block in self.up_blocks: + if image is not None and mask is not None: + sample_ = im_x[str(tuple(sample.shape))] + mask_ = nn.functional.interpolate(mask, size=sample.shape[-2:], mode="nearest") + sample = sample * mask_ + sample_ * (1 - mask_) + sample = up_block(sample, latent_embeds) + if image is not None and mask is not None: + sample = sample * mask + im_x[str(tuple(sample.shape))] * (1 - mask) + + # post-process + if latent_embeds is None: + sample = self.conv_norm_out(sample) + else: + sample = self.conv_norm_out(sample, latent_embeds) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + return sample + + +class VectorQuantizer(nn.Module): + """ + Improved version over VectorQuantizer, can be used as a drop-in replacement. Mostly avoids costly matrix + multiplications and allows for post-hoc remapping of indices. + """ + + # NOTE: due to a bug the beta term was applied to the wrong term. for + # backwards compatibility we use the buggy version by default, but you can + # specify legacy=False to fix it. + def __init__( + self, + n_e: int, + vq_embed_dim: int, + beta: float, + remap=None, + unknown_index: str = "random", + sane_index_shape: bool = False, + legacy: bool = True, + ): + super().__init__() + self.n_e = n_e + self.vq_embed_dim = vq_embed_dim + self.beta = beta + self.legacy = legacy + + self.embedding = nn.Embedding(self.n_e, self.vq_embed_dim) + self.embedding.weight.data.uniform_(-1.0 / self.n_e, 1.0 / self.n_e) + + self.remap = remap + if self.remap is not None: + self.register_buffer("used", torch.tensor(np.load(self.remap))) + self.used: torch.Tensor + self.re_embed = self.used.shape[0] + self.unknown_index = unknown_index # "random" or "extra" or integer + if self.unknown_index == "extra": + self.unknown_index = self.re_embed + self.re_embed = self.re_embed + 1 + print( + f"Remapping {self.n_e} indices to {self.re_embed} indices. " + f"Using {self.unknown_index} for unknown indices." + ) + else: + self.re_embed = n_e + + self.sane_index_shape = sane_index_shape + + def remap_to_used(self, inds: torch.LongTensor) -> torch.LongTensor: + ishape = inds.shape + assert len(ishape) > 1 + inds = inds.reshape(ishape[0], -1) + used = self.used.to(inds) + match = (inds[:, :, None] == used[None, None, ...]).long() + new = match.argmax(-1) + unknown = match.sum(2) < 1 + if self.unknown_index == "random": + new[unknown] = torch.randint(0, self.re_embed, size=new[unknown].shape).to(device=new.device) + else: + new[unknown] = self.unknown_index + return new.reshape(ishape) + + def unmap_to_all(self, inds: torch.LongTensor) -> torch.LongTensor: + ishape = inds.shape + assert len(ishape) > 1 + inds = inds.reshape(ishape[0], -1) + used = self.used.to(inds) + if self.re_embed > self.used.shape[0]: # extra token + inds[inds >= self.used.shape[0]] = 0 # simply set to zero + back = torch.gather(used[None, :][inds.shape[0] * [0], :], 1, inds) + return back.reshape(ishape) + + def forward(self, z: torch.FloatTensor) -> Tuple[torch.FloatTensor, torch.FloatTensor, Tuple]: + # reshape z -> (batch, height, width, channel) and flatten + z = z.permute(0, 2, 3, 1).contiguous() + z_flattened = z.view(-1, self.vq_embed_dim) + + # distances from z to embeddings e_j (z - e)^2 = z^2 + e^2 - 2 e * z + min_encoding_indices = torch.argmin(torch.cdist(z_flattened, self.embedding.weight), dim=1) + + z_q = self.embedding(min_encoding_indices).view(z.shape) + perplexity = None + min_encodings = None + + # compute loss for embedding + if not self.legacy: + loss = self.beta * torch.mean((z_q.detach() - z) ** 2) + torch.mean((z_q - z.detach()) ** 2) + else: + loss = torch.mean((z_q.detach() - z) ** 2) + self.beta * torch.mean((z_q - z.detach()) ** 2) + + # preserve gradients + z_q: torch.FloatTensor = z + (z_q - z).detach() + + # reshape back to match original input shape + z_q = z_q.permute(0, 3, 1, 2).contiguous() + + if self.remap is not None: + min_encoding_indices = min_encoding_indices.reshape(z.shape[0], -1) # add batch axis + min_encoding_indices = self.remap_to_used(min_encoding_indices) + min_encoding_indices = min_encoding_indices.reshape(-1, 1) # flatten + + if self.sane_index_shape: + min_encoding_indices = min_encoding_indices.reshape(z_q.shape[0], z_q.shape[2], z_q.shape[3]) + + return z_q, loss, (perplexity, min_encodings, min_encoding_indices) + + def get_codebook_entry(self, indices: torch.LongTensor, shape: Tuple[int, ...]) -> torch.FloatTensor: + # shape specifying (batch, height, width, channel) + if self.remap is not None: + indices = indices.reshape(shape[0], -1) # add batch axis + indices = self.unmap_to_all(indices) + indices = indices.reshape(-1) # flatten again + + # get quantized latent vectors + z_q: torch.FloatTensor = self.embedding(indices) + + if shape is not None: + z_q = z_q.view(shape) + # reshape back to match original input shape + z_q = z_q.permute(0, 3, 1, 2).contiguous() + + return z_q + + +class DiagonalGaussianDistribution(object): + def __init__(self, parameters: torch.Tensor, deterministic: bool = False): + self.parameters = parameters + self.mean, self.logvar = torch.chunk(parameters, 2, dim=1) + self.logvar = torch.clamp(self.logvar, -30.0, 20.0) + self.deterministic = deterministic + self.std = torch.exp(0.5 * self.logvar) + self.var = torch.exp(self.logvar) + if self.deterministic: + self.var = self.std = torch.zeros_like( + self.mean, device=self.parameters.device, dtype=self.parameters.dtype + ) + + def sample(self, generator: Optional[torch.Generator] = None) -> torch.FloatTensor: + # make sure sample is on the same device as the parameters and has same dtype + sample = randn_tensor( + self.mean.shape, + generator=generator, + device=self.parameters.device, + dtype=self.parameters.dtype, + ) + x = self.mean + self.std * sample + return x + + def kl(self, other: "DiagonalGaussianDistribution" = None) -> torch.Tensor: + if self.deterministic: + return torch.Tensor([0.0]) + else: + if other is None: + return 0.5 * torch.sum( + torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar, + dim=[1, 2, 3], + ) + else: + return 0.5 * torch.sum( + torch.pow(self.mean - other.mean, 2) / other.var + + self.var / other.var + - 1.0 + - self.logvar + + other.logvar, + dim=[1, 2, 3], + ) + + def nll(self, sample: torch.Tensor, dims: Tuple[int, ...] = [1, 2, 3]) -> torch.Tensor: + if self.deterministic: + return torch.Tensor([0.0]) + logtwopi = np.log(2.0 * np.pi) + return 0.5 * torch.sum( + logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var, + dim=dims, + ) + + def mode(self) -> torch.Tensor: + return self.mean + + +class EncoderTiny(nn.Module): + r""" + The `EncoderTiny` layer is a simpler version of the `Encoder` layer. + + Args: + in_channels (`int`): + The number of input channels. + out_channels (`int`): + The number of output channels. + num_blocks (`Tuple[int, ...]`): + Each value of the tuple represents a Conv2d layer followed by `value` number of `AutoencoderTinyBlock`'s to + use. + block_out_channels (`Tuple[int, ...]`): + The number of output channels for each block. + act_fn (`str`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + num_blocks: Tuple[int, ...], + block_out_channels: Tuple[int, ...], + act_fn: str, + ): + super().__init__() + + layers = [] + for i, num_block in enumerate(num_blocks): + num_channels = block_out_channels[i] + + if i == 0: + layers.append(nn.Conv2d(in_channels, num_channels, kernel_size=3, padding=1)) + else: + layers.append( + nn.Conv2d( + num_channels, + num_channels, + kernel_size=3, + padding=1, + stride=2, + bias=False, + ) + ) + + for _ in range(num_block): + layers.append(AutoencoderTinyBlock(num_channels, num_channels, act_fn)) + + layers.append(nn.Conv2d(block_out_channels[-1], out_channels, kernel_size=3, padding=1)) + + self.layers = nn.Sequential(*layers) + self.gradient_checkpointing = False + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + r"""The forward method of the `EncoderTiny` class.""" + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.layers), x, use_reentrant=False) + else: + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.layers), x) + + else: + # scale image from [-1, 1] to [0, 1] to match TAESD convention + x = self.layers(x.add(1).div(2)) + + return x + + +class DecoderTiny(nn.Module): + r""" + The `DecoderTiny` layer is a simpler version of the `Decoder` layer. + + Args: + in_channels (`int`): + The number of input channels. + out_channels (`int`): + The number of output channels. + num_blocks (`Tuple[int, ...]`): + Each value of the tuple represents a Conv2d layer followed by `value` number of `AutoencoderTinyBlock`'s to + use. + block_out_channels (`Tuple[int, ...]`): + The number of output channels for each block. + upsampling_scaling_factor (`int`): + The scaling factor to use for upsampling. + act_fn (`str`): + The activation function to use. See `~diffusers.models.activations.get_activation` for available options. + """ + + def __init__( + self, + in_channels: int, + out_channels: int, + num_blocks: Tuple[int, ...], + block_out_channels: Tuple[int, ...], + upsampling_scaling_factor: int, + act_fn: str, + ): + super().__init__() + + layers = [ + nn.Conv2d(in_channels, block_out_channels[0], kernel_size=3, padding=1), + get_activation(act_fn), + ] + + for i, num_block in enumerate(num_blocks): + is_final_block = i == (len(num_blocks) - 1) + num_channels = block_out_channels[i] + + for _ in range(num_block): + layers.append(AutoencoderTinyBlock(num_channels, num_channels, act_fn)) + + if not is_final_block: + layers.append(nn.Upsample(scale_factor=upsampling_scaling_factor)) + + conv_out_channel = num_channels if not is_final_block else out_channels + layers.append( + nn.Conv2d( + num_channels, + conv_out_channel, + kernel_size=3, + padding=1, + bias=is_final_block, + ) + ) + + self.layers = nn.Sequential(*layers) + self.gradient_checkpointing = False + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + r"""The forward method of the `DecoderTiny` class.""" + # Clamp. + x = torch.tanh(x / 3) * 3 + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.layers), x, use_reentrant=False) + else: + x = torch.utils.checkpoint.checkpoint(create_custom_forward(self.layers), x) + + else: + x = self.layers(x) + + # scale image from [0, 1] to [-1, 1] to match diffusers convention + return x.mul(2).sub(1) diff --git a/diffusers-0.27.0/src/diffusers/models/controlnet.py b/diffusers-0.27.0/src/diffusers/models/controlnet.py new file mode 100755 index 0000000..130e643 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/controlnet.py @@ -0,0 +1,868 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +from torch import nn +from torch.nn import functional as F + +from ..configuration_utils import ConfigMixin, register_to_config +from ..loaders import FromOriginalControlNetMixin +from ..utils import BaseOutput, logging +from .attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from .embeddings import TextImageProjection, TextImageTimeEmbedding, TextTimeEmbedding, TimestepEmbedding, Timesteps +from .modeling_utils import ModelMixin +from .unets.unet_2d_blocks import ( + CrossAttnDownBlock2D, + DownBlock2D, + UNetMidBlock2D, + UNetMidBlock2DCrossAttn, + get_down_block, +) +from .unets.unet_2d_condition import UNet2DConditionModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class ControlNetOutput(BaseOutput): + """ + The output of [`ControlNetModel`]. + + Args: + down_block_res_samples (`tuple[torch.Tensor]`): + A tuple of downsample activations at different resolutions for each downsampling block. Each tensor should + be of shape `(batch_size, channel * resolution, height //resolution, width // resolution)`. Output can be + used to condition the original UNet's downsampling activations. + mid_down_block_re_sample (`torch.Tensor`): + The activation of the midde block (the lowest sample resolution). Each tensor should be of shape + `(batch_size, channel * lowest_resolution, height // lowest_resolution, width // lowest_resolution)`. + Output can be used to condition the original UNet's middle block activation. + """ + + down_block_res_samples: Tuple[torch.Tensor] + mid_block_res_sample: torch.Tensor + + +class ControlNetConditioningEmbedding(nn.Module): + """ + Quoting from https://arxiv.org/abs/2302.05543: "Stable Diffusion uses a pre-processing method similar to VQ-GAN + [11] to convert the entire dataset of 512 × 512 images into smaller 64 × 64 “latent images” for stabilized + training. This requires ControlNets to convert image-based conditions to 64 × 64 feature space to match the + convolution size. We use a tiny network E(·) of four convolution layers with 4 × 4 kernels and 2 × 2 strides + (activated by ReLU, channels are 16, 32, 64, 128, initialized with Gaussian weights, trained jointly with the full + model) to encode image-space conditions ... into feature maps ..." + """ + + def __init__( + self, + conditioning_embedding_channels: int, + conditioning_channels: int = 3, + block_out_channels: Tuple[int, ...] = (16, 32, 96, 256), + ): + super().__init__() + + self.conv_in = nn.Conv2d(conditioning_channels, block_out_channels[0], kernel_size=3, padding=1) + + self.blocks = nn.ModuleList([]) + + for i in range(len(block_out_channels) - 1): + channel_in = block_out_channels[i] + channel_out = block_out_channels[i + 1] + self.blocks.append(nn.Conv2d(channel_in, channel_in, kernel_size=3, padding=1)) + self.blocks.append(nn.Conv2d(channel_in, channel_out, kernel_size=3, padding=1, stride=2)) + + self.conv_out = zero_module( + nn.Conv2d(block_out_channels[-1], conditioning_embedding_channels, kernel_size=3, padding=1) + ) + + def forward(self, conditioning): + embedding = self.conv_in(conditioning) + embedding = F.silu(embedding) + + for block in self.blocks: + embedding = block(embedding) + embedding = F.silu(embedding) + + embedding = self.conv_out(embedding) + + return embedding + + +class ControlNetModel(ModelMixin, ConfigMixin, FromOriginalControlNetMixin): + """ + A ControlNet model. + + Args: + in_channels (`int`, defaults to 4): + The number of channels in the input sample. + flip_sin_to_cos (`bool`, defaults to `True`): + Whether to flip the sin to cos in the time embedding. + freq_shift (`int`, defaults to 0): + The frequency shift to apply to the time embedding. + down_block_types (`tuple[str]`, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): + The tuple of downsample blocks to use. + only_cross_attention (`Union[bool, Tuple[bool]]`, defaults to `False`): + block_out_channels (`tuple[int]`, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, defaults to 2): + The number of layers per block. + downsample_padding (`int`, defaults to 1): + The padding to use for the downsampling convolution. + mid_block_scale_factor (`float`, defaults to 1): + The scale factor to use for the mid block. + act_fn (`str`, defaults to "silu"): + The activation function to use. + norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups to use for the normalization. If None, normalization and activation layers is skipped + in post-processing. + norm_eps (`float`, defaults to 1e-5): + The epsilon to use for the normalization. + cross_attention_dim (`int`, defaults to 1280): + The dimension of the cross attention features. + transformer_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 1): + The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for + [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], + [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. + encoder_hid_dim (`int`, *optional*, defaults to None): + If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim` + dimension to `cross_attention_dim`. + encoder_hid_dim_type (`str`, *optional*, defaults to `None`): + If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text + embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`. + attention_head_dim (`Union[int, Tuple[int]]`, defaults to 8): + The dimension of the attention heads. + use_linear_projection (`bool`, defaults to `False`): + class_embed_type (`str`, *optional*, defaults to `None`): + The type of class embedding to use which is ultimately summed with the time embeddings. Choose from None, + `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`. + addition_embed_type (`str`, *optional*, defaults to `None`): + Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or + "text". "text" will use the `TextTimeEmbedding` layer. + num_class_embeds (`int`, *optional*, defaults to 0): + Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing + class conditioning with `class_embed_type` equal to `None`. + upcast_attention (`bool`, defaults to `False`): + resnet_time_scale_shift (`str`, defaults to `"default"`): + Time scale shift config for ResNet blocks (see `ResnetBlock2D`). Choose from `default` or `scale_shift`. + projection_class_embeddings_input_dim (`int`, *optional*, defaults to `None`): + The dimension of the `class_labels` input when `class_embed_type="projection"`. Required when + `class_embed_type="projection"`. + controlnet_conditioning_channel_order (`str`, defaults to `"rgb"`): + The channel order of conditional image. Will convert to `rgb` if it's `bgr`. + conditioning_embedding_out_channels (`tuple[int]`, *optional*, defaults to `(16, 32, 96, 256)`): + The tuple of output channel for each block in the `conditioning_embedding` layer. + global_pool_conditions (`bool`, defaults to `False`): + TODO(Patrick) - unused parameter. + addition_embed_type_num_heads (`int`, defaults to 64): + The number of heads to use for the `TextTimeEmbedding` layer. + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + in_channels: int = 4, + conditioning_channels: int = 3, + flip_sin_to_cos: bool = True, + freq_shift: int = 0, + down_block_types: Tuple[str, ...] = ( + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "DownBlock2D", + ), + mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn", + only_cross_attention: Union[bool, Tuple[bool]] = False, + block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + layers_per_block: int = 2, + downsample_padding: int = 1, + mid_block_scale_factor: float = 1, + act_fn: str = "silu", + norm_num_groups: Optional[int] = 32, + norm_eps: float = 1e-5, + cross_attention_dim: int = 1280, + transformer_layers_per_block: Union[int, Tuple[int, ...]] = 1, + encoder_hid_dim: Optional[int] = None, + encoder_hid_dim_type: Optional[str] = None, + attention_head_dim: Union[int, Tuple[int, ...]] = 8, + num_attention_heads: Optional[Union[int, Tuple[int, ...]]] = None, + use_linear_projection: bool = False, + class_embed_type: Optional[str] = None, + addition_embed_type: Optional[str] = None, + addition_time_embed_dim: Optional[int] = None, + num_class_embeds: Optional[int] = None, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + projection_class_embeddings_input_dim: Optional[int] = None, + controlnet_conditioning_channel_order: str = "rgb", + conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (16, 32, 96, 256), + global_pool_conditions: bool = False, + addition_embed_type_num_heads: int = 64, + ): + super().__init__() + + # If `num_attention_heads` is not defined (which is the case for most models) + # it will default to `attention_head_dim`. This looks weird upon first reading it and it is. + # The reason for this behavior is to correct for incorrectly named variables that were introduced + # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131 + # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking + # which is why we correct for the naming here. + num_attention_heads = num_attention_heads or attention_head_dim + + # Check inputs + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(only_cross_attention, bool) and len(only_cross_attention) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}." + ) + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * len(down_block_types) + + # input + conv_in_kernel = 3 + conv_in_padding = (conv_in_kernel - 1) // 2 + self.conv_in = nn.Conv2d( + in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding + ) + + # time + time_embed_dim = block_out_channels[0] * 4 + self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift) + timestep_input_dim = block_out_channels[0] + self.time_embedding = TimestepEmbedding( + timestep_input_dim, + time_embed_dim, + act_fn=act_fn, + ) + + if encoder_hid_dim_type is None and encoder_hid_dim is not None: + encoder_hid_dim_type = "text_proj" + self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type) + logger.info("encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined.") + + if encoder_hid_dim is None and encoder_hid_dim_type is not None: + raise ValueError( + f"`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}." + ) + + if encoder_hid_dim_type == "text_proj": + self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim) + elif encoder_hid_dim_type == "text_image_proj": + # image_embed_dim DOESN'T have to be `cross_attention_dim`. To not clutter the __init__ too much + # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use + # case when `addition_embed_type == "text_image_proj"` (Kadinsky 2.1)` + self.encoder_hid_proj = TextImageProjection( + text_embed_dim=encoder_hid_dim, + image_embed_dim=cross_attention_dim, + cross_attention_dim=cross_attention_dim, + ) + + elif encoder_hid_dim_type is not None: + raise ValueError( + f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'." + ) + else: + self.encoder_hid_proj = None + + # class embedding + if class_embed_type is None and num_class_embeds is not None: + self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim) + elif class_embed_type == "timestep": + self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim) + elif class_embed_type == "identity": + self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim) + elif class_embed_type == "projection": + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set" + ) + # The projection `class_embed_type` is the same as the timestep `class_embed_type` except + # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings + # 2. it projects from an arbitrary input dimension. + # + # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations. + # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings. + # As a result, `TimestepEmbedding` can be passed arbitrary vectors. + self.class_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim) + else: + self.class_embedding = None + + if addition_embed_type == "text": + if encoder_hid_dim is not None: + text_time_embedding_from_dim = encoder_hid_dim + else: + text_time_embedding_from_dim = cross_attention_dim + + self.add_embedding = TextTimeEmbedding( + text_time_embedding_from_dim, time_embed_dim, num_heads=addition_embed_type_num_heads + ) + elif addition_embed_type == "text_image": + # text_embed_dim and image_embed_dim DON'T have to be `cross_attention_dim`. To not clutter the __init__ too much + # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use + # case when `addition_embed_type == "text_image"` (Kadinsky 2.1)` + self.add_embedding = TextImageTimeEmbedding( + text_embed_dim=cross_attention_dim, image_embed_dim=cross_attention_dim, time_embed_dim=time_embed_dim + ) + elif addition_embed_type == "text_time": + self.add_time_proj = Timesteps(addition_time_embed_dim, flip_sin_to_cos, freq_shift) + self.add_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim) + + elif addition_embed_type is not None: + raise ValueError(f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'.") + + # control net conditioning embedding + self.controlnet_cond_embedding = ControlNetConditioningEmbedding( + conditioning_embedding_channels=block_out_channels[0], + block_out_channels=conditioning_embedding_out_channels, + conditioning_channels=conditioning_channels, + ) + + self.down_blocks = nn.ModuleList([]) + self.controlnet_down_blocks = nn.ModuleList([]) + + if isinstance(only_cross_attention, bool): + only_cross_attention = [only_cross_attention] * len(down_block_types) + + if isinstance(attention_head_dim, int): + attention_head_dim = (attention_head_dim,) * len(down_block_types) + + if isinstance(num_attention_heads, int): + num_attention_heads = (num_attention_heads,) * len(down_block_types) + + # down + output_channel = block_out_channels[0] + + controlnet_block = nn.Conv2d(output_channel, output_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_down_blocks.append(controlnet_block) + + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block, + transformer_layers_per_block=transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + temb_channels=time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads[i], + attention_head_dim=attention_head_dim[i] if attention_head_dim[i] is not None else output_channel, + downsample_padding=downsample_padding, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + self.down_blocks.append(down_block) + + for _ in range(layers_per_block): + controlnet_block = nn.Conv2d(output_channel, output_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_down_blocks.append(controlnet_block) + + if not is_final_block: + controlnet_block = nn.Conv2d(output_channel, output_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_down_blocks.append(controlnet_block) + + # mid + mid_block_channel = block_out_channels[-1] + + controlnet_block = nn.Conv2d(mid_block_channel, mid_block_channel, kernel_size=1) + controlnet_block = zero_module(controlnet_block) + self.controlnet_mid_block = controlnet_block + + if mid_block_type == "UNetMidBlock2DCrossAttn": + self.mid_block = UNetMidBlock2DCrossAttn( + transformer_layers_per_block=transformer_layers_per_block[-1], + in_channels=mid_block_channel, + temb_channels=time_embed_dim, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_time_scale_shift=resnet_time_scale_shift, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads[-1], + resnet_groups=norm_num_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + ) + elif mid_block_type == "UNetMidBlock2D": + self.mid_block = UNetMidBlock2D( + in_channels=block_out_channels[-1], + temb_channels=time_embed_dim, + num_layers=0, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_groups=norm_num_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + add_attention=False, + ) + else: + raise ValueError(f"unknown mid_block_type : {mid_block_type}") + + @classmethod + def from_unet( + cls, + unet: UNet2DConditionModel, + controlnet_conditioning_channel_order: str = "rgb", + conditioning_embedding_out_channels: Optional[Tuple[int, ...]] = (16, 32, 96, 256), + load_weights_from_unet: bool = True, + conditioning_channels: int = 3, + ): + r""" + Instantiate a [`ControlNetModel`] from [`UNet2DConditionModel`]. + + Parameters: + unet (`UNet2DConditionModel`): + The UNet model weights to copy to the [`ControlNetModel`]. All configuration options are also copied + where applicable. + """ + transformer_layers_per_block = ( + unet.config.transformer_layers_per_block if "transformer_layers_per_block" in unet.config else 1 + ) + encoder_hid_dim = unet.config.encoder_hid_dim if "encoder_hid_dim" in unet.config else None + encoder_hid_dim_type = unet.config.encoder_hid_dim_type if "encoder_hid_dim_type" in unet.config else None + addition_embed_type = unet.config.addition_embed_type if "addition_embed_type" in unet.config else None + addition_time_embed_dim = ( + unet.config.addition_time_embed_dim if "addition_time_embed_dim" in unet.config else None + ) + + controlnet = cls( + encoder_hid_dim=encoder_hid_dim, + encoder_hid_dim_type=encoder_hid_dim_type, + addition_embed_type=addition_embed_type, + addition_time_embed_dim=addition_time_embed_dim, + transformer_layers_per_block=transformer_layers_per_block, + in_channels=unet.config.in_channels, + flip_sin_to_cos=unet.config.flip_sin_to_cos, + freq_shift=unet.config.freq_shift, + down_block_types=unet.config.down_block_types, + only_cross_attention=unet.config.only_cross_attention, + block_out_channels=unet.config.block_out_channels, + layers_per_block=unet.config.layers_per_block, + downsample_padding=unet.config.downsample_padding, + mid_block_scale_factor=unet.config.mid_block_scale_factor, + act_fn=unet.config.act_fn, + norm_num_groups=unet.config.norm_num_groups, + norm_eps=unet.config.norm_eps, + cross_attention_dim=unet.config.cross_attention_dim, + attention_head_dim=unet.config.attention_head_dim, + num_attention_heads=unet.config.num_attention_heads, + use_linear_projection=unet.config.use_linear_projection, + class_embed_type=unet.config.class_embed_type, + num_class_embeds=unet.config.num_class_embeds, + upcast_attention=unet.config.upcast_attention, + resnet_time_scale_shift=unet.config.resnet_time_scale_shift, + projection_class_embeddings_input_dim=unet.config.projection_class_embeddings_input_dim, + mid_block_type=unet.config.mid_block_type, + controlnet_conditioning_channel_order=controlnet_conditioning_channel_order, + conditioning_embedding_out_channels=conditioning_embedding_out_channels, + conditioning_channels=conditioning_channels, + ) + + if load_weights_from_unet: + controlnet.conv_in.load_state_dict(unet.conv_in.state_dict()) + controlnet.time_proj.load_state_dict(unet.time_proj.state_dict()) + controlnet.time_embedding.load_state_dict(unet.time_embedding.state_dict()) + + if controlnet.class_embedding: + controlnet.class_embedding.load_state_dict(unet.class_embedding.state_dict()) + + controlnet.down_blocks.load_state_dict(unet.down_blocks.state_dict()) + controlnet.mid_block.load_state_dict(unet.mid_block.state_dict()) + + return controlnet + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attention_slice + def set_attention_slice(self, slice_size: Union[str, int, List[int]]) -> None: + r""" + Enable sliced attention computation. + + When this option is enabled, the attention module splits the input tensor in slices to compute attention in + several steps. This is useful for saving some memory in exchange for a small decrease in speed. + + Args: + slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`): + When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If + `"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is + provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim` + must be a multiple of `slice_size`. + """ + sliceable_head_dims = [] + + def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): + if hasattr(module, "set_attention_slice"): + sliceable_head_dims.append(module.sliceable_head_dim) + + for child in module.children(): + fn_recursive_retrieve_sliceable_dims(child) + + # retrieve number of attention layers + for module in self.children(): + fn_recursive_retrieve_sliceable_dims(module) + + num_sliceable_layers = len(sliceable_head_dims) + + if slice_size == "auto": + # half the attention head size is usually a good trade-off between + # speed and memory + slice_size = [dim // 2 for dim in sliceable_head_dims] + elif slice_size == "max": + # make smallest slice possible + slice_size = num_sliceable_layers * [1] + + slice_size = num_sliceable_layers * [slice_size] if not isinstance(slice_size, list) else slice_size + + if len(slice_size) != len(sliceable_head_dims): + raise ValueError( + f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different" + f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}." + ) + + for i in range(len(slice_size)): + size = slice_size[i] + dim = sliceable_head_dims[i] + if size is not None and size > dim: + raise ValueError(f"size {size} has to be smaller or equal to {dim}.") + + # Recursively walk through all the children. + # Any children which exposes the set_attention_slice method + # gets the message + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + if hasattr(module, "set_attention_slice"): + module.set_attention_slice(slice_size.pop()) + + for child in module.children(): + fn_recursive_set_attention_slice(child, slice_size) + + reversed_slice_size = list(reversed(slice_size)) + for module in self.children(): + fn_recursive_set_attention_slice(module, reversed_slice_size) + + def _set_gradient_checkpointing(self, module, value: bool = False) -> None: + if isinstance(module, (CrossAttnDownBlock2D, DownBlock2D)): + module.gradient_checkpointing = value + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + controlnet_cond: torch.FloatTensor, + conditioning_scale: float = 1.0, + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guess_mode: bool = False, + return_dict: bool = True, + ) -> Union[ControlNetOutput, Tuple[Tuple[torch.FloatTensor, ...], torch.FloatTensor]]: + """ + The [`ControlNetModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor. + timestep (`Union[torch.Tensor, float, int]`): + The number of timesteps to denoise an input. + encoder_hidden_states (`torch.Tensor`): + The encoder hidden states. + controlnet_cond (`torch.FloatTensor`): + The conditional input tensor of shape `(batch_size, sequence_length, hidden_size)`. + conditioning_scale (`float`, defaults to `1.0`): + The scale factor for ControlNet outputs. + class_labels (`torch.Tensor`, *optional*, defaults to `None`): + Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings. + timestep_cond (`torch.Tensor`, *optional*, defaults to `None`): + Additional conditional embeddings for timestep. If provided, the embeddings will be summed with the + timestep_embedding passed through the `self.time_embedding` layer to obtain the final timestep + embeddings. + attention_mask (`torch.Tensor`, *optional*, defaults to `None`): + An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask + is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large + negative values to the attention scores corresponding to "discard" tokens. + added_cond_kwargs (`dict`): + Additional conditions for the Stable Diffusion XL UNet. + cross_attention_kwargs (`dict[str]`, *optional*, defaults to `None`): + A kwargs dictionary that if specified is passed along to the `AttnProcessor`. + guess_mode (`bool`, defaults to `False`): + In this mode, the ControlNet encoder tries its best to recognize the input content of the input even if + you remove all prompts. A `guidance_scale` between 3.0 and 5.0 is recommended. + return_dict (`bool`, defaults to `True`): + Whether or not to return a [`~models.controlnet.ControlNetOutput`] instead of a plain tuple. + + Returns: + [`~models.controlnet.ControlNetOutput`] **or** `tuple`: + If `return_dict` is `True`, a [`~models.controlnet.ControlNetOutput`] is returned, otherwise a tuple is + returned where the first element is the sample tensor. + """ + # check channel order + channel_order = self.config.controlnet_conditioning_channel_order + + if channel_order == "rgb": + # in rgb order by default + ... + elif channel_order == "bgr": + controlnet_cond = torch.flip(controlnet_cond, dims=[1]) + else: + raise ValueError(f"unknown `controlnet_conditioning_channel_order`: {channel_order}") + + # prepare attention_mask + if attention_mask is not None: + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == "mps" + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # timesteps does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=sample.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + aug_emb = None + + if self.class_embedding is not None: + if class_labels is None: + raise ValueError("class_labels should be provided when num_class_embeds > 0") + + if self.config.class_embed_type == "timestep": + class_labels = self.time_proj(class_labels) + + class_emb = self.class_embedding(class_labels).to(dtype=self.dtype) + emb = emb + class_emb + + if self.config.addition_embed_type is not None: + if self.config.addition_embed_type == "text": + aug_emb = self.add_embedding(encoder_hidden_states) + + elif self.config.addition_embed_type == "text_time": + if "text_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`" + ) + text_embeds = added_cond_kwargs.get("text_embeds") + if "time_ids" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`" + ) + time_ids = added_cond_kwargs.get("time_ids") + time_embeds = self.add_time_proj(time_ids.flatten()) + time_embeds = time_embeds.reshape((text_embeds.shape[0], -1)) + + add_embeds = torch.concat([text_embeds, time_embeds], dim=-1) + add_embeds = add_embeds.to(emb.dtype) + aug_emb = self.add_embedding(add_embeds) + + emb = emb + aug_emb if aug_emb is not None else emb + + # 2. pre-process + sample = self.conv_in(sample) + + controlnet_cond = self.controlnet_cond_embedding(controlnet_cond) + sample = sample + controlnet_cond + + # 3. down + down_block_res_samples = (sample,) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb) + + down_block_res_samples += res_samples + + # 4. mid + if self.mid_block is not None: + if hasattr(self.mid_block, "has_cross_attention") and self.mid_block.has_cross_attention: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample = self.mid_block(sample, emb) + + # 5. Control net blocks + + controlnet_down_block_res_samples = () + + for down_block_res_sample, controlnet_block in zip(down_block_res_samples, self.controlnet_down_blocks): + down_block_res_sample = controlnet_block(down_block_res_sample) + controlnet_down_block_res_samples = controlnet_down_block_res_samples + (down_block_res_sample,) + + down_block_res_samples = controlnet_down_block_res_samples + + mid_block_res_sample = self.controlnet_mid_block(sample) + + # 6. scaling + if guess_mode and not self.config.global_pool_conditions: + scales = torch.logspace(-1, 0, len(down_block_res_samples) + 1, device=sample.device) # 0.1 to 1.0 + scales = scales * conditioning_scale + down_block_res_samples = [sample * scale for sample, scale in zip(down_block_res_samples, scales)] + mid_block_res_sample = mid_block_res_sample * scales[-1] # last one + else: + down_block_res_samples = [sample * conditioning_scale for sample in down_block_res_samples] + mid_block_res_sample = mid_block_res_sample * conditioning_scale + + if self.config.global_pool_conditions: + down_block_res_samples = [ + torch.mean(sample, dim=(2, 3), keepdim=True) for sample in down_block_res_samples + ] + mid_block_res_sample = torch.mean(mid_block_res_sample, dim=(2, 3), keepdim=True) + + if not return_dict: + return (down_block_res_samples, mid_block_res_sample) + + return ControlNetOutput( + down_block_res_samples=down_block_res_samples, mid_block_res_sample=mid_block_res_sample + ) + + +def zero_module(module): + for p in module.parameters(): + nn.init.zeros_(p) + return module diff --git a/diffusers-0.27.0/src/diffusers/models/controlnet_flax.py b/diffusers-0.27.0/src/diffusers/models/controlnet_flax.py new file mode 100755 index 0000000..6f9b201 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/controlnet_flax.py @@ -0,0 +1,395 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, Tuple, Union + +import flax +import flax.linen as nn +import jax +import jax.numpy as jnp +from flax.core.frozen_dict import FrozenDict + +from ..configuration_utils import ConfigMixin, flax_register_to_config +from ..utils import BaseOutput +from .embeddings_flax import FlaxTimestepEmbedding, FlaxTimesteps +from .modeling_flax_utils import FlaxModelMixin +from .unets.unet_2d_blocks_flax import ( + FlaxCrossAttnDownBlock2D, + FlaxDownBlock2D, + FlaxUNetMidBlock2DCrossAttn, +) + + +@flax.struct.dataclass +class FlaxControlNetOutput(BaseOutput): + """ + The output of [`FlaxControlNetModel`]. + + Args: + down_block_res_samples (`jnp.ndarray`): + mid_block_res_sample (`jnp.ndarray`): + """ + + down_block_res_samples: jnp.ndarray + mid_block_res_sample: jnp.ndarray + + +class FlaxControlNetConditioningEmbedding(nn.Module): + conditioning_embedding_channels: int + block_out_channels: Tuple[int, ...] = (16, 32, 96, 256) + dtype: jnp.dtype = jnp.float32 + + def setup(self) -> None: + self.conv_in = nn.Conv( + self.block_out_channels[0], + kernel_size=(3, 3), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + blocks = [] + for i in range(len(self.block_out_channels) - 1): + channel_in = self.block_out_channels[i] + channel_out = self.block_out_channels[i + 1] + conv1 = nn.Conv( + channel_in, + kernel_size=(3, 3), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + blocks.append(conv1) + conv2 = nn.Conv( + channel_out, + kernel_size=(3, 3), + strides=(2, 2), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + blocks.append(conv2) + self.blocks = blocks + + self.conv_out = nn.Conv( + self.conditioning_embedding_channels, + kernel_size=(3, 3), + padding=((1, 1), (1, 1)), + kernel_init=nn.initializers.zeros_init(), + bias_init=nn.initializers.zeros_init(), + dtype=self.dtype, + ) + + def __call__(self, conditioning: jnp.ndarray) -> jnp.ndarray: + embedding = self.conv_in(conditioning) + embedding = nn.silu(embedding) + + for block in self.blocks: + embedding = block(embedding) + embedding = nn.silu(embedding) + + embedding = self.conv_out(embedding) + + return embedding + + +@flax_register_to_config +class FlaxControlNetModel(nn.Module, FlaxModelMixin, ConfigMixin): + r""" + A ControlNet model. + + This model inherits from [`FlaxModelMixin`]. Check the superclass documentation for it’s generic methods + implemented for all models (such as downloading or saving). + + This model is also a Flax Linen [`flax.linen.Module`](https://flax.readthedocs.io/en/latest/flax.linen.html#module) + subclass. Use it as a regular Flax Linen module and refer to the Flax documentation for all matters related to its + general usage and behavior. + + Inherent JAX features such as the following are supported: + + - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit) + - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation) + - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap) + - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap) + + Parameters: + sample_size (`int`, *optional*): + The size of the input sample. + in_channels (`int`, *optional*, defaults to 4): + The number of channels in the input sample. + down_block_types (`Tuple[str]`, *optional*, defaults to `("FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxDownBlock2D")`): + The tuple of downsample blocks to use. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + attention_head_dim (`int` or `Tuple[int]`, *optional*, defaults to 8): + The dimension of the attention heads. + num_attention_heads (`int` or `Tuple[int]`, *optional*): + The number of attention heads. + cross_attention_dim (`int`, *optional*, defaults to 768): + The dimension of the cross attention features. + dropout (`float`, *optional*, defaults to 0): + Dropout probability for down, up and bottleneck blocks. + flip_sin_to_cos (`bool`, *optional*, defaults to `True`): + Whether to flip the sin to cos in the time embedding. + freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding. + controlnet_conditioning_channel_order (`str`, *optional*, defaults to `rgb`): + The channel order of conditional image. Will convert to `rgb` if it's `bgr`. + conditioning_embedding_out_channels (`tuple`, *optional*, defaults to `(16, 32, 96, 256)`): + The tuple of output channel for each block in the `conditioning_embedding` layer. + """ + + sample_size: int = 32 + in_channels: int = 4 + down_block_types: Tuple[str, ...] = ( + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "DownBlock2D", + ) + only_cross_attention: Union[bool, Tuple[bool, ...]] = False + block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280) + layers_per_block: int = 2 + attention_head_dim: Union[int, Tuple[int, ...]] = 8 + num_attention_heads: Optional[Union[int, Tuple[int, ...]]] = None + cross_attention_dim: int = 1280 + dropout: float = 0.0 + use_linear_projection: bool = False + dtype: jnp.dtype = jnp.float32 + flip_sin_to_cos: bool = True + freq_shift: int = 0 + controlnet_conditioning_channel_order: str = "rgb" + conditioning_embedding_out_channels: Tuple[int, ...] = (16, 32, 96, 256) + + def init_weights(self, rng: jax.Array) -> FrozenDict: + # init input tensors + sample_shape = (1, self.in_channels, self.sample_size, self.sample_size) + sample = jnp.zeros(sample_shape, dtype=jnp.float32) + timesteps = jnp.ones((1,), dtype=jnp.int32) + encoder_hidden_states = jnp.zeros((1, 1, self.cross_attention_dim), dtype=jnp.float32) + controlnet_cond_shape = (1, 3, self.sample_size * 8, self.sample_size * 8) + controlnet_cond = jnp.zeros(controlnet_cond_shape, dtype=jnp.float32) + + params_rng, dropout_rng = jax.random.split(rng) + rngs = {"params": params_rng, "dropout": dropout_rng} + + return self.init(rngs, sample, timesteps, encoder_hidden_states, controlnet_cond)["params"] + + def setup(self) -> None: + block_out_channels = self.block_out_channels + time_embed_dim = block_out_channels[0] * 4 + + # If `num_attention_heads` is not defined (which is the case for most models) + # it will default to `attention_head_dim`. This looks weird upon first reading it and it is. + # The reason for this behavior is to correct for incorrectly named variables that were introduced + # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131 + # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking + # which is why we correct for the naming here. + num_attention_heads = self.num_attention_heads or self.attention_head_dim + + # input + self.conv_in = nn.Conv( + block_out_channels[0], + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + # time + self.time_proj = FlaxTimesteps( + block_out_channels[0], flip_sin_to_cos=self.flip_sin_to_cos, freq_shift=self.config.freq_shift + ) + self.time_embedding = FlaxTimestepEmbedding(time_embed_dim, dtype=self.dtype) + + self.controlnet_cond_embedding = FlaxControlNetConditioningEmbedding( + conditioning_embedding_channels=block_out_channels[0], + block_out_channels=self.conditioning_embedding_out_channels, + ) + + only_cross_attention = self.only_cross_attention + if isinstance(only_cross_attention, bool): + only_cross_attention = (only_cross_attention,) * len(self.down_block_types) + + if isinstance(num_attention_heads, int): + num_attention_heads = (num_attention_heads,) * len(self.down_block_types) + + # down + down_blocks = [] + controlnet_down_blocks = [] + + output_channel = block_out_channels[0] + + controlnet_block = nn.Conv( + output_channel, + kernel_size=(1, 1), + padding="VALID", + kernel_init=nn.initializers.zeros_init(), + bias_init=nn.initializers.zeros_init(), + dtype=self.dtype, + ) + controlnet_down_blocks.append(controlnet_block) + + for i, down_block_type in enumerate(self.down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + if down_block_type == "CrossAttnDownBlock2D": + down_block = FlaxCrossAttnDownBlock2D( + in_channels=input_channel, + out_channels=output_channel, + dropout=self.dropout, + num_layers=self.layers_per_block, + num_attention_heads=num_attention_heads[i], + add_downsample=not is_final_block, + use_linear_projection=self.use_linear_projection, + only_cross_attention=only_cross_attention[i], + dtype=self.dtype, + ) + else: + down_block = FlaxDownBlock2D( + in_channels=input_channel, + out_channels=output_channel, + dropout=self.dropout, + num_layers=self.layers_per_block, + add_downsample=not is_final_block, + dtype=self.dtype, + ) + + down_blocks.append(down_block) + + for _ in range(self.layers_per_block): + controlnet_block = nn.Conv( + output_channel, + kernel_size=(1, 1), + padding="VALID", + kernel_init=nn.initializers.zeros_init(), + bias_init=nn.initializers.zeros_init(), + dtype=self.dtype, + ) + controlnet_down_blocks.append(controlnet_block) + + if not is_final_block: + controlnet_block = nn.Conv( + output_channel, + kernel_size=(1, 1), + padding="VALID", + kernel_init=nn.initializers.zeros_init(), + bias_init=nn.initializers.zeros_init(), + dtype=self.dtype, + ) + controlnet_down_blocks.append(controlnet_block) + + self.down_blocks = down_blocks + self.controlnet_down_blocks = controlnet_down_blocks + + # mid + mid_block_channel = block_out_channels[-1] + self.mid_block = FlaxUNetMidBlock2DCrossAttn( + in_channels=mid_block_channel, + dropout=self.dropout, + num_attention_heads=num_attention_heads[-1], + use_linear_projection=self.use_linear_projection, + dtype=self.dtype, + ) + + self.controlnet_mid_block = nn.Conv( + mid_block_channel, + kernel_size=(1, 1), + padding="VALID", + kernel_init=nn.initializers.zeros_init(), + bias_init=nn.initializers.zeros_init(), + dtype=self.dtype, + ) + + def __call__( + self, + sample: jnp.ndarray, + timesteps: Union[jnp.ndarray, float, int], + encoder_hidden_states: jnp.ndarray, + controlnet_cond: jnp.ndarray, + conditioning_scale: float = 1.0, + return_dict: bool = True, + train: bool = False, + ) -> Union[FlaxControlNetOutput, Tuple[Tuple[jnp.ndarray, ...], jnp.ndarray]]: + r""" + Args: + sample (`jnp.ndarray`): (batch, channel, height, width) noisy inputs tensor + timestep (`jnp.ndarray` or `float` or `int`): timesteps + encoder_hidden_states (`jnp.ndarray`): (batch_size, sequence_length, hidden_size) encoder hidden states + controlnet_cond (`jnp.ndarray`): (batch, channel, height, width) the conditional input tensor + conditioning_scale (`float`, *optional*, defaults to `1.0`): the scale factor for controlnet outputs + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`models.unets.unet_2d_condition_flax.FlaxUNet2DConditionOutput`] instead of a + plain tuple. + train (`bool`, *optional*, defaults to `False`): + Use deterministic functions and disable dropout when not training. + + Returns: + [`~models.unets.unet_2d_condition_flax.FlaxUNet2DConditionOutput`] or `tuple`: + [`~models.unets.unet_2d_condition_flax.FlaxUNet2DConditionOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is the sample tensor. + """ + channel_order = self.controlnet_conditioning_channel_order + if channel_order == "bgr": + controlnet_cond = jnp.flip(controlnet_cond, axis=1) + + # 1. time + if not isinstance(timesteps, jnp.ndarray): + timesteps = jnp.array([timesteps], dtype=jnp.int32) + elif isinstance(timesteps, jnp.ndarray) and len(timesteps.shape) == 0: + timesteps = timesteps.astype(dtype=jnp.float32) + timesteps = jnp.expand_dims(timesteps, 0) + + t_emb = self.time_proj(timesteps) + t_emb = self.time_embedding(t_emb) + + # 2. pre-process + sample = jnp.transpose(sample, (0, 2, 3, 1)) + sample = self.conv_in(sample) + + controlnet_cond = jnp.transpose(controlnet_cond, (0, 2, 3, 1)) + controlnet_cond = self.controlnet_cond_embedding(controlnet_cond) + sample += controlnet_cond + + # 3. down + down_block_res_samples = (sample,) + for down_block in self.down_blocks: + if isinstance(down_block, FlaxCrossAttnDownBlock2D): + sample, res_samples = down_block(sample, t_emb, encoder_hidden_states, deterministic=not train) + else: + sample, res_samples = down_block(sample, t_emb, deterministic=not train) + down_block_res_samples += res_samples + + # 4. mid + sample = self.mid_block(sample, t_emb, encoder_hidden_states, deterministic=not train) + + # 5. contronet blocks + controlnet_down_block_res_samples = () + for down_block_res_sample, controlnet_block in zip(down_block_res_samples, self.controlnet_down_blocks): + down_block_res_sample = controlnet_block(down_block_res_sample) + controlnet_down_block_res_samples += (down_block_res_sample,) + + down_block_res_samples = controlnet_down_block_res_samples + + mid_block_res_sample = self.controlnet_mid_block(sample) + + # 6. scaling + down_block_res_samples = [sample * conditioning_scale for sample in down_block_res_samples] + mid_block_res_sample *= conditioning_scale + + if not return_dict: + return (down_block_res_samples, mid_block_res_sample) + + return FlaxControlNetOutput( + down_block_res_samples=down_block_res_samples, mid_block_res_sample=mid_block_res_sample + ) diff --git a/diffusers-0.27.0/src/diffusers/models/downsampling.py b/diffusers-0.27.0/src/diffusers/models/downsampling.py new file mode 100755 index 0000000..9ae28e9 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/downsampling.py @@ -0,0 +1,334 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..utils import deprecate +from .normalization import RMSNorm +from .upsampling import upfirdn2d_native + + +class Downsample1D(nn.Module): + """A 1D downsampling layer with an optional convolution. + + Parameters: + channels (`int`): + number of channels in the inputs and outputs. + use_conv (`bool`, default `False`): + option to use a convolution. + out_channels (`int`, optional): + number of output channels. Defaults to `channels`. + padding (`int`, default `1`): + padding for the convolution. + name (`str`, default `conv`): + name of the downsampling 1D layer. + """ + + def __init__( + self, + channels: int, + use_conv: bool = False, + out_channels: Optional[int] = None, + padding: int = 1, + name: str = "conv", + ): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.padding = padding + stride = 2 + self.name = name + + if use_conv: + self.conv = nn.Conv1d(self.channels, self.out_channels, 3, stride=stride, padding=padding) + else: + assert self.channels == self.out_channels + self.conv = nn.AvgPool1d(kernel_size=stride, stride=stride) + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + assert inputs.shape[1] == self.channels + return self.conv(inputs) + + +class Downsample2D(nn.Module): + """A 2D downsampling layer with an optional convolution. + + Parameters: + channels (`int`): + number of channels in the inputs and outputs. + use_conv (`bool`, default `False`): + option to use a convolution. + out_channels (`int`, optional): + number of output channels. Defaults to `channels`. + padding (`int`, default `1`): + padding for the convolution. + name (`str`, default `conv`): + name of the downsampling 2D layer. + """ + + def __init__( + self, + channels: int, + use_conv: bool = False, + out_channels: Optional[int] = None, + padding: int = 1, + name: str = "conv", + kernel_size=3, + norm_type=None, + eps=None, + elementwise_affine=None, + bias=True, + ): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.padding = padding + stride = 2 + self.name = name + conv_cls = nn.Conv2d + + if norm_type == "ln_norm": + self.norm = nn.LayerNorm(channels, eps, elementwise_affine) + elif norm_type == "rms_norm": + self.norm = RMSNorm(channels, eps, elementwise_affine) + elif norm_type is None: + self.norm = None + else: + raise ValueError(f"unknown norm_type: {norm_type}") + + if use_conv: + conv = conv_cls( + self.channels, self.out_channels, kernel_size=kernel_size, stride=stride, padding=padding, bias=bias + ) + else: + assert self.channels == self.out_channels + conv = nn.AvgPool2d(kernel_size=stride, stride=stride) + + # TODO(Suraj, Patrick) - clean up after weight dicts are correctly renamed + if name == "conv": + self.Conv2d_0 = conv + self.conv = conv + elif name == "Conv2d_0": + self.conv = conv + else: + self.conv = conv + + def forward(self, hidden_states: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + assert hidden_states.shape[1] == self.channels + + if self.norm is not None: + hidden_states = self.norm(hidden_states.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) + + if self.use_conv and self.padding == 0: + pad = (0, 1, 0, 1) + hidden_states = F.pad(hidden_states, pad, mode="constant", value=0) + + assert hidden_states.shape[1] == self.channels + + hidden_states = self.conv(hidden_states) + + return hidden_states + + +class FirDownsample2D(nn.Module): + """A 2D FIR downsampling layer with an optional convolution. + + Parameters: + channels (`int`): + number of channels in the inputs and outputs. + use_conv (`bool`, default `False`): + option to use a convolution. + out_channels (`int`, optional): + number of output channels. Defaults to `channels`. + fir_kernel (`tuple`, default `(1, 3, 3, 1)`): + kernel for the FIR filter. + """ + + def __init__( + self, + channels: Optional[int] = None, + out_channels: Optional[int] = None, + use_conv: bool = False, + fir_kernel: Tuple[int, int, int, int] = (1, 3, 3, 1), + ): + super().__init__() + out_channels = out_channels if out_channels else channels + if use_conv: + self.Conv2d_0 = nn.Conv2d(channels, out_channels, kernel_size=3, stride=1, padding=1) + self.fir_kernel = fir_kernel + self.use_conv = use_conv + self.out_channels = out_channels + + def _downsample_2d( + self, + hidden_states: torch.FloatTensor, + weight: Optional[torch.FloatTensor] = None, + kernel: Optional[torch.FloatTensor] = None, + factor: int = 2, + gain: float = 1, + ) -> torch.FloatTensor: + """Fused `Conv2d()` followed by `downsample_2d()`. + Padding is performed only once at the beginning, not between the operations. The fused op is considerably more + efficient than performing the same calculation using standard TensorFlow ops. It supports gradients of + arbitrary order. + + Args: + hidden_states (`torch.FloatTensor`): + Input tensor of the shape `[N, C, H, W]` or `[N, H, W, C]`. + weight (`torch.FloatTensor`, *optional*): + Weight tensor of the shape `[filterH, filterW, inChannels, outChannels]`. Grouped convolution can be + performed by `inChannels = x.shape[0] // numGroups`. + kernel (`torch.FloatTensor`, *optional*): + FIR filter of the shape `[firH, firW]` or `[firN]` (separable). The default is `[1] * factor`, which + corresponds to average pooling. + factor (`int`, *optional*, default to `2`): + Integer downsampling factor. + gain (`float`, *optional*, default to `1.0`): + Scaling factor for signal magnitude. + + Returns: + output (`torch.FloatTensor`): + Tensor of the shape `[N, C, H // factor, W // factor]` or `[N, H // factor, W // factor, C]`, and same + datatype as `x`. + """ + + assert isinstance(factor, int) and factor >= 1 + if kernel is None: + kernel = [1] * factor + + # setup kernel + kernel = torch.tensor(kernel, dtype=torch.float32) + if kernel.ndim == 1: + kernel = torch.outer(kernel, kernel) + kernel /= torch.sum(kernel) + + kernel = kernel * gain + + if self.use_conv: + _, _, convH, convW = weight.shape + pad_value = (kernel.shape[0] - factor) + (convW - 1) + stride_value = [factor, factor] + upfirdn_input = upfirdn2d_native( + hidden_states, + torch.tensor(kernel, device=hidden_states.device), + pad=((pad_value + 1) // 2, pad_value // 2), + ) + output = F.conv2d(upfirdn_input, weight, stride=stride_value, padding=0) + else: + pad_value = kernel.shape[0] - factor + output = upfirdn2d_native( + hidden_states, + torch.tensor(kernel, device=hidden_states.device), + down=factor, + pad=((pad_value + 1) // 2, pad_value // 2), + ) + + return output + + def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + if self.use_conv: + downsample_input = self._downsample_2d(hidden_states, weight=self.Conv2d_0.weight, kernel=self.fir_kernel) + hidden_states = downsample_input + self.Conv2d_0.bias.reshape(1, -1, 1, 1) + else: + hidden_states = self._downsample_2d(hidden_states, kernel=self.fir_kernel, factor=2) + + return hidden_states + + +# downsample/upsample layer used in k-upscaler, might be able to use FirDownsample2D/DirUpsample2D instead +class KDownsample2D(nn.Module): + r"""A 2D K-downsampling layer. + + Parameters: + pad_mode (`str`, *optional*, default to `"reflect"`): the padding mode to use. + """ + + def __init__(self, pad_mode: str = "reflect"): + super().__init__() + self.pad_mode = pad_mode + kernel_1d = torch.tensor([[1 / 8, 3 / 8, 3 / 8, 1 / 8]]) + self.pad = kernel_1d.shape[1] // 2 - 1 + self.register_buffer("kernel", kernel_1d.T @ kernel_1d, persistent=False) + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + inputs = F.pad(inputs, (self.pad,) * 4, self.pad_mode) + weight = inputs.new_zeros( + [ + inputs.shape[1], + inputs.shape[1], + self.kernel.shape[0], + self.kernel.shape[1], + ] + ) + indices = torch.arange(inputs.shape[1], device=inputs.device) + kernel = self.kernel.to(weight)[None, :].expand(inputs.shape[1], -1, -1) + weight[indices, indices] = kernel + return F.conv2d(inputs, weight, stride=2) + + +def downsample_2d( + hidden_states: torch.FloatTensor, + kernel: Optional[torch.FloatTensor] = None, + factor: int = 2, + gain: float = 1, +) -> torch.FloatTensor: + r"""Downsample2D a batch of 2D images with the given filter. + Accepts a batch of 2D images of the shape `[N, C, H, W]` or `[N, H, W, C]` and downsamples each image with the + given filter. The filter is normalized so that if the input pixels are constant, they will be scaled by the + specified `gain`. Pixels outside the image are assumed to be zero, and the filter is padded with zeros so that its + shape is a multiple of the downsampling factor. + + Args: + hidden_states (`torch.FloatTensor`) + Input tensor of the shape `[N, C, H, W]` or `[N, H, W, C]`. + kernel (`torch.FloatTensor`, *optional*): + FIR filter of the shape `[firH, firW]` or `[firN]` (separable). The default is `[1] * factor`, which + corresponds to average pooling. + factor (`int`, *optional*, default to `2`): + Integer downsampling factor. + gain (`float`, *optional*, default to `1.0`): + Scaling factor for signal magnitude. + + Returns: + output (`torch.FloatTensor`): + Tensor of the shape `[N, C, H // factor, W // factor]` + """ + + assert isinstance(factor, int) and factor >= 1 + if kernel is None: + kernel = [1] * factor + + kernel = torch.tensor(kernel, dtype=torch.float32) + if kernel.ndim == 1: + kernel = torch.outer(kernel, kernel) + kernel /= torch.sum(kernel) + + kernel = kernel * gain + pad_value = kernel.shape[0] - factor + output = upfirdn2d_native( + hidden_states, + kernel.to(device=hidden_states.device), + down=factor, + pad=((pad_value + 1) // 2, pad_value // 2), + ) + return output diff --git a/diffusers-0.27.0/src/diffusers/models/dual_transformer_2d.py b/diffusers-0.27.0/src/diffusers/models/dual_transformer_2d.py new file mode 100755 index 0000000..b8e40f1 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/dual_transformer_2d.py @@ -0,0 +1,20 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ..utils import deprecate +from .transformers.dual_transformer_2d import DualTransformer2DModel + + +class DualTransformer2DModel(DualTransformer2DModel): + deprecation_message = "Importing `DualTransformer2DModel` from `diffusers.models.dual_transformer_2d` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.dual_transformer_2d import DualTransformer2DModel`, instead." + deprecate("DualTransformer2DModel", "0.29", deprecation_message) diff --git a/diffusers-0.27.0/src/diffusers/models/embeddings.py b/diffusers-0.27.0/src/diffusers/models/embeddings.py new file mode 100755 index 0000000..c15ff24 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/embeddings.py @@ -0,0 +1,914 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch +from torch import nn + +from ..utils import deprecate +from .activations import get_activation +from .attention_processor import Attention + + +def get_timestep_embedding( + timesteps: torch.Tensor, + embedding_dim: int, + flip_sin_to_cos: bool = False, + downscale_freq_shift: float = 1, + scale: float = 1, + max_period: int = 10000, +): + """ + This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings. + + :param timesteps: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + :param embedding_dim: the dimension of the output. :param max_period: controls the minimum frequency of the + embeddings. :return: an [N x dim] Tensor of positional embeddings. + """ + assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array" + + half_dim = embedding_dim // 2 + exponent = -math.log(max_period) * torch.arange( + start=0, end=half_dim, dtype=torch.float32, device=timesteps.device + ) + exponent = exponent / (half_dim - downscale_freq_shift) + + emb = torch.exp(exponent) + emb = timesteps[:, None].float() * emb[None, :] + + # scale embeddings + emb = scale * emb + + # concat sine and cosine embeddings + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1) + + # flip sine and cosine embeddings + if flip_sin_to_cos: + emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1) + + # zero pad + if embedding_dim % 2 == 1: + emb = torch.nn.functional.pad(emb, (0, 1, 0, 0)) + return emb + + +def get_2d_sincos_pos_embed( + embed_dim, grid_size, cls_token=False, extra_tokens=0, interpolation_scale=1.0, base_size=16 +): + """ + grid_size: int of the grid height and width return: pos_embed: [grid_size*grid_size, embed_dim] or + [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token) + """ + if isinstance(grid_size, int): + grid_size = (grid_size, grid_size) + + grid_h = np.arange(grid_size[0], dtype=np.float32) / (grid_size[0] / base_size) / interpolation_scale + grid_w = np.arange(grid_size[1], dtype=np.float32) / (grid_size[1] / base_size) / interpolation_scale + grid = np.meshgrid(grid_w, grid_h) # here w goes first + grid = np.stack(grid, axis=0) + + grid = grid.reshape([2, 1, grid_size[1], grid_size[0]]) + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid) + if cls_token and extra_tokens > 0: + pos_embed = np.concatenate([np.zeros([extra_tokens, embed_dim]), pos_embed], axis=0) + return pos_embed + + +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid): + if embed_dim % 2 != 0: + raise ValueError("embed_dim must be divisible by 2") + + # use half of dimensions to encode grid_h + emb_h = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[0]) # (H*W, D/2) + emb_w = get_1d_sincos_pos_embed_from_grid(embed_dim // 2, grid[1]) # (H*W, D/2) + + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D) + return emb + + +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos): + """ + embed_dim: output dimension for each position pos: a list of positions to be encoded: size (M,) out: (M, D) + """ + if embed_dim % 2 != 0: + raise ValueError("embed_dim must be divisible by 2") + + omega = np.arange(embed_dim // 2, dtype=np.float64) + omega /= embed_dim / 2.0 + omega = 1.0 / 10000**omega # (D/2,) + + pos = pos.reshape(-1) # (M,) + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product + + emb_sin = np.sin(out) # (M, D/2) + emb_cos = np.cos(out) # (M, D/2) + + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D) + return emb + + +class PatchEmbed(nn.Module): + """2D Image to Patch Embedding""" + + def __init__( + self, + height=224, + width=224, + patch_size=16, + in_channels=3, + embed_dim=768, + layer_norm=False, + flatten=True, + bias=True, + interpolation_scale=1, + ): + super().__init__() + + num_patches = (height // patch_size) * (width // patch_size) + self.flatten = flatten + self.layer_norm = layer_norm + + self.proj = nn.Conv2d( + in_channels, embed_dim, kernel_size=(patch_size, patch_size), stride=patch_size, bias=bias + ) + if layer_norm: + self.norm = nn.LayerNorm(embed_dim, elementwise_affine=False, eps=1e-6) + else: + self.norm = None + + self.patch_size = patch_size + # See: + # https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L161 + self.height, self.width = height // patch_size, width // patch_size + self.base_size = height // patch_size + self.interpolation_scale = interpolation_scale + pos_embed = get_2d_sincos_pos_embed( + embed_dim, int(num_patches**0.5), base_size=self.base_size, interpolation_scale=self.interpolation_scale + ) + self.register_buffer("pos_embed", torch.from_numpy(pos_embed).float().unsqueeze(0), persistent=False) + + def forward(self, latent): + height, width = latent.shape[-2] // self.patch_size, latent.shape[-1] // self.patch_size + + latent = self.proj(latent) + if self.flatten: + latent = latent.flatten(2).transpose(1, 2) # BCHW -> BNC + if self.layer_norm: + latent = self.norm(latent) + + # Interpolate positional embeddings if needed. + # (For PixArt-Alpha: https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L162C151-L162C160) + if self.height != height or self.width != width: + pos_embed = get_2d_sincos_pos_embed( + embed_dim=self.pos_embed.shape[-1], + grid_size=(height, width), + base_size=self.base_size, + interpolation_scale=self.interpolation_scale, + ) + pos_embed = torch.from_numpy(pos_embed) + pos_embed = pos_embed.float().unsqueeze(0).to(latent.device) + else: + pos_embed = self.pos_embed + + return (latent + pos_embed).to(latent.dtype) + + +class TimestepEmbedding(nn.Module): + def __init__( + self, + in_channels: int, + time_embed_dim: int, + act_fn: str = "silu", + out_dim: int = None, + post_act_fn: Optional[str] = None, + cond_proj_dim=None, + sample_proj_bias=True, + ): + super().__init__() + linear_cls = nn.Linear + + self.linear_1 = linear_cls(in_channels, time_embed_dim, sample_proj_bias) + + if cond_proj_dim is not None: + self.cond_proj = nn.Linear(cond_proj_dim, in_channels, bias=False) + else: + self.cond_proj = None + + self.act = get_activation(act_fn) + + if out_dim is not None: + time_embed_dim_out = out_dim + else: + time_embed_dim_out = time_embed_dim + self.linear_2 = linear_cls(time_embed_dim, time_embed_dim_out, sample_proj_bias) + + if post_act_fn is None: + self.post_act = None + else: + self.post_act = get_activation(post_act_fn) + + def forward(self, sample, condition=None): + if condition is not None: + sample = sample + self.cond_proj(condition) + sample = self.linear_1(sample) + + if self.act is not None: + sample = self.act(sample) + + sample = self.linear_2(sample) + + if self.post_act is not None: + sample = self.post_act(sample) + return sample + + +class Timesteps(nn.Module): + def __init__(self, num_channels: int, flip_sin_to_cos: bool, downscale_freq_shift: float): + super().__init__() + self.num_channels = num_channels + self.flip_sin_to_cos = flip_sin_to_cos + self.downscale_freq_shift = downscale_freq_shift + + def forward(self, timesteps): + t_emb = get_timestep_embedding( + timesteps, + self.num_channels, + flip_sin_to_cos=self.flip_sin_to_cos, + downscale_freq_shift=self.downscale_freq_shift, + ) + return t_emb + + +class GaussianFourierProjection(nn.Module): + """Gaussian Fourier embeddings for noise levels.""" + + def __init__( + self, embedding_size: int = 256, scale: float = 1.0, set_W_to_weight=True, log=True, flip_sin_to_cos=False + ): + super().__init__() + self.weight = nn.Parameter(torch.randn(embedding_size) * scale, requires_grad=False) + self.log = log + self.flip_sin_to_cos = flip_sin_to_cos + + if set_W_to_weight: + # to delete later + self.W = nn.Parameter(torch.randn(embedding_size) * scale, requires_grad=False) + + self.weight = self.W + + def forward(self, x): + if self.log: + x = torch.log(x) + + x_proj = x[:, None] * self.weight[None, :] * 2 * np.pi + + if self.flip_sin_to_cos: + out = torch.cat([torch.cos(x_proj), torch.sin(x_proj)], dim=-1) + else: + out = torch.cat([torch.sin(x_proj), torch.cos(x_proj)], dim=-1) + return out + + +class SinusoidalPositionalEmbedding(nn.Module): + """Apply positional information to a sequence of embeddings. + + Takes in a sequence of embeddings with shape (batch_size, seq_length, embed_dim) and adds positional embeddings to + them + + Args: + embed_dim: (int): Dimension of the positional embedding. + max_seq_length: Maximum sequence length to apply positional embeddings + + """ + + def __init__(self, embed_dim: int, max_seq_length: int = 32): + super().__init__() + position = torch.arange(max_seq_length).unsqueeze(1) + div_term = torch.exp(torch.arange(0, embed_dim, 2) * (-math.log(10000.0) / embed_dim)) + pe = torch.zeros(1, max_seq_length, embed_dim) + pe[0, :, 0::2] = torch.sin(position * div_term) + pe[0, :, 1::2] = torch.cos(position * div_term) + self.register_buffer("pe", pe) + + def forward(self, x): + _, seq_length, _ = x.shape + x = x + self.pe[:, :seq_length] + return x + + +class ImagePositionalEmbeddings(nn.Module): + """ + Converts latent image classes into vector embeddings. Sums the vector embeddings with positional embeddings for the + height and width of the latent space. + + For more details, see figure 10 of the dall-e paper: https://arxiv.org/abs/2102.12092 + + For VQ-diffusion: + + Output vector embeddings are used as input for the transformer. + + Note that the vector embeddings for the transformer are different than the vector embeddings from the VQVAE. + + Args: + num_embed (`int`): + Number of embeddings for the latent pixels embeddings. + height (`int`): + Height of the latent image i.e. the number of height embeddings. + width (`int`): + Width of the latent image i.e. the number of width embeddings. + embed_dim (`int`): + Dimension of the produced vector embeddings. Used for the latent pixel, height, and width embeddings. + """ + + def __init__( + self, + num_embed: int, + height: int, + width: int, + embed_dim: int, + ): + super().__init__() + + self.height = height + self.width = width + self.num_embed = num_embed + self.embed_dim = embed_dim + + self.emb = nn.Embedding(self.num_embed, embed_dim) + self.height_emb = nn.Embedding(self.height, embed_dim) + self.width_emb = nn.Embedding(self.width, embed_dim) + + def forward(self, index): + emb = self.emb(index) + + height_emb = self.height_emb(torch.arange(self.height, device=index.device).view(1, self.height)) + + # 1 x H x D -> 1 x H x 1 x D + height_emb = height_emb.unsqueeze(2) + + width_emb = self.width_emb(torch.arange(self.width, device=index.device).view(1, self.width)) + + # 1 x W x D -> 1 x 1 x W x D + width_emb = width_emb.unsqueeze(1) + + pos_emb = height_emb + width_emb + + # 1 x H x W x D -> 1 x L xD + pos_emb = pos_emb.view(1, self.height * self.width, -1) + + emb = emb + pos_emb[:, : emb.shape[1], :] + + return emb + + +class LabelEmbedding(nn.Module): + """ + Embeds class labels into vector representations. Also handles label dropout for classifier-free guidance. + + Args: + num_classes (`int`): The number of classes. + hidden_size (`int`): The size of the vector embeddings. + dropout_prob (`float`): The probability of dropping a label. + """ + + def __init__(self, num_classes, hidden_size, dropout_prob): + super().__init__() + use_cfg_embedding = dropout_prob > 0 + self.embedding_table = nn.Embedding(num_classes + use_cfg_embedding, hidden_size) + self.num_classes = num_classes + self.dropout_prob = dropout_prob + + def token_drop(self, labels, force_drop_ids=None): + """ + Drops labels to enable classifier-free guidance. + """ + if force_drop_ids is None: + drop_ids = torch.rand(labels.shape[0], device=labels.device) < self.dropout_prob + else: + drop_ids = torch.tensor(force_drop_ids == 1) + labels = torch.where(drop_ids, self.num_classes, labels) + return labels + + def forward(self, labels: torch.LongTensor, force_drop_ids=None): + use_dropout = self.dropout_prob > 0 + if (self.training and use_dropout) or (force_drop_ids is not None): + labels = self.token_drop(labels, force_drop_ids) + embeddings = self.embedding_table(labels) + return embeddings + + +class TextImageProjection(nn.Module): + def __init__( + self, + text_embed_dim: int = 1024, + image_embed_dim: int = 768, + cross_attention_dim: int = 768, + num_image_text_embeds: int = 10, + ): + super().__init__() + + self.num_image_text_embeds = num_image_text_embeds + self.image_embeds = nn.Linear(image_embed_dim, self.num_image_text_embeds * cross_attention_dim) + self.text_proj = nn.Linear(text_embed_dim, cross_attention_dim) + + def forward(self, text_embeds: torch.FloatTensor, image_embeds: torch.FloatTensor): + batch_size = text_embeds.shape[0] + + # image + image_text_embeds = self.image_embeds(image_embeds) + image_text_embeds = image_text_embeds.reshape(batch_size, self.num_image_text_embeds, -1) + + # text + text_embeds = self.text_proj(text_embeds) + + return torch.cat([image_text_embeds, text_embeds], dim=1) + + +class ImageProjection(nn.Module): + def __init__( + self, + image_embed_dim: int = 768, + cross_attention_dim: int = 768, + num_image_text_embeds: int = 32, + ): + super().__init__() + + self.num_image_text_embeds = num_image_text_embeds + self.image_embeds = nn.Linear(image_embed_dim, self.num_image_text_embeds * cross_attention_dim) + self.norm = nn.LayerNorm(cross_attention_dim) + + def forward(self, image_embeds: torch.FloatTensor): + batch_size = image_embeds.shape[0] + + # image + image_embeds = self.image_embeds(image_embeds) + image_embeds = image_embeds.reshape(batch_size, self.num_image_text_embeds, -1) + image_embeds = self.norm(image_embeds) + return image_embeds + + +class IPAdapterFullImageProjection(nn.Module): + def __init__(self, image_embed_dim=1024, cross_attention_dim=1024): + super().__init__() + from .attention import FeedForward + + self.ff = FeedForward(image_embed_dim, cross_attention_dim, mult=1, activation_fn="gelu") + self.norm = nn.LayerNorm(cross_attention_dim) + + def forward(self, image_embeds: torch.FloatTensor): + return self.norm(self.ff(image_embeds)) + + +class CombinedTimestepLabelEmbeddings(nn.Module): + def __init__(self, num_classes, embedding_dim, class_dropout_prob=0.1): + super().__init__() + + self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=1) + self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim) + self.class_embedder = LabelEmbedding(num_classes, embedding_dim, class_dropout_prob) + + def forward(self, timestep, class_labels, hidden_dtype=None): + timesteps_proj = self.time_proj(timestep) + timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_dtype)) # (N, D) + + class_labels = self.class_embedder(class_labels) # (N, D) + + conditioning = timesteps_emb + class_labels # (N, D) + + return conditioning + + +class TextTimeEmbedding(nn.Module): + def __init__(self, encoder_dim: int, time_embed_dim: int, num_heads: int = 64): + super().__init__() + self.norm1 = nn.LayerNorm(encoder_dim) + self.pool = AttentionPooling(num_heads, encoder_dim) + self.proj = nn.Linear(encoder_dim, time_embed_dim) + self.norm2 = nn.LayerNorm(time_embed_dim) + + def forward(self, hidden_states): + hidden_states = self.norm1(hidden_states) + hidden_states = self.pool(hidden_states) + hidden_states = self.proj(hidden_states) + hidden_states = self.norm2(hidden_states) + return hidden_states + + +class TextImageTimeEmbedding(nn.Module): + def __init__(self, text_embed_dim: int = 768, image_embed_dim: int = 768, time_embed_dim: int = 1536): + super().__init__() + self.text_proj = nn.Linear(text_embed_dim, time_embed_dim) + self.text_norm = nn.LayerNorm(time_embed_dim) + self.image_proj = nn.Linear(image_embed_dim, time_embed_dim) + + def forward(self, text_embeds: torch.FloatTensor, image_embeds: torch.FloatTensor): + # text + time_text_embeds = self.text_proj(text_embeds) + time_text_embeds = self.text_norm(time_text_embeds) + + # image + time_image_embeds = self.image_proj(image_embeds) + + return time_image_embeds + time_text_embeds + + +class ImageTimeEmbedding(nn.Module): + def __init__(self, image_embed_dim: int = 768, time_embed_dim: int = 1536): + super().__init__() + self.image_proj = nn.Linear(image_embed_dim, time_embed_dim) + self.image_norm = nn.LayerNorm(time_embed_dim) + + def forward(self, image_embeds: torch.FloatTensor): + # image + time_image_embeds = self.image_proj(image_embeds) + time_image_embeds = self.image_norm(time_image_embeds) + return time_image_embeds + + +class ImageHintTimeEmbedding(nn.Module): + def __init__(self, image_embed_dim: int = 768, time_embed_dim: int = 1536): + super().__init__() + self.image_proj = nn.Linear(image_embed_dim, time_embed_dim) + self.image_norm = nn.LayerNorm(time_embed_dim) + self.input_hint_block = nn.Sequential( + nn.Conv2d(3, 16, 3, padding=1), + nn.SiLU(), + nn.Conv2d(16, 16, 3, padding=1), + nn.SiLU(), + nn.Conv2d(16, 32, 3, padding=1, stride=2), + nn.SiLU(), + nn.Conv2d(32, 32, 3, padding=1), + nn.SiLU(), + nn.Conv2d(32, 96, 3, padding=1, stride=2), + nn.SiLU(), + nn.Conv2d(96, 96, 3, padding=1), + nn.SiLU(), + nn.Conv2d(96, 256, 3, padding=1, stride=2), + nn.SiLU(), + nn.Conv2d(256, 4, 3, padding=1), + ) + + def forward(self, image_embeds: torch.FloatTensor, hint: torch.FloatTensor): + # image + time_image_embeds = self.image_proj(image_embeds) + time_image_embeds = self.image_norm(time_image_embeds) + hint = self.input_hint_block(hint) + return time_image_embeds, hint + + +class AttentionPooling(nn.Module): + # Copied from https://github.com/deep-floyd/IF/blob/2f91391f27dd3c468bf174be5805b4cc92980c0b/deepfloyd_if/model/nn.py#L54 + + def __init__(self, num_heads, embed_dim, dtype=None): + super().__init__() + self.dtype = dtype + self.positional_embedding = nn.Parameter(torch.randn(1, embed_dim) / embed_dim**0.5) + self.k_proj = nn.Linear(embed_dim, embed_dim, dtype=self.dtype) + self.q_proj = nn.Linear(embed_dim, embed_dim, dtype=self.dtype) + self.v_proj = nn.Linear(embed_dim, embed_dim, dtype=self.dtype) + self.num_heads = num_heads + self.dim_per_head = embed_dim // self.num_heads + + def forward(self, x): + bs, length, width = x.size() + + def shape(x): + # (bs, length, width) --> (bs, length, n_heads, dim_per_head) + x = x.view(bs, -1, self.num_heads, self.dim_per_head) + # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head) + x = x.transpose(1, 2) + # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head) + x = x.reshape(bs * self.num_heads, -1, self.dim_per_head) + # (bs*n_heads, length, dim_per_head) --> (bs*n_heads, dim_per_head, length) + x = x.transpose(1, 2) + return x + + class_token = x.mean(dim=1, keepdim=True) + self.positional_embedding.to(x.dtype) + x = torch.cat([class_token, x], dim=1) # (bs, length+1, width) + + # (bs*n_heads, class_token_length, dim_per_head) + q = shape(self.q_proj(class_token)) + # (bs*n_heads, length+class_token_length, dim_per_head) + k = shape(self.k_proj(x)) + v = shape(self.v_proj(x)) + + # (bs*n_heads, class_token_length, length+class_token_length): + scale = 1 / math.sqrt(math.sqrt(self.dim_per_head)) + weight = torch.einsum("bct,bcs->bts", q * scale, k * scale) # More stable with f16 than dividing afterwards + weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype) + + # (bs*n_heads, dim_per_head, class_token_length) + a = torch.einsum("bts,bcs->bct", weight, v) + + # (bs, length+1, width) + a = a.reshape(bs, -1, 1).transpose(1, 2) + + return a[:, 0, :] # cls_token + + +def get_fourier_embeds_from_boundingbox(embed_dim, box): + """ + Args: + embed_dim: int + box: a 3-D tensor [B x N x 4] representing the bounding boxes for GLIGEN pipeline + Returns: + [B x N x embed_dim] tensor of positional embeddings + """ + + batch_size, num_boxes = box.shape[:2] + + emb = 100 ** (torch.arange(embed_dim) / embed_dim) + emb = emb[None, None, None].to(device=box.device, dtype=box.dtype) + emb = emb * box.unsqueeze(-1) + + emb = torch.stack((emb.sin(), emb.cos()), dim=-1) + emb = emb.permute(0, 1, 3, 4, 2).reshape(batch_size, num_boxes, embed_dim * 2 * 4) + + return emb + + +class GLIGENTextBoundingboxProjection(nn.Module): + def __init__(self, positive_len, out_dim, feature_type="text-only", fourier_freqs=8): + super().__init__() + self.positive_len = positive_len + self.out_dim = out_dim + + self.fourier_embedder_dim = fourier_freqs + self.position_dim = fourier_freqs * 2 * 4 # 2: sin/cos, 4: xyxy + + if isinstance(out_dim, tuple): + out_dim = out_dim[0] + + if feature_type == "text-only": + self.linears = nn.Sequential( + nn.Linear(self.positive_len + self.position_dim, 512), + nn.SiLU(), + nn.Linear(512, 512), + nn.SiLU(), + nn.Linear(512, out_dim), + ) + self.null_positive_feature = torch.nn.Parameter(torch.zeros([self.positive_len])) + + elif feature_type == "text-image": + self.linears_text = nn.Sequential( + nn.Linear(self.positive_len + self.position_dim, 512), + nn.SiLU(), + nn.Linear(512, 512), + nn.SiLU(), + nn.Linear(512, out_dim), + ) + self.linears_image = nn.Sequential( + nn.Linear(self.positive_len + self.position_dim, 512), + nn.SiLU(), + nn.Linear(512, 512), + nn.SiLU(), + nn.Linear(512, out_dim), + ) + self.null_text_feature = torch.nn.Parameter(torch.zeros([self.positive_len])) + self.null_image_feature = torch.nn.Parameter(torch.zeros([self.positive_len])) + + self.null_position_feature = torch.nn.Parameter(torch.zeros([self.position_dim])) + + def forward( + self, + boxes, + masks, + positive_embeddings=None, + phrases_masks=None, + image_masks=None, + phrases_embeddings=None, + image_embeddings=None, + ): + masks = masks.unsqueeze(-1) + + # embedding position (it may includes padding as placeholder) + xyxy_embedding = get_fourier_embeds_from_boundingbox(self.fourier_embedder_dim, boxes) # B*N*4 -> B*N*C + + # learnable null embedding + xyxy_null = self.null_position_feature.view(1, 1, -1) + + # replace padding with learnable null embedding + xyxy_embedding = xyxy_embedding * masks + (1 - masks) * xyxy_null + + # positionet with text only information + if positive_embeddings is not None: + # learnable null embedding + positive_null = self.null_positive_feature.view(1, 1, -1) + + # replace padding with learnable null embedding + positive_embeddings = positive_embeddings * masks + (1 - masks) * positive_null + + objs = self.linears(torch.cat([positive_embeddings, xyxy_embedding], dim=-1)) + + # positionet with text and image infomation + else: + phrases_masks = phrases_masks.unsqueeze(-1) + image_masks = image_masks.unsqueeze(-1) + + # learnable null embedding + text_null = self.null_text_feature.view(1, 1, -1) + image_null = self.null_image_feature.view(1, 1, -1) + + # replace padding with learnable null embedding + phrases_embeddings = phrases_embeddings * phrases_masks + (1 - phrases_masks) * text_null + image_embeddings = image_embeddings * image_masks + (1 - image_masks) * image_null + + objs_text = self.linears_text(torch.cat([phrases_embeddings, xyxy_embedding], dim=-1)) + objs_image = self.linears_image(torch.cat([image_embeddings, xyxy_embedding], dim=-1)) + objs = torch.cat([objs_text, objs_image], dim=1) + + return objs + + +class PixArtAlphaCombinedTimestepSizeEmbeddings(nn.Module): + """ + For PixArt-Alpha. + + Reference: + https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L164C9-L168C29 + """ + + def __init__(self, embedding_dim, size_emb_dim, use_additional_conditions: bool = False): + super().__init__() + + self.outdim = size_emb_dim + self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0) + self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim) + + self.use_additional_conditions = use_additional_conditions + if use_additional_conditions: + self.additional_condition_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0) + self.resolution_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=size_emb_dim) + self.aspect_ratio_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=size_emb_dim) + + def forward(self, timestep, resolution, aspect_ratio, batch_size, hidden_dtype): + timesteps_proj = self.time_proj(timestep) + timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_dtype)) # (N, D) + + if self.use_additional_conditions: + resolution_emb = self.additional_condition_proj(resolution.flatten()).to(hidden_dtype) + resolution_emb = self.resolution_embedder(resolution_emb).reshape(batch_size, -1) + aspect_ratio_emb = self.additional_condition_proj(aspect_ratio.flatten()).to(hidden_dtype) + aspect_ratio_emb = self.aspect_ratio_embedder(aspect_ratio_emb).reshape(batch_size, -1) + conditioning = timesteps_emb + torch.cat([resolution_emb, aspect_ratio_emb], dim=1) + else: + conditioning = timesteps_emb + + return conditioning + + +class PixArtAlphaTextProjection(nn.Module): + """ + Projects caption embeddings. Also handles dropout for classifier-free guidance. + + Adapted from https://github.com/PixArt-alpha/PixArt-alpha/blob/master/diffusion/model/nets/PixArt_blocks.py + """ + + def __init__(self, in_features, hidden_size, num_tokens=120): + super().__init__() + self.linear_1 = nn.Linear(in_features=in_features, out_features=hidden_size, bias=True) + self.act_1 = nn.GELU(approximate="tanh") + self.linear_2 = nn.Linear(in_features=hidden_size, out_features=hidden_size, bias=True) + + def forward(self, caption): + hidden_states = self.linear_1(caption) + hidden_states = self.act_1(hidden_states) + hidden_states = self.linear_2(hidden_states) + return hidden_states + + +class IPAdapterPlusImageProjection(nn.Module): + """Resampler of IP-Adapter Plus. + + Args: + ---- + embed_dims (int): The feature dimension. Defaults to 768. + output_dims (int): The number of output channels, that is the same + number of the channels in the + `unet.config.cross_attention_dim`. Defaults to 1024. + hidden_dims (int): The number of hidden channels. Defaults to 1280. + depth (int): The number of blocks. Defaults to 8. + dim_head (int): The number of head channels. Defaults to 64. + heads (int): Parallel attention heads. Defaults to 16. + num_queries (int): The number of queries. Defaults to 8. + ffn_ratio (float): The expansion ratio of feedforward network hidden + layer channels. Defaults to 4. + """ + + def __init__( + self, + embed_dims: int = 768, + output_dims: int = 1024, + hidden_dims: int = 1280, + depth: int = 4, + dim_head: int = 64, + heads: int = 16, + num_queries: int = 8, + ffn_ratio: float = 4, + ) -> None: + super().__init__() + from .attention import FeedForward # Lazy import to avoid circular import + + self.latents = nn.Parameter(torch.randn(1, num_queries, hidden_dims) / hidden_dims**0.5) + + self.proj_in = nn.Linear(embed_dims, hidden_dims) + + self.proj_out = nn.Linear(hidden_dims, output_dims) + self.norm_out = nn.LayerNorm(output_dims) + + self.layers = nn.ModuleList([]) + for _ in range(depth): + self.layers.append( + nn.ModuleList( + [ + nn.LayerNorm(hidden_dims), + nn.LayerNorm(hidden_dims), + Attention( + query_dim=hidden_dims, + dim_head=dim_head, + heads=heads, + out_bias=False, + ), + nn.Sequential( + nn.LayerNorm(hidden_dims), + FeedForward(hidden_dims, hidden_dims, activation_fn="gelu", mult=ffn_ratio, bias=False), + ), + ] + ) + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + """Forward pass. + + Args: + ---- + x (torch.Tensor): Input Tensor. + + Returns: + ------- + torch.Tensor: Output Tensor. + """ + latents = self.latents.repeat(x.size(0), 1, 1) + + x = self.proj_in(x) + + for ln0, ln1, attn, ff in self.layers: + residual = latents + + encoder_hidden_states = ln0(x) + latents = ln1(latents) + encoder_hidden_states = torch.cat([encoder_hidden_states, latents], dim=-2) + latents = attn(latents, encoder_hidden_states) + residual + latents = ff(latents) + latents + + latents = self.proj_out(latents) + return self.norm_out(latents) + + +class MultiIPAdapterImageProjection(nn.Module): + def __init__(self, IPAdapterImageProjectionLayers: Union[List[nn.Module], Tuple[nn.Module]]): + super().__init__() + self.image_projection_layers = nn.ModuleList(IPAdapterImageProjectionLayers) + + def forward(self, image_embeds: List[torch.FloatTensor]): + projected_image_embeds = [] + + # currently, we accept `image_embeds` as + # 1. a tensor (deprecated) with shape [batch_size, embed_dim] or [batch_size, sequence_length, embed_dim] + # 2. list of `n` tensors where `n` is number of ip-adapters, each tensor can hae shape [batch_size, num_images, embed_dim] or [batch_size, num_images, sequence_length, embed_dim] + if not isinstance(image_embeds, list): + deprecation_message = ( + "You have passed a tensor as `image_embeds`.This is deprecated and will be removed in a future release." + " Please make sure to update your script to pass `image_embeds` as a list of tensors to supress this warning." + ) + deprecate("image_embeds not a list", "1.0.0", deprecation_message, standard_warn=False) + image_embeds = [image_embeds.unsqueeze(1)] + + if len(image_embeds) != len(self.image_projection_layers): + raise ValueError( + f"image_embeds must have the same length as image_projection_layers, got {len(image_embeds)} and {len(self.image_projection_layers)}" + ) + + for image_embed, image_projection_layer in zip(image_embeds, self.image_projection_layers): + batch_size, num_images = image_embed.shape[0], image_embed.shape[1] + image_embed = image_embed.reshape((batch_size * num_images,) + image_embed.shape[2:]) + image_embed = image_projection_layer(image_embed) + image_embed = image_embed.reshape((batch_size, num_images) + image_embed.shape[1:]) + + projected_image_embeds.append(image_embed) + + return projected_image_embeds diff --git a/diffusers-0.27.0/src/diffusers/models/embeddings_flax.py b/diffusers-0.27.0/src/diffusers/models/embeddings_flax.py new file mode 100755 index 0000000..8e343be --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/embeddings_flax.py @@ -0,0 +1,97 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math + +import flax.linen as nn +import jax.numpy as jnp + + +def get_sinusoidal_embeddings( + timesteps: jnp.ndarray, + embedding_dim: int, + freq_shift: float = 1, + min_timescale: float = 1, + max_timescale: float = 1.0e4, + flip_sin_to_cos: bool = False, + scale: float = 1.0, +) -> jnp.ndarray: + """Returns the positional encoding (same as Tensor2Tensor). + + Args: + timesteps: a 1-D Tensor of N indices, one per batch element. + These may be fractional. + embedding_dim: The number of output channels. + min_timescale: The smallest time unit (should probably be 0.0). + max_timescale: The largest time unit. + Returns: + a Tensor of timing signals [N, num_channels] + """ + assert timesteps.ndim == 1, "Timesteps should be a 1d-array" + assert embedding_dim % 2 == 0, f"Embedding dimension {embedding_dim} should be even" + num_timescales = float(embedding_dim // 2) + log_timescale_increment = math.log(max_timescale / min_timescale) / (num_timescales - freq_shift) + inv_timescales = min_timescale * jnp.exp(jnp.arange(num_timescales, dtype=jnp.float32) * -log_timescale_increment) + emb = jnp.expand_dims(timesteps, 1) * jnp.expand_dims(inv_timescales, 0) + + # scale embeddings + scaled_time = scale * emb + + if flip_sin_to_cos: + signal = jnp.concatenate([jnp.cos(scaled_time), jnp.sin(scaled_time)], axis=1) + else: + signal = jnp.concatenate([jnp.sin(scaled_time), jnp.cos(scaled_time)], axis=1) + signal = jnp.reshape(signal, [jnp.shape(timesteps)[0], embedding_dim]) + return signal + + +class FlaxTimestepEmbedding(nn.Module): + r""" + Time step Embedding Module. Learns embeddings for input time steps. + + Args: + time_embed_dim (`int`, *optional*, defaults to `32`): + Time step embedding dimension + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + time_embed_dim: int = 32 + dtype: jnp.dtype = jnp.float32 + + @nn.compact + def __call__(self, temb): + temb = nn.Dense(self.time_embed_dim, dtype=self.dtype, name="linear_1")(temb) + temb = nn.silu(temb) + temb = nn.Dense(self.time_embed_dim, dtype=self.dtype, name="linear_2")(temb) + return temb + + +class FlaxTimesteps(nn.Module): + r""" + Wrapper Module for sinusoidal Time step Embeddings as described in https://arxiv.org/abs/2006.11239 + + Args: + dim (`int`, *optional*, defaults to `32`): + Time step embedding dimension + """ + + dim: int = 32 + flip_sin_to_cos: bool = False + freq_shift: float = 1 + + @nn.compact + def __call__(self, timesteps): + return get_sinusoidal_embeddings( + timesteps, embedding_dim=self.dim, flip_sin_to_cos=self.flip_sin_to_cos, freq_shift=self.freq_shift + ) diff --git a/diffusers-0.27.0/src/diffusers/models/lora.py b/diffusers-0.27.0/src/diffusers/models/lora.py new file mode 100755 index 0000000..4e9e0c0 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/lora.py @@ -0,0 +1,457 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# IMPORTANT: # +################################################################### +# ----------------------------------------------------------------# +# This file is deprecated and will be removed soon # +# (as soon as PEFT will become a required dependency for LoRA) # +# ----------------------------------------------------------------# +################################################################### + +from typing import Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from ..utils import deprecate, logging +from ..utils.import_utils import is_transformers_available + + +if is_transformers_available(): + from transformers import CLIPTextModel, CLIPTextModelWithProjection + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def text_encoder_attn_modules(text_encoder): + attn_modules = [] + + if isinstance(text_encoder, (CLIPTextModel, CLIPTextModelWithProjection)): + for i, layer in enumerate(text_encoder.text_model.encoder.layers): + name = f"text_model.encoder.layers.{i}.self_attn" + mod = layer.self_attn + attn_modules.append((name, mod)) + else: + raise ValueError(f"do not know how to get attention modules for: {text_encoder.__class__.__name__}") + + return attn_modules + + +def text_encoder_mlp_modules(text_encoder): + mlp_modules = [] + + if isinstance(text_encoder, (CLIPTextModel, CLIPTextModelWithProjection)): + for i, layer in enumerate(text_encoder.text_model.encoder.layers): + mlp_mod = layer.mlp + name = f"text_model.encoder.layers.{i}.mlp" + mlp_modules.append((name, mlp_mod)) + else: + raise ValueError(f"do not know how to get mlp modules for: {text_encoder.__class__.__name__}") + + return mlp_modules + + +def adjust_lora_scale_text_encoder(text_encoder, lora_scale: float = 1.0): + for _, attn_module in text_encoder_attn_modules(text_encoder): + if isinstance(attn_module.q_proj, PatchedLoraProjection): + attn_module.q_proj.lora_scale = lora_scale + attn_module.k_proj.lora_scale = lora_scale + attn_module.v_proj.lora_scale = lora_scale + attn_module.out_proj.lora_scale = lora_scale + + for _, mlp_module in text_encoder_mlp_modules(text_encoder): + if isinstance(mlp_module.fc1, PatchedLoraProjection): + mlp_module.fc1.lora_scale = lora_scale + mlp_module.fc2.lora_scale = lora_scale + + +class PatchedLoraProjection(torch.nn.Module): + def __init__(self, regular_linear_layer, lora_scale=1, network_alpha=None, rank=4, dtype=None): + deprecation_message = "Use of `PatchedLoraProjection` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`." + deprecate("PatchedLoraProjection", "1.0.0", deprecation_message) + + super().__init__() + from ..models.lora import LoRALinearLayer + + self.regular_linear_layer = regular_linear_layer + + device = self.regular_linear_layer.weight.device + + if dtype is None: + dtype = self.regular_linear_layer.weight.dtype + + self.lora_linear_layer = LoRALinearLayer( + self.regular_linear_layer.in_features, + self.regular_linear_layer.out_features, + network_alpha=network_alpha, + device=device, + dtype=dtype, + rank=rank, + ) + + self.lora_scale = lora_scale + + # overwrite PyTorch's `state_dict` to be sure that only the 'regular_linear_layer' weights are saved + # when saving the whole text encoder model and when LoRA is unloaded or fused + def state_dict(self, *args, destination=None, prefix="", keep_vars=False): + if self.lora_linear_layer is None: + return self.regular_linear_layer.state_dict( + *args, destination=destination, prefix=prefix, keep_vars=keep_vars + ) + + return super().state_dict(*args, destination=destination, prefix=prefix, keep_vars=keep_vars) + + def _fuse_lora(self, lora_scale=1.0, safe_fusing=False): + if self.lora_linear_layer is None: + return + + dtype, device = self.regular_linear_layer.weight.data.dtype, self.regular_linear_layer.weight.data.device + + w_orig = self.regular_linear_layer.weight.data.float() + w_up = self.lora_linear_layer.up.weight.data.float() + w_down = self.lora_linear_layer.down.weight.data.float() + + if self.lora_linear_layer.network_alpha is not None: + w_up = w_up * self.lora_linear_layer.network_alpha / self.lora_linear_layer.rank + + fused_weight = w_orig + (lora_scale * torch.bmm(w_up[None, :], w_down[None, :])[0]) + + if safe_fusing and torch.isnan(fused_weight).any().item(): + raise ValueError( + "This LoRA weight seems to be broken. " + f"Encountered NaN values when trying to fuse LoRA weights for {self}." + "LoRA weights will not be fused." + ) + + self.regular_linear_layer.weight.data = fused_weight.to(device=device, dtype=dtype) + + # we can drop the lora layer now + self.lora_linear_layer = None + + # offload the up and down matrices to CPU to not blow the memory + self.w_up = w_up.cpu() + self.w_down = w_down.cpu() + self.lora_scale = lora_scale + + def _unfuse_lora(self): + if not (getattr(self, "w_up", None) is not None and getattr(self, "w_down", None) is not None): + return + + fused_weight = self.regular_linear_layer.weight.data + dtype, device = fused_weight.dtype, fused_weight.device + + w_up = self.w_up.to(device=device).float() + w_down = self.w_down.to(device).float() + + unfused_weight = fused_weight.float() - (self.lora_scale * torch.bmm(w_up[None, :], w_down[None, :])[0]) + self.regular_linear_layer.weight.data = unfused_weight.to(device=device, dtype=dtype) + + self.w_up = None + self.w_down = None + + def forward(self, input): + if self.lora_scale is None: + self.lora_scale = 1.0 + if self.lora_linear_layer is None: + return self.regular_linear_layer(input) + return self.regular_linear_layer(input) + (self.lora_scale * self.lora_linear_layer(input)) + + +class LoRALinearLayer(nn.Module): + r""" + A linear layer that is used with LoRA. + + Parameters: + in_features (`int`): + Number of input features. + out_features (`int`): + Number of output features. + rank (`int`, `optional`, defaults to 4): + The rank of the LoRA layer. + network_alpha (`float`, `optional`, defaults to `None`): + The value of the network alpha used for stable learning and preventing underflow. This value has the same + meaning as the `--network_alpha` option in the kohya-ss trainer script. See + https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning + device (`torch.device`, `optional`, defaults to `None`): + The device to use for the layer's weights. + dtype (`torch.dtype`, `optional`, defaults to `None`): + The dtype to use for the layer's weights. + """ + + def __init__( + self, + in_features: int, + out_features: int, + rank: int = 4, + network_alpha: Optional[float] = None, + device: Optional[Union[torch.device, str]] = None, + dtype: Optional[torch.dtype] = None, + ): + super().__init__() + + deprecation_message = "Use of `LoRALinearLayer` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`." + deprecate("LoRALinearLayer", "1.0.0", deprecation_message) + + self.down = nn.Linear(in_features, rank, bias=False, device=device, dtype=dtype) + self.up = nn.Linear(rank, out_features, bias=False, device=device, dtype=dtype) + # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. + # See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning + self.network_alpha = network_alpha + self.rank = rank + self.out_features = out_features + self.in_features = in_features + + nn.init.normal_(self.down.weight, std=1 / rank) + nn.init.zeros_(self.up.weight) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + orig_dtype = hidden_states.dtype + dtype = self.down.weight.dtype + + down_hidden_states = self.down(hidden_states.to(dtype)) + up_hidden_states = self.up(down_hidden_states) + + if self.network_alpha is not None: + up_hidden_states *= self.network_alpha / self.rank + + return up_hidden_states.to(orig_dtype) + + +class LoRAConv2dLayer(nn.Module): + r""" + A convolutional layer that is used with LoRA. + + Parameters: + in_features (`int`): + Number of input features. + out_features (`int`): + Number of output features. + rank (`int`, `optional`, defaults to 4): + The rank of the LoRA layer. + kernel_size (`int` or `tuple` of two `int`, `optional`, defaults to 1): + The kernel size of the convolution. + stride (`int` or `tuple` of two `int`, `optional`, defaults to 1): + The stride of the convolution. + padding (`int` or `tuple` of two `int` or `str`, `optional`, defaults to 0): + The padding of the convolution. + network_alpha (`float`, `optional`, defaults to `None`): + The value of the network alpha used for stable learning and preventing underflow. This value has the same + meaning as the `--network_alpha` option in the kohya-ss trainer script. See + https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning + """ + + def __init__( + self, + in_features: int, + out_features: int, + rank: int = 4, + kernel_size: Union[int, Tuple[int, int]] = (1, 1), + stride: Union[int, Tuple[int, int]] = (1, 1), + padding: Union[int, Tuple[int, int], str] = 0, + network_alpha: Optional[float] = None, + ): + super().__init__() + + deprecation_message = "Use of `LoRAConv2dLayer` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`." + deprecate("LoRAConv2dLayer", "1.0.0", deprecation_message) + + self.down = nn.Conv2d(in_features, rank, kernel_size=kernel_size, stride=stride, padding=padding, bias=False) + # according to the official kohya_ss trainer kernel_size are always fixed for the up layer + # # see: https://github.com/bmaltais/kohya_ss/blob/2accb1305979ba62f5077a23aabac23b4c37e935/networks/lora_diffusers.py#L129 + self.up = nn.Conv2d(rank, out_features, kernel_size=(1, 1), stride=(1, 1), bias=False) + + # This value has the same meaning as the `--network_alpha` option in the kohya-ss trainer script. + # See https://github.com/darkstorm2150/sd-scripts/blob/main/docs/train_network_README-en.md#execute-learning + self.network_alpha = network_alpha + self.rank = rank + + nn.init.normal_(self.down.weight, std=1 / rank) + nn.init.zeros_(self.up.weight) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + orig_dtype = hidden_states.dtype + dtype = self.down.weight.dtype + + down_hidden_states = self.down(hidden_states.to(dtype)) + up_hidden_states = self.up(down_hidden_states) + + if self.network_alpha is not None: + up_hidden_states *= self.network_alpha / self.rank + + return up_hidden_states.to(orig_dtype) + + +class LoRACompatibleConv(nn.Conv2d): + """ + A convolutional layer that can be used with LoRA. + """ + + def __init__(self, *args, lora_layer: Optional[LoRAConv2dLayer] = None, **kwargs): + deprecation_message = "Use of `LoRACompatibleConv` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`." + deprecate("LoRACompatibleConv", "1.0.0", deprecation_message) + + super().__init__(*args, **kwargs) + self.lora_layer = lora_layer + + def set_lora_layer(self, lora_layer: Optional[LoRAConv2dLayer]): + deprecation_message = "Use of `set_lora_layer()` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`." + deprecate("set_lora_layer", "1.0.0", deprecation_message) + + self.lora_layer = lora_layer + + def _fuse_lora(self, lora_scale: float = 1.0, safe_fusing: bool = False): + if self.lora_layer is None: + return + + dtype, device = self.weight.data.dtype, self.weight.data.device + + w_orig = self.weight.data.float() + w_up = self.lora_layer.up.weight.data.float() + w_down = self.lora_layer.down.weight.data.float() + + if self.lora_layer.network_alpha is not None: + w_up = w_up * self.lora_layer.network_alpha / self.lora_layer.rank + + fusion = torch.mm(w_up.flatten(start_dim=1), w_down.flatten(start_dim=1)) + fusion = fusion.reshape((w_orig.shape)) + fused_weight = w_orig + (lora_scale * fusion) + + if safe_fusing and torch.isnan(fused_weight).any().item(): + raise ValueError( + "This LoRA weight seems to be broken. " + f"Encountered NaN values when trying to fuse LoRA weights for {self}." + "LoRA weights will not be fused." + ) + + self.weight.data = fused_weight.to(device=device, dtype=dtype) + + # we can drop the lora layer now + self.lora_layer = None + + # offload the up and down matrices to CPU to not blow the memory + self.w_up = w_up.cpu() + self.w_down = w_down.cpu() + self._lora_scale = lora_scale + + def _unfuse_lora(self): + if not (getattr(self, "w_up", None) is not None and getattr(self, "w_down", None) is not None): + return + + fused_weight = self.weight.data + dtype, device = fused_weight.data.dtype, fused_weight.data.device + + self.w_up = self.w_up.to(device=device).float() + self.w_down = self.w_down.to(device).float() + + fusion = torch.mm(self.w_up.flatten(start_dim=1), self.w_down.flatten(start_dim=1)) + fusion = fusion.reshape((fused_weight.shape)) + unfused_weight = fused_weight.float() - (self._lora_scale * fusion) + self.weight.data = unfused_weight.to(device=device, dtype=dtype) + + self.w_up = None + self.w_down = None + + def forward(self, hidden_states: torch.Tensor, scale: float = 1.0) -> torch.Tensor: + if self.padding_mode != "zeros": + hidden_states = F.pad(hidden_states, self._reversed_padding_repeated_twice, mode=self.padding_mode) + padding = (0, 0) + else: + padding = self.padding + + original_outputs = F.conv2d( + hidden_states, self.weight, self.bias, self.stride, padding, self.dilation, self.groups + ) + + if self.lora_layer is None: + return original_outputs + else: + return original_outputs + (scale * self.lora_layer(hidden_states)) + + +class LoRACompatibleLinear(nn.Linear): + """ + A Linear layer that can be used with LoRA. + """ + + def __init__(self, *args, lora_layer: Optional[LoRALinearLayer] = None, **kwargs): + deprecation_message = "Use of `LoRACompatibleLinear` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`." + deprecate("LoRACompatibleLinear", "1.0.0", deprecation_message) + + super().__init__(*args, **kwargs) + self.lora_layer = lora_layer + + def set_lora_layer(self, lora_layer: Optional[LoRALinearLayer]): + deprecation_message = "Use of `set_lora_layer()` is deprecated. Please switch to PEFT backend by installing PEFT: `pip install peft`." + deprecate("set_lora_layer", "1.0.0", deprecation_message) + self.lora_layer = lora_layer + + def _fuse_lora(self, lora_scale: float = 1.0, safe_fusing: bool = False): + if self.lora_layer is None: + return + + dtype, device = self.weight.data.dtype, self.weight.data.device + + w_orig = self.weight.data.float() + w_up = self.lora_layer.up.weight.data.float() + w_down = self.lora_layer.down.weight.data.float() + + if self.lora_layer.network_alpha is not None: + w_up = w_up * self.lora_layer.network_alpha / self.lora_layer.rank + + fused_weight = w_orig + (lora_scale * torch.bmm(w_up[None, :], w_down[None, :])[0]) + + if safe_fusing and torch.isnan(fused_weight).any().item(): + raise ValueError( + "This LoRA weight seems to be broken. " + f"Encountered NaN values when trying to fuse LoRA weights for {self}." + "LoRA weights will not be fused." + ) + + self.weight.data = fused_weight.to(device=device, dtype=dtype) + + # we can drop the lora layer now + self.lora_layer = None + + # offload the up and down matrices to CPU to not blow the memory + self.w_up = w_up.cpu() + self.w_down = w_down.cpu() + self._lora_scale = lora_scale + + def _unfuse_lora(self): + if not (getattr(self, "w_up", None) is not None and getattr(self, "w_down", None) is not None): + return + + fused_weight = self.weight.data + dtype, device = fused_weight.dtype, fused_weight.device + + w_up = self.w_up.to(device=device).float() + w_down = self.w_down.to(device).float() + + unfused_weight = fused_weight.float() - (self._lora_scale * torch.bmm(w_up[None, :], w_down[None, :])[0]) + self.weight.data = unfused_weight.to(device=device, dtype=dtype) + + self.w_up = None + self.w_down = None + + def forward(self, hidden_states: torch.Tensor, scale: float = 1.0) -> torch.Tensor: + if self.lora_layer is None: + out = super().forward(hidden_states) + return out + else: + out = super().forward(hidden_states) + (scale * self.lora_layer(hidden_states)) + return out diff --git a/diffusers-0.27.0/src/diffusers/models/modeling_flax_pytorch_utils.py b/diffusers-0.27.0/src/diffusers/models/modeling_flax_pytorch_utils.py new file mode 100755 index 0000000..4a48713 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/modeling_flax_pytorch_utils.py @@ -0,0 +1,134 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch - Flax general utilities.""" +import re + +import jax.numpy as jnp +from flax.traverse_util import flatten_dict, unflatten_dict +from jax.random import PRNGKey + +from ..utils import logging + + +logger = logging.get_logger(__name__) + + +def rename_key(key): + regex = r"\w+[.]\d+" + pats = re.findall(regex, key) + for pat in pats: + key = key.replace(pat, "_".join(pat.split("."))) + return key + + +##################### +# PyTorch => Flax # +##################### + + +# Adapted from https://github.com/huggingface/transformers/blob/c603c80f46881ae18b2ca50770ef65fa4033eacd/src/transformers/modeling_flax_pytorch_utils.py#L69 +# and https://github.com/patil-suraj/stable-diffusion-jax/blob/main/stable_diffusion_jax/convert_diffusers_to_jax.py +def rename_key_and_reshape_tensor(pt_tuple_key, pt_tensor, random_flax_state_dict): + """Rename PT weight names to corresponding Flax weight names and reshape tensor if necessary""" + # conv norm or layer norm + renamed_pt_tuple_key = pt_tuple_key[:-1] + ("scale",) + + # rename attention layers + if len(pt_tuple_key) > 1: + for rename_from, rename_to in ( + ("to_out_0", "proj_attn"), + ("to_k", "key"), + ("to_v", "value"), + ("to_q", "query"), + ): + if pt_tuple_key[-2] == rename_from: + weight_name = pt_tuple_key[-1] + weight_name = "kernel" if weight_name == "weight" else weight_name + renamed_pt_tuple_key = pt_tuple_key[:-2] + (rename_to, weight_name) + if renamed_pt_tuple_key in random_flax_state_dict: + assert random_flax_state_dict[renamed_pt_tuple_key].shape == pt_tensor.T.shape + return renamed_pt_tuple_key, pt_tensor.T + + if ( + any("norm" in str_ for str_ in pt_tuple_key) + and (pt_tuple_key[-1] == "bias") + and (pt_tuple_key[:-1] + ("bias",) not in random_flax_state_dict) + and (pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict) + ): + renamed_pt_tuple_key = pt_tuple_key[:-1] + ("scale",) + return renamed_pt_tuple_key, pt_tensor + elif pt_tuple_key[-1] in ["weight", "gamma"] and pt_tuple_key[:-1] + ("scale",) in random_flax_state_dict: + renamed_pt_tuple_key = pt_tuple_key[:-1] + ("scale",) + return renamed_pt_tuple_key, pt_tensor + + # embedding + if pt_tuple_key[-1] == "weight" and pt_tuple_key[:-1] + ("embedding",) in random_flax_state_dict: + pt_tuple_key = pt_tuple_key[:-1] + ("embedding",) + return renamed_pt_tuple_key, pt_tensor + + # conv layer + renamed_pt_tuple_key = pt_tuple_key[:-1] + ("kernel",) + if pt_tuple_key[-1] == "weight" and pt_tensor.ndim == 4: + pt_tensor = pt_tensor.transpose(2, 3, 1, 0) + return renamed_pt_tuple_key, pt_tensor + + # linear layer + renamed_pt_tuple_key = pt_tuple_key[:-1] + ("kernel",) + if pt_tuple_key[-1] == "weight": + pt_tensor = pt_tensor.T + return renamed_pt_tuple_key, pt_tensor + + # old PyTorch layer norm weight + renamed_pt_tuple_key = pt_tuple_key[:-1] + ("weight",) + if pt_tuple_key[-1] == "gamma": + return renamed_pt_tuple_key, pt_tensor + + # old PyTorch layer norm bias + renamed_pt_tuple_key = pt_tuple_key[:-1] + ("bias",) + if pt_tuple_key[-1] == "beta": + return renamed_pt_tuple_key, pt_tensor + + return pt_tuple_key, pt_tensor + + +def convert_pytorch_state_dict_to_flax(pt_state_dict, flax_model, init_key=42): + # Step 1: Convert pytorch tensor to numpy + pt_state_dict = {k: v.numpy() for k, v in pt_state_dict.items()} + + # Step 2: Since the model is stateless, get random Flax params + random_flax_params = flax_model.init_weights(PRNGKey(init_key)) + + random_flax_state_dict = flatten_dict(random_flax_params) + flax_state_dict = {} + + # Need to change some parameters name to match Flax names + for pt_key, pt_tensor in pt_state_dict.items(): + renamed_pt_key = rename_key(pt_key) + pt_tuple_key = tuple(renamed_pt_key.split(".")) + + # Correctly rename weight parameters + flax_key, flax_tensor = rename_key_and_reshape_tensor(pt_tuple_key, pt_tensor, random_flax_state_dict) + + if flax_key in random_flax_state_dict: + if flax_tensor.shape != random_flax_state_dict[flax_key].shape: + raise ValueError( + f"PyTorch checkpoint seems to be incorrect. Weight {pt_key} was expected to be of shape " + f"{random_flax_state_dict[flax_key].shape}, but is {flax_tensor.shape}." + ) + + # also add unexpected weight so that warning is thrown + flax_state_dict[flax_key] = jnp.asarray(flax_tensor) + + return unflatten_dict(flax_state_dict) diff --git a/diffusers-0.27.0/src/diffusers/models/modeling_flax_utils.py b/diffusers-0.27.0/src/diffusers/models/modeling_flax_utils.py new file mode 100755 index 0000000..1ddcda9 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/modeling_flax_utils.py @@ -0,0 +1,566 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from pickle import UnpicklingError +from typing import Any, Dict, Union + +import jax +import jax.numpy as jnp +import msgpack.exceptions +from flax.core.frozen_dict import FrozenDict, unfreeze +from flax.serialization import from_bytes, to_bytes +from flax.traverse_util import flatten_dict, unflatten_dict +from huggingface_hub import create_repo, hf_hub_download +from huggingface_hub.utils import ( + EntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, + validate_hf_hub_args, +) +from requests import HTTPError + +from .. import __version__, is_torch_available +from ..utils import ( + CONFIG_NAME, + FLAX_WEIGHTS_NAME, + HUGGINGFACE_CO_RESOLVE_ENDPOINT, + WEIGHTS_NAME, + PushToHubMixin, + logging, +) +from .modeling_flax_pytorch_utils import convert_pytorch_state_dict_to_flax + + +logger = logging.get_logger(__name__) + + +class FlaxModelMixin(PushToHubMixin): + r""" + Base class for all Flax models. + + [`FlaxModelMixin`] takes care of storing the model configuration and provides methods for loading, downloading and + saving models. + + - **config_name** ([`str`]) -- Filename to save a model to when calling [`~FlaxModelMixin.save_pretrained`]. + """ + + config_name = CONFIG_NAME + _automatically_saved_args = ["_diffusers_version", "_class_name", "_name_or_path"] + _flax_internal_args = ["name", "parent", "dtype"] + + @classmethod + def _from_config(cls, config, **kwargs): + """ + All context managers that the model should be initialized under go here. + """ + return cls(config, **kwargs) + + def _cast_floating_to(self, params: Union[Dict, FrozenDict], dtype: jnp.dtype, mask: Any = None) -> Any: + """ + Helper method to cast floating-point values of given parameter `PyTree` to given `dtype`. + """ + + # taken from https://github.com/deepmind/jmp/blob/3a8318abc3292be38582794dbf7b094e6583b192/jmp/_src/policy.py#L27 + def conditional_cast(param): + if isinstance(param, jnp.ndarray) and jnp.issubdtype(param.dtype, jnp.floating): + param = param.astype(dtype) + return param + + if mask is None: + return jax.tree_map(conditional_cast, params) + + flat_params = flatten_dict(params) + flat_mask, _ = jax.tree_flatten(mask) + + for masked, key in zip(flat_mask, flat_params.keys()): + if masked: + param = flat_params[key] + flat_params[key] = conditional_cast(param) + + return unflatten_dict(flat_params) + + def to_bf16(self, params: Union[Dict, FrozenDict], mask: Any = None): + r""" + Cast the floating-point `params` to `jax.numpy.bfloat16`. This returns a new `params` tree and does not cast + the `params` in place. + + This method can be used on a TPU to explicitly convert the model parameters to bfloat16 precision to do full + half-precision training or to save weights in bfloat16 for inference in order to save memory and improve speed. + + Arguments: + params (`Union[Dict, FrozenDict]`): + A `PyTree` of model parameters. + mask (`Union[Dict, FrozenDict]`): + A `PyTree` with same structure as the `params` tree. The leaves should be booleans. It should be `True` + for params you want to cast, and `False` for those you want to skip. + + Examples: + + ```python + >>> from diffusers import FlaxUNet2DConditionModel + + >>> # load model + >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> # By default, the model parameters will be in fp32 precision, to cast these to bfloat16 precision + >>> params = model.to_bf16(params) + >>> # If you don't want to cast certain parameters (for example layer norm bias and scale) + >>> # then pass the mask as follows + >>> from flax import traverse_util + + >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> flat_params = traverse_util.flatten_dict(params) + >>> mask = { + ... path: (path[-2] != ("LayerNorm", "bias") and path[-2:] != ("LayerNorm", "scale")) + ... for path in flat_params + ... } + >>> mask = traverse_util.unflatten_dict(mask) + >>> params = model.to_bf16(params, mask) + ```""" + return self._cast_floating_to(params, jnp.bfloat16, mask) + + def to_fp32(self, params: Union[Dict, FrozenDict], mask: Any = None): + r""" + Cast the floating-point `params` to `jax.numpy.float32`. This method can be used to explicitly convert the + model parameters to fp32 precision. This returns a new `params` tree and does not cast the `params` in place. + + Arguments: + params (`Union[Dict, FrozenDict]`): + A `PyTree` of model parameters. + mask (`Union[Dict, FrozenDict]`): + A `PyTree` with same structure as the `params` tree. The leaves should be booleans. It should be `True` + for params you want to cast, and `False` for those you want to skip. + + Examples: + + ```python + >>> from diffusers import FlaxUNet2DConditionModel + + >>> # Download model and configuration from huggingface.co + >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> # By default, the model params will be in fp32, to illustrate the use of this method, + >>> # we'll first cast to fp16 and back to fp32 + >>> params = model.to_f16(params) + >>> # now cast back to fp32 + >>> params = model.to_fp32(params) + ```""" + return self._cast_floating_to(params, jnp.float32, mask) + + def to_fp16(self, params: Union[Dict, FrozenDict], mask: Any = None): + r""" + Cast the floating-point `params` to `jax.numpy.float16`. This returns a new `params` tree and does not cast the + `params` in place. + + This method can be used on a GPU to explicitly convert the model parameters to float16 precision to do full + half-precision training or to save weights in float16 for inference in order to save memory and improve speed. + + Arguments: + params (`Union[Dict, FrozenDict]`): + A `PyTree` of model parameters. + mask (`Union[Dict, FrozenDict]`): + A `PyTree` with same structure as the `params` tree. The leaves should be booleans. It should be `True` + for params you want to cast, and `False` for those you want to skip. + + Examples: + + ```python + >>> from diffusers import FlaxUNet2DConditionModel + + >>> # load model + >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> # By default, the model params will be in fp32, to cast these to float16 + >>> params = model.to_fp16(params) + >>> # If you want don't want to cast certain parameters (for example layer norm bias and scale) + >>> # then pass the mask as follows + >>> from flax import traverse_util + + >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> flat_params = traverse_util.flatten_dict(params) + >>> mask = { + ... path: (path[-2] != ("LayerNorm", "bias") and path[-2:] != ("LayerNorm", "scale")) + ... for path in flat_params + ... } + >>> mask = traverse_util.unflatten_dict(mask) + >>> params = model.to_fp16(params, mask) + ```""" + return self._cast_floating_to(params, jnp.float16, mask) + + def init_weights(self, rng: jax.Array) -> Dict: + raise NotImplementedError(f"init_weights method has to be implemented for {self}") + + @classmethod + @validate_hf_hub_args + def from_pretrained( + cls, + pretrained_model_name_or_path: Union[str, os.PathLike], + dtype: jnp.dtype = jnp.float32, + *model_args, + **kwargs, + ): + r""" + Instantiate a pretrained Flax model from a pretrained model configuration. + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`): + Can be either: + + - A string, the *model id* (for example `runwayml/stable-diffusion-v1-5`) of a pretrained model + hosted on the Hub. + - A path to a *directory* (for example `./my_model_directory`) containing the model weights saved + using [`~FlaxModelMixin.save_pretrained`]. + dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`): + The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and + `jax.numpy.bfloat16` (on TPUs). + + This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If + specified, all the computation will be performed with the given `dtype`. + + + + This only specifies the dtype of the *computation* and does not influence the dtype of model + parameters. + + If you wish to change the dtype of the model parameters, see [`~FlaxModelMixin.to_fp16`] and + [`~FlaxModelMixin.to_bf16`]. + + + + model_args (sequence of positional arguments, *optional*): + All remaining positional arguments are passed to the underlying model's `__init__` method. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + from_pt (`bool`, *optional*, defaults to `False`): + Load the model weights from a PyTorch checkpoint save file. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to update the configuration object (after it is loaded) and initiate the model (for + example, `output_attentions=True`). Behaves differently depending on whether a `config` is provided or + automatically loaded: + + - If a configuration is provided with `config`, `kwargs` are directly passed to the underlying + model's `__init__` method (we assume all relevant updates to the configuration have already been + done). + - If a configuration is not provided, `kwargs` are first passed to the configuration class + initialization function [`~ConfigMixin.from_config`]. Each key of the `kwargs` that corresponds + to a configuration attribute is used to override said attribute with the supplied `kwargs` value. + Remaining keys that do not correspond to any configuration attribute are passed to the underlying + model's `__init__` function. + + Examples: + + ```python + >>> from diffusers import FlaxUNet2DConditionModel + + >>> # Download model and configuration from huggingface.co and cache. + >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> # Model was saved using *save_pretrained('./test/saved_model/')* (for example purposes, not runnable). + >>> model, params = FlaxUNet2DConditionModel.from_pretrained("./test/saved_model/") + ``` + + If you get the error message below, you need to finetune the weights for your downstream task: + + ```bash + Some weights of UNet2DConditionModel were not initialized from the model checkpoint at runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated + You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. + ``` + """ + config = kwargs.pop("config", None) + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + from_pt = kwargs.pop("from_pt", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", False) + token = kwargs.pop("token", None) + revision = kwargs.pop("revision", None) + subfolder = kwargs.pop("subfolder", None) + + user_agent = { + "diffusers": __version__, + "file_type": "model", + "framework": "flax", + } + + # Load config if we don't provide one + if config is None: + config, unused_kwargs = cls.load_config( + pretrained_model_name_or_path, + cache_dir=cache_dir, + return_unused_kwargs=True, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + **kwargs, + ) + + model, model_kwargs = cls.from_config(config, dtype=dtype, return_unused_kwargs=True, **unused_kwargs) + + # Load model + pretrained_path_with_subfolder = ( + pretrained_model_name_or_path + if subfolder is None + else os.path.join(pretrained_model_name_or_path, subfolder) + ) + if os.path.isdir(pretrained_path_with_subfolder): + if from_pt: + if not os.path.isfile(os.path.join(pretrained_path_with_subfolder, WEIGHTS_NAME)): + raise EnvironmentError( + f"Error no file named {WEIGHTS_NAME} found in directory {pretrained_path_with_subfolder} " + ) + model_file = os.path.join(pretrained_path_with_subfolder, WEIGHTS_NAME) + elif os.path.isfile(os.path.join(pretrained_path_with_subfolder, FLAX_WEIGHTS_NAME)): + # Load from a Flax checkpoint + model_file = os.path.join(pretrained_path_with_subfolder, FLAX_WEIGHTS_NAME) + # Check if pytorch weights exist instead + elif os.path.isfile(os.path.join(pretrained_path_with_subfolder, WEIGHTS_NAME)): + raise EnvironmentError( + f"{WEIGHTS_NAME} file found in directory {pretrained_path_with_subfolder}. Please load the model" + " using `from_pt=True`." + ) + else: + raise EnvironmentError( + f"Error no file named {FLAX_WEIGHTS_NAME} or {WEIGHTS_NAME} found in directory " + f"{pretrained_path_with_subfolder}." + ) + else: + try: + model_file = hf_hub_download( + pretrained_model_name_or_path, + filename=FLAX_WEIGHTS_NAME if not from_pt else WEIGHTS_NAME, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + token=token, + user_agent=user_agent, + subfolder=subfolder, + revision=revision, + ) + + except RepositoryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier " + "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a " + "token having permission to this repo with `token` or log in with `huggingface-cli " + "login`." + ) + except RevisionNotFoundError: + raise EnvironmentError( + f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for " + "this model name. Check the model page at " + f"'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions." + ) + except EntryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} does not appear to have a file named {FLAX_WEIGHTS_NAME}." + ) + except HTTPError as err: + raise EnvironmentError( + f"There was a specific connection error when trying to load {pretrained_model_name_or_path}:\n" + f"{err}" + ) + except ValueError: + raise EnvironmentError( + f"We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it" + f" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a" + f" directory containing a file named {FLAX_WEIGHTS_NAME} or {WEIGHTS_NAME}.\nCheckout your" + " internet connection or see how to run the library in offline mode at" + " 'https://huggingface.co/docs/transformers/installation#offline-mode'." + ) + except EnvironmentError: + raise EnvironmentError( + f"Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it from " + "'https://huggingface.co/models', make sure you don't have a local directory with the same name. " + f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory " + f"containing a file named {FLAX_WEIGHTS_NAME} or {WEIGHTS_NAME}." + ) + + if from_pt: + if is_torch_available(): + from .modeling_utils import load_state_dict + else: + raise EnvironmentError( + "Can't load the model in PyTorch format because PyTorch is not installed. " + "Please, install PyTorch or use native Flax weights." + ) + + # Step 1: Get the pytorch file + pytorch_model_file = load_state_dict(model_file) + + # Step 2: Convert the weights + state = convert_pytorch_state_dict_to_flax(pytorch_model_file, model) + else: + try: + with open(model_file, "rb") as state_f: + state = from_bytes(cls, state_f.read()) + except (UnpicklingError, msgpack.exceptions.ExtraData) as e: + try: + with open(model_file) as f: + if f.read().startswith("version"): + raise OSError( + "You seem to have cloned a repository without having git-lfs installed. Please" + " install git-lfs and run `git lfs install` followed by `git lfs pull` in the" + " folder you cloned." + ) + else: + raise ValueError from e + except (UnicodeDecodeError, ValueError): + raise EnvironmentError(f"Unable to convert {model_file} to Flax deserializable object. ") + # make sure all arrays are stored as jnp.ndarray + # NOTE: This is to prevent a bug this will be fixed in Flax >= v0.3.4: + # https://github.com/google/flax/issues/1261 + state = jax.tree_util.tree_map(lambda x: jax.device_put(x, jax.local_devices(backend="cpu")[0]), state) + + # flatten dicts + state = flatten_dict(state) + + params_shape_tree = jax.eval_shape(model.init_weights, rng=jax.random.PRNGKey(0)) + required_params = set(flatten_dict(unfreeze(params_shape_tree)).keys()) + + shape_state = flatten_dict(unfreeze(params_shape_tree)) + + missing_keys = required_params - set(state.keys()) + unexpected_keys = set(state.keys()) - required_params + + if missing_keys: + logger.warning( + f"The checkpoint {pretrained_model_name_or_path} is missing required keys: {missing_keys}. " + "Make sure to call model.init_weights to initialize the missing weights." + ) + cls._missing_keys = missing_keys + + for key in state.keys(): + if key in shape_state and state[key].shape != shape_state[key].shape: + raise ValueError( + f"Trying to load the pretrained weight for {key} failed: checkpoint has shape " + f"{state[key].shape} which is incompatible with the model shape {shape_state[key].shape}. " + ) + + # remove unexpected keys to not be saved again + for unexpected_key in unexpected_keys: + del state[unexpected_key] + + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when" + f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are" + f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task or" + " with another architecture." + ) + else: + logger.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n") + + if len(missing_keys) > 0: + logger.warning( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably" + " TRAIN this model on a down-stream task to be able to use it for predictions and inference." + ) + else: + logger.info( + f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at" + f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the checkpoint" + f" was trained on, you can already use {model.__class__.__name__} for predictions without further" + " training." + ) + + return model, unflatten_dict(state) + + def save_pretrained( + self, + save_directory: Union[str, os.PathLike], + params: Union[Dict, FrozenDict], + is_main_process: bool = True, + push_to_hub: bool = False, + **kwargs, + ): + """ + Save a model and its configuration file to a directory so that it can be reloaded using the + [`~FlaxModelMixin.from_pretrained`] class method. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to save a model and its configuration file to. Will be created if it doesn't exist. + params (`Union[Dict, FrozenDict]`): + A `PyTree` of model parameters. + is_main_process (`bool`, *optional*, defaults to `True`): + Whether the process calling this is the main process or not. Useful during distributed training and you + need to call this function on all processes. In this case, set `is_main_process=True` only on the main + process to avoid race conditions. + push_to_hub (`bool`, *optional*, defaults to `False`): + Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the + repository you want to push to with `repo_id` (will default to the name of `save_directory` in your + namespace). + kwargs (`Dict[str, Any]`, *optional*): + Additional key word arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. + """ + if os.path.isfile(save_directory): + logger.error(f"Provided path ({save_directory}) should be a directory, not a file") + return + + os.makedirs(save_directory, exist_ok=True) + + if push_to_hub: + commit_message = kwargs.pop("commit_message", None) + private = kwargs.pop("private", False) + create_pr = kwargs.pop("create_pr", False) + token = kwargs.pop("token", None) + repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1]) + repo_id = create_repo(repo_id, exist_ok=True, private=private, token=token).repo_id + + model_to_save = self + + # Attach architecture to the config + # Save the config + if is_main_process: + model_to_save.save_config(save_directory) + + # save model + output_model_file = os.path.join(save_directory, FLAX_WEIGHTS_NAME) + with open(output_model_file, "wb") as f: + model_bytes = to_bytes(params) + f.write(model_bytes) + + logger.info(f"Model weights saved in {output_model_file}") + + if push_to_hub: + self._upload_folder( + save_directory, + repo_id, + token=token, + commit_message=commit_message, + create_pr=create_pr, + ) diff --git a/diffusers-0.27.0/src/diffusers/models/modeling_outputs.py b/diffusers-0.27.0/src/diffusers/models/modeling_outputs.py new file mode 100755 index 0000000..8dfee5f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/modeling_outputs.py @@ -0,0 +1,17 @@ +from dataclasses import dataclass + +from ..utils import BaseOutput + + +@dataclass +class AutoencoderKLOutput(BaseOutput): + """ + Output of AutoencoderKL encoding method. + + Args: + latent_dist (`DiagonalGaussianDistribution`): + Encoded outputs of `Encoder` represented as the mean and logvar of `DiagonalGaussianDistribution`. + `DiagonalGaussianDistribution` allows for sampling latents from the distribution. + """ + + latent_dist: "DiagonalGaussianDistribution" # noqa: F821 diff --git a/diffusers-0.27.0/src/diffusers/models/modeling_pytorch_flax_utils.py b/diffusers-0.27.0/src/diffusers/models/modeling_pytorch_flax_utils.py new file mode 100755 index 0000000..9d820bb --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/modeling_pytorch_flax_utils.py @@ -0,0 +1,161 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" PyTorch - Flax general utilities.""" + +from pickle import UnpicklingError + +import jax +import jax.numpy as jnp +import numpy as np +from flax.serialization import from_bytes +from flax.traverse_util import flatten_dict + +from ..utils import logging + + +logger = logging.get_logger(__name__) + + +##################### +# Flax => PyTorch # +##################### + + +# from https://github.com/huggingface/transformers/blob/main/src/transformers/modeling_flax_pytorch_utils.py#L224-L352 +def load_flax_checkpoint_in_pytorch_model(pt_model, model_file): + try: + with open(model_file, "rb") as flax_state_f: + flax_state = from_bytes(None, flax_state_f.read()) + except UnpicklingError as e: + try: + with open(model_file) as f: + if f.read().startswith("version"): + raise OSError( + "You seem to have cloned a repository without having git-lfs installed. Please" + " install git-lfs and run `git lfs install` followed by `git lfs pull` in the" + " folder you cloned." + ) + else: + raise ValueError from e + except (UnicodeDecodeError, ValueError): + raise EnvironmentError(f"Unable to convert {model_file} to Flax deserializable object. ") + + return load_flax_weights_in_pytorch_model(pt_model, flax_state) + + +def load_flax_weights_in_pytorch_model(pt_model, flax_state): + """Load flax checkpoints in a PyTorch model""" + + try: + import torch # noqa: F401 + except ImportError: + logger.error( + "Loading Flax weights in PyTorch requires both PyTorch and Flax to be installed. Please see" + " https://pytorch.org/ and https://flax.readthedocs.io/en/latest/installation.html for installation" + " instructions." + ) + raise + + # check if we have bf16 weights + is_type_bf16 = flatten_dict(jax.tree_util.tree_map(lambda x: x.dtype == jnp.bfloat16, flax_state)).values() + if any(is_type_bf16): + # convert all weights to fp32 if they are bf16 since torch.from_numpy can-not handle bf16 + + # and bf16 is not fully supported in PT yet. + logger.warning( + "Found ``bfloat16`` weights in Flax model. Casting all ``bfloat16`` weights to ``float32`` " + "before loading those in PyTorch model." + ) + flax_state = jax.tree_util.tree_map( + lambda params: params.astype(np.float32) if params.dtype == jnp.bfloat16 else params, flax_state + ) + + pt_model.base_model_prefix = "" + + flax_state_dict = flatten_dict(flax_state, sep=".") + pt_model_dict = pt_model.state_dict() + + # keep track of unexpected & missing keys + unexpected_keys = [] + missing_keys = set(pt_model_dict.keys()) + + for flax_key_tuple, flax_tensor in flax_state_dict.items(): + flax_key_tuple_array = flax_key_tuple.split(".") + + if flax_key_tuple_array[-1] == "kernel" and flax_tensor.ndim == 4: + flax_key_tuple_array = flax_key_tuple_array[:-1] + ["weight"] + flax_tensor = jnp.transpose(flax_tensor, (3, 2, 0, 1)) + elif flax_key_tuple_array[-1] == "kernel": + flax_key_tuple_array = flax_key_tuple_array[:-1] + ["weight"] + flax_tensor = flax_tensor.T + elif flax_key_tuple_array[-1] == "scale": + flax_key_tuple_array = flax_key_tuple_array[:-1] + ["weight"] + + if "time_embedding" not in flax_key_tuple_array: + for i, flax_key_tuple_string in enumerate(flax_key_tuple_array): + flax_key_tuple_array[i] = ( + flax_key_tuple_string.replace("_0", ".0") + .replace("_1", ".1") + .replace("_2", ".2") + .replace("_3", ".3") + .replace("_4", ".4") + .replace("_5", ".5") + .replace("_6", ".6") + .replace("_7", ".7") + .replace("_8", ".8") + .replace("_9", ".9") + ) + + flax_key = ".".join(flax_key_tuple_array) + + if flax_key in pt_model_dict: + if flax_tensor.shape != pt_model_dict[flax_key].shape: + raise ValueError( + f"Flax checkpoint seems to be incorrect. Weight {flax_key_tuple} was expected " + f"to be of shape {pt_model_dict[flax_key].shape}, but is {flax_tensor.shape}." + ) + else: + # add weight to pytorch dict + flax_tensor = np.asarray(flax_tensor) if not isinstance(flax_tensor, np.ndarray) else flax_tensor + pt_model_dict[flax_key] = torch.from_numpy(flax_tensor) + # remove from missing keys + missing_keys.remove(flax_key) + else: + # weight is not expected by PyTorch model + unexpected_keys.append(flax_key) + + pt_model.load_state_dict(pt_model_dict) + + # re-transform missing_keys to list + missing_keys = list(missing_keys) + + if len(unexpected_keys) > 0: + logger.warning( + "Some weights of the Flax model were not used when initializing the PyTorch model" + f" {pt_model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are initializing" + f" {pt_model.__class__.__name__} from a Flax model trained on another task or with another architecture" + " (e.g. initializing a BertForSequenceClassification model from a FlaxBertForPreTraining model).\n- This" + f" IS NOT expected if you are initializing {pt_model.__class__.__name__} from a Flax model that you expect" + " to be exactly identical (e.g. initializing a BertForSequenceClassification model from a" + " FlaxBertForSequenceClassification model)." + ) + if len(missing_keys) > 0: + logger.warning( + f"Some weights of {pt_model.__class__.__name__} were not initialized from the Flax model and are newly" + f" initialized: {missing_keys}\nYou should probably TRAIN this model on a down-stream task to be able to" + " use it for predictions and inference." + ) + + return pt_model diff --git a/diffusers-0.27.0/src/diffusers/models/modeling_utils.py b/diffusers-0.27.0/src/diffusers/models/modeling_utils.py new file mode 100755 index 0000000..73ea5fb --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/modeling_utils.py @@ -0,0 +1,1021 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import itertools +import os +import re +from collections import OrderedDict +from functools import partial +from typing import Any, Callable, List, Optional, Tuple, Union + +import safetensors +import torch +from huggingface_hub import create_repo +from huggingface_hub.utils import validate_hf_hub_args +from torch import Tensor, nn + +from .. import __version__ +from ..utils import ( + CONFIG_NAME, + FLAX_WEIGHTS_NAME, + SAFETENSORS_FILE_EXTENSION, + SAFETENSORS_WEIGHTS_NAME, + WEIGHTS_NAME, + _add_variant, + _get_model_file, + deprecate, + is_accelerate_available, + is_torch_version, + logging, +) +from ..utils.hub_utils import PushToHubMixin, load_or_create_model_card, populate_model_card + + +logger = logging.get_logger(__name__) + + +if is_torch_version(">=", "1.9.0"): + _LOW_CPU_MEM_USAGE_DEFAULT = True +else: + _LOW_CPU_MEM_USAGE_DEFAULT = False + + +if is_accelerate_available(): + import accelerate + from accelerate.utils import set_module_tensor_to_device + from accelerate.utils.versions import is_torch_version + + +def get_parameter_device(parameter: torch.nn.Module) -> torch.device: + try: + parameters_and_buffers = itertools.chain(parameter.parameters(), parameter.buffers()) + return next(parameters_and_buffers).device + except StopIteration: + # For torch.nn.DataParallel compatibility in PyTorch 1.5 + + def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]: + tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)] + return tuples + + gen = parameter._named_members(get_members_fn=find_tensor_attributes) + first_tuple = next(gen) + return first_tuple[1].device + + +def get_parameter_dtype(parameter: torch.nn.Module) -> torch.dtype: + try: + params = tuple(parameter.parameters()) + if len(params) > 0: + return params[0].dtype + + buffers = tuple(parameter.buffers()) + if len(buffers) > 0: + return buffers[0].dtype + + except StopIteration: + # For torch.nn.DataParallel compatibility in PyTorch 1.5 + + def find_tensor_attributes(module: torch.nn.Module) -> List[Tuple[str, Tensor]]: + tuples = [(k, v) for k, v in module.__dict__.items() if torch.is_tensor(v)] + return tuples + + gen = parameter._named_members(get_members_fn=find_tensor_attributes) + first_tuple = next(gen) + return first_tuple[1].dtype + + +def load_state_dict(checkpoint_file: Union[str, os.PathLike], variant: Optional[str] = None): + """ + Reads a checkpoint file, returning properly formatted errors if they arise. + """ + try: + file_extension = os.path.basename(checkpoint_file).split(".")[-1] + if file_extension == SAFETENSORS_FILE_EXTENSION: + return safetensors.torch.load_file(checkpoint_file, device="cpu") + else: + return torch.load(checkpoint_file, map_location="cpu") + except Exception as e: + try: + with open(checkpoint_file) as f: + if f.read().startswith("version"): + raise OSError( + "You seem to have cloned a repository without having git-lfs installed. Please install " + "git-lfs and run `git lfs install` followed by `git lfs pull` in the folder " + "you cloned." + ) + else: + raise ValueError( + f"Unable to locate the file {checkpoint_file} which is necessary to load this pretrained " + "model. Make sure you have saved the model properly." + ) from e + except (UnicodeDecodeError, ValueError): + raise OSError( + f"Unable to load weights from checkpoint file for '{checkpoint_file}' " f"at '{checkpoint_file}'. " + ) + + +def load_model_dict_into_meta( + model, + state_dict: OrderedDict, + device: Optional[Union[str, torch.device]] = None, + dtype: Optional[Union[str, torch.dtype]] = None, + model_name_or_path: Optional[str] = None, +) -> List[str]: + device = device or torch.device("cpu") + dtype = dtype or torch.float32 + + accepts_dtype = "dtype" in set(inspect.signature(set_module_tensor_to_device).parameters.keys()) + + unexpected_keys = [] + empty_state_dict = model.state_dict() + for param_name, param in state_dict.items(): + if param_name not in empty_state_dict: + unexpected_keys.append(param_name) + continue + + if empty_state_dict[param_name].shape != param.shape: + model_name_or_path_str = f"{model_name_or_path} " if model_name_or_path is not None else "" + raise ValueError( + f"Cannot load {model_name_or_path_str}because {param_name} expected shape {empty_state_dict[param_name]}, but got {param.shape}. If you want to instead overwrite randomly initialized weights, please make sure to pass both `low_cpu_mem_usage=False` and `ignore_mismatched_sizes=True`. For more information, see also: https://github.com/huggingface/diffusers/issues/1619#issuecomment-1345604389 as an example." + ) + + if accepts_dtype: + set_module_tensor_to_device(model, param_name, device, value=param, dtype=dtype) + else: + set_module_tensor_to_device(model, param_name, device, value=param) + return unexpected_keys + + +def _load_state_dict_into_model(model_to_load, state_dict: OrderedDict) -> List[str]: + # Convert old format to new format if needed from a PyTorch state_dict + # copy state_dict so _load_from_state_dict can modify it + state_dict = state_dict.copy() + error_msgs = [] + + # PyTorch's `_load_from_state_dict` does not copy parameters in a module's descendants + # so we need to apply the function recursively. + def load(module: torch.nn.Module, prefix: str = ""): + args = (state_dict, prefix, {}, True, [], [], error_msgs) + module._load_from_state_dict(*args) + + for name, child in module._modules.items(): + if child is not None: + load(child, prefix + name + ".") + + load(model_to_load) + + return error_msgs + + +class ModelMixin(torch.nn.Module, PushToHubMixin): + r""" + Base class for all models. + + [`ModelMixin`] takes care of storing the model configuration and provides methods for loading, downloading and + saving models. + + - **config_name** ([`str`]) -- Filename to save a model to when calling [`~models.ModelMixin.save_pretrained`]. + """ + + config_name = CONFIG_NAME + _automatically_saved_args = ["_diffusers_version", "_class_name", "_name_or_path"] + _supports_gradient_checkpointing = False + _keys_to_ignore_on_load_unexpected = None + + def __init__(self): + super().__init__() + + def __getattr__(self, name: str) -> Any: + """The only reason we overwrite `getattr` here is to gracefully deprecate accessing + config attributes directly. See https://github.com/huggingface/diffusers/pull/3129 We need to overwrite + __getattr__ here in addition so that we don't trigger `torch.nn.Module`'s __getattr__': + https://pytorch.org/docs/stable/_modules/torch/nn/modules/module.html#Module + """ + + is_in_config = "_internal_dict" in self.__dict__ and hasattr(self.__dict__["_internal_dict"], name) + is_attribute = name in self.__dict__ + + if is_in_config and not is_attribute: + deprecation_message = f"Accessing config attribute `{name}` directly via '{type(self).__name__}' object attribute is deprecated. Please access '{name}' over '{type(self).__name__}'s config object instead, e.g. 'unet.config.{name}'." + deprecate("direct config name access", "1.0.0", deprecation_message, standard_warn=False, stacklevel=3) + return self._internal_dict[name] + + # call PyTorch's https://pytorch.org/docs/stable/_modules/torch/nn/modules/module.html#Module + return super().__getattr__(name) + + @property + def is_gradient_checkpointing(self) -> bool: + """ + Whether gradient checkpointing is activated for this model or not. + """ + return any(hasattr(m, "gradient_checkpointing") and m.gradient_checkpointing for m in self.modules()) + + def enable_gradient_checkpointing(self) -> None: + """ + Activates gradient checkpointing for the current model (may be referred to as *activation checkpointing* or + *checkpoint activations* in other frameworks). + """ + if not self._supports_gradient_checkpointing: + raise ValueError(f"{self.__class__.__name__} does not support gradient checkpointing.") + self.apply(partial(self._set_gradient_checkpointing, value=True)) + + def disable_gradient_checkpointing(self) -> None: + """ + Deactivates gradient checkpointing for the current model (may be referred to as *activation checkpointing* or + *checkpoint activations* in other frameworks). + """ + if self._supports_gradient_checkpointing: + self.apply(partial(self._set_gradient_checkpointing, value=False)) + + def set_use_memory_efficient_attention_xformers( + self, valid: bool, attention_op: Optional[Callable] = None + ) -> None: + # Recursively walk through all the children. + # Any children which exposes the set_use_memory_efficient_attention_xformers method + # gets the message + def fn_recursive_set_mem_eff(module: torch.nn.Module): + if hasattr(module, "set_use_memory_efficient_attention_xformers"): + module.set_use_memory_efficient_attention_xformers(valid, attention_op) + + for child in module.children(): + fn_recursive_set_mem_eff(child) + + for module in self.children(): + if isinstance(module, torch.nn.Module): + fn_recursive_set_mem_eff(module) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None) -> None: + r""" + Enable memory efficient attention from [xFormers](https://facebookresearch.github.io/xformers/). + + When this option is enabled, you should observe lower GPU memory usage and a potential speed up during + inference. Speed up during training is not guaranteed. + + + + ⚠️ When memory efficient attention and sliced attention are both enabled, memory efficient attention takes + precedent. + + + + Parameters: + attention_op (`Callable`, *optional*): + Override the default `None` operator for use as `op` argument to the + [`memory_efficient_attention()`](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.memory_efficient_attention) + function of xFormers. + + Examples: + + ```py + >>> import torch + >>> from diffusers import UNet2DConditionModel + >>> from xformers.ops import MemoryEfficientAttentionFlashAttentionOp + + >>> model = UNet2DConditionModel.from_pretrained( + ... "stabilityai/stable-diffusion-2-1", subfolder="unet", torch_dtype=torch.float16 + ... ) + >>> model = model.to("cuda") + >>> model.enable_xformers_memory_efficient_attention(attention_op=MemoryEfficientAttentionFlashAttentionOp) + ``` + """ + self.set_use_memory_efficient_attention_xformers(True, attention_op) + + def disable_xformers_memory_efficient_attention(self) -> None: + r""" + Disable memory efficient attention from [xFormers](https://facebookresearch.github.io/xformers/). + """ + self.set_use_memory_efficient_attention_xformers(False) + + def save_pretrained( + self, + save_directory: Union[str, os.PathLike], + is_main_process: bool = True, + save_function: Optional[Callable] = None, + safe_serialization: bool = True, + variant: Optional[str] = None, + push_to_hub: bool = False, + **kwargs, + ): + """ + Save a model and its configuration file to a directory so that it can be reloaded using the + [`~models.ModelMixin.from_pretrained`] class method. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to save a model and its configuration file to. Will be created if it doesn't exist. + is_main_process (`bool`, *optional*, defaults to `True`): + Whether the process calling this is the main process or not. Useful during distributed training and you + need to call this function on all processes. In this case, set `is_main_process=True` only on the main + process to avoid race conditions. + save_function (`Callable`): + The function to use to save the state dictionary. Useful during distributed training when you need to + replace `torch.save` with another method. Can be configured with the environment variable + `DIFFUSERS_SAVE_MODE`. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether to save the model using `safetensors` or the traditional PyTorch way with `pickle`. + variant (`str`, *optional*): + If specified, weights are saved in the format `pytorch_model..bin`. + push_to_hub (`bool`, *optional*, defaults to `False`): + Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the + repository you want to push to with `repo_id` (will default to the name of `save_directory` in your + namespace). + kwargs (`Dict[str, Any]`, *optional*): + Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. + """ + if os.path.isfile(save_directory): + logger.error(f"Provided path ({save_directory}) should be a directory, not a file") + return + + os.makedirs(save_directory, exist_ok=True) + + if push_to_hub: + commit_message = kwargs.pop("commit_message", None) + private = kwargs.pop("private", False) + create_pr = kwargs.pop("create_pr", False) + token = kwargs.pop("token", None) + repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1]) + repo_id = create_repo(repo_id, exist_ok=True, private=private, token=token).repo_id + + # Only save the model itself if we are using distributed training + model_to_save = self + + # Attach architecture to the config + # Save the config + if is_main_process: + model_to_save.save_config(save_directory) + + # Save the model + state_dict = model_to_save.state_dict() + + weights_name = SAFETENSORS_WEIGHTS_NAME if safe_serialization else WEIGHTS_NAME + weights_name = _add_variant(weights_name, variant) + + # Save the model + if safe_serialization: + safetensors.torch.save_file( + state_dict, os.path.join(save_directory, weights_name), metadata={"format": "pt"} + ) + else: + torch.save(state_dict, os.path.join(save_directory, weights_name)) + + logger.info(f"Model weights saved in {os.path.join(save_directory, weights_name)}") + + if push_to_hub: + # Create a new empty model card and eventually tag it + model_card = load_or_create_model_card(repo_id, token=token) + model_card = populate_model_card(model_card) + model_card.save(os.path.join(save_directory, "README.md")) + + self._upload_folder( + save_directory, + repo_id, + token=token, + commit_message=commit_message, + create_pr=create_pr, + ) + + @classmethod + @validate_hf_hub_args + def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): + r""" + Instantiate a pretrained PyTorch model from a pretrained model configuration. + + The model is set in evaluation mode - `model.eval()` - by default, and dropout modules are deactivated. To + train the model, set it back in training mode with `model.train()`. + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on + the Hub. + - A path to a *directory* (for example `./my_model_directory`) containing the model weights saved + with [`~ModelMixin.save_pretrained`]. + + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If `"auto"` is passed, the + dtype is automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info (`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + from_flax (`bool`, *optional*, defaults to `False`): + Load the model weights from a Flax checkpoint save file. + subfolder (`str`, *optional*, defaults to `""`): + The subfolder location of a model file within a larger model repository on the Hub or locally. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you're downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn't need to be defined for each + parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the + same device. + + Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier for the maximum memory. Will default to the maximum memory available for + each GPU and the available CPU RAM if unset. + offload_folder (`str` or `os.PathLike`, *optional*): + The path to offload weights if `device_map` contains the value `"disk"`. + offload_state_dict (`bool`, *optional*): + If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if + the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` + when there is some disk offload. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + variant (`str`, *optional*): + Load weights from a specified `variant` filename such as `"fp16"` or `"ema"`. This is ignored when + loading `from_flax`. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the `safetensors` weights are downloaded if they're available **and** if the + `safetensors` library is installed. If set to `True`, the model is forcibly loaded from `safetensors` + weights. If set to `False`, `safetensors` weights are not loaded. + + + + To use private or [gated models](https://huggingface.co/docs/hub/models-gated#gated-models), log-in with + `huggingface-cli login`. You can also activate the special + ["offline-mode"](https://huggingface.co/diffusers/installation.html#offline-mode) to use this method in a + firewalled environment. + + + + Example: + + ```py + from diffusers import UNet2DConditionModel + + unet = UNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="unet") + ``` + + If you get the error message below, you need to finetune the weights for your downstream task: + + ```bash + Some weights of UNet2DConditionModel were not initialized from the model checkpoint at runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated + You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. + ``` + """ + cache_dir = kwargs.pop("cache_dir", None) + ignore_mismatched_sizes = kwargs.pop("ignore_mismatched_sizes", False) + force_download = kwargs.pop("force_download", False) + from_flax = kwargs.pop("from_flax", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + output_loading_info = kwargs.pop("output_loading_info", False) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) + revision = kwargs.pop("revision", None) + torch_dtype = kwargs.pop("torch_dtype", None) + subfolder = kwargs.pop("subfolder", None) + device_map = kwargs.pop("device_map", None) + max_memory = kwargs.pop("max_memory", None) + offload_folder = kwargs.pop("offload_folder", None) + offload_state_dict = kwargs.pop("offload_state_dict", False) + low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", _LOW_CPU_MEM_USAGE_DEFAULT) + variant = kwargs.pop("variant", None) + use_safetensors = kwargs.pop("use_safetensors", None) + + allow_pickle = False + if use_safetensors is None: + use_safetensors = True + allow_pickle = True + + if low_cpu_mem_usage and not is_accelerate_available(): + low_cpu_mem_usage = False + logger.warning( + "Cannot initialize model with low cpu memory usage because `accelerate` was not found in the" + " environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install" + " `accelerate` for faster and less memory-intense model loading. You can do so with: \n```\npip" + " install accelerate\n```\n." + ) + + if device_map is not None and not is_accelerate_available(): + raise NotImplementedError( + "Loading and dispatching requires `accelerate`. Please make sure to install accelerate or set" + " `device_map=None`. You can install accelerate with `pip install accelerate`." + ) + + # Check if we can handle device_map and dispatching the weights + if device_map is not None and not is_torch_version(">=", "1.9.0"): + raise NotImplementedError( + "Loading and dispatching requires torch >= 1.9.0. Please either update your PyTorch version or set" + " `device_map=None`." + ) + + if low_cpu_mem_usage is True and not is_torch_version(">=", "1.9.0"): + raise NotImplementedError( + "Low memory initialization requires torch >= 1.9.0. Please either update your PyTorch version or set" + " `low_cpu_mem_usage=False`." + ) + + if low_cpu_mem_usage is False and device_map is not None: + raise ValueError( + f"You cannot set `low_cpu_mem_usage` to `False` while using device_map={device_map} for loading and" + " dispatching. Please make sure to set `low_cpu_mem_usage=True`." + ) + + # Load config if we don't provide a configuration + config_path = pretrained_model_name_or_path + + user_agent = { + "diffusers": __version__, + "file_type": "model", + "framework": "pytorch", + } + + # load config + config, unused_kwargs, commit_hash = cls.load_config( + config_path, + cache_dir=cache_dir, + return_unused_kwargs=True, + return_commit_hash=True, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + device_map=device_map, + max_memory=max_memory, + offload_folder=offload_folder, + offload_state_dict=offload_state_dict, + user_agent=user_agent, + **kwargs, + ) + + # load model + model_file = None + if from_flax: + model_file = _get_model_file( + pretrained_model_name_or_path, + weights_name=FLAX_WEIGHTS_NAME, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + commit_hash=commit_hash, + ) + model = cls.from_config(config, **unused_kwargs) + + # Convert the weights + from .modeling_pytorch_flax_utils import load_flax_checkpoint_in_pytorch_model + + model = load_flax_checkpoint_in_pytorch_model(model, model_file) + else: + if use_safetensors: + try: + model_file = _get_model_file( + pretrained_model_name_or_path, + weights_name=_add_variant(SAFETENSORS_WEIGHTS_NAME, variant), + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + commit_hash=commit_hash, + ) + except IOError as e: + if not allow_pickle: + raise e + pass + if model_file is None: + model_file = _get_model_file( + pretrained_model_name_or_path, + weights_name=_add_variant(WEIGHTS_NAME, variant), + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + subfolder=subfolder, + user_agent=user_agent, + commit_hash=commit_hash, + ) + + if low_cpu_mem_usage: + # Instantiate model with empty weights + with accelerate.init_empty_weights(): + model = cls.from_config(config, **unused_kwargs) + + # if device_map is None, load the state dict and move the params from meta device to the cpu + if device_map is None: + param_device = "cpu" + state_dict = load_state_dict(model_file, variant=variant) + model._convert_deprecated_attention_blocks(state_dict) + # move the params from meta device to cpu + missing_keys = set(model.state_dict().keys()) - set(state_dict.keys()) + if len(missing_keys) > 0: + raise ValueError( + f"Cannot load {cls} from {pretrained_model_name_or_path} because the following keys are" + f" missing: \n {', '.join(missing_keys)}. \n Please make sure to pass" + " `low_cpu_mem_usage=False` and `device_map=None` if you want to randomly initialize" + " those weights or else make sure your checkpoint file is correct." + ) + + unexpected_keys = load_model_dict_into_meta( + model, + state_dict, + device=param_device, + dtype=torch_dtype, + model_name_or_path=pretrained_model_name_or_path, + ) + + if cls._keys_to_ignore_on_load_unexpected is not None: + for pat in cls._keys_to_ignore_on_load_unexpected: + unexpected_keys = [k for k in unexpected_keys if re.search(pat, k) is None] + + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint were not used when initializing {cls.__name__}: \n {[', '.join(unexpected_keys)]}" + ) + + else: # else let accelerate handle loading and dispatching. + # Load weights and dispatch according to the device_map + # by default the device_map is None and the weights are loaded on the CPU + try: + accelerate.load_checkpoint_and_dispatch( + model, + model_file, + device_map, + max_memory=max_memory, + offload_folder=offload_folder, + offload_state_dict=offload_state_dict, + dtype=torch_dtype, + ) + except AttributeError as e: + # When using accelerate loading, we do not have the ability to load the state + # dict and rename the weight names manually. Additionally, accelerate skips + # torch loading conventions and directly writes into `module.{_buffers, _parameters}` + # (which look like they should be private variables?), so we can't use the standard hooks + # to rename parameters on load. We need to mimic the original weight names so the correct + # attributes are available. After we have loaded the weights, we convert the deprecated + # names to the new non-deprecated names. Then we _greatly encourage_ the user to convert + # the weights so we don't have to do this again. + + if "'Attention' object has no attribute" in str(e): + logger.warning( + f"Taking `{str(e)}` while using `accelerate.load_checkpoint_and_dispatch` to mean {pretrained_model_name_or_path}" + " was saved with deprecated attention block weight names. We will load it with the deprecated attention block" + " names and convert them on the fly to the new attention block format. Please re-save the model after this conversion," + " so we don't have to do the on the fly renaming in the future. If the model is from a hub checkpoint," + " please also re-upload it or open a PR on the original repository." + ) + model._temp_convert_self_to_deprecated_attention_blocks() + accelerate.load_checkpoint_and_dispatch( + model, + model_file, + device_map, + max_memory=max_memory, + offload_folder=offload_folder, + offload_state_dict=offload_state_dict, + dtype=torch_dtype, + ) + model._undo_temp_convert_self_to_deprecated_attention_blocks() + else: + raise e + + loading_info = { + "missing_keys": [], + "unexpected_keys": [], + "mismatched_keys": [], + "error_msgs": [], + } + else: + model = cls.from_config(config, **unused_kwargs) + + state_dict = load_state_dict(model_file, variant=variant) + model._convert_deprecated_attention_blocks(state_dict) + + model, missing_keys, unexpected_keys, mismatched_keys, error_msgs = cls._load_pretrained_model( + model, + state_dict, + model_file, + pretrained_model_name_or_path, + ignore_mismatched_sizes=ignore_mismatched_sizes, + ) + + loading_info = { + "missing_keys": missing_keys, + "unexpected_keys": unexpected_keys, + "mismatched_keys": mismatched_keys, + "error_msgs": error_msgs, + } + + if torch_dtype is not None and not isinstance(torch_dtype, torch.dtype): + raise ValueError( + f"{torch_dtype} needs to be of type `torch.dtype`, e.g. `torch.float16`, but is {type(torch_dtype)}." + ) + elif torch_dtype is not None: + model = model.to(torch_dtype) + + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + + # Set model in evaluation mode to deactivate DropOut modules by default + model.eval() + if output_loading_info: + return model, loading_info + + return model + + @classmethod + def _load_pretrained_model( + cls, + model, + state_dict: OrderedDict, + resolved_archive_file, + pretrained_model_name_or_path: Union[str, os.PathLike], + ignore_mismatched_sizes: bool = False, + ): + # Retrieve missing & unexpected_keys + model_state_dict = model.state_dict() + loaded_keys = list(state_dict.keys()) + + expected_keys = list(model_state_dict.keys()) + + original_loaded_keys = loaded_keys + + missing_keys = list(set(expected_keys) - set(loaded_keys)) + unexpected_keys = list(set(loaded_keys) - set(expected_keys)) + + # Make sure we are able to load base models as well as derived models (with heads) + model_to_load = model + + def _find_mismatched_keys( + state_dict, + model_state_dict, + loaded_keys, + ignore_mismatched_sizes, + ): + mismatched_keys = [] + if ignore_mismatched_sizes: + for checkpoint_key in loaded_keys: + model_key = checkpoint_key + + if ( + model_key in model_state_dict + and state_dict[checkpoint_key].shape != model_state_dict[model_key].shape + ): + mismatched_keys.append( + (checkpoint_key, state_dict[checkpoint_key].shape, model_state_dict[model_key].shape) + ) + del state_dict[checkpoint_key] + return mismatched_keys + + if state_dict is not None: + # Whole checkpoint + mismatched_keys = _find_mismatched_keys( + state_dict, + model_state_dict, + original_loaded_keys, + ignore_mismatched_sizes, + ) + error_msgs = _load_state_dict_into_model(model_to_load, state_dict) + + if len(error_msgs) > 0: + error_msg = "\n\t".join(error_msgs) + if "size mismatch" in error_msg: + error_msg += ( + "\n\tYou may consider adding `ignore_mismatched_sizes=True` in the model `from_pretrained` method." + ) + raise RuntimeError(f"Error(s) in loading state_dict for {model.__class__.__name__}:\n\t{error_msg}") + + if len(unexpected_keys) > 0: + logger.warning( + f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when" + f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are" + f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task" + " or with another architecture (e.g. initializing a BertForSequenceClassification model from a" + " BertForPreTraining model).\n- This IS NOT expected if you are initializing" + f" {model.__class__.__name__} from the checkpoint of a model that you expect to be exactly" + " identical (initializing a BertForSequenceClassification model from a" + " BertForSequenceClassification model)." + ) + else: + logger.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n") + if len(missing_keys) > 0: + logger.warning( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably" + " TRAIN this model on a down-stream task to be able to use it for predictions and inference." + ) + elif len(mismatched_keys) == 0: + logger.info( + f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at" + f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the" + f" checkpoint was trained on, you can already use {model.__class__.__name__} for predictions" + " without further training." + ) + if len(mismatched_keys) > 0: + mismatched_warning = "\n".join( + [ + f"- {key}: found shape {shape1} in the checkpoint and {shape2} in the model instantiated" + for key, shape1, shape2 in mismatched_keys + ] + ) + logger.warning( + f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at" + f" {pretrained_model_name_or_path} and are newly initialized because the shapes did not" + f" match:\n{mismatched_warning}\nYou should probably TRAIN this model on a down-stream task to be" + " able to use it for predictions and inference." + ) + + return model, missing_keys, unexpected_keys, mismatched_keys, error_msgs + + @property + def device(self) -> torch.device: + """ + `torch.device`: The device on which the module is (assuming that all the module parameters are on the same + device). + """ + return get_parameter_device(self) + + @property + def dtype(self) -> torch.dtype: + """ + `torch.dtype`: The dtype of the module (assuming that all the module parameters have the same dtype). + """ + return get_parameter_dtype(self) + + def num_parameters(self, only_trainable: bool = False, exclude_embeddings: bool = False) -> int: + """ + Get number of (trainable or non-embedding) parameters in the module. + + Args: + only_trainable (`bool`, *optional*, defaults to `False`): + Whether or not to return only the number of trainable parameters. + exclude_embeddings (`bool`, *optional*, defaults to `False`): + Whether or not to return only the number of non-embedding parameters. + + Returns: + `int`: The number of parameters. + + Example: + + ```py + from diffusers import UNet2DConditionModel + + model_id = "runwayml/stable-diffusion-v1-5" + unet = UNet2DConditionModel.from_pretrained(model_id, subfolder="unet") + unet.num_parameters(only_trainable=True) + 859520964 + ``` + """ + + if exclude_embeddings: + embedding_param_names = [ + f"{name}.weight" + for name, module_type in self.named_modules() + if isinstance(module_type, torch.nn.Embedding) + ] + non_embedding_parameters = [ + parameter for name, parameter in self.named_parameters() if name not in embedding_param_names + ] + return sum(p.numel() for p in non_embedding_parameters if p.requires_grad or not only_trainable) + else: + return sum(p.numel() for p in self.parameters() if p.requires_grad or not only_trainable) + + def _convert_deprecated_attention_blocks(self, state_dict: OrderedDict) -> None: + deprecated_attention_block_paths = [] + + def recursive_find_attn_block(name, module): + if hasattr(module, "_from_deprecated_attn_block") and module._from_deprecated_attn_block: + deprecated_attention_block_paths.append(name) + + for sub_name, sub_module in module.named_children(): + sub_name = sub_name if name == "" else f"{name}.{sub_name}" + recursive_find_attn_block(sub_name, sub_module) + + recursive_find_attn_block("", self) + + # NOTE: we have to check if the deprecated parameters are in the state dict + # because it is possible we are loading from a state dict that was already + # converted + + for path in deprecated_attention_block_paths: + # group_norm path stays the same + + # query -> to_q + if f"{path}.query.weight" in state_dict: + state_dict[f"{path}.to_q.weight"] = state_dict.pop(f"{path}.query.weight") + if f"{path}.query.bias" in state_dict: + state_dict[f"{path}.to_q.bias"] = state_dict.pop(f"{path}.query.bias") + + # key -> to_k + if f"{path}.key.weight" in state_dict: + state_dict[f"{path}.to_k.weight"] = state_dict.pop(f"{path}.key.weight") + if f"{path}.key.bias" in state_dict: + state_dict[f"{path}.to_k.bias"] = state_dict.pop(f"{path}.key.bias") + + # value -> to_v + if f"{path}.value.weight" in state_dict: + state_dict[f"{path}.to_v.weight"] = state_dict.pop(f"{path}.value.weight") + if f"{path}.value.bias" in state_dict: + state_dict[f"{path}.to_v.bias"] = state_dict.pop(f"{path}.value.bias") + + # proj_attn -> to_out.0 + if f"{path}.proj_attn.weight" in state_dict: + state_dict[f"{path}.to_out.0.weight"] = state_dict.pop(f"{path}.proj_attn.weight") + if f"{path}.proj_attn.bias" in state_dict: + state_dict[f"{path}.to_out.0.bias"] = state_dict.pop(f"{path}.proj_attn.bias") + + def _temp_convert_self_to_deprecated_attention_blocks(self) -> None: + deprecated_attention_block_modules = [] + + def recursive_find_attn_block(module): + if hasattr(module, "_from_deprecated_attn_block") and module._from_deprecated_attn_block: + deprecated_attention_block_modules.append(module) + + for sub_module in module.children(): + recursive_find_attn_block(sub_module) + + recursive_find_attn_block(self) + + for module in deprecated_attention_block_modules: + module.query = module.to_q + module.key = module.to_k + module.value = module.to_v + module.proj_attn = module.to_out[0] + + # We don't _have_ to delete the old attributes, but it's helpful to ensure + # that _all_ the weights are loaded into the new attributes and we're not + # making an incorrect assumption that this model should be converted when + # it really shouldn't be. + del module.to_q + del module.to_k + del module.to_v + del module.to_out + + def _undo_temp_convert_self_to_deprecated_attention_blocks(self) -> None: + deprecated_attention_block_modules = [] + + def recursive_find_attn_block(module) -> None: + if hasattr(module, "_from_deprecated_attn_block") and module._from_deprecated_attn_block: + deprecated_attention_block_modules.append(module) + + for sub_module in module.children(): + recursive_find_attn_block(sub_module) + + recursive_find_attn_block(self) + + for module in deprecated_attention_block_modules: + module.to_q = module.query + module.to_k = module.key + module.to_v = module.value + module.to_out = nn.ModuleList([module.proj_attn, nn.Dropout(module.dropout)]) + + del module.query + del module.key + del module.value + del module.proj_attn diff --git a/diffusers-0.27.0/src/diffusers/models/normalization.py b/diffusers-0.27.0/src/diffusers/models/normalization.py new file mode 100755 index 0000000..036a668 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/normalization.py @@ -0,0 +1,254 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numbers +from typing import Dict, Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..utils import is_torch_version +from .activations import get_activation +from .embeddings import CombinedTimestepLabelEmbeddings, PixArtAlphaCombinedTimestepSizeEmbeddings + + +class AdaLayerNorm(nn.Module): + r""" + Norm layer modified to incorporate timestep embeddings. + + Parameters: + embedding_dim (`int`): The size of each embedding vector. + num_embeddings (`int`): The size of the embeddings dictionary. + """ + + def __init__(self, embedding_dim: int, num_embeddings: int): + super().__init__() + self.emb = nn.Embedding(num_embeddings, embedding_dim) + self.silu = nn.SiLU() + self.linear = nn.Linear(embedding_dim, embedding_dim * 2) + self.norm = nn.LayerNorm(embedding_dim, elementwise_affine=False) + + def forward(self, x: torch.Tensor, timestep: torch.Tensor) -> torch.Tensor: + emb = self.linear(self.silu(self.emb(timestep))) + scale, shift = torch.chunk(emb, 2) + x = self.norm(x) * (1 + scale) + shift + return x + + +class AdaLayerNormZero(nn.Module): + r""" + Norm layer adaptive layer norm zero (adaLN-Zero). + + Parameters: + embedding_dim (`int`): The size of each embedding vector. + num_embeddings (`int`): The size of the embeddings dictionary. + """ + + def __init__(self, embedding_dim: int, num_embeddings: int): + super().__init__() + + self.emb = CombinedTimestepLabelEmbeddings(num_embeddings, embedding_dim) + + self.silu = nn.SiLU() + self.linear = nn.Linear(embedding_dim, 6 * embedding_dim, bias=True) + self.norm = nn.LayerNorm(embedding_dim, elementwise_affine=False, eps=1e-6) + + def forward( + self, + x: torch.Tensor, + timestep: torch.Tensor, + class_labels: torch.LongTensor, + hidden_dtype: Optional[torch.dtype] = None, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + emb = self.linear(self.silu(self.emb(timestep, class_labels, hidden_dtype=hidden_dtype))) + shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = emb.chunk(6, dim=1) + x = self.norm(x) * (1 + scale_msa[:, None]) + shift_msa[:, None] + return x, gate_msa, shift_mlp, scale_mlp, gate_mlp + + +class AdaLayerNormSingle(nn.Module): + r""" + Norm layer adaptive layer norm single (adaLN-single). + + As proposed in PixArt-Alpha (see: https://arxiv.org/abs/2310.00426; Section 2.3). + + Parameters: + embedding_dim (`int`): The size of each embedding vector. + use_additional_conditions (`bool`): To use additional conditions for normalization or not. + """ + + def __init__(self, embedding_dim: int, use_additional_conditions: bool = False): + super().__init__() + + self.emb = PixArtAlphaCombinedTimestepSizeEmbeddings( + embedding_dim, size_emb_dim=embedding_dim // 3, use_additional_conditions=use_additional_conditions + ) + + self.silu = nn.SiLU() + self.linear = nn.Linear(embedding_dim, 6 * embedding_dim, bias=True) + + def forward( + self, + timestep: torch.Tensor, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + batch_size: Optional[int] = None, + hidden_dtype: Optional[torch.dtype] = None, + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + # No modulation happening here. + embedded_timestep = self.emb(timestep, **added_cond_kwargs, batch_size=batch_size, hidden_dtype=hidden_dtype) + return self.linear(self.silu(embedded_timestep)), embedded_timestep + + +class AdaGroupNorm(nn.Module): + r""" + GroupNorm layer modified to incorporate timestep embeddings. + + Parameters: + embedding_dim (`int`): The size of each embedding vector. + num_embeddings (`int`): The size of the embeddings dictionary. + num_groups (`int`): The number of groups to separate the channels into. + act_fn (`str`, *optional*, defaults to `None`): The activation function to use. + eps (`float`, *optional*, defaults to `1e-5`): The epsilon value to use for numerical stability. + """ + + def __init__( + self, embedding_dim: int, out_dim: int, num_groups: int, act_fn: Optional[str] = None, eps: float = 1e-5 + ): + super().__init__() + self.num_groups = num_groups + self.eps = eps + + if act_fn is None: + self.act = None + else: + self.act = get_activation(act_fn) + + self.linear = nn.Linear(embedding_dim, out_dim * 2) + + def forward(self, x: torch.Tensor, emb: torch.Tensor) -> torch.Tensor: + if self.act: + emb = self.act(emb) + emb = self.linear(emb) + emb = emb[:, :, None, None] + scale, shift = emb.chunk(2, dim=1) + + x = F.group_norm(x, self.num_groups, eps=self.eps) + x = x * (1 + scale) + shift + return x + + +class AdaLayerNormContinuous(nn.Module): + def __init__( + self, + embedding_dim: int, + conditioning_embedding_dim: int, + # NOTE: It is a bit weird that the norm layer can be configured to have scale and shift parameters + # because the output is immediately scaled and shifted by the projected conditioning embeddings. + # Note that AdaLayerNorm does not let the norm layer have scale and shift parameters. + # However, this is how it was implemented in the original code, and it's rather likely you should + # set `elementwise_affine` to False. + elementwise_affine=True, + eps=1e-5, + bias=True, + norm_type="layer_norm", + ): + super().__init__() + self.silu = nn.SiLU() + self.linear = nn.Linear(conditioning_embedding_dim, embedding_dim * 2, bias=bias) + if norm_type == "layer_norm": + self.norm = LayerNorm(embedding_dim, eps, elementwise_affine, bias) + elif norm_type == "rms_norm": + self.norm = RMSNorm(embedding_dim, eps, elementwise_affine) + else: + raise ValueError(f"unknown norm_type {norm_type}") + + def forward(self, x: torch.Tensor, conditioning_embedding: torch.Tensor) -> torch.Tensor: + emb = self.linear(self.silu(conditioning_embedding)) + scale, shift = torch.chunk(emb, 2, dim=1) + x = self.norm(x) * (1 + scale)[:, None, :] + shift[:, None, :] + return x + + +if is_torch_version(">=", "2.1.0"): + LayerNorm = nn.LayerNorm +else: + # Has optional bias parameter compared to torch layer norm + # TODO: replace with torch layernorm once min required torch version >= 2.1 + class LayerNorm(nn.Module): + def __init__(self, dim, eps: float = 1e-5, elementwise_affine: bool = True, bias: bool = True): + super().__init__() + + self.eps = eps + + if isinstance(dim, numbers.Integral): + dim = (dim,) + + self.dim = torch.Size(dim) + + if elementwise_affine: + self.weight = nn.Parameter(torch.ones(dim)) + self.bias = nn.Parameter(torch.zeros(dim)) if bias else None + else: + self.weight = None + self.bias = None + + def forward(self, input): + return F.layer_norm(input, self.dim, self.weight, self.bias, self.eps) + + +class RMSNorm(nn.Module): + def __init__(self, dim, eps: float, elementwise_affine: bool = True): + super().__init__() + + self.eps = eps + + if isinstance(dim, numbers.Integral): + dim = (dim,) + + self.dim = torch.Size(dim) + + if elementwise_affine: + self.weight = nn.Parameter(torch.ones(dim)) + else: + self.weight = None + + def forward(self, hidden_states): + input_dtype = hidden_states.dtype + variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.eps) + + if self.weight is not None: + # convert into half-precision if necessary + if self.weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(self.weight.dtype) + hidden_states = hidden_states * self.weight + else: + hidden_states = hidden_states.to(input_dtype) + + return hidden_states + + +class GlobalResponseNorm(nn.Module): + # Taken from https://github.com/facebookresearch/ConvNeXt-V2/blob/3608f67cc1dae164790c5d0aead7bf2d73d9719b/models/utils.py#L105 + def __init__(self, dim): + super().__init__() + self.gamma = nn.Parameter(torch.zeros(1, 1, 1, dim)) + self.beta = nn.Parameter(torch.zeros(1, 1, 1, dim)) + + def forward(self, x): + gx = torch.norm(x, p=2, dim=(1, 2), keepdim=True) + nx = gx / (gx.mean(dim=-1, keepdim=True) + 1e-6) + return self.gamma * (x * nx) + self.beta + x diff --git a/diffusers-0.27.0/src/diffusers/models/prior_transformer.py b/diffusers-0.27.0/src/diffusers/models/prior_transformer.py new file mode 100755 index 0000000..328835a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/prior_transformer.py @@ -0,0 +1,12 @@ +from ..utils import deprecate +from .transformers.prior_transformer import PriorTransformer, PriorTransformerOutput + + +class PriorTransformerOutput(PriorTransformerOutput): + deprecation_message = "Importing `PriorTransformerOutput` from `diffusers.models.prior_transformer` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.prior_transformer import PriorTransformerOutput`, instead." + deprecate("PriorTransformerOutput", "0.29", deprecation_message) + + +class PriorTransformer(PriorTransformer): + deprecation_message = "Importing `PriorTransformer` from `diffusers.models.prior_transformer` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.prior_transformer import PriorTransformer`, instead." + deprecate("PriorTransformer", "0.29", deprecation_message) diff --git a/diffusers-0.27.0/src/diffusers/models/resnet.py b/diffusers-0.27.0/src/diffusers/models/resnet.py new file mode 100755 index 0000000..ec75861 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/resnet.py @@ -0,0 +1,802 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# `TemporalConvLayer` Copyright 2024 Alibaba DAMO-VILAB, The ModelScope Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import partial +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..utils import deprecate +from .activations import get_activation +from .attention_processor import SpatialNorm +from .downsampling import ( # noqa + Downsample1D, + Downsample2D, + FirDownsample2D, + KDownsample2D, + downsample_2d, +) +from .normalization import AdaGroupNorm +from .upsampling import ( # noqa + FirUpsample2D, + KUpsample2D, + Upsample1D, + Upsample2D, + upfirdn2d_native, + upsample_2d, +) + + +class ResnetBlockCondNorm2D(nn.Module): + r""" + A Resnet block that use normalization layer that incorporate conditioning information. + + Parameters: + in_channels (`int`): The number of channels in the input. + out_channels (`int`, *optional*, default to be `None`): + The number of output channels for the first conv2d layer. If None, same as `in_channels`. + dropout (`float`, *optional*, defaults to `0.0`): The dropout probability to use. + temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. + groups (`int`, *optional*, default to `32`): The number of groups to use for the first normalization layer. + groups_out (`int`, *optional*, default to None): + The number of groups to use for the second normalization layer. if set to None, same as `groups`. + eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the normalization. + non_linearity (`str`, *optional*, default to `"swish"`): the activation function to use. + time_embedding_norm (`str`, *optional*, default to `"ada_group"` ): + The normalization layer for time embedding `temb`. Currently only support "ada_group" or "spatial". + kernel (`torch.FloatTensor`, optional, default to None): FIR filter, see + [`~models.resnet.FirUpsample2D`] and [`~models.resnet.FirDownsample2D`]. + output_scale_factor (`float`, *optional*, default to be `1.0`): the scale factor to use for the output. + use_in_shortcut (`bool`, *optional*, default to `True`): + If `True`, add a 1x1 nn.conv2d layer for skip-connection. + up (`bool`, *optional*, default to `False`): If `True`, add an upsample layer. + down (`bool`, *optional*, default to `False`): If `True`, add a downsample layer. + conv_shortcut_bias (`bool`, *optional*, default to `True`): If `True`, adds a learnable bias to the + `conv_shortcut` output. + conv_2d_out_channels (`int`, *optional*, default to `None`): the number of channels in the output. + If None, same as `out_channels`. + """ + + def __init__( + self, + *, + in_channels: int, + out_channels: Optional[int] = None, + conv_shortcut: bool = False, + dropout: float = 0.0, + temb_channels: int = 512, + groups: int = 32, + groups_out: Optional[int] = None, + eps: float = 1e-6, + non_linearity: str = "swish", + time_embedding_norm: str = "ada_group", # ada_group, spatial + output_scale_factor: float = 1.0, + use_in_shortcut: Optional[bool] = None, + up: bool = False, + down: bool = False, + conv_shortcut_bias: bool = True, + conv_2d_out_channels: Optional[int] = None, + ): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.use_conv_shortcut = conv_shortcut + self.up = up + self.down = down + self.output_scale_factor = output_scale_factor + self.time_embedding_norm = time_embedding_norm + + conv_cls = nn.Conv2d + + if groups_out is None: + groups_out = groups + + if self.time_embedding_norm == "ada_group": # ada_group + self.norm1 = AdaGroupNorm(temb_channels, in_channels, groups, eps=eps) + elif self.time_embedding_norm == "spatial": + self.norm1 = SpatialNorm(in_channels, temb_channels) + else: + raise ValueError(f" unsupported time_embedding_norm: {self.time_embedding_norm}") + + self.conv1 = conv_cls(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + + if self.time_embedding_norm == "ada_group": # ada_group + self.norm2 = AdaGroupNorm(temb_channels, out_channels, groups_out, eps=eps) + elif self.time_embedding_norm == "spatial": # spatial + self.norm2 = SpatialNorm(out_channels, temb_channels) + else: + raise ValueError(f" unsupported time_embedding_norm: {self.time_embedding_norm}") + + self.dropout = torch.nn.Dropout(dropout) + + conv_2d_out_channels = conv_2d_out_channels or out_channels + self.conv2 = conv_cls(out_channels, conv_2d_out_channels, kernel_size=3, stride=1, padding=1) + + self.nonlinearity = get_activation(non_linearity) + + self.upsample = self.downsample = None + if self.up: + self.upsample = Upsample2D(in_channels, use_conv=False) + elif self.down: + self.downsample = Downsample2D(in_channels, use_conv=False, padding=1, name="op") + + self.use_in_shortcut = self.in_channels != conv_2d_out_channels if use_in_shortcut is None else use_in_shortcut + + self.conv_shortcut = None + if self.use_in_shortcut: + self.conv_shortcut = conv_cls( + in_channels, + conv_2d_out_channels, + kernel_size=1, + stride=1, + padding=0, + bias=conv_shortcut_bias, + ) + + def forward(self, input_tensor: torch.FloatTensor, temb: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + hidden_states = input_tensor + + hidden_states = self.norm1(hidden_states, temb) + + hidden_states = self.nonlinearity(hidden_states) + + if self.upsample is not None: + # upsample_nearest_nhwc fails with large batch sizes. see https://github.com/huggingface/diffusers/issues/984 + if hidden_states.shape[0] >= 64: + input_tensor = input_tensor.contiguous() + hidden_states = hidden_states.contiguous() + input_tensor = self.upsample(input_tensor) + hidden_states = self.upsample(hidden_states) + + elif self.downsample is not None: + input_tensor = self.downsample(input_tensor) + hidden_states = self.downsample(hidden_states) + + hidden_states = self.conv1(hidden_states) + + hidden_states = self.norm2(hidden_states, temb) + + hidden_states = self.nonlinearity(hidden_states) + + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.conv_shortcut is not None: + input_tensor = self.conv_shortcut(input_tensor) + + output_tensor = (input_tensor + hidden_states) / self.output_scale_factor + + return output_tensor + + +class ResnetBlock2D(nn.Module): + r""" + A Resnet block. + + Parameters: + in_channels (`int`): The number of channels in the input. + out_channels (`int`, *optional*, default to be `None`): + The number of output channels for the first conv2d layer. If None, same as `in_channels`. + dropout (`float`, *optional*, defaults to `0.0`): The dropout probability to use. + temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. + groups (`int`, *optional*, default to `32`): The number of groups to use for the first normalization layer. + groups_out (`int`, *optional*, default to None): + The number of groups to use for the second normalization layer. if set to None, same as `groups`. + eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the normalization. + non_linearity (`str`, *optional*, default to `"swish"`): the activation function to use. + time_embedding_norm (`str`, *optional*, default to `"default"` ): Time scale shift config. + By default, apply timestep embedding conditioning with a simple shift mechanism. Choose "scale_shift" + for a stronger conditioning with scale and shift. + kernel (`torch.FloatTensor`, optional, default to None): FIR filter, see + [`~models.resnet.FirUpsample2D`] and [`~models.resnet.FirDownsample2D`]. + output_scale_factor (`float`, *optional*, default to be `1.0`): the scale factor to use for the output. + use_in_shortcut (`bool`, *optional*, default to `True`): + If `True`, add a 1x1 nn.conv2d layer for skip-connection. + up (`bool`, *optional*, default to `False`): If `True`, add an upsample layer. + down (`bool`, *optional*, default to `False`): If `True`, add a downsample layer. + conv_shortcut_bias (`bool`, *optional*, default to `True`): If `True`, adds a learnable bias to the + `conv_shortcut` output. + conv_2d_out_channels (`int`, *optional*, default to `None`): the number of channels in the output. + If None, same as `out_channels`. + """ + + def __init__( + self, + *, + in_channels: int, + out_channels: Optional[int] = None, + conv_shortcut: bool = False, + dropout: float = 0.0, + temb_channels: int = 512, + groups: int = 32, + groups_out: Optional[int] = None, + pre_norm: bool = True, + eps: float = 1e-6, + non_linearity: str = "swish", + skip_time_act: bool = False, + time_embedding_norm: str = "default", # default, scale_shift, + kernel: Optional[torch.FloatTensor] = None, + output_scale_factor: float = 1.0, + use_in_shortcut: Optional[bool] = None, + up: bool = False, + down: bool = False, + conv_shortcut_bias: bool = True, + conv_2d_out_channels: Optional[int] = None, + ): + super().__init__() + if time_embedding_norm == "ada_group": + raise ValueError( + "This class cannot be used with `time_embedding_norm==ada_group`, please use `ResnetBlockCondNorm2D` instead", + ) + if time_embedding_norm == "spatial": + raise ValueError( + "This class cannot be used with `time_embedding_norm==spatial`, please use `ResnetBlockCondNorm2D` instead", + ) + + self.pre_norm = True + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.use_conv_shortcut = conv_shortcut + self.up = up + self.down = down + self.output_scale_factor = output_scale_factor + self.time_embedding_norm = time_embedding_norm + self.skip_time_act = skip_time_act + + linear_cls = nn.Linear + conv_cls = nn.Conv2d + + if groups_out is None: + groups_out = groups + + self.norm1 = torch.nn.GroupNorm(num_groups=groups, num_channels=in_channels, eps=eps, affine=True) + + self.conv1 = conv_cls(in_channels, out_channels, kernel_size=3, stride=1, padding=1) + + if temb_channels is not None: + if self.time_embedding_norm == "default": + self.time_emb_proj = linear_cls(temb_channels, out_channels) + elif self.time_embedding_norm == "scale_shift": + self.time_emb_proj = linear_cls(temb_channels, 2 * out_channels) + else: + raise ValueError(f"unknown time_embedding_norm : {self.time_embedding_norm} ") + else: + self.time_emb_proj = None + + self.norm2 = torch.nn.GroupNorm(num_groups=groups_out, num_channels=out_channels, eps=eps, affine=True) + + self.dropout = torch.nn.Dropout(dropout) + conv_2d_out_channels = conv_2d_out_channels or out_channels + self.conv2 = conv_cls(out_channels, conv_2d_out_channels, kernel_size=3, stride=1, padding=1) + + self.nonlinearity = get_activation(non_linearity) + + self.upsample = self.downsample = None + if self.up: + if kernel == "fir": + fir_kernel = (1, 3, 3, 1) + self.upsample = lambda x: upsample_2d(x, kernel=fir_kernel) + elif kernel == "sde_vp": + self.upsample = partial(F.interpolate, scale_factor=2.0, mode="nearest") + else: + self.upsample = Upsample2D(in_channels, use_conv=False) + elif self.down: + if kernel == "fir": + fir_kernel = (1, 3, 3, 1) + self.downsample = lambda x: downsample_2d(x, kernel=fir_kernel) + elif kernel == "sde_vp": + self.downsample = partial(F.avg_pool2d, kernel_size=2, stride=2) + else: + self.downsample = Downsample2D(in_channels, use_conv=False, padding=1, name="op") + + self.use_in_shortcut = self.in_channels != conv_2d_out_channels if use_in_shortcut is None else use_in_shortcut + + self.conv_shortcut = None + if self.use_in_shortcut: + self.conv_shortcut = conv_cls( + in_channels, + conv_2d_out_channels, + kernel_size=1, + stride=1, + padding=0, + bias=conv_shortcut_bias, + ) + + def forward(self, input_tensor: torch.FloatTensor, temb: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + hidden_states = input_tensor + + hidden_states = self.norm1(hidden_states) + hidden_states = self.nonlinearity(hidden_states) + + if self.upsample is not None: + # upsample_nearest_nhwc fails with large batch sizes. see https://github.com/huggingface/diffusers/issues/984 + if hidden_states.shape[0] >= 64: + input_tensor = input_tensor.contiguous() + hidden_states = hidden_states.contiguous() + input_tensor = self.upsample(input_tensor) + hidden_states = self.upsample(hidden_states) + elif self.downsample is not None: + input_tensor = self.downsample(input_tensor) + hidden_states = self.downsample(hidden_states) + + hidden_states = self.conv1(hidden_states) + + if self.time_emb_proj is not None: + if not self.skip_time_act: + temb = self.nonlinearity(temb) + temb = self.time_emb_proj(temb)[:, :, None, None] + + if self.time_embedding_norm == "default": + if temb is not None: + hidden_states = hidden_states + temb + hidden_states = self.norm2(hidden_states) + elif self.time_embedding_norm == "scale_shift": + if temb is None: + raise ValueError( + f" `temb` should not be None when `time_embedding_norm` is {self.time_embedding_norm}" + ) + time_scale, time_shift = torch.chunk(temb, 2, dim=1) + hidden_states = self.norm2(hidden_states) + hidden_states = hidden_states * (1 + time_scale) + time_shift + else: + hidden_states = self.norm2(hidden_states) + + hidden_states = self.nonlinearity(hidden_states) + + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.conv_shortcut is not None: + input_tensor = self.conv_shortcut(input_tensor) + + output_tensor = (input_tensor + hidden_states) / self.output_scale_factor + + return output_tensor + + +# unet_rl.py +def rearrange_dims(tensor: torch.Tensor) -> torch.Tensor: + if len(tensor.shape) == 2: + return tensor[:, :, None] + if len(tensor.shape) == 3: + return tensor[:, :, None, :] + elif len(tensor.shape) == 4: + return tensor[:, :, 0, :] + else: + raise ValueError(f"`len(tensor)`: {len(tensor)} has to be 2, 3 or 4.") + + +class Conv1dBlock(nn.Module): + """ + Conv1d --> GroupNorm --> Mish + + Parameters: + inp_channels (`int`): Number of input channels. + out_channels (`int`): Number of output channels. + kernel_size (`int` or `tuple`): Size of the convolving kernel. + n_groups (`int`, default `8`): Number of groups to separate the channels into. + activation (`str`, defaults to `mish`): Name of the activation function. + """ + + def __init__( + self, + inp_channels: int, + out_channels: int, + kernel_size: Union[int, Tuple[int, int]], + n_groups: int = 8, + activation: str = "mish", + ): + super().__init__() + + self.conv1d = nn.Conv1d(inp_channels, out_channels, kernel_size, padding=kernel_size // 2) + self.group_norm = nn.GroupNorm(n_groups, out_channels) + self.mish = get_activation(activation) + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + intermediate_repr = self.conv1d(inputs) + intermediate_repr = rearrange_dims(intermediate_repr) + intermediate_repr = self.group_norm(intermediate_repr) + intermediate_repr = rearrange_dims(intermediate_repr) + output = self.mish(intermediate_repr) + return output + + +# unet_rl.py +class ResidualTemporalBlock1D(nn.Module): + """ + Residual 1D block with temporal convolutions. + + Parameters: + inp_channels (`int`): Number of input channels. + out_channels (`int`): Number of output channels. + embed_dim (`int`): Embedding dimension. + kernel_size (`int` or `tuple`): Size of the convolving kernel. + activation (`str`, defaults `mish`): It is possible to choose the right activation function. + """ + + def __init__( + self, + inp_channels: int, + out_channels: int, + embed_dim: int, + kernel_size: Union[int, Tuple[int, int]] = 5, + activation: str = "mish", + ): + super().__init__() + self.conv_in = Conv1dBlock(inp_channels, out_channels, kernel_size) + self.conv_out = Conv1dBlock(out_channels, out_channels, kernel_size) + + self.time_emb_act = get_activation(activation) + self.time_emb = nn.Linear(embed_dim, out_channels) + + self.residual_conv = ( + nn.Conv1d(inp_channels, out_channels, 1) if inp_channels != out_channels else nn.Identity() + ) + + def forward(self, inputs: torch.Tensor, t: torch.Tensor) -> torch.Tensor: + """ + Args: + inputs : [ batch_size x inp_channels x horizon ] + t : [ batch_size x embed_dim ] + + returns: + out : [ batch_size x out_channels x horizon ] + """ + t = self.time_emb_act(t) + t = self.time_emb(t) + out = self.conv_in(inputs) + rearrange_dims(t) + out = self.conv_out(out) + return out + self.residual_conv(inputs) + + +class TemporalConvLayer(nn.Module): + """ + Temporal convolutional layer that can be used for video (sequence of images) input Code mostly copied from: + https://github.com/modelscope/modelscope/blob/1509fdb973e5871f37148a4b5e5964cafd43e64d/modelscope/models/multi_modal/video_synthesis/unet_sd.py#L1016 + + Parameters: + in_dim (`int`): Number of input channels. + out_dim (`int`): Number of output channels. + dropout (`float`, *optional*, defaults to `0.0`): The dropout probability to use. + """ + + def __init__( + self, + in_dim: int, + out_dim: Optional[int] = None, + dropout: float = 0.0, + norm_num_groups: int = 32, + ): + super().__init__() + out_dim = out_dim or in_dim + self.in_dim = in_dim + self.out_dim = out_dim + + # conv layers + self.conv1 = nn.Sequential( + nn.GroupNorm(norm_num_groups, in_dim), + nn.SiLU(), + nn.Conv3d(in_dim, out_dim, (3, 1, 1), padding=(1, 0, 0)), + ) + self.conv2 = nn.Sequential( + nn.GroupNorm(norm_num_groups, out_dim), + nn.SiLU(), + nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0)), + ) + self.conv3 = nn.Sequential( + nn.GroupNorm(norm_num_groups, out_dim), + nn.SiLU(), + nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0)), + ) + self.conv4 = nn.Sequential( + nn.GroupNorm(norm_num_groups, out_dim), + nn.SiLU(), + nn.Dropout(dropout), + nn.Conv3d(out_dim, in_dim, (3, 1, 1), padding=(1, 0, 0)), + ) + + # zero out the last layer params,so the conv block is identity + nn.init.zeros_(self.conv4[-1].weight) + nn.init.zeros_(self.conv4[-1].bias) + + def forward(self, hidden_states: torch.Tensor, num_frames: int = 1) -> torch.Tensor: + hidden_states = ( + hidden_states[None, :].reshape((-1, num_frames) + hidden_states.shape[1:]).permute(0, 2, 1, 3, 4) + ) + + identity = hidden_states + hidden_states = self.conv1(hidden_states) + hidden_states = self.conv2(hidden_states) + hidden_states = self.conv3(hidden_states) + hidden_states = self.conv4(hidden_states) + + hidden_states = identity + hidden_states + + hidden_states = hidden_states.permute(0, 2, 1, 3, 4).reshape( + (hidden_states.shape[0] * hidden_states.shape[2], -1) + hidden_states.shape[3:] + ) + return hidden_states + + +class TemporalResnetBlock(nn.Module): + r""" + A Resnet block. + + Parameters: + in_channels (`int`): The number of channels in the input. + out_channels (`int`, *optional*, default to be `None`): + The number of output channels for the first conv2d layer. If None, same as `in_channels`. + temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. + eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the normalization. + """ + + def __init__( + self, + in_channels: int, + out_channels: Optional[int] = None, + temb_channels: int = 512, + eps: float = 1e-6, + ): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + + kernel_size = (3, 1, 1) + padding = [k // 2 for k in kernel_size] + + self.norm1 = torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=eps, affine=True) + self.conv1 = nn.Conv3d( + in_channels, + out_channels, + kernel_size=kernel_size, + stride=1, + padding=padding, + ) + + if temb_channels is not None: + self.time_emb_proj = nn.Linear(temb_channels, out_channels) + else: + self.time_emb_proj = None + + self.norm2 = torch.nn.GroupNorm(num_groups=32, num_channels=out_channels, eps=eps, affine=True) + + self.dropout = torch.nn.Dropout(0.0) + self.conv2 = nn.Conv3d( + out_channels, + out_channels, + kernel_size=kernel_size, + stride=1, + padding=padding, + ) + + self.nonlinearity = get_activation("silu") + + self.use_in_shortcut = self.in_channels != out_channels + + self.conv_shortcut = None + if self.use_in_shortcut: + self.conv_shortcut = nn.Conv3d( + in_channels, + out_channels, + kernel_size=1, + stride=1, + padding=0, + ) + + def forward(self, input_tensor: torch.FloatTensor, temb: torch.FloatTensor) -> torch.FloatTensor: + hidden_states = input_tensor + + hidden_states = self.norm1(hidden_states) + hidden_states = self.nonlinearity(hidden_states) + hidden_states = self.conv1(hidden_states) + + if self.time_emb_proj is not None: + temb = self.nonlinearity(temb) + temb = self.time_emb_proj(temb)[:, :, :, None, None] + temb = temb.permute(0, 2, 1, 3, 4) + hidden_states = hidden_states + temb + + hidden_states = self.norm2(hidden_states) + hidden_states = self.nonlinearity(hidden_states) + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.conv_shortcut is not None: + input_tensor = self.conv_shortcut(input_tensor) + + output_tensor = input_tensor + hidden_states + + return output_tensor + + +# VideoResBlock +class SpatioTemporalResBlock(nn.Module): + r""" + A SpatioTemporal Resnet block. + + Parameters: + in_channels (`int`): The number of channels in the input. + out_channels (`int`, *optional*, default to be `None`): + The number of output channels for the first conv2d layer. If None, same as `in_channels`. + temb_channels (`int`, *optional*, default to `512`): the number of channels in timestep embedding. + eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the spatial resenet. + temporal_eps (`float`, *optional*, defaults to `eps`): The epsilon to use for the temporal resnet. + merge_factor (`float`, *optional*, defaults to `0.5`): The merge factor to use for the temporal mixing. + merge_strategy (`str`, *optional*, defaults to `learned_with_images`): + The merge strategy to use for the temporal mixing. + switch_spatial_to_temporal_mix (`bool`, *optional*, defaults to `False`): + If `True`, switch the spatial and temporal mixing. + """ + + def __init__( + self, + in_channels: int, + out_channels: Optional[int] = None, + temb_channels: int = 512, + eps: float = 1e-6, + temporal_eps: Optional[float] = None, + merge_factor: float = 0.5, + merge_strategy="learned_with_images", + switch_spatial_to_temporal_mix: bool = False, + ): + super().__init__() + + self.spatial_res_block = ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=eps, + ) + + self.temporal_res_block = TemporalResnetBlock( + in_channels=out_channels if out_channels is not None else in_channels, + out_channels=out_channels if out_channels is not None else in_channels, + temb_channels=temb_channels, + eps=temporal_eps if temporal_eps is not None else eps, + ) + + self.time_mixer = AlphaBlender( + alpha=merge_factor, + merge_strategy=merge_strategy, + switch_spatial_to_temporal_mix=switch_spatial_to_temporal_mix, + ) + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + image_only_indicator: Optional[torch.Tensor] = None, + ): + num_frames = image_only_indicator.shape[-1] + hidden_states = self.spatial_res_block(hidden_states, temb) + + batch_frames, channels, height, width = hidden_states.shape + batch_size = batch_frames // num_frames + + hidden_states_mix = ( + hidden_states[None, :].reshape(batch_size, num_frames, channels, height, width).permute(0, 2, 1, 3, 4) + ) + hidden_states = ( + hidden_states[None, :].reshape(batch_size, num_frames, channels, height, width).permute(0, 2, 1, 3, 4) + ) + + if temb is not None: + temb = temb.reshape(batch_size, num_frames, -1) + + hidden_states = self.temporal_res_block(hidden_states, temb) + hidden_states = self.time_mixer( + x_spatial=hidden_states_mix, + x_temporal=hidden_states, + image_only_indicator=image_only_indicator, + ) + + hidden_states = hidden_states.permute(0, 2, 1, 3, 4).reshape(batch_frames, channels, height, width) + return hidden_states + + +class AlphaBlender(nn.Module): + r""" + A module to blend spatial and temporal features. + + Parameters: + alpha (`float`): The initial value of the blending factor. + merge_strategy (`str`, *optional*, defaults to `learned_with_images`): + The merge strategy to use for the temporal mixing. + switch_spatial_to_temporal_mix (`bool`, *optional*, defaults to `False`): + If `True`, switch the spatial and temporal mixing. + """ + + strategies = ["learned", "fixed", "learned_with_images"] + + def __init__( + self, + alpha: float, + merge_strategy: str = "learned_with_images", + switch_spatial_to_temporal_mix: bool = False, + ): + super().__init__() + self.merge_strategy = merge_strategy + self.switch_spatial_to_temporal_mix = switch_spatial_to_temporal_mix # For TemporalVAE + + if merge_strategy not in self.strategies: + raise ValueError(f"merge_strategy needs to be in {self.strategies}") + + if self.merge_strategy == "fixed": + self.register_buffer("mix_factor", torch.Tensor([alpha])) + elif self.merge_strategy == "learned" or self.merge_strategy == "learned_with_images": + self.register_parameter("mix_factor", torch.nn.Parameter(torch.Tensor([alpha]))) + else: + raise ValueError(f"Unknown merge strategy {self.merge_strategy}") + + def get_alpha(self, image_only_indicator: torch.Tensor, ndims: int) -> torch.Tensor: + if self.merge_strategy == "fixed": + alpha = self.mix_factor + + elif self.merge_strategy == "learned": + alpha = torch.sigmoid(self.mix_factor) + + elif self.merge_strategy == "learned_with_images": + if image_only_indicator is None: + raise ValueError("Please provide image_only_indicator to use learned_with_images merge strategy") + + alpha = torch.where( + image_only_indicator.bool(), + torch.ones(1, 1, device=image_only_indicator.device), + torch.sigmoid(self.mix_factor)[..., None], + ) + + # (batch, channel, frames, height, width) + if ndims == 5: + alpha = alpha[:, None, :, None, None] + # (batch*frames, height*width, channels) + elif ndims == 3: + alpha = alpha.reshape(-1)[:, None, None] + else: + raise ValueError(f"Unexpected ndims {ndims}. Dimensions should be 3 or 5") + + else: + raise NotImplementedError + + return alpha + + def forward( + self, + x_spatial: torch.Tensor, + x_temporal: torch.Tensor, + image_only_indicator: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + alpha = self.get_alpha(image_only_indicator, x_spatial.ndim) + alpha = alpha.to(x_spatial.dtype) + + if self.switch_spatial_to_temporal_mix: + alpha = 1.0 - alpha + + x = alpha * x_spatial + (1.0 - alpha) * x_temporal + return x diff --git a/diffusers-0.27.0/src/diffusers/models/resnet_flax.py b/diffusers-0.27.0/src/diffusers/models/resnet_flax.py new file mode 100755 index 0000000..f8bb478 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/resnet_flax.py @@ -0,0 +1,124 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import flax.linen as nn +import jax +import jax.numpy as jnp + + +class FlaxUpsample2D(nn.Module): + out_channels: int + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.conv = nn.Conv( + self.out_channels, + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + def __call__(self, hidden_states): + batch, height, width, channels = hidden_states.shape + hidden_states = jax.image.resize( + hidden_states, + shape=(batch, height * 2, width * 2, channels), + method="nearest", + ) + hidden_states = self.conv(hidden_states) + return hidden_states + + +class FlaxDownsample2D(nn.Module): + out_channels: int + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.conv = nn.Conv( + self.out_channels, + kernel_size=(3, 3), + strides=(2, 2), + padding=((1, 1), (1, 1)), # padding="VALID", + dtype=self.dtype, + ) + + def __call__(self, hidden_states): + # pad = ((0, 0), (0, 1), (0, 1), (0, 0)) # pad height and width dim + # hidden_states = jnp.pad(hidden_states, pad_width=pad) + hidden_states = self.conv(hidden_states) + return hidden_states + + +class FlaxResnetBlock2D(nn.Module): + in_channels: int + out_channels: int = None + dropout_prob: float = 0.0 + use_nin_shortcut: bool = None + dtype: jnp.dtype = jnp.float32 + + def setup(self): + out_channels = self.in_channels if self.out_channels is None else self.out_channels + + self.norm1 = nn.GroupNorm(num_groups=32, epsilon=1e-5) + self.conv1 = nn.Conv( + out_channels, + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + self.time_emb_proj = nn.Dense(out_channels, dtype=self.dtype) + + self.norm2 = nn.GroupNorm(num_groups=32, epsilon=1e-5) + self.dropout = nn.Dropout(self.dropout_prob) + self.conv2 = nn.Conv( + out_channels, + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + use_nin_shortcut = self.in_channels != out_channels if self.use_nin_shortcut is None else self.use_nin_shortcut + + self.conv_shortcut = None + if use_nin_shortcut: + self.conv_shortcut = nn.Conv( + out_channels, + kernel_size=(1, 1), + strides=(1, 1), + padding="VALID", + dtype=self.dtype, + ) + + def __call__(self, hidden_states, temb, deterministic=True): + residual = hidden_states + hidden_states = self.norm1(hidden_states) + hidden_states = nn.swish(hidden_states) + hidden_states = self.conv1(hidden_states) + + temb = self.time_emb_proj(nn.swish(temb)) + temb = jnp.expand_dims(jnp.expand_dims(temb, 1), 1) + hidden_states = hidden_states + temb + + hidden_states = self.norm2(hidden_states) + hidden_states = nn.swish(hidden_states) + hidden_states = self.dropout(hidden_states, deterministic) + hidden_states = self.conv2(hidden_states) + + if self.conv_shortcut is not None: + residual = self.conv_shortcut(residual) + + return hidden_states + residual diff --git a/diffusers-0.27.0/src/diffusers/models/t5_film_transformer.py b/diffusers-0.27.0/src/diffusers/models/t5_film_transformer.py new file mode 100755 index 0000000..6aa5ff7 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/t5_film_transformer.py @@ -0,0 +1,70 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ..utils import deprecate +from .transformers.t5_film_transformer import ( + DecoderLayer, + NewGELUActivation, + T5DenseGatedActDense, + T5FilmDecoder, + T5FiLMLayer, + T5LayerCrossAttention, + T5LayerFFCond, + T5LayerNorm, + T5LayerSelfAttentionCond, +) + + +class T5FilmDecoder(T5FilmDecoder): + deprecation_message = "Importing `T5FilmDecoder` from `diffusers.models.t5_film_transformer` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.t5_film_transformer import T5FilmDecoder`, instead." + deprecate("T5FilmDecoder", "0.29", deprecation_message) + + +class DecoderLayer(DecoderLayer): + deprecation_message = "Importing `DecoderLayer` from `diffusers.models.t5_film_transformer` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.t5_film_transformer import DecoderLayer`, instead." + deprecate("DecoderLayer", "0.29", deprecation_message) + + +class T5LayerSelfAttentionCond(T5LayerSelfAttentionCond): + deprecation_message = "Importing `T5LayerSelfAttentionCond` from `diffusers.models.t5_film_transformer` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.t5_film_transformer import T5LayerSelfAttentionCond`, instead." + deprecate("T5LayerSelfAttentionCond", "0.29", deprecation_message) + + +class T5LayerCrossAttention(T5LayerCrossAttention): + deprecation_message = "Importing `T5LayerCrossAttention` from `diffusers.models.t5_film_transformer` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.t5_film_transformer import T5LayerCrossAttention`, instead." + deprecate("T5LayerCrossAttention", "0.29", deprecation_message) + + +class T5LayerFFCond(T5LayerFFCond): + deprecation_message = "Importing `T5LayerFFCond` from `diffusers.models.t5_film_transformer` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.t5_film_transformer import T5LayerFFCond`, instead." + deprecate("T5LayerFFCond", "0.29", deprecation_message) + + +class T5DenseGatedActDense(T5DenseGatedActDense): + deprecation_message = "Importing `T5DenseGatedActDense` from `diffusers.models.t5_film_transformer` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.t5_film_transformer import T5DenseGatedActDense`, instead." + deprecate("T5DenseGatedActDense", "0.29", deprecation_message) + + +class T5LayerNorm(T5LayerNorm): + deprecation_message = "Importing `T5LayerNorm` from `diffusers.models.t5_film_transformer` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.t5_film_transformer import T5LayerNorm`, instead." + deprecate("T5LayerNorm", "0.29", deprecation_message) + + +class NewGELUActivation(NewGELUActivation): + deprecation_message = "Importing `T5LayerNorm` from `diffusers.models.t5_film_transformer` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.t5_film_transformer import NewGELUActivation`, instead." + deprecate("NewGELUActivation", "0.29", deprecation_message) + + +class T5FiLMLayer(T5FiLMLayer): + deprecation_message = "Importing `T5FiLMLayer` from `diffusers.models.t5_film_transformer` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.t5_film_transformer import T5FiLMLayer`, instead." + deprecate("T5FiLMLayer", "0.29", deprecation_message) diff --git a/diffusers-0.27.0/src/diffusers/models/transformer_2d.py b/diffusers-0.27.0/src/diffusers/models/transformer_2d.py new file mode 100755 index 0000000..5d8ef13 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/transformer_2d.py @@ -0,0 +1,25 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ..utils import deprecate +from .transformers.transformer_2d import Transformer2DModel, Transformer2DModelOutput + + +class Transformer2DModelOutput(Transformer2DModelOutput): + deprecation_message = "Importing `Transformer2DModelOutput` from `diffusers.models.transformer_2d` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.transformer_2d import Transformer2DModelOutput`, instead." + deprecate("Transformer2DModelOutput", "0.29", deprecation_message) + + +class Transformer2DModel(Transformer2DModel): + deprecation_message = "Importing `Transformer2DModel` from `diffusers.models.transformer_2d` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.transformer_2d import Transformer2DModel`, instead." + deprecate("Transformer2DModel", "0.29", deprecation_message) diff --git a/diffusers-0.27.0/src/diffusers/models/transformer_temporal.py b/diffusers-0.27.0/src/diffusers/models/transformer_temporal.py new file mode 100755 index 0000000..83c7a8e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/transformer_temporal.py @@ -0,0 +1,34 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ..utils import deprecate +from .transformers.transformer_temporal import ( + TransformerSpatioTemporalModel, + TransformerTemporalModel, + TransformerTemporalModelOutput, +) + + +class TransformerTemporalModelOutput(TransformerTemporalModelOutput): + deprecation_message = "Importing `TransformerTemporalModelOutput` from `diffusers.models.transformer_temporal` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.tranformer_temporal import TransformerTemporalModelOutput`, instead." + deprecate("TransformerTemporalModelOutput", "0.29", deprecation_message) + + +class TransformerTemporalModel(TransformerTemporalModel): + deprecation_message = "Importing `TransformerTemporalModel` from `diffusers.models.transformer_temporal` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.tranformer_temporal import TransformerTemporalModel`, instead." + deprecate("TransformerTemporalModel", "0.29", deprecation_message) + + +class TransformerSpatioTemporalModel(TransformerSpatioTemporalModel): + deprecation_message = "Importing `TransformerSpatioTemporalModel` from `diffusers.models.transformer_temporal` is deprecated and this will be removed in a future version. Please use `from diffusers.models.transformers.tranformer_temporal import TransformerSpatioTemporalModel`, instead." + deprecate("TransformerTemporalModelOutput", "0.29", deprecation_message) diff --git a/diffusers-0.27.0/src/diffusers/models/transformers/__init__.py b/diffusers-0.27.0/src/diffusers/models/transformers/__init__.py new file mode 100755 index 0000000..dc78a72 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/transformers/__init__.py @@ -0,0 +1,9 @@ +from ...utils import is_torch_available + + +if is_torch_available(): + from .dual_transformer_2d import DualTransformer2DModel + from .prior_transformer import PriorTransformer + from .t5_film_transformer import T5FilmDecoder + from .transformer_2d import Transformer2DModel + from .transformer_temporal import TransformerTemporalModel diff --git a/diffusers-0.27.0/src/diffusers/models/transformers/dual_transformer_2d.py b/diffusers-0.27.0/src/diffusers/models/transformers/dual_transformer_2d.py new file mode 100755 index 0000000..96849bd --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/transformers/dual_transformer_2d.py @@ -0,0 +1,155 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional + +from torch import nn + +from .transformer_2d import Transformer2DModel, Transformer2DModelOutput + + +class DualTransformer2DModel(nn.Module): + """ + Dual transformer wrapper that combines two `Transformer2DModel`s for mixed inference. + + Parameters: + num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention. + attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head. + in_channels (`int`, *optional*): + Pass if the input is continuous. The number of channels in the input and output. + num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use. + dropout (`float`, *optional*, defaults to 0.1): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The number of encoder_hidden_states dimensions to use. + sample_size (`int`, *optional*): Pass if the input is discrete. The width of the latent images. + Note that this is fixed at training time as it is used for learning a number of position embeddings. See + `ImagePositionalEmbeddings`. + num_vector_embeds (`int`, *optional*): + Pass if the input is discrete. The number of classes of the vector embeddings of the latent pixels. + Includes the class for the masked latent pixel. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + num_embeds_ada_norm ( `int`, *optional*): Pass if at least one of the norm_layers is `AdaLayerNorm`. + The number of diffusion steps used during training. Note that this is fixed at training time as it is used + to learn a number of embeddings that are added to the hidden states. During inference, you can denoise for + up to but not more than steps than `num_embeds_ada_norm`. + attention_bias (`bool`, *optional*): + Configure if the TransformerBlocks' attention should contain a bias parameter. + """ + + def __init__( + self, + num_attention_heads: int = 16, + attention_head_dim: int = 88, + in_channels: Optional[int] = None, + num_layers: int = 1, + dropout: float = 0.0, + norm_num_groups: int = 32, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + sample_size: Optional[int] = None, + num_vector_embeds: Optional[int] = None, + activation_fn: str = "geglu", + num_embeds_ada_norm: Optional[int] = None, + ): + super().__init__() + self.transformers = nn.ModuleList( + [ + Transformer2DModel( + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + in_channels=in_channels, + num_layers=num_layers, + dropout=dropout, + norm_num_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + attention_bias=attention_bias, + sample_size=sample_size, + num_vector_embeds=num_vector_embeds, + activation_fn=activation_fn, + num_embeds_ada_norm=num_embeds_ada_norm, + ) + for _ in range(2) + ] + ) + + # Variables that can be set by a pipeline: + + # The ratio of transformer1 to transformer2's output states to be combined during inference + self.mix_ratio = 0.5 + + # The shape of `encoder_hidden_states` is expected to be + # `(batch_size, condition_lengths[0]+condition_lengths[1], num_features)` + self.condition_lengths = [77, 257] + + # Which transformer to use to encode which condition. + # E.g. `(1, 0)` means that we'll use `transformers[1](conditions[0])` and `transformers[0](conditions[1])` + self.transformer_index_for_condition = [1, 0] + + def forward( + self, + hidden_states, + encoder_hidden_states, + timestep=None, + attention_mask=None, + cross_attention_kwargs=None, + return_dict: bool = True, + ): + """ + Args: + hidden_states ( When discrete, `torch.LongTensor` of shape `(batch size, num latent pixels)`. + When continuous, `torch.FloatTensor` of shape `(batch size, channel, height, width)`): Input + hidden_states. + encoder_hidden_states ( `torch.LongTensor` of shape `(batch size, encoder_hidden_states dim)`, *optional*): + Conditional embeddings for cross attention layer. If not given, cross-attention defaults to + self-attention. + timestep ( `torch.long`, *optional*): + Optional timestep to be applied as an embedding in AdaLayerNorm's. Used to indicate denoising step. + attention_mask (`torch.FloatTensor`, *optional*): + Optional attention mask to be applied in Attention. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`models.unets.unet_2d_condition.UNet2DConditionOutput`] instead of a plain tuple. + + Returns: + [`~models.transformer_2d.Transformer2DModelOutput`] or `tuple`: + [`~models.transformer_2d.Transformer2DModelOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is the sample tensor. + """ + input_states = hidden_states + + encoded_states = [] + tokens_start = 0 + # attention_mask is not used yet + for i in range(2): + # for each of the two transformers, pass the corresponding condition tokens + condition_state = encoder_hidden_states[:, tokens_start : tokens_start + self.condition_lengths[i]] + transformer_index = self.transformer_index_for_condition[i] + encoded_state = self.transformers[transformer_index]( + input_states, + encoder_hidden_states=condition_state, + timestep=timestep, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + encoded_states.append(encoded_state - input_states) + tokens_start += self.condition_lengths[i] + + output_states = encoded_states[0] * self.mix_ratio + encoded_states[1] * (1 - self.mix_ratio) + output_states = output_states + input_states + + if not return_dict: + return (output_states,) + + return Transformer2DModelOutput(sample=output_states) diff --git a/diffusers-0.27.0/src/diffusers/models/transformers/prior_transformer.py b/diffusers-0.27.0/src/diffusers/models/transformers/prior_transformer.py new file mode 100755 index 0000000..990eabe --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/transformers/prior_transformer.py @@ -0,0 +1,380 @@ +from dataclasses import dataclass +from typing import Dict, Optional, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...loaders import PeftAdapterMixin, UNet2DConditionLoadersMixin +from ...utils import BaseOutput +from ..attention import BasicTransformerBlock +from ..attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from ..embeddings import TimestepEmbedding, Timesteps +from ..modeling_utils import ModelMixin + + +@dataclass +class PriorTransformerOutput(BaseOutput): + """ + The output of [`PriorTransformer`]. + + Args: + predicted_image_embedding (`torch.FloatTensor` of shape `(batch_size, embedding_dim)`): + The predicted CLIP image embedding conditioned on the CLIP text embedding input. + """ + + predicted_image_embedding: torch.FloatTensor + + +class PriorTransformer(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin, PeftAdapterMixin): + """ + A Prior Transformer model. + + Parameters: + num_attention_heads (`int`, *optional*, defaults to 32): The number of heads to use for multi-head attention. + attention_head_dim (`int`, *optional*, defaults to 64): The number of channels in each head. + num_layers (`int`, *optional*, defaults to 20): The number of layers of Transformer blocks to use. + embedding_dim (`int`, *optional*, defaults to 768): The dimension of the model input `hidden_states` + num_embeddings (`int`, *optional*, defaults to 77): + The number of embeddings of the model input `hidden_states` + additional_embeddings (`int`, *optional*, defaults to 4): The number of additional tokens appended to the + projected `hidden_states`. The actual length of the used `hidden_states` is `num_embeddings + + additional_embeddings`. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + time_embed_act_fn (`str`, *optional*, defaults to 'silu'): + The activation function to use to create timestep embeddings. + norm_in_type (`str`, *optional*, defaults to None): The normalization layer to apply on hidden states before + passing to Transformer blocks. Set it to `None` if normalization is not needed. + embedding_proj_norm_type (`str`, *optional*, defaults to None): + The normalization layer to apply on the input `proj_embedding`. Set it to `None` if normalization is not + needed. + encoder_hid_proj_type (`str`, *optional*, defaults to `linear`): + The projection layer to apply on the input `encoder_hidden_states`. Set it to `None` if + `encoder_hidden_states` is `None`. + added_emb_type (`str`, *optional*, defaults to `prd`): Additional embeddings to condition the model. + Choose from `prd` or `None`. if choose `prd`, it will prepend a token indicating the (quantized) dot + product between the text embedding and image embedding as proposed in the unclip paper + https://arxiv.org/abs/2204.06125 If it is `None`, no additional embeddings will be prepended. + time_embed_dim (`int, *optional*, defaults to None): The dimension of timestep embeddings. + If None, will be set to `num_attention_heads * attention_head_dim` + embedding_proj_dim (`int`, *optional*, default to None): + The dimension of `proj_embedding`. If None, will be set to `embedding_dim`. + clip_embed_dim (`int`, *optional*, default to None): + The dimension of the output. If None, will be set to `embedding_dim`. + """ + + @register_to_config + def __init__( + self, + num_attention_heads: int = 32, + attention_head_dim: int = 64, + num_layers: int = 20, + embedding_dim: int = 768, + num_embeddings=77, + additional_embeddings=4, + dropout: float = 0.0, + time_embed_act_fn: str = "silu", + norm_in_type: Optional[str] = None, # layer + embedding_proj_norm_type: Optional[str] = None, # layer + encoder_hid_proj_type: Optional[str] = "linear", # linear + added_emb_type: Optional[str] = "prd", # prd + time_embed_dim: Optional[int] = None, + embedding_proj_dim: Optional[int] = None, + clip_embed_dim: Optional[int] = None, + ): + super().__init__() + self.num_attention_heads = num_attention_heads + self.attention_head_dim = attention_head_dim + inner_dim = num_attention_heads * attention_head_dim + self.additional_embeddings = additional_embeddings + + time_embed_dim = time_embed_dim or inner_dim + embedding_proj_dim = embedding_proj_dim or embedding_dim + clip_embed_dim = clip_embed_dim or embedding_dim + + self.time_proj = Timesteps(inner_dim, True, 0) + self.time_embedding = TimestepEmbedding(inner_dim, time_embed_dim, out_dim=inner_dim, act_fn=time_embed_act_fn) + + self.proj_in = nn.Linear(embedding_dim, inner_dim) + + if embedding_proj_norm_type is None: + self.embedding_proj_norm = None + elif embedding_proj_norm_type == "layer": + self.embedding_proj_norm = nn.LayerNorm(embedding_proj_dim) + else: + raise ValueError(f"unsupported embedding_proj_norm_type: {embedding_proj_norm_type}") + + self.embedding_proj = nn.Linear(embedding_proj_dim, inner_dim) + + if encoder_hid_proj_type is None: + self.encoder_hidden_states_proj = None + elif encoder_hid_proj_type == "linear": + self.encoder_hidden_states_proj = nn.Linear(embedding_dim, inner_dim) + else: + raise ValueError(f"unsupported encoder_hid_proj_type: {encoder_hid_proj_type}") + + self.positional_embedding = nn.Parameter(torch.zeros(1, num_embeddings + additional_embeddings, inner_dim)) + + if added_emb_type == "prd": + self.prd_embedding = nn.Parameter(torch.zeros(1, 1, inner_dim)) + elif added_emb_type is None: + self.prd_embedding = None + else: + raise ValueError( + f"`added_emb_type`: {added_emb_type} is not supported. Make sure to choose one of `'prd'` or `None`." + ) + + self.transformer_blocks = nn.ModuleList( + [ + BasicTransformerBlock( + inner_dim, + num_attention_heads, + attention_head_dim, + dropout=dropout, + activation_fn="gelu", + attention_bias=True, + ) + for d in range(num_layers) + ] + ) + + if norm_in_type == "layer": + self.norm_in = nn.LayerNorm(inner_dim) + elif norm_in_type is None: + self.norm_in = None + else: + raise ValueError(f"Unsupported norm_in_type: {norm_in_type}.") + + self.norm_out = nn.LayerNorm(inner_dim) + + self.proj_to_clip_embeddings = nn.Linear(inner_dim, clip_embed_dim) + + causal_attention_mask = torch.full( + [num_embeddings + additional_embeddings, num_embeddings + additional_embeddings], -10000.0 + ) + causal_attention_mask.triu_(1) + causal_attention_mask = causal_attention_mask[None, ...] + self.register_buffer("causal_attention_mask", causal_attention_mask, persistent=False) + + self.clip_mean = nn.Parameter(torch.zeros(1, clip_embed_dim)) + self.clip_std = nn.Parameter(torch.zeros(1, clip_embed_dim)) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + def forward( + self, + hidden_states, + timestep: Union[torch.Tensor, float, int], + proj_embedding: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.BoolTensor] = None, + return_dict: bool = True, + ): + """ + The [`PriorTransformer`] forward method. + + Args: + hidden_states (`torch.FloatTensor` of shape `(batch_size, embedding_dim)`): + The currently predicted image embeddings. + timestep (`torch.LongTensor`): + Current denoising step. + proj_embedding (`torch.FloatTensor` of shape `(batch_size, embedding_dim)`): + Projected embedding vector the denoising process is conditioned on. + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, num_embeddings, embedding_dim)`): + Hidden states of the text embeddings the denoising process is conditioned on. + attention_mask (`torch.BoolTensor` of shape `(batch_size, num_embeddings)`): + Text mask for the text embeddings. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.prior_transformer.PriorTransformerOutput`] instead of a plain + tuple. + + Returns: + [`~models.prior_transformer.PriorTransformerOutput`] or `tuple`: + If return_dict is True, a [`~models.prior_transformer.PriorTransformerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + """ + batch_size = hidden_states.shape[0] + + timesteps = timestep + if not torch.is_tensor(timesteps): + timesteps = torch.tensor([timesteps], dtype=torch.long, device=hidden_states.device) + elif torch.is_tensor(timesteps) and len(timesteps.shape) == 0: + timesteps = timesteps[None].to(hidden_states.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps * torch.ones(batch_size, dtype=timesteps.dtype, device=timesteps.device) + + timesteps_projected = self.time_proj(timesteps) + + # timesteps does not contain any weights and will always return f32 tensors + # but time_embedding might be fp16, so we need to cast here. + timesteps_projected = timesteps_projected.to(dtype=self.dtype) + time_embeddings = self.time_embedding(timesteps_projected) + + if self.embedding_proj_norm is not None: + proj_embedding = self.embedding_proj_norm(proj_embedding) + + proj_embeddings = self.embedding_proj(proj_embedding) + if self.encoder_hidden_states_proj is not None and encoder_hidden_states is not None: + encoder_hidden_states = self.encoder_hidden_states_proj(encoder_hidden_states) + elif self.encoder_hidden_states_proj is not None and encoder_hidden_states is None: + raise ValueError("`encoder_hidden_states_proj` requires `encoder_hidden_states` to be set") + + hidden_states = self.proj_in(hidden_states) + + positional_embeddings = self.positional_embedding.to(hidden_states.dtype) + + additional_embeds = [] + additional_embeddings_len = 0 + + if encoder_hidden_states is not None: + additional_embeds.append(encoder_hidden_states) + additional_embeddings_len += encoder_hidden_states.shape[1] + + if len(proj_embeddings.shape) == 2: + proj_embeddings = proj_embeddings[:, None, :] + + if len(hidden_states.shape) == 2: + hidden_states = hidden_states[:, None, :] + + additional_embeds = additional_embeds + [ + proj_embeddings, + time_embeddings[:, None, :], + hidden_states, + ] + + if self.prd_embedding is not None: + prd_embedding = self.prd_embedding.to(hidden_states.dtype).expand(batch_size, -1, -1) + additional_embeds.append(prd_embedding) + + hidden_states = torch.cat( + additional_embeds, + dim=1, + ) + + # Allow positional_embedding to not include the `addtional_embeddings` and instead pad it with zeros for these additional tokens + additional_embeddings_len = additional_embeddings_len + proj_embeddings.shape[1] + 1 + if positional_embeddings.shape[1] < hidden_states.shape[1]: + positional_embeddings = F.pad( + positional_embeddings, + ( + 0, + 0, + additional_embeddings_len, + self.prd_embedding.shape[1] if self.prd_embedding is not None else 0, + ), + value=0.0, + ) + + hidden_states = hidden_states + positional_embeddings + + if attention_mask is not None: + attention_mask = (1 - attention_mask.to(hidden_states.dtype)) * -10000.0 + attention_mask = F.pad(attention_mask, (0, self.additional_embeddings), value=0.0) + attention_mask = (attention_mask[:, None, :] + self.causal_attention_mask).to(hidden_states.dtype) + attention_mask = attention_mask.repeat_interleave(self.config.num_attention_heads, dim=0) + + if self.norm_in is not None: + hidden_states = self.norm_in(hidden_states) + + for block in self.transformer_blocks: + hidden_states = block(hidden_states, attention_mask=attention_mask) + + hidden_states = self.norm_out(hidden_states) + + if self.prd_embedding is not None: + hidden_states = hidden_states[:, -1] + else: + hidden_states = hidden_states[:, additional_embeddings_len:] + + predicted_image_embedding = self.proj_to_clip_embeddings(hidden_states) + + if not return_dict: + return (predicted_image_embedding,) + + return PriorTransformerOutput(predicted_image_embedding=predicted_image_embedding) + + def post_process_latents(self, prior_latents): + prior_latents = (prior_latents * self.clip_std) + self.clip_mean + return prior_latents diff --git a/diffusers-0.27.0/src/diffusers/models/transformers/t5_film_transformer.py b/diffusers-0.27.0/src/diffusers/models/transformers/t5_film_transformer.py new file mode 100755 index 0000000..bff98db --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/transformers/t5_film_transformer.py @@ -0,0 +1,438 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from typing import Optional, Tuple + +import torch +from torch import nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ..attention_processor import Attention +from ..embeddings import get_timestep_embedding +from ..modeling_utils import ModelMixin + + +class T5FilmDecoder(ModelMixin, ConfigMixin): + r""" + T5 style decoder with FiLM conditioning. + + Args: + input_dims (`int`, *optional*, defaults to `128`): + The number of input dimensions. + targets_length (`int`, *optional*, defaults to `256`): + The length of the targets. + d_model (`int`, *optional*, defaults to `768`): + Size of the input hidden states. + num_layers (`int`, *optional*, defaults to `12`): + The number of `DecoderLayer`'s to use. + num_heads (`int`, *optional*, defaults to `12`): + The number of attention heads to use. + d_kv (`int`, *optional*, defaults to `64`): + Size of the key-value projection vectors. + d_ff (`int`, *optional*, defaults to `2048`): + The number of dimensions in the intermediate feed-forward layer of `DecoderLayer`'s. + dropout_rate (`float`, *optional*, defaults to `0.1`): + Dropout probability. + """ + + @register_to_config + def __init__( + self, + input_dims: int = 128, + targets_length: int = 256, + max_decoder_noise_time: float = 2000.0, + d_model: int = 768, + num_layers: int = 12, + num_heads: int = 12, + d_kv: int = 64, + d_ff: int = 2048, + dropout_rate: float = 0.1, + ): + super().__init__() + + self.conditioning_emb = nn.Sequential( + nn.Linear(d_model, d_model * 4, bias=False), + nn.SiLU(), + nn.Linear(d_model * 4, d_model * 4, bias=False), + nn.SiLU(), + ) + + self.position_encoding = nn.Embedding(targets_length, d_model) + self.position_encoding.weight.requires_grad = False + + self.continuous_inputs_projection = nn.Linear(input_dims, d_model, bias=False) + + self.dropout = nn.Dropout(p=dropout_rate) + + self.decoders = nn.ModuleList() + for lyr_num in range(num_layers): + # FiLM conditional T5 decoder + lyr = DecoderLayer(d_model=d_model, d_kv=d_kv, num_heads=num_heads, d_ff=d_ff, dropout_rate=dropout_rate) + self.decoders.append(lyr) + + self.decoder_norm = T5LayerNorm(d_model) + + self.post_dropout = nn.Dropout(p=dropout_rate) + self.spec_out = nn.Linear(d_model, input_dims, bias=False) + + def encoder_decoder_mask(self, query_input: torch.FloatTensor, key_input: torch.FloatTensor) -> torch.FloatTensor: + mask = torch.mul(query_input.unsqueeze(-1), key_input.unsqueeze(-2)) + return mask.unsqueeze(-3) + + def forward(self, encodings_and_masks, decoder_input_tokens, decoder_noise_time): + batch, _, _ = decoder_input_tokens.shape + assert decoder_noise_time.shape == (batch,) + + # decoder_noise_time is in [0, 1), so rescale to expected timing range. + time_steps = get_timestep_embedding( + decoder_noise_time * self.config.max_decoder_noise_time, + embedding_dim=self.config.d_model, + max_period=self.config.max_decoder_noise_time, + ).to(dtype=self.dtype) + + conditioning_emb = self.conditioning_emb(time_steps).unsqueeze(1) + + assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4) + + seq_length = decoder_input_tokens.shape[1] + + # If we want to use relative positions for audio context, we can just offset + # this sequence by the length of encodings_and_masks. + decoder_positions = torch.broadcast_to( + torch.arange(seq_length, device=decoder_input_tokens.device), + (batch, seq_length), + ) + + position_encodings = self.position_encoding(decoder_positions) + + inputs = self.continuous_inputs_projection(decoder_input_tokens) + inputs += position_encodings + y = self.dropout(inputs) + + # decoder: No padding present. + decoder_mask = torch.ones( + decoder_input_tokens.shape[:2], device=decoder_input_tokens.device, dtype=inputs.dtype + ) + + # Translate encoding masks to encoder-decoder masks. + encodings_and_encdec_masks = [(x, self.encoder_decoder_mask(decoder_mask, y)) for x, y in encodings_and_masks] + + # cross attend style: concat encodings + encoded = torch.cat([x[0] for x in encodings_and_encdec_masks], dim=1) + encoder_decoder_mask = torch.cat([x[1] for x in encodings_and_encdec_masks], dim=-1) + + for lyr in self.decoders: + y = lyr( + y, + conditioning_emb=conditioning_emb, + encoder_hidden_states=encoded, + encoder_attention_mask=encoder_decoder_mask, + )[0] + + y = self.decoder_norm(y) + y = self.post_dropout(y) + + spec_out = self.spec_out(y) + return spec_out + + +class DecoderLayer(nn.Module): + r""" + T5 decoder layer. + + Args: + d_model (`int`): + Size of the input hidden states. + d_kv (`int`): + Size of the key-value projection vectors. + num_heads (`int`): + Number of attention heads. + d_ff (`int`): + Size of the intermediate feed-forward layer. + dropout_rate (`float`): + Dropout probability. + layer_norm_epsilon (`float`, *optional*, defaults to `1e-6`): + A small value used for numerical stability to avoid dividing by zero. + """ + + def __init__( + self, d_model: int, d_kv: int, num_heads: int, d_ff: int, dropout_rate: float, layer_norm_epsilon: float = 1e-6 + ): + super().__init__() + self.layer = nn.ModuleList() + + # cond self attention: layer 0 + self.layer.append( + T5LayerSelfAttentionCond(d_model=d_model, d_kv=d_kv, num_heads=num_heads, dropout_rate=dropout_rate) + ) + + # cross attention: layer 1 + self.layer.append( + T5LayerCrossAttention( + d_model=d_model, + d_kv=d_kv, + num_heads=num_heads, + dropout_rate=dropout_rate, + layer_norm_epsilon=layer_norm_epsilon, + ) + ) + + # Film Cond MLP + dropout: last layer + self.layer.append( + T5LayerFFCond(d_model=d_model, d_ff=d_ff, dropout_rate=dropout_rate, layer_norm_epsilon=layer_norm_epsilon) + ) + + def forward( + self, + hidden_states: torch.FloatTensor, + conditioning_emb: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + encoder_decoder_position_bias=None, + ) -> Tuple[torch.FloatTensor]: + hidden_states = self.layer[0]( + hidden_states, + conditioning_emb=conditioning_emb, + attention_mask=attention_mask, + ) + + if encoder_hidden_states is not None: + encoder_extended_attention_mask = torch.where(encoder_attention_mask > 0, 0, -1e10).to( + encoder_hidden_states.dtype + ) + + hidden_states = self.layer[1]( + hidden_states, + key_value_states=encoder_hidden_states, + attention_mask=encoder_extended_attention_mask, + ) + + # Apply Film Conditional Feed Forward layer + hidden_states = self.layer[-1](hidden_states, conditioning_emb) + + return (hidden_states,) + + +class T5LayerSelfAttentionCond(nn.Module): + r""" + T5 style self-attention layer with conditioning. + + Args: + d_model (`int`): + Size of the input hidden states. + d_kv (`int`): + Size of the key-value projection vectors. + num_heads (`int`): + Number of attention heads. + dropout_rate (`float`): + Dropout probability. + """ + + def __init__(self, d_model: int, d_kv: int, num_heads: int, dropout_rate: float): + super().__init__() + self.layer_norm = T5LayerNorm(d_model) + self.FiLMLayer = T5FiLMLayer(in_features=d_model * 4, out_features=d_model) + self.attention = Attention(query_dim=d_model, heads=num_heads, dim_head=d_kv, out_bias=False, scale_qk=False) + self.dropout = nn.Dropout(dropout_rate) + + def forward( + self, + hidden_states: torch.FloatTensor, + conditioning_emb: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + # pre_self_attention_layer_norm + normed_hidden_states = self.layer_norm(hidden_states) + + if conditioning_emb is not None: + normed_hidden_states = self.FiLMLayer(normed_hidden_states, conditioning_emb) + + # Self-attention block + attention_output = self.attention(normed_hidden_states) + + hidden_states = hidden_states + self.dropout(attention_output) + + return hidden_states + + +class T5LayerCrossAttention(nn.Module): + r""" + T5 style cross-attention layer. + + Args: + d_model (`int`): + Size of the input hidden states. + d_kv (`int`): + Size of the key-value projection vectors. + num_heads (`int`): + Number of attention heads. + dropout_rate (`float`): + Dropout probability. + layer_norm_epsilon (`float`): + A small value used for numerical stability to avoid dividing by zero. + """ + + def __init__(self, d_model: int, d_kv: int, num_heads: int, dropout_rate: float, layer_norm_epsilon: float): + super().__init__() + self.attention = Attention(query_dim=d_model, heads=num_heads, dim_head=d_kv, out_bias=False, scale_qk=False) + self.layer_norm = T5LayerNorm(d_model, eps=layer_norm_epsilon) + self.dropout = nn.Dropout(dropout_rate) + + def forward( + self, + hidden_states: torch.FloatTensor, + key_value_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + normed_hidden_states = self.layer_norm(hidden_states) + attention_output = self.attention( + normed_hidden_states, + encoder_hidden_states=key_value_states, + attention_mask=attention_mask.squeeze(1), + ) + layer_output = hidden_states + self.dropout(attention_output) + return layer_output + + +class T5LayerFFCond(nn.Module): + r""" + T5 style feed-forward conditional layer. + + Args: + d_model (`int`): + Size of the input hidden states. + d_ff (`int`): + Size of the intermediate feed-forward layer. + dropout_rate (`float`): + Dropout probability. + layer_norm_epsilon (`float`): + A small value used for numerical stability to avoid dividing by zero. + """ + + def __init__(self, d_model: int, d_ff: int, dropout_rate: float, layer_norm_epsilon: float): + super().__init__() + self.DenseReluDense = T5DenseGatedActDense(d_model=d_model, d_ff=d_ff, dropout_rate=dropout_rate) + self.film = T5FiLMLayer(in_features=d_model * 4, out_features=d_model) + self.layer_norm = T5LayerNorm(d_model, eps=layer_norm_epsilon) + self.dropout = nn.Dropout(dropout_rate) + + def forward( + self, hidden_states: torch.FloatTensor, conditioning_emb: Optional[torch.FloatTensor] = None + ) -> torch.FloatTensor: + forwarded_states = self.layer_norm(hidden_states) + if conditioning_emb is not None: + forwarded_states = self.film(forwarded_states, conditioning_emb) + + forwarded_states = self.DenseReluDense(forwarded_states) + hidden_states = hidden_states + self.dropout(forwarded_states) + return hidden_states + + +class T5DenseGatedActDense(nn.Module): + r""" + T5 style feed-forward layer with gated activations and dropout. + + Args: + d_model (`int`): + Size of the input hidden states. + d_ff (`int`): + Size of the intermediate feed-forward layer. + dropout_rate (`float`): + Dropout probability. + """ + + def __init__(self, d_model: int, d_ff: int, dropout_rate: float): + super().__init__() + self.wi_0 = nn.Linear(d_model, d_ff, bias=False) + self.wi_1 = nn.Linear(d_model, d_ff, bias=False) + self.wo = nn.Linear(d_ff, d_model, bias=False) + self.dropout = nn.Dropout(dropout_rate) + self.act = NewGELUActivation() + + def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + hidden_gelu = self.act(self.wi_0(hidden_states)) + hidden_linear = self.wi_1(hidden_states) + hidden_states = hidden_gelu * hidden_linear + hidden_states = self.dropout(hidden_states) + + hidden_states = self.wo(hidden_states) + return hidden_states + + +class T5LayerNorm(nn.Module): + r""" + T5 style layer normalization module. + + Args: + hidden_size (`int`): + Size of the input hidden states. + eps (`float`, `optional`, defaults to `1e-6`): + A small value used for numerical stability to avoid dividing by zero. + """ + + def __init__(self, hidden_size: int, eps: float = 1e-6): + """ + Construct a layernorm module in the T5 style. No bias and no subtraction of mean. + """ + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden_size)) + self.variance_epsilon = eps + + def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + # T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean + # Square Layer Normalization https://arxiv.org/abs/1910.07467 thus variance is calculated + # w/o mean and there is no bias. Additionally we want to make sure that the accumulation for + # half-precision inputs is done in fp32 + + variance = hidden_states.to(torch.float32).pow(2).mean(-1, keepdim=True) + hidden_states = hidden_states * torch.rsqrt(variance + self.variance_epsilon) + + # convert into half-precision if necessary + if self.weight.dtype in [torch.float16, torch.bfloat16]: + hidden_states = hidden_states.to(self.weight.dtype) + + return self.weight * hidden_states + + +class NewGELUActivation(nn.Module): + """ + Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT). Also see + the Gaussian Error Linear Units paper: https://arxiv.org/abs/1606.08415 + """ + + def forward(self, input: torch.Tensor) -> torch.Tensor: + return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (input + 0.044715 * torch.pow(input, 3.0)))) + + +class T5FiLMLayer(nn.Module): + """ + T5 style FiLM Layer. + + Args: + in_features (`int`): + Number of input features. + out_features (`int`): + Number of output features. + """ + + def __init__(self, in_features: int, out_features: int): + super().__init__() + self.scale_bias = nn.Linear(in_features, out_features * 2, bias=False) + + def forward(self, x: torch.FloatTensor, conditioning_emb: torch.FloatTensor) -> torch.FloatTensor: + emb = self.scale_bias(conditioning_emb) + scale, shift = torch.chunk(emb, 2, -1) + x = x * (1 + scale) + shift + return x diff --git a/diffusers-0.27.0/src/diffusers/models/transformers/transformer_2d.py b/diffusers-0.27.0/src/diffusers/models/transformers/transformer_2d.py new file mode 100755 index 0000000..555ea4f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/transformers/transformer_2d.py @@ -0,0 +1,456 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import torch +import torch.nn.functional as F +from torch import nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...utils import BaseOutput, deprecate, is_torch_version, logging +from ..attention import BasicTransformerBlock +from ..embeddings import ImagePositionalEmbeddings, PatchEmbed, PixArtAlphaTextProjection +from ..modeling_utils import ModelMixin +from ..normalization import AdaLayerNormSingle + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class Transformer2DModelOutput(BaseOutput): + """ + The output of [`Transformer2DModel`]. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` or `(batch size, num_vector_embeds - 1, num_latent_pixels)` if [`Transformer2DModel`] is discrete): + The hidden states output conditioned on the `encoder_hidden_states` input. If discrete, returns probability + distributions for the unnoised latent pixels. + """ + + sample: torch.FloatTensor + + +class Transformer2DModel(ModelMixin, ConfigMixin): + """ + A 2D Transformer model for image-like data. + + Parameters: + num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention. + attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head. + in_channels (`int`, *optional*): + The number of channels in the input and output (specify if the input is **continuous**). + num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The number of `encoder_hidden_states` dimensions to use. + sample_size (`int`, *optional*): The width of the latent images (specify if the input is **discrete**). + This is fixed during training since it is used to learn a number of position embeddings. + num_vector_embeds (`int`, *optional*): + The number of classes of the vector embeddings of the latent pixels (specify if the input is **discrete**). + Includes the class for the masked latent pixel. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to use in feed-forward. + num_embeds_ada_norm ( `int`, *optional*): + The number of diffusion steps used during training. Pass if at least one of the norm_layers is + `AdaLayerNorm`. This is fixed during training since it is used to learn a number of embeddings that are + added to the hidden states. + + During inference, you can denoise for up to but not more steps than `num_embeds_ada_norm`. + attention_bias (`bool`, *optional*): + Configure if the `TransformerBlocks` attention should contain a bias parameter. + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + num_attention_heads: int = 16, + attention_head_dim: int = 88, + in_channels: Optional[int] = None, + out_channels: Optional[int] = None, + num_layers: int = 1, + dropout: float = 0.0, + norm_num_groups: int = 32, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + sample_size: Optional[int] = None, + num_vector_embeds: Optional[int] = None, + patch_size: Optional[int] = None, + activation_fn: str = "geglu", + num_embeds_ada_norm: Optional[int] = None, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + double_self_attention: bool = False, + upcast_attention: bool = False, + norm_type: str = "layer_norm", # 'layer_norm', 'ada_norm', 'ada_norm_zero', 'ada_norm_single', 'ada_norm_continuous', 'layer_norm_i2vgen' + norm_elementwise_affine: bool = True, + norm_eps: float = 1e-5, + attention_type: str = "default", + caption_channels: int = None, + interpolation_scale: float = None, + ): + super().__init__() + if patch_size is not None: + if norm_type not in ["ada_norm", "ada_norm_zero", "ada_norm_single"]: + raise NotImplementedError( + f"Forward pass is not implemented when `patch_size` is not None and `norm_type` is '{norm_type}'." + ) + elif norm_type in ["ada_norm", "ada_norm_zero"] and num_embeds_ada_norm is None: + raise ValueError( + f"When using a `patch_size` and this `norm_type` ({norm_type}), `num_embeds_ada_norm` cannot be None." + ) + + self.use_linear_projection = use_linear_projection + self.num_attention_heads = num_attention_heads + self.attention_head_dim = attention_head_dim + inner_dim = num_attention_heads * attention_head_dim + + conv_cls = nn.Conv2d + linear_cls = nn.Linear + + # 1. Transformer2DModel can process both standard continuous images of shape `(batch_size, num_channels, width, height)` as well as quantized image embeddings of shape `(batch_size, num_image_vectors)` + # Define whether input is continuous or discrete depending on configuration + self.is_input_continuous = (in_channels is not None) and (patch_size is None) + self.is_input_vectorized = num_vector_embeds is not None + self.is_input_patches = in_channels is not None and patch_size is not None + + if norm_type == "layer_norm" and num_embeds_ada_norm is not None: + deprecation_message = ( + f"The configuration file of this model: {self.__class__} is outdated. `norm_type` is either not set or" + " incorrectly set to `'layer_norm'`.Make sure to set `norm_type` to `'ada_norm'` in the config." + " Please make sure to update the config accordingly as leaving `norm_type` might led to incorrect" + " results in future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it" + " would be very nice if you could open a Pull request for the `transformer/config.json` file" + ) + deprecate("norm_type!=num_embeds_ada_norm", "1.0.0", deprecation_message, standard_warn=False) + norm_type = "ada_norm" + + if self.is_input_continuous and self.is_input_vectorized: + raise ValueError( + f"Cannot define both `in_channels`: {in_channels} and `num_vector_embeds`: {num_vector_embeds}. Make" + " sure that either `in_channels` or `num_vector_embeds` is None." + ) + elif self.is_input_vectorized and self.is_input_patches: + raise ValueError( + f"Cannot define both `num_vector_embeds`: {num_vector_embeds} and `patch_size`: {patch_size}. Make" + " sure that either `num_vector_embeds` or `num_patches` is None." + ) + elif not self.is_input_continuous and not self.is_input_vectorized and not self.is_input_patches: + raise ValueError( + f"Has to define `in_channels`: {in_channels}, `num_vector_embeds`: {num_vector_embeds}, or patch_size:" + f" {patch_size}. Make sure that `in_channels`, `num_vector_embeds` or `num_patches` is not None." + ) + + # 2. Define input layers + if self.is_input_continuous: + self.in_channels = in_channels + + self.norm = torch.nn.GroupNorm(num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True) + if use_linear_projection: + self.proj_in = linear_cls(in_channels, inner_dim) + else: + self.proj_in = conv_cls(in_channels, inner_dim, kernel_size=1, stride=1, padding=0) + elif self.is_input_vectorized: + assert sample_size is not None, "Transformer2DModel over discrete input must provide sample_size" + assert num_vector_embeds is not None, "Transformer2DModel over discrete input must provide num_embed" + + self.height = sample_size + self.width = sample_size + self.num_vector_embeds = num_vector_embeds + self.num_latent_pixels = self.height * self.width + + self.latent_image_embedding = ImagePositionalEmbeddings( + num_embed=num_vector_embeds, embed_dim=inner_dim, height=self.height, width=self.width + ) + elif self.is_input_patches: + assert sample_size is not None, "Transformer2DModel over patched input must provide sample_size" + + self.height = sample_size + self.width = sample_size + + self.patch_size = patch_size + interpolation_scale = ( + interpolation_scale if interpolation_scale is not None else max(self.config.sample_size // 64, 1) + ) + self.pos_embed = PatchEmbed( + height=sample_size, + width=sample_size, + patch_size=patch_size, + in_channels=in_channels, + embed_dim=inner_dim, + interpolation_scale=interpolation_scale, + ) + + # 3. Define transformers blocks + self.transformer_blocks = nn.ModuleList( + [ + BasicTransformerBlock( + inner_dim, + num_attention_heads, + attention_head_dim, + dropout=dropout, + cross_attention_dim=cross_attention_dim, + activation_fn=activation_fn, + num_embeds_ada_norm=num_embeds_ada_norm, + attention_bias=attention_bias, + only_cross_attention=only_cross_attention, + double_self_attention=double_self_attention, + upcast_attention=upcast_attention, + norm_type=norm_type, + norm_elementwise_affine=norm_elementwise_affine, + norm_eps=norm_eps, + attention_type=attention_type, + ) + for d in range(num_layers) + ] + ) + + # 4. Define output layers + self.out_channels = in_channels if out_channels is None else out_channels + if self.is_input_continuous: + # TODO: should use out_channels for continuous projections + if use_linear_projection: + self.proj_out = linear_cls(inner_dim, in_channels) + else: + self.proj_out = conv_cls(inner_dim, in_channels, kernel_size=1, stride=1, padding=0) + elif self.is_input_vectorized: + self.norm_out = nn.LayerNorm(inner_dim) + self.out = nn.Linear(inner_dim, self.num_vector_embeds - 1) + elif self.is_input_patches and norm_type != "ada_norm_single": + self.norm_out = nn.LayerNorm(inner_dim, elementwise_affine=False, eps=1e-6) + self.proj_out_1 = nn.Linear(inner_dim, 2 * inner_dim) + self.proj_out_2 = nn.Linear(inner_dim, patch_size * patch_size * self.out_channels) + elif self.is_input_patches and norm_type == "ada_norm_single": + self.norm_out = nn.LayerNorm(inner_dim, elementwise_affine=False, eps=1e-6) + self.scale_shift_table = nn.Parameter(torch.randn(2, inner_dim) / inner_dim**0.5) + self.proj_out = nn.Linear(inner_dim, patch_size * patch_size * self.out_channels) + + # 5. PixArt-Alpha blocks. + self.adaln_single = None + self.use_additional_conditions = False + if norm_type == "ada_norm_single": + self.use_additional_conditions = self.config.sample_size == 128 + # TODO(Sayak, PVP) clean this, for now we use sample size to determine whether to use + # additional conditions until we find better name + self.adaln_single = AdaLayerNormSingle(inner_dim, use_additional_conditions=self.use_additional_conditions) + + self.caption_projection = None + if caption_channels is not None: + self.caption_projection = PixArtAlphaTextProjection(in_features=caption_channels, hidden_size=inner_dim) + + self.gradient_checkpointing = False + + def _set_gradient_checkpointing(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + timestep: Optional[torch.LongTensor] = None, + added_cond_kwargs: Dict[str, torch.Tensor] = None, + class_labels: Optional[torch.LongTensor] = None, + cross_attention_kwargs: Dict[str, Any] = None, + attention_mask: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + return_dict: bool = True, + ): + """ + The [`Transformer2DModel`] forward method. + + Args: + hidden_states (`torch.LongTensor` of shape `(batch size, num latent pixels)` if discrete, `torch.FloatTensor` of shape `(batch size, channel, height, width)` if continuous): + Input `hidden_states`. + encoder_hidden_states ( `torch.FloatTensor` of shape `(batch size, sequence len, embed dims)`, *optional*): + Conditional embeddings for cross attention layer. If not given, cross-attention defaults to + self-attention. + timestep ( `torch.LongTensor`, *optional*): + Used to indicate denoising step. Optional timestep to be applied as an embedding in `AdaLayerNorm`. + class_labels ( `torch.LongTensor` of shape `(batch size, num classes)`, *optional*): + Used to indicate class labels conditioning. Optional class labels to be applied as an embedding in + `AdaLayerZeroNorm`. + cross_attention_kwargs ( `Dict[str, Any]`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + attention_mask ( `torch.Tensor`, *optional*): + An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask + is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large + negative values to the attention scores corresponding to "discard" tokens. + encoder_attention_mask ( `torch.Tensor`, *optional*): + Cross-attention mask applied to `encoder_hidden_states`. Two formats supported: + + * Mask `(batch, sequence_length)` True = keep, False = discard. + * Bias `(batch, 1, sequence_length)` 0 = keep, -10000 = discard. + + If `ndim == 2`: will be interpreted as a mask, then converted into a bias consistent with the format + above. This bias will be added to the cross-attention scores. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] instead of a plain + tuple. + + Returns: + If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a + `tuple` where the first element is the sample tensor. + """ + if cross_attention_kwargs is not None: + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + # ensure attention_mask is a bias, and give it a singleton query_tokens dimension. + # we may have done this conversion already, e.g. if we came here via UNet2DConditionModel#forward. + # we can tell by counting dims; if ndim == 2: it's a mask rather than a bias. + # expects mask of shape: + # [batch, key_tokens] + # adds singleton query_tokens dimension: + # [batch, 1, key_tokens] + # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes: + # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn) + # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn) + if attention_mask is not None and attention_mask.ndim == 2: + # assume that mask is expressed as: + # (1 = keep, 0 = discard) + # convert mask into a bias that can be added to attention scores: + # (keep = +0, discard = -10000.0) + attention_mask = (1 - attention_mask.to(hidden_states.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # convert encoder_attention_mask to a bias the same way we do for attention_mask + if encoder_attention_mask is not None and encoder_attention_mask.ndim == 2: + encoder_attention_mask = (1 - encoder_attention_mask.to(hidden_states.dtype)) * -10000.0 + encoder_attention_mask = encoder_attention_mask.unsqueeze(1) + + # 1. Input + if self.is_input_continuous: + batch, _, height, width = hidden_states.shape + residual = hidden_states + + hidden_states = self.norm(hidden_states) + if not self.use_linear_projection: + hidden_states = self.proj_in(hidden_states) + inner_dim = hidden_states.shape[1] + hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * width, inner_dim) + else: + inner_dim = hidden_states.shape[1] + hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch, height * width, inner_dim) + hidden_states = self.proj_in(hidden_states) + + elif self.is_input_vectorized: + hidden_states = self.latent_image_embedding(hidden_states) + elif self.is_input_patches: + height, width = hidden_states.shape[-2] // self.patch_size, hidden_states.shape[-1] // self.patch_size + hidden_states = self.pos_embed(hidden_states) + + if self.adaln_single is not None: + if self.use_additional_conditions and added_cond_kwargs is None: + raise ValueError( + "`added_cond_kwargs` cannot be None when using additional conditions for `adaln_single`." + ) + batch_size = hidden_states.shape[0] + timestep, embedded_timestep = self.adaln_single( + timestep, added_cond_kwargs, batch_size=batch_size, hidden_dtype=hidden_states.dtype + ) + + # 2. Blocks + if self.caption_projection is not None: + batch_size = hidden_states.shape[0] + encoder_hidden_states = self.caption_projection(encoder_hidden_states) + encoder_hidden_states = encoder_hidden_states.view(batch_size, -1, hidden_states.shape[-1]) + + for block in self.transformer_blocks: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), + hidden_states, + attention_mask, + encoder_hidden_states, + encoder_attention_mask, + timestep, + cross_attention_kwargs, + class_labels, + **ckpt_kwargs, + ) + else: + hidden_states = block( + hidden_states, + attention_mask=attention_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + timestep=timestep, + cross_attention_kwargs=cross_attention_kwargs, + class_labels=class_labels, + ) + + # 3. Output + if self.is_input_continuous: + if not self.use_linear_projection: + hidden_states = hidden_states.reshape(batch, height, width, inner_dim).permute(0, 3, 1, 2).contiguous() + hidden_states = self.proj_out(hidden_states) + else: + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states.reshape(batch, height, width, inner_dim).permute(0, 3, 1, 2).contiguous() + + output = hidden_states + residual + elif self.is_input_vectorized: + hidden_states = self.norm_out(hidden_states) + logits = self.out(hidden_states) + # (batch, self.num_vector_embeds - 1, self.num_latent_pixels) + logits = logits.permute(0, 2, 1) + + # log(p(x_0)) + output = F.log_softmax(logits.double(), dim=1).float() + + if self.is_input_patches: + if self.config.norm_type != "ada_norm_single": + conditioning = self.transformer_blocks[0].norm1.emb( + timestep, class_labels, hidden_dtype=hidden_states.dtype + ) + shift, scale = self.proj_out_1(F.silu(conditioning)).chunk(2, dim=1) + hidden_states = self.norm_out(hidden_states) * (1 + scale[:, None]) + shift[:, None] + hidden_states = self.proj_out_2(hidden_states) + elif self.config.norm_type == "ada_norm_single": + shift, scale = (self.scale_shift_table[None] + embedded_timestep[:, None]).chunk(2, dim=1) + hidden_states = self.norm_out(hidden_states) + # Modulation + hidden_states = hidden_states * (1 + scale) + shift + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states.squeeze(1) + + # unpatchify + if self.adaln_single is None: + height = width = int(hidden_states.shape[1] ** 0.5) + hidden_states = hidden_states.reshape( + shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels) + ) + hidden_states = torch.einsum("nhwpqc->nchpwq", hidden_states) + output = hidden_states.reshape( + shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size) + ) + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) diff --git a/diffusers-0.27.0/src/diffusers/models/transformers/transformer_temporal.py b/diffusers-0.27.0/src/diffusers/models/transformers/transformer_temporal.py new file mode 100755 index 0000000..9c61eae --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/transformers/transformer_temporal.py @@ -0,0 +1,379 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import torch +from torch import nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...utils import BaseOutput +from ..attention import BasicTransformerBlock, TemporalBasicTransformerBlock +from ..embeddings import TimestepEmbedding, Timesteps +from ..modeling_utils import ModelMixin +from ..resnet import AlphaBlender + + +@dataclass +class TransformerTemporalModelOutput(BaseOutput): + """ + The output of [`TransformerTemporalModel`]. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size x num_frames, num_channels, height, width)`): + The hidden states output conditioned on `encoder_hidden_states` input. + """ + + sample: torch.FloatTensor + + +class TransformerTemporalModel(ModelMixin, ConfigMixin): + """ + A Transformer model for video-like data. + + Parameters: + num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention. + attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head. + in_channels (`int`, *optional*): + The number of channels in the input and output (specify if the input is **continuous**). + num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The number of `encoder_hidden_states` dimensions to use. + attention_bias (`bool`, *optional*): + Configure if the `TransformerBlock` attention should contain a bias parameter. + sample_size (`int`, *optional*): The width of the latent images (specify if the input is **discrete**). + This is fixed during training since it is used to learn a number of position embeddings. + activation_fn (`str`, *optional*, defaults to `"geglu"`): + Activation function to use in feed-forward. See `diffusers.models.activations.get_activation` for supported + activation functions. + norm_elementwise_affine (`bool`, *optional*): + Configure if the `TransformerBlock` should use learnable elementwise affine parameters for normalization. + double_self_attention (`bool`, *optional*): + Configure if each `TransformerBlock` should contain two self-attention layers. + positional_embeddings: (`str`, *optional*): + The type of positional embeddings to apply to the sequence input before passing use. + num_positional_embeddings: (`int`, *optional*): + The maximum length of the sequence over which to apply positional embeddings. + """ + + @register_to_config + def __init__( + self, + num_attention_heads: int = 16, + attention_head_dim: int = 88, + in_channels: Optional[int] = None, + out_channels: Optional[int] = None, + num_layers: int = 1, + dropout: float = 0.0, + norm_num_groups: int = 32, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + sample_size: Optional[int] = None, + activation_fn: str = "geglu", + norm_elementwise_affine: bool = True, + double_self_attention: bool = True, + positional_embeddings: Optional[str] = None, + num_positional_embeddings: Optional[int] = None, + ): + super().__init__() + self.num_attention_heads = num_attention_heads + self.attention_head_dim = attention_head_dim + inner_dim = num_attention_heads * attention_head_dim + + self.in_channels = in_channels + + self.norm = torch.nn.GroupNorm(num_groups=norm_num_groups, num_channels=in_channels, eps=1e-6, affine=True) + self.proj_in = nn.Linear(in_channels, inner_dim) + + # 3. Define transformers blocks + self.transformer_blocks = nn.ModuleList( + [ + BasicTransformerBlock( + inner_dim, + num_attention_heads, + attention_head_dim, + dropout=dropout, + cross_attention_dim=cross_attention_dim, + activation_fn=activation_fn, + attention_bias=attention_bias, + double_self_attention=double_self_attention, + norm_elementwise_affine=norm_elementwise_affine, + positional_embeddings=positional_embeddings, + num_positional_embeddings=num_positional_embeddings, + ) + for d in range(num_layers) + ] + ) + + self.proj_out = nn.Linear(inner_dim, in_channels) + + def forward( + self, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.LongTensor] = None, + timestep: Optional[torch.LongTensor] = None, + class_labels: torch.LongTensor = None, + num_frames: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + return_dict: bool = True, + ) -> TransformerTemporalModelOutput: + """ + The [`TransformerTemporal`] forward method. + + Args: + hidden_states (`torch.LongTensor` of shape `(batch size, num latent pixels)` if discrete, `torch.FloatTensor` of shape `(batch size, channel, height, width)` if continuous): + Input hidden_states. + encoder_hidden_states ( `torch.LongTensor` of shape `(batch size, encoder_hidden_states dim)`, *optional*): + Conditional embeddings for cross attention layer. If not given, cross-attention defaults to + self-attention. + timestep ( `torch.LongTensor`, *optional*): + Used to indicate denoising step. Optional timestep to be applied as an embedding in `AdaLayerNorm`. + class_labels ( `torch.LongTensor` of shape `(batch size, num classes)`, *optional*): + Used to indicate class labels conditioning. Optional class labels to be applied as an embedding in + `AdaLayerZeroNorm`. + num_frames (`int`, *optional*, defaults to 1): + The number of frames to be processed per batch. This is used to reshape the hidden states. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] instead of a plain + tuple. + + Returns: + [`~models.transformer_temporal.TransformerTemporalModelOutput`] or `tuple`: + If `return_dict` is True, an [`~models.transformer_temporal.TransformerTemporalModelOutput`] is + returned, otherwise a `tuple` where the first element is the sample tensor. + """ + # 1. Input + batch_frames, channel, height, width = hidden_states.shape + batch_size = batch_frames // num_frames + + residual = hidden_states + + hidden_states = hidden_states[None, :].reshape(batch_size, num_frames, channel, height, width) + hidden_states = hidden_states.permute(0, 2, 1, 3, 4) + + hidden_states = self.norm(hidden_states) + hidden_states = hidden_states.permute(0, 3, 4, 2, 1).reshape(batch_size * height * width, num_frames, channel) + + hidden_states = self.proj_in(hidden_states) + + # 2. Blocks + for block in self.transformer_blocks: + hidden_states = block( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + cross_attention_kwargs=cross_attention_kwargs, + class_labels=class_labels, + ) + + # 3. Output + hidden_states = self.proj_out(hidden_states) + hidden_states = ( + hidden_states[None, None, :] + .reshape(batch_size, height, width, num_frames, channel) + .permute(0, 3, 4, 1, 2) + .contiguous() + ) + hidden_states = hidden_states.reshape(batch_frames, channel, height, width) + + output = hidden_states + residual + + if not return_dict: + return (output,) + + return TransformerTemporalModelOutput(sample=output) + + +class TransformerSpatioTemporalModel(nn.Module): + """ + A Transformer model for video-like data. + + Parameters: + num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention. + attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head. + in_channels (`int`, *optional*): + The number of channels in the input and output (specify if the input is **continuous**). + out_channels (`int`, *optional*): + The number of channels in the output (specify if the input is **continuous**). + num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use. + cross_attention_dim (`int`, *optional*): The number of `encoder_hidden_states` dimensions to use. + """ + + def __init__( + self, + num_attention_heads: int = 16, + attention_head_dim: int = 88, + in_channels: int = 320, + out_channels: Optional[int] = None, + num_layers: int = 1, + cross_attention_dim: Optional[int] = None, + ): + super().__init__() + self.num_attention_heads = num_attention_heads + self.attention_head_dim = attention_head_dim + + inner_dim = num_attention_heads * attention_head_dim + self.inner_dim = inner_dim + + # 2. Define input layers + self.in_channels = in_channels + self.norm = torch.nn.GroupNorm(num_groups=32, num_channels=in_channels, eps=1e-6) + self.proj_in = nn.Linear(in_channels, inner_dim) + + # 3. Define transformers blocks + self.transformer_blocks = nn.ModuleList( + [ + BasicTransformerBlock( + inner_dim, + num_attention_heads, + attention_head_dim, + cross_attention_dim=cross_attention_dim, + ) + for d in range(num_layers) + ] + ) + + time_mix_inner_dim = inner_dim + self.temporal_transformer_blocks = nn.ModuleList( + [ + TemporalBasicTransformerBlock( + inner_dim, + time_mix_inner_dim, + num_attention_heads, + attention_head_dim, + cross_attention_dim=cross_attention_dim, + ) + for _ in range(num_layers) + ] + ) + + time_embed_dim = in_channels * 4 + self.time_pos_embed = TimestepEmbedding(in_channels, time_embed_dim, out_dim=in_channels) + self.time_proj = Timesteps(in_channels, True, 0) + self.time_mixer = AlphaBlender(alpha=0.5, merge_strategy="learned_with_images") + + # 4. Define output layers + self.out_channels = in_channels if out_channels is None else out_channels + # TODO: should use out_channels for continuous projections + self.proj_out = nn.Linear(inner_dim, in_channels) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.Tensor, + encoder_hidden_states: Optional[torch.Tensor] = None, + image_only_indicator: Optional[torch.Tensor] = None, + return_dict: bool = True, + ): + """ + Args: + hidden_states (`torch.FloatTensor` of shape `(batch size, channel, height, width)`): + Input hidden_states. + num_frames (`int`): + The number of frames to be processed per batch. This is used to reshape the hidden states. + encoder_hidden_states ( `torch.LongTensor` of shape `(batch size, encoder_hidden_states dim)`, *optional*): + Conditional embeddings for cross attention layer. If not given, cross-attention defaults to + self-attention. + image_only_indicator (`torch.LongTensor` of shape `(batch size, num_frames)`, *optional*): + A tensor indicating whether the input contains only images. 1 indicates that the input contains only + images, 0 indicates that the input contains video frames. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.transformer_temporal.TransformerTemporalModelOutput`] instead of a plain + tuple. + + Returns: + [`~models.transformer_temporal.TransformerTemporalModelOutput`] or `tuple`: + If `return_dict` is True, an [`~models.transformer_temporal.TransformerTemporalModelOutput`] is + returned, otherwise a `tuple` where the first element is the sample tensor. + """ + # 1. Input + batch_frames, _, height, width = hidden_states.shape + num_frames = image_only_indicator.shape[-1] + batch_size = batch_frames // num_frames + + time_context = encoder_hidden_states + time_context_first_timestep = time_context[None, :].reshape( + batch_size, num_frames, -1, time_context.shape[-1] + )[:, 0] + time_context = time_context_first_timestep[None, :].broadcast_to( + height * width, batch_size, 1, time_context.shape[-1] + ) + time_context = time_context.reshape(height * width * batch_size, 1, time_context.shape[-1]) + + residual = hidden_states + + hidden_states = self.norm(hidden_states) + inner_dim = hidden_states.shape[1] + hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch_frames, height * width, inner_dim) + hidden_states = self.proj_in(hidden_states) + + num_frames_emb = torch.arange(num_frames, device=hidden_states.device) + num_frames_emb = num_frames_emb.repeat(batch_size, 1) + num_frames_emb = num_frames_emb.reshape(-1) + t_emb = self.time_proj(num_frames_emb) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=hidden_states.dtype) + + emb = self.time_pos_embed(t_emb) + emb = emb[:, None, :] + + # 2. Blocks + for block, temporal_block in zip(self.transformer_blocks, self.temporal_transformer_blocks): + if self.training and self.gradient_checkpointing: + hidden_states = torch.utils.checkpoint.checkpoint( + block, + hidden_states, + None, + encoder_hidden_states, + None, + use_reentrant=False, + ) + else: + hidden_states = block( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + ) + + hidden_states_mix = hidden_states + hidden_states_mix = hidden_states_mix + emb + + hidden_states_mix = temporal_block( + hidden_states_mix, + num_frames=num_frames, + encoder_hidden_states=time_context, + ) + hidden_states = self.time_mixer( + x_spatial=hidden_states, + x_temporal=hidden_states_mix, + image_only_indicator=image_only_indicator, + ) + + # 3. Output + hidden_states = self.proj_out(hidden_states) + hidden_states = hidden_states.reshape(batch_frames, height, width, inner_dim).permute(0, 3, 1, 2).contiguous() + + output = hidden_states + residual + + if not return_dict: + return (output,) + + return TransformerTemporalModelOutput(sample=output) diff --git a/diffusers-0.27.0/src/diffusers/models/unet_1d.py b/diffusers-0.27.0/src/diffusers/models/unet_1d.py new file mode 100755 index 0000000..e857c90 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unet_1d.py @@ -0,0 +1,26 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..utils import deprecate +from .unets.unet_1d import UNet1DModel, UNet1DOutput + + +class UNet1DOutput(UNet1DOutput): + deprecation_message = "Importing `UNet1DOutput` from `diffusers.models.unet_1d` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d import UNet1DOutput`, instead." + deprecate("UNet1DOutput", "0.29", deprecation_message) + + +class UNet1DModel(UNet1DModel): + deprecation_message = "Importing `UNet1DModel` from `diffusers.models.unet_1d` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d import UNet1DModel`, instead." + deprecate("UNet1DModel", "0.29", deprecation_message) diff --git a/diffusers-0.27.0/src/diffusers/models/unet_1d_blocks.py b/diffusers-0.27.0/src/diffusers/models/unet_1d_blocks.py new file mode 100755 index 0000000..6b0f094 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unet_1d_blocks.py @@ -0,0 +1,203 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ..utils import deprecate +from .unets.unet_1d_blocks import ( + AttnDownBlock1D, + AttnUpBlock1D, + DownBlock1D, + DownBlock1DNoSkip, + DownResnetBlock1D, + Downsample1d, + MidResTemporalBlock1D, + OutConv1DBlock, + OutValueFunctionBlock, + ResConvBlock, + SelfAttention1d, + UNetMidBlock1D, + UpBlock1D, + UpBlock1DNoSkip, + UpResnetBlock1D, + Upsample1d, + ValueFunctionMidBlock1D, +) + + +class DownResnetBlock1D(DownResnetBlock1D): + deprecation_message = "Importing `DownResnetBlock1D` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import DownResnetBlock1D`, instead." + deprecate("DownResnetBlock1D", "0.29", deprecation_message) + + +class UpResnetBlock1D(UpResnetBlock1D): + deprecation_message = "Importing `UpResnetBlock1D` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import UpResnetBlock1D`, instead." + deprecate("UpResnetBlock1D", "0.29", deprecation_message) + + +class ValueFunctionMidBlock1D(ValueFunctionMidBlock1D): + deprecation_message = "Importing `ValueFunctionMidBlock1D` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import ValueFunctionMidBlock1D`, instead." + deprecate("ValueFunctionMidBlock1D", "0.29", deprecation_message) + + +class OutConv1DBlock(OutConv1DBlock): + deprecation_message = "Importing `OutConv1DBlock` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import OutConv1DBlock`, instead." + deprecate("OutConv1DBlock", "0.29", deprecation_message) + + +class OutValueFunctionBlock(OutValueFunctionBlock): + deprecation_message = "Importing `OutValueFunctionBlock` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import OutValueFunctionBlock`, instead." + deprecate("OutValueFunctionBlock", "0.29", deprecation_message) + + +class Downsample1d(Downsample1d): + deprecation_message = "Importing `Downsample1d` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import Downsample1d`, instead." + deprecate("Downsample1d", "0.29", deprecation_message) + + +class Upsample1d(Upsample1d): + deprecation_message = "Importing `Upsample1d` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import Upsample1d`, instead." + deprecate("Upsample1d", "0.29", deprecation_message) + + +class SelfAttention1d(SelfAttention1d): + deprecation_message = "Importing `SelfAttention1d` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import SelfAttention1d`, instead." + deprecate("SelfAttention1d", "0.29", deprecation_message) + + +class ResConvBlock(ResConvBlock): + deprecation_message = "Importing `ResConvBlock` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import ResConvBlock`, instead." + deprecate("ResConvBlock", "0.29", deprecation_message) + + +class UNetMidBlock1D(UNetMidBlock1D): + deprecation_message = "Importing `UNetMidBlock1D` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import UNetMidBlock1D`, instead." + deprecate("UNetMidBlock1D", "0.29", deprecation_message) + + +class AttnDownBlock1D(AttnDownBlock1D): + deprecation_message = "Importing `AttnDownBlock1D` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import AttnDownBlock1D`, instead." + deprecate("AttnDownBlock1D", "0.29", deprecation_message) + + +class DownBlock1D(DownBlock1D): + deprecation_message = "Importing `DownBlock1D` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import DownBlock1D`, instead." + deprecate("DownBlock1D", "0.29", deprecation_message) + + +class DownBlock1DNoSkip(DownBlock1DNoSkip): + deprecation_message = "Importing `DownBlock1DNoSkip` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import DownBlock1DNoSkip`, instead." + deprecate("DownBlock1DNoSkip", "0.29", deprecation_message) + + +class AttnUpBlock1D(AttnUpBlock1D): + deprecation_message = "Importing `AttnUpBlock1D` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import AttnUpBlock1D`, instead." + deprecate("AttnUpBlock1D", "0.29", deprecation_message) + + +class UpBlock1D(UpBlock1D): + deprecation_message = "Importing `UpBlock1D` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import UpBlock1D`, instead." + deprecate("UpBlock1D", "0.29", deprecation_message) + + +class UpBlock1DNoSkip(UpBlock1DNoSkip): + deprecation_message = "Importing `UpBlock1DNoSkip` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import UpBlock1DNoSkip`, instead." + deprecate("UpBlock1DNoSkip", "0.29", deprecation_message) + + +class MidResTemporalBlock1D(MidResTemporalBlock1D): + deprecation_message = "Importing `MidResTemporalBlock1D` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import MidResTemporalBlock1D`, instead." + deprecate("MidResTemporalBlock1D", "0.29", deprecation_message) + + +def get_down_block( + down_block_type: str, + num_layers: int, + in_channels: int, + out_channels: int, + temb_channels: int, + add_downsample: bool, +): + deprecation_message = "Importing `get_down_block` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import get_down_block`, instead." + deprecate("get_down_block", "0.29", deprecation_message) + + from .unets.unet_1d_blocks import get_down_block + + return get_down_block( + down_block_type=down_block_type, + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + ) + + +def get_up_block( + up_block_type: str, num_layers: int, in_channels: int, out_channels: int, temb_channels: int, add_upsample: bool +): + deprecation_message = "Importing `get_up_block` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import get_up_block`, instead." + deprecate("get_up_block", "0.29", deprecation_message) + + from .unets.unet_1d_blocks import get_up_block + + return get_up_block( + up_block_type=up_block_type, + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_upsample=add_upsample, + ) + + +def get_mid_block( + mid_block_type: str, + num_layers: int, + in_channels: int, + mid_channels: int, + out_channels: int, + embed_dim: int, + add_downsample: bool, +): + deprecation_message = "Importing `get_mid_block` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import get_mid_block`, instead." + deprecate("get_mid_block", "0.29", deprecation_message) + + from .unets.unet_1d_blocks import get_mid_block + + return get_mid_block( + mid_block_type=mid_block_type, + num_layers=num_layers, + in_channels=in_channels, + mid_channels=mid_channels, + out_channels=out_channels, + embed_dim=embed_dim, + add_downsample=add_downsample, + ) + + +def get_out_block( + *, out_block_type: str, num_groups_out: int, embed_dim: int, out_channels: int, act_fn: str, fc_dim: int +): + deprecation_message = "Importing `get_out_block` from `diffusers.models.unet_1d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_1d_blocks import get_out_block`, instead." + deprecate("get_out_block", "0.29", deprecation_message) + + from .unets.unet_1d_blocks import get_out_block + + return get_out_block( + out_block_type=out_block_type, + num_groups_out=num_groups_out, + embed_dim=embed_dim, + out_channels=out_channels, + act_fn=act_fn, + fc_dim=fc_dim, + ) diff --git a/diffusers-0.27.0/src/diffusers/models/unet_2d.py b/diffusers-0.27.0/src/diffusers/models/unet_2d.py new file mode 100755 index 0000000..21f1fea --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unet_2d.py @@ -0,0 +1,27 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from ..utils import deprecate +from .unets.unet_2d import UNet2DModel, UNet2DOutput + + +class UNet2DOutput(UNet2DOutput): + deprecation_message = "Importing `UNet2DOutput` from `diffusers.models.unet_2d` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d import UNet2DOutput`, instead." + deprecate("UNet2DOutput", "0.29", deprecation_message) + + +class UNet2DModel(UNet2DModel): + deprecation_message = "Importing `UNet2DModel` from `diffusers.models.unet_2d` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d import UNet2DModel`, instead." + deprecate("UNet2DModel", "0.29", deprecation_message) diff --git a/diffusers-0.27.0/src/diffusers/models/unet_2d_blocks.py b/diffusers-0.27.0/src/diffusers/models/unet_2d_blocks.py new file mode 100755 index 0000000..931fa89 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unet_2d_blocks.py @@ -0,0 +1,375 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional + +from ..utils import deprecate +from .unets.unet_2d_blocks import ( + AttnDownBlock2D, + AttnDownEncoderBlock2D, + AttnSkipDownBlock2D, + AttnSkipUpBlock2D, + AttnUpBlock2D, + AttnUpDecoderBlock2D, + AutoencoderTinyBlock, + CrossAttnDownBlock2D, + CrossAttnUpBlock2D, + DownBlock2D, + KAttentionBlock, + KCrossAttnDownBlock2D, + KCrossAttnUpBlock2D, + KDownBlock2D, + KUpBlock2D, + ResnetDownsampleBlock2D, + ResnetUpsampleBlock2D, + SimpleCrossAttnDownBlock2D, + SimpleCrossAttnUpBlock2D, + SkipDownBlock2D, + SkipUpBlock2D, + UNetMidBlock2D, + UNetMidBlock2DCrossAttn, + UNetMidBlock2DSimpleCrossAttn, + UpBlock2D, + UpDecoderBlock2D, +) + + +def get_down_block( + down_block_type: str, + num_layers: int, + in_channels: int, + out_channels: int, + temb_channels: int, + add_downsample: bool, + resnet_eps: float, + resnet_act_fn: str, + transformer_layers_per_block: int = 1, + num_attention_heads: Optional[int] = None, + resnet_groups: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + downsample_padding: Optional[int] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + attention_type: str = "default", + resnet_skip_time_act: bool = False, + resnet_out_scale_factor: float = 1.0, + cross_attention_norm: Optional[str] = None, + attention_head_dim: Optional[int] = None, + downsample_type: Optional[str] = None, + dropout: float = 0.0, +): + deprecation_message = "Importing `get_down_block` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import get_down_block`, instead." + deprecate("get_down_block", "0.29", deprecation_message) + + from .unets.unet_2d_blocks import get_down_block + + return get_down_block( + down_block_type=down_block_type, + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + transformer_layers_per_block=transformer_layers_per_block, + num_attention_heads=num_attention_heads, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + downsample_padding=downsample_padding, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + attention_head_dim=attention_head_dim, + downsample_type=downsample_type, + dropout=dropout, + ) + + +def get_mid_block( + mid_block_type: str, + temb_channels: int, + in_channels: int, + resnet_eps: float, + resnet_act_fn: str, + resnet_groups: int, + output_scale_factor: float = 1.0, + transformer_layers_per_block: int = 1, + num_attention_heads: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + mid_block_only_cross_attention: bool = False, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + attention_type: str = "default", + resnet_skip_time_act: bool = False, + cross_attention_norm: Optional[str] = None, + attention_head_dim: Optional[int] = 1, + dropout: float = 0.0, +): + if mid_block_type == "UNetMidBlock2DCrossAttn": + return UNetMidBlock2DCrossAttn( + transformer_layers_per_block=transformer_layers_per_block, + in_channels=in_channels, + temb_channels=temb_channels, + dropout=dropout, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + output_scale_factor=output_scale_factor, + resnet_time_scale_shift=resnet_time_scale_shift, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + resnet_groups=resnet_groups, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + elif mid_block_type == "UNetMidBlock2DSimpleCrossAttn": + return UNetMidBlock2DSimpleCrossAttn( + in_channels=in_channels, + temb_channels=temb_channels, + dropout=dropout, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + output_scale_factor=output_scale_factor, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + only_cross_attention=mid_block_only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif mid_block_type == "UNetMidBlock2D": + return UNetMidBlock2D( + in_channels=in_channels, + temb_channels=temb_channels, + dropout=dropout, + num_layers=0, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + output_scale_factor=output_scale_factor, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + add_attention=False, + ) + elif mid_block_type is None: + return None + else: + raise ValueError(f"unknown mid_block_type : {mid_block_type}") + + +def get_up_block( + up_block_type: str, + num_layers: int, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + add_upsample: bool, + resnet_eps: float, + resnet_act_fn: str, + resolution_idx: Optional[int] = None, + transformer_layers_per_block: int = 1, + num_attention_heads: Optional[int] = None, + resnet_groups: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + attention_type: str = "default", + resnet_skip_time_act: bool = False, + resnet_out_scale_factor: float = 1.0, + cross_attention_norm: Optional[str] = None, + attention_head_dim: Optional[int] = None, + upsample_type: Optional[str] = None, + dropout: float = 0.0, +): + deprecation_message = "Importing `get_up_block` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import get_up_block`, instead." + deprecate("get_up_block", "0.29", deprecation_message) + + from .unets.unet_2d_blocks import get_up_block + + return get_up_block( + up_block_type=up_block_type, + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resolution_idx=resolution_idx, + transformer_layers_per_block=transformer_layers_per_block, + num_attention_heads=num_attention_heads, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + attention_head_dim=attention_head_dim, + upsample_type=upsample_type, + dropout=dropout, + ) + + +class AutoencoderTinyBlock(AutoencoderTinyBlock): + deprecation_message = "Importing `AutoencoderTinyBlock` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import AutoencoderTinyBlock`, instead." + deprecate("AutoencoderTinyBlock", "0.29", deprecation_message) + + +class UNetMidBlock2D(UNetMidBlock2D): + deprecation_message = "Importing `UNetMidBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import UNetMidBlock2D`, instead." + deprecate("UNetMidBlock2D", "0.29", deprecation_message) + + +class UNetMidBlock2DCrossAttn(UNetMidBlock2DCrossAttn): + deprecation_message = "Importing `UNetMidBlock2DCrossAttn` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import UNetMidBlock2DCrossAttn`, instead." + deprecate("UNetMidBlock2DCrossAttn", "0.29", deprecation_message) + + +class UNetMidBlock2DSimpleCrossAttn(UNetMidBlock2DSimpleCrossAttn): + deprecation_message = "Importing `UNetMidBlock2DSimpleCrossAttn` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import UNetMidBlock2DSimpleCrossAttn`, instead." + deprecate("UNetMidBlock2DSimpleCrossAttn", "0.29", deprecation_message) + + +class AttnDownBlock2D(AttnDownBlock2D): + deprecation_message = "Importing `AttnDownBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import AttnDownBlock2D`, instead." + deprecate("AttnDownBlock2D", "0.29", deprecation_message) + + +class CrossAttnDownBlock2D(CrossAttnDownBlock2D): + deprecation_message = "Importing `AttnDownBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import CrossAttnDownBlock2D`, instead." + deprecate("CrossAttnDownBlock2D", "0.29", deprecation_message) + + +class DownBlock2D(DownBlock2D): + deprecation_message = "Importing `DownBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import DownBlock2D`, instead." + deprecate("DownBlock2D", "0.29", deprecation_message) + + +class AttnDownEncoderBlock2D(AttnDownEncoderBlock2D): + deprecation_message = "Importing `AttnDownEncoderBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import AttnDownEncoderBlock2D`, instead." + deprecate("AttnDownEncoderBlock2D", "0.29", deprecation_message) + + +class AttnSkipDownBlock2D(AttnSkipDownBlock2D): + deprecation_message = "Importing `AttnSkipDownBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import AttnSkipDownBlock2D`, instead." + deprecate("AttnSkipDownBlock2D", "0.29", deprecation_message) + + +class SkipDownBlock2D(SkipDownBlock2D): + deprecation_message = "Importing `SkipDownBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import SkipDownBlock2D`, instead." + deprecate("SkipDownBlock2D", "0.29", deprecation_message) + + +class ResnetDownsampleBlock2D(ResnetDownsampleBlock2D): + deprecation_message = "Importing `ResnetDownsampleBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import ResnetDownsampleBlock2D`, instead." + deprecate("ResnetDownsampleBlock2D", "0.29", deprecation_message) + + +class SimpleCrossAttnDownBlock2D(SimpleCrossAttnDownBlock2D): + deprecation_message = "Importing `SimpleCrossAttnDownBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import SimpleCrossAttnDownBlock2D`, instead." + deprecate("SimpleCrossAttnDownBlock2D", "0.29", deprecation_message) + + +class KDownBlock2D(KDownBlock2D): + deprecation_message = "Importing `KDownBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import KDownBlock2D`, instead." + deprecate("KDownBlock2D", "0.29", deprecation_message) + + +class KCrossAttnDownBlock2D(KCrossAttnDownBlock2D): + deprecation_message = "Importing `KCrossAttnDownBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import KCrossAttnDownBlock2D`, instead." + deprecate("KCrossAttnDownBlock2D", "0.29", deprecation_message) + + +class AttnUpBlock2D(AttnUpBlock2D): + deprecation_message = "Importing `AttnUpBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import AttnUpBlock2D`, instead." + deprecate("AttnUpBlock2D", "0.29", deprecation_message) + + +class CrossAttnUpBlock2D(CrossAttnUpBlock2D): + deprecation_message = "Importing `CrossAttnUpBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import CrossAttnUpBlock2D`, instead." + deprecate("CrossAttnUpBlock2D", "0.29", deprecation_message) + + +class UpBlock2D(UpBlock2D): + deprecation_message = "Importing `UpBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import UpBlock2D`, instead." + deprecate("UpBlock2D", "0.29", deprecation_message) + + +class UpDecoderBlock2D(UpDecoderBlock2D): + deprecation_message = "Importing `UpDecoderBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import UpDecoderBlock2D`, instead." + deprecate("UpDecoderBlock2D", "0.29", deprecation_message) + + +class AttnUpDecoderBlock2D(AttnUpDecoderBlock2D): + deprecation_message = "Importing `AttnUpDecoderBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import AttnUpDecoderBlock2D`, instead." + deprecate("AttnUpDecoderBlock2D", "0.29", deprecation_message) + + +class AttnSkipUpBlock2D(AttnSkipUpBlock2D): + deprecation_message = "Importing `AttnSkipUpBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import AttnSkipUpBlock2D`, instead." + deprecate("AttnSkipUpBlock2D", "0.29", deprecation_message) + + +class SkipUpBlock2D(SkipUpBlock2D): + deprecation_message = "Importing `SkipUpBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import SkipUpBlock2D`, instead." + deprecate("SkipUpBlock2D", "0.29", deprecation_message) + + +class ResnetUpsampleBlock2D(ResnetUpsampleBlock2D): + deprecation_message = "Importing `ResnetUpsampleBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import ResnetUpsampleBlock2D`, instead." + deprecate("ResnetUpsampleBlock2D", "0.29", deprecation_message) + + +class SimpleCrossAttnUpBlock2D(SimpleCrossAttnUpBlock2D): + deprecation_message = "Importing `SimpleCrossAttnUpBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import SimpleCrossAttnUpBlock2D`, instead." + deprecate("SimpleCrossAttnUpBlock2D", "0.29", deprecation_message) + + +class KUpBlock2D(KUpBlock2D): + deprecation_message = "Importing `KUpBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import KUpBlock2D`, instead." + deprecate("KUpBlock2D", "0.29", deprecation_message) + + +class KCrossAttnUpBlock2D(KCrossAttnUpBlock2D): + deprecation_message = "Importing `KCrossAttnUpBlock2D` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import KCrossAttnUpBlock2D`, instead." + deprecate("KCrossAttnUpBlock2D", "0.29", deprecation_message) + + +# can potentially later be renamed to `No-feed-forward` attention +class KAttentionBlock(KAttentionBlock): + deprecation_message = "Importing `KAttentionBlock` from `diffusers.models.unet_2d_blocks` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_blocks import KAttentionBlock`, instead." + deprecate("KAttentionBlock", "0.29", deprecation_message) diff --git a/diffusers-0.27.0/src/diffusers/models/unet_2d_condition.py b/diffusers-0.27.0/src/diffusers/models/unet_2d_condition.py new file mode 100755 index 0000000..85a3e7b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unet_2d_condition.py @@ -0,0 +1,25 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from ..utils import deprecate +from .unets.unet_2d_condition import UNet2DConditionModel, UNet2DConditionOutput + + +class UNet2DConditionOutput(UNet2DConditionOutput): + deprecation_message = "Importing `UNet2DConditionOutput` from `diffusers.models.unet_2d_condition` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_condition import UNet2DConditionOutput`, instead." + deprecate("UNet2DConditionOutput", "0.29", deprecation_message) + + +class UNet2DConditionModel(UNet2DConditionModel): + deprecation_message = "Importing `UNet2DConditionModel` from `diffusers.models.unet_2d_condition` is deprecated and this will be removed in a future version. Please use `from diffusers.models.unets.unet_2d_condition import UNet2DConditionModel`, instead." + deprecate("UNet2DConditionModel", "0.29", deprecation_message) diff --git a/diffusers-0.27.0/src/diffusers/models/unets/__init__.py b/diffusers-0.27.0/src/diffusers/models/unets/__init__.py new file mode 100755 index 0000000..9ef04fb --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/__init__.py @@ -0,0 +1,18 @@ +from ...utils import is_flax_available, is_torch_available + + +if is_torch_available(): + from .unet_1d import UNet1DModel + from .unet_2d import UNet2DModel + from .unet_2d_condition import UNet2DConditionModel + from .unet_3d_condition import UNet3DConditionModel + from .unet_i2vgen_xl import I2VGenXLUNet + from .unet_kandinsky3 import Kandinsky3UNet + from .unet_motion_model import MotionAdapter, UNetMotionModel + from .unet_spatio_temporal_condition import UNetSpatioTemporalConditionModel + from .unet_stable_cascade import StableCascadeUNet + from .uvit_2d import UVit2DModel + + +if is_flax_available(): + from .unet_2d_condition_flax import FlaxUNet2DConditionModel diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_1d.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_1d.py new file mode 100755 index 0000000..59d70f6 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_1d.py @@ -0,0 +1,255 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...utils import BaseOutput +from ..embeddings import GaussianFourierProjection, TimestepEmbedding, Timesteps +from ..modeling_utils import ModelMixin +from .unet_1d_blocks import get_down_block, get_mid_block, get_out_block, get_up_block + + +@dataclass +class UNet1DOutput(BaseOutput): + """ + The output of [`UNet1DModel`]. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, sample_size)`): + The hidden states output from the last layer of the model. + """ + + sample: torch.FloatTensor + + +class UNet1DModel(ModelMixin, ConfigMixin): + r""" + A 1D UNet model that takes a noisy sample and a timestep and returns a sample shaped output. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + sample_size (`int`, *optional*): Default length of sample. Should be adaptable at runtime. + in_channels (`int`, *optional*, defaults to 2): Number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 2): Number of channels in the output. + extra_in_channels (`int`, *optional*, defaults to 0): + Number of additional channels to be added to the input of the first down block. Useful for cases where the + input data has more channels than what the model was initially designed for. + time_embedding_type (`str`, *optional*, defaults to `"fourier"`): Type of time embedding to use. + freq_shift (`float`, *optional*, defaults to 0.0): Frequency shift for Fourier time embedding. + flip_sin_to_cos (`bool`, *optional*, defaults to `False`): + Whether to flip sin to cos for Fourier time embedding. + down_block_types (`Tuple[str]`, *optional*, defaults to `("DownBlock1DNoSkip", "DownBlock1D", "AttnDownBlock1D")`): + Tuple of downsample block types. + up_block_types (`Tuple[str]`, *optional*, defaults to `("AttnUpBlock1D", "UpBlock1D", "UpBlock1DNoSkip")`): + Tuple of upsample block types. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(32, 32, 64)`): + Tuple of block output channels. + mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock1D"`): Block type for middle of UNet. + out_block_type (`str`, *optional*, defaults to `None`): Optional output processing block of UNet. + act_fn (`str`, *optional*, defaults to `None`): Optional activation function in UNet blocks. + norm_num_groups (`int`, *optional*, defaults to 8): The number of groups for normalization. + layers_per_block (`int`, *optional*, defaults to 1): The number of layers per block. + downsample_each_block (`int`, *optional*, defaults to `False`): + Experimental feature for using a UNet without upsampling. + """ + + @register_to_config + def __init__( + self, + sample_size: int = 65536, + sample_rate: Optional[int] = None, + in_channels: int = 2, + out_channels: int = 2, + extra_in_channels: int = 0, + time_embedding_type: str = "fourier", + flip_sin_to_cos: bool = True, + use_timestep_embedding: bool = False, + freq_shift: float = 0.0, + down_block_types: Tuple[str] = ("DownBlock1DNoSkip", "DownBlock1D", "AttnDownBlock1D"), + up_block_types: Tuple[str] = ("AttnUpBlock1D", "UpBlock1D", "UpBlock1DNoSkip"), + mid_block_type: Tuple[str] = "UNetMidBlock1D", + out_block_type: str = None, + block_out_channels: Tuple[int] = (32, 32, 64), + act_fn: str = None, + norm_num_groups: int = 8, + layers_per_block: int = 1, + downsample_each_block: bool = False, + ): + super().__init__() + self.sample_size = sample_size + + # time + if time_embedding_type == "fourier": + self.time_proj = GaussianFourierProjection( + embedding_size=8, set_W_to_weight=False, log=False, flip_sin_to_cos=flip_sin_to_cos + ) + timestep_input_dim = 2 * block_out_channels[0] + elif time_embedding_type == "positional": + self.time_proj = Timesteps( + block_out_channels[0], flip_sin_to_cos=flip_sin_to_cos, downscale_freq_shift=freq_shift + ) + timestep_input_dim = block_out_channels[0] + + if use_timestep_embedding: + time_embed_dim = block_out_channels[0] * 4 + self.time_mlp = TimestepEmbedding( + in_channels=timestep_input_dim, + time_embed_dim=time_embed_dim, + act_fn=act_fn, + out_dim=block_out_channels[0], + ) + + self.down_blocks = nn.ModuleList([]) + self.mid_block = None + self.up_blocks = nn.ModuleList([]) + self.out_block = None + + # down + output_channel = in_channels + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + + if i == 0: + input_channel += extra_in_channels + + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + temb_channels=block_out_channels[0], + add_downsample=not is_final_block or downsample_each_block, + ) + self.down_blocks.append(down_block) + + # mid + self.mid_block = get_mid_block( + mid_block_type, + in_channels=block_out_channels[-1], + mid_channels=block_out_channels[-1], + out_channels=block_out_channels[-1], + embed_dim=block_out_channels[0], + num_layers=layers_per_block, + add_downsample=downsample_each_block, + ) + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + if out_block_type is None: + final_upsample_channels = out_channels + else: + final_upsample_channels = block_out_channels[0] + + for i, up_block_type in enumerate(up_block_types): + prev_output_channel = output_channel + output_channel = ( + reversed_block_out_channels[i + 1] if i < len(up_block_types) - 1 else final_upsample_channels + ) + + is_final_block = i == len(block_out_channels) - 1 + + up_block = get_up_block( + up_block_type, + num_layers=layers_per_block, + in_channels=prev_output_channel, + out_channels=output_channel, + temb_channels=block_out_channels[0], + add_upsample=not is_final_block, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + num_groups_out = norm_num_groups if norm_num_groups is not None else min(block_out_channels[0] // 4, 32) + self.out_block = get_out_block( + out_block_type=out_block_type, + num_groups_out=num_groups_out, + embed_dim=block_out_channels[0], + out_channels=out_channels, + act_fn=act_fn, + fc_dim=block_out_channels[-1] // 4, + ) + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + return_dict: bool = True, + ) -> Union[UNet1DOutput, Tuple]: + r""" + The [`UNet1DModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor with the following shape `(batch_size, num_channels, sample_size)`. + timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unet_1d.UNet1DOutput`] instead of a plain tuple. + + Returns: + [`~models.unet_1d.UNet1DOutput`] or `tuple`: + If `return_dict` is True, an [`~models.unet_1d.UNet1DOutput`] is returned, otherwise a `tuple` is + returned where the first element is the sample tensor. + """ + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + timesteps = torch.tensor([timesteps], dtype=torch.long, device=sample.device) + elif torch.is_tensor(timesteps) and len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + timestep_embed = self.time_proj(timesteps) + if self.config.use_timestep_embedding: + timestep_embed = self.time_mlp(timestep_embed) + else: + timestep_embed = timestep_embed[..., None] + timestep_embed = timestep_embed.repeat([1, 1, sample.shape[2]]).to(sample.dtype) + timestep_embed = timestep_embed.broadcast_to((sample.shape[:1] + timestep_embed.shape[1:])) + + # 2. down + down_block_res_samples = () + for downsample_block in self.down_blocks: + sample, res_samples = downsample_block(hidden_states=sample, temb=timestep_embed) + down_block_res_samples += res_samples + + # 3. mid + if self.mid_block: + sample = self.mid_block(sample, timestep_embed) + + # 4. up + for i, upsample_block in enumerate(self.up_blocks): + res_samples = down_block_res_samples[-1:] + down_block_res_samples = down_block_res_samples[:-1] + sample = upsample_block(sample, res_hidden_states_tuple=res_samples, temb=timestep_embed) + + # 5. post-process + if self.out_block: + sample = self.out_block(sample, timestep_embed) + + if not return_dict: + return (sample,) + + return UNet1DOutput(sample=sample) diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_1d_blocks.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_1d_blocks.py new file mode 100755 index 0000000..e3163cd --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_1d_blocks.py @@ -0,0 +1,702 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +from typing import Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from torch import nn + +from ..activations import get_activation +from ..resnet import Downsample1D, ResidualTemporalBlock1D, Upsample1D, rearrange_dims + + +class DownResnetBlock1D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: Optional[int] = None, + num_layers: int = 1, + conv_shortcut: bool = False, + temb_channels: int = 32, + groups: int = 32, + groups_out: Optional[int] = None, + non_linearity: Optional[str] = None, + time_embedding_norm: str = "default", + output_scale_factor: float = 1.0, + add_downsample: bool = True, + ): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.use_conv_shortcut = conv_shortcut + self.time_embedding_norm = time_embedding_norm + self.add_downsample = add_downsample + self.output_scale_factor = output_scale_factor + + if groups_out is None: + groups_out = groups + + # there will always be at least one resnet + resnets = [ResidualTemporalBlock1D(in_channels, out_channels, embed_dim=temb_channels)] + + for _ in range(num_layers): + resnets.append(ResidualTemporalBlock1D(out_channels, out_channels, embed_dim=temb_channels)) + + self.resnets = nn.ModuleList(resnets) + + if non_linearity is None: + self.nonlinearity = None + else: + self.nonlinearity = get_activation(non_linearity) + + self.downsample = None + if add_downsample: + self.downsample = Downsample1D(out_channels, use_conv=True, padding=1) + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + output_states = () + + hidden_states = self.resnets[0](hidden_states, temb) + for resnet in self.resnets[1:]: + hidden_states = resnet(hidden_states, temb) + + output_states += (hidden_states,) + + if self.nonlinearity is not None: + hidden_states = self.nonlinearity(hidden_states) + + if self.downsample is not None: + hidden_states = self.downsample(hidden_states) + + return hidden_states, output_states + + +class UpResnetBlock1D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: Optional[int] = None, + num_layers: int = 1, + temb_channels: int = 32, + groups: int = 32, + groups_out: Optional[int] = None, + non_linearity: Optional[str] = None, + time_embedding_norm: str = "default", + output_scale_factor: float = 1.0, + add_upsample: bool = True, + ): + super().__init__() + self.in_channels = in_channels + out_channels = in_channels if out_channels is None else out_channels + self.out_channels = out_channels + self.time_embedding_norm = time_embedding_norm + self.add_upsample = add_upsample + self.output_scale_factor = output_scale_factor + + if groups_out is None: + groups_out = groups + + # there will always be at least one resnet + resnets = [ResidualTemporalBlock1D(2 * in_channels, out_channels, embed_dim=temb_channels)] + + for _ in range(num_layers): + resnets.append(ResidualTemporalBlock1D(out_channels, out_channels, embed_dim=temb_channels)) + + self.resnets = nn.ModuleList(resnets) + + if non_linearity is None: + self.nonlinearity = None + else: + self.nonlinearity = get_activation(non_linearity) + + self.upsample = None + if add_upsample: + self.upsample = Upsample1D(out_channels, use_conv_transpose=True) + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Optional[Tuple[torch.FloatTensor, ...]] = None, + temb: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + if res_hidden_states_tuple is not None: + res_hidden_states = res_hidden_states_tuple[-1] + hidden_states = torch.cat((hidden_states, res_hidden_states), dim=1) + + hidden_states = self.resnets[0](hidden_states, temb) + for resnet in self.resnets[1:]: + hidden_states = resnet(hidden_states, temb) + + if self.nonlinearity is not None: + hidden_states = self.nonlinearity(hidden_states) + + if self.upsample is not None: + hidden_states = self.upsample(hidden_states) + + return hidden_states + + +class ValueFunctionMidBlock1D(nn.Module): + def __init__(self, in_channels: int, out_channels: int, embed_dim: int): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.embed_dim = embed_dim + + self.res1 = ResidualTemporalBlock1D(in_channels, in_channels // 2, embed_dim=embed_dim) + self.down1 = Downsample1D(out_channels // 2, use_conv=True) + self.res2 = ResidualTemporalBlock1D(in_channels // 2, in_channels // 4, embed_dim=embed_dim) + self.down2 = Downsample1D(out_channels // 4, use_conv=True) + + def forward(self, x: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + x = self.res1(x, temb) + x = self.down1(x) + x = self.res2(x, temb) + x = self.down2(x) + return x + + +class MidResTemporalBlock1D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + embed_dim: int, + num_layers: int = 1, + add_downsample: bool = False, + add_upsample: bool = False, + non_linearity: Optional[str] = None, + ): + super().__init__() + self.in_channels = in_channels + self.out_channels = out_channels + self.add_downsample = add_downsample + + # there will always be at least one resnet + resnets = [ResidualTemporalBlock1D(in_channels, out_channels, embed_dim=embed_dim)] + + for _ in range(num_layers): + resnets.append(ResidualTemporalBlock1D(out_channels, out_channels, embed_dim=embed_dim)) + + self.resnets = nn.ModuleList(resnets) + + if non_linearity is None: + self.nonlinearity = None + else: + self.nonlinearity = get_activation(non_linearity) + + self.upsample = None + if add_upsample: + self.upsample = Downsample1D(out_channels, use_conv=True) + + self.downsample = None + if add_downsample: + self.downsample = Downsample1D(out_channels, use_conv=True) + + if self.upsample and self.downsample: + raise ValueError("Block cannot downsample and upsample") + + def forward(self, hidden_states: torch.FloatTensor, temb: torch.FloatTensor) -> torch.FloatTensor: + hidden_states = self.resnets[0](hidden_states, temb) + for resnet in self.resnets[1:]: + hidden_states = resnet(hidden_states, temb) + + if self.upsample: + hidden_states = self.upsample(hidden_states) + if self.downsample: + self.downsample = self.downsample(hidden_states) + + return hidden_states + + +class OutConv1DBlock(nn.Module): + def __init__(self, num_groups_out: int, out_channels: int, embed_dim: int, act_fn: str): + super().__init__() + self.final_conv1d_1 = nn.Conv1d(embed_dim, embed_dim, 5, padding=2) + self.final_conv1d_gn = nn.GroupNorm(num_groups_out, embed_dim) + self.final_conv1d_act = get_activation(act_fn) + self.final_conv1d_2 = nn.Conv1d(embed_dim, out_channels, 1) + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + hidden_states = self.final_conv1d_1(hidden_states) + hidden_states = rearrange_dims(hidden_states) + hidden_states = self.final_conv1d_gn(hidden_states) + hidden_states = rearrange_dims(hidden_states) + hidden_states = self.final_conv1d_act(hidden_states) + hidden_states = self.final_conv1d_2(hidden_states) + return hidden_states + + +class OutValueFunctionBlock(nn.Module): + def __init__(self, fc_dim: int, embed_dim: int, act_fn: str = "mish"): + super().__init__() + self.final_block = nn.ModuleList( + [ + nn.Linear(fc_dim + embed_dim, fc_dim // 2), + get_activation(act_fn), + nn.Linear(fc_dim // 2, 1), + ] + ) + + def forward(self, hidden_states: torch.FloatTensor, temb: torch.FloatTensor) -> torch.FloatTensor: + hidden_states = hidden_states.view(hidden_states.shape[0], -1) + hidden_states = torch.cat((hidden_states, temb), dim=-1) + for layer in self.final_block: + hidden_states = layer(hidden_states) + + return hidden_states + + +_kernels = { + "linear": [1 / 8, 3 / 8, 3 / 8, 1 / 8], + "cubic": [-0.01171875, -0.03515625, 0.11328125, 0.43359375, 0.43359375, 0.11328125, -0.03515625, -0.01171875], + "lanczos3": [ + 0.003689131001010537, + 0.015056144446134567, + -0.03399861603975296, + -0.066637322306633, + 0.13550527393817902, + 0.44638532400131226, + 0.44638532400131226, + 0.13550527393817902, + -0.066637322306633, + -0.03399861603975296, + 0.015056144446134567, + 0.003689131001010537, + ], +} + + +class Downsample1d(nn.Module): + def __init__(self, kernel: str = "linear", pad_mode: str = "reflect"): + super().__init__() + self.pad_mode = pad_mode + kernel_1d = torch.tensor(_kernels[kernel]) + self.pad = kernel_1d.shape[0] // 2 - 1 + self.register_buffer("kernel", kernel_1d) + + def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + hidden_states = F.pad(hidden_states, (self.pad,) * 2, self.pad_mode) + weight = hidden_states.new_zeros([hidden_states.shape[1], hidden_states.shape[1], self.kernel.shape[0]]) + indices = torch.arange(hidden_states.shape[1], device=hidden_states.device) + kernel = self.kernel.to(weight)[None, :].expand(hidden_states.shape[1], -1) + weight[indices, indices] = kernel + return F.conv1d(hidden_states, weight, stride=2) + + +class Upsample1d(nn.Module): + def __init__(self, kernel: str = "linear", pad_mode: str = "reflect"): + super().__init__() + self.pad_mode = pad_mode + kernel_1d = torch.tensor(_kernels[kernel]) * 2 + self.pad = kernel_1d.shape[0] // 2 - 1 + self.register_buffer("kernel", kernel_1d) + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + hidden_states = F.pad(hidden_states, ((self.pad + 1) // 2,) * 2, self.pad_mode) + weight = hidden_states.new_zeros([hidden_states.shape[1], hidden_states.shape[1], self.kernel.shape[0]]) + indices = torch.arange(hidden_states.shape[1], device=hidden_states.device) + kernel = self.kernel.to(weight)[None, :].expand(hidden_states.shape[1], -1) + weight[indices, indices] = kernel + return F.conv_transpose1d(hidden_states, weight, stride=2, padding=self.pad * 2 + 1) + + +class SelfAttention1d(nn.Module): + def __init__(self, in_channels: int, n_head: int = 1, dropout_rate: float = 0.0): + super().__init__() + self.channels = in_channels + self.group_norm = nn.GroupNorm(1, num_channels=in_channels) + self.num_heads = n_head + + self.query = nn.Linear(self.channels, self.channels) + self.key = nn.Linear(self.channels, self.channels) + self.value = nn.Linear(self.channels, self.channels) + + self.proj_attn = nn.Linear(self.channels, self.channels, bias=True) + + self.dropout = nn.Dropout(dropout_rate, inplace=True) + + def transpose_for_scores(self, projection: torch.Tensor) -> torch.Tensor: + new_projection_shape = projection.size()[:-1] + (self.num_heads, -1) + # move heads to 2nd position (B, T, H * D) -> (B, T, H, D) -> (B, H, T, D) + new_projection = projection.view(new_projection_shape).permute(0, 2, 1, 3) + return new_projection + + def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + residual = hidden_states + batch, channel_dim, seq = hidden_states.shape + + hidden_states = self.group_norm(hidden_states) + hidden_states = hidden_states.transpose(1, 2) + + query_proj = self.query(hidden_states) + key_proj = self.key(hidden_states) + value_proj = self.value(hidden_states) + + query_states = self.transpose_for_scores(query_proj) + key_states = self.transpose_for_scores(key_proj) + value_states = self.transpose_for_scores(value_proj) + + scale = 1 / math.sqrt(math.sqrt(key_states.shape[-1])) + + attention_scores = torch.matmul(query_states * scale, key_states.transpose(-1, -2) * scale) + attention_probs = torch.softmax(attention_scores, dim=-1) + + # compute attention output + hidden_states = torch.matmul(attention_probs, value_states) + + hidden_states = hidden_states.permute(0, 2, 1, 3).contiguous() + new_hidden_states_shape = hidden_states.size()[:-2] + (self.channels,) + hidden_states = hidden_states.view(new_hidden_states_shape) + + # compute next hidden_states + hidden_states = self.proj_attn(hidden_states) + hidden_states = hidden_states.transpose(1, 2) + hidden_states = self.dropout(hidden_states) + + output = hidden_states + residual + + return output + + +class ResConvBlock(nn.Module): + def __init__(self, in_channels: int, mid_channels: int, out_channels: int, is_last: bool = False): + super().__init__() + self.is_last = is_last + self.has_conv_skip = in_channels != out_channels + + if self.has_conv_skip: + self.conv_skip = nn.Conv1d(in_channels, out_channels, 1, bias=False) + + self.conv_1 = nn.Conv1d(in_channels, mid_channels, 5, padding=2) + self.group_norm_1 = nn.GroupNorm(1, mid_channels) + self.gelu_1 = nn.GELU() + self.conv_2 = nn.Conv1d(mid_channels, out_channels, 5, padding=2) + + if not self.is_last: + self.group_norm_2 = nn.GroupNorm(1, out_channels) + self.gelu_2 = nn.GELU() + + def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + residual = self.conv_skip(hidden_states) if self.has_conv_skip else hidden_states + + hidden_states = self.conv_1(hidden_states) + hidden_states = self.group_norm_1(hidden_states) + hidden_states = self.gelu_1(hidden_states) + hidden_states = self.conv_2(hidden_states) + + if not self.is_last: + hidden_states = self.group_norm_2(hidden_states) + hidden_states = self.gelu_2(hidden_states) + + output = hidden_states + residual + return output + + +class UNetMidBlock1D(nn.Module): + def __init__(self, mid_channels: int, in_channels: int, out_channels: Optional[int] = None): + super().__init__() + + out_channels = in_channels if out_channels is None else out_channels + + # there is always at least one resnet + self.down = Downsample1d("cubic") + resnets = [ + ResConvBlock(in_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, out_channels), + ] + attentions = [ + SelfAttention1d(mid_channels, mid_channels // 32), + SelfAttention1d(mid_channels, mid_channels // 32), + SelfAttention1d(mid_channels, mid_channels // 32), + SelfAttention1d(mid_channels, mid_channels // 32), + SelfAttention1d(mid_channels, mid_channels // 32), + SelfAttention1d(out_channels, out_channels // 32), + ] + self.up = Upsample1d(kernel="cubic") + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + hidden_states = self.down(hidden_states) + for attn, resnet in zip(self.attentions, self.resnets): + hidden_states = resnet(hidden_states) + hidden_states = attn(hidden_states) + + hidden_states = self.up(hidden_states) + + return hidden_states + + +class AttnDownBlock1D(nn.Module): + def __init__(self, out_channels: int, in_channels: int, mid_channels: Optional[int] = None): + super().__init__() + mid_channels = out_channels if mid_channels is None else mid_channels + + self.down = Downsample1d("cubic") + resnets = [ + ResConvBlock(in_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, out_channels), + ] + attentions = [ + SelfAttention1d(mid_channels, mid_channels // 32), + SelfAttention1d(mid_channels, mid_channels // 32), + SelfAttention1d(out_channels, out_channels // 32), + ] + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + hidden_states = self.down(hidden_states) + + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states) + hidden_states = attn(hidden_states) + + return hidden_states, (hidden_states,) + + +class DownBlock1D(nn.Module): + def __init__(self, out_channels: int, in_channels: int, mid_channels: Optional[int] = None): + super().__init__() + mid_channels = out_channels if mid_channels is None else mid_channels + + self.down = Downsample1d("cubic") + resnets = [ + ResConvBlock(in_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, out_channels), + ] + + self.resnets = nn.ModuleList(resnets) + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + hidden_states = self.down(hidden_states) + + for resnet in self.resnets: + hidden_states = resnet(hidden_states) + + return hidden_states, (hidden_states,) + + +class DownBlock1DNoSkip(nn.Module): + def __init__(self, out_channels: int, in_channels: int, mid_channels: Optional[int] = None): + super().__init__() + mid_channels = out_channels if mid_channels is None else mid_channels + + resnets = [ + ResConvBlock(in_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, out_channels), + ] + + self.resnets = nn.ModuleList(resnets) + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + hidden_states = torch.cat([hidden_states, temb], dim=1) + for resnet in self.resnets: + hidden_states = resnet(hidden_states) + + return hidden_states, (hidden_states,) + + +class AttnUpBlock1D(nn.Module): + def __init__(self, in_channels: int, out_channels: int, mid_channels: Optional[int] = None): + super().__init__() + mid_channels = out_channels if mid_channels is None else mid_channels + + resnets = [ + ResConvBlock(2 * in_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, out_channels), + ] + attentions = [ + SelfAttention1d(mid_channels, mid_channels // 32), + SelfAttention1d(mid_channels, mid_channels // 32), + SelfAttention1d(out_channels, out_channels // 32), + ] + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + self.up = Upsample1d(kernel="cubic") + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + res_hidden_states = res_hidden_states_tuple[-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states) + hidden_states = attn(hidden_states) + + hidden_states = self.up(hidden_states) + + return hidden_states + + +class UpBlock1D(nn.Module): + def __init__(self, in_channels: int, out_channels: int, mid_channels: Optional[int] = None): + super().__init__() + mid_channels = in_channels if mid_channels is None else mid_channels + + resnets = [ + ResConvBlock(2 * in_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, out_channels), + ] + + self.resnets = nn.ModuleList(resnets) + self.up = Upsample1d(kernel="cubic") + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + res_hidden_states = res_hidden_states_tuple[-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + for resnet in self.resnets: + hidden_states = resnet(hidden_states) + + hidden_states = self.up(hidden_states) + + return hidden_states + + +class UpBlock1DNoSkip(nn.Module): + def __init__(self, in_channels: int, out_channels: int, mid_channels: Optional[int] = None): + super().__init__() + mid_channels = in_channels if mid_channels is None else mid_channels + + resnets = [ + ResConvBlock(2 * in_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, mid_channels), + ResConvBlock(mid_channels, mid_channels, out_channels, is_last=True), + ] + + self.resnets = nn.ModuleList(resnets) + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + res_hidden_states = res_hidden_states_tuple[-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + for resnet in self.resnets: + hidden_states = resnet(hidden_states) + + return hidden_states + + +DownBlockType = Union[DownResnetBlock1D, DownBlock1D, AttnDownBlock1D, DownBlock1DNoSkip] +MidBlockType = Union[MidResTemporalBlock1D, ValueFunctionMidBlock1D, UNetMidBlock1D] +OutBlockType = Union[OutConv1DBlock, OutValueFunctionBlock] +UpBlockType = Union[UpResnetBlock1D, UpBlock1D, AttnUpBlock1D, UpBlock1DNoSkip] + + +def get_down_block( + down_block_type: str, + num_layers: int, + in_channels: int, + out_channels: int, + temb_channels: int, + add_downsample: bool, +) -> DownBlockType: + if down_block_type == "DownResnetBlock1D": + return DownResnetBlock1D( + in_channels=in_channels, + num_layers=num_layers, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + ) + elif down_block_type == "DownBlock1D": + return DownBlock1D(out_channels=out_channels, in_channels=in_channels) + elif down_block_type == "AttnDownBlock1D": + return AttnDownBlock1D(out_channels=out_channels, in_channels=in_channels) + elif down_block_type == "DownBlock1DNoSkip": + return DownBlock1DNoSkip(out_channels=out_channels, in_channels=in_channels) + raise ValueError(f"{down_block_type} does not exist.") + + +def get_up_block( + up_block_type: str, num_layers: int, in_channels: int, out_channels: int, temb_channels: int, add_upsample: bool +) -> UpBlockType: + if up_block_type == "UpResnetBlock1D": + return UpResnetBlock1D( + in_channels=in_channels, + num_layers=num_layers, + out_channels=out_channels, + temb_channels=temb_channels, + add_upsample=add_upsample, + ) + elif up_block_type == "UpBlock1D": + return UpBlock1D(in_channels=in_channels, out_channels=out_channels) + elif up_block_type == "AttnUpBlock1D": + return AttnUpBlock1D(in_channels=in_channels, out_channels=out_channels) + elif up_block_type == "UpBlock1DNoSkip": + return UpBlock1DNoSkip(in_channels=in_channels, out_channels=out_channels) + raise ValueError(f"{up_block_type} does not exist.") + + +def get_mid_block( + mid_block_type: str, + num_layers: int, + in_channels: int, + mid_channels: int, + out_channels: int, + embed_dim: int, + add_downsample: bool, +) -> MidBlockType: + if mid_block_type == "MidResTemporalBlock1D": + return MidResTemporalBlock1D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + embed_dim=embed_dim, + add_downsample=add_downsample, + ) + elif mid_block_type == "ValueFunctionMidBlock1D": + return ValueFunctionMidBlock1D(in_channels=in_channels, out_channels=out_channels, embed_dim=embed_dim) + elif mid_block_type == "UNetMidBlock1D": + return UNetMidBlock1D(in_channels=in_channels, mid_channels=mid_channels, out_channels=out_channels) + raise ValueError(f"{mid_block_type} does not exist.") + + +def get_out_block( + *, out_block_type: str, num_groups_out: int, embed_dim: int, out_channels: int, act_fn: str, fc_dim: int +) -> Optional[OutBlockType]: + if out_block_type == "OutConv1DBlock": + return OutConv1DBlock(num_groups_out, out_channels, embed_dim, act_fn) + elif out_block_type == "ValueFunction": + return OutValueFunctionBlock(fc_dim, embed_dim, act_fn) + return None diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_2d.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_2d.py new file mode 100755 index 0000000..5efb638 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_2d.py @@ -0,0 +1,346 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...utils import BaseOutput +from ..embeddings import GaussianFourierProjection, TimestepEmbedding, Timesteps +from ..modeling_utils import ModelMixin +from .unet_2d_blocks import UNetMidBlock2D, get_down_block, get_up_block + + +@dataclass +class UNet2DOutput(BaseOutput): + """ + The output of [`UNet2DModel`]. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + The hidden states output from the last layer of the model. + """ + + sample: torch.FloatTensor + + +class UNet2DModel(ModelMixin, ConfigMixin): + r""" + A 2D UNet model that takes a noisy sample and a timestep and returns a sample shaped output. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + Height and width of input/output sample. Dimensions must be a multiple of `2 ** (len(block_out_channels) - + 1)`. + in_channels (`int`, *optional*, defaults to 3): Number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 3): Number of channels in the output. + center_input_sample (`bool`, *optional*, defaults to `False`): Whether to center the input sample. + time_embedding_type (`str`, *optional*, defaults to `"positional"`): Type of time embedding to use. + freq_shift (`int`, *optional*, defaults to 0): Frequency shift for Fourier time embedding. + flip_sin_to_cos (`bool`, *optional*, defaults to `True`): + Whether to flip sin to cos for Fourier time embedding. + down_block_types (`Tuple[str]`, *optional*, defaults to `("DownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D")`): + Tuple of downsample block types. + mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2D"`): + Block type for middle of UNet, it can be either `UNetMidBlock2D` or `UnCLIPUNetMidBlock2D`. + up_block_types (`Tuple[str]`, *optional*, defaults to `("AttnUpBlock2D", "AttnUpBlock2D", "AttnUpBlock2D", "UpBlock2D")`): + Tuple of upsample block types. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(224, 448, 672, 896)`): + Tuple of block output channels. + layers_per_block (`int`, *optional*, defaults to `2`): The number of layers per block. + mid_block_scale_factor (`float`, *optional*, defaults to `1`): The scale factor for the mid block. + downsample_padding (`int`, *optional*, defaults to `1`): The padding for the downsample convolution. + downsample_type (`str`, *optional*, defaults to `conv`): + The downsample type for downsampling layers. Choose between "conv" and "resnet" + upsample_type (`str`, *optional*, defaults to `conv`): + The upsample type for upsampling layers. Choose between "conv" and "resnet" + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + attention_head_dim (`int`, *optional*, defaults to `8`): The attention head dimension. + norm_num_groups (`int`, *optional*, defaults to `32`): The number of groups for normalization. + attn_norm_num_groups (`int`, *optional*, defaults to `None`): + If set to an integer, a group norm layer will be created in the mid block's [`Attention`] layer with the + given number of groups. If left as `None`, the group norm layer will only be created if + `resnet_time_scale_shift` is set to `default`, and if created will have `norm_num_groups` groups. + norm_eps (`float`, *optional*, defaults to `1e-5`): The epsilon for normalization. + resnet_time_scale_shift (`str`, *optional*, defaults to `"default"`): Time scale shift config + for ResNet blocks (see [`~models.resnet.ResnetBlock2D`]). Choose from `default` or `scale_shift`. + class_embed_type (`str`, *optional*, defaults to `None`): + The type of class embedding to use which is ultimately summed with the time embeddings. Choose from `None`, + `"timestep"`, or `"identity"`. + num_class_embeds (`int`, *optional*, defaults to `None`): + Input dimension of the learnable embedding matrix to be projected to `time_embed_dim` when performing class + conditioning with `class_embed_type` equal to `None`. + """ + + @register_to_config + def __init__( + self, + sample_size: Optional[Union[int, Tuple[int, int]]] = None, + in_channels: int = 3, + out_channels: int = 3, + center_input_sample: bool = False, + time_embedding_type: str = "positional", + freq_shift: int = 0, + flip_sin_to_cos: bool = True, + down_block_types: Tuple[str, ...] = ("DownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D", "AttnDownBlock2D"), + up_block_types: Tuple[str, ...] = ("AttnUpBlock2D", "AttnUpBlock2D", "AttnUpBlock2D", "UpBlock2D"), + block_out_channels: Tuple[int, ...] = (224, 448, 672, 896), + layers_per_block: int = 2, + mid_block_scale_factor: float = 1, + downsample_padding: int = 1, + downsample_type: str = "conv", + upsample_type: str = "conv", + dropout: float = 0.0, + act_fn: str = "silu", + attention_head_dim: Optional[int] = 8, + norm_num_groups: int = 32, + attn_norm_num_groups: Optional[int] = None, + norm_eps: float = 1e-5, + resnet_time_scale_shift: str = "default", + add_attention: bool = True, + class_embed_type: Optional[str] = None, + num_class_embeds: Optional[int] = None, + num_train_timesteps: Optional[int] = None, + ): + super().__init__() + + self.sample_size = sample_size + time_embed_dim = block_out_channels[0] * 4 + + # Check inputs + if len(down_block_types) != len(up_block_types): + raise ValueError( + f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}." + ) + + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}." + ) + + # input + self.conv_in = nn.Conv2d(in_channels, block_out_channels[0], kernel_size=3, padding=(1, 1)) + + # time + if time_embedding_type == "fourier": + self.time_proj = GaussianFourierProjection(embedding_size=block_out_channels[0], scale=16) + timestep_input_dim = 2 * block_out_channels[0] + elif time_embedding_type == "positional": + self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift) + timestep_input_dim = block_out_channels[0] + elif time_embedding_type == "learned": + self.time_proj = nn.Embedding(num_train_timesteps, block_out_channels[0]) + timestep_input_dim = block_out_channels[0] + + self.time_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim) + + # class embedding + if class_embed_type is None and num_class_embeds is not None: + self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim) + elif class_embed_type == "timestep": + self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim) + elif class_embed_type == "identity": + self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim) + else: + self.class_embedding = None + + self.down_blocks = nn.ModuleList([]) + self.mid_block = None + self.up_blocks = nn.ModuleList([]) + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + temb_channels=time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attention_head_dim=attention_head_dim if attention_head_dim is not None else output_channel, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + downsample_type=downsample_type, + dropout=dropout, + ) + self.down_blocks.append(down_block) + + # mid + self.mid_block = UNetMidBlock2D( + in_channels=block_out_channels[-1], + temb_channels=time_embed_dim, + dropout=dropout, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_head_dim=attention_head_dim if attention_head_dim is not None else block_out_channels[-1], + resnet_groups=norm_num_groups, + attn_groups=attn_norm_num_groups, + add_attention=add_attention, + ) + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)] + + is_final_block = i == len(block_out_channels) - 1 + + up_block = get_up_block( + up_block_type, + num_layers=layers_per_block + 1, + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=time_embed_dim, + add_upsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + attention_head_dim=attention_head_dim if attention_head_dim is not None else output_channel, + resnet_time_scale_shift=resnet_time_scale_shift, + upsample_type=upsample_type, + dropout=dropout, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + num_groups_out = norm_num_groups if norm_num_groups is not None else min(block_out_channels[0] // 4, 32) + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=num_groups_out, eps=norm_eps) + self.conv_act = nn.SiLU() + self.conv_out = nn.Conv2d(block_out_channels[0], out_channels, kernel_size=3, padding=1) + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + class_labels: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[UNet2DOutput, Tuple]: + r""" + The [`UNet2DModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor with the following shape `(batch, channel, height, width)`. + timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input. + class_labels (`torch.FloatTensor`, *optional*, defaults to `None`): + Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unet_2d.UNet2DOutput`] instead of a plain tuple. + + Returns: + [`~models.unet_2d.UNet2DOutput`] or `tuple`: + If `return_dict` is True, an [`~models.unet_2d.UNet2DOutput`] is returned, otherwise a `tuple` is + returned where the first element is the sample tensor. + """ + # 0. center input if necessary + if self.config.center_input_sample: + sample = 2 * sample - 1.0 + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + timesteps = torch.tensor([timesteps], dtype=torch.long, device=sample.device) + elif torch.is_tensor(timesteps) and len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps * torch.ones(sample.shape[0], dtype=timesteps.dtype, device=timesteps.device) + + t_emb = self.time_proj(timesteps) + + # timesteps does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=self.dtype) + emb = self.time_embedding(t_emb) + + if self.class_embedding is not None: + if class_labels is None: + raise ValueError("class_labels should be provided when doing class conditioning") + + if self.config.class_embed_type == "timestep": + class_labels = self.time_proj(class_labels) + + class_emb = self.class_embedding(class_labels).to(dtype=self.dtype) + emb = emb + class_emb + elif self.class_embedding is None and class_labels is not None: + raise ValueError("class_embedding needs to be initialized in order to use class conditioning") + + # 2. pre-process + skip_sample = sample + sample = self.conv_in(sample) + + # 3. down + down_block_res_samples = (sample,) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, "skip_conv"): + sample, res_samples, skip_sample = downsample_block( + hidden_states=sample, temb=emb, skip_sample=skip_sample + ) + else: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb) + + down_block_res_samples += res_samples + + # 4. mid + sample = self.mid_block(sample, emb) + + # 5. up + skip_sample = None + for upsample_block in self.up_blocks: + res_samples = down_block_res_samples[-len(upsample_block.resnets) :] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + if hasattr(upsample_block, "skip_conv"): + sample, skip_sample = upsample_block(sample, res_samples, emb, skip_sample) + else: + sample = upsample_block(sample, res_samples, emb) + + # 6. post-process + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + if skip_sample is not None: + sample += skip_sample + + if self.config.time_embedding_type == "fourier": + timesteps = timesteps.reshape((sample.shape[0], *([1] * len(sample.shape[1:])))) + sample = sample / timesteps + + if not return_dict: + return (sample,) + + return UNet2DOutput(sample=sample) diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_blocks.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_blocks.py new file mode 100755 index 0000000..b9e9e63 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_blocks.py @@ -0,0 +1,3731 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Dict, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from ...utils import deprecate, is_torch_version, logging +from ...utils.torch_utils import apply_freeu +from ..activations import get_activation +from ..attention_processor import Attention, AttnAddedKVProcessor, AttnAddedKVProcessor2_0 +from ..normalization import AdaGroupNorm +from ..resnet import ( + Downsample2D, + FirDownsample2D, + FirUpsample2D, + KDownsample2D, + KUpsample2D, + ResnetBlock2D, + ResnetBlockCondNorm2D, + Upsample2D, +) +from ..transformers.dual_transformer_2d import DualTransformer2DModel +from ..transformers.transformer_2d import Transformer2DModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def get_down_block( + down_block_type: str, + num_layers: int, + in_channels: int, + out_channels: int, + temb_channels: int, + add_downsample: bool, + resnet_eps: float, + resnet_act_fn: str, + transformer_layers_per_block: int = 1, + num_attention_heads: Optional[int] = None, + resnet_groups: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + downsample_padding: Optional[int] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + attention_type: str = "default", + resnet_skip_time_act: bool = False, + resnet_out_scale_factor: float = 1.0, + cross_attention_norm: Optional[str] = None, + attention_head_dim: Optional[int] = None, + downsample_type: Optional[str] = None, + dropout: float = 0.0, +): + # If attn head dim is not defined, we default it to the number of heads + if attention_head_dim is None: + logger.warning( + f"It is recommended to provide `attention_head_dim` when calling `get_down_block`. Defaulting `attention_head_dim` to {num_attention_heads}." + ) + attention_head_dim = num_attention_heads + + down_block_type = down_block_type[7:] if down_block_type.startswith("UNetRes") else down_block_type + if down_block_type == "DownBlock2D": + return DownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "ResnetDownsampleBlock2D": + return ResnetDownsampleBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + ) + elif down_block_type == "AttnDownBlock2D": + if add_downsample is False: + downsample_type = None + else: + downsample_type = downsample_type or "conv" # default to 'conv' + return AttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + downsample_type=downsample_type, + ) + elif down_block_type == "CrossAttnDownBlock2D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnDownBlock2D") + return CrossAttnDownBlock2D( + num_layers=num_layers, + transformer_layers_per_block=transformer_layers_per_block, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + ) + elif down_block_type == "SimpleCrossAttnDownBlock2D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for SimpleCrossAttnDownBlock2D") + return SimpleCrossAttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif down_block_type == "SkipDownBlock2D": + return SkipDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "AttnSkipDownBlock2D": + return AttnSkipDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "DownEncoderBlock2D": + return DownEncoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "AttnDownEncoderBlock2D": + return AttnDownEncoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "KDownBlock2D": + return KDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + ) + elif down_block_type == "KCrossAttnDownBlock2D": + return KCrossAttnDownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + add_self_attention=True if not add_downsample else False, + ) + raise ValueError(f"{down_block_type} does not exist.") + + +def get_mid_block( + mid_block_type: str, + temb_channels: int, + in_channels: int, + resnet_eps: float, + resnet_act_fn: str, + resnet_groups: int, + output_scale_factor: float = 1.0, + transformer_layers_per_block: int = 1, + num_attention_heads: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + mid_block_only_cross_attention: bool = False, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + attention_type: str = "default", + resnet_skip_time_act: bool = False, + cross_attention_norm: Optional[str] = None, + attention_head_dim: Optional[int] = 1, + dropout: float = 0.0, +): + if mid_block_type == "UNetMidBlock2DCrossAttn": + return UNetMidBlock2DCrossAttn( + transformer_layers_per_block=transformer_layers_per_block, + in_channels=in_channels, + temb_channels=temb_channels, + dropout=dropout, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + output_scale_factor=output_scale_factor, + resnet_time_scale_shift=resnet_time_scale_shift, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + resnet_groups=resnet_groups, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + elif mid_block_type == "UNetMidBlock2DSimpleCrossAttn": + return UNetMidBlock2DSimpleCrossAttn( + in_channels=in_channels, + temb_channels=temb_channels, + dropout=dropout, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + output_scale_factor=output_scale_factor, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + only_cross_attention=mid_block_only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif mid_block_type == "UNetMidBlock2D": + return UNetMidBlock2D( + in_channels=in_channels, + temb_channels=temb_channels, + dropout=dropout, + num_layers=0, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + output_scale_factor=output_scale_factor, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + add_attention=False, + ) + elif mid_block_type is None: + return None + else: + raise ValueError(f"unknown mid_block_type : {mid_block_type}") + + +def get_up_block( + up_block_type: str, + num_layers: int, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + add_upsample: bool, + resnet_eps: float, + resnet_act_fn: str, + resolution_idx: Optional[int] = None, + transformer_layers_per_block: int = 1, + num_attention_heads: Optional[int] = None, + resnet_groups: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + attention_type: str = "default", + resnet_skip_time_act: bool = False, + resnet_out_scale_factor: float = 1.0, + cross_attention_norm: Optional[str] = None, + attention_head_dim: Optional[int] = None, + upsample_type: Optional[str] = None, + dropout: float = 0.0, +) -> nn.Module: + # If attn head dim is not defined, we default it to the number of heads + if attention_head_dim is None: + logger.warning( + f"It is recommended to provide `attention_head_dim` when calling `get_up_block`. Defaulting `attention_head_dim` to {num_attention_heads}." + ) + attention_head_dim = num_attention_heads + + up_block_type = up_block_type[7:] if up_block_type.startswith("UNetRes") else up_block_type + if up_block_type == "UpBlock2D": + return UpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == "ResnetUpsampleBlock2D": + return ResnetUpsampleBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + ) + elif up_block_type == "CrossAttnUpBlock2D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnUpBlock2D") + return CrossAttnUpBlock2D( + num_layers=num_layers, + transformer_layers_per_block=transformer_layers_per_block, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + ) + elif up_block_type == "SimpleCrossAttnUpBlock2D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for SimpleCrossAttnUpBlock2D") + return SimpleCrossAttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + output_scale_factor=resnet_out_scale_factor, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif up_block_type == "AttnUpBlock2D": + if add_upsample is False: + upsample_type = None + else: + upsample_type = upsample_type or "conv" # default to 'conv' + + return AttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + upsample_type=upsample_type, + ) + elif up_block_type == "SkipUpBlock2D": + return SkipUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == "AttnSkipUpBlock2D": + return AttnSkipUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == "UpDecoderBlock2D": + return UpDecoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + temb_channels=temb_channels, + ) + elif up_block_type == "AttnUpDecoderBlock2D": + return AttnUpDecoderBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + attention_head_dim=attention_head_dim, + resnet_time_scale_shift=resnet_time_scale_shift, + temb_channels=temb_channels, + ) + elif up_block_type == "KUpBlock2D": + return KUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + ) + elif up_block_type == "KCrossAttnUpBlock2D": + return KCrossAttnUpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + cross_attention_dim=cross_attention_dim, + attention_head_dim=attention_head_dim, + ) + + raise ValueError(f"{up_block_type} does not exist.") + + +class AutoencoderTinyBlock(nn.Module): + """ + Tiny Autoencoder block used in [`AutoencoderTiny`]. It is a mini residual module consisting of plain conv + ReLU + blocks. + + Args: + in_channels (`int`): The number of input channels. + out_channels (`int`): The number of output channels. + act_fn (`str`): + ` The activation function to use. Supported values are `"swish"`, `"mish"`, `"gelu"`, and `"relu"`. + + Returns: + `torch.FloatTensor`: A tensor with the same shape as the input tensor, but with the number of channels equal to + `out_channels`. + """ + + def __init__(self, in_channels: int, out_channels: int, act_fn: str): + super().__init__() + act_fn = get_activation(act_fn) + self.conv = nn.Sequential( + nn.Conv2d(in_channels, out_channels, kernel_size=3, padding=1), + act_fn, + nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1), + act_fn, + nn.Conv2d(out_channels, out_channels, kernel_size=3, padding=1), + ) + self.skip = ( + nn.Conv2d(in_channels, out_channels, kernel_size=1, bias=False) + if in_channels != out_channels + else nn.Identity() + ) + self.fuse = nn.ReLU() + + def forward(self, x: torch.FloatTensor) -> torch.FloatTensor: + return self.fuse(self.conv(x) + self.skip(x)) + + +class UNetMidBlock2D(nn.Module): + """ + A 2D UNet mid-block [`UNetMidBlock2D`] with multiple residual blocks and optional attention blocks. + + Args: + in_channels (`int`): The number of input channels. + temb_channels (`int`): The number of temporal embedding channels. + dropout (`float`, *optional*, defaults to 0.0): The dropout rate. + num_layers (`int`, *optional*, defaults to 1): The number of residual blocks. + resnet_eps (`float`, *optional*, 1e-6 ): The epsilon value for the resnet blocks. + resnet_time_scale_shift (`str`, *optional*, defaults to `default`): + The type of normalization to apply to the time embeddings. This can help to improve the performance of the + model on tasks with long-range temporal dependencies. + resnet_act_fn (`str`, *optional*, defaults to `swish`): The activation function for the resnet blocks. + resnet_groups (`int`, *optional*, defaults to 32): + The number of groups to use in the group normalization layers of the resnet blocks. + attn_groups (`Optional[int]`, *optional*, defaults to None): The number of groups for the attention blocks. + resnet_pre_norm (`bool`, *optional*, defaults to `True`): + Whether to use pre-normalization for the resnet blocks. + add_attention (`bool`, *optional*, defaults to `True`): Whether to add attention blocks. + attention_head_dim (`int`, *optional*, defaults to 1): + Dimension of a single attention head. The number of attention heads is determined based on this value and + the number of input channels. + output_scale_factor (`float`, *optional*, defaults to 1.0): The output scale factor. + + Returns: + `torch.FloatTensor`: The output of the last residual block, which is a tensor of shape `(batch_size, + in_channels, height, width)`. + + """ + + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", # default, spatial + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + attn_groups: Optional[int] = None, + resnet_pre_norm: bool = True, + add_attention: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + ): + super().__init__() + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + self.add_attention = add_attention + + if attn_groups is None: + attn_groups = resnet_groups if resnet_time_scale_shift == "default" else None + + # there is always at least one resnet + if resnet_time_scale_shift == "spatial": + resnets = [ + ResnetBlockCondNorm2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm="spatial", + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + ) + ] + else: + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + + if attention_head_dim is None: + logger.warning( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {in_channels}." + ) + attention_head_dim = in_channels + + for _ in range(num_layers): + if self.add_attention: + attentions.append( + Attention( + in_channels, + heads=in_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=attn_groups, + spatial_norm_dim=temb_channels if resnet_time_scale_shift == "spatial" else None, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + else: + attentions.append(None) + + if resnet_time_scale_shift == "spatial": + resnets.append( + ResnetBlockCondNorm2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm="spatial", + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + ) + ) + else: + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if attn is not None: + hidden_states = attn(hidden_states, temb=temb) + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +class UNetMidBlock2DCrossAttn(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + output_scale_factor: float = 1.0, + cross_attention_dim: int = 1280, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + upcast_attention: bool = False, + attention_type: str = "default", + ): + super().__init__() + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + + # support for variable transformer layers per block + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + + for i in range(num_layers): + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + if cross_attention_kwargs is not None: + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + else: + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +class UNetMidBlock2DSimpleCrossAttn(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + cross_attention_dim: int = 1280, + skip_time_act: bool = False, + only_cross_attention: bool = False, + cross_attention_norm: Optional[str] = None, + ): + super().__init__() + + self.has_cross_attention = True + + self.attention_head_dim = attention_head_dim + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + + self.num_heads = in_channels // self.attention_head_dim + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ] + attentions = [] + + for _ in range(num_layers): + processor = ( + AttnAddedKVProcessor2_0() if hasattr(F, "scaled_dot_product_attention") else AttnAddedKVProcessor() + ) + + attentions.append( + Attention( + query_dim=in_channels, + cross_attention_dim=in_channels, + heads=self.num_heads, + dim_head=self.attention_head_dim, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + ) + ) + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + if attention_mask is None: + # if encoder_hidden_states is defined: we are doing cross-attn, so we should use cross-attn mask. + mask = None if encoder_hidden_states is None else encoder_attention_mask + else: + # when attention_mask is defined: we don't even check for encoder_attention_mask. + # this is to maintain compatibility with UnCLIP, which uses 'attention_mask' param for cross-attn masks. + # TODO: UnCLIP should express cross-attn mask via encoder_attention_mask param instead of via attention_mask. + # then we can simplify this whole if/else block to: + # mask = attention_mask if encoder_hidden_states is None else encoder_attention_mask + mask = attention_mask + + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + # attn + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + + # resnet + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +class AttnDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + downsample_padding: int = 1, + downsample_type: str = "conv", + ): + super().__init__() + resnets = [] + attentions = [] + self.downsample_type = downsample_type + + if attention_head_dim is None: + logger.warning( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if downsample_type == "conv": + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + elif downsample_type == "resnet": + self.downsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + down=True, + ) + ] + ) + else: + self.downsamplers = None + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb) + hidden_states = attn(hidden_states, **cross_attention_kwargs) + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + if self.downsample_type == "resnet": + hidden_states = downsampler(hidden_states, temb=temb) + else: + hidden_states = downsampler(hidden_states) + + output_states += (hidden_states,) + + return hidden_states, output_states + + +class CrossAttnDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + downsample_padding: int = 1, + add_downsample: bool = True, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + attention_type: str = "default", + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + additional_residuals: Optional[torch.FloatTensor] = None, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + if cross_attention_kwargs is not None: + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + output_states = () + + blocks = list(zip(self.resnets, self.attentions)) + + for i, (resnet, attn) in enumerate(blocks): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + + # apply additional residuals to the output of the last pair of resnet and attention blocks + if i == len(blocks) - 1 and additional_residuals is not None: + hidden_states = hidden_states + additional_residuals + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class DownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + downsample_padding: int = 1, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None, *args, **kwargs + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb) + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class DownEncoderBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + downsample_padding: int = 1, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + if resnet_time_scale_shift == "spatial": + resnets.append( + ResnetBlockCondNorm2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm="spatial", + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + ) + ) + else: + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + else: + self.downsamplers = None + + def forward(self, hidden_states: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb=None) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states + + +class AttnDownEncoderBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + downsample_padding: int = 1, + ): + super().__init__() + resnets = [] + attentions = [] + + if attention_head_dim is None: + logger.warning( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + if resnet_time_scale_shift == "spatial": + resnets.append( + ResnetBlockCondNorm2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm="spatial", + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + ) + ) + else: + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=None, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + else: + self.downsamplers = None + + def forward(self, hidden_states: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb=None) + hidden_states = attn(hidden_states) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states + + +class AttnSkipDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = np.sqrt(2.0), + add_downsample: bool = True, + ): + super().__init__() + self.attentions = nn.ModuleList([]) + self.resnets = nn.ModuleList([]) + + if attention_head_dim is None: + logger.warning( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + self.resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(in_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + self.attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=32, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + if add_downsample: + self.resnet_down = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + down=True, + kernel="fir", + ) + self.downsamplers = nn.ModuleList([FirDownsample2D(out_channels, out_channels=out_channels)]) + self.skip_conv = nn.Conv2d(3, out_channels, kernel_size=(1, 1), stride=(1, 1)) + else: + self.resnet_down = None + self.downsamplers = None + self.skip_conv = None + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + skip_sample: Optional[torch.FloatTensor] = None, + *args, + **kwargs, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...], torch.FloatTensor]: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb) + hidden_states = attn(hidden_states) + output_states += (hidden_states,) + + if self.downsamplers is not None: + hidden_states = self.resnet_down(hidden_states, temb) + for downsampler in self.downsamplers: + skip_sample = downsampler(skip_sample) + + hidden_states = self.skip_conv(skip_sample) + hidden_states + + output_states += (hidden_states,) + + return hidden_states, output_states, skip_sample + + +class SkipDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_pre_norm: bool = True, + output_scale_factor: float = np.sqrt(2.0), + add_downsample: bool = True, + downsample_padding: int = 1, + ): + super().__init__() + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + self.resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(in_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + if add_downsample: + self.resnet_down = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + down=True, + kernel="fir", + ) + self.downsamplers = nn.ModuleList([FirDownsample2D(out_channels, out_channels=out_channels)]) + self.skip_conv = nn.Conv2d(3, out_channels, kernel_size=(1, 1), stride=(1, 1)) + else: + self.resnet_down = None + self.downsamplers = None + self.skip_conv = None + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + skip_sample: Optional[torch.FloatTensor] = None, + *args, + **kwargs, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...], torch.FloatTensor]: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + output_states = () + + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb) + output_states += (hidden_states,) + + if self.downsamplers is not None: + hidden_states = self.resnet_down(hidden_states, temb) + for downsampler in self.downsamplers: + skip_sample = downsampler(skip_sample) + + hidden_states = self.skip_conv(skip_sample) + hidden_states + + output_states += (hidden_states,) + + return hidden_states, output_states, skip_sample + + +class ResnetDownsampleBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + skip_time_act: bool = False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + down=True, + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None, *args, **kwargs + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb) + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, temb) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class SimpleCrossAttnDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + skip_time_act: bool = False, + only_cross_attention: bool = False, + cross_attention_norm: Optional[str] = None, + ): + super().__init__() + + self.has_cross_attention = True + + resnets = [] + attentions = [] + + self.attention_head_dim = attention_head_dim + self.num_heads = out_channels // self.attention_head_dim + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ) + + processor = ( + AttnAddedKVProcessor2_0() if hasattr(F, "scaled_dot_product_attention") else AttnAddedKVProcessor() + ) + + attentions.append( + Attention( + query_dim=out_channels, + cross_attention_dim=out_channels, + heads=self.num_heads, + dim_head=attention_head_dim, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + down=True, + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + output_states = () + + if attention_mask is None: + # if encoder_hidden_states is defined: we are doing cross-attn, so we should use cross-attn mask. + mask = None if encoder_hidden_states is None else encoder_attention_mask + else: + # when attention_mask is defined: we don't even check for encoder_attention_mask. + # this is to maintain compatibility with UnCLIP, which uses 'attention_mask' param for cross-attn masks. + # TODO: UnCLIP should express cross-attn mask via encoder_attention_mask param instead of via attention_mask. + # then we can simplify this whole if/else block to: + # mask = attention_mask if encoder_hidden_states is None else encoder_attention_mask + mask = attention_mask + + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + else: + hidden_states = resnet(hidden_states, temb) + + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states, temb) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class KDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_eps: float = 1e-5, + resnet_act_fn: str = "gelu", + resnet_group_size: int = 32, + add_downsample: bool = False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlockCondNorm2D( + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + temb_channels=temb_channels, + groups=groups, + groups_out=groups_out, + eps=resnet_eps, + non_linearity=resnet_act_fn, + time_embedding_norm="ada_group", + conv_shortcut_bias=False, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + # YiYi's comments- might be able to use FirDownsample2D, look into details later + self.downsamplers = nn.ModuleList([KDownsample2D()]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None, *args, **kwargs + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb) + + output_states += (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states, output_states + + +class KCrossAttnDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + cross_attention_dim: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_group_size: int = 32, + add_downsample: bool = True, + attention_head_dim: int = 64, + add_self_attention: bool = False, + resnet_eps: float = 1e-5, + resnet_act_fn: str = "gelu", + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlockCondNorm2D( + in_channels=in_channels, + out_channels=out_channels, + dropout=dropout, + temb_channels=temb_channels, + groups=groups, + groups_out=groups_out, + eps=resnet_eps, + non_linearity=resnet_act_fn, + time_embedding_norm="ada_group", + conv_shortcut_bias=False, + ) + ) + attentions.append( + KAttentionBlock( + out_channels, + out_channels // attention_head_dim, + attention_head_dim, + cross_attention_dim=cross_attention_dim, + temb_channels=temb_channels, + attention_bias=True, + add_self_attention=add_self_attention, + cross_attention_norm="layer_norm", + group_size=resnet_group_size, + ) + ) + + self.resnets = nn.ModuleList(resnets) + self.attentions = nn.ModuleList(attentions) + + if add_downsample: + self.downsamplers = nn.ModuleList([KDownsample2D()]) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + + if self.downsamplers is None: + output_states += (None,) + else: + output_states += (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + return hidden_states, output_states + + +class AttnUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: int = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + upsample_type: str = "conv", + ): + super().__init__() + resnets = [] + attentions = [] + + self.upsample_type = upsample_type + + if attention_head_dim is None: + logger.warning( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if upsample_type == "conv": + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + elif upsample_type == "resnet": + self.upsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + up=True, + ) + ] + ) + else: + self.upsamplers = None + + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + *args, + **kwargs, + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + hidden_states = resnet(hidden_states, temb) + hidden_states = attn(hidden_states) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + if self.upsample_type == "resnet": + hidden_states = upsampler(hidden_states, temb=temb) + else: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class CrossAttnUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + attention_type: str = "default", + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + if cross_attention_kwargs is not None: + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + is_freeu_enabled = ( + getattr(self, "s1", None) + and getattr(self, "s2", None) + and getattr(self, "b1", None) + and getattr(self, "b2", None) + ) + + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # FreeU: Only operate on the first two stages + if is_freeu_enabled: + hidden_states, res_hidden_states = apply_freeu( + self.resolution_idx, + hidden_states, + res_hidden_states, + s1=self.s1, + s2=self.s2, + b1=self.b1, + b2=self.b2, + ) + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +class UpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + *args, + **kwargs, + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + is_freeu_enabled = ( + getattr(self, "s1", None) + and getattr(self, "s2", None) + and getattr(self, "b1", None) + and getattr(self, "b2", None) + ) + + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # FreeU: Only operate on the first two stages + if is_freeu_enabled: + hidden_states, res_hidden_states = apply_freeu( + self.resolution_idx, + hidden_states, + res_hidden_states, + s1=self.s1, + s2=self.s2, + b1=self.b1, + b2=self.b2, + ) + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +class UpDecoderBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", # default, spatial + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + temb_channels: Optional[int] = None, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + if resnet_time_scale_shift == "spatial": + resnets.append( + ResnetBlockCondNorm2D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm="spatial", + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + ) + ) + else: + resnets.append( + ResnetBlock2D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.resolution_idx = resolution_idx + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb=temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class AttnUpDecoderBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + temb_channels: Optional[int] = None, + ): + super().__init__() + resnets = [] + attentions = [] + + if attention_head_dim is None: + logger.warning( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `out_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + if resnet_time_scale_shift == "spatial": + resnets.append( + ResnetBlockCondNorm2D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm="spatial", + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + ) + ) + else: + resnets.append( + ResnetBlock2D( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=resnet_groups if resnet_time_scale_shift != "spatial" else None, + spatial_norm_dim=temb_channels if resnet_time_scale_shift == "spatial" else None, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.resolution_idx = resolution_idx + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb=temb) + hidden_states = attn(hidden_states, temb=temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class AttnSkipUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = np.sqrt(2.0), + add_upsample: bool = True, + ): + super().__init__() + self.attentions = nn.ModuleList([]) + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + self.resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(resnet_in_channels + res_skip_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + if attention_head_dim is None: + logger.warning( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `out_channels`: {out_channels}." + ) + attention_head_dim = out_channels + + self.attentions.append( + Attention( + out_channels, + heads=out_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=32, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + + self.upsampler = FirUpsample2D(in_channels, out_channels=out_channels) + if add_upsample: + self.resnet_up = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + up=True, + kernel="fir", + ) + self.skip_conv = nn.Conv2d(out_channels, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + self.skip_norm = torch.nn.GroupNorm( + num_groups=min(out_channels // 4, 32), num_channels=out_channels, eps=resnet_eps, affine=True + ) + self.act = nn.SiLU() + else: + self.resnet_up = None + self.skip_conv = None + self.skip_norm = None + self.act = None + + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + skip_sample=None, + *args, + **kwargs, + ) -> Tuple[torch.FloatTensor, torch.FloatTensor]: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + hidden_states = resnet(hidden_states, temb) + + hidden_states = self.attentions[0](hidden_states) + + if skip_sample is not None: + skip_sample = self.upsampler(skip_sample) + else: + skip_sample = 0 + + if self.resnet_up is not None: + skip_sample_states = self.skip_norm(hidden_states) + skip_sample_states = self.act(skip_sample_states) + skip_sample_states = self.skip_conv(skip_sample_states) + + skip_sample = skip_sample + skip_sample_states + + hidden_states = self.resnet_up(hidden_states, temb) + + return hidden_states, skip_sample + + +class SkipUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_pre_norm: bool = True, + output_scale_factor: float = np.sqrt(2.0), + add_upsample: bool = True, + upsample_padding: int = 1, + ): + super().__init__() + self.resnets = nn.ModuleList([]) + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + self.resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min((resnet_in_channels + res_skip_channels) // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.upsampler = FirUpsample2D(in_channels, out_channels=out_channels) + if add_upsample: + self.resnet_up = ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=min(out_channels // 4, 32), + groups_out=min(out_channels // 4, 32), + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + use_in_shortcut=True, + up=True, + kernel="fir", + ) + self.skip_conv = nn.Conv2d(out_channels, 3, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1)) + self.skip_norm = torch.nn.GroupNorm( + num_groups=min(out_channels // 4, 32), num_channels=out_channels, eps=resnet_eps, affine=True + ) + self.act = nn.SiLU() + else: + self.resnet_up = None + self.skip_conv = None + self.skip_norm = None + self.act = None + + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + skip_sample=None, + *args, + **kwargs, + ) -> Tuple[torch.FloatTensor, torch.FloatTensor]: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + hidden_states = resnet(hidden_states, temb) + + if skip_sample is not None: + skip_sample = self.upsampler(skip_sample) + else: + skip_sample = 0 + + if self.resnet_up is not None: + skip_sample_states = self.skip_norm(hidden_states) + skip_sample_states = self.act(skip_sample_states) + skip_sample_states = self.skip_conv(skip_sample_states) + + skip_sample = skip_sample + skip_sample_states + + hidden_states = self.resnet_up(hidden_states, temb) + + return hidden_states, skip_sample + + +class ResnetUpsampleBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + skip_time_act: bool = False, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + up=True, + ) + ] + ) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + *args, + **kwargs, + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, temb) + + return hidden_states + + +class SimpleCrossAttnUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + skip_time_act: bool = False, + only_cross_attention: bool = False, + cross_attention_norm: Optional[str] = None, + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.attention_head_dim = attention_head_dim + + self.num_heads = out_channels // self.attention_head_dim + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ) + + processor = ( + AttnAddedKVProcessor2_0() if hasattr(F, "scaled_dot_product_attention") else AttnAddedKVProcessor() + ) + + attentions.append( + Attention( + query_dim=out_channels, + cross_attention_dim=out_channels, + heads=self.num_heads, + dim_head=self.attention_head_dim, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList( + [ + ResnetBlock2D( + in_channels=out_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + up=True, + ) + ] + ) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + if attention_mask is None: + # if encoder_hidden_states is defined: we are doing cross-attn, so we should use cross-attn mask. + mask = None if encoder_hidden_states is None else encoder_attention_mask + else: + # when attention_mask is defined: we don't even check for encoder_attention_mask. + # this is to maintain compatibility with UnCLIP, which uses 'attention_mask' param for cross-attn masks. + # TODO: UnCLIP should express cross-attn mask via encoder_attention_mask param instead of via attention_mask. + # then we can simplify this whole if/else block to: + # mask = attention_mask if encoder_hidden_states is None else encoder_attention_mask + mask = attention_mask + + for resnet, attn in zip(self.resnets, self.attentions): + # resnet + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + hidden_states = torch.utils.checkpoint.checkpoint(create_custom_forward(resnet), hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + else: + hidden_states = resnet(hidden_states, temb) + + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, temb) + + return hidden_states + + +class KUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + resolution_idx: int, + dropout: float = 0.0, + num_layers: int = 5, + resnet_eps: float = 1e-5, + resnet_act_fn: str = "gelu", + resnet_group_size: Optional[int] = 32, + add_upsample: bool = True, + ): + super().__init__() + resnets = [] + k_in_channels = 2 * out_channels + k_out_channels = in_channels + num_layers = num_layers - 1 + + for i in range(num_layers): + in_channels = k_in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + resnets.append( + ResnetBlockCondNorm2D( + in_channels=in_channels, + out_channels=k_out_channels if (i == num_layers - 1) else out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=groups, + groups_out=groups_out, + dropout=dropout, + non_linearity=resnet_act_fn, + time_embedding_norm="ada_group", + conv_shortcut_bias=False, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([KUpsample2D()]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + *args, + **kwargs, + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + res_hidden_states_tuple = res_hidden_states_tuple[-1] + if res_hidden_states_tuple is not None: + hidden_states = torch.cat([hidden_states, res_hidden_states_tuple], dim=1) + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class KCrossAttnUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + resolution_idx: int, + dropout: float = 0.0, + num_layers: int = 4, + resnet_eps: float = 1e-5, + resnet_act_fn: str = "gelu", + resnet_group_size: int = 32, + attention_head_dim: int = 1, # attention dim_head + cross_attention_dim: int = 768, + add_upsample: bool = True, + upcast_attention: bool = False, + ): + super().__init__() + resnets = [] + attentions = [] + + is_first_block = in_channels == out_channels == temb_channels + is_middle_block = in_channels != out_channels + add_self_attention = True if is_first_block else False + + self.has_cross_attention = True + self.attention_head_dim = attention_head_dim + + # in_channels, and out_channels for the block (k-unet) + k_in_channels = out_channels if is_first_block else 2 * out_channels + k_out_channels = in_channels + + num_layers = num_layers - 1 + + for i in range(num_layers): + in_channels = k_in_channels if i == 0 else out_channels + groups = in_channels // resnet_group_size + groups_out = out_channels // resnet_group_size + + if is_middle_block and (i == num_layers - 1): + conv_2d_out_channels = k_out_channels + else: + conv_2d_out_channels = None + + resnets.append( + ResnetBlockCondNorm2D( + in_channels=in_channels, + out_channels=out_channels, + conv_2d_out_channels=conv_2d_out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=groups, + groups_out=groups_out, + dropout=dropout, + non_linearity=resnet_act_fn, + time_embedding_norm="ada_group", + conv_shortcut_bias=False, + ) + ) + attentions.append( + KAttentionBlock( + k_out_channels if (i == num_layers - 1) else out_channels, + k_out_channels // attention_head_dim + if (i == num_layers - 1) + else out_channels // attention_head_dim, + attention_head_dim, + cross_attention_dim=cross_attention_dim, + temb_channels=temb_channels, + attention_bias=True, + add_self_attention=add_self_attention, + cross_attention_norm="layer_norm", + upcast_attention=upcast_attention, + ) + ) + + self.resnets = nn.ModuleList(resnets) + self.attentions = nn.ModuleList(attentions) + + if add_upsample: + self.upsamplers = nn.ModuleList([KUpsample2D()]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + res_hidden_states_tuple = res_hidden_states_tuple[-1] + if res_hidden_states_tuple is not None: + hidden_states = torch.cat([hidden_states, res_hidden_states_tuple], dim=1) + + for resnet, attn in zip(self.resnets, self.attentions): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + emb=temb, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +# can potentially later be renamed to `No-feed-forward` attention +class KAttentionBlock(nn.Module): + r""" + A basic Transformer block. + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + attention_bias (`bool`, *optional*, defaults to `False`): + Configure if the attention layers should contain a bias parameter. + upcast_attention (`bool`, *optional*, defaults to `False`): + Set to `True` to upcast the attention computation to `float32`. + temb_channels (`int`, *optional*, defaults to 768): + The number of channels in the token embedding. + add_self_attention (`bool`, *optional*, defaults to `False`): + Set to `True` to add self-attention to the block. + cross_attention_norm (`str`, *optional*, defaults to `None`): + The type of normalization to use for the cross attention. Can be `None`, `layer_norm`, or `group_norm`. + group_size (`int`, *optional*, defaults to 32): + The number of groups to separate the channels into for group normalization. + """ + + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + dropout: float = 0.0, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + upcast_attention: bool = False, + temb_channels: int = 768, # for ada_group_norm + add_self_attention: bool = False, + cross_attention_norm: Optional[str] = None, + group_size: int = 32, + ): + super().__init__() + self.add_self_attention = add_self_attention + + # 1. Self-Attn + if add_self_attention: + self.norm1 = AdaGroupNorm(temb_channels, dim, max(1, dim // group_size)) + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=None, + cross_attention_norm=None, + ) + + # 2. Cross-Attn + self.norm2 = AdaGroupNorm(temb_channels, dim, max(1, dim // group_size)) + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + upcast_attention=upcast_attention, + cross_attention_norm=cross_attention_norm, + ) + + def _to_3d(self, hidden_states: torch.FloatTensor, height: int, weight: int) -> torch.FloatTensor: + return hidden_states.permute(0, 2, 3, 1).reshape(hidden_states.shape[0], height * weight, -1) + + def _to_4d(self, hidden_states: torch.FloatTensor, height: int, weight: int) -> torch.FloatTensor: + return hidden_states.permute(0, 2, 1).reshape(hidden_states.shape[0], -1, height, weight) + + def forward( + self, + hidden_states: torch.FloatTensor, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + # TODO: mark emb as non-optional (self.norm2 requires it). + # requires assessing impact of change to positional param interface. + emb: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + # 1. Self-Attention + if self.add_self_attention: + norm_hidden_states = self.norm1(hidden_states, emb) + + height, weight = norm_hidden_states.shape[2:] + norm_hidden_states = self._to_3d(norm_hidden_states, height, weight) + + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + attn_output = self._to_4d(attn_output, height, weight) + + hidden_states = attn_output + hidden_states + + # 2. Cross-Attention/None + norm_hidden_states = self.norm2(hidden_states, emb) + + height, weight = norm_hidden_states.shape[2:] + norm_hidden_states = self._to_3d(norm_hidden_states, height, weight) + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask if encoder_hidden_states is None else encoder_attention_mask, + **cross_attention_kwargs, + ) + attn_output = self._to_4d(attn_output, height, weight) + + hidden_states = attn_output + hidden_states + + return hidden_states diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_blocks_flax.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_blocks_flax.py new file mode 100755 index 0000000..a4585db --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_blocks_flax.py @@ -0,0 +1,400 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import flax.linen as nn +import jax.numpy as jnp + +from ..attention_flax import FlaxTransformer2DModel +from ..resnet_flax import FlaxDownsample2D, FlaxResnetBlock2D, FlaxUpsample2D + + +class FlaxCrossAttnDownBlock2D(nn.Module): + r""" + Cross Attention 2D Downsizing block - original architecture from Unet transformers: + https://arxiv.org/abs/2103.06104 + + Parameters: + in_channels (:obj:`int`): + Input channels + out_channels (:obj:`int`): + Output channels + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + num_layers (:obj:`int`, *optional*, defaults to 1): + Number of attention blocks layers + num_attention_heads (:obj:`int`, *optional*, defaults to 1): + Number of attention heads of each spatial transformer block + add_downsample (:obj:`bool`, *optional*, defaults to `True`): + Whether to add downsampling layer before each final output + use_memory_efficient_attention (`bool`, *optional*, defaults to `False`): + enable memory efficient attention https://arxiv.org/abs/2112.05682 + split_head_dim (`bool`, *optional*, defaults to `False`): + Whether to split the head dimension into a new axis for the self-attention computation. In most cases, + enabling this flag should speed up the computation for Stable Diffusion 2.x and Stable Diffusion XL. + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int + out_channels: int + dropout: float = 0.0 + num_layers: int = 1 + num_attention_heads: int = 1 + add_downsample: bool = True + use_linear_projection: bool = False + only_cross_attention: bool = False + use_memory_efficient_attention: bool = False + split_head_dim: bool = False + dtype: jnp.dtype = jnp.float32 + transformer_layers_per_block: int = 1 + + def setup(self): + resnets = [] + attentions = [] + + for i in range(self.num_layers): + in_channels = self.in_channels if i == 0 else self.out_channels + + res_block = FlaxResnetBlock2D( + in_channels=in_channels, + out_channels=self.out_channels, + dropout_prob=self.dropout, + dtype=self.dtype, + ) + resnets.append(res_block) + + attn_block = FlaxTransformer2DModel( + in_channels=self.out_channels, + n_heads=self.num_attention_heads, + d_head=self.out_channels // self.num_attention_heads, + depth=self.transformer_layers_per_block, + use_linear_projection=self.use_linear_projection, + only_cross_attention=self.only_cross_attention, + use_memory_efficient_attention=self.use_memory_efficient_attention, + split_head_dim=self.split_head_dim, + dtype=self.dtype, + ) + attentions.append(attn_block) + + self.resnets = resnets + self.attentions = attentions + + if self.add_downsample: + self.downsamplers_0 = FlaxDownsample2D(self.out_channels, dtype=self.dtype) + + def __call__(self, hidden_states, temb, encoder_hidden_states, deterministic=True): + output_states = () + + for resnet, attn in zip(self.resnets, self.attentions): + hidden_states = resnet(hidden_states, temb, deterministic=deterministic) + hidden_states = attn(hidden_states, encoder_hidden_states, deterministic=deterministic) + output_states += (hidden_states,) + + if self.add_downsample: + hidden_states = self.downsamplers_0(hidden_states) + output_states += (hidden_states,) + + return hidden_states, output_states + + +class FlaxDownBlock2D(nn.Module): + r""" + Flax 2D downsizing block + + Parameters: + in_channels (:obj:`int`): + Input channels + out_channels (:obj:`int`): + Output channels + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + num_layers (:obj:`int`, *optional*, defaults to 1): + Number of attention blocks layers + add_downsample (:obj:`bool`, *optional*, defaults to `True`): + Whether to add downsampling layer before each final output + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int + out_channels: int + dropout: float = 0.0 + num_layers: int = 1 + add_downsample: bool = True + dtype: jnp.dtype = jnp.float32 + + def setup(self): + resnets = [] + + for i in range(self.num_layers): + in_channels = self.in_channels if i == 0 else self.out_channels + + res_block = FlaxResnetBlock2D( + in_channels=in_channels, + out_channels=self.out_channels, + dropout_prob=self.dropout, + dtype=self.dtype, + ) + resnets.append(res_block) + self.resnets = resnets + + if self.add_downsample: + self.downsamplers_0 = FlaxDownsample2D(self.out_channels, dtype=self.dtype) + + def __call__(self, hidden_states, temb, deterministic=True): + output_states = () + + for resnet in self.resnets: + hidden_states = resnet(hidden_states, temb, deterministic=deterministic) + output_states += (hidden_states,) + + if self.add_downsample: + hidden_states = self.downsamplers_0(hidden_states) + output_states += (hidden_states,) + + return hidden_states, output_states + + +class FlaxCrossAttnUpBlock2D(nn.Module): + r""" + Cross Attention 2D Upsampling block - original architecture from Unet transformers: + https://arxiv.org/abs/2103.06104 + + Parameters: + in_channels (:obj:`int`): + Input channels + out_channels (:obj:`int`): + Output channels + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + num_layers (:obj:`int`, *optional*, defaults to 1): + Number of attention blocks layers + num_attention_heads (:obj:`int`, *optional*, defaults to 1): + Number of attention heads of each spatial transformer block + add_upsample (:obj:`bool`, *optional*, defaults to `True`): + Whether to add upsampling layer before each final output + use_memory_efficient_attention (`bool`, *optional*, defaults to `False`): + enable memory efficient attention https://arxiv.org/abs/2112.05682 + split_head_dim (`bool`, *optional*, defaults to `False`): + Whether to split the head dimension into a new axis for the self-attention computation. In most cases, + enabling this flag should speed up the computation for Stable Diffusion 2.x and Stable Diffusion XL. + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int + out_channels: int + prev_output_channel: int + dropout: float = 0.0 + num_layers: int = 1 + num_attention_heads: int = 1 + add_upsample: bool = True + use_linear_projection: bool = False + only_cross_attention: bool = False + use_memory_efficient_attention: bool = False + split_head_dim: bool = False + dtype: jnp.dtype = jnp.float32 + transformer_layers_per_block: int = 1 + + def setup(self): + resnets = [] + attentions = [] + + for i in range(self.num_layers): + res_skip_channels = self.in_channels if (i == self.num_layers - 1) else self.out_channels + resnet_in_channels = self.prev_output_channel if i == 0 else self.out_channels + + res_block = FlaxResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=self.out_channels, + dropout_prob=self.dropout, + dtype=self.dtype, + ) + resnets.append(res_block) + + attn_block = FlaxTransformer2DModel( + in_channels=self.out_channels, + n_heads=self.num_attention_heads, + d_head=self.out_channels // self.num_attention_heads, + depth=self.transformer_layers_per_block, + use_linear_projection=self.use_linear_projection, + only_cross_attention=self.only_cross_attention, + use_memory_efficient_attention=self.use_memory_efficient_attention, + split_head_dim=self.split_head_dim, + dtype=self.dtype, + ) + attentions.append(attn_block) + + self.resnets = resnets + self.attentions = attentions + + if self.add_upsample: + self.upsamplers_0 = FlaxUpsample2D(self.out_channels, dtype=self.dtype) + + def __call__(self, hidden_states, res_hidden_states_tuple, temb, encoder_hidden_states, deterministic=True): + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = jnp.concatenate((hidden_states, res_hidden_states), axis=-1) + + hidden_states = resnet(hidden_states, temb, deterministic=deterministic) + hidden_states = attn(hidden_states, encoder_hidden_states, deterministic=deterministic) + + if self.add_upsample: + hidden_states = self.upsamplers_0(hidden_states) + + return hidden_states + + +class FlaxUpBlock2D(nn.Module): + r""" + Flax 2D upsampling block + + Parameters: + in_channels (:obj:`int`): + Input channels + out_channels (:obj:`int`): + Output channels + prev_output_channel (:obj:`int`): + Output channels from the previous block + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + num_layers (:obj:`int`, *optional*, defaults to 1): + Number of attention blocks layers + add_downsample (:obj:`bool`, *optional*, defaults to `True`): + Whether to add downsampling layer before each final output + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int + out_channels: int + prev_output_channel: int + dropout: float = 0.0 + num_layers: int = 1 + add_upsample: bool = True + dtype: jnp.dtype = jnp.float32 + + def setup(self): + resnets = [] + + for i in range(self.num_layers): + res_skip_channels = self.in_channels if (i == self.num_layers - 1) else self.out_channels + resnet_in_channels = self.prev_output_channel if i == 0 else self.out_channels + + res_block = FlaxResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=self.out_channels, + dropout_prob=self.dropout, + dtype=self.dtype, + ) + resnets.append(res_block) + + self.resnets = resnets + + if self.add_upsample: + self.upsamplers_0 = FlaxUpsample2D(self.out_channels, dtype=self.dtype) + + def __call__(self, hidden_states, res_hidden_states_tuple, temb, deterministic=True): + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = jnp.concatenate((hidden_states, res_hidden_states), axis=-1) + + hidden_states = resnet(hidden_states, temb, deterministic=deterministic) + + if self.add_upsample: + hidden_states = self.upsamplers_0(hidden_states) + + return hidden_states + + +class FlaxUNetMidBlock2DCrossAttn(nn.Module): + r""" + Cross Attention 2D Mid-level block - original architecture from Unet transformers: https://arxiv.org/abs/2103.06104 + + Parameters: + in_channels (:obj:`int`): + Input channels + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + num_layers (:obj:`int`, *optional*, defaults to 1): + Number of attention blocks layers + num_attention_heads (:obj:`int`, *optional*, defaults to 1): + Number of attention heads of each spatial transformer block + use_memory_efficient_attention (`bool`, *optional*, defaults to `False`): + enable memory efficient attention https://arxiv.org/abs/2112.05682 + split_head_dim (`bool`, *optional*, defaults to `False`): + Whether to split the head dimension into a new axis for the self-attention computation. In most cases, + enabling this flag should speed up the computation for Stable Diffusion 2.x and Stable Diffusion XL. + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int + dropout: float = 0.0 + num_layers: int = 1 + num_attention_heads: int = 1 + use_linear_projection: bool = False + use_memory_efficient_attention: bool = False + split_head_dim: bool = False + dtype: jnp.dtype = jnp.float32 + transformer_layers_per_block: int = 1 + + def setup(self): + # there is always at least one resnet + resnets = [ + FlaxResnetBlock2D( + in_channels=self.in_channels, + out_channels=self.in_channels, + dropout_prob=self.dropout, + dtype=self.dtype, + ) + ] + + attentions = [] + + for _ in range(self.num_layers): + attn_block = FlaxTransformer2DModel( + in_channels=self.in_channels, + n_heads=self.num_attention_heads, + d_head=self.in_channels // self.num_attention_heads, + depth=self.transformer_layers_per_block, + use_linear_projection=self.use_linear_projection, + use_memory_efficient_attention=self.use_memory_efficient_attention, + split_head_dim=self.split_head_dim, + dtype=self.dtype, + ) + attentions.append(attn_block) + + res_block = FlaxResnetBlock2D( + in_channels=self.in_channels, + out_channels=self.in_channels, + dropout_prob=self.dropout, + dtype=self.dtype, + ) + resnets.append(res_block) + + self.resnets = resnets + self.attentions = attentions + + def __call__(self, hidden_states, temb, encoder_hidden_states, deterministic=True): + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + hidden_states = attn(hidden_states, encoder_hidden_states, deterministic=deterministic) + hidden_states = resnet(hidden_states, temb, deterministic=deterministic) + + return hidden_states diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_condition.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_condition.py new file mode 100755 index 0000000..9f69b03 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_condition.py @@ -0,0 +1,1315 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.utils.checkpoint + +from ...configuration_utils import ConfigMixin, register_to_config +from ...loaders import PeftAdapterMixin, UNet2DConditionLoadersMixin +from ...utils import USE_PEFT_BACKEND, BaseOutput, deprecate, logging, scale_lora_layers, unscale_lora_layers +from ..activations import get_activation +from ..attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + Attention, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from ..embeddings import ( + GaussianFourierProjection, + GLIGENTextBoundingboxProjection, + ImageHintTimeEmbedding, + ImageProjection, + ImageTimeEmbedding, + TextImageProjection, + TextImageTimeEmbedding, + TextTimeEmbedding, + TimestepEmbedding, + Timesteps, +) +from ..modeling_utils import ModelMixin +from .unet_2d_blocks import ( + get_down_block, + get_mid_block, + get_up_block, +) + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class UNet2DConditionOutput(BaseOutput): + """ + The output of [`UNet2DConditionModel`]. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + The hidden states output conditioned on `encoder_hidden_states` input. Output of last layer of model. + """ + + sample: torch.FloatTensor = None + + +class UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin, PeftAdapterMixin): + r""" + A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample + shaped output. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + Height and width of input/output sample. + in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 4): Number of channels in the output. + center_input_sample (`bool`, *optional*, defaults to `False`): Whether to center the input sample. + flip_sin_to_cos (`bool`, *optional*, defaults to `True`): + Whether to flip the sin to cos in the time embedding. + freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding. + down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): + The tuple of downsample blocks to use. + mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`): + Block type for middle of UNet, it can be one of `UNetMidBlock2DCrossAttn`, `UNetMidBlock2D`, or + `UNetMidBlock2DSimpleCrossAttn`. If `None`, the mid block layer is skipped. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): + The tuple of upsample blocks to use. + only_cross_attention(`bool` or `Tuple[bool]`, *optional*, default to `False`): + Whether to include self-attention in the basic transformer blocks, see + [`~models.attention.BasicTransformerBlock`]. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. + downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution. + mid_block_scale_factor (`float`, *optional*, defaults to 1.0): The scale factor to use for the mid block. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. + If `None`, normalization and activation layers is skipped in post-processing. + norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization. + cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280): + The dimension of the cross attention features. + transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple]` , *optional*, defaults to 1): + The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for + [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], + [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. + reverse_transformer_layers_per_block : (`Tuple[Tuple]`, *optional*, defaults to None): + The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`], in the upsampling + blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `Tuple[Tuple]` and for + [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], + [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. + encoder_hid_dim (`int`, *optional*, defaults to None): + If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim` + dimension to `cross_attention_dim`. + encoder_hid_dim_type (`str`, *optional*, defaults to `None`): + If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text + embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`. + attention_head_dim (`int`, *optional*, defaults to 8): The dimension of the attention heads. + num_attention_heads (`int`, *optional*): + The number of attention heads. If not defined, defaults to `attention_head_dim` + resnet_time_scale_shift (`str`, *optional*, defaults to `"default"`): Time scale shift config + for ResNet blocks (see [`~models.resnet.ResnetBlock2D`]). Choose from `default` or `scale_shift`. + class_embed_type (`str`, *optional*, defaults to `None`): + The type of class embedding to use which is ultimately summed with the time embeddings. Choose from `None`, + `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`. + addition_embed_type (`str`, *optional*, defaults to `None`): + Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or + "text". "text" will use the `TextTimeEmbedding` layer. + addition_time_embed_dim: (`int`, *optional*, defaults to `None`): + Dimension for the timestep embeddings. + num_class_embeds (`int`, *optional*, defaults to `None`): + Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing + class conditioning with `class_embed_type` equal to `None`. + time_embedding_type (`str`, *optional*, defaults to `positional`): + The type of position embedding to use for timesteps. Choose from `positional` or `fourier`. + time_embedding_dim (`int`, *optional*, defaults to `None`): + An optional override for the dimension of the projected time embedding. + time_embedding_act_fn (`str`, *optional*, defaults to `None`): + Optional activation function to use only once on the time embeddings before they are passed to the rest of + the UNet. Choose from `silu`, `mish`, `gelu`, and `swish`. + timestep_post_act (`str`, *optional*, defaults to `None`): + The second activation function to use in timestep embedding. Choose from `silu`, `mish` and `gelu`. + time_cond_proj_dim (`int`, *optional*, defaults to `None`): + The dimension of `cond_proj` layer in the timestep embedding. + conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer. + conv_out_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_out` layer. + projection_class_embeddings_input_dim (`int`, *optional*): The dimension of the `class_labels` input when + `class_embed_type="projection"`. Required when `class_embed_type="projection"`. + class_embeddings_concat (`bool`, *optional*, defaults to `False`): Whether to concatenate the time + embeddings with the class embeddings. + mid_block_only_cross_attention (`bool`, *optional*, defaults to `None`): + Whether to use cross attention with the mid block when using the `UNetMidBlock2DSimpleCrossAttn`. If + `only_cross_attention` is given as a single boolean and `mid_block_only_cross_attention` is `None`, the + `only_cross_attention` value is used as the value for `mid_block_only_cross_attention`. Default to `False` + otherwise. + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + sample_size: Optional[int] = None, + in_channels: int = 4, + out_channels: int = 4, + center_input_sample: bool = False, + flip_sin_to_cos: bool = True, + freq_shift: int = 0, + down_block_types: Tuple[str] = ( + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "DownBlock2D", + ), + mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn", + up_block_types: Tuple[str] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + only_cross_attention: Union[bool, Tuple[bool]] = False, + block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + layers_per_block: Union[int, Tuple[int]] = 2, + downsample_padding: int = 1, + mid_block_scale_factor: float = 1, + dropout: float = 0.0, + act_fn: str = "silu", + norm_num_groups: Optional[int] = 32, + norm_eps: float = 1e-5, + cross_attention_dim: Union[int, Tuple[int]] = 1280, + transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1, + reverse_transformer_layers_per_block: Optional[Tuple[Tuple[int]]] = None, + encoder_hid_dim: Optional[int] = None, + encoder_hid_dim_type: Optional[str] = None, + attention_head_dim: Union[int, Tuple[int]] = 8, + num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + class_embed_type: Optional[str] = None, + addition_embed_type: Optional[str] = None, + addition_time_embed_dim: Optional[int] = None, + num_class_embeds: Optional[int] = None, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + resnet_skip_time_act: bool = False, + resnet_out_scale_factor: float = 1.0, + time_embedding_type: str = "positional", + time_embedding_dim: Optional[int] = None, + time_embedding_act_fn: Optional[str] = None, + timestep_post_act: Optional[str] = None, + time_cond_proj_dim: Optional[int] = None, + conv_in_kernel: int = 3, + conv_out_kernel: int = 3, + projection_class_embeddings_input_dim: Optional[int] = None, + attention_type: str = "default", + class_embeddings_concat: bool = False, + mid_block_only_cross_attention: Optional[bool] = None, + cross_attention_norm: Optional[str] = None, + addition_embed_type_num_heads: int = 64, + ): + super().__init__() + + self.sample_size = sample_size + + if num_attention_heads is not None: + raise ValueError( + "At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19." + ) + + # If `num_attention_heads` is not defined (which is the case for most models) + # it will default to `attention_head_dim`. This looks weird upon first reading it and it is. + # The reason for this behavior is to correct for incorrectly named variables that were introduced + # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131 + # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking + # which is why we correct for the naming here. + num_attention_heads = num_attention_heads or attention_head_dim + + # Check inputs + self._check_config( + down_block_types=down_block_types, + up_block_types=up_block_types, + only_cross_attention=only_cross_attention, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + cross_attention_dim=cross_attention_dim, + transformer_layers_per_block=transformer_layers_per_block, + reverse_transformer_layers_per_block=reverse_transformer_layers_per_block, + attention_head_dim=attention_head_dim, + num_attention_heads=num_attention_heads, + ) + + # input + conv_in_padding = (conv_in_kernel - 1) // 2 + self.conv_in = nn.Conv2d( + in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding + ) + + # time + time_embed_dim, timestep_input_dim = self._set_time_proj( + time_embedding_type, + block_out_channels=block_out_channels, + flip_sin_to_cos=flip_sin_to_cos, + freq_shift=freq_shift, + time_embedding_dim=time_embedding_dim, + ) + + self.time_embedding = TimestepEmbedding( + timestep_input_dim, + time_embed_dim, + act_fn=act_fn, + post_act_fn=timestep_post_act, + cond_proj_dim=time_cond_proj_dim, + ) + + self._set_encoder_hid_proj( + encoder_hid_dim_type, + cross_attention_dim=cross_attention_dim, + encoder_hid_dim=encoder_hid_dim, + ) + + # class embedding + self._set_class_embedding( + class_embed_type, + act_fn=act_fn, + num_class_embeds=num_class_embeds, + projection_class_embeddings_input_dim=projection_class_embeddings_input_dim, + time_embed_dim=time_embed_dim, + timestep_input_dim=timestep_input_dim, + ) + + self._set_add_embedding( + addition_embed_type, + addition_embed_type_num_heads=addition_embed_type_num_heads, + addition_time_embed_dim=addition_time_embed_dim, + cross_attention_dim=cross_attention_dim, + encoder_hid_dim=encoder_hid_dim, + flip_sin_to_cos=flip_sin_to_cos, + freq_shift=freq_shift, + projection_class_embeddings_input_dim=projection_class_embeddings_input_dim, + time_embed_dim=time_embed_dim, + ) + + if time_embedding_act_fn is None: + self.time_embed_act = None + else: + self.time_embed_act = get_activation(time_embedding_act_fn) + + self.down_blocks = nn.ModuleList([]) + self.up_blocks = nn.ModuleList([]) + + if isinstance(only_cross_attention, bool): + if mid_block_only_cross_attention is None: + mid_block_only_cross_attention = only_cross_attention + + only_cross_attention = [only_cross_attention] * len(down_block_types) + + if mid_block_only_cross_attention is None: + mid_block_only_cross_attention = False + + if isinstance(num_attention_heads, int): + num_attention_heads = (num_attention_heads,) * len(down_block_types) + + if isinstance(attention_head_dim, int): + attention_head_dim = (attention_head_dim,) * len(down_block_types) + + if isinstance(cross_attention_dim, int): + cross_attention_dim = (cross_attention_dim,) * len(down_block_types) + + if isinstance(layers_per_block, int): + layers_per_block = [layers_per_block] * len(down_block_types) + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * len(down_block_types) + + if class_embeddings_concat: + # The time embeddings are concatenated with the class embeddings. The dimension of the + # time embeddings passed to the down, middle, and up blocks is twice the dimension of the + # regular time embeddings + blocks_time_embed_dim = time_embed_dim * 2 + else: + blocks_time_embed_dim = time_embed_dim + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block[i], + transformer_layers_per_block=transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + temb_channels=blocks_time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim[i], + num_attention_heads=num_attention_heads[i], + downsample_padding=downsample_padding, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + attention_head_dim=attention_head_dim[i] if attention_head_dim[i] is not None else output_channel, + dropout=dropout, + ) + self.down_blocks.append(down_block) + + # mid + self.mid_block = get_mid_block( + mid_block_type, + temb_channels=blocks_time_embed_dim, + in_channels=block_out_channels[-1], + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + output_scale_factor=mid_block_scale_factor, + transformer_layers_per_block=transformer_layers_per_block[-1], + num_attention_heads=num_attention_heads[-1], + cross_attention_dim=cross_attention_dim[-1], + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + mid_block_only_cross_attention=mid_block_only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + resnet_skip_time_act=resnet_skip_time_act, + cross_attention_norm=cross_attention_norm, + attention_head_dim=attention_head_dim[-1], + dropout=dropout, + ) + + # count how many layers upsample the images + self.num_upsamplers = 0 + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_num_attention_heads = list(reversed(num_attention_heads)) + reversed_layers_per_block = list(reversed(layers_per_block)) + reversed_cross_attention_dim = list(reversed(cross_attention_dim)) + reversed_transformer_layers_per_block = ( + list(reversed(transformer_layers_per_block)) + if reverse_transformer_layers_per_block is None + else reverse_transformer_layers_per_block + ) + only_cross_attention = list(reversed(only_cross_attention)) + + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + is_final_block = i == len(block_out_channels) - 1 + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)] + + # add upsample block for all BUT final layer + if not is_final_block: + add_upsample = True + self.num_upsamplers += 1 + else: + add_upsample = False + + up_block = get_up_block( + up_block_type, + num_layers=reversed_layers_per_block[i] + 1, + transformer_layers_per_block=reversed_transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=blocks_time_embed_dim, + add_upsample=add_upsample, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resolution_idx=i, + resnet_groups=norm_num_groups, + cross_attention_dim=reversed_cross_attention_dim[i], + num_attention_heads=reversed_num_attention_heads[i], + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + attention_head_dim=attention_head_dim[i] if attention_head_dim[i] is not None else output_channel, + dropout=dropout, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_num_groups is not None: + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps + ) + + self.conv_act = get_activation(act_fn) + + else: + self.conv_norm_out = None + self.conv_act = None + + conv_out_padding = (conv_out_kernel - 1) // 2 + self.conv_out = nn.Conv2d( + block_out_channels[0], out_channels, kernel_size=conv_out_kernel, padding=conv_out_padding + ) + + self._set_pos_net_if_use_gligen(attention_type=attention_type, cross_attention_dim=cross_attention_dim) + + def _check_config( + self, + down_block_types: Tuple[str], + up_block_types: Tuple[str], + only_cross_attention: Union[bool, Tuple[bool]], + block_out_channels: Tuple[int], + layers_per_block: Union[int, Tuple[int]], + cross_attention_dim: Union[int, Tuple[int]], + transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple[int]]], + reverse_transformer_layers_per_block: bool, + attention_head_dim: int, + num_attention_heads: Optional[Union[int, Tuple[int]]], + ): + if len(down_block_types) != len(up_block_types): + raise ValueError( + f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}." + ) + + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(only_cross_attention, bool) and len(only_cross_attention) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(attention_head_dim, int) and len(attention_head_dim) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `attention_head_dim` as `down_block_types`. `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}." + ) + + if isinstance(cross_attention_dim, list) and len(cross_attention_dim) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `cross_attention_dim` as `down_block_types`. `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(layers_per_block, int) and len(layers_per_block) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `layers_per_block` as `down_block_types`. `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}." + ) + if isinstance(transformer_layers_per_block, list) and reverse_transformer_layers_per_block is None: + for layer_number_per_block in transformer_layers_per_block: + if isinstance(layer_number_per_block, list): + raise ValueError("Must provide 'reverse_transformer_layers_per_block` if using asymmetrical UNet.") + + def _set_time_proj( + self, + time_embedding_type: str, + block_out_channels: int, + flip_sin_to_cos: bool, + freq_shift: float, + time_embedding_dim: int, + ) -> Tuple[int, int]: + if time_embedding_type == "fourier": + time_embed_dim = time_embedding_dim or block_out_channels[0] * 2 + if time_embed_dim % 2 != 0: + raise ValueError(f"`time_embed_dim` should be divisible by 2, but is {time_embed_dim}.") + self.time_proj = GaussianFourierProjection( + time_embed_dim // 2, set_W_to_weight=False, log=False, flip_sin_to_cos=flip_sin_to_cos + ) + timestep_input_dim = time_embed_dim + elif time_embedding_type == "positional": + time_embed_dim = time_embedding_dim or block_out_channels[0] * 4 + + self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift) + timestep_input_dim = block_out_channels[0] + else: + raise ValueError( + f"{time_embedding_type} does not exist. Please make sure to use one of `fourier` or `positional`." + ) + + return time_embed_dim, timestep_input_dim + + def _set_encoder_hid_proj( + self, + encoder_hid_dim_type: Optional[str], + cross_attention_dim: Union[int, Tuple[int]], + encoder_hid_dim: Optional[int], + ): + if encoder_hid_dim_type is None and encoder_hid_dim is not None: + encoder_hid_dim_type = "text_proj" + self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type) + logger.info("encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined.") + + if encoder_hid_dim is None and encoder_hid_dim_type is not None: + raise ValueError( + f"`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}." + ) + + if encoder_hid_dim_type == "text_proj": + self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim) + elif encoder_hid_dim_type == "text_image_proj": + # image_embed_dim DOESN'T have to be `cross_attention_dim`. To not clutter the __init__ too much + # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use + # case when `addition_embed_type == "text_image_proj"` (Kadinsky 2.1)` + self.encoder_hid_proj = TextImageProjection( + text_embed_dim=encoder_hid_dim, + image_embed_dim=cross_attention_dim, + cross_attention_dim=cross_attention_dim, + ) + elif encoder_hid_dim_type == "image_proj": + # Kandinsky 2.2 + self.encoder_hid_proj = ImageProjection( + image_embed_dim=encoder_hid_dim, + cross_attention_dim=cross_attention_dim, + ) + elif encoder_hid_dim_type is not None: + raise ValueError( + f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'." + ) + else: + self.encoder_hid_proj = None + + def _set_class_embedding( + self, + class_embed_type: Optional[str], + act_fn: str, + num_class_embeds: Optional[int], + projection_class_embeddings_input_dim: Optional[int], + time_embed_dim: int, + timestep_input_dim: int, + ): + if class_embed_type is None and num_class_embeds is not None: + self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim) + elif class_embed_type == "timestep": + self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim, act_fn=act_fn) + elif class_embed_type == "identity": + self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim) + elif class_embed_type == "projection": + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set" + ) + # The projection `class_embed_type` is the same as the timestep `class_embed_type` except + # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings + # 2. it projects from an arbitrary input dimension. + # + # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations. + # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings. + # As a result, `TimestepEmbedding` can be passed arbitrary vectors. + self.class_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim) + elif class_embed_type == "simple_projection": + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'simple_projection' requires `projection_class_embeddings_input_dim` be set" + ) + self.class_embedding = nn.Linear(projection_class_embeddings_input_dim, time_embed_dim) + else: + self.class_embedding = None + + def _set_add_embedding( + self, + addition_embed_type: str, + addition_embed_type_num_heads: int, + addition_time_embed_dim: Optional[int], + flip_sin_to_cos: bool, + freq_shift: float, + cross_attention_dim: Optional[int], + encoder_hid_dim: Optional[int], + projection_class_embeddings_input_dim: Optional[int], + time_embed_dim: int, + ): + if addition_embed_type == "text": + if encoder_hid_dim is not None: + text_time_embedding_from_dim = encoder_hid_dim + else: + text_time_embedding_from_dim = cross_attention_dim + + self.add_embedding = TextTimeEmbedding( + text_time_embedding_from_dim, time_embed_dim, num_heads=addition_embed_type_num_heads + ) + elif addition_embed_type == "text_image": + # text_embed_dim and image_embed_dim DON'T have to be `cross_attention_dim`. To not clutter the __init__ too much + # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use + # case when `addition_embed_type == "text_image"` (Kadinsky 2.1)` + self.add_embedding = TextImageTimeEmbedding( + text_embed_dim=cross_attention_dim, image_embed_dim=cross_attention_dim, time_embed_dim=time_embed_dim + ) + elif addition_embed_type == "text_time": + self.add_time_proj = Timesteps(addition_time_embed_dim, flip_sin_to_cos, freq_shift) + self.add_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim) + elif addition_embed_type == "image": + # Kandinsky 2.2 + self.add_embedding = ImageTimeEmbedding(image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim) + elif addition_embed_type == "image_hint": + # Kandinsky 2.2 ControlNet + self.add_embedding = ImageHintTimeEmbedding(image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim) + elif addition_embed_type is not None: + raise ValueError(f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'.") + + def _set_pos_net_if_use_gligen(self, attention_type: str, cross_attention_dim: int): + if attention_type in ["gated", "gated-text-image"]: + positive_len = 768 + if isinstance(cross_attention_dim, int): + positive_len = cross_attention_dim + elif isinstance(cross_attention_dim, tuple) or isinstance(cross_attention_dim, list): + positive_len = cross_attention_dim[0] + + feature_type = "text-only" if attention_type == "gated" else "text-image" + self.position_net = GLIGENTextBoundingboxProjection( + positive_len=positive_len, out_dim=cross_attention_dim, feature_type=feature_type + ) + + @property + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + def set_attention_slice(self, slice_size: Union[str, int, List[int]] = "auto"): + r""" + Enable sliced attention computation. + + When this option is enabled, the attention module splits the input tensor in slices to compute attention in + several steps. This is useful for saving some memory in exchange for a small decrease in speed. + + Args: + slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`): + When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If + `"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is + provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim` + must be a multiple of `slice_size`. + """ + sliceable_head_dims = [] + + def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): + if hasattr(module, "set_attention_slice"): + sliceable_head_dims.append(module.sliceable_head_dim) + + for child in module.children(): + fn_recursive_retrieve_sliceable_dims(child) + + # retrieve number of attention layers + for module in self.children(): + fn_recursive_retrieve_sliceable_dims(module) + + num_sliceable_layers = len(sliceable_head_dims) + + if slice_size == "auto": + # half the attention head size is usually a good trade-off between + # speed and memory + slice_size = [dim // 2 for dim in sliceable_head_dims] + elif slice_size == "max": + # make smallest slice possible + slice_size = num_sliceable_layers * [1] + + slice_size = num_sliceable_layers * [slice_size] if not isinstance(slice_size, list) else slice_size + + if len(slice_size) != len(sliceable_head_dims): + raise ValueError( + f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different" + f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}." + ) + + for i in range(len(slice_size)): + size = slice_size[i] + dim = sliceable_head_dims[i] + if size is not None and size > dim: + raise ValueError(f"size {size} has to be smaller or equal to {dim}.") + + # Recursively walk through all the children. + # Any children which exposes the set_attention_slice method + # gets the message + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + if hasattr(module, "set_attention_slice"): + module.set_attention_slice(slice_size.pop()) + + for child in module.children(): + fn_recursive_set_attention_slice(child, slice_size) + + reversed_slice_size = list(reversed(slice_size)) + for module in self.children(): + fn_recursive_set_attention_slice(module, reversed_slice_size) + + def _set_gradient_checkpointing(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + + def enable_freeu(self, s1: float, s2: float, b1: float, b2: float): + r"""Enables the FreeU mechanism from https://arxiv.org/abs/2309.11497. + + The suffixes after the scaling factors represent the stage blocks where they are being applied. + + Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of values that + are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL. + + Args: + s1 (`float`): + Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + s2 (`float`): + Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. + b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. + """ + for i, upsample_block in enumerate(self.up_blocks): + setattr(upsample_block, "s1", s1) + setattr(upsample_block, "s2", s2) + setattr(upsample_block, "b1", b1) + setattr(upsample_block, "b2", b2) + + def disable_freeu(self): + """Disables the FreeU mechanism.""" + freeu_keys = {"s1", "s2", "b1", "b2"} + for i, upsample_block in enumerate(self.up_blocks): + for k in freeu_keys: + if hasattr(upsample_block, k) or getattr(upsample_block, k, None) is not None: + setattr(upsample_block, k, None) + + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, + key, value) are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is 🧪 experimental. + + + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.") + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + def unfuse_qkv_projections(self): + """Disables the fused QKV projection if enabled. + + + + This API is 🧪 experimental. + + + + """ + if self.original_attn_processors is not None: + self.set_attn_processor(self.original_attn_processors) + + def unload_lora(self): + """Unloads LoRA weights.""" + deprecate( + "unload_lora", + "0.28.0", + "Calling `unload_lora()` is deprecated and will be removed in a future version. Please install `peft` and then call `disable_adapters().", + ) + for module in self.modules(): + if hasattr(module, "set_lora_layer"): + module.set_lora_layer(None) + + def get_time_embed( + self, sample: torch.Tensor, timestep: Union[torch.Tensor, float, int] + ) -> Optional[torch.Tensor]: + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == "mps" + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + # `Timesteps` does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=sample.dtype) + return t_emb + + def get_class_embed(self, sample: torch.Tensor, class_labels: Optional[torch.Tensor]) -> Optional[torch.Tensor]: + class_emb = None + if self.class_embedding is not None: + if class_labels is None: + raise ValueError("class_labels should be provided when num_class_embeds > 0") + + if self.config.class_embed_type == "timestep": + class_labels = self.time_proj(class_labels) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # there might be better ways to encapsulate this. + class_labels = class_labels.to(dtype=sample.dtype) + + class_emb = self.class_embedding(class_labels).to(dtype=sample.dtype) + return class_emb + + def get_aug_embed( + self, emb: torch.Tensor, encoder_hidden_states: torch.Tensor, added_cond_kwargs: Dict[str, Any] + ) -> Optional[torch.Tensor]: + aug_emb = None + if self.config.addition_embed_type == "text": + aug_emb = self.add_embedding(encoder_hidden_states) + elif self.config.addition_embed_type == "text_image": + # Kandinsky 2.1 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`" + ) + + image_embs = added_cond_kwargs.get("image_embeds") + text_embs = added_cond_kwargs.get("text_embeds", encoder_hidden_states) + aug_emb = self.add_embedding(text_embs, image_embs) + elif self.config.addition_embed_type == "text_time": + # SDXL - style + if "text_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`" + ) + text_embeds = added_cond_kwargs.get("text_embeds") + if "time_ids" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`" + ) + time_ids = added_cond_kwargs.get("time_ids") + time_embeds = self.add_time_proj(time_ids.flatten()) + time_embeds = time_embeds.reshape((text_embeds.shape[0], -1)) + add_embeds = torch.concat([text_embeds, time_embeds], dim=-1) + add_embeds = add_embeds.to(emb.dtype) + aug_emb = self.add_embedding(add_embeds) + elif self.config.addition_embed_type == "image": + # Kandinsky 2.2 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`" + ) + image_embs = added_cond_kwargs.get("image_embeds") + aug_emb = self.add_embedding(image_embs) + elif self.config.addition_embed_type == "image_hint": + # Kandinsky 2.2 - style + if "image_embeds" not in added_cond_kwargs or "hint" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'image_hint' which requires the keyword arguments `image_embeds` and `hint` to be passed in `added_cond_kwargs`" + ) + image_embs = added_cond_kwargs.get("image_embeds") + hint = added_cond_kwargs.get("hint") + aug_emb = self.add_embedding(image_embs, hint) + return aug_emb + + def process_encoder_hidden_states( + self, encoder_hidden_states: torch.Tensor, added_cond_kwargs: Dict[str, Any] + ) -> torch.Tensor: + if self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "text_proj": + encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states) + elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "text_image_proj": + # Kadinsky 2.1 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'text_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + + image_embeds = added_cond_kwargs.get("image_embeds") + encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states, image_embeds) + elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "image_proj": + # Kandinsky 2.2 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + image_embeds = added_cond_kwargs.get("image_embeds") + encoder_hidden_states = self.encoder_hid_proj(image_embeds) + elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "ip_image_proj": + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'ip_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + image_embeds = added_cond_kwargs.get("image_embeds") + image_embeds = self.encoder_hid_proj(image_embeds) + encoder_hidden_states = (encoder_hidden_states, image_embeds) + return encoder_hidden_states + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + mid_block_additional_residual: Optional[torch.Tensor] = None, + down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[UNet2DConditionOutput, Tuple]: + r""" + The [`UNet2DConditionModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor with the following shape `(batch, channel, height, width)`. + timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input. + encoder_hidden_states (`torch.FloatTensor`): + The encoder hidden states with shape `(batch, sequence_length, feature_dim)`. + class_labels (`torch.Tensor`, *optional*, defaults to `None`): + Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings. + timestep_cond: (`torch.Tensor`, *optional*, defaults to `None`): + Conditional embeddings for timestep. If provided, the embeddings will be summed with the samples passed + through the `self.time_embedding` layer to obtain the timestep embeddings. + attention_mask (`torch.Tensor`, *optional*, defaults to `None`): + An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask + is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large + negative values to the attention scores corresponding to "discard" tokens. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + added_cond_kwargs: (`dict`, *optional*): + A kwargs dictionary containing additional embeddings that if specified are added to the embeddings that + are passed along to the UNet blocks. + down_block_additional_residuals: (`tuple` of `torch.Tensor`, *optional*): + A tuple of tensors that if specified are added to the residuals of down unet blocks. + mid_block_additional_residual: (`torch.Tensor`, *optional*): + A tensor that if specified is added to the residual of the middle unet block. + encoder_attention_mask (`torch.Tensor`): + A cross-attention mask of shape `(batch, sequence_length)` is applied to `encoder_hidden_states`. If + `True` the mask is kept, otherwise if `False` it is discarded. Mask will be converted into a bias, + which adds large negative values to the attention scores corresponding to "discard" tokens. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] instead of a plain + tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttnProcessor`]. + added_cond_kwargs: (`dict`, *optional*): + A kwargs dictionary containin additional embeddings that if specified are added to the embeddings that + are passed along to the UNet blocks. + down_block_additional_residuals (`tuple` of `torch.Tensor`, *optional*): + additional residuals to be added to UNet long skip connections from down blocks to up blocks for + example from ControlNet side model(s) + mid_block_additional_residual (`torch.Tensor`, *optional*): + additional residual to be added to UNet mid block output, for example from ControlNet side model + down_intrablock_additional_residuals (`tuple` of `torch.Tensor`, *optional*): + additional residuals to be added within UNet down blocks, for example from T2I-Adapter side model(s) + + Returns: + [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] or `tuple`: + If `return_dict` is True, an [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] is returned, otherwise + a `tuple` is returned where the first element is the sample tensor. + """ + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layers). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2**self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + for dim in sample.shape[-2:]: + if dim % default_overall_up_factor != 0: + # Forward upsample size to force interpolation output size. + forward_upsample_size = True + break + + # ensure attention_mask is a bias, and give it a singleton query_tokens dimension + # expects mask of shape: + # [batch, key_tokens] + # adds singleton query_tokens dimension: + # [batch, 1, key_tokens] + # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes: + # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn) + # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn) + if attention_mask is not None: + # assume that mask is expressed as: + # (1 = keep, 0 = discard) + # convert mask into a bias that can be added to attention scores: + # (keep = +0, discard = -10000.0) + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # convert encoder_attention_mask to a bias the same way we do for attention_mask + if encoder_attention_mask is not None: + encoder_attention_mask = (1 - encoder_attention_mask.to(sample.dtype)) * -10000.0 + encoder_attention_mask = encoder_attention_mask.unsqueeze(1) + + # 0. center input if necessary + if self.config.center_input_sample: + sample = 2 * sample - 1.0 + + # 1. time + t_emb = self.get_time_embed(sample=sample, timestep=timestep) + emb = self.time_embedding(t_emb, timestep_cond) + aug_emb = None + + class_emb = self.get_class_embed(sample=sample, class_labels=class_labels) + if class_emb is not None: + if self.config.class_embeddings_concat: + emb = torch.cat([emb, class_emb], dim=-1) + else: + emb = emb + class_emb + + aug_emb = self.get_aug_embed( + emb=emb, encoder_hidden_states=encoder_hidden_states, added_cond_kwargs=added_cond_kwargs + ) + if self.config.addition_embed_type == "image_hint": + aug_emb, hint = aug_emb + sample = torch.cat([sample, hint], dim=1) + + emb = emb + aug_emb if aug_emb is not None else emb + + if self.time_embed_act is not None: + emb = self.time_embed_act(emb) + + encoder_hidden_states = self.process_encoder_hidden_states( + encoder_hidden_states=encoder_hidden_states, added_cond_kwargs=added_cond_kwargs + ) + + # 2. pre-process + sample = self.conv_in(sample) + + # 2.5 GLIGEN position net + if cross_attention_kwargs is not None and cross_attention_kwargs.get("gligen", None) is not None: + cross_attention_kwargs = cross_attention_kwargs.copy() + gligen_args = cross_attention_kwargs.pop("gligen") + cross_attention_kwargs["gligen"] = {"objs": self.position_net(**gligen_args)} + + # 3. down + lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + if USE_PEFT_BACKEND: + # weight the lora layers by setting `lora_scale` for each PEFT layer + scale_lora_layers(self, lora_scale) + + is_controlnet = mid_block_additional_residual is not None and down_block_additional_residuals is not None + # using new arg down_intrablock_additional_residuals for T2I-Adapters, to distinguish from controlnets + is_adapter = down_intrablock_additional_residuals is not None + # maintain backward compatibility for legacy usage, where + # T2I-Adapter and ControlNet both use down_block_additional_residuals arg + # but can only use one or the other + if not is_adapter and mid_block_additional_residual is None and down_block_additional_residuals is not None: + deprecate( + "T2I should not use down_block_additional_residuals", + "1.3.0", + "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated \ + and will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used \ + for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ", + standard_warn=False, + ) + down_intrablock_additional_residuals = down_block_additional_residuals + is_adapter = True + + down_block_res_samples = (sample,) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: + # For t2i-adapter CrossAttnDownBlock2D + additional_residuals = {} + if is_adapter and len(down_intrablock_additional_residuals) > 0: + additional_residuals["additional_residuals"] = down_intrablock_additional_residuals.pop(0) + + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + **additional_residuals, + ) + else: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb) + if is_adapter and len(down_intrablock_additional_residuals) > 0: + sample += down_intrablock_additional_residuals.pop(0) + + down_block_res_samples += res_samples + + if is_controlnet: + new_down_block_res_samples = () + + for down_block_res_sample, down_block_additional_residual in zip( + down_block_res_samples, down_block_additional_residuals + ): + down_block_res_sample = down_block_res_sample + down_block_additional_residual + new_down_block_res_samples = new_down_block_res_samples + (down_block_res_sample,) + + down_block_res_samples = new_down_block_res_samples + + # 4. mid + if self.mid_block is not None: + if hasattr(self.mid_block, "has_cross_attention") and self.mid_block.has_cross_attention: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + else: + sample = self.mid_block(sample, emb) + + # To support T2I-Adapter-XL + if ( + is_adapter + and len(down_intrablock_additional_residuals) > 0 + and sample.shape == down_intrablock_additional_residuals[0].shape + ): + sample += down_intrablock_additional_residuals.pop(0) + + if is_controlnet: + sample = sample + mid_block_additional_residual + + # 5. up + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets) :] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + upsample_size=upsample_size, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + ) + else: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + upsample_size=upsample_size, + ) + + # 6. post-process + if self.conv_norm_out: + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + if USE_PEFT_BACKEND: + # remove `lora_scale` from each PEFT layer + unscale_lora_layers(self, lora_scale) + + if not return_dict: + return (sample,) + + return UNet2DConditionOutput(sample=sample) diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_condition_flax.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_condition_flax.py new file mode 100755 index 0000000..a5ec287 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_2d_condition_flax.py @@ -0,0 +1,453 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Dict, Optional, Tuple, Union + +import flax +import flax.linen as nn +import jax +import jax.numpy as jnp +from flax.core.frozen_dict import FrozenDict + +from ...configuration_utils import ConfigMixin, flax_register_to_config +from ...utils import BaseOutput +from ..embeddings_flax import FlaxTimestepEmbedding, FlaxTimesteps +from ..modeling_flax_utils import FlaxModelMixin +from .unet_2d_blocks_flax import ( + FlaxCrossAttnDownBlock2D, + FlaxCrossAttnUpBlock2D, + FlaxDownBlock2D, + FlaxUNetMidBlock2DCrossAttn, + FlaxUpBlock2D, +) + + +@flax.struct.dataclass +class FlaxUNet2DConditionOutput(BaseOutput): + """ + The output of [`FlaxUNet2DConditionModel`]. + + Args: + sample (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)`): + The hidden states output conditioned on `encoder_hidden_states` input. Output of last layer of model. + """ + + sample: jnp.ndarray + + +@flax_register_to_config +class FlaxUNet2DConditionModel(nn.Module, FlaxModelMixin, ConfigMixin): + r""" + A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample + shaped output. + + This model inherits from [`FlaxModelMixin`]. Check the superclass documentation for it's generic methods + implemented for all models (such as downloading or saving). + + This model is also a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) + subclass. Use it as a regular Flax Linen module and refer to the Flax documentation for all matters related to its + general usage and behavior. + + Inherent JAX features such as the following are supported: + - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit) + - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation) + - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap) + - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap) + + Parameters: + sample_size (`int`, *optional*): + The size of the input sample. + in_channels (`int`, *optional*, defaults to 4): + The number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 4): + The number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxCrossAttnDownBlock2D", "FlaxDownBlock2D")`): + The tuple of downsample blocks to use. + up_block_types (`Tuple[str]`, *optional*, defaults to `("FlaxUpBlock2D", "FlaxCrossAttnUpBlock2D", "FlaxCrossAttnUpBlock2D", "FlaxCrossAttnUpBlock2D")`): + The tuple of upsample blocks to use. + mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`): + Block type for middle of UNet, it can be one of `UNetMidBlock2DCrossAttn`. If `None`, the mid block layer is skipped. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): + The number of layers per block. + attention_head_dim (`int` or `Tuple[int]`, *optional*, defaults to 8): + The dimension of the attention heads. + num_attention_heads (`int` or `Tuple[int]`, *optional*): + The number of attention heads. + cross_attention_dim (`int`, *optional*, defaults to 768): + The dimension of the cross attention features. + dropout (`float`, *optional*, defaults to 0): + Dropout probability for down, up and bottleneck blocks. + flip_sin_to_cos (`bool`, *optional*, defaults to `True`): + Whether to flip the sin to cos in the time embedding. + freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding. + use_memory_efficient_attention (`bool`, *optional*, defaults to `False`): + Enable memory efficient attention as described [here](https://arxiv.org/abs/2112.05682). + split_head_dim (`bool`, *optional*, defaults to `False`): + Whether to split the head dimension into a new axis for the self-attention computation. In most cases, + enabling this flag should speed up the computation for Stable Diffusion 2.x and Stable Diffusion XL. + """ + + sample_size: int = 32 + in_channels: int = 4 + out_channels: int = 4 + down_block_types: Tuple[str, ...] = ( + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "DownBlock2D", + ) + up_block_types: Tuple[str, ...] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D") + mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn" + only_cross_attention: Union[bool, Tuple[bool]] = False + block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280) + layers_per_block: int = 2 + attention_head_dim: Union[int, Tuple[int, ...]] = 8 + num_attention_heads: Optional[Union[int, Tuple[int, ...]]] = None + cross_attention_dim: int = 1280 + dropout: float = 0.0 + use_linear_projection: bool = False + dtype: jnp.dtype = jnp.float32 + flip_sin_to_cos: bool = True + freq_shift: int = 0 + use_memory_efficient_attention: bool = False + split_head_dim: bool = False + transformer_layers_per_block: Union[int, Tuple[int, ...]] = 1 + addition_embed_type: Optional[str] = None + addition_time_embed_dim: Optional[int] = None + addition_embed_type_num_heads: int = 64 + projection_class_embeddings_input_dim: Optional[int] = None + + def init_weights(self, rng: jax.Array) -> FrozenDict: + # init input tensors + sample_shape = (1, self.in_channels, self.sample_size, self.sample_size) + sample = jnp.zeros(sample_shape, dtype=jnp.float32) + timesteps = jnp.ones((1,), dtype=jnp.int32) + encoder_hidden_states = jnp.zeros((1, 1, self.cross_attention_dim), dtype=jnp.float32) + + params_rng, dropout_rng = jax.random.split(rng) + rngs = {"params": params_rng, "dropout": dropout_rng} + + added_cond_kwargs = None + if self.addition_embed_type == "text_time": + # we retrieve the expected `text_embeds_dim` by first checking if the architecture is a refiner + # or non-refiner architecture and then by "reverse-computing" from `projection_class_embeddings_input_dim` + is_refiner = ( + 5 * self.config.addition_time_embed_dim + self.config.cross_attention_dim + == self.config.projection_class_embeddings_input_dim + ) + num_micro_conditions = 5 if is_refiner else 6 + + text_embeds_dim = self.config.projection_class_embeddings_input_dim - ( + num_micro_conditions * self.config.addition_time_embed_dim + ) + + time_ids_channels = self.projection_class_embeddings_input_dim - text_embeds_dim + time_ids_dims = time_ids_channels // self.addition_time_embed_dim + added_cond_kwargs = { + "text_embeds": jnp.zeros((1, text_embeds_dim), dtype=jnp.float32), + "time_ids": jnp.zeros((1, time_ids_dims), dtype=jnp.float32), + } + return self.init(rngs, sample, timesteps, encoder_hidden_states, added_cond_kwargs)["params"] + + def setup(self) -> None: + block_out_channels = self.block_out_channels + time_embed_dim = block_out_channels[0] * 4 + + if self.num_attention_heads is not None: + raise ValueError( + "At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19." + ) + + # If `num_attention_heads` is not defined (which is the case for most models) + # it will default to `attention_head_dim`. This looks weird upon first reading it and it is. + # The reason for this behavior is to correct for incorrectly named variables that were introduced + # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131 + # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking + # which is why we correct for the naming here. + num_attention_heads = self.num_attention_heads or self.attention_head_dim + + # input + self.conv_in = nn.Conv( + block_out_channels[0], + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + # time + self.time_proj = FlaxTimesteps( + block_out_channels[0], flip_sin_to_cos=self.flip_sin_to_cos, freq_shift=self.config.freq_shift + ) + self.time_embedding = FlaxTimestepEmbedding(time_embed_dim, dtype=self.dtype) + + only_cross_attention = self.only_cross_attention + if isinstance(only_cross_attention, bool): + only_cross_attention = (only_cross_attention,) * len(self.down_block_types) + + if isinstance(num_attention_heads, int): + num_attention_heads = (num_attention_heads,) * len(self.down_block_types) + + # transformer layers per block + transformer_layers_per_block = self.transformer_layers_per_block + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * len(self.down_block_types) + + # addition embed types + if self.addition_embed_type is None: + self.add_embedding = None + elif self.addition_embed_type == "text_time": + if self.addition_time_embed_dim is None: + raise ValueError( + f"addition_embed_type {self.addition_embed_type} requires `addition_time_embed_dim` to not be None" + ) + self.add_time_proj = FlaxTimesteps(self.addition_time_embed_dim, self.flip_sin_to_cos, self.freq_shift) + self.add_embedding = FlaxTimestepEmbedding(time_embed_dim, dtype=self.dtype) + else: + raise ValueError(f"addition_embed_type: {self.addition_embed_type} must be None or `text_time`.") + + # down + down_blocks = [] + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(self.down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + if down_block_type == "CrossAttnDownBlock2D": + down_block = FlaxCrossAttnDownBlock2D( + in_channels=input_channel, + out_channels=output_channel, + dropout=self.dropout, + num_layers=self.layers_per_block, + transformer_layers_per_block=transformer_layers_per_block[i], + num_attention_heads=num_attention_heads[i], + add_downsample=not is_final_block, + use_linear_projection=self.use_linear_projection, + only_cross_attention=only_cross_attention[i], + use_memory_efficient_attention=self.use_memory_efficient_attention, + split_head_dim=self.split_head_dim, + dtype=self.dtype, + ) + else: + down_block = FlaxDownBlock2D( + in_channels=input_channel, + out_channels=output_channel, + dropout=self.dropout, + num_layers=self.layers_per_block, + add_downsample=not is_final_block, + dtype=self.dtype, + ) + + down_blocks.append(down_block) + self.down_blocks = down_blocks + + # mid + if self.config.mid_block_type == "UNetMidBlock2DCrossAttn": + self.mid_block = FlaxUNetMidBlock2DCrossAttn( + in_channels=block_out_channels[-1], + dropout=self.dropout, + num_attention_heads=num_attention_heads[-1], + transformer_layers_per_block=transformer_layers_per_block[-1], + use_linear_projection=self.use_linear_projection, + use_memory_efficient_attention=self.use_memory_efficient_attention, + split_head_dim=self.split_head_dim, + dtype=self.dtype, + ) + elif self.config.mid_block_type is None: + self.mid_block = None + else: + raise ValueError(f"Unexpected mid_block_type {self.config.mid_block_type}") + + # up + up_blocks = [] + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_num_attention_heads = list(reversed(num_attention_heads)) + only_cross_attention = list(reversed(only_cross_attention)) + output_channel = reversed_block_out_channels[0] + reversed_transformer_layers_per_block = list(reversed(transformer_layers_per_block)) + for i, up_block_type in enumerate(self.up_block_types): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)] + + is_final_block = i == len(block_out_channels) - 1 + + if up_block_type == "CrossAttnUpBlock2D": + up_block = FlaxCrossAttnUpBlock2D( + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + num_layers=self.layers_per_block + 1, + transformer_layers_per_block=reversed_transformer_layers_per_block[i], + num_attention_heads=reversed_num_attention_heads[i], + add_upsample=not is_final_block, + dropout=self.dropout, + use_linear_projection=self.use_linear_projection, + only_cross_attention=only_cross_attention[i], + use_memory_efficient_attention=self.use_memory_efficient_attention, + split_head_dim=self.split_head_dim, + dtype=self.dtype, + ) + else: + up_block = FlaxUpBlock2D( + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + num_layers=self.layers_per_block + 1, + add_upsample=not is_final_block, + dropout=self.dropout, + dtype=self.dtype, + ) + + up_blocks.append(up_block) + prev_output_channel = output_channel + self.up_blocks = up_blocks + + # out + self.conv_norm_out = nn.GroupNorm(num_groups=32, epsilon=1e-5) + self.conv_out = nn.Conv( + self.out_channels, + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + def __call__( + self, + sample: jnp.ndarray, + timesteps: Union[jnp.ndarray, float, int], + encoder_hidden_states: jnp.ndarray, + added_cond_kwargs: Optional[Union[Dict, FrozenDict]] = None, + down_block_additional_residuals: Optional[Tuple[jnp.ndarray, ...]] = None, + mid_block_additional_residual: Optional[jnp.ndarray] = None, + return_dict: bool = True, + train: bool = False, + ) -> Union[FlaxUNet2DConditionOutput, Tuple[jnp.ndarray]]: + r""" + Args: + sample (`jnp.ndarray`): (batch, channel, height, width) noisy inputs tensor + timestep (`jnp.ndarray` or `float` or `int`): timesteps + encoder_hidden_states (`jnp.ndarray`): (batch_size, sequence_length, hidden_size) encoder hidden states + added_cond_kwargs: (`dict`, *optional*): + A kwargs dictionary containing additional embeddings that if specified are added to the embeddings that + are passed along to the UNet blocks. + down_block_additional_residuals: (`tuple` of `torch.Tensor`, *optional*): + A tuple of tensors that if specified are added to the residuals of down unet blocks. + mid_block_additional_residual: (`torch.Tensor`, *optional*): + A tensor that if specified is added to the residual of the middle unet block. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`models.unets.unet_2d_condition_flax.FlaxUNet2DConditionOutput`] instead of a + plain tuple. + train (`bool`, *optional*, defaults to `False`): + Use deterministic functions and disable dropout when not training. + + Returns: + [`~models.unets.unet_2d_condition_flax.FlaxUNet2DConditionOutput`] or `tuple`: + [`~models.unets.unet_2d_condition_flax.FlaxUNet2DConditionOutput`] if `return_dict` is True, otherwise a `tuple`. + When returning a tuple, the first element is the sample tensor. + """ + # 1. time + if not isinstance(timesteps, jnp.ndarray): + timesteps = jnp.array([timesteps], dtype=jnp.int32) + elif isinstance(timesteps, jnp.ndarray) and len(timesteps.shape) == 0: + timesteps = timesteps.astype(dtype=jnp.float32) + timesteps = jnp.expand_dims(timesteps, 0) + + t_emb = self.time_proj(timesteps) + t_emb = self.time_embedding(t_emb) + + # additional embeddings + aug_emb = None + if self.addition_embed_type == "text_time": + if added_cond_kwargs is None: + raise ValueError( + f"Need to provide argument `added_cond_kwargs` for {self.__class__} when using `addition_embed_type={self.addition_embed_type}`" + ) + text_embeds = added_cond_kwargs.get("text_embeds") + if text_embeds is None: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`" + ) + time_ids = added_cond_kwargs.get("time_ids") + if time_ids is None: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`" + ) + # compute time embeds + time_embeds = self.add_time_proj(jnp.ravel(time_ids)) # (1, 6) => (6,) => (6, 256) + time_embeds = jnp.reshape(time_embeds, (text_embeds.shape[0], -1)) + add_embeds = jnp.concatenate([text_embeds, time_embeds], axis=-1) + aug_emb = self.add_embedding(add_embeds) + + t_emb = t_emb + aug_emb if aug_emb is not None else t_emb + + # 2. pre-process + sample = jnp.transpose(sample, (0, 2, 3, 1)) + sample = self.conv_in(sample) + + # 3. down + down_block_res_samples = (sample,) + for down_block in self.down_blocks: + if isinstance(down_block, FlaxCrossAttnDownBlock2D): + sample, res_samples = down_block(sample, t_emb, encoder_hidden_states, deterministic=not train) + else: + sample, res_samples = down_block(sample, t_emb, deterministic=not train) + down_block_res_samples += res_samples + + if down_block_additional_residuals is not None: + new_down_block_res_samples = () + + for down_block_res_sample, down_block_additional_residual in zip( + down_block_res_samples, down_block_additional_residuals + ): + down_block_res_sample += down_block_additional_residual + new_down_block_res_samples += (down_block_res_sample,) + + down_block_res_samples = new_down_block_res_samples + + # 4. mid + if self.mid_block is not None: + sample = self.mid_block(sample, t_emb, encoder_hidden_states, deterministic=not train) + + if mid_block_additional_residual is not None: + sample += mid_block_additional_residual + + # 5. up + for up_block in self.up_blocks: + res_samples = down_block_res_samples[-(self.layers_per_block + 1) :] + down_block_res_samples = down_block_res_samples[: -(self.layers_per_block + 1)] + if isinstance(up_block, FlaxCrossAttnUpBlock2D): + sample = up_block( + sample, + temb=t_emb, + encoder_hidden_states=encoder_hidden_states, + res_hidden_states_tuple=res_samples, + deterministic=not train, + ) + else: + sample = up_block(sample, temb=t_emb, res_hidden_states_tuple=res_samples, deterministic=not train) + + # 6. post-process + sample = self.conv_norm_out(sample) + sample = nn.silu(sample) + sample = self.conv_out(sample) + sample = jnp.transpose(sample, (0, 3, 1, 2)) + + if not return_dict: + return (sample,) + + return FlaxUNet2DConditionOutput(sample=sample) diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_3d_blocks.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_3d_blocks.py new file mode 100755 index 0000000..a48f184 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_3d_blocks.py @@ -0,0 +1,2405 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, Optional, Tuple, Union + +import torch +from torch import nn + +from ...utils import deprecate, is_torch_version, logging +from ...utils.torch_utils import apply_freeu +from ..attention import Attention +from ..resnet import ( + Downsample2D, + ResnetBlock2D, + SpatioTemporalResBlock, + TemporalConvLayer, + Upsample2D, +) +from ..transformers.dual_transformer_2d import DualTransformer2DModel +from ..transformers.transformer_2d import Transformer2DModel +from ..transformers.transformer_temporal import ( + TransformerSpatioTemporalModel, + TransformerTemporalModel, +) + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def get_down_block( + down_block_type: str, + num_layers: int, + in_channels: int, + out_channels: int, + temb_channels: int, + add_downsample: bool, + resnet_eps: float, + resnet_act_fn: str, + num_attention_heads: int, + resnet_groups: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + downsample_padding: Optional[int] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = True, + only_cross_attention: bool = False, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + temporal_num_attention_heads: int = 8, + temporal_max_seq_length: int = 32, + transformer_layers_per_block: int = 1, +) -> Union[ + "DownBlock3D", + "CrossAttnDownBlock3D", + "DownBlockMotion", + "CrossAttnDownBlockMotion", + "DownBlockSpatioTemporal", + "CrossAttnDownBlockSpatioTemporal", +]: + if down_block_type == "DownBlock3D": + return DownBlock3D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "CrossAttnDownBlock3D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnDownBlock3D") + return CrossAttnDownBlock3D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + if down_block_type == "DownBlockMotion": + return DownBlockMotion( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + temporal_num_attention_heads=temporal_num_attention_heads, + temporal_max_seq_length=temporal_max_seq_length, + ) + elif down_block_type == "CrossAttnDownBlockMotion": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnDownBlockMotion") + return CrossAttnDownBlockMotion( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + temporal_num_attention_heads=temporal_num_attention_heads, + temporal_max_seq_length=temporal_max_seq_length, + ) + elif down_block_type == "DownBlockSpatioTemporal": + # added for SDV + return DownBlockSpatioTemporal( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + ) + elif down_block_type == "CrossAttnDownBlockSpatioTemporal": + # added for SDV + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnDownBlockSpatioTemporal") + return CrossAttnDownBlockSpatioTemporal( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + num_layers=num_layers, + transformer_layers_per_block=transformer_layers_per_block, + add_downsample=add_downsample, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + ) + + raise ValueError(f"{down_block_type} does not exist.") + + +def get_up_block( + up_block_type: str, + num_layers: int, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + add_upsample: bool, + resnet_eps: float, + resnet_act_fn: str, + num_attention_heads: int, + resolution_idx: Optional[int] = None, + resnet_groups: Optional[int] = None, + cross_attention_dim: Optional[int] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = True, + only_cross_attention: bool = False, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + temporal_num_attention_heads: int = 8, + temporal_cross_attention_dim: Optional[int] = None, + temporal_max_seq_length: int = 32, + transformer_layers_per_block: int = 1, + dropout: float = 0.0, +) -> Union[ + "UpBlock3D", + "CrossAttnUpBlock3D", + "UpBlockMotion", + "CrossAttnUpBlockMotion", + "UpBlockSpatioTemporal", + "CrossAttnUpBlockSpatioTemporal", +]: + if up_block_type == "UpBlock3D": + return UpBlock3D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + resolution_idx=resolution_idx, + ) + elif up_block_type == "CrossAttnUpBlock3D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnUpBlock3D") + return CrossAttnUpBlock3D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + resolution_idx=resolution_idx, + ) + if up_block_type == "UpBlockMotion": + return UpBlockMotion( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + resolution_idx=resolution_idx, + temporal_num_attention_heads=temporal_num_attention_heads, + temporal_max_seq_length=temporal_max_seq_length, + ) + elif up_block_type == "CrossAttnUpBlockMotion": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnUpBlockMotion") + return CrossAttnUpBlockMotion( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + resolution_idx=resolution_idx, + temporal_num_attention_heads=temporal_num_attention_heads, + temporal_max_seq_length=temporal_max_seq_length, + ) + elif up_block_type == "UpBlockSpatioTemporal": + # added for SDV + return UpBlockSpatioTemporal( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + resolution_idx=resolution_idx, + add_upsample=add_upsample, + ) + elif up_block_type == "CrossAttnUpBlockSpatioTemporal": + # added for SDV + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnUpBlockSpatioTemporal") + return CrossAttnUpBlockSpatioTemporal( + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + num_layers=num_layers, + transformer_layers_per_block=transformer_layers_per_block, + add_upsample=add_upsample, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + resolution_idx=resolution_idx, + ) + + raise ValueError(f"{up_block_type} does not exist.") + + +class UNetMidBlock3DCrossAttn(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + output_scale_factor: float = 1.0, + cross_attention_dim: int = 1280, + dual_cross_attention: bool = False, + use_linear_projection: bool = True, + upcast_attention: bool = False, + ): + super().__init__() + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + temp_convs = [ + TemporalConvLayer( + in_channels, + in_channels, + dropout=0.1, + norm_num_groups=resnet_groups, + ) + ] + attentions = [] + temp_attentions = [] + + for _ in range(num_layers): + attentions.append( + Transformer2DModel( + in_channels // num_attention_heads, + num_attention_heads, + in_channels=in_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + ) + ) + temp_attentions.append( + TransformerTemporalModel( + in_channels // num_attention_heads, + num_attention_heads, + in_channels=in_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + temp_convs.append( + TemporalConvLayer( + in_channels, + in_channels, + dropout=0.1, + norm_num_groups=resnet_groups, + ) + ) + + self.resnets = nn.ModuleList(resnets) + self.temp_convs = nn.ModuleList(temp_convs) + self.attentions = nn.ModuleList(attentions) + self.temp_attentions = nn.ModuleList(temp_attentions) + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + num_frames: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + ) -> torch.FloatTensor: + hidden_states = self.resnets[0](hidden_states, temb) + hidden_states = self.temp_convs[0](hidden_states, num_frames=num_frames) + for attn, temp_attn, resnet, temp_conv in zip( + self.attentions, self.temp_attentions, self.resnets[1:], self.temp_convs[1:] + ): + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + hidden_states = temp_attn( + hidden_states, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + hidden_states = resnet(hidden_states, temb) + hidden_states = temp_conv(hidden_states, num_frames=num_frames) + + return hidden_states + + +class CrossAttnDownBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + downsample_padding: int = 1, + add_downsample: bool = True, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + ): + super().__init__() + resnets = [] + attentions = [] + temp_attentions = [] + temp_convs = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + temp_convs.append( + TemporalConvLayer( + out_channels, + out_channels, + dropout=0.1, + norm_num_groups=resnet_groups, + ) + ) + attentions.append( + Transformer2DModel( + out_channels // num_attention_heads, + num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + ) + ) + temp_attentions.append( + TransformerTemporalModel( + out_channels // num_attention_heads, + num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + self.resnets = nn.ModuleList(resnets) + self.temp_convs = nn.ModuleList(temp_convs) + self.attentions = nn.ModuleList(attentions) + self.temp_attentions = nn.ModuleList(temp_attentions) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name="op", + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + num_frames: int = 1, + cross_attention_kwargs: Dict[str, Any] = None, + ) -> Union[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + # TODO(Patrick, William) - attention mask is not used + output_states = () + + for resnet, temp_conv, attn, temp_attn in zip( + self.resnets, self.temp_convs, self.attentions, self.temp_attentions + ): + hidden_states = resnet(hidden_states, temb) + hidden_states = temp_conv(hidden_states, num_frames=num_frames) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + hidden_states = temp_attn( + hidden_states, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + output_states += (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states += (hidden_states,) + + return hidden_states, output_states + + +class DownBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + downsample_padding: int = 1, + ): + super().__init__() + resnets = [] + temp_convs = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + temp_convs.append( + TemporalConvLayer( + out_channels, + out_channels, + dropout=0.1, + norm_num_groups=resnet_groups, + ) + ) + + self.resnets = nn.ModuleList(resnets) + self.temp_convs = nn.ModuleList(temp_convs) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name="op", + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + num_frames: int = 1, + ) -> Union[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + output_states = () + + for resnet, temp_conv in zip(self.resnets, self.temp_convs): + hidden_states = resnet(hidden_states, temb) + hidden_states = temp_conv(hidden_states, num_frames=num_frames) + + output_states += (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states += (hidden_states,) + + return hidden_states, output_states + + +class CrossAttnUpBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + resolution_idx: Optional[int] = None, + ): + super().__init__() + resnets = [] + temp_convs = [] + attentions = [] + temp_attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + temp_convs.append( + TemporalConvLayer( + out_channels, + out_channels, + dropout=0.1, + norm_num_groups=resnet_groups, + ) + ) + attentions.append( + Transformer2DModel( + out_channels // num_attention_heads, + num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + ) + ) + temp_attentions.append( + TransformerTemporalModel( + out_channels // num_attention_heads, + num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + self.resnets = nn.ModuleList(resnets) + self.temp_convs = nn.ModuleList(temp_convs) + self.attentions = nn.ModuleList(attentions) + self.temp_attentions = nn.ModuleList(temp_attentions) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + num_frames: int = 1, + cross_attention_kwargs: Dict[str, Any] = None, + ) -> torch.FloatTensor: + is_freeu_enabled = ( + getattr(self, "s1", None) + and getattr(self, "s2", None) + and getattr(self, "b1", None) + and getattr(self, "b2", None) + ) + + # TODO(Patrick, William) - attention mask is not used + for resnet, temp_conv, attn, temp_attn in zip( + self.resnets, self.temp_convs, self.attentions, self.temp_attentions + ): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # FreeU: Only operate on the first two stages + if is_freeu_enabled: + hidden_states, res_hidden_states = apply_freeu( + self.resolution_idx, + hidden_states, + res_hidden_states, + s1=self.s1, + s2=self.s2, + b1=self.b1, + b2=self.b2, + ) + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + hidden_states = resnet(hidden_states, temb) + hidden_states = temp_conv(hidden_states, num_frames=num_frames) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + hidden_states = temp_attn( + hidden_states, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +class UpBlock3D(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + resolution_idx: Optional[int] = None, + ): + super().__init__() + resnets = [] + temp_convs = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + temp_convs.append( + TemporalConvLayer( + out_channels, + out_channels, + dropout=0.1, + norm_num_groups=resnet_groups, + ) + ) + + self.resnets = nn.ModuleList(resnets) + self.temp_convs = nn.ModuleList(temp_convs) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + num_frames: int = 1, + ) -> torch.FloatTensor: + is_freeu_enabled = ( + getattr(self, "s1", None) + and getattr(self, "s2", None) + and getattr(self, "b1", None) + and getattr(self, "b2", None) + ) + for resnet, temp_conv in zip(self.resnets, self.temp_convs): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # FreeU: Only operate on the first two stages + if is_freeu_enabled: + hidden_states, res_hidden_states = apply_freeu( + self.resolution_idx, + hidden_states, + res_hidden_states, + s1=self.s1, + s2=self.s2, + b1=self.b1, + b2=self.b2, + ) + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + hidden_states = resnet(hidden_states, temb) + hidden_states = temp_conv(hidden_states, num_frames=num_frames) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +class DownBlockMotion(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + downsample_padding: int = 1, + temporal_num_attention_heads: int = 1, + temporal_cross_attention_dim: Optional[int] = None, + temporal_max_seq_length: int = 32, + ): + super().__init__() + resnets = [] + motion_modules = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + motion_modules.append( + TransformerTemporalModel( + num_attention_heads=temporal_num_attention_heads, + in_channels=out_channels, + norm_num_groups=resnet_groups, + cross_attention_dim=temporal_cross_attention_dim, + attention_bias=False, + activation_fn="geglu", + positional_embeddings="sinusoidal", + num_positional_embeddings=temporal_max_seq_length, + attention_head_dim=out_channels // temporal_num_attention_heads, + ) + ) + + self.resnets = nn.ModuleList(resnets) + self.motion_modules = nn.ModuleList(motion_modules) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name="op", + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + num_frames: int = 1, + *args, + **kwargs, + ) -> Union[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + output_states = () + + blocks = zip(self.resnets, self.motion_modules) + for resnet, motion_module in blocks: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + use_reentrant=False, + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = motion_module(hidden_states, num_frames=num_frames)[0] + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class CrossAttnDownBlockMotion(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + downsample_padding: int = 1, + add_downsample: bool = True, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + attention_type: str = "default", + temporal_cross_attention_dim: Optional[int] = None, + temporal_num_attention_heads: int = 8, + temporal_max_seq_length: int = 32, + ): + super().__init__() + resnets = [] + attentions = [] + motion_modules = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + + motion_modules.append( + TransformerTemporalModel( + num_attention_heads=temporal_num_attention_heads, + in_channels=out_channels, + norm_num_groups=resnet_groups, + cross_attention_dim=temporal_cross_attention_dim, + attention_bias=False, + activation_fn="geglu", + positional_embeddings="sinusoidal", + num_positional_embeddings=temporal_max_seq_length, + attention_head_dim=out_channels // temporal_num_attention_heads, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + self.motion_modules = nn.ModuleList(motion_modules) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=downsample_padding, + name="op", + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + num_frames: int = 1, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + additional_residuals: Optional[torch.FloatTensor] = None, + ): + if cross_attention_kwargs is not None: + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + output_states = () + + blocks = list(zip(self.resnets, self.attentions, self.motion_modules)) + for i, (resnet, attn, motion_module) in enumerate(blocks): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = motion_module( + hidden_states, + num_frames=num_frames, + )[0] + + # apply additional residuals to the output of the last pair of resnet and attention blocks + if i == len(blocks) - 1 and additional_residuals is not None: + hidden_states = hidden_states + additional_residuals + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class CrossAttnUpBlockMotion(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + attention_type: str = "default", + temporal_cross_attention_dim: Optional[int] = None, + temporal_num_attention_heads: int = 8, + temporal_max_seq_length: int = 32, + ): + super().__init__() + resnets = [] + attentions = [] + motion_modules = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + motion_modules.append( + TransformerTemporalModel( + num_attention_heads=temporal_num_attention_heads, + in_channels=out_channels, + norm_num_groups=resnet_groups, + cross_attention_dim=temporal_cross_attention_dim, + attention_bias=False, + activation_fn="geglu", + positional_embeddings="sinusoidal", + num_positional_embeddings=temporal_max_seq_length, + attention_head_dim=out_channels // temporal_num_attention_heads, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + self.motion_modules = nn.ModuleList(motion_modules) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + num_frames: int = 1, + ) -> torch.FloatTensor: + if cross_attention_kwargs is not None: + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + is_freeu_enabled = ( + getattr(self, "s1", None) + and getattr(self, "s2", None) + and getattr(self, "b1", None) + and getattr(self, "b2", None) + ) + + blocks = zip(self.resnets, self.attentions, self.motion_modules) + for resnet, attn, motion_module in blocks: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # FreeU: Only operate on the first two stages + if is_freeu_enabled: + hidden_states, res_hidden_states = apply_freeu( + self.resolution_idx, + hidden_states, + res_hidden_states, + s1=self.s1, + s2=self.s2, + b1=self.b1, + b2=self.b2, + ) + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = motion_module( + hidden_states, + num_frames=num_frames, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +class UpBlockMotion(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + temporal_norm_num_groups: int = 32, + temporal_cross_attention_dim: Optional[int] = None, + temporal_num_attention_heads: int = 8, + temporal_max_seq_length: int = 32, + ): + super().__init__() + resnets = [] + motion_modules = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + motion_modules.append( + TransformerTemporalModel( + num_attention_heads=temporal_num_attention_heads, + in_channels=out_channels, + norm_num_groups=temporal_norm_num_groups, + cross_attention_dim=temporal_cross_attention_dim, + attention_bias=False, + activation_fn="geglu", + positional_embeddings="sinusoidal", + num_positional_embeddings=temporal_max_seq_length, + attention_head_dim=out_channels // temporal_num_attention_heads, + ) + ) + + self.resnets = nn.ModuleList(resnets) + self.motion_modules = nn.ModuleList(motion_modules) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + upsample_size=None, + num_frames: int = 1, + *args, + **kwargs, + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + is_freeu_enabled = ( + getattr(self, "s1", None) + and getattr(self, "s2", None) + and getattr(self, "b1", None) + and getattr(self, "b2", None) + ) + + blocks = zip(self.resnets, self.motion_modules) + + for resnet, motion_module in blocks: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # FreeU: Only operate on the first two stages + if is_freeu_enabled: + hidden_states, res_hidden_states = apply_freeu( + self.resolution_idx, + hidden_states, + res_hidden_states, + s1=self.s1, + s2=self.s2, + b1=self.b1, + b2=self.b2, + ) + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + use_reentrant=False, + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = motion_module(hidden_states, num_frames=num_frames)[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +class UNetMidBlockCrossAttnMotion(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + output_scale_factor: float = 1.0, + cross_attention_dim: int = 1280, + dual_cross_attention: float = False, + use_linear_projection: float = False, + upcast_attention: float = False, + attention_type: str = "default", + temporal_num_attention_heads: int = 1, + temporal_cross_attention_dim: Optional[int] = None, + temporal_max_seq_length: int = 32, + ): + super().__init__() + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + motion_modules = [] + + for _ in range(num_layers): + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=transformer_layers_per_block, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + motion_modules.append( + TransformerTemporalModel( + num_attention_heads=temporal_num_attention_heads, + attention_head_dim=in_channels // temporal_num_attention_heads, + in_channels=in_channels, + norm_num_groups=resnet_groups, + cross_attention_dim=temporal_cross_attention_dim, + attention_bias=False, + positional_embeddings="sinusoidal", + num_positional_embeddings=temporal_max_seq_length, + activation_fn="geglu", + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + self.motion_modules = nn.ModuleList(motion_modules) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + num_frames: int = 1, + ) -> torch.FloatTensor: + if cross_attention_kwargs is not None: + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + hidden_states = self.resnets[0](hidden_states, temb) + + blocks = zip(self.attentions, self.resnets[1:], self.motion_modules) + for attn, resnet, motion_module in blocks: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(motion_module), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + else: + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = motion_module( + hidden_states, + num_frames=num_frames, + )[0] + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +class MidBlockTemporalDecoder(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + attention_head_dim: int = 512, + num_layers: int = 1, + upcast_attention: bool = False, + ): + super().__init__() + + resnets = [] + attentions = [] + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + resnets.append( + SpatioTemporalResBlock( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=None, + eps=1e-6, + temporal_eps=1e-5, + merge_factor=0.0, + merge_strategy="learned", + switch_spatial_to_temporal_mix=True, + ) + ) + + attentions.append( + Attention( + query_dim=in_channels, + heads=in_channels // attention_head_dim, + dim_head=attention_head_dim, + eps=1e-6, + upcast_attention=upcast_attention, + norm_num_groups=32, + bias=True, + residual_connection=True, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward( + self, + hidden_states: torch.FloatTensor, + image_only_indicator: torch.FloatTensor, + ): + hidden_states = self.resnets[0]( + hidden_states, + image_only_indicator=image_only_indicator, + ) + for resnet, attn in zip(self.resnets[1:], self.attentions): + hidden_states = attn(hidden_states) + hidden_states = resnet( + hidden_states, + image_only_indicator=image_only_indicator, + ) + + return hidden_states + + +class UpBlockTemporalDecoder(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + num_layers: int = 1, + add_upsample: bool = True, + ): + super().__init__() + resnets = [] + for i in range(num_layers): + input_channels = in_channels if i == 0 else out_channels + + resnets.append( + SpatioTemporalResBlock( + in_channels=input_channels, + out_channels=out_channels, + temb_channels=None, + eps=1e-6, + temporal_eps=1e-5, + merge_factor=0.0, + merge_strategy="learned", + switch_spatial_to_temporal_mix=True, + ) + ) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + def forward( + self, + hidden_states: torch.FloatTensor, + image_only_indicator: torch.FloatTensor, + ) -> torch.FloatTensor: + for resnet in self.resnets: + hidden_states = resnet( + hidden_states, + image_only_indicator=image_only_indicator, + ) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class UNetMidBlockSpatioTemporal(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + ): + super().__init__() + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + # support for variable transformer layers per block + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + # there is always at least one resnet + resnets = [ + SpatioTemporalResBlock( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=1e-5, + ) + ] + attentions = [] + + for i in range(num_layers): + attentions.append( + TransformerSpatioTemporalModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + ) + ) + + resnets.append( + SpatioTemporalResBlock( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=1e-5, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + image_only_indicator: Optional[torch.Tensor] = None, + ) -> torch.FloatTensor: + hidden_states = self.resnets[0]( + hidden_states, + temb, + image_only_indicator=image_only_indicator, + ) + + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if self.training and self.gradient_checkpointing: # TODO + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + image_only_indicator=image_only_indicator, + return_dict=False, + )[0] + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + image_only_indicator, + **ckpt_kwargs, + ) + else: + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + image_only_indicator=image_only_indicator, + return_dict=False, + )[0] + hidden_states = resnet( + hidden_states, + temb, + image_only_indicator=image_only_indicator, + ) + + return hidden_states + + +class DownBlockSpatioTemporal(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + num_layers: int = 1, + add_downsample: bool = True, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + SpatioTemporalResBlock( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=1e-5, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + name="op", + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + image_only_indicator: Optional[torch.Tensor] = None, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + output_states = () + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + image_only_indicator, + use_reentrant=False, + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + image_only_indicator, + ) + else: + hidden_states = resnet( + hidden_states, + temb, + image_only_indicator=image_only_indicator, + ) + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class CrossAttnDownBlockSpatioTemporal(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + add_downsample: bool = True, + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + SpatioTemporalResBlock( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=1e-6, + ) + ) + attentions.append( + TransformerSpatioTemporalModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, + use_conv=True, + out_channels=out_channels, + padding=1, + name="op", + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + image_only_indicator: Optional[torch.Tensor] = None, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + output_states = () + + blocks = list(zip(self.resnets, self.attentions)) + for resnet, attn in blocks: + if self.training and self.gradient_checkpointing: # TODO + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + image_only_indicator, + **ckpt_kwargs, + ) + + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + image_only_indicator=image_only_indicator, + return_dict=False, + )[0] + else: + hidden_states = resnet( + hidden_states, + temb, + image_only_indicator=image_only_indicator, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + image_only_indicator=image_only_indicator, + return_dict=False, + )[0] + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class UpBlockSpatioTemporal(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + num_layers: int = 1, + resnet_eps: float = 1e-6, + add_upsample: bool = True, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + SpatioTemporalResBlock( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + image_only_indicator: Optional[torch.Tensor] = None, + ) -> torch.FloatTensor: + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + image_only_indicator, + use_reentrant=False, + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + image_only_indicator, + ) + else: + hidden_states = resnet( + hidden_states, + temb, + image_only_indicator=image_only_indicator, + ) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states + + +class CrossAttnUpBlockSpatioTemporal(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + resnet_eps: float = 1e-6, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + add_upsample: bool = True, + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + SpatioTemporalResBlock( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + ) + ) + attentions.append( + TransformerSpatioTemporalModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + image_only_indicator: Optional[torch.Tensor] = None, + ) -> torch.FloatTensor: + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: # TODO + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + image_only_indicator, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + image_only_indicator=image_only_indicator, + return_dict=False, + )[0] + else: + hidden_states = resnet( + hidden_states, + temb, + image_only_indicator=image_only_indicator, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + image_only_indicator=image_only_indicator, + return_dict=False, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states) + + return hidden_states diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_3d_condition.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_3d_condition.py new file mode 100755 index 0000000..b7641a9 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_3d_condition.py @@ -0,0 +1,753 @@ +# Copyright 2024 Alibaba DAMO-VILAB and The HuggingFace Team. All rights reserved. +# Copyright 2024 The ModelScope Team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.utils.checkpoint + +from ...configuration_utils import ConfigMixin, register_to_config +from ...loaders import UNet2DConditionLoadersMixin +from ...utils import BaseOutput, deprecate, logging +from ..activations import get_activation +from ..attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + Attention, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from ..embeddings import TimestepEmbedding, Timesteps +from ..modeling_utils import ModelMixin +from ..transformers.transformer_temporal import TransformerTemporalModel +from .unet_3d_blocks import ( + CrossAttnDownBlock3D, + CrossAttnUpBlock3D, + DownBlock3D, + UNetMidBlock3DCrossAttn, + UpBlock3D, + get_down_block, + get_up_block, +) + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class UNet3DConditionOutput(BaseOutput): + """ + The output of [`UNet3DConditionModel`]. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_channels, num_frames, height, width)`): + The hidden states output conditioned on `encoder_hidden_states` input. Output of last layer of model. + """ + + sample: torch.FloatTensor + + +class UNet3DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin): + r""" + A conditional 3D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample + shaped output. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + Height and width of input/output sample. + in_channels (`int`, *optional*, defaults to 4): The number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 4): The number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "CrossAttnDownBlock3D", "DownBlock3D")`): + The tuple of downsample blocks to use. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D", "CrossAttnUpBlock3D")`): + The tuple of upsample blocks to use. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. + downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution. + mid_block_scale_factor (`float`, *optional*, defaults to 1.0): The scale factor to use for the mid block. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. + If `None`, normalization and activation layers is skipped in post-processing. + norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization. + cross_attention_dim (`int`, *optional*, defaults to 1024): The dimension of the cross attention features. + attention_head_dim (`int`, *optional*, defaults to 64): The dimension of the attention heads. + num_attention_heads (`int`, *optional*): The number of attention heads. + """ + + _supports_gradient_checkpointing = False + + @register_to_config + def __init__( + self, + sample_size: Optional[int] = None, + in_channels: int = 4, + out_channels: int = 4, + down_block_types: Tuple[str, ...] = ( + "CrossAttnDownBlock3D", + "CrossAttnDownBlock3D", + "CrossAttnDownBlock3D", + "DownBlock3D", + ), + up_block_types: Tuple[str, ...] = ( + "UpBlock3D", + "CrossAttnUpBlock3D", + "CrossAttnUpBlock3D", + "CrossAttnUpBlock3D", + ), + block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + layers_per_block: int = 2, + downsample_padding: int = 1, + mid_block_scale_factor: float = 1, + act_fn: str = "silu", + norm_num_groups: Optional[int] = 32, + norm_eps: float = 1e-5, + cross_attention_dim: int = 1024, + attention_head_dim: Union[int, Tuple[int]] = 64, + num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + ): + super().__init__() + + self.sample_size = sample_size + + if num_attention_heads is not None: + raise NotImplementedError( + "At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19." + ) + + # If `num_attention_heads` is not defined (which is the case for most models) + # it will default to `attention_head_dim`. This looks weird upon first reading it and it is. + # The reason for this behavior is to correct for incorrectly named variables that were introduced + # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131 + # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking + # which is why we correct for the naming here. + num_attention_heads = num_attention_heads or attention_head_dim + + # Check inputs + if len(down_block_types) != len(up_block_types): + raise ValueError( + f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}." + ) + + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}." + ) + + # input + conv_in_kernel = 3 + conv_out_kernel = 3 + conv_in_padding = (conv_in_kernel - 1) // 2 + self.conv_in = nn.Conv2d( + in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding + ) + + # time + time_embed_dim = block_out_channels[0] * 4 + self.time_proj = Timesteps(block_out_channels[0], True, 0) + timestep_input_dim = block_out_channels[0] + + self.time_embedding = TimestepEmbedding( + timestep_input_dim, + time_embed_dim, + act_fn=act_fn, + ) + + self.transformer_in = TransformerTemporalModel( + num_attention_heads=8, + attention_head_dim=attention_head_dim, + in_channels=block_out_channels[0], + num_layers=1, + norm_num_groups=norm_num_groups, + ) + + # class embedding + self.down_blocks = nn.ModuleList([]) + self.up_blocks = nn.ModuleList([]) + + if isinstance(num_attention_heads, int): + num_attention_heads = (num_attention_heads,) * len(down_block_types) + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + temb_channels=time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads[i], + downsample_padding=downsample_padding, + dual_cross_attention=False, + ) + self.down_blocks.append(down_block) + + # mid + self.mid_block = UNetMidBlock3DCrossAttn( + in_channels=block_out_channels[-1], + temb_channels=time_embed_dim, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads[-1], + resnet_groups=norm_num_groups, + dual_cross_attention=False, + ) + + # count how many layers upsample the images + self.num_upsamplers = 0 + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_num_attention_heads = list(reversed(num_attention_heads)) + + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + is_final_block = i == len(block_out_channels) - 1 + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)] + + # add upsample block for all BUT final layer + if not is_final_block: + add_upsample = True + self.num_upsamplers += 1 + else: + add_upsample = False + + up_block = get_up_block( + up_block_type, + num_layers=layers_per_block + 1, + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=time_embed_dim, + add_upsample=add_upsample, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=reversed_num_attention_heads[i], + dual_cross_attention=False, + resolution_idx=i, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_num_groups is not None: + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps + ) + self.conv_act = get_activation("silu") + else: + self.conv_norm_out = None + self.conv_act = None + + conv_out_padding = (conv_out_kernel - 1) // 2 + self.conv_out = nn.Conv2d( + block_out_channels[0], out_channels, kernel_size=conv_out_kernel, padding=conv_out_padding + ) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attention_slice + def set_attention_slice(self, slice_size: Union[str, int, List[int]]) -> None: + r""" + Enable sliced attention computation. + + When this option is enabled, the attention module splits the input tensor in slices to compute attention in + several steps. This is useful for saving some memory in exchange for a small decrease in speed. + + Args: + slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`): + When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If + `"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is + provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim` + must be a multiple of `slice_size`. + """ + sliceable_head_dims = [] + + def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): + if hasattr(module, "set_attention_slice"): + sliceable_head_dims.append(module.sliceable_head_dim) + + for child in module.children(): + fn_recursive_retrieve_sliceable_dims(child) + + # retrieve number of attention layers + for module in self.children(): + fn_recursive_retrieve_sliceable_dims(module) + + num_sliceable_layers = len(sliceable_head_dims) + + if slice_size == "auto": + # half the attention head size is usually a good trade-off between + # speed and memory + slice_size = [dim // 2 for dim in sliceable_head_dims] + elif slice_size == "max": + # make smallest slice possible + slice_size = num_sliceable_layers * [1] + + slice_size = num_sliceable_layers * [slice_size] if not isinstance(slice_size, list) else slice_size + + if len(slice_size) != len(sliceable_head_dims): + raise ValueError( + f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different" + f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}." + ) + + for i in range(len(slice_size)): + size = slice_size[i] + dim = sliceable_head_dims[i] + if size is not None and size > dim: + raise ValueError(f"size {size} has to be smaller or equal to {dim}.") + + # Recursively walk through all the children. + # Any children which exposes the set_attention_slice method + # gets the message + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + if hasattr(module, "set_attention_slice"): + module.set_attention_slice(slice_size.pop()) + + for child in module.children(): + fn_recursive_set_attention_slice(child, slice_size) + + reversed_slice_size = list(reversed(slice_size)) + for module in self.children(): + fn_recursive_set_attention_slice(module, reversed_slice_size) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def enable_forward_chunking(self, chunk_size: Optional[int] = None, dim: int = 0) -> None: + """ + Sets the attention processor to use [feed forward + chunking](https://huggingface.co/blog/reformer#2-chunked-feed-forward-layers). + + Parameters: + chunk_size (`int`, *optional*): + The chunk size of the feed-forward layers. If not specified, will run feed-forward layer individually + over each tensor of dim=`dim`. + dim (`int`, *optional*, defaults to `0`): + The dimension over which the feed-forward computation should be chunked. Choose between dim=0 (batch) + or dim=1 (sequence length). + """ + if dim not in [0, 1]: + raise ValueError(f"Make sure to set `dim` to either 0 or 1, not {dim}") + + # By default chunk size is 1 + chunk_size = chunk_size or 1 + + def fn_recursive_feed_forward(module: torch.nn.Module, chunk_size: int, dim: int): + if hasattr(module, "set_chunk_feed_forward"): + module.set_chunk_feed_forward(chunk_size=chunk_size, dim=dim) + + for child in module.children(): + fn_recursive_feed_forward(child, chunk_size, dim) + + for module in self.children(): + fn_recursive_feed_forward(module, chunk_size, dim) + + def disable_forward_chunking(self): + def fn_recursive_feed_forward(module: torch.nn.Module, chunk_size: int, dim: int): + if hasattr(module, "set_chunk_feed_forward"): + module.set_chunk_feed_forward(chunk_size=chunk_size, dim=dim) + + for child in module.children(): + fn_recursive_feed_forward(child, chunk_size, dim) + + for module in self.children(): + fn_recursive_feed_forward(module, None, 0) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + def _set_gradient_checkpointing(self, module, value: bool = False) -> None: + if isinstance(module, (CrossAttnDownBlock3D, DownBlock3D, CrossAttnUpBlock3D, UpBlock3D)): + module.gradient_checkpointing = value + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.enable_freeu + def enable_freeu(self, s1, s2, b1, b2): + r"""Enables the FreeU mechanism from https://arxiv.org/abs/2309.11497. + + The suffixes after the scaling factors represent the stage blocks where they are being applied. + + Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of values that + are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL. + + Args: + s1 (`float`): + Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + s2 (`float`): + Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. + b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. + """ + for i, upsample_block in enumerate(self.up_blocks): + setattr(upsample_block, "s1", s1) + setattr(upsample_block, "s2", s2) + setattr(upsample_block, "b1", b1) + setattr(upsample_block, "b2", b2) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.disable_freeu + def disable_freeu(self): + """Disables the FreeU mechanism.""" + freeu_keys = {"s1", "s2", "b1", "b2"} + for i, upsample_block in enumerate(self.up_blocks): + for k in freeu_keys: + if hasattr(upsample_block, k) or getattr(upsample_block, k, None) is not None: + setattr(upsample_block, k, None) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.fuse_qkv_projections + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, + key, value) are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is 🧪 experimental. + + + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.") + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.unfuse_qkv_projections + def unfuse_qkv_projections(self): + """Disables the fused QKV projection if enabled. + + + + This API is 🧪 experimental. + + + + """ + if self.original_attn_processors is not None: + self.set_attn_processor(self.original_attn_processors) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.unload_lora + def unload_lora(self): + """Unloads LoRA weights.""" + deprecate( + "unload_lora", + "0.28.0", + "Calling `unload_lora()` is deprecated and will be removed in a future version. Please install `peft` and then call `disable_adapters().", + ) + for module in self.modules(): + if hasattr(module, "set_lora_layer"): + module.set_lora_layer(None) + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + mid_block_additional_residual: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[UNet3DConditionOutput, Tuple[torch.FloatTensor]]: + r""" + The [`UNet3DConditionModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor with the following shape `(batch, num_channels, num_frames, height, width`. + timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input. + encoder_hidden_states (`torch.FloatTensor`): + The encoder hidden states with shape `(batch, sequence_length, feature_dim)`. + class_labels (`torch.Tensor`, *optional*, defaults to `None`): + Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings. + timestep_cond: (`torch.Tensor`, *optional*, defaults to `None`): + Conditional embeddings for timestep. If provided, the embeddings will be summed with the samples passed + through the `self.time_embedding` layer to obtain the timestep embeddings. + attention_mask (`torch.Tensor`, *optional*, defaults to `None`): + An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask + is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large + negative values to the attention scores corresponding to "discard" tokens. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + down_block_additional_residuals: (`tuple` of `torch.Tensor`, *optional*): + A tuple of tensors that if specified are added to the residuals of down unet blocks. + mid_block_additional_residual: (`torch.Tensor`, *optional*): + A tensor that if specified is added to the residual of the middle unet block. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unet_3d_condition.UNet3DConditionOutput`] instead of a plain + tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttnProcessor`]. + + Returns: + [`~models.unet_3d_condition.UNet3DConditionOutput`] or `tuple`: + If `return_dict` is True, an [`~models.unet_3d_condition.UNet3DConditionOutput`] is returned, otherwise + a `tuple` is returned where the first element is the sample tensor. + """ + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layears). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2**self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]): + logger.info("Forward upsample size to force interpolation output size.") + forward_upsample_size = True + + # prepare attention_mask + if attention_mask is not None: + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == "mps" + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + num_frames = sample.shape[2] + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # timesteps does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=self.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + emb = emb.repeat_interleave(repeats=num_frames, dim=0) + encoder_hidden_states = encoder_hidden_states.repeat_interleave(repeats=num_frames, dim=0) + + # 2. pre-process + sample = sample.permute(0, 2, 1, 3, 4).reshape((sample.shape[0] * num_frames, -1) + sample.shape[3:]) + sample = self.conv_in(sample) + + sample = self.transformer_in( + sample, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # 3. down + down_block_res_samples = (sample,) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb, num_frames=num_frames) + + down_block_res_samples += res_samples + + if down_block_additional_residuals is not None: + new_down_block_res_samples = () + + for down_block_res_sample, down_block_additional_residual in zip( + down_block_res_samples, down_block_additional_residuals + ): + down_block_res_sample = down_block_res_sample + down_block_additional_residual + new_down_block_res_samples += (down_block_res_sample,) + + down_block_res_samples = new_down_block_res_samples + + # 4. mid + if self.mid_block is not None: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + + if mid_block_additional_residual is not None: + sample = sample + mid_block_additional_residual + + # 5. up + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets) :] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=encoder_hidden_states, + upsample_size=upsample_size, + attention_mask=attention_mask, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + upsample_size=upsample_size, + num_frames=num_frames, + ) + + # 6. post-process + if self.conv_norm_out: + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + + sample = self.conv_out(sample) + + # reshape to (batch, channel, framerate, width, height) + sample = sample[None, :].reshape((-1, num_frames) + sample.shape[1:]).permute(0, 2, 1, 3, 4) + + if not return_dict: + return (sample,) + + return UNet3DConditionOutput(sample=sample) diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_i2vgen_xl.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_i2vgen_xl.py new file mode 100755 index 0000000..5c5c6a2 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_i2vgen_xl.py @@ -0,0 +1,724 @@ +# Copyright 2024 Alibaba DAMO-VILAB and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Dict, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.utils.checkpoint + +from ...configuration_utils import ConfigMixin, register_to_config +from ...loaders import UNet2DConditionLoadersMixin +from ...utils import logging +from ..activations import get_activation +from ..attention import Attention, FeedForward +from ..attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from ..embeddings import TimestepEmbedding, Timesteps +from ..modeling_utils import ModelMixin +from ..transformers.transformer_temporal import TransformerTemporalModel +from .unet_3d_blocks import ( + CrossAttnDownBlock3D, + CrossAttnUpBlock3D, + DownBlock3D, + UNetMidBlock3DCrossAttn, + UpBlock3D, + get_down_block, + get_up_block, +) +from .unet_3d_condition import UNet3DConditionOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class I2VGenXLTransformerTemporalEncoder(nn.Module): + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + activation_fn: str = "geglu", + upcast_attention: bool = False, + ff_inner_dim: Optional[int] = None, + dropout: int = 0.0, + ): + super().__init__() + self.norm1 = nn.LayerNorm(dim, elementwise_affine=True, eps=1e-5) + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=False, + upcast_attention=upcast_attention, + out_bias=True, + ) + self.ff = FeedForward( + dim, + dropout=dropout, + activation_fn=activation_fn, + final_dropout=False, + inner_dim=ff_inner_dim, + bias=True, + ) + + def forward( + self, + hidden_states: torch.FloatTensor, + ) -> torch.FloatTensor: + norm_hidden_states = self.norm1(hidden_states) + attn_output = self.attn1(norm_hidden_states, encoder_hidden_states=None) + hidden_states = attn_output + hidden_states + if hidden_states.ndim == 4: + hidden_states = hidden_states.squeeze(1) + + ff_output = self.ff(hidden_states) + hidden_states = ff_output + hidden_states + if hidden_states.ndim == 4: + hidden_states = hidden_states.squeeze(1) + + return hidden_states + + +class I2VGenXLUNet(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin): + r""" + I2VGenXL UNet. It is a conditional 3D UNet model that takes a noisy sample, conditional state, and a timestep + and returns a sample-shaped output. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + Height and width of input/output sample. + in_channels (`int`, *optional*, defaults to 4): The number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 4): The number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): + The tuple of downsample blocks to use. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): + The tuple of upsample blocks to use. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. + norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. + If `None`, normalization and activation layers is skipped in post-processing. + cross_attention_dim (`int`, *optional*, defaults to 1280): The dimension of the cross attention features. + attention_head_dim (`int`, *optional*, defaults to 64): Attention head dim. + num_attention_heads (`int`, *optional*): The number of attention heads. + """ + + _supports_gradient_checkpointing = False + + @register_to_config + def __init__( + self, + sample_size: Optional[int] = None, + in_channels: int = 4, + out_channels: int = 4, + down_block_types: Tuple[str, ...] = ( + "CrossAttnDownBlock3D", + "CrossAttnDownBlock3D", + "CrossAttnDownBlock3D", + "DownBlock3D", + ), + up_block_types: Tuple[str, ...] = ( + "UpBlock3D", + "CrossAttnUpBlock3D", + "CrossAttnUpBlock3D", + "CrossAttnUpBlock3D", + ), + block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + layers_per_block: int = 2, + norm_num_groups: Optional[int] = 32, + cross_attention_dim: int = 1024, + attention_head_dim: Union[int, Tuple[int]] = 64, + num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + ): + super().__init__() + + # When we first integrated the UNet into the library, we didn't have `attention_head_dim`. As a consequence + # of that, we used `num_attention_heads` for arguments that actually denote attention head dimension. This + # is why we ignore `num_attention_heads` and calculate it from `attention_head_dims` below. + # This is still an incorrect way of calculating `num_attention_heads` but we need to stick to it + # without running proper depcrecation cycles for the {down,mid,up} blocks which are a + # part of the public API. + num_attention_heads = attention_head_dim + + # Check inputs + if len(down_block_types) != len(up_block_types): + raise ValueError( + f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}." + ) + + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}." + ) + + # input + self.conv_in = nn.Conv2d(in_channels + in_channels, block_out_channels[0], kernel_size=3, padding=1) + + self.transformer_in = TransformerTemporalModel( + num_attention_heads=8, + attention_head_dim=num_attention_heads, + in_channels=block_out_channels[0], + num_layers=1, + norm_num_groups=norm_num_groups, + ) + + # image embedding + self.image_latents_proj_in = nn.Sequential( + nn.Conv2d(4, in_channels * 4, 3, padding=1), + nn.SiLU(), + nn.Conv2d(in_channels * 4, in_channels * 4, 3, stride=1, padding=1), + nn.SiLU(), + nn.Conv2d(in_channels * 4, in_channels, 3, stride=1, padding=1), + ) + self.image_latents_temporal_encoder = I2VGenXLTransformerTemporalEncoder( + dim=in_channels, + num_attention_heads=2, + ff_inner_dim=in_channels * 4, + attention_head_dim=in_channels, + activation_fn="gelu", + ) + self.image_latents_context_embedding = nn.Sequential( + nn.Conv2d(4, in_channels * 8, 3, padding=1), + nn.SiLU(), + nn.AdaptiveAvgPool2d((32, 32)), + nn.Conv2d(in_channels * 8, in_channels * 16, 3, stride=2, padding=1), + nn.SiLU(), + nn.Conv2d(in_channels * 16, cross_attention_dim, 3, stride=2, padding=1), + ) + + # other embeddings -- time, context, fps, etc. + time_embed_dim = block_out_channels[0] * 4 + self.time_proj = Timesteps(block_out_channels[0], True, 0) + timestep_input_dim = block_out_channels[0] + + self.time_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim, act_fn="silu") + self.context_embedding = nn.Sequential( + nn.Linear(cross_attention_dim, time_embed_dim), + nn.SiLU(), + nn.Linear(time_embed_dim, cross_attention_dim * in_channels), + ) + self.fps_embedding = nn.Sequential( + nn.Linear(timestep_input_dim, time_embed_dim), nn.SiLU(), nn.Linear(time_embed_dim, time_embed_dim) + ) + + # blocks + self.down_blocks = nn.ModuleList([]) + self.up_blocks = nn.ModuleList([]) + + if isinstance(num_attention_heads, int): + num_attention_heads = (num_attention_heads,) * len(down_block_types) + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + temb_channels=time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=1e-05, + resnet_act_fn="silu", + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads[i], + downsample_padding=1, + dual_cross_attention=False, + ) + self.down_blocks.append(down_block) + + # mid + self.mid_block = UNetMidBlock3DCrossAttn( + in_channels=block_out_channels[-1], + temb_channels=time_embed_dim, + resnet_eps=1e-05, + resnet_act_fn="silu", + output_scale_factor=1, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads[-1], + resnet_groups=norm_num_groups, + dual_cross_attention=False, + ) + + # count how many layers upsample the images + self.num_upsamplers = 0 + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_num_attention_heads = list(reversed(num_attention_heads)) + + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + is_final_block = i == len(block_out_channels) - 1 + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)] + + # add upsample block for all BUT final layer + if not is_final_block: + add_upsample = True + self.num_upsamplers += 1 + else: + add_upsample = False + + up_block = get_up_block( + up_block_type, + num_layers=layers_per_block + 1, + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=time_embed_dim, + add_upsample=add_upsample, + resnet_eps=1e-05, + resnet_act_fn="silu", + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=reversed_num_attention_heads[i], + dual_cross_attention=False, + resolution_idx=i, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=1e-05) + self.conv_act = get_activation("silu") + self.conv_out = nn.Conv2d(block_out_channels[0], out_channels, kernel_size=3, padding=1) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_3d_condition.UNet3DConditionModel.enable_forward_chunking + def enable_forward_chunking(self, chunk_size: Optional[int] = None, dim: int = 0) -> None: + """ + Sets the attention processor to use [feed forward + chunking](https://huggingface.co/blog/reformer#2-chunked-feed-forward-layers). + + Parameters: + chunk_size (`int`, *optional*): + The chunk size of the feed-forward layers. If not specified, will run feed-forward layer individually + over each tensor of dim=`dim`. + dim (`int`, *optional*, defaults to `0`): + The dimension over which the feed-forward computation should be chunked. Choose between dim=0 (batch) + or dim=1 (sequence length). + """ + if dim not in [0, 1]: + raise ValueError(f"Make sure to set `dim` to either 0 or 1, not {dim}") + + # By default chunk size is 1 + chunk_size = chunk_size or 1 + + def fn_recursive_feed_forward(module: torch.nn.Module, chunk_size: int, dim: int): + if hasattr(module, "set_chunk_feed_forward"): + module.set_chunk_feed_forward(chunk_size=chunk_size, dim=dim) + + for child in module.children(): + fn_recursive_feed_forward(child, chunk_size, dim) + + for module in self.children(): + fn_recursive_feed_forward(module, chunk_size, dim) + + # Copied from diffusers.models.unets.unet_3d_condition.UNet3DConditionModel.disable_forward_chunking + def disable_forward_chunking(self): + def fn_recursive_feed_forward(module: torch.nn.Module, chunk_size: int, dim: int): + if hasattr(module, "set_chunk_feed_forward"): + module.set_chunk_feed_forward(chunk_size=chunk_size, dim=dim) + + for child in module.children(): + fn_recursive_feed_forward(child, chunk_size, dim) + + for module in self.children(): + fn_recursive_feed_forward(module, None, 0) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + # Copied from diffusers.models.unets.unet_3d_condition.UNet3DConditionModel._set_gradient_checkpointing + def _set_gradient_checkpointing(self, module, value: bool = False) -> None: + if isinstance(module, (CrossAttnDownBlock3D, DownBlock3D, CrossAttnUpBlock3D, UpBlock3D)): + module.gradient_checkpointing = value + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.enable_freeu + def enable_freeu(self, s1, s2, b1, b2): + r"""Enables the FreeU mechanism from https://arxiv.org/abs/2309.11497. + + The suffixes after the scaling factors represent the stage blocks where they are being applied. + + Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of values that + are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL. + + Args: + s1 (`float`): + Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + s2 (`float`): + Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. + b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. + """ + for i, upsample_block in enumerate(self.up_blocks): + setattr(upsample_block, "s1", s1) + setattr(upsample_block, "s2", s2) + setattr(upsample_block, "b1", b1) + setattr(upsample_block, "b2", b2) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.disable_freeu + def disable_freeu(self): + """Disables the FreeU mechanism.""" + freeu_keys = {"s1", "s2", "b1", "b2"} + for i, upsample_block in enumerate(self.up_blocks): + for k in freeu_keys: + if hasattr(upsample_block, k) or getattr(upsample_block, k, None) is not None: + setattr(upsample_block, k, None) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.fuse_qkv_projections + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, + key, value) are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is 🧪 experimental. + + + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.") + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.unfuse_qkv_projections + def unfuse_qkv_projections(self): + """Disables the fused QKV projection if enabled. + + + + This API is 🧪 experimental. + + + + """ + if self.original_attn_processors is not None: + self.set_attn_processor(self.original_attn_processors) + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + fps: torch.Tensor, + image_latents: torch.Tensor, + image_embeddings: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + return_dict: bool = True, + ) -> Union[UNet3DConditionOutput, Tuple[torch.FloatTensor]]: + r""" + The [`I2VGenXLUNet`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor with the following shape `(batch, num_frames, channel, height, width`. + timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input. + fps (`torch.Tensor`): Frames per second for the video being generated. Used as a "micro-condition". + image_latents (`torch.FloatTensor`): Image encodings from the VAE. + image_embeddings (`torch.FloatTensor`): Projection embeddings of the conditioning image computed with a vision encoder. + encoder_hidden_states (`torch.FloatTensor`): + The encoder hidden states with shape `(batch, sequence_length, feature_dim)`. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unet_3d_condition.UNet3DConditionOutput`] instead of a plain + tuple. + + Returns: + [`~models.unet_3d_condition.UNet3DConditionOutput`] or `tuple`: + If `return_dict` is True, an [`~models.unet_3d_condition.UNet3DConditionOutput`] is returned, otherwise + a `tuple` is returned where the first element is the sample tensor. + """ + batch_size, channels, num_frames, height, width = sample.shape + + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layears). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2**self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]): + logger.info("Forward upsample size to force interpolation output size.") + forward_upsample_size = True + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass `timesteps` as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == "mps" + if isinstance(timesteps, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + t_emb = self.time_proj(timesteps) + + # timesteps does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=self.dtype) + t_emb = self.time_embedding(t_emb, timestep_cond) + + # 2. FPS + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + fps = fps.expand(fps.shape[0]) + fps_emb = self.fps_embedding(self.time_proj(fps).to(dtype=self.dtype)) + + # 3. time + FPS embeddings. + emb = t_emb + fps_emb + emb = emb.repeat_interleave(repeats=num_frames, dim=0) + + # 4. context embeddings. + # The context embeddings consist of both text embeddings from the input prompt + # AND the image embeddings from the input image. For images, both VAE encodings + # and the CLIP image embeddings are incorporated. + # So the final `context_embeddings` becomes the query for cross-attention. + context_emb = sample.new_zeros(batch_size, 0, self.config.cross_attention_dim) + context_emb = torch.cat([context_emb, encoder_hidden_states], dim=1) + + image_latents_for_context_embds = image_latents[:, :, :1, :] + image_latents_context_embs = image_latents_for_context_embds.permute(0, 2, 1, 3, 4).reshape( + image_latents_for_context_embds.shape[0] * image_latents_for_context_embds.shape[2], + image_latents_for_context_embds.shape[1], + image_latents_for_context_embds.shape[3], + image_latents_for_context_embds.shape[4], + ) + image_latents_context_embs = self.image_latents_context_embedding(image_latents_context_embs) + + _batch_size, _channels, _height, _width = image_latents_context_embs.shape + image_latents_context_embs = image_latents_context_embs.permute(0, 2, 3, 1).reshape( + _batch_size, _height * _width, _channels + ) + context_emb = torch.cat([context_emb, image_latents_context_embs], dim=1) + + image_emb = self.context_embedding(image_embeddings) + image_emb = image_emb.view(-1, self.config.in_channels, self.config.cross_attention_dim) + context_emb = torch.cat([context_emb, image_emb], dim=1) + context_emb = context_emb.repeat_interleave(repeats=num_frames, dim=0) + + image_latents = image_latents.permute(0, 2, 1, 3, 4).reshape( + image_latents.shape[0] * image_latents.shape[2], + image_latents.shape[1], + image_latents.shape[3], + image_latents.shape[4], + ) + image_latents = self.image_latents_proj_in(image_latents) + image_latents = ( + image_latents[None, :] + .reshape(batch_size, num_frames, channels, height, width) + .permute(0, 3, 4, 1, 2) + .reshape(batch_size * height * width, num_frames, channels) + ) + image_latents = self.image_latents_temporal_encoder(image_latents) + image_latents = image_latents.reshape(batch_size, height, width, num_frames, channels).permute(0, 4, 3, 1, 2) + + # 5. pre-process + sample = torch.cat([sample, image_latents], dim=1) + sample = sample.permute(0, 2, 1, 3, 4).reshape((sample.shape[0] * num_frames, -1) + sample.shape[3:]) + sample = self.conv_in(sample) + sample = self.transformer_in( + sample, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # 6. down + down_block_res_samples = (sample,) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=context_emb, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb, num_frames=num_frames) + + down_block_res_samples += res_samples + + # 7. mid + if self.mid_block is not None: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=context_emb, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + # 8. up + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets) :] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=context_emb, + upsample_size=upsample_size, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + upsample_size=upsample_size, + num_frames=num_frames, + ) + + # 9. post-process + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + + sample = self.conv_out(sample) + + # reshape to (batch, channel, framerate, width, height) + sample = sample[None, :].reshape((-1, num_frames) + sample.shape[1:]).permute(0, 2, 1, 3, 4) + + if not return_dict: + return (sample,) + + return UNet3DConditionOutput(sample=sample) diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_kandinsky3.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_kandinsky3.py new file mode 100755 index 0000000..b981c8e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_kandinsky3.py @@ -0,0 +1,535 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Dict, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...utils import BaseOutput, logging +from ..attention_processor import Attention, AttentionProcessor, AttnProcessor +from ..embeddings import TimestepEmbedding, Timesteps +from ..modeling_utils import ModelMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class Kandinsky3UNetOutput(BaseOutput): + sample: torch.FloatTensor = None + + +class Kandinsky3EncoderProj(nn.Module): + def __init__(self, encoder_hid_dim, cross_attention_dim): + super().__init__() + self.projection_linear = nn.Linear(encoder_hid_dim, cross_attention_dim, bias=False) + self.projection_norm = nn.LayerNorm(cross_attention_dim) + + def forward(self, x): + x = self.projection_linear(x) + x = self.projection_norm(x) + return x + + +class Kandinsky3UNet(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + in_channels: int = 4, + time_embedding_dim: int = 1536, + groups: int = 32, + attention_head_dim: int = 64, + layers_per_block: Union[int, Tuple[int]] = 3, + block_out_channels: Tuple[int] = (384, 768, 1536, 3072), + cross_attention_dim: Union[int, Tuple[int]] = 4096, + encoder_hid_dim: int = 4096, + ): + super().__init__() + + # TOOD(Yiyi): Give better name and put into config for the following 4 parameters + expansion_ratio = 4 + compression_ratio = 2 + add_cross_attention = (False, True, True, True) + add_self_attention = (False, True, True, True) + + out_channels = in_channels + init_channels = block_out_channels[0] // 2 + self.time_proj = Timesteps(init_channels, flip_sin_to_cos=False, downscale_freq_shift=1) + + self.time_embedding = TimestepEmbedding( + init_channels, + time_embedding_dim, + ) + + self.add_time_condition = Kandinsky3AttentionPooling( + time_embedding_dim, cross_attention_dim, attention_head_dim + ) + + self.conv_in = nn.Conv2d(in_channels, init_channels, kernel_size=3, padding=1) + + self.encoder_hid_proj = Kandinsky3EncoderProj(encoder_hid_dim, cross_attention_dim) + + hidden_dims = [init_channels] + list(block_out_channels) + in_out_dims = list(zip(hidden_dims[:-1], hidden_dims[1:])) + text_dims = [cross_attention_dim if is_exist else None for is_exist in add_cross_attention] + num_blocks = len(block_out_channels) * [layers_per_block] + layer_params = [num_blocks, text_dims, add_self_attention] + rev_layer_params = map(reversed, layer_params) + + cat_dims = [] + self.num_levels = len(in_out_dims) + self.down_blocks = nn.ModuleList([]) + for level, ((in_dim, out_dim), res_block_num, text_dim, self_attention) in enumerate( + zip(in_out_dims, *layer_params) + ): + down_sample = level != (self.num_levels - 1) + cat_dims.append(out_dim if level != (self.num_levels - 1) else 0) + self.down_blocks.append( + Kandinsky3DownSampleBlock( + in_dim, + out_dim, + time_embedding_dim, + text_dim, + res_block_num, + groups, + attention_head_dim, + expansion_ratio, + compression_ratio, + down_sample, + self_attention, + ) + ) + + self.up_blocks = nn.ModuleList([]) + for level, ((out_dim, in_dim), res_block_num, text_dim, self_attention) in enumerate( + zip(reversed(in_out_dims), *rev_layer_params) + ): + up_sample = level != 0 + self.up_blocks.append( + Kandinsky3UpSampleBlock( + in_dim, + cat_dims.pop(), + out_dim, + time_embedding_dim, + text_dim, + res_block_num, + groups, + attention_head_dim, + expansion_ratio, + compression_ratio, + up_sample, + self_attention, + ) + ) + + self.conv_norm_out = nn.GroupNorm(groups, init_channels) + self.conv_act_out = nn.SiLU() + self.conv_out = nn.Conv2d(init_channels, out_channels, kernel_size=3, padding=1) + + @property + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "set_processor"): + processors[f"{name}.processor"] = module.processor + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + self.set_attn_processor(AttnProcessor()) + + def _set_gradient_checkpointing(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + + def forward(self, sample, timestep, encoder_hidden_states=None, encoder_attention_mask=None, return_dict=True): + if encoder_attention_mask is not None: + encoder_attention_mask = (1 - encoder_attention_mask.to(sample.dtype)) * -10000.0 + encoder_attention_mask = encoder_attention_mask.unsqueeze(1) + + if not torch.is_tensor(timestep): + dtype = torch.float32 if isinstance(timestep, float) else torch.int32 + timestep = torch.tensor([timestep], dtype=dtype, device=sample.device) + elif len(timestep.shape) == 0: + timestep = timestep[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep = timestep.expand(sample.shape[0]) + time_embed_input = self.time_proj(timestep).to(sample.dtype) + time_embed = self.time_embedding(time_embed_input) + + encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states) + + if encoder_hidden_states is not None: + time_embed = self.add_time_condition(time_embed, encoder_hidden_states, encoder_attention_mask) + + hidden_states = [] + sample = self.conv_in(sample) + for level, down_sample in enumerate(self.down_blocks): + sample = down_sample(sample, time_embed, encoder_hidden_states, encoder_attention_mask) + if level != self.num_levels - 1: + hidden_states.append(sample) + + for level, up_sample in enumerate(self.up_blocks): + if level != 0: + sample = torch.cat([sample, hidden_states.pop()], dim=1) + sample = up_sample(sample, time_embed, encoder_hidden_states, encoder_attention_mask) + + sample = self.conv_norm_out(sample) + sample = self.conv_act_out(sample) + sample = self.conv_out(sample) + + if not return_dict: + return (sample,) + return Kandinsky3UNetOutput(sample=sample) + + +class Kandinsky3UpSampleBlock(nn.Module): + def __init__( + self, + in_channels, + cat_dim, + out_channels, + time_embed_dim, + context_dim=None, + num_blocks=3, + groups=32, + head_dim=64, + expansion_ratio=4, + compression_ratio=2, + up_sample=True, + self_attention=True, + ): + super().__init__() + up_resolutions = [[None, True if up_sample else None, None, None]] + [[None] * 4] * (num_blocks - 1) + hidden_channels = ( + [(in_channels + cat_dim, in_channels)] + + [(in_channels, in_channels)] * (num_blocks - 2) + + [(in_channels, out_channels)] + ) + attentions = [] + resnets_in = [] + resnets_out = [] + + self.self_attention = self_attention + self.context_dim = context_dim + + if self_attention: + attentions.append( + Kandinsky3AttentionBlock(out_channels, time_embed_dim, None, groups, head_dim, expansion_ratio) + ) + else: + attentions.append(nn.Identity()) + + for (in_channel, out_channel), up_resolution in zip(hidden_channels, up_resolutions): + resnets_in.append( + Kandinsky3ResNetBlock(in_channel, in_channel, time_embed_dim, groups, compression_ratio, up_resolution) + ) + + if context_dim is not None: + attentions.append( + Kandinsky3AttentionBlock( + in_channel, time_embed_dim, context_dim, groups, head_dim, expansion_ratio + ) + ) + else: + attentions.append(nn.Identity()) + + resnets_out.append( + Kandinsky3ResNetBlock(in_channel, out_channel, time_embed_dim, groups, compression_ratio) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets_in = nn.ModuleList(resnets_in) + self.resnets_out = nn.ModuleList(resnets_out) + + def forward(self, x, time_embed, context=None, context_mask=None, image_mask=None): + for attention, resnet_in, resnet_out in zip(self.attentions[1:], self.resnets_in, self.resnets_out): + x = resnet_in(x, time_embed) + if self.context_dim is not None: + x = attention(x, time_embed, context, context_mask, image_mask) + x = resnet_out(x, time_embed) + + if self.self_attention: + x = self.attentions[0](x, time_embed, image_mask=image_mask) + return x + + +class Kandinsky3DownSampleBlock(nn.Module): + def __init__( + self, + in_channels, + out_channels, + time_embed_dim, + context_dim=None, + num_blocks=3, + groups=32, + head_dim=64, + expansion_ratio=4, + compression_ratio=2, + down_sample=True, + self_attention=True, + ): + super().__init__() + attentions = [] + resnets_in = [] + resnets_out = [] + + self.self_attention = self_attention + self.context_dim = context_dim + + if self_attention: + attentions.append( + Kandinsky3AttentionBlock(in_channels, time_embed_dim, None, groups, head_dim, expansion_ratio) + ) + else: + attentions.append(nn.Identity()) + + up_resolutions = [[None] * 4] * (num_blocks - 1) + [[None, None, False if down_sample else None, None]] + hidden_channels = [(in_channels, out_channels)] + [(out_channels, out_channels)] * (num_blocks - 1) + for (in_channel, out_channel), up_resolution in zip(hidden_channels, up_resolutions): + resnets_in.append( + Kandinsky3ResNetBlock(in_channel, out_channel, time_embed_dim, groups, compression_ratio) + ) + + if context_dim is not None: + attentions.append( + Kandinsky3AttentionBlock( + out_channel, time_embed_dim, context_dim, groups, head_dim, expansion_ratio + ) + ) + else: + attentions.append(nn.Identity()) + + resnets_out.append( + Kandinsky3ResNetBlock( + out_channel, out_channel, time_embed_dim, groups, compression_ratio, up_resolution + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets_in = nn.ModuleList(resnets_in) + self.resnets_out = nn.ModuleList(resnets_out) + + def forward(self, x, time_embed, context=None, context_mask=None, image_mask=None): + if self.self_attention: + x = self.attentions[0](x, time_embed, image_mask=image_mask) + + for attention, resnet_in, resnet_out in zip(self.attentions[1:], self.resnets_in, self.resnets_out): + x = resnet_in(x, time_embed) + if self.context_dim is not None: + x = attention(x, time_embed, context, context_mask, image_mask) + x = resnet_out(x, time_embed) + return x + + +class Kandinsky3ConditionalGroupNorm(nn.Module): + def __init__(self, groups, normalized_shape, context_dim): + super().__init__() + self.norm = nn.GroupNorm(groups, normalized_shape, affine=False) + self.context_mlp = nn.Sequential(nn.SiLU(), nn.Linear(context_dim, 2 * normalized_shape)) + self.context_mlp[1].weight.data.zero_() + self.context_mlp[1].bias.data.zero_() + + def forward(self, x, context): + context = self.context_mlp(context) + + for _ in range(len(x.shape[2:])): + context = context.unsqueeze(-1) + + scale, shift = context.chunk(2, dim=1) + x = self.norm(x) * (scale + 1.0) + shift + return x + + +class Kandinsky3Block(nn.Module): + def __init__(self, in_channels, out_channels, time_embed_dim, kernel_size=3, norm_groups=32, up_resolution=None): + super().__init__() + self.group_norm = Kandinsky3ConditionalGroupNorm(norm_groups, in_channels, time_embed_dim) + self.activation = nn.SiLU() + if up_resolution is not None and up_resolution: + self.up_sample = nn.ConvTranspose2d(in_channels, in_channels, kernel_size=2, stride=2) + else: + self.up_sample = nn.Identity() + + padding = int(kernel_size > 1) + self.projection = nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, padding=padding) + + if up_resolution is not None and not up_resolution: + self.down_sample = nn.Conv2d(out_channels, out_channels, kernel_size=2, stride=2) + else: + self.down_sample = nn.Identity() + + def forward(self, x, time_embed): + x = self.group_norm(x, time_embed) + x = self.activation(x) + x = self.up_sample(x) + x = self.projection(x) + x = self.down_sample(x) + return x + + +class Kandinsky3ResNetBlock(nn.Module): + def __init__( + self, in_channels, out_channels, time_embed_dim, norm_groups=32, compression_ratio=2, up_resolutions=4 * [None] + ): + super().__init__() + kernel_sizes = [1, 3, 3, 1] + hidden_channel = max(in_channels, out_channels) // compression_ratio + hidden_channels = ( + [(in_channels, hidden_channel)] + [(hidden_channel, hidden_channel)] * 2 + [(hidden_channel, out_channels)] + ) + self.resnet_blocks = nn.ModuleList( + [ + Kandinsky3Block(in_channel, out_channel, time_embed_dim, kernel_size, norm_groups, up_resolution) + for (in_channel, out_channel), kernel_size, up_resolution in zip( + hidden_channels, kernel_sizes, up_resolutions + ) + ] + ) + self.shortcut_up_sample = ( + nn.ConvTranspose2d(in_channels, in_channels, kernel_size=2, stride=2) + if True in up_resolutions + else nn.Identity() + ) + self.shortcut_projection = ( + nn.Conv2d(in_channels, out_channels, kernel_size=1) if in_channels != out_channels else nn.Identity() + ) + self.shortcut_down_sample = ( + nn.Conv2d(out_channels, out_channels, kernel_size=2, stride=2) + if False in up_resolutions + else nn.Identity() + ) + + def forward(self, x, time_embed): + out = x + for resnet_block in self.resnet_blocks: + out = resnet_block(out, time_embed) + + x = self.shortcut_up_sample(x) + x = self.shortcut_projection(x) + x = self.shortcut_down_sample(x) + x = x + out + return x + + +class Kandinsky3AttentionPooling(nn.Module): + def __init__(self, num_channels, context_dim, head_dim=64): + super().__init__() + self.attention = Attention( + context_dim, + context_dim, + dim_head=head_dim, + out_dim=num_channels, + out_bias=False, + ) + + def forward(self, x, context, context_mask=None): + context_mask = context_mask.to(dtype=context.dtype) + context = self.attention(context.mean(dim=1, keepdim=True), context, context_mask) + return x + context.squeeze(1) + + +class Kandinsky3AttentionBlock(nn.Module): + def __init__(self, num_channels, time_embed_dim, context_dim=None, norm_groups=32, head_dim=64, expansion_ratio=4): + super().__init__() + self.in_norm = Kandinsky3ConditionalGroupNorm(norm_groups, num_channels, time_embed_dim) + self.attention = Attention( + num_channels, + context_dim or num_channels, + dim_head=head_dim, + out_dim=num_channels, + out_bias=False, + ) + + hidden_channels = expansion_ratio * num_channels + self.out_norm = Kandinsky3ConditionalGroupNorm(norm_groups, num_channels, time_embed_dim) + self.feed_forward = nn.Sequential( + nn.Conv2d(num_channels, hidden_channels, kernel_size=1, bias=False), + nn.SiLU(), + nn.Conv2d(hidden_channels, num_channels, kernel_size=1, bias=False), + ) + + def forward(self, x, time_embed, context=None, context_mask=None, image_mask=None): + height, width = x.shape[-2:] + out = self.in_norm(x, time_embed) + out = out.reshape(x.shape[0], -1, height * width).permute(0, 2, 1) + context = context if context is not None else out + if context_mask is not None: + context_mask = context_mask.to(dtype=context.dtype) + + out = self.attention(out, context, context_mask) + out = out.permute(0, 2, 1).unsqueeze(-1).reshape(out.shape[0], -1, height, width) + x = x + out + + out = self.out_norm(x, time_embed) + out = self.feed_forward(out) + x = x + out + return x diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_motion_model.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_motion_model.py new file mode 100755 index 0000000..ab2eac4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_motion_model.py @@ -0,0 +1,948 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Any, Dict, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.utils.checkpoint + +from ...configuration_utils import ConfigMixin, register_to_config +from ...loaders import UNet2DConditionLoadersMixin +from ...utils import logging +from ..attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + Attention, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from ..embeddings import TimestepEmbedding, Timesteps +from ..modeling_utils import ModelMixin +from ..transformers.transformer_temporal import TransformerTemporalModel +from .unet_2d_blocks import UNetMidBlock2DCrossAttn +from .unet_2d_condition import UNet2DConditionModel +from .unet_3d_blocks import ( + CrossAttnDownBlockMotion, + CrossAttnUpBlockMotion, + DownBlockMotion, + UNetMidBlockCrossAttnMotion, + UpBlockMotion, + get_down_block, + get_up_block, +) +from .unet_3d_condition import UNet3DConditionOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class MotionModules(nn.Module): + def __init__( + self, + in_channels: int, + layers_per_block: int = 2, + num_attention_heads: int = 8, + attention_bias: bool = False, + cross_attention_dim: Optional[int] = None, + activation_fn: str = "geglu", + norm_num_groups: int = 32, + max_seq_length: int = 32, + ): + super().__init__() + self.motion_modules = nn.ModuleList([]) + + for i in range(layers_per_block): + self.motion_modules.append( + TransformerTemporalModel( + in_channels=in_channels, + norm_num_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + activation_fn=activation_fn, + attention_bias=attention_bias, + num_attention_heads=num_attention_heads, + attention_head_dim=in_channels // num_attention_heads, + positional_embeddings="sinusoidal", + num_positional_embeddings=max_seq_length, + ) + ) + + +class MotionAdapter(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + motion_layers_per_block: int = 2, + motion_mid_block_layers_per_block: int = 1, + motion_num_attention_heads: int = 8, + motion_norm_num_groups: int = 32, + motion_max_seq_length: int = 32, + use_motion_mid_block: bool = True, + conv_in_channels: Optional[int] = None, + ): + """Container to store AnimateDiff Motion Modules + + Args: + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each UNet block. + motion_layers_per_block (`int`, *optional*, defaults to 2): + The number of motion layers per UNet block. + motion_mid_block_layers_per_block (`int`, *optional*, defaults to 1): + The number of motion layers in the middle UNet block. + motion_num_attention_heads (`int`, *optional*, defaults to 8): + The number of heads to use in each attention layer of the motion module. + motion_norm_num_groups (`int`, *optional*, defaults to 32): + The number of groups to use in each group normalization layer of the motion module. + motion_max_seq_length (`int`, *optional*, defaults to 32): + The maximum sequence length to use in the motion module. + use_motion_mid_block (`bool`, *optional*, defaults to True): + Whether to use a motion module in the middle of the UNet. + """ + + super().__init__() + down_blocks = [] + up_blocks = [] + + if conv_in_channels: + # input + self.conv_in = nn.Conv2d(conv_in_channels, block_out_channels[0], kernel_size=3, padding=1) + else: + self.conv_in = None + + for i, channel in enumerate(block_out_channels): + output_channel = block_out_channels[i] + down_blocks.append( + MotionModules( + in_channels=output_channel, + norm_num_groups=motion_norm_num_groups, + cross_attention_dim=None, + activation_fn="geglu", + attention_bias=False, + num_attention_heads=motion_num_attention_heads, + max_seq_length=motion_max_seq_length, + layers_per_block=motion_layers_per_block, + ) + ) + + if use_motion_mid_block: + self.mid_block = MotionModules( + in_channels=block_out_channels[-1], + norm_num_groups=motion_norm_num_groups, + cross_attention_dim=None, + activation_fn="geglu", + attention_bias=False, + num_attention_heads=motion_num_attention_heads, + layers_per_block=motion_mid_block_layers_per_block, + max_seq_length=motion_max_seq_length, + ) + else: + self.mid_block = None + + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + for i, channel in enumerate(reversed_block_out_channels): + output_channel = reversed_block_out_channels[i] + up_blocks.append( + MotionModules( + in_channels=output_channel, + norm_num_groups=motion_norm_num_groups, + cross_attention_dim=None, + activation_fn="geglu", + attention_bias=False, + num_attention_heads=motion_num_attention_heads, + max_seq_length=motion_max_seq_length, + layers_per_block=motion_layers_per_block + 1, + ) + ) + + self.down_blocks = nn.ModuleList(down_blocks) + self.up_blocks = nn.ModuleList(up_blocks) + + def forward(self, sample): + pass + + +class UNetMotionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin): + r""" + A modified conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a + sample shaped output. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + sample_size: Optional[int] = None, + in_channels: int = 4, + out_channels: int = 4, + down_block_types: Tuple[str, ...] = ( + "CrossAttnDownBlockMotion", + "CrossAttnDownBlockMotion", + "CrossAttnDownBlockMotion", + "DownBlockMotion", + ), + up_block_types: Tuple[str, ...] = ( + "UpBlockMotion", + "CrossAttnUpBlockMotion", + "CrossAttnUpBlockMotion", + "CrossAttnUpBlockMotion", + ), + block_out_channels: Tuple[int, ...] = (320, 640, 1280, 1280), + layers_per_block: int = 2, + downsample_padding: int = 1, + mid_block_scale_factor: float = 1, + act_fn: str = "silu", + norm_num_groups: int = 32, + norm_eps: float = 1e-5, + cross_attention_dim: int = 1280, + use_linear_projection: bool = False, + num_attention_heads: Union[int, Tuple[int, ...]] = 8, + motion_max_seq_length: int = 32, + motion_num_attention_heads: int = 8, + use_motion_mid_block: int = True, + encoder_hid_dim: Optional[int] = None, + encoder_hid_dim_type: Optional[str] = None, + time_cond_proj_dim: Optional[int] = None, + ): + super().__init__() + + self.sample_size = sample_size + + # Check inputs + if len(down_block_types) != len(up_block_types): + raise ValueError( + f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}." + ) + + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}." + ) + + # input + conv_in_kernel = 3 + conv_out_kernel = 3 + conv_in_padding = (conv_in_kernel - 1) // 2 + self.conv_in = nn.Conv2d( + in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding + ) + + # time + time_embed_dim = block_out_channels[0] * 4 + self.time_proj = Timesteps(block_out_channels[0], True, 0) + timestep_input_dim = block_out_channels[0] + + self.time_embedding = TimestepEmbedding( + timestep_input_dim, time_embed_dim, act_fn=act_fn, cond_proj_dim=time_cond_proj_dim + ) + + if encoder_hid_dim_type is None: + self.encoder_hid_proj = None + + # class embedding + self.down_blocks = nn.ModuleList([]) + self.up_blocks = nn.ModuleList([]) + + if isinstance(num_attention_heads, int): + num_attention_heads = (num_attention_heads,) * len(down_block_types) + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block, + in_channels=input_channel, + out_channels=output_channel, + temb_channels=time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads[i], + downsample_padding=downsample_padding, + use_linear_projection=use_linear_projection, + dual_cross_attention=False, + temporal_num_attention_heads=motion_num_attention_heads, + temporal_max_seq_length=motion_max_seq_length, + ) + self.down_blocks.append(down_block) + + # mid + if use_motion_mid_block: + self.mid_block = UNetMidBlockCrossAttnMotion( + in_channels=block_out_channels[-1], + temb_channels=time_embed_dim, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads[-1], + resnet_groups=norm_num_groups, + dual_cross_attention=False, + use_linear_projection=use_linear_projection, + temporal_num_attention_heads=motion_num_attention_heads, + temporal_max_seq_length=motion_max_seq_length, + ) + + else: + self.mid_block = UNetMidBlock2DCrossAttn( + in_channels=block_out_channels[-1], + temb_channels=time_embed_dim, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads[-1], + resnet_groups=norm_num_groups, + dual_cross_attention=False, + use_linear_projection=use_linear_projection, + ) + + # count how many layers upsample the images + self.num_upsamplers = 0 + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_num_attention_heads = list(reversed(num_attention_heads)) + + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + is_final_block = i == len(block_out_channels) - 1 + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)] + + # add upsample block for all BUT final layer + if not is_final_block: + add_upsample = True + self.num_upsamplers += 1 + else: + add_upsample = False + + up_block = get_up_block( + up_block_type, + num_layers=layers_per_block + 1, + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=time_embed_dim, + add_upsample=add_upsample, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=reversed_num_attention_heads[i], + dual_cross_attention=False, + resolution_idx=i, + use_linear_projection=use_linear_projection, + temporal_num_attention_heads=motion_num_attention_heads, + temporal_max_seq_length=motion_max_seq_length, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_num_groups is not None: + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps + ) + self.conv_act = nn.SiLU() + else: + self.conv_norm_out = None + self.conv_act = None + + conv_out_padding = (conv_out_kernel - 1) // 2 + self.conv_out = nn.Conv2d( + block_out_channels[0], out_channels, kernel_size=conv_out_kernel, padding=conv_out_padding + ) + + @classmethod + def from_unet2d( + cls, + unet: UNet2DConditionModel, + motion_adapter: Optional[MotionAdapter] = None, + load_weights: bool = True, + ): + has_motion_adapter = motion_adapter is not None + + # based on https://github.com/guoyww/AnimateDiff/blob/895f3220c06318ea0760131ec70408b466c49333/animatediff/models/unet.py#L459 + config = unet.config + config["_class_name"] = cls.__name__ + + down_blocks = [] + for down_blocks_type in config["down_block_types"]: + if "CrossAttn" in down_blocks_type: + down_blocks.append("CrossAttnDownBlockMotion") + else: + down_blocks.append("DownBlockMotion") + config["down_block_types"] = down_blocks + + up_blocks = [] + for down_blocks_type in config["up_block_types"]: + if "CrossAttn" in down_blocks_type: + up_blocks.append("CrossAttnUpBlockMotion") + else: + up_blocks.append("UpBlockMotion") + + config["up_block_types"] = up_blocks + + if has_motion_adapter: + config["motion_num_attention_heads"] = motion_adapter.config["motion_num_attention_heads"] + config["motion_max_seq_length"] = motion_adapter.config["motion_max_seq_length"] + config["use_motion_mid_block"] = motion_adapter.config["use_motion_mid_block"] + + # For PIA UNets we need to set the number input channels to 9 + if motion_adapter.config["conv_in_channels"]: + config["in_channels"] = motion_adapter.config["conv_in_channels"] + + # Need this for backwards compatibility with UNet2DConditionModel checkpoints + if not config.get("num_attention_heads"): + config["num_attention_heads"] = config["attention_head_dim"] + + model = cls.from_config(config) + + if not load_weights: + return model + + # Logic for loading PIA UNets which allow the first 4 channels to be any UNet2DConditionModel conv_in weight + # while the last 5 channels must be PIA conv_in weights. + if has_motion_adapter and motion_adapter.config["conv_in_channels"]: + model.conv_in = motion_adapter.conv_in + updated_conv_in_weight = torch.cat( + [unet.conv_in.weight, motion_adapter.conv_in.weight[:, 4:, :, :]], dim=1 + ) + model.conv_in.load_state_dict({"weight": updated_conv_in_weight, "bias": unet.conv_in.bias}) + else: + model.conv_in.load_state_dict(unet.conv_in.state_dict()) + + model.time_proj.load_state_dict(unet.time_proj.state_dict()) + model.time_embedding.load_state_dict(unet.time_embedding.state_dict()) + + for i, down_block in enumerate(unet.down_blocks): + model.down_blocks[i].resnets.load_state_dict(down_block.resnets.state_dict()) + if hasattr(model.down_blocks[i], "attentions"): + model.down_blocks[i].attentions.load_state_dict(down_block.attentions.state_dict()) + if model.down_blocks[i].downsamplers: + model.down_blocks[i].downsamplers.load_state_dict(down_block.downsamplers.state_dict()) + + for i, up_block in enumerate(unet.up_blocks): + model.up_blocks[i].resnets.load_state_dict(up_block.resnets.state_dict()) + if hasattr(model.up_blocks[i], "attentions"): + model.up_blocks[i].attentions.load_state_dict(up_block.attentions.state_dict()) + if model.up_blocks[i].upsamplers: + model.up_blocks[i].upsamplers.load_state_dict(up_block.upsamplers.state_dict()) + + model.mid_block.resnets.load_state_dict(unet.mid_block.resnets.state_dict()) + model.mid_block.attentions.load_state_dict(unet.mid_block.attentions.state_dict()) + + if unet.conv_norm_out is not None: + model.conv_norm_out.load_state_dict(unet.conv_norm_out.state_dict()) + if unet.conv_act is not None: + model.conv_act.load_state_dict(unet.conv_act.state_dict()) + model.conv_out.load_state_dict(unet.conv_out.state_dict()) + + if has_motion_adapter: + model.load_motion_modules(motion_adapter) + + # ensure that the Motion UNet is the same dtype as the UNet2DConditionModel + model.to(unet.dtype) + + return model + + def freeze_unet2d_params(self) -> None: + """Freeze the weights of just the UNet2DConditionModel, and leave the motion modules + unfrozen for fine tuning. + """ + # Freeze everything + for param in self.parameters(): + param.requires_grad = False + + # Unfreeze Motion Modules + for down_block in self.down_blocks: + motion_modules = down_block.motion_modules + for param in motion_modules.parameters(): + param.requires_grad = True + + for up_block in self.up_blocks: + motion_modules = up_block.motion_modules + for param in motion_modules.parameters(): + param.requires_grad = True + + if hasattr(self.mid_block, "motion_modules"): + motion_modules = self.mid_block.motion_modules + for param in motion_modules.parameters(): + param.requires_grad = True + + def load_motion_modules(self, motion_adapter: Optional[MotionAdapter]) -> None: + for i, down_block in enumerate(motion_adapter.down_blocks): + self.down_blocks[i].motion_modules.load_state_dict(down_block.motion_modules.state_dict()) + for i, up_block in enumerate(motion_adapter.up_blocks): + self.up_blocks[i].motion_modules.load_state_dict(up_block.motion_modules.state_dict()) + + # to support older motion modules that don't have a mid_block + if hasattr(self.mid_block, "motion_modules"): + self.mid_block.motion_modules.load_state_dict(motion_adapter.mid_block.motion_modules.state_dict()) + + def save_motion_modules( + self, + save_directory: str, + is_main_process: bool = True, + safe_serialization: bool = True, + variant: Optional[str] = None, + push_to_hub: bool = False, + **kwargs, + ) -> None: + state_dict = self.state_dict() + + # Extract all motion modules + motion_state_dict = {} + for k, v in state_dict.items(): + if "motion_modules" in k: + motion_state_dict[k] = v + + adapter = MotionAdapter( + block_out_channels=self.config["block_out_channels"], + motion_layers_per_block=self.config["layers_per_block"], + motion_norm_num_groups=self.config["norm_num_groups"], + motion_num_attention_heads=self.config["motion_num_attention_heads"], + motion_max_seq_length=self.config["motion_max_seq_length"], + use_motion_mid_block=self.config["use_motion_mid_block"], + ) + adapter.load_state_dict(motion_state_dict) + adapter.save_pretrained( + save_directory=save_directory, + is_main_process=is_main_process, + safe_serialization=safe_serialization, + variant=variant, + push_to_hub=push_to_hub, + **kwargs, + ) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_3d_condition.UNet3DConditionModel.enable_forward_chunking + def enable_forward_chunking(self, chunk_size: Optional[int] = None, dim: int = 0) -> None: + """ + Sets the attention processor to use [feed forward + chunking](https://huggingface.co/blog/reformer#2-chunked-feed-forward-layers). + + Parameters: + chunk_size (`int`, *optional*): + The chunk size of the feed-forward layers. If not specified, will run feed-forward layer individually + over each tensor of dim=`dim`. + dim (`int`, *optional*, defaults to `0`): + The dimension over which the feed-forward computation should be chunked. Choose between dim=0 (batch) + or dim=1 (sequence length). + """ + if dim not in [0, 1]: + raise ValueError(f"Make sure to set `dim` to either 0 or 1, not {dim}") + + # By default chunk size is 1 + chunk_size = chunk_size or 1 + + def fn_recursive_feed_forward(module: torch.nn.Module, chunk_size: int, dim: int): + if hasattr(module, "set_chunk_feed_forward"): + module.set_chunk_feed_forward(chunk_size=chunk_size, dim=dim) + + for child in module.children(): + fn_recursive_feed_forward(child, chunk_size, dim) + + for module in self.children(): + fn_recursive_feed_forward(module, chunk_size, dim) + + # Copied from diffusers.models.unets.unet_3d_condition.UNet3DConditionModel.disable_forward_chunking + def disable_forward_chunking(self) -> None: + def fn_recursive_feed_forward(module: torch.nn.Module, chunk_size: int, dim: int): + if hasattr(module, "set_chunk_feed_forward"): + module.set_chunk_feed_forward(chunk_size=chunk_size, dim=dim) + + for child in module.children(): + fn_recursive_feed_forward(child, chunk_size, dim) + + for module in self.children(): + fn_recursive_feed_forward(module, None, 0) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self) -> None: + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + def _set_gradient_checkpointing(self, module, value: bool = False) -> None: + if isinstance(module, (CrossAttnDownBlockMotion, DownBlockMotion, CrossAttnUpBlockMotion, UpBlockMotion)): + module.gradient_checkpointing = value + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.enable_freeu + def enable_freeu(self, s1: float, s2: float, b1: float, b2: float) -> None: + r"""Enables the FreeU mechanism from https://arxiv.org/abs/2309.11497. + + The suffixes after the scaling factors represent the stage blocks where they are being applied. + + Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of values that + are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL. + + Args: + s1 (`float`): + Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + s2 (`float`): + Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. + b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. + """ + for i, upsample_block in enumerate(self.up_blocks): + setattr(upsample_block, "s1", s1) + setattr(upsample_block, "s2", s2) + setattr(upsample_block, "b1", b1) + setattr(upsample_block, "b2", b2) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.disable_freeu + def disable_freeu(self) -> None: + """Disables the FreeU mechanism.""" + freeu_keys = {"s1", "s2", "b1", "b2"} + for i, upsample_block in enumerate(self.up_blocks): + for k in freeu_keys: + if hasattr(upsample_block, k) or getattr(upsample_block, k, None) is not None: + setattr(upsample_block, k, None) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.fuse_qkv_projections + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, + key, value) are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is 🧪 experimental. + + + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.") + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.unfuse_qkv_projections + def unfuse_qkv_projections(self): + """Disables the fused QKV projection if enabled. + + + + This API is 🧪 experimental. + + + + """ + if self.original_attn_processors is not None: + self.set_attn_processor(self.original_attn_processors) + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + mid_block_additional_residual: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[UNet3DConditionOutput, Tuple[torch.Tensor]]: + r""" + The [`UNetMotionModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor with the following shape `(batch, num_frames, channel, height, width`. + timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input. + encoder_hidden_states (`torch.FloatTensor`): + The encoder hidden states with shape `(batch, sequence_length, feature_dim)`. + timestep_cond: (`torch.Tensor`, *optional*, defaults to `None`): + Conditional embeddings for timestep. If provided, the embeddings will be summed with the samples passed + through the `self.time_embedding` layer to obtain the timestep embeddings. + attention_mask (`torch.Tensor`, *optional*, defaults to `None`): + An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask + is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large + negative values to the attention scores corresponding to "discard" tokens. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + down_block_additional_residuals: (`tuple` of `torch.Tensor`, *optional*): + A tuple of tensors that if specified are added to the residuals of down unet blocks. + mid_block_additional_residual: (`torch.Tensor`, *optional*): + A tensor that if specified is added to the residual of the middle unet block. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unet_3d_condition.UNet3DConditionOutput`] instead of a plain + tuple. + + Returns: + [`~models.unet_3d_condition.UNet3DConditionOutput`] or `tuple`: + If `return_dict` is True, an [`~models.unet_3d_condition.UNet3DConditionOutput`] is returned, otherwise + a `tuple` is returned where the first element is the sample tensor. + """ + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layears). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2**self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]): + logger.info("Forward upsample size to force interpolation output size.") + forward_upsample_size = True + + # prepare attention_mask + if attention_mask is not None: + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == "mps" + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + num_frames = sample.shape[2] + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # timesteps does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=self.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + emb = emb.repeat_interleave(repeats=num_frames, dim=0) + encoder_hidden_states = encoder_hidden_states.repeat_interleave(repeats=num_frames, dim=0) + + if self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "ip_image_proj": + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'ip_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + image_embeds = added_cond_kwargs.get("image_embeds") + image_embeds = self.encoder_hid_proj(image_embeds) + image_embeds = [image_embed.repeat_interleave(repeats=num_frames, dim=0) for image_embed in image_embeds] + encoder_hidden_states = (encoder_hidden_states, image_embeds) + + # 2. pre-process + sample = sample.permute(0, 2, 1, 3, 4).reshape((sample.shape[0] * num_frames, -1) + sample.shape[3:]) + sample = self.conv_in(sample) + + # 3. down + down_block_res_samples = (sample,) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb, num_frames=num_frames) + + down_block_res_samples += res_samples + + if down_block_additional_residuals is not None: + new_down_block_res_samples = () + + for down_block_res_sample, down_block_additional_residual in zip( + down_block_res_samples, down_block_additional_residuals + ): + down_block_res_sample = down_block_res_sample + down_block_additional_residual + new_down_block_res_samples += (down_block_res_sample,) + + down_block_res_samples = new_down_block_res_samples + + # 4. mid + if self.mid_block is not None: + # To support older versions of motion modules that don't have a mid_block + if hasattr(self.mid_block, "motion_modules"): + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + ) + + if mid_block_additional_residual is not None: + sample = sample + mid_block_additional_residual + + # 5. up + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets) :] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=encoder_hidden_states, + upsample_size=upsample_size, + attention_mask=attention_mask, + num_frames=num_frames, + cross_attention_kwargs=cross_attention_kwargs, + ) + else: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + upsample_size=upsample_size, + num_frames=num_frames, + ) + + # 6. post-process + if self.conv_norm_out: + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + + sample = self.conv_out(sample) + + # reshape to (batch, channel, framerate, width, height) + sample = sample[None, :].reshape((-1, num_frames) + sample.shape[1:]).permute(0, 2, 1, 3, 4) + + if not return_dict: + return (sample,) + + return UNet3DConditionOutput(sample=sample) diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_spatio_temporal_condition.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_spatio_temporal_condition.py new file mode 100755 index 0000000..5fe265e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_spatio_temporal_condition.py @@ -0,0 +1,489 @@ +from dataclasses import dataclass +from typing import Dict, Optional, Tuple, Union + +import torch +import torch.nn as nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...loaders import UNet2DConditionLoadersMixin +from ...utils import BaseOutput, logging +from ..attention_processor import CROSS_ATTENTION_PROCESSORS, AttentionProcessor, AttnProcessor +from ..embeddings import TimestepEmbedding, Timesteps +from ..modeling_utils import ModelMixin +from .unet_3d_blocks import UNetMidBlockSpatioTemporal, get_down_block, get_up_block + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class UNetSpatioTemporalConditionOutput(BaseOutput): + """ + The output of [`UNetSpatioTemporalConditionModel`]. + + Args: + sample (`torch.FloatTensor` of shape `(batch_size, num_frames, num_channels, height, width)`): + The hidden states output conditioned on `encoder_hidden_states` input. Output of last layer of model. + """ + + sample: torch.FloatTensor = None + + +class UNetSpatioTemporalConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin): + r""" + A conditional Spatio-Temporal UNet model that takes a noisy video frames, conditional state, and a timestep and returns a sample + shaped output. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + Height and width of input/output sample. + in_channels (`int`, *optional*, defaults to 8): Number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 4): Number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlockSpatioTemporal", "CrossAttnDownBlockSpatioTemporal", "CrossAttnDownBlockSpatioTemporal", "DownBlockSpatioTemporal")`): + The tuple of downsample blocks to use. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal")`): + The tuple of upsample blocks to use. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + addition_time_embed_dim: (`int`, defaults to 256): + Dimension to to encode the additional time ids. + projection_class_embeddings_input_dim (`int`, defaults to 768): + The dimension of the projection of encoded `added_time_ids`. + layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. + cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280): + The dimension of the cross attention features. + transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple]` , *optional*, defaults to 1): + The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for + [`~models.unet_3d_blocks.CrossAttnDownBlockSpatioTemporal`], [`~models.unet_3d_blocks.CrossAttnUpBlockSpatioTemporal`], + [`~models.unet_3d_blocks.UNetMidBlockSpatioTemporal`]. + num_attention_heads (`int`, `Tuple[int]`, defaults to `(5, 10, 10, 20)`): + The number of attention heads. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + sample_size: Optional[int] = None, + in_channels: int = 8, + out_channels: int = 4, + down_block_types: Tuple[str] = ( + "CrossAttnDownBlockSpatioTemporal", + "CrossAttnDownBlockSpatioTemporal", + "CrossAttnDownBlockSpatioTemporal", + "DownBlockSpatioTemporal", + ), + up_block_types: Tuple[str] = ( + "UpBlockSpatioTemporal", + "CrossAttnUpBlockSpatioTemporal", + "CrossAttnUpBlockSpatioTemporal", + "CrossAttnUpBlockSpatioTemporal", + ), + block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + addition_time_embed_dim: int = 256, + projection_class_embeddings_input_dim: int = 768, + layers_per_block: Union[int, Tuple[int]] = 2, + cross_attention_dim: Union[int, Tuple[int]] = 1024, + transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1, + num_attention_heads: Union[int, Tuple[int]] = (5, 10, 20, 20), + num_frames: int = 25, + ): + super().__init__() + + self.sample_size = sample_size + + # Check inputs + if len(down_block_types) != len(up_block_types): + raise ValueError( + f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}." + ) + + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}." + ) + + if isinstance(cross_attention_dim, list) and len(cross_attention_dim) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `cross_attention_dim` as `down_block_types`. `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(layers_per_block, int) and len(layers_per_block) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `layers_per_block` as `down_block_types`. `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}." + ) + + # input + self.conv_in = nn.Conv2d( + in_channels, + block_out_channels[0], + kernel_size=3, + padding=1, + ) + + # time + time_embed_dim = block_out_channels[0] * 4 + + self.time_proj = Timesteps(block_out_channels[0], True, downscale_freq_shift=0) + timestep_input_dim = block_out_channels[0] + + self.time_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim) + + self.add_time_proj = Timesteps(addition_time_embed_dim, True, downscale_freq_shift=0) + self.add_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim) + + self.down_blocks = nn.ModuleList([]) + self.up_blocks = nn.ModuleList([]) + + if isinstance(num_attention_heads, int): + num_attention_heads = (num_attention_heads,) * len(down_block_types) + + if isinstance(cross_attention_dim, int): + cross_attention_dim = (cross_attention_dim,) * len(down_block_types) + + if isinstance(layers_per_block, int): + layers_per_block = [layers_per_block] * len(down_block_types) + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * len(down_block_types) + + blocks_time_embed_dim = time_embed_dim + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block[i], + transformer_layers_per_block=transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + temb_channels=blocks_time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=1e-5, + cross_attention_dim=cross_attention_dim[i], + num_attention_heads=num_attention_heads[i], + resnet_act_fn="silu", + ) + self.down_blocks.append(down_block) + + # mid + self.mid_block = UNetMidBlockSpatioTemporal( + block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + transformer_layers_per_block=transformer_layers_per_block[-1], + cross_attention_dim=cross_attention_dim[-1], + num_attention_heads=num_attention_heads[-1], + ) + + # count how many layers upsample the images + self.num_upsamplers = 0 + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_num_attention_heads = list(reversed(num_attention_heads)) + reversed_layers_per_block = list(reversed(layers_per_block)) + reversed_cross_attention_dim = list(reversed(cross_attention_dim)) + reversed_transformer_layers_per_block = list(reversed(transformer_layers_per_block)) + + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + is_final_block = i == len(block_out_channels) - 1 + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)] + + # add upsample block for all BUT final layer + if not is_final_block: + add_upsample = True + self.num_upsamplers += 1 + else: + add_upsample = False + + up_block = get_up_block( + up_block_type, + num_layers=reversed_layers_per_block[i] + 1, + transformer_layers_per_block=reversed_transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=blocks_time_embed_dim, + add_upsample=add_upsample, + resnet_eps=1e-5, + resolution_idx=i, + cross_attention_dim=reversed_cross_attention_dim[i], + num_attention_heads=reversed_num_attention_heads[i], + resnet_act_fn="silu", + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + self.conv_norm_out = nn.GroupNorm(num_channels=block_out_channels[0], num_groups=32, eps=1e-5) + self.conv_act = nn.SiLU() + + self.conv_out = nn.Conv2d( + block_out_channels[0], + out_channels, + kernel_size=3, + padding=1, + ) + + @property + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors( + name: str, + module: torch.nn.Module, + processors: Dict[str, AttentionProcessor], + ): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + def _set_gradient_checkpointing(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + + # Copied from diffusers.models.unets.unet_3d_condition.UNet3DConditionModel.enable_forward_chunking + def enable_forward_chunking(self, chunk_size: Optional[int] = None, dim: int = 0) -> None: + """ + Sets the attention processor to use [feed forward + chunking](https://huggingface.co/blog/reformer#2-chunked-feed-forward-layers). + + Parameters: + chunk_size (`int`, *optional*): + The chunk size of the feed-forward layers. If not specified, will run feed-forward layer individually + over each tensor of dim=`dim`. + dim (`int`, *optional*, defaults to `0`): + The dimension over which the feed-forward computation should be chunked. Choose between dim=0 (batch) + or dim=1 (sequence length). + """ + if dim not in [0, 1]: + raise ValueError(f"Make sure to set `dim` to either 0 or 1, not {dim}") + + # By default chunk size is 1 + chunk_size = chunk_size or 1 + + def fn_recursive_feed_forward(module: torch.nn.Module, chunk_size: int, dim: int): + if hasattr(module, "set_chunk_feed_forward"): + module.set_chunk_feed_forward(chunk_size=chunk_size, dim=dim) + + for child in module.children(): + fn_recursive_feed_forward(child, chunk_size, dim) + + for module in self.children(): + fn_recursive_feed_forward(module, chunk_size, dim) + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + added_time_ids: torch.Tensor, + return_dict: bool = True, + ) -> Union[UNetSpatioTemporalConditionOutput, Tuple]: + r""" + The [`UNetSpatioTemporalConditionModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor with the following shape `(batch, num_frames, channel, height, width)`. + timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input. + encoder_hidden_states (`torch.FloatTensor`): + The encoder hidden states with shape `(batch, sequence_length, cross_attention_dim)`. + added_time_ids: (`torch.FloatTensor`): + The additional time ids with shape `(batch, num_additional_ids)`. These are encoded with sinusoidal + embeddings and added to the time embeddings. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unet_slatio_temporal.UNetSpatioTemporalConditionOutput`] instead of a plain + tuple. + Returns: + [`~models.unet_slatio_temporal.UNetSpatioTemporalConditionOutput`] or `tuple`: + If `return_dict` is True, an [`~models.unet_slatio_temporal.UNetSpatioTemporalConditionOutput`] is returned, otherwise + a `tuple` is returned where the first element is the sample tensor. + """ + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == "mps" + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + batch_size, num_frames = sample.shape[:2] + timesteps = timesteps.expand(batch_size) + + t_emb = self.time_proj(timesteps) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=sample.dtype) + + emb = self.time_embedding(t_emb) + + time_embeds = self.add_time_proj(added_time_ids.flatten()) + time_embeds = time_embeds.reshape((batch_size, -1)) + time_embeds = time_embeds.to(emb.dtype) + aug_emb = self.add_embedding(time_embeds) + emb = emb + aug_emb + + # Flatten the batch and frames dimensions + # sample: [batch, frames, channels, height, width] -> [batch * frames, channels, height, width] + sample = sample.flatten(0, 1) + # Repeat the embeddings num_video_frames times + # emb: [batch, channels] -> [batch * frames, channels] + emb = emb.repeat_interleave(num_frames, dim=0) + # encoder_hidden_states: [batch, 1, channels] -> [batch * frames, 1, channels] + encoder_hidden_states = encoder_hidden_states.repeat_interleave(num_frames, dim=0) + + # 2. pre-process + sample = self.conv_in(sample) + + image_only_indicator = torch.zeros(batch_size, num_frames, dtype=sample.dtype, device=sample.device) + + down_block_res_samples = (sample,) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + image_only_indicator=image_only_indicator, + ) + else: + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + image_only_indicator=image_only_indicator, + ) + + down_block_res_samples += res_samples + + # 4. mid + sample = self.mid_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + image_only_indicator=image_only_indicator, + ) + + # 5. up + for i, upsample_block in enumerate(self.up_blocks): + res_samples = down_block_res_samples[-len(upsample_block.resnets) :] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=encoder_hidden_states, + image_only_indicator=image_only_indicator, + ) + else: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + image_only_indicator=image_only_indicator, + ) + + # 6. post-process + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + # 7. Reshape back to original shape + sample = sample.reshape(batch_size, num_frames, *sample.shape[1:]) + + if not return_dict: + return (sample,) + + return UNetSpatioTemporalConditionOutput(sample=sample) diff --git a/diffusers-0.27.0/src/diffusers/models/unets/unet_stable_cascade.py b/diffusers-0.27.0/src/diffusers/models/unets/unet_stable_cascade.py new file mode 100755 index 0000000..9f81e50 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/unet_stable_cascade.py @@ -0,0 +1,610 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn as nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...loaders.unet import FromOriginalUNetMixin +from ...utils import BaseOutput +from ..attention_processor import Attention +from ..modeling_utils import ModelMixin + + +# Copied from diffusers.pipelines.wuerstchen.modeling_wuerstchen_common.WuerstchenLayerNorm with WuerstchenLayerNorm -> SDCascadeLayerNorm +class SDCascadeLayerNorm(nn.LayerNorm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def forward(self, x): + x = x.permute(0, 2, 3, 1) + x = super().forward(x) + return x.permute(0, 3, 1, 2) + + +class SDCascadeTimestepBlock(nn.Module): + def __init__(self, c, c_timestep, conds=[]): + super().__init__() + linear_cls = nn.Linear + self.mapper = linear_cls(c_timestep, c * 2) + self.conds = conds + for cname in conds: + setattr(self, f"mapper_{cname}", linear_cls(c_timestep, c * 2)) + + def forward(self, x, t): + t = t.chunk(len(self.conds) + 1, dim=1) + a, b = self.mapper(t[0])[:, :, None, None].chunk(2, dim=1) + for i, c in enumerate(self.conds): + ac, bc = getattr(self, f"mapper_{c}")(t[i + 1])[:, :, None, None].chunk(2, dim=1) + a, b = a + ac, b + bc + return x * (1 + a) + b + + +class SDCascadeResBlock(nn.Module): + def __init__(self, c, c_skip=0, kernel_size=3, dropout=0.0): + super().__init__() + self.depthwise = nn.Conv2d(c, c, kernel_size=kernel_size, padding=kernel_size // 2, groups=c) + self.norm = SDCascadeLayerNorm(c, elementwise_affine=False, eps=1e-6) + self.channelwise = nn.Sequential( + nn.Linear(c + c_skip, c * 4), + nn.GELU(), + GlobalResponseNorm(c * 4), + nn.Dropout(dropout), + nn.Linear(c * 4, c), + ) + + def forward(self, x, x_skip=None): + x_res = x + x = self.norm(self.depthwise(x)) + if x_skip is not None: + x = torch.cat([x, x_skip], dim=1) + x = self.channelwise(x.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) + return x + x_res + + +# from https://github.com/facebookresearch/ConvNeXt-V2/blob/3608f67cc1dae164790c5d0aead7bf2d73d9719b/models/utils.py#L105 +class GlobalResponseNorm(nn.Module): + def __init__(self, dim): + super().__init__() + self.gamma = nn.Parameter(torch.zeros(1, 1, 1, dim)) + self.beta = nn.Parameter(torch.zeros(1, 1, 1, dim)) + + def forward(self, x): + agg_norm = torch.norm(x, p=2, dim=(1, 2), keepdim=True) + stand_div_norm = agg_norm / (agg_norm.mean(dim=-1, keepdim=True) + 1e-6) + return self.gamma * (x * stand_div_norm) + self.beta + x + + +class SDCascadeAttnBlock(nn.Module): + def __init__(self, c, c_cond, nhead, self_attn=True, dropout=0.0): + super().__init__() + linear_cls = nn.Linear + + self.self_attn = self_attn + self.norm = SDCascadeLayerNorm(c, elementwise_affine=False, eps=1e-6) + self.attention = Attention(query_dim=c, heads=nhead, dim_head=c // nhead, dropout=dropout, bias=True) + self.kv_mapper = nn.Sequential(nn.SiLU(), linear_cls(c_cond, c)) + + def forward(self, x, kv): + kv = self.kv_mapper(kv) + norm_x = self.norm(x) + if self.self_attn: + batch_size, channel, _, _ = x.shape + kv = torch.cat([norm_x.view(batch_size, channel, -1).transpose(1, 2), kv], dim=1) + x = x + self.attention(norm_x, encoder_hidden_states=kv) + return x + + +class UpDownBlock2d(nn.Module): + def __init__(self, in_channels, out_channels, mode, enabled=True): + super().__init__() + if mode not in ["up", "down"]: + raise ValueError(f"{mode} not supported") + interpolation = ( + nn.Upsample(scale_factor=2 if mode == "up" else 0.5, mode="bilinear", align_corners=True) + if enabled + else nn.Identity() + ) + mapping = nn.Conv2d(in_channels, out_channels, kernel_size=1) + self.blocks = nn.ModuleList([interpolation, mapping] if mode == "up" else [mapping, interpolation]) + + def forward(self, x): + for block in self.blocks: + x = block(x) + return x + + +@dataclass +class StableCascadeUNetOutput(BaseOutput): + sample: torch.FloatTensor = None + + +class StableCascadeUNet(ModelMixin, ConfigMixin, FromOriginalUNetMixin): + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + in_channels: int = 16, + out_channels: int = 16, + timestep_ratio_embedding_dim: int = 64, + patch_size: int = 1, + conditioning_dim: int = 2048, + block_out_channels: Tuple[int] = (2048, 2048), + num_attention_heads: Tuple[int] = (32, 32), + down_num_layers_per_block: Tuple[int] = (8, 24), + up_num_layers_per_block: Tuple[int] = (24, 8), + down_blocks_repeat_mappers: Optional[Tuple[int]] = ( + 1, + 1, + ), + up_blocks_repeat_mappers: Optional[Tuple[int]] = (1, 1), + block_types_per_layer: Tuple[Tuple[str]] = ( + ("SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"), + ("SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"), + ), + clip_text_in_channels: Optional[int] = None, + clip_text_pooled_in_channels=1280, + clip_image_in_channels: Optional[int] = None, + clip_seq=4, + effnet_in_channels: Optional[int] = None, + pixel_mapper_in_channels: Optional[int] = None, + kernel_size=3, + dropout: Union[float, Tuple[float]] = (0.1, 0.1), + self_attn: Union[bool, Tuple[bool]] = True, + timestep_conditioning_type: Tuple[str] = ("sca", "crp"), + switch_level: Optional[Tuple[bool]] = None, + ): + """ + + Parameters: + in_channels (`int`, defaults to 16): + Number of channels in the input sample. + out_channels (`int`, defaults to 16): + Number of channels in the output sample. + timestep_ratio_embedding_dim (`int`, defaults to 64): + Dimension of the projected time embedding. + patch_size (`int`, defaults to 1): + Patch size to use for pixel unshuffling layer + conditioning_dim (`int`, defaults to 2048): + Dimension of the image and text conditional embedding. + block_out_channels (Tuple[int], defaults to (2048, 2048)): + Tuple of output channels for each block. + num_attention_heads (Tuple[int], defaults to (32, 32)): + Number of attention heads in each attention block. Set to -1 to if block types in a layer do not have attention. + down_num_layers_per_block (Tuple[int], defaults to [8, 24]): + Number of layers in each down block. + up_num_layers_per_block (Tuple[int], defaults to [24, 8]): + Number of layers in each up block. + down_blocks_repeat_mappers (Tuple[int], optional, defaults to [1, 1]): + Number of 1x1 Convolutional layers to repeat in each down block. + up_blocks_repeat_mappers (Tuple[int], optional, defaults to [1, 1]): + Number of 1x1 Convolutional layers to repeat in each up block. + block_types_per_layer (Tuple[Tuple[str]], optional, + defaults to ( + ("SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"), + ("SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock") + ): + Block types used in each layer of the up/down blocks. + clip_text_in_channels (`int`, *optional*, defaults to `None`): + Number of input channels for CLIP based text conditioning. + clip_text_pooled_in_channels (`int`, *optional*, defaults to 1280): + Number of input channels for pooled CLIP text embeddings. + clip_image_in_channels (`int`, *optional*): + Number of input channels for CLIP based image conditioning. + clip_seq (`int`, *optional*, defaults to 4): + effnet_in_channels (`int`, *optional*, defaults to `None`): + Number of input channels for effnet conditioning. + pixel_mapper_in_channels (`int`, defaults to `None`): + Number of input channels for pixel mapper conditioning. + kernel_size (`int`, *optional*, defaults to 3): + Kernel size to use in the block convolutional layers. + dropout (Tuple[float], *optional*, defaults to (0.1, 0.1)): + Dropout to use per block. + self_attn (Union[bool, Tuple[bool]]): + Tuple of booleans that determine whether to use self attention in a block or not. + timestep_conditioning_type (Tuple[str], defaults to ("sca", "crp")): + Timestep conditioning type. + switch_level (Optional[Tuple[bool]], *optional*, defaults to `None`): + Tuple that indicates whether upsampling or downsampling should be applied in a block + """ + + super().__init__() + + if len(block_out_channels) != len(down_num_layers_per_block): + raise ValueError( + f"Number of elements in `down_num_layers_per_block` must match the length of `block_out_channels`: {len(block_out_channels)}" + ) + + elif len(block_out_channels) != len(up_num_layers_per_block): + raise ValueError( + f"Number of elements in `up_num_layers_per_block` must match the length of `block_out_channels`: {len(block_out_channels)}" + ) + + elif len(block_out_channels) != len(down_blocks_repeat_mappers): + raise ValueError( + f"Number of elements in `down_blocks_repeat_mappers` must match the length of `block_out_channels`: {len(block_out_channels)}" + ) + + elif len(block_out_channels) != len(up_blocks_repeat_mappers): + raise ValueError( + f"Number of elements in `up_blocks_repeat_mappers` must match the length of `block_out_channels`: {len(block_out_channels)}" + ) + + elif len(block_out_channels) != len(block_types_per_layer): + raise ValueError( + f"Number of elements in `block_types_per_layer` must match the length of `block_out_channels`: {len(block_out_channels)}" + ) + + if isinstance(dropout, float): + dropout = (dropout,) * len(block_out_channels) + if isinstance(self_attn, bool): + self_attn = (self_attn,) * len(block_out_channels) + + # CONDITIONING + if effnet_in_channels is not None: + self.effnet_mapper = nn.Sequential( + nn.Conv2d(effnet_in_channels, block_out_channels[0] * 4, kernel_size=1), + nn.GELU(), + nn.Conv2d(block_out_channels[0] * 4, block_out_channels[0], kernel_size=1), + SDCascadeLayerNorm(block_out_channels[0], elementwise_affine=False, eps=1e-6), + ) + if pixel_mapper_in_channels is not None: + self.pixels_mapper = nn.Sequential( + nn.Conv2d(pixel_mapper_in_channels, block_out_channels[0] * 4, kernel_size=1), + nn.GELU(), + nn.Conv2d(block_out_channels[0] * 4, block_out_channels[0], kernel_size=1), + SDCascadeLayerNorm(block_out_channels[0], elementwise_affine=False, eps=1e-6), + ) + + self.clip_txt_pooled_mapper = nn.Linear(clip_text_pooled_in_channels, conditioning_dim * clip_seq) + if clip_text_in_channels is not None: + self.clip_txt_mapper = nn.Linear(clip_text_in_channels, conditioning_dim) + if clip_image_in_channels is not None: + self.clip_img_mapper = nn.Linear(clip_image_in_channels, conditioning_dim * clip_seq) + self.clip_norm = nn.LayerNorm(conditioning_dim, elementwise_affine=False, eps=1e-6) + + self.embedding = nn.Sequential( + nn.PixelUnshuffle(patch_size), + nn.Conv2d(in_channels * (patch_size**2), block_out_channels[0], kernel_size=1), + SDCascadeLayerNorm(block_out_channels[0], elementwise_affine=False, eps=1e-6), + ) + + def get_block(block_type, in_channels, nhead, c_skip=0, dropout=0, self_attn=True): + if block_type == "SDCascadeResBlock": + return SDCascadeResBlock(in_channels, c_skip, kernel_size=kernel_size, dropout=dropout) + elif block_type == "SDCascadeAttnBlock": + return SDCascadeAttnBlock(in_channels, conditioning_dim, nhead, self_attn=self_attn, dropout=dropout) + elif block_type == "SDCascadeTimestepBlock": + return SDCascadeTimestepBlock( + in_channels, timestep_ratio_embedding_dim, conds=timestep_conditioning_type + ) + else: + raise ValueError(f"Block type {block_type} not supported") + + # BLOCKS + # -- down blocks + self.down_blocks = nn.ModuleList() + self.down_downscalers = nn.ModuleList() + self.down_repeat_mappers = nn.ModuleList() + for i in range(len(block_out_channels)): + if i > 0: + self.down_downscalers.append( + nn.Sequential( + SDCascadeLayerNorm(block_out_channels[i - 1], elementwise_affine=False, eps=1e-6), + UpDownBlock2d( + block_out_channels[i - 1], block_out_channels[i], mode="down", enabled=switch_level[i - 1] + ) + if switch_level is not None + else nn.Conv2d(block_out_channels[i - 1], block_out_channels[i], kernel_size=2, stride=2), + ) + ) + else: + self.down_downscalers.append(nn.Identity()) + + down_block = nn.ModuleList() + for _ in range(down_num_layers_per_block[i]): + for block_type in block_types_per_layer[i]: + block = get_block( + block_type, + block_out_channels[i], + num_attention_heads[i], + dropout=dropout[i], + self_attn=self_attn[i], + ) + down_block.append(block) + self.down_blocks.append(down_block) + + if down_blocks_repeat_mappers is not None: + block_repeat_mappers = nn.ModuleList() + for _ in range(down_blocks_repeat_mappers[i] - 1): + block_repeat_mappers.append(nn.Conv2d(block_out_channels[i], block_out_channels[i], kernel_size=1)) + self.down_repeat_mappers.append(block_repeat_mappers) + + # -- up blocks + self.up_blocks = nn.ModuleList() + self.up_upscalers = nn.ModuleList() + self.up_repeat_mappers = nn.ModuleList() + for i in reversed(range(len(block_out_channels))): + if i > 0: + self.up_upscalers.append( + nn.Sequential( + SDCascadeLayerNorm(block_out_channels[i], elementwise_affine=False, eps=1e-6), + UpDownBlock2d( + block_out_channels[i], block_out_channels[i - 1], mode="up", enabled=switch_level[i - 1] + ) + if switch_level is not None + else nn.ConvTranspose2d( + block_out_channels[i], block_out_channels[i - 1], kernel_size=2, stride=2 + ), + ) + ) + else: + self.up_upscalers.append(nn.Identity()) + + up_block = nn.ModuleList() + for j in range(up_num_layers_per_block[::-1][i]): + for k, block_type in enumerate(block_types_per_layer[i]): + c_skip = block_out_channels[i] if i < len(block_out_channels) - 1 and j == k == 0 else 0 + block = get_block( + block_type, + block_out_channels[i], + num_attention_heads[i], + c_skip=c_skip, + dropout=dropout[i], + self_attn=self_attn[i], + ) + up_block.append(block) + self.up_blocks.append(up_block) + + if up_blocks_repeat_mappers is not None: + block_repeat_mappers = nn.ModuleList() + for _ in range(up_blocks_repeat_mappers[::-1][i] - 1): + block_repeat_mappers.append(nn.Conv2d(block_out_channels[i], block_out_channels[i], kernel_size=1)) + self.up_repeat_mappers.append(block_repeat_mappers) + + # OUTPUT + self.clf = nn.Sequential( + SDCascadeLayerNorm(block_out_channels[0], elementwise_affine=False, eps=1e-6), + nn.Conv2d(block_out_channels[0], out_channels * (patch_size**2), kernel_size=1), + nn.PixelShuffle(patch_size), + ) + + self.gradient_checkpointing = False + + def _set_gradient_checkpointing(self, value=False): + self.gradient_checkpointing = value + + def _init_weights(self, m): + if isinstance(m, (nn.Conv2d, nn.Linear)): + torch.nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + + nn.init.normal_(self.clip_txt_pooled_mapper.weight, std=0.02) + nn.init.normal_(self.clip_txt_mapper.weight, std=0.02) if hasattr(self, "clip_txt_mapper") else None + nn.init.normal_(self.clip_img_mapper.weight, std=0.02) if hasattr(self, "clip_img_mapper") else None + + if hasattr(self, "effnet_mapper"): + nn.init.normal_(self.effnet_mapper[0].weight, std=0.02) # conditionings + nn.init.normal_(self.effnet_mapper[2].weight, std=0.02) # conditionings + + if hasattr(self, "pixels_mapper"): + nn.init.normal_(self.pixels_mapper[0].weight, std=0.02) # conditionings + nn.init.normal_(self.pixels_mapper[2].weight, std=0.02) # conditionings + + torch.nn.init.xavier_uniform_(self.embedding[1].weight, 0.02) # inputs + nn.init.constant_(self.clf[1].weight, 0) # outputs + + # blocks + for level_block in self.down_blocks + self.up_blocks: + for block in level_block: + if isinstance(block, SDCascadeResBlock): + block.channelwise[-1].weight.data *= np.sqrt(1 / sum(self.config.blocks[0])) + elif isinstance(block, SDCascadeTimestepBlock): + nn.init.constant_(block.mapper.weight, 0) + + def get_timestep_ratio_embedding(self, timestep_ratio, max_positions=10000): + r = timestep_ratio * max_positions + half_dim = self.config.timestep_ratio_embedding_dim // 2 + + emb = math.log(max_positions) / (half_dim - 1) + emb = torch.arange(half_dim, device=r.device).float().mul(-emb).exp() + emb = r[:, None] * emb[None, :] + emb = torch.cat([emb.sin(), emb.cos()], dim=1) + + if self.config.timestep_ratio_embedding_dim % 2 == 1: # zero pad + emb = nn.functional.pad(emb, (0, 1), mode="constant") + + return emb.to(dtype=r.dtype) + + def get_clip_embeddings(self, clip_txt_pooled, clip_txt=None, clip_img=None): + if len(clip_txt_pooled.shape) == 2: + clip_txt_pool = clip_txt_pooled.unsqueeze(1) + clip_txt_pool = self.clip_txt_pooled_mapper(clip_txt_pooled).view( + clip_txt_pooled.size(0), clip_txt_pooled.size(1) * self.config.clip_seq, -1 + ) + if clip_txt is not None and clip_img is not None: + clip_txt = self.clip_txt_mapper(clip_txt) + if len(clip_img.shape) == 2: + clip_img = clip_img.unsqueeze(1) + clip_img = self.clip_img_mapper(clip_img).view( + clip_img.size(0), clip_img.size(1) * self.config.clip_seq, -1 + ) + clip = torch.cat([clip_txt, clip_txt_pool, clip_img], dim=1) + else: + clip = clip_txt_pool + return self.clip_norm(clip) + + def _down_encode(self, x, r_embed, clip): + level_outputs = [] + block_group = zip(self.down_blocks, self.down_downscalers, self.down_repeat_mappers) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + for down_block, downscaler, repmap in block_group: + x = downscaler(x) + for i in range(len(repmap) + 1): + for block in down_block: + if isinstance(block, SDCascadeResBlock): + x = torch.utils.checkpoint.checkpoint(create_custom_forward(block), x, use_reentrant=False) + elif isinstance(block, SDCascadeAttnBlock): + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), x, clip, use_reentrant=False + ) + elif isinstance(block, SDCascadeTimestepBlock): + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), x, r_embed, use_reentrant=False + ) + else: + x = x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), use_reentrant=False + ) + if i < len(repmap): + x = repmap[i](x) + level_outputs.insert(0, x) + else: + for down_block, downscaler, repmap in block_group: + x = downscaler(x) + for i in range(len(repmap) + 1): + for block in down_block: + if isinstance(block, SDCascadeResBlock): + x = block(x) + elif isinstance(block, SDCascadeAttnBlock): + x = block(x, clip) + elif isinstance(block, SDCascadeTimestepBlock): + x = block(x, r_embed) + else: + x = block(x) + if i < len(repmap): + x = repmap[i](x) + level_outputs.insert(0, x) + return level_outputs + + def _up_decode(self, level_outputs, r_embed, clip): + x = level_outputs[0] + block_group = zip(self.up_blocks, self.up_upscalers, self.up_repeat_mappers) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + for i, (up_block, upscaler, repmap) in enumerate(block_group): + for j in range(len(repmap) + 1): + for k, block in enumerate(up_block): + if isinstance(block, SDCascadeResBlock): + skip = level_outputs[i] if k == 0 and i > 0 else None + if skip is not None and (x.size(-1) != skip.size(-1) or x.size(-2) != skip.size(-2)): + x = torch.nn.functional.interpolate( + x.float(), skip.shape[-2:], mode="bilinear", align_corners=True + ) + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), x, skip, use_reentrant=False + ) + elif isinstance(block, SDCascadeAttnBlock): + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), x, clip, use_reentrant=False + ) + elif isinstance(block, SDCascadeTimestepBlock): + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), x, r_embed, use_reentrant=False + ) + else: + x = torch.utils.checkpoint.checkpoint(create_custom_forward(block), x, use_reentrant=False) + if j < len(repmap): + x = repmap[j](x) + x = upscaler(x) + else: + for i, (up_block, upscaler, repmap) in enumerate(block_group): + for j in range(len(repmap) + 1): + for k, block in enumerate(up_block): + if isinstance(block, SDCascadeResBlock): + skip = level_outputs[i] if k == 0 and i > 0 else None + if skip is not None and (x.size(-1) != skip.size(-1) or x.size(-2) != skip.size(-2)): + x = torch.nn.functional.interpolate( + x.float(), skip.shape[-2:], mode="bilinear", align_corners=True + ) + x = block(x, skip) + elif isinstance(block, SDCascadeAttnBlock): + x = block(x, clip) + elif isinstance(block, SDCascadeTimestepBlock): + x = block(x, r_embed) + else: + x = block(x) + if j < len(repmap): + x = repmap[j](x) + x = upscaler(x) + return x + + def forward( + self, + sample, + timestep_ratio, + clip_text_pooled, + clip_text=None, + clip_img=None, + effnet=None, + pixels=None, + sca=None, + crp=None, + return_dict=True, + ): + if pixels is None: + pixels = sample.new_zeros(sample.size(0), 3, 8, 8) + + # Process the conditioning embeddings + timestep_ratio_embed = self.get_timestep_ratio_embedding(timestep_ratio) + for c in self.config.timestep_conditioning_type: + if c == "sca": + cond = sca + elif c == "crp": + cond = crp + else: + cond = None + t_cond = cond or torch.zeros_like(timestep_ratio) + timestep_ratio_embed = torch.cat([timestep_ratio_embed, self.get_timestep_ratio_embedding(t_cond)], dim=1) + clip = self.get_clip_embeddings(clip_txt_pooled=clip_text_pooled, clip_txt=clip_text, clip_img=clip_img) + + # Model Blocks + x = self.embedding(sample) + if hasattr(self, "effnet_mapper") and effnet is not None: + x = x + self.effnet_mapper( + nn.functional.interpolate(effnet, size=x.shape[-2:], mode="bilinear", align_corners=True) + ) + if hasattr(self, "pixels_mapper"): + x = x + nn.functional.interpolate( + self.pixels_mapper(pixels), size=x.shape[-2:], mode="bilinear", align_corners=True + ) + level_outputs = self._down_encode(x, timestep_ratio_embed, clip) + x = self._up_decode(level_outputs, timestep_ratio_embed, clip) + sample = self.clf(x) + + if not return_dict: + return (sample,) + return StableCascadeUNetOutput(sample=sample) diff --git a/diffusers-0.27.0/src/diffusers/models/unets/uvit_2d.py b/diffusers-0.27.0/src/diffusers/models/unets/uvit_2d.py new file mode 100755 index 0000000..bfd865d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/unets/uvit_2d.py @@ -0,0 +1,470 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, Union + +import torch +import torch.nn.functional as F +from torch import nn +from torch.utils.checkpoint import checkpoint + +from ...configuration_utils import ConfigMixin, register_to_config +from ...loaders import PeftAdapterMixin +from ..attention import BasicTransformerBlock, SkipFFTransformerBlock +from ..attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from ..embeddings import TimestepEmbedding, get_timestep_embedding +from ..modeling_utils import ModelMixin +from ..normalization import GlobalResponseNorm, RMSNorm +from ..resnet import Downsample2D, Upsample2D + + +class UVit2DModel(ModelMixin, ConfigMixin, PeftAdapterMixin): + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + # global config + hidden_size: int = 1024, + use_bias: bool = False, + hidden_dropout: float = 0.0, + # conditioning dimensions + cond_embed_dim: int = 768, + micro_cond_encode_dim: int = 256, + micro_cond_embed_dim: int = 1280, + encoder_hidden_size: int = 768, + # num tokens + vocab_size: int = 8256, # codebook_size + 1 (for the mask token) rounded + codebook_size: int = 8192, + # `UVit2DConvEmbed` + in_channels: int = 768, + block_out_channels: int = 768, + num_res_blocks: int = 3, + downsample: bool = False, + upsample: bool = False, + block_num_heads: int = 12, + # `TransformerLayer` + num_hidden_layers: int = 22, + num_attention_heads: int = 16, + # `Attention` + attention_dropout: float = 0.0, + # `FeedForward` + intermediate_size: int = 2816, + # `Norm` + layer_norm_eps: float = 1e-6, + ln_elementwise_affine: bool = True, + sample_size: int = 64, + ): + super().__init__() + + self.encoder_proj = nn.Linear(encoder_hidden_size, hidden_size, bias=use_bias) + self.encoder_proj_layer_norm = RMSNorm(hidden_size, layer_norm_eps, ln_elementwise_affine) + + self.embed = UVit2DConvEmbed( + in_channels, block_out_channels, vocab_size, ln_elementwise_affine, layer_norm_eps, use_bias + ) + + self.cond_embed = TimestepEmbedding( + micro_cond_embed_dim + cond_embed_dim, hidden_size, sample_proj_bias=use_bias + ) + + self.down_block = UVitBlock( + block_out_channels, + num_res_blocks, + hidden_size, + hidden_dropout, + ln_elementwise_affine, + layer_norm_eps, + use_bias, + block_num_heads, + attention_dropout, + downsample, + False, + ) + + self.project_to_hidden_norm = RMSNorm(block_out_channels, layer_norm_eps, ln_elementwise_affine) + self.project_to_hidden = nn.Linear(block_out_channels, hidden_size, bias=use_bias) + + self.transformer_layers = nn.ModuleList( + [ + BasicTransformerBlock( + dim=hidden_size, + num_attention_heads=num_attention_heads, + attention_head_dim=hidden_size // num_attention_heads, + dropout=hidden_dropout, + cross_attention_dim=hidden_size, + attention_bias=use_bias, + norm_type="ada_norm_continuous", + ada_norm_continous_conditioning_embedding_dim=hidden_size, + norm_elementwise_affine=ln_elementwise_affine, + norm_eps=layer_norm_eps, + ada_norm_bias=use_bias, + ff_inner_dim=intermediate_size, + ff_bias=use_bias, + attention_out_bias=use_bias, + ) + for _ in range(num_hidden_layers) + ] + ) + + self.project_from_hidden_norm = RMSNorm(hidden_size, layer_norm_eps, ln_elementwise_affine) + self.project_from_hidden = nn.Linear(hidden_size, block_out_channels, bias=use_bias) + + self.up_block = UVitBlock( + block_out_channels, + num_res_blocks, + hidden_size, + hidden_dropout, + ln_elementwise_affine, + layer_norm_eps, + use_bias, + block_num_heads, + attention_dropout, + downsample=False, + upsample=upsample, + ) + + self.mlm_layer = ConvMlmLayer( + block_out_channels, in_channels, use_bias, ln_elementwise_affine, layer_norm_eps, codebook_size + ) + + self.gradient_checkpointing = False + + def _set_gradient_checkpointing(self, module, value: bool = False) -> None: + pass + + def forward(self, input_ids, encoder_hidden_states, pooled_text_emb, micro_conds, cross_attention_kwargs=None): + encoder_hidden_states = self.encoder_proj(encoder_hidden_states) + encoder_hidden_states = self.encoder_proj_layer_norm(encoder_hidden_states) + + micro_cond_embeds = get_timestep_embedding( + micro_conds.flatten(), self.config.micro_cond_encode_dim, flip_sin_to_cos=True, downscale_freq_shift=0 + ) + + micro_cond_embeds = micro_cond_embeds.reshape((input_ids.shape[0], -1)) + + pooled_text_emb = torch.cat([pooled_text_emb, micro_cond_embeds], dim=1) + pooled_text_emb = pooled_text_emb.to(dtype=self.dtype) + pooled_text_emb = self.cond_embed(pooled_text_emb).to(encoder_hidden_states.dtype) + + hidden_states = self.embed(input_ids) + + hidden_states = self.down_block( + hidden_states, + pooled_text_emb=pooled_text_emb, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + ) + + batch_size, channels, height, width = hidden_states.shape + hidden_states = hidden_states.permute(0, 2, 3, 1).reshape(batch_size, height * width, channels) + + hidden_states = self.project_to_hidden_norm(hidden_states) + hidden_states = self.project_to_hidden(hidden_states) + + for layer in self.transformer_layers: + if self.training and self.gradient_checkpointing: + + def layer_(*args): + return checkpoint(layer, *args) + + else: + layer_ = layer + + hidden_states = layer_( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs={"pooled_text_emb": pooled_text_emb}, + ) + + hidden_states = self.project_from_hidden_norm(hidden_states) + hidden_states = self.project_from_hidden(hidden_states) + + hidden_states = hidden_states.reshape(batch_size, height, width, channels).permute(0, 3, 1, 2) + + hidden_states = self.up_block( + hidden_states, + pooled_text_emb=pooled_text_emb, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + ) + + logits = self.mlm_layer(hidden_states) + + return logits + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + +class UVit2DConvEmbed(nn.Module): + def __init__(self, in_channels, block_out_channels, vocab_size, elementwise_affine, eps, bias): + super().__init__() + self.embeddings = nn.Embedding(vocab_size, in_channels) + self.layer_norm = RMSNorm(in_channels, eps, elementwise_affine) + self.conv = nn.Conv2d(in_channels, block_out_channels, kernel_size=1, bias=bias) + + def forward(self, input_ids): + embeddings = self.embeddings(input_ids) + embeddings = self.layer_norm(embeddings) + embeddings = embeddings.permute(0, 3, 1, 2) + embeddings = self.conv(embeddings) + return embeddings + + +class UVitBlock(nn.Module): + def __init__( + self, + channels, + num_res_blocks: int, + hidden_size, + hidden_dropout, + ln_elementwise_affine, + layer_norm_eps, + use_bias, + block_num_heads, + attention_dropout, + downsample: bool, + upsample: bool, + ): + super().__init__() + + if downsample: + self.downsample = Downsample2D( + channels, + use_conv=True, + padding=0, + name="Conv2d_0", + kernel_size=2, + norm_type="rms_norm", + eps=layer_norm_eps, + elementwise_affine=ln_elementwise_affine, + bias=use_bias, + ) + else: + self.downsample = None + + self.res_blocks = nn.ModuleList( + [ + ConvNextBlock( + channels, + layer_norm_eps, + ln_elementwise_affine, + use_bias, + hidden_dropout, + hidden_size, + ) + for i in range(num_res_blocks) + ] + ) + + self.attention_blocks = nn.ModuleList( + [ + SkipFFTransformerBlock( + channels, + block_num_heads, + channels // block_num_heads, + hidden_size, + use_bias, + attention_dropout, + channels, + attention_bias=use_bias, + attention_out_bias=use_bias, + ) + for _ in range(num_res_blocks) + ] + ) + + if upsample: + self.upsample = Upsample2D( + channels, + use_conv_transpose=True, + kernel_size=2, + padding=0, + name="conv", + norm_type="rms_norm", + eps=layer_norm_eps, + elementwise_affine=ln_elementwise_affine, + bias=use_bias, + interpolate=False, + ) + else: + self.upsample = None + + def forward(self, x, pooled_text_emb, encoder_hidden_states, cross_attention_kwargs): + if self.downsample is not None: + x = self.downsample(x) + + for res_block, attention_block in zip(self.res_blocks, self.attention_blocks): + x = res_block(x, pooled_text_emb) + + batch_size, channels, height, width = x.shape + x = x.view(batch_size, channels, height * width).permute(0, 2, 1) + x = attention_block( + x, encoder_hidden_states=encoder_hidden_states, cross_attention_kwargs=cross_attention_kwargs + ) + x = x.permute(0, 2, 1).view(batch_size, channels, height, width) + + if self.upsample is not None: + x = self.upsample(x) + + return x + + +class ConvNextBlock(nn.Module): + def __init__( + self, channels, layer_norm_eps, ln_elementwise_affine, use_bias, hidden_dropout, hidden_size, res_ffn_factor=4 + ): + super().__init__() + self.depthwise = nn.Conv2d( + channels, + channels, + kernel_size=3, + padding=1, + groups=channels, + bias=use_bias, + ) + self.norm = RMSNorm(channels, layer_norm_eps, ln_elementwise_affine) + self.channelwise_linear_1 = nn.Linear(channels, int(channels * res_ffn_factor), bias=use_bias) + self.channelwise_act = nn.GELU() + self.channelwise_norm = GlobalResponseNorm(int(channels * res_ffn_factor)) + self.channelwise_linear_2 = nn.Linear(int(channels * res_ffn_factor), channels, bias=use_bias) + self.channelwise_dropout = nn.Dropout(hidden_dropout) + self.cond_embeds_mapper = nn.Linear(hidden_size, channels * 2, use_bias) + + def forward(self, x, cond_embeds): + x_res = x + + x = self.depthwise(x) + + x = x.permute(0, 2, 3, 1) + x = self.norm(x) + + x = self.channelwise_linear_1(x) + x = self.channelwise_act(x) + x = self.channelwise_norm(x) + x = self.channelwise_linear_2(x) + x = self.channelwise_dropout(x) + + x = x.permute(0, 3, 1, 2) + + x = x + x_res + + scale, shift = self.cond_embeds_mapper(F.silu(cond_embeds)).chunk(2, dim=1) + x = x * (1 + scale[:, :, None, None]) + shift[:, :, None, None] + + return x + + +class ConvMlmLayer(nn.Module): + def __init__( + self, + block_out_channels: int, + in_channels: int, + use_bias: bool, + ln_elementwise_affine: bool, + layer_norm_eps: float, + codebook_size: int, + ): + super().__init__() + self.conv1 = nn.Conv2d(block_out_channels, in_channels, kernel_size=1, bias=use_bias) + self.layer_norm = RMSNorm(in_channels, layer_norm_eps, ln_elementwise_affine) + self.conv2 = nn.Conv2d(in_channels, codebook_size, kernel_size=1, bias=use_bias) + + def forward(self, hidden_states): + hidden_states = self.conv1(hidden_states) + hidden_states = self.layer_norm(hidden_states.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) + logits = self.conv2(hidden_states) + return logits diff --git a/diffusers-0.27.0/src/diffusers/models/upsampling.py b/diffusers-0.27.0/src/diffusers/models/upsampling.py new file mode 100755 index 0000000..4ecf6eb --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/upsampling.py @@ -0,0 +1,448 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Tuple + +import torch +import torch.nn as nn +import torch.nn.functional as F + +from ..utils import deprecate +from .normalization import RMSNorm + + +class Upsample1D(nn.Module): + """A 1D upsampling layer with an optional convolution. + + Parameters: + channels (`int`): + number of channels in the inputs and outputs. + use_conv (`bool`, default `False`): + option to use a convolution. + use_conv_transpose (`bool`, default `False`): + option to use a convolution transpose. + out_channels (`int`, optional): + number of output channels. Defaults to `channels`. + name (`str`, default `conv`): + name of the upsampling 1D layer. + """ + + def __init__( + self, + channels: int, + use_conv: bool = False, + use_conv_transpose: bool = False, + out_channels: Optional[int] = None, + name: str = "conv", + ): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.use_conv_transpose = use_conv_transpose + self.name = name + + self.conv = None + if use_conv_transpose: + self.conv = nn.ConvTranspose1d(channels, self.out_channels, 4, 2, 1) + elif use_conv: + self.conv = nn.Conv1d(self.channels, self.out_channels, 3, padding=1) + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + assert inputs.shape[1] == self.channels + if self.use_conv_transpose: + return self.conv(inputs) + + outputs = F.interpolate(inputs, scale_factor=2.0, mode="nearest") + + if self.use_conv: + outputs = self.conv(outputs) + + return outputs + + +class Upsample2D(nn.Module): + """A 2D upsampling layer with an optional convolution. + + Parameters: + channels (`int`): + number of channels in the inputs and outputs. + use_conv (`bool`, default `False`): + option to use a convolution. + use_conv_transpose (`bool`, default `False`): + option to use a convolution transpose. + out_channels (`int`, optional): + number of output channels. Defaults to `channels`. + name (`str`, default `conv`): + name of the upsampling 2D layer. + """ + + def __init__( + self, + channels: int, + use_conv: bool = False, + use_conv_transpose: bool = False, + out_channels: Optional[int] = None, + name: str = "conv", + kernel_size: Optional[int] = None, + padding=1, + norm_type=None, + eps=None, + elementwise_affine=None, + bias=True, + interpolate=True, + ): + super().__init__() + self.channels = channels + self.out_channels = out_channels or channels + self.use_conv = use_conv + self.use_conv_transpose = use_conv_transpose + self.name = name + self.interpolate = interpolate + conv_cls = nn.Conv2d + + if norm_type == "ln_norm": + self.norm = nn.LayerNorm(channels, eps, elementwise_affine) + elif norm_type == "rms_norm": + self.norm = RMSNorm(channels, eps, elementwise_affine) + elif norm_type is None: + self.norm = None + else: + raise ValueError(f"unknown norm_type: {norm_type}") + + conv = None + if use_conv_transpose: + if kernel_size is None: + kernel_size = 4 + conv = nn.ConvTranspose2d( + channels, self.out_channels, kernel_size=kernel_size, stride=2, padding=padding, bias=bias + ) + elif use_conv: + if kernel_size is None: + kernel_size = 3 + conv = conv_cls(self.channels, self.out_channels, kernel_size=kernel_size, padding=padding, bias=bias) + + # TODO(Suraj, Patrick) - clean up after weight dicts are correctly renamed + if name == "conv": + self.conv = conv + else: + self.Conv2d_0 = conv + + def forward( + self, hidden_states: torch.FloatTensor, output_size: Optional[int] = None, *args, **kwargs + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + assert hidden_states.shape[1] == self.channels + + if self.norm is not None: + hidden_states = self.norm(hidden_states.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) + + if self.use_conv_transpose: + return self.conv(hidden_states) + + # Cast to float32 to as 'upsample_nearest2d_out_frame' op does not support bfloat16 + # TODO(Suraj): Remove this cast once the issue is fixed in PyTorch + # https://github.com/pytorch/pytorch/issues/86679 + dtype = hidden_states.dtype + if dtype == torch.bfloat16: + hidden_states = hidden_states.to(torch.float32) + + # upsample_nearest_nhwc fails with large batch sizes. see https://github.com/huggingface/diffusers/issues/984 + if hidden_states.shape[0] >= 64: + hidden_states = hidden_states.contiguous() + + # if `output_size` is passed we force the interpolation output + # size and do not make use of `scale_factor=2` + if self.interpolate: + if output_size is None: + hidden_states = F.interpolate(hidden_states, scale_factor=2.0, mode="nearest") + else: + hidden_states = F.interpolate(hidden_states, size=output_size, mode="nearest") + + # If the input is bfloat16, we cast back to bfloat16 + if dtype == torch.bfloat16: + hidden_states = hidden_states.to(dtype) + + # TODO(Suraj, Patrick) - clean up after weight dicts are correctly renamed + if self.use_conv: + if self.name == "conv": + hidden_states = self.conv(hidden_states) + else: + hidden_states = self.Conv2d_0(hidden_states) + + return hidden_states + + +class FirUpsample2D(nn.Module): + """A 2D FIR upsampling layer with an optional convolution. + + Parameters: + channels (`int`, optional): + number of channels in the inputs and outputs. + use_conv (`bool`, default `False`): + option to use a convolution. + out_channels (`int`, optional): + number of output channels. Defaults to `channels`. + fir_kernel (`tuple`, default `(1, 3, 3, 1)`): + kernel for the FIR filter. + """ + + def __init__( + self, + channels: Optional[int] = None, + out_channels: Optional[int] = None, + use_conv: bool = False, + fir_kernel: Tuple[int, int, int, int] = (1, 3, 3, 1), + ): + super().__init__() + out_channels = out_channels if out_channels else channels + if use_conv: + self.Conv2d_0 = nn.Conv2d(channels, out_channels, kernel_size=3, stride=1, padding=1) + self.use_conv = use_conv + self.fir_kernel = fir_kernel + self.out_channels = out_channels + + def _upsample_2d( + self, + hidden_states: torch.FloatTensor, + weight: Optional[torch.FloatTensor] = None, + kernel: Optional[torch.FloatTensor] = None, + factor: int = 2, + gain: float = 1, + ) -> torch.FloatTensor: + """Fused `upsample_2d()` followed by `Conv2d()`. + + Padding is performed only once at the beginning, not between the operations. The fused op is considerably more + efficient than performing the same calculation using standard TensorFlow ops. It supports gradients of + arbitrary order. + + Args: + hidden_states (`torch.FloatTensor`): + Input tensor of the shape `[N, C, H, W]` or `[N, H, W, C]`. + weight (`torch.FloatTensor`, *optional*): + Weight tensor of the shape `[filterH, filterW, inChannels, outChannels]`. Grouped convolution can be + performed by `inChannels = x.shape[0] // numGroups`. + kernel (`torch.FloatTensor`, *optional*): + FIR filter of the shape `[firH, firW]` or `[firN]` (separable). The default is `[1] * factor`, which + corresponds to nearest-neighbor upsampling. + factor (`int`, *optional*): Integer upsampling factor (default: 2). + gain (`float`, *optional*): Scaling factor for signal magnitude (default: 1.0). + + Returns: + output (`torch.FloatTensor`): + Tensor of the shape `[N, C, H * factor, W * factor]` or `[N, H * factor, W * factor, C]`, and same + datatype as `hidden_states`. + """ + + assert isinstance(factor, int) and factor >= 1 + + # Setup filter kernel. + if kernel is None: + kernel = [1] * factor + + # setup kernel + kernel = torch.tensor(kernel, dtype=torch.float32) + if kernel.ndim == 1: + kernel = torch.outer(kernel, kernel) + kernel /= torch.sum(kernel) + + kernel = kernel * (gain * (factor**2)) + + if self.use_conv: + convH = weight.shape[2] + convW = weight.shape[3] + inC = weight.shape[1] + + pad_value = (kernel.shape[0] - factor) - (convW - 1) + + stride = (factor, factor) + # Determine data dimensions. + output_shape = ( + (hidden_states.shape[2] - 1) * factor + convH, + (hidden_states.shape[3] - 1) * factor + convW, + ) + output_padding = ( + output_shape[0] - (hidden_states.shape[2] - 1) * stride[0] - convH, + output_shape[1] - (hidden_states.shape[3] - 1) * stride[1] - convW, + ) + assert output_padding[0] >= 0 and output_padding[1] >= 0 + num_groups = hidden_states.shape[1] // inC + + # Transpose weights. + weight = torch.reshape(weight, (num_groups, -1, inC, convH, convW)) + weight = torch.flip(weight, dims=[3, 4]).permute(0, 2, 1, 3, 4) + weight = torch.reshape(weight, (num_groups * inC, -1, convH, convW)) + + inverse_conv = F.conv_transpose2d( + hidden_states, + weight, + stride=stride, + output_padding=output_padding, + padding=0, + ) + + output = upfirdn2d_native( + inverse_conv, + torch.tensor(kernel, device=inverse_conv.device), + pad=((pad_value + 1) // 2 + factor - 1, pad_value // 2 + 1), + ) + else: + pad_value = kernel.shape[0] - factor + output = upfirdn2d_native( + hidden_states, + torch.tensor(kernel, device=hidden_states.device), + up=factor, + pad=((pad_value + 1) // 2 + factor - 1, pad_value // 2), + ) + + return output + + def forward(self, hidden_states: torch.FloatTensor) -> torch.FloatTensor: + if self.use_conv: + height = self._upsample_2d(hidden_states, self.Conv2d_0.weight, kernel=self.fir_kernel) + height = height + self.Conv2d_0.bias.reshape(1, -1, 1, 1) + else: + height = self._upsample_2d(hidden_states, kernel=self.fir_kernel, factor=2) + + return height + + +class KUpsample2D(nn.Module): + r"""A 2D K-upsampling layer. + + Parameters: + pad_mode (`str`, *optional*, default to `"reflect"`): the padding mode to use. + """ + + def __init__(self, pad_mode: str = "reflect"): + super().__init__() + self.pad_mode = pad_mode + kernel_1d = torch.tensor([[1 / 8, 3 / 8, 3 / 8, 1 / 8]]) * 2 + self.pad = kernel_1d.shape[1] // 2 - 1 + self.register_buffer("kernel", kernel_1d.T @ kernel_1d, persistent=False) + + def forward(self, inputs: torch.Tensor) -> torch.Tensor: + inputs = F.pad(inputs, ((self.pad + 1) // 2,) * 4, self.pad_mode) + weight = inputs.new_zeros( + [ + inputs.shape[1], + inputs.shape[1], + self.kernel.shape[0], + self.kernel.shape[1], + ] + ) + indices = torch.arange(inputs.shape[1], device=inputs.device) + kernel = self.kernel.to(weight)[None, :].expand(inputs.shape[1], -1, -1) + weight[indices, indices] = kernel + return F.conv_transpose2d(inputs, weight, stride=2, padding=self.pad * 2 + 1) + + +def upfirdn2d_native( + tensor: torch.Tensor, + kernel: torch.Tensor, + up: int = 1, + down: int = 1, + pad: Tuple[int, int] = (0, 0), +) -> torch.Tensor: + up_x = up_y = up + down_x = down_y = down + pad_x0 = pad_y0 = pad[0] + pad_x1 = pad_y1 = pad[1] + + _, channel, in_h, in_w = tensor.shape + tensor = tensor.reshape(-1, in_h, in_w, 1) + + _, in_h, in_w, minor = tensor.shape + kernel_h, kernel_w = kernel.shape + + out = tensor.view(-1, in_h, 1, in_w, 1, minor) + out = F.pad(out, [0, 0, 0, up_x - 1, 0, 0, 0, up_y - 1]) + out = out.view(-1, in_h * up_y, in_w * up_x, minor) + + out = F.pad(out, [0, 0, max(pad_x0, 0), max(pad_x1, 0), max(pad_y0, 0), max(pad_y1, 0)]) + out = out.to(tensor.device) # Move back to mps if necessary + out = out[ + :, + max(-pad_y0, 0) : out.shape[1] - max(-pad_y1, 0), + max(-pad_x0, 0) : out.shape[2] - max(-pad_x1, 0), + :, + ] + + out = out.permute(0, 3, 1, 2) + out = out.reshape([-1, 1, in_h * up_y + pad_y0 + pad_y1, in_w * up_x + pad_x0 + pad_x1]) + w = torch.flip(kernel, [0, 1]).view(1, 1, kernel_h, kernel_w) + out = F.conv2d(out, w) + out = out.reshape( + -1, + minor, + in_h * up_y + pad_y0 + pad_y1 - kernel_h + 1, + in_w * up_x + pad_x0 + pad_x1 - kernel_w + 1, + ) + out = out.permute(0, 2, 3, 1) + out = out[:, ::down_y, ::down_x, :] + + out_h = (in_h * up_y + pad_y0 + pad_y1 - kernel_h) // down_y + 1 + out_w = (in_w * up_x + pad_x0 + pad_x1 - kernel_w) // down_x + 1 + + return out.view(-1, channel, out_h, out_w) + + +def upsample_2d( + hidden_states: torch.FloatTensor, + kernel: Optional[torch.FloatTensor] = None, + factor: int = 2, + gain: float = 1, +) -> torch.FloatTensor: + r"""Upsample2D a batch of 2D images with the given filter. + Accepts a batch of 2D images of the shape `[N, C, H, W]` or `[N, H, W, C]` and upsamples each image with the given + filter. The filter is normalized so that if the input pixels are constant, they will be scaled by the specified + `gain`. Pixels outside the image are assumed to be zero, and the filter is padded with zeros so that its shape is + a: multiple of the upsampling factor. + + Args: + hidden_states (`torch.FloatTensor`): + Input tensor of the shape `[N, C, H, W]` or `[N, H, W, C]`. + kernel (`torch.FloatTensor`, *optional*): + FIR filter of the shape `[firH, firW]` or `[firN]` (separable). The default is `[1] * factor`, which + corresponds to nearest-neighbor upsampling. + factor (`int`, *optional*, default to `2`): + Integer upsampling factor. + gain (`float`, *optional*, default to `1.0`): + Scaling factor for signal magnitude (default: 1.0). + + Returns: + output (`torch.FloatTensor`): + Tensor of the shape `[N, C, H * factor, W * factor]` + """ + assert isinstance(factor, int) and factor >= 1 + if kernel is None: + kernel = [1] * factor + + kernel = torch.tensor(kernel, dtype=torch.float32) + if kernel.ndim == 1: + kernel = torch.outer(kernel, kernel) + kernel /= torch.sum(kernel) + + kernel = kernel * (gain * (factor**2)) + pad_value = kernel.shape[0] - factor + output = upfirdn2d_native( + hidden_states, + kernel.to(device=hidden_states.device), + up=factor, + pad=((pad_value + 1) // 2 + factor - 1, pad_value // 2), + ) + return output diff --git a/diffusers-0.27.0/src/diffusers/models/vae_flax.py b/diffusers-0.27.0/src/diffusers/models/vae_flax.py new file mode 100755 index 0000000..5027f42 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/vae_flax.py @@ -0,0 +1,876 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# JAX implementation of VQGAN from taming-transformers https://github.com/CompVis/taming-transformers + +import math +from functools import partial +from typing import Tuple + +import flax +import flax.linen as nn +import jax +import jax.numpy as jnp +from flax.core.frozen_dict import FrozenDict + +from ..configuration_utils import ConfigMixin, flax_register_to_config +from ..utils import BaseOutput +from .modeling_flax_utils import FlaxModelMixin + + +@flax.struct.dataclass +class FlaxDecoderOutput(BaseOutput): + """ + Output of decoding method. + + Args: + sample (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)`): + The decoded output sample from the last layer of the model. + dtype (`jnp.dtype`, *optional*, defaults to `jnp.float32`): + The `dtype` of the parameters. + """ + + sample: jnp.ndarray + + +@flax.struct.dataclass +class FlaxAutoencoderKLOutput(BaseOutput): + """ + Output of AutoencoderKL encoding method. + + Args: + latent_dist (`FlaxDiagonalGaussianDistribution`): + Encoded outputs of `Encoder` represented as the mean and logvar of `FlaxDiagonalGaussianDistribution`. + `FlaxDiagonalGaussianDistribution` allows for sampling latents from the distribution. + """ + + latent_dist: "FlaxDiagonalGaussianDistribution" + + +class FlaxUpsample2D(nn.Module): + """ + Flax implementation of 2D Upsample layer + + Args: + in_channels (`int`): + Input channels + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.conv = nn.Conv( + self.in_channels, + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + def __call__(self, hidden_states): + batch, height, width, channels = hidden_states.shape + hidden_states = jax.image.resize( + hidden_states, + shape=(batch, height * 2, width * 2, channels), + method="nearest", + ) + hidden_states = self.conv(hidden_states) + return hidden_states + + +class FlaxDownsample2D(nn.Module): + """ + Flax implementation of 2D Downsample layer + + Args: + in_channels (`int`): + Input channels + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.conv = nn.Conv( + self.in_channels, + kernel_size=(3, 3), + strides=(2, 2), + padding="VALID", + dtype=self.dtype, + ) + + def __call__(self, hidden_states): + pad = ((0, 0), (0, 1), (0, 1), (0, 0)) # pad height and width dim + hidden_states = jnp.pad(hidden_states, pad_width=pad) + hidden_states = self.conv(hidden_states) + return hidden_states + + +class FlaxResnetBlock2D(nn.Module): + """ + Flax implementation of 2D Resnet Block. + + Args: + in_channels (`int`): + Input channels + out_channels (`int`): + Output channels + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + groups (:obj:`int`, *optional*, defaults to `32`): + The number of groups to use for group norm. + use_nin_shortcut (:obj:`bool`, *optional*, defaults to `None`): + Whether to use `nin_shortcut`. This activates a new layer inside ResNet block + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int + out_channels: int = None + dropout: float = 0.0 + groups: int = 32 + use_nin_shortcut: bool = None + dtype: jnp.dtype = jnp.float32 + + def setup(self): + out_channels = self.in_channels if self.out_channels is None else self.out_channels + + self.norm1 = nn.GroupNorm(num_groups=self.groups, epsilon=1e-6) + self.conv1 = nn.Conv( + out_channels, + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + self.norm2 = nn.GroupNorm(num_groups=self.groups, epsilon=1e-6) + self.dropout_layer = nn.Dropout(self.dropout) + self.conv2 = nn.Conv( + out_channels, + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + use_nin_shortcut = self.in_channels != out_channels if self.use_nin_shortcut is None else self.use_nin_shortcut + + self.conv_shortcut = None + if use_nin_shortcut: + self.conv_shortcut = nn.Conv( + out_channels, + kernel_size=(1, 1), + strides=(1, 1), + padding="VALID", + dtype=self.dtype, + ) + + def __call__(self, hidden_states, deterministic=True): + residual = hidden_states + hidden_states = self.norm1(hidden_states) + hidden_states = nn.swish(hidden_states) + hidden_states = self.conv1(hidden_states) + + hidden_states = self.norm2(hidden_states) + hidden_states = nn.swish(hidden_states) + hidden_states = self.dropout_layer(hidden_states, deterministic) + hidden_states = self.conv2(hidden_states) + + if self.conv_shortcut is not None: + residual = self.conv_shortcut(residual) + + return hidden_states + residual + + +class FlaxAttentionBlock(nn.Module): + r""" + Flax Convolutional based multi-head attention block for diffusion-based VAE. + + Parameters: + channels (:obj:`int`): + Input channels + num_head_channels (:obj:`int`, *optional*, defaults to `None`): + Number of attention heads + num_groups (:obj:`int`, *optional*, defaults to `32`): + The number of groups to use for group norm + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + + """ + + channels: int + num_head_channels: int = None + num_groups: int = 32 + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.num_heads = self.channels // self.num_head_channels if self.num_head_channels is not None else 1 + + dense = partial(nn.Dense, self.channels, dtype=self.dtype) + + self.group_norm = nn.GroupNorm(num_groups=self.num_groups, epsilon=1e-6) + self.query, self.key, self.value = dense(), dense(), dense() + self.proj_attn = dense() + + def transpose_for_scores(self, projection): + new_projection_shape = projection.shape[:-1] + (self.num_heads, -1) + # move heads to 2nd position (B, T, H * D) -> (B, T, H, D) + new_projection = projection.reshape(new_projection_shape) + # (B, T, H, D) -> (B, H, T, D) + new_projection = jnp.transpose(new_projection, (0, 2, 1, 3)) + return new_projection + + def __call__(self, hidden_states): + residual = hidden_states + batch, height, width, channels = hidden_states.shape + + hidden_states = self.group_norm(hidden_states) + + hidden_states = hidden_states.reshape((batch, height * width, channels)) + + query = self.query(hidden_states) + key = self.key(hidden_states) + value = self.value(hidden_states) + + # transpose + query = self.transpose_for_scores(query) + key = self.transpose_for_scores(key) + value = self.transpose_for_scores(value) + + # compute attentions + scale = 1 / math.sqrt(math.sqrt(self.channels / self.num_heads)) + attn_weights = jnp.einsum("...qc,...kc->...qk", query * scale, key * scale) + attn_weights = nn.softmax(attn_weights, axis=-1) + + # attend to values + hidden_states = jnp.einsum("...kc,...qk->...qc", value, attn_weights) + + hidden_states = jnp.transpose(hidden_states, (0, 2, 1, 3)) + new_hidden_states_shape = hidden_states.shape[:-2] + (self.channels,) + hidden_states = hidden_states.reshape(new_hidden_states_shape) + + hidden_states = self.proj_attn(hidden_states) + hidden_states = hidden_states.reshape((batch, height, width, channels)) + hidden_states = hidden_states + residual + return hidden_states + + +class FlaxDownEncoderBlock2D(nn.Module): + r""" + Flax Resnet blocks-based Encoder block for diffusion-based VAE. + + Parameters: + in_channels (:obj:`int`): + Input channels + out_channels (:obj:`int`): + Output channels + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + num_layers (:obj:`int`, *optional*, defaults to 1): + Number of Resnet layer block + resnet_groups (:obj:`int`, *optional*, defaults to `32`): + The number of groups to use for the Resnet block group norm + add_downsample (:obj:`bool`, *optional*, defaults to `True`): + Whether to add downsample layer + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int + out_channels: int + dropout: float = 0.0 + num_layers: int = 1 + resnet_groups: int = 32 + add_downsample: bool = True + dtype: jnp.dtype = jnp.float32 + + def setup(self): + resnets = [] + for i in range(self.num_layers): + in_channels = self.in_channels if i == 0 else self.out_channels + + res_block = FlaxResnetBlock2D( + in_channels=in_channels, + out_channels=self.out_channels, + dropout=self.dropout, + groups=self.resnet_groups, + dtype=self.dtype, + ) + resnets.append(res_block) + self.resnets = resnets + + if self.add_downsample: + self.downsamplers_0 = FlaxDownsample2D(self.out_channels, dtype=self.dtype) + + def __call__(self, hidden_states, deterministic=True): + for resnet in self.resnets: + hidden_states = resnet(hidden_states, deterministic=deterministic) + + if self.add_downsample: + hidden_states = self.downsamplers_0(hidden_states) + + return hidden_states + + +class FlaxUpDecoderBlock2D(nn.Module): + r""" + Flax Resnet blocks-based Decoder block for diffusion-based VAE. + + Parameters: + in_channels (:obj:`int`): + Input channels + out_channels (:obj:`int`): + Output channels + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + num_layers (:obj:`int`, *optional*, defaults to 1): + Number of Resnet layer block + resnet_groups (:obj:`int`, *optional*, defaults to `32`): + The number of groups to use for the Resnet block group norm + add_upsample (:obj:`bool`, *optional*, defaults to `True`): + Whether to add upsample layer + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int + out_channels: int + dropout: float = 0.0 + num_layers: int = 1 + resnet_groups: int = 32 + add_upsample: bool = True + dtype: jnp.dtype = jnp.float32 + + def setup(self): + resnets = [] + for i in range(self.num_layers): + in_channels = self.in_channels if i == 0 else self.out_channels + res_block = FlaxResnetBlock2D( + in_channels=in_channels, + out_channels=self.out_channels, + dropout=self.dropout, + groups=self.resnet_groups, + dtype=self.dtype, + ) + resnets.append(res_block) + + self.resnets = resnets + + if self.add_upsample: + self.upsamplers_0 = FlaxUpsample2D(self.out_channels, dtype=self.dtype) + + def __call__(self, hidden_states, deterministic=True): + for resnet in self.resnets: + hidden_states = resnet(hidden_states, deterministic=deterministic) + + if self.add_upsample: + hidden_states = self.upsamplers_0(hidden_states) + + return hidden_states + + +class FlaxUNetMidBlock2D(nn.Module): + r""" + Flax Unet Mid-Block module. + + Parameters: + in_channels (:obj:`int`): + Input channels + dropout (:obj:`float`, *optional*, defaults to 0.0): + Dropout rate + num_layers (:obj:`int`, *optional*, defaults to 1): + Number of Resnet layer block + resnet_groups (:obj:`int`, *optional*, defaults to `32`): + The number of groups to use for the Resnet and Attention block group norm + num_attention_heads (:obj:`int`, *optional*, defaults to `1`): + Number of attention heads for each attention block + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int + dropout: float = 0.0 + num_layers: int = 1 + resnet_groups: int = 32 + num_attention_heads: int = 1 + dtype: jnp.dtype = jnp.float32 + + def setup(self): + resnet_groups = self.resnet_groups if self.resnet_groups is not None else min(self.in_channels // 4, 32) + + # there is always at least one resnet + resnets = [ + FlaxResnetBlock2D( + in_channels=self.in_channels, + out_channels=self.in_channels, + dropout=self.dropout, + groups=resnet_groups, + dtype=self.dtype, + ) + ] + + attentions = [] + + for _ in range(self.num_layers): + attn_block = FlaxAttentionBlock( + channels=self.in_channels, + num_head_channels=self.num_attention_heads, + num_groups=resnet_groups, + dtype=self.dtype, + ) + attentions.append(attn_block) + + res_block = FlaxResnetBlock2D( + in_channels=self.in_channels, + out_channels=self.in_channels, + dropout=self.dropout, + groups=resnet_groups, + dtype=self.dtype, + ) + resnets.append(res_block) + + self.resnets = resnets + self.attentions = attentions + + def __call__(self, hidden_states, deterministic=True): + hidden_states = self.resnets[0](hidden_states, deterministic=deterministic) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + hidden_states = attn(hidden_states) + hidden_states = resnet(hidden_states, deterministic=deterministic) + + return hidden_states + + +class FlaxEncoder(nn.Module): + r""" + Flax Implementation of VAE Encoder. + + This model is a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) + subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to + general usage and behavior. + + Finally, this model supports inherent JAX features such as: + - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit) + - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation) + - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap) + - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap) + + Parameters: + in_channels (:obj:`int`, *optional*, defaults to 3): + Input channels + out_channels (:obj:`int`, *optional*, defaults to 3): + Output channels + down_block_types (:obj:`Tuple[str]`, *optional*, defaults to `(DownEncoderBlock2D)`): + DownEncoder block type + block_out_channels (:obj:`Tuple[str]`, *optional*, defaults to `(64,)`): + Tuple containing the number of output channels for each block + layers_per_block (:obj:`int`, *optional*, defaults to `2`): + Number of Resnet layer for each block + norm_num_groups (:obj:`int`, *optional*, defaults to `32`): + norm num group + act_fn (:obj:`str`, *optional*, defaults to `silu`): + Activation function + double_z (:obj:`bool`, *optional*, defaults to `False`): + Whether to double the last output channels + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + Parameters `dtype` + """ + + in_channels: int = 3 + out_channels: int = 3 + down_block_types: Tuple[str] = ("DownEncoderBlock2D",) + block_out_channels: Tuple[int] = (64,) + layers_per_block: int = 2 + norm_num_groups: int = 32 + act_fn: str = "silu" + double_z: bool = False + dtype: jnp.dtype = jnp.float32 + + def setup(self): + block_out_channels = self.block_out_channels + # in + self.conv_in = nn.Conv( + block_out_channels[0], + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + # downsampling + down_blocks = [] + output_channel = block_out_channels[0] + for i, _ in enumerate(self.down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = FlaxDownEncoderBlock2D( + in_channels=input_channel, + out_channels=output_channel, + num_layers=self.layers_per_block, + resnet_groups=self.norm_num_groups, + add_downsample=not is_final_block, + dtype=self.dtype, + ) + down_blocks.append(down_block) + self.down_blocks = down_blocks + + # middle + self.mid_block = FlaxUNetMidBlock2D( + in_channels=block_out_channels[-1], + resnet_groups=self.norm_num_groups, + num_attention_heads=None, + dtype=self.dtype, + ) + + # end + conv_out_channels = 2 * self.out_channels if self.double_z else self.out_channels + self.conv_norm_out = nn.GroupNorm(num_groups=self.norm_num_groups, epsilon=1e-6) + self.conv_out = nn.Conv( + conv_out_channels, + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + def __call__(self, sample, deterministic: bool = True): + # in + sample = self.conv_in(sample) + + # downsampling + for block in self.down_blocks: + sample = block(sample, deterministic=deterministic) + + # middle + sample = self.mid_block(sample, deterministic=deterministic) + + # end + sample = self.conv_norm_out(sample) + sample = nn.swish(sample) + sample = self.conv_out(sample) + + return sample + + +class FlaxDecoder(nn.Module): + r""" + Flax Implementation of VAE Decoder. + + This model is a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) + subclass. Use it as a regular Flax linen Module and refer to the Flax documentation for all matter related to + general usage and behavior. + + Finally, this model supports inherent JAX features such as: + - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit) + - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation) + - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap) + - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap) + + Parameters: + in_channels (:obj:`int`, *optional*, defaults to 3): + Input channels + out_channels (:obj:`int`, *optional*, defaults to 3): + Output channels + up_block_types (:obj:`Tuple[str]`, *optional*, defaults to `(UpDecoderBlock2D)`): + UpDecoder block type + block_out_channels (:obj:`Tuple[str]`, *optional*, defaults to `(64,)`): + Tuple containing the number of output channels for each block + layers_per_block (:obj:`int`, *optional*, defaults to `2`): + Number of Resnet layer for each block + norm_num_groups (:obj:`int`, *optional*, defaults to `32`): + norm num group + act_fn (:obj:`str`, *optional*, defaults to `silu`): + Activation function + double_z (:obj:`bool`, *optional*, defaults to `False`): + Whether to double the last output channels + dtype (:obj:`jnp.dtype`, *optional*, defaults to jnp.float32): + parameters `dtype` + """ + + in_channels: int = 3 + out_channels: int = 3 + up_block_types: Tuple[str] = ("UpDecoderBlock2D",) + block_out_channels: int = (64,) + layers_per_block: int = 2 + norm_num_groups: int = 32 + act_fn: str = "silu" + dtype: jnp.dtype = jnp.float32 + + def setup(self): + block_out_channels = self.block_out_channels + + # z to block_in + self.conv_in = nn.Conv( + block_out_channels[-1], + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + # middle + self.mid_block = FlaxUNetMidBlock2D( + in_channels=block_out_channels[-1], + resnet_groups=self.norm_num_groups, + num_attention_heads=None, + dtype=self.dtype, + ) + + # upsampling + reversed_block_out_channels = list(reversed(block_out_channels)) + output_channel = reversed_block_out_channels[0] + up_blocks = [] + for i, _ in enumerate(self.up_block_types): + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + + is_final_block = i == len(block_out_channels) - 1 + + up_block = FlaxUpDecoderBlock2D( + in_channels=prev_output_channel, + out_channels=output_channel, + num_layers=self.layers_per_block + 1, + resnet_groups=self.norm_num_groups, + add_upsample=not is_final_block, + dtype=self.dtype, + ) + up_blocks.append(up_block) + prev_output_channel = output_channel + + self.up_blocks = up_blocks + + # end + self.conv_norm_out = nn.GroupNorm(num_groups=self.norm_num_groups, epsilon=1e-6) + self.conv_out = nn.Conv( + self.out_channels, + kernel_size=(3, 3), + strides=(1, 1), + padding=((1, 1), (1, 1)), + dtype=self.dtype, + ) + + def __call__(self, sample, deterministic: bool = True): + # z to block_in + sample = self.conv_in(sample) + + # middle + sample = self.mid_block(sample, deterministic=deterministic) + + # upsampling + for block in self.up_blocks: + sample = block(sample, deterministic=deterministic) + + sample = self.conv_norm_out(sample) + sample = nn.swish(sample) + sample = self.conv_out(sample) + + return sample + + +class FlaxDiagonalGaussianDistribution(object): + def __init__(self, parameters, deterministic=False): + # Last axis to account for channels-last + self.mean, self.logvar = jnp.split(parameters, 2, axis=-1) + self.logvar = jnp.clip(self.logvar, -30.0, 20.0) + self.deterministic = deterministic + self.std = jnp.exp(0.5 * self.logvar) + self.var = jnp.exp(self.logvar) + if self.deterministic: + self.var = self.std = jnp.zeros_like(self.mean) + + def sample(self, key): + return self.mean + self.std * jax.random.normal(key, self.mean.shape) + + def kl(self, other=None): + if self.deterministic: + return jnp.array([0.0]) + + if other is None: + return 0.5 * jnp.sum(self.mean**2 + self.var - 1.0 - self.logvar, axis=[1, 2, 3]) + + return 0.5 * jnp.sum( + jnp.square(self.mean - other.mean) / other.var + self.var / other.var - 1.0 - self.logvar + other.logvar, + axis=[1, 2, 3], + ) + + def nll(self, sample, axis=[1, 2, 3]): + if self.deterministic: + return jnp.array([0.0]) + + logtwopi = jnp.log(2.0 * jnp.pi) + return 0.5 * jnp.sum(logtwopi + self.logvar + jnp.square(sample - self.mean) / self.var, axis=axis) + + def mode(self): + return self.mean + + +@flax_register_to_config +class FlaxAutoencoderKL(nn.Module, FlaxModelMixin, ConfigMixin): + r""" + Flax implementation of a VAE model with KL loss for decoding latent representations. + + This model inherits from [`FlaxModelMixin`]. Check the superclass documentation for it's generic methods + implemented for all models (such as downloading or saving). + + This model is a Flax Linen [flax.linen.Module](https://flax.readthedocs.io/en/latest/flax.linen.html#module) + subclass. Use it as a regular Flax Linen module and refer to the Flax documentation for all matter related to its + general usage and behavior. + + Inherent JAX features such as the following are supported: + + - [Just-In-Time (JIT) compilation](https://jax.readthedocs.io/en/latest/jax.html#just-in-time-compilation-jit) + - [Automatic Differentiation](https://jax.readthedocs.io/en/latest/jax.html#automatic-differentiation) + - [Vectorization](https://jax.readthedocs.io/en/latest/jax.html#vectorization-vmap) + - [Parallelization](https://jax.readthedocs.io/en/latest/jax.html#parallelization-pmap) + + Parameters: + in_channels (`int`, *optional*, defaults to 3): + Number of channels in the input image. + out_channels (`int`, *optional*, defaults to 3): + Number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `(DownEncoderBlock2D)`): + Tuple of downsample block types. + up_block_types (`Tuple[str]`, *optional*, defaults to `(UpDecoderBlock2D)`): + Tuple of upsample block types. + block_out_channels (`Tuple[str]`, *optional*, defaults to `(64,)`): + Tuple of block output channels. + layers_per_block (`int`, *optional*, defaults to `2`): + Number of ResNet layer for each block. + act_fn (`str`, *optional*, defaults to `silu`): + The activation function to use. + latent_channels (`int`, *optional*, defaults to `4`): + Number of channels in the latent space. + norm_num_groups (`int`, *optional*, defaults to `32`): + The number of groups for normalization. + sample_size (`int`, *optional*, defaults to 32): + Sample input size. + scaling_factor (`float`, *optional*, defaults to 0.18215): + The component-wise standard deviation of the trained latent space computed using the first batch of the + training set. This is used to scale the latent space to have unit variance when training the diffusion + model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the + diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1 + / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image + Synthesis with Latent Diffusion Models](https://arxiv.org/abs/2112.10752) paper. + dtype (`jnp.dtype`, *optional*, defaults to `jnp.float32`): + The `dtype` of the parameters. + """ + + in_channels: int = 3 + out_channels: int = 3 + down_block_types: Tuple[str] = ("DownEncoderBlock2D",) + up_block_types: Tuple[str] = ("UpDecoderBlock2D",) + block_out_channels: Tuple[int] = (64,) + layers_per_block: int = 1 + act_fn: str = "silu" + latent_channels: int = 4 + norm_num_groups: int = 32 + sample_size: int = 32 + scaling_factor: float = 0.18215 + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.encoder = FlaxEncoder( + in_channels=self.config.in_channels, + out_channels=self.config.latent_channels, + down_block_types=self.config.down_block_types, + block_out_channels=self.config.block_out_channels, + layers_per_block=self.config.layers_per_block, + act_fn=self.config.act_fn, + norm_num_groups=self.config.norm_num_groups, + double_z=True, + dtype=self.dtype, + ) + self.decoder = FlaxDecoder( + in_channels=self.config.latent_channels, + out_channels=self.config.out_channels, + up_block_types=self.config.up_block_types, + block_out_channels=self.config.block_out_channels, + layers_per_block=self.config.layers_per_block, + norm_num_groups=self.config.norm_num_groups, + act_fn=self.config.act_fn, + dtype=self.dtype, + ) + self.quant_conv = nn.Conv( + 2 * self.config.latent_channels, + kernel_size=(1, 1), + strides=(1, 1), + padding="VALID", + dtype=self.dtype, + ) + self.post_quant_conv = nn.Conv( + self.config.latent_channels, + kernel_size=(1, 1), + strides=(1, 1), + padding="VALID", + dtype=self.dtype, + ) + + def init_weights(self, rng: jax.Array) -> FrozenDict: + # init input tensors + sample_shape = (1, self.in_channels, self.sample_size, self.sample_size) + sample = jnp.zeros(sample_shape, dtype=jnp.float32) + + params_rng, dropout_rng, gaussian_rng = jax.random.split(rng, 3) + rngs = {"params": params_rng, "dropout": dropout_rng, "gaussian": gaussian_rng} + + return self.init(rngs, sample)["params"] + + def encode(self, sample, deterministic: bool = True, return_dict: bool = True): + sample = jnp.transpose(sample, (0, 2, 3, 1)) + + hidden_states = self.encoder(sample, deterministic=deterministic) + moments = self.quant_conv(hidden_states) + posterior = FlaxDiagonalGaussianDistribution(moments) + + if not return_dict: + return (posterior,) + + return FlaxAutoencoderKLOutput(latent_dist=posterior) + + def decode(self, latents, deterministic: bool = True, return_dict: bool = True): + if latents.shape[-1] != self.config.latent_channels: + latents = jnp.transpose(latents, (0, 2, 3, 1)) + + hidden_states = self.post_quant_conv(latents) + hidden_states = self.decoder(hidden_states, deterministic=deterministic) + + hidden_states = jnp.transpose(hidden_states, (0, 3, 1, 2)) + + if not return_dict: + return (hidden_states,) + + return FlaxDecoderOutput(sample=hidden_states) + + def __call__(self, sample, sample_posterior=False, deterministic: bool = True, return_dict: bool = True): + posterior = self.encode(sample, deterministic=deterministic, return_dict=return_dict) + if sample_posterior: + rng = self.make_rng("gaussian") + hidden_states = posterior.latent_dist.sample(rng) + else: + hidden_states = posterior.latent_dist.mode() + + sample = self.decode(hidden_states, return_dict=return_dict).sample + + if not return_dict: + return (sample,) + + return FlaxDecoderOutput(sample=sample) diff --git a/diffusers-0.27.0/src/diffusers/models/vq_model.py b/diffusers-0.27.0/src/diffusers/models/vq_model.py new file mode 100755 index 0000000..e518444 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/models/vq_model.py @@ -0,0 +1,181 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import torch +import torch.nn as nn + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from ..utils.accelerate_utils import apply_forward_hook +from .autoencoders.vae import Decoder, DecoderOutput, Encoder, VectorQuantizer +from .modeling_utils import ModelMixin + + +@dataclass +class VQEncoderOutput(BaseOutput): + """ + Output of VQModel encoding method. + + Args: + latents (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): + The encoded output sample from the last layer of the model. + """ + + latents: torch.FloatTensor + + +class VQModel(ModelMixin, ConfigMixin): + r""" + A VQ-VAE model for decoding latent representations. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + in_channels (int, *optional*, defaults to 3): Number of channels in the input image. + out_channels (int, *optional*, defaults to 3): Number of channels in the output. + down_block_types (`Tuple[str]`, *optional*, defaults to `("DownEncoderBlock2D",)`): + Tuple of downsample block types. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpDecoderBlock2D",)`): + Tuple of upsample block types. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(64,)`): + Tuple of block output channels. + layers_per_block (`int`, *optional*, defaults to `1`): Number of layers per block. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + latent_channels (`int`, *optional*, defaults to `3`): Number of channels in the latent space. + sample_size (`int`, *optional*, defaults to `32`): Sample input size. + num_vq_embeddings (`int`, *optional*, defaults to `256`): Number of codebook vectors in the VQ-VAE. + norm_num_groups (`int`, *optional*, defaults to `32`): Number of groups for normalization layers. + vq_embed_dim (`int`, *optional*): Hidden dim of codebook vectors in the VQ-VAE. + scaling_factor (`float`, *optional*, defaults to `0.18215`): + The component-wise standard deviation of the trained latent space computed using the first batch of the + training set. This is used to scale the latent space to have unit variance when training the diffusion + model. The latents are scaled with the formula `z = z * scaling_factor` before being passed to the + diffusion model. When decoding, the latents are scaled back to the original scale with the formula: `z = 1 + / scaling_factor * z`. For more details, refer to sections 4.3.2 and D.1 of the [High-Resolution Image + Synthesis with Latent Diffusion Models](https://arxiv.org/abs/2112.10752) paper. + norm_type (`str`, *optional*, defaults to `"group"`): + Type of normalization layer to use. Can be one of `"group"` or `"spatial"`. + """ + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + down_block_types: Tuple[str, ...] = ("DownEncoderBlock2D",), + up_block_types: Tuple[str, ...] = ("UpDecoderBlock2D",), + block_out_channels: Tuple[int, ...] = (64,), + layers_per_block: int = 1, + act_fn: str = "silu", + latent_channels: int = 3, + sample_size: int = 32, + num_vq_embeddings: int = 256, + norm_num_groups: int = 32, + vq_embed_dim: Optional[int] = None, + scaling_factor: float = 0.18215, + norm_type: str = "group", # group, spatial + mid_block_add_attention=True, + lookup_from_codebook=False, + force_upcast=False, + ): + super().__init__() + + # pass init params to Encoder + self.encoder = Encoder( + in_channels=in_channels, + out_channels=latent_channels, + down_block_types=down_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn=act_fn, + norm_num_groups=norm_num_groups, + double_z=False, + mid_block_add_attention=mid_block_add_attention, + ) + + vq_embed_dim = vq_embed_dim if vq_embed_dim is not None else latent_channels + + self.quant_conv = nn.Conv2d(latent_channels, vq_embed_dim, 1) + self.quantize = VectorQuantizer(num_vq_embeddings, vq_embed_dim, beta=0.25, remap=None, sane_index_shape=False) + self.post_quant_conv = nn.Conv2d(vq_embed_dim, latent_channels, 1) + + # pass init params to Decoder + self.decoder = Decoder( + in_channels=latent_channels, + out_channels=out_channels, + up_block_types=up_block_types, + block_out_channels=block_out_channels, + layers_per_block=layers_per_block, + act_fn=act_fn, + norm_num_groups=norm_num_groups, + norm_type=norm_type, + mid_block_add_attention=mid_block_add_attention, + ) + + @apply_forward_hook + def encode(self, x: torch.FloatTensor, return_dict: bool = True) -> VQEncoderOutput: + h = self.encoder(x) + h = self.quant_conv(h) + + if not return_dict: + return (h,) + + return VQEncoderOutput(latents=h) + + @apply_forward_hook + def decode( + self, h: torch.FloatTensor, force_not_quantize: bool = False, return_dict: bool = True, shape=None + ) -> Union[DecoderOutput, torch.FloatTensor]: + # also go through quantization layer + if not force_not_quantize: + quant, _, _ = self.quantize(h) + elif self.config.lookup_from_codebook: + quant = self.quantize.get_codebook_entry(h, shape) + else: + quant = h + quant2 = self.post_quant_conv(quant) + dec = self.decoder(quant2, quant if self.config.norm_type == "spatial" else None) + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + def forward( + self, sample: torch.FloatTensor, return_dict: bool = True + ) -> Union[DecoderOutput, Tuple[torch.FloatTensor, ...]]: + r""" + The [`VQModel`] forward method. + + Args: + sample (`torch.FloatTensor`): Input sample. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`models.vq_model.VQEncoderOutput`] instead of a plain tuple. + + Returns: + [`~models.vq_model.VQEncoderOutput`] or `tuple`: + If return_dict is True, a [`~models.vq_model.VQEncoderOutput`] is returned, otherwise a plain `tuple` + is returned. + """ + + h = self.encode(sample).latents + dec = self.decode(h).sample + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) diff --git a/diffusers-0.27.0/src/diffusers/optimization.py b/diffusers-0.27.0/src/diffusers/optimization.py new file mode 100755 index 0000000..fbaa143 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/optimization.py @@ -0,0 +1,361 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""PyTorch optimization for diffusion models.""" + +import math +from enum import Enum +from typing import Optional, Union + +from torch.optim import Optimizer +from torch.optim.lr_scheduler import LambdaLR + +from .utils import logging + + +logger = logging.get_logger(__name__) + + +class SchedulerType(Enum): + LINEAR = "linear" + COSINE = "cosine" + COSINE_WITH_RESTARTS = "cosine_with_restarts" + POLYNOMIAL = "polynomial" + CONSTANT = "constant" + CONSTANT_WITH_WARMUP = "constant_with_warmup" + PIECEWISE_CONSTANT = "piecewise_constant" + + +def get_constant_schedule(optimizer: Optimizer, last_epoch: int = -1) -> LambdaLR: + """ + Create a schedule with a constant learning rate, using the learning rate set in optimizer. + + Args: + optimizer ([`~torch.optim.Optimizer`]): + The optimizer for which to schedule the learning rate. + last_epoch (`int`, *optional*, defaults to -1): + The index of the last epoch when resuming training. + + Return: + `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. + """ + return LambdaLR(optimizer, lambda _: 1, last_epoch=last_epoch) + + +def get_constant_schedule_with_warmup(optimizer: Optimizer, num_warmup_steps: int, last_epoch: int = -1) -> LambdaLR: + """ + Create a schedule with a constant learning rate preceded by a warmup period during which the learning rate + increases linearly between 0 and the initial lr set in the optimizer. + + Args: + optimizer ([`~torch.optim.Optimizer`]): + The optimizer for which to schedule the learning rate. + num_warmup_steps (`int`): + The number of steps for the warmup phase. + last_epoch (`int`, *optional*, defaults to -1): + The index of the last epoch when resuming training. + + Return: + `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. + """ + + def lr_lambda(current_step: int): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1.0, num_warmup_steps)) + return 1.0 + + return LambdaLR(optimizer, lr_lambda, last_epoch=last_epoch) + + +def get_piecewise_constant_schedule(optimizer: Optimizer, step_rules: str, last_epoch: int = -1) -> LambdaLR: + """ + Create a schedule with a constant learning rate, using the learning rate set in optimizer. + + Args: + optimizer ([`~torch.optim.Optimizer`]): + The optimizer for which to schedule the learning rate. + step_rules (`string`): + The rules for the learning rate. ex: rule_steps="1:10,0.1:20,0.01:30,0.005" it means that the learning rate + if multiple 1 for the first 10 steps, mutiple 0.1 for the next 20 steps, multiple 0.01 for the next 30 + steps and multiple 0.005 for the other steps. + last_epoch (`int`, *optional*, defaults to -1): + The index of the last epoch when resuming training. + + Return: + `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. + """ + + rules_dict = {} + rule_list = step_rules.split(",") + for rule_str in rule_list[:-1]: + value_str, steps_str = rule_str.split(":") + steps = int(steps_str) + value = float(value_str) + rules_dict[steps] = value + last_lr_multiple = float(rule_list[-1]) + + def create_rules_function(rules_dict, last_lr_multiple): + def rule_func(steps: int) -> float: + sorted_steps = sorted(rules_dict.keys()) + for i, sorted_step in enumerate(sorted_steps): + if steps < sorted_step: + return rules_dict[sorted_steps[i]] + return last_lr_multiple + + return rule_func + + rules_func = create_rules_function(rules_dict, last_lr_multiple) + + return LambdaLR(optimizer, rules_func, last_epoch=last_epoch) + + +def get_linear_schedule_with_warmup( + optimizer: Optimizer, num_warmup_steps: int, num_training_steps: int, last_epoch: int = -1 +) -> LambdaLR: + """ + Create a schedule with a learning rate that decreases linearly from the initial lr set in the optimizer to 0, after + a warmup period during which it increases linearly from 0 to the initial lr set in the optimizer. + + Args: + optimizer ([`~torch.optim.Optimizer`]): + The optimizer for which to schedule the learning rate. + num_warmup_steps (`int`): + The number of steps for the warmup phase. + num_training_steps (`int`): + The total number of training steps. + last_epoch (`int`, *optional*, defaults to -1): + The index of the last epoch when resuming training. + + Return: + `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. + """ + + def lr_lambda(current_step: int): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + return max( + 0.0, float(num_training_steps - current_step) / float(max(1, num_training_steps - num_warmup_steps)) + ) + + return LambdaLR(optimizer, lr_lambda, last_epoch) + + +def get_cosine_schedule_with_warmup( + optimizer: Optimizer, num_warmup_steps: int, num_training_steps: int, num_cycles: float = 0.5, last_epoch: int = -1 +) -> LambdaLR: + """ + Create a schedule with a learning rate that decreases following the values of the cosine function between the + initial lr set in the optimizer to 0, after a warmup period during which it increases linearly between 0 and the + initial lr set in the optimizer. + + Args: + optimizer ([`~torch.optim.Optimizer`]): + The optimizer for which to schedule the learning rate. + num_warmup_steps (`int`): + The number of steps for the warmup phase. + num_training_steps (`int`): + The total number of training steps. + num_periods (`float`, *optional*, defaults to 0.5): + The number of periods of the cosine function in a schedule (the default is to just decrease from the max + value to 0 following a half-cosine). + last_epoch (`int`, *optional*, defaults to -1): + The index of the last epoch when resuming training. + + Return: + `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. + """ + + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps)) + return max(0.0, 0.5 * (1.0 + math.cos(math.pi * float(num_cycles) * 2.0 * progress))) + + return LambdaLR(optimizer, lr_lambda, last_epoch) + + +def get_cosine_with_hard_restarts_schedule_with_warmup( + optimizer: Optimizer, num_warmup_steps: int, num_training_steps: int, num_cycles: int = 1, last_epoch: int = -1 +) -> LambdaLR: + """ + Create a schedule with a learning rate that decreases following the values of the cosine function between the + initial lr set in the optimizer to 0, with several hard restarts, after a warmup period during which it increases + linearly between 0 and the initial lr set in the optimizer. + + Args: + optimizer ([`~torch.optim.Optimizer`]): + The optimizer for which to schedule the learning rate. + num_warmup_steps (`int`): + The number of steps for the warmup phase. + num_training_steps (`int`): + The total number of training steps. + num_cycles (`int`, *optional*, defaults to 1): + The number of hard restarts to use. + last_epoch (`int`, *optional*, defaults to -1): + The index of the last epoch when resuming training. + + Return: + `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. + """ + + def lr_lambda(current_step): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + progress = float(current_step - num_warmup_steps) / float(max(1, num_training_steps - num_warmup_steps)) + if progress >= 1.0: + return 0.0 + return max(0.0, 0.5 * (1.0 + math.cos(math.pi * ((float(num_cycles) * progress) % 1.0)))) + + return LambdaLR(optimizer, lr_lambda, last_epoch) + + +def get_polynomial_decay_schedule_with_warmup( + optimizer: Optimizer, + num_warmup_steps: int, + num_training_steps: int, + lr_end: float = 1e-7, + power: float = 1.0, + last_epoch: int = -1, +) -> LambdaLR: + """ + Create a schedule with a learning rate that decreases as a polynomial decay from the initial lr set in the + optimizer to end lr defined by *lr_end*, after a warmup period during which it increases linearly from 0 to the + initial lr set in the optimizer. + + Args: + optimizer ([`~torch.optim.Optimizer`]): + The optimizer for which to schedule the learning rate. + num_warmup_steps (`int`): + The number of steps for the warmup phase. + num_training_steps (`int`): + The total number of training steps. + lr_end (`float`, *optional*, defaults to 1e-7): + The end LR. + power (`float`, *optional*, defaults to 1.0): + Power factor. + last_epoch (`int`, *optional*, defaults to -1): + The index of the last epoch when resuming training. + + Note: *power* defaults to 1.0 as in the fairseq implementation, which in turn is based on the original BERT + implementation at + https://github.com/google-research/bert/blob/f39e881b169b9d53bea03d2d341b31707a6c052b/optimization.py#L37 + + Return: + `torch.optim.lr_scheduler.LambdaLR` with the appropriate schedule. + + """ + + lr_init = optimizer.defaults["lr"] + if not (lr_init > lr_end): + raise ValueError(f"lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})") + + def lr_lambda(current_step: int): + if current_step < num_warmup_steps: + return float(current_step) / float(max(1, num_warmup_steps)) + elif current_step > num_training_steps: + return lr_end / lr_init # as LambdaLR multiplies by lr_init + else: + lr_range = lr_init - lr_end + decay_steps = num_training_steps - num_warmup_steps + pct_remaining = 1 - (current_step - num_warmup_steps) / decay_steps + decay = lr_range * pct_remaining**power + lr_end + return decay / lr_init # as LambdaLR multiplies by lr_init + + return LambdaLR(optimizer, lr_lambda, last_epoch) + + +TYPE_TO_SCHEDULER_FUNCTION = { + SchedulerType.LINEAR: get_linear_schedule_with_warmup, + SchedulerType.COSINE: get_cosine_schedule_with_warmup, + SchedulerType.COSINE_WITH_RESTARTS: get_cosine_with_hard_restarts_schedule_with_warmup, + SchedulerType.POLYNOMIAL: get_polynomial_decay_schedule_with_warmup, + SchedulerType.CONSTANT: get_constant_schedule, + SchedulerType.CONSTANT_WITH_WARMUP: get_constant_schedule_with_warmup, + SchedulerType.PIECEWISE_CONSTANT: get_piecewise_constant_schedule, +} + + +def get_scheduler( + name: Union[str, SchedulerType], + optimizer: Optimizer, + step_rules: Optional[str] = None, + num_warmup_steps: Optional[int] = None, + num_training_steps: Optional[int] = None, + num_cycles: int = 1, + power: float = 1.0, + last_epoch: int = -1, +) -> LambdaLR: + """ + Unified API to get any scheduler from its name. + + Args: + name (`str` or `SchedulerType`): + The name of the scheduler to use. + optimizer (`torch.optim.Optimizer`): + The optimizer that will be used during training. + step_rules (`str`, *optional*): + A string representing the step rules to use. This is only used by the `PIECEWISE_CONSTANT` scheduler. + num_warmup_steps (`int`, *optional*): + The number of warmup steps to do. This is not required by all schedulers (hence the argument being + optional), the function will raise an error if it's unset and the scheduler type requires it. + num_training_steps (`int``, *optional*): + The number of training steps to do. This is not required by all schedulers (hence the argument being + optional), the function will raise an error if it's unset and the scheduler type requires it. + num_cycles (`int`, *optional*): + The number of hard restarts used in `COSINE_WITH_RESTARTS` scheduler. + power (`float`, *optional*, defaults to 1.0): + Power factor. See `POLYNOMIAL` scheduler + last_epoch (`int`, *optional*, defaults to -1): + The index of the last epoch when resuming training. + """ + name = SchedulerType(name) + schedule_func = TYPE_TO_SCHEDULER_FUNCTION[name] + if name == SchedulerType.CONSTANT: + return schedule_func(optimizer, last_epoch=last_epoch) + + if name == SchedulerType.PIECEWISE_CONSTANT: + return schedule_func(optimizer, step_rules=step_rules, last_epoch=last_epoch) + + # All other schedulers require `num_warmup_steps` + if num_warmup_steps is None: + raise ValueError(f"{name} requires `num_warmup_steps`, please provide that argument.") + + if name == SchedulerType.CONSTANT_WITH_WARMUP: + return schedule_func(optimizer, num_warmup_steps=num_warmup_steps, last_epoch=last_epoch) + + # All other schedulers require `num_training_steps` + if num_training_steps is None: + raise ValueError(f"{name} requires `num_training_steps`, please provide that argument.") + + if name == SchedulerType.COSINE_WITH_RESTARTS: + return schedule_func( + optimizer, + num_warmup_steps=num_warmup_steps, + num_training_steps=num_training_steps, + num_cycles=num_cycles, + last_epoch=last_epoch, + ) + + if name == SchedulerType.POLYNOMIAL: + return schedule_func( + optimizer, + num_warmup_steps=num_warmup_steps, + num_training_steps=num_training_steps, + power=power, + last_epoch=last_epoch, + ) + + return schedule_func( + optimizer, num_warmup_steps=num_warmup_steps, num_training_steps=num_training_steps, last_epoch=last_epoch + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/README.md b/diffusers-0.27.0/src/diffusers/pipelines/README.md new file mode 100755 index 0000000..d5125ae --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/README.md @@ -0,0 +1,171 @@ +# 🧨 Diffusers Pipelines + +Pipelines provide a simple way to run state-of-the-art diffusion models in inference. +Most diffusion systems consist of multiple independently-trained models and highly adaptable scheduler +components - all of which are needed to have a functioning end-to-end diffusion system. + +As an example, [Stable Diffusion](https://huggingface.co/blog/stable_diffusion) has three independently trained models: +- [Autoencoder](https://github.com/huggingface/diffusers/blob/5cbed8e0d157f65d3ddc2420dfd09f2df630e978/src/diffusers/models/vae.py#L392) +- [Conditional Unet](https://github.com/huggingface/diffusers/blob/5cbed8e0d157f65d3ddc2420dfd09f2df630e978/src/diffusers/models/unet_2d_condition.py#L12) +- [CLIP text encoder](https://huggingface.co/docs/transformers/main/en/model_doc/clip#transformers.CLIPTextModel) +- a scheduler component, [scheduler](https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_pndm.py), +- a [CLIPImageProcessor](https://huggingface.co/docs/transformers/main/en/model_doc/clip#transformers.CLIPImageProcessor), +- as well as a [safety checker](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/safety_checker.py). +All of these components are necessary to run stable diffusion in inference even though they were trained +or created independently from each other. + +To that end, we strive to offer all open-sourced, state-of-the-art diffusion system under a unified API. +More specifically, we strive to provide pipelines that +- 1. can load the officially published weights and yield 1-to-1 the same outputs as the original implementation according to the corresponding paper (*e.g.* [LDMTextToImagePipeline](https://github.com/huggingface/diffusers/tree/main/src/diffusers/pipelines/latent_diffusion), uses the officially released weights of [High-Resolution Image Synthesis with Latent Diffusion Models](https://arxiv.org/abs/2112.10752)), +- 2. have a simple user interface to run the model in inference (see the [Pipelines API](#pipelines-api) section), +- 3. are easy to understand with code that is self-explanatory and can be read along-side the official paper (see [Pipelines summary](#pipelines-summary)), +- 4. can easily be contributed by the community (see the [Contribution](#contribution) section). + +**Note** that pipelines do not (and should not) offer any training functionality. +If you are looking for *official* training examples, please have a look at [examples](https://github.com/huggingface/diffusers/tree/main/examples). + + +## Pipelines Summary + +The following table summarizes all officially supported pipelines, their corresponding paper, and if +available a colab notebook to directly try them out. + +| Pipeline | Source | Tasks | Colab +|-------------------------------------------------------------------------------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------|:---:|:---:| +| [dance diffusion](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/dance_diffusion) | [**Dance Diffusion**](https://github.com/Harmonai-org/sample-generator) | *Unconditional Audio Generation* | +| [ddpm](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/ddpm) | [**Denoising Diffusion Probabilistic Models**](https://arxiv.org/abs/2006.11239) | *Unconditional Image Generation* | +| [ddim](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/ddim) | [**Denoising Diffusion Implicit Models**](https://arxiv.org/abs/2010.02502) | *Unconditional Image Generation* | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/training_example.ipynb) +| [latent_diffusion](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/latent_diffusion) | [**High-Resolution Image Synthesis with Latent Diffusion Models**](https://arxiv.org/abs/2112.10752) | *Text-to-Image Generation* | +| [latent_diffusion_uncond](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/latent_diffusion_uncond) | [**High-Resolution Image Synthesis with Latent Diffusion Models**](https://arxiv.org/abs/2112.10752) | *Unconditional Image Generation* | +| [pndm](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/pndm) | [**Pseudo Numerical Methods for Diffusion Models on Manifolds**](https://arxiv.org/abs/2202.09778) | *Unconditional Image Generation* | +| [score_sde_ve](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/score_sde_ve) | [**Score-Based Generative Modeling through Stochastic Differential Equations**](https://openreview.net/forum?id=PxTIG12RRHS) | *Unconditional Image Generation* | +| [score_sde_vp](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/score_sde_vp) | [**Score-Based Generative Modeling through Stochastic Differential Equations**](https://openreview.net/forum?id=PxTIG12RRHS) | *Unconditional Image Generation* | +| [stable_diffusion](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion) | [**Stable Diffusion**](https://stability.ai/blog/stable-diffusion-public-release) | *Text-to-Image Generation* | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/stable_diffusion.ipynb) +| [stable_diffusion](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion) | [**Stable Diffusion**](https://stability.ai/blog/stable-diffusion-public-release) | *Image-to-Image Text-Guided Generation* | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/image_2_image_using_diffusers.ipynb) +| [stable_diffusion](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion) | [**Stable Diffusion**](https://stability.ai/blog/stable-diffusion-public-release) | *Text-Guided Image Inpainting* | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/in_painting_with_stable_diffusion_using_diffusers.ipynb) +| [stochastic_karras_ve](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stochastic_karras_ve) | [**Elucidating the Design Space of Diffusion-Based Generative Models**](https://arxiv.org/abs/2206.00364) | *Unconditional Image Generation* | + +**Note**: Pipelines are simple examples of how to play around with the diffusion systems as described in the corresponding papers. +However, most of them can be adapted to use different scheduler components or even different model components. Some pipeline examples are shown in the [Examples](#examples) below. + +## Pipelines API + +Diffusion models often consist of multiple independently-trained models or other previously existing components. + + +Each model has been trained independently on a different task and the scheduler can easily be swapped out and replaced with a different one. +During inference, we however want to be able to easily load all components and use them in inference - even if one component, *e.g.* CLIP's text encoder, originates from a different library, such as [Transformers](https://github.com/huggingface/transformers). To that end, all pipelines provide the following functionality: + +- [`from_pretrained` method](https://github.com/huggingface/diffusers/blob/5cbed8e0d157f65d3ddc2420dfd09f2df630e978/src/diffusers/pipeline_utils.py#L139) that accepts a Hugging Face Hub repository id, *e.g.* [runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5) or a path to a local directory, *e.g.* +"./stable-diffusion". To correctly retrieve which models and components should be loaded, one has to provide a `model_index.json` file, *e.g.* [runwayml/stable-diffusion-v1-5/model_index.json](https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/model_index.json), which defines all components that should be +loaded into the pipelines. More specifically, for each model/component one needs to define the format `: ["", ""]`. `` is the attribute name given to the loaded instance of `` which can be found in the library or pipeline folder called `""`. +- [`save_pretrained`](https://github.com/huggingface/diffusers/blob/5cbed8e0d157f65d3ddc2420dfd09f2df630e978/src/diffusers/pipeline_utils.py#L90) that accepts a local path, *e.g.* `./stable-diffusion` under which all models/components of the pipeline will be saved. For each component/model a folder is created inside the local path that is named after the given attribute name, *e.g.* `./stable_diffusion/unet`. +In addition, a `model_index.json` file is created at the root of the local path, *e.g.* `./stable_diffusion/model_index.json` so that the complete pipeline can again be instantiated +from the local path. +- [`to`](https://github.com/huggingface/diffusers/blob/5cbed8e0d157f65d3ddc2420dfd09f2df630e978/src/diffusers/pipeline_utils.py#L118) which accepts a `string` or `torch.device` to move all models that are of type `torch.nn.Module` to the passed device. The behavior is fully analogous to [PyTorch's `to` method](https://pytorch.org/docs/stable/generated/torch.nn.Module.html#torch.nn.Module.to). +- [`__call__`] method to use the pipeline in inference. `__call__` defines inference logic of the pipeline and should ideally encompass all aspects of it, from pre-processing to forwarding tensors to the different models and schedulers, as well as post-processing. The API of the `__call__` method can strongly vary from pipeline to pipeline. *E.g.* a text-to-image pipeline, such as [`StableDiffusionPipeline`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py) should accept among other things the text prompt to generate the image. A pure image generation pipeline, such as [DDPMPipeline](https://github.com/huggingface/diffusers/tree/main/src/diffusers/pipelines/ddpm) on the other hand can be run without providing any inputs. To better understand what inputs can be adapted for +each pipeline, one should look directly into the respective pipeline. + +**Note**: All pipelines have PyTorch's autograd disabled by decorating the `__call__` method with a [`torch.no_grad`](https://pytorch.org/docs/stable/generated/torch.no_grad.html) decorator because pipelines should +not be used for training. If you want to store the gradients during the forward pass, we recommend writing your own pipeline, see also our [community-examples](https://github.com/huggingface/diffusers/tree/main/examples/community) + +## Contribution + +We are more than happy about any contribution to the officially supported pipelines 🤗. We aspire +all of our pipelines to be **self-contained**, **easy-to-tweak**, **beginner-friendly** and for **one-purpose-only**. + +- **Self-contained**: A pipeline shall be as self-contained as possible. More specifically, this means that all functionality should be either directly defined in the pipeline file itself, should be inherited from (and only from) the [`DiffusionPipeline` class](https://github.com/huggingface/diffusers/blob/5cbed8e0d157f65d3ddc2420dfd09f2df630e978/src/diffusers/pipeline_utils.py#L56) or be directly attached to the model and scheduler components of the pipeline. +- **Easy-to-use**: Pipelines should be extremely easy to use - one should be able to load the pipeline and +use it for its designated task, *e.g.* text-to-image generation, in just a couple of lines of code. Most +logic including pre-processing, an unrolled diffusion loop, and post-processing should all happen inside the `__call__` method. +- **Easy-to-tweak**: Certain pipelines will not be able to handle all use cases and tasks that you might like them to. If you want to use a certain pipeline for a specific use case that is not yet supported, you might have to copy the pipeline file and tweak the code to your needs. We try to make the pipeline code as readable as possible so that each part –from pre-processing to diffusing to post-processing– can easily be adapted. If you would like the community to benefit from your customized pipeline, we would love to see a contribution to our [community-examples](https://github.com/huggingface/diffusers/tree/main/examples/community). If you feel that an important pipeline should be part of the official pipelines but isn't, a contribution to the [official pipelines](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines) would be even better. +- **One-purpose-only**: Pipelines should be used for one task and one task only. Even if two tasks are very similar from a modeling point of view, *e.g.* image2image translation and in-painting, pipelines shall be used for one task only to keep them *easy-to-tweak* and *readable*. + +## Examples + +### Text-to-Image generation with Stable Diffusion + +```python +# make sure you're logged in with `huggingface-cli login` +from diffusers import StableDiffusionPipeline, LMSDiscreteScheduler + +pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") +pipe = pipe.to("cuda") + +prompt = "a photo of an astronaut riding a horse on mars" +image = pipe(prompt).images[0] + +image.save("astronaut_rides_horse.png") +``` + +### Image-to-Image text-guided generation with Stable Diffusion + +The `StableDiffusionImg2ImgPipeline` lets you pass a text prompt and an initial image to condition the generation of new images. + +```python +import requests +from PIL import Image +from io import BytesIO + +from diffusers import StableDiffusionImg2ImgPipeline + +# load the pipeline +device = "cuda" +pipe = StableDiffusionImg2ImgPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + torch_dtype=torch.float16, +).to(device) + +# let's download an initial image +url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + +response = requests.get(url) +init_image = Image.open(BytesIO(response.content)).convert("RGB") +init_image = init_image.resize((768, 512)) + +prompt = "A fantasy landscape, trending on artstation" + +images = pipe(prompt=prompt, image=init_image, strength=0.75, guidance_scale=7.5).images + +images[0].save("fantasy_landscape.png") +``` +You can also run this example on colab [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/image_2_image_using_diffusers.ipynb) + +### Tweak prompts reusing seeds and latents + +You can generate your own latents to reproduce results, or tweak your prompt on a specific result you liked. [This notebook](https://github.com/pcuenca/diffusers-examples/blob/main/notebooks/stable-diffusion-seeds.ipynb) shows how to do it step by step. You can also run it in Google Colab [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/pcuenca/diffusers-examples/blob/main/notebooks/stable-diffusion-seeds.ipynb). + + +### In-painting using Stable Diffusion + +The `StableDiffusionInpaintPipeline` lets you edit specific parts of an image by providing a mask and text prompt. + +```python +import PIL +import requests +import torch +from io import BytesIO + +from diffusers import StableDiffusionInpaintPipeline + +def download_image(url): + response = requests.get(url) + return PIL.Image.open(BytesIO(response.content)).convert("RGB") + +img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png" +mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png" + +init_image = download_image(img_url).resize((512, 512)) +mask_image = download_image(mask_url).resize((512, 512)) + +pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", + torch_dtype=torch.float16, +) +pipe = pipe.to("cuda") + +prompt = "Face of a yellow cat, high resolution, sitting on a park bench" +image = pipe(prompt=prompt, image=init_image, mask_image=mask_image).images[0] +``` + +You can also run this example on colab [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/in_painting_with_stable_diffusion_using_diffusers.ipynb) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/__init__.py new file mode 100755 index 0000000..2b22778 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/__init__.py @@ -0,0 +1,581 @@ +from typing import TYPE_CHECKING + +from ..utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_flax_available, + is_k_diffusion_available, + is_librosa_available, + is_note_seq_available, + is_onnx_available, + is_torch_available, + is_torch_npu_available, + is_transformers_available, +) + + +# These modules contain pipelines from multiple libraries/frameworks +_dummy_objects = {} +_import_structure = { + "controlnet": [], + "controlnet_xs": [], + "deprecated": [], + "latent_diffusion": [], + "ledits_pp": [], + "stable_diffusion": [], + "stable_diffusion_xl": [], +} + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_pt_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_pt_objects)) +else: + _import_structure["auto_pipeline"] = [ + "AutoPipelineForImage2Image", + "AutoPipelineForInpainting", + "AutoPipelineForText2Image", + ] + _import_structure["consistency_models"] = ["ConsistencyModelPipeline"] + _import_structure["dance_diffusion"] = ["DanceDiffusionPipeline"] + _import_structure["ddim"] = ["DDIMPipeline"] + _import_structure["ddpm"] = ["DDPMPipeline"] + _import_structure["dit"] = ["DiTPipeline"] + _import_structure["latent_diffusion"].extend(["LDMSuperResolutionPipeline"]) + _import_structure["pipeline_utils"] = [ + "AudioPipelineOutput", + "DiffusionPipeline", + "StableDiffusionMixin", + "ImagePipelineOutput", + ] + _import_structure["deprecated"].extend( + [ + "PNDMPipeline", + "LDMPipeline", + "RePaintPipeline", + "ScoreSdeVePipeline", + "KarrasVePipeline", + ] + ) +try: + if not (is_torch_available() and is_librosa_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_torch_and_librosa_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_librosa_objects)) +else: + _import_structure["deprecated"].extend(["AudioDiffusionPipeline", "Mel"]) + +try: + if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_transformers_and_torch_and_note_seq_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_transformers_and_torch_and_note_seq_objects)) +else: + _import_structure["deprecated"].extend( + [ + "MidiProcessor", + "SpectrogramDiffusionPipeline", + ] + ) + +try: + if not (is_torch_available() and is_transformers_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["deprecated"].extend( + [ + "VQDiffusionPipeline", + "AltDiffusionPipeline", + "AltDiffusionImg2ImgPipeline", + "CycleDiffusionPipeline", + "StableDiffusionInpaintPipelineLegacy", + "StableDiffusionPix2PixZeroPipeline", + "StableDiffusionParadigmsPipeline", + "StableDiffusionModelEditingPipeline", + "VersatileDiffusionDualGuidedPipeline", + "VersatileDiffusionImageVariationPipeline", + "VersatileDiffusionPipeline", + "VersatileDiffusionTextToImagePipeline", + ] + ) + _import_structure["amused"] = ["AmusedImg2ImgPipeline", "AmusedInpaintPipeline", "AmusedPipeline"] + _import_structure["animatediff"] = [ + "AnimateDiffPipeline", + "AnimateDiffVideoToVideoPipeline", + ] + _import_structure["audioldm"] = ["AudioLDMPipeline"] + _import_structure["audioldm2"] = [ + "AudioLDM2Pipeline", + "AudioLDM2ProjectionModel", + "AudioLDM2UNet2DConditionModel", + ] + _import_structure["blip_diffusion"] = ["BlipDiffusionPipeline"] + _import_structure["controlnet"].extend( + [ + "BlipDiffusionControlNetPipeline", + "StableDiffusionControlNetImg2ImgPipeline", + "StableDiffusionControlNetInpaintPipeline", + "StableDiffusionControlNetPipeline", + "StableDiffusionXLControlNetImg2ImgPipeline", + "StableDiffusionXLControlNetInpaintPipeline", + "StableDiffusionXLControlNetPipeline", + ] + ) + _import_structure["deepfloyd_if"] = [ + "IFImg2ImgPipeline", + "IFImg2ImgSuperResolutionPipeline", + "IFInpaintingPipeline", + "IFInpaintingSuperResolutionPipeline", + "IFPipeline", + "IFSuperResolutionPipeline", + ] + _import_structure["kandinsky"] = [ + "KandinskyCombinedPipeline", + "KandinskyImg2ImgCombinedPipeline", + "KandinskyImg2ImgPipeline", + "KandinskyInpaintCombinedPipeline", + "KandinskyInpaintPipeline", + "KandinskyPipeline", + "KandinskyPriorPipeline", + ] + _import_structure["kandinsky2_2"] = [ + "KandinskyV22CombinedPipeline", + "KandinskyV22ControlnetImg2ImgPipeline", + "KandinskyV22ControlnetPipeline", + "KandinskyV22Img2ImgCombinedPipeline", + "KandinskyV22Img2ImgPipeline", + "KandinskyV22InpaintCombinedPipeline", + "KandinskyV22InpaintPipeline", + "KandinskyV22Pipeline", + "KandinskyV22PriorEmb2EmbPipeline", + "KandinskyV22PriorPipeline", + ] + _import_structure["kandinsky3"] = [ + "Kandinsky3Img2ImgPipeline", + "Kandinsky3Pipeline", + ] + _import_structure["latent_consistency_models"] = [ + "LatentConsistencyModelImg2ImgPipeline", + "LatentConsistencyModelPipeline", + ] + _import_structure["latent_diffusion"].extend(["LDMTextToImagePipeline"]) + _import_structure["ledits_pp"].extend( + [ + "LEditsPPPipelineStableDiffusion", + "LEditsPPPipelineStableDiffusionXL", + ] + ) + _import_structure["musicldm"] = ["MusicLDMPipeline"] + _import_structure["paint_by_example"] = ["PaintByExamplePipeline"] + _import_structure["pia"] = ["PIAPipeline"] + _import_structure["pixart_alpha"] = ["PixArtAlphaPipeline"] + _import_structure["semantic_stable_diffusion"] = ["SemanticStableDiffusionPipeline"] + _import_structure["shap_e"] = ["ShapEImg2ImgPipeline", "ShapEPipeline"] + _import_structure["stable_cascade"] = [ + "StableCascadeCombinedPipeline", + "StableCascadeDecoderPipeline", + "StableCascadePriorPipeline", + ] + _import_structure["stable_diffusion"].extend( + [ + "CLIPImageProjection", + "StableDiffusionDepth2ImgPipeline", + "StableDiffusionImageVariationPipeline", + "StableDiffusionImg2ImgPipeline", + "StableDiffusionInpaintPipeline", + "StableDiffusionInstructPix2PixPipeline", + "StableDiffusionLatentUpscalePipeline", + "StableDiffusionPipeline", + "StableDiffusionUpscalePipeline", + "StableUnCLIPImg2ImgPipeline", + "StableUnCLIPPipeline", + "StableDiffusionLDM3DPipeline", + ] + ) + _import_structure["stable_diffusion_attend_and_excite"] = ["StableDiffusionAttendAndExcitePipeline"] + _import_structure["stable_diffusion_safe"] = ["StableDiffusionPipelineSafe"] + _import_structure["stable_diffusion_sag"] = ["StableDiffusionSAGPipeline"] + _import_structure["stable_diffusion_gligen"] = [ + "StableDiffusionGLIGENPipeline", + "StableDiffusionGLIGENTextImagePipeline", + ] + _import_structure["stable_video_diffusion"] = ["StableVideoDiffusionPipeline"] + _import_structure["stable_diffusion_xl"].extend( + [ + "StableDiffusionXLImg2ImgPipeline", + "StableDiffusionXLInpaintPipeline", + "StableDiffusionXLInstructPix2PixPipeline", + "StableDiffusionXLPipeline", + ] + ) + _import_structure["stable_diffusion_diffedit"] = ["StableDiffusionDiffEditPipeline"] + _import_structure["stable_diffusion_ldm3d"] = ["StableDiffusionLDM3DPipeline"] + _import_structure["stable_diffusion_panorama"] = ["StableDiffusionPanoramaPipeline"] + _import_structure["t2i_adapter"] = [ + "StableDiffusionAdapterPipeline", + "StableDiffusionXLAdapterPipeline", + ] + _import_structure["text_to_video_synthesis"] = [ + "TextToVideoSDPipeline", + "TextToVideoZeroPipeline", + "TextToVideoZeroSDXLPipeline", + "VideoToVideoSDPipeline", + ] + _import_structure["i2vgen_xl"] = ["I2VGenXLPipeline"] + _import_structure["unclip"] = ["UnCLIPImageVariationPipeline", "UnCLIPPipeline"] + _import_structure["unidiffuser"] = [ + "ImageTextPipelineOutput", + "UniDiffuserModel", + "UniDiffuserPipeline", + "UniDiffuserTextDecoder", + ] + _import_structure["wuerstchen"] = [ + "WuerstchenCombinedPipeline", + "WuerstchenDecoderPipeline", + "WuerstchenPriorPipeline", + ] +try: + if not is_onnx_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_onnx_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_onnx_objects)) +else: + _import_structure["onnx_utils"] = ["OnnxRuntimeModel"] +try: + if not (is_torch_available() and is_transformers_available() and is_onnx_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_torch_and_transformers_and_onnx_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_and_onnx_objects)) +else: + _import_structure["stable_diffusion"].extend( + [ + "OnnxStableDiffusionImg2ImgPipeline", + "OnnxStableDiffusionInpaintPipeline", + "OnnxStableDiffusionPipeline", + "OnnxStableDiffusionUpscalePipeline", + "StableDiffusionOnnxPipeline", + ] + ) + +try: + if not (is_torch_available() and is_transformers_available() and is_k_diffusion_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import ( + dummy_torch_and_transformers_and_k_diffusion_objects, + ) + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_and_k_diffusion_objects)) +else: + _import_structure["stable_diffusion_k_diffusion"] = [ + "StableDiffusionKDiffusionPipeline", + "StableDiffusionXLKDiffusionPipeline", + ] +try: + if not is_flax_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_flax_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_flax_objects)) +else: + _import_structure["pipeline_flax_utils"] = ["FlaxDiffusionPipeline"] +try: + if not (is_flax_available() and is_transformers_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_flax_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_flax_and_transformers_objects)) +else: + _import_structure["controlnet"].extend(["FlaxStableDiffusionControlNetPipeline"]) + _import_structure["stable_diffusion"].extend( + [ + "FlaxStableDiffusionImg2ImgPipeline", + "FlaxStableDiffusionInpaintPipeline", + "FlaxStableDiffusionPipeline", + ] + ) + _import_structure["stable_diffusion_xl"].extend( + [ + "FlaxStableDiffusionXLPipeline", + ] + ) + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_pt_objects import * # noqa F403 + + else: + from .auto_pipeline import ( + AutoPipelineForImage2Image, + AutoPipelineForInpainting, + AutoPipelineForText2Image, + ) + from .consistency_models import ConsistencyModelPipeline + from .dance_diffusion import DanceDiffusionPipeline + from .ddim import DDIMPipeline + from .ddpm import DDPMPipeline + from .deprecated import KarrasVePipeline, LDMPipeline, PNDMPipeline, RePaintPipeline, ScoreSdeVePipeline + from .dit import DiTPipeline + from .latent_diffusion import LDMSuperResolutionPipeline + from .pipeline_utils import ( + AudioPipelineOutput, + DiffusionPipeline, + ImagePipelineOutput, + StableDiffusionMixin, + ) + + try: + if not (is_torch_available() and is_librosa_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_torch_and_librosa_objects import * + else: + from .deprecated import AudioDiffusionPipeline, Mel + + try: + if not (is_torch_available() and is_transformers_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_torch_and_transformers_objects import * + else: + from .amused import AmusedImg2ImgPipeline, AmusedInpaintPipeline, AmusedPipeline + from .animatediff import AnimateDiffPipeline, AnimateDiffVideoToVideoPipeline + from .audioldm import AudioLDMPipeline + from .audioldm2 import ( + AudioLDM2Pipeline, + AudioLDM2ProjectionModel, + AudioLDM2UNet2DConditionModel, + ) + from .blip_diffusion import BlipDiffusionPipeline + from .controlnet import ( + BlipDiffusionControlNetPipeline, + StableDiffusionControlNetImg2ImgPipeline, + StableDiffusionControlNetInpaintPipeline, + StableDiffusionControlNetPipeline, + StableDiffusionXLControlNetImg2ImgPipeline, + StableDiffusionXLControlNetInpaintPipeline, + StableDiffusionXLControlNetPipeline, + ) + from .deepfloyd_if import ( + IFImg2ImgPipeline, + IFImg2ImgSuperResolutionPipeline, + IFInpaintingPipeline, + IFInpaintingSuperResolutionPipeline, + IFPipeline, + IFSuperResolutionPipeline, + ) + from .deprecated import ( + AltDiffusionImg2ImgPipeline, + AltDiffusionPipeline, + CycleDiffusionPipeline, + StableDiffusionInpaintPipelineLegacy, + StableDiffusionModelEditingPipeline, + StableDiffusionParadigmsPipeline, + StableDiffusionPix2PixZeroPipeline, + VersatileDiffusionDualGuidedPipeline, + VersatileDiffusionImageVariationPipeline, + VersatileDiffusionPipeline, + VersatileDiffusionTextToImagePipeline, + VQDiffusionPipeline, + ) + from .i2vgen_xl import I2VGenXLPipeline + from .kandinsky import ( + KandinskyCombinedPipeline, + KandinskyImg2ImgCombinedPipeline, + KandinskyImg2ImgPipeline, + KandinskyInpaintCombinedPipeline, + KandinskyInpaintPipeline, + KandinskyPipeline, + KandinskyPriorPipeline, + ) + from .kandinsky2_2 import ( + KandinskyV22CombinedPipeline, + KandinskyV22ControlnetImg2ImgPipeline, + KandinskyV22ControlnetPipeline, + KandinskyV22Img2ImgCombinedPipeline, + KandinskyV22Img2ImgPipeline, + KandinskyV22InpaintCombinedPipeline, + KandinskyV22InpaintPipeline, + KandinskyV22Pipeline, + KandinskyV22PriorEmb2EmbPipeline, + KandinskyV22PriorPipeline, + ) + from .kandinsky3 import ( + Kandinsky3Img2ImgPipeline, + Kandinsky3Pipeline, + ) + from .latent_consistency_models import ( + LatentConsistencyModelImg2ImgPipeline, + LatentConsistencyModelPipeline, + ) + from .latent_diffusion import LDMTextToImagePipeline + from .ledits_pp import ( + LEditsPPDiffusionPipelineOutput, + LEditsPPInversionPipelineOutput, + LEditsPPPipelineStableDiffusion, + LEditsPPPipelineStableDiffusionXL, + ) + from .musicldm import MusicLDMPipeline + from .paint_by_example import PaintByExamplePipeline + from .pia import PIAPipeline + from .pixart_alpha import PixArtAlphaPipeline + from .semantic_stable_diffusion import SemanticStableDiffusionPipeline + from .shap_e import ShapEImg2ImgPipeline, ShapEPipeline + from .stable_cascade import ( + StableCascadeCombinedPipeline, + StableCascadeDecoderPipeline, + StableCascadePriorPipeline, + ) + from .stable_diffusion import ( + CLIPImageProjection, + StableDiffusionDepth2ImgPipeline, + StableDiffusionImageVariationPipeline, + StableDiffusionImg2ImgPipeline, + StableDiffusionInpaintPipeline, + StableDiffusionInstructPix2PixPipeline, + StableDiffusionLatentUpscalePipeline, + StableDiffusionPipeline, + StableDiffusionUpscalePipeline, + StableUnCLIPImg2ImgPipeline, + StableUnCLIPPipeline, + ) + from .stable_diffusion_attend_and_excite import StableDiffusionAttendAndExcitePipeline + from .stable_diffusion_diffedit import StableDiffusionDiffEditPipeline + from .stable_diffusion_gligen import StableDiffusionGLIGENPipeline, StableDiffusionGLIGENTextImagePipeline + from .stable_diffusion_ldm3d import StableDiffusionLDM3DPipeline + from .stable_diffusion_panorama import StableDiffusionPanoramaPipeline + from .stable_diffusion_safe import StableDiffusionPipelineSafe + from .stable_diffusion_sag import StableDiffusionSAGPipeline + from .stable_diffusion_xl import ( + StableDiffusionXLImg2ImgPipeline, + StableDiffusionXLInpaintPipeline, + StableDiffusionXLInstructPix2PixPipeline, + StableDiffusionXLPipeline, + ) + from .stable_video_diffusion import StableVideoDiffusionPipeline + from .t2i_adapter import ( + StableDiffusionAdapterPipeline, + StableDiffusionXLAdapterPipeline, + ) + from .text_to_video_synthesis import ( + TextToVideoSDPipeline, + TextToVideoZeroPipeline, + TextToVideoZeroSDXLPipeline, + VideoToVideoSDPipeline, + ) + from .unclip import UnCLIPImageVariationPipeline, UnCLIPPipeline + from .unidiffuser import ( + ImageTextPipelineOutput, + UniDiffuserModel, + UniDiffuserPipeline, + UniDiffuserTextDecoder, + ) + from .wuerstchen import ( + WuerstchenCombinedPipeline, + WuerstchenDecoderPipeline, + WuerstchenPriorPipeline, + ) + + try: + if not is_onnx_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_onnx_objects import * # noqa F403 + + else: + from .onnx_utils import OnnxRuntimeModel + + try: + if not (is_torch_available() and is_transformers_available() and is_onnx_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_torch_and_transformers_and_onnx_objects import * + else: + from .stable_diffusion import ( + OnnxStableDiffusionImg2ImgPipeline, + OnnxStableDiffusionInpaintPipeline, + OnnxStableDiffusionPipeline, + OnnxStableDiffusionUpscalePipeline, + StableDiffusionOnnxPipeline, + ) + + try: + if not (is_torch_available() and is_transformers_available() and is_k_diffusion_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_torch_and_transformers_and_k_diffusion_objects import * + else: + from .stable_diffusion_k_diffusion import ( + StableDiffusionKDiffusionPipeline, + StableDiffusionXLKDiffusionPipeline, + ) + + try: + if not is_flax_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_flax_objects import * # noqa F403 + else: + from .pipeline_flax_utils import FlaxDiffusionPipeline + + try: + if not (is_flax_available() and is_transformers_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_flax_and_transformers_objects import * + else: + from .controlnet import FlaxStableDiffusionControlNetPipeline + from .stable_diffusion import ( + FlaxStableDiffusionImg2ImgPipeline, + FlaxStableDiffusionInpaintPipeline, + FlaxStableDiffusionPipeline, + ) + from .stable_diffusion_xl import ( + FlaxStableDiffusionXLPipeline, + ) + + try: + if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_transformers_and_torch_and_note_seq_objects import * # noqa F403 + + else: + from .deprecated import ( + MidiProcessor, + SpectrogramDiffusionPipeline, + ) + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/amused/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/amused/__init__.py new file mode 100755 index 0000000..3c4d07a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/amused/__init__.py @@ -0,0 +1,62 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import ( + AmusedImg2ImgPipeline, + AmusedInpaintPipeline, + AmusedPipeline, + ) + + _dummy_objects.update( + { + "AmusedPipeline": AmusedPipeline, + "AmusedImg2ImgPipeline": AmusedImg2ImgPipeline, + "AmusedInpaintPipeline": AmusedInpaintPipeline, + } + ) +else: + _import_structure["pipeline_amused"] = ["AmusedPipeline"] + _import_structure["pipeline_amused_img2img"] = ["AmusedImg2ImgPipeline"] + _import_structure["pipeline_amused_inpaint"] = ["AmusedInpaintPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import ( + AmusedPipeline, + ) + else: + from .pipeline_amused import AmusedPipeline + from .pipeline_amused_img2img import AmusedImg2ImgPipeline + from .pipeline_amused_inpaint import AmusedInpaintPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused.py b/diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused.py new file mode 100755 index 0000000..aa682b4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused.py @@ -0,0 +1,328 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from transformers import CLIPTextModelWithProjection, CLIPTokenizer + +from ...image_processor import VaeImageProcessor +from ...models import UVit2DModel, VQModel +from ...schedulers import AmusedScheduler +from ...utils import replace_example_docstring +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import AmusedPipeline + + >>> pipe = AmusedPipeline.from_pretrained( + ... "amused/amused-512", variant="fp16", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" + + +class AmusedPipeline(DiffusionPipeline): + image_processor: VaeImageProcessor + vqvae: VQModel + tokenizer: CLIPTokenizer + text_encoder: CLIPTextModelWithProjection + transformer: UVit2DModel + scheduler: AmusedScheduler + + model_cpu_offload_seq = "text_encoder->transformer->vqvae" + + def __init__( + self, + vqvae: VQModel, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModelWithProjection, + transformer: UVit2DModel, + scheduler: AmusedScheduler, + ): + super().__init__() + + self.register_modules( + vqvae=vqvae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vqvae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_normalize=False) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[List[str], str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 12, + guidance_scale: float = 10.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[torch.Generator] = None, + latents: Optional[torch.IntTensor] = None, + prompt_embeds: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_encoder_hidden_states: Optional[torch.Tensor] = None, + output_type="pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + micro_conditioning_aesthetic_score: int = 6, + micro_conditioning_crop_coord: Tuple[int, int] = (0, 0), + temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), + ): + """ + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.transformer.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 16): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 10.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.IntTensor`, *optional*): + Pre-generated tokens representing latent vectors in `self.vqvae`, to be used as inputs for image + gneration. If not provided, the starting latents will be completely masked. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. A single vector from the + pooled and projected final hidden states. + encoder_hidden_states (`torch.FloatTensor`, *optional*): + Pre-generated penultimate hidden states from the text encoder providing additional text conditioning. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + negative_encoder_hidden_states (`torch.FloatTensor`, *optional*): + Analogous to `encoder_hidden_states` for the positive prompt. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + micro_conditioning_aesthetic_score (`int`, *optional*, defaults to 6): + The targeted aesthetic score according to the laion aesthetic classifier. See https://laion.ai/blog/laion-aesthetics/ + and the micro-conditioning section of https://arxiv.org/abs/2307.01952. + micro_conditioning_crop_coord (`Tuple[int]`, *optional*, defaults to (0, 0)): + The targeted height, width crop coordinates. See the micro-conditioning section of https://arxiv.org/abs/2307.01952. + temperature (`Union[int, Tuple[int, int], List[int]]`, *optional*, defaults to (2, 0)): + Configures the temperature scheduler on `self.scheduler` see `AmusedScheduler#set_timesteps`. + + Examples: + + Returns: + [`~pipelines.pipeline_utils.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.pipeline_utils.ImagePipelineOutput`] is returned, otherwise a + `tuple` is returned where the first element is a list with the generated images. + """ + if (prompt_embeds is not None and encoder_hidden_states is None) or ( + prompt_embeds is None and encoder_hidden_states is not None + ): + raise ValueError("pass either both `prompt_embeds` and `encoder_hidden_states` or neither") + + if (negative_prompt_embeds is not None and negative_encoder_hidden_states is None) or ( + negative_prompt_embeds is None and negative_encoder_hidden_states is not None + ): + raise ValueError( + "pass either both `negatve_prompt_embeds` and `negative_encoder_hidden_states` or neither" + ) + + if (prompt is None and prompt_embeds is None) or (prompt is not None and prompt_embeds is not None): + raise ValueError("pass only one of `prompt` or `prompt_embeds`") + + if isinstance(prompt, str): + prompt = [prompt] + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + batch_size = batch_size * num_images_per_prompt + + if height is None: + height = self.transformer.config.sample_size * self.vae_scale_factor + + if width is None: + width = self.transformer.config.sample_size * self.vae_scale_factor + + if prompt_embeds is None: + input_ids = self.tokenizer( + prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + prompt_embeds = outputs.text_embeds + encoder_hidden_states = outputs.hidden_states[-2] + + prompt_embeds = prompt_embeds.repeat(num_images_per_prompt, 1) + encoder_hidden_states = encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + if guidance_scale > 1.0: + if negative_prompt_embeds is None: + if negative_prompt is None: + negative_prompt = [""] * len(prompt) + + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + + input_ids = self.tokenizer( + negative_prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + negative_prompt_embeds = outputs.text_embeds + negative_encoder_hidden_states = outputs.hidden_states[-2] + + negative_prompt_embeds = negative_prompt_embeds.repeat(num_images_per_prompt, 1) + negative_encoder_hidden_states = negative_encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + prompt_embeds = torch.concat([negative_prompt_embeds, prompt_embeds]) + encoder_hidden_states = torch.concat([negative_encoder_hidden_states, encoder_hidden_states]) + + # Note that the micro conditionings _do_ flip the order of width, height for the original size + # and the crop coordinates. This is how it was done in the original code base + micro_conds = torch.tensor( + [ + width, + height, + micro_conditioning_crop_coord[0], + micro_conditioning_crop_coord[1], + micro_conditioning_aesthetic_score, + ], + device=self._execution_device, + dtype=encoder_hidden_states.dtype, + ) + micro_conds = micro_conds.unsqueeze(0) + micro_conds = micro_conds.expand(2 * batch_size if guidance_scale > 1.0 else batch_size, -1) + + shape = (batch_size, height // self.vae_scale_factor, width // self.vae_scale_factor) + + if latents is None: + latents = torch.full( + shape, self.scheduler.config.mask_token_id, dtype=torch.long, device=self._execution_device + ) + + self.scheduler.set_timesteps(num_inference_steps, temperature, self._execution_device) + + num_warmup_steps = len(self.scheduler.timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, timestep in enumerate(self.scheduler.timesteps): + if guidance_scale > 1.0: + model_input = torch.cat([latents] * 2) + else: + model_input = latents + + model_output = self.transformer( + model_input, + micro_conds=micro_conds, + pooled_text_emb=prompt_embeds, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + ) + + if guidance_scale > 1.0: + uncond_logits, cond_logits = model_output.chunk(2) + model_output = uncond_logits + guidance_scale * (cond_logits - uncond_logits) + + latents = self.scheduler.step( + model_output=model_output, + timestep=timestep, + sample=latents, + generator=generator, + ).prev_sample + + if i == len(self.scheduler.timesteps) - 1 or ( + (i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0 + ): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, timestep, latents) + + if output_type == "latent": + output = latents + else: + needs_upcasting = self.vqvae.dtype == torch.float16 and self.vqvae.config.force_upcast + + if needs_upcasting: + self.vqvae.float() + + output = self.vqvae.decode( + latents, + force_not_quantize=True, + shape=( + batch_size, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + self.vqvae.config.latent_channels, + ), + ).sample.clip(0, 1) + output = self.image_processor.postprocess(output, output_type) + + if needs_upcasting: + self.vqvae.half() + + self.maybe_free_model_hooks() + + if not return_dict: + return (output,) + + return ImagePipelineOutput(output) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused_img2img.py new file mode 100755 index 0000000..444d635 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused_img2img.py @@ -0,0 +1,347 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from transformers import CLIPTextModelWithProjection, CLIPTokenizer + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...models import UVit2DModel, VQModel +from ...schedulers import AmusedScheduler +from ...utils import replace_example_docstring +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import AmusedImg2ImgPipeline + >>> from diffusers.utils import load_image + + >>> pipe = AmusedImg2ImgPipeline.from_pretrained( + ... "amused/amused-512", variant="fp16", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "winter mountains" + >>> input_image = ( + ... load_image( + ... "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains.jpg" + ... ) + ... .resize((512, 512)) + ... .convert("RGB") + ... ) + >>> image = pipe(prompt, input_image).images[0] + ``` +""" + + +class AmusedImg2ImgPipeline(DiffusionPipeline): + image_processor: VaeImageProcessor + vqvae: VQModel + tokenizer: CLIPTokenizer + text_encoder: CLIPTextModelWithProjection + transformer: UVit2DModel + scheduler: AmusedScheduler + + model_cpu_offload_seq = "text_encoder->transformer->vqvae" + + # TODO - when calling self.vqvae.quantize, it uses self.vqvae.quantize.embedding.weight before + # the forward method of self.vqvae.quantize, so the hook doesn't get called to move the parameter + # off the meta device. There should be a way to fix this instead of just not offloading it + _exclude_from_cpu_offload = ["vqvae"] + + def __init__( + self, + vqvae: VQModel, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModelWithProjection, + transformer: UVit2DModel, + scheduler: AmusedScheduler, + ): + super().__init__() + + self.register_modules( + vqvae=vqvae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vqvae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_normalize=False) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[List[str], str]] = None, + image: PipelineImageInput = None, + strength: float = 0.5, + num_inference_steps: int = 12, + guidance_scale: float = 10.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[torch.Generator] = None, + prompt_embeds: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_encoder_hidden_states: Optional[torch.Tensor] = None, + output_type="pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + micro_conditioning_aesthetic_score: int = 6, + micro_conditioning_crop_coord: Tuple[int, int] = (0, 0), + temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), + ): + """ + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + strength (`float`, *optional*, defaults to 0.5): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 16): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 10.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. A single vector from the + pooled and projected final hidden states. + encoder_hidden_states (`torch.FloatTensor`, *optional*): + Pre-generated penultimate hidden states from the text encoder providing additional text conditioning. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + negative_encoder_hidden_states (`torch.FloatTensor`, *optional*): + Analogous to `encoder_hidden_states` for the positive prompt. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + micro_conditioning_aesthetic_score (`int`, *optional*, defaults to 6): + The targeted aesthetic score according to the laion aesthetic classifier. See https://laion.ai/blog/laion-aesthetics/ + and the micro-conditioning section of https://arxiv.org/abs/2307.01952. + micro_conditioning_crop_coord (`Tuple[int]`, *optional*, defaults to (0, 0)): + The targeted height, width crop coordinates. See the micro-conditioning section of https://arxiv.org/abs/2307.01952. + temperature (`Union[int, Tuple[int, int], List[int]]`, *optional*, defaults to (2, 0)): + Configures the temperature scheduler on `self.scheduler` see `AmusedScheduler#set_timesteps`. + + Examples: + + Returns: + [`~pipelines.pipeline_utils.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.pipeline_utils.ImagePipelineOutput`] is returned, otherwise a + `tuple` is returned where the first element is a list with the generated images. + """ + + if (prompt_embeds is not None and encoder_hidden_states is None) or ( + prompt_embeds is None and encoder_hidden_states is not None + ): + raise ValueError("pass either both `prompt_embeds` and `encoder_hidden_states` or neither") + + if (negative_prompt_embeds is not None and negative_encoder_hidden_states is None) or ( + negative_prompt_embeds is None and negative_encoder_hidden_states is not None + ): + raise ValueError( + "pass either both `negatve_prompt_embeds` and `negative_encoder_hidden_states` or neither" + ) + + if (prompt is None and prompt_embeds is None) or (prompt is not None and prompt_embeds is not None): + raise ValueError("pass only one of `prompt` or `prompt_embeds`") + + if isinstance(prompt, str): + prompt = [prompt] + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + batch_size = batch_size * num_images_per_prompt + + if prompt_embeds is None: + input_ids = self.tokenizer( + prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + prompt_embeds = outputs.text_embeds + encoder_hidden_states = outputs.hidden_states[-2] + + prompt_embeds = prompt_embeds.repeat(num_images_per_prompt, 1) + encoder_hidden_states = encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + if guidance_scale > 1.0: + if negative_prompt_embeds is None: + if negative_prompt is None: + negative_prompt = [""] * len(prompt) + + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + + input_ids = self.tokenizer( + negative_prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + negative_prompt_embeds = outputs.text_embeds + negative_encoder_hidden_states = outputs.hidden_states[-2] + + negative_prompt_embeds = negative_prompt_embeds.repeat(num_images_per_prompt, 1) + negative_encoder_hidden_states = negative_encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + prompt_embeds = torch.concat([negative_prompt_embeds, prompt_embeds]) + encoder_hidden_states = torch.concat([negative_encoder_hidden_states, encoder_hidden_states]) + + image = self.image_processor.preprocess(image) + + height, width = image.shape[-2:] + + # Note that the micro conditionings _do_ flip the order of width, height for the original size + # and the crop coordinates. This is how it was done in the original code base + micro_conds = torch.tensor( + [ + width, + height, + micro_conditioning_crop_coord[0], + micro_conditioning_crop_coord[1], + micro_conditioning_aesthetic_score, + ], + device=self._execution_device, + dtype=encoder_hidden_states.dtype, + ) + + micro_conds = micro_conds.unsqueeze(0) + micro_conds = micro_conds.expand(2 * batch_size if guidance_scale > 1.0 else batch_size, -1) + + self.scheduler.set_timesteps(num_inference_steps, temperature, self._execution_device) + num_inference_steps = int(len(self.scheduler.timesteps) * strength) + start_timestep_idx = len(self.scheduler.timesteps) - num_inference_steps + + needs_upcasting = self.vqvae.dtype == torch.float16 and self.vqvae.config.force_upcast + + if needs_upcasting: + self.vqvae.float() + + latents = self.vqvae.encode(image.to(dtype=self.vqvae.dtype, device=self._execution_device)).latents + latents_bsz, channels, latents_height, latents_width = latents.shape + latents = self.vqvae.quantize(latents)[2][2].reshape(latents_bsz, latents_height, latents_width) + latents = self.scheduler.add_noise( + latents, self.scheduler.timesteps[start_timestep_idx - 1], generator=generator + ) + latents = latents.repeat(num_images_per_prompt, 1, 1) + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i in range(start_timestep_idx, len(self.scheduler.timesteps)): + timestep = self.scheduler.timesteps[i] + + if guidance_scale > 1.0: + model_input = torch.cat([latents] * 2) + else: + model_input = latents + + model_output = self.transformer( + model_input, + micro_conds=micro_conds, + pooled_text_emb=prompt_embeds, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + ) + + if guidance_scale > 1.0: + uncond_logits, cond_logits = model_output.chunk(2) + model_output = uncond_logits + guidance_scale * (cond_logits - uncond_logits) + + latents = self.scheduler.step( + model_output=model_output, + timestep=timestep, + sample=latents, + generator=generator, + ).prev_sample + + if i == len(self.scheduler.timesteps) - 1 or ((i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, timestep, latents) + + if output_type == "latent": + output = latents + else: + output = self.vqvae.decode( + latents, + force_not_quantize=True, + shape=( + batch_size, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + self.vqvae.config.latent_channels, + ), + ).sample.clip(0, 1) + output = self.image_processor.postprocess(output, output_type) + + if needs_upcasting: + self.vqvae.half() + + self.maybe_free_model_hooks() + + if not return_dict: + return (output,) + + return ImagePipelineOutput(output) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused_inpaint.py b/diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused_inpaint.py new file mode 100755 index 0000000..423f573 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/amused/pipeline_amused_inpaint.py @@ -0,0 +1,378 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from transformers import CLIPTextModelWithProjection, CLIPTokenizer + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...models import UVit2DModel, VQModel +from ...schedulers import AmusedScheduler +from ...utils import replace_example_docstring +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import AmusedInpaintPipeline + >>> from diffusers.utils import load_image + + >>> pipe = AmusedInpaintPipeline.from_pretrained( + ... "amused/amused-512", variant="fp16", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "fall mountains" + >>> input_image = ( + ... load_image( + ... "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains_1.jpg" + ... ) + ... .resize((512, 512)) + ... .convert("RGB") + ... ) + >>> mask = ( + ... load_image( + ... "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains_1_mask.png" + ... ) + ... .resize((512, 512)) + ... .convert("L") + ... ) + >>> pipe(prompt, input_image, mask).images[0].save("out.png") + ``` +""" + + +class AmusedInpaintPipeline(DiffusionPipeline): + image_processor: VaeImageProcessor + vqvae: VQModel + tokenizer: CLIPTokenizer + text_encoder: CLIPTextModelWithProjection + transformer: UVit2DModel + scheduler: AmusedScheduler + + model_cpu_offload_seq = "text_encoder->transformer->vqvae" + + # TODO - when calling self.vqvae.quantize, it uses self.vqvae.quantize.embedding.weight before + # the forward method of self.vqvae.quantize, so the hook doesn't get called to move the parameter + # off the meta device. There should be a way to fix this instead of just not offloading it + _exclude_from_cpu_offload = ["vqvae"] + + def __init__( + self, + vqvae: VQModel, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModelWithProjection, + transformer: UVit2DModel, + scheduler: AmusedScheduler, + ): + super().__init__() + + self.register_modules( + vqvae=vqvae, + tokenizer=tokenizer, + text_encoder=text_encoder, + transformer=transformer, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vqvae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_normalize=False) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, + do_normalize=False, + do_binarize=True, + do_convert_grayscale=True, + do_resize=True, + ) + self.scheduler.register_to_config(masking_schedule="linear") + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[List[str], str]] = None, + image: PipelineImageInput = None, + mask_image: PipelineImageInput = None, + strength: float = 1.0, + num_inference_steps: int = 12, + guidance_scale: float = 10.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[torch.Generator] = None, + prompt_embeds: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.Tensor] = None, + negative_encoder_hidden_states: Optional[torch.Tensor] = None, + output_type="pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + micro_conditioning_aesthetic_score: int = 6, + micro_conditioning_crop_coord: Tuple[int, int] = (0, 0), + temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), + ): + """ + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + mask_image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask + are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a + single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one + color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B, + H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, + 1)`, or `(H, W)`. + strength (`float`, *optional*, defaults to 1.0): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 16): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 10.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. A single vector from the + pooled and projected final hidden states. + encoder_hidden_states (`torch.FloatTensor`, *optional*): + Pre-generated penultimate hidden states from the text encoder providing additional text conditioning. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + negative_encoder_hidden_states (`torch.FloatTensor`, *optional*): + Analogous to `encoder_hidden_states` for the positive prompt. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + micro_conditioning_aesthetic_score (`int`, *optional*, defaults to 6): + The targeted aesthetic score according to the laion aesthetic classifier. See https://laion.ai/blog/laion-aesthetics/ + and the micro-conditioning section of https://arxiv.org/abs/2307.01952. + micro_conditioning_crop_coord (`Tuple[int]`, *optional*, defaults to (0, 0)): + The targeted height, width crop coordinates. See the micro-conditioning section of https://arxiv.org/abs/2307.01952. + temperature (`Union[int, Tuple[int, int], List[int]]`, *optional*, defaults to (2, 0)): + Configures the temperature scheduler on `self.scheduler` see `AmusedScheduler#set_timesteps`. + + Examples: + + Returns: + [`~pipelines.pipeline_utils.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.pipeline_utils.ImagePipelineOutput`] is returned, otherwise a + `tuple` is returned where the first element is a list with the generated images. + """ + + if (prompt_embeds is not None and encoder_hidden_states is None) or ( + prompt_embeds is None and encoder_hidden_states is not None + ): + raise ValueError("pass either both `prompt_embeds` and `encoder_hidden_states` or neither") + + if (negative_prompt_embeds is not None and negative_encoder_hidden_states is None) or ( + negative_prompt_embeds is None and negative_encoder_hidden_states is not None + ): + raise ValueError( + "pass either both `negatve_prompt_embeds` and `negative_encoder_hidden_states` or neither" + ) + + if (prompt is None and prompt_embeds is None) or (prompt is not None and prompt_embeds is not None): + raise ValueError("pass only one of `prompt` or `prompt_embeds`") + + if isinstance(prompt, str): + prompt = [prompt] + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + batch_size = batch_size * num_images_per_prompt + + if prompt_embeds is None: + input_ids = self.tokenizer( + prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + prompt_embeds = outputs.text_embeds + encoder_hidden_states = outputs.hidden_states[-2] + + prompt_embeds = prompt_embeds.repeat(num_images_per_prompt, 1) + encoder_hidden_states = encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + if guidance_scale > 1.0: + if negative_prompt_embeds is None: + if negative_prompt is None: + negative_prompt = [""] * len(prompt) + + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + + input_ids = self.tokenizer( + negative_prompt, + return_tensors="pt", + padding="max_length", + truncation=True, + max_length=self.tokenizer.model_max_length, + ).input_ids.to(self._execution_device) + + outputs = self.text_encoder(input_ids, return_dict=True, output_hidden_states=True) + negative_prompt_embeds = outputs.text_embeds + negative_encoder_hidden_states = outputs.hidden_states[-2] + + negative_prompt_embeds = negative_prompt_embeds.repeat(num_images_per_prompt, 1) + negative_encoder_hidden_states = negative_encoder_hidden_states.repeat(num_images_per_prompt, 1, 1) + + prompt_embeds = torch.concat([negative_prompt_embeds, prompt_embeds]) + encoder_hidden_states = torch.concat([negative_encoder_hidden_states, encoder_hidden_states]) + + image = self.image_processor.preprocess(image) + + height, width = image.shape[-2:] + + # Note that the micro conditionings _do_ flip the order of width, height for the original size + # and the crop coordinates. This is how it was done in the original code base + micro_conds = torch.tensor( + [ + width, + height, + micro_conditioning_crop_coord[0], + micro_conditioning_crop_coord[1], + micro_conditioning_aesthetic_score, + ], + device=self._execution_device, + dtype=encoder_hidden_states.dtype, + ) + + micro_conds = micro_conds.unsqueeze(0) + micro_conds = micro_conds.expand(2 * batch_size if guidance_scale > 1.0 else batch_size, -1) + + self.scheduler.set_timesteps(num_inference_steps, temperature, self._execution_device) + num_inference_steps = int(len(self.scheduler.timesteps) * strength) + start_timestep_idx = len(self.scheduler.timesteps) - num_inference_steps + + needs_upcasting = self.vqvae.dtype == torch.float16 and self.vqvae.config.force_upcast + + if needs_upcasting: + self.vqvae.float() + + latents = self.vqvae.encode(image.to(dtype=self.vqvae.dtype, device=self._execution_device)).latents + latents_bsz, channels, latents_height, latents_width = latents.shape + latents = self.vqvae.quantize(latents)[2][2].reshape(latents_bsz, latents_height, latents_width) + + mask = self.mask_processor.preprocess( + mask_image, height // self.vae_scale_factor, width // self.vae_scale_factor + ) + mask = mask.reshape(mask.shape[0], latents_height, latents_width).bool().to(latents.device) + latents[mask] = self.scheduler.config.mask_token_id + + starting_mask_ratio = mask.sum() / latents.numel() + + latents = latents.repeat(num_images_per_prompt, 1, 1) + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i in range(start_timestep_idx, len(self.scheduler.timesteps)): + timestep = self.scheduler.timesteps[i] + + if guidance_scale > 1.0: + model_input = torch.cat([latents] * 2) + else: + model_input = latents + + model_output = self.transformer( + model_input, + micro_conds=micro_conds, + pooled_text_emb=prompt_embeds, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + ) + + if guidance_scale > 1.0: + uncond_logits, cond_logits = model_output.chunk(2) + model_output = uncond_logits + guidance_scale * (cond_logits - uncond_logits) + + latents = self.scheduler.step( + model_output=model_output, + timestep=timestep, + sample=latents, + generator=generator, + starting_mask_ratio=starting_mask_ratio, + ).prev_sample + + if i == len(self.scheduler.timesteps) - 1 or ((i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, timestep, latents) + + if output_type == "latent": + output = latents + else: + output = self.vqvae.decode( + latents, + force_not_quantize=True, + shape=( + batch_size, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + self.vqvae.config.latent_channels, + ), + ).sample.clip(0, 1) + output = self.image_processor.postprocess(output, output_type) + + if needs_upcasting: + self.vqvae.half() + + self.maybe_free_model_hooks() + + if not return_dict: + return (output,) + + return ImagePipelineOutput(output) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/animatediff/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/animatediff/__init__.py new file mode 100755 index 0000000..35b99a7 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/animatediff/__init__.py @@ -0,0 +1,49 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {"pipeline_output": ["AnimateDiffPipelineOutput"]} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_animatediff"] = ["AnimateDiffPipeline"] + _import_structure["pipeline_animatediff_video2video"] = ["AnimateDiffVideoToVideoPipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + + else: + from .pipeline_animatediff import AnimateDiffPipeline + from .pipeline_animatediff_video2video import AnimateDiffVideoToVideoPipeline + from .pipeline_output import AnimateDiffPipelineOutput + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_animatediff.py b/diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_animatediff.py new file mode 100755 index 0000000..cd7f0a2 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_animatediff.py @@ -0,0 +1,847 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel, UNetMotionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...models.unets.unet_motion_model import MotionAdapter +from ...schedulers import ( + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, +) +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..free_init_utils import FreeInitMixin +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from .pipeline_output import AnimateDiffPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import MotionAdapter, AnimateDiffPipeline, DDIMScheduler + >>> from diffusers.utils import export_to_gif + + >>> adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-v1-5-2") + >>> pipe = AnimateDiffPipeline.from_pretrained("frankjoshua/toonyou_beta6", motion_adapter=adapter) + >>> pipe.scheduler = DDIMScheduler(beta_schedule="linear", steps_offset=1, clip_sample=False) + >>> output = pipe(prompt="A corgi walking in the park") + >>> frames = output.frames[0] + >>> export_to_gif(frames, "animation.gif") + ``` +""" + + +def tensor2vid(video: torch.Tensor, processor: "VaeImageProcessor", output_type: str = "np"): + batch_size, channels, num_frames, height, width = video.shape + outputs = [] + for batch_idx in range(batch_size): + batch_vid = video[batch_idx].permute(1, 0, 2, 3) + batch_output = processor.postprocess(batch_vid, output_type) + + outputs.append(batch_output) + + if output_type == "np": + outputs = np.stack(outputs) + + elif output_type == "pt": + outputs = torch.stack(outputs) + + elif not output_type == "pil": + raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil']") + + return outputs + + +class AnimateDiffPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, + LoraLoaderMixin, + FreeInitMixin, +): + r""" + Pipeline for text-to-video generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (`CLIPTokenizer`): + A [`~transformers.CLIPTokenizer`] to tokenize text. + unet ([`UNet2DConditionModel`]): + A [`UNet2DConditionModel`] used to create a UNetMotionModel to denoise the encoded video latents. + motion_adapter ([`MotionAdapter`]): + A [`MotionAdapter`] to be used in combination with `unet` to denoise the encoded video latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + _optional_components = ["feature_extractor", "image_encoder", "motion_adapter"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + motion_adapter: MotionAdapter, + scheduler: Union[ + DDIMScheduler, + PNDMScheduler, + LMSDiscreteScheduler, + EulerDiscreteScheduler, + EulerAncestralDiscreteScheduler, + DPMSolverMultistepScheduler, + ], + feature_extractor: CLIPImageProcessor = None, + image_encoder: CLIPVisionModelWithProjection = None, + ): + super().__init__() + if isinstance(unet, UNet2DConditionModel): + unet = UNetMotionModel.from_unet2d(unet, motion_adapter) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + motion_adapter=motion_adapter, + scheduler=scheduler, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt with num_images_per_prompt -> num_videos_per_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.text_to_video_synthesis/pipeline_text_to_video_synth.TextToVideoSDPipeline.decode_latents + def decode_latents(self, latents): + latents = 1 / self.vae.config.scaling_factor * latents + + batch_size, channels, num_frames, height, width = latents.shape + latents = latents.permute(0, 2, 1, 3, 4).reshape(batch_size * num_frames, channels, height, width) + + image = self.vae.decode(latents).sample + video = image[None, :].reshape((batch_size, num_frames, -1) + image.shape[2:]).permute(0, 2, 1, 3, 4) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + video = video.float() + return video + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + # Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_synth.TextToVideoSDPipeline.prepare_latents + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + shape = ( + batch_size, + num_channels_latents, + num_frames, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + num_frames: Optional[int] = 16, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_videos_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated video. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated video. + num_frames (`int`, *optional*, defaults to 16): + The number of video frames that are generated. Defaults to 16 frames which at 8 frames per seconds + amounts to 2 seconds of video. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality videos at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. Latents should be of shape + `(batch_size, num_channel, num_frames, height, width)`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): + Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated video. Choose between `torch.FloatTensor`, `PIL.Image` or + `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] instead + of a plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeine class. + + Examples: + + Returns: + [`~pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] is + returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_videos_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_videos_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + num_channels_latents, + num_frames, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + num_free_init_iters = self._free_init_num_iters if self.free_init_enabled else 1 + for free_init_iter in range(num_free_init_iters): + if self.free_init_enabled: + latents, timesteps = self._apply_free_init( + latents, free_init_iter, num_inference_steps, device, latents.dtype, generator + ) + + self._num_timesteps = len(timesteps) + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + + # 8. Denoising loop + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + ).sample + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents) + + # 9. Post processing + if output_type == "latent": + video = latents + else: + video_tensor = self.decode_latents(latents) + video = tensor2vid(video_tensor, self.image_processor, output_type=output_type) + + # 10. Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (video,) + + return AnimateDiffPipelineOutput(frames=video) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py b/diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py new file mode 100755 index 0000000..cb6b713 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_animatediff_video2video.py @@ -0,0 +1,997 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel, UNetMotionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...models.unets.unet_motion_model import MotionAdapter +from ...schedulers import ( + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, +) +from ...utils import USE_PEFT_BACKEND, logging, scale_lora_layers, unscale_lora_layers +from ...utils.torch_utils import randn_tensor +from ..free_init_utils import FreeInitMixin +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from .pipeline_output import AnimateDiffPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import imageio + >>> import requests + >>> import torch + >>> from diffusers import AnimateDiffVideoToVideoPipeline, DDIMScheduler, MotionAdapter + >>> from diffusers.utils import export_to_gif + >>> from io import BytesIO + >>> from PIL import Image + + >>> adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-v1-5-2", torch_dtype=torch.float16) + >>> pipe = AnimateDiffVideoToVideoPipeline.from_pretrained("SG161222/Realistic_Vision_V5.1_noVAE", motion_adapter=adapter).to("cuda") + >>> pipe.scheduler = DDIMScheduler(beta_schedule="linear", steps_offset=1, clip_sample=False, timespace_spacing="linspace") + + >>> def load_video(file_path: str): + ... images = [] + ... + ... if file_path.startswith(('http://', 'https://')): + ... # If the file_path is a URL + ... response = requests.get(file_path) + ... response.raise_for_status() + ... content = BytesIO(response.content) + ... vid = imageio.get_reader(content) + ... else: + ... # Assuming it's a local file path + ... vid = imageio.get_reader(file_path) + ... + ... for frame in vid: + ... pil_image = Image.fromarray(frame) + ... images.append(pil_image) + ... + ... return images + + >>> video = load_video("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/animatediff-vid2vid-input-1.gif") + >>> output = pipe(video=video, prompt="panda playing a guitar, on a boat, in the ocean, high quality", strength=0.5) + >>> frames = output.frames[0] + >>> export_to_gif(frames, "animation.gif") + ``` +""" + + +# Copied from diffusers.pipelines.animatediff.pipeline_animatediff.tensor2vid +def tensor2vid(video: torch.Tensor, processor, output_type="np"): + batch_size, channels, num_frames, height, width = video.shape + outputs = [] + for batch_idx in range(batch_size): + batch_vid = video[batch_idx].permute(1, 0, 2, 3) + batch_output = processor.postprocess(batch_vid, output_type) + + outputs.append(batch_output) + + if output_type == "np": + outputs = np.stack(outputs) + + elif output_type == "pt": + outputs = torch.stack(outputs) + + elif not output_type == "pil": + raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil']") + + return outputs + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class AnimateDiffVideoToVideoPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, + LoraLoaderMixin, + FreeInitMixin, +): + r""" + Pipeline for video-to-video generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (`CLIPTokenizer`): + A [`~transformers.CLIPTokenizer`] to tokenize text. + unet ([`UNet2DConditionModel`]): + A [`UNet2DConditionModel`] used to create a UNetMotionModel to denoise the encoded video latents. + motion_adapter ([`MotionAdapter`]): + A [`MotionAdapter`] to be used in combination with `unet` to denoise the encoded video latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + _optional_components = ["feature_extractor", "image_encoder", "motion_adapter"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + motion_adapter: MotionAdapter, + scheduler: Union[ + DDIMScheduler, + PNDMScheduler, + LMSDiscreteScheduler, + EulerDiscreteScheduler, + EulerAncestralDiscreteScheduler, + DPMSolverMultistepScheduler, + ], + feature_extractor: CLIPImageProcessor = None, + image_encoder: CLIPVisionModelWithProjection = None, + ): + super().__init__() + if isinstance(unet, UNet2DConditionModel): + unet = UNetMotionModel.from_unet2d(unet, motion_adapter) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + motion_adapter=motion_adapter, + scheduler=scheduler, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt with num_images_per_prompt -> num_videos_per_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.text_to_video_synthesis/pipeline_text_to_video_synth.TextToVideoSDPipeline.decode_latents + def decode_latents(self, latents): + latents = 1 / self.vae.config.scaling_factor * latents + + batch_size, channels, num_frames, height, width = latents.shape + latents = latents.permute(0, 2, 1, 3, 4).reshape(batch_size * num_frames, channels, height, width) + + image = self.vae.decode(latents).sample + video = image[None, :].reshape((batch_size, num_frames, -1) + image.shape[2:]).permute(0, 2, 1, 3, 4) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + video = video.float() + return video + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + strength, + height, + width, + video=None, + latents=None, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if video is not None and latents is not None: + raise ValueError("Only one of `video` or `latents` should be provided") + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + def get_timesteps(self, num_inference_steps, timesteps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = timesteps[t_start * self.scheduler.order :] + + return timesteps, num_inference_steps - t_start + + def prepare_latents( + self, + video, + height, + width, + num_channels_latents, + batch_size, + timestep, + dtype, + device, + generator, + latents=None, + ): + # video must be a list of list of images + # the outer list denotes having multiple videos as input, whereas inner list means the frames of the video + # as a list of images + if not isinstance(video[0], list): + video = [video] + if latents is None: + video = torch.cat( + [self.image_processor.preprocess(vid, height=height, width=width).unsqueeze(0) for vid in video], dim=0 + ) + video = video.to(device=device, dtype=dtype) + num_frames = video.shape[1] + else: + num_frames = latents.shape[2] + + shape = ( + batch_size, + num_channels_latents, + num_frames, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + # make sure the VAE is in float32 mode, as it overflows in float16 + if self.vae.config.force_upcast: + video = video.float() + self.vae.to(dtype=torch.float32) + + if isinstance(generator, list): + if len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + init_latents = [ + retrieve_latents(self.vae.encode(video[i]), generator=generator[i]).unsqueeze(0) + for i in range(batch_size) + ] + else: + init_latents = [ + retrieve_latents(self.vae.encode(vid), generator=generator).unsqueeze(0) for vid in video + ] + + init_latents = torch.cat(init_latents, dim=0) + + # restore vae to original dtype + if self.vae.config.force_upcast: + self.vae.to(dtype) + + init_latents = init_latents.to(dtype) + init_latents = self.vae.config.scaling_factor * init_latents + + if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0: + # expand init_latents for batch_size + error_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {init_latents.shape[0]} initial" + " images (`image`). Please make sure to update your script to pass as many initial images as text prompts" + ) + raise ValueError(error_message) + elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts." + ) + else: + init_latents = torch.cat([init_latents], dim=0) + + noise = randn_tensor(init_latents.shape, generator=generator, device=device, dtype=dtype) + latents = self.scheduler.add_noise(init_latents, noise, timestep).permute(0, 2, 1, 3, 4) + else: + if shape != latents.shape: + # [B, C, F, H, W] + raise ValueError(f"`latents` expected to have {shape=}, but found {latents.shape=}") + latents = latents.to(device, dtype=dtype) + + return latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + def __call__( + self, + video: List[List[PipelineImageInput]] = None, + prompt: Optional[Union[str, List[str]]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + timesteps: Optional[List[int]] = None, + guidance_scale: float = 7.5, + strength: float = 0.8, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_videos_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + ): + r""" + The call function to the pipeline for generation. + + Args: + video (`List[PipelineImageInput]`): + The input video to condition the generation on. Must be a list of images/frames of the video. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated video. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated video. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality videos at the + expense of slower inference. + strength (`float`, *optional*, defaults to 0.8): + Higher strength leads to more differences between original video and generated video. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. Latents should be of shape + `(batch_size, num_channel, num_frames, height, width)`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): + Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated video. Choose between `torch.FloatTensor`, `PIL.Image` or + `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`AnimateDiffPipelineOutput`] instead + of a plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeine class. + + Examples: + + Returns: + [`pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`pipelines.animatediff.pipeline_output.AnimateDiffPipelineOutput`] is + returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. + """ + + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt=prompt, + strength=strength, + height=height, + width=width, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + video=video, + latents=latents, + ip_adapter_image=ip_adapter_image, + ip_adapter_image_embeds=ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_videos_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_videos_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, timesteps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_videos_per_prompt) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + video=video, + height=height, + width=width, + num_channels_latents=num_channels_latents, + batch_size=batch_size * num_videos_per_prompt, + timestep=latent_timestep, + dtype=prompt_embeds.dtype, + device=device, + generator=generator, + latents=latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + num_free_init_iters = self._free_init_num_iters if self.free_init_enabled else 1 + for free_init_iter in range(num_free_init_iters): + if self.free_init_enabled: + latents, timesteps = self._apply_free_init( + latents, free_init_iter, num_inference_steps, device, latents.dtype, generator + ) + num_inference_steps = len(timesteps) + # make sure to readjust timesteps based on strength + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, timesteps, strength, device) + + self._num_timesteps = len(timesteps) + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + + # 8. Denoising loop + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + ).sample + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + # 9. Post-processing + if output_type == "latent": + video = latents + else: + video_tensor = self.decode_latents(latents) + video = tensor2vid(video_tensor, self.image_processor, output_type=output_type) + + # 10. Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (video,) + + return AnimateDiffPipelineOutput(frames=video) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_output.py b/diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_output.py new file mode 100755 index 0000000..184a458 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/animatediff/pipeline_output.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass +from typing import List, Union + +import numpy as np +import PIL.Image +import torch + +from ...utils import BaseOutput + + +@dataclass +class AnimateDiffPipelineOutput(BaseOutput): + r""" + Output class for AnimateDiff pipelines. + + Args: + frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised + PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)` + """ + + frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]] diff --git a/diffusers-0.27.0/src/diffusers/pipelines/audioldm/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/audioldm/__init__.py new file mode 100755 index 0000000..a002b4a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/audioldm/__init__.py @@ -0,0 +1,51 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + is_torch_available, + is_transformers_available, + is_transformers_version, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import ( + AudioLDMPipeline, + ) + + _dummy_objects.update({"AudioLDMPipeline": AudioLDMPipeline}) +else: + _import_structure["pipeline_audioldm"] = ["AudioLDMPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import ( + AudioLDMPipeline, + ) + + else: + from .pipeline_audioldm import AudioLDMPipeline +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/audioldm/pipeline_audioldm.py b/diffusers-0.27.0/src/diffusers/pipelines/audioldm/pipeline_audioldm.py new file mode 100755 index 0000000..69bebdd --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/audioldm/pipeline_audioldm.py @@ -0,0 +1,546 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import torch +import torch.nn.functional as F +from transformers import ClapTextModelWithProjection, RobertaTokenizer, RobertaTokenizerFast, SpeechT5HifiGan + +from ...models import AutoencoderKL, UNet2DConditionModel +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import logging, replace_example_docstring +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import AudioPipelineOutput, DiffusionPipeline, StableDiffusionMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import AudioLDMPipeline + >>> import torch + >>> import scipy + + >>> repo_id = "cvssp/audioldm-s-full-v2" + >>> pipe = AudioLDMPipeline.from_pretrained(repo_id, torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + + >>> prompt = "Techno music with a strong, upbeat tempo and high melodic riffs" + >>> audio = pipe(prompt, num_inference_steps=10, audio_length_in_s=5.0).audios[0] + + >>> # save the audio sample as a .wav file + >>> scipy.io.wavfile.write("techno.wav", rate=16000, data=audio) + ``` +""" + + +class AudioLDMPipeline(DiffusionPipeline, StableDiffusionMixin): + r""" + Pipeline for text-to-audio generation using AudioLDM. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.ClapTextModelWithProjection`]): + Frozen text-encoder (`ClapTextModelWithProjection`, specifically the + [laion/clap-htsat-unfused](https://huggingface.co/laion/clap-htsat-unfused) variant. + tokenizer ([`PreTrainedTokenizer`]): + A [`~transformers.RobertaTokenizer`] to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded audio latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded audio latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + vocoder ([`~transformers.SpeechT5HifiGan`]): + Vocoder of class `SpeechT5HifiGan`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: ClapTextModelWithProjection, + tokenizer: Union[RobertaTokenizer, RobertaTokenizerFast], + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + vocoder: SpeechT5HifiGan, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + vocoder=vocoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + + def _encode_prompt( + self, + prompt, + device, + num_waveforms_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device (`torch.device`): + torch device + num_waveforms_per_prompt (`int`): + number of waveforms that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the audio generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + """ + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLAP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask.to(device), + ) + prompt_embeds = prompt_embeds.text_embeds + # additional L_2 normalization over each hidden-state + prompt_embeds = F.normalize(prompt_embeds, dim=-1) + + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device) + + ( + bs_embed, + seq_len, + ) = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_waveforms_per_prompt) + prompt_embeds = prompt_embeds.view(bs_embed * num_waveforms_per_prompt, seq_len) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + uncond_input_ids = uncond_input.input_ids.to(device) + attention_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds = self.text_encoder( + uncond_input_ids, + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds.text_embeds + # additional L_2 normalization over each hidden-state + negative_prompt_embeds = F.normalize(negative_prompt_embeds, dim=-1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_waveforms_per_prompt) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_waveforms_per_prompt, seq_len) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + def decode_latents(self, latents): + latents = 1 / self.vae.config.scaling_factor * latents + mel_spectrogram = self.vae.decode(latents).sample + return mel_spectrogram + + def mel_spectrogram_to_waveform(self, mel_spectrogram): + if mel_spectrogram.dim() == 4: + mel_spectrogram = mel_spectrogram.squeeze(1) + + waveform = self.vocoder(mel_spectrogram) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + waveform = waveform.cpu().float() + return waveform + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + audio_length_in_s, + vocoder_upsample_factor, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + min_audio_length_in_s = vocoder_upsample_factor * self.vae_scale_factor + if audio_length_in_s < min_audio_length_in_s: + raise ValueError( + f"`audio_length_in_s` has to be a positive value greater than or equal to {min_audio_length_in_s}, but " + f"is {audio_length_in_s}." + ) + + if self.vocoder.config.model_in_dim % self.vae_scale_factor != 0: + raise ValueError( + f"The number of frequency bins in the vocoder's log-mel spectrogram has to be divisible by the " + f"VAE scale factor, but got {self.vocoder.config.model_in_dim} bins and a scale factor of " + f"{self.vae_scale_factor}." + ) + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents with width->self.vocoder.config.model_in_dim + def prepare_latents(self, batch_size, num_channels_latents, height, dtype, device, generator, latents=None): + shape = ( + batch_size, + num_channels_latents, + height // self.vae_scale_factor, + self.vocoder.config.model_in_dim // self.vae_scale_factor, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + audio_length_in_s: Optional[float] = None, + num_inference_steps: int = 10, + guidance_scale: float = 2.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_waveforms_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: Optional[int] = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + output_type: Optional[str] = "np", + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide audio generation. If not defined, you need to pass `prompt_embeds`. + audio_length_in_s (`int`, *optional*, defaults to 5.12): + The length of the generated audio sample in seconds. + num_inference_steps (`int`, *optional*, defaults to 10): + The number of denoising steps. More denoising steps usually lead to a higher quality audio at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 2.5): + A higher guidance scale value encourages the model to generate audio that is closely linked to the text + `prompt` at the expense of lower sound quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in audio generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_waveforms_per_prompt (`int`, *optional*, defaults to 1): + The number of waveforms to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.AudioPipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + output_type (`str`, *optional*, defaults to `"np"`): + The output format of the generated image. Choose between `"np"` to return a NumPy `np.ndarray` or + `"pt"` to return a PyTorch `torch.Tensor` object. + + Examples: + + Returns: + [`~pipelines.AudioPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.AudioPipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated audio. + """ + # 0. Convert audio input length from seconds to spectrogram height + vocoder_upsample_factor = np.prod(self.vocoder.config.upsample_rates) / self.vocoder.config.sampling_rate + + if audio_length_in_s is None: + audio_length_in_s = self.unet.config.sample_size * self.vae_scale_factor * vocoder_upsample_factor + + height = int(audio_length_in_s / vocoder_upsample_factor) + + original_waveform_length = int(audio_length_in_s * self.vocoder.config.sampling_rate) + if height % self.vae_scale_factor != 0: + height = int(np.ceil(height / self.vae_scale_factor)) * self.vae_scale_factor + logger.info( + f"Audio length in seconds {audio_length_in_s} is increased to {height * vocoder_upsample_factor} " + f"so that it can be handled by the model. It will be cut to {audio_length_in_s} after the " + f"denoising process." + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + audio_length_in_s, + vocoder_upsample_factor, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds = self._encode_prompt( + prompt, + device, + num_waveforms_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_waveforms_per_prompt, + num_channels_latents, + height, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=None, + class_labels=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 8. Post-processing + mel_spectrogram = self.decode_latents(latents) + + audio = self.mel_spectrogram_to_waveform(mel_spectrogram) + + audio = audio[:, :original_waveform_length] + + if output_type == "np": + audio = audio.numpy() + + if not return_dict: + return (audio,) + + return AudioPipelineOutput(audios=audio) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/audioldm2/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/audioldm2/__init__.py new file mode 100755 index 0000000..23cd0e4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/audioldm2/__init__.py @@ -0,0 +1,50 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, + is_transformers_version, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["modeling_audioldm2"] = ["AudioLDM2ProjectionModel", "AudioLDM2UNet2DConditionModel"] + _import_structure["pipeline_audioldm2"] = ["AudioLDM2Pipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + + else: + from .modeling_audioldm2 import AudioLDM2ProjectionModel, AudioLDM2UNet2DConditionModel + from .pipeline_audioldm2 import AudioLDM2Pipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py b/diffusers-0.27.0/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py new file mode 100755 index 0000000..c0b85e4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/audioldm2/modeling_audioldm2.py @@ -0,0 +1,1511 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.utils.checkpoint + +from ...configuration_utils import ConfigMixin, register_to_config +from ...loaders import UNet2DConditionLoadersMixin +from ...models.activations import get_activation +from ...models.attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from ...models.embeddings import ( + TimestepEmbedding, + Timesteps, +) +from ...models.modeling_utils import ModelMixin +from ...models.resnet import Downsample2D, ResnetBlock2D, Upsample2D +from ...models.transformers.transformer_2d import Transformer2DModel +from ...models.unets.unet_2d_blocks import DownBlock2D, UpBlock2D +from ...models.unets.unet_2d_condition import UNet2DConditionOutput +from ...utils import BaseOutput, is_torch_version, logging + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def add_special_tokens(hidden_states, attention_mask, sos_token, eos_token): + batch_size = hidden_states.shape[0] + + if attention_mask is not None: + # Add two more steps to attn mask + new_attn_mask_step = attention_mask.new_ones((batch_size, 1)) + attention_mask = torch.concat([new_attn_mask_step, attention_mask, new_attn_mask_step], dim=-1) + + # Add the SOS / EOS tokens at the start / end of the sequence respectively + sos_token = sos_token.expand(batch_size, 1, -1) + eos_token = eos_token.expand(batch_size, 1, -1) + hidden_states = torch.concat([sos_token, hidden_states, eos_token], dim=1) + return hidden_states, attention_mask + + +@dataclass +class AudioLDM2ProjectionModelOutput(BaseOutput): + """ + Args: + Class for AudioLDM2 projection layer's outputs. + hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states obtained by linearly projecting the hidden-states for each of the text + encoders and subsequently concatenating them together. + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices, formed by concatenating the attention masks + for the two text encoders together. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + """ + + hidden_states: torch.FloatTensor + attention_mask: Optional[torch.LongTensor] = None + + +class AudioLDM2ProjectionModel(ModelMixin, ConfigMixin): + """ + A simple linear projection model to map two text embeddings to a shared latent space. It also inserts learned + embedding vectors at the start and end of each text embedding sequence respectively. Each variable appended with + `_1` refers to that corresponding to the second text encoder. Otherwise, it is from the first. + + Args: + text_encoder_dim (`int`): + Dimensionality of the text embeddings from the first text encoder (CLAP). + text_encoder_1_dim (`int`): + Dimensionality of the text embeddings from the second text encoder (T5 or VITS). + langauge_model_dim (`int`): + Dimensionality of the text embeddings from the language model (GPT2). + """ + + @register_to_config + def __init__(self, text_encoder_dim, text_encoder_1_dim, langauge_model_dim): + super().__init__() + # additional projection layers for each text encoder + self.projection = nn.Linear(text_encoder_dim, langauge_model_dim) + self.projection_1 = nn.Linear(text_encoder_1_dim, langauge_model_dim) + + # learnable SOS / EOS token embeddings for each text encoder + self.sos_embed = nn.Parameter(torch.ones(langauge_model_dim)) + self.eos_embed = nn.Parameter(torch.ones(langauge_model_dim)) + + self.sos_embed_1 = nn.Parameter(torch.ones(langauge_model_dim)) + self.eos_embed_1 = nn.Parameter(torch.ones(langauge_model_dim)) + + def forward( + self, + hidden_states: Optional[torch.FloatTensor] = None, + hidden_states_1: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + attention_mask_1: Optional[torch.LongTensor] = None, + ): + hidden_states = self.projection(hidden_states) + hidden_states, attention_mask = add_special_tokens( + hidden_states, attention_mask, sos_token=self.sos_embed, eos_token=self.eos_embed + ) + + hidden_states_1 = self.projection_1(hidden_states_1) + hidden_states_1, attention_mask_1 = add_special_tokens( + hidden_states_1, attention_mask_1, sos_token=self.sos_embed_1, eos_token=self.eos_embed_1 + ) + + # concatenate clap and t5 text encoding + hidden_states = torch.cat([hidden_states, hidden_states_1], dim=1) + + # concatenate attention masks + if attention_mask is None and attention_mask_1 is not None: + attention_mask = attention_mask_1.new_ones((hidden_states[:2])) + elif attention_mask is not None and attention_mask_1 is None: + attention_mask_1 = attention_mask.new_ones((hidden_states_1[:2])) + + if attention_mask is not None and attention_mask_1 is not None: + attention_mask = torch.cat([attention_mask, attention_mask_1], dim=-1) + else: + attention_mask = None + + return AudioLDM2ProjectionModelOutput( + hidden_states=hidden_states, + attention_mask=attention_mask, + ) + + +class AudioLDM2UNet2DConditionModel(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin): + r""" + A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample + shaped output. Compared to the vanilla [`UNet2DConditionModel`], this variant optionally includes an additional + self-attention layer in each Transformer block, as well as multiple cross-attention layers. It also allows for up + to two cross-attention embeddings, `encoder_hidden_states` and `encoder_hidden_states_1`. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + Height and width of input/output sample. + in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 4): Number of channels in the output. + flip_sin_to_cos (`bool`, *optional*, defaults to `False`): + Whether to flip the sin to cos in the time embedding. + freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding. + down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D")`): + The tuple of downsample blocks to use. + mid_block_type (`str`, *optional*, defaults to `"UNetMidBlock2DCrossAttn"`): + Block type for middle of UNet, it can only be `UNetMidBlock2DCrossAttn` for AudioLDM2. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D")`): + The tuple of upsample blocks to use. + only_cross_attention (`bool` or `Tuple[bool]`, *optional*, default to `False`): + Whether to include self-attention in the basic transformer blocks, see + [`~models.attention.BasicTransformerBlock`]. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. + downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution. + mid_block_scale_factor (`float`, *optional*, defaults to 1.0): The scale factor to use for the mid block. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. + If `None`, normalization and activation layers is skipped in post-processing. + norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization. + cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280): + The dimension of the cross attention features. + transformer_layers_per_block (`int` or `Tuple[int]`, *optional*, defaults to 1): + The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for + [`~models.unet_2d_blocks.CrossAttnDownBlock2D`], [`~models.unet_2d_blocks.CrossAttnUpBlock2D`], + [`~models.unet_2d_blocks.UNetMidBlock2DCrossAttn`]. + attention_head_dim (`int`, *optional*, defaults to 8): The dimension of the attention heads. + num_attention_heads (`int`, *optional*): + The number of attention heads. If not defined, defaults to `attention_head_dim` + resnet_time_scale_shift (`str`, *optional*, defaults to `"default"`): Time scale shift config + for ResNet blocks (see [`~models.resnet.ResnetBlock2D`]). Choose from `default` or `scale_shift`. + class_embed_type (`str`, *optional*, defaults to `None`): + The type of class embedding to use which is ultimately summed with the time embeddings. Choose from `None`, + `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`. + num_class_embeds (`int`, *optional*, defaults to `None`): + Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing + class conditioning with `class_embed_type` equal to `None`. + time_embedding_type (`str`, *optional*, defaults to `positional`): + The type of position embedding to use for timesteps. Choose from `positional` or `fourier`. + time_embedding_dim (`int`, *optional*, defaults to `None`): + An optional override for the dimension of the projected time embedding. + time_embedding_act_fn (`str`, *optional*, defaults to `None`): + Optional activation function to use only once on the time embeddings before they are passed to the rest of + the UNet. Choose from `silu`, `mish`, `gelu`, and `swish`. + timestep_post_act (`str`, *optional*, defaults to `None`): + The second activation function to use in timestep embedding. Choose from `silu`, `mish` and `gelu`. + time_cond_proj_dim (`int`, *optional*, defaults to `None`): + The dimension of `cond_proj` layer in the timestep embedding. + conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer. + conv_out_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_out` layer. + projection_class_embeddings_input_dim (`int`, *optional*): The dimension of the `class_labels` input when + `class_embed_type="projection"`. Required when `class_embed_type="projection"`. + class_embeddings_concat (`bool`, *optional*, defaults to `False`): Whether to concatenate the time + embeddings with the class embeddings. + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + sample_size: Optional[int] = None, + in_channels: int = 4, + out_channels: int = 4, + flip_sin_to_cos: bool = True, + freq_shift: int = 0, + down_block_types: Tuple[str] = ( + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "CrossAttnDownBlock2D", + "DownBlock2D", + ), + mid_block_type: Optional[str] = "UNetMidBlock2DCrossAttn", + up_block_types: Tuple[str] = ("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + only_cross_attention: Union[bool, Tuple[bool]] = False, + block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + layers_per_block: Union[int, Tuple[int]] = 2, + downsample_padding: int = 1, + mid_block_scale_factor: float = 1, + act_fn: str = "silu", + norm_num_groups: Optional[int] = 32, + norm_eps: float = 1e-5, + cross_attention_dim: Union[int, Tuple[int]] = 1280, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + attention_head_dim: Union[int, Tuple[int]] = 8, + num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + use_linear_projection: bool = False, + class_embed_type: Optional[str] = None, + num_class_embeds: Optional[int] = None, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + time_embedding_type: str = "positional", + time_embedding_dim: Optional[int] = None, + time_embedding_act_fn: Optional[str] = None, + timestep_post_act: Optional[str] = None, + time_cond_proj_dim: Optional[int] = None, + conv_in_kernel: int = 3, + conv_out_kernel: int = 3, + projection_class_embeddings_input_dim: Optional[int] = None, + class_embeddings_concat: bool = False, + ): + super().__init__() + + self.sample_size = sample_size + + if num_attention_heads is not None: + raise ValueError( + "At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19." + ) + + # If `num_attention_heads` is not defined (which is the case for most models) + # it will default to `attention_head_dim`. This looks weird upon first reading it and it is. + # The reason for this behavior is to correct for incorrectly named variables that were introduced + # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131 + # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking + # which is why we correct for the naming here. + num_attention_heads = num_attention_heads or attention_head_dim + + # Check inputs + if len(down_block_types) != len(up_block_types): + raise ValueError( + f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}." + ) + + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(only_cross_attention, bool) and len(only_cross_attention) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(attention_head_dim, int) and len(attention_head_dim) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `attention_head_dim` as `down_block_types`. `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}." + ) + + if isinstance(cross_attention_dim, list) and len(cross_attention_dim) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `cross_attention_dim` as `down_block_types`. `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(layers_per_block, int) and len(layers_per_block) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `layers_per_block` as `down_block_types`. `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}." + ) + + # input + conv_in_padding = (conv_in_kernel - 1) // 2 + self.conv_in = nn.Conv2d( + in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding + ) + + # time + if time_embedding_type == "positional": + time_embed_dim = time_embedding_dim or block_out_channels[0] * 4 + + self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift) + timestep_input_dim = block_out_channels[0] + else: + raise ValueError(f"{time_embedding_type} does not exist. Please make sure to use `positional`.") + + self.time_embedding = TimestepEmbedding( + timestep_input_dim, + time_embed_dim, + act_fn=act_fn, + post_act_fn=timestep_post_act, + cond_proj_dim=time_cond_proj_dim, + ) + + # class embedding + if class_embed_type is None and num_class_embeds is not None: + self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim) + elif class_embed_type == "timestep": + self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim, act_fn=act_fn) + elif class_embed_type == "identity": + self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim) + elif class_embed_type == "projection": + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set" + ) + # The projection `class_embed_type` is the same as the timestep `class_embed_type` except + # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings + # 2. it projects from an arbitrary input dimension. + # + # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations. + # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings. + # As a result, `TimestepEmbedding` can be passed arbitrary vectors. + self.class_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim) + elif class_embed_type == "simple_projection": + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'simple_projection' requires `projection_class_embeddings_input_dim` be set" + ) + self.class_embedding = nn.Linear(projection_class_embeddings_input_dim, time_embed_dim) + else: + self.class_embedding = None + + if time_embedding_act_fn is None: + self.time_embed_act = None + else: + self.time_embed_act = get_activation(time_embedding_act_fn) + + self.down_blocks = nn.ModuleList([]) + self.up_blocks = nn.ModuleList([]) + + if isinstance(only_cross_attention, bool): + only_cross_attention = [only_cross_attention] * len(down_block_types) + + if isinstance(num_attention_heads, int): + num_attention_heads = (num_attention_heads,) * len(down_block_types) + + if isinstance(cross_attention_dim, int): + cross_attention_dim = (cross_attention_dim,) * len(down_block_types) + + if isinstance(layers_per_block, int): + layers_per_block = [layers_per_block] * len(down_block_types) + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * len(down_block_types) + + if class_embeddings_concat: + # The time embeddings are concatenated with the class embeddings. The dimension of the + # time embeddings passed to the down, middle, and up blocks is twice the dimension of the + # regular time embeddings + blocks_time_embed_dim = time_embed_dim * 2 + else: + blocks_time_embed_dim = time_embed_dim + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block[i], + transformer_layers_per_block=transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + temb_channels=blocks_time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim[i], + num_attention_heads=num_attention_heads[i], + downsample_padding=downsample_padding, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + self.down_blocks.append(down_block) + + # mid + if mid_block_type == "UNetMidBlock2DCrossAttn": + self.mid_block = UNetMidBlock2DCrossAttn( + transformer_layers_per_block=transformer_layers_per_block[-1], + in_channels=block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_time_scale_shift=resnet_time_scale_shift, + cross_attention_dim=cross_attention_dim[-1], + num_attention_heads=num_attention_heads[-1], + resnet_groups=norm_num_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + ) + else: + raise ValueError( + f"unknown mid_block_type : {mid_block_type}. Should be `UNetMidBlock2DCrossAttn` for AudioLDM2." + ) + + # count how many layers upsample the images + self.num_upsamplers = 0 + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_num_attention_heads = list(reversed(num_attention_heads)) + reversed_layers_per_block = list(reversed(layers_per_block)) + reversed_cross_attention_dim = list(reversed(cross_attention_dim)) + reversed_transformer_layers_per_block = list(reversed(transformer_layers_per_block)) + only_cross_attention = list(reversed(only_cross_attention)) + + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + is_final_block = i == len(block_out_channels) - 1 + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)] + + # add upsample block for all BUT final layer + if not is_final_block: + add_upsample = True + self.num_upsamplers += 1 + else: + add_upsample = False + + up_block = get_up_block( + up_block_type, + num_layers=reversed_layers_per_block[i] + 1, + transformer_layers_per_block=reversed_transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=blocks_time_embed_dim, + add_upsample=add_upsample, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=reversed_cross_attention_dim[i], + num_attention_heads=reversed_num_attention_heads[i], + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_num_groups is not None: + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps + ) + + self.conv_act = get_activation(act_fn) + + else: + self.conv_norm_out = None + self.conv_act = None + + conv_out_padding = (conv_out_kernel - 1) // 2 + self.conv_out = nn.Conv2d( + block_out_channels[0], out_channels, kernel_size=conv_out_kernel, padding=conv_out_padding + ) + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attention_slice + def set_attention_slice(self, slice_size): + r""" + Enable sliced attention computation. + + When this option is enabled, the attention module splits the input tensor in slices to compute attention in + several steps. This is useful for saving some memory in exchange for a small decrease in speed. + + Args: + slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`): + When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If + `"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is + provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim` + must be a multiple of `slice_size`. + """ + sliceable_head_dims = [] + + def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): + if hasattr(module, "set_attention_slice"): + sliceable_head_dims.append(module.sliceable_head_dim) + + for child in module.children(): + fn_recursive_retrieve_sliceable_dims(child) + + # retrieve number of attention layers + for module in self.children(): + fn_recursive_retrieve_sliceable_dims(module) + + num_sliceable_layers = len(sliceable_head_dims) + + if slice_size == "auto": + # half the attention head size is usually a good trade-off between + # speed and memory + slice_size = [dim // 2 for dim in sliceable_head_dims] + elif slice_size == "max": + # make smallest slice possible + slice_size = num_sliceable_layers * [1] + + slice_size = num_sliceable_layers * [slice_size] if not isinstance(slice_size, list) else slice_size + + if len(slice_size) != len(sliceable_head_dims): + raise ValueError( + f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different" + f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}." + ) + + for i in range(len(slice_size)): + size = slice_size[i] + dim = sliceable_head_dims[i] + if size is not None and size > dim: + raise ValueError(f"size {size} has to be smaller or equal to {dim}.") + + # Recursively walk through all the children. + # Any children which exposes the set_attention_slice method + # gets the message + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + if hasattr(module, "set_attention_slice"): + module.set_attention_slice(slice_size.pop()) + + for child in module.children(): + fn_recursive_set_attention_slice(child, slice_size) + + reversed_slice_size = list(reversed(slice_size)) + for module in self.children(): + fn_recursive_set_attention_slice(module, reversed_slice_size) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel._set_gradient_checkpointing + def _set_gradient_checkpointing(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + return_dict: bool = True, + encoder_hidden_states_1: Optional[torch.Tensor] = None, + encoder_attention_mask_1: Optional[torch.Tensor] = None, + ) -> Union[UNet2DConditionOutput, Tuple]: + r""" + The [`AudioLDM2UNet2DConditionModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor with the following shape `(batch, channel, height, width)`. + timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input. + encoder_hidden_states (`torch.FloatTensor`): + The encoder hidden states with shape `(batch, sequence_length, feature_dim)`. + encoder_attention_mask (`torch.Tensor`): + A cross-attention mask of shape `(batch, sequence_length)` is applied to `encoder_hidden_states`. If + `True` the mask is kept, otherwise if `False` it is discarded. Mask will be converted into a bias, + which adds large negative values to the attention scores corresponding to "discard" tokens. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] instead of a plain + tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttnProcessor`]. + encoder_hidden_states_1 (`torch.FloatTensor`, *optional*): + A second set of encoder hidden states with shape `(batch, sequence_length_2, feature_dim_2)`. Can be + used to condition the model on a different set of embeddings to `encoder_hidden_states`. + encoder_attention_mask_1 (`torch.Tensor`, *optional*): + A cross-attention mask of shape `(batch, sequence_length_2)` is applied to `encoder_hidden_states_1`. + If `True` the mask is kept, otherwise if `False` it is discarded. Mask will be converted into a bias, + which adds large negative values to the attention scores corresponding to "discard" tokens. + + Returns: + [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] or `tuple`: + If `return_dict` is True, an [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] is returned, otherwise + a `tuple` is returned where the first element is the sample tensor. + """ + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layers). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2**self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + if any(s % default_overall_up_factor != 0 for s in sample.shape[-2:]): + logger.info("Forward upsample size to force interpolation output size.") + forward_upsample_size = True + + # ensure attention_mask is a bias, and give it a singleton query_tokens dimension + # expects mask of shape: + # [batch, key_tokens] + # adds singleton query_tokens dimension: + # [batch, 1, key_tokens] + # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes: + # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn) + # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn) + if attention_mask is not None: + # assume that mask is expressed as: + # (1 = keep, 0 = discard) + # convert mask into a bias that can be added to attention scores: + # (keep = +0, discard = -10000.0) + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # convert encoder_attention_mask to a bias the same way we do for attention_mask + if encoder_attention_mask is not None: + encoder_attention_mask = (1 - encoder_attention_mask.to(sample.dtype)) * -10000.0 + encoder_attention_mask = encoder_attention_mask.unsqueeze(1) + + if encoder_attention_mask_1 is not None: + encoder_attention_mask_1 = (1 - encoder_attention_mask_1.to(sample.dtype)) * -10000.0 + encoder_attention_mask_1 = encoder_attention_mask_1.unsqueeze(1) + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == "mps" + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=sample.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + aug_emb = None + + if self.class_embedding is not None: + if class_labels is None: + raise ValueError("class_labels should be provided when num_class_embeds > 0") + + if self.config.class_embed_type == "timestep": + class_labels = self.time_proj(class_labels) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # there might be better ways to encapsulate this. + class_labels = class_labels.to(dtype=sample.dtype) + + class_emb = self.class_embedding(class_labels).to(dtype=sample.dtype) + + if self.config.class_embeddings_concat: + emb = torch.cat([emb, class_emb], dim=-1) + else: + emb = emb + class_emb + + emb = emb + aug_emb if aug_emb is not None else emb + + if self.time_embed_act is not None: + emb = self.time_embed_act(emb) + + # 2. pre-process + sample = self.conv_in(sample) + + # 3. down + down_block_res_samples = (sample,) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + encoder_hidden_states_1=encoder_hidden_states_1, + encoder_attention_mask_1=encoder_attention_mask_1, + ) + else: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb) + + down_block_res_samples += res_samples + + # 4. mid + if self.mid_block is not None: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + encoder_hidden_states_1=encoder_hidden_states_1, + encoder_attention_mask_1=encoder_attention_mask_1, + ) + + # 5. up + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets) :] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + upsample_size=upsample_size, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + encoder_hidden_states_1=encoder_hidden_states_1, + encoder_attention_mask_1=encoder_attention_mask_1, + ) + else: + sample = upsample_block( + hidden_states=sample, temb=emb, res_hidden_states_tuple=res_samples, upsample_size=upsample_size + ) + + # 6. post-process + if self.conv_norm_out: + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + if not return_dict: + return (sample,) + + return UNet2DConditionOutput(sample=sample) + + +def get_down_block( + down_block_type, + num_layers, + in_channels, + out_channels, + temb_channels, + add_downsample, + resnet_eps, + resnet_act_fn, + transformer_layers_per_block=1, + num_attention_heads=None, + resnet_groups=None, + cross_attention_dim=None, + downsample_padding=None, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + resnet_time_scale_shift="default", +): + down_block_type = down_block_type[7:] if down_block_type.startswith("UNetRes") else down_block_type + if down_block_type == "DownBlock2D": + return DownBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "CrossAttnDownBlock2D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnDownBlock2D") + return CrossAttnDownBlock2D( + num_layers=num_layers, + transformer_layers_per_block=transformer_layers_per_block, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + raise ValueError(f"{down_block_type} does not exist.") + + +def get_up_block( + up_block_type, + num_layers, + in_channels, + out_channels, + prev_output_channel, + temb_channels, + add_upsample, + resnet_eps, + resnet_act_fn, + transformer_layers_per_block=1, + num_attention_heads=None, + resnet_groups=None, + cross_attention_dim=None, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + resnet_time_scale_shift="default", +): + up_block_type = up_block_type[7:] if up_block_type.startswith("UNetRes") else up_block_type + if up_block_type == "UpBlock2D": + return UpBlock2D( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == "CrossAttnUpBlock2D": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnUpBlock2D") + return CrossAttnUpBlock2D( + num_layers=num_layers, + transformer_layers_per_block=transformer_layers_per_block, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + raise ValueError(f"{up_block_type} does not exist.") + + +class CrossAttnDownBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads=1, + cross_attention_dim=1280, + output_scale_factor=1.0, + downsample_padding=1, + add_downsample=True, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + if isinstance(cross_attention_dim, int): + cross_attention_dim = (cross_attention_dim,) + if isinstance(cross_attention_dim, (list, tuple)) and len(cross_attention_dim) > 4: + raise ValueError( + "Only up to 4 cross-attention layers are supported. Ensure that the length of cross-attention " + f"dims is less than or equal to 4. Got cross-attention dims {cross_attention_dim} of length {len(cross_attention_dim)}" + ) + self.cross_attention_dim = cross_attention_dim + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + for j in range(len(cross_attention_dim)): + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block, + cross_attention_dim=cross_attention_dim[j], + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + double_self_attention=True if cross_attention_dim[j] is None else False, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + Downsample2D( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states_1: Optional[torch.FloatTensor] = None, + encoder_attention_mask_1: Optional[torch.FloatTensor] = None, + ): + output_states = () + num_layers = len(self.resnets) + num_attention_per_layer = len(self.attentions) // num_layers + + encoder_hidden_states_1 = ( + encoder_hidden_states_1 if encoder_hidden_states_1 is not None else encoder_hidden_states + ) + encoder_attention_mask_1 = ( + encoder_attention_mask_1 if encoder_hidden_states_1 is not None else encoder_attention_mask + ) + + for i in range(num_layers): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.resnets[i]), + hidden_states, + temb, + **ckpt_kwargs, + ) + for idx, cross_attention_dim in enumerate(self.cross_attention_dim): + if cross_attention_dim is not None and idx <= 1: + forward_encoder_hidden_states = encoder_hidden_states + forward_encoder_attention_mask = encoder_attention_mask + elif cross_attention_dim is not None and idx > 1: + forward_encoder_hidden_states = encoder_hidden_states_1 + forward_encoder_attention_mask = encoder_attention_mask_1 + else: + forward_encoder_hidden_states = None + forward_encoder_attention_mask = None + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.attentions[i * num_attention_per_layer + idx], return_dict=False), + hidden_states, + forward_encoder_hidden_states, + None, # timestep + None, # class_labels + cross_attention_kwargs, + attention_mask, + forward_encoder_attention_mask, + **ckpt_kwargs, + )[0] + else: + hidden_states = self.resnets[i](hidden_states, temb) + for idx, cross_attention_dim in enumerate(self.cross_attention_dim): + if cross_attention_dim is not None and idx <= 1: + forward_encoder_hidden_states = encoder_hidden_states + forward_encoder_attention_mask = encoder_attention_mask + elif cross_attention_dim is not None and idx > 1: + forward_encoder_hidden_states = encoder_hidden_states_1 + forward_encoder_attention_mask = encoder_attention_mask_1 + else: + forward_encoder_hidden_states = None + forward_encoder_attention_mask = None + hidden_states = self.attentions[i * num_attention_per_layer + idx]( + hidden_states, + attention_mask=attention_mask, + encoder_hidden_states=forward_encoder_hidden_states, + encoder_attention_mask=forward_encoder_attention_mask, + return_dict=False, + )[0] + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class UNetMidBlock2DCrossAttn(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads=1, + output_scale_factor=1.0, + cross_attention_dim=1280, + use_linear_projection=False, + upcast_attention=False, + ): + super().__init__() + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + + if isinstance(cross_attention_dim, int): + cross_attention_dim = (cross_attention_dim,) + if isinstance(cross_attention_dim, (list, tuple)) and len(cross_attention_dim) > 4: + raise ValueError( + "Only up to 4 cross-attention layers are supported. Ensure that the length of cross-attention " + f"dims is less than or equal to 4. Got cross-attention dims {cross_attention_dim} of length {len(cross_attention_dim)}" + ) + self.cross_attention_dim = cross_attention_dim + + # there is always at least one resnet + resnets = [ + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + + for i in range(num_layers): + for j in range(len(cross_attention_dim)): + attentions.append( + Transformer2DModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=transformer_layers_per_block, + cross_attention_dim=cross_attention_dim[j], + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + double_self_attention=True if cross_attention_dim[j] is None else False, + ) + ) + resnets.append( + ResnetBlock2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states_1: Optional[torch.FloatTensor] = None, + encoder_attention_mask_1: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + hidden_states = self.resnets[0](hidden_states, temb) + num_attention_per_layer = len(self.attentions) // (len(self.resnets) - 1) + + encoder_hidden_states_1 = ( + encoder_hidden_states_1 if encoder_hidden_states_1 is not None else encoder_hidden_states + ) + encoder_attention_mask_1 = ( + encoder_attention_mask_1 if encoder_hidden_states_1 is not None else encoder_attention_mask + ) + + for i in range(len(self.resnets[1:])): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + for idx, cross_attention_dim in enumerate(self.cross_attention_dim): + if cross_attention_dim is not None and idx <= 1: + forward_encoder_hidden_states = encoder_hidden_states + forward_encoder_attention_mask = encoder_attention_mask + elif cross_attention_dim is not None and idx > 1: + forward_encoder_hidden_states = encoder_hidden_states_1 + forward_encoder_attention_mask = encoder_attention_mask_1 + else: + forward_encoder_hidden_states = None + forward_encoder_attention_mask = None + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.attentions[i * num_attention_per_layer + idx], return_dict=False), + hidden_states, + forward_encoder_hidden_states, + None, # timestep + None, # class_labels + cross_attention_kwargs, + attention_mask, + forward_encoder_attention_mask, + **ckpt_kwargs, + )[0] + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.resnets[i + 1]), + hidden_states, + temb, + **ckpt_kwargs, + ) + else: + for idx, cross_attention_dim in enumerate(self.cross_attention_dim): + if cross_attention_dim is not None and idx <= 1: + forward_encoder_hidden_states = encoder_hidden_states + forward_encoder_attention_mask = encoder_attention_mask + elif cross_attention_dim is not None and idx > 1: + forward_encoder_hidden_states = encoder_hidden_states_1 + forward_encoder_attention_mask = encoder_attention_mask_1 + else: + forward_encoder_hidden_states = None + forward_encoder_attention_mask = None + hidden_states = self.attentions[i * num_attention_per_layer + idx]( + hidden_states, + attention_mask=attention_mask, + encoder_hidden_states=forward_encoder_hidden_states, + encoder_attention_mask=forward_encoder_attention_mask, + return_dict=False, + )[0] + + hidden_states = self.resnets[i + 1](hidden_states, temb) + + return hidden_states + + +class CrossAttnUpBlock2D(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads=1, + cross_attention_dim=1280, + output_scale_factor=1.0, + add_upsample=True, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + if isinstance(cross_attention_dim, int): + cross_attention_dim = (cross_attention_dim,) + if isinstance(cross_attention_dim, (list, tuple)) and len(cross_attention_dim) > 4: + raise ValueError( + "Only up to 4 cross-attention layers are supported. Ensure that the length of cross-attention " + f"dims is less than or equal to 4. Got cross-attention dims {cross_attention_dim} of length {len(cross_attention_dim)}" + ) + self.cross_attention_dim = cross_attention_dim + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlock2D( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + for j in range(len(cross_attention_dim)): + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block, + cross_attention_dim=cross_attention_dim[j], + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + double_self_attention=True if cross_attention_dim[j] is None else False, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([Upsample2D(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + encoder_hidden_states_1: Optional[torch.FloatTensor] = None, + encoder_attention_mask_1: Optional[torch.FloatTensor] = None, + ): + num_layers = len(self.resnets) + num_attention_per_layer = len(self.attentions) // num_layers + + encoder_hidden_states_1 = ( + encoder_hidden_states_1 if encoder_hidden_states_1 is not None else encoder_hidden_states + ) + encoder_attention_mask_1 = ( + encoder_attention_mask_1 if encoder_hidden_states_1 is not None else encoder_attention_mask + ) + + for i in range(num_layers): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.resnets[i]), + hidden_states, + temb, + **ckpt_kwargs, + ) + for idx, cross_attention_dim in enumerate(self.cross_attention_dim): + if cross_attention_dim is not None and idx <= 1: + forward_encoder_hidden_states = encoder_hidden_states + forward_encoder_attention_mask = encoder_attention_mask + elif cross_attention_dim is not None and idx > 1: + forward_encoder_hidden_states = encoder_hidden_states_1 + forward_encoder_attention_mask = encoder_attention_mask_1 + else: + forward_encoder_hidden_states = None + forward_encoder_attention_mask = None + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(self.attentions[i * num_attention_per_layer + idx], return_dict=False), + hidden_states, + forward_encoder_hidden_states, + None, # timestep + None, # class_labels + cross_attention_kwargs, + attention_mask, + forward_encoder_attention_mask, + **ckpt_kwargs, + )[0] + else: + hidden_states = self.resnets[i](hidden_states, temb) + for idx, cross_attention_dim in enumerate(self.cross_attention_dim): + if cross_attention_dim is not None and idx <= 1: + forward_encoder_hidden_states = encoder_hidden_states + forward_encoder_attention_mask = encoder_attention_mask + elif cross_attention_dim is not None and idx > 1: + forward_encoder_hidden_states = encoder_hidden_states_1 + forward_encoder_attention_mask = encoder_attention_mask_1 + else: + forward_encoder_hidden_states = None + forward_encoder_attention_mask = None + hidden_states = self.attentions[i * num_attention_per_layer + idx]( + hidden_states, + attention_mask=attention_mask, + encoder_hidden_states=forward_encoder_hidden_states, + encoder_attention_mask=forward_encoder_attention_mask, + return_dict=False, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states diff --git a/diffusers-0.27.0/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py b/diffusers-0.27.0/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py new file mode 100755 index 0000000..e01aa99 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/audioldm2/pipeline_audioldm2.py @@ -0,0 +1,980 @@ +# Copyright 2024 CVSSP, ByteDance and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import torch +from transformers import ( + ClapFeatureExtractor, + ClapModel, + GPT2Model, + RobertaTokenizer, + RobertaTokenizerFast, + SpeechT5HifiGan, + T5EncoderModel, + T5Tokenizer, + T5TokenizerFast, +) + +from ...models import AutoencoderKL +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + is_accelerate_available, + is_accelerate_version, + is_librosa_available, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import AudioPipelineOutput, DiffusionPipeline +from .modeling_audioldm2 import AudioLDM2ProjectionModel, AudioLDM2UNet2DConditionModel + + +if is_librosa_available(): + import librosa + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import scipy + >>> import torch + >>> from diffusers import AudioLDM2Pipeline + + >>> repo_id = "cvssp/audioldm2" + >>> pipe = AudioLDM2Pipeline.from_pretrained(repo_id, torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + + >>> # define the prompts + >>> prompt = "The sound of a hammer hitting a wooden surface." + >>> negative_prompt = "Low quality." + + >>> # set the seed for generator + >>> generator = torch.Generator("cuda").manual_seed(0) + + >>> # run the generation + >>> audio = pipe( + ... prompt, + ... negative_prompt=negative_prompt, + ... num_inference_steps=200, + ... audio_length_in_s=10.0, + ... num_waveforms_per_prompt=3, + ... generator=generator, + ... ).audios + + >>> # save the best audio sample (index 0) as a .wav file + >>> scipy.io.wavfile.write("techno.wav", rate=16000, data=audio[0]) + ``` +""" + + +def prepare_inputs_for_generation( + inputs_embeds, + attention_mask=None, + past_key_values=None, + **kwargs, +): + if past_key_values is not None: + # only last token for inputs_embeds if past is defined in kwargs + inputs_embeds = inputs_embeds[:, -1:] + + return { + "inputs_embeds": inputs_embeds, + "attention_mask": attention_mask, + "past_key_values": past_key_values, + "use_cache": kwargs.get("use_cache"), + } + + +class AudioLDM2Pipeline(DiffusionPipeline): + r""" + Pipeline for text-to-audio generation using AudioLDM2. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.ClapModel`]): + First frozen text-encoder. AudioLDM2 uses the joint audio-text embedding model + [CLAP](https://huggingface.co/docs/transformers/model_doc/clap#transformers.CLAPTextModelWithProjection), + specifically the [laion/clap-htsat-unfused](https://huggingface.co/laion/clap-htsat-unfused) variant. The + text branch is used to encode the text prompt to a prompt embedding. The full audio-text model is used to + rank generated waveforms against the text prompt by computing similarity scores. + text_encoder_2 ([`~transformers.T5EncoderModel`]): + Second frozen text-encoder. AudioLDM2 uses the encoder of + [T5](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5EncoderModel), specifically the + [google/flan-t5-large](https://huggingface.co/google/flan-t5-large) variant. + projection_model ([`AudioLDM2ProjectionModel`]): + A trained model used to linearly project the hidden-states from the first and second text encoder models + and insert learned SOS and EOS token embeddings. The projected hidden-states from the two text encoders are + concatenated to give the input to the language model. + language_model ([`~transformers.GPT2Model`]): + An auto-regressive language model used to generate a sequence of hidden-states conditioned on the projected + outputs from the two text encoders. + tokenizer ([`~transformers.RobertaTokenizer`]): + Tokenizer to tokenize text for the first frozen text-encoder. + tokenizer_2 ([`~transformers.T5Tokenizer`]): + Tokenizer to tokenize text for the second frozen text-encoder. + feature_extractor ([`~transformers.ClapFeatureExtractor`]): + Feature extractor to pre-process generated audio waveforms to log-mel spectrograms for automatic scoring. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded audio latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded audio latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + vocoder ([`~transformers.SpeechT5HifiGan`]): + Vocoder of class `SpeechT5HifiGan` to convert the mel-spectrogram latents to the final audio waveform. + """ + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: ClapModel, + text_encoder_2: T5EncoderModel, + projection_model: AudioLDM2ProjectionModel, + language_model: GPT2Model, + tokenizer: Union[RobertaTokenizer, RobertaTokenizerFast], + tokenizer_2: Union[T5Tokenizer, T5TokenizerFast], + feature_extractor: ClapFeatureExtractor, + unet: AudioLDM2UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + vocoder: SpeechT5HifiGan, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + projection_model=projection_model, + language_model=language_model, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + feature_extractor=feature_extractor, + unet=unet, + scheduler=scheduler, + vocoder=vocoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + + # Copied from diffusers.pipelines.pipeline_utils.StableDiffusionMixin.enable_vae_slicing + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + # Copied from diffusers.pipelines.pipeline_utils.StableDiffusionMixin.disable_vae_slicing + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + def enable_model_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared + to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` + method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with + `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`. + """ + if is_accelerate_available() and is_accelerate_version(">=", "0.17.0.dev0"): + from accelerate import cpu_offload_with_hook + else: + raise ImportError("`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.") + + device = torch.device(f"cuda:{gpu_id}") + + if self.device.type != "cpu": + self.to("cpu", silence_dtype_warnings=True) + torch.cuda.empty_cache() # otherwise we don't see the memory savings (but they probably exist) + + model_sequence = [ + self.text_encoder.text_model, + self.text_encoder.text_projection, + self.text_encoder_2, + self.projection_model, + self.language_model, + self.unet, + self.vae, + self.vocoder, + self.text_encoder, + ] + + hook = None + for cpu_offloaded_model in model_sequence: + _, hook = cpu_offload_with_hook(cpu_offloaded_model, device, prev_module_hook=hook) + + # We'll offload the last model manually. + self.final_offload_hook = hook + + def generate_language_model( + self, + inputs_embeds: torch.Tensor = None, + max_new_tokens: int = 8, + **model_kwargs, + ): + """ + + Generates a sequence of hidden-states from the language model, conditioned on the embedding inputs. + + Parameters: + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + The sequence used as a prompt for the generation. + max_new_tokens (`int`): + Number of new tokens to generate. + model_kwargs (`Dict[str, Any]`, *optional*): + Ad hoc parametrization of additional model-specific kwargs that will be forwarded to the `forward` + function of the model. + + Return: + `inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + The sequence of generated hidden-states. + """ + max_new_tokens = max_new_tokens if max_new_tokens is not None else self.language_model.config.max_new_tokens + for _ in range(max_new_tokens): + # prepare model inputs + model_inputs = prepare_inputs_for_generation(inputs_embeds, **model_kwargs) + + # forward pass to get next hidden states + output = self.language_model(**model_inputs, return_dict=True) + + next_hidden_states = output.last_hidden_state + + # Update the model input + inputs_embeds = torch.cat([inputs_embeds, next_hidden_states[:, -1:, :]], dim=1) + + # Update generated hidden states, model inputs, and length for next step + model_kwargs = self.language_model._update_model_kwargs_for_generation(output, model_kwargs) + + return inputs_embeds[:, -max_new_tokens:, :] + + def encode_prompt( + self, + prompt, + device, + num_waveforms_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + generated_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_generated_prompt_embeds: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + negative_attention_mask: Optional[torch.LongTensor] = None, + max_new_tokens: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device (`torch.device`): + torch device + num_waveforms_per_prompt (`int`): + number of waveforms that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the audio generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-computed text embeddings from the Flan T5 model. Can be used to easily tweak text inputs, *e.g.* + prompt weighting. If not provided, text embeddings will be computed from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-computed negative text embeddings from the Flan T5 model. Can be used to easily tweak text inputs, + *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be computed from + `negative_prompt` input argument. + generated_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings from the GPT2 langauge model. Can be used to easily tweak text inputs, + *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input + argument. + negative_generated_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings from the GPT2 language model. Can be used to easily tweak text + inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be computed from + `negative_prompt` input argument. + attention_mask (`torch.LongTensor`, *optional*): + Pre-computed attention mask to be applied to the `prompt_embeds`. If not provided, attention mask will + be computed from `prompt` input argument. + negative_attention_mask (`torch.LongTensor`, *optional*): + Pre-computed attention mask to be applied to the `negative_prompt_embeds`. If not provided, attention + mask will be computed from `negative_prompt` input argument. + max_new_tokens (`int`, *optional*, defaults to None): + The number of new tokens to generate with the GPT2 language model. + Returns: + prompt_embeds (`torch.FloatTensor`): + Text embeddings from the Flan T5 model. + attention_mask (`torch.LongTensor`): + Attention mask to be applied to the `prompt_embeds`. + generated_prompt_embeds (`torch.FloatTensor`): + Text embeddings generated from the GPT2 langauge model. + + Example: + + ```python + >>> import scipy + >>> import torch + >>> from diffusers import AudioLDM2Pipeline + + >>> repo_id = "cvssp/audioldm2" + >>> pipe = AudioLDM2Pipeline.from_pretrained(repo_id, torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + + >>> # Get text embedding vectors + >>> prompt_embeds, attention_mask, generated_prompt_embeds = pipe.encode_prompt( + ... prompt="Techno music with a strong, upbeat tempo and high melodic riffs", + ... device="cuda", + ... do_classifier_free_guidance=True, + ... ) + + >>> # Pass text embeddings to pipeline for text-conditional audio generation + >>> audio = pipe( + ... prompt_embeds=prompt_embeds, + ... attention_mask=attention_mask, + ... generated_prompt_embeds=generated_prompt_embeds, + ... num_inference_steps=200, + ... audio_length_in_s=10.0, + ... ).audios[0] + + >>> # save generated audio sample + >>> scipy.io.wavfile.write("techno.wav", rate=16000, data=audio) + ```""" + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] + text_encoders = [self.text_encoder, self.text_encoder_2] + + if prompt_embeds is None: + prompt_embeds_list = [] + attention_mask_list = [] + + for tokenizer, text_encoder in zip(tokenizers, text_encoders): + text_inputs = tokenizer( + prompt, + padding="max_length" if isinstance(tokenizer, (RobertaTokenizer, RobertaTokenizerFast)) else True, + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + attention_mask = text_inputs.attention_mask + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + f"The following part of your input was truncated because {text_encoder.config.model_type} can " + f"only handle sequences up to {tokenizer.model_max_length} tokens: {removed_text}" + ) + + text_input_ids = text_input_ids.to(device) + attention_mask = attention_mask.to(device) + + if text_encoder.config.model_type == "clap": + prompt_embeds = text_encoder.get_text_features( + text_input_ids, + attention_mask=attention_mask, + ) + # append the seq-len dim: (bs, hidden_size) -> (bs, seq_len, hidden_size) + prompt_embeds = prompt_embeds[:, None, :] + # make sure that we attend to this single hidden-state + attention_mask = attention_mask.new_ones((batch_size, 1)) + else: + prompt_embeds = text_encoder( + text_input_ids, + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + + prompt_embeds_list.append(prompt_embeds) + attention_mask_list.append(attention_mask) + + projection_output = self.projection_model( + hidden_states=prompt_embeds_list[0], + hidden_states_1=prompt_embeds_list[1], + attention_mask=attention_mask_list[0], + attention_mask_1=attention_mask_list[1], + ) + projected_prompt_embeds = projection_output.hidden_states + projected_attention_mask = projection_output.attention_mask + + generated_prompt_embeds = self.generate_language_model( + projected_prompt_embeds, + attention_mask=projected_attention_mask, + max_new_tokens=max_new_tokens, + ) + + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + attention_mask = ( + attention_mask.to(device=device) + if attention_mask is not None + else torch.ones(prompt_embeds.shape[:2], dtype=torch.long, device=device) + ) + generated_prompt_embeds = generated_prompt_embeds.to(dtype=self.language_model.dtype, device=device) + + bs_embed, seq_len, hidden_size = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_waveforms_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_waveforms_per_prompt, seq_len, hidden_size) + + # duplicate attention mask for each generation per prompt + attention_mask = attention_mask.repeat(1, num_waveforms_per_prompt) + attention_mask = attention_mask.view(bs_embed * num_waveforms_per_prompt, seq_len) + + bs_embed, seq_len, hidden_size = generated_prompt_embeds.shape + # duplicate generated embeddings for each generation per prompt, using mps friendly method + generated_prompt_embeds = generated_prompt_embeds.repeat(1, num_waveforms_per_prompt, 1) + generated_prompt_embeds = generated_prompt_embeds.view( + bs_embed * num_waveforms_per_prompt, seq_len, hidden_size + ) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + negative_prompt_embeds_list = [] + negative_attention_mask_list = [] + max_length = prompt_embeds.shape[1] + for tokenizer, text_encoder in zip(tokenizers, text_encoders): + uncond_input = tokenizer( + uncond_tokens, + padding="max_length", + max_length=tokenizer.model_max_length + if isinstance(tokenizer, (RobertaTokenizer, RobertaTokenizerFast)) + else max_length, + truncation=True, + return_tensors="pt", + ) + + uncond_input_ids = uncond_input.input_ids.to(device) + negative_attention_mask = uncond_input.attention_mask.to(device) + + if text_encoder.config.model_type == "clap": + negative_prompt_embeds = text_encoder.get_text_features( + uncond_input_ids, + attention_mask=negative_attention_mask, + ) + # append the seq-len dim: (bs, hidden_size) -> (bs, seq_len, hidden_size) + negative_prompt_embeds = negative_prompt_embeds[:, None, :] + # make sure that we attend to this single hidden-state + negative_attention_mask = negative_attention_mask.new_ones((batch_size, 1)) + else: + negative_prompt_embeds = text_encoder( + uncond_input_ids, + attention_mask=negative_attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + negative_attention_mask_list.append(negative_attention_mask) + + projection_output = self.projection_model( + hidden_states=negative_prompt_embeds_list[0], + hidden_states_1=negative_prompt_embeds_list[1], + attention_mask=negative_attention_mask_list[0], + attention_mask_1=negative_attention_mask_list[1], + ) + negative_projected_prompt_embeds = projection_output.hidden_states + negative_projected_attention_mask = projection_output.attention_mask + + negative_generated_prompt_embeds = self.generate_language_model( + negative_projected_prompt_embeds, + attention_mask=negative_projected_attention_mask, + max_new_tokens=max_new_tokens, + ) + + if do_classifier_free_guidance: + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + negative_attention_mask = ( + negative_attention_mask.to(device=device) + if negative_attention_mask is not None + else torch.ones(negative_prompt_embeds.shape[:2], dtype=torch.long, device=device) + ) + negative_generated_prompt_embeds = negative_generated_prompt_embeds.to( + dtype=self.language_model.dtype, device=device + ) + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_waveforms_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_waveforms_per_prompt, seq_len, -1) + + # duplicate unconditional attention mask for each generation per prompt + negative_attention_mask = negative_attention_mask.repeat(1, num_waveforms_per_prompt) + negative_attention_mask = negative_attention_mask.view(batch_size * num_waveforms_per_prompt, seq_len) + + # duplicate unconditional generated embeddings for each generation per prompt + seq_len = negative_generated_prompt_embeds.shape[1] + negative_generated_prompt_embeds = negative_generated_prompt_embeds.repeat(1, num_waveforms_per_prompt, 1) + negative_generated_prompt_embeds = negative_generated_prompt_embeds.view( + batch_size * num_waveforms_per_prompt, seq_len, -1 + ) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + attention_mask = torch.cat([negative_attention_mask, attention_mask]) + generated_prompt_embeds = torch.cat([negative_generated_prompt_embeds, generated_prompt_embeds]) + + return prompt_embeds, attention_mask, generated_prompt_embeds + + # Copied from diffusers.pipelines.audioldm.pipeline_audioldm.AudioLDMPipeline.mel_spectrogram_to_waveform + def mel_spectrogram_to_waveform(self, mel_spectrogram): + if mel_spectrogram.dim() == 4: + mel_spectrogram = mel_spectrogram.squeeze(1) + + waveform = self.vocoder(mel_spectrogram) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + waveform = waveform.cpu().float() + return waveform + + def score_waveforms(self, text, audio, num_waveforms_per_prompt, device, dtype): + if not is_librosa_available(): + logger.info( + "Automatic scoring of the generated audio waveforms against the input prompt text requires the " + "`librosa` package to resample the generated waveforms. Returning the audios in the order they were " + "generated. To enable automatic scoring, install `librosa` with: `pip install librosa`." + ) + return audio + inputs = self.tokenizer(text, return_tensors="pt", padding=True) + resampled_audio = librosa.resample( + audio.numpy(), orig_sr=self.vocoder.config.sampling_rate, target_sr=self.feature_extractor.sampling_rate + ) + inputs["input_features"] = self.feature_extractor( + list(resampled_audio), return_tensors="pt", sampling_rate=self.feature_extractor.sampling_rate + ).input_features.type(dtype) + inputs = inputs.to(device) + + # compute the audio-text similarity score using the CLAP model + logits_per_text = self.text_encoder(**inputs).logits_per_text + # sort by the highest matching generations per prompt + indices = torch.argsort(logits_per_text, dim=1, descending=True)[:, :num_waveforms_per_prompt] + audio = torch.index_select(audio, 0, indices.reshape(-1).cpu()) + return audio + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + audio_length_in_s, + vocoder_upsample_factor, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + generated_prompt_embeds=None, + negative_generated_prompt_embeds=None, + attention_mask=None, + negative_attention_mask=None, + ): + min_audio_length_in_s = vocoder_upsample_factor * self.vae_scale_factor + if audio_length_in_s < min_audio_length_in_s: + raise ValueError( + f"`audio_length_in_s` has to be a positive value greater than or equal to {min_audio_length_in_s}, but " + f"is {audio_length_in_s}." + ) + + if self.vocoder.config.model_in_dim % self.vae_scale_factor != 0: + raise ValueError( + f"The number of frequency bins in the vocoder's log-mel spectrogram has to be divisible by the " + f"VAE scale factor, but got {self.vocoder.config.model_in_dim} bins and a scale factor of " + f"{self.vae_scale_factor}." + ) + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and (prompt_embeds is None or generated_prompt_embeds is None): + raise ValueError( + "Provide either `prompt`, or `prompt_embeds` and `generated_prompt_embeds`. Cannot leave " + "`prompt` undefined without specifying both `prompt_embeds` and `generated_prompt_embeds`." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_embeds is not None and negative_generated_prompt_embeds is None: + raise ValueError( + "Cannot forward `negative_prompt_embeds` without `negative_generated_prompt_embeds`. Ensure that" + "both arguments are specified" + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + if attention_mask is not None and attention_mask.shape != prompt_embeds.shape[:2]: + raise ValueError( + "`attention_mask should have the same batch size and sequence length as `prompt_embeds`, but got:" + f"`attention_mask: {attention_mask.shape} != `prompt_embeds` {prompt_embeds.shape}" + ) + + if generated_prompt_embeds is not None and negative_generated_prompt_embeds is not None: + if generated_prompt_embeds.shape != negative_generated_prompt_embeds.shape: + raise ValueError( + "`generated_prompt_embeds` and `negative_generated_prompt_embeds` must have the same shape when " + f"passed directly, but got: `generated_prompt_embeds` {generated_prompt_embeds.shape} != " + f"`negative_generated_prompt_embeds` {negative_generated_prompt_embeds.shape}." + ) + if ( + negative_attention_mask is not None + and negative_attention_mask.shape != negative_prompt_embeds.shape[:2] + ): + raise ValueError( + "`attention_mask should have the same batch size and sequence length as `prompt_embeds`, but got:" + f"`attention_mask: {negative_attention_mask.shape} != `prompt_embeds` {negative_prompt_embeds.shape}" + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents with width->self.vocoder.config.model_in_dim + def prepare_latents(self, batch_size, num_channels_latents, height, dtype, device, generator, latents=None): + shape = ( + batch_size, + num_channels_latents, + height // self.vae_scale_factor, + self.vocoder.config.model_in_dim // self.vae_scale_factor, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + audio_length_in_s: Optional[float] = None, + num_inference_steps: int = 200, + guidance_scale: float = 3.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_waveforms_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + generated_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_generated_prompt_embeds: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.LongTensor] = None, + negative_attention_mask: Optional[torch.LongTensor] = None, + max_new_tokens: Optional[int] = None, + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: Optional[int] = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + output_type: Optional[str] = "np", + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide audio generation. If not defined, you need to pass `prompt_embeds`. + audio_length_in_s (`int`, *optional*, defaults to 10.24): + The length of the generated audio sample in seconds. + num_inference_steps (`int`, *optional*, defaults to 200): + The number of denoising steps. More denoising steps usually lead to a higher quality audio at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 3.5): + A higher guidance scale value encourages the model to generate audio that is closely linked to the text + `prompt` at the expense of lower sound quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in audio generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_waveforms_per_prompt (`int`, *optional*, defaults to 1): + The number of waveforms to generate per prompt. If `num_waveforms_per_prompt > 1`, then automatic + scoring is performed between the generated outputs and the text prompt. This scoring ranks the + generated waveforms based on their cosine similarity with the text input in the joint text-audio + embedding space. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for spectrogram + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + generated_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings from the GPT2 langauge model. Can be used to easily tweak text inputs, + *e.g.* prompt weighting. If not provided, text embeddings will be generated from `prompt` input + argument. + negative_generated_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings from the GPT2 language model. Can be used to easily tweak text + inputs, *e.g.* prompt weighting. If not provided, negative_prompt_embeds will be computed from + `negative_prompt` input argument. + attention_mask (`torch.LongTensor`, *optional*): + Pre-computed attention mask to be applied to the `prompt_embeds`. If not provided, attention mask will + be computed from `prompt` input argument. + negative_attention_mask (`torch.LongTensor`, *optional*): + Pre-computed attention mask to be applied to the `negative_prompt_embeds`. If not provided, attention + mask will be computed from `negative_prompt` input argument. + max_new_tokens (`int`, *optional*, defaults to None): + Number of new tokens to generate with the GPT2 language model. If not provided, number of tokens will + be taken from the config of the model. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + output_type (`str`, *optional*, defaults to `"np"`): + The output format of the generated audio. Choose between `"np"` to return a NumPy `np.ndarray` or + `"pt"` to return a PyTorch `torch.Tensor` object. Set to `"latent"` to return the latent diffusion + model (LDM) output. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated audio. + """ + # 0. Convert audio input length from seconds to spectrogram height + vocoder_upsample_factor = np.prod(self.vocoder.config.upsample_rates) / self.vocoder.config.sampling_rate + + if audio_length_in_s is None: + audio_length_in_s = self.unet.config.sample_size * self.vae_scale_factor * vocoder_upsample_factor + + height = int(audio_length_in_s / vocoder_upsample_factor) + + original_waveform_length = int(audio_length_in_s * self.vocoder.config.sampling_rate) + if height % self.vae_scale_factor != 0: + height = int(np.ceil(height / self.vae_scale_factor)) * self.vae_scale_factor + logger.info( + f"Audio length in seconds {audio_length_in_s} is increased to {height * vocoder_upsample_factor} " + f"so that it can be handled by the model. It will be cut to {audio_length_in_s} after the " + f"denoising process." + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + audio_length_in_s, + vocoder_upsample_factor, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + generated_prompt_embeds, + negative_generated_prompt_embeds, + attention_mask, + negative_attention_mask, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, attention_mask, generated_prompt_embeds = self.encode_prompt( + prompt, + device, + num_waveforms_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + generated_prompt_embeds=generated_prompt_embeds, + negative_generated_prompt_embeds=negative_generated_prompt_embeds, + attention_mask=attention_mask, + negative_attention_mask=negative_attention_mask, + max_new_tokens=max_new_tokens, + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_waveforms_per_prompt, + num_channels_latents, + height, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=generated_prompt_embeds, + encoder_hidden_states_1=prompt_embeds, + encoder_attention_mask_1=attention_mask, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + self.maybe_free_model_hooks() + + # 8. Post-processing + if not output_type == "latent": + latents = 1 / self.vae.config.scaling_factor * latents + mel_spectrogram = self.vae.decode(latents).sample + else: + return AudioPipelineOutput(audios=latents) + + audio = self.mel_spectrogram_to_waveform(mel_spectrogram) + + audio = audio[:, :original_waveform_length] + + # 9. Automatic scoring + if num_waveforms_per_prompt > 1 and prompt is not None: + audio = self.score_waveforms( + text=prompt, + audio=audio, + num_waveforms_per_prompt=num_waveforms_per_prompt, + device=device, + dtype=prompt_embeds.dtype, + ) + + if output_type == "np": + audio = audio.numpy() + + if not return_dict: + return (audio,) + + return AudioPipelineOutput(audios=audio) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/auto_pipeline.py b/diffusers-0.27.0/src/diffusers/pipelines/auto_pipeline.py new file mode 100755 index 0000000..fc30fc4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/auto_pipeline.py @@ -0,0 +1,987 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import OrderedDict + +from huggingface_hub.utils import validate_hf_hub_args + +from ..configuration_utils import ConfigMixin +from .controlnet import ( + StableDiffusionControlNetImg2ImgPipeline, + StableDiffusionControlNetInpaintPipeline, + StableDiffusionControlNetPipeline, + StableDiffusionXLControlNetImg2ImgPipeline, + StableDiffusionXLControlNetInpaintPipeline, + StableDiffusionXLControlNetPipeline, +) +from .deepfloyd_if import IFImg2ImgPipeline, IFInpaintingPipeline, IFPipeline +from .kandinsky import ( + KandinskyCombinedPipeline, + KandinskyImg2ImgCombinedPipeline, + KandinskyImg2ImgPipeline, + KandinskyInpaintCombinedPipeline, + KandinskyInpaintPipeline, + KandinskyPipeline, +) +from .kandinsky2_2 import ( + KandinskyV22CombinedPipeline, + KandinskyV22Img2ImgCombinedPipeline, + KandinskyV22Img2ImgPipeline, + KandinskyV22InpaintCombinedPipeline, + KandinskyV22InpaintPipeline, + KandinskyV22Pipeline, +) +from .kandinsky3 import Kandinsky3Img2ImgPipeline, Kandinsky3Pipeline +from .latent_consistency_models import LatentConsistencyModelImg2ImgPipeline, LatentConsistencyModelPipeline +from .pixart_alpha import PixArtAlphaPipeline +from .stable_diffusion import ( + StableDiffusionImg2ImgPipeline, + StableDiffusionInpaintPipeline, + StableDiffusionPipeline, +) +from .stable_diffusion_xl import ( + StableDiffusionXLImg2ImgPipeline, + StableDiffusionXLInpaintPipeline, + StableDiffusionXLPipeline, +) +from .wuerstchen import WuerstchenCombinedPipeline, WuerstchenDecoderPipeline + + +AUTO_TEXT2IMAGE_PIPELINES_MAPPING = OrderedDict( + [ + ("stable-diffusion", StableDiffusionPipeline), + ("stable-diffusion-xl", StableDiffusionXLPipeline), + ("if", IFPipeline), + ("kandinsky", KandinskyCombinedPipeline), + ("kandinsky22", KandinskyV22CombinedPipeline), + ("kandinsky3", Kandinsky3Pipeline), + ("stable-diffusion-controlnet", StableDiffusionControlNetPipeline), + ("stable-diffusion-xl-controlnet", StableDiffusionXLControlNetPipeline), + ("wuerstchen", WuerstchenCombinedPipeline), + ("lcm", LatentConsistencyModelPipeline), + ("pixart", PixArtAlphaPipeline), + ] +) + +AUTO_IMAGE2IMAGE_PIPELINES_MAPPING = OrderedDict( + [ + ("stable-diffusion", StableDiffusionImg2ImgPipeline), + ("stable-diffusion-xl", StableDiffusionXLImg2ImgPipeline), + ("if", IFImg2ImgPipeline), + ("kandinsky", KandinskyImg2ImgCombinedPipeline), + ("kandinsky22", KandinskyV22Img2ImgCombinedPipeline), + ("kandinsky3", Kandinsky3Img2ImgPipeline), + ("stable-diffusion-controlnet", StableDiffusionControlNetImg2ImgPipeline), + ("stable-diffusion-xl-controlnet", StableDiffusionXLControlNetImg2ImgPipeline), + ("lcm", LatentConsistencyModelImg2ImgPipeline), + ] +) + +AUTO_INPAINT_PIPELINES_MAPPING = OrderedDict( + [ + ("stable-diffusion", StableDiffusionInpaintPipeline), + ("stable-diffusion-xl", StableDiffusionXLInpaintPipeline), + ("if", IFInpaintingPipeline), + ("kandinsky", KandinskyInpaintCombinedPipeline), + ("kandinsky22", KandinskyV22InpaintCombinedPipeline), + ("stable-diffusion-controlnet", StableDiffusionControlNetInpaintPipeline), + ("stable-diffusion-xl-controlnet", StableDiffusionXLControlNetInpaintPipeline), + ] +) + +_AUTO_TEXT2IMAGE_DECODER_PIPELINES_MAPPING = OrderedDict( + [ + ("kandinsky", KandinskyPipeline), + ("kandinsky22", KandinskyV22Pipeline), + ("wuerstchen", WuerstchenDecoderPipeline), + ] +) +_AUTO_IMAGE2IMAGE_DECODER_PIPELINES_MAPPING = OrderedDict( + [ + ("kandinsky", KandinskyImg2ImgPipeline), + ("kandinsky22", KandinskyV22Img2ImgPipeline), + ] +) +_AUTO_INPAINT_DECODER_PIPELINES_MAPPING = OrderedDict( + [ + ("kandinsky", KandinskyInpaintPipeline), + ("kandinsky22", KandinskyV22InpaintPipeline), + ] +) + +SUPPORTED_TASKS_MAPPINGS = [ + AUTO_TEXT2IMAGE_PIPELINES_MAPPING, + AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, + AUTO_INPAINT_PIPELINES_MAPPING, + _AUTO_TEXT2IMAGE_DECODER_PIPELINES_MAPPING, + _AUTO_IMAGE2IMAGE_DECODER_PIPELINES_MAPPING, + _AUTO_INPAINT_DECODER_PIPELINES_MAPPING, +] + + +def _get_connected_pipeline(pipeline_cls): + # for now connected pipelines can only be loaded from decoder pipelines, such as kandinsky-community/kandinsky-2-2-decoder + if pipeline_cls in _AUTO_TEXT2IMAGE_DECODER_PIPELINES_MAPPING.values(): + return _get_task_class( + AUTO_TEXT2IMAGE_PIPELINES_MAPPING, pipeline_cls.__name__, throw_error_if_not_exist=False + ) + if pipeline_cls in _AUTO_IMAGE2IMAGE_DECODER_PIPELINES_MAPPING.values(): + return _get_task_class( + AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, pipeline_cls.__name__, throw_error_if_not_exist=False + ) + if pipeline_cls in _AUTO_INPAINT_DECODER_PIPELINES_MAPPING.values(): + return _get_task_class(AUTO_INPAINT_PIPELINES_MAPPING, pipeline_cls.__name__, throw_error_if_not_exist=False) + + +def _get_task_class(mapping, pipeline_class_name, throw_error_if_not_exist: bool = True): + def get_model(pipeline_class_name): + for task_mapping in SUPPORTED_TASKS_MAPPINGS: + for model_name, pipeline in task_mapping.items(): + if pipeline.__name__ == pipeline_class_name: + return model_name + + model_name = get_model(pipeline_class_name) + + if model_name is not None: + task_class = mapping.get(model_name, None) + if task_class is not None: + return task_class + + if throw_error_if_not_exist: + raise ValueError(f"AutoPipeline can't find a pipeline linked to {pipeline_class_name} for {model_name}") + + +class AutoPipelineForText2Image(ConfigMixin): + r""" + + [`AutoPipelineForText2Image`] is a generic pipeline class that instantiates a text-to-image pipeline class. The + specific underlying pipeline class is automatically selected from either the + [`~AutoPipelineForText2Image.from_pretrained`] or [`~AutoPipelineForText2Image.from_pipe`] methods. + + This class cannot be instantiated using `__init__()` (throws an error). + + Class attributes: + + - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the + diffusion pipeline's components. + + """ + + config_name = "model_index.json" + + def __init__(self, *args, **kwargs): + raise EnvironmentError( + f"{self.__class__.__name__} is designed to be instantiated " + f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or " + f"`{self.__class__.__name__}.from_pipe(pipeline)` methods." + ) + + @classmethod + @validate_hf_hub_args + def from_pretrained(cls, pretrained_model_or_path, **kwargs): + r""" + Instantiates a text-to-image Pytorch diffusion pipeline from pretrained pipeline weight. + + The from_pretrained() method takes care of returning the correct pipeline class instance by: + 1. Detect the pipeline class of the pretrained_model_or_path based on the _class_name property of its + config object + 2. Find the text-to-image pipeline linked to the pipeline class using pattern matching on pipeline class + name. + + If a `controlnet` argument is passed, it will instantiate a [`StableDiffusionControlNetPipeline`] object. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + If you get the error message below, you need to finetune the weights for your downstream task: + + ``` + Some weights of UNet2DConditionModel were not initialized from the model checkpoint at runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated + You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. + ``` + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *repo id* (for example `CompVis/ldm-text2im-large-256`) of a pretrained pipeline + hosted on the Hub. + - A path to a *directory* (for example `./my_pipeline_directory/`) containing pipeline weights + saved using + [`~DiffusionPipeline.save_pretrained`]. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the + dtype is automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + custom_revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id similar to + `revision` when loading a custom pipeline from the Hub. It can be a 🤗 Diffusers version when loading a + custom pipeline from GitHub, otherwise it defaults to `"main"` when loading from the Hub. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn’t need to be defined for each + parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the + same device. + + Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier for the maximum memory. Will default to the maximum memory available for + each GPU and the available CPU RAM if unset. + offload_folder (`str` or `os.PathLike`, *optional*): + The path to offload weights if device_map contains the value `"disk"`. + offload_state_dict (`bool`, *optional*): + If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if + the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` + when there is some disk offload. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the safetensors weights are downloaded if they're available **and** if the + safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors + weights. If set to `False`, safetensors weights are not loaded. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline + class). The overwritten components are passed directly to the pipelines `__init__` method. See example + below for more information. + variant (`str`, *optional*): + Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when + loading `from_flax`. + + + + To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with + `huggingface-cli login`. + + + + Examples: + + ```py + >>> from diffusers import AutoPipelineForText2Image + + >>> pipeline = AutoPipelineForText2Image.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> image = pipeline(prompt).images[0] + ``` + """ + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + + load_config_kwargs = { + "cache_dir": cache_dir, + "force_download": force_download, + "resume_download": resume_download, + "proxies": proxies, + "token": token, + "local_files_only": local_files_only, + "revision": revision, + } + + config = cls.load_config(pretrained_model_or_path, **load_config_kwargs) + orig_class_name = config["_class_name"] + + if "controlnet" in kwargs: + orig_class_name = config["_class_name"].replace("Pipeline", "ControlNetPipeline") + + text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, orig_class_name) + + kwargs = {**load_config_kwargs, **kwargs} + return text_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) + + @classmethod + def from_pipe(cls, pipeline, **kwargs): + r""" + Instantiates a text-to-image Pytorch diffusion pipeline from another instantiated diffusion pipeline class. + + The from_pipe() method takes care of returning the correct pipeline class instance by finding the text-to-image + pipeline linked to the pipeline class using pattern matching on pipeline class name. + + All the modules the pipeline contains will be used to initialize the new pipeline without reallocating + additional memory. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pipeline (`DiffusionPipeline`): + an instantiated `DiffusionPipeline` object + + ```py + >>> from diffusers import AutoPipelineForText2Image, AutoPipelineForImage2Image + + >>> pipe_i2i = AutoPipelineForImage2Image.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", requires_safety_checker=False + ... ) + + >>> pipe_t2i = AutoPipelineForText2Image.from_pipe(pipe_i2i) + >>> image = pipe_t2i(prompt).images[0] + ``` + """ + + original_config = dict(pipeline.config) + original_cls_name = pipeline.__class__.__name__ + + # derive the pipeline class to instantiate + text_2_image_cls = _get_task_class(AUTO_TEXT2IMAGE_PIPELINES_MAPPING, original_cls_name) + + if "controlnet" in kwargs: + if kwargs["controlnet"] is not None: + text_2_image_cls = _get_task_class( + AUTO_TEXT2IMAGE_PIPELINES_MAPPING, + text_2_image_cls.__name__.replace("ControlNet", "").replace("Pipeline", "ControlNetPipeline"), + ) + else: + text_2_image_cls = _get_task_class( + AUTO_TEXT2IMAGE_PIPELINES_MAPPING, + text_2_image_cls.__name__.replace("ControlNetPipeline", "Pipeline"), + ) + + # define expected module and optional kwargs given the pipeline signature + expected_modules, optional_kwargs = text_2_image_cls._get_signature_keys(text_2_image_cls) + + pretrained_model_name_or_path = original_config.pop("_name_or_path", None) + + # allow users pass modules in `kwargs` to override the original pipeline's components + passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} + original_class_obj = { + k: pipeline.components[k] + for k, v in pipeline.components.items() + if k in expected_modules and k not in passed_class_obj + } + + # allow users pass optional kwargs to override the original pipelines config attribute + passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} + original_pipe_kwargs = { + k: original_config[k] + for k, v in original_config.items() + if k in optional_kwargs and k not in passed_pipe_kwargs + } + + # config that were not expected by original pipeline is stored as private attribute + # we will pass them as optional arguments if they can be accepted by the pipeline + additional_pipe_kwargs = [ + k[1:] + for k in original_config.keys() + if k.startswith("_") and k[1:] in optional_kwargs and k[1:] not in passed_pipe_kwargs + ] + for k in additional_pipe_kwargs: + original_pipe_kwargs[k] = original_config.pop(f"_{k}") + + text_2_image_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} + + # store unused config as private attribute + unused_original_config = { + f"{'' if k.startswith('_') else '_'}{k}": original_config[k] + for k, v in original_config.items() + if k not in text_2_image_kwargs + } + + missing_modules = set(expected_modules) - set(pipeline._optional_components) - set(text_2_image_kwargs.keys()) + + if len(missing_modules) > 0: + raise ValueError( + f"Pipeline {text_2_image_cls} expected {expected_modules}, but only {set(list(passed_class_obj.keys()) + list(original_class_obj.keys()))} were passed" + ) + + model = text_2_image_cls(**text_2_image_kwargs) + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + model.register_to_config(**unused_original_config) + + return model + + +class AutoPipelineForImage2Image(ConfigMixin): + r""" + + [`AutoPipelineForImage2Image`] is a generic pipeline class that instantiates an image-to-image pipeline class. The + specific underlying pipeline class is automatically selected from either the + [`~AutoPipelineForImage2Image.from_pretrained`] or [`~AutoPipelineForImage2Image.from_pipe`] methods. + + This class cannot be instantiated using `__init__()` (throws an error). + + Class attributes: + + - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the + diffusion pipeline's components. + + """ + + config_name = "model_index.json" + + def __init__(self, *args, **kwargs): + raise EnvironmentError( + f"{self.__class__.__name__} is designed to be instantiated " + f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or " + f"`{self.__class__.__name__}.from_pipe(pipeline)` methods." + ) + + @classmethod + @validate_hf_hub_args + def from_pretrained(cls, pretrained_model_or_path, **kwargs): + r""" + Instantiates a image-to-image Pytorch diffusion pipeline from pretrained pipeline weight. + + The from_pretrained() method takes care of returning the correct pipeline class instance by: + 1. Detect the pipeline class of the pretrained_model_or_path based on the _class_name property of its + config object + 2. Find the image-to-image pipeline linked to the pipeline class using pattern matching on pipeline class + name. + + If a `controlnet` argument is passed, it will instantiate a [`StableDiffusionControlNetImg2ImgPipeline`] + object. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + If you get the error message below, you need to finetune the weights for your downstream task: + + ``` + Some weights of UNet2DConditionModel were not initialized from the model checkpoint at runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated + You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. + ``` + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *repo id* (for example `CompVis/ldm-text2im-large-256`) of a pretrained pipeline + hosted on the Hub. + - A path to a *directory* (for example `./my_pipeline_directory/`) containing pipeline weights + saved using + [`~DiffusionPipeline.save_pretrained`]. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the + dtype is automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + custom_revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id similar to + `revision` when loading a custom pipeline from the Hub. It can be a 🤗 Diffusers version when loading a + custom pipeline from GitHub, otherwise it defaults to `"main"` when loading from the Hub. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn’t need to be defined for each + parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the + same device. + + Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier for the maximum memory. Will default to the maximum memory available for + each GPU and the available CPU RAM if unset. + offload_folder (`str` or `os.PathLike`, *optional*): + The path to offload weights if device_map contains the value `"disk"`. + offload_state_dict (`bool`, *optional*): + If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if + the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` + when there is some disk offload. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the safetensors weights are downloaded if they're available **and** if the + safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors + weights. If set to `False`, safetensors weights are not loaded. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline + class). The overwritten components are passed directly to the pipelines `__init__` method. See example + below for more information. + variant (`str`, *optional*): + Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when + loading `from_flax`. + + + + To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with + `huggingface-cli login`. + + + + Examples: + + ```py + >>> from diffusers import AutoPipelineForImage2Image + + >>> pipeline = AutoPipelineForImage2Image.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> image = pipeline(prompt, image).images[0] + ``` + """ + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + + load_config_kwargs = { + "cache_dir": cache_dir, + "force_download": force_download, + "resume_download": resume_download, + "proxies": proxies, + "token": token, + "local_files_only": local_files_only, + "revision": revision, + } + + config = cls.load_config(pretrained_model_or_path, **load_config_kwargs) + orig_class_name = config["_class_name"] + + if "controlnet" in kwargs: + orig_class_name = config["_class_name"].replace("Pipeline", "ControlNetPipeline") + + image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, orig_class_name) + + kwargs = {**load_config_kwargs, **kwargs} + return image_2_image_cls.from_pretrained(pretrained_model_or_path, **kwargs) + + @classmethod + def from_pipe(cls, pipeline, **kwargs): + r""" + Instantiates a image-to-image Pytorch diffusion pipeline from another instantiated diffusion pipeline class. + + The from_pipe() method takes care of returning the correct pipeline class instance by finding the + image-to-image pipeline linked to the pipeline class using pattern matching on pipeline class name. + + All the modules the pipeline contains will be used to initialize the new pipeline without reallocating + additional memory. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pipeline (`DiffusionPipeline`): + an instantiated `DiffusionPipeline` object + + Examples: + + ```py + >>> from diffusers import AutoPipelineForText2Image, AutoPipelineForImage2Image + + >>> pipe_t2i = AutoPipelineForText2Image.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", requires_safety_checker=False + ... ) + + >>> pipe_i2i = AutoPipelineForImage2Image.from_pipe(pipe_t2i) + >>> image = pipe_i2i(prompt, image).images[0] + ``` + """ + + original_config = dict(pipeline.config) + original_cls_name = pipeline.__class__.__name__ + + # derive the pipeline class to instantiate + image_2_image_cls = _get_task_class(AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, original_cls_name) + + if "controlnet" in kwargs: + if kwargs["controlnet"] is not None: + image_2_image_cls = _get_task_class( + AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, + image_2_image_cls.__name__.replace("ControlNet", "").replace( + "Img2ImgPipeline", "ControlNetImg2ImgPipeline" + ), + ) + else: + image_2_image_cls = _get_task_class( + AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, + image_2_image_cls.__name__.replace("ControlNetImg2ImgPipeline", "Img2ImgPipeline"), + ) + + # define expected module and optional kwargs given the pipeline signature + expected_modules, optional_kwargs = image_2_image_cls._get_signature_keys(image_2_image_cls) + + pretrained_model_name_or_path = original_config.pop("_name_or_path", None) + + # allow users pass modules in `kwargs` to override the original pipeline's components + passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} + original_class_obj = { + k: pipeline.components[k] + for k, v in pipeline.components.items() + if k in expected_modules and k not in passed_class_obj + } + + # allow users pass optional kwargs to override the original pipelines config attribute + passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} + original_pipe_kwargs = { + k: original_config[k] + for k, v in original_config.items() + if k in optional_kwargs and k not in passed_pipe_kwargs + } + + # config attribute that were not expected by original pipeline is stored as its private attribute + # we will pass them as optional arguments if they can be accepted by the pipeline + additional_pipe_kwargs = [ + k[1:] + for k in original_config.keys() + if k.startswith("_") and k[1:] in optional_kwargs and k[1:] not in passed_pipe_kwargs + ] + for k in additional_pipe_kwargs: + original_pipe_kwargs[k] = original_config.pop(f"_{k}") + + image_2_image_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} + + # store unused config as private attribute + unused_original_config = { + f"{'' if k.startswith('_') else '_'}{k}": original_config[k] + for k, v in original_config.items() + if k not in image_2_image_kwargs + } + + missing_modules = set(expected_modules) - set(pipeline._optional_components) - set(image_2_image_kwargs.keys()) + + if len(missing_modules) > 0: + raise ValueError( + f"Pipeline {image_2_image_cls} expected {expected_modules}, but only {set(list(passed_class_obj.keys()) + list(original_class_obj.keys()))} were passed" + ) + + model = image_2_image_cls(**image_2_image_kwargs) + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + model.register_to_config(**unused_original_config) + + return model + + +class AutoPipelineForInpainting(ConfigMixin): + r""" + + [`AutoPipelineForInpainting`] is a generic pipeline class that instantiates an inpainting pipeline class. The + specific underlying pipeline class is automatically selected from either the + [`~AutoPipelineForInpainting.from_pretrained`] or [`~AutoPipelineForInpainting.from_pipe`] methods. + + This class cannot be instantiated using `__init__()` (throws an error). + + Class attributes: + + - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the + diffusion pipeline's components. + + """ + + config_name = "model_index.json" + + def __init__(self, *args, **kwargs): + raise EnvironmentError( + f"{self.__class__.__name__} is designed to be instantiated " + f"using the `{self.__class__.__name__}.from_pretrained(pretrained_model_name_or_path)` or " + f"`{self.__class__.__name__}.from_pipe(pipeline)` methods." + ) + + @classmethod + @validate_hf_hub_args + def from_pretrained(cls, pretrained_model_or_path, **kwargs): + r""" + Instantiates a inpainting Pytorch diffusion pipeline from pretrained pipeline weight. + + The from_pretrained() method takes care of returning the correct pipeline class instance by: + 1. Detect the pipeline class of the pretrained_model_or_path based on the _class_name property of its + config object + 2. Find the inpainting pipeline linked to the pipeline class using pattern matching on pipeline class name. + + If a `controlnet` argument is passed, it will instantiate a [`StableDiffusionControlNetInpaintPipeline`] + object. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + If you get the error message below, you need to finetune the weights for your downstream task: + + ``` + Some weights of UNet2DConditionModel were not initialized from the model checkpoint at runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated + You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. + ``` + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *repo id* (for example `CompVis/ldm-text2im-large-256`) of a pretrained pipeline + hosted on the Hub. + - A path to a *directory* (for example `./my_pipeline_directory/`) containing pipeline weights + saved using + [`~DiffusionPipeline.save_pretrained`]. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the + dtype is automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + custom_revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id similar to + `revision` when loading a custom pipeline from the Hub. It can be a 🤗 Diffusers version when loading a + custom pipeline from GitHub, otherwise it defaults to `"main"` when loading from the Hub. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn’t need to be defined for each + parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the + same device. + + Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier for the maximum memory. Will default to the maximum memory available for + each GPU and the available CPU RAM if unset. + offload_folder (`str` or `os.PathLike`, *optional*): + The path to offload weights if device_map contains the value `"disk"`. + offload_state_dict (`bool`, *optional*): + If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if + the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` + when there is some disk offload. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the safetensors weights are downloaded if they're available **and** if the + safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors + weights. If set to `False`, safetensors weights are not loaded. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline + class). The overwritten components are passed directly to the pipelines `__init__` method. See example + below for more information. + variant (`str`, *optional*): + Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when + loading `from_flax`. + + + + To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with + `huggingface-cli login`. + + + + Examples: + + ```py + >>> from diffusers import AutoPipelineForInpainting + + >>> pipeline = AutoPipelineForInpainting.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> image = pipeline(prompt, image=init_image, mask_image=mask_image).images[0] + ``` + """ + cache_dir = kwargs.pop("cache_dir", None) + force_download = kwargs.pop("force_download", False) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + token = kwargs.pop("token", None) + local_files_only = kwargs.pop("local_files_only", False) + revision = kwargs.pop("revision", None) + + load_config_kwargs = { + "cache_dir": cache_dir, + "force_download": force_download, + "resume_download": resume_download, + "proxies": proxies, + "token": token, + "local_files_only": local_files_only, + "revision": revision, + } + + config = cls.load_config(pretrained_model_or_path, **load_config_kwargs) + orig_class_name = config["_class_name"] + + if "controlnet" in kwargs: + orig_class_name = config["_class_name"].replace("Pipeline", "ControlNetPipeline") + + inpainting_cls = _get_task_class(AUTO_INPAINT_PIPELINES_MAPPING, orig_class_name) + + kwargs = {**load_config_kwargs, **kwargs} + return inpainting_cls.from_pretrained(pretrained_model_or_path, **kwargs) + + @classmethod + def from_pipe(cls, pipeline, **kwargs): + r""" + Instantiates a inpainting Pytorch diffusion pipeline from another instantiated diffusion pipeline class. + + The from_pipe() method takes care of returning the correct pipeline class instance by finding the inpainting + pipeline linked to the pipeline class using pattern matching on pipeline class name. + + All the modules the pipeline class contain will be used to initialize the new pipeline without reallocating + additional memory. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + Parameters: + pipeline (`DiffusionPipeline`): + an instantiated `DiffusionPipeline` object + + Examples: + + ```py + >>> from diffusers import AutoPipelineForText2Image, AutoPipelineForInpainting + + >>> pipe_t2i = AutoPipelineForText2Image.from_pretrained( + ... "DeepFloyd/IF-I-XL-v1.0", requires_safety_checker=False + ... ) + + >>> pipe_inpaint = AutoPipelineForInpainting.from_pipe(pipe_t2i) + >>> image = pipe_inpaint(prompt, image=init_image, mask_image=mask_image).images[0] + ``` + """ + original_config = dict(pipeline.config) + original_cls_name = pipeline.__class__.__name__ + + # derive the pipeline class to instantiate + inpainting_cls = _get_task_class(AUTO_INPAINT_PIPELINES_MAPPING, original_cls_name) + + if "controlnet" in kwargs: + if kwargs["controlnet"] is not None: + inpainting_cls = _get_task_class( + AUTO_INPAINT_PIPELINES_MAPPING, + inpainting_cls.__name__.replace("ControlNet", "").replace( + "InpaintPipeline", "ControlNetInpaintPipeline" + ), + ) + else: + inpainting_cls = _get_task_class( + AUTO_INPAINT_PIPELINES_MAPPING, + inpainting_cls.__name__.replace("ControlNetInpaintPipeline", "InpaintPipeline"), + ) + + # define expected module and optional kwargs given the pipeline signature + expected_modules, optional_kwargs = inpainting_cls._get_signature_keys(inpainting_cls) + + pretrained_model_name_or_path = original_config.pop("_name_or_path", None) + + # allow users pass modules in `kwargs` to override the original pipeline's components + passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} + original_class_obj = { + k: pipeline.components[k] + for k, v in pipeline.components.items() + if k in expected_modules and k not in passed_class_obj + } + + # allow users pass optional kwargs to override the original pipelines config attribute + passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} + original_pipe_kwargs = { + k: original_config[k] + for k, v in original_config.items() + if k in optional_kwargs and k not in passed_pipe_kwargs + } + + # config that were not expected by original pipeline is stored as private attribute + # we will pass them as optional arguments if they can be accepted by the pipeline + additional_pipe_kwargs = [ + k[1:] + for k in original_config.keys() + if k.startswith("_") and k[1:] in optional_kwargs and k[1:] not in passed_pipe_kwargs + ] + for k in additional_pipe_kwargs: + original_pipe_kwargs[k] = original_config.pop(f"_{k}") + + inpainting_kwargs = {**passed_class_obj, **original_class_obj, **passed_pipe_kwargs, **original_pipe_kwargs} + + # store unused config as private attribute + unused_original_config = { + f"{'' if k.startswith('_') else '_'}{k}": original_config[k] + for k, v in original_config.items() + if k not in inpainting_kwargs + } + + missing_modules = set(expected_modules) - set(pipeline._optional_components) - set(inpainting_kwargs.keys()) + + if len(missing_modules) > 0: + raise ValueError( + f"Pipeline {inpainting_cls} expected {expected_modules}, but only {set(list(passed_class_obj.keys()) + list(original_class_obj.keys()))} were passed" + ) + + model = inpainting_cls(**inpainting_kwargs) + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + model.register_to_config(**unused_original_config) + + return model diff --git a/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/__init__.py new file mode 100755 index 0000000..af6c879 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/__init__.py @@ -0,0 +1,20 @@ +from dataclasses import dataclass +from typing import List, Optional, Union + +import numpy as np +import PIL +from PIL import Image + +from ...utils import OptionalDependencyNotAvailable, is_torch_available, is_transformers_available + + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import ShapEPipeline +else: + from .blip_image_processing import BlipImageProcessor + from .modeling_blip2 import Blip2QFormerModel + from .modeling_ctx_clip import ContextCLIPTextModel + from .pipeline_blip_diffusion import BlipDiffusionPipeline diff --git a/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/blip_image_processing.py b/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/blip_image_processing.py new file mode 100755 index 0000000..d71a148 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/blip_image_processing.py @@ -0,0 +1,318 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Image processor class for BLIP.""" + +from typing import Dict, List, Optional, Union + +import numpy as np +import torch +from transformers.image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict +from transformers.image_transforms import convert_to_rgb, resize, to_channel_dimension_format +from transformers.image_utils import ( + OPENAI_CLIP_MEAN, + OPENAI_CLIP_STD, + ChannelDimension, + ImageInput, + PILImageResampling, + infer_channel_dimension_format, + is_scaled_image, + make_list_of_images, + to_numpy_array, + valid_images, +) +from transformers.utils import TensorType, is_vision_available, logging + +from diffusers.utils import numpy_to_pil + + +if is_vision_available(): + import PIL.Image + + +logger = logging.get_logger(__name__) + + +# We needed some extra functions on top of the ones in transformers.image_processing_utils.BaseImageProcessor, namely center crop +# Copy-pasted from transformers.models.blip.image_processing_blip.BlipImageProcessor +class BlipImageProcessor(BaseImageProcessor): + r""" + Constructs a BLIP image processor. + + Args: + do_resize (`bool`, *optional*, defaults to `True`): + Whether to resize the image's (height, width) dimensions to the specified `size`. Can be overridden by the + `do_resize` parameter in the `preprocess` method. + size (`dict`, *optional*, defaults to `{"height": 384, "width": 384}`): + Size of the output image after resizing. Can be overridden by the `size` parameter in the `preprocess` + method. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): + Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. Can be + overridden by the `resample` parameter in the `preprocess` method. + do_rescale (`bool`, *optional*, defaults to `True`): + Wwhether to rescale the image by the specified scale `rescale_factor`. Can be overridden by the + `do_rescale` parameter in the `preprocess` method. + rescale_factor (`int` or `float`, *optional*, defaults to `1/255`): + Scale factor to use if rescaling the image. Only has an effect if `do_rescale` is set to `True`. Can be + overridden by the `rescale_factor` parameter in the `preprocess` method. + do_normalize (`bool`, *optional*, defaults to `True`): + Whether to normalize the image. Can be overridden by the `do_normalize` parameter in the `preprocess` + method. Can be overridden by the `do_normalize` parameter in the `preprocess` method. + image_mean (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_MEAN`): + Mean to use if normalizing the image. This is a float or list of floats the length of the number of + channels in the image. Can be overridden by the `image_mean` parameter in the `preprocess` method. Can be + overridden by the `image_mean` parameter in the `preprocess` method. + image_std (`float` or `List[float]`, *optional*, defaults to `IMAGENET_STANDARD_STD`): + Standard deviation to use if normalizing the image. This is a float or list of floats the length of the + number of channels in the image. Can be overridden by the `image_std` parameter in the `preprocess` method. + Can be overridden by the `image_std` parameter in the `preprocess` method. + do_convert_rgb (`bool`, *optional*, defaults to `True`): + Whether to convert the image to RGB. + """ + + model_input_names = ["pixel_values"] + + def __init__( + self, + do_resize: bool = True, + size: Dict[str, int] = None, + resample: PILImageResampling = PILImageResampling.BICUBIC, + do_rescale: bool = True, + rescale_factor: Union[int, float] = 1 / 255, + do_normalize: bool = True, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + do_convert_rgb: bool = True, + do_center_crop: bool = True, + **kwargs, + ) -> None: + super().__init__(**kwargs) + size = size if size is not None else {"height": 224, "width": 224} + size = get_size_dict(size, default_to_square=True) + + self.do_resize = do_resize + self.size = size + self.resample = resample + self.do_rescale = do_rescale + self.rescale_factor = rescale_factor + self.do_normalize = do_normalize + self.image_mean = image_mean if image_mean is not None else OPENAI_CLIP_MEAN + self.image_std = image_std if image_std is not None else OPENAI_CLIP_STD + self.do_convert_rgb = do_convert_rgb + self.do_center_crop = do_center_crop + + # Copy-pasted from transformers.models.vit.image_processing_vit.ViTImageProcessor.resize with PILImageResampling.BILINEAR->PILImageResampling.BICUBIC + def resize( + self, + image: np.ndarray, + size: Dict[str, int], + resample: PILImageResampling = PILImageResampling.BICUBIC, + data_format: Optional[Union[str, ChannelDimension]] = None, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> np.ndarray: + """ + Resize an image to `(size["height"], size["width"])`. + + Args: + image (`np.ndarray`): + Image to resize. + size (`Dict[str, int]`): + Dictionary in the format `{"height": int, "width": int}` specifying the size of the output image. + resample (`PILImageResampling`, *optional*, defaults to `PILImageResampling.BICUBIC`): + `PILImageResampling` filter to use when resizing the image e.g. `PILImageResampling.BICUBIC`. + data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the output image. If unset, the channel dimension format of the input + image is used. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + + Returns: + `np.ndarray`: The resized image. + """ + size = get_size_dict(size) + if "height" not in size or "width" not in size: + raise ValueError(f"The `size` dictionary must contain the keys `height` and `width`. Got {size.keys()}") + output_size = (size["height"], size["width"]) + return resize( + image, + size=output_size, + resample=resample, + data_format=data_format, + input_data_format=input_data_format, + **kwargs, + ) + + def preprocess( + self, + images: ImageInput, + do_resize: Optional[bool] = None, + size: Optional[Dict[str, int]] = None, + resample: PILImageResampling = None, + do_rescale: Optional[bool] = None, + do_center_crop: Optional[bool] = None, + rescale_factor: Optional[float] = None, + do_normalize: Optional[bool] = None, + image_mean: Optional[Union[float, List[float]]] = None, + image_std: Optional[Union[float, List[float]]] = None, + return_tensors: Optional[Union[str, TensorType]] = None, + do_convert_rgb: bool = None, + data_format: ChannelDimension = ChannelDimension.FIRST, + input_data_format: Optional[Union[str, ChannelDimension]] = None, + **kwargs, + ) -> PIL.Image.Image: + """ + Preprocess an image or batch of images. + + Args: + images (`ImageInput`): + Image to preprocess. Expects a single or batch of images with pixel values ranging from 0 to 255. If + passing in images with pixel values between 0 and 1, set `do_rescale=False`. + do_resize (`bool`, *optional*, defaults to `self.do_resize`): + Whether to resize the image. + size (`Dict[str, int]`, *optional*, defaults to `self.size`): + Controls the size of the image after `resize`. The shortest edge of the image is resized to + `size["shortest_edge"]` whilst preserving the aspect ratio. If the longest edge of this resized image + is > `int(size["shortest_edge"] * (1333 / 800))`, then the image is resized again to make the longest + edge equal to `int(size["shortest_edge"] * (1333 / 800))`. + resample (`PILImageResampling`, *optional*, defaults to `self.resample`): + Resampling filter to use if resizing the image. Only has an effect if `do_resize` is set to `True`. + do_rescale (`bool`, *optional*, defaults to `self.do_rescale`): + Whether to rescale the image values between [0 - 1]. + rescale_factor (`float`, *optional*, defaults to `self.rescale_factor`): + Rescale factor to rescale the image by if `do_rescale` is set to `True`. + do_normalize (`bool`, *optional*, defaults to `self.do_normalize`): + Whether to normalize the image. + image_mean (`float` or `List[float]`, *optional*, defaults to `self.image_mean`): + Image mean to normalize the image by if `do_normalize` is set to `True`. + image_std (`float` or `List[float]`, *optional*, defaults to `self.image_std`): + Image standard deviation to normalize the image by if `do_normalize` is set to `True`. + do_convert_rgb (`bool`, *optional*, defaults to `self.do_convert_rgb`): + Whether to convert the image to RGB. + return_tensors (`str` or `TensorType`, *optional*): + The type of tensors to return. Can be one of: + - Unset: Return a list of `np.ndarray`. + - `TensorType.TENSORFLOW` or `'tf'`: Return a batch of type `tf.Tensor`. + - `TensorType.PYTORCH` or `'pt'`: Return a batch of type `torch.Tensor`. + - `TensorType.NUMPY` or `'np'`: Return a batch of type `np.ndarray`. + - `TensorType.JAX` or `'jax'`: Return a batch of type `jax.numpy.ndarray`. + data_format (`ChannelDimension` or `str`, *optional*, defaults to `ChannelDimension.FIRST`): + The channel dimension format for the output image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - Unset: Use the channel dimension format of the input image. + input_data_format (`ChannelDimension` or `str`, *optional*): + The channel dimension format for the input image. If unset, the channel dimension format is inferred + from the input image. Can be one of: + - `"channels_first"` or `ChannelDimension.FIRST`: image in (num_channels, height, width) format. + - `"channels_last"` or `ChannelDimension.LAST`: image in (height, width, num_channels) format. + - `"none"` or `ChannelDimension.NONE`: image in (height, width) format. + """ + do_resize = do_resize if do_resize is not None else self.do_resize + resample = resample if resample is not None else self.resample + do_rescale = do_rescale if do_rescale is not None else self.do_rescale + rescale_factor = rescale_factor if rescale_factor is not None else self.rescale_factor + do_normalize = do_normalize if do_normalize is not None else self.do_normalize + image_mean = image_mean if image_mean is not None else self.image_mean + image_std = image_std if image_std is not None else self.image_std + do_convert_rgb = do_convert_rgb if do_convert_rgb is not None else self.do_convert_rgb + do_center_crop = do_center_crop if do_center_crop is not None else self.do_center_crop + + size = size if size is not None else self.size + size = get_size_dict(size, default_to_square=False) + images = make_list_of_images(images) + + if not valid_images(images): + raise ValueError( + "Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, " + "torch.Tensor, tf.Tensor or jax.ndarray." + ) + + if do_resize and size is None or resample is None: + raise ValueError("Size and resample must be specified if do_resize is True.") + + if do_rescale and rescale_factor is None: + raise ValueError("Rescale factor must be specified if do_rescale is True.") + + if do_normalize and (image_mean is None or image_std is None): + raise ValueError("Image mean and std must be specified if do_normalize is True.") + + # PIL RGBA images are converted to RGB + if do_convert_rgb: + images = [convert_to_rgb(image) for image in images] + + # All transformations expect numpy arrays. + images = [to_numpy_array(image) for image in images] + + if is_scaled_image(images[0]) and do_rescale: + logger.warning_once( + "It looks like you are trying to rescale already rescaled images. If the input" + " images have pixel values between 0 and 1, set `do_rescale=False` to avoid rescaling them again." + ) + if input_data_format is None: + # We assume that all images have the same channel dimension format. + input_data_format = infer_channel_dimension_format(images[0]) + + if do_resize: + images = [ + self.resize(image=image, size=size, resample=resample, input_data_format=input_data_format) + for image in images + ] + + if do_rescale: + images = [ + self.rescale(image=image, scale=rescale_factor, input_data_format=input_data_format) + for image in images + ] + if do_normalize: + images = [ + self.normalize(image=image, mean=image_mean, std=image_std, input_data_format=input_data_format) + for image in images + ] + if do_center_crop: + images = [self.center_crop(image, size, input_data_format=input_data_format) for image in images] + + images = [ + to_channel_dimension_format(image, data_format, input_channel_dim=input_data_format) for image in images + ] + + encoded_outputs = BatchFeature(data={"pixel_values": images}, tensor_type=return_tensors) + return encoded_outputs + + # Follows diffusers.VaeImageProcessor.postprocess + def postprocess(self, sample: torch.FloatTensor, output_type: str = "pil"): + if output_type not in ["pt", "np", "pil"]: + raise ValueError( + f"output_type={output_type} is not supported. Make sure to choose one of ['pt', 'np', or 'pil']" + ) + + # Equivalent to diffusers.VaeImageProcessor.denormalize + sample = (sample / 2 + 0.5).clamp(0, 1) + if output_type == "pt": + return sample + + # Equivalent to diffusers.VaeImageProcessor.pt_to_numpy + sample = sample.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "np": + return sample + # Output_type must be 'pil' + sample = numpy_to_pil(sample) + return sample diff --git a/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py b/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py new file mode 100755 index 0000000..c8869ad --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/modeling_blip2.py @@ -0,0 +1,642 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, Tuple, Union + +import torch +import torch.utils.checkpoint +from torch import nn +from transformers import BertTokenizer +from transformers.activations import QuickGELUActivation as QuickGELU +from transformers.modeling_outputs import ( + BaseModelOutputWithPastAndCrossAttentions, + BaseModelOutputWithPooling, + BaseModelOutputWithPoolingAndCrossAttentions, +) +from transformers.models.blip_2.configuration_blip_2 import Blip2Config, Blip2VisionConfig +from transformers.models.blip_2.modeling_blip_2 import ( + Blip2Encoder, + Blip2PreTrainedModel, + Blip2QFormerAttention, + Blip2QFormerIntermediate, + Blip2QFormerOutput, +) +from transformers.pytorch_utils import apply_chunking_to_forward +from transformers.utils import ( + logging, + replace_return_docstrings, +) + + +logger = logging.get_logger(__name__) + + +# There is an implementation of Blip2 in `transformers` : https://github.com/huggingface/transformers/blob/main/src/transformers/models/blip_2/modeling_blip_2.py. +# But it doesn't support getting multimodal embeddings. So, this module can be +# replaced with a future `transformers` version supports that. +class Blip2TextEmbeddings(nn.Module): + """Construct the embeddings from word and position embeddings.""" + + def __init__(self, config): + super().__init__() + self.word_embeddings = nn.Embedding(config.vocab_size, config.hidden_size, padding_idx=config.pad_token_id) + self.position_embeddings = nn.Embedding(config.max_position_embeddings, config.hidden_size) + + # self.LayerNorm is not snake-cased to stick with TensorFlow model variable name and be able to load + # any TensorFlow checkpoint file + self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.dropout = nn.Dropout(config.hidden_dropout_prob) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1))) + self.position_embedding_type = getattr(config, "position_embedding_type", "absolute") + + self.config = config + + def forward( + self, + input_ids=None, + position_ids=None, + query_embeds=None, + past_key_values_length=0, + ): + if input_ids is not None: + seq_length = input_ids.size()[1] + else: + seq_length = 0 + + if position_ids is None: + position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length].clone() + + if input_ids is not None: + embeddings = self.word_embeddings(input_ids) + if self.position_embedding_type == "absolute": + position_embeddings = self.position_embeddings(position_ids) + embeddings = embeddings + position_embeddings + + if query_embeds is not None: + batch_size = embeddings.shape[0] + # repeat the query embeddings for batch size + query_embeds = query_embeds.repeat(batch_size, 1, 1) + embeddings = torch.cat((query_embeds, embeddings), dim=1) + else: + embeddings = query_embeds + embeddings = embeddings.to(query_embeds.dtype) + embeddings = self.LayerNorm(embeddings) + embeddings = self.dropout(embeddings) + return embeddings + + +# Copy-pasted from transformers.models.blip.modeling_blip.BlipVisionEmbeddings with Blip->Blip2 +class Blip2VisionEmbeddings(nn.Module): + def __init__(self, config: Blip2VisionConfig): + super().__init__() + self.config = config + self.embed_dim = config.hidden_size + self.image_size = config.image_size + self.patch_size = config.patch_size + + self.class_embedding = nn.Parameter(torch.randn(1, 1, self.embed_dim)) + + self.patch_embedding = nn.Conv2d( + in_channels=3, out_channels=self.embed_dim, kernel_size=self.patch_size, stride=self.patch_size, bias=False + ) + + self.num_patches = (self.image_size // self.patch_size) ** 2 + self.num_positions = self.num_patches + 1 + + self.position_embedding = nn.Parameter(torch.randn(1, self.num_positions, self.embed_dim)) + + def forward(self, pixel_values: torch.FloatTensor) -> torch.Tensor: + batch_size = pixel_values.shape[0] + target_dtype = self.patch_embedding.weight.dtype + patch_embeds = self.patch_embedding(pixel_values.to(dtype=target_dtype)) # shape = [*, width, grid, grid] + patch_embeds = patch_embeds.flatten(2).transpose(1, 2) + + class_embeds = self.class_embedding.expand(batch_size, 1, -1).to(target_dtype) + embeddings = torch.cat([class_embeds, patch_embeds], dim=1) + embeddings = embeddings + self.position_embedding[:, : embeddings.size(1), :].to(target_dtype) + return embeddings + + +# The Qformer encoder, which takes the visual embeddings, and the text input, to get multimodal embeddings +class Blip2QFormerEncoder(nn.Module): + def __init__(self, config): + super().__init__() + self.config = config + self.layer = nn.ModuleList( + [Blip2QFormerLayer(config, layer_idx) for layer_idx in range(config.num_hidden_layers)] + ) + self.gradient_checkpointing = False + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=False, + output_hidden_states=False, + return_dict=True, + query_length=0, + ): + all_hidden_states = () if output_hidden_states else None + all_self_attentions = () if output_attentions else None + all_cross_attentions = () if output_attentions else None + + next_decoder_cache = () if use_cache else None + + for i in range(self.config.num_hidden_layers): + layer_module = self.layer[i] + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + layer_head_mask = head_mask[i] if head_mask is not None else None + past_key_value = past_key_values[i] if past_key_values is not None else None + + if getattr(self.config, "gradient_checkpointing", False) and self.training: + if use_cache: + logger.warning( + "`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`..." + ) + use_cache = False + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs, past_key_value, output_attentions, query_length) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(layer_module), + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + ) + else: + layer_outputs = layer_module( + hidden_states, + attention_mask, + layer_head_mask, + encoder_hidden_states, + encoder_attention_mask, + past_key_value, + output_attentions, + query_length, + ) + + hidden_states = layer_outputs[0] + if use_cache: + next_decoder_cache += (layer_outputs[-1],) + if output_attentions: + all_self_attentions = all_self_attentions + (layer_outputs[1],) + if layer_module.has_cross_attention: + all_cross_attentions = all_cross_attentions + (layer_outputs[2],) + + if output_hidden_states: + all_hidden_states = all_hidden_states + (hidden_states,) + + if not return_dict: + return tuple( + v + for v in [ + hidden_states, + next_decoder_cache, + all_hidden_states, + all_self_attentions, + all_cross_attentions, + ] + if v is not None + ) + return BaseModelOutputWithPastAndCrossAttentions( + last_hidden_state=hidden_states, + past_key_values=next_decoder_cache, + hidden_states=all_hidden_states, + attentions=all_self_attentions, + cross_attentions=all_cross_attentions, + ) + + +# The layers making up the Qformer encoder +class Blip2QFormerLayer(nn.Module): + def __init__(self, config, layer_idx): + super().__init__() + self.chunk_size_feed_forward = config.chunk_size_feed_forward + self.seq_len_dim = 1 + self.attention = Blip2QFormerAttention(config) + + self.layer_idx = layer_idx + + if layer_idx % config.cross_attention_frequency == 0: + self.crossattention = Blip2QFormerAttention(config, is_cross_attention=True) + self.has_cross_attention = True + else: + self.has_cross_attention = False + + self.intermediate = Blip2QFormerIntermediate(config) + self.intermediate_query = Blip2QFormerIntermediate(config) + self.output_query = Blip2QFormerOutput(config) + self.output = Blip2QFormerOutput(config) + + def forward( + self, + hidden_states, + attention_mask=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_value=None, + output_attentions=False, + query_length=0, + ): + # decoder uni-directional self-attention cached key/values tuple is at positions 1,2 + self_attn_past_key_value = past_key_value[:2] if past_key_value is not None else None + self_attention_outputs = self.attention( + hidden_states, + attention_mask, + head_mask, + output_attentions=output_attentions, + past_key_value=self_attn_past_key_value, + ) + attention_output = self_attention_outputs[0] + outputs = self_attention_outputs[1:-1] + + present_key_value = self_attention_outputs[-1] + + if query_length > 0: + query_attention_output = attention_output[:, :query_length, :] + + if self.has_cross_attention: + if encoder_hidden_states is None: + raise ValueError("encoder_hidden_states must be given for cross-attention layers") + cross_attention_outputs = self.crossattention( + query_attention_output, + attention_mask, + head_mask, + encoder_hidden_states, + encoder_attention_mask, + output_attentions=output_attentions, + ) + query_attention_output = cross_attention_outputs[0] + # add cross attentions if we output attention weights + outputs = outputs + cross_attention_outputs[1:-1] + + layer_output = apply_chunking_to_forward( + self.feed_forward_chunk_query, + self.chunk_size_feed_forward, + self.seq_len_dim, + query_attention_output, + ) + + if attention_output.shape[1] > query_length: + layer_output_text = apply_chunking_to_forward( + self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output[:, query_length:, :], + ) + layer_output = torch.cat([layer_output, layer_output_text], dim=1) + else: + layer_output = apply_chunking_to_forward( + self.feed_forward_chunk, + self.chunk_size_feed_forward, + self.seq_len_dim, + attention_output, + ) + outputs = (layer_output,) + outputs + + outputs = outputs + (present_key_value,) + + return outputs + + def feed_forward_chunk(self, attention_output): + intermediate_output = self.intermediate(attention_output) + layer_output = self.output(intermediate_output, attention_output) + return layer_output + + def feed_forward_chunk_query(self, attention_output): + intermediate_output = self.intermediate_query(attention_output) + layer_output = self.output_query(intermediate_output, attention_output) + return layer_output + + +# ProjLayer used to project the multimodal Blip2 embeddings to be used in the text encoder +class ProjLayer(nn.Module): + def __init__(self, in_dim, out_dim, hidden_dim, drop_p=0.1, eps=1e-12): + super().__init__() + + # Dense1 -> Act -> Dense2 -> Drop -> Res -> Norm + self.dense1 = nn.Linear(in_dim, hidden_dim) + self.act_fn = QuickGELU() + self.dense2 = nn.Linear(hidden_dim, out_dim) + self.dropout = nn.Dropout(drop_p) + + self.LayerNorm = nn.LayerNorm(out_dim, eps=eps) + + def forward(self, x): + x_in = x + + x = self.LayerNorm(x) + x = self.dropout(self.dense2(self.act_fn(self.dense1(x)))) + x_in + + return x + + +# Copy-pasted from transformers.models.blip.modeling_blip.BlipVisionModel with Blip->Blip2, BLIP->BLIP_2 +class Blip2VisionModel(Blip2PreTrainedModel): + main_input_name = "pixel_values" + config_class = Blip2VisionConfig + + def __init__(self, config: Blip2VisionConfig): + super().__init__(config) + self.config = config + embed_dim = config.hidden_size + self.embeddings = Blip2VisionEmbeddings(config) + self.pre_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) + self.encoder = Blip2Encoder(config) + self.post_layernorm = nn.LayerNorm(embed_dim, eps=config.layer_norm_eps) + + self.post_init() + + @replace_return_docstrings(output_type=BaseModelOutputWithPooling, config_class=Blip2VisionConfig) + def forward( + self, + pixel_values: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + Returns: + + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if pixel_values is None: + raise ValueError("You have to specify pixel_values") + + hidden_states = self.embeddings(pixel_values) + hidden_states = self.pre_layernorm(hidden_states) + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + last_hidden_state = encoder_outputs[0] + last_hidden_state = self.post_layernorm(last_hidden_state) + + pooled_output = last_hidden_state[:, 0, :] + pooled_output = self.post_layernorm(pooled_output) + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + def get_input_embeddings(self): + return self.embeddings + + +# Qformer model, used to get multimodal embeddings from the text and image inputs +class Blip2QFormerModel(Blip2PreTrainedModel): + """ + Querying Transformer (Q-Former), used in BLIP-2. + """ + + def __init__(self, config: Blip2Config): + super().__init__(config) + self.config = config + self.embeddings = Blip2TextEmbeddings(config.qformer_config) + self.visual_encoder = Blip2VisionModel(config.vision_config) + self.query_tokens = nn.Parameter(torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)) + if not hasattr(config, "tokenizer") or config.tokenizer is None: + self.tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", truncation_side="right") + else: + self.tokenizer = BertTokenizer.from_pretrained(config.tokenizer, truncation_side="right") + self.tokenizer.add_special_tokens({"bos_token": "[DEC]"}) + self.proj_layer = ProjLayer( + in_dim=config.qformer_config.hidden_size, + out_dim=config.qformer_config.hidden_size, + hidden_dim=config.qformer_config.hidden_size * 4, + drop_p=0.1, + eps=1e-12, + ) + + self.encoder = Blip2QFormerEncoder(config.qformer_config) + + self.post_init() + + def get_input_embeddings(self): + return self.embeddings.word_embeddings + + def set_input_embeddings(self, value): + self.embeddings.word_embeddings = value + + def _prune_heads(self, heads_to_prune): + """ + Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base + class PreTrainedModel + """ + for layer, heads in heads_to_prune.items(): + self.encoder.layer[layer].attention.prune_heads(heads) + + def get_extended_attention_mask( + self, + attention_mask: torch.Tensor, + input_shape: Tuple[int], + device: torch.device, + has_query: bool = False, + ) -> torch.Tensor: + """ + Makes broadcastable attention and causal masks so that future and masked tokens are ignored. + + Arguments: + attention_mask (`torch.Tensor`): + Mask with ones indicating tokens to attend to, zeros for tokens to ignore. + input_shape (`Tuple[int]`): + The shape of the input to the model. + device (`torch.device`): + The device of the input to the model. + + Returns: + `torch.Tensor` The extended attention mask, with a the same dtype as `attention_mask.dtype`. + """ + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + if attention_mask.dim() == 3: + extended_attention_mask = attention_mask[:, None, :, :] + elif attention_mask.dim() == 2: + # Provided a padding mask of dimensions [batch_size, seq_length] + # - the model is an encoder, so make the mask broadcastable to [batch_size, num_heads, seq_length, seq_length] + extended_attention_mask = attention_mask[:, None, None, :] + else: + raise ValueError( + "Wrong shape for input_ids (shape {}) or attention_mask (shape {})".format( + input_shape, attention_mask.shape + ) + ) + + # Since attention_mask is 1.0 for positions we want to attend and 0.0 for + # masked positions, this operation will create a tensor which is 0.0 for + # positions we want to attend and -10000.0 for masked positions. + # Since we are adding it to the raw scores before the softmax, this is + # effectively the same as removing these entirely. + extended_attention_mask = extended_attention_mask.to(dtype=self.dtype) # fp16 compatibility + extended_attention_mask = (1.0 - extended_attention_mask) * -10000.0 + return extended_attention_mask + + def forward( + self, + text_input=None, + image_input=None, + head_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + past_key_values=None, + use_cache=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + r""" + encoder_hidden_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, `optional`): + Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if + the model is configured as a decoder. + encoder_attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, `optional`): + Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in + the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`: + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + past_key_values (`tuple(tuple(torch.FloatTensor))` of length `config.n_layers` with each tuple having 4 tensors of: + shape `(batch_size, num_heads, sequence_length - 1, embed_size_per_head)`): Contains precomputed key and + value hidden states of the attention blocks. Can be used to speed up decoding. If `past_key_values` are + used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key + value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape + `(batch_size, sequence_length)`. + use_cache (`bool`, `optional`): + If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see + `past_key_values`). + """ + + text = self.tokenizer(text_input, return_tensors="pt", padding=True) + text = text.to(self.device) + input_ids = text.input_ids + batch_size = input_ids.shape[0] + query_atts = torch.ones((batch_size, self.query_tokens.size()[1]), dtype=torch.long).to(self.device) + attention_mask = torch.cat([query_atts, text.attention_mask], dim=1) + + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # past_key_values_length + past_key_values_length = ( + past_key_values[0][0].shape[2] - self.config.query_length if past_key_values is not None else 0 + ) + + query_length = self.query_tokens.shape[1] + + embedding_output = self.embeddings( + input_ids=input_ids, + query_embeds=self.query_tokens, + past_key_values_length=past_key_values_length, + ) + + # embedding_output = self.layernorm(query_embeds) + # embedding_output = self.dropout(embedding_output) + + input_shape = embedding_output.size()[:-1] + batch_size, seq_length = input_shape + device = embedding_output.device + + image_embeds_frozen = self.visual_encoder(image_input).last_hidden_state + # image_embeds_frozen = torch.ones_like(image_embeds_frozen) + encoder_hidden_states = image_embeds_frozen + + if attention_mask is None: + attention_mask = torch.ones(((batch_size, seq_length + past_key_values_length)), device=device) + + # We can provide a self-attention mask of dimensions [batch_size, from_seq_length, to_seq_length] + # ourselves in which case we just need to make it broadcastable to all heads. + extended_attention_mask = self.get_extended_attention_mask(attention_mask, input_shape, device) + + # If a 2D or 3D attention mask is provided for the cross-attention + # we need to make broadcastable to [batch_size, num_heads, seq_length, seq_length] + if encoder_hidden_states is not None: + if isinstance(encoder_hidden_states, list): + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states[0].size() + else: + encoder_batch_size, encoder_sequence_length, _ = encoder_hidden_states.size() + encoder_hidden_shape = (encoder_batch_size, encoder_sequence_length) + + if isinstance(encoder_attention_mask, list): + encoder_extended_attention_mask = [self.invert_attention_mask(mask) for mask in encoder_attention_mask] + elif encoder_attention_mask is None: + encoder_attention_mask = torch.ones(encoder_hidden_shape, device=device) + encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) + else: + encoder_extended_attention_mask = self.invert_attention_mask(encoder_attention_mask) + else: + encoder_extended_attention_mask = None + + # Prepare head mask if needed + # 1.0 in head_mask indicate we keep the head + # attention_probs has shape bsz x n_heads x N x N + # input head_mask has shape [num_heads] or [num_hidden_layers x num_heads] + # and head_mask is converted to shape [num_hidden_layers x batch x num_heads x seq_length x seq_length] + head_mask = self.get_head_mask(head_mask, self.config.qformer_config.num_hidden_layers) + + encoder_outputs = self.encoder( + embedding_output, + attention_mask=extended_attention_mask, + head_mask=head_mask, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_extended_attention_mask, + past_key_values=past_key_values, + use_cache=use_cache, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + query_length=query_length, + ) + sequence_output = encoder_outputs[0] + pooled_output = sequence_output[:, 0, :] + + if not return_dict: + return self.proj_layer(sequence_output[:, :query_length, :]) + + return BaseModelOutputWithPoolingAndCrossAttentions( + last_hidden_state=sequence_output, + pooler_output=pooled_output, + past_key_values=encoder_outputs.past_key_values, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + cross_attentions=encoder_outputs.cross_attentions, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py b/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py new file mode 100755 index 0000000..c6772fc --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/modeling_ctx_clip.py @@ -0,0 +1,223 @@ +# Copyright 2024 Salesforce.com, inc. +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Optional, Tuple, Union + +import torch +from torch import nn +from transformers import CLIPPreTrainedModel +from transformers.modeling_outputs import BaseModelOutputWithPooling +from transformers.models.clip.configuration_clip import CLIPTextConfig +from transformers.models.clip.modeling_clip import CLIPEncoder + + +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +# This is a modified version of the CLIPTextModel from transformers.models.clip.modeling_clip +# Which allows for an extra input of "context embeddings", which are the query embeddings used in Qformer +# They pass through the clip model, along with the text embeddings, and interact with them using self attention +class ContextCLIPTextModel(CLIPPreTrainedModel): + config_class = CLIPTextConfig + + _no_split_modules = ["CLIPEncoderLayer"] + + def __init__(self, config: CLIPTextConfig): + super().__init__(config) + self.text_model = ContextCLIPTextTransformer(config) + # Initialize weights and apply final processing + self.post_init() + + def forward( + self, + ctx_embeddings: torch.Tensor = None, + ctx_begin_pos: list = None, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + return self.text_model( + ctx_embeddings=ctx_embeddings, + ctx_begin_pos=ctx_begin_pos, + input_ids=input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + +class ContextCLIPTextTransformer(nn.Module): + def __init__(self, config: CLIPTextConfig): + super().__init__() + self.config = config + embed_dim = config.hidden_size + self.embeddings = ContextCLIPTextEmbeddings(config) + self.encoder = CLIPEncoder(config) + self.final_layer_norm = nn.LayerNorm(embed_dim) + + def forward( + self, + ctx_embeddings: torch.Tensor, + ctx_begin_pos: list, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutputWithPooling]: + r""" + Returns: + + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + if input_ids is None: + raise ValueError("You have to specify either input_ids") + + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + + hidden_states = self.embeddings( + input_ids=input_ids, + position_ids=position_ids, + ctx_embeddings=ctx_embeddings, + ctx_begin_pos=ctx_begin_pos, + ) + + bsz, seq_len = input_shape + if ctx_embeddings is not None: + seq_len += ctx_embeddings.size(1) + # CLIP's text model uses causal mask, prepare it here. + # https://github.com/openai/CLIP/blob/cfcffb90e69f37bf2ff1e988237a0fbe41f33c04/clip/model.py#L324 + causal_attention_mask = self._build_causal_attention_mask(bsz, seq_len, hidden_states.dtype).to( + hidden_states.device + ) + # expand attention_mask + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + attention_mask = _expand_mask(attention_mask, hidden_states.dtype) + + encoder_outputs = self.encoder( + inputs_embeds=hidden_states, + attention_mask=attention_mask, + causal_attention_mask=causal_attention_mask, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + + last_hidden_state = encoder_outputs[0] + last_hidden_state = self.final_layer_norm(last_hidden_state) + + # text_embeds.shape = [batch_size, sequence_length, transformer.width] + # take features from the eot embedding (eot_token is the highest number in each sequence) + # casting to torch.int for onnx compatibility: argmax doesn't support int64 inputs with opset 14 + pooled_output = last_hidden_state[ + torch.arange(last_hidden_state.shape[0], device=input_ids.device), + input_ids.to(torch.int).argmax(dim=-1), + ] + + if not return_dict: + return (last_hidden_state, pooled_output) + encoder_outputs[1:] + + return BaseModelOutputWithPooling( + last_hidden_state=last_hidden_state, + pooler_output=pooled_output, + hidden_states=encoder_outputs.hidden_states, + attentions=encoder_outputs.attentions, + ) + + def _build_causal_attention_mask(self, bsz, seq_len, dtype): + # lazily create causal attention mask, with full attention between the vision tokens + # pytorch uses additive attention mask; fill with -inf + mask = torch.empty(bsz, seq_len, seq_len, dtype=dtype) + mask.fill_(torch.tensor(torch.finfo(dtype).min)) + mask.triu_(1) # zero out the lower diagonal + mask = mask.unsqueeze(1) # expand mask + return mask + + +class ContextCLIPTextEmbeddings(nn.Module): + def __init__(self, config: CLIPTextConfig): + super().__init__() + embed_dim = config.hidden_size + + self.token_embedding = nn.Embedding(config.vocab_size, embed_dim) + self.position_embedding = nn.Embedding(config.max_position_embeddings, embed_dim) + + # position_ids (1, len position emb) is contiguous in memory and exported when serialized + self.register_buffer("position_ids", torch.arange(config.max_position_embeddings).expand((1, -1))) + + def forward( + self, + ctx_embeddings: torch.Tensor, + ctx_begin_pos: list, + input_ids: Optional[torch.LongTensor] = None, + position_ids: Optional[torch.LongTensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + ) -> torch.Tensor: + if ctx_embeddings is None: + ctx_len = 0 + else: + ctx_len = ctx_embeddings.shape[1] + + seq_length = (input_ids.shape[-1] if input_ids is not None else inputs_embeds.shape[-2]) + ctx_len + + if position_ids is None: + position_ids = self.position_ids[:, :seq_length] + + if inputs_embeds is None: + inputs_embeds = self.token_embedding(input_ids) + + # for each input embeddings, add the ctx embeddings at the correct position + input_embeds_ctx = [] + bsz = inputs_embeds.shape[0] + + if ctx_embeddings is not None: + for i in range(bsz): + cbp = ctx_begin_pos[i] + + prefix = inputs_embeds[i, :cbp] + # remove the special token embedding + suffix = inputs_embeds[i, cbp:] + + input_embeds_ctx.append(torch.cat([prefix, ctx_embeddings[i], suffix], dim=0)) + + inputs_embeds = torch.stack(input_embeds_ctx, dim=0) + + position_embeddings = self.position_embedding(position_ids) + embeddings = inputs_embeds + position_embeddings + + return embeddings diff --git a/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py new file mode 100755 index 0000000..ba43b2e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/blip_diffusion/pipeline_blip_diffusion.py @@ -0,0 +1,348 @@ +# Copyright 2024 Salesforce.com, inc. +# Copyright 2024 The HuggingFace Team. All rights reserved.# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List, Optional, Union + +import PIL.Image +import torch +from transformers import CLIPTokenizer + +from ...models import AutoencoderKL, UNet2DConditionModel +from ...schedulers import PNDMScheduler +from ...utils import ( + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from .blip_image_processing import BlipImageProcessor +from .modeling_blip2 import Blip2QFormerModel +from .modeling_ctx_clip import ContextCLIPTextModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers.pipelines import BlipDiffusionPipeline + >>> from diffusers.utils import load_image + >>> import torch + + >>> blip_diffusion_pipe = BlipDiffusionPipeline.from_pretrained( + ... "Salesforce/blipdiffusion", torch_dtype=torch.float16 + ... ).to("cuda") + + + >>> cond_subject = "dog" + >>> tgt_subject = "dog" + >>> text_prompt_input = "swimming underwater" + + >>> cond_image = load_image( + ... "https://huggingface.co/datasets/ayushtues/blipdiffusion_images/resolve/main/dog.jpg" + ... ) + >>> guidance_scale = 7.5 + >>> num_inference_steps = 25 + >>> negative_prompt = "over-exposure, under-exposure, saturated, duplicate, out of frame, lowres, cropped, worst quality, low quality, jpeg artifacts, morbid, mutilated, out of frame, ugly, bad anatomy, bad proportions, deformed, blurry, duplicate" + + + >>> output = blip_diffusion_pipe( + ... text_prompt_input, + ... cond_image, + ... cond_subject, + ... tgt_subject, + ... guidance_scale=guidance_scale, + ... num_inference_steps=num_inference_steps, + ... neg_prompt=negative_prompt, + ... height=512, + ... width=512, + ... ).images + >>> output[0].save("image.png") + ``` +""" + + +class BlipDiffusionPipeline(DiffusionPipeline): + """ + Pipeline for Zero-Shot Subject Driven Generation using Blip Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + tokenizer ([`CLIPTokenizer`]): + Tokenizer for the text encoder + text_encoder ([`ContextCLIPTextModel`]): + Text encoder to encode the text prompt + vae ([`AutoencoderKL`]): + VAE model to map the latents to the image + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + scheduler ([`PNDMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + qformer ([`Blip2QFormerModel`]): + QFormer model to get multi-modal embeddings from the text and image. + image_processor ([`BlipImageProcessor`]): + Image Processor to preprocess and postprocess the image. + ctx_begin_pos (int, `optional`, defaults to 2): + Position of the context token in the text encoder. + """ + + model_cpu_offload_seq = "qformer->text_encoder->unet->vae" + + def __init__( + self, + tokenizer: CLIPTokenizer, + text_encoder: ContextCLIPTextModel, + vae: AutoencoderKL, + unet: UNet2DConditionModel, + scheduler: PNDMScheduler, + qformer: Blip2QFormerModel, + image_processor: BlipImageProcessor, + ctx_begin_pos: int = 2, + mean: List[float] = None, + std: List[float] = None, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + vae=vae, + unet=unet, + scheduler=scheduler, + qformer=qformer, + image_processor=image_processor, + ) + self.register_to_config(ctx_begin_pos=ctx_begin_pos, mean=mean, std=std) + + def get_query_embeddings(self, input_image, src_subject): + return self.qformer(image_input=input_image, text_input=src_subject, return_dict=False) + + # from the original Blip Diffusion code, speciefies the target subject and augments the prompt by repeating it + def _build_prompt(self, prompts, tgt_subjects, prompt_strength=1.0, prompt_reps=20): + rv = [] + for prompt, tgt_subject in zip(prompts, tgt_subjects): + prompt = f"a {tgt_subject} {prompt.strip()}" + # a trick to amplify the prompt + rv.append(", ".join([prompt] * int(prompt_strength * prompt_reps))) + + return rv + + # Copied from diffusers.pipelines.consistency_models.pipeline_consistency_models.ConsistencyModelPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels, height, width) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device=device, dtype=dtype) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def encode_prompt(self, query_embeds, prompt, device=None): + device = device or self._execution_device + + # embeddings for prompt, with query_embeds as context + max_len = self.text_encoder.text_model.config.max_position_embeddings + max_len -= self.qformer.config.num_query_tokens + + tokenized_prompt = self.tokenizer( + prompt, + padding="max_length", + truncation=True, + max_length=max_len, + return_tensors="pt", + ).to(device) + + batch_size = query_embeds.shape[0] + ctx_begin_pos = [self.config.ctx_begin_pos] * batch_size + + text_embeddings = self.text_encoder( + input_ids=tokenized_prompt.input_ids, + ctx_embeddings=query_embeds, + ctx_begin_pos=ctx_begin_pos, + )[0] + + return text_embeddings + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: List[str], + reference_image: PIL.Image.Image, + source_subject_category: List[str], + target_subject_category: List[str], + latents: Optional[torch.FloatTensor] = None, + guidance_scale: float = 7.5, + height: int = 512, + width: int = 512, + num_inference_steps: int = 50, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + neg_prompt: Optional[str] = "", + prompt_strength: float = 1.0, + prompt_reps: int = 20, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`List[str]`): + The prompt or prompts to guide the image generation. + reference_image (`PIL.Image.Image`): + The reference image to condition the generation on. + source_subject_category (`List[str]`): + The source subject category. + target_subject_category (`List[str]`): + The target subject category. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by random sampling. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + height (`int`, *optional*, defaults to 512): + The height of the generated image. + width (`int`, *optional*, defaults to 512): + The width of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + neg_prompt (`str`, *optional*, defaults to ""): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + prompt_strength (`float`, *optional*, defaults to 1.0): + The strength of the prompt. Specifies the number of times the prompt is repeated along with prompt_reps + to amplify the prompt. + prompt_reps (`int`, *optional*, defaults to 20): + The number of times the prompt is repeated along with prompt_strength to amplify the prompt. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + device = self._execution_device + + reference_image = self.image_processor.preprocess( + reference_image, image_mean=self.config.mean, image_std=self.config.std, return_tensors="pt" + )["pixel_values"] + reference_image = reference_image.to(device) + + if isinstance(prompt, str): + prompt = [prompt] + if isinstance(source_subject_category, str): + source_subject_category = [source_subject_category] + if isinstance(target_subject_category, str): + target_subject_category = [target_subject_category] + + batch_size = len(prompt) + + prompt = self._build_prompt( + prompts=prompt, + tgt_subjects=target_subject_category, + prompt_strength=prompt_strength, + prompt_reps=prompt_reps, + ) + query_embeds = self.get_query_embeddings(reference_image, source_subject_category) + text_embeddings = self.encode_prompt(query_embeds, prompt, device) + do_classifier_free_guidance = guidance_scale > 1.0 + if do_classifier_free_guidance: + max_length = self.text_encoder.text_model.config.max_position_embeddings + + uncond_input = self.tokenizer( + [neg_prompt] * batch_size, + padding="max_length", + max_length=max_length, + return_tensors="pt", + ) + uncond_embeddings = self.text_encoder( + input_ids=uncond_input.input_ids.to(device), + ctx_embeddings=None, + )[0] + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + text_embeddings = torch.cat([uncond_embeddings, text_embeddings]) + + scale_down_factor = 2 ** (len(self.unet.config.block_out_channels) - 1) + latents = self.prepare_latents( + batch_size=batch_size, + num_channels=self.unet.config.in_channels, + height=height // scale_down_factor, + width=width // scale_down_factor, + generator=generator, + latents=latents, + dtype=self.unet.dtype, + device=device, + ) + # set timesteps + extra_set_kwargs = {} + self.scheduler.set_timesteps(num_inference_steps, **extra_set_kwargs) + + for i, t in enumerate(self.progress_bar(self.scheduler.timesteps)): + # expand the latents if we are doing classifier free guidance + do_classifier_free_guidance = guidance_scale > 1.0 + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + noise_pred = self.unet( + latent_model_input, + timestep=t, + encoder_hidden_states=text_embeddings, + down_block_additional_residuals=None, + mid_block_additional_residual=None, + )["sample"] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + latents = self.scheduler.step( + noise_pred, + t, + latents, + )["prev_sample"] + + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/consistency_models/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/consistency_models/__init__.py new file mode 100755 index 0000000..162d91c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/consistency_models/__init__.py @@ -0,0 +1,24 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + _LazyModule, +) + + +_import_structure = { + "pipeline_consistency_models": ["ConsistencyModelPipeline"], +} + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .pipeline_consistency_models import ConsistencyModelPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/consistency_models/pipeline_consistency_models.py b/diffusers-0.27.0/src/diffusers/pipelines/consistency_models/pipeline_consistency_models.py new file mode 100755 index 0000000..befac79 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/consistency_models/pipeline_consistency_models.py @@ -0,0 +1,275 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, List, Optional, Union + +import torch + +from ...models import UNet2DModel +from ...schedulers import CMStochasticIterativeScheduler +from ...utils import ( + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + + >>> from diffusers import ConsistencyModelPipeline + + >>> device = "cuda" + >>> # Load the cd_imagenet64_l2 checkpoint. + >>> model_id_or_path = "openai/diffusers-cd_imagenet64_l2" + >>> pipe = ConsistencyModelPipeline.from_pretrained(model_id_or_path, torch_dtype=torch.float16) + >>> pipe.to(device) + + >>> # Onestep Sampling + >>> image = pipe(num_inference_steps=1).images[0] + >>> image.save("cd_imagenet64_l2_onestep_sample.png") + + >>> # Onestep sampling, class-conditional image generation + >>> # ImageNet-64 class label 145 corresponds to king penguins + >>> image = pipe(num_inference_steps=1, class_labels=145).images[0] + >>> image.save("cd_imagenet64_l2_onestep_sample_penguin.png") + + >>> # Multistep sampling, class-conditional image generation + >>> # Timesteps can be explicitly specified; the particular timesteps below are from the original Github repo: + >>> # https://github.com/openai/consistency_models/blob/main/scripts/launch.sh#L77 + >>> image = pipe(num_inference_steps=None, timesteps=[22, 0], class_labels=145).images[0] + >>> image.save("cd_imagenet64_l2_multistep_sample_penguin.png") + ``` +""" + + +class ConsistencyModelPipeline(DiffusionPipeline): + r""" + Pipeline for unconditional or class-conditional image generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + unet ([`UNet2DModel`]): + A `UNet2DModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Currently only + compatible with [`CMStochasticIterativeScheduler`]. + """ + + model_cpu_offload_seq = "unet" + + def __init__(self, unet: UNet2DModel, scheduler: CMStochasticIterativeScheduler) -> None: + super().__init__() + + self.register_modules( + unet=unet, + scheduler=scheduler, + ) + + self.safety_checker = None + + def prepare_latents(self, batch_size, num_channels, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels, height, width) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device=device, dtype=dtype) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + # Follows diffusers.VaeImageProcessor.postprocess + def postprocess_image(self, sample: torch.FloatTensor, output_type: str = "pil"): + if output_type not in ["pt", "np", "pil"]: + raise ValueError( + f"output_type={output_type} is not supported. Make sure to choose one of ['pt', 'np', or 'pil']" + ) + + # Equivalent to diffusers.VaeImageProcessor.denormalize + sample = (sample / 2 + 0.5).clamp(0, 1) + if output_type == "pt": + return sample + + # Equivalent to diffusers.VaeImageProcessor.pt_to_numpy + sample = sample.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "np": + return sample + + # Output_type must be 'pil' + sample = self.numpy_to_pil(sample) + return sample + + def prepare_class_labels(self, batch_size, device, class_labels=None): + if self.unet.config.num_class_embeds is not None: + if isinstance(class_labels, list): + class_labels = torch.tensor(class_labels, dtype=torch.int) + elif isinstance(class_labels, int): + assert batch_size == 1, "Batch size must be 1 if classes is an int" + class_labels = torch.tensor([class_labels], dtype=torch.int) + elif class_labels is None: + # Randomly generate batch_size class labels + # TODO: should use generator here? int analogue of randn_tensor is not exposed in ...utils + class_labels = torch.randint(0, self.unet.config.num_class_embeds, size=(batch_size,)) + class_labels = class_labels.to(device) + else: + class_labels = None + return class_labels + + def check_inputs(self, num_inference_steps, timesteps, latents, batch_size, img_size, callback_steps): + if num_inference_steps is None and timesteps is None: + raise ValueError("Exactly one of `num_inference_steps` or `timesteps` must be supplied.") + + if num_inference_steps is not None and timesteps is not None: + logger.warning( + f"Both `num_inference_steps`: {num_inference_steps} and `timesteps`: {timesteps} are supplied;" + " `timesteps` will be used over `num_inference_steps`." + ) + + if latents is not None: + expected_shape = (batch_size, 3, img_size, img_size) + if latents.shape != expected_shape: + raise ValueError(f"The shape of latents is {latents.shape} but is expected to be {expected_shape}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + batch_size: int = 1, + class_labels: Optional[Union[torch.Tensor, List[int], int]] = None, + num_inference_steps: int = 1, + timesteps: List[int] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + ): + r""" + Args: + batch_size (`int`, *optional*, defaults to 1): + The number of images to generate. + class_labels (`torch.Tensor` or `List[int]` or `int`, *optional*): + Optional class labels for conditioning class-conditional consistency models. Not used if the model is + not class-conditional. + num_inference_steps (`int`, *optional*, defaults to 1): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps are used. Must be in descending order. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images. + """ + # 0. Prepare call parameters + img_size = self.unet.config.sample_size + device = self._execution_device + + # 1. Check inputs + self.check_inputs(num_inference_steps, timesteps, latents, batch_size, img_size, callback_steps) + + # 2. Prepare image latents + # Sample image latents x_0 ~ N(0, sigma_0^2 * I) + sample = self.prepare_latents( + batch_size=batch_size, + num_channels=self.unet.config.in_channels, + height=img_size, + width=img_size, + dtype=self.unet.dtype, + device=device, + generator=generator, + latents=latents, + ) + + # 3. Handle class_labels for class-conditional models + class_labels = self.prepare_class_labels(batch_size, device, class_labels=class_labels) + + # 4. Prepare timesteps + if timesteps is not None: + self.scheduler.set_timesteps(timesteps=timesteps, device=device) + timesteps = self.scheduler.timesteps + num_inference_steps = len(timesteps) + else: + self.scheduler.set_timesteps(num_inference_steps) + timesteps = self.scheduler.timesteps + + # 5. Denoising loop + # Multistep sampling: implements Algorithm 1 in the paper + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + scaled_sample = self.scheduler.scale_model_input(sample, t) + model_output = self.unet(scaled_sample, t, class_labels=class_labels, return_dict=False)[0] + + sample = self.scheduler.step(model_output, t, sample, generator=generator)[0] + + # call the callback, if provided + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, sample) + + # 6. Post-process image sample + image = self.postprocess_image(sample, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/controlnet/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/__init__.py new file mode 100755 index 0000000..b167105 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/__init__.py @@ -0,0 +1,80 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_flax_available, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["multicontrolnet"] = ["MultiControlNetModel"] + _import_structure["pipeline_controlnet"] = ["StableDiffusionControlNetPipeline"] + _import_structure["pipeline_controlnet_blip_diffusion"] = ["BlipDiffusionControlNetPipeline"] + _import_structure["pipeline_controlnet_img2img"] = ["StableDiffusionControlNetImg2ImgPipeline"] + _import_structure["pipeline_controlnet_inpaint"] = ["StableDiffusionControlNetInpaintPipeline"] + _import_structure["pipeline_controlnet_inpaint_sd_xl"] = ["StableDiffusionXLControlNetInpaintPipeline"] + _import_structure["pipeline_controlnet_sd_xl"] = ["StableDiffusionXLControlNetPipeline"] + _import_structure["pipeline_controlnet_sd_xl_img2img"] = ["StableDiffusionXLControlNetImg2ImgPipeline"] +try: + if not (is_transformers_available() and is_flax_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_flax_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_flax_and_transformers_objects)) +else: + _import_structure["pipeline_flax_controlnet"] = ["FlaxStableDiffusionControlNetPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .multicontrolnet import MultiControlNetModel + from .pipeline_controlnet import StableDiffusionControlNetPipeline + from .pipeline_controlnet_blip_diffusion import BlipDiffusionControlNetPipeline + from .pipeline_controlnet_img2img import StableDiffusionControlNetImg2ImgPipeline + from .pipeline_controlnet_inpaint import StableDiffusionControlNetInpaintPipeline + from .pipeline_controlnet_inpaint_sd_xl import StableDiffusionXLControlNetInpaintPipeline + from .pipeline_controlnet_sd_xl import StableDiffusionXLControlNetPipeline + from .pipeline_controlnet_sd_xl_img2img import StableDiffusionXLControlNetImg2ImgPipeline + + try: + if not (is_transformers_available() and is_flax_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_flax_and_transformers_objects import * # noqa F403 + else: + from .pipeline_flax_controlnet import FlaxStableDiffusionControlNetPipeline + + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/controlnet/multicontrolnet.py b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/multicontrolnet.py new file mode 100755 index 0000000..7d284f2 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/multicontrolnet.py @@ -0,0 +1,187 @@ +import os +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from torch import nn + +from ...models.controlnet import ControlNetModel, ControlNetOutput +from ...models.modeling_utils import ModelMixin +from ...utils import logging + + +logger = logging.get_logger(__name__) + + +class MultiControlNetModel(ModelMixin): + r""" + Multiple `ControlNetModel` wrapper class for Multi-ControlNet + + This module is a wrapper for multiple instances of the `ControlNetModel`. The `forward()` API is designed to be + compatible with `ControlNetModel`. + + Args: + controlnets (`List[ControlNetModel]`): + Provides additional conditioning to the unet during the denoising process. You must set multiple + `ControlNetModel` as a list. + """ + + def __init__(self, controlnets: Union[List[ControlNetModel], Tuple[ControlNetModel]]): + super().__init__() + self.nets = nn.ModuleList(controlnets) + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + controlnet_cond: List[torch.tensor], + conditioning_scale: List[float], + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guess_mode: bool = False, + return_dict: bool = True, + ) -> Union[ControlNetOutput, Tuple]: + for i, (image, scale, controlnet) in enumerate(zip(controlnet_cond, conditioning_scale, self.nets)): + down_samples, mid_sample = controlnet( + sample=sample, + timestep=timestep, + encoder_hidden_states=encoder_hidden_states, + controlnet_cond=image, + conditioning_scale=scale, + class_labels=class_labels, + timestep_cond=timestep_cond, + attention_mask=attention_mask, + added_cond_kwargs=added_cond_kwargs, + cross_attention_kwargs=cross_attention_kwargs, + guess_mode=guess_mode, + return_dict=return_dict, + ) + + # merge samples + if i == 0: + down_block_res_samples, mid_block_res_sample = down_samples, mid_sample + else: + down_block_res_samples = [ + samples_prev + samples_curr + for samples_prev, samples_curr in zip(down_block_res_samples, down_samples) + ] + mid_block_res_sample += mid_sample + + return down_block_res_samples, mid_block_res_sample + + def save_pretrained( + self, + save_directory: Union[str, os.PathLike], + is_main_process: bool = True, + save_function: Callable = None, + safe_serialization: bool = True, + variant: Optional[str] = None, + ): + """ + Save a model and its configuration file to a directory, so that it can be re-loaded using the + `[`~pipelines.controlnet.MultiControlNetModel.from_pretrained`]` class method. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to which to save. Will be created if it doesn't exist. + is_main_process (`bool`, *optional*, defaults to `True`): + Whether the process calling this is the main process or not. Useful when in distributed training like + TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on + the main process to avoid race conditions. + save_function (`Callable`): + The function to use to save the state dictionary. Useful on distributed training like TPUs when one + need to replace `torch.save` by another method. Can be configured with the environment variable + `DIFFUSERS_SAVE_MODE`. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether to save the model using `safetensors` or the traditional PyTorch way (that uses `pickle`). + variant (`str`, *optional*): + If specified, weights are saved in the format pytorch_model..bin. + """ + idx = 0 + model_path_to_save = save_directory + for controlnet in self.nets: + controlnet.save_pretrained( + model_path_to_save, + is_main_process=is_main_process, + save_function=save_function, + safe_serialization=safe_serialization, + variant=variant, + ) + + idx += 1 + model_path_to_save = model_path_to_save + f"_{idx}" + + @classmethod + def from_pretrained(cls, pretrained_model_path: Optional[Union[str, os.PathLike]], **kwargs): + r""" + Instantiate a pretrained MultiControlNet model from multiple pre-trained controlnet models. + + The model is set in evaluation mode by default using `model.eval()` (Dropout modules are deactivated). To train + the model, you should first set it back in training mode with `model.train()`. + + The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come + pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning + task. + + The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those + weights are discarded. + + Parameters: + pretrained_model_path (`os.PathLike`): + A path to a *directory* containing model weights saved using + [`~diffusers.pipelines.controlnet.MultiControlNetModel.save_pretrained`], e.g., + `./my_model_directory/controlnet`. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model under this dtype. If `"auto"` is passed the dtype + will be automatically derived from the model's weights. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn't need to be refined to each + parameter/buffer name, once a given module name is inside, every submodule of it will be sent to the + same device. + + To have Accelerate compute the most optimized `device_map` automatically, set `device_map="auto"`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier to maximum memory. Will default to the maximum memory available for each + GPU and the available CPU RAM if unset. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading by not initializing the weights and only loading the pre-trained weights. This + also tries to not use more than 1x model size in CPU memory (including peak memory) while loading the + model. This is only supported when torch version >= 1.9.0. If you are using an older version of torch, + setting this argument to `True` will raise an error. + variant (`str`, *optional*): + If specified load weights from `variant` filename, *e.g.* pytorch_model..bin. `variant` is + ignored when using `from_flax`. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the `safetensors` weights will be downloaded if they're available **and** if the + `safetensors` library is installed. If set to `True`, the model will be forcibly loaded from + `safetensors` weights. If set to `False`, loading will *not* use `safetensors`. + """ + idx = 0 + controlnets = [] + + # load controlnet and append to list until no controlnet directory exists anymore + # first controlnet has to be saved under `./mydirectory/controlnet` to be compliant with `DiffusionPipeline.from_prertained` + # second, third, ... controlnets have to be saved under `./mydirectory/controlnet_1`, `./mydirectory/controlnet_2`, ... + model_path_to_load = pretrained_model_path + while os.path.isdir(model_path_to_load): + controlnet = ControlNetModel.from_pretrained(model_path_to_load, **kwargs) + controlnets.append(controlnet) + + idx += 1 + model_path_to_load = pretrained_model_path + f"_{idx}" + + logger.info(f"{len(controlnets)} controlnets loaded from {pretrained_model_path}.") + + if len(controlnets) == 0: + raise ValueError( + f"No ControlNets found under {os.path.dirname(pretrained_model_path)}. Expected at least {pretrained_model_path + '_0'}." + ) + + return cls(controlnets) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet.py b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet.py new file mode 100755 index 0000000..8f31dfc --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet.py @@ -0,0 +1,1318 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ControlNetModel, ImageProjection, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import is_compiled_module, is_torch_version, randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion.pipeline_output import StableDiffusionPipelineOutput +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from .multicontrolnet import MultiControlNetModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> # !pip install opencv-python transformers accelerate + >>> from diffusers import StableDiffusionControlNetPipeline, ControlNetModel, UniPCMultistepScheduler + >>> from diffusers.utils import load_image + >>> import numpy as np + >>> import torch + + >>> import cv2 + >>> from PIL import Image + + >>> # download an image + >>> image = load_image( + ... "https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png" + ... ) + >>> image = np.array(image) + + >>> # get canny image + >>> image = cv2.Canny(image, 100, 200) + >>> image = image[:, :, None] + >>> image = np.concatenate([image, image, image], axis=2) + >>> canny_image = Image.fromarray(image) + + >>> # load control net and stable diffusion v1-5 + >>> controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16) + >>> pipe = StableDiffusionControlNetPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16 + ... ) + + >>> # speed up diffusion process with faster scheduler and memory optimization + >>> pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) + >>> # remove following line if xformers is not installed + >>> pipe.enable_xformers_memory_efficient_attention() + + >>> pipe.enable_model_cpu_offload() + + >>> # generate image + >>> generator = torch.manual_seed(0) + >>> image = pipe( + ... "futuristic-looking woman", num_inference_steps=20, generator=generator, image=canny_image + ... ).images[0] + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class StableDiffusionControlNetPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + LoraLoaderMixin, + IPAdapterMixin, + FromSingleFileMixin, +): + r""" + Pipeline for text-to-image generation using Stable Diffusion with ControlNet guidance. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + Provides additional conditioning to the `unet` during the denoising process. If you set multiple + ControlNets as a list, the outputs from each ControlNet are added together to create one combined + additional conditioning. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: CLIPVisionModelWithProjection = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + if isinstance(controlnet, (list, tuple)): + controlnet = MultiControlNetModel(controlnet) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + controlnet=controlnet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True) + self.control_image_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=False + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + image, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + controlnet_conditioning_scale=1.0, + control_guidance_start=0.0, + control_guidance_end=1.0, + callback_on_step_end_tensor_inputs=None, + ): + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Check `image` + is_compiled = hasattr(F, "scaled_dot_product_attention") and isinstance( + self.controlnet, torch._dynamo.eval_frame.OptimizedModule + ) + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + self.check_image(image, prompt, prompt_embeds) + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if not isinstance(image, list): + raise TypeError("For multiple controlnets: `image` must be type `list`") + + # When `image` is a nested list: + # (e.g. [[canny_image_1, pose_image_1], [canny_image_2, pose_image_2]]) + elif any(isinstance(i, list) for i in image): + transposed_image = [list(t) for t in zip(*image)] + if len(transposed_image) != len(self.controlnet.nets): + raise ValueError( + f"For multiple controlnets: if you pass`image` as a list of list, each sublist must have the same length as the number of controlnets, but the sublists in `image` got {len(transposed_image)} images and {len(self.controlnet.nets)} ControlNets." + ) + for image_ in transposed_image: + self.check_image(image_, prompt, prompt_embeds) + elif len(image) != len(self.controlnet.nets): + raise ValueError( + f"For multiple controlnets: `image` must have the same length as the number of controlnets, but got {len(image)} images and {len(self.controlnet.nets)} ControlNets." + ) + + for image_ in image: + self.check_image(image_, prompt, prompt_embeds) + else: + assert False + + # Check `controlnet_conditioning_scale` + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + if not isinstance(controlnet_conditioning_scale, float): + raise TypeError("For single controlnet: `controlnet_conditioning_scale` must be type `float`.") + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if isinstance(controlnet_conditioning_scale, list): + if any(isinstance(i, list) for i in controlnet_conditioning_scale): + raise ValueError( + "A single batch of varying conditioning scale settings (e.g. [[1.0, 0.5], [0.2, 0.8]]) is not supported at the moment. " + "The conditioning scale must be fixed across the batch." + ) + elif isinstance(controlnet_conditioning_scale, list) and len(controlnet_conditioning_scale) != len( + self.controlnet.nets + ): + raise ValueError( + "For multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have" + " the same length as the number of controlnets" + ) + else: + assert False + + if not isinstance(control_guidance_start, (tuple, list)): + control_guidance_start = [control_guidance_start] + + if not isinstance(control_guidance_end, (tuple, list)): + control_guidance_end = [control_guidance_end] + + if len(control_guidance_start) != len(control_guidance_end): + raise ValueError( + f"`control_guidance_start` has {len(control_guidance_start)} elements, but `control_guidance_end` has {len(control_guidance_end)} elements. Make sure to provide the same number of elements to each list." + ) + + if isinstance(self.controlnet, MultiControlNetModel): + if len(control_guidance_start) != len(self.controlnet.nets): + raise ValueError( + f"`control_guidance_start`: {control_guidance_start} has {len(control_guidance_start)} elements but there are {len(self.controlnet.nets)} controlnets available. Make sure to provide {len(self.controlnet.nets)}." + ) + + for start, end in zip(control_guidance_start, control_guidance_end): + if start >= end: + raise ValueError( + f"control guidance start: {start} cannot be larger or equal to control guidance end: {end}." + ) + if start < 0.0: + raise ValueError(f"control guidance start: {start} can't be smaller than 0.") + if end > 1.0: + raise ValueError(f"control guidance end: {end} can't be larger than 1.0.") + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + def check_image(self, image, prompt, prompt_embeds): + image_is_pil = isinstance(image, PIL.Image.Image) + image_is_tensor = isinstance(image, torch.Tensor) + image_is_np = isinstance(image, np.ndarray) + image_is_pil_list = isinstance(image, list) and isinstance(image[0], PIL.Image.Image) + image_is_tensor_list = isinstance(image, list) and isinstance(image[0], torch.Tensor) + image_is_np_list = isinstance(image, list) and isinstance(image[0], np.ndarray) + + if ( + not image_is_pil + and not image_is_tensor + and not image_is_np + and not image_is_pil_list + and not image_is_tensor_list + and not image_is_np_list + ): + raise TypeError( + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" + ) + + if image_is_pil: + image_batch_size = 1 + else: + image_batch_size = len(image) + + if prompt is not None and isinstance(prompt, str): + prompt_batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + prompt_batch_size = len(prompt) + elif prompt_embeds is not None: + prompt_batch_size = prompt_embeds.shape[0] + + if image_batch_size != 1 and image_batch_size != prompt_batch_size: + raise ValueError( + f"If image batch size is not 1, image batch size must be same as prompt batch size. image batch size: {image_batch_size}, prompt batch size: {prompt_batch_size}" + ) + + def prepare_image( + self, + image, + width, + height, + batch_size, + num_images_per_prompt, + device, + dtype, + do_classifier_free_guidance=False, + guess_mode=False, + ): + image = self.control_image_processor.preprocess(image, height=height, width=width).to(dtype=torch.float32) + image_batch_size = image.shape[0] + + if image_batch_size == 1: + repeat_by = batch_size + else: + # image batch size is the same as prompt batch size + repeat_by = num_images_per_prompt + + image = image.repeat_interleave(repeat_by, dim=0) + + image = image.to(device=device, dtype=dtype) + + if do_classifier_free_guidance and not guess_mode: + image = torch.cat([image] * 2) + + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + timesteps: List[int] = None, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + guess_mode: bool = False, + control_guidance_start: Union[float, List[float]] = 0.0, + control_guidance_end: Union[float, List[float]] = 1.0, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: + `List[List[torch.FloatTensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + The ControlNet input condition to provide guidance to the `unet` for generation. If the type is + specified as `torch.FloatTensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be + accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height + and/or width are passed, `image` is resized accordingly. If multiple ControlNets are specified in + `init`, images must be passed as a list such that each element of the list can be correctly batched for + input to a single ControlNet. When `prompt` is a list, and if a list of images is passed for a single ControlNet, + each will be paired with each prompt in the `prompt` list. This also applies to multiple ControlNets, + where a list of image lists can be passed to batch for each prompt and each ControlNet. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added + to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set + the corresponding scale as a list. + guess_mode (`bool`, *optional*, defaults to `False`): + The ControlNet encoder tries to recognize the content of the input image even if you remove all + prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. + control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + The percentage of total steps at which the ControlNet starts applying. + control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + The percentage of total steps at which the ControlNet stops applying. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeine class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + + controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet + + # align format for control guidance + if not isinstance(control_guidance_start, list) and isinstance(control_guidance_end, list): + control_guidance_start = len(control_guidance_end) * [control_guidance_start] + elif not isinstance(control_guidance_end, list) and isinstance(control_guidance_start, list): + control_guidance_end = len(control_guidance_start) * [control_guidance_end] + elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list): + mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1 + control_guidance_start, control_guidance_end = ( + mult * [control_guidance_start], + mult * [control_guidance_end], + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + image, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + controlnet_conditioning_scale, + control_guidance_start, + control_guidance_end, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float): + controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets) + + global_pool_conditions = ( + controlnet.config.global_pool_conditions + if isinstance(controlnet, ControlNetModel) + else controlnet.nets[0].config.global_pool_conditions + ) + guess_mode = guess_mode or global_pool_conditions + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Prepare image + if isinstance(controlnet, ControlNetModel): + image = self.prepare_image( + image=image, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + height, width = image.shape[-2:] + elif isinstance(controlnet, MultiControlNetModel): + images = [] + + # Nested lists as ControlNet condition + if isinstance(image[0], list): + # Transpose the nested image list + image = [list(t) for t in zip(*image)] + + for image_ in image: + image_ = self.prepare_image( + image=image_, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + + images.append(image_) + + image = images + height, width = image[0].shape[-2:] + else: + assert False + + # 5. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + self._num_timesteps = len(timesteps) + + # 6. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6.5 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + # 7.2 Create tensor stating which controlnets to keep + controlnet_keep = [] + for i in range(len(timesteps)): + keeps = [ + 1.0 - float(i / len(timesteps) < s or (i + 1) / len(timesteps) > e) + for s, e in zip(control_guidance_start, control_guidance_end) + ] + controlnet_keep.append(keeps[0] if isinstance(controlnet, ControlNetModel) else keeps) + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + is_unet_compiled = is_compiled_module(self.unet) + is_controlnet_compiled = is_compiled_module(self.controlnet) + is_torch_higher_equal_2_1 = is_torch_version(">=", "2.1") + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # Relevant thread: + # https://dev-discuss.pytorch.org/t/cudagraphs-in-pytorch-2-0/1428 + if (is_unet_compiled and is_controlnet_compiled) and is_torch_higher_equal_2_1: + torch._inductor.cudagraph_mark_step_begin() + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # controlnet(s) inference + if guess_mode and self.do_classifier_free_guidance: + # Infer ControlNet only for the conditional batch. + control_model_input = latents + control_model_input = self.scheduler.scale_model_input(control_model_input, t) + controlnet_prompt_embeds = prompt_embeds.chunk(2)[1] + else: + control_model_input = latent_model_input + controlnet_prompt_embeds = prompt_embeds + + if isinstance(controlnet_keep[i], list): + cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] + else: + controlnet_cond_scale = controlnet_conditioning_scale + if isinstance(controlnet_cond_scale, list): + controlnet_cond_scale = controlnet_cond_scale[0] + cond_scale = controlnet_cond_scale * controlnet_keep[i] + + down_block_res_samples, mid_block_res_sample = self.controlnet( + control_model_input, + t, + encoder_hidden_states=controlnet_prompt_embeds, + controlnet_cond=image, + conditioning_scale=cond_scale, + guess_mode=guess_mode, + return_dict=False, + ) + + if guess_mode and self.do_classifier_free_guidance: + # Infered ControlNet only for the conditional batch. + # To apply the output of ControlNet to both the unconditional and conditional batches, + # add 0 to the unconditional batch to keep it unchanged. + down_block_res_samples = [torch.cat([torch.zeros_like(d), d]) for d in down_block_res_samples] + mid_block_res_sample = torch.cat([torch.zeros_like(mid_block_res_sample), mid_block_res_sample]) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # If we do sequential model offloading, let's offload unet and controlnet + # manually for max memory savings + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.unet.to("cpu") + self.controlnet.to("cpu") + torch.cuda.empty_cache() + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ + 0 + ] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py new file mode 100755 index 0000000..b983a3f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_blip_diffusion.py @@ -0,0 +1,413 @@ +# Copyright 2024 Salesforce.com, inc. +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import List, Optional, Union + +import PIL.Image +import torch +from transformers import CLIPTokenizer + +from ...models import AutoencoderKL, ControlNetModel, UNet2DConditionModel +from ...schedulers import PNDMScheduler +from ...utils import ( + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..blip_diffusion.blip_image_processing import BlipImageProcessor +from ..blip_diffusion.modeling_blip2 import Blip2QFormerModel +from ..blip_diffusion.modeling_ctx_clip import ContextCLIPTextModel +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers.pipelines import BlipDiffusionControlNetPipeline + >>> from diffusers.utils import load_image + >>> from controlnet_aux import CannyDetector + >>> import torch + + >>> blip_diffusion_pipe = BlipDiffusionControlNetPipeline.from_pretrained( + ... "Salesforce/blipdiffusion-controlnet", torch_dtype=torch.float16 + ... ).to("cuda") + + >>> style_subject = "flower" + >>> tgt_subject = "teapot" + >>> text_prompt = "on a marble table" + + >>> cldm_cond_image = load_image( + ... "https://huggingface.co/datasets/ayushtues/blipdiffusion_images/resolve/main/kettle.jpg" + ... ).resize((512, 512)) + >>> canny = CannyDetector() + >>> cldm_cond_image = canny(cldm_cond_image, 30, 70, output_type="pil") + >>> style_image = load_image( + ... "https://huggingface.co/datasets/ayushtues/blipdiffusion_images/resolve/main/flower.jpg" + ... ) + >>> guidance_scale = 7.5 + >>> num_inference_steps = 50 + >>> negative_prompt = "over-exposure, under-exposure, saturated, duplicate, out of frame, lowres, cropped, worst quality, low quality, jpeg artifacts, morbid, mutilated, out of frame, ugly, bad anatomy, bad proportions, deformed, blurry, duplicate" + + + >>> output = blip_diffusion_pipe( + ... text_prompt, + ... style_image, + ... cldm_cond_image, + ... style_subject, + ... tgt_subject, + ... guidance_scale=guidance_scale, + ... num_inference_steps=num_inference_steps, + ... neg_prompt=negative_prompt, + ... height=512, + ... width=512, + ... ).images + >>> output[0].save("image.png") + ``` +""" + + +class BlipDiffusionControlNetPipeline(DiffusionPipeline): + """ + Pipeline for Canny Edge based Controlled subject-driven generation using Blip Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + tokenizer ([`CLIPTokenizer`]): + Tokenizer for the text encoder + text_encoder ([`ContextCLIPTextModel`]): + Text encoder to encode the text prompt + vae ([`AutoencoderKL`]): + VAE model to map the latents to the image + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + scheduler ([`PNDMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + qformer ([`Blip2QFormerModel`]): + QFormer model to get multi-modal embeddings from the text and image. + controlnet ([`ControlNetModel`]): + ControlNet model to get the conditioning image embedding. + image_processor ([`BlipImageProcessor`]): + Image Processor to preprocess and postprocess the image. + ctx_begin_pos (int, `optional`, defaults to 2): + Position of the context token in the text encoder. + """ + + model_cpu_offload_seq = "qformer->text_encoder->unet->vae" + + def __init__( + self, + tokenizer: CLIPTokenizer, + text_encoder: ContextCLIPTextModel, + vae: AutoencoderKL, + unet: UNet2DConditionModel, + scheduler: PNDMScheduler, + qformer: Blip2QFormerModel, + controlnet: ControlNetModel, + image_processor: BlipImageProcessor, + ctx_begin_pos: int = 2, + mean: List[float] = None, + std: List[float] = None, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + vae=vae, + unet=unet, + scheduler=scheduler, + qformer=qformer, + controlnet=controlnet, + image_processor=image_processor, + ) + self.register_to_config(ctx_begin_pos=ctx_begin_pos, mean=mean, std=std) + + def get_query_embeddings(self, input_image, src_subject): + return self.qformer(image_input=input_image, text_input=src_subject, return_dict=False) + + # from the original Blip Diffusion code, speciefies the target subject and augments the prompt by repeating it + def _build_prompt(self, prompts, tgt_subjects, prompt_strength=1.0, prompt_reps=20): + rv = [] + for prompt, tgt_subject in zip(prompts, tgt_subjects): + prompt = f"a {tgt_subject} {prompt.strip()}" + # a trick to amplify the prompt + rv.append(", ".join([prompt] * int(prompt_strength * prompt_reps))) + + return rv + + # Copied from diffusers.pipelines.consistency_models.pipeline_consistency_models.ConsistencyModelPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels, height, width) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device=device, dtype=dtype) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def encode_prompt(self, query_embeds, prompt, device=None): + device = device or self._execution_device + + # embeddings for prompt, with query_embeds as context + max_len = self.text_encoder.text_model.config.max_position_embeddings + max_len -= self.qformer.config.num_query_tokens + + tokenized_prompt = self.tokenizer( + prompt, + padding="max_length", + truncation=True, + max_length=max_len, + return_tensors="pt", + ).to(device) + + batch_size = query_embeds.shape[0] + ctx_begin_pos = [self.config.ctx_begin_pos] * batch_size + + text_embeddings = self.text_encoder( + input_ids=tokenized_prompt.input_ids, + ctx_embeddings=query_embeds, + ctx_begin_pos=ctx_begin_pos, + )[0] + + return text_embeddings + + # Adapted from diffusers.pipelines.controlnet.pipeline_controlnet.StableDiffusionControlNetPipeline.prepare_image + def prepare_control_image( + self, + image, + width, + height, + batch_size, + num_images_per_prompt, + device, + dtype, + do_classifier_free_guidance=False, + ): + image = self.image_processor.preprocess( + image, + size={"width": width, "height": height}, + do_rescale=True, + do_center_crop=False, + do_normalize=False, + return_tensors="pt", + )["pixel_values"].to(device) + image_batch_size = image.shape[0] + + if image_batch_size == 1: + repeat_by = batch_size + else: + # image batch size is the same as prompt batch size + repeat_by = num_images_per_prompt + + image = image.repeat_interleave(repeat_by, dim=0) + + image = image.to(device=device, dtype=dtype) + + if do_classifier_free_guidance: + image = torch.cat([image] * 2) + + return image + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: List[str], + reference_image: PIL.Image.Image, + condtioning_image: PIL.Image.Image, + source_subject_category: List[str], + target_subject_category: List[str], + latents: Optional[torch.FloatTensor] = None, + guidance_scale: float = 7.5, + height: int = 512, + width: int = 512, + num_inference_steps: int = 50, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + neg_prompt: Optional[str] = "", + prompt_strength: float = 1.0, + prompt_reps: int = 20, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`List[str]`): + The prompt or prompts to guide the image generation. + reference_image (`PIL.Image.Image`): + The reference image to condition the generation on. + condtioning_image (`PIL.Image.Image`): + The conditioning canny edge image to condition the generation on. + source_subject_category (`List[str]`): + The source subject category. + target_subject_category (`List[str]`): + The target subject category. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by random sampling. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + height (`int`, *optional*, defaults to 512): + The height of the generated image. + width (`int`, *optional*, defaults to 512): + The width of the generated image. + seed (`int`, *optional*, defaults to 42): + The seed to use for random generation. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + neg_prompt (`str`, *optional*, defaults to ""): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + prompt_strength (`float`, *optional*, defaults to 1.0): + The strength of the prompt. Specifies the number of times the prompt is repeated along with prompt_reps + to amplify the prompt. + prompt_reps (`int`, *optional*, defaults to 20): + The number of times the prompt is repeated along with prompt_strength to amplify the prompt. + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + device = self._execution_device + + reference_image = self.image_processor.preprocess( + reference_image, image_mean=self.config.mean, image_std=self.config.std, return_tensors="pt" + )["pixel_values"] + reference_image = reference_image.to(device) + + if isinstance(prompt, str): + prompt = [prompt] + if isinstance(source_subject_category, str): + source_subject_category = [source_subject_category] + if isinstance(target_subject_category, str): + target_subject_category = [target_subject_category] + + batch_size = len(prompt) + + prompt = self._build_prompt( + prompts=prompt, + tgt_subjects=target_subject_category, + prompt_strength=prompt_strength, + prompt_reps=prompt_reps, + ) + query_embeds = self.get_query_embeddings(reference_image, source_subject_category) + text_embeddings = self.encode_prompt(query_embeds, prompt, device) + # 3. unconditional embedding + do_classifier_free_guidance = guidance_scale > 1.0 + if do_classifier_free_guidance: + max_length = self.text_encoder.text_model.config.max_position_embeddings + + uncond_input = self.tokenizer( + [neg_prompt] * batch_size, + padding="max_length", + max_length=max_length, + return_tensors="pt", + ) + uncond_embeddings = self.text_encoder( + input_ids=uncond_input.input_ids.to(device), + ctx_embeddings=None, + )[0] + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + text_embeddings = torch.cat([uncond_embeddings, text_embeddings]) + scale_down_factor = 2 ** (len(self.unet.config.block_out_channels) - 1) + latents = self.prepare_latents( + batch_size=batch_size, + num_channels=self.unet.config.in_channels, + height=height // scale_down_factor, + width=width // scale_down_factor, + generator=generator, + latents=latents, + dtype=self.unet.dtype, + device=device, + ) + # set timesteps + extra_set_kwargs = {} + self.scheduler.set_timesteps(num_inference_steps, **extra_set_kwargs) + + cond_image = self.prepare_control_image( + image=condtioning_image, + width=width, + height=height, + batch_size=batch_size, + num_images_per_prompt=1, + device=device, + dtype=self.controlnet.dtype, + do_classifier_free_guidance=do_classifier_free_guidance, + ) + + for i, t in enumerate(self.progress_bar(self.scheduler.timesteps)): + # expand the latents if we are doing classifier free guidance + do_classifier_free_guidance = guidance_scale > 1.0 + + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + down_block_res_samples, mid_block_res_sample = self.controlnet( + latent_model_input, + t, + encoder_hidden_states=text_embeddings, + controlnet_cond=cond_image, + return_dict=False, + ) + + noise_pred = self.unet( + latent_model_input, + timestep=t, + encoder_hidden_states=text_embeddings, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + )["sample"] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + latents = self.scheduler.step( + noise_pred, + t, + latents, + )["prev_sample"] + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py new file mode 100755 index 0000000..9d2c76f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_img2img.py @@ -0,0 +1,1310 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ControlNetModel, ImageProjection, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import is_compiled_module, randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from .multicontrolnet import MultiControlNetModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> # !pip install opencv-python transformers accelerate + >>> from diffusers import StableDiffusionControlNetImg2ImgPipeline, ControlNetModel, UniPCMultistepScheduler + >>> from diffusers.utils import load_image + >>> import numpy as np + >>> import torch + + >>> import cv2 + >>> from PIL import Image + + >>> # download an image + >>> image = load_image( + ... "https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/input_image_vermeer.png" + ... ) + >>> np_image = np.array(image) + + >>> # get canny image + >>> np_image = cv2.Canny(np_image, 100, 200) + >>> np_image = np_image[:, :, None] + >>> np_image = np.concatenate([np_image, np_image, np_image], axis=2) + >>> canny_image = Image.fromarray(np_image) + + >>> # load control net and stable diffusion v1-5 + >>> controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny", torch_dtype=torch.float16) + >>> pipe = StableDiffusionControlNetImg2ImgPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16 + ... ) + + >>> # speed up diffusion process with faster scheduler and memory optimization + >>> pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) + >>> pipe.enable_model_cpu_offload() + + >>> # generate image + >>> generator = torch.manual_seed(0) + >>> image = pipe( + ... "futuristic-looking woman", + ... num_inference_steps=20, + ... generator=generator, + ... image=image, + ... control_image=canny_image, + ... ).images[0] + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +def prepare_image(image): + if isinstance(image, torch.Tensor): + # Batch single image + if image.ndim == 3: + image = image.unsqueeze(0) + + image = image.to(dtype=torch.float32) + else: + # preprocess image + if isinstance(image, (PIL.Image.Image, np.ndarray)): + image = [image] + + if isinstance(image, list) and isinstance(image[0], PIL.Image.Image): + image = [np.array(i.convert("RGB"))[None, :] for i in image] + image = np.concatenate(image, axis=0) + elif isinstance(image, list) and isinstance(image[0], np.ndarray): + image = np.concatenate([i[None, :] for i in image], axis=0) + + image = image.transpose(0, 3, 1, 2) + image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0 + + return image + + +class StableDiffusionControlNetImg2ImgPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + LoraLoaderMixin, + IPAdapterMixin, + FromSingleFileMixin, +): + r""" + Pipeline for image-to-image generation using Stable Diffusion with ControlNet guidance. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + Provides additional conditioning to the `unet` during the denoising process. If you set multiple + ControlNets as a list, the outputs from each ControlNet are added together to create one combined + additional conditioning. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: CLIPVisionModelWithProjection = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + if isinstance(controlnet, (list, tuple)): + controlnet = MultiControlNetModel(controlnet) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + controlnet=controlnet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True) + self.control_image_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=False + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + image, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + controlnet_conditioning_scale=1.0, + control_guidance_start=0.0, + control_guidance_end=1.0, + callback_on_step_end_tensor_inputs=None, + ): + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # `prompt` needs more sophisticated handling when there are multiple + # conditionings. + if isinstance(self.controlnet, MultiControlNetModel): + if isinstance(prompt, list): + logger.warning( + f"You have {len(self.controlnet.nets)} ControlNets and you have passed {len(prompt)}" + " prompts. The conditionings will be fixed across the prompts." + ) + + # Check `image` + is_compiled = hasattr(F, "scaled_dot_product_attention") and isinstance( + self.controlnet, torch._dynamo.eval_frame.OptimizedModule + ) + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + self.check_image(image, prompt, prompt_embeds) + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if not isinstance(image, list): + raise TypeError("For multiple controlnets: `image` must be type `list`") + + # When `image` is a nested list: + # (e.g. [[canny_image_1, pose_image_1], [canny_image_2, pose_image_2]]) + elif any(isinstance(i, list) for i in image): + raise ValueError("A single batch of multiple conditionings are supported at the moment.") + elif len(image) != len(self.controlnet.nets): + raise ValueError( + f"For multiple controlnets: `image` must have the same length as the number of controlnets, but got {len(image)} images and {len(self.controlnet.nets)} ControlNets." + ) + + for image_ in image: + self.check_image(image_, prompt, prompt_embeds) + else: + assert False + + # Check `controlnet_conditioning_scale` + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + if not isinstance(controlnet_conditioning_scale, float): + raise TypeError("For single controlnet: `controlnet_conditioning_scale` must be type `float`.") + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if isinstance(controlnet_conditioning_scale, list): + if any(isinstance(i, list) for i in controlnet_conditioning_scale): + raise ValueError("A single batch of multiple conditionings are supported at the moment.") + elif isinstance(controlnet_conditioning_scale, list) and len(controlnet_conditioning_scale) != len( + self.controlnet.nets + ): + raise ValueError( + "For multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have" + " the same length as the number of controlnets" + ) + else: + assert False + + if len(control_guidance_start) != len(control_guidance_end): + raise ValueError( + f"`control_guidance_start` has {len(control_guidance_start)} elements, but `control_guidance_end` has {len(control_guidance_end)} elements. Make sure to provide the same number of elements to each list." + ) + + if isinstance(self.controlnet, MultiControlNetModel): + if len(control_guidance_start) != len(self.controlnet.nets): + raise ValueError( + f"`control_guidance_start`: {control_guidance_start} has {len(control_guidance_start)} elements but there are {len(self.controlnet.nets)} controlnets available. Make sure to provide {len(self.controlnet.nets)}." + ) + + for start, end in zip(control_guidance_start, control_guidance_end): + if start >= end: + raise ValueError( + f"control guidance start: {start} cannot be larger or equal to control guidance end: {end}." + ) + if start < 0.0: + raise ValueError(f"control guidance start: {start} can't be smaller than 0.") + if end > 1.0: + raise ValueError(f"control guidance end: {end} can't be larger than 1.0.") + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + # Copied from diffusers.pipelines.controlnet.pipeline_controlnet.StableDiffusionControlNetPipeline.check_image + def check_image(self, image, prompt, prompt_embeds): + image_is_pil = isinstance(image, PIL.Image.Image) + image_is_tensor = isinstance(image, torch.Tensor) + image_is_np = isinstance(image, np.ndarray) + image_is_pil_list = isinstance(image, list) and isinstance(image[0], PIL.Image.Image) + image_is_tensor_list = isinstance(image, list) and isinstance(image[0], torch.Tensor) + image_is_np_list = isinstance(image, list) and isinstance(image[0], np.ndarray) + + if ( + not image_is_pil + and not image_is_tensor + and not image_is_np + and not image_is_pil_list + and not image_is_tensor_list + and not image_is_np_list + ): + raise TypeError( + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" + ) + + if image_is_pil: + image_batch_size = 1 + else: + image_batch_size = len(image) + + if prompt is not None and isinstance(prompt, str): + prompt_batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + prompt_batch_size = len(prompt) + elif prompt_embeds is not None: + prompt_batch_size = prompt_embeds.shape[0] + + if image_batch_size != 1 and image_batch_size != prompt_batch_size: + raise ValueError( + f"If image batch size is not 1, image batch size must be same as prompt batch size. image batch size: {image_batch_size}, prompt batch size: {prompt_batch_size}" + ) + + # Copied from diffusers.pipelines.controlnet.pipeline_controlnet.StableDiffusionControlNetPipeline.prepare_image + def prepare_control_image( + self, + image, + width, + height, + batch_size, + num_images_per_prompt, + device, + dtype, + do_classifier_free_guidance=False, + guess_mode=False, + ): + image = self.control_image_processor.preprocess(image, height=height, width=width).to(dtype=torch.float32) + image_batch_size = image.shape[0] + + if image_batch_size == 1: + repeat_by = batch_size + else: + # image batch size is the same as prompt batch size + repeat_by = num_images_per_prompt + + image = image.repeat_interleave(repeat_by, dim=0) + + image = image.to(device=device, dtype=dtype) + + if do_classifier_free_guidance and not guess_mode: + image = torch.cat([image] * 2) + + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.prepare_latents + def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + init_latents = image + + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + elif isinstance(generator, list): + init_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(batch_size) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + init_latents = self.vae.config.scaling_factor * init_latents + + if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0: + # expand init_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {init_latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_image_per_prompt = batch_size // init_latents.shape[0] + init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0) + elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts." + ) + else: + init_latents = torch.cat([init_latents], dim=0) + + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + latents = init_latents + + return latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + control_image: PipelineImageInput = None, + height: Optional[int] = None, + width: Optional[int] = None, + strength: float = 0.8, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_conditioning_scale: Union[float, List[float]] = 0.8, + guess_mode: bool = False, + control_guidance_start: Union[float, List[float]] = 0.0, + control_guidance_end: Union[float, List[float]] = 1.0, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: + `List[List[torch.FloatTensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + The initial image to be used as the starting point for the image generation process. Can also accept + image latents as `image`, and if passing latents directly they are not encoded again. + control_image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: + `List[List[torch.FloatTensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + The ControlNet input condition to provide guidance to the `unet` for generation. If the type is + specified as `torch.FloatTensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be + accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height + and/or width are passed, `image` is resized accordingly. If multiple ControlNets are specified in + `init`, images must be passed as a list such that each element of the list can be correctly batched for + input to a single ControlNet. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + strength (`float`, *optional*, defaults to 0.8): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added + to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set + the corresponding scale as a list. + guess_mode (`bool`, *optional*, defaults to `False`): + The ControlNet encoder tries to recognize the content of the input image even if you remove all + prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. + control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + The percentage of total steps at which the ControlNet starts applying. + control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + The percentage of total steps at which the ControlNet stops applying. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeine class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + + controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet + + # align format for control guidance + if not isinstance(control_guidance_start, list) and isinstance(control_guidance_end, list): + control_guidance_start = len(control_guidance_end) * [control_guidance_start] + elif not isinstance(control_guidance_end, list) and isinstance(control_guidance_start, list): + control_guidance_end = len(control_guidance_start) * [control_guidance_end] + elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list): + mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1 + control_guidance_start, control_guidance_end = ( + mult * [control_guidance_start], + mult * [control_guidance_end], + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + control_image, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + controlnet_conditioning_scale, + control_guidance_start, + control_guidance_end, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float): + controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets) + + global_pool_conditions = ( + controlnet.config.global_pool_conditions + if isinstance(controlnet, ControlNetModel) + else controlnet.nets[0].config.global_pool_conditions + ) + guess_mode = guess_mode or global_pool_conditions + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Prepare image + image = self.image_processor.preprocess(image, height=height, width=width).to(dtype=torch.float32) + + # 5. Prepare controlnet_conditioning_image + if isinstance(controlnet, ControlNetModel): + control_image = self.prepare_control_image( + image=control_image, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + elif isinstance(controlnet, MultiControlNetModel): + control_images = [] + + for control_image_ in control_image: + control_image_ = self.prepare_control_image( + image=control_image_, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + + control_images.append(control_image_) + + control_image = control_images + else: + assert False + + # 5. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + self._num_timesteps = len(timesteps) + + # 6. Prepare latent variables + latents = self.prepare_latents( + image, + latent_timestep, + batch_size, + num_images_per_prompt, + prompt_embeds.dtype, + device, + generator, + ) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + # 7.2 Create tensor stating which controlnets to keep + controlnet_keep = [] + for i in range(len(timesteps)): + keeps = [ + 1.0 - float(i / len(timesteps) < s or (i + 1) / len(timesteps) > e) + for s, e in zip(control_guidance_start, control_guidance_end) + ] + controlnet_keep.append(keeps[0] if isinstance(controlnet, ControlNetModel) else keeps) + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # controlnet(s) inference + if guess_mode and self.do_classifier_free_guidance: + # Infer ControlNet only for the conditional batch. + control_model_input = latents + control_model_input = self.scheduler.scale_model_input(control_model_input, t) + controlnet_prompt_embeds = prompt_embeds.chunk(2)[1] + else: + control_model_input = latent_model_input + controlnet_prompt_embeds = prompt_embeds + + if isinstance(controlnet_keep[i], list): + cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] + else: + controlnet_cond_scale = controlnet_conditioning_scale + if isinstance(controlnet_cond_scale, list): + controlnet_cond_scale = controlnet_cond_scale[0] + cond_scale = controlnet_cond_scale * controlnet_keep[i] + + down_block_res_samples, mid_block_res_sample = self.controlnet( + control_model_input, + t, + encoder_hidden_states=controlnet_prompt_embeds, + controlnet_cond=control_image, + conditioning_scale=cond_scale, + guess_mode=guess_mode, + return_dict=False, + ) + + if guess_mode and self.do_classifier_free_guidance: + # Infered ControlNet only for the conditional batch. + # To apply the output of ControlNet to both the unconditional and conditional batches, + # add 0 to the unconditional batch to keep it unchanged. + down_block_res_samples = [torch.cat([torch.zeros_like(d), d]) for d in down_block_res_samples] + mid_block_res_sample = torch.cat([torch.zeros_like(mid_block_res_sample), mid_block_res_sample]) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=self.cross_attention_kwargs, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # If we do sequential model offloading, let's offload unet and controlnet + # manually for max memory savings + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.unet.to("cpu") + self.controlnet.to("cpu") + torch.cuda.empty_cache() + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ + 0 + ] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py new file mode 100755 index 0000000..c4f1bff --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint.py @@ -0,0 +1,1620 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This model implementation is heavily inspired by https://github.com/haofanwang/ControlNet-for-Diffusers/ + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ControlNetModel, ImageProjection, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import is_compiled_module, randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from .multicontrolnet import MultiControlNetModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> # !pip install transformers accelerate + >>> from diffusers import StableDiffusionControlNetInpaintPipeline, ControlNetModel, DDIMScheduler + >>> from diffusers.utils import load_image + >>> import numpy as np + >>> import torch + + >>> init_image = load_image( + ... "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy.png" + ... ) + >>> init_image = init_image.resize((512, 512)) + + >>> generator = torch.Generator(device="cpu").manual_seed(1) + + >>> mask_image = load_image( + ... "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy_mask.png" + ... ) + >>> mask_image = mask_image.resize((512, 512)) + + + >>> def make_canny_condition(image): + ... image = np.array(image) + ... image = cv2.Canny(image, 100, 200) + ... image = image[:, :, None] + ... image = np.concatenate([image, image, image], axis=2) + ... image = Image.fromarray(image) + ... return image + + + >>> control_image = make_canny_condition(init_image) + + >>> controlnet = ControlNetModel.from_pretrained( + ... "lllyasviel/control_v11p_sd15_inpaint", torch_dtype=torch.float16 + ... ) + >>> pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", controlnet=controlnet, torch_dtype=torch.float16 + ... ) + + >>> pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + >>> pipe.enable_model_cpu_offload() + + >>> # generate image + >>> image = pipe( + ... "a handsome man with ray-ban sunglasses", + ... num_inference_steps=20, + ... generator=generator, + ... eta=1.0, + ... image=init_image, + ... mask_image=mask_image, + ... control_image=control_image, + ... ).images[0] + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_inpaint.prepare_mask_and_masked_image +def prepare_mask_and_masked_image(image, mask, height, width, return_image=False): + """ + Prepares a pair (image, mask) to be consumed by the Stable Diffusion pipeline. This means that those inputs will be + converted to ``torch.Tensor`` with shapes ``batch x channels x height x width`` where ``channels`` is ``3`` for the + ``image`` and ``1`` for the ``mask``. + + The ``image`` will be converted to ``torch.float32`` and normalized to be in ``[-1, 1]``. The ``mask`` will be + binarized (``mask > 0.5``) and cast to ``torch.float32`` too. + + Args: + image (Union[np.array, PIL.Image, torch.Tensor]): The image to inpaint. + It can be a ``PIL.Image``, or a ``height x width x 3`` ``np.array`` or a ``channels x height x width`` + ``torch.Tensor`` or a ``batch x channels x height x width`` ``torch.Tensor``. + mask (_type_): The mask to apply to the image, i.e. regions to inpaint. + It can be a ``PIL.Image``, or a ``height x width`` ``np.array`` or a ``1 x height x width`` + ``torch.Tensor`` or a ``batch x 1 x height x width`` ``torch.Tensor``. + + + Raises: + ValueError: ``torch.Tensor`` images should be in the ``[-1, 1]`` range. ValueError: ``torch.Tensor`` mask + should be in the ``[0, 1]`` range. ValueError: ``mask`` and ``image`` should have the same spatial dimensions. + TypeError: ``mask`` is a ``torch.Tensor`` but ``image`` is not + (ot the other way around). + + Returns: + tuple[torch.Tensor]: The pair (mask, masked_image) as ``torch.Tensor`` with 4 + dimensions: ``batch x channels x height x width``. + """ + deprecation_message = "The prepare_mask_and_masked_image method is deprecated and will be removed in a future version. Please use VaeImageProcessor.preprocess instead" + deprecate( + "prepare_mask_and_masked_image", + "0.30.0", + deprecation_message, + ) + if image is None: + raise ValueError("`image` input cannot be undefined.") + + if mask is None: + raise ValueError("`mask_image` input cannot be undefined.") + + if isinstance(image, torch.Tensor): + if not isinstance(mask, torch.Tensor): + raise TypeError(f"`image` is a torch.Tensor but `mask` (type: {type(mask)} is not") + + # Batch single image + if image.ndim == 3: + assert image.shape[0] == 3, "Image outside a batch should be of shape (3, H, W)" + image = image.unsqueeze(0) + + # Batch and add channel dim for single mask + if mask.ndim == 2: + mask = mask.unsqueeze(0).unsqueeze(0) + + # Batch single mask or add channel dim + if mask.ndim == 3: + # Single batched mask, no channel dim or single mask not batched but channel dim + if mask.shape[0] == 1: + mask = mask.unsqueeze(0) + + # Batched masks no channel dim + else: + mask = mask.unsqueeze(1) + + assert image.ndim == 4 and mask.ndim == 4, "Image and Mask must have 4 dimensions" + assert image.shape[-2:] == mask.shape[-2:], "Image and Mask must have the same spatial dimensions" + assert image.shape[0] == mask.shape[0], "Image and Mask must have the same batch size" + + # Check image is in [-1, 1] + if image.min() < -1 or image.max() > 1: + raise ValueError("Image should be in [-1, 1] range") + + # Check mask is in [0, 1] + if mask.min() < 0 or mask.max() > 1: + raise ValueError("Mask should be in [0, 1] range") + + # Binarize mask + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + + # Image as float32 + image = image.to(dtype=torch.float32) + elif isinstance(mask, torch.Tensor): + raise TypeError(f"`mask` is a torch.Tensor but `image` (type: {type(image)} is not") + else: + # preprocess image + if isinstance(image, (PIL.Image.Image, np.ndarray)): + image = [image] + if isinstance(image, list) and isinstance(image[0], PIL.Image.Image): + # resize all images w.r.t passed height an width + image = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in image] + image = [np.array(i.convert("RGB"))[None, :] for i in image] + image = np.concatenate(image, axis=0) + elif isinstance(image, list) and isinstance(image[0], np.ndarray): + image = np.concatenate([i[None, :] for i in image], axis=0) + + image = image.transpose(0, 3, 1, 2) + image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0 + + # preprocess mask + if isinstance(mask, (PIL.Image.Image, np.ndarray)): + mask = [mask] + + if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image): + mask = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in mask] + mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0) + mask = mask.astype(np.float32) / 255.0 + elif isinstance(mask, list) and isinstance(mask[0], np.ndarray): + mask = np.concatenate([m[None, None, :] for m in mask], axis=0) + + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + mask = torch.from_numpy(mask) + + masked_image = image * (mask < 0.5) + + # n.b. ensure backwards compatibility as old function does not return image + if return_image: + return mask, masked_image, image + + return mask, masked_image + + +class StableDiffusionControlNetInpaintPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + LoraLoaderMixin, + IPAdapterMixin, + FromSingleFileMixin, +): + r""" + Pipeline for image inpainting using Stable Diffusion with ControlNet guidance. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + + + This pipeline can be used with checkpoints that have been specifically fine-tuned for inpainting + ([runwayml/stable-diffusion-inpainting](https://huggingface.co/runwayml/stable-diffusion-inpainting)) as well as + default text-to-image Stable Diffusion checkpoints + ([runwayml/stable-diffusion-v1-5](https://huggingface.co/runwayml/stable-diffusion-v1-5)). Default text-to-image + Stable Diffusion checkpoints might be preferable for ControlNets that have been fine-tuned on those, such as + [lllyasviel/control_v11p_sd15_inpaint](https://huggingface.co/lllyasviel/control_v11p_sd15_inpaint). + + + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + Provides additional conditioning to the `unet` during the denoising process. If you set multiple + ControlNets as a list, the outputs from each ControlNet are added together to create one combined + additional conditioning. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: CLIPVisionModelWithProjection = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + if isinstance(controlnet, (list, tuple)): + controlnet = MultiControlNetModel(controlnet) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + controlnet=controlnet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + self.control_image_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=False + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + def check_inputs( + self, + prompt, + image, + mask_image, + height, + width, + callback_steps, + output_type, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + controlnet_conditioning_scale=1.0, + control_guidance_start=0.0, + control_guidance_end=1.0, + callback_on_step_end_tensor_inputs=None, + padding_mask_crop=None, + ): + if height is not None and height % 8 != 0 or width is not None and width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if padding_mask_crop is not None: + if not isinstance(image, PIL.Image.Image): + raise ValueError( + f"The image should be a PIL image when inpainting mask crop, but is of type" f" {type(image)}." + ) + if not isinstance(mask_image, PIL.Image.Image): + raise ValueError( + f"The mask image should be a PIL image when inpainting mask crop, but is of type" + f" {type(mask_image)}." + ) + if output_type != "pil": + raise ValueError(f"The output type should be PIL when inpainting mask crop, but is" f" {output_type}.") + + # `prompt` needs more sophisticated handling when there are multiple + # conditionings. + if isinstance(self.controlnet, MultiControlNetModel): + if isinstance(prompt, list): + logger.warning( + f"You have {len(self.controlnet.nets)} ControlNets and you have passed {len(prompt)}" + " prompts. The conditionings will be fixed across the prompts." + ) + + # Check `image` + is_compiled = hasattr(F, "scaled_dot_product_attention") and isinstance( + self.controlnet, torch._dynamo.eval_frame.OptimizedModule + ) + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + self.check_image(image, prompt, prompt_embeds) + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if not isinstance(image, list): + raise TypeError("For multiple controlnets: `image` must be type `list`") + + # When `image` is a nested list: + # (e.g. [[canny_image_1, pose_image_1], [canny_image_2, pose_image_2]]) + elif any(isinstance(i, list) for i in image): + raise ValueError("A single batch of multiple conditionings are supported at the moment.") + elif len(image) != len(self.controlnet.nets): + raise ValueError( + f"For multiple controlnets: `image` must have the same length as the number of controlnets, but got {len(image)} images and {len(self.controlnet.nets)} ControlNets." + ) + + for image_ in image: + self.check_image(image_, prompt, prompt_embeds) + else: + assert False + + # Check `controlnet_conditioning_scale` + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + if not isinstance(controlnet_conditioning_scale, float): + raise TypeError("For single controlnet: `controlnet_conditioning_scale` must be type `float`.") + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if isinstance(controlnet_conditioning_scale, list): + if any(isinstance(i, list) for i in controlnet_conditioning_scale): + raise ValueError("A single batch of multiple conditionings are supported at the moment.") + elif isinstance(controlnet_conditioning_scale, list) and len(controlnet_conditioning_scale) != len( + self.controlnet.nets + ): + raise ValueError( + "For multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have" + " the same length as the number of controlnets" + ) + else: + assert False + + if len(control_guidance_start) != len(control_guidance_end): + raise ValueError( + f"`control_guidance_start` has {len(control_guidance_start)} elements, but `control_guidance_end` has {len(control_guidance_end)} elements. Make sure to provide the same number of elements to each list." + ) + + if isinstance(self.controlnet, MultiControlNetModel): + if len(control_guidance_start) != len(self.controlnet.nets): + raise ValueError( + f"`control_guidance_start`: {control_guidance_start} has {len(control_guidance_start)} elements but there are {len(self.controlnet.nets)} controlnets available. Make sure to provide {len(self.controlnet.nets)}." + ) + + for start, end in zip(control_guidance_start, control_guidance_end): + if start >= end: + raise ValueError( + f"control guidance start: {start} cannot be larger or equal to control guidance end: {end}." + ) + if start < 0.0: + raise ValueError(f"control guidance start: {start} can't be smaller than 0.") + if end > 1.0: + raise ValueError(f"control guidance end: {end} can't be larger than 1.0.") + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + # Copied from diffusers.pipelines.controlnet.pipeline_controlnet.StableDiffusionControlNetPipeline.check_image + def check_image(self, image, prompt, prompt_embeds): + image_is_pil = isinstance(image, PIL.Image.Image) + image_is_tensor = isinstance(image, torch.Tensor) + image_is_np = isinstance(image, np.ndarray) + image_is_pil_list = isinstance(image, list) and isinstance(image[0], PIL.Image.Image) + image_is_tensor_list = isinstance(image, list) and isinstance(image[0], torch.Tensor) + image_is_np_list = isinstance(image, list) and isinstance(image[0], np.ndarray) + + if ( + not image_is_pil + and not image_is_tensor + and not image_is_np + and not image_is_pil_list + and not image_is_tensor_list + and not image_is_np_list + ): + raise TypeError( + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" + ) + + if image_is_pil: + image_batch_size = 1 + else: + image_batch_size = len(image) + + if prompt is not None and isinstance(prompt, str): + prompt_batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + prompt_batch_size = len(prompt) + elif prompt_embeds is not None: + prompt_batch_size = prompt_embeds.shape[0] + + if image_batch_size != 1 and image_batch_size != prompt_batch_size: + raise ValueError( + f"If image batch size is not 1, image batch size must be same as prompt batch size. image batch size: {image_batch_size}, prompt batch size: {prompt_batch_size}" + ) + + def prepare_control_image( + self, + image, + width, + height, + batch_size, + num_images_per_prompt, + device, + dtype, + crops_coords, + resize_mode, + do_classifier_free_guidance=False, + guess_mode=False, + ): + image = self.control_image_processor.preprocess( + image, height=height, width=width, crops_coords=crops_coords, resize_mode=resize_mode + ).to(dtype=torch.float32) + image_batch_size = image.shape[0] + + if image_batch_size == 1: + repeat_by = batch_size + else: + # image batch size is the same as prompt batch size + repeat_by = num_images_per_prompt + + image = image.repeat_interleave(repeat_by, dim=0) + + image = image.to(device=device, dtype=dtype) + + if do_classifier_free_guidance and not guess_mode: + image = torch.cat([image] * 2) + + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_inpaint.StableDiffusionInpaintPipeline.prepare_latents + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + image=None, + timestep=None, + is_strength_max=True, + return_noise=False, + return_image_latents=False, + ): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if (image is None or timestep is None) and not is_strength_max: + raise ValueError( + "Since strength < 1. initial latents are to be initialised as a combination of Image + Noise." + "However, either the image or the noise timestep has not been provided." + ) + + if return_image_latents or (latents is None and not is_strength_max): + image = image.to(device=device, dtype=dtype) + + if image.shape[1] == 4: + image_latents = image + else: + image_latents = self._encode_vae_image(image=image, generator=generator) + image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1) + + if latents is None: + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + # if strength is 1. then initialise the latents to noise, else initial to image + noise + latents = noise if is_strength_max else self.scheduler.add_noise(image_latents, noise, timestep) + # if pure noise then scale the initial latents by the Scheduler's init sigma + latents = latents * self.scheduler.init_noise_sigma if is_strength_max else latents + else: + noise = latents.to(device) + latents = noise * self.scheduler.init_noise_sigma + + outputs = (latents,) + + if return_noise: + outputs += (noise,) + + if return_image_latents: + outputs += (image_latents,) + + return outputs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_inpaint.StableDiffusionInpaintPipeline.prepare_mask_latents + def prepare_mask_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + mask = torch.nn.functional.interpolate( + mask, size=(height // self.vae_scale_factor, width // self.vae_scale_factor) + ) + mask = mask.to(device=device, dtype=dtype) + + masked_image = masked_image.to(device=device, dtype=dtype) + + if masked_image.shape[1] == 4: + masked_image_latents = masked_image + else: + masked_image_latents = self._encode_vae_image(masked_image, generator=generator) + + # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method + if mask.shape[0] < batch_size: + if not batch_size % mask.shape[0] == 0: + raise ValueError( + "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to" + f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number" + " of masks that you pass is divisible by the total requested batch size." + ) + mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1) + if masked_image_latents.shape[0] < batch_size: + if not batch_size % masked_image_latents.shape[0] == 0: + raise ValueError( + "The passed images and the required batch size don't match. Images are supposed to be duplicated" + f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed." + " Make sure the number of images that you pass is divisible by the total requested batch size." + ) + masked_image_latents = masked_image_latents.repeat(batch_size // masked_image_latents.shape[0], 1, 1, 1) + + mask = torch.cat([mask] * 2) if do_classifier_free_guidance else mask + masked_image_latents = ( + torch.cat([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents + ) + + # aligning device to prevent device errors when concating it with the latent model input + masked_image_latents = masked_image_latents.to(device=device, dtype=dtype) + return mask, masked_image_latents + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_inpaint.StableDiffusionInpaintPipeline._encode_vae_image + def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator): + if isinstance(generator, list): + image_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(image.shape[0]) + ] + image_latents = torch.cat(image_latents, dim=0) + else: + image_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + image_latents = self.vae.config.scaling_factor * image_latents + + return image_latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + mask_image: PipelineImageInput = None, + control_image: PipelineImageInput = None, + height: Optional[int] = None, + width: Optional[int] = None, + padding_mask_crop: Optional[int] = None, + strength: float = 1.0, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_conditioning_scale: Union[float, List[float]] = 0.5, + guess_mode: bool = False, + control_guidance_start: Union[float, List[float]] = 0.0, + control_guidance_end: Union[float, List[float]] = 1.0, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, + `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, NumPy array or tensor representing an image batch to be used as the starting point. For both + NumPy array and PyTorch tensor, the expected value range is between `[0, 1]`. If it's a tensor or a + list or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a NumPy array or + a list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)`. It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + mask_image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, + `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, NumPy array or tensor representing an image batch to mask `image`. White pixels in the mask + are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a + single channel (luminance) before use. If it's a NumPy array or PyTorch tensor, it should contain one + color channel (L) instead of 3, so the expected shape for PyTorch tensor would be `(B, 1, H, W)`, `(B, + H, W)`, `(1, H, W)`, `(H, W)`. And for NumPy array, it would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, + W, 1)`, or `(H, W)`. + control_image (`torch.FloatTensor`, `PIL.Image.Image`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, + `List[List[torch.FloatTensor]]`, or `List[List[PIL.Image.Image]]`): + The ControlNet input condition to provide guidance to the `unet` for generation. If the type is + specified as `torch.FloatTensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be + accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height + and/or width are passed, `image` is resized accordingly. If multiple ControlNets are specified in + `init`, images must be passed as a list such that each element of the list can be correctly batched for + input to a single ControlNet. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + padding_mask_crop (`int`, *optional*, defaults to `None`): + The size of margin in the crop to be applied to the image and masking. If `None`, no crop is applied to image and mask_image. If + `padding_mask_crop` is not `None`, it will first find a rectangular region with the same aspect ration of the image and + contains all masked area, and then expand that area based on `padding_mask_crop`. The image and mask_image will then be cropped based on + the expanded area before resizing to the original image size for inpainting. This is useful when the masked area is small while the image is large + and contain information inreleant for inpainging, such as background. + strength (`float`, *optional*, defaults to 1.0): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 0.5): + The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added + to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set + the corresponding scale as a list. + guess_mode (`bool`, *optional*, defaults to `False`): + The ControlNet encoder tries to recognize the content of the input image even if you remove all + prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. + control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + The percentage of total steps at which the ControlNet starts applying. + control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + The percentage of total steps at which the ControlNet stops applying. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeine class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + + controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet + + # align format for control guidance + if not isinstance(control_guidance_start, list) and isinstance(control_guidance_end, list): + control_guidance_start = len(control_guidance_end) * [control_guidance_start] + elif not isinstance(control_guidance_end, list) and isinstance(control_guidance_start, list): + control_guidance_end = len(control_guidance_start) * [control_guidance_end] + elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list): + mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1 + control_guidance_start, control_guidance_end = ( + mult * [control_guidance_start], + mult * [control_guidance_end], + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + control_image, + mask_image, + height, + width, + callback_steps, + output_type, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + controlnet_conditioning_scale, + control_guidance_start, + control_guidance_end, + callback_on_step_end_tensor_inputs, + padding_mask_crop, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if padding_mask_crop is not None: + height, width = self.image_processor.get_default_height_width(image, height, width) + crops_coords = self.mask_processor.get_crop_region(mask_image, width, height, pad=padding_mask_crop) + resize_mode = "fill" + else: + crops_coords = None + resize_mode = "default" + + device = self._execution_device + + if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float): + controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets) + + global_pool_conditions = ( + controlnet.config.global_pool_conditions + if isinstance(controlnet, ControlNetModel) + else controlnet.nets[0].config.global_pool_conditions + ) + guess_mode = guess_mode or global_pool_conditions + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Prepare image + if isinstance(controlnet, ControlNetModel): + control_image = self.prepare_control_image( + image=control_image, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + crops_coords=crops_coords, + resize_mode=resize_mode, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + elif isinstance(controlnet, MultiControlNetModel): + control_images = [] + + for control_image_ in control_image: + control_image_ = self.prepare_control_image( + image=control_image_, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + crops_coords=crops_coords, + resize_mode=resize_mode, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + + control_images.append(control_image_) + + control_image = control_images + else: + assert False + + # 4.1 Preprocess mask and image - resizes image and mask w.r.t height and width + original_image = image + init_image = self.image_processor.preprocess( + image, height=height, width=width, crops_coords=crops_coords, resize_mode=resize_mode + ) + init_image = init_image.to(dtype=torch.float32) + + mask = self.mask_processor.preprocess( + mask_image, height=height, width=width, resize_mode=resize_mode, crops_coords=crops_coords + ) + + masked_image = init_image * (mask < 0.5) + _, _, height, width = init_image.shape + + # 5. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps( + num_inference_steps=num_inference_steps, strength=strength, device=device + ) + # at which timestep to set the initial noise (n.b. 50% if strength is 0.5) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + # create a boolean to check if the strength is set to 1. if so then initialise the latents with pure noise + is_strength_max = strength == 1.0 + self._num_timesteps = len(timesteps) + + # 6. Prepare latent variables + num_channels_latents = self.vae.config.latent_channels + num_channels_unet = self.unet.config.in_channels + return_image_latents = num_channels_unet == 4 + latents_outputs = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + image=init_image, + timestep=latent_timestep, + is_strength_max=is_strength_max, + return_noise=True, + return_image_latents=return_image_latents, + ) + + if return_image_latents: + latents, noise, image_latents = latents_outputs + else: + latents, noise = latents_outputs + + # 7. Prepare mask latent variables + mask, masked_image_latents = self.prepare_mask_latents( + mask, + masked_image, + batch_size * num_images_per_prompt, + height, + width, + prompt_embeds.dtype, + device, + generator, + self.do_classifier_free_guidance, + ) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + # 7.2 Create tensor stating which controlnets to keep + controlnet_keep = [] + for i in range(len(timesteps)): + keeps = [ + 1.0 - float(i / len(timesteps) < s or (i + 1) / len(timesteps) > e) + for s, e in zip(control_guidance_start, control_guidance_end) + ] + controlnet_keep.append(keeps[0] if isinstance(controlnet, ControlNetModel) else keeps) + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # controlnet(s) inference + if guess_mode and self.do_classifier_free_guidance: + # Infer ControlNet only for the conditional batch. + control_model_input = latents + control_model_input = self.scheduler.scale_model_input(control_model_input, t) + controlnet_prompt_embeds = prompt_embeds.chunk(2)[1] + else: + control_model_input = latent_model_input + controlnet_prompt_embeds = prompt_embeds + + if isinstance(controlnet_keep[i], list): + cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] + else: + controlnet_cond_scale = controlnet_conditioning_scale + if isinstance(controlnet_cond_scale, list): + controlnet_cond_scale = controlnet_cond_scale[0] + cond_scale = controlnet_cond_scale * controlnet_keep[i] + + down_block_res_samples, mid_block_res_sample = self.controlnet( + control_model_input, + t, + encoder_hidden_states=controlnet_prompt_embeds, + controlnet_cond=control_image, + conditioning_scale=cond_scale, + guess_mode=guess_mode, + return_dict=False, + ) + + if guess_mode and self.do_classifier_free_guidance: + # Infered ControlNet only for the conditional batch. + # To apply the output of ControlNet to both the unconditional and conditional batches, + # add 0 to the unconditional batch to keep it unchanged. + down_block_res_samples = [torch.cat([torch.zeros_like(d), d]) for d in down_block_res_samples] + mid_block_res_sample = torch.cat([torch.zeros_like(mid_block_res_sample), mid_block_res_sample]) + + # predict the noise residual + if num_channels_unet == 9: + latent_model_input = torch.cat([latent_model_input, mask, masked_image_latents], dim=1) + + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=self.cross_attention_kwargs, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if num_channels_unet == 4: + init_latents_proper = image_latents + if self.do_classifier_free_guidance: + init_mask, _ = mask.chunk(2) + else: + init_mask = mask + + if i < len(timesteps) - 1: + noise_timestep = timesteps[i + 1] + init_latents_proper = self.scheduler.add_noise( + init_latents_proper, noise, torch.tensor([noise_timestep]) + ) + + latents = (1 - init_mask) * init_latents_proper + init_mask * latents + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # If we do sequential model offloading, let's offload unet and controlnet + # manually for max memory savings + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.unet.to("cpu") + self.controlnet.to("cpu") + torch.cuda.empty_cache() + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ + 0 + ] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + if padding_mask_crop is not None: + image = [self.image_processor.apply_overlay(mask_image, original_image, i, crops_coords) for i in image] + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py new file mode 100755 index 0000000..52ffe5a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_inpaint_sd_xl.py @@ -0,0 +1,1818 @@ +# Copyright 2024 Harutatsu Akiyama, Jinbin Bai, and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import ( + FromSingleFileMixin, + IPAdapterMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, +) +from ...models import AutoencoderKL, ControlNetModel, ImageProjection, UNet2DConditionModel +from ...models.attention_processor import ( + AttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + is_invisible_watermark_available, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import is_compiled_module, randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput +from .multicontrolnet import MultiControlNetModel + + +if is_invisible_watermark_available(): + from diffusers.pipelines.stable_diffusion_xl.watermark import StableDiffusionXLWatermarker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> # !pip install transformers accelerate + >>> from diffusers import StableDiffusionXLControlNetInpaintPipeline, ControlNetModel, DDIMScheduler + >>> from diffusers.utils import load_image + >>> import numpy as np + >>> import torch + + >>> init_image = load_image( + ... "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy.png" + ... ) + >>> init_image = init_image.resize((1024, 1024)) + + >>> generator = torch.Generator(device="cpu").manual_seed(1) + + >>> mask_image = load_image( + ... "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy_mask.png" + ... ) + >>> mask_image = mask_image.resize((1024, 1024)) + + + >>> def make_canny_condition(image): + ... image = np.array(image) + ... image = cv2.Canny(image, 100, 200) + ... image = image[:, :, None] + ... image = np.concatenate([image, image, image], axis=2) + ... image = Image.fromarray(image) + ... return image + + + >>> control_image = make_canny_condition(init_image) + + >>> controlnet = ControlNetModel.from_pretrained( + ... "diffusers/controlnet-canny-sdxl-1.0", torch_dtype=torch.float16 + ... ) + >>> pipe = StableDiffusionXLControlNetInpaintPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-xl-base-1.0", controlnet=controlnet, torch_dtype=torch.float16 + ... ) + + >>> pipe.enable_model_cpu_offload() + + >>> # generate image + >>> image = pipe( + ... "a handsome man with ray-ban sunglasses", + ... num_inference_steps=20, + ... generator=generator, + ... eta=1.0, + ... image=init_image, + ... mask_image=mask_image, + ... control_image=control_image, + ... ).images[0] + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class StableDiffusionXLControlNetInpaintPipeline( + DiffusionPipeline, StableDiffusionMixin, StableDiffusionXLLoraLoaderMixin, FromSingleFileMixin, IPAdapterMixin +): + r""" + Pipeline for text-to-image generation using Stable Diffusion XL. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + The pipeline also inherits the following loading methods: + - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->unet->vae" + _optional_components = ["tokenizer", "tokenizer_2", "text_encoder", "text_encoder_2"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + controlnet: ControlNetModel, + scheduler: KarrasDiffusionSchedulers, + requires_aesthetics_score: bool = False, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + feature_extractor: Optional[CLIPImageProcessor] = None, + image_encoder: Optional[CLIPVisionModelWithProjection] = None, + ): + super().__init__() + + if isinstance(controlnet, (list, tuple)): + controlnet = MultiControlNetModel(controlnet) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + controlnet=controlnet, + scheduler=scheduler, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.register_to_config(requires_aesthetics_score=requires_aesthetics_score) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + self.control_image_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=False + ) + + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() + + if add_watermarker: + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # textual inversion: process multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_2 = ( + batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 + ) + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_image(self, image, prompt, prompt_embeds): + image_is_pil = isinstance(image, PIL.Image.Image) + image_is_tensor = isinstance(image, torch.Tensor) + image_is_np = isinstance(image, np.ndarray) + image_is_pil_list = isinstance(image, list) and isinstance(image[0], PIL.Image.Image) + image_is_tensor_list = isinstance(image, list) and isinstance(image[0], torch.Tensor) + image_is_np_list = isinstance(image, list) and isinstance(image[0], np.ndarray) + + if ( + not image_is_pil + and not image_is_tensor + and not image_is_np + and not image_is_pil_list + and not image_is_tensor_list + and not image_is_np_list + ): + raise TypeError( + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" + ) + + if image_is_pil: + image_batch_size = 1 + else: + image_batch_size = len(image) + + if prompt is not None and isinstance(prompt, str): + prompt_batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + prompt_batch_size = len(prompt) + elif prompt_embeds is not None: + prompt_batch_size = prompt_embeds.shape[0] + + if image_batch_size != 1 and image_batch_size != prompt_batch_size: + raise ValueError( + f"If image batch size is not 1, image batch size must be same as prompt batch size. image batch size: {image_batch_size}, prompt batch size: {prompt_batch_size}" + ) + + def check_inputs( + self, + prompt, + prompt_2, + image, + mask_image, + strength, + num_inference_steps, + callback_steps, + output_type, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + controlnet_conditioning_scale=1.0, + control_guidance_start=0.0, + control_guidance_end=1.0, + callback_on_step_end_tensor_inputs=None, + padding_mask_crop=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + if num_inference_steps is None: + raise ValueError("`num_inference_steps` cannot be None.") + elif not isinstance(num_inference_steps, int) or num_inference_steps <= 0: + raise ValueError( + f"`num_inference_steps` has to be a positive integer but is {num_inference_steps} of type" + f" {type(num_inference_steps)}." + ) + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if padding_mask_crop is not None: + if not isinstance(image, PIL.Image.Image): + raise ValueError( + f"The image should be a PIL image when inpainting mask crop, but is of type" f" {type(image)}." + ) + if not isinstance(mask_image, PIL.Image.Image): + raise ValueError( + f"The mask image should be a PIL image when inpainting mask crop, but is of type" + f" {type(mask_image)}." + ) + if output_type != "pil": + raise ValueError(f"The output type should be PIL when inpainting mask crop, but is" f" {output_type}.") + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + # `prompt` needs more sophisticated handling when there are multiple + # conditionings. + if isinstance(self.controlnet, MultiControlNetModel): + if isinstance(prompt, list): + logger.warning( + f"You have {len(self.controlnet.nets)} ControlNets and you have passed {len(prompt)}" + " prompts. The conditionings will be fixed across the prompts." + ) + + # Check `image` + is_compiled = hasattr(F, "scaled_dot_product_attention") and isinstance( + self.controlnet, torch._dynamo.eval_frame.OptimizedModule + ) + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + self.check_image(image, prompt, prompt_embeds) + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if not isinstance(image, list): + raise TypeError("For multiple controlnets: `image` must be type `list`") + + # When `image` is a nested list: + # (e.g. [[canny_image_1, pose_image_1], [canny_image_2, pose_image_2]]) + elif any(isinstance(i, list) for i in image): + raise ValueError("A single batch of multiple conditionings are supported at the moment.") + elif len(image) != len(self.controlnet.nets): + raise ValueError( + f"For multiple controlnets: `image` must have the same length as the number of controlnets, but got {len(image)} images and {len(self.controlnet.nets)} ControlNets." + ) + + for image_ in image: + self.check_image(image_, prompt, prompt_embeds) + else: + assert False + + # Check `controlnet_conditioning_scale` + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + if not isinstance(controlnet_conditioning_scale, float): + raise TypeError("For single controlnet: `controlnet_conditioning_scale` must be type `float`.") + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if isinstance(controlnet_conditioning_scale, list): + if any(isinstance(i, list) for i in controlnet_conditioning_scale): + raise ValueError("A single batch of multiple conditionings are supported at the moment.") + elif isinstance(controlnet_conditioning_scale, list) and len(controlnet_conditioning_scale) != len( + self.controlnet.nets + ): + raise ValueError( + "For multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have" + " the same length as the number of controlnets" + ) + else: + assert False + + if not isinstance(control_guidance_start, (tuple, list)): + control_guidance_start = [control_guidance_start] + + if not isinstance(control_guidance_end, (tuple, list)): + control_guidance_end = [control_guidance_end] + + if len(control_guidance_start) != len(control_guidance_end): + raise ValueError( + f"`control_guidance_start` has {len(control_guidance_start)} elements, but `control_guidance_end` has {len(control_guidance_end)} elements. Make sure to provide the same number of elements to each list." + ) + + if isinstance(self.controlnet, MultiControlNetModel): + if len(control_guidance_start) != len(self.controlnet.nets): + raise ValueError( + f"`control_guidance_start`: {control_guidance_start} has {len(control_guidance_start)} elements but there are {len(self.controlnet.nets)} controlnets available. Make sure to provide {len(self.controlnet.nets)}." + ) + + for start, end in zip(control_guidance_start, control_guidance_end): + if start >= end: + raise ValueError( + f"control guidance start: {start} cannot be larger or equal to control guidance end: {end}." + ) + if start < 0.0: + raise ValueError(f"control guidance start: {start} can't be smaller than 0.") + if end > 1.0: + raise ValueError(f"control guidance end: {end} can't be larger than 1.0.") + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + def prepare_control_image( + self, + image, + width, + height, + batch_size, + num_images_per_prompt, + device, + dtype, + crops_coords, + resize_mode, + do_classifier_free_guidance=False, + guess_mode=False, + ): + image = self.control_image_processor.preprocess( + image, height=height, width=width, crops_coords=crops_coords, resize_mode=resize_mode + ).to(dtype=torch.float32) + image_batch_size = image.shape[0] + + if image_batch_size == 1: + repeat_by = batch_size + else: + # image batch size is the same as prompt batch size + repeat_by = num_images_per_prompt + + image = image.repeat_interleave(repeat_by, dim=0) + + image = image.to(device=device, dtype=dtype) + + if do_classifier_free_guidance and not guess_mode: + image = torch.cat([image] * 2) + + return image + + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + image=None, + timestep=None, + is_strength_max=True, + add_noise=True, + return_noise=False, + return_image_latents=False, + ): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if (image is None or timestep is None) and not is_strength_max: + raise ValueError( + "Since strength < 1. initial latents are to be initialised as a combination of Image + Noise." + "However, either the image or the noise timestep has not been provided." + ) + + if return_image_latents or (latents is None and not is_strength_max): + image = image.to(device=device, dtype=dtype) + + if image.shape[1] == 4: + image_latents = image + else: + image_latents = self._encode_vae_image(image=image, generator=generator) + image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1) + + if latents is None and add_noise: + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + # if strength is 1. then initialise the latents to noise, else initial to image + noise + latents = noise if is_strength_max else self.scheduler.add_noise(image_latents, noise, timestep) + # if pure noise then scale the initial latents by the Scheduler's init sigma + latents = latents * self.scheduler.init_noise_sigma if is_strength_max else latents + elif add_noise: + noise = latents.to(device) + latents = noise * self.scheduler.init_noise_sigma + else: + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + latents = image_latents.to(device) + + outputs = (latents,) + + if return_noise: + outputs += (noise,) + + if return_image_latents: + outputs += (image_latents,) + + return outputs + + def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator): + dtype = image.dtype + if self.vae.config.force_upcast: + image = image.float() + self.vae.to(dtype=torch.float32) + + if isinstance(generator, list): + image_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(image.shape[0]) + ] + image_latents = torch.cat(image_latents, dim=0) + else: + image_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + if self.vae.config.force_upcast: + self.vae.to(dtype) + + image_latents = image_latents.to(dtype) + image_latents = self.vae.config.scaling_factor * image_latents + + return image_latents + + def prepare_mask_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + mask = torch.nn.functional.interpolate( + mask, size=(height // self.vae_scale_factor, width // self.vae_scale_factor) + ) + mask = mask.to(device=device, dtype=dtype) + + # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method + if mask.shape[0] < batch_size: + if not batch_size % mask.shape[0] == 0: + raise ValueError( + "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to" + f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number" + " of masks that you pass is divisible by the total requested batch size." + ) + mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1) + + mask = torch.cat([mask] * 2) if do_classifier_free_guidance else mask + + masked_image_latents = None + if masked_image is not None: + masked_image = masked_image.to(device=device, dtype=dtype) + masked_image_latents = self._encode_vae_image(masked_image, generator=generator) + if masked_image_latents.shape[0] < batch_size: + if not batch_size % masked_image_latents.shape[0] == 0: + raise ValueError( + "The passed images and the required batch size don't match. Images are supposed to be duplicated" + f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed." + " Make sure the number of images that you pass is divisible by the total requested batch size." + ) + masked_image_latents = masked_image_latents.repeat( + batch_size // masked_image_latents.shape[0], 1, 1, 1 + ) + + masked_image_latents = ( + torch.cat([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents + ) + + # aligning device to prevent device errors when concating it with the latent model input + masked_image_latents = masked_image_latents.to(device=device, dtype=dtype) + + return mask, masked_image_latents + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img.StableDiffusionXLImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device, denoising_start=None): + # get the original timestep using init_timestep + if denoising_start is None: + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + t_start = max(num_inference_steps - init_timestep, 0) + else: + t_start = 0 + + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + + # Strength is irrelevant if we directly request a timestep to start at; + # that is, strength is determined by the denoising_start instead. + if denoising_start is not None: + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (denoising_start * self.scheduler.config.num_train_timesteps) + ) + ) + + num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() + if self.scheduler.order == 2 and num_inference_steps % 2 == 0: + # if the scheduler is a 2nd order scheduler we might have to do +1 + # because `num_inference_steps` might be even given that every timestep + # (except the highest one) is duplicated. If `num_inference_steps` is even it would + # mean that we cut the timesteps in the middle of the denoising step + # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1 + # we ensure that the denoising process always ends after the 2nd derivate step of the scheduler + num_inference_steps = num_inference_steps + 1 + + # because t_n+1 >= t_n, we slice the timesteps starting from the end + timesteps = timesteps[-num_inference_steps:] + return timesteps, num_inference_steps + + return timesteps, num_inference_steps - t_start + + def _get_add_time_ids( + self, + original_size, + crops_coords_top_left, + target_size, + aesthetic_score, + negative_aesthetic_score, + dtype, + text_encoder_projection_dim=None, + ): + if self.config.requires_aesthetics_score: + add_time_ids = list(original_size + crops_coords_top_left + (aesthetic_score,)) + add_neg_time_ids = list(original_size + crops_coords_top_left + (negative_aesthetic_score,)) + else: + add_time_ids = list(original_size + crops_coords_top_left + target_size) + add_neg_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if ( + expected_add_embed_dim > passed_add_embed_dim + and (expected_add_embed_dim - passed_add_embed_dim) == self.unet.config.addition_time_embed_dim + ): + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to enable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=True)` to make sure `aesthetic_score` {aesthetic_score} and `negative_aesthetic_score` {negative_aesthetic_score} is correctly used by the model." + ) + elif ( + expected_add_embed_dim < passed_add_embed_dim + and (passed_add_embed_dim - expected_add_embed_dim) == self.unet.config.addition_time_embed_dim + ): + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to disable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=False)` to make sure `target_size` {target_size} is correctly used by the model." + ) + elif expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + add_neg_time_ids = torch.tensor([add_neg_time_ids], dtype=dtype) + + return add_time_ids, add_neg_time_ids + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + image: PipelineImageInput = None, + mask_image: PipelineImageInput = None, + control_image: Union[ + PipelineImageInput, + List[PipelineImageInput], + ] = None, + height: Optional[int] = None, + width: Optional[int] = None, + padding_mask_crop: Optional[int] = None, + strength: float = 0.9999, + num_inference_steps: int = 50, + denoising_start: Optional[float] = None, + denoising_end: Optional[float] = None, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + guess_mode: bool = False, + control_guidance_start: Union[float, List[float]] = 0.0, + control_guidance_end: Union[float, List[float]] = 1.0, + guidance_rescale: float = 0.0, + original_size: Tuple[int, int] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Tuple[int, int] = None, + aesthetic_score: float = 6.0, + negative_aesthetic_score: float = 2.5, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + image (`PIL.Image.Image`): + `Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will + be masked out with `mask_image` and repainted according to `prompt`. + mask_image (`PIL.Image.Image`): + `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be + repainted, while black pixels will be preserved. If `mask_image` is a PIL image, it will be converted + to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) + instead of 3, so the expected shape would be `(B, H, W, 1)`. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. + padding_mask_crop (`int`, *optional*, defaults to `None`): + The size of margin in the crop to be applied to the image and masking. If `None`, no crop is applied to image and mask_image. If + `padding_mask_crop` is not `None`, it will first find a rectangular region with the same aspect ration of the image and + contains all masked area, and then expand that area based on `padding_mask_crop`. The image and mask_image will then be cropped based on + the expanded area before resizing to the original image size for inpainting. This is useful when the masked area is small while the image is large + and contain information inreleant for inpainging, such as background. + strength (`float`, *optional*, defaults to 0.9999): + Conceptually, indicates how much to transform the masked portion of the reference `image`. Must be + between 0 and 1. `image` will be used as a starting point, adding more noise to it the larger the + `strength`. The number of denoising steps depends on the amount of noise initially added. When + `strength` is 1, added noise will be maximum and the denoising process will run for the full number of + iterations specified in `num_inference_steps`. A value of 1, therefore, essentially ignores the masked + portion of the reference `image`. Note that in the case of `denoising_start` being declared as an + integer, the value of `strength` will be ignored. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + denoising_start (`float`, *optional*): + When specified, indicates the fraction (between 0.0 and 1.0) of the total denoising process to be + bypassed before it is initiated. Consequently, the initial part of the denoising process is skipped and + it is assumed that the passed `image` is a partly denoised image. Note that when this is specified, + strength will be ignored. The `denoising_start` parameter is particularly beneficial when this pipeline + is integrated into a "Mixture of Denoisers" multi-pipeline setup, as detailed in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output). + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise (ca. final 20% of timesteps still needed) and should be + denoised by a successor pipeline that has `denoising_start` set to 0.8 so that it only denoises the + final 20% of the scheduler. The denoising_end parameter should ideally be utilized when this pipeline + forms a part of a "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output). + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(width, height)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(width, height)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + aesthetic_score (`float`, *optional*, defaults to 6.0): + Used to simulate an aesthetic score of the generated image by influencing the positive text condition. + Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_aesthetic_score (`float`, *optional*, defaults to 2.5): + Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). Can be used to + simulate an aesthetic score of the generated image by influencing the negative text condition. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeine class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a + `tuple. `tuple. When returning a tuple, the first element is a list with the generated images. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + + controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet + + # align format for control guidance + if not isinstance(control_guidance_start, list) and isinstance(control_guidance_end, list): + control_guidance_start = len(control_guidance_end) * [control_guidance_start] + elif not isinstance(control_guidance_end, list) and isinstance(control_guidance_start, list): + control_guidance_end = len(control_guidance_start) * [control_guidance_end] + elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list): + mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1 + control_guidance_start, control_guidance_end = ( + mult * [control_guidance_start], + mult * [control_guidance_end], + ) + + # # 0.0 Default height and width to unet + # height = height or self.unet.config.sample_size * self.vae_scale_factor + # width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 0.1 align format for control guidance + if not isinstance(control_guidance_start, list) and isinstance(control_guidance_end, list): + control_guidance_start = len(control_guidance_end) * [control_guidance_start] + elif not isinstance(control_guidance_end, list) and isinstance(control_guidance_start, list): + control_guidance_end = len(control_guidance_start) * [control_guidance_end] + elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list): + mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1 + control_guidance_start, control_guidance_end = ( + mult * [control_guidance_start], + mult * [control_guidance_end], + ) + + # 1. Check inputs + self.check_inputs( + prompt, + prompt_2, + control_image, + mask_image, + strength, + num_inference_steps, + callback_steps, + output_type, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + controlnet_conditioning_scale, + control_guidance_start, + control_guidance_end, + callback_on_step_end_tensor_inputs, + padding_mask_crop, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float): + controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets) + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + + # 3.1 Encode ip_adapter_image + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. set timesteps + def denoising_value_valid(dnv): + return isinstance(dnv, float) and 0 < dnv < 1 + + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps( + num_inference_steps, + strength, + device, + denoising_start=denoising_start if denoising_value_valid(denoising_start) else None, + ) + # check that number of inference steps is not < 1 - as this doesn't make sense + if num_inference_steps < 1: + raise ValueError( + f"After adjusting the num_inference_steps by strength parameter: {strength}, the number of pipeline" + f"steps is {num_inference_steps} which is < 1 and not appropriate for this pipeline." + ) + # at which timestep to set the initial noise (n.b. 50% if strength is 0.5) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + # create a boolean to check if the strength is set to 1. if so then initialise the latents with pure noise + is_strength_max = strength == 1.0 + self._num_timesteps = len(timesteps) + + # 5. Preprocess mask and image - resizes image and mask w.r.t height and width + # 5.1 Prepare init image + if padding_mask_crop is not None: + height, width = self.image_processor.get_default_height_width(image, height, width) + crops_coords = self.mask_processor.get_crop_region(mask_image, width, height, pad=padding_mask_crop) + resize_mode = "fill" + else: + crops_coords = None + resize_mode = "default" + + original_image = image + init_image = self.image_processor.preprocess( + image, height=height, width=width, crops_coords=crops_coords, resize_mode=resize_mode + ) + init_image = init_image.to(dtype=torch.float32) + + # 5.2 Prepare control images + if isinstance(controlnet, ControlNetModel): + control_image = self.prepare_control_image( + image=control_image, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + crops_coords=crops_coords, + resize_mode=resize_mode, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + elif isinstance(controlnet, MultiControlNetModel): + control_images = [] + + for control_image_ in control_image: + control_image_ = self.prepare_control_image( + image=control_image_, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + crops_coords=crops_coords, + resize_mode=resize_mode, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + + control_images.append(control_image_) + + control_image = control_images + else: + raise ValueError(f"{controlnet.__class__} is not supported.") + + # 5.3 Prepare mask + mask = self.mask_processor.preprocess( + mask_image, height=height, width=width, resize_mode=resize_mode, crops_coords=crops_coords + ) + + masked_image = init_image * (mask < 0.5) + _, _, height, width = init_image.shape + + # 6. Prepare latent variables + num_channels_latents = self.vae.config.latent_channels + num_channels_unet = self.unet.config.in_channels + return_image_latents = num_channels_unet == 4 + + add_noise = True if denoising_start is None else False + latents_outputs = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + image=init_image, + timestep=latent_timestep, + is_strength_max=is_strength_max, + add_noise=add_noise, + return_noise=True, + return_image_latents=return_image_latents, + ) + + if return_image_latents: + latents, noise, image_latents = latents_outputs + else: + latents, noise = latents_outputs + + # 7. Prepare mask latent variables + mask, masked_image_latents = self.prepare_mask_latents( + mask, + masked_image, + batch_size * num_images_per_prompt, + height, + width, + prompt_embeds.dtype, + device, + generator, + self.do_classifier_free_guidance, + ) + + # 8. Check that sizes of mask, masked image and latents match + if num_channels_unet == 9: + # default case for runwayml/stable-diffusion-inpainting + num_channels_mask = mask.shape[1] + num_channels_masked_image = masked_image_latents.shape[1] + if num_channels_latents + num_channels_mask + num_channels_masked_image != self.unet.config.in_channels: + raise ValueError( + f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects" + f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +" + f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}" + f" = {num_channels_latents+num_channels_masked_image+num_channels_mask}. Please verify the config of" + " `pipeline.unet` or your `mask_image` or `image` input." + ) + elif num_channels_unet != 4: + raise ValueError( + f"The unet {self.unet.__class__} should have either 4 or 9 input channels, not {self.unet.config.in_channels}." + ) + # 8.1 Prepare extra step kwargs. + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 8.2 Create tensor stating which controlnets to keep + controlnet_keep = [] + for i in range(len(timesteps)): + keeps = [ + 1.0 - float(i / len(timesteps) < s or (i + 1) / len(timesteps) > e) + for s, e in zip(control_guidance_start, control_guidance_end) + ] + if isinstance(self.controlnet, MultiControlNetModel): + controlnet_keep.append(keeps) + else: + controlnet_keep.append(keeps[0]) + + # 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + height, width = latents.shape[-2:] + height = height * self.vae_scale_factor + width = width * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 10. Prepare added time ids & embeddings + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids, add_neg_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + aesthetic_score, + negative_aesthetic_score, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + add_time_ids = add_time_ids.repeat(batch_size * num_images_per_prompt, 1) + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_neg_time_ids = add_neg_time_ids.repeat(batch_size * num_images_per_prompt, 1) + add_time_ids = torch.cat([add_neg_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device) + + # 11. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + if ( + denoising_end is not None + and denoising_start is not None + and denoising_value_valid(denoising_end) + and denoising_value_valid(denoising_start) + and denoising_start >= denoising_end + ): + raise ValueError( + f"`denoising_start`: {denoising_start} cannot be larger than or equal to `denoising_end`: " + + f" {denoising_end} when using type float." + ) + elif denoising_end is not None and denoising_value_valid(denoising_end): + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + # concat latents, mask, masked_image_latents in the channel dimension + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + + # controlnet(s) inference + if guess_mode and self.do_classifier_free_guidance: + # Infer ControlNet only for the conditional batch. + control_model_input = latents + control_model_input = self.scheduler.scale_model_input(control_model_input, t) + controlnet_prompt_embeds = prompt_embeds.chunk(2)[1] + controlnet_added_cond_kwargs = { + "text_embeds": add_text_embeds.chunk(2)[1], + "time_ids": add_time_ids.chunk(2)[1], + } + else: + control_model_input = latent_model_input + controlnet_prompt_embeds = prompt_embeds + controlnet_added_cond_kwargs = added_cond_kwargs + + if isinstance(controlnet_keep[i], list): + cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] + else: + controlnet_cond_scale = controlnet_conditioning_scale + if isinstance(controlnet_cond_scale, list): + controlnet_cond_scale = controlnet_cond_scale[0] + cond_scale = controlnet_cond_scale * controlnet_keep[i] + + # # Resize control_image to match the size of the input to the controlnet + # if control_image.shape[-2:] != control_model_input.shape[-2:]: + # control_image = F.interpolate(control_image, size=control_model_input.shape[-2:], mode="bilinear", align_corners=False) + + down_block_res_samples, mid_block_res_sample = self.controlnet( + control_model_input, + t, + encoder_hidden_states=controlnet_prompt_embeds, + controlnet_cond=control_image, + conditioning_scale=cond_scale, + guess_mode=guess_mode, + added_cond_kwargs=controlnet_added_cond_kwargs, + return_dict=False, + ) + + if guess_mode and self.do_classifier_free_guidance: + # Infered ControlNet only for the conditional batch. + # To apply the output of ControlNet to both the unconditional and conditional batches, + # add 0 to the unconditional batch to keep it unchanged. + down_block_res_samples = [torch.cat([torch.zeros_like(d), d]) for d in down_block_res_samples] + mid_block_res_sample = torch.cat([torch.zeros_like(mid_block_res_sample), mid_block_res_sample]) + + if ip_adapter_image is not None: + added_cond_kwargs["image_embeds"] = image_embeds + + if num_channels_unet == 9: + latent_model_input = torch.cat([latent_model_input, mask, masked_image_latents], dim=1) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=self.cross_attention_kwargs, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if self.do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if num_channels_unet == 4: + init_latents_proper = image_latents + if self.do_classifier_free_guidance: + init_mask, _ = mask.chunk(2) + else: + init_mask = mask + + if i < len(timesteps) - 1: + noise_timestep = timesteps[i + 1] + init_latents_proper = self.scheduler.add_noise( + init_latents_proper, noise, torch.tensor([noise_timestep]) + ) + + latents = (1 - init_mask) * init_latents_proper + init_mask * latents + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # make sure the VAE is in float32 mode, as it overflows in float16 + if self.vae.dtype == torch.float16 and self.vae.config.force_upcast: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + # If we do sequential model offloading, let's offload unet and controlnet + # manually for max memory savings + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.unet.to("cpu") + self.controlnet.to("cpu") + torch.cuda.empty_cache() + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + return StableDiffusionXLPipelineOutput(images=latents) + + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + if padding_mask_crop is not None: + image = [self.image_processor.apply_overlay(mask_image, original_image, i, crops_coords) for i in image] + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return StableDiffusionXLPipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py new file mode 100755 index 0000000..eca8108 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl.py @@ -0,0 +1,1499 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from diffusers.utils.import_utils import is_invisible_watermark_available + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import ( + FromSingleFileMixin, + IPAdapterMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, +) +from ...models import AutoencoderKL, ControlNetModel, ImageProjection, UNet2DConditionModel +from ...models.attention_processor import ( + AttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import is_compiled_module, is_torch_version, randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput + + +if is_invisible_watermark_available(): + from ..stable_diffusion_xl.watermark import StableDiffusionXLWatermarker + +from .multicontrolnet import MultiControlNetModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> # !pip install opencv-python transformers accelerate + >>> from diffusers import StableDiffusionXLControlNetPipeline, ControlNetModel, AutoencoderKL + >>> from diffusers.utils import load_image + >>> import numpy as np + >>> import torch + + >>> import cv2 + >>> from PIL import Image + + >>> prompt = "aerial view, a futuristic research complex in a bright foggy jungle, hard lighting" + >>> negative_prompt = "low quality, bad quality, sketches" + + >>> # download an image + >>> image = load_image( + ... "https://hf.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/hf-logo.png" + ... ) + + >>> # initialize the models and pipeline + >>> controlnet_conditioning_scale = 0.5 # recommended for good generalization + >>> controlnet = ControlNetModel.from_pretrained( + ... "diffusers/controlnet-canny-sdxl-1.0", torch_dtype=torch.float16 + ... ) + >>> vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16) + >>> pipe = StableDiffusionXLControlNetPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-xl-base-1.0", controlnet=controlnet, vae=vae, torch_dtype=torch.float16 + ... ) + >>> pipe.enable_model_cpu_offload() + + >>> # get canny image + >>> image = np.array(image) + >>> image = cv2.Canny(image, 100, 200) + >>> image = image[:, :, None] + >>> image = np.concatenate([image, image, image], axis=2) + >>> canny_image = Image.fromarray(image) + + >>> # generate image + >>> image = pipe( + ... prompt, controlnet_conditioning_scale=controlnet_conditioning_scale, image=canny_image + ... ).images[0] + ``` +""" + + +class StableDiffusionXLControlNetPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + StableDiffusionXLLoraLoaderMixin, + IPAdapterMixin, + FromSingleFileMixin, +): + r""" + Pipeline for text-to-image generation using Stable Diffusion XL with ControlNet guidance. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + text_encoder_2 ([`~transformers.CLIPTextModelWithProjection`]): + Second frozen text-encoder + ([laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + tokenizer_2 ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + Provides additional conditioning to the `unet` during the denoising process. If you set multiple + ControlNets as a list, the outputs from each ControlNet are added together to create one combined + additional conditioning. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): + Whether the negative prompt embeddings should always be set to 0. Also see the config of + `stabilityai/stable-diffusion-xl-base-1-0`. + add_watermarker (`bool`, *optional*): + Whether to use the [invisible_watermark](https://github.com/ShieldMnt/invisible-watermark/) library to + watermark output images. If not defined, it defaults to `True` if the package is installed; otherwise no + watermarker is used. + """ + + # leave controlnet out on purpose because it iterates with unet + model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae" + _optional_components = [ + "tokenizer", + "tokenizer_2", + "text_encoder", + "text_encoder_2", + "feature_extractor", + "image_encoder", + ] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + scheduler: KarrasDiffusionSchedulers, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + feature_extractor: CLIPImageProcessor = None, + image_encoder: CLIPVisionModelWithProjection = None, + ): + super().__init__() + + if isinstance(controlnet, (list, tuple)): + controlnet = MultiControlNetModel(controlnet) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + controlnet=controlnet, + scheduler=scheduler, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True) + self.control_image_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=False + ) + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() + + if add_watermarker: + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None + + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # textual inversion: process multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_2 = ( + batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 + ) + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + prompt_2, + image, + callback_steps, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + negative_pooled_prompt_embeds=None, + controlnet_conditioning_scale=1.0, + control_guidance_start=0.0, + control_guidance_end=1.0, + callback_on_step_end_tensor_inputs=None, + ): + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + # `prompt` needs more sophisticated handling when there are multiple + # conditionings. + if isinstance(self.controlnet, MultiControlNetModel): + if isinstance(prompt, list): + logger.warning( + f"You have {len(self.controlnet.nets)} ControlNets and you have passed {len(prompt)}" + " prompts. The conditionings will be fixed across the prompts." + ) + + # Check `image` + is_compiled = hasattr(F, "scaled_dot_product_attention") and isinstance( + self.controlnet, torch._dynamo.eval_frame.OptimizedModule + ) + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + self.check_image(image, prompt, prompt_embeds) + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if not isinstance(image, list): + raise TypeError("For multiple controlnets: `image` must be type `list`") + + # When `image` is a nested list: + # (e.g. [[canny_image_1, pose_image_1], [canny_image_2, pose_image_2]]) + elif any(isinstance(i, list) for i in image): + raise ValueError("A single batch of multiple conditionings are supported at the moment.") + elif len(image) != len(self.controlnet.nets): + raise ValueError( + f"For multiple controlnets: `image` must have the same length as the number of controlnets, but got {len(image)} images and {len(self.controlnet.nets)} ControlNets." + ) + + for image_ in image: + self.check_image(image_, prompt, prompt_embeds) + else: + assert False + + # Check `controlnet_conditioning_scale` + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + if not isinstance(controlnet_conditioning_scale, float): + raise TypeError("For single controlnet: `controlnet_conditioning_scale` must be type `float`.") + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if isinstance(controlnet_conditioning_scale, list): + if any(isinstance(i, list) for i in controlnet_conditioning_scale): + raise ValueError("A single batch of multiple conditionings are supported at the moment.") + elif isinstance(controlnet_conditioning_scale, list) and len(controlnet_conditioning_scale) != len( + self.controlnet.nets + ): + raise ValueError( + "For multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have" + " the same length as the number of controlnets" + ) + else: + assert False + + if not isinstance(control_guidance_start, (tuple, list)): + control_guidance_start = [control_guidance_start] + + if not isinstance(control_guidance_end, (tuple, list)): + control_guidance_end = [control_guidance_end] + + if len(control_guidance_start) != len(control_guidance_end): + raise ValueError( + f"`control_guidance_start` has {len(control_guidance_start)} elements, but `control_guidance_end` has {len(control_guidance_end)} elements. Make sure to provide the same number of elements to each list." + ) + + if isinstance(self.controlnet, MultiControlNetModel): + if len(control_guidance_start) != len(self.controlnet.nets): + raise ValueError( + f"`control_guidance_start`: {control_guidance_start} has {len(control_guidance_start)} elements but there are {len(self.controlnet.nets)} controlnets available. Make sure to provide {len(self.controlnet.nets)}." + ) + + for start, end in zip(control_guidance_start, control_guidance_end): + if start >= end: + raise ValueError( + f"control guidance start: {start} cannot be larger or equal to control guidance end: {end}." + ) + if start < 0.0: + raise ValueError(f"control guidance start: {start} can't be smaller than 0.") + if end > 1.0: + raise ValueError(f"control guidance end: {end} can't be larger than 1.0.") + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + # Copied from diffusers.pipelines.controlnet.pipeline_controlnet.StableDiffusionControlNetPipeline.check_image + def check_image(self, image, prompt, prompt_embeds): + image_is_pil = isinstance(image, PIL.Image.Image) + image_is_tensor = isinstance(image, torch.Tensor) + image_is_np = isinstance(image, np.ndarray) + image_is_pil_list = isinstance(image, list) and isinstance(image[0], PIL.Image.Image) + image_is_tensor_list = isinstance(image, list) and isinstance(image[0], torch.Tensor) + image_is_np_list = isinstance(image, list) and isinstance(image[0], np.ndarray) + + if ( + not image_is_pil + and not image_is_tensor + and not image_is_np + and not image_is_pil_list + and not image_is_tensor_list + and not image_is_np_list + ): + raise TypeError( + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" + ) + + if image_is_pil: + image_batch_size = 1 + else: + image_batch_size = len(image) + + if prompt is not None and isinstance(prompt, str): + prompt_batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + prompt_batch_size = len(prompt) + elif prompt_embeds is not None: + prompt_batch_size = prompt_embeds.shape[0] + + if image_batch_size != 1 and image_batch_size != prompt_batch_size: + raise ValueError( + f"If image batch size is not 1, image batch size must be same as prompt batch size. image batch size: {image_batch_size}, prompt batch size: {prompt_batch_size}" + ) + + # Copied from diffusers.pipelines.controlnet.pipeline_controlnet.StableDiffusionControlNetPipeline.prepare_image + def prepare_image( + self, + image, + width, + height, + batch_size, + num_images_per_prompt, + device, + dtype, + do_classifier_free_guidance=False, + guess_mode=False, + ): + image = self.control_image_processor.preprocess(image, height=height, width=width).to(dtype=torch.float32) + image_batch_size = image.shape[0] + + if image_batch_size == 1: + repeat_by = batch_size + else: + # image batch size is the same as prompt batch size + repeat_by = num_images_per_prompt + + image = image.repeat_interleave(repeat_by, dim=0) + + image = image.to(device=device, dtype=dtype) + + if do_classifier_free_guidance and not guess_mode: + image = torch.cat([image] * 2) + + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._get_add_time_ids + def _get_add_time_ids( + self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None + ): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + return add_time_ids + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def denoising_end(self): + return self._denoising_end + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + image: PipelineImageInput = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + denoising_end: Optional[float] = None, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_conditioning_scale: Union[float, List[float]] = 1.0, + guess_mode: bool = False, + control_guidance_start: Union[float, List[float]] = 0.0, + control_guidance_end: Union[float, List[float]] = 1.0, + original_size: Tuple[int, int] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Tuple[int, int] = None, + negative_original_size: Optional[Tuple[int, int]] = None, + negative_crops_coords_top_left: Tuple[int, int] = (0, 0), + negative_target_size: Optional[Tuple[int, int]] = None, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: + `List[List[torch.FloatTensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + The ControlNet input condition to provide guidance to the `unet` for generation. If the type is + specified as `torch.FloatTensor`, it is passed to ControlNet as is. `PIL.Image.Image` can also be + accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If height + and/or width are passed, `image` is resized accordingly. If multiple ControlNets are specified in + `init`, images must be passed as a list such that each element of the list can be correctly batched for + input to a single ControlNet. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise as determined by the discrete timesteps selected by the + scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a + "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) + guidance_scale (`float`, *optional*, defaults to 5.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. This is sent to `tokenizer_2` + and `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, pooled text embeddings are generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs (prompt + weighting). If not provided, pooled `negative_prompt_embeds` are generated from `negative_prompt` input + argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added + to the residual in the original `unet`. If multiple ControlNets are specified in `init`, you can set + the corresponding scale as a list. + guess_mode (`bool`, *optional*, defaults to `False`): + The ControlNet encoder tries to recognize the content of the input image even if you remove all + prompts. A `guidance_scale` value between 3.0 and 5.0 is recommended. + control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + The percentage of total steps at which the ControlNet starts applying. + control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + The percentage of total steps at which the ControlNet stops applying. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a specific image resolution. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a target image resolution. It should be as same + as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeine class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned containing the output images. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + + controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet + + # align format for control guidance + if not isinstance(control_guidance_start, list) and isinstance(control_guidance_end, list): + control_guidance_start = len(control_guidance_end) * [control_guidance_start] + elif not isinstance(control_guidance_end, list) and isinstance(control_guidance_start, list): + control_guidance_end = len(control_guidance_start) * [control_guidance_end] + elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list): + mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1 + control_guidance_start, control_guidance_end = ( + mult * [control_guidance_start], + mult * [control_guidance_end], + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + image, + callback_steps, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + negative_pooled_prompt_embeds, + controlnet_conditioning_scale, + control_guidance_start, + control_guidance_end, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._denoising_end = denoising_end + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float): + controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets) + + global_pool_conditions = ( + controlnet.config.global_pool_conditions + if isinstance(controlnet, ControlNetModel) + else controlnet.nets[0].config.global_pool_conditions + ) + guess_mode = guess_mode or global_pool_conditions + + # 3.1 Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt, + prompt_2, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + + # 3.2 Encode ip_adapter_image + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Prepare image + if isinstance(controlnet, ControlNetModel): + image = self.prepare_image( + image=image, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + height, width = image.shape[-2:] + elif isinstance(controlnet, MultiControlNetModel): + images = [] + + for image_ in image: + image_ = self.prepare_image( + image=image_, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + + images.append(image_) + + image = images + height, width = image[0].shape[-2:] + else: + assert False + + # 5. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + self._num_timesteps = len(timesteps) + + # 6. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6.5 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7.1 Create tensor stating which controlnets to keep + controlnet_keep = [] + for i in range(len(timesteps)): + keeps = [ + 1.0 - float(i / len(timesteps) < s or (i + 1) / len(timesteps) > e) + for s, e in zip(control_guidance_start, control_guidance_end) + ] + controlnet_keep.append(keeps[0] if isinstance(controlnet, ControlNetModel) else keeps) + + # 7.2 Prepare added time ids & embeddings + if isinstance(image, list): + original_size = original_size or image[0].shape[-2:] + else: + original_size = original_size or image.shape[-2:] + target_size = target_size or (height, width) + + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + + if negative_original_size is not None and negative_target_size is not None: + negative_add_time_ids = self._get_add_time_ids( + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + else: + negative_add_time_ids = add_time_ids + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1) + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + + # 8.1 Apply denoising_end + if ( + self.denoising_end is not None + and isinstance(self.denoising_end, float) + and self.denoising_end > 0 + and self.denoising_end < 1 + ): + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (self.denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + is_unet_compiled = is_compiled_module(self.unet) + is_controlnet_compiled = is_compiled_module(self.controlnet) + is_torch_higher_equal_2_1 = is_torch_version(">=", "2.1") + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # Relevant thread: + # https://dev-discuss.pytorch.org/t/cudagraphs-in-pytorch-2-0/1428 + if (is_unet_compiled and is_controlnet_compiled) and is_torch_higher_equal_2_1: + torch._inductor.cudagraph_mark_step_begin() + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + + # controlnet(s) inference + if guess_mode and self.do_classifier_free_guidance: + # Infer ControlNet only for the conditional batch. + control_model_input = latents + control_model_input = self.scheduler.scale_model_input(control_model_input, t) + controlnet_prompt_embeds = prompt_embeds.chunk(2)[1] + controlnet_added_cond_kwargs = { + "text_embeds": add_text_embeds.chunk(2)[1], + "time_ids": add_time_ids.chunk(2)[1], + } + else: + control_model_input = latent_model_input + controlnet_prompt_embeds = prompt_embeds + controlnet_added_cond_kwargs = added_cond_kwargs + + if isinstance(controlnet_keep[i], list): + cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] + else: + controlnet_cond_scale = controlnet_conditioning_scale + if isinstance(controlnet_cond_scale, list): + controlnet_cond_scale = controlnet_cond_scale[0] + cond_scale = controlnet_cond_scale * controlnet_keep[i] + + down_block_res_samples, mid_block_res_sample = self.controlnet( + control_model_input, + t, + encoder_hidden_states=controlnet_prompt_embeds, + controlnet_cond=image, + conditioning_scale=cond_scale, + guess_mode=guess_mode, + added_cond_kwargs=controlnet_added_cond_kwargs, + return_dict=False, + ) + + if guess_mode and self.do_classifier_free_guidance: + # Infered ControlNet only for the conditional batch. + # To apply the output of ControlNet to both the unconditional and conditional batches, + # add 0 to the unconditional batch to keep it unchanged. + down_block_res_samples = [torch.cat([torch.zeros_like(d), d]) for d in down_block_res_samples] + mid_block_res_sample = torch.cat([torch.zeros_like(mid_block_res_sample), mid_block_res_sample]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + added_cond_kwargs["image_embeds"] = image_embeds + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + # unscale/denormalize the latents + # denormalize with the mean and std if available and not None + has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None + has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None + if has_latents_mean and has_latents_std: + latents_mean = ( + torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents_std = ( + torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean + else: + latents = latents / self.vae.config.scaling_factor + + image = self.vae.decode(latents, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + image = latents + + if not output_type == "latent": + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return StableDiffusionXLPipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py new file mode 100755 index 0000000..86a0e2c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_controlnet_sd_xl_img2img.py @@ -0,0 +1,1626 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from diffusers.utils.import_utils import is_invisible_watermark_available + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import ( + IPAdapterMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, +) +from ...models import AutoencoderKL, ControlNetModel, ImageProjection, UNet2DConditionModel +from ...models.attention_processor import ( + AttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import is_compiled_module, randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput + + +if is_invisible_watermark_available(): + from ..stable_diffusion_xl.watermark import StableDiffusionXLWatermarker + +from .multicontrolnet import MultiControlNetModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> # pip install accelerate transformers safetensors diffusers + + >>> import torch + >>> import numpy as np + >>> from PIL import Image + + >>> from transformers import DPTFeatureExtractor, DPTForDepthEstimation + >>> from diffusers import ControlNetModel, StableDiffusionXLControlNetImg2ImgPipeline, AutoencoderKL + >>> from diffusers.utils import load_image + + + >>> depth_estimator = DPTForDepthEstimation.from_pretrained("Intel/dpt-hybrid-midas").to("cuda") + >>> feature_extractor = DPTFeatureExtractor.from_pretrained("Intel/dpt-hybrid-midas") + >>> controlnet = ControlNetModel.from_pretrained( + ... "diffusers/controlnet-depth-sdxl-1.0-small", + ... variant="fp16", + ... use_safetensors=True, + ... torch_dtype=torch.float16, + ... ).to("cuda") + >>> vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16).to("cuda") + >>> pipe = StableDiffusionXLControlNetImg2ImgPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-xl-base-1.0", + ... controlnet=controlnet, + ... vae=vae, + ... variant="fp16", + ... use_safetensors=True, + ... torch_dtype=torch.float16, + ... ).to("cuda") + >>> pipe.enable_model_cpu_offload() + + + >>> def get_depth_map(image): + ... image = feature_extractor(images=image, return_tensors="pt").pixel_values.to("cuda") + ... with torch.no_grad(), torch.autocast("cuda"): + ... depth_map = depth_estimator(image).predicted_depth + + ... depth_map = torch.nn.functional.interpolate( + ... depth_map.unsqueeze(1), + ... size=(1024, 1024), + ... mode="bicubic", + ... align_corners=False, + ... ) + ... depth_min = torch.amin(depth_map, dim=[1, 2, 3], keepdim=True) + ... depth_max = torch.amax(depth_map, dim=[1, 2, 3], keepdim=True) + ... depth_map = (depth_map - depth_min) / (depth_max - depth_min) + ... image = torch.cat([depth_map] * 3, dim=1) + ... image = image.permute(0, 2, 3, 1).cpu().numpy()[0] + ... image = Image.fromarray((image * 255.0).clip(0, 255).astype(np.uint8)) + ... return image + + + >>> prompt = "A robot, 4k photo" + >>> image = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/cat.png" + ... ).resize((1024, 1024)) + >>> controlnet_conditioning_scale = 0.5 # recommended for good generalization + >>> depth_image = get_depth_map(image) + + >>> images = pipe( + ... prompt, + ... image=image, + ... control_image=depth_image, + ... strength=0.99, + ... num_inference_steps=50, + ... controlnet_conditioning_scale=controlnet_conditioning_scale, + ... ).images + >>> images[0].save(f"robot_cat.png") + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +class StableDiffusionXLControlNetImg2ImgPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + StableDiffusionXLLoraLoaderMixin, + IPAdapterMixin, +): + r""" + Pipeline for image-to-image generation using Stable Diffusion XL with ControlNet guidance. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + controlnet ([`ControlNetModel`] or `List[ControlNetModel]`): + Provides additional conditioning to the unet during the denoising process. If you set multiple ControlNets + as a list, the outputs from each ControlNet are added together to create one combined additional + conditioning. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + requires_aesthetics_score (`bool`, *optional*, defaults to `"False"`): + Whether the `unet` requires an `aesthetic_score` condition to be passed during inference. Also see the + config of `stabilityai/stable-diffusion-xl-refiner-1-0`. + force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): + Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of + `stabilityai/stable-diffusion-xl-base-1-0`. + add_watermarker (`bool`, *optional*): + Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to + watermark output images. If not defined, it will default to True if the package is installed, otherwise no + watermarker will be used. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae" + _optional_components = [ + "tokenizer", + "tokenizer_2", + "text_encoder", + "text_encoder_2", + "feature_extractor", + "image_encoder", + ] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + controlnet: Union[ControlNetModel, List[ControlNetModel], Tuple[ControlNetModel], MultiControlNetModel], + scheduler: KarrasDiffusionSchedulers, + requires_aesthetics_score: bool = False, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + feature_extractor: CLIPImageProcessor = None, + image_encoder: CLIPVisionModelWithProjection = None, + ): + super().__init__() + + if isinstance(controlnet, (list, tuple)): + controlnet = MultiControlNetModel(controlnet) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + controlnet=controlnet, + scheduler=scheduler, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True) + self.control_image_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True, do_normalize=False + ) + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() + + if add_watermarker: + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None + + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.register_to_config(requires_aesthetics_score=requires_aesthetics_score) + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # textual inversion: process multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_2 = ( + batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 + ) + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + prompt_2, + image, + strength, + num_inference_steps, + callback_steps, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + controlnet_conditioning_scale=1.0, + control_guidance_start=0.0, + control_guidance_end=1.0, + callback_on_step_end_tensor_inputs=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + if num_inference_steps is None: + raise ValueError("`num_inference_steps` cannot be None.") + elif not isinstance(num_inference_steps, int) or num_inference_steps <= 0: + raise ValueError( + f"`num_inference_steps` has to be a positive integer but is {num_inference_steps} of type" + f" {type(num_inference_steps)}." + ) + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + # `prompt` needs more sophisticated handling when there are multiple + # conditionings. + if isinstance(self.controlnet, MultiControlNetModel): + if isinstance(prompt, list): + logger.warning( + f"You have {len(self.controlnet.nets)} ControlNets and you have passed {len(prompt)}" + " prompts. The conditionings will be fixed across the prompts." + ) + + # Check `image` + is_compiled = hasattr(F, "scaled_dot_product_attention") and isinstance( + self.controlnet, torch._dynamo.eval_frame.OptimizedModule + ) + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + self.check_image(image, prompt, prompt_embeds) + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if not isinstance(image, list): + raise TypeError("For multiple controlnets: `image` must be type `list`") + + # When `image` is a nested list: + # (e.g. [[canny_image_1, pose_image_1], [canny_image_2, pose_image_2]]) + elif any(isinstance(i, list) for i in image): + raise ValueError("A single batch of multiple conditionings are supported at the moment.") + elif len(image) != len(self.controlnet.nets): + raise ValueError( + f"For multiple controlnets: `image` must have the same length as the number of controlnets, but got {len(image)} images and {len(self.controlnet.nets)} ControlNets." + ) + + for image_ in image: + self.check_image(image_, prompt, prompt_embeds) + else: + assert False + + # Check `controlnet_conditioning_scale` + if ( + isinstance(self.controlnet, ControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, ControlNetModel) + ): + if not isinstance(controlnet_conditioning_scale, float): + raise TypeError("For single controlnet: `controlnet_conditioning_scale` must be type `float`.") + elif ( + isinstance(self.controlnet, MultiControlNetModel) + or is_compiled + and isinstance(self.controlnet._orig_mod, MultiControlNetModel) + ): + if isinstance(controlnet_conditioning_scale, list): + if any(isinstance(i, list) for i in controlnet_conditioning_scale): + raise ValueError("A single batch of multiple conditionings are supported at the moment.") + elif isinstance(controlnet_conditioning_scale, list) and len(controlnet_conditioning_scale) != len( + self.controlnet.nets + ): + raise ValueError( + "For multiple controlnets: When `controlnet_conditioning_scale` is specified as `list`, it must have" + " the same length as the number of controlnets" + ) + else: + assert False + + if not isinstance(control_guidance_start, (tuple, list)): + control_guidance_start = [control_guidance_start] + + if not isinstance(control_guidance_end, (tuple, list)): + control_guidance_end = [control_guidance_end] + + if len(control_guidance_start) != len(control_guidance_end): + raise ValueError( + f"`control_guidance_start` has {len(control_guidance_start)} elements, but `control_guidance_end` has {len(control_guidance_end)} elements. Make sure to provide the same number of elements to each list." + ) + + if isinstance(self.controlnet, MultiControlNetModel): + if len(control_guidance_start) != len(self.controlnet.nets): + raise ValueError( + f"`control_guidance_start`: {control_guidance_start} has {len(control_guidance_start)} elements but there are {len(self.controlnet.nets)} controlnets available. Make sure to provide {len(self.controlnet.nets)}." + ) + + for start, end in zip(control_guidance_start, control_guidance_end): + if start >= end: + raise ValueError( + f"control guidance start: {start} cannot be larger or equal to control guidance end: {end}." + ) + if start < 0.0: + raise ValueError(f"control guidance start: {start} can't be smaller than 0.") + if end > 1.0: + raise ValueError(f"control guidance end: {end} can't be larger than 1.0.") + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + # Copied from diffusers.pipelines.controlnet.pipeline_controlnet_sd_xl.StableDiffusionXLControlNetPipeline.check_image + def check_image(self, image, prompt, prompt_embeds): + image_is_pil = isinstance(image, PIL.Image.Image) + image_is_tensor = isinstance(image, torch.Tensor) + image_is_np = isinstance(image, np.ndarray) + image_is_pil_list = isinstance(image, list) and isinstance(image[0], PIL.Image.Image) + image_is_tensor_list = isinstance(image, list) and isinstance(image[0], torch.Tensor) + image_is_np_list = isinstance(image, list) and isinstance(image[0], np.ndarray) + + if ( + not image_is_pil + and not image_is_tensor + and not image_is_np + and not image_is_pil_list + and not image_is_tensor_list + and not image_is_np_list + ): + raise TypeError( + f"image must be passed and be one of PIL image, numpy array, torch tensor, list of PIL images, list of numpy arrays or list of torch tensors, but is {type(image)}" + ) + + if image_is_pil: + image_batch_size = 1 + else: + image_batch_size = len(image) + + if prompt is not None and isinstance(prompt, str): + prompt_batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + prompt_batch_size = len(prompt) + elif prompt_embeds is not None: + prompt_batch_size = prompt_embeds.shape[0] + + if image_batch_size != 1 and image_batch_size != prompt_batch_size: + raise ValueError( + f"If image batch size is not 1, image batch size must be same as prompt batch size. image batch size: {image_batch_size}, prompt batch size: {prompt_batch_size}" + ) + + # Copied from diffusers.pipelines.controlnet.pipeline_controlnet_sd_xl.StableDiffusionXLControlNetPipeline.prepare_image + def prepare_control_image( + self, + image, + width, + height, + batch_size, + num_images_per_prompt, + device, + dtype, + do_classifier_free_guidance=False, + guess_mode=False, + ): + image = self.control_image_processor.preprocess(image, height=height, width=width).to(dtype=torch.float32) + image_batch_size = image.shape[0] + + if image_batch_size == 1: + repeat_by = batch_size + else: + # image batch size is the same as prompt batch size + repeat_by = num_images_per_prompt + + image = image.repeat_interleave(repeat_by, dim=0) + + image = image.to(device=device, dtype=dtype) + + if do_classifier_free_guidance and not guess_mode: + image = torch.cat([image] * 2) + + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img.StableDiffusionXLImg2ImgPipeline.prepare_latents + def prepare_latents( + self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None, add_noise=True + ): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + # Offload text encoder if `enable_model_cpu_offload` was enabled + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.text_encoder_2.to("cpu") + torch.cuda.empty_cache() + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + init_latents = image + + else: + # make sure the VAE is in float32 mode, as it overflows in float16 + if self.vae.config.force_upcast: + image = image.float() + self.vae.to(dtype=torch.float32) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + elif isinstance(generator, list): + init_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(batch_size) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + if self.vae.config.force_upcast: + self.vae.to(dtype) + + init_latents = init_latents.to(dtype) + init_latents = self.vae.config.scaling_factor * init_latents + + if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0: + # expand init_latents for batch_size + additional_image_per_prompt = batch_size // init_latents.shape[0] + init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0) + elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts." + ) + else: + init_latents = torch.cat([init_latents], dim=0) + + if add_noise: + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + + latents = init_latents + + return latents + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img.StableDiffusionXLImg2ImgPipeline._get_add_time_ids + def _get_add_time_ids( + self, + original_size, + crops_coords_top_left, + target_size, + aesthetic_score, + negative_aesthetic_score, + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype, + text_encoder_projection_dim=None, + ): + if self.config.requires_aesthetics_score: + add_time_ids = list(original_size + crops_coords_top_left + (aesthetic_score,)) + add_neg_time_ids = list( + negative_original_size + negative_crops_coords_top_left + (negative_aesthetic_score,) + ) + else: + add_time_ids = list(original_size + crops_coords_top_left + target_size) + add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if ( + expected_add_embed_dim > passed_add_embed_dim + and (expected_add_embed_dim - passed_add_embed_dim) == self.unet.config.addition_time_embed_dim + ): + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to enable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=True)` to make sure `aesthetic_score` {aesthetic_score} and `negative_aesthetic_score` {negative_aesthetic_score} is correctly used by the model." + ) + elif ( + expected_add_embed_dim < passed_add_embed_dim + and (passed_add_embed_dim - expected_add_embed_dim) == self.unet.config.addition_time_embed_dim + ): + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to disable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=False)` to make sure `target_size` {target_size} is correctly used by the model." + ) + elif expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + add_neg_time_ids = torch.tensor([add_neg_time_ids], dtype=dtype) + + return add_time_ids, add_neg_time_ids + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + image: PipelineImageInput = None, + control_image: PipelineImageInput = None, + height: Optional[int] = None, + width: Optional[int] = None, + strength: float = 0.8, + num_inference_steps: int = 50, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + controlnet_conditioning_scale: Union[float, List[float]] = 0.8, + guess_mode: bool = False, + control_guidance_start: Union[float, List[float]] = 0.0, + control_guidance_end: Union[float, List[float]] = 1.0, + original_size: Tuple[int, int] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Tuple[int, int] = None, + negative_original_size: Optional[Tuple[int, int]] = None, + negative_crops_coords_top_left: Tuple[int, int] = (0, 0), + negative_target_size: Optional[Tuple[int, int]] = None, + aesthetic_score: float = 6.0, + negative_aesthetic_score: float = 2.5, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: + `List[List[torch.FloatTensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + The initial image will be used as the starting point for the image generation process. Can also accept + image latents as `image`, if passing latents directly, it will not be encoded again. + control_image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, `List[np.ndarray]`,: + `List[List[torch.FloatTensor]]`, `List[List[np.ndarray]]` or `List[List[PIL.Image.Image]]`): + The ControlNet input condition. ControlNet uses this input condition to generate guidance to Unet. If + the type is specified as `Torch.FloatTensor`, it is passed to ControlNet as is. `PIL.Image.Image` can + also be accepted as an image. The dimensions of the output image defaults to `image`'s dimensions. If + height and/or width are passed, `image` is resized according to them. If multiple ControlNets are + specified in init, images must be passed as a list such that each element of the list can be correctly + batched for input to a single controlnet. + height (`int`, *optional*, defaults to the size of control_image): + The height in pixels of the generated image. Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + width (`int`, *optional*, defaults to the size of control_image): + The width in pixels of the generated image. Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + strength (`float`, *optional*, defaults to 0.8): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + controlnet_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + The outputs of the controlnet are multiplied by `controlnet_conditioning_scale` before they are added + to the residual in the original unet. If multiple ControlNets are specified in init, you can set the + corresponding scale as a list. + guess_mode (`bool`, *optional*, defaults to `False`): + In this mode, the ControlNet encoder will try best to recognize the content of the input image even if + you remove all prompts. The `guidance_scale` between 3.0 and 5.0 is recommended. + control_guidance_start (`float` or `List[float]`, *optional*, defaults to 0.0): + The percentage of total steps at which the controlnet starts applying. + control_guidance_end (`float` or `List[float]`, *optional*, defaults to 1.0): + The percentage of total steps at which the controlnet stops applying. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a specific image resolution. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a target image resolution. It should be as same + as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + aesthetic_score (`float`, *optional*, defaults to 6.0): + Used to simulate an aesthetic score of the generated image by influencing the positive text condition. + Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_aesthetic_score (`float`, *optional*, defaults to 2.5): + Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). Can be used to + simulate an aesthetic score of the generated image by influencing the negative text condition. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeine class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple` + containing the output images. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + + controlnet = self.controlnet._orig_mod if is_compiled_module(self.controlnet) else self.controlnet + + # align format for control guidance + if not isinstance(control_guidance_start, list) and isinstance(control_guidance_end, list): + control_guidance_start = len(control_guidance_end) * [control_guidance_start] + elif not isinstance(control_guidance_end, list) and isinstance(control_guidance_start, list): + control_guidance_end = len(control_guidance_start) * [control_guidance_end] + elif not isinstance(control_guidance_start, list) and not isinstance(control_guidance_end, list): + mult = len(controlnet.nets) if isinstance(controlnet, MultiControlNetModel) else 1 + control_guidance_start, control_guidance_end = ( + mult * [control_guidance_start], + mult * [control_guidance_end], + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + control_image, + strength, + num_inference_steps, + callback_steps, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + controlnet_conditioning_scale, + control_guidance_start, + control_guidance_end, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + if isinstance(controlnet, MultiControlNetModel) and isinstance(controlnet_conditioning_scale, float): + controlnet_conditioning_scale = [controlnet_conditioning_scale] * len(controlnet.nets) + + global_pool_conditions = ( + controlnet.config.global_pool_conditions + if isinstance(controlnet, ControlNetModel) + else controlnet.nets[0].config.global_pool_conditions + ) + guess_mode = guess_mode or global_pool_conditions + + # 3.1. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt, + prompt_2, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + + # 3.2 Encode ip_adapter_image + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Prepare image and controlnet_conditioning_image + image = self.image_processor.preprocess(image, height=height, width=width).to(dtype=torch.float32) + + if isinstance(controlnet, ControlNetModel): + control_image = self.prepare_control_image( + image=control_image, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + height, width = control_image.shape[-2:] + elif isinstance(controlnet, MultiControlNetModel): + control_images = [] + + for control_image_ in control_image: + control_image_ = self.prepare_control_image( + image=control_image_, + width=width, + height=height, + batch_size=batch_size * num_images_per_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + dtype=controlnet.dtype, + do_classifier_free_guidance=self.do_classifier_free_guidance, + guess_mode=guess_mode, + ) + + control_images.append(control_image_) + + control_image = control_images + height, width = control_image[0].shape[-2:] + else: + assert False + + # 5. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + self._num_timesteps = len(timesteps) + + # 6. Prepare latent variables + latents = self.prepare_latents( + image, + latent_timestep, + batch_size, + num_images_per_prompt, + prompt_embeds.dtype, + device, + generator, + True, + ) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7.1 Create tensor stating which controlnets to keep + controlnet_keep = [] + for i in range(len(timesteps)): + keeps = [ + 1.0 - float(i / len(timesteps) < s or (i + 1) / len(timesteps) > e) + for s, e in zip(control_guidance_start, control_guidance_end) + ] + controlnet_keep.append(keeps[0] if isinstance(controlnet, ControlNetModel) else keeps) + + # 7.2 Prepare added time ids & embeddings + if isinstance(control_image, list): + original_size = original_size or control_image[0].shape[-2:] + else: + original_size = original_size or control_image.shape[-2:] + target_size = target_size or (height, width) + + if negative_original_size is None: + negative_original_size = original_size + if negative_target_size is None: + negative_target_size = target_size + add_text_embeds = pooled_prompt_embeds + + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids, add_neg_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + aesthetic_score, + negative_aesthetic_score, + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + add_time_ids = add_time_ids.repeat(batch_size * num_images_per_prompt, 1) + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_neg_time_ids = add_neg_time_ids.repeat(batch_size * num_images_per_prompt, 1) + add_time_ids = torch.cat([add_neg_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device) + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + + # controlnet(s) inference + if guess_mode and self.do_classifier_free_guidance: + # Infer ControlNet only for the conditional batch. + control_model_input = latents + control_model_input = self.scheduler.scale_model_input(control_model_input, t) + controlnet_prompt_embeds = prompt_embeds.chunk(2)[1] + controlnet_added_cond_kwargs = { + "text_embeds": add_text_embeds.chunk(2)[1], + "time_ids": add_time_ids.chunk(2)[1], + } + else: + control_model_input = latent_model_input + controlnet_prompt_embeds = prompt_embeds + controlnet_added_cond_kwargs = added_cond_kwargs + + if isinstance(controlnet_keep[i], list): + cond_scale = [c * s for c, s in zip(controlnet_conditioning_scale, controlnet_keep[i])] + else: + controlnet_cond_scale = controlnet_conditioning_scale + if isinstance(controlnet_cond_scale, list): + controlnet_cond_scale = controlnet_cond_scale[0] + cond_scale = controlnet_cond_scale * controlnet_keep[i] + + down_block_res_samples, mid_block_res_sample = self.controlnet( + control_model_input, + t, + encoder_hidden_states=controlnet_prompt_embeds, + controlnet_cond=control_image, + conditioning_scale=cond_scale, + guess_mode=guess_mode, + added_cond_kwargs=controlnet_added_cond_kwargs, + return_dict=False, + ) + + if guess_mode and self.do_classifier_free_guidance: + # Infered ControlNet only for the conditional batch. + # To apply the output of ControlNet to both the unconditional and conditional batches, + # add 0 to the unconditional batch to keep it unchanged. + down_block_res_samples = [torch.cat([torch.zeros_like(d), d]) for d in down_block_res_samples] + mid_block_res_sample = torch.cat([torch.zeros_like(mid_block_res_sample), mid_block_res_sample]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + added_cond_kwargs["image_embeds"] = image_embeds + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=self.cross_attention_kwargs, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # If we do sequential model offloading, let's offload unet and controlnet + # manually for max memory savings + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.unet.to("cpu") + self.controlnet.to("cpu") + torch.cuda.empty_cache() + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + # unscale/denormalize the latents + # denormalize with the mean and std if available and not None + has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None + has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None + if has_latents_mean and has_latents_std: + latents_mean = ( + torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents_std = ( + torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean + else: + latents = latents / self.vae.config.scaling_factor + + image = self.vae.decode(latents, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + image = latents + return StableDiffusionXLPipelineOutput(images=image) + + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return StableDiffusionXLPipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_flax_controlnet.py b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_flax_controlnet.py new file mode 100755 index 0000000..5b6fc2b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/controlnet/pipeline_flax_controlnet.py @@ -0,0 +1,532 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from functools import partial +from typing import Dict, List, Optional, Union + +import jax +import jax.numpy as jnp +import numpy as np +from flax.core.frozen_dict import FrozenDict +from flax.jax_utils import unreplicate +from flax.training.common_utils import shard +from PIL import Image +from transformers import CLIPFeatureExtractor, CLIPTokenizer, FlaxCLIPTextModel + +from ...models import FlaxAutoencoderKL, FlaxControlNetModel, FlaxUNet2DConditionModel +from ...schedulers import ( + FlaxDDIMScheduler, + FlaxDPMSolverMultistepScheduler, + FlaxLMSDiscreteScheduler, + FlaxPNDMScheduler, +) +from ...utils import PIL_INTERPOLATION, logging, replace_example_docstring +from ..pipeline_flax_utils import FlaxDiffusionPipeline +from ..stable_diffusion import FlaxStableDiffusionPipelineOutput +from ..stable_diffusion.safety_checker_flax import FlaxStableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +# Set to True to use python for loop instead of jax.fori_loop for easier debugging +DEBUG = False + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import jax + >>> import numpy as np + >>> import jax.numpy as jnp + >>> from flax.jax_utils import replicate + >>> from flax.training.common_utils import shard + >>> from diffusers.utils import load_image, make_image_grid + >>> from PIL import Image + >>> from diffusers import FlaxStableDiffusionControlNetPipeline, FlaxControlNetModel + + + >>> def create_key(seed=0): + ... return jax.random.PRNGKey(seed) + + + >>> rng = create_key(0) + + >>> # get canny image + >>> canny_image = load_image( + ... "https://huggingface.co/datasets/YiYiXu/test-doc-assets/resolve/main/blog_post_cell_10_output_0.jpeg" + ... ) + + >>> prompts = "best quality, extremely detailed" + >>> negative_prompts = "monochrome, lowres, bad anatomy, worst quality, low quality" + + >>> # load control net and stable diffusion v1-5 + >>> controlnet, controlnet_params = FlaxControlNetModel.from_pretrained( + ... "lllyasviel/sd-controlnet-canny", from_pt=True, dtype=jnp.float32 + ... ) + >>> pipe, params = FlaxStableDiffusionControlNetPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", controlnet=controlnet, revision="flax", dtype=jnp.float32 + ... ) + >>> params["controlnet"] = controlnet_params + + >>> num_samples = jax.device_count() + >>> rng = jax.random.split(rng, jax.device_count()) + + >>> prompt_ids = pipe.prepare_text_inputs([prompts] * num_samples) + >>> negative_prompt_ids = pipe.prepare_text_inputs([negative_prompts] * num_samples) + >>> processed_image = pipe.prepare_image_inputs([canny_image] * num_samples) + + >>> p_params = replicate(params) + >>> prompt_ids = shard(prompt_ids) + >>> negative_prompt_ids = shard(negative_prompt_ids) + >>> processed_image = shard(processed_image) + + >>> output = pipe( + ... prompt_ids=prompt_ids, + ... image=processed_image, + ... params=p_params, + ... prng_seed=rng, + ... num_inference_steps=50, + ... neg_prompt_ids=negative_prompt_ids, + ... jit=True, + ... ).images + + >>> output_images = pipe.numpy_to_pil(np.asarray(output.reshape((num_samples,) + output.shape[-3:]))) + >>> output_images = make_image_grid(output_images, num_samples // 4, 4) + >>> output_images.save("generated_image.png") + ``` +""" + + +class FlaxStableDiffusionControlNetPipeline(FlaxDiffusionPipeline): + r""" + Flax-based pipeline for text-to-image generation using Stable Diffusion with ControlNet Guidance. + + This model inherits from [`FlaxDiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`FlaxAutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.FlaxCLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`FlaxUNet2DConditionModel`]): + A `FlaxUNet2DConditionModel` to denoise the encoded image latents. + controlnet ([`FlaxControlNetModel`]: + Provides additional conditioning to the `unet` during the denoising process. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`FlaxDDIMScheduler`], [`FlaxLMSDiscreteScheduler`], [`FlaxPNDMScheduler`], or + [`FlaxDPMSolverMultistepScheduler`]. + safety_checker ([`FlaxStableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + def __init__( + self, + vae: FlaxAutoencoderKL, + text_encoder: FlaxCLIPTextModel, + tokenizer: CLIPTokenizer, + unet: FlaxUNet2DConditionModel, + controlnet: FlaxControlNetModel, + scheduler: Union[ + FlaxDDIMScheduler, FlaxPNDMScheduler, FlaxLMSDiscreteScheduler, FlaxDPMSolverMultistepScheduler + ], + safety_checker: FlaxStableDiffusionSafetyChecker, + feature_extractor: CLIPFeatureExtractor, + dtype: jnp.dtype = jnp.float32, + ): + super().__init__() + self.dtype = dtype + + if safety_checker is None: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + controlnet=controlnet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + + def prepare_text_inputs(self, prompt: Union[str, List[str]]): + if not isinstance(prompt, (str, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + text_input = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + + return text_input.input_ids + + def prepare_image_inputs(self, image: Union[Image.Image, List[Image.Image]]): + if not isinstance(image, (Image.Image, list)): + raise ValueError(f"image has to be of type `PIL.Image.Image` or list but is {type(image)}") + + if isinstance(image, Image.Image): + image = [image] + + processed_images = jnp.concatenate([preprocess(img, jnp.float32) for img in image]) + + return processed_images + + def _get_has_nsfw_concepts(self, features, params): + has_nsfw_concepts = self.safety_checker(features, params) + return has_nsfw_concepts + + def _run_safety_checker(self, images, safety_model_params, jit=False): + # safety_model_params should already be replicated when jit is True + pil_images = [Image.fromarray(image) for image in images] + features = self.feature_extractor(pil_images, return_tensors="np").pixel_values + + if jit: + features = shard(features) + has_nsfw_concepts = _p_get_has_nsfw_concepts(self, features, safety_model_params) + has_nsfw_concepts = unshard(has_nsfw_concepts) + safety_model_params = unreplicate(safety_model_params) + else: + has_nsfw_concepts = self._get_has_nsfw_concepts(features, safety_model_params) + + images_was_copied = False + for idx, has_nsfw_concept in enumerate(has_nsfw_concepts): + if has_nsfw_concept: + if not images_was_copied: + images_was_copied = True + images = images.copy() + + images[idx] = np.zeros(images[idx].shape, dtype=np.uint8) # black image + + if any(has_nsfw_concepts): + warnings.warn( + "Potential NSFW content was detected in one or more images. A black image will be returned" + " instead. Try again with a different prompt and/or seed." + ) + + return images, has_nsfw_concepts + + def _generate( + self, + prompt_ids: jnp.ndarray, + image: jnp.ndarray, + params: Union[Dict, FrozenDict], + prng_seed: jax.Array, + num_inference_steps: int, + guidance_scale: float, + latents: Optional[jnp.ndarray] = None, + neg_prompt_ids: Optional[jnp.ndarray] = None, + controlnet_conditioning_scale: float = 1.0, + ): + height, width = image.shape[-2:] + if height % 64 != 0 or width % 64 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 64 but are {height} and {width}.") + + # get prompt text embeddings + prompt_embeds = self.text_encoder(prompt_ids, params=params["text_encoder"])[0] + + # TODO: currently it is assumed `do_classifier_free_guidance = guidance_scale > 1.0` + # implement this conditional `do_classifier_free_guidance = guidance_scale > 1.0` + batch_size = prompt_ids.shape[0] + + max_length = prompt_ids.shape[-1] + + if neg_prompt_ids is None: + uncond_input = self.tokenizer( + [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="np" + ).input_ids + else: + uncond_input = neg_prompt_ids + negative_prompt_embeds = self.text_encoder(uncond_input, params=params["text_encoder"])[0] + context = jnp.concatenate([negative_prompt_embeds, prompt_embeds]) + + image = jnp.concatenate([image] * 2) + + latents_shape = ( + batch_size, + self.unet.config.in_channels, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + if latents is None: + latents = jax.random.normal(prng_seed, shape=latents_shape, dtype=jnp.float32) + else: + if latents.shape != latents_shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}") + + def loop_body(step, args): + latents, scheduler_state = args + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + latents_input = jnp.concatenate([latents] * 2) + + t = jnp.array(scheduler_state.timesteps, dtype=jnp.int32)[step] + timestep = jnp.broadcast_to(t, latents_input.shape[0]) + + latents_input = self.scheduler.scale_model_input(scheduler_state, latents_input, t) + + down_block_res_samples, mid_block_res_sample = self.controlnet.apply( + {"params": params["controlnet"]}, + jnp.array(latents_input), + jnp.array(timestep, dtype=jnp.int32), + encoder_hidden_states=context, + controlnet_cond=image, + conditioning_scale=controlnet_conditioning_scale, + return_dict=False, + ) + + # predict the noise residual + noise_pred = self.unet.apply( + {"params": params["unet"]}, + jnp.array(latents_input), + jnp.array(timestep, dtype=jnp.int32), + encoder_hidden_states=context, + down_block_additional_residuals=down_block_res_samples, + mid_block_additional_residual=mid_block_res_sample, + ).sample + + # perform guidance + noise_pred_uncond, noise_prediction_text = jnp.split(noise_pred, 2, axis=0) + noise_pred = noise_pred_uncond + guidance_scale * (noise_prediction_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents, scheduler_state = self.scheduler.step(scheduler_state, noise_pred, t, latents).to_tuple() + return latents, scheduler_state + + scheduler_state = self.scheduler.set_timesteps( + params["scheduler"], num_inference_steps=num_inference_steps, shape=latents_shape + ) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * params["scheduler"].init_noise_sigma + + if DEBUG: + # run with python for loop + for i in range(num_inference_steps): + latents, scheduler_state = loop_body(i, (latents, scheduler_state)) + else: + latents, _ = jax.lax.fori_loop(0, num_inference_steps, loop_body, (latents, scheduler_state)) + + # scale and decode the image latents with vae + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.apply({"params": params["vae"]}, latents, method=self.vae.decode).sample + + image = (image / 2 + 0.5).clip(0, 1).transpose(0, 2, 3, 1) + return image + + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt_ids: jnp.ndarray, + image: jnp.ndarray, + params: Union[Dict, FrozenDict], + prng_seed: jax.Array, + num_inference_steps: int = 50, + guidance_scale: Union[float, jnp.ndarray] = 7.5, + latents: jnp.ndarray = None, + neg_prompt_ids: jnp.ndarray = None, + controlnet_conditioning_scale: Union[float, jnp.ndarray] = 1.0, + return_dict: bool = True, + jit: bool = False, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt_ids (`jnp.ndarray`): + The prompt or prompts to guide the image generation. + image (`jnp.ndarray`): + Array representing the ControlNet input condition to provide guidance to the `unet` for generation. + params (`Dict` or `FrozenDict`): + Dictionary containing the model parameters/weights. + prng_seed (`jax.Array`): + Array containing random number generator key. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + latents (`jnp.ndarray`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + array is generated by sampling using the supplied random `generator`. + controlnet_conditioning_scale (`float` or `jnp.ndarray`, *optional*, defaults to 1.0): + The outputs of the ControlNet are multiplied by `controlnet_conditioning_scale` before they are added + to the residual in the original `unet`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] instead of + a plain tuple. + jit (`bool`, defaults to `False`): + Whether to run `pmap` versions of the generation and safety scoring functions. + + + + This argument exists because `__call__` is not yet end-to-end pmap-able. It will be removed in a + future release. + + + + Examples: + + Returns: + [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] is + returned, otherwise a `tuple` is returned where the first element is a list with the generated images + and the second element is a list of `bool`s indicating whether the corresponding generated image + contains "not-safe-for-work" (nsfw) content. + """ + + height, width = image.shape[-2:] + + if isinstance(guidance_scale, float): + # Convert to a tensor so each device gets a copy. Follow the prompt_ids for + # shape information, as they may be sharded (when `jit` is `True`), or not. + guidance_scale = jnp.array([guidance_scale] * prompt_ids.shape[0]) + if len(prompt_ids.shape) > 2: + # Assume sharded + guidance_scale = guidance_scale[:, None] + + if isinstance(controlnet_conditioning_scale, float): + # Convert to a tensor so each device gets a copy. Follow the prompt_ids for + # shape information, as they may be sharded (when `jit` is `True`), or not. + controlnet_conditioning_scale = jnp.array([controlnet_conditioning_scale] * prompt_ids.shape[0]) + if len(prompt_ids.shape) > 2: + # Assume sharded + controlnet_conditioning_scale = controlnet_conditioning_scale[:, None] + + if jit: + images = _p_generate( + self, + prompt_ids, + image, + params, + prng_seed, + num_inference_steps, + guidance_scale, + latents, + neg_prompt_ids, + controlnet_conditioning_scale, + ) + else: + images = self._generate( + prompt_ids, + image, + params, + prng_seed, + num_inference_steps, + guidance_scale, + latents, + neg_prompt_ids, + controlnet_conditioning_scale, + ) + + if self.safety_checker is not None: + safety_params = params["safety_checker"] + images_uint8_casted = (images * 255).round().astype("uint8") + num_devices, batch_size = images.shape[:2] + + images_uint8_casted = np.asarray(images_uint8_casted).reshape(num_devices * batch_size, height, width, 3) + images_uint8_casted, has_nsfw_concept = self._run_safety_checker(images_uint8_casted, safety_params, jit) + images = np.array(images) + + # block images + if any(has_nsfw_concept): + for i, is_nsfw in enumerate(has_nsfw_concept): + if is_nsfw: + images[i] = np.asarray(images_uint8_casted[i]) + + images = images.reshape(num_devices, batch_size, height, width, 3) + else: + images = np.asarray(images) + has_nsfw_concept = False + + if not return_dict: + return (images, has_nsfw_concept) + + return FlaxStableDiffusionPipelineOutput(images=images, nsfw_content_detected=has_nsfw_concept) + + +# Static argnums are pipe, num_inference_steps. A change would trigger recompilation. +# Non-static args are (sharded) input tensors mapped over their first dimension (hence, `0`). +@partial( + jax.pmap, + in_axes=(None, 0, 0, 0, 0, None, 0, 0, 0, 0), + static_broadcasted_argnums=(0, 5), +) +def _p_generate( + pipe, + prompt_ids, + image, + params, + prng_seed, + num_inference_steps, + guidance_scale, + latents, + neg_prompt_ids, + controlnet_conditioning_scale, +): + return pipe._generate( + prompt_ids, + image, + params, + prng_seed, + num_inference_steps, + guidance_scale, + latents, + neg_prompt_ids, + controlnet_conditioning_scale, + ) + + +@partial(jax.pmap, static_broadcasted_argnums=(0,)) +def _p_get_has_nsfw_concepts(pipe, features, params): + return pipe._get_has_nsfw_concepts(features, params) + + +def unshard(x: jnp.ndarray): + # einops.rearrange(x, 'd b ... -> (d b) ...') + num_devices, batch_size = x.shape[:2] + rest = x.shape[2:] + return x.reshape(num_devices * batch_size, *rest) + + +def preprocess(image, dtype): + image = image.convert("RGB") + w, h = image.size + w, h = (x - x % 64 for x in (w, h)) # resize to integer multiple of 64 + image = image.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]) + image = jnp.array(image).astype(dtype) / 255.0 + image = image[None].transpose(0, 3, 1, 2) + return image diff --git a/diffusers-0.27.0/src/diffusers/pipelines/dance_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/dance_diffusion/__init__.py new file mode 100755 index 0000000..0d3e466 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/dance_diffusion/__init__.py @@ -0,0 +1,18 @@ +from typing import TYPE_CHECKING + +from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule + + +_import_structure = {"pipeline_dance_diffusion": ["DanceDiffusionPipeline"]} + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .pipeline_dance_diffusion import DanceDiffusionPipeline +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py new file mode 100755 index 0000000..bcd36c4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/dance_diffusion/pipeline_dance_diffusion.py @@ -0,0 +1,156 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import List, Optional, Tuple, Union + +import torch + +from ...utils import logging +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import AudioPipelineOutput, DiffusionPipeline + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class DanceDiffusionPipeline(DiffusionPipeline): + r""" + Pipeline for audio generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + unet ([`UNet1DModel`]): + A `UNet1DModel` to denoise the encoded audio. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded audio latents. Can be one of + [`IPNDMScheduler`]. + """ + + model_cpu_offload_seq = "unet" + + def __init__(self, unet, scheduler): + super().__init__() + self.register_modules(unet=unet, scheduler=scheduler) + + @torch.no_grad() + def __call__( + self, + batch_size: int = 1, + num_inference_steps: int = 100, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + audio_length_in_s: Optional[float] = None, + return_dict: bool = True, + ) -> Union[AudioPipelineOutput, Tuple]: + r""" + The call function to the pipeline for generation. + + Args: + batch_size (`int`, *optional*, defaults to 1): + The number of audio samples to generate. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher-quality audio sample at + the expense of slower inference. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + audio_length_in_s (`float`, *optional*, defaults to `self.unet.config.sample_size/self.unet.config.sample_rate`): + The length of the generated audio sample in seconds. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.AudioPipelineOutput`] instead of a plain tuple. + + Example: + + ```py + from diffusers import DiffusionPipeline + from scipy.io.wavfile import write + + model_id = "harmonai/maestro-150k" + pipe = DiffusionPipeline.from_pretrained(model_id) + pipe = pipe.to("cuda") + + audios = pipe(audio_length_in_s=4.0).audios + + # To save locally + for i, audio in enumerate(audios): + write(f"maestro_test_{i}.wav", pipe.unet.sample_rate, audio.transpose()) + + # To dislay in google colab + import IPython.display as ipd + + for audio in audios: + display(ipd.Audio(audio, rate=pipe.unet.sample_rate)) + ``` + + Returns: + [`~pipelines.AudioPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.AudioPipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated audio. + """ + + if audio_length_in_s is None: + audio_length_in_s = self.unet.config.sample_size / self.unet.config.sample_rate + + sample_size = audio_length_in_s * self.unet.config.sample_rate + + down_scale_factor = 2 ** len(self.unet.up_blocks) + if sample_size < 3 * down_scale_factor: + raise ValueError( + f"{audio_length_in_s} is too small. Make sure it's bigger or equal to" + f" {3 * down_scale_factor / self.unet.config.sample_rate}." + ) + + original_sample_size = int(sample_size) + if sample_size % down_scale_factor != 0: + sample_size = ( + (audio_length_in_s * self.unet.config.sample_rate) // down_scale_factor + 1 + ) * down_scale_factor + logger.info( + f"{audio_length_in_s} is increased to {sample_size / self.unet.config.sample_rate} so that it can be handled" + f" by the model. It will be cut to {original_sample_size / self.unet.config.sample_rate} after the denoising" + " process." + ) + sample_size = int(sample_size) + + dtype = next(self.unet.parameters()).dtype + shape = (batch_size, self.unet.config.in_channels, sample_size) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + audio = randn_tensor(shape, generator=generator, device=self._execution_device, dtype=dtype) + + # set step values + self.scheduler.set_timesteps(num_inference_steps, device=audio.device) + self.scheduler.timesteps = self.scheduler.timesteps.to(dtype) + + for t in self.progress_bar(self.scheduler.timesteps): + # 1. predict noise model_output + model_output = self.unet(audio, t).sample + + # 2. compute previous audio sample: x_t -> t_t-1 + audio = self.scheduler.step(model_output, t, audio).prev_sample + + audio = audio.clamp(-1, 1).float().cpu().numpy() + + audio = audio[:, :, :original_sample_size] + + if not return_dict: + return (audio,) + + return AudioPipelineOutput(audios=audio) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/ddim/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/ddim/__init__.py new file mode 100755 index 0000000..d9eede4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/ddim/__init__.py @@ -0,0 +1,18 @@ +from typing import TYPE_CHECKING + +from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule + + +_import_structure = {"pipeline_ddim": ["DDIMPipeline"]} + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .pipeline_ddim import DDIMPipeline +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/ddim/pipeline_ddim.py b/diffusers-0.27.0/src/diffusers/pipelines/ddim/pipeline_ddim.py new file mode 100755 index 0000000..a3b967e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/ddim/pipeline_ddim.py @@ -0,0 +1,154 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Tuple, Union + +import torch + +from ...schedulers import DDIMScheduler +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +class DDIMPipeline(DiffusionPipeline): + r""" + Pipeline for image generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + unet ([`UNet2DModel`]): + A `UNet2DModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image. Can be one of + [`DDPMScheduler`], or [`DDIMScheduler`]. + """ + + model_cpu_offload_seq = "unet" + + def __init__(self, unet, scheduler): + super().__init__() + + # make sure scheduler can always be converted to DDIM + scheduler = DDIMScheduler.from_config(scheduler.config) + + self.register_modules(unet=unet, scheduler=scheduler) + + @torch.no_grad() + def __call__( + self, + batch_size: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + eta: float = 0.0, + num_inference_steps: int = 50, + use_clipped_model_output: Optional[bool] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + r""" + The call function to the pipeline for generation. + + Args: + batch_size (`int`, *optional*, defaults to 1): + The number of images to generate. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. A value of `0` corresponds to + DDIM and `1` corresponds to DDPM. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + use_clipped_model_output (`bool`, *optional*, defaults to `None`): + If `True` or `False`, see documentation for [`DDIMScheduler.step`]. If `None`, nothing is passed + downstream to the scheduler (use `None` for schedulers which don't support this argument). + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Example: + + ```py + >>> from diffusers import DDIMPipeline + >>> import PIL.Image + >>> import numpy as np + + >>> # load model and scheduler + >>> pipe = DDIMPipeline.from_pretrained("fusing/ddim-lsun-bedroom") + + >>> # run pipeline in inference (sample random noise and denoise) + >>> image = pipe(eta=0.0, num_inference_steps=50) + + >>> # process image to PIL + >>> image_processed = image.cpu().permute(0, 2, 3, 1) + >>> image_processed = (image_processed + 1.0) * 127.5 + >>> image_processed = image_processed.numpy().astype(np.uint8) + >>> image_pil = PIL.Image.fromarray(image_processed[0]) + + >>> # save image + >>> image_pil.save("test.png") + ``` + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images + """ + + # Sample gaussian noise to begin loop + if isinstance(self.unet.config.sample_size, int): + image_shape = ( + batch_size, + self.unet.config.in_channels, + self.unet.config.sample_size, + self.unet.config.sample_size, + ) + else: + image_shape = (batch_size, self.unet.config.in_channels, *self.unet.config.sample_size) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + image = randn_tensor(image_shape, generator=generator, device=self._execution_device, dtype=self.unet.dtype) + + # set step values + self.scheduler.set_timesteps(num_inference_steps) + + for t in self.progress_bar(self.scheduler.timesteps): + # 1. predict noise model_output + model_output = self.unet(image, t).sample + + # 2. predict previous mean of image x_t-1 and add variance depending on eta + # eta corresponds to η in paper and should be between [0, 1] + # do x_t -> x_t-1 + image = self.scheduler.step( + model_output, t, image, eta=eta, use_clipped_model_output=use_clipped_model_output, generator=generator + ).prev_sample + + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/ddpm/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/ddpm/__init__.py new file mode 100755 index 0000000..eb41dd1 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/ddpm/__init__.py @@ -0,0 +1,22 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + _LazyModule, +) + + +_import_structure = {"pipeline_ddpm": ["DDPMPipeline"]} + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .pipeline_ddpm import DDPMPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/ddpm/pipeline_ddpm.py b/diffusers-0.27.0/src/diffusers/pipelines/ddpm/pipeline_ddpm.py new file mode 100755 index 0000000..093a3cd --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/ddpm/pipeline_ddpm.py @@ -0,0 +1,127 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import List, Optional, Tuple, Union + +import torch + +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +class DDPMPipeline(DiffusionPipeline): + r""" + Pipeline for image generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + unet ([`UNet2DModel`]): + A `UNet2DModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image. Can be one of + [`DDPMScheduler`], or [`DDIMScheduler`]. + """ + + model_cpu_offload_seq = "unet" + + def __init__(self, unet, scheduler): + super().__init__() + self.register_modules(unet=unet, scheduler=scheduler) + + @torch.no_grad() + def __call__( + self, + batch_size: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + num_inference_steps: int = 1000, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + r""" + The call function to the pipeline for generation. + + Args: + batch_size (`int`, *optional*, defaults to 1): + The number of images to generate. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + num_inference_steps (`int`, *optional*, defaults to 1000): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Example: + + ```py + >>> from diffusers import DDPMPipeline + + >>> # load model and scheduler + >>> pipe = DDPMPipeline.from_pretrained("google/ddpm-cat-256") + + >>> # run pipeline in inference (sample random noise and denoise) + >>> image = pipe().images[0] + + >>> # save image + >>> image.save("ddpm_generated_image.png") + ``` + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images + """ + # Sample gaussian noise to begin loop + if isinstance(self.unet.config.sample_size, int): + image_shape = ( + batch_size, + self.unet.config.in_channels, + self.unet.config.sample_size, + self.unet.config.sample_size, + ) + else: + image_shape = (batch_size, self.unet.config.in_channels, *self.unet.config.sample_size) + + if self.device.type == "mps": + # randn does not work reproducibly on mps + image = randn_tensor(image_shape, generator=generator) + image = image.to(self.device) + else: + image = randn_tensor(image_shape, generator=generator, device=self.device) + + # set step values + self.scheduler.set_timesteps(num_inference_steps) + + for t in self.progress_bar(self.scheduler.timesteps): + # 1. predict noise model_output + model_output = self.unet(image, t).sample + + # 2. compute previous image: x_t -> x_t-1 + image = self.scheduler.step(model_output, t, image, generator=generator).prev_sample + + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/__init__.py new file mode 100755 index 0000000..79aab1f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/__init__.py @@ -0,0 +1,85 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = { + "timesteps": [ + "fast27_timesteps", + "smart100_timesteps", + "smart185_timesteps", + "smart27_timesteps", + "smart50_timesteps", + "super100_timesteps", + "super27_timesteps", + "super40_timesteps", + ] +} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_if"] = ["IFPipeline"] + _import_structure["pipeline_if_img2img"] = ["IFImg2ImgPipeline"] + _import_structure["pipeline_if_img2img_superresolution"] = ["IFImg2ImgSuperResolutionPipeline"] + _import_structure["pipeline_if_inpainting"] = ["IFInpaintingPipeline"] + _import_structure["pipeline_if_inpainting_superresolution"] = ["IFInpaintingSuperResolutionPipeline"] + _import_structure["pipeline_if_superresolution"] = ["IFSuperResolutionPipeline"] + _import_structure["pipeline_output"] = ["IFPipelineOutput"] + _import_structure["safety_checker"] = ["IFSafetyChecker"] + _import_structure["watermark"] = ["IFWatermarker"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_if import IFPipeline + from .pipeline_if_img2img import IFImg2ImgPipeline + from .pipeline_if_img2img_superresolution import IFImg2ImgSuperResolutionPipeline + from .pipeline_if_inpainting import IFInpaintingPipeline + from .pipeline_if_inpainting_superresolution import IFInpaintingSuperResolutionPipeline + from .pipeline_if_superresolution import IFSuperResolutionPipeline + from .pipeline_output import IFPipelineOutput + from .safety_checker import IFSafetyChecker + from .timesteps import ( + fast27_timesteps, + smart27_timesteps, + smart50_timesteps, + smart100_timesteps, + smart185_timesteps, + super27_timesteps, + super40_timesteps, + super100_timesteps, + ) + from .watermark import IFWatermarker + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if.py b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if.py new file mode 100755 index 0000000..7adf9e9 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if.py @@ -0,0 +1,788 @@ +import html +import inspect +import re +import urllib.parse as ul +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from transformers import CLIPImageProcessor, T5EncoderModel, T5Tokenizer + +from ...loaders import LoraLoaderMixin +from ...models import UNet2DConditionModel +from ...schedulers import DDPMScheduler +from ...utils import ( + BACKENDS_MAPPING, + is_accelerate_available, + is_bs4_available, + is_ftfy_available, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .pipeline_output import IFPipelineOutput +from .safety_checker import IFSafetyChecker +from .watermark import IFWatermarker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +if is_bs4_available(): + from bs4 import BeautifulSoup + +if is_ftfy_available(): + import ftfy + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import IFPipeline, IFSuperResolutionPipeline, DiffusionPipeline + >>> from diffusers.utils import pt_to_pil + >>> import torch + + >>> pipe = IFPipeline.from_pretrained("DeepFloyd/IF-I-XL-v1.0", variant="fp16", torch_dtype=torch.float16) + >>> pipe.enable_model_cpu_offload() + + >>> prompt = 'a photo of a kangaroo wearing an orange hoodie and blue sunglasses standing in front of the eiffel tower holding a sign that says "very deep learning"' + >>> prompt_embeds, negative_embeds = pipe.encode_prompt(prompt) + + >>> image = pipe(prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, output_type="pt").images + + >>> # save intermediate image + >>> pil_image = pt_to_pil(image) + >>> pil_image[0].save("./if_stage_I.png") + + >>> super_res_1_pipe = IFSuperResolutionPipeline.from_pretrained( + ... "DeepFloyd/IF-II-L-v1.0", text_encoder=None, variant="fp16", torch_dtype=torch.float16 + ... ) + >>> super_res_1_pipe.enable_model_cpu_offload() + + >>> image = super_res_1_pipe( + ... image=image, prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, output_type="pt" + ... ).images + + >>> # save intermediate image + >>> pil_image = pt_to_pil(image) + >>> pil_image[0].save("./if_stage_I.png") + + >>> safety_modules = { + ... "feature_extractor": pipe.feature_extractor, + ... "safety_checker": pipe.safety_checker, + ... "watermarker": pipe.watermarker, + ... } + >>> super_res_2_pipe = DiffusionPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-x4-upscaler", **safety_modules, torch_dtype=torch.float16 + ... ) + >>> super_res_2_pipe.enable_model_cpu_offload() + + >>> image = super_res_2_pipe( + ... prompt=prompt, + ... image=image, + ... ).images + >>> image[0].save("./if_stage_II.png") + ``` +""" + + +class IFPipeline(DiffusionPipeline, LoraLoaderMixin): + tokenizer: T5Tokenizer + text_encoder: T5EncoderModel + + unet: UNet2DConditionModel + scheduler: DDPMScheduler + + feature_extractor: Optional[CLIPImageProcessor] + safety_checker: Optional[IFSafetyChecker] + + watermarker: Optional[IFWatermarker] + + bad_punct_regex = re.compile( + r"[" + + "#®•©™&@·º½¾¿¡§~" + + r"\)" + + r"\(" + + r"\]" + + r"\[" + + r"\}" + + r"\{" + + r"\|" + + "\\" + + r"\/" + + r"\*" + + r"]{1,}" + ) # noqa + + _optional_components = ["tokenizer", "text_encoder", "safety_checker", "feature_extractor", "watermarker"] + model_cpu_offload_seq = "text_encoder->unet" + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + safety_checker: Optional[IFSafetyChecker], + feature_extractor: Optional[CLIPImageProcessor], + watermarker: Optional[IFWatermarker], + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the IF license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + watermarker=watermarker, + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + def remove_all_hooks(self): + if is_accelerate_available(): + from accelerate.hooks import remove_hook_from_module + else: + raise ImportError("Please install accelerate via `pip install accelerate`") + + for model in [self.text_encoder, self.unet, self.safety_checker]: + if model is not None: + remove_hook_from_module(model, recurse=True) + + self.unet_offload_hook = None + self.text_encoder_offload_hook = None + self.final_offload_hook = None + + @torch.no_grad() + def encode_prompt( + self, + prompt: Union[str, List[str]], + do_classifier_free_guidance: bool = True, + num_images_per_prompt: int = 1, + device: Optional[torch.device] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + clean_caption: bool = False, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + whether to use classifier free guidance or not + num_images_per_prompt (`int`, *optional*, defaults to 1): + number of images that should be generated per prompt + device: (`torch.device`, *optional*): + torch device to place the resulting embeddings on + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. + Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + clean_caption (bool, defaults to `False`): + If `True`, the function will preprocess and clean the provided caption before encoding. + """ + if prompt is not None and negative_prompt is not None: + if type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + + if device is None: + device = self._execution_device + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # while T5 can handle much longer input sequences than 77, the text encoder was trained with a max length of 77 for IF + max_length = 77 + + if prompt_embeds is None: + prompt = self._text_preprocessing(prompt, clean_caption=clean_caption) + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {max_length} tokens: {removed_text}" + ) + + attention_mask = text_inputs.attention_mask.to(device) + + prompt_embeds = self.text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + + if self.text_encoder is not None: + dtype = self.text_encoder.dtype + elif self.unet is not None: + dtype = self.unet.dtype + else: + dtype = None + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_tokens = self._text_preprocessing(uncond_tokens, clean_caption=clean_caption) + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + attention_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + else: + negative_prompt_embeds = None + + return prompt_embeds, negative_prompt_embeds + + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is not None: + safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device) + image, nsfw_detected, watermark_detected = self.safety_checker( + images=image, + clip_input=safety_checker_input.pixel_values.to(dtype=dtype), + ) + else: + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + + return image, nsfw_detected, watermark_detected + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def prepare_intermediate_images(self, batch_size, num_channels, height, width, dtype, device, generator): + shape = (batch_size, num_channels, height, width) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + intermediate_images = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # scale the initial noise by the standard deviation required by the scheduler + intermediate_images = intermediate_images * self.scheduler.init_noise_sigma + return intermediate_images + + def _text_preprocessing(self, text, clean_caption=False): + if clean_caption and not is_bs4_available(): + logger.warning(BACKENDS_MAPPING["bs4"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if clean_caption and not is_ftfy_available(): + logger.warning(BACKENDS_MAPPING["ftfy"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if not isinstance(text, (tuple, list)): + text = [text] + + def process(text: str): + if clean_caption: + text = self._clean_caption(text) + text = self._clean_caption(text) + else: + text = text.lower().strip() + return text + + return [process(t) for t in text] + + def _clean_caption(self, caption): + caption = str(caption) + caption = ul.unquote_plus(caption) + caption = caption.strip().lower() + caption = re.sub("", "person", caption) + # urls: + caption = re.sub( + r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + caption = re.sub( + r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + # html: + caption = BeautifulSoup(caption, features="html.parser").text + + # @ + caption = re.sub(r"@[\w\d]+\b", "", caption) + + # 31C0—31EF CJK Strokes + # 31F0—31FF Katakana Phonetic Extensions + # 3200—32FF Enclosed CJK Letters and Months + # 3300—33FF CJK Compatibility + # 3400—4DBF CJK Unified Ideographs Extension A + # 4DC0—4DFF Yijing Hexagram Symbols + # 4E00—9FFF CJK Unified Ideographs + caption = re.sub(r"[\u31c0-\u31ef]+", "", caption) + caption = re.sub(r"[\u31f0-\u31ff]+", "", caption) + caption = re.sub(r"[\u3200-\u32ff]+", "", caption) + caption = re.sub(r"[\u3300-\u33ff]+", "", caption) + caption = re.sub(r"[\u3400-\u4dbf]+", "", caption) + caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption) + caption = re.sub(r"[\u4e00-\u9fff]+", "", caption) + ####################################################### + + # все виды тире / all types of dash --> "-" + caption = re.sub( + r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa + "-", + caption, + ) + + # кавычки к одному стандарту + caption = re.sub(r"[`´«»“”¨]", '"', caption) + caption = re.sub(r"[‘’]", "'", caption) + + # " + caption = re.sub(r""?", "", caption) + # & + caption = re.sub(r"&", "", caption) + + # ip adresses: + caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption) + + # article ids: + caption = re.sub(r"\d:\d\d\s+$", "", caption) + + # \n + caption = re.sub(r"\\n", " ", caption) + + # "#123" + caption = re.sub(r"#\d{1,3}\b", "", caption) + # "#12345.." + caption = re.sub(r"#\d{5,}\b", "", caption) + # "123456.." + caption = re.sub(r"\b\d{6,}\b", "", caption) + # filenames: + caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption) + + # + caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT""" + caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT""" + + caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT + caption = re.sub(r"\s+\.\s+", r" ", caption) # " . " + + # this-is-my-cute-cat / this_is_my_cute_cat + regex2 = re.compile(r"(?:\-|\_)") + if len(re.findall(regex2, caption)) > 3: + caption = re.sub(regex2, " ", caption) + + caption = ftfy.fix_text(caption) + caption = html.unescape(html.unescape(caption)) + + caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640 + caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc + caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231 + + caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption) + caption = re.sub(r"(free\s)?download(\sfree)?", "", caption) + caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption) + caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption) + caption = re.sub(r"\bpage\s+\d+\b", "", caption) + + caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a... + + caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption) + + caption = re.sub(r"\b\s+\:\s+", r": ", caption) + caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption) + caption = re.sub(r"\s+", " ", caption) + + caption.strip() + + caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption) + caption = re.sub(r"^[\'\_,\-\:;]", r"", caption) + caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption) + caption = re.sub(r"^\.\S+$", "", caption) + + return caption.strip() + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + num_inference_steps: int = 100, + timesteps: List[int] = None, + guidance_scale: float = 7.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + height: Optional[int] = None, + width: Optional[int] = None, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + clean_caption: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps are used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + height (`int`, *optional*, defaults to self.unet.config.sample_size): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size): + The width in pixels of the generated image. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + clean_caption (`bool`, *optional*, defaults to `True`): + Whether or not to clean the caption before creating embeddings. Requires `beautifulsoup4` and `ftfy` to + be installed. If the dependencies are not installed, the embeddings will be created from the raw + prompt. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + Examples: + + Returns: + [`~pipelines.stable_diffusion.IFPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.IFPipelineOutput`] if `return_dict` is True, otherwise a `tuple. When + returning a tuple, the first element is a list with the generated images, and the second element is a list + of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" (nsfw) + or watermarked content, according to the `safety_checker`. + """ + # 1. Check inputs. Raise error if not correct + self.check_inputs(prompt, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds) + + # 2. Define call parameters + height = height or self.unet.config.sample_size + width = width or self.unet.config.sample_size + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + do_classifier_free_guidance, + num_images_per_prompt=num_images_per_prompt, + device=device, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clean_caption=clean_caption, + ) + + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + if timesteps is not None: + self.scheduler.set_timesteps(timesteps=timesteps, device=device) + timesteps = self.scheduler.timesteps + num_inference_steps = len(timesteps) + else: + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare intermediate images + intermediate_images = self.prepare_intermediate_images( + batch_size * num_images_per_prompt, + self.unet.config.in_channels, + height, + width, + prompt_embeds.dtype, + device, + generator, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # HACK: see comment in `enable_model_cpu_offload` + if hasattr(self, "text_encoder_offload_hook") and self.text_encoder_offload_hook is not None: + self.text_encoder_offload_hook.offload() + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + model_input = ( + torch.cat([intermediate_images] * 2) if do_classifier_free_guidance else intermediate_images + ) + model_input = self.scheduler.scale_model_input(model_input, t) + + # predict the noise residual + noise_pred = self.unet( + model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred_uncond, _ = noise_pred_uncond.split(model_input.shape[1], dim=1) + noise_pred_text, predicted_variance = noise_pred_text.split(model_input.shape[1], dim=1) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, predicted_variance], dim=1) + + if self.scheduler.config.variance_type not in ["learned", "learned_range"]: + noise_pred, _ = noise_pred.split(model_input.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + intermediate_images = self.scheduler.step( + noise_pred, t, intermediate_images, **extra_step_kwargs, return_dict=False + )[0] + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, intermediate_images) + + image = intermediate_images + + if output_type == "pil": + # 8. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 9. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # 10. Convert to PIL + image = self.numpy_to_pil(image) + + # 11. Apply watermark + if self.watermarker is not None: + image = self.watermarker.apply_watermark(image, self.unet.config.sample_size) + elif output_type == "pt": + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + else: + # 8. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 9. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, nsfw_detected, watermark_detected) + + return IFPipelineOutput(images=image, nsfw_detected=nsfw_detected, watermark_detected=watermark_detected) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py new file mode 100755 index 0000000..ccc7b1d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img.py @@ -0,0 +1,910 @@ +import html +import inspect +import re +import urllib.parse as ul +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, T5EncoderModel, T5Tokenizer + +from ...loaders import LoraLoaderMixin +from ...models import UNet2DConditionModel +from ...schedulers import DDPMScheduler +from ...utils import ( + BACKENDS_MAPPING, + PIL_INTERPOLATION, + is_accelerate_available, + is_bs4_available, + is_ftfy_available, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .pipeline_output import IFPipelineOutput +from .safety_checker import IFSafetyChecker +from .watermark import IFWatermarker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +if is_bs4_available(): + from bs4 import BeautifulSoup + +if is_ftfy_available(): + import ftfy + + +def resize(images: PIL.Image.Image, img_size: int) -> PIL.Image.Image: + w, h = images.size + + coef = w / h + + w, h = img_size, img_size + + if coef >= 1: + w = int(round(img_size / 8 * coef) * 8) + else: + h = int(round(img_size / 8 / coef) * 8) + + images = images.resize((w, h), resample=PIL_INTERPOLATION["bicubic"], reducing_gap=None) + + return images + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import IFImg2ImgPipeline, IFImg2ImgSuperResolutionPipeline, DiffusionPipeline + >>> from diffusers.utils import pt_to_pil + >>> import torch + >>> from PIL import Image + >>> import requests + >>> from io import BytesIO + + >>> url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + >>> response = requests.get(url) + >>> original_image = Image.open(BytesIO(response.content)).convert("RGB") + >>> original_image = original_image.resize((768, 512)) + + >>> pipe = IFImg2ImgPipeline.from_pretrained( + ... "DeepFloyd/IF-I-XL-v1.0", + ... variant="fp16", + ... torch_dtype=torch.float16, + ... ) + >>> pipe.enable_model_cpu_offload() + + >>> prompt = "A fantasy landscape in style minecraft" + >>> prompt_embeds, negative_embeds = pipe.encode_prompt(prompt) + + >>> image = pipe( + ... image=original_image, + ... prompt_embeds=prompt_embeds, + ... negative_prompt_embeds=negative_embeds, + ... output_type="pt", + ... ).images + + >>> # save intermediate image + >>> pil_image = pt_to_pil(image) + >>> pil_image[0].save("./if_stage_I.png") + + >>> super_res_1_pipe = IFImg2ImgSuperResolutionPipeline.from_pretrained( + ... "DeepFloyd/IF-II-L-v1.0", + ... text_encoder=None, + ... variant="fp16", + ... torch_dtype=torch.float16, + ... ) + >>> super_res_1_pipe.enable_model_cpu_offload() + + >>> image = super_res_1_pipe( + ... image=image, + ... original_image=original_image, + ... prompt_embeds=prompt_embeds, + ... negative_prompt_embeds=negative_embeds, + ... ).images + >>> image[0].save("./if_stage_II.png") + ``` +""" + + +class IFImg2ImgPipeline(DiffusionPipeline, LoraLoaderMixin): + tokenizer: T5Tokenizer + text_encoder: T5EncoderModel + + unet: UNet2DConditionModel + scheduler: DDPMScheduler + + feature_extractor: Optional[CLIPImageProcessor] + safety_checker: Optional[IFSafetyChecker] + + watermarker: Optional[IFWatermarker] + + bad_punct_regex = re.compile( + r"[" + + "#®•©™&@·º½¾¿¡§~" + + r"\)" + + r"\(" + + r"\]" + + r"\[" + + r"\}" + + r"\{" + + r"\|" + + "\\" + + r"\/" + + r"\*" + + r"]{1,}" + ) # noqa + + _optional_components = ["tokenizer", "text_encoder", "safety_checker", "feature_extractor", "watermarker"] + model_cpu_offload_seq = "text_encoder->unet" + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + safety_checker: Optional[IFSafetyChecker], + feature_extractor: Optional[CLIPImageProcessor], + watermarker: Optional[IFWatermarker], + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the IF license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + watermarker=watermarker, + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.remove_all_hooks + def remove_all_hooks(self): + if is_accelerate_available(): + from accelerate.hooks import remove_hook_from_module + else: + raise ImportError("Please install accelerate via `pip install accelerate`") + + for model in [self.text_encoder, self.unet, self.safety_checker]: + if model is not None: + remove_hook_from_module(model, recurse=True) + + self.unet_offload_hook = None + self.text_encoder_offload_hook = None + self.final_offload_hook = None + + @torch.no_grad() + def encode_prompt( + self, + prompt: Union[str, List[str]], + do_classifier_free_guidance: bool = True, + num_images_per_prompt: int = 1, + device: Optional[torch.device] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + clean_caption: bool = False, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + whether to use classifier free guidance or not + num_images_per_prompt (`int`, *optional*, defaults to 1): + number of images that should be generated per prompt + device: (`torch.device`, *optional*): + torch device to place the resulting embeddings on + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. + Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + clean_caption (bool, defaults to `False`): + If `True`, the function will preprocess and clean the provided caption before encoding. + """ + if prompt is not None and negative_prompt is not None: + if type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + + if device is None: + device = self._execution_device + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # while T5 can handle much longer input sequences than 77, the text encoder was trained with a max length of 77 for IF + max_length = 77 + + if prompt_embeds is None: + prompt = self._text_preprocessing(prompt, clean_caption=clean_caption) + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {max_length} tokens: {removed_text}" + ) + + attention_mask = text_inputs.attention_mask.to(device) + + prompt_embeds = self.text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + + if self.text_encoder is not None: + dtype = self.text_encoder.dtype + elif self.unet is not None: + dtype = self.unet.dtype + else: + dtype = None + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_tokens = self._text_preprocessing(uncond_tokens, clean_caption=clean_caption) + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + attention_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + else: + negative_prompt_embeds = None + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is not None: + safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device) + image, nsfw_detected, watermark_detected = self.safety_checker( + images=image, + clip_input=safety_checker_input.pixel_values.to(dtype=dtype), + ) + else: + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + + return image, nsfw_detected, watermark_detected + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + image, + batch_size, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if isinstance(image, list): + check_image_type = image[0] + else: + check_image_type = image + + if ( + not isinstance(check_image_type, torch.Tensor) + and not isinstance(check_image_type, PIL.Image.Image) + and not isinstance(check_image_type, np.ndarray) + ): + raise ValueError( + "`image` has to be of type `torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + f" {type(check_image_type)}" + ) + + if isinstance(image, list): + image_batch_size = len(image) + elif isinstance(image, torch.Tensor): + image_batch_size = image.shape[0] + elif isinstance(image, PIL.Image.Image): + image_batch_size = 1 + elif isinstance(image, np.ndarray): + image_batch_size = image.shape[0] + else: + assert False + + if batch_size != image_batch_size: + raise ValueError(f"image batch size: {image_batch_size} must be same as prompt batch size {batch_size}") + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._text_preprocessing + def _text_preprocessing(self, text, clean_caption=False): + if clean_caption and not is_bs4_available(): + logger.warning(BACKENDS_MAPPING["bs4"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if clean_caption and not is_ftfy_available(): + logger.warning(BACKENDS_MAPPING["ftfy"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if not isinstance(text, (tuple, list)): + text = [text] + + def process(text: str): + if clean_caption: + text = self._clean_caption(text) + text = self._clean_caption(text) + else: + text = text.lower().strip() + return text + + return [process(t) for t in text] + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._clean_caption + def _clean_caption(self, caption): + caption = str(caption) + caption = ul.unquote_plus(caption) + caption = caption.strip().lower() + caption = re.sub("", "person", caption) + # urls: + caption = re.sub( + r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + caption = re.sub( + r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + # html: + caption = BeautifulSoup(caption, features="html.parser").text + + # @ + caption = re.sub(r"@[\w\d]+\b", "", caption) + + # 31C0—31EF CJK Strokes + # 31F0—31FF Katakana Phonetic Extensions + # 3200—32FF Enclosed CJK Letters and Months + # 3300—33FF CJK Compatibility + # 3400—4DBF CJK Unified Ideographs Extension A + # 4DC0—4DFF Yijing Hexagram Symbols + # 4E00—9FFF CJK Unified Ideographs + caption = re.sub(r"[\u31c0-\u31ef]+", "", caption) + caption = re.sub(r"[\u31f0-\u31ff]+", "", caption) + caption = re.sub(r"[\u3200-\u32ff]+", "", caption) + caption = re.sub(r"[\u3300-\u33ff]+", "", caption) + caption = re.sub(r"[\u3400-\u4dbf]+", "", caption) + caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption) + caption = re.sub(r"[\u4e00-\u9fff]+", "", caption) + ####################################################### + + # все виды тире / all types of dash --> "-" + caption = re.sub( + r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa + "-", + caption, + ) + + # кавычки к одному стандарту + caption = re.sub(r"[`´«»“”¨]", '"', caption) + caption = re.sub(r"[‘’]", "'", caption) + + # " + caption = re.sub(r""?", "", caption) + # & + caption = re.sub(r"&", "", caption) + + # ip adresses: + caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption) + + # article ids: + caption = re.sub(r"\d:\d\d\s+$", "", caption) + + # \n + caption = re.sub(r"\\n", " ", caption) + + # "#123" + caption = re.sub(r"#\d{1,3}\b", "", caption) + # "#12345.." + caption = re.sub(r"#\d{5,}\b", "", caption) + # "123456.." + caption = re.sub(r"\b\d{6,}\b", "", caption) + # filenames: + caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption) + + # + caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT""" + caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT""" + + caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT + caption = re.sub(r"\s+\.\s+", r" ", caption) # " . " + + # this-is-my-cute-cat / this_is_my_cute_cat + regex2 = re.compile(r"(?:\-|\_)") + if len(re.findall(regex2, caption)) > 3: + caption = re.sub(regex2, " ", caption) + + caption = ftfy.fix_text(caption) + caption = html.unescape(html.unescape(caption)) + + caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640 + caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc + caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231 + + caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption) + caption = re.sub(r"(free\s)?download(\sfree)?", "", caption) + caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption) + caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption) + caption = re.sub(r"\bpage\s+\d+\b", "", caption) + + caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a... + + caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption) + + caption = re.sub(r"\b\s+\:\s+", r": ", caption) + caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption) + caption = re.sub(r"\s+", " ", caption) + + caption.strip() + + caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption) + caption = re.sub(r"^[\'\_,\-\:;]", r"", caption) + caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption) + caption = re.sub(r"^\.\S+$", "", caption) + + return caption.strip() + + def preprocess_image(self, image: PIL.Image.Image) -> torch.Tensor: + if not isinstance(image, list): + image = [image] + + def numpy_to_pt(images): + if images.ndim == 3: + images = images[..., None] + + images = torch.from_numpy(images.transpose(0, 3, 1, 2)) + return images + + if isinstance(image[0], PIL.Image.Image): + new_image = [] + + for image_ in image: + image_ = image_.convert("RGB") + image_ = resize(image_, self.unet.sample_size) + image_ = np.array(image_) + image_ = image_.astype(np.float32) + image_ = image_ / 127.5 - 1 + new_image.append(image_) + + image = new_image + + image = np.stack(image, axis=0) # to np + image = numpy_to_pt(image) # to pt + + elif isinstance(image[0], np.ndarray): + image = np.concatenate(image, axis=0) if image[0].ndim == 4 else np.stack(image, axis=0) + image = numpy_to_pt(image) + + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, axis=0) if image[0].ndim == 4 else torch.stack(image, axis=0) + + return image + + def get_timesteps(self, num_inference_steps, strength): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start:] + + return timesteps, num_inference_steps - t_start + + def prepare_intermediate_images( + self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None + ): + _, channels, height, width = image.shape + + batch_size = batch_size * num_images_per_prompt + + shape = (batch_size, channels, height, width) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + image = image.repeat_interleave(num_images_per_prompt, dim=0) + image = self.scheduler.add_noise(image, noise, timestep) + + return image + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: Union[ + PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] + ] = None, + strength: float = 0.7, + num_inference_steps: int = 80, + timesteps: List[int] = None, + guidance_scale: float = 10.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + clean_caption: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + image (`torch.FloatTensor` or `PIL.Image.Image`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. + strength (`float`, *optional*, defaults to 0.7): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 80): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps are used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 10.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + clean_caption (`bool`, *optional*, defaults to `True`): + Whether or not to clean the caption before creating embeddings. Requires `beautifulsoup4` and `ftfy` to + be installed. If the dependencies are not installed, the embeddings will be created from the raw + prompt. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + Examples: + + Returns: + [`~pipelines.stable_diffusion.IFPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.IFPipelineOutput`] if `return_dict` is True, otherwise a `tuple. When + returning a tuple, the first element is a list with the generated images, and the second element is a list + of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" (nsfw) + or watermarked content, according to the `safety_checker`. + """ + # 1. Check inputs. Raise error if not correct + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + self.check_inputs( + prompt, image, batch_size, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # 2. Define call parameters + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + do_classifier_free_guidance, + num_images_per_prompt=num_images_per_prompt, + device=device, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clean_caption=clean_caption, + ) + + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + dtype = prompt_embeds.dtype + + # 4. Prepare timesteps + if timesteps is not None: + self.scheduler.set_timesteps(timesteps=timesteps, device=device) + timesteps = self.scheduler.timesteps + num_inference_steps = len(timesteps) + else: + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength) + + # 5. Prepare intermediate images + image = self.preprocess_image(image) + image = image.to(device=device, dtype=dtype) + + noise_timestep = timesteps[0:1] + noise_timestep = noise_timestep.repeat(batch_size * num_images_per_prompt) + + intermediate_images = self.prepare_intermediate_images( + image, noise_timestep, batch_size, num_images_per_prompt, dtype, device, generator + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # HACK: see comment in `enable_model_cpu_offload` + if hasattr(self, "text_encoder_offload_hook") and self.text_encoder_offload_hook is not None: + self.text_encoder_offload_hook.offload() + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + model_input = ( + torch.cat([intermediate_images] * 2) if do_classifier_free_guidance else intermediate_images + ) + model_input = self.scheduler.scale_model_input(model_input, t) + + # predict the noise residual + noise_pred = self.unet( + model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred_uncond, _ = noise_pred_uncond.split(model_input.shape[1], dim=1) + noise_pred_text, predicted_variance = noise_pred_text.split(model_input.shape[1], dim=1) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, predicted_variance], dim=1) + + if self.scheduler.config.variance_type not in ["learned", "learned_range"]: + noise_pred, _ = noise_pred.split(model_input.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + intermediate_images = self.scheduler.step( + noise_pred, t, intermediate_images, **extra_step_kwargs, return_dict=False + )[0] + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, intermediate_images) + + image = intermediate_images + + if output_type == "pil": + # 8. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 9. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # 10. Convert to PIL + image = self.numpy_to_pil(image) + + # 11. Apply watermark + if self.watermarker is not None: + self.watermarker.apply_watermark(image, self.unet.config.sample_size) + elif output_type == "pt": + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + else: + # 8. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 9. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, nsfw_detected, watermark_detected) + + return IFPipelineOutput(images=image, nsfw_detected=nsfw_detected, watermark_detected=watermark_detected) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py new file mode 100755 index 0000000..b4ce583 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_img2img_superresolution.py @@ -0,0 +1,1029 @@ +import html +import inspect +import re +import urllib.parse as ul +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from transformers import CLIPImageProcessor, T5EncoderModel, T5Tokenizer + +from ...loaders import LoraLoaderMixin +from ...models import UNet2DConditionModel +from ...schedulers import DDPMScheduler +from ...utils import ( + BACKENDS_MAPPING, + PIL_INTERPOLATION, + is_accelerate_available, + is_bs4_available, + is_ftfy_available, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .pipeline_output import IFPipelineOutput +from .safety_checker import IFSafetyChecker +from .watermark import IFWatermarker + + +if is_bs4_available(): + from bs4 import BeautifulSoup + +if is_ftfy_available(): + import ftfy + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_img2img.resize +def resize(images: PIL.Image.Image, img_size: int) -> PIL.Image.Image: + w, h = images.size + + coef = w / h + + w, h = img_size, img_size + + if coef >= 1: + w = int(round(img_size / 8 * coef) * 8) + else: + h = int(round(img_size / 8 / coef) * 8) + + images = images.resize((w, h), resample=PIL_INTERPOLATION["bicubic"], reducing_gap=None) + + return images + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import IFImg2ImgPipeline, IFImg2ImgSuperResolutionPipeline, DiffusionPipeline + >>> from diffusers.utils import pt_to_pil + >>> import torch + >>> from PIL import Image + >>> import requests + >>> from io import BytesIO + + >>> url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + >>> response = requests.get(url) + >>> original_image = Image.open(BytesIO(response.content)).convert("RGB") + >>> original_image = original_image.resize((768, 512)) + + >>> pipe = IFImg2ImgPipeline.from_pretrained( + ... "DeepFloyd/IF-I-XL-v1.0", + ... variant="fp16", + ... torch_dtype=torch.float16, + ... ) + >>> pipe.enable_model_cpu_offload() + + >>> prompt = "A fantasy landscape in style minecraft" + >>> prompt_embeds, negative_embeds = pipe.encode_prompt(prompt) + + >>> image = pipe( + ... image=original_image, + ... prompt_embeds=prompt_embeds, + ... negative_prompt_embeds=negative_embeds, + ... output_type="pt", + ... ).images + + >>> # save intermediate image + >>> pil_image = pt_to_pil(image) + >>> pil_image[0].save("./if_stage_I.png") + + >>> super_res_1_pipe = IFImg2ImgSuperResolutionPipeline.from_pretrained( + ... "DeepFloyd/IF-II-L-v1.0", + ... text_encoder=None, + ... variant="fp16", + ... torch_dtype=torch.float16, + ... ) + >>> super_res_1_pipe.enable_model_cpu_offload() + + >>> image = super_res_1_pipe( + ... image=image, + ... original_image=original_image, + ... prompt_embeds=prompt_embeds, + ... negative_prompt_embeds=negative_embeds, + ... ).images + >>> image[0].save("./if_stage_II.png") + ``` +""" + + +class IFImg2ImgSuperResolutionPipeline(DiffusionPipeline, LoraLoaderMixin): + tokenizer: T5Tokenizer + text_encoder: T5EncoderModel + + unet: UNet2DConditionModel + scheduler: DDPMScheduler + image_noising_scheduler: DDPMScheduler + + feature_extractor: Optional[CLIPImageProcessor] + safety_checker: Optional[IFSafetyChecker] + + watermarker: Optional[IFWatermarker] + + bad_punct_regex = re.compile( + r"[" + + "#®•©™&@·º½¾¿¡§~" + + r"\)" + + r"\(" + + r"\]" + + r"\[" + + r"\}" + + r"\{" + + r"\|" + + "\\" + + r"\/" + + r"\*" + + r"]{1,}" + ) # noqa + + _optional_components = ["tokenizer", "text_encoder", "safety_checker", "feature_extractor"] + model_cpu_offload_seq = "text_encoder->unet" + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + image_noising_scheduler: DDPMScheduler, + safety_checker: Optional[IFSafetyChecker], + feature_extractor: Optional[CLIPImageProcessor], + watermarker: Optional[IFWatermarker], + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the IF license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + if unet.config.in_channels != 6: + logger.warning( + "It seems like you have loaded a checkpoint that shall not be used for super resolution from {unet.config._name_or_path} as it accepts {unet.config.in_channels} input channels instead of 6. Please make sure to pass a super resolution checkpoint as the `'unet'`: IFSuperResolutionPipeline.from_pretrained(unet=super_resolution_unet, ...)`." + ) + + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + unet=unet, + scheduler=scheduler, + image_noising_scheduler=image_noising_scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + watermarker=watermarker, + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.remove_all_hooks + def remove_all_hooks(self): + if is_accelerate_available(): + from accelerate.hooks import remove_hook_from_module + else: + raise ImportError("Please install accelerate via `pip install accelerate`") + + for model in [self.text_encoder, self.unet, self.safety_checker]: + if model is not None: + remove_hook_from_module(model, recurse=True) + + self.unet_offload_hook = None + self.text_encoder_offload_hook = None + self.final_offload_hook = None + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._text_preprocessing + def _text_preprocessing(self, text, clean_caption=False): + if clean_caption and not is_bs4_available(): + logger.warning(BACKENDS_MAPPING["bs4"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if clean_caption and not is_ftfy_available(): + logger.warning(BACKENDS_MAPPING["ftfy"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if not isinstance(text, (tuple, list)): + text = [text] + + def process(text: str): + if clean_caption: + text = self._clean_caption(text) + text = self._clean_caption(text) + else: + text = text.lower().strip() + return text + + return [process(t) for t in text] + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._clean_caption + def _clean_caption(self, caption): + caption = str(caption) + caption = ul.unquote_plus(caption) + caption = caption.strip().lower() + caption = re.sub("", "person", caption) + # urls: + caption = re.sub( + r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + caption = re.sub( + r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + # html: + caption = BeautifulSoup(caption, features="html.parser").text + + # @ + caption = re.sub(r"@[\w\d]+\b", "", caption) + + # 31C0—31EF CJK Strokes + # 31F0—31FF Katakana Phonetic Extensions + # 3200—32FF Enclosed CJK Letters and Months + # 3300—33FF CJK Compatibility + # 3400—4DBF CJK Unified Ideographs Extension A + # 4DC0—4DFF Yijing Hexagram Symbols + # 4E00—9FFF CJK Unified Ideographs + caption = re.sub(r"[\u31c0-\u31ef]+", "", caption) + caption = re.sub(r"[\u31f0-\u31ff]+", "", caption) + caption = re.sub(r"[\u3200-\u32ff]+", "", caption) + caption = re.sub(r"[\u3300-\u33ff]+", "", caption) + caption = re.sub(r"[\u3400-\u4dbf]+", "", caption) + caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption) + caption = re.sub(r"[\u4e00-\u9fff]+", "", caption) + ####################################################### + + # все виды тире / all types of dash --> "-" + caption = re.sub( + r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa + "-", + caption, + ) + + # кавычки к одному стандарту + caption = re.sub(r"[`´«»“”¨]", '"', caption) + caption = re.sub(r"[‘’]", "'", caption) + + # " + caption = re.sub(r""?", "", caption) + # & + caption = re.sub(r"&", "", caption) + + # ip adresses: + caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption) + + # article ids: + caption = re.sub(r"\d:\d\d\s+$", "", caption) + + # \n + caption = re.sub(r"\\n", " ", caption) + + # "#123" + caption = re.sub(r"#\d{1,3}\b", "", caption) + # "#12345.." + caption = re.sub(r"#\d{5,}\b", "", caption) + # "123456.." + caption = re.sub(r"\b\d{6,}\b", "", caption) + # filenames: + caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption) + + # + caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT""" + caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT""" + + caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT + caption = re.sub(r"\s+\.\s+", r" ", caption) # " . " + + # this-is-my-cute-cat / this_is_my_cute_cat + regex2 = re.compile(r"(?:\-|\_)") + if len(re.findall(regex2, caption)) > 3: + caption = re.sub(regex2, " ", caption) + + caption = ftfy.fix_text(caption) + caption = html.unescape(html.unescape(caption)) + + caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640 + caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc + caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231 + + caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption) + caption = re.sub(r"(free\s)?download(\sfree)?", "", caption) + caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption) + caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption) + caption = re.sub(r"\bpage\s+\d+\b", "", caption) + + caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a... + + caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption) + + caption = re.sub(r"\b\s+\:\s+", r": ", caption) + caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption) + caption = re.sub(r"\s+", " ", caption) + + caption.strip() + + caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption) + caption = re.sub(r"^[\'\_,\-\:;]", r"", caption) + caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption) + caption = re.sub(r"^\.\S+$", "", caption) + + return caption.strip() + + @torch.no_grad() + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.encode_prompt + def encode_prompt( + self, + prompt: Union[str, List[str]], + do_classifier_free_guidance: bool = True, + num_images_per_prompt: int = 1, + device: Optional[torch.device] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + clean_caption: bool = False, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + whether to use classifier free guidance or not + num_images_per_prompt (`int`, *optional*, defaults to 1): + number of images that should be generated per prompt + device: (`torch.device`, *optional*): + torch device to place the resulting embeddings on + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. + Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + clean_caption (bool, defaults to `False`): + If `True`, the function will preprocess and clean the provided caption before encoding. + """ + if prompt is not None and negative_prompt is not None: + if type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + + if device is None: + device = self._execution_device + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # while T5 can handle much longer input sequences than 77, the text encoder was trained with a max length of 77 for IF + max_length = 77 + + if prompt_embeds is None: + prompt = self._text_preprocessing(prompt, clean_caption=clean_caption) + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {max_length} tokens: {removed_text}" + ) + + attention_mask = text_inputs.attention_mask.to(device) + + prompt_embeds = self.text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + + if self.text_encoder is not None: + dtype = self.text_encoder.dtype + elif self.unet is not None: + dtype = self.unet.dtype + else: + dtype = None + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_tokens = self._text_preprocessing(uncond_tokens, clean_caption=clean_caption) + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + attention_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + else: + negative_prompt_embeds = None + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is not None: + safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device) + image, nsfw_detected, watermark_detected = self.safety_checker( + images=image, + clip_input=safety_checker_input.pixel_values.to(dtype=dtype), + ) + else: + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + + return image, nsfw_detected, watermark_detected + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + image, + original_image, + batch_size, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # image + + if isinstance(image, list): + check_image_type = image[0] + else: + check_image_type = image + + if ( + not isinstance(check_image_type, torch.Tensor) + and not isinstance(check_image_type, PIL.Image.Image) + and not isinstance(check_image_type, np.ndarray) + ): + raise ValueError( + "`image` has to be of type `torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + f" {type(check_image_type)}" + ) + + if isinstance(image, list): + image_batch_size = len(image) + elif isinstance(image, torch.Tensor): + image_batch_size = image.shape[0] + elif isinstance(image, PIL.Image.Image): + image_batch_size = 1 + elif isinstance(image, np.ndarray): + image_batch_size = image.shape[0] + else: + assert False + + if batch_size != image_batch_size: + raise ValueError(f"image batch size: {image_batch_size} must be same as prompt batch size {batch_size}") + + # original_image + + if isinstance(original_image, list): + check_image_type = original_image[0] + else: + check_image_type = original_image + + if ( + not isinstance(check_image_type, torch.Tensor) + and not isinstance(check_image_type, PIL.Image.Image) + and not isinstance(check_image_type, np.ndarray) + ): + raise ValueError( + "`original_image` has to be of type `torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + f" {type(check_image_type)}" + ) + + if isinstance(original_image, list): + image_batch_size = len(original_image) + elif isinstance(original_image, torch.Tensor): + image_batch_size = original_image.shape[0] + elif isinstance(original_image, PIL.Image.Image): + image_batch_size = 1 + elif isinstance(original_image, np.ndarray): + image_batch_size = original_image.shape[0] + else: + assert False + + if batch_size != image_batch_size: + raise ValueError( + f"original_image batch size: {image_batch_size} must be same as prompt batch size {batch_size}" + ) + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_img2img.IFImg2ImgPipeline.preprocess_image with preprocess_image -> preprocess_original_image + def preprocess_original_image(self, image: PIL.Image.Image) -> torch.Tensor: + if not isinstance(image, list): + image = [image] + + def numpy_to_pt(images): + if images.ndim == 3: + images = images[..., None] + + images = torch.from_numpy(images.transpose(0, 3, 1, 2)) + return images + + if isinstance(image[0], PIL.Image.Image): + new_image = [] + + for image_ in image: + image_ = image_.convert("RGB") + image_ = resize(image_, self.unet.sample_size) + image_ = np.array(image_) + image_ = image_.astype(np.float32) + image_ = image_ / 127.5 - 1 + new_image.append(image_) + + image = new_image + + image = np.stack(image, axis=0) # to np + image = numpy_to_pt(image) # to pt + + elif isinstance(image[0], np.ndarray): + image = np.concatenate(image, axis=0) if image[0].ndim == 4 else np.stack(image, axis=0) + image = numpy_to_pt(image) + + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, axis=0) if image[0].ndim == 4 else torch.stack(image, axis=0) + + return image + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_superresolution.IFSuperResolutionPipeline.preprocess_image + def preprocess_image(self, image: PIL.Image.Image, num_images_per_prompt, device) -> torch.Tensor: + if not isinstance(image, torch.Tensor) and not isinstance(image, list): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + image = [np.array(i).astype(np.float32) / 127.5 - 1.0 for i in image] + + image = np.stack(image, axis=0) # to np + image = torch.from_numpy(image.transpose(0, 3, 1, 2)) + elif isinstance(image[0], np.ndarray): + image = np.stack(image, axis=0) # to np + if image.ndim == 5: + image = image[0] + + image = torch.from_numpy(image.transpose(0, 3, 1, 2)) + elif isinstance(image, list) and isinstance(image[0], torch.Tensor): + dims = image[0].ndim + + if dims == 3: + image = torch.stack(image, dim=0) + elif dims == 4: + image = torch.concat(image, dim=0) + else: + raise ValueError(f"Image must have 3 or 4 dimensions, instead got {dims}") + + image = image.to(device=device, dtype=self.unet.dtype) + + image = image.repeat_interleave(num_images_per_prompt, dim=0) + + return image + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_img2img.IFImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start:] + + return timesteps, num_inference_steps - t_start + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_img2img.IFImg2ImgPipeline.prepare_intermediate_images + def prepare_intermediate_images( + self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None + ): + _, channels, height, width = image.shape + + batch_size = batch_size * num_images_per_prompt + + shape = (batch_size, channels, height, width) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + image = image.repeat_interleave(num_images_per_prompt, dim=0) + image = self.scheduler.add_noise(image, noise, timestep) + + return image + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image: Union[PIL.Image.Image, np.ndarray, torch.FloatTensor], + original_image: Union[ + PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] + ] = None, + strength: float = 0.8, + prompt: Union[str, List[str]] = None, + num_inference_steps: int = 50, + timesteps: List[int] = None, + guidance_scale: float = 4.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + noise_level: int = 250, + clean_caption: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + image (`torch.FloatTensor` or `PIL.Image.Image`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. + original_image (`torch.FloatTensor` or `PIL.Image.Image`): + The original image that `image` was varied from. + strength (`float`, *optional*, defaults to 0.8): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps are used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + noise_level (`int`, *optional*, defaults to 250): + The amount of noise to add to the upscaled image. Must be in the range `[0, 1000)` + clean_caption (`bool`, *optional*, defaults to `True`): + Whether or not to clean the caption before creating embeddings. Requires `beautifulsoup4` and `ftfy` to + be installed. If the dependencies are not installed, the embeddings will be created from the raw + prompt. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.IFPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.IFPipelineOutput`] if `return_dict` is True, otherwise a `tuple. When + returning a tuple, the first element is a list with the generated images, and the second element is a list + of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" (nsfw) + or watermarked content, according to the `safety_checker`. + """ + # 1. Check inputs. Raise error if not correct + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + self.check_inputs( + prompt, + image, + original_image, + batch_size, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + # 2. Define call parameters + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + device = self._execution_device + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + do_classifier_free_guidance, + num_images_per_prompt=num_images_per_prompt, + device=device, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clean_caption=clean_caption, + ) + + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + dtype = prompt_embeds.dtype + + # 4. Prepare timesteps + if timesteps is not None: + self.scheduler.set_timesteps(timesteps=timesteps, device=device) + timesteps = self.scheduler.timesteps + num_inference_steps = len(timesteps) + else: + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength) + + # 5. prepare original image + original_image = self.preprocess_original_image(original_image) + original_image = original_image.to(device=device, dtype=dtype) + + # 6. Prepare intermediate images + noise_timestep = timesteps[0:1] + noise_timestep = noise_timestep.repeat(batch_size * num_images_per_prompt) + + intermediate_images = self.prepare_intermediate_images( + original_image, + noise_timestep, + batch_size, + num_images_per_prompt, + dtype, + device, + generator, + ) + + # 7. Prepare upscaled image and noise level + _, _, height, width = original_image.shape + + image = self.preprocess_image(image, num_images_per_prompt, device) + + upscaled = F.interpolate(image, (height, width), mode="bilinear", align_corners=True) + + noise_level = torch.tensor([noise_level] * upscaled.shape[0], device=upscaled.device) + noise = randn_tensor(upscaled.shape, generator=generator, device=upscaled.device, dtype=upscaled.dtype) + upscaled = self.image_noising_scheduler.add_noise(upscaled, noise, timesteps=noise_level) + + if do_classifier_free_guidance: + noise_level = torch.cat([noise_level] * 2) + + # 8. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # HACK: see comment in `enable_model_cpu_offload` + if hasattr(self, "text_encoder_offload_hook") and self.text_encoder_offload_hook is not None: + self.text_encoder_offload_hook.offload() + + # 9. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + model_input = torch.cat([intermediate_images, upscaled], dim=1) + + model_input = torch.cat([model_input] * 2) if do_classifier_free_guidance else model_input + model_input = self.scheduler.scale_model_input(model_input, t) + + # predict the noise residual + noise_pred = self.unet( + model_input, + t, + encoder_hidden_states=prompt_embeds, + class_labels=noise_level, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred_uncond, _ = noise_pred_uncond.split(model_input.shape[1] // 2, dim=1) + noise_pred_text, predicted_variance = noise_pred_text.split(model_input.shape[1] // 2, dim=1) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, predicted_variance], dim=1) + + if self.scheduler.config.variance_type not in ["learned", "learned_range"]: + noise_pred, _ = noise_pred.split(intermediate_images.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + intermediate_images = self.scheduler.step( + noise_pred, t, intermediate_images, **extra_step_kwargs, return_dict=False + )[0] + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, intermediate_images) + + image = intermediate_images + + if output_type == "pil": + # 10. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 11. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # 12. Convert to PIL + image = self.numpy_to_pil(image) + + # 13. Apply watermark + if self.watermarker is not None: + self.watermarker.apply_watermark(image, self.unet.config.sample_size) + elif output_type == "pt": + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + else: + # 10. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 11. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, nsfw_detected, watermark_detected) + + return IFPipelineOutput(images=image, nsfw_detected=nsfw_detected, watermark_detected=watermark_detected) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py new file mode 100755 index 0000000..180e530 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting.py @@ -0,0 +1,1030 @@ +import html +import inspect +import re +import urllib.parse as ul +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, T5EncoderModel, T5Tokenizer + +from ...loaders import LoraLoaderMixin +from ...models import UNet2DConditionModel +from ...schedulers import DDPMScheduler +from ...utils import ( + BACKENDS_MAPPING, + PIL_INTERPOLATION, + is_accelerate_available, + is_bs4_available, + is_ftfy_available, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .pipeline_output import IFPipelineOutput +from .safety_checker import IFSafetyChecker +from .watermark import IFWatermarker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +if is_bs4_available(): + from bs4 import BeautifulSoup + +if is_ftfy_available(): + import ftfy + + +# Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_img2img.resize +def resize(images: PIL.Image.Image, img_size: int) -> PIL.Image.Image: + w, h = images.size + + coef = w / h + + w, h = img_size, img_size + + if coef >= 1: + w = int(round(img_size / 8 * coef) * 8) + else: + h = int(round(img_size / 8 / coef) * 8) + + images = images.resize((w, h), resample=PIL_INTERPOLATION["bicubic"], reducing_gap=None) + + return images + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import IFInpaintingPipeline, IFInpaintingSuperResolutionPipeline, DiffusionPipeline + >>> from diffusers.utils import pt_to_pil + >>> import torch + >>> from PIL import Image + >>> import requests + >>> from io import BytesIO + + >>> url = "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/if/person.png" + >>> response = requests.get(url) + >>> original_image = Image.open(BytesIO(response.content)).convert("RGB") + >>> original_image = original_image + + >>> url = "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/if/glasses_mask.png" + >>> response = requests.get(url) + >>> mask_image = Image.open(BytesIO(response.content)) + >>> mask_image = mask_image + + >>> pipe = IFInpaintingPipeline.from_pretrained( + ... "DeepFloyd/IF-I-XL-v1.0", variant="fp16", torch_dtype=torch.float16 + ... ) + >>> pipe.enable_model_cpu_offload() + + >>> prompt = "blue sunglasses" + >>> prompt_embeds, negative_embeds = pipe.encode_prompt(prompt) + + >>> image = pipe( + ... image=original_image, + ... mask_image=mask_image, + ... prompt_embeds=prompt_embeds, + ... negative_prompt_embeds=negative_embeds, + ... output_type="pt", + ... ).images + + >>> # save intermediate image + >>> pil_image = pt_to_pil(image) + >>> pil_image[0].save("./if_stage_I.png") + + >>> super_res_1_pipe = IFInpaintingSuperResolutionPipeline.from_pretrained( + ... "DeepFloyd/IF-II-L-v1.0", text_encoder=None, variant="fp16", torch_dtype=torch.float16 + ... ) + >>> super_res_1_pipe.enable_model_cpu_offload() + + >>> image = super_res_1_pipe( + ... image=image, + ... mask_image=mask_image, + ... original_image=original_image, + ... prompt_embeds=prompt_embeds, + ... negative_prompt_embeds=negative_embeds, + ... ).images + >>> image[0].save("./if_stage_II.png") + ``` +""" + + +class IFInpaintingPipeline(DiffusionPipeline, LoraLoaderMixin): + tokenizer: T5Tokenizer + text_encoder: T5EncoderModel + + unet: UNet2DConditionModel + scheduler: DDPMScheduler + + feature_extractor: Optional[CLIPImageProcessor] + safety_checker: Optional[IFSafetyChecker] + + watermarker: Optional[IFWatermarker] + + bad_punct_regex = re.compile( + r"[" + + "#®•©™&@·º½¾¿¡§~" + + r"\)" + + r"\(" + + r"\]" + + r"\[" + + r"\}" + + r"\{" + + r"\|" + + "\\" + + r"\/" + + r"\*" + + r"]{1,}" + ) # noqa + + _optional_components = ["tokenizer", "text_encoder", "safety_checker", "feature_extractor", "watermarker"] + model_cpu_offload_seq = "text_encoder->unet" + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + safety_checker: Optional[IFSafetyChecker], + feature_extractor: Optional[CLIPImageProcessor], + watermarker: Optional[IFWatermarker], + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the IF license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + watermarker=watermarker, + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.remove_all_hooks + def remove_all_hooks(self): + if is_accelerate_available(): + from accelerate.hooks import remove_hook_from_module + else: + raise ImportError("Please install accelerate via `pip install accelerate`") + + for model in [self.text_encoder, self.unet, self.safety_checker]: + if model is not None: + remove_hook_from_module(model, recurse=True) + + self.unet_offload_hook = None + self.text_encoder_offload_hook = None + self.final_offload_hook = None + + @torch.no_grad() + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.encode_prompt + def encode_prompt( + self, + prompt: Union[str, List[str]], + do_classifier_free_guidance: bool = True, + num_images_per_prompt: int = 1, + device: Optional[torch.device] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + clean_caption: bool = False, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + whether to use classifier free guidance or not + num_images_per_prompt (`int`, *optional*, defaults to 1): + number of images that should be generated per prompt + device: (`torch.device`, *optional*): + torch device to place the resulting embeddings on + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. + Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + clean_caption (bool, defaults to `False`): + If `True`, the function will preprocess and clean the provided caption before encoding. + """ + if prompt is not None and negative_prompt is not None: + if type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + + if device is None: + device = self._execution_device + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # while T5 can handle much longer input sequences than 77, the text encoder was trained with a max length of 77 for IF + max_length = 77 + + if prompt_embeds is None: + prompt = self._text_preprocessing(prompt, clean_caption=clean_caption) + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {max_length} tokens: {removed_text}" + ) + + attention_mask = text_inputs.attention_mask.to(device) + + prompt_embeds = self.text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + + if self.text_encoder is not None: + dtype = self.text_encoder.dtype + elif self.unet is not None: + dtype = self.unet.dtype + else: + dtype = None + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_tokens = self._text_preprocessing(uncond_tokens, clean_caption=clean_caption) + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + attention_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + else: + negative_prompt_embeds = None + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is not None: + safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device) + image, nsfw_detected, watermark_detected = self.safety_checker( + images=image, + clip_input=safety_checker_input.pixel_values.to(dtype=dtype), + ) + else: + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + + return image, nsfw_detected, watermark_detected + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + image, + mask_image, + batch_size, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # image + + if isinstance(image, list): + check_image_type = image[0] + else: + check_image_type = image + + if ( + not isinstance(check_image_type, torch.Tensor) + and not isinstance(check_image_type, PIL.Image.Image) + and not isinstance(check_image_type, np.ndarray) + ): + raise ValueError( + "`image` has to be of type `torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + f" {type(check_image_type)}" + ) + + if isinstance(image, list): + image_batch_size = len(image) + elif isinstance(image, torch.Tensor): + image_batch_size = image.shape[0] + elif isinstance(image, PIL.Image.Image): + image_batch_size = 1 + elif isinstance(image, np.ndarray): + image_batch_size = image.shape[0] + else: + assert False + + if batch_size != image_batch_size: + raise ValueError(f"image batch size: {image_batch_size} must be same as prompt batch size {batch_size}") + + # mask_image + + if isinstance(mask_image, list): + check_image_type = mask_image[0] + else: + check_image_type = mask_image + + if ( + not isinstance(check_image_type, torch.Tensor) + and not isinstance(check_image_type, PIL.Image.Image) + and not isinstance(check_image_type, np.ndarray) + ): + raise ValueError( + "`mask_image` has to be of type `torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + f" {type(check_image_type)}" + ) + + if isinstance(mask_image, list): + image_batch_size = len(mask_image) + elif isinstance(mask_image, torch.Tensor): + image_batch_size = mask_image.shape[0] + elif isinstance(mask_image, PIL.Image.Image): + image_batch_size = 1 + elif isinstance(mask_image, np.ndarray): + image_batch_size = mask_image.shape[0] + else: + assert False + + if image_batch_size != 1 and batch_size != image_batch_size: + raise ValueError( + f"mask_image batch size: {image_batch_size} must be `1` or the same as prompt batch size {batch_size}" + ) + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._text_preprocessing + def _text_preprocessing(self, text, clean_caption=False): + if clean_caption and not is_bs4_available(): + logger.warning(BACKENDS_MAPPING["bs4"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if clean_caption and not is_ftfy_available(): + logger.warning(BACKENDS_MAPPING["ftfy"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if not isinstance(text, (tuple, list)): + text = [text] + + def process(text: str): + if clean_caption: + text = self._clean_caption(text) + text = self._clean_caption(text) + else: + text = text.lower().strip() + return text + + return [process(t) for t in text] + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._clean_caption + def _clean_caption(self, caption): + caption = str(caption) + caption = ul.unquote_plus(caption) + caption = caption.strip().lower() + caption = re.sub("", "person", caption) + # urls: + caption = re.sub( + r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + caption = re.sub( + r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + # html: + caption = BeautifulSoup(caption, features="html.parser").text + + # @ + caption = re.sub(r"@[\w\d]+\b", "", caption) + + # 31C0—31EF CJK Strokes + # 31F0—31FF Katakana Phonetic Extensions + # 3200—32FF Enclosed CJK Letters and Months + # 3300—33FF CJK Compatibility + # 3400—4DBF CJK Unified Ideographs Extension A + # 4DC0—4DFF Yijing Hexagram Symbols + # 4E00—9FFF CJK Unified Ideographs + caption = re.sub(r"[\u31c0-\u31ef]+", "", caption) + caption = re.sub(r"[\u31f0-\u31ff]+", "", caption) + caption = re.sub(r"[\u3200-\u32ff]+", "", caption) + caption = re.sub(r"[\u3300-\u33ff]+", "", caption) + caption = re.sub(r"[\u3400-\u4dbf]+", "", caption) + caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption) + caption = re.sub(r"[\u4e00-\u9fff]+", "", caption) + ####################################################### + + # все виды тире / all types of dash --> "-" + caption = re.sub( + r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa + "-", + caption, + ) + + # кавычки к одному стандарту + caption = re.sub(r"[`´«»“”¨]", '"', caption) + caption = re.sub(r"[‘’]", "'", caption) + + # " + caption = re.sub(r""?", "", caption) + # & + caption = re.sub(r"&", "", caption) + + # ip adresses: + caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption) + + # article ids: + caption = re.sub(r"\d:\d\d\s+$", "", caption) + + # \n + caption = re.sub(r"\\n", " ", caption) + + # "#123" + caption = re.sub(r"#\d{1,3}\b", "", caption) + # "#12345.." + caption = re.sub(r"#\d{5,}\b", "", caption) + # "123456.." + caption = re.sub(r"\b\d{6,}\b", "", caption) + # filenames: + caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption) + + # + caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT""" + caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT""" + + caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT + caption = re.sub(r"\s+\.\s+", r" ", caption) # " . " + + # this-is-my-cute-cat / this_is_my_cute_cat + regex2 = re.compile(r"(?:\-|\_)") + if len(re.findall(regex2, caption)) > 3: + caption = re.sub(regex2, " ", caption) + + caption = ftfy.fix_text(caption) + caption = html.unescape(html.unescape(caption)) + + caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640 + caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc + caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231 + + caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption) + caption = re.sub(r"(free\s)?download(\sfree)?", "", caption) + caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption) + caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption) + caption = re.sub(r"\bpage\s+\d+\b", "", caption) + + caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a... + + caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption) + + caption = re.sub(r"\b\s+\:\s+", r": ", caption) + caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption) + caption = re.sub(r"\s+", " ", caption) + + caption.strip() + + caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption) + caption = re.sub(r"^[\'\_,\-\:;]", r"", caption) + caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption) + caption = re.sub(r"^\.\S+$", "", caption) + + return caption.strip() + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_img2img.IFImg2ImgPipeline.preprocess_image + def preprocess_image(self, image: PIL.Image.Image) -> torch.Tensor: + if not isinstance(image, list): + image = [image] + + def numpy_to_pt(images): + if images.ndim == 3: + images = images[..., None] + + images = torch.from_numpy(images.transpose(0, 3, 1, 2)) + return images + + if isinstance(image[0], PIL.Image.Image): + new_image = [] + + for image_ in image: + image_ = image_.convert("RGB") + image_ = resize(image_, self.unet.sample_size) + image_ = np.array(image_) + image_ = image_.astype(np.float32) + image_ = image_ / 127.5 - 1 + new_image.append(image_) + + image = new_image + + image = np.stack(image, axis=0) # to np + image = numpy_to_pt(image) # to pt + + elif isinstance(image[0], np.ndarray): + image = np.concatenate(image, axis=0) if image[0].ndim == 4 else np.stack(image, axis=0) + image = numpy_to_pt(image) + + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, axis=0) if image[0].ndim == 4 else torch.stack(image, axis=0) + + return image + + def preprocess_mask_image(self, mask_image) -> torch.Tensor: + if not isinstance(mask_image, list): + mask_image = [mask_image] + + if isinstance(mask_image[0], torch.Tensor): + mask_image = torch.cat(mask_image, axis=0) if mask_image[0].ndim == 4 else torch.stack(mask_image, axis=0) + + if mask_image.ndim == 2: + # Batch and add channel dim for single mask + mask_image = mask_image.unsqueeze(0).unsqueeze(0) + elif mask_image.ndim == 3 and mask_image.shape[0] == 1: + # Single mask, the 0'th dimension is considered to be + # the existing batch size of 1 + mask_image = mask_image.unsqueeze(0) + elif mask_image.ndim == 3 and mask_image.shape[0] != 1: + # Batch of mask, the 0'th dimension is considered to be + # the batching dimension + mask_image = mask_image.unsqueeze(1) + + mask_image[mask_image < 0.5] = 0 + mask_image[mask_image >= 0.5] = 1 + + elif isinstance(mask_image[0], PIL.Image.Image): + new_mask_image = [] + + for mask_image_ in mask_image: + mask_image_ = mask_image_.convert("L") + mask_image_ = resize(mask_image_, self.unet.sample_size) + mask_image_ = np.array(mask_image_) + mask_image_ = mask_image_[None, None, :] + new_mask_image.append(mask_image_) + + mask_image = new_mask_image + + mask_image = np.concatenate(mask_image, axis=0) + mask_image = mask_image.astype(np.float32) / 255.0 + mask_image[mask_image < 0.5] = 0 + mask_image[mask_image >= 0.5] = 1 + mask_image = torch.from_numpy(mask_image) + + elif isinstance(mask_image[0], np.ndarray): + mask_image = np.concatenate([m[None, None, :] for m in mask_image], axis=0) + + mask_image[mask_image < 0.5] = 0 + mask_image[mask_image >= 0.5] = 1 + mask_image = torch.from_numpy(mask_image) + + return mask_image + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_img2img.IFImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start:] + + return timesteps, num_inference_steps - t_start + + def prepare_intermediate_images( + self, image, timestep, batch_size, num_images_per_prompt, dtype, device, mask_image, generator=None + ): + image_batch_size, channels, height, width = image.shape + + batch_size = batch_size * num_images_per_prompt + + shape = (batch_size, channels, height, width) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + image = image.repeat_interleave(num_images_per_prompt, dim=0) + noised_image = self.scheduler.add_noise(image, noise, timestep) + + image = (1 - mask_image) * image + mask_image * noised_image + + return image + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: Union[ + PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] + ] = None, + mask_image: Union[ + PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] + ] = None, + strength: float = 1.0, + num_inference_steps: int = 50, + timesteps: List[int] = None, + guidance_scale: float = 7.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + clean_caption: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + image (`torch.FloatTensor` or `PIL.Image.Image`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. + mask_image (`PIL.Image.Image`): + `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be + repainted, while black pixels will be preserved. If `mask_image` is a PIL image, it will be converted + to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) + instead of 3, so the expected shape would be `(B, H, W, 1)`. + strength (`float`, *optional*, defaults to 1.0): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps are used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + clean_caption (`bool`, *optional*, defaults to `True`): + Whether or not to clean the caption before creating embeddings. Requires `beautifulsoup4` and `ftfy` to + be installed. If the dependencies are not installed, the embeddings will be created from the raw + prompt. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + Examples: + + Returns: + [`~pipelines.stable_diffusion.IFPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.IFPipelineOutput`] if `return_dict` is True, otherwise a `tuple. When + returning a tuple, the first element is a list with the generated images, and the second element is a list + of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" (nsfw) + or watermarked content, according to the `safety_checker`. + """ + # 1. Check inputs. Raise error if not correct + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + self.check_inputs( + prompt, + image, + mask_image, + batch_size, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + # 2. Define call parameters + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + do_classifier_free_guidance, + num_images_per_prompt=num_images_per_prompt, + device=device, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clean_caption=clean_caption, + ) + + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + dtype = prompt_embeds.dtype + + # 4. Prepare timesteps + if timesteps is not None: + self.scheduler.set_timesteps(timesteps=timesteps, device=device) + timesteps = self.scheduler.timesteps + num_inference_steps = len(timesteps) + else: + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength) + + # 5. Prepare intermediate images + image = self.preprocess_image(image) + image = image.to(device=device, dtype=dtype) + + mask_image = self.preprocess_mask_image(mask_image) + mask_image = mask_image.to(device=device, dtype=dtype) + + if mask_image.shape[0] == 1: + mask_image = mask_image.repeat_interleave(batch_size * num_images_per_prompt, dim=0) + else: + mask_image = mask_image.repeat_interleave(num_images_per_prompt, dim=0) + + noise_timestep = timesteps[0:1] + noise_timestep = noise_timestep.repeat(batch_size * num_images_per_prompt) + + intermediate_images = self.prepare_intermediate_images( + image, noise_timestep, batch_size, num_images_per_prompt, dtype, device, mask_image, generator + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # HACK: see comment in `enable_model_cpu_offload` + if hasattr(self, "text_encoder_offload_hook") and self.text_encoder_offload_hook is not None: + self.text_encoder_offload_hook.offload() + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + model_input = ( + torch.cat([intermediate_images] * 2) if do_classifier_free_guidance else intermediate_images + ) + model_input = self.scheduler.scale_model_input(model_input, t) + + # predict the noise residual + noise_pred = self.unet( + model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred_uncond, _ = noise_pred_uncond.split(model_input.shape[1], dim=1) + noise_pred_text, predicted_variance = noise_pred_text.split(model_input.shape[1], dim=1) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, predicted_variance], dim=1) + + if self.scheduler.config.variance_type not in ["learned", "learned_range"]: + noise_pred, _ = noise_pred.split(model_input.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + prev_intermediate_images = intermediate_images + + intermediate_images = self.scheduler.step( + noise_pred, t, intermediate_images, **extra_step_kwargs, return_dict=False + )[0] + + intermediate_images = (1 - mask_image) * prev_intermediate_images + mask_image * intermediate_images + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, intermediate_images) + + image = intermediate_images + + if output_type == "pil": + # 8. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 9. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # 10. Convert to PIL + image = self.numpy_to_pil(image) + + # 11. Apply watermark + if self.watermarker is not None: + self.watermarker.apply_watermark(image, self.unet.config.sample_size) + elif output_type == "pt": + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + else: + # 8. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 9. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, nsfw_detected, watermark_detected) + + return IFPipelineOutput(images=image, nsfw_detected=nsfw_detected, watermark_detected=watermark_detected) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py new file mode 100755 index 0000000..b67907c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_inpainting_superresolution.py @@ -0,0 +1,1137 @@ +import html +import inspect +import re +import urllib.parse as ul +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from transformers import CLIPImageProcessor, T5EncoderModel, T5Tokenizer + +from ...loaders import LoraLoaderMixin +from ...models import UNet2DConditionModel +from ...schedulers import DDPMScheduler +from ...utils import ( + BACKENDS_MAPPING, + PIL_INTERPOLATION, + is_accelerate_available, + is_bs4_available, + is_ftfy_available, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .pipeline_output import IFPipelineOutput +from .safety_checker import IFSafetyChecker +from .watermark import IFWatermarker + + +if is_bs4_available(): + from bs4 import BeautifulSoup + +if is_ftfy_available(): + import ftfy + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_img2img.resize +def resize(images: PIL.Image.Image, img_size: int) -> PIL.Image.Image: + w, h = images.size + + coef = w / h + + w, h = img_size, img_size + + if coef >= 1: + w = int(round(img_size / 8 * coef) * 8) + else: + h = int(round(img_size / 8 / coef) * 8) + + images = images.resize((w, h), resample=PIL_INTERPOLATION["bicubic"], reducing_gap=None) + + return images + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import IFInpaintingPipeline, IFInpaintingSuperResolutionPipeline, DiffusionPipeline + >>> from diffusers.utils import pt_to_pil + >>> import torch + >>> from PIL import Image + >>> import requests + >>> from io import BytesIO + + >>> url = "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/if/person.png" + >>> response = requests.get(url) + >>> original_image = Image.open(BytesIO(response.content)).convert("RGB") + >>> original_image = original_image + + >>> url = "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/if/glasses_mask.png" + >>> response = requests.get(url) + >>> mask_image = Image.open(BytesIO(response.content)) + >>> mask_image = mask_image + + >>> pipe = IFInpaintingPipeline.from_pretrained( + ... "DeepFloyd/IF-I-XL-v1.0", variant="fp16", torch_dtype=torch.float16 + ... ) + >>> pipe.enable_model_cpu_offload() + + >>> prompt = "blue sunglasses" + + >>> prompt_embeds, negative_embeds = pipe.encode_prompt(prompt) + >>> image = pipe( + ... image=original_image, + ... mask_image=mask_image, + ... prompt_embeds=prompt_embeds, + ... negative_prompt_embeds=negative_embeds, + ... output_type="pt", + ... ).images + + >>> # save intermediate image + >>> pil_image = pt_to_pil(image) + >>> pil_image[0].save("./if_stage_I.png") + + >>> super_res_1_pipe = IFInpaintingSuperResolutionPipeline.from_pretrained( + ... "DeepFloyd/IF-II-L-v1.0", text_encoder=None, variant="fp16", torch_dtype=torch.float16 + ... ) + >>> super_res_1_pipe.enable_model_cpu_offload() + + >>> image = super_res_1_pipe( + ... image=image, + ... mask_image=mask_image, + ... original_image=original_image, + ... prompt_embeds=prompt_embeds, + ... negative_prompt_embeds=negative_embeds, + ... ).images + >>> image[0].save("./if_stage_II.png") + ``` + """ + + +class IFInpaintingSuperResolutionPipeline(DiffusionPipeline, LoraLoaderMixin): + tokenizer: T5Tokenizer + text_encoder: T5EncoderModel + + unet: UNet2DConditionModel + scheduler: DDPMScheduler + image_noising_scheduler: DDPMScheduler + + feature_extractor: Optional[CLIPImageProcessor] + safety_checker: Optional[IFSafetyChecker] + + watermarker: Optional[IFWatermarker] + + bad_punct_regex = re.compile( + r"[" + + "#®•©™&@·º½¾¿¡§~" + + r"\)" + + r"\(" + + r"\]" + + r"\[" + + r"\}" + + r"\{" + + r"\|" + + "\\" + + r"\/" + + r"\*" + + r"]{1,}" + ) # noqa + + model_cpu_offload_seq = "text_encoder->unet" + _optional_components = ["tokenizer", "text_encoder", "safety_checker", "feature_extractor", "watermarker"] + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + image_noising_scheduler: DDPMScheduler, + safety_checker: Optional[IFSafetyChecker], + feature_extractor: Optional[CLIPImageProcessor], + watermarker: Optional[IFWatermarker], + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the IF license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + if unet.config.in_channels != 6: + logger.warning( + "It seems like you have loaded a checkpoint that shall not be used for super resolution from {unet.config._name_or_path} as it accepts {unet.config.in_channels} input channels instead of 6. Please make sure to pass a super resolution checkpoint as the `'unet'`: IFSuperResolutionPipeline.from_pretrained(unet=super_resolution_unet, ...)`." + ) + + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + unet=unet, + scheduler=scheduler, + image_noising_scheduler=image_noising_scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + watermarker=watermarker, + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.remove_all_hooks + def remove_all_hooks(self): + if is_accelerate_available(): + from accelerate.hooks import remove_hook_from_module + else: + raise ImportError("Please install accelerate via `pip install accelerate`") + + for model in [self.text_encoder, self.unet, self.safety_checker]: + if model is not None: + remove_hook_from_module(model, recurse=True) + + self.unet_offload_hook = None + self.text_encoder_offload_hook = None + self.final_offload_hook = None + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._text_preprocessing + def _text_preprocessing(self, text, clean_caption=False): + if clean_caption and not is_bs4_available(): + logger.warning(BACKENDS_MAPPING["bs4"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if clean_caption and not is_ftfy_available(): + logger.warning(BACKENDS_MAPPING["ftfy"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if not isinstance(text, (tuple, list)): + text = [text] + + def process(text: str): + if clean_caption: + text = self._clean_caption(text) + text = self._clean_caption(text) + else: + text = text.lower().strip() + return text + + return [process(t) for t in text] + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._clean_caption + def _clean_caption(self, caption): + caption = str(caption) + caption = ul.unquote_plus(caption) + caption = caption.strip().lower() + caption = re.sub("", "person", caption) + # urls: + caption = re.sub( + r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + caption = re.sub( + r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + # html: + caption = BeautifulSoup(caption, features="html.parser").text + + # @ + caption = re.sub(r"@[\w\d]+\b", "", caption) + + # 31C0—31EF CJK Strokes + # 31F0—31FF Katakana Phonetic Extensions + # 3200—32FF Enclosed CJK Letters and Months + # 3300—33FF CJK Compatibility + # 3400—4DBF CJK Unified Ideographs Extension A + # 4DC0—4DFF Yijing Hexagram Symbols + # 4E00—9FFF CJK Unified Ideographs + caption = re.sub(r"[\u31c0-\u31ef]+", "", caption) + caption = re.sub(r"[\u31f0-\u31ff]+", "", caption) + caption = re.sub(r"[\u3200-\u32ff]+", "", caption) + caption = re.sub(r"[\u3300-\u33ff]+", "", caption) + caption = re.sub(r"[\u3400-\u4dbf]+", "", caption) + caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption) + caption = re.sub(r"[\u4e00-\u9fff]+", "", caption) + ####################################################### + + # все виды тире / all types of dash --> "-" + caption = re.sub( + r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa + "-", + caption, + ) + + # кавычки к одному стандарту + caption = re.sub(r"[`´«»“”¨]", '"', caption) + caption = re.sub(r"[‘’]", "'", caption) + + # " + caption = re.sub(r""?", "", caption) + # & + caption = re.sub(r"&", "", caption) + + # ip adresses: + caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption) + + # article ids: + caption = re.sub(r"\d:\d\d\s+$", "", caption) + + # \n + caption = re.sub(r"\\n", " ", caption) + + # "#123" + caption = re.sub(r"#\d{1,3}\b", "", caption) + # "#12345.." + caption = re.sub(r"#\d{5,}\b", "", caption) + # "123456.." + caption = re.sub(r"\b\d{6,}\b", "", caption) + # filenames: + caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption) + + # + caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT""" + caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT""" + + caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT + caption = re.sub(r"\s+\.\s+", r" ", caption) # " . " + + # this-is-my-cute-cat / this_is_my_cute_cat + regex2 = re.compile(r"(?:\-|\_)") + if len(re.findall(regex2, caption)) > 3: + caption = re.sub(regex2, " ", caption) + + caption = ftfy.fix_text(caption) + caption = html.unescape(html.unescape(caption)) + + caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640 + caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc + caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231 + + caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption) + caption = re.sub(r"(free\s)?download(\sfree)?", "", caption) + caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption) + caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption) + caption = re.sub(r"\bpage\s+\d+\b", "", caption) + + caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a... + + caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption) + + caption = re.sub(r"\b\s+\:\s+", r": ", caption) + caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption) + caption = re.sub(r"\s+", " ", caption) + + caption.strip() + + caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption) + caption = re.sub(r"^[\'\_,\-\:;]", r"", caption) + caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption) + caption = re.sub(r"^\.\S+$", "", caption) + + return caption.strip() + + @torch.no_grad() + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.encode_prompt + def encode_prompt( + self, + prompt: Union[str, List[str]], + do_classifier_free_guidance: bool = True, + num_images_per_prompt: int = 1, + device: Optional[torch.device] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + clean_caption: bool = False, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + whether to use classifier free guidance or not + num_images_per_prompt (`int`, *optional*, defaults to 1): + number of images that should be generated per prompt + device: (`torch.device`, *optional*): + torch device to place the resulting embeddings on + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. + Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + clean_caption (bool, defaults to `False`): + If `True`, the function will preprocess and clean the provided caption before encoding. + """ + if prompt is not None and negative_prompt is not None: + if type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + + if device is None: + device = self._execution_device + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # while T5 can handle much longer input sequences than 77, the text encoder was trained with a max length of 77 for IF + max_length = 77 + + if prompt_embeds is None: + prompt = self._text_preprocessing(prompt, clean_caption=clean_caption) + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {max_length} tokens: {removed_text}" + ) + + attention_mask = text_inputs.attention_mask.to(device) + + prompt_embeds = self.text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + + if self.text_encoder is not None: + dtype = self.text_encoder.dtype + elif self.unet is not None: + dtype = self.unet.dtype + else: + dtype = None + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_tokens = self._text_preprocessing(uncond_tokens, clean_caption=clean_caption) + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + attention_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + else: + negative_prompt_embeds = None + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is not None: + safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device) + image, nsfw_detected, watermark_detected = self.safety_checker( + images=image, + clip_input=safety_checker_input.pixel_values.to(dtype=dtype), + ) + else: + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + + return image, nsfw_detected, watermark_detected + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + image, + original_image, + mask_image, + batch_size, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # image + + if isinstance(image, list): + check_image_type = image[0] + else: + check_image_type = image + + if ( + not isinstance(check_image_type, torch.Tensor) + and not isinstance(check_image_type, PIL.Image.Image) + and not isinstance(check_image_type, np.ndarray) + ): + raise ValueError( + "`image` has to be of type `torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + f" {type(check_image_type)}" + ) + + if isinstance(image, list): + image_batch_size = len(image) + elif isinstance(image, torch.Tensor): + image_batch_size = image.shape[0] + elif isinstance(image, PIL.Image.Image): + image_batch_size = 1 + elif isinstance(image, np.ndarray): + image_batch_size = image.shape[0] + else: + assert False + + if batch_size != image_batch_size: + raise ValueError(f"image batch size: {image_batch_size} must be same as prompt batch size {batch_size}") + + # original_image + + if isinstance(original_image, list): + check_image_type = original_image[0] + else: + check_image_type = original_image + + if ( + not isinstance(check_image_type, torch.Tensor) + and not isinstance(check_image_type, PIL.Image.Image) + and not isinstance(check_image_type, np.ndarray) + ): + raise ValueError( + "`original_image` has to be of type `torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + f" {type(check_image_type)}" + ) + + if isinstance(original_image, list): + image_batch_size = len(original_image) + elif isinstance(original_image, torch.Tensor): + image_batch_size = original_image.shape[0] + elif isinstance(original_image, PIL.Image.Image): + image_batch_size = 1 + elif isinstance(original_image, np.ndarray): + image_batch_size = original_image.shape[0] + else: + assert False + + if batch_size != image_batch_size: + raise ValueError( + f"original_image batch size: {image_batch_size} must be same as prompt batch size {batch_size}" + ) + + # mask_image + + if isinstance(mask_image, list): + check_image_type = mask_image[0] + else: + check_image_type = mask_image + + if ( + not isinstance(check_image_type, torch.Tensor) + and not isinstance(check_image_type, PIL.Image.Image) + and not isinstance(check_image_type, np.ndarray) + ): + raise ValueError( + "`mask_image` has to be of type `torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + f" {type(check_image_type)}" + ) + + if isinstance(mask_image, list): + image_batch_size = len(mask_image) + elif isinstance(mask_image, torch.Tensor): + image_batch_size = mask_image.shape[0] + elif isinstance(mask_image, PIL.Image.Image): + image_batch_size = 1 + elif isinstance(mask_image, np.ndarray): + image_batch_size = mask_image.shape[0] + else: + assert False + + if image_batch_size != 1 and batch_size != image_batch_size: + raise ValueError( + f"mask_image batch size: {image_batch_size} must be `1` or the same as prompt batch size {batch_size}" + ) + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_img2img.IFImg2ImgPipeline.preprocess_image with preprocess_image -> preprocess_original_image + def preprocess_original_image(self, image: PIL.Image.Image) -> torch.Tensor: + if not isinstance(image, list): + image = [image] + + def numpy_to_pt(images): + if images.ndim == 3: + images = images[..., None] + + images = torch.from_numpy(images.transpose(0, 3, 1, 2)) + return images + + if isinstance(image[0], PIL.Image.Image): + new_image = [] + + for image_ in image: + image_ = image_.convert("RGB") + image_ = resize(image_, self.unet.sample_size) + image_ = np.array(image_) + image_ = image_.astype(np.float32) + image_ = image_ / 127.5 - 1 + new_image.append(image_) + + image = new_image + + image = np.stack(image, axis=0) # to np + image = numpy_to_pt(image) # to pt + + elif isinstance(image[0], np.ndarray): + image = np.concatenate(image, axis=0) if image[0].ndim == 4 else np.stack(image, axis=0) + image = numpy_to_pt(image) + + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, axis=0) if image[0].ndim == 4 else torch.stack(image, axis=0) + + return image + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_superresolution.IFSuperResolutionPipeline.preprocess_image + def preprocess_image(self, image: PIL.Image.Image, num_images_per_prompt, device) -> torch.Tensor: + if not isinstance(image, torch.Tensor) and not isinstance(image, list): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + image = [np.array(i).astype(np.float32) / 127.5 - 1.0 for i in image] + + image = np.stack(image, axis=0) # to np + image = torch.from_numpy(image.transpose(0, 3, 1, 2)) + elif isinstance(image[0], np.ndarray): + image = np.stack(image, axis=0) # to np + if image.ndim == 5: + image = image[0] + + image = torch.from_numpy(image.transpose(0, 3, 1, 2)) + elif isinstance(image, list) and isinstance(image[0], torch.Tensor): + dims = image[0].ndim + + if dims == 3: + image = torch.stack(image, dim=0) + elif dims == 4: + image = torch.concat(image, dim=0) + else: + raise ValueError(f"Image must have 3 or 4 dimensions, instead got {dims}") + + image = image.to(device=device, dtype=self.unet.dtype) + + image = image.repeat_interleave(num_images_per_prompt, dim=0) + + return image + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_inpainting.IFInpaintingPipeline.preprocess_mask_image + def preprocess_mask_image(self, mask_image) -> torch.Tensor: + if not isinstance(mask_image, list): + mask_image = [mask_image] + + if isinstance(mask_image[0], torch.Tensor): + mask_image = torch.cat(mask_image, axis=0) if mask_image[0].ndim == 4 else torch.stack(mask_image, axis=0) + + if mask_image.ndim == 2: + # Batch and add channel dim for single mask + mask_image = mask_image.unsqueeze(0).unsqueeze(0) + elif mask_image.ndim == 3 and mask_image.shape[0] == 1: + # Single mask, the 0'th dimension is considered to be + # the existing batch size of 1 + mask_image = mask_image.unsqueeze(0) + elif mask_image.ndim == 3 and mask_image.shape[0] != 1: + # Batch of mask, the 0'th dimension is considered to be + # the batching dimension + mask_image = mask_image.unsqueeze(1) + + mask_image[mask_image < 0.5] = 0 + mask_image[mask_image >= 0.5] = 1 + + elif isinstance(mask_image[0], PIL.Image.Image): + new_mask_image = [] + + for mask_image_ in mask_image: + mask_image_ = mask_image_.convert("L") + mask_image_ = resize(mask_image_, self.unet.sample_size) + mask_image_ = np.array(mask_image_) + mask_image_ = mask_image_[None, None, :] + new_mask_image.append(mask_image_) + + mask_image = new_mask_image + + mask_image = np.concatenate(mask_image, axis=0) + mask_image = mask_image.astype(np.float32) / 255.0 + mask_image[mask_image < 0.5] = 0 + mask_image[mask_image >= 0.5] = 1 + mask_image = torch.from_numpy(mask_image) + + elif isinstance(mask_image[0], np.ndarray): + mask_image = np.concatenate([m[None, None, :] for m in mask_image], axis=0) + + mask_image[mask_image < 0.5] = 0 + mask_image[mask_image >= 0.5] = 1 + mask_image = torch.from_numpy(mask_image) + + return mask_image + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_img2img.IFImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start:] + + return timesteps, num_inference_steps - t_start + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if_inpainting.IFInpaintingPipeline.prepare_intermediate_images + def prepare_intermediate_images( + self, image, timestep, batch_size, num_images_per_prompt, dtype, device, mask_image, generator=None + ): + image_batch_size, channels, height, width = image.shape + + batch_size = batch_size * num_images_per_prompt + + shape = (batch_size, channels, height, width) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + image = image.repeat_interleave(num_images_per_prompt, dim=0) + noised_image = self.scheduler.add_noise(image, noise, timestep) + + image = (1 - mask_image) * image + mask_image * noised_image + + return image + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image: Union[PIL.Image.Image, np.ndarray, torch.FloatTensor], + original_image: Union[ + PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] + ] = None, + mask_image: Union[ + PIL.Image.Image, torch.Tensor, np.ndarray, List[PIL.Image.Image], List[torch.Tensor], List[np.ndarray] + ] = None, + strength: float = 0.8, + prompt: Union[str, List[str]] = None, + num_inference_steps: int = 100, + timesteps: List[int] = None, + guidance_scale: float = 4.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + noise_level: int = 0, + clean_caption: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + image (`torch.FloatTensor` or `PIL.Image.Image`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. + original_image (`torch.FloatTensor` or `PIL.Image.Image`): + The original image that `image` was varied from. + mask_image (`PIL.Image.Image`): + `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be + repainted, while black pixels will be preserved. If `mask_image` is a PIL image, it will be converted + to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) + instead of 3, so the expected shape would be `(B, H, W, 1)`. + strength (`float`, *optional*, defaults to 0.8): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps are used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + noise_level (`int`, *optional*, defaults to 0): + The amount of noise to add to the upscaled image. Must be in the range `[0, 1000)` + clean_caption (`bool`, *optional*, defaults to `True`): + Whether or not to clean the caption before creating embeddings. Requires `beautifulsoup4` and `ftfy` to + be installed. If the dependencies are not installed, the embeddings will be created from the raw + prompt. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.IFPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.IFPipelineOutput`] if `return_dict` is True, otherwise a `tuple. When + returning a tuple, the first element is a list with the generated images, and the second element is a list + of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" (nsfw) + or watermarked content, according to the `safety_checker`. + """ + # 1. Check inputs. Raise error if not correct + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + self.check_inputs( + prompt, + image, + original_image, + mask_image, + batch_size, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + # 2. Define call parameters + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + device = self._execution_device + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + do_classifier_free_guidance, + num_images_per_prompt=num_images_per_prompt, + device=device, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clean_caption=clean_caption, + ) + + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + dtype = prompt_embeds.dtype + + # 4. Prepare timesteps + if timesteps is not None: + self.scheduler.set_timesteps(timesteps=timesteps, device=device) + timesteps = self.scheduler.timesteps + num_inference_steps = len(timesteps) + else: + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength) + + # 5. prepare original image + original_image = self.preprocess_original_image(original_image) + original_image = original_image.to(device=device, dtype=dtype) + + # 6. prepare mask image + mask_image = self.preprocess_mask_image(mask_image) + mask_image = mask_image.to(device=device, dtype=dtype) + + if mask_image.shape[0] == 1: + mask_image = mask_image.repeat_interleave(batch_size * num_images_per_prompt, dim=0) + else: + mask_image = mask_image.repeat_interleave(num_images_per_prompt, dim=0) + + # 6. Prepare intermediate images + noise_timestep = timesteps[0:1] + noise_timestep = noise_timestep.repeat(batch_size * num_images_per_prompt) + + intermediate_images = self.prepare_intermediate_images( + original_image, + noise_timestep, + batch_size, + num_images_per_prompt, + dtype, + device, + mask_image, + generator, + ) + + # 7. Prepare upscaled image and noise level + _, _, height, width = original_image.shape + + image = self.preprocess_image(image, num_images_per_prompt, device) + + upscaled = F.interpolate(image, (height, width), mode="bilinear", align_corners=True) + + noise_level = torch.tensor([noise_level] * upscaled.shape[0], device=upscaled.device) + noise = randn_tensor(upscaled.shape, generator=generator, device=upscaled.device, dtype=upscaled.dtype) + upscaled = self.image_noising_scheduler.add_noise(upscaled, noise, timesteps=noise_level) + + if do_classifier_free_guidance: + noise_level = torch.cat([noise_level] * 2) + + # 8. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # HACK: see comment in `enable_model_cpu_offload` + if hasattr(self, "text_encoder_offload_hook") and self.text_encoder_offload_hook is not None: + self.text_encoder_offload_hook.offload() + + # 9. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + model_input = torch.cat([intermediate_images, upscaled], dim=1) + + model_input = torch.cat([model_input] * 2) if do_classifier_free_guidance else model_input + model_input = self.scheduler.scale_model_input(model_input, t) + + # predict the noise residual + noise_pred = self.unet( + model_input, + t, + encoder_hidden_states=prompt_embeds, + class_labels=noise_level, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred_uncond, _ = noise_pred_uncond.split(model_input.shape[1] // 2, dim=1) + noise_pred_text, predicted_variance = noise_pred_text.split(model_input.shape[1] // 2, dim=1) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, predicted_variance], dim=1) + + if self.scheduler.config.variance_type not in ["learned", "learned_range"]: + noise_pred, _ = noise_pred.split(intermediate_images.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + prev_intermediate_images = intermediate_images + + intermediate_images = self.scheduler.step( + noise_pred, t, intermediate_images, **extra_step_kwargs, return_dict=False + )[0] + + intermediate_images = (1 - mask_image) * prev_intermediate_images + mask_image * intermediate_images + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, intermediate_images) + + image = intermediate_images + + if output_type == "pil": + # 10. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 11. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # 12. Convert to PIL + image = self.numpy_to_pil(image) + + # 13. Apply watermark + if self.watermarker is not None: + self.watermarker.apply_watermark(image, self.unet.config.sample_size) + elif output_type == "pt": + nsfw_detected = None + watermark_detected = None + + else: + # 10. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 11. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + self.maybe_free_model_hooks() + + if not return_dict: + return (image, nsfw_detected, watermark_detected) + + return IFPipelineOutput(images=image, nsfw_detected=nsfw_detected, watermark_detected=watermark_detected) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py new file mode 100755 index 0000000..a293343 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_if_superresolution.py @@ -0,0 +1,885 @@ +import html +import inspect +import re +import urllib.parse as ul +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from transformers import CLIPImageProcessor, T5EncoderModel, T5Tokenizer + +from ...loaders import LoraLoaderMixin +from ...models import UNet2DConditionModel +from ...schedulers import DDPMScheduler +from ...utils import ( + BACKENDS_MAPPING, + is_accelerate_available, + is_bs4_available, + is_ftfy_available, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .pipeline_output import IFPipelineOutput +from .safety_checker import IFSafetyChecker +from .watermark import IFWatermarker + + +if is_bs4_available(): + from bs4 import BeautifulSoup + +if is_ftfy_available(): + import ftfy + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import IFPipeline, IFSuperResolutionPipeline, DiffusionPipeline + >>> from diffusers.utils import pt_to_pil + >>> import torch + + >>> pipe = IFPipeline.from_pretrained("DeepFloyd/IF-I-XL-v1.0", variant="fp16", torch_dtype=torch.float16) + >>> pipe.enable_model_cpu_offload() + + >>> prompt = 'a photo of a kangaroo wearing an orange hoodie and blue sunglasses standing in front of the eiffel tower holding a sign that says "very deep learning"' + >>> prompt_embeds, negative_embeds = pipe.encode_prompt(prompt) + + >>> image = pipe(prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds, output_type="pt").images + + >>> # save intermediate image + >>> pil_image = pt_to_pil(image) + >>> pil_image[0].save("./if_stage_I.png") + + >>> super_res_1_pipe = IFSuperResolutionPipeline.from_pretrained( + ... "DeepFloyd/IF-II-L-v1.0", text_encoder=None, variant="fp16", torch_dtype=torch.float16 + ... ) + >>> super_res_1_pipe.enable_model_cpu_offload() + + >>> image = super_res_1_pipe( + ... image=image, prompt_embeds=prompt_embeds, negative_prompt_embeds=negative_embeds + ... ).images + >>> image[0].save("./if_stage_II.png") + ``` +""" + + +class IFSuperResolutionPipeline(DiffusionPipeline, LoraLoaderMixin): + tokenizer: T5Tokenizer + text_encoder: T5EncoderModel + + unet: UNet2DConditionModel + scheduler: DDPMScheduler + image_noising_scheduler: DDPMScheduler + + feature_extractor: Optional[CLIPImageProcessor] + safety_checker: Optional[IFSafetyChecker] + + watermarker: Optional[IFWatermarker] + + bad_punct_regex = re.compile( + r"[" + + "#®•©™&@·º½¾¿¡§~" + + r"\)" + + r"\(" + + r"\]" + + r"\[" + + r"\}" + + r"\{" + + r"\|" + + "\\" + + r"\/" + + r"\*" + + r"]{1,}" + ) # noqa + + _optional_components = ["tokenizer", "text_encoder", "safety_checker", "feature_extractor", "watermarker"] + model_cpu_offload_seq = "text_encoder->unet" + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + image_noising_scheduler: DDPMScheduler, + safety_checker: Optional[IFSafetyChecker], + feature_extractor: Optional[CLIPImageProcessor], + watermarker: Optional[IFWatermarker], + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the IF license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + if unet.config.in_channels != 6: + logger.warning( + "It seems like you have loaded a checkpoint that shall not be used for super resolution from {unet.config._name_or_path} as it accepts {unet.config.in_channels} input channels instead of 6. Please make sure to pass a super resolution checkpoint as the `'unet'`: IFSuperResolutionPipeline.from_pretrained(unet=super_resolution_unet, ...)`." + ) + + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + unet=unet, + scheduler=scheduler, + image_noising_scheduler=image_noising_scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + watermarker=watermarker, + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.remove_all_hooks + def remove_all_hooks(self): + if is_accelerate_available(): + from accelerate.hooks import remove_hook_from_module + else: + raise ImportError("Please install accelerate via `pip install accelerate`") + + for model in [self.text_encoder, self.unet, self.safety_checker]: + if model is not None: + remove_hook_from_module(model, recurse=True) + + self.unet_offload_hook = None + self.text_encoder_offload_hook = None + self.final_offload_hook = None + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._text_preprocessing + def _text_preprocessing(self, text, clean_caption=False): + if clean_caption and not is_bs4_available(): + logger.warning(BACKENDS_MAPPING["bs4"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if clean_caption and not is_ftfy_available(): + logger.warning(BACKENDS_MAPPING["ftfy"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if not isinstance(text, (tuple, list)): + text = [text] + + def process(text: str): + if clean_caption: + text = self._clean_caption(text) + text = self._clean_caption(text) + else: + text = text.lower().strip() + return text + + return [process(t) for t in text] + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._clean_caption + def _clean_caption(self, caption): + caption = str(caption) + caption = ul.unquote_plus(caption) + caption = caption.strip().lower() + caption = re.sub("", "person", caption) + # urls: + caption = re.sub( + r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + caption = re.sub( + r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + # html: + caption = BeautifulSoup(caption, features="html.parser").text + + # @ + caption = re.sub(r"@[\w\d]+\b", "", caption) + + # 31C0—31EF CJK Strokes + # 31F0—31FF Katakana Phonetic Extensions + # 3200—32FF Enclosed CJK Letters and Months + # 3300—33FF CJK Compatibility + # 3400—4DBF CJK Unified Ideographs Extension A + # 4DC0—4DFF Yijing Hexagram Symbols + # 4E00—9FFF CJK Unified Ideographs + caption = re.sub(r"[\u31c0-\u31ef]+", "", caption) + caption = re.sub(r"[\u31f0-\u31ff]+", "", caption) + caption = re.sub(r"[\u3200-\u32ff]+", "", caption) + caption = re.sub(r"[\u3300-\u33ff]+", "", caption) + caption = re.sub(r"[\u3400-\u4dbf]+", "", caption) + caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption) + caption = re.sub(r"[\u4e00-\u9fff]+", "", caption) + ####################################################### + + # все виды тире / all types of dash --> "-" + caption = re.sub( + r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa + "-", + caption, + ) + + # кавычки к одному стандарту + caption = re.sub(r"[`´«»“”¨]", '"', caption) + caption = re.sub(r"[‘’]", "'", caption) + + # " + caption = re.sub(r""?", "", caption) + # & + caption = re.sub(r"&", "", caption) + + # ip adresses: + caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption) + + # article ids: + caption = re.sub(r"\d:\d\d\s+$", "", caption) + + # \n + caption = re.sub(r"\\n", " ", caption) + + # "#123" + caption = re.sub(r"#\d{1,3}\b", "", caption) + # "#12345.." + caption = re.sub(r"#\d{5,}\b", "", caption) + # "123456.." + caption = re.sub(r"\b\d{6,}\b", "", caption) + # filenames: + caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption) + + # + caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT""" + caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT""" + + caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT + caption = re.sub(r"\s+\.\s+", r" ", caption) # " . " + + # this-is-my-cute-cat / this_is_my_cute_cat + regex2 = re.compile(r"(?:\-|\_)") + if len(re.findall(regex2, caption)) > 3: + caption = re.sub(regex2, " ", caption) + + caption = ftfy.fix_text(caption) + caption = html.unescape(html.unescape(caption)) + + caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640 + caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc + caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231 + + caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption) + caption = re.sub(r"(free\s)?download(\sfree)?", "", caption) + caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption) + caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption) + caption = re.sub(r"\bpage\s+\d+\b", "", caption) + + caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a... + + caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption) + + caption = re.sub(r"\b\s+\:\s+", r": ", caption) + caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption) + caption = re.sub(r"\s+", " ", caption) + + caption.strip() + + caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption) + caption = re.sub(r"^[\'\_,\-\:;]", r"", caption) + caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption) + caption = re.sub(r"^\.\S+$", "", caption) + + return caption.strip() + + @torch.no_grad() + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.encode_prompt + def encode_prompt( + self, + prompt: Union[str, List[str]], + do_classifier_free_guidance: bool = True, + num_images_per_prompt: int = 1, + device: Optional[torch.device] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + clean_caption: bool = False, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + whether to use classifier free guidance or not + num_images_per_prompt (`int`, *optional*, defaults to 1): + number of images that should be generated per prompt + device: (`torch.device`, *optional*): + torch device to place the resulting embeddings on + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. + Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + clean_caption (bool, defaults to `False`): + If `True`, the function will preprocess and clean the provided caption before encoding. + """ + if prompt is not None and negative_prompt is not None: + if type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + + if device is None: + device = self._execution_device + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # while T5 can handle much longer input sequences than 77, the text encoder was trained with a max length of 77 for IF + max_length = 77 + + if prompt_embeds is None: + prompt = self._text_preprocessing(prompt, clean_caption=clean_caption) + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {max_length} tokens: {removed_text}" + ) + + attention_mask = text_inputs.attention_mask.to(device) + + prompt_embeds = self.text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + + if self.text_encoder is not None: + dtype = self.text_encoder.dtype + elif self.unet is not None: + dtype = self.unet.dtype + else: + dtype = None + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_tokens = self._text_preprocessing(uncond_tokens, clean_caption=clean_caption) + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + attention_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + else: + negative_prompt_embeds = None + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is not None: + safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device) + image, nsfw_detected, watermark_detected = self.safety_checker( + images=image, + clip_input=safety_checker_input.pixel_values.to(dtype=dtype), + ) + else: + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + + return image, nsfw_detected, watermark_detected + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + image, + batch_size, + noise_level, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if noise_level < 0 or noise_level >= self.image_noising_scheduler.config.num_train_timesteps: + raise ValueError( + f"`noise_level`: {noise_level} must be a valid timestep in `self.noising_scheduler`, [0, {self.image_noising_scheduler.config.num_train_timesteps})" + ) + + if isinstance(image, list): + check_image_type = image[0] + else: + check_image_type = image + + if ( + not isinstance(check_image_type, torch.Tensor) + and not isinstance(check_image_type, PIL.Image.Image) + and not isinstance(check_image_type, np.ndarray) + ): + raise ValueError( + "`image` has to be of type `torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, or List[...] but is" + f" {type(check_image_type)}" + ) + + if isinstance(image, list): + image_batch_size = len(image) + elif isinstance(image, torch.Tensor): + image_batch_size = image.shape[0] + elif isinstance(image, PIL.Image.Image): + image_batch_size = 1 + elif isinstance(image, np.ndarray): + image_batch_size = image.shape[0] + else: + assert False + + if batch_size != image_batch_size: + raise ValueError(f"image batch size: {image_batch_size} must be same as prompt batch size {batch_size}") + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline.prepare_intermediate_images + def prepare_intermediate_images(self, batch_size, num_channels, height, width, dtype, device, generator): + shape = (batch_size, num_channels, height, width) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + intermediate_images = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # scale the initial noise by the standard deviation required by the scheduler + intermediate_images = intermediate_images * self.scheduler.init_noise_sigma + return intermediate_images + + def preprocess_image(self, image, num_images_per_prompt, device): + if not isinstance(image, torch.Tensor) and not isinstance(image, list): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + image = [np.array(i).astype(np.float32) / 127.5 - 1.0 for i in image] + + image = np.stack(image, axis=0) # to np + image = torch.from_numpy(image.transpose(0, 3, 1, 2)) + elif isinstance(image[0], np.ndarray): + image = np.stack(image, axis=0) # to np + if image.ndim == 5: + image = image[0] + + image = torch.from_numpy(image.transpose(0, 3, 1, 2)) + elif isinstance(image, list) and isinstance(image[0], torch.Tensor): + dims = image[0].ndim + + if dims == 3: + image = torch.stack(image, dim=0) + elif dims == 4: + image = torch.concat(image, dim=0) + else: + raise ValueError(f"Image must have 3 or 4 dimensions, instead got {dims}") + + image = image.to(device=device, dtype=self.unet.dtype) + + image = image.repeat_interleave(num_images_per_prompt, dim=0) + + return image + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: int = None, + width: int = None, + image: Union[PIL.Image.Image, np.ndarray, torch.FloatTensor] = None, + num_inference_steps: int = 50, + timesteps: List[int] = None, + guidance_scale: float = 4.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + noise_level: int = 250, + clean_caption: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + height (`int`, *optional*, defaults to None): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to None): + The width in pixels of the generated image. + image (`PIL.Image.Image`, `np.ndarray`, `torch.FloatTensor`): + The image to be upscaled. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*, defaults to None): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps are used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + noise_level (`int`, *optional*, defaults to 250): + The amount of noise to add to the upscaled image. Must be in the range `[0, 1000)` + clean_caption (`bool`, *optional*, defaults to `True`): + Whether or not to clean the caption before creating embeddings. Requires `beautifulsoup4` and `ftfy` to + be installed. If the dependencies are not installed, the embeddings will be created from the raw + prompt. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.IFPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.IFPipelineOutput`] if `return_dict` is True, otherwise a `tuple. When + returning a tuple, the first element is a list with the generated images, and the second element is a list + of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" (nsfw) + or watermarked content, according to the `safety_checker`. + """ + # 1. Check inputs. Raise error if not correct + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + self.check_inputs( + prompt, + image, + batch_size, + noise_level, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + # 2. Define call parameters + + height = height or self.unet.config.sample_size + width = width or self.unet.config.sample_size + + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + do_classifier_free_guidance, + num_images_per_prompt=num_images_per_prompt, + device=device, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clean_caption=clean_caption, + ) + + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + if timesteps is not None: + self.scheduler.set_timesteps(timesteps=timesteps, device=device) + timesteps = self.scheduler.timesteps + num_inference_steps = len(timesteps) + else: + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare intermediate images + num_channels = self.unet.config.in_channels // 2 + intermediate_images = self.prepare_intermediate_images( + batch_size * num_images_per_prompt, + num_channels, + height, + width, + prompt_embeds.dtype, + device, + generator, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Prepare upscaled image and noise level + image = self.preprocess_image(image, num_images_per_prompt, device) + upscaled = F.interpolate(image, (height, width), mode="bilinear", align_corners=True) + + noise_level = torch.tensor([noise_level] * upscaled.shape[0], device=upscaled.device) + noise = randn_tensor(upscaled.shape, generator=generator, device=upscaled.device, dtype=upscaled.dtype) + upscaled = self.image_noising_scheduler.add_noise(upscaled, noise, timesteps=noise_level) + + if do_classifier_free_guidance: + noise_level = torch.cat([noise_level] * 2) + + # HACK: see comment in `enable_model_cpu_offload` + if hasattr(self, "text_encoder_offload_hook") and self.text_encoder_offload_hook is not None: + self.text_encoder_offload_hook.offload() + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + model_input = torch.cat([intermediate_images, upscaled], dim=1) + + model_input = torch.cat([model_input] * 2) if do_classifier_free_guidance else model_input + model_input = self.scheduler.scale_model_input(model_input, t) + + # predict the noise residual + noise_pred = self.unet( + model_input, + t, + encoder_hidden_states=prompt_embeds, + class_labels=noise_level, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred_uncond, _ = noise_pred_uncond.split(model_input.shape[1] // 2, dim=1) + noise_pred_text, predicted_variance = noise_pred_text.split(model_input.shape[1] // 2, dim=1) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, predicted_variance], dim=1) + + if self.scheduler.config.variance_type not in ["learned", "learned_range"]: + noise_pred, _ = noise_pred.split(intermediate_images.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + intermediate_images = self.scheduler.step( + noise_pred, t, intermediate_images, **extra_step_kwargs, return_dict=False + )[0] + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, intermediate_images) + + image = intermediate_images + + if output_type == "pil": + # 9. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 10. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # 11. Convert to PIL + image = self.numpy_to_pil(image) + + # 12. Apply watermark + if self.watermarker is not None: + self.watermarker.apply_watermark(image, self.unet.config.sample_size) + elif output_type == "pt": + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + else: + # 9. Post-processing + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + # 10. Run safety checker + image, nsfw_detected, watermark_detected = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, nsfw_detected, watermark_detected) + + return IFPipelineOutput(images=image, nsfw_detected=nsfw_detected, watermark_detected=watermark_detected) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_output.py b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_output.py new file mode 100755 index 0000000..7f39ab5 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/pipeline_output.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +from typing import List, Optional, Union + +import numpy as np +import PIL.Image + +from ...utils import BaseOutput + + +@dataclass +class IFPipelineOutput(BaseOutput): + """ + Args: + Output class for Stable Diffusion pipelines. + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. + nsfw_detected (`List[bool]`) + List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content or a watermark. `None` if safety checking could not be performed. + watermark_detected (`List[bool]`) + List of flags denoting whether the corresponding generated image likely has a watermark. `None` if safety + checking could not be performed. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + nsfw_detected: Optional[List[bool]] + watermark_detected: Optional[List[bool]] diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/safety_checker.py b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/safety_checker.py new file mode 100755 index 0000000..8ffeed5 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/safety_checker.py @@ -0,0 +1,59 @@ +import numpy as np +import torch +import torch.nn as nn +from transformers import CLIPConfig, CLIPVisionModelWithProjection, PreTrainedModel + +from ...utils import logging + + +logger = logging.get_logger(__name__) + + +class IFSafetyChecker(PreTrainedModel): + config_class = CLIPConfig + + _no_split_modules = ["CLIPEncoderLayer"] + + def __init__(self, config: CLIPConfig): + super().__init__(config) + + self.vision_model = CLIPVisionModelWithProjection(config.vision_config) + + self.p_head = nn.Linear(config.vision_config.projection_dim, 1) + self.w_head = nn.Linear(config.vision_config.projection_dim, 1) + + @torch.no_grad() + def forward(self, clip_input, images, p_threshold=0.5, w_threshold=0.5): + image_embeds = self.vision_model(clip_input)[0] + + nsfw_detected = self.p_head(image_embeds) + nsfw_detected = nsfw_detected.flatten() + nsfw_detected = nsfw_detected > p_threshold + nsfw_detected = nsfw_detected.tolist() + + if any(nsfw_detected): + logger.warning( + "Potential NSFW content was detected in one or more images. A black image will be returned instead." + " Try again with a different prompt and/or seed." + ) + + for idx, nsfw_detected_ in enumerate(nsfw_detected): + if nsfw_detected_: + images[idx] = np.zeros(images[idx].shape) + + watermark_detected = self.w_head(image_embeds) + watermark_detected = watermark_detected.flatten() + watermark_detected = watermark_detected > w_threshold + watermark_detected = watermark_detected.tolist() + + if any(watermark_detected): + logger.warning( + "Potential watermarked content was detected in one or more images. A black image will be returned instead." + " Try again with a different prompt and/or seed." + ) + + for idx, watermark_detected_ in enumerate(watermark_detected): + if watermark_detected_: + images[idx] = np.zeros(images[idx].shape) + + return images, nsfw_detected, watermark_detected diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/timesteps.py b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/timesteps.py new file mode 100755 index 0000000..d44285c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/timesteps.py @@ -0,0 +1,579 @@ +fast27_timesteps = [ + 999, + 800, + 799, + 600, + 599, + 500, + 400, + 399, + 377, + 355, + 333, + 311, + 288, + 266, + 244, + 222, + 200, + 199, + 177, + 155, + 133, + 111, + 88, + 66, + 44, + 22, + 0, +] + +smart27_timesteps = [ + 999, + 976, + 952, + 928, + 905, + 882, + 858, + 857, + 810, + 762, + 715, + 714, + 572, + 429, + 428, + 286, + 285, + 238, + 190, + 143, + 142, + 118, + 95, + 71, + 47, + 24, + 0, +] + +smart50_timesteps = [ + 999, + 988, + 977, + 966, + 955, + 944, + 933, + 922, + 911, + 900, + 899, + 879, + 859, + 840, + 820, + 800, + 799, + 766, + 733, + 700, + 699, + 650, + 600, + 599, + 500, + 499, + 400, + 399, + 350, + 300, + 299, + 266, + 233, + 200, + 199, + 179, + 159, + 140, + 120, + 100, + 99, + 88, + 77, + 66, + 55, + 44, + 33, + 22, + 11, + 0, +] + +smart100_timesteps = [ + 999, + 995, + 992, + 989, + 985, + 981, + 978, + 975, + 971, + 967, + 964, + 961, + 957, + 956, + 951, + 947, + 942, + 937, + 933, + 928, + 923, + 919, + 914, + 913, + 908, + 903, + 897, + 892, + 887, + 881, + 876, + 871, + 870, + 864, + 858, + 852, + 846, + 840, + 834, + 828, + 827, + 820, + 813, + 806, + 799, + 792, + 785, + 784, + 777, + 770, + 763, + 756, + 749, + 742, + 741, + 733, + 724, + 716, + 707, + 699, + 698, + 688, + 677, + 666, + 656, + 655, + 645, + 634, + 623, + 613, + 612, + 598, + 584, + 570, + 569, + 555, + 541, + 527, + 526, + 505, + 484, + 483, + 462, + 440, + 439, + 396, + 395, + 352, + 351, + 308, + 307, + 264, + 263, + 220, + 219, + 176, + 132, + 88, + 44, + 0, +] + +smart185_timesteps = [ + 999, + 997, + 995, + 992, + 990, + 988, + 986, + 984, + 981, + 979, + 977, + 975, + 972, + 970, + 968, + 966, + 964, + 961, + 959, + 957, + 956, + 954, + 951, + 949, + 946, + 944, + 941, + 939, + 936, + 934, + 931, + 929, + 926, + 924, + 921, + 919, + 916, + 914, + 913, + 910, + 907, + 905, + 902, + 899, + 896, + 893, + 891, + 888, + 885, + 882, + 879, + 877, + 874, + 871, + 870, + 867, + 864, + 861, + 858, + 855, + 852, + 849, + 846, + 843, + 840, + 837, + 834, + 831, + 828, + 827, + 824, + 821, + 817, + 814, + 811, + 808, + 804, + 801, + 798, + 795, + 791, + 788, + 785, + 784, + 780, + 777, + 774, + 770, + 766, + 763, + 760, + 756, + 752, + 749, + 746, + 742, + 741, + 737, + 733, + 730, + 726, + 722, + 718, + 714, + 710, + 707, + 703, + 699, + 698, + 694, + 690, + 685, + 681, + 677, + 673, + 669, + 664, + 660, + 656, + 655, + 650, + 646, + 641, + 636, + 632, + 627, + 622, + 618, + 613, + 612, + 607, + 602, + 596, + 591, + 586, + 580, + 575, + 570, + 569, + 563, + 557, + 551, + 545, + 539, + 533, + 527, + 526, + 519, + 512, + 505, + 498, + 491, + 484, + 483, + 474, + 466, + 457, + 449, + 440, + 439, + 428, + 418, + 407, + 396, + 395, + 381, + 366, + 352, + 351, + 330, + 308, + 307, + 286, + 264, + 263, + 242, + 220, + 219, + 176, + 175, + 132, + 131, + 88, + 44, + 0, +] + +super27_timesteps = [ + 999, + 991, + 982, + 974, + 966, + 958, + 950, + 941, + 933, + 925, + 916, + 908, + 900, + 899, + 874, + 850, + 825, + 800, + 799, + 700, + 600, + 500, + 400, + 300, + 200, + 100, + 0, +] + +super40_timesteps = [ + 999, + 992, + 985, + 978, + 971, + 964, + 957, + 949, + 942, + 935, + 928, + 921, + 914, + 907, + 900, + 899, + 879, + 859, + 840, + 820, + 800, + 799, + 766, + 733, + 700, + 699, + 650, + 600, + 599, + 500, + 499, + 400, + 399, + 300, + 299, + 200, + 199, + 100, + 99, + 0, +] + +super100_timesteps = [ + 999, + 996, + 992, + 989, + 985, + 982, + 979, + 975, + 972, + 968, + 965, + 961, + 958, + 955, + 951, + 948, + 944, + 941, + 938, + 934, + 931, + 927, + 924, + 920, + 917, + 914, + 910, + 907, + 903, + 900, + 899, + 891, + 884, + 876, + 869, + 861, + 853, + 846, + 838, + 830, + 823, + 815, + 808, + 800, + 799, + 788, + 777, + 766, + 755, + 744, + 733, + 722, + 711, + 700, + 699, + 688, + 677, + 666, + 655, + 644, + 633, + 622, + 611, + 600, + 599, + 585, + 571, + 557, + 542, + 528, + 514, + 500, + 499, + 485, + 471, + 457, + 442, + 428, + 414, + 400, + 399, + 379, + 359, + 340, + 320, + 300, + 299, + 279, + 259, + 240, + 220, + 200, + 199, + 166, + 133, + 100, + 99, + 66, + 33, + 0, +] diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/watermark.py b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/watermark.py new file mode 100755 index 0000000..ca10413 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deepfloyd_if/watermark.py @@ -0,0 +1,46 @@ +from typing import List + +import PIL.Image +import torch +from PIL import Image + +from ...configuration_utils import ConfigMixin +from ...models.modeling_utils import ModelMixin +from ...utils import PIL_INTERPOLATION + + +class IFWatermarker(ModelMixin, ConfigMixin): + def __init__(self): + super().__init__() + + self.register_buffer("watermark_image", torch.zeros((62, 62, 4))) + self.watermark_image_as_pil = None + + def apply_watermark(self, images: List[PIL.Image.Image], sample_size=None): + # copied from https://github.com/deep-floyd/IF/blob/b77482e36ca2031cb94dbca1001fc1e6400bf4ab/deepfloyd_if/modules/base.py#L287 + + h = images[0].height + w = images[0].width + + sample_size = sample_size or h + + coef = min(h / sample_size, w / sample_size) + img_h, img_w = (int(h / coef), int(w / coef)) if coef < 1 else (h, w) + + S1, S2 = 1024**2, img_w * img_h + K = (S2 / S1) ** 0.5 + wm_size, wm_x, wm_y = int(K * 62), img_w - int(14 * K), img_h - int(14 * K) + + if self.watermark_image_as_pil is None: + watermark_image = self.watermark_image.to(torch.uint8).cpu().numpy() + watermark_image = Image.fromarray(watermark_image, mode="RGBA") + self.watermark_image_as_pil = watermark_image + + wm_img = self.watermark_image_as_pil.resize( + (wm_size, wm_size), PIL_INTERPOLATION["bicubic"], reducing_gap=None + ) + + for pil_img in images: + pil_img.paste(wm_img, box=(wm_x - wm_size, wm_y - wm_size, wm_x, wm_y), mask=wm_img.split()[-1]) + + return images diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/README.md b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/README.md new file mode 100755 index 0000000..1e21dbb --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/README.md @@ -0,0 +1,3 @@ +# Deprecated Pipelines + +This folder contains pipelines that have very low usage as measured by model downloads, issues and PRs. While you can still use the pipelines just as before, we will stop testing the pipelines and will not accept any changes to existing files. \ No newline at end of file diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/__init__.py new file mode 100755 index 0000000..9936323 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/__init__.py @@ -0,0 +1,153 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_librosa_available, + is_note_seq_available, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_pt_objects + + _dummy_objects.update(get_objects_from_module(dummy_pt_objects)) +else: + _import_structure["latent_diffusion_uncond"] = ["LDMPipeline"] + _import_structure["pndm"] = ["PNDMPipeline"] + _import_structure["repaint"] = ["RePaintPipeline"] + _import_structure["score_sde_ve"] = ["ScoreSdeVePipeline"] + _import_structure["stochastic_karras_ve"] = ["KarrasVePipeline"] + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["alt_diffusion"] = [ + "AltDiffusionImg2ImgPipeline", + "AltDiffusionPipeline", + "AltDiffusionPipelineOutput", + ] + _import_structure["versatile_diffusion"] = [ + "VersatileDiffusionDualGuidedPipeline", + "VersatileDiffusionImageVariationPipeline", + "VersatileDiffusionPipeline", + "VersatileDiffusionTextToImagePipeline", + ] + _import_structure["vq_diffusion"] = ["VQDiffusionPipeline"] + _import_structure["stable_diffusion_variants"] = [ + "CycleDiffusionPipeline", + "StableDiffusionInpaintPipelineLegacy", + "StableDiffusionPix2PixZeroPipeline", + "StableDiffusionParadigmsPipeline", + "StableDiffusionModelEditingPipeline", + ] + +try: + if not (is_torch_available() and is_librosa_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_librosa_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_librosa_objects)) + +else: + _import_structure["audio_diffusion"] = ["AudioDiffusionPipeline", "Mel"] + +try: + if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_transformers_and_torch_and_note_seq_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_transformers_and_torch_and_note_seq_objects)) + +else: + _import_structure["spectrogram_diffusion"] = ["MidiProcessor", "SpectrogramDiffusionPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_pt_objects import * + + else: + from .latent_diffusion_uncond import LDMPipeline + from .pndm import PNDMPipeline + from .repaint import RePaintPipeline + from .score_sde_ve import ScoreSdeVePipeline + from .stochastic_karras_ve import KarrasVePipeline + + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + + else: + from .alt_diffusion import AltDiffusionImg2ImgPipeline, AltDiffusionPipeline, AltDiffusionPipelineOutput + from .audio_diffusion import AudioDiffusionPipeline, Mel + from .spectrogram_diffusion import SpectrogramDiffusionPipeline + from .stable_diffusion_variants import ( + CycleDiffusionPipeline, + StableDiffusionInpaintPipelineLegacy, + StableDiffusionModelEditingPipeline, + StableDiffusionParadigmsPipeline, + StableDiffusionPix2PixZeroPipeline, + ) + from .stochastic_karras_ve import KarrasVePipeline + from .versatile_diffusion import ( + VersatileDiffusionDualGuidedPipeline, + VersatileDiffusionImageVariationPipeline, + VersatileDiffusionPipeline, + VersatileDiffusionTextToImagePipeline, + ) + from .vq_diffusion import VQDiffusionPipeline + + try: + if not (is_torch_available() and is_librosa_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_librosa_objects import * + else: + from .audio_diffusion import AudioDiffusionPipeline, Mel + + try: + if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_transformers_and_torch_and_note_seq_objects import * # noqa F403 + else: + from .spectrogram_diffusion import ( + MidiProcessor, + SpectrogramDiffusionPipeline, + ) + + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/__init__.py new file mode 100755 index 0000000..71fa15b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/__init__.py @@ -0,0 +1,53 @@ +from typing import TYPE_CHECKING + +from ....utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ....utils import dummy_torch_and_transformers_objects + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["modeling_roberta_series"] = ["RobertaSeriesModelWithTransformation"] + _import_structure["pipeline_alt_diffusion"] = ["AltDiffusionPipeline"] + _import_structure["pipeline_alt_diffusion_img2img"] = ["AltDiffusionImg2ImgPipeline"] + + _import_structure["pipeline_output"] = ["AltDiffusionPipelineOutput"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ....utils.dummy_torch_and_transformers_objects import * + + else: + from .modeling_roberta_series import RobertaSeriesModelWithTransformation + from .pipeline_alt_diffusion import AltDiffusionPipeline + from .pipeline_alt_diffusion_img2img import AltDiffusionImg2ImgPipeline + from .pipeline_output import AltDiffusionPipelineOutput + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/modeling_roberta_series.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/modeling_roberta_series.py new file mode 100755 index 0000000..f73ef15 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/modeling_roberta_series.py @@ -0,0 +1,124 @@ +from dataclasses import dataclass +from typing import Optional, Tuple + +import torch +from torch import nn +from transformers import RobertaPreTrainedModel, XLMRobertaConfig, XLMRobertaModel +from transformers.utils import ModelOutput + + +@dataclass +class TransformationModelOutput(ModelOutput): + """ + Base class for text model's outputs that also contains a pooling of the last hidden states. + + Args: + text_embeds (`torch.FloatTensor` of shape `(batch_size, output_dim)` *optional* returned when model is initialized with `with_projection=True`): + The text embeddings obtained by applying the projection layer to the pooler_output. + last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): + Sequence of hidden-states at the output of the last layer of the model. + hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): + Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. + + Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. + attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): + Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, + sequence_length)`. + + Attentions weights after the attention softmax, used to compute the weighted average in the self-attention + heads. + """ + + projection_state: Optional[torch.FloatTensor] = None + last_hidden_state: torch.FloatTensor = None + hidden_states: Optional[Tuple[torch.FloatTensor]] = None + attentions: Optional[Tuple[torch.FloatTensor]] = None + + +class RobertaSeriesConfig(XLMRobertaConfig): + def __init__( + self, + pad_token_id=1, + bos_token_id=0, + eos_token_id=2, + project_dim=512, + pooler_fn="cls", + learn_encoder=False, + use_attention_mask=True, + **kwargs, + ): + super().__init__(pad_token_id=pad_token_id, bos_token_id=bos_token_id, eos_token_id=eos_token_id, **kwargs) + self.project_dim = project_dim + self.pooler_fn = pooler_fn + self.learn_encoder = learn_encoder + self.use_attention_mask = use_attention_mask + + +class RobertaSeriesModelWithTransformation(RobertaPreTrainedModel): + _keys_to_ignore_on_load_unexpected = [r"pooler", r"logit_scale"] + _keys_to_ignore_on_load_missing = [r"position_ids", r"predictions.decoder.bias"] + base_model_prefix = "roberta" + config_class = RobertaSeriesConfig + + def __init__(self, config): + super().__init__(config) + self.roberta = XLMRobertaModel(config) + self.transformation = nn.Linear(config.hidden_size, config.project_dim) + self.has_pre_transformation = getattr(config, "has_pre_transformation", False) + if self.has_pre_transformation: + self.transformation_pre = nn.Linear(config.hidden_size, config.project_dim) + self.pre_LN = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps) + self.post_init() + + def forward( + self, + input_ids: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + token_type_ids: Optional[torch.Tensor] = None, + position_ids: Optional[torch.Tensor] = None, + head_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.Tensor] = None, + encoder_hidden_states: Optional[torch.Tensor] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + output_attentions: Optional[bool] = None, + return_dict: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + ): + r""" """ + + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + outputs = self.base_model( + input_ids=input_ids, + attention_mask=attention_mask, + token_type_ids=token_type_ids, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + encoder_hidden_states=encoder_hidden_states, + encoder_attention_mask=encoder_attention_mask, + output_attentions=output_attentions, + output_hidden_states=True if self.has_pre_transformation else output_hidden_states, + return_dict=return_dict, + ) + + if self.has_pre_transformation: + sequence_output2 = outputs["hidden_states"][-2] + sequence_output2 = self.pre_LN(sequence_output2) + projection_state2 = self.transformation_pre(sequence_output2) + + return TransformationModelOutput( + projection_state=projection_state2, + last_hidden_state=outputs.last_hidden_state, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) + else: + projection_state = self.transformation(outputs.last_hidden_state) + return TransformationModelOutput( + projection_state=projection_state, + last_hidden_state=outputs.last_hidden_state, + hidden_states=outputs.hidden_states, + attentions=outputs.attentions, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py new file mode 100755 index 0000000..e458369 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion.py @@ -0,0 +1,946 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from packaging import version +from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection, XLMRobertaTokenizer + +from ....configuration_utils import FrozenDict +from ....image_processor import PipelineImageInput, VaeImageProcessor +from ....loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ....models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ....models.lora import adjust_lora_scale_text_encoder +from ....schedulers import KarrasDiffusionSchedulers +from ....utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ...stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from .modeling_roberta_series import RobertaSeriesModelWithTransformation +from .pipeline_output import AltDiffusionPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import AltDiffusionPipeline + + >>> pipe = AltDiffusionPipeline.from_pretrained("BAAI/AltDiffusion-m9", torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + + >>> # "dark elf princess, highly detailed, d & d, fantasy, highly detailed, digital painting, trending on artstation, concept art, sharp focus, illustration, art by artgerm and greg rutkowski and fuji choko and viktoria gavrilenko and hoang lap" + >>> prompt = "黑暗精灵公主，非常详细，幻想，非常详细，数字绘画，概念艺术，敏锐的焦点，插图" + >>> image = pipe(prompt).images[0] + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class AltDiffusionPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + LoraLoaderMixin, + IPAdapterMixin, + FromSingleFileMixin, +): + r""" + Pipeline for text-to-image generation using Alt Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.RobertaSeriesModelWithTransformation`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.XLMRobertaTokenizer`]): + A `XLMRobertaTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: RobertaSeriesModelWithTransformation, + tokenizer: XLMRobertaTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: CLIPVisionModelWithProjection = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Alt Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + timesteps: List[int] = None, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.AltDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.0): + Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf). Guidance rescale factor should fix overexposure when + using zero terminal SNR. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.AltDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.AltDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + # to deal with lora scaling and other possible forward hooks + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._guidance_rescale = guidance_rescale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + clip_skip=self.clip_skip, + ) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None: + output_hidden_state = False if isinstance(self.unet.encoder_hid_proj, ImageProjection) else True + image_embeds, negative_image_embeds = self.encode_image( + ip_adapter_image, device, num_images_per_prompt, output_hidden_state + ) + if self.do_classifier_free_guidance: + image_embeds = torch.cat([negative_image_embeds, image_embeds]) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 6.1 Add image embeds for IP-Adapter + added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None + + # 6.2 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ + 0 + ] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return AltDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py new file mode 100755 index 0000000..156e52c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_alt_diffusion_img2img.py @@ -0,0 +1,1018 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from packaging import version +from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection, XLMRobertaTokenizer + +from ....configuration_utils import FrozenDict +from ....image_processor import PipelineImageInput, VaeImageProcessor +from ....loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ....models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ....models.lora import adjust_lora_scale_text_encoder +from ....schedulers import KarrasDiffusionSchedulers +from ....utils import ( + PIL_INTERPOLATION, + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ...stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from .modeling_roberta_series import RobertaSeriesModelWithTransformation +from .pipeline_output import AltDiffusionPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import requests + >>> import torch + >>> from PIL import Image + >>> from io import BytesIO + + >>> from diffusers import AltDiffusionImg2ImgPipeline + + >>> device = "cuda" + >>> model_id_or_path = "BAAI/AltDiffusion-m9" + >>> pipe = AltDiffusionImg2ImgPipeline.from_pretrained(model_id_or_path, torch_dtype=torch.float16) + >>> pipe = pipe.to(device) + + >>> url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + + >>> response = requests.get(url) + >>> init_image = Image.open(BytesIO(response.content)).convert("RGB") + >>> init_image = init_image.resize((768, 512)) + + >>> # "A fantasy landscape, trending on artstation" + >>> prompt = "幻想风景, artstation" + + >>> images = pipe(prompt=prompt, image=init_image, strength=0.75, guidance_scale=7.5).images + >>> images[0].save("幻想风景.png") + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.preprocess +def preprocess(image): + deprecation_message = "The preprocess method is deprecated and will be removed in diffusers 1.0.0. Please use VaeImageProcessor.preprocess(...) instead" + deprecate("preprocess", "1.0.0", deprecation_message, standard_warn=False) + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 8 for x in (w, h)) # resize to integer multiple of 8 + + image = [np.array(i.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + return image + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class AltDiffusionImg2ImgPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, + LoraLoaderMixin, + FromSingleFileMixin, +): + r""" + Pipeline for text-guided image-to-image generation using Alt Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.RobertaSeriesModelWithTransformation`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.XLMRobertaTokenizer`]): + A `XLMRobertaTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: RobertaSeriesModelWithTransformation, + tokenizer: XLMRobertaTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: CLIPVisionModelWithProjection = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Alt Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + strength, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + + return timesteps, num_inference_steps - t_start + + def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + init_latents = image + + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + elif isinstance(generator, list): + init_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(batch_size) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + init_latents = self.vae.config.scaling_factor * init_latents + + if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0: + # expand init_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {init_latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_image_per_prompt = batch_size // init_latents.shape[0] + init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0) + elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts." + ) + else: + init_latents = torch.cat([init_latents], dim=0) + + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + latents = init_latents + + return latents + + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + strength: float = 0.8, + num_inference_steps: Optional[int] = 50, + timesteps: List[int] = None, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: int = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + strength (`float`, *optional*, defaults to 0.8): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.AltDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + Examples: + + Returns: + [`~pipelines.stable_diffusion.AltDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.AltDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + strength, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None: + output_hidden_state = False if isinstance(self.unet.encoder_hid_proj, ImageProjection) else True + image_embeds, negative_image_embeds = self.encode_image( + ip_adapter_image, device, num_images_per_prompt, output_hidden_state + ) + if self.do_classifier_free_guidance: + image_embeds = torch.cat([negative_image_embeds, image_embeds]) + + # 4. Preprocess image + image = self.image_processor.preprocess(image) + + # 5. set timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + + # 6. Prepare latent variables + latents = self.prepare_latents( + image, + latent_timestep, + batch_size, + num_images_per_prompt, + prompt_embeds.dtype, + device, + generator, + ) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7.1 Add image embeds for IP-Adapter + added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None + + # 7.2 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ + 0 + ] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return AltDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_output.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_output.py new file mode 100755 index 0000000..dd174ae --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/alt_diffusion/pipeline_output.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +from typing import List, Optional, Union + +import numpy as np +import PIL.Image + +from ....utils import ( + BaseOutput, +) + + +@dataclass +# Copied from diffusers.pipelines.stable_diffusion.pipeline_output.StableDiffusionPipelineOutput with Stable->Alt +class AltDiffusionPipelineOutput(BaseOutput): + """ + Output class for Alt Diffusion pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + num_channels)`. + nsfw_content_detected (`List[bool]`) + List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or + `None` if safety checking could not be performed. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + nsfw_content_detected: Optional[List[bool]] diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/__init__.py new file mode 100755 index 0000000..3127951 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/__init__.py @@ -0,0 +1,23 @@ +from typing import TYPE_CHECKING + +from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule + + +_import_structure = { + "mel": ["Mel"], + "pipeline_audio_diffusion": ["AudioDiffusionPipeline"], +} + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .mel import Mel + from .pipeline_audio_diffusion import AudioDiffusionPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/mel.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/mel.py new file mode 100755 index 0000000..3426c3a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/mel.py @@ -0,0 +1,179 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import numpy as np # noqa: E402 + +from ....configuration_utils import ConfigMixin, register_to_config +from ....schedulers.scheduling_utils import SchedulerMixin + + +try: + import librosa # noqa: E402 + + _librosa_can_be_imported = True + _import_error = "" +except Exception as e: + _librosa_can_be_imported = False + _import_error = ( + f"Cannot import librosa because {e}. Make sure to correctly install librosa to be able to install it." + ) + + +from PIL import Image # noqa: E402 + + +class Mel(ConfigMixin, SchedulerMixin): + """ + Parameters: + x_res (`int`): + x resolution of spectrogram (time). + y_res (`int`): + y resolution of spectrogram (frequency bins). + sample_rate (`int`): + Sample rate of audio. + n_fft (`int`): + Number of Fast Fourier Transforms. + hop_length (`int`): + Hop length (a higher number is recommended if `y_res` < 256). + top_db (`int`): + Loudest decibel value. + n_iter (`int`): + Number of iterations for Griffin-Lim Mel inversion. + """ + + config_name = "mel_config.json" + + @register_to_config + def __init__( + self, + x_res: int = 256, + y_res: int = 256, + sample_rate: int = 22050, + n_fft: int = 2048, + hop_length: int = 512, + top_db: int = 80, + n_iter: int = 32, + ): + self.hop_length = hop_length + self.sr = sample_rate + self.n_fft = n_fft + self.top_db = top_db + self.n_iter = n_iter + self.set_resolution(x_res, y_res) + self.audio = None + + if not _librosa_can_be_imported: + raise ValueError(_import_error) + + def set_resolution(self, x_res: int, y_res: int): + """Set resolution. + + Args: + x_res (`int`): + x resolution of spectrogram (time). + y_res (`int`): + y resolution of spectrogram (frequency bins). + """ + self.x_res = x_res + self.y_res = y_res + self.n_mels = self.y_res + self.slice_size = self.x_res * self.hop_length - 1 + + def load_audio(self, audio_file: str = None, raw_audio: np.ndarray = None): + """Load audio. + + Args: + audio_file (`str`): + An audio file that must be on disk due to [Librosa](https://librosa.org/) limitation. + raw_audio (`np.ndarray`): + The raw audio file as a NumPy array. + """ + if audio_file is not None: + self.audio, _ = librosa.load(audio_file, mono=True, sr=self.sr) + else: + self.audio = raw_audio + + # Pad with silence if necessary. + if len(self.audio) < self.x_res * self.hop_length: + self.audio = np.concatenate([self.audio, np.zeros((self.x_res * self.hop_length - len(self.audio),))]) + + def get_number_of_slices(self) -> int: + """Get number of slices in audio. + + Returns: + `int`: + Number of spectograms audio can be sliced into. + """ + return len(self.audio) // self.slice_size + + def get_audio_slice(self, slice: int = 0) -> np.ndarray: + """Get slice of audio. + + Args: + slice (`int`): + Slice number of audio (out of `get_number_of_slices()`). + + Returns: + `np.ndarray`: + The audio slice as a NumPy array. + """ + return self.audio[self.slice_size * slice : self.slice_size * (slice + 1)] + + def get_sample_rate(self) -> int: + """Get sample rate. + + Returns: + `int`: + Sample rate of audio. + """ + return self.sr + + def audio_slice_to_image(self, slice: int) -> Image.Image: + """Convert slice of audio to spectrogram. + + Args: + slice (`int`): + Slice number of audio to convert (out of `get_number_of_slices()`). + + Returns: + `PIL Image`: + A grayscale image of `x_res x y_res`. + """ + S = librosa.feature.melspectrogram( + y=self.get_audio_slice(slice), sr=self.sr, n_fft=self.n_fft, hop_length=self.hop_length, n_mels=self.n_mels + ) + log_S = librosa.power_to_db(S, ref=np.max, top_db=self.top_db) + bytedata = (((log_S + self.top_db) * 255 / self.top_db).clip(0, 255) + 0.5).astype(np.uint8) + image = Image.fromarray(bytedata) + return image + + def image_to_audio(self, image: Image.Image) -> np.ndarray: + """Converts spectrogram to audio. + + Args: + image (`PIL Image`): + An grayscale image of `x_res x y_res`. + + Returns: + audio (`np.ndarray`): + The audio as a NumPy array. + """ + bytedata = np.frombuffer(image.tobytes(), dtype="uint8").reshape((image.height, image.width)) + log_S = bytedata.astype("float") * self.top_db / 255 - self.top_db + S = librosa.db_to_power(log_S) + audio = librosa.feature.inverse.mel_to_audio( + S, sr=self.sr, n_fft=self.n_fft, hop_length=self.hop_length, n_iter=self.n_iter + ) + return audio diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/pipeline_audio_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/pipeline_audio_diffusion.py new file mode 100755 index 0000000..47044e0 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/audio_diffusion/pipeline_audio_diffusion.py @@ -0,0 +1,329 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from math import acos, sin +from typing import List, Tuple, Union + +import numpy as np +import torch +from PIL import Image + +from ....models import AutoencoderKL, UNet2DConditionModel +from ....schedulers import DDIMScheduler, DDPMScheduler +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import AudioPipelineOutput, BaseOutput, DiffusionPipeline, ImagePipelineOutput +from .mel import Mel + + +class AudioDiffusionPipeline(DiffusionPipeline): + """ + Pipeline for audio diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + vqae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + mel ([`Mel`]): + Transform audio into a spectrogram. + scheduler ([`DDIMScheduler`] or [`DDPMScheduler`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`] or [`DDPMScheduler`]. + """ + + _optional_components = ["vqvae"] + + def __init__( + self, + vqvae: AutoencoderKL, + unet: UNet2DConditionModel, + mel: Mel, + scheduler: Union[DDIMScheduler, DDPMScheduler], + ): + super().__init__() + self.register_modules(unet=unet, scheduler=scheduler, mel=mel, vqvae=vqvae) + + def get_default_steps(self) -> int: + """Returns default number of steps recommended for inference. + + Returns: + `int`: + The number of steps. + """ + return 50 if isinstance(self.scheduler, DDIMScheduler) else 1000 + + @torch.no_grad() + def __call__( + self, + batch_size: int = 1, + audio_file: str = None, + raw_audio: np.ndarray = None, + slice: int = 0, + start_step: int = 0, + steps: int = None, + generator: torch.Generator = None, + mask_start_secs: float = 0, + mask_end_secs: float = 0, + step_generator: torch.Generator = None, + eta: float = 0, + noise: torch.Tensor = None, + encoding: torch.Tensor = None, + return_dict=True, + ) -> Union[ + Union[AudioPipelineOutput, ImagePipelineOutput], + Tuple[List[Image.Image], Tuple[int, List[np.ndarray]]], + ]: + """ + The call function to the pipeline for generation. + + Args: + batch_size (`int`): + Number of samples to generate. + audio_file (`str`): + An audio file that must be on disk due to [Librosa](https://librosa.org/) limitation. + raw_audio (`np.ndarray`): + The raw audio file as a NumPy array. + slice (`int`): + Slice number of audio to convert. + start_step (int): + Step to start diffusion from. + steps (`int`): + Number of denoising steps (defaults to `50` for DDIM and `1000` for DDPM). + generator (`torch.Generator`): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + mask_start_secs (`float`): + Number of seconds of audio to mask (not generate) at start. + mask_end_secs (`float`): + Number of seconds of audio to mask (not generate) at end. + step_generator (`torch.Generator`): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) used to denoise. + None + eta (`float`): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + noise (`torch.Tensor`): + A noise tensor of shape `(batch_size, 1, height, width)` or `None`. + encoding (`torch.Tensor`): + A tensor for [`UNet2DConditionModel`] of shape `(batch_size, seq_length, cross_attention_dim)`. + return_dict (`bool`): + Whether or not to return a [`AudioPipelineOutput`], [`ImagePipelineOutput`] or a plain tuple. + + Examples: + + For audio diffusion: + + ```py + import torch + from IPython.display import Audio + from diffusers import DiffusionPipeline + + device = "cuda" if torch.cuda.is_available() else "cpu" + pipe = DiffusionPipeline.from_pretrained("teticio/audio-diffusion-256").to(device) + + output = pipe() + display(output.images[0]) + display(Audio(output.audios[0], rate=mel.get_sample_rate())) + ``` + + For latent audio diffusion: + + ```py + import torch + from IPython.display import Audio + from diffusers import DiffusionPipeline + + device = "cuda" if torch.cuda.is_available() else "cpu" + pipe = DiffusionPipeline.from_pretrained("teticio/latent-audio-diffusion-256").to(device) + + output = pipe() + display(output.images[0]) + display(Audio(output.audios[0], rate=pipe.mel.get_sample_rate())) + ``` + + For other tasks like variation, inpainting, outpainting, etc: + + ```py + output = pipe( + raw_audio=output.audios[0, 0], + start_step=int(pipe.get_default_steps() / 2), + mask_start_secs=1, + mask_end_secs=1, + ) + display(output.images[0]) + display(Audio(output.audios[0], rate=pipe.mel.get_sample_rate())) + ``` + + Returns: + `List[PIL Image]`: + A list of Mel spectrograms (`float`, `List[np.ndarray]`) with the sample rate and raw audio. + """ + + steps = steps or self.get_default_steps() + self.scheduler.set_timesteps(steps) + step_generator = step_generator or generator + # For backwards compatibility + if isinstance(self.unet.config.sample_size, int): + self.unet.config.sample_size = (self.unet.config.sample_size, self.unet.config.sample_size) + if noise is None: + noise = randn_tensor( + ( + batch_size, + self.unet.config.in_channels, + self.unet.config.sample_size[0], + self.unet.config.sample_size[1], + ), + generator=generator, + device=self.device, + ) + images = noise + mask = None + + if audio_file is not None or raw_audio is not None: + self.mel.load_audio(audio_file, raw_audio) + input_image = self.mel.audio_slice_to_image(slice) + input_image = np.frombuffer(input_image.tobytes(), dtype="uint8").reshape( + (input_image.height, input_image.width) + ) + input_image = (input_image / 255) * 2 - 1 + input_images = torch.tensor(input_image[np.newaxis, :, :], dtype=torch.float).to(self.device) + + if self.vqvae is not None: + input_images = self.vqvae.encode(torch.unsqueeze(input_images, 0)).latent_dist.sample( + generator=generator + )[0] + input_images = self.vqvae.config.scaling_factor * input_images + + if start_step > 0: + images[0, 0] = self.scheduler.add_noise(input_images, noise, self.scheduler.timesteps[start_step - 1]) + + pixels_per_second = ( + self.unet.config.sample_size[1] * self.mel.get_sample_rate() / self.mel.x_res / self.mel.hop_length + ) + mask_start = int(mask_start_secs * pixels_per_second) + mask_end = int(mask_end_secs * pixels_per_second) + mask = self.scheduler.add_noise(input_images, noise, torch.tensor(self.scheduler.timesteps[start_step:])) + + for step, t in enumerate(self.progress_bar(self.scheduler.timesteps[start_step:])): + if isinstance(self.unet, UNet2DConditionModel): + model_output = self.unet(images, t, encoding)["sample"] + else: + model_output = self.unet(images, t)["sample"] + + if isinstance(self.scheduler, DDIMScheduler): + images = self.scheduler.step( + model_output=model_output, + timestep=t, + sample=images, + eta=eta, + generator=step_generator, + )["prev_sample"] + else: + images = self.scheduler.step( + model_output=model_output, + timestep=t, + sample=images, + generator=step_generator, + )["prev_sample"] + + if mask is not None: + if mask_start > 0: + images[:, :, :, :mask_start] = mask[:, step, :, :mask_start] + if mask_end > 0: + images[:, :, :, -mask_end:] = mask[:, step, :, -mask_end:] + + if self.vqvae is not None: + # 0.18215 was scaling factor used in training to ensure unit variance + images = 1 / self.vqvae.config.scaling_factor * images + images = self.vqvae.decode(images)["sample"] + + images = (images / 2 + 0.5).clamp(0, 1) + images = images.cpu().permute(0, 2, 3, 1).numpy() + images = (images * 255).round().astype("uint8") + images = list( + (Image.fromarray(_[:, :, 0]) for _ in images) + if images.shape[3] == 1 + else (Image.fromarray(_, mode="RGB").convert("L") for _ in images) + ) + + audios = [self.mel.image_to_audio(_) for _ in images] + if not return_dict: + return images, (self.mel.get_sample_rate(), audios) + + return BaseOutput(**AudioPipelineOutput(np.array(audios)[:, np.newaxis, :]), **ImagePipelineOutput(images)) + + @torch.no_grad() + def encode(self, images: List[Image.Image], steps: int = 50) -> np.ndarray: + """ + Reverse the denoising step process to recover a noisy image from the generated image. + + Args: + images (`List[PIL Image]`): + List of images to encode. + steps (`int`): + Number of encoding steps to perform (defaults to `50`). + + Returns: + `np.ndarray`: + A noise tensor of shape `(batch_size, 1, height, width)`. + """ + + # Only works with DDIM as this method is deterministic + assert isinstance(self.scheduler, DDIMScheduler) + self.scheduler.set_timesteps(steps) + sample = np.array( + [np.frombuffer(image.tobytes(), dtype="uint8").reshape((1, image.height, image.width)) for image in images] + ) + sample = (sample / 255) * 2 - 1 + sample = torch.Tensor(sample).to(self.device) + + for t in self.progress_bar(torch.flip(self.scheduler.timesteps, (0,))): + prev_timestep = t - self.scheduler.config.num_train_timesteps // self.scheduler.num_inference_steps + alpha_prod_t = self.scheduler.alphas_cumprod[t] + alpha_prod_t_prev = ( + self.scheduler.alphas_cumprod[prev_timestep] + if prev_timestep >= 0 + else self.scheduler.final_alpha_cumprod + ) + beta_prod_t = 1 - alpha_prod_t + model_output = self.unet(sample, t)["sample"] + pred_sample_direction = (1 - alpha_prod_t_prev) ** (0.5) * model_output + sample = (sample - pred_sample_direction) * alpha_prod_t_prev ** (-0.5) + sample = sample * alpha_prod_t ** (0.5) + beta_prod_t ** (0.5) * model_output + + return sample + + @staticmethod + def slerp(x0: torch.Tensor, x1: torch.Tensor, alpha: float) -> torch.Tensor: + """Spherical Linear intERPolation. + + Args: + x0 (`torch.Tensor`): + The first tensor to interpolate between. + x1 (`torch.Tensor`): + Second tensor to interpolate between. + alpha (`float`): + Interpolation between 0 and 1 + + Returns: + `torch.Tensor`: + The interpolated tensor. + """ + + theta = acos(torch.dot(torch.flatten(x0), torch.flatten(x1)) / torch.norm(x0) / torch.norm(x1)) + return sin((1 - alpha) * theta) * x0 / sin(theta) + sin(alpha * theta) * x1 / sin(theta) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/__init__.py new file mode 100755 index 0000000..214f5bb --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/__init__.py @@ -0,0 +1,18 @@ +from typing import TYPE_CHECKING + +from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule + + +_import_structure = {"pipeline_latent_diffusion_uncond": ["LDMPipeline"]} + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .pipeline_latent_diffusion_uncond import LDMPipeline +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py new file mode 100755 index 0000000..7fe5d59 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/latent_diffusion_uncond/pipeline_latent_diffusion_uncond.py @@ -0,0 +1,130 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import List, Optional, Tuple, Union + +import torch + +from ....models import UNet2DModel, VQModel +from ....schedulers import DDIMScheduler +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +class LDMPipeline(DiffusionPipeline): + r""" + Pipeline for unconditional image generation using latent diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + vqvae ([`VQModel`]): + Vector-quantized (VQ) model to encode and decode images to and from latent representations. + unet ([`UNet2DModel`]): + A `UNet2DModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + [`DDIMScheduler`] is used in combination with `unet` to denoise the encoded image latents. + """ + + def __init__(self, vqvae: VQModel, unet: UNet2DModel, scheduler: DDIMScheduler): + super().__init__() + self.register_modules(vqvae=vqvae, unet=unet, scheduler=scheduler) + + @torch.no_grad() + def __call__( + self, + batch_size: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + eta: float = 0.0, + num_inference_steps: int = 50, + output_type: Optional[str] = "pil", + return_dict: bool = True, + **kwargs, + ) -> Union[Tuple, ImagePipelineOutput]: + r""" + The call function to the pipeline for generation. + + Args: + batch_size (`int`, *optional*, defaults to 1): + Number of images to generate. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Example: + + ```py + >>> from diffusers import LDMPipeline + + >>> # load model and scheduler + >>> pipe = LDMPipeline.from_pretrained("CompVis/ldm-celebahq-256") + + >>> # run pipeline in inference (sample random noise and denoise) + >>> image = pipe().images[0] + ``` + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images + """ + + latents = randn_tensor( + (batch_size, self.unet.config.in_channels, self.unet.config.sample_size, self.unet.config.sample_size), + generator=generator, + ) + latents = latents.to(self.device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + + self.scheduler.set_timesteps(num_inference_steps) + + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + + extra_kwargs = {} + if accepts_eta: + extra_kwargs["eta"] = eta + + for t in self.progress_bar(self.scheduler.timesteps): + latent_model_input = self.scheduler.scale_model_input(latents, t) + # predict the noise residual + noise_prediction = self.unet(latent_model_input, t).sample + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_prediction, t, latents, **extra_kwargs).prev_sample + + # adjust latents with inverse of vae scale + latents = latents / self.vqvae.config.scaling_factor + # decode the image latents with the VAE + image = self.vqvae.decode(latents).sample + + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/pndm/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/pndm/__init__.py new file mode 100755 index 0000000..5e3bdba --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/pndm/__init__.py @@ -0,0 +1,18 @@ +from typing import TYPE_CHECKING + +from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule + + +_import_structure = {"pipeline_pndm": ["PNDMPipeline"]} + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .pipeline_pndm import PNDMPipeline +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/pndm/pipeline_pndm.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/pndm/pipeline_pndm.py new file mode 100755 index 0000000..ef78af1 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/pndm/pipeline_pndm.py @@ -0,0 +1,121 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import List, Optional, Tuple, Union + +import torch + +from ....models import UNet2DModel +from ....schedulers import PNDMScheduler +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +class PNDMPipeline(DiffusionPipeline): + r""" + Pipeline for unconditional image generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + unet ([`UNet2DModel`]): + A `UNet2DModel` to denoise the encoded image latents. + scheduler ([`PNDMScheduler`]): + A `PNDMScheduler` to be used in combination with `unet` to denoise the encoded image. + """ + + unet: UNet2DModel + scheduler: PNDMScheduler + + def __init__(self, unet: UNet2DModel, scheduler: PNDMScheduler): + super().__init__() + + scheduler = PNDMScheduler.from_config(scheduler.config) + + self.register_modules(unet=unet, scheduler=scheduler) + + @torch.no_grad() + def __call__( + self, + batch_size: int = 1, + num_inference_steps: int = 50, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + **kwargs, + ) -> Union[ImagePipelineOutput, Tuple]: + r""" + The call function to the pipeline for generation. + + Args: + batch_size (`int`, `optional`, defaults to 1): + The number of images to generate. + num_inference_steps (`int`, `optional`, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator`, `optional`): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + output_type (`str`, `optional`, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`ImagePipelineOutput`] instead of a plain tuple. + + Example: + + ```py + >>> from diffusers import PNDMPipeline + + >>> # load model and scheduler + >>> pndm = PNDMPipeline.from_pretrained("google/ddpm-cifar10-32") + + >>> # run pipeline in inference (sample random noise and denoise) + >>> image = pndm().images[0] + + >>> # save image + >>> image.save("pndm_generated_image.png") + ``` + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images. + """ + # For more information on the sampling method you can take a look at Algorithm 2 of + # the official paper: https://arxiv.org/pdf/2202.09778.pdf + + # Sample gaussian noise to begin loop + image = randn_tensor( + (batch_size, self.unet.config.in_channels, self.unet.config.sample_size, self.unet.config.sample_size), + generator=generator, + device=self.device, + ) + + self.scheduler.set_timesteps(num_inference_steps) + for t in self.progress_bar(self.scheduler.timesteps): + model_output = self.unet(image, t).sample + + image = self.scheduler.step(model_output, t, image).prev_sample + + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/repaint/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/repaint/__init__.py new file mode 100755 index 0000000..2c6b04a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/repaint/__init__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule + + +_import_structure = {"pipeline_repaint": ["RePaintPipeline"]} + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .pipeline_repaint import RePaintPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/repaint/pipeline_repaint.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/repaint/pipeline_repaint.py new file mode 100755 index 0000000..c03a3d8 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/repaint/pipeline_repaint.py @@ -0,0 +1,230 @@ +# Copyright 2024 ETH Zurich Computer Vision Lab and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from typing import List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch + +from ....models import UNet2DModel +from ....schedulers import RePaintScheduler +from ....utils import PIL_INTERPOLATION, deprecate, logging +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.preprocess +def _preprocess_image(image: Union[List, PIL.Image.Image, torch.Tensor]): + deprecation_message = "The preprocess method is deprecated and will be removed in diffusers 1.0.0. Please use VaeImageProcessor.preprocess(...) instead" + deprecate("preprocess", "1.0.0", deprecation_message, standard_warn=False) + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 8 for x in (w, h)) # resize to integer multiple of 8 + + image = [np.array(i.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + return image + + +def _preprocess_mask(mask: Union[List, PIL.Image.Image, torch.Tensor]): + if isinstance(mask, torch.Tensor): + return mask + elif isinstance(mask, PIL.Image.Image): + mask = [mask] + + if isinstance(mask[0], PIL.Image.Image): + w, h = mask[0].size + w, h = (x - x % 32 for x in (w, h)) # resize to integer multiple of 32 + mask = [np.array(m.convert("L").resize((w, h), resample=PIL_INTERPOLATION["nearest"]))[None, :] for m in mask] + mask = np.concatenate(mask, axis=0) + mask = mask.astype(np.float32) / 255.0 + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + mask = torch.from_numpy(mask) + elif isinstance(mask[0], torch.Tensor): + mask = torch.cat(mask, dim=0) + return mask + + +class RePaintPipeline(DiffusionPipeline): + r""" + Pipeline for image inpainting using RePaint. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + unet ([`UNet2DModel`]): + A `UNet2DModel` to denoise the encoded image latents. + scheduler ([`RePaintScheduler`]): + A `RePaintScheduler` to be used in combination with `unet` to denoise the encoded image. + """ + + unet: UNet2DModel + scheduler: RePaintScheduler + model_cpu_offload_seq = "unet" + + def __init__(self, unet, scheduler): + super().__init__() + self.register_modules(unet=unet, scheduler=scheduler) + + @torch.no_grad() + def __call__( + self, + image: Union[torch.Tensor, PIL.Image.Image], + mask_image: Union[torch.Tensor, PIL.Image.Image], + num_inference_steps: int = 250, + eta: float = 0.0, + jump_length: int = 10, + jump_n_sample: int = 10, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + r""" + The call function to the pipeline for generation. + + Args: + image (`torch.FloatTensor` or `PIL.Image.Image`): + The original image to inpaint on. + mask_image (`torch.FloatTensor` or `PIL.Image.Image`): + The mask_image where 0.0 define which part of the original image to inpaint. + num_inference_steps (`int`, *optional*, defaults to 1000): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + eta (`float`): + The weight of the added noise in a diffusion step. Its value is between 0.0 and 1.0; 0.0 corresponds to + DDIM and 1.0 is the DDPM scheduler. + jump_length (`int`, *optional*, defaults to 10): + The number of steps taken forward in time before going backward in time for a single jump ("j" in + RePaint paper). Take a look at Figure 9 and 10 in the [paper](https://arxiv.org/pdf/2201.09865.pdf). + jump_n_sample (`int`, *optional*, defaults to 10): + The number of times to make a forward time jump for a given chosen time sample. Take a look at Figure 9 + and 10 in the [paper](https://arxiv.org/pdf/2201.09865.pdf). + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + output_type (`str`, `optional`, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`ImagePipelineOutput`] instead of a plain tuple. + + Example: + + ```py + >>> from io import BytesIO + >>> import torch + >>> import PIL + >>> import requests + >>> from diffusers import RePaintPipeline, RePaintScheduler + + + >>> def download_image(url): + ... response = requests.get(url) + ... return PIL.Image.open(BytesIO(response.content)).convert("RGB") + + + >>> img_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/repaint/celeba_hq_256.png" + >>> mask_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/repaint/mask_256.png" + + >>> # Load the original image and the mask as PIL images + >>> original_image = download_image(img_url).resize((256, 256)) + >>> mask_image = download_image(mask_url).resize((256, 256)) + + >>> # Load the RePaint scheduler and pipeline based on a pretrained DDPM model + >>> scheduler = RePaintScheduler.from_pretrained("google/ddpm-ema-celebahq-256") + >>> pipe = RePaintPipeline.from_pretrained("google/ddpm-ema-celebahq-256", scheduler=scheduler) + >>> pipe = pipe.to("cuda") + + >>> generator = torch.Generator(device="cuda").manual_seed(0) + >>> output = pipe( + ... image=original_image, + ... mask_image=mask_image, + ... num_inference_steps=250, + ... eta=0.0, + ... jump_length=10, + ... jump_n_sample=10, + ... generator=generator, + ... ) + >>> inpainted_image = output.images[0] + ``` + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images. + """ + + original_image = image + + original_image = _preprocess_image(original_image) + original_image = original_image.to(device=self._execution_device, dtype=self.unet.dtype) + mask_image = _preprocess_mask(mask_image) + mask_image = mask_image.to(device=self._execution_device, dtype=self.unet.dtype) + + batch_size = original_image.shape[0] + + # sample gaussian noise to begin the loop + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + image_shape = original_image.shape + image = randn_tensor(image_shape, generator=generator, device=self._execution_device, dtype=self.unet.dtype) + + # set step values + self.scheduler.set_timesteps(num_inference_steps, jump_length, jump_n_sample, self._execution_device) + self.scheduler.eta = eta + + t_last = self.scheduler.timesteps[0] + 1 + generator = generator[0] if isinstance(generator, list) else generator + for i, t in enumerate(self.progress_bar(self.scheduler.timesteps)): + if t < t_last: + # predict the noise residual + model_output = self.unet(image, t).sample + # compute previous image: x_t -> x_t-1 + image = self.scheduler.step(model_output, t, image, original_image, mask_image, generator).prev_sample + + else: + # compute the reverse: x_t-1 -> x_t + image = self.scheduler.undo_step(image, t_last, generator) + t_last = t + + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/score_sde_ve/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/score_sde_ve/__init__.py new file mode 100755 index 0000000..87c167c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/score_sde_ve/__init__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule + + +_import_structure = {"pipeline_score_sde_ve": ["ScoreSdeVePipeline"]} + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .pipeline_score_sde_ve import ScoreSdeVePipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/score_sde_ve/pipeline_score_sde_ve.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/score_sde_ve/pipeline_score_sde_ve.py new file mode 100755 index 0000000..b0bb114 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/score_sde_ve/pipeline_score_sde_ve.py @@ -0,0 +1,109 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Tuple, Union + +import torch + +from ....models import UNet2DModel +from ....schedulers import ScoreSdeVeScheduler +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +class ScoreSdeVePipeline(DiffusionPipeline): + r""" + Pipeline for unconditional image generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + unet ([`UNet2DModel`]): + A `UNet2DModel` to denoise the encoded image. + scheduler ([`ScoreSdeVeScheduler`]): + A `ScoreSdeVeScheduler` to be used in combination with `unet` to denoise the encoded image. + """ + + unet: UNet2DModel + scheduler: ScoreSdeVeScheduler + + def __init__(self, unet: UNet2DModel, scheduler: ScoreSdeVeScheduler): + super().__init__() + self.register_modules(unet=unet, scheduler=scheduler) + + @torch.no_grad() + def __call__( + self, + batch_size: int = 1, + num_inference_steps: int = 2000, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + **kwargs, + ) -> Union[ImagePipelineOutput, Tuple]: + r""" + The call function to the pipeline for generation. + + Args: + batch_size (`int`, *optional*, defaults to 1): + The number of images to generate. + generator (`torch.Generator`, `optional`): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + output_type (`str`, `optional`, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`ImagePipelineOutput`] instead of a plain tuple. + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images. + """ + + img_size = self.unet.config.sample_size + shape = (batch_size, 3, img_size, img_size) + + model = self.unet + + sample = randn_tensor(shape, generator=generator) * self.scheduler.init_noise_sigma + sample = sample.to(self.device) + + self.scheduler.set_timesteps(num_inference_steps) + self.scheduler.set_sigmas(num_inference_steps) + + for i, t in enumerate(self.progress_bar(self.scheduler.timesteps)): + sigma_t = self.scheduler.sigmas[i] * torch.ones(shape[0], device=self.device) + + # correction step + for _ in range(self.scheduler.config.correct_steps): + model_output = self.unet(sample, sigma_t).sample + sample = self.scheduler.step_correct(model_output, sample, generator=generator).prev_sample + + # prediction step + model_output = model(sample, sigma_t).sample + output = self.scheduler.step_pred(model_output, t, sample, generator=generator) + + sample, sample_mean = output.prev_sample, output.prev_sample_mean + + sample = sample_mean.clamp(0, 1) + sample = sample.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "pil": + sample = self.numpy_to_pil(sample) + + if not return_dict: + return (sample,) + + return ImagePipelineOutput(images=sample) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/__init__.py new file mode 100755 index 0000000..150954b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/__init__.py @@ -0,0 +1,75 @@ +# flake8: noqa +from typing import TYPE_CHECKING +from ....utils import ( + DIFFUSERS_SLOW_IMPORT, + _LazyModule, + is_note_seq_available, + OptionalDependencyNotAvailable, + is_torch_available, + is_transformers_available, + get_objects_from_module, +) + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ....utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["continous_encoder"] = ["SpectrogramContEncoder"] + _import_structure["notes_encoder"] = ["SpectrogramNotesEncoder"] + _import_structure["pipeline_spectrogram_diffusion"] = [ + "SpectrogramContEncoder", + "SpectrogramDiffusionPipeline", + "T5FilmDecoder", + ] +try: + if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ....utils import dummy_transformers_and_torch_and_note_seq_objects + + _dummy_objects.update(get_objects_from_module(dummy_transformers_and_torch_and_note_seq_objects)) +else: + _import_structure["midi_utils"] = ["MidiProcessor"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ....utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_spectrogram_diffusion import SpectrogramDiffusionPipeline + from .pipeline_spectrogram_diffusion import SpectrogramContEncoder + from .pipeline_spectrogram_diffusion import SpectrogramNotesEncoder + from .pipeline_spectrogram_diffusion import T5FilmDecoder + + try: + if not (is_transformers_available() and is_torch_available() and is_note_seq_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ....utils.dummy_transformers_and_torch_and_note_seq_objects import * + + else: + from .midi_utils import MidiProcessor + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/continuous_encoder.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/continuous_encoder.py new file mode 100755 index 0000000..8664c2f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/continuous_encoder.py @@ -0,0 +1,92 @@ +# Copyright 2022 The Music Spectrogram Diffusion Authors. +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +from transformers.modeling_utils import ModuleUtilsMixin +from transformers.models.t5.modeling_t5 import ( + T5Block, + T5Config, + T5LayerNorm, +) + +from ....configuration_utils import ConfigMixin, register_to_config +from ....models import ModelMixin + + +class SpectrogramContEncoder(ModelMixin, ConfigMixin, ModuleUtilsMixin): + @register_to_config + def __init__( + self, + input_dims: int, + targets_context_length: int, + d_model: int, + dropout_rate: float, + num_layers: int, + num_heads: int, + d_kv: int, + d_ff: int, + feed_forward_proj: str, + is_decoder: bool = False, + ): + super().__init__() + + self.input_proj = nn.Linear(input_dims, d_model, bias=False) + + self.position_encoding = nn.Embedding(targets_context_length, d_model) + self.position_encoding.weight.requires_grad = False + + self.dropout_pre = nn.Dropout(p=dropout_rate) + + t5config = T5Config( + d_model=d_model, + num_heads=num_heads, + d_kv=d_kv, + d_ff=d_ff, + feed_forward_proj=feed_forward_proj, + dropout_rate=dropout_rate, + is_decoder=is_decoder, + is_encoder_decoder=False, + ) + self.encoders = nn.ModuleList() + for lyr_num in range(num_layers): + lyr = T5Block(t5config) + self.encoders.append(lyr) + + self.layer_norm = T5LayerNorm(d_model) + self.dropout_post = nn.Dropout(p=dropout_rate) + + def forward(self, encoder_inputs, encoder_inputs_mask): + x = self.input_proj(encoder_inputs) + + # terminal relative positional encodings + max_positions = encoder_inputs.shape[1] + input_positions = torch.arange(max_positions, device=encoder_inputs.device) + + seq_lens = encoder_inputs_mask.sum(-1) + input_positions = torch.roll(input_positions.unsqueeze(0), tuple(seq_lens.tolist()), dims=0) + x += self.position_encoding(input_positions) + + x = self.dropout_pre(x) + + # inverted the attention mask + input_shape = encoder_inputs.size() + extended_attention_mask = self.get_extended_attention_mask(encoder_inputs_mask, input_shape) + + for lyr in self.encoders: + x = lyr(x, extended_attention_mask)[0] + x = self.layer_norm(x) + + return self.dropout_post(x), encoder_inputs_mask diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/midi_utils.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/midi_utils.py new file mode 100755 index 0000000..e777e84 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/midi_utils.py @@ -0,0 +1,667 @@ +# Copyright 2022 The Music Spectrogram Diffusion Authors. +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import dataclasses +import math +import os +from typing import Any, Callable, List, Mapping, MutableMapping, Optional, Sequence, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F + +from ....utils import is_note_seq_available +from .pipeline_spectrogram_diffusion import TARGET_FEATURE_LENGTH + + +if is_note_seq_available(): + import note_seq +else: + raise ImportError("Please install note-seq via `pip install note-seq`") + + +INPUT_FEATURE_LENGTH = 2048 + +SAMPLE_RATE = 16000 +HOP_SIZE = 320 +FRAME_RATE = int(SAMPLE_RATE // HOP_SIZE) + +DEFAULT_STEPS_PER_SECOND = 100 +DEFAULT_MAX_SHIFT_SECONDS = 10 +DEFAULT_NUM_VELOCITY_BINS = 1 + +SLAKH_CLASS_PROGRAMS = { + "Acoustic Piano": 0, + "Electric Piano": 4, + "Chromatic Percussion": 8, + "Organ": 16, + "Acoustic Guitar": 24, + "Clean Electric Guitar": 26, + "Distorted Electric Guitar": 29, + "Acoustic Bass": 32, + "Electric Bass": 33, + "Violin": 40, + "Viola": 41, + "Cello": 42, + "Contrabass": 43, + "Orchestral Harp": 46, + "Timpani": 47, + "String Ensemble": 48, + "Synth Strings": 50, + "Choir and Voice": 52, + "Orchestral Hit": 55, + "Trumpet": 56, + "Trombone": 57, + "Tuba": 58, + "French Horn": 60, + "Brass Section": 61, + "Soprano/Alto Sax": 64, + "Tenor Sax": 66, + "Baritone Sax": 67, + "Oboe": 68, + "English Horn": 69, + "Bassoon": 70, + "Clarinet": 71, + "Pipe": 73, + "Synth Lead": 80, + "Synth Pad": 88, +} + + +@dataclasses.dataclass +class NoteRepresentationConfig: + """Configuration note representations.""" + + onsets_only: bool + include_ties: bool + + +@dataclasses.dataclass +class NoteEventData: + pitch: int + velocity: Optional[int] = None + program: Optional[int] = None + is_drum: Optional[bool] = None + instrument: Optional[int] = None + + +@dataclasses.dataclass +class NoteEncodingState: + """Encoding state for note transcription, keeping track of active pitches.""" + + # velocity bin for active pitches and programs + active_pitches: MutableMapping[Tuple[int, int], int] = dataclasses.field(default_factory=dict) + + +@dataclasses.dataclass +class EventRange: + type: str + min_value: int + max_value: int + + +@dataclasses.dataclass +class Event: + type: str + value: int + + +class Tokenizer: + def __init__(self, regular_ids: int): + # The special tokens: 0=PAD, 1=EOS, and 2=UNK + self._num_special_tokens = 3 + self._num_regular_tokens = regular_ids + + def encode(self, token_ids): + encoded = [] + for token_id in token_ids: + if not 0 <= token_id < self._num_regular_tokens: + raise ValueError( + f"token_id {token_id} does not fall within valid range of [0, {self._num_regular_tokens})" + ) + encoded.append(token_id + self._num_special_tokens) + + # Add EOS token + encoded.append(1) + + # Pad to till INPUT_FEATURE_LENGTH + encoded = encoded + [0] * (INPUT_FEATURE_LENGTH - len(encoded)) + + return encoded + + +class Codec: + """Encode and decode events. + + Useful for declaring what certain ranges of a vocabulary should be used for. This is intended to be used from + Python before encoding or after decoding with GenericTokenVocabulary. This class is more lightweight and does not + include things like EOS or UNK token handling. + + To ensure that 'shift' events are always the first block of the vocab and start at 0, that event type is required + and specified separately. + """ + + def __init__(self, max_shift_steps: int, steps_per_second: float, event_ranges: List[EventRange]): + """Define Codec. + + Args: + max_shift_steps: Maximum number of shift steps that can be encoded. + steps_per_second: Shift steps will be interpreted as having a duration of + 1 / steps_per_second. + event_ranges: Other supported event types and their ranges. + """ + self.steps_per_second = steps_per_second + self._shift_range = EventRange(type="shift", min_value=0, max_value=max_shift_steps) + self._event_ranges = [self._shift_range] + event_ranges + # Ensure all event types have unique names. + assert len(self._event_ranges) == len({er.type for er in self._event_ranges}) + + @property + def num_classes(self) -> int: + return sum(er.max_value - er.min_value + 1 for er in self._event_ranges) + + # The next couple methods are simplified special case methods just for shift + # events that are intended to be used from within autograph functions. + + def is_shift_event_index(self, index: int) -> bool: + return (self._shift_range.min_value <= index) and (index <= self._shift_range.max_value) + + @property + def max_shift_steps(self) -> int: + return self._shift_range.max_value + + def encode_event(self, event: Event) -> int: + """Encode an event to an index.""" + offset = 0 + for er in self._event_ranges: + if event.type == er.type: + if not er.min_value <= event.value <= er.max_value: + raise ValueError( + f"Event value {event.value} is not within valid range " + f"[{er.min_value}, {er.max_value}] for type {event.type}" + ) + return offset + event.value - er.min_value + offset += er.max_value - er.min_value + 1 + + raise ValueError(f"Unknown event type: {event.type}") + + def event_type_range(self, event_type: str) -> Tuple[int, int]: + """Return [min_id, max_id] for an event type.""" + offset = 0 + for er in self._event_ranges: + if event_type == er.type: + return offset, offset + (er.max_value - er.min_value) + offset += er.max_value - er.min_value + 1 + + raise ValueError(f"Unknown event type: {event_type}") + + def decode_event_index(self, index: int) -> Event: + """Decode an event index to an Event.""" + offset = 0 + for er in self._event_ranges: + if offset <= index <= offset + er.max_value - er.min_value: + return Event(type=er.type, value=er.min_value + index - offset) + offset += er.max_value - er.min_value + 1 + + raise ValueError(f"Unknown event index: {index}") + + +@dataclasses.dataclass +class ProgramGranularity: + # both tokens_map_fn and program_map_fn should be idempotent + tokens_map_fn: Callable[[Sequence[int], Codec], Sequence[int]] + program_map_fn: Callable[[int], int] + + +def drop_programs(tokens, codec: Codec): + """Drops program change events from a token sequence.""" + min_program_id, max_program_id = codec.event_type_range("program") + return tokens[(tokens < min_program_id) | (tokens > max_program_id)] + + +def programs_to_midi_classes(tokens, codec): + """Modifies program events to be the first program in the MIDI class.""" + min_program_id, max_program_id = codec.event_type_range("program") + is_program = (tokens >= min_program_id) & (tokens <= max_program_id) + return np.where(is_program, min_program_id + 8 * ((tokens - min_program_id) // 8), tokens) + + +PROGRAM_GRANULARITIES = { + # "flat" granularity; drop program change tokens and set NoteSequence + # programs to zero + "flat": ProgramGranularity(tokens_map_fn=drop_programs, program_map_fn=lambda program: 0), + # map each program to the first program in its MIDI class + "midi_class": ProgramGranularity( + tokens_map_fn=programs_to_midi_classes, program_map_fn=lambda program: 8 * (program // 8) + ), + # leave programs as is + "full": ProgramGranularity(tokens_map_fn=lambda tokens, codec: tokens, program_map_fn=lambda program: program), +} + + +def frame(signal, frame_length, frame_step, pad_end=False, pad_value=0, axis=-1): + """ + equivalent of tf.signal.frame + """ + signal_length = signal.shape[axis] + if pad_end: + frames_overlap = frame_length - frame_step + rest_samples = np.abs(signal_length - frames_overlap) % np.abs(frame_length - frames_overlap) + pad_size = int(frame_length - rest_samples) + + if pad_size != 0: + pad_axis = [0] * signal.ndim + pad_axis[axis] = pad_size + signal = F.pad(signal, pad_axis, "constant", pad_value) + frames = signal.unfold(axis, frame_length, frame_step) + return frames + + +def program_to_slakh_program(program): + # this is done very hackily, probably should use a custom mapping + for slakh_program in sorted(SLAKH_CLASS_PROGRAMS.values(), reverse=True): + if program >= slakh_program: + return slakh_program + + +def audio_to_frames( + samples, + hop_size: int, + frame_rate: int, +) -> Tuple[Sequence[Sequence[int]], torch.Tensor]: + """Convert audio samples to non-overlapping frames and frame times.""" + frame_size = hop_size + samples = np.pad(samples, [0, frame_size - len(samples) % frame_size], mode="constant") + + # Split audio into frames. + frames = frame( + torch.Tensor(samples).unsqueeze(0), + frame_length=frame_size, + frame_step=frame_size, + pad_end=False, # TODO check why its off by 1 here when True + ) + + num_frames = len(samples) // frame_size + + times = np.arange(num_frames) / frame_rate + return frames, times + + +def note_sequence_to_onsets_and_offsets_and_programs( + ns: note_seq.NoteSequence, +) -> Tuple[Sequence[float], Sequence[NoteEventData]]: + """Extract onset & offset times and pitches & programs from a NoteSequence. + + The onset & offset times will not necessarily be in sorted order. + + Args: + ns: NoteSequence from which to extract onsets and offsets. + + Returns: + times: A list of note onset and offset times. values: A list of NoteEventData objects where velocity is zero for + note + offsets. + """ + # Sort by program and pitch and put offsets before onsets as a tiebreaker for + # subsequent stable sort. + notes = sorted(ns.notes, key=lambda note: (note.is_drum, note.program, note.pitch)) + times = [note.end_time for note in notes if not note.is_drum] + [note.start_time for note in notes] + values = [ + NoteEventData(pitch=note.pitch, velocity=0, program=note.program, is_drum=False) + for note in notes + if not note.is_drum + ] + [ + NoteEventData(pitch=note.pitch, velocity=note.velocity, program=note.program, is_drum=note.is_drum) + for note in notes + ] + return times, values + + +def num_velocity_bins_from_codec(codec: Codec): + """Get number of velocity bins from event codec.""" + lo, hi = codec.event_type_range("velocity") + return hi - lo + + +# segment an array into segments of length n +def segment(a, n): + return [a[i : i + n] for i in range(0, len(a), n)] + + +def velocity_to_bin(velocity, num_velocity_bins): + if velocity == 0: + return 0 + else: + return math.ceil(num_velocity_bins * velocity / note_seq.MAX_MIDI_VELOCITY) + + +def note_event_data_to_events( + state: Optional[NoteEncodingState], + value: NoteEventData, + codec: Codec, +) -> Sequence[Event]: + """Convert note event data to a sequence of events.""" + if value.velocity is None: + # onsets only, no program or velocity + return [Event("pitch", value.pitch)] + else: + num_velocity_bins = num_velocity_bins_from_codec(codec) + velocity_bin = velocity_to_bin(value.velocity, num_velocity_bins) + if value.program is None: + # onsets + offsets + velocities only, no programs + if state is not None: + state.active_pitches[(value.pitch, 0)] = velocity_bin + return [Event("velocity", velocity_bin), Event("pitch", value.pitch)] + else: + if value.is_drum: + # drum events use a separate vocabulary + return [Event("velocity", velocity_bin), Event("drum", value.pitch)] + else: + # program + velocity + pitch + if state is not None: + state.active_pitches[(value.pitch, value.program)] = velocity_bin + return [ + Event("program", value.program), + Event("velocity", velocity_bin), + Event("pitch", value.pitch), + ] + + +def note_encoding_state_to_events(state: NoteEncodingState) -> Sequence[Event]: + """Output program and pitch events for active notes plus a final tie event.""" + events = [] + for pitch, program in sorted(state.active_pitches.keys(), key=lambda k: k[::-1]): + if state.active_pitches[(pitch, program)]: + events += [Event("program", program), Event("pitch", pitch)] + events.append(Event("tie", 0)) + return events + + +def encode_and_index_events( + state, event_times, event_values, codec, frame_times, encode_event_fn, encoding_state_to_events_fn=None +): + """Encode a sequence of timed events and index to audio frame times. + + Encodes time shifts as repeated single step shifts for later run length encoding. + + Optionally, also encodes a sequence of "state events", keeping track of the current encoding state at each audio + frame. This can be used e.g. to prepend events representing the current state to a targets segment. + + Args: + state: Initial event encoding state. + event_times: Sequence of event times. + event_values: Sequence of event values. + encode_event_fn: Function that transforms event value into a sequence of one + or more Event objects. + codec: An Codec object that maps Event objects to indices. + frame_times: Time for every audio frame. + encoding_state_to_events_fn: Function that transforms encoding state into a + sequence of one or more Event objects. + + Returns: + events: Encoded events and shifts. event_start_indices: Corresponding start event index for every audio frame. + Note: one event can correspond to multiple audio indices due to sampling rate differences. This makes + splitting sequences tricky because the same event can appear at the end of one sequence and the beginning of + another. + event_end_indices: Corresponding end event index for every audio frame. Used + to ensure when slicing that one chunk ends where the next begins. Should always be true that + event_end_indices[i] = event_start_indices[i + 1]. + state_events: Encoded "state" events representing the encoding state before + each event. + state_event_indices: Corresponding state event index for every audio frame. + """ + indices = np.argsort(event_times, kind="stable") + event_steps = [round(event_times[i] * codec.steps_per_second) for i in indices] + event_values = [event_values[i] for i in indices] + + events = [] + state_events = [] + event_start_indices = [] + state_event_indices = [] + + cur_step = 0 + cur_event_idx = 0 + cur_state_event_idx = 0 + + def fill_event_start_indices_to_cur_step(): + while ( + len(event_start_indices) < len(frame_times) + and frame_times[len(event_start_indices)] < cur_step / codec.steps_per_second + ): + event_start_indices.append(cur_event_idx) + state_event_indices.append(cur_state_event_idx) + + for event_step, event_value in zip(event_steps, event_values): + while event_step > cur_step: + events.append(codec.encode_event(Event(type="shift", value=1))) + cur_step += 1 + fill_event_start_indices_to_cur_step() + cur_event_idx = len(events) + cur_state_event_idx = len(state_events) + if encoding_state_to_events_fn: + # Dump state to state events *before* processing the next event, because + # we want to capture the state prior to the occurrence of the event. + for e in encoding_state_to_events_fn(state): + state_events.append(codec.encode_event(e)) + + for e in encode_event_fn(state, event_value, codec): + events.append(codec.encode_event(e)) + + # After the last event, continue filling out the event_start_indices array. + # The inequality is not strict because if our current step lines up exactly + # with (the start of) an audio frame, we need to add an additional shift event + # to "cover" that frame. + while cur_step / codec.steps_per_second <= frame_times[-1]: + events.append(codec.encode_event(Event(type="shift", value=1))) + cur_step += 1 + fill_event_start_indices_to_cur_step() + cur_event_idx = len(events) + + # Now fill in event_end_indices. We need this extra array to make sure that + # when we slice events, each slice ends exactly where the subsequent slice + # begins. + event_end_indices = event_start_indices[1:] + [len(events)] + + events = np.array(events).astype(np.int32) + state_events = np.array(state_events).astype(np.int32) + event_start_indices = segment(np.array(event_start_indices).astype(np.int32), TARGET_FEATURE_LENGTH) + event_end_indices = segment(np.array(event_end_indices).astype(np.int32), TARGET_FEATURE_LENGTH) + state_event_indices = segment(np.array(state_event_indices).astype(np.int32), TARGET_FEATURE_LENGTH) + + outputs = [] + for start_indices, end_indices, event_indices in zip(event_start_indices, event_end_indices, state_event_indices): + outputs.append( + { + "inputs": events, + "event_start_indices": start_indices, + "event_end_indices": end_indices, + "state_events": state_events, + "state_event_indices": event_indices, + } + ) + + return outputs + + +def extract_sequence_with_indices(features, state_events_end_token=None, feature_key="inputs"): + """Extract target sequence corresponding to audio token segment.""" + features = features.copy() + start_idx = features["event_start_indices"][0] + end_idx = features["event_end_indices"][-1] + + features[feature_key] = features[feature_key][start_idx:end_idx] + + if state_events_end_token is not None: + # Extract the state events corresponding to the audio start token, and + # prepend them to the targets array. + state_event_start_idx = features["state_event_indices"][0] + state_event_end_idx = state_event_start_idx + 1 + while features["state_events"][state_event_end_idx - 1] != state_events_end_token: + state_event_end_idx += 1 + features[feature_key] = np.concatenate( + [ + features["state_events"][state_event_start_idx:state_event_end_idx], + features[feature_key], + ], + axis=0, + ) + + return features + + +def map_midi_programs( + feature, codec: Codec, granularity_type: str = "full", feature_key: str = "inputs" +) -> Mapping[str, Any]: + """Apply MIDI program map to token sequences.""" + granularity = PROGRAM_GRANULARITIES[granularity_type] + + feature[feature_key] = granularity.tokens_map_fn(feature[feature_key], codec) + return feature + + +def run_length_encode_shifts_fn( + features, + codec: Codec, + feature_key: str = "inputs", + state_change_event_types: Sequence[str] = (), +) -> Callable[[Mapping[str, Any]], Mapping[str, Any]]: + """Return a function that run-length encodes shifts for a given codec. + + Args: + codec: The Codec to use for shift events. + feature_key: The feature key for which to run-length encode shifts. + state_change_event_types: A list of event types that represent state + changes; tokens corresponding to these event types will be interpreted as state changes and redundant ones + will be removed. + + Returns: + A preprocessing function that run-length encodes single-step shifts. + """ + state_change_event_ranges = [codec.event_type_range(event_type) for event_type in state_change_event_types] + + def run_length_encode_shifts(features: MutableMapping[str, Any]) -> Mapping[str, Any]: + """Combine leading/interior shifts, trim trailing shifts. + + Args: + features: Dict of features to process. + + Returns: + A dict of features. + """ + events = features[feature_key] + + shift_steps = 0 + total_shift_steps = 0 + output = np.array([], dtype=np.int32) + + current_state = np.zeros(len(state_change_event_ranges), dtype=np.int32) + + for event in events: + if codec.is_shift_event_index(event): + shift_steps += 1 + total_shift_steps += 1 + + else: + # If this event is a state change and has the same value as the current + # state, we can skip it entirely. + is_redundant = False + for i, (min_index, max_index) in enumerate(state_change_event_ranges): + if (min_index <= event) and (event <= max_index): + if current_state[i] == event: + is_redundant = True + current_state[i] = event + if is_redundant: + continue + + # Once we've reached a non-shift event, RLE all previous shift events + # before outputting the non-shift event. + if shift_steps > 0: + shift_steps = total_shift_steps + while shift_steps > 0: + output_steps = np.minimum(codec.max_shift_steps, shift_steps) + output = np.concatenate([output, [output_steps]], axis=0) + shift_steps -= output_steps + output = np.concatenate([output, [event]], axis=0) + + features[feature_key] = output + return features + + return run_length_encode_shifts(features) + + +def note_representation_processor_chain(features, codec: Codec, note_representation_config: NoteRepresentationConfig): + tie_token = codec.encode_event(Event("tie", 0)) + state_events_end_token = tie_token if note_representation_config.include_ties else None + + features = extract_sequence_with_indices( + features, state_events_end_token=state_events_end_token, feature_key="inputs" + ) + + features = map_midi_programs(features, codec) + + features = run_length_encode_shifts_fn(features, codec, state_change_event_types=["velocity", "program"]) + + return features + + +class MidiProcessor: + def __init__(self): + self.codec = Codec( + max_shift_steps=DEFAULT_MAX_SHIFT_SECONDS * DEFAULT_STEPS_PER_SECOND, + steps_per_second=DEFAULT_STEPS_PER_SECOND, + event_ranges=[ + EventRange("pitch", note_seq.MIN_MIDI_PITCH, note_seq.MAX_MIDI_PITCH), + EventRange("velocity", 0, DEFAULT_NUM_VELOCITY_BINS), + EventRange("tie", 0, 0), + EventRange("program", note_seq.MIN_MIDI_PROGRAM, note_seq.MAX_MIDI_PROGRAM), + EventRange("drum", note_seq.MIN_MIDI_PITCH, note_seq.MAX_MIDI_PITCH), + ], + ) + self.tokenizer = Tokenizer(self.codec.num_classes) + self.note_representation_config = NoteRepresentationConfig(onsets_only=False, include_ties=True) + + def __call__(self, midi: Union[bytes, os.PathLike, str]): + if not isinstance(midi, bytes): + with open(midi, "rb") as f: + midi = f.read() + + ns = note_seq.midi_to_note_sequence(midi) + ns_sus = note_seq.apply_sustain_control_changes(ns) + + for note in ns_sus.notes: + if not note.is_drum: + note.program = program_to_slakh_program(note.program) + + samples = np.zeros(int(ns_sus.total_time * SAMPLE_RATE)) + + _, frame_times = audio_to_frames(samples, HOP_SIZE, FRAME_RATE) + times, values = note_sequence_to_onsets_and_offsets_and_programs(ns_sus) + + events = encode_and_index_events( + state=NoteEncodingState(), + event_times=times, + event_values=values, + frame_times=frame_times, + codec=self.codec, + encode_event_fn=note_event_data_to_events, + encoding_state_to_events_fn=note_encoding_state_to_events, + ) + + events = [ + note_representation_processor_chain(event, self.codec, self.note_representation_config) for event in events + ] + input_tokens = [self.tokenizer.encode(event["inputs"]) for event in events] + + return input_tokens diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/notes_encoder.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/notes_encoder.py new file mode 100755 index 0000000..1259f0b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/notes_encoder.py @@ -0,0 +1,86 @@ +# Copyright 2022 The Music Spectrogram Diffusion Authors. +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +from transformers.modeling_utils import ModuleUtilsMixin +from transformers.models.t5.modeling_t5 import T5Block, T5Config, T5LayerNorm + +from ....configuration_utils import ConfigMixin, register_to_config +from ....models import ModelMixin + + +class SpectrogramNotesEncoder(ModelMixin, ConfigMixin, ModuleUtilsMixin): + @register_to_config + def __init__( + self, + max_length: int, + vocab_size: int, + d_model: int, + dropout_rate: float, + num_layers: int, + num_heads: int, + d_kv: int, + d_ff: int, + feed_forward_proj: str, + is_decoder: bool = False, + ): + super().__init__() + + self.token_embedder = nn.Embedding(vocab_size, d_model) + + self.position_encoding = nn.Embedding(max_length, d_model) + self.position_encoding.weight.requires_grad = False + + self.dropout_pre = nn.Dropout(p=dropout_rate) + + t5config = T5Config( + vocab_size=vocab_size, + d_model=d_model, + num_heads=num_heads, + d_kv=d_kv, + d_ff=d_ff, + dropout_rate=dropout_rate, + feed_forward_proj=feed_forward_proj, + is_decoder=is_decoder, + is_encoder_decoder=False, + ) + + self.encoders = nn.ModuleList() + for lyr_num in range(num_layers): + lyr = T5Block(t5config) + self.encoders.append(lyr) + + self.layer_norm = T5LayerNorm(d_model) + self.dropout_post = nn.Dropout(p=dropout_rate) + + def forward(self, encoder_input_tokens, encoder_inputs_mask): + x = self.token_embedder(encoder_input_tokens) + + seq_length = encoder_input_tokens.shape[1] + inputs_positions = torch.arange(seq_length, device=encoder_input_tokens.device) + x += self.position_encoding(inputs_positions) + + x = self.dropout_pre(x) + + # inverted the attention mask + input_shape = encoder_input_tokens.size() + extended_attention_mask = self.get_extended_attention_mask(encoder_inputs_mask, input_shape) + + for lyr in self.encoders: + x = lyr(x, extended_attention_mask)[0] + x = self.layer_norm(x) + + return self.dropout_post(x), encoder_inputs_mask diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py new file mode 100755 index 0000000..496a1f7 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/spectrogram_diffusion/pipeline_spectrogram_diffusion.py @@ -0,0 +1,269 @@ +# Copyright 2022 The Music Spectrogram Diffusion Authors. +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Any, Callable, List, Optional, Tuple, Union + +import numpy as np +import torch + +from ....models import T5FilmDecoder +from ....schedulers import DDPMScheduler +from ....utils import is_onnx_available, logging +from ....utils.torch_utils import randn_tensor + + +if is_onnx_available(): + from ...onnx_utils import OnnxRuntimeModel + +from ...pipeline_utils import AudioPipelineOutput, DiffusionPipeline +from .continuous_encoder import SpectrogramContEncoder +from .notes_encoder import SpectrogramNotesEncoder + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +TARGET_FEATURE_LENGTH = 256 + + +class SpectrogramDiffusionPipeline(DiffusionPipeline): + r""" + Pipeline for unconditional audio generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + notes_encoder ([`SpectrogramNotesEncoder`]): + continuous_encoder ([`SpectrogramContEncoder`]): + decoder ([`T5FilmDecoder`]): + A [`T5FilmDecoder`] to denoise the encoded audio latents. + scheduler ([`DDPMScheduler`]): + A scheduler to be used in combination with `decoder` to denoise the encoded audio latents. + melgan ([`OnnxRuntimeModel`]): + """ + + _optional_components = ["melgan"] + + def __init__( + self, + notes_encoder: SpectrogramNotesEncoder, + continuous_encoder: SpectrogramContEncoder, + decoder: T5FilmDecoder, + scheduler: DDPMScheduler, + melgan: OnnxRuntimeModel if is_onnx_available() else Any, + ) -> None: + super().__init__() + + # From MELGAN + self.min_value = math.log(1e-5) # Matches MelGAN training. + self.max_value = 4.0 # Largest value for most examples + self.n_dims = 128 + + self.register_modules( + notes_encoder=notes_encoder, + continuous_encoder=continuous_encoder, + decoder=decoder, + scheduler=scheduler, + melgan=melgan, + ) + + def scale_features(self, features, output_range=(-1.0, 1.0), clip=False): + """Linearly scale features to network outputs range.""" + min_out, max_out = output_range + if clip: + features = torch.clip(features, self.min_value, self.max_value) + # Scale to [0, 1]. + zero_one = (features - self.min_value) / (self.max_value - self.min_value) + # Scale to [min_out, max_out]. + return zero_one * (max_out - min_out) + min_out + + def scale_to_features(self, outputs, input_range=(-1.0, 1.0), clip=False): + """Invert by linearly scaling network outputs to features range.""" + min_out, max_out = input_range + outputs = torch.clip(outputs, min_out, max_out) if clip else outputs + # Scale to [0, 1]. + zero_one = (outputs - min_out) / (max_out - min_out) + # Scale to [self.min_value, self.max_value]. + return zero_one * (self.max_value - self.min_value) + self.min_value + + def encode(self, input_tokens, continuous_inputs, continuous_mask): + tokens_mask = input_tokens > 0 + tokens_encoded, tokens_mask = self.notes_encoder( + encoder_input_tokens=input_tokens, encoder_inputs_mask=tokens_mask + ) + + continuous_encoded, continuous_mask = self.continuous_encoder( + encoder_inputs=continuous_inputs, encoder_inputs_mask=continuous_mask + ) + + return [(tokens_encoded, tokens_mask), (continuous_encoded, continuous_mask)] + + def decode(self, encodings_and_masks, input_tokens, noise_time): + timesteps = noise_time + if not torch.is_tensor(timesteps): + timesteps = torch.tensor([timesteps], dtype=torch.long, device=input_tokens.device) + elif torch.is_tensor(timesteps) and len(timesteps.shape) == 0: + timesteps = timesteps[None].to(input_tokens.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps * torch.ones(input_tokens.shape[0], dtype=timesteps.dtype, device=timesteps.device) + + logits = self.decoder( + encodings_and_masks=encodings_and_masks, decoder_input_tokens=input_tokens, decoder_noise_time=timesteps + ) + return logits + + @torch.no_grad() + def __call__( + self, + input_tokens: List[List[int]], + generator: Optional[torch.Generator] = None, + num_inference_steps: int = 100, + return_dict: bool = True, + output_type: str = "numpy", + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + ) -> Union[AudioPipelineOutput, Tuple]: + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + r""" + The call function to the pipeline for generation. + + Args: + input_tokens (`List[List[int]]`): + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality audio at the + expense of slower inference. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.AudioPipelineOutput`] instead of a plain tuple. + output_type (`str`, *optional*, defaults to `"numpy"`): + The output format of the generated audio. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Example: + + ```py + >>> from diffusers import SpectrogramDiffusionPipeline, MidiProcessor + + >>> pipe = SpectrogramDiffusionPipeline.from_pretrained("google/music-spectrogram-diffusion") + >>> pipe = pipe.to("cuda") + >>> processor = MidiProcessor() + + >>> # Download MIDI from: wget http://www.piano-midi.de/midis/beethoven/beethoven_hammerklavier_2.mid + >>> output = pipe(processor("beethoven_hammerklavier_2.mid")) + + >>> audio = output.audios[0] + ``` + + Returns: + [`pipelines.AudioPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`pipelines.AudioPipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated audio. + """ + + pred_mel = np.zeros([1, TARGET_FEATURE_LENGTH, self.n_dims], dtype=np.float32) + full_pred_mel = np.zeros([1, 0, self.n_dims], np.float32) + ones = torch.ones((1, TARGET_FEATURE_LENGTH), dtype=bool, device=self.device) + + for i, encoder_input_tokens in enumerate(input_tokens): + if i == 0: + encoder_continuous_inputs = torch.from_numpy(pred_mel[:1].copy()).to( + device=self.device, dtype=self.decoder.dtype + ) + # The first chunk has no previous context. + encoder_continuous_mask = torch.zeros((1, TARGET_FEATURE_LENGTH), dtype=bool, device=self.device) + else: + # The full song pipeline does not feed in a context feature, so the mask + # will be all 0s after the feature converter. Because we know we're + # feeding in a full context chunk from the previous prediction, set it + # to all 1s. + encoder_continuous_mask = ones + + encoder_continuous_inputs = self.scale_features( + encoder_continuous_inputs, output_range=[-1.0, 1.0], clip=True + ) + + encodings_and_masks = self.encode( + input_tokens=torch.IntTensor([encoder_input_tokens]).to(device=self.device), + continuous_inputs=encoder_continuous_inputs, + continuous_mask=encoder_continuous_mask, + ) + + # Sample encoder_continuous_inputs shaped gaussian noise to begin loop + x = randn_tensor( + shape=encoder_continuous_inputs.shape, + generator=generator, + device=self.device, + dtype=self.decoder.dtype, + ) + + # set step values + self.scheduler.set_timesteps(num_inference_steps) + + # Denoising diffusion loop + for j, t in enumerate(self.progress_bar(self.scheduler.timesteps)): + output = self.decode( + encodings_and_masks=encodings_and_masks, + input_tokens=x, + noise_time=t / self.scheduler.config.num_train_timesteps, # rescale to [0, 1) + ) + + # Compute previous output: x_t -> x_t-1 + x = self.scheduler.step(output, t, x, generator=generator).prev_sample + + mel = self.scale_to_features(x, input_range=[-1.0, 1.0]) + encoder_continuous_inputs = mel[:1] + pred_mel = mel.cpu().float().numpy() + + full_pred_mel = np.concatenate([full_pred_mel, pred_mel[:1]], axis=1) + + # call the callback, if provided + if callback is not None and i % callback_steps == 0: + callback(i, full_pred_mel) + + logger.info("Generated segment", i) + + if output_type == "numpy" and not is_onnx_available(): + raise ValueError( + "Cannot return output in 'np' format if ONNX is not available. Make sure to have ONNX installed or set 'output_type' to 'mel'." + ) + elif output_type == "numpy" and self.melgan is None: + raise ValueError( + "Cannot return output in 'np' format if melgan component is not defined. Make sure to define `self.melgan` or set 'output_type' to 'mel'." + ) + + if output_type == "numpy": + output = self.melgan(input_features=full_pred_mel.astype(np.float32)) + else: + output = full_pred_mel + + if not return_dict: + return (output,) + + return AudioPipelineOutput(audios=output) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/__init__.py new file mode 100755 index 0000000..36cf1a3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/__init__.py @@ -0,0 +1,55 @@ +from typing import TYPE_CHECKING + +from ....utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ....utils import dummy_torch_and_transformers_objects + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_cycle_diffusion"] = ["CycleDiffusionPipeline"] + _import_structure["pipeline_stable_diffusion_inpaint_legacy"] = ["StableDiffusionInpaintPipelineLegacy"] + _import_structure["pipeline_stable_diffusion_model_editing"] = ["StableDiffusionModelEditingPipeline"] + + _import_structure["pipeline_stable_diffusion_paradigms"] = ["StableDiffusionParadigmsPipeline"] + _import_structure["pipeline_stable_diffusion_pix2pix_zero"] = ["StableDiffusionPix2PixZeroPipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ....utils.dummy_torch_and_transformers_objects import * + + else: + from .pipeline_cycle_diffusion import CycleDiffusionPipeline + from .pipeline_stable_diffusion_inpaint_legacy import StableDiffusionInpaintPipelineLegacy + from .pipeline_stable_diffusion_model_editing import StableDiffusionModelEditingPipeline + from .pipeline_stable_diffusion_paradigms import StableDiffusionParadigmsPipeline + from .pipeline_stable_diffusion_pix2pix_zero import StableDiffusionPix2PixZeroPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py new file mode 100755 index 0000000..0581eff --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_cycle_diffusion.py @@ -0,0 +1,948 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from packaging import version +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ....configuration_utils import FrozenDict +from ....image_processor import PipelineImageInput, VaeImageProcessor +from ....loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ....models import AutoencoderKL, UNet2DConditionModel +from ....models.lora import adjust_lora_scale_text_encoder +from ....schedulers import DDIMScheduler +from ....utils import PIL_INTERPOLATION, USE_PEFT_BACKEND, deprecate, logging, scale_lora_layers, unscale_lora_layers +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline +from ...stable_diffusion.pipeline_output import StableDiffusionPipelineOutput +from ...stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.preprocess +def preprocess(image): + deprecation_message = "The preprocess method is deprecated and will be removed in diffusers 1.0.0. Please use VaeImageProcessor.preprocess(...) instead" + deprecate("preprocess", "1.0.0", deprecation_message, standard_warn=False) + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 8 for x in (w, h)) # resize to integer multiple of 8 + + image = [np.array(i.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + return image + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +def posterior_sample(scheduler, latents, timestep, clean_latents, generator, eta): + # 1. get previous step value (=t-1) + prev_timestep = timestep - scheduler.config.num_train_timesteps // scheduler.num_inference_steps + + if prev_timestep <= 0: + return clean_latents + + # 2. compute alphas, betas + alpha_prod_t = scheduler.alphas_cumprod[timestep] + alpha_prod_t_prev = ( + scheduler.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else scheduler.final_alpha_cumprod + ) + + variance = scheduler._get_variance(timestep, prev_timestep) + std_dev_t = eta * variance ** (0.5) + + # direction pointing to x_t + e_t = (latents - alpha_prod_t ** (0.5) * clean_latents) / (1 - alpha_prod_t) ** (0.5) + dir_xt = (1.0 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * e_t + noise = std_dev_t * randn_tensor( + clean_latents.shape, dtype=clean_latents.dtype, device=clean_latents.device, generator=generator + ) + prev_latents = alpha_prod_t_prev ** (0.5) * clean_latents + dir_xt + noise + + return prev_latents + + +def compute_noise(scheduler, prev_latents, latents, timestep, noise_pred, eta): + # 1. get previous step value (=t-1) + prev_timestep = timestep - scheduler.config.num_train_timesteps // scheduler.num_inference_steps + + # 2. compute alphas, betas + alpha_prod_t = scheduler.alphas_cumprod[timestep] + alpha_prod_t_prev = ( + scheduler.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else scheduler.final_alpha_cumprod + ) + + beta_prod_t = 1 - alpha_prod_t + + # 3. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_original_sample = (latents - beta_prod_t ** (0.5) * noise_pred) / alpha_prod_t ** (0.5) + + # 4. Clip "predicted x_0" + if scheduler.config.clip_sample: + pred_original_sample = torch.clamp(pred_original_sample, -1, 1) + + # 5. compute variance: "sigma_t(η)" -> see formula (16) + # σ_t = sqrt((1 − α_t−1)/(1 − α_t)) * sqrt(1 − α_t/α_t−1) + variance = scheduler._get_variance(timestep, prev_timestep) + std_dev_t = eta * variance ** (0.5) + + # 6. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * noise_pred + + noise = (prev_latents - (alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction)) / ( + variance ** (0.5) * eta + ) + return noise + + +class CycleDiffusionPipeline(DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin): + r""" + Pipeline for text-guided image to image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can only be an + instance of [`DDIMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: DDIMScheduler, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely .If you're checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + def check_inputs( + self, + prompt, + strength, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): + image = image.to(device=device, dtype=dtype) + + batch_size = image.shape[0] + + if image.shape[1] == 4: + init_latents = image + + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if isinstance(generator, list): + init_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(image.shape[0]) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + init_latents = self.vae.config.scaling_factor * init_latents + + if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0: + # expand init_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {init_latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_image_per_prompt = batch_size // init_latents.shape[0] + init_latents = torch.cat([init_latents] * additional_image_per_prompt * num_images_per_prompt, dim=0) + elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts." + ) + else: + init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0) + + # add noise to latents using the timestep + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # get latents + clean_latents = init_latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + latents = init_latents + + return latents, clean_latents + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + source_prompt: Union[str, List[str]], + image: PipelineImageInput = None, + strength: float = 0.8, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + source_guidance_scale: Optional[float] = 1, + num_images_per_prompt: Optional[int] = 1, + eta: Optional[float] = 0.1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`torch.FloatTensor` `np.ndarray`, `PIL.Image.Image`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image` or tensor representing an image batch to be used as the starting point. Can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + strength (`float`, *optional*, defaults to 0.8): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + source_guidance_scale (`float`, *optional*, defaults to 1): + Guidance scale for the source prompt. This is useful to control the amount of influence the source + prompt has for encoding. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Example: + + ```py + import requests + import torch + from PIL import Image + from io import BytesIO + + from diffusers import CycleDiffusionPipeline, DDIMScheduler + + # load the pipeline + # make sure you're logged in with `huggingface-cli login` + model_id_or_path = "CompVis/stable-diffusion-v1-4" + scheduler = DDIMScheduler.from_pretrained(model_id_or_path, subfolder="scheduler") + pipe = CycleDiffusionPipeline.from_pretrained(model_id_or_path, scheduler=scheduler).to("cuda") + + # let's download an initial image + url = "https://raw.githubusercontent.com/ChenWu98/cycle-diffusion/main/data/dalle2/An%20astronaut%20riding%20a%20horse.png" + response = requests.get(url) + init_image = Image.open(BytesIO(response.content)).convert("RGB") + init_image = init_image.resize((512, 512)) + init_image.save("horse.png") + + # let's specify a prompt + source_prompt = "An astronaut riding a horse" + prompt = "An astronaut riding an elephant" + + # call the pipeline + image = pipe( + prompt=prompt, + source_prompt=source_prompt, + image=init_image, + num_inference_steps=100, + eta=0.1, + strength=0.8, + guidance_scale=2, + source_guidance_scale=1, + ).images[0] + + image.save("horse_to_elephant.png") + + # let's try another example + # See more samples at the original repo: https://github.com/ChenWu98/cycle-diffusion + url = ( + "https://raw.githubusercontent.com/ChenWu98/cycle-diffusion/main/data/dalle2/A%20black%20colored%20car.png" + ) + response = requests.get(url) + init_image = Image.open(BytesIO(response.content)).convert("RGB") + init_image = init_image.resize((512, 512)) + init_image.save("black.png") + + source_prompt = "A black colored car" + prompt = "A blue colored car" + + # call the pipeline + torch.manual_seed(0) + image = pipe( + prompt=prompt, + source_prompt=source_prompt, + image=init_image, + num_inference_steps=100, + eta=0.1, + strength=0.85, + guidance_scale=3, + source_guidance_scale=1, + ).images[0] + + image.save("black_to_blue.png") + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 1. Check inputs + self.check_inputs(prompt, strength, callback_steps) + + # 2. Define call parameters + batch_size = 1 if isinstance(prompt, str) else len(prompt) + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + prompt_embeds_tuple = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + prompt_embeds=prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=clip_skip, + ) + source_prompt_embeds_tuple = self.encode_prompt( + source_prompt, device, num_images_per_prompt, do_classifier_free_guidance, None, clip_skip=clip_skip + ) + if prompt_embeds_tuple[1] is not None: + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + else: + prompt_embeds = prompt_embeds_tuple[0] + if source_prompt_embeds_tuple[1] is not None: + source_prompt_embeds = torch.cat([source_prompt_embeds_tuple[1], source_prompt_embeds_tuple[0]]) + else: + source_prompt_embeds = source_prompt_embeds_tuple[0] + + # 4. Preprocess image + image = self.image_processor.preprocess(image) + + # 5. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + + # 6. Prepare latent variables + latents, clean_latents = self.prepare_latents( + image, latent_timestep, batch_size, num_images_per_prompt, prompt_embeds.dtype, device, generator + ) + source_latents = latents + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + generator = extra_step_kwargs.pop("generator", None) + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + source_latent_model_input = ( + torch.cat([source_latents] * 2) if do_classifier_free_guidance else source_latents + ) + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + source_latent_model_input = self.scheduler.scale_model_input(source_latent_model_input, t) + + # predict the noise residual + if do_classifier_free_guidance: + concat_latent_model_input = torch.stack( + [ + source_latent_model_input[0], + latent_model_input[0], + source_latent_model_input[1], + latent_model_input[1], + ], + dim=0, + ) + concat_prompt_embeds = torch.stack( + [ + source_prompt_embeds[0], + prompt_embeds[0], + source_prompt_embeds[1], + prompt_embeds[1], + ], + dim=0, + ) + else: + concat_latent_model_input = torch.cat( + [ + source_latent_model_input, + latent_model_input, + ], + dim=0, + ) + concat_prompt_embeds = torch.cat( + [ + source_prompt_embeds, + prompt_embeds, + ], + dim=0, + ) + + concat_noise_pred = self.unet( + concat_latent_model_input, + t, + cross_attention_kwargs=cross_attention_kwargs, + encoder_hidden_states=concat_prompt_embeds, + ).sample + + # perform guidance + if do_classifier_free_guidance: + ( + source_noise_pred_uncond, + noise_pred_uncond, + source_noise_pred_text, + noise_pred_text, + ) = concat_noise_pred.chunk(4, dim=0) + + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + source_noise_pred = source_noise_pred_uncond + source_guidance_scale * ( + source_noise_pred_text - source_noise_pred_uncond + ) + + else: + (source_noise_pred, noise_pred) = concat_noise_pred.chunk(2, dim=0) + + # Sample source_latents from the posterior distribution. + prev_source_latents = posterior_sample( + self.scheduler, source_latents, t, clean_latents, generator=generator, **extra_step_kwargs + ) + # Compute noise. + noise = compute_noise( + self.scheduler, prev_source_latents, source_latents, t, source_noise_pred, **extra_step_kwargs + ) + source_latents = prev_source_latents + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, t, latents, variance_noise=noise, **extra_step_kwargs + ).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 9. Post-processing + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_onnx_stable_diffusion_inpaint_legacy.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_onnx_stable_diffusion_inpaint_legacy.py new file mode 100755 index 0000000..0aa5e68 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_onnx_stable_diffusion_inpaint_legacy.py @@ -0,0 +1,542 @@ +import inspect +from typing import Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTokenizer + +from ....configuration_utils import FrozenDict +from ....schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler +from ....utils import deprecate, logging +from ...onnx_utils import ORT_TO_NP_TYPE, OnnxRuntimeModel +from ...pipeline_utils import DiffusionPipeline +from ...stable_diffusion.pipeline_output import StableDiffusionPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def preprocess(image): + w, h = image.size + w, h = (x - x % 32 for x in (w, h)) # resize to integer multiple of 32 + image = image.resize((w, h), resample=PIL.Image.LANCZOS) + image = np.array(image).astype(np.float32) / 255.0 + image = image[None].transpose(0, 3, 1, 2) + return 2.0 * image - 1.0 + + +def preprocess_mask(mask, scale_factor=8): + mask = mask.convert("L") + w, h = mask.size + w, h = (x - x % 32 for x in (w, h)) # resize to integer multiple of 32 + mask = mask.resize((w // scale_factor, h // scale_factor), resample=PIL.Image.NEAREST) + mask = np.array(mask).astype(np.float32) / 255.0 + mask = np.tile(mask, (4, 1, 1)) + mask = mask[None].transpose(0, 1, 2, 3) # what does this step do? + mask = 1 - mask # repaint white, keep black + return mask + + +class OnnxStableDiffusionInpaintPipelineLegacy(DiffusionPipeline): + r""" + Pipeline for text-guided image inpainting using Stable Diffusion. This is a *legacy feature* for Onnx pipelines to + provide compatibility with StableDiffusionInpaintPipelineLegacy and may be removed in the future. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details. + feature_extractor ([`CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `safety_checker`. + """ + + _optional_components = ["safety_checker", "feature_extractor"] + _is_onnx = True + + vae_encoder: OnnxRuntimeModel + vae_decoder: OnnxRuntimeModel + text_encoder: OnnxRuntimeModel + tokenizer: CLIPTokenizer + unet: OnnxRuntimeModel + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler] + safety_checker: OnnxRuntimeModel + feature_extractor: CLIPImageProcessor + + def __init__( + self, + vae_encoder: OnnxRuntimeModel, + vae_decoder: OnnxRuntimeModel, + text_encoder: OnnxRuntimeModel, + tokenizer: CLIPTokenizer, + unet: OnnxRuntimeModel, + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + safety_checker: OnnxRuntimeModel, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae_encoder=vae_encoder, + vae_decoder=vae_decoder, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_onnx_stable_diffusion.OnnxStableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt: Union[str, List[str]], + num_images_per_prompt: Optional[int], + do_classifier_free_guidance: bool, + negative_prompt: Optional[str], + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`): + prompt to be encoded + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + prompt_embeds (`np.ndarray`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`np.ndarray`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + """ + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="max_length", return_tensors="np").input_ids + + if not np.array_equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder(input_ids=text_input_ids.astype(np.int32))[0] + + prompt_embeds = np.repeat(prompt_embeds, num_images_per_prompt, axis=0) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] * batch_size + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="np", + ) + negative_prompt_embeds = self.text_encoder(input_ids=uncond_input.input_ids.astype(np.int32))[0] + + if do_classifier_free_guidance: + negative_prompt_embeds = np.repeat(negative_prompt_embeds, num_images_per_prompt, axis=0) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = np.concatenate([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + def check_inputs( + self, + prompt, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def __call__( + self, + prompt: Union[str, List[str]], + image: Union[np.ndarray, PIL.Image.Image] = None, + mask_image: Union[np.ndarray, PIL.Image.Image] = None, + strength: float = 0.8, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: Optional[float] = 0.0, + generator: Optional[np.random.RandomState] = None, + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, np.ndarray], None]] = None, + callback_steps: int = 1, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`nd.ndarray` or `PIL.Image.Image`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. This is the image whose masked region will be inpainted. + mask_image (`nd.ndarray` or `PIL.Image.Image`): + `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be + replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a + PIL image, it will be converted to a single channel (luminance) before use. If it's a tensor, it should + contain one color channel (L) instead of 3, so the expected shape would be `(B, H, W, 1)`.uu + strength (`float`, *optional*, defaults to 0.8): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter will be modulated by `strength`. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (?) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`np.random.RandomState`, *optional*): + A np.random.RandomState to make generation deterministic. + prompt_embeds (`np.ndarray`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`np.ndarray`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: np.ndarray)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. + When returning a tuple, the first element is a list with the generated images, and the second element is a + list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, according to the `safety_checker`. + """ + + # check inputs. Raise error if not correct + self.check_inputs(prompt, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds) + + # define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if generator is None: + generator = np.random + + # set timesteps + self.scheduler.set_timesteps(num_inference_steps) + + if isinstance(image, PIL.Image.Image): + image = preprocess(image) + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + prompt_embeds = self._encode_prompt( + prompt, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + latents_dtype = prompt_embeds.dtype + image = image.astype(latents_dtype) + + # encode the init image into latents and scale the latents + init_latents = self.vae_encoder(sample=image)[0] + init_latents = 0.18215 * init_latents + + # Expand init_latents for batch_size and num_images_per_prompt + init_latents = np.concatenate([init_latents] * num_images_per_prompt, axis=0) + init_latents_orig = init_latents + + # preprocess mask + if not isinstance(mask_image, np.ndarray): + mask_image = preprocess_mask(mask_image, 8) + mask_image = mask_image.astype(latents_dtype) + mask = np.concatenate([mask_image] * num_images_per_prompt, axis=0) + + # check sizes + if not mask.shape == init_latents.shape: + raise ValueError("The mask and image should be the same size!") + + # get the original timestep using init_timestep + offset = self.scheduler.config.get("steps_offset", 0) + init_timestep = int(num_inference_steps * strength) + offset + init_timestep = min(init_timestep, num_inference_steps) + + timesteps = self.scheduler.timesteps.numpy()[-init_timestep] + timesteps = np.array([timesteps] * batch_size * num_images_per_prompt) + + # add noise to latents using the timesteps + noise = generator.randn(*init_latents.shape).astype(latents_dtype) + init_latents = self.scheduler.add_noise( + torch.from_numpy(init_latents), torch.from_numpy(noise), torch.from_numpy(timesteps) + ) + init_latents = init_latents.numpy() + + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (?) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to ? in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + latents = init_latents + + t_start = max(num_inference_steps - init_timestep + offset, 0) + timesteps = self.scheduler.timesteps[t_start:].numpy() + timestep_dtype = next( + (input.type for input in self.unet.model.get_inputs() if input.name == "timestep"), "tensor(float)" + ) + timestep_dtype = ORT_TO_NP_TYPE[timestep_dtype] + + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = np.concatenate([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + timestep = np.array([t], dtype=timestep_dtype) + noise_pred = self.unet(sample=latent_model_input, timestep=timestep, encoder_hidden_states=prompt_embeds)[ + 0 + ] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = np.split(noise_pred, 2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + torch.from_numpy(noise_pred), t, torch.from_numpy(latents), **extra_step_kwargs + ).prev_sample + + latents = latents.numpy() + + init_latents_proper = self.scheduler.add_noise( + torch.from_numpy(init_latents_orig), torch.from_numpy(noise), torch.from_numpy(np.array([t])) + ) + + init_latents_proper = init_latents_proper.numpy() + + latents = (init_latents_proper * mask) + (latents * (1 - mask)) + + # call the callback, if provided + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + latents = 1 / 0.18215 * latents + # image = self.vae_decoder(latent_sample=latents)[0] + # it seems likes there is a strange result for using half-precision vae decoder if batchsize>1 + image = np.concatenate( + [self.vae_decoder(latent_sample=latents[i : i + 1])[0] for i in range(latents.shape[0])] + ) + + image = np.clip(image / 2 + 0.5, 0, 1) + image = image.transpose((0, 2, 3, 1)) + + if self.safety_checker is not None: + safety_checker_input = self.feature_extractor( + self.numpy_to_pil(image), return_tensors="np" + ).pixel_values.astype(image.dtype) + # There will throw an error if use safety_checker batchsize>1 + images, has_nsfw_concept = [], [] + for i in range(image.shape[0]): + image_i, has_nsfw_concept_i = self.safety_checker( + clip_input=safety_checker_input[i : i + 1], images=image[i : i + 1] + ) + images.append(image_i) + has_nsfw_concept.append(has_nsfw_concept_i[0]) + image = np.concatenate(images) + else: + has_nsfw_concept = None + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py new file mode 100755 index 0000000..980adf2 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_inpaint_legacy.py @@ -0,0 +1,786 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from packaging import version +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ....configuration_utils import FrozenDict +from ....image_processor import VaeImageProcessor +from ....loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ....models import AutoencoderKL, UNet2DConditionModel +from ....models.lora import adjust_lora_scale_text_encoder +from ....schedulers import KarrasDiffusionSchedulers +from ....utils import PIL_INTERPOLATION, USE_PEFT_BACKEND, deprecate, logging, scale_lora_layers, unscale_lora_layers +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline +from ...stable_diffusion import StableDiffusionPipelineOutput +from ...stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) + + +def preprocess_image(image, batch_size): + w, h = image.size + w, h = (x - x % 8 for x in (w, h)) # resize to integer multiple of 8 + image = image.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]) + image = np.array(image).astype(np.float32) / 255.0 + image = np.vstack([image[None].transpose(0, 3, 1, 2)] * batch_size) + image = torch.from_numpy(image) + return 2.0 * image - 1.0 + + +def preprocess_mask(mask, batch_size, scale_factor=8): + if not isinstance(mask, torch.FloatTensor): + mask = mask.convert("L") + w, h = mask.size + w, h = (x - x % 8 for x in (w, h)) # resize to integer multiple of 8 + mask = mask.resize((w // scale_factor, h // scale_factor), resample=PIL_INTERPOLATION["nearest"]) + mask = np.array(mask).astype(np.float32) / 255.0 + mask = np.tile(mask, (4, 1, 1)) + mask = np.vstack([mask[None]] * batch_size) + mask = 1 - mask # repaint white, keep black + mask = torch.from_numpy(mask) + return mask + + else: + valid_mask_channel_sizes = [1, 3] + # if mask channel is fourth tensor dimension, permute dimensions to pytorch standard (B, C, H, W) + if mask.shape[3] in valid_mask_channel_sizes: + mask = mask.permute(0, 3, 1, 2) + elif mask.shape[1] not in valid_mask_channel_sizes: + raise ValueError( + f"Mask channel dimension of size in {valid_mask_channel_sizes} should be second or fourth dimension," + f" but received mask of shape {tuple(mask.shape)}" + ) + # (potentially) reduce mask channel dimension from 3 to 1 for broadcasting to latent shape + mask = mask.mean(dim=1, keepdim=True) + h, w = mask.shape[-2:] + h, w = (x - x % 8 for x in (h, w)) # resize to integer multiple of 8 + mask = torch.nn.functional.interpolate(mask, (h // scale_factor, w // scale_factor)) + return mask + + +class StableDiffusionInpaintPipelineLegacy( + DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, FromSingleFileMixin +): + r""" + Pipeline for text-guided image inpainting using Stable Diffusion. *This is an experimental feature*. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + In addition the pipeline inherits the following loading methods: + - *Textual-Inversion*: [`loaders.TextualInversionLoaderMixin.load_textual_inversion`] + - *LoRA*: [`loaders.LoraLoaderMixin.load_lora_weights`] + - *Ckpt*: [`loaders.FromSingleFileMixin.from_single_file`] + + as well as the following saving methods: + - *LoRA*: [`loaders.LoraLoaderMixin.save_lora_weights`] + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details. + feature_extractor ([`CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["feature_extractor"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + + deprecation_message = ( + f"The class {self.__class__} is deprecated and will be removed in v1.0.0. You can achieve exactly the same functionality" + "by loading your model into `StableDiffusionInpaintPipeline` instead. See https://github.com/huggingface/diffusers/pull/3533" + "for more information." + ) + deprecate("legacy is outdated", "1.0.0", deprecation_message, standard_warn=False) + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + strength, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + def prepare_latents(self, image, timestep, num_images_per_prompt, dtype, device, generator): + image = image.to(device=device, dtype=dtype) + init_latent_dist = self.vae.encode(image).latent_dist + init_latents = init_latent_dist.sample(generator=generator) + init_latents = self.vae.config.scaling_factor * init_latents + + # Expand init_latents for batch_size and num_images_per_prompt + init_latents = torch.cat([init_latents] * num_images_per_prompt, dim=0) + init_latents_orig = init_latents + + # add noise to latents using the timesteps + noise = randn_tensor(init_latents.shape, generator=generator, device=device, dtype=dtype) + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + latents = init_latents + return latents, init_latents_orig, noise + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: Union[torch.FloatTensor, PIL.Image.Image] = None, + mask_image: Union[torch.FloatTensor, PIL.Image.Image] = None, + strength: float = 0.8, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + add_predicted_noise: Optional[bool] = False, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + image (`torch.FloatTensor` or `PIL.Image.Image`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. This is the image whose masked region will be inpainted. + mask_image (`torch.FloatTensor` or `PIL.Image.Image`): + `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be + replaced by noise and therefore repainted, while black pixels will be preserved. If `mask_image` is a + PIL image, it will be converted to a single channel (luminance) before use. If mask is a tensor, the + expected shape should be either `(B, H, W, C)` or `(B, C, H, W)`, where C is 1 or 3. + strength (`float`, *optional*, defaults to 0.8): + Conceptually, indicates how much to inpaint the masked area. Must be between 0 and 1. When `strength` + is 1, the denoising process will be run on the masked area for the full number of iterations specified + in `num_inference_steps`. `image` will be used as a reference for the masked area, adding more noise to + that region the larger the `strength`. If `strength` is 0, no inpainting will occur. + num_inference_steps (`int`, *optional*, defaults to 50): + The reference number of denoising steps. More denoising steps usually lead to a higher quality image at + the expense of slower inference. This parameter will be modulated by `strength`, as explained above. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` + is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + add_predicted_noise (`bool`, *optional*, defaults to True): + Use predicted noise instead of random noise when constructing noisy versions of the original image in + the reverse diffusion process + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. + When returning a tuple, the first element is a list with the generated images, and the second element is a + list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, according to the `safety_checker`. + """ + # 1. Check inputs + self.check_inputs(prompt, strength, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Preprocess image and mask + if not isinstance(image, torch.FloatTensor): + image = preprocess_image(image, batch_size) + + mask_image = preprocess_mask(mask_image, batch_size, self.vae_scale_factor) + + # 5. set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + + # 6. Prepare latent variables + # encode the init image into latents and scale the latents + latents, init_latents_orig, noise = self.prepare_latents( + image, latent_timestep, num_images_per_prompt, prompt_embeds.dtype, device, generator + ) + + # 7. Prepare mask latent + mask = mask_image.to(device=device, dtype=latents.dtype) + mask = torch.cat([mask] * num_images_per_prompt) + + # 8. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 9. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + # masking + if add_predicted_noise: + init_latents_proper = self.scheduler.add_noise( + init_latents_orig, noise_pred_uncond, torch.tensor([t]) + ) + else: + init_latents_proper = self.scheduler.add_noise(init_latents_orig, noise, torch.tensor([t])) + + latents = (init_latents_proper * mask) + (latents * (1 - mask)) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # use original latents corresponding to unmasked portions of the image + latents = (init_latents_orig * mask) + (latents * (1 - mask)) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py new file mode 100755 index 0000000..dee93fc --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_model_editing.py @@ -0,0 +1,824 @@ +# Copyright 2024 TIME Authors and The HuggingFace Team. All rights reserved." +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer + +from ....image_processor import VaeImageProcessor +from ....loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ....models import AutoencoderKL, UNet2DConditionModel +from ....models.lora import adjust_lora_scale_text_encoder +from ....schedulers import PNDMScheduler +from ....schedulers.scheduling_utils import SchedulerMixin +from ....utils import USE_PEFT_BACKEND, deprecate, logging, scale_lora_layers, unscale_lora_layers +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ...stable_diffusion.pipeline_output import StableDiffusionPipelineOutput +from ...stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +AUGS_CONST = ["A photo of ", "An image of ", "A picture of "] + + +class StableDiffusionModelEditingPipeline( + DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin +): + r""" + Pipeline for text-to-image model editing. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPFeatureExtractor`]): + A `CLIPFeatureExtractor` to extract features from generated images; used as inputs to the `safety_checker`. + with_to_k ([`bool`]): + Whether to edit the key projection matrices along with the value projection matrices. + with_augs ([`list`]): + Textual augmentations to apply while editing the text-to-image model. Set to `[]` for no augmentations. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: SchedulerMixin, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPFeatureExtractor, + requires_safety_checker: bool = True, + with_to_k: bool = True, + with_augs: list = AUGS_CONST, + ): + super().__init__() + + if isinstance(scheduler, PNDMScheduler): + logger.error("PNDMScheduler for this pipeline is currently not supported.") + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + self.with_to_k = with_to_k + self.with_augs = with_augs + + # get cross-attention layers + ca_layers = [] + + def append_ca(net_): + if net_.__class__.__name__ == "CrossAttention": + ca_layers.append(net_) + elif hasattr(net_, "children"): + for net__ in net_.children(): + append_ca(net__) + + # recursively find all cross-attention layers in unet + for net in self.unet.named_children(): + if "down" in net[0]: + append_ca(net[1]) + elif "up" in net[0]: + append_ca(net[1]) + elif "mid" in net[0]: + append_ca(net[1]) + + # get projection matrices + self.ca_clip_layers = [l for l in ca_layers if l.to_v.in_features == 768] + self.projection_matrices = [l.to_v for l in self.ca_clip_layers] + self.og_matrices = [copy.deepcopy(l.to_v) for l in self.ca_clip_layers] + if self.with_to_k: + self.projection_matrices = self.projection_matrices + [l.to_k for l in self.ca_clip_layers] + self.og_matrices = self.og_matrices + [copy.deepcopy(l.to_k) for l in self.ca_clip_layers] + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + def edit_model( + self, + source_prompt: str, + destination_prompt: str, + lamb: float = 0.1, + restart_params: bool = True, + ): + r""" + Apply model editing via closed-form solution (see Eq. 5 in the TIME [paper](https://arxiv.org/abs/2303.08084)). + + Args: + source_prompt (`str`): + The source prompt containing the concept to be edited. + destination_prompt (`str`): + The destination prompt. Must contain all words from `source_prompt` with additional ones to specify the + target edit. + lamb (`float`, *optional*, defaults to 0.1): + The lambda parameter specifying the regularization intesity. Smaller values increase the editing power. + restart_params (`bool`, *optional*, defaults to True): + Restart the model parameters to their pre-trained version before editing. This is done to avoid edit + compounding. When it is `False`, edits accumulate. + """ + + # restart LDM parameters + if restart_params: + num_ca_clip_layers = len(self.ca_clip_layers) + for idx_, l in enumerate(self.ca_clip_layers): + l.to_v = copy.deepcopy(self.og_matrices[idx_]) + self.projection_matrices[idx_] = l.to_v + if self.with_to_k: + l.to_k = copy.deepcopy(self.og_matrices[num_ca_clip_layers + idx_]) + self.projection_matrices[num_ca_clip_layers + idx_] = l.to_k + + # set up sentences + old_texts = [source_prompt] + new_texts = [destination_prompt] + # add augmentations + base = old_texts[0] if old_texts[0][0:1] != "A" else "a" + old_texts[0][1:] + for aug in self.with_augs: + old_texts.append(aug + base) + base = new_texts[0] if new_texts[0][0:1] != "A" else "a" + new_texts[0][1:] + for aug in self.with_augs: + new_texts.append(aug + base) + + # prepare input k* and v* + old_embs, new_embs = [], [] + for old_text, new_text in zip(old_texts, new_texts): + text_input = self.tokenizer( + [old_text, new_text], + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_embeddings = self.text_encoder(text_input.input_ids.to(self.device))[0] + old_emb, new_emb = text_embeddings + old_embs.append(old_emb) + new_embs.append(new_emb) + + # identify corresponding destinations for each token in old_emb + idxs_replaces = [] + for old_text, new_text in zip(old_texts, new_texts): + tokens_a = self.tokenizer(old_text).input_ids + tokens_b = self.tokenizer(new_text).input_ids + tokens_a = [self.tokenizer.encode("a ")[1] if self.tokenizer.decode(t) == "an" else t for t in tokens_a] + tokens_b = [self.tokenizer.encode("a ")[1] if self.tokenizer.decode(t) == "an" else t for t in tokens_b] + num_orig_tokens = len(tokens_a) + idxs_replace = [] + j = 0 + for i in range(num_orig_tokens): + curr_token = tokens_a[i] + while tokens_b[j] != curr_token: + j += 1 + idxs_replace.append(j) + j += 1 + while j < 77: + idxs_replace.append(j) + j += 1 + while len(idxs_replace) < 77: + idxs_replace.append(76) + idxs_replaces.append(idxs_replace) + + # prepare batch: for each pair of setences, old context and new values + contexts, valuess = [], [] + for old_emb, new_emb, idxs_replace in zip(old_embs, new_embs, idxs_replaces): + context = old_emb.detach() + values = [] + with torch.no_grad(): + for layer in self.projection_matrices: + values.append(layer(new_emb[idxs_replace]).detach()) + contexts.append(context) + valuess.append(values) + + # edit the model + for layer_num in range(len(self.projection_matrices)): + # mat1 = \lambda W + \sum{v k^T} + mat1 = lamb * self.projection_matrices[layer_num].weight + + # mat2 = \lambda I + \sum{k k^T} + mat2 = lamb * torch.eye( + self.projection_matrices[layer_num].weight.shape[1], + device=self.projection_matrices[layer_num].weight.device, + ) + + # aggregate sums for mat1, mat2 + for context, values in zip(contexts, valuess): + context_vector = context.reshape(context.shape[0], context.shape[1], 1) + context_vector_T = context.reshape(context.shape[0], 1, context.shape[1]) + value_vector = values[layer_num].reshape(values[layer_num].shape[0], values[layer_num].shape[1], 1) + for_mat1 = (value_vector @ context_vector_T).sum(dim=0) + for_mat2 = (context_vector @ context_vector_T).sum(dim=0) + mat1 += for_mat1 + mat2 += for_mat2 + + # update projection matrix + self.projection_matrices[layer_num].weight = torch.nn.Parameter(mat1 @ torch.inverse(mat2)) + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + + Examples: + + ```py + >>> import torch + >>> from diffusers import StableDiffusionModelEditingPipeline + + >>> model_ckpt = "CompVis/stable-diffusion-v1-4" + >>> pipe = StableDiffusionModelEditingPipeline.from_pretrained(model_ckpt) + + >>> pipe = pipe.to("cuda") + + >>> source_prompt = "A pack of roses" + >>> destination_prompt = "A pack of blue roses" + >>> pipe.edit_model(source_prompt, destination_prompt) + + >>> prompt = "A field of roses" + >>> image = pipe(prompt).images[0] + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py new file mode 100755 index 0000000..ddc866e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_paradigms.py @@ -0,0 +1,786 @@ +# Copyright 2024 ParaDiGMS authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ....image_processor import VaeImageProcessor +from ....loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ....models import AutoencoderKL, UNet2DConditionModel +from ....models.lora import adjust_lora_scale_text_encoder +from ....schedulers import KarrasDiffusionSchedulers +from ....utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ...stable_diffusion.pipeline_output import StableDiffusionPipelineOutput +from ...stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import DDPMParallelScheduler + >>> from diffusers import StableDiffusionParadigmsPipeline + + >>> scheduler = DDPMParallelScheduler.from_pretrained("runwayml/stable-diffusion-v1-5", subfolder="scheduler") + + >>> pipe = StableDiffusionParadigmsPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", scheduler=scheduler, torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> ngpu, batch_per_device = torch.cuda.device_count(), 5 + >>> pipe.wrapped_unet = torch.nn.DataParallel(pipe.unet, device_ids=[d for d in range(ngpu)]) + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt, parallel=ngpu * batch_per_device, num_inference_steps=1000).images[0] + ``` +""" + + +class StableDiffusionParadigmsPipeline( + DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin, FromSingleFileMixin +): + r""" + Pipeline for text-to-image generation using a parallelized version of Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # attribute to wrap the unet with torch.nn.DataParallel when running multiple denoising steps on multiple GPUs + self.wrapped_unet = self.unet + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def _cumsum(self, input, dim, debug=False): + if debug: + # cumsum_cuda_kernel does not have a deterministic implementation + # so perform cumsum on cpu for debugging purposes + return torch.cumsum(input.cpu().float(), dim=dim).to(input.device) + else: + return torch.cumsum(input, dim=dim) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + parallel: int = 10, + tolerance: float = 0.1, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + debug: bool = False, + clip_skip: int = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + parallel (`int`, *optional*, defaults to 10): + The batch size to use when doing parallel sampling. More parallelism may lead to faster inference but + requires higher memory usage and can also require more total FLOPs. + tolerance (`float`, *optional*, defaults to 0.1): + The error tolerance for determining when to slide the batch window forward for parallel sampling. Lower + tolerance usually leads to less or no degradation. Higher tolerance is faster but can risk degradation + of sample quality. The tolerance is specified as a ratio of the scheduler's noise magnitude. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + debug (`bool`, *optional*, defaults to `False`): + Whether or not to run in debug mode. In debug mode, `torch.cumsum` is evaluated using the CPU. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + extra_step_kwargs.pop("generator", None) + + # # 7. Denoising loop + scheduler = self.scheduler + parallel = min(parallel, len(scheduler.timesteps)) + + begin_idx = 0 + end_idx = parallel + latents_time_evolution_buffer = torch.stack([latents] * (len(scheduler.timesteps) + 1)) + + # We must make sure the noise of stochastic schedulers such as DDPM is sampled only once per timestep. + # Sampling inside the parallel denoising loop will mess this up, so we pre-sample the noise vectors outside the denoising loop. + noise_array = torch.zeros_like(latents_time_evolution_buffer) + for j in range(len(scheduler.timesteps)): + base_noise = randn_tensor( + shape=latents.shape, generator=generator, device=latents.device, dtype=prompt_embeds.dtype + ) + noise = (self.scheduler._get_variance(scheduler.timesteps[j]) ** 0.5) * base_noise + noise_array[j] = noise.clone() + + # We specify the error tolerance as a ratio of the scheduler's noise magnitude. We similarly compute the error tolerance + # outside of the denoising loop to avoid recomputing it at every step. + # We will be dividing the norm of the noise, so we store its inverse here to avoid a division at every step. + inverse_variance_norm = 1.0 / torch.tensor( + [scheduler._get_variance(scheduler.timesteps[j]) for j in range(len(scheduler.timesteps))] + [0] + ).to(noise_array.device) + latent_dim = noise_array[0, 0].numel() + inverse_variance_norm = inverse_variance_norm[:, None] / latent_dim + + scaled_tolerance = tolerance**2 + + with self.progress_bar(total=num_inference_steps) as progress_bar: + steps = 0 + while begin_idx < len(scheduler.timesteps): + # these have shape (parallel_dim, 2*batch_size, ...) + # parallel_len is at most parallel, but could be less if we are at the end of the timesteps + # we are processing batch window of timesteps spanning [begin_idx, end_idx) + parallel_len = end_idx - begin_idx + + block_prompt_embeds = torch.stack([prompt_embeds] * parallel_len) + block_latents = latents_time_evolution_buffer[begin_idx:end_idx] + block_t = scheduler.timesteps[begin_idx:end_idx, None].repeat(1, batch_size * num_images_per_prompt) + t_vec = block_t + if do_classifier_free_guidance: + t_vec = t_vec.repeat(1, 2) + + # expand the latents if we are doing classifier free guidance + latent_model_input = ( + torch.cat([block_latents] * 2, dim=1) if do_classifier_free_guidance else block_latents + ) + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t_vec) + + # if parallel_len is small, no need to use multiple GPUs + net = self.wrapped_unet if parallel_len > 3 else self.unet + # predict the noise residual, shape is now [parallel_len * 2 * batch_size * num_images_per_prompt, ...] + model_output = net( + latent_model_input.flatten(0, 1), + t_vec.flatten(0, 1), + encoder_hidden_states=block_prompt_embeds.flatten(0, 1), + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + per_latent_shape = model_output.shape[1:] + if do_classifier_free_guidance: + model_output = model_output.reshape( + parallel_len, 2, batch_size * num_images_per_prompt, *per_latent_shape + ) + noise_pred_uncond, noise_pred_text = model_output[:, 0], model_output[:, 1] + model_output = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + model_output = model_output.reshape( + parallel_len * batch_size * num_images_per_prompt, *per_latent_shape + ) + + block_latents_denoise = scheduler.batch_step_no_noise( + model_output=model_output, + timesteps=block_t.flatten(0, 1), + sample=block_latents.flatten(0, 1), + **extra_step_kwargs, + ).reshape(block_latents.shape) + + # back to shape (parallel_dim, batch_size, ...) + # now we want to add the pre-sampled noise + # parallel sampling algorithm requires computing the cumulative drift from the beginning + # of the window, so we need to compute cumulative sum of the deltas and the pre-sampled noises. + delta = block_latents_denoise - block_latents + cumulative_delta = self._cumsum(delta, dim=0, debug=debug) + cumulative_noise = self._cumsum(noise_array[begin_idx:end_idx], dim=0, debug=debug) + + # if we are using an ODE-like scheduler (like DDIM), we don't want to add noise + if scheduler._is_ode_scheduler: + cumulative_noise = 0 + + block_latents_new = ( + latents_time_evolution_buffer[begin_idx][None,] + cumulative_delta + cumulative_noise + ) + cur_error = torch.linalg.norm( + (block_latents_new - latents_time_evolution_buffer[begin_idx + 1 : end_idx + 1]).reshape( + parallel_len, batch_size * num_images_per_prompt, -1 + ), + dim=-1, + ).pow(2) + error_ratio = cur_error * inverse_variance_norm[begin_idx + 1 : end_idx + 1] + + # find the first index of the vector error_ratio that is greater than error tolerance + # we can shift the window for the next iteration up to this index + error_ratio = torch.nn.functional.pad( + error_ratio, (0, 0, 0, 1), value=1e9 + ) # handle the case when everything is below ratio, by padding the end of parallel_len dimension + any_error_at_time = torch.max(error_ratio > scaled_tolerance, dim=1).values.int() + ind = torch.argmax(any_error_at_time).item() + + # compute the new begin and end idxs for the window + new_begin_idx = begin_idx + min(1 + ind, parallel) + new_end_idx = min(new_begin_idx + parallel, len(scheduler.timesteps)) + + # store the computed latents for the current window in the global buffer + latents_time_evolution_buffer[begin_idx + 1 : end_idx + 1] = block_latents_new + # initialize the new sliding window latents with the end of the current window, + # should be better than random initialization + latents_time_evolution_buffer[end_idx : new_end_idx + 1] = latents_time_evolution_buffer[end_idx][ + None, + ] + + steps += 1 + + progress_bar.update(new_begin_idx - begin_idx) + if callback is not None and steps % callback_steps == 0: + callback(begin_idx, block_t[begin_idx], latents_time_evolution_buffer[begin_idx]) + + begin_idx = new_begin_idx + end_idx = new_end_idx + + latents = latents_time_evolution_buffer[-1] + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py new file mode 100755 index 0000000..c819e57 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stable_diffusion_variants/pipeline_stable_diffusion_pix2pix_zero.py @@ -0,0 +1,1304 @@ +# Copyright 2024 Pix2Pix Zero Authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from transformers import ( + BlipForConditionalGeneration, + BlipProcessor, + CLIPImageProcessor, + CLIPTextModel, + CLIPTokenizer, +) + +from ....image_processor import PipelineImageInput, VaeImageProcessor +from ....loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ....models import AutoencoderKL, UNet2DConditionModel +from ....models.attention_processor import Attention +from ....models.lora import adjust_lora_scale_text_encoder +from ....schedulers import DDIMScheduler, DDPMScheduler, EulerAncestralDiscreteScheduler, LMSDiscreteScheduler +from ....schedulers.scheduling_ddim_inverse import DDIMInverseScheduler +from ....utils import ( + PIL_INTERPOLATION, + USE_PEFT_BACKEND, + BaseOutput, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ...stable_diffusion.pipeline_output import StableDiffusionPipelineOutput +from ...stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class Pix2PixInversionPipelineOutput(BaseOutput, TextualInversionLoaderMixin): + """ + Output class for Stable Diffusion pipelines. + + Args: + latents (`torch.FloatTensor`) + inverted latents tensor + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. + """ + + latents: torch.FloatTensor + images: Union[List[PIL.Image.Image], np.ndarray] + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import requests + >>> import torch + + >>> from diffusers import DDIMScheduler, StableDiffusionPix2PixZeroPipeline + + + >>> def download(embedding_url, local_filepath): + ... r = requests.get(embedding_url) + ... with open(local_filepath, "wb") as f: + ... f.write(r.content) + + + >>> model_ckpt = "CompVis/stable-diffusion-v1-4" + >>> pipeline = StableDiffusionPix2PixZeroPipeline.from_pretrained(model_ckpt, torch_dtype=torch.float16) + >>> pipeline.scheduler = DDIMScheduler.from_config(pipeline.scheduler.config) + >>> pipeline.to("cuda") + + >>> prompt = "a high resolution painting of a cat in the style of van gough" + >>> source_emb_url = "https://hf.co/datasets/sayakpaul/sample-datasets/resolve/main/cat.pt" + >>> target_emb_url = "https://hf.co/datasets/sayakpaul/sample-datasets/resolve/main/dog.pt" + + >>> for url in [source_emb_url, target_emb_url]: + ... download(url, url.split("/")[-1]) + + >>> src_embeds = torch.load(source_emb_url.split("/")[-1]) + >>> target_embeds = torch.load(target_emb_url.split("/")[-1]) + >>> images = pipeline( + ... prompt, + ... source_embeds=src_embeds, + ... target_embeds=target_embeds, + ... num_inference_steps=50, + ... cross_attention_guidance_amount=0.15, + ... ).images + + >>> images[0].save("edited_image_dog.png") + ``` +""" + +EXAMPLE_INVERT_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from transformers import BlipForConditionalGeneration, BlipProcessor + >>> from diffusers import DDIMScheduler, DDIMInverseScheduler, StableDiffusionPix2PixZeroPipeline + + >>> import requests + >>> from PIL import Image + + >>> captioner_id = "Salesforce/blip-image-captioning-base" + >>> processor = BlipProcessor.from_pretrained(captioner_id) + >>> model = BlipForConditionalGeneration.from_pretrained( + ... captioner_id, torch_dtype=torch.float16, low_cpu_mem_usage=True + ... ) + + >>> sd_model_ckpt = "CompVis/stable-diffusion-v1-4" + >>> pipeline = StableDiffusionPix2PixZeroPipeline.from_pretrained( + ... sd_model_ckpt, + ... caption_generator=model, + ... caption_processor=processor, + ... torch_dtype=torch.float16, + ... safety_checker=None, + ... ) + + >>> pipeline.scheduler = DDIMScheduler.from_config(pipeline.scheduler.config) + >>> pipeline.inverse_scheduler = DDIMInverseScheduler.from_config(pipeline.scheduler.config) + >>> pipeline.enable_model_cpu_offload() + + >>> img_url = "https://github.com/pix2pixzero/pix2pix-zero/raw/main/assets/test_images/cats/cat_6.png" + + >>> raw_image = Image.open(requests.get(img_url, stream=True).raw).convert("RGB").resize((512, 512)) + >>> # generate caption + >>> caption = pipeline.generate_caption(raw_image) + + >>> # "a photography of a cat with flowers and dai dai daie - daie - daie kasaii" + >>> inv_latents = pipeline.invert(caption, image=raw_image).latents + >>> # we need to generate source and target embeds + + >>> source_prompts = ["a cat sitting on the street", "a cat playing in the field", "a face of a cat"] + + >>> target_prompts = ["a dog sitting on the street", "a dog playing in the field", "a face of a dog"] + + >>> source_embeds = pipeline.get_embeds(source_prompts) + >>> target_embeds = pipeline.get_embeds(target_prompts) + >>> # the latents can then be used to edit a real image + >>> # when using Stable Diffusion 2 or other models that use v-prediction + >>> # set `cross_attention_guidance_amount` to 0.01 or less to avoid input latent gradient explosion + + >>> image = pipeline( + ... caption, + ... source_embeds=source_embeds, + ... target_embeds=target_embeds, + ... num_inference_steps=50, + ... cross_attention_guidance_amount=0.15, + ... generator=generator, + ... latents=inv_latents, + ... negative_prompt=caption, + ... ).images[0] + >>> image.save("edited_image.png") + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.preprocess +def preprocess(image): + deprecation_message = "The preprocess method is deprecated and will be removed in diffusers 1.0.0. Please use VaeImageProcessor.preprocess(...) instead" + deprecate("preprocess", "1.0.0", deprecation_message, standard_warn=False) + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 8 for x in (w, h)) # resize to integer multiple of 8 + + image = [np.array(i.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + return image + + +def prepare_unet(unet: UNet2DConditionModel): + """Modifies the UNet (`unet`) to perform Pix2Pix Zero optimizations.""" + pix2pix_zero_attn_procs = {} + for name in unet.attn_processors.keys(): + module_name = name.replace(".processor", "") + module = unet.get_submodule(module_name) + if "attn2" in name: + pix2pix_zero_attn_procs[name] = Pix2PixZeroAttnProcessor(is_pix2pix_zero=True) + module.requires_grad_(True) + else: + pix2pix_zero_attn_procs[name] = Pix2PixZeroAttnProcessor(is_pix2pix_zero=False) + module.requires_grad_(False) + + unet.set_attn_processor(pix2pix_zero_attn_procs) + return unet + + +class Pix2PixZeroL2Loss: + def __init__(self): + self.loss = 0.0 + + def compute_loss(self, predictions, targets): + self.loss += ((predictions - targets) ** 2).sum((1, 2)).mean(0) + + +class Pix2PixZeroAttnProcessor: + """An attention processor class to store the attention weights. + In Pix2Pix Zero, it happens during computations in the cross-attention blocks.""" + + def __init__(self, is_pix2pix_zero=False): + self.is_pix2pix_zero = is_pix2pix_zero + if self.is_pix2pix_zero: + self.reference_cross_attn_map = {} + + def __call__( + self, + attn: Attention, + hidden_states, + encoder_hidden_states=None, + attention_mask=None, + timestep=None, + loss=None, + ): + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + if self.is_pix2pix_zero and timestep is not None: + # new bookkeeping to save the attention weights. + if loss is None: + self.reference_cross_attn_map[timestep.item()] = attention_probs.detach().cpu() + # compute loss + elif loss is not None: + prev_attn_probs = self.reference_cross_attn_map.pop(timestep.item()) + loss.compute_loss(attention_probs, prev_attn_probs.to(attention_probs.device)) + + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +class StableDiffusionPix2PixZeroPipeline(DiffusionPipeline, StableDiffusionMixin): + r""" + Pipeline for pixel-level image editing using Pix2Pix Zero. Based on Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], [`EulerAncestralDiscreteScheduler`], or [`DDPMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details. + feature_extractor ([`CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `safety_checker`. + requires_safety_checker (bool): + Whether the pipeline requires a safety checker. We recommend setting it to True if you're using the + pipeline publicly. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = [ + "safety_checker", + "feature_extractor", + "caption_generator", + "caption_processor", + "inverse_scheduler", + ] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: Union[DDPMScheduler, DDIMScheduler, EulerAncestralDiscreteScheduler, LMSDiscreteScheduler], + feature_extractor: CLIPImageProcessor, + safety_checker: StableDiffusionSafetyChecker, + inverse_scheduler: DDIMInverseScheduler, + caption_generator: BlipForConditionalGeneration, + caption_processor: BlipProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + caption_processor=caption_processor, + caption_generator=caption_generator, + inverse_scheduler=inverse_scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + source_embeds, + target_embeds, + callback_steps, + prompt_embeds=None, + ): + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if source_embeds is None and target_embeds is None: + raise ValueError("`source_embeds` and `target_embeds` cannot be undefined.") + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + def generate_caption(self, images): + """Generates caption for a given image.""" + text = "a photography of" + + prev_device = self.caption_generator.device + + device = self._execution_device + inputs = self.caption_processor(images, text, return_tensors="pt").to( + device=device, dtype=self.caption_generator.dtype + ) + self.caption_generator.to(device) + outputs = self.caption_generator.generate(**inputs, max_new_tokens=128) + + # offload caption generator + self.caption_generator.to(prev_device) + + caption = self.caption_processor.batch_decode(outputs, skip_special_tokens=True)[0] + return caption + + def construct_direction(self, embs_source: torch.Tensor, embs_target: torch.Tensor): + """Constructs the edit direction to steer the image generation process semantically.""" + return (embs_target.mean(0) - embs_source.mean(0)).unsqueeze(0) + + @torch.no_grad() + def get_embeds(self, prompt: List[str], batch_size: int = 16) -> torch.FloatTensor: + num_prompts = len(prompt) + embeds = [] + for i in range(0, num_prompts, batch_size): + prompt_slice = prompt[i : i + batch_size] + + input_ids = self.tokenizer( + prompt_slice, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ).input_ids + + input_ids = input_ids.to(self.text_encoder.device) + embeds.append(self.text_encoder(input_ids)[0]) + + return torch.cat(embeds, dim=0).mean(0)[None] + + def prepare_image_latents(self, image, batch_size, dtype, device, generator=None): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + if image.shape[1] == 4: + latents = image + + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if isinstance(generator, list): + latents = [ + self.vae.encode(image[i : i + 1]).latent_dist.sample(generator[i]) for i in range(batch_size) + ] + latents = torch.cat(latents, dim=0) + else: + latents = self.vae.encode(image).latent_dist.sample(generator) + + latents = self.vae.config.scaling_factor * latents + + if batch_size != latents.shape[0]: + if batch_size % latents.shape[0] == 0: + # expand image_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_latents_per_image = batch_size // latents.shape[0] + latents = torch.cat([latents] * additional_latents_per_image, dim=0) + else: + raise ValueError( + f"Cannot duplicate `image` of batch size {latents.shape[0]} to {batch_size} text prompts." + ) + else: + latents = torch.cat([latents], dim=0) + + return latents + + def get_epsilon(self, model_output: torch.Tensor, sample: torch.Tensor, timestep: int): + pred_type = self.inverse_scheduler.config.prediction_type + alpha_prod_t = self.inverse_scheduler.alphas_cumprod[timestep] + + beta_prod_t = 1 - alpha_prod_t + + if pred_type == "epsilon": + return model_output + elif pred_type == "sample": + return (sample - alpha_prod_t ** (0.5) * model_output) / beta_prod_t ** (0.5) + elif pred_type == "v_prediction": + return (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + else: + raise ValueError( + f"prediction_type given as {pred_type} must be one of `epsilon`, `sample`, or `v_prediction`" + ) + + def auto_corr_loss(self, hidden_states, generator=None): + reg_loss = 0.0 + for i in range(hidden_states.shape[0]): + for j in range(hidden_states.shape[1]): + noise = hidden_states[i : i + 1, j : j + 1, :, :] + while True: + roll_amount = torch.randint(noise.shape[2] // 2, (1,), generator=generator).item() + reg_loss += (noise * torch.roll(noise, shifts=roll_amount, dims=2)).mean() ** 2 + reg_loss += (noise * torch.roll(noise, shifts=roll_amount, dims=3)).mean() ** 2 + + if noise.shape[2] <= 8: + break + noise = F.avg_pool2d(noise, kernel_size=2) + return reg_loss + + def kl_divergence(self, hidden_states): + mean = hidden_states.mean() + var = hidden_states.var() + return var + mean**2 - 1 - torch.log(var + 1e-7) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + source_embeds: torch.Tensor = None, + target_embeds: torch.Tensor = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + cross_attention_guidance_amount: float = 0.1, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: Optional[int] = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + source_embeds (`torch.Tensor`): + Source concept embeddings. Generation of the embeddings as per the [original + paper](https://arxiv.org/abs/2302.03027). Used in discovering the edit direction. + target_embeds (`torch.Tensor`): + Target concept embeddings. Generation of the embeddings as per the [original + paper](https://arxiv.org/abs/2302.03027). Used in discovering the edit direction. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + cross_attention_guidance_amount (`float`, defaults to 0.1): + Amount of guidance needed from the reference cross-attention maps. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. + When returning a tuple, the first element is a list with the generated images, and the second element is a + list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, according to the `safety_checker`. + """ + # 0. Define the spatial resolutions. + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + source_embeds, + target_embeds, + callback_steps, + prompt_embeds, + ) + + # 3. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + if cross_attention_kwargs is None: + cross_attention_kwargs = {} + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Generate the inverted noise from the input image or any other image + # generated from the input prompt. + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + latents_init = latents.clone() + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 8. Rejig the UNet so that we can obtain the cross-attenion maps and + # use them for guiding the subsequent image generation. + self.unet = prepare_unet(self.unet) + + # 7. Denoising loop where we obtain the cross-attention maps. + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs={"timestep": t}, + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 8. Compute the edit directions. + edit_direction = self.construct_direction(source_embeds, target_embeds).to(prompt_embeds.device) + + # 9. Edit the prompt embeddings as per the edit directions discovered. + prompt_embeds_edit = prompt_embeds.clone() + prompt_embeds_edit[1:2] += edit_direction + + # 10. Second denoising loop to generate the edited image. + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + latents = latents_init + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # we want to learn the latent such that it steers the generation + # process towards the edited direction, so make the make initial + # noise learnable + x_in = latent_model_input.detach().clone() + x_in.requires_grad = True + + # optimizer + opt = torch.optim.SGD([x_in], lr=cross_attention_guidance_amount) + + with torch.enable_grad(): + # initialize loss + loss = Pix2PixZeroL2Loss() + + # predict the noise residual + noise_pred = self.unet( + x_in, + t, + encoder_hidden_states=prompt_embeds_edit.detach(), + cross_attention_kwargs={"timestep": t, "loss": loss}, + ).sample + + loss.loss.backward(retain_graph=False) + opt.step() + + # recompute the noise + noise_pred = self.unet( + x_in.detach(), + t, + encoder_hidden_states=prompt_embeds_edit, + cross_attention_kwargs={"timestep": None}, + ).sample + + latents = x_in.detach().chunk(2)[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_INVERT_DOC_STRING) + def invert( + self, + prompt: Optional[str] = None, + image: PipelineImageInput = None, + num_inference_steps: int = 50, + guidance_scale: float = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + cross_attention_guidance_amount: float = 0.1, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: Optional[int] = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + lambda_auto_corr: float = 20.0, + lambda_kl: float = 20.0, + num_reg_steps: int = 5, + num_auto_corr_rolls: int = 5, + ): + r""" + Function used to generate inverted latents given a prompt and image. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + image (`torch.FloatTensor` `np.ndarray`, `PIL.Image.Image`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, or tensor representing an image batch which will be used for conditioning. Can also accept + image latents as `image`, if passing latents directly, it will not be encoded again. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 1): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + cross_attention_guidance_amount (`float`, defaults to 0.1): + Amount of guidance needed from the reference cross-attention maps. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + lambda_auto_corr (`float`, *optional*, defaults to 20.0): + Lambda parameter to control auto correction + lambda_kl (`float`, *optional*, defaults to 20.0): + Lambda parameter to control Kullback–Leibler divergence output + num_reg_steps (`int`, *optional*, defaults to 5): + Number of regularization loss steps + num_auto_corr_rolls (`int`, *optional*, defaults to 5): + Number of auto correction roll steps + + Examples: + + Returns: + [`~pipelines.stable_diffusion.pipeline_stable_diffusion_pix2pix_zero.Pix2PixInversionPipelineOutput`] or + `tuple`: + [`~pipelines.stable_diffusion.pipeline_stable_diffusion_pix2pix_zero.Pix2PixInversionPipelineOutput`] if + `return_dict` is True, otherwise a `tuple. When returning a tuple, the first element is the inverted + latents tensor and then second is the corresponding decoded image. + """ + # 1. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + if cross_attention_kwargs is None: + cross_attention_kwargs = {} + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Preprocess image + image = self.image_processor.preprocess(image) + + # 4. Prepare latent variables + latents = self.prepare_image_latents(image, batch_size, self.vae.dtype, device, generator) + + # 5. Encode input prompt + num_images_per_prompt = 1 + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + prompt_embeds=prompt_embeds, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + self.inverse_scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.inverse_scheduler.timesteps + + # 6. Rejig the UNet so that we can obtain the cross-attenion maps and + # use them for guiding the subsequent image generation. + self.unet = prepare_unet(self.unet) + + # 7. Denoising loop where we obtain the cross-attention maps. + num_warmup_steps = len(timesteps) - num_inference_steps * self.inverse_scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.inverse_scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs={"timestep": t}, + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # regularization of the noise prediction + with torch.enable_grad(): + for _ in range(num_reg_steps): + if lambda_auto_corr > 0: + for _ in range(num_auto_corr_rolls): + var = torch.autograd.Variable(noise_pred.detach().clone(), requires_grad=True) + + # Derive epsilon from model output before regularizing to IID standard normal + var_epsilon = self.get_epsilon(var, latent_model_input.detach(), t) + + l_ac = self.auto_corr_loss(var_epsilon, generator=generator) + l_ac.backward() + + grad = var.grad.detach() / num_auto_corr_rolls + noise_pred = noise_pred - lambda_auto_corr * grad + + if lambda_kl > 0: + var = torch.autograd.Variable(noise_pred.detach().clone(), requires_grad=True) + + # Derive epsilon from model output before regularizing to IID standard normal + var_epsilon = self.get_epsilon(var, latent_model_input.detach(), t) + + l_kld = self.kl_divergence(var_epsilon) + l_kld.backward() + + grad = var.grad.detach() + noise_pred = noise_pred - lambda_kl * grad + + noise_pred = noise_pred.detach() + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.inverse_scheduler.step(noise_pred, t, latents).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ( + (i + 1) > num_warmup_steps and (i + 1) % self.inverse_scheduler.order == 0 + ): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + inverted_latents = latents.detach().clone() + + # 8. Post-processing + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (inverted_latents, image) + + return Pix2PixInversionPipelineOutput(latents=inverted_latents, images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stochastic_karras_ve/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stochastic_karras_ve/__init__.py new file mode 100755 index 0000000..15c9a8c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stochastic_karras_ve/__init__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from ....utils import DIFFUSERS_SLOW_IMPORT, _LazyModule + + +_import_structure = {"pipeline_stochastic_karras_ve": ["KarrasVePipeline"]} + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .pipeline_stochastic_karras_ve import KarrasVePipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stochastic_karras_ve/pipeline_stochastic_karras_ve.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stochastic_karras_ve/pipeline_stochastic_karras_ve.py new file mode 100755 index 0000000..023edb4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/stochastic_karras_ve/pipeline_stochastic_karras_ve.py @@ -0,0 +1,128 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List, Optional, Tuple, Union + +import torch + +from ....models import UNet2DModel +from ....schedulers import KarrasVeScheduler +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +class KarrasVePipeline(DiffusionPipeline): + r""" + Pipeline for unconditional image generation. + + Parameters: + unet ([`UNet2DModel`]): + A `UNet2DModel` to denoise the encoded image. + scheduler ([`KarrasVeScheduler`]): + A scheduler to be used in combination with `unet` to denoise the encoded image. + """ + + # add type hints for linting + unet: UNet2DModel + scheduler: KarrasVeScheduler + + def __init__(self, unet: UNet2DModel, scheduler: KarrasVeScheduler): + super().__init__() + self.register_modules(unet=unet, scheduler=scheduler) + + @torch.no_grad() + def __call__( + self, + batch_size: int = 1, + num_inference_steps: int = 50, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + **kwargs, + ) -> Union[Tuple, ImagePipelineOutput]: + r""" + The call function to the pipeline for generation. + + Args: + batch_size (`int`, *optional*, defaults to 1): + The number of images to generate. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`ImagePipelineOutput`] instead of a plain tuple. + + Example: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images. + """ + + img_size = self.unet.config.sample_size + shape = (batch_size, 3, img_size, img_size) + + model = self.unet + + # sample x_0 ~ N(0, sigma_0^2 * I) + sample = randn_tensor(shape, generator=generator, device=self.device) * self.scheduler.init_noise_sigma + + self.scheduler.set_timesteps(num_inference_steps) + + for t in self.progress_bar(self.scheduler.timesteps): + # here sigma_t == t_i from the paper + sigma = self.scheduler.schedule[t] + sigma_prev = self.scheduler.schedule[t - 1] if t > 0 else 0 + + # 1. Select temporarily increased noise level sigma_hat + # 2. Add new noise to move from sample_i to sample_hat + sample_hat, sigma_hat = self.scheduler.add_noise_to_input(sample, sigma, generator=generator) + + # 3. Predict the noise residual given the noise magnitude `sigma_hat` + # The model inputs and output are adjusted by following eq. (213) in [1]. + model_output = (sigma_hat / 2) * model((sample_hat + 1) / 2, sigma_hat / 2).sample + + # 4. Evaluate dx/dt at sigma_hat + # 5. Take Euler step from sigma to sigma_prev + step_output = self.scheduler.step(model_output, sigma_hat, sigma_prev, sample_hat) + + if sigma_prev != 0: + # 6. Apply 2nd order correction + # The model inputs and output are adjusted by following eq. (213) in [1]. + model_output = (sigma_prev / 2) * model((step_output.prev_sample + 1) / 2, sigma_prev / 2).sample + step_output = self.scheduler.step_correct( + model_output, + sigma_hat, + sigma_prev, + sample_hat, + step_output.prev_sample, + step_output["derivative"], + ) + sample = step_output.prev_sample + + sample = (sample / 2 + 0.5).clamp(0, 1) + image = sample.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/__init__.py new file mode 100755 index 0000000..8ea6ef6 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/__init__.py @@ -0,0 +1,71 @@ +from typing import TYPE_CHECKING + +from ....utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + is_torch_available, + is_transformers_available, + is_transformers_version, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ....utils.dummy_torch_and_transformers_objects import ( + VersatileDiffusionDualGuidedPipeline, + VersatileDiffusionImageVariationPipeline, + VersatileDiffusionPipeline, + VersatileDiffusionTextToImagePipeline, + ) + + _dummy_objects.update( + { + "VersatileDiffusionDualGuidedPipeline": VersatileDiffusionDualGuidedPipeline, + "VersatileDiffusionImageVariationPipeline": VersatileDiffusionImageVariationPipeline, + "VersatileDiffusionPipeline": VersatileDiffusionPipeline, + "VersatileDiffusionTextToImagePipeline": VersatileDiffusionTextToImagePipeline, + } + ) +else: + _import_structure["modeling_text_unet"] = ["UNetFlatConditionModel"] + _import_structure["pipeline_versatile_diffusion"] = ["VersatileDiffusionPipeline"] + _import_structure["pipeline_versatile_diffusion_dual_guided"] = ["VersatileDiffusionDualGuidedPipeline"] + _import_structure["pipeline_versatile_diffusion_image_variation"] = ["VersatileDiffusionImageVariationPipeline"] + _import_structure["pipeline_versatile_diffusion_text_to_image"] = ["VersatileDiffusionTextToImagePipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ....utils.dummy_torch_and_transformers_objects import ( + VersatileDiffusionDualGuidedPipeline, + VersatileDiffusionImageVariationPipeline, + VersatileDiffusionPipeline, + VersatileDiffusionTextToImagePipeline, + ) + else: + from .pipeline_versatile_diffusion import VersatileDiffusionPipeline + from .pipeline_versatile_diffusion_dual_guided import VersatileDiffusionDualGuidedPipeline + from .pipeline_versatile_diffusion_image_variation import VersatileDiffusionImageVariationPipeline + from .pipeline_versatile_diffusion_text_to_image import VersatileDiffusionTextToImagePipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py new file mode 100755 index 0000000..62a3a87 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/modeling_text_unet.py @@ -0,0 +1,2508 @@ +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F + +from diffusers.utils import deprecate + +from ....configuration_utils import ConfigMixin, register_to_config +from ....models import ModelMixin +from ....models.activations import get_activation +from ....models.attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + Attention, + AttentionProcessor, + AttnAddedKVProcessor, + AttnAddedKVProcessor2_0, + AttnProcessor, +) +from ....models.embeddings import ( + GaussianFourierProjection, + ImageHintTimeEmbedding, + ImageProjection, + ImageTimeEmbedding, + TextImageProjection, + TextImageTimeEmbedding, + TextTimeEmbedding, + TimestepEmbedding, + Timesteps, +) +from ....models.resnet import ResnetBlockCondNorm2D +from ....models.transformers.dual_transformer_2d import DualTransformer2DModel +from ....models.transformers.transformer_2d import Transformer2DModel +from ....models.unets.unet_2d_condition import UNet2DConditionOutput +from ....utils import USE_PEFT_BACKEND, is_torch_version, logging, scale_lora_layers, unscale_lora_layers +from ....utils.torch_utils import apply_freeu + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def get_down_block( + down_block_type, + num_layers, + in_channels, + out_channels, + temb_channels, + add_downsample, + resnet_eps, + resnet_act_fn, + num_attention_heads, + transformer_layers_per_block, + attention_type, + attention_head_dim, + resnet_groups=None, + cross_attention_dim=None, + downsample_padding=None, + dual_cross_attention=False, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + resnet_time_scale_shift="default", + resnet_skip_time_act=False, + resnet_out_scale_factor=1.0, + cross_attention_norm=None, + dropout=0.0, +): + down_block_type = down_block_type[7:] if down_block_type.startswith("UNetRes") else down_block_type + if down_block_type == "DownBlockFlat": + return DownBlockFlat( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif down_block_type == "CrossAttnDownBlockFlat": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnDownBlockFlat") + return CrossAttnDownBlockFlat( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + dropout=dropout, + add_downsample=add_downsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + downsample_padding=downsample_padding, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + raise ValueError(f"{down_block_type} is not supported.") + + +def get_up_block( + up_block_type, + num_layers, + in_channels, + out_channels, + prev_output_channel, + temb_channels, + add_upsample, + resnet_eps, + resnet_act_fn, + num_attention_heads, + transformer_layers_per_block, + resolution_idx, + attention_type, + attention_head_dim, + resnet_groups=None, + cross_attention_dim=None, + dual_cross_attention=False, + use_linear_projection=False, + only_cross_attention=False, + upcast_attention=False, + resnet_time_scale_shift="default", + resnet_skip_time_act=False, + resnet_out_scale_factor=1.0, + cross_attention_norm=None, + dropout=0.0, +): + up_block_type = up_block_type[7:] if up_block_type.startswith("UNetRes") else up_block_type + if up_block_type == "UpBlockFlat": + return UpBlockFlat( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + elif up_block_type == "CrossAttnUpBlockFlat": + if cross_attention_dim is None: + raise ValueError("cross_attention_dim must be specified for CrossAttnUpBlockFlat") + return CrossAttnUpBlockFlat( + num_layers=num_layers, + in_channels=in_channels, + out_channels=out_channels, + prev_output_channel=prev_output_channel, + temb_channels=temb_channels, + dropout=dropout, + add_upsample=add_upsample, + resnet_eps=resnet_eps, + resnet_act_fn=resnet_act_fn, + resnet_groups=resnet_groups, + cross_attention_dim=cross_attention_dim, + num_attention_heads=num_attention_heads, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + ) + raise ValueError(f"{up_block_type} is not supported.") + + +class FourierEmbedder(nn.Module): + def __init__(self, num_freqs=64, temperature=100): + super().__init__() + + self.num_freqs = num_freqs + self.temperature = temperature + + freq_bands = temperature ** (torch.arange(num_freqs) / num_freqs) + freq_bands = freq_bands[None, None, None] + self.register_buffer("freq_bands", freq_bands, persistent=False) + + def __call__(self, x): + x = self.freq_bands * x.unsqueeze(-1) + return torch.stack((x.sin(), x.cos()), dim=-1).permute(0, 1, 3, 4, 2).reshape(*x.shape[:2], -1) + + +class GLIGENTextBoundingboxProjection(nn.Module): + def __init__(self, positive_len, out_dim, feature_type, fourier_freqs=8): + super().__init__() + self.positive_len = positive_len + self.out_dim = out_dim + + self.fourier_embedder = FourierEmbedder(num_freqs=fourier_freqs) + self.position_dim = fourier_freqs * 2 * 4 # 2: sin/cos, 4: xyxy + + if isinstance(out_dim, tuple): + out_dim = out_dim[0] + + if feature_type == "text-only": + self.linears = nn.Sequential( + nn.Linear(self.positive_len + self.position_dim, 512), + nn.SiLU(), + nn.Linear(512, 512), + nn.SiLU(), + nn.Linear(512, out_dim), + ) + self.null_positive_feature = torch.nn.Parameter(torch.zeros([self.positive_len])) + + elif feature_type == "text-image": + self.linears_text = nn.Sequential( + nn.Linear(self.positive_len + self.position_dim, 512), + nn.SiLU(), + nn.Linear(512, 512), + nn.SiLU(), + nn.Linear(512, out_dim), + ) + self.linears_image = nn.Sequential( + nn.Linear(self.positive_len + self.position_dim, 512), + nn.SiLU(), + nn.Linear(512, 512), + nn.SiLU(), + nn.Linear(512, out_dim), + ) + self.null_text_feature = torch.nn.Parameter(torch.zeros([self.positive_len])) + self.null_image_feature = torch.nn.Parameter(torch.zeros([self.positive_len])) + + self.null_position_feature = torch.nn.Parameter(torch.zeros([self.position_dim])) + + def forward( + self, + boxes, + masks, + positive_embeddings=None, + phrases_masks=None, + image_masks=None, + phrases_embeddings=None, + image_embeddings=None, + ): + masks = masks.unsqueeze(-1) + + xyxy_embedding = self.fourier_embedder(boxes) + xyxy_null = self.null_position_feature.view(1, 1, -1) + xyxy_embedding = xyxy_embedding * masks + (1 - masks) * xyxy_null + + if positive_embeddings: + positive_null = self.null_positive_feature.view(1, 1, -1) + positive_embeddings = positive_embeddings * masks + (1 - masks) * positive_null + + objs = self.linears(torch.cat([positive_embeddings, xyxy_embedding], dim=-1)) + else: + phrases_masks = phrases_masks.unsqueeze(-1) + image_masks = image_masks.unsqueeze(-1) + + text_null = self.null_text_feature.view(1, 1, -1) + image_null = self.null_image_feature.view(1, 1, -1) + + phrases_embeddings = phrases_embeddings * phrases_masks + (1 - phrases_masks) * text_null + image_embeddings = image_embeddings * image_masks + (1 - image_masks) * image_null + + objs_text = self.linears_text(torch.cat([phrases_embeddings, xyxy_embedding], dim=-1)) + objs_image = self.linears_image(torch.cat([image_embeddings, xyxy_embedding], dim=-1)) + objs = torch.cat([objs_text, objs_image], dim=1) + + return objs + + +class UNetFlatConditionModel(ModelMixin, ConfigMixin): + r""" + A conditional 2D UNet model that takes a noisy sample, conditional state, and a timestep and returns a sample + shaped output. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for it's generic methods implemented + for all models (such as downloading or saving). + + Parameters: + sample_size (`int` or `Tuple[int, int]`, *optional*, defaults to `None`): + Height and width of input/output sample. + in_channels (`int`, *optional*, defaults to 4): Number of channels in the input sample. + out_channels (`int`, *optional*, defaults to 4): Number of channels in the output. + center_input_sample (`bool`, *optional*, defaults to `False`): Whether to center the input sample. + flip_sin_to_cos (`bool`, *optional*, defaults to `False`): + Whether to flip the sin to cos in the time embedding. + freq_shift (`int`, *optional*, defaults to 0): The frequency shift to apply to the time embedding. + down_block_types (`Tuple[str]`, *optional*, defaults to `("CrossAttnDownBlockFlat", "CrossAttnDownBlockFlat", "CrossAttnDownBlockFlat", "DownBlockFlat")`): + The tuple of downsample blocks to use. + mid_block_type (`str`, *optional*, defaults to `"UNetMidBlockFlatCrossAttn"`): + Block type for middle of UNet, it can be one of `UNetMidBlockFlatCrossAttn`, `UNetMidBlockFlat`, or + `UNetMidBlockFlatSimpleCrossAttn`. If `None`, the mid block layer is skipped. + up_block_types (`Tuple[str]`, *optional*, defaults to `("UpBlockFlat", "CrossAttnUpBlockFlat", "CrossAttnUpBlockFlat", "CrossAttnUpBlockFlat")`): + The tuple of upsample blocks to use. + only_cross_attention(`bool` or `Tuple[bool]`, *optional*, default to `False`): + Whether to include self-attention in the basic transformer blocks, see + [`~models.attention.BasicTransformerBlock`]. + block_out_channels (`Tuple[int]`, *optional*, defaults to `(320, 640, 1280, 1280)`): + The tuple of output channels for each block. + layers_per_block (`int`, *optional*, defaults to 2): The number of layers per block. + downsample_padding (`int`, *optional*, defaults to 1): The padding to use for the downsampling convolution. + mid_block_scale_factor (`float`, *optional*, defaults to 1.0): The scale factor to use for the mid block. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + act_fn (`str`, *optional*, defaults to `"silu"`): The activation function to use. + norm_num_groups (`int`, *optional*, defaults to 32): The number of groups to use for the normalization. + If `None`, normalization and activation layers is skipped in post-processing. + norm_eps (`float`, *optional*, defaults to 1e-5): The epsilon to use for the normalization. + cross_attention_dim (`int` or `Tuple[int]`, *optional*, defaults to 1280): + The dimension of the cross attention features. + transformer_layers_per_block (`int`, `Tuple[int]`, or `Tuple[Tuple]` , *optional*, defaults to 1): + The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`]. Only relevant for + [`~models.unet_2d_blocks.CrossAttnDownBlockFlat`], [`~models.unet_2d_blocks.CrossAttnUpBlockFlat`], + [`~models.unet_2d_blocks.UNetMidBlockFlatCrossAttn`]. + reverse_transformer_layers_per_block : (`Tuple[Tuple]`, *optional*, defaults to None): + The number of transformer blocks of type [`~models.attention.BasicTransformerBlock`], in the upsampling + blocks of the U-Net. Only relevant if `transformer_layers_per_block` is of type `Tuple[Tuple]` and for + [`~models.unet_2d_blocks.CrossAttnDownBlockFlat`], [`~models.unet_2d_blocks.CrossAttnUpBlockFlat`], + [`~models.unet_2d_blocks.UNetMidBlockFlatCrossAttn`]. + encoder_hid_dim (`int`, *optional*, defaults to None): + If `encoder_hid_dim_type` is defined, `encoder_hidden_states` will be projected from `encoder_hid_dim` + dimension to `cross_attention_dim`. + encoder_hid_dim_type (`str`, *optional*, defaults to `None`): + If given, the `encoder_hidden_states` and potentially other embeddings are down-projected to text + embeddings of dimension `cross_attention` according to `encoder_hid_dim_type`. + attention_head_dim (`int`, *optional*, defaults to 8): The dimension of the attention heads. + num_attention_heads (`int`, *optional*): + The number of attention heads. If not defined, defaults to `attention_head_dim` + resnet_time_scale_shift (`str`, *optional*, defaults to `"default"`): Time scale shift config + for ResNet blocks (see [`~models.resnet.ResnetBlockFlat`]). Choose from `default` or `scale_shift`. + class_embed_type (`str`, *optional*, defaults to `None`): + The type of class embedding to use which is ultimately summed with the time embeddings. Choose from `None`, + `"timestep"`, `"identity"`, `"projection"`, or `"simple_projection"`. + addition_embed_type (`str`, *optional*, defaults to `None`): + Configures an optional embedding which will be summed with the time embeddings. Choose from `None` or + "text". "text" will use the `TextTimeEmbedding` layer. + addition_time_embed_dim: (`int`, *optional*, defaults to `None`): + Dimension for the timestep embeddings. + num_class_embeds (`int`, *optional*, defaults to `None`): + Input dimension of the learnable embedding matrix to be projected to `time_embed_dim`, when performing + class conditioning with `class_embed_type` equal to `None`. + time_embedding_type (`str`, *optional*, defaults to `positional`): + The type of position embedding to use for timesteps. Choose from `positional` or `fourier`. + time_embedding_dim (`int`, *optional*, defaults to `None`): + An optional override for the dimension of the projected time embedding. + time_embedding_act_fn (`str`, *optional*, defaults to `None`): + Optional activation function to use only once on the time embeddings before they are passed to the rest of + the UNet. Choose from `silu`, `mish`, `gelu`, and `swish`. + timestep_post_act (`str`, *optional*, defaults to `None`): + The second activation function to use in timestep embedding. Choose from `silu`, `mish` and `gelu`. + time_cond_proj_dim (`int`, *optional*, defaults to `None`): + The dimension of `cond_proj` layer in the timestep embedding. + conv_in_kernel (`int`, *optional*, default to `3`): The kernel size of `conv_in` layer. conv_out_kernel (`int`, + *optional*, default to `3`): The kernel size of `conv_out` layer. projection_class_embeddings_input_dim (`int`, + *optional*): The dimension of the `class_labels` input when + `class_embed_type="projection"`. Required when `class_embed_type="projection"`. + class_embeddings_concat (`bool`, *optional*, defaults to `False`): Whether to concatenate the time + embeddings with the class embeddings. + mid_block_only_cross_attention (`bool`, *optional*, defaults to `None`): + Whether to use cross attention with the mid block when using the `UNetMidBlockFlatSimpleCrossAttn`. If + `only_cross_attention` is given as a single boolean and `mid_block_only_cross_attention` is `None`, the + `only_cross_attention` value is used as the value for `mid_block_only_cross_attention`. Default to `False` + otherwise. + """ + + _supports_gradient_checkpointing = True + + @register_to_config + def __init__( + self, + sample_size: Optional[int] = None, + in_channels: int = 4, + out_channels: int = 4, + center_input_sample: bool = False, + flip_sin_to_cos: bool = True, + freq_shift: int = 0, + down_block_types: Tuple[str] = ( + "CrossAttnDownBlockFlat", + "CrossAttnDownBlockFlat", + "CrossAttnDownBlockFlat", + "DownBlockFlat", + ), + mid_block_type: Optional[str] = "UNetMidBlockFlatCrossAttn", + up_block_types: Tuple[str] = ( + "UpBlockFlat", + "CrossAttnUpBlockFlat", + "CrossAttnUpBlockFlat", + "CrossAttnUpBlockFlat", + ), + only_cross_attention: Union[bool, Tuple[bool]] = False, + block_out_channels: Tuple[int] = (320, 640, 1280, 1280), + layers_per_block: Union[int, Tuple[int]] = 2, + downsample_padding: int = 1, + mid_block_scale_factor: float = 1, + dropout: float = 0.0, + act_fn: str = "silu", + norm_num_groups: Optional[int] = 32, + norm_eps: float = 1e-5, + cross_attention_dim: Union[int, Tuple[int]] = 1280, + transformer_layers_per_block: Union[int, Tuple[int], Tuple[Tuple]] = 1, + reverse_transformer_layers_per_block: Optional[Tuple[Tuple[int]]] = None, + encoder_hid_dim: Optional[int] = None, + encoder_hid_dim_type: Optional[str] = None, + attention_head_dim: Union[int, Tuple[int]] = 8, + num_attention_heads: Optional[Union[int, Tuple[int]]] = None, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + class_embed_type: Optional[str] = None, + addition_embed_type: Optional[str] = None, + addition_time_embed_dim: Optional[int] = None, + num_class_embeds: Optional[int] = None, + upcast_attention: bool = False, + resnet_time_scale_shift: str = "default", + resnet_skip_time_act: bool = False, + resnet_out_scale_factor: int = 1.0, + time_embedding_type: str = "positional", + time_embedding_dim: Optional[int] = None, + time_embedding_act_fn: Optional[str] = None, + timestep_post_act: Optional[str] = None, + time_cond_proj_dim: Optional[int] = None, + conv_in_kernel: int = 3, + conv_out_kernel: int = 3, + projection_class_embeddings_input_dim: Optional[int] = None, + attention_type: str = "default", + class_embeddings_concat: bool = False, + mid_block_only_cross_attention: Optional[bool] = None, + cross_attention_norm: Optional[str] = None, + addition_embed_type_num_heads=64, + ): + super().__init__() + + self.sample_size = sample_size + + if num_attention_heads is not None: + raise ValueError( + "At the moment it is not possible to define the number of attention heads via `num_attention_heads` because of a naming issue as described in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131. Passing `num_attention_heads` will only be supported in diffusers v0.19." + ) + + # If `num_attention_heads` is not defined (which is the case for most models) + # it will default to `attention_head_dim`. This looks weird upon first reading it and it is. + # The reason for this behavior is to correct for incorrectly named variables that were introduced + # when this library was created. The incorrect naming was only discovered much later in https://github.com/huggingface/diffusers/issues/2011#issuecomment-1547958131 + # Changing `attention_head_dim` to `num_attention_heads` for 40,000+ configurations is too backwards breaking + # which is why we correct for the naming here. + num_attention_heads = num_attention_heads or attention_head_dim + + # Check inputs + if len(down_block_types) != len(up_block_types): + raise ValueError( + f"Must provide the same number of `down_block_types` as `up_block_types`. `down_block_types`: {down_block_types}. `up_block_types`: {up_block_types}." + ) + + if len(block_out_channels) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `block_out_channels` as `down_block_types`. `block_out_channels`: {block_out_channels}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(only_cross_attention, bool) and len(only_cross_attention) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `only_cross_attention` as `down_block_types`. `only_cross_attention`: {only_cross_attention}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(num_attention_heads, int) and len(num_attention_heads) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `num_attention_heads` as `down_block_types`. `num_attention_heads`: {num_attention_heads}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(attention_head_dim, int) and len(attention_head_dim) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `attention_head_dim` as `down_block_types`. `attention_head_dim`: {attention_head_dim}. `down_block_types`: {down_block_types}." + ) + + if isinstance(cross_attention_dim, list) and len(cross_attention_dim) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `cross_attention_dim` as `down_block_types`. `cross_attention_dim`: {cross_attention_dim}. `down_block_types`: {down_block_types}." + ) + + if not isinstance(layers_per_block, int) and len(layers_per_block) != len(down_block_types): + raise ValueError( + f"Must provide the same number of `layers_per_block` as `down_block_types`. `layers_per_block`: {layers_per_block}. `down_block_types`: {down_block_types}." + ) + if isinstance(transformer_layers_per_block, list) and reverse_transformer_layers_per_block is None: + for layer_number_per_block in transformer_layers_per_block: + if isinstance(layer_number_per_block, list): + raise ValueError("Must provide 'reverse_transformer_layers_per_block` if using asymmetrical UNet.") + + # input + conv_in_padding = (conv_in_kernel - 1) // 2 + self.conv_in = LinearMultiDim( + in_channels, block_out_channels[0], kernel_size=conv_in_kernel, padding=conv_in_padding + ) + + # time + if time_embedding_type == "fourier": + time_embed_dim = time_embedding_dim or block_out_channels[0] * 2 + if time_embed_dim % 2 != 0: + raise ValueError(f"`time_embed_dim` should be divisible by 2, but is {time_embed_dim}.") + self.time_proj = GaussianFourierProjection( + time_embed_dim // 2, set_W_to_weight=False, log=False, flip_sin_to_cos=flip_sin_to_cos + ) + timestep_input_dim = time_embed_dim + elif time_embedding_type == "positional": + time_embed_dim = time_embedding_dim or block_out_channels[0] * 4 + + self.time_proj = Timesteps(block_out_channels[0], flip_sin_to_cos, freq_shift) + timestep_input_dim = block_out_channels[0] + else: + raise ValueError( + f"{time_embedding_type} does not exist. Please make sure to use one of `fourier` or `positional`." + ) + + self.time_embedding = TimestepEmbedding( + timestep_input_dim, + time_embed_dim, + act_fn=act_fn, + post_act_fn=timestep_post_act, + cond_proj_dim=time_cond_proj_dim, + ) + + if encoder_hid_dim_type is None and encoder_hid_dim is not None: + encoder_hid_dim_type = "text_proj" + self.register_to_config(encoder_hid_dim_type=encoder_hid_dim_type) + logger.info("encoder_hid_dim_type defaults to 'text_proj' as `encoder_hid_dim` is defined.") + + if encoder_hid_dim is None and encoder_hid_dim_type is not None: + raise ValueError( + f"`encoder_hid_dim` has to be defined when `encoder_hid_dim_type` is set to {encoder_hid_dim_type}." + ) + + if encoder_hid_dim_type == "text_proj": + self.encoder_hid_proj = nn.Linear(encoder_hid_dim, cross_attention_dim) + elif encoder_hid_dim_type == "text_image_proj": + # image_embed_dim DOESN'T have to be `cross_attention_dim`. To not clutter the __init__ too much + # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use + # case when `addition_embed_type == "text_image_proj"` (Kadinsky 2.1)` + self.encoder_hid_proj = TextImageProjection( + text_embed_dim=encoder_hid_dim, + image_embed_dim=cross_attention_dim, + cross_attention_dim=cross_attention_dim, + ) + elif encoder_hid_dim_type == "image_proj": + # Kandinsky 2.2 + self.encoder_hid_proj = ImageProjection( + image_embed_dim=encoder_hid_dim, + cross_attention_dim=cross_attention_dim, + ) + elif encoder_hid_dim_type is not None: + raise ValueError( + f"encoder_hid_dim_type: {encoder_hid_dim_type} must be None, 'text_proj' or 'text_image_proj'." + ) + else: + self.encoder_hid_proj = None + + # class embedding + if class_embed_type is None and num_class_embeds is not None: + self.class_embedding = nn.Embedding(num_class_embeds, time_embed_dim) + elif class_embed_type == "timestep": + self.class_embedding = TimestepEmbedding(timestep_input_dim, time_embed_dim, act_fn=act_fn) + elif class_embed_type == "identity": + self.class_embedding = nn.Identity(time_embed_dim, time_embed_dim) + elif class_embed_type == "projection": + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'projection' requires `projection_class_embeddings_input_dim` be set" + ) + # The projection `class_embed_type` is the same as the timestep `class_embed_type` except + # 1. the `class_labels` inputs are not first converted to sinusoidal embeddings + # 2. it projects from an arbitrary input dimension. + # + # Note that `TimestepEmbedding` is quite general, being mainly linear layers and activations. + # When used for embedding actual timesteps, the timesteps are first converted to sinusoidal embeddings. + # As a result, `TimestepEmbedding` can be passed arbitrary vectors. + self.class_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim) + elif class_embed_type == "simple_projection": + if projection_class_embeddings_input_dim is None: + raise ValueError( + "`class_embed_type`: 'simple_projection' requires `projection_class_embeddings_input_dim` be set" + ) + self.class_embedding = nn.Linear(projection_class_embeddings_input_dim, time_embed_dim) + else: + self.class_embedding = None + + if addition_embed_type == "text": + if encoder_hid_dim is not None: + text_time_embedding_from_dim = encoder_hid_dim + else: + text_time_embedding_from_dim = cross_attention_dim + + self.add_embedding = TextTimeEmbedding( + text_time_embedding_from_dim, time_embed_dim, num_heads=addition_embed_type_num_heads + ) + elif addition_embed_type == "text_image": + # text_embed_dim and image_embed_dim DON'T have to be `cross_attention_dim`. To not clutter the __init__ too much + # they are set to `cross_attention_dim` here as this is exactly the required dimension for the currently only use + # case when `addition_embed_type == "text_image"` (Kadinsky 2.1)` + self.add_embedding = TextImageTimeEmbedding( + text_embed_dim=cross_attention_dim, image_embed_dim=cross_attention_dim, time_embed_dim=time_embed_dim + ) + elif addition_embed_type == "text_time": + self.add_time_proj = Timesteps(addition_time_embed_dim, flip_sin_to_cos, freq_shift) + self.add_embedding = TimestepEmbedding(projection_class_embeddings_input_dim, time_embed_dim) + elif addition_embed_type == "image": + # Kandinsky 2.2 + self.add_embedding = ImageTimeEmbedding(image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim) + elif addition_embed_type == "image_hint": + # Kandinsky 2.2 ControlNet + self.add_embedding = ImageHintTimeEmbedding(image_embed_dim=encoder_hid_dim, time_embed_dim=time_embed_dim) + elif addition_embed_type is not None: + raise ValueError(f"addition_embed_type: {addition_embed_type} must be None, 'text' or 'text_image'.") + + if time_embedding_act_fn is None: + self.time_embed_act = None + else: + self.time_embed_act = get_activation(time_embedding_act_fn) + + self.down_blocks = nn.ModuleList([]) + self.up_blocks = nn.ModuleList([]) + + if isinstance(only_cross_attention, bool): + if mid_block_only_cross_attention is None: + mid_block_only_cross_attention = only_cross_attention + + only_cross_attention = [only_cross_attention] * len(down_block_types) + + if mid_block_only_cross_attention is None: + mid_block_only_cross_attention = False + + if isinstance(num_attention_heads, int): + num_attention_heads = (num_attention_heads,) * len(down_block_types) + + if isinstance(attention_head_dim, int): + attention_head_dim = (attention_head_dim,) * len(down_block_types) + + if isinstance(cross_attention_dim, int): + cross_attention_dim = (cross_attention_dim,) * len(down_block_types) + + if isinstance(layers_per_block, int): + layers_per_block = [layers_per_block] * len(down_block_types) + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * len(down_block_types) + + if class_embeddings_concat: + # The time embeddings are concatenated with the class embeddings. The dimension of the + # time embeddings passed to the down, middle, and up blocks is twice the dimension of the + # regular time embeddings + blocks_time_embed_dim = time_embed_dim * 2 + else: + blocks_time_embed_dim = time_embed_dim + + # down + output_channel = block_out_channels[0] + for i, down_block_type in enumerate(down_block_types): + input_channel = output_channel + output_channel = block_out_channels[i] + is_final_block = i == len(block_out_channels) - 1 + + down_block = get_down_block( + down_block_type, + num_layers=layers_per_block[i], + transformer_layers_per_block=transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + temb_channels=blocks_time_embed_dim, + add_downsample=not is_final_block, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resnet_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim[i], + num_attention_heads=num_attention_heads[i], + downsample_padding=downsample_padding, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + attention_head_dim=attention_head_dim[i] if attention_head_dim[i] is not None else output_channel, + dropout=dropout, + ) + self.down_blocks.append(down_block) + + # mid + if mid_block_type == "UNetMidBlockFlatCrossAttn": + self.mid_block = UNetMidBlockFlatCrossAttn( + transformer_layers_per_block=transformer_layers_per_block[-1], + in_channels=block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + dropout=dropout, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_time_scale_shift=resnet_time_scale_shift, + cross_attention_dim=cross_attention_dim[-1], + num_attention_heads=num_attention_heads[-1], + resnet_groups=norm_num_groups, + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + elif mid_block_type == "UNetMidBlockFlatSimpleCrossAttn": + self.mid_block = UNetMidBlockFlatSimpleCrossAttn( + in_channels=block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + dropout=dropout, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + cross_attention_dim=cross_attention_dim[-1], + attention_head_dim=attention_head_dim[-1], + resnet_groups=norm_num_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + skip_time_act=resnet_skip_time_act, + only_cross_attention=mid_block_only_cross_attention, + cross_attention_norm=cross_attention_norm, + ) + elif mid_block_type == "UNetMidBlockFlat": + self.mid_block = UNetMidBlockFlat( + in_channels=block_out_channels[-1], + temb_channels=blocks_time_embed_dim, + dropout=dropout, + num_layers=0, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + output_scale_factor=mid_block_scale_factor, + resnet_groups=norm_num_groups, + resnet_time_scale_shift=resnet_time_scale_shift, + add_attention=False, + ) + elif mid_block_type is None: + self.mid_block = None + else: + raise ValueError(f"unknown mid_block_type : {mid_block_type}") + + # count how many layers upsample the images + self.num_upsamplers = 0 + + # up + reversed_block_out_channels = list(reversed(block_out_channels)) + reversed_num_attention_heads = list(reversed(num_attention_heads)) + reversed_layers_per_block = list(reversed(layers_per_block)) + reversed_cross_attention_dim = list(reversed(cross_attention_dim)) + reversed_transformer_layers_per_block = ( + list(reversed(transformer_layers_per_block)) + if reverse_transformer_layers_per_block is None + else reverse_transformer_layers_per_block + ) + only_cross_attention = list(reversed(only_cross_attention)) + + output_channel = reversed_block_out_channels[0] + for i, up_block_type in enumerate(up_block_types): + is_final_block = i == len(block_out_channels) - 1 + + prev_output_channel = output_channel + output_channel = reversed_block_out_channels[i] + input_channel = reversed_block_out_channels[min(i + 1, len(block_out_channels) - 1)] + + # add upsample block for all BUT final layer + if not is_final_block: + add_upsample = True + self.num_upsamplers += 1 + else: + add_upsample = False + + up_block = get_up_block( + up_block_type, + num_layers=reversed_layers_per_block[i] + 1, + transformer_layers_per_block=reversed_transformer_layers_per_block[i], + in_channels=input_channel, + out_channels=output_channel, + prev_output_channel=prev_output_channel, + temb_channels=blocks_time_embed_dim, + add_upsample=add_upsample, + resnet_eps=norm_eps, + resnet_act_fn=act_fn, + resolution_idx=i, + resnet_groups=norm_num_groups, + cross_attention_dim=reversed_cross_attention_dim[i], + num_attention_heads=reversed_num_attention_heads[i], + dual_cross_attention=dual_cross_attention, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention[i], + upcast_attention=upcast_attention, + resnet_time_scale_shift=resnet_time_scale_shift, + attention_type=attention_type, + resnet_skip_time_act=resnet_skip_time_act, + resnet_out_scale_factor=resnet_out_scale_factor, + cross_attention_norm=cross_attention_norm, + attention_head_dim=attention_head_dim[i] if attention_head_dim[i] is not None else output_channel, + dropout=dropout, + ) + self.up_blocks.append(up_block) + prev_output_channel = output_channel + + # out + if norm_num_groups is not None: + self.conv_norm_out = nn.GroupNorm( + num_channels=block_out_channels[0], num_groups=norm_num_groups, eps=norm_eps + ) + + self.conv_act = get_activation(act_fn) + + else: + self.conv_norm_out = None + self.conv_act = None + + conv_out_padding = (conv_out_kernel - 1) // 2 + self.conv_out = LinearMultiDim( + block_out_channels[0], out_channels, kernel_size=conv_out_kernel, padding=conv_out_padding + ) + + if attention_type in ["gated", "gated-text-image"]: + positive_len = 768 + if isinstance(cross_attention_dim, int): + positive_len = cross_attention_dim + elif isinstance(cross_attention_dim, tuple) or isinstance(cross_attention_dim, list): + positive_len = cross_attention_dim[0] + + feature_type = "text-only" if attention_type == "gated" else "text-image" + self.position_net = GLIGENTextBoundingboxProjection( + positive_len=positive_len, out_dim=cross_attention_dim, feature_type=feature_type + ) + + @property + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + def set_attention_slice(self, slice_size): + r""" + Enable sliced attention computation. + + When this option is enabled, the attention module splits the input tensor in slices to compute attention in + several steps. This is useful for saving some memory in exchange for a small decrease in speed. + + Args: + slice_size (`str` or `int` or `list(int)`, *optional*, defaults to `"auto"`): + When `"auto"`, input to the attention heads is halved, so attention is computed in two steps. If + `"max"`, maximum amount of memory is saved by running only one slice at a time. If a number is + provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim` + must be a multiple of `slice_size`. + """ + sliceable_head_dims = [] + + def fn_recursive_retrieve_sliceable_dims(module: torch.nn.Module): + if hasattr(module, "set_attention_slice"): + sliceable_head_dims.append(module.sliceable_head_dim) + + for child in module.children(): + fn_recursive_retrieve_sliceable_dims(child) + + # retrieve number of attention layers + for module in self.children(): + fn_recursive_retrieve_sliceable_dims(module) + + num_sliceable_layers = len(sliceable_head_dims) + + if slice_size == "auto": + # half the attention head size is usually a good trade-off between + # speed and memory + slice_size = [dim // 2 for dim in sliceable_head_dims] + elif slice_size == "max": + # make smallest slice possible + slice_size = num_sliceable_layers * [1] + + slice_size = num_sliceable_layers * [slice_size] if not isinstance(slice_size, list) else slice_size + + if len(slice_size) != len(sliceable_head_dims): + raise ValueError( + f"You have provided {len(slice_size)}, but {self.config} has {len(sliceable_head_dims)} different" + f" attention layers. Make sure to match `len(slice_size)` to be {len(sliceable_head_dims)}." + ) + + for i in range(len(slice_size)): + size = slice_size[i] + dim = sliceable_head_dims[i] + if size is not None and size > dim: + raise ValueError(f"size {size} has to be smaller or equal to {dim}.") + + # Recursively walk through all the children. + # Any children which exposes the set_attention_slice method + # gets the message + def fn_recursive_set_attention_slice(module: torch.nn.Module, slice_size: List[int]): + if hasattr(module, "set_attention_slice"): + module.set_attention_slice(slice_size.pop()) + + for child in module.children(): + fn_recursive_set_attention_slice(child, slice_size) + + reversed_slice_size = list(reversed(slice_size)) + for module in self.children(): + fn_recursive_set_attention_slice(module, reversed_slice_size) + + def _set_gradient_checkpointing(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + + def enable_freeu(self, s1, s2, b1, b2): + r"""Enables the FreeU mechanism from https://arxiv.org/abs/2309.11497. + + The suffixes after the scaling factors represent the stage blocks where they are being applied. + + Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of values that + are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL. + + Args: + s1 (`float`): + Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + s2 (`float`): + Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to + mitigate the "oversmoothing effect" in the enhanced denoising process. + b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. + b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. + """ + for i, upsample_block in enumerate(self.up_blocks): + setattr(upsample_block, "s1", s1) + setattr(upsample_block, "s2", s2) + setattr(upsample_block, "b1", b1) + setattr(upsample_block, "b2", b2) + + def disable_freeu(self): + """Disables the FreeU mechanism.""" + freeu_keys = {"s1", "s2", "b1", "b2"} + for i, upsample_block in enumerate(self.up_blocks): + for k in freeu_keys: + if hasattr(upsample_block, k) or getattr(upsample_block, k, None) is not None: + setattr(upsample_block, k, None) + + def fuse_qkv_projections(self): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, + key, value) are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is 🧪 experimental. + + + """ + self.original_attn_processors = None + + for _, attn_processor in self.attn_processors.items(): + if "Added" in str(attn_processor.__class__.__name__): + raise ValueError("`fuse_qkv_projections()` is not supported for models having added KV projections.") + + self.original_attn_processors = self.attn_processors + + for module in self.modules(): + if isinstance(module, Attention): + module.fuse_projections(fuse=True) + + def unfuse_qkv_projections(self): + """Disables the fused QKV projection if enabled. + + + + This API is 🧪 experimental. + + + + """ + if self.original_attn_processors is not None: + self.set_attn_processor(self.original_attn_processors) + + def unload_lora(self): + """Unloads LoRA weights.""" + deprecate( + "unload_lora", + "0.28.0", + "Calling `unload_lora()` is deprecated and will be removed in a future version. Please install `peft` and then call `disable_adapters().", + ) + for module in self.modules(): + if hasattr(module, "set_lora_layer"): + module.set_lora_layer(None) + + def forward( + self, + sample: torch.FloatTensor, + timestep: Union[torch.Tensor, float, int], + encoder_hidden_states: torch.Tensor, + class_labels: Optional[torch.Tensor] = None, + timestep_cond: Optional[torch.Tensor] = None, + attention_mask: Optional[torch.Tensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None, + down_block_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + mid_block_additional_residual: Optional[torch.Tensor] = None, + down_intrablock_additional_residuals: Optional[Tuple[torch.Tensor]] = None, + encoder_attention_mask: Optional[torch.Tensor] = None, + return_dict: bool = True, + ) -> Union[UNet2DConditionOutput, Tuple]: + r""" + The [`UNetFlatConditionModel`] forward method. + + Args: + sample (`torch.FloatTensor`): + The noisy input tensor with the following shape `(batch, channel, height, width)`. + timestep (`torch.FloatTensor` or `float` or `int`): The number of timesteps to denoise an input. + encoder_hidden_states (`torch.FloatTensor`): + The encoder hidden states with shape `(batch, sequence_length, feature_dim)`. + class_labels (`torch.Tensor`, *optional*, defaults to `None`): + Optional class labels for conditioning. Their embeddings will be summed with the timestep embeddings. + timestep_cond: (`torch.Tensor`, *optional*, defaults to `None`): + Conditional embeddings for timestep. If provided, the embeddings will be summed with the samples passed + through the `self.time_embedding` layer to obtain the timestep embeddings. + attention_mask (`torch.Tensor`, *optional*, defaults to `None`): + An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask + is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large + negative values to the attention scores corresponding to "discard" tokens. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + added_cond_kwargs: (`dict`, *optional*): + A kwargs dictionary containing additional embeddings that if specified are added to the embeddings that + are passed along to the UNet blocks. + down_block_additional_residuals: (`tuple` of `torch.Tensor`, *optional*): + A tuple of tensors that if specified are added to the residuals of down unet blocks. + mid_block_additional_residual: (`torch.Tensor`, *optional*): + A tensor that if specified is added to the residual of the middle unet block. + encoder_attention_mask (`torch.Tensor`): + A cross-attention mask of shape `(batch, sequence_length)` is applied to `encoder_hidden_states`. If + `True` the mask is kept, otherwise if `False` it is discarded. Mask will be converted into a bias, + which adds large negative values to the attention scores corresponding to "discard" tokens. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] instead of a plain + tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttnProcessor`]. + added_cond_kwargs: (`dict`, *optional*): + A kwargs dictionary containin additional embeddings that if specified are added to the embeddings that + are passed along to the UNet blocks. + down_block_additional_residuals (`tuple` of `torch.Tensor`, *optional*): + additional residuals to be added to UNet long skip connections from down blocks to up blocks for + example from ControlNet side model(s) + mid_block_additional_residual (`torch.Tensor`, *optional*): + additional residual to be added to UNet mid block output, for example from ControlNet side model + down_intrablock_additional_residuals (`tuple` of `torch.Tensor`, *optional*): + additional residuals to be added within UNet down blocks, for example from T2I-Adapter side model(s) + + Returns: + [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] or `tuple`: + If `return_dict` is True, an [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] is returned, otherwise + a `tuple` is returned where the first element is the sample tensor. + """ + # By default samples have to be AT least a multiple of the overall upsampling factor. + # The overall upsampling factor is equal to 2 ** (# num of upsampling layers). + # However, the upsampling interpolation output size can be forced to fit any upsampling size + # on the fly if necessary. + default_overall_up_factor = 2**self.num_upsamplers + + # upsample size should be forwarded when sample is not a multiple of `default_overall_up_factor` + forward_upsample_size = False + upsample_size = None + + for dim in sample.shape[-2:]: + if dim % default_overall_up_factor != 0: + # Forward upsample size to force interpolation output size. + forward_upsample_size = True + break + + # ensure attention_mask is a bias, and give it a singleton query_tokens dimension + # expects mask of shape: + # [batch, key_tokens] + # adds singleton query_tokens dimension: + # [batch, 1, key_tokens] + # this helps to broadcast it as a bias over attention scores, which will be in one of the following shapes: + # [batch, heads, query_tokens, key_tokens] (e.g. torch sdp attn) + # [batch * heads, query_tokens, key_tokens] (e.g. xformers or classic attn) + if attention_mask is not None: + # assume that mask is expressed as: + # (1 = keep, 0 = discard) + # convert mask into a bias that can be added to attention scores: + # (keep = +0, discard = -10000.0) + attention_mask = (1 - attention_mask.to(sample.dtype)) * -10000.0 + attention_mask = attention_mask.unsqueeze(1) + + # convert encoder_attention_mask to a bias the same way we do for attention_mask + if encoder_attention_mask is not None: + encoder_attention_mask = (1 - encoder_attention_mask.to(sample.dtype)) * -10000.0 + encoder_attention_mask = encoder_attention_mask.unsqueeze(1) + + # 0. center input if necessary + if self.config.center_input_sample: + sample = 2 * sample - 1.0 + + # 1. time + timesteps = timestep + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = sample.device.type == "mps" + if isinstance(timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=sample.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(sample.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(sample.shape[0]) + + t_emb = self.time_proj(timesteps) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # but time_embedding might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + t_emb = t_emb.to(dtype=sample.dtype) + + emb = self.time_embedding(t_emb, timestep_cond) + aug_emb = None + + if self.class_embedding is not None: + if class_labels is None: + raise ValueError("class_labels should be provided when num_class_embeds > 0") + + if self.config.class_embed_type == "timestep": + class_labels = self.time_proj(class_labels) + + # `Timesteps` does not contain any weights and will always return f32 tensors + # there might be better ways to encapsulate this. + class_labels = class_labels.to(dtype=sample.dtype) + + class_emb = self.class_embedding(class_labels).to(dtype=sample.dtype) + + if self.config.class_embeddings_concat: + emb = torch.cat([emb, class_emb], dim=-1) + else: + emb = emb + class_emb + + if self.config.addition_embed_type == "text": + aug_emb = self.add_embedding(encoder_hidden_states) + elif self.config.addition_embed_type == "text_image": + # Kandinsky 2.1 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`" + ) + + image_embs = added_cond_kwargs.get("image_embeds") + text_embs = added_cond_kwargs.get("text_embeds", encoder_hidden_states) + aug_emb = self.add_embedding(text_embs, image_embs) + elif self.config.addition_embed_type == "text_time": + # SDXL - style + if "text_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `text_embeds` to be passed in `added_cond_kwargs`" + ) + text_embeds = added_cond_kwargs.get("text_embeds") + if "time_ids" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'text_time' which requires the keyword argument `time_ids` to be passed in `added_cond_kwargs`" + ) + time_ids = added_cond_kwargs.get("time_ids") + time_embeds = self.add_time_proj(time_ids.flatten()) + time_embeds = time_embeds.reshape((text_embeds.shape[0], -1)) + add_embeds = torch.concat([text_embeds, time_embeds], dim=-1) + add_embeds = add_embeds.to(emb.dtype) + aug_emb = self.add_embedding(add_embeds) + elif self.config.addition_embed_type == "image": + # Kandinsky 2.2 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'image' which requires the keyword argument `image_embeds` to be passed in `added_cond_kwargs`" + ) + image_embs = added_cond_kwargs.get("image_embeds") + aug_emb = self.add_embedding(image_embs) + elif self.config.addition_embed_type == "image_hint": + # Kandinsky 2.2 - style + if "image_embeds" not in added_cond_kwargs or "hint" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `addition_embed_type` set to 'image_hint' which requires the keyword arguments `image_embeds` and `hint` to be passed in `added_cond_kwargs`" + ) + image_embs = added_cond_kwargs.get("image_embeds") + hint = added_cond_kwargs.get("hint") + aug_emb, hint = self.add_embedding(image_embs, hint) + sample = torch.cat([sample, hint], dim=1) + + emb = emb + aug_emb if aug_emb is not None else emb + + if self.time_embed_act is not None: + emb = self.time_embed_act(emb) + + if self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "text_proj": + encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states) + elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "text_image_proj": + # Kadinsky 2.1 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'text_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + + image_embeds = added_cond_kwargs.get("image_embeds") + encoder_hidden_states = self.encoder_hid_proj(encoder_hidden_states, image_embeds) + elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "image_proj": + # Kandinsky 2.2 - style + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + image_embeds = added_cond_kwargs.get("image_embeds") + encoder_hidden_states = self.encoder_hid_proj(image_embeds) + elif self.encoder_hid_proj is not None and self.config.encoder_hid_dim_type == "ip_image_proj": + if "image_embeds" not in added_cond_kwargs: + raise ValueError( + f"{self.__class__} has the config param `encoder_hid_dim_type` set to 'ip_image_proj' which requires the keyword argument `image_embeds` to be passed in `added_conditions`" + ) + image_embeds = added_cond_kwargs.get("image_embeds") + image_embeds = self.encoder_hid_proj(image_embeds) + encoder_hidden_states = (encoder_hidden_states, image_embeds) + + # 2. pre-process + sample = self.conv_in(sample) + + # 2.5 GLIGEN position net + if cross_attention_kwargs is not None and cross_attention_kwargs.get("gligen", None) is not None: + cross_attention_kwargs = cross_attention_kwargs.copy() + gligen_args = cross_attention_kwargs.pop("gligen") + cross_attention_kwargs["gligen"] = {"objs": self.position_net(**gligen_args)} + + # 3. down + lora_scale = cross_attention_kwargs.get("scale", 1.0) if cross_attention_kwargs is not None else 1.0 + if USE_PEFT_BACKEND: + # weight the lora layers by setting `lora_scale` for each PEFT layer + scale_lora_layers(self, lora_scale) + + is_controlnet = mid_block_additional_residual is not None and down_block_additional_residuals is not None + # using new arg down_intrablock_additional_residuals for T2I-Adapters, to distinguish from controlnets + is_adapter = down_intrablock_additional_residuals is not None + # maintain backward compatibility for legacy usage, where + # T2I-Adapter and ControlNet both use down_block_additional_residuals arg + # but can only use one or the other + if not is_adapter and mid_block_additional_residual is None and down_block_additional_residuals is not None: + deprecate( + "T2I should not use down_block_additional_residuals", + "1.3.0", + "Passing intrablock residual connections with `down_block_additional_residuals` is deprecated \ + and will be removed in diffusers 1.3.0. `down_block_additional_residuals` should only be used \ + for ControlNet. Please make sure use `down_intrablock_additional_residuals` instead. ", + standard_warn=False, + ) + down_intrablock_additional_residuals = down_block_additional_residuals + is_adapter = True + + down_block_res_samples = (sample,) + for downsample_block in self.down_blocks: + if hasattr(downsample_block, "has_cross_attention") and downsample_block.has_cross_attention: + # For t2i-adapter CrossAttnDownBlockFlat + additional_residuals = {} + if is_adapter and len(down_intrablock_additional_residuals) > 0: + additional_residuals["additional_residuals"] = down_intrablock_additional_residuals.pop(0) + + sample, res_samples = downsample_block( + hidden_states=sample, + temb=emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + **additional_residuals, + ) + else: + sample, res_samples = downsample_block(hidden_states=sample, temb=emb) + if is_adapter and len(down_intrablock_additional_residuals) > 0: + sample += down_intrablock_additional_residuals.pop(0) + + down_block_res_samples += res_samples + + if is_controlnet: + new_down_block_res_samples = () + + for down_block_res_sample, down_block_additional_residual in zip( + down_block_res_samples, down_block_additional_residuals + ): + down_block_res_sample = down_block_res_sample + down_block_additional_residual + new_down_block_res_samples = new_down_block_res_samples + (down_block_res_sample,) + + down_block_res_samples = new_down_block_res_samples + + # 4. mid + if self.mid_block is not None: + if hasattr(self.mid_block, "has_cross_attention") and self.mid_block.has_cross_attention: + sample = self.mid_block( + sample, + emb, + encoder_hidden_states=encoder_hidden_states, + attention_mask=attention_mask, + cross_attention_kwargs=cross_attention_kwargs, + encoder_attention_mask=encoder_attention_mask, + ) + else: + sample = self.mid_block(sample, emb) + + # To support T2I-Adapter-XL + if ( + is_adapter + and len(down_intrablock_additional_residuals) > 0 + and sample.shape == down_intrablock_additional_residuals[0].shape + ): + sample += down_intrablock_additional_residuals.pop(0) + + if is_controlnet: + sample = sample + mid_block_additional_residual + + # 5. up + for i, upsample_block in enumerate(self.up_blocks): + is_final_block = i == len(self.up_blocks) - 1 + + res_samples = down_block_res_samples[-len(upsample_block.resnets) :] + down_block_res_samples = down_block_res_samples[: -len(upsample_block.resnets)] + + # if we have not reached the final block and need to forward the + # upsample size, we do it here + if not is_final_block and forward_upsample_size: + upsample_size = down_block_res_samples[-1].shape[2:] + + if hasattr(upsample_block, "has_cross_attention") and upsample_block.has_cross_attention: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + upsample_size=upsample_size, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + ) + else: + sample = upsample_block( + hidden_states=sample, + temb=emb, + res_hidden_states_tuple=res_samples, + upsample_size=upsample_size, + scale=lora_scale, + ) + + # 6. post-process + if self.conv_norm_out: + sample = self.conv_norm_out(sample) + sample = self.conv_act(sample) + sample = self.conv_out(sample) + + if USE_PEFT_BACKEND: + # remove `lora_scale` from each PEFT layer + unscale_lora_layers(self, lora_scale) + + if not return_dict: + return (sample,) + + return UNet2DConditionOutput(sample=sample) + + +class LinearMultiDim(nn.Linear): + def __init__(self, in_features, out_features=None, second_dim=4, *args, **kwargs): + in_features = [in_features, second_dim, 1] if isinstance(in_features, int) else list(in_features) + if out_features is None: + out_features = in_features + out_features = [out_features, second_dim, 1] if isinstance(out_features, int) else list(out_features) + self.in_features_multidim = in_features + self.out_features_multidim = out_features + super().__init__(np.array(in_features).prod(), np.array(out_features).prod()) + + def forward(self, input_tensor, *args, **kwargs): + shape = input_tensor.shape + n_dim = len(self.in_features_multidim) + input_tensor = input_tensor.reshape(*shape[0:-n_dim], self.in_features) + output_tensor = super().forward(input_tensor) + output_tensor = output_tensor.view(*shape[0:-n_dim], *self.out_features_multidim) + return output_tensor + + +class ResnetBlockFlat(nn.Module): + def __init__( + self, + *, + in_channels, + out_channels=None, + dropout=0.0, + temb_channels=512, + groups=32, + groups_out=None, + pre_norm=True, + eps=1e-6, + time_embedding_norm="default", + use_in_shortcut=None, + second_dim=4, + **kwargs, + ): + super().__init__() + self.pre_norm = pre_norm + self.pre_norm = True + + in_channels = [in_channels, second_dim, 1] if isinstance(in_channels, int) else list(in_channels) + self.in_channels_prod = np.array(in_channels).prod() + self.channels_multidim = in_channels + + if out_channels is not None: + out_channels = [out_channels, second_dim, 1] if isinstance(out_channels, int) else list(out_channels) + out_channels_prod = np.array(out_channels).prod() + self.out_channels_multidim = out_channels + else: + out_channels_prod = self.in_channels_prod + self.out_channels_multidim = self.channels_multidim + self.time_embedding_norm = time_embedding_norm + + if groups_out is None: + groups_out = groups + + self.norm1 = torch.nn.GroupNorm(num_groups=groups, num_channels=self.in_channels_prod, eps=eps, affine=True) + self.conv1 = torch.nn.Conv2d(self.in_channels_prod, out_channels_prod, kernel_size=1, padding=0) + + if temb_channels is not None: + self.time_emb_proj = torch.nn.Linear(temb_channels, out_channels_prod) + else: + self.time_emb_proj = None + + self.norm2 = torch.nn.GroupNorm(num_groups=groups_out, num_channels=out_channels_prod, eps=eps, affine=True) + self.dropout = torch.nn.Dropout(dropout) + self.conv2 = torch.nn.Conv2d(out_channels_prod, out_channels_prod, kernel_size=1, padding=0) + + self.nonlinearity = nn.SiLU() + + self.use_in_shortcut = ( + self.in_channels_prod != out_channels_prod if use_in_shortcut is None else use_in_shortcut + ) + + self.conv_shortcut = None + if self.use_in_shortcut: + self.conv_shortcut = torch.nn.Conv2d( + self.in_channels_prod, out_channels_prod, kernel_size=1, stride=1, padding=0 + ) + + def forward(self, input_tensor, temb): + shape = input_tensor.shape + n_dim = len(self.channels_multidim) + input_tensor = input_tensor.reshape(*shape[0:-n_dim], self.in_channels_prod, 1, 1) + input_tensor = input_tensor.view(-1, self.in_channels_prod, 1, 1) + + hidden_states = input_tensor + + hidden_states = self.norm1(hidden_states) + hidden_states = self.nonlinearity(hidden_states) + hidden_states = self.conv1(hidden_states) + + if temb is not None: + temb = self.time_emb_proj(self.nonlinearity(temb))[:, :, None, None] + hidden_states = hidden_states + temb + + hidden_states = self.norm2(hidden_states) + hidden_states = self.nonlinearity(hidden_states) + + hidden_states = self.dropout(hidden_states) + hidden_states = self.conv2(hidden_states) + + if self.conv_shortcut is not None: + input_tensor = self.conv_shortcut(input_tensor) + + output_tensor = input_tensor + hidden_states + + output_tensor = output_tensor.view(*shape[0:-n_dim], -1) + output_tensor = output_tensor.view(*shape[0:-n_dim], *self.out_channels_multidim) + + return output_tensor + + +class DownBlockFlat(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_downsample: bool = True, + downsample_padding: int = 1, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlockFlat( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + LinearMultiDim( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + output_states = () + + for resnet in self.resnets: + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb) + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +class CrossAttnDownBlockFlat(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + downsample_padding: int = 1, + add_downsample: bool = True, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + attention_type: str = "default", + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + for i in range(num_layers): + in_channels = in_channels if i == 0 else out_channels + resnets.append( + ResnetBlockFlat( + in_channels=in_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_downsample: + self.downsamplers = nn.ModuleList( + [ + LinearMultiDim( + out_channels, use_conv=True, out_channels=out_channels, padding=downsample_padding, name="op" + ) + ] + ) + else: + self.downsamplers = None + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + additional_residuals: Optional[torch.FloatTensor] = None, + ) -> Tuple[torch.FloatTensor, Tuple[torch.FloatTensor, ...]]: + output_states = () + + blocks = list(zip(self.resnets, self.attentions)) + + for i, (resnet, attn) in enumerate(blocks): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + + # apply additional residuals to the output of the last pair of resnet and attention blocks + if i == len(blocks) - 1 and additional_residuals is not None: + hidden_states = hidden_states + additional_residuals + + output_states = output_states + (hidden_states,) + + if self.downsamplers is not None: + for downsampler in self.downsamplers: + hidden_states = downsampler(hidden_states) + + output_states = output_states + (hidden_states,) + + return hidden_states, output_states + + +# Copied from diffusers.models.unets.unet_2d_blocks.UpBlock2D with UpBlock2D->UpBlockFlat, ResnetBlock2D->ResnetBlockFlat, Upsample2D->LinearMultiDim +class UpBlockFlat(nn.Module): + def __init__( + self, + in_channels: int, + prev_output_channel: int, + out_channels: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + ): + super().__init__() + resnets = [] + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlockFlat( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([LinearMultiDim(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + upsample_size: Optional[int] = None, + *args, + **kwargs, + ) -> torch.FloatTensor: + if len(args) > 0 or kwargs.get("scale", None) is not None: + deprecation_message = "The `scale` argument is deprecated and will be ignored. Please remove it, as passing it will raise an error in the future. `scale` should directly be passed while calling the underlying pipeline component i.e., via `cross_attention_kwargs`." + deprecate("scale", "1.0.0", deprecation_message) + + is_freeu_enabled = ( + getattr(self, "s1", None) + and getattr(self, "s2", None) + and getattr(self, "b1", None) + and getattr(self, "b2", None) + ) + + for resnet in self.resnets: + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # FreeU: Only operate on the first two stages + if is_freeu_enabled: + hidden_states, res_hidden_states = apply_freeu( + self.resolution_idx, + hidden_states, + res_hidden_states, + s1=self.s1, + s2=self.s2, + b1=self.b1, + b2=self.b2, + ) + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb, use_reentrant=False + ) + else: + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), hidden_states, temb + ) + else: + hidden_states = resnet(hidden_states, temb) + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +# Copied from diffusers.models.unets.unet_2d_blocks.CrossAttnUpBlock2D with CrossAttnUpBlock2D->CrossAttnUpBlockFlat, ResnetBlock2D->ResnetBlockFlat, Upsample2D->LinearMultiDim +class CrossAttnUpBlockFlat(nn.Module): + def __init__( + self, + in_channels: int, + out_channels: int, + prev_output_channel: int, + temb_channels: int, + resolution_idx: Optional[int] = None, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + cross_attention_dim: int = 1280, + output_scale_factor: float = 1.0, + add_upsample: bool = True, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + attention_type: str = "default", + ): + super().__init__() + resnets = [] + attentions = [] + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + for i in range(num_layers): + res_skip_channels = in_channels if (i == num_layers - 1) else out_channels + resnet_in_channels = prev_output_channel if i == 0 else out_channels + + resnets.append( + ResnetBlockFlat( + in_channels=resnet_in_channels + res_skip_channels, + out_channels=out_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + out_channels // num_attention_heads, + in_channels=out_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + if add_upsample: + self.upsamplers = nn.ModuleList([LinearMultiDim(out_channels, use_conv=True, out_channels=out_channels)]) + else: + self.upsamplers = None + + self.gradient_checkpointing = False + self.resolution_idx = resolution_idx + + def forward( + self, + hidden_states: torch.FloatTensor, + res_hidden_states_tuple: Tuple[torch.FloatTensor, ...], + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + upsample_size: Optional[int] = None, + attention_mask: Optional[torch.FloatTensor] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + if cross_attention_kwargs is not None: + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + is_freeu_enabled = ( + getattr(self, "s1", None) + and getattr(self, "s2", None) + and getattr(self, "b1", None) + and getattr(self, "b2", None) + ) + + for resnet, attn in zip(self.resnets, self.attentions): + # pop res hidden states + res_hidden_states = res_hidden_states_tuple[-1] + res_hidden_states_tuple = res_hidden_states_tuple[:-1] + + # FreeU: Only operate on the first two stages + if is_freeu_enabled: + hidden_states, res_hidden_states = apply_freeu( + self.resolution_idx, + hidden_states, + res_hidden_states, + s1=self.s1, + s2=self.s2, + b1=self.b1, + b2=self.b2, + ) + + hidden_states = torch.cat([hidden_states, res_hidden_states], dim=1) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + else: + hidden_states = resnet(hidden_states, temb) + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + + if self.upsamplers is not None: + for upsampler in self.upsamplers: + hidden_states = upsampler(hidden_states, upsample_size) + + return hidden_states + + +# Copied from diffusers.models.unets.unet_2d_blocks.UNetMidBlock2D with UNetMidBlock2D->UNetMidBlockFlat, ResnetBlock2D->ResnetBlockFlat +class UNetMidBlockFlat(nn.Module): + """ + A 2D UNet mid-block [`UNetMidBlockFlat`] with multiple residual blocks and optional attention blocks. + + Args: + in_channels (`int`): The number of input channels. + temb_channels (`int`): The number of temporal embedding channels. + dropout (`float`, *optional*, defaults to 0.0): The dropout rate. + num_layers (`int`, *optional*, defaults to 1): The number of residual blocks. + resnet_eps (`float`, *optional*, 1e-6 ): The epsilon value for the resnet blocks. + resnet_time_scale_shift (`str`, *optional*, defaults to `default`): + The type of normalization to apply to the time embeddings. This can help to improve the performance of the + model on tasks with long-range temporal dependencies. + resnet_act_fn (`str`, *optional*, defaults to `swish`): The activation function for the resnet blocks. + resnet_groups (`int`, *optional*, defaults to 32): + The number of groups to use in the group normalization layers of the resnet blocks. + attn_groups (`Optional[int]`, *optional*, defaults to None): The number of groups for the attention blocks. + resnet_pre_norm (`bool`, *optional*, defaults to `True`): + Whether to use pre-normalization for the resnet blocks. + add_attention (`bool`, *optional*, defaults to `True`): Whether to add attention blocks. + attention_head_dim (`int`, *optional*, defaults to 1): + Dimension of a single attention head. The number of attention heads is determined based on this value and + the number of input channels. + output_scale_factor (`float`, *optional*, defaults to 1.0): The output scale factor. + + Returns: + `torch.FloatTensor`: The output of the last residual block, which is a tensor of shape `(batch_size, + in_channels, height, width)`. + + """ + + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", # default, spatial + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + attn_groups: Optional[int] = None, + resnet_pre_norm: bool = True, + add_attention: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + ): + super().__init__() + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + self.add_attention = add_attention + + if attn_groups is None: + attn_groups = resnet_groups if resnet_time_scale_shift == "default" else None + + # there is always at least one resnet + if resnet_time_scale_shift == "spatial": + resnets = [ + ResnetBlockCondNorm2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm="spatial", + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + ) + ] + else: + resnets = [ + ResnetBlockFlat( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + + if attention_head_dim is None: + logger.warning( + f"It is not recommend to pass `attention_head_dim=None`. Defaulting `attention_head_dim` to `in_channels`: {in_channels}." + ) + attention_head_dim = in_channels + + for _ in range(num_layers): + if self.add_attention: + attentions.append( + Attention( + in_channels, + heads=in_channels // attention_head_dim, + dim_head=attention_head_dim, + rescale_output_factor=output_scale_factor, + eps=resnet_eps, + norm_num_groups=attn_groups, + spatial_norm_dim=temb_channels if resnet_time_scale_shift == "spatial" else None, + residual_connection=True, + bias=True, + upcast_softmax=True, + _from_deprecated_attn_block=True, + ) + ) + else: + attentions.append(None) + + if resnet_time_scale_shift == "spatial": + resnets.append( + ResnetBlockCondNorm2D( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm="spatial", + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + ) + ) + else: + resnets.append( + ResnetBlockFlat( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward(self, hidden_states: torch.FloatTensor, temb: Optional[torch.FloatTensor] = None) -> torch.FloatTensor: + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if attn is not None: + hidden_states = attn(hidden_states, temb=temb) + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +# Copied from diffusers.models.unets.unet_2d_blocks.UNetMidBlock2DCrossAttn with UNetMidBlock2DCrossAttn->UNetMidBlockFlatCrossAttn, ResnetBlock2D->ResnetBlockFlat +class UNetMidBlockFlatCrossAttn(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + transformer_layers_per_block: Union[int, Tuple[int]] = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + num_attention_heads: int = 1, + output_scale_factor: float = 1.0, + cross_attention_dim: int = 1280, + dual_cross_attention: bool = False, + use_linear_projection: bool = False, + upcast_attention: bool = False, + attention_type: str = "default", + ): + super().__init__() + + self.has_cross_attention = True + self.num_attention_heads = num_attention_heads + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + + # support for variable transformer layers per block + if isinstance(transformer_layers_per_block, int): + transformer_layers_per_block = [transformer_layers_per_block] * num_layers + + # there is always at least one resnet + resnets = [ + ResnetBlockFlat( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ] + attentions = [] + + for i in range(num_layers): + if not dual_cross_attention: + attentions.append( + Transformer2DModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=transformer_layers_per_block[i], + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + use_linear_projection=use_linear_projection, + upcast_attention=upcast_attention, + attention_type=attention_type, + ) + ) + else: + attentions.append( + DualTransformer2DModel( + num_attention_heads, + in_channels // num_attention_heads, + in_channels=in_channels, + num_layers=1, + cross_attention_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + ) + ) + resnets.append( + ResnetBlockFlat( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + self.gradient_checkpointing = False + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + if cross_attention_kwargs is not None: + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module, return_dict=None): + def custom_forward(*inputs): + if return_dict is not None: + return module(*inputs, return_dict=return_dict) + else: + return module(*inputs) + + return custom_forward + + ckpt_kwargs: Dict[str, Any] = {"use_reentrant": False} if is_torch_version(">=", "1.11.0") else {} + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = torch.utils.checkpoint.checkpoint( + create_custom_forward(resnet), + hidden_states, + temb, + **ckpt_kwargs, + ) + else: + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + cross_attention_kwargs=cross_attention_kwargs, + attention_mask=attention_mask, + encoder_attention_mask=encoder_attention_mask, + return_dict=False, + )[0] + hidden_states = resnet(hidden_states, temb) + + return hidden_states + + +# Copied from diffusers.models.unets.unet_2d_blocks.UNetMidBlock2DSimpleCrossAttn with UNetMidBlock2DSimpleCrossAttn->UNetMidBlockFlatSimpleCrossAttn, ResnetBlock2D->ResnetBlockFlat +class UNetMidBlockFlatSimpleCrossAttn(nn.Module): + def __init__( + self, + in_channels: int, + temb_channels: int, + dropout: float = 0.0, + num_layers: int = 1, + resnet_eps: float = 1e-6, + resnet_time_scale_shift: str = "default", + resnet_act_fn: str = "swish", + resnet_groups: int = 32, + resnet_pre_norm: bool = True, + attention_head_dim: int = 1, + output_scale_factor: float = 1.0, + cross_attention_dim: int = 1280, + skip_time_act: bool = False, + only_cross_attention: bool = False, + cross_attention_norm: Optional[str] = None, + ): + super().__init__() + + self.has_cross_attention = True + + self.attention_head_dim = attention_head_dim + resnet_groups = resnet_groups if resnet_groups is not None else min(in_channels // 4, 32) + + self.num_heads = in_channels // self.attention_head_dim + + # there is always at least one resnet + resnets = [ + ResnetBlockFlat( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ] + attentions = [] + + for _ in range(num_layers): + processor = ( + AttnAddedKVProcessor2_0() if hasattr(F, "scaled_dot_product_attention") else AttnAddedKVProcessor() + ) + + attentions.append( + Attention( + query_dim=in_channels, + cross_attention_dim=in_channels, + heads=self.num_heads, + dim_head=self.attention_head_dim, + added_kv_proj_dim=cross_attention_dim, + norm_num_groups=resnet_groups, + bias=True, + upcast_softmax=True, + only_cross_attention=only_cross_attention, + cross_attention_norm=cross_attention_norm, + processor=processor, + ) + ) + resnets.append( + ResnetBlockFlat( + in_channels=in_channels, + out_channels=in_channels, + temb_channels=temb_channels, + eps=resnet_eps, + groups=resnet_groups, + dropout=dropout, + time_embedding_norm=resnet_time_scale_shift, + non_linearity=resnet_act_fn, + output_scale_factor=output_scale_factor, + pre_norm=resnet_pre_norm, + skip_time_act=skip_time_act, + ) + ) + + self.attentions = nn.ModuleList(attentions) + self.resnets = nn.ModuleList(resnets) + + def forward( + self, + hidden_states: torch.FloatTensor, + temb: Optional[torch.FloatTensor] = None, + encoder_hidden_states: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + encoder_attention_mask: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + if cross_attention_kwargs.get("scale", None) is not None: + logger.warning("Passing `scale` to `cross_attention_kwargs` is depcrecated. `scale` will be ignored.") + + if attention_mask is None: + # if encoder_hidden_states is defined: we are doing cross-attn, so we should use cross-attn mask. + mask = None if encoder_hidden_states is None else encoder_attention_mask + else: + # when attention_mask is defined: we don't even check for encoder_attention_mask. + # this is to maintain compatibility with UnCLIP, which uses 'attention_mask' param for cross-attn masks. + # TODO: UnCLIP should express cross-attn mask via encoder_attention_mask param instead of via attention_mask. + # then we can simplify this whole if/else block to: + # mask = attention_mask if encoder_hidden_states is None else encoder_attention_mask + mask = attention_mask + + hidden_states = self.resnets[0](hidden_states, temb) + for attn, resnet in zip(self.attentions, self.resnets[1:]): + # attn + hidden_states = attn( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=mask, + **cross_attention_kwargs, + ) + + # resnet + hidden_states = resnet(hidden_states, temb) + + return hidden_states diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py new file mode 100755 index 0000000..4455d20 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion.py @@ -0,0 +1,421 @@ +import inspect +from typing import Callable, List, Optional, Union + +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModel + +from ....models import AutoencoderKL, UNet2DConditionModel +from ....schedulers import KarrasDiffusionSchedulers +from ....utils import logging +from ...pipeline_utils import DiffusionPipeline +from .pipeline_versatile_diffusion_dual_guided import VersatileDiffusionDualGuidedPipeline +from .pipeline_versatile_diffusion_image_variation import VersatileDiffusionImageVariationPipeline +from .pipeline_versatile_diffusion_text_to_image import VersatileDiffusionTextToImagePipeline + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class VersatileDiffusionPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + tokenizer: CLIPTokenizer + image_feature_extractor: CLIPImageProcessor + text_encoder: CLIPTextModel + image_encoder: CLIPVisionModel + image_unet: UNet2DConditionModel + text_unet: UNet2DConditionModel + vae: AutoencoderKL + scheduler: KarrasDiffusionSchedulers + + def __init__( + self, + tokenizer: CLIPTokenizer, + image_feature_extractor: CLIPImageProcessor, + text_encoder: CLIPTextModel, + image_encoder: CLIPVisionModel, + image_unet: UNet2DConditionModel, + text_unet: UNet2DConditionModel, + vae: AutoencoderKL, + scheduler: KarrasDiffusionSchedulers, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, + image_feature_extractor=image_feature_extractor, + text_encoder=text_encoder, + image_encoder=image_encoder, + image_unet=image_unet, + text_unet=text_unet, + vae=vae, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + + @torch.no_grad() + def image_variation( + self, + image: Union[torch.FloatTensor, PIL.Image.Image], + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + ): + r""" + The call function to the pipeline for generation. + + Args: + image (`PIL.Image.Image`, `List[PIL.Image.Image]` or `torch.Tensor`): + The image prompt or prompts to guide the image generation. + height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Examples: + + ```py + >>> from diffusers import VersatileDiffusionPipeline + >>> import torch + >>> import requests + >>> from io import BytesIO + >>> from PIL import Image + + >>> # let's download an initial image + >>> url = "https://huggingface.co/datasets/diffusers/images/resolve/main/benz.jpg" + + >>> response = requests.get(url) + >>> image = Image.open(BytesIO(response.content)).convert("RGB") + + >>> pipe = VersatileDiffusionPipeline.from_pretrained( + ... "shi-labs/versatile-diffusion", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> generator = torch.Generator(device="cuda").manual_seed(0) + >>> image = pipe.image_variation(image, generator=generator).images[0] + >>> image.save("./car_variation.png") + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + expected_components = inspect.signature(VersatileDiffusionImageVariationPipeline.__init__).parameters.keys() + components = {name: component for name, component in self.components.items() if name in expected_components} + return VersatileDiffusionImageVariationPipeline(**components)( + image=image, + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + eta=eta, + generator=generator, + latents=latents, + output_type=output_type, + return_dict=return_dict, + callback=callback, + callback_steps=callback_steps, + ) + + @torch.no_grad() + def text_to_image( + self, + prompt: Union[str, List[str]], + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide image generation. + height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Examples: + + ```py + >>> from diffusers import VersatileDiffusionPipeline + >>> import torch + + >>> pipe = VersatileDiffusionPipeline.from_pretrained( + ... "shi-labs/versatile-diffusion", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> generator = torch.Generator(device="cuda").manual_seed(0) + >>> image = pipe.text_to_image("an astronaut riding on a horse on mars", generator=generator).images[0] + >>> image.save("./astronaut.png") + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + expected_components = inspect.signature(VersatileDiffusionTextToImagePipeline.__init__).parameters.keys() + components = {name: component for name, component in self.components.items() if name in expected_components} + temp_pipeline = VersatileDiffusionTextToImagePipeline(**components) + output = temp_pipeline( + prompt=prompt, + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + eta=eta, + generator=generator, + latents=latents, + output_type=output_type, + return_dict=return_dict, + callback=callback, + callback_steps=callback_steps, + ) + # swap the attention blocks back to the original state + temp_pipeline._swap_unet_attention_blocks() + + return output + + @torch.no_grad() + def dual_guided( + self, + prompt: Union[PIL.Image.Image, List[PIL.Image.Image]], + image: Union[str, List[str]], + text_to_image_strength: float = 0.5, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide image generation. + height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Examples: + + ```py + >>> from diffusers import VersatileDiffusionPipeline + >>> import torch + >>> import requests + >>> from io import BytesIO + >>> from PIL import Image + + >>> # let's download an initial image + >>> url = "https://huggingface.co/datasets/diffusers/images/resolve/main/benz.jpg" + + >>> response = requests.get(url) + >>> image = Image.open(BytesIO(response.content)).convert("RGB") + >>> text = "a red car in the sun" + + >>> pipe = VersatileDiffusionPipeline.from_pretrained( + ... "shi-labs/versatile-diffusion", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> generator = torch.Generator(device="cuda").manual_seed(0) + >>> text_to_image_strength = 0.75 + + >>> image = pipe.dual_guided( + ... prompt=text, image=image, text_to_image_strength=text_to_image_strength, generator=generator + ... ).images[0] + >>> image.save("./car_variation.png") + ``` + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images. + """ + + expected_components = inspect.signature(VersatileDiffusionDualGuidedPipeline.__init__).parameters.keys() + components = {name: component for name, component in self.components.items() if name in expected_components} + temp_pipeline = VersatileDiffusionDualGuidedPipeline(**components) + output = temp_pipeline( + prompt=prompt, + image=image, + text_to_image_strength=text_to_image_strength, + height=height, + width=width, + num_inference_steps=num_inference_steps, + guidance_scale=guidance_scale, + num_images_per_prompt=num_images_per_prompt, + eta=eta, + generator=generator, + latents=latents, + output_type=output_type, + return_dict=return_dict, + callback=callback, + callback_steps=callback_steps, + ) + temp_pipeline._revert_dual_attention() + + return output diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py new file mode 100755 index 0000000..8af739b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_dual_guided.py @@ -0,0 +1,556 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Callable, List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +import torch.utils.checkpoint +from transformers import ( + CLIPImageProcessor, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from ....image_processor import VaeImageProcessor +from ....models import AutoencoderKL, DualTransformer2DModel, Transformer2DModel, UNet2DConditionModel +from ....schedulers import KarrasDiffusionSchedulers +from ....utils import deprecate, logging +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from .modeling_text_unet import UNetFlatConditionModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class VersatileDiffusionDualGuidedPipeline(DiffusionPipeline): + r""" + Pipeline for image-text dual-guided generation using Versatile Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + vqvae ([`VQModel`]): + Vector-quantized (VQ) model to encode and decode images to and from latent representations. + bert ([`LDMBertModel`]): + Text-encoder model based on [`~transformers.BERT`]. + tokenizer ([`~transformers.BertTokenizer`]): + A `BertTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "bert->unet->vqvae" + + tokenizer: CLIPTokenizer + image_feature_extractor: CLIPImageProcessor + text_encoder: CLIPTextModelWithProjection + image_encoder: CLIPVisionModelWithProjection + image_unet: UNet2DConditionModel + text_unet: UNetFlatConditionModel + vae: AutoencoderKL + scheduler: KarrasDiffusionSchedulers + + _optional_components = ["text_unet"] + + def __init__( + self, + tokenizer: CLIPTokenizer, + image_feature_extractor: CLIPImageProcessor, + text_encoder: CLIPTextModelWithProjection, + image_encoder: CLIPVisionModelWithProjection, + image_unet: UNet2DConditionModel, + text_unet: UNetFlatConditionModel, + vae: AutoencoderKL, + scheduler: KarrasDiffusionSchedulers, + ): + super().__init__() + self.register_modules( + tokenizer=tokenizer, + image_feature_extractor=image_feature_extractor, + text_encoder=text_encoder, + image_encoder=image_encoder, + image_unet=image_unet, + text_unet=text_unet, + vae=vae, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + if self.text_unet is not None and ( + "dual_cross_attention" not in self.image_unet.config or not self.image_unet.config.dual_cross_attention + ): + # if loading from a universal checkpoint rather than a saved dual-guided pipeline + self._convert_to_dual_attention() + + def remove_unused_weights(self): + self.register_modules(text_unet=None) + + def _convert_to_dual_attention(self): + """ + Replace image_unet's `Transformer2DModel` blocks with `DualTransformer2DModel` that contains transformer blocks + from both `image_unet` and `text_unet` + """ + for name, module in self.image_unet.named_modules(): + if isinstance(module, Transformer2DModel): + parent_name, index = name.rsplit(".", 1) + index = int(index) + + image_transformer = self.image_unet.get_submodule(parent_name)[index] + text_transformer = self.text_unet.get_submodule(parent_name)[index] + + config = image_transformer.config + dual_transformer = DualTransformer2DModel( + num_attention_heads=config.num_attention_heads, + attention_head_dim=config.attention_head_dim, + in_channels=config.in_channels, + num_layers=config.num_layers, + dropout=config.dropout, + norm_num_groups=config.norm_num_groups, + cross_attention_dim=config.cross_attention_dim, + attention_bias=config.attention_bias, + sample_size=config.sample_size, + num_vector_embeds=config.num_vector_embeds, + activation_fn=config.activation_fn, + num_embeds_ada_norm=config.num_embeds_ada_norm, + ) + dual_transformer.transformers[0] = image_transformer + dual_transformer.transformers[1] = text_transformer + + self.image_unet.get_submodule(parent_name)[index] = dual_transformer + self.image_unet.register_to_config(dual_cross_attention=True) + + def _revert_dual_attention(self): + """ + Revert the image_unet `DualTransformer2DModel` blocks back to `Transformer2DModel` with image_unet weights Call + this function if you reuse `image_unet` in another pipeline, e.g. `VersatileDiffusionPipeline` + """ + for name, module in self.image_unet.named_modules(): + if isinstance(module, DualTransformer2DModel): + parent_name, index = name.rsplit(".", 1) + index = int(index) + self.image_unet.get_submodule(parent_name)[index] = module.transformers[0] + + self.image_unet.register_to_config(dual_cross_attention=False) + + def _encode_text_prompt(self, prompt, device, num_images_per_prompt, do_classifier_free_guidance): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + """ + + def normalize_embeddings(encoder_output): + embeds = self.text_encoder.text_projection(encoder_output.last_hidden_state) + embeds_pooled = encoder_output.text_embeds + embeds = embeds / torch.norm(embeds_pooled.unsqueeze(1), dim=-1, keepdim=True) + return embeds + + batch_size = len(prompt) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="max_length", return_tensors="pt").input_ids + + if not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + prompt_embeds = self.text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = normalize_embeddings(prompt_embeds) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + bs_embed, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance: + uncond_tokens = [""] * batch_size + max_length = text_input_ids.shape[-1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = normalize_embeddings(negative_prompt_embeds) + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + def _encode_image_prompt(self, prompt, device, num_images_per_prompt, do_classifier_free_guidance): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + """ + + def normalize_embeddings(encoder_output): + embeds = self.image_encoder.vision_model.post_layernorm(encoder_output.last_hidden_state) + embeds = self.image_encoder.visual_projection(embeds) + embeds_pooled = embeds[:, 0:1] + embeds = embeds / torch.norm(embeds_pooled, dim=-1, keepdim=True) + return embeds + + batch_size = len(prompt) if isinstance(prompt, list) else 1 + + # get prompt text embeddings + image_input = self.image_feature_extractor(images=prompt, return_tensors="pt") + pixel_values = image_input.pixel_values.to(device).to(self.image_encoder.dtype) + image_embeddings = self.image_encoder(pixel_values) + image_embeddings = normalize_embeddings(image_embeddings) + + # duplicate image embeddings for each generation per prompt, using mps friendly method + bs_embed, seq_len, _ = image_embeddings.shape + image_embeddings = image_embeddings.repeat(1, num_images_per_prompt, 1) + image_embeddings = image_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance: + uncond_images = [np.zeros((512, 512, 3)) + 0.5] * batch_size + uncond_images = self.image_feature_extractor(images=uncond_images, return_tensors="pt") + pixel_values = uncond_images.pixel_values.to(device).to(self.image_encoder.dtype) + negative_prompt_embeds = self.image_encoder(pixel_values) + negative_prompt_embeds = normalize_embeddings(negative_prompt_embeds) + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and conditional embeddings into a single batch + # to avoid doing two forward passes + image_embeddings = torch.cat([negative_prompt_embeds, image_embeddings]) + + return image_embeddings + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs(self, prompt, image, height, width, callback_steps): + if not isinstance(prompt, str) and not isinstance(prompt, PIL.Image.Image) and not isinstance(prompt, list): + raise ValueError(f"`prompt` has to be of type `str` `PIL.Image` or `list` but is {type(prompt)}") + if not isinstance(image, str) and not isinstance(image, PIL.Image.Image) and not isinstance(image, list): + raise ValueError(f"`image` has to be of type `str` `PIL.Image` or `list` but is {type(image)}") + + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def set_transformer_params(self, mix_ratio: float = 0.5, condition_types: Tuple = ("text", "image")): + for name, module in self.image_unet.named_modules(): + if isinstance(module, DualTransformer2DModel): + module.mix_ratio = mix_ratio + + for i, type in enumerate(condition_types): + if type == "text": + module.condition_lengths[i] = self.text_encoder.config.max_position_embeddings + module.transformer_index_for_condition[i] = 1 # use the second (text) transformer + else: + module.condition_lengths[i] = 257 + module.transformer_index_for_condition[i] = 0 # use the first (image) transformer + + @torch.no_grad() + def __call__( + self, + prompt: Union[PIL.Image.Image, List[PIL.Image.Image]], + image: Union[str, List[str]], + text_to_image_strength: float = 0.5, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide image generation. + height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Examples: + + ```py + >>> from diffusers import VersatileDiffusionDualGuidedPipeline + >>> import torch + >>> import requests + >>> from io import BytesIO + >>> from PIL import Image + + >>> # let's download an initial image + >>> url = "https://huggingface.co/datasets/diffusers/images/resolve/main/benz.jpg" + + >>> response = requests.get(url) + >>> image = Image.open(BytesIO(response.content)).convert("RGB") + >>> text = "a red car in the sun" + + >>> pipe = VersatileDiffusionDualGuidedPipeline.from_pretrained( + ... "shi-labs/versatile-diffusion", torch_dtype=torch.float16 + ... ) + >>> pipe.remove_unused_weights() + >>> pipe = pipe.to("cuda") + + >>> generator = torch.Generator(device="cuda").manual_seed(0) + >>> text_to_image_strength = 0.75 + + >>> image = pipe( + ... prompt=text, image=image, text_to_image_strength=text_to_image_strength, generator=generator + ... ).images[0] + >>> image.save("./car_variation.png") + ``` + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images. + """ + # 0. Default height and width to unet + height = height or self.image_unet.config.sample_size * self.vae_scale_factor + width = width or self.image_unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs(prompt, image, height, width, callback_steps) + + # 2. Define call parameters + prompt = [prompt] if not isinstance(prompt, list) else prompt + image = [image] if not isinstance(image, list) else image + batch_size = len(prompt) + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompts + prompt_embeds = self._encode_text_prompt(prompt, device, num_images_per_prompt, do_classifier_free_guidance) + image_embeddings = self._encode_image_prompt(image, device, num_images_per_prompt, do_classifier_free_guidance) + dual_prompt_embeddings = torch.cat([prompt_embeds, image_embeddings], dim=1) + prompt_types = ("text", "image") + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.image_unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + dual_prompt_embeddings.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Combine the attention blocks of the image and text UNets + self.set_transformer_params(text_to_image_strength, prompt_types) + + # 8. Denoising loop + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.image_unet(latent_model_input, t, encoder_hidden_states=dual_prompt_embeddings).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = latents + + image = self.image_processor.postprocess(image, output_type=output_type) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py new file mode 100755 index 0000000..345c15f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_image_variation.py @@ -0,0 +1,397 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +import torch.utils.checkpoint +from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection + +from ....image_processor import VaeImageProcessor +from ....models import AutoencoderKL, UNet2DConditionModel +from ....schedulers import KarrasDiffusionSchedulers +from ....utils import deprecate, logging +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class VersatileDiffusionImageVariationPipeline(DiffusionPipeline): + r""" + Pipeline for image variation using Versatile Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + vqvae ([`VQModel`]): + Vector-quantized (VQ) model to encode and decode images to and from latent representations. + bert ([`LDMBertModel`]): + Text-encoder model based on [`~transformers.BERT`]. + tokenizer ([`~transformers.BertTokenizer`]): + A `BertTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "bert->unet->vqvae" + + image_feature_extractor: CLIPImageProcessor + image_encoder: CLIPVisionModelWithProjection + image_unet: UNet2DConditionModel + vae: AutoencoderKL + scheduler: KarrasDiffusionSchedulers + + def __init__( + self, + image_feature_extractor: CLIPImageProcessor, + image_encoder: CLIPVisionModelWithProjection, + image_unet: UNet2DConditionModel, + vae: AutoencoderKL, + scheduler: KarrasDiffusionSchedulers, + ): + super().__init__() + self.register_modules( + image_feature_extractor=image_feature_extractor, + image_encoder=image_encoder, + image_unet=image_unet, + vae=vae, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + def _encode_prompt(self, prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + """ + + def normalize_embeddings(encoder_output): + embeds = self.image_encoder.vision_model.post_layernorm(encoder_output.last_hidden_state) + embeds = self.image_encoder.visual_projection(embeds) + embeds_pooled = embeds[:, 0:1] + embeds = embeds / torch.norm(embeds_pooled, dim=-1, keepdim=True) + return embeds + + if isinstance(prompt, torch.Tensor) and len(prompt.shape) == 4: + prompt = list(prompt) + + batch_size = len(prompt) if isinstance(prompt, list) else 1 + + # get prompt text embeddings + image_input = self.image_feature_extractor(images=prompt, return_tensors="pt") + pixel_values = image_input.pixel_values.to(device).to(self.image_encoder.dtype) + image_embeddings = self.image_encoder(pixel_values) + image_embeddings = normalize_embeddings(image_embeddings) + + # duplicate image embeddings for each generation per prompt, using mps friendly method + bs_embed, seq_len, _ = image_embeddings.shape + image_embeddings = image_embeddings.repeat(1, num_images_per_prompt, 1) + image_embeddings = image_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance: + uncond_images: List[str] + if negative_prompt is None: + uncond_images = [np.zeros((512, 512, 3)) + 0.5] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, PIL.Image.Image): + uncond_images = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_images = negative_prompt + + uncond_images = self.image_feature_extractor(images=uncond_images, return_tensors="pt") + pixel_values = uncond_images.pixel_values.to(device).to(self.image_encoder.dtype) + negative_prompt_embeds = self.image_encoder(pixel_values) + negative_prompt_embeds = normalize_embeddings(negative_prompt_embeds) + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and conditional embeddings into a single batch + # to avoid doing two forward passes + image_embeddings = torch.cat([negative_prompt_embeds, image_embeddings]) + + return image_embeddings + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_image_variation.StableDiffusionImageVariationPipeline.check_inputs + def check_inputs(self, image, height, width, callback_steps): + if ( + not isinstance(image, torch.Tensor) + and not isinstance(image, PIL.Image.Image) + and not isinstance(image, list) + ): + raise ValueError( + "`image` has to be of type `torch.FloatTensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + f" {type(image)}" + ) + + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + def __call__( + self, + image: Union[PIL.Image.Image, List[PIL.Image.Image], torch.Tensor], + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + image (`PIL.Image.Image`, `List[PIL.Image.Image]` or `torch.Tensor`): + The image prompt or prompts to guide the image generation. + height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Examples: + + ```py + >>> from diffusers import VersatileDiffusionImageVariationPipeline + >>> import torch + >>> import requests + >>> from io import BytesIO + >>> from PIL import Image + + >>> # let's download an initial image + >>> url = "https://huggingface.co/datasets/diffusers/images/resolve/main/benz.jpg" + + >>> response = requests.get(url) + >>> image = Image.open(BytesIO(response.content)).convert("RGB") + + >>> pipe = VersatileDiffusionImageVariationPipeline.from_pretrained( + ... "shi-labs/versatile-diffusion", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> generator = torch.Generator(device="cuda").manual_seed(0) + >>> image = pipe(image, generator=generator).images[0] + >>> image.save("./car_variation.png") + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images. + """ + # 0. Default height and width to unet + height = height or self.image_unet.config.sample_size * self.vae_scale_factor + width = width or self.image_unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs(image, height, width, callback_steps) + + # 2. Define call parameters + batch_size = 1 if isinstance(image, PIL.Image.Image) else len(image) + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + image_embeddings = self._encode_prompt( + image, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.image_unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + image_embeddings.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.image_unet(latent_model_input, t, encoder_hidden_states=image_embeddings).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = latents + + image = self.image_processor.postprocess(image, output_type=output_type) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py new file mode 100755 index 0000000..0b2518f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/versatile_diffusion/pipeline_versatile_diffusion_text_to_image.py @@ -0,0 +1,475 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Callable, List, Optional, Union + +import torch +import torch.utils.checkpoint +from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer + +from ....image_processor import VaeImageProcessor +from ....models import AutoencoderKL, Transformer2DModel, UNet2DConditionModel +from ....schedulers import KarrasDiffusionSchedulers +from ....utils import deprecate, logging +from ....utils.torch_utils import randn_tensor +from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from .modeling_text_unet import UNetFlatConditionModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class VersatileDiffusionTextToImagePipeline(DiffusionPipeline): + r""" + Pipeline for text-to-image generation using Versatile Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + vqvae ([`VQModel`]): + Vector-quantized (VQ) model to encode and decode images to and from latent representations. + bert ([`LDMBertModel`]): + Text-encoder model based on [`~transformers.BERT`]. + tokenizer ([`~transformers.BertTokenizer`]): + A `BertTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "bert->unet->vqvae" + + tokenizer: CLIPTokenizer + image_feature_extractor: CLIPImageProcessor + text_encoder: CLIPTextModelWithProjection + image_unet: UNet2DConditionModel + text_unet: UNetFlatConditionModel + vae: AutoencoderKL + scheduler: KarrasDiffusionSchedulers + + _optional_components = ["text_unet"] + + def __init__( + self, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModelWithProjection, + image_unet: UNet2DConditionModel, + text_unet: UNetFlatConditionModel, + vae: AutoencoderKL, + scheduler: KarrasDiffusionSchedulers, + ): + super().__init__() + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + image_unet=image_unet, + text_unet=text_unet, + vae=vae, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + if self.text_unet is not None: + self._swap_unet_attention_blocks() + + def _swap_unet_attention_blocks(self): + """ + Swap the `Transformer2DModel` blocks between the image and text UNets + """ + for name, module in self.image_unet.named_modules(): + if isinstance(module, Transformer2DModel): + parent_name, index = name.rsplit(".", 1) + index = int(index) + self.image_unet.get_submodule(parent_name)[index], self.text_unet.get_submodule(parent_name)[index] = ( + self.text_unet.get_submodule(parent_name)[index], + self.image_unet.get_submodule(parent_name)[index], + ) + + def remove_unused_weights(self): + self.register_modules(text_unet=None) + + def _encode_prompt(self, prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + """ + + def normalize_embeddings(encoder_output): + embeds = self.text_encoder.text_projection(encoder_output.last_hidden_state) + embeds_pooled = encoder_output.text_embeds + embeds = embeds / torch.norm(embeds_pooled.unsqueeze(1), dim=-1, keepdim=True) + return embeds + + batch_size = len(prompt) if isinstance(prompt, list) else 1 + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="max_length", return_tensors="pt").input_ids + + if not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + prompt_embeds = self.text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = normalize_embeddings(prompt_embeds) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + bs_embed, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = text_input_ids.shape[-1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = normalize_embeddings(negative_prompt_embeds) + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide image generation. + height (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.image_unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Examples: + + ```py + >>> from diffusers import VersatileDiffusionTextToImagePipeline + >>> import torch + + >>> pipe = VersatileDiffusionTextToImagePipeline.from_pretrained( + ... "shi-labs/versatile-diffusion", torch_dtype=torch.float16 + ... ) + >>> pipe.remove_unused_weights() + >>> pipe = pipe.to("cuda") + + >>> generator = torch.Generator(device="cuda").manual_seed(0) + >>> image = pipe("an astronaut riding on a horse on mars", generator=generator).images[0] + >>> image.save("./astronaut.png") + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images. + """ + # 0. Default height and width to unet + height = height or self.image_unet.config.sample_size * self.vae_scale_factor + width = width or self.image_unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs(prompt, height, width, callback_steps) + + # 2. Define call parameters + batch_size = 1 if isinstance(prompt, str) else len(prompt) + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds = self._encode_prompt( + prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.image_unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.image_unet(latent_model_input, t, encoder_hidden_states=prompt_embeds).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = latents + + image = self.image_processor.postprocess(image, output_type=output_type) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/vq_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/vq_diffusion/__init__.py new file mode 100755 index 0000000..0709033 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/vq_diffusion/__init__.py @@ -0,0 +1,57 @@ +from typing import TYPE_CHECKING + +from ....utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ....utils.dummy_torch_and_transformers_objects import ( + LearnedClassifierFreeSamplingEmbeddings, + VQDiffusionPipeline, + ) + + _dummy_objects.update( + { + "LearnedClassifierFreeSamplingEmbeddings": LearnedClassifierFreeSamplingEmbeddings, + "VQDiffusionPipeline": VQDiffusionPipeline, + } + ) +else: + _import_structure["pipeline_vq_diffusion"] = ["LearnedClassifierFreeSamplingEmbeddings", "VQDiffusionPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ....utils.dummy_torch_and_transformers_objects import ( + LearnedClassifierFreeSamplingEmbeddings, + VQDiffusionPipeline, + ) + else: + from .pipeline_vq_diffusion import LearnedClassifierFreeSamplingEmbeddings, VQDiffusionPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py new file mode 100755 index 0000000..0c55d04 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/deprecated/vq_diffusion/pipeline_vq_diffusion.py @@ -0,0 +1,325 @@ +# Copyright 2024 Microsoft and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, List, Optional, Tuple, Union + +import torch +from transformers import CLIPTextModel, CLIPTokenizer + +from ....configuration_utils import ConfigMixin, register_to_config +from ....models import ModelMixin, Transformer2DModel, VQModel +from ....schedulers import VQDiffusionScheduler +from ....utils import logging +from ...pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class LearnedClassifierFreeSamplingEmbeddings(ModelMixin, ConfigMixin): + """ + Utility class for storing learned text embeddings for classifier free sampling + """ + + @register_to_config + def __init__(self, learnable: bool, hidden_size: Optional[int] = None, length: Optional[int] = None): + super().__init__() + + self.learnable = learnable + + if self.learnable: + assert hidden_size is not None, "learnable=True requires `hidden_size` to be set" + assert length is not None, "learnable=True requires `length` to be set" + + embeddings = torch.zeros(length, hidden_size) + else: + embeddings = None + + self.embeddings = torch.nn.Parameter(embeddings) + + +class VQDiffusionPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-image generation using VQ Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vqvae ([`VQModel`]): + Vector Quantized Variational Auto-Encoder (VAE) model to encode and decode images to and from latent + representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-base-patch32](https://huggingface.co/openai/clip-vit-base-patch32)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + transformer ([`Transformer2DModel`]): + A conditional `Transformer2DModel` to denoise the encoded image latents. + scheduler ([`VQDiffusionScheduler`]): + A scheduler to be used in combination with `transformer` to denoise the encoded image latents. + """ + + vqvae: VQModel + text_encoder: CLIPTextModel + tokenizer: CLIPTokenizer + transformer: Transformer2DModel + learned_classifier_free_sampling_embeddings: LearnedClassifierFreeSamplingEmbeddings + scheduler: VQDiffusionScheduler + + def __init__( + self, + vqvae: VQModel, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + transformer: Transformer2DModel, + scheduler: VQDiffusionScheduler, + learned_classifier_free_sampling_embeddings: LearnedClassifierFreeSamplingEmbeddings, + ): + super().__init__() + + self.register_modules( + vqvae=vqvae, + transformer=transformer, + text_encoder=text_encoder, + tokenizer=tokenizer, + scheduler=scheduler, + learned_classifier_free_sampling_embeddings=learned_classifier_free_sampling_embeddings, + ) + + def _encode_prompt(self, prompt, num_images_per_prompt, do_classifier_free_guidance): + batch_size = len(prompt) if isinstance(prompt, list) else 1 + + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + + if text_input_ids.shape[-1] > self.tokenizer.model_max_length: + removed_text = self.tokenizer.batch_decode(text_input_ids[:, self.tokenizer.model_max_length :]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + text_input_ids = text_input_ids[:, : self.tokenizer.model_max_length] + prompt_embeds = self.text_encoder(text_input_ids.to(self.device))[0] + + # NOTE: This additional step of normalizing the text embeddings is from VQ-Diffusion. + # While CLIP does normalize the pooled output of the text transformer when combining + # the image and text embeddings, CLIP does not directly normalize the last hidden state. + # + # CLIP normalizing the pooled output. + # https://github.com/huggingface/transformers/blob/d92e22d1f28324f513f3080e5c47c071a3916721/src/transformers/models/clip/modeling_clip.py#L1052-L1053 + prompt_embeds = prompt_embeds / prompt_embeds.norm(dim=-1, keepdim=True) + + # duplicate text embeddings for each generation per prompt + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + + if do_classifier_free_guidance: + if self.learned_classifier_free_sampling_embeddings.learnable: + negative_prompt_embeds = self.learned_classifier_free_sampling_embeddings.embeddings + negative_prompt_embeds = negative_prompt_embeds.unsqueeze(0).repeat(batch_size, 1, 1) + else: + uncond_tokens = [""] * batch_size + + max_length = text_input_ids.shape[-1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + negative_prompt_embeds = self.text_encoder(uncond_input.input_ids.to(self.device))[0] + # See comment for normalizing text embeddings + negative_prompt_embeds = negative_prompt_embeds / negative_prompt_embeds.norm(dim=-1, keepdim=True) + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + num_inference_steps: int = 100, + guidance_scale: float = 5.0, + truncation_rate: float = 1.0, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + ) -> Union[ImagePipelineOutput, Tuple]: + """ + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide image generation. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + truncation_rate (`float`, *optional*, defaults to 1.0 (equivalent to no truncation)): + Used to "truncate" the predicted classes for x_0 such that the cumulative probability for a pixel is at + most `truncation_rate`. The lowest probabilities that would increase the cumulative probability above + `truncation_rate` are set to zero. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor` of shape (batch), *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Must be valid embedding indices.If not provided, a latents tensor will be generated of + completely masked latent pixels. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images. + """ + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + batch_size = batch_size * num_images_per_prompt + + do_classifier_free_guidance = guidance_scale > 1.0 + + prompt_embeds = self._encode_prompt(prompt, num_images_per_prompt, do_classifier_free_guidance) + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + # get the initial completely masked latents unless the user supplied it + + latents_shape = (batch_size, self.transformer.num_latent_pixels) + if latents is None: + mask_class = self.transformer.num_vector_embeds - 1 + latents = torch.full(latents_shape, mask_class).to(self.device) + else: + if latents.shape != latents_shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}") + if (latents < 0).any() or (latents >= self.transformer.num_vector_embeds).any(): + raise ValueError( + "Unexpected latents value(s). All latents be valid embedding indices i.e. in the range 0," + f" {self.transformer.num_vector_embeds - 1} (inclusive)." + ) + latents = latents.to(self.device) + + # set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=self.device) + + timesteps_tensor = self.scheduler.timesteps.to(self.device) + + sample = latents + + for i, t in enumerate(self.progress_bar(timesteps_tensor)): + # expand the sample if we are doing classifier free guidance + latent_model_input = torch.cat([sample] * 2) if do_classifier_free_guidance else sample + + # predict the un-noised image + # model_output == `log_p_x_0` + model_output = self.transformer(latent_model_input, encoder_hidden_states=prompt_embeds, timestep=t).sample + + if do_classifier_free_guidance: + model_output_uncond, model_output_text = model_output.chunk(2) + model_output = model_output_uncond + guidance_scale * (model_output_text - model_output_uncond) + model_output -= torch.logsumexp(model_output, dim=1, keepdim=True) + + model_output = self.truncate(model_output, truncation_rate) + + # remove `log(0)`'s (`-inf`s) + model_output = model_output.clamp(-70) + + # compute the previous noisy sample x_t -> x_t-1 + sample = self.scheduler.step(model_output, timestep=t, sample=sample, generator=generator).prev_sample + + # call the callback, if provided + if callback is not None and i % callback_steps == 0: + callback(i, t, sample) + + embedding_channels = self.vqvae.config.vq_embed_dim + embeddings_shape = (batch_size, self.transformer.height, self.transformer.width, embedding_channels) + embeddings = self.vqvae.quantize.get_codebook_entry(sample, shape=embeddings_shape) + image = self.vqvae.decode(embeddings, force_not_quantize=True).sample + + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) + + def truncate(self, log_p_x_0: torch.FloatTensor, truncation_rate: float) -> torch.FloatTensor: + """ + Truncates `log_p_x_0` such that for each column vector, the total cumulative probability is `truncation_rate` + The lowest probabilities that would increase the cumulative probability above `truncation_rate` are set to + zero. + """ + sorted_log_p_x_0, indices = torch.sort(log_p_x_0, 1, descending=True) + sorted_p_x_0 = torch.exp(sorted_log_p_x_0) + keep_mask = sorted_p_x_0.cumsum(dim=1) < truncation_rate + + # Ensure that at least the largest probability is not zeroed out + all_true = torch.full_like(keep_mask[:, 0:1, :], True) + keep_mask = torch.cat((all_true, keep_mask), dim=1) + keep_mask = keep_mask[:, :-1, :] + + keep_mask = keep_mask.gather(1, indices.argsort(1)) + + rv = log_p_x_0.clone() + + rv[~keep_mask] = -torch.inf # -inf = log(0) + + return rv diff --git a/diffusers-0.27.0/src/diffusers/pipelines/dit/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/dit/__init__.py new file mode 100755 index 0000000..fe2a94f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/dit/__init__.py @@ -0,0 +1,19 @@ +from typing import TYPE_CHECKING + +from ...utils import DIFFUSERS_SLOW_IMPORT, _LazyModule + + +_import_structure = {"pipeline_dit": ["DiTPipeline"]} + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from .pipeline_dit import DiTPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/dit/pipeline_dit.py b/diffusers-0.27.0/src/diffusers/pipelines/dit/pipeline_dit.py new file mode 100755 index 0000000..289ea49 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/dit/pipeline_dit.py @@ -0,0 +1,233 @@ +# Attribution-NonCommercial 4.0 International (CC BY-NC 4.0) +# William Peebles and Saining Xie +# +# Copyright (c) 2021 OpenAI +# MIT License +# +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Dict, List, Optional, Tuple, Union + +import torch + +from ...models import AutoencoderKL, Transformer2DModel +from ...schedulers import KarrasDiffusionSchedulers +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +class DiTPipeline(DiffusionPipeline): + r""" + Pipeline for image generation based on a Transformer backbone instead of a UNet. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + transformer ([`Transformer2DModel`]): + A class conditioned `Transformer2DModel` to denoise the encoded image latents. + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + scheduler ([`DDIMScheduler`]): + A scheduler to be used in combination with `transformer` to denoise the encoded image latents. + """ + + model_cpu_offload_seq = "transformer->vae" + + def __init__( + self, + transformer: Transformer2DModel, + vae: AutoencoderKL, + scheduler: KarrasDiffusionSchedulers, + id2label: Optional[Dict[int, str]] = None, + ): + super().__init__() + self.register_modules(transformer=transformer, vae=vae, scheduler=scheduler) + + # create a imagenet -> id dictionary for easier use + self.labels = {} + if id2label is not None: + for key, value in id2label.items(): + for label in value.split(","): + self.labels[label.lstrip().rstrip()] = int(key) + self.labels = dict(sorted(self.labels.items())) + + def get_label_ids(self, label: Union[str, List[str]]) -> List[int]: + r""" + + Map label strings from ImageNet to corresponding class ids. + + Parameters: + label (`str` or `dict` of `str`): + Label strings to be mapped to class ids. + + Returns: + `list` of `int`: + Class ids to be processed by pipeline. + """ + + if not isinstance(label, list): + label = list(label) + + for l in label: + if l not in self.labels: + raise ValueError( + f"{l} does not exist. Please make sure to select one of the following labels: \n {self.labels}." + ) + + return [self.labels[l] for l in label] + + @torch.no_grad() + def __call__( + self, + class_labels: List[int], + guidance_scale: float = 4.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + num_inference_steps: int = 50, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[ImagePipelineOutput, Tuple]: + r""" + The call function to the pipeline for generation. + + Args: + class_labels (List[int]): + List of ImageNet class labels for the images to be generated. + guidance_scale (`float`, *optional*, defaults to 4.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + num_inference_steps (`int`, *optional*, defaults to 250): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + ```py + >>> from diffusers import DiTPipeline, DPMSolverMultistepScheduler + >>> import torch + + >>> pipe = DiTPipeline.from_pretrained("facebook/DiT-XL-2-256", torch_dtype=torch.float16) + >>> pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + >>> pipe = pipe.to("cuda") + + >>> # pick words from Imagenet class labels + >>> pipe.labels # to print all available words + + >>> # pick words that exist in ImageNet + >>> words = ["white shark", "umbrella"] + + >>> class_ids = pipe.get_label_ids(words) + + >>> generator = torch.manual_seed(33) + >>> output = pipe(class_labels=class_ids, num_inference_steps=25, generator=generator) + + >>> image = output.images[0] # label 'white shark' + ``` + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images + """ + + batch_size = len(class_labels) + latent_size = self.transformer.config.sample_size + latent_channels = self.transformer.config.in_channels + + latents = randn_tensor( + shape=(batch_size, latent_channels, latent_size, latent_size), + generator=generator, + device=self._execution_device, + dtype=self.transformer.dtype, + ) + latent_model_input = torch.cat([latents] * 2) if guidance_scale > 1 else latents + + class_labels = torch.tensor(class_labels, device=self._execution_device).reshape(-1) + class_null = torch.tensor([1000] * batch_size, device=self._execution_device) + class_labels_input = torch.cat([class_labels, class_null], 0) if guidance_scale > 1 else class_labels + + # set step values + self.scheduler.set_timesteps(num_inference_steps) + for t in self.progress_bar(self.scheduler.timesteps): + if guidance_scale > 1: + half = latent_model_input[: len(latent_model_input) // 2] + latent_model_input = torch.cat([half, half], dim=0) + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + timesteps = t + if not torch.is_tensor(timesteps): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = latent_model_input.device.type == "mps" + if isinstance(timesteps, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + timesteps = torch.tensor([timesteps], dtype=dtype, device=latent_model_input.device) + elif len(timesteps.shape) == 0: + timesteps = timesteps[None].to(latent_model_input.device) + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timesteps = timesteps.expand(latent_model_input.shape[0]) + # predict noise model_output + noise_pred = self.transformer( + latent_model_input, timestep=timesteps, class_labels=class_labels_input + ).sample + + # perform guidance + if guidance_scale > 1: + eps, rest = noise_pred[:, :latent_channels], noise_pred[:, latent_channels:] + cond_eps, uncond_eps = torch.split(eps, len(eps) // 2, dim=0) + + half_eps = uncond_eps + guidance_scale * (cond_eps - uncond_eps) + eps = torch.cat([half_eps, half_eps], dim=0) + + noise_pred = torch.cat([eps, rest], dim=1) + + # learned sigma + if self.transformer.config.out_channels // 2 == latent_channels: + model_output, _ = torch.split(noise_pred, latent_channels, dim=1) + else: + model_output = noise_pred + + # compute previous image: x_t -> x_t-1 + latent_model_input = self.scheduler.step(model_output, t, latent_model_input).prev_sample + + if guidance_scale > 1: + latents, _ = latent_model_input.chunk(2, dim=0) + else: + latents = latent_model_input + + latents = 1 / self.vae.config.scaling_factor * latents + samples = self.vae.decode(latents).sample + + samples = (samples / 2 + 0.5).clamp(0, 1) + + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + samples = samples.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + samples = self.numpy_to_pil(samples) + + if not return_dict: + return (samples,) + + return ImagePipelineOutput(images=samples) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/free_init_utils.py b/diffusers-0.27.0/src/diffusers/pipelines/free_init_utils.py new file mode 100755 index 0000000..50c28cc --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/free_init_utils.py @@ -0,0 +1,184 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Tuple, Union + +import torch +import torch.fft as fft + +from ..utils.torch_utils import randn_tensor + + +class FreeInitMixin: + r"""Mixin class for FreeInit.""" + + def enable_free_init( + self, + num_iters: int = 3, + use_fast_sampling: bool = False, + method: str = "butterworth", + order: int = 4, + spatial_stop_frequency: float = 0.25, + temporal_stop_frequency: float = 0.25, + ): + """Enables the FreeInit mechanism as in https://arxiv.org/abs/2312.07537. + + This implementation has been adapted from the [official repository](https://github.com/TianxingWu/FreeInit). + + Args: + num_iters (`int`, *optional*, defaults to `3`): + Number of FreeInit noise re-initialization iterations. + use_fast_sampling (`bool`, *optional*, defaults to `False`): + Whether or not to speedup sampling procedure at the cost of probably lower quality results. Enables + the "Coarse-to-Fine Sampling" strategy, as mentioned in the paper, if set to `True`. + method (`str`, *optional*, defaults to `butterworth`): + Must be one of `butterworth`, `ideal` or `gaussian` to use as the filtering method for the + FreeInit low pass filter. + order (`int`, *optional*, defaults to `4`): + Order of the filter used in `butterworth` method. Larger values lead to `ideal` method behaviour + whereas lower values lead to `gaussian` method behaviour. + spatial_stop_frequency (`float`, *optional*, defaults to `0.25`): + Normalized stop frequency for spatial dimensions. Must be between 0 to 1. Referred to as `d_s` in + the original implementation. + temporal_stop_frequency (`float`, *optional*, defaults to `0.25`): + Normalized stop frequency for temporal dimensions. Must be between 0 to 1. Referred to as `d_t` in + the original implementation. + """ + self._free_init_num_iters = num_iters + self._free_init_use_fast_sampling = use_fast_sampling + self._free_init_method = method + self._free_init_order = order + self._free_init_spatial_stop_frequency = spatial_stop_frequency + self._free_init_temporal_stop_frequency = temporal_stop_frequency + + def disable_free_init(self): + """Disables the FreeInit mechanism if enabled.""" + self._free_init_num_iters = None + + @property + def free_init_enabled(self): + return hasattr(self, "_free_init_num_iters") and self._free_init_num_iters is not None + + def _get_free_init_freq_filter( + self, + shape: Tuple[int, ...], + device: Union[str, torch.dtype], + filter_type: str, + order: float, + spatial_stop_frequency: float, + temporal_stop_frequency: float, + ) -> torch.Tensor: + r"""Returns the FreeInit filter based on filter type and other input conditions.""" + + time, height, width = shape[-3], shape[-2], shape[-1] + mask = torch.zeros(shape) + + if spatial_stop_frequency == 0 or temporal_stop_frequency == 0: + return mask + + if filter_type == "butterworth": + + def retrieve_mask(x): + return 1 / (1 + (x / spatial_stop_frequency**2) ** order) + elif filter_type == "gaussian": + + def retrieve_mask(x): + return math.exp(-1 / (2 * spatial_stop_frequency**2) * x) + elif filter_type == "ideal": + + def retrieve_mask(x): + return 1 if x <= spatial_stop_frequency * 2 else 0 + else: + raise NotImplementedError("`filter_type` must be one of gaussian, butterworth or ideal") + + for t in range(time): + for h in range(height): + for w in range(width): + d_square = ( + ((spatial_stop_frequency / temporal_stop_frequency) * (2 * t / time - 1)) ** 2 + + (2 * h / height - 1) ** 2 + + (2 * w / width - 1) ** 2 + ) + mask[..., t, h, w] = retrieve_mask(d_square) + + return mask.to(device) + + def _apply_freq_filter(self, x: torch.Tensor, noise: torch.Tensor, low_pass_filter: torch.Tensor) -> torch.Tensor: + r"""Noise reinitialization.""" + # FFT + x_freq = fft.fftn(x, dim=(-3, -2, -1)) + x_freq = fft.fftshift(x_freq, dim=(-3, -2, -1)) + noise_freq = fft.fftn(noise, dim=(-3, -2, -1)) + noise_freq = fft.fftshift(noise_freq, dim=(-3, -2, -1)) + + # frequency mix + high_pass_filter = 1 - low_pass_filter + x_freq_low = x_freq * low_pass_filter + noise_freq_high = noise_freq * high_pass_filter + x_freq_mixed = x_freq_low + noise_freq_high # mix in freq domain + + # IFFT + x_freq_mixed = fft.ifftshift(x_freq_mixed, dim=(-3, -2, -1)) + x_mixed = fft.ifftn(x_freq_mixed, dim=(-3, -2, -1)).real + + return x_mixed + + def _apply_free_init( + self, + latents: torch.Tensor, + free_init_iteration: int, + num_inference_steps: int, + device: torch.device, + dtype: torch.dtype, + generator: torch.Generator, + ): + if free_init_iteration == 0: + self._free_init_initial_noise = latents.detach().clone() + return latents, self.scheduler.timesteps + + latent_shape = latents.shape + + free_init_filter_shape = (1, *latent_shape[1:]) + free_init_freq_filter = self._get_free_init_freq_filter( + shape=free_init_filter_shape, + device=device, + filter_type=self._free_init_method, + order=self._free_init_order, + spatial_stop_frequency=self._free_init_spatial_stop_frequency, + temporal_stop_frequency=self._free_init_temporal_stop_frequency, + ) + + current_diffuse_timestep = self.scheduler.config.num_train_timesteps - 1 + diffuse_timesteps = torch.full((latent_shape[0],), current_diffuse_timestep).long() + + z_t = self.scheduler.add_noise( + original_samples=latents, noise=self._free_init_initial_noise, timesteps=diffuse_timesteps.to(device) + ).to(dtype=torch.float32) + + z_rand = randn_tensor( + shape=latent_shape, + generator=generator, + device=device, + dtype=torch.float32, + ) + latents = self._apply_freq_filter(z_t, z_rand, low_pass_filter=free_init_freq_filter) + latents = latents.to(dtype) + + # Coarse-to-Fine Sampling for faster inference (can lead to lower quality) + if self._free_init_use_fast_sampling: + num_inference_steps = int(num_inference_steps / self._free_init_num_iters * (free_init_iteration + 1)) + self.scheduler.set_timesteps(num_inference_steps, device=device) + + return latents, self.scheduler.timesteps diff --git a/diffusers-0.27.0/src/diffusers/pipelines/i2vgen_xl/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/i2vgen_xl/__init__.py new file mode 100755 index 0000000..b24a7e4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/i2vgen_xl/__init__.py @@ -0,0 +1,46 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_i2vgen_xl"] = ["I2VGenXLPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 + else: + from .pipeline_i2vgen_xl import I2VGenXLPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py b/diffusers-0.27.0/src/diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py new file mode 100755 index 0000000..cb6f3e3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/i2vgen_xl/pipeline_i2vgen_xl.py @@ -0,0 +1,798 @@ +# Copyright 2024 Alibaba DAMO-VILAB and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...models import AutoencoderKL +from ...models.unets.unet_i2vgen_xl import I2VGenXLUNet +from ...schedulers import DDIMScheduler +from ...utils import ( + BaseOutput, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import I2VGenXLPipeline + >>> from diffusers.utils import export_to_gif, load_image + + >>> pipeline = I2VGenXLPipeline.from_pretrained("ali-vilab/i2vgen-xl", torch_dtype=torch.float16, variant="fp16") + >>> pipeline.enable_model_cpu_offload() + + >>> image_url = "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/i2vgen_xl_images/img_0009.png" + >>> image = load_image(image_url).convert("RGB") + + >>> prompt = "Papers were floating in the air on a table in the library" + >>> negative_prompt = "Distorted, discontinuous, Ugly, blurry, low resolution, motionless, static, disfigured, disconnected limbs, Ugly faces, incomplete arms" + >>> generator = torch.manual_seed(8888) + + >>> frames = pipeline( + ... prompt=prompt, + ... image=image, + ... num_inference_steps=50, + ... negative_prompt=negative_prompt, + ... guidance_scale=9.0, + ... generator=generator + ... ).frames[0] + >>> video_path = export_to_gif(frames, "i2v.gif") + ``` +""" + + +# Copied from diffusers.pipelines.animatediff.pipeline_animatediff.tensor2vid +def tensor2vid(video: torch.Tensor, processor: "VaeImageProcessor", output_type: str = "np"): + batch_size, channels, num_frames, height, width = video.shape + outputs = [] + for batch_idx in range(batch_size): + batch_vid = video[batch_idx].permute(1, 0, 2, 3) + batch_output = processor.postprocess(batch_vid, output_type) + + outputs.append(batch_output) + + if output_type == "np": + outputs = np.stack(outputs) + + elif output_type == "pt": + outputs = torch.stack(outputs) + + elif not output_type == "pil": + raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil']") + + return outputs + + +@dataclass +class I2VGenXLPipelineOutput(BaseOutput): + r""" + Output class for image-to-video pipeline. + + Args: + frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised + PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)` + """ + + frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]] + + +class I2VGenXLPipeline( + DiffusionPipeline, + StableDiffusionMixin, +): + r""" + Pipeline for image-to-video generation as proposed in [I2VGenXL](https://i2vgen-xl.github.io/). + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (`CLIPTokenizer`): + A [`~transformers.CLIPTokenizer`] to tokenize text. + unet ([`I2VGenXLUNet`]): + A [`I2VGenXLUNet`] to denoise the encoded video latents. + scheduler ([`DDIMScheduler`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + image_encoder: CLIPVisionModelWithProjection, + feature_extractor: CLIPImageProcessor, + unet: I2VGenXLUNet, + scheduler: DDIMScheduler, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + unet=unet, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + # `do_resize=False` as we do custom resizing. + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_resize=False) + + @property + def guidance_scale(self): + return self._guidance_scale + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + def encode_prompt( + self, + prompt, + device, + num_videos_per_prompt, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_videos_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_videos_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if self.do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + # Apply clip_skip to negative prompt embeds + if clip_skip is None: + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + else: + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + negative_prompt_embeds = negative_prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + negative_prompt_embeds = self.text_encoder.text_model.final_layer_norm(negative_prompt_embeds) + + if self.do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_videos_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1) + + return prompt_embeds, negative_prompt_embeds + + def _encode_image(self, image, device, num_videos_per_prompt): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.image_processor.pil_to_numpy(image) + image = self.image_processor.numpy_to_pt(image) + + # Normalize the image with CLIP training stats. + image = self.feature_extractor( + images=image, + do_normalize=True, + do_center_crop=False, + do_resize=False, + do_rescale=False, + return_tensors="pt", + ).pixel_values + + image = image.to(device=device, dtype=dtype) + image_embeddings = self.image_encoder(image).image_embeds + image_embeddings = image_embeddings.unsqueeze(1) + + # duplicate image embeddings for each generation per prompt, using mps friendly method + bs_embed, seq_len, _ = image_embeddings.shape + image_embeddings = image_embeddings.repeat(1, num_videos_per_prompt, 1) + image_embeddings = image_embeddings.view(bs_embed * num_videos_per_prompt, seq_len, -1) + + if self.do_classifier_free_guidance: + negative_image_embeddings = torch.zeros_like(image_embeddings) + image_embeddings = torch.cat([negative_image_embeddings, image_embeddings]) + + return image_embeddings + + def decode_latents(self, latents, decode_chunk_size=None): + latents = 1 / self.vae.config.scaling_factor * latents + + batch_size, channels, num_frames, height, width = latents.shape + latents = latents.permute(0, 2, 1, 3, 4).reshape(batch_size * num_frames, channels, height, width) + + if decode_chunk_size is not None: + frames = [] + for i in range(0, latents.shape[0], decode_chunk_size): + frame = self.vae.decode(latents[i : i + decode_chunk_size]).sample + frames.append(frame) + image = torch.cat(frames, dim=0) + else: + image = self.vae.decode(latents).sample + + decode_shape = (batch_size, num_frames, -1) + image.shape[2:] + video = image[None, :].reshape(decode_shape).permute(0, 2, 1, 3, 4) + + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + video = video.float() + return video + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + image, + height, + width, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if ( + not isinstance(image, torch.Tensor) + and not isinstance(image, PIL.Image.Image) + and not isinstance(image, list) + ): + raise ValueError( + "`image` has to be of type `torch.FloatTensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + f" {type(image)}" + ) + + def prepare_image_latents( + self, + image, + device, + num_frames, + num_videos_per_prompt, + ): + image = image.to(device=device) + image_latents = self.vae.encode(image).latent_dist.sample() + image_latents = image_latents * self.vae.config.scaling_factor + + # Add frames dimension to image latents + image_latents = image_latents.unsqueeze(2) + + # Append a position mask for each subsequent frame + # after the intial image latent frame + frame_position_mask = [] + for frame_idx in range(num_frames - 1): + scale = (frame_idx + 1) / (num_frames - 1) + frame_position_mask.append(torch.ones_like(image_latents[:, :, :1]) * scale) + if frame_position_mask: + frame_position_mask = torch.cat(frame_position_mask, dim=2) + image_latents = torch.cat([image_latents, frame_position_mask], dim=2) + + # duplicate image_latents for each generation per prompt, using mps friendly method + image_latents = image_latents.repeat(num_videos_per_prompt, 1, 1, 1, 1) + + if self.do_classifier_free_guidance: + image_latents = torch.cat([image_latents] * 2) + + return image_latents + + # Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_synth.TextToVideoSDPipeline.prepare_latents + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + shape = ( + batch_size, + num_channels_latents, + num_frames, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + height: Optional[int] = 704, + width: Optional[int] = 1280, + target_fps: Optional[int] = 16, + num_frames: int = 16, + num_inference_steps: int = 50, + guidance_scale: float = 9.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + eta: float = 0.0, + num_videos_per_prompt: Optional[int] = 1, + decode_chunk_size: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = 1, + ): + r""" + The call function to the pipeline for image-to-video generation with [`I2VGenXLPipeline`]. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.FloatTensor`): + Image or images to guide image generation. If you provide a tensor, it needs to be compatible with + [`CLIPImageProcessor`](https://huggingface.co/lambdalabs/sd-image-variations-diffusers/blob/main/feature_extractor/preprocessor_config.json). + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + target_fps (`int`, *optional*): + Frames per second. The rate at which the generated images shall be exported to a video after generation. This is also used as a "micro-condition" while generation. + num_frames (`int`, *optional*): + The number of video frames to generate. + num_inference_steps (`int`, *optional*): + The number of denoising steps. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + eta (`float`, *optional*): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + num_videos_per_prompt (`int`, *optional*): + The number of images to generate per prompt. + decode_chunk_size (`int`, *optional*): + The number of frames to decode at a time. The higher the chunk size, the higher the temporal consistency + between frames, but also the higher the memory consumption. By default, the decoder will decode all frames at once + for maximal quality. Reduce `decode_chunk_size` to reduce memory usage. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + + Examples: + + Returns: + [`pipelines.i2vgen_xl.pipeline_i2vgen_xl.I2VGenXLPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`pipelines.i2vgen_xl.pipeline_i2vgen_xl.I2VGenXLPipelineOutput`] is + returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs(prompt, image, height, width, negative_prompt, prompt_embeds, negative_prompt_embeds) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + self._guidance_scale = guidance_scale + + # 3.1 Encode input text prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_videos_per_prompt, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 3.2 Encode image prompt + # 3.2.1 Image encodings. + # https://github.com/ali-vilab/i2vgen-xl/blob/2539c9262ff8a2a22fa9daecbfd13f0a2dbc32d0/tools/inferences/inference_i2vgen_entrance.py#L114 + cropped_image = _center_crop_wide(image, (width, width)) + cropped_image = _resize_bilinear( + cropped_image, (self.feature_extractor.crop_size["width"], self.feature_extractor.crop_size["height"]) + ) + image_embeddings = self._encode_image(cropped_image, device, num_videos_per_prompt) + + # 3.2.2 Image latents. + resized_image = _center_crop_wide(image, (width, height)) + image = self.image_processor.preprocess(resized_image).to(device=device, dtype=image_embeddings.dtype) + image_latents = self.prepare_image_latents( + image, + device=device, + num_frames=num_frames, + num_videos_per_prompt=num_videos_per_prompt, + ) + + # 3.3 Prepare additional conditions for the UNet. + if self.do_classifier_free_guidance: + fps_tensor = torch.tensor([target_fps, target_fps]).to(device) + else: + fps_tensor = torch.tensor([target_fps]).to(device) + fps_tensor = fps_tensor.repeat(batch_size * num_videos_per_prompt, 1).ravel() + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + num_channels_latents, + num_frames, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + fps=fps_tensor, + image_latents=image_latents, + image_embeddings=image_embeddings, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # reshape latents + batch_size, channel, frames, width, height = latents.shape + latents = latents.permute(0, 2, 1, 3, 4).reshape(batch_size * frames, channel, width, height) + noise_pred = noise_pred.permute(0, 2, 1, 3, 4).reshape(batch_size * frames, channel, width, height) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # reshape latents back + latents = latents[None, :].reshape(batch_size, frames, channel, width, height).permute(0, 2, 1, 3, 4) + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + # 8. Post processing + if output_type == "latent": + video = latents + else: + video_tensor = self.decode_latents(latents, decode_chunk_size=decode_chunk_size) + video = tensor2vid(video_tensor, self.image_processor, output_type=output_type) + + # 9. Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (video,) + + return I2VGenXLPipelineOutput(frames=video) + + +# The following utilities are taken and adapted from +# https://github.com/ali-vilab/i2vgen-xl/blob/main/utils/transforms.py. + + +def _convert_pt_to_pil(image: Union[torch.Tensor, List[torch.Tensor]]): + if isinstance(image, list) and isinstance(image[0], torch.Tensor): + image = torch.cat(image, 0) + + if isinstance(image, torch.Tensor): + if image.ndim == 3: + image = image.unsqueeze(0) + + image_numpy = VaeImageProcessor.pt_to_numpy(image) + image_pil = VaeImageProcessor.numpy_to_pil(image_numpy) + image = image_pil + + return image + + +def _resize_bilinear( + image: Union[torch.Tensor, List[torch.Tensor], PIL.Image.Image, List[PIL.Image.Image]], resolution: Tuple[int, int] +): + # First convert the images to PIL in case they are float tensors (only relevant for tests now). + image = _convert_pt_to_pil(image) + + if isinstance(image, list): + image = [u.resize(resolution, PIL.Image.BILINEAR) for u in image] + else: + image = image.resize(resolution, PIL.Image.BILINEAR) + return image + + +def _center_crop_wide( + image: Union[torch.Tensor, List[torch.Tensor], PIL.Image.Image, List[PIL.Image.Image]], resolution: Tuple[int, int] +): + # First convert the images to PIL in case they are float tensors (only relevant for tests now). + image = _convert_pt_to_pil(image) + + if isinstance(image, list): + scale = min(image[0].size[0] / resolution[0], image[0].size[1] / resolution[1]) + image = [u.resize((round(u.width // scale), round(u.height // scale)), resample=PIL.Image.BOX) for u in image] + + # center crop + x1 = (image[0].width - resolution[0]) // 2 + y1 = (image[0].height - resolution[1]) // 2 + image = [u.crop((x1, y1, x1 + resolution[0], y1 + resolution[1])) for u in image] + return image + else: + scale = min(image.size[0] / resolution[0], image.size[1] / resolution[1]) + image = image.resize((round(image.width // scale), round(image.height // scale)), resample=PIL.Image.BOX) + x1 = (image.width - resolution[0]) // 2 + y1 = (image.height - resolution[1]) // 2 + image = image.crop((x1, y1, x1 + resolution[0], y1 + resolution[1])) + return image diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/__init__.py new file mode 100755 index 0000000..606f7b3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/__init__.py @@ -0,0 +1,66 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_kandinsky"] = ["KandinskyPipeline"] + _import_structure["pipeline_kandinsky_combined"] = [ + "KandinskyCombinedPipeline", + "KandinskyImg2ImgCombinedPipeline", + "KandinskyInpaintCombinedPipeline", + ] + _import_structure["pipeline_kandinsky_img2img"] = ["KandinskyImg2ImgPipeline"] + _import_structure["pipeline_kandinsky_inpaint"] = ["KandinskyInpaintPipeline"] + _import_structure["pipeline_kandinsky_prior"] = ["KandinskyPriorPipeline", "KandinskyPriorPipelineOutput"] + _import_structure["text_encoder"] = ["MultilingualCLIP"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + + else: + from .pipeline_kandinsky import KandinskyPipeline + from .pipeline_kandinsky_combined import ( + KandinskyCombinedPipeline, + KandinskyImg2ImgCombinedPipeline, + KandinskyInpaintCombinedPipeline, + ) + from .pipeline_kandinsky_img2img import KandinskyImg2ImgPipeline + from .pipeline_kandinsky_inpaint import KandinskyInpaintPipeline + from .pipeline_kandinsky_prior import KandinskyPriorPipeline, KandinskyPriorPipelineOutput + from .text_encoder import MultilingualCLIP + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py new file mode 100755 index 0000000..34b5a47 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky.py @@ -0,0 +1,407 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, List, Optional, Union + +import torch +from transformers import ( + XLMRobertaTokenizer, +) + +from ...models import UNet2DConditionModel, VQModel +from ...schedulers import DDIMScheduler, DDPMScheduler +from ...utils import ( + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from .text_encoder import MultilingualCLIP + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyPipeline, KandinskyPriorPipeline + >>> import torch + + >>> pipe_prior = KandinskyPriorPipeline.from_pretrained("kandinsky-community/Kandinsky-2-1-prior") + >>> pipe_prior.to("cuda") + + >>> prompt = "red cat, 4k photo" + >>> out = pipe_prior(prompt) + >>> image_emb = out.image_embeds + >>> negative_image_emb = out.negative_image_embeds + + >>> pipe = KandinskyPipeline.from_pretrained("kandinsky-community/kandinsky-2-1") + >>> pipe.to("cuda") + + >>> image = pipe( + ... prompt, + ... image_embeds=image_emb, + ... negative_image_embeds=negative_image_emb, + ... height=768, + ... width=768, + ... num_inference_steps=100, + ... ).images + + >>> image[0].save("cat.png") + ``` +""" + + +def get_new_h_w(h, w, scale_factor=8): + new_h = h // scale_factor**2 + if h % scale_factor**2 != 0: + new_h += 1 + new_w = w // scale_factor**2 + if w % scale_factor**2 != 0: + new_w += 1 + return new_h * scale_factor, new_w * scale_factor + + +class KandinskyPipeline(DiffusionPipeline): + """ + Pipeline for text-to-image generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + text_encoder ([`MultilingualCLIP`]): + Frozen text-encoder. + tokenizer ([`XLMRobertaTokenizer`]): + Tokenizer of class + scheduler (Union[`DDIMScheduler`,`DDPMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + """ + + model_cpu_offload_seq = "text_encoder->unet->movq" + + def __init__( + self, + text_encoder: MultilingualCLIP, + tokenizer: XLMRobertaTokenizer, + unet: UNet2DConditionModel, + scheduler: Union[DDIMScheduler, DDPMScheduler], + movq: VQModel, + ): + super().__init__() + + self.register_modules( + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + movq=movq, + ) + self.movq_scale_factor = 2 ** (len(self.movq.config.block_out_channels) - 1) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + ): + batch_size = len(prompt) if isinstance(prompt, list) else 1 + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + truncation=True, + max_length=77, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + text_input_ids = text_input_ids.to(device) + text_mask = text_inputs.attention_mask.to(device) + + prompt_embeds, text_encoder_hidden_states = self.text_encoder( + input_ids=text_input_ids, attention_mask=text_mask + ) + + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + text_encoder_hidden_states = text_encoder_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + if do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=77, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + uncond_text_input_ids = uncond_input.input_ids.to(device) + uncond_text_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds, uncond_text_encoder_hidden_states = self.text_encoder( + input_ids=uncond_text_input_ids, attention_mask=uncond_text_mask + ) + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len) + + seq_len = uncond_text_encoder_hidden_states.shape[1] + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.repeat(1, num_images_per_prompt, 1) + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + uncond_text_mask = uncond_text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + # done duplicates + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + text_encoder_hidden_states = torch.cat([uncond_text_encoder_hidden_states, text_encoder_hidden_states]) + + text_mask = torch.cat([uncond_text_mask, text_mask]) + + return prompt_embeds, text_encoder_hidden_states, text_mask + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + negative_image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 512, + width: int = 512, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for text prompt, that will be used to condition the image generation. + negative_image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for negative text prompt, will be used to condition the image generation. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + device = self._execution_device + + batch_size = batch_size * num_images_per_prompt + do_classifier_free_guidance = guidance_scale > 1.0 + + prompt_embeds, text_encoder_hidden_states, _ = self._encode_prompt( + prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + + if isinstance(image_embeds, list): + image_embeds = torch.cat(image_embeds, dim=0) + if isinstance(negative_image_embeds, list): + negative_image_embeds = torch.cat(negative_image_embeds, dim=0) + + if do_classifier_free_guidance: + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + negative_image_embeds = negative_image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + + image_embeds = torch.cat([negative_image_embeds, image_embeds], dim=0).to( + dtype=prompt_embeds.dtype, device=device + ) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps_tensor = self.scheduler.timesteps + + num_channels_latents = self.unet.config.in_channels + + height, width = get_new_h_w(height, width, self.movq_scale_factor) + + # create initial latent + latents = self.prepare_latents( + (batch_size, num_channels_latents, height, width), + text_encoder_hidden_states.dtype, + device, + generator, + latents, + self.scheduler, + ) + + for i, t in enumerate(self.progress_bar(timesteps_tensor)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + added_cond_kwargs = {"text_embeds": prompt_embeds, "image_embeds": image_embeds} + noise_pred = self.unet( + sample=latent_model_input, + timestep=t, + encoder_hidden_states=text_encoder_hidden_states, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + if do_classifier_free_guidance: + noise_pred, variance_pred = noise_pred.split(latents.shape[1], dim=1) + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + _, variance_pred_text = variance_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, variance_pred_text], dim=1) + + if not ( + hasattr(self.scheduler.config, "variance_type") + and self.scheduler.config.variance_type in ["learned", "learned_range"] + ): + noise_pred, _ = noise_pred.split(latents.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, + t, + latents, + generator=generator, + ).prev_sample + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # post-processing + image = self.movq.decode(latents, force_not_quantize=True)["sample"] + + self.maybe_free_model_hooks() + + if output_type not in ["pt", "np", "pil"]: + raise ValueError(f"Only the output types `pt`, `pil` and `np` are supported not output_type={output_type}") + + if output_type in ["np", "pil"]: + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py new file mode 100755 index 0000000..da5ff52 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_combined.py @@ -0,0 +1,814 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Callable, List, Optional, Union + +import PIL.Image +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, + XLMRobertaTokenizer, +) + +from ...models import PriorTransformer, UNet2DConditionModel, VQModel +from ...schedulers import DDIMScheduler, DDPMScheduler, UnCLIPScheduler +from ...utils import ( + replace_example_docstring, +) +from ..pipeline_utils import DiffusionPipeline +from .pipeline_kandinsky import KandinskyPipeline +from .pipeline_kandinsky_img2img import KandinskyImg2ImgPipeline +from .pipeline_kandinsky_inpaint import KandinskyInpaintPipeline +from .pipeline_kandinsky_prior import KandinskyPriorPipeline +from .text_encoder import MultilingualCLIP + + +TEXT2IMAGE_EXAMPLE_DOC_STRING = """ + Examples: + ```py + from diffusers import AutoPipelineForText2Image + import torch + + pipe = AutoPipelineForText2Image.from_pretrained( + "kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16 + ) + pipe.enable_model_cpu_offload() + + prompt = "A lion in galaxies, spirals, nebulae, stars, smoke, iridescent, intricate detail, octane render, 8k" + + image = pipe(prompt=prompt, num_inference_steps=25).images[0] + ``` +""" + +IMAGE2IMAGE_EXAMPLE_DOC_STRING = """ + Examples: + ```py + from diffusers import AutoPipelineForImage2Image + import torch + import requests + from io import BytesIO + from PIL import Image + import os + + pipe = AutoPipelineForImage2Image.from_pretrained( + "kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16 + ) + pipe.enable_model_cpu_offload() + + prompt = "A fantasy landscape, Cinematic lighting" + negative_prompt = "low quality, bad quality" + + url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + + response = requests.get(url) + image = Image.open(BytesIO(response.content)).convert("RGB") + image.thumbnail((768, 768)) + + image = pipe(prompt=prompt, image=original_image, num_inference_steps=25).images[0] + ``` +""" + +INPAINT_EXAMPLE_DOC_STRING = """ + Examples: + ```py + from diffusers import AutoPipelineForInpainting + from diffusers.utils import load_image + import torch + import numpy as np + + pipe = AutoPipelineForInpainting.from_pretrained( + "kandinsky-community/kandinsky-2-1-inpaint", torch_dtype=torch.float16 + ) + pipe.enable_model_cpu_offload() + + prompt = "A fantasy landscape, Cinematic lighting" + negative_prompt = "low quality, bad quality" + + original_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/cat.png" + ) + + mask = np.zeros((768, 768), dtype=np.float32) + # Let's mask out an area above the cat's head + mask[:250, 250:-250] = 1 + + image = pipe(prompt=prompt, image=original_image, mask_image=mask, num_inference_steps=25).images[0] + ``` +""" + + +class KandinskyCombinedPipeline(DiffusionPipeline): + """ + Combined Pipeline for text-to-image generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + text_encoder ([`MultilingualCLIP`]): + Frozen text-encoder. + tokenizer ([`XLMRobertaTokenizer`]): + Tokenizer of class + scheduler (Union[`DDIMScheduler`,`DDPMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + prior_prior ([`PriorTransformer`]): + The canonincal unCLIP prior to approximate the image embedding from the text embedding. + prior_image_encoder ([`CLIPVisionModelWithProjection`]): + Frozen image-encoder. + prior_text_encoder ([`CLIPTextModelWithProjection`]): + Frozen text-encoder. + prior_tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + prior_scheduler ([`UnCLIPScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + """ + + _load_connected_pipes = True + model_cpu_offload_seq = "text_encoder->unet->movq->prior_prior->prior_image_encoder->prior_text_encoder" + + def __init__( + self, + text_encoder: MultilingualCLIP, + tokenizer: XLMRobertaTokenizer, + unet: UNet2DConditionModel, + scheduler: Union[DDIMScheduler, DDPMScheduler], + movq: VQModel, + prior_prior: PriorTransformer, + prior_image_encoder: CLIPVisionModelWithProjection, + prior_text_encoder: CLIPTextModelWithProjection, + prior_tokenizer: CLIPTokenizer, + prior_scheduler: UnCLIPScheduler, + prior_image_processor: CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + movq=movq, + prior_prior=prior_prior, + prior_image_encoder=prior_image_encoder, + prior_text_encoder=prior_text_encoder, + prior_tokenizer=prior_tokenizer, + prior_scheduler=prior_scheduler, + prior_image_processor=prior_image_processor, + ) + self.prior_pipe = KandinskyPriorPipeline( + prior=prior_prior, + image_encoder=prior_image_encoder, + text_encoder=prior_text_encoder, + tokenizer=prior_tokenizer, + scheduler=prior_scheduler, + image_processor=prior_image_processor, + ) + self.decoder_pipe = KandinskyPipeline( + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + movq=movq, + ) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) + + def enable_sequential_cpu_offload(self, gpu_id=0): + r""" + Offloads all models (`unet`, `text_encoder`, `vae`, and `safety checker` state dicts) to CPU using 🤗 + Accelerate, significantly reducing memory usage. Models are moved to a `torch.device('meta')` and loaded on a + GPU only when their specific submodule's `forward` method is called. Offloading happens on a submodule basis. + Memory savings are higher than using `enable_model_cpu_offload`, but performance is lower. + """ + self.prior_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + self.decoder_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + + def progress_bar(self, iterable=None, total=None): + self.prior_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.enable_model_cpu_offload() + + def set_progress_bar_config(self, **kwargs): + self.prior_pipe.set_progress_bar_config(**kwargs) + self.decoder_pipe.set_progress_bar_config(**kwargs) + + @torch.no_grad() + @replace_example_docstring(TEXT2IMAGE_EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + num_images_per_prompt: int = 1, + height: int = 512, + width: int = 512, + prior_guidance_scale: float = 4.0, + prior_num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + prior_guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + prior_num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + prior_outputs = self.prior_pipe( + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + num_inference_steps=prior_num_inference_steps, + generator=generator, + latents=latents, + guidance_scale=prior_guidance_scale, + output_type="pt", + return_dict=False, + ) + image_embeds = prior_outputs[0] + negative_image_embeds = prior_outputs[1] + + prompt = [prompt] if not isinstance(prompt, (list, tuple)) else prompt + + if len(prompt) < image_embeds.shape[0] and image_embeds.shape[0] % len(prompt) == 0: + prompt = (image_embeds.shape[0] // len(prompt)) * prompt + + outputs = self.decoder_pipe( + prompt=prompt, + image_embeds=image_embeds, + negative_image_embeds=negative_image_embeds, + width=width, + height=height, + num_inference_steps=num_inference_steps, + generator=generator, + guidance_scale=guidance_scale, + output_type=output_type, + callback=callback, + callback_steps=callback_steps, + return_dict=return_dict, + ) + + self.maybe_free_model_hooks() + + return outputs + + +class KandinskyImg2ImgCombinedPipeline(DiffusionPipeline): + """ + Combined Pipeline for image-to-image generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + text_encoder ([`MultilingualCLIP`]): + Frozen text-encoder. + tokenizer ([`XLMRobertaTokenizer`]): + Tokenizer of class + scheduler (Union[`DDIMScheduler`,`DDPMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + prior_prior ([`PriorTransformer`]): + The canonincal unCLIP prior to approximate the image embedding from the text embedding. + prior_image_encoder ([`CLIPVisionModelWithProjection`]): + Frozen image-encoder. + prior_text_encoder ([`CLIPTextModelWithProjection`]): + Frozen text-encoder. + prior_tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + prior_scheduler ([`UnCLIPScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + """ + + _load_connected_pipes = True + model_cpu_offload_seq = "prior_text_encoder->prior_image_encoder->prior_prior->" "text_encoder->unet->movq" + + def __init__( + self, + text_encoder: MultilingualCLIP, + tokenizer: XLMRobertaTokenizer, + unet: UNet2DConditionModel, + scheduler: Union[DDIMScheduler, DDPMScheduler], + movq: VQModel, + prior_prior: PriorTransformer, + prior_image_encoder: CLIPVisionModelWithProjection, + prior_text_encoder: CLIPTextModelWithProjection, + prior_tokenizer: CLIPTokenizer, + prior_scheduler: UnCLIPScheduler, + prior_image_processor: CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + movq=movq, + prior_prior=prior_prior, + prior_image_encoder=prior_image_encoder, + prior_text_encoder=prior_text_encoder, + prior_tokenizer=prior_tokenizer, + prior_scheduler=prior_scheduler, + prior_image_processor=prior_image_processor, + ) + self.prior_pipe = KandinskyPriorPipeline( + prior=prior_prior, + image_encoder=prior_image_encoder, + text_encoder=prior_text_encoder, + tokenizer=prior_tokenizer, + scheduler=prior_scheduler, + image_processor=prior_image_processor, + ) + self.decoder_pipe = KandinskyImg2ImgPipeline( + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + movq=movq, + ) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) + + def enable_sequential_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet, + text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a + `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called. + Note that offloading happens on a submodule basis. Memory savings are higher than with + `enable_model_cpu_offload`, but performance is lower. + """ + self.prior_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + self.decoder_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + + def progress_bar(self, iterable=None, total=None): + self.prior_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.enable_model_cpu_offload() + + def set_progress_bar_config(self, **kwargs): + self.prior_pipe.set_progress_bar_config(**kwargs) + self.decoder_pipe.set_progress_bar_config(**kwargs) + + @torch.no_grad() + @replace_example_docstring(IMAGE2IMAGE_EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]], + negative_prompt: Optional[Union[str, List[str]]] = None, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + num_images_per_prompt: int = 1, + strength: float = 0.3, + height: int = 512, + width: int = 512, + prior_guidance_scale: float = 4.0, + prior_num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded + again. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + strength (`float`, *optional*, defaults to 0.3): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. + prior_guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + prior_num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + prior_outputs = self.prior_pipe( + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + num_inference_steps=prior_num_inference_steps, + generator=generator, + latents=latents, + guidance_scale=prior_guidance_scale, + output_type="pt", + return_dict=False, + ) + image_embeds = prior_outputs[0] + negative_image_embeds = prior_outputs[1] + + prompt = [prompt] if not isinstance(prompt, (list, tuple)) else prompt + image = [image] if isinstance(prompt, PIL.Image.Image) else image + + if len(prompt) < image_embeds.shape[0] and image_embeds.shape[0] % len(prompt) == 0: + prompt = (image_embeds.shape[0] // len(prompt)) * prompt + + if ( + isinstance(image, (list, tuple)) + and len(image) < image_embeds.shape[0] + and image_embeds.shape[0] % len(image) == 0 + ): + image = (image_embeds.shape[0] // len(image)) * image + + outputs = self.decoder_pipe( + prompt=prompt, + image=image, + image_embeds=image_embeds, + negative_image_embeds=negative_image_embeds, + strength=strength, + width=width, + height=height, + num_inference_steps=num_inference_steps, + generator=generator, + guidance_scale=guidance_scale, + output_type=output_type, + callback=callback, + callback_steps=callback_steps, + return_dict=return_dict, + ) + + self.maybe_free_model_hooks() + + return outputs + + +class KandinskyInpaintCombinedPipeline(DiffusionPipeline): + """ + Combined Pipeline for generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + text_encoder ([`MultilingualCLIP`]): + Frozen text-encoder. + tokenizer ([`XLMRobertaTokenizer`]): + Tokenizer of class + scheduler (Union[`DDIMScheduler`,`DDPMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + prior_prior ([`PriorTransformer`]): + The canonincal unCLIP prior to approximate the image embedding from the text embedding. + prior_image_encoder ([`CLIPVisionModelWithProjection`]): + Frozen image-encoder. + prior_text_encoder ([`CLIPTextModelWithProjection`]): + Frozen text-encoder. + prior_tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + prior_scheduler ([`UnCLIPScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + """ + + _load_connected_pipes = True + model_cpu_offload_seq = "prior_text_encoder->prior_image_encoder->prior_prior->text_encoder->unet->movq" + + def __init__( + self, + text_encoder: MultilingualCLIP, + tokenizer: XLMRobertaTokenizer, + unet: UNet2DConditionModel, + scheduler: Union[DDIMScheduler, DDPMScheduler], + movq: VQModel, + prior_prior: PriorTransformer, + prior_image_encoder: CLIPVisionModelWithProjection, + prior_text_encoder: CLIPTextModelWithProjection, + prior_tokenizer: CLIPTokenizer, + prior_scheduler: UnCLIPScheduler, + prior_image_processor: CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + movq=movq, + prior_prior=prior_prior, + prior_image_encoder=prior_image_encoder, + prior_text_encoder=prior_text_encoder, + prior_tokenizer=prior_tokenizer, + prior_scheduler=prior_scheduler, + prior_image_processor=prior_image_processor, + ) + self.prior_pipe = KandinskyPriorPipeline( + prior=prior_prior, + image_encoder=prior_image_encoder, + text_encoder=prior_text_encoder, + tokenizer=prior_tokenizer, + scheduler=prior_scheduler, + image_processor=prior_image_processor, + ) + self.decoder_pipe = KandinskyInpaintPipeline( + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + movq=movq, + ) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) + + def enable_sequential_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet, + text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a + `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called. + Note that offloading happens on a submodule basis. Memory savings are higher than with + `enable_model_cpu_offload`, but performance is lower. + """ + self.prior_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + self.decoder_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + + def progress_bar(self, iterable=None, total=None): + self.prior_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.enable_model_cpu_offload() + + def set_progress_bar_config(self, **kwargs): + self.prior_pipe.set_progress_bar_config(**kwargs) + self.decoder_pipe.set_progress_bar_config(**kwargs) + + @torch.no_grad() + @replace_example_docstring(INPAINT_EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]], + mask_image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]], + negative_prompt: Optional[Union[str, List[str]]] = None, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + num_images_per_prompt: int = 1, + height: int = 512, + width: int = 512, + prior_guidance_scale: float = 4.0, + prior_num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded + again. + mask_image (`np.array`): + Tensor representing an image batch, to mask `image`. White pixels in the mask will be repainted, while + black pixels will be preserved. If `mask_image` is a PIL image, it will be converted to a single + channel (luminance) before use. If it's a tensor, it should contain one color channel (L) instead of 3, + so the expected shape would be `(B, H, W, 1)`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + prior_guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + prior_num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + prior_outputs = self.prior_pipe( + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + num_inference_steps=prior_num_inference_steps, + generator=generator, + latents=latents, + guidance_scale=prior_guidance_scale, + output_type="pt", + return_dict=False, + ) + image_embeds = prior_outputs[0] + negative_image_embeds = prior_outputs[1] + + prompt = [prompt] if not isinstance(prompt, (list, tuple)) else prompt + image = [image] if isinstance(prompt, PIL.Image.Image) else image + mask_image = [mask_image] if isinstance(mask_image, PIL.Image.Image) else mask_image + + if len(prompt) < image_embeds.shape[0] and image_embeds.shape[0] % len(prompt) == 0: + prompt = (image_embeds.shape[0] // len(prompt)) * prompt + + if ( + isinstance(image, (list, tuple)) + and len(image) < image_embeds.shape[0] + and image_embeds.shape[0] % len(image) == 0 + ): + image = (image_embeds.shape[0] // len(image)) * image + + if ( + isinstance(mask_image, (list, tuple)) + and len(mask_image) < image_embeds.shape[0] + and image_embeds.shape[0] % len(mask_image) == 0 + ): + mask_image = (image_embeds.shape[0] // len(mask_image)) * mask_image + + outputs = self.decoder_pipe( + prompt=prompt, + image=image, + mask_image=mask_image, + image_embeds=image_embeds, + negative_image_embeds=negative_image_embeds, + width=width, + height=height, + num_inference_steps=num_inference_steps, + generator=generator, + guidance_scale=guidance_scale, + output_type=output_type, + callback=callback, + callback_steps=callback_steps, + return_dict=return_dict, + ) + + self.maybe_free_model_hooks() + + return outputs diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py new file mode 100755 index 0000000..4d091e7 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_img2img.py @@ -0,0 +1,500 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from PIL import Image +from transformers import ( + XLMRobertaTokenizer, +) + +from ...models import UNet2DConditionModel, VQModel +from ...schedulers import DDIMScheduler +from ...utils import ( + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from .text_encoder import MultilingualCLIP + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyImg2ImgPipeline, KandinskyPriorPipeline + >>> from diffusers.utils import load_image + >>> import torch + + >>> pipe_prior = KandinskyPriorPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16 + ... ) + >>> pipe_prior.to("cuda") + + >>> prompt = "A red cartoon frog, 4k" + >>> image_emb, zero_image_emb = pipe_prior(prompt, return_dict=False) + + >>> pipe = KandinskyImg2ImgPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16 + ... ) + >>> pipe.to("cuda") + + >>> init_image = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/frog.png" + ... ) + + >>> image = pipe( + ... prompt, + ... image=init_image, + ... image_embeds=image_emb, + ... negative_image_embeds=zero_image_emb, + ... height=768, + ... width=768, + ... num_inference_steps=100, + ... strength=0.2, + ... ).images + + >>> image[0].save("red_frog.png") + ``` +""" + + +def get_new_h_w(h, w, scale_factor=8): + new_h = h // scale_factor**2 + if h % scale_factor**2 != 0: + new_h += 1 + new_w = w // scale_factor**2 + if w % scale_factor**2 != 0: + new_w += 1 + return new_h * scale_factor, new_w * scale_factor + + +def prepare_image(pil_image, w=512, h=512): + pil_image = pil_image.resize((w, h), resample=Image.BICUBIC, reducing_gap=1) + arr = np.array(pil_image.convert("RGB")) + arr = arr.astype(np.float32) / 127.5 - 1 + arr = np.transpose(arr, [2, 0, 1]) + image = torch.from_numpy(arr).unsqueeze(0) + return image + + +class KandinskyImg2ImgPipeline(DiffusionPipeline): + """ + Pipeline for image-to-image generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + text_encoder ([`MultilingualCLIP`]): + Frozen text-encoder. + tokenizer ([`XLMRobertaTokenizer`]): + Tokenizer of class + scheduler ([`DDIMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ image encoder and decoder + """ + + model_cpu_offload_seq = "text_encoder->unet->movq" + + def __init__( + self, + text_encoder: MultilingualCLIP, + movq: VQModel, + tokenizer: XLMRobertaTokenizer, + unet: UNet2DConditionModel, + scheduler: DDIMScheduler, + ): + super().__init__() + + self.register_modules( + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + movq=movq, + ) + self.movq_scale_factor = 2 ** (len(self.movq.config.block_out_channels) - 1) + + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start:] + + return timesteps, num_inference_steps - t_start + + def prepare_latents(self, latents, latent_timestep, shape, dtype, device, generator, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + + shape = latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + latents = self.add_noise(latents, noise, latent_timestep) + return latents + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + ): + batch_size = len(prompt) if isinstance(prompt, list) else 1 + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=77, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + text_input_ids = text_input_ids.to(device) + text_mask = text_inputs.attention_mask.to(device) + + prompt_embeds, text_encoder_hidden_states = self.text_encoder( + input_ids=text_input_ids, attention_mask=text_mask + ) + + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + text_encoder_hidden_states = text_encoder_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + if do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=77, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + uncond_text_input_ids = uncond_input.input_ids.to(device) + uncond_text_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds, uncond_text_encoder_hidden_states = self.text_encoder( + input_ids=uncond_text_input_ids, attention_mask=uncond_text_mask + ) + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len) + + seq_len = uncond_text_encoder_hidden_states.shape[1] + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.repeat(1, num_images_per_prompt, 1) + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + uncond_text_mask = uncond_text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + # done duplicates + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + text_encoder_hidden_states = torch.cat([uncond_text_encoder_hidden_states, text_encoder_hidden_states]) + + text_mask = torch.cat([uncond_text_mask, text_mask]) + + return prompt_embeds, text_encoder_hidden_states, text_mask + + # add_noise method to overwrite the one in schedule because it use a different beta schedule for adding noise vs sampling + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + betas = torch.linspace(0.0001, 0.02, 1000, dtype=torch.float32) + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_cumprod = alphas_cumprod.to(device=original_samples.device, dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + + return noisy_samples + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]], + image_embeds: torch.FloatTensor, + negative_image_embeds: torch.FloatTensor, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 512, + width: int = 512, + num_inference_steps: int = 100, + strength: float = 0.3, + guidance_scale: float = 7.0, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`torch.FloatTensor`, `PIL.Image.Image`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. + image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for text prompt, that will be used to condition the image generation. + negative_image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for negative text prompt, will be used to condition the image generation. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + strength (`float`, *optional*, defaults to 0.3): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + # 1. Define call parameters + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + device = self._execution_device + + batch_size = batch_size * num_images_per_prompt + + do_classifier_free_guidance = guidance_scale > 1.0 + + # 2. get text and image embeddings + prompt_embeds, text_encoder_hidden_states, _ = self._encode_prompt( + prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + + if isinstance(image_embeds, list): + image_embeds = torch.cat(image_embeds, dim=0) + if isinstance(negative_image_embeds, list): + negative_image_embeds = torch.cat(negative_image_embeds, dim=0) + + if do_classifier_free_guidance: + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + negative_image_embeds = negative_image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + + image_embeds = torch.cat([negative_image_embeds, image_embeds], dim=0).to( + dtype=prompt_embeds.dtype, device=device + ) + + # 3. pre-processing initial image + if not isinstance(image, list): + image = [image] + if not all(isinstance(i, (PIL.Image.Image, torch.Tensor)) for i in image): + raise ValueError( + f"Input is in incorrect format: {[type(i) for i in image]}. Currently, we only support PIL image and pytorch tensor" + ) + + image = torch.cat([prepare_image(i, width, height) for i in image], dim=0) + image = image.to(dtype=prompt_embeds.dtype, device=device) + + latents = self.movq.encode(image)["latents"] + latents = latents.repeat_interleave(num_images_per_prompt, dim=0) + + # 4. set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + + timesteps_tensor, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + + # the formular to calculate timestep for add_noise is taken from the original kandinsky repo + latent_timestep = int(self.scheduler.config.num_train_timesteps * strength) - 2 + + latent_timestep = torch.tensor([latent_timestep] * batch_size, dtype=timesteps_tensor.dtype, device=device) + + num_channels_latents = self.unet.config.in_channels + + height, width = get_new_h_w(height, width, self.movq_scale_factor) + + # 5. Create initial latent + latents = self.prepare_latents( + latents, + latent_timestep, + (batch_size, num_channels_latents, height, width), + text_encoder_hidden_states.dtype, + device, + generator, + self.scheduler, + ) + + # 6. Denoising loop + for i, t in enumerate(self.progress_bar(timesteps_tensor)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + added_cond_kwargs = {"text_embeds": prompt_embeds, "image_embeds": image_embeds} + noise_pred = self.unet( + sample=latent_model_input, + timestep=t, + encoder_hidden_states=text_encoder_hidden_states, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + if do_classifier_free_guidance: + noise_pred, variance_pred = noise_pred.split(latents.shape[1], dim=1) + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + _, variance_pred_text = variance_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, variance_pred_text], dim=1) + + if not ( + hasattr(self.scheduler.config, "variance_type") + and self.scheduler.config.variance_type in ["learned", "learned_range"] + ): + noise_pred, _ = noise_pred.split(latents.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, + t, + latents, + generator=generator, + ).prev_sample + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 7. post-processing + image = self.movq.decode(latents, force_not_quantize=True)["sample"] + + self.maybe_free_model_hooks() + + if output_type not in ["pt", "np", "pil"]: + raise ValueError(f"Only the output types `pt`, `pil` and `np` are supported not output_type={output_type}") + + if output_type in ["np", "pil"]: + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py new file mode 100755 index 0000000..d8d9e96 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_inpaint.py @@ -0,0 +1,635 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from copy import deepcopy +from typing import Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from packaging import version +from PIL import Image +from transformers import ( + XLMRobertaTokenizer, +) + +from ... import __version__ +from ...models import UNet2DConditionModel, VQModel +from ...schedulers import DDIMScheduler +from ...utils import ( + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from .text_encoder import MultilingualCLIP + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyInpaintPipeline, KandinskyPriorPipeline + >>> from diffusers.utils import load_image + >>> import torch + >>> import numpy as np + + >>> pipe_prior = KandinskyPriorPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16 + ... ) + >>> pipe_prior.to("cuda") + + >>> prompt = "a hat" + >>> image_emb, zero_image_emb = pipe_prior(prompt, return_dict=False) + + >>> pipe = KandinskyInpaintPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-1-inpaint", torch_dtype=torch.float16 + ... ) + >>> pipe.to("cuda") + + >>> init_image = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/cat.png" + ... ) + + >>> mask = np.zeros((768, 768), dtype=np.float32) + >>> mask[:250, 250:-250] = 1 + + >>> out = pipe( + ... prompt, + ... image=init_image, + ... mask_image=mask, + ... image_embeds=image_emb, + ... negative_image_embeds=zero_image_emb, + ... height=768, + ... width=768, + ... num_inference_steps=50, + ... ) + + >>> image = out.images[0] + >>> image.save("cat_with_hat.png") + ``` +""" + + +def get_new_h_w(h, w, scale_factor=8): + new_h = h // scale_factor**2 + if h % scale_factor**2 != 0: + new_h += 1 + new_w = w // scale_factor**2 + if w % scale_factor**2 != 0: + new_w += 1 + return new_h * scale_factor, new_w * scale_factor + + +def prepare_mask(masks): + prepared_masks = [] + for mask in masks: + old_mask = deepcopy(mask) + for i in range(mask.shape[1]): + for j in range(mask.shape[2]): + if old_mask[0][i][j] == 1: + continue + if i != 0: + mask[:, i - 1, j] = 0 + if j != 0: + mask[:, i, j - 1] = 0 + if i != 0 and j != 0: + mask[:, i - 1, j - 1] = 0 + if i != mask.shape[1] - 1: + mask[:, i + 1, j] = 0 + if j != mask.shape[2] - 1: + mask[:, i, j + 1] = 0 + if i != mask.shape[1] - 1 and j != mask.shape[2] - 1: + mask[:, i + 1, j + 1] = 0 + prepared_masks.append(mask) + return torch.stack(prepared_masks, dim=0) + + +def prepare_mask_and_masked_image(image, mask, height, width): + r""" + Prepares a pair (mask, image) to be consumed by the Kandinsky inpaint pipeline. This means that those inputs will + be converted to ``torch.Tensor`` with shapes ``batch x channels x height x width`` where ``channels`` is ``3`` for + the ``image`` and ``1`` for the ``mask``. + + The ``image`` will be converted to ``torch.float32`` and normalized to be in ``[-1, 1]``. The ``mask`` will be + binarized (``mask > 0.5``) and cast to ``torch.float32`` too. + + Args: + image (Union[np.array, PIL.Image, torch.Tensor]): The image to inpaint. + It can be a ``PIL.Image``, or a ``height x width x 3`` ``np.array`` or a ``channels x height x width`` + ``torch.Tensor`` or a ``batch x channels x height x width`` ``torch.Tensor``. + mask (_type_): The mask to apply to the image, i.e. regions to inpaint. + It can be a ``PIL.Image``, or a ``height x width`` ``np.array`` or a ``1 x height x width`` + ``torch.Tensor`` or a ``batch x 1 x height x width`` ``torch.Tensor``. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + + + Raises: + ValueError: ``torch.Tensor`` images should be in the ``[-1, 1]`` range. ValueError: ``torch.Tensor`` mask + should be in the ``[0, 1]`` range. ValueError: ``mask`` and ``image`` should have the same spatial dimensions. + TypeError: ``mask`` is a ``torch.Tensor`` but ``image`` is not + (ot the other way around). + + Returns: + tuple[torch.Tensor]: The pair (mask, image) as ``torch.Tensor`` with 4 + dimensions: ``batch x channels x height x width``. + """ + + if image is None: + raise ValueError("`image` input cannot be undefined.") + + if mask is None: + raise ValueError("`mask_image` input cannot be undefined.") + + if isinstance(image, torch.Tensor): + if not isinstance(mask, torch.Tensor): + raise TypeError(f"`image` is a torch.Tensor but `mask` (type: {type(mask)} is not") + + # Batch single image + if image.ndim == 3: + assert image.shape[0] == 3, "Image outside a batch should be of shape (3, H, W)" + image = image.unsqueeze(0) + + # Batch and add channel dim for single mask + if mask.ndim == 2: + mask = mask.unsqueeze(0).unsqueeze(0) + + # Batch single mask or add channel dim + if mask.ndim == 3: + # Single batched mask, no channel dim or single mask not batched but channel dim + if mask.shape[0] == 1: + mask = mask.unsqueeze(0) + + # Batched masks no channel dim + else: + mask = mask.unsqueeze(1) + + assert image.ndim == 4 and mask.ndim == 4, "Image and Mask must have 4 dimensions" + assert image.shape[-2:] == mask.shape[-2:], "Image and Mask must have the same spatial dimensions" + assert image.shape[0] == mask.shape[0], "Image and Mask must have the same batch size" + + # Check image is in [-1, 1] + if image.min() < -1 or image.max() > 1: + raise ValueError("Image should be in [-1, 1] range") + + # Check mask is in [0, 1] + if mask.min() < 0 or mask.max() > 1: + raise ValueError("Mask should be in [0, 1] range") + + # Binarize mask + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + + # Image as float32 + image = image.to(dtype=torch.float32) + elif isinstance(mask, torch.Tensor): + raise TypeError(f"`mask` is a torch.Tensor but `image` (type: {type(image)} is not") + else: + # preprocess image + if isinstance(image, (PIL.Image.Image, np.ndarray)): + image = [image] + + if isinstance(image, list) and isinstance(image[0], PIL.Image.Image): + # resize all images w.r.t passed height an width + image = [i.resize((width, height), resample=Image.BICUBIC, reducing_gap=1) for i in image] + image = [np.array(i.convert("RGB"))[None, :] for i in image] + image = np.concatenate(image, axis=0) + elif isinstance(image, list) and isinstance(image[0], np.ndarray): + image = np.concatenate([i[None, :] for i in image], axis=0) + + image = image.transpose(0, 3, 1, 2) + image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0 + + # preprocess mask + if isinstance(mask, (PIL.Image.Image, np.ndarray)): + mask = [mask] + + if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image): + mask = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in mask] + mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0) + mask = mask.astype(np.float32) / 255.0 + elif isinstance(mask, list) and isinstance(mask[0], np.ndarray): + mask = np.concatenate([m[None, None, :] for m in mask], axis=0) + + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + mask = torch.from_numpy(mask) + + mask = 1 - mask + + return mask, image + + +class KandinskyInpaintPipeline(DiffusionPipeline): + """ + Pipeline for text-guided image inpainting using Kandinsky2.1 + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + text_encoder ([`MultilingualCLIP`]): + Frozen text-encoder. + tokenizer ([`XLMRobertaTokenizer`]): + Tokenizer of class + scheduler ([`DDIMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ image encoder and decoder + """ + + model_cpu_offload_seq = "text_encoder->unet->movq" + + def __init__( + self, + text_encoder: MultilingualCLIP, + movq: VQModel, + tokenizer: XLMRobertaTokenizer, + unet: UNet2DConditionModel, + scheduler: DDIMScheduler, + ): + super().__init__() + + self.register_modules( + text_encoder=text_encoder, + movq=movq, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + ) + self.movq_scale_factor = 2 ** (len(self.movq.config.block_out_channels) - 1) + self._warn_has_been_called = False + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + ): + batch_size = len(prompt) if isinstance(prompt, list) else 1 + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=77, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + text_input_ids = text_input_ids.to(device) + text_mask = text_inputs.attention_mask.to(device) + + prompt_embeds, text_encoder_hidden_states = self.text_encoder( + input_ids=text_input_ids, attention_mask=text_mask + ) + + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + text_encoder_hidden_states = text_encoder_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + if do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=77, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + uncond_text_input_ids = uncond_input.input_ids.to(device) + uncond_text_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds, uncond_text_encoder_hidden_states = self.text_encoder( + input_ids=uncond_text_input_ids, attention_mask=uncond_text_mask + ) + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len) + + seq_len = uncond_text_encoder_hidden_states.shape[1] + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.repeat(1, num_images_per_prompt, 1) + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + uncond_text_mask = uncond_text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + # done duplicates + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + text_encoder_hidden_states = torch.cat([uncond_text_encoder_hidden_states, text_encoder_hidden_states]) + + text_mask = torch.cat([uncond_text_mask, text_mask]) + + return prompt_embeds, text_encoder_hidden_states, text_mask + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + image: Union[torch.FloatTensor, PIL.Image.Image], + mask_image: Union[torch.FloatTensor, PIL.Image.Image, np.ndarray], + image_embeds: torch.FloatTensor, + negative_image_embeds: torch.FloatTensor, + negative_prompt: Optional[Union[str, List[str]]] = None, + height: int = 512, + width: int = 512, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`torch.FloatTensor`, `PIL.Image.Image` or `np.ndarray`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. + mask_image (`PIL.Image.Image`,`torch.FloatTensor` or `np.ndarray`): + `Image`, or a tensor representing an image batch, to mask `image`. White pixels in the mask will be + repainted, while black pixels will be preserved. You can pass a pytorch tensor as mask only if the + image you passed is a pytorch tensor, and it should contain one color channel (L) instead of 3, so the + expected shape would be either `(B, 1, H, W,)`, `(B, H, W)`, `(1, H, W)` or `(H, W)` If image is an PIL + image or numpy array, mask should also be a either PIL image or numpy array. If it is a PIL image, it + will be converted to a single channel (luminance) before use. If it is a nummpy array, the expected + shape is `(H, W)`. + image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for text prompt, that will be used to condition the image generation. + negative_image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for negative text prompt, will be used to condition the image generation. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + if not self._warn_has_been_called and version.parse(version.parse(__version__).base_version) < version.parse( + "0.23.0.dev0" + ): + logger.warning( + "Please note that the expected format of `mask_image` has recently been changed. " + "Before diffusers == 0.19.0, Kandinsky Inpainting pipelines repainted black pixels and preserved black pixels. " + "As of diffusers==0.19.0 this behavior has been inverted. Now white pixels are repainted and black pixels are preserved. " + "This way, Kandinsky's masking behavior is aligned with Stable Diffusion. " + "THIS means that you HAVE to invert the input mask to have the same behavior as before as explained in https://github.com/huggingface/diffusers/pull/4207. " + "This warning will be surpressed after the first inference call and will be removed in diffusers>0.23.0" + ) + self._warn_has_been_called = True + + # Define call parameters + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + device = self._execution_device + + batch_size = batch_size * num_images_per_prompt + do_classifier_free_guidance = guidance_scale > 1.0 + + prompt_embeds, text_encoder_hidden_states, _ = self._encode_prompt( + prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + + if isinstance(image_embeds, list): + image_embeds = torch.cat(image_embeds, dim=0) + if isinstance(negative_image_embeds, list): + negative_image_embeds = torch.cat(negative_image_embeds, dim=0) + + if do_classifier_free_guidance: + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + negative_image_embeds = negative_image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + + image_embeds = torch.cat([negative_image_embeds, image_embeds], dim=0).to( + dtype=prompt_embeds.dtype, device=device + ) + + # preprocess image and mask + mask_image, image = prepare_mask_and_masked_image(image, mask_image, height, width) + + image = image.to(dtype=prompt_embeds.dtype, device=device) + image = self.movq.encode(image)["latents"] + + mask_image = mask_image.to(dtype=prompt_embeds.dtype, device=device) + + image_shape = tuple(image.shape[-2:]) + mask_image = F.interpolate( + mask_image, + image_shape, + mode="nearest", + ) + mask_image = prepare_mask(mask_image) + masked_image = image * mask_image + + mask_image = mask_image.repeat_interleave(num_images_per_prompt, dim=0) + masked_image = masked_image.repeat_interleave(num_images_per_prompt, dim=0) + if do_classifier_free_guidance: + mask_image = mask_image.repeat(2, 1, 1, 1) + masked_image = masked_image.repeat(2, 1, 1, 1) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps_tensor = self.scheduler.timesteps + + num_channels_latents = self.movq.config.latent_channels + + # get h, w for latents + sample_height, sample_width = get_new_h_w(height, width, self.movq_scale_factor) + + # create initial latent + latents = self.prepare_latents( + (batch_size, num_channels_latents, sample_height, sample_width), + text_encoder_hidden_states.dtype, + device, + generator, + latents, + self.scheduler, + ) + + # Check that sizes of mask, masked image and latents match with expected + num_channels_mask = mask_image.shape[1] + num_channels_masked_image = masked_image.shape[1] + if num_channels_latents + num_channels_mask + num_channels_masked_image != self.unet.config.in_channels: + raise ValueError( + f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects" + f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +" + f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}" + f" = {num_channels_latents+num_channels_masked_image+num_channels_mask}. Please verify the config of" + " `pipeline.unet` or your `mask_image` or `image` input." + ) + + for i, t in enumerate(self.progress_bar(timesteps_tensor)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = torch.cat([latent_model_input, masked_image, mask_image], dim=1) + + added_cond_kwargs = {"text_embeds": prompt_embeds, "image_embeds": image_embeds} + noise_pred = self.unet( + sample=latent_model_input, + timestep=t, + encoder_hidden_states=text_encoder_hidden_states, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + if do_classifier_free_guidance: + noise_pred, variance_pred = noise_pred.split(latents.shape[1], dim=1) + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + _, variance_pred_text = variance_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, variance_pred_text], dim=1) + + if not ( + hasattr(self.scheduler.config, "variance_type") + and self.scheduler.config.variance_type in ["learned", "learned_range"] + ): + noise_pred, _ = noise_pred.split(latents.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, + t, + latents, + generator=generator, + ).prev_sample + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # post-processing + image = self.movq.decode(latents, force_not_quantize=True)["sample"] + + self.maybe_free_model_hooks() + + if output_type not in ["pt", "np", "pil"]: + raise ValueError(f"Only the output types `pt`, `pil` and `np` are supported not output_type={output_type}") + + if output_type in ["np", "pil"]: + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py new file mode 100755 index 0000000..0d9f543 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/pipeline_kandinsky_prior.py @@ -0,0 +1,547 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...models import PriorTransformer +from ...schedulers import UnCLIPScheduler +from ...utils import ( + BaseOutput, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyPipeline, KandinskyPriorPipeline + >>> import torch + + >>> pipe_prior = KandinskyPriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-1-prior") + >>> pipe_prior.to("cuda") + + >>> prompt = "red cat, 4k photo" + >>> out = pipe_prior(prompt) + >>> image_emb = out.image_embeds + >>> negative_image_emb = out.negative_image_embeds + + >>> pipe = KandinskyPipeline.from_pretrained("kandinsky-community/kandinsky-2-1") + >>> pipe.to("cuda") + + >>> image = pipe( + ... prompt, + ... image_embeds=image_emb, + ... negative_image_embeds=negative_image_emb, + ... height=768, + ... width=768, + ... num_inference_steps=100, + ... ).images + + >>> image[0].save("cat.png") + ``` +""" + +EXAMPLE_INTERPOLATE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyPriorPipeline, KandinskyPipeline + >>> from diffusers.utils import load_image + >>> import PIL + + >>> import torch + >>> from torchvision import transforms + + >>> pipe_prior = KandinskyPriorPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16 + ... ) + >>> pipe_prior.to("cuda") + + >>> img1 = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/cat.png" + ... ) + + >>> img2 = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/starry_night.jpeg" + ... ) + + >>> images_texts = ["a cat", img1, img2] + >>> weights = [0.3, 0.3, 0.4] + >>> image_emb, zero_image_emb = pipe_prior.interpolate(images_texts, weights) + + >>> pipe = KandinskyPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16) + >>> pipe.to("cuda") + + >>> image = pipe( + ... "", + ... image_embeds=image_emb, + ... negative_image_embeds=zero_image_emb, + ... height=768, + ... width=768, + ... num_inference_steps=150, + ... ).images[0] + + >>> image.save("starry_cat.png") + ``` +""" + + +@dataclass +class KandinskyPriorPipelineOutput(BaseOutput): + """ + Output class for KandinskyPriorPipeline. + + Args: + image_embeds (`torch.FloatTensor`) + clip image embeddings for text prompt + negative_image_embeds (`List[PIL.Image.Image]` or `np.ndarray`) + clip image embeddings for unconditional tokens + """ + + image_embeds: Union[torch.FloatTensor, np.ndarray] + negative_image_embeds: Union[torch.FloatTensor, np.ndarray] + + +class KandinskyPriorPipeline(DiffusionPipeline): + """ + Pipeline for generating image prior for Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + prior ([`PriorTransformer`]): + The canonincal unCLIP prior to approximate the image embedding from the text embedding. + image_encoder ([`CLIPVisionModelWithProjection`]): + Frozen image-encoder. + text_encoder ([`CLIPTextModelWithProjection`]): + Frozen text-encoder. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + scheduler ([`UnCLIPScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + """ + + _exclude_from_cpu_offload = ["prior"] + model_cpu_offload_seq = "text_encoder->prior" + + def __init__( + self, + prior: PriorTransformer, + image_encoder: CLIPVisionModelWithProjection, + text_encoder: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + scheduler: UnCLIPScheduler, + image_processor: CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + prior=prior, + text_encoder=text_encoder, + tokenizer=tokenizer, + scheduler=scheduler, + image_encoder=image_encoder, + image_processor=image_processor, + ) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_INTERPOLATE_DOC_STRING) + def interpolate( + self, + images_and_prompts: List[Union[str, PIL.Image.Image, torch.FloatTensor]], + weights: List[float], + num_images_per_prompt: int = 1, + num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + negative_prior_prompt: Optional[str] = None, + negative_prompt: str = "", + guidance_scale: float = 4.0, + device=None, + ): + """ + Function invoked when using the prior pipeline for interpolation. + + Args: + images_and_prompts (`List[Union[str, PIL.Image.Image, torch.FloatTensor]]`): + list of prompts and images to guide the image generation. + weights: (`List[float]`): + list of weights for each condition in `images_and_prompts` + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 25): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + negative_prior_prompt (`str`, *optional*): + The prompt not to guide the prior diffusion process. Ignored when not using guidance (i.e., ignored if + `guidance_scale` is less than `1`). + negative_prompt (`str` or `List[str]`, *optional*): + The prompt not to guide the image generation. Ignored when not using guidance (i.e., ignored if + `guidance_scale` is less than `1`). + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + + Examples: + + Returns: + [`KandinskyPriorPipelineOutput`] or `tuple` + """ + + device = device or self.device + + if len(images_and_prompts) != len(weights): + raise ValueError( + f"`images_and_prompts` contains {len(images_and_prompts)} items and `weights` contains {len(weights)} items - they should be lists of same length" + ) + + image_embeddings = [] + for cond, weight in zip(images_and_prompts, weights): + if isinstance(cond, str): + image_emb = self( + cond, + num_inference_steps=num_inference_steps, + num_images_per_prompt=num_images_per_prompt, + generator=generator, + latents=latents, + negative_prompt=negative_prior_prompt, + guidance_scale=guidance_scale, + ).image_embeds + + elif isinstance(cond, (PIL.Image.Image, torch.Tensor)): + if isinstance(cond, PIL.Image.Image): + cond = ( + self.image_processor(cond, return_tensors="pt") + .pixel_values[0] + .unsqueeze(0) + .to(dtype=self.image_encoder.dtype, device=device) + ) + + image_emb = self.image_encoder(cond)["image_embeds"] + + else: + raise ValueError( + f"`images_and_prompts` can only contains elements to be of type `str`, `PIL.Image.Image` or `torch.Tensor` but is {type(cond)}" + ) + + image_embeddings.append(image_emb * weight) + + image_emb = torch.cat(image_embeddings).sum(dim=0, keepdim=True) + + out_zero = self( + negative_prompt, + num_inference_steps=num_inference_steps, + num_images_per_prompt=num_images_per_prompt, + generator=generator, + latents=latents, + negative_prompt=negative_prior_prompt, + guidance_scale=guidance_scale, + ) + zero_image_emb = out_zero.negative_image_embeds if negative_prompt == "" else out_zero.image_embeds + + return KandinskyPriorPipelineOutput(image_embeds=image_emb, negative_image_embeds=zero_image_emb) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def get_zero_embed(self, batch_size=1, device=None): + device = device or self.device + zero_img = torch.zeros(1, 3, self.image_encoder.config.image_size, self.image_encoder.config.image_size).to( + device=device, dtype=self.image_encoder.dtype + ) + zero_image_emb = self.image_encoder(zero_img)["image_embeds"] + zero_image_emb = zero_image_emb.repeat(batch_size, 1) + return zero_image_emb + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + ): + batch_size = len(prompt) if isinstance(prompt, list) else 1 + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + text_mask = text_inputs.attention_mask.bool().to(device) + + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + text_input_ids = text_input_ids[:, : self.tokenizer.model_max_length] + + text_encoder_output = self.text_encoder(text_input_ids.to(device)) + + prompt_embeds = text_encoder_output.text_embeds + text_encoder_hidden_states = text_encoder_output.last_hidden_state + + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + text_encoder_hidden_states = text_encoder_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + if do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + uncond_text_mask = uncond_input.attention_mask.bool().to(device) + negative_prompt_embeds_text_encoder_output = self.text_encoder(uncond_input.input_ids.to(device)) + + negative_prompt_embeds = negative_prompt_embeds_text_encoder_output.text_embeds + uncond_text_encoder_hidden_states = negative_prompt_embeds_text_encoder_output.last_hidden_state + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len) + + seq_len = uncond_text_encoder_hidden_states.shape[1] + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.repeat(1, num_images_per_prompt, 1) + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + uncond_text_mask = uncond_text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + # done duplicates + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + text_encoder_hidden_states = torch.cat([uncond_text_encoder_hidden_states, text_encoder_hidden_states]) + + text_mask = torch.cat([uncond_text_mask, text_mask]) + + return prompt_embeds, text_encoder_hidden_states, text_mask + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + guidance_scale: float = 4.0, + output_type: Optional[str] = "pt", + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 25): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + output_type (`str`, *optional*, defaults to `"pt"`): + The output format of the generate image. Choose between: `"np"` (`np.array`) or `"pt"` + (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + Returns: + [`KandinskyPriorPipelineOutput`] or `tuple` + """ + + if isinstance(prompt, str): + prompt = [prompt] + elif not isinstance(prompt, list): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + elif not isinstance(negative_prompt, list) and negative_prompt is not None: + raise ValueError(f"`negative_prompt` has to be of type `str` or `list` but is {type(negative_prompt)}") + + # if the negative prompt is defined we double the batch size to + # directly retrieve the negative prompt embedding + if negative_prompt is not None: + prompt = prompt + negative_prompt + negative_prompt = 2 * negative_prompt + + device = self._execution_device + + batch_size = len(prompt) + batch_size = batch_size * num_images_per_prompt + + do_classifier_free_guidance = guidance_scale > 1.0 + prompt_embeds, text_encoder_hidden_states, text_mask = self._encode_prompt( + prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + + # prior + self.scheduler.set_timesteps(num_inference_steps, device=device) + prior_timesteps_tensor = self.scheduler.timesteps + + embedding_dim = self.prior.config.embedding_dim + + latents = self.prepare_latents( + (batch_size, embedding_dim), + prompt_embeds.dtype, + device, + generator, + latents, + self.scheduler, + ) + + for i, t in enumerate(self.progress_bar(prior_timesteps_tensor)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + predicted_image_embedding = self.prior( + latent_model_input, + timestep=t, + proj_embedding=prompt_embeds, + encoder_hidden_states=text_encoder_hidden_states, + attention_mask=text_mask, + ).predicted_image_embedding + + if do_classifier_free_guidance: + predicted_image_embedding_uncond, predicted_image_embedding_text = predicted_image_embedding.chunk(2) + predicted_image_embedding = predicted_image_embedding_uncond + guidance_scale * ( + predicted_image_embedding_text - predicted_image_embedding_uncond + ) + + if i + 1 == prior_timesteps_tensor.shape[0]: + prev_timestep = None + else: + prev_timestep = prior_timesteps_tensor[i + 1] + + latents = self.scheduler.step( + predicted_image_embedding, + timestep=t, + sample=latents, + generator=generator, + prev_timestep=prev_timestep, + ).prev_sample + + latents = self.prior.post_process_latents(latents) + + image_embeddings = latents + + # if negative prompt has been defined, we retrieve split the image embedding into two + if negative_prompt is None: + zero_embeds = self.get_zero_embed(latents.shape[0], device=latents.device) + + self.maybe_free_model_hooks() + else: + image_embeddings, zero_embeds = image_embeddings.chunk(2) + + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.prior_hook.offload() + + if output_type not in ["pt", "np"]: + raise ValueError(f"Only the output types `pt` and `np` are supported not output_type={output_type}") + + if output_type == "np": + image_embeddings = image_embeddings.cpu().numpy() + zero_embeds = zero_embeds.cpu().numpy() + + if not return_dict: + return (image_embeddings, zero_embeds) + + return KandinskyPriorPipelineOutput(image_embeds=image_embeddings, negative_image_embeds=zero_embeds) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/text_encoder.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/text_encoder.py new file mode 100755 index 0000000..caa0029 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky/text_encoder.py @@ -0,0 +1,27 @@ +import torch +from transformers import PreTrainedModel, XLMRobertaConfig, XLMRobertaModel + + +class MCLIPConfig(XLMRobertaConfig): + model_type = "M-CLIP" + + def __init__(self, transformerDimSize=1024, imageDimSize=768, **kwargs): + self.transformerDimensions = transformerDimSize + self.numDims = imageDimSize + super().__init__(**kwargs) + + +class MultilingualCLIP(PreTrainedModel): + config_class = MCLIPConfig + + def __init__(self, config, *args, **kwargs): + super().__init__(config, *args, **kwargs) + self.transformer = XLMRobertaModel(config) + self.LinearTransformation = torch.nn.Linear( + in_features=config.transformerDimensions, out_features=config.numDims + ) + + def forward(self, input_ids, attention_mask): + embs = self.transformer(input_ids=input_ids, attention_mask=attention_mask)[0] + embs2 = (embs * attention_mask.unsqueeze(2)).sum(dim=1) / attention_mask.sum(dim=1)[:, None] + return self.LinearTransformation(embs2), embs diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/__init__.py new file mode 100755 index 0000000..67e97f1 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/__init__.py @@ -0,0 +1,70 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_kandinsky2_2"] = ["KandinskyV22Pipeline"] + _import_structure["pipeline_kandinsky2_2_combined"] = [ + "KandinskyV22CombinedPipeline", + "KandinskyV22Img2ImgCombinedPipeline", + "KandinskyV22InpaintCombinedPipeline", + ] + _import_structure["pipeline_kandinsky2_2_controlnet"] = ["KandinskyV22ControlnetPipeline"] + _import_structure["pipeline_kandinsky2_2_controlnet_img2img"] = ["KandinskyV22ControlnetImg2ImgPipeline"] + _import_structure["pipeline_kandinsky2_2_img2img"] = ["KandinskyV22Img2ImgPipeline"] + _import_structure["pipeline_kandinsky2_2_inpainting"] = ["KandinskyV22InpaintPipeline"] + _import_structure["pipeline_kandinsky2_2_prior"] = ["KandinskyV22PriorPipeline"] + _import_structure["pipeline_kandinsky2_2_prior_emb2emb"] = ["KandinskyV22PriorEmb2EmbPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_kandinsky2_2 import KandinskyV22Pipeline + from .pipeline_kandinsky2_2_combined import ( + KandinskyV22CombinedPipeline, + KandinskyV22Img2ImgCombinedPipeline, + KandinskyV22InpaintCombinedPipeline, + ) + from .pipeline_kandinsky2_2_controlnet import KandinskyV22ControlnetPipeline + from .pipeline_kandinsky2_2_controlnet_img2img import KandinskyV22ControlnetImg2ImgPipeline + from .pipeline_kandinsky2_2_img2img import KandinskyV22Img2ImgPipeline + from .pipeline_kandinsky2_2_inpainting import KandinskyV22InpaintPipeline + from .pipeline_kandinsky2_2_prior import KandinskyV22PriorPipeline + from .pipeline_kandinsky2_2_prior_emb2emb import KandinskyV22PriorEmb2EmbPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py new file mode 100755 index 0000000..4b977af --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2.py @@ -0,0 +1,320 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Dict, List, Optional, Union + +import torch + +from ...models import UNet2DConditionModel, VQModel +from ...schedulers import DDPMScheduler +from ...utils import deprecate, logging, replace_example_docstring +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyV22Pipeline, KandinskyV22PriorPipeline + >>> import torch + + >>> pipe_prior = KandinskyV22PriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-prior") + >>> pipe_prior.to("cuda") + >>> prompt = "red cat, 4k photo" + >>> out = pipe_prior(prompt) + >>> image_emb = out.image_embeds + >>> zero_image_emb = out.negative_image_embeds + >>> pipe = KandinskyV22Pipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder") + >>> pipe.to("cuda") + >>> image = pipe( + ... image_embeds=image_emb, + ... negative_image_embeds=zero_image_emb, + ... height=768, + ... width=768, + ... num_inference_steps=50, + ... ).images + >>> image[0].save("cat.png") + ``` +""" + + +def downscale_height_and_width(height, width, scale_factor=8): + new_height = height // scale_factor**2 + if height % scale_factor**2 != 0: + new_height += 1 + new_width = width // scale_factor**2 + if width % scale_factor**2 != 0: + new_width += 1 + return new_height * scale_factor, new_width * scale_factor + + +class KandinskyV22Pipeline(DiffusionPipeline): + """ + Pipeline for text-to-image generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + scheduler (Union[`DDIMScheduler`,`DDPMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + """ + + model_cpu_offload_seq = "unet->movq" + _callback_tensor_inputs = ["latents", "image_embeds", "negative_image_embeds"] + + def __init__( + self, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + movq: VQModel, + ): + super().__init__() + + self.register_modules( + unet=unet, + scheduler=scheduler, + movq=movq, + ) + self.movq_scale_factor = 2 ** (len(self.movq.config.block_out_channels) - 1) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + negative_image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + height: int = 512, + width: int = 512, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for text prompt, that will be used to condition the image generation. + negative_image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for negative text prompt, will be used to condition the image generation. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + device = self._execution_device + + self._guidance_scale = guidance_scale + + if isinstance(image_embeds, list): + image_embeds = torch.cat(image_embeds, dim=0) + batch_size = image_embeds.shape[0] * num_images_per_prompt + if isinstance(negative_image_embeds, list): + negative_image_embeds = torch.cat(negative_image_embeds, dim=0) + + if self.do_classifier_free_guidance: + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + negative_image_embeds = negative_image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + + image_embeds = torch.cat([negative_image_embeds, image_embeds], dim=0).to( + dtype=self.unet.dtype, device=device + ) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + num_channels_latents = self.unet.config.in_channels + + height, width = downscale_height_and_width(height, width, self.movq_scale_factor) + + # create initial latent + latents = self.prepare_latents( + (batch_size, num_channels_latents, height, width), + image_embeds.dtype, + device, + generator, + latents, + self.scheduler, + ) + + self._num_timesteps = len(timesteps) + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + added_cond_kwargs = {"image_embeds": image_embeds} + noise_pred = self.unet( + sample=latent_model_input, + timestep=t, + encoder_hidden_states=None, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + if self.do_classifier_free_guidance: + noise_pred, variance_pred = noise_pred.split(latents.shape[1], dim=1) + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + _, variance_pred_text = variance_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, variance_pred_text], dim=1) + + if not ( + hasattr(self.scheduler.config, "variance_type") + and self.scheduler.config.variance_type in ["learned", "learned_range"] + ): + noise_pred, _ = noise_pred.split(latents.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, + t, + latents, + generator=generator, + )[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + image_embeds = callback_outputs.pop("image_embeds", image_embeds) + negative_image_embeds = callback_outputs.pop("negative_image_embeds", negative_image_embeds) + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if output_type not in ["pt", "np", "pil", "latent"]: + raise ValueError(f"Only the output types `pt`, `pil` and `np` are supported not output_type={output_type}") + + if not output_type == "latent": + # post-processing + image = self.movq.decode(latents, force_not_quantize=True)["sample"] + if output_type in ["np", "pil"]: + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + else: + image = latents + + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py new file mode 100755 index 0000000..65ba22c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_combined.py @@ -0,0 +1,851 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Dict, List, Optional, Union + +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...models import PriorTransformer, UNet2DConditionModel, VQModel +from ...schedulers import DDPMScheduler, UnCLIPScheduler +from ...utils import deprecate, logging, replace_example_docstring +from ..pipeline_utils import DiffusionPipeline +from .pipeline_kandinsky2_2 import KandinskyV22Pipeline +from .pipeline_kandinsky2_2_img2img import KandinskyV22Img2ImgPipeline +from .pipeline_kandinsky2_2_inpainting import KandinskyV22InpaintPipeline +from .pipeline_kandinsky2_2_prior import KandinskyV22PriorPipeline + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +TEXT2IMAGE_EXAMPLE_DOC_STRING = """ + Examples: + ```py + from diffusers import AutoPipelineForText2Image + import torch + + pipe = AutoPipelineForText2Image.from_pretrained( + "kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16 + ) + pipe.enable_model_cpu_offload() + + prompt = "A lion in galaxies, spirals, nebulae, stars, smoke, iridescent, intricate detail, octane render, 8k" + + image = pipe(prompt=prompt, num_inference_steps=25).images[0] + ``` +""" + +IMAGE2IMAGE_EXAMPLE_DOC_STRING = """ + Examples: + ```py + from diffusers import AutoPipelineForImage2Image + import torch + import requests + from io import BytesIO + from PIL import Image + import os + + pipe = AutoPipelineForImage2Image.from_pretrained( + "kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16 + ) + pipe.enable_model_cpu_offload() + + prompt = "A fantasy landscape, Cinematic lighting" + negative_prompt = "low quality, bad quality" + + url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + + response = requests.get(url) + image = Image.open(BytesIO(response.content)).convert("RGB") + image.thumbnail((768, 768)) + + image = pipe(prompt=prompt, image=original_image, num_inference_steps=25).images[0] + ``` +""" + +INPAINT_EXAMPLE_DOC_STRING = """ + Examples: + ```py + from diffusers import AutoPipelineForInpainting + from diffusers.utils import load_image + import torch + import numpy as np + + pipe = AutoPipelineForInpainting.from_pretrained( + "kandinsky-community/kandinsky-2-2-decoder-inpaint", torch_dtype=torch.float16 + ) + pipe.enable_model_cpu_offload() + + prompt = "A fantasy landscape, Cinematic lighting" + negative_prompt = "low quality, bad quality" + + original_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/cat.png" + ) + + mask = np.zeros((768, 768), dtype=np.float32) + # Let's mask out an area above the cat's head + mask[:250, 250:-250] = 1 + + image = pipe(prompt=prompt, image=original_image, mask_image=mask, num_inference_steps=25).images[0] + ``` +""" + + +class KandinskyV22CombinedPipeline(DiffusionPipeline): + """ + Combined Pipeline for text-to-image generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + scheduler (Union[`DDIMScheduler`,`DDPMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + prior_prior ([`PriorTransformer`]): + The canonincal unCLIP prior to approximate the image embedding from the text embedding. + prior_image_encoder ([`CLIPVisionModelWithProjection`]): + Frozen image-encoder. + prior_text_encoder ([`CLIPTextModelWithProjection`]): + Frozen text-encoder. + prior_tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + prior_scheduler ([`UnCLIPScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + prior_image_processor ([`CLIPImageProcessor`]): + A image_processor to be used to preprocess image from clip. + """ + + model_cpu_offload_seq = "prior_text_encoder->prior_image_encoder->unet->movq" + _load_connected_pipes = True + + def __init__( + self, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + movq: VQModel, + prior_prior: PriorTransformer, + prior_image_encoder: CLIPVisionModelWithProjection, + prior_text_encoder: CLIPTextModelWithProjection, + prior_tokenizer: CLIPTokenizer, + prior_scheduler: UnCLIPScheduler, + prior_image_processor: CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + unet=unet, + scheduler=scheduler, + movq=movq, + prior_prior=prior_prior, + prior_image_encoder=prior_image_encoder, + prior_text_encoder=prior_text_encoder, + prior_tokenizer=prior_tokenizer, + prior_scheduler=prior_scheduler, + prior_image_processor=prior_image_processor, + ) + self.prior_pipe = KandinskyV22PriorPipeline( + prior=prior_prior, + image_encoder=prior_image_encoder, + text_encoder=prior_text_encoder, + tokenizer=prior_tokenizer, + scheduler=prior_scheduler, + image_processor=prior_image_processor, + ) + self.decoder_pipe = KandinskyV22Pipeline( + unet=unet, + scheduler=scheduler, + movq=movq, + ) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) + + def enable_sequential_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet, + text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a + `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called. + Note that offloading happens on a submodule basis. Memory savings are higher than with + `enable_model_cpu_offload`, but performance is lower. + """ + self.prior_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + self.decoder_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + + def progress_bar(self, iterable=None, total=None): + self.prior_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.enable_model_cpu_offload() + + def set_progress_bar_config(self, **kwargs): + self.prior_pipe.set_progress_bar_config(**kwargs) + self.decoder_pipe.set_progress_bar_config(**kwargs) + + @torch.no_grad() + @replace_example_docstring(TEXT2IMAGE_EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + num_images_per_prompt: int = 1, + height: int = 512, + width: int = 512, + prior_guidance_scale: float = 4.0, + prior_num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + return_dict: bool = True, + prior_callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + prior_callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + prior_guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + prior_num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + prior_callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference of the prior pipeline. + The function is called with the following arguments: `prior_callback_on_step_end(self: + DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. + prior_callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `prior_callback_on_step_end` function. The tensors specified in the + list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in + the `._callback_tensor_inputs` attribute of your prior pipeline class. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference of the decoder pipeline. + The function is called with the following arguments: `callback_on_step_end(self: DiffusionPipeline, + step: int, timestep: int, callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors + as specified by `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + prior_outputs = self.prior_pipe( + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + num_inference_steps=prior_num_inference_steps, + generator=generator, + latents=latents, + guidance_scale=prior_guidance_scale, + output_type="pt", + return_dict=False, + callback_on_step_end=prior_callback_on_step_end, + callback_on_step_end_tensor_inputs=prior_callback_on_step_end_tensor_inputs, + ) + image_embeds = prior_outputs[0] + negative_image_embeds = prior_outputs[1] + + prompt = [prompt] if not isinstance(prompt, (list, tuple)) else prompt + + if len(prompt) < image_embeds.shape[0] and image_embeds.shape[0] % len(prompt) == 0: + prompt = (image_embeds.shape[0] // len(prompt)) * prompt + + outputs = self.decoder_pipe( + image_embeds=image_embeds, + negative_image_embeds=negative_image_embeds, + width=width, + height=height, + num_inference_steps=num_inference_steps, + generator=generator, + guidance_scale=guidance_scale, + output_type=output_type, + callback=callback, + callback_steps=callback_steps, + return_dict=return_dict, + callback_on_step_end=callback_on_step_end, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + ) + self.maybe_free_model_hooks() + + return outputs + + +class KandinskyV22Img2ImgCombinedPipeline(DiffusionPipeline): + """ + Combined Pipeline for image-to-image generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + scheduler (Union[`DDIMScheduler`,`DDPMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + prior_prior ([`PriorTransformer`]): + The canonincal unCLIP prior to approximate the image embedding from the text embedding. + prior_image_encoder ([`CLIPVisionModelWithProjection`]): + Frozen image-encoder. + prior_text_encoder ([`CLIPTextModelWithProjection`]): + Frozen text-encoder. + prior_tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + prior_scheduler ([`UnCLIPScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + prior_image_processor ([`CLIPImageProcessor`]): + A image_processor to be used to preprocess image from clip. + """ + + model_cpu_offload_seq = "prior_text_encoder->prior_image_encoder->unet->movq" + _load_connected_pipes = True + + def __init__( + self, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + movq: VQModel, + prior_prior: PriorTransformer, + prior_image_encoder: CLIPVisionModelWithProjection, + prior_text_encoder: CLIPTextModelWithProjection, + prior_tokenizer: CLIPTokenizer, + prior_scheduler: UnCLIPScheduler, + prior_image_processor: CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + unet=unet, + scheduler=scheduler, + movq=movq, + prior_prior=prior_prior, + prior_image_encoder=prior_image_encoder, + prior_text_encoder=prior_text_encoder, + prior_tokenizer=prior_tokenizer, + prior_scheduler=prior_scheduler, + prior_image_processor=prior_image_processor, + ) + self.prior_pipe = KandinskyV22PriorPipeline( + prior=prior_prior, + image_encoder=prior_image_encoder, + text_encoder=prior_text_encoder, + tokenizer=prior_tokenizer, + scheduler=prior_scheduler, + image_processor=prior_image_processor, + ) + self.decoder_pipe = KandinskyV22Img2ImgPipeline( + unet=unet, + scheduler=scheduler, + movq=movq, + ) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) + + def enable_model_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared + to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` + method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with + `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`. + """ + self.prior_pipe.enable_model_cpu_offload() + self.decoder_pipe.enable_model_cpu_offload() + + def enable_sequential_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet, + text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a + `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called. + Note that offloading happens on a submodule basis. Memory savings are higher than with + `enable_model_cpu_offload`, but performance is lower. + """ + self.prior_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + self.decoder_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + + def progress_bar(self, iterable=None, total=None): + self.prior_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.enable_model_cpu_offload() + + def set_progress_bar_config(self, **kwargs): + self.prior_pipe.set_progress_bar_config(**kwargs) + self.decoder_pipe.set_progress_bar_config(**kwargs) + + @torch.no_grad() + @replace_example_docstring(IMAGE2IMAGE_EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]], + negative_prompt: Optional[Union[str, List[str]]] = None, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + strength: float = 0.3, + num_images_per_prompt: int = 1, + height: int = 512, + width: int = 512, + prior_guidance_scale: float = 4.0, + prior_num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + return_dict: bool = True, + prior_callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + prior_callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded + again. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + strength (`float`, *optional*, defaults to 0.3): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + prior_guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + prior_num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + prior_outputs = self.prior_pipe( + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + num_inference_steps=prior_num_inference_steps, + generator=generator, + latents=latents, + guidance_scale=prior_guidance_scale, + output_type="pt", + return_dict=False, + callback_on_step_end=prior_callback_on_step_end, + callback_on_step_end_tensor_inputs=prior_callback_on_step_end_tensor_inputs, + ) + image_embeds = prior_outputs[0] + negative_image_embeds = prior_outputs[1] + + prompt = [prompt] if not isinstance(prompt, (list, tuple)) else prompt + image = [image] if isinstance(prompt, PIL.Image.Image) else image + + if len(prompt) < image_embeds.shape[0] and image_embeds.shape[0] % len(prompt) == 0: + prompt = (image_embeds.shape[0] // len(prompt)) * prompt + + if ( + isinstance(image, (list, tuple)) + and len(image) < image_embeds.shape[0] + and image_embeds.shape[0] % len(image) == 0 + ): + image = (image_embeds.shape[0] // len(image)) * image + + outputs = self.decoder_pipe( + image=image, + image_embeds=image_embeds, + negative_image_embeds=negative_image_embeds, + width=width, + height=height, + strength=strength, + num_inference_steps=num_inference_steps, + generator=generator, + guidance_scale=guidance_scale, + output_type=output_type, + callback=callback, + callback_steps=callback_steps, + return_dict=return_dict, + callback_on_step_end=callback_on_step_end, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + ) + + self.maybe_free_model_hooks() + return outputs + + +class KandinskyV22InpaintCombinedPipeline(DiffusionPipeline): + """ + Combined Pipeline for inpainting generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + scheduler (Union[`DDIMScheduler`,`DDPMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + prior_prior ([`PriorTransformer`]): + The canonincal unCLIP prior to approximate the image embedding from the text embedding. + prior_image_encoder ([`CLIPVisionModelWithProjection`]): + Frozen image-encoder. + prior_text_encoder ([`CLIPTextModelWithProjection`]): + Frozen text-encoder. + prior_tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + prior_scheduler ([`UnCLIPScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + prior_image_processor ([`CLIPImageProcessor`]): + A image_processor to be used to preprocess image from clip. + """ + + model_cpu_offload_seq = "prior_text_encoder->prior_image_encoder->unet->movq" + _load_connected_pipes = True + + def __init__( + self, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + movq: VQModel, + prior_prior: PriorTransformer, + prior_image_encoder: CLIPVisionModelWithProjection, + prior_text_encoder: CLIPTextModelWithProjection, + prior_tokenizer: CLIPTokenizer, + prior_scheduler: UnCLIPScheduler, + prior_image_processor: CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + unet=unet, + scheduler=scheduler, + movq=movq, + prior_prior=prior_prior, + prior_image_encoder=prior_image_encoder, + prior_text_encoder=prior_text_encoder, + prior_tokenizer=prior_tokenizer, + prior_scheduler=prior_scheduler, + prior_image_processor=prior_image_processor, + ) + self.prior_pipe = KandinskyV22PriorPipeline( + prior=prior_prior, + image_encoder=prior_image_encoder, + text_encoder=prior_text_encoder, + tokenizer=prior_tokenizer, + scheduler=prior_scheduler, + image_processor=prior_image_processor, + ) + self.decoder_pipe = KandinskyV22InpaintPipeline( + unet=unet, + scheduler=scheduler, + movq=movq, + ) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) + + def enable_sequential_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, significantly reducing memory usage. When called, unet, + text_encoder, vae and safety checker have their state dicts saved to CPU and then are moved to a + `torch.device('meta') and loaded to GPU only when their specific submodule has its `forward` method called. + Note that offloading happens on a submodule basis. Memory savings are higher than with + `enable_model_cpu_offload`, but performance is lower. + """ + self.prior_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + self.decoder_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + + def progress_bar(self, iterable=None, total=None): + self.prior_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.enable_model_cpu_offload() + + def set_progress_bar_config(self, **kwargs): + self.prior_pipe.set_progress_bar_config(**kwargs) + self.decoder_pipe.set_progress_bar_config(**kwargs) + + @torch.no_grad() + @replace_example_docstring(INPAINT_EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]], + mask_image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]], + negative_prompt: Optional[Union[str, List[str]]] = None, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + num_images_per_prompt: int = 1, + height: int = 512, + width: int = 512, + prior_guidance_scale: float = 4.0, + prior_num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + prior_callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + prior_callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded + again. + mask_image (`np.array`): + Tensor representing an image batch, to mask `image`. White pixels in the mask will be repainted, while + black pixels will be preserved. If `mask_image` is a PIL image, it will be converted to a single + channel (luminance) before use. If it's a tensor, it should contain one color channel (L) instead of 3, + so the expected shape would be `(B, H, W, 1)`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + prior_guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + prior_num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + prior_callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `prior_callback_on_step_end(self: DiffusionPipeline, step: int, timestep: + int, callback_kwargs: Dict)`. + prior_callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `prior_callback_on_step_end` function. The tensors specified in the + list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in + the `._callback_tensor_inputs` attribute of your pipeline class. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + prior_kwargs = {} + if kwargs.get("prior_callback", None) is not None: + prior_kwargs["callback"] = kwargs.pop("prior_callback") + deprecate( + "prior_callback", + "1.0.0", + "Passing `prior_callback` as an input argument to `__call__` is deprecated, consider use `prior_callback_on_step_end`", + ) + if kwargs.get("prior_callback_steps", None) is not None: + deprecate( + "prior_callback_steps", + "1.0.0", + "Passing `prior_callback_steps` as an input argument to `__call__` is deprecated, consider use `prior_callback_on_step_end`", + ) + prior_kwargs["callback_steps"] = kwargs.pop("prior_callback_steps") + + prior_outputs = self.prior_pipe( + prompt=prompt, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + num_inference_steps=prior_num_inference_steps, + generator=generator, + latents=latents, + guidance_scale=prior_guidance_scale, + output_type="pt", + return_dict=False, + callback_on_step_end=prior_callback_on_step_end, + callback_on_step_end_tensor_inputs=prior_callback_on_step_end_tensor_inputs, + **prior_kwargs, + ) + image_embeds = prior_outputs[0] + negative_image_embeds = prior_outputs[1] + + prompt = [prompt] if not isinstance(prompt, (list, tuple)) else prompt + image = [image] if isinstance(prompt, PIL.Image.Image) else image + mask_image = [mask_image] if isinstance(mask_image, PIL.Image.Image) else mask_image + + if len(prompt) < image_embeds.shape[0] and image_embeds.shape[0] % len(prompt) == 0: + prompt = (image_embeds.shape[0] // len(prompt)) * prompt + + if ( + isinstance(image, (list, tuple)) + and len(image) < image_embeds.shape[0] + and image_embeds.shape[0] % len(image) == 0 + ): + image = (image_embeds.shape[0] // len(image)) * image + + if ( + isinstance(mask_image, (list, tuple)) + and len(mask_image) < image_embeds.shape[0] + and image_embeds.shape[0] % len(mask_image) == 0 + ): + mask_image = (image_embeds.shape[0] // len(mask_image)) * mask_image + + outputs = self.decoder_pipe( + image=image, + mask_image=mask_image, + image_embeds=image_embeds, + negative_image_embeds=negative_image_embeds, + width=width, + height=height, + num_inference_steps=num_inference_steps, + generator=generator, + guidance_scale=guidance_scale, + output_type=output_type, + return_dict=return_dict, + callback_on_step_end=callback_on_step_end, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + **kwargs, + ) + self.maybe_free_model_hooks() + + return outputs diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py new file mode 100755 index 0000000..de87dd3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet.py @@ -0,0 +1,320 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, List, Optional, Union + +import torch + +from ...models import UNet2DConditionModel, VQModel +from ...schedulers import DDPMScheduler +from ...utils import ( + logging, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> import numpy as np + + >>> from diffusers import KandinskyV22PriorPipeline, KandinskyV22ControlnetPipeline + >>> from transformers import pipeline + >>> from diffusers.utils import load_image + + + >>> def make_hint(image, depth_estimator): + ... image = depth_estimator(image)["depth"] + ... image = np.array(image) + ... image = image[:, :, None] + ... image = np.concatenate([image, image, image], axis=2) + ... detected_map = torch.from_numpy(image).float() / 255.0 + ... hint = detected_map.permute(2, 0, 1) + ... return hint + + + >>> depth_estimator = pipeline("depth-estimation") + + >>> pipe_prior = KandinskyV22PriorPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ... ) + >>> pipe_prior = pipe_prior.to("cuda") + + >>> pipe = KandinskyV22ControlnetPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-controlnet-depth", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + + >>> img = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/cat.png" + ... ).resize((768, 768)) + + >>> hint = make_hint(img, depth_estimator).unsqueeze(0).half().to("cuda") + + >>> prompt = "A robot, 4k photo" + >>> negative_prior_prompt = "lowres, text, error, cropped, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature" + + >>> generator = torch.Generator(device="cuda").manual_seed(43) + + >>> image_emb, zero_image_emb = pipe_prior( + ... prompt=prompt, negative_prompt=negative_prior_prompt, generator=generator + ... ).to_tuple() + + >>> images = pipe( + ... image_embeds=image_emb, + ... negative_image_embeds=zero_image_emb, + ... hint=hint, + ... num_inference_steps=50, + ... generator=generator, + ... height=768, + ... width=768, + ... ).images + + >>> images[0].save("robot_cat.png") + ``` +""" + + +# Copied from diffusers.pipelines.kandinsky2_2.pipeline_kandinsky2_2.downscale_height_and_width +def downscale_height_and_width(height, width, scale_factor=8): + new_height = height // scale_factor**2 + if height % scale_factor**2 != 0: + new_height += 1 + new_width = width // scale_factor**2 + if width % scale_factor**2 != 0: + new_width += 1 + return new_height * scale_factor, new_width * scale_factor + + +class KandinskyV22ControlnetPipeline(DiffusionPipeline): + """ + Pipeline for text-to-image generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + scheduler ([`DDIMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + """ + + model_cpu_offload_seq = "unet->movq" + + def __init__( + self, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + movq: VQModel, + ): + super().__init__() + + self.register_modules( + unet=unet, + scheduler=scheduler, + movq=movq, + ) + self.movq_scale_factor = 2 ** (len(self.movq.config.block_out_channels) - 1) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + @torch.no_grad() + def __call__( + self, + image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + negative_image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + hint: torch.FloatTensor, + height: int = 512, + width: int = 512, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + hint (`torch.FloatTensor`): + The controlnet condition. + image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for text prompt, that will be used to condition the image generation. + negative_image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for negative text prompt, will be used to condition the image generation. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + device = self._execution_device + + do_classifier_free_guidance = guidance_scale > 1.0 + + if isinstance(image_embeds, list): + image_embeds = torch.cat(image_embeds, dim=0) + if isinstance(negative_image_embeds, list): + negative_image_embeds = torch.cat(negative_image_embeds, dim=0) + if isinstance(hint, list): + hint = torch.cat(hint, dim=0) + + batch_size = image_embeds.shape[0] * num_images_per_prompt + + if do_classifier_free_guidance: + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + negative_image_embeds = negative_image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + hint = hint.repeat_interleave(num_images_per_prompt, dim=0) + + image_embeds = torch.cat([negative_image_embeds, image_embeds], dim=0).to( + dtype=self.unet.dtype, device=device + ) + hint = torch.cat([hint, hint], dim=0).to(dtype=self.unet.dtype, device=device) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps_tensor = self.scheduler.timesteps + + num_channels_latents = self.movq.config.latent_channels + + height, width = downscale_height_and_width(height, width, self.movq_scale_factor) + + # create initial latent + latents = self.prepare_latents( + (batch_size, num_channels_latents, height, width), + image_embeds.dtype, + device, + generator, + latents, + self.scheduler, + ) + + for i, t in enumerate(self.progress_bar(timesteps_tensor)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + added_cond_kwargs = {"image_embeds": image_embeds, "hint": hint} + noise_pred = self.unet( + sample=latent_model_input, + timestep=t, + encoder_hidden_states=None, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + if do_classifier_free_guidance: + noise_pred, variance_pred = noise_pred.split(latents.shape[1], dim=1) + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + _, variance_pred_text = variance_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, variance_pred_text], dim=1) + + if not ( + hasattr(self.scheduler.config, "variance_type") + and self.scheduler.config.variance_type in ["learned", "learned_range"] + ): + noise_pred, _ = noise_pred.split(latents.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, + t, + latents, + generator=generator, + )[0] + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + # post-processing + image = self.movq.decode(latents, force_not_quantize=True)["sample"] + + # Offload all models + self.maybe_free_model_hooks() + + if output_type not in ["pt", "np", "pil"]: + raise ValueError(f"Only the output types `pt`, `pil` and `np` are supported not output_type={output_type}") + + if output_type in ["np", "pil"]: + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py new file mode 100755 index 0000000..c3ac7bc --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_controlnet_img2img.py @@ -0,0 +1,381 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from PIL import Image + +from ...models import UNet2DConditionModel, VQModel +from ...schedulers import DDPMScheduler +from ...utils import ( + logging, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> import numpy as np + + >>> from diffusers import KandinskyV22PriorEmb2EmbPipeline, KandinskyV22ControlnetImg2ImgPipeline + >>> from transformers import pipeline + >>> from diffusers.utils import load_image + + + >>> def make_hint(image, depth_estimator): + ... image = depth_estimator(image)["depth"] + ... image = np.array(image) + ... image = image[:, :, None] + ... image = np.concatenate([image, image, image], axis=2) + ... detected_map = torch.from_numpy(image).float() / 255.0 + ... hint = detected_map.permute(2, 0, 1) + ... return hint + + + >>> depth_estimator = pipeline("depth-estimation") + + >>> pipe_prior = KandinskyV22PriorEmb2EmbPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ... ) + >>> pipe_prior = pipe_prior.to("cuda") + + >>> pipe = KandinskyV22ControlnetImg2ImgPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-controlnet-depth", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> img = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/cat.png" + ... ).resize((768, 768)) + + + >>> hint = make_hint(img, depth_estimator).unsqueeze(0).half().to("cuda") + + >>> prompt = "A robot, 4k photo" + >>> negative_prior_prompt = "lowres, text, error, cropped, worst quality, low quality, jpeg artifacts, ugly, duplicate, morbid, mutilated, out of frame, extra fingers, mutated hands, poorly drawn hands, poorly drawn face, mutation, deformed, blurry, dehydrated, bad anatomy, bad proportions, extra limbs, cloned face, disfigured, gross proportions, malformed limbs, missing arms, missing legs, extra arms, extra legs, fused fingers, too many fingers, long neck, username, watermark, signature" + + >>> generator = torch.Generator(device="cuda").manual_seed(43) + + >>> img_emb = pipe_prior(prompt=prompt, image=img, strength=0.85, generator=generator) + >>> negative_emb = pipe_prior(prompt=negative_prior_prompt, image=img, strength=1, generator=generator) + + >>> images = pipe( + ... image=img, + ... strength=0.5, + ... image_embeds=img_emb.image_embeds, + ... negative_image_embeds=negative_emb.image_embeds, + ... hint=hint, + ... num_inference_steps=50, + ... generator=generator, + ... height=768, + ... width=768, + ... ).images + + >>> images[0].save("robot_cat.png") + ``` +""" + + +# Copied from diffusers.pipelines.kandinsky2_2.pipeline_kandinsky2_2.downscale_height_and_width +def downscale_height_and_width(height, width, scale_factor=8): + new_height = height // scale_factor**2 + if height % scale_factor**2 != 0: + new_height += 1 + new_width = width // scale_factor**2 + if width % scale_factor**2 != 0: + new_width += 1 + return new_height * scale_factor, new_width * scale_factor + + +# Copied from diffusers.pipelines.kandinsky.pipeline_kandinsky_img2img.prepare_image +def prepare_image(pil_image, w=512, h=512): + pil_image = pil_image.resize((w, h), resample=Image.BICUBIC, reducing_gap=1) + arr = np.array(pil_image.convert("RGB")) + arr = arr.astype(np.float32) / 127.5 - 1 + arr = np.transpose(arr, [2, 0, 1]) + image = torch.from_numpy(arr).unsqueeze(0) + return image + + +class KandinskyV22ControlnetImg2ImgPipeline(DiffusionPipeline): + """ + Pipeline for image-to-image generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + scheduler ([`DDIMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + """ + + model_cpu_offload_seq = "unet->movq" + + def __init__( + self, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + movq: VQModel, + ): + super().__init__() + + self.register_modules( + unet=unet, + scheduler=scheduler, + movq=movq, + ) + self.movq_scale_factor = 2 ** (len(self.movq.config.block_out_channels) - 1) + + # Copied from diffusers.pipelines.kandinsky.pipeline_kandinsky_img2img.KandinskyImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start:] + + return timesteps, num_inference_steps - t_start + + # Copied from diffusers.pipelines.kandinsky2_2.pipeline_kandinsky2_2_img2img.KandinskyV22Img2ImgPipeline.prepare_latents + def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + init_latents = image + + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + elif isinstance(generator, list): + init_latents = [ + self.movq.encode(image[i : i + 1]).latent_dist.sample(generator[i]) for i in range(batch_size) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = self.movq.encode(image).latent_dist.sample(generator) + + init_latents = self.movq.config.scaling_factor * init_latents + + init_latents = torch.cat([init_latents], dim=0) + + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + + latents = init_latents + + return latents + + @torch.no_grad() + def __call__( + self, + image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]], + negative_image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + hint: torch.FloatTensor, + height: int = 512, + width: int = 512, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + strength: float = 0.3, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for text prompt, that will be used to condition the image generation. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded + again. + strength (`float`, *optional*, defaults to 0.8): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. + hint (`torch.FloatTensor`): + The controlnet condition. + negative_image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for negative text prompt, will be used to condition the image generation. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + device = self._execution_device + + do_classifier_free_guidance = guidance_scale > 1.0 + + if isinstance(image_embeds, list): + image_embeds = torch.cat(image_embeds, dim=0) + if isinstance(negative_image_embeds, list): + negative_image_embeds = torch.cat(negative_image_embeds, dim=0) + if isinstance(hint, list): + hint = torch.cat(hint, dim=0) + + batch_size = image_embeds.shape[0] + + if do_classifier_free_guidance: + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + negative_image_embeds = negative_image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + hint = hint.repeat_interleave(num_images_per_prompt, dim=0) + + image_embeds = torch.cat([negative_image_embeds, image_embeds], dim=0).to( + dtype=self.unet.dtype, device=device + ) + hint = torch.cat([hint, hint], dim=0).to(dtype=self.unet.dtype, device=device) + + if not isinstance(image, list): + image = [image] + if not all(isinstance(i, (PIL.Image.Image, torch.Tensor)) for i in image): + raise ValueError( + f"Input is in incorrect format: {[type(i) for i in image]}. Currently, we only support PIL image and pytorch tensor" + ) + + image = torch.cat([prepare_image(i, width, height) for i in image], dim=0) + image = image.to(dtype=image_embeds.dtype, device=device) + + latents = self.movq.encode(image)["latents"] + latents = latents.repeat_interleave(num_images_per_prompt, dim=0) + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + height, width = downscale_height_and_width(height, width, self.movq_scale_factor) + latents = self.prepare_latents( + latents, latent_timestep, batch_size, num_images_per_prompt, image_embeds.dtype, device, generator + ) + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + added_cond_kwargs = {"image_embeds": image_embeds, "hint": hint} + noise_pred = self.unet( + sample=latent_model_input, + timestep=t, + encoder_hidden_states=None, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + if do_classifier_free_guidance: + noise_pred, variance_pred = noise_pred.split(latents.shape[1], dim=1) + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + _, variance_pred_text = variance_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, variance_pred_text], dim=1) + + if not ( + hasattr(self.scheduler.config, "variance_type") + and self.scheduler.config.variance_type in ["learned", "learned_range"] + ): + noise_pred, _ = noise_pred.split(latents.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + + latents = self.scheduler.step( + noise_pred, + t, + latents, + generator=generator, + )[0] + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # post-processing + image = self.movq.decode(latents, force_not_quantize=True)["sample"] + + # Offload all models + self.maybe_free_model_hooks() + + if output_type not in ["pt", "np", "pil"]: + raise ValueError(f"Only the output types `pt`, `pil` and `np` are supported not output_type={output_type}") + + if output_type in ["np", "pil"]: + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py new file mode 100755 index 0000000..3fdae93 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_img2img.py @@ -0,0 +1,399 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from PIL import Image + +from ...models import UNet2DConditionModel, VQModel +from ...schedulers import DDPMScheduler +from ...utils import deprecate, logging +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyV22Img2ImgPipeline, KandinskyV22PriorPipeline + >>> from diffusers.utils import load_image + >>> import torch + + >>> pipe_prior = KandinskyV22PriorPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ... ) + >>> pipe_prior.to("cuda") + + >>> prompt = "A red cartoon frog, 4k" + >>> image_emb, zero_image_emb = pipe_prior(prompt, return_dict=False) + + >>> pipe = KandinskyV22Img2ImgPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16 + ... ) + >>> pipe.to("cuda") + + >>> init_image = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/frog.png" + ... ) + + >>> image = pipe( + ... image=init_image, + ... image_embeds=image_emb, + ... negative_image_embeds=zero_image_emb, + ... height=768, + ... width=768, + ... num_inference_steps=100, + ... strength=0.2, + ... ).images + + >>> image[0].save("red_frog.png") + ``` +""" + + +# Copied from diffusers.pipelines.kandinsky2_2.pipeline_kandinsky2_2.downscale_height_and_width +def downscale_height_and_width(height, width, scale_factor=8): + new_height = height // scale_factor**2 + if height % scale_factor**2 != 0: + new_height += 1 + new_width = width // scale_factor**2 + if width % scale_factor**2 != 0: + new_width += 1 + return new_height * scale_factor, new_width * scale_factor + + +# Copied from diffusers.pipelines.kandinsky.pipeline_kandinsky_img2img.prepare_image +def prepare_image(pil_image, w=512, h=512): + pil_image = pil_image.resize((w, h), resample=Image.BICUBIC, reducing_gap=1) + arr = np.array(pil_image.convert("RGB")) + arr = arr.astype(np.float32) / 127.5 - 1 + arr = np.transpose(arr, [2, 0, 1]) + image = torch.from_numpy(arr).unsqueeze(0) + return image + + +class KandinskyV22Img2ImgPipeline(DiffusionPipeline): + """ + Pipeline for image-to-image generation using Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + scheduler ([`DDIMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + """ + + model_cpu_offload_seq = "unet->movq" + _callback_tensor_inputs = ["latents", "image_embeds", "negative_image_embeds"] + + def __init__( + self, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + movq: VQModel, + ): + super().__init__() + + self.register_modules( + unet=unet, + scheduler=scheduler, + movq=movq, + ) + self.movq_scale_factor = 2 ** (len(self.movq.config.block_out_channels) - 1) + + # Copied from diffusers.pipelines.kandinsky.pipeline_kandinsky_img2img.KandinskyImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start:] + + return timesteps, num_inference_steps - t_start + + def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + init_latents = image + + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + elif isinstance(generator, list): + init_latents = [ + self.movq.encode(image[i : i + 1]).latent_dist.sample(generator[i]) for i in range(batch_size) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = self.movq.encode(image).latent_dist.sample(generator) + + init_latents = self.movq.config.scaling_factor * init_latents + + init_latents = torch.cat([init_latents], dim=0) + + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + + latents = init_latents + + return latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + def __call__( + self, + image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]], + negative_image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + height: int = 512, + width: int = 512, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + strength: float = 0.3, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for text prompt, that will be used to condition the image generation. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. Can also accept image latents as `image`, if passing latents directly, it will not be encoded + again. + strength (`float`, *optional*, defaults to 0.8): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. + negative_image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for negative text prompt, will be used to condition the image generation. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + device = self._execution_device + + self._guidance_scale = guidance_scale + + if isinstance(image_embeds, list): + image_embeds = torch.cat(image_embeds, dim=0) + batch_size = image_embeds.shape[0] + if isinstance(negative_image_embeds, list): + negative_image_embeds = torch.cat(negative_image_embeds, dim=0) + + if self.do_classifier_free_guidance: + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + negative_image_embeds = negative_image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + + image_embeds = torch.cat([negative_image_embeds, image_embeds], dim=0).to( + dtype=self.unet.dtype, device=device + ) + + if not isinstance(image, list): + image = [image] + if not all(isinstance(i, (PIL.Image.Image, torch.Tensor)) for i in image): + raise ValueError( + f"Input is in incorrect format: {[type(i) for i in image]}. Currently, we only support PIL image and pytorch tensor" + ) + + image = torch.cat([prepare_image(i, width, height) for i in image], dim=0) + image = image.to(dtype=image_embeds.dtype, device=device) + + latents = self.movq.encode(image)["latents"] + latents = latents.repeat_interleave(num_images_per_prompt, dim=0) + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + height, width = downscale_height_and_width(height, width, self.movq_scale_factor) + latents = self.prepare_latents( + latents, latent_timestep, batch_size, num_images_per_prompt, image_embeds.dtype, device, generator + ) + self._num_timesteps = len(timesteps) + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + added_cond_kwargs = {"image_embeds": image_embeds} + noise_pred = self.unet( + sample=latent_model_input, + timestep=t, + encoder_hidden_states=None, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + if self.do_classifier_free_guidance: + noise_pred, variance_pred = noise_pred.split(latents.shape[1], dim=1) + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + _, variance_pred_text = variance_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, variance_pred_text], dim=1) + + if not ( + hasattr(self.scheduler.config, "variance_type") + and self.scheduler.config.variance_type in ["learned", "learned_range"] + ): + noise_pred, _ = noise_pred.split(latents.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, + t, + latents, + generator=generator, + )[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + image_embeds = callback_outputs.pop("image_embeds", image_embeds) + negative_image_embeds = callback_outputs.pop("negative_image_embeds", negative_image_embeds) + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if output_type not in ["pt", "np", "pil", "latent"]: + raise ValueError( + f"Only the output types `pt`, `pil` ,`np` and `latent` are supported not output_type={output_type}" + ) + + if not output_type == "latent": + # post-processing + image = self.movq.decode(latents, force_not_quantize=True)["sample"] + if output_type in ["np", "pil"]: + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + else: + image = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py new file mode 100755 index 0000000..2fb8731 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_inpainting.py @@ -0,0 +1,556 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from copy import deepcopy +from typing import Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from packaging import version +from PIL import Image + +from ... import __version__ +from ...models import UNet2DConditionModel, VQModel +from ...schedulers import DDPMScheduler +from ...utils import deprecate, logging +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyV22InpaintPipeline, KandinskyV22PriorPipeline + >>> from diffusers.utils import load_image + >>> import torch + >>> import numpy as np + + >>> pipe_prior = KandinskyV22PriorPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ... ) + >>> pipe_prior.to("cuda") + + >>> prompt = "a hat" + >>> image_emb, zero_image_emb = pipe_prior(prompt, return_dict=False) + + >>> pipe = KandinskyV22InpaintPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-decoder-inpaint", torch_dtype=torch.float16 + ... ) + >>> pipe.to("cuda") + + >>> init_image = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/cat.png" + ... ) + + >>> mask = np.zeros((768, 768), dtype=np.float32) + >>> mask[:250, 250:-250] = 1 + + >>> out = pipe( + ... image=init_image, + ... mask_image=mask, + ... image_embeds=image_emb, + ... negative_image_embeds=zero_image_emb, + ... height=768, + ... width=768, + ... num_inference_steps=50, + ... ) + + >>> image = out.images[0] + >>> image.save("cat_with_hat.png") + ``` +""" + + +# Copied from diffusers.pipelines.kandinsky2_2.pipeline_kandinsky2_2.downscale_height_and_width +def downscale_height_and_width(height, width, scale_factor=8): + new_height = height // scale_factor**2 + if height % scale_factor**2 != 0: + new_height += 1 + new_width = width // scale_factor**2 + if width % scale_factor**2 != 0: + new_width += 1 + return new_height * scale_factor, new_width * scale_factor + + +# Copied from diffusers.pipelines.kandinsky.pipeline_kandinsky_inpaint.prepare_mask +def prepare_mask(masks): + prepared_masks = [] + for mask in masks: + old_mask = deepcopy(mask) + for i in range(mask.shape[1]): + for j in range(mask.shape[2]): + if old_mask[0][i][j] == 1: + continue + if i != 0: + mask[:, i - 1, j] = 0 + if j != 0: + mask[:, i, j - 1] = 0 + if i != 0 and j != 0: + mask[:, i - 1, j - 1] = 0 + if i != mask.shape[1] - 1: + mask[:, i + 1, j] = 0 + if j != mask.shape[2] - 1: + mask[:, i, j + 1] = 0 + if i != mask.shape[1] - 1 and j != mask.shape[2] - 1: + mask[:, i + 1, j + 1] = 0 + prepared_masks.append(mask) + return torch.stack(prepared_masks, dim=0) + + +# Copied from diffusers.pipelines.kandinsky.pipeline_kandinsky_inpaint.prepare_mask_and_masked_image +def prepare_mask_and_masked_image(image, mask, height, width): + r""" + Prepares a pair (mask, image) to be consumed by the Kandinsky inpaint pipeline. This means that those inputs will + be converted to ``torch.Tensor`` with shapes ``batch x channels x height x width`` where ``channels`` is ``3`` for + the ``image`` and ``1`` for the ``mask``. + + The ``image`` will be converted to ``torch.float32`` and normalized to be in ``[-1, 1]``. The ``mask`` will be + binarized (``mask > 0.5``) and cast to ``torch.float32`` too. + + Args: + image (Union[np.array, PIL.Image, torch.Tensor]): The image to inpaint. + It can be a ``PIL.Image``, or a ``height x width x 3`` ``np.array`` or a ``channels x height x width`` + ``torch.Tensor`` or a ``batch x channels x height x width`` ``torch.Tensor``. + mask (_type_): The mask to apply to the image, i.e. regions to inpaint. + It can be a ``PIL.Image``, or a ``height x width`` ``np.array`` or a ``1 x height x width`` + ``torch.Tensor`` or a ``batch x 1 x height x width`` ``torch.Tensor``. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + + + Raises: + ValueError: ``torch.Tensor`` images should be in the ``[-1, 1]`` range. ValueError: ``torch.Tensor`` mask + should be in the ``[0, 1]`` range. ValueError: ``mask`` and ``image`` should have the same spatial dimensions. + TypeError: ``mask`` is a ``torch.Tensor`` but ``image`` is not + (ot the other way around). + + Returns: + tuple[torch.Tensor]: The pair (mask, image) as ``torch.Tensor`` with 4 + dimensions: ``batch x channels x height x width``. + """ + + if image is None: + raise ValueError("`image` input cannot be undefined.") + + if mask is None: + raise ValueError("`mask_image` input cannot be undefined.") + + if isinstance(image, torch.Tensor): + if not isinstance(mask, torch.Tensor): + raise TypeError(f"`image` is a torch.Tensor but `mask` (type: {type(mask)} is not") + + # Batch single image + if image.ndim == 3: + assert image.shape[0] == 3, "Image outside a batch should be of shape (3, H, W)" + image = image.unsqueeze(0) + + # Batch and add channel dim for single mask + if mask.ndim == 2: + mask = mask.unsqueeze(0).unsqueeze(0) + + # Batch single mask or add channel dim + if mask.ndim == 3: + # Single batched mask, no channel dim or single mask not batched but channel dim + if mask.shape[0] == 1: + mask = mask.unsqueeze(0) + + # Batched masks no channel dim + else: + mask = mask.unsqueeze(1) + + assert image.ndim == 4 and mask.ndim == 4, "Image and Mask must have 4 dimensions" + assert image.shape[-2:] == mask.shape[-2:], "Image and Mask must have the same spatial dimensions" + assert image.shape[0] == mask.shape[0], "Image and Mask must have the same batch size" + + # Check image is in [-1, 1] + if image.min() < -1 or image.max() > 1: + raise ValueError("Image should be in [-1, 1] range") + + # Check mask is in [0, 1] + if mask.min() < 0 or mask.max() > 1: + raise ValueError("Mask should be in [0, 1] range") + + # Binarize mask + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + + # Image as float32 + image = image.to(dtype=torch.float32) + elif isinstance(mask, torch.Tensor): + raise TypeError(f"`mask` is a torch.Tensor but `image` (type: {type(image)} is not") + else: + # preprocess image + if isinstance(image, (PIL.Image.Image, np.ndarray)): + image = [image] + + if isinstance(image, list) and isinstance(image[0], PIL.Image.Image): + # resize all images w.r.t passed height an width + image = [i.resize((width, height), resample=Image.BICUBIC, reducing_gap=1) for i in image] + image = [np.array(i.convert("RGB"))[None, :] for i in image] + image = np.concatenate(image, axis=0) + elif isinstance(image, list) and isinstance(image[0], np.ndarray): + image = np.concatenate([i[None, :] for i in image], axis=0) + + image = image.transpose(0, 3, 1, 2) + image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0 + + # preprocess mask + if isinstance(mask, (PIL.Image.Image, np.ndarray)): + mask = [mask] + + if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image): + mask = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in mask] + mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0) + mask = mask.astype(np.float32) / 255.0 + elif isinstance(mask, list) and isinstance(mask[0], np.ndarray): + mask = np.concatenate([m[None, None, :] for m in mask], axis=0) + + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + mask = torch.from_numpy(mask) + + mask = 1 - mask + + return mask, image + + +class KandinskyV22InpaintPipeline(DiffusionPipeline): + """ + Pipeline for text-guided image inpainting using Kandinsky2.1 + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + scheduler ([`DDIMScheduler`]): + A scheduler to be used in combination with `unet` to generate image latents. + unet ([`UNet2DConditionModel`]): + Conditional U-Net architecture to denoise the image embedding. + movq ([`VQModel`]): + MoVQ Decoder to generate the image from the latents. + """ + + model_cpu_offload_seq = "unet->movq" + _callback_tensor_inputs = ["latents", "image_embeds", "negative_image_embeds", "masked_image", "mask_image"] + + def __init__( + self, + unet: UNet2DConditionModel, + scheduler: DDPMScheduler, + movq: VQModel, + ): + super().__init__() + + self.register_modules( + unet=unet, + scheduler=scheduler, + movq=movq, + ) + self.movq_scale_factor = 2 ** (len(self.movq.config.block_out_channels) - 1) + self._warn_has_been_called = False + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + def __call__( + self, + image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + image: Union[torch.FloatTensor, PIL.Image.Image], + mask_image: Union[torch.FloatTensor, PIL.Image.Image, np.ndarray], + negative_image_embeds: Union[torch.FloatTensor, List[torch.FloatTensor]], + height: int = 512, + width: int = 512, + num_inference_steps: int = 100, + guidance_scale: float = 4.0, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for text prompt, that will be used to condition the image generation. + image (`PIL.Image.Image`): + `Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will + be masked out with `mask_image` and repainted according to `prompt`. + mask_image (`np.array`): + Tensor representing an image batch, to mask `image`. White pixels in the mask will be repainted, while + black pixels will be preserved. If `mask_image` is a PIL image, it will be converted to a single + channel (luminance) before use. If it's a tensor, it should contain one color channel (L) instead of 3, + so the expected shape would be `(B, H, W, 1)`. + negative_image_embeds (`torch.FloatTensor` or `List[torch.FloatTensor]`): + The clip image embeddings for negative text prompt, will be used to condition the image generation. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + """ + if not self._warn_has_been_called and version.parse(version.parse(__version__).base_version) < version.parse( + "0.23.0.dev0" + ): + logger.warning( + "Please note that the expected format of `mask_image` has recently been changed. " + "Before diffusers == 0.19.0, Kandinsky Inpainting pipelines repainted black pixels and preserved black pixels. " + "As of diffusers==0.19.0 this behavior has been inverted. Now white pixels are repainted and black pixels are preserved. " + "This way, Kandinsky's masking behavior is aligned with Stable Diffusion. " + "THIS means that you HAVE to invert the input mask to have the same behavior as before as explained in https://github.com/huggingface/diffusers/pull/4207. " + "This warning will be surpressed after the first inference call and will be removed in diffusers>0.23.0" + ) + self._warn_has_been_called = True + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + self._guidance_scale = guidance_scale + + device = self._execution_device + + if isinstance(image_embeds, list): + image_embeds = torch.cat(image_embeds, dim=0) + batch_size = image_embeds.shape[0] * num_images_per_prompt + if isinstance(negative_image_embeds, list): + negative_image_embeds = torch.cat(negative_image_embeds, dim=0) + + if self.do_classifier_free_guidance: + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + negative_image_embeds = negative_image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + + image_embeds = torch.cat([negative_image_embeds, image_embeds], dim=0).to( + dtype=self.unet.dtype, device=device + ) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # preprocess image and mask + mask_image, image = prepare_mask_and_masked_image(image, mask_image, height, width) + + image = image.to(dtype=image_embeds.dtype, device=device) + image = self.movq.encode(image)["latents"] + + mask_image = mask_image.to(dtype=image_embeds.dtype, device=device) + + image_shape = tuple(image.shape[-2:]) + mask_image = F.interpolate( + mask_image, + image_shape, + mode="nearest", + ) + mask_image = prepare_mask(mask_image) + masked_image = image * mask_image + + mask_image = mask_image.repeat_interleave(num_images_per_prompt, dim=0) + masked_image = masked_image.repeat_interleave(num_images_per_prompt, dim=0) + if self.do_classifier_free_guidance: + mask_image = mask_image.repeat(2, 1, 1, 1) + masked_image = masked_image.repeat(2, 1, 1, 1) + + num_channels_latents = self.movq.config.latent_channels + + height, width = downscale_height_and_width(height, width, self.movq_scale_factor) + + # create initial latent + latents = self.prepare_latents( + (batch_size, num_channels_latents, height, width), + image_embeds.dtype, + device, + generator, + latents, + self.scheduler, + ) + noise = torch.clone(latents) + + self._num_timesteps = len(timesteps) + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = torch.cat([latent_model_input, masked_image, mask_image], dim=1) + + added_cond_kwargs = {"image_embeds": image_embeds} + noise_pred = self.unet( + sample=latent_model_input, + timestep=t, + encoder_hidden_states=None, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + if self.do_classifier_free_guidance: + noise_pred, variance_pred = noise_pred.split(latents.shape[1], dim=1) + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + _, variance_pred_text = variance_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, variance_pred_text], dim=1) + + if not ( + hasattr(self.scheduler.config, "variance_type") + and self.scheduler.config.variance_type in ["learned", "learned_range"] + ): + noise_pred, _ = noise_pred.split(latents.shape[1], dim=1) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, + t, + latents, + generator=generator, + )[0] + init_latents_proper = image[:1] + init_mask = mask_image[:1] + + if i < len(timesteps) - 1: + noise_timestep = timesteps[i + 1] + init_latents_proper = self.scheduler.add_noise( + init_latents_proper, noise, torch.tensor([noise_timestep]) + ) + + latents = init_mask * init_latents_proper + (1 - init_mask) * latents + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + image_embeds = callback_outputs.pop("image_embeds", image_embeds) + negative_image_embeds = callback_outputs.pop("negative_image_embeds", negative_image_embeds) + masked_image = callback_outputs.pop("masked_image", masked_image) + mask_image = callback_outputs.pop("mask_image", mask_image) + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # post-processing + latents = mask_image[:1] * image[:1] + (1 - mask_image[:1]) * latents + + if output_type not in ["pt", "np", "pil", "latent"]: + raise ValueError( + f"Only the output types `pt`, `pil`, `np` and `latent` are supported not output_type={output_type}" + ) + + if not output_type == "latent": + image = self.movq.decode(latents, force_not_quantize=True)["sample"] + + if output_type in ["np", "pil"]: + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + else: + image = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py new file mode 100755 index 0000000..83427c6 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior.py @@ -0,0 +1,549 @@ +from typing import Callable, Dict, List, Optional, Union + +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...models import PriorTransformer +from ...schedulers import UnCLIPScheduler +from ...utils import ( + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..kandinsky import KandinskyPriorPipelineOutput +from ..pipeline_utils import DiffusionPipeline + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyV22Pipeline, KandinskyV22PriorPipeline + >>> import torch + + >>> pipe_prior = KandinskyV22PriorPipeline.from_pretrained("kandinsky-community/kandinsky-2-2-prior") + >>> pipe_prior.to("cuda") + >>> prompt = "red cat, 4k photo" + >>> image_emb, negative_image_emb = pipe_prior(prompt).to_tuple() + + >>> pipe = KandinskyV22Pipeline.from_pretrained("kandinsky-community/kandinsky-2-2-decoder") + >>> pipe.to("cuda") + >>> image = pipe( + ... image_embeds=image_emb, + ... negative_image_embeds=negative_image_emb, + ... height=768, + ... width=768, + ... num_inference_steps=50, + ... ).images + >>> image[0].save("cat.png") + ``` +""" + +EXAMPLE_INTERPOLATE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyV22PriorPipeline, KandinskyV22Pipeline + >>> from diffusers.utils import load_image + >>> import PIL + >>> import torch + >>> from torchvision import transforms + + >>> pipe_prior = KandinskyV22PriorPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ... ) + >>> pipe_prior.to("cuda") + >>> img1 = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/cat.png" + ... ) + >>> img2 = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/starry_night.jpeg" + ... ) + >>> images_texts = ["a cat", img1, img2] + >>> weights = [0.3, 0.3, 0.4] + >>> out = pipe_prior.interpolate(images_texts, weights) + >>> pipe = KandinskyV22Pipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16 + ... ) + >>> pipe.to("cuda") + >>> image = pipe( + ... image_embeds=out.image_embeds, + ... negative_image_embeds=out.negative_image_embeds, + ... height=768, + ... width=768, + ... num_inference_steps=50, + ... ).images[0] + >>> image.save("starry_cat.png") + ``` +""" + + +class KandinskyV22PriorPipeline(DiffusionPipeline): + """ + Pipeline for generating image prior for Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + prior ([`PriorTransformer`]): + The canonincal unCLIP prior to approximate the image embedding from the text embedding. + image_encoder ([`CLIPVisionModelWithProjection`]): + Frozen image-encoder. + text_encoder ([`CLIPTextModelWithProjection`]): + Frozen text-encoder. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + scheduler ([`UnCLIPScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + image_processor ([`CLIPImageProcessor`]): + A image_processor to be used to preprocess image from clip. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->prior" + _exclude_from_cpu_offload = ["prior"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "text_encoder_hidden_states", "text_mask"] + + def __init__( + self, + prior: PriorTransformer, + image_encoder: CLIPVisionModelWithProjection, + text_encoder: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + scheduler: UnCLIPScheduler, + image_processor: CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + prior=prior, + text_encoder=text_encoder, + tokenizer=tokenizer, + scheduler=scheduler, + image_encoder=image_encoder, + image_processor=image_processor, + ) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_INTERPOLATE_DOC_STRING) + def interpolate( + self, + images_and_prompts: List[Union[str, PIL.Image.Image, torch.FloatTensor]], + weights: List[float], + num_images_per_prompt: int = 1, + num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + negative_prior_prompt: Optional[str] = None, + negative_prompt: str = "", + guidance_scale: float = 4.0, + device=None, + ): + """ + Function invoked when using the prior pipeline for interpolation. + + Args: + images_and_prompts (`List[Union[str, PIL.Image.Image, torch.FloatTensor]]`): + list of prompts and images to guide the image generation. + weights: (`List[float]`): + list of weights for each condition in `images_and_prompts` + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + negative_prior_prompt (`str`, *optional*): + The prompt not to guide the prior diffusion process. Ignored when not using guidance (i.e., ignored if + `guidance_scale` is less than `1`). + negative_prompt (`str` or `List[str]`, *optional*): + The prompt not to guide the image generation. Ignored when not using guidance (i.e., ignored if + `guidance_scale` is less than `1`). + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + + Examples: + + Returns: + [`KandinskyPriorPipelineOutput`] or `tuple` + """ + + device = device or self.device + + if len(images_and_prompts) != len(weights): + raise ValueError( + f"`images_and_prompts` contains {len(images_and_prompts)} items and `weights` contains {len(weights)} items - they should be lists of same length" + ) + + image_embeddings = [] + for cond, weight in zip(images_and_prompts, weights): + if isinstance(cond, str): + image_emb = self( + cond, + num_inference_steps=num_inference_steps, + num_images_per_prompt=num_images_per_prompt, + generator=generator, + latents=latents, + negative_prompt=negative_prior_prompt, + guidance_scale=guidance_scale, + ).image_embeds.unsqueeze(0) + + elif isinstance(cond, (PIL.Image.Image, torch.Tensor)): + if isinstance(cond, PIL.Image.Image): + cond = ( + self.image_processor(cond, return_tensors="pt") + .pixel_values[0] + .unsqueeze(0) + .to(dtype=self.image_encoder.dtype, device=device) + ) + + image_emb = self.image_encoder(cond)["image_embeds"].repeat(num_images_per_prompt, 1).unsqueeze(0) + + else: + raise ValueError( + f"`images_and_prompts` can only contains elements to be of type `str`, `PIL.Image.Image` or `torch.Tensor` but is {type(cond)}" + ) + + image_embeddings.append(image_emb * weight) + + image_emb = torch.cat(image_embeddings).sum(dim=0) + + out_zero = self( + negative_prompt, + num_inference_steps=num_inference_steps, + num_images_per_prompt=num_images_per_prompt, + generator=generator, + latents=latents, + negative_prompt=negative_prior_prompt, + guidance_scale=guidance_scale, + ) + zero_image_emb = out_zero.negative_image_embeds if negative_prompt == "" else out_zero.image_embeds + + return KandinskyPriorPipelineOutput(image_embeds=image_emb, negative_image_embeds=zero_image_emb) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + # Copied from diffusers.pipelines.kandinsky.pipeline_kandinsky_prior.KandinskyPriorPipeline.get_zero_embed + def get_zero_embed(self, batch_size=1, device=None): + device = device or self.device + zero_img = torch.zeros(1, 3, self.image_encoder.config.image_size, self.image_encoder.config.image_size).to( + device=device, dtype=self.image_encoder.dtype + ) + zero_image_emb = self.image_encoder(zero_img)["image_embeds"] + zero_image_emb = zero_image_emb.repeat(batch_size, 1) + return zero_image_emb + + # Copied from diffusers.pipelines.kandinsky.pipeline_kandinsky_prior.KandinskyPriorPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + ): + batch_size = len(prompt) if isinstance(prompt, list) else 1 + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + text_mask = text_inputs.attention_mask.bool().to(device) + + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + text_input_ids = text_input_ids[:, : self.tokenizer.model_max_length] + + text_encoder_output = self.text_encoder(text_input_ids.to(device)) + + prompt_embeds = text_encoder_output.text_embeds + text_encoder_hidden_states = text_encoder_output.last_hidden_state + + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + text_encoder_hidden_states = text_encoder_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + if do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + uncond_text_mask = uncond_input.attention_mask.bool().to(device) + negative_prompt_embeds_text_encoder_output = self.text_encoder(uncond_input.input_ids.to(device)) + + negative_prompt_embeds = negative_prompt_embeds_text_encoder_output.text_embeds + uncond_text_encoder_hidden_states = negative_prompt_embeds_text_encoder_output.last_hidden_state + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len) + + seq_len = uncond_text_encoder_hidden_states.shape[1] + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.repeat(1, num_images_per_prompt, 1) + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + uncond_text_mask = uncond_text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + # done duplicates + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + text_encoder_hidden_states = torch.cat([uncond_text_encoder_hidden_states, text_encoder_hidden_states]) + + text_mask = torch.cat([uncond_text_mask, text_mask]) + + return prompt_embeds, text_encoder_hidden_states, text_mask + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + guidance_scale: float = 4.0, + output_type: Optional[str] = "pt", # pt only + return_dict: bool = True, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + output_type (`str`, *optional*, defaults to `"pt"`): + The output format of the generate image. Choose between: `"np"` (`np.array`) or `"pt"` + (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`KandinskyPriorPipelineOutput`] or `tuple` + """ + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if isinstance(prompt, str): + prompt = [prompt] + elif not isinstance(prompt, list): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + elif not isinstance(negative_prompt, list) and negative_prompt is not None: + raise ValueError(f"`negative_prompt` has to be of type `str` or `list` but is {type(negative_prompt)}") + + # if the negative prompt is defined we double the batch size to + # directly retrieve the negative prompt embedding + if negative_prompt is not None: + prompt = prompt + negative_prompt + negative_prompt = 2 * negative_prompt + + device = self._execution_device + + batch_size = len(prompt) + batch_size = batch_size * num_images_per_prompt + + self._guidance_scale = guidance_scale + + prompt_embeds, text_encoder_hidden_states, text_mask = self._encode_prompt( + prompt, device, num_images_per_prompt, self.do_classifier_free_guidance, negative_prompt + ) + + # prior + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + embedding_dim = self.prior.config.embedding_dim + + latents = self.prepare_latents( + (batch_size, embedding_dim), + prompt_embeds.dtype, + device, + generator, + latents, + self.scheduler, + ) + self._num_timesteps = len(timesteps) + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + predicted_image_embedding = self.prior( + latent_model_input, + timestep=t, + proj_embedding=prompt_embeds, + encoder_hidden_states=text_encoder_hidden_states, + attention_mask=text_mask, + ).predicted_image_embedding + + if self.do_classifier_free_guidance: + predicted_image_embedding_uncond, predicted_image_embedding_text = predicted_image_embedding.chunk(2) + predicted_image_embedding = predicted_image_embedding_uncond + self.guidance_scale * ( + predicted_image_embedding_text - predicted_image_embedding_uncond + ) + + if i + 1 == timesteps.shape[0]: + prev_timestep = None + else: + prev_timestep = timesteps[i + 1] + + latents = self.scheduler.step( + predicted_image_embedding, + timestep=t, + sample=latents, + generator=generator, + prev_timestep=prev_timestep, + ).prev_sample + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + text_encoder_hidden_states = callback_outputs.pop( + "text_encoder_hidden_states", text_encoder_hidden_states + ) + text_mask = callback_outputs.pop("text_mask", text_mask) + + latents = self.prior.post_process_latents(latents) + + image_embeddings = latents + + # if negative prompt has been defined, we retrieve split the image embedding into two + if negative_prompt is None: + zero_embeds = self.get_zero_embed(latents.shape[0], device=latents.device) + else: + image_embeddings, zero_embeds = image_embeddings.chunk(2) + + self.maybe_free_model_hooks() + + if output_type not in ["pt", "np"]: + raise ValueError(f"Only the output types `pt` and `np` are supported not output_type={output_type}") + + if output_type == "np": + image_embeddings = image_embeddings.cpu().numpy() + zero_embeds = zero_embeds.cpu().numpy() + + if not return_dict: + return (image_embeddings, zero_embeds) + + return KandinskyPriorPipelineOutput(image_embeds=image_embeddings, negative_image_embeds=zero_embeds) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py new file mode 100755 index 0000000..bef7082 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky2_2/pipeline_kandinsky2_2_prior_emb2emb.py @@ -0,0 +1,563 @@ +from typing import List, Optional, Union + +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...models import PriorTransformer +from ...schedulers import UnCLIPScheduler +from ...utils import ( + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..kandinsky import KandinskyPriorPipelineOutput +from ..pipeline_utils import DiffusionPipeline + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyV22Pipeline, KandinskyV22PriorEmb2EmbPipeline + >>> import torch + + >>> pipe_prior = KandinskyPriorPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ... ) + >>> pipe_prior.to("cuda") + + >>> prompt = "red cat, 4k photo" + >>> img = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/cat.png" + ... ) + >>> image_emb, nagative_image_emb = pipe_prior(prompt, image=img, strength=0.2).to_tuple() + + >>> pipe = KandinskyPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-decoder, torch_dtype=torch.float16" + ... ) + >>> pipe.to("cuda") + + >>> image = pipe( + ... image_embeds=image_emb, + ... negative_image_embeds=negative_image_emb, + ... height=768, + ... width=768, + ... num_inference_steps=100, + ... ).images + + >>> image[0].save("cat.png") + ``` +""" + +EXAMPLE_INTERPOLATE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import KandinskyV22PriorEmb2EmbPipeline, KandinskyV22Pipeline + >>> from diffusers.utils import load_image + >>> import PIL + + >>> import torch + >>> from torchvision import transforms + + >>> pipe_prior = KandinskyV22PriorPipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ... ) + >>> pipe_prior.to("cuda") + + >>> img1 = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/cat.png" + ... ) + + >>> img2 = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + ... "/kandinsky/starry_night.jpeg" + ... ) + + >>> images_texts = ["a cat", img1, img2] + >>> weights = [0.3, 0.3, 0.4] + >>> image_emb, zero_image_emb = pipe_prior.interpolate(images_texts, weights) + + >>> pipe = KandinskyV22Pipeline.from_pretrained( + ... "kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16 + ... ) + >>> pipe.to("cuda") + + >>> image = pipe( + ... image_embeds=image_emb, + ... negative_image_embeds=zero_image_emb, + ... height=768, + ... width=768, + ... num_inference_steps=150, + ... ).images[0] + + >>> image.save("starry_cat.png") + ``` +""" + + +class KandinskyV22PriorEmb2EmbPipeline(DiffusionPipeline): + """ + Pipeline for generating image prior for Kandinsky + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + prior ([`PriorTransformer`]): + The canonincal unCLIP prior to approximate the image embedding from the text embedding. + image_encoder ([`CLIPVisionModelWithProjection`]): + Frozen image-encoder. + text_encoder ([`CLIPTextModelWithProjection`]): + Frozen text-encoder. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + scheduler ([`UnCLIPScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->prior" + _exclude_from_cpu_offload = ["prior"] + + def __init__( + self, + prior: PriorTransformer, + image_encoder: CLIPVisionModelWithProjection, + text_encoder: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + scheduler: UnCLIPScheduler, + image_processor: CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + prior=prior, + text_encoder=text_encoder, + tokenizer=tokenizer, + scheduler=scheduler, + image_encoder=image_encoder, + image_processor=image_processor, + ) + + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start:] + + return timesteps, num_inference_steps - t_start + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_INTERPOLATE_DOC_STRING) + def interpolate( + self, + images_and_prompts: List[Union[str, PIL.Image.Image, torch.FloatTensor]], + weights: List[float], + num_images_per_prompt: int = 1, + num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + negative_prior_prompt: Optional[str] = None, + negative_prompt: str = "", + guidance_scale: float = 4.0, + device=None, + ): + """ + Function invoked when using the prior pipeline for interpolation. + + Args: + images_and_prompts (`List[Union[str, PIL.Image.Image, torch.FloatTensor]]`): + list of prompts and images to guide the image generation. + weights: (`List[float]`): + list of weights for each condition in `images_and_prompts` + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + negative_prior_prompt (`str`, *optional*): + The prompt not to guide the prior diffusion process. Ignored when not using guidance (i.e., ignored if + `guidance_scale` is less than `1`). + negative_prompt (`str` or `List[str]`, *optional*): + The prompt not to guide the image generation. Ignored when not using guidance (i.e., ignored if + `guidance_scale` is less than `1`). + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + + Examples: + + Returns: + [`KandinskyPriorPipelineOutput`] or `tuple` + """ + + device = device or self.device + + if len(images_and_prompts) != len(weights): + raise ValueError( + f"`images_and_prompts` contains {len(images_and_prompts)} items and `weights` contains {len(weights)} items - they should be lists of same length" + ) + + image_embeddings = [] + for cond, weight in zip(images_and_prompts, weights): + if isinstance(cond, str): + image_emb = self( + cond, + num_inference_steps=num_inference_steps, + num_images_per_prompt=num_images_per_prompt, + generator=generator, + latents=latents, + negative_prompt=negative_prior_prompt, + guidance_scale=guidance_scale, + ).image_embeds.unsqueeze(0) + + elif isinstance(cond, (PIL.Image.Image, torch.Tensor)): + image_emb = self._encode_image( + cond, device=device, num_images_per_prompt=num_images_per_prompt + ).unsqueeze(0) + + else: + raise ValueError( + f"`images_and_prompts` can only contains elements to be of type `str`, `PIL.Image.Image` or `torch.Tensor` but is {type(cond)}" + ) + + image_embeddings.append(image_emb * weight) + + image_emb = torch.cat(image_embeddings).sum(dim=0) + + return KandinskyPriorPipelineOutput(image_embeds=image_emb, negative_image_embeds=torch.randn_like(image_emb)) + + def _encode_image( + self, + image: Union[torch.Tensor, List[PIL.Image.Image]], + device, + num_images_per_prompt, + ): + if not isinstance(image, torch.Tensor): + image = self.image_processor(image, return_tensors="pt").pixel_values.to( + dtype=self.image_encoder.dtype, device=device + ) + + image_emb = self.image_encoder(image)["image_embeds"] # B, D + image_emb = image_emb.repeat_interleave(num_images_per_prompt, dim=0) + image_emb.to(device=device) + + return image_emb + + def prepare_latents(self, emb, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): + emb = emb.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + init_latents = emb + + if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0: + additional_image_per_prompt = batch_size // init_latents.shape[0] + init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0) + elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts." + ) + else: + init_latents = torch.cat([init_latents], dim=0) + + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + latents = init_latents + + return latents + + # Copied from diffusers.pipelines.kandinsky.pipeline_kandinsky_prior.KandinskyPriorPipeline.get_zero_embed + def get_zero_embed(self, batch_size=1, device=None): + device = device or self.device + zero_img = torch.zeros(1, 3, self.image_encoder.config.image_size, self.image_encoder.config.image_size).to( + device=device, dtype=self.image_encoder.dtype + ) + zero_image_emb = self.image_encoder(zero_img)["image_embeds"] + zero_image_emb = zero_image_emb.repeat(batch_size, 1) + return zero_image_emb + + # Copied from diffusers.pipelines.kandinsky.pipeline_kandinsky_prior.KandinskyPriorPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + ): + batch_size = len(prompt) if isinstance(prompt, list) else 1 + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + text_mask = text_inputs.attention_mask.bool().to(device) + + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + text_input_ids = text_input_ids[:, : self.tokenizer.model_max_length] + + text_encoder_output = self.text_encoder(text_input_ids.to(device)) + + prompt_embeds = text_encoder_output.text_embeds + text_encoder_hidden_states = text_encoder_output.last_hidden_state + + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + text_encoder_hidden_states = text_encoder_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + if do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + uncond_text_mask = uncond_input.attention_mask.bool().to(device) + negative_prompt_embeds_text_encoder_output = self.text_encoder(uncond_input.input_ids.to(device)) + + negative_prompt_embeds = negative_prompt_embeds_text_encoder_output.text_embeds + uncond_text_encoder_hidden_states = negative_prompt_embeds_text_encoder_output.last_hidden_state + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len) + + seq_len = uncond_text_encoder_hidden_states.shape[1] + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.repeat(1, num_images_per_prompt, 1) + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + uncond_text_mask = uncond_text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + # done duplicates + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + text_encoder_hidden_states = torch.cat([uncond_text_encoder_hidden_states, text_encoder_hidden_states]) + + text_mask = torch.cat([uncond_text_mask, text_mask]) + + return prompt_embeds, text_encoder_hidden_states, text_mask + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + image: Union[torch.Tensor, List[torch.Tensor], PIL.Image.Image, List[PIL.Image.Image]], + strength: float = 0.3, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + guidance_scale: float = 4.0, + output_type: Optional[str] = "pt", # pt only + return_dict: bool = True, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + strength (`float`, *optional*, defaults to 0.8): + Conceptually, indicates how much to transform the reference `emb`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. + emb (`torch.FloatTensor`): + The image embedding. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + output_type (`str`, *optional*, defaults to `"pt"`): + The output format of the generate image. Choose between: `"np"` (`np.array`) or `"pt"` + (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Examples: + + Returns: + [`KandinskyPriorPipelineOutput`] or `tuple` + """ + + if isinstance(prompt, str): + prompt = [prompt] + elif not isinstance(prompt, list): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + elif not isinstance(negative_prompt, list) and negative_prompt is not None: + raise ValueError(f"`negative_prompt` has to be of type `str` or `list` but is {type(negative_prompt)}") + + # if the negative prompt is defined we double the batch size to + # directly retrieve the negative prompt embedding + if negative_prompt is not None: + prompt = prompt + negative_prompt + negative_prompt = 2 * negative_prompt + + device = self._execution_device + + batch_size = len(prompt) + batch_size = batch_size * num_images_per_prompt + + do_classifier_free_guidance = guidance_scale > 1.0 + prompt_embeds, text_encoder_hidden_states, text_mask = self._encode_prompt( + prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + + if not isinstance(image, List): + image = [image] + + if isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + + if isinstance(image, torch.Tensor) and image.ndim == 2: + # allow user to pass image_embeds directly + image_embeds = image.repeat_interleave(num_images_per_prompt, dim=0) + elif isinstance(image, torch.Tensor) and image.ndim != 4: + raise ValueError( + f" if pass `image` as pytorch tensor, or a list of pytorch tensor, please make sure each tensor has shape [batch_size, channels, height, width], currently {image[0].unsqueeze(0).shape}" + ) + else: + image_embeds = self._encode_image(image, device, num_images_per_prompt) + + # prior + self.scheduler.set_timesteps(num_inference_steps, device=device) + + latents = image_embeds + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size) + latents = self.prepare_latents( + latents, + latent_timestep, + batch_size // num_images_per_prompt, + num_images_per_prompt, + prompt_embeds.dtype, + device, + generator, + ) + + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + predicted_image_embedding = self.prior( + latent_model_input, + timestep=t, + proj_embedding=prompt_embeds, + encoder_hidden_states=text_encoder_hidden_states, + attention_mask=text_mask, + ).predicted_image_embedding + + if do_classifier_free_guidance: + predicted_image_embedding_uncond, predicted_image_embedding_text = predicted_image_embedding.chunk(2) + predicted_image_embedding = predicted_image_embedding_uncond + guidance_scale * ( + predicted_image_embedding_text - predicted_image_embedding_uncond + ) + + if i + 1 == timesteps.shape[0]: + prev_timestep = None + else: + prev_timestep = timesteps[i + 1] + + latents = self.scheduler.step( + predicted_image_embedding, + timestep=t, + sample=latents, + generator=generator, + prev_timestep=prev_timestep, + ).prev_sample + + latents = self.prior.post_process_latents(latents) + + image_embeddings = latents + + # if negative prompt has been defined, we retrieve split the image embedding into two + if negative_prompt is None: + zero_embeds = self.get_zero_embed(latents.shape[0], device=latents.device) + else: + image_embeddings, zero_embeds = image_embeddings.chunk(2) + + self.maybe_free_model_hooks() + + if output_type not in ["pt", "np"]: + raise ValueError(f"Only the output types `pt` and `np` are supported not output_type={output_type}") + + if output_type == "np": + image_embeddings = image_embeddings.cpu().numpy() + zero_embeds = zero_embeds.cpu().numpy() + + if not return_dict: + return (image_embeddings, zero_embeds) + + return KandinskyPriorPipelineOutput(image_embeds=image_embeddings, negative_image_embeds=zero_embeds) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/__init__.py new file mode 100755 index 0000000..e8a3063 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/__init__.py @@ -0,0 +1,49 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_kandinsky3"] = ["Kandinsky3Pipeline"] + _import_structure["pipeline_kandinsky3_img2img"] = ["Kandinsky3Img2ImgPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_kandinsky3 import Kandinsky3Pipeline + from .pipeline_kandinsky3_img2img import Kandinsky3Img2ImgPipeline +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/convert_kandinsky3_unet.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/convert_kandinsky3_unet.py new file mode 100755 index 0000000..4fe8c54 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/convert_kandinsky3_unet.py @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 +import argparse +import fnmatch + +from safetensors.torch import load_file + +from diffusers import Kandinsky3UNet + + +MAPPING = { + "to_time_embed.1": "time_embedding.linear_1", + "to_time_embed.3": "time_embedding.linear_2", + "in_layer": "conv_in", + "out_layer.0": "conv_norm_out", + "out_layer.2": "conv_out", + "down_samples": "down_blocks", + "up_samples": "up_blocks", + "projection_lin": "encoder_hid_proj.projection_linear", + "projection_ln": "encoder_hid_proj.projection_norm", + "feature_pooling": "add_time_condition", + "to_query": "to_q", + "to_key": "to_k", + "to_value": "to_v", + "output_layer": "to_out.0", + "self_attention_block": "attentions.0", +} + +DYNAMIC_MAP = { + "resnet_attn_blocks.*.0": "resnets_in.*", + "resnet_attn_blocks.*.1": ("attentions.*", 1), + "resnet_attn_blocks.*.2": "resnets_out.*", +} +# MAPPING = {} + + +def convert_state_dict(unet_state_dict): + """ + Convert the state dict of a U-Net model to match the key format expected by Kandinsky3UNet model. + Args: + unet_model (torch.nn.Module): The original U-Net model. + unet_kandi3_model (torch.nn.Module): The Kandinsky3UNet model to match keys with. + + Returns: + OrderedDict: The converted state dictionary. + """ + # Example of renaming logic (this will vary based on your model's architecture) + converted_state_dict = {} + for key in unet_state_dict: + new_key = key + for pattern, new_pattern in MAPPING.items(): + new_key = new_key.replace(pattern, new_pattern) + + for dyn_pattern, dyn_new_pattern in DYNAMIC_MAP.items(): + has_matched = False + if fnmatch.fnmatch(new_key, f"*.{dyn_pattern}.*") and not has_matched: + star = int(new_key.split(dyn_pattern.split(".")[0])[-1].split(".")[1]) + + if isinstance(dyn_new_pattern, tuple): + new_star = star + dyn_new_pattern[-1] + dyn_new_pattern = dyn_new_pattern[0] + else: + new_star = star + + pattern = dyn_pattern.replace("*", str(star)) + new_pattern = dyn_new_pattern.replace("*", str(new_star)) + + new_key = new_key.replace(pattern, new_pattern) + has_matched = True + + converted_state_dict[new_key] = unet_state_dict[key] + + return converted_state_dict + + +def main(model_path, output_path): + # Load your original U-Net model + unet_state_dict = load_file(model_path) + + # Initialize your Kandinsky3UNet model + config = {} + + # Convert the state dict + converted_state_dict = convert_state_dict(unet_state_dict) + + unet = Kandinsky3UNet(config) + unet.load_state_dict(converted_state_dict) + + unet.save_pretrained(output_path) + print(f"Converted model saved to {output_path}") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Convert U-Net PyTorch model to Kandinsky3UNet format") + parser.add_argument("--model_path", type=str, required=True, help="Path to the original U-Net PyTorch model") + parser.add_argument("--output_path", type=str, required=True, help="Path to save the converted model") + + args = parser.parse_args() + main(args.model_path, args.output_path) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py new file mode 100755 index 0000000..fcf7ddc --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3.py @@ -0,0 +1,589 @@ +from typing import Callable, Dict, List, Optional, Union + +import torch +from transformers import T5EncoderModel, T5Tokenizer + +from ...loaders import LoraLoaderMixin +from ...models import Kandinsky3UNet, VQModel +from ...schedulers import DDPMScheduler +from ...utils import ( + deprecate, + is_accelerate_available, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import AutoPipelineForText2Image + >>> import torch + + >>> pipe = AutoPipelineForText2Image.from_pretrained("kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16) + >>> pipe.enable_model_cpu_offload() + + >>> prompt = "A photograph of the inside of a subway train. There are raccoons sitting on the seats. One of them is reading a newspaper. The window shows the city in the background." + + >>> generator = torch.Generator(device="cpu").manual_seed(0) + >>> image = pipe(prompt, num_inference_steps=25, generator=generator).images[0] + ``` + +""" + + +def downscale_height_and_width(height, width, scale_factor=8): + new_height = height // scale_factor**2 + if height % scale_factor**2 != 0: + new_height += 1 + new_width = width // scale_factor**2 + if width % scale_factor**2 != 0: + new_width += 1 + return new_height * scale_factor, new_width * scale_factor + + +class Kandinsky3Pipeline(DiffusionPipeline, LoraLoaderMixin): + model_cpu_offload_seq = "text_encoder->unet->movq" + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + "negative_attention_mask", + "attention_mask", + ] + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + unet: Kandinsky3UNet, + scheduler: DDPMScheduler, + movq: VQModel, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, unet=unet, scheduler=scheduler, movq=movq + ) + + def remove_all_hooks(self): + if is_accelerate_available(): + from accelerate.hooks import remove_hook_from_module + else: + raise ImportError("Please install accelerate via `pip install accelerate`") + + for model in [self.text_encoder, self.unet, self.movq]: + if model is not None: + remove_hook_from_module(model, recurse=True) + + self.unet_offload_hook = None + self.text_encoder_offload_hook = None + self.final_offload_hook = None + + def process_embeds(self, embeddings, attention_mask, cut_context): + if cut_context: + embeddings[attention_mask == 0] = torch.zeros_like(embeddings[attention_mask == 0]) + max_seq_length = attention_mask.sum(-1).max() + 1 + embeddings = embeddings[:, :max_seq_length] + attention_mask = attention_mask[:, :max_seq_length] + return embeddings, attention_mask + + @torch.no_grad() + def encode_prompt( + self, + prompt, + do_classifier_free_guidance=True, + num_images_per_prompt=1, + device=None, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + _cut_context=False, + attention_mask: Optional[torch.FloatTensor] = None, + negative_attention_mask: Optional[torch.FloatTensor] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`, *optional*): + torch device to place the resulting embeddings on + num_images_per_prompt (`int`, *optional*, defaults to 1): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. + Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + attention_mask (`torch.FloatTensor`, *optional*): + Pre-generated attention mask. Must provide if passing `prompt_embeds` directly. + negative_attention_mask (`torch.FloatTensor`, *optional*): + Pre-generated negative attention mask. Must provide if passing `negative_prompt_embeds` directly. + """ + if prompt is not None and negative_prompt is not None: + if type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + + if device is None: + device = self._execution_device + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + max_length = 128 + + if prompt_embeds is None: + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids.to(device) + attention_mask = text_inputs.attention_mask.to(device) + prompt_embeds = self.text_encoder( + text_input_ids, + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + prompt_embeds, attention_mask = self.process_embeds(prompt_embeds, attention_mask, _cut_context) + prompt_embeds = prompt_embeds * attention_mask.unsqueeze(2) + + if self.text_encoder is not None: + dtype = self.text_encoder.dtype + else: + dtype = None + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + attention_mask = attention_mask.repeat(num_images_per_prompt, 1) + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + if negative_prompt is not None: + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=128, + truncation=True, + return_attention_mask=True, + return_tensors="pt", + ) + text_input_ids = uncond_input.input_ids.to(device) + negative_attention_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds = self.text_encoder( + text_input_ids, + attention_mask=negative_attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds[:, : prompt_embeds.shape[1]] + negative_attention_mask = negative_attention_mask[:, : prompt_embeds.shape[1]] + negative_prompt_embeds = negative_prompt_embeds * negative_attention_mask.unsqueeze(2) + + else: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_attention_mask = torch.zeros_like(attention_mask) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device) + if negative_prompt_embeds.shape != prompt_embeds.shape: + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + negative_attention_mask = negative_attention_mask.repeat(num_images_per_prompt, 1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + else: + negative_prompt_embeds = None + negative_attention_mask = None + return prompt_embeds, negative_prompt_embeds, attention_mask, negative_attention_mask + + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def check_inputs( + self, + prompt, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + attention_mask=None, + negative_attention_mask=None, + ): + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + if negative_prompt_embeds is not None and negative_attention_mask is None: + raise ValueError("Please provide `negative_attention_mask` along with `negative_prompt_embeds`") + + if negative_prompt_embeds is not None and negative_attention_mask is not None: + if negative_prompt_embeds.shape[:2] != negative_attention_mask.shape: + raise ValueError( + "`negative_prompt_embeds` and `negative_attention_mask` must have the same batch_size and token length when passed directly, but" + f" got: `negative_prompt_embeds` {negative_prompt_embeds.shape[:2]} != `negative_attention_mask`" + f" {negative_attention_mask.shape}." + ) + + if prompt_embeds is not None and attention_mask is None: + raise ValueError("Please provide `attention_mask` along with `prompt_embeds`") + + if prompt_embeds is not None and attention_mask is not None: + if prompt_embeds.shape[:2] != attention_mask.shape: + raise ValueError( + "`prompt_embeds` and `attention_mask` must have the same batch_size and token length when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape[:2]} != `attention_mask`" + f" {attention_mask.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + num_inference_steps: int = 25, + guidance_scale: float = 3.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + height: Optional[int] = 1024, + width: Optional[int] = 1024, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + negative_attention_mask: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + latents=None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + num_inference_steps (`int`, *optional*, defaults to 25): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps are used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 3.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + height (`int`, *optional*, defaults to self.unet.config.sample_size): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size): + The width in pixels of the generated image. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + attention_mask (`torch.FloatTensor`, *optional*): + Pre-generated attention mask. Must provide if passing `prompt_embeds` directly. + negative_attention_mask (`torch.FloatTensor`, *optional*): + Pre-generated negative attention mask. Must provide if passing `negative_prompt_embeds` directly. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + clean_caption (`bool`, *optional*, defaults to `True`): + Whether or not to clean the caption before creating embeddings. Requires `beautifulsoup4` and `ftfy` to + be installed. If the dependencies are not installed, the embeddings will be created from the raw + prompt. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + cut_context = True + device = self._execution_device + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + callback_on_step_end_tensor_inputs, + attention_mask, + negative_attention_mask, + ) + + self._guidance_scale = guidance_scale + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds, attention_mask, negative_attention_mask = self.encode_prompt( + prompt, + self.do_classifier_free_guidance, + num_images_per_prompt=num_images_per_prompt, + device=device, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + _cut_context=cut_context, + attention_mask=attention_mask, + negative_attention_mask=negative_attention_mask, + ) + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + attention_mask = torch.cat([negative_attention_mask, attention_mask]).bool() + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latents + height, width = downscale_height_and_width(height, width, 8) + + latents = self.prepare_latents( + (batch_size * num_images_per_prompt, 4, height, width), + prompt_embeds.dtype, + device, + generator, + latents, + self.scheduler, + ) + + if hasattr(self, "text_encoder_offload_hook") and self.text_encoder_offload_hook is not None: + self.text_encoder_offload_hook.offload() + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + encoder_attention_mask=attention_mask, + return_dict=False, + )[0] + + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + + noise_pred = (guidance_scale + 1.0) * noise_pred_text - guidance_scale * noise_pred_uncond + # noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, + t, + latents, + generator=generator, + ).prev_sample + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + attention_mask = callback_outputs.pop("attention_mask", attention_mask) + negative_attention_mask = callback_outputs.pop("negative_attention_mask", negative_attention_mask) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # post-processing + if output_type not in ["pt", "np", "pil", "latent"]: + raise ValueError( + f"Only the output types `pt`, `pil`, `np` and `latent` are supported not output_type={output_type}" + ) + + if not output_type == "latent": + image = self.movq.decode(latents, force_not_quantize=True)["sample"] + + if output_type in ["np", "pil"]: + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + else: + image = latents + + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py new file mode 100755 index 0000000..7f4164a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/kandinsky3/pipeline_kandinsky3_img2img.py @@ -0,0 +1,654 @@ +import inspect +from typing import Callable, Dict, List, Optional, Union + +import numpy as np +import PIL +import PIL.Image +import torch +from transformers import T5EncoderModel, T5Tokenizer + +from ...loaders import LoraLoaderMixin +from ...models import Kandinsky3UNet, VQModel +from ...schedulers import DDPMScheduler +from ...utils import ( + deprecate, + is_accelerate_available, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import AutoPipelineForImage2Image + >>> from diffusers.utils import load_image + >>> import torch + + >>> pipe = AutoPipelineForImage2Image.from_pretrained("kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16) + >>> pipe.enable_model_cpu_offload() + + >>> prompt = "A painting of the inside of a subway train with tiny raccoons." + >>> image = load_image("https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky3/t2i.png") + + >>> generator = torch.Generator(device="cpu").manual_seed(0) + >>> image = pipe(prompt, image=image, strength=0.75, num_inference_steps=25, generator=generator).images[0] + ``` +""" + + +def downscale_height_and_width(height, width, scale_factor=8): + new_height = height // scale_factor**2 + if height % scale_factor**2 != 0: + new_height += 1 + new_width = width // scale_factor**2 + if width % scale_factor**2 != 0: + new_width += 1 + return new_height * scale_factor, new_width * scale_factor + + +def prepare_image(pil_image): + arr = np.array(pil_image.convert("RGB")) + arr = arr.astype(np.float32) / 127.5 - 1 + arr = np.transpose(arr, [2, 0, 1]) + image = torch.from_numpy(arr).unsqueeze(0) + return image + + +class Kandinsky3Img2ImgPipeline(DiffusionPipeline, LoraLoaderMixin): + model_cpu_offload_seq = "text_encoder->movq->unet->movq" + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + "negative_attention_mask", + "attention_mask", + ] + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + unet: Kandinsky3UNet, + scheduler: DDPMScheduler, + movq: VQModel, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, unet=unet, scheduler=scheduler, movq=movq + ) + + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start:] + + return timesteps, num_inference_steps - t_start + + def remove_all_hooks(self): + if is_accelerate_available(): + from accelerate.hooks import remove_hook_from_module + else: + raise ImportError("Please install accelerate via `pip install accelerate`") + + for model in [self.text_encoder, self.unet]: + if model is not None: + remove_hook_from_module(model, recurse=True) + + self.unet_offload_hook = None + self.text_encoder_offload_hook = None + self.final_offload_hook = None + + def _process_embeds(self, embeddings, attention_mask, cut_context): + # return embeddings, attention_mask + if cut_context: + embeddings[attention_mask == 0] = torch.zeros_like(embeddings[attention_mask == 0]) + max_seq_length = attention_mask.sum(-1).max() + 1 + embeddings = embeddings[:, :max_seq_length] + attention_mask = attention_mask[:, :max_seq_length] + return embeddings, attention_mask + + @torch.no_grad() + def encode_prompt( + self, + prompt, + do_classifier_free_guidance=True, + num_images_per_prompt=1, + device=None, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + _cut_context=False, + attention_mask: Optional[torch.FloatTensor] = None, + negative_attention_mask: Optional[torch.FloatTensor] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`, *optional*): + torch device to place the resulting embeddings on + num_images_per_prompt (`int`, *optional*, defaults to 1): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. + Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + attention_mask (`torch.FloatTensor`, *optional*): + Pre-generated attention mask. Must provide if passing `prompt_embeds` directly. + negative_attention_mask (`torch.FloatTensor`, *optional*): + Pre-generated negative attention mask. Must provide if passing `negative_prompt_embeds` directly. + """ + if prompt is not None and negative_prompt is not None: + if type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + + if device is None: + device = self._execution_device + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + max_length = 128 + + if prompt_embeds is None: + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids.to(device) + attention_mask = text_inputs.attention_mask.to(device) + prompt_embeds = self.text_encoder( + text_input_ids, + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + prompt_embeds, attention_mask = self._process_embeds(prompt_embeds, attention_mask, _cut_context) + prompt_embeds = prompt_embeds * attention_mask.unsqueeze(2) + + if self.text_encoder is not None: + dtype = self.text_encoder.dtype + else: + dtype = None + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + attention_mask = attention_mask.repeat(num_images_per_prompt, 1) + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + if negative_prompt is not None: + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=128, + truncation=True, + return_attention_mask=True, + return_tensors="pt", + ) + text_input_ids = uncond_input.input_ids.to(device) + negative_attention_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds = self.text_encoder( + text_input_ids, + attention_mask=negative_attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds[:, : prompt_embeds.shape[1]] + negative_attention_mask = negative_attention_mask[:, : prompt_embeds.shape[1]] + negative_prompt_embeds = negative_prompt_embeds * negative_attention_mask.unsqueeze(2) + + else: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_attention_mask = torch.zeros_like(attention_mask) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device) + if negative_prompt_embeds.shape != prompt_embeds.shape: + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + negative_attention_mask = negative_attention_mask.repeat(num_images_per_prompt, 1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + else: + negative_prompt_embeds = None + negative_attention_mask = None + return prompt_embeds, negative_prompt_embeds, attention_mask, negative_attention_mask + + def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + init_latents = image + + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + elif isinstance(generator, list): + init_latents = [ + self.movq.encode(image[i : i + 1]).latent_dist.sample(generator[i]) for i in range(batch_size) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = self.movq.encode(image).latent_dist.sample(generator) + + init_latents = self.movq.config.scaling_factor * init_latents + + init_latents = torch.cat([init_latents], dim=0) + + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + + latents = init_latents + + return latents + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + attention_mask=None, + negative_attention_mask=None, + ): + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if negative_prompt_embeds is not None and negative_attention_mask is None: + raise ValueError("Please provide `negative_attention_mask` along with `negative_prompt_embeds`") + + if negative_prompt_embeds is not None and negative_attention_mask is not None: + if negative_prompt_embeds.shape[:2] != negative_attention_mask.shape: + raise ValueError( + "`negative_prompt_embeds` and `negative_attention_mask` must have the same batch_size and token length when passed directly, but" + f" got: `negative_prompt_embeds` {negative_prompt_embeds.shape[:2]} != `negative_attention_mask`" + f" {negative_attention_mask.shape}." + ) + + if prompt_embeds is not None and attention_mask is None: + raise ValueError("Please provide `attention_mask` along with `prompt_embeds`") + + if prompt_embeds is not None and attention_mask is not None: + if prompt_embeds.shape[:2] != attention_mask.shape: + raise ValueError( + "`prompt_embeds` and `attention_mask` must have the same batch_size and token length when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape[:2]} != `attention_mask`" + f" {attention_mask.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: Union[torch.FloatTensor, PIL.Image.Image, List[torch.FloatTensor], List[PIL.Image.Image]] = None, + strength: float = 0.3, + num_inference_steps: int = 25, + guidance_scale: float = 3.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + attention_mask: Optional[torch.FloatTensor] = None, + negative_attention_mask: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. + strength (`float`, *optional*, defaults to 0.8): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 3.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + attention_mask (`torch.FloatTensor`, *optional*): + Pre-generated attention mask. Must provide if passing `prompt_embeds` directly. + negative_attention_mask (`torch.FloatTensor`, *optional*): + Pre-generated negative attention mask. Must provide if passing `negative_prompt_embeds` directly. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` + + """ + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + cut_context = True + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + callback_on_step_end_tensor_inputs, + attention_mask, + negative_attention_mask, + ) + + self._guidance_scale = guidance_scale + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds, attention_mask, negative_attention_mask = self.encode_prompt( + prompt, + self.do_classifier_free_guidance, + num_images_per_prompt=num_images_per_prompt, + device=device, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + _cut_context=cut_context, + attention_mask=attention_mask, + negative_attention_mask=negative_attention_mask, + ) + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + attention_mask = torch.cat([negative_attention_mask, attention_mask]).bool() + if not isinstance(image, list): + image = [image] + if not all(isinstance(i, (PIL.Image.Image, torch.Tensor)) for i in image): + raise ValueError( + f"Input is in incorrect format: {[type(i) for i in image]}. Currently, we only support PIL image and pytorch tensor" + ) + + image = torch.cat([prepare_image(i) for i in image], dim=0) + image = image.to(dtype=prompt_embeds.dtype, device=device) + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + # 5. Prepare latents + latents = self.movq.encode(image)["latents"] + latents = latents.repeat_interleave(num_images_per_prompt, dim=0) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + latents = self.prepare_latents( + latents, latent_timestep, batch_size, num_images_per_prompt, prompt_embeds.dtype, device, generator + ) + if hasattr(self, "text_encoder_offload_hook") and self.text_encoder_offload_hook is not None: + self.text_encoder_offload_hook.offload() + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + encoder_attention_mask=attention_mask, + )[0] + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + + noise_pred = (guidance_scale + 1.0) * noise_pred_text - guidance_scale * noise_pred_uncond + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + noise_pred, + t, + latents, + generator=generator, + ).prev_sample + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + attention_mask = callback_outputs.pop("attention_mask", attention_mask) + negative_attention_mask = callback_outputs.pop("negative_attention_mask", negative_attention_mask) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # post-processing + if output_type not in ["pt", "np", "pil", "latent"]: + raise ValueError( + f"Only the output types `pt`, `pil`, `np` and `latent` are supported not output_type={output_type}" + ) + if not output_type == "latent": + image = self.movq.decode(latents, force_not_quantize=True)["sample"] + + if output_type in ["np", "pil"]: + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + else: + image = latents + + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/__init__.py new file mode 100755 index 0000000..8f79d3c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/__init__.py @@ -0,0 +1,50 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_latent_consistency_img2img"] = ["LatentConsistencyModelImg2ImgPipeline"] + _import_structure["pipeline_latent_consistency_text2img"] = ["LatentConsistencyModelPipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_latent_consistency_img2img import LatentConsistencyModelImg2ImgPipeline + from .pipeline_latent_consistency_text2img import LatentConsistencyModelPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py new file mode 100755 index 0000000..f64854e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_img2img.py @@ -0,0 +1,956 @@ +# Copyright 2024 Stanford University Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion +# and https://github.com/hojonathanho/diffusion + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import LCMScheduler +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput, StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import AutoPipelineForImage2Image + >>> import torch + >>> import PIL + + >>> pipe = AutoPipelineForImage2Image.from_pretrained("SimianLuo/LCM_Dreamshaper_v7") + >>> # To save GPU memory, torch.float16 can be used, but it may compromise image quality. + >>> pipe.to(torch_device="cuda", torch_dtype=torch.float32) + + >>> prompt = "High altitude snowy mountains" + >>> image = PIL.Image.open("./snowy_mountains.png") + + >>> # Can be set to 1~50 steps. LCM support fast inference even <= 4 steps. Recommend: 1~8 steps. + >>> num_inference_steps = 4 + >>> images = pipe( + ... prompt=prompt, image=image, num_inference_steps=num_inference_steps, guidance_scale=8.0 + ... ).images + + >>> images[0].save("image.png") + ``` + +""" + + +class LatentConsistencyModelImg2ImgPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, + LoraLoaderMixin, + FromSingleFileMixin, +): + r""" + Pipeline for image-to-image generation using a latent consistency model. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Currently only + supports [`LCMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + requires_safety_checker (`bool`, *optional*, defaults to `True`): + Whether the pipeline requires a safety checker component. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "denoised", "prompt_embeds", "w_embedding"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: LCMScheduler, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: Optional[CLIPVisionModelWithProjection] = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.prepare_latents + def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + init_latents = image + + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + elif isinstance(generator, list): + init_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(batch_size) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + init_latents = self.vae.config.scaling_factor * init_latents + + if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0: + # expand init_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {init_latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_image_per_prompt = batch_size // init_latents.shape[0] + init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0) + elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts." + ) + else: + init_latents = torch.cat([init_latents], dim=0) + + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + latents = init_latents + + return latents + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + def check_inputs( + self, + prompt: Union[str, List[str]], + strength: float, + callback_steps: int, + prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def clip_skip(self): + return self._clip_skip + + @property + def do_classifier_free_guidance(self): + return False + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + num_inference_steps: int = 4, + strength: float = 0.8, + original_inference_steps: int = None, + timesteps: List[int] = None, + guidance_scale: float = 8.5, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + original_inference_steps (`int`, *optional*): + The original number of inference steps use to generate a linearly-spaced timestep schedule, from which + we will draw `num_inference_steps` evenly spaced timesteps from as our final timestep schedule, + following the Skipping-Step method in the paper (see Section 4.3). If not set this will default to the + scheduler's `original_inference_steps` attribute. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps on the original LCM training/distillation timestep schedule are used. Must be in descending + order. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + Note that the original latent consistency models paper uses a different CFG formulation where the + guidance scales are decreased by 1 (so in the paper formulation CFG is enabled when `guidance_scale > + 0`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): + Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + strength, + callback_steps, + prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 3. Encode input prompt + lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + + # NOTE: when a LCM is distilled from an LDM via latent consistency distillation (Algorithm 1) with guided + # distillation, the forward pass of the LCM learns to approximate sampling from the LDM using CFG with the + # unconditional prompt "" (the empty string). Due to this, LCMs currently do not support negative prompts. + prompt_embeds, _ = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=None, + lora_scale=lora_scale, + clip_skip=self.clip_skip, + ) + + # 4. Encode image + image = self.image_processor.preprocess(image) + + # 5. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, + num_inference_steps, + device, + timesteps, + original_inference_steps=original_inference_steps, + strength=strength, + ) + + # 6. Prepare latent variables + original_inference_steps = ( + original_inference_steps + if original_inference_steps is not None + else self.scheduler.config.original_inference_steps + ) + latent_timestep = timesteps[:1] + latents = self.prepare_latents( + image, latent_timestep, batch_size, num_images_per_prompt, prompt_embeds.dtype, device, generator + ) + bs = batch_size * num_images_per_prompt + + # 6. Get Guidance Scale Embedding + # NOTE: We use the Imagen CFG formulation that StableDiffusionPipeline uses rather than the original LCM paper + # CFG formulation, so we need to subtract 1 from the input guidance_scale. + # LCM CFG formulation: cfg_noise = noise_cond + cfg_scale * (noise_cond - noise_uncond), (cfg_scale > 0.0 using CFG) + w = torch.tensor(self.guidance_scale - 1).repeat(bs) + w_embedding = self.get_guidance_scale_embedding(w, embedding_dim=self.unet.config.time_cond_proj_dim).to( + device=device, dtype=latents.dtype + ) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, None) + + # 7.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + # 8. LCM Multistep Sampling Loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + latents = latents.to(prompt_embeds.dtype) + + # model prediction (v-prediction, eps, x) + model_pred = self.unet( + latents, + t, + timestep_cond=w_embedding, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # compute the previous noisy sample x_t -> x_t-1 + latents, denoised = self.scheduler.step(model_pred, t, latents, **extra_step_kwargs, return_dict=False) + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + w_embedding = callback_outputs.pop("w_embedding", w_embedding) + denoised = callback_outputs.pop("denoised", denoised) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + denoised = denoised.to(prompt_embeds.dtype) + if not output_type == "latent": + image = self.vae.decode(denoised / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = denoised + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py b/diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py new file mode 100755 index 0000000..e9bacaa --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/latent_consistency_models/pipeline_latent_consistency_text2img.py @@ -0,0 +1,888 @@ +# Copyright 2024 Stanford University Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion +# and https://github.com/hojonathanho/diffusion + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import LCMScheduler +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput, StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import DiffusionPipeline + >>> import torch + + >>> pipe = DiffusionPipeline.from_pretrained("SimianLuo/LCM_Dreamshaper_v7") + >>> # To save GPU memory, torch.float16 can be used, but it may compromise image quality. + >>> pipe.to(torch_device="cuda", torch_dtype=torch.float32) + + >>> prompt = "Self-portrait oil painting, a beautiful cyborg with golden hair, 8k" + + >>> # Can be set to 1~50 steps. LCM support fast inference even <= 4 steps. Recommend: 1~8 steps. + >>> num_inference_steps = 4 + >>> images = pipe(prompt=prompt, num_inference_steps=num_inference_steps, guidance_scale=8.0).images + >>> images[0].save("image.png") + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class LatentConsistencyModelPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, + LoraLoaderMixin, + FromSingleFileMixin, +): + r""" + Pipeline for text-to-image generation using a latent consistency model. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Currently only + supports [`LCMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + requires_safety_checker (`bool`, *optional*, defaults to `True`): + Whether the pipeline requires a safety checker component. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "denoised", "prompt_embeds", "w_embedding"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: LCMScheduler, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: Optional[CLIPVisionModelWithProjection] = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Currently StableDiffusionPipeline.check_inputs with negative prompt stuff removed + def check_inputs( + self, + prompt: Union[str, List[str]], + height: int, + width: int, + callback_steps: int, + prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def clip_skip(self): + return self._clip_skip + + @property + def do_classifier_free_guidance(self): + return False + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 4, + original_inference_steps: int = None, + timesteps: List[int] = None, + guidance_scale: float = 8.5, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + original_inference_steps (`int`, *optional*): + The original number of inference steps use to generate a linearly-spaced timestep schedule, from which + we will draw `num_inference_steps` evenly spaced timesteps from as our final timestep schedule, + following the Skipping-Step method in the paper (see Section 4.3). If not set this will default to the + scheduler's `original_inference_steps` attribute. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps on the original LCM training/distillation timestep schedule are used. Must be in descending + order. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + Note that the original latent consistency models paper uses a different CFG formulation where the + guidance scales are decreased by 1 (so in the paper formulation CFG is enabled when `guidance_scale > + 0`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): + Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + callback_steps, + prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 3. Encode input prompt + lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + + # NOTE: when a LCM is distilled from an LDM via latent consistency distillation (Algorithm 1) with guided + # distillation, the forward pass of the LCM learns to approximate sampling from the LDM using CFG with the + # unconditional prompt "" (the empty string). Due to this, LCMs currently do not support negative prompts. + prompt_embeds, _ = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=None, + lora_scale=lora_scale, + clip_skip=self.clip_skip, + ) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps( + self.scheduler, num_inference_steps, device, timesteps, original_inference_steps=original_inference_steps + ) + + # 5. Prepare latent variable + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + bs = batch_size * num_images_per_prompt + + # 6. Get Guidance Scale Embedding + # NOTE: We use the Imagen CFG formulation that StableDiffusionPipeline uses rather than the original LCM paper + # CFG formulation, so we need to subtract 1 from the input guidance_scale. + # LCM CFG formulation: cfg_noise = noise_cond + cfg_scale * (noise_cond - noise_uncond), (cfg_scale > 0.0 using CFG) + w = torch.tensor(self.guidance_scale - 1).repeat(bs) + w_embedding = self.get_guidance_scale_embedding(w, embedding_dim=self.unet.config.time_cond_proj_dim).to( + device=device, dtype=latents.dtype + ) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, None) + + # 7.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + # 8. LCM MultiStep Sampling Loop: + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + latents = latents.to(prompt_embeds.dtype) + + # model prediction (v-prediction, eps, x) + model_pred = self.unet( + latents, + t, + timestep_cond=w_embedding, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # compute the previous noisy sample x_t -> x_t-1 + latents, denoised = self.scheduler.step(model_pred, t, latents, **extra_step_kwargs, return_dict=False) + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + w_embedding = callback_outputs.pop("w_embedding", w_embedding) + denoised = callback_outputs.pop("denoised", denoised) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + denoised = denoised.to(prompt_embeds.dtype) + if not output_type == "latent": + image = self.vae.decode(denoised / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = denoised + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/__init__.py new file mode 100755 index 0000000..561f96f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/__init__.py @@ -0,0 +1,50 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_latent_diffusion"] = ["LDMBertModel", "LDMTextToImagePipeline"] + _import_structure["pipeline_latent_diffusion_superresolution"] = ["LDMSuperResolutionPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_latent_diffusion import LDMBertModel, LDMTextToImagePipeline + from .pipeline_latent_diffusion_superresolution import LDMSuperResolutionPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py new file mode 100755 index 0000000..f39cbc8 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion.py @@ -0,0 +1,746 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import List, Optional, Tuple, Union + +import torch +import torch.nn as nn +import torch.utils.checkpoint +from transformers import PretrainedConfig, PreTrainedModel, PreTrainedTokenizer +from transformers.activations import ACT2FN +from transformers.modeling_outputs import BaseModelOutput +from transformers.utils import logging + +from ...models import AutoencoderKL, UNet2DConditionModel, UNet2DModel, VQModel +from ...schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +class LDMTextToImagePipeline(DiffusionPipeline): + r""" + Pipeline for text-to-image generation using latent diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + vqvae ([`VQModel`]): + Vector-quantized (VQ) model to encode and decode images to and from latent representations. + bert ([`LDMBertModel`]): + Text-encoder model based on [`~transformers.BERT`]. + tokenizer ([`~transformers.BertTokenizer`]): + A `BertTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "bert->unet->vqvae" + + def __init__( + self, + vqvae: Union[VQModel, AutoencoderKL], + bert: PreTrainedModel, + tokenizer: PreTrainedTokenizer, + unet: Union[UNet2DModel, UNet2DConditionModel], + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + ): + super().__init__() + self.register_modules(vqvae=vqvae, bert=bert, tokenizer=tokenizer, unet=unet, scheduler=scheduler) + self.vae_scale_factor = 2 ** (len(self.vqvae.config.block_out_channels) - 1) + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 1.0, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + **kwargs, + ) -> Union[Tuple, ImagePipelineOutput]: + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 1.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`ImagePipelineOutput`] instead of a plain tuple. + + Example: + + ```py + >>> from diffusers import DiffusionPipeline + + >>> # load model and scheduler + >>> ldm = DiffusionPipeline.from_pretrained("CompVis/ldm-text2im-large-256") + + >>> # run pipeline in inference (sample random noise and denoise) + >>> prompt = "A painting of a squirrel eating a burger" + >>> images = ldm([prompt], num_inference_steps=50, eta=0.3, guidance_scale=6).images + + >>> # save images + >>> for idx, image in enumerate(images): + ... image.save(f"squirrel-{idx}.png") + ``` + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + # get unconditional embeddings for classifier free guidance + if guidance_scale != 1.0: + uncond_input = self.tokenizer( + [""] * batch_size, padding="max_length", max_length=77, truncation=True, return_tensors="pt" + ) + negative_prompt_embeds = self.bert(uncond_input.input_ids.to(self._execution_device))[0] + + # get prompt text embeddings + text_input = self.tokenizer(prompt, padding="max_length", max_length=77, truncation=True, return_tensors="pt") + prompt_embeds = self.bert(text_input.input_ids.to(self._execution_device))[0] + + # get the initial random noise unless the user supplied it + latents_shape = (batch_size, self.unet.config.in_channels, height // 8, width // 8) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor( + latents_shape, generator=generator, device=self._execution_device, dtype=prompt_embeds.dtype + ) + else: + if latents.shape != latents_shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}") + latents = latents.to(self._execution_device) + + self.scheduler.set_timesteps(num_inference_steps) + + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + + extra_kwargs = {} + if accepts_eta: + extra_kwargs["eta"] = eta + + for t in self.progress_bar(self.scheduler.timesteps): + if guidance_scale == 1.0: + # guidance_scale of 1 means no guidance + latents_input = latents + context = prompt_embeds + else: + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + latents_input = torch.cat([latents] * 2) + context = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # predict the noise residual + noise_pred = self.unet(latents_input, t, encoder_hidden_states=context).sample + # perform guidance + if guidance_scale != 1.0: + noise_pred_uncond, noise_prediction_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_prediction_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_kwargs).prev_sample + + # scale and decode the image latents with vae + latents = 1 / self.vqvae.config.scaling_factor * latents + image = self.vqvae.decode(latents).sample + + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) + + +################################################################################ +# Code for the text transformer model +################################################################################ +""" PyTorch LDMBERT model.""" + + +logger = logging.get_logger(__name__) + +LDMBERT_PRETRAINED_MODEL_ARCHIVE_LIST = [ + "ldm-bert", + # See all LDMBert models at https://huggingface.co/models?filter=ldmbert +] + + +LDMBERT_PRETRAINED_CONFIG_ARCHIVE_MAP = { + "ldm-bert": "https://huggingface.co/valhalla/ldm-bert/blob/main/config.json", +} + + +""" LDMBERT model configuration""" + + +class LDMBertConfig(PretrainedConfig): + model_type = "ldmbert" + keys_to_ignore_at_inference = ["past_key_values"] + attribute_map = {"num_attention_heads": "encoder_attention_heads", "hidden_size": "d_model"} + + def __init__( + self, + vocab_size=30522, + max_position_embeddings=77, + encoder_layers=32, + encoder_ffn_dim=5120, + encoder_attention_heads=8, + head_dim=64, + encoder_layerdrop=0.0, + activation_function="gelu", + d_model=1280, + dropout=0.1, + attention_dropout=0.0, + activation_dropout=0.0, + init_std=0.02, + classifier_dropout=0.0, + scale_embedding=False, + use_cache=True, + pad_token_id=0, + **kwargs, + ): + self.vocab_size = vocab_size + self.max_position_embeddings = max_position_embeddings + self.d_model = d_model + self.encoder_ffn_dim = encoder_ffn_dim + self.encoder_layers = encoder_layers + self.encoder_attention_heads = encoder_attention_heads + self.head_dim = head_dim + self.dropout = dropout + self.attention_dropout = attention_dropout + self.activation_dropout = activation_dropout + self.activation_function = activation_function + self.init_std = init_std + self.encoder_layerdrop = encoder_layerdrop + self.classifier_dropout = classifier_dropout + self.use_cache = use_cache + self.num_hidden_layers = encoder_layers + self.scale_embedding = scale_embedding # scale factor will be sqrt(d_model) if True + + super().__init__(pad_token_id=pad_token_id, **kwargs) + + +def _expand_mask(mask: torch.Tensor, dtype: torch.dtype, tgt_len: Optional[int] = None): + """ + Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. + """ + bsz, src_len = mask.size() + tgt_len = tgt_len if tgt_len is not None else src_len + + expanded_mask = mask[:, None, None, :].expand(bsz, 1, tgt_len, src_len).to(dtype) + + inverted_mask = 1.0 - expanded_mask + + return inverted_mask.masked_fill(inverted_mask.to(torch.bool), torch.finfo(dtype).min) + + +# Copied from transformers.models.bart.modeling_bart.BartAttention with Bart->LDMBert +class LDMBertAttention(nn.Module): + """Multi-headed attention from 'Attention Is All You Need' paper""" + + def __init__( + self, + embed_dim: int, + num_heads: int, + head_dim: int, + dropout: float = 0.0, + is_decoder: bool = False, + bias: bool = False, + ): + super().__init__() + self.embed_dim = embed_dim + self.num_heads = num_heads + self.dropout = dropout + self.head_dim = head_dim + self.inner_dim = head_dim * num_heads + + self.scaling = self.head_dim**-0.5 + self.is_decoder = is_decoder + + self.k_proj = nn.Linear(embed_dim, self.inner_dim, bias=bias) + self.v_proj = nn.Linear(embed_dim, self.inner_dim, bias=bias) + self.q_proj = nn.Linear(embed_dim, self.inner_dim, bias=bias) + self.out_proj = nn.Linear(self.inner_dim, embed_dim) + + def _shape(self, tensor: torch.Tensor, seq_len: int, bsz: int): + return tensor.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2).contiguous() + + def forward( + self, + hidden_states: torch.Tensor, + key_value_states: Optional[torch.Tensor] = None, + past_key_value: Optional[Tuple[torch.Tensor]] = None, + attention_mask: Optional[torch.Tensor] = None, + layer_head_mask: Optional[torch.Tensor] = None, + output_attentions: bool = False, + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[Tuple[torch.Tensor]]]: + """Input shape: Batch x Time x Channel""" + + # if key_value_states are provided this layer is used as a cross-attention layer + # for the decoder + is_cross_attention = key_value_states is not None + + bsz, tgt_len, _ = hidden_states.size() + + # get query proj + query_states = self.q_proj(hidden_states) * self.scaling + # get key, value proj + if is_cross_attention and past_key_value is not None: + # reuse k,v, cross_attentions + key_states = past_key_value[0] + value_states = past_key_value[1] + elif is_cross_attention: + # cross_attentions + key_states = self._shape(self.k_proj(key_value_states), -1, bsz) + value_states = self._shape(self.v_proj(key_value_states), -1, bsz) + elif past_key_value is not None: + # reuse k, v, self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + key_states = torch.cat([past_key_value[0], key_states], dim=2) + value_states = torch.cat([past_key_value[1], value_states], dim=2) + else: + # self_attention + key_states = self._shape(self.k_proj(hidden_states), -1, bsz) + value_states = self._shape(self.v_proj(hidden_states), -1, bsz) + + if self.is_decoder: + # if cross_attention save Tuple(torch.Tensor, torch.Tensor) of all cross attention key/value_states. + # Further calls to cross_attention layer can then reuse all cross-attention + # key/value_states (first "if" case) + # if uni-directional self-attention (decoder) save Tuple(torch.Tensor, torch.Tensor) of + # all previous decoder key/value_states. Further calls to uni-directional self-attention + # can concat previous decoder key/value_states to current projected key/value_states (third "elif" case) + # if encoder bi-directional self-attention `past_key_value` is always `None` + past_key_value = (key_states, value_states) + + proj_shape = (bsz * self.num_heads, -1, self.head_dim) + query_states = self._shape(query_states, tgt_len, bsz).view(*proj_shape) + key_states = key_states.view(*proj_shape) + value_states = value_states.view(*proj_shape) + + src_len = key_states.size(1) + attn_weights = torch.bmm(query_states, key_states.transpose(1, 2)) + + if attn_weights.size() != (bsz * self.num_heads, tgt_len, src_len): + raise ValueError( + f"Attention weights should be of size {(bsz * self.num_heads, tgt_len, src_len)}, but is" + f" {attn_weights.size()}" + ) + + if attention_mask is not None: + if attention_mask.size() != (bsz, 1, tgt_len, src_len): + raise ValueError( + f"Attention mask should be of size {(bsz, 1, tgt_len, src_len)}, but is {attention_mask.size()}" + ) + attn_weights = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attention_mask + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + attn_weights = nn.functional.softmax(attn_weights, dim=-1) + + if layer_head_mask is not None: + if layer_head_mask.size() != (self.num_heads,): + raise ValueError( + f"Head mask for a single layer should be of size {(self.num_heads,)}, but is" + f" {layer_head_mask.size()}" + ) + attn_weights = layer_head_mask.view(1, -1, 1, 1) * attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights.view(bsz * self.num_heads, tgt_len, src_len) + + if output_attentions: + # this operation is a bit awkward, but it's required to + # make sure that attn_weights keeps its gradient. + # In order to do so, attn_weights have to be reshaped + # twice and have to be reused in the following + attn_weights_reshaped = attn_weights.view(bsz, self.num_heads, tgt_len, src_len) + attn_weights = attn_weights_reshaped.view(bsz * self.num_heads, tgt_len, src_len) + else: + attn_weights_reshaped = None + + attn_probs = nn.functional.dropout(attn_weights, p=self.dropout, training=self.training) + + attn_output = torch.bmm(attn_probs, value_states) + + if attn_output.size() != (bsz * self.num_heads, tgt_len, self.head_dim): + raise ValueError( + f"`attn_output` should be of size {(bsz, self.num_heads, tgt_len, self.head_dim)}, but is" + f" {attn_output.size()}" + ) + + attn_output = attn_output.view(bsz, self.num_heads, tgt_len, self.head_dim) + attn_output = attn_output.transpose(1, 2) + + # Use the `embed_dim` from the config (stored in the class) rather than `hidden_state` because `attn_output` can be + # partitioned across GPUs when using tensor-parallelism. + attn_output = attn_output.reshape(bsz, tgt_len, self.inner_dim) + + attn_output = self.out_proj(attn_output) + + return attn_output, attn_weights_reshaped, past_key_value + + +class LDMBertEncoderLayer(nn.Module): + def __init__(self, config: LDMBertConfig): + super().__init__() + self.embed_dim = config.d_model + self.self_attn = LDMBertAttention( + embed_dim=self.embed_dim, + num_heads=config.encoder_attention_heads, + head_dim=config.head_dim, + dropout=config.attention_dropout, + ) + self.self_attn_layer_norm = nn.LayerNorm(self.embed_dim) + self.dropout = config.dropout + self.activation_fn = ACT2FN[config.activation_function] + self.activation_dropout = config.activation_dropout + self.fc1 = nn.Linear(self.embed_dim, config.encoder_ffn_dim) + self.fc2 = nn.Linear(config.encoder_ffn_dim, self.embed_dim) + self.final_layer_norm = nn.LayerNorm(self.embed_dim) + + def forward( + self, + hidden_states: torch.FloatTensor, + attention_mask: torch.FloatTensor, + layer_head_mask: torch.FloatTensor, + output_attentions: Optional[bool] = False, + ) -> Tuple[torch.FloatTensor, Optional[torch.FloatTensor]]: + """ + Args: + hidden_states (`torch.FloatTensor`): input to the layer of shape `(seq_len, batch, embed_dim)` + attention_mask (`torch.FloatTensor`): attention mask of size + `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. + layer_head_mask (`torch.FloatTensor`): mask for attention heads in a given layer of size + `(encoder_attention_heads,)`. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + """ + residual = hidden_states + hidden_states = self.self_attn_layer_norm(hidden_states) + hidden_states, attn_weights, _ = self.self_attn( + hidden_states=hidden_states, + attention_mask=attention_mask, + layer_head_mask=layer_head_mask, + output_attentions=output_attentions, + ) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + + residual = hidden_states + hidden_states = self.final_layer_norm(hidden_states) + hidden_states = self.activation_fn(self.fc1(hidden_states)) + hidden_states = nn.functional.dropout(hidden_states, p=self.activation_dropout, training=self.training) + hidden_states = self.fc2(hidden_states) + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + hidden_states = residual + hidden_states + + if hidden_states.dtype == torch.float16 and ( + torch.isinf(hidden_states).any() or torch.isnan(hidden_states).any() + ): + clamp_value = torch.finfo(hidden_states.dtype).max - 1000 + hidden_states = torch.clamp(hidden_states, min=-clamp_value, max=clamp_value) + + outputs = (hidden_states,) + + if output_attentions: + outputs += (attn_weights,) + + return outputs + + +# Copied from transformers.models.bart.modeling_bart.BartPretrainedModel with Bart->LDMBert +class LDMBertPreTrainedModel(PreTrainedModel): + config_class = LDMBertConfig + base_model_prefix = "model" + _supports_gradient_checkpointing = True + _keys_to_ignore_on_load_unexpected = [r"encoder\.version", r"decoder\.version"] + + def _init_weights(self, module): + std = self.config.init_std + if isinstance(module, nn.Linear): + module.weight.data.normal_(mean=0.0, std=std) + if module.bias is not None: + module.bias.data.zero_() + elif isinstance(module, nn.Embedding): + module.weight.data.normal_(mean=0.0, std=std) + if module.padding_idx is not None: + module.weight.data[module.padding_idx].zero_() + + def _set_gradient_checkpointing(self, module, value=False): + if isinstance(module, (LDMBertEncoder,)): + module.gradient_checkpointing = value + + @property + def dummy_inputs(self): + pad_token = self.config.pad_token_id + input_ids = torch.tensor([[0, 6, 10, 4, 2], [0, 8, 12, 2, pad_token]], device=self.device) + dummy_inputs = { + "attention_mask": input_ids.ne(pad_token), + "input_ids": input_ids, + } + return dummy_inputs + + +class LDMBertEncoder(LDMBertPreTrainedModel): + """ + Transformer encoder consisting of *config.encoder_layers* self attention layers. Each layer is a + [`LDMBertEncoderLayer`]. + + Args: + config: LDMBertConfig + embed_tokens (nn.Embedding): output embedding + """ + + def __init__(self, config: LDMBertConfig): + super().__init__(config) + + self.dropout = config.dropout + + embed_dim = config.d_model + self.padding_idx = config.pad_token_id + self.max_source_positions = config.max_position_embeddings + + self.embed_tokens = nn.Embedding(config.vocab_size, embed_dim) + self.embed_positions = nn.Embedding(config.max_position_embeddings, embed_dim) + self.layers = nn.ModuleList([LDMBertEncoderLayer(config) for _ in range(config.encoder_layers)]) + self.layer_norm = nn.LayerNorm(embed_dim) + + self.gradient_checkpointing = False + # Initialize weights and apply final processing + self.post_init() + + def get_input_embeddings(self): + return self.embed_tokens + + def set_input_embeddings(self, value): + self.embed_tokens = value + + def forward( + self, + input_ids: torch.LongTensor = None, + attention_mask: Optional[torch.Tensor] = None, + position_ids: Optional[torch.LongTensor] = None, + head_mask: Optional[torch.Tensor] = None, + inputs_embeds: Optional[torch.FloatTensor] = None, + output_attentions: Optional[bool] = None, + output_hidden_states: Optional[bool] = None, + return_dict: Optional[bool] = None, + ) -> Union[Tuple, BaseModelOutput]: + r""" + Args: + input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): + Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you + provide it. + + Indices can be obtained using [`BartTokenizer`]. See [`PreTrainedTokenizer.encode`] and + [`PreTrainedTokenizer.__call__`] for details. + + [What are input IDs?](../glossary#input-ids) + attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): + Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: + + - 1 for tokens that are **not masked**, + - 0 for tokens that are **masked**. + + [What are attention masks?](../glossary#attention-mask) + head_mask (`torch.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): + Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: + + - 1 indicates the head is **not masked**, + - 0 indicates the head is **masked**. + + inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): + Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. + This is useful if you want more control over how to convert `input_ids` indices into associated vectors + than the model's internal embedding lookup matrix. + output_attentions (`bool`, *optional*): + Whether or not to return the attentions tensors of all attention layers. See `attentions` under + returned tensors for more detail. + output_hidden_states (`bool`, *optional*): + Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors + for more detail. + return_dict (`bool`, *optional*): + Whether or not to return a [`~utils.BaseModelOutput`] instead of a plain tuple. + """ + output_attentions = output_attentions if output_attentions is not None else self.config.output_attentions + output_hidden_states = ( + output_hidden_states if output_hidden_states is not None else self.config.output_hidden_states + ) + return_dict = return_dict if return_dict is not None else self.config.use_return_dict + + # retrieve input_ids and inputs_embeds + if input_ids is not None and inputs_embeds is not None: + raise ValueError("You cannot specify both input_ids and inputs_embeds at the same time") + elif input_ids is not None: + input_shape = input_ids.size() + input_ids = input_ids.view(-1, input_shape[-1]) + elif inputs_embeds is not None: + input_shape = inputs_embeds.size()[:-1] + else: + raise ValueError("You have to specify either input_ids or inputs_embeds") + + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + + seq_len = input_shape[1] + if position_ids is None: + position_ids = torch.arange(seq_len, dtype=torch.long, device=inputs_embeds.device).expand((1, -1)) + embed_pos = self.embed_positions(position_ids) + + hidden_states = inputs_embeds + embed_pos + hidden_states = nn.functional.dropout(hidden_states, p=self.dropout, training=self.training) + + # expand attention_mask + if attention_mask is not None: + # [bsz, seq_len] -> [bsz, 1, tgt_seq_len, src_seq_len] + attention_mask = _expand_mask(attention_mask, inputs_embeds.dtype) + + encoder_states = () if output_hidden_states else None + all_attentions = () if output_attentions else None + + # check if head_mask has a correct number of layers specified if desired + if head_mask is not None: + if head_mask.size()[0] != (len(self.layers)): + raise ValueError( + f"The head_mask should be specified for {len(self.layers)} layers, but it is for" + f" {head_mask.size()[0]}." + ) + + for idx, encoder_layer in enumerate(self.layers): + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + if self.gradient_checkpointing and self.training: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs, output_attentions) + + return custom_forward + + layer_outputs = torch.utils.checkpoint.checkpoint( + create_custom_forward(encoder_layer), + hidden_states, + attention_mask, + (head_mask[idx] if head_mask is not None else None), + ) + else: + layer_outputs = encoder_layer( + hidden_states, + attention_mask, + layer_head_mask=(head_mask[idx] if head_mask is not None else None), + output_attentions=output_attentions, + ) + + hidden_states = layer_outputs[0] + + if output_attentions: + all_attentions = all_attentions + (layer_outputs[1],) + + hidden_states = self.layer_norm(hidden_states) + + if output_hidden_states: + encoder_states = encoder_states + (hidden_states,) + + if not return_dict: + return tuple(v for v in [hidden_states, encoder_states, all_attentions] if v is not None) + return BaseModelOutput( + last_hidden_state=hidden_states, hidden_states=encoder_states, attentions=all_attentions + ) + + +class LDMBertModel(LDMBertPreTrainedModel): + _no_split_modules = [] + + def __init__(self, config: LDMBertConfig): + super().__init__(config) + self.model = LDMBertEncoder(config) + self.to_logits = nn.Linear(config.hidden_size, config.vocab_size) + + def forward( + self, + input_ids=None, + attention_mask=None, + position_ids=None, + head_mask=None, + inputs_embeds=None, + output_attentions=None, + output_hidden_states=None, + return_dict=None, + ): + outputs = self.model( + input_ids, + attention_mask=attention_mask, + position_ids=position_ids, + head_mask=head_mask, + inputs_embeds=inputs_embeds, + output_attentions=output_attentions, + output_hidden_states=output_hidden_states, + return_dict=return_dict, + ) + return outputs diff --git a/diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py b/diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py new file mode 100755 index 0000000..bb72b4d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/latent_diffusion/pipeline_latent_diffusion_superresolution.py @@ -0,0 +1,189 @@ +import inspect +from typing import List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +import torch.utils.checkpoint + +from ...models import UNet2DModel, VQModel +from ...schedulers import ( + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, +) +from ...utils import PIL_INTERPOLATION +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +def preprocess(image): + w, h = image.size + w, h = (x - x % 32 for x in (w, h)) # resize to integer multiple of 32 + image = image.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]) + image = np.array(image).astype(np.float32) / 255.0 + image = image[None].transpose(0, 3, 1, 2) + image = torch.from_numpy(image) + return 2.0 * image - 1.0 + + +class LDMSuperResolutionPipeline(DiffusionPipeline): + r""" + A pipeline for image super-resolution using latent diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Parameters: + vqvae ([`VQModel`]): + Vector-quantized (VQ) model to encode and decode images to and from latent representations. + unet ([`UNet2DModel`]): + A `UNet2DModel` to denoise the encoded image. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latens. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], [`EulerDiscreteScheduler`], + [`EulerAncestralDiscreteScheduler`], [`DPMSolverMultistepScheduler`], or [`PNDMScheduler`]. + """ + + def __init__( + self, + vqvae: VQModel, + unet: UNet2DModel, + scheduler: Union[ + DDIMScheduler, + PNDMScheduler, + LMSDiscreteScheduler, + EulerDiscreteScheduler, + EulerAncestralDiscreteScheduler, + DPMSolverMultistepScheduler, + ], + ): + super().__init__() + self.register_modules(vqvae=vqvae, unet=unet, scheduler=scheduler) + + @torch.no_grad() + def __call__( + self, + image: Union[torch.Tensor, PIL.Image.Image] = None, + batch_size: Optional[int] = 1, + num_inference_steps: Optional[int] = 100, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ) -> Union[Tuple, ImagePipelineOutput]: + r""" + The call function to the pipeline for generation. + + Args: + image (`torch.Tensor` or `PIL.Image.Image`): + `Image` or tensor representing an image batch to be used as the starting point for the process. + batch_size (`int`, *optional*, defaults to 1): + Number of images to generate. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`ImagePipelineOutput`] instead of a plain tuple. + + Example: + + ```py + >>> import requests + >>> from PIL import Image + >>> from io import BytesIO + >>> from diffusers import LDMSuperResolutionPipeline + >>> import torch + + >>> # load model and scheduler + >>> pipeline = LDMSuperResolutionPipeline.from_pretrained("CompVis/ldm-super-resolution-4x-openimages") + >>> pipeline = pipeline.to("cuda") + + >>> # let's download an image + >>> url = ( + ... "https://user-images.githubusercontent.com/38061659/199705896-b48e17b8-b231-47cd-a270-4ffa5a93fa3e.png" + ... ) + >>> response = requests.get(url) + >>> low_res_img = Image.open(BytesIO(response.content)).convert("RGB") + >>> low_res_img = low_res_img.resize((128, 128)) + + >>> # run pipeline in inference (sample random noise and denoise) + >>> upscaled_image = pipeline(low_res_img, num_inference_steps=100, eta=1).images[0] + >>> # save image + >>> upscaled_image.save("ldm_generated_image.png") + ``` + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images + """ + if isinstance(image, PIL.Image.Image): + batch_size = 1 + elif isinstance(image, torch.Tensor): + batch_size = image.shape[0] + else: + raise ValueError(f"`image` has to be of type `PIL.Image.Image` or `torch.Tensor` but is {type(image)}") + + if isinstance(image, PIL.Image.Image): + image = preprocess(image) + + height, width = image.shape[-2:] + + # in_channels should be 6: 3 for latents, 3 for low resolution image + latents_shape = (batch_size, self.unet.config.in_channels // 2, height, width) + latents_dtype = next(self.unet.parameters()).dtype + + latents = randn_tensor(latents_shape, generator=generator, device=self.device, dtype=latents_dtype) + + image = image.to(device=self.device, dtype=latents_dtype) + + # set timesteps and move to the correct device + self.scheduler.set_timesteps(num_inference_steps, device=self.device) + timesteps_tensor = self.scheduler.timesteps + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature. + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_kwargs = {} + if accepts_eta: + extra_kwargs["eta"] = eta + + for t in self.progress_bar(timesteps_tensor): + # concat latents and low resolution image in the channel dimension. + latents_input = torch.cat([latents, image], dim=1) + latents_input = self.scheduler.scale_model_input(latents_input, t) + # predict the noise residual + noise_pred = self.unet(latents_input, t).sample + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_kwargs).prev_sample + + # decode the image latents with the VQVAE + image = self.vqvae.decode(latents).sample + image = torch.clamp(image, -1.0, 1.0) + image = image / 2 + 0.5 + image = image.cpu().permute(0, 2, 3, 1).numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/__init__.py new file mode 100755 index 0000000..aae3b1c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/__init__.py @@ -0,0 +1,55 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_leditspp_stable_diffusion"] = ["LEditsPPPipelineStableDiffusion"] + _import_structure["pipeline_leditspp_stable_diffusion_xl"] = ["LEditsPPPipelineStableDiffusionXL"] + + _import_structure["pipeline_output"] = ["LEditsPPDiffusionPipelineOutput", "LEditsPPDiffusionPipelineOutput"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_leditspp_stable_diffusion import ( + LEditsPPDiffusionPipelineOutput, + LEditsPPInversionPipelineOutput, + LEditsPPPipelineStableDiffusion, + ) + from .pipeline_leditspp_stable_diffusion_xl import LEditsPPPipelineStableDiffusionXL + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py new file mode 100755 index 0000000..a6357c4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion.py @@ -0,0 +1,1505 @@ +import inspect +import math +from itertools import repeat +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from packaging import version +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ...configuration_utils import FrozenDict +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.attention_processor import Attention, AttnProcessor +from ...models.lora import adjust_lora_scale_text_encoder +from ...pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from ...schedulers import DDIMScheduler, DPMSolverMultistepScheduler +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .pipeline_output import LEditsPPDiffusionPipelineOutput, LEditsPPInversionPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import PIL + >>> import requests + >>> import torch + >>> from io import BytesIO + + >>> from diffusers import LEditsPPPipelineStableDiffusion + + >>> pipe = LEditsPPPipelineStableDiffusion.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> def download_image(url): + ... response = requests.get(url) + ... return PIL.Image.open(BytesIO(response.content)).convert("RGB") + + >>> img_url = "https://www.aiml.informatik.tu-darmstadt.de/people/mbrack/cherry_blossom.png" + >>> image = download_image(img_url) + + >>> _ = pipe.invert( + ... image = image, + ... num_inversion_steps=50, + ... skip=0.1 + ... ) + + >>> edited_image = pipe( + ... editing_prompt=["cherry blossom"], + ... edit_guidance_scale=10.0, + ... edit_threshold=0.75, + ).images[0] + ``` +""" + + +# Modified from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionAttendAndExcitePipeline.AttentionStore +class LeditsAttentionStore: + @staticmethod + def get_empty_store(): + return {"down_cross": [], "mid_cross": [], "up_cross": [], "down_self": [], "mid_self": [], "up_self": []} + + def __call__(self, attn, is_cross: bool, place_in_unet: str, editing_prompts, PnP=False): + # attn.shape = batch_size * head_size, seq_len query, seq_len_key + if attn.shape[1] <= self.max_size: + bs = 1 + int(PnP) + editing_prompts + skip = 2 if PnP else 1 # skip PnP & unconditional + attn = torch.stack(attn.split(self.batch_size)).permute(1, 0, 2, 3) + source_batch_size = int(attn.shape[1] // bs) + self.forward(attn[:, skip * source_batch_size :], is_cross, place_in_unet) + + def forward(self, attn, is_cross: bool, place_in_unet: str): + key = f"{place_in_unet}_{'cross' if is_cross else 'self'}" + + self.step_store[key].append(attn) + + def between_steps(self, store_step=True): + if store_step: + if self.average: + if len(self.attention_store) == 0: + self.attention_store = self.step_store + else: + for key in self.attention_store: + for i in range(len(self.attention_store[key])): + self.attention_store[key][i] += self.step_store[key][i] + else: + if len(self.attention_store) == 0: + self.attention_store = [self.step_store] + else: + self.attention_store.append(self.step_store) + + self.cur_step += 1 + self.step_store = self.get_empty_store() + + def get_attention(self, step: int): + if self.average: + attention = { + key: [item / self.cur_step for item in self.attention_store[key]] for key in self.attention_store + } + else: + assert step is not None + attention = self.attention_store[step] + return attention + + def aggregate_attention( + self, attention_maps, prompts, res: Union[int, Tuple[int]], from_where: List[str], is_cross: bool, select: int + ): + out = [[] for x in range(self.batch_size)] + if isinstance(res, int): + num_pixels = res**2 + resolution = (res, res) + else: + num_pixels = res[0] * res[1] + resolution = res[:2] + + for location in from_where: + for bs_item in attention_maps[f"{location}_{'cross' if is_cross else 'self'}"]: + for batch, item in enumerate(bs_item): + if item.shape[1] == num_pixels: + cross_maps = item.reshape(len(prompts), -1, *resolution, item.shape[-1])[select] + out[batch].append(cross_maps) + + out = torch.stack([torch.cat(x, dim=0) for x in out]) + # average over heads + out = out.sum(1) / out.shape[1] + return out + + def __init__(self, average: bool, batch_size=1, max_resolution=16, max_size: int = None): + self.step_store = self.get_empty_store() + self.attention_store = [] + self.cur_step = 0 + self.average = average + self.batch_size = batch_size + if max_size is None: + self.max_size = max_resolution**2 + elif max_size is not None and max_resolution is None: + self.max_size = max_size + else: + raise ValueError("Only allowed to set one of max_resolution or max_size") + + +# Modified from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionAttendAndExcitePipeline.GaussianSmoothing +class LeditsGaussianSmoothing: + def __init__(self, device): + kernel_size = [3, 3] + sigma = [0.5, 0.5] + + # The gaussian kernel is the product of the gaussian function of each dimension. + kernel = 1 + meshgrids = torch.meshgrid([torch.arange(size, dtype=torch.float32) for size in kernel_size]) + for size, std, mgrid in zip(kernel_size, sigma, meshgrids): + mean = (size - 1) / 2 + kernel *= 1 / (std * math.sqrt(2 * math.pi)) * torch.exp(-(((mgrid - mean) / (2 * std)) ** 2)) + + # Make sure sum of values in gaussian kernel equals 1. + kernel = kernel / torch.sum(kernel) + + # Reshape to depthwise convolutional weight + kernel = kernel.view(1, 1, *kernel.size()) + kernel = kernel.repeat(1, *[1] * (kernel.dim() - 1)) + + self.weight = kernel.to(device) + + def __call__(self, input): + """ + Arguments: + Apply gaussian filter to input. + input (torch.Tensor): Input to apply gaussian filter on. + Returns: + filtered (torch.Tensor): Filtered output. + """ + return F.conv2d(input, weight=self.weight.to(input.dtype)) + + +class LEDITSCrossAttnProcessor: + def __init__(self, attention_store, place_in_unet, pnp, editing_prompts): + self.attnstore = attention_store + self.place_in_unet = place_in_unet + self.editing_prompts = editing_prompts + self.pnp = pnp + + def __call__( + self, + attn: Attention, + hidden_states, + encoder_hidden_states, + attention_mask=None, + temb=None, + ): + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + self.attnstore( + attention_probs, + is_cross=True, + place_in_unet=self.place_in_unet, + editing_prompts=self.editing_prompts, + PnP=self.pnp, + ) + + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + hidden_states = hidden_states / attn.rescale_output_factor + return hidden_states + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class LEditsPPPipelineStableDiffusion( + DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin, IPAdapterMixin, FromSingleFileMixin +): + """ + Pipeline for textual image editing using LEDits++ with Stable Diffusion. + + This model inherits from [`DiffusionPipeline`] and builds on the [`StableDiffusionPipeline`]. Check the superclass + documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular + device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer ([`~transformers.CLIPTokenizer`]): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`DPMSolverMultistepScheduler`] or [`DDIMScheduler`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latens. Can be one of + [`DPMSolverMultistepScheduler`] or [`DDIMScheduler`]. If any other scheduler is passed it will automatically + be set to [`DPMSolverMultistepScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please, refer to the [model card](https://huggingface.co/CompVis/stable-diffusion-v1-4) for details. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: Union[DDIMScheduler, DPMSolverMultistepScheduler], + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if not isinstance(scheduler, DDIMScheduler) and not isinstance(scheduler, DPMSolverMultistepScheduler): + scheduler = DPMSolverMultistepScheduler.from_config( + scheduler.config, algorithm_type="sde-dpmsolver++", solver_order=2 + ) + logger.warning( + "This pipeline only supports DDIMScheduler and DPMSolverMultistepScheduler. " + "The scheduler has been changed to DPMSolverMultistepScheduler." + ) + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + self.inversion_steps = None + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, eta, generator=None): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Modified from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.check_inputs + def check_inputs( + self, + negative_prompt=None, + editing_prompt_embeddings=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if editing_prompt_embeddings is not None and negative_prompt_embeds is not None: + if editing_prompt_embeddings.shape != negative_prompt_embeds.shape: + raise ValueError( + "`editing_prompt_embeddings` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `editing_prompt_embeddings` {editing_prompt_embeddings.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Modified from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, latents): + # shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + + # if latents.shape != shape: + # raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_unet(self, attention_store, PnP: bool = False): + attn_procs = {} + for name in self.unet.attn_processors.keys(): + if name.startswith("mid_block"): + place_in_unet = "mid" + elif name.startswith("up_blocks"): + place_in_unet = "up" + elif name.startswith("down_blocks"): + place_in_unet = "down" + else: + continue + + if "attn2" in name and place_in_unet != "mid": + attn_procs[name] = LEDITSCrossAttnProcessor( + attention_store=attention_store, + place_in_unet=place_in_unet, + pnp=PnP, + editing_prompts=self.enabled_editing_prompts, + ) + else: + attn_procs[name] = AttnProcessor() + + self.unet.set_attn_processor(attn_procs) + + def encode_prompt( + self, + device, + num_images_per_prompt, + enable_edit_guidance, + negative_prompt=None, + editing_prompt=None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + editing_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + enable_edit_guidance (`bool`): + whether to perform any editing or reconstruct the input image instead + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + editing_prompt (`str` or `List[str]`, *optional*): + Editing prompt(s) to be encoded. If not defined, one has to pass + `editing_prompt_embeds` instead. + editing_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + batch_size = self.batch_size + num_edit_tokens = None + + if negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but exoected" + f"{batch_size} based on the input images. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: procecss multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = negative_prompt_embeds.dtype + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + if enable_edit_guidance: + if editing_prompt_embeds is None: + # textual inversion: procecss multi-vector tokens if necessary + # if isinstance(self, TextualInversionLoaderMixin): + # prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + if isinstance(editing_prompt, str): + editing_prompt = [editing_prompt] + + max_length = negative_prompt_embeds.shape[1] + text_inputs = self.tokenizer( + [x for item in editing_prompt for x in repeat(item, batch_size)], + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + return_length=True, + ) + + num_edit_tokens = text_inputs.length - 2 # not counting startoftext and endoftext + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer( + [x for item in editing_prompt for x in repeat(item, batch_size)], + padding="longest", + return_tensors="pt", + ).input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if ( + hasattr(self.text_encoder.config, "use_attention_mask") + and self.text_encoder.config.use_attention_mask + ): + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + editing_prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + editing_prompt_embeds = editing_prompt_embeds[0] + else: + editing_prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + editing_prompt_embeds = editing_prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + editing_prompt_embeds = self.text_encoder.text_model.final_layer_norm(editing_prompt_embeds) + + editing_prompt_embeds = editing_prompt_embeds.to(dtype=negative_prompt_embeds.dtype, device=device) + + bs_embed_edit, seq_len, _ = editing_prompt_embeds.shape + editing_prompt_embeds = editing_prompt_embeds.to(dtype=negative_prompt_embeds.dtype, device=device) + editing_prompt_embeds = editing_prompt_embeds.repeat(1, num_images_per_prompt, 1) + editing_prompt_embeds = editing_prompt_embeds.view(bs_embed_edit * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return editing_prompt_embeds, negative_prompt_embeds, num_edit_tokens + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def clip_skip(self): + return self._clip_skip + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + negative_prompt: Optional[Union[str, List[str]]] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + editing_prompt: Optional[Union[str, List[str]]] = None, + editing_prompt_embeds: Optional[torch.Tensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + reverse_editing_direction: Optional[Union[bool, List[bool]]] = False, + edit_guidance_scale: Optional[Union[float, List[float]]] = 5, + edit_warmup_steps: Optional[Union[int, List[int]]] = 0, + edit_cooldown_steps: Optional[Union[int, List[int]]] = None, + edit_threshold: Optional[Union[float, List[float]]] = 0.9, + user_mask: Optional[torch.FloatTensor] = None, + sem_guidance: Optional[List[torch.Tensor]] = None, + use_cross_attn_mask: bool = False, + use_intersect_mask: bool = True, + attn_store_steps: Optional[List[int]] = [], + store_averaged_over_steps: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for editing. The [`~pipelines.ledits_pp.LEditsPPPipelineStableDiffusion.invert`] + method has to be called beforehand. Edits will always be performed for the last inverted image(s). + + Args: + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + generator (`torch.Generator`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ledits_pp.LEditsPPDiffusionPipelineOutput`] instead of a + plain tuple. + editing_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. The image is reconstructed by setting + `editing_prompt = None`. Guidance direction of prompt should be specified via `reverse_editing_direction`. + editing_prompt_embeds (`torch.Tensor>`, *optional*): + Pre-computed embeddings to use for guiding the image generation. Guidance direction of embedding should be + specified via `reverse_editing_direction`. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + reverse_editing_direction (`bool` or `List[bool]`, *optional*, defaults to `False`): + Whether the corresponding prompt in `editing_prompt` should be increased or decreased. + edit_guidance_scale (`float` or `List[float]`, *optional*, defaults to 5): + Guidance scale for guiding the image generation. If provided as list values should correspond to `editing_prompt`. + `edit_guidance_scale` is defined as `s_e` of equation 12 of + [LEDITS++ Paper](https://arxiv.org/abs/2301.12247). + edit_warmup_steps (`float` or `List[float]`, *optional*, defaults to 10): + Number of diffusion steps (for each prompt) for which guidance will not be applied. + edit_cooldown_steps (`float` or `List[float]`, *optional*, defaults to `None`): + Number of diffusion steps (for each prompt) after which guidance will no longer be applied. + edit_threshold (`float` or `List[float]`, *optional*, defaults to 0.9): + Masking threshold of guidance. Threshold should be proportional to the image region that is modified. + 'edit_threshold' is defined as 'λ' of equation 12 of [LEDITS++ Paper](https://arxiv.org/abs/2301.12247). + user_mask (`torch.FloatTensor`, *optional*): + User-provided mask for even better control over the editing process. This is helpful when LEDITS++'s implicit + masks do not meet user preferences. + sem_guidance (`List[torch.Tensor]`, *optional*): + List of pre-generated guidance vectors to be applied at generation. Length of the list has to + correspond to `num_inference_steps`. + use_cross_attn_mask (`bool`, defaults to `False`): + Whether cross-attention masks are used. Cross-attention masks are always used when use_intersect_mask + is set to true. Cross-attention masks are defined as 'M^1' of equation 12 of + [LEDITS++ paper](https://arxiv.org/pdf/2311.16711.pdf). + use_intersect_mask (`bool`, defaults to `True`): + Whether the masking term is calculated as intersection of cross-attention masks and masks derived + from the noise estimate. Cross-attention mask are defined as 'M^1' and masks derived from the noise + estimate are defined as 'M^2' of equation 12 of [LEDITS++ paper](https://arxiv.org/pdf/2311.16711.pdf). + attn_store_steps (`List[int]`, *optional*): + Steps for which the attention maps are stored in the AttentionStore. Just for visualization purposes. + store_averaged_over_steps (`bool`, defaults to `True`): + Whether the attention maps for the 'attn_store_steps' are stored averaged over the diffusion steps. + If False, attention maps for each step are stores separately. Just for visualization purposes. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.0): + Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf). Guidance rescale factor should fix overexposure when + using zero terminal SNR. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.ledits_pp.LEditsPPDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.ledits_pp.LEditsPPDiffusionPipelineOutput`] if `return_dict` is True, + otherwise a `tuple. When returning a tuple, the first element is a list with the generated images, and the + second element is a list of `bool`s denoting whether the corresponding generated image likely represents + "not-safe-for-work" (nsfw) content, according to the `safety_checker`. + """ + + if self.inversion_steps is None: + raise ValueError( + "You need to invert an input image first before calling the pipeline. The `invert` method has to be called beforehand. Edits will always be performed for the last inverted image(s)." + ) + + eta = self.eta + num_images_per_prompt = 1 + latents = self.init_latents + + zs = self.zs + self.scheduler.set_timesteps(len(self.scheduler.timesteps)) + + if use_intersect_mask: + use_cross_attn_mask = True + + if use_cross_attn_mask: + self.smoothing = LeditsGaussianSmoothing(self.device) + + if user_mask is not None: + user_mask = user_mask.to(self.device) + + org_prompt = "" + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + negative_prompt, + editing_prompt_embeds, + negative_prompt_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_rescale = guidance_rescale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + batch_size = self.batch_size + + if editing_prompt: + enable_edit_guidance = True + if isinstance(editing_prompt, str): + editing_prompt = [editing_prompt] + self.enabled_editing_prompts = len(editing_prompt) + elif editing_prompt_embeds is not None: + enable_edit_guidance = True + self.enabled_editing_prompts = editing_prompt_embeds.shape[0] + else: + self.enabled_editing_prompts = 0 + enable_edit_guidance = False + + # 3. Encode input prompt + lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + + edit_concepts, uncond_embeddings, num_edit_tokens = self.encode_prompt( + editing_prompt=editing_prompt, + device=self.device, + num_images_per_prompt=num_images_per_prompt, + enable_edit_guidance=enable_edit_guidance, + negative_prompt=negative_prompt, + editing_prompt_embeds=editing_prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + clip_skip=self.clip_skip, + ) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if enable_edit_guidance: + text_embeddings = torch.cat([uncond_embeddings, edit_concepts]) + self.text_cross_attention_maps = [editing_prompt] if isinstance(editing_prompt, str) else editing_prompt + else: + text_embeddings = torch.cat([uncond_embeddings]) + + # 4. Prepare timesteps + # self.scheduler.set_timesteps(num_inference_steps, device=self.device) + timesteps = self.inversion_steps + t_to_idx = {int(v): k for k, v in enumerate(timesteps[-zs.shape[0] :])} + + if use_cross_attn_mask: + self.attention_store = LeditsAttentionStore( + average=store_averaged_over_steps, + batch_size=batch_size, + max_size=(latents.shape[-2] / 4.0) * (latents.shape[-1] / 4.0), + max_resolution=None, + ) + self.prepare_unet(self.attention_store, PnP=False) + resolution = latents.shape[-2:] + att_res = (int(resolution[0] / 4), int(resolution[1] / 4)) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + None, + None, + text_embeddings.dtype, + self.device, + latents, + ) + + # 6. Prepare extra step kwargs. + extra_step_kwargs = self.prepare_extra_step_kwargs(eta) + + self.sem_guidance = None + self.activation_mask = None + + # 7. Denoising loop + num_warmup_steps = 0 + with self.progress_bar(total=len(timesteps)) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + + if enable_edit_guidance: + latent_model_input = torch.cat([latents] * (1 + self.enabled_editing_prompts)) + else: + latent_model_input = latents + + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + text_embed_input = text_embeddings + + # predict the noise residual + noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embed_input).sample + + noise_pred_out = noise_pred.chunk(1 + self.enabled_editing_prompts) # [b,4, 64, 64] + noise_pred_uncond = noise_pred_out[0] + noise_pred_edit_concepts = noise_pred_out[1:] + + noise_guidance_edit = torch.zeros( + noise_pred_uncond.shape, + device=self.device, + dtype=noise_pred_uncond.dtype, + ) + + if sem_guidance is not None and len(sem_guidance) > i: + noise_guidance_edit += sem_guidance[i].to(self.device) + + elif enable_edit_guidance: + if self.activation_mask is None: + self.activation_mask = torch.zeros( + (len(timesteps), len(noise_pred_edit_concepts), *noise_pred_edit_concepts[0].shape) + ) + + if self.sem_guidance is None: + self.sem_guidance = torch.zeros((len(timesteps), *noise_pred_uncond.shape)) + + for c, noise_pred_edit_concept in enumerate(noise_pred_edit_concepts): + if isinstance(edit_warmup_steps, list): + edit_warmup_steps_c = edit_warmup_steps[c] + else: + edit_warmup_steps_c = edit_warmup_steps + if i < edit_warmup_steps_c: + continue + + if isinstance(edit_guidance_scale, list): + edit_guidance_scale_c = edit_guidance_scale[c] + else: + edit_guidance_scale_c = edit_guidance_scale + + if isinstance(edit_threshold, list): + edit_threshold_c = edit_threshold[c] + else: + edit_threshold_c = edit_threshold + if isinstance(reverse_editing_direction, list): + reverse_editing_direction_c = reverse_editing_direction[c] + else: + reverse_editing_direction_c = reverse_editing_direction + + if isinstance(edit_cooldown_steps, list): + edit_cooldown_steps_c = edit_cooldown_steps[c] + elif edit_cooldown_steps is None: + edit_cooldown_steps_c = i + 1 + else: + edit_cooldown_steps_c = edit_cooldown_steps + + if i >= edit_cooldown_steps_c: + continue + + noise_guidance_edit_tmp = noise_pred_edit_concept - noise_pred_uncond + + if reverse_editing_direction_c: + noise_guidance_edit_tmp = noise_guidance_edit_tmp * -1 + + noise_guidance_edit_tmp = noise_guidance_edit_tmp * edit_guidance_scale_c + + if user_mask is not None: + noise_guidance_edit_tmp = noise_guidance_edit_tmp * user_mask + + if use_cross_attn_mask: + out = self.attention_store.aggregate_attention( + attention_maps=self.attention_store.step_store, + prompts=self.text_cross_attention_maps, + res=att_res, + from_where=["up", "down"], + is_cross=True, + select=self.text_cross_attention_maps.index(editing_prompt[c]), + ) + attn_map = out[:, :, :, 1 : 1 + num_edit_tokens[c]] # 0 -> startoftext + + # average over all tokens + if attn_map.shape[3] != num_edit_tokens[c]: + raise ValueError( + f"Incorrect shape of attention_map. Expected size {num_edit_tokens[c]}, but found {attn_map.shape[3]}!" + ) + + attn_map = torch.sum(attn_map, dim=3) + + # gaussian_smoothing + attn_map = F.pad(attn_map.unsqueeze(1), (1, 1, 1, 1), mode="reflect") + attn_map = self.smoothing(attn_map).squeeze(1) + + # torch.quantile function expects float32 + if attn_map.dtype == torch.float32: + tmp = torch.quantile(attn_map.flatten(start_dim=1), edit_threshold_c, dim=1) + else: + tmp = torch.quantile( + attn_map.flatten(start_dim=1).to(torch.float32), edit_threshold_c, dim=1 + ).to(attn_map.dtype) + attn_mask = torch.where( + attn_map >= tmp.unsqueeze(1).unsqueeze(1).repeat(1, *att_res), 1.0, 0.0 + ) + + # resolution must match latent space dimension + attn_mask = F.interpolate( + attn_mask.unsqueeze(1), + noise_guidance_edit_tmp.shape[-2:], # 64,64 + ).repeat(1, 4, 1, 1) + self.activation_mask[i, c] = attn_mask.detach().cpu() + if not use_intersect_mask: + noise_guidance_edit_tmp = noise_guidance_edit_tmp * attn_mask + + if use_intersect_mask: + if t <= 800: + noise_guidance_edit_tmp_quantile = torch.abs(noise_guidance_edit_tmp) + noise_guidance_edit_tmp_quantile = torch.sum( + noise_guidance_edit_tmp_quantile, dim=1, keepdim=True + ) + noise_guidance_edit_tmp_quantile = noise_guidance_edit_tmp_quantile.repeat( + 1, self.unet.config.in_channels, 1, 1 + ) + + # torch.quantile function expects float32 + if noise_guidance_edit_tmp_quantile.dtype == torch.float32: + tmp = torch.quantile( + noise_guidance_edit_tmp_quantile.flatten(start_dim=2), + edit_threshold_c, + dim=2, + keepdim=False, + ) + else: + tmp = torch.quantile( + noise_guidance_edit_tmp_quantile.flatten(start_dim=2).to(torch.float32), + edit_threshold_c, + dim=2, + keepdim=False, + ).to(noise_guidance_edit_tmp_quantile.dtype) + + intersect_mask = ( + torch.where( + noise_guidance_edit_tmp_quantile >= tmp[:, :, None, None], + torch.ones_like(noise_guidance_edit_tmp), + torch.zeros_like(noise_guidance_edit_tmp), + ) + * attn_mask + ) + + self.activation_mask[i, c] = intersect_mask.detach().cpu() + + noise_guidance_edit_tmp = noise_guidance_edit_tmp * intersect_mask + + else: + # print(f"only attention mask for step {i}") + noise_guidance_edit_tmp = noise_guidance_edit_tmp * attn_mask + + elif not use_cross_attn_mask: + # calculate quantile + noise_guidance_edit_tmp_quantile = torch.abs(noise_guidance_edit_tmp) + noise_guidance_edit_tmp_quantile = torch.sum( + noise_guidance_edit_tmp_quantile, dim=1, keepdim=True + ) + noise_guidance_edit_tmp_quantile = noise_guidance_edit_tmp_quantile.repeat(1, 4, 1, 1) + + # torch.quantile function expects float32 + if noise_guidance_edit_tmp_quantile.dtype == torch.float32: + tmp = torch.quantile( + noise_guidance_edit_tmp_quantile.flatten(start_dim=2), + edit_threshold_c, + dim=2, + keepdim=False, + ) + else: + tmp = torch.quantile( + noise_guidance_edit_tmp_quantile.flatten(start_dim=2).to(torch.float32), + edit_threshold_c, + dim=2, + keepdim=False, + ).to(noise_guidance_edit_tmp_quantile.dtype) + + self.activation_mask[i, c] = ( + torch.where( + noise_guidance_edit_tmp_quantile >= tmp[:, :, None, None], + torch.ones_like(noise_guidance_edit_tmp), + torch.zeros_like(noise_guidance_edit_tmp), + ) + .detach() + .cpu() + ) + + noise_guidance_edit_tmp = torch.where( + noise_guidance_edit_tmp_quantile >= tmp[:, :, None, None], + noise_guidance_edit_tmp, + torch.zeros_like(noise_guidance_edit_tmp), + ) + + noise_guidance_edit += noise_guidance_edit_tmp + + self.sem_guidance[i] = noise_guidance_edit.detach().cpu() + + noise_pred = noise_pred_uncond + noise_guidance_edit + + if enable_edit_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg( + noise_pred, + noise_pred_edit_concepts.mean(dim=0, keepdim=False), + guidance_rescale=self.guidance_rescale, + ) + + idx = t_to_idx[int(t)] + latents = self.scheduler.step( + noise_pred, t, latents, variance_noise=zs[idx], **extra_step_kwargs + ).prev_sample + + # step callback + if use_cross_attn_mask: + store_step = i in attn_store_steps + self.attention_store.between_steps(store_step) + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + # prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + # 8. Post-processing + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ + 0 + ] + image, has_nsfw_concept = self.run_safety_checker(image, self.device, text_embeddings.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return LEditsPPDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) + + @torch.no_grad() + def invert( + self, + image: PipelineImageInput, + source_prompt: str = "", + source_guidance_scale: float = 3.5, + num_inversion_steps: int = 30, + skip: float = 0.15, + generator: Optional[torch.Generator] = None, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + height: Optional[int] = None, + width: Optional[int] = None, + resize_mode: Optional[str] = "default", + crops_coords: Optional[Tuple[int, int, int, int]] = None, + ): + r""" + The function to the pipeline for image inversion as described by the [LEDITS++ Paper](https://arxiv.org/abs/2301.12247). + If the scheduler is set to [`~schedulers.DDIMScheduler`] the inversion proposed by [edit-friendly DPDM](https://arxiv.org/abs/2304.06140) + will be performed instead. + + Args: + image (`PipelineImageInput`): + Input for the image(s) that are to be edited. Multiple input images have to default to the same aspect + ratio. + source_prompt (`str`, defaults to `""`): + Prompt describing the input image that will be used for guidance during inversion. Guidance is disabled + if the `source_prompt` is `""`. + source_guidance_scale (`float`, defaults to `3.5`): + Strength of guidance during inversion. + num_inversion_steps (`int`, defaults to `30`): + Number of total performed inversion steps after discarding the initial `skip` steps. + skip (`float`, defaults to `0.15`): + Portion of initial steps that will be ignored for inversion and subsequent generation. Lower values + will lead to stronger changes to the input image. `skip` has to be between `0` and `1`. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + inversion deterministic. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + height (`int`, *optional*, defaults to `None`): + The height in preprocessed image. If `None`, will use the `get_default_height_width()` to get default height. + width (`int`, *optional*`, defaults to `None`): + The width in preprocessed. If `None`, will use get_default_height_width()` to get the default width. + resize_mode (`str`, *optional*, defaults to `default`): + The resize mode, can be one of `default` or `fill`. If `default`, will resize the image to fit + within the specified width and height, and it may not maintaining the original aspect ratio. + If `fill`, will resize the image to fit within the specified width and height, maintaining the aspect ratio, and then center the image + within the dimensions, filling empty with data from image. + If `crop`, will resize the image to fit within the specified width and height, maintaining the aspect ratio, and then center the image + within the dimensions, cropping the excess. + Note that resize_mode `fill` and `crop` are only supported for PIL image input. + crops_coords (`List[Tuple[int, int, int, int]]`, *optional*, defaults to `None`): + The crop coordinates for each image in the batch. If `None`, will not crop the image. + + Returns: + [`~pipelines.ledits_pp.LEditsPPInversionPipelineOutput`]: + Output will contain the resized input image(s) and respective VAE reconstruction(s). + """ + # Reset attn processor, we do not want to store attn maps during inversion + self.unet.set_attn_processor(AttnProcessor()) + + self.eta = 1.0 + + self.scheduler.config.timestep_spacing = "leading" + self.scheduler.set_timesteps(int(num_inversion_steps * (1 + skip))) + self.inversion_steps = self.scheduler.timesteps[-num_inversion_steps:] + timesteps = self.inversion_steps + + # 1. encode image + x0, resized = self.encode_image( + image, + dtype=self.text_encoder.dtype, + height=height, + width=width, + resize_mode=resize_mode, + crops_coords=crops_coords, + ) + self.batch_size = x0.shape[0] + + # autoencoder reconstruction + image_rec = self.vae.decode(x0 / self.vae.config.scaling_factor, return_dict=False, generator=generator)[0] + image_rec = self.image_processor.postprocess(image_rec, output_type="pil") + + # 2. get embeddings + do_classifier_free_guidance = source_guidance_scale > 1.0 + + lora_scale = cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + + uncond_embedding, text_embeddings, _ = self.encode_prompt( + num_images_per_prompt=1, + device=self.device, + negative_prompt=None, + enable_edit_guidance=do_classifier_free_guidance, + editing_prompt=source_prompt, + lora_scale=lora_scale, + clip_skip=clip_skip, + ) + + # 3. find zs and xts + variance_noise_shape = (num_inversion_steps, *x0.shape) + + # intermediate latents + t_to_idx = {int(v): k for k, v in enumerate(timesteps)} + xts = torch.zeros(size=variance_noise_shape, device=self.device, dtype=uncond_embedding.dtype) + + for t in reversed(timesteps): + idx = num_inversion_steps - t_to_idx[int(t)] - 1 + noise = randn_tensor(shape=x0.shape, generator=generator, device=self.device, dtype=x0.dtype) + xts[idx] = self.scheduler.add_noise(x0, noise, torch.Tensor([t])) + xts = torch.cat([x0.unsqueeze(0), xts], dim=0) + + self.scheduler.set_timesteps(len(self.scheduler.timesteps)) + # noise maps + zs = torch.zeros(size=variance_noise_shape, device=self.device, dtype=uncond_embedding.dtype) + + with self.progress_bar(total=len(timesteps)) as progress_bar: + for t in timesteps: + idx = num_inversion_steps - t_to_idx[int(t)] - 1 + # 1. predict noise residual + xt = xts[idx + 1] + + noise_pred = self.unet(xt, timestep=t, encoder_hidden_states=uncond_embedding).sample + + if not source_prompt == "": + noise_pred_cond = self.unet(xt, timestep=t, encoder_hidden_states=text_embeddings).sample + noise_pred = noise_pred + source_guidance_scale * (noise_pred_cond - noise_pred) + + xtm1 = xts[idx] + z, xtm1_corrected = compute_noise(self.scheduler, xtm1, xt, t, noise_pred, self.eta) + zs[idx] = z + + # correction to avoid error accumulation + xts[idx] = xtm1_corrected + + progress_bar.update() + + self.init_latents = xts[-1].expand(self.batch_size, -1, -1, -1) + zs = zs.flip(0) + self.zs = zs + + return LEditsPPInversionPipelineOutput(images=resized, vae_reconstruction_images=image_rec) + + @torch.no_grad() + def encode_image(self, image, dtype=None, height=None, width=None, resize_mode="default", crops_coords=None): + image = self.image_processor.preprocess( + image=image, height=height, width=width, resize_mode=resize_mode, crops_coords=crops_coords + ) + resized = self.image_processor.postprocess(image=image, output_type="pil") + + if max(image.shape[-2:]) > self.vae.config["sample_size"] * 1.5: + logger.warning( + "Your input images far exceed the default resolution of the underlying diffusion model. " + "The output images may contain severe artifacts! " + "Consider down-sampling the input using the `height` and `width` parameters" + ) + image = image.to(dtype) + + x0 = self.vae.encode(image.to(self.device)).latent_dist.mode() + x0 = x0.to(dtype) + x0 = self.vae.config.scaling_factor * x0 + return x0, resized + + +def compute_noise_ddim(scheduler, prev_latents, latents, timestep, noise_pred, eta): + # 1. get previous step value (=t-1) + prev_timestep = timestep - scheduler.config.num_train_timesteps // scheduler.num_inference_steps + + # 2. compute alphas, betas + alpha_prod_t = scheduler.alphas_cumprod[timestep] + alpha_prod_t_prev = ( + scheduler.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else scheduler.final_alpha_cumprod + ) + + beta_prod_t = 1 - alpha_prod_t + + # 3. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_original_sample = (latents - beta_prod_t ** (0.5) * noise_pred) / alpha_prod_t ** (0.5) + + # 4. Clip "predicted x_0" + if scheduler.config.clip_sample: + pred_original_sample = torch.clamp(pred_original_sample, -1, 1) + + # 5. compute variance: "sigma_t(η)" -> see formula (16) + # σ_t = sqrt((1 − α_t−1)/(1 − α_t)) * sqrt(1 − α_t/α_t−1) + variance = scheduler._get_variance(timestep, prev_timestep) + std_dev_t = eta * variance ** (0.5) + + # 6. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * noise_pred + + # modifed so that updated xtm1 is returned as well (to avoid error accumulation) + mu_xt = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction + if variance > 0.0: + noise = (prev_latents - mu_xt) / (variance ** (0.5) * eta) + else: + noise = torch.tensor([0.0]).to(latents.device) + + return noise, mu_xt + (eta * variance**0.5) * noise + + +def compute_noise_sde_dpm_pp_2nd(scheduler, prev_latents, latents, timestep, noise_pred, eta): + def first_order_update(model_output, sample): # timestep, prev_timestep, sample): + sigma_t, sigma_s = scheduler.sigmas[scheduler.step_index + 1], scheduler.sigmas[scheduler.step_index] + alpha_t, sigma_t = scheduler._sigma_to_alpha_sigma_t(sigma_t) + alpha_s, sigma_s = scheduler._sigma_to_alpha_sigma_t(sigma_s) + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s = torch.log(alpha_s) - torch.log(sigma_s) + + h = lambda_t - lambda_s + + mu_xt = (sigma_t / sigma_s * torch.exp(-h)) * sample + (alpha_t * (1 - torch.exp(-2.0 * h))) * model_output + + mu_xt = scheduler.dpm_solver_first_order_update( + model_output=model_output, sample=sample, noise=torch.zeros_like(sample) + ) + + sigma = sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) + if sigma > 0.0: + noise = (prev_latents - mu_xt) / sigma + else: + noise = torch.tensor([0.0]).to(sample.device) + + prev_sample = mu_xt + sigma * noise + return noise, prev_sample + + def second_order_update(model_output_list, sample): # timestep_list, prev_timestep, sample): + sigma_t, sigma_s0, sigma_s1 = ( + scheduler.sigmas[scheduler.step_index + 1], + scheduler.sigmas[scheduler.step_index], + scheduler.sigmas[scheduler.step_index - 1], + ) + + alpha_t, sigma_t = scheduler._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = scheduler._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = scheduler._sigma_to_alpha_sigma_t(sigma_s1) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + + m0, m1 = model_output_list[-1], model_output_list[-2] + + h, h_0 = lambda_t - lambda_s0, lambda_s0 - lambda_s1 + r0 = h_0 / h + D0, D1 = m0, (1.0 / r0) * (m0 - m1) + + mu_xt = ( + (sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + + 0.5 * (alpha_t * (1 - torch.exp(-2.0 * h))) * D1 + ) + + sigma = sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) + if sigma > 0.0: + noise = (prev_latents - mu_xt) / sigma + else: + noise = torch.tensor([0.0]).to(sample.device) + + prev_sample = mu_xt + sigma * noise + + return noise, prev_sample + + if scheduler.step_index is None: + scheduler._init_step_index(timestep) + + model_output = scheduler.convert_model_output(model_output=noise_pred, sample=latents) + for i in range(scheduler.config.solver_order - 1): + scheduler.model_outputs[i] = scheduler.model_outputs[i + 1] + scheduler.model_outputs[-1] = model_output + + if scheduler.lower_order_nums < 1: + noise, prev_sample = first_order_update(model_output, latents) + else: + noise, prev_sample = second_order_update(scheduler.model_outputs, latents) + + if scheduler.lower_order_nums < scheduler.config.solver_order: + scheduler.lower_order_nums += 1 + + # upon completion increase step index by one + scheduler._step_index += 1 + + return noise, prev_sample + + +def compute_noise(scheduler, *args): + if isinstance(scheduler, DDIMScheduler): + return compute_noise_ddim(scheduler, *args) + elif ( + isinstance(scheduler, DPMSolverMultistepScheduler) + and scheduler.config.algorithm_type == "sde-dpmsolver++" + and scheduler.config.solver_order == 2 + ): + return compute_noise_sde_dpm_pp_2nd(scheduler, *args) + else: + raise NotImplementedError diff --git a/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py b/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py new file mode 100755 index 0000000..874a10a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_leditspp_stable_diffusion_xl.py @@ -0,0 +1,1797 @@ +# Copyright 2023 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import math +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import ( + FromSingleFileMixin, + IPAdapterMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, +) +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.attention_processor import ( + Attention, + AttnProcessor, + AttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import DDIMScheduler, DPMSolverMultistepScheduler +from ...utils import ( + USE_PEFT_BACKEND, + is_invisible_watermark_available, + is_torch_xla_available, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .pipeline_output import LEditsPPDiffusionPipelineOutput, LEditsPPInversionPipelineOutput + + +if is_invisible_watermark_available(): + from ..stable_diffusion_xl.watermark import StableDiffusionXLWatermarker + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> import PIL + >>> import requests + >>> from io import BytesIO + + >>> from diffusers import LEditsPPPipelineStableDiffusionXL + + >>> pipe = LEditsPPPipelineStableDiffusionXL.from_pretrained( + ... "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> def download_image(url): + ... response = requests.get(url) + ... return PIL.Image.open(BytesIO(response.content)).convert("RGB") + + >>> img_url = "https://www.aiml.informatik.tu-darmstadt.de/people/mbrack/tennis.jpg" + >>> image = download_image(img_url) + + >>> _ = pipe.invert( + ... image = image, + ... num_inversion_steps=50, + ... skip=0.2 + ... ) + + >>> edited_image = pipe( + ... editing_prompt=["tennis ball","tomato"], + ... reverse_editing_direction=[True,False], + ... edit_guidance_scale=[5.0,10.0], + ... edit_threshold=[0.9,0.85], + ).images[0] + ``` +""" + + +# Copied from diffusers.pipelines.ledits_pp.pipeline_leditspp_stable_diffusion.LeditsAttentionStore +class LeditsAttentionStore: + @staticmethod + def get_empty_store(): + return {"down_cross": [], "mid_cross": [], "up_cross": [], "down_self": [], "mid_self": [], "up_self": []} + + def __call__(self, attn, is_cross: bool, place_in_unet: str, editing_prompts, PnP=False): + # attn.shape = batch_size * head_size, seq_len query, seq_len_key + if attn.shape[1] <= self.max_size: + bs = 1 + int(PnP) + editing_prompts + skip = 2 if PnP else 1 # skip PnP & unconditional + attn = torch.stack(attn.split(self.batch_size)).permute(1, 0, 2, 3) + source_batch_size = int(attn.shape[1] // bs) + self.forward(attn[:, skip * source_batch_size :], is_cross, place_in_unet) + + def forward(self, attn, is_cross: bool, place_in_unet: str): + key = f"{place_in_unet}_{'cross' if is_cross else 'self'}" + + self.step_store[key].append(attn) + + def between_steps(self, store_step=True): + if store_step: + if self.average: + if len(self.attention_store) == 0: + self.attention_store = self.step_store + else: + for key in self.attention_store: + for i in range(len(self.attention_store[key])): + self.attention_store[key][i] += self.step_store[key][i] + else: + if len(self.attention_store) == 0: + self.attention_store = [self.step_store] + else: + self.attention_store.append(self.step_store) + + self.cur_step += 1 + self.step_store = self.get_empty_store() + + def get_attention(self, step: int): + if self.average: + attention = { + key: [item / self.cur_step for item in self.attention_store[key]] for key in self.attention_store + } + else: + assert step is not None + attention = self.attention_store[step] + return attention + + def aggregate_attention( + self, attention_maps, prompts, res: Union[int, Tuple[int]], from_where: List[str], is_cross: bool, select: int + ): + out = [[] for x in range(self.batch_size)] + if isinstance(res, int): + num_pixels = res**2 + resolution = (res, res) + else: + num_pixels = res[0] * res[1] + resolution = res[:2] + + for location in from_where: + for bs_item in attention_maps[f"{location}_{'cross' if is_cross else 'self'}"]: + for batch, item in enumerate(bs_item): + if item.shape[1] == num_pixels: + cross_maps = item.reshape(len(prompts), -1, *resolution, item.shape[-1])[select] + out[batch].append(cross_maps) + + out = torch.stack([torch.cat(x, dim=0) for x in out]) + # average over heads + out = out.sum(1) / out.shape[1] + return out + + def __init__(self, average: bool, batch_size=1, max_resolution=16, max_size: int = None): + self.step_store = self.get_empty_store() + self.attention_store = [] + self.cur_step = 0 + self.average = average + self.batch_size = batch_size + if max_size is None: + self.max_size = max_resolution**2 + elif max_size is not None and max_resolution is None: + self.max_size = max_size + else: + raise ValueError("Only allowed to set one of max_resolution or max_size") + + +# Copied from diffusers.pipelines.ledits_pp.pipeline_leditspp_stable_diffusion.LeditsGaussianSmoothing +class LeditsGaussianSmoothing: + def __init__(self, device): + kernel_size = [3, 3] + sigma = [0.5, 0.5] + + # The gaussian kernel is the product of the gaussian function of each dimension. + kernel = 1 + meshgrids = torch.meshgrid([torch.arange(size, dtype=torch.float32) for size in kernel_size]) + for size, std, mgrid in zip(kernel_size, sigma, meshgrids): + mean = (size - 1) / 2 + kernel *= 1 / (std * math.sqrt(2 * math.pi)) * torch.exp(-(((mgrid - mean) / (2 * std)) ** 2)) + + # Make sure sum of values in gaussian kernel equals 1. + kernel = kernel / torch.sum(kernel) + + # Reshape to depthwise convolutional weight + kernel = kernel.view(1, 1, *kernel.size()) + kernel = kernel.repeat(1, *[1] * (kernel.dim() - 1)) + + self.weight = kernel.to(device) + + def __call__(self, input): + """ + Arguments: + Apply gaussian filter to input. + input (torch.Tensor): Input to apply gaussian filter on. + Returns: + filtered (torch.Tensor): Filtered output. + """ + return F.conv2d(input, weight=self.weight.to(input.dtype)) + + +# Copied from diffusers.pipelines.ledits_pp.pipeline_leditspp_stable_diffusion.LEDITSCrossAttnProcessor +class LEDITSCrossAttnProcessor: + def __init__(self, attention_store, place_in_unet, pnp, editing_prompts): + self.attnstore = attention_store + self.place_in_unet = place_in_unet + self.editing_prompts = editing_prompts + self.pnp = pnp + + def __call__( + self, + attn: Attention, + hidden_states, + encoder_hidden_states, + attention_mask=None, + temb=None, + ): + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + self.attnstore( + attention_probs, + is_cross=True, + place_in_unet=self.place_in_unet, + editing_prompts=self.editing_prompts, + PnP=self.pnp, + ) + + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + hidden_states = hidden_states / attn.rescale_output_factor + return hidden_states + + +class LEditsPPPipelineStableDiffusionXL( + DiffusionPipeline, + FromSingleFileMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, +): + """ + Pipeline for textual image editing using LEDits++ with Stable Diffusion XL. + + This model inherits from [`DiffusionPipeline`] and builds on the [`StableDiffusionXLPipeline`]. Check the superclass + documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular + device, etc.). + + In addition the pipeline inherits the following loading methods: + - *LoRA*: [`LEditsPPPipelineStableDiffusionXL.load_lora_weights`] + - *Ckpt*: [`loaders.FromSingleFileMixin.from_single_file`] + + as well as the following saving methods: + - *LoRA*: [`loaders.StableDiffusionXLPipeline.save_lora_weights`] + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([`~transformers.CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer ([`~transformers.CLIPTokenizer`]): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 ([`~transformers.CLIPTokenizer`]): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`DPMSolverMultistepScheduler`] or [`DDIMScheduler`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latens. Can be one of + [`DPMSolverMultistepScheduler`] or [`DDIMScheduler`]. If any other scheduler is passed it will automatically + be set to [`DPMSolverMultistepScheduler`]. + force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): + Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of + `stabilityai/stable-diffusion-xl-base-1-0`. + add_watermarker (`bool`, *optional*): + Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to + watermark output images. If not defined, it will default to True if the package is installed, otherwise no + watermarker will be used. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->unet->vae" + _optional_components = [ + "tokenizer", + "tokenizer_2", + "text_encoder", + "text_encoder_2", + "image_encoder", + "feature_extractor", + ] + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + "add_text_embeds", + "add_time_ids", + "negative_pooled_prompt_embeds", + "negative_add_time_ids", + ] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: Union[DPMSolverMultistepScheduler, DDIMScheduler], + image_encoder: CLIPVisionModelWithProjection = None, + feature_extractor: CLIPImageProcessor = None, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + scheduler=scheduler, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + if not isinstance(scheduler, DDIMScheduler) and not isinstance(scheduler, DPMSolverMultistepScheduler): + self.scheduler = DPMSolverMultistepScheduler.from_config( + scheduler.config, algorithm_type="sde-dpmsolver++", solver_order=2 + ) + logger.warning( + "This pipeline only supports DDIMScheduler and DPMSolverMultistepScheduler. " + "The scheduler has been changed to DPMSolverMultistepScheduler." + ) + + self.default_sample_size = self.unet.config.sample_size + + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() + + if add_watermarker: + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None + self.inversion_steps = None + + def encode_prompt( + self, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + enable_edit_guidance: bool = True, + editing_prompt: Optional[str] = None, + editing_prompt_embeds: Optional[torch.FloatTensor] = None, + editing_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ) -> object: + r""" + Encodes the prompt into text encoder hidden states. + + Args: + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + enable_edit_guidance (`bool`): + Whether to guide towards an editing prompt or not. + editing_prompt (`str` or `List[str]`, *optional*): + Editing prompt(s) to be encoded. If not defined and 'enable_edit_guidance' is True, one has to pass + `editing_prompt_embeds` instead. + editing_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated edit text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided and 'enable_edit_guidance' is True, editing_prompt_embeds will be generated from `editing_prompt` input + argument. + editing_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated edit pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled editing_pooled_prompt_embeds will be generated from `editing_prompt` + input argument. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + batch_size = self.batch_size + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + num_edit_tokens = 0 + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + + if negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_2 = ( + batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 + ) + + uncond_tokens: List[str] + + if batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but image inversion " + f" has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of the input images." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + if zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(negative_prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(negative_pooled_prompt_embeds) + + if enable_edit_guidance and editing_prompt_embeds is None: + editing_prompt_2 = editing_prompt + + editing_prompts = [editing_prompt, editing_prompt_2] + edit_prompt_embeds_list = [] + + for editing_prompt, tokenizer, text_encoder in zip(editing_prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + editing_prompt = self.maybe_convert_prompt(editing_prompt, tokenizer) + + max_length = negative_prompt_embeds.shape[1] + edit_concepts_input = tokenizer( + # [x for item in editing_prompt for x in repeat(item, batch_size)], + editing_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + return_length=True, + ) + num_edit_tokens = edit_concepts_input.length - 2 + + edit_concepts_embeds = text_encoder( + edit_concepts_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + editing_pooled_prompt_embeds = edit_concepts_embeds[0] + if clip_skip is None: + edit_concepts_embeds = edit_concepts_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + edit_concepts_embeds = edit_concepts_embeds.hidden_states[-(clip_skip + 2)] + + edit_prompt_embeds_list.append(edit_concepts_embeds) + + edit_concepts_embeds = torch.concat(edit_prompt_embeds_list, dim=-1) + elif not enable_edit_guidance: + edit_concepts_embeds = None + editing_pooled_prompt_embeds = None + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + bs_embed, seq_len, _ = negative_prompt_embeds.shape + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if enable_edit_guidance: + bs_embed_edit, seq_len, _ = edit_concepts_embeds.shape + edit_concepts_embeds = edit_concepts_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + edit_concepts_embeds = edit_concepts_embeds.repeat(1, num_images_per_prompt, 1) + edit_concepts_embeds = edit_concepts_embeds.view(bs_embed_edit * num_images_per_prompt, seq_len, -1) + + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if enable_edit_guidance: + editing_pooled_prompt_embeds = editing_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed_edit * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return ( + negative_prompt_embeds, + edit_concepts_embeds, + negative_pooled_prompt_embeds, + editing_pooled_prompt_embeds, + num_edit_tokens, + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, eta, generator=None): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + negative_prompt=None, + negative_prompt_2=None, + negative_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + ): + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + # Modified from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, device, latents): + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def _get_add_time_ids( + self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None + ): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + return add_time_ids + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def denoising_end(self): + return self._denoising_end + + @property + def num_timesteps(self): + return self._num_timesteps + + # Copied from diffusers.pipelines.ledits_pp.pipeline_leditspp_stable_diffusion.LEditsPPPipelineStableDiffusion.prepare_unet + def prepare_unet(self, attention_store, PnP: bool = False): + attn_procs = {} + for name in self.unet.attn_processors.keys(): + if name.startswith("mid_block"): + place_in_unet = "mid" + elif name.startswith("up_blocks"): + place_in_unet = "up" + elif name.startswith("down_blocks"): + place_in_unet = "down" + else: + continue + + if "attn2" in name and place_in_unet != "mid": + attn_procs[name] = LEDITSCrossAttnProcessor( + attention_store=attention_store, + place_in_unet=place_in_unet, + pnp=PnP, + editing_prompts=self.enabled_editing_prompts, + ) + else: + attn_procs[name] = AttnProcessor() + + self.unet.set_attn_processor(attn_procs) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + denoising_end: Optional[float] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Optional[Tuple[int, int]] = None, + editing_prompt: Optional[Union[str, List[str]]] = None, + editing_prompt_embeddings: Optional[torch.Tensor] = None, + editing_pooled_prompt_embeds: Optional[torch.Tensor] = None, + reverse_editing_direction: Optional[Union[bool, List[bool]]] = False, + edit_guidance_scale: Optional[Union[float, List[float]]] = 5, + edit_warmup_steps: Optional[Union[int, List[int]]] = 0, + edit_cooldown_steps: Optional[Union[int, List[int]]] = None, + edit_threshold: Optional[Union[float, List[float]]] = 0.9, + sem_guidance: Optional[List[torch.Tensor]] = None, + use_cross_attn_mask: bool = False, + use_intersect_mask: bool = False, + user_mask: Optional[torch.FloatTensor] = None, + attn_store_steps: Optional[List[int]] = [], + store_averaged_over_steps: bool = True, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for editing. The [`~pipelines.ledits_pp.LEditsPPPipelineStableDiffusionXL.invert`] + method has to be called beforehand. Edits will always be performed for the last inverted image(s). + + Args: + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise as determined by the discrete timesteps selected by the + scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a + "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): + Optional image input to work with IP Adapters. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.7): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(width, height)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + editing_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. The image is reconstructed by setting + `editing_prompt = None`. Guidance direction of prompt should be specified via `reverse_editing_direction`. + editing_prompt_embeddings (`torch.Tensor`, *optional*): + Pre-generated edit text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, editing_prompt_embeddings will be generated from `editing_prompt` input + argument. + editing_pooled_prompt_embeddings (`torch.Tensor`, *optional*): + Pre-generated pooled edit text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, editing_prompt_embeddings will be generated from `editing_prompt` input + argument. + reverse_editing_direction (`bool` or `List[bool]`, *optional*, defaults to `False`): + Whether the corresponding prompt in `editing_prompt` should be increased or decreased. + edit_guidance_scale (`float` or `List[float]`, *optional*, defaults to 5): + Guidance scale for guiding the image generation. If provided as list values should correspond to `editing_prompt`. + `edit_guidance_scale` is defined as `s_e` of equation 12 of + [LEDITS++ Paper](https://arxiv.org/abs/2301.12247). + edit_warmup_steps (`float` or `List[float]`, *optional*, defaults to 10): + Number of diffusion steps (for each prompt) for which guidance is not applied. + edit_cooldown_steps (`float` or `List[float]`, *optional*, defaults to `None`): + Number of diffusion steps (for each prompt) after which guidance is no longer applied. + edit_threshold (`float` or `List[float]`, *optional*, defaults to 0.9): + Masking threshold of guidance. Threshold should be proportional to the image region that is modified. + 'edit_threshold' is defined as 'λ' of equation 12 of [LEDITS++ Paper](https://arxiv.org/abs/2301.12247). + sem_guidance (`List[torch.Tensor]`, *optional*): + List of pre-generated guidance vectors to be applied at generation. Length of the list has to + correspond to `num_inference_steps`. + use_cross_attn_mask: + Whether cross-attention masks are used. Cross-attention masks are always used when use_intersect_mask + is set to true. Cross-attention masks are defined as 'M^1' of equation 12 of + [LEDITS++ paper](https://arxiv.org/pdf/2311.16711.pdf). + use_intersect_mask: + Whether the masking term is calculated as intersection of cross-attention masks and masks derived + from the noise estimate. Cross-attention mask are defined as 'M^1' and masks derived from the noise + estimate are defined as 'M^2' of equation 12 of [LEDITS++ paper](https://arxiv.org/pdf/2311.16711.pdf). + user_mask: + User-provided mask for even better control over the editing process. This is helpful when LEDITS++'s implicit + masks do not meet user preferences. + attn_store_steps: + Steps for which the attention maps are stored in the AttentionStore. Just for visualization purposes. + store_averaged_over_steps: + Whether the attention maps for the 'attn_store_steps' are stored averaged over the diffusion steps. + If False, attention maps for each step are stores separately. Just for visualization purposes. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.ledits_pp.LEditsPPDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.ledits_pp.LEditsPPDiffusionPipelineOutput`] if `return_dict` is True, + otherwise a `tuple. When returning a tuple, the first element is a list with the generated images. + """ + if self.inversion_steps is None: + raise ValueError( + "You need to invert an input image first before calling the pipeline. The `invert` method has to be called beforehand. Edits will always be performed for the last inverted image(s)." + ) + + eta = self.eta + num_images_per_prompt = 1 + latents = self.init_latents + + zs = self.zs + self.scheduler.set_timesteps(len(self.scheduler.timesteps)) + + if use_intersect_mask: + use_cross_attn_mask = True + + if use_cross_attn_mask: + self.smoothing = LeditsGaussianSmoothing(self.device) + + if user_mask is not None: + user_mask = user_mask.to(self.device) + + # TODO: Check inputs + # 1. Check inputs. Raise error if not correct + # self.check_inputs( + # callback_steps, + # negative_prompt, + # negative_prompt_2, + # prompt_embeds, + # negative_prompt_embeds, + # pooled_prompt_embeds, + # negative_pooled_prompt_embeds, + # ) + self._guidance_rescale = guidance_rescale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._denoising_end = denoising_end + + # 2. Define call parameters + batch_size = self.batch_size + + device = self._execution_device + + if editing_prompt: + enable_edit_guidance = True + if isinstance(editing_prompt, str): + editing_prompt = [editing_prompt] + self.enabled_editing_prompts = len(editing_prompt) + elif editing_prompt_embeddings is not None: + enable_edit_guidance = True + self.enabled_editing_prompts = editing_prompt_embeddings.shape[0] + else: + self.enabled_editing_prompts = 0 + enable_edit_guidance = False + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + ( + prompt_embeds, + edit_prompt_embeds, + negative_pooled_prompt_embeds, + pooled_edit_embeds, + num_edit_tokens, + ) = self.encode_prompt( + device=device, + num_images_per_prompt=num_images_per_prompt, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + negative_prompt_embeds=negative_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + enable_edit_guidance=enable_edit_guidance, + editing_prompt=editing_prompt, + editing_prompt_embeds=editing_prompt_embeddings, + editing_pooled_prompt_embeds=editing_pooled_prompt_embeds, + ) + + # 4. Prepare timesteps + # self.scheduler.set_timesteps(num_inference_steps, device=device) + + timesteps = self.inversion_steps + t_to_idx = {int(v): k for k, v in enumerate(timesteps)} + + if use_cross_attn_mask: + self.attention_store = LeditsAttentionStore( + average=store_averaged_over_steps, + batch_size=batch_size, + max_size=(latents.shape[-2] / 4.0) * (latents.shape[-1] / 4.0), + max_resolution=None, + ) + self.prepare_unet(self.attention_store) + resolution = latents.shape[-2:] + att_res = (int(resolution[0] / 4), int(resolution[1] / 4)) + + # 5. Prepare latent variables + latents = self.prepare_latents(device=device, latents=latents) + + # 6. Prepare extra step kwargs. + extra_step_kwargs = self.prepare_extra_step_kwargs(eta) + + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(negative_pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + # 7. Prepare added time ids & embeddings + add_text_embeds = negative_pooled_prompt_embeds + add_time_ids = self._get_add_time_ids( + self.size, + crops_coords_top_left, + self.size, + dtype=negative_pooled_prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + + if enable_edit_guidance: + prompt_embeds = torch.cat([prompt_embeds, edit_prompt_embeds], dim=0) + add_text_embeds = torch.cat([add_text_embeds, pooled_edit_embeds], dim=0) + edit_concepts_time_ids = add_time_ids.repeat(edit_prompt_embeds.shape[0], 1) + add_time_ids = torch.cat([add_time_ids, edit_concepts_time_ids], dim=0) + self.text_cross_attention_maps = [editing_prompt] if isinstance(editing_prompt, str) else editing_prompt + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1) + + if ip_adapter_image is not None: + # TODO: fix image encoding + image_embeds, negative_image_embeds = self.encode_image(ip_adapter_image, device, num_images_per_prompt) + if self.do_classifier_free_guidance: + image_embeds = torch.cat([negative_image_embeds, image_embeds]) + image_embeds = image_embeds.to(device) + + # 8. Denoising loop + self.sem_guidance = None + self.activation_mask = None + + if ( + self.denoising_end is not None + and isinstance(self.denoising_end, float) + and self.denoising_end > 0 + and self.denoising_end < 1 + ): + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (self.denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + # 9. Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + self._num_timesteps = len(timesteps) + with self.progress_bar(total=self._num_timesteps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * (1 + self.enabled_editing_prompts)) + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + if ip_adapter_image is not None: + added_cond_kwargs["image_embeds"] = image_embeds + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + noise_pred_out = noise_pred.chunk(1 + self.enabled_editing_prompts) # [b,4, 64, 64] + noise_pred_uncond = noise_pred_out[0] + noise_pred_edit_concepts = noise_pred_out[1:] + + noise_guidance_edit = torch.zeros( + noise_pred_uncond.shape, + device=self.device, + dtype=noise_pred_uncond.dtype, + ) + + if sem_guidance is not None and len(sem_guidance) > i: + noise_guidance_edit += sem_guidance[i].to(self.device) + + elif enable_edit_guidance: + if self.activation_mask is None: + self.activation_mask = torch.zeros( + (len(timesteps), self.enabled_editing_prompts, *noise_pred_edit_concepts[0].shape) + ) + if self.sem_guidance is None: + self.sem_guidance = torch.zeros((len(timesteps), *noise_pred_uncond.shape)) + + # noise_guidance_edit = torch.zeros_like(noise_guidance) + for c, noise_pred_edit_concept in enumerate(noise_pred_edit_concepts): + if isinstance(edit_warmup_steps, list): + edit_warmup_steps_c = edit_warmup_steps[c] + else: + edit_warmup_steps_c = edit_warmup_steps + if i < edit_warmup_steps_c: + continue + + if isinstance(edit_guidance_scale, list): + edit_guidance_scale_c = edit_guidance_scale[c] + else: + edit_guidance_scale_c = edit_guidance_scale + + if isinstance(edit_threshold, list): + edit_threshold_c = edit_threshold[c] + else: + edit_threshold_c = edit_threshold + if isinstance(reverse_editing_direction, list): + reverse_editing_direction_c = reverse_editing_direction[c] + else: + reverse_editing_direction_c = reverse_editing_direction + + if isinstance(edit_cooldown_steps, list): + edit_cooldown_steps_c = edit_cooldown_steps[c] + elif edit_cooldown_steps is None: + edit_cooldown_steps_c = i + 1 + else: + edit_cooldown_steps_c = edit_cooldown_steps + + if i >= edit_cooldown_steps_c: + continue + + noise_guidance_edit_tmp = noise_pred_edit_concept - noise_pred_uncond + + if reverse_editing_direction_c: + noise_guidance_edit_tmp = noise_guidance_edit_tmp * -1 + + noise_guidance_edit_tmp = noise_guidance_edit_tmp * edit_guidance_scale_c + + if user_mask is not None: + noise_guidance_edit_tmp = noise_guidance_edit_tmp * user_mask + + if use_cross_attn_mask: + out = self.attention_store.aggregate_attention( + attention_maps=self.attention_store.step_store, + prompts=self.text_cross_attention_maps, + res=att_res, + from_where=["up", "down"], + is_cross=True, + select=self.text_cross_attention_maps.index(editing_prompt[c]), + ) + attn_map = out[:, :, :, 1 : 1 + num_edit_tokens[c]] # 0 -> startoftext + + # average over all tokens + if attn_map.shape[3] != num_edit_tokens[c]: + raise ValueError( + f"Incorrect shape of attention_map. Expected size {num_edit_tokens[c]}, but found {attn_map.shape[3]}!" + ) + attn_map = torch.sum(attn_map, dim=3) + + # gaussian_smoothing + attn_map = F.pad(attn_map.unsqueeze(1), (1, 1, 1, 1), mode="reflect") + attn_map = self.smoothing(attn_map).squeeze(1) + + # torch.quantile function expects float32 + if attn_map.dtype == torch.float32: + tmp = torch.quantile(attn_map.flatten(start_dim=1), edit_threshold_c, dim=1) + else: + tmp = torch.quantile( + attn_map.flatten(start_dim=1).to(torch.float32), edit_threshold_c, dim=1 + ).to(attn_map.dtype) + attn_mask = torch.where( + attn_map >= tmp.unsqueeze(1).unsqueeze(1).repeat(1, *att_res), 1.0, 0.0 + ) + + # resolution must match latent space dimension + attn_mask = F.interpolate( + attn_mask.unsqueeze(1), + noise_guidance_edit_tmp.shape[-2:], # 64,64 + ).repeat(1, 4, 1, 1) + self.activation_mask[i, c] = attn_mask.detach().cpu() + if not use_intersect_mask: + noise_guidance_edit_tmp = noise_guidance_edit_tmp * attn_mask + + if use_intersect_mask: + noise_guidance_edit_tmp_quantile = torch.abs(noise_guidance_edit_tmp) + noise_guidance_edit_tmp_quantile = torch.sum( + noise_guidance_edit_tmp_quantile, dim=1, keepdim=True + ) + noise_guidance_edit_tmp_quantile = noise_guidance_edit_tmp_quantile.repeat( + 1, self.unet.config.in_channels, 1, 1 + ) + + # torch.quantile function expects float32 + if noise_guidance_edit_tmp_quantile.dtype == torch.float32: + tmp = torch.quantile( + noise_guidance_edit_tmp_quantile.flatten(start_dim=2), + edit_threshold_c, + dim=2, + keepdim=False, + ) + else: + tmp = torch.quantile( + noise_guidance_edit_tmp_quantile.flatten(start_dim=2).to(torch.float32), + edit_threshold_c, + dim=2, + keepdim=False, + ).to(noise_guidance_edit_tmp_quantile.dtype) + + intersect_mask = ( + torch.where( + noise_guidance_edit_tmp_quantile >= tmp[:, :, None, None], + torch.ones_like(noise_guidance_edit_tmp), + torch.zeros_like(noise_guidance_edit_tmp), + ) + * attn_mask + ) + + self.activation_mask[i, c] = intersect_mask.detach().cpu() + + noise_guidance_edit_tmp = noise_guidance_edit_tmp * intersect_mask + + elif not use_cross_attn_mask: + # calculate quantile + noise_guidance_edit_tmp_quantile = torch.abs(noise_guidance_edit_tmp) + noise_guidance_edit_tmp_quantile = torch.sum( + noise_guidance_edit_tmp_quantile, dim=1, keepdim=True + ) + noise_guidance_edit_tmp_quantile = noise_guidance_edit_tmp_quantile.repeat(1, 4, 1, 1) + + # torch.quantile function expects float32 + if noise_guidance_edit_tmp_quantile.dtype == torch.float32: + tmp = torch.quantile( + noise_guidance_edit_tmp_quantile.flatten(start_dim=2), + edit_threshold_c, + dim=2, + keepdim=False, + ) + else: + tmp = torch.quantile( + noise_guidance_edit_tmp_quantile.flatten(start_dim=2).to(torch.float32), + edit_threshold_c, + dim=2, + keepdim=False, + ).to(noise_guidance_edit_tmp_quantile.dtype) + + self.activation_mask[i, c] = ( + torch.where( + noise_guidance_edit_tmp_quantile >= tmp[:, :, None, None], + torch.ones_like(noise_guidance_edit_tmp), + torch.zeros_like(noise_guidance_edit_tmp), + ) + .detach() + .cpu() + ) + + noise_guidance_edit_tmp = torch.where( + noise_guidance_edit_tmp_quantile >= tmp[:, :, None, None], + noise_guidance_edit_tmp, + torch.zeros_like(noise_guidance_edit_tmp), + ) + + noise_guidance_edit += noise_guidance_edit_tmp + + self.sem_guidance[i] = noise_guidance_edit.detach().cpu() + + noise_pred = noise_pred_uncond + noise_guidance_edit + + # compute the previous noisy sample x_t -> x_t-1 + if enable_edit_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg( + noise_pred, + noise_pred_edit_concepts.mean(dim=0, keepdim=False), + guidance_rescale=self.guidance_rescale, + ) + + idx = t_to_idx[int(t)] + latents = self.scheduler.step( + noise_pred, t, latents, variance_noise=zs[idx], **extra_step_kwargs, return_dict=False + )[0] + + # step callback + if use_cross_attn_mask: + store_step = i in attn_store_steps + self.attention_store.between_steps(store_step) + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds) + negative_pooled_prompt_embeds = callback_outputs.pop( + "negative_pooled_prompt_embeds", negative_pooled_prompt_embeds + ) + add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids) + # negative_add_time_ids = callback_outputs.pop("negative_add_time_ids", negative_add_time_ids) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > 0 and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if XLA_AVAILABLE: + xm.mark_step() + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + image = latents + + if not output_type == "latent": + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return LEditsPPDiffusionPipelineOutput(images=image, nsfw_content_detected=None) + + @torch.no_grad() + # Modified from diffusers.pipelines.ledits_pp.pipeline_leditspp_stable_diffusion.LEditsPPPipelineStableDiffusion.encode_image + def encode_image(self, image, dtype=None, height=None, width=None, resize_mode="default", crops_coords=None): + image = self.image_processor.preprocess( + image=image, height=height, width=width, resize_mode=resize_mode, crops_coords=crops_coords + ) + resized = self.image_processor.postprocess(image=image, output_type="pil") + + if max(image.shape[-2:]) > self.vae.config["sample_size"] * 1.5: + logger.warning( + "Your input images far exceed the default resolution of the underlying diffusion model. " + "The output images may contain severe artifacts! " + "Consider down-sampling the input using the `height` and `width` parameters" + ) + image = image.to(self.device, dtype=dtype) + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + image = image.float() + self.upcast_vae() + image = image.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + x0 = self.vae.encode(image).latent_dist.mode() + x0 = x0.to(dtype) + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + + x0 = self.vae.config.scaling_factor * x0 + return x0, resized + + @torch.no_grad() + def invert( + self, + image: PipelineImageInput, + source_prompt: str = "", + source_guidance_scale=3.5, + negative_prompt: str = None, + negative_prompt_2: str = None, + num_inversion_steps: int = 50, + skip: float = 0.15, + generator: Optional[torch.Generator] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + num_zero_noise_steps: int = 3, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + ): + r""" + The function to the pipeline for image inversion as described by the [LEDITS++ Paper](https://arxiv.org/abs/2301.12247). + If the scheduler is set to [`~schedulers.DDIMScheduler`] the inversion proposed by [edit-friendly DPDM](https://arxiv.org/abs/2304.06140) + will be performed instead. + + Args: + image (`PipelineImageInput`): + Input for the image(s) that are to be edited. Multiple input images have to default to the same aspect + ratio. + source_prompt (`str`, defaults to `""`): + Prompt describing the input image that will be used for guidance during inversion. Guidance is disabled + if the `source_prompt` is `""`. + source_guidance_scale (`float`, defaults to `3.5`): + Strength of guidance during inversion. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_inversion_steps (`int`, defaults to `50`): + Number of total performed inversion steps after discarding the initial `skip` steps. + skip (`float`, defaults to `0.15`): + Portion of initial steps that will be ignored for inversion and subsequent generation. Lower values + will lead to stronger changes to the input image. `skip` has to be between `0` and `1`. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + inversion deterministic. + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + num_zero_noise_steps (`int`, defaults to `3`): + Number of final diffusion steps that will not renoise the current image. If no steps are set to zero + SD-XL in combination with [`DPMSolverMultistepScheduler`] will produce noise artifacts. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + Returns: + [`~pipelines.ledits_pp.LEditsPPInversionPipelineOutput`]: + Output will contain the resized input image(s) and respective VAE reconstruction(s). + """ + + # Reset attn processor, we do not want to store attn maps during inversion + self.unet.set_attn_processor(AttnProcessor()) + + self.eta = 1.0 + + self.scheduler.config.timestep_spacing = "leading" + self.scheduler.set_timesteps(int(num_inversion_steps * (1 + skip))) + self.inversion_steps = self.scheduler.timesteps[-num_inversion_steps:] + timesteps = self.inversion_steps + + num_images_per_prompt = 1 + + device = self._execution_device + + # 0. Ensure that only uncond embedding is used if prompt = "" + if source_prompt == "": + # noise pred should only be noise_pred_uncond + source_guidance_scale = 0.0 + do_classifier_free_guidance = False + else: + do_classifier_free_guidance = source_guidance_scale > 1.0 + + # 1. prepare image + x0, resized = self.encode_image(image, dtype=self.text_encoder_2.dtype) + width = x0.shape[2] * self.vae_scale_factor + height = x0.shape[3] * self.vae_scale_factor + self.size = (height, width) + + self.batch_size = x0.shape[0] + + # 2. get embeddings + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + + if isinstance(source_prompt, str): + source_prompt = [source_prompt] * self.batch_size + + ( + negative_prompt_embeds, + prompt_embeds, + negative_pooled_prompt_embeds, + edit_pooled_prompt_embeds, + _, + ) = self.encode_prompt( + device=device, + num_images_per_prompt=num_images_per_prompt, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + editing_prompt=source_prompt, + lora_scale=text_encoder_lora_scale, + enable_edit_guidance=do_classifier_free_guidance, + ) + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(negative_pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + # 3. Prepare added time ids & embeddings + add_text_embeds = negative_pooled_prompt_embeds + add_time_ids = self._get_add_time_ids( + self.size, + crops_coords_top_left, + self.size, + dtype=negative_prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + + if do_classifier_free_guidance: + negative_prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([add_text_embeds, edit_pooled_prompt_embeds], dim=0) + add_time_ids = torch.cat([add_time_ids, add_time_ids], dim=0) + + negative_prompt_embeds = negative_prompt_embeds.to(device) + + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(self.batch_size * num_images_per_prompt, 1) + + # autoencoder reconstruction + if self.vae.dtype == torch.float16 and self.vae.config.force_upcast: + self.upcast_vae() + x0_tmp = x0.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + image_rec = self.vae.decode( + x0_tmp / self.vae.config.scaling_factor, return_dict=False, generator=generator + )[0] + elif self.vae.config.force_upcast: + x0_tmp = x0.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + image_rec = self.vae.decode( + x0_tmp / self.vae.config.scaling_factor, return_dict=False, generator=generator + )[0] + else: + image_rec = self.vae.decode(x0 / self.vae.config.scaling_factor, return_dict=False, generator=generator)[0] + + image_rec = self.image_processor.postprocess(image_rec, output_type="pil") + + # 5. find zs and xts + variance_noise_shape = (num_inversion_steps, *x0.shape) + + # intermediate latents + t_to_idx = {int(v): k for k, v in enumerate(timesteps)} + xts = torch.zeros(size=variance_noise_shape, device=self.device, dtype=negative_prompt_embeds.dtype) + + for t in reversed(timesteps): + idx = num_inversion_steps - t_to_idx[int(t)] - 1 + noise = randn_tensor(shape=x0.shape, generator=generator, device=self.device, dtype=x0.dtype) + xts[idx] = self.scheduler.add_noise(x0, noise, t.unsqueeze(0)) + xts = torch.cat([x0.unsqueeze(0), xts], dim=0) + + # noise maps + zs = torch.zeros(size=variance_noise_shape, device=self.device, dtype=negative_prompt_embeds.dtype) + + self.scheduler.set_timesteps(len(self.scheduler.timesteps)) + + for t in self.progress_bar(timesteps): + idx = num_inversion_steps - t_to_idx[int(t)] - 1 + # 1. predict noise residual + xt = xts[idx + 1] + + latent_model_input = torch.cat([xt] * 2) if do_classifier_free_guidance else xt + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=negative_prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # 2. perform guidance + if do_classifier_free_guidance: + noise_pred_out = noise_pred.chunk(2) + noise_pred_uncond, noise_pred_text = noise_pred_out[0], noise_pred_out[1] + noise_pred = noise_pred_uncond + source_guidance_scale * (noise_pred_text - noise_pred_uncond) + + xtm1 = xts[idx] + z, xtm1_corrected = compute_noise(self.scheduler, xtm1, xt, t, noise_pred, self.eta) + zs[idx] = z + + # correction to avoid error accumulation + xts[idx] = xtm1_corrected + + self.init_latents = xts[-1] + zs = zs.flip(0) + + if num_zero_noise_steps > 0: + zs[-num_zero_noise_steps:] = torch.zeros_like(zs[-num_zero_noise_steps:]) + self.zs = zs + return LEditsPPInversionPipelineOutput(images=resized, vae_reconstruction_images=image_rec) + + +# Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +# Copied from diffusers.pipelines.ledits_pp.pipeline_leditspp_stable_diffusion.compute_noise_ddim +def compute_noise_ddim(scheduler, prev_latents, latents, timestep, noise_pred, eta): + # 1. get previous step value (=t-1) + prev_timestep = timestep - scheduler.config.num_train_timesteps // scheduler.num_inference_steps + + # 2. compute alphas, betas + alpha_prod_t = scheduler.alphas_cumprod[timestep] + alpha_prod_t_prev = ( + scheduler.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else scheduler.final_alpha_cumprod + ) + + beta_prod_t = 1 - alpha_prod_t + + # 3. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_original_sample = (latents - beta_prod_t ** (0.5) * noise_pred) / alpha_prod_t ** (0.5) + + # 4. Clip "predicted x_0" + if scheduler.config.clip_sample: + pred_original_sample = torch.clamp(pred_original_sample, -1, 1) + + # 5. compute variance: "sigma_t(η)" -> see formula (16) + # σ_t = sqrt((1 − α_t−1)/(1 − α_t)) * sqrt(1 − α_t/α_t−1) + variance = scheduler._get_variance(timestep, prev_timestep) + std_dev_t = eta * variance ** (0.5) + + # 6. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * noise_pred + + # modifed so that updated xtm1 is returned as well (to avoid error accumulation) + mu_xt = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction + if variance > 0.0: + noise = (prev_latents - mu_xt) / (variance ** (0.5) * eta) + else: + noise = torch.tensor([0.0]).to(latents.device) + + return noise, mu_xt + (eta * variance**0.5) * noise + + +# Copied from diffusers.pipelines.ledits_pp.pipeline_leditspp_stable_diffusion.compute_noise_sde_dpm_pp_2nd +def compute_noise_sde_dpm_pp_2nd(scheduler, prev_latents, latents, timestep, noise_pred, eta): + def first_order_update(model_output, sample): # timestep, prev_timestep, sample): + sigma_t, sigma_s = scheduler.sigmas[scheduler.step_index + 1], scheduler.sigmas[scheduler.step_index] + alpha_t, sigma_t = scheduler._sigma_to_alpha_sigma_t(sigma_t) + alpha_s, sigma_s = scheduler._sigma_to_alpha_sigma_t(sigma_s) + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s = torch.log(alpha_s) - torch.log(sigma_s) + + h = lambda_t - lambda_s + + mu_xt = (sigma_t / sigma_s * torch.exp(-h)) * sample + (alpha_t * (1 - torch.exp(-2.0 * h))) * model_output + + mu_xt = scheduler.dpm_solver_first_order_update( + model_output=model_output, sample=sample, noise=torch.zeros_like(sample) + ) + + sigma = sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) + if sigma > 0.0: + noise = (prev_latents - mu_xt) / sigma + else: + noise = torch.tensor([0.0]).to(sample.device) + + prev_sample = mu_xt + sigma * noise + return noise, prev_sample + + def second_order_update(model_output_list, sample): # timestep_list, prev_timestep, sample): + sigma_t, sigma_s0, sigma_s1 = ( + scheduler.sigmas[scheduler.step_index + 1], + scheduler.sigmas[scheduler.step_index], + scheduler.sigmas[scheduler.step_index - 1], + ) + + alpha_t, sigma_t = scheduler._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = scheduler._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = scheduler._sigma_to_alpha_sigma_t(sigma_s1) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + + m0, m1 = model_output_list[-1], model_output_list[-2] + + h, h_0 = lambda_t - lambda_s0, lambda_s0 - lambda_s1 + r0 = h_0 / h + D0, D1 = m0, (1.0 / r0) * (m0 - m1) + + mu_xt = ( + (sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + + 0.5 * (alpha_t * (1 - torch.exp(-2.0 * h))) * D1 + ) + + sigma = sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) + if sigma > 0.0: + noise = (prev_latents - mu_xt) / sigma + else: + noise = torch.tensor([0.0]).to(sample.device) + + prev_sample = mu_xt + sigma * noise + + return noise, prev_sample + + if scheduler.step_index is None: + scheduler._init_step_index(timestep) + + model_output = scheduler.convert_model_output(model_output=noise_pred, sample=latents) + for i in range(scheduler.config.solver_order - 1): + scheduler.model_outputs[i] = scheduler.model_outputs[i + 1] + scheduler.model_outputs[-1] = model_output + + if scheduler.lower_order_nums < 1: + noise, prev_sample = first_order_update(model_output, latents) + else: + noise, prev_sample = second_order_update(scheduler.model_outputs, latents) + + if scheduler.lower_order_nums < scheduler.config.solver_order: + scheduler.lower_order_nums += 1 + + # upon completion increase step index by one + scheduler._step_index += 1 + + return noise, prev_sample + + +# Copied from diffusers.pipelines.ledits_pp.pipeline_leditspp_stable_diffusion.compute_noise +def compute_noise(scheduler, *args): + if isinstance(scheduler, DDIMScheduler): + return compute_noise_ddim(scheduler, *args) + elif ( + isinstance(scheduler, DPMSolverMultistepScheduler) + and scheduler.config.algorithm_type == "sde-dpmsolver++" + and scheduler.config.solver_order == 2 + ): + return compute_noise_sde_dpm_pp_2nd(scheduler, *args) + else: + raise NotImplementedError diff --git a/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_output.py b/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_output.py new file mode 100755 index 0000000..b90005c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/ledits_pp/pipeline_output.py @@ -0,0 +1,43 @@ +from dataclasses import dataclass +from typing import List, Optional, Union + +import numpy as np +import PIL.Image + +from ...utils import BaseOutput + + +@dataclass +class LEditsPPDiffusionPipelineOutput(BaseOutput): + """ + Output class for LEdits++ Diffusion pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + num_channels)`. + nsfw_content_detected (`List[bool]`) + List indicating whether the corresponding generated image contains “not-safe-for-work” (nsfw) content or + `None` if safety checking could not be performed. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + nsfw_content_detected: Optional[List[bool]] + + +@dataclass +class LEditsPPInversionPipelineOutput(BaseOutput): + """ + Output class for LEdits++ Diffusion pipelines. + + Args: + input_images (`List[PIL.Image.Image]` or `np.ndarray`) + List of the cropped and resized input images as PIL images of length `batch_size` or NumPy array of shape ` + (batch_size, height, width, num_channels)`. + vae_reconstruction_images (`List[PIL.Image.Image]` or `np.ndarray`) + List of VAE reconstruction of all input images as PIL images of length `batch_size` or NumPy array of shape ` + (batch_size, height, width, num_channels)`. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + vae_reconstruction_images: Union[List[PIL.Image.Image], np.ndarray] diff --git a/diffusers-0.27.0/src/diffusers/pipelines/musicldm/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/musicldm/__init__.py new file mode 100755 index 0000000..ed71eeb --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/musicldm/__init__.py @@ -0,0 +1,49 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, + is_transformers_version, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_musicldm"] = ["MusicLDMPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.27.0")): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_musicldm import MusicLDMPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/musicldm/pipeline_musicldm.py b/diffusers-0.27.0/src/diffusers/pipelines/musicldm/pipeline_musicldm.py new file mode 100755 index 0000000..5fde345 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/musicldm/pipeline_musicldm.py @@ -0,0 +1,635 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import torch +from transformers import ( + ClapFeatureExtractor, + ClapModel, + ClapTextModelWithProjection, + RobertaTokenizer, + RobertaTokenizerFast, + SpeechT5HifiGan, +) + +from ...models import AutoencoderKL, UNet2DConditionModel +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + is_accelerate_available, + is_accelerate_version, + is_librosa_available, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import AudioPipelineOutput, DiffusionPipeline, StableDiffusionMixin + + +if is_librosa_available(): + import librosa + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import MusicLDMPipeline + >>> import torch + >>> import scipy + + >>> repo_id = "ucsd-reach/musicldm" + >>> pipe = MusicLDMPipeline.from_pretrained(repo_id, torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + + >>> prompt = "Techno music with a strong, upbeat tempo and high melodic riffs" + >>> audio = pipe(prompt, num_inference_steps=10, audio_length_in_s=5.0).audios[0] + + >>> # save the audio sample as a .wav file + >>> scipy.io.wavfile.write("techno.wav", rate=16000, data=audio) + ``` +""" + + +class MusicLDMPipeline(DiffusionPipeline, StableDiffusionMixin): + r""" + Pipeline for text-to-audio generation using MusicLDM. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.ClapModel`]): + Frozen text-audio embedding model (`ClapTextModel`), specifically the + [laion/clap-htsat-unfused](https://huggingface.co/laion/clap-htsat-unfused) variant. + tokenizer ([`PreTrainedTokenizer`]): + A [`~transformers.RobertaTokenizer`] to tokenize text. + feature_extractor ([`~transformers.ClapFeatureExtractor`]): + Feature extractor to compute mel-spectrograms from audio waveforms. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded audio latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded audio latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + vocoder ([`~transformers.SpeechT5HifiGan`]): + Vocoder of class `SpeechT5HifiGan`. + """ + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: Union[ClapTextModelWithProjection, ClapModel], + tokenizer: Union[RobertaTokenizer, RobertaTokenizerFast], + feature_extractor: Optional[ClapFeatureExtractor], + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + vocoder: SpeechT5HifiGan, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + feature_extractor=feature_extractor, + unet=unet, + scheduler=scheduler, + vocoder=vocoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + + def _encode_prompt( + self, + prompt, + device, + num_waveforms_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device (`torch.device`): + torch device + num_waveforms_per_prompt (`int`): + number of waveforms that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the audio generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + """ + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + attention_mask = text_inputs.attention_mask + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLAP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder.get_text_features( + text_input_ids.to(device), + attention_mask=attention_mask.to(device), + ) + + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.text_model.dtype, device=device) + + ( + bs_embed, + seq_len, + ) = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_waveforms_per_prompt) + prompt_embeds = prompt_embeds.view(bs_embed * num_waveforms_per_prompt, seq_len) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + uncond_input_ids = uncond_input.input_ids.to(device) + attention_mask = uncond_input.attention_mask.to(device) + + negative_prompt_embeds = self.text_encoder.get_text_features( + uncond_input_ids, + attention_mask=attention_mask, + ) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.text_model.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_waveforms_per_prompt) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_waveforms_per_prompt, seq_len) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + # Copied from diffusers.pipelines.audioldm.pipeline_audioldm.AudioLDMPipeline.mel_spectrogram_to_waveform + def mel_spectrogram_to_waveform(self, mel_spectrogram): + if mel_spectrogram.dim() == 4: + mel_spectrogram = mel_spectrogram.squeeze(1) + + waveform = self.vocoder(mel_spectrogram) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + waveform = waveform.cpu().float() + return waveform + + # Copied from diffusers.pipelines.audioldm2.pipeline_audioldm2.AudioLDM2Pipeline.score_waveforms + def score_waveforms(self, text, audio, num_waveforms_per_prompt, device, dtype): + if not is_librosa_available(): + logger.info( + "Automatic scoring of the generated audio waveforms against the input prompt text requires the " + "`librosa` package to resample the generated waveforms. Returning the audios in the order they were " + "generated. To enable automatic scoring, install `librosa` with: `pip install librosa`." + ) + return audio + inputs = self.tokenizer(text, return_tensors="pt", padding=True) + resampled_audio = librosa.resample( + audio.numpy(), orig_sr=self.vocoder.config.sampling_rate, target_sr=self.feature_extractor.sampling_rate + ) + inputs["input_features"] = self.feature_extractor( + list(resampled_audio), return_tensors="pt", sampling_rate=self.feature_extractor.sampling_rate + ).input_features.type(dtype) + inputs = inputs.to(device) + + # compute the audio-text similarity score using the CLAP model + logits_per_text = self.text_encoder(**inputs).logits_per_text + # sort by the highest matching generations per prompt + indices = torch.argsort(logits_per_text, dim=1, descending=True)[:, :num_waveforms_per_prompt] + audio = torch.index_select(audio, 0, indices.reshape(-1).cpu()) + return audio + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.audioldm.pipeline_audioldm.AudioLDMPipeline.check_inputs + def check_inputs( + self, + prompt, + audio_length_in_s, + vocoder_upsample_factor, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + min_audio_length_in_s = vocoder_upsample_factor * self.vae_scale_factor + if audio_length_in_s < min_audio_length_in_s: + raise ValueError( + f"`audio_length_in_s` has to be a positive value greater than or equal to {min_audio_length_in_s}, but " + f"is {audio_length_in_s}." + ) + + if self.vocoder.config.model_in_dim % self.vae_scale_factor != 0: + raise ValueError( + f"The number of frequency bins in the vocoder's log-mel spectrogram has to be divisible by the " + f"VAE scale factor, but got {self.vocoder.config.model_in_dim} bins and a scale factor of " + f"{self.vae_scale_factor}." + ) + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.audioldm.pipeline_audioldm.AudioLDMPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, dtype, device, generator, latents=None): + shape = ( + batch_size, + num_channels_latents, + height // self.vae_scale_factor, + self.vocoder.config.model_in_dim // self.vae_scale_factor, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def enable_model_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared + to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` + method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with + `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`. + """ + if is_accelerate_available() and is_accelerate_version(">=", "0.17.0.dev0"): + from accelerate import cpu_offload_with_hook + else: + raise ImportError("`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.") + + device = torch.device(f"cuda:{gpu_id}") + + if self.device.type != "cpu": + self.to("cpu", silence_dtype_warnings=True) + torch.cuda.empty_cache() # otherwise we don't see the memory savings (but they probably exist) + + model_sequence = [ + self.text_encoder.text_model, + self.text_encoder.text_projection, + self.unet, + self.vae, + self.vocoder, + self.text_encoder, + ] + + hook = None + for cpu_offloaded_model in model_sequence: + _, hook = cpu_offload_with_hook(cpu_offloaded_model, device, prev_module_hook=hook) + + # We'll offload the last model manually. + self.final_offload_hook = hook + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + audio_length_in_s: Optional[float] = None, + num_inference_steps: int = 200, + guidance_scale: float = 2.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_waveforms_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: Optional[int] = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + output_type: Optional[str] = "np", + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide audio generation. If not defined, you need to pass `prompt_embeds`. + audio_length_in_s (`int`, *optional*, defaults to 10.24): + The length of the generated audio sample in seconds. + num_inference_steps (`int`, *optional*, defaults to 200): + The number of denoising steps. More denoising steps usually lead to a higher quality audio at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 2.0): + A higher guidance scale value encourages the model to generate audio that is closely linked to the text + `prompt` at the expense of lower sound quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in audio generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_waveforms_per_prompt (`int`, *optional*, defaults to 1): + The number of waveforms to generate per prompt. If `num_waveforms_per_prompt > 1`, the text encoding + model is a joint text-audio model ([`~transformers.ClapModel`]), and the tokenizer is a + `[~transformers.ClapProcessor]`, then automatic scoring will be performed between the generated outputs + and the input text. This scoring ranks the generated waveforms based on their cosine similarity to text + input in the joint text-audio embedding space. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.AudioPipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + output_type (`str`, *optional*, defaults to `"np"`): + The output format of the generated audio. Choose between `"np"` to return a NumPy `np.ndarray` or + `"pt"` to return a PyTorch `torch.Tensor` object. Set to `"latent"` to return the latent diffusion + model (LDM) output. + + Examples: + + Returns: + [`~pipelines.AudioPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.AudioPipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated audio. + """ + # 0. Convert audio input length from seconds to spectrogram height + vocoder_upsample_factor = np.prod(self.vocoder.config.upsample_rates) / self.vocoder.config.sampling_rate + + if audio_length_in_s is None: + audio_length_in_s = self.unet.config.sample_size * self.vae_scale_factor * vocoder_upsample_factor + + height = int(audio_length_in_s / vocoder_upsample_factor) + + original_waveform_length = int(audio_length_in_s * self.vocoder.config.sampling_rate) + if height % self.vae_scale_factor != 0: + height = int(np.ceil(height / self.vae_scale_factor)) * self.vae_scale_factor + logger.info( + f"Audio length in seconds {audio_length_in_s} is increased to {height * vocoder_upsample_factor} " + f"so that it can be handled by the model. It will be cut to {audio_length_in_s} after the " + f"denoising process." + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + audio_length_in_s, + vocoder_upsample_factor, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds = self._encode_prompt( + prompt, + device, + num_waveforms_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_waveforms_per_prompt, + num_channels_latents, + height, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=None, + class_labels=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + self.maybe_free_model_hooks() + + # 8. Post-processing + if not output_type == "latent": + latents = 1 / self.vae.config.scaling_factor * latents + mel_spectrogram = self.vae.decode(latents).sample + else: + return AudioPipelineOutput(audios=latents) + + audio = self.mel_spectrogram_to_waveform(mel_spectrogram) + + audio = audio[:, :original_waveform_length] + + # 9. Automatic scoring + if num_waveforms_per_prompt > 1 and prompt is not None: + audio = self.score_waveforms( + text=prompt, + audio=audio, + num_waveforms_per_prompt=num_waveforms_per_prompt, + device=device, + dtype=prompt_embeds.dtype, + ) + + if output_type == "np": + audio = audio.numpy() + + if not return_dict: + return (audio,) + + return AudioPipelineOutput(audios=audio) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/onnx_utils.py b/diffusers-0.27.0/src/diffusers/pipelines/onnx_utils.py new file mode 100755 index 0000000..11f2241 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/onnx_utils.py @@ -0,0 +1,215 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import shutil +from pathlib import Path +from typing import Optional, Union + +import numpy as np +from huggingface_hub import hf_hub_download +from huggingface_hub.utils import validate_hf_hub_args + +from ..utils import ONNX_EXTERNAL_WEIGHTS_NAME, ONNX_WEIGHTS_NAME, is_onnx_available, logging + + +if is_onnx_available(): + import onnxruntime as ort + + +logger = logging.get_logger(__name__) + +ORT_TO_NP_TYPE = { + "tensor(bool)": np.bool_, + "tensor(int8)": np.int8, + "tensor(uint8)": np.uint8, + "tensor(int16)": np.int16, + "tensor(uint16)": np.uint16, + "tensor(int32)": np.int32, + "tensor(uint32)": np.uint32, + "tensor(int64)": np.int64, + "tensor(uint64)": np.uint64, + "tensor(float16)": np.float16, + "tensor(float)": np.float32, + "tensor(double)": np.float64, +} + + +class OnnxRuntimeModel: + def __init__(self, model=None, **kwargs): + logger.info("`diffusers.OnnxRuntimeModel` is experimental and might change in the future.") + self.model = model + self.model_save_dir = kwargs.get("model_save_dir", None) + self.latest_model_name = kwargs.get("latest_model_name", ONNX_WEIGHTS_NAME) + + def __call__(self, **kwargs): + inputs = {k: np.array(v) for k, v in kwargs.items()} + return self.model.run(None, inputs) + + @staticmethod + def load_model(path: Union[str, Path], provider=None, sess_options=None): + """ + Loads an ONNX Inference session with an ExecutionProvider. Default provider is `CPUExecutionProvider` + + Arguments: + path (`str` or `Path`): + Directory from which to load + provider(`str`, *optional*): + Onnxruntime execution provider to use for loading the model, defaults to `CPUExecutionProvider` + """ + if provider is None: + logger.info("No onnxruntime provider specified, using CPUExecutionProvider") + provider = "CPUExecutionProvider" + + return ort.InferenceSession(path, providers=[provider], sess_options=sess_options) + + def _save_pretrained(self, save_directory: Union[str, Path], file_name: Optional[str] = None, **kwargs): + """ + Save a model and its configuration file to a directory, so that it can be re-loaded using the + [`~optimum.onnxruntime.modeling_ort.ORTModel.from_pretrained`] class method. It will always save the + latest_model_name. + + Arguments: + save_directory (`str` or `Path`): + Directory where to save the model file. + file_name(`str`, *optional*): + Overwrites the default model file name from `"model.onnx"` to `file_name`. This allows you to save the + model with a different name. + """ + model_file_name = file_name if file_name is not None else ONNX_WEIGHTS_NAME + + src_path = self.model_save_dir.joinpath(self.latest_model_name) + dst_path = Path(save_directory).joinpath(model_file_name) + try: + shutil.copyfile(src_path, dst_path) + except shutil.SameFileError: + pass + + # copy external weights (for models >2GB) + src_path = self.model_save_dir.joinpath(ONNX_EXTERNAL_WEIGHTS_NAME) + if src_path.exists(): + dst_path = Path(save_directory).joinpath(ONNX_EXTERNAL_WEIGHTS_NAME) + try: + shutil.copyfile(src_path, dst_path) + except shutil.SameFileError: + pass + + def save_pretrained( + self, + save_directory: Union[str, os.PathLike], + **kwargs, + ): + """ + Save a model to a directory, so that it can be re-loaded using the [`~OnnxModel.from_pretrained`] class + method.: + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to which to save. Will be created if it doesn't exist. + """ + if os.path.isfile(save_directory): + logger.error(f"Provided path ({save_directory}) should be a directory, not a file") + return + + os.makedirs(save_directory, exist_ok=True) + + # saving model weights/files + self._save_pretrained(save_directory, **kwargs) + + @classmethod + @validate_hf_hub_args + def _from_pretrained( + cls, + model_id: Union[str, Path], + token: Optional[Union[bool, str, None]] = None, + revision: Optional[Union[str, None]] = None, + force_download: bool = False, + cache_dir: Optional[str] = None, + file_name: Optional[str] = None, + provider: Optional[str] = None, + sess_options: Optional["ort.SessionOptions"] = None, + **kwargs, + ): + """ + Load a model from a directory or the HF Hub. + + Arguments: + model_id (`str` or `Path`): + Directory from which to load + token (`str` or `bool`): + Is needed to load models from a private or gated repository + revision (`str`): + Revision is the specific model version to use. It can be a branch name, a tag name, or a commit id + cache_dir (`Union[str, Path]`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + file_name(`str`): + Overwrites the default model file name from `"model.onnx"` to `file_name`. This allows you to load + different model files from the same repository or directory. + provider(`str`): + The ONNX runtime provider, e.g. `CPUExecutionProvider` or `CUDAExecutionProvider`. + kwargs (`Dict`, *optional*): + kwargs will be passed to the model during initialization + """ + model_file_name = file_name if file_name is not None else ONNX_WEIGHTS_NAME + # load model from local directory + if os.path.isdir(model_id): + model = OnnxRuntimeModel.load_model( + Path(model_id, model_file_name).as_posix(), provider=provider, sess_options=sess_options + ) + kwargs["model_save_dir"] = Path(model_id) + # load model from hub + else: + # download model + model_cache_path = hf_hub_download( + repo_id=model_id, + filename=model_file_name, + token=token, + revision=revision, + cache_dir=cache_dir, + force_download=force_download, + ) + kwargs["model_save_dir"] = Path(model_cache_path).parent + kwargs["latest_model_name"] = Path(model_cache_path).name + model = OnnxRuntimeModel.load_model(model_cache_path, provider=provider, sess_options=sess_options) + return cls(model=model, **kwargs) + + @classmethod + @validate_hf_hub_args + def from_pretrained( + cls, + model_id: Union[str, Path], + force_download: bool = True, + token: Optional[str] = None, + cache_dir: Optional[str] = None, + **model_kwargs, + ): + revision = None + if len(str(model_id).split("@")) == 2: + model_id, revision = model_id.split("@") + + return cls._from_pretrained( + model_id=model_id, + revision=revision, + cache_dir=cache_dir, + force_download=force_download, + token=token, + **model_kwargs, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/__init__.py new file mode 100755 index 0000000..aaa775f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/__init__.py @@ -0,0 +1,55 @@ +from dataclasses import dataclass +from typing import TYPE_CHECKING, List, Optional, Union + +import numpy as np +import PIL +from PIL import Image + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["image_encoder"] = ["PaintByExampleImageEncoder"] + _import_structure["pipeline_paint_by_example"] = ["PaintByExamplePipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .image_encoder import PaintByExampleImageEncoder + from .pipeline_paint_by_example import PaintByExamplePipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/image_encoder.py b/diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/image_encoder.py new file mode 100755 index 0000000..2fd0338 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/image_encoder.py @@ -0,0 +1,67 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import torch +from torch import nn +from transformers import CLIPPreTrainedModel, CLIPVisionModel + +from ...models.attention import BasicTransformerBlock +from ...utils import logging + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class PaintByExampleImageEncoder(CLIPPreTrainedModel): + def __init__(self, config, proj_size=None): + super().__init__(config) + self.proj_size = proj_size or getattr(config, "projection_dim", 768) + + self.model = CLIPVisionModel(config) + self.mapper = PaintByExampleMapper(config) + self.final_layer_norm = nn.LayerNorm(config.hidden_size) + self.proj_out = nn.Linear(config.hidden_size, self.proj_size) + + # uncondition for scaling + self.uncond_vector = nn.Parameter(torch.randn((1, 1, self.proj_size))) + + def forward(self, pixel_values, return_uncond_vector=False): + clip_output = self.model(pixel_values=pixel_values) + latent_states = clip_output.pooler_output + latent_states = self.mapper(latent_states[:, None]) + latent_states = self.final_layer_norm(latent_states) + latent_states = self.proj_out(latent_states) + if return_uncond_vector: + return latent_states, self.uncond_vector + + return latent_states + + +class PaintByExampleMapper(nn.Module): + def __init__(self, config): + super().__init__() + num_layers = (config.num_hidden_layers + 1) // 5 + hid_size = config.hidden_size + num_heads = 1 + self.blocks = nn.ModuleList( + [ + BasicTransformerBlock(hid_size, num_heads, hid_size, activation_fn="gelu", attention_bias=True) + for _ in range(num_layers) + ] + ) + + def forward(self, hidden_states): + for block in self.blocks: + hidden_states = block(hidden_states) + + return hidden_states diff --git a/diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py b/diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py new file mode 100755 index 0000000..8a24f13 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/paint_by_example/pipeline_paint_by_example.py @@ -0,0 +1,621 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor + +from ...image_processor import VaeImageProcessor +from ...models import AutoencoderKL, UNet2DConditionModel +from ...schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler +from ...utils import deprecate, logging +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from .image_encoder import PaintByExampleImageEncoder + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +def prepare_mask_and_masked_image(image, mask): + """ + Prepares a pair (image, mask) to be consumed by the Paint by Example pipeline. This means that those inputs will be + converted to ``torch.Tensor`` with shapes ``batch x channels x height x width`` where ``channels`` is ``3`` for the + ``image`` and ``1`` for the ``mask``. + + The ``image`` will be converted to ``torch.float32`` and normalized to be in ``[-1, 1]``. The ``mask`` will be + binarized (``mask > 0.5``) and cast to ``torch.float32`` too. + + Args: + image (Union[np.array, PIL.Image, torch.Tensor]): The image to inpaint. + It can be a ``PIL.Image``, or a ``height x width x 3`` ``np.array`` or a ``channels x height x width`` + ``torch.Tensor`` or a ``batch x channels x height x width`` ``torch.Tensor``. + mask (_type_): The mask to apply to the image, i.e. regions to inpaint. + It can be a ``PIL.Image``, or a ``height x width`` ``np.array`` or a ``1 x height x width`` + ``torch.Tensor`` or a ``batch x 1 x height x width`` ``torch.Tensor``. + + + Raises: + ValueError: ``torch.Tensor`` images should be in the ``[-1, 1]`` range. ValueError: ``torch.Tensor`` mask + should be in the ``[0, 1]`` range. ValueError: ``mask`` and ``image`` should have the same spatial dimensions. + TypeError: ``mask`` is a ``torch.Tensor`` but ``image`` is not + (ot the other way around). + + Returns: + tuple[torch.Tensor]: The pair (mask, masked_image) as ``torch.Tensor`` with 4 + dimensions: ``batch x channels x height x width``. + """ + if isinstance(image, torch.Tensor): + if not isinstance(mask, torch.Tensor): + raise TypeError(f"`image` is a torch.Tensor but `mask` (type: {type(mask)} is not") + + # Batch single image + if image.ndim == 3: + assert image.shape[0] == 3, "Image outside a batch should be of shape (3, H, W)" + image = image.unsqueeze(0) + + # Batch and add channel dim for single mask + if mask.ndim == 2: + mask = mask.unsqueeze(0).unsqueeze(0) + + # Batch single mask or add channel dim + if mask.ndim == 3: + # Batched mask + if mask.shape[0] == image.shape[0]: + mask = mask.unsqueeze(1) + else: + mask = mask.unsqueeze(0) + + assert image.ndim == 4 and mask.ndim == 4, "Image and Mask must have 4 dimensions" + assert image.shape[-2:] == mask.shape[-2:], "Image and Mask must have the same spatial dimensions" + assert image.shape[0] == mask.shape[0], "Image and Mask must have the same batch size" + assert mask.shape[1] == 1, "Mask image must have a single channel" + + # Check image is in [-1, 1] + if image.min() < -1 or image.max() > 1: + raise ValueError("Image should be in [-1, 1] range") + + # Check mask is in [0, 1] + if mask.min() < 0 or mask.max() > 1: + raise ValueError("Mask should be in [0, 1] range") + + # paint-by-example inverses the mask + mask = 1 - mask + + # Binarize mask + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + + # Image as float32 + image = image.to(dtype=torch.float32) + elif isinstance(mask, torch.Tensor): + raise TypeError(f"`mask` is a torch.Tensor but `image` (type: {type(image)} is not") + else: + if isinstance(image, PIL.Image.Image): + image = [image] + + image = np.concatenate([np.array(i.convert("RGB"))[None, :] for i in image], axis=0) + image = image.transpose(0, 3, 1, 2) + image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0 + + # preprocess mask + if isinstance(mask, PIL.Image.Image): + mask = [mask] + + mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0) + mask = mask.astype(np.float32) / 255.0 + + # paint-by-example inverses the mask + mask = 1 - mask + + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + mask = torch.from_numpy(mask) + + masked_image = image * mask + + return mask, masked_image + + +class PaintByExamplePipeline(DiffusionPipeline, StableDiffusionMixin): + r""" + + + 🧪 This is an experimental feature! + + + + Pipeline for image-guided image inpainting using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + image_encoder ([`PaintByExampleImageEncoder`]): + Encodes the example input image. The `unet` is conditioned on the example image instead of a text prompt. + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + + """ + + # TODO: feature_extractor is required to encode initial images (if they are in PIL format), + # we should give a descriptive message if the pipeline doesn't have one. + + model_cpu_offload_seq = "unet->vae" + _exclude_from_cpu_offload = ["image_encoder"] + _optional_components = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + image_encoder: PaintByExampleImageEncoder, + unet: UNet2DConditionModel, + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = False, + ): + super().__init__() + + self.register_modules( + vae=vae, + image_encoder=image_encoder, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_image_variation.StableDiffusionImageVariationPipeline.check_inputs + def check_inputs(self, image, height, width, callback_steps): + if ( + not isinstance(image, torch.Tensor) + and not isinstance(image, PIL.Image.Image) + and not isinstance(image, list) + ): + raise ValueError( + "`image` has to be of type `torch.FloatTensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + f" {type(image)}" + ) + + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_inpaint.StableDiffusionInpaintPipeline.prepare_mask_latents + def prepare_mask_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + mask = torch.nn.functional.interpolate( + mask, size=(height // self.vae_scale_factor, width // self.vae_scale_factor) + ) + mask = mask.to(device=device, dtype=dtype) + + masked_image = masked_image.to(device=device, dtype=dtype) + + if masked_image.shape[1] == 4: + masked_image_latents = masked_image + else: + masked_image_latents = self._encode_vae_image(masked_image, generator=generator) + + # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method + if mask.shape[0] < batch_size: + if not batch_size % mask.shape[0] == 0: + raise ValueError( + "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to" + f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number" + " of masks that you pass is divisible by the total requested batch size." + ) + mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1) + if masked_image_latents.shape[0] < batch_size: + if not batch_size % masked_image_latents.shape[0] == 0: + raise ValueError( + "The passed images and the required batch size don't match. Images are supposed to be duplicated" + f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed." + " Make sure the number of images that you pass is divisible by the total requested batch size." + ) + masked_image_latents = masked_image_latents.repeat(batch_size // masked_image_latents.shape[0], 1, 1, 1) + + mask = torch.cat([mask] * 2) if do_classifier_free_guidance else mask + masked_image_latents = ( + torch.cat([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents + ) + + # aligning device to prevent device errors when concating it with the latent model input + masked_image_latents = masked_image_latents.to(device=device, dtype=dtype) + return mask, masked_image_latents + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_inpaint.StableDiffusionInpaintPipeline._encode_vae_image + def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator): + if isinstance(generator, list): + image_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(image.shape[0]) + ] + image_latents = torch.cat(image_latents, dim=0) + else: + image_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + image_latents = self.vae.config.scaling_factor * image_latents + + return image_latents + + def _encode_image(self, image, device, num_images_per_prompt, do_classifier_free_guidance): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(images=image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + image_embeddings, negative_prompt_embeds = self.image_encoder(image, return_uncond_vector=True) + + # duplicate image embeddings for each generation per prompt, using mps friendly method + bs_embed, seq_len, _ = image_embeddings.shape + image_embeddings = image_embeddings.repeat(1, num_images_per_prompt, 1) + image_embeddings = image_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + negative_prompt_embeds = negative_prompt_embeds.repeat(1, image_embeddings.shape[0], 1) + negative_prompt_embeds = negative_prompt_embeds.view(bs_embed * num_images_per_prompt, 1, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + image_embeddings = torch.cat([negative_prompt_embeds, image_embeddings]) + + return image_embeddings + + @torch.no_grad() + def __call__( + self, + example_image: Union[torch.FloatTensor, PIL.Image.Image], + image: Union[torch.FloatTensor, PIL.Image.Image], + mask_image: Union[torch.FloatTensor, PIL.Image.Image], + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + ): + r""" + The call function to the pipeline for generation. + + Args: + example_image (`torch.FloatTensor` or `PIL.Image.Image` or `List[PIL.Image.Image]`): + An example image to guide image generation. + image (`torch.FloatTensor` or `PIL.Image.Image` or `List[PIL.Image.Image]`): + `Image` or tensor representing an image batch to be inpainted (parts of the image are masked out with + `mask_image` and repainted according to `prompt`). + mask_image (`torch.FloatTensor` or `PIL.Image.Image` or `List[PIL.Image.Image]`): + `Image` or tensor representing an image batch to mask `image`. White pixels in the mask are repainted, + while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a single channel + (luminance) before use. If it's a tensor, it should contain one color channel (L) instead of 3, so the + expected shape would be `(B, H, W, 1)`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Example: + + ```py + >>> import PIL + >>> import requests + >>> import torch + >>> from io import BytesIO + >>> from diffusers import PaintByExamplePipeline + + + >>> def download_image(url): + ... response = requests.get(url) + ... return PIL.Image.open(BytesIO(response.content)).convert("RGB") + + + >>> img_url = ( + ... "https://raw.githubusercontent.com/Fantasy-Studio/Paint-by-Example/main/examples/image/example_1.png" + ... ) + >>> mask_url = ( + ... "https://raw.githubusercontent.com/Fantasy-Studio/Paint-by-Example/main/examples/mask/example_1.png" + ... ) + >>> example_url = "https://raw.githubusercontent.com/Fantasy-Studio/Paint-by-Example/main/examples/reference/example_1.jpg" + + >>> init_image = download_image(img_url).resize((512, 512)) + >>> mask_image = download_image(mask_url).resize((512, 512)) + >>> example_image = download_image(example_url).resize((512, 512)) + + >>> pipe = PaintByExamplePipeline.from_pretrained( + ... "Fantasy-Studio/Paint-by-Example", + ... torch_dtype=torch.float16, + ... ) + >>> pipe = pipe.to("cuda") + + >>> image = pipe(image=init_image, mask_image=mask_image, example_image=example_image).images[0] + >>> image + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 1. Define call parameters + if isinstance(image, PIL.Image.Image): + batch_size = 1 + elif isinstance(image, list): + batch_size = len(image) + else: + batch_size = image.shape[0] + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 2. Preprocess mask and image + mask, masked_image = prepare_mask_and_masked_image(image, mask_image) + height, width = masked_image.shape[-2:] + + # 3. Check inputs + self.check_inputs(example_image, height, width, callback_steps) + + # 4. Encode input image + image_embeddings = self._encode_image( + example_image, device, num_images_per_prompt, do_classifier_free_guidance + ) + + # 5. set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 6. Prepare latent variables + num_channels_latents = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + image_embeddings.dtype, + device, + generator, + latents, + ) + + # 7. Prepare mask latent variables + mask, masked_image_latents = self.prepare_mask_latents( + mask, + masked_image, + batch_size * num_images_per_prompt, + height, + width, + image_embeddings.dtype, + device, + generator, + do_classifier_free_guidance, + ) + + # 8. Check that sizes of mask, masked image and latents match + num_channels_mask = mask.shape[1] + num_channels_masked_image = masked_image_latents.shape[1] + if num_channels_latents + num_channels_mask + num_channels_masked_image != self.unet.config.in_channels: + raise ValueError( + f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects" + f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +" + f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}" + f" = {num_channels_latents+num_channels_masked_image+num_channels_mask}. Please verify the config of" + " `pipeline.unet` or your `mask_image` or `image` input." + ) + + # 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 10. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + # concat latents, mask, masked_image_latents in the channel dimension + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + latent_model_input = torch.cat([latent_model_input, masked_image_latents, mask], dim=1) + + # predict the noise residual + noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=image_embeddings).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + self.maybe_free_model_hooks() + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, image_embeddings.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/pia/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/pia/__init__.py new file mode 100755 index 0000000..16e8004 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/pia/__init__.py @@ -0,0 +1,46 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_pia"] = ["PIAPipeline", "PIAPipelineOutput"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + + else: + from .pipeline_pia import PIAPipeline, PIAPipelineOutput + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/pia/pipeline_pia.py b/diffusers-0.27.0/src/diffusers/pipelines/pia/pipeline_pia.py new file mode 100755 index 0000000..5070889 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/pia/pipeline_pia.py @@ -0,0 +1,1034 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import math +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL +import torch +import torch.fft as fft +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel, UNetMotionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...models.unets.unet_motion_model import MotionAdapter +from ...schedulers import ( + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, +) +from ...utils import ( + USE_PEFT_BACKEND, + BaseOutput, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..free_init_utils import FreeInitMixin +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import ( + ... EulerDiscreteScheduler, + ... MotionAdapter, + ... PIAPipeline, + ... ) + >>> from diffusers.utils import export_to_gif, load_image + >>> adapter = MotionAdapter.from_pretrained("../checkpoints/pia-diffusers") + >>> pipe = PIAPipeline.from_pretrained("SG161222/Realistic_Vision_V6.0_B1_noVAE", motion_adapter=adapter) + >>> pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config) + >>> image = load_image( + ... "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/pix2pix/cat_6.png?download=true" + ... ) + >>> image = image.resize((512, 512)) + >>> prompt = "cat in a hat" + >>> negative_prompt = "wrong white balance, dark, sketches,worst quality,low quality, deformed, distorted, disfigured, bad eyes, wrong lips,weird mouth, bad teeth, mutated hands and fingers, bad anatomy,wrong anatomy, amputation, extra limb, missing limb, floating,limbs, disconnected limbs, mutation, ugly, disgusting, bad_pictures, negative_hand-neg" + >>> generator = torch.Generator("cpu").manual_seed(0) + >>> output = pipe(image=image, prompt=prompt, negative_prompt=negative_prompt, generator=generator) + >>> frames = output.frames[0] + >>> export_to_gif(frames, "pia-animation.gif") + ``` +""" + +RANGE_LIST = [ + [1.0, 0.9, 0.85, 0.85, 0.85, 0.8], # 0 Small Motion + [1.0, 0.8, 0.8, 0.8, 0.79, 0.78, 0.75], # Moderate Motion + [1.0, 0.8, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.7, 0.6, 0.5, 0.5], # Large Motion + [1.0, 0.9, 0.85, 0.85, 0.85, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.8, 0.85, 0.85, 0.9, 1.0], # Loop + [1.0, 0.8, 0.8, 0.8, 0.79, 0.78, 0.75, 0.75, 0.75, 0.75, 0.75, 0.78, 0.79, 0.8, 0.8, 1.0], # Loop + [1.0, 0.8, 0.7, 0.7, 0.7, 0.7, 0.6, 0.5, 0.5, 0.6, 0.7, 0.7, 0.7, 0.7, 0.8, 1.0], # Loop + [0.5, 0.4, 0.4, 0.4, 0.35, 0.3], # Style Transfer Candidate Small Motion + [0.5, 0.4, 0.4, 0.4, 0.35, 0.35, 0.3, 0.25, 0.2], # Style Transfer Moderate Motion + [0.5, 0.2], # Style Transfer Large Motion +] + + +# Copied from diffusers.pipelines.animatediff.pipeline_animatediff.tensor2vid +def tensor2vid(video: torch.Tensor, processor: "VaeImageProcessor", output_type: str = "np"): + batch_size, channels, num_frames, height, width = video.shape + outputs = [] + for batch_idx in range(batch_size): + batch_vid = video[batch_idx].permute(1, 0, 2, 3) + batch_output = processor.postprocess(batch_vid, output_type) + + outputs.append(batch_output) + + if output_type == "np": + outputs = np.stack(outputs) + + elif output_type == "pt": + outputs = torch.stack(outputs) + + elif not output_type == "pil": + raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil']") + + return outputs + + +def prepare_mask_coef_by_statistics(num_frames: int, cond_frame: int, motion_scale: int): + assert num_frames > 0, "video_length should be greater than 0" + + assert num_frames > cond_frame, "video_length should be greater than cond_frame" + + range_list = RANGE_LIST + + assert motion_scale < len(range_list), f"motion_scale type{motion_scale} not implemented" + + coef = range_list[motion_scale] + coef = coef + ([coef[-1]] * (num_frames - len(coef))) + + order = [abs(i - cond_frame) for i in range(num_frames)] + coef = [coef[order[i]] for i in range(num_frames)] + + return coef + + +def _get_freeinit_freq_filter( + shape: Tuple[int, ...], + device: Union[str, torch.dtype], + filter_type: str, + order: float, + spatial_stop_frequency: float, + temporal_stop_frequency: float, +) -> torch.Tensor: + r"""Returns the FreeInit filter based on filter type and other input conditions.""" + + time, height, width = shape[-3], shape[-2], shape[-1] + mask = torch.zeros(shape) + + if spatial_stop_frequency == 0 or temporal_stop_frequency == 0: + return mask + + if filter_type == "butterworth": + + def retrieve_mask(x): + return 1 / (1 + (x / spatial_stop_frequency**2) ** order) + elif filter_type == "gaussian": + + def retrieve_mask(x): + return math.exp(-1 / (2 * spatial_stop_frequency**2) * x) + elif filter_type == "ideal": + + def retrieve_mask(x): + return 1 if x <= spatial_stop_frequency * 2 else 0 + else: + raise NotImplementedError("`filter_type` must be one of gaussian, butterworth or ideal") + + for t in range(time): + for h in range(height): + for w in range(width): + d_square = ( + ((spatial_stop_frequency / temporal_stop_frequency) * (2 * t / time - 1)) ** 2 + + (2 * h / height - 1) ** 2 + + (2 * w / width - 1) ** 2 + ) + mask[..., t, h, w] = retrieve_mask(d_square) + + return mask.to(device) + + +def _freq_mix_3d(x: torch.Tensor, noise: torch.Tensor, LPF: torch.Tensor) -> torch.Tensor: + r"""Noise reinitialization.""" + # FFT + x_freq = fft.fftn(x, dim=(-3, -2, -1)) + x_freq = fft.fftshift(x_freq, dim=(-3, -2, -1)) + noise_freq = fft.fftn(noise, dim=(-3, -2, -1)) + noise_freq = fft.fftshift(noise_freq, dim=(-3, -2, -1)) + + # frequency mix + HPF = 1 - LPF + x_freq_low = x_freq * LPF + noise_freq_high = noise_freq * HPF + x_freq_mixed = x_freq_low + noise_freq_high # mix in freq domain + + # IFFT + x_freq_mixed = fft.ifftshift(x_freq_mixed, dim=(-3, -2, -1)) + x_mixed = fft.ifftn(x_freq_mixed, dim=(-3, -2, -1)).real + + return x_mixed + + +@dataclass +class PIAPipelineOutput(BaseOutput): + r""" + Output class for PIAPipeline. + + Args: + frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + Nested list of length `batch_size` with denoised PIL image sequences of length `num_frames`, + NumPy array of shape `(batch_size, num_frames, channels, height, width, + Torch tensor of shape `(batch_size, num_frames, channels, height, width)`. + """ + + frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]] + + +class PIAPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, + LoraLoaderMixin, + FromSingleFileMixin, + FreeInitMixin, +): + r""" + Pipeline for text-to-video generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (`CLIPTokenizer`): + A [`~transformers.CLIPTokenizer`] to tokenize text. + unet ([`UNet2DConditionModel`]): + A [`UNet2DConditionModel`] used to create a UNetMotionModel to denoise the encoded video latents. + motion_adapter ([`MotionAdapter`]): + A [`MotionAdapter`] to be used in combination with `unet` to denoise the encoded video latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + _optional_components = ["feature_extractor", "image_encoder", "motion_adapter"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: Union[UNet2DConditionModel, UNetMotionModel], + scheduler: Union[ + DDIMScheduler, + PNDMScheduler, + LMSDiscreteScheduler, + EulerDiscreteScheduler, + EulerAncestralDiscreteScheduler, + DPMSolverMultistepScheduler, + ], + motion_adapter: Optional[MotionAdapter] = None, + feature_extractor: CLIPImageProcessor = None, + image_encoder: CLIPVisionModelWithProjection = None, + ): + super().__init__() + if isinstance(unet, UNet2DConditionModel): + unet = UNetMotionModel.from_unet2d(unet, motion_adapter) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + motion_adapter=motion_adapter, + scheduler=scheduler, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt with num_images_per_prompt -> num_videos_per_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.text_to_video_synthesis/pipeline_text_to_video_synth.TextToVideoSDPipeline.decode_latents + def decode_latents(self, latents): + latents = 1 / self.vae.config.scaling_factor * latents + + batch_size, channels, num_frames, height, width = latents.shape + latents = latents.permute(0, 2, 1, 3, 4).reshape(batch_size * num_frames, channels, height, width) + + image = self.vae.decode(latents).sample + video = image[None, :].reshape((batch_size, num_frames, -1) + image.shape[2:]).permute(0, 2, 1, 3, 4) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + video = video.float() + return video + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_synth.TextToVideoSDPipeline.prepare_latents + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + shape = ( + batch_size, + num_channels_latents, + num_frames, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_masked_condition( + self, + image, + batch_size, + num_channels_latents, + num_frames, + height, + width, + dtype, + device, + generator, + motion_scale=0, + ): + shape = ( + batch_size, + num_channels_latents, + num_frames, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + _, _, _, scaled_height, scaled_width = shape + + image = self.image_processor.preprocess(image) + image = image.to(device, dtype) + + if isinstance(generator, list): + image_latent = [ + self.vae.encode(image[k : k + 1]).latent_dist.sample(generator[k]) for k in range(batch_size) + ] + image_latent = torch.cat(image_latent, dim=0) + else: + image_latent = self.vae.encode(image).latent_dist.sample(generator) + + image_latent = image_latent.to(device=device, dtype=dtype) + image_latent = torch.nn.functional.interpolate(image_latent, size=[scaled_height, scaled_width]) + image_latent_padding = image_latent.clone() * self.vae.config.scaling_factor + + mask = torch.zeros((batch_size, 1, num_frames, scaled_height, scaled_width)).to(device=device, dtype=dtype) + mask_coef = prepare_mask_coef_by_statistics(num_frames, 0, motion_scale) + masked_image = torch.zeros(batch_size, 4, num_frames, scaled_height, scaled_width).to( + device=device, dtype=self.unet.dtype + ) + for f in range(num_frames): + mask[:, :, f, :, :] = mask_coef[f] + masked_image[:, :, f, :, :] = image_latent_padding.clone() + + mask = torch.cat([mask] * 2) if self.do_classifier_free_guidance else mask + masked_image = torch.cat([masked_image] * 2) if self.do_classifier_free_guidance else masked_image + + return mask, masked_image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image: PipelineImageInput, + prompt: Union[str, List[str]] = None, + strength: float = 1.0, + num_frames: Optional[int] = 16, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_videos_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + motion_scale: int = 0, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + ): + r""" + The call function to the pipeline for generation. + + Args: + image (`PipelineImageInput`): + The input image to be used for video generation. + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + strength (`float`, *optional*, defaults to 1.0): Indicates extent to transform the reference `image`. Must be between 0 and 1. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated video. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated video. + num_frames (`int`, *optional*, defaults to 16): + The number of video frames that are generated. Defaults to 16 frames which at 8 frames per seconds + amounts to 2 seconds of video. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality videos at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. Latents should be of shape + `(batch_size, num_channel, num_frames, height, width)`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): + Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + motion_scale: (`int`, *optional*, defaults to 0): + Parameter that controls the amount and type of motion that is added to the image. Increasing the value increases the amount of motion, while specific + ranges of values control the type of motion that is added. Must be between 0 and 8. + Set between 0-2 to only increase the amount of motion. + Set between 3-5 to create looping motion. + Set between 6-8 to perform motion with image style transfer. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated video. Choose between `torch.FloatTensor`, `PIL.Image` or + `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] instead + of a plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeine class. + + Examples: + + Returns: + [`~pipelines.pia.pipeline_pia.PIAPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.pia.pipeline_pia.PIAPipelineOutput`] is + returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + num_videos_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_videos_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_videos_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_videos_per_prompt) + self._num_timesteps = len(timesteps) + + # 5. Prepare latent variables + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + 4, + num_frames, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents=latents, + ) + mask, masked_image = self.prepare_masked_condition( + image, + batch_size * num_videos_per_prompt, + 4, + num_frames=num_frames, + height=height, + width=width, + dtype=self.unet.dtype, + device=device, + generator=generator, + motion_scale=motion_scale, + ) + if strength < 1.0: + noise = randn_tensor(latents.shape, generator=generator, device=device, dtype=latents.dtype) + latents = self.scheduler.add_noise(masked_image[0], noise, latent_timestep) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + # 8. Denoising loop + num_free_init_iters = self._free_init_num_iters if self.free_init_enabled else 1 + for free_init_iter in range(num_free_init_iters): + if self.free_init_enabled: + latents, timesteps = self._apply_free_init( + latents, free_init_iter, num_inference_steps, device, latents.dtype, generator + ) + + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + latent_model_input = torch.cat([latent_model_input, mask, masked_image], dim=1) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + ).sample + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + # 9. Post processing + if output_type == "latent": + video = latents + else: + video_tensor = self.decode_latents(latents) + video = tensor2vid(video_tensor, self.image_processor, output_type=output_type) + + # 10. Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (video,) + + return PIAPipelineOutput(frames=video) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/pipeline_flax_utils.py b/diffusers-0.27.0/src/diffusers/pipelines/pipeline_flax_utils.py new file mode 100755 index 0000000..b1035c1 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/pipeline_flax_utils.py @@ -0,0 +1,616 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import inspect +import os +from typing import Any, Dict, List, Optional, Union + +import flax +import numpy as np +import PIL.Image +from flax.core.frozen_dict import FrozenDict +from huggingface_hub import create_repo, snapshot_download +from huggingface_hub.utils import validate_hf_hub_args +from PIL import Image +from tqdm.auto import tqdm + +from ..configuration_utils import ConfigMixin +from ..models.modeling_flax_utils import FLAX_WEIGHTS_NAME, FlaxModelMixin +from ..schedulers.scheduling_utils_flax import SCHEDULER_CONFIG_NAME, FlaxSchedulerMixin +from ..utils import ( + CONFIG_NAME, + BaseOutput, + PushToHubMixin, + http_user_agent, + is_transformers_available, + logging, +) + + +if is_transformers_available(): + from transformers import FlaxPreTrainedModel + +INDEX_FILE = "diffusion_flax_model.bin" + + +logger = logging.get_logger(__name__) + + +LOADABLE_CLASSES = { + "diffusers": { + "FlaxModelMixin": ["save_pretrained", "from_pretrained"], + "FlaxSchedulerMixin": ["save_pretrained", "from_pretrained"], + "FlaxDiffusionPipeline": ["save_pretrained", "from_pretrained"], + }, + "transformers": { + "PreTrainedTokenizer": ["save_pretrained", "from_pretrained"], + "PreTrainedTokenizerFast": ["save_pretrained", "from_pretrained"], + "FlaxPreTrainedModel": ["save_pretrained", "from_pretrained"], + "FeatureExtractionMixin": ["save_pretrained", "from_pretrained"], + "ProcessorMixin": ["save_pretrained", "from_pretrained"], + "ImageProcessingMixin": ["save_pretrained", "from_pretrained"], + }, +} + +ALL_IMPORTABLE_CLASSES = {} +for library in LOADABLE_CLASSES: + ALL_IMPORTABLE_CLASSES.update(LOADABLE_CLASSES[library]) + + +def import_flax_or_no_model(module, class_name): + try: + # 1. First make sure that if a Flax object is present, import this one + class_obj = getattr(module, "Flax" + class_name) + except AttributeError: + # 2. If this doesn't work, it's not a model and we don't append "Flax" + class_obj = getattr(module, class_name) + except AttributeError: + raise ValueError(f"Neither Flax{class_name} nor {class_name} exist in {module}") + + return class_obj + + +@flax.struct.dataclass +class FlaxImagePipelineOutput(BaseOutput): + """ + Output class for image pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + num_channels)`. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +class FlaxDiffusionPipeline(ConfigMixin, PushToHubMixin): + r""" + Base class for Flax-based pipelines. + + [`FlaxDiffusionPipeline`] stores all components (models, schedulers, and processors) for diffusion pipelines and + provides methods for loading, downloading and saving models. It also includes methods to: + + - enable/disable the progress bar for the denoising iteration + + Class attributes: + + - **config_name** ([`str`]) -- The configuration filename that stores the class and module names of all the + diffusion pipeline's components. + """ + + config_name = "model_index.json" + + def register_modules(self, **kwargs): + # import it here to avoid circular import + from diffusers import pipelines + + for name, module in kwargs.items(): + if module is None: + register_dict = {name: (None, None)} + else: + # retrieve library + library = module.__module__.split(".")[0] + + # check if the module is a pipeline module + pipeline_dir = module.__module__.split(".")[-2] + path = module.__module__.split(".") + is_pipeline_module = pipeline_dir in path and hasattr(pipelines, pipeline_dir) + + # if library is not in LOADABLE_CLASSES, then it is a custom module. + # Or if it's a pipeline module, then the module is inside the pipeline + # folder so we set the library to module name. + if library not in LOADABLE_CLASSES or is_pipeline_module: + library = pipeline_dir + + # retrieve class_name + class_name = module.__class__.__name__ + + register_dict = {name: (library, class_name)} + + # save model index config + self.register_to_config(**register_dict) + + # set models + setattr(self, name, module) + + def save_pretrained( + self, + save_directory: Union[str, os.PathLike], + params: Union[Dict, FrozenDict], + push_to_hub: bool = False, + **kwargs, + ): + # TODO: handle inference_state + """ + Save all saveable variables of the pipeline to a directory. A pipeline variable can be saved and loaded if its + class implements both a save and loading method. The pipeline is easily reloaded using the + [`~FlaxDiffusionPipeline.from_pretrained`] class method. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to which to save. Will be created if it doesn't exist. + push_to_hub (`bool`, *optional*, defaults to `False`): + Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the + repository you want to push to with `repo_id` (will default to the name of `save_directory` in your + namespace). + kwargs (`Dict[str, Any]`, *optional*): + Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. + """ + self.save_config(save_directory) + + model_index_dict = dict(self.config) + model_index_dict.pop("_class_name") + model_index_dict.pop("_diffusers_version") + model_index_dict.pop("_module", None) + + if push_to_hub: + commit_message = kwargs.pop("commit_message", None) + private = kwargs.pop("private", False) + create_pr = kwargs.pop("create_pr", False) + token = kwargs.pop("token", None) + repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1]) + repo_id = create_repo(repo_id, exist_ok=True, private=private, token=token).repo_id + + for pipeline_component_name in model_index_dict.keys(): + sub_model = getattr(self, pipeline_component_name) + if sub_model is None: + # edge case for saving a pipeline with safety_checker=None + continue + + model_cls = sub_model.__class__ + + save_method_name = None + # search for the model's base class in LOADABLE_CLASSES + for library_name, library_classes in LOADABLE_CLASSES.items(): + library = importlib.import_module(library_name) + for base_class, save_load_methods in library_classes.items(): + class_candidate = getattr(library, base_class, None) + if class_candidate is not None and issubclass(model_cls, class_candidate): + # if we found a suitable base class in LOADABLE_CLASSES then grab its save method + save_method_name = save_load_methods[0] + break + if save_method_name is not None: + break + + save_method = getattr(sub_model, save_method_name) + expects_params = "params" in set(inspect.signature(save_method).parameters.keys()) + + if expects_params: + save_method( + os.path.join(save_directory, pipeline_component_name), params=params[pipeline_component_name] + ) + else: + save_method(os.path.join(save_directory, pipeline_component_name)) + + if push_to_hub: + self._upload_folder( + save_directory, + repo_id, + token=token, + commit_message=commit_message, + create_pr=create_pr, + ) + + @classmethod + @validate_hf_hub_args + def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): + r""" + Instantiate a Flax-based diffusion pipeline from pretrained pipeline weights. + + The pipeline is set in evaluation mode (`model.eval()) by default and dropout modules are deactivated. + + If you get the error message below, you need to finetune the weights for your downstream task: + + ``` + Some weights of FlaxUNet2DConditionModel were not initialized from the model checkpoint at runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + ``` + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *repo id* (for example `runwayml/stable-diffusion-v1-5`) of a pretrained pipeline + hosted on the Hub. + - A path to a *directory* (for example `./my_model_directory`) containing the model weights saved + using [`~FlaxDiffusionPipeline.save_pretrained`]. + dtype (`str` or `jnp.dtype`, *optional*): + Override the default `jnp.dtype` and load the model under this dtype. If `"auto"`, the dtype is + automatically derived from the model's weights. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you're downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (the pipeline components) of the specific pipeline + class. The overwritten components are passed directly to the pipelines `__init__` method. + + + + To use private or [gated models](https://huggingface.co/docs/hub/models-gated#gated-models), log-in with + `huggingface-cli login`. + + + + Examples: + + ```py + >>> from diffusers import FlaxDiffusionPipeline + + >>> # Download pipeline from huggingface.co and cache. + >>> # Requires to be logged in to Hugging Face hub, + >>> # see more in [the documentation](https://huggingface.co/docs/hub/security-tokens) + >>> pipeline, params = FlaxDiffusionPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", + ... revision="bf16", + ... dtype=jnp.bfloat16, + ... ) + + >>> # Download pipeline, but use a different scheduler + >>> from diffusers import FlaxDPMSolverMultistepScheduler + + >>> model_id = "runwayml/stable-diffusion-v1-5" + >>> dpmpp, dpmpp_state = FlaxDPMSolverMultistepScheduler.from_pretrained( + ... model_id, + ... subfolder="scheduler", + ... ) + + >>> dpm_pipe, dpm_params = FlaxStableDiffusionPipeline.from_pretrained( + ... model_id, revision="bf16", dtype=jnp.bfloat16, scheduler=dpmpp + ... ) + >>> dpm_params["scheduler"] = dpmpp_state + ``` + """ + cache_dir = kwargs.pop("cache_dir", None) + resume_download = kwargs.pop("resume_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", False) + token = kwargs.pop("token", None) + revision = kwargs.pop("revision", None) + from_pt = kwargs.pop("from_pt", False) + use_memory_efficient_attention = kwargs.pop("use_memory_efficient_attention", False) + split_head_dim = kwargs.pop("split_head_dim", False) + dtype = kwargs.pop("dtype", None) + + # 1. Download the checkpoints and configs + # use snapshot download here to get it working from from_pretrained + if not os.path.isdir(pretrained_model_name_or_path): + config_dict = cls.load_config( + pretrained_model_name_or_path, + cache_dir=cache_dir, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + ) + # make sure we only download sub-folders and `diffusers` filenames + folder_names = [k for k in config_dict.keys() if not k.startswith("_")] + allow_patterns = [os.path.join(k, "*") for k in folder_names] + allow_patterns += [FLAX_WEIGHTS_NAME, SCHEDULER_CONFIG_NAME, CONFIG_NAME, cls.config_name] + + ignore_patterns = ["*.bin", "*.safetensors"] if not from_pt else [] + ignore_patterns += ["*.onnx", "*.onnx_data", "*.xml", "*.pb"] + + if cls != FlaxDiffusionPipeline: + requested_pipeline_class = cls.__name__ + else: + requested_pipeline_class = config_dict.get("_class_name", cls.__name__) + requested_pipeline_class = ( + requested_pipeline_class + if requested_pipeline_class.startswith("Flax") + else "Flax" + requested_pipeline_class + ) + + user_agent = {"pipeline_class": requested_pipeline_class} + user_agent = http_user_agent(user_agent) + + # download all allow_patterns + cached_folder = snapshot_download( + pretrained_model_name_or_path, + cache_dir=cache_dir, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + user_agent=user_agent, + ) + else: + cached_folder = pretrained_model_name_or_path + + config_dict = cls.load_config(cached_folder) + + # 2. Load the pipeline class, if using custom module then load it from the hub + # if we load from explicit class, let's use it + if cls != FlaxDiffusionPipeline: + pipeline_class = cls + else: + diffusers_module = importlib.import_module(cls.__module__.split(".")[0]) + class_name = ( + config_dict["_class_name"] + if config_dict["_class_name"].startswith("Flax") + else "Flax" + config_dict["_class_name"] + ) + pipeline_class = getattr(diffusers_module, class_name) + + # some modules can be passed directly to the init + # in this case they are already instantiated in `kwargs` + # extract them here + expected_modules, optional_kwargs = cls._get_signature_keys(pipeline_class) + passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} + passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} + + init_dict, unused_kwargs, _ = pipeline_class.extract_init_dict(config_dict, **kwargs) + + # define init kwargs + init_kwargs = {k: init_dict.pop(k) for k in optional_kwargs if k in init_dict} + init_kwargs = {**init_kwargs, **passed_pipe_kwargs} + + # remove `null` components + def load_module(name, value): + if value[0] is None: + return False + if name in passed_class_obj and passed_class_obj[name] is None: + return False + return True + + init_dict = {k: v for k, v in init_dict.items() if load_module(k, v)} + + # Throw nice warnings / errors for fast accelerate loading + if len(unused_kwargs) > 0: + logger.warning( + f"Keyword arguments {unused_kwargs} are not expected by {pipeline_class.__name__} and will be ignored." + ) + + # inference_params + params = {} + + # import it here to avoid circular import + from diffusers import pipelines + + # 3. Load each module in the pipeline + for name, (library_name, class_name) in init_dict.items(): + if class_name is None: + # edge case for when the pipeline was saved with safety_checker=None + init_kwargs[name] = None + continue + + is_pipeline_module = hasattr(pipelines, library_name) + loaded_sub_model = None + sub_model_should_be_defined = True + + # if the model is in a pipeline module, then we load it from the pipeline + if name in passed_class_obj: + # 1. check that passed_class_obj has correct parent class + if not is_pipeline_module: + library = importlib.import_module(library_name) + class_obj = getattr(library, class_name) + importable_classes = LOADABLE_CLASSES[library_name] + class_candidates = {c: getattr(library, c, None) for c in importable_classes.keys()} + + expected_class_obj = None + for class_name, class_candidate in class_candidates.items(): + if class_candidate is not None and issubclass(class_obj, class_candidate): + expected_class_obj = class_candidate + + if not issubclass(passed_class_obj[name].__class__, expected_class_obj): + raise ValueError( + f"{passed_class_obj[name]} is of type: {type(passed_class_obj[name])}, but should be" + f" {expected_class_obj}" + ) + elif passed_class_obj[name] is None: + logger.warning( + f"You have passed `None` for {name} to disable its functionality in {pipeline_class}. Note" + f" that this might lead to problems when using {pipeline_class} and is not recommended." + ) + sub_model_should_be_defined = False + else: + logger.warning( + f"You have passed a non-standard module {passed_class_obj[name]}. We cannot verify whether it" + " has the correct type" + ) + + # set passed class object + loaded_sub_model = passed_class_obj[name] + elif is_pipeline_module: + pipeline_module = getattr(pipelines, library_name) + class_obj = import_flax_or_no_model(pipeline_module, class_name) + + importable_classes = ALL_IMPORTABLE_CLASSES + class_candidates = {c: class_obj for c in importable_classes.keys()} + else: + # else we just import it from the library. + library = importlib.import_module(library_name) + class_obj = import_flax_or_no_model(library, class_name) + + importable_classes = LOADABLE_CLASSES[library_name] + class_candidates = {c: getattr(library, c, None) for c in importable_classes.keys()} + + if loaded_sub_model is None and sub_model_should_be_defined: + load_method_name = None + for class_name, class_candidate in class_candidates.items(): + if class_candidate is not None and issubclass(class_obj, class_candidate): + load_method_name = importable_classes[class_name][1] + + load_method = getattr(class_obj, load_method_name) + + # check if the module is in a subdirectory + if os.path.isdir(os.path.join(cached_folder, name)): + loadable_folder = os.path.join(cached_folder, name) + else: + loaded_sub_model = cached_folder + + if issubclass(class_obj, FlaxModelMixin): + loaded_sub_model, loaded_params = load_method( + loadable_folder, + from_pt=from_pt, + use_memory_efficient_attention=use_memory_efficient_attention, + split_head_dim=split_head_dim, + dtype=dtype, + ) + params[name] = loaded_params + elif is_transformers_available() and issubclass(class_obj, FlaxPreTrainedModel): + if from_pt: + # TODO(Suraj): Fix this in Transformers. We should be able to use `_do_init=False` here + loaded_sub_model = load_method(loadable_folder, from_pt=from_pt) + loaded_params = loaded_sub_model.params + del loaded_sub_model._params + else: + loaded_sub_model, loaded_params = load_method(loadable_folder, _do_init=False) + params[name] = loaded_params + elif issubclass(class_obj, FlaxSchedulerMixin): + loaded_sub_model, scheduler_state = load_method(loadable_folder) + params[name] = scheduler_state + else: + loaded_sub_model = load_method(loadable_folder) + + init_kwargs[name] = loaded_sub_model # UNet(...), # DiffusionSchedule(...) + + # 4. Potentially add passed objects if expected + missing_modules = set(expected_modules) - set(init_kwargs.keys()) + passed_modules = list(passed_class_obj.keys()) + + if len(missing_modules) > 0 and missing_modules <= set(passed_modules): + for module in missing_modules: + init_kwargs[module] = passed_class_obj.get(module, None) + elif len(missing_modules) > 0: + passed_modules = set(list(init_kwargs.keys()) + list(passed_class_obj.keys())) - optional_kwargs + raise ValueError( + f"Pipeline {pipeline_class} expected {expected_modules}, but only {passed_modules} were passed." + ) + + model = pipeline_class(**init_kwargs, dtype=dtype) + return model, params + + @classmethod + def _get_signature_keys(cls, obj): + parameters = inspect.signature(obj.__init__).parameters + required_parameters = {k: v for k, v in parameters.items() if v.default == inspect._empty} + optional_parameters = set({k for k, v in parameters.items() if v.default != inspect._empty}) + expected_modules = set(required_parameters.keys()) - {"self"} + + return expected_modules, optional_parameters + + @property + def components(self) -> Dict[str, Any]: + r""" + + The `self.components` property can be useful to run different pipelines with the same weights and + configurations to not have to re-allocate memory. + + Examples: + + ```py + >>> from diffusers import ( + ... FlaxStableDiffusionPipeline, + ... FlaxStableDiffusionImg2ImgPipeline, + ... ) + + >>> text2img = FlaxStableDiffusionPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", revision="bf16", dtype=jnp.bfloat16 + ... ) + >>> img2img = FlaxStableDiffusionImg2ImgPipeline(**text2img.components) + ``` + + Returns: + A dictionary containing all the modules needed to initialize the pipeline. + """ + expected_modules, optional_parameters = self._get_signature_keys(self) + components = { + k: getattr(self, k) for k in self.config.keys() if not k.startswith("_") and k not in optional_parameters + } + + if set(components.keys()) != expected_modules: + raise ValueError( + f"{self} has been incorrectly initialized or {self.__class__} is incorrectly implemented. Expected" + f" {expected_modules} to be defined, but {components} are defined." + ) + + return components + + @staticmethod + def numpy_to_pil(images): + """ + Convert a NumPy image or a batch of images to a PIL image. + """ + if images.ndim == 3: + images = images[None, ...] + images = (images * 255).round().astype("uint8") + if images.shape[-1] == 1: + # special case for grayscale (single channel) images + pil_images = [Image.fromarray(image.squeeze(), mode="L") for image in images] + else: + pil_images = [Image.fromarray(image) for image in images] + + return pil_images + + # TODO: make it compatible with jax.lax + def progress_bar(self, iterable): + if not hasattr(self, "_progress_bar_config"): + self._progress_bar_config = {} + elif not isinstance(self._progress_bar_config, dict): + raise ValueError( + f"`self._progress_bar_config` should be of type `dict`, but is {type(self._progress_bar_config)}." + ) + + return tqdm(iterable, **self._progress_bar_config) + + def set_progress_bar_config(self, **kwargs): + self._progress_bar_config = kwargs diff --git a/diffusers-0.27.0/src/diffusers/pipelines/pipeline_loading_utils.py b/diffusers-0.27.0/src/diffusers/pipelines/pipeline_loading_utils.py new file mode 100755 index 0000000..30c17ee --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/pipeline_loading_utils.py @@ -0,0 +1,508 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import importlib +import os +import re +import warnings +from pathlib import Path +from typing import Any, Dict, List, Optional, Union + +import torch +from huggingface_hub import ( + model_info, +) +from packaging import version + +from ..utils import ( + SAFETENSORS_WEIGHTS_NAME, + WEIGHTS_NAME, + get_class_from_dynamic_module, + is_peft_available, + is_transformers_available, + logging, +) +from ..utils.torch_utils import is_compiled_module + + +if is_transformers_available(): + import transformers + from transformers import PreTrainedModel + from transformers.utils import FLAX_WEIGHTS_NAME as TRANSFORMERS_FLAX_WEIGHTS_NAME + from transformers.utils import SAFE_WEIGHTS_NAME as TRANSFORMERS_SAFE_WEIGHTS_NAME + from transformers.utils import WEIGHTS_NAME as TRANSFORMERS_WEIGHTS_NAME +from huggingface_hub.utils import validate_hf_hub_args + +from ..utils import FLAX_WEIGHTS_NAME, ONNX_EXTERNAL_WEIGHTS_NAME, ONNX_WEIGHTS_NAME + + +INDEX_FILE = "diffusion_pytorch_model.bin" +CUSTOM_PIPELINE_FILE_NAME = "pipeline.py" +DUMMY_MODULES_FOLDER = "diffusers.utils" +TRANSFORMERS_DUMMY_MODULES_FOLDER = "transformers.utils" +CONNECTED_PIPES_KEYS = ["prior"] + +logger = logging.get_logger(__name__) + +LOADABLE_CLASSES = { + "diffusers": { + "ModelMixin": ["save_pretrained", "from_pretrained"], + "SchedulerMixin": ["save_pretrained", "from_pretrained"], + "DiffusionPipeline": ["save_pretrained", "from_pretrained"], + "OnnxRuntimeModel": ["save_pretrained", "from_pretrained"], + }, + "transformers": { + "PreTrainedTokenizer": ["save_pretrained", "from_pretrained"], + "PreTrainedTokenizerFast": ["save_pretrained", "from_pretrained"], + "PreTrainedModel": ["save_pretrained", "from_pretrained"], + "FeatureExtractionMixin": ["save_pretrained", "from_pretrained"], + "ProcessorMixin": ["save_pretrained", "from_pretrained"], + "ImageProcessingMixin": ["save_pretrained", "from_pretrained"], + }, + "onnxruntime.training": { + "ORTModule": ["save_pretrained", "from_pretrained"], + }, +} + +ALL_IMPORTABLE_CLASSES = {} +for library in LOADABLE_CLASSES: + ALL_IMPORTABLE_CLASSES.update(LOADABLE_CLASSES[library]) + + +def is_safetensors_compatible(filenames, variant=None, passed_components=None) -> bool: + """ + Checking for safetensors compatibility: + - By default, all models are saved with the default pytorch serialization, so we use the list of default pytorch + files to know which safetensors files are needed. + - The model is safetensors compatible only if there is a matching safetensors file for every default pytorch file. + + Converting default pytorch serialized filenames to safetensors serialized filenames: + - For models from the diffusers library, just replace the ".bin" extension with ".safetensors" + - For models from the transformers library, the filename changes from "pytorch_model" to "model", and the ".bin" + extension is replaced with ".safetensors" + """ + pt_filenames = [] + + sf_filenames = set() + + passed_components = passed_components or [] + + for filename in filenames: + _, extension = os.path.splitext(filename) + + if len(filename.split("/")) == 2 and filename.split("/")[0] in passed_components: + continue + + if extension == ".bin": + pt_filenames.append(os.path.normpath(filename)) + elif extension == ".safetensors": + sf_filenames.add(os.path.normpath(filename)) + + for filename in pt_filenames: + # filename = 'foo/bar/baz.bam' -> path = 'foo/bar', filename = 'baz', extension = '.bam' + path, filename = os.path.split(filename) + filename, extension = os.path.splitext(filename) + + if filename.startswith("pytorch_model"): + filename = filename.replace("pytorch_model", "model") + else: + filename = filename + + expected_sf_filename = os.path.normpath(os.path.join(path, filename)) + expected_sf_filename = f"{expected_sf_filename}.safetensors" + if expected_sf_filename not in sf_filenames: + logger.warning(f"{expected_sf_filename} not found") + return False + + return True + + +def variant_compatible_siblings(filenames, variant=None) -> Union[List[os.PathLike], str]: + weight_names = [ + WEIGHTS_NAME, + SAFETENSORS_WEIGHTS_NAME, + FLAX_WEIGHTS_NAME, + ONNX_WEIGHTS_NAME, + ONNX_EXTERNAL_WEIGHTS_NAME, + ] + + if is_transformers_available(): + weight_names += [TRANSFORMERS_WEIGHTS_NAME, TRANSFORMERS_SAFE_WEIGHTS_NAME, TRANSFORMERS_FLAX_WEIGHTS_NAME] + + # model_pytorch, diffusion_model_pytorch, ... + weight_prefixes = [w.split(".")[0] for w in weight_names] + # .bin, .safetensors, ... + weight_suffixs = [w.split(".")[-1] for w in weight_names] + # -00001-of-00002 + transformers_index_format = r"\d{5}-of-\d{5}" + + if variant is not None: + # `diffusion_pytorch_model.fp16.bin` as well as `model.fp16-00001-of-00002.safetensors` + variant_file_re = re.compile( + rf"({'|'.join(weight_prefixes)})\.({variant}|{variant}-{transformers_index_format})\.({'|'.join(weight_suffixs)})$" + ) + # `text_encoder/pytorch_model.bin.index.fp16.json` + variant_index_re = re.compile( + rf"({'|'.join(weight_prefixes)})\.({'|'.join(weight_suffixs)})\.index\.{variant}\.json$" + ) + + # `diffusion_pytorch_model.bin` as well as `model-00001-of-00002.safetensors` + non_variant_file_re = re.compile( + rf"({'|'.join(weight_prefixes)})(-{transformers_index_format})?\.({'|'.join(weight_suffixs)})$" + ) + # `text_encoder/pytorch_model.bin.index.json` + non_variant_index_re = re.compile(rf"({'|'.join(weight_prefixes)})\.({'|'.join(weight_suffixs)})\.index\.json") + + if variant is not None: + variant_weights = {f for f in filenames if variant_file_re.match(f.split("/")[-1]) is not None} + variant_indexes = {f for f in filenames if variant_index_re.match(f.split("/")[-1]) is not None} + variant_filenames = variant_weights | variant_indexes + else: + variant_filenames = set() + + non_variant_weights = {f for f in filenames if non_variant_file_re.match(f.split("/")[-1]) is not None} + non_variant_indexes = {f for f in filenames if non_variant_index_re.match(f.split("/")[-1]) is not None} + non_variant_filenames = non_variant_weights | non_variant_indexes + + # all variant filenames will be used by default + usable_filenames = set(variant_filenames) + + def convert_to_variant(filename): + if "index" in filename: + variant_filename = filename.replace("index", f"index.{variant}") + elif re.compile(f"^(.*?){transformers_index_format}").match(filename) is not None: + variant_filename = f"{filename.split('-')[0]}.{variant}-{'-'.join(filename.split('-')[1:])}" + else: + variant_filename = f"{filename.split('.')[0]}.{variant}.{filename.split('.')[1]}" + return variant_filename + + for f in non_variant_filenames: + variant_filename = convert_to_variant(f) + if variant_filename not in usable_filenames: + usable_filenames.add(f) + + return usable_filenames, variant_filenames + + +@validate_hf_hub_args +def warn_deprecated_model_variant(pretrained_model_name_or_path, token, variant, revision, model_filenames): + info = model_info( + pretrained_model_name_or_path, + token=token, + revision=None, + ) + filenames = {sibling.rfilename for sibling in info.siblings} + comp_model_filenames, _ = variant_compatible_siblings(filenames, variant=revision) + comp_model_filenames = [".".join(f.split(".")[:1] + f.split(".")[2:]) for f in comp_model_filenames] + + if set(model_filenames).issubset(set(comp_model_filenames)): + warnings.warn( + f"You are loading the variant {revision} from {pretrained_model_name_or_path} via `revision='{revision}'` even though you can load it via `variant=`{revision}`. Loading model variants via `revision='{revision}'` is deprecated and will be removed in diffusers v1. Please use `variant='{revision}'` instead.", + FutureWarning, + ) + else: + warnings.warn( + f"You are loading the variant {revision} from {pretrained_model_name_or_path} via `revision='{revision}'`. This behavior is deprecated and will be removed in diffusers v1. One should use `variant='{revision}'` instead. However, it appears that {pretrained_model_name_or_path} currently does not have the required variant filenames in the 'main' branch. \n The Diffusers team and community would be very grateful if you could open an issue: https://github.com/huggingface/diffusers/issues/new with the title '{pretrained_model_name_or_path} is missing {revision} files' so that the correct variant file can be added.", + FutureWarning, + ) + + +def _unwrap_model(model): + """Unwraps a model.""" + if is_compiled_module(model): + model = model._orig_mod + + if is_peft_available(): + from peft import PeftModel + + if isinstance(model, PeftModel): + model = model.base_model.model + + return model + + +def maybe_raise_or_warn( + library_name, library, class_name, importable_classes, passed_class_obj, name, is_pipeline_module +): + """Simple helper method to raise or warn in case incorrect module has been passed""" + if not is_pipeline_module: + library = importlib.import_module(library_name) + class_obj = getattr(library, class_name) + class_candidates = {c: getattr(library, c, None) for c in importable_classes.keys()} + + expected_class_obj = None + for class_name, class_candidate in class_candidates.items(): + if class_candidate is not None and issubclass(class_obj, class_candidate): + expected_class_obj = class_candidate + + # Dynamo wraps the original model in a private class. + # I didn't find a public API to get the original class. + sub_model = passed_class_obj[name] + unwrapped_sub_model = _unwrap_model(sub_model) + model_cls = unwrapped_sub_model.__class__ + + if not issubclass(model_cls, expected_class_obj): + raise ValueError( + f"{passed_class_obj[name]} is of type: {model_cls}, but should be" f" {expected_class_obj}" + ) + else: + logger.warning( + f"You have passed a non-standard module {passed_class_obj[name]}. We cannot verify whether it" + " has the correct type" + ) + + +def get_class_obj_and_candidates( + library_name, class_name, importable_classes, pipelines, is_pipeline_module, component_name=None, cache_dir=None +): + """Simple helper method to retrieve class object of module as well as potential parent class objects""" + component_folder = os.path.join(cache_dir, component_name) + + if is_pipeline_module: + pipeline_module = getattr(pipelines, library_name) + + class_obj = getattr(pipeline_module, class_name) + class_candidates = {c: class_obj for c in importable_classes.keys()} + elif os.path.isfile(os.path.join(component_folder, library_name + ".py")): + # load custom component + class_obj = get_class_from_dynamic_module( + component_folder, module_file=library_name + ".py", class_name=class_name + ) + class_candidates = {c: class_obj for c in importable_classes.keys()} + else: + # else we just import it from the library. + library = importlib.import_module(library_name) + + class_obj = getattr(library, class_name) + class_candidates = {c: getattr(library, c, None) for c in importable_classes.keys()} + + return class_obj, class_candidates + + +def _get_pipeline_class( + class_obj, + config=None, + load_connected_pipeline=False, + custom_pipeline=None, + repo_id=None, + hub_revision=None, + class_name=None, + cache_dir=None, + revision=None, +): + if custom_pipeline is not None: + if custom_pipeline.endswith(".py"): + path = Path(custom_pipeline) + # decompose into folder & file + file_name = path.name + custom_pipeline = path.parent.absolute() + elif repo_id is not None: + file_name = f"{custom_pipeline}.py" + custom_pipeline = repo_id + else: + file_name = CUSTOM_PIPELINE_FILE_NAME + + if repo_id is not None and hub_revision is not None: + # if we load the pipeline code from the Hub + # make sure to overwrite the `revision` + revision = hub_revision + + return get_class_from_dynamic_module( + custom_pipeline, + module_file=file_name, + class_name=class_name, + cache_dir=cache_dir, + revision=revision, + ) + + if class_obj.__name__ != "DiffusionPipeline": + return class_obj + + diffusers_module = importlib.import_module(class_obj.__module__.split(".")[0]) + class_name = class_name or config["_class_name"] + if not class_name: + raise ValueError( + "The class name could not be found in the configuration file. Please make sure to pass the correct `class_name`." + ) + + class_name = class_name[4:] if class_name.startswith("Flax") else class_name + + pipeline_cls = getattr(diffusers_module, class_name) + + if load_connected_pipeline: + from .auto_pipeline import _get_connected_pipeline + + connected_pipeline_cls = _get_connected_pipeline(pipeline_cls) + if connected_pipeline_cls is not None: + logger.info( + f"Loading connected pipeline {connected_pipeline_cls.__name__} instead of {pipeline_cls.__name__} as specified via `load_connected_pipeline=True`" + ) + else: + logger.info(f"{pipeline_cls.__name__} has no connected pipeline class. Loading {pipeline_cls.__name__}.") + + pipeline_cls = connected_pipeline_cls or pipeline_cls + + return pipeline_cls + + +def load_sub_model( + library_name: str, + class_name: str, + importable_classes: List[Any], + pipelines: Any, + is_pipeline_module: bool, + pipeline_class: Any, + torch_dtype: torch.dtype, + provider: Any, + sess_options: Any, + device_map: Optional[Union[Dict[str, torch.device], str]], + max_memory: Optional[Dict[Union[int, str], Union[int, str]]], + offload_folder: Optional[Union[str, os.PathLike]], + offload_state_dict: bool, + model_variants: Dict[str, str], + name: str, + from_flax: bool, + variant: str, + low_cpu_mem_usage: bool, + cached_folder: Union[str, os.PathLike], +): + """Helper method to load the module `name` from `library_name` and `class_name`""" + # retrieve class candidates + class_obj, class_candidates = get_class_obj_and_candidates( + library_name, + class_name, + importable_classes, + pipelines, + is_pipeline_module, + component_name=name, + cache_dir=cached_folder, + ) + + load_method_name = None + # retrieve load method name + for class_name, class_candidate in class_candidates.items(): + if class_candidate is not None and issubclass(class_obj, class_candidate): + load_method_name = importable_classes[class_name][1] + + # if load method name is None, then we have a dummy module -> raise Error + if load_method_name is None: + none_module = class_obj.__module__ + is_dummy_path = none_module.startswith(DUMMY_MODULES_FOLDER) or none_module.startswith( + TRANSFORMERS_DUMMY_MODULES_FOLDER + ) + if is_dummy_path and "dummy" in none_module: + # call class_obj for nice error message of missing requirements + class_obj() + + raise ValueError( + f"The component {class_obj} of {pipeline_class} cannot be loaded as it does not seem to have" + f" any of the loading methods defined in {ALL_IMPORTABLE_CLASSES}." + ) + + load_method = getattr(class_obj, load_method_name) + + # add kwargs to loading method + diffusers_module = importlib.import_module(__name__.split(".")[0]) + loading_kwargs = {} + if issubclass(class_obj, torch.nn.Module): + loading_kwargs["torch_dtype"] = torch_dtype + if issubclass(class_obj, diffusers_module.OnnxRuntimeModel): + loading_kwargs["provider"] = provider + loading_kwargs["sess_options"] = sess_options + + is_diffusers_model = issubclass(class_obj, diffusers_module.ModelMixin) + + if is_transformers_available(): + transformers_version = version.parse(version.parse(transformers.__version__).base_version) + else: + transformers_version = "N/A" + + is_transformers_model = ( + is_transformers_available() + and issubclass(class_obj, PreTrainedModel) + and transformers_version >= version.parse("4.20.0") + ) + + # When loading a transformers model, if the device_map is None, the weights will be initialized as opposed to diffusers. + # To make default loading faster we set the `low_cpu_mem_usage=low_cpu_mem_usage` flag which is `True` by default. + # This makes sure that the weights won't be initialized which significantly speeds up loading. + if is_diffusers_model or is_transformers_model: + loading_kwargs["device_map"] = device_map + loading_kwargs["max_memory"] = max_memory + loading_kwargs["offload_folder"] = offload_folder + loading_kwargs["offload_state_dict"] = offload_state_dict + loading_kwargs["variant"] = model_variants.pop(name, None) + + if from_flax: + loading_kwargs["from_flax"] = True + + # the following can be deleted once the minimum required `transformers` version + # is higher than 4.27 + if ( + is_transformers_model + and loading_kwargs["variant"] is not None + and transformers_version < version.parse("4.27.0") + ): + raise ImportError( + f"When passing `variant='{variant}'`, please make sure to upgrade your `transformers` version to at least 4.27.0.dev0" + ) + elif is_transformers_model and loading_kwargs["variant"] is None: + loading_kwargs.pop("variant") + + # if `from_flax` and model is transformer model, can currently not load with `low_cpu_mem_usage` + if not (from_flax and is_transformers_model): + loading_kwargs["low_cpu_mem_usage"] = low_cpu_mem_usage + else: + loading_kwargs["low_cpu_mem_usage"] = False + + # check if the module is in a subdirectory + if os.path.isdir(os.path.join(cached_folder, name)): + loaded_sub_model = load_method(os.path.join(cached_folder, name), **loading_kwargs) + else: + # else load from the root directory + loaded_sub_model = load_method(cached_folder, **loading_kwargs) + + return loaded_sub_model + + +def _fetch_class_library_tuple(module): + # import it here to avoid circular import + diffusers_module = importlib.import_module(__name__.split(".")[0]) + pipelines = getattr(diffusers_module, "pipelines") + + # register the config from the original module, not the dynamo compiled one + not_compiled_module = _unwrap_model(module) + library = not_compiled_module.__module__.split(".")[0] + + # check if the module is a pipeline module + module_path_items = not_compiled_module.__module__.split(".") + pipeline_dir = module_path_items[-2] if len(module_path_items) > 2 else None + + path = not_compiled_module.__module__.split(".") + is_pipeline_module = pipeline_dir in path and hasattr(pipelines, pipeline_dir) + + # if library is not in LOADABLE_CLASSES, then it is a custom module. + # Or if it's a pipeline module, then the module is inside the pipeline + # folder so we set the library to module name. + if is_pipeline_module: + library = pipeline_dir + elif library not in LOADABLE_CLASSES: + library = not_compiled_module.__module__ + + # retrieve class_name + class_name = not_compiled_module.__class__.__name__ + + return (library, class_name) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/pipeline_utils.py b/diffusers-0.27.0/src/diffusers/pipelines/pipeline_utils.py new file mode 100755 index 0000000..341360d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/pipeline_utils.py @@ -0,0 +1,1771 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import fnmatch +import importlib +import inspect +import os +import re +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import requests +import torch +from huggingface_hub import ( + ModelCard, + create_repo, + hf_hub_download, + model_info, + snapshot_download, +) +from huggingface_hub.utils import OfflineModeIsEnabled, validate_hf_hub_args +from packaging import version +from requests.exceptions import HTTPError +from tqdm.auto import tqdm + +from .. import __version__ +from ..configuration_utils import ConfigMixin +from ..models import AutoencoderKL +from ..models.attention_processor import FusedAttnProcessor2_0 +from ..models.modeling_utils import _LOW_CPU_MEM_USAGE_DEFAULT +from ..schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME +from ..utils import ( + CONFIG_NAME, + DEPRECATED_REVISION_ARGS, + BaseOutput, + PushToHubMixin, + deprecate, + is_accelerate_available, + is_accelerate_version, + is_torch_npu_available, + is_torch_version, + logging, + numpy_to_pil, +) +from ..utils.hub_utils import load_or_create_model_card, populate_model_card +from ..utils.torch_utils import is_compiled_module + + +if is_torch_npu_available(): + import torch_npu # noqa: F401 + + +from .pipeline_loading_utils import ( + ALL_IMPORTABLE_CLASSES, + CONNECTED_PIPES_KEYS, + CUSTOM_PIPELINE_FILE_NAME, + LOADABLE_CLASSES, + _fetch_class_library_tuple, + _get_pipeline_class, + _unwrap_model, + is_safetensors_compatible, + load_sub_model, + maybe_raise_or_warn, + variant_compatible_siblings, + warn_deprecated_model_variant, +) + + +if is_accelerate_available(): + import accelerate + + +LIBRARIES = [] +for library in LOADABLE_CLASSES: + LIBRARIES.append(library) + +logger = logging.get_logger(__name__) + + +@dataclass +class ImagePipelineOutput(BaseOutput): + """ + Output class for image pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + num_channels)`. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +@dataclass +class AudioPipelineOutput(BaseOutput): + """ + Output class for audio pipelines. + + Args: + audios (`np.ndarray`) + List of denoised audio samples of a NumPy array of shape `(batch_size, num_channels, sample_rate)`. + """ + + audios: np.ndarray + + +class DiffusionPipeline(ConfigMixin, PushToHubMixin): + r""" + Base class for all pipelines. + + [`DiffusionPipeline`] stores all components (models, schedulers, and processors) for diffusion pipelines and + provides methods for loading, downloading and saving models. It also includes methods to: + + - move all PyTorch modules to the device of your choice + - enable/disable the progress bar for the denoising iteration + + Class attributes: + + - **config_name** (`str`) -- The configuration filename that stores the class and module names of all the + diffusion pipeline's components. + - **_optional_components** (`List[str]`) -- List of all optional components that don't have to be passed to the + pipeline to function (should be overridden by subclasses). + """ + + config_name = "model_index.json" + model_cpu_offload_seq = None + _optional_components = [] + _exclude_from_cpu_offload = [] + _load_connected_pipes = False + _is_onnx = False + + def register_modules(self, **kwargs): + for name, module in kwargs.items(): + # retrieve library + if module is None or isinstance(module, (tuple, list)) and module[0] is None: + register_dict = {name: (None, None)} + else: + library, class_name = _fetch_class_library_tuple(module) + register_dict = {name: (library, class_name)} + + # save model index config + self.register_to_config(**register_dict) + + # set models + setattr(self, name, module) + + def __setattr__(self, name: str, value: Any): + if name in self.__dict__ and hasattr(self.config, name): + # We need to overwrite the config if name exists in config + if isinstance(getattr(self.config, name), (tuple, list)): + if value is not None and self.config[name][0] is not None: + class_library_tuple = _fetch_class_library_tuple(value) + else: + class_library_tuple = (None, None) + + self.register_to_config(**{name: class_library_tuple}) + else: + self.register_to_config(**{name: value}) + + super().__setattr__(name, value) + + def save_pretrained( + self, + save_directory: Union[str, os.PathLike], + safe_serialization: bool = True, + variant: Optional[str] = None, + push_to_hub: bool = False, + **kwargs, + ): + """ + Save all saveable variables of the pipeline to a directory. A pipeline variable can be saved and loaded if its + class implements both a save and loading method. The pipeline is easily reloaded using the + [`~DiffusionPipeline.from_pretrained`] class method. + + Arguments: + save_directory (`str` or `os.PathLike`): + Directory to save a pipeline to. Will be created if it doesn't exist. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether to save the model using `safetensors` or the traditional PyTorch way with `pickle`. + variant (`str`, *optional*): + If specified, weights are saved in the format `pytorch_model..bin`. + push_to_hub (`bool`, *optional*, defaults to `False`): + Whether or not to push your model to the Hugging Face model hub after saving it. You can specify the + repository you want to push to with `repo_id` (will default to the name of `save_directory` in your + namespace). + kwargs (`Dict[str, Any]`, *optional*): + Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. + """ + model_index_dict = dict(self.config) + model_index_dict.pop("_class_name", None) + model_index_dict.pop("_diffusers_version", None) + model_index_dict.pop("_module", None) + model_index_dict.pop("_name_or_path", None) + + if push_to_hub: + commit_message = kwargs.pop("commit_message", None) + private = kwargs.pop("private", False) + create_pr = kwargs.pop("create_pr", False) + token = kwargs.pop("token", None) + repo_id = kwargs.pop("repo_id", save_directory.split(os.path.sep)[-1]) + repo_id = create_repo(repo_id, exist_ok=True, private=private, token=token).repo_id + + expected_modules, optional_kwargs = self._get_signature_keys(self) + + def is_saveable_module(name, value): + if name not in expected_modules: + return False + if name in self._optional_components and value[0] is None: + return False + return True + + model_index_dict = {k: v for k, v in model_index_dict.items() if is_saveable_module(k, v)} + for pipeline_component_name in model_index_dict.keys(): + sub_model = getattr(self, pipeline_component_name) + model_cls = sub_model.__class__ + + # Dynamo wraps the original model in a private class. + # I didn't find a public API to get the original class. + if is_compiled_module(sub_model): + sub_model = _unwrap_model(sub_model) + model_cls = sub_model.__class__ + + save_method_name = None + # search for the model's base class in LOADABLE_CLASSES + for library_name, library_classes in LOADABLE_CLASSES.items(): + if library_name in sys.modules: + library = importlib.import_module(library_name) + else: + logger.info( + f"{library_name} is not installed. Cannot save {pipeline_component_name} as {library_classes} from {library_name}" + ) + + for base_class, save_load_methods in library_classes.items(): + class_candidate = getattr(library, base_class, None) + if class_candidate is not None and issubclass(model_cls, class_candidate): + # if we found a suitable base class in LOADABLE_CLASSES then grab its save method + save_method_name = save_load_methods[0] + break + if save_method_name is not None: + break + + if save_method_name is None: + logger.warning( + f"self.{pipeline_component_name}={sub_model} of type {type(sub_model)} cannot be saved." + ) + # make sure that unsaveable components are not tried to be loaded afterward + self.register_to_config(**{pipeline_component_name: (None, None)}) + continue + + save_method = getattr(sub_model, save_method_name) + + # Call the save method with the argument safe_serialization only if it's supported + save_method_signature = inspect.signature(save_method) + save_method_accept_safe = "safe_serialization" in save_method_signature.parameters + save_method_accept_variant = "variant" in save_method_signature.parameters + + save_kwargs = {} + if save_method_accept_safe: + save_kwargs["safe_serialization"] = safe_serialization + if save_method_accept_variant: + save_kwargs["variant"] = variant + + save_method(os.path.join(save_directory, pipeline_component_name), **save_kwargs) + + # finally save the config + self.save_config(save_directory) + + if push_to_hub: + # Create a new empty model card and eventually tag it + model_card = load_or_create_model_card(repo_id, token=token, is_pipeline=True) + model_card = populate_model_card(model_card) + model_card.save(os.path.join(save_directory, "README.md")) + + self._upload_folder( + save_directory, + repo_id, + token=token, + commit_message=commit_message, + create_pr=create_pr, + ) + + def to(self, *args, **kwargs): + r""" + Performs Pipeline dtype and/or device conversion. A torch.dtype and torch.device are inferred from the + arguments of `self.to(*args, **kwargs).` + + + + If the pipeline already has the correct torch.dtype and torch.device, then it is returned as is. Otherwise, + the returned pipeline is a copy of self with the desired torch.dtype and torch.device. + + + + + Here are the ways to call `to`: + + - `to(dtype, silence_dtype_warnings=False) → DiffusionPipeline` to return a pipeline with the specified + [`dtype`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.dtype) + - `to(device, silence_dtype_warnings=False) → DiffusionPipeline` to return a pipeline with the specified + [`device`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.device) + - `to(device=None, dtype=None, silence_dtype_warnings=False) → DiffusionPipeline` to return a pipeline with the + specified [`device`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.device) and + [`dtype`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.dtype) + + Arguments: + dtype (`torch.dtype`, *optional*): + Returns a pipeline with the specified + [`dtype`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.dtype) + device (`torch.Device`, *optional*): + Returns a pipeline with the specified + [`device`](https://pytorch.org/docs/stable/tensor_attributes.html#torch.device) + silence_dtype_warnings (`str`, *optional*, defaults to `False`): + Whether to omit warnings if the target `dtype` is not compatible with the target `device`. + + Returns: + [`DiffusionPipeline`]: The pipeline converted to specified `dtype` and/or `dtype`. + """ + dtype = kwargs.pop("dtype", None) + device = kwargs.pop("device", None) + silence_dtype_warnings = kwargs.pop("silence_dtype_warnings", False) + + dtype_arg = None + device_arg = None + if len(args) == 1: + if isinstance(args[0], torch.dtype): + dtype_arg = args[0] + else: + device_arg = torch.device(args[0]) if args[0] is not None else None + elif len(args) == 2: + if isinstance(args[0], torch.dtype): + raise ValueError( + "When passing two arguments, make sure the first corresponds to `device` and the second to `dtype`." + ) + device_arg = torch.device(args[0]) if args[0] is not None else None + dtype_arg = args[1] + elif len(args) > 2: + raise ValueError("Please make sure to pass at most two arguments (`device` and `dtype`) `.to(...)`") + + if dtype is not None and dtype_arg is not None: + raise ValueError( + "You have passed `dtype` both as an argument and as a keyword argument. Please only pass one of the two." + ) + + dtype = dtype or dtype_arg + + if device is not None and device_arg is not None: + raise ValueError( + "You have passed `device` both as an argument and as a keyword argument. Please only pass one of the two." + ) + + device = device or device_arg + + # throw warning if pipeline is in "offloaded"-mode but user tries to manually set to GPU. + def module_is_sequentially_offloaded(module): + if not is_accelerate_available() or is_accelerate_version("<", "0.14.0"): + return False + + return hasattr(module, "_hf_hook") and not isinstance( + module._hf_hook, (accelerate.hooks.CpuOffload, accelerate.hooks.AlignDevicesHook) + ) + + def module_is_offloaded(module): + if not is_accelerate_available() or is_accelerate_version("<", "0.17.0.dev0"): + return False + + return hasattr(module, "_hf_hook") and isinstance(module._hf_hook, accelerate.hooks.CpuOffload) + + # .to("cuda") would raise an error if the pipeline is sequentially offloaded, so we raise our own to make it clearer + pipeline_is_sequentially_offloaded = any( + module_is_sequentially_offloaded(module) for _, module in self.components.items() + ) + if pipeline_is_sequentially_offloaded and device and torch.device(device).type == "cuda": + raise ValueError( + "It seems like you have activated sequential model offloading by calling `enable_sequential_cpu_offload`, but are now attempting to move the pipeline to GPU. This is not compatible with offloading. Please, move your pipeline `.to('cpu')` or consider removing the move altogether if you use sequential offloading." + ) + + # Display a warning in this case (the operation succeeds but the benefits are lost) + pipeline_is_offloaded = any(module_is_offloaded(module) for _, module in self.components.items()) + if pipeline_is_offloaded and device and torch.device(device).type == "cuda": + logger.warning( + f"It seems like you have activated model offloading by calling `enable_model_cpu_offload`, but are now manually moving the pipeline to GPU. It is strongly recommended against doing so as memory gains from offloading are likely to be lost. Offloading automatically takes care of moving the individual components {', '.join(self.components.keys())} to GPU when needed. To make sure offloading works as expected, you should consider moving the pipeline back to CPU: `pipeline.to('cpu')` or removing the move altogether if you use offloading." + ) + + module_names, _ = self._get_signature_keys(self) + modules = [getattr(self, n, None) for n in module_names] + modules = [m for m in modules if isinstance(m, torch.nn.Module)] + + is_offloaded = pipeline_is_offloaded or pipeline_is_sequentially_offloaded + for module in modules: + is_loaded_in_8bit = hasattr(module, "is_loaded_in_8bit") and module.is_loaded_in_8bit + + if is_loaded_in_8bit and dtype is not None: + logger.warning( + f"The module '{module.__class__.__name__}' has been loaded in 8bit and conversion to {dtype} is not yet supported. Module is still in 8bit precision." + ) + + if is_loaded_in_8bit and device is not None: + logger.warning( + f"The module '{module.__class__.__name__}' has been loaded in 8bit and moving it to {dtype} via `.to()` is not yet supported. Module is still on {module.device}." + ) + else: + module.to(device, dtype) + + if ( + module.dtype == torch.float16 + and str(device) in ["cpu"] + and not silence_dtype_warnings + and not is_offloaded + ): + logger.warning( + "Pipelines loaded with `dtype=torch.float16` cannot run with `cpu` device. It" + " is not recommended to move them to `cpu` as running them will fail. Please make" + " sure to use an accelerator to run the pipeline in inference, due to the lack of" + " support for`float16` operations on this device in PyTorch. Please, remove the" + " `torch_dtype=torch.float16` argument, or use another device for inference." + ) + return self + + @property + def device(self) -> torch.device: + r""" + Returns: + `torch.device`: The torch device on which the pipeline is located. + """ + module_names, _ = self._get_signature_keys(self) + modules = [getattr(self, n, None) for n in module_names] + modules = [m for m in modules if isinstance(m, torch.nn.Module)] + + for module in modules: + return module.device + + return torch.device("cpu") + + @property + def dtype(self) -> torch.dtype: + r""" + Returns: + `torch.dtype`: The torch dtype on which the pipeline is located. + """ + module_names, _ = self._get_signature_keys(self) + modules = [getattr(self, n, None) for n in module_names] + modules = [m for m in modules if isinstance(m, torch.nn.Module)] + + for module in modules: + return module.dtype + + return torch.float32 + + @classmethod + @validate_hf_hub_args + def from_pretrained(cls, pretrained_model_name_or_path: Optional[Union[str, os.PathLike]], **kwargs): + r""" + Instantiate a PyTorch diffusion pipeline from pretrained pipeline weights. + + The pipeline is set in evaluation mode (`model.eval()`) by default. + + If you get the error message below, you need to finetune the weights for your downstream task: + + ``` + Some weights of UNet2DConditionModel were not initialized from the model checkpoint at runwayml/stable-diffusion-v1-5 and are newly initialized because the shapes did not match: + - conv_in.weight: found shape torch.Size([320, 4, 3, 3]) in the checkpoint and torch.Size([320, 9, 3, 3]) in the model instantiated + You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference. + ``` + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *repo id* (for example `CompVis/ldm-text2im-large-256`) of a pretrained pipeline + hosted on the Hub. + - A path to a *directory* (for example `./my_pipeline_directory/`) containing pipeline weights + saved using + [`~DiffusionPipeline.save_pretrained`]. + torch_dtype (`str` or `torch.dtype`, *optional*): + Override the default `torch.dtype` and load the model with another dtype. If "auto" is passed, the + dtype is automatically derived from the model's weights. + custom_pipeline (`str`, *optional*): + + + + 🧪 This is an experimental feature and may change in the future. + + + + Can be either: + + - A string, the *repo id* (for example `hf-internal-testing/diffusers-dummy-pipeline`) of a custom + pipeline hosted on the Hub. The repository must contain a file called pipeline.py that defines + the custom pipeline. + - A string, the *file name* of a community pipeline hosted on GitHub under + [Community](https://github.com/huggingface/diffusers/tree/main/examples/community). Valid file + names must match the file name and not the pipeline script (`clip_guided_stable_diffusion` + instead of `clip_guided_stable_diffusion.py`). Community pipelines are always loaded from the + current main branch of GitHub. + - A path to a directory (`./my_pipeline_directory/`) containing a custom pipeline. The directory + must contain a file called `pipeline.py` that defines the custom pipeline. + + For more information on how to load and create custom pipelines, please have a look at [Loading and + Adding Custom + Pipelines](https://huggingface.co/docs/diffusers/using-diffusers/custom_pipeline_overview) + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + custom_revision (`str`, *optional*): + The specific model version to use. It can be a branch name, a tag name, or a commit id similar to + `revision` when loading a custom pipeline from the Hub. Defaults to the latest stable 🤗 Diffusers version. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you’re downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + device_map (`str` or `Dict[str, Union[int, str, torch.device]]`, *optional*): + A map that specifies where each submodule should go. It doesn’t need to be defined for each + parameter/buffer name; once a given module name is inside, every submodule of it will be sent to the + same device. + + Set `device_map="auto"` to have 🤗 Accelerate automatically compute the most optimized `device_map`. For + more information about each option see [designing a device + map](https://hf.co/docs/accelerate/main/en/usage_guides/big_modeling#designing-a-device-map). + max_memory (`Dict`, *optional*): + A dictionary device identifier for the maximum memory. Will default to the maximum memory available for + each GPU and the available CPU RAM if unset. + offload_folder (`str` or `os.PathLike`, *optional*): + The path to offload weights if device_map contains the value `"disk"`. + offload_state_dict (`bool`, *optional*): + If `True`, temporarily offloads the CPU state dict to the hard drive to avoid running out of CPU RAM if + the weight of the CPU state dict + the biggest shard of the checkpoint does not fit. Defaults to `True` + when there is some disk offload. + low_cpu_mem_usage (`bool`, *optional*, defaults to `True` if torch version >= 1.9.0 else `False`): + Speed up model loading only loading the pretrained weights and not initializing the weights. This also + tries to not use more than 1x model size in CPU memory (including peak memory) while loading the model. + Only supported for PyTorch >= 1.9.0. If you are using an older version of PyTorch, setting this + argument to `True` will raise an error. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the safetensors weights are downloaded if they're available **and** if the + safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors + weights. If set to `False`, safetensors weights are not loaded. + use_onnx (`bool`, *optional*, defaults to `None`): + If set to `True`, ONNX weights will always be downloaded if present. If set to `False`, ONNX weights + will never be downloaded. By default `use_onnx` defaults to the `_is_onnx` class attribute which is + `False` for non-ONNX pipelines and `True` for ONNX pipelines. ONNX weights include both files ending + with `.onnx` and `.pb`. + kwargs (remaining dictionary of keyword arguments, *optional*): + Can be used to overwrite load and saveable variables (the pipeline components of the specific pipeline + class). The overwritten components are passed directly to the pipelines `__init__` method. See example + below for more information. + variant (`str`, *optional*): + Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when + loading `from_flax`. + + + + To use private or [gated](https://huggingface.co/docs/hub/models-gated#gated-models) models, log-in with + `huggingface-cli login`. + + + + Examples: + + ```py + >>> from diffusers import DiffusionPipeline + + >>> # Download pipeline from huggingface.co and cache. + >>> pipeline = DiffusionPipeline.from_pretrained("CompVis/ldm-text2im-large-256") + + >>> # Download pipeline that requires an authorization token + >>> # For more information on access tokens, please refer to this section + >>> # of the documentation](https://huggingface.co/docs/hub/security-tokens) + >>> pipeline = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + + >>> # Use a different scheduler + >>> from diffusers import LMSDiscreteScheduler + + >>> scheduler = LMSDiscreteScheduler.from_config(pipeline.scheduler.config) + >>> pipeline.scheduler = scheduler + ``` + """ + cache_dir = kwargs.pop("cache_dir", None) + resume_download = kwargs.pop("resume_download", False) + force_download = kwargs.pop("force_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) + revision = kwargs.pop("revision", None) + from_flax = kwargs.pop("from_flax", False) + torch_dtype = kwargs.pop("torch_dtype", None) + custom_pipeline = kwargs.pop("custom_pipeline", None) + custom_revision = kwargs.pop("custom_revision", None) + provider = kwargs.pop("provider", None) + sess_options = kwargs.pop("sess_options", None) + device_map = kwargs.pop("device_map", None) + max_memory = kwargs.pop("max_memory", None) + offload_folder = kwargs.pop("offload_folder", None) + offload_state_dict = kwargs.pop("offload_state_dict", False) + low_cpu_mem_usage = kwargs.pop("low_cpu_mem_usage", _LOW_CPU_MEM_USAGE_DEFAULT) + variant = kwargs.pop("variant", None) + use_safetensors = kwargs.pop("use_safetensors", None) + use_onnx = kwargs.pop("use_onnx", None) + load_connected_pipeline = kwargs.pop("load_connected_pipeline", False) + + if low_cpu_mem_usage and not is_accelerate_available(): + low_cpu_mem_usage = False + logger.warning( + "Cannot initialize model with low cpu memory usage because `accelerate` was not found in the" + " environment. Defaulting to `low_cpu_mem_usage=False`. It is strongly recommended to install" + " `accelerate` for faster and less memory-intense model loading. You can do so with: \n```\npip" + " install accelerate\n```\n." + ) + + if device_map is not None and not is_torch_version(">=", "1.9.0"): + raise NotImplementedError( + "Loading and dispatching requires torch >= 1.9.0. Please either update your PyTorch version or set" + " `device_map=None`." + ) + + if low_cpu_mem_usage is True and not is_torch_version(">=", "1.9.0"): + raise NotImplementedError( + "Low memory initialization requires torch >= 1.9.0. Please either update your PyTorch version or set" + " `low_cpu_mem_usage=False`." + ) + + if low_cpu_mem_usage is False and device_map is not None: + raise ValueError( + f"You cannot set `low_cpu_mem_usage` to False while using device_map={device_map} for loading and" + " dispatching. Please make sure to set `low_cpu_mem_usage=True`." + ) + + # 1. Download the checkpoints and configs + # use snapshot download here to get it working from from_pretrained + if not os.path.isdir(pretrained_model_name_or_path): + if pretrained_model_name_or_path.count("/") > 1: + raise ValueError( + f'The provided pretrained_model_name_or_path "{pretrained_model_name_or_path}"' + " is neither a valid local path nor a valid repo id. Please check the parameter." + ) + cached_folder = cls.download( + pretrained_model_name_or_path, + cache_dir=cache_dir, + resume_download=resume_download, + force_download=force_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + from_flax=from_flax, + use_safetensors=use_safetensors, + use_onnx=use_onnx, + custom_pipeline=custom_pipeline, + custom_revision=custom_revision, + variant=variant, + load_connected_pipeline=load_connected_pipeline, + **kwargs, + ) + else: + cached_folder = pretrained_model_name_or_path + + config_dict = cls.load_config(cached_folder) + + # pop out "_ignore_files" as it is only needed for download + config_dict.pop("_ignore_files", None) + + # 2. Define which model components should load variants + # We retrieve the information by matching whether variant + # model checkpoints exist in the subfolders + model_variants = {} + if variant is not None: + for folder in os.listdir(cached_folder): + folder_path = os.path.join(cached_folder, folder) + is_folder = os.path.isdir(folder_path) and folder in config_dict + variant_exists = is_folder and any( + p.split(".")[1].startswith(variant) for p in os.listdir(folder_path) + ) + if variant_exists: + model_variants[folder] = variant + + # 3. Load the pipeline class, if using custom module then load it from the hub + # if we load from explicit class, let's use it + custom_class_name = None + if os.path.isfile(os.path.join(cached_folder, f"{custom_pipeline}.py")): + custom_pipeline = os.path.join(cached_folder, f"{custom_pipeline}.py") + elif isinstance(config_dict["_class_name"], (list, tuple)) and os.path.isfile( + os.path.join(cached_folder, f"{config_dict['_class_name'][0]}.py") + ): + custom_pipeline = os.path.join(cached_folder, f"{config_dict['_class_name'][0]}.py") + custom_class_name = config_dict["_class_name"][1] + + pipeline_class = _get_pipeline_class( + cls, + config_dict, + load_connected_pipeline=load_connected_pipeline, + custom_pipeline=custom_pipeline, + class_name=custom_class_name, + cache_dir=cache_dir, + revision=custom_revision, + ) + + # DEPRECATED: To be removed in 1.0.0 + if pipeline_class.__name__ == "StableDiffusionInpaintPipeline" and version.parse( + version.parse(config_dict["_diffusers_version"]).base_version + ) <= version.parse("0.5.1"): + from diffusers import StableDiffusionInpaintPipeline, StableDiffusionInpaintPipelineLegacy + + pipeline_class = StableDiffusionInpaintPipelineLegacy + + deprecation_message = ( + "You are using a legacy checkpoint for inpainting with Stable Diffusion, therefore we are loading the" + f" {StableDiffusionInpaintPipelineLegacy} class instead of {StableDiffusionInpaintPipeline}. For" + " better inpainting results, we strongly suggest using Stable Diffusion's official inpainting" + " checkpoint: https://huggingface.co/runwayml/stable-diffusion-inpainting instead or adapting your" + f" checkpoint {pretrained_model_name_or_path} to the format of" + " https://huggingface.co/runwayml/stable-diffusion-inpainting. Note that we do not actively maintain" + " the {StableDiffusionInpaintPipelineLegacy} class and will likely remove it in version 1.0.0." + ) + deprecate("StableDiffusionInpaintPipelineLegacy", "1.0.0", deprecation_message, standard_warn=False) + + # 4. Define expected modules given pipeline signature + # and define non-None initialized modules (=`init_kwargs`) + + # some modules can be passed directly to the init + # in this case they are already instantiated in `kwargs` + # extract them here + expected_modules, optional_kwargs = cls._get_signature_keys(pipeline_class) + passed_class_obj = {k: kwargs.pop(k) for k in expected_modules if k in kwargs} + passed_pipe_kwargs = {k: kwargs.pop(k) for k in optional_kwargs if k in kwargs} + + init_dict, unused_kwargs, _ = pipeline_class.extract_init_dict(config_dict, **kwargs) + + # define init kwargs and make sure that optional component modules are filtered out + init_kwargs = { + k: init_dict.pop(k) + for k in optional_kwargs + if k in init_dict and k not in pipeline_class._optional_components + } + init_kwargs = {**init_kwargs, **passed_pipe_kwargs} + + # remove `null` components + def load_module(name, value): + if value[0] is None: + return False + if name in passed_class_obj and passed_class_obj[name] is None: + return False + return True + + init_dict = {k: v for k, v in init_dict.items() if load_module(k, v)} + + # Special case: safety_checker must be loaded separately when using `from_flax` + if from_flax and "safety_checker" in init_dict and "safety_checker" not in passed_class_obj: + raise NotImplementedError( + "The safety checker cannot be automatically loaded when loading weights `from_flax`." + " Please, pass `safety_checker=None` to `from_pretrained`, and load the safety checker" + " separately if you need it." + ) + + # 5. Throw nice warnings / errors for fast accelerate loading + if len(unused_kwargs) > 0: + logger.warning( + f"Keyword arguments {unused_kwargs} are not expected by {pipeline_class.__name__} and will be ignored." + ) + + # import it here to avoid circular import + from diffusers import pipelines + + # 6. Load each module in the pipeline + for name, (library_name, class_name) in logging.tqdm(init_dict.items(), desc="Loading pipeline components..."): + # 6.1 - now that JAX/Flax is an official framework of the library, we might load from Flax names + class_name = class_name[4:] if class_name.startswith("Flax") else class_name + + # 6.2 Define all importable classes + is_pipeline_module = hasattr(pipelines, library_name) + importable_classes = ALL_IMPORTABLE_CLASSES + loaded_sub_model = None + + # 6.3 Use passed sub model or load class_name from library_name + if name in passed_class_obj: + # if the model is in a pipeline module, then we load it from the pipeline + # check that passed_class_obj has correct parent class + maybe_raise_or_warn( + library_name, library, class_name, importable_classes, passed_class_obj, name, is_pipeline_module + ) + + loaded_sub_model = passed_class_obj[name] + else: + # load sub model + loaded_sub_model = load_sub_model( + library_name=library_name, + class_name=class_name, + importable_classes=importable_classes, + pipelines=pipelines, + is_pipeline_module=is_pipeline_module, + pipeline_class=pipeline_class, + torch_dtype=torch_dtype, + provider=provider, + sess_options=sess_options, + device_map=device_map, + max_memory=max_memory, + offload_folder=offload_folder, + offload_state_dict=offload_state_dict, + model_variants=model_variants, + name=name, + from_flax=from_flax, + variant=variant, + low_cpu_mem_usage=low_cpu_mem_usage, + cached_folder=cached_folder, + ) + logger.info( + f"Loaded {name} as {class_name} from `{name}` subfolder of {pretrained_model_name_or_path}." + ) + + init_kwargs[name] = loaded_sub_model # UNet(...), # DiffusionSchedule(...) + + if pipeline_class._load_connected_pipes and os.path.isfile(os.path.join(cached_folder, "README.md")): + modelcard = ModelCard.load(os.path.join(cached_folder, "README.md")) + connected_pipes = {prefix: getattr(modelcard.data, prefix, [None])[0] for prefix in CONNECTED_PIPES_KEYS} + load_kwargs = { + "cache_dir": cache_dir, + "resume_download": resume_download, + "force_download": force_download, + "proxies": proxies, + "local_files_only": local_files_only, + "token": token, + "revision": revision, + "torch_dtype": torch_dtype, + "custom_pipeline": custom_pipeline, + "custom_revision": custom_revision, + "provider": provider, + "sess_options": sess_options, + "device_map": device_map, + "max_memory": max_memory, + "offload_folder": offload_folder, + "offload_state_dict": offload_state_dict, + "low_cpu_mem_usage": low_cpu_mem_usage, + "variant": variant, + "use_safetensors": use_safetensors, + } + + def get_connected_passed_kwargs(prefix): + connected_passed_class_obj = { + k.replace(f"{prefix}_", ""): w for k, w in passed_class_obj.items() if k.split("_")[0] == prefix + } + connected_passed_pipe_kwargs = { + k.replace(f"{prefix}_", ""): w for k, w in passed_pipe_kwargs.items() if k.split("_")[0] == prefix + } + + connected_passed_kwargs = {**connected_passed_class_obj, **connected_passed_pipe_kwargs} + return connected_passed_kwargs + + connected_pipes = { + prefix: DiffusionPipeline.from_pretrained( + repo_id, **load_kwargs.copy(), **get_connected_passed_kwargs(prefix) + ) + for prefix, repo_id in connected_pipes.items() + if repo_id is not None + } + + for prefix, connected_pipe in connected_pipes.items(): + # add connected pipes to `init_kwargs` with _, e.g. "prior_text_encoder" + init_kwargs.update( + {"_".join([prefix, name]): component for name, component in connected_pipe.components.items()} + ) + + # 7. Potentially add passed objects if expected + missing_modules = set(expected_modules) - set(init_kwargs.keys()) + passed_modules = list(passed_class_obj.keys()) + optional_modules = pipeline_class._optional_components + if len(missing_modules) > 0 and missing_modules <= set(passed_modules + optional_modules): + for module in missing_modules: + init_kwargs[module] = passed_class_obj.get(module, None) + elif len(missing_modules) > 0: + passed_modules = set(list(init_kwargs.keys()) + list(passed_class_obj.keys())) - optional_kwargs + raise ValueError( + f"Pipeline {pipeline_class} expected {expected_modules}, but only {passed_modules} were passed." + ) + + # 8. Instantiate the pipeline + model = pipeline_class(**init_kwargs) + + # 9. Save where the model was instantiated from + model.register_to_config(_name_or_path=pretrained_model_name_or_path) + return model + + @property + def name_or_path(self) -> str: + return getattr(self.config, "_name_or_path", None) + + @property + def _execution_device(self): + r""" + Returns the device on which the pipeline's models will be executed. After calling + [`~DiffusionPipeline.enable_sequential_cpu_offload`] the execution device can only be inferred from + Accelerate's module hooks. + """ + for name, model in self.components.items(): + if not isinstance(model, torch.nn.Module) or name in self._exclude_from_cpu_offload: + continue + + if not hasattr(model, "_hf_hook"): + return self.device + for module in model.modules(): + if ( + hasattr(module, "_hf_hook") + and hasattr(module._hf_hook, "execution_device") + and module._hf_hook.execution_device is not None + ): + return torch.device(module._hf_hook.execution_device) + return self.device + + def enable_model_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = "cuda"): + r""" + Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared + to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` + method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with + `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`. + + Arguments: + gpu_id (`int`, *optional*): + The ID of the accelerator that shall be used in inference. If not specified, it will default to 0. + device (`torch.Device` or `str`, *optional*, defaults to "cuda"): + The PyTorch device type of the accelerator that shall be used in inference. If not specified, it will + default to "cuda". + """ + if self.model_cpu_offload_seq is None: + raise ValueError( + "Model CPU offload cannot be enabled because no `model_cpu_offload_seq` class attribute is set." + ) + + if is_accelerate_available() and is_accelerate_version(">=", "0.17.0.dev0"): + from accelerate import cpu_offload_with_hook + else: + raise ImportError("`enable_model_cpu_offload` requires `accelerate v0.17.0` or higher.") + + torch_device = torch.device(device) + device_index = torch_device.index + + if gpu_id is not None and device_index is not None: + raise ValueError( + f"You have passed both `gpu_id`={gpu_id} and an index as part of the passed device `device`={device}" + f"Cannot pass both. Please make sure to either not define `gpu_id` or not pass the index as part of the device: `device`={torch_device.type}" + ) + + # _offload_gpu_id should be set to passed gpu_id (or id in passed `device`) or default to previously set id or default to 0 + self._offload_gpu_id = gpu_id or torch_device.index or getattr(self, "_offload_gpu_id", 0) + + device_type = torch_device.type + device = torch.device(f"{device_type}:{self._offload_gpu_id}") + self._offload_device = device + + if self.device.type != "cpu": + self.to("cpu", silence_dtype_warnings=True) + device_mod = getattr(torch, self.device.type, None) + if hasattr(device_mod, "empty_cache") and device_mod.is_available(): + device_mod.empty_cache() # otherwise we don't see the memory savings (but they probably exist) + + all_model_components = {k: v for k, v in self.components.items() if isinstance(v, torch.nn.Module)} + + self._all_hooks = [] + hook = None + for model_str in self.model_cpu_offload_seq.split("->"): + model = all_model_components.pop(model_str, None) + if not isinstance(model, torch.nn.Module): + continue + + _, hook = cpu_offload_with_hook(model, device, prev_module_hook=hook) + self._all_hooks.append(hook) + + # CPU offload models that are not in the seq chain unless they are explicitly excluded + # these models will stay on CPU until maybe_free_model_hooks is called + # some models cannot be in the seq chain because they are iteratively called, such as controlnet + for name, model in all_model_components.items(): + if not isinstance(model, torch.nn.Module): + continue + + if name in self._exclude_from_cpu_offload: + model.to(device) + else: + _, hook = cpu_offload_with_hook(model, device) + self._all_hooks.append(hook) + + def maybe_free_model_hooks(self): + r""" + Function that offloads all components, removes all model hooks that were added when using + `enable_model_cpu_offload` and then applies them again. In case the model has not been offloaded this function + is a no-op. Make sure to add this function to the end of the `__call__` function of your pipeline so that it + functions correctly when applying enable_model_cpu_offload. + """ + if not hasattr(self, "_all_hooks") or len(self._all_hooks) == 0: + # `enable_model_cpu_offload` has not be called, so silently do nothing + return + + for hook in self._all_hooks: + # offload model and remove hook from model + hook.offload() + hook.remove() + + # make sure the model is in the same state as before calling it + self.enable_model_cpu_offload(device=getattr(self, "_offload_device", "cuda")) + + def enable_sequential_cpu_offload(self, gpu_id: Optional[int] = None, device: Union[torch.device, str] = "cuda"): + r""" + Offloads all models to CPU using 🤗 Accelerate, significantly reducing memory usage. When called, the state + dicts of all `torch.nn.Module` components (except those in `self._exclude_from_cpu_offload`) are saved to CPU + and then moved to `torch.device('meta')` and loaded to GPU only when their specific submodule has its `forward` + method called. Offloading happens on a submodule basis. Memory savings are higher than with + `enable_model_cpu_offload`, but performance is lower. + + Arguments: + gpu_id (`int`, *optional*): + The ID of the accelerator that shall be used in inference. If not specified, it will default to 0. + device (`torch.Device` or `str`, *optional*, defaults to "cuda"): + The PyTorch device type of the accelerator that shall be used in inference. If not specified, it will + default to "cuda". + """ + if is_accelerate_available() and is_accelerate_version(">=", "0.14.0"): + from accelerate import cpu_offload + else: + raise ImportError("`enable_sequential_cpu_offload` requires `accelerate v0.14.0` or higher") + + torch_device = torch.device(device) + device_index = torch_device.index + + if gpu_id is not None and device_index is not None: + raise ValueError( + f"You have passed both `gpu_id`={gpu_id} and an index as part of the passed device `device`={device}" + f"Cannot pass both. Please make sure to either not define `gpu_id` or not pass the index as part of the device: `device`={torch_device.type}" + ) + + # _offload_gpu_id should be set to passed gpu_id (or id in passed `device`) or default to previously set id or default to 0 + self._offload_gpu_id = gpu_id or torch_device.index or getattr(self, "_offload_gpu_id", 0) + + device_type = torch_device.type + device = torch.device(f"{device_type}:{self._offload_gpu_id}") + self._offload_device = device + + if self.device.type != "cpu": + self.to("cpu", silence_dtype_warnings=True) + device_mod = getattr(torch, self.device.type, None) + if hasattr(device_mod, "empty_cache") and device_mod.is_available(): + device_mod.empty_cache() # otherwise we don't see the memory savings (but they probably exist) + + for name, model in self.components.items(): + if not isinstance(model, torch.nn.Module): + continue + + if name in self._exclude_from_cpu_offload: + model.to(device) + else: + # make sure to offload buffers if not all high level weights + # are of type nn.Module + offload_buffers = len(model._parameters) > 0 + cpu_offload(model, device, offload_buffers=offload_buffers) + + @classmethod + @validate_hf_hub_args + def download(cls, pretrained_model_name, **kwargs) -> Union[str, os.PathLike]: + r""" + Download and cache a PyTorch diffusion pipeline from pretrained pipeline weights. + + Parameters: + pretrained_model_name (`str` or `os.PathLike`, *optional*): + A string, the *repository id* (for example `CompVis/ldm-text2im-large-256`) of a pretrained pipeline + hosted on the Hub. + custom_pipeline (`str`, *optional*): + Can be either: + + - A string, the *repository id* (for example `CompVis/ldm-text2im-large-256`) of a pretrained + pipeline hosted on the Hub. The repository must contain a file called `pipeline.py` that defines + the custom pipeline. + + - A string, the *file name* of a community pipeline hosted on GitHub under + [Community](https://github.com/huggingface/diffusers/tree/main/examples/community). Valid file + names must match the file name and not the pipeline script (`clip_guided_stable_diffusion` + instead of `clip_guided_stable_diffusion.py`). Community pipelines are always loaded from the + current `main` branch of GitHub. + + - A path to a *directory* (`./my_pipeline_directory/`) containing a custom pipeline. The directory + must contain a file called `pipeline.py` that defines the custom pipeline. + + + + 🧪 This is an experimental feature and may change in the future. + + + + For more information on how to load and create custom pipelines, take a look at [How to contribute a + community pipeline](https://huggingface.co/docs/diffusers/main/en/using-diffusers/contribute_pipeline). + + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + custom_revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id similar to + `revision` when loading a custom pipeline from the Hub. It can be a 🤗 Diffusers version when loading a + custom pipeline from GitHub, otherwise it defaults to `"main"` when loading from the Hub. + mirror (`str`, *optional*): + Mirror source to resolve accessibility issues if you're downloading a model in China. We do not + guarantee the timeliness or safety of the source, and you should refer to the mirror site for more + information. + variant (`str`, *optional*): + Load weights from a specified variant filename such as `"fp16"` or `"ema"`. This is ignored when + loading `from_flax`. + use_safetensors (`bool`, *optional*, defaults to `None`): + If set to `None`, the safetensors weights are downloaded if they're available **and** if the + safetensors library is installed. If set to `True`, the model is forcibly loaded from safetensors + weights. If set to `False`, safetensors weights are not loaded. + use_onnx (`bool`, *optional*, defaults to `False`): + If set to `True`, ONNX weights will always be downloaded if present. If set to `False`, ONNX weights + will never be downloaded. By default `use_onnx` defaults to the `_is_onnx` class attribute which is + `False` for non-ONNX pipelines and `True` for ONNX pipelines. ONNX weights include both files ending + with `.onnx` and `.pb`. + trust_remote_code (`bool`, *optional*, defaults to `False`): + Whether or not to allow for custom pipelines and components defined on the Hub in their own files. This + option should only be set to `True` for repositories you trust and in which you have read the code, as + it will execute code present on the Hub on your local machine. + + Returns: + `os.PathLike`: + A path to the downloaded pipeline. + + + + To use private or [gated models](https://huggingface.co/docs/hub/models-gated#gated-models), log-in with + `huggingface-cli login`. + + + + """ + cache_dir = kwargs.pop("cache_dir", None) + resume_download = kwargs.pop("resume_download", False) + force_download = kwargs.pop("force_download", False) + proxies = kwargs.pop("proxies", None) + local_files_only = kwargs.pop("local_files_only", None) + token = kwargs.pop("token", None) + revision = kwargs.pop("revision", None) + from_flax = kwargs.pop("from_flax", False) + custom_pipeline = kwargs.pop("custom_pipeline", None) + custom_revision = kwargs.pop("custom_revision", None) + variant = kwargs.pop("variant", None) + use_safetensors = kwargs.pop("use_safetensors", None) + use_onnx = kwargs.pop("use_onnx", None) + load_connected_pipeline = kwargs.pop("load_connected_pipeline", False) + trust_remote_code = kwargs.pop("trust_remote_code", False) + + allow_pickle = False + if use_safetensors is None: + use_safetensors = True + allow_pickle = True + + allow_patterns = None + ignore_patterns = None + + model_info_call_error: Optional[Exception] = None + if not local_files_only: + try: + info = model_info(pretrained_model_name, token=token, revision=revision) + except (HTTPError, OfflineModeIsEnabled, requests.ConnectionError) as e: + logger.warning(f"Couldn't connect to the Hub: {e}.\nWill try to load from local cache.") + local_files_only = True + model_info_call_error = e # save error to reraise it if model is not cached locally + + if not local_files_only: + config_file = hf_hub_download( + pretrained_model_name, + cls.config_name, + cache_dir=cache_dir, + revision=revision, + proxies=proxies, + force_download=force_download, + resume_download=resume_download, + token=token, + ) + + config_dict = cls._dict_from_json_file(config_file) + ignore_filenames = config_dict.pop("_ignore_files", []) + + # retrieve all folder_names that contain relevant files + folder_names = [k for k, v in config_dict.items() if isinstance(v, list) and k != "_class_name"] + + filenames = {sibling.rfilename for sibling in info.siblings} + model_filenames, variant_filenames = variant_compatible_siblings(filenames, variant=variant) + + diffusers_module = importlib.import_module(__name__.split(".")[0]) + pipelines = getattr(diffusers_module, "pipelines") + + # optionally create a custom component <> custom file mapping + custom_components = {} + for component in folder_names: + module_candidate = config_dict[component][0] + + if module_candidate is None or not isinstance(module_candidate, str): + continue + + # We compute candidate file path on the Hub. Do not use `os.path.join`. + candidate_file = f"{component}/{module_candidate}.py" + + if candidate_file in filenames: + custom_components[component] = module_candidate + elif module_candidate not in LOADABLE_CLASSES and not hasattr(pipelines, module_candidate): + raise ValueError( + f"{candidate_file} as defined in `model_index.json` does not exist in {pretrained_model_name} and is not a module in 'diffusers/pipelines'." + ) + + if len(variant_filenames) == 0 and variant is not None: + deprecation_message = ( + f"You are trying to load the model files of the `variant={variant}`, but no such modeling files are available." + f"The default model files: {model_filenames} will be loaded instead. Make sure to not load from `variant={variant}`" + "if such variant modeling files are not available. Doing so will lead to an error in v0.24.0 as defaulting to non-variant" + "modeling files is deprecated." + ) + deprecate("no variant default", "0.24.0", deprecation_message, standard_warn=False) + + # remove ignored filenames + model_filenames = set(model_filenames) - set(ignore_filenames) + variant_filenames = set(variant_filenames) - set(ignore_filenames) + + # if the whole pipeline is cached we don't have to ping the Hub + if revision in DEPRECATED_REVISION_ARGS and version.parse( + version.parse(__version__).base_version + ) >= version.parse("0.22.0"): + warn_deprecated_model_variant(pretrained_model_name, token, variant, revision, model_filenames) + + model_folder_names = {os.path.split(f)[0] for f in model_filenames if os.path.split(f)[0] in folder_names} + + custom_class_name = None + if custom_pipeline is None and isinstance(config_dict["_class_name"], (list, tuple)): + custom_pipeline = config_dict["_class_name"][0] + custom_class_name = config_dict["_class_name"][1] + + # all filenames compatible with variant will be added + allow_patterns = list(model_filenames) + + # allow all patterns from non-model folders + # this enables downloading schedulers, tokenizers, ... + allow_patterns += [f"{k}/*" for k in folder_names if k not in model_folder_names] + # add custom component files + allow_patterns += [f"{k}/{f}.py" for k, f in custom_components.items()] + # add custom pipeline file + allow_patterns += [f"{custom_pipeline}.py"] if f"{custom_pipeline}.py" in filenames else [] + # also allow downloading config.json files with the model + allow_patterns += [os.path.join(k, "config.json") for k in model_folder_names] + + allow_patterns += [ + SCHEDULER_CONFIG_NAME, + CONFIG_NAME, + cls.config_name, + CUSTOM_PIPELINE_FILE_NAME, + ] + + load_pipe_from_hub = custom_pipeline is not None and f"{custom_pipeline}.py" in filenames + load_components_from_hub = len(custom_components) > 0 + + if load_pipe_from_hub and not trust_remote_code: + raise ValueError( + f"The repository for {pretrained_model_name} contains custom code in {custom_pipeline}.py which must be executed to correctly " + f"load the model. You can inspect the repository content at https://hf.co/{pretrained_model_name}/blob/main/{custom_pipeline}.py.\n" + f"Please pass the argument `trust_remote_code=True` to allow custom code to be run." + ) + + if load_components_from_hub and not trust_remote_code: + raise ValueError( + f"The repository for {pretrained_model_name} contains custom code in {'.py, '.join([os.path.join(k, v) for k,v in custom_components.items()])} which must be executed to correctly " + f"load the model. You can inspect the repository content at {', '.join([f'https://hf.co/{pretrained_model_name}/{k}/{v}.py' for k,v in custom_components.items()])}.\n" + f"Please pass the argument `trust_remote_code=True` to allow custom code to be run." + ) + + # retrieve passed components that should not be downloaded + pipeline_class = _get_pipeline_class( + cls, + config_dict, + load_connected_pipeline=load_connected_pipeline, + custom_pipeline=custom_pipeline, + repo_id=pretrained_model_name if load_pipe_from_hub else None, + hub_revision=revision, + class_name=custom_class_name, + cache_dir=cache_dir, + revision=custom_revision, + ) + expected_components, _ = cls._get_signature_keys(pipeline_class) + passed_components = [k for k in expected_components if k in kwargs] + + if ( + use_safetensors + and not allow_pickle + and not is_safetensors_compatible( + model_filenames, variant=variant, passed_components=passed_components + ) + ): + raise EnvironmentError( + f"Could not find the necessary `safetensors` weights in {model_filenames} (variant={variant})" + ) + if from_flax: + ignore_patterns = ["*.bin", "*.safetensors", "*.onnx", "*.pb"] + elif use_safetensors and is_safetensors_compatible( + model_filenames, variant=variant, passed_components=passed_components + ): + ignore_patterns = ["*.bin", "*.msgpack"] + + use_onnx = use_onnx if use_onnx is not None else pipeline_class._is_onnx + if not use_onnx: + ignore_patterns += ["*.onnx", "*.pb"] + + safetensors_variant_filenames = {f for f in variant_filenames if f.endswith(".safetensors")} + safetensors_model_filenames = {f for f in model_filenames if f.endswith(".safetensors")} + if ( + len(safetensors_variant_filenames) > 0 + and safetensors_model_filenames != safetensors_variant_filenames + ): + logger.warning( + f"\nA mixture of {variant} and non-{variant} filenames will be loaded.\nLoaded {variant} filenames:\n[{', '.join(safetensors_variant_filenames)}]\nLoaded non-{variant} filenames:\n[{', '.join(safetensors_model_filenames - safetensors_variant_filenames)}\nIf this behavior is not expected, please check your folder structure." + ) + else: + ignore_patterns = ["*.safetensors", "*.msgpack"] + + use_onnx = use_onnx if use_onnx is not None else pipeline_class._is_onnx + if not use_onnx: + ignore_patterns += ["*.onnx", "*.pb"] + + bin_variant_filenames = {f for f in variant_filenames if f.endswith(".bin")} + bin_model_filenames = {f for f in model_filenames if f.endswith(".bin")} + if len(bin_variant_filenames) > 0 and bin_model_filenames != bin_variant_filenames: + logger.warning( + f"\nA mixture of {variant} and non-{variant} filenames will be loaded.\nLoaded {variant} filenames:\n[{', '.join(bin_variant_filenames)}]\nLoaded non-{variant} filenames:\n[{', '.join(bin_model_filenames - bin_variant_filenames)}\nIf this behavior is not expected, please check your folder structure." + ) + + # Don't download any objects that are passed + allow_patterns = [ + p for p in allow_patterns if not (len(p.split("/")) == 2 and p.split("/")[0] in passed_components) + ] + + if pipeline_class._load_connected_pipes: + allow_patterns.append("README.md") + + # Don't download index files of forbidden patterns either + ignore_patterns = ignore_patterns + [f"{i}.index.*json" for i in ignore_patterns] + + re_ignore_pattern = [re.compile(fnmatch.translate(p)) for p in ignore_patterns] + re_allow_pattern = [re.compile(fnmatch.translate(p)) for p in allow_patterns] + + expected_files = [f for f in filenames if not any(p.match(f) for p in re_ignore_pattern)] + expected_files = [f for f in expected_files if any(p.match(f) for p in re_allow_pattern)] + + snapshot_folder = Path(config_file).parent + pipeline_is_cached = all((snapshot_folder / f).is_file() for f in expected_files) + + if pipeline_is_cached and not force_download: + # if the pipeline is cached, we can directly return it + # else call snapshot_download + return snapshot_folder + + user_agent = {"pipeline_class": cls.__name__} + if custom_pipeline is not None and not custom_pipeline.endswith(".py"): + user_agent["custom_pipeline"] = custom_pipeline + + # download all allow_patterns - ignore_patterns + try: + cached_folder = snapshot_download( + pretrained_model_name, + cache_dir=cache_dir, + resume_download=resume_download, + proxies=proxies, + local_files_only=local_files_only, + token=token, + revision=revision, + allow_patterns=allow_patterns, + ignore_patterns=ignore_patterns, + user_agent=user_agent, + ) + + # retrieve pipeline class from local file + cls_name = cls.load_config(os.path.join(cached_folder, "model_index.json")).get("_class_name", None) + cls_name = cls_name[4:] if isinstance(cls_name, str) and cls_name.startswith("Flax") else cls_name + + diffusers_module = importlib.import_module(__name__.split(".")[0]) + pipeline_class = getattr(diffusers_module, cls_name, None) if isinstance(cls_name, str) else None + + if pipeline_class is not None and pipeline_class._load_connected_pipes: + modelcard = ModelCard.load(os.path.join(cached_folder, "README.md")) + connected_pipes = sum([getattr(modelcard.data, k, []) for k in CONNECTED_PIPES_KEYS], []) + for connected_pipe_repo_id in connected_pipes: + download_kwargs = { + "cache_dir": cache_dir, + "resume_download": resume_download, + "force_download": force_download, + "proxies": proxies, + "local_files_only": local_files_only, + "token": token, + "variant": variant, + "use_safetensors": use_safetensors, + } + DiffusionPipeline.download(connected_pipe_repo_id, **download_kwargs) + + return cached_folder + + except FileNotFoundError: + # Means we tried to load pipeline with `local_files_only=True` but the files have not been found in local cache. + # This can happen in two cases: + # 1. If the user passed `local_files_only=True` => we raise the error directly + # 2. If we forced `local_files_only=True` when `model_info` failed => we raise the initial error + if model_info_call_error is None: + # 1. user passed `local_files_only=True` + raise + else: + # 2. we forced `local_files_only=True` when `model_info` failed + raise EnvironmentError( + f"Cannot load model {pretrained_model_name}: model is not cached locally and an error occurred" + " while trying to fetch metadata from the Hub. Please check out the root cause in the stacktrace" + " above." + ) from model_info_call_error + + @classmethod + def _get_signature_keys(cls, obj): + parameters = inspect.signature(obj.__init__).parameters + required_parameters = {k: v for k, v in parameters.items() if v.default == inspect._empty} + optional_parameters = set({k for k, v in parameters.items() if v.default != inspect._empty}) + expected_modules = set(required_parameters.keys()) - {"self"} + + optional_names = list(optional_parameters) + for name in optional_names: + if name in cls._optional_components: + expected_modules.add(name) + optional_parameters.remove(name) + + return expected_modules, optional_parameters + + @property + def components(self) -> Dict[str, Any]: + r""" + The `self.components` property can be useful to run different pipelines with the same weights and + configurations without reallocating additional memory. + + Returns (`dict`): + A dictionary containing all the modules needed to initialize the pipeline. + + Examples: + + ```py + >>> from diffusers import ( + ... StableDiffusionPipeline, + ... StableDiffusionImg2ImgPipeline, + ... StableDiffusionInpaintPipeline, + ... ) + + >>> text2img = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + >>> img2img = StableDiffusionImg2ImgPipeline(**text2img.components) + >>> inpaint = StableDiffusionInpaintPipeline(**text2img.components) + ``` + """ + expected_modules, optional_parameters = self._get_signature_keys(self) + components = { + k: getattr(self, k) for k in self.config.keys() if not k.startswith("_") and k not in optional_parameters + } + + if set(components.keys()) != expected_modules: + raise ValueError( + f"{self} has been incorrectly initialized or {self.__class__} is incorrectly implemented. Expected" + f" {expected_modules} to be defined, but {components.keys()} are defined." + ) + + return components + + @staticmethod + def numpy_to_pil(images): + """ + Convert a NumPy image or a batch of images to a PIL image. + """ + return numpy_to_pil(images) + + def progress_bar(self, iterable=None, total=None): + if not hasattr(self, "_progress_bar_config"): + self._progress_bar_config = {} + elif not isinstance(self._progress_bar_config, dict): + raise ValueError( + f"`self._progress_bar_config` should be of type `dict`, but is {type(self._progress_bar_config)}." + ) + + if iterable is not None: + return tqdm(iterable, **self._progress_bar_config) + elif total is not None: + return tqdm(total=total, **self._progress_bar_config) + else: + raise ValueError("Either `total` or `iterable` has to be defined.") + + def set_progress_bar_config(self, **kwargs): + self._progress_bar_config = kwargs + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + r""" + Enable memory efficient attention from [xFormers](https://facebookresearch.github.io/xformers/). When this + option is enabled, you should observe lower GPU memory usage and a potential speed up during inference. Speed + up during training is not guaranteed. + + + + ⚠️ When memory efficient attention and sliced attention are both enabled, memory efficient attention takes + precedent. + + + + Parameters: + attention_op (`Callable`, *optional*): + Override the default `None` operator for use as `op` argument to the + [`memory_efficient_attention()`](https://facebookresearch.github.io/xformers/components/ops.html#xformers.ops.memory_efficient_attention) + function of xFormers. + + Examples: + + ```py + >>> import torch + >>> from diffusers import DiffusionPipeline + >>> from xformers.ops import MemoryEfficientAttentionFlashAttentionOp + + >>> pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1", torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + >>> pipe.enable_xformers_memory_efficient_attention(attention_op=MemoryEfficientAttentionFlashAttentionOp) + >>> # Workaround for not accepting attention shape using VAE for Flash Attention + >>> pipe.vae.enable_xformers_memory_efficient_attention(attention_op=None) + ``` + """ + self.set_use_memory_efficient_attention_xformers(True, attention_op) + + def disable_xformers_memory_efficient_attention(self): + r""" + Disable memory efficient attention from [xFormers](https://facebookresearch.github.io/xformers/). + """ + self.set_use_memory_efficient_attention_xformers(False) + + def set_use_memory_efficient_attention_xformers( + self, valid: bool, attention_op: Optional[Callable] = None + ) -> None: + # Recursively walk through all the children. + # Any children which exposes the set_use_memory_efficient_attention_xformers method + # gets the message + def fn_recursive_set_mem_eff(module: torch.nn.Module): + if hasattr(module, "set_use_memory_efficient_attention_xformers"): + module.set_use_memory_efficient_attention_xformers(valid, attention_op) + + for child in module.children(): + fn_recursive_set_mem_eff(child) + + module_names, _ = self._get_signature_keys(self) + modules = [getattr(self, n, None) for n in module_names] + modules = [m for m in modules if isinstance(m, torch.nn.Module)] + + for module in modules: + fn_recursive_set_mem_eff(module) + + def enable_attention_slicing(self, slice_size: Optional[Union[str, int]] = "auto"): + r""" + Enable sliced attention computation. When this option is enabled, the attention module splits the input tensor + in slices to compute attention in several steps. For more than one attention head, the computation is performed + sequentially over each head. This is useful to save some memory in exchange for a small speed decrease. + + + + ⚠️ Don't enable attention slicing if you're already using `scaled_dot_product_attention` (SDPA) from PyTorch + 2.0 or xFormers. These attention computations are already very memory efficient so you won't need to enable + this function. If you enable attention slicing with SDPA or xFormers, it can lead to serious slow downs! + + + + Args: + slice_size (`str` or `int`, *optional*, defaults to `"auto"`): + When `"auto"`, halves the input to the attention heads, so attention will be computed in two steps. If + `"max"`, maximum amount of memory will be saved by running only one slice at a time. If a number is + provided, uses as many slices as `attention_head_dim // slice_size`. In this case, `attention_head_dim` + must be a multiple of `slice_size`. + + Examples: + + ```py + >>> import torch + >>> from diffusers import StableDiffusionPipeline + + >>> pipe = StableDiffusionPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", + ... torch_dtype=torch.float16, + ... use_safetensors=True, + ... ) + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> pipe.enable_attention_slicing() + >>> image = pipe(prompt).images[0] + ``` + """ + self.set_attention_slice(slice_size) + + def disable_attention_slicing(self): + r""" + Disable sliced attention computation. If `enable_attention_slicing` was previously called, attention is + computed in one step. + """ + # set slice_size = `None` to disable `attention slicing` + self.enable_attention_slicing(None) + + def set_attention_slice(self, slice_size: Optional[int]): + module_names, _ = self._get_signature_keys(self) + modules = [getattr(self, n, None) for n in module_names] + modules = [m for m in modules if isinstance(m, torch.nn.Module) and hasattr(m, "set_attention_slice")] + + for module in modules: + module.set_attention_slice(slice_size) + + +class StableDiffusionMixin: + r""" + Helper for DiffusionPipeline with vae and unet.(mainly for LDM such as stable diffusion) + """ + + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.vae.enable_tiling() + + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_tiling() + + def enable_freeu(self, s1: float, s2: float, b1: float, b2: float): + r"""Enables the FreeU mechanism as in https://arxiv.org/abs/2309.11497. + + The suffixes after the scaling factors represent the stages where they are being applied. + + Please refer to the [official repository](https://github.com/ChenyangSi/FreeU) for combinations of the values + that are known to work well for different pipelines such as Stable Diffusion v1, v2, and Stable Diffusion XL. + + Args: + s1 (`float`): + Scaling factor for stage 1 to attenuate the contributions of the skip features. This is done to + mitigate "oversmoothing effect" in the enhanced denoising process. + s2 (`float`): + Scaling factor for stage 2 to attenuate the contributions of the skip features. This is done to + mitigate "oversmoothing effect" in the enhanced denoising process. + b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. + b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. + """ + if not hasattr(self, "unet"): + raise ValueError("The pipeline must have `unet` for using FreeU.") + self.unet.enable_freeu(s1=s1, s2=s2, b1=b1, b2=b2) + + def disable_freeu(self): + """Disables the FreeU mechanism if enabled.""" + self.unet.disable_freeu() + + def fuse_qkv_projections(self, unet: bool = True, vae: bool = True): + """ + Enables fused QKV projections. For self-attention modules, all projection matrices (i.e., query, + key, value) are fused. For cross-attention modules, key and value projection matrices are fused. + + + + This API is 🧪 experimental. + + + + Args: + unet (`bool`, defaults to `True`): To apply fusion on the UNet. + vae (`bool`, defaults to `True`): To apply fusion on the VAE. + """ + self.fusing_unet = False + self.fusing_vae = False + + if unet: + self.fusing_unet = True + self.unet.fuse_qkv_projections() + self.unet.set_attn_processor(FusedAttnProcessor2_0()) + + if vae: + if not isinstance(self.vae, AutoencoderKL): + raise ValueError("`fuse_qkv_projections()` is only supported for the VAE of type `AutoencoderKL`.") + + self.fusing_vae = True + self.vae.fuse_qkv_projections() + self.vae.set_attn_processor(FusedAttnProcessor2_0()) + + def unfuse_qkv_projections(self, unet: bool = True, vae: bool = True): + """Disable QKV projection fusion if enabled. + + + + This API is 🧪 experimental. + + + + Args: + unet (`bool`, defaults to `True`): To apply fusion on the UNet. + vae (`bool`, defaults to `True`): To apply fusion on the VAE. + + """ + if unet: + if not self.fusing_unet: + logger.warning("The UNet was not initially fused for QKV projections. Doing nothing.") + else: + self.unet.unfuse_qkv_projections() + self.fusing_unet = False + + if vae: + if not self.fusing_vae: + logger.warning("The VAE was not initially fused for QKV projections. Doing nothing.") + else: + self.vae.unfuse_qkv_projections() + self.fusing_vae = False diff --git a/diffusers-0.27.0/src/diffusers/pipelines/pixart_alpha/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/pixart_alpha/__init__.py new file mode 100755 index 0000000..0bfa28f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/pixart_alpha/__init__.py @@ -0,0 +1,48 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_pixart_alpha"] = ["PixArtAlphaPipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_pixart_alpha import PixArtAlphaPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py b/diffusers-0.27.0/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py new file mode 100755 index 0000000..e7213a3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/pixart_alpha/pipeline_pixart_alpha.py @@ -0,0 +1,979 @@ +# Copyright 2024 PixArt-Alpha Authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import html +import inspect +import re +import urllib.parse as ul +from typing import Callable, List, Optional, Tuple, Union + +import torch +import torch.nn.functional as F +from transformers import T5EncoderModel, T5Tokenizer + +from ...image_processor import VaeImageProcessor +from ...models import AutoencoderKL, Transformer2DModel +from ...schedulers import DPMSolverMultistepScheduler +from ...utils import ( + BACKENDS_MAPPING, + deprecate, + is_bs4_available, + is_ftfy_available, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +if is_bs4_available(): + from bs4 import BeautifulSoup + +if is_ftfy_available(): + import ftfy + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import PixArtAlphaPipeline + + >>> # You can replace the checkpoint id with "PixArt-alpha/PixArt-XL-2-512x512" too. + >>> pipe = PixArtAlphaPipeline.from_pretrained("PixArt-alpha/PixArt-XL-2-1024-MS", torch_dtype=torch.float16) + >>> # Enable memory optimizations. + >>> pipe.enable_model_cpu_offload() + + >>> prompt = "A small cactus with a happy face in the Sahara desert." + >>> image = pipe(prompt).images[0] + ``` +""" + +ASPECT_RATIO_1024_BIN = { + "0.25": [512.0, 2048.0], + "0.28": [512.0, 1856.0], + "0.32": [576.0, 1792.0], + "0.33": [576.0, 1728.0], + "0.35": [576.0, 1664.0], + "0.4": [640.0, 1600.0], + "0.42": [640.0, 1536.0], + "0.48": [704.0, 1472.0], + "0.5": [704.0, 1408.0], + "0.52": [704.0, 1344.0], + "0.57": [768.0, 1344.0], + "0.6": [768.0, 1280.0], + "0.68": [832.0, 1216.0], + "0.72": [832.0, 1152.0], + "0.78": [896.0, 1152.0], + "0.82": [896.0, 1088.0], + "0.88": [960.0, 1088.0], + "0.94": [960.0, 1024.0], + "1.0": [1024.0, 1024.0], + "1.07": [1024.0, 960.0], + "1.13": [1088.0, 960.0], + "1.21": [1088.0, 896.0], + "1.29": [1152.0, 896.0], + "1.38": [1152.0, 832.0], + "1.46": [1216.0, 832.0], + "1.67": [1280.0, 768.0], + "1.75": [1344.0, 768.0], + "2.0": [1408.0, 704.0], + "2.09": [1472.0, 704.0], + "2.4": [1536.0, 640.0], + "2.5": [1600.0, 640.0], + "3.0": [1728.0, 576.0], + "4.0": [2048.0, 512.0], +} + +ASPECT_RATIO_512_BIN = { + "0.25": [256.0, 1024.0], + "0.28": [256.0, 928.0], + "0.32": [288.0, 896.0], + "0.33": [288.0, 864.0], + "0.35": [288.0, 832.0], + "0.4": [320.0, 800.0], + "0.42": [320.0, 768.0], + "0.48": [352.0, 736.0], + "0.5": [352.0, 704.0], + "0.52": [352.0, 672.0], + "0.57": [384.0, 672.0], + "0.6": [384.0, 640.0], + "0.68": [416.0, 608.0], + "0.72": [416.0, 576.0], + "0.78": [448.0, 576.0], + "0.82": [448.0, 544.0], + "0.88": [480.0, 544.0], + "0.94": [480.0, 512.0], + "1.0": [512.0, 512.0], + "1.07": [512.0, 480.0], + "1.13": [544.0, 480.0], + "1.21": [544.0, 448.0], + "1.29": [576.0, 448.0], + "1.38": [576.0, 416.0], + "1.46": [608.0, 416.0], + "1.67": [640.0, 384.0], + "1.75": [672.0, 384.0], + "2.0": [704.0, 352.0], + "2.09": [736.0, 352.0], + "2.4": [768.0, 320.0], + "2.5": [800.0, 320.0], + "3.0": [864.0, 288.0], + "4.0": [1024.0, 256.0], +} + +ASPECT_RATIO_256_BIN = { + "0.25": [128.0, 512.0], + "0.28": [128.0, 464.0], + "0.32": [144.0, 448.0], + "0.33": [144.0, 432.0], + "0.35": [144.0, 416.0], + "0.4": [160.0, 400.0], + "0.42": [160.0, 384.0], + "0.48": [176.0, 368.0], + "0.5": [176.0, 352.0], + "0.52": [176.0, 336.0], + "0.57": [192.0, 336.0], + "0.6": [192.0, 320.0], + "0.68": [208.0, 304.0], + "0.72": [208.0, 288.0], + "0.78": [224.0, 288.0], + "0.82": [224.0, 272.0], + "0.88": [240.0, 272.0], + "0.94": [240.0, 256.0], + "1.0": [256.0, 256.0], + "1.07": [256.0, 240.0], + "1.13": [272.0, 240.0], + "1.21": [272.0, 224.0], + "1.29": [288.0, 224.0], + "1.38": [288.0, 208.0], + "1.46": [304.0, 208.0], + "1.67": [320.0, 192.0], + "1.75": [336.0, 192.0], + "2.0": [352.0, 176.0], + "2.09": [368.0, 176.0], + "2.4": [384.0, 160.0], + "2.5": [400.0, 160.0], + "3.0": [432.0, 144.0], + "4.0": [512.0, 128.0], +} + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class PixArtAlphaPipeline(DiffusionPipeline): + r""" + Pipeline for text-to-image generation using PixArt-Alpha. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`T5EncoderModel`]): + Frozen text-encoder. PixArt-Alpha uses + [T5](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5EncoderModel), specifically the + [t5-v1_1-xxl](https://huggingface.co/PixArt-alpha/PixArt-alpha/tree/main/t5-v1_1-xxl) variant. + tokenizer (`T5Tokenizer`): + Tokenizer of class + [T5Tokenizer](https://huggingface.co/docs/transformers/model_doc/t5#transformers.T5Tokenizer). + transformer ([`Transformer2DModel`]): + A text conditioned `Transformer2DModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `transformer` to denoise the encoded image latents. + """ + + bad_punct_regex = re.compile( + r"[" + + "#®•©™&@·º½¾¿¡§~" + + r"\)" + + r"\(" + + r"\]" + + r"\[" + + r"\}" + + r"\{" + + r"\|" + + "\\" + + r"\/" + + r"\*" + + r"]{1,}" + ) # noqa + + _optional_components = ["tokenizer", "text_encoder"] + model_cpu_offload_seq = "text_encoder->transformer->vae" + + def __init__( + self, + tokenizer: T5Tokenizer, + text_encoder: T5EncoderModel, + vae: AutoencoderKL, + transformer: Transformer2DModel, + scheduler: DPMSolverMultistepScheduler, + ): + super().__init__() + + self.register_modules( + tokenizer=tokenizer, text_encoder=text_encoder, vae=vae, transformer=transformer, scheduler=scheduler + ) + + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + # Adapted from https://github.com/PixArt-alpha/PixArt-alpha/blob/master/diffusion/model/utils.py + def mask_text_embeddings(self, emb, mask): + if emb.shape[0] == 1: + keep_index = mask.sum().item() + return emb[:, :, :keep_index, :], keep_index + else: + masked_feature = emb * mask[:, None, :, None] + return masked_feature, emb.shape[2] + + # Adapted from diffusers.pipelines.deepfloyd_if.pipeline_if.encode_prompt + def encode_prompt( + self, + prompt: Union[str, List[str]], + do_classifier_free_guidance: bool = True, + negative_prompt: str = "", + num_images_per_prompt: int = 1, + device: Optional[torch.device] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + prompt_attention_mask: Optional[torch.FloatTensor] = None, + negative_prompt_attention_mask: Optional[torch.FloatTensor] = None, + clean_caption: bool = False, + max_sequence_length: int = 120, + **kwargs, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + negative_prompt (`str` or `List[str]`, *optional*): + The prompt not to guide the image generation. If not defined, one has to pass `negative_prompt_embeds` + instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). For + PixArt-Alpha, this should be "". + do_classifier_free_guidance (`bool`, *optional*, defaults to `True`): + whether to use classifier free guidance or not + num_images_per_prompt (`int`, *optional*, defaults to 1): + number of images that should be generated per prompt + device: (`torch.device`, *optional*): + torch device to place the resulting embeddings on + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. For PixArt-Alpha, it's should be the embeddings of the "" + string. + clean_caption (`bool`, defaults to `False`): + If `True`, the function will preprocess and clean the provided caption before encoding. + max_sequence_length (`int`, defaults to 120): Maximum sequence length to use for the prompt. + """ + + if "mask_feature" in kwargs: + deprecation_message = "The use of `mask_feature` is deprecated. It is no longer used in any computation and that doesn't affect the end results. It will be removed in a future version." + deprecate("mask_feature", "1.0.0", deprecation_message, standard_warn=False) + + if device is None: + device = self._execution_device + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # See Section 3.1. of the paper. + max_length = max_sequence_length + + if prompt_embeds is None: + prompt = self._text_preprocessing(prompt, clean_caption=clean_caption) + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=max_length, + truncation=True, + add_special_tokens=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {max_length} tokens: {removed_text}" + ) + + prompt_attention_mask = text_inputs.attention_mask + prompt_attention_mask = prompt_attention_mask.to(device) + + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=prompt_attention_mask) + prompt_embeds = prompt_embeds[0] + + if self.text_encoder is not None: + dtype = self.text_encoder.dtype + elif self.transformer is not None: + dtype = self.transformer.dtype + else: + dtype = None + + prompt_embeds = prompt_embeds.to(dtype=dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings and attention mask for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + prompt_attention_mask = prompt_attention_mask.view(bs_embed, -1) + prompt_attention_mask = prompt_attention_mask.repeat(num_images_per_prompt, 1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens = [negative_prompt] * batch_size + uncond_tokens = self._text_preprocessing(uncond_tokens, clean_caption=clean_caption) + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_attention_mask=True, + add_special_tokens=True, + return_tensors="pt", + ) + negative_prompt_attention_mask = uncond_input.attention_mask + negative_prompt_attention_mask = negative_prompt_attention_mask.to(device) + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), attention_mask=negative_prompt_attention_mask + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + negative_prompt_attention_mask = negative_prompt_attention_mask.view(bs_embed, -1) + negative_prompt_attention_mask = negative_prompt_attention_mask.repeat(num_images_per_prompt, 1) + else: + negative_prompt_embeds = None + negative_prompt_attention_mask = None + + return prompt_embeds, prompt_attention_mask, negative_prompt_embeds, negative_prompt_attention_mask + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + negative_prompt, + callback_steps, + prompt_embeds=None, + negative_prompt_embeds=None, + prompt_attention_mask=None, + negative_prompt_attention_mask=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and prompt_attention_mask is None: + raise ValueError("Must provide `prompt_attention_mask` when specifying `prompt_embeds`.") + + if negative_prompt_embeds is not None and negative_prompt_attention_mask is None: + raise ValueError("Must provide `negative_prompt_attention_mask` when specifying `negative_prompt_embeds`.") + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + if prompt_attention_mask.shape != negative_prompt_attention_mask.shape: + raise ValueError( + "`prompt_attention_mask` and `negative_prompt_attention_mask` must have the same shape when passed directly, but" + f" got: `prompt_attention_mask` {prompt_attention_mask.shape} != `negative_prompt_attention_mask`" + f" {negative_prompt_attention_mask.shape}." + ) + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._text_preprocessing + def _text_preprocessing(self, text, clean_caption=False): + if clean_caption and not is_bs4_available(): + logger.warning(BACKENDS_MAPPING["bs4"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if clean_caption and not is_ftfy_available(): + logger.warning(BACKENDS_MAPPING["ftfy"][-1].format("Setting `clean_caption=True`")) + logger.warning("Setting `clean_caption` to False...") + clean_caption = False + + if not isinstance(text, (tuple, list)): + text = [text] + + def process(text: str): + if clean_caption: + text = self._clean_caption(text) + text = self._clean_caption(text) + else: + text = text.lower().strip() + return text + + return [process(t) for t in text] + + # Copied from diffusers.pipelines.deepfloyd_if.pipeline_if.IFPipeline._clean_caption + def _clean_caption(self, caption): + caption = str(caption) + caption = ul.unquote_plus(caption) + caption = caption.strip().lower() + caption = re.sub("", "person", caption) + # urls: + caption = re.sub( + r"\b((?:https?:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + caption = re.sub( + r"\b((?:www:(?:\/{1,3}|[a-zA-Z0-9%])|[a-zA-Z0-9.\-]+[.](?:com|co|ru|net|org|edu|gov|it)[\w/-]*\b\/?(?!@)))", # noqa + "", + caption, + ) # regex for urls + # html: + caption = BeautifulSoup(caption, features="html.parser").text + + # @ + caption = re.sub(r"@[\w\d]+\b", "", caption) + + # 31C0—31EF CJK Strokes + # 31F0—31FF Katakana Phonetic Extensions + # 3200—32FF Enclosed CJK Letters and Months + # 3300—33FF CJK Compatibility + # 3400—4DBF CJK Unified Ideographs Extension A + # 4DC0—4DFF Yijing Hexagram Symbols + # 4E00—9FFF CJK Unified Ideographs + caption = re.sub(r"[\u31c0-\u31ef]+", "", caption) + caption = re.sub(r"[\u31f0-\u31ff]+", "", caption) + caption = re.sub(r"[\u3200-\u32ff]+", "", caption) + caption = re.sub(r"[\u3300-\u33ff]+", "", caption) + caption = re.sub(r"[\u3400-\u4dbf]+", "", caption) + caption = re.sub(r"[\u4dc0-\u4dff]+", "", caption) + caption = re.sub(r"[\u4e00-\u9fff]+", "", caption) + ####################################################### + + # все виды тире / all types of dash --> "-" + caption = re.sub( + r"[\u002D\u058A\u05BE\u1400\u1806\u2010-\u2015\u2E17\u2E1A\u2E3A\u2E3B\u2E40\u301C\u3030\u30A0\uFE31\uFE32\uFE58\uFE63\uFF0D]+", # noqa + "-", + caption, + ) + + # кавычки к одному стандарту + caption = re.sub(r"[`´«»“”¨]", '"', caption) + caption = re.sub(r"[‘’]", "'", caption) + + # " + caption = re.sub(r""?", "", caption) + # & + caption = re.sub(r"&", "", caption) + + # ip adresses: + caption = re.sub(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}", " ", caption) + + # article ids: + caption = re.sub(r"\d:\d\d\s+$", "", caption) + + # \n + caption = re.sub(r"\\n", " ", caption) + + # "#123" + caption = re.sub(r"#\d{1,3}\b", "", caption) + # "#12345.." + caption = re.sub(r"#\d{5,}\b", "", caption) + # "123456.." + caption = re.sub(r"\b\d{6,}\b", "", caption) + # filenames: + caption = re.sub(r"[\S]+\.(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)", "", caption) + + # + caption = re.sub(r"[\"\']{2,}", r'"', caption) # """AUSVERKAUFT""" + caption = re.sub(r"[\.]{2,}", r" ", caption) # """AUSVERKAUFT""" + + caption = re.sub(self.bad_punct_regex, r" ", caption) # ***AUSVERKAUFT***, #AUSVERKAUFT + caption = re.sub(r"\s+\.\s+", r" ", caption) # " . " + + # this-is-my-cute-cat / this_is_my_cute_cat + regex2 = re.compile(r"(?:\-|\_)") + if len(re.findall(regex2, caption)) > 3: + caption = re.sub(regex2, " ", caption) + + caption = ftfy.fix_text(caption) + caption = html.unescape(html.unescape(caption)) + + caption = re.sub(r"\b[a-zA-Z]{1,3}\d{3,15}\b", "", caption) # jc6640 + caption = re.sub(r"\b[a-zA-Z]+\d+[a-zA-Z]+\b", "", caption) # jc6640vc + caption = re.sub(r"\b\d+[a-zA-Z]+\d+\b", "", caption) # 6640vc231 + + caption = re.sub(r"(worldwide\s+)?(free\s+)?shipping", "", caption) + caption = re.sub(r"(free\s)?download(\sfree)?", "", caption) + caption = re.sub(r"\bclick\b\s(?:for|on)\s\w+", "", caption) + caption = re.sub(r"\b(?:png|jpg|jpeg|bmp|webp|eps|pdf|apk|mp4)(\simage[s]?)?", "", caption) + caption = re.sub(r"\bpage\s+\d+\b", "", caption) + + caption = re.sub(r"\b\d*[a-zA-Z]+\d+[a-zA-Z]+\d+[a-zA-Z\d]*\b", r" ", caption) # j2d1a2a... + + caption = re.sub(r"\b\d+\.?\d*[xх×]\d+\.?\d*\b", "", caption) + + caption = re.sub(r"\b\s+\:\s+", r": ", caption) + caption = re.sub(r"(\D[,\./])\b", r"\1 ", caption) + caption = re.sub(r"\s+", " ", caption) + + caption.strip() + + caption = re.sub(r"^[\"\']([\w\W]+)[\"\']$", r"\1", caption) + caption = re.sub(r"^[\'\_,\-\:;]", r"", caption) + caption = re.sub(r"[\'\_,\-\:\-\+]$", r"", caption) + caption = re.sub(r"^\.\S+$", "", caption) + + return caption.strip() + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @staticmethod + def classify_height_width_bin(height: int, width: int, ratios: dict) -> Tuple[int, int]: + """Returns binned height and width.""" + ar = float(height / width) + closest_ratio = min(ratios.keys(), key=lambda ratio: abs(float(ratio) - ar)) + default_hw = ratios[closest_ratio] + return int(default_hw[0]), int(default_hw[1]) + + @staticmethod + def resize_and_crop_tensor(samples: torch.Tensor, new_width: int, new_height: int) -> torch.Tensor: + orig_height, orig_width = samples.shape[2], samples.shape[3] + + # Check if resizing is needed + if orig_height != new_height or orig_width != new_width: + ratio = max(new_height / orig_height, new_width / orig_width) + resized_width = int(orig_width * ratio) + resized_height = int(orig_height * ratio) + + # Resize + samples = F.interpolate( + samples, size=(resized_height, resized_width), mode="bilinear", align_corners=False + ) + + # Center Crop + start_x = (resized_width - new_width) // 2 + end_x = start_x + new_width + start_y = (resized_height - new_height) // 2 + end_y = start_y + new_height + samples = samples[:, :, start_y:end_y, start_x:end_x] + + return samples + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + negative_prompt: str = "", + num_inference_steps: int = 20, + timesteps: List[int] = None, + guidance_scale: float = 4.5, + num_images_per_prompt: Optional[int] = 1, + height: Optional[int] = None, + width: Optional[int] = None, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + prompt_attention_mask: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_attention_mask: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + clean_caption: bool = True, + use_resolution_binning: bool = True, + max_sequence_length: int = 120, + **kwargs, + ) -> Union[ImagePipelineOutput, Tuple]: + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps are used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 4.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + height (`int`, *optional*, defaults to self.unet.config.sample_size): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size): + The width in pixels of the generated image. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + prompt_attention_mask (`torch.FloatTensor`, *optional*): Pre-generated attention mask for text embeddings. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. For PixArt-Alpha this negative prompt should be "". If not + provided, negative_prompt_embeds will be generated from `negative_prompt` input argument. + negative_prompt_attention_mask (`torch.FloatTensor`, *optional*): + Pre-generated attention mask for negative text embeddings. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.IFPipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + clean_caption (`bool`, *optional*, defaults to `True`): + Whether or not to clean the caption before creating embeddings. Requires `beautifulsoup4` and `ftfy` to + be installed. If the dependencies are not installed, the embeddings will be created from the raw + prompt. + use_resolution_binning (`bool` defaults to `True`): + If set to `True`, the requested height and width are first mapped to the closest resolutions using + `ASPECT_RATIO_1024_BIN`. After the produced latents are decoded into images, they are resized back to + the requested resolution. Useful for generating non-square images. + max_sequence_length (`int` defaults to 120): Maximum sequence length to use with the `prompt`. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images + """ + if "mask_feature" in kwargs: + deprecation_message = "The use of `mask_feature` is deprecated. It is no longer used in any computation and that doesn't affect the end results. It will be removed in a future version." + deprecate("mask_feature", "1.0.0", deprecation_message, standard_warn=False) + # 1. Check inputs. Raise error if not correct + height = height or self.transformer.config.sample_size * self.vae_scale_factor + width = width or self.transformer.config.sample_size * self.vae_scale_factor + if use_resolution_binning: + if self.transformer.config.sample_size == 128: + aspect_ratio_bin = ASPECT_RATIO_1024_BIN + elif self.transformer.config.sample_size == 64: + aspect_ratio_bin = ASPECT_RATIO_512_BIN + elif self.transformer.config.sample_size == 32: + aspect_ratio_bin = ASPECT_RATIO_256_BIN + else: + raise ValueError("Invalid sample size") + orig_height, orig_width = height, width + height, width = self.classify_height_width_bin(height, width, ratios=aspect_ratio_bin) + + self.check_inputs( + prompt, + height, + width, + negative_prompt, + callback_steps, + prompt_embeds, + negative_prompt_embeds, + prompt_attention_mask, + negative_prompt_attention_mask, + ) + + # 2. Default height and width to transformer + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + ( + prompt_embeds, + prompt_attention_mask, + negative_prompt_embeds, + negative_prompt_attention_mask, + ) = self.encode_prompt( + prompt, + do_classifier_free_guidance, + negative_prompt=negative_prompt, + num_images_per_prompt=num_images_per_prompt, + device=device, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + prompt_attention_mask=prompt_attention_mask, + negative_prompt_attention_mask=negative_prompt_attention_mask, + clean_caption=clean_caption, + max_sequence_length=max_sequence_length, + ) + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + prompt_attention_mask = torch.cat([negative_prompt_attention_mask, prompt_attention_mask], dim=0) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + + # 5. Prepare latents. + latent_channels = self.transformer.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + latent_channels, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 6.1 Prepare micro-conditions. + added_cond_kwargs = {"resolution": None, "aspect_ratio": None} + if self.transformer.config.sample_size == 128: + resolution = torch.tensor([height, width]).repeat(batch_size * num_images_per_prompt, 1) + aspect_ratio = torch.tensor([float(height / width)]).repeat(batch_size * num_images_per_prompt, 1) + resolution = resolution.to(dtype=prompt_embeds.dtype, device=device) + aspect_ratio = aspect_ratio.to(dtype=prompt_embeds.dtype, device=device) + + if do_classifier_free_guidance: + resolution = torch.cat([resolution, resolution], dim=0) + aspect_ratio = torch.cat([aspect_ratio, aspect_ratio], dim=0) + + added_cond_kwargs = {"resolution": resolution, "aspect_ratio": aspect_ratio} + + # 7. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + current_timestep = t + if not torch.is_tensor(current_timestep): + # TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can + # This would be a good case for the `match` statement (Python 3.10+) + is_mps = latent_model_input.device.type == "mps" + if isinstance(current_timestep, float): + dtype = torch.float32 if is_mps else torch.float64 + else: + dtype = torch.int32 if is_mps else torch.int64 + current_timestep = torch.tensor([current_timestep], dtype=dtype, device=latent_model_input.device) + elif len(current_timestep.shape) == 0: + current_timestep = current_timestep[None].to(latent_model_input.device) + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + current_timestep = current_timestep.expand(latent_model_input.shape[0]) + + # predict noise model_output + noise_pred = self.transformer( + latent_model_input, + encoder_hidden_states=prompt_embeds, + encoder_attention_mask=prompt_attention_mask, + timestep=current_timestep, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # learned sigma + if self.transformer.config.out_channels // 2 == latent_channels: + noise_pred = noise_pred.chunk(2, dim=1)[0] + else: + noise_pred = noise_pred + + # compute previous image: x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + if use_resolution_binning: + image = self.resize_and_crop_tensor(image, orig_width, orig_height) + else: + image = latents + + if not output_type == "latent": + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/__init__.py new file mode 100755 index 0000000..70f5b1a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/__init__.py @@ -0,0 +1,49 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_output"] = ["SemanticStableDiffusionPipelineOutput"] + _import_structure["pipeline_semantic_stable_diffusion"] = ["SemanticStableDiffusionPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_semantic_stable_diffusion import SemanticStableDiffusionPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_output.py b/diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_output.py new file mode 100755 index 0000000..3499129 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_output.py @@ -0,0 +1,25 @@ +from dataclasses import dataclass +from typing import List, Optional, Union + +import numpy as np +import PIL.Image + +from ...utils import BaseOutput + + +@dataclass +class SemanticStableDiffusionPipelineOutput(BaseOutput): + """ + Output class for Stable Diffusion pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + num_channels)`. + nsfw_content_detected (`List[bool]`) + List indicating whether the corresponding generated image contains “not-safe-for-work” (nsfw) content or + `None` if safety checking could not be performed. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + nsfw_content_detected: Optional[List[bool]] diff --git a/diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py new file mode 100755 index 0000000..9687342 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/semantic_stable_diffusion/pipeline_semantic_stable_diffusion.py @@ -0,0 +1,718 @@ +import inspect +from itertools import repeat +from typing import Callable, List, Optional, Union + +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ...image_processor import VaeImageProcessor +from ...models import AutoencoderKL, UNet2DConditionModel +from ...pipelines.stable_diffusion.safety_checker import StableDiffusionSafetyChecker +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import deprecate, logging +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from .pipeline_output import SemanticStableDiffusionPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class SemanticStableDiffusionPipeline(DiffusionPipeline, StableDiffusionMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion with latent editing. + + This model inherits from [`DiffusionPipeline`] and builds on the [`StableDiffusionPipeline`]. Check the superclass + documentation for the generic methods implemented for all pipelines (downloading, saving, running on a particular + device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`Q16SafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion_k_diffusion.pipeline_stable_diffusion_k_diffusion.StableDiffusionKDiffusionPipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + editing_prompt: Optional[Union[str, List[str]]] = None, + editing_prompt_embeddings: Optional[torch.Tensor] = None, + reverse_editing_direction: Optional[Union[bool, List[bool]]] = False, + edit_guidance_scale: Optional[Union[float, List[float]]] = 5, + edit_warmup_steps: Optional[Union[int, List[int]]] = 10, + edit_cooldown_steps: Optional[Union[int, List[int]]] = None, + edit_threshold: Optional[Union[float, List[float]]] = 0.9, + edit_momentum_scale: Optional[float] = 0.1, + edit_mom_beta: Optional[float] = 0.4, + edit_weights: Optional[List[float]] = None, + sem_guidance: Optional[List[torch.Tensor]] = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide image generation. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + editing_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to use for semantic guidance. Semantic guidance is disabled by setting + `editing_prompt = None`. Guidance direction of prompt should be specified via + `reverse_editing_direction`. + editing_prompt_embeddings (`torch.Tensor`, *optional*): + Pre-computed embeddings to use for semantic guidance. Guidance direction of embedding should be + specified via `reverse_editing_direction`. + reverse_editing_direction (`bool` or `List[bool]`, *optional*, defaults to `False`): + Whether the corresponding prompt in `editing_prompt` should be increased or decreased. + edit_guidance_scale (`float` or `List[float]`, *optional*, defaults to 5): + Guidance scale for semantic guidance. If provided as a list, values should correspond to + `editing_prompt`. + edit_warmup_steps (`float` or `List[float]`, *optional*, defaults to 10): + Number of diffusion steps (for each prompt) for which semantic guidance is not applied. Momentum is + calculated for those steps and applied once all warmup periods are over. + edit_cooldown_steps (`float` or `List[float]`, *optional*, defaults to `None`): + Number of diffusion steps (for each prompt) after which semantic guidance is longer applied. + edit_threshold (`float` or `List[float]`, *optional*, defaults to 0.9): + Threshold of semantic guidance. + edit_momentum_scale (`float`, *optional*, defaults to 0.1): + Scale of the momentum to be added to the semantic guidance at each diffusion step. If set to 0.0, + momentum is disabled. Momentum is already built up during warmup (for diffusion steps smaller than + `sld_warmup_steps`). Momentum is only added to latent guidance once all warmup periods are finished. + edit_mom_beta (`float`, *optional*, defaults to 0.4): + Defines how semantic guidance momentum builds up. `edit_mom_beta` indicates how much of the previous + momentum is kept. Momentum is already built up during warmup (for diffusion steps smaller than + `edit_warmup_steps`). + edit_weights (`List[float]`, *optional*, defaults to `None`): + Indicates how much each individual concept should influence the overall guidance. If no weights are + provided all concepts are applied equally. + sem_guidance (`List[torch.Tensor]`, *optional*): + List of pre-generated guidance vectors to be applied at generation. Length of the list has to + correspond to `num_inference_steps`. + + Examples: + + ```py + >>> import torch + >>> from diffusers import SemanticStableDiffusionPipeline + + >>> pipe = SemanticStableDiffusionPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> out = pipe( + ... prompt="a photo of the face of a woman", + ... num_images_per_prompt=1, + ... guidance_scale=7, + ... editing_prompt=[ + ... "smiling, smile", # Concepts to apply + ... "glasses, wearing glasses", + ... "curls, wavy hair, curly hair", + ... "beard, full beard, mustache", + ... ], + ... reverse_editing_direction=[ + ... False, + ... False, + ... False, + ... False, + ... ], # Direction of guidance i.e. increase all concepts + ... edit_warmup_steps=[10, 10, 10, 10], # Warmup period for each concept + ... edit_guidance_scale=[4, 5, 5, 5.4], # Guidance scale for each concept + ... edit_threshold=[ + ... 0.99, + ... 0.975, + ... 0.925, + ... 0.96, + ... ], # Threshold for each concept. Threshold equals the percentile of the latent space that will be discarded. I.e. threshold=0.99 uses 1% of the latent dimensions + ... edit_momentum_scale=0.3, # Momentum scale that will be added to the latent guidance + ... edit_mom_beta=0.6, # Momentum beta + ... edit_weights=[1, 1, 1, 1, 1], # Weights of the individual concepts against each other + ... ) + >>> image = out.images[0] + ``` + + Returns: + [`~pipelines.semantic_stable_diffusion.SemanticStableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, + [`~pipelines.semantic_stable_diffusion.SemanticStableDiffusionPipelineOutput`] is returned, otherwise a + `tuple` is returned where the first element is a list with the generated images and the second element + is a list of `bool`s indicating whether the corresponding generated image contains "not-safe-for-work" + (nsfw) content. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs(prompt, height, width, callback_steps) + + # 2. Define call parameters + batch_size = 1 if isinstance(prompt, str) else len(prompt) + + if editing_prompt: + enable_edit_guidance = True + if isinstance(editing_prompt, str): + editing_prompt = [editing_prompt] + enabled_editing_prompts = len(editing_prompt) + elif editing_prompt_embeddings is not None: + enable_edit_guidance = True + enabled_editing_prompts = editing_prompt_embeddings.shape[0] + else: + enabled_editing_prompts = 0 + enable_edit_guidance = False + + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + + if text_input_ids.shape[-1] > self.tokenizer.model_max_length: + removed_text = self.tokenizer.batch_decode(text_input_ids[:, self.tokenizer.model_max_length :]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + text_input_ids = text_input_ids[:, : self.tokenizer.model_max_length] + text_embeddings = self.text_encoder(text_input_ids.to(self.device))[0] + + # duplicate text embeddings for each generation per prompt, using mps friendly method + bs_embed, seq_len, _ = text_embeddings.shape + text_embeddings = text_embeddings.repeat(1, num_images_per_prompt, 1) + text_embeddings = text_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if enable_edit_guidance: + # get safety text embeddings + if editing_prompt_embeddings is None: + edit_concepts_input = self.tokenizer( + [x for item in editing_prompt for x in repeat(item, batch_size)], + padding="max_length", + max_length=self.tokenizer.model_max_length, + return_tensors="pt", + ) + + edit_concepts_input_ids = edit_concepts_input.input_ids + + if edit_concepts_input_ids.shape[-1] > self.tokenizer.model_max_length: + removed_text = self.tokenizer.batch_decode( + edit_concepts_input_ids[:, self.tokenizer.model_max_length :] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + edit_concepts_input_ids = edit_concepts_input_ids[:, : self.tokenizer.model_max_length] + edit_concepts = self.text_encoder(edit_concepts_input_ids.to(self.device))[0] + else: + edit_concepts = editing_prompt_embeddings.to(self.device).repeat(batch_size, 1, 1) + + # duplicate text embeddings for each generation per prompt, using mps friendly method + bs_embed_edit, seq_len_edit, _ = edit_concepts.shape + edit_concepts = edit_concepts.repeat(1, num_images_per_prompt, 1) + edit_concepts = edit_concepts.view(bs_embed_edit * num_images_per_prompt, seq_len_edit, -1) + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + # get unconditional embeddings for classifier free guidance + + if do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = text_input_ids.shape[-1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + uncond_embeddings = self.text_encoder(uncond_input.input_ids.to(self.device))[0] + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = uncond_embeddings.shape[1] + uncond_embeddings = uncond_embeddings.repeat(1, num_images_per_prompt, 1) + uncond_embeddings = uncond_embeddings.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if enable_edit_guidance: + text_embeddings = torch.cat([uncond_embeddings, text_embeddings, edit_concepts]) + else: + text_embeddings = torch.cat([uncond_embeddings, text_embeddings]) + # get the initial random noise unless the user supplied it + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=self.device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + text_embeddings.dtype, + self.device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # Initialize edit_momentum to None + edit_momentum = None + + self.uncond_estimates = None + self.text_estimates = None + self.edit_estimates = None + self.sem_guidance = None + + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = ( + torch.cat([latents] * (2 + enabled_editing_prompts)) if do_classifier_free_guidance else latents + ) + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=text_embeddings).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_out = noise_pred.chunk(2 + enabled_editing_prompts) # [b,4, 64, 64] + noise_pred_uncond, noise_pred_text = noise_pred_out[0], noise_pred_out[1] + noise_pred_edit_concepts = noise_pred_out[2:] + + # default text guidance + noise_guidance = guidance_scale * (noise_pred_text - noise_pred_uncond) + # noise_guidance = (noise_pred_text - noise_pred_edit_concepts[0]) + + if self.uncond_estimates is None: + self.uncond_estimates = torch.zeros((num_inference_steps + 1, *noise_pred_uncond.shape)) + self.uncond_estimates[i] = noise_pred_uncond.detach().cpu() + + if self.text_estimates is None: + self.text_estimates = torch.zeros((num_inference_steps + 1, *noise_pred_text.shape)) + self.text_estimates[i] = noise_pred_text.detach().cpu() + + if self.edit_estimates is None and enable_edit_guidance: + self.edit_estimates = torch.zeros( + (num_inference_steps + 1, len(noise_pred_edit_concepts), *noise_pred_edit_concepts[0].shape) + ) + + if self.sem_guidance is None: + self.sem_guidance = torch.zeros((num_inference_steps + 1, *noise_pred_text.shape)) + + if edit_momentum is None: + edit_momentum = torch.zeros_like(noise_guidance) + + if enable_edit_guidance: + concept_weights = torch.zeros( + (len(noise_pred_edit_concepts), noise_guidance.shape[0]), + device=self.device, + dtype=noise_guidance.dtype, + ) + noise_guidance_edit = torch.zeros( + (len(noise_pred_edit_concepts), *noise_guidance.shape), + device=self.device, + dtype=noise_guidance.dtype, + ) + # noise_guidance_edit = torch.zeros_like(noise_guidance) + warmup_inds = [] + for c, noise_pred_edit_concept in enumerate(noise_pred_edit_concepts): + self.edit_estimates[i, c] = noise_pred_edit_concept + if isinstance(edit_guidance_scale, list): + edit_guidance_scale_c = edit_guidance_scale[c] + else: + edit_guidance_scale_c = edit_guidance_scale + + if isinstance(edit_threshold, list): + edit_threshold_c = edit_threshold[c] + else: + edit_threshold_c = edit_threshold + if isinstance(reverse_editing_direction, list): + reverse_editing_direction_c = reverse_editing_direction[c] + else: + reverse_editing_direction_c = reverse_editing_direction + if edit_weights: + edit_weight_c = edit_weights[c] + else: + edit_weight_c = 1.0 + if isinstance(edit_warmup_steps, list): + edit_warmup_steps_c = edit_warmup_steps[c] + else: + edit_warmup_steps_c = edit_warmup_steps + + if isinstance(edit_cooldown_steps, list): + edit_cooldown_steps_c = edit_cooldown_steps[c] + elif edit_cooldown_steps is None: + edit_cooldown_steps_c = i + 1 + else: + edit_cooldown_steps_c = edit_cooldown_steps + if i >= edit_warmup_steps_c: + warmup_inds.append(c) + if i >= edit_cooldown_steps_c: + noise_guidance_edit[c, :, :, :, :] = torch.zeros_like(noise_pred_edit_concept) + continue + + noise_guidance_edit_tmp = noise_pred_edit_concept - noise_pred_uncond + # tmp_weights = (noise_pred_text - noise_pred_edit_concept).sum(dim=(1, 2, 3)) + tmp_weights = (noise_guidance - noise_pred_edit_concept).sum(dim=(1, 2, 3)) + + tmp_weights = torch.full_like(tmp_weights, edit_weight_c) # * (1 / enabled_editing_prompts) + if reverse_editing_direction_c: + noise_guidance_edit_tmp = noise_guidance_edit_tmp * -1 + concept_weights[c, :] = tmp_weights + + noise_guidance_edit_tmp = noise_guidance_edit_tmp * edit_guidance_scale_c + + # torch.quantile function expects float32 + if noise_guidance_edit_tmp.dtype == torch.float32: + tmp = torch.quantile( + torch.abs(noise_guidance_edit_tmp).flatten(start_dim=2), + edit_threshold_c, + dim=2, + keepdim=False, + ) + else: + tmp = torch.quantile( + torch.abs(noise_guidance_edit_tmp).flatten(start_dim=2).to(torch.float32), + edit_threshold_c, + dim=2, + keepdim=False, + ).to(noise_guidance_edit_tmp.dtype) + + noise_guidance_edit_tmp = torch.where( + torch.abs(noise_guidance_edit_tmp) >= tmp[:, :, None, None], + noise_guidance_edit_tmp, + torch.zeros_like(noise_guidance_edit_tmp), + ) + noise_guidance_edit[c, :, :, :, :] = noise_guidance_edit_tmp + + # noise_guidance_edit = noise_guidance_edit + noise_guidance_edit_tmp + + warmup_inds = torch.tensor(warmup_inds).to(self.device) + if len(noise_pred_edit_concepts) > warmup_inds.shape[0] > 0: + concept_weights = concept_weights.to("cpu") # Offload to cpu + noise_guidance_edit = noise_guidance_edit.to("cpu") + + concept_weights_tmp = torch.index_select(concept_weights.to(self.device), 0, warmup_inds) + concept_weights_tmp = torch.where( + concept_weights_tmp < 0, torch.zeros_like(concept_weights_tmp), concept_weights_tmp + ) + concept_weights_tmp = concept_weights_tmp / concept_weights_tmp.sum(dim=0) + # concept_weights_tmp = torch.nan_to_num(concept_weights_tmp) + + noise_guidance_edit_tmp = torch.index_select( + noise_guidance_edit.to(self.device), 0, warmup_inds + ) + noise_guidance_edit_tmp = torch.einsum( + "cb,cbijk->bijk", concept_weights_tmp, noise_guidance_edit_tmp + ) + noise_guidance_edit_tmp = noise_guidance_edit_tmp + noise_guidance = noise_guidance + noise_guidance_edit_tmp + + self.sem_guidance[i] = noise_guidance_edit_tmp.detach().cpu() + + del noise_guidance_edit_tmp + del concept_weights_tmp + concept_weights = concept_weights.to(self.device) + noise_guidance_edit = noise_guidance_edit.to(self.device) + + concept_weights = torch.where( + concept_weights < 0, torch.zeros_like(concept_weights), concept_weights + ) + + concept_weights = torch.nan_to_num(concept_weights) + + noise_guidance_edit = torch.einsum("cb,cbijk->bijk", concept_weights, noise_guidance_edit) + + noise_guidance_edit = noise_guidance_edit + edit_momentum_scale * edit_momentum + + edit_momentum = edit_mom_beta * edit_momentum + (1 - edit_mom_beta) * noise_guidance_edit + + if warmup_inds.shape[0] == len(noise_pred_edit_concepts): + noise_guidance = noise_guidance + noise_guidance_edit + self.sem_guidance[i] = noise_guidance_edit.detach().cpu() + + if sem_guidance is not None: + edit_guidance = sem_guidance[i].to(self.device) + noise_guidance = noise_guidance + edit_guidance + + noise_pred = noise_pred_uncond + noise_guidance + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 8. Post-processing + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, self.device, text_embeddings.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + if not return_dict: + return (image, has_nsfw_concept) + + return SemanticStableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/shap_e/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/shap_e/__init__.py new file mode 100755 index 0000000..4ed563c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/shap_e/__init__.py @@ -0,0 +1,71 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["camera"] = ["create_pan_cameras"] + _import_structure["pipeline_shap_e"] = ["ShapEPipeline"] + _import_structure["pipeline_shap_e_img2img"] = ["ShapEImg2ImgPipeline"] + _import_structure["renderer"] = [ + "BoundingBoxVolume", + "ImportanceRaySampler", + "MLPNeRFModelOutput", + "MLPNeRSTFModel", + "ShapEParamsProjModel", + "ShapERenderer", + "StratifiedRaySampler", + "VoidNeRFModel", + ] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .camera import create_pan_cameras + from .pipeline_shap_e import ShapEPipeline + from .pipeline_shap_e_img2img import ShapEImg2ImgPipeline + from .renderer import ( + BoundingBoxVolume, + ImportanceRaySampler, + MLPNeRFModelOutput, + MLPNeRSTFModel, + ShapEParamsProjModel, + ShapERenderer, + StratifiedRaySampler, + VoidNeRFModel, + ) + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/shap_e/camera.py b/diffusers-0.27.0/src/diffusers/pipelines/shap_e/camera.py new file mode 100755 index 0000000..d4b94c3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/shap_e/camera.py @@ -0,0 +1,147 @@ +# Copyright 2024 Open AI and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Tuple + +import numpy as np +import torch + + +@dataclass +class DifferentiableProjectiveCamera: + """ + Implements a batch, differentiable, standard pinhole camera + """ + + origin: torch.Tensor # [batch_size x 3] + x: torch.Tensor # [batch_size x 3] + y: torch.Tensor # [batch_size x 3] + z: torch.Tensor # [batch_size x 3] + width: int + height: int + x_fov: float + y_fov: float + shape: Tuple[int] + + def __post_init__(self): + assert self.x.shape[0] == self.y.shape[0] == self.z.shape[0] == self.origin.shape[0] + assert self.x.shape[1] == self.y.shape[1] == self.z.shape[1] == self.origin.shape[1] == 3 + assert len(self.x.shape) == len(self.y.shape) == len(self.z.shape) == len(self.origin.shape) == 2 + + def resolution(self): + return torch.from_numpy(np.array([self.width, self.height], dtype=np.float32)) + + def fov(self): + return torch.from_numpy(np.array([self.x_fov, self.y_fov], dtype=np.float32)) + + def get_image_coords(self) -> torch.Tensor: + """ + :return: coords of shape (width * height, 2) + """ + pixel_indices = torch.arange(self.height * self.width) + coords = torch.stack( + [ + pixel_indices % self.width, + torch.div(pixel_indices, self.width, rounding_mode="trunc"), + ], + axis=1, + ) + return coords + + @property + def camera_rays(self): + batch_size, *inner_shape = self.shape + inner_batch_size = int(np.prod(inner_shape)) + + coords = self.get_image_coords() + coords = torch.broadcast_to(coords.unsqueeze(0), [batch_size * inner_batch_size, *coords.shape]) + rays = self.get_camera_rays(coords) + + rays = rays.view(batch_size, inner_batch_size * self.height * self.width, 2, 3) + + return rays + + def get_camera_rays(self, coords: torch.Tensor) -> torch.Tensor: + batch_size, *shape, n_coords = coords.shape + assert n_coords == 2 + assert batch_size == self.origin.shape[0] + + flat = coords.view(batch_size, -1, 2) + + res = self.resolution() + fov = self.fov() + + fracs = (flat.float() / (res - 1)) * 2 - 1 + fracs = fracs * torch.tan(fov / 2) + + fracs = fracs.view(batch_size, -1, 2) + directions = ( + self.z.view(batch_size, 1, 3) + + self.x.view(batch_size, 1, 3) * fracs[:, :, :1] + + self.y.view(batch_size, 1, 3) * fracs[:, :, 1:] + ) + directions = directions / directions.norm(dim=-1, keepdim=True) + rays = torch.stack( + [ + torch.broadcast_to(self.origin.view(batch_size, 1, 3), [batch_size, directions.shape[1], 3]), + directions, + ], + dim=2, + ) + return rays.view(batch_size, *shape, 2, 3) + + def resize_image(self, width: int, height: int) -> "DifferentiableProjectiveCamera": + """ + Creates a new camera for the resized view assuming the aspect ratio does not change. + """ + assert width * self.height == height * self.width, "The aspect ratio should not change." + return DifferentiableProjectiveCamera( + origin=self.origin, + x=self.x, + y=self.y, + z=self.z, + width=width, + height=height, + x_fov=self.x_fov, + y_fov=self.y_fov, + ) + + +def create_pan_cameras(size: int) -> DifferentiableProjectiveCamera: + origins = [] + xs = [] + ys = [] + zs = [] + for theta in np.linspace(0, 2 * np.pi, num=20): + z = np.array([np.sin(theta), np.cos(theta), -0.5]) + z /= np.sqrt(np.sum(z**2)) + origin = -z * 4 + x = np.array([np.cos(theta), -np.sin(theta), 0.0]) + y = np.cross(z, x) + origins.append(origin) + xs.append(x) + ys.append(y) + zs.append(z) + return DifferentiableProjectiveCamera( + origin=torch.from_numpy(np.stack(origins, axis=0)).float(), + x=torch.from_numpy(np.stack(xs, axis=0)).float(), + y=torch.from_numpy(np.stack(ys, axis=0)).float(), + z=torch.from_numpy(np.stack(zs, axis=0)).float(), + width=size, + height=size, + x_fov=0.7, + y_fov=0.7, + shape=(1, len(xs)), + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/shap_e/pipeline_shap_e.py b/diffusers-0.27.0/src/diffusers/pipelines/shap_e/pipeline_shap_e.py new file mode 100755 index 0000000..1ef10e1 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/shap_e/pipeline_shap_e.py @@ -0,0 +1,334 @@ +# Copyright 2024 Open AI and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from dataclasses import dataclass +from typing import List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPTextModelWithProjection, CLIPTokenizer + +from ...models import PriorTransformer +from ...schedulers import HeunDiscreteScheduler +from ...utils import ( + BaseOutput, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .renderer import ShapERenderer + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import DiffusionPipeline + >>> from diffusers.utils import export_to_gif + + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + >>> repo = "openai/shap-e" + >>> pipe = DiffusionPipeline.from_pretrained(repo, torch_dtype=torch.float16) + >>> pipe = pipe.to(device) + + >>> guidance_scale = 15.0 + >>> prompt = "a shark" + + >>> images = pipe( + ... prompt, + ... guidance_scale=guidance_scale, + ... num_inference_steps=64, + ... frame_size=256, + ... ).images + + >>> gif_path = export_to_gif(images[0], "shark_3d.gif") + ``` +""" + + +@dataclass +class ShapEPipelineOutput(BaseOutput): + """ + Output class for [`ShapEPipeline`] and [`ShapEImg2ImgPipeline`]. + + Args: + images (`torch.FloatTensor`) + A list of images for 3D rendering. + """ + + images: Union[List[List[PIL.Image.Image]], List[List[np.ndarray]]] + + +class ShapEPipeline(DiffusionPipeline): + """ + Pipeline for generating latent representation of a 3D asset and rendering with the NeRF method. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + prior ([`PriorTransformer`]): + The canonical unCLIP prior to approximate the image embedding from the text embedding. + text_encoder ([`~transformers.CLIPTextModelWithProjection`]): + Frozen text-encoder. + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + scheduler ([`HeunDiscreteScheduler`]): + A scheduler to be used in combination with the `prior` model to generate image embedding. + shap_e_renderer ([`ShapERenderer`]): + Shap-E renderer projects the generated latents into parameters of a MLP to create 3D objects with the NeRF + rendering method. + """ + + model_cpu_offload_seq = "text_encoder->prior" + _exclude_from_cpu_offload = ["shap_e_renderer"] + + def __init__( + self, + prior: PriorTransformer, + text_encoder: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + scheduler: HeunDiscreteScheduler, + shap_e_renderer: ShapERenderer, + ): + super().__init__() + + self.register_modules( + prior=prior, + text_encoder=text_encoder, + tokenizer=tokenizer, + scheduler=scheduler, + shap_e_renderer=shap_e_renderer, + ) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + ): + len(prompt) if isinstance(prompt, list) else 1 + + # YiYi Notes: set pad_token_id to be 0, not sure why I can't set in the config file + self.tokenizer.pad_token_id = 0 + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + text_encoder_output = self.text_encoder(text_input_ids.to(device)) + prompt_embeds = text_encoder_output.text_embeds + + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + # in Shap-E it normalize the prompt_embeds and then later rescale it + prompt_embeds = prompt_embeds / torch.linalg.norm(prompt_embeds, dim=-1, keepdim=True) + + if do_classifier_free_guidance: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # Rescale the features to have unit variance + prompt_embeds = math.sqrt(prompt_embeds.shape[1]) * prompt_embeds + + return prompt_embeds + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: str, + num_images_per_prompt: int = 1, + num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + guidance_scale: float = 4.0, + frame_size: int = 64, + output_type: Optional[str] = "pil", # pil, np, latent, mesh + return_dict: bool = True, + ): + """ + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 25): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + guidance_scale (`float`, *optional*, defaults to 4.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + frame_size (`int`, *optional*, default to 64): + The width and height of each image frame of the generated 3D output. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`), `"latent"` (`torch.Tensor`), or mesh ([`MeshDecoderOutput`]). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.shap_e.pipeline_shap_e.ShapEPipelineOutput`] instead of a plain + tuple. + + Examples: + + Returns: + [`~pipelines.shap_e.pipeline_shap_e.ShapEPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.shap_e.pipeline_shap_e.ShapEPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images. + """ + + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + device = self._execution_device + + batch_size = batch_size * num_images_per_prompt + + do_classifier_free_guidance = guidance_scale > 1.0 + prompt_embeds = self._encode_prompt(prompt, device, num_images_per_prompt, do_classifier_free_guidance) + + # prior + + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + num_embeddings = self.prior.config.num_embeddings + embedding_dim = self.prior.config.embedding_dim + + latents = self.prepare_latents( + (batch_size, num_embeddings * embedding_dim), + prompt_embeds.dtype, + device, + generator, + latents, + self.scheduler, + ) + + # YiYi notes: for testing only to match ldm, we can directly create a latents with desired shape: batch_size, num_embeddings, embedding_dim + latents = latents.reshape(latents.shape[0], num_embeddings, embedding_dim) + + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + scaled_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + noise_pred = self.prior( + scaled_model_input, + timestep=t, + proj_embedding=prompt_embeds, + ).predicted_image_embedding + + # remove the variance + noise_pred, _ = noise_pred.split( + scaled_model_input.shape[2], dim=2 + ) # batch_size, num_embeddings, embedding_dim + + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred - noise_pred_uncond) + + latents = self.scheduler.step( + noise_pred, + timestep=t, + sample=latents, + ).prev_sample + + # Offload all models + self.maybe_free_model_hooks() + + if output_type not in ["np", "pil", "latent", "mesh"]: + raise ValueError( + f"Only the output types `pil`, `np`, `latent` and `mesh` are supported not output_type={output_type}" + ) + + if output_type == "latent": + return ShapEPipelineOutput(images=latents) + + images = [] + if output_type == "mesh": + for i, latent in enumerate(latents): + mesh = self.shap_e_renderer.decode_to_mesh( + latent[None, :], + device, + ) + images.append(mesh) + + else: + # np, pil + for i, latent in enumerate(latents): + image = self.shap_e_renderer.decode_to_image( + latent[None, :], + device, + size=frame_size, + ) + images.append(image) + + images = torch.stack(images) + + images = images.cpu().numpy() + + if output_type == "pil": + images = [self.numpy_to_pil(image) for image in images] + + if not return_dict: + return (images,) + + return ShapEPipelineOutput(images=images) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py new file mode 100755 index 0000000..641ec56 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/shap_e/pipeline_shap_e_img2img.py @@ -0,0 +1,321 @@ +# Copyright 2024 Open AI and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPVisionModel + +from ...models import PriorTransformer +from ...schedulers import HeunDiscreteScheduler +from ...utils import ( + BaseOutput, + logging, + replace_example_docstring, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .renderer import ShapERenderer + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from PIL import Image + >>> import torch + >>> from diffusers import DiffusionPipeline + >>> from diffusers.utils import export_to_gif, load_image + + >>> device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + + >>> repo = "openai/shap-e-img2img" + >>> pipe = DiffusionPipeline.from_pretrained(repo, torch_dtype=torch.float16) + >>> pipe = pipe.to(device) + + >>> guidance_scale = 3.0 + >>> image_url = "https://hf.co/datasets/diffusers/docs-images/resolve/main/shap-e/corgi.png" + >>> image = load_image(image_url).convert("RGB") + + >>> images = pipe( + ... image, + ... guidance_scale=guidance_scale, + ... num_inference_steps=64, + ... frame_size=256, + ... ).images + + >>> gif_path = export_to_gif(images[0], "corgi_3d.gif") + ``` +""" + + +@dataclass +class ShapEPipelineOutput(BaseOutput): + """ + Output class for [`ShapEPipeline`] and [`ShapEImg2ImgPipeline`]. + + Args: + images (`torch.FloatTensor`) + A list of images for 3D rendering. + """ + + images: Union[PIL.Image.Image, np.ndarray] + + +class ShapEImg2ImgPipeline(DiffusionPipeline): + """ + Pipeline for generating latent representation of a 3D asset and rendering with the NeRF method from an image. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + prior ([`PriorTransformer`]): + The canonincal unCLIP prior to approximate the image embedding from the text embedding. + image_encoder ([`~transformers.CLIPVisionModel`]): + Frozen image-encoder. + image_processor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to process images. + scheduler ([`HeunDiscreteScheduler`]): + A scheduler to be used in combination with the `prior` model to generate image embedding. + shap_e_renderer ([`ShapERenderer`]): + Shap-E renderer projects the generated latents into parameters of a MLP to create 3D objects with the NeRF + rendering method. + """ + + model_cpu_offload_seq = "image_encoder->prior" + _exclude_from_cpu_offload = ["shap_e_renderer"] + + def __init__( + self, + prior: PriorTransformer, + image_encoder: CLIPVisionModel, + image_processor: CLIPImageProcessor, + scheduler: HeunDiscreteScheduler, + shap_e_renderer: ShapERenderer, + ): + super().__init__() + + self.register_modules( + prior=prior, + image_encoder=image_encoder, + image_processor=image_processor, + scheduler=scheduler, + shap_e_renderer=shap_e_renderer, + ) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def _encode_image( + self, + image, + device, + num_images_per_prompt, + do_classifier_free_guidance, + ): + if isinstance(image, List) and isinstance(image[0], torch.Tensor): + image = torch.cat(image, axis=0) if image[0].ndim == 4 else torch.stack(image, axis=0) + + if not isinstance(image, torch.Tensor): + image = self.image_processor(image, return_tensors="pt").pixel_values[0].unsqueeze(0) + + image = image.to(dtype=self.image_encoder.dtype, device=device) + + image_embeds = self.image_encoder(image)["last_hidden_state"] + image_embeds = image_embeds[:, 1:, :].contiguous() # batch_size, dim, 256 + + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + + if do_classifier_free_guidance: + negative_image_embeds = torch.zeros_like(image_embeds) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + image_embeds = torch.cat([negative_image_embeds, image_embeds]) + + return image_embeds + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image: Union[PIL.Image.Image, List[PIL.Image.Image]], + num_images_per_prompt: int = 1, + num_inference_steps: int = 25, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + guidance_scale: float = 4.0, + frame_size: int = 64, + output_type: Optional[str] = "pil", # pil, np, latent, mesh + return_dict: bool = True, + ): + """ + The call function to the pipeline for generation. + + Args: + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image` or tensor representing an image batch to be used as the starting point. Can also accept image + latents as image, but if passing latents directly it is not encoded again. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + num_inference_steps (`int`, *optional*, defaults to 25): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + guidance_scale (`float`, *optional*, defaults to 4.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + frame_size (`int`, *optional*, default to 64): + The width and height of each image frame of the generated 3D output. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`), `"latent"` (`torch.Tensor`), or mesh ([`MeshDecoderOutput`]). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.shap_e.pipeline_shap_e.ShapEPipelineOutput`] instead of a plain + tuple. + + Examples: + + Returns: + [`~pipelines.shap_e.pipeline_shap_e.ShapEPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.shap_e.pipeline_shap_e.ShapEPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images. + """ + + if isinstance(image, PIL.Image.Image): + batch_size = 1 + elif isinstance(image, torch.Tensor): + batch_size = image.shape[0] + elif isinstance(image, list) and isinstance(image[0], (torch.Tensor, PIL.Image.Image)): + batch_size = len(image) + else: + raise ValueError( + f"`image` has to be of type `PIL.Image.Image`, `torch.Tensor`, `List[PIL.Image.Image]` or `List[torch.Tensor]` but is {type(image)}" + ) + + device = self._execution_device + + batch_size = batch_size * num_images_per_prompt + + do_classifier_free_guidance = guidance_scale > 1.0 + image_embeds = self._encode_image(image, device, num_images_per_prompt, do_classifier_free_guidance) + + # prior + + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + num_embeddings = self.prior.config.num_embeddings + embedding_dim = self.prior.config.embedding_dim + + latents = self.prepare_latents( + (batch_size, num_embeddings * embedding_dim), + image_embeds.dtype, + device, + generator, + latents, + self.scheduler, + ) + + # YiYi notes: for testing only to match ldm, we can directly create a latents with desired shape: batch_size, num_embeddings, embedding_dim + latents = latents.reshape(latents.shape[0], num_embeddings, embedding_dim) + + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + scaled_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + noise_pred = self.prior( + scaled_model_input, + timestep=t, + proj_embedding=image_embeds, + ).predicted_image_embedding + + # remove the variance + noise_pred, _ = noise_pred.split( + scaled_model_input.shape[2], dim=2 + ) # batch_size, num_embeddings, embedding_dim + + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred - noise_pred_uncond) + + latents = self.scheduler.step( + noise_pred, + timestep=t, + sample=latents, + ).prev_sample + + if output_type not in ["np", "pil", "latent", "mesh"]: + raise ValueError( + f"Only the output types `pil`, `np`, `latent` and `mesh` are supported not output_type={output_type}" + ) + + # Offload all models + self.maybe_free_model_hooks() + + if output_type == "latent": + return ShapEPipelineOutput(images=latents) + + images = [] + if output_type == "mesh": + for i, latent in enumerate(latents): + mesh = self.shap_e_renderer.decode_to_mesh( + latent[None, :], + device, + ) + images.append(mesh) + + else: + # np, pil + for i, latent in enumerate(latents): + image = self.shap_e_renderer.decode_to_image( + latent[None, :], + device, + size=frame_size, + ) + images.append(image) + + images = torch.stack(images) + + images = images.cpu().numpy() + + if output_type == "pil": + images = [self.numpy_to_pil(image) for image in images] + + if not return_dict: + return (images,) + + return ShapEPipelineOutput(images=images) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/shap_e/renderer.py b/diffusers-0.27.0/src/diffusers/pipelines/shap_e/renderer.py new file mode 100755 index 0000000..047c6f7 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/shap_e/renderer.py @@ -0,0 +1,1050 @@ +# Copyright 2024 Open AI and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from dataclasses import dataclass +from typing import Dict, Optional, Tuple + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...models import ModelMixin +from ...utils import BaseOutput +from .camera import create_pan_cameras + + +def sample_pmf(pmf: torch.Tensor, n_samples: int) -> torch.Tensor: + r""" + Sample from the given discrete probability distribution with replacement. + + The i-th bin is assumed to have mass pmf[i]. + + Args: + pmf: [batch_size, *shape, n_samples, 1] where (pmf.sum(dim=-2) == 1).all() + n_samples: number of samples + + Return: + indices sampled with replacement + """ + + *shape, support_size, last_dim = pmf.shape + assert last_dim == 1 + + cdf = torch.cumsum(pmf.view(-1, support_size), dim=1) + inds = torch.searchsorted(cdf, torch.rand(cdf.shape[0], n_samples, device=cdf.device)) + + return inds.view(*shape, n_samples, 1).clamp(0, support_size - 1) + + +def posenc_nerf(x: torch.Tensor, min_deg: int = 0, max_deg: int = 15) -> torch.Tensor: + """ + Concatenate x and its positional encodings, following NeRF. + + Reference: https://arxiv.org/pdf/2210.04628.pdf + """ + if min_deg == max_deg: + return x + + scales = 2.0 ** torch.arange(min_deg, max_deg, dtype=x.dtype, device=x.device) + *shape, dim = x.shape + xb = (x.reshape(-1, 1, dim) * scales.view(1, -1, 1)).reshape(*shape, -1) + assert xb.shape[-1] == dim * (max_deg - min_deg) + emb = torch.cat([xb, xb + math.pi / 2.0], axis=-1).sin() + return torch.cat([x, emb], dim=-1) + + +def encode_position(position): + return posenc_nerf(position, min_deg=0, max_deg=15) + + +def encode_direction(position, direction=None): + if direction is None: + return torch.zeros_like(posenc_nerf(position, min_deg=0, max_deg=8)) + else: + return posenc_nerf(direction, min_deg=0, max_deg=8) + + +def _sanitize_name(x: str) -> str: + return x.replace(".", "__") + + +def integrate_samples(volume_range, ts, density, channels): + r""" + Function integrating the model output. + + Args: + volume_range: Specifies the integral range [t0, t1] + ts: timesteps + density: torch.Tensor [batch_size, *shape, n_samples, 1] + channels: torch.Tensor [batch_size, *shape, n_samples, n_channels] + returns: + channels: integrated rgb output weights: torch.Tensor [batch_size, *shape, n_samples, 1] (density + *transmittance)[i] weight for each rgb output at [..., i, :]. transmittance: transmittance of this volume + ) + """ + + # 1. Calculate the weights + _, _, dt = volume_range.partition(ts) + ddensity = density * dt + + mass = torch.cumsum(ddensity, dim=-2) + transmittance = torch.exp(-mass[..., -1, :]) + + alphas = 1.0 - torch.exp(-ddensity) + Ts = torch.exp(torch.cat([torch.zeros_like(mass[..., :1, :]), -mass[..., :-1, :]], dim=-2)) + # This is the probability of light hitting and reflecting off of + # something at depth [..., i, :]. + weights = alphas * Ts + + # 2. Integrate channels + channels = torch.sum(channels * weights, dim=-2) + + return channels, weights, transmittance + + +def volume_query_points(volume, grid_size): + indices = torch.arange(grid_size**3, device=volume.bbox_min.device) + zs = indices % grid_size + ys = torch.div(indices, grid_size, rounding_mode="trunc") % grid_size + xs = torch.div(indices, grid_size**2, rounding_mode="trunc") % grid_size + combined = torch.stack([xs, ys, zs], dim=1) + return (combined.float() / (grid_size - 1)) * (volume.bbox_max - volume.bbox_min) + volume.bbox_min + + +def _convert_srgb_to_linear(u: torch.Tensor): + return torch.where(u <= 0.04045, u / 12.92, ((u + 0.055) / 1.055) ** 2.4) + + +def _create_flat_edge_indices( + flat_cube_indices: torch.Tensor, + grid_size: Tuple[int, int, int], +): + num_xs = (grid_size[0] - 1) * grid_size[1] * grid_size[2] + y_offset = num_xs + num_ys = grid_size[0] * (grid_size[1] - 1) * grid_size[2] + z_offset = num_xs + num_ys + return torch.stack( + [ + # Edges spanning x-axis. + flat_cube_indices[:, 0] * grid_size[1] * grid_size[2] + + flat_cube_indices[:, 1] * grid_size[2] + + flat_cube_indices[:, 2], + flat_cube_indices[:, 0] * grid_size[1] * grid_size[2] + + (flat_cube_indices[:, 1] + 1) * grid_size[2] + + flat_cube_indices[:, 2], + flat_cube_indices[:, 0] * grid_size[1] * grid_size[2] + + flat_cube_indices[:, 1] * grid_size[2] + + flat_cube_indices[:, 2] + + 1, + flat_cube_indices[:, 0] * grid_size[1] * grid_size[2] + + (flat_cube_indices[:, 1] + 1) * grid_size[2] + + flat_cube_indices[:, 2] + + 1, + # Edges spanning y-axis. + ( + y_offset + + flat_cube_indices[:, 0] * (grid_size[1] - 1) * grid_size[2] + + flat_cube_indices[:, 1] * grid_size[2] + + flat_cube_indices[:, 2] + ), + ( + y_offset + + (flat_cube_indices[:, 0] + 1) * (grid_size[1] - 1) * grid_size[2] + + flat_cube_indices[:, 1] * grid_size[2] + + flat_cube_indices[:, 2] + ), + ( + y_offset + + flat_cube_indices[:, 0] * (grid_size[1] - 1) * grid_size[2] + + flat_cube_indices[:, 1] * grid_size[2] + + flat_cube_indices[:, 2] + + 1 + ), + ( + y_offset + + (flat_cube_indices[:, 0] + 1) * (grid_size[1] - 1) * grid_size[2] + + flat_cube_indices[:, 1] * grid_size[2] + + flat_cube_indices[:, 2] + + 1 + ), + # Edges spanning z-axis. + ( + z_offset + + flat_cube_indices[:, 0] * grid_size[1] * (grid_size[2] - 1) + + flat_cube_indices[:, 1] * (grid_size[2] - 1) + + flat_cube_indices[:, 2] + ), + ( + z_offset + + (flat_cube_indices[:, 0] + 1) * grid_size[1] * (grid_size[2] - 1) + + flat_cube_indices[:, 1] * (grid_size[2] - 1) + + flat_cube_indices[:, 2] + ), + ( + z_offset + + flat_cube_indices[:, 0] * grid_size[1] * (grid_size[2] - 1) + + (flat_cube_indices[:, 1] + 1) * (grid_size[2] - 1) + + flat_cube_indices[:, 2] + ), + ( + z_offset + + (flat_cube_indices[:, 0] + 1) * grid_size[1] * (grid_size[2] - 1) + + (flat_cube_indices[:, 1] + 1) * (grid_size[2] - 1) + + flat_cube_indices[:, 2] + ), + ], + dim=-1, + ) + + +class VoidNeRFModel(nn.Module): + """ + Implements the default empty space model where all queries are rendered as background. + """ + + def __init__(self, background, channel_scale=255.0): + super().__init__() + background = nn.Parameter(torch.from_numpy(np.array(background)).to(dtype=torch.float32) / channel_scale) + + self.register_buffer("background", background) + + def forward(self, position): + background = self.background[None].to(position.device) + + shape = position.shape[:-1] + ones = [1] * (len(shape) - 1) + n_channels = background.shape[-1] + background = torch.broadcast_to(background.view(background.shape[0], *ones, n_channels), [*shape, n_channels]) + + return background + + +@dataclass +class VolumeRange: + t0: torch.Tensor + t1: torch.Tensor + intersected: torch.Tensor + + def __post_init__(self): + assert self.t0.shape == self.t1.shape == self.intersected.shape + + def partition(self, ts): + """ + Partitions t0 and t1 into n_samples intervals. + + Args: + ts: [batch_size, *shape, n_samples, 1] + + Return: + + lower: [batch_size, *shape, n_samples, 1] upper: [batch_size, *shape, n_samples, 1] delta: [batch_size, + *shape, n_samples, 1] + + where + ts \\in [lower, upper] deltas = upper - lower + """ + + mids = (ts[..., 1:, :] + ts[..., :-1, :]) * 0.5 + lower = torch.cat([self.t0[..., None, :], mids], dim=-2) + upper = torch.cat([mids, self.t1[..., None, :]], dim=-2) + delta = upper - lower + assert lower.shape == upper.shape == delta.shape == ts.shape + return lower, upper, delta + + +class BoundingBoxVolume(nn.Module): + """ + Axis-aligned bounding box defined by the two opposite corners. + """ + + def __init__( + self, + *, + bbox_min, + bbox_max, + min_dist: float = 0.0, + min_t_range: float = 1e-3, + ): + """ + Args: + bbox_min: the left/bottommost corner of the bounding box + bbox_max: the other corner of the bounding box + min_dist: all rays should start at least this distance away from the origin. + """ + super().__init__() + + self.min_dist = min_dist + self.min_t_range = min_t_range + + self.bbox_min = torch.tensor(bbox_min) + self.bbox_max = torch.tensor(bbox_max) + self.bbox = torch.stack([self.bbox_min, self.bbox_max]) + assert self.bbox.shape == (2, 3) + assert min_dist >= 0.0 + assert min_t_range > 0.0 + + def intersect( + self, + origin: torch.Tensor, + direction: torch.Tensor, + t0_lower: Optional[torch.Tensor] = None, + epsilon=1e-6, + ): + """ + Args: + origin: [batch_size, *shape, 3] + direction: [batch_size, *shape, 3] + t0_lower: Optional [batch_size, *shape, 1] lower bound of t0 when intersecting this volume. + params: Optional meta parameters in case Volume is parametric + epsilon: to stabilize calculations + + Return: + A tuple of (t0, t1, intersected) where each has a shape [batch_size, *shape, 1]. If a ray intersects with + the volume, `o + td` is in the volume for all t in [t0, t1]. If the volume is bounded, t1 is guaranteed to + be on the boundary of the volume. + """ + + batch_size, *shape, _ = origin.shape + ones = [1] * len(shape) + bbox = self.bbox.view(1, *ones, 2, 3).to(origin.device) + + def _safe_divide(a, b, epsilon=1e-6): + return a / torch.where(b < 0, b - epsilon, b + epsilon) + + ts = _safe_divide(bbox - origin[..., None, :], direction[..., None, :], epsilon=epsilon) + + # Cases to think about: + # + # 1. t1 <= t0: the ray does not pass through the AABB. + # 2. t0 < t1 <= 0: the ray intersects but the BB is behind the origin. + # 3. t0 <= 0 <= t1: the ray starts from inside the BB + # 4. 0 <= t0 < t1: the ray is not inside and intersects with the BB twice. + # + # 1 and 4 are clearly handled from t0 < t1 below. + # Making t0 at least min_dist (>= 0) takes care of 2 and 3. + t0 = ts.min(dim=-2).values.max(dim=-1, keepdim=True).values.clamp(self.min_dist) + t1 = ts.max(dim=-2).values.min(dim=-1, keepdim=True).values + assert t0.shape == t1.shape == (batch_size, *shape, 1) + if t0_lower is not None: + assert t0.shape == t0_lower.shape + t0 = torch.maximum(t0, t0_lower) + + intersected = t0 + self.min_t_range < t1 + t0 = torch.where(intersected, t0, torch.zeros_like(t0)) + t1 = torch.where(intersected, t1, torch.ones_like(t1)) + + return VolumeRange(t0=t0, t1=t1, intersected=intersected) + + +class StratifiedRaySampler(nn.Module): + """ + Instead of fixed intervals, a sample is drawn uniformly at random from each interval. + """ + + def __init__(self, depth_mode: str = "linear"): + """ + :param depth_mode: linear samples ts linearly in depth. harmonic ensures + closer points are sampled more densely. + """ + self.depth_mode = depth_mode + assert self.depth_mode in ("linear", "geometric", "harmonic") + + def sample( + self, + t0: torch.Tensor, + t1: torch.Tensor, + n_samples: int, + epsilon: float = 1e-3, + ) -> torch.Tensor: + """ + Args: + t0: start time has shape [batch_size, *shape, 1] + t1: finish time has shape [batch_size, *shape, 1] + n_samples: number of ts to sample + Return: + sampled ts of shape [batch_size, *shape, n_samples, 1] + """ + ones = [1] * (len(t0.shape) - 1) + ts = torch.linspace(0, 1, n_samples).view(*ones, n_samples).to(t0.dtype).to(t0.device) + + if self.depth_mode == "linear": + ts = t0 * (1.0 - ts) + t1 * ts + elif self.depth_mode == "geometric": + ts = (t0.clamp(epsilon).log() * (1.0 - ts) + t1.clamp(epsilon).log() * ts).exp() + elif self.depth_mode == "harmonic": + # The original NeRF recommends this interpolation scheme for + # spherical scenes, but there could be some weird edge cases when + # the observer crosses from the inner to outer volume. + ts = 1.0 / (1.0 / t0.clamp(epsilon) * (1.0 - ts) + 1.0 / t1.clamp(epsilon) * ts) + + mids = 0.5 * (ts[..., 1:] + ts[..., :-1]) + upper = torch.cat([mids, t1], dim=-1) + lower = torch.cat([t0, mids], dim=-1) + # yiyi notes: add a random seed here for testing, don't forget to remove + torch.manual_seed(0) + t_rand = torch.rand_like(ts) + + ts = lower + (upper - lower) * t_rand + return ts.unsqueeze(-1) + + +class ImportanceRaySampler(nn.Module): + """ + Given the initial estimate of densities, this samples more from regions/bins expected to have objects. + """ + + def __init__( + self, + volume_range: VolumeRange, + ts: torch.Tensor, + weights: torch.Tensor, + blur_pool: bool = False, + alpha: float = 1e-5, + ): + """ + Args: + volume_range: the range in which a ray intersects the given volume. + ts: earlier samples from the coarse rendering step + weights: discretized version of density * transmittance + blur_pool: if true, use 2-tap max + 2-tap blur filter from mip-NeRF. + alpha: small value to add to weights. + """ + self.volume_range = volume_range + self.ts = ts.clone().detach() + self.weights = weights.clone().detach() + self.blur_pool = blur_pool + self.alpha = alpha + + @torch.no_grad() + def sample(self, t0: torch.Tensor, t1: torch.Tensor, n_samples: int) -> torch.Tensor: + """ + Args: + t0: start time has shape [batch_size, *shape, 1] + t1: finish time has shape [batch_size, *shape, 1] + n_samples: number of ts to sample + Return: + sampled ts of shape [batch_size, *shape, n_samples, 1] + """ + lower, upper, _ = self.volume_range.partition(self.ts) + + batch_size, *shape, n_coarse_samples, _ = self.ts.shape + + weights = self.weights + if self.blur_pool: + padded = torch.cat([weights[..., :1, :], weights, weights[..., -1:, :]], dim=-2) + maxes = torch.maximum(padded[..., :-1, :], padded[..., 1:, :]) + weights = 0.5 * (maxes[..., :-1, :] + maxes[..., 1:, :]) + weights = weights + self.alpha + pmf = weights / weights.sum(dim=-2, keepdim=True) + inds = sample_pmf(pmf, n_samples) + assert inds.shape == (batch_size, *shape, n_samples, 1) + assert (inds >= 0).all() and (inds < n_coarse_samples).all() + + t_rand = torch.rand(inds.shape, device=inds.device) + lower_ = torch.gather(lower, -2, inds) + upper_ = torch.gather(upper, -2, inds) + + ts = lower_ + (upper_ - lower_) * t_rand + ts = torch.sort(ts, dim=-2).values + return ts + + +@dataclass +class MeshDecoderOutput(BaseOutput): + """ + A 3D triangle mesh with optional data at the vertices and faces. + + Args: + verts (`torch.Tensor` of shape `(N, 3)`): + array of vertext coordinates + faces (`torch.Tensor` of shape `(N, 3)`): + array of triangles, pointing to indices in verts. + vertext_channels (Dict): + vertext coordinates for each color channel + """ + + verts: torch.Tensor + faces: torch.Tensor + vertex_channels: Dict[str, torch.Tensor] + + +class MeshDecoder(nn.Module): + """ + Construct meshes from Signed distance functions (SDFs) using marching cubes method + """ + + def __init__(self): + super().__init__() + cases = torch.zeros(256, 5, 3, dtype=torch.long) + masks = torch.zeros(256, 5, dtype=torch.bool) + + self.register_buffer("cases", cases) + self.register_buffer("masks", masks) + + def forward(self, field: torch.Tensor, min_point: torch.Tensor, size: torch.Tensor): + """ + For a signed distance field, produce a mesh using marching cubes. + + :param field: a 3D tensor of field values, where negative values correspond + to the outside of the shape. The dimensions correspond to the x, y, and z directions, respectively. + :param min_point: a tensor of shape [3] containing the point corresponding + to (0, 0, 0) in the field. + :param size: a tensor of shape [3] containing the per-axis distance from the + (0, 0, 0) field corner and the (-1, -1, -1) field corner. + """ + assert len(field.shape) == 3, "input must be a 3D scalar field" + dev = field.device + + cases = self.cases.to(dev) + masks = self.masks.to(dev) + + min_point = min_point.to(dev) + size = size.to(dev) + + grid_size = field.shape + grid_size_tensor = torch.tensor(grid_size).to(size) + + # Create bitmasks between 0 and 255 (inclusive) indicating the state + # of the eight corners of each cube. + bitmasks = (field > 0).to(torch.uint8) + bitmasks = bitmasks[:-1, :, :] | (bitmasks[1:, :, :] << 1) + bitmasks = bitmasks[:, :-1, :] | (bitmasks[:, 1:, :] << 2) + bitmasks = bitmasks[:, :, :-1] | (bitmasks[:, :, 1:] << 4) + + # Compute corner coordinates across the entire grid. + corner_coords = torch.empty(*grid_size, 3, device=dev, dtype=field.dtype) + corner_coords[range(grid_size[0]), :, :, 0] = torch.arange(grid_size[0], device=dev, dtype=field.dtype)[ + :, None, None + ] + corner_coords[:, range(grid_size[1]), :, 1] = torch.arange(grid_size[1], device=dev, dtype=field.dtype)[ + :, None + ] + corner_coords[:, :, range(grid_size[2]), 2] = torch.arange(grid_size[2], device=dev, dtype=field.dtype) + + # Compute all vertices across all edges in the grid, even though we will + # throw some out later. We have (X-1)*Y*Z + X*(Y-1)*Z + X*Y*(Z-1) vertices. + # These are all midpoints, and don't account for interpolation (which is + # done later based on the used edge midpoints). + edge_midpoints = torch.cat( + [ + ((corner_coords[:-1] + corner_coords[1:]) / 2).reshape(-1, 3), + ((corner_coords[:, :-1] + corner_coords[:, 1:]) / 2).reshape(-1, 3), + ((corner_coords[:, :, :-1] + corner_coords[:, :, 1:]) / 2).reshape(-1, 3), + ], + dim=0, + ) + + # Create a flat array of [X, Y, Z] indices for each cube. + cube_indices = torch.zeros( + grid_size[0] - 1, grid_size[1] - 1, grid_size[2] - 1, 3, device=dev, dtype=torch.long + ) + cube_indices[range(grid_size[0] - 1), :, :, 0] = torch.arange(grid_size[0] - 1, device=dev)[:, None, None] + cube_indices[:, range(grid_size[1] - 1), :, 1] = torch.arange(grid_size[1] - 1, device=dev)[:, None] + cube_indices[:, :, range(grid_size[2] - 1), 2] = torch.arange(grid_size[2] - 1, device=dev) + flat_cube_indices = cube_indices.reshape(-1, 3) + + # Create a flat array mapping each cube to 12 global edge indices. + edge_indices = _create_flat_edge_indices(flat_cube_indices, grid_size) + + # Apply the LUT to figure out the triangles. + flat_bitmasks = bitmasks.reshape(-1).long() # must cast to long for indexing to believe this not a mask + local_tris = cases[flat_bitmasks] + local_masks = masks[flat_bitmasks] + # Compute the global edge indices for the triangles. + global_tris = torch.gather(edge_indices, 1, local_tris.reshape(local_tris.shape[0], -1)).reshape( + local_tris.shape + ) + # Select the used triangles for each cube. + selected_tris = global_tris.reshape(-1, 3)[local_masks.reshape(-1)] + + # Now we have a bunch of indices into the full list of possible vertices, + # but we want to reduce this list to only the used vertices. + used_vertex_indices = torch.unique(selected_tris.view(-1)) + used_edge_midpoints = edge_midpoints[used_vertex_indices] + old_index_to_new_index = torch.zeros(len(edge_midpoints), device=dev, dtype=torch.long) + old_index_to_new_index[used_vertex_indices] = torch.arange( + len(used_vertex_indices), device=dev, dtype=torch.long + ) + + # Rewrite the triangles to use the new indices + faces = torch.gather(old_index_to_new_index, 0, selected_tris.view(-1)).reshape(selected_tris.shape) + + # Compute the actual interpolated coordinates corresponding to edge midpoints. + v1 = torch.floor(used_edge_midpoints).to(torch.long) + v2 = torch.ceil(used_edge_midpoints).to(torch.long) + s1 = field[v1[:, 0], v1[:, 1], v1[:, 2]] + s2 = field[v2[:, 0], v2[:, 1], v2[:, 2]] + p1 = (v1.float() / (grid_size_tensor - 1)) * size + min_point + p2 = (v2.float() / (grid_size_tensor - 1)) * size + min_point + # The signs of s1 and s2 should be different. We want to find + # t such that t*s2 + (1-t)*s1 = 0. + t = (s1 / (s1 - s2))[:, None] + verts = t * p2 + (1 - t) * p1 + + return MeshDecoderOutput(verts=verts, faces=faces, vertex_channels=None) + + +@dataclass +class MLPNeRFModelOutput(BaseOutput): + density: torch.Tensor + signed_distance: torch.Tensor + channels: torch.Tensor + ts: torch.Tensor + + +class MLPNeRSTFModel(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + d_hidden: int = 256, + n_output: int = 12, + n_hidden_layers: int = 6, + act_fn: str = "swish", + insert_direction_at: int = 4, + ): + super().__init__() + + # Instantiate the MLP + + # Find out the dimension of encoded position and direction + dummy = torch.eye(1, 3) + d_posenc_pos = encode_position(position=dummy).shape[-1] + d_posenc_dir = encode_direction(position=dummy).shape[-1] + + mlp_widths = [d_hidden] * n_hidden_layers + input_widths = [d_posenc_pos] + mlp_widths + output_widths = mlp_widths + [n_output] + + if insert_direction_at is not None: + input_widths[insert_direction_at] += d_posenc_dir + + self.mlp = nn.ModuleList([nn.Linear(d_in, d_out) for d_in, d_out in zip(input_widths, output_widths)]) + + if act_fn == "swish": + # self.activation = swish + # yiyi testing: + self.activation = lambda x: F.silu(x) + else: + raise ValueError(f"Unsupported activation function {act_fn}") + + self.sdf_activation = torch.tanh + self.density_activation = torch.nn.functional.relu + self.channel_activation = torch.sigmoid + + def map_indices_to_keys(self, output): + h_map = { + "sdf": (0, 1), + "density_coarse": (1, 2), + "density_fine": (2, 3), + "stf": (3, 6), + "nerf_coarse": (6, 9), + "nerf_fine": (9, 12), + } + + mapped_output = {k: output[..., start:end] for k, (start, end) in h_map.items()} + + return mapped_output + + def forward(self, *, position, direction, ts, nerf_level="coarse", rendering_mode="nerf"): + h = encode_position(position) + + h_preact = h + h_directionless = None + for i, layer in enumerate(self.mlp): + if i == self.config.insert_direction_at: # 4 in the config + h_directionless = h_preact + h_direction = encode_direction(position, direction=direction) + h = torch.cat([h, h_direction], dim=-1) + + h = layer(h) + + h_preact = h + + if i < len(self.mlp) - 1: + h = self.activation(h) + + h_final = h + if h_directionless is None: + h_directionless = h_preact + + activation = self.map_indices_to_keys(h_final) + + if nerf_level == "coarse": + h_density = activation["density_coarse"] + else: + h_density = activation["density_fine"] + + if rendering_mode == "nerf": + if nerf_level == "coarse": + h_channels = activation["nerf_coarse"] + else: + h_channels = activation["nerf_fine"] + + elif rendering_mode == "stf": + h_channels = activation["stf"] + + density = self.density_activation(h_density) + signed_distance = self.sdf_activation(activation["sdf"]) + channels = self.channel_activation(h_channels) + + # yiyi notes: I think signed_distance is not used + return MLPNeRFModelOutput(density=density, signed_distance=signed_distance, channels=channels, ts=ts) + + +class ChannelsProj(nn.Module): + def __init__( + self, + *, + vectors: int, + channels: int, + d_latent: int, + ): + super().__init__() + self.proj = nn.Linear(d_latent, vectors * channels) + self.norm = nn.LayerNorm(channels) + self.d_latent = d_latent + self.vectors = vectors + self.channels = channels + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x_bvd = x + w_vcd = self.proj.weight.view(self.vectors, self.channels, self.d_latent) + b_vc = self.proj.bias.view(1, self.vectors, self.channels) + h = torch.einsum("bvd,vcd->bvc", x_bvd, w_vcd) + h = self.norm(h) + + h = h + b_vc + return h + + +class ShapEParamsProjModel(ModelMixin, ConfigMixin): + """ + project the latent representation of a 3D asset to obtain weights of a multi-layer perceptron (MLP). + + For more details, see the original paper: + """ + + @register_to_config + def __init__( + self, + *, + param_names: Tuple[str] = ( + "nerstf.mlp.0.weight", + "nerstf.mlp.1.weight", + "nerstf.mlp.2.weight", + "nerstf.mlp.3.weight", + ), + param_shapes: Tuple[Tuple[int]] = ( + (256, 93), + (256, 256), + (256, 256), + (256, 256), + ), + d_latent: int = 1024, + ): + super().__init__() + + # check inputs + if len(param_names) != len(param_shapes): + raise ValueError("Must provide same number of `param_names` as `param_shapes`") + self.projections = nn.ModuleDict({}) + for k, (vectors, channels) in zip(param_names, param_shapes): + self.projections[_sanitize_name(k)] = ChannelsProj( + vectors=vectors, + channels=channels, + d_latent=d_latent, + ) + + def forward(self, x: torch.Tensor): + out = {} + start = 0 + for k, shape in zip(self.config.param_names, self.config.param_shapes): + vectors, _ = shape + end = start + vectors + x_bvd = x[:, start:end] + out[k] = self.projections[_sanitize_name(k)](x_bvd).reshape(len(x), *shape) + start = end + return out + + +class ShapERenderer(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + *, + param_names: Tuple[str] = ( + "nerstf.mlp.0.weight", + "nerstf.mlp.1.weight", + "nerstf.mlp.2.weight", + "nerstf.mlp.3.weight", + ), + param_shapes: Tuple[Tuple[int]] = ( + (256, 93), + (256, 256), + (256, 256), + (256, 256), + ), + d_latent: int = 1024, + d_hidden: int = 256, + n_output: int = 12, + n_hidden_layers: int = 6, + act_fn: str = "swish", + insert_direction_at: int = 4, + background: Tuple[float] = ( + 255.0, + 255.0, + 255.0, + ), + ): + super().__init__() + + self.params_proj = ShapEParamsProjModel( + param_names=param_names, + param_shapes=param_shapes, + d_latent=d_latent, + ) + self.mlp = MLPNeRSTFModel(d_hidden, n_output, n_hidden_layers, act_fn, insert_direction_at) + self.void = VoidNeRFModel(background=background, channel_scale=255.0) + self.volume = BoundingBoxVolume(bbox_max=[1.0, 1.0, 1.0], bbox_min=[-1.0, -1.0, -1.0]) + self.mesh_decoder = MeshDecoder() + + @torch.no_grad() + def render_rays(self, rays, sampler, n_samples, prev_model_out=None, render_with_direction=False): + """ + Perform volumetric rendering over a partition of possible t's in the union of rendering volumes (written below + with some abuse of notations) + + C(r) := sum( + transmittance(t[i]) * integrate( + lambda t: density(t) * channels(t) * transmittance(t), [t[i], t[i + 1]], + ) for i in range(len(parts)) + ) + transmittance(t[-1]) * void_model(t[-1]).channels + + where + + 1) transmittance(s) := exp(-integrate(density, [t[0], s])) calculates the probability of light passing through + the volume specified by [t[0], s]. (transmittance of 1 means light can pass freely) 2) density and channels are + obtained by evaluating the appropriate part.model at time t. 3) [t[i], t[i + 1]] is defined as the range of t + where the ray intersects (parts[i].volume \\ union(part.volume for part in parts[:i])) at the surface of the + shell (if bounded). If the ray does not intersect, the integral over this segment is evaluated as 0 and + transmittance(t[i + 1]) := transmittance(t[i]). 4) The last term is integration to infinity (e.g. [t[-1], + math.inf]) that is evaluated by the void_model (i.e. we consider this space to be empty). + + args: + rays: [batch_size x ... x 2 x 3] origin and direction. sampler: disjoint volume integrals. n_samples: + number of ts to sample. prev_model_outputs: model outputs from the previous rendering step, including + + :return: A tuple of + - `channels` + - A importance samplers for additional fine-grained rendering + - raw model output + """ + origin, direction = rays[..., 0, :], rays[..., 1, :] + + # Integrate over [t[i], t[i + 1]] + + # 1 Intersect the rays with the current volume and sample ts to integrate along. + vrange = self.volume.intersect(origin, direction, t0_lower=None) + ts = sampler.sample(vrange.t0, vrange.t1, n_samples) + ts = ts.to(rays.dtype) + + if prev_model_out is not None: + # Append the previous ts now before fprop because previous + # rendering used a different model and we can't reuse the output. + ts = torch.sort(torch.cat([ts, prev_model_out.ts], dim=-2), dim=-2).values + + batch_size, *_shape, _t0_dim = vrange.t0.shape + _, *ts_shape, _ts_dim = ts.shape + + # 2. Get the points along the ray and query the model + directions = torch.broadcast_to(direction.unsqueeze(-2), [batch_size, *ts_shape, 3]) + positions = origin.unsqueeze(-2) + ts * directions + + directions = directions.to(self.mlp.dtype) + positions = positions.to(self.mlp.dtype) + + optional_directions = directions if render_with_direction else None + + model_out = self.mlp( + position=positions, + direction=optional_directions, + ts=ts, + nerf_level="coarse" if prev_model_out is None else "fine", + ) + + # 3. Integrate the model results + channels, weights, transmittance = integrate_samples( + vrange, model_out.ts, model_out.density, model_out.channels + ) + + # 4. Clean up results that do not intersect with the volume. + transmittance = torch.where(vrange.intersected, transmittance, torch.ones_like(transmittance)) + channels = torch.where(vrange.intersected, channels, torch.zeros_like(channels)) + # 5. integration to infinity (e.g. [t[-1], math.inf]) that is evaluated by the void_model (i.e. we consider this space to be empty). + channels = channels + transmittance * self.void(origin) + + weighted_sampler = ImportanceRaySampler(vrange, ts=model_out.ts, weights=weights) + + return channels, weighted_sampler, model_out + + @torch.no_grad() + def decode_to_image( + self, + latents, + device, + size: int = 64, + ray_batch_size: int = 4096, + n_coarse_samples=64, + n_fine_samples=128, + ): + # project the parameters from the generated latents + projected_params = self.params_proj(latents) + + # update the mlp layers of the renderer + for name, param in self.mlp.state_dict().items(): + if f"nerstf.{name}" in projected_params.keys(): + param.copy_(projected_params[f"nerstf.{name}"].squeeze(0)) + + # create cameras object + camera = create_pan_cameras(size) + rays = camera.camera_rays + rays = rays.to(device) + n_batches = rays.shape[1] // ray_batch_size + + coarse_sampler = StratifiedRaySampler() + + images = [] + + for idx in range(n_batches): + rays_batch = rays[:, idx * ray_batch_size : (idx + 1) * ray_batch_size] + + # render rays with coarse, stratified samples. + _, fine_sampler, coarse_model_out = self.render_rays(rays_batch, coarse_sampler, n_coarse_samples) + # Then, render with additional importance-weighted ray samples. + channels, _, _ = self.render_rays( + rays_batch, fine_sampler, n_fine_samples, prev_model_out=coarse_model_out + ) + + images.append(channels) + + images = torch.cat(images, dim=1) + images = images.view(*camera.shape, camera.height, camera.width, -1).squeeze(0) + + return images + + @torch.no_grad() + def decode_to_mesh( + self, + latents, + device, + grid_size: int = 128, + query_batch_size: int = 4096, + texture_channels: Tuple = ("R", "G", "B"), + ): + # 1. project the parameters from the generated latents + projected_params = self.params_proj(latents) + + # 2. update the mlp layers of the renderer + for name, param in self.mlp.state_dict().items(): + if f"nerstf.{name}" in projected_params.keys(): + param.copy_(projected_params[f"nerstf.{name}"].squeeze(0)) + + # 3. decoding with STF rendering + # 3.1 query the SDF values at vertices along a regular 128**3 grid + + query_points = volume_query_points(self.volume, grid_size) + query_positions = query_points[None].repeat(1, 1, 1).to(device=device, dtype=self.mlp.dtype) + + fields = [] + + for idx in range(0, query_positions.shape[1], query_batch_size): + query_batch = query_positions[:, idx : idx + query_batch_size] + + model_out = self.mlp( + position=query_batch, direction=None, ts=None, nerf_level="fine", rendering_mode="stf" + ) + fields.append(model_out.signed_distance) + + # predicted SDF values + fields = torch.cat(fields, dim=1) + fields = fields.float() + + assert ( + len(fields.shape) == 3 and fields.shape[-1] == 1 + ), f"expected [meta_batch x inner_batch] SDF results, but got {fields.shape}" + + fields = fields.reshape(1, *([grid_size] * 3)) + + # create grid 128 x 128 x 128 + # - force a negative border around the SDFs to close off all the models. + full_grid = torch.zeros( + 1, + grid_size + 2, + grid_size + 2, + grid_size + 2, + device=fields.device, + dtype=fields.dtype, + ) + full_grid.fill_(-1.0) + full_grid[:, 1:-1, 1:-1, 1:-1] = fields + fields = full_grid + + # apply a differentiable implementation of Marching Cubes to construct meshs + raw_meshes = [] + mesh_mask = [] + + for field in fields: + raw_mesh = self.mesh_decoder(field, self.volume.bbox_min, self.volume.bbox_max - self.volume.bbox_min) + mesh_mask.append(True) + raw_meshes.append(raw_mesh) + + mesh_mask = torch.tensor(mesh_mask, device=fields.device) + max_vertices = max(len(m.verts) for m in raw_meshes) + + # 3.2. query the texture color head at each vertex of the resulting mesh. + texture_query_positions = torch.stack( + [m.verts[torch.arange(0, max_vertices) % len(m.verts)] for m in raw_meshes], + dim=0, + ) + texture_query_positions = texture_query_positions.to(device=device, dtype=self.mlp.dtype) + + textures = [] + + for idx in range(0, texture_query_positions.shape[1], query_batch_size): + query_batch = texture_query_positions[:, idx : idx + query_batch_size] + + texture_model_out = self.mlp( + position=query_batch, direction=None, ts=None, nerf_level="fine", rendering_mode="stf" + ) + textures.append(texture_model_out.channels) + + # predict texture color + textures = torch.cat(textures, dim=1) + + textures = _convert_srgb_to_linear(textures) + textures = textures.float() + + # 3.3 augument the mesh with texture data + assert len(textures.shape) == 3 and textures.shape[-1] == len( + texture_channels + ), f"expected [meta_batch x inner_batch x texture_channels] field results, but got {textures.shape}" + + for m, texture in zip(raw_meshes, textures): + texture = texture[: len(m.verts)] + m.vertex_channels = dict(zip(texture_channels, texture.unbind(-1))) + + return raw_meshes[0] diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/__init__.py new file mode 100755 index 0000000..5270cb9 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/__init__.py @@ -0,0 +1,50 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_stable_cascade"] = ["StableCascadeDecoderPipeline"] + _import_structure["pipeline_stable_cascade_combined"] = ["StableCascadeCombinedPipeline"] + _import_structure["pipeline_stable_cascade_prior"] = ["StableCascadePriorPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 + else: + from .pipeline_stable_cascade import StableCascadeDecoderPipeline + from .pipeline_stable_cascade_combined import StableCascadeCombinedPipeline + from .pipeline_stable_cascade_prior import StableCascadePriorPipeline +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py new file mode 100755 index 0000000..a05fb90 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade.py @@ -0,0 +1,482 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Dict, List, Optional, Union + +import torch +from transformers import CLIPTextModel, CLIPTokenizer + +from ...models import StableCascadeUNet +from ...schedulers import DDPMWuerstchenScheduler +from ...utils import is_torch_version, logging, replace_example_docstring +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from ..wuerstchen.modeling_paella_vq_model import PaellaVQModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableCascadePriorPipeline, StableCascadeDecoderPipeline + + >>> prior_pipe = StableCascadePriorPipeline.from_pretrained( + ... "stabilityai/stable-cascade-prior", torch_dtype=torch.bfloat16 + ... ).to("cuda") + >>> gen_pipe = StableCascadeDecoderPipeline.from_pretrain( + ... "stabilityai/stable-cascade", torch_dtype=torch.float16 + ... ).to("cuda") + + >>> prompt = "an image of a shiba inu, donning a spacesuit and helmet" + >>> prior_output = pipe(prompt) + >>> images = gen_pipe(prior_output.image_embeddings, prompt=prompt) + ``` +""" + + +class StableCascadeDecoderPipeline(DiffusionPipeline): + """ + Pipeline for generating images from the Stable Cascade model. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + tokenizer (`CLIPTokenizer`): + The CLIP tokenizer. + text_encoder (`CLIPTextModel`): + The CLIP text encoder. + decoder ([`StableCascadeUNet`]): + The Stable Cascade decoder unet. + vqgan ([`PaellaVQModel`]): + The VQGAN model. + scheduler ([`DDPMWuerstchenScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + latent_dim_scale (float, `optional`, defaults to 10.67): + Multiplier to determine the VQ latent space size from the image embeddings. If the image embeddings are + height=24 and width=24, the VQ latent shape needs to be height=int(24*10.67)=256 and + width=int(24*10.67)=256 in order to match the training conditions. + """ + + unet_name = "decoder" + text_encoder_name = "text_encoder" + model_cpu_offload_seq = "text_encoder->decoder->vqgan" + _callback_tensor_inputs = [ + "latents", + "prompt_embeds_pooled", + "negative_prompt_embeds", + "image_embeddings", + ] + + def __init__( + self, + decoder: StableCascadeUNet, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModel, + scheduler: DDPMWuerstchenScheduler, + vqgan: PaellaVQModel, + latent_dim_scale: float = 10.67, + ) -> None: + super().__init__() + self.register_modules( + decoder=decoder, + tokenizer=tokenizer, + text_encoder=text_encoder, + scheduler=scheduler, + vqgan=vqgan, + ) + self.register_to_config(latent_dim_scale=latent_dim_scale) + + def prepare_latents(self, image_embeddings, num_images_per_prompt, dtype, device, generator, latents, scheduler): + batch_size, channels, height, width = image_embeddings.shape + latents_shape = ( + batch_size * num_images_per_prompt, + 4, + int(height * self.config.latent_dim_scale), + int(width * self.config.latent_dim_scale), + ) + + if latents is None: + latents = randn_tensor(latents_shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != latents_shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def encode_prompt( + self, + device, + batch_size, + num_images_per_prompt, + do_classifier_free_guidance, + prompt=None, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + prompt_embeds_pooled: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds_pooled: Optional[torch.FloatTensor] = None, + ): + if prompt_embeds is None: + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + attention_mask = text_inputs.attention_mask + + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + text_input_ids = text_input_ids[:, : self.tokenizer.model_max_length] + attention_mask = attention_mask[:, : self.tokenizer.model_max_length] + + text_encoder_output = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask.to(device), output_hidden_states=True + ) + prompt_embeds = text_encoder_output.hidden_states[-1] + if prompt_embeds_pooled is None: + prompt_embeds_pooled = text_encoder_output.text_embeds.unsqueeze(1) + + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device) + prompt_embeds_pooled = prompt_embeds_pooled.to(dtype=self.text_encoder.dtype, device=device) + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + prompt_embeds_pooled = prompt_embeds_pooled.repeat_interleave(num_images_per_prompt, dim=0) + + if negative_prompt_embeds is None and do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + negative_prompt_embeds_text_encoder_output = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=uncond_input.attention_mask.to(device), + output_hidden_states=True, + ) + + negative_prompt_embeds = negative_prompt_embeds_text_encoder_output.hidden_states[-1] + negative_prompt_embeds_pooled = negative_prompt_embeds_text_encoder_output.text_embeds.unsqueeze(1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device) + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + seq_len = negative_prompt_embeds_pooled.shape[1] + negative_prompt_embeds_pooled = negative_prompt_embeds_pooled.to( + dtype=self.text_encoder.dtype, device=device + ) + negative_prompt_embeds_pooled = negative_prompt_embeds_pooled.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds_pooled = negative_prompt_embeds_pooled.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + # done duplicates + + return prompt_embeds, prompt_embeds_pooled, negative_prompt_embeds, negative_prompt_embeds_pooled + + def check_inputs( + self, + prompt, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image_embeddings: Union[torch.FloatTensor, List[torch.FloatTensor]], + prompt: Union[str, List[str]] = None, + num_inference_steps: int = 10, + guidance_scale: float = 0.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + prompt_embeds_pooled: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds_pooled: Optional[torch.FloatTensor] = None, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + image_embedding (`torch.FloatTensor` or `List[torch.FloatTensor]`): + Image Embeddings either extracted from an image or generated by a Prior Model. + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + num_inference_steps (`int`, *optional*, defaults to 12): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 0.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `decoder_guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting + `decoder_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely + linked to the text `prompt`, usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `decoder_guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + prompt_embeds_pooled (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + negative_prompt_embeds_pooled (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds_pooled will be generated from `negative_prompt` input + argument. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` [`~pipelines.ImagePipelineOutput`] if `return_dict` is True, + otherwise a `tuple`. When returning a tuple, the first element is a list with the generated image + embeddings. + """ + + # 0. Define commonly used variables + device = self._execution_device + dtype = self.decoder.dtype + self._guidance_scale = guidance_scale + if is_torch_version("<", "2.2.0") and dtype == torch.bfloat16: + raise ValueError("`StableCascadeDecoderPipeline` requires torch>=2.2.0 when using `torch.bfloat16` dtype.") + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + ) + if isinstance(image_embeddings, list): + image_embeddings = torch.cat(image_embeddings, dim=0) + batch_size = image_embeddings.shape[0] + + # 2. Encode caption + if prompt_embeds is None and negative_prompt_embeds is None: + _, prompt_embeds_pooled, _, negative_prompt_embeds_pooled = self.encode_prompt( + prompt=prompt, + device=device, + batch_size=batch_size, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + prompt_embeds_pooled=prompt_embeds_pooled, + negative_prompt_embeds=negative_prompt_embeds, + negative_prompt_embeds_pooled=negative_prompt_embeds_pooled, + ) + + # The pooled embeds from the prior are pooled again before being passed to the decoder + prompt_embeds_pooled = ( + torch.cat([prompt_embeds_pooled, negative_prompt_embeds_pooled]) + if self.do_classifier_free_guidance + else prompt_embeds_pooled + ) + effnet = ( + torch.cat([image_embeddings, torch.zeros_like(image_embeddings)]) + if self.do_classifier_free_guidance + else image_embeddings + ) + + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latents + latents = self.prepare_latents( + image_embeddings, num_images_per_prompt, dtype, device, generator, latents, self.scheduler + ) + + # 6. Run denoising loop + self._num_timesteps = len(timesteps[:-1]) + for i, t in enumerate(self.progress_bar(timesteps[:-1])): + timestep_ratio = t.expand(latents.size(0)).to(dtype) + + # 7. Denoise latents + predicted_latents = self.decoder( + sample=torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents, + timestep_ratio=torch.cat([timestep_ratio] * 2) if self.do_classifier_free_guidance else timestep_ratio, + clip_text_pooled=prompt_embeds_pooled, + effnet=effnet, + return_dict=False, + )[0] + + # 8. Check for classifier free guidance and apply it + if self.do_classifier_free_guidance: + predicted_latents_text, predicted_latents_uncond = predicted_latents.chunk(2) + predicted_latents = torch.lerp(predicted_latents_uncond, predicted_latents_text, self.guidance_scale) + + # 9. Renoise latents to next timestep + latents = self.scheduler.step( + model_output=predicted_latents, + timestep=timestep_ratio, + sample=latents, + generator=generator, + ).prev_sample + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if output_type not in ["pt", "np", "pil", "latent"]: + raise ValueError( + f"Only the output types `pt`, `np`, `pil` and `latent` are supported not output_type={output_type}" + ) + + if not output_type == "latent": + # 10. Scale and decode the image latents with vq-vae + latents = self.vqgan.config.scale_factor * latents + images = self.vqgan.decode(latents).sample.clamp(0, 1) + if output_type == "np": + images = images.permute(0, 2, 3, 1).cpu().float().numpy() # float() as bfloat16-> numpy doesnt work + elif output_type == "pil": + images = images.permute(0, 2, 3, 1).cpu().float().numpy() # float() as bfloat16-> numpy doesnt work + images = self.numpy_to_pil(images) + else: + images = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return images + return ImagePipelineOutput(images) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py new file mode 100755 index 0000000..07afded --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_combined.py @@ -0,0 +1,311 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Callable, Dict, List, Optional, Union + +import PIL +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...models import StableCascadeUNet +from ...schedulers import DDPMWuerstchenScheduler +from ...utils import is_torch_version, replace_example_docstring +from ..pipeline_utils import DiffusionPipeline +from ..wuerstchen.modeling_paella_vq_model import PaellaVQModel +from .pipeline_stable_cascade import StableCascadeDecoderPipeline +from .pipeline_stable_cascade_prior import StableCascadePriorPipeline + + +TEXT2IMAGE_EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableCascadeCombinedPipeline + >>> pipe = StableCascadeCombinedPipeline.from_pretrained("stabilityai/stable-cascade", variant="bf16", torch_dtype=torch.bfloat16) + >>> pipe.enable_model_cpu_offload() + >>> prompt = "an image of a shiba inu, donning a spacesuit and helmet" + >>> images = pipe(prompt=prompt) + ``` +""" + + +class StableCascadeCombinedPipeline(DiffusionPipeline): + """ + Combined Pipeline for text-to-image generation using Stable Cascade. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + tokenizer (`CLIPTokenizer`): + The decoder tokenizer to be used for text inputs. + text_encoder (`CLIPTextModel`): + The decoder text encoder to be used for text inputs. + decoder (`StableCascadeUNet`): + The decoder model to be used for decoder image generation pipeline. + scheduler (`DDPMWuerstchenScheduler`): + The scheduler to be used for decoder image generation pipeline. + vqgan (`PaellaVQModel`): + The VQGAN model to be used for decoder image generation pipeline. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `image_encoder`. + image_encoder ([`CLIPVisionModelWithProjection`]): + Frozen CLIP image-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + prior_prior (`StableCascadeUNet`): + The prior model to be used for prior pipeline. + prior_scheduler (`DDPMWuerstchenScheduler`): + The scheduler to be used for prior pipeline. + """ + + _load_connected_pipes = True + + def __init__( + self, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModel, + decoder: StableCascadeUNet, + scheduler: DDPMWuerstchenScheduler, + vqgan: PaellaVQModel, + prior_prior: StableCascadeUNet, + prior_text_encoder: CLIPTextModel, + prior_tokenizer: CLIPTokenizer, + prior_scheduler: DDPMWuerstchenScheduler, + prior_feature_extractor: Optional[CLIPImageProcessor] = None, + prior_image_encoder: Optional[CLIPVisionModelWithProjection] = None, + ): + super().__init__() + + self.register_modules( + text_encoder=text_encoder, + tokenizer=tokenizer, + decoder=decoder, + scheduler=scheduler, + vqgan=vqgan, + prior_text_encoder=prior_text_encoder, + prior_tokenizer=prior_tokenizer, + prior_prior=prior_prior, + prior_scheduler=prior_scheduler, + prior_feature_extractor=prior_feature_extractor, + prior_image_encoder=prior_image_encoder, + ) + self.prior_pipe = StableCascadePriorPipeline( + prior=prior_prior, + text_encoder=prior_text_encoder, + tokenizer=prior_tokenizer, + scheduler=prior_scheduler, + image_encoder=prior_image_encoder, + feature_extractor=prior_feature_extractor, + ) + self.decoder_pipe = StableCascadeDecoderPipeline( + text_encoder=text_encoder, + tokenizer=tokenizer, + decoder=decoder, + scheduler=scheduler, + vqgan=vqgan, + ) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) + + def enable_model_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared + to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` + method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with + `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`. + """ + self.prior_pipe.enable_model_cpu_offload(gpu_id=gpu_id) + self.decoder_pipe.enable_model_cpu_offload(gpu_id=gpu_id) + + def enable_sequential_cpu_offload(self, gpu_id=0): + r""" + Offloads all models (`unet`, `text_encoder`, `vae`, and `safety checker` state dicts) to CPU using 🤗 + Accelerate, significantly reducing memory usage. Models are moved to a `torch.device('meta')` and loaded on a + GPU only when their specific submodule's `forward` method is called. Offloading happens on a submodule basis. + Memory savings are higher than using `enable_model_cpu_offload`, but performance is lower. + """ + self.prior_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + self.decoder_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + + def progress_bar(self, iterable=None, total=None): + self.prior_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.progress_bar(iterable=iterable, total=total) + + def set_progress_bar_config(self, **kwargs): + self.prior_pipe.set_progress_bar_config(**kwargs) + self.decoder_pipe.set_progress_bar_config(**kwargs) + + @torch.no_grad() + @replace_example_docstring(TEXT2IMAGE_EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + images: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]] = None, + height: int = 512, + width: int = 512, + prior_num_inference_steps: int = 60, + prior_guidance_scale: float = 4.0, + num_inference_steps: int = 12, + decoder_guidance_scale: float = 0.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + prompt_embeds_pooled: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds_pooled: Optional[torch.FloatTensor] = None, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + prior_callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + prior_callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation for the prior and decoder. + images (`torch.Tensor`, `PIL.Image.Image`, `List[torch.Tensor]`, `List[PIL.Image.Image]`, *optional*): + The images to guide the image generation for the prior. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings for the prior. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, text embeddings will be generated from `prompt` input argument. + prompt_embeds_pooled (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings for the prior. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings for the prior. Can be used to easily tweak text inputs, *e.g.* + prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` + input argument. + negative_prompt_embeds_pooled (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings for the prior. Can be used to easily tweak text inputs, *e.g.* + prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` + input argument. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + prior_guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `prior_guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting + `prior_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked + to the text `prompt`, usually at the expense of lower image quality. + prior_num_inference_steps (`Union[int, Dict[float, int]]`, *optional*, defaults to 60): + The number of prior denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. For more specific timestep spacing, you can pass customized + `prior_timesteps` + num_inference_steps (`int`, *optional*, defaults to 12): + The number of decoder denoising steps. More denoising steps usually lead to a higher quality image at + the expense of slower inference. For more specific timestep spacing, you can pass customized + `timesteps` + decoder_guidance_scale (`float`, *optional*, defaults to 0.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + prior_callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `prior_callback_on_step_end(self: DiffusionPipeline, step: int, timestep: + int, callback_kwargs: Dict)`. + prior_callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `prior_callback_on_step_end` function. The tensors specified in the + list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in + the `._callback_tensor_inputs` attribute of your pipeine class. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeine class. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` [`~pipelines.ImagePipelineOutput`] if `return_dict` is True, + otherwise a `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + dtype = self.decoder_pipe.decoder.dtype + if is_torch_version("<", "2.2.0") and dtype == torch.bfloat16: + raise ValueError( + "`StableCascadeCombinedPipeline` requires torch>=2.2.0 when using `torch.bfloat16` dtype." + ) + + prior_outputs = self.prior_pipe( + prompt=prompt if prompt_embeds is None else None, + images=images, + height=height, + width=width, + num_inference_steps=prior_num_inference_steps, + guidance_scale=prior_guidance_scale, + negative_prompt=negative_prompt if negative_prompt_embeds is None else None, + prompt_embeds=prompt_embeds, + prompt_embeds_pooled=prompt_embeds_pooled, + negative_prompt_embeds=negative_prompt_embeds, + negative_prompt_embeds_pooled=negative_prompt_embeds_pooled, + num_images_per_prompt=num_images_per_prompt, + generator=generator, + latents=latents, + output_type="pt", + return_dict=True, + callback_on_step_end=prior_callback_on_step_end, + callback_on_step_end_tensor_inputs=prior_callback_on_step_end_tensor_inputs, + ) + image_embeddings = prior_outputs.image_embeddings + prompt_embeds = prior_outputs.get("prompt_embeds", None) + prompt_embeds_pooled = prior_outputs.get("prompt_embeds_pooled", None) + negative_prompt_embeds = prior_outputs.get("negative_prompt_embeds", None) + negative_prompt_embeds_pooled = prior_outputs.get("negative_prompt_embeds_pooled", None) + + outputs = self.decoder_pipe( + image_embeddings=image_embeddings, + prompt=prompt if prompt_embeds is None else None, + num_inference_steps=num_inference_steps, + guidance_scale=decoder_guidance_scale, + negative_prompt=negative_prompt if negative_prompt_embeds is None else None, + prompt_embeds=prompt_embeds, + prompt_embeds_pooled=prompt_embeds_pooled, + negative_prompt_embeds=negative_prompt_embeds, + negative_prompt_embeds_pooled=negative_prompt_embeds_pooled, + generator=generator, + output_type=output_type, + return_dict=return_dict, + callback_on_step_end=callback_on_step_end, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + ) + + return outputs diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py new file mode 100755 index 0000000..24ccc4b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_cascade/pipeline_stable_cascade_prior.py @@ -0,0 +1,638 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from math import ceil +from typing import Callable, Dict, List, Optional, Union + +import numpy as np +import PIL +import torch +from transformers import CLIPImageProcessor, CLIPTextModelWithProjection, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...models import StableCascadeUNet +from ...schedulers import DDPMWuerstchenScheduler +from ...utils import BaseOutput, logging, replace_example_docstring +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +DEFAULT_STAGE_C_TIMESTEPS = list(np.linspace(1.0, 2 / 3, 20)) + list(np.linspace(2 / 3, 0.0, 11))[1:] + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableCascadePriorPipeline + + >>> prior_pipe = StableCascadePriorPipeline.from_pretrained( + ... "stabilityai/stable-cascade-prior", torch_dtype=torch.bfloat16 + ... ).to("cuda") + + >>> prompt = "an image of a shiba inu, donning a spacesuit and helmet" + >>> prior_output = pipe(prompt) + ``` +""" + + +@dataclass +class StableCascadePriorPipelineOutput(BaseOutput): + """ + Output class for WuerstchenPriorPipeline. + + Args: + image_embeddings (`torch.FloatTensor` or `np.ndarray`) + Prior image embeddings for text prompt + prompt_embeds (`torch.FloatTensor`): + Text embeddings for the prompt. + negative_prompt_embeds (`torch.FloatTensor`): + Text embeddings for the negative prompt. + """ + + image_embeddings: Union[torch.FloatTensor, np.ndarray] + prompt_embeds: Union[torch.FloatTensor, np.ndarray] + prompt_embeds_pooled: Union[torch.FloatTensor, np.ndarray] + negative_prompt_embeds: Union[torch.FloatTensor, np.ndarray] + negative_prompt_embeds_pooled: Union[torch.FloatTensor, np.ndarray] + + +class StableCascadePriorPipeline(DiffusionPipeline): + """ + Pipeline for generating image prior for Stable Cascade. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + prior ([`StableCascadeUNet`]): + The Stable Cascade prior to approximate the image embedding from the text and/or image embedding. + text_encoder ([`CLIPTextModelWithProjection`]): + Frozen text-encoder ([laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k)). + feature_extractor ([`~transformers.CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `image_encoder`. + image_encoder ([`CLIPVisionModelWithProjection`]): + Frozen CLIP image-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + scheduler ([`DDPMWuerstchenScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + resolution_multiple ('float', *optional*, defaults to 42.67): + Default resolution for multiple images generated. + """ + + unet_name = "prior" + text_encoder_name = "text_encoder" + model_cpu_offload_seq = "image_encoder->text_encoder->prior" + _optional_components = ["image_encoder", "feature_extractor"] + _callback_tensor_inputs = ["latents", "text_encoder_hidden_states", "negative_prompt_embeds"] + + def __init__( + self, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModelWithProjection, + prior: StableCascadeUNet, + scheduler: DDPMWuerstchenScheduler, + resolution_multiple: float = 42.67, + feature_extractor: Optional[CLIPImageProcessor] = None, + image_encoder: Optional[CLIPVisionModelWithProjection] = None, + ) -> None: + super().__init__() + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + prior=prior, + scheduler=scheduler, + ) + self.register_to_config(resolution_multiple=resolution_multiple) + + def prepare_latents( + self, batch_size, height, width, num_images_per_prompt, dtype, device, generator, latents, scheduler + ): + latent_shape = ( + num_images_per_prompt * batch_size, + self.prior.config.in_channels, + ceil(height / self.config.resolution_multiple), + ceil(width / self.config.resolution_multiple), + ) + + if latents is None: + latents = randn_tensor(latent_shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != latent_shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latent_shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def encode_prompt( + self, + device, + batch_size, + num_images_per_prompt, + do_classifier_free_guidance, + prompt=None, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + prompt_embeds_pooled: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds_pooled: Optional[torch.FloatTensor] = None, + ): + if prompt_embeds is None: + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + attention_mask = text_inputs.attention_mask + + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + text_input_ids = text_input_ids[:, : self.tokenizer.model_max_length] + attention_mask = attention_mask[:, : self.tokenizer.model_max_length] + + text_encoder_output = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask.to(device), output_hidden_states=True + ) + prompt_embeds = text_encoder_output.hidden_states[-1] + if prompt_embeds_pooled is None: + prompt_embeds_pooled = text_encoder_output.text_embeds.unsqueeze(1) + + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device) + prompt_embeds_pooled = prompt_embeds_pooled.to(dtype=self.text_encoder.dtype, device=device) + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + prompt_embeds_pooled = prompt_embeds_pooled.repeat_interleave(num_images_per_prompt, dim=0) + + if negative_prompt_embeds is None and do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + negative_prompt_embeds_text_encoder_output = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=uncond_input.attention_mask.to(device), + output_hidden_states=True, + ) + + negative_prompt_embeds = negative_prompt_embeds_text_encoder_output.hidden_states[-1] + negative_prompt_embeds_pooled = negative_prompt_embeds_text_encoder_output.text_embeds.unsqueeze(1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device) + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + seq_len = negative_prompt_embeds_pooled.shape[1] + negative_prompt_embeds_pooled = negative_prompt_embeds_pooled.to( + dtype=self.text_encoder.dtype, device=device + ) + negative_prompt_embeds_pooled = negative_prompt_embeds_pooled.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds_pooled = negative_prompt_embeds_pooled.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + # done duplicates + + return prompt_embeds, prompt_embeds_pooled, negative_prompt_embeds, negative_prompt_embeds_pooled + + def encode_image(self, images, device, dtype, batch_size, num_images_per_prompt): + image_embeds = [] + for image in images: + image = self.feature_extractor(image, return_tensors="pt").pixel_values + image = image.to(device=device, dtype=dtype) + image_embed = self.image_encoder(image).image_embeds.unsqueeze(1) + image_embeds.append(image_embed) + image_embeds = torch.cat(image_embeds, dim=1) + + image_embeds = image_embeds.repeat(batch_size * num_images_per_prompt, 1, 1) + negative_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, negative_image_embeds + + def check_inputs( + self, + prompt, + images=None, + image_embeds=None, + negative_prompt=None, + prompt_embeds=None, + prompt_embeds_pooled=None, + negative_prompt_embeds=None, + negative_prompt_embeds_pooled=None, + callback_on_step_end_tensor_inputs=None, + ): + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if prompt_embeds is not None and prompt_embeds_pooled is None: + raise ValueError( + "If `prompt_embeds` are provided, `prompt_embeds_pooled` must also be provided. Make sure to generate `prompt_embeds_pooled` from the same text encoder that was used to generate `prompt_embeds`" + ) + + if negative_prompt_embeds is not None and negative_prompt_embeds_pooled is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_prompt_embeds_pooled` must also be provided. Make sure to generate `prompt_embeds_pooled` from the same text encoder that was used to generate `prompt_embeds`" + ) + + if prompt_embeds_pooled is not None and negative_prompt_embeds_pooled is not None: + if prompt_embeds_pooled.shape != negative_prompt_embeds_pooled.shape: + raise ValueError( + "`prompt_embeds_pooled` and `negative_prompt_embeds_pooled` must have the same shape when passed" + f"directly, but got: `prompt_embeds_pooled` {prompt_embeds_pooled.shape} !=" + f"`negative_prompt_embeds_pooled` {negative_prompt_embeds_pooled.shape}." + ) + + if image_embeds is not None and images is not None: + raise ValueError( + f"Cannot forward both `images`: {images} and `image_embeds`: {image_embeds}. Please make sure to" + " only forward one of the two." + ) + + if images: + for i, image in enumerate(images): + if not isinstance(image, torch.Tensor) and not isinstance(image, PIL.Image.Image): + raise TypeError( + f"'images' must contain images of type 'torch.Tensor' or 'PIL.Image.Image, but got" + f"{type(image)} for image number {i}." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def num_timesteps(self): + return self._num_timesteps + + def get_timestep_ratio_conditioning(self, t, alphas_cumprod): + s = torch.tensor([0.003]) + clamp_range = [0, 1] + min_var = torch.cos(s / (1 + s) * torch.pi * 0.5) ** 2 + var = alphas_cumprod[t] + var = var.clamp(*clamp_range) + s, min_var = s.to(var.device), min_var.to(var.device) + ratio = (((var * min_var) ** 0.5).acos() / (torch.pi * 0.5)) * (1 + s) - s + return ratio + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + images: Union[torch.Tensor, PIL.Image.Image, List[torch.Tensor], List[PIL.Image.Image]] = None, + height: int = 1024, + width: int = 1024, + num_inference_steps: int = 20, + timesteps: List[float] = None, + guidance_scale: float = 4.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + prompt_embeds_pooled: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds_pooled: Optional[torch.FloatTensor] = None, + image_embeds: Optional[torch.FloatTensor] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pt", + return_dict: bool = True, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + height (`int`, *optional*, defaults to 1024): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 1024): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 60): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 8.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `decoder_guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting + `decoder_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely + linked to the text `prompt`, usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `decoder_guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + prompt_embeds_pooled (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + negative_prompt_embeds_pooled (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds_pooled will be generated from `negative_prompt` input + argument. + image_embeds (`torch.FloatTensor`, *optional*): + Pre-generated image embeddings. Can be used to easily tweak image inputs, *e.g.* prompt weighting. + If not provided, image embeddings will be generated from `image` input argument if existing. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`StableCascadePriorPipelineOutput`] or `tuple` [`StableCascadePriorPipelineOutput`] if + `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the + generated image embeddings. + """ + + # 0. Define commonly used variables + device = self._execution_device + dtype = next(self.prior.parameters()).dtype + self._guidance_scale = guidance_scale + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + images=images, + image_embeds=image_embeds, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + prompt_embeds_pooled=prompt_embeds_pooled, + negative_prompt_embeds=negative_prompt_embeds, + negative_prompt_embeds_pooled=negative_prompt_embeds_pooled, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + ) + + # 2. Encode caption + images + ( + prompt_embeds, + prompt_embeds_pooled, + negative_prompt_embeds, + negative_prompt_embeds_pooled, + ) = self.encode_prompt( + prompt=prompt, + device=device, + batch_size=batch_size, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + prompt_embeds_pooled=prompt_embeds_pooled, + negative_prompt_embeds=negative_prompt_embeds, + negative_prompt_embeds_pooled=negative_prompt_embeds_pooled, + ) + + if images is not None: + image_embeds_pooled, uncond_image_embeds_pooled = self.encode_image( + images=images, + device=device, + dtype=dtype, + batch_size=batch_size, + num_images_per_prompt=num_images_per_prompt, + ) + elif image_embeds is not None: + image_embeds_pooled = image_embeds.repeat(batch_size * num_images_per_prompt, 1, 1) + uncond_image_embeds_pooled = torch.zeros_like(image_embeds_pooled) + else: + image_embeds_pooled = torch.zeros( + batch_size * num_images_per_prompt, + 1, + self.prior.config.clip_image_in_channels, + device=device, + dtype=dtype, + ) + uncond_image_embeds_pooled = torch.zeros( + batch_size * num_images_per_prompt, + 1, + self.prior.config.clip_image_in_channels, + device=device, + dtype=dtype, + ) + + if self.do_classifier_free_guidance: + image_embeds = torch.cat([image_embeds_pooled, uncond_image_embeds_pooled], dim=0) + else: + image_embeds = image_embeds_pooled + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + text_encoder_hidden_states = ( + torch.cat([prompt_embeds, negative_prompt_embeds]) if negative_prompt_embeds is not None else prompt_embeds + ) + text_encoder_pooled = ( + torch.cat([prompt_embeds_pooled, negative_prompt_embeds_pooled]) + if negative_prompt_embeds is not None + else prompt_embeds_pooled + ) + + # 4. Prepare and set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latents + latents = self.prepare_latents( + batch_size, height, width, num_images_per_prompt, dtype, device, generator, latents, self.scheduler + ) + + if isinstance(self.scheduler, DDPMWuerstchenScheduler): + timesteps = timesteps[:-1] + else: + if self.scheduler.config.clip_sample: + self.scheduler.config.clip_sample = False # disample sample clipping + logger.warning(" set `clip_sample` to be False") + # 6. Run denoising loop + if hasattr(self.scheduler, "betas"): + alphas = 1.0 - self.scheduler.betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + else: + alphas_cumprod = [] + + self._num_timesteps = len(timesteps) + for i, t in enumerate(self.progress_bar(timesteps)): + if not isinstance(self.scheduler, DDPMWuerstchenScheduler): + if len(alphas_cumprod) > 0: + timestep_ratio = self.get_timestep_ratio_conditioning(t.long().cpu(), alphas_cumprod) + timestep_ratio = timestep_ratio.expand(latents.size(0)).to(dtype).to(device) + else: + timestep_ratio = t.float().div(self.scheduler.timesteps[-1]).expand(latents.size(0)).to(dtype) + else: + timestep_ratio = t.expand(latents.size(0)).to(dtype) + # 7. Denoise image embeddings + predicted_image_embedding = self.prior( + sample=torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents, + timestep_ratio=torch.cat([timestep_ratio] * 2) if self.do_classifier_free_guidance else timestep_ratio, + clip_text_pooled=text_encoder_pooled, + clip_text=text_encoder_hidden_states, + clip_img=image_embeds, + return_dict=False, + )[0] + + # 8. Check for classifier free guidance and apply it + if self.do_classifier_free_guidance: + predicted_image_embedding_text, predicted_image_embedding_uncond = predicted_image_embedding.chunk(2) + predicted_image_embedding = torch.lerp( + predicted_image_embedding_uncond, predicted_image_embedding_text, self.guidance_scale + ) + + # 9. Renoise latents to next timestep + if not isinstance(self.scheduler, DDPMWuerstchenScheduler): + timestep_ratio = t + latents = self.scheduler.step( + model_output=predicted_image_embedding, timestep=timestep_ratio, sample=latents, generator=generator + ).prev_sample + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # Offload all models + self.maybe_free_model_hooks() + + if output_type == "np": + latents = latents.cpu().float().numpy() # float() as bfloat16-> numpy doesnt work + prompt_embeds = prompt_embeds.cpu().float().numpy() # float() as bfloat16-> numpy doesnt work + negative_prompt_embeds = ( + negative_prompt_embeds.cpu().float().numpy() if negative_prompt_embeds is not None else None + ) # float() as bfloat16-> numpy doesnt work + + if not return_dict: + return ( + latents, + prompt_embeds, + prompt_embeds_pooled, + negative_prompt_embeds, + negative_prompt_embeds_pooled, + ) + + return StableCascadePriorPipelineOutput( + image_embeddings=latents, + prompt_embeds=prompt_embeds, + prompt_embeds_pooled=prompt_embeds_pooled, + negative_prompt_embeds=negative_prompt_embeds, + negative_prompt_embeds_pooled=negative_prompt_embeds_pooled, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/README.md b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/README.md new file mode 100755 index 0000000..5b64243 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/README.md @@ -0,0 +1,176 @@ +# Stable Diffusion + +## Overview + +Stable Diffusion was proposed in [Stable Diffusion Announcement](https://stability.ai/blog/stable-diffusion-announcement) by Patrick Esser and Robin Rombach and the Stability AI team. + +The summary of the model is the following: + +*Stable Diffusion is a text-to-image model that will empower billions of people to create stunning art within seconds. It is a breakthrough in speed and quality meaning that it can run on consumer GPUs. You can see some of the amazing output that has been created by this model without pre or post-processing on this page. The model itself builds upon the work of the team at CompVis and Runway in their widely used latent diffusion model combined with insights from the conditional diffusion models by our lead generative AI developer Katherine Crowson, Dall-E 2 by Open AI, Imagen by Google Brain and many others. We are delighted that AI media generation is a cooperative field and hope it can continue this way to bring the gift of creativity to all.* + +## Tips: + +- Stable Diffusion has the same architecture as [Latent Diffusion](https://arxiv.org/abs/2112.10752) but uses a frozen CLIP Text Encoder instead of training the text encoder jointly with the diffusion model. +- An in-detail explanation of the Stable Diffusion model can be found under [Stable Diffusion with 🧨 Diffusers](https://huggingface.co/blog/stable_diffusion). +- If you don't want to rely on the Hugging Face Hub and having to pass a authentication token, you can +download the weights with `git lfs install; git clone https://huggingface.co/runwayml/stable-diffusion-v1-5` and instead pass the local path to the cloned folder to `from_pretrained` as shown below. +- Stable Diffusion can work with a variety of different samplers as is shown below. + +## Available Pipelines: + +| Pipeline | Tasks | Colab +|---|---|:---:| +| [pipeline_stable_diffusion.py](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py) | *Text-to-Image Generation* | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/training_example.ipynb) +| [pipeline_stable_diffusion_img2img](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py) | *Image-to-Image Text-Guided Generation* | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/image_2_image_using_diffusers.ipynb) +| [pipeline_stable_diffusion_inpaint](https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py) | *Text-Guided Image Inpainting* | [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/huggingface/notebooks/blob/main/diffusers/in_painting_with_stable_diffusion_using_diffusers.ipynb) + +## Examples: + +### Using Stable Diffusion without being logged into the Hub. + +If you want to download the model weights using a single Python line, you need to be logged in via `huggingface-cli login`. + +```python +from diffusers import DiffusionPipeline + +pipeline = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") +``` + +This however can make it difficult to build applications on top of `diffusers` as you will always have to pass the token around. A potential way to solve this issue is by downloading the weights to a local path `"./stable-diffusion-v1-5"`: + +``` +git lfs install +git clone https://huggingface.co/runwayml/stable-diffusion-v1-5 +``` + +and simply passing the local path to `from_pretrained`: + +```python +from diffusers import StableDiffusionPipeline + +pipe = StableDiffusionPipeline.from_pretrained("./stable-diffusion-v1-5") +``` + +### Text-to-Image with default PLMS scheduler + +```python +# make sure you're logged in with `huggingface-cli login` +from diffusers import StableDiffusionPipeline + +pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") +pipe = pipe.to("cuda") + +prompt = "a photo of an astronaut riding a horse on mars" +image = pipe(prompt).images[0] + +image.save("astronaut_rides_horse.png") +``` + +### Text-to-Image with DDIM scheduler + +```python +# make sure you're logged in with `huggingface-cli login` +from diffusers import StableDiffusionPipeline, DDIMScheduler + +scheduler = DDIMScheduler.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="scheduler") + +pipe = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + scheduler=scheduler, +).to("cuda") + +prompt = "a photo of an astronaut riding a horse on mars" +image = pipe(prompt).images[0] + +image.save("astronaut_rides_horse.png") +``` + +### Text-to-Image with K-LMS scheduler + +```python +# make sure you're logged in with `huggingface-cli login` +from diffusers import StableDiffusionPipeline, LMSDiscreteScheduler + +lms = LMSDiscreteScheduler.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="scheduler") + +pipe = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + scheduler=lms, +).to("cuda") + +prompt = "a photo of an astronaut riding a horse on mars" +image = pipe(prompt).images[0] + +image.save("astronaut_rides_horse.png") +``` + +### CycleDiffusion using Stable Diffusion and DDIM scheduler + +```python +import requests +import torch +from PIL import Image +from io import BytesIO + +from diffusers import CycleDiffusionPipeline, DDIMScheduler + + +# load the scheduler. CycleDiffusion only supports stochastic schedulers. + +# load the pipeline +# make sure you're logged in with `huggingface-cli login` +model_id_or_path = "CompVis/stable-diffusion-v1-4" +scheduler = DDIMScheduler.from_pretrained(model_id_or_path, subfolder="scheduler") +pipe = CycleDiffusionPipeline.from_pretrained(model_id_or_path, scheduler=scheduler).to("cuda") + +# let's download an initial image +url = "https://raw.githubusercontent.com/ChenWu98/cycle-diffusion/main/data/dalle2/An%20astronaut%20riding%20a%20horse.png" +response = requests.get(url) +init_image = Image.open(BytesIO(response.content)).convert("RGB") +init_image = init_image.resize((512, 512)) +init_image.save("horse.png") + +# let's specify a prompt +source_prompt = "An astronaut riding a horse" +prompt = "An astronaut riding an elephant" + +# call the pipeline +image = pipe( + prompt=prompt, + source_prompt=source_prompt, + image=init_image, + num_inference_steps=100, + eta=0.1, + strength=0.8, + guidance_scale=2, + source_guidance_scale=1, +).images[0] + +image.save("horse_to_elephant.png") + +# let's try another example +# See more samples at the original repo: https://github.com/ChenWu98/cycle-diffusion +url = "https://raw.githubusercontent.com/ChenWu98/cycle-diffusion/main/data/dalle2/A%20black%20colored%20car.png" +response = requests.get(url) +init_image = Image.open(BytesIO(response.content)).convert("RGB") +init_image = init_image.resize((512, 512)) +init_image.save("black.png") + +source_prompt = "A black colored car" +prompt = "A blue colored car" + +# call the pipeline +torch.manual_seed(0) +image = pipe( + prompt=prompt, + source_prompt=source_prompt, + image=init_image, + num_inference_steps=100, + eta=0.1, + strength=0.85, + guidance_scale=3, + source_guidance_scale=1, +).images[0] + +image.save("black_to_blue.png") +``` diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/__init__.py new file mode 100755 index 0000000..0eda32d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/__init__.py @@ -0,0 +1,203 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_flax_available, + is_k_diffusion_available, + is_k_diffusion_version, + is_onnx_available, + is_torch_available, + is_transformers_available, + is_transformers_version, +) + + +_dummy_objects = {} +_additional_imports = {} +_import_structure = {"pipeline_output": ["StableDiffusionPipelineOutput"]} + +if is_transformers_available() and is_flax_available(): + _import_structure["pipeline_output"].extend(["FlaxStableDiffusionPipelineOutput"]) +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["clip_image_project_model"] = ["CLIPImageProjection"] + _import_structure["pipeline_cycle_diffusion"] = ["CycleDiffusionPipeline"] + _import_structure["pipeline_stable_diffusion"] = ["StableDiffusionPipeline"] + _import_structure["pipeline_stable_diffusion_attend_and_excite"] = ["StableDiffusionAttendAndExcitePipeline"] + _import_structure["pipeline_stable_diffusion_gligen"] = ["StableDiffusionGLIGENPipeline"] + _import_structure["pipeline_stable_diffusion_gligen_text_image"] = ["StableDiffusionGLIGENTextImagePipeline"] + _import_structure["pipeline_stable_diffusion_img2img"] = ["StableDiffusionImg2ImgPipeline"] + _import_structure["pipeline_stable_diffusion_inpaint"] = ["StableDiffusionInpaintPipeline"] + _import_structure["pipeline_stable_diffusion_inpaint_legacy"] = ["StableDiffusionInpaintPipelineLegacy"] + _import_structure["pipeline_stable_diffusion_instruct_pix2pix"] = ["StableDiffusionInstructPix2PixPipeline"] + _import_structure["pipeline_stable_diffusion_latent_upscale"] = ["StableDiffusionLatentUpscalePipeline"] + _import_structure["pipeline_stable_diffusion_model_editing"] = ["StableDiffusionModelEditingPipeline"] + _import_structure["pipeline_stable_diffusion_paradigms"] = ["StableDiffusionParadigmsPipeline"] + _import_structure["pipeline_stable_diffusion_upscale"] = ["StableDiffusionUpscalePipeline"] + _import_structure["pipeline_stable_unclip"] = ["StableUnCLIPPipeline"] + _import_structure["pipeline_stable_unclip_img2img"] = ["StableUnCLIPImg2ImgPipeline"] + _import_structure["safety_checker"] = ["StableDiffusionSafetyChecker"] + _import_structure["stable_unclip_image_normalizer"] = ["StableUnCLIPImageNormalizer"] +try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import ( + StableDiffusionImageVariationPipeline, + ) + + _dummy_objects.update({"StableDiffusionImageVariationPipeline": StableDiffusionImageVariationPipeline}) +else: + _import_structure["pipeline_stable_diffusion_image_variation"] = ["StableDiffusionImageVariationPipeline"] +try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.26.0")): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import ( + StableDiffusionDepth2ImgPipeline, + ) + + _dummy_objects.update( + { + "StableDiffusionDepth2ImgPipeline": StableDiffusionDepth2ImgPipeline, + } + ) +else: + _import_structure["pipeline_stable_diffusion_depth2img"] = ["StableDiffusionDepth2ImgPipeline"] + +try: + if not (is_transformers_available() and is_onnx_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_onnx_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_onnx_objects)) +else: + _import_structure["pipeline_onnx_stable_diffusion"] = [ + "OnnxStableDiffusionPipeline", + "StableDiffusionOnnxPipeline", + ] + _import_structure["pipeline_onnx_stable_diffusion_img2img"] = ["OnnxStableDiffusionImg2ImgPipeline"] + _import_structure["pipeline_onnx_stable_diffusion_inpaint"] = ["OnnxStableDiffusionInpaintPipeline"] + _import_structure["pipeline_onnx_stable_diffusion_inpaint_legacy"] = ["OnnxStableDiffusionInpaintPipelineLegacy"] + _import_structure["pipeline_onnx_stable_diffusion_upscale"] = ["OnnxStableDiffusionUpscalePipeline"] + +if is_transformers_available() and is_flax_available(): + from ...schedulers.scheduling_pndm_flax import PNDMSchedulerState + + _additional_imports.update({"PNDMSchedulerState": PNDMSchedulerState}) + _import_structure["pipeline_flax_stable_diffusion"] = ["FlaxStableDiffusionPipeline"] + _import_structure["pipeline_flax_stable_diffusion_img2img"] = ["FlaxStableDiffusionImg2ImgPipeline"] + _import_structure["pipeline_flax_stable_diffusion_inpaint"] = ["FlaxStableDiffusionInpaintPipeline"] + _import_structure["safety_checker_flax"] = ["FlaxStableDiffusionSafetyChecker"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + + else: + from .clip_image_project_model import CLIPImageProjection + from .pipeline_stable_diffusion import ( + StableDiffusionPipeline, + StableDiffusionPipelineOutput, + StableDiffusionSafetyChecker, + ) + from .pipeline_stable_diffusion_img2img import StableDiffusionImg2ImgPipeline + from .pipeline_stable_diffusion_inpaint import StableDiffusionInpaintPipeline + from .pipeline_stable_diffusion_instruct_pix2pix import ( + StableDiffusionInstructPix2PixPipeline, + ) + from .pipeline_stable_diffusion_latent_upscale import ( + StableDiffusionLatentUpscalePipeline, + ) + from .pipeline_stable_diffusion_upscale import StableDiffusionUpscalePipeline + from .pipeline_stable_unclip import StableUnCLIPPipeline + from .pipeline_stable_unclip_img2img import StableUnCLIPImg2ImgPipeline + from .safety_checker import StableDiffusionSafetyChecker + from .stable_unclip_image_normalizer import StableUnCLIPImageNormalizer + + try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import ( + StableDiffusionImageVariationPipeline, + ) + else: + from .pipeline_stable_diffusion_image_variation import ( + StableDiffusionImageVariationPipeline, + ) + + try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.26.0")): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import StableDiffusionDepth2ImgPipeline + else: + from .pipeline_stable_diffusion_depth2img import ( + StableDiffusionDepth2ImgPipeline, + ) + + try: + if not (is_transformers_available() and is_onnx_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_onnx_objects import * + else: + from .pipeline_onnx_stable_diffusion import ( + OnnxStableDiffusionPipeline, + StableDiffusionOnnxPipeline, + ) + from .pipeline_onnx_stable_diffusion_img2img import ( + OnnxStableDiffusionImg2ImgPipeline, + ) + from .pipeline_onnx_stable_diffusion_inpaint import ( + OnnxStableDiffusionInpaintPipeline, + ) + from .pipeline_onnx_stable_diffusion_upscale import ( + OnnxStableDiffusionUpscalePipeline, + ) + + try: + if not (is_transformers_available() and is_flax_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_flax_objects import * + else: + from .pipeline_flax_stable_diffusion import FlaxStableDiffusionPipeline + from .pipeline_flax_stable_diffusion_img2img import ( + FlaxStableDiffusionImg2ImgPipeline, + ) + from .pipeline_flax_stable_diffusion_inpaint import ( + FlaxStableDiffusionInpaintPipeline, + ) + from .pipeline_output import FlaxStableDiffusionPipelineOutput + from .safety_checker_flax import FlaxStableDiffusionSafetyChecker + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) + for name, value in _additional_imports.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/clip_image_project_model.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/clip_image_project_model.py new file mode 100755 index 0000000..71f9d97 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/clip_image_project_model.py @@ -0,0 +1,29 @@ +# Copyright 2024 The GLIGEN Authors and HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from torch import nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...models.modeling_utils import ModelMixin + + +class CLIPImageProjection(ModelMixin, ConfigMixin): + @register_to_config + def __init__(self, hidden_size: int = 768): + super().__init__() + self.hidden_size = hidden_size + self.project = nn.Linear(self.hidden_size, self.hidden_size, bias=False) + + def forward(self, x): + return self.project(x) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py new file mode 100755 index 0000000..30c3c5b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/convert_from_ckpt.py @@ -0,0 +1,1860 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Conversion script for the Stable Diffusion checkpoints.""" + +import re +from contextlib import nullcontext +from io import BytesIO +from typing import Dict, Optional, Union + +import requests +import torch +import yaml +from transformers import ( + AutoFeatureExtractor, + BertTokenizerFast, + CLIPImageProcessor, + CLIPTextConfig, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +from ...models import ( + AutoencoderKL, + ControlNetModel, + PriorTransformer, + UNet2DConditionModel, +) +from ...schedulers import ( + DDIMScheduler, + DDPMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + UnCLIPScheduler, +) +from ...utils import is_accelerate_available, logging +from ..latent_diffusion.pipeline_latent_diffusion import LDMBertConfig, LDMBertModel +from ..paint_by_example import PaintByExampleImageEncoder +from ..pipeline_utils import DiffusionPipeline +from .safety_checker import StableDiffusionSafetyChecker +from .stable_unclip_image_normalizer import StableUnCLIPImageNormalizer + + +if is_accelerate_available(): + from accelerate import init_empty_weights + from accelerate.utils import set_module_tensor_to_device + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def shave_segments(path, n_shave_prefix_segments=1): + """ + Removes segments. Positive values shave the first segments, negative shave the last segments. + """ + if n_shave_prefix_segments >= 0: + return ".".join(path.split(".")[n_shave_prefix_segments:]) + else: + return ".".join(path.split(".")[:n_shave_prefix_segments]) + + +def renew_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item.replace("in_layers.0", "norm1") + new_item = new_item.replace("in_layers.2", "conv1") + + new_item = new_item.replace("out_layers.0", "norm2") + new_item = new_item.replace("out_layers.3", "conv2") + + new_item = new_item.replace("emb_layers.1", "time_emb_proj") + new_item = new_item.replace("skip_connection", "conv_shortcut") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_vae_resnet_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside resnets to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("nin_shortcut", "conv_shortcut") + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + # new_item = new_item.replace('norm.weight', 'group_norm.weight') + # new_item = new_item.replace('norm.bias', 'group_norm.bias') + + # new_item = new_item.replace('proj_out.weight', 'proj_attn.weight') + # new_item = new_item.replace('proj_out.bias', 'proj_attn.bias') + + # new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def renew_vae_attention_paths(old_list, n_shave_prefix_segments=0): + """ + Updates paths inside attentions to the new naming scheme (local renaming) + """ + mapping = [] + for old_item in old_list: + new_item = old_item + + new_item = new_item.replace("norm.weight", "group_norm.weight") + new_item = new_item.replace("norm.bias", "group_norm.bias") + + new_item = new_item.replace("q.weight", "to_q.weight") + new_item = new_item.replace("q.bias", "to_q.bias") + + new_item = new_item.replace("k.weight", "to_k.weight") + new_item = new_item.replace("k.bias", "to_k.bias") + + new_item = new_item.replace("v.weight", "to_v.weight") + new_item = new_item.replace("v.bias", "to_v.bias") + + new_item = new_item.replace("proj_out.weight", "to_out.0.weight") + new_item = new_item.replace("proj_out.bias", "to_out.0.bias") + + new_item = shave_segments(new_item, n_shave_prefix_segments=n_shave_prefix_segments) + + mapping.append({"old": old_item, "new": new_item}) + + return mapping + + +def assign_to_checkpoint( + paths, checkpoint, old_checkpoint, attention_paths_to_split=None, additional_replacements=None, config=None +): + """ + This does the final conversion step: take locally converted weights and apply a global renaming to them. It splits + attention layers, and takes into account additional replacements that may arise. + + Assigns the weights to the new checkpoint. + """ + assert isinstance(paths, list), "Paths should be a list of dicts containing 'old' and 'new' keys." + + # Splits the attention layers into three variables. + if attention_paths_to_split is not None: + for path, path_map in attention_paths_to_split.items(): + old_tensor = old_checkpoint[path] + channels = old_tensor.shape[0] // 3 + + target_shape = (-1, channels) if len(old_tensor.shape) == 3 else (-1) + + num_heads = old_tensor.shape[0] // config["num_head_channels"] // 3 + + old_tensor = old_tensor.reshape((num_heads, 3 * channels // num_heads) + old_tensor.shape[1:]) + query, key, value = old_tensor.split(channels // num_heads, dim=1) + + checkpoint[path_map["query"]] = query.reshape(target_shape) + checkpoint[path_map["key"]] = key.reshape(target_shape) + checkpoint[path_map["value"]] = value.reshape(target_shape) + + for path in paths: + new_path = path["new"] + + # These have already been assigned + if attention_paths_to_split is not None and new_path in attention_paths_to_split: + continue + + # Global renaming happens here + new_path = new_path.replace("middle_block.0", "mid_block.resnets.0") + new_path = new_path.replace("middle_block.1", "mid_block.attentions.0") + new_path = new_path.replace("middle_block.2", "mid_block.resnets.1") + + if additional_replacements is not None: + for replacement in additional_replacements: + new_path = new_path.replace(replacement["old"], replacement["new"]) + + # proj_attn.weight has to be converted from conv 1D to linear + is_attn_weight = "proj_attn.weight" in new_path or ("attentions" in new_path and "to_" in new_path) + shape = old_checkpoint[path["old"]].shape + if is_attn_weight and len(shape) == 3: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0] + elif is_attn_weight and len(shape) == 4: + checkpoint[new_path] = old_checkpoint[path["old"]][:, :, 0, 0] + else: + checkpoint[new_path] = old_checkpoint[path["old"]] + + +def conv_attn_to_linear(checkpoint): + keys = list(checkpoint.keys()) + attn_keys = ["query.weight", "key.weight", "value.weight"] + for key in keys: + if ".".join(key.split(".")[-2:]) in attn_keys: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0, 0] + elif "proj_attn.weight" in key: + if checkpoint[key].ndim > 2: + checkpoint[key] = checkpoint[key][:, :, 0] + + +def create_unet_diffusers_config(original_config, image_size: int, controlnet=False): + """ + Creates a config for the diffusers based on the config of the LDM model. + """ + if controlnet: + unet_params = original_config["model"]["params"]["control_stage_config"]["params"] + else: + if ( + "unet_config" in original_config["model"]["params"] + and original_config["model"]["params"]["unet_config"] is not None + ): + unet_params = original_config["model"]["params"]["unet_config"]["params"] + else: + unet_params = original_config["model"]["params"]["network_config"]["params"] + + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + + block_out_channels = [unet_params["model_channels"] * mult for mult in unet_params["channel_mult"]] + + down_block_types = [] + resolution = 1 + for i in range(len(block_out_channels)): + block_type = "CrossAttnDownBlock2D" if resolution in unet_params["attention_resolutions"] else "DownBlock2D" + down_block_types.append(block_type) + if i != len(block_out_channels) - 1: + resolution *= 2 + + up_block_types = [] + for i in range(len(block_out_channels)): + block_type = "CrossAttnUpBlock2D" if resolution in unet_params["attention_resolutions"] else "UpBlock2D" + up_block_types.append(block_type) + resolution //= 2 + + if unet_params["transformer_depth"] is not None: + transformer_layers_per_block = ( + unet_params["transformer_depth"] + if isinstance(unet_params["transformer_depth"], int) + else list(unet_params["transformer_depth"]) + ) + else: + transformer_layers_per_block = 1 + + vae_scale_factor = 2 ** (len(vae_params["ch_mult"]) - 1) + + head_dim = unet_params["num_heads"] if "num_heads" in unet_params else None + use_linear_projection = ( + unet_params["use_linear_in_transformer"] if "use_linear_in_transformer" in unet_params else False + ) + if use_linear_projection: + # stable diffusion 2-base-512 and 2-768 + if head_dim is None: + head_dim_mult = unet_params["model_channels"] // unet_params["num_head_channels"] + head_dim = [head_dim_mult * c for c in list(unet_params["channel_mult"])] + + class_embed_type = None + addition_embed_type = None + addition_time_embed_dim = None + projection_class_embeddings_input_dim = None + context_dim = None + + if unet_params["context_dim"] is not None: + context_dim = ( + unet_params["context_dim"] + if isinstance(unet_params["context_dim"], int) + else unet_params["context_dim"][0] + ) + + if "num_classes" in unet_params: + if unet_params["num_classes"] == "sequential": + if context_dim in [2048, 1280]: + # SDXL + addition_embed_type = "text_time" + addition_time_embed_dim = 256 + else: + class_embed_type = "projection" + assert "adm_in_channels" in unet_params + projection_class_embeddings_input_dim = unet_params["adm_in_channels"] + + config = { + "sample_size": image_size // vae_scale_factor, + "in_channels": unet_params["in_channels"], + "down_block_types": tuple(down_block_types), + "block_out_channels": tuple(block_out_channels), + "layers_per_block": unet_params["num_res_blocks"], + "cross_attention_dim": context_dim, + "attention_head_dim": head_dim, + "use_linear_projection": use_linear_projection, + "class_embed_type": class_embed_type, + "addition_embed_type": addition_embed_type, + "addition_time_embed_dim": addition_time_embed_dim, + "projection_class_embeddings_input_dim": projection_class_embeddings_input_dim, + "transformer_layers_per_block": transformer_layers_per_block, + } + + if "disable_self_attentions" in unet_params: + config["only_cross_attention"] = unet_params["disable_self_attentions"] + + if "num_classes" in unet_params and isinstance(unet_params["num_classes"], int): + config["num_class_embeds"] = unet_params["num_classes"] + + if controlnet: + config["conditioning_channels"] = unet_params["hint_channels"] + else: + config["out_channels"] = unet_params["out_channels"] + config["up_block_types"] = tuple(up_block_types) + + return config + + +def create_vae_diffusers_config(original_config, image_size: int): + """ + Creates a config for the diffusers based on the config of the LDM model. + """ + vae_params = original_config["model"]["params"]["first_stage_config"]["params"]["ddconfig"] + _ = original_config["model"]["params"]["first_stage_config"]["params"]["embed_dim"] + + block_out_channels = [vae_params["ch"] * mult for mult in vae_params["ch_mult"]] + down_block_types = ["DownEncoderBlock2D"] * len(block_out_channels) + up_block_types = ["UpDecoderBlock2D"] * len(block_out_channels) + + config = { + "sample_size": image_size, + "in_channels": vae_params["in_channels"], + "out_channels": vae_params["out_ch"], + "down_block_types": tuple(down_block_types), + "up_block_types": tuple(up_block_types), + "block_out_channels": tuple(block_out_channels), + "latent_channels": vae_params["z_channels"], + "layers_per_block": vae_params["num_res_blocks"], + } + return config + + +def create_diffusers_schedular(original_config): + schedular = DDIMScheduler( + num_train_timesteps=original_config["model"]["params"]["timesteps"], + beta_start=original_config["model"]["params"]["linear_start"], + beta_end=original_config["model"]["params"]["linear_end"], + beta_schedule="scaled_linear", + ) + return schedular + + +def create_ldm_bert_config(original_config): + bert_params = original_config["model"]["params"]["cond_stage_config"]["params"] + config = LDMBertConfig( + d_model=bert_params.n_embed, + encoder_layers=bert_params.n_layer, + encoder_ffn_dim=bert_params.n_embed * 4, + ) + return config + + +def convert_ldm_unet_checkpoint( + checkpoint, config, path=None, extract_ema=False, controlnet=False, skip_extract_state_dict=False +): + """ + Takes a state dict and a config, and returns a converted checkpoint. + """ + + if skip_extract_state_dict: + unet_state_dict = checkpoint + else: + # extract state_dict for UNet + unet_state_dict = {} + keys = list(checkpoint.keys()) + + if controlnet: + unet_key = "control_model." + else: + unet_key = "model.diffusion_model." + + # at least a 100 parameters have to start with `model_ema` in order for the checkpoint to be EMA + if sum(k.startswith("model_ema") for k in keys) > 100 and extract_ema: + logger.warning(f"Checkpoint {path} has both EMA and non-EMA weights.") + logger.warning( + "In this conversion only the EMA weights are extracted. If you want to instead extract the non-EMA" + " weights (useful to continue fine-tuning), please make sure to remove the `--extract_ema` flag." + ) + for key in keys: + if key.startswith("model.diffusion_model"): + flat_ema_key = "model_ema." + "".join(key.split(".")[1:]) + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(flat_ema_key) + else: + if sum(k.startswith("model_ema") for k in keys) > 100: + logger.warning( + "In this conversion only the non-EMA weights are extracted. If you want to instead extract the EMA" + " weights (usually better for inference), please make sure to add the `--extract_ema` flag." + ) + + for key in keys: + if key.startswith(unet_key): + unet_state_dict[key.replace(unet_key, "")] = checkpoint.pop(key) + + new_checkpoint = {} + + new_checkpoint["time_embedding.linear_1.weight"] = unet_state_dict["time_embed.0.weight"] + new_checkpoint["time_embedding.linear_1.bias"] = unet_state_dict["time_embed.0.bias"] + new_checkpoint["time_embedding.linear_2.weight"] = unet_state_dict["time_embed.2.weight"] + new_checkpoint["time_embedding.linear_2.bias"] = unet_state_dict["time_embed.2.bias"] + + if config["class_embed_type"] is None: + # No parameters to port + ... + elif config["class_embed_type"] == "timestep" or config["class_embed_type"] == "projection": + new_checkpoint["class_embedding.linear_1.weight"] = unet_state_dict["label_emb.0.0.weight"] + new_checkpoint["class_embedding.linear_1.bias"] = unet_state_dict["label_emb.0.0.bias"] + new_checkpoint["class_embedding.linear_2.weight"] = unet_state_dict["label_emb.0.2.weight"] + new_checkpoint["class_embedding.linear_2.bias"] = unet_state_dict["label_emb.0.2.bias"] + else: + raise NotImplementedError(f"Not implemented `class_embed_type`: {config['class_embed_type']}") + + if config["addition_embed_type"] == "text_time": + new_checkpoint["add_embedding.linear_1.weight"] = unet_state_dict["label_emb.0.0.weight"] + new_checkpoint["add_embedding.linear_1.bias"] = unet_state_dict["label_emb.0.0.bias"] + new_checkpoint["add_embedding.linear_2.weight"] = unet_state_dict["label_emb.0.2.weight"] + new_checkpoint["add_embedding.linear_2.bias"] = unet_state_dict["label_emb.0.2.bias"] + + # Relevant to StableDiffusionUpscalePipeline + if "num_class_embeds" in config: + if (config["num_class_embeds"] is not None) and ("label_emb.weight" in unet_state_dict): + new_checkpoint["class_embedding.weight"] = unet_state_dict["label_emb.weight"] + + new_checkpoint["conv_in.weight"] = unet_state_dict["input_blocks.0.0.weight"] + new_checkpoint["conv_in.bias"] = unet_state_dict["input_blocks.0.0.bias"] + + if not controlnet: + new_checkpoint["conv_norm_out.weight"] = unet_state_dict["out.0.weight"] + new_checkpoint["conv_norm_out.bias"] = unet_state_dict["out.0.bias"] + new_checkpoint["conv_out.weight"] = unet_state_dict["out.2.weight"] + new_checkpoint["conv_out.bias"] = unet_state_dict["out.2.bias"] + + # Retrieves the keys for the input blocks only + num_input_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "input_blocks" in layer}) + input_blocks = { + layer_id: [key for key in unet_state_dict if f"input_blocks.{layer_id}" in key] + for layer_id in range(num_input_blocks) + } + + # Retrieves the keys for the middle blocks only + num_middle_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "middle_block" in layer}) + middle_blocks = { + layer_id: [key for key in unet_state_dict if f"middle_block.{layer_id}" in key] + for layer_id in range(num_middle_blocks) + } + + # Retrieves the keys for the output blocks only + num_output_blocks = len({".".join(layer.split(".")[:2]) for layer in unet_state_dict if "output_blocks" in layer}) + output_blocks = { + layer_id: [key for key in unet_state_dict if f"output_blocks.{layer_id}" in key] + for layer_id in range(num_output_blocks) + } + + for i in range(1, num_input_blocks): + block_id = (i - 1) // (config["layers_per_block"] + 1) + layer_in_block_id = (i - 1) % (config["layers_per_block"] + 1) + + resnets = [ + key for key in input_blocks[i] if f"input_blocks.{i}.0" in key and f"input_blocks.{i}.0.op" not in key + ] + attentions = [key for key in input_blocks[i] if f"input_blocks.{i}.1" in key] + + if f"input_blocks.{i}.0.op.weight" in unet_state_dict: + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.weight"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.weight" + ) + new_checkpoint[f"down_blocks.{block_id}.downsamplers.0.conv.bias"] = unet_state_dict.pop( + f"input_blocks.{i}.0.op.bias" + ) + + paths = renew_resnet_paths(resnets) + meta_path = {"old": f"input_blocks.{i}.0", "new": f"down_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + if len(attentions): + paths = renew_attention_paths(attentions) + + meta_path = {"old": f"input_blocks.{i}.1", "new": f"down_blocks.{block_id}.attentions.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + resnet_0 = middle_blocks[0] + attentions = middle_blocks[1] + resnet_1 = middle_blocks[2] + + resnet_0_paths = renew_resnet_paths(resnet_0) + assign_to_checkpoint(resnet_0_paths, new_checkpoint, unet_state_dict, config=config) + + resnet_1_paths = renew_resnet_paths(resnet_1) + assign_to_checkpoint(resnet_1_paths, new_checkpoint, unet_state_dict, config=config) + + attentions_paths = renew_attention_paths(attentions) + meta_path = {"old": "middle_block.1", "new": "mid_block.attentions.0"} + assign_to_checkpoint( + attentions_paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + for i in range(num_output_blocks): + block_id = i // (config["layers_per_block"] + 1) + layer_in_block_id = i % (config["layers_per_block"] + 1) + output_block_layers = [shave_segments(name, 2) for name in output_blocks[i]] + output_block_list = {} + + for layer in output_block_layers: + layer_id, layer_name = layer.split(".")[0], shave_segments(layer, 1) + if layer_id in output_block_list: + output_block_list[layer_id].append(layer_name) + else: + output_block_list[layer_id] = [layer_name] + + if len(output_block_list) > 1: + resnets = [key for key in output_blocks[i] if f"output_blocks.{i}.0" in key] + attentions = [key for key in output_blocks[i] if f"output_blocks.{i}.1" in key] + + resnet_0_paths = renew_resnet_paths(resnets) + paths = renew_resnet_paths(resnets) + + meta_path = {"old": f"output_blocks.{i}.0", "new": f"up_blocks.{block_id}.resnets.{layer_in_block_id}"} + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + + output_block_list = {k: sorted(v) for k, v in output_block_list.items()} + if ["conv.bias", "conv.weight"] in output_block_list.values(): + index = list(output_block_list.values()).index(["conv.bias", "conv.weight"]) + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.weight"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.weight" + ] + new_checkpoint[f"up_blocks.{block_id}.upsamplers.0.conv.bias"] = unet_state_dict[ + f"output_blocks.{i}.{index}.conv.bias" + ] + + # Clear attentions as they have been attributed above. + if len(attentions) == 2: + attentions = [] + + if len(attentions): + paths = renew_attention_paths(attentions) + meta_path = { + "old": f"output_blocks.{i}.1", + "new": f"up_blocks.{block_id}.attentions.{layer_in_block_id}", + } + assign_to_checkpoint( + paths, new_checkpoint, unet_state_dict, additional_replacements=[meta_path], config=config + ) + else: + resnet_0_paths = renew_resnet_paths(output_block_layers, n_shave_prefix_segments=1) + for path in resnet_0_paths: + old_path = ".".join(["output_blocks", str(i), path["old"]]) + new_path = ".".join(["up_blocks", str(block_id), "resnets", str(layer_in_block_id), path["new"]]) + + new_checkpoint[new_path] = unet_state_dict[old_path] + + if controlnet: + # conditioning embedding + + orig_index = 0 + + new_checkpoint["controlnet_cond_embedding.conv_in.weight"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.weight" + ) + new_checkpoint["controlnet_cond_embedding.conv_in.bias"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.bias" + ) + + orig_index += 2 + + diffusers_index = 0 + + while diffusers_index < 6: + new_checkpoint[f"controlnet_cond_embedding.blocks.{diffusers_index}.weight"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.weight" + ) + new_checkpoint[f"controlnet_cond_embedding.blocks.{diffusers_index}.bias"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.bias" + ) + diffusers_index += 1 + orig_index += 2 + + new_checkpoint["controlnet_cond_embedding.conv_out.weight"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.weight" + ) + new_checkpoint["controlnet_cond_embedding.conv_out.bias"] = unet_state_dict.pop( + f"input_hint_block.{orig_index}.bias" + ) + + # down blocks + for i in range(num_input_blocks): + new_checkpoint[f"controlnet_down_blocks.{i}.weight"] = unet_state_dict.pop(f"zero_convs.{i}.0.weight") + new_checkpoint[f"controlnet_down_blocks.{i}.bias"] = unet_state_dict.pop(f"zero_convs.{i}.0.bias") + + # mid block + new_checkpoint["controlnet_mid_block.weight"] = unet_state_dict.pop("middle_block_out.0.weight") + new_checkpoint["controlnet_mid_block.bias"] = unet_state_dict.pop("middle_block_out.0.bias") + + return new_checkpoint + + +def convert_ldm_vae_checkpoint(checkpoint, config): + # extract state dict for VAE + vae_state_dict = {} + keys = list(checkpoint.keys()) + vae_key = "first_stage_model." if any(k.startswith("first_stage_model.") for k in keys) else "" + for key in keys: + if key.startswith(vae_key): + vae_state_dict[key.replace(vae_key, "")] = checkpoint.get(key) + + new_checkpoint = {} + + new_checkpoint["encoder.conv_in.weight"] = vae_state_dict["encoder.conv_in.weight"] + new_checkpoint["encoder.conv_in.bias"] = vae_state_dict["encoder.conv_in.bias"] + new_checkpoint["encoder.conv_out.weight"] = vae_state_dict["encoder.conv_out.weight"] + new_checkpoint["encoder.conv_out.bias"] = vae_state_dict["encoder.conv_out.bias"] + new_checkpoint["encoder.conv_norm_out.weight"] = vae_state_dict["encoder.norm_out.weight"] + new_checkpoint["encoder.conv_norm_out.bias"] = vae_state_dict["encoder.norm_out.bias"] + + new_checkpoint["decoder.conv_in.weight"] = vae_state_dict["decoder.conv_in.weight"] + new_checkpoint["decoder.conv_in.bias"] = vae_state_dict["decoder.conv_in.bias"] + new_checkpoint["decoder.conv_out.weight"] = vae_state_dict["decoder.conv_out.weight"] + new_checkpoint["decoder.conv_out.bias"] = vae_state_dict["decoder.conv_out.bias"] + new_checkpoint["decoder.conv_norm_out.weight"] = vae_state_dict["decoder.norm_out.weight"] + new_checkpoint["decoder.conv_norm_out.bias"] = vae_state_dict["decoder.norm_out.bias"] + + new_checkpoint["quant_conv.weight"] = vae_state_dict["quant_conv.weight"] + new_checkpoint["quant_conv.bias"] = vae_state_dict["quant_conv.bias"] + new_checkpoint["post_quant_conv.weight"] = vae_state_dict["post_quant_conv.weight"] + new_checkpoint["post_quant_conv.bias"] = vae_state_dict["post_quant_conv.bias"] + + # Retrieves the keys for the encoder down blocks only + num_down_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "encoder.down" in layer}) + down_blocks = { + layer_id: [key for key in vae_state_dict if f"down.{layer_id}" in key] for layer_id in range(num_down_blocks) + } + + # Retrieves the keys for the decoder up blocks only + num_up_blocks = len({".".join(layer.split(".")[:3]) for layer in vae_state_dict if "decoder.up" in layer}) + up_blocks = { + layer_id: [key for key in vae_state_dict if f"up.{layer_id}" in key] for layer_id in range(num_up_blocks) + } + + for i in range(num_down_blocks): + resnets = [key for key in down_blocks[i] if f"down.{i}" in key and f"down.{i}.downsample" not in key] + + if f"encoder.down.{i}.downsample.conv.weight" in vae_state_dict: + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.weight"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.weight" + ) + new_checkpoint[f"encoder.down_blocks.{i}.downsamplers.0.conv.bias"] = vae_state_dict.pop( + f"encoder.down.{i}.downsample.conv.bias" + ) + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"down.{i}.block", "new": f"down_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "encoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"encoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "encoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + + for i in range(num_up_blocks): + block_id = num_up_blocks - 1 - i + resnets = [ + key for key in up_blocks[block_id] if f"up.{block_id}" in key and f"up.{block_id}.upsample" not in key + ] + + if f"decoder.up.{block_id}.upsample.conv.weight" in vae_state_dict: + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.weight"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.weight" + ] + new_checkpoint[f"decoder.up_blocks.{i}.upsamplers.0.conv.bias"] = vae_state_dict[ + f"decoder.up.{block_id}.upsample.conv.bias" + ] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"up.{block_id}.block", "new": f"up_blocks.{i}.resnets"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_resnets = [key for key in vae_state_dict if "decoder.mid.block" in key] + num_mid_res_blocks = 2 + for i in range(1, num_mid_res_blocks + 1): + resnets = [key for key in mid_resnets if f"decoder.mid.block_{i}" in key] + + paths = renew_vae_resnet_paths(resnets) + meta_path = {"old": f"mid.block_{i}", "new": f"mid_block.resnets.{i - 1}"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + + mid_attentions = [key for key in vae_state_dict if "decoder.mid.attn" in key] + paths = renew_vae_attention_paths(mid_attentions) + meta_path = {"old": "mid.attn_1", "new": "mid_block.attentions.0"} + assign_to_checkpoint(paths, new_checkpoint, vae_state_dict, additional_replacements=[meta_path], config=config) + conv_attn_to_linear(new_checkpoint) + return new_checkpoint + + +def convert_ldm_bert_checkpoint(checkpoint, config): + def _copy_attn_layer(hf_attn_layer, pt_attn_layer): + hf_attn_layer.q_proj.weight.data = pt_attn_layer.to_q.weight + hf_attn_layer.k_proj.weight.data = pt_attn_layer.to_k.weight + hf_attn_layer.v_proj.weight.data = pt_attn_layer.to_v.weight + + hf_attn_layer.out_proj.weight = pt_attn_layer.to_out.weight + hf_attn_layer.out_proj.bias = pt_attn_layer.to_out.bias + + def _copy_linear(hf_linear, pt_linear): + hf_linear.weight = pt_linear.weight + hf_linear.bias = pt_linear.bias + + def _copy_layer(hf_layer, pt_layer): + # copy layer norms + _copy_linear(hf_layer.self_attn_layer_norm, pt_layer[0][0]) + _copy_linear(hf_layer.final_layer_norm, pt_layer[1][0]) + + # copy attn + _copy_attn_layer(hf_layer.self_attn, pt_layer[0][1]) + + # copy MLP + pt_mlp = pt_layer[1][1] + _copy_linear(hf_layer.fc1, pt_mlp.net[0][0]) + _copy_linear(hf_layer.fc2, pt_mlp.net[2]) + + def _copy_layers(hf_layers, pt_layers): + for i, hf_layer in enumerate(hf_layers): + if i != 0: + i += i + pt_layer = pt_layers[i : i + 2] + _copy_layer(hf_layer, pt_layer) + + hf_model = LDMBertModel(config).eval() + + # copy embeds + hf_model.model.embed_tokens.weight = checkpoint.transformer.token_emb.weight + hf_model.model.embed_positions.weight.data = checkpoint.transformer.pos_emb.emb.weight + + # copy layer norm + _copy_linear(hf_model.model.layer_norm, checkpoint.transformer.norm) + + # copy hidden layers + _copy_layers(hf_model.model.layers, checkpoint.transformer.attn_layers.layers) + + _copy_linear(hf_model.to_logits, checkpoint.transformer.to_logits) + + return hf_model + + +def convert_ldm_clip_checkpoint(checkpoint, local_files_only=False, text_encoder=None): + if text_encoder is None: + config_name = "openai/clip-vit-large-patch14" + try: + config = CLIPTextConfig.from_pretrained(config_name, local_files_only=local_files_only) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the configuration in the following path: 'openai/clip-vit-large-patch14'." + ) + + ctx = init_empty_weights if is_accelerate_available() else nullcontext + with ctx(): + text_model = CLIPTextModel(config) + else: + text_model = text_encoder + + keys = list(checkpoint.keys()) + + text_model_dict = {} + + remove_prefixes = ["cond_stage_model.transformer", "conditioner.embedders.0.transformer"] + + for key in keys: + for prefix in remove_prefixes: + if key.startswith(prefix): + text_model_dict[key[len(prefix + ".") :]] = checkpoint[key] + + if is_accelerate_available(): + for param_name, param in text_model_dict.items(): + set_module_tensor_to_device(text_model, param_name, "cpu", value=param) + else: + if not (hasattr(text_model, "embeddings") and hasattr(text_model.embeddings.position_ids)): + text_model_dict.pop("text_model.embeddings.position_ids", None) + + text_model.load_state_dict(text_model_dict) + + return text_model + + +textenc_conversion_lst = [ + ("positional_embedding", "text_model.embeddings.position_embedding.weight"), + ("token_embedding.weight", "text_model.embeddings.token_embedding.weight"), + ("ln_final.weight", "text_model.final_layer_norm.weight"), + ("ln_final.bias", "text_model.final_layer_norm.bias"), + ("text_projection", "text_projection.weight"), +] +textenc_conversion_map = {x[0]: x[1] for x in textenc_conversion_lst} + +textenc_transformer_conversion_lst = [ + # (stable-diffusion, HF Diffusers) + ("resblocks.", "text_model.encoder.layers."), + ("ln_1", "layer_norm1"), + ("ln_2", "layer_norm2"), + (".c_fc.", ".fc1."), + (".c_proj.", ".fc2."), + (".attn", ".self_attn"), + ("ln_final.", "transformer.text_model.final_layer_norm."), + ("token_embedding.weight", "transformer.text_model.embeddings.token_embedding.weight"), + ("positional_embedding", "transformer.text_model.embeddings.position_embedding.weight"), +] +protected = {re.escape(x[0]): x[1] for x in textenc_transformer_conversion_lst} +textenc_pattern = re.compile("|".join(protected.keys())) + + +def convert_paint_by_example_checkpoint(checkpoint, local_files_only=False): + config = CLIPVisionConfig.from_pretrained("openai/clip-vit-large-patch14", local_files_only=local_files_only) + model = PaintByExampleImageEncoder(config) + + keys = list(checkpoint.keys()) + + text_model_dict = {} + + for key in keys: + if key.startswith("cond_stage_model.transformer"): + text_model_dict[key[len("cond_stage_model.transformer.") :]] = checkpoint[key] + + # load clip vision + model.model.load_state_dict(text_model_dict) + + # load mapper + keys_mapper = { + k[len("cond_stage_model.mapper.res") :]: v + for k, v in checkpoint.items() + if k.startswith("cond_stage_model.mapper") + } + + MAPPING = { + "attn.c_qkv": ["attn1.to_q", "attn1.to_k", "attn1.to_v"], + "attn.c_proj": ["attn1.to_out.0"], + "ln_1": ["norm1"], + "ln_2": ["norm3"], + "mlp.c_fc": ["ff.net.0.proj"], + "mlp.c_proj": ["ff.net.2"], + } + + mapped_weights = {} + for key, value in keys_mapper.items(): + prefix = key[: len("blocks.i")] + suffix = key.split(prefix)[-1].split(".")[-1] + name = key.split(prefix)[-1].split(suffix)[0][1:-1] + mapped_names = MAPPING[name] + + num_splits = len(mapped_names) + for i, mapped_name in enumerate(mapped_names): + new_name = ".".join([prefix, mapped_name, suffix]) + shape = value.shape[0] // num_splits + mapped_weights[new_name] = value[i * shape : (i + 1) * shape] + + model.mapper.load_state_dict(mapped_weights) + + # load final layer norm + model.final_layer_norm.load_state_dict( + { + "bias": checkpoint["cond_stage_model.final_ln.bias"], + "weight": checkpoint["cond_stage_model.final_ln.weight"], + } + ) + + # load final proj + model.proj_out.load_state_dict( + { + "bias": checkpoint["proj_out.bias"], + "weight": checkpoint["proj_out.weight"], + } + ) + + # load uncond vector + model.uncond_vector.data = torch.nn.Parameter(checkpoint["learnable_vector"]) + return model + + +def convert_open_clip_checkpoint( + checkpoint, + config_name, + prefix="cond_stage_model.model.", + has_projection=False, + local_files_only=False, + **config_kwargs, +): + # text_model = CLIPTextModel.from_pretrained("stabilityai/stable-diffusion-2", subfolder="text_encoder") + # text_model = CLIPTextModelWithProjection.from_pretrained( + # "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", projection_dim=1280 + # ) + try: + config = CLIPTextConfig.from_pretrained(config_name, **config_kwargs, local_files_only=local_files_only) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the configuration in the following path: '{config_name}'." + ) + + ctx = init_empty_weights if is_accelerate_available() else nullcontext + with ctx(): + text_model = CLIPTextModelWithProjection(config) if has_projection else CLIPTextModel(config) + + keys = list(checkpoint.keys()) + + keys_to_ignore = [] + if config_name == "stabilityai/stable-diffusion-2" and config.num_hidden_layers == 23: + # make sure to remove all keys > 22 + keys_to_ignore += [k for k in keys if k.startswith("cond_stage_model.model.transformer.resblocks.23")] + keys_to_ignore += ["cond_stage_model.model.text_projection"] + + text_model_dict = {} + + if prefix + "text_projection" in checkpoint: + d_model = int(checkpoint[prefix + "text_projection"].shape[0]) + else: + d_model = 1024 + + text_model_dict["text_model.embeddings.position_ids"] = text_model.text_model.embeddings.get_buffer("position_ids") + + for key in keys: + if key in keys_to_ignore: + continue + if key[len(prefix) :] in textenc_conversion_map: + if key.endswith("text_projection"): + value = checkpoint[key].T.contiguous() + else: + value = checkpoint[key] + + text_model_dict[textenc_conversion_map[key[len(prefix) :]]] = value + + if key.startswith(prefix + "transformer."): + new_key = key[len(prefix + "transformer.") :] + if new_key.endswith(".in_proj_weight"): + new_key = new_key[: -len(".in_proj_weight")] + new_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], new_key) + text_model_dict[new_key + ".q_proj.weight"] = checkpoint[key][:d_model, :] + text_model_dict[new_key + ".k_proj.weight"] = checkpoint[key][d_model : d_model * 2, :] + text_model_dict[new_key + ".v_proj.weight"] = checkpoint[key][d_model * 2 :, :] + elif new_key.endswith(".in_proj_bias"): + new_key = new_key[: -len(".in_proj_bias")] + new_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], new_key) + text_model_dict[new_key + ".q_proj.bias"] = checkpoint[key][:d_model] + text_model_dict[new_key + ".k_proj.bias"] = checkpoint[key][d_model : d_model * 2] + text_model_dict[new_key + ".v_proj.bias"] = checkpoint[key][d_model * 2 :] + else: + new_key = textenc_pattern.sub(lambda m: protected[re.escape(m.group(0))], new_key) + + text_model_dict[new_key] = checkpoint[key] + + if is_accelerate_available(): + for param_name, param in text_model_dict.items(): + set_module_tensor_to_device(text_model, param_name, "cpu", value=param) + else: + if not (hasattr(text_model, "embeddings") and hasattr(text_model.embeddings.position_ids)): + text_model_dict.pop("text_model.embeddings.position_ids", None) + + text_model.load_state_dict(text_model_dict) + + return text_model + + +def stable_unclip_image_encoder(original_config, local_files_only=False): + """ + Returns the image processor and clip image encoder for the img2img unclip pipeline. + + We currently know of two types of stable unclip models which separately use the clip and the openclip image + encoders. + """ + + image_embedder_config = original_config["model"]["params"]["embedder_config"] + + sd_clip_image_embedder_class = image_embedder_config["target"] + sd_clip_image_embedder_class = sd_clip_image_embedder_class.split(".")[-1] + + if sd_clip_image_embedder_class == "ClipImageEmbedder": + clip_model_name = image_embedder_config.params.model + + if clip_model_name == "ViT-L/14": + feature_extractor = CLIPImageProcessor() + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + "openai/clip-vit-large-patch14", local_files_only=local_files_only + ) + else: + raise NotImplementedError(f"Unknown CLIP checkpoint name in stable diffusion checkpoint {clip_model_name}") + + elif sd_clip_image_embedder_class == "FrozenOpenCLIPImageEmbedder": + feature_extractor = CLIPImageProcessor() + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + "laion/CLIP-ViT-H-14-laion2B-s32B-b79K", local_files_only=local_files_only + ) + else: + raise NotImplementedError( + f"Unknown CLIP image embedder class in stable diffusion checkpoint {sd_clip_image_embedder_class}" + ) + + return feature_extractor, image_encoder + + +def stable_unclip_image_noising_components( + original_config, clip_stats_path: Optional[str] = None, device: Optional[str] = None +): + """ + Returns the noising components for the img2img and txt2img unclip pipelines. + + Converts the stability noise augmentor into + 1. a `StableUnCLIPImageNormalizer` for holding the CLIP stats + 2. a `DDPMScheduler` for holding the noise schedule + + If the noise augmentor config specifies a clip stats path, the `clip_stats_path` must be provided. + """ + noise_aug_config = original_config["model"]["params"]["noise_aug_config"] + noise_aug_class = noise_aug_config["target"] + noise_aug_class = noise_aug_class.split(".")[-1] + + if noise_aug_class == "CLIPEmbeddingNoiseAugmentation": + noise_aug_config = noise_aug_config.params + embedding_dim = noise_aug_config.timestep_dim + max_noise_level = noise_aug_config.noise_schedule_config.timesteps + beta_schedule = noise_aug_config.noise_schedule_config.beta_schedule + + image_normalizer = StableUnCLIPImageNormalizer(embedding_dim=embedding_dim) + image_noising_scheduler = DDPMScheduler(num_train_timesteps=max_noise_level, beta_schedule=beta_schedule) + + if "clip_stats_path" in noise_aug_config: + if clip_stats_path is None: + raise ValueError("This stable unclip config requires a `clip_stats_path`") + + clip_mean, clip_std = torch.load(clip_stats_path, map_location=device) + clip_mean = clip_mean[None, :] + clip_std = clip_std[None, :] + + clip_stats_state_dict = { + "mean": clip_mean, + "std": clip_std, + } + + image_normalizer.load_state_dict(clip_stats_state_dict) + else: + raise NotImplementedError(f"Unknown noise augmentor class: {noise_aug_class}") + + return image_normalizer, image_noising_scheduler + + +def convert_controlnet_checkpoint( + checkpoint, + original_config, + checkpoint_path, + image_size, + upcast_attention, + extract_ema, + use_linear_projection=None, + cross_attention_dim=None, +): + ctrlnet_config = create_unet_diffusers_config(original_config, image_size=image_size, controlnet=True) + ctrlnet_config["upcast_attention"] = upcast_attention + + ctrlnet_config.pop("sample_size") + + if use_linear_projection is not None: + ctrlnet_config["use_linear_projection"] = use_linear_projection + + if cross_attention_dim is not None: + ctrlnet_config["cross_attention_dim"] = cross_attention_dim + + ctx = init_empty_weights if is_accelerate_available() else nullcontext + with ctx(): + controlnet = ControlNetModel(**ctrlnet_config) + + # Some controlnet ckpt files are distributed independently from the rest of the + # model components i.e. https://huggingface.co/thibaud/controlnet-sd21/ + if "time_embed.0.weight" in checkpoint: + skip_extract_state_dict = True + else: + skip_extract_state_dict = False + + converted_ctrl_checkpoint = convert_ldm_unet_checkpoint( + checkpoint, + ctrlnet_config, + path=checkpoint_path, + extract_ema=extract_ema, + controlnet=True, + skip_extract_state_dict=skip_extract_state_dict, + ) + + if is_accelerate_available(): + for param_name, param in converted_ctrl_checkpoint.items(): + set_module_tensor_to_device(controlnet, param_name, "cpu", value=param) + else: + controlnet.load_state_dict(converted_ctrl_checkpoint) + + return controlnet + + +def download_from_original_stable_diffusion_ckpt( + checkpoint_path_or_dict: Union[str, Dict[str, torch.Tensor]], + original_config_file: str = None, + image_size: Optional[int] = None, + prediction_type: str = None, + model_type: str = None, + extract_ema: bool = False, + scheduler_type: str = "pndm", + num_in_channels: Optional[int] = None, + upcast_attention: Optional[bool] = None, + device: str = None, + from_safetensors: bool = False, + stable_unclip: Optional[str] = None, + stable_unclip_prior: Optional[str] = None, + clip_stats_path: Optional[str] = None, + controlnet: Optional[bool] = None, + adapter: Optional[bool] = None, + load_safety_checker: bool = True, + pipeline_class: DiffusionPipeline = None, + local_files_only=False, + vae_path=None, + vae=None, + text_encoder=None, + text_encoder_2=None, + tokenizer=None, + tokenizer_2=None, + config_files=None, +) -> DiffusionPipeline: + """ + Load a Stable Diffusion pipeline object from a CompVis-style `.ckpt`/`.safetensors` file and (ideally) a `.yaml` + config file. + + Although many of the arguments can be automatically inferred, some of these rely on brittle checks against the + global step count, which will likely fail for models that have undergone further fine-tuning. Therefore, it is + recommended that you override the default values and/or supply an `original_config_file` wherever possible. + + Args: + checkpoint_path_or_dict (`str` or `dict`): Path to `.ckpt` file, or the state dict. + original_config_file (`str`): + Path to `.yaml` config file corresponding to the original architecture. If `None`, will be automatically + inferred by looking for a key that only exists in SD2.0 models. + image_size (`int`, *optional*, defaults to 512): + The image size that the model was trained on. Use 512 for Stable Diffusion v1.X and Stable Diffusion v2 + Base. Use 768 for Stable Diffusion v2. + prediction_type (`str`, *optional*): + The prediction type that the model was trained on. Use `'epsilon'` for Stable Diffusion v1.X and Stable + Diffusion v2 Base. Use `'v_prediction'` for Stable Diffusion v2. + num_in_channels (`int`, *optional*, defaults to None): + The number of input channels. If `None`, it will be automatically inferred. + scheduler_type (`str`, *optional*, defaults to 'pndm'): + Type of scheduler to use. Should be one of `["pndm", "lms", "heun", "euler", "euler-ancestral", "dpm", + "ddim"]`. + model_type (`str`, *optional*, defaults to `None`): + The pipeline type. `None` to automatically infer, or one of `["FrozenOpenCLIPEmbedder", + "FrozenCLIPEmbedder", "PaintByExample"]`. + is_img2img (`bool`, *optional*, defaults to `False`): + Whether the model should be loaded as an img2img pipeline. + extract_ema (`bool`, *optional*, defaults to `False`): Only relevant for + checkpoints that have both EMA and non-EMA weights. Whether to extract the EMA weights or not. Defaults to + `False`. Pass `True` to extract the EMA weights. EMA weights usually yield higher quality images for + inference. Non-EMA weights are usually better to continue fine-tuning. + upcast_attention (`bool`, *optional*, defaults to `None`): + Whether the attention computation should always be upcasted. This is necessary when running stable + diffusion 2.1. + device (`str`, *optional*, defaults to `None`): + The device to use. Pass `None` to determine automatically. + from_safetensors (`str`, *optional*, defaults to `False`): + If `checkpoint_path` is in `safetensors` format, load checkpoint with safetensors instead of PyTorch. + load_safety_checker (`bool`, *optional*, defaults to `True`): + Whether to load the safety checker or not. Defaults to `True`. + pipeline_class (`str`, *optional*, defaults to `None`): + The pipeline class to use. Pass `None` to determine automatically. + local_files_only (`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (i.e., do not try to download the model). + vae (`AutoencoderKL`, *optional*, defaults to `None`): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. If + this parameter is `None`, the function will load a new instance of [CLIP] by itself, if needed. + text_encoder (`CLIPTextModel`, *optional*, defaults to `None`): + An instance of [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel) + to use, specifically the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) + variant. If this parameter is `None`, the function will load a new instance of [CLIP] by itself, if needed. + tokenizer (`CLIPTokenizer`, *optional*, defaults to `None`): + An instance of + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer) + to use. If this parameter is `None`, the function will load a new instance of [CLIPTokenizer] by itself, if + needed. + config_files (`Dict[str, str]`, *optional*, defaults to `None`): + A dictionary mapping from config file names to their contents. If this parameter is `None`, the function + will load the config files by itself, if needed. Valid keys are: + - `v1`: Config file for Stable Diffusion v1 + - `v2`: Config file for Stable Diffusion v2 + - `xl`: Config file for Stable Diffusion XL + - `xl_refiner`: Config file for Stable Diffusion XL Refiner + return: A StableDiffusionPipeline object representing the passed-in `.ckpt`/`.safetensors` file. + """ + + # import pipelines here to avoid circular import error when using from_single_file method + from diffusers import ( + LDMTextToImagePipeline, + PaintByExamplePipeline, + StableDiffusionControlNetPipeline, + StableDiffusionInpaintPipeline, + StableDiffusionPipeline, + StableDiffusionUpscalePipeline, + StableDiffusionXLControlNetInpaintPipeline, + StableDiffusionXLImg2ImgPipeline, + StableDiffusionXLInpaintPipeline, + StableDiffusionXLPipeline, + StableUnCLIPImg2ImgPipeline, + StableUnCLIPPipeline, + ) + + if prediction_type == "v-prediction": + prediction_type = "v_prediction" + + if isinstance(checkpoint_path_or_dict, str): + if from_safetensors: + from safetensors.torch import load_file as safe_load + + checkpoint = safe_load(checkpoint_path_or_dict, device="cpu") + else: + if device is None: + device = "cuda" if torch.cuda.is_available() else "cpu" + checkpoint = torch.load(checkpoint_path_or_dict, map_location=device) + else: + checkpoint = torch.load(checkpoint_path_or_dict, map_location=device) + elif isinstance(checkpoint_path_or_dict, dict): + checkpoint = checkpoint_path_or_dict + + # Sometimes models don't have the global_step item + if "global_step" in checkpoint: + global_step = checkpoint["global_step"] + else: + logger.debug("global_step key not found in model") + global_step = None + + # NOTE: this while loop isn't great but this controlnet checkpoint has one additional + # "state_dict" key https://huggingface.co/thibaud/controlnet-canny-sd21 + while "state_dict" in checkpoint: + checkpoint = checkpoint["state_dict"] + + if original_config_file is None: + key_name_v2_1 = "model.diffusion_model.input_blocks.2.1.transformer_blocks.0.attn2.to_k.weight" + key_name_sd_xl_base = "conditioner.embedders.1.model.transformer.resblocks.9.mlp.c_proj.bias" + key_name_sd_xl_refiner = "conditioner.embedders.0.model.transformer.resblocks.9.mlp.c_proj.bias" + is_upscale = pipeline_class == StableDiffusionUpscalePipeline + + config_url = None + + # model_type = "v1" + if config_files is not None and "v1" in config_files: + original_config_file = config_files["v1"] + else: + config_url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/configs/stable-diffusion/v1-inference.yaml" + + if key_name_v2_1 in checkpoint and checkpoint[key_name_v2_1].shape[-1] == 1024: + # model_type = "v2" + if config_files is not None and "v2" in config_files: + original_config_file = config_files["v2"] + else: + config_url = "https://raw.githubusercontent.com/Stability-AI/stablediffusion/main/configs/stable-diffusion/v2-inference-v.yaml" + if global_step == 110000: + # v2.1 needs to upcast attention + upcast_attention = True + elif key_name_sd_xl_base in checkpoint: + # only base xl has two text embedders + if config_files is not None and "xl" in config_files: + original_config_file = config_files["xl"] + else: + config_url = "https://raw.githubusercontent.com/Stability-AI/generative-models/main/configs/inference/sd_xl_base.yaml" + elif key_name_sd_xl_refiner in checkpoint: + # only refiner xl has embedder and one text embedders + if config_files is not None and "xl_refiner" in config_files: + original_config_file = config_files["xl_refiner"] + else: + config_url = "https://raw.githubusercontent.com/Stability-AI/generative-models/main/configs/inference/sd_xl_refiner.yaml" + + if is_upscale: + config_url = "https://raw.githubusercontent.com/Stability-AI/stablediffusion/main/configs/stable-diffusion/x4-upscaling.yaml" + + if config_url is not None: + original_config_file = BytesIO(requests.get(config_url).content) + else: + with open(original_config_file, "r") as f: + original_config_file = f.read() + else: + with open(original_config_file, "r") as f: + original_config_file = f.read() + + original_config = yaml.safe_load(original_config_file) + + # Convert the text model. + if ( + model_type is None + and "cond_stage_config" in original_config["model"]["params"] + and original_config["model"]["params"]["cond_stage_config"] is not None + ): + model_type = original_config["model"]["params"]["cond_stage_config"]["target"].split(".")[-1] + logger.debug(f"no `model_type` given, `model_type` inferred as: {model_type}") + elif model_type is None and original_config["model"]["params"]["network_config"] is not None: + if original_config["model"]["params"]["network_config"]["params"]["context_dim"] == 2048: + model_type = "SDXL" + else: + model_type = "SDXL-Refiner" + if image_size is None: + image_size = 1024 + + if pipeline_class is None: + # Check if we have a SDXL or SD model and initialize default pipeline + if model_type not in ["SDXL", "SDXL-Refiner"]: + pipeline_class = StableDiffusionPipeline if not controlnet else StableDiffusionControlNetPipeline + else: + pipeline_class = StableDiffusionXLPipeline if model_type == "SDXL" else StableDiffusionXLImg2ImgPipeline + + if num_in_channels is None and pipeline_class in [ + StableDiffusionInpaintPipeline, + StableDiffusionXLInpaintPipeline, + StableDiffusionXLControlNetInpaintPipeline, + ]: + num_in_channels = 9 + if num_in_channels is None and pipeline_class == StableDiffusionUpscalePipeline: + num_in_channels = 7 + elif num_in_channels is None: + num_in_channels = 4 + + if "unet_config" in original_config["model"]["params"]: + original_config["model"]["params"]["unet_config"]["params"]["in_channels"] = num_in_channels + + if ( + "parameterization" in original_config["model"]["params"] + and original_config["model"]["params"]["parameterization"] == "v" + ): + if prediction_type is None: + # NOTE: For stable diffusion 2 base it is recommended to pass `prediction_type=="epsilon"` + # as it relies on a brittle global step parameter here + prediction_type = "epsilon" if global_step == 875000 else "v_prediction" + if image_size is None: + # NOTE: For stable diffusion 2 base one has to pass `image_size==512` + # as it relies on a brittle global step parameter here + image_size = 512 if global_step == 875000 else 768 + else: + if prediction_type is None: + prediction_type = "epsilon" + if image_size is None: + image_size = 512 + + if controlnet is None and "control_stage_config" in original_config["model"]["params"]: + path = checkpoint_path_or_dict if isinstance(checkpoint_path_or_dict, str) else "" + controlnet = convert_controlnet_checkpoint( + checkpoint, original_config, path, image_size, upcast_attention, extract_ema + ) + + if "timesteps" in original_config["model"]["params"]: + num_train_timesteps = original_config["model"]["params"]["timesteps"] + else: + num_train_timesteps = 1000 + + if model_type in ["SDXL", "SDXL-Refiner"]: + scheduler_dict = { + "beta_schedule": "scaled_linear", + "beta_start": 0.00085, + "beta_end": 0.012, + "interpolation_type": "linear", + "num_train_timesteps": num_train_timesteps, + "prediction_type": "epsilon", + "sample_max_value": 1.0, + "set_alpha_to_one": False, + "skip_prk_steps": True, + "steps_offset": 1, + "timestep_spacing": "leading", + } + scheduler = EulerDiscreteScheduler.from_config(scheduler_dict) + scheduler_type = "euler" + else: + if "linear_start" in original_config["model"]["params"]: + beta_start = original_config["model"]["params"]["linear_start"] + else: + beta_start = 0.02 + + if "linear_end" in original_config["model"]["params"]: + beta_end = original_config["model"]["params"]["linear_end"] + else: + beta_end = 0.085 + scheduler = DDIMScheduler( + beta_end=beta_end, + beta_schedule="scaled_linear", + beta_start=beta_start, + num_train_timesteps=num_train_timesteps, + steps_offset=1, + clip_sample=False, + set_alpha_to_one=False, + prediction_type=prediction_type, + ) + # make sure scheduler works correctly with DDIM + scheduler.register_to_config(clip_sample=False) + + if scheduler_type == "pndm": + config = dict(scheduler.config) + config["skip_prk_steps"] = True + scheduler = PNDMScheduler.from_config(config) + elif scheduler_type == "lms": + scheduler = LMSDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "heun": + scheduler = HeunDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "euler": + scheduler = EulerDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "euler-ancestral": + scheduler = EulerAncestralDiscreteScheduler.from_config(scheduler.config) + elif scheduler_type == "dpm": + scheduler = DPMSolverMultistepScheduler.from_config(scheduler.config) + elif scheduler_type == "ddim": + scheduler = scheduler + else: + raise ValueError(f"Scheduler of type {scheduler_type} doesn't exist!") + + if pipeline_class == StableDiffusionUpscalePipeline: + image_size = original_config["model"]["params"]["unet_config"]["params"]["image_size"] + + # Convert the UNet2DConditionModel model. + unet_config = create_unet_diffusers_config(original_config, image_size=image_size) + unet_config["upcast_attention"] = upcast_attention + + path = checkpoint_path_or_dict if isinstance(checkpoint_path_or_dict, str) else "" + converted_unet_checkpoint = convert_ldm_unet_checkpoint( + checkpoint, unet_config, path=path, extract_ema=extract_ema + ) + + ctx = init_empty_weights if is_accelerate_available() else nullcontext + with ctx(): + unet = UNet2DConditionModel(**unet_config) + + if is_accelerate_available(): + if model_type not in ["SDXL", "SDXL-Refiner"]: # SBM Delay this. + for param_name, param in converted_unet_checkpoint.items(): + set_module_tensor_to_device(unet, param_name, "cpu", value=param) + else: + unet.load_state_dict(converted_unet_checkpoint) + + # Convert the VAE model. + if vae_path is None and vae is None: + vae_config = create_vae_diffusers_config(original_config, image_size=image_size) + converted_vae_checkpoint = convert_ldm_vae_checkpoint(checkpoint, vae_config) + + if ( + "model" in original_config + and "params" in original_config["model"] + and "scale_factor" in original_config["model"]["params"] + ): + vae_scaling_factor = original_config["model"]["params"]["scale_factor"] + else: + vae_scaling_factor = 0.18215 # default SD scaling factor + + vae_config["scaling_factor"] = vae_scaling_factor + + ctx = init_empty_weights if is_accelerate_available() else nullcontext + with ctx(): + vae = AutoencoderKL(**vae_config) + + if is_accelerate_available(): + for param_name, param in converted_vae_checkpoint.items(): + set_module_tensor_to_device(vae, param_name, "cpu", value=param) + else: + vae.load_state_dict(converted_vae_checkpoint) + elif vae is None: + vae = AutoencoderKL.from_pretrained(vae_path, local_files_only=local_files_only) + + if model_type == "FrozenOpenCLIPEmbedder": + config_name = "stabilityai/stable-diffusion-2" + config_kwargs = {"subfolder": "text_encoder"} + + if text_encoder is None: + text_model = convert_open_clip_checkpoint( + checkpoint, config_name, local_files_only=local_files_only, **config_kwargs + ) + else: + text_model = text_encoder + + try: + tokenizer = CLIPTokenizer.from_pretrained( + "stabilityai/stable-diffusion-2", subfolder="tokenizer", local_files_only=local_files_only + ) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the tokenizer in the following path: 'stabilityai/stable-diffusion-2'." + ) + + if stable_unclip is None: + if controlnet: + pipe = pipeline_class( + vae=vae, + text_encoder=text_model, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + controlnet=controlnet, + safety_checker=None, + feature_extractor=None, + ) + if hasattr(pipe, "requires_safety_checker"): + pipe.requires_safety_checker = False + + elif pipeline_class == StableDiffusionUpscalePipeline: + scheduler = DDIMScheduler.from_pretrained( + "stabilityai/stable-diffusion-x4-upscaler", subfolder="scheduler" + ) + low_res_scheduler = DDPMScheduler.from_pretrained( + "stabilityai/stable-diffusion-x4-upscaler", subfolder="low_res_scheduler" + ) + + pipe = pipeline_class( + vae=vae, + text_encoder=text_model, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + low_res_scheduler=low_res_scheduler, + safety_checker=None, + feature_extractor=None, + ) + + else: + pipe = pipeline_class( + vae=vae, + text_encoder=text_model, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=None, + feature_extractor=None, + ) + if hasattr(pipe, "requires_safety_checker"): + pipe.requires_safety_checker = False + + else: + image_normalizer, image_noising_scheduler = stable_unclip_image_noising_components( + original_config, clip_stats_path=clip_stats_path, device=device + ) + + if stable_unclip == "img2img": + feature_extractor, image_encoder = stable_unclip_image_encoder(original_config) + + pipe = StableUnCLIPImg2ImgPipeline( + # image encoding components + feature_extractor=feature_extractor, + image_encoder=image_encoder, + # image noising components + image_normalizer=image_normalizer, + image_noising_scheduler=image_noising_scheduler, + # regular denoising components + tokenizer=tokenizer, + text_encoder=text_model, + unet=unet, + scheduler=scheduler, + # vae + vae=vae, + ) + elif stable_unclip == "txt2img": + if stable_unclip_prior is None or stable_unclip_prior == "karlo": + karlo_model = "kakaobrain/karlo-v1-alpha" + prior = PriorTransformer.from_pretrained( + karlo_model, subfolder="prior", local_files_only=local_files_only + ) + + try: + prior_tokenizer = CLIPTokenizer.from_pretrained( + "openai/clip-vit-large-patch14", local_files_only=local_files_only + ) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the tokenizer in the following path: 'openai/clip-vit-large-patch14'." + ) + prior_text_model = CLIPTextModelWithProjection.from_pretrained( + "openai/clip-vit-large-patch14", local_files_only=local_files_only + ) + + prior_scheduler = UnCLIPScheduler.from_pretrained( + karlo_model, subfolder="prior_scheduler", local_files_only=local_files_only + ) + prior_scheduler = DDPMScheduler.from_config(prior_scheduler.config) + else: + raise NotImplementedError(f"unknown prior for stable unclip model: {stable_unclip_prior}") + + pipe = StableUnCLIPPipeline( + # prior components + prior_tokenizer=prior_tokenizer, + prior_text_encoder=prior_text_model, + prior=prior, + prior_scheduler=prior_scheduler, + # image noising components + image_normalizer=image_normalizer, + image_noising_scheduler=image_noising_scheduler, + # regular denoising components + tokenizer=tokenizer, + text_encoder=text_model, + unet=unet, + scheduler=scheduler, + # vae + vae=vae, + ) + else: + raise NotImplementedError(f"unknown `stable_unclip` type: {stable_unclip}") + elif model_type == "PaintByExample": + vision_model = convert_paint_by_example_checkpoint(checkpoint) + try: + tokenizer = CLIPTokenizer.from_pretrained( + "openai/clip-vit-large-patch14", local_files_only=local_files_only + ) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the tokenizer in the following path: 'openai/clip-vit-large-patch14'." + ) + try: + feature_extractor = AutoFeatureExtractor.from_pretrained( + "CompVis/stable-diffusion-safety-checker", local_files_only=local_files_only + ) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the feature_extractor in the following path: 'CompVis/stable-diffusion-safety-checker'." + ) + pipe = PaintByExamplePipeline( + vae=vae, + image_encoder=vision_model, + unet=unet, + scheduler=scheduler, + safety_checker=None, + feature_extractor=feature_extractor, + ) + elif model_type == "FrozenCLIPEmbedder": + text_model = convert_ldm_clip_checkpoint( + checkpoint, local_files_only=local_files_only, text_encoder=text_encoder + ) + try: + tokenizer = ( + CLIPTokenizer.from_pretrained("openai/clip-vit-large-patch14", local_files_only=local_files_only) + if tokenizer is None + else tokenizer + ) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the tokenizer in the following path: 'openai/clip-vit-large-patch14'." + ) + + if load_safety_checker: + safety_checker = StableDiffusionSafetyChecker.from_pretrained( + "CompVis/stable-diffusion-safety-checker", local_files_only=local_files_only + ) + feature_extractor = AutoFeatureExtractor.from_pretrained( + "CompVis/stable-diffusion-safety-checker", local_files_only=local_files_only + ) + else: + safety_checker = None + feature_extractor = None + + if controlnet: + pipe = pipeline_class( + vae=vae, + text_encoder=text_model, + tokenizer=tokenizer, + unet=unet, + controlnet=controlnet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + else: + pipe = pipeline_class( + vae=vae, + text_encoder=text_model, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + elif model_type in ["SDXL", "SDXL-Refiner"]: + is_refiner = model_type == "SDXL-Refiner" + + if (is_refiner is False) and (tokenizer is None): + try: + tokenizer = CLIPTokenizer.from_pretrained( + "openai/clip-vit-large-patch14", local_files_only=local_files_only + ) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the tokenizer in the following path: 'openai/clip-vit-large-patch14'." + ) + + if (is_refiner is False) and (text_encoder is None): + text_encoder = convert_ldm_clip_checkpoint(checkpoint, local_files_only=local_files_only) + + if tokenizer_2 is None: + try: + tokenizer_2 = CLIPTokenizer.from_pretrained( + "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k", pad_token="!", local_files_only=local_files_only + ) + except Exception: + raise ValueError( + f"With local_files_only set to {local_files_only}, you must first locally save the tokenizer in the following path: 'laion/CLIP-ViT-bigG-14-laion2B-39B-b160k' with `pad_token` set to '!'." + ) + + if text_encoder_2 is None: + config_name = "laion/CLIP-ViT-bigG-14-laion2B-39B-b160k" + config_kwargs = {"projection_dim": 1280} + prefix = "conditioner.embedders.0.model." if is_refiner else "conditioner.embedders.1.model." + + text_encoder_2 = convert_open_clip_checkpoint( + checkpoint, + config_name, + prefix=prefix, + has_projection=True, + local_files_only=local_files_only, + **config_kwargs, + ) + + if is_accelerate_available(): # SBM Now move model to cpu. + for param_name, param in converted_unet_checkpoint.items(): + set_module_tensor_to_device(unet, param_name, "cpu", value=param) + + if controlnet: + pipe = pipeline_class( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + text_encoder_2=text_encoder_2, + tokenizer_2=tokenizer_2, + unet=unet, + controlnet=controlnet, + scheduler=scheduler, + force_zeros_for_empty_prompt=True, + ) + elif adapter: + pipe = pipeline_class( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + text_encoder_2=text_encoder_2, + tokenizer_2=tokenizer_2, + unet=unet, + adapter=adapter, + scheduler=scheduler, + force_zeros_for_empty_prompt=True, + ) + + else: + pipeline_kwargs = { + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "unet": unet, + "scheduler": scheduler, + } + + if (pipeline_class == StableDiffusionXLImg2ImgPipeline) or ( + pipeline_class == StableDiffusionXLInpaintPipeline + ): + pipeline_kwargs.update({"requires_aesthetics_score": is_refiner}) + + if is_refiner: + pipeline_kwargs.update({"force_zeros_for_empty_prompt": False}) + + pipe = pipeline_class(**pipeline_kwargs) + else: + text_config = create_ldm_bert_config(original_config) + text_model = convert_ldm_bert_checkpoint(checkpoint, text_config) + tokenizer = BertTokenizerFast.from_pretrained("bert-base-uncased", local_files_only=local_files_only) + pipe = LDMTextToImagePipeline(vqvae=vae, bert=text_model, tokenizer=tokenizer, unet=unet, scheduler=scheduler) + + return pipe + + +def download_controlnet_from_original_ckpt( + checkpoint_path: str, + original_config_file: str, + image_size: int = 512, + extract_ema: bool = False, + num_in_channels: Optional[int] = None, + upcast_attention: Optional[bool] = None, + device: str = None, + from_safetensors: bool = False, + use_linear_projection: Optional[bool] = None, + cross_attention_dim: Optional[bool] = None, +) -> DiffusionPipeline: + if from_safetensors: + from safetensors import safe_open + + checkpoint = {} + with safe_open(checkpoint_path, framework="pt", device="cpu") as f: + for key in f.keys(): + checkpoint[key] = f.get_tensor(key) + else: + if device is None: + device = "cuda" if torch.cuda.is_available() else "cpu" + checkpoint = torch.load(checkpoint_path, map_location=device) + else: + checkpoint = torch.load(checkpoint_path, map_location=device) + + # NOTE: this while loop isn't great but this controlnet checkpoint has one additional + # "state_dict" key https://huggingface.co/thibaud/controlnet-canny-sd21 + while "state_dict" in checkpoint: + checkpoint = checkpoint["state_dict"] + + original_config = yaml.safe_load(original_config_file) + + if num_in_channels is not None: + original_config["model"]["params"]["unet_config"]["params"]["in_channels"] = num_in_channels + + if "control_stage_config" not in original_config["model"]["params"]: + raise ValueError("`control_stage_config` not present in original config") + + controlnet = convert_controlnet_checkpoint( + checkpoint, + original_config, + checkpoint_path, + image_size, + upcast_attention, + extract_ema, + use_linear_projection=use_linear_projection, + cross_attention_dim=cross_attention_dim, + ) + + return controlnet diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py new file mode 100755 index 0000000..55ff51c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion.py @@ -0,0 +1,473 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from functools import partial +from typing import Dict, List, Optional, Union + +import jax +import jax.numpy as jnp +import numpy as np +from flax.core.frozen_dict import FrozenDict +from flax.jax_utils import unreplicate +from flax.training.common_utils import shard +from packaging import version +from PIL import Image +from transformers import CLIPImageProcessor, CLIPTokenizer, FlaxCLIPTextModel + +from ...models import FlaxAutoencoderKL, FlaxUNet2DConditionModel +from ...schedulers import ( + FlaxDDIMScheduler, + FlaxDPMSolverMultistepScheduler, + FlaxLMSDiscreteScheduler, + FlaxPNDMScheduler, +) +from ...utils import deprecate, logging, replace_example_docstring +from ..pipeline_flax_utils import FlaxDiffusionPipeline +from .pipeline_output import FlaxStableDiffusionPipelineOutput +from .safety_checker_flax import FlaxStableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +# Set to True to use python for loop instead of jax.fori_loop for easier debugging +DEBUG = False + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import jax + >>> import numpy as np + >>> from flax.jax_utils import replicate + >>> from flax.training.common_utils import shard + + >>> from diffusers import FlaxStableDiffusionPipeline + + >>> pipeline, params = FlaxStableDiffusionPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", revision="bf16", dtype=jax.numpy.bfloat16 + ... ) + + >>> prompt = "a photo of an astronaut riding a horse on mars" + + >>> prng_seed = jax.random.PRNGKey(0) + >>> num_inference_steps = 50 + + >>> num_samples = jax.device_count() + >>> prompt = num_samples * [prompt] + >>> prompt_ids = pipeline.prepare_inputs(prompt) + # shard inputs and rng + + >>> params = replicate(params) + >>> prng_seed = jax.random.split(prng_seed, jax.device_count()) + >>> prompt_ids = shard(prompt_ids) + + >>> images = pipeline(prompt_ids, params, prng_seed, num_inference_steps, jit=True).images + >>> images = pipeline.numpy_to_pil(np.asarray(images.reshape((num_samples,) + images.shape[-3:]))) + ``` +""" + + +class FlaxStableDiffusionPipeline(FlaxDiffusionPipeline): + r""" + Flax-based pipeline for text-to-image generation using Stable Diffusion. + + This model inherits from [`FlaxDiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`FlaxAutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.FlaxCLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`FlaxUNet2DConditionModel`]): + A `FlaxUNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`FlaxDDIMScheduler`], [`FlaxLMSDiscreteScheduler`], [`FlaxPNDMScheduler`], or + [`FlaxDPMSolverMultistepScheduler`]. + safety_checker ([`FlaxStableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + def __init__( + self, + vae: FlaxAutoencoderKL, + text_encoder: FlaxCLIPTextModel, + tokenizer: CLIPTokenizer, + unet: FlaxUNet2DConditionModel, + scheduler: Union[ + FlaxDDIMScheduler, FlaxPNDMScheduler, FlaxLMSDiscreteScheduler, FlaxDPMSolverMultistepScheduler + ], + safety_checker: FlaxStableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + dtype: jnp.dtype = jnp.float32, + ): + super().__init__() + self.dtype = dtype + + if safety_checker is None: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely .If you're checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + + def prepare_inputs(self, prompt: Union[str, List[str]]): + if not isinstance(prompt, (str, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + text_input = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + return text_input.input_ids + + def _get_has_nsfw_concepts(self, features, params): + has_nsfw_concepts = self.safety_checker(features, params) + return has_nsfw_concepts + + def _run_safety_checker(self, images, safety_model_params, jit=False): + # safety_model_params should already be replicated when jit is True + pil_images = [Image.fromarray(image) for image in images] + features = self.feature_extractor(pil_images, return_tensors="np").pixel_values + + if jit: + features = shard(features) + has_nsfw_concepts = _p_get_has_nsfw_concepts(self, features, safety_model_params) + has_nsfw_concepts = unshard(has_nsfw_concepts) + safety_model_params = unreplicate(safety_model_params) + else: + has_nsfw_concepts = self._get_has_nsfw_concepts(features, safety_model_params) + + images_was_copied = False + for idx, has_nsfw_concept in enumerate(has_nsfw_concepts): + if has_nsfw_concept: + if not images_was_copied: + images_was_copied = True + images = images.copy() + + images[idx] = np.zeros(images[idx].shape, dtype=np.uint8) # black image + + if any(has_nsfw_concepts): + warnings.warn( + "Potential NSFW content was detected in one or more images. A black image will be returned" + " instead. Try again with a different prompt and/or seed." + ) + + return images, has_nsfw_concepts + + def _generate( + self, + prompt_ids: jnp.array, + params: Union[Dict, FrozenDict], + prng_seed: jax.Array, + num_inference_steps: int, + height: int, + width: int, + guidance_scale: float, + latents: Optional[jnp.ndarray] = None, + neg_prompt_ids: Optional[jnp.ndarray] = None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + # get prompt text embeddings + prompt_embeds = self.text_encoder(prompt_ids, params=params["text_encoder"])[0] + + # TODO: currently it is assumed `do_classifier_free_guidance = guidance_scale > 1.0` + # implement this conditional `do_classifier_free_guidance = guidance_scale > 1.0` + batch_size = prompt_ids.shape[0] + + max_length = prompt_ids.shape[-1] + + if neg_prompt_ids is None: + uncond_input = self.tokenizer( + [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="np" + ).input_ids + else: + uncond_input = neg_prompt_ids + negative_prompt_embeds = self.text_encoder(uncond_input, params=params["text_encoder"])[0] + context = jnp.concatenate([negative_prompt_embeds, prompt_embeds]) + + # Ensure model output will be `float32` before going into the scheduler + guidance_scale = jnp.array([guidance_scale], dtype=jnp.float32) + + latents_shape = ( + batch_size, + self.unet.config.in_channels, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + if latents is None: + latents = jax.random.normal(prng_seed, shape=latents_shape, dtype=jnp.float32) + else: + if latents.shape != latents_shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}") + + def loop_body(step, args): + latents, scheduler_state = args + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + latents_input = jnp.concatenate([latents] * 2) + + t = jnp.array(scheduler_state.timesteps, dtype=jnp.int32)[step] + timestep = jnp.broadcast_to(t, latents_input.shape[0]) + + latents_input = self.scheduler.scale_model_input(scheduler_state, latents_input, t) + + # predict the noise residual + noise_pred = self.unet.apply( + {"params": params["unet"]}, + jnp.array(latents_input), + jnp.array(timestep, dtype=jnp.int32), + encoder_hidden_states=context, + ).sample + # perform guidance + noise_pred_uncond, noise_prediction_text = jnp.split(noise_pred, 2, axis=0) + noise_pred = noise_pred_uncond + guidance_scale * (noise_prediction_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents, scheduler_state = self.scheduler.step(scheduler_state, noise_pred, t, latents).to_tuple() + return latents, scheduler_state + + scheduler_state = self.scheduler.set_timesteps( + params["scheduler"], num_inference_steps=num_inference_steps, shape=latents.shape + ) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * params["scheduler"].init_noise_sigma + + if DEBUG: + # run with python for loop + for i in range(num_inference_steps): + latents, scheduler_state = loop_body(i, (latents, scheduler_state)) + else: + latents, _ = jax.lax.fori_loop(0, num_inference_steps, loop_body, (latents, scheduler_state)) + + # scale and decode the image latents with vae + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.apply({"params": params["vae"]}, latents, method=self.vae.decode).sample + + image = (image / 2 + 0.5).clip(0, 1).transpose(0, 2, 3, 1) + return image + + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt_ids: jnp.array, + params: Union[Dict, FrozenDict], + prng_seed: jax.Array, + num_inference_steps: int = 50, + height: Optional[int] = None, + width: Optional[int] = None, + guidance_scale: Union[float, jnp.ndarray] = 7.5, + latents: jnp.ndarray = None, + neg_prompt_ids: jnp.ndarray = None, + return_dict: bool = True, + jit: bool = False, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + latents (`jnp.ndarray`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + array is generated by sampling using the supplied random `generator`. + jit (`bool`, defaults to `False`): + Whether to run `pmap` versions of the generation and safety scoring functions. + + + + This argument exists because `__call__` is not yet end-to-end pmap-able. It will be removed in a + future release. + + + + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] instead of + a plain tuple. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] is + returned, otherwise a `tuple` is returned where the first element is a list with the generated images + and the second element is a list of `bool`s indicating whether the corresponding generated image + contains "not-safe-for-work" (nsfw) content. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + if isinstance(guidance_scale, float): + # Convert to a tensor so each device gets a copy. Follow the prompt_ids for + # shape information, as they may be sharded (when `jit` is `True`), or not. + guidance_scale = jnp.array([guidance_scale] * prompt_ids.shape[0]) + if len(prompt_ids.shape) > 2: + # Assume sharded + guidance_scale = guidance_scale[:, None] + + if jit: + images = _p_generate( + self, + prompt_ids, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, + ) + else: + images = self._generate( + prompt_ids, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, + ) + + if self.safety_checker is not None: + safety_params = params["safety_checker"] + images_uint8_casted = (images * 255).round().astype("uint8") + num_devices, batch_size = images.shape[:2] + + images_uint8_casted = np.asarray(images_uint8_casted).reshape(num_devices * batch_size, height, width, 3) + images_uint8_casted, has_nsfw_concept = self._run_safety_checker(images_uint8_casted, safety_params, jit) + images = np.asarray(images).copy() + + # block images + if any(has_nsfw_concept): + for i, is_nsfw in enumerate(has_nsfw_concept): + if is_nsfw: + images[i, 0] = np.asarray(images_uint8_casted[i]) + + images = images.reshape(num_devices, batch_size, height, width, 3) + else: + images = np.asarray(images) + has_nsfw_concept = False + + if not return_dict: + return (images, has_nsfw_concept) + + return FlaxStableDiffusionPipelineOutput(images=images, nsfw_content_detected=has_nsfw_concept) + + +# Static argnums are pipe, num_inference_steps, height, width. A change would trigger recompilation. +# Non-static args are (sharded) input tensors mapped over their first dimension (hence, `0`). +@partial( + jax.pmap, + in_axes=(None, 0, 0, 0, None, None, None, 0, 0, 0), + static_broadcasted_argnums=(0, 4, 5, 6), +) +def _p_generate( + pipe, + prompt_ids, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, +): + return pipe._generate( + prompt_ids, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, + ) + + +@partial(jax.pmap, static_broadcasted_argnums=(0,)) +def _p_get_has_nsfw_concepts(pipe, features, params): + return pipe._get_has_nsfw_concepts(features, params) + + +def unshard(x: jnp.ndarray): + # einops.rearrange(x, 'd b ... -> (d b) ...') + num_devices, batch_size = x.shape[:2] + rest = x.shape[2:] + return x.reshape(num_devices * batch_size, *rest) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_img2img.py new file mode 100755 index 0000000..7792bc0 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_img2img.py @@ -0,0 +1,532 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from functools import partial +from typing import Dict, List, Optional, Union + +import jax +import jax.numpy as jnp +import numpy as np +from flax.core.frozen_dict import FrozenDict +from flax.jax_utils import unreplicate +from flax.training.common_utils import shard +from PIL import Image +from transformers import CLIPImageProcessor, CLIPTokenizer, FlaxCLIPTextModel + +from ...models import FlaxAutoencoderKL, FlaxUNet2DConditionModel +from ...schedulers import ( + FlaxDDIMScheduler, + FlaxDPMSolverMultistepScheduler, + FlaxLMSDiscreteScheduler, + FlaxPNDMScheduler, +) +from ...utils import PIL_INTERPOLATION, logging, replace_example_docstring +from ..pipeline_flax_utils import FlaxDiffusionPipeline +from .pipeline_output import FlaxStableDiffusionPipelineOutput +from .safety_checker_flax import FlaxStableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +# Set to True to use python for loop instead of jax.fori_loop for easier debugging +DEBUG = False + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import jax + >>> import numpy as np + >>> import jax.numpy as jnp + >>> from flax.jax_utils import replicate + >>> from flax.training.common_utils import shard + >>> import requests + >>> from io import BytesIO + >>> from PIL import Image + >>> from diffusers import FlaxStableDiffusionImg2ImgPipeline + + + >>> def create_key(seed=0): + ... return jax.random.PRNGKey(seed) + + + >>> rng = create_key(0) + + >>> url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + >>> response = requests.get(url) + >>> init_img = Image.open(BytesIO(response.content)).convert("RGB") + >>> init_img = init_img.resize((768, 512)) + + >>> prompts = "A fantasy landscape, trending on artstation" + + >>> pipeline, params = FlaxStableDiffusionImg2ImgPipeline.from_pretrained( + ... "CompVis/stable-diffusion-v1-4", + ... revision="flax", + ... dtype=jnp.bfloat16, + ... ) + + >>> num_samples = jax.device_count() + >>> rng = jax.random.split(rng, jax.device_count()) + >>> prompt_ids, processed_image = pipeline.prepare_inputs( + ... prompt=[prompts] * num_samples, image=[init_img] * num_samples + ... ) + >>> p_params = replicate(params) + >>> prompt_ids = shard(prompt_ids) + >>> processed_image = shard(processed_image) + + >>> output = pipeline( + ... prompt_ids=prompt_ids, + ... image=processed_image, + ... params=p_params, + ... prng_seed=rng, + ... strength=0.75, + ... num_inference_steps=50, + ... jit=True, + ... height=512, + ... width=768, + ... ).images + + >>> output_images = pipeline.numpy_to_pil(np.asarray(output.reshape((num_samples,) + output.shape[-3:]))) + ``` +""" + + +class FlaxStableDiffusionImg2ImgPipeline(FlaxDiffusionPipeline): + r""" + Flax-based pipeline for text-guided image-to-image generation using Stable Diffusion. + + This model inherits from [`FlaxDiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`FlaxAutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.FlaxCLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`FlaxUNet2DConditionModel`]): + A `FlaxUNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`FlaxDDIMScheduler`], [`FlaxLMSDiscreteScheduler`], [`FlaxPNDMScheduler`], or + [`FlaxDPMSolverMultistepScheduler`]. + safety_checker ([`FlaxStableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + def __init__( + self, + vae: FlaxAutoencoderKL, + text_encoder: FlaxCLIPTextModel, + tokenizer: CLIPTokenizer, + unet: FlaxUNet2DConditionModel, + scheduler: Union[ + FlaxDDIMScheduler, FlaxPNDMScheduler, FlaxLMSDiscreteScheduler, FlaxDPMSolverMultistepScheduler + ], + safety_checker: FlaxStableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + dtype: jnp.dtype = jnp.float32, + ): + super().__init__() + self.dtype = dtype + + if safety_checker is None: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + + def prepare_inputs(self, prompt: Union[str, List[str]], image: Union[Image.Image, List[Image.Image]]): + if not isinstance(prompt, (str, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if not isinstance(image, (Image.Image, list)): + raise ValueError(f"image has to be of type `PIL.Image.Image` or list but is {type(image)}") + + if isinstance(image, Image.Image): + image = [image] + + processed_images = jnp.concatenate([preprocess(img, jnp.float32) for img in image]) + + text_input = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + return text_input.input_ids, processed_images + + def _get_has_nsfw_concepts(self, features, params): + has_nsfw_concepts = self.safety_checker(features, params) + return has_nsfw_concepts + + def _run_safety_checker(self, images, safety_model_params, jit=False): + # safety_model_params should already be replicated when jit is True + pil_images = [Image.fromarray(image) for image in images] + features = self.feature_extractor(pil_images, return_tensors="np").pixel_values + + if jit: + features = shard(features) + has_nsfw_concepts = _p_get_has_nsfw_concepts(self, features, safety_model_params) + has_nsfw_concepts = unshard(has_nsfw_concepts) + safety_model_params = unreplicate(safety_model_params) + else: + has_nsfw_concepts = self._get_has_nsfw_concepts(features, safety_model_params) + + images_was_copied = False + for idx, has_nsfw_concept in enumerate(has_nsfw_concepts): + if has_nsfw_concept: + if not images_was_copied: + images_was_copied = True + images = images.copy() + + images[idx] = np.zeros(images[idx].shape, dtype=np.uint8) # black image + + if any(has_nsfw_concepts): + warnings.warn( + "Potential NSFW content was detected in one or more images. A black image will be returned" + " instead. Try again with a different prompt and/or seed." + ) + + return images, has_nsfw_concepts + + def get_timestep_start(self, num_inference_steps, strength): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + + return t_start + + def _generate( + self, + prompt_ids: jnp.ndarray, + image: jnp.ndarray, + params: Union[Dict, FrozenDict], + prng_seed: jax.Array, + start_timestep: int, + num_inference_steps: int, + height: int, + width: int, + guidance_scale: float, + noise: Optional[jnp.ndarray] = None, + neg_prompt_ids: Optional[jnp.ndarray] = None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + # get prompt text embeddings + prompt_embeds = self.text_encoder(prompt_ids, params=params["text_encoder"])[0] + + # TODO: currently it is assumed `do_classifier_free_guidance = guidance_scale > 1.0` + # implement this conditional `do_classifier_free_guidance = guidance_scale > 1.0` + batch_size = prompt_ids.shape[0] + + max_length = prompt_ids.shape[-1] + + if neg_prompt_ids is None: + uncond_input = self.tokenizer( + [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="np" + ).input_ids + else: + uncond_input = neg_prompt_ids + negative_prompt_embeds = self.text_encoder(uncond_input, params=params["text_encoder"])[0] + context = jnp.concatenate([negative_prompt_embeds, prompt_embeds]) + + latents_shape = ( + batch_size, + self.unet.config.in_channels, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + if noise is None: + noise = jax.random.normal(prng_seed, shape=latents_shape, dtype=jnp.float32) + else: + if noise.shape != latents_shape: + raise ValueError(f"Unexpected latents shape, got {noise.shape}, expected {latents_shape}") + + # Create init_latents + init_latent_dist = self.vae.apply({"params": params["vae"]}, image, method=self.vae.encode).latent_dist + init_latents = init_latent_dist.sample(key=prng_seed).transpose((0, 3, 1, 2)) + init_latents = self.vae.config.scaling_factor * init_latents + + def loop_body(step, args): + latents, scheduler_state = args + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + latents_input = jnp.concatenate([latents] * 2) + + t = jnp.array(scheduler_state.timesteps, dtype=jnp.int32)[step] + timestep = jnp.broadcast_to(t, latents_input.shape[0]) + + latents_input = self.scheduler.scale_model_input(scheduler_state, latents_input, t) + + # predict the noise residual + noise_pred = self.unet.apply( + {"params": params["unet"]}, + jnp.array(latents_input), + jnp.array(timestep, dtype=jnp.int32), + encoder_hidden_states=context, + ).sample + # perform guidance + noise_pred_uncond, noise_prediction_text = jnp.split(noise_pred, 2, axis=0) + noise_pred = noise_pred_uncond + guidance_scale * (noise_prediction_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents, scheduler_state = self.scheduler.step(scheduler_state, noise_pred, t, latents).to_tuple() + return latents, scheduler_state + + scheduler_state = self.scheduler.set_timesteps( + params["scheduler"], num_inference_steps=num_inference_steps, shape=latents_shape + ) + + latent_timestep = scheduler_state.timesteps[start_timestep : start_timestep + 1].repeat(batch_size) + + latents = self.scheduler.add_noise(params["scheduler"], init_latents, noise, latent_timestep) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * params["scheduler"].init_noise_sigma + + if DEBUG: + # run with python for loop + for i in range(start_timestep, num_inference_steps): + latents, scheduler_state = loop_body(i, (latents, scheduler_state)) + else: + latents, _ = jax.lax.fori_loop(start_timestep, num_inference_steps, loop_body, (latents, scheduler_state)) + + # scale and decode the image latents with vae + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.apply({"params": params["vae"]}, latents, method=self.vae.decode).sample + + image = (image / 2 + 0.5).clip(0, 1).transpose(0, 2, 3, 1) + return image + + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt_ids: jnp.ndarray, + image: jnp.ndarray, + params: Union[Dict, FrozenDict], + prng_seed: jax.Array, + strength: float = 0.8, + num_inference_steps: int = 50, + height: Optional[int] = None, + width: Optional[int] = None, + guidance_scale: Union[float, jnp.ndarray] = 7.5, + noise: jnp.ndarray = None, + neg_prompt_ids: jnp.ndarray = None, + return_dict: bool = True, + jit: bool = False, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt_ids (`jnp.ndarray`): + The prompt or prompts to guide image generation. + image (`jnp.ndarray`): + Array representing an image batch to be used as the starting point. + params (`Dict` or `FrozenDict`): + Dictionary containing the model parameters/weights. + prng_seed (`jax.Array` or `jax.Array`): + Array containing random number generator key. + strength (`float`, *optional*, defaults to 0.8): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + noise (`jnp.ndarray`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. The array is generated by + sampling using the supplied random `generator`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] instead of + a plain tuple. + jit (`bool`, defaults to `False`): + Whether to run `pmap` versions of the generation and safety scoring functions. + + + + This argument exists because `__call__` is not yet end-to-end pmap-able. It will be removed in a + future release. + + + + Examples: + + Returns: + [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] is + returned, otherwise a `tuple` is returned where the first element is a list with the generated images + and the second element is a list of `bool`s indicating whether the corresponding generated image + contains "not-safe-for-work" (nsfw) content. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + if isinstance(guidance_scale, float): + # Convert to a tensor so each device gets a copy. Follow the prompt_ids for + # shape information, as they may be sharded (when `jit` is `True`), or not. + guidance_scale = jnp.array([guidance_scale] * prompt_ids.shape[0]) + if len(prompt_ids.shape) > 2: + # Assume sharded + guidance_scale = guidance_scale[:, None] + + start_timestep = self.get_timestep_start(num_inference_steps, strength) + + if jit: + images = _p_generate( + self, + prompt_ids, + image, + params, + prng_seed, + start_timestep, + num_inference_steps, + height, + width, + guidance_scale, + noise, + neg_prompt_ids, + ) + else: + images = self._generate( + prompt_ids, + image, + params, + prng_seed, + start_timestep, + num_inference_steps, + height, + width, + guidance_scale, + noise, + neg_prompt_ids, + ) + + if self.safety_checker is not None: + safety_params = params["safety_checker"] + images_uint8_casted = (images * 255).round().astype("uint8") + num_devices, batch_size = images.shape[:2] + + images_uint8_casted = np.asarray(images_uint8_casted).reshape(num_devices * batch_size, height, width, 3) + images_uint8_casted, has_nsfw_concept = self._run_safety_checker(images_uint8_casted, safety_params, jit) + images = np.asarray(images) + + # block images + if any(has_nsfw_concept): + for i, is_nsfw in enumerate(has_nsfw_concept): + if is_nsfw: + images[i] = np.asarray(images_uint8_casted[i]) + + images = images.reshape(num_devices, batch_size, height, width, 3) + else: + images = np.asarray(images) + has_nsfw_concept = False + + if not return_dict: + return (images, has_nsfw_concept) + + return FlaxStableDiffusionPipelineOutput(images=images, nsfw_content_detected=has_nsfw_concept) + + +# Static argnums are pipe, start_timestep, num_inference_steps, height, width. A change would trigger recompilation. +# Non-static args are (sharded) input tensors mapped over their first dimension (hence, `0`). +@partial( + jax.pmap, + in_axes=(None, 0, 0, 0, 0, None, None, None, None, 0, 0, 0), + static_broadcasted_argnums=(0, 5, 6, 7, 8), +) +def _p_generate( + pipe, + prompt_ids, + image, + params, + prng_seed, + start_timestep, + num_inference_steps, + height, + width, + guidance_scale, + noise, + neg_prompt_ids, +): + return pipe._generate( + prompt_ids, + image, + params, + prng_seed, + start_timestep, + num_inference_steps, + height, + width, + guidance_scale, + noise, + neg_prompt_ids, + ) + + +@partial(jax.pmap, static_broadcasted_argnums=(0,)) +def _p_get_has_nsfw_concepts(pipe, features, params): + return pipe._get_has_nsfw_concepts(features, params) + + +def unshard(x: jnp.ndarray): + # einops.rearrange(x, 'd b ... -> (d b) ...') + num_devices, batch_size = x.shape[:2] + rest = x.shape[2:] + return x.reshape(num_devices * batch_size, *rest) + + +def preprocess(image, dtype): + w, h = image.size + w, h = (x - x % 32 for x in (w, h)) # resize to integer multiple of 32 + image = image.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]) + image = jnp.array(image).astype(dtype) / 255.0 + image = image[None].transpose(0, 3, 1, 2) + return 2.0 * image - 1.0 diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_inpaint.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_inpaint.py new file mode 100755 index 0000000..f6bb0ac --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_flax_stable_diffusion_inpaint.py @@ -0,0 +1,589 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from functools import partial +from typing import Dict, List, Optional, Union + +import jax +import jax.numpy as jnp +import numpy as np +from flax.core.frozen_dict import FrozenDict +from flax.jax_utils import unreplicate +from flax.training.common_utils import shard +from packaging import version +from PIL import Image +from transformers import CLIPImageProcessor, CLIPTokenizer, FlaxCLIPTextModel + +from ...models import FlaxAutoencoderKL, FlaxUNet2DConditionModel +from ...schedulers import ( + FlaxDDIMScheduler, + FlaxDPMSolverMultistepScheduler, + FlaxLMSDiscreteScheduler, + FlaxPNDMScheduler, +) +from ...utils import PIL_INTERPOLATION, deprecate, logging, replace_example_docstring +from ..pipeline_flax_utils import FlaxDiffusionPipeline +from .pipeline_output import FlaxStableDiffusionPipelineOutput +from .safety_checker_flax import FlaxStableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +# Set to True to use python for loop instead of jax.fori_loop for easier debugging +DEBUG = False + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import jax + >>> import numpy as np + >>> from flax.jax_utils import replicate + >>> from flax.training.common_utils import shard + >>> import PIL + >>> import requests + >>> from io import BytesIO + >>> from diffusers import FlaxStableDiffusionInpaintPipeline + + + >>> def download_image(url): + ... response = requests.get(url) + ... return PIL.Image.open(BytesIO(response.content)).convert("RGB") + + + >>> img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png" + >>> mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png" + + >>> init_image = download_image(img_url).resize((512, 512)) + >>> mask_image = download_image(mask_url).resize((512, 512)) + + >>> pipeline, params = FlaxStableDiffusionInpaintPipeline.from_pretrained( + ... "xvjiarui/stable-diffusion-2-inpainting" + ... ) + + >>> prompt = "Face of a yellow cat, high resolution, sitting on a park bench" + >>> prng_seed = jax.random.PRNGKey(0) + >>> num_inference_steps = 50 + + >>> num_samples = jax.device_count() + >>> prompt = num_samples * [prompt] + >>> init_image = num_samples * [init_image] + >>> mask_image = num_samples * [mask_image] + >>> prompt_ids, processed_masked_images, processed_masks = pipeline.prepare_inputs( + ... prompt, init_image, mask_image + ... ) + # shard inputs and rng + + >>> params = replicate(params) + >>> prng_seed = jax.random.split(prng_seed, jax.device_count()) + >>> prompt_ids = shard(prompt_ids) + >>> processed_masked_images = shard(processed_masked_images) + >>> processed_masks = shard(processed_masks) + + >>> images = pipeline( + ... prompt_ids, processed_masks, processed_masked_images, params, prng_seed, num_inference_steps, jit=True + ... ).images + >>> images = pipeline.numpy_to_pil(np.asarray(images.reshape((num_samples,) + images.shape[-3:]))) + ``` +""" + + +class FlaxStableDiffusionInpaintPipeline(FlaxDiffusionPipeline): + r""" + Flax-based pipeline for text-guided image inpainting using Stable Diffusion. + + + + 🧪 This is an experimental feature! + + + + This model inherits from [`FlaxDiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`FlaxAutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.FlaxCLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`FlaxUNet2DConditionModel`]): + A `FlaxUNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`FlaxDDIMScheduler`], [`FlaxLMSDiscreteScheduler`], [`FlaxPNDMScheduler`], or + [`FlaxDPMSolverMultistepScheduler`]. + safety_checker ([`FlaxStableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + def __init__( + self, + vae: FlaxAutoencoderKL, + text_encoder: FlaxCLIPTextModel, + tokenizer: CLIPTokenizer, + unet: FlaxUNet2DConditionModel, + scheduler: Union[ + FlaxDDIMScheduler, FlaxPNDMScheduler, FlaxLMSDiscreteScheduler, FlaxDPMSolverMultistepScheduler + ], + safety_checker: FlaxStableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + dtype: jnp.dtype = jnp.float32, + ): + super().__init__() + self.dtype = dtype + + if safety_checker is None: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely .If you're checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + + def prepare_inputs( + self, + prompt: Union[str, List[str]], + image: Union[Image.Image, List[Image.Image]], + mask: Union[Image.Image, List[Image.Image]], + ): + if not isinstance(prompt, (str, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if not isinstance(image, (Image.Image, list)): + raise ValueError(f"image has to be of type `PIL.Image.Image` or list but is {type(image)}") + + if isinstance(image, Image.Image): + image = [image] + + if not isinstance(mask, (Image.Image, list)): + raise ValueError(f"image has to be of type `PIL.Image.Image` or list but is {type(image)}") + + if isinstance(mask, Image.Image): + mask = [mask] + + processed_images = jnp.concatenate([preprocess_image(img, jnp.float32) for img in image]) + processed_masks = jnp.concatenate([preprocess_mask(m, jnp.float32) for m in mask]) + # processed_masks[processed_masks < 0.5] = 0 + processed_masks = processed_masks.at[processed_masks < 0.5].set(0) + # processed_masks[processed_masks >= 0.5] = 1 + processed_masks = processed_masks.at[processed_masks >= 0.5].set(1) + + processed_masked_images = processed_images * (processed_masks < 0.5) + + text_input = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + return text_input.input_ids, processed_masked_images, processed_masks + + def _get_has_nsfw_concepts(self, features, params): + has_nsfw_concepts = self.safety_checker(features, params) + return has_nsfw_concepts + + def _run_safety_checker(self, images, safety_model_params, jit=False): + # safety_model_params should already be replicated when jit is True + pil_images = [Image.fromarray(image) for image in images] + features = self.feature_extractor(pil_images, return_tensors="np").pixel_values + + if jit: + features = shard(features) + has_nsfw_concepts = _p_get_has_nsfw_concepts(self, features, safety_model_params) + has_nsfw_concepts = unshard(has_nsfw_concepts) + safety_model_params = unreplicate(safety_model_params) + else: + has_nsfw_concepts = self._get_has_nsfw_concepts(features, safety_model_params) + + images_was_copied = False + for idx, has_nsfw_concept in enumerate(has_nsfw_concepts): + if has_nsfw_concept: + if not images_was_copied: + images_was_copied = True + images = images.copy() + + images[idx] = np.zeros(images[idx].shape, dtype=np.uint8) # black image + + if any(has_nsfw_concepts): + warnings.warn( + "Potential NSFW content was detected in one or more images. A black image will be returned" + " instead. Try again with a different prompt and/or seed." + ) + + return images, has_nsfw_concepts + + def _generate( + self, + prompt_ids: jnp.ndarray, + mask: jnp.ndarray, + masked_image: jnp.ndarray, + params: Union[Dict, FrozenDict], + prng_seed: jax.Array, + num_inference_steps: int, + height: int, + width: int, + guidance_scale: float, + latents: Optional[jnp.ndarray] = None, + neg_prompt_ids: Optional[jnp.ndarray] = None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + # get prompt text embeddings + prompt_embeds = self.text_encoder(prompt_ids, params=params["text_encoder"])[0] + + # TODO: currently it is assumed `do_classifier_free_guidance = guidance_scale > 1.0` + # implement this conditional `do_classifier_free_guidance = guidance_scale > 1.0` + batch_size = prompt_ids.shape[0] + + max_length = prompt_ids.shape[-1] + + if neg_prompt_ids is None: + uncond_input = self.tokenizer( + [""] * batch_size, padding="max_length", max_length=max_length, return_tensors="np" + ).input_ids + else: + uncond_input = neg_prompt_ids + negative_prompt_embeds = self.text_encoder(uncond_input, params=params["text_encoder"])[0] + context = jnp.concatenate([negative_prompt_embeds, prompt_embeds]) + + latents_shape = ( + batch_size, + self.vae.config.latent_channels, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + if latents is None: + latents = jax.random.normal(prng_seed, shape=latents_shape, dtype=self.dtype) + else: + if latents.shape != latents_shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}") + + prng_seed, mask_prng_seed = jax.random.split(prng_seed) + + masked_image_latent_dist = self.vae.apply( + {"params": params["vae"]}, masked_image, method=self.vae.encode + ).latent_dist + masked_image_latents = masked_image_latent_dist.sample(key=mask_prng_seed).transpose((0, 3, 1, 2)) + masked_image_latents = self.vae.config.scaling_factor * masked_image_latents + del mask_prng_seed + + mask = jax.image.resize(mask, (*mask.shape[:-2], *masked_image_latents.shape[-2:]), method="nearest") + + # 8. Check that sizes of mask, masked image and latents match + num_channels_latents = self.vae.config.latent_channels + num_channels_mask = mask.shape[1] + num_channels_masked_image = masked_image_latents.shape[1] + if num_channels_latents + num_channels_mask + num_channels_masked_image != self.unet.config.in_channels: + raise ValueError( + f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects" + f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +" + f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}" + f" = {num_channels_latents+num_channels_masked_image+num_channels_mask}. Please verify the config of" + " `pipeline.unet` or your `mask_image` or `image` input." + ) + + def loop_body(step, args): + latents, mask, masked_image_latents, scheduler_state = args + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + latents_input = jnp.concatenate([latents] * 2) + mask_input = jnp.concatenate([mask] * 2) + masked_image_latents_input = jnp.concatenate([masked_image_latents] * 2) + + t = jnp.array(scheduler_state.timesteps, dtype=jnp.int32)[step] + timestep = jnp.broadcast_to(t, latents_input.shape[0]) + + latents_input = self.scheduler.scale_model_input(scheduler_state, latents_input, t) + # concat latents, mask, masked_image_latents in the channel dimension + latents_input = jnp.concatenate([latents_input, mask_input, masked_image_latents_input], axis=1) + + # predict the noise residual + noise_pred = self.unet.apply( + {"params": params["unet"]}, + jnp.array(latents_input), + jnp.array(timestep, dtype=jnp.int32), + encoder_hidden_states=context, + ).sample + # perform guidance + noise_pred_uncond, noise_prediction_text = jnp.split(noise_pred, 2, axis=0) + noise_pred = noise_pred_uncond + guidance_scale * (noise_prediction_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents, scheduler_state = self.scheduler.step(scheduler_state, noise_pred, t, latents).to_tuple() + return latents, mask, masked_image_latents, scheduler_state + + scheduler_state = self.scheduler.set_timesteps( + params["scheduler"], num_inference_steps=num_inference_steps, shape=latents.shape + ) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * params["scheduler"].init_noise_sigma + + if DEBUG: + # run with python for loop + for i in range(num_inference_steps): + latents, mask, masked_image_latents, scheduler_state = loop_body( + i, (latents, mask, masked_image_latents, scheduler_state) + ) + else: + latents, _, _, _ = jax.lax.fori_loop( + 0, num_inference_steps, loop_body, (latents, mask, masked_image_latents, scheduler_state) + ) + + # scale and decode the image latents with vae + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.apply({"params": params["vae"]}, latents, method=self.vae.decode).sample + + image = (image / 2 + 0.5).clip(0, 1).transpose(0, 2, 3, 1) + return image + + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt_ids: jnp.ndarray, + mask: jnp.ndarray, + masked_image: jnp.ndarray, + params: Union[Dict, FrozenDict], + prng_seed: jax.Array, + num_inference_steps: int = 50, + height: Optional[int] = None, + width: Optional[int] = None, + guidance_scale: Union[float, jnp.ndarray] = 7.5, + latents: jnp.ndarray = None, + neg_prompt_ids: jnp.ndarray = None, + return_dict: bool = True, + jit: bool = False, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide image generation. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + latents (`jnp.ndarray`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + array is generated by sampling using the supplied random `generator`. + jit (`bool`, defaults to `False`): + Whether to run `pmap` versions of the generation and safety scoring functions. + + + + This argument exists because `__call__` is not yet end-to-end pmap-able. It will be removed in a + future release. + + + + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] instead of + a plain tuple. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.FlaxStableDiffusionPipelineOutput`] is + returned, otherwise a `tuple` is returned where the first element is a list with the generated images + and the second element is a list of `bool`s indicating whether the corresponding generated image + contains "not-safe-for-work" (nsfw) content. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + masked_image = jax.image.resize(masked_image, (*masked_image.shape[:-2], height, width), method="bicubic") + mask = jax.image.resize(mask, (*mask.shape[:-2], height, width), method="nearest") + + if isinstance(guidance_scale, float): + # Convert to a tensor so each device gets a copy. Follow the prompt_ids for + # shape information, as they may be sharded (when `jit` is `True`), or not. + guidance_scale = jnp.array([guidance_scale] * prompt_ids.shape[0]) + if len(prompt_ids.shape) > 2: + # Assume sharded + guidance_scale = guidance_scale[:, None] + + if jit: + images = _p_generate( + self, + prompt_ids, + mask, + masked_image, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, + ) + else: + images = self._generate( + prompt_ids, + mask, + masked_image, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, + ) + + if self.safety_checker is not None: + safety_params = params["safety_checker"] + images_uint8_casted = (images * 255).round().astype("uint8") + num_devices, batch_size = images.shape[:2] + + images_uint8_casted = np.asarray(images_uint8_casted).reshape(num_devices * batch_size, height, width, 3) + images_uint8_casted, has_nsfw_concept = self._run_safety_checker(images_uint8_casted, safety_params, jit) + images = np.asarray(images) + + # block images + if any(has_nsfw_concept): + for i, is_nsfw in enumerate(has_nsfw_concept): + if is_nsfw: + images[i] = np.asarray(images_uint8_casted[i]) + + images = images.reshape(num_devices, batch_size, height, width, 3) + else: + images = np.asarray(images) + has_nsfw_concept = False + + if not return_dict: + return (images, has_nsfw_concept) + + return FlaxStableDiffusionPipelineOutput(images=images, nsfw_content_detected=has_nsfw_concept) + + +# Static argnums are pipe, num_inference_steps, height, width. A change would trigger recompilation. +# Non-static args are (sharded) input tensors mapped over their first dimension (hence, `0`). +@partial( + jax.pmap, + in_axes=(None, 0, 0, 0, 0, 0, None, None, None, 0, 0, 0), + static_broadcasted_argnums=(0, 6, 7, 8), +) +def _p_generate( + pipe, + prompt_ids, + mask, + masked_image, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, +): + return pipe._generate( + prompt_ids, + mask, + masked_image, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, + ) + + +@partial(jax.pmap, static_broadcasted_argnums=(0,)) +def _p_get_has_nsfw_concepts(pipe, features, params): + return pipe._get_has_nsfw_concepts(features, params) + + +def unshard(x: jnp.ndarray): + # einops.rearrange(x, 'd b ... -> (d b) ...') + num_devices, batch_size = x.shape[:2] + rest = x.shape[2:] + return x.reshape(num_devices * batch_size, *rest) + + +def preprocess_image(image, dtype): + w, h = image.size + w, h = (x - x % 32 for x in (w, h)) # resize to integer multiple of 32 + image = image.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]) + image = jnp.array(image).astype(dtype) / 255.0 + image = image[None].transpose(0, 3, 1, 2) + return 2.0 * image - 1.0 + + +def preprocess_mask(mask, dtype): + w, h = mask.size + w, h = (x - x % 32 for x in (w, h)) # resize to integer multiple of 32 + mask = mask.resize((w, h)) + mask = jnp.array(mask.convert("L")).astype(dtype) / 255.0 + mask = jnp.expand_dims(mask, axis=(0, 1)) + + return mask diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py new file mode 100755 index 0000000..311347d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion.py @@ -0,0 +1,487 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Callable, List, Optional, Union + +import numpy as np +import torch +from transformers import CLIPImageProcessor, CLIPTokenizer + +from ...configuration_utils import FrozenDict +from ...schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler +from ...utils import deprecate, logging +from ..onnx_utils import ORT_TO_NP_TYPE, OnnxRuntimeModel +from ..pipeline_utils import DiffusionPipeline +from . import StableDiffusionPipelineOutput + + +logger = logging.get_logger(__name__) + + +class OnnxStableDiffusionPipeline(DiffusionPipeline): + vae_encoder: OnnxRuntimeModel + vae_decoder: OnnxRuntimeModel + text_encoder: OnnxRuntimeModel + tokenizer: CLIPTokenizer + unet: OnnxRuntimeModel + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler] + safety_checker: OnnxRuntimeModel + feature_extractor: CLIPImageProcessor + + _optional_components = ["safety_checker", "feature_extractor"] + _is_onnx = True + + def __init__( + self, + vae_encoder: OnnxRuntimeModel, + vae_decoder: OnnxRuntimeModel, + text_encoder: OnnxRuntimeModel, + tokenizer: CLIPTokenizer, + unet: OnnxRuntimeModel, + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + safety_checker: OnnxRuntimeModel, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae_encoder=vae_encoder, + vae_decoder=vae_decoder, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + def _encode_prompt( + self, + prompt: Union[str, List[str]], + num_images_per_prompt: Optional[int], + do_classifier_free_guidance: bool, + negative_prompt: Optional[str], + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`): + prompt to be encoded + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + prompt_embeds (`np.ndarray`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`np.ndarray`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + """ + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="max_length", return_tensors="np").input_ids + + if not np.array_equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder(input_ids=text_input_ids.astype(np.int32))[0] + + prompt_embeds = np.repeat(prompt_embeds, num_images_per_prompt, axis=0) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] * batch_size + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="np", + ) + negative_prompt_embeds = self.text_encoder(input_ids=uncond_input.input_ids.astype(np.int32))[0] + + if do_classifier_free_guidance: + negative_prompt_embeds = np.repeat(negative_prompt_embeds, num_images_per_prompt, axis=0) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = np.concatenate([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + def check_inputs( + self, + prompt: Union[str, List[str]], + height: Optional[int], + width: Optional[int], + callback_steps: int, + negative_prompt: Optional[str] = None, + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = 512, + width: Optional[int] = 512, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: Optional[float] = 0.0, + generator: Optional[np.random.RandomState] = None, + latents: Optional[np.ndarray] = None, + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, np.ndarray], None]] = None, + callback_steps: int = 1, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + image (`PIL.Image.Image` or List[`PIL.Image.Image`] or `torch.FloatTensor`): + `Image`, or tensor representing an image batch which will be upscaled. * + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` + is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`np.random.RandomState`, *optional*): + One or a list of [numpy generator(s)](TODO) to make generation deterministic. + latents (`np.ndarray`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`np.ndarray`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`np.ndarray`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. + When returning a tuple, the first element is a list with the generated images, and the second element is a + list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, according to the `safety_checker`. + """ + + # check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if generator is None: + generator = np.random + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + prompt_embeds = self._encode_prompt( + prompt, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + # get the initial random noise unless the user supplied it + latents_dtype = prompt_embeds.dtype + latents_shape = (batch_size * num_images_per_prompt, 4, height // 8, width // 8) + if latents is None: + latents = generator.randn(*latents_shape).astype(latents_dtype) + elif latents.shape != latents_shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}") + + # set timesteps + self.scheduler.set_timesteps(num_inference_steps) + + latents = latents * np.float64(self.scheduler.init_noise_sigma) + + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + timestep_dtype = next( + (input.type for input in self.unet.model.get_inputs() if input.name == "timestep"), "tensor(float)" + ) + timestep_dtype = ORT_TO_NP_TYPE[timestep_dtype] + + for i, t in enumerate(self.progress_bar(self.scheduler.timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = np.concatenate([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(torch.from_numpy(latent_model_input), t) + latent_model_input = latent_model_input.cpu().numpy() + + # predict the noise residual + timestep = np.array([t], dtype=timestep_dtype) + noise_pred = self.unet(sample=latent_model_input, timestep=timestep, encoder_hidden_states=prompt_embeds) + noise_pred = noise_pred[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = np.split(noise_pred, 2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + scheduler_output = self.scheduler.step( + torch.from_numpy(noise_pred), t, torch.from_numpy(latents), **extra_step_kwargs + ) + latents = scheduler_output.prev_sample.numpy() + + # call the callback, if provided + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + latents = 1 / 0.18215 * latents + # image = self.vae_decoder(latent_sample=latents)[0] + # it seems likes there is a strange result for using half-precision vae decoder if batchsize>1 + image = np.concatenate( + [self.vae_decoder(latent_sample=latents[i : i + 1])[0] for i in range(latents.shape[0])] + ) + + image = np.clip(image / 2 + 0.5, 0, 1) + image = image.transpose((0, 2, 3, 1)) + + if self.safety_checker is not None: + safety_checker_input = self.feature_extractor( + self.numpy_to_pil(image), return_tensors="np" + ).pixel_values.astype(image.dtype) + + images, has_nsfw_concept = [], [] + for i in range(image.shape[0]): + image_i, has_nsfw_concept_i = self.safety_checker( + clip_input=safety_checker_input[i : i + 1], images=image[i : i + 1] + ) + images.append(image_i) + has_nsfw_concept.append(has_nsfw_concept_i[0]) + image = np.concatenate(images) + else: + has_nsfw_concept = None + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) + + +class StableDiffusionOnnxPipeline(OnnxStableDiffusionPipeline): + def __init__( + self, + vae_encoder: OnnxRuntimeModel, + vae_decoder: OnnxRuntimeModel, + text_encoder: OnnxRuntimeModel, + tokenizer: CLIPTokenizer, + unet: OnnxRuntimeModel, + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + safety_checker: OnnxRuntimeModel, + feature_extractor: CLIPImageProcessor, + ): + deprecation_message = "Please use `OnnxStableDiffusionPipeline` instead of `StableDiffusionOnnxPipeline`." + deprecate("StableDiffusionOnnxPipeline", "1.0.0", deprecation_message) + super().__init__( + vae_encoder=vae_encoder, + vae_decoder=vae_decoder, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py new file mode 100755 index 0000000..c394098 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_img2img.py @@ -0,0 +1,549 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTokenizer + +from ...configuration_utils import FrozenDict +from ...schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler +from ...utils import PIL_INTERPOLATION, deprecate, logging +from ..onnx_utils import ORT_TO_NP_TYPE, OnnxRuntimeModel +from ..pipeline_utils import DiffusionPipeline +from . import StableDiffusionPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.preprocess with 8->64 +def preprocess(image): + deprecation_message = "The preprocess method is deprecated and will be removed in diffusers 1.0.0. Please use VaeImageProcessor.preprocess(...) instead" + deprecate("preprocess", "1.0.0", deprecation_message, standard_warn=False) + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 64 for x in (w, h)) # resize to integer multiple of 64 + + image = [np.array(i.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + return image + + +class OnnxStableDiffusionImg2ImgPipeline(DiffusionPipeline): + r""" + Pipeline for text-guided image to image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details. + feature_extractor ([`CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `safety_checker`. + """ + + vae_encoder: OnnxRuntimeModel + vae_decoder: OnnxRuntimeModel + text_encoder: OnnxRuntimeModel + tokenizer: CLIPTokenizer + unet: OnnxRuntimeModel + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler] + safety_checker: OnnxRuntimeModel + feature_extractor: CLIPImageProcessor + + _optional_components = ["safety_checker", "feature_extractor"] + _is_onnx = True + + def __init__( + self, + vae_encoder: OnnxRuntimeModel, + vae_decoder: OnnxRuntimeModel, + text_encoder: OnnxRuntimeModel, + tokenizer: CLIPTokenizer, + unet: OnnxRuntimeModel, + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + safety_checker: OnnxRuntimeModel, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae_encoder=vae_encoder, + vae_decoder=vae_decoder, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_onnx_stable_diffusion.OnnxStableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt: Union[str, List[str]], + num_images_per_prompt: Optional[int], + do_classifier_free_guidance: bool, + negative_prompt: Optional[str], + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`): + prompt to be encoded + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + prompt_embeds (`np.ndarray`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`np.ndarray`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + """ + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="max_length", return_tensors="np").input_ids + + if not np.array_equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder(input_ids=text_input_ids.astype(np.int32))[0] + + prompt_embeds = np.repeat(prompt_embeds, num_images_per_prompt, axis=0) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] * batch_size + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="np", + ) + negative_prompt_embeds = self.text_encoder(input_ids=uncond_input.input_ids.astype(np.int32))[0] + + if do_classifier_free_guidance: + negative_prompt_embeds = np.repeat(negative_prompt_embeds, num_images_per_prompt, axis=0) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = np.concatenate([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + def check_inputs( + self, + prompt: Union[str, List[str]], + callback_steps: int, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + ): + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def __call__( + self, + prompt: Union[str, List[str]], + image: Union[np.ndarray, PIL.Image.Image] = None, + strength: float = 0.8, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: Optional[float] = 0.0, + generator: Optional[np.random.RandomState] = None, + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, np.ndarray], None]] = None, + callback_steps: int = 1, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`np.ndarray` or `PIL.Image.Image`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. + strength (`float`, *optional*, defaults to 0.8): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter will be modulated by `strength`. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`np.random.RandomState`, *optional*): + A np.random.RandomState to make generation deterministic. + prompt_embeds (`np.ndarray`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`np.ndarray`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: np.ndarray)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. + When returning a tuple, the first element is a list with the generated images, and the second element is a + list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, according to the `safety_checker`. + """ + + # check inputs. Raise error if not correct + self.check_inputs(prompt, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds) + + # define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if generator is None: + generator = np.random + + # set timesteps + self.scheduler.set_timesteps(num_inference_steps) + + image = preprocess(image).cpu().numpy() + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + prompt_embeds = self._encode_prompt( + prompt, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + latents_dtype = prompt_embeds.dtype + image = image.astype(latents_dtype) + # encode the init image into latents and scale the latents + init_latents = self.vae_encoder(sample=image)[0] + init_latents = 0.18215 * init_latents + + if isinstance(prompt, str): + prompt = [prompt] + if len(prompt) > init_latents.shape[0] and len(prompt) % init_latents.shape[0] == 0: + # expand init_latents for batch_size + deprecation_message = ( + f"You have passed {len(prompt)} text prompts (`prompt`), but only {init_latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_image_per_prompt = len(prompt) // init_latents.shape[0] + init_latents = np.concatenate([init_latents] * additional_image_per_prompt * num_images_per_prompt, axis=0) + elif len(prompt) > init_latents.shape[0] and len(prompt) % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {len(prompt)} text prompts." + ) + else: + init_latents = np.concatenate([init_latents] * num_images_per_prompt, axis=0) + + # get the original timestep using init_timestep + offset = self.scheduler.config.get("steps_offset", 0) + init_timestep = int(num_inference_steps * strength) + offset + init_timestep = min(init_timestep, num_inference_steps) + + timesteps = self.scheduler.timesteps.numpy()[-init_timestep] + timesteps = np.array([timesteps] * batch_size * num_images_per_prompt) + + # add noise to latents using the timesteps + noise = generator.randn(*init_latents.shape).astype(latents_dtype) + init_latents = self.scheduler.add_noise( + torch.from_numpy(init_latents), torch.from_numpy(noise), torch.from_numpy(timesteps) + ) + init_latents = init_latents.numpy() + + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + latents = init_latents + + t_start = max(num_inference_steps - init_timestep + offset, 0) + timesteps = self.scheduler.timesteps[t_start:].numpy() + + timestep_dtype = next( + (input.type for input in self.unet.model.get_inputs() if input.name == "timestep"), "tensor(float)" + ) + timestep_dtype = ORT_TO_NP_TYPE[timestep_dtype] + + for i, t in enumerate(self.progress_bar(timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = np.concatenate([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(torch.from_numpy(latent_model_input), t) + latent_model_input = latent_model_input.cpu().numpy() + + # predict the noise residual + timestep = np.array([t], dtype=timestep_dtype) + noise_pred = self.unet(sample=latent_model_input, timestep=timestep, encoder_hidden_states=prompt_embeds)[ + 0 + ] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = np.split(noise_pred, 2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + scheduler_output = self.scheduler.step( + torch.from_numpy(noise_pred), t, torch.from_numpy(latents), **extra_step_kwargs + ) + latents = scheduler_output.prev_sample.numpy() + + # call the callback, if provided + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + latents = 1 / 0.18215 * latents + # image = self.vae_decoder(latent_sample=latents)[0] + # it seems likes there is a strange result for using half-precision vae decoder if batchsize>1 + image = np.concatenate( + [self.vae_decoder(latent_sample=latents[i : i + 1])[0] for i in range(latents.shape[0])] + ) + + image = np.clip(image / 2 + 0.5, 0, 1) + image = image.transpose((0, 2, 3, 1)) + + if self.safety_checker is not None: + safety_checker_input = self.feature_extractor( + self.numpy_to_pil(image), return_tensors="np" + ).pixel_values.astype(image.dtype) + # safety_checker does not support batched inputs yet + images, has_nsfw_concept = [], [] + for i in range(image.shape[0]): + image_i, has_nsfw_concept_i = self.safety_checker( + clip_input=safety_checker_input[i : i + 1], images=image[i : i + 1] + ) + images.append(image_i) + has_nsfw_concept.append(has_nsfw_concept_i[0]) + image = np.concatenate(images) + else: + has_nsfw_concept = None + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py new file mode 100755 index 0000000..18d8050 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_inpaint.py @@ -0,0 +1,563 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTokenizer + +from ...configuration_utils import FrozenDict +from ...schedulers import DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler +from ...utils import PIL_INTERPOLATION, deprecate, logging +from ..onnx_utils import ORT_TO_NP_TYPE, OnnxRuntimeModel +from ..pipeline_utils import DiffusionPipeline +from . import StableDiffusionPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +NUM_UNET_INPUT_CHANNELS = 9 +NUM_LATENT_CHANNELS = 4 + + +def prepare_mask_and_masked_image(image, mask, latents_shape): + image = np.array(image.convert("RGB").resize((latents_shape[1] * 8, latents_shape[0] * 8))) + image = image[None].transpose(0, 3, 1, 2) + image = image.astype(np.float32) / 127.5 - 1.0 + + image_mask = np.array(mask.convert("L").resize((latents_shape[1] * 8, latents_shape[0] * 8))) + masked_image = image * (image_mask < 127.5) + + mask = mask.resize((latents_shape[1], latents_shape[0]), PIL_INTERPOLATION["nearest"]) + mask = np.array(mask.convert("L")) + mask = mask.astype(np.float32) / 255.0 + mask = mask[None, None] + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + + return mask, masked_image + + +class OnnxStableDiffusionInpaintPipeline(DiffusionPipeline): + r""" + Pipeline for text-guided image inpainting using Stable Diffusion. *This is an experimental feature*. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details. + feature_extractor ([`CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `safety_checker`. + """ + + vae_encoder: OnnxRuntimeModel + vae_decoder: OnnxRuntimeModel + text_encoder: OnnxRuntimeModel + tokenizer: CLIPTokenizer + unet: OnnxRuntimeModel + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler] + safety_checker: OnnxRuntimeModel + feature_extractor: CLIPImageProcessor + + _optional_components = ["safety_checker", "feature_extractor"] + _is_onnx = True + + def __init__( + self, + vae_encoder: OnnxRuntimeModel, + vae_decoder: OnnxRuntimeModel, + text_encoder: OnnxRuntimeModel, + tokenizer: CLIPTokenizer, + unet: OnnxRuntimeModel, + scheduler: Union[DDIMScheduler, PNDMScheduler, LMSDiscreteScheduler], + safety_checker: OnnxRuntimeModel, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + logger.info("`OnnxStableDiffusionInpaintPipeline` is experimental and will very likely change in the future.") + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae_encoder=vae_encoder, + vae_decoder=vae_decoder, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_onnx_stable_diffusion.OnnxStableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt: Union[str, List[str]], + num_images_per_prompt: Optional[int], + do_classifier_free_guidance: bool, + negative_prompt: Optional[str], + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`): + prompt to be encoded + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + prompt_embeds (`np.ndarray`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`np.ndarray`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + """ + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="max_length", return_tensors="np").input_ids + + if not np.array_equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder(input_ids=text_input_ids.astype(np.int32))[0] + + prompt_embeds = np.repeat(prompt_embeds, num_images_per_prompt, axis=0) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] * batch_size + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="np", + ) + negative_prompt_embeds = self.text_encoder(input_ids=uncond_input.input_ids.astype(np.int32))[0] + + if do_classifier_free_guidance: + negative_prompt_embeds = np.repeat(negative_prompt_embeds, num_images_per_prompt, axis=0) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = np.concatenate([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_onnx_stable_diffusion.OnnxStableDiffusionPipeline.check_inputs + def check_inputs( + self, + prompt: Union[str, List[str]], + height: Optional[int], + width: Optional[int], + callback_steps: int, + negative_prompt: Optional[str] = None, + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + image: PIL.Image.Image, + mask_image: PIL.Image.Image, + height: Optional[int] = 512, + width: Optional[int] = 512, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[np.random.RandomState] = None, + latents: Optional[np.ndarray] = None, + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, np.ndarray], None]] = None, + callback_steps: int = 1, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`PIL.Image.Image`): + `Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will + be masked out with `mask_image` and repainted according to `prompt`. + mask_image (`PIL.Image.Image`): + `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be + repainted, while black pixels will be preserved. If `mask_image` is a PIL image, it will be converted + to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) + instead of 3, so the expected shape would be `(B, H, W, 1)`. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`np.random.RandomState`, *optional*): + A np.random.RandomState to make generation deterministic. + latents (`np.ndarray`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`np.ndarray`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`np.ndarray`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: np.ndarray)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. + When returning a tuple, the first element is a list with the generated images, and the second element is a + list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, according to the `safety_checker`. + """ + + # check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if generator is None: + generator = np.random + + # set timesteps + self.scheduler.set_timesteps(num_inference_steps) + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + prompt_embeds = self._encode_prompt( + prompt, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + num_channels_latents = NUM_LATENT_CHANNELS + latents_shape = (batch_size * num_images_per_prompt, num_channels_latents, height // 8, width // 8) + latents_dtype = prompt_embeds.dtype + if latents is None: + latents = generator.randn(*latents_shape).astype(latents_dtype) + else: + if latents.shape != latents_shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}") + + # prepare mask and masked_image + mask, masked_image = prepare_mask_and_masked_image(image, mask_image, latents_shape[-2:]) + mask = mask.astype(latents.dtype) + masked_image = masked_image.astype(latents.dtype) + + masked_image_latents = self.vae_encoder(sample=masked_image)[0] + masked_image_latents = 0.18215 * masked_image_latents + + # duplicate mask and masked_image_latents for each generation per prompt + mask = mask.repeat(batch_size * num_images_per_prompt, 0) + masked_image_latents = masked_image_latents.repeat(batch_size * num_images_per_prompt, 0) + + mask = np.concatenate([mask] * 2) if do_classifier_free_guidance else mask + masked_image_latents = ( + np.concatenate([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents + ) + + num_channels_mask = mask.shape[1] + num_channels_masked_image = masked_image_latents.shape[1] + + unet_input_channels = NUM_UNET_INPUT_CHANNELS + if num_channels_latents + num_channels_mask + num_channels_masked_image != unet_input_channels: + raise ValueError( + "Incorrect configuration settings! The config of `pipeline.unet` expects" + f" {unet_input_channels} but received `num_channels_latents`: {num_channels_latents} +" + f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}" + f" = {num_channels_latents+num_channels_masked_image+num_channels_mask}. Please verify the config of" + " `pipeline.unet` or your `mask_image` or `image` input." + ) + + # set timesteps + self.scheduler.set_timesteps(num_inference_steps) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * np.float64(self.scheduler.init_noise_sigma) + + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + timestep_dtype = next( + (input.type for input in self.unet.model.get_inputs() if input.name == "timestep"), "tensor(float)" + ) + timestep_dtype = ORT_TO_NP_TYPE[timestep_dtype] + + for i, t in enumerate(self.progress_bar(self.scheduler.timesteps)): + # expand the latents if we are doing classifier free guidance + latent_model_input = np.concatenate([latents] * 2) if do_classifier_free_guidance else latents + # concat latents, mask, masked_image_latnets in the channel dimension + latent_model_input = self.scheduler.scale_model_input(torch.from_numpy(latent_model_input), t) + latent_model_input = latent_model_input.cpu().numpy() + latent_model_input = np.concatenate([latent_model_input, mask, masked_image_latents], axis=1) + + # predict the noise residual + timestep = np.array([t], dtype=timestep_dtype) + noise_pred = self.unet(sample=latent_model_input, timestep=timestep, encoder_hidden_states=prompt_embeds)[ + 0 + ] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = np.split(noise_pred, 2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + scheduler_output = self.scheduler.step( + torch.from_numpy(noise_pred), t, torch.from_numpy(latents), **extra_step_kwargs + ) + latents = scheduler_output.prev_sample.numpy() + + # call the callback, if provided + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + latents = 1 / 0.18215 * latents + # image = self.vae_decoder(latent_sample=latents)[0] + # it seems likes there is a strange result for using half-precision vae decoder if batchsize>1 + image = np.concatenate( + [self.vae_decoder(latent_sample=latents[i : i + 1])[0] for i in range(latents.shape[0])] + ) + + image = np.clip(image / 2 + 0.5, 0, 1) + image = image.transpose((0, 2, 3, 1)) + + if self.safety_checker is not None: + safety_checker_input = self.feature_extractor( + self.numpy_to_pil(image), return_tensors="np" + ).pixel_values.astype(image.dtype) + # safety_checker does not support batched inputs yet + images, has_nsfw_concept = [], [] + for i in range(image.shape[0]): + image_i, has_nsfw_concept_i = self.safety_checker( + clip_input=safety_checker_input[i : i + 1], images=image[i : i + 1] + ) + images.append(image_i) + has_nsfw_concept.append(has_nsfw_concept_i[0]) + image = np.concatenate(images) + else: + has_nsfw_concept = None + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py new file mode 100755 index 0000000..58d83de --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_onnx_stable_diffusion_upscale.py @@ -0,0 +1,586 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTokenizer + +from ...configuration_utils import FrozenDict +from ...schedulers import DDPMScheduler, KarrasDiffusionSchedulers +from ...utils import deprecate, logging +from ..onnx_utils import ORT_TO_NP_TYPE, OnnxRuntimeModel +from ..pipeline_utils import DiffusionPipeline +from . import StableDiffusionPipelineOutput + + +logger = logging.get_logger(__name__) + + +def preprocess(image): + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 64 for x in (w, h)) # resize to integer multiple of 32 + + image = [np.array(i.resize((w, h)))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + + return image + + +class OnnxStableDiffusionUpscalePipeline(DiffusionPipeline): + vae: OnnxRuntimeModel + text_encoder: OnnxRuntimeModel + tokenizer: CLIPTokenizer + unet: OnnxRuntimeModel + low_res_scheduler: DDPMScheduler + scheduler: KarrasDiffusionSchedulers + safety_checker: OnnxRuntimeModel + feature_extractor: CLIPImageProcessor + + _optional_components = ["safety_checker", "feature_extractor"] + _is_onnx = True + + def __init__( + self, + vae: OnnxRuntimeModel, + text_encoder: OnnxRuntimeModel, + tokenizer: Any, + unet: OnnxRuntimeModel, + low_res_scheduler: DDPMScheduler, + scheduler: KarrasDiffusionSchedulers, + safety_checker: Optional[OnnxRuntimeModel] = None, + feature_extractor: Optional[CLIPImageProcessor] = None, + max_noise_level: int = 350, + num_latent_channels=4, + num_unet_input_channels=7, + requires_safety_checker: bool = True, + ): + super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + low_res_scheduler=low_res_scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.register_to_config( + max_noise_level=max_noise_level, + num_latent_channels=num_latent_channels, + num_unet_input_channels=num_unet_input_channels, + ) + + def check_inputs( + self, + prompt: Union[str, List[str]], + image, + noise_level, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if ( + not isinstance(image, torch.Tensor) + and not isinstance(image, PIL.Image.Image) + and not isinstance(image, np.ndarray) + and not isinstance(image, list) + ): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `np.ndarray`, `PIL.Image.Image` or `list` but is {type(image)}" + ) + + # verify batch size of prompt and image are same if image is a list or tensor or numpy array + if isinstance(image, list) or isinstance(image, np.ndarray): + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if isinstance(image, list): + image_batch_size = len(image) + else: + image_batch_size = image.shape[0] + if batch_size != image_batch_size: + raise ValueError( + f"`prompt` has batch size {batch_size} and `image` has batch size {image_batch_size}." + " Please make sure that passed `prompt` matches the batch size of `image`." + ) + + # check noise level + if noise_level > self.config.max_noise_level: + raise ValueError(f"`noise_level` has to be <= {self.config.max_noise_level} but is {noise_level}") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, generator, latents=None): + shape = (batch_size, num_channels_latents, height, width) + if latents is None: + latents = generator.randn(*shape).astype(dtype) + elif latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + + return latents + + def decode_latents(self, latents): + latents = 1 / 0.08333 * latents + image = self.vae(latent_sample=latents)[0] + image = np.clip(image / 2 + 0.5, 0, 1) + image = image.transpose((0, 2, 3, 1)) + return image + + def _encode_prompt( + self, + prompt: Union[str, List[str]], + num_images_per_prompt: Optional[int], + do_classifier_free_guidance: bool, + negative_prompt: Optional[str], + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`): + prompt to be encoded + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + prompt_embeds (`np.ndarray`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`np.ndarray`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + """ + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="max_length", return_tensors="np").input_ids + + if not np.array_equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = self.text_encoder(input_ids=text_input_ids.astype(np.int32))[0] + + prompt_embeds = np.repeat(prompt_embeds, num_images_per_prompt, axis=0) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] * batch_size + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="np", + ) + negative_prompt_embeds = self.text_encoder(input_ids=uncond_input.input_ids.astype(np.int32))[0] + + if do_classifier_free_guidance: + negative_prompt_embeds = np.repeat(negative_prompt_embeds, num_images_per_prompt, axis=0) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = np.concatenate([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + def __call__( + self, + prompt: Union[str, List[str]], + image: Union[np.ndarray, PIL.Image.Image, List[PIL.Image.Image]], + num_inference_steps: int = 75, + guidance_scale: float = 9.0, + noise_level: int = 20, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[np.random.RandomState, List[np.random.RandomState]]] = None, + latents: Optional[np.ndarray] = None, + prompt_embeds: Optional[np.ndarray] = None, + negative_prompt_embeds: Optional[np.ndarray] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, np.ndarray], None]] = None, + callback_steps: Optional[int] = 1, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + image (`np.ndarray` or `PIL.Image.Image`): + `Image`, or tensor representing an image batch, that will be used as the starting point for the + process. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter will be modulated by `strength`. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + noise_level (`float`, defaults to 0.2): + Deteremines the amount of noise to add to the initial image before performing upscaling. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`np.random.RandomState`, *optional*): + A np.random.RandomState to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`np.ndarray`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`np.ndarray`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: np.ndarray)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. + When returning a tuple, the first element is a list with the generated images, and the second element is a + list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, according to the `safety_checker`. + """ + + # 1. Check inputs + self.check_inputs( + prompt, + image, + noise_level, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if generator is None: + generator = np.random + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + prompt_embeds = self._encode_prompt( + prompt, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + latents_dtype = prompt_embeds.dtype + image = preprocess(image).cpu().numpy() + height, width = image.shape[2:] + + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + self.num_latent_channels, + height, + width, + latents_dtype, + generator, + ) + image = image.astype(latents_dtype) + + self.scheduler.set_timesteps(num_inference_steps) + timesteps = self.scheduler.timesteps + + # Scale the initial noise by the standard deviation required by the scheduler + latents = latents * np.float64(self.scheduler.init_noise_sigma) + + # 5. Add noise to image + noise_level = np.array([noise_level]).astype(np.int64) + noise = generator.randn(*image.shape).astype(latents_dtype) + + image = self.low_res_scheduler.add_noise( + torch.from_numpy(image), torch.from_numpy(noise), torch.from_numpy(noise_level) + ) + image = image.numpy() + + batch_multiplier = 2 if do_classifier_free_guidance else 1 + image = np.concatenate([image] * batch_multiplier * num_images_per_prompt) + noise_level = np.concatenate([noise_level] * image.shape[0]) + + # 7. Check that sizes of image and latents match + num_channels_image = image.shape[1] + if self.num_latent_channels + num_channels_image != self.num_unet_input_channels: + raise ValueError( + "Incorrect configuration settings! The config of `pipeline.unet` expects" + f" {self.num_unet_input_channels} but received `num_channels_latents`: {self.num_latent_channels} +" + f" `num_channels_image`: {num_channels_image} " + f" = {self.num_latent_channels + num_channels_image}. Please verify the config of" + " `pipeline.unet` or your `image` input." + ) + + # 8. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + timestep_dtype = next( + (input.type for input in self.unet.model.get_inputs() if input.name == "timestep"), "tensor(float)" + ) + timestep_dtype = ORT_TO_NP_TYPE[timestep_dtype] + + # 9. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = np.concatenate([latents] * 2) if do_classifier_free_guidance else latents + + # concat latents, mask, masked_image_latents in the channel dimension + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + latent_model_input = np.concatenate([latent_model_input, image], axis=1) + + # timestep to tensor + timestep = np.array([t], dtype=timestep_dtype) + + # predict the noise residual + noise_pred = self.unet( + sample=latent_model_input, + timestep=timestep, + encoder_hidden_states=prompt_embeds, + class_labels=noise_level, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = np.split(noise_pred, 2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step( + torch.from_numpy(noise_pred), t, torch.from_numpy(latents), **extra_step_kwargs + ).prev_sample + latents = latents.numpy() + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 10. Post-processing + image = self.decode_latents(latents) + + if self.safety_checker is not None: + safety_checker_input = self.feature_extractor( + self.numpy_to_pil(image), return_tensors="np" + ).pixel_values.astype(image.dtype) + + images, has_nsfw_concept = [], [] + for i in range(image.shape[0]): + image_i, has_nsfw_concept_i = self.safety_checker( + clip_input=safety_checker_input[i : i + 1], images=image[i : i + 1] + ) + images.append(image_i) + has_nsfw_concept.append(has_nsfw_concept_i[0]) + image = np.concatenate(images) + else: + has_nsfw_concept = None + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_output.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_output.py new file mode 100755 index 0000000..5fb9b1a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_output.py @@ -0,0 +1,45 @@ +from dataclasses import dataclass +from typing import List, Optional, Union + +import numpy as np +import PIL.Image + +from ...utils import BaseOutput, is_flax_available + + +@dataclass +class StableDiffusionPipelineOutput(BaseOutput): + """ + Output class for Stable Diffusion pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + num_channels)`. + nsfw_content_detected (`List[bool]`) + List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or + `None` if safety checking could not be performed. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + nsfw_content_detected: Optional[List[bool]] + + +if is_flax_available(): + import flax + + @flax.struct.dataclass + class FlaxStableDiffusionPipelineOutput(BaseOutput): + """ + Output class for Flax-based Stable Diffusion pipelines. + + Args: + images (`np.ndarray`): + Denoised images of array shape of `(batch_size, height, width, num_channels)`. + nsfw_content_detected (`List[bool]`): + List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content + or `None` if safety checking could not be performed. + """ + + images: np.ndarray + nsfw_content_detected: List[bool] diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py new file mode 100755 index 0000000..9e4e6c1 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion.py @@ -0,0 +1,1032 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from packaging import version +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...configuration_utils import FrozenDict +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from .pipeline_output import StableDiffusionPipelineOutput +from .safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionPipeline + + >>> pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" + + +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class StableDiffusionPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + LoraLoaderMixin, + IPAdapterMixin, + FromSingleFileMixin, +): + r""" + Pipeline for text-to-image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: CLIPVisionModelWithProjection = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + timesteps: List[int] = None, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.0): + Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf). Guidance rescale factor should fix overexposure when + using zero terminal SNR. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + # to deal with lora scaling and other possible forward hooks + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._guidance_rescale = guidance_rescale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + clip_skip=self.clip_skip, + ) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 6.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if (ip_adapter_image is not None or ip_adapter_image_embeds is not None) + else None + ) + + # 6.2 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ + 0 + ] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py new file mode 100755 index 0000000..c410acb --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_depth2img.py @@ -0,0 +1,860 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from packaging import version +from transformers import CLIPTextModel, CLIPTokenizer, DPTFeatureExtractor, DPTForDepthEstimation + +from ...configuration_utils import FrozenDict +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import PIL_INTERPOLATION, USE_PEFT_BACKEND, deprecate, logging, scale_lora_layers, unscale_lora_layers +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.preprocess +def preprocess(image): + deprecation_message = "The preprocess method is deprecated and will be removed in diffusers 1.0.0. Please use VaeImageProcessor.preprocess(...) instead" + deprecate("preprocess", "1.0.0", deprecation_message, standard_warn=False) + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 8 for x in (w, h)) # resize to integer multiple of 8 + + image = [np.array(i.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + return image + + +class StableDiffusionDepth2ImgPipeline(DiffusionPipeline, TextualInversionLoaderMixin, LoraLoaderMixin): + r""" + Pipeline for text-guided depth-based image-to-image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds", "depth_mask"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + depth_estimator: DPTForDepthEstimation, + feature_extractor: DPTFeatureExtractor, + ): + super().__init__() + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely .If you're checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + depth_estimator=depth_estimator, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + strength, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.prepare_latents + def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + init_latents = image + + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + elif isinstance(generator, list): + init_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(batch_size) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + init_latents = self.vae.config.scaling_factor * init_latents + + if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0: + # expand init_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {init_latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_image_per_prompt = batch_size // init_latents.shape[0] + init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0) + elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts." + ) + else: + init_latents = torch.cat([init_latents], dim=0) + + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + latents = init_latents + + return latents + + def prepare_depth_map(self, image, depth_map, batch_size, do_classifier_free_guidance, dtype, device): + if isinstance(image, PIL.Image.Image): + image = [image] + else: + image = list(image) + + if isinstance(image[0], PIL.Image.Image): + width, height = image[0].size + elif isinstance(image[0], np.ndarray): + width, height = image[0].shape[:-1] + else: + height, width = image[0].shape[-2:] + + if depth_map is None: + pixel_values = self.feature_extractor(images=image, return_tensors="pt").pixel_values + pixel_values = pixel_values.to(device=device) + # The DPT-Hybrid model uses batch-norm layers which are not compatible with fp16. + # So we use `torch.autocast` here for half precision inference. + context_manger = torch.autocast("cuda", dtype=dtype) if device.type == "cuda" else contextlib.nullcontext() + with context_manger: + depth_map = self.depth_estimator(pixel_values).predicted_depth + else: + depth_map = depth_map.to(device=device, dtype=dtype) + + depth_map = torch.nn.functional.interpolate( + depth_map.unsqueeze(1), + size=(height // self.vae_scale_factor, width // self.vae_scale_factor), + mode="bicubic", + align_corners=False, + ) + + depth_min = torch.amin(depth_map, dim=[1, 2, 3], keepdim=True) + depth_max = torch.amax(depth_map, dim=[1, 2, 3], keepdim=True) + depth_map = 2.0 * (depth_map - depth_min) / (depth_max - depth_min) - 1.0 + depth_map = depth_map.to(dtype) + + # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method + if depth_map.shape[0] < batch_size: + repeat_by = batch_size // depth_map.shape[0] + depth_map = depth_map.repeat(repeat_by, 1, 1, 1) + + depth_map = torch.cat([depth_map] * 2) if do_classifier_free_guidance else depth_map + return depth_map + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + depth_map: Optional[torch.FloatTensor] = None, + strength: float = 0.8, + num_inference_steps: Optional[int] = 50, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image` or tensor representing an image batch to be used as the starting point. Can accept image + latents as `image` only if `depth_map` is not `None`. + depth_map (`torch.FloatTensor`, *optional*): + Depth prediction to be used as additional conditioning for the image generation process. If not + defined, it automatically predicts the depth with `self.depth_estimator`. + strength (`float`, *optional*, defaults to 0.8): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + Examples: + + ```py + >>> import torch + >>> import requests + >>> from PIL import Image + + >>> from diffusers import StableDiffusionDepth2ImgPipeline + + >>> pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-2-depth", + ... torch_dtype=torch.float16, + ... ) + >>> pipe.to("cuda") + + + >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" + >>> init_image = Image.open(requests.get(url, stream=True).raw) + >>> prompt = "two tigers" + >>> n_propmt = "bad, deformed, ugly, bad anotomy" + >>> image = pipe(prompt=prompt, image=init_image, negative_prompt=n_propmt, strength=0.7).images[0] + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 1. Check inputs + self.check_inputs( + prompt, + strength, + callback_steps, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + + if image is None: + raise ValueError("`image` input cannot be undefined.") + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare depth mask + depth_mask = self.prepare_depth_map( + image, + depth_map, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + prompt_embeds.dtype, + device, + ) + + # 5. Preprocess image + image = self.image_processor.preprocess(image) + + # 6. Set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + + # 7. Prepare latent variables + latents = self.prepare_latents( + image, latent_timestep, batch_size, num_images_per_prompt, prompt_embeds.dtype, device, generator + ) + + # 8. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 9. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + latent_model_input = torch.cat([latent_model_input, depth_mask], dim=1) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=self.cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + depth_mask = callback_outputs.pop("depth_mask", depth_mask) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = latents + + image = self.image_processor.postprocess(image, output_type=output_type) + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py new file mode 100755 index 0000000..afd8729 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_image_variation.py @@ -0,0 +1,420 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Callable, List, Optional, Union + +import PIL.Image +import torch +from packaging import version +from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection + +from ...configuration_utils import FrozenDict +from ...image_processor import VaeImageProcessor +from ...models import AutoencoderKL, UNet2DConditionModel +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import deprecate, logging +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from . import StableDiffusionPipelineOutput +from .safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class StableDiffusionImageVariationPipeline(DiffusionPipeline, StableDiffusionMixin): + r""" + Pipeline to generate image variations from an input image using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + image_encoder ([`~transformers.CLIPVisionModelWithProjection`]): + Frozen CLIP image-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + # TODO: feature_extractor is required to encode images (if they are in PIL format), + # we should give a descriptive message if the pipeline doesn't have one. + _optional_components = ["safety_checker"] + model_cpu_offload_seq = "image_encoder->unet->vae" + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + image_encoder: CLIPVisionModelWithProjection, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely .If you're checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + image_encoder=image_encoder, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + def _encode_image(self, image, device, num_images_per_prompt, do_classifier_free_guidance): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(images=image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + image_embeddings = self.image_encoder(image).image_embeds + image_embeddings = image_embeddings.unsqueeze(1) + + # duplicate image embeddings for each generation per prompt, using mps friendly method + bs_embed, seq_len, _ = image_embeddings.shape + image_embeddings = image_embeddings.repeat(1, num_images_per_prompt, 1) + image_embeddings = image_embeddings.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + negative_prompt_embeds = torch.zeros_like(image_embeddings) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + image_embeddings = torch.cat([negative_prompt_embeds, image_embeddings]) + + return image_embeddings + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs(self, image, height, width, callback_steps): + if ( + not isinstance(image, torch.Tensor) + and not isinstance(image, PIL.Image.Image) + and not isinstance(image, list) + ): + raise ValueError( + "`image` has to be of type `torch.FloatTensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + f" {type(image)}" + ) + + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + def __call__( + self, + image: Union[PIL.Image.Image, List[PIL.Image.Image], torch.FloatTensor], + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + ): + r""" + The call function to the pipeline for generation. + + Args: + image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.FloatTensor`): + Image or images to guide image generation. If you provide a tensor, it needs to be compatible with + [`CLIPImageProcessor`](https://huggingface.co/lambdalabs/sd-image-variations-diffusers/blob/main/feature_extractor/preprocessor_config.json). + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + + Examples: + + ```py + from diffusers import StableDiffusionImageVariationPipeline + from PIL import Image + from io import BytesIO + import requests + + pipe = StableDiffusionImageVariationPipeline.from_pretrained( + "lambdalabs/sd-image-variations-diffusers", revision="v2.0" + ) + pipe = pipe.to("cuda") + + url = "https://lh3.googleusercontent.com/y-iFOHfLTwkuQSUegpwDdgKmOjRSTvPxat63dQLB25xkTs4lhIbRUFeNBWZzYf370g=s1200" + + response = requests.get(url) + image = Image.open(BytesIO(response.content)).convert("RGB") + + out = pipe(image, num_images_per_prompt=3, guidance_scale=15) + out["images"][0].save("result.jpg") + ``` + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs(image, height, width, callback_steps) + + # 2. Define call parameters + if isinstance(image, PIL.Image.Image): + batch_size = 1 + elif isinstance(image, list): + batch_size = len(image) + else: + batch_size = image.shape[0] + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input image + image_embeddings = self._encode_image(image, device, num_images_per_prompt, do_classifier_free_guidance) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + image_embeddings.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet(latent_model_input, t, encoder_hidden_states=image_embeddings).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + self.maybe_free_model_hooks() + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, image_embeddings.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py new file mode 100755 index 0000000..b43e0eb --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_img2img.py @@ -0,0 +1,1113 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from packaging import version +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...configuration_utils import FrozenDict +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + PIL_INTERPOLATION, + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from . import StableDiffusionPipelineOutput +from .safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import requests + >>> import torch + >>> from PIL import Image + >>> from io import BytesIO + + >>> from diffusers import StableDiffusionImg2ImgPipeline + + >>> device = "cuda" + >>> model_id_or_path = "runwayml/stable-diffusion-v1-5" + >>> pipe = StableDiffusionImg2ImgPipeline.from_pretrained(model_id_or_path, torch_dtype=torch.float16) + >>> pipe = pipe.to(device) + + >>> url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + + >>> response = requests.get(url) + >>> init_image = Image.open(BytesIO(response.content)).convert("RGB") + >>> init_image = init_image.resize((768, 512)) + + >>> prompt = "A fantasy landscape, trending on artstation" + + >>> images = pipe(prompt=prompt, image=init_image, strength=0.75, guidance_scale=7.5).images + >>> images[0].save("fantasy_landscape.png") + ``` +""" + + +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +def preprocess(image): + deprecation_message = "The preprocess method is deprecated and will be removed in diffusers 1.0.0. Please use VaeImageProcessor.preprocess(...) instead" + deprecate("preprocess", "1.0.0", deprecation_message, standard_warn=False) + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 8 for x in (w, h)) # resize to integer multiple of 8 + + image = [np.array(i.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + return image + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class StableDiffusionImg2ImgPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, + LoraLoaderMixin, + FromSingleFileMixin, +): + r""" + Pipeline for text-guided image-to-image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: CLIPVisionModelWithProjection = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely. If your checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + strength, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + def prepare_latents(self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + init_latents = image + + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + elif isinstance(generator, list): + init_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(batch_size) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + init_latents = self.vae.config.scaling_factor * init_latents + + if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0: + # expand init_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {init_latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_image_per_prompt = batch_size // init_latents.shape[0] + init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0) + elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts." + ) + else: + init_latents = torch.cat([init_latents], dim=0) + + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + latents = init_latents + + return latents + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + strength: float = 0.8, + num_inference_steps: Optional[int] = 50, + timesteps: List[int] = None, + guidance_scale: Optional[float] = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: Optional[float] = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: int = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be used as the starting point. For both + numpy array and pytorch tensor, the expected value range is between `[0, 1]` If it's a tensor or a list + or tensors, the expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a + list of arrays, the expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image + latents as `image`, but if passing latents directly it is not encoded again. + strength (`float`, *optional*, defaults to 0.8): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + strength, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Preprocess image + image = self.image_processor.preprocess(image) + + # 5. set timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + + # 6. Prepare latent variables + latents = self.prepare_latents( + image, + latent_timestep, + batch_size, + num_images_per_prompt, + prompt_embeds.dtype, + device, + generator, + ) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + # 7.2 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False, generator=generator)[ + 0 + ] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py new file mode 100755 index 0000000..221d5c2 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_inpaint.py @@ -0,0 +1,1430 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from packaging import version +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...configuration_utils import FrozenDict +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AsymmetricAutoencoderKL, AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import USE_PEFT_BACKEND, deprecate, logging, scale_lora_layers, unscale_lora_layers +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from . import StableDiffusionPipelineOutput +from .safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def prepare_mask_and_masked_image(image, mask, height, width, return_image: bool = False): + """ + Prepares a pair (image, mask) to be consumed by the Stable Diffusion pipeline. This means that those inputs will be + converted to ``torch.Tensor`` with shapes ``batch x channels x height x width`` where ``channels`` is ``3`` for the + ``image`` and ``1`` for the ``mask``. + + The ``image`` will be converted to ``torch.float32`` and normalized to be in ``[-1, 1]``. The ``mask`` will be + binarized (``mask > 0.5``) and cast to ``torch.float32`` too. + + Args: + image (Union[np.array, PIL.Image, torch.Tensor]): The image to inpaint. + It can be a ``PIL.Image``, or a ``height x width x 3`` ``np.array`` or a ``channels x height x width`` + ``torch.Tensor`` or a ``batch x channels x height x width`` ``torch.Tensor``. + mask (_type_): The mask to apply to the image, i.e. regions to inpaint. + It can be a ``PIL.Image``, or a ``height x width`` ``np.array`` or a ``1 x height x width`` + ``torch.Tensor`` or a ``batch x 1 x height x width`` ``torch.Tensor``. + + + Raises: + ValueError: ``torch.Tensor`` images should be in the ``[-1, 1]`` range. ValueError: ``torch.Tensor`` mask + should be in the ``[0, 1]`` range. ValueError: ``mask`` and ``image`` should have the same spatial dimensions. + TypeError: ``mask`` is a ``torch.Tensor`` but ``image`` is not + (ot the other way around). + + Returns: + tuple[torch.Tensor]: The pair (mask, masked_image) as ``torch.Tensor`` with 4 + dimensions: ``batch x channels x height x width``. + """ + deprecation_message = "The prepare_mask_and_masked_image method is deprecated and will be removed in a future version. Please use VaeImageProcessor.preprocess instead" + deprecate( + "prepare_mask_and_masked_image", + "0.30.0", + deprecation_message, + ) + if image is None: + raise ValueError("`image` input cannot be undefined.") + + if mask is None: + raise ValueError("`mask_image` input cannot be undefined.") + + if isinstance(image, torch.Tensor): + if not isinstance(mask, torch.Tensor): + raise TypeError(f"`image` is a torch.Tensor but `mask` (type: {type(mask)} is not") + + # Batch single image + if image.ndim == 3: + assert image.shape[0] == 3, "Image outside a batch should be of shape (3, H, W)" + image = image.unsqueeze(0) + + # Batch and add channel dim for single mask + if mask.ndim == 2: + mask = mask.unsqueeze(0).unsqueeze(0) + + # Batch single mask or add channel dim + if mask.ndim == 3: + # Single batched mask, no channel dim or single mask not batched but channel dim + if mask.shape[0] == 1: + mask = mask.unsqueeze(0) + + # Batched masks no channel dim + else: + mask = mask.unsqueeze(1) + + assert image.ndim == 4 and mask.ndim == 4, "Image and Mask must have 4 dimensions" + assert image.shape[-2:] == mask.shape[-2:], "Image and Mask must have the same spatial dimensions" + assert image.shape[0] == mask.shape[0], "Image and Mask must have the same batch size" + + # Check image is in [-1, 1] + if image.min() < -1 or image.max() > 1: + raise ValueError("Image should be in [-1, 1] range") + + # Check mask is in [0, 1] + if mask.min() < 0 or mask.max() > 1: + raise ValueError("Mask should be in [0, 1] range") + + # Binarize mask + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + + # Image as float32 + image = image.to(dtype=torch.float32) + elif isinstance(mask, torch.Tensor): + raise TypeError(f"`mask` is a torch.Tensor but `image` (type: {type(image)} is not") + else: + # preprocess image + if isinstance(image, (PIL.Image.Image, np.ndarray)): + image = [image] + if isinstance(image, list) and isinstance(image[0], PIL.Image.Image): + # resize all images w.r.t passed height an width + image = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in image] + image = [np.array(i.convert("RGB"))[None, :] for i in image] + image = np.concatenate(image, axis=0) + elif isinstance(image, list) and isinstance(image[0], np.ndarray): + image = np.concatenate([i[None, :] for i in image], axis=0) + + image = image.transpose(0, 3, 1, 2) + image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0 + + # preprocess mask + if isinstance(mask, (PIL.Image.Image, np.ndarray)): + mask = [mask] + + if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image): + mask = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in mask] + mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0) + mask = mask.astype(np.float32) / 255.0 + elif isinstance(mask, list) and isinstance(mask[0], np.ndarray): + mask = np.concatenate([m[None, None, :] for m in mask], axis=0) + + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + mask = torch.from_numpy(mask) + + masked_image = image * (mask < 0.5) + + # n.b. ensure backwards compatibility as old function does not return image + if return_image: + return mask, masked_image, image + + return mask, masked_image + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class StableDiffusionInpaintPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, + LoraLoaderMixin, + FromSingleFileMixin, +): + r""" + Pipeline for text-guided image inpainting using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + + Args: + vae ([`AutoencoderKL`, `AsymmetricAutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds", "mask", "masked_image_latents"] + + def __init__( + self, + vae: Union[AutoencoderKL, AsymmetricAutoencoderKL], + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: CLIPVisionModelWithProjection = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "skip_prk_steps") and scheduler.config.skip_prk_steps is False: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration" + " `skip_prk_steps`. `skip_prk_steps` should be set to True in the configuration file. Please make" + " sure to update the config accordingly as not setting `skip_prk_steps` in the config might lead to" + " incorrect results in future versions. If you have downloaded this checkpoint from the Hugging Face" + " Hub, it would be very nice if you could open a Pull request for the" + " `scheduler/scheduler_config.json` file" + ) + deprecate("skip_prk_steps not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["skip_prk_steps"] = True + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely .If you're checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + # Check shapes, assume num_channels_latents == 4, num_channels_mask == 1, num_channels_masked == 4 + if unet.config.in_channels != 9: + logger.info(f"You have loaded a UNet with {unet.config.in_channels} input channels which.") + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + image, + mask_image, + height, + width, + strength, + callback_steps, + output_type, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + padding_mask_crop=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if height % self.vae_scale_factor != 0 or width % self.vae_scale_factor != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + if padding_mask_crop is not None: + if not isinstance(image, PIL.Image.Image): + raise ValueError( + f"The image should be a PIL image when inpainting mask crop, but is of type" f" {type(image)}." + ) + if not isinstance(mask_image, PIL.Image.Image): + raise ValueError( + f"The mask image should be a PIL image when inpainting mask crop, but is of type" + f" {type(mask_image)}." + ) + if output_type != "pil": + raise ValueError(f"The output type should be PIL when inpainting mask crop, but is" f" {output_type}.") + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + image=None, + timestep=None, + is_strength_max=True, + return_noise=False, + return_image_latents=False, + ): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if (image is None or timestep is None) and not is_strength_max: + raise ValueError( + "Since strength < 1. initial latents are to be initialised as a combination of Image + Noise." + "However, either the image or the noise timestep has not been provided." + ) + + if return_image_latents or (latents is None and not is_strength_max): + image = image.to(device=device, dtype=dtype) + + if image.shape[1] == 4: + image_latents = image + else: + image_latents = self._encode_vae_image(image=image, generator=generator) + image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1) + + if latents is None: + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + # if strength is 1. then initialise the latents to noise, else initial to image + noise + latents = noise if is_strength_max else self.scheduler.add_noise(image_latents, noise, timestep) + # if pure noise then scale the initial latents by the Scheduler's init sigma + latents = latents * self.scheduler.init_noise_sigma if is_strength_max else latents + else: + noise = latents.to(device) + latents = noise * self.scheduler.init_noise_sigma + + outputs = (latents,) + + if return_noise: + outputs += (noise,) + + if return_image_latents: + outputs += (image_latents,) + + return outputs + + def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator): + if isinstance(generator, list): + image_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(image.shape[0]) + ] + image_latents = torch.cat(image_latents, dim=0) + else: + image_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + image_latents = self.vae.config.scaling_factor * image_latents + + return image_latents + + def prepare_mask_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + mask = torch.nn.functional.interpolate( + mask, size=(height // self.vae_scale_factor, width // self.vae_scale_factor) + ) + mask = mask.to(device=device, dtype=dtype) + + masked_image = masked_image.to(device=device, dtype=dtype) + + if masked_image.shape[1] == 4: + masked_image_latents = masked_image + else: + masked_image_latents = self._encode_vae_image(masked_image, generator=generator) + + # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method + if mask.shape[0] < batch_size: + if not batch_size % mask.shape[0] == 0: + raise ValueError( + "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to" + f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number" + " of masks that you pass is divisible by the total requested batch size." + ) + mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1) + if masked_image_latents.shape[0] < batch_size: + if not batch_size % masked_image_latents.shape[0] == 0: + raise ValueError( + "The passed images and the required batch size don't match. Images are supposed to be duplicated" + f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed." + " Make sure the number of images that you pass is divisible by the total requested batch size." + ) + masked_image_latents = masked_image_latents.repeat(batch_size // masked_image_latents.shape[0], 1, 1, 1) + + mask = torch.cat([mask] * 2) if do_classifier_free_guidance else mask + masked_image_latents = ( + torch.cat([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents + ) + + # aligning device to prevent device errors when concating it with the latent model input + masked_image_latents = masked_image_latents.to(device=device, dtype=dtype) + return mask, masked_image_latents + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + mask_image: PipelineImageInput = None, + masked_image_latents: torch.FloatTensor = None, + height: Optional[int] = None, + width: Optional[int] = None, + padding_mask_crop: Optional[int] = None, + strength: float = 1.0, + num_inference_steps: int = 50, + timesteps: List[int] = None, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: int = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to be inpainted (which parts of the image to + be masked out with `mask_image` and repainted according to `prompt`). For both numpy array and pytorch + tensor, the expected value range is between `[0, 1]` If it's a tensor or a list or tensors, the + expected shape should be `(B, C, H, W)` or `(C, H, W)`. If it is a numpy array or a list of arrays, the + expected shape should be `(B, H, W, C)` or `(H, W, C)` It can also accept image latents as `image`, but + if passing latents directly it is not encoded again. + mask_image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image`, numpy array or tensor representing an image batch to mask `image`. White pixels in the mask + are repainted while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a + single channel (luminance) before use. If it's a numpy array or pytorch tensor, it should contain one + color channel (L) instead of 3, so the expected shape for pytorch tensor would be `(B, 1, H, W)`, `(B, + H, W)`, `(1, H, W)`, `(H, W)`. And for numpy array would be for `(B, H, W, 1)`, `(B, H, W)`, `(H, W, + 1)`, or `(H, W)`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + padding_mask_crop (`int`, *optional*, defaults to `None`): + The size of margin in the crop to be applied to the image and masking. If `None`, no crop is applied to image and mask_image. If + `padding_mask_crop` is not `None`, it will first find a rectangular region with the same aspect ration of the image and + contains all masked area, and then expand that area based on `padding_mask_crop`. The image and mask_image will then be cropped based on + the expanded area before resizing to the original image size for inpainting. This is useful when the masked area is small while the image is large + and contain information inreleant for inpainging, such as background. + strength (`float`, *optional*, defaults to 1.0): + Indicates extent to transform the reference `image`. Must be between 0 and 1. `image` is used as a + starting point and more noise is added the higher the `strength`. The number of denoising steps depends + on the amount of noise initially added. When `strength` is 1, added noise is maximum and the denoising + process runs for the full number of iterations specified in `num_inference_steps`. A value of 1 + essentially ignores `image`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. This parameter is modulated by `strength`. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + Examples: + + ```py + >>> import PIL + >>> import requests + >>> import torch + >>> from io import BytesIO + + >>> from diffusers import StableDiffusionInpaintPipeline + + + >>> def download_image(url): + ... response = requests.get(url) + ... return PIL.Image.open(BytesIO(response.content)).convert("RGB") + + + >>> img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png" + >>> mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png" + + >>> init_image = download_image(img_url).resize((512, 512)) + >>> mask_image = download_image(mask_url).resize((512, 512)) + + >>> pipe = StableDiffusionInpaintPipeline.from_pretrained( + ... "runwayml/stable-diffusion-inpainting", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "Face of a yellow cat, high resolution, sitting on a park bench" + >>> image = pipe(prompt=prompt, image=init_image, mask_image=mask_image).images[0] + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs + self.check_inputs( + prompt, + image, + mask_image, + height, + width, + strength, + callback_steps, + output_type, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + padding_mask_crop, + ) + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. set timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + timesteps, num_inference_steps = self.get_timesteps( + num_inference_steps=num_inference_steps, strength=strength, device=device + ) + # check that number of inference steps is not < 1 - as this doesn't make sense + if num_inference_steps < 1: + raise ValueError( + f"After adjusting the num_inference_steps by strength parameter: {strength}, the number of pipeline" + f"steps is {num_inference_steps} which is < 1 and not appropriate for this pipeline." + ) + # at which timestep to set the initial noise (n.b. 50% if strength is 0.5) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + # create a boolean to check if the strength is set to 1. if so then initialise the latents with pure noise + is_strength_max = strength == 1.0 + + # 5. Preprocess mask and image + + if padding_mask_crop is not None: + crops_coords = self.mask_processor.get_crop_region(mask_image, width, height, pad=padding_mask_crop) + resize_mode = "fill" + else: + crops_coords = None + resize_mode = "default" + + original_image = image + init_image = self.image_processor.preprocess( + image, height=height, width=width, crops_coords=crops_coords, resize_mode=resize_mode + ) + init_image = init_image.to(dtype=torch.float32) + + # 6. Prepare latent variables + num_channels_latents = self.vae.config.latent_channels + num_channels_unet = self.unet.config.in_channels + return_image_latents = num_channels_unet == 4 + + latents_outputs = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + image=init_image, + timestep=latent_timestep, + is_strength_max=is_strength_max, + return_noise=True, + return_image_latents=return_image_latents, + ) + + if return_image_latents: + latents, noise, image_latents = latents_outputs + else: + latents, noise = latents_outputs + + # 7. Prepare mask latent variables + mask_condition = self.mask_processor.preprocess( + mask_image, height=height, width=width, resize_mode=resize_mode, crops_coords=crops_coords + ) + + if masked_image_latents is None: + masked_image = init_image * (mask_condition < 0.5) + else: + masked_image = masked_image_latents + + mask, masked_image_latents = self.prepare_mask_latents( + mask_condition, + masked_image, + batch_size * num_images_per_prompt, + height, + width, + prompt_embeds.dtype, + device, + generator, + self.do_classifier_free_guidance, + ) + + # 8. Check that sizes of mask, masked image and latents match + if num_channels_unet == 9: + # default case for runwayml/stable-diffusion-inpainting + num_channels_mask = mask.shape[1] + num_channels_masked_image = masked_image_latents.shape[1] + if num_channels_latents + num_channels_mask + num_channels_masked_image != self.unet.config.in_channels: + raise ValueError( + f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects" + f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +" + f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}" + f" = {num_channels_latents+num_channels_masked_image+num_channels_mask}. Please verify the config of" + " `pipeline.unet` or your `mask_image` or `image` input." + ) + elif num_channels_unet != 4: + raise ValueError( + f"The unet {self.unet.__class__} should have either 4 or 9 input channels, not {self.unet.config.in_channels}." + ) + + # 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 9.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + # 9.2 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + # 10. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + # concat latents, mask, masked_image_latents in the channel dimension + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + if num_channels_unet == 9: + latent_model_input = torch.cat([latent_model_input, mask, masked_image_latents], dim=1) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + if num_channels_unet == 4: + init_latents_proper = image_latents + if self.do_classifier_free_guidance: + init_mask, _ = mask.chunk(2) + else: + init_mask = mask + + if i < len(timesteps) - 1: + noise_timestep = timesteps[i + 1] + init_latents_proper = self.scheduler.add_noise( + init_latents_proper, noise, torch.tensor([noise_timestep]) + ) + + latents = (1 - init_mask) * init_latents_proper + init_mask * latents + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + mask = callback_outputs.pop("mask", mask) + masked_image_latents = callback_outputs.pop("masked_image_latents", masked_image_latents) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + condition_kwargs = {} + if isinstance(self.vae, AsymmetricAutoencoderKL): + init_image = init_image.to(device=device, dtype=masked_image_latents.dtype) + init_image_condition = init_image.clone() + init_image = self._encode_vae_image(init_image, generator=generator) + mask_condition = mask_condition.to(device=device, dtype=masked_image_latents.dtype) + condition_kwargs = {"image": init_image_condition, "mask": mask_condition} + image = self.vae.decode( + latents / self.vae.config.scaling_factor, return_dict=False, generator=generator, **condition_kwargs + )[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + if padding_mask_crop is not None: + image = [self.image_processor.apply_overlay(mask_image, original_image, i, crops_coords) for i in image] + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py new file mode 100755 index 0000000..cbb6ed4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_instruct_pix2pix.py @@ -0,0 +1,807 @@ +# Copyright 2024 The InstructPix2Pix Authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import PIL_INTERPOLATION, deprecate, logging +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from . import StableDiffusionPipelineOutput +from .safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.preprocess +def preprocess(image): + deprecation_message = "The preprocess method is deprecated and will be removed in diffusers 1.0.0. Please use VaeImageProcessor.preprocess(...) instead" + deprecate("preprocess", "1.0.0", deprecation_message, standard_warn=False) + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 8 for x in (w, h)) # resize to integer multiple of 8 + + image = [np.array(i.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + return image + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +class StableDiffusionInstructPix2PixPipeline( + DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin, IPAdapterMixin +): + r""" + Pipeline for pixel-level image editing by following text instructions (based on Stable Diffusion). + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "image_latents"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: Optional[CLIPVisionModelWithProjection] = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + num_inference_steps: int = 100, + guidance_scale: float = 7.5, + image_guidance_scale: float = 1.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor` `np.ndarray`, `PIL.Image.Image`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image` or tensor representing an image batch to be repainted according to `prompt`. Can also accept + image latents as `image`, but if passing latents directly it is not encoded again. + num_inference_steps (`int`, *optional*, defaults to 100): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + image_guidance_scale (`float`, *optional*, defaults to 1.5): + Push the generated image towards the inital `image`. Image guidance scale is enabled by setting + `image_guidance_scale > 1`. Higher image guidance scale encourages generated images that are closely + linked to the source `image`, usually at the expense of lower image quality. This pipeline requires a + value of at least `1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): + Optional image input to work with IP Adapters. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + ```py + >>> import PIL + >>> import requests + >>> import torch + >>> from io import BytesIO + + >>> from diffusers import StableDiffusionInstructPix2PixPipeline + + + >>> def download_image(url): + ... response = requests.get(url) + ... return PIL.Image.open(BytesIO(response.content)).convert("RGB") + + + >>> img_url = "https://huggingface.co/datasets/diffusers/diffusers-images-docs/resolve/main/mountain.png" + + >>> image = download_image(img_url).resize((512, 512)) + + >>> pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained( + ... "timbrooks/instruct-pix2pix", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "make the mountains snowy" + >>> image = pipe(prompt=prompt, image=image).images[0] + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 0. Check inputs + self.check_inputs( + prompt, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + callback_on_step_end_tensor_inputs, + ) + self._guidance_scale = guidance_scale + self._image_guidance_scale = image_guidance_scale + + device = self._execution_device + + if ip_adapter_image is not None: + output_hidden_state = False if isinstance(self.unet.encoder_hid_proj, ImageProjection) else True + image_embeds, negative_image_embeds = self.encode_image( + ip_adapter_image, device, num_images_per_prompt, output_hidden_state + ) + if self.do_classifier_free_guidance: + image_embeds = torch.cat([image_embeds, negative_image_embeds, negative_image_embeds]) + + if image is None: + raise ValueError("`image` input cannot be undefined.") + + # 1. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 2. Encode input prompt + prompt_embeds = self._encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + # 3. Preprocess image + image = self.image_processor.preprocess(image) + + # 4. set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare Image latents + image_latents = self.prepare_image_latents( + image, + batch_size, + num_images_per_prompt, + prompt_embeds.dtype, + device, + self.do_classifier_free_guidance, + ) + + height, width = image_latents.shape[-2:] + height = height * self.vae_scale_factor + width = width * self.vae_scale_factor + + # 6. Prepare latent variables + num_channels_latents = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 7. Check that shapes of latents and image match the UNet channels + num_channels_image = image_latents.shape[1] + if num_channels_latents + num_channels_image != self.unet.config.in_channels: + raise ValueError( + f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects" + f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +" + f" `num_channels_image`: {num_channels_image} " + f" = {num_channels_latents+num_channels_image}. Please verify the config of" + " `pipeline.unet` or your `image` input." + ) + + # 8. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 8.1 Add image embeds for IP-Adapter + added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None + + # 9. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # Expand the latents if we are doing classifier free guidance. + # The latents are expanded 3 times because for pix2pix the guidance\ + # is applied for both the text and the input image. + latent_model_input = torch.cat([latents] * 3) if self.do_classifier_free_guidance else latents + + # concat latents, image_latents in the channel dimension + scaled_latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + scaled_latent_model_input = torch.cat([scaled_latent_model_input, image_latents], dim=1) + + # predict the noise residual + noise_pred = self.unet( + scaled_latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_text, noise_pred_image, noise_pred_uncond = noise_pred.chunk(3) + noise_pred = ( + noise_pred_uncond + + self.guidance_scale * (noise_pred_text - noise_pred_image) + + self.image_guidance_scale * (noise_pred_image - noise_pred_uncond) + ) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + image_latents = callback_outputs.pop("image_latents", image_latents) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_ prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + """ + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + else: + prompt_embeds_dtype = self.unet.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + # pix2pix has two negative embeddings, and unlike in other pipelines latents are ordered [prompt_embeds, negative_prompt_embeds, negative_prompt_embeds] + prompt_embeds = torch.cat([prompt_embeds, negative_prompt_embeds, negative_prompt_embeds]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def check_inputs( + self, + prompt, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_image_latents( + self, image, batch_size, num_images_per_prompt, dtype, device, do_classifier_free_guidance, generator=None + ): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + image_latents = image + else: + image_latents = retrieve_latents(self.vae.encode(image), sample_mode="argmax") + + if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0: + # expand image_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {image_latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_image_per_prompt = batch_size // image_latents.shape[0] + image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0) + elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts." + ) + else: + image_latents = torch.cat([image_latents], dim=0) + + if do_classifier_free_guidance: + uncond_image_latents = torch.zeros_like(image_latents) + image_latents = torch.cat([image_latents, image_latents, uncond_image_latents], dim=0) + + return image_latents + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def image_guidance_scale(self): + return self._image_guidance_scale + + @property + def num_timesteps(self): + return self._num_timesteps + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self.guidance_scale > 1.0 and self.image_guidance_scale >= 1.0 diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py new file mode 100755 index 0000000..918dffe --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_latent_upscale.py @@ -0,0 +1,495 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import warnings +from typing import Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from transformers import CLIPTextModel, CLIPTokenizer + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...schedulers import EulerDiscreteScheduler +from ...utils import deprecate, logging +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput, StableDiffusionMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.preprocess +def preprocess(image): + warnings.warn( + "The preprocess method is deprecated and will be removed in a future version. Please" + " use VaeImageProcessor.preprocess instead", + FutureWarning, + ) + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 64 for x in (w, h)) # resize to integer multiple of 64 + + image = [np.array(i.resize((w, h)))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + return image + + +class StableDiffusionLatentUpscalePipeline(DiffusionPipeline, StableDiffusionMixin, FromSingleFileMixin): + r""" + Pipeline for upscaling Stable Diffusion output image resolution by a factor of 2. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A [`EulerDiscreteScheduler`] to be used in combination with `unet` to denoise the encoded image latents. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: EulerDiscreteScheduler, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, resample="bicubic") + + def _encode_prompt(self, prompt, device, do_classifier_free_guidance, negative_prompt): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `list(int)`): + prompt to be encoded + device: (`torch.device`): + torch device + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + """ + batch_size = len(prompt) if isinstance(prompt, list) else 1 + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_length=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + text_encoder_out = self.text_encoder( + text_input_ids.to(device), + output_hidden_states=True, + ) + text_embeddings = text_encoder_out.hidden_states[-1] + text_pooler_out = text_encoder_out.pooler_output + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = text_input_ids.shape[-1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_length=True, + return_tensors="pt", + ) + + uncond_encoder_out = self.text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + + uncond_embeddings = uncond_encoder_out.hidden_states[-1] + uncond_pooler_out = uncond_encoder_out.pooler_output + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + text_embeddings = torch.cat([uncond_embeddings, text_embeddings]) + text_pooler_out = torch.cat([uncond_pooler_out, text_pooler_out]) + + return text_embeddings, text_pooler_out + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def check_inputs(self, prompt, image, callback_steps): + if not isinstance(prompt, str) and not isinstance(prompt, list): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if ( + not isinstance(image, torch.Tensor) + and not isinstance(image, PIL.Image.Image) + and not isinstance(image, list) + ): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or `list` but is {type(image)}" + ) + + # verify batch size of prompt and image are same if image is a list or tensor + if isinstance(image, list) or isinstance(image, torch.Tensor): + if isinstance(prompt, str): + batch_size = 1 + else: + batch_size = len(prompt) + if isinstance(image, list): + image_batch_size = len(image) + else: + image_batch_size = image.shape[0] if image.ndim == 4 else 1 + if batch_size != image_batch_size: + raise ValueError( + f"`prompt` has batch size {batch_size} and `image` has batch size {image_batch_size}." + " Please make sure that passed `prompt` matches the batch size of `image`." + ) + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height, width) + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + image: PipelineImageInput = None, + num_inference_steps: int = 75, + guidance_scale: float = 9.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide image upscaling. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image` or tensor representing an image batch to be upscaled. If it's a tensor, it can be either a + latent output from a Stable Diffusion model or an image tensor in the range `[-1, 1]`. It is considered + a `latent` if `image.shape[1]` is `4`; otherwise, it is considered to be an image representation and + encoded using this pipeline's `vae` encoder. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Examples: + ```py + >>> from diffusers import StableDiffusionLatentUpscalePipeline, StableDiffusionPipeline + >>> import torch + + + >>> pipeline = StableDiffusionPipeline.from_pretrained( + ... "CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16 + ... ) + >>> pipeline.to("cuda") + + >>> model_id = "stabilityai/sd-x2-latent-upscaler" + >>> upscaler = StableDiffusionLatentUpscalePipeline.from_pretrained(model_id, torch_dtype=torch.float16) + >>> upscaler.to("cuda") + + >>> prompt = "a photo of an astronaut high resolution, unreal engine, ultra realistic" + >>> generator = torch.manual_seed(33) + + >>> low_res_latents = pipeline(prompt, generator=generator, output_type="latent").images + + >>> with torch.no_grad(): + ... image = pipeline.decode_latents(low_res_latents) + >>> image = pipeline.numpy_to_pil(image)[0] + + >>> image.save("../images/a1.png") + + >>> upscaled_image = upscaler( + ... prompt=prompt, + ... image=low_res_latents, + ... num_inference_steps=20, + ... guidance_scale=0, + ... generator=generator, + ... ).images[0] + + >>> upscaled_image.save("../images/a2.png") + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images. + """ + + # 1. Check inputs + self.check_inputs(prompt, image, callback_steps) + + # 2. Define call parameters + batch_size = 1 if isinstance(prompt, str) else len(prompt) + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + if guidance_scale == 0: + prompt = [""] * batch_size + + # 3. Encode input prompt + text_embeddings, text_pooler_out = self._encode_prompt( + prompt, device, do_classifier_free_guidance, negative_prompt + ) + + # 4. Preprocess image + image = self.image_processor.preprocess(image) + image = image.to(dtype=text_embeddings.dtype, device=device) + if image.shape[1] == 3: + # encode image if not in latent-space yet + image = self.vae.encode(image).latent_dist.sample() * self.vae.config.scaling_factor + + # 5. set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + batch_multiplier = 2 if do_classifier_free_guidance else 1 + image = image[None, :] if image.ndim == 3 else image + image = torch.cat([image] * batch_multiplier) + + # 5. Add noise to image (set to be 0): + # (see below notes from the author): + # "the This step theoretically can make the model work better on out-of-distribution inputs, but mostly just seems to make it match the input less, so it's turned off by default." + noise_level = torch.tensor([0.0], dtype=torch.float32, device=device) + noise_level = torch.cat([noise_level] * image.shape[0]) + inv_noise_level = (noise_level**2 + 1) ** (-0.5) + + image_cond = F.interpolate(image, scale_factor=2, mode="nearest") * inv_noise_level[:, None, None, None] + image_cond = image_cond.to(text_embeddings.dtype) + + noise_level_embed = torch.cat( + [ + torch.ones(text_pooler_out.shape[0], 64, dtype=text_pooler_out.dtype, device=device), + torch.zeros(text_pooler_out.shape[0], 64, dtype=text_pooler_out.dtype, device=device), + ], + dim=1, + ) + + timestep_condition = torch.cat([noise_level_embed, text_pooler_out], dim=1) + + # 6. Prepare latent variables + height, width = image.shape[2:] + num_channels_latents = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size, + num_channels_latents, + height * 2, # 2x upscale + width * 2, + text_embeddings.dtype, + device, + generator, + latents, + ) + + # 7. Check that sizes of image and latents match + num_channels_image = image.shape[1] + if num_channels_latents + num_channels_image != self.unet.config.in_channels: + raise ValueError( + f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects" + f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +" + f" `num_channels_image`: {num_channels_image} " + f" = {num_channels_latents+num_channels_image}. Please verify the config of" + " `pipeline.unet` or your `image` input." + ) + + # 9. Denoising loop + num_warmup_steps = 0 + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + sigma = self.scheduler.sigmas[i] + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + scaled_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + scaled_model_input = torch.cat([scaled_model_input, image_cond], dim=1) + # preconditioning parameter based on Karras et al. (2022) (table 1) + timestep = torch.log(sigma) * 0.25 + + noise_pred = self.unet( + scaled_model_input, + timestep, + encoder_hidden_states=text_embeddings, + timestep_cond=timestep_condition, + ).sample + + # in original repo, the output contains a variance channel that's not used + noise_pred = noise_pred[:, :-1] + + # apply preconditioning, based on table 1 in Karras et al. (2022) + inv_sigma = 1 / (sigma**2 + 1) + noise_pred = inv_sigma * latent_model_input + self.scheduler.scale_model_input(sigma, t) * noise_pred + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = latents + + image = self.image_processor.postprocess(image, output_type=output_type) + + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py new file mode 100755 index 0000000..2d04cf4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_diffusion_upscale.py @@ -0,0 +1,808 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import warnings +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.attention_processor import ( + AttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import DDPMScheduler, KarrasDiffusionSchedulers +from ...utils import USE_PEFT_BACKEND, deprecate, logging, scale_lora_layers, unscale_lora_layers +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from . import StableDiffusionPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def preprocess(image): + warnings.warn( + "The preprocess method is deprecated and will be removed in a future version. Please" + " use VaeImageProcessor.preprocess instead", + FutureWarning, + ) + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 64 for x in (w, h)) # resize to integer multiple of 64 + + image = [np.array(i.resize((w, h)))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + return image + + +class StableDiffusionUpscalePipeline( + DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin, FromSingleFileMixin +): + r""" + Pipeline for text-guided image super-resolution using Stable Diffusion 2. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + low_res_scheduler ([`SchedulerMixin`]): + A scheduler used to add initial noise to the low resolution conditioning image. It must be an instance of + [`DDPMScheduler`]. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["watermarker", "safety_checker", "feature_extractor"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + low_res_scheduler: DDPMScheduler, + scheduler: KarrasDiffusionSchedulers, + safety_checker: Optional[Any] = None, + feature_extractor: Optional[CLIPImageProcessor] = None, + watermarker: Optional[Any] = None, + max_noise_level: int = 350, + ): + super().__init__() + + if hasattr( + vae, "config" + ): # check if vae has a config attribute `scaling_factor` and if it is set to 0.08333, else set it to 0.08333 and deprecate + is_vae_scaling_factor_set_to_0_08333 = ( + hasattr(vae.config, "scaling_factor") and vae.config.scaling_factor == 0.08333 + ) + if not is_vae_scaling_factor_set_to_0_08333: + deprecation_message = ( + "The configuration file of the vae does not contain `scaling_factor` or it is set to" + f" {vae.config.scaling_factor}, which seems highly unlikely. If your checkpoint is a fine-tuned" + " version of `stabilityai/stable-diffusion-x4-upscaler` you should change 'scaling_factor' to" + " 0.08333 Please make sure to update the config accordingly, as not doing so might lead to" + " incorrect results in future versions. If you have downloaded this checkpoint from the Hugging" + " Face Hub, it would be very nice if you could open a Pull Request for the `vae/config.json` file" + ) + deprecate("wrong scaling_factor", "1.0.0", deprecation_message, standard_warn=False) + vae.register_to_config(scaling_factor=0.08333) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + low_res_scheduler=low_res_scheduler, + scheduler=scheduler, + safety_checker=safety_checker, + watermarker=watermarker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, resample="bicubic") + self.register_to_config(max_noise_level=max_noise_level) + + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is not None: + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, nsfw_detected, watermark_detected = self.safety_checker( + images=image, + clip_input=safety_checker_input.pixel_values.to(dtype=dtype), + ) + else: + nsfw_detected = None + watermark_detected = None + + if hasattr(self, "unet_offload_hook") and self.unet_offload_hook is not None: + self.unet_offload_hook.offload() + + return image, nsfw_detected, watermark_detected + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def check_inputs( + self, + prompt, + image, + noise_level, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if ( + not isinstance(image, torch.Tensor) + and not isinstance(image, PIL.Image.Image) + and not isinstance(image, np.ndarray) + and not isinstance(image, list) + ): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `np.ndarray`, `PIL.Image.Image` or `list` but is {type(image)}" + ) + + # verify batch size of prompt and image are same if image is a list or tensor or numpy array + if isinstance(image, list) or isinstance(image, torch.Tensor) or isinstance(image, np.ndarray): + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if isinstance(image, list): + image_batch_size = len(image) + else: + image_batch_size = image.shape[0] + if batch_size != image_batch_size: + raise ValueError( + f"`prompt` has batch size {batch_size} and `image` has batch size {image_batch_size}." + " Please make sure that passed `prompt` matches the batch size of `image`." + ) + + # check noise level + if noise_level > self.config.max_noise_level: + raise ValueError(f"`noise_level` has to be <= {self.config.max_noise_level} but is {noise_level}") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height, width) + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: PipelineImageInput = None, + num_inference_steps: int = 75, + guidance_scale: float = 9.0, + noise_level: int = 20, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: int = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`torch.FloatTensor`, `PIL.Image.Image`, `np.ndarray`, `List[torch.FloatTensor]`, `List[PIL.Image.Image]`, or `List[np.ndarray]`): + `Image` or tensor representing an image batch to be upscaled. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Examples: + ```py + >>> import requests + >>> from PIL import Image + >>> from io import BytesIO + >>> from diffusers import StableDiffusionUpscalePipeline + >>> import torch + + >>> # load model and scheduler + >>> model_id = "stabilityai/stable-diffusion-x4-upscaler" + >>> pipeline = StableDiffusionUpscalePipeline.from_pretrained( + ... model_id, revision="fp16", torch_dtype=torch.float16 + ... ) + >>> pipeline = pipeline.to("cuda") + + >>> # let's download an image + >>> url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale/low_res_cat.png" + >>> response = requests.get(url) + >>> low_res_img = Image.open(BytesIO(response.content)).convert("RGB") + >>> low_res_img = low_res_img.resize((128, 128)) + >>> prompt = "a white cat" + + >>> upscaled_image = pipeline(prompt=prompt, image=low_res_img).images[0] + >>> upscaled_image.save("upsampled_cat.png") + ``` + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + # 1. Check inputs + self.check_inputs( + prompt, + image, + noise_level, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + if image is None: + raise ValueError("`image` input cannot be undefined.") + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Preprocess image + image = self.image_processor.preprocess(image) + image = image.to(dtype=prompt_embeds.dtype, device=device) + + # 5. set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Add noise to image + noise_level = torch.tensor([noise_level], dtype=torch.long, device=device) + noise = randn_tensor(image.shape, generator=generator, device=device, dtype=prompt_embeds.dtype) + image = self.low_res_scheduler.add_noise(image, noise, noise_level) + + batch_multiplier = 2 if do_classifier_free_guidance else 1 + image = torch.cat([image] * batch_multiplier * num_images_per_prompt) + noise_level = torch.cat([noise_level] * image.shape[0]) + + # 6. Prepare latent variables + height, width = image.shape[2:] + num_channels_latents = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 7. Check that sizes of image and latents match + num_channels_image = image.shape[1] + if num_channels_latents + num_channels_image != self.unet.config.in_channels: + raise ValueError( + f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects" + f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +" + f" `num_channels_image`: {num_channels_image} " + f" = {num_channels_latents+num_channels_image}. Please verify the config of" + " `pipeline.unet` or your `image` input." + ) + + # 8. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 9. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + + # concat latents, mask, masked_image_latents in the channel dimension + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + latent_model_input = torch.cat([latent_model_input, image], dim=1) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + class_labels=noise_level, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + + # Ensure latents are always the same type as the VAE + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + + image, has_nsfw_concept, _ = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # 11. Apply watermark + if output_type == "pil" and self.watermarker is not None: + image = self.watermarker.apply_watermark(image) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py new file mode 100755 index 0000000..c62e0f4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip.py @@ -0,0 +1,932 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer +from transformers.models.clip.modeling_clip import CLIPTextModelOutput + +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, PriorTransformer, UNet2DConditionModel +from ...models.embeddings import get_timestep_embedding +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput, StableDiffusionMixin +from .stable_unclip_image_normalizer import StableUnCLIPImageNormalizer + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableUnCLIPPipeline + + >>> pipe = StableUnCLIPPipeline.from_pretrained( + ... "fusing/stable-unclip-2-1-l", torch_dtype=torch.float16 + ... ) # TODO update model path + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> images = pipe(prompt).images + >>> images[0].save("astronaut_horse.png") + ``` +""" + + +class StableUnCLIPPipeline(DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin): + """ + Pipeline for text-to-image generation using stable unCLIP. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + + Args: + prior_tokenizer ([`CLIPTokenizer`]): + A [`CLIPTokenizer`]. + prior_text_encoder ([`CLIPTextModelWithProjection`]): + Frozen [`CLIPTextModelWithProjection`] text-encoder. + prior ([`PriorTransformer`]): + The canonincal unCLIP prior to approximate the image embedding from the text embedding. + prior_scheduler ([`KarrasDiffusionSchedulers`]): + Scheduler used in the prior denoising process. + image_normalizer ([`StableUnCLIPImageNormalizer`]): + Used to normalize the predicted image embeddings before the noise is applied and un-normalize the image + embeddings after the noise has been applied. + image_noising_scheduler ([`KarrasDiffusionSchedulers`]): + Noise schedule for adding noise to the predicted image embeddings. The amount of noise to add is determined + by the `noise_level`. + tokenizer ([`CLIPTokenizer`]): + A [`CLIPTokenizer`]. + text_encoder ([`CLIPTextModel`]): + Frozen [`CLIPTextModel`] text-encoder. + unet ([`UNet2DConditionModel`]): + A [`UNet2DConditionModel`] to denoise the encoded image latents. + scheduler ([`KarrasDiffusionSchedulers`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + """ + + _exclude_from_cpu_offload = ["prior", "image_normalizer"] + model_cpu_offload_seq = "text_encoder->prior_text_encoder->unet->vae" + + # prior components + prior_tokenizer: CLIPTokenizer + prior_text_encoder: CLIPTextModelWithProjection + prior: PriorTransformer + prior_scheduler: KarrasDiffusionSchedulers + + # image noising components + image_normalizer: StableUnCLIPImageNormalizer + image_noising_scheduler: KarrasDiffusionSchedulers + + # regular denoising components + tokenizer: CLIPTokenizer + text_encoder: CLIPTextModel + unet: UNet2DConditionModel + scheduler: KarrasDiffusionSchedulers + + vae: AutoencoderKL + + def __init__( + self, + # prior components + prior_tokenizer: CLIPTokenizer, + prior_text_encoder: CLIPTextModelWithProjection, + prior: PriorTransformer, + prior_scheduler: KarrasDiffusionSchedulers, + # image noising components + image_normalizer: StableUnCLIPImageNormalizer, + image_noising_scheduler: KarrasDiffusionSchedulers, + # regular denoising components + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModelWithProjection, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + # vae + vae: AutoencoderKL, + ): + super().__init__() + + self.register_modules( + prior_tokenizer=prior_tokenizer, + prior_text_encoder=prior_text_encoder, + prior=prior, + prior_scheduler=prior_scheduler, + image_normalizer=image_normalizer, + image_noising_scheduler=image_noising_scheduler, + tokenizer=tokenizer, + text_encoder=text_encoder, + unet=unet, + scheduler=scheduler, + vae=vae, + ) + + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline._encode_prompt with _encode_prompt->_encode_prior_prompt, tokenizer->prior_tokenizer, text_encoder->prior_text_encoder + def _encode_prior_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + text_model_output: Optional[Union[CLIPTextModelOutput, Tuple]] = None, + text_attention_mask: Optional[torch.Tensor] = None, + ): + if text_model_output is None: + batch_size = len(prompt) if isinstance(prompt, list) else 1 + # get prompt text embeddings + text_inputs = self.prior_tokenizer( + prompt, + padding="max_length", + max_length=self.prior_tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + text_mask = text_inputs.attention_mask.bool().to(device) + + untruncated_ids = self.prior_tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.prior_tokenizer.batch_decode( + untruncated_ids[:, self.prior_tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.prior_tokenizer.model_max_length} tokens: {removed_text}" + ) + text_input_ids = text_input_ids[:, : self.prior_tokenizer.model_max_length] + + prior_text_encoder_output = self.prior_text_encoder(text_input_ids.to(device)) + + prompt_embeds = prior_text_encoder_output.text_embeds + text_enc_hid_states = prior_text_encoder_output.last_hidden_state + + else: + batch_size = text_model_output[0].shape[0] + prompt_embeds, text_enc_hid_states = text_model_output[0], text_model_output[1] + text_mask = text_attention_mask + + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + text_enc_hid_states = text_enc_hid_states.repeat_interleave(num_images_per_prompt, dim=0) + text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + if do_classifier_free_guidance: + uncond_tokens = [""] * batch_size + + uncond_input = self.prior_tokenizer( + uncond_tokens, + padding="max_length", + max_length=self.prior_tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + uncond_text_mask = uncond_input.attention_mask.bool().to(device) + negative_prompt_embeds_prior_text_encoder_output = self.prior_text_encoder( + uncond_input.input_ids.to(device) + ) + + negative_prompt_embeds = negative_prompt_embeds_prior_text_encoder_output.text_embeds + uncond_text_enc_hid_states = negative_prompt_embeds_prior_text_encoder_output.last_hidden_state + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len) + + seq_len = uncond_text_enc_hid_states.shape[1] + uncond_text_enc_hid_states = uncond_text_enc_hid_states.repeat(1, num_images_per_prompt, 1) + uncond_text_enc_hid_states = uncond_text_enc_hid_states.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + uncond_text_mask = uncond_text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + # done duplicates + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + text_enc_hid_states = torch.cat([uncond_text_enc_hid_states, text_enc_hid_states]) + + text_mask = torch.cat([uncond_text_mask, text_mask]) + + return prompt_embeds, text_enc_hid_states, text_mask + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs with prepare_extra_step_kwargs->prepare_prior_extra_step_kwargs, scheduler->prior_scheduler + def prepare_prior_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the prior_scheduler step, since not all prior_schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other prior_schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.prior_scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the prior_scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.prior_scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + noise_level, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Please make sure to define only one of the two." + ) + + if prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + + if prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + "Provide either `negative_prompt` or `negative_prompt_embeds`. Cannot leave both `negative_prompt` and `negative_prompt_embeds` undefined." + ) + + if prompt is not None and negative_prompt is not None: + if type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if noise_level < 0 or noise_level >= self.image_noising_scheduler.config.num_train_timesteps: + raise ValueError( + f"`noise_level` must be between 0 and {self.image_noising_scheduler.config.num_train_timesteps - 1}, inclusive." + ) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def noise_image_embeddings( + self, + image_embeds: torch.Tensor, + noise_level: int, + noise: Optional[torch.FloatTensor] = None, + generator: Optional[torch.Generator] = None, + ): + """ + Add noise to the image embeddings. The amount of noise is controlled by a `noise_level` input. A higher + `noise_level` increases the variance in the final un-noised images. + + The noise is applied in two ways: + 1. A noise schedule is applied directly to the embeddings. + 2. A vector of sinusoidal time embeddings are appended to the output. + + In both cases, the amount of noise is controlled by the same `noise_level`. + + The embeddings are normalized before the noise is applied and un-normalized after the noise is applied. + """ + if noise is None: + noise = randn_tensor( + image_embeds.shape, generator=generator, device=image_embeds.device, dtype=image_embeds.dtype + ) + + noise_level = torch.tensor([noise_level] * image_embeds.shape[0], device=image_embeds.device) + + self.image_normalizer.to(image_embeds.device) + image_embeds = self.image_normalizer.scale(image_embeds) + + image_embeds = self.image_noising_scheduler.add_noise(image_embeds, timesteps=noise_level, noise=noise) + + image_embeds = self.image_normalizer.unscale(image_embeds) + + noise_level = get_timestep_embedding( + timesteps=noise_level, embedding_dim=image_embeds.shape[-1], flip_sin_to_cos=True, downscale_freq_shift=0 + ) + + # `get_timestep_embeddings` does not contain any weights and will always return f32 tensors, + # but we might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + noise_level = noise_level.to(image_embeds.dtype) + + image_embeds = torch.cat((image_embeds, noise_level), 1) + + return image_embeds + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + # regular denoising process args + prompt: Optional[Union[str, List[str]]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 20, + guidance_scale: float = 10.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[torch.Generator] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + noise_level: int = 0, + # prior args + prior_num_inference_steps: int = 25, + prior_guidance_scale: float = 4.0, + prior_latents: Optional[torch.FloatTensor] = None, + clip_skip: Optional[int] = None, + ): + """ + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 20): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 10.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + noise_level (`int`, *optional*, defaults to `0`): + The amount of noise to add to the image embeddings. A higher `noise_level` increases the variance in + the final un-noised images. See [`StableUnCLIPPipeline.noise_image_embeddings`] for more details. + prior_num_inference_steps (`int`, *optional*, defaults to 25): + The number of denoising steps in the prior denoising process. More denoising steps usually lead to a + higher quality image at the expense of slower inference. + prior_guidance_scale (`float`, *optional*, defaults to 4.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + prior_latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + embedding generation in the prior denoising process. Can be used to tweak the same generation with + different prompts. If not provided, a latents tensor is generated by sampling using the supplied random + `generator`. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + [`~ pipeline_utils.ImagePipelineOutput`] if `return_dict` is True, otherwise a `tuple`. When returning + a tuple, the first element is a list with the generated images. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt=prompt, + height=height, + width=width, + callback_steps=callback_steps, + noise_level=noise_level, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + batch_size = batch_size * num_images_per_prompt + + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + prior_do_classifier_free_guidance = prior_guidance_scale > 1.0 + + # 3. Encode input prompt + prior_prompt_embeds, prior_text_encoder_hidden_states, prior_text_mask = self._encode_prior_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=prior_do_classifier_free_guidance, + ) + + # 4. Prepare prior timesteps + self.prior_scheduler.set_timesteps(prior_num_inference_steps, device=device) + prior_timesteps_tensor = self.prior_scheduler.timesteps + + # 5. Prepare prior latent variables + embedding_dim = self.prior.config.embedding_dim + prior_latents = self.prepare_latents( + (batch_size, embedding_dim), + prior_prompt_embeds.dtype, + device, + generator, + prior_latents, + self.prior_scheduler, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + prior_extra_step_kwargs = self.prepare_prior_extra_step_kwargs(generator, eta) + + # 7. Prior denoising loop + for i, t in enumerate(self.progress_bar(prior_timesteps_tensor)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([prior_latents] * 2) if prior_do_classifier_free_guidance else prior_latents + latent_model_input = self.prior_scheduler.scale_model_input(latent_model_input, t) + + predicted_image_embedding = self.prior( + latent_model_input, + timestep=t, + proj_embedding=prior_prompt_embeds, + encoder_hidden_states=prior_text_encoder_hidden_states, + attention_mask=prior_text_mask, + ).predicted_image_embedding + + if prior_do_classifier_free_guidance: + predicted_image_embedding_uncond, predicted_image_embedding_text = predicted_image_embedding.chunk(2) + predicted_image_embedding = predicted_image_embedding_uncond + prior_guidance_scale * ( + predicted_image_embedding_text - predicted_image_embedding_uncond + ) + + prior_latents = self.prior_scheduler.step( + predicted_image_embedding, + timestep=t, + sample=prior_latents, + **prior_extra_step_kwargs, + return_dict=False, + )[0] + + if callback is not None and i % callback_steps == 0: + callback(i, t, prior_latents) + + prior_latents = self.prior.post_process_latents(prior_latents) + + image_embeds = prior_latents + + # done prior + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 8. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 9. Prepare image embeddings + image_embeds = self.noise_image_embeddings( + image_embeds=image_embeds, + noise_level=noise_level, + generator=generator, + ) + + if do_classifier_free_guidance: + negative_prompt_embeds = torch.zeros_like(image_embeds) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + image_embeds = torch.cat([negative_prompt_embeds, image_embeds]) + + # 10. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 11. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + latents = self.prepare_latents( + shape=shape, + dtype=prompt_embeds.dtype, + device=device, + generator=generator, + latents=latents, + scheduler=self.scheduler, + ) + + # 12. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 13. Denoising loop + for i, t in enumerate(self.progress_bar(timesteps)): + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + class_labels=image_embeds, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = latents + + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py new file mode 100755 index 0000000..9b85d9e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/pipeline_stable_unclip_img2img.py @@ -0,0 +1,839 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.embeddings import get_timestep_embedding +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput, StableDiffusionMixin +from .stable_unclip_image_normalizer import StableUnCLIPImageNormalizer + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import requests + >>> import torch + >>> from PIL import Image + >>> from io import BytesIO + + >>> from diffusers import StableUnCLIPImg2ImgPipeline + + >>> pipe = StableUnCLIPImg2ImgPipeline.from_pretrained( + ... "fusing/stable-unclip-2-1-l-img2img", torch_dtype=torch.float16 + ... ) # TODO update model path + >>> pipe = pipe.to("cuda") + + >>> url = "https://raw.githubusercontent.com/CompVis/stable-diffusion/main/assets/stable-samples/img2img/sketch-mountains-input.jpg" + + >>> response = requests.get(url) + >>> init_image = Image.open(BytesIO(response.content)).convert("RGB") + >>> init_image = init_image.resize((768, 512)) + + >>> prompt = "A fantasy landscape, trending on artstation" + + >>> images = pipe(prompt, init_image).images + >>> images[0].save("fantasy_landscape.png") + ``` +""" + + +class StableUnCLIPImg2ImgPipeline( + DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin +): + """ + Pipeline for text-guided image-to-image generation using stable unCLIP. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + + Args: + feature_extractor ([`CLIPImageProcessor`]): + Feature extractor for image pre-processing before being encoded. + image_encoder ([`CLIPVisionModelWithProjection`]): + CLIP vision model for encoding images. + image_normalizer ([`StableUnCLIPImageNormalizer`]): + Used to normalize the predicted image embeddings before the noise is applied and un-normalize the image + embeddings after the noise has been applied. + image_noising_scheduler ([`KarrasDiffusionSchedulers`]): + Noise schedule for adding noise to the predicted image embeddings. The amount of noise to add is determined + by the `noise_level`. + tokenizer (`~transformers.CLIPTokenizer`): + A [`~transformers.CLIPTokenizer`)]. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen [`~transformers.CLIPTextModel`] text-encoder. + unet ([`UNet2DConditionModel`]): + A [`UNet2DConditionModel`] to denoise the encoded image latents. + scheduler ([`KarrasDiffusionSchedulers`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + """ + + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae" + _exclude_from_cpu_offload = ["image_normalizer"] + + # image encoding components + feature_extractor: CLIPImageProcessor + image_encoder: CLIPVisionModelWithProjection + + # image noising components + image_normalizer: StableUnCLIPImageNormalizer + image_noising_scheduler: KarrasDiffusionSchedulers + + # regular denoising components + tokenizer: CLIPTokenizer + text_encoder: CLIPTextModel + unet: UNet2DConditionModel + scheduler: KarrasDiffusionSchedulers + + vae: AutoencoderKL + + def __init__( + self, + # image encoding components + feature_extractor: CLIPImageProcessor, + image_encoder: CLIPVisionModelWithProjection, + # image noising components + image_normalizer: StableUnCLIPImageNormalizer, + image_noising_scheduler: KarrasDiffusionSchedulers, + # regular denoising components + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModel, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + # vae + vae: AutoencoderKL, + ): + super().__init__() + + self.register_modules( + feature_extractor=feature_extractor, + image_encoder=image_encoder, + image_normalizer=image_normalizer, + image_noising_scheduler=image_noising_scheduler, + tokenizer=tokenizer, + text_encoder=text_encoder, + unet=unet, + scheduler=scheduler, + vae=vae, + ) + + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + def _encode_image( + self, + image, + device, + batch_size, + num_images_per_prompt, + do_classifier_free_guidance, + noise_level, + generator, + image_embeds, + ): + dtype = next(self.image_encoder.parameters()).dtype + + if isinstance(image, PIL.Image.Image): + # the image embedding should repeated so it matches the total batch size of the prompt + repeat_by = batch_size + else: + # assume the image input is already properly batched and just needs to be repeated so + # it matches the num_images_per_prompt. + # + # NOTE(will) this is probably missing a few number of side cases. I.e. batched/non-batched + # `image_embeds`. If those happen to be common use cases, let's think harder about + # what the expected dimensions of inputs should be and how we handle the encoding. + repeat_by = num_images_per_prompt + + if image_embeds is None: + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(images=image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + image_embeds = self.image_encoder(image).image_embeds + + image_embeds = self.noise_image_embeddings( + image_embeds=image_embeds, + noise_level=noise_level, + generator=generator, + ) + + # duplicate image embeddings for each generation per prompt, using mps friendly method + image_embeds = image_embeds.unsqueeze(1) + bs_embed, seq_len, _ = image_embeds.shape + image_embeds = image_embeds.repeat(1, repeat_by, 1) + image_embeds = image_embeds.view(bs_embed * repeat_by, seq_len, -1) + image_embeds = image_embeds.squeeze(1) + + if do_classifier_free_guidance: + negative_prompt_embeds = torch.zeros_like(image_embeds) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + image_embeds = torch.cat([negative_prompt_embeds, image_embeds]) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + image, + height, + width, + callback_steps, + noise_level, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + image_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Please make sure to define only one of the two." + ) + + if prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + + if prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + "Provide either `negative_prompt` or `negative_prompt_embeds`. Cannot leave both `negative_prompt` and `negative_prompt_embeds` undefined." + ) + + if prompt is not None and negative_prompt is not None: + if type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if noise_level < 0 or noise_level >= self.image_noising_scheduler.config.num_train_timesteps: + raise ValueError( + f"`noise_level` must be between 0 and {self.image_noising_scheduler.config.num_train_timesteps - 1}, inclusive." + ) + + if image is not None and image_embeds is not None: + raise ValueError( + "Provide either `image` or `image_embeds`. Please make sure to define only one of the two." + ) + + if image is None and image_embeds is None: + raise ValueError( + "Provide either `image` or `image_embeds`. Cannot leave both `image` and `image_embeds` undefined." + ) + + if image is not None: + if ( + not isinstance(image, torch.Tensor) + and not isinstance(image, PIL.Image.Image) + and not isinstance(image, list) + ): + raise ValueError( + "`image` has to be of type `torch.FloatTensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + f" {type(image)}" + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_unclip.StableUnCLIPPipeline.noise_image_embeddings + def noise_image_embeddings( + self, + image_embeds: torch.Tensor, + noise_level: int, + noise: Optional[torch.FloatTensor] = None, + generator: Optional[torch.Generator] = None, + ): + """ + Add noise to the image embeddings. The amount of noise is controlled by a `noise_level` input. A higher + `noise_level` increases the variance in the final un-noised images. + + The noise is applied in two ways: + 1. A noise schedule is applied directly to the embeddings. + 2. A vector of sinusoidal time embeddings are appended to the output. + + In both cases, the amount of noise is controlled by the same `noise_level`. + + The embeddings are normalized before the noise is applied and un-normalized after the noise is applied. + """ + if noise is None: + noise = randn_tensor( + image_embeds.shape, generator=generator, device=image_embeds.device, dtype=image_embeds.dtype + ) + + noise_level = torch.tensor([noise_level] * image_embeds.shape[0], device=image_embeds.device) + + self.image_normalizer.to(image_embeds.device) + image_embeds = self.image_normalizer.scale(image_embeds) + + image_embeds = self.image_noising_scheduler.add_noise(image_embeds, timesteps=noise_level, noise=noise) + + image_embeds = self.image_normalizer.unscale(image_embeds) + + noise_level = get_timestep_embedding( + timesteps=noise_level, embedding_dim=image_embeds.shape[-1], flip_sin_to_cos=True, downscale_freq_shift=0 + ) + + # `get_timestep_embeddings` does not contain any weights and will always return f32 tensors, + # but we might actually be running in fp16. so we need to cast here. + # there might be better ways to encapsulate this. + noise_level = noise_level.to(image_embeds.dtype) + + image_embeds = torch.cat((image_embeds, noise_level), 1) + + return image_embeds + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image: Union[torch.FloatTensor, PIL.Image.Image] = None, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 20, + guidance_scale: float = 10, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[torch.Generator] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + noise_level: int = 0, + image_embeds: Optional[torch.FloatTensor] = None, + clip_skip: Optional[int] = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, either `prompt_embeds` will be + used or prompt is initialized to `""`. + image (`torch.FloatTensor` or `PIL.Image.Image`): + `Image` or tensor representing an image batch. The image is encoded to its CLIP embedding which the + `unet` is conditioned on. The image is _not_ encoded by the `vae` and then used as the latents in the + denoising process like it is in the standard Stable Diffusion text-guided image variation process. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 20): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 10.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + noise_level (`int`, *optional*, defaults to `0`): + The amount of noise to add to the image embeddings. A higher `noise_level` increases the variance in + the final un-noised images. See [`StableUnCLIPPipeline.noise_image_embeddings`] for more details. + image_embeds (`torch.FloatTensor`, *optional*): + Pre-generated CLIP embeddings to condition the `unet` on. These latents are not used in the denoising + process. If you want to provide pre-generated latents, pass them to `__call__` as `latents`. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + [`~ pipeline_utils.ImagePipelineOutput`] if `return_dict` is True, otherwise a `tuple`. When returning + a tuple, the first element is a list with the generated images. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + if prompt is None and prompt_embeds is None: + prompt = len(image) * [""] if isinstance(image, list) else "" + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt=prompt, + image=image, + height=height, + width=width, + callback_steps=callback_steps, + noise_level=noise_level, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + image_embeds=image_embeds, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + batch_size = batch_size * num_images_per_prompt + + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Encoder input image + noise_level = torch.tensor([noise_level], device=device) + image_embeds = self._encode_image( + image=image, + device=device, + batch_size=batch_size, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + noise_level=noise_level, + generator=generator, + image_embeds=image_embeds, + ) + + # 5. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 6. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size=batch_size, + num_channels_latents=num_channels_latents, + height=height, + width=width, + dtype=prompt_embeds.dtype, + device=device, + generator=generator, + latents=latents, + ) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 8. Denoising loop + for i, t in enumerate(self.progress_bar(timesteps)): + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + class_labels=image_embeds, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 9. Post-processing + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = latents + + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/safety_checker.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/safety_checker.py new file mode 100755 index 0000000..6cc4d26 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/safety_checker.py @@ -0,0 +1,125 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import numpy as np +import torch +import torch.nn as nn +from transformers import CLIPConfig, CLIPVisionModel, PreTrainedModel + +from ...utils import logging + + +logger = logging.get_logger(__name__) + + +def cosine_distance(image_embeds, text_embeds): + normalized_image_embeds = nn.functional.normalize(image_embeds) + normalized_text_embeds = nn.functional.normalize(text_embeds) + return torch.mm(normalized_image_embeds, normalized_text_embeds.t()) + + +class StableDiffusionSafetyChecker(PreTrainedModel): + config_class = CLIPConfig + + _no_split_modules = ["CLIPEncoderLayer"] + + def __init__(self, config: CLIPConfig): + super().__init__(config) + + self.vision_model = CLIPVisionModel(config.vision_config) + self.visual_projection = nn.Linear(config.vision_config.hidden_size, config.projection_dim, bias=False) + + self.concept_embeds = nn.Parameter(torch.ones(17, config.projection_dim), requires_grad=False) + self.special_care_embeds = nn.Parameter(torch.ones(3, config.projection_dim), requires_grad=False) + + self.concept_embeds_weights = nn.Parameter(torch.ones(17), requires_grad=False) + self.special_care_embeds_weights = nn.Parameter(torch.ones(3), requires_grad=False) + + @torch.no_grad() + def forward(self, clip_input, images): + pooled_output = self.vision_model(clip_input)[1] # pooled_output + image_embeds = self.visual_projection(pooled_output) + + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + special_cos_dist = cosine_distance(image_embeds, self.special_care_embeds).cpu().float().numpy() + cos_dist = cosine_distance(image_embeds, self.concept_embeds).cpu().float().numpy() + + result = [] + batch_size = image_embeds.shape[0] + for i in range(batch_size): + result_img = {"special_scores": {}, "special_care": [], "concept_scores": {}, "bad_concepts": []} + + # increase this value to create a stronger `nfsw` filter + # at the cost of increasing the possibility of filtering benign images + adjustment = 0.0 + + for concept_idx in range(len(special_cos_dist[0])): + concept_cos = special_cos_dist[i][concept_idx] + concept_threshold = self.special_care_embeds_weights[concept_idx].item() + result_img["special_scores"][concept_idx] = round(concept_cos - concept_threshold + adjustment, 3) + if result_img["special_scores"][concept_idx] > 0: + result_img["special_care"].append({concept_idx, result_img["special_scores"][concept_idx]}) + adjustment = 0.01 + + for concept_idx in range(len(cos_dist[0])): + concept_cos = cos_dist[i][concept_idx] + concept_threshold = self.concept_embeds_weights[concept_idx].item() + result_img["concept_scores"][concept_idx] = round(concept_cos - concept_threshold + adjustment, 3) + if result_img["concept_scores"][concept_idx] > 0: + result_img["bad_concepts"].append(concept_idx) + + result.append(result_img) + + has_nsfw_concepts = [len(res["bad_concepts"]) > 0 for res in result] + + for idx, has_nsfw_concept in enumerate(has_nsfw_concepts): + if has_nsfw_concept: + if torch.is_tensor(images) or torch.is_tensor(images[0]): + images[idx] = torch.zeros_like(images[idx]) # black image + else: + images[idx] = np.zeros(images[idx].shape) # black image + + if any(has_nsfw_concepts): + logger.warning( + "Potential NSFW content was detected in one or more images. A black image will be returned instead." + " Try again with a different prompt and/or seed." + ) + + return images, has_nsfw_concepts + + @torch.no_grad() + def forward_onnx(self, clip_input: torch.FloatTensor, images: torch.FloatTensor): + pooled_output = self.vision_model(clip_input)[1] # pooled_output + image_embeds = self.visual_projection(pooled_output) + + special_cos_dist = cosine_distance(image_embeds, self.special_care_embeds) + cos_dist = cosine_distance(image_embeds, self.concept_embeds) + + # increase this value to create a stronger `nsfw` filter + # at the cost of increasing the possibility of filtering benign images + adjustment = 0.0 + + special_scores = special_cos_dist - self.special_care_embeds_weights + adjustment + # special_scores = special_scores.round(decimals=3) + special_care = torch.any(special_scores > 0, dim=1) + special_adjustment = special_care * 0.01 + special_adjustment = special_adjustment.unsqueeze(1).expand(-1, cos_dist.shape[1]) + + concept_scores = (cos_dist - self.concept_embeds_weights) + special_adjustment + # concept_scores = concept_scores.round(decimals=3) + has_nsfw_concepts = torch.any(concept_scores > 0, dim=1) + + images[has_nsfw_concepts] = 0.0 # black image + + return images, has_nsfw_concepts diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/safety_checker_flax.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/safety_checker_flax.py new file mode 100755 index 0000000..571a4f2 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/safety_checker_flax.py @@ -0,0 +1,112 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Tuple + +import jax +import jax.numpy as jnp +from flax import linen as nn +from flax.core.frozen_dict import FrozenDict +from transformers import CLIPConfig, FlaxPreTrainedModel +from transformers.models.clip.modeling_flax_clip import FlaxCLIPVisionModule + + +def jax_cosine_distance(emb_1, emb_2, eps=1e-12): + norm_emb_1 = jnp.divide(emb_1.T, jnp.clip(jnp.linalg.norm(emb_1, axis=1), a_min=eps)).T + norm_emb_2 = jnp.divide(emb_2.T, jnp.clip(jnp.linalg.norm(emb_2, axis=1), a_min=eps)).T + return jnp.matmul(norm_emb_1, norm_emb_2.T) + + +class FlaxStableDiffusionSafetyCheckerModule(nn.Module): + config: CLIPConfig + dtype: jnp.dtype = jnp.float32 + + def setup(self): + self.vision_model = FlaxCLIPVisionModule(self.config.vision_config) + self.visual_projection = nn.Dense(self.config.projection_dim, use_bias=False, dtype=self.dtype) + + self.concept_embeds = self.param("concept_embeds", jax.nn.initializers.ones, (17, self.config.projection_dim)) + self.special_care_embeds = self.param( + "special_care_embeds", jax.nn.initializers.ones, (3, self.config.projection_dim) + ) + + self.concept_embeds_weights = self.param("concept_embeds_weights", jax.nn.initializers.ones, (17,)) + self.special_care_embeds_weights = self.param("special_care_embeds_weights", jax.nn.initializers.ones, (3,)) + + def __call__(self, clip_input): + pooled_output = self.vision_model(clip_input)[1] + image_embeds = self.visual_projection(pooled_output) + + special_cos_dist = jax_cosine_distance(image_embeds, self.special_care_embeds) + cos_dist = jax_cosine_distance(image_embeds, self.concept_embeds) + + # increase this value to create a stronger `nfsw` filter + # at the cost of increasing the possibility of filtering benign image inputs + adjustment = 0.0 + + special_scores = special_cos_dist - self.special_care_embeds_weights[None, :] + adjustment + special_scores = jnp.round(special_scores, 3) + is_special_care = jnp.any(special_scores > 0, axis=1, keepdims=True) + # Use a lower threshold if an image has any special care concept + special_adjustment = is_special_care * 0.01 + + concept_scores = cos_dist - self.concept_embeds_weights[None, :] + special_adjustment + concept_scores = jnp.round(concept_scores, 3) + has_nsfw_concepts = jnp.any(concept_scores > 0, axis=1) + + return has_nsfw_concepts + + +class FlaxStableDiffusionSafetyChecker(FlaxPreTrainedModel): + config_class = CLIPConfig + main_input_name = "clip_input" + module_class = FlaxStableDiffusionSafetyCheckerModule + + def __init__( + self, + config: CLIPConfig, + input_shape: Optional[Tuple] = None, + seed: int = 0, + dtype: jnp.dtype = jnp.float32, + _do_init: bool = True, + **kwargs, + ): + if input_shape is None: + input_shape = (1, 224, 224, 3) + module = self.module_class(config=config, dtype=dtype, **kwargs) + super().__init__(config, module, input_shape=input_shape, seed=seed, dtype=dtype, _do_init=_do_init) + + def init_weights(self, rng: jax.Array, input_shape: Tuple, params: FrozenDict = None) -> FrozenDict: + # init input tensor + clip_input = jax.random.normal(rng, input_shape) + + params_rng, dropout_rng = jax.random.split(rng) + rngs = {"params": params_rng, "dropout": dropout_rng} + + random_params = self.module.init(rngs, clip_input)["params"] + + return random_params + + def __call__( + self, + clip_input, + params: dict = None, + ): + clip_input = jnp.transpose(clip_input, (0, 2, 3, 1)) + + return self.module.apply( + {"params": params or self.params}, + jnp.array(clip_input, dtype=jnp.float32), + rngs={}, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py new file mode 100755 index 0000000..3fc6b3a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion/stable_unclip_image_normalizer.py @@ -0,0 +1,57 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Optional, Union + +import torch +from torch import nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...models.modeling_utils import ModelMixin + + +class StableUnCLIPImageNormalizer(ModelMixin, ConfigMixin): + """ + This class is used to hold the mean and standard deviation of the CLIP embedder used in stable unCLIP. + + It is used to normalize the image embeddings before the noise is applied and un-normalize the noised image + embeddings. + """ + + @register_to_config + def __init__( + self, + embedding_dim: int = 768, + ): + super().__init__() + + self.mean = nn.Parameter(torch.zeros(1, embedding_dim)) + self.std = nn.Parameter(torch.ones(1, embedding_dim)) + + def to( + self, + torch_device: Optional[Union[str, torch.device]] = None, + torch_dtype: Optional[torch.dtype] = None, + ): + self.mean = nn.Parameter(self.mean.to(torch_device).to(torch_dtype)) + self.std = nn.Parameter(self.std.to(torch_device).to(torch_dtype)) + return self + + def scale(self, embeds): + embeds = (embeds - self.mean) * 1.0 / self.std + return embeds + + def unscale(self, embeds): + embeds = (embeds * self.std) + self.mean + return embeds diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_attend_and_excite/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_attend_and_excite/__init__.py new file mode 100755 index 0000000..cce556f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_attend_and_excite/__init__.py @@ -0,0 +1,48 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_stable_diffusion_attend_and_excite"] = ["StableDiffusionAttendAndExcitePipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_stable_diffusion_attend_and_excite import StableDiffusionAttendAndExcitePipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py new file mode 100755 index 0000000..03c80b4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_attend_and_excite/pipeline_stable_diffusion_attend_and_excite.py @@ -0,0 +1,1088 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import math +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch +from torch.nn import functional as F +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.attention_processor import Attention +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionAttendAndExcitePipeline + + >>> pipe = StableDiffusionAttendAndExcitePipeline.from_pretrained( + ... "CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16 + ... ).to("cuda") + + + >>> prompt = "a cat and a frog" + + >>> # use get_indices function to find out indices of the tokens you want to alter + >>> pipe.get_indices(prompt) + {0: '<|startoftext|>', 1: 'a', 2: 'cat', 3: 'and', 4: 'a', 5: 'frog', 6: '<|endoftext|>'} + + >>> token_indices = [2, 5] + >>> seed = 6141 + >>> generator = torch.Generator("cuda").manual_seed(seed) + + >>> images = pipe( + ... prompt=prompt, + ... token_indices=token_indices, + ... guidance_scale=7.5, + ... generator=generator, + ... num_inference_steps=50, + ... max_iter_to_alter=25, + ... ).images + + >>> image = images[0] + >>> image.save(f"../images/{prompt}_{seed}.png") + ``` +""" + + +class AttentionStore: + @staticmethod + def get_empty_store(): + return {"down": [], "mid": [], "up": []} + + def __call__(self, attn, is_cross: bool, place_in_unet: str): + if self.cur_att_layer >= 0 and is_cross: + if attn.shape[1] == np.prod(self.attn_res): + self.step_store[place_in_unet].append(attn) + + self.cur_att_layer += 1 + if self.cur_att_layer == self.num_att_layers: + self.cur_att_layer = 0 + self.between_steps() + + def between_steps(self): + self.attention_store = self.step_store + self.step_store = self.get_empty_store() + + def get_average_attention(self): + average_attention = self.attention_store + return average_attention + + def aggregate_attention(self, from_where: List[str]) -> torch.Tensor: + """Aggregates the attention across the different layers and heads at the specified resolution.""" + out = [] + attention_maps = self.get_average_attention() + for location in from_where: + for item in attention_maps[location]: + cross_maps = item.reshape(-1, self.attn_res[0], self.attn_res[1], item.shape[-1]) + out.append(cross_maps) + out = torch.cat(out, dim=0) + out = out.sum(0) / out.shape[0] + return out + + def reset(self): + self.cur_att_layer = 0 + self.step_store = self.get_empty_store() + self.attention_store = {} + + def __init__(self, attn_res): + """ + Initialize an empty AttentionStore :param step_index: used to visualize only a specific step in the diffusion + process + """ + self.num_att_layers = -1 + self.cur_att_layer = 0 + self.step_store = self.get_empty_store() + self.attention_store = {} + self.curr_step_index = 0 + self.attn_res = attn_res + + +class AttendExciteAttnProcessor: + def __init__(self, attnstore, place_in_unet): + super().__init__() + self.attnstore = attnstore + self.place_in_unet = place_in_unet + + def __call__(self, attn: Attention, hidden_states, encoder_hidden_states=None, attention_mask=None): + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + query = attn.to_q(hidden_states) + + is_cross = encoder_hidden_states is not None + encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + + # only need to store attention maps during the Attend and Excite process + if attention_probs.requires_grad: + self.attnstore(attention_probs, is_cross, self.place_in_unet) + + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +class StableDiffusionAttendAndExcitePipeline(DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion and Attend-and-Excite. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + indices, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + indices_is_list_ints = isinstance(indices, list) and isinstance(indices[0], int) + indices_is_list_list_ints = ( + isinstance(indices, list) and isinstance(indices[0], list) and isinstance(indices[0][0], int) + ) + + if not indices_is_list_ints and not indices_is_list_list_ints: + raise TypeError("`indices` must be a list of ints or a list of a list of ints") + + if indices_is_list_ints: + indices_batch_size = 1 + elif indices_is_list_list_ints: + indices_batch_size = len(indices) + + if prompt is not None and isinstance(prompt, str): + prompt_batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + prompt_batch_size = len(prompt) + elif prompt_embeds is not None: + prompt_batch_size = prompt_embeds.shape[0] + + if indices_batch_size != prompt_batch_size: + raise ValueError( + f"indices batch size must be same as prompt batch size. indices batch size: {indices_batch_size}, prompt batch size: {prompt_batch_size}" + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @staticmethod + def _compute_max_attention_per_index( + attention_maps: torch.Tensor, + indices: List[int], + ) -> List[torch.Tensor]: + """Computes the maximum attention value for each of the tokens we wish to alter.""" + attention_for_text = attention_maps[:, :, 1:-1] + attention_for_text *= 100 + attention_for_text = torch.nn.functional.softmax(attention_for_text, dim=-1) + + # Shift indices since we removed the first token + indices = [index - 1 for index in indices] + + # Extract the maximum values + max_indices_list = [] + for i in indices: + image = attention_for_text[:, :, i] + smoothing = GaussianSmoothing().to(attention_maps.device) + input = F.pad(image.unsqueeze(0).unsqueeze(0), (1, 1, 1, 1), mode="reflect") + image = smoothing(input).squeeze(0).squeeze(0) + max_indices_list.append(image.max()) + return max_indices_list + + def _aggregate_and_get_max_attention_per_token( + self, + indices: List[int], + ): + """Aggregates the attention for each token and computes the max activation value for each token to alter.""" + attention_maps = self.attention_store.aggregate_attention( + from_where=("up", "down", "mid"), + ) + max_attention_per_index = self._compute_max_attention_per_index( + attention_maps=attention_maps, + indices=indices, + ) + return max_attention_per_index + + @staticmethod + def _compute_loss(max_attention_per_index: List[torch.Tensor]) -> torch.Tensor: + """Computes the attend-and-excite loss using the maximum attention value for each token.""" + losses = [max(0, 1.0 - curr_max) for curr_max in max_attention_per_index] + loss = max(losses) + return loss + + @staticmethod + def _update_latent(latents: torch.Tensor, loss: torch.Tensor, step_size: float) -> torch.Tensor: + """Update the latent according to the computed loss.""" + grad_cond = torch.autograd.grad(loss.requires_grad_(True), [latents], retain_graph=True)[0] + latents = latents - step_size * grad_cond + return latents + + def _perform_iterative_refinement_step( + self, + latents: torch.Tensor, + indices: List[int], + loss: torch.Tensor, + threshold: float, + text_embeddings: torch.Tensor, + step_size: float, + t: int, + max_refinement_steps: int = 20, + ): + """ + Performs the iterative latent refinement introduced in the paper. Here, we continuously update the latent code + according to our loss objective until the given threshold is reached for all tokens. + """ + iteration = 0 + target_loss = max(0, 1.0 - threshold) + while loss > target_loss: + iteration += 1 + + latents = latents.clone().detach().requires_grad_(True) + self.unet(latents, t, encoder_hidden_states=text_embeddings).sample + self.unet.zero_grad() + + # Get max activation value for each subject token + max_attention_per_index = self._aggregate_and_get_max_attention_per_token( + indices=indices, + ) + + loss = self._compute_loss(max_attention_per_index) + + if loss != 0: + latents = self._update_latent(latents, loss, step_size) + + logger.info(f"\t Try {iteration}. loss: {loss}") + + if iteration >= max_refinement_steps: + logger.info(f"\t Exceeded max number of iterations ({max_refinement_steps})! ") + break + + # Run one more time but don't compute gradients and update the latents. + # We just need to compute the new loss - the grad update will occur below + latents = latents.clone().detach().requires_grad_(True) + _ = self.unet(latents, t, encoder_hidden_states=text_embeddings).sample + self.unet.zero_grad() + + # Get max activation value for each subject token + max_attention_per_index = self._aggregate_and_get_max_attention_per_token( + indices=indices, + ) + loss = self._compute_loss(max_attention_per_index) + logger.info(f"\t Finished with loss of: {loss}") + return loss, latents, max_attention_per_index + + def register_attention_control(self): + attn_procs = {} + cross_att_count = 0 + for name in self.unet.attn_processors.keys(): + if name.startswith("mid_block"): + place_in_unet = "mid" + elif name.startswith("up_blocks"): + place_in_unet = "up" + elif name.startswith("down_blocks"): + place_in_unet = "down" + else: + continue + + cross_att_count += 1 + attn_procs[name] = AttendExciteAttnProcessor(attnstore=self.attention_store, place_in_unet=place_in_unet) + + self.unet.set_attn_processor(attn_procs) + self.attention_store.num_att_layers = cross_att_count + + def get_indices(self, prompt: str) -> Dict[str, int]: + """Utility function to list the indices of the tokens you wish to alte""" + ids = self.tokenizer(prompt).input_ids + indices = {i: tok for tok, i in zip(self.tokenizer.convert_ids_to_tokens(ids), range(len(ids)))} + return indices + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]], + token_indices: Union[List[int], List[List[int]]], + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + max_iter_to_alter: int = 25, + thresholds: dict = {0: 0.05, 10: 0.5, 20: 0.8}, + scale_factor: int = 20, + attn_res: Optional[Tuple[int]] = (16, 16), + clip_skip: Optional[int] = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + token_indices (`List[int]`): + The token indices to alter with attend-and-excite. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + max_iter_to_alter (`int`, *optional*, defaults to `25`): + Number of denoising steps to apply attend-and-excite. The `max_iter_to_alter` denoising steps are when + attend-and-excite is applied. For example, if `max_iter_to_alter` is `25` and there are a total of `30` + denoising steps, the first `25` denoising steps applies attend-and-excite and the last `5` will not. + thresholds (`dict`, *optional*, defaults to `{0: 0.05, 10: 0.5, 20: 0.8}`): + Dictionary defining the iterations and desired thresholds to apply iterative latent refinement in. + scale_factor (`int`, *optional*, default to 20): + Scale factor to control the step size of each attend-and-excite update. + attn_res (`tuple`, *optional*, default computed from width and height): + The 2D resolution of the semantic attention map. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + token_indices, + height, + width, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + if attn_res is None: + attn_res = int(np.ceil(width / 32)), int(np.ceil(height / 32)) + self.attention_store = AttentionStore(attn_res) + self.register_attention_control() + + # default config for step size from original repo + scale_range = np.linspace(1.0, 0.5, len(self.scheduler.timesteps)) + step_size = scale_factor * np.sqrt(scale_range) + + text_embeddings = ( + prompt_embeds[batch_size * num_images_per_prompt :] if do_classifier_free_guidance else prompt_embeds + ) + + if isinstance(token_indices[0], int): + token_indices = [token_indices] + + indices = [] + + for ind in token_indices: + indices = indices + [ind] * num_images_per_prompt + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # Attend and excite process + with torch.enable_grad(): + latents = latents.clone().detach().requires_grad_(True) + updated_latents = [] + for latent, index, text_embedding in zip(latents, indices, text_embeddings): + # Forward pass of denoising with text conditioning + latent = latent.unsqueeze(0) + text_embedding = text_embedding.unsqueeze(0) + + self.unet( + latent, + t, + encoder_hidden_states=text_embedding, + cross_attention_kwargs=cross_attention_kwargs, + ).sample + self.unet.zero_grad() + + # Get max activation value for each subject token + max_attention_per_index = self._aggregate_and_get_max_attention_per_token( + indices=index, + ) + + loss = self._compute_loss(max_attention_per_index=max_attention_per_index) + + # If this is an iterative refinement step, verify we have reached the desired threshold for all + if i in thresholds.keys() and loss > 1.0 - thresholds[i]: + loss, latent, max_attention_per_index = self._perform_iterative_refinement_step( + latents=latent, + indices=index, + loss=loss, + threshold=thresholds[i], + text_embeddings=text_embedding, + step_size=step_size[i], + t=t, + ) + + # Perform gradient update + if i < max_iter_to_alter: + if loss != 0: + latent = self._update_latent( + latents=latent, + loss=loss, + step_size=step_size[i], + ) + logger.info(f"Iteration {i} | Loss: {loss:0.4f}") + + updated_latents.append(latent) + + latents = torch.cat(updated_latents, dim=0) + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 8. Post-processing + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) + + +class GaussianSmoothing(torch.nn.Module): + """ + Arguments: + Apply gaussian smoothing on a 1d, 2d or 3d tensor. Filtering is performed seperately for each channel in the input + using a depthwise convolution. + channels (int, sequence): Number of channels of the input tensors. Output will + have this number of channels as well. + kernel_size (int, sequence): Size of the gaussian kernel. sigma (float, sequence): Standard deviation of the + gaussian kernel. dim (int, optional): The number of dimensions of the data. + Default value is 2 (spatial). + """ + + # channels=1, kernel_size=kernel_size, sigma=sigma, dim=2 + def __init__( + self, + channels: int = 1, + kernel_size: int = 3, + sigma: float = 0.5, + dim: int = 2, + ): + super().__init__() + + if isinstance(kernel_size, int): + kernel_size = [kernel_size] * dim + if isinstance(sigma, float): + sigma = [sigma] * dim + + # The gaussian kernel is the product of the + # gaussian function of each dimension. + kernel = 1 + meshgrids = torch.meshgrid([torch.arange(size, dtype=torch.float32) for size in kernel_size]) + for size, std, mgrid in zip(kernel_size, sigma, meshgrids): + mean = (size - 1) / 2 + kernel *= 1 / (std * math.sqrt(2 * math.pi)) * torch.exp(-(((mgrid - mean) / (2 * std)) ** 2)) + + # Make sure sum of values in gaussian kernel equals 1. + kernel = kernel / torch.sum(kernel) + + # Reshape to depthwise convolutional weight + kernel = kernel.view(1, 1, *kernel.size()) + kernel = kernel.repeat(channels, *[1] * (kernel.dim() - 1)) + + self.register_buffer("weight", kernel) + self.groups = channels + + if dim == 1: + self.conv = F.conv1d + elif dim == 2: + self.conv = F.conv2d + elif dim == 3: + self.conv = F.conv3d + else: + raise RuntimeError("Only 1, 2 and 3 dimensions are supported. Received {}.".format(dim)) + + def forward(self, input): + """ + Arguments: + Apply gaussian filter to input. + input (torch.Tensor): Input to apply gaussian filter on. + Returns: + filtered (torch.Tensor): Filtered output. + """ + return self.conv(input, weight=self.weight.to(input.dtype), groups=self.groups) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_diffedit/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_diffedit/__init__.py new file mode 100755 index 0000000..e2145ed --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_diffedit/__init__.py @@ -0,0 +1,48 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_stable_diffusion_diffedit"] = ["StableDiffusionDiffEditPipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_stable_diffusion_diffedit import StableDiffusionDiffEditPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py new file mode 100755 index 0000000..4c90ce0 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_diffedit/pipeline_stable_diffusion_diffedit.py @@ -0,0 +1,1530 @@ +# Copyright 2024 DiffEdit Authors and Pix2Pix Zero Authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from packaging import version +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ...configuration_utils import FrozenDict +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import DDIMInverseScheduler, KarrasDiffusionSchedulers +from ...utils import ( + PIL_INTERPOLATION, + USE_PEFT_BACKEND, + BaseOutput, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class DiffEditInversionPipelineOutput(BaseOutput): + """ + Output class for Stable Diffusion pipelines. + + Args: + latents (`torch.FloatTensor`) + inverted latents tensor + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `num_timesteps * batch_size` or numpy array of shape `(num_timesteps, + batch_size, height, width, num_channels)`. PIL images or numpy array present the denoised images of the + diffusion pipeline. + """ + + latents: torch.FloatTensor + images: Union[List[PIL.Image.Image], np.ndarray] + + +EXAMPLE_DOC_STRING = """ + + ```py + >>> import PIL + >>> import requests + >>> import torch + >>> from io import BytesIO + + >>> from diffusers import StableDiffusionDiffEditPipeline + + + >>> def download_image(url): + ... response = requests.get(url) + ... return PIL.Image.open(BytesIO(response.content)).convert("RGB") + + + >>> img_url = "https://github.com/Xiang-cd/DiffEdit-stable-diffusion/raw/main/assets/origin.png" + + >>> init_image = download_image(img_url).resize((768, 768)) + + >>> pipe = StableDiffusionDiffEditPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-2-1", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> pipeline.scheduler = DDIMScheduler.from_config(pipeline.scheduler.config) + >>> pipeline.inverse_scheduler = DDIMInverseScheduler.from_config(pipeline.scheduler.config) + >>> pipeline.enable_model_cpu_offload() + + >>> mask_prompt = "A bowl of fruits" + >>> prompt = "A bowl of pears" + + >>> mask_image = pipe.generate_mask(image=init_image, source_prompt=prompt, target_prompt=mask_prompt) + >>> image_latents = pipe.invert(image=init_image, prompt=mask_prompt).latents + >>> image = pipe(prompt=prompt, mask_image=mask_image, image_latents=image_latents).images[0] + ``` +""" + +EXAMPLE_INVERT_DOC_STRING = """ + ```py + >>> import PIL + >>> import requests + >>> import torch + >>> from io import BytesIO + + >>> from diffusers import StableDiffusionDiffEditPipeline + + + >>> def download_image(url): + ... response = requests.get(url) + ... return PIL.Image.open(BytesIO(response.content)).convert("RGB") + + + >>> img_url = "https://github.com/Xiang-cd/DiffEdit-stable-diffusion/raw/main/assets/origin.png" + + >>> init_image = download_image(img_url).resize((768, 768)) + + >>> pipe = StableDiffusionDiffEditPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-2-1", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> pipeline.scheduler = DDIMScheduler.from_config(pipeline.scheduler.config) + >>> pipeline.inverse_scheduler = DDIMInverseScheduler.from_config(pipeline.scheduler.config) + >>> pipeline.enable_model_cpu_offload() + + >>> prompt = "A bowl of fruits" + + >>> inverted_latents = pipe.invert(image=init_image, prompt=prompt).latents + ``` +""" + + +def auto_corr_loss(hidden_states, generator=None): + reg_loss = 0.0 + for i in range(hidden_states.shape[0]): + for j in range(hidden_states.shape[1]): + noise = hidden_states[i : i + 1, j : j + 1, :, :] + while True: + roll_amount = torch.randint(noise.shape[2] // 2, (1,), generator=generator).item() + reg_loss += (noise * torch.roll(noise, shifts=roll_amount, dims=2)).mean() ** 2 + reg_loss += (noise * torch.roll(noise, shifts=roll_amount, dims=3)).mean() ** 2 + + if noise.shape[2] <= 8: + break + noise = torch.nn.functional.avg_pool2d(noise, kernel_size=2) + return reg_loss + + +def kl_divergence(hidden_states): + return hidden_states.var() + hidden_states.mean() ** 2 - 1 - torch.log(hidden_states.var() + 1e-7) + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.preprocess +def preprocess(image): + deprecation_message = "The preprocess method is deprecated and will be removed in diffusers 1.0.0. Please use VaeImageProcessor.preprocess(...) instead" + deprecate("preprocess", "1.0.0", deprecation_message, standard_warn=False) + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + w, h = image[0].size + w, h = (x - x % 8 for x in (w, h)) # resize to integer multiple of 8 + + image = [np.array(i.resize((w, h), resample=PIL_INTERPOLATION["lanczos"]))[None, :] for i in image] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = 2.0 * image - 1.0 + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + image = torch.cat(image, dim=0) + return image + + +def preprocess_mask(mask, batch_size: int = 1): + if not isinstance(mask, torch.Tensor): + # preprocess mask + if isinstance(mask, PIL.Image.Image) or isinstance(mask, np.ndarray): + mask = [mask] + + if isinstance(mask, list): + if isinstance(mask[0], PIL.Image.Image): + mask = [np.array(m.convert("L")).astype(np.float32) / 255.0 for m in mask] + if isinstance(mask[0], np.ndarray): + mask = np.stack(mask, axis=0) if mask[0].ndim < 3 else np.concatenate(mask, axis=0) + mask = torch.from_numpy(mask) + elif isinstance(mask[0], torch.Tensor): + mask = torch.stack(mask, dim=0) if mask[0].ndim < 3 else torch.cat(mask, dim=0) + + # Batch and add channel dim for single mask + if mask.ndim == 2: + mask = mask.unsqueeze(0).unsqueeze(0) + + # Batch single mask or add channel dim + if mask.ndim == 3: + # Single batched mask, no channel dim or single mask not batched but channel dim + if mask.shape[0] == 1: + mask = mask.unsqueeze(0) + + # Batched masks no channel dim + else: + mask = mask.unsqueeze(1) + + # Check mask shape + if batch_size > 1: + if mask.shape[0] == 1: + mask = torch.cat([mask] * batch_size) + elif mask.shape[0] > 1 and mask.shape[0] != batch_size: + raise ValueError( + f"`mask_image` with batch size {mask.shape[0]} cannot be broadcasted to batch size {batch_size} " + f"inferred by prompt inputs" + ) + + if mask.shape[1] != 1: + raise ValueError(f"`mask_image` must have 1 channel, but has {mask.shape[1]} channels") + + # Check mask is in [0, 1] + if mask.min() < 0 or mask.max() > 1: + raise ValueError("`mask_image` should be in [0, 1] range") + + # Binarize mask + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + + return mask + + +class StableDiffusionDiffEditPipeline( + DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin +): + r""" + + + This is an experimental feature! + + + + Pipeline for text-guided image inpainting using Stable Diffusion and DiffEdit. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading and saving methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. + inverse_scheduler ([`DDIMInverseScheduler`]): + A scheduler to be used in combination with `unet` to fill in the unmasked part of the input latents. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "inverse_scheduler"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + inverse_scheduler: DDIMInverseScheduler, + requires_safety_checker: bool = True, + ): + super().__init__() + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "skip_prk_steps") and scheduler.config.skip_prk_steps is False: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration" + " `skip_prk_steps`. `skip_prk_steps` should be set to True in the configuration file. Please make" + " sure to update the config accordingly as not setting `skip_prk_steps` in the config might lead to" + " incorrect results in future versions. If you have downloaded this checkpoint from the Hugging Face" + " Hub, it would be very nice if you could open a Pull request for the" + " `scheduler/scheduler_config.json` file" + ) + deprecate("skip_prk_steps not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["skip_prk_steps"] = True + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely .If you're checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + inverse_scheduler=inverse_scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def check_inputs( + self, + prompt, + strength, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if (strength is None) or (strength is not None and (strength < 0 or strength > 1)): + raise ValueError( + f"The value of `strength` should in [0.0, 1.0] but is, but is {strength} of type {type(strength)}." + ) + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def check_source_inputs( + self, + source_prompt=None, + source_negative_prompt=None, + source_prompt_embeds=None, + source_negative_prompt_embeds=None, + ): + if source_prompt is not None and source_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `source_prompt`: {source_prompt} and `source_prompt_embeds`: {source_prompt_embeds}." + " Please make sure to only forward one of the two." + ) + elif source_prompt is None and source_prompt_embeds is None: + raise ValueError( + "Provide either `source_image` or `source_prompt_embeds`. Cannot leave all both of the arguments undefined." + ) + elif source_prompt is not None and ( + not isinstance(source_prompt, str) and not isinstance(source_prompt, list) + ): + raise ValueError(f"`source_prompt` has to be of type `str` or `list` but is {type(source_prompt)}") + + if source_negative_prompt is not None and source_negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `source_negative_prompt`: {source_negative_prompt} and `source_negative_prompt_embeds`:" + f" {source_negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if source_prompt_embeds is not None and source_negative_prompt_embeds is not None: + if source_prompt_embeds.shape != source_negative_prompt_embeds.shape: + raise ValueError( + "`source_prompt_embeds` and `source_negative_prompt_embeds` must have the same shape when passed" + f" directly, but got: `source_prompt_embeds` {source_prompt_embeds.shape} !=" + f" `source_negative_prompt_embeds` {source_negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + def get_inverse_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + + # safety for t_start overflow to prevent empty timsteps slice + if t_start == 0: + return self.inverse_scheduler.timesteps, num_inference_steps + timesteps = self.inverse_scheduler.timesteps[:-t_start] + + return timesteps, num_inference_steps - t_start + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_image_latents(self, image, batch_size, dtype, device, generator=None): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + if image.shape[1] == 4: + latents = image + + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if isinstance(generator, list): + latents = [ + self.vae.encode(image[i : i + 1]).latent_dist.sample(generator[i]) for i in range(batch_size) + ] + latents = torch.cat(latents, dim=0) + else: + latents = self.vae.encode(image).latent_dist.sample(generator) + + latents = self.vae.config.scaling_factor * latents + + if batch_size != latents.shape[0]: + if batch_size % latents.shape[0] == 0: + # expand image_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_latents_per_image = batch_size // latents.shape[0] + latents = torch.cat([latents] * additional_latents_per_image, dim=0) + else: + raise ValueError( + f"Cannot duplicate `image` of batch size {latents.shape[0]} to {batch_size} text prompts." + ) + else: + latents = torch.cat([latents], dim=0) + + return latents + + def get_epsilon(self, model_output: torch.Tensor, sample: torch.Tensor, timestep: int): + pred_type = self.inverse_scheduler.config.prediction_type + alpha_prod_t = self.inverse_scheduler.alphas_cumprod[timestep] + + beta_prod_t = 1 - alpha_prod_t + + if pred_type == "epsilon": + return model_output + elif pred_type == "sample": + return (sample - alpha_prod_t ** (0.5) * model_output) / beta_prod_t ** (0.5) + elif pred_type == "v_prediction": + return (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + else: + raise ValueError( + f"prediction_type given as {pred_type} must be one of `epsilon`, `sample`, or `v_prediction`" + ) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def generate_mask( + self, + image: Union[torch.FloatTensor, PIL.Image.Image] = None, + target_prompt: Optional[Union[str, List[str]]] = None, + target_negative_prompt: Optional[Union[str, List[str]]] = None, + target_prompt_embeds: Optional[torch.FloatTensor] = None, + target_negative_prompt_embeds: Optional[torch.FloatTensor] = None, + source_prompt: Optional[Union[str, List[str]]] = None, + source_negative_prompt: Optional[Union[str, List[str]]] = None, + source_prompt_embeds: Optional[torch.FloatTensor] = None, + source_negative_prompt_embeds: Optional[torch.FloatTensor] = None, + num_maps_per_mask: Optional[int] = 10, + mask_encode_strength: Optional[float] = 0.5, + mask_thresholding_ratio: Optional[float] = 3.0, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + output_type: Optional[str] = "np", + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + ): + r""" + Generate a latent mask given a mask prompt, a target prompt, and an image. + + Args: + image (`PIL.Image.Image`): + `Image` or tensor representing an image batch to be used for computing the mask. + target_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide semantic mask generation. If not defined, you need to pass + `prompt_embeds`. + target_negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + target_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + target_negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + source_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide semantic mask generation using DiffEdit. If not defined, you need to + pass `source_prompt_embeds` or `source_image` instead. + source_negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide semantic mask generation away from using DiffEdit. If not defined, you + need to pass `source_negative_prompt_embeds` or `source_image` instead. + source_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings to guide the semantic mask generation. Can be used to easily tweak text + inputs (prompt weighting). If not provided, text embeddings are generated from `source_prompt` input + argument. + source_negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings to negatively guide the semantic mask generation. Can be used to easily + tweak text inputs (prompt weighting). If not provided, text embeddings are generated from + `source_negative_prompt` input argument. + num_maps_per_mask (`int`, *optional*, defaults to 10): + The number of noise maps sampled to generate the semantic mask using DiffEdit. + mask_encode_strength (`float`, *optional*, defaults to 0.5): + The strength of the noise maps sampled to generate the semantic mask using DiffEdit. Must be between 0 + and 1. + mask_thresholding_ratio (`float`, *optional*, defaults to 3.0): + The maximum multiple of the mean absolute difference used to clamp the semantic guidance map before + mask binarization. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the + [`~models.attention_processor.AttnProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + + Examples: + + Returns: + `List[PIL.Image.Image]` or `np.array`: + When returning a `List[PIL.Image.Image]`, the list consists of a batch of single-channel binary images + with dimensions `(height // self.vae_scale_factor, width // self.vae_scale_factor)`. If it's + `np.array`, the shape is `(batch_size, height // self.vae_scale_factor, width // + self.vae_scale_factor)`. + """ + + # 1. Check inputs (Provide dummy argument for callback_steps) + self.check_inputs( + target_prompt, + mask_encode_strength, + 1, + target_negative_prompt, + target_prompt_embeds, + target_negative_prompt_embeds, + ) + + self.check_source_inputs( + source_prompt, + source_negative_prompt, + source_prompt_embeds, + source_negative_prompt_embeds, + ) + + if (num_maps_per_mask is None) or ( + num_maps_per_mask is not None and (not isinstance(num_maps_per_mask, int) or num_maps_per_mask <= 0) + ): + raise ValueError( + f"`num_maps_per_mask` has to be a positive integer but is {num_maps_per_mask} of type" + f" {type(num_maps_per_mask)}." + ) + + if mask_thresholding_ratio is None or mask_thresholding_ratio <= 0: + raise ValueError( + f"`mask_thresholding_ratio` has to be positive but is {mask_thresholding_ratio} of type" + f" {type(mask_thresholding_ratio)}." + ) + + # 2. Define call parameters + if target_prompt is not None and isinstance(target_prompt, str): + batch_size = 1 + elif target_prompt is not None and isinstance(target_prompt, list): + batch_size = len(target_prompt) + else: + batch_size = target_prompt_embeds.shape[0] + if cross_attention_kwargs is None: + cross_attention_kwargs = {} + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompts + (cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None) + target_negative_prompt_embeds, target_prompt_embeds = self.encode_prompt( + target_prompt, + device, + num_maps_per_mask, + do_classifier_free_guidance, + target_negative_prompt, + prompt_embeds=target_prompt_embeds, + negative_prompt_embeds=target_negative_prompt_embeds, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + target_prompt_embeds = torch.cat([target_negative_prompt_embeds, target_prompt_embeds]) + + source_negative_prompt_embeds, source_prompt_embeds = self.encode_prompt( + source_prompt, + device, + num_maps_per_mask, + do_classifier_free_guidance, + source_negative_prompt, + prompt_embeds=source_prompt_embeds, + negative_prompt_embeds=source_negative_prompt_embeds, + ) + if do_classifier_free_guidance: + source_prompt_embeds = torch.cat([source_negative_prompt_embeds, source_prompt_embeds]) + + # 4. Preprocess image + image = self.image_processor.preprocess(image).repeat_interleave(num_maps_per_mask, dim=0) + + # 5. Set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, _ = self.get_timesteps(num_inference_steps, mask_encode_strength, device) + encode_timestep = timesteps[0] + + # 6. Prepare image latents and add noise with specified strength + image_latents = self.prepare_image_latents( + image, batch_size * num_maps_per_mask, self.vae.dtype, device, generator + ) + noise = randn_tensor(image_latents.shape, generator=generator, device=device, dtype=self.vae.dtype) + image_latents = self.scheduler.add_noise(image_latents, noise, encode_timestep) + + latent_model_input = torch.cat([image_latents] * (4 if do_classifier_free_guidance else 2)) + latent_model_input = self.scheduler.scale_model_input(latent_model_input, encode_timestep) + + # 7. Predict the noise residual + prompt_embeds = torch.cat([source_prompt_embeds, target_prompt_embeds]) + noise_pred = self.unet( + latent_model_input, + encode_timestep, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + ).sample + + if do_classifier_free_guidance: + noise_pred_neg_src, noise_pred_source, noise_pred_uncond, noise_pred_target = noise_pred.chunk(4) + noise_pred_source = noise_pred_neg_src + guidance_scale * (noise_pred_source - noise_pred_neg_src) + noise_pred_target = noise_pred_uncond + guidance_scale * (noise_pred_target - noise_pred_uncond) + else: + noise_pred_source, noise_pred_target = noise_pred.chunk(2) + + # 8. Compute the mask from the absolute difference of predicted noise residuals + # TODO: Consider smoothing mask guidance map + mask_guidance_map = ( + torch.abs(noise_pred_target - noise_pred_source) + .reshape(batch_size, num_maps_per_mask, *noise_pred_target.shape[-3:]) + .mean([1, 2]) + ) + clamp_magnitude = mask_guidance_map.mean() * mask_thresholding_ratio + semantic_mask_image = mask_guidance_map.clamp(0, clamp_magnitude) / clamp_magnitude + semantic_mask_image = torch.where(semantic_mask_image <= 0.5, 0, 1) + mask_image = semantic_mask_image.cpu().numpy() + + # 9. Convert to Numpy array or PIL. + if output_type == "pil": + mask_image = self.image_processor.numpy_to_pil(mask_image) + + # Offload all models + self.maybe_free_model_hooks() + + return mask_image + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_INVERT_DOC_STRING) + def invert( + self, + prompt: Optional[Union[str, List[str]]] = None, + image: Union[torch.FloatTensor, PIL.Image.Image] = None, + num_inference_steps: int = 50, + inpaint_strength: float = 0.8, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + decode_latents: bool = False, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: Optional[int] = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + lambda_auto_corr: float = 20.0, + lambda_kl: float = 20.0, + num_reg_steps: int = 0, + num_auto_corr_rolls: int = 5, + ): + r""" + Generate inverted latents given a prompt and image. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + image (`PIL.Image.Image`): + `Image` or tensor representing an image batch to produce the inverted latents guided by `prompt`. + inpaint_strength (`float`, *optional*, defaults to 0.8): + Indicates extent of the noising process to run latent inversion. Must be between 0 and 1. When + `inpaint_strength` is 1, the inversion process is run for the full number of iterations specified in + `num_inference_steps`. `image` is used as a reference for the inversion process, and adding more noise + increases `inpaint_strength`. If `inpaint_strength` is 0, no inpainting occurs. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + decode_latents (`bool`, *optional*, defaults to `False`): + Whether or not to decode the inverted latents into a generated image. Setting this argument to `True` + decodes all inverted latents for each timestep into a list of generated images. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.DiffEditInversionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the + [`~models.attention_processor.AttnProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + lambda_auto_corr (`float`, *optional*, defaults to 20.0): + Lambda parameter to control auto correction. + lambda_kl (`float`, *optional*, defaults to 20.0): + Lambda parameter to control Kullback-Leibler divergence output. + num_reg_steps (`int`, *optional*, defaults to 0): + Number of regularization loss steps. + num_auto_corr_rolls (`int`, *optional*, defaults to 5): + Number of auto correction roll steps. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.pipeline_stable_diffusion_diffedit.DiffEditInversionPipelineOutput`] or + `tuple`: + If `return_dict` is `True`, + [`~pipelines.stable_diffusion.pipeline_stable_diffusion_diffedit.DiffEditInversionPipelineOutput`] is + returned, otherwise a `tuple` is returned where the first element is the inverted latents tensors + ordered by increasing noise, and the second is the corresponding decoded images if `decode_latents` is + `True`, otherwise `None`. + """ + + # 1. Check inputs + self.check_inputs( + prompt, + inpaint_strength, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + if image is None: + raise ValueError("`image` input cannot be undefined.") + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + if cross_attention_kwargs is None: + cross_attention_kwargs = {} + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Preprocess image + image = self.image_processor.preprocess(image) + + # 4. Prepare latent variables + num_images_per_prompt = 1 + latents = self.prepare_image_latents( + image, batch_size * num_images_per_prompt, self.vae.dtype, device, generator + ) + + # 5. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 6. Prepare timesteps + self.inverse_scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_inverse_timesteps(num_inference_steps, inpaint_strength, device) + + # 7. Noising loop where we obtain the intermediate noised latent image for each timestep. + num_warmup_steps = len(timesteps) - num_inference_steps * self.inverse_scheduler.order + inverted_latents = [] + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.inverse_scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # regularization of the noise prediction (not in original code or paper but borrowed from Pix2PixZero) + if num_reg_steps > 0: + with torch.enable_grad(): + for _ in range(num_reg_steps): + if lambda_auto_corr > 0: + for _ in range(num_auto_corr_rolls): + var = torch.autograd.Variable(noise_pred.detach().clone(), requires_grad=True) + + # Derive epsilon from model output before regularizing to IID standard normal + var_epsilon = self.get_epsilon(var, latent_model_input.detach(), t) + + l_ac = auto_corr_loss(var_epsilon, generator=generator) + l_ac.backward() + + grad = var.grad.detach() / num_auto_corr_rolls + noise_pred = noise_pred - lambda_auto_corr * grad + + if lambda_kl > 0: + var = torch.autograd.Variable(noise_pred.detach().clone(), requires_grad=True) + + # Derive epsilon from model output before regularizing to IID standard normal + var_epsilon = self.get_epsilon(var, latent_model_input.detach(), t) + + l_kld = kl_divergence(var_epsilon) + l_kld.backward() + + grad = var.grad.detach() + noise_pred = noise_pred - lambda_kl * grad + + noise_pred = noise_pred.detach() + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.inverse_scheduler.step(noise_pred, t, latents).prev_sample + inverted_latents.append(latents.detach().clone()) + + # call the callback, if provided + if i == len(timesteps) - 1 or ( + (i + 1) > num_warmup_steps and (i + 1) % self.inverse_scheduler.order == 0 + ): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + assert len(inverted_latents) == len(timesteps) + latents = torch.stack(list(reversed(inverted_latents)), 1) + + # 8. Post-processing + image = None + if decode_latents: + image = self.decode_latents(latents.flatten(0, 1)) + + # 9. Convert to PIL. + if decode_latents and output_type == "pil": + image = self.image_processor.numpy_to_pil(image) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (latents, image) + + return DiffEditInversionPipelineOutput(latents=latents, images=image) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + mask_image: Union[torch.FloatTensor, PIL.Image.Image] = None, + image_latents: Union[torch.FloatTensor, PIL.Image.Image] = None, + inpaint_strength: Optional[float] = 0.8, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_ckip: int = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + mask_image (`PIL.Image.Image`): + `Image` or tensor representing an image batch to mask the generated image. White pixels in the mask are + repainted, while black pixels are preserved. If `mask_image` is a PIL image, it is converted to a + single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) + instead of 3, so the expected shape would be `(B, 1, H, W)`. + image_latents (`PIL.Image.Image` or `torch.FloatTensor`): + Partially noised image latents from the inversion process to be used as inputs for image generation. + inpaint_strength (`float`, *optional*, defaults to 0.8): + Indicates extent to inpaint the masked area. Must be between 0 and 1. When `inpaint_strength` is 1, the + denoising process is run on the masked area for the full number of iterations specified in + `num_inference_steps`. `image_latents` is used as a reference for the masked area, and adding more + noise to a region increases `inpaint_strength`. If `inpaint_strength` is 0, no inpainting occurs. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + + # 1. Check inputs + self.check_inputs( + prompt, + inpaint_strength, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + if mask_image is None: + raise ValueError( + "`mask_image` input cannot be undefined. Use `generate_mask()` to compute `mask_image` from text prompts." + ) + if image_latents is None: + raise ValueError( + "`image_latents` input cannot be undefined. Use `invert()` to compute `image_latents` from input images." + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + if cross_attention_kwargs is None: + cross_attention_kwargs = {} + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=clip_ckip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Preprocess mask + mask_image = preprocess_mask(mask_image, batch_size) + latent_height, latent_width = mask_image.shape[-2:] + mask_image = torch.cat([mask_image] * num_images_per_prompt) + mask_image = mask_image.to(device=device, dtype=prompt_embeds.dtype) + + # 5. Set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, inpaint_strength, device) + + # 6. Preprocess image latents + if isinstance(image_latents, list) and any(isinstance(l, torch.Tensor) and l.ndim == 5 for l in image_latents): + image_latents = torch.cat(image_latents).detach() + elif isinstance(image_latents, torch.Tensor) and image_latents.ndim == 5: + image_latents = image_latents.detach() + else: + image_latents = self.image_processor.preprocess(image_latents).detach() + + latent_shape = (self.vae.config.latent_channels, latent_height, latent_width) + if image_latents.shape[-3:] != latent_shape: + raise ValueError( + f"Each latent image in `image_latents` must have shape {latent_shape}, " + f"but has shape {image_latents.shape[-3:]}" + ) + if image_latents.ndim == 4: + image_latents = image_latents.reshape(batch_size, len(timesteps), *latent_shape) + if image_latents.shape[:2] != (batch_size, len(timesteps)): + raise ValueError( + f"`image_latents` must have batch size {batch_size} with latent images from {len(timesteps)}" + f" timesteps, but has batch size {image_latents.shape[0]} with latent images from" + f" {image_latents.shape[1]} timesteps." + ) + image_latents = image_latents.transpose(0, 1).repeat_interleave(num_images_per_prompt, dim=1) + image_latents = image_latents.to(device=device, dtype=prompt_embeds.dtype) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 8. Denoising loop + latents = image_latents[0].clone() + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # mask with inverted latents from appropriate timestep - use original image latent for last step + latents = latents * mask_image + image_latents[i] * (1 - mask_image) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/__init__.py new file mode 100755 index 0000000..147980c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/__init__.py @@ -0,0 +1,50 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_stable_diffusion_gligen"] = ["StableDiffusionGLIGENPipeline"] + _import_structure["pipeline_stable_diffusion_gligen_text_image"] = ["StableDiffusionGLIGENTextImagePipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_stable_diffusion_gligen import StableDiffusionGLIGENPipeline + from .pipeline_stable_diffusion_gligen_text_image import StableDiffusionGLIGENTextImagePipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py new file mode 100755 index 0000000..9f0d119 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen.py @@ -0,0 +1,845 @@ +# Copyright 2024 The GLIGEN Authors and HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import warnings +from typing import Any, Callable, Dict, List, Optional, Union + +import PIL.Image +import torch +from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer + +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.attention import GatedSelfAttentionDense +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionGLIGENPipeline + >>> from diffusers.utils import load_image + + >>> # Insert objects described by text at the region defined by bounding boxes + >>> pipe = StableDiffusionGLIGENPipeline.from_pretrained( + ... "masterful/gligen-1-4-inpainting-text-box", variant="fp16", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> input_image = load_image( + ... "https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/gligen/livingroom_modern.png" + ... ) + >>> prompt = "a birthday cake" + >>> boxes = [[0.2676, 0.6088, 0.4773, 0.7183]] + >>> phrases = ["a birthday cake"] + + >>> images = pipe( + ... prompt=prompt, + ... gligen_phrases=phrases, + ... gligen_inpaint_image=input_image, + ... gligen_boxes=boxes, + ... gligen_scheduled_sampling_beta=1, + ... output_type="pil", + ... num_inference_steps=50, + ... ).images + + >>> images[0].save("./gligen-1-4-inpainting-text-box.jpg") + + >>> # Generate an image described by the prompt and + >>> # insert objects described by text at the region defined by bounding boxes + >>> pipe = StableDiffusionGLIGENPipeline.from_pretrained( + ... "masterful/gligen-1-4-generation-text-box", variant="fp16", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a waterfall and a modern high speed train running through the tunnel in a beautiful forest with fall foliage" + >>> boxes = [[0.1387, 0.2051, 0.4277, 0.7090], [0.4980, 0.4355, 0.8516, 0.7266]] + >>> phrases = ["a waterfall", "a modern high speed train running through the tunnel"] + + >>> images = pipe( + ... prompt=prompt, + ... gligen_phrases=phrases, + ... gligen_boxes=boxes, + ... gligen_scheduled_sampling_beta=1, + ... output_type="pil", + ... num_inference_steps=50, + ... ).images + + >>> images[0].save("./gligen-1-4-generation-text-box.jpg") + ``` +""" + + +class StableDiffusionGLIGENPipeline(DiffusionPipeline, StableDiffusionMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion with Grounded-Language-to-Image Generation (GLIGEN). + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + _optional_components = ["safety_checker", "feature_extractor"] + model_cpu_offload_seq = "text_encoder->unet->vae" + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPFeatureExtractor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + gligen_phrases, + gligen_boxes, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if len(gligen_phrases) != len(gligen_boxes): + ValueError( + "length of `gligen_phrases` and `gligen_boxes` has to be same, but" + f" got: `gligen_phrases` {len(gligen_phrases)} != `gligen_boxes` {len(gligen_boxes)}" + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def enable_fuser(self, enabled=True): + for module in self.unet.modules(): + if type(module) is GatedSelfAttentionDense: + module.enabled = enabled + + def draw_inpaint_mask_from_boxes(self, boxes, size): + inpaint_mask = torch.ones(size[0], size[1]) + for box in boxes: + x0, x1 = box[0] * size[0], box[2] * size[0] + y0, y1 = box[1] * size[1], box[3] * size[1] + inpaint_mask[int(y0) : int(y1), int(x0) : int(x1)] = 0 + return inpaint_mask + + def crop(self, im, new_width, new_height): + width, height = im.size + left = (width - new_width) / 2 + top = (height - new_height) / 2 + right = (width + new_width) / 2 + bottom = (height + new_height) / 2 + return im.crop((left, top, right, bottom)) + + def target_size_center_crop(self, im, new_hw): + width, height = im.size + if width != height: + im = self.crop(im, min(height, width), min(height, width)) + return im.resize((new_hw, new_hw), PIL.Image.LANCZOS) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + gligen_scheduled_sampling_beta: float = 0.3, + gligen_phrases: List[str] = None, + gligen_boxes: List[List[float]] = None, + gligen_inpaint_image: Optional[PIL.Image.Image] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + gligen_phrases (`List[str]`): + The phrases to guide what to include in each of the regions defined by the corresponding + `gligen_boxes`. There should only be one phrase per bounding box. + gligen_boxes (`List[List[float]]`): + The bounding boxes that identify rectangular regions of the image that are going to be filled with the + content described by the corresponding `gligen_phrases`. Each rectangular box is defined as a + `List[float]` of 4 elements `[xmin, ymin, xmax, ymax]` where each value is between [0,1]. + gligen_inpaint_image (`PIL.Image.Image`, *optional*): + The input image, if provided, is inpainted with objects described by the `gligen_boxes` and + `gligen_phrases`. Otherwise, it is treated as a generation task on a blank input image. + gligen_scheduled_sampling_beta (`float`, defaults to 0.3): + Scheduled Sampling factor from [GLIGEN: Open-Set Grounded Text-to-Image + Generation](https://arxiv.org/pdf/2301.07093.pdf). Scheduled Sampling factor is only varied for + scheduled sampling during inference for improved quality and controllability. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.0): + Guidance rescale factor from [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf). Guidance rescale factor should fix overexposure when + using zero terminal SNR. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + callback_steps, + gligen_phrases, + gligen_boxes, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 5.1 Prepare GLIGEN variables + max_objs = 30 + if len(gligen_boxes) > max_objs: + warnings.warn( + f"More that {max_objs} objects found. Only first {max_objs} objects will be processed.", + FutureWarning, + ) + gligen_phrases = gligen_phrases[:max_objs] + gligen_boxes = gligen_boxes[:max_objs] + # prepare batched input to the GLIGENTextBoundingboxProjection (boxes, phrases, mask) + # Get tokens for phrases from pre-trained CLIPTokenizer + tokenizer_inputs = self.tokenizer(gligen_phrases, padding=True, return_tensors="pt").to(device) + # For the token, we use the same pre-trained text encoder + # to obtain its text feature + _text_embeddings = self.text_encoder(**tokenizer_inputs).pooler_output + n_objs = len(gligen_boxes) + # For each entity, described in phrases, is denoted with a bounding box, + # we represent the location information as (xmin,ymin,xmax,ymax) + boxes = torch.zeros(max_objs, 4, device=device, dtype=self.text_encoder.dtype) + boxes[:n_objs] = torch.tensor(gligen_boxes) + text_embeddings = torch.zeros( + max_objs, self.unet.cross_attention_dim, device=device, dtype=self.text_encoder.dtype + ) + text_embeddings[:n_objs] = _text_embeddings + # Generate a mask for each object that is entity described by phrases + masks = torch.zeros(max_objs, device=device, dtype=self.text_encoder.dtype) + masks[:n_objs] = 1 + + repeat_batch = batch_size * num_images_per_prompt + boxes = boxes.unsqueeze(0).expand(repeat_batch, -1, -1).clone() + text_embeddings = text_embeddings.unsqueeze(0).expand(repeat_batch, -1, -1).clone() + masks = masks.unsqueeze(0).expand(repeat_batch, -1).clone() + if do_classifier_free_guidance: + repeat_batch = repeat_batch * 2 + boxes = torch.cat([boxes] * 2) + text_embeddings = torch.cat([text_embeddings] * 2) + masks = torch.cat([masks] * 2) + masks[: repeat_batch // 2] = 0 + if cross_attention_kwargs is None: + cross_attention_kwargs = {} + cross_attention_kwargs["gligen"] = {"boxes": boxes, "positive_embeddings": text_embeddings, "masks": masks} + + # Prepare latent variables for GLIGEN inpainting + if gligen_inpaint_image is not None: + # if the given input image is not of the same size as expected by VAE + # center crop and resize the input image to expected shape + if gligen_inpaint_image.size != (self.vae.sample_size, self.vae.sample_size): + gligen_inpaint_image = self.target_size_center_crop(gligen_inpaint_image, self.vae.sample_size) + # Convert a single image into a batch of images with a batch size of 1 + # The resulting shape becomes (1, C, H, W), where C is the number of channels, + # and H and W are the height and width of the image. + # scales the pixel values to a range [-1, 1] + gligen_inpaint_image = self.image_processor.preprocess(gligen_inpaint_image) + gligen_inpaint_image = gligen_inpaint_image.to(dtype=self.vae.dtype, device=self.vae.device) + # Run AutoEncoder to get corresponding latents + gligen_inpaint_latent = self.vae.encode(gligen_inpaint_image).latent_dist.sample() + gligen_inpaint_latent = self.vae.config.scaling_factor * gligen_inpaint_latent + # Generate an inpainting mask + # pixel value = 0, where the object is present (defined by bounding boxes above) + # 1, everywhere else + gligen_inpaint_mask = self.draw_inpaint_mask_from_boxes(gligen_boxes, gligen_inpaint_latent.shape[2:]) + gligen_inpaint_mask = gligen_inpaint_mask.to( + dtype=gligen_inpaint_latent.dtype, device=gligen_inpaint_latent.device + ) + gligen_inpaint_mask = gligen_inpaint_mask[None, None] + gligen_inpaint_mask_addition = torch.cat( + (gligen_inpaint_latent * gligen_inpaint_mask, gligen_inpaint_mask), dim=1 + ) + # Convert a single mask into a batch of masks with a batch size of 1 + gligen_inpaint_mask_addition = gligen_inpaint_mask_addition.expand(repeat_batch, -1, -1, -1).clone() + + num_grounding_steps = int(gligen_scheduled_sampling_beta * len(timesteps)) + self.enable_fuser(True) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # Scheduled sampling + if i == num_grounding_steps: + self.enable_fuser(False) + + if latents.shape[1] != 4: + latents = torch.randn_like(latents[:, :4]) + + if gligen_inpaint_image is not None: + gligen_inpaint_latent_with_noise = ( + self.scheduler.add_noise( + gligen_inpaint_latent, torch.randn_like(gligen_inpaint_latent), torch.tensor([t]) + ) + .expand(latents.shape[0], -1, -1, -1) + .clone() + ) + latents = gligen_inpaint_latent_with_noise * gligen_inpaint_mask + latents * ( + 1 - gligen_inpaint_mask + ) + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + if gligen_inpaint_image is not None: + latent_model_input = torch.cat((latent_model_input, gligen_inpaint_mask_addition), dim=1) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py new file mode 100755 index 0000000..296ecae --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_gligen/pipeline_stable_diffusion_gligen_text_image.py @@ -0,0 +1,1017 @@ +# Copyright 2024 The GLIGEN Authors and HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import warnings +from typing import Any, Callable, Dict, List, Optional, Union + +import PIL.Image +import torch +from transformers import ( + CLIPFeatureExtractor, + CLIPProcessor, + CLIPTextModel, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.attention import GatedSelfAttentionDense +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import USE_PEFT_BACKEND, logging, replace_example_docstring, scale_lora_layers, unscale_lora_layers +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput +from ..stable_diffusion.clip_image_project_model import CLIPImageProjection +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionGLIGENTextImagePipeline + >>> from diffusers.utils import load_image + + >>> # Insert objects described by image at the region defined by bounding boxes + >>> pipe = StableDiffusionGLIGENTextImagePipeline.from_pretrained( + ... "anhnct/Gligen_Inpainting_Text_Image", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> input_image = load_image( + ... "https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/gligen/livingroom_modern.png" + ... ) + >>> prompt = "a backpack" + >>> boxes = [[0.2676, 0.4088, 0.4773, 0.7183]] + >>> phrases = None + >>> gligen_image = load_image( + ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/gligen/backpack.jpeg" + ... ) + + >>> images = pipe( + ... prompt=prompt, + ... gligen_phrases=phrases, + ... gligen_inpaint_image=input_image, + ... gligen_boxes=boxes, + ... gligen_images=[gligen_image], + ... gligen_scheduled_sampling_beta=1, + ... output_type="pil", + ... num_inference_steps=50, + ... ).images + + >>> images[0].save("./gligen-inpainting-text-image-box.jpg") + + >>> # Generate an image described by the prompt and + >>> # insert objects described by text and image at the region defined by bounding boxes + >>> pipe = StableDiffusionGLIGENTextImagePipeline.from_pretrained( + ... "anhnct/Gligen_Text_Image", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a flower sitting on the beach" + >>> boxes = [[0.0, 0.09, 0.53, 0.76]] + >>> phrases = ["flower"] + >>> gligen_image = load_image( + ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/gligen/pexels-pixabay-60597.jpg" + ... ) + + >>> images = pipe( + ... prompt=prompt, + ... gligen_phrases=phrases, + ... gligen_images=[gligen_image], + ... gligen_boxes=boxes, + ... gligen_scheduled_sampling_beta=1, + ... output_type="pil", + ... num_inference_steps=50, + ... ).images + + >>> images[0].save("./gligen-generation-text-image-box.jpg") + + >>> # Generate an image described by the prompt and + >>> # transfer style described by image at the region defined by bounding boxes + >>> pipe = StableDiffusionGLIGENTextImagePipeline.from_pretrained( + ... "anhnct/Gligen_Text_Image", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a dragon flying on the sky" + >>> boxes = [[0.4, 0.2, 1.0, 0.8], [0.0, 1.0, 0.0, 1.0]] # Set `[0.0, 1.0, 0.0, 1.0]` for the style + + >>> gligen_image = load_image( + ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/landscape.png" + ... ) + + >>> gligen_placeholder = load_image( + ... "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/landscape.png" + ... ) + + >>> images = pipe( + ... prompt=prompt, + ... gligen_phrases=[ + ... "dragon", + ... "placeholder", + ... ], # Can use any text instead of `placeholder` token, because we will use mask here + ... gligen_images=[ + ... gligen_placeholder, + ... gligen_image, + ... ], # Can use any image in gligen_placeholder, because we will use mask here + ... input_phrases_mask=[1, 0], # Set 0 for the placeholder token + ... input_images_mask=[0, 1], # Set 0 for the placeholder image + ... gligen_boxes=boxes, + ... gligen_scheduled_sampling_beta=1, + ... output_type="pil", + ... num_inference_steps=50, + ... ).images + + >>> images[0].save("./gligen-generation-text-image-box-style-transfer.jpg") + ``` +""" + + +class StableDiffusionGLIGENTextImagePipeline(DiffusionPipeline, StableDiffusionMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion with Grounded-Language-to-Image Generation (GLIGEN). + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + processor ([`~transformers.CLIPProcessor`]): + A `CLIPProcessor` to procces reference image. + image_encoder ([`~transformers.CLIPVisionModelWithProjection`]): + Frozen image-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + image_project ([`CLIPImageProjection`]): + A `CLIPImageProjection` to project image embedding into phrases embedding space. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + processor: CLIPProcessor, + image_encoder: CLIPVisionModelWithProjection, + image_project: CLIPImageProjection, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPFeatureExtractor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + image_encoder=image_encoder, + processor=processor, + image_project=image_project, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor, do_convert_rgb=True) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion_k_diffusion.pipeline_stable_diffusion_k_diffusion.StableDiffusionKDiffusionPipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def enable_fuser(self, enabled=True): + for module in self.unet.modules(): + if type(module) is GatedSelfAttentionDense: + module.enabled = enabled + + def draw_inpaint_mask_from_boxes(self, boxes, size): + """ + Create an inpainting mask based on given boxes. This function generates an inpainting mask using the provided + boxes to mark regions that need to be inpainted. + """ + inpaint_mask = torch.ones(size[0], size[1]) + for box in boxes: + x0, x1 = box[0] * size[0], box[2] * size[0] + y0, y1 = box[1] * size[1], box[3] * size[1] + inpaint_mask[int(y0) : int(y1), int(x0) : int(x1)] = 0 + return inpaint_mask + + def crop(self, im, new_width, new_height): + """ + Crop the input image to the specified dimensions. + """ + width, height = im.size + left = (width - new_width) / 2 + top = (height - new_height) / 2 + right = (width + new_width) / 2 + bottom = (height + new_height) / 2 + return im.crop((left, top, right, bottom)) + + def target_size_center_crop(self, im, new_hw): + """ + Crop and resize the image to the target size while keeping the center. + """ + width, height = im.size + if width != height: + im = self.crop(im, min(height, width), min(height, width)) + return im.resize((new_hw, new_hw), PIL.Image.LANCZOS) + + def complete_mask(self, has_mask, max_objs, device): + """ + Based on the input mask corresponding value `0 or 1` for each phrases and image, mask the features + corresponding to phrases and images. + """ + mask = torch.ones(1, max_objs).type(self.text_encoder.dtype).to(device) + if has_mask is None: + return mask + + if isinstance(has_mask, int): + return mask * has_mask + else: + for idx, value in enumerate(has_mask): + mask[0, idx] = value + return mask + + def get_clip_feature(self, input, normalize_constant, device, is_image=False): + """ + Get image and phrases embedding by using CLIP pretrain model. The image embedding is transformed into the + phrases embedding space through a projection. + """ + if is_image: + if input is None: + return None + inputs = self.processor(images=[input], return_tensors="pt").to(device) + inputs["pixel_values"] = inputs["pixel_values"].to(self.image_encoder.dtype) + + outputs = self.image_encoder(**inputs) + feature = outputs.image_embeds + feature = self.image_project(feature).squeeze(0) + feature = (feature / feature.norm()) * normalize_constant + feature = feature.unsqueeze(0) + else: + if input is None: + return None + inputs = self.tokenizer(input, return_tensors="pt", padding=True).to(device) + outputs = self.text_encoder(**inputs) + feature = outputs.pooler_output + return feature + + def get_cross_attention_kwargs_with_grounded( + self, + hidden_size, + gligen_phrases, + gligen_images, + gligen_boxes, + input_phrases_mask, + input_images_mask, + repeat_batch, + normalize_constant, + max_objs, + device, + ): + """ + Prepare the cross-attention kwargs containing information about the grounded input (boxes, mask, image + embedding, phrases embedding). + """ + phrases, images = gligen_phrases, gligen_images + images = [None] * len(phrases) if images is None else images + phrases = [None] * len(images) if phrases is None else phrases + + boxes = torch.zeros(max_objs, 4, device=device, dtype=self.text_encoder.dtype) + masks = torch.zeros(max_objs, device=device, dtype=self.text_encoder.dtype) + phrases_masks = torch.zeros(max_objs, device=device, dtype=self.text_encoder.dtype) + image_masks = torch.zeros(max_objs, device=device, dtype=self.text_encoder.dtype) + phrases_embeddings = torch.zeros(max_objs, hidden_size, device=device, dtype=self.text_encoder.dtype) + image_embeddings = torch.zeros(max_objs, hidden_size, device=device, dtype=self.text_encoder.dtype) + + text_features = [] + image_features = [] + for phrase, image in zip(phrases, images): + text_features.append(self.get_clip_feature(phrase, normalize_constant, device, is_image=False)) + image_features.append(self.get_clip_feature(image, normalize_constant, device, is_image=True)) + + for idx, (box, text_feature, image_feature) in enumerate(zip(gligen_boxes, text_features, image_features)): + boxes[idx] = torch.tensor(box) + masks[idx] = 1 + if text_feature is not None: + phrases_embeddings[idx] = text_feature + phrases_masks[idx] = 1 + if image_feature is not None: + image_embeddings[idx] = image_feature + image_masks[idx] = 1 + + input_phrases_mask = self.complete_mask(input_phrases_mask, max_objs, device) + phrases_masks = phrases_masks.unsqueeze(0).repeat(repeat_batch, 1) * input_phrases_mask + input_images_mask = self.complete_mask(input_images_mask, max_objs, device) + image_masks = image_masks.unsqueeze(0).repeat(repeat_batch, 1) * input_images_mask + boxes = boxes.unsqueeze(0).repeat(repeat_batch, 1, 1) + masks = masks.unsqueeze(0).repeat(repeat_batch, 1) + phrases_embeddings = phrases_embeddings.unsqueeze(0).repeat(repeat_batch, 1, 1) + image_embeddings = image_embeddings.unsqueeze(0).repeat(repeat_batch, 1, 1) + + out = { + "boxes": boxes, + "masks": masks, + "phrases_masks": phrases_masks, + "image_masks": image_masks, + "phrases_embeddings": phrases_embeddings, + "image_embeddings": image_embeddings, + } + + return out + + def get_cross_attention_kwargs_without_grounded(self, hidden_size, repeat_batch, max_objs, device): + """ + Prepare the cross-attention kwargs without information about the grounded input (boxes, mask, image embedding, + phrases embedding) (All are zero tensor). + """ + boxes = torch.zeros(max_objs, 4, device=device, dtype=self.text_encoder.dtype) + masks = torch.zeros(max_objs, device=device, dtype=self.text_encoder.dtype) + phrases_masks = torch.zeros(max_objs, device=device, dtype=self.text_encoder.dtype) + image_masks = torch.zeros(max_objs, device=device, dtype=self.text_encoder.dtype) + phrases_embeddings = torch.zeros(max_objs, hidden_size, device=device, dtype=self.text_encoder.dtype) + image_embeddings = torch.zeros(max_objs, hidden_size, device=device, dtype=self.text_encoder.dtype) + + out = { + "boxes": boxes.unsqueeze(0).repeat(repeat_batch, 1, 1), + "masks": masks.unsqueeze(0).repeat(repeat_batch, 1), + "phrases_masks": phrases_masks.unsqueeze(0).repeat(repeat_batch, 1), + "image_masks": image_masks.unsqueeze(0).repeat(repeat_batch, 1), + "phrases_embeddings": phrases_embeddings.unsqueeze(0).repeat(repeat_batch, 1, 1), + "image_embeddings": image_embeddings.unsqueeze(0).repeat(repeat_batch, 1, 1), + } + + return out + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + gligen_scheduled_sampling_beta: float = 0.3, + gligen_phrases: List[str] = None, + gligen_images: List[PIL.Image.Image] = None, + input_phrases_mask: Union[int, List[int]] = None, + input_images_mask: Union[int, List[int]] = None, + gligen_boxes: List[List[float]] = None, + gligen_inpaint_image: Optional[PIL.Image.Image] = None, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + gligen_normalize_constant: float = 28.7, + clip_skip: int = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + gligen_phrases (`List[str]`): + The phrases to guide what to include in each of the regions defined by the corresponding + `gligen_boxes`. There should only be one phrase per bounding box. + gligen_images (`List[PIL.Image.Image]`): + The images to guide what to include in each of the regions defined by the corresponding `gligen_boxes`. + There should only be one image per bounding box + input_phrases_mask (`int` or `List[int]`): + pre phrases mask input defined by the correspongding `input_phrases_mask` + input_images_mask (`int` or `List[int]`): + pre images mask input defined by the correspongding `input_images_mask` + gligen_boxes (`List[List[float]]`): + The bounding boxes that identify rectangular regions of the image that are going to be filled with the + content described by the corresponding `gligen_phrases`. Each rectangular box is defined as a + `List[float]` of 4 elements `[xmin, ymin, xmax, ymax]` where each value is between [0,1]. + gligen_inpaint_image (`PIL.Image.Image`, *optional*): + The input image, if provided, is inpainted with objects described by the `gligen_boxes` and + `gligen_phrases`. Otherwise, it is treated as a generation task on a blank input image. + gligen_scheduled_sampling_beta (`float`, defaults to 0.3): + Scheduled Sampling factor from [GLIGEN: Open-Set Grounded Text-to-Image + Generation](https://arxiv.org/pdf/2301.07093.pdf). Scheduled Sampling factor is only varied for + scheduled sampling during inference for improved quality and controllability. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + gligen_normalize_constant (`float`, *optional*, defaults to 28.7): + The normalize value of the image embedding. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clip_skip=clip_skip, + ) + + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 5.1 Prepare GLIGEN variables + max_objs = 30 + if len(gligen_boxes) > max_objs: + warnings.warn( + f"More that {max_objs} objects found. Only first {max_objs} objects will be processed.", + FutureWarning, + ) + gligen_phrases = gligen_phrases[:max_objs] + gligen_boxes = gligen_boxes[:max_objs] + gligen_images = gligen_images[:max_objs] + + repeat_batch = batch_size * num_images_per_prompt + + if do_classifier_free_guidance: + repeat_batch = repeat_batch * 2 + + if cross_attention_kwargs is None: + cross_attention_kwargs = {} + + hidden_size = prompt_embeds.shape[2] + + cross_attention_kwargs["gligen"] = self.get_cross_attention_kwargs_with_grounded( + hidden_size=hidden_size, + gligen_phrases=gligen_phrases, + gligen_images=gligen_images, + gligen_boxes=gligen_boxes, + input_phrases_mask=input_phrases_mask, + input_images_mask=input_images_mask, + repeat_batch=repeat_batch, + normalize_constant=gligen_normalize_constant, + max_objs=max_objs, + device=device, + ) + + cross_attention_kwargs_without_grounded = {} + cross_attention_kwargs_without_grounded["gligen"] = self.get_cross_attention_kwargs_without_grounded( + hidden_size=hidden_size, repeat_batch=repeat_batch, max_objs=max_objs, device=device + ) + + # Prepare latent variables for GLIGEN inpainting + if gligen_inpaint_image is not None: + # if the given input image is not of the same size as expected by VAE + # center crop and resize the input image to expected shape + if gligen_inpaint_image.size != (self.vae.sample_size, self.vae.sample_size): + gligen_inpaint_image = self.target_size_center_crop(gligen_inpaint_image, self.vae.sample_size) + # Convert a single image into a batch of images with a batch size of 1 + # The resulting shape becomes (1, C, H, W), where C is the number of channels, + # and H and W are the height and width of the image. + # scales the pixel values to a range [-1, 1] + gligen_inpaint_image = self.image_processor.preprocess(gligen_inpaint_image) + gligen_inpaint_image = gligen_inpaint_image.to(dtype=self.vae.dtype, device=self.vae.device) + # Run AutoEncoder to get corresponding latents + gligen_inpaint_latent = self.vae.encode(gligen_inpaint_image).latent_dist.sample() + gligen_inpaint_latent = self.vae.config.scaling_factor * gligen_inpaint_latent + # Generate an inpainting mask + # pixel value = 0, where the object is present (defined by bounding boxes above) + # 1, everywhere else + gligen_inpaint_mask = self.draw_inpaint_mask_from_boxes(gligen_boxes, gligen_inpaint_latent.shape[2:]) + gligen_inpaint_mask = gligen_inpaint_mask.to( + dtype=gligen_inpaint_latent.dtype, device=gligen_inpaint_latent.device + ) + gligen_inpaint_mask = gligen_inpaint_mask[None, None] + gligen_inpaint_mask_addition = torch.cat( + (gligen_inpaint_latent * gligen_inpaint_mask, gligen_inpaint_mask), dim=1 + ) + # Convert a single mask into a batch of masks with a batch size of 1 + gligen_inpaint_mask_addition = gligen_inpaint_mask_addition.expand(repeat_batch, -1, -1, -1).clone() + + int(gligen_scheduled_sampling_beta * len(timesteps)) + self.enable_fuser(True) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if latents.shape[1] != 4: + latents = torch.randn_like(latents[:, :4]) + + if gligen_inpaint_image is not None: + gligen_inpaint_latent_with_noise = ( + self.scheduler.add_noise( + gligen_inpaint_latent, torch.randn_like(gligen_inpaint_latent), torch.tensor([t]) + ) + .expand(latents.shape[0], -1, -1, -1) + .clone() + ) + latents = gligen_inpaint_latent_with_noise * gligen_inpaint_mask + latents * ( + 1 - gligen_inpaint_mask + ) + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + if gligen_inpaint_image is not None: + latent_model_input = torch.cat((latent_model_input, gligen_inpaint_mask_addition), dim=1) + + # predict the noise residual with grounded information + noise_pred_with_grounding = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + ).sample + + # predict the noise residual without grounded information + noise_pred_without_grounding = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs_without_grounded, + ).sample + + # perform guidance + if do_classifier_free_guidance: + # Using noise_pred_text from noise residual with grounded information and noise_pred_uncond from noise residual without grounded information + _, noise_pred_text = noise_pred_with_grounding.chunk(2) + noise_pred_uncond, _ = noise_pred_without_grounding.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + else: + noise_pred = noise_pred_with_grounding + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/__init__.py new file mode 100755 index 0000000..7eb5bf8 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/__init__.py @@ -0,0 +1,62 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_k_diffusion_available, + is_k_diffusion_version, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + + +try: + if not ( + is_transformers_available() + and is_torch_available() + and is_k_diffusion_available() + and is_k_diffusion_version(">=", "0.0.12") + ): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_and_k_diffusion_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_and_k_diffusion_objects)) +else: + _import_structure["pipeline_stable_diffusion_k_diffusion"] = ["StableDiffusionKDiffusionPipeline"] + _import_structure["pipeline_stable_diffusion_xl_k_diffusion"] = ["StableDiffusionXLKDiffusionPipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not ( + is_transformers_available() + and is_torch_available() + and is_k_diffusion_available() + and is_k_diffusion_version(">=", "0.0.12") + ): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_and_k_diffusion_objects import * + else: + from .pipeline_stable_diffusion_k_diffusion import StableDiffusionKDiffusionPipeline + from .pipeline_stable_diffusion_xl_k_diffusion import StableDiffusionXLKDiffusionPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py new file mode 100755 index 0000000..bc565c9 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_k_diffusion.py @@ -0,0 +1,664 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import inspect +from typing import Callable, List, Optional, Union + +import torch +from k_diffusion.external import CompVisDenoiser, CompVisVDenoiser +from k_diffusion.sampling import BrownianTreeNoiseSampler, get_sigmas_karras + +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import LMSDiscreteScheduler +from ...utils import USE_PEFT_BACKEND, deprecate, logging, scale_lora_layers, unscale_lora_layers +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class ModelWrapper: + def __init__(self, model, alphas_cumprod): + self.model = model + self.alphas_cumprod = alphas_cumprod + + def apply_model(self, *args, **kwargs): + if len(args) == 3: + encoder_hidden_states = args[-1] + args = args[:2] + if kwargs.get("cond", None) is not None: + encoder_hidden_states = kwargs.pop("cond") + return self.model(*args, encoder_hidden_states=encoder_hidden_states, **kwargs).sample + + +class StableDiffusionKDiffusionPipeline( + DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin +): + r""" + Pipeline for text-to-image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + + + + This is an experimental pipeline and is likely to change in the future. + + + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details. + feature_extractor ([`CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae, + text_encoder, + tokenizer, + unet, + scheduler, + safety_checker, + feature_extractor, + requires_safety_checker: bool = True, + ): + super().__init__() + + logger.info( + f"{self.__class__} is an experimntal pipeline and is likely to change in the future. We recommend to use" + " this pipeline for fast experimentation / iteration if needed, but advice to rely on existing pipelines" + " as defined in https://huggingface.co/docs/diffusers/api/schedulers#implemented-schedulers for" + " production settings." + ) + + # get correct sigmas from LMS + scheduler = LMSDiscreteScheduler.from_config(scheduler.config) + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.register_to_config(requires_safety_checker=requires_safety_checker) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + model = ModelWrapper(unet, scheduler.alphas_cumprod) + if scheduler.config.prediction_type == "v_prediction": + self.k_diffusion_model = CompVisVDenoiser(model) + else: + self.k_diffusion_model = CompVisDenoiser(model) + + def set_scheduler(self, scheduler_type: str): + library = importlib.import_module("k_diffusion") + sampling = getattr(library, "sampling") + try: + self.sampler = getattr(sampling, scheduler_type) + except Exception: + valid_samplers = [] + for s in dir(sampling): + if "sample_" in s: + valid_samplers.append(s) + + raise ValueError(f"Invalid scheduler type {scheduler_type}. Please choose one of {valid_samplers}.") + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + return latents + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + use_karras_sigmas: Optional[bool] = False, + noise_sampler_seed: Optional[int] = None, + clip_skip: int = None, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` + is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Use karras sigmas. For example, specifying `sample_dpmpp_2m` to `set_scheduler` will be equivalent to + `DPM++2M` in stable-diffusion-webui. On top of that, setting this option to True will make it `DPM++2M + Karras`. + noise_sampler_seed (`int`, *optional*, defaults to `None`): + The random seed to use for the noise sampler. If `None`, a random seed will be generated. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] if `return_dict` is True, otherwise a `tuple. + When returning a tuple, the first element is a list with the generated images, and the second element is a + list of `bool`s denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, according to the `safety_checker`. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = True + if guidance_scale <= 1.0: + raise ValueError("has to use guidance_scale") + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=prompt_embeds.device) + + # 5. Prepare sigmas + if use_karras_sigmas: + sigma_min: float = self.k_diffusion_model.sigmas[0].item() + sigma_max: float = self.k_diffusion_model.sigmas[-1].item() + sigmas = get_sigmas_karras(n=num_inference_steps, sigma_min=sigma_min, sigma_max=sigma_max) + sigmas = sigmas.to(device) + else: + sigmas = self.scheduler.sigmas + sigmas = sigmas.to(prompt_embeds.dtype) + + # 6. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + latents = latents * sigmas[0] + self.k_diffusion_model.sigmas = self.k_diffusion_model.sigmas.to(latents.device) + self.k_diffusion_model.log_sigmas = self.k_diffusion_model.log_sigmas.to(latents.device) + + # 7. Define model function + def model_fn(x, t): + latent_model_input = torch.cat([x] * 2) + t = torch.cat([t] * 2) + + noise_pred = self.k_diffusion_model(latent_model_input, t, cond=prompt_embeds) + + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + return noise_pred + + # 8. Run k-diffusion solver + sampler_kwargs = {} + + if "noise_sampler" in inspect.signature(self.sampler).parameters: + min_sigma, max_sigma = sigmas[sigmas > 0].min(), sigmas.max() + noise_sampler = BrownianTreeNoiseSampler(latents, min_sigma, max_sigma, noise_sampler_seed) + sampler_kwargs["noise_sampler"] = noise_sampler + + if "generator" in inspect.signature(self.sampler).parameters: + sampler_kwargs["generator"] = generator + + latents = self.sampler(model_fn, latents, sigmas, **sampler_kwargs) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py new file mode 100755 index 0000000..ed46a1e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_k_diffusion/pipeline_stable_diffusion_xl_k_diffusion.py @@ -0,0 +1,891 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import inspect +from typing import List, Optional, Tuple, Union + +import torch +from k_diffusion.external import CompVisDenoiser, CompVisVDenoiser +from k_diffusion.sampling import BrownianTreeNoiseSampler, get_sigmas_karras +from transformers import ( + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, +) + +from ...image_processor import VaeImageProcessor +from ...loaders import ( + FromSingleFileMixin, + IPAdapterMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, +) +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.attention_processor import ( + AttnProcessor2_0, + FusedAttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers, LMSDiscreteScheduler +from ...utils import ( + USE_PEFT_BACKEND, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionXLKDiffusionPipeline + + >>> pipe = StableDiffusionXLKDiffusionPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + >>> pipe.set_scheduler("sample_dpmpp_2m_sde") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion_k_diffusion.pipeline_stable_diffusion_k_diffusion.ModelWrapper +class ModelWrapper: + def __init__(self, model, alphas_cumprod): + self.model = model + self.alphas_cumprod = alphas_cumprod + + def apply_model(self, *args, **kwargs): + if len(args) == 3: + encoder_hidden_states = args[-1] + args = args[:2] + if kwargs.get("cond", None) is not None: + encoder_hidden_states = kwargs.pop("cond") + return self.model(*args, encoder_hidden_states=encoder_hidden_states, **kwargs).sample + + +class StableDiffusionXLKDiffusionPipeline( + DiffusionPipeline, + StableDiffusionMixin, + FromSingleFileMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, +): + r""" + Pipeline for text-to-image generation using Stable Diffusion XL and k-diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): + Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of + `stabilityai/stable-diffusion-xl-base-1-0`. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->unet->vae" + _optional_components = [ + "tokenizer", + "tokenizer_2", + "text_encoder", + "text_encoder_2", + "feature_extractor", + ] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + force_zeros_for_empty_prompt: bool = True, + ): + super().__init__() + + # get correct sigmas from LMS + scheduler = LMSDiscreteScheduler.from_config(scheduler.config) + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + scheduler=scheduler, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + self.default_sample_size = self.unet.config.sample_size + + model = ModelWrapper(unet, scheduler.alphas_cumprod) + if scheduler.config.prediction_type == "v_prediction": + self.k_diffusion_model = CompVisVDenoiser(model) + else: + self.k_diffusion_model = CompVisDenoiser(model) + + # Copied from diffusers.pipelines.stable_diffusion_k_diffusion.pipeline_stable_diffusion_k_diffusion.StableDiffusionKDiffusionPipeline.set_scheduler + def set_scheduler(self, scheduler_type: str): + library = importlib.import_module("k_diffusion") + sampling = getattr(library, "sampling") + try: + self.sampler = getattr(sampling, scheduler_type) + except Exception: + valid_samplers = [] + for s in dir(sampling): + if "sample_" in s: + valid_samplers.append(s) + + raise ValueError(f"Invalid scheduler type {scheduler_type}. Please choose one of {valid_samplers}.") + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # textual inversion: process multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_2 = ( + batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 + ) + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + return latents + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._get_add_time_ids + def _get_add_time_ids( + self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None + ): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + return add_time_ids + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + FusedAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + original_size: Optional[Tuple[int, int]] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Optional[Tuple[int, int]] = None, + negative_original_size: Optional[Tuple[int, int]] = None, + negative_crops_coords_top_left: Tuple[int, int] = (0, 0), + negative_target_size: Optional[Tuple[int, int]] = None, + use_karras_sigmas: Optional[bool] = False, + noise_sampler_seed: Optional[int] = None, + clip_skip: Optional[int] = None, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 5.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a specific image resolution. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a target image resolution. It should be as same + as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + + Examples: + + Returns: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + + # 0. Default height and width to unet + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) + + if guidance_scale <= 1.0: + raise ValueError("has to use guidance_scale") + + self._guidance_scale = guidance_scale + self._clip_skip = clip_skip + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + lora_scale = None + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=lora_scale, + clip_skip=self.clip_skip, + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=prompt_embeds.device) + + # 5. Prepare sigmas + if use_karras_sigmas: + sigma_min: float = self.k_diffusion_model.sigmas[0].item() + sigma_max: float = self.k_diffusion_model.sigmas[-1].item() + sigmas = get_sigmas_karras(n=num_inference_steps, sigma_min=sigma_min, sigma_max=sigma_max) + else: + sigmas = self.scheduler.sigmas + sigmas = sigmas.to(dtype=prompt_embeds.dtype, device=device) + + # 6. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + latents = latents * sigmas[0] + + self.k_diffusion_model.sigmas = self.k_diffusion_model.sigmas.to(latents.device) + self.k_diffusion_model.log_sigmas = self.k_diffusion_model.log_sigmas.to(latents.device) + + # 7. Prepare added time ids & embeddings + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + if negative_original_size is not None and negative_target_size is not None: + negative_add_time_ids = self._get_add_time_ids( + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + else: + negative_add_time_ids = add_time_ids + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1) + + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + + # 8. Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + # 9. Define model function + def model_fn(x, t): + latent_model_input = torch.cat([x] * 2) + t = torch.cat([t] * 2) + + noise_pred = self.k_diffusion_model( + latent_model_input, + t, + cond=prompt_embeds, + timestep_cond=timestep_cond, + added_cond_kwargs=added_cond_kwargs, + ) + + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + return noise_pred + + # 10. Run k-diffusion solver + sampler_kwargs = {} + + if "noise_sampler" in inspect.signature(self.sampler).parameters: + min_sigma, max_sigma = sigmas[sigmas > 0].min(), sigmas.max() + noise_sampler = BrownianTreeNoiseSampler(latents, min_sigma, max_sigma, noise_sampler_seed) + sampler_kwargs["noise_sampler"] = noise_sampler + + if "generator" in inspect.signature(self.sampler).parameters: + sampler_kwargs["generator"] = generator + + latents = self.sampler(model_fn, latents, sigmas, **sampler_kwargs) + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + image = latents + + if not output_type == "latent": + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return StableDiffusionXLPipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_ldm3d/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_ldm3d/__init__.py new file mode 100755 index 0000000..dae2aff --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_ldm3d/__init__.py @@ -0,0 +1,48 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_stable_diffusion_ldm3d"] = ["StableDiffusionLDM3DPipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_stable_diffusion_ldm3d import StableDiffusionLDM3DPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py new file mode 100755 index 0000000..c7c05fe --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_ldm3d/pipeline_stable_diffusion_ldm3d.py @@ -0,0 +1,985 @@ +# Copyright 2024 The Intel Labs Team Authors and the HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessorLDM3D +from ...loaders import FromSingleFileMixin, IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + BaseOutput, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```python + >>> from diffusers import StableDiffusionLDM3DPipeline + + >>> pipe = StableDiffusionLDM3DPipeline.from_pretrained("Intel/ldm3d-4c") + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> output = pipe(prompt) + >>> rgb_image, depth_image = output.rgb, output.depth + >>> rgb_image[0].save("astronaut_ldm3d_rgb.jpg") + >>> depth_image[0].save("astronaut_ldm3d_depth.png") + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +@dataclass +class LDM3DPipelineOutput(BaseOutput): + """ + Output class for Stable Diffusion pipelines. + + Args: + rgb (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + num_channels)`. + depth (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + num_channels)`. + nsfw_content_detected (`List[bool]`) + List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or + `None` if safety checking could not be performed. + """ + + rgb: Union[List[PIL.Image.Image], np.ndarray] + depth: Union[List[PIL.Image.Image], np.ndarray] + nsfw_content_detected: Optional[List[bool]] + + +class StableDiffusionLDM3DPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, + LoraLoaderMixin, + FromSingleFileMixin, +): + r""" + Pipeline for text-to-image and 3D generation using LDM3D. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + _callback_tensor_inputs = ["latents", "prompt_embeds", "negative_prompt_embeds"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: Optional[CLIPVisionModelWithProjection], + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessorLDM3D(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + rgb_feature_extractor_input = feature_extractor_input[0] + safety_checker_input = self.feature_extractor(rgb_feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 49, + timesteps: List[int] = None, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 5.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): + Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`", + ) + + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._guidance_rescale = guidance_rescale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 6.1 Add image embeds for IP-Adapter + added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None + + # 6.2 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + rgb, depth = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return ((rgb, depth), has_nsfw_concept) + + return LDM3DPipelineOutput(rgb=rgb, depth=depth, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_panorama/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_panorama/__init__.py new file mode 100755 index 0000000..f7572db --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_panorama/__init__.py @@ -0,0 +1,48 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_stable_diffusion_panorama"] = ["StableDiffusionPanoramaPipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_stable_diffusion_panorama import StableDiffusionPanoramaPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py new file mode 100755 index 0000000..feda710 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_panorama/pipeline_stable_diffusion_panorama.py @@ -0,0 +1,933 @@ +# Copyright 2024 MultiDiffusion Authors and The HuggingFace Team. All rights reserved." +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import DDIMScheduler +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionPanoramaPipeline, DDIMScheduler + + >>> model_ckpt = "stabilityai/stable-diffusion-2-base" + >>> scheduler = DDIMScheduler.from_pretrained(model_ckpt, subfolder="scheduler") + >>> pipe = StableDiffusionPanoramaPipeline.from_pretrained( + ... model_ckpt, scheduler=scheduler, torch_dtype=torch.float16 + ... ) + + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of the dolomites" + >>> image = pipe(prompt).images[0] + ``` +""" + + +class StableDiffusionPanoramaPipeline( + DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin, IPAdapterMixin +): + r""" + Pipeline for text-to-image generation using MultiDiffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: DDIMScheduler, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: Optional[CLIPVisionModelWithProjection] = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + def decode_latents_with_padding(self, latents, padding=8): + # Add padding to latents for circular inference + # padding is the number of latents to add on each side + # it would slightly increase the memory usage, but remove the boundary artifacts + latents = 1 / self.vae.config.scaling_factor * latents + latents_left = latents[..., :padding] + latents_right = latents[..., -padding:] + latents = torch.cat((latents_right, latents, latents_left), axis=-1) + image = self.vae.decode(latents, return_dict=False)[0] + padding_pix = self.vae_scale_factor * padding + image = image[..., padding_pix:-padding_pix] + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def get_views(self, panorama_height, panorama_width, window_size=64, stride=8, circular_padding=False): + # Here, we define the mappings F_i (see Eq. 7 in the MultiDiffusion paper https://arxiv.org/abs/2302.08113) + # if panorama's height/width < window_size, num_blocks of height/width should return 1 + panorama_height /= 8 + panorama_width /= 8 + num_blocks_height = (panorama_height - window_size) // stride + 1 if panorama_height > window_size else 1 + if circular_padding: + num_blocks_width = panorama_width // stride if panorama_width > window_size else 1 + else: + num_blocks_width = (panorama_width - window_size) // stride + 1 if panorama_width > window_size else 1 + total_num_blocks = int(num_blocks_height * num_blocks_width) + views = [] + for i in range(total_num_blocks): + h_start = int((i // num_blocks_width) * stride) + h_end = h_start + window_size + w_start = int((i % num_blocks_width) * stride) + w_end = w_start + window_size + views.append((h_start, h_end, w_start, w_end)) + return views + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = 512, + width: Optional[int] = 2048, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + view_batch_size: int = 1, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: Optional[int] = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + circular_padding: bool = False, + clip_skip: Optional[int] = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 2048): + The width in pixels of the generated image. The width is kept high because the pipeline is supposed + generate panorama-like images. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + view_batch_size (`int`, *optional*, defaults to 1): + The batch size to denoise split views. For some GPUs with high performance, higher view batch size can + speedup the generation and increase the VRAM usage. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): + Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + circular_padding (`bool`, *optional*, defaults to `False`): + If set to `True`, circular padding is applied to ensure there are no stitching artifacts. Circular + padding allows the model to seamlessly generate a transition from the rightmost part of the image to + the leftmost part, maintaining consistency in a 360-degree sense. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + height, + width, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Define panorama grid and initialize views for synthesis. + # prepare batch grid + views = self.get_views(height, width, circular_padding=circular_padding) + views_batch = [views[i : i + view_batch_size] for i in range(0, len(views), view_batch_size)] + views_scheduler_status = [copy.deepcopy(self.scheduler.__dict__)] * len(views_batch) + count = torch.zeros_like(latents) + value = torch.zeros_like(latents) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7.1 Add image embeds for IP-Adapter + added_cond_kwargs = ( + {"image_embeds": image_embeds} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None + else None + ) + + # 8. Denoising loop + # Each denoising step also includes refinement of the latents with respect to the + # views. + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + count.zero_() + value.zero_() + + # generate views + # Here, we iterate through different spatial crops of the latents and denoise them. These + # denoised (latent) crops are then averaged to produce the final latent + # for the current timestep via MultiDiffusion. Please see Sec. 4.1 in the + # MultiDiffusion paper for more details: https://arxiv.org/abs/2302.08113 + # Batch views denoise + for j, batch_view in enumerate(views_batch): + vb_size = len(batch_view) + # get the latents corresponding to the current view coordinates + if circular_padding: + latents_for_view = [] + for h_start, h_end, w_start, w_end in batch_view: + if w_end > latents.shape[3]: + # Add circular horizontal padding + latent_view = torch.cat( + ( + latents[:, :, h_start:h_end, w_start:], + latents[:, :, h_start:h_end, : w_end - latents.shape[3]], + ), + axis=-1, + ) + else: + latent_view = latents[:, :, h_start:h_end, w_start:w_end] + latents_for_view.append(latent_view) + latents_for_view = torch.cat(latents_for_view) + else: + latents_for_view = torch.cat( + [ + latents[:, :, h_start:h_end, w_start:w_end] + for h_start, h_end, w_start, w_end in batch_view + ] + ) + + # rematch block's scheduler status + self.scheduler.__dict__.update(views_scheduler_status[j]) + + # expand the latents if we are doing classifier free guidance + latent_model_input = ( + latents_for_view.repeat_interleave(2, dim=0) + if do_classifier_free_guidance + else latents_for_view + ) + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # repeat prompt_embeds for batch + prompt_embeds_input = torch.cat([prompt_embeds] * vb_size) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds_input, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred[::2], noise_pred[1::2] + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents_denoised_batch = self.scheduler.step( + noise_pred, t, latents_for_view, **extra_step_kwargs + ).prev_sample + + # save views scheduler status after sample + views_scheduler_status[j] = copy.deepcopy(self.scheduler.__dict__) + + # extract value from batch + for latents_view_denoised, (h_start, h_end, w_start, w_end) in zip( + latents_denoised_batch.chunk(vb_size), batch_view + ): + if circular_padding and w_end > latents.shape[3]: + # Case for circular padding + value[:, :, h_start:h_end, w_start:] += latents_view_denoised[ + :, :, h_start:h_end, : latents.shape[3] - w_start + ] + value[:, :, h_start:h_end, : w_end - latents.shape[3]] += latents_view_denoised[ + :, :, h_start:h_end, latents.shape[3] - w_start : + ] + count[:, :, h_start:h_end, w_start:] += 1 + count[:, :, h_start:h_end, : w_end - latents.shape[3]] += 1 + else: + value[:, :, h_start:h_end, w_start:w_end] += latents_view_denoised + count[:, :, h_start:h_end, w_start:w_end] += 1 + + # take the MultiDiffusion step. Eq. 5 in MultiDiffusion paper: https://arxiv.org/abs/2302.08113 + latents = torch.where(count > 0, value / count, value) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + if circular_padding: + image = self.decode_latents_with_padding(latents) + else: + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/__init__.py new file mode 100755 index 0000000..b432b94 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/__init__.py @@ -0,0 +1,99 @@ +from dataclasses import dataclass +from enum import Enum +from typing import TYPE_CHECKING, List, Optional, Union + +import numpy as np +import PIL +from PIL import Image + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + BaseOutput, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +@dataclass +class SafetyConfig(object): + WEAK = { + "sld_warmup_steps": 15, + "sld_guidance_scale": 20, + "sld_threshold": 0.0, + "sld_momentum_scale": 0.0, + "sld_mom_beta": 0.0, + } + MEDIUM = { + "sld_warmup_steps": 10, + "sld_guidance_scale": 1000, + "sld_threshold": 0.01, + "sld_momentum_scale": 0.3, + "sld_mom_beta": 0.4, + } + STRONG = { + "sld_warmup_steps": 7, + "sld_guidance_scale": 2000, + "sld_threshold": 0.025, + "sld_momentum_scale": 0.5, + "sld_mom_beta": 0.7, + } + MAX = { + "sld_warmup_steps": 0, + "sld_guidance_scale": 5000, + "sld_threshold": 1.0, + "sld_momentum_scale": 0.5, + "sld_mom_beta": 0.7, + } + + +_dummy_objects = {} +_additional_imports = {} +_import_structure = {} + +_additional_imports.update({"SafetyConfig": SafetyConfig}) + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure.update( + { + "pipeline_output": ["StableDiffusionSafePipelineOutput"], + "pipeline_stable_diffusion_safe": ["StableDiffusionPipelineSafe"], + "safety_checker": ["StableDiffusionSafetyChecker"], + } + ) + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_output import StableDiffusionSafePipelineOutput + from .pipeline_stable_diffusion_safe import StableDiffusionPipelineSafe + from .safety_checker import SafeStableDiffusionSafetyChecker + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) + for name, value in _additional_imports.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/pipeline_output.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/pipeline_output.py new file mode 100755 index 0000000..69a064d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/pipeline_output.py @@ -0,0 +1,34 @@ +from dataclasses import dataclass +from typing import List, Optional, Union + +import numpy as np +import PIL.Image + +from ...utils import ( + BaseOutput, +) + + +@dataclass +class StableDiffusionSafePipelineOutput(BaseOutput): + """ + Output class for Safe Stable Diffusion pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. + nsfw_content_detected (`List[bool]`) + List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, or `None` if safety checking could not be performed. + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images that were flagged by the safety checker any may contain "not-safe-for-work" + (nsfw) content, or `None` if no safety check was performed or no images were flagged. + applied_safety_concept (`str`) + The safety concept that was applied for safety guidance, or `None` if safety guidance was disabled + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + nsfw_content_detected: Optional[List[bool]] + unsafe_images: Optional[Union[List[PIL.Image.Image], np.ndarray]] + applied_safety_concept: Optional[str] diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py new file mode 100755 index 0000000..ae74e09 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/pipeline_stable_diffusion_safe.py @@ -0,0 +1,764 @@ +import inspect +import warnings +from typing import Callable, List, Optional, Union + +import numpy as np +import torch +from packaging import version +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...configuration_utils import FrozenDict +from ...image_processor import PipelineImageInput +from ...loaders import IPAdapterMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import deprecate, logging +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from . import StableDiffusionSafePipelineOutput +from .safety_checker import SafeStableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class StableDiffusionPipelineSafe(DiffusionPipeline, StableDiffusionMixin, IPAdapterMixin): + r""" + Pipeline based on the [`StableDiffusionPipeline`] for text-to-image generation using Safe Latent Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: SafeStableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: Optional[CLIPVisionModelWithProjection] = None, + requires_safety_checker: bool = True, + ): + super().__init__() + safety_concept: Optional[str] = ( + "an image showing hate, harassment, violence, suffering, humiliation, harm, suicide, sexual, nudity," + " bodily fluids, blood, obscene gestures, illegal activity, drug use, theft, vandalism, weapons, child" + " abuse, brutality, cruelty" + ) + + if hasattr(scheduler.config, "steps_offset") and scheduler.config.steps_offset != 1: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} is outdated. `steps_offset`" + f" should be set to 1 instead of {scheduler.config.steps_offset}. Please make sure " + "to update the config accordingly as leaving `steps_offset` might led to incorrect results" + " in future versions. If you have downloaded this checkpoint from the Hugging Face Hub," + " it would be very nice if you could open a Pull request for the `scheduler/scheduler_config.json`" + " file" + ) + deprecate("steps_offset!=1", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["steps_offset"] = 1 + scheduler._internal_dict = FrozenDict(new_config) + + if hasattr(scheduler.config, "clip_sample") and scheduler.config.clip_sample is True: + deprecation_message = ( + f"The configuration file of this scheduler: {scheduler} has not set the configuration `clip_sample`." + " `clip_sample` should be set to False in the configuration file. Please make sure to update the" + " config accordingly as not setting `clip_sample` in the config might lead to incorrect results in" + " future versions. If you have downloaded this checkpoint from the Hugging Face Hub, it would be very" + " nice if you could open a Pull request for the `scheduler/scheduler_config.json` file" + ) + deprecate("clip_sample not set", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(scheduler.config) + new_config["clip_sample"] = False + scheduler._internal_dict = FrozenDict(new_config) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + is_unet_version_less_0_9_0 = hasattr(unet.config, "_diffusers_version") and version.parse( + version.parse(unet.config._diffusers_version).base_version + ) < version.parse("0.9.0.dev0") + is_unet_sample_size_less_64 = hasattr(unet.config, "sample_size") and unet.config.sample_size < 64 + if is_unet_version_less_0_9_0 and is_unet_sample_size_less_64: + deprecation_message = ( + "The configuration file of the unet has set the default `sample_size` to smaller than" + " 64 which seems highly unlikely .If you're checkpoint is a fine-tuned version of any of the" + " following: \n- CompVis/stable-diffusion-v1-4 \n- CompVis/stable-diffusion-v1-3 \n-" + " CompVis/stable-diffusion-v1-2 \n- CompVis/stable-diffusion-v1-1 \n- runwayml/stable-diffusion-v1-5" + " \n- runwayml/stable-diffusion-inpainting \n you should change 'sample_size' to 64 in the" + " configuration file. Please make sure to update the config accordingly as leaving `sample_size=32`" + " in the config might lead to incorrect results in future versions. If you have downloaded this" + " checkpoint from the Hugging Face Hub, it would be very nice if you could open a Pull request for" + " the `unet/config.json` file" + ) + deprecate("sample_size<64", "1.0.0", deprecation_message, standard_warn=False) + new_config = dict(unet.config) + new_config["sample_size"] = 64 + unet._internal_dict = FrozenDict(new_config) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self._safety_text_concept = safety_concept + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + @property + def safety_concept(self): + r""" + Getter method for the safety concept used with SLD + + Returns: + `str`: The text describing the safety concept + """ + return self._safety_text_concept + + @safety_concept.setter + def safety_concept(self, concept): + r""" + Setter method for the safety concept used with SLD + + Args: + concept (`str`): + The text of the new safety concept + """ + self._safety_text_concept = concept + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + enable_safety_guidance, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + """ + batch_size = len(prompt) if isinstance(prompt, list) else 1 + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="max_length", return_tensors="pt").input_ids + + if not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + prompt_embeds = self.text_encoder( + text_input_ids.to(device), + attention_mask=attention_mask, + ) + prompt_embeds = prompt_embeds[0] + + # duplicate text embeddings for each generation per prompt, using mps friendly method + bs_embed, seq_len, _ = prompt_embeds.shape + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + max_length = text_input_ids.shape[-1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + # Encode the safety concept text + if enable_safety_guidance: + safety_concept_input = self.tokenizer( + [self._safety_text_concept], + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + safety_embeddings = self.text_encoder(safety_concept_input.input_ids.to(self.device))[0] + + # duplicate safety embeddings for each generation per prompt, using mps friendly method + seq_len = safety_embeddings.shape[1] + safety_embeddings = safety_embeddings.repeat(batch_size, num_images_per_prompt, 1) + safety_embeddings = safety_embeddings.view(batch_size * num_images_per_prompt, seq_len, -1) + + # For classifier free guidance + sld, we need to do three forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing three forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds, safety_embeddings]) + + else: + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + return prompt_embeds + + def run_safety_checker(self, image, device, dtype, enable_safety_guidance): + if self.safety_checker is not None: + images = image.copy() + safety_checker_input = self.feature_extractor(self.numpy_to_pil(image), return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + flagged_images = np.zeros((2, *image.shape[1:])) + if any(has_nsfw_concept): + logger.warning( + "Potential NSFW content was detected in one or more images. A black image will be returned" + " instead." + f"{'You may look at this images in the `unsafe_images` variable of the output at your own discretion.' if enable_safety_guidance else 'Try again with a different prompt and/or seed.'}" + ) + for idx, has_nsfw_concept in enumerate(has_nsfw_concept): + if has_nsfw_concept: + flagged_images[idx] = images[idx] + image[idx] = np.zeros(image[idx].shape) # black image + else: + has_nsfw_concept = None + flagged_images = None + return image, has_nsfw_concept, flagged_images + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion_k_diffusion.pipeline_stable_diffusion_k_diffusion.StableDiffusionKDiffusionPipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def perform_safety_guidance( + self, + enable_safety_guidance, + safety_momentum, + noise_guidance, + noise_pred_out, + i, + sld_guidance_scale, + sld_warmup_steps, + sld_threshold, + sld_momentum_scale, + sld_mom_beta, + ): + # Perform SLD guidance + if enable_safety_guidance: + if safety_momentum is None: + safety_momentum = torch.zeros_like(noise_guidance) + noise_pred_text, noise_pred_uncond = noise_pred_out[0], noise_pred_out[1] + noise_pred_safety_concept = noise_pred_out[2] + + # Equation 6 + scale = torch.clamp(torch.abs((noise_pred_text - noise_pred_safety_concept)) * sld_guidance_scale, max=1.0) + + # Equation 6 + safety_concept_scale = torch.where( + (noise_pred_text - noise_pred_safety_concept) >= sld_threshold, torch.zeros_like(scale), scale + ) + + # Equation 4 + noise_guidance_safety = torch.mul((noise_pred_safety_concept - noise_pred_uncond), safety_concept_scale) + + # Equation 7 + noise_guidance_safety = noise_guidance_safety + sld_momentum_scale * safety_momentum + + # Equation 8 + safety_momentum = sld_mom_beta * safety_momentum + (1 - sld_mom_beta) * noise_guidance_safety + + if i >= sld_warmup_steps: # Warmup + # Equation 3 + noise_guidance = noise_guidance - noise_guidance_safety + return noise_guidance, safety_momentum + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + sld_guidance_scale: Optional[float] = 1000, + sld_warmup_steps: Optional[int] = 10, + sld_threshold: Optional[float] = 0.01, + sld_momentum_scale: Optional[float] = 0.3, + sld_mom_beta: Optional[float] = 0.4, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + ip_adapter_image: (`PipelineImageInput`, *optional*): + Optional image input to work with IP Adapters. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + sld_guidance_scale (`float`, *optional*, defaults to 1000): + If `sld_guidance_scale < 1`, safety guidance is disabled. + sld_warmup_steps (`int`, *optional*, defaults to 10): + Number of warmup steps for safety guidance. SLD is only be applied for diffusion steps greater than + `sld_warmup_steps`. + sld_threshold (`float`, *optional*, defaults to 0.01): + Threshold that separates the hyperplane between appropriate and inappropriate images. + sld_momentum_scale (`float`, *optional*, defaults to 0.3): + Scale of the SLD momentum to be added to the safety guidance at each diffusion step. If set to 0.0, + momentum is disabled. Momentum is built up during warmup for diffusion steps smaller than + `sld_warmup_steps`. + sld_mom_beta (`float`, *optional*, defaults to 0.4): + Defines how safety guidance momentum builds up. `sld_mom_beta` indicates how much of the previous + momentum is kept. Momentum is built up during warmup for diffusion steps smaller than + `sld_warmup_steps`. + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + + Examples: + + ```py + import torch + from diffusers import StableDiffusionPipelineSafe + from diffusers.pipelines.stable_diffusion_safe import SafetyConfig + + pipeline = StableDiffusionPipelineSafe.from_pretrained( + "AIML-TUDA/stable-diffusion-safe", torch_dtype=torch.float16 + ).to("cuda") + prompt = "the four horsewomen of the apocalypse, painting by tom of finland, gaston bussiere, craig mullins, j. c. leyendecker" + image = pipeline(prompt=prompt, **SafetyConfig.MEDIUM).images[0] + ``` + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs(prompt, height, width, callback_steps) + + # 2. Define call parameters + batch_size = 1 if isinstance(prompt, str) else len(prompt) + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + enable_safety_guidance = sld_guidance_scale > 1.0 and do_classifier_free_guidance + if not enable_safety_guidance: + warnings.warn("Safety checker disabled!") + + if ip_adapter_image is not None: + output_hidden_state = False if isinstance(self.unet.encoder_hid_proj, ImageProjection) else True + image_embeds, negative_image_embeds = self.encode_image( + ip_adapter_image, device, num_images_per_prompt, output_hidden_state + ) + if do_classifier_free_guidance: + if enable_safety_guidance: + image_embeds = torch.cat([negative_image_embeds, image_embeds, image_embeds]) + else: + image_embeds = torch.cat([negative_image_embeds, image_embeds]) + + # 3. Encode input prompt + prompt_embeds = self._encode_prompt( + prompt, device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt, enable_safety_guidance + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 6.1 Add image embeds for IP-Adapter + added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None + + safety_momentum = None + + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = ( + torch.cat([latents] * (3 if enable_safety_guidance else 2)) + if do_classifier_free_guidance + else latents + ) + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, t, encoder_hidden_states=prompt_embeds, added_cond_kwargs=added_cond_kwargs + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_out = noise_pred.chunk((3 if enable_safety_guidance else 2)) + noise_pred_uncond, noise_pred_text = noise_pred_out[0], noise_pred_out[1] + + # default classifier free guidance + noise_guidance = noise_pred_text - noise_pred_uncond + + # Perform SLD guidance + if enable_safety_guidance: + if safety_momentum is None: + safety_momentum = torch.zeros_like(noise_guidance) + noise_pred_safety_concept = noise_pred_out[2] + + # Equation 6 + scale = torch.clamp( + torch.abs((noise_pred_text - noise_pred_safety_concept)) * sld_guidance_scale, max=1.0 + ) + + # Equation 6 + safety_concept_scale = torch.where( + (noise_pred_text - noise_pred_safety_concept) >= sld_threshold, + torch.zeros_like(scale), + scale, + ) + + # Equation 4 + noise_guidance_safety = torch.mul( + (noise_pred_safety_concept - noise_pred_uncond), safety_concept_scale + ) + + # Equation 7 + noise_guidance_safety = noise_guidance_safety + sld_momentum_scale * safety_momentum + + # Equation 8 + safety_momentum = sld_mom_beta * safety_momentum + (1 - sld_mom_beta) * noise_guidance_safety + + if i >= sld_warmup_steps: # Warmup + # Equation 3 + noise_guidance = noise_guidance - noise_guidance_safety + + noise_pred = noise_pred_uncond + guidance_scale * noise_guidance + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 8. Post-processing + image = self.decode_latents(latents) + + # 9. Run safety checker + image, has_nsfw_concept, flagged_images = self.run_safety_checker( + image, device, prompt_embeds.dtype, enable_safety_guidance + ) + + # 10. Convert to PIL + if output_type == "pil": + image = self.numpy_to_pil(image) + if flagged_images is not None: + flagged_images = self.numpy_to_pil(flagged_images) + + if not return_dict: + return ( + image, + has_nsfw_concept, + self._safety_text_concept if enable_safety_guidance else None, + flagged_images, + ) + + return StableDiffusionSafePipelineOutput( + images=image, + nsfw_content_detected=has_nsfw_concept, + applied_safety_concept=self._safety_text_concept if enable_safety_guidance else None, + unsafe_images=flagged_images, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/safety_checker.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/safety_checker.py new file mode 100755 index 0000000..549747e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_safe/safety_checker.py @@ -0,0 +1,109 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +import torch.nn as nn +from transformers import CLIPConfig, CLIPVisionModel, PreTrainedModel + +from ...utils import logging + + +logger = logging.get_logger(__name__) + + +def cosine_distance(image_embeds, text_embeds): + normalized_image_embeds = nn.functional.normalize(image_embeds) + normalized_text_embeds = nn.functional.normalize(text_embeds) + return torch.mm(normalized_image_embeds, normalized_text_embeds.t()) + + +class SafeStableDiffusionSafetyChecker(PreTrainedModel): + config_class = CLIPConfig + + _no_split_modules = ["CLIPEncoderLayer"] + + def __init__(self, config: CLIPConfig): + super().__init__(config) + + self.vision_model = CLIPVisionModel(config.vision_config) + self.visual_projection = nn.Linear(config.vision_config.hidden_size, config.projection_dim, bias=False) + + self.concept_embeds = nn.Parameter(torch.ones(17, config.projection_dim), requires_grad=False) + self.special_care_embeds = nn.Parameter(torch.ones(3, config.projection_dim), requires_grad=False) + + self.concept_embeds_weights = nn.Parameter(torch.ones(17), requires_grad=False) + self.special_care_embeds_weights = nn.Parameter(torch.ones(3), requires_grad=False) + + @torch.no_grad() + def forward(self, clip_input, images): + pooled_output = self.vision_model(clip_input)[1] # pooled_output + image_embeds = self.visual_projection(pooled_output) + + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + special_cos_dist = cosine_distance(image_embeds, self.special_care_embeds).cpu().float().numpy() + cos_dist = cosine_distance(image_embeds, self.concept_embeds).cpu().float().numpy() + + result = [] + batch_size = image_embeds.shape[0] + for i in range(batch_size): + result_img = {"special_scores": {}, "special_care": [], "concept_scores": {}, "bad_concepts": []} + + # increase this value to create a stronger `nfsw` filter + # at the cost of increasing the possibility of filtering benign images + adjustment = 0.0 + + for concept_idx in range(len(special_cos_dist[0])): + concept_cos = special_cos_dist[i][concept_idx] + concept_threshold = self.special_care_embeds_weights[concept_idx].item() + result_img["special_scores"][concept_idx] = round(concept_cos - concept_threshold + adjustment, 3) + if result_img["special_scores"][concept_idx] > 0: + result_img["special_care"].append({concept_idx, result_img["special_scores"][concept_idx]}) + adjustment = 0.01 + + for concept_idx in range(len(cos_dist[0])): + concept_cos = cos_dist[i][concept_idx] + concept_threshold = self.concept_embeds_weights[concept_idx].item() + result_img["concept_scores"][concept_idx] = round(concept_cos - concept_threshold + adjustment, 3) + if result_img["concept_scores"][concept_idx] > 0: + result_img["bad_concepts"].append(concept_idx) + + result.append(result_img) + + has_nsfw_concepts = [len(res["bad_concepts"]) > 0 for res in result] + + return images, has_nsfw_concepts + + @torch.no_grad() + def forward_onnx(self, clip_input: torch.FloatTensor, images: torch.FloatTensor): + pooled_output = self.vision_model(clip_input)[1] # pooled_output + image_embeds = self.visual_projection(pooled_output) + + special_cos_dist = cosine_distance(image_embeds, self.special_care_embeds) + cos_dist = cosine_distance(image_embeds, self.concept_embeds) + + # increase this value to create a stronger `nsfw` filter + # at the cost of increasing the possibility of filtering benign images + adjustment = 0.0 + + special_scores = special_cos_dist - self.special_care_embeds_weights + adjustment + # special_scores = special_scores.round(decimals=3) + special_care = torch.any(special_scores > 0, dim=1) + special_adjustment = special_care * 0.01 + special_adjustment = special_adjustment.unsqueeze(1).expand(-1, cos_dist.shape[1]) + + concept_scores = (cos_dist - self.concept_embeds_weights) + special_adjustment + # concept_scores = concept_scores.round(decimals=3) + has_nsfw_concepts = torch.any(concept_scores > 0, dim=1) + + return images, has_nsfw_concepts diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_sag/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_sag/__init__.py new file mode 100755 index 0000000..378e0e5 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_sag/__init__.py @@ -0,0 +1,48 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_stable_diffusion_sag"] = ["StableDiffusionSAGPipeline"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_stable_diffusion_sag import StableDiffusionSAGPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py new file mode 100755 index 0000000..96aa006 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_sag/pipeline_stable_diffusion_sag.py @@ -0,0 +1,886 @@ +# Copyright 2024 Susung Hong and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import torch +import torch.nn.functional as F +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import IPAdapterMixin, LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionPipelineOutput +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionSAGPipeline + + >>> pipe = StableDiffusionSAGPipeline.from_pretrained( + ... "runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt, sag_scale=0.75).images[0] + ``` +""" + + +# processes and stores attention probabilities +class CrossAttnStoreProcessor: + def __init__(self): + self.attention_probs = None + + def __call__( + self, + attn, + hidden_states, + encoder_hidden_states=None, + attention_mask=None, + ): + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + query = attn.to_q(hidden_states) + + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + self.attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(self.attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +# Modified to get self-attention guidance scale in this paper (https://arxiv.org/pdf/2210.00939.pdf) as an input +class StableDiffusionSAGPipeline(DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, IPAdapterMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + text_encoder ([`~transformers.CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + unet ([`UNet2DConditionModel`]): + A `UNet2DConditionModel` to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images; used as inputs to the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + _optional_components = ["safety_checker", "feature_extractor", "image_encoder"] + _exclude_from_cpu_offload = ["safety_checker"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + image_encoder: Optional[CLIPVisionModelWithProjection] = None, + requires_safety_checker: bool = True, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion_k_diffusion.pipeline_stable_diffusion_k_diffusion.StableDiffusionKDiffusionPipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + sag_scale: float = 0.75, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: Optional[int] = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + sag_scale (`float`, *optional*, defaults to 0.75): + Chosen between [0, 1.0] for better quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): + Optional image input to work with IP Adapters. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] is returned, + otherwise a `tuple` is returned where the first element is a list with the generated images and the + second element is a list of `bool`s indicating whether the corresponding generated image contains + "not-safe-for-work" (nsfw) content. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + # and `sag_scale` is` `s` of equation (16) + # of the self-attentnion guidance paper: https://arxiv.org/pdf/2210.00939.pdf + # `sag_scale = 0` means no self-attention guidance + do_self_attention_guidance = sag_scale > 0.0 + + if ip_adapter_image is not None: + output_hidden_state = False if isinstance(self.unet.encoder_hid_proj, ImageProjection) else True + image_embeds, negative_image_embeds = self.encode_image( + ip_adapter_image, device, num_images_per_prompt, output_hidden_state + ) + if do_classifier_free_guidance: + image_embeds = torch.cat([negative_image_embeds, image_embeds]) + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + if timesteps.dtype not in [torch.int16, torch.int32, torch.int64]: + raise ValueError( + f"{self.__class__.__name__} does not support using a scheduler of type {self.scheduler.__class__.__name__}. Please make sure to use one of 'DDIMScheduler, PNDMScheduler, DDPMScheduler, DEISMultistepScheduler, UniPCMultistepScheduler, DPMSolverMultistepScheduler, DPMSolverSinlgestepScheduler'." + ) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 6.1 Add image embeds for IP-Adapter + added_cond_kwargs = {"image_embeds": image_embeds} if ip_adapter_image is not None else None + added_uncond_kwargs = {"image_embeds": negative_image_embeds} if ip_adapter_image is not None else None + + # 7. Denoising loop + store_processor = CrossAttnStoreProcessor() + self.unet.mid_block.attentions[0].transformer_blocks[0].attn1.processor = store_processor + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + + map_size = None + + def get_map_size(module, input, output): + nonlocal map_size + map_size = output[0].shape[-2:] + + with self.unet.mid_block.attentions[0].register_forward_hook(get_map_size): + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # perform self-attention guidance with the stored self-attentnion map + if do_self_attention_guidance: + # classifier-free guidance produces two chunks of attention map + # and we only use unconditional one according to equation (25) + # in https://arxiv.org/pdf/2210.00939.pdf + if do_classifier_free_guidance: + # DDIM-like prediction of x0 + pred_x0 = self.pred_x0(latents, noise_pred_uncond, t) + # get the stored attention maps + uncond_attn, cond_attn = store_processor.attention_probs.chunk(2) + # self-attention-based degrading of latents + degraded_latents = self.sag_masking( + pred_x0, uncond_attn, map_size, t, self.pred_epsilon(latents, noise_pred_uncond, t) + ) + uncond_emb, _ = prompt_embeds.chunk(2) + # forward and give guidance + degraded_pred = self.unet( + degraded_latents, + t, + encoder_hidden_states=uncond_emb, + added_cond_kwargs=added_uncond_kwargs, + ).sample + noise_pred += sag_scale * (noise_pred_uncond - degraded_pred) + else: + # DDIM-like prediction of x0 + pred_x0 = self.pred_x0(latents, noise_pred, t) + # get the stored attention maps + cond_attn = store_processor.attention_probs + # self-attention-based degrading of latents + degraded_latents = self.sag_masking( + pred_x0, cond_attn, map_size, t, self.pred_epsilon(latents, noise_pred, t) + ) + # forward and give guidance + degraded_pred = self.unet( + degraded_latents, + t, + encoder_hidden_states=prompt_embeds, + added_cond_kwargs=added_cond_kwargs, + ).sample + noise_pred += sag_scale * (noise_pred - degraded_pred) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + else: + image = latents + has_nsfw_concept = None + + if has_nsfw_concept is None: + do_denormalize = [True] * image.shape[0] + else: + do_denormalize = [not has_nsfw for has_nsfw in has_nsfw_concept] + + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) + + def sag_masking(self, original_latents, attn_map, map_size, t, eps): + # Same masking process as in SAG paper: https://arxiv.org/pdf/2210.00939.pdf + bh, hw1, hw2 = attn_map.shape + b, latent_channel, latent_h, latent_w = original_latents.shape + h = self.unet.config.attention_head_dim + if isinstance(h, list): + h = h[-1] + + # Produce attention mask + attn_map = attn_map.reshape(b, h, hw1, hw2) + attn_mask = attn_map.mean(1, keepdim=False).sum(1, keepdim=False) > 1.0 + attn_mask = ( + attn_mask.reshape(b, map_size[0], map_size[1]) + .unsqueeze(1) + .repeat(1, latent_channel, 1, 1) + .type(attn_map.dtype) + ) + attn_mask = F.interpolate(attn_mask, (latent_h, latent_w)) + + # Blur according to the self-attention mask + degraded_latents = gaussian_blur_2d(original_latents, kernel_size=9, sigma=1.0) + degraded_latents = degraded_latents * attn_mask + original_latents * (1 - attn_mask) + + # Noise it again to match the noise level + degraded_latents = self.scheduler.add_noise(degraded_latents, noise=eps, timesteps=t[None]) + + return degraded_latents + + # Modified from diffusers.schedulers.scheduling_ddim.DDIMScheduler.step + # Note: there are some schedulers that clip or do not return x_0 (PNDMScheduler, DDIMScheduler, etc.) + def pred_x0(self, sample, model_output, timestep): + alpha_prod_t = self.scheduler.alphas_cumprod[timestep].to(sample.device) + + beta_prod_t = 1 - alpha_prod_t + if self.scheduler.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + elif self.scheduler.config.prediction_type == "sample": + pred_original_sample = model_output + elif self.scheduler.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + # predict V + model_output = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + else: + raise ValueError( + f"prediction_type given as {self.scheduler.config.prediction_type} must be one of `epsilon`, `sample`," + " or `v_prediction`" + ) + + return pred_original_sample + + def pred_epsilon(self, sample, model_output, timestep): + alpha_prod_t = self.scheduler.alphas_cumprod[timestep] + + beta_prod_t = 1 - alpha_prod_t + if self.scheduler.config.prediction_type == "epsilon": + pred_eps = model_output + elif self.scheduler.config.prediction_type == "sample": + pred_eps = (sample - (alpha_prod_t**0.5) * model_output) / (beta_prod_t**0.5) + elif self.scheduler.config.prediction_type == "v_prediction": + pred_eps = (beta_prod_t**0.5) * sample + (alpha_prod_t**0.5) * model_output + else: + raise ValueError( + f"prediction_type given as {self.scheduler.config.prediction_type} must be one of `epsilon`, `sample`," + " or `v_prediction`" + ) + + return pred_eps + + +# Gaussian blur +def gaussian_blur_2d(img, kernel_size, sigma): + ksize_half = (kernel_size - 1) * 0.5 + + x = torch.linspace(-ksize_half, ksize_half, steps=kernel_size) + + pdf = torch.exp(-0.5 * (x / sigma).pow(2)) + + x_kernel = pdf / pdf.sum() + x_kernel = x_kernel.to(device=img.device, dtype=img.dtype) + + kernel2d = torch.mm(x_kernel[:, None], x_kernel[None, :]) + kernel2d = kernel2d.expand(img.shape[-3], 1, kernel2d.shape[0], kernel2d.shape[1]) + + padding = [kernel_size // 2, kernel_size // 2, kernel_size // 2, kernel_size // 2] + + img = F.pad(img, padding, mode="reflect") + img = F.conv2d(img, kernel2d, groups=img.shape[-3]) + + return img diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/__init__.py new file mode 100755 index 0000000..8088fbc --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/__init__.py @@ -0,0 +1,76 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_flax_available, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_additional_imports = {} +_import_structure = {"pipeline_output": ["StableDiffusionXLPipelineOutput"]} + +if is_transformers_available() and is_flax_available(): + _import_structure["pipeline_output"].extend(["FlaxStableDiffusionXLPipelineOutput"]) +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_stable_diffusion_xl"] = ["StableDiffusionXLPipeline"] + _import_structure["pipeline_stable_diffusion_xl_img2img"] = ["StableDiffusionXLImg2ImgPipeline"] + _import_structure["pipeline_stable_diffusion_xl_inpaint"] = ["StableDiffusionXLInpaintPipeline"] + _import_structure["pipeline_stable_diffusion_xl_instruct_pix2pix"] = ["StableDiffusionXLInstructPix2PixPipeline"] + +if is_transformers_available() and is_flax_available(): + from ...schedulers.scheduling_pndm_flax import PNDMSchedulerState + + _additional_imports.update({"PNDMSchedulerState": PNDMSchedulerState}) + _import_structure["pipeline_flax_stable_diffusion_xl"] = ["FlaxStableDiffusionXLPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 + else: + from .pipeline_stable_diffusion_xl import StableDiffusionXLPipeline + from .pipeline_stable_diffusion_xl_img2img import StableDiffusionXLImg2ImgPipeline + from .pipeline_stable_diffusion_xl_inpaint import StableDiffusionXLInpaintPipeline + from .pipeline_stable_diffusion_xl_instruct_pix2pix import StableDiffusionXLInstructPix2PixPipeline + + try: + if not (is_transformers_available() and is_flax_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_flax_objects import * + else: + from .pipeline_flax_stable_diffusion_xl import ( + FlaxStableDiffusionXLPipeline, + ) + from .pipeline_output import FlaxStableDiffusionXLPipelineOutput + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) + for name, value in _additional_imports.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py new file mode 100755 index 0000000..77363b2 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_flax_stable_diffusion_xl.py @@ -0,0 +1,308 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from functools import partial +from typing import Dict, List, Optional, Union + +import jax +import jax.numpy as jnp +from flax.core.frozen_dict import FrozenDict +from transformers import CLIPTokenizer, FlaxCLIPTextModel + +from diffusers.utils import logging + +from ...models import FlaxAutoencoderKL, FlaxUNet2DConditionModel +from ...schedulers import ( + FlaxDDIMScheduler, + FlaxDPMSolverMultistepScheduler, + FlaxLMSDiscreteScheduler, + FlaxPNDMScheduler, +) +from ..pipeline_flax_utils import FlaxDiffusionPipeline +from .pipeline_output import FlaxStableDiffusionXLPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +# Set to True to use python for loop instead of jax.fori_loop for easier debugging +DEBUG = False + + +class FlaxStableDiffusionXLPipeline(FlaxDiffusionPipeline): + def __init__( + self, + text_encoder: FlaxCLIPTextModel, + text_encoder_2: FlaxCLIPTextModel, + vae: FlaxAutoencoderKL, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: FlaxUNet2DConditionModel, + scheduler: Union[ + FlaxDDIMScheduler, FlaxPNDMScheduler, FlaxLMSDiscreteScheduler, FlaxDPMSolverMultistepScheduler + ], + dtype: jnp.dtype = jnp.float32, + ): + super().__init__() + self.dtype = dtype + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + + def prepare_inputs(self, prompt: Union[str, List[str]]): + if not isinstance(prompt, (str, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + # Assume we have the two encoders + inputs = [] + for tokenizer in [self.tokenizer, self.tokenizer_2]: + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + inputs.append(text_inputs.input_ids) + inputs = jnp.stack(inputs, axis=1) + return inputs + + def __call__( + self, + prompt_ids: jax.Array, + params: Union[Dict, FrozenDict], + prng_seed: jax.Array, + num_inference_steps: int = 50, + guidance_scale: Union[float, jax.Array] = 7.5, + height: Optional[int] = None, + width: Optional[int] = None, + latents: jnp.array = None, + neg_prompt_ids: jnp.array = None, + return_dict: bool = True, + output_type: str = None, + jit: bool = False, + ): + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + if isinstance(guidance_scale, float) and jit: + # Convert to a tensor so each device gets a copy. + guidance_scale = jnp.array([guidance_scale] * prompt_ids.shape[0]) + guidance_scale = guidance_scale[:, None] + + return_latents = output_type == "latent" + + if jit: + images = _p_generate( + self, + prompt_ids, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, + return_latents, + ) + else: + images = self._generate( + prompt_ids, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, + return_latents, + ) + + if not return_dict: + return (images,) + + return FlaxStableDiffusionXLPipelineOutput(images=images) + + def get_embeddings(self, prompt_ids: jnp.array, params): + # We assume we have the two encoders + + # bs, encoder_input, seq_length + te_1_inputs = prompt_ids[:, 0, :] + te_2_inputs = prompt_ids[:, 1, :] + + prompt_embeds = self.text_encoder(te_1_inputs, params=params["text_encoder"], output_hidden_states=True) + prompt_embeds = prompt_embeds["hidden_states"][-2] + prompt_embeds_2_out = self.text_encoder_2( + te_2_inputs, params=params["text_encoder_2"], output_hidden_states=True + ) + prompt_embeds_2 = prompt_embeds_2_out["hidden_states"][-2] + text_embeds = prompt_embeds_2_out["text_embeds"] + prompt_embeds = jnp.concatenate([prompt_embeds, prompt_embeds_2], axis=-1) + return prompt_embeds, text_embeds + + def _get_add_time_ids(self, original_size, crops_coords_top_left, target_size, bs, dtype): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + add_time_ids = jnp.array([add_time_ids] * bs, dtype=dtype) + return add_time_ids + + def _generate( + self, + prompt_ids: jnp.array, + params: Union[Dict, FrozenDict], + prng_seed: jax.Array, + num_inference_steps: int, + height: int, + width: int, + guidance_scale: float, + latents: Optional[jnp.array] = None, + neg_prompt_ids: Optional[jnp.array] = None, + return_latents=False, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + # Encode input prompt + prompt_embeds, pooled_embeds = self.get_embeddings(prompt_ids, params) + + # Get unconditional embeddings + batch_size = prompt_embeds.shape[0] + if neg_prompt_ids is None: + neg_prompt_embeds = jnp.zeros_like(prompt_embeds) + negative_pooled_embeds = jnp.zeros_like(pooled_embeds) + else: + neg_prompt_embeds, negative_pooled_embeds = self.get_embeddings(neg_prompt_ids, params) + + add_time_ids = self._get_add_time_ids( + (height, width), (0, 0), (height, width), prompt_embeds.shape[0], dtype=prompt_embeds.dtype + ) + + prompt_embeds = jnp.concatenate([neg_prompt_embeds, prompt_embeds], axis=0) # (2, 77, 2048) + add_text_embeds = jnp.concatenate([negative_pooled_embeds, pooled_embeds], axis=0) + add_time_ids = jnp.concatenate([add_time_ids, add_time_ids], axis=0) + + # Ensure model output will be `float32` before going into the scheduler + guidance_scale = jnp.array([guidance_scale], dtype=jnp.float32) + + # Create random latents + latents_shape = ( + batch_size, + self.unet.config.in_channels, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + if latents is None: + latents = jax.random.normal(prng_seed, shape=latents_shape, dtype=jnp.float32) + else: + if latents.shape != latents_shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {latents_shape}") + + # Prepare scheduler state + scheduler_state = self.scheduler.set_timesteps( + params["scheduler"], num_inference_steps=num_inference_steps, shape=latents.shape + ) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * scheduler_state.init_noise_sigma + + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + + # Denoising loop + def loop_body(step, args): + latents, scheduler_state = args + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + latents_input = jnp.concatenate([latents] * 2) + + t = jnp.array(scheduler_state.timesteps, dtype=jnp.int32)[step] + timestep = jnp.broadcast_to(t, latents_input.shape[0]) + + latents_input = self.scheduler.scale_model_input(scheduler_state, latents_input, t) + + # predict the noise residual + noise_pred = self.unet.apply( + {"params": params["unet"]}, + jnp.array(latents_input), + jnp.array(timestep, dtype=jnp.int32), + encoder_hidden_states=prompt_embeds, + added_cond_kwargs=added_cond_kwargs, + ).sample + # perform guidance + noise_pred_uncond, noise_prediction_text = jnp.split(noise_pred, 2, axis=0) + noise_pred = noise_pred_uncond + guidance_scale * (noise_prediction_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents, scheduler_state = self.scheduler.step(scheduler_state, noise_pred, t, latents).to_tuple() + return latents, scheduler_state + + if DEBUG: + # run with python for loop + for i in range(num_inference_steps): + latents, scheduler_state = loop_body(i, (latents, scheduler_state)) + else: + latents, _ = jax.lax.fori_loop(0, num_inference_steps, loop_body, (latents, scheduler_state)) + + if return_latents: + return latents + + # Decode latents + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.apply({"params": params["vae"]}, latents, method=self.vae.decode).sample + + image = (image / 2 + 0.5).clip(0, 1).transpose(0, 2, 3, 1) + return image + + +# Static argnums are pipe, num_inference_steps, height, width, return_latents. A change would trigger recompilation. +# Non-static args are (sharded) input tensors mapped over their first dimension (hence, `0`). +@partial( + jax.pmap, + in_axes=(None, 0, 0, 0, None, None, None, 0, 0, 0, None), + static_broadcasted_argnums=(0, 4, 5, 6, 10), +) +def _p_generate( + pipe, + prompt_ids, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, + return_latents, +): + return pipe._generate( + prompt_ids, + params, + prng_seed, + num_inference_steps, + height, + width, + guidance_scale, + latents, + neg_prompt_ids, + return_latents, + ) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_output.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_output.py new file mode 100755 index 0000000..0783f44 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_output.py @@ -0,0 +1,37 @@ +from dataclasses import dataclass +from typing import List, Union + +import numpy as np +import PIL.Image + +from ...utils import BaseOutput, is_flax_available + + +@dataclass +class StableDiffusionXLPipelineOutput(BaseOutput): + """ + Output class for Stable Diffusion pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +if is_flax_available(): + import flax + + @flax.struct.dataclass + class FlaxStableDiffusionXLPipelineOutput(BaseOutput): + """ + Output class for Flax Stable Diffusion XL pipelines. + + Args: + images (`np.ndarray`) + Array of shape `(batch_size, height, width, num_channels)` with images from the diffusion pipeline. + """ + + images: np.ndarray diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py new file mode 100755 index 0000000..776696e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl.py @@ -0,0 +1,1266 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import ( + FromSingleFileMixin, + IPAdapterMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, +) +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...models.attention_processor import ( + AttnProcessor2_0, + FusedAttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + is_invisible_watermark_available, + is_torch_xla_available, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from .pipeline_output import StableDiffusionXLPipelineOutput + + +if is_invisible_watermark_available(): + from .watermark import StableDiffusionXLWatermarker + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionXLPipeline + + >>> pipe = StableDiffusionXLPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt).images[0] + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class StableDiffusionXLPipeline( + DiffusionPipeline, + StableDiffusionMixin, + FromSingleFileMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, + IPAdapterMixin, +): + r""" + Pipeline for text-to-image generation using Stable Diffusion XL. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): + Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of + `stabilityai/stable-diffusion-xl-base-1-0`. + add_watermarker (`bool`, *optional*): + Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to + watermark output images. If not defined, it will default to True if the package is installed, otherwise no + watermarker will be used. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae" + _optional_components = [ + "tokenizer", + "tokenizer_2", + "text_encoder", + "text_encoder_2", + "image_encoder", + "feature_extractor", + ] + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + "add_text_embeds", + "add_time_ids", + "negative_pooled_prompt_embeds", + "negative_add_time_ids", + ] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + image_encoder: CLIPVisionModelWithProjection = None, + feature_extractor: CLIPImageProcessor = None, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + scheduler=scheduler, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + self.default_sample_size = self.unet.config.sample_size + + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() + + if add_watermarker: + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None + + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # textual inversion: process multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_2 = ( + batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 + ) + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def _get_add_time_ids( + self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None + ): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + return add_time_ids + + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + FusedAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def denoising_end(self): + return self._denoising_end + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + timesteps: List[int] = None, + denoising_end: Optional[float] = None, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + original_size: Optional[Tuple[int, int]] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Optional[Tuple[int, int]] = None, + negative_original_size: Optional[Tuple[int, int]] = None, + negative_crops_coords_top_left: Tuple[int, int] = (0, 0), + negative_target_size: Optional[Tuple[int, int]] = None, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise as determined by the discrete timesteps selected by the + scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a + "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) + guidance_scale (`float`, *optional*, defaults to 5.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.0): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a specific image resolution. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a target image resolution. It should be as same + as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 0. Default height and width to unet + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._guidance_rescale = guidance_rescale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._denoising_end = denoising_end + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=lora_scale, + clip_skip=self.clip_skip, + ) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Prepare added time ids & embeddings + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + if negative_original_size is not None and negative_target_size is not None: + negative_add_time_ids = self._get_add_time_ids( + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + else: + negative_add_time_ids = add_time_ids + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 8. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + # 8.1 Apply denoising_end + if ( + self.denoising_end is not None + and isinstance(self.denoising_end, float) + and self.denoising_end > 0 + and self.denoising_end < 1 + ): + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (self.denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + # 9. Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + added_cond_kwargs["image_embeds"] = image_embeds + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds) + negative_pooled_prompt_embeds = callback_outputs.pop( + "negative_pooled_prompt_embeds", negative_pooled_prompt_embeds + ) + add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids) + negative_add_time_ids = callback_outputs.pop("negative_add_time_ids", negative_add_time_ids) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if XLA_AVAILABLE: + xm.mark_step() + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + # unscale/denormalize the latents + # denormalize with the mean and std if available and not None + has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None + has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None + if has_latents_mean and has_latents_std: + latents_mean = ( + torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents_std = ( + torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean + else: + latents = latents / self.vae.config.scaling_factor + + image = self.vae.decode(latents, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + image = latents + + if not output_type == "latent": + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return StableDiffusionXLPipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py new file mode 100755 index 0000000..fd4c412 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_img2img.py @@ -0,0 +1,1442 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import PIL.Image +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import ( + FromSingleFileMixin, + IPAdapterMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, +) +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...models.attention_processor import ( + AttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + is_invisible_watermark_available, + is_torch_xla_available, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from .pipeline_output import StableDiffusionXLPipelineOutput + + +if is_invisible_watermark_available(): + from .watermark import StableDiffusionXLWatermarker + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionXLImg2ImgPipeline + >>> from diffusers.utils import load_image + + >>> pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-xl-refiner-1.0", torch_dtype=torch.float16 + ... ) + >>> pipe = pipe.to("cuda") + >>> url = "https://huggingface.co/datasets/patrickvonplaten/images/resolve/main/aa_xl/000000009.png" + + >>> init_image = load_image(url).convert("RGB") + >>> prompt = "a photo of an astronaut riding a horse on mars" + >>> image = pipe(prompt, image=init_image).images[0] + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class StableDiffusionXLImg2ImgPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + FromSingleFileMixin, + StableDiffusionXLLoraLoaderMixin, + IPAdapterMixin, +): + r""" + Pipeline for text-to-image generation using Stable Diffusion XL. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + requires_aesthetics_score (`bool`, *optional*, defaults to `"False"`): + Whether the `unet` requires an `aesthetic_score` condition to be passed during inference. Also see the + config of `stabilityai/stable-diffusion-xl-refiner-1-0`. + force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): + Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of + `stabilityai/stable-diffusion-xl-base-1-0`. + add_watermarker (`bool`, *optional*): + Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to + watermark output images. If not defined, it will default to True if the package is installed, otherwise no + watermarker will be used. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae" + _optional_components = [ + "tokenizer", + "tokenizer_2", + "text_encoder", + "text_encoder_2", + "image_encoder", + "feature_extractor", + ] + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + "add_text_embeds", + "add_time_ids", + "negative_pooled_prompt_embeds", + "add_neg_time_ids", + ] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + image_encoder: CLIPVisionModelWithProjection = None, + feature_extractor: CLIPImageProcessor = None, + requires_aesthetics_score: bool = False, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + scheduler=scheduler, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.register_to_config(requires_aesthetics_score=requires_aesthetics_score) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() + + if add_watermarker: + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # textual inversion: process multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_2 = ( + batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 + ) + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + prompt_2, + strength, + num_inference_steps, + callback_steps, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + if num_inference_steps is None: + raise ValueError("`num_inference_steps` cannot be None.") + elif not isinstance(num_inference_steps, int) or num_inference_steps <= 0: + raise ValueError( + f"`num_inference_steps` has to be a positive integer but is {num_inference_steps} of type" + f" {type(num_inference_steps)}." + ) + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + def get_timesteps(self, num_inference_steps, strength, device, denoising_start=None): + # get the original timestep using init_timestep + if denoising_start is None: + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + t_start = max(num_inference_steps - init_timestep, 0) + else: + t_start = 0 + + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + + # Strength is irrelevant if we directly request a timestep to start at; + # that is, strength is determined by the denoising_start instead. + if denoising_start is not None: + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (denoising_start * self.scheduler.config.num_train_timesteps) + ) + ) + + num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() + if self.scheduler.order == 2 and num_inference_steps % 2 == 0: + # if the scheduler is a 2nd order scheduler we might have to do +1 + # because `num_inference_steps` might be even given that every timestep + # (except the highest one) is duplicated. If `num_inference_steps` is even it would + # mean that we cut the timesteps in the middle of the denoising step + # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1 + # we ensure that the denoising process always ends after the 2nd derivate step of the scheduler + num_inference_steps = num_inference_steps + 1 + + # because t_n+1 >= t_n, we slice the timesteps starting from the end + timesteps = timesteps[-num_inference_steps:] + return timesteps, num_inference_steps + + return timesteps, num_inference_steps - t_start + + def prepare_latents( + self, image, timestep, batch_size, num_images_per_prompt, dtype, device, generator=None, add_noise=True + ): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + # Offload text encoder if `enable_model_cpu_offload` was enabled + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.text_encoder_2.to("cpu") + torch.cuda.empty_cache() + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + init_latents = image + + else: + # make sure the VAE is in float32 mode, as it overflows in float16 + if self.vae.config.force_upcast: + image = image.float() + self.vae.to(dtype=torch.float32) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + elif isinstance(generator, list): + init_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(batch_size) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + if self.vae.config.force_upcast: + self.vae.to(dtype) + + init_latents = init_latents.to(dtype) + init_latents = self.vae.config.scaling_factor * init_latents + + if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] == 0: + # expand init_latents for batch_size + additional_image_per_prompt = batch_size // init_latents.shape[0] + init_latents = torch.cat([init_latents] * additional_image_per_prompt, dim=0) + elif batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {init_latents.shape[0]} to {batch_size} text prompts." + ) + else: + init_latents = torch.cat([init_latents], dim=0) + + if add_noise: + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + + latents = init_latents + + return latents + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + def _get_add_time_ids( + self, + original_size, + crops_coords_top_left, + target_size, + aesthetic_score, + negative_aesthetic_score, + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype, + text_encoder_projection_dim=None, + ): + if self.config.requires_aesthetics_score: + add_time_ids = list(original_size + crops_coords_top_left + (aesthetic_score,)) + add_neg_time_ids = list( + negative_original_size + negative_crops_coords_top_left + (negative_aesthetic_score,) + ) + else: + add_time_ids = list(original_size + crops_coords_top_left + target_size) + add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if ( + expected_add_embed_dim > passed_add_embed_dim + and (expected_add_embed_dim - passed_add_embed_dim) == self.unet.config.addition_time_embed_dim + ): + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to enable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=True)` to make sure `aesthetic_score` {aesthetic_score} and `negative_aesthetic_score` {negative_aesthetic_score} is correctly used by the model." + ) + elif ( + expected_add_embed_dim < passed_add_embed_dim + and (passed_add_embed_dim - expected_add_embed_dim) == self.unet.config.addition_time_embed_dim + ): + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to disable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=False)` to make sure `target_size` {target_size} is correctly used by the model." + ) + elif expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + add_neg_time_ids = torch.tensor([add_neg_time_ids], dtype=dtype) + + return add_time_ids, add_neg_time_ids + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def denoising_end(self): + return self._denoising_end + + @property + def denoising_start(self): + return self._denoising_start + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + image: PipelineImageInput = None, + strength: float = 0.3, + num_inference_steps: int = 50, + timesteps: List[int] = None, + denoising_start: Optional[float] = None, + denoising_end: Optional[float] = None, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + original_size: Tuple[int, int] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Tuple[int, int] = None, + negative_original_size: Optional[Tuple[int, int]] = None, + negative_crops_coords_top_left: Tuple[int, int] = (0, 0), + negative_target_size: Optional[Tuple[int, int]] = None, + aesthetic_score: float = 6.0, + negative_aesthetic_score: float = 2.5, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + image (`torch.FloatTensor` or `PIL.Image.Image` or `np.ndarray` or `List[torch.FloatTensor]` or `List[PIL.Image.Image]` or `List[np.ndarray]`): + The image(s) to modify with the pipeline. + strength (`float`, *optional*, defaults to 0.3): + Conceptually, indicates how much to transform the reference `image`. Must be between 0 and 1. `image` + will be used as a starting point, adding more noise to it the larger the `strength`. The number of + denoising steps depends on the amount of noise initially added. When `strength` is 1, added noise will + be maximum and the denoising process will run for the full number of iterations specified in + `num_inference_steps`. A value of 1, therefore, essentially ignores `image`. Note that in the case of + `denoising_start` being declared as an integer, the value of `strength` will be ignored. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + denoising_start (`float`, *optional*): + When specified, indicates the fraction (between 0.0 and 1.0) of the total denoising process to be + bypassed before it is initiated. Consequently, the initial part of the denoising process is skipped and + it is assumed that the passed `image` is a partly denoised image. Note that when this is specified, + strength will be ignored. The `denoising_start` parameter is particularly beneficial when this pipeline + is integrated into a "Mixture of Denoisers" multi-pipeline setup, as detailed in [**Refine Image + Quality**](https://huggingface.co/docs/diffusers/using-diffusers/sdxl#refine-image-quality). + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise (ca. final 20% of timesteps still needed) and should be + denoised by a successor pipeline that has `denoising_start` set to 0.8 so that it only denoises the + final 20% of the scheduler. The denoising_end parameter should ideally be utilized when this pipeline + forms a part of a "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refine Image + Quality**](https://huggingface.co/docs/diffusers/using-diffusers/sdxl#refine-image-quality). + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.0): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a specific image resolution. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a target image resolution. It should be as same + as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + aesthetic_score (`float`, *optional*, defaults to 6.0): + Used to simulate an aesthetic score of the generated image by influencing the positive text condition. + Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_aesthetic_score (`float`, *optional*, defaults to 2.5): + Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). Can be used to + simulate an aesthetic score of the generated image by influencing the negative text condition. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a + `tuple. When returning a tuple, the first element is a list with the generated images. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + strength, + num_inference_steps, + callback_steps, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + ) + + self._guidance_scale = guidance_scale + self._guidance_rescale = guidance_rescale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._denoising_end = denoising_end + self._denoising_start = denoising_start + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + + # 4. Preprocess image + image = self.image_processor.preprocess(image) + + # 5. Prepare timesteps + def denoising_value_valid(dnv): + return isinstance(dnv, float) and 0 < dnv < 1 + + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + timesteps, num_inference_steps = self.get_timesteps( + num_inference_steps, + strength, + device, + denoising_start=self.denoising_start if denoising_value_valid(self.denoising_start) else None, + ) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + + add_noise = True if self.denoising_start is None else False + # 6. Prepare latent variables + latents = self.prepare_latents( + image, + latent_timestep, + batch_size, + num_images_per_prompt, + prompt_embeds.dtype, + device, + generator, + add_noise, + ) + # 7. Prepare extra step kwargs. + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + height, width = latents.shape[-2:] + height = height * self.vae_scale_factor + width = width * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 8. Prepare added time ids & embeddings + if negative_original_size is None: + negative_original_size = original_size + if negative_target_size is None: + negative_target_size = target_size + + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids, add_neg_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + aesthetic_score, + negative_aesthetic_score, + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + add_time_ids = add_time_ids.repeat(batch_size * num_images_per_prompt, 1) + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_neg_time_ids = add_neg_time_ids.repeat(batch_size * num_images_per_prompt, 1) + add_time_ids = torch.cat([add_neg_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 9. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + # 9.1 Apply denoising_end + if ( + self.denoising_end is not None + and self.denoising_start is not None + and denoising_value_valid(self.denoising_end) + and denoising_value_valid(self.denoising_start) + and self.denoising_start >= self.denoising_end + ): + raise ValueError( + f"`denoising_start`: {self.denoising_start} cannot be larger than or equal to `denoising_end`: " + + f" {self.denoising_end} when using type float." + ) + elif self.denoising_end is not None and denoising_value_valid(self.denoising_end): + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (self.denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + # 9.2 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + added_cond_kwargs["image_embeds"] = image_embeds + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds) + negative_pooled_prompt_embeds = callback_outputs.pop( + "negative_pooled_prompt_embeds", negative_pooled_prompt_embeds + ) + add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids) + add_neg_time_ids = callback_outputs.pop("add_neg_time_ids", add_neg_time_ids) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if XLA_AVAILABLE: + xm.mark_step() + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + # unscale/denormalize the latents + # denormalize with the mean and std if available and not None + has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None + has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None + if has_latents_mean and has_latents_std: + latents_mean = ( + torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents_std = ( + torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean + else: + latents = latents / self.vae.config.scaling_factor + + image = self.vae.decode(latents, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + image = latents + + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return StableDiffusionXLPipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py new file mode 100755 index 0000000..c25628c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_inpaint.py @@ -0,0 +1,1812 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import ( + FromSingleFileMixin, + IPAdapterMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, +) +from ...models import AutoencoderKL, ImageProjection, UNet2DConditionModel +from ...models.attention_processor import ( + AttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + is_invisible_watermark_available, + is_torch_xla_available, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from .pipeline_output import StableDiffusionXLPipelineOutput + + +if is_invisible_watermark_available(): + from .watermark import StableDiffusionXLWatermarker + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionXLInpaintPipeline + >>> from diffusers.utils import load_image + + >>> pipe = StableDiffusionXLInpaintPipeline.from_pretrained( + ... "stabilityai/stable-diffusion-xl-base-1.0", + ... torch_dtype=torch.float16, + ... variant="fp16", + ... use_safetensors=True, + ... ) + >>> pipe.to("cuda") + + >>> img_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png" + >>> mask_url = "https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png" + + >>> init_image = load_image(img_url).convert("RGB") + >>> mask_image = load_image(mask_url).convert("RGB") + + >>> prompt = "A majestic tiger sitting on a bench" + >>> image = pipe( + ... prompt=prompt, image=init_image, mask_image=mask_image, num_inference_steps=50, strength=0.80 + ... ).images[0] + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +def mask_pil_to_torch(mask, height, width): + # preprocess mask + if isinstance(mask, (PIL.Image.Image, np.ndarray)): + mask = [mask] + + if isinstance(mask, list) and isinstance(mask[0], PIL.Image.Image): + mask = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in mask] + mask = np.concatenate([np.array(m.convert("L"))[None, None, :] for m in mask], axis=0) + mask = mask.astype(np.float32) / 255.0 + elif isinstance(mask, list) and isinstance(mask[0], np.ndarray): + mask = np.concatenate([m[None, None, :] for m in mask], axis=0) + + mask = torch.from_numpy(mask) + return mask + + +def prepare_mask_and_masked_image(image, mask, height, width, return_image: bool = False): + """ + Prepares a pair (image, mask) to be consumed by the Stable Diffusion pipeline. This means that those inputs will be + converted to ``torch.Tensor`` with shapes ``batch x channels x height x width`` where ``channels`` is ``3`` for the + ``image`` and ``1`` for the ``mask``. + + The ``image`` will be converted to ``torch.float32`` and normalized to be in ``[-1, 1]``. The ``mask`` will be + binarized (``mask > 0.5``) and cast to ``torch.float32`` too. + + Args: + image (Union[np.array, PIL.Image, torch.Tensor]): The image to inpaint. + It can be a ``PIL.Image``, or a ``height x width x 3`` ``np.array`` or a ``channels x height x width`` + ``torch.Tensor`` or a ``batch x channels x height x width`` ``torch.Tensor``. + mask (_type_): The mask to apply to the image, i.e. regions to inpaint. + It can be a ``PIL.Image``, or a ``height x width`` ``np.array`` or a ``1 x height x width`` + ``torch.Tensor`` or a ``batch x 1 x height x width`` ``torch.Tensor``. + + + Raises: + ValueError: ``torch.Tensor`` images should be in the ``[-1, 1]`` range. ValueError: ``torch.Tensor`` mask + should be in the ``[0, 1]`` range. ValueError: ``mask`` and ``image`` should have the same spatial dimensions. + TypeError: ``mask`` is a ``torch.Tensor`` but ``image`` is not + (ot the other way around). + + Returns: + tuple[torch.Tensor]: The pair (mask, masked_image) as ``torch.Tensor`` with 4 + dimensions: ``batch x channels x height x width``. + """ + + # checkpoint. TOD(Yiyi) - need to clean this up later + deprecation_message = "The prepare_mask_and_masked_image method is deprecated and will be removed in a future version. Please use VaeImageProcessor.preprocess instead" + deprecate( + "prepare_mask_and_masked_image", + "0.30.0", + deprecation_message, + ) + if image is None: + raise ValueError("`image` input cannot be undefined.") + + if mask is None: + raise ValueError("`mask_image` input cannot be undefined.") + + if isinstance(image, torch.Tensor): + if not isinstance(mask, torch.Tensor): + mask = mask_pil_to_torch(mask, height, width) + + if image.ndim == 3: + image = image.unsqueeze(0) + + # Batch and add channel dim for single mask + if mask.ndim == 2: + mask = mask.unsqueeze(0).unsqueeze(0) + + # Batch single mask or add channel dim + if mask.ndim == 3: + # Single batched mask, no channel dim or single mask not batched but channel dim + if mask.shape[0] == 1: + mask = mask.unsqueeze(0) + + # Batched masks no channel dim + else: + mask = mask.unsqueeze(1) + + assert image.ndim == 4 and mask.ndim == 4, "Image and Mask must have 4 dimensions" + # assert image.shape[-2:] == mask.shape[-2:], "Image and Mask must have the same spatial dimensions" + assert image.shape[0] == mask.shape[0], "Image and Mask must have the same batch size" + + # Check image is in [-1, 1] + # if image.min() < -1 or image.max() > 1: + # raise ValueError("Image should be in [-1, 1] range") + + # Check mask is in [0, 1] + if mask.min() < 0 or mask.max() > 1: + raise ValueError("Mask should be in [0, 1] range") + + # Binarize mask + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + + # Image as float32 + image = image.to(dtype=torch.float32) + elif isinstance(mask, torch.Tensor): + raise TypeError(f"`mask` is a torch.Tensor but `image` (type: {type(image)} is not") + else: + # preprocess image + if isinstance(image, (PIL.Image.Image, np.ndarray)): + image = [image] + if isinstance(image, list) and isinstance(image[0], PIL.Image.Image): + # resize all images w.r.t passed height an width + image = [i.resize((width, height), resample=PIL.Image.LANCZOS) for i in image] + image = [np.array(i.convert("RGB"))[None, :] for i in image] + image = np.concatenate(image, axis=0) + elif isinstance(image, list) and isinstance(image[0], np.ndarray): + image = np.concatenate([i[None, :] for i in image], axis=0) + + image = image.transpose(0, 3, 1, 2) + image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0 + + mask = mask_pil_to_torch(mask, height, width) + mask[mask < 0.5] = 0 + mask[mask >= 0.5] = 1 + + if image.shape[1] == 4: + # images are in latent space and thus can't + # be masked set masked_image to None + # we assume that the checkpoint is not an inpainting + # checkpoint. TOD(Yiyi) - need to clean this up later + masked_image = None + else: + masked_image = image * (mask < 0.5) + + # n.b. ensure backwards compatibility as old function does not return image + if return_image: + return mask, masked_image, image + + return mask, masked_image + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class StableDiffusionXLInpaintPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + StableDiffusionXLLoraLoaderMixin, + FromSingleFileMixin, + IPAdapterMixin, +): + r""" + Pipeline for text-to-image generation using Stable Diffusion XL. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + requires_aesthetics_score (`bool`, *optional*, defaults to `"False"`): + Whether the `unet` requires a aesthetic_score condition to be passed during inference. Also see the config + of `stabilityai/stable-diffusion-xl-refiner-1-0`. + force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): + Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of + `stabilityai/stable-diffusion-xl-base-1-0`. + add_watermarker (`bool`, *optional*): + Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to + watermark output images. If not defined, it will default to True if the package is installed, otherwise no + watermarker will be used. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae" + + _optional_components = [ + "tokenizer", + "tokenizer_2", + "text_encoder", + "text_encoder_2", + "image_encoder", + "feature_extractor", + ] + _callback_tensor_inputs = [ + "latents", + "prompt_embeds", + "negative_prompt_embeds", + "add_text_embeds", + "add_time_ids", + "negative_pooled_prompt_embeds", + "add_neg_time_ids", + "mask", + "masked_image_latents", + ] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + image_encoder: CLIPVisionModelWithProjection = None, + feature_extractor: CLIPImageProcessor = None, + requires_aesthetics_score: bool = False, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + scheduler=scheduler, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.register_to_config(requires_aesthetics_score=requires_aesthetics_score) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.mask_processor = VaeImageProcessor( + vae_scale_factor=self.vae_scale_factor, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() + + if add_watermarker: + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # textual inversion: process multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_2 = ( + batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 + ) + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + prompt_2, + image, + mask_image, + height, + width, + strength, + callback_steps, + output_type, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + padding_mask_crop=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + if padding_mask_crop is not None: + if not isinstance(image, PIL.Image.Image): + raise ValueError( + f"The image should be a PIL image when inpainting mask crop, but is of type" f" {type(image)}." + ) + if not isinstance(mask_image, PIL.Image.Image): + raise ValueError( + f"The mask image should be a PIL image when inpainting mask crop, but is of type" + f" {type(mask_image)}." + ) + if output_type != "pil": + raise ValueError(f"The output type should be PIL when inpainting mask crop, but is" f" {output_type}.") + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + def prepare_latents( + self, + batch_size, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + image=None, + timestep=None, + is_strength_max=True, + add_noise=True, + return_noise=False, + return_image_latents=False, + ): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if (image is None or timestep is None) and not is_strength_max: + raise ValueError( + "Since strength < 1. initial latents are to be initialised as a combination of Image + Noise." + "However, either the image or the noise timestep has not been provided." + ) + + if image.shape[1] == 4: + image_latents = image.to(device=device, dtype=dtype) + image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1) + elif return_image_latents or (latents is None and not is_strength_max): + image = image.to(device=device, dtype=dtype) + image_latents = self._encode_vae_image(image=image, generator=generator) + image_latents = image_latents.repeat(batch_size // image_latents.shape[0], 1, 1, 1) + + if latents is None and add_noise: + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + # if strength is 1. then initialise the latents to noise, else initial to image + noise + latents = noise if is_strength_max else self.scheduler.add_noise(image_latents, noise, timestep) + # if pure noise then scale the initial latents by the Scheduler's init sigma + latents = latents * self.scheduler.init_noise_sigma if is_strength_max else latents + elif add_noise: + noise = latents.to(device) + latents = noise * self.scheduler.init_noise_sigma + else: + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + latents = image_latents.to(device) + + outputs = (latents,) + + if return_noise: + outputs += (noise,) + + if return_image_latents: + outputs += (image_latents,) + + return outputs + + def _encode_vae_image(self, image: torch.Tensor, generator: torch.Generator): + dtype = image.dtype + if self.vae.config.force_upcast: + image = image.float() + self.vae.to(dtype=torch.float32) + + if isinstance(generator, list): + image_latents = [ + retrieve_latents(self.vae.encode(image[i : i + 1]), generator=generator[i]) + for i in range(image.shape[0]) + ] + image_latents = torch.cat(image_latents, dim=0) + else: + image_latents = retrieve_latents(self.vae.encode(image), generator=generator) + + if self.vae.config.force_upcast: + self.vae.to(dtype) + + image_latents = image_latents.to(dtype) + image_latents = self.vae.config.scaling_factor * image_latents + + return image_latents + + def prepare_mask_latents( + self, mask, masked_image, batch_size, height, width, dtype, device, generator, do_classifier_free_guidance + ): + # resize the mask to latents shape as we concatenate the mask to the latents + # we do that before converting to dtype to avoid breaking in case we're using cpu_offload + # and half precision + mask = torch.nn.functional.interpolate( + mask, size=(height // self.vae_scale_factor, width // self.vae_scale_factor) + ) + mask = mask.to(device=device, dtype=dtype) + + # duplicate mask and masked_image_latents for each generation per prompt, using mps friendly method + if mask.shape[0] < batch_size: + if not batch_size % mask.shape[0] == 0: + raise ValueError( + "The passed mask and the required batch size don't match. Masks are supposed to be duplicated to" + f" a total batch size of {batch_size}, but {mask.shape[0]} masks were passed. Make sure the number" + " of masks that you pass is divisible by the total requested batch size." + ) + mask = mask.repeat(batch_size // mask.shape[0], 1, 1, 1) + + mask = torch.cat([mask] * 2) if do_classifier_free_guidance else mask + + if masked_image is not None and masked_image.shape[1] == 4: + masked_image_latents = masked_image + else: + masked_image_latents = None + + if masked_image is not None: + if masked_image_latents is None: + masked_image = masked_image.to(device=device, dtype=dtype) + masked_image_latents = self._encode_vae_image(masked_image, generator=generator) + + if masked_image_latents.shape[0] < batch_size: + if not batch_size % masked_image_latents.shape[0] == 0: + raise ValueError( + "The passed images and the required batch size don't match. Images are supposed to be duplicated" + f" to a total batch size of {batch_size}, but {masked_image_latents.shape[0]} images were passed." + " Make sure the number of images that you pass is divisible by the total requested batch size." + ) + masked_image_latents = masked_image_latents.repeat( + batch_size // masked_image_latents.shape[0], 1, 1, 1 + ) + + masked_image_latents = ( + torch.cat([masked_image_latents] * 2) if do_classifier_free_guidance else masked_image_latents + ) + + # aligning device to prevent device errors when concating it with the latent model input + masked_image_latents = masked_image_latents.to(device=device, dtype=dtype) + + return mask, masked_image_latents + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img.StableDiffusionXLImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device, denoising_start=None): + # get the original timestep using init_timestep + if denoising_start is None: + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + t_start = max(num_inference_steps - init_timestep, 0) + else: + t_start = 0 + + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + + # Strength is irrelevant if we directly request a timestep to start at; + # that is, strength is determined by the denoising_start instead. + if denoising_start is not None: + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (denoising_start * self.scheduler.config.num_train_timesteps) + ) + ) + + num_inference_steps = (timesteps < discrete_timestep_cutoff).sum().item() + if self.scheduler.order == 2 and num_inference_steps % 2 == 0: + # if the scheduler is a 2nd order scheduler we might have to do +1 + # because `num_inference_steps` might be even given that every timestep + # (except the highest one) is duplicated. If `num_inference_steps` is even it would + # mean that we cut the timesteps in the middle of the denoising step + # (between 1st and 2nd devirative) which leads to incorrect results. By adding 1 + # we ensure that the denoising process always ends after the 2nd derivate step of the scheduler + num_inference_steps = num_inference_steps + 1 + + # because t_n+1 >= t_n, we slice the timesteps starting from the end + timesteps = timesteps[-num_inference_steps:] + return timesteps, num_inference_steps + + return timesteps, num_inference_steps - t_start + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_img2img.StableDiffusionXLImg2ImgPipeline._get_add_time_ids + def _get_add_time_ids( + self, + original_size, + crops_coords_top_left, + target_size, + aesthetic_score, + negative_aesthetic_score, + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype, + text_encoder_projection_dim=None, + ): + if self.config.requires_aesthetics_score: + add_time_ids = list(original_size + crops_coords_top_left + (aesthetic_score,)) + add_neg_time_ids = list( + negative_original_size + negative_crops_coords_top_left + (negative_aesthetic_score,) + ) + else: + add_time_ids = list(original_size + crops_coords_top_left + target_size) + add_neg_time_ids = list(negative_original_size + crops_coords_top_left + negative_target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if ( + expected_add_embed_dim > passed_add_embed_dim + and (expected_add_embed_dim - passed_add_embed_dim) == self.unet.config.addition_time_embed_dim + ): + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to enable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=True)` to make sure `aesthetic_score` {aesthetic_score} and `negative_aesthetic_score` {negative_aesthetic_score} is correctly used by the model." + ) + elif ( + expected_add_embed_dim < passed_add_embed_dim + and (passed_add_embed_dim - expected_add_embed_dim) == self.unet.config.addition_time_embed_dim + ): + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. Please make sure to disable `requires_aesthetics_score` with `pipe.register_to_config(requires_aesthetics_score=False)` to make sure `target_size` {target_size} is correctly used by the model." + ) + elif expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + add_neg_time_ids = torch.tensor([add_neg_time_ids], dtype=dtype) + + return add_time_ids, add_neg_time_ids + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def guidance_rescale(self): + return self._guidance_rescale + + @property + def clip_skip(self): + return self._clip_skip + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @property + def cross_attention_kwargs(self): + return self._cross_attention_kwargs + + @property + def denoising_end(self): + return self._denoising_end + + @property + def denoising_start(self): + return self._denoising_start + + @property + def num_timesteps(self): + return self._num_timesteps + + @property + def interrupt(self): + return self._interrupt + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + image: PipelineImageInput = None, + mask_image: PipelineImageInput = None, + masked_image_latents: torch.FloatTensor = None, + height: Optional[int] = None, + width: Optional[int] = None, + padding_mask_crop: Optional[int] = None, + strength: float = 0.9999, + num_inference_steps: int = 50, + timesteps: List[int] = None, + denoising_start: Optional[float] = None, + denoising_end: Optional[float] = None, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + original_size: Tuple[int, int] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Tuple[int, int] = None, + negative_original_size: Optional[Tuple[int, int]] = None, + negative_crops_coords_top_left: Tuple[int, int] = (0, 0), + negative_target_size: Optional[Tuple[int, int]] = None, + aesthetic_score: float = 6.0, + negative_aesthetic_score: float = 2.5, + clip_skip: Optional[int] = None, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + image (`PIL.Image.Image`): + `Image`, or tensor representing an image batch which will be inpainted, *i.e.* parts of the image will + be masked out with `mask_image` and repainted according to `prompt`. + mask_image (`PIL.Image.Image`): + `Image`, or tensor representing an image batch, to mask `image`. White pixels in the mask will be + repainted, while black pixels will be preserved. If `mask_image` is a PIL image, it will be converted + to a single channel (luminance) before use. If it's a tensor, it should contain one color channel (L) + instead of 3, so the expected shape would be `(B, H, W, 1)`. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. This is set to 1024 by default for the best results. + Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + padding_mask_crop (`int`, *optional*, defaults to `None`): + The size of margin in the crop to be applied to the image and masking. If `None`, no crop is applied to image and mask_image. If + `padding_mask_crop` is not `None`, it will first find a rectangular region with the same aspect ration of the image and + contains all masked area, and then expand that area based on `padding_mask_crop`. The image and mask_image will then be cropped based on + the expanded area before resizing to the original image size for inpainting. This is useful when the masked area is small while the image is large + and contain information inreleant for inpainging, such as background. + strength (`float`, *optional*, defaults to 0.9999): + Conceptually, indicates how much to transform the masked portion of the reference `image`. Must be + between 0 and 1. `image` will be used as a starting point, adding more noise to it the larger the + `strength`. The number of denoising steps depends on the amount of noise initially added. When + `strength` is 1, added noise will be maximum and the denoising process will run for the full number of + iterations specified in `num_inference_steps`. A value of 1, therefore, essentially ignores the masked + portion of the reference `image`. Note that in the case of `denoising_start` being declared as an + integer, the value of `strength` will be ignored. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + denoising_start (`float`, *optional*): + When specified, indicates the fraction (between 0.0 and 1.0) of the total denoising process to be + bypassed before it is initiated. Consequently, the initial part of the denoising process is skipped and + it is assumed that the passed `image` is a partly denoised image. Note that when this is specified, + strength will be ignored. The `denoising_start` parameter is particularly beneficial when this pipeline + is integrated into a "Mixture of Denoisers" multi-pipeline setup, as detailed in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output). + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise (ca. final 20% of timesteps still needed) and should be + denoised by a successor pipeline that has `denoising_start` set to 0.8 so that it only denoises the + final 20% of the scheduler. The denoising_end parameter should ideally be utilized when this pipeline + forms a part of a "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output). + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a specific image resolution. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a target image resolution. It should be as same + as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + aesthetic_score (`float`, *optional*, defaults to 6.0): + Used to simulate an aesthetic score of the generated image by influencing the positive text condition. + Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_aesthetic_score (`float`, *optional*, defaults to 2.5): + Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). Can be used to + simulate an aesthetic score of the generated image by influencing the negative text condition. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a + `tuple. `tuple. When returning a tuple, the first element is a list with the generated images. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # 1. Check inputs + self.check_inputs( + prompt, + prompt_2, + image, + mask_image, + height, + width, + strength, + callback_steps, + output_type, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + callback_on_step_end_tensor_inputs, + padding_mask_crop, + ) + + self._guidance_scale = guidance_scale + self._guidance_rescale = guidance_rescale + self._clip_skip = clip_skip + self._cross_attention_kwargs = cross_attention_kwargs + self._denoising_end = denoising_end + self._denoising_start = denoising_start + self._interrupt = False + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3. Encode input prompt + text_encoder_lora_scale = ( + self.cross_attention_kwargs.get("scale", None) if self.cross_attention_kwargs is not None else None + ) + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=self.clip_skip, + ) + + # 4. set timesteps + def denoising_value_valid(dnv): + return isinstance(dnv, float) and 0 < dnv < 1 + + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + timesteps, num_inference_steps = self.get_timesteps( + num_inference_steps, + strength, + device, + denoising_start=self.denoising_start if denoising_value_valid(self.denoising_start) else None, + ) + # check that number of inference steps is not < 1 - as this doesn't make sense + if num_inference_steps < 1: + raise ValueError( + f"After adjusting the num_inference_steps by strength parameter: {strength}, the number of pipeline" + f"steps is {num_inference_steps} which is < 1 and not appropriate for this pipeline." + ) + # at which timestep to set the initial noise (n.b. 50% if strength is 0.5) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + # create a boolean to check if the strength is set to 1. if so then initialise the latents with pure noise + is_strength_max = strength == 1.0 + + # 5. Preprocess mask and image + if padding_mask_crop is not None: + crops_coords = self.mask_processor.get_crop_region(mask_image, width, height, pad=padding_mask_crop) + resize_mode = "fill" + else: + crops_coords = None + resize_mode = "default" + + original_image = image + init_image = self.image_processor.preprocess( + image, height=height, width=width, crops_coords=crops_coords, resize_mode=resize_mode + ) + init_image = init_image.to(dtype=torch.float32) + + mask = self.mask_processor.preprocess( + mask_image, height=height, width=width, resize_mode=resize_mode, crops_coords=crops_coords + ) + + if masked_image_latents is not None: + masked_image = masked_image_latents + elif init_image.shape[1] == 4: + # if images are in latent space, we can't mask it + masked_image = None + else: + masked_image = init_image * (mask < 0.5) + + # 6. Prepare latent variables + num_channels_latents = self.vae.config.latent_channels + num_channels_unet = self.unet.config.in_channels + return_image_latents = num_channels_unet == 4 + + add_noise = True if self.denoising_start is None else False + latents_outputs = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + image=init_image, + timestep=latent_timestep, + is_strength_max=is_strength_max, + add_noise=add_noise, + return_noise=True, + return_image_latents=return_image_latents, + ) + + if return_image_latents: + latents, noise, image_latents = latents_outputs + else: + latents, noise = latents_outputs + + # 7. Prepare mask latent variables + mask, masked_image_latents = self.prepare_mask_latents( + mask, + masked_image, + batch_size * num_images_per_prompt, + height, + width, + prompt_embeds.dtype, + device, + generator, + self.do_classifier_free_guidance, + ) + + # 8. Check that sizes of mask, masked image and latents match + if num_channels_unet == 9: + # default case for runwayml/stable-diffusion-inpainting + num_channels_mask = mask.shape[1] + num_channels_masked_image = masked_image_latents.shape[1] + if num_channels_latents + num_channels_mask + num_channels_masked_image != self.unet.config.in_channels: + raise ValueError( + f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects" + f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +" + f" `num_channels_mask`: {num_channels_mask} + `num_channels_masked_image`: {num_channels_masked_image}" + f" = {num_channels_latents+num_channels_masked_image+num_channels_mask}. Please verify the config of" + " `pipeline.unet` or your `mask_image` or `image` input." + ) + elif num_channels_unet != 4: + raise ValueError( + f"The unet {self.unet.__class__} should have either 4 or 9 input channels, not {self.unet.config.in_channels}." + ) + # 8.1 Prepare extra step kwargs. + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + height, width = latents.shape[-2:] + height = height * self.vae_scale_factor + width = width * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 10. Prepare added time ids & embeddings + if negative_original_size is None: + negative_original_size = original_size + if negative_target_size is None: + negative_target_size = target_size + + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids, add_neg_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + aesthetic_score, + negative_aesthetic_score, + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + add_time_ids = add_time_ids.repeat(batch_size * num_images_per_prompt, 1) + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_neg_time_ids = add_neg_time_ids.repeat(batch_size * num_images_per_prompt, 1) + add_time_ids = torch.cat([add_neg_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device) + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 11. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + + if ( + self.denoising_end is not None + and self.denoising_start is not None + and denoising_value_valid(self.denoising_end) + and denoising_value_valid(self.denoising_start) + and self.denoising_start >= self.denoising_end + ): + raise ValueError( + f"`denoising_start`: {self.denoising_start} cannot be larger than or equal to `denoising_end`: " + + f" {self.denoising_end} when using type float." + ) + elif self.denoising_end is not None and denoising_value_valid(self.denoising_end): + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (self.denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + # 11.1 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + if self.interrupt: + continue + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + # concat latents, mask, masked_image_latents in the channel dimension + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + if num_channels_unet == 9: + latent_model_input = torch.cat([latent_model_input, mask, masked_image_latents], dim=1) + + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + added_cond_kwargs["image_embeds"] = image_embeds + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=self.cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_text - noise_pred_uncond) + + if self.do_classifier_free_guidance and self.guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=self.guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + if num_channels_unet == 4: + init_latents_proper = image_latents + if self.do_classifier_free_guidance: + init_mask, _ = mask.chunk(2) + else: + init_mask = mask + + if i < len(timesteps) - 1: + noise_timestep = timesteps[i + 1] + init_latents_proper = self.scheduler.add_noise( + init_latents_proper, noise, torch.tensor([noise_timestep]) + ) + + latents = (1 - init_mask) * init_latents_proper + init_mask * latents + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + prompt_embeds = callback_outputs.pop("prompt_embeds", prompt_embeds) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + add_text_embeds = callback_outputs.pop("add_text_embeds", add_text_embeds) + negative_pooled_prompt_embeds = callback_outputs.pop( + "negative_pooled_prompt_embeds", negative_pooled_prompt_embeds + ) + add_time_ids = callback_outputs.pop("add_time_ids", add_time_ids) + add_neg_time_ids = callback_outputs.pop("add_neg_time_ids", add_neg_time_ids) + mask = callback_outputs.pop("mask", mask) + masked_image_latents = callback_outputs.pop("masked_image_latents", masked_image_latents) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if XLA_AVAILABLE: + xm.mark_step() + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + # unscale/denormalize the latents + # denormalize with the mean and std if available and not None + has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None + has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None + if has_latents_mean and has_latents_std: + latents_mean = ( + torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents_std = ( + torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean + else: + latents = latents / self.vae.config.scaling_factor + + image = self.vae.decode(latents, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + return StableDiffusionXLPipelineOutput(images=latents) + + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + if padding_mask_crop is not None: + image = [self.image_processor.apply_overlay(mask_image, original_image, i, crops_coords) for i in image] + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return StableDiffusionXLPipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py new file mode 100755 index 0000000..51e413d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/pipeline_stable_diffusion_xl_instruct_pix2pix.py @@ -0,0 +1,976 @@ +# Copyright 2024 Harutatsu Akiyama and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import PIL.Image +import torch +from transformers import CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import FromSingleFileMixin, StableDiffusionXLLoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.attention_processor import ( + AttnProcessor2_0, + FusedAttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + is_invisible_watermark_available, + is_torch_xla_available, + logging, + replace_example_docstring, + scale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from .pipeline_output import StableDiffusionXLPipelineOutput + + +if is_invisible_watermark_available(): + from .watermark import StableDiffusionXLWatermarker + +if is_torch_xla_available(): + import torch_xla.core.xla_model as xm + + XLA_AVAILABLE = True +else: + XLA_AVAILABLE = False + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import StableDiffusionXLInstructPix2PixPipeline + >>> from diffusers.utils import load_image + + >>> resolution = 768 + >>> image = load_image( + ... "https://hf.co/datasets/diffusers/diffusers-images-docs/resolve/main/mountain.png" + ... ).resize((resolution, resolution)) + >>> edit_instruction = "Turn sky into a cloudy one" + + >>> pipe = StableDiffusionXLInstructPix2PixPipeline.from_pretrained( + ... "diffusers/sdxl-instructpix2pix-768", torch_dtype=torch.float16 + ... ).to("cuda") + + >>> edited_image = pipe( + ... prompt=edit_instruction, + ... image=image, + ... height=resolution, + ... width=resolution, + ... guidance_scale=3.0, + ... image_guidance_scale=1.5, + ... num_inference_steps=30, + ... ).images[0] + >>> edited_image + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class StableDiffusionXLInstructPix2PixPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + FromSingleFileMixin, + StableDiffusionXLLoraLoaderMixin, +): + r""" + Pipeline for pixel-level image editing by following text instructions. Based on Stable Diffusion XL. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + requires_aesthetics_score (`bool`, *optional*, defaults to `"False"`): + Whether the `unet` requires a aesthetic_score condition to be passed during inference. Also see the config + of `stabilityai/stable-diffusion-xl-refiner-1-0`. + force_zeros_for_empty_prompt (`bool`, *optional*, defaults to `"True"`): + Whether the negative prompt embeddings shall be forced to always be set to 0. Also see the config of + `stabilityai/stable-diffusion-xl-base-1-0`. + add_watermarker (`bool`, *optional*): + Whether to use the [invisible_watermark library](https://github.com/ShieldMnt/invisible-watermark/) to + watermark output images. If not defined, it will default to True if the package is installed, otherwise no + watermarker will be used. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->unet->vae" + _optional_components = ["tokenizer", "tokenizer_2", "text_encoder", "text_encoder_2"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + scheduler=scheduler, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.default_sample_size = self.unet.config.sample_size + + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() + + if add_watermarker: + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None + + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + # textual inversion: process multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder( + text_input_ids.to(device), + output_hidden_states=True, + ) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + prompt_embeds = prompt_embeds.hidden_states[-2] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt, negative_prompt_2] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + prompt_embeds_dtype = self.text_encoder_2.dtype if self.text_encoder_2 is not None else self.unet.dtype + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_instruct_pix2pix.StableDiffusionInstructPix2PixPipeline.check_inputs + def check_inputs( + self, + prompt, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_image_latents( + self, image, batch_size, num_images_per_prompt, dtype, device, do_classifier_free_guidance, generator=None + ): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_images_per_prompt + + if image.shape[1] == 4: + image_latents = image + else: + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + if needs_upcasting: + self.upcast_vae() + image = image.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + image_latents = retrieve_latents(self.vae.encode(image), sample_mode="argmax") + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + + if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0: + # expand image_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {image_latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_image_per_prompt = batch_size // image_latents.shape[0] + image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0) + elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts." + ) + else: + image_latents = torch.cat([image_latents], dim=0) + + if do_classifier_free_guidance: + uncond_image_latents = torch.zeros_like(image_latents) + image_latents = torch.cat([image_latents, image_latents, uncond_image_latents], dim=0) + + if image_latents.dtype != self.vae.dtype: + image_latents = image_latents.to(dtype=self.vae.dtype) + + return image_latents + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._get_add_time_ids + def _get_add_time_ids( + self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None + ): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + return add_time_ids + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + FusedAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + image: PipelineImageInput = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 100, + denoising_end: Optional[float] = None, + guidance_scale: float = 5.0, + image_guidance_scale: float = 1.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + original_size: Tuple[int, int] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Tuple[int, int] = None, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + image (`torch.FloatTensor` or `PIL.Image.Image` or `np.ndarray` or `List[torch.FloatTensor]` or `List[PIL.Image.Image]` or `List[np.ndarray]`): + The image(s) to modify with the pipeline. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise as determined by the discrete timesteps selected by the + scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a + "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) + guidance_scale (`float`, *optional*, defaults to 5.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + image_guidance_scale (`float`, *optional*, defaults to 1.5): + Image guidance scale is to push the generated image towards the inital image `image`. Image guidance + scale is enabled by setting `image_guidance_scale > 1`. Higher image guidance scale encourages to + generate images that are closely linked to the source image `image`, usually at the expense of lower + image quality. This pipeline requires a value of at least `1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionXLPipelineOutput`] instead of a + plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.0): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + aesthetic_score (`float`, *optional*, defaults to 6.0): + Used to simulate an aesthetic score of the generated image by influencing the positive text condition. + Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_aesthetic_score (`float`, *optional*, defaults to 2.5): + Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). Can be used to + simulate an aesthetic score of the generated image by influencing the negative text condition. + + Examples: + + Returns: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + # 0. Default height and width to unet + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 1. Check inputs. Raise error if not correct + self.check_inputs(prompt, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds) + + if image is None: + raise ValueError("`image` input cannot be undefined.") + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 and image_guidance_scale >= 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=text_encoder_lora_scale, + ) + + # 4. Preprocess image + image = self.image_processor.preprocess(image, height=height, width=width).to(device) + + # 5. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 6. Prepare Image latents + image_latents = self.prepare_image_latents( + image, + batch_size, + num_images_per_prompt, + prompt_embeds.dtype, + device, + do_classifier_free_guidance, + ) + + # 7. Prepare latent variables + num_channels_latents = self.vae.config.latent_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 8. Check that shapes of latents and image match the UNet channels + num_channels_image = image_latents.shape[1] + if num_channels_latents + num_channels_image != self.unet.config.in_channels: + raise ValueError( + f"Incorrect configuration settings! The config of `pipeline.unet`: {self.unet.config} expects" + f" {self.unet.config.in_channels} but received `num_channels_latents`: {num_channels_latents} +" + f" `num_channels_image`: {num_channels_image} " + f" = {num_channels_latents + num_channels_image}. Please verify the config of" + " `pipeline.unet` or your `image` input." + ) + + # 9. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 10. Prepare added time ids & embeddings + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + + if do_classifier_free_guidance: + # The extra concat similar to how it's done in SD InstructPix2Pix. + prompt_embeds = torch.cat([prompt_embeds, negative_prompt_embeds, negative_prompt_embeds], dim=0) + add_text_embeds = torch.cat( + [add_text_embeds, negative_pooled_prompt_embeds, negative_pooled_prompt_embeds], dim=0 + ) + add_time_ids = torch.cat([add_time_ids, add_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1) + + # 11. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + if denoising_end is not None and isinstance(denoising_end, float) and denoising_end > 0 and denoising_end < 1: + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # Expand the latents if we are doing classifier free guidance. + # The latents are expanded 3 times because for pix2pix the guidance + # is applied for both the text and the input image. + latent_model_input = torch.cat([latents] * 3) if do_classifier_free_guidance else latents + + # concat latents, image_latents in the channel dimension + scaled_latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + scaled_latent_model_input = torch.cat([scaled_latent_model_input, image_latents], dim=1) + + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + noise_pred = self.unet( + scaled_latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_text, noise_pred_image, noise_pred_uncond = noise_pred.chunk(3) + noise_pred = ( + noise_pred_uncond + + guidance_scale * (noise_pred_text - noise_pred_image) + + image_guidance_scale * (noise_pred_image - noise_pred_uncond) + ) + + if do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if XLA_AVAILABLE: + xm.mark_step() + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + # unscale/denormalize the latents + # denormalize with the mean and std if available and not None + has_latents_mean = hasattr(self.vae.config, "latents_mean") and self.vae.config.latents_mean is not None + has_latents_std = hasattr(self.vae.config, "latents_std") and self.vae.config.latents_std is not None + if has_latents_mean and has_latents_std: + latents_mean = ( + torch.tensor(self.vae.config.latents_mean).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents_std = ( + torch.tensor(self.vae.config.latents_std).view(1, 4, 1, 1).to(latents.device, latents.dtype) + ) + latents = latents * latents_std / self.vae.config.scaling_factor + latents_mean + else: + latents = latents / self.vae.config.scaling_factor + + image = self.vae.decode(latents, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + return StableDiffusionXLPipelineOutput(images=latents) + + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return StableDiffusionXLPipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/watermark.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/watermark.py new file mode 100755 index 0000000..5b6e36d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_diffusion_xl/watermark.py @@ -0,0 +1,36 @@ +import numpy as np +import torch + +from ...utils import is_invisible_watermark_available + + +if is_invisible_watermark_available(): + from imwatermark import WatermarkEncoder + + +# Copied from https://github.com/Stability-AI/generative-models/blob/613af104c6b85184091d42d374fef420eddb356d/scripts/demo/streamlit_helpers.py#L66 +WATERMARK_MESSAGE = 0b101100111110110010010000011110111011000110011110 +# bin(x)[2:] gives bits of x as str, use int to convert them to 0/1 +WATERMARK_BITS = [int(bit) for bit in bin(WATERMARK_MESSAGE)[2:]] + + +class StableDiffusionXLWatermarker: + def __init__(self): + self.watermark = WATERMARK_BITS + self.encoder = WatermarkEncoder() + + self.encoder.set_watermark("bits", self.watermark) + + def apply_watermark(self, images: torch.FloatTensor): + # can't encode images that are smaller than 256 + if images.shape[-1] < 256: + return images + + images = (255 * (images / 2 + 0.5)).cpu().permute(0, 2, 3, 1).float().numpy() + + images = [self.encoder.encode(image, "dwtDct") for image in images] + + images = torch.from_numpy(np.array(images)).permute(0, 3, 1, 2) + + images = torch.clamp(2 * (images / 255 - 0.5), min=-1.0, max=1.0) + return images diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_video_diffusion/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_video_diffusion/__init__.py new file mode 100755 index 0000000..3bd4dc7 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_video_diffusion/__init__.py @@ -0,0 +1,58 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + BaseOutput, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure.update( + { + "pipeline_stable_video_diffusion": [ + "StableVideoDiffusionPipeline", + "StableVideoDiffusionPipelineOutput", + ], + } + ) + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * + else: + from .pipeline_stable_video_diffusion import ( + StableVideoDiffusionPipeline, + StableVideoDiffusionPipelineOutput, + ) + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py b/diffusers-0.27.0/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py new file mode 100755 index 0000000..1342fe4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/stable_video_diffusion/pipeline_stable_video_diffusion.py @@ -0,0 +1,673 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPImageProcessor, CLIPVisionModelWithProjection + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...models import AutoencoderKLTemporalDecoder, UNetSpatioTemporalConditionModel +from ...schedulers import EulerDiscreteScheduler +from ...utils import BaseOutput, logging, replace_example_docstring +from ...utils.torch_utils import is_compiled_module, randn_tensor +from ..pipeline_utils import DiffusionPipeline + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusers import StableVideoDiffusionPipeline + >>> from diffusers.utils import load_image, export_to_video + + >>> pipe = StableVideoDiffusionPipeline.from_pretrained("stabilityai/stable-video-diffusion-img2vid-xt", torch_dtype=torch.float16, variant="fp16") + >>> pipe.to("cuda") + + >>> image = load_image("https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/svd-docstring-example.jpeg") + >>> image = image.resize((1024, 576)) + + >>> frames = pipe(image, num_frames=25, decode_chunk_size=8).frames[0] + >>> export_to_video(frames, "generated.mp4", fps=7) + ``` +""" + + +def _append_dims(x, target_dims): + """Appends dimensions to the end of a tensor until it has target_dims dimensions.""" + dims_to_append = target_dims - x.ndim + if dims_to_append < 0: + raise ValueError(f"input has {x.ndim} dims but target_dims is {target_dims}, which is less") + return x[(...,) + (None,) * dims_to_append] + + +# Copied from diffusers.pipelines.animatediff.pipeline_animatediff.tensor2vid +def tensor2vid(video: torch.Tensor, processor: VaeImageProcessor, output_type: str = "np"): + batch_size, channels, num_frames, height, width = video.shape + outputs = [] + for batch_idx in range(batch_size): + batch_vid = video[batch_idx].permute(1, 0, 2, 3) + batch_output = processor.postprocess(batch_vid, output_type) + + outputs.append(batch_output) + + if output_type == "np": + outputs = np.stack(outputs) + + elif output_type == "pt": + outputs = torch.stack(outputs) + + elif not output_type == "pil": + raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil']") + + return outputs + + +@dataclass +class StableVideoDiffusionPipelineOutput(BaseOutput): + r""" + Output class for Stable Video Diffusion pipeline. + + Args: + frames (`[List[List[PIL.Image.Image]]`, `np.ndarray`, `torch.FloatTensor`]): + List of denoised PIL images of length `batch_size` or numpy array or torch tensor + of shape `(batch_size, num_frames, height, width, num_channels)`. + """ + + frames: Union[List[List[PIL.Image.Image]], np.ndarray, torch.FloatTensor] + + +class StableVideoDiffusionPipeline(DiffusionPipeline): + r""" + Pipeline to generate video from an input image using Stable Video Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKLTemporalDecoder`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. + image_encoder ([`~transformers.CLIPVisionModelWithProjection`]): + Frozen CLIP image-encoder ([laion/CLIP-ViT-H-14-laion2B-s32B-b79K](https://huggingface.co/laion/CLIP-ViT-H-14-laion2B-s32B-b79K)). + unet ([`UNetSpatioTemporalConditionModel`]): + A `UNetSpatioTemporalConditionModel` to denoise the encoded image latents. + scheduler ([`EulerDiscreteScheduler`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + A `CLIPImageProcessor` to extract features from generated images. + """ + + model_cpu_offload_seq = "image_encoder->unet->vae" + _callback_tensor_inputs = ["latents"] + + def __init__( + self, + vae: AutoencoderKLTemporalDecoder, + image_encoder: CLIPVisionModelWithProjection, + unet: UNetSpatioTemporalConditionModel, + scheduler: EulerDiscreteScheduler, + feature_extractor: CLIPImageProcessor, + ): + super().__init__() + + self.register_modules( + vae=vae, + image_encoder=image_encoder, + unet=unet, + scheduler=scheduler, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + def _encode_image( + self, + image: PipelineImageInput, + device: Union[str, torch.device], + num_videos_per_prompt: int, + do_classifier_free_guidance: bool, + ) -> torch.FloatTensor: + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.image_processor.pil_to_numpy(image) + image = self.image_processor.numpy_to_pt(image) + + # We normalize the image before resizing to match with the original implementation. + # Then we unnormalize it after resizing. + image = image * 2.0 - 1.0 + image = _resize_with_antialiasing(image, (224, 224)) + image = (image + 1.0) / 2.0 + + # Normalize the image with for CLIP input + image = self.feature_extractor( + images=image, + do_normalize=True, + do_center_crop=False, + do_resize=False, + do_rescale=False, + return_tensors="pt", + ).pixel_values + + image = image.to(device=device, dtype=dtype) + image_embeddings = self.image_encoder(image).image_embeds + image_embeddings = image_embeddings.unsqueeze(1) + + # duplicate image embeddings for each generation per prompt, using mps friendly method + bs_embed, seq_len, _ = image_embeddings.shape + image_embeddings = image_embeddings.repeat(1, num_videos_per_prompt, 1) + image_embeddings = image_embeddings.view(bs_embed * num_videos_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + negative_image_embeddings = torch.zeros_like(image_embeddings) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + image_embeddings = torch.cat([negative_image_embeddings, image_embeddings]) + + return image_embeddings + + def _encode_vae_image( + self, + image: torch.Tensor, + device: Union[str, torch.device], + num_videos_per_prompt: int, + do_classifier_free_guidance: bool, + ): + image = image.to(device=device) + image_latents = self.vae.encode(image).latent_dist.mode() + + if do_classifier_free_guidance: + negative_image_latents = torch.zeros_like(image_latents) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + image_latents = torch.cat([negative_image_latents, image_latents]) + + # duplicate image_latents for each generation per prompt, using mps friendly method + image_latents = image_latents.repeat(num_videos_per_prompt, 1, 1, 1) + + return image_latents + + def _get_add_time_ids( + self, + fps: int, + motion_bucket_id: int, + noise_aug_strength: float, + dtype: torch.dtype, + batch_size: int, + num_videos_per_prompt: int, + do_classifier_free_guidance: bool, + ): + add_time_ids = [fps, motion_bucket_id, noise_aug_strength] + + passed_add_embed_dim = self.unet.config.addition_time_embed_dim * len(add_time_ids) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + add_time_ids = add_time_ids.repeat(batch_size * num_videos_per_prompt, 1) + + if do_classifier_free_guidance: + add_time_ids = torch.cat([add_time_ids, add_time_ids]) + + return add_time_ids + + def decode_latents(self, latents: torch.FloatTensor, num_frames: int, decode_chunk_size: int = 14): + # [batch, frames, channels, height, width] -> [batch*frames, channels, height, width] + latents = latents.flatten(0, 1) + + latents = 1 / self.vae.config.scaling_factor * latents + + forward_vae_fn = self.vae._orig_mod.forward if is_compiled_module(self.vae) else self.vae.forward + accepts_num_frames = "num_frames" in set(inspect.signature(forward_vae_fn).parameters.keys()) + + # decode decode_chunk_size frames at a time to avoid OOM + frames = [] + for i in range(0, latents.shape[0], decode_chunk_size): + num_frames_in = latents[i : i + decode_chunk_size].shape[0] + decode_kwargs = {} + if accepts_num_frames: + # we only pass num_frames_in if it's expected + decode_kwargs["num_frames"] = num_frames_in + + frame = self.vae.decode(latents[i : i + decode_chunk_size], **decode_kwargs).sample + frames.append(frame) + frames = torch.cat(frames, dim=0) + + # [batch*frames, channels, height, width] -> [batch, channels, frames, height, width] + frames = frames.reshape(-1, num_frames, *frames.shape[1:]).permute(0, 2, 1, 3, 4) + + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + frames = frames.float() + return frames + + def check_inputs(self, image, height, width): + if ( + not isinstance(image, torch.Tensor) + and not isinstance(image, PIL.Image.Image) + and not isinstance(image, list) + ): + raise ValueError( + "`image` has to be of type `torch.FloatTensor` or `PIL.Image.Image` or `List[PIL.Image.Image]` but is" + f" {type(image)}" + ) + + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + def prepare_latents( + self, + batch_size: int, + num_frames: int, + num_channels_latents: int, + height: int, + width: int, + dtype: torch.dtype, + device: Union[str, torch.device], + generator: torch.Generator, + latents: Optional[torch.FloatTensor] = None, + ): + shape = ( + batch_size, + num_frames, + num_channels_latents // 2, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @property + def guidance_scale(self): + return self._guidance_scale + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + if isinstance(self.guidance_scale, (int, float)): + return self.guidance_scale > 1 + return self.guidance_scale.max() > 1 + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image: Union[PIL.Image.Image, List[PIL.Image.Image], torch.FloatTensor], + height: int = 576, + width: int = 1024, + num_frames: Optional[int] = None, + num_inference_steps: int = 25, + min_guidance_scale: float = 1.0, + max_guidance_scale: float = 3.0, + fps: int = 7, + motion_bucket_id: int = 127, + noise_aug_strength: float = 0.02, + decode_chunk_size: Optional[int] = None, + num_videos_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + return_dict: bool = True, + ): + r""" + The call function to the pipeline for generation. + + Args: + image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.FloatTensor`): + Image(s) to guide image generation. If you provide a tensor, the expected value range is between `[0, 1]`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_frames (`int`, *optional*): + The number of video frames to generate. Defaults to `self.unet.config.num_frames` + (14 for `stable-video-diffusion-img2vid` and to 25 for `stable-video-diffusion-img2vid-xt`). + num_inference_steps (`int`, *optional*, defaults to 25): + The number of denoising steps. More denoising steps usually lead to a higher quality video at the + expense of slower inference. This parameter is modulated by `strength`. + min_guidance_scale (`float`, *optional*, defaults to 1.0): + The minimum guidance scale. Used for the classifier free guidance with first frame. + max_guidance_scale (`float`, *optional*, defaults to 3.0): + The maximum guidance scale. Used for the classifier free guidance with last frame. + fps (`int`, *optional*, defaults to 7): + Frames per second. The rate at which the generated images shall be exported to a video after generation. + Note that Stable Diffusion Video's UNet was micro-conditioned on fps-1 during training. + motion_bucket_id (`int`, *optional*, defaults to 127): + Used for conditioning the amount of motion for the generation. The higher the number the more motion + will be in the video. + noise_aug_strength (`float`, *optional*, defaults to 0.02): + The amount of noise added to the init image, the higher it is the less the video will look like the init image. Increase it for more motion. + decode_chunk_size (`int`, *optional*): + The number of frames to decode at a time. Higher chunk size leads to better temporal consistency at the expense of more memory usage. By default, the decoder decodes all frames at once for maximal + quality. For lower memory usage, reduce `decode_chunk_size`. + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of videos to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `pil`, `np` or `pt`. + callback_on_step_end (`Callable`, *optional*): + A function that is called at the end of each denoising step during inference. The function is called + with the following arguments: + `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, callback_kwargs: Dict)`. + `callback_kwargs` will include a list of all tensors as specified by `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionPipelineOutput`] instead of a + plain tuple. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableVideoDiffusionPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.stable_diffusion.StableVideoDiffusionPipelineOutput`] is returned, + otherwise a `tuple` of (`List[List[PIL.Image.Image]]` or `np.ndarray` or `torch.FloatTensor`) is returned. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + num_frames = num_frames if num_frames is not None else self.unet.config.num_frames + decode_chunk_size = decode_chunk_size if decode_chunk_size is not None else num_frames + + # 1. Check inputs. Raise error if not correct + self.check_inputs(image, height, width) + + # 2. Define call parameters + if isinstance(image, PIL.Image.Image): + batch_size = 1 + elif isinstance(image, list): + batch_size = len(image) + else: + batch_size = image.shape[0] + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + self._guidance_scale = max_guidance_scale + + # 3. Encode input image + image_embeddings = self._encode_image(image, device, num_videos_per_prompt, self.do_classifier_free_guidance) + + # NOTE: Stable Video Diffusion was conditioned on fps - 1, which is why it is reduced here. + # See: https://github.com/Stability-AI/generative-models/blob/ed0997173f98eaf8f4edf7ba5fe8f15c6b877fd3/scripts/sampling/simple_video_sample.py#L188 + fps = fps - 1 + + # 4. Encode input image using VAE + image = self.image_processor.preprocess(image, height=height, width=width).to(device) + noise = randn_tensor(image.shape, generator=generator, device=device, dtype=image.dtype) + image = image + noise_aug_strength * noise + + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + if needs_upcasting: + self.vae.to(dtype=torch.float32) + + image_latents = self._encode_vae_image( + image, + device=device, + num_videos_per_prompt=num_videos_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + ) + image_latents = image_latents.to(image_embeddings.dtype) + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + + # Repeat the image latents for each frame so we can concatenate them with the noise + # image_latents [batch, channels, height, width] ->[batch, num_frames, channels, height, width] + image_latents = image_latents.unsqueeze(1).repeat(1, num_frames, 1, 1, 1) + + # 5. Get Added Time IDs + added_time_ids = self._get_add_time_ids( + fps, + motion_bucket_id, + noise_aug_strength, + image_embeddings.dtype, + batch_size, + num_videos_per_prompt, + self.do_classifier_free_guidance, + ) + added_time_ids = added_time_ids.to(device) + + # 6. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 7. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + num_frames, + num_channels_latents, + height, + width, + image_embeddings.dtype, + device, + generator, + latents, + ) + + # 8. Prepare guidance scale + guidance_scale = torch.linspace(min_guidance_scale, max_guidance_scale, num_frames).unsqueeze(0) + guidance_scale = guidance_scale.to(device, latents.dtype) + guidance_scale = guidance_scale.repeat(batch_size * num_videos_per_prompt, 1) + guidance_scale = _append_dims(guidance_scale, latents.ndim) + + self._guidance_scale = guidance_scale + + # 9. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + self._num_timesteps = len(timesteps) + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # Concatenate image_latents over channels dimension + latent_model_input = torch.cat([latent_model_input, image_latents], dim=2) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=image_embeddings, + added_time_ids=added_time_ids, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_cond = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + self.guidance_scale * (noise_pred_cond - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents).prev_sample + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + + if not output_type == "latent": + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + frames = self.decode_latents(latents, num_frames, decode_chunk_size) + frames = tensor2vid(frames, self.image_processor, output_type=output_type) + else: + frames = latents + + self.maybe_free_model_hooks() + + if not return_dict: + return frames + + return StableVideoDiffusionPipelineOutput(frames=frames) + + +# resizing utils +# TODO: clean up later +def _resize_with_antialiasing(input, size, interpolation="bicubic", align_corners=True): + h, w = input.shape[-2:] + factors = (h / size[0], w / size[1]) + + # First, we have to determine sigma + # Taken from skimage: https://github.com/scikit-image/scikit-image/blob/v0.19.2/skimage/transform/_warps.py#L171 + sigmas = ( + max((factors[0] - 1.0) / 2.0, 0.001), + max((factors[1] - 1.0) / 2.0, 0.001), + ) + + # Now kernel size. Good results are for 3 sigma, but that is kind of slow. Pillow uses 1 sigma + # https://github.com/python-pillow/Pillow/blob/master/src/libImaging/Resample.c#L206 + # But they do it in the 2 passes, which gives better results. Let's try 2 sigmas for now + ks = int(max(2.0 * 2 * sigmas[0], 3)), int(max(2.0 * 2 * sigmas[1], 3)) + + # Make sure it is odd + if (ks[0] % 2) == 0: + ks = ks[0] + 1, ks[1] + + if (ks[1] % 2) == 0: + ks = ks[0], ks[1] + 1 + + input = _gaussian_blur2d(input, ks, sigmas) + + output = torch.nn.functional.interpolate(input, size=size, mode=interpolation, align_corners=align_corners) + return output + + +def _compute_padding(kernel_size): + """Compute padding tuple.""" + # 4 or 6 ints: (padding_left, padding_right,padding_top,padding_bottom) + # https://pytorch.org/docs/stable/nn.html#torch.nn.functional.pad + if len(kernel_size) < 2: + raise AssertionError(kernel_size) + computed = [k - 1 for k in kernel_size] + + # for even kernels we need to do asymmetric padding :( + out_padding = 2 * len(kernel_size) * [0] + + for i in range(len(kernel_size)): + computed_tmp = computed[-(i + 1)] + + pad_front = computed_tmp // 2 + pad_rear = computed_tmp - pad_front + + out_padding[2 * i + 0] = pad_front + out_padding[2 * i + 1] = pad_rear + + return out_padding + + +def _filter2d(input, kernel): + # prepare kernel + b, c, h, w = input.shape + tmp_kernel = kernel[:, None, ...].to(device=input.device, dtype=input.dtype) + + tmp_kernel = tmp_kernel.expand(-1, c, -1, -1) + + height, width = tmp_kernel.shape[-2:] + + padding_shape: list[int] = _compute_padding([height, width]) + input = torch.nn.functional.pad(input, padding_shape, mode="reflect") + + # kernel and input tensor reshape to align element-wise or batch-wise params + tmp_kernel = tmp_kernel.reshape(-1, 1, height, width) + input = input.view(-1, tmp_kernel.size(0), input.size(-2), input.size(-1)) + + # convolve the tensor with the kernel. + output = torch.nn.functional.conv2d(input, tmp_kernel, groups=tmp_kernel.size(0), padding=0, stride=1) + + out = output.view(b, c, h, w) + return out + + +def _gaussian(window_size: int, sigma): + if isinstance(sigma, float): + sigma = torch.tensor([[sigma]]) + + batch_size = sigma.shape[0] + + x = (torch.arange(window_size, device=sigma.device, dtype=sigma.dtype) - window_size // 2).expand(batch_size, -1) + + if window_size % 2 == 0: + x = x + 0.5 + + gauss = torch.exp(-x.pow(2.0) / (2 * sigma.pow(2.0))) + + return gauss / gauss.sum(-1, keepdim=True) + + +def _gaussian_blur2d(input, kernel_size, sigma): + if isinstance(sigma, tuple): + sigma = torch.tensor([sigma], dtype=input.dtype) + else: + sigma = sigma.to(dtype=input.dtype) + + ky, kx = int(kernel_size[0]), int(kernel_size[1]) + bs = sigma.shape[0] + kernel_x = _gaussian(kx, sigma[:, 1].view(bs, 1)) + kernel_y = _gaussian(ky, sigma[:, 0].view(bs, 1)) + out_x = _filter2d(input, kernel_x[..., None, :]) + out = _filter2d(out_x, kernel_y[..., None]) + + return out diff --git a/diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/__init__.py new file mode 100755 index 0000000..08c22a2 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/__init__.py @@ -0,0 +1,47 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_stable_diffusion_adapter"] = ["StableDiffusionAdapterPipeline"] + _import_structure["pipeline_stable_diffusion_xl_adapter"] = ["StableDiffusionXLAdapterPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 + else: + from .pipeline_stable_diffusion_adapter import StableDiffusionAdapterPipeline + from .pipeline_stable_diffusion_xl_adapter import StableDiffusionXLAdapterPipeline +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py b/diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py new file mode 100755 index 0000000..0b55bb3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_adapter.py @@ -0,0 +1,912 @@ +# Copyright 2024 TencentARC and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPFeatureExtractor, CLIPTextModel, CLIPTokenizer + +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, MultiAdapter, T2IAdapter, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + PIL_INTERPOLATION, + USE_PEFT_BACKEND, + BaseOutput, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion.safety_checker import StableDiffusionSafetyChecker + + +@dataclass +class StableDiffusionAdapterPipelineOutput(BaseOutput): + """ + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. + nsfw_content_detected (`List[bool]`) + List of flags denoting whether the corresponding generated image likely represents "not-safe-for-work" + (nsfw) content, or `None` if safety checking could not be performed. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + nsfw_content_detected: Optional[List[bool]] + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from PIL import Image + >>> from diffusers.utils import load_image + >>> import torch + >>> from diffusers import StableDiffusionAdapterPipeline, T2IAdapter + + >>> image = load_image( + ... "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/t2i-adapter/color_ref.png" + ... ) + + >>> color_palette = image.resize((8, 8)) + >>> color_palette = color_palette.resize((512, 512), resample=Image.Resampling.NEAREST) + + >>> adapter = T2IAdapter.from_pretrained("TencentARC/t2iadapter_color_sd14v1", torch_dtype=torch.float16) + >>> pipe = StableDiffusionAdapterPipeline.from_pretrained( + ... "CompVis/stable-diffusion-v1-4", + ... adapter=adapter, + ... torch_dtype=torch.float16, + ... ) + + >>> pipe.to("cuda") + + >>> out_image = pipe( + ... "At night, glowing cubes in front of the beach", + ... image=color_palette, + ... ).images[0] + ``` +""" + + +def _preprocess_adapter_image(image, height, width): + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + image = [np.array(i.resize((width, height), resample=PIL_INTERPOLATION["lanczos"])) for i in image] + image = [ + i[None, ..., None] if i.ndim == 2 else i[None, ...] for i in image + ] # expand [h, w] or [h, w, c] to [b, h, w, c] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + if image[0].ndim == 3: + image = torch.stack(image, dim=0) + elif image[0].ndim == 4: + image = torch.cat(image, dim=0) + else: + raise ValueError( + f"Invalid image tensor! Expecting image tensor with 3 or 4 dimension, but recive: {image[0].ndim}" + ) + return image + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class StableDiffusionAdapterPipeline(DiffusionPipeline, StableDiffusionMixin): + r""" + Pipeline for text-to-image generation using Stable Diffusion augmented with T2I-Adapter + https://arxiv.org/abs/2302.08453 + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + adapter ([`T2IAdapter`] or [`MultiAdapter`] or `List[T2IAdapter]`): + Provides additional conditioning to the unet during the denoising process. If you set multiple Adapter as a + list, the outputs from each Adapter are added together to create one combined additional conditioning. + adapter_weights (`List[float]`, *optional*, defaults to None): + List of floats representing the weight which will be multiply to each adapter's output before adding them + together. + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details. + feature_extractor ([`CLIPFeatureExtractor`]): + Model that extracts features from generated images to be used as inputs for the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->adapter->unet->vae" + _optional_components = ["safety_checker", "feature_extractor"] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + adapter: Union[T2IAdapter, MultiAdapter, List[T2IAdapter]], + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPFeatureExtractor, + requires_safety_checker: bool = True, + ): + super().__init__() + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + + if safety_checker is not None and feature_extractor is None: + raise ValueError( + "Make sure to define a feature extractor when loading {self.__class__} if you want to use the safety" + " checker. If you do not want to use the safety checker, you can pass `'safety_checker=None'` instead." + ) + + if isinstance(adapter, (list, tuple)): + adapter = MultiAdapter(adapter) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + adapter=adapter, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.register_to_config(requires_safety_checker=requires_safety_checker) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.decode_latents + def decode_latents(self, latents): + deprecation_message = "The decode_latents method is deprecated and will be removed in 1.0.0. Please use VaeImageProcessor.postprocess(...) instead" + deprecate("decode_latents", "1.0.0", deprecation_message, standard_warn=False) + + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + image, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if isinstance(self.adapter, MultiAdapter): + if not isinstance(image, list): + raise ValueError( + "MultiAdapter is enabled, but `image` is not a list. Please pass a list of images to `image`." + ) + + if len(image) != len(self.adapter.adapters): + raise ValueError( + f"MultiAdapter requires passing the same number of images as adapters. Given {len(image)} images and {len(self.adapter.adapters)} adapters." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def _default_height_width(self, height, width, image): + # NOTE: It is possible that a list of images have different + # dimensions for each image, so just checking the first image + # is not _exactly_ correct, but it is simple. + while isinstance(image, list): + image = image[0] + + if height is None: + if isinstance(image, PIL.Image.Image): + height = image.height + elif isinstance(image, torch.Tensor): + height = image.shape[-2] + + # round down to nearest multiple of `self.adapter.downscale_factor` + height = (height // self.adapter.downscale_factor) * self.adapter.downscale_factor + + if width is None: + if isinstance(image, PIL.Image.Image): + width = image.width + elif isinstance(image, torch.Tensor): + width = image.shape[-1] + + # round down to nearest multiple of `self.adapter.downscale_factor` + width = (width // self.adapter.downscale_factor) * self.adapter.downscale_factor + + return height, width + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + image: Union[torch.Tensor, PIL.Image.Image, List[PIL.Image.Image]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + timesteps: List[int] = None, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + adapter_conditioning_scale: Union[float, List[float]] = 1.0, + clip_skip: Optional[int] = None, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + image (`torch.FloatTensor`, `PIL.Image.Image`, `List[torch.FloatTensor]` or `List[PIL.Image.Image]` or `List[List[PIL.Image.Image]]`): + The Adapter input condition. Adapter uses this input condition to generate guidance to Unet. If the + type is specified as `Torch.FloatTensor`, it is passed to Adapter as is. PIL.Image.Image` can also be + accepted as an image. The control image is automatically resized to fit the output image. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds`. instead. If not defined, one has to pass `negative_prompt_embeds`. instead. + Ignored when not using guidance (i.e., ignored if `guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion.StableDiffusionAdapterPipelineOutput`] instead + of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttnProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + adapter_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + The outputs of the adapter are multiplied by `adapter_conditioning_scale` before they are added to the + residual in the original unet. If multiple adapters are specified in init, you can set the + corresponding scale as a list. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionAdapterPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionAdapterPipelineOutput`] if `return_dict` is True, otherwise a + `tuple. When returning a tuple, the first element is a list with the generated images, and the second + element is a list of `bool`s denoting whether the corresponding generated image likely represents + "not-safe-for-work" (nsfw) content, according to the `safety_checker`. + """ + # 0. Default height and width to unet + height, width = self._default_height_width(height, width, image) + device = self._execution_device + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, image, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + self._guidance_scale = guidance_scale + + if isinstance(self.adapter, MultiAdapter): + adapter_input = [] + + for one_image in image: + one_image = _preprocess_adapter_image(one_image, height, width) + one_image = one_image.to(device=device, dtype=self.adapter.dtype) + adapter_input.append(one_image) + else: + adapter_input = _preprocess_adapter_image(image, height, width) + adapter_input = adapter_input.to(device=device, dtype=self.adapter.dtype) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # 3. Encode input prompt + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 6.5 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + # 7. Denoising loop + if isinstance(self.adapter, MultiAdapter): + adapter_state = self.adapter(adapter_input, adapter_conditioning_scale) + for k, v in enumerate(adapter_state): + adapter_state[k] = v + else: + adapter_state = self.adapter(adapter_input) + for k, v in enumerate(adapter_state): + adapter_state[k] = v * adapter_conditioning_scale + if num_images_per_prompt > 1: + for k, v in enumerate(adapter_state): + adapter_state[k] = v.repeat(num_images_per_prompt, 1, 1, 1) + if self.do_classifier_free_guidance: + for k, v in enumerate(adapter_state): + adapter_state[k] = torch.cat([v] * 2, dim=0) + + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=cross_attention_kwargs, + down_intrablock_additional_residuals=[state.clone() for state in adapter_state], + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if output_type == "latent": + image = latents + has_nsfw_concept = None + elif output_type == "pil": + # 8. Post-processing + image = self.decode_latents(latents) + + # 9. Run safety checker + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # 10. Convert to PIL + image = self.numpy_to_pil(image) + else: + # 8. Post-processing + image = self.decode_latents(latents) + + # 9. Run safety checker + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return StableDiffusionAdapterPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py b/diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py new file mode 100755 index 0000000..4e0cc61 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/t2i_adapter/pipeline_stable_diffusion_xl_adapter.py @@ -0,0 +1,1258 @@ +# Copyright 2024 TencentARC and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL.Image +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from ...image_processor import PipelineImageInput, VaeImageProcessor +from ...loaders import ( + FromSingleFileMixin, + IPAdapterMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, +) +from ...models import AutoencoderKL, ImageProjection, MultiAdapter, T2IAdapter, UNet2DConditionModel +from ...models.attention_processor import ( + AttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + PIL_INTERPOLATION, + USE_PEFT_BACKEND, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion_xl.pipeline_output import StableDiffusionXLPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import T2IAdapter, StableDiffusionXLAdapterPipeline, DDPMScheduler + >>> from diffusers.utils import load_image + + >>> sketch_image = load_image("https://huggingface.co/Adapter/t2iadapter/resolve/main/sketch.png").convert("L") + + >>> model_id = "stabilityai/stable-diffusion-xl-base-1.0" + + >>> adapter = T2IAdapter.from_pretrained( + ... "Adapter/t2iadapter", + ... subfolder="sketch_sdxl_1.0", + ... torch_dtype=torch.float16, + ... adapter_type="full_adapter_xl", + ... ) + >>> scheduler = DDPMScheduler.from_pretrained(model_id, subfolder="scheduler") + + >>> pipe = StableDiffusionXLAdapterPipeline.from_pretrained( + ... model_id, adapter=adapter, torch_dtype=torch.float16, variant="fp16", scheduler=scheduler + ... ).to("cuda") + + >>> generator = torch.manual_seed(42) + >>> sketch_image_out = pipe( + ... prompt="a photo of a dog in real world, high quality", + ... negative_prompt="extra digit, fewer digits, cropped, worst quality, low quality", + ... image=sketch_image, + ... generator=generator, + ... guidance_scale=7.5, + ... ).images[0] + ``` +""" + + +def _preprocess_adapter_image(image, height, width): + if isinstance(image, torch.Tensor): + return image + elif isinstance(image, PIL.Image.Image): + image = [image] + + if isinstance(image[0], PIL.Image.Image): + image = [np.array(i.resize((width, height), resample=PIL_INTERPOLATION["lanczos"])) for i in image] + image = [ + i[None, ..., None] if i.ndim == 2 else i[None, ...] for i in image + ] # expand [h, w] or [h, w, c] to [b, h, w, c] + image = np.concatenate(image, axis=0) + image = np.array(image).astype(np.float32) / 255.0 + image = image.transpose(0, 3, 1, 2) + image = torch.from_numpy(image) + elif isinstance(image[0], torch.Tensor): + if image[0].ndim == 3: + image = torch.stack(image, dim=0) + elif image[0].ndim == 4: + image = torch.cat(image, dim=0) + else: + raise ValueError( + f"Invalid image tensor! Expecting image tensor with 3 or 4 dimension, but recive: {image[0].ndim}" + ) + return image + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.retrieve_timesteps +def retrieve_timesteps( + scheduler, + num_inference_steps: Optional[int] = None, + device: Optional[Union[str, torch.device]] = None, + timesteps: Optional[List[int]] = None, + **kwargs, +): + """ + Calls the scheduler's `set_timesteps` method and retrieves timesteps from the scheduler after the call. Handles + custom timesteps. Any kwargs will be supplied to `scheduler.set_timesteps`. + + Args: + scheduler (`SchedulerMixin`): + The scheduler to get timesteps from. + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of the scheduler is used. If `timesteps` is passed, `num_inference_steps` + must be `None`. + + Returns: + `Tuple[torch.Tensor, int]`: A tuple where the first element is the timestep schedule from the scheduler and the + second element is the number of inference steps. + """ + if timesteps is not None: + accepts_timesteps = "timesteps" in set(inspect.signature(scheduler.set_timesteps).parameters.keys()) + if not accepts_timesteps: + raise ValueError( + f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom" + f" timestep schedules. Please check whether you are using the correct scheduler." + ) + scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs) + timesteps = scheduler.timesteps + num_inference_steps = len(timesteps) + else: + scheduler.set_timesteps(num_inference_steps, device=device, **kwargs) + timesteps = scheduler.timesteps + return timesteps, num_inference_steps + + +class StableDiffusionXLAdapterPipeline( + DiffusionPipeline, + StableDiffusionMixin, + TextualInversionLoaderMixin, + StableDiffusionXLLoraLoaderMixin, + IPAdapterMixin, + FromSingleFileMixin, +): + r""" + Pipeline for text-to-image generation using Stable Diffusion augmented with T2I-Adapter + https://arxiv.org/abs/2302.08453 + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.FromSingleFileMixin.from_single_file`] for loading `.ckpt` files + - [`~loaders.StableDiffusionXLLoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.StableDiffusionXLLoraLoaderMixin.save_lora_weights`] for saving LoRA weights + - [`~loaders.IPAdapterMixin.load_ip_adapter`] for loading IP Adapters + + Args: + adapter ([`T2IAdapter`] or [`MultiAdapter`] or `List[T2IAdapter]`): + Provides additional conditioning to the unet during the denoising process. If you set multiple Adapter as a + list, the outputs from each Adapter are added together to create one combined additional conditioning. + adapter_weights (`List[float]`, *optional*, defaults to None): + List of floats representing the weight which will be multiply to each adapter's output before adding them + together. + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please, refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for details. + feature_extractor ([`CLIPFeatureExtractor`]): + Model that extracts features from generated images to be used as inputs for the `safety_checker`. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->image_encoder->unet->vae" + _optional_components = [ + "tokenizer", + "tokenizer_2", + "text_encoder", + "text_encoder_2", + "feature_extractor", + "image_encoder", + ] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + adapter: Union[T2IAdapter, MultiAdapter, List[T2IAdapter]], + scheduler: KarrasDiffusionSchedulers, + force_zeros_for_empty_prompt: bool = True, + feature_extractor: CLIPImageProcessor = None, + image_encoder: CLIPVisionModelWithProjection = None, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + adapter=adapter, + scheduler=scheduler, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + self.default_sample_size = self.unet.config.sample_size + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # textual inversion: process multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_2 = ( + batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 + ) + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_image + def encode_image(self, image, device, num_images_per_prompt, output_hidden_states=None): + dtype = next(self.image_encoder.parameters()).dtype + + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + if output_hidden_states: + image_enc_hidden_states = self.image_encoder(image, output_hidden_states=True).hidden_states[-2] + image_enc_hidden_states = image_enc_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_enc_hidden_states = self.image_encoder( + torch.zeros_like(image), output_hidden_states=True + ).hidden_states[-2] + uncond_image_enc_hidden_states = uncond_image_enc_hidden_states.repeat_interleave( + num_images_per_prompt, dim=0 + ) + return image_enc_hidden_states, uncond_image_enc_hidden_states + else: + image_embeds = self.image_encoder(image).image_embeds + image_embeds = image_embeds.repeat_interleave(num_images_per_prompt, dim=0) + uncond_image_embeds = torch.zeros_like(image_embeds) + + return image_embeds, uncond_image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_ip_adapter_image_embeds + def prepare_ip_adapter_image_embeds( + self, ip_adapter_image, ip_adapter_image_embeds, device, num_images_per_prompt, do_classifier_free_guidance + ): + if ip_adapter_image_embeds is None: + if not isinstance(ip_adapter_image, list): + ip_adapter_image = [ip_adapter_image] + + if len(ip_adapter_image) != len(self.unet.encoder_hid_proj.image_projection_layers): + raise ValueError( + f"`ip_adapter_image` must have same length as the number of IP Adapters. Got {len(ip_adapter_image)} images and {len(self.unet.encoder_hid_proj.image_projection_layers)} IP Adapters." + ) + + image_embeds = [] + for single_ip_adapter_image, image_proj_layer in zip( + ip_adapter_image, self.unet.encoder_hid_proj.image_projection_layers + ): + output_hidden_state = not isinstance(image_proj_layer, ImageProjection) + single_image_embeds, single_negative_image_embeds = self.encode_image( + single_ip_adapter_image, device, 1, output_hidden_state + ) + single_image_embeds = torch.stack([single_image_embeds] * num_images_per_prompt, dim=0) + single_negative_image_embeds = torch.stack( + [single_negative_image_embeds] * num_images_per_prompt, dim=0 + ) + + if do_classifier_free_guidance: + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + single_image_embeds = single_image_embeds.to(device) + + image_embeds.append(single_image_embeds) + else: + repeat_dims = [1] + image_embeds = [] + for single_image_embeds in ip_adapter_image_embeds: + if do_classifier_free_guidance: + single_negative_image_embeds, single_image_embeds = single_image_embeds.chunk(2) + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + single_negative_image_embeds = single_negative_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_negative_image_embeds.shape[1:])) + ) + single_image_embeds = torch.cat([single_negative_image_embeds, single_image_embeds]) + else: + single_image_embeds = single_image_embeds.repeat( + num_images_per_prompt, *(repeat_dims * len(single_image_embeds.shape[1:])) + ) + image_embeds.append(single_image_embeds) + + return image_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.check_inputs + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + ip_adapter_image=None, + ip_adapter_image_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + if ip_adapter_image is not None and ip_adapter_image_embeds is not None: + raise ValueError( + "Provide either `ip_adapter_image` or `ip_adapter_image_embeds`. Cannot leave both `ip_adapter_image` and `ip_adapter_image_embeds` defined." + ) + + if ip_adapter_image_embeds is not None: + if not isinstance(ip_adapter_image_embeds, list): + raise ValueError( + f"`ip_adapter_image_embeds` has to be of type `list` but is {type(ip_adapter_image_embeds)}" + ) + elif ip_adapter_image_embeds[0].ndim not in [3, 4]: + raise ValueError( + f"`ip_adapter_image_embeds` has to be a list of 3D or 4D tensors but is {ip_adapter_image_embeds[0].ndim}D" + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._get_add_time_ids + def _get_add_time_ids( + self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None + ): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + return add_time_ids + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_upscale.StableDiffusionUpscalePipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + # Copied from diffusers.pipelines.t2i_adapter.pipeline_stable_diffusion_adapter.StableDiffusionAdapterPipeline._default_height_width + def _default_height_width(self, height, width, image): + # NOTE: It is possible that a list of images have different + # dimensions for each image, so just checking the first image + # is not _exactly_ correct, but it is simple. + while isinstance(image, list): + image = image[0] + + if height is None: + if isinstance(image, PIL.Image.Image): + height = image.height + elif isinstance(image, torch.Tensor): + height = image.shape[-2] + + # round down to nearest multiple of `self.adapter.downscale_factor` + height = (height // self.adapter.downscale_factor) * self.adapter.downscale_factor + + if width is None: + if isinstance(image, PIL.Image.Image): + width = image.width + elif isinstance(image, torch.Tensor): + width = image.shape[-1] + + # round down to nearest multiple of `self.adapter.downscale_factor` + width = (width // self.adapter.downscale_factor) * self.adapter.downscale_factor + + return height, width + + # Copied from diffusers.pipelines.latent_consistency_models.pipeline_latent_consistency_text2img.LatentConsistencyModelPipeline.get_guidance_scale_embedding + def get_guidance_scale_embedding(self, w, embedding_dim=512, dtype=torch.float32): + """ + See https://github.com/google-research/vdm/blob/dc27b98a554f65cdc654b800da5aa1846545d41b/model_vdm.py#L298 + + Args: + timesteps (`torch.Tensor`): + generate embedding vectors at these timesteps + embedding_dim (`int`, *optional*, defaults to 512): + dimension of the embeddings to generate + dtype: + data type of the generated embeddings + + Returns: + `torch.FloatTensor`: Embedding vectors with shape `(len(timesteps), embedding_dim)` + """ + assert len(w.shape) == 1 + w = w * 1000.0 + + half_dim = embedding_dim // 2 + emb = torch.log(torch.tensor(10000.0)) / (half_dim - 1) + emb = torch.exp(torch.arange(half_dim, dtype=dtype) * -emb) + emb = w.to(dtype)[:, None] * emb[None, :] + emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=1) + if embedding_dim % 2 == 1: # zero pad + emb = torch.nn.functional.pad(emb, (0, 1)) + assert emb.shape == (w.shape[0], embedding_dim) + return emb + + @property + def guidance_scale(self): + return self._guidance_scale + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 and self.unet.config.time_cond_proj_dim is None + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + prompt_2: Optional[Union[str, List[str]]] = None, + image: PipelineImageInput = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + timesteps: List[int] = None, + denoising_end: Optional[float] = None, + guidance_scale: float = 5.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + ip_adapter_image: Optional[PipelineImageInput] = None, + ip_adapter_image_embeds: Optional[List[torch.FloatTensor]] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + original_size: Optional[Tuple[int, int]] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Optional[Tuple[int, int]] = None, + negative_original_size: Optional[Tuple[int, int]] = None, + negative_crops_coords_top_left: Tuple[int, int] = (0, 0), + negative_target_size: Optional[Tuple[int, int]] = None, + adapter_conditioning_scale: Union[float, List[float]] = 1.0, + adapter_conditioning_factor: float = 1.0, + clip_skip: Optional[int] = None, + ): + r""" + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + image (`torch.FloatTensor`, `PIL.Image.Image`, `List[torch.FloatTensor]` or `List[PIL.Image.Image]` or `List[List[PIL.Image.Image]]`): + The Adapter input condition. Adapter uses this input condition to generate guidance to Unet. If the + type is specified as `Torch.FloatTensor`, it is passed to Adapter as is. PIL.Image.Image` can also be + accepted as an image. The control image is automatically resized to fit the output image. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. Anything below 512 pixels won't work well for + [stabilityai/stable-diffusion-xl-base-1.0](https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0) + and checkpoints that are not specifically fine-tuned on low resolutions. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process with schedulers which support a `timesteps` argument + in their `set_timesteps` method. If not defined, the default behavior when `num_inference_steps` is + passed will be used. Must be in descending order. + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise as determined by the discrete timesteps selected by the + scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a + "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) + guidance_scale (`float`, *optional*, defaults to 5.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + ip_adapter_image: (`PipelineImageInput`, *optional*): Optional image input to work with IP Adapters. + ip_adapter_image_embeds (`List[torch.FloatTensor]`, *optional*): + Pre-generated image embeddings for IP-Adapter. It should be a list of length same as number of IP-adapters. + Each element should be a tensor of shape `(batch_size, num_images, emb_dim)`. It should contain the negative image embedding + if `do_classifier_free_guidance` is set to `True`. + If not provided, embeddings are computed from the `ip_adapter_image` input argument. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionAdapterPipelineOutput`] + instead of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.models.attention_processor](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + guidance_rescale (`float`, *optional*, defaults to 0.0): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(height, width)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(height, width)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + negative_original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a specific image resolution. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + To negatively condition the generation process based on a specific crop coordinates. Part of SDXL's + micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + negative_target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + To negatively condition the generation process based on a target image resolution. It should be as same + as the `target_size` for most cases. Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). For more + information, refer to this issue thread: https://github.com/huggingface/diffusers/issues/4208. + adapter_conditioning_scale (`float` or `List[float]`, *optional*, defaults to 1.0): + The outputs of the adapter are multiplied by `adapter_conditioning_scale` before they are added to the + residual in the original unet. If multiple adapters are specified in init, you can set the + corresponding scale as a list. + adapter_conditioning_factor (`float`, *optional*, defaults to 1.0): + The fraction of timesteps for which adapter should be applied. If `adapter_conditioning_factor` is + `0.0`, adapter is not applied at all. If `adapter_conditioning_factor` is `1.0`, adapter is applied for + all timesteps. If `adapter_conditioning_factor` is `0.5`, adapter is applied for half of the timesteps. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + + Examples: + + Returns: + [`~pipelines.stable_diffusion.StableDiffusionAdapterPipelineOutput`] or `tuple`: + [`~pipelines.stable_diffusion.StableDiffusionAdapterPipelineOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + # 0. Default height and width to unet + + height, width = self._default_height_width(height, width, image) + device = self._execution_device + + if isinstance(self.adapter, MultiAdapter): + adapter_input = [] + + for one_image in image: + one_image = _preprocess_adapter_image(one_image, height, width) + one_image = one_image.to(device=device, dtype=self.adapter.dtype) + adapter_input.append(one_image) + else: + adapter_input = _preprocess_adapter_image(image, height, width) + adapter_input = adapter_input.to(device=device, dtype=self.adapter.dtype) + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ip_adapter_image, + ip_adapter_image_embeds, + ) + + self._guidance_scale = guidance_scale + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + + # 3.1 Encode input prompt + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + clip_skip=clip_skip, + ) + + # 3.2 Encode ip_adapter_image + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + image_embeds = self.prepare_ip_adapter_image_embeds( + ip_adapter_image, + ip_adapter_image_embeds, + device, + batch_size * num_images_per_prompt, + self.do_classifier_free_guidance, + ) + + # 4. Prepare timesteps + timesteps, num_inference_steps = retrieve_timesteps(self.scheduler, num_inference_steps, device, timesteps) + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6.1 Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 6.2 Optionally get Guidance Scale Embedding + timestep_cond = None + if self.unet.config.time_cond_proj_dim is not None: + guidance_scale_tensor = torch.tensor(self.guidance_scale - 1).repeat(batch_size * num_images_per_prompt) + timestep_cond = self.get_guidance_scale_embedding( + guidance_scale_tensor, embedding_dim=self.unet.config.time_cond_proj_dim + ).to(device=device, dtype=latents.dtype) + + # 7. Prepare added time ids & embeddings & adapter features + if isinstance(self.adapter, MultiAdapter): + adapter_state = self.adapter(adapter_input, adapter_conditioning_scale) + for k, v in enumerate(adapter_state): + adapter_state[k] = v + else: + adapter_state = self.adapter(adapter_input) + for k, v in enumerate(adapter_state): + adapter_state[k] = v * adapter_conditioning_scale + if num_images_per_prompt > 1: + for k, v in enumerate(adapter_state): + adapter_state[k] = v.repeat(num_images_per_prompt, 1, 1, 1) + if self.do_classifier_free_guidance: + for k, v in enumerate(adapter_state): + adapter_state[k] = torch.cat([v] * 2, dim=0) + + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + if negative_original_size is not None and negative_target_size is not None: + negative_add_time_ids = self._get_add_time_ids( + negative_original_size, + negative_crops_coords_top_left, + negative_target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + else: + negative_add_time_ids = add_time_ids + + if self.do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([negative_add_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(batch_size * num_images_per_prompt, 1) + + # 8. Denoising loop + num_warmup_steps = max(len(timesteps) - num_inference_steps * self.scheduler.order, 0) + # Apply denoising_end + if denoising_end is not None and isinstance(denoising_end, float) and denoising_end > 0 and denoising_end < 1: + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents + + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + + if ip_adapter_image is not None or ip_adapter_image_embeds is not None: + added_cond_kwargs["image_embeds"] = image_embeds + + # predict the noise residual + if i < int(num_inference_steps * adapter_conditioning_factor): + down_intrablock_additional_residuals = [state.clone() for state in adapter_state] + else: + down_intrablock_additional_residuals = None + + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + timestep_cond=timestep_cond, + cross_attention_kwargs=cross_attention_kwargs, + down_intrablock_additional_residuals=down_intrablock_additional_residuals, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if self.do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if self.do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs, return_dict=False)[0] + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + image = latents + return StableDiffusionXLPipelineOutput(images=image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image,) + + return StableDiffusionXLPipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/__init__.py new file mode 100755 index 0000000..8d8fdb9 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/__init__.py @@ -0,0 +1,54 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["pipeline_output"] = ["TextToVideoSDPipelineOutput"] + _import_structure["pipeline_text_to_video_synth"] = ["TextToVideoSDPipeline"] + _import_structure["pipeline_text_to_video_synth_img2img"] = ["VideoToVideoSDPipeline"] + _import_structure["pipeline_text_to_video_zero"] = ["TextToVideoZeroPipeline"] + _import_structure["pipeline_text_to_video_zero_sdxl"] = ["TextToVideoZeroSDXLPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 + else: + from .pipeline_output import TextToVideoSDPipelineOutput + from .pipeline_text_to_video_synth import TextToVideoSDPipeline + from .pipeline_text_to_video_synth_img2img import VideoToVideoSDPipeline + from .pipeline_text_to_video_zero import TextToVideoZeroPipeline + from .pipeline_text_to_video_zero_sdxl import TextToVideoZeroSDXLPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_output.py b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_output.py new file mode 100755 index 0000000..c155386 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_output.py @@ -0,0 +1,25 @@ +from dataclasses import dataclass +from typing import List, Union + +import numpy as np +import PIL +import torch + +from ...utils import ( + BaseOutput, +) + + +@dataclass +class TextToVideoSDPipelineOutput(BaseOutput): + """ + Output class for text-to-video pipelines. + + Args: + frames (`torch.Tensor`, `np.ndarray`, or List[List[PIL.Image.Image]]): + List of video outputs - It can be a nested list of length `batch_size,` with each sub-list containing denoised + PIL image sequences of length `num_frames.` It can also be a NumPy array or Torch tensor of shape + `(batch_size, num_frames, channels, height, width)` + """ + + frames: Union[torch.Tensor, np.ndarray, List[List[PIL.Image.Image]]] diff --git a/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py new file mode 100755 index 0000000..6c33836 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth.py @@ -0,0 +1,663 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import torch +from transformers import CLIPTextModel, CLIPTokenizer + +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet3DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from . import TextToVideoSDPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import TextToVideoSDPipeline + >>> from diffusers.utils import export_to_video + + >>> pipe = TextToVideoSDPipeline.from_pretrained( + ... "damo-vilab/text-to-video-ms-1.7b", torch_dtype=torch.float16, variant="fp16" + ... ) + >>> pipe.enable_model_cpu_offload() + + >>> prompt = "Spiderman is surfing" + >>> video_frames = pipe(prompt).frames[0] + >>> video_path = export_to_video(video_frames) + >>> video_path + ``` +""" + + +# Copied from diffusers.pipelines.animatediff.pipeline_animatediff.tensor2vid +def tensor2vid(video: torch.Tensor, processor: "VaeImageProcessor", output_type: str = "np"): + batch_size, channels, num_frames, height, width = video.shape + outputs = [] + for batch_idx in range(batch_size): + batch_vid = video[batch_idx].permute(1, 0, 2, 3) + batch_output = processor.postprocess(batch_vid, output_type) + + outputs.append(batch_output) + + if output_type == "np": + outputs = np.stack(outputs) + + elif output_type == "pt": + outputs = torch.stack(outputs) + + elif not output_type == "pil": + raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil']") + + return outputs + + +class TextToVideoSDPipeline(DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin): + r""" + Pipeline for text-to-video generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (`CLIPTokenizer`): + A [`~transformers.CLIPTokenizer`] to tokenize text. + unet ([`UNet3DConditionModel`]): + A [`UNet3DConditionModel`] to denoise the encoded video latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet3DConditionModel, + scheduler: KarrasDiffusionSchedulers, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + def decode_latents(self, latents): + latents = 1 / self.vae.config.scaling_factor * latents + + batch_size, channels, num_frames, height, width = latents.shape + latents = latents.permute(0, 2, 1, 3, 4).reshape(batch_size * num_frames, channels, height, width) + + image = self.vae.decode(latents).sample + video = image[None, :].reshape((batch_size, num_frames, -1) + image.shape[2:]).permute(0, 2, 1, 3, 4) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + video = video.float() + return video + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion_k_diffusion.pipeline_stable_diffusion_k_diffusion.StableDiffusionKDiffusionPipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + def prepare_latents( + self, batch_size, num_channels_latents, num_frames, height, width, dtype, device, generator, latents=None + ): + shape = ( + batch_size, + num_channels_latents, + num_frames, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + num_frames: int = 16, + num_inference_steps: int = 50, + guidance_scale: float = 9.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "np", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated video. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated video. + num_frames (`int`, *optional*, defaults to 16): + The number of video frames that are generated. Defaults to 16 frames which at 8 frames per seconds + amounts to 2 seconds of video. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality videos at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. Latents should be of shape + `(batch_size, num_channel, num_frames, height, width)`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"np"`): + The output format of the generated video. Choose between `torch.FloatTensor` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] instead + of a plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Examples: + + Returns: + [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] is + returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. + """ + # 0. Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + num_images_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, height, width, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds + ) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_images_per_prompt, + num_channels_latents, + num_frames, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # reshape latents + bsz, channel, frames, width, height = latents.shape + latents = latents.permute(0, 2, 1, 3, 4).reshape(bsz * frames, channel, width, height) + noise_pred = noise_pred.permute(0, 2, 1, 3, 4).reshape(bsz * frames, channel, width, height) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # reshape latents back + latents = latents[None, :].reshape(bsz, frames, channel, width, height).permute(0, 2, 1, 3, 4) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 8. Post processing + if output_type == "latent": + video = latents + else: + video_tensor = self.decode_latents(latents) + video = tensor2vid(video_tensor, self.image_processor, output_type) + + # 9. Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (video,) + + return TextToVideoSDPipelineOutput(frames=video) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py new file mode 100755 index 0000000..3901946 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_synth_img2img.py @@ -0,0 +1,760 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import Any, Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import CLIPTextModel, CLIPTokenizer + +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet3DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + deprecate, + logging, + replace_example_docstring, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from . import TextToVideoSDPipelineOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import DiffusionPipeline, DPMSolverMultistepScheduler + >>> from diffusers.utils import export_to_video + + >>> pipe = DiffusionPipeline.from_pretrained("cerspense/zeroscope_v2_576w", torch_dtype=torch.float16) + >>> pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + >>> pipe.to("cuda") + + >>> prompt = "spiderman running in the desert" + >>> video_frames = pipe(prompt, num_inference_steps=40, height=320, width=576, num_frames=24).frames[0] + >>> # safe low-res video + >>> video_path = export_to_video(video_frames, output_video_path="./video_576_spiderman.mp4") + + >>> # let's offload the text-to-image model + >>> pipe.to("cpu") + + >>> # and load the image-to-image model + >>> pipe = DiffusionPipeline.from_pretrained( + ... "cerspense/zeroscope_v2_XL", torch_dtype=torch.float16, revision="refs/pr/15" + ... ) + >>> pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + >>> pipe.enable_model_cpu_offload() + + >>> # The VAE consumes A LOT of memory, let's make sure we run it in sliced mode + >>> pipe.vae.enable_slicing() + + >>> # now let's upscale it + >>> video = [Image.fromarray(frame).resize((1024, 576)) for frame in video_frames] + + >>> # and denoise it + >>> video_frames = pipe(prompt, video=video, strength=0.6).frames[0] + >>> video_path = export_to_video(video_frames, output_video_path="./video_1024_spiderman.mp4") + >>> video_path + ``` +""" + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.retrieve_latents +def retrieve_latents( + encoder_output: torch.Tensor, generator: Optional[torch.Generator] = None, sample_mode: str = "sample" +): + if hasattr(encoder_output, "latent_dist") and sample_mode == "sample": + return encoder_output.latent_dist.sample(generator) + elif hasattr(encoder_output, "latent_dist") and sample_mode == "argmax": + return encoder_output.latent_dist.mode() + elif hasattr(encoder_output, "latents"): + return encoder_output.latents + else: + raise AttributeError("Could not access latents of provided encoder_output") + + +# Copied from diffusers.pipelines.animatediff.pipeline_animatediff.tensor2vid +def tensor2vid(video: torch.Tensor, processor: "VaeImageProcessor", output_type: str = "np"): + batch_size, channels, num_frames, height, width = video.shape + outputs = [] + for batch_idx in range(batch_size): + batch_vid = video[batch_idx].permute(1, 0, 2, 3) + batch_output = processor.postprocess(batch_vid, output_type) + + outputs.append(batch_output) + + if output_type == "np": + outputs = np.stack(outputs) + + elif output_type == "pt": + outputs = torch.stack(outputs) + + elif not output_type == "pil": + raise ValueError(f"{output_type} does not exist. Please choose one of ['np', 'pt', 'pil']") + + return outputs + + +def preprocess_video(video): + supported_formats = (np.ndarray, torch.Tensor, PIL.Image.Image) + + if isinstance(video, supported_formats): + video = [video] + elif not (isinstance(video, list) and all(isinstance(i, supported_formats) for i in video)): + raise ValueError( + f"Input is in incorrect format: {[type(i) for i in video]}. Currently, we only support {', '.join(supported_formats)}" + ) + + if isinstance(video[0], PIL.Image.Image): + video = [np.array(frame) for frame in video] + + if isinstance(video[0], np.ndarray): + video = np.concatenate(video, axis=0) if video[0].ndim == 5 else np.stack(video, axis=0) + + if video.dtype == np.uint8: + video = np.array(video).astype(np.float32) / 255.0 + + if video.ndim == 4: + video = video[None, ...] + + video = torch.from_numpy(video.transpose(0, 4, 1, 2, 3)) + + elif isinstance(video[0], torch.Tensor): + video = torch.cat(video, axis=0) if video[0].ndim == 5 else torch.stack(video, axis=0) + + # don't need any preprocess if the video is latents + channel = video.shape[1] + if channel == 4: + return video + + # move channels before num_frames + video = video.permute(0, 2, 1, 3, 4) + + # normalize video + video = 2.0 * video - 1.0 + + return video + + +class VideoToVideoSDPipeline(DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin): + r""" + Pipeline for text-guided video-to-video generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + The pipeline also inherits the following loading methods: + - [`~loaders.TextualInversionLoaderMixin.load_textual_inversion`] for loading textual inversion embeddings + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode videos to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (`CLIPTokenizer`): + A [`~transformers.CLIPTokenizer`] to tokenize text. + unet ([`UNet3DConditionModel`]): + A [`UNet3DConditionModel`] to denoise the encoded video latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "text_encoder->unet->vae" + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet3DConditionModel, + scheduler: KarrasDiffusionSchedulers, + ): + super().__init__() + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_synth.TextToVideoSDPipeline.decode_latents + def decode_latents(self, latents): + latents = 1 / self.vae.config.scaling_factor * latents + + batch_size, channels, num_frames, height, width = latents.shape + latents = latents.permute(0, 2, 1, 3, 4).reshape(batch_size * num_frames, channels, height, width) + + image = self.vae.decode(latents).sample + video = image[None, :].reshape((batch_size, num_frames, -1) + image.shape[2:]).permute(0, 2, 1, 3, 4) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + video = video.float() + return video + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def check_inputs( + self, + prompt, + strength, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if strength < 0 or strength > 1: + raise ValueError(f"The value of strength should in [0.0, 1.0] but is {strength}") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_img2img.StableDiffusionImg2ImgPipeline.get_timesteps + def get_timesteps(self, num_inference_steps, strength, device): + # get the original timestep using init_timestep + init_timestep = min(int(num_inference_steps * strength), num_inference_steps) + + t_start = max(num_inference_steps - init_timestep, 0) + timesteps = self.scheduler.timesteps[t_start * self.scheduler.order :] + if hasattr(self.scheduler, "set_begin_index"): + self.scheduler.set_begin_index(t_start * self.scheduler.order) + + return timesteps, num_inference_steps - t_start + + def prepare_latents(self, video, timestep, batch_size, dtype, device, generator=None): + video = video.to(device=device, dtype=dtype) + + # change from (b, c, f, h, w) -> (b * f, c, w, h) + bsz, channel, frames, width, height = video.shape + video = video.permute(0, 2, 1, 3, 4).reshape(bsz * frames, channel, width, height) + + if video.shape[1] == 4: + init_latents = video + else: + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + elif isinstance(generator, list): + init_latents = [ + retrieve_latents(self.vae.encode(video[i : i + 1]), generator=generator[i]) + for i in range(batch_size) + ] + init_latents = torch.cat(init_latents, dim=0) + else: + init_latents = retrieve_latents(self.vae.encode(video), generator=generator) + + init_latents = self.vae.config.scaling_factor * init_latents + + if batch_size > init_latents.shape[0] and batch_size % init_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `video` of batch size {init_latents.shape[0]} to {batch_size} text prompts." + ) + else: + init_latents = torch.cat([init_latents], dim=0) + + shape = init_latents.shape + noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + + # get latents + init_latents = self.scheduler.add_noise(init_latents, noise, timestep) + latents = init_latents + + latents = latents[None, :].reshape((bsz, frames, latents.shape[1]) + latents.shape[2:]).permute(0, 2, 1, 3, 4) + + return latents + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Union[str, List[str]] = None, + video: Union[List[np.ndarray], torch.FloatTensor] = None, + strength: float = 0.6, + num_inference_steps: int = 50, + guidance_scale: float = 15.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "np", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + clip_skip: Optional[int] = None, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + video (`List[np.ndarray]` or `torch.FloatTensor`): + `video` frames or tensor representing a video batch to be used as the starting point for the process. + Can also accept video latents as `image`, if passing latents directly, it will not be encoded again. + strength (`float`, *optional*, defaults to 0.8): + Indicates extent to transform the reference `video`. Must be between 0 and 1. `video` is used as a + starting point, adding more noise to it the larger the `strength`. The number of denoising steps + depends on the amount of noise initially added. When `strength` is 1, added noise is maximum and the + denoising process runs for the full number of iterations specified in `num_inference_steps`. A value of + 1 essentially ignores `video`. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality videos at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in video generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. Latents should be of shape + `(batch_size, num_channel, num_frames, height, width)`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are generated from the `negative_prompt` input argument. + output_type (`str`, *optional*, defaults to `"np"`): + The output format of the generated video. Choose between `torch.FloatTensor` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] instead + of a plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + Examples: + + Returns: + [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.text_to_video_synthesis.TextToVideoSDPipelineOutput`] is + returned, otherwise a `tuple` is returned where the first element is a list with the generated frames. + """ + # 0. Default height and width to unet + num_images_per_prompt = 1 + + # 1. Check inputs. Raise error if not correct + self.check_inputs(prompt, strength, callback_steps, negative_prompt, prompt_embeds, negative_prompt_embeds) + + # 2. Define call parameters + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=text_encoder_lora_scale, + clip_skip=clip_skip, + ) + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + # 4. Preprocess video + video = preprocess_video(video) + + # 5. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps, num_inference_steps = self.get_timesteps(num_inference_steps, strength, device) + latent_timestep = timesteps[:1].repeat(batch_size * num_images_per_prompt) + + # 6. Prepare latent variables + latents = self.prepare_latents(video, latent_timestep, batch_size, prompt_embeds.dtype, device, generator) + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # reshape latents + bsz, channel, frames, width, height = latents.shape + latents = latents.permute(0, 2, 1, 3, 4).reshape(bsz * frames, channel, width, height) + noise_pred = noise_pred.permute(0, 2, 1, 3, 4).reshape(bsz * frames, channel, width, height) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # reshape latents back + latents = latents[None, :].reshape(bsz, frames, channel, width, height).permute(0, 2, 1, 3, 4) + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # manually for max memory savings + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.unet.to("cpu") + + # 9. Post processing + if output_type == "latent": + video = latents + else: + video_tensor = self.decode_latents(latents) + video = tensor2vid(video_tensor, self.image_processor, output_type) + + # 10. Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (video,) + + return TextToVideoSDPipelineOutput(frames=video) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py new file mode 100755 index 0000000..d3ff372 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero.py @@ -0,0 +1,969 @@ +import copy +import inspect +from dataclasses import dataclass +from typing import Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +import torch.nn.functional as F +from torch.nn.functional import grid_sample +from transformers import CLIPImageProcessor, CLIPTextModel, CLIPTokenizer + +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import USE_PEFT_BACKEND, BaseOutput, logging, scale_lora_layers, unscale_lora_layers +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin +from ..stable_diffusion import StableDiffusionSafetyChecker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def rearrange_0(tensor, f): + F, C, H, W = tensor.size() + tensor = torch.permute(torch.reshape(tensor, (F // f, f, C, H, W)), (0, 2, 1, 3, 4)) + return tensor + + +def rearrange_1(tensor): + B, C, F, H, W = tensor.size() + return torch.reshape(torch.permute(tensor, (0, 2, 1, 3, 4)), (B * F, C, H, W)) + + +def rearrange_3(tensor, f): + F, D, C = tensor.size() + return torch.reshape(tensor, (F // f, f, D, C)) + + +def rearrange_4(tensor): + B, F, D, C = tensor.size() + return torch.reshape(tensor, (B * F, D, C)) + + +class CrossFrameAttnProcessor: + """ + Cross frame attention processor. Each frame attends the first frame. + + Args: + batch_size: The number that represents actual batch size, other than the frames. + For example, calling unet with a single prompt and num_images_per_prompt=1, batch_size should be equal to + 2, due to classifier-free guidance. + """ + + def __init__(self, batch_size=2): + self.batch_size = batch_size + + def __call__(self, attn, hidden_states, encoder_hidden_states=None, attention_mask=None): + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + query = attn.to_q(hidden_states) + + is_cross_attention = encoder_hidden_states is not None + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + # Cross Frame Attention + if not is_cross_attention: + video_length = key.size()[0] // self.batch_size + first_frame_index = [0] * video_length + + # rearrange keys to have batch and frames in the 1st and 2nd dims respectively + key = rearrange_3(key, video_length) + key = key[:, first_frame_index] + # rearrange values to have batch and frames in the 1st and 2nd dims respectively + value = rearrange_3(value, video_length) + value = value[:, first_frame_index] + + # rearrange back to original shape + key = rearrange_4(key) + value = rearrange_4(value) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +class CrossFrameAttnProcessor2_0: + """ + Cross frame attention processor with scaled_dot_product attention of Pytorch 2.0. + + Args: + batch_size: The number that represents actual batch size, other than the frames. + For example, calling unet with a single prompt and num_images_per_prompt=1, batch_size should be equal to + 2, due to classifier-free guidance. + """ + + def __init__(self, batch_size=2): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.") + self.batch_size = batch_size + + def __call__(self, attn, hidden_states, encoder_hidden_states=None, attention_mask=None): + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + inner_dim = hidden_states.shape[-1] + + if attention_mask is not None: + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + # scaled_dot_product_attention expects attention_mask shape to be + # (batch, heads, source_length, target_length) + attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) + + query = attn.to_q(hidden_states) + + is_cross_attention = encoder_hidden_states is not None + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + # Cross Frame Attention + if not is_cross_attention: + video_length = max(1, key.size()[0] // self.batch_size) + first_frame_index = [0] * video_length + + # rearrange keys to have batch and frames in the 1st and 2nd dims respectively + key = rearrange_3(key, video_length) + key = key[:, first_frame_index] + # rearrange values to have batch and frames in the 1st and 2nd dims respectively + value = rearrange_3(value, video_length) + value = value[:, first_frame_index] + + # rearrange back to original shape + key = rearrange_4(key) + value = rearrange_4(value) + + head_dim = inner_dim // attn.heads + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # the output of sdp = (batch, num_heads, seq_len, head_dim) + # TODO: add support for attn.scale when we move to Torch 2.1 + hidden_states = F.scaled_dot_product_attention( + query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False + ) + + hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + hidden_states = hidden_states.to(query.dtype) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + return hidden_states + + +@dataclass +class TextToVideoPipelineOutput(BaseOutput): + r""" + Output class for zero-shot text-to-video pipeline. + + Args: + images (`[List[PIL.Image.Image]`, `np.ndarray`]): + List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + num_channels)`. + nsfw_content_detected (`[List[bool]]`): + List indicating whether the corresponding generated image contains "not-safe-for-work" (nsfw) content or + `None` if safety checking could not be performed. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + nsfw_content_detected: Optional[List[bool]] + + +def coords_grid(batch, ht, wd, device): + # Adapted from https://github.com/princeton-vl/RAFT/blob/master/core/utils/utils.py + coords = torch.meshgrid(torch.arange(ht, device=device), torch.arange(wd, device=device)) + coords = torch.stack(coords[::-1], dim=0).float() + return coords[None].repeat(batch, 1, 1, 1) + + +def warp_single_latent(latent, reference_flow): + """ + Warp latent of a single frame with given flow + + Args: + latent: latent code of a single frame + reference_flow: flow which to warp the latent with + + Returns: + warped: warped latent + """ + _, _, H, W = reference_flow.size() + _, _, h, w = latent.size() + coords0 = coords_grid(1, H, W, device=latent.device).to(latent.dtype) + + coords_t0 = coords0 + reference_flow + coords_t0[:, 0] /= W + coords_t0[:, 1] /= H + + coords_t0 = coords_t0 * 2.0 - 1.0 + coords_t0 = F.interpolate(coords_t0, size=(h, w), mode="bilinear") + coords_t0 = torch.permute(coords_t0, (0, 2, 3, 1)) + + warped = grid_sample(latent, coords_t0, mode="nearest", padding_mode="reflection") + return warped + + +def create_motion_field(motion_field_strength_x, motion_field_strength_y, frame_ids, device, dtype): + """ + Create translation motion field + + Args: + motion_field_strength_x: motion strength along x-axis + motion_field_strength_y: motion strength along y-axis + frame_ids: indexes of the frames the latents of which are being processed. + This is needed when we perform chunk-by-chunk inference + device: device + dtype: dtype + + Returns: + + """ + seq_length = len(frame_ids) + reference_flow = torch.zeros((seq_length, 2, 512, 512), device=device, dtype=dtype) + for fr_idx in range(seq_length): + reference_flow[fr_idx, 0, :, :] = motion_field_strength_x * (frame_ids[fr_idx]) + reference_flow[fr_idx, 1, :, :] = motion_field_strength_y * (frame_ids[fr_idx]) + return reference_flow + + +def create_motion_field_and_warp_latents(motion_field_strength_x, motion_field_strength_y, frame_ids, latents): + """ + Creates translation motion and warps the latents accordingly + + Args: + motion_field_strength_x: motion strength along x-axis + motion_field_strength_y: motion strength along y-axis + frame_ids: indexes of the frames the latents of which are being processed. + This is needed when we perform chunk-by-chunk inference + latents: latent codes of frames + + Returns: + warped_latents: warped latents + """ + motion_field = create_motion_field( + motion_field_strength_x=motion_field_strength_x, + motion_field_strength_y=motion_field_strength_y, + frame_ids=frame_ids, + device=latents.device, + dtype=latents.dtype, + ) + warped_latents = latents.clone().detach() + for i in range(len(warped_latents)): + warped_latents[i] = warp_single_latent(latents[i][None], motion_field[i][None]) + return warped_latents + + +class TextToVideoZeroPipeline(DiffusionPipeline, StableDiffusionMixin, TextualInversionLoaderMixin, LoraLoaderMixin): + r""" + Pipeline for zero-shot text-to-video generation using Stable Diffusion. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + tokenizer (`CLIPTokenizer`): + A [`~transformers.CLIPTokenizer`] to tokenize text. + unet ([`UNet2DConditionModel`]): + A [`UNet3DConditionModel`] to denoise the encoded video latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + safety_checker ([`StableDiffusionSafetyChecker`]): + Classification module that estimates whether generated images could be considered offensive or harmful. + Please refer to the [model card](https://huggingface.co/runwayml/stable-diffusion-v1-5) for more details + about a model's potential harms. + feature_extractor ([`CLIPImageProcessor`]): + A [`CLIPImageProcessor`] to extract features from generated images; used as inputs to the `safety_checker`. + """ + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + tokenizer: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + safety_checker: StableDiffusionSafetyChecker, + feature_extractor: CLIPImageProcessor, + requires_safety_checker: bool = True, + ): + super().__init__() + self.register_modules( + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + unet=unet, + scheduler=scheduler, + safety_checker=safety_checker, + feature_extractor=feature_extractor, + ) + processor = ( + CrossFrameAttnProcessor2_0(batch_size=2) + if hasattr(F, "scaled_dot_product_attention") + else CrossFrameAttnProcessor(batch_size=2) + ) + self.unet.set_attn_processor(processor) + + if safety_checker is None and requires_safety_checker: + logger.warning( + f"You have disabled the safety checker for {self.__class__} by passing `safety_checker=None`. Ensure" + " that you abide to the conditions of the Stable Diffusion license and do not expose unfiltered" + " results in services or applications open to the public. Both the diffusers team and Hugging Face" + " strongly recommend to keep the safety filter enabled in all public facing circumstances, disabling" + " it only for use-cases that involve analyzing network behavior or auditing its results. For more" + " information, please have a look at https://github.com/huggingface/diffusers/pull/254 ." + ) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + def forward_loop(self, x_t0, t0, t1, generator): + """ + Perform DDPM forward process from time t0 to t1. This is the same as adding noise with corresponding variance. + + Args: + x_t0: + Latent code at time t0. + t0: + Timestep at t0. + t1: + Timestamp at t1. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + + Returns: + x_t1: + Forward process applied to x_t0 from time t0 to t1. + """ + eps = randn_tensor(x_t0.size(), generator=generator, dtype=x_t0.dtype, device=x_t0.device) + alpha_vec = torch.prod(self.scheduler.alphas[t0:t1]) + x_t1 = torch.sqrt(alpha_vec) * x_t0 + torch.sqrt(1 - alpha_vec) * eps + return x_t1 + + def backward_loop( + self, + latents, + timesteps, + prompt_embeds, + guidance_scale, + callback, + callback_steps, + num_warmup_steps, + extra_step_kwargs, + cross_attention_kwargs=None, + ): + """ + Perform backward process given list of time steps. + + Args: + latents: + Latents at time timesteps[0]. + timesteps: + Time steps along which to perform backward process. + prompt_embeds: + Pre-generated text embeddings. + guidance_scale: + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + extra_step_kwargs: + Extra_step_kwargs. + cross_attention_kwargs: + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + num_warmup_steps: + number of warmup steps. + + Returns: + latents: + Latents of backward process output at time timesteps[-1]. + """ + do_classifier_free_guidance = guidance_scale > 1.0 + num_steps = (len(timesteps) - num_warmup_steps) // self.scheduler.order + with self.progress_bar(total=num_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + ).sample + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + return latents.clone().detach() + + # Copied from diffusers.pipelines.stable_diffusion_k_diffusion.pipeline_stable_diffusion_k_diffusion.StableDiffusionKDiffusionPipeline.check_inputs + def check_inputs( + self, + prompt, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + video_length: Optional[int] = 8, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_videos_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + motion_field_strength_x: float = 12, + motion_field_strength_y: float = 12, + output_type: Optional[str] = "tensor", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: Optional[int] = 1, + t0: int = 44, + t1: int = 47, + frame_ids: Optional[List[int]] = None, + ): + """ + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + video_length (`int`, *optional*, defaults to 8): + The number of generated video frames. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 7.5): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in video generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of videos to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for video + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"numpy"`): + The output format of the generated video. Choose between `"latent"` and `"numpy"`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a + [`~pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.TextToVideoPipelineOutput`] instead of + a plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + motion_field_strength_x (`float`, *optional*, defaults to 12): + Strength of motion in generated video along x-axis. See the [paper](https://arxiv.org/abs/2303.13439), + Sect. 3.3.1. + motion_field_strength_y (`float`, *optional*, defaults to 12): + Strength of motion in generated video along y-axis. See the [paper](https://arxiv.org/abs/2303.13439), + Sect. 3.3.1. + t0 (`int`, *optional*, defaults to 44): + Timestep t0. Should be in the range [0, num_inference_steps - 1]. See the + [paper](https://arxiv.org/abs/2303.13439), Sect. 3.3.1. + t1 (`int`, *optional*, defaults to 47): + Timestep t0. Should be in the range [t0 + 1, num_inference_steps - 1]. See the + [paper](https://arxiv.org/abs/2303.13439), Sect. 3.3.1. + frame_ids (`List[int]`, *optional*): + Indexes of the frames that are being generated. This is used when generating longer videos + chunk-by-chunk. + + Returns: + [`~pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.TextToVideoPipelineOutput`]: + The output contains a `ndarray` of the generated video, when `output_type` != `"latent"`, otherwise a + latent code of generated videos and a list of `bool`s indicating whether the corresponding generated + video contains "not-safe-for-work" (nsfw) content.. + """ + assert video_length > 0 + if frame_ids is None: + frame_ids = list(range(video_length)) + assert len(frame_ids) == video_length + + assert num_videos_per_prompt == 1 + + if isinstance(prompt, str): + prompt = [prompt] + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + + # Default height and width to unet + height = height or self.unet.config.sample_size * self.vae_scale_factor + width = width or self.unet.config.sample_size * self.vae_scale_factor + + # Check inputs. Raise error if not correct + self.check_inputs(prompt, height, width, callback_steps) + + # Define call parameters + batch_size = 1 if isinstance(prompt, str) else len(prompt) + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # Encode input prompt + prompt_embeds_tuple = self.encode_prompt( + prompt, device, num_videos_per_prompt, do_classifier_free_guidance, negative_prompt + ) + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + # Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # Prepare latent variables + num_channels_latents = self.unet.config.in_channels + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + # Prepare extra step kwargs. + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + + # Perform the first backward process up to time T_1 + x_1_t1 = self.backward_loop( + timesteps=timesteps[: -t1 - 1], + prompt_embeds=prompt_embeds, + latents=latents, + guidance_scale=guidance_scale, + callback=callback, + callback_steps=callback_steps, + extra_step_kwargs=extra_step_kwargs, + num_warmup_steps=num_warmup_steps, + ) + scheduler_copy = copy.deepcopy(self.scheduler) + + # Perform the second backward process up to time T_0 + x_1_t0 = self.backward_loop( + timesteps=timesteps[-t1 - 1 : -t0 - 1], + prompt_embeds=prompt_embeds, + latents=x_1_t1, + guidance_scale=guidance_scale, + callback=callback, + callback_steps=callback_steps, + extra_step_kwargs=extra_step_kwargs, + num_warmup_steps=0, + ) + + # Propagate first frame latents at time T_0 to remaining frames + x_2k_t0 = x_1_t0.repeat(video_length - 1, 1, 1, 1) + + # Add motion in latents at time T_0 + x_2k_t0 = create_motion_field_and_warp_latents( + motion_field_strength_x=motion_field_strength_x, + motion_field_strength_y=motion_field_strength_y, + latents=x_2k_t0, + frame_ids=frame_ids[1:], + ) + + # Perform forward process up to time T_1 + x_2k_t1 = self.forward_loop( + x_t0=x_2k_t0, + t0=timesteps[-t0 - 1].item(), + t1=timesteps[-t1 - 1].item(), + generator=generator, + ) + + # Perform backward process from time T_1 to 0 + x_1k_t1 = torch.cat([x_1_t1, x_2k_t1]) + b, l, d = prompt_embeds.size() + prompt_embeds = prompt_embeds[:, None].repeat(1, video_length, 1, 1).reshape(b * video_length, l, d) + + self.scheduler = scheduler_copy + x_1k_0 = self.backward_loop( + timesteps=timesteps[-t1 - 1 :], + prompt_embeds=prompt_embeds, + latents=x_1k_t1, + guidance_scale=guidance_scale, + callback=callback, + callback_steps=callback_steps, + extra_step_kwargs=extra_step_kwargs, + num_warmup_steps=0, + ) + latents = x_1k_0 + + # manually for max memory savings + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.unet.to("cpu") + torch.cuda.empty_cache() + + if output_type == "latent": + image = latents + has_nsfw_concept = None + else: + image = self.decode_latents(latents) + # Run safety checker + image, has_nsfw_concept = self.run_safety_checker(image, device, prompt_embeds.dtype) + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return (image, has_nsfw_concept) + + return TextToVideoPipelineOutput(images=image, nsfw_content_detected=has_nsfw_concept) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.run_safety_checker + def run_safety_checker(self, image, device, dtype): + if self.safety_checker is None: + has_nsfw_concept = None + else: + if torch.is_tensor(image): + feature_extractor_input = self.image_processor.postprocess(image, output_type="pil") + else: + feature_extractor_input = self.image_processor.numpy_to_pil(image) + safety_checker_input = self.feature_extractor(feature_extractor_input, return_tensors="pt").to(device) + image, has_nsfw_concept = self.safety_checker( + images=image, clip_input=safety_checker_input.pixel_values.to(dtype) + ) + return image, has_nsfw_concept + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.tokenizer) + + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + def decode_latents(self, latents): + latents = 1 / self.vae.config.scaling_factor * latents + image = self.vae.decode(latents, return_dict=False)[0] + image = (image / 2 + 0.5).clamp(0, 1) + # we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16 + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + return image diff --git a/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py new file mode 100755 index 0000000..eaa2760 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/text_to_video_synthesis/pipeline_text_to_video_zero_sdxl.py @@ -0,0 +1,1315 @@ +import copy +import inspect +from dataclasses import dataclass +from typing import Any, Callable, Dict, List, Optional, Tuple, Union + +import numpy as np +import PIL +import torch +import torch.nn.functional as F +from torch.nn.functional import grid_sample +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from ...image_processor import VaeImageProcessor +from ...loaders import StableDiffusionXLLoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL, UNet2DConditionModel +from ...models.attention_processor import ( + AttnProcessor2_0, + FusedAttnProcessor2_0, + LoRAAttnProcessor2_0, + LoRAXFormersAttnProcessor, + XFormersAttnProcessor, +) +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import ( + USE_PEFT_BACKEND, + BaseOutput, + is_invisible_watermark_available, + logging, + scale_lora_layers, + unscale_lora_layers, +) +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, StableDiffusionMixin + + +if is_invisible_watermark_available(): + from ..stable_diffusion_xl.watermark import StableDiffusionXLWatermarker + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.rearrange_0 +def rearrange_0(tensor, f): + F, C, H, W = tensor.size() + tensor = torch.permute(torch.reshape(tensor, (F // f, f, C, H, W)), (0, 2, 1, 3, 4)) + return tensor + + +# Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.rearrange_1 +def rearrange_1(tensor): + B, C, F, H, W = tensor.size() + return torch.reshape(torch.permute(tensor, (0, 2, 1, 3, 4)), (B * F, C, H, W)) + + +# Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.rearrange_3 +def rearrange_3(tensor, f): + F, D, C = tensor.size() + return torch.reshape(tensor, (F // f, f, D, C)) + + +# Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.rearrange_4 +def rearrange_4(tensor): + B, F, D, C = tensor.size() + return torch.reshape(tensor, (B * F, D, C)) + + +# Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.CrossFrameAttnProcessor +class CrossFrameAttnProcessor: + """ + Cross frame attention processor. Each frame attends the first frame. + + Args: + batch_size: The number that represents actual batch size, other than the frames. + For example, calling unet with a single prompt and num_images_per_prompt=1, batch_size should be equal to + 2, due to classifier-free guidance. + """ + + def __init__(self, batch_size=2): + self.batch_size = batch_size + + def __call__(self, attn, hidden_states, encoder_hidden_states=None, attention_mask=None): + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + query = attn.to_q(hidden_states) + + is_cross_attention = encoder_hidden_states is not None + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + # Cross Frame Attention + if not is_cross_attention: + video_length = key.size()[0] // self.batch_size + first_frame_index = [0] * video_length + + # rearrange keys to have batch and frames in the 1st and 2nd dims respectively + key = rearrange_3(key, video_length) + key = key[:, first_frame_index] + # rearrange values to have batch and frames in the 1st and 2nd dims respectively + value = rearrange_3(value, video_length) + value = value[:, first_frame_index] + + # rearrange back to original shape + key = rearrange_4(key) + value = rearrange_4(value) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + return hidden_states + + +# Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.CrossFrameAttnProcessor2_0 +class CrossFrameAttnProcessor2_0: + """ + Cross frame attention processor with scaled_dot_product attention of Pytorch 2.0. + + Args: + batch_size: The number that represents actual batch size, other than the frames. + For example, calling unet with a single prompt and num_images_per_prompt=1, batch_size should be equal to + 2, due to classifier-free guidance. + """ + + def __init__(self, batch_size=2): + if not hasattr(F, "scaled_dot_product_attention"): + raise ImportError("AttnProcessor2_0 requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.") + self.batch_size = batch_size + + def __call__(self, attn, hidden_states, encoder_hidden_states=None, attention_mask=None): + batch_size, sequence_length, _ = ( + hidden_states.shape if encoder_hidden_states is None else encoder_hidden_states.shape + ) + inner_dim = hidden_states.shape[-1] + + if attention_mask is not None: + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + # scaled_dot_product_attention expects attention_mask shape to be + # (batch, heads, source_length, target_length) + attention_mask = attention_mask.view(batch_size, attn.heads, -1, attention_mask.shape[-1]) + + query = attn.to_q(hidden_states) + + is_cross_attention = encoder_hidden_states is not None + if encoder_hidden_states is None: + encoder_hidden_states = hidden_states + elif attn.norm_cross: + encoder_hidden_states = attn.norm_encoder_hidden_states(encoder_hidden_states) + + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + # Cross Frame Attention + if not is_cross_attention: + video_length = max(1, key.size()[0] // self.batch_size) + first_frame_index = [0] * video_length + + # rearrange keys to have batch and frames in the 1st and 2nd dims respectively + key = rearrange_3(key, video_length) + key = key[:, first_frame_index] + # rearrange values to have batch and frames in the 1st and 2nd dims respectively + value = rearrange_3(value, video_length) + value = value[:, first_frame_index] + + # rearrange back to original shape + key = rearrange_4(key) + value = rearrange_4(value) + + head_dim = inner_dim // attn.heads + query = query.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + key = key.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + value = value.view(batch_size, -1, attn.heads, head_dim).transpose(1, 2) + + # the output of sdp = (batch, num_heads, seq_len, head_dim) + # TODO: add support for attn.scale when we move to Torch 2.1 + hidden_states = F.scaled_dot_product_attention( + query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False + ) + + hidden_states = hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim) + hidden_states = hidden_states.to(query.dtype) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + return hidden_states + + +@dataclass +class TextToVideoSDXLPipelineOutput(BaseOutput): + """ + Output class for zero-shot text-to-video pipeline. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or numpy array of shape `(batch_size, height, width, + num_channels)`. PIL images or numpy array present the denoised images of the diffusion pipeline. + """ + + images: Union[List[PIL.Image.Image], np.ndarray] + + +# Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.coords_grid +def coords_grid(batch, ht, wd, device): + # Adapted from https://github.com/princeton-vl/RAFT/blob/master/core/utils/utils.py + coords = torch.meshgrid(torch.arange(ht, device=device), torch.arange(wd, device=device)) + coords = torch.stack(coords[::-1], dim=0).float() + return coords[None].repeat(batch, 1, 1, 1) + + +# Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.warp_single_latent +def warp_single_latent(latent, reference_flow): + """ + Warp latent of a single frame with given flow + + Args: + latent: latent code of a single frame + reference_flow: flow which to warp the latent with + + Returns: + warped: warped latent + """ + _, _, H, W = reference_flow.size() + _, _, h, w = latent.size() + coords0 = coords_grid(1, H, W, device=latent.device).to(latent.dtype) + + coords_t0 = coords0 + reference_flow + coords_t0[:, 0] /= W + coords_t0[:, 1] /= H + + coords_t0 = coords_t0 * 2.0 - 1.0 + coords_t0 = F.interpolate(coords_t0, size=(h, w), mode="bilinear") + coords_t0 = torch.permute(coords_t0, (0, 2, 3, 1)) + + warped = grid_sample(latent, coords_t0, mode="nearest", padding_mode="reflection") + return warped + + +# Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.create_motion_field +def create_motion_field(motion_field_strength_x, motion_field_strength_y, frame_ids, device, dtype): + """ + Create translation motion field + + Args: + motion_field_strength_x: motion strength along x-axis + motion_field_strength_y: motion strength along y-axis + frame_ids: indexes of the frames the latents of which are being processed. + This is needed when we perform chunk-by-chunk inference + device: device + dtype: dtype + + Returns: + + """ + seq_length = len(frame_ids) + reference_flow = torch.zeros((seq_length, 2, 512, 512), device=device, dtype=dtype) + for fr_idx in range(seq_length): + reference_flow[fr_idx, 0, :, :] = motion_field_strength_x * (frame_ids[fr_idx]) + reference_flow[fr_idx, 1, :, :] = motion_field_strength_y * (frame_ids[fr_idx]) + return reference_flow + + +# Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.create_motion_field_and_warp_latents +def create_motion_field_and_warp_latents(motion_field_strength_x, motion_field_strength_y, frame_ids, latents): + """ + Creates translation motion and warps the latents accordingly + + Args: + motion_field_strength_x: motion strength along x-axis + motion_field_strength_y: motion strength along y-axis + frame_ids: indexes of the frames the latents of which are being processed. + This is needed when we perform chunk-by-chunk inference + latents: latent codes of frames + + Returns: + warped_latents: warped latents + """ + motion_field = create_motion_field( + motion_field_strength_x=motion_field_strength_x, + motion_field_strength_y=motion_field_strength_y, + frame_ids=frame_ids, + device=latents.device, + dtype=latents.dtype, + ) + warped_latents = latents.clone().detach() + for i in range(len(warped_latents)): + warped_latents[i] = warp_single_latent(latents[i][None], motion_field[i][None]) + return warped_latents + + +# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.rescale_noise_cfg +def rescale_noise_cfg(noise_cfg, noise_pred_text, guidance_rescale=0.0): + """ + Rescale `noise_cfg` according to `guidance_rescale`. Based on findings of [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). See Section 3.4 + """ + std_text = noise_pred_text.std(dim=list(range(1, noise_pred_text.ndim)), keepdim=True) + std_cfg = noise_cfg.std(dim=list(range(1, noise_cfg.ndim)), keepdim=True) + # rescale the results from guidance (fixes overexposure) + noise_pred_rescaled = noise_cfg * (std_text / std_cfg) + # mix with the original results from guidance by factor guidance_rescale to avoid "plain looking" images + noise_cfg = guidance_rescale * noise_pred_rescaled + (1 - guidance_rescale) * noise_cfg + return noise_cfg + + +class TextToVideoZeroSDXLPipeline( + DiffusionPipeline, + StableDiffusionMixin, + StableDiffusionXLLoraLoaderMixin, + TextualInversionLoaderMixin, +): + r""" + Pipeline for zero-shot text-to-video generation using Stable Diffusion XL. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) Model to encode and decode images to and from latent representations. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder. Stable Diffusion XL uses the text portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModel), specifically + the [clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14) variant. + text_encoder_2 ([` CLIPTextModelWithProjection`]): + Second frozen text-encoder. Stable Diffusion XL uses the text and pool portion of + [CLIP](https://huggingface.co/docs/transformers/model_doc/clip#transformers.CLIPTextModelWithProjection), + specifically the + [laion/CLIP-ViT-bigG-14-laion2B-39B-b160k](https://huggingface.co/laion/CLIP-ViT-bigG-14-laion2B-39B-b160k) + variant. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + tokenizer_2 (`CLIPTokenizer`): + Second Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + unet ([`UNet2DConditionModel`]): Conditional U-Net architecture to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image latents. Can be one of + [`DDIMScheduler`], [`LMSDiscreteScheduler`], or [`PNDMScheduler`]. + """ + + model_cpu_offload_seq = "text_encoder->text_encoder_2->unet->vae" + _optional_components = [ + "tokenizer", + "tokenizer_2", + "text_encoder", + "text_encoder_2", + "image_encoder", + "feature_extractor", + ] + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + text_encoder_2: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + tokenizer_2: CLIPTokenizer, + unet: UNet2DConditionModel, + scheduler: KarrasDiffusionSchedulers, + image_encoder: CLIPVisionModelWithProjection = None, + feature_extractor: CLIPImageProcessor = None, + force_zeros_for_empty_prompt: bool = True, + add_watermarker: Optional[bool] = None, + ): + super().__init__() + self.register_modules( + vae=vae, + text_encoder=text_encoder, + text_encoder_2=text_encoder_2, + tokenizer=tokenizer, + tokenizer_2=tokenizer_2, + unet=unet, + scheduler=scheduler, + image_encoder=image_encoder, + feature_extractor=feature_extractor, + ) + self.register_to_config(force_zeros_for_empty_prompt=force_zeros_for_empty_prompt) + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + self.default_sample_size = self.unet.config.sample_size + + add_watermarker = add_watermarker if add_watermarker is not None else is_invisible_watermark_available() + + if add_watermarker: + self.watermark = StableDiffusionXLWatermarker() + else: + self.watermark = None + + processor = ( + CrossFrameAttnProcessor2_0(batch_size=2) + if hasattr(F, "scaled_dot_product_attention") + else CrossFrameAttnProcessor(batch_size=2) + ) + + self.unet.set_attn_processor(processor) + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.upcast_vae + def upcast_vae(self): + dtype = self.vae.dtype + self.vae.to(dtype=torch.float32) + use_torch_2_0_or_xformers = isinstance( + self.vae.decoder.mid_block.attentions[0].processor, + ( + AttnProcessor2_0, + XFormersAttnProcessor, + LoRAXFormersAttnProcessor, + LoRAAttnProcessor2_0, + FusedAttnProcessor2_0, + ), + ) + # if xformers or torch_2_0 is used attention block does not need + # to be in float32 which can save lots of memory + if use_torch_2_0_or_xformers: + self.vae.post_quant_conv.to(dtype) + self.vae.decoder.conv_in.to(dtype) + self.vae.decoder.mid_block.to(dtype) + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline._get_add_time_ids + def _get_add_time_ids( + self, original_size, crops_coords_top_left, target_size, dtype, text_encoder_projection_dim=None + ): + add_time_ids = list(original_size + crops_coords_top_left + target_size) + + passed_add_embed_dim = ( + self.unet.config.addition_time_embed_dim * len(add_time_ids) + text_encoder_projection_dim + ) + expected_add_embed_dim = self.unet.add_embedding.linear_1.in_features + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], dtype=dtype) + return add_time_ids + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + def prepare_latents(self, batch_size, num_channels_latents, height, width, dtype, device, generator, latents=None): + shape = (batch_size, num_channels_latents, height // self.vae_scale_factor, width // self.vae_scale_factor) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + latents = latents.to(device) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def check_inputs( + self, + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt=None, + negative_prompt_2=None, + prompt_embeds=None, + negative_prompt_embeds=None, + pooled_prompt_embeds=None, + negative_pooled_prompt_embeds=None, + callback_on_step_end_tensor_inputs=None, + ): + if height % 8 != 0 or width % 8 != 0: + raise ValueError(f"`height` and `width` have to be divisible by 8 but are {height} and {width}.") + + if callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt_2 is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt_2`: {prompt_2} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + elif prompt_2 is not None and (not isinstance(prompt_2, str) and not isinstance(prompt_2, list)): + raise ValueError(f"`prompt_2` has to be of type `str` or `list` but is {type(prompt_2)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + elif negative_prompt_2 is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt_2`: {negative_prompt_2} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if prompt_embeds is not None and pooled_prompt_embeds is None: + raise ValueError( + "If `prompt_embeds` are provided, `pooled_prompt_embeds` also have to be passed. Make sure to generate `pooled_prompt_embeds` from the same text encoder that was used to generate `prompt_embeds`." + ) + + if negative_prompt_embeds is not None and negative_pooled_prompt_embeds is None: + raise ValueError( + "If `negative_prompt_embeds` are provided, `negative_pooled_prompt_embeds` also have to be passed. Make sure to generate `negative_pooled_prompt_embeds` from the same text encoder that was used to generate `negative_prompt_embeds`." + ) + + # Copied from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl.StableDiffusionXLPipeline.encode_prompt + def encode_prompt( + self, + prompt: str, + prompt_2: Optional[str] = None, + device: Optional[torch.device] = None, + num_images_per_prompt: int = 1, + do_classifier_free_guidance: bool = True, + negative_prompt: Optional[str] = None, + negative_prompt_2: Optional[str] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + lora_scale (`float`, *optional*): + A lora scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + device = device or self._execution_device + + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, StableDiffusionXLLoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if self.text_encoder is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder_2, lora_scale) + else: + scale_lora_layers(self.text_encoder_2, lora_scale) + + prompt = [prompt] if isinstance(prompt, str) else prompt + + if prompt is not None: + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # Define tokenizers and text encoders + tokenizers = [self.tokenizer, self.tokenizer_2] if self.tokenizer is not None else [self.tokenizer_2] + text_encoders = ( + [self.text_encoder, self.text_encoder_2] if self.text_encoder is not None else [self.text_encoder_2] + ) + + if prompt_embeds is None: + prompt_2 = prompt_2 or prompt + prompt_2 = [prompt_2] if isinstance(prompt_2, str) else prompt_2 + + # textual inversion: process multi-vector tokens if necessary + prompt_embeds_list = [] + prompts = [prompt, prompt_2] + for prompt, tokenizer, text_encoder in zip(prompts, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, tokenizer) + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + untruncated_ids = tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = tokenizer.batch_decode(untruncated_ids[:, tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {tokenizer.model_max_length} tokens: {removed_text}" + ) + + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + + # We are only ALWAYS interested in the pooled output of the final text encoder + pooled_prompt_embeds = prompt_embeds[0] + if clip_skip is None: + prompt_embeds = prompt_embeds.hidden_states[-2] + else: + # "2" because SDXL always indexes from the penultimate layer. + prompt_embeds = prompt_embeds.hidden_states[-(clip_skip + 2)] + + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + # get unconditional embeddings for classifier free guidance + zero_out_negative_prompt = negative_prompt is None and self.config.force_zeros_for_empty_prompt + if do_classifier_free_guidance and negative_prompt_embeds is None and zero_out_negative_prompt: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + elif do_classifier_free_guidance and negative_prompt_embeds is None: + negative_prompt = negative_prompt or "" + negative_prompt_2 = negative_prompt_2 or negative_prompt + + # normalize str to list + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + negative_prompt_2 = ( + batch_size * [negative_prompt_2] if isinstance(negative_prompt_2, str) else negative_prompt_2 + ) + + uncond_tokens: List[str] + if prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = [negative_prompt, negative_prompt_2] + + negative_prompt_embeds_list = [] + for negative_prompt, tokenizer, text_encoder in zip(uncond_tokens, tokenizers, text_encoders): + if isinstance(self, TextualInversionLoaderMixin): + negative_prompt = self.maybe_convert_prompt(negative_prompt, tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder( + uncond_input.input_ids.to(device), + output_hidden_states=True, + ) + # We are only ALWAYS interested in the pooled output of the final text encoder + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + if self.text_encoder_2 is not None: + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + prompt_embeds = prompt_embeds.to(dtype=self.unet.dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + if self.text_encoder_2 is not None: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder_2.dtype, device=device) + else: + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.unet.dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + if do_classifier_free_guidance: + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + if self.text_encoder is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + if self.text_encoder_2 is not None: + if isinstance(self, StableDiffusionXLLoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder_2, lora_scale) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + # Copied from diffusers.pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.TextToVideoZeroPipeline.forward_loop + def forward_loop(self, x_t0, t0, t1, generator): + """ + Perform DDPM forward process from time t0 to t1. This is the same as adding noise with corresponding variance. + + Args: + x_t0: + Latent code at time t0. + t0: + Timestep at t0. + t1: + Timestamp at t1. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + + Returns: + x_t1: + Forward process applied to x_t0 from time t0 to t1. + """ + eps = randn_tensor(x_t0.size(), generator=generator, dtype=x_t0.dtype, device=x_t0.device) + alpha_vec = torch.prod(self.scheduler.alphas[t0:t1]) + x_t1 = torch.sqrt(alpha_vec) * x_t0 + torch.sqrt(1 - alpha_vec) * eps + return x_t1 + + def backward_loop( + self, + latents, + timesteps, + prompt_embeds, + guidance_scale, + callback, + callback_steps, + num_warmup_steps, + extra_step_kwargs, + add_text_embeds, + add_time_ids, + cross_attention_kwargs=None, + guidance_rescale: float = 0.0, + ): + """ + Perform backward process given list of time steps + + Args: + latents: + Latents at time timesteps[0]. + timesteps: + Time steps along which to perform backward process. + prompt_embeds: + Pre-generated text embeddings. + guidance_scale: + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + extra_step_kwargs: + Extra_step_kwargs. + cross_attention_kwargs: + A kwargs dictionary that if specified is passed along to the [`AttentionProcessor`] as defined in + [`self.processor`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/attention_processor.py). + num_warmup_steps: + number of warmup steps. + + Returns: + latents: latents of backward process output at time timesteps[-1] + """ + + do_classifier_free_guidance = guidance_scale > 1.0 + num_steps = (len(timesteps) - num_warmup_steps) // self.scheduler.order + + with self.progress_bar(total=num_steps) as progress_bar: + for i, t in enumerate(timesteps): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([latents] * 2) if do_classifier_free_guidance else latents + latent_model_input = self.scheduler.scale_model_input(latent_model_input, t) + + # predict the noise residual + added_cond_kwargs = {"text_embeds": add_text_embeds, "time_ids": add_time_ids} + noise_pred = self.unet( + latent_model_input, + t, + encoder_hidden_states=prompt_embeds, + cross_attention_kwargs=cross_attention_kwargs, + added_cond_kwargs=added_cond_kwargs, + return_dict=False, + )[0] + + # perform guidance + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond) + + if do_classifier_free_guidance and guidance_rescale > 0.0: + # Based on 3.4. in https://arxiv.org/pdf/2305.08891.pdf + noise_pred = rescale_noise_cfg(noise_pred, noise_pred_text, guidance_rescale=guidance_rescale) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + callback(i, t, latents) + return latents.clone().detach() + + @torch.no_grad() + def __call__( + self, + prompt: Union[str, List[str]], + prompt_2: Optional[Union[str, List[str]]] = None, + video_length: Optional[int] = 8, + height: Optional[int] = None, + width: Optional[int] = None, + num_inference_steps: int = 50, + denoising_end: Optional[float] = None, + guidance_scale: float = 7.5, + negative_prompt: Optional[Union[str, List[str]]] = None, + negative_prompt_2: Optional[Union[str, List[str]]] = None, + num_videos_per_prompt: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + frame_ids: Optional[List[int]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + negative_pooled_prompt_embeds: Optional[torch.FloatTensor] = None, + latents: Optional[torch.FloatTensor] = None, + motion_field_strength_x: float = 12, + motion_field_strength_y: float = 12, + output_type: Optional[str] = "tensor", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + cross_attention_kwargs: Optional[Dict[str, Any]] = None, + guidance_rescale: float = 0.0, + original_size: Optional[Tuple[int, int]] = None, + crops_coords_top_left: Tuple[int, int] = (0, 0), + target_size: Optional[Tuple[int, int]] = None, + t0: int = 44, + t1: int = 47, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide the image generation. If not defined, one has to pass `prompt_embeds`. + instead. + prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts to be sent to the `tokenizer_2` and `text_encoder_2`. If not defined, `prompt` is + used in both text-encoders + video_length (`int`, *optional*, defaults to 8): + The number of generated video frames. + height (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to self.unet.config.sample_size * self.vae_scale_factor): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + denoising_end (`float`, *optional*): + When specified, determines the fraction (between 0.0 and 1.0) of the total denoising process to be + completed before it is intentionally prematurely terminated. As a result, the returned sample will + still retain a substantial amount of noise as determined by the discrete timesteps selected by the + scheduler. The denoising_end parameter should ideally be utilized when this pipeline forms a part of a + "Mixture of Denoisers" multi-pipeline setup, as elaborated in [**Refining the Image + Output**](https://huggingface.co/docs/diffusers/api/pipelines/stable_diffusion/stable_diffusion_xl#refining-the-image-output) + guidance_scale (`float`, *optional*, defaults to 7.5): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + negative_prompt_2 (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation to be sent to `tokenizer_2` and + `text_encoder_2`. If not defined, `negative_prompt` is used in both text-encoders + num_videos_per_prompt (`int`, *optional*, defaults to 1): + The number of videos to generate per prompt. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) in the DDIM paper: https://arxiv.org/abs/2010.02502. Only applies to + [`schedulers.DDIMScheduler`], will be ignored for others. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + frame_ids (`List[int]`, *optional*): + Indexes of the frames that are being generated. This is used when generating longer videos + chunk-by-chunk. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. + If not provided, pooled text embeddings will be generated from `prompt` input argument. + negative_pooled_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative pooled text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, pooled negative_prompt_embeds will be generated from `negative_prompt` + input argument. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + motion_field_strength_x (`float`, *optional*, defaults to 12): + Strength of motion in generated video along x-axis. See the [paper](https://arxiv.org/abs/2303.13439), + Sect. 3.3.1. + motion_field_strength_y (`float`, *optional*, defaults to 12): + Strength of motion in generated video along y-axis. See the [paper](https://arxiv.org/abs/2303.13439), + Sect. 3.3.1. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.stable_diffusion_xl.StableDiffusionXLPipelineOutput`] instead + of a plain tuple. + callback (`Callable`, *optional*): + A function that will be called every `callback_steps` steps during inference. The function will be + called with the following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function will be called. If not specified, the callback will be + called at every step. + cross_attention_kwargs (`dict`, *optional*): + A kwargs dictionary that if specified is passed along to the `AttentionProcessor` as defined under + `self.processor` in + [diffusers.cross_attention](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/cross_attention.py). + guidance_rescale (`float`, *optional*, defaults to 0.7): + Guidance rescale factor proposed by [Common Diffusion Noise Schedules and Sample Steps are + Flawed](https://arxiv.org/pdf/2305.08891.pdf) `guidance_scale` is defined as `φ` in equation 16. of + [Common Diffusion Noise Schedules and Sample Steps are Flawed](https://arxiv.org/pdf/2305.08891.pdf). + Guidance rescale factor should fix overexposure when using zero terminal SNR. + original_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + If `original_size` is not the same as `target_size` the image will appear to be down- or upsampled. + `original_size` defaults to `(width, height)` if not specified. Part of SDXL's micro-conditioning as + explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + crops_coords_top_left (`Tuple[int]`, *optional*, defaults to (0, 0)): + `crops_coords_top_left` can be used to generate an image that appears to be "cropped" from the position + `crops_coords_top_left` downwards. Favorable, well-centered images are usually achieved by setting + `crops_coords_top_left` to (0, 0). Part of SDXL's micro-conditioning as explained in section 2.2 of + [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + target_size (`Tuple[int]`, *optional*, defaults to (1024, 1024)): + For most cases, `target_size` should be set to the desired height and width of the generated image. If + not specified it will default to `(width, height)`. Part of SDXL's micro-conditioning as explained in + section 2.2 of [https://huggingface.co/papers/2307.01952](https://huggingface.co/papers/2307.01952). + t0 (`int`, *optional*, defaults to 44): + Timestep t0. Should be in the range [0, num_inference_steps - 1]. See the + [paper](https://arxiv.org/abs/2303.13439), Sect. 3.3.1. + t1 (`int`, *optional*, defaults to 47): + Timestep t0. Should be in the range [t0 + 1, num_inference_steps - 1]. See the + [paper](https://arxiv.org/abs/2303.13439), Sect. 3.3.1. + + Returns: + [`~pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.TextToVideoSDXLPipelineOutput`] or + `tuple`: [`~pipelines.text_to_video_synthesis.pipeline_text_to_video_zero.TextToVideoSDXLPipelineOutput`] + if `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the + generated images. + """ + assert video_length > 0 + if frame_ids is None: + frame_ids = list(range(video_length)) + assert len(frame_ids) == video_length + + assert num_videos_per_prompt == 1 + + if isinstance(prompt, str): + prompt = [prompt] + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + + # 0. Default height and width to unet + height = height or self.default_sample_size * self.vae_scale_factor + width = width or self.default_sample_size * self.vae_scale_factor + + original_size = original_size or (height, width) + target_size = target_size or (height, width) + + # 1. Check inputs. Raise error if not correct + self.check_inputs( + prompt, + prompt_2, + height, + width, + callback_steps, + negative_prompt, + negative_prompt_2, + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) + + # 2. Define call parameters + batch_size = ( + 1 if isinstance(prompt, str) else len(prompt) if isinstance(prompt, list) else prompt_embeds.shape[0] + ) + device = self._execution_device + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + do_classifier_free_guidance = guidance_scale > 1.0 + + # 3. Encode input prompt + text_encoder_lora_scale = ( + cross_attention_kwargs.get("scale", None) if cross_attention_kwargs is not None else None + ) + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt( + prompt=prompt, + prompt_2=prompt_2, + device=device, + num_images_per_prompt=num_videos_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + negative_prompt_2=negative_prompt_2, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + lora_scale=text_encoder_lora_scale, + ) + + # 4. Prepare timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latent variables + num_channels_latents = self.unet.config.in_channels + + latents = self.prepare_latents( + batch_size * num_videos_per_prompt, + num_channels_latents, + height, + width, + prompt_embeds.dtype, + device, + generator, + latents, + ) + + # 6. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + # 7. Prepare added time ids & embeddings + add_text_embeds = pooled_prompt_embeds + if self.text_encoder_2 is None: + text_encoder_projection_dim = int(pooled_prompt_embeds.shape[-1]) + else: + text_encoder_projection_dim = self.text_encoder_2.config.projection_dim + + add_time_ids = self._get_add_time_ids( + original_size, + crops_coords_top_left, + target_size, + dtype=prompt_embeds.dtype, + text_encoder_projection_dim=text_encoder_projection_dim, + ) + + if do_classifier_free_guidance: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0) + add_text_embeds = torch.cat([negative_pooled_prompt_embeds, add_text_embeds], dim=0) + add_time_ids = torch.cat([add_time_ids, add_time_ids], dim=0) + + prompt_embeds = prompt_embeds.to(device) + add_text_embeds = add_text_embeds.to(device) + add_time_ids = add_time_ids.to(device).repeat(batch_size * num_videos_per_prompt, 1) + + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + + # Perform the first backward process up to time T_1 + x_1_t1 = self.backward_loop( + timesteps=timesteps[: -t1 - 1], + prompt_embeds=prompt_embeds, + latents=latents, + guidance_scale=guidance_scale, + callback=callback, + callback_steps=callback_steps, + extra_step_kwargs=extra_step_kwargs, + num_warmup_steps=num_warmup_steps, + add_text_embeds=add_text_embeds, + add_time_ids=add_time_ids, + ) + + scheduler_copy = copy.deepcopy(self.scheduler) + + # Perform the second backward process up to time T_0 + x_1_t0 = self.backward_loop( + timesteps=timesteps[-t1 - 1 : -t0 - 1], + prompt_embeds=prompt_embeds, + latents=x_1_t1, + guidance_scale=guidance_scale, + callback=callback, + callback_steps=callback_steps, + extra_step_kwargs=extra_step_kwargs, + num_warmup_steps=0, + add_text_embeds=add_text_embeds, + add_time_ids=add_time_ids, + ) + + # Propagate first frame latents at time T_0 to remaining frames + x_2k_t0 = x_1_t0.repeat(video_length - 1, 1, 1, 1) + + # Add motion in latents at time T_0 + x_2k_t0 = create_motion_field_and_warp_latents( + motion_field_strength_x=motion_field_strength_x, + motion_field_strength_y=motion_field_strength_y, + latents=x_2k_t0, + frame_ids=frame_ids[1:], + ) + + # Perform forward process up to time T_1 + x_2k_t1 = self.forward_loop( + x_t0=x_2k_t0, + t0=timesteps[-t0 - 1].to(torch.long), + t1=timesteps[-t1 - 1].to(torch.long), + generator=generator, + ) + + # Perform backward process from time T_1 to 0 + latents = torch.cat([x_1_t1, x_2k_t1]) + + self.scheduler = scheduler_copy + timesteps = timesteps[-t1 - 1 :] + + b, l, d = prompt_embeds.size() + prompt_embeds = prompt_embeds[:, None].repeat(1, video_length, 1, 1).reshape(b * video_length, l, d) + + b, k = add_text_embeds.size() + add_text_embeds = add_text_embeds[:, None].repeat(1, video_length, 1).reshape(b * video_length, k) + + b, k = add_time_ids.size() + add_time_ids = add_time_ids[:, None].repeat(1, video_length, 1).reshape(b * video_length, k) + + # 7.1 Apply denoising_end + if denoising_end is not None and isinstance(denoising_end, float) and denoising_end > 0 and denoising_end < 1: + discrete_timestep_cutoff = int( + round( + self.scheduler.config.num_train_timesteps + - (denoising_end * self.scheduler.config.num_train_timesteps) + ) + ) + num_inference_steps = len(list(filter(lambda ts: ts >= discrete_timestep_cutoff, timesteps))) + timesteps = timesteps[:num_inference_steps] + + x_1k_0 = self.backward_loop( + timesteps=timesteps, + prompt_embeds=prompt_embeds, + latents=latents, + guidance_scale=guidance_scale, + callback=callback, + callback_steps=callback_steps, + extra_step_kwargs=extra_step_kwargs, + num_warmup_steps=0, + add_text_embeds=add_text_embeds, + add_time_ids=add_time_ids, + ) + + latents = x_1k_0 + + if not output_type == "latent": + # make sure the VAE is in float32 mode, as it overflows in float16 + needs_upcasting = self.vae.dtype == torch.float16 and self.vae.config.force_upcast + + if needs_upcasting: + self.upcast_vae() + latents = latents.to(next(iter(self.vae.post_quant_conv.parameters())).dtype) + + image = self.vae.decode(latents / self.vae.config.scaling_factor, return_dict=False)[0] + + # cast back to fp16 if needed + if needs_upcasting: + self.vae.to(dtype=torch.float16) + else: + image = latents + return TextToVideoSDXLPipelineOutput(images=image) + + # apply watermark if available + if self.watermark is not None: + image = self.watermark.apply_watermark(image) + + image = self.image_processor.postprocess(image, output_type=output_type) + + # Offload last model to CPU manually for max memory savings + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.final_offload_hook.offload() + + if not return_dict: + return (image,) + + return TextToVideoSDXLPipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/unclip/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/unclip/__init__.py new file mode 100755 index 0000000..c89e899 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/unclip/__init__.py @@ -0,0 +1,52 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + is_torch_available, + is_transformers_available, + is_transformers_version, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import UnCLIPImageVariationPipeline, UnCLIPPipeline + + _dummy_objects.update( + {"UnCLIPImageVariationPipeline": UnCLIPImageVariationPipeline, "UnCLIPPipeline": UnCLIPPipeline} + ) +else: + _import_structure["pipeline_unclip"] = ["UnCLIPPipeline"] + _import_structure["pipeline_unclip_image_variation"] = ["UnCLIPImageVariationPipeline"] + _import_structure["text_proj"] = ["UnCLIPTextProjModel"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available() and is_transformers_version(">=", "4.25.0")): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 + else: + from .pipeline_unclip import UnCLIPPipeline + from .pipeline_unclip_image_variation import UnCLIPImageVariationPipeline + from .text_proj import UnCLIPTextProjModel + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/unclip/pipeline_unclip.py b/diffusers-0.27.0/src/diffusers/pipelines/unclip/pipeline_unclip.py new file mode 100755 index 0000000..72e5b31 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/unclip/pipeline_unclip.py @@ -0,0 +1,493 @@ +# Copyright 2024 Kakao Brain and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import List, Optional, Tuple, Union + +import torch +from torch.nn import functional as F +from transformers import CLIPTextModelWithProjection, CLIPTokenizer +from transformers.models.clip.modeling_clip import CLIPTextModelOutput + +from ...models import PriorTransformer, UNet2DConditionModel, UNet2DModel +from ...schedulers import UnCLIPScheduler +from ...utils import logging +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from .text_proj import UnCLIPTextProjModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class UnCLIPPipeline(DiffusionPipeline): + """ + Pipeline for text-to-image generation using unCLIP. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + text_encoder ([`~transformers.CLIPTextModelWithProjection`]): + Frozen text-encoder. + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + prior ([`PriorTransformer`]): + The canonical unCLIP prior to approximate the image embedding from the text embedding. + text_proj ([`UnCLIPTextProjModel`]): + Utility class to prepare and combine the embeddings before they are passed to the decoder. + decoder ([`UNet2DConditionModel`]): + The decoder to invert the image embedding into an image. + super_res_first ([`UNet2DModel`]): + Super resolution UNet. Used in all but the last step of the super resolution diffusion process. + super_res_last ([`UNet2DModel`]): + Super resolution UNet. Used in the last step of the super resolution diffusion process. + prior_scheduler ([`UnCLIPScheduler`]): + Scheduler used in the prior denoising process (a modified [`DDPMScheduler`]). + decoder_scheduler ([`UnCLIPScheduler`]): + Scheduler used in the decoder denoising process (a modified [`DDPMScheduler`]). + super_res_scheduler ([`UnCLIPScheduler`]): + Scheduler used in the super resolution denoising process (a modified [`DDPMScheduler`]). + + """ + + _exclude_from_cpu_offload = ["prior"] + + prior: PriorTransformer + decoder: UNet2DConditionModel + text_proj: UnCLIPTextProjModel + text_encoder: CLIPTextModelWithProjection + tokenizer: CLIPTokenizer + super_res_first: UNet2DModel + super_res_last: UNet2DModel + + prior_scheduler: UnCLIPScheduler + decoder_scheduler: UnCLIPScheduler + super_res_scheduler: UnCLIPScheduler + + model_cpu_offload_seq = "text_encoder->text_proj->decoder->super_res_first->super_res_last" + + def __init__( + self, + prior: PriorTransformer, + decoder: UNet2DConditionModel, + text_encoder: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + text_proj: UnCLIPTextProjModel, + super_res_first: UNet2DModel, + super_res_last: UNet2DModel, + prior_scheduler: UnCLIPScheduler, + decoder_scheduler: UnCLIPScheduler, + super_res_scheduler: UnCLIPScheduler, + ): + super().__init__() + + self.register_modules( + prior=prior, + decoder=decoder, + text_encoder=text_encoder, + tokenizer=tokenizer, + text_proj=text_proj, + super_res_first=super_res_first, + super_res_last=super_res_last, + prior_scheduler=prior_scheduler, + decoder_scheduler=decoder_scheduler, + super_res_scheduler=super_res_scheduler, + ) + + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + text_model_output: Optional[Union[CLIPTextModelOutput, Tuple]] = None, + text_attention_mask: Optional[torch.Tensor] = None, + ): + if text_model_output is None: + batch_size = len(prompt) if isinstance(prompt, list) else 1 + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + text_mask = text_inputs.attention_mask.bool().to(device) + + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + text_input_ids = text_input_ids[:, : self.tokenizer.model_max_length] + + text_encoder_output = self.text_encoder(text_input_ids.to(device)) + + prompt_embeds = text_encoder_output.text_embeds + text_enc_hid_states = text_encoder_output.last_hidden_state + + else: + batch_size = text_model_output[0].shape[0] + prompt_embeds, text_enc_hid_states = text_model_output[0], text_model_output[1] + text_mask = text_attention_mask + + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + text_enc_hid_states = text_enc_hid_states.repeat_interleave(num_images_per_prompt, dim=0) + text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + if do_classifier_free_guidance: + uncond_tokens = [""] * batch_size + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + uncond_text_mask = uncond_input.attention_mask.bool().to(device) + negative_prompt_embeds_text_encoder_output = self.text_encoder(uncond_input.input_ids.to(device)) + + negative_prompt_embeds = negative_prompt_embeds_text_encoder_output.text_embeds + uncond_text_enc_hid_states = negative_prompt_embeds_text_encoder_output.last_hidden_state + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len) + + seq_len = uncond_text_enc_hid_states.shape[1] + uncond_text_enc_hid_states = uncond_text_enc_hid_states.repeat(1, num_images_per_prompt, 1) + uncond_text_enc_hid_states = uncond_text_enc_hid_states.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + uncond_text_mask = uncond_text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + # done duplicates + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + text_enc_hid_states = torch.cat([uncond_text_enc_hid_states, text_enc_hid_states]) + + text_mask = torch.cat([uncond_text_mask, text_mask]) + + return prompt_embeds, text_enc_hid_states, text_mask + + @torch.no_grad() + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + prior_num_inference_steps: int = 25, + decoder_num_inference_steps: int = 25, + super_res_num_inference_steps: int = 7, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + prior_latents: Optional[torch.FloatTensor] = None, + decoder_latents: Optional[torch.FloatTensor] = None, + super_res_latents: Optional[torch.FloatTensor] = None, + text_model_output: Optional[Union[CLIPTextModelOutput, Tuple]] = None, + text_attention_mask: Optional[torch.Tensor] = None, + prior_guidance_scale: float = 4.0, + decoder_guidance_scale: float = 8.0, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ): + """ + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide image generation. This can only be left undefined if `text_model_output` + and `text_attention_mask` is passed. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + prior_num_inference_steps (`int`, *optional*, defaults to 25): + The number of denoising steps for the prior. More denoising steps usually lead to a higher quality + image at the expense of slower inference. + decoder_num_inference_steps (`int`, *optional*, defaults to 25): + The number of denoising steps for the decoder. More denoising steps usually lead to a higher quality + image at the expense of slower inference. + super_res_num_inference_steps (`int`, *optional*, defaults to 7): + The number of denoising steps for super resolution. More denoising steps usually lead to a higher + quality image at the expense of slower inference. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + prior_latents (`torch.FloatTensor` of shape (batch size, embeddings dimension), *optional*): + Pre-generated noisy latents to be used as inputs for the prior. + decoder_latents (`torch.FloatTensor` of shape (batch size, channels, height, width), *optional*): + Pre-generated noisy latents to be used as inputs for the decoder. + super_res_latents (`torch.FloatTensor` of shape (batch size, channels, super res height, super res width), *optional*): + Pre-generated noisy latents to be used as inputs for the decoder. + prior_guidance_scale (`float`, *optional*, defaults to 4.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + decoder_guidance_scale (`float`, *optional*, defaults to 4.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + text_model_output (`CLIPTextModelOutput`, *optional*): + Pre-defined [`CLIPTextModel`] outputs that can be derived from the text encoder. Pre-defined text + outputs can be passed for tasks like text embedding interpolations. Make sure to also pass + `text_attention_mask` in this case. `prompt` can the be left `None`. + text_attention_mask (`torch.Tensor`, *optional*): + Pre-defined CLIP text attention mask that can be derived from the tokenizer. Pre-defined text attention + masks are necessary when passing `text_model_output`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images. + """ + if prompt is not None: + if isinstance(prompt, str): + batch_size = 1 + elif isinstance(prompt, list): + batch_size = len(prompt) + else: + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + else: + batch_size = text_model_output[0].shape[0] + + device = self._execution_device + + batch_size = batch_size * num_images_per_prompt + + do_classifier_free_guidance = prior_guidance_scale > 1.0 or decoder_guidance_scale > 1.0 + + prompt_embeds, text_enc_hid_states, text_mask = self._encode_prompt( + prompt, device, num_images_per_prompt, do_classifier_free_guidance, text_model_output, text_attention_mask + ) + + # prior + + self.prior_scheduler.set_timesteps(prior_num_inference_steps, device=device) + prior_timesteps_tensor = self.prior_scheduler.timesteps + + embedding_dim = self.prior.config.embedding_dim + + prior_latents = self.prepare_latents( + (batch_size, embedding_dim), + prompt_embeds.dtype, + device, + generator, + prior_latents, + self.prior_scheduler, + ) + + for i, t in enumerate(self.progress_bar(prior_timesteps_tensor)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([prior_latents] * 2) if do_classifier_free_guidance else prior_latents + + predicted_image_embedding = self.prior( + latent_model_input, + timestep=t, + proj_embedding=prompt_embeds, + encoder_hidden_states=text_enc_hid_states, + attention_mask=text_mask, + ).predicted_image_embedding + + if do_classifier_free_guidance: + predicted_image_embedding_uncond, predicted_image_embedding_text = predicted_image_embedding.chunk(2) + predicted_image_embedding = predicted_image_embedding_uncond + prior_guidance_scale * ( + predicted_image_embedding_text - predicted_image_embedding_uncond + ) + + if i + 1 == prior_timesteps_tensor.shape[0]: + prev_timestep = None + else: + prev_timestep = prior_timesteps_tensor[i + 1] + + prior_latents = self.prior_scheduler.step( + predicted_image_embedding, + timestep=t, + sample=prior_latents, + generator=generator, + prev_timestep=prev_timestep, + ).prev_sample + + prior_latents = self.prior.post_process_latents(prior_latents) + + image_embeddings = prior_latents + + # done prior + + # decoder + + text_enc_hid_states, additive_clip_time_embeddings = self.text_proj( + image_embeddings=image_embeddings, + prompt_embeds=prompt_embeds, + text_encoder_hidden_states=text_enc_hid_states, + do_classifier_free_guidance=do_classifier_free_guidance, + ) + + if device.type == "mps": + # HACK: MPS: There is a panic when padding bool tensors, + # so cast to int tensor for the pad and back to bool afterwards + text_mask = text_mask.type(torch.int) + decoder_text_mask = F.pad(text_mask, (self.text_proj.clip_extra_context_tokens, 0), value=1) + decoder_text_mask = decoder_text_mask.type(torch.bool) + else: + decoder_text_mask = F.pad(text_mask, (self.text_proj.clip_extra_context_tokens, 0), value=True) + + self.decoder_scheduler.set_timesteps(decoder_num_inference_steps, device=device) + decoder_timesteps_tensor = self.decoder_scheduler.timesteps + + num_channels_latents = self.decoder.config.in_channels + height = self.decoder.config.sample_size + width = self.decoder.config.sample_size + + decoder_latents = self.prepare_latents( + (batch_size, num_channels_latents, height, width), + text_enc_hid_states.dtype, + device, + generator, + decoder_latents, + self.decoder_scheduler, + ) + + for i, t in enumerate(self.progress_bar(decoder_timesteps_tensor)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([decoder_latents] * 2) if do_classifier_free_guidance else decoder_latents + + noise_pred = self.decoder( + sample=latent_model_input, + timestep=t, + encoder_hidden_states=text_enc_hid_states, + class_labels=additive_clip_time_embeddings, + attention_mask=decoder_text_mask, + ).sample + + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred_uncond, _ = noise_pred_uncond.split(latent_model_input.shape[1], dim=1) + noise_pred_text, predicted_variance = noise_pred_text.split(latent_model_input.shape[1], dim=1) + noise_pred = noise_pred_uncond + decoder_guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, predicted_variance], dim=1) + + if i + 1 == decoder_timesteps_tensor.shape[0]: + prev_timestep = None + else: + prev_timestep = decoder_timesteps_tensor[i + 1] + + # compute the previous noisy sample x_t -> x_t-1 + decoder_latents = self.decoder_scheduler.step( + noise_pred, t, decoder_latents, prev_timestep=prev_timestep, generator=generator + ).prev_sample + + decoder_latents = decoder_latents.clamp(-1, 1) + + image_small = decoder_latents + + # done decoder + + # super res + + self.super_res_scheduler.set_timesteps(super_res_num_inference_steps, device=device) + super_res_timesteps_tensor = self.super_res_scheduler.timesteps + + channels = self.super_res_first.config.in_channels // 2 + height = self.super_res_first.config.sample_size + width = self.super_res_first.config.sample_size + + super_res_latents = self.prepare_latents( + (batch_size, channels, height, width), + image_small.dtype, + device, + generator, + super_res_latents, + self.super_res_scheduler, + ) + + if device.type == "mps": + # MPS does not support many interpolations + image_upscaled = F.interpolate(image_small, size=[height, width]) + else: + interpolate_antialias = {} + if "antialias" in inspect.signature(F.interpolate).parameters: + interpolate_antialias["antialias"] = True + + image_upscaled = F.interpolate( + image_small, size=[height, width], mode="bicubic", align_corners=False, **interpolate_antialias + ) + + for i, t in enumerate(self.progress_bar(super_res_timesteps_tensor)): + # no classifier free guidance + + if i == super_res_timesteps_tensor.shape[0] - 1: + unet = self.super_res_last + else: + unet = self.super_res_first + + latent_model_input = torch.cat([super_res_latents, image_upscaled], dim=1) + + noise_pred = unet( + sample=latent_model_input, + timestep=t, + ).sample + + if i + 1 == super_res_timesteps_tensor.shape[0]: + prev_timestep = None + else: + prev_timestep = super_res_timesteps_tensor[i + 1] + + # compute the previous noisy sample x_t -> x_t-1 + super_res_latents = self.super_res_scheduler.step( + noise_pred, t, super_res_latents, prev_timestep=prev_timestep, generator=generator + ).prev_sample + + image = super_res_latents + # done super res + + self.maybe_free_model_hooks() + + # post processing + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py b/diffusers-0.27.0/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py new file mode 100755 index 0000000..6c646a7 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/unclip/pipeline_unclip_image_variation.py @@ -0,0 +1,420 @@ +# Copyright 2024 Kakao Brain and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +from typing import List, Optional, Union + +import PIL.Image +import torch +from torch.nn import functional as F +from transformers import ( + CLIPImageProcessor, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionModelWithProjection, +) + +from ...models import UNet2DConditionModel, UNet2DModel +from ...schedulers import UnCLIPScheduler +from ...utils import logging +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from .text_proj import UnCLIPTextProjModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class UnCLIPImageVariationPipeline(DiffusionPipeline): + """ + Pipeline to generate image variations from an input image using UnCLIP. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + text_encoder ([`~transformers.CLIPTextModelWithProjection`]): + Frozen text-encoder. + tokenizer ([`~transformers.CLIPTokenizer`]): + A `CLIPTokenizer` to tokenize text. + feature_extractor ([`~transformers.CLIPImageProcessor`]): + Model that extracts features from generated images to be used as inputs for the `image_encoder`. + image_encoder ([`~transformers.CLIPVisionModelWithProjection`]): + Frozen CLIP image-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + text_proj ([`UnCLIPTextProjModel`]): + Utility class to prepare and combine the embeddings before they are passed to the decoder. + decoder ([`UNet2DConditionModel`]): + The decoder to invert the image embedding into an image. + super_res_first ([`UNet2DModel`]): + Super resolution UNet. Used in all but the last step of the super resolution diffusion process. + super_res_last ([`UNet2DModel`]): + Super resolution UNet. Used in the last step of the super resolution diffusion process. + decoder_scheduler ([`UnCLIPScheduler`]): + Scheduler used in the decoder denoising process (a modified [`DDPMScheduler`]). + super_res_scheduler ([`UnCLIPScheduler`]): + Scheduler used in the super resolution denoising process (a modified [`DDPMScheduler`]). + """ + + decoder: UNet2DConditionModel + text_proj: UnCLIPTextProjModel + text_encoder: CLIPTextModelWithProjection + tokenizer: CLIPTokenizer + feature_extractor: CLIPImageProcessor + image_encoder: CLIPVisionModelWithProjection + super_res_first: UNet2DModel + super_res_last: UNet2DModel + + decoder_scheduler: UnCLIPScheduler + super_res_scheduler: UnCLIPScheduler + model_cpu_offload_seq = "text_encoder->image_encoder->text_proj->decoder->super_res_first->super_res_last" + + def __init__( + self, + decoder: UNet2DConditionModel, + text_encoder: CLIPTextModelWithProjection, + tokenizer: CLIPTokenizer, + text_proj: UnCLIPTextProjModel, + feature_extractor: CLIPImageProcessor, + image_encoder: CLIPVisionModelWithProjection, + super_res_first: UNet2DModel, + super_res_last: UNet2DModel, + decoder_scheduler: UnCLIPScheduler, + super_res_scheduler: UnCLIPScheduler, + ): + super().__init__() + + self.register_modules( + decoder=decoder, + text_encoder=text_encoder, + tokenizer=tokenizer, + text_proj=text_proj, + feature_extractor=feature_extractor, + image_encoder=image_encoder, + super_res_first=super_res_first, + super_res_last=super_res_last, + decoder_scheduler=decoder_scheduler, + super_res_scheduler=super_res_scheduler, + ) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def _encode_prompt(self, prompt, device, num_images_per_prompt, do_classifier_free_guidance): + batch_size = len(prompt) if isinstance(prompt, list) else 1 + + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + text_mask = text_inputs.attention_mask.bool().to(device) + text_encoder_output = self.text_encoder(text_input_ids.to(device)) + + prompt_embeds = text_encoder_output.text_embeds + text_encoder_hidden_states = text_encoder_output.last_hidden_state + + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + text_encoder_hidden_states = text_encoder_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + text_mask = text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + if do_classifier_free_guidance: + uncond_tokens = [""] * batch_size + + max_length = text_input_ids.shape[-1] + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + uncond_text_mask = uncond_input.attention_mask.bool().to(device) + negative_prompt_embeds_text_encoder_output = self.text_encoder(uncond_input.input_ids.to(device)) + + negative_prompt_embeds = negative_prompt_embeds_text_encoder_output.text_embeds + uncond_text_encoder_hidden_states = negative_prompt_embeds_text_encoder_output.last_hidden_state + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len) + + seq_len = uncond_text_encoder_hidden_states.shape[1] + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.repeat(1, num_images_per_prompt, 1) + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + uncond_text_mask = uncond_text_mask.repeat_interleave(num_images_per_prompt, dim=0) + + # done duplicates + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + text_encoder_hidden_states = torch.cat([uncond_text_encoder_hidden_states, text_encoder_hidden_states]) + + text_mask = torch.cat([uncond_text_mask, text_mask]) + + return prompt_embeds, text_encoder_hidden_states, text_mask + + def _encode_image(self, image, device, num_images_per_prompt, image_embeddings: Optional[torch.Tensor] = None): + dtype = next(self.image_encoder.parameters()).dtype + + if image_embeddings is None: + if not isinstance(image, torch.Tensor): + image = self.feature_extractor(images=image, return_tensors="pt").pixel_values + + image = image.to(device=device, dtype=dtype) + image_embeddings = self.image_encoder(image).image_embeds + + image_embeddings = image_embeddings.repeat_interleave(num_images_per_prompt, dim=0) + + return image_embeddings + + @torch.no_grad() + def __call__( + self, + image: Optional[Union[PIL.Image.Image, List[PIL.Image.Image], torch.FloatTensor]] = None, + num_images_per_prompt: int = 1, + decoder_num_inference_steps: int = 25, + super_res_num_inference_steps: int = 7, + generator: Optional[torch.Generator] = None, + decoder_latents: Optional[torch.FloatTensor] = None, + super_res_latents: Optional[torch.FloatTensor] = None, + image_embeddings: Optional[torch.Tensor] = None, + decoder_guidance_scale: float = 8.0, + output_type: Optional[str] = "pil", + return_dict: bool = True, + ): + """ + The call function to the pipeline for generation. + + Args: + image (`PIL.Image.Image` or `List[PIL.Image.Image]` or `torch.FloatTensor`): + `Image` or tensor representing an image batch to be used as the starting point. If you provide a + tensor, it needs to be compatible with the [`CLIPImageProcessor`] + [configuration](https://huggingface.co/fusing/karlo-image-variations-diffusers/blob/main/feature_extractor/preprocessor_config.json). + Can be left as `None` only when `image_embeddings` are passed. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + decoder_num_inference_steps (`int`, *optional*, defaults to 25): + The number of denoising steps for the decoder. More denoising steps usually lead to a higher quality + image at the expense of slower inference. + super_res_num_inference_steps (`int`, *optional*, defaults to 7): + The number of denoising steps for super resolution. More denoising steps usually lead to a higher + quality image at the expense of slower inference. + generator (`torch.Generator`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + decoder_latents (`torch.FloatTensor` of shape (batch size, channels, height, width), *optional*): + Pre-generated noisy latents to be used as inputs for the decoder. + super_res_latents (`torch.FloatTensor` of shape (batch size, channels, super res height, super res width), *optional*): + Pre-generated noisy latents to be used as inputs for the decoder. + decoder_guidance_scale (`float`, *optional*, defaults to 4.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + image_embeddings (`torch.Tensor`, *optional*): + Pre-defined image embeddings that can be derived from the image encoder. Pre-defined image embeddings + can be passed for tasks like image interpolations. `image` can be left as `None`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.ImagePipelineOutput`] is returned, otherwise a `tuple` is + returned where the first element is a list with the generated images. + """ + if image is not None: + if isinstance(image, PIL.Image.Image): + batch_size = 1 + elif isinstance(image, list): + batch_size = len(image) + else: + batch_size = image.shape[0] + else: + batch_size = image_embeddings.shape[0] + + prompt = [""] * batch_size + + device = self._execution_device + + batch_size = batch_size * num_images_per_prompt + + do_classifier_free_guidance = decoder_guidance_scale > 1.0 + + prompt_embeds, text_encoder_hidden_states, text_mask = self._encode_prompt( + prompt, device, num_images_per_prompt, do_classifier_free_guidance + ) + + image_embeddings = self._encode_image(image, device, num_images_per_prompt, image_embeddings) + + # decoder + text_encoder_hidden_states, additive_clip_time_embeddings = self.text_proj( + image_embeddings=image_embeddings, + prompt_embeds=prompt_embeds, + text_encoder_hidden_states=text_encoder_hidden_states, + do_classifier_free_guidance=do_classifier_free_guidance, + ) + + if device.type == "mps": + # HACK: MPS: There is a panic when padding bool tensors, + # so cast to int tensor for the pad and back to bool afterwards + text_mask = text_mask.type(torch.int) + decoder_text_mask = F.pad(text_mask, (self.text_proj.clip_extra_context_tokens, 0), value=1) + decoder_text_mask = decoder_text_mask.type(torch.bool) + else: + decoder_text_mask = F.pad(text_mask, (self.text_proj.clip_extra_context_tokens, 0), value=True) + + self.decoder_scheduler.set_timesteps(decoder_num_inference_steps, device=device) + decoder_timesteps_tensor = self.decoder_scheduler.timesteps + + num_channels_latents = self.decoder.config.in_channels + height = self.decoder.config.sample_size + width = self.decoder.config.sample_size + + if decoder_latents is None: + decoder_latents = self.prepare_latents( + (batch_size, num_channels_latents, height, width), + text_encoder_hidden_states.dtype, + device, + generator, + decoder_latents, + self.decoder_scheduler, + ) + + for i, t in enumerate(self.progress_bar(decoder_timesteps_tensor)): + # expand the latents if we are doing classifier free guidance + latent_model_input = torch.cat([decoder_latents] * 2) if do_classifier_free_guidance else decoder_latents + + noise_pred = self.decoder( + sample=latent_model_input, + timestep=t, + encoder_hidden_states=text_encoder_hidden_states, + class_labels=additive_clip_time_embeddings, + attention_mask=decoder_text_mask, + ).sample + + if do_classifier_free_guidance: + noise_pred_uncond, noise_pred_text = noise_pred.chunk(2) + noise_pred_uncond, _ = noise_pred_uncond.split(latent_model_input.shape[1], dim=1) + noise_pred_text, predicted_variance = noise_pred_text.split(latent_model_input.shape[1], dim=1) + noise_pred = noise_pred_uncond + decoder_guidance_scale * (noise_pred_text - noise_pred_uncond) + noise_pred = torch.cat([noise_pred, predicted_variance], dim=1) + + if i + 1 == decoder_timesteps_tensor.shape[0]: + prev_timestep = None + else: + prev_timestep = decoder_timesteps_tensor[i + 1] + + # compute the previous noisy sample x_t -> x_t-1 + decoder_latents = self.decoder_scheduler.step( + noise_pred, t, decoder_latents, prev_timestep=prev_timestep, generator=generator + ).prev_sample + + decoder_latents = decoder_latents.clamp(-1, 1) + + image_small = decoder_latents + + # done decoder + + # super res + + self.super_res_scheduler.set_timesteps(super_res_num_inference_steps, device=device) + super_res_timesteps_tensor = self.super_res_scheduler.timesteps + + channels = self.super_res_first.config.in_channels // 2 + height = self.super_res_first.config.sample_size + width = self.super_res_first.config.sample_size + + if super_res_latents is None: + super_res_latents = self.prepare_latents( + (batch_size, channels, height, width), + image_small.dtype, + device, + generator, + super_res_latents, + self.super_res_scheduler, + ) + + if device.type == "mps": + # MPS does not support many interpolations + image_upscaled = F.interpolate(image_small, size=[height, width]) + else: + interpolate_antialias = {} + if "antialias" in inspect.signature(F.interpolate).parameters: + interpolate_antialias["antialias"] = True + + image_upscaled = F.interpolate( + image_small, size=[height, width], mode="bicubic", align_corners=False, **interpolate_antialias + ) + + for i, t in enumerate(self.progress_bar(super_res_timesteps_tensor)): + # no classifier free guidance + + if i == super_res_timesteps_tensor.shape[0] - 1: + unet = self.super_res_last + else: + unet = self.super_res_first + + latent_model_input = torch.cat([super_res_latents, image_upscaled], dim=1) + + noise_pred = unet( + sample=latent_model_input, + timestep=t, + ).sample + + if i + 1 == super_res_timesteps_tensor.shape[0]: + prev_timestep = None + else: + prev_timestep = super_res_timesteps_tensor[i + 1] + + # compute the previous noisy sample x_t -> x_t-1 + super_res_latents = self.super_res_scheduler.step( + noise_pred, t, super_res_latents, prev_timestep=prev_timestep, generator=generator + ).prev_sample + + image = super_res_latents + + # done super res + self.maybe_free_model_hooks() + + # post processing + + image = image * 0.5 + 0.5 + image = image.clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).float().numpy() + + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,) + + return ImagePipelineOutput(images=image) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/unclip/text_proj.py b/diffusers-0.27.0/src/diffusers/pipelines/unclip/text_proj.py new file mode 100755 index 0000000..5a86d0c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/unclip/text_proj.py @@ -0,0 +1,86 @@ +# Copyright 2024 Kakao Brain and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch +from torch import nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...models import ModelMixin + + +class UnCLIPTextProjModel(ModelMixin, ConfigMixin): + """ + Utility class for CLIP embeddings. Used to combine the image and text embeddings into a format usable by the + decoder. + + For more details, see the original paper: https://arxiv.org/abs/2204.06125 section 2.1 + """ + + @register_to_config + def __init__( + self, + *, + clip_extra_context_tokens: int = 4, + clip_embeddings_dim: int = 768, + time_embed_dim: int, + cross_attention_dim, + ): + super().__init__() + + self.learned_classifier_free_guidance_embeddings = nn.Parameter(torch.zeros(clip_embeddings_dim)) + + # parameters for additional clip time embeddings + self.embedding_proj = nn.Linear(clip_embeddings_dim, time_embed_dim) + self.clip_image_embeddings_project_to_time_embeddings = nn.Linear(clip_embeddings_dim, time_embed_dim) + + # parameters for encoder hidden states + self.clip_extra_context_tokens = clip_extra_context_tokens + self.clip_extra_context_tokens_proj = nn.Linear( + clip_embeddings_dim, self.clip_extra_context_tokens * cross_attention_dim + ) + self.encoder_hidden_states_proj = nn.Linear(clip_embeddings_dim, cross_attention_dim) + self.text_encoder_hidden_states_norm = nn.LayerNorm(cross_attention_dim) + + def forward(self, *, image_embeddings, prompt_embeds, text_encoder_hidden_states, do_classifier_free_guidance): + if do_classifier_free_guidance: + # Add the classifier free guidance embeddings to the image embeddings + image_embeddings_batch_size = image_embeddings.shape[0] + classifier_free_guidance_embeddings = self.learned_classifier_free_guidance_embeddings.unsqueeze(0) + classifier_free_guidance_embeddings = classifier_free_guidance_embeddings.expand( + image_embeddings_batch_size, -1 + ) + image_embeddings = torch.cat([classifier_free_guidance_embeddings, image_embeddings], dim=0) + + # The image embeddings batch size and the text embeddings batch size are equal + assert image_embeddings.shape[0] == prompt_embeds.shape[0] + + batch_size = prompt_embeds.shape[0] + + # "Specifically, we modify the architecture described in Nichol et al. (2021) by projecting and + # adding CLIP embeddings to the existing timestep embedding, ... + time_projected_prompt_embeds = self.embedding_proj(prompt_embeds) + time_projected_image_embeddings = self.clip_image_embeddings_project_to_time_embeddings(image_embeddings) + additive_clip_time_embeddings = time_projected_image_embeddings + time_projected_prompt_embeds + + # ... and by projecting CLIP embeddings into four + # extra tokens of context that are concatenated to the sequence of outputs from the GLIDE text encoder" + clip_extra_context_tokens = self.clip_extra_context_tokens_proj(image_embeddings) + clip_extra_context_tokens = clip_extra_context_tokens.reshape(batch_size, -1, self.clip_extra_context_tokens) + clip_extra_context_tokens = clip_extra_context_tokens.permute(0, 2, 1) + + text_encoder_hidden_states = self.encoder_hidden_states_proj(text_encoder_hidden_states) + text_encoder_hidden_states = self.text_encoder_hidden_states_norm(text_encoder_hidden_states) + text_encoder_hidden_states = torch.cat([clip_extra_context_tokens, text_encoder_hidden_states], dim=1) + + return text_encoder_hidden_states, additive_clip_time_embeddings diff --git a/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/__init__.py new file mode 100755 index 0000000..1ac2b09 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/__init__.py @@ -0,0 +1,58 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import ( + ImageTextPipelineOutput, + UniDiffuserPipeline, + ) + + _dummy_objects.update( + {"ImageTextPipelineOutput": ImageTextPipelineOutput, "UniDiffuserPipeline": UniDiffuserPipeline} + ) +else: + _import_structure["modeling_text_decoder"] = ["UniDiffuserTextDecoder"] + _import_structure["modeling_uvit"] = ["UniDiffuserModel", "UTransformer2DModel"] + _import_structure["pipeline_unidiffuser"] = ["ImageTextPipelineOutput", "UniDiffuserPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import ( + ImageTextPipelineOutput, + UniDiffuserPipeline, + ) + else: + from .modeling_text_decoder import UniDiffuserTextDecoder + from .modeling_uvit import UniDiffuserModel, UTransformer2DModel + from .pipeline_unidiffuser import ImageTextPipelineOutput, UniDiffuserPipeline + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/modeling_text_decoder.py b/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/modeling_text_decoder.py new file mode 100755 index 0000000..bf0a4eb --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/modeling_text_decoder.py @@ -0,0 +1,296 @@ +from typing import Optional + +import numpy as np +import torch +from torch import nn +from transformers import GPT2Config, GPT2LMHeadModel +from transformers.modeling_utils import ModuleUtilsMixin + +from ...configuration_utils import ConfigMixin, register_to_config +from ...models import ModelMixin + + +# Modified from ClipCaptionModel in https://github.com/thu-ml/unidiffuser/blob/main/libs/caption_decoder.py +class UniDiffuserTextDecoder(ModelMixin, ConfigMixin, ModuleUtilsMixin): + """ + Text decoder model for a image-text [UniDiffuser](https://arxiv.org/pdf/2303.06555.pdf) model. This is used to + generate text from the UniDiffuser image-text embedding. + + Parameters: + prefix_length (`int`): + Max number of prefix tokens that will be supplied to the model. + prefix_inner_dim (`int`): + The hidden size of the incoming prefix embeddings. For UniDiffuser, this would be the hidden dim of the + CLIP text encoder. + prefix_hidden_dim (`int`, *optional*): + Hidden dim of the MLP if we encode the prefix. + vocab_size (`int`, *optional*, defaults to 50257): + Vocabulary size of the GPT-2 model. Defines the number of different tokens that can be represented by the + `inputs_ids` passed when calling [`GPT2Model`] or [`TFGPT2Model`]. + n_positions (`int`, *optional*, defaults to 1024): + The maximum sequence length that this model might ever be used with. Typically set this to something large + just in case (e.g., 512 or 1024 or 2048). + n_embd (`int`, *optional*, defaults to 768): + Dimensionality of the embeddings and hidden states. + n_layer (`int`, *optional*, defaults to 12): + Number of hidden layers in the Transformer encoder. + n_head (`int`, *optional*, defaults to 12): + Number of attention heads for each attention layer in the Transformer encoder. + n_inner (`int`, *optional*, defaults to None): + Dimensionality of the inner feed-forward layers. `None` will set it to 4 times n_embd + activation_function (`str`, *optional*, defaults to `"gelu"`): + Activation function, to be selected in the list `["relu", "silu", "gelu", "tanh", "gelu_new"]`. + resid_pdrop (`float`, *optional*, defaults to 0.1): + The dropout probability for all fully connected layers in the embeddings, encoder, and pooler. + embd_pdrop (`float`, *optional*, defaults to 0.1): + The dropout ratio for the embeddings. + attn_pdrop (`float`, *optional*, defaults to 0.1): + The dropout ratio for the attention. + layer_norm_epsilon (`float`, *optional*, defaults to 1e-5): + The epsilon to use in the layer normalization layers. + initializer_range (`float`, *optional*, defaults to 0.02): + The standard deviation of the truncated_normal_initializer for initializing all weight matrices. + scale_attn_weights (`bool`, *optional*, defaults to `True`): + Scale attention weights by dividing by sqrt(hidden_size).. + use_cache (`bool`, *optional*, defaults to `True`): + Whether or not the model should return the last key/values attentions (not used by all models). + scale_attn_by_inverse_layer_idx (`bool`, *optional*, defaults to `False`): + Whether to additionally scale attention weights by `1 / layer_idx + 1`. + reorder_and_upcast_attn (`bool`, *optional*, defaults to `False`): + Whether to scale keys (K) prior to computing attention (dot-product) and upcast attention + dot-product/softmax to float() when training with mixed precision. + """ + + _keys_to_ignore_on_load_unexpected = [r"h\.\d+\.attn\.bias", r"h\.\d+\.attn\.masked_bias"] + + @register_to_config + def __init__( + self, + prefix_length: int, + prefix_inner_dim: int, + prefix_hidden_dim: Optional[int] = None, + vocab_size: int = 50257, # Start of GPT2 config args + n_positions: int = 1024, + n_embd: int = 768, + n_layer: int = 12, + n_head: int = 12, + n_inner: Optional[int] = None, + activation_function: str = "gelu_new", + resid_pdrop: float = 0.1, + embd_pdrop: float = 0.1, + attn_pdrop: float = 0.1, + layer_norm_epsilon: float = 1e-5, + initializer_range: float = 0.02, + scale_attn_weights: bool = True, + use_cache: bool = True, + scale_attn_by_inverse_layer_idx: bool = False, + reorder_and_upcast_attn: bool = False, + ): + super().__init__() + + self.prefix_length = prefix_length + + if prefix_inner_dim != n_embd and prefix_hidden_dim is None: + raise ValueError( + f"`prefix_hidden_dim` cannot be `None` when `prefix_inner_dim`: {prefix_hidden_dim} and" + f" `n_embd`: {n_embd} are not equal." + ) + + self.prefix_inner_dim = prefix_inner_dim + self.prefix_hidden_dim = prefix_hidden_dim + + self.encode_prefix = ( + nn.Linear(self.prefix_inner_dim, self.prefix_hidden_dim) + if self.prefix_hidden_dim is not None + else nn.Identity() + ) + self.decode_prefix = ( + nn.Linear(self.prefix_hidden_dim, n_embd) if self.prefix_hidden_dim is not None else nn.Identity() + ) + + gpt_config = GPT2Config( + vocab_size=vocab_size, + n_positions=n_positions, + n_embd=n_embd, + n_layer=n_layer, + n_head=n_head, + n_inner=n_inner, + activation_function=activation_function, + resid_pdrop=resid_pdrop, + embd_pdrop=embd_pdrop, + attn_pdrop=attn_pdrop, + layer_norm_epsilon=layer_norm_epsilon, + initializer_range=initializer_range, + scale_attn_weights=scale_attn_weights, + use_cache=use_cache, + scale_attn_by_inverse_layer_idx=scale_attn_by_inverse_layer_idx, + reorder_and_upcast_attn=reorder_and_upcast_attn, + ) + self.transformer = GPT2LMHeadModel(gpt_config) + + def forward( + self, + input_ids: torch.Tensor, + prefix_embeds: torch.Tensor, + attention_mask: Optional[torch.Tensor] = None, + labels: Optional[torch.Tensor] = None, + ): + """ + Args: + input_ids (`torch.Tensor` of shape `(N, max_seq_len)`): + Text tokens to use for inference. + prefix_embeds (`torch.Tensor` of shape `(N, prefix_length, 768)`): + Prefix embedding to preprend to the embedded tokens. + attention_mask (`torch.Tensor` of shape `(N, prefix_length + max_seq_len, 768)`, *optional*): + Attention mask for the prefix embedding. + labels (`torch.Tensor`, *optional*): + Labels to use for language modeling. + """ + embedding_text = self.transformer.transformer.wte(input_ids) + hidden = self.encode_prefix(prefix_embeds) + prefix_embeds = self.decode_prefix(hidden) + embedding_cat = torch.cat((prefix_embeds, embedding_text), dim=1) + + if labels is not None: + dummy_token = self.get_dummy_token(input_ids.shape[0], input_ids.device) + labels = torch.cat((dummy_token, input_ids), dim=1) + out = self.transformer(inputs_embeds=embedding_cat, labels=labels, attention_mask=attention_mask) + if self.prefix_hidden_dim is not None: + return out, hidden + else: + return out + + def get_dummy_token(self, batch_size: int, device: torch.device) -> torch.Tensor: + return torch.zeros(batch_size, self.prefix_length, dtype=torch.int64, device=device) + + def encode(self, prefix): + return self.encode_prefix(prefix) + + @torch.no_grad() + def generate_captions(self, features, eos_token_id, device): + """ + Generate captions given text embedding features. Returns list[L]. + + Args: + features (`torch.Tensor` of shape `(B, L, D)`): + Text embedding features to generate captions from. + eos_token_id (`int`): + The token ID of the EOS token for the text decoder model. + device: + Device to perform text generation on. + + Returns: + `List[str]`: A list of strings generated from the decoder model. + """ + + features = torch.split(features, 1, dim=0) + generated_tokens = [] + generated_seq_lengths = [] + for feature in features: + feature = self.decode_prefix(feature.to(device)) # back to the clip feature + # Only support beam search for now + output_tokens, seq_lengths = self.generate_beam( + input_embeds=feature, device=device, eos_token_id=eos_token_id + ) + generated_tokens.append(output_tokens[0]) + generated_seq_lengths.append(seq_lengths[0]) + generated_tokens = torch.stack(generated_tokens) + generated_seq_lengths = torch.stack(generated_seq_lengths) + return generated_tokens, generated_seq_lengths + + @torch.no_grad() + def generate_beam( + self, + input_ids=None, + input_embeds=None, + device=None, + beam_size: int = 5, + entry_length: int = 67, + temperature: float = 1.0, + eos_token_id: Optional[int] = None, + ): + """ + Generates text using the given tokenizer and text prompt or token embedding via beam search. This + implementation is based on the beam search implementation from the [original UniDiffuser + code](https://github.com/thu-ml/unidiffuser/blob/main/libs/caption_decoder.py#L89). + + Args: + eos_token_id (`int`, *optional*): + The token ID of the EOS token for the text decoder model. + input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): + Tokenizer indices of input sequence tokens in the vocabulary. One of `input_ids` and `input_embeds` + must be supplied. + input_embeds (`torch.FloatTensor` of shape `(batch_size, seq_len, hidden_size)`, *optional*): + An embedded representation to directly pass to the transformer as a prefix for beam search. One of + `input_ids` and `input_embeds` must be supplied. + device: + The device to perform beam search on. + beam_size (`int`, *optional*, defaults to `5`): + The number of best states to store during beam search. + entry_length (`int`, *optional*, defaults to `67`): + The number of iterations to run beam search. + temperature (`float`, *optional*, defaults to 1.0): + The temperature to use when performing the softmax over logits from the decoding model. + + Returns: + `Tuple(torch.Tensor, torch.Tensor)`: A tuple of tensors where the first element is a tensor of generated + token sequences sorted by score in descending order, and the second element is the sequence lengths + corresponding to those sequences. + """ + # Generates text until stop_token is reached using beam search with the desired beam size. + stop_token_index = eos_token_id + tokens = None + scores = None + seq_lengths = torch.ones(beam_size, device=device, dtype=torch.int) + is_stopped = torch.zeros(beam_size, device=device, dtype=torch.bool) + + if input_embeds is not None: + generated = input_embeds + else: + generated = self.transformer.transformer.wte(input_ids) + + for i in range(entry_length): + outputs = self.transformer(inputs_embeds=generated) + logits = outputs.logits + logits = logits[:, -1, :] / (temperature if temperature > 0 else 1.0) + logits = logits.softmax(-1).log() + + if scores is None: + scores, next_tokens = logits.topk(beam_size, -1) + generated = generated.expand(beam_size, *generated.shape[1:]) + next_tokens, scores = next_tokens.permute(1, 0), scores.squeeze(0) + if tokens is None: + tokens = next_tokens + else: + tokens = tokens.expand(beam_size, *tokens.shape[1:]) + tokens = torch.cat((tokens, next_tokens), dim=1) + else: + logits[is_stopped] = -float(np.inf) + logits[is_stopped, 0] = 0 + scores_sum = scores[:, None] + logits + seq_lengths[~is_stopped] += 1 + scores_sum_average = scores_sum / seq_lengths[:, None] + scores_sum_average, next_tokens = scores_sum_average.view(-1).topk(beam_size, -1) + next_tokens_source = next_tokens // scores_sum.shape[1] + seq_lengths = seq_lengths[next_tokens_source] + next_tokens = next_tokens % scores_sum.shape[1] + next_tokens = next_tokens.unsqueeze(1) + tokens = tokens[next_tokens_source] + tokens = torch.cat((tokens, next_tokens), dim=1) + generated = generated[next_tokens_source] + scores = scores_sum_average * seq_lengths + is_stopped = is_stopped[next_tokens_source] + + next_token_embed = self.transformer.transformer.wte(next_tokens.squeeze()).view(generated.shape[0], 1, -1) + generated = torch.cat((generated, next_token_embed), dim=1) + is_stopped = is_stopped + next_tokens.eq(stop_token_index).squeeze() + if is_stopped.all(): + break + + scores = scores / seq_lengths + order = scores.argsort(descending=True) + # tokens tensors are already padded to max_seq_length + output_texts = [tokens[i] for i in order] + output_texts = torch.stack(output_texts, dim=0) + seq_lengths = torch.tensor([seq_lengths[i] for i in order], dtype=seq_lengths.dtype) + return output_texts, seq_lengths diff --git a/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/modeling_uvit.py b/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/modeling_uvit.py new file mode 100755 index 0000000..c074b99 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/modeling_uvit.py @@ -0,0 +1,1197 @@ +import math +from typing import Optional, Union + +import torch +from torch import nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...models import ModelMixin +from ...models.attention import FeedForward +from ...models.attention_processor import Attention +from ...models.embeddings import TimestepEmbedding, Timesteps, get_2d_sincos_pos_embed +from ...models.normalization import AdaLayerNorm +from ...models.transformers.transformer_2d import Transformer2DModelOutput +from ...utils import logging + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def _no_grad_trunc_normal_(tensor, mean, std, a, b): + # Cut & paste from PyTorch official master until it's in a few official releases - RW + # Method based on https://people.sc.fsu.edu/~jburkardt/presentations/truncated_normal.pdf + def norm_cdf(x): + # Computes standard normal cumulative distribution function + return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0 + + if (mean < a - 2 * std) or (mean > b + 2 * std): + logger.warning( + "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. " + "The distribution of values may be incorrect." + ) + + with torch.no_grad(): + # Values are generated by using a truncated uniform distribution and + # then using the inverse CDF for the normal distribution. + # Get upper and lower cdf values + l = norm_cdf((a - mean) / std) + u = norm_cdf((b - mean) / std) + + # Uniformly fill tensor with values from [l, u], then translate to + # [2l-1, 2u-1]. + tensor.uniform_(2 * l - 1, 2 * u - 1) + + # Use inverse cdf transform for normal distribution to get truncated + # standard normal + tensor.erfinv_() + + # Transform to proper mean, std + tensor.mul_(std * math.sqrt(2.0)) + tensor.add_(mean) + + # Clamp to ensure it's in the proper range + tensor.clamp_(min=a, max=b) + return tensor + + +def trunc_normal_(tensor, mean=0.0, std=1.0, a=-2.0, b=2.0): + # type: (torch.Tensor, float, float, float, float) -> torch.Tensor + r"""Fills the input Tensor with values drawn from a truncated + normal distribution. The values are effectively drawn from the normal distribution :math:`\mathcal{N}(\text{mean}, + \text{std}^2)` with values outside :math:`[a, b]` redrawn until they are within the bounds. The method used for + generating the random values works best when :math:`a \leq \text{mean} \leq b`. + + Args: + tensor: an n-dimensional `torch.Tensor` + mean: the mean of the normal distribution + std: the standard deviation of the normal distribution + a: the minimum cutoff value + b: the maximum cutoff value + Examples: + >>> w = torch.empty(3, 5) >>> nn.init.trunc_normal_(w) + """ + return _no_grad_trunc_normal_(tensor, mean, std, a, b) + + +class PatchEmbed(nn.Module): + """2D Image to Patch Embedding""" + + def __init__( + self, + height=224, + width=224, + patch_size=16, + in_channels=3, + embed_dim=768, + layer_norm=False, + flatten=True, + bias=True, + use_pos_embed=True, + ): + super().__init__() + + num_patches = (height // patch_size) * (width // patch_size) + self.flatten = flatten + self.layer_norm = layer_norm + + self.proj = nn.Conv2d( + in_channels, embed_dim, kernel_size=(patch_size, patch_size), stride=patch_size, bias=bias + ) + if layer_norm: + self.norm = nn.LayerNorm(embed_dim, elementwise_affine=False, eps=1e-6) + else: + self.norm = None + + self.use_pos_embed = use_pos_embed + if self.use_pos_embed: + pos_embed = get_2d_sincos_pos_embed(embed_dim, int(num_patches**0.5)) + self.register_buffer("pos_embed", torch.from_numpy(pos_embed).float().unsqueeze(0), persistent=False) + + def forward(self, latent): + latent = self.proj(latent) + if self.flatten: + latent = latent.flatten(2).transpose(1, 2) # BCHW -> BNC + if self.layer_norm: + latent = self.norm(latent) + if self.use_pos_embed: + return latent + self.pos_embed + else: + return latent + + +class SkipBlock(nn.Module): + def __init__(self, dim: int): + super().__init__() + + self.skip_linear = nn.Linear(2 * dim, dim) + + # Use torch.nn.LayerNorm for now, following the original code + self.norm = nn.LayerNorm(dim) + + def forward(self, x, skip): + x = self.skip_linear(torch.cat([x, skip], dim=-1)) + x = self.norm(x) + + return x + + +# Modified to support both pre-LayerNorm and post-LayerNorm configurations +# Don't support AdaLayerNormZero for now +# Modified from diffusers.models.attention.BasicTransformerBlock +class UTransformerBlock(nn.Module): + r""" + A modification of BasicTransformerBlock which supports pre-LayerNorm and post-LayerNorm configurations. + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + activation_fn (`str`, *optional*, defaults to `"geglu"`): + Activation function to be used in feed-forward. + num_embeds_ada_norm (:obj: `int`, *optional*): + The number of diffusion steps used during training. See `Transformer2DModel`. + attention_bias (:obj: `bool`, *optional*, defaults to `False`): + Configure if the attentions should contain a bias parameter. + only_cross_attention (`bool`, *optional*): + Whether to use only cross-attention layers. In this case two cross attention layers are used. + double_self_attention (`bool`, *optional*): + Whether to use two self-attention layers. In this case no cross attention layers are used. + upcast_attention (`bool`, *optional*): + Whether to upcast the query and key to float32 when performing the attention calculation. + norm_elementwise_affine (`bool`, *optional*): + Whether to use learnable per-element affine parameters during layer normalization. + norm_type (`str`, defaults to `"layer_norm"`): + The layer norm implementation to use. + pre_layer_norm (`bool`, *optional*): + Whether to perform layer normalization before the attention and feedforward operations ("pre-LayerNorm"), + as opposed to after ("post-LayerNorm"). Note that `BasicTransformerBlock` uses pre-LayerNorm, e.g. + `pre_layer_norm = True`. + final_dropout (`bool`, *optional*): + Whether to use a final Dropout layer after the feedforward network. + """ + + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + dropout=0.0, + cross_attention_dim: Optional[int] = None, + activation_fn: str = "geglu", + num_embeds_ada_norm: Optional[int] = None, + attention_bias: bool = False, + only_cross_attention: bool = False, + double_self_attention: bool = False, + upcast_attention: bool = False, + norm_elementwise_affine: bool = True, + norm_type: str = "layer_norm", + pre_layer_norm: bool = True, + final_dropout: bool = False, + ): + super().__init__() + self.only_cross_attention = only_cross_attention + + self.use_ada_layer_norm = (num_embeds_ada_norm is not None) and norm_type == "ada_norm" + + self.pre_layer_norm = pre_layer_norm + + if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None: + raise ValueError( + f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to" + f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}." + ) + + # 1. Self-Attn + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=cross_attention_dim if only_cross_attention else None, + upcast_attention=upcast_attention, + ) + + # 2. Cross-Attn + if cross_attention_dim is not None or double_self_attention: + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim if not double_self_attention else None, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + upcast_attention=upcast_attention, + ) # is self-attn if encoder_hidden_states is none + else: + self.attn2 = None + + if self.use_ada_layer_norm: + self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm) + else: + self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine) + + if cross_attention_dim is not None or double_self_attention: + # We currently only use AdaLayerNormZero for self attention where there will only be one attention block. + # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during + # the second cross attention block. + self.norm2 = ( + AdaLayerNorm(dim, num_embeds_ada_norm) + if self.use_ada_layer_norm + else nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine) + ) + else: + self.norm2 = None + + # 3. Feed-forward + self.norm3 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine) + self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn, final_dropout=final_dropout) + + def forward( + self, + hidden_states, + attention_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + timestep=None, + cross_attention_kwargs=None, + class_labels=None, + ): + # Pre-LayerNorm + if self.pre_layer_norm: + if self.use_ada_layer_norm: + norm_hidden_states = self.norm1(hidden_states, timestep) + else: + norm_hidden_states = self.norm1(hidden_states) + else: + norm_hidden_states = hidden_states + + # 1. Self-Attention + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + attn_output = self.attn1( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + + # Post-LayerNorm + if not self.pre_layer_norm: + if self.use_ada_layer_norm: + attn_output = self.norm1(attn_output, timestep) + else: + attn_output = self.norm1(attn_output) + + hidden_states = attn_output + hidden_states + + if self.attn2 is not None: + # Pre-LayerNorm + if self.pre_layer_norm: + norm_hidden_states = ( + self.norm2(hidden_states, timestep) if self.use_ada_layer_norm else self.norm2(hidden_states) + ) + else: + norm_hidden_states = hidden_states + # TODO (Birch-San): Here we should prepare the encoder_attention mask correctly + # prepare attention mask here + + # 2. Cross-Attention + attn_output = self.attn2( + norm_hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + **cross_attention_kwargs, + ) + + # Post-LayerNorm + if not self.pre_layer_norm: + attn_output = self.norm2(attn_output, timestep) if self.use_ada_layer_norm else self.norm2(attn_output) + + hidden_states = attn_output + hidden_states + + # 3. Feed-forward + # Pre-LayerNorm + if self.pre_layer_norm: + norm_hidden_states = self.norm3(hidden_states) + else: + norm_hidden_states = hidden_states + + ff_output = self.ff(norm_hidden_states) + + # Post-LayerNorm + if not self.pre_layer_norm: + ff_output = self.norm3(ff_output) + + hidden_states = ff_output + hidden_states + + return hidden_states + + +# Like UTransformerBlock except with LayerNorms on the residual backbone of the block +# Modified from diffusers.models.attention.BasicTransformerBlock +class UniDiffuserBlock(nn.Module): + r""" + A modification of BasicTransformerBlock which supports pre-LayerNorm and post-LayerNorm configurations and puts the + LayerNorms on the residual backbone of the block. This matches the transformer block in the [original UniDiffuser + implementation](https://github.com/thu-ml/unidiffuser/blob/main/libs/uvit_multi_post_ln_v1.py#L104). + + Parameters: + dim (`int`): The number of channels in the input and output. + num_attention_heads (`int`): The number of heads to use for multi-head attention. + attention_head_dim (`int`): The number of channels in each head. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + cross_attention_dim (`int`, *optional*): The size of the encoder_hidden_states vector for cross attention. + activation_fn (`str`, *optional*, defaults to `"geglu"`): + Activation function to be used in feed-forward. + num_embeds_ada_norm (:obj: `int`, *optional*): + The number of diffusion steps used during training. See `Transformer2DModel`. + attention_bias (:obj: `bool`, *optional*, defaults to `False`): + Configure if the attentions should contain a bias parameter. + only_cross_attention (`bool`, *optional*): + Whether to use only cross-attention layers. In this case two cross attention layers are used. + double_self_attention (`bool`, *optional*): + Whether to use two self-attention layers. In this case no cross attention layers are used. + upcast_attention (`bool`, *optional*): + Whether to upcast the query and key to float() when performing the attention calculation. + norm_elementwise_affine (`bool`, *optional*): + Whether to use learnable per-element affine parameters during layer normalization. + norm_type (`str`, defaults to `"layer_norm"`): + The layer norm implementation to use. + pre_layer_norm (`bool`, *optional*): + Whether to perform layer normalization before the attention and feedforward operations ("pre-LayerNorm"), + as opposed to after ("post-LayerNorm"). The original UniDiffuser implementation is post-LayerNorm + (`pre_layer_norm = False`). + final_dropout (`bool`, *optional*): + Whether to use a final Dropout layer after the feedforward network. + """ + + def __init__( + self, + dim: int, + num_attention_heads: int, + attention_head_dim: int, + dropout=0.0, + cross_attention_dim: Optional[int] = None, + activation_fn: str = "geglu", + num_embeds_ada_norm: Optional[int] = None, + attention_bias: bool = False, + only_cross_attention: bool = False, + double_self_attention: bool = False, + upcast_attention: bool = False, + norm_elementwise_affine: bool = True, + norm_type: str = "layer_norm", + pre_layer_norm: bool = False, + final_dropout: bool = True, + ): + super().__init__() + self.only_cross_attention = only_cross_attention + + self.use_ada_layer_norm = (num_embeds_ada_norm is not None) and norm_type == "ada_norm" + + self.pre_layer_norm = pre_layer_norm + + if norm_type in ("ada_norm", "ada_norm_zero") and num_embeds_ada_norm is None: + raise ValueError( + f"`norm_type` is set to {norm_type}, but `num_embeds_ada_norm` is not defined. Please make sure to" + f" define `num_embeds_ada_norm` if setting `norm_type` to {norm_type}." + ) + + # 1. Self-Attn + self.attn1 = Attention( + query_dim=dim, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + cross_attention_dim=cross_attention_dim if only_cross_attention else None, + upcast_attention=upcast_attention, + ) + + # 2. Cross-Attn + if cross_attention_dim is not None or double_self_attention: + self.attn2 = Attention( + query_dim=dim, + cross_attention_dim=cross_attention_dim if not double_self_attention else None, + heads=num_attention_heads, + dim_head=attention_head_dim, + dropout=dropout, + bias=attention_bias, + upcast_attention=upcast_attention, + ) # is self-attn if encoder_hidden_states is none + else: + self.attn2 = None + + if self.use_ada_layer_norm: + self.norm1 = AdaLayerNorm(dim, num_embeds_ada_norm) + else: + self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine) + + if cross_attention_dim is not None or double_self_attention: + # We currently only use AdaLayerNormZero for self attention where there will only be one attention block. + # I.e. the number of returned modulation chunks from AdaLayerZero would not make sense if returned during + # the second cross attention block. + self.norm2 = ( + AdaLayerNorm(dim, num_embeds_ada_norm) + if self.use_ada_layer_norm + else nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine) + ) + else: + self.norm2 = None + + # 3. Feed-forward + self.norm3 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine) + self.ff = FeedForward(dim, dropout=dropout, activation_fn=activation_fn, final_dropout=final_dropout) + + def forward( + self, + hidden_states, + attention_mask=None, + encoder_hidden_states=None, + encoder_attention_mask=None, + timestep=None, + cross_attention_kwargs=None, + class_labels=None, + ): + # Following the diffusers transformer block implementation, put the LayerNorm on the + # residual backbone + # Pre-LayerNorm + if self.pre_layer_norm: + if self.use_ada_layer_norm: + hidden_states = self.norm1(hidden_states, timestep) + else: + hidden_states = self.norm1(hidden_states) + + # 1. Self-Attention + cross_attention_kwargs = cross_attention_kwargs if cross_attention_kwargs is not None else {} + attn_output = self.attn1( + hidden_states, + encoder_hidden_states=encoder_hidden_states if self.only_cross_attention else None, + attention_mask=attention_mask, + **cross_attention_kwargs, + ) + + hidden_states = attn_output + hidden_states + + # Following the diffusers transformer block implementation, put the LayerNorm on the + # residual backbone + # Post-LayerNorm + if not self.pre_layer_norm: + if self.use_ada_layer_norm: + hidden_states = self.norm1(hidden_states, timestep) + else: + hidden_states = self.norm1(hidden_states) + + if self.attn2 is not None: + # Pre-LayerNorm + if self.pre_layer_norm: + hidden_states = ( + self.norm2(hidden_states, timestep) if self.use_ada_layer_norm else self.norm2(hidden_states) + ) + # TODO (Birch-San): Here we should prepare the encoder_attention mask correctly + # prepare attention mask here + + # 2. Cross-Attention + attn_output = self.attn2( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + attention_mask=encoder_attention_mask, + **cross_attention_kwargs, + ) + + hidden_states = attn_output + hidden_states + + # Post-LayerNorm + if not self.pre_layer_norm: + hidden_states = ( + self.norm2(hidden_states, timestep) if self.use_ada_layer_norm else self.norm2(hidden_states) + ) + + # 3. Feed-forward + # Pre-LayerNorm + if self.pre_layer_norm: + hidden_states = self.norm3(hidden_states) + + ff_output = self.ff(hidden_states) + + hidden_states = ff_output + hidden_states + + # Post-LayerNorm + if not self.pre_layer_norm: + hidden_states = self.norm3(hidden_states) + + return hidden_states + + +# Modified from diffusers.models.transformer_2d.Transformer2DModel +# Modify the transformer block structure to be U-Net like following U-ViT +# Only supports patch-style input and torch.nn.LayerNorm currently +# https://github.com/baofff/U-ViT +class UTransformer2DModel(ModelMixin, ConfigMixin): + """ + Transformer model based on the [U-ViT](https://github.com/baofff/U-ViT) architecture for image-like data. Compared + to [`Transformer2DModel`], this model has skip connections between transformer blocks in a "U"-shaped fashion, + similar to a U-Net. Supports only continuous (actual embeddings) inputs, which are embedded via a [`PatchEmbed`] + layer and then reshaped to (b, t, d). + + Parameters: + num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention. + attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head. + in_channels (`int`, *optional*): + Pass if the input is continuous. The number of channels in the input. + out_channels (`int`, *optional*): + The number of output channels; if `None`, defaults to `in_channels`. + num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + norm_num_groups (`int`, *optional*, defaults to `32`): + The number of groups to use when performing Group Normalization. + cross_attention_dim (`int`, *optional*): The number of encoder_hidden_states dimensions to use. + attention_bias (`bool`, *optional*): + Configure if the TransformerBlocks' attention should contain a bias parameter. + sample_size (`int`, *optional*): Pass if the input is discrete. The width of the latent images. + Note that this is fixed at training time as it is used for learning a number of position embeddings. See + `ImagePositionalEmbeddings`. + num_vector_embeds (`int`, *optional*): + Pass if the input is discrete. The number of classes of the vector embeddings of the latent pixels. + Includes the class for the masked latent pixel. + patch_size (`int`, *optional*, defaults to 2): + The patch size to use in the patch embedding. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + num_embeds_ada_norm ( `int`, *optional*): Pass if at least one of the norm_layers is `AdaLayerNorm`. + The number of diffusion steps used during training. Note that this is fixed at training time as it is used + to learn a number of embeddings that are added to the hidden states. During inference, you can denoise for + up to but not more than steps than `num_embeds_ada_norm`. + use_linear_projection (int, *optional*): TODO: Not used + only_cross_attention (`bool`, *optional*): + Whether to use only cross-attention layers. In this case two cross attention layers are used in each + transformer block. + upcast_attention (`bool`, *optional*): + Whether to upcast the query and key to float() when performing the attention calculation. + norm_type (`str`, *optional*, defaults to `"layer_norm"`): + The Layer Normalization implementation to use. Defaults to `torch.nn.LayerNorm`. + block_type (`str`, *optional*, defaults to `"unidiffuser"`): + The transformer block implementation to use. If `"unidiffuser"`, has the LayerNorms on the residual + backbone of each transformer block; otherwise has them in the attention/feedforward branches (the standard + behavior in `diffusers`.) + pre_layer_norm (`bool`, *optional*): + Whether to perform layer normalization before the attention and feedforward operations ("pre-LayerNorm"), + as opposed to after ("post-LayerNorm"). The original UniDiffuser implementation is post-LayerNorm + (`pre_layer_norm = False`). + norm_elementwise_affine (`bool`, *optional*): + Whether to use learnable per-element affine parameters during layer normalization. + use_patch_pos_embed (`bool`, *optional*): + Whether to use position embeddings inside the patch embedding layer (`PatchEmbed`). + final_dropout (`bool`, *optional*): + Whether to use a final Dropout layer after the feedforward network. + """ + + @register_to_config + def __init__( + self, + num_attention_heads: int = 16, + attention_head_dim: int = 88, + in_channels: Optional[int] = None, + out_channels: Optional[int] = None, + num_layers: int = 1, + dropout: float = 0.0, + norm_num_groups: int = 32, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + sample_size: Optional[int] = None, + num_vector_embeds: Optional[int] = None, + patch_size: Optional[int] = 2, + activation_fn: str = "geglu", + num_embeds_ada_norm: Optional[int] = None, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + norm_type: str = "layer_norm", + block_type: str = "unidiffuser", + pre_layer_norm: bool = False, + norm_elementwise_affine: bool = True, + use_patch_pos_embed=False, + ff_final_dropout: bool = False, + ): + super().__init__() + self.use_linear_projection = use_linear_projection + self.num_attention_heads = num_attention_heads + self.attention_head_dim = attention_head_dim + inner_dim = num_attention_heads * attention_head_dim + + # 1. Input + # Only support patch input of shape (batch_size, num_channels, height, width) for now + assert in_channels is not None and patch_size is not None, "Patch input requires in_channels and patch_size." + + assert sample_size is not None, "UTransformer2DModel over patched input must provide sample_size" + + # 2. Define input layers + self.height = sample_size + self.width = sample_size + + self.patch_size = patch_size + self.pos_embed = PatchEmbed( + height=sample_size, + width=sample_size, + patch_size=patch_size, + in_channels=in_channels, + embed_dim=inner_dim, + use_pos_embed=use_patch_pos_embed, + ) + + # 3. Define transformers blocks + # Modify this to have in_blocks ("downsample" blocks, even though we don't actually downsample), a mid_block, + # and out_blocks ("upsample" blocks). Like a U-Net, there are skip connections from in_blocks to out_blocks in + # a "U"-shaped fashion (e.g. first in_block to last out_block, etc.). + # Quick hack to make the transformer block type configurable + if block_type == "unidiffuser": + block_cls = UniDiffuserBlock + else: + block_cls = UTransformerBlock + self.transformer_in_blocks = nn.ModuleList( + [ + block_cls( + inner_dim, + num_attention_heads, + attention_head_dim, + dropout=dropout, + cross_attention_dim=cross_attention_dim, + activation_fn=activation_fn, + num_embeds_ada_norm=num_embeds_ada_norm, + attention_bias=attention_bias, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + norm_type=norm_type, + pre_layer_norm=pre_layer_norm, + norm_elementwise_affine=norm_elementwise_affine, + final_dropout=ff_final_dropout, + ) + for d in range(num_layers // 2) + ] + ) + + self.transformer_mid_block = block_cls( + inner_dim, + num_attention_heads, + attention_head_dim, + dropout=dropout, + cross_attention_dim=cross_attention_dim, + activation_fn=activation_fn, + num_embeds_ada_norm=num_embeds_ada_norm, + attention_bias=attention_bias, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + norm_type=norm_type, + pre_layer_norm=pre_layer_norm, + norm_elementwise_affine=norm_elementwise_affine, + final_dropout=ff_final_dropout, + ) + + # For each skip connection, we use a SkipBlock (concatenation + Linear + LayerNorm) to process the inputs + # before each transformer out_block. + self.transformer_out_blocks = nn.ModuleList( + [ + nn.ModuleDict( + { + "skip": SkipBlock( + inner_dim, + ), + "block": block_cls( + inner_dim, + num_attention_heads, + attention_head_dim, + dropout=dropout, + cross_attention_dim=cross_attention_dim, + activation_fn=activation_fn, + num_embeds_ada_norm=num_embeds_ada_norm, + attention_bias=attention_bias, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + norm_type=norm_type, + pre_layer_norm=pre_layer_norm, + norm_elementwise_affine=norm_elementwise_affine, + final_dropout=ff_final_dropout, + ), + } + ) + for d in range(num_layers // 2) + ] + ) + + # 4. Define output layers + self.out_channels = in_channels if out_channels is None else out_channels + + # Following the UniDiffuser U-ViT implementation, we process the transformer output with + # a LayerNorm layer with per-element affine params + self.norm_out = nn.LayerNorm(inner_dim) + + def forward( + self, + hidden_states, + encoder_hidden_states=None, + timestep=None, + class_labels=None, + cross_attention_kwargs=None, + return_dict: bool = True, + hidden_states_is_embedding: bool = False, + unpatchify: bool = True, + ): + """ + Args: + hidden_states ( When discrete, `torch.LongTensor` of shape `(batch size, num latent pixels)`. + When continuous, `torch.FloatTensor` of shape `(batch size, channel, height, width)`): Input + hidden_states + encoder_hidden_states ( `torch.LongTensor` of shape `(batch size, encoder_hidden_states dim)`, *optional*): + Conditional embeddings for cross attention layer. If not given, cross-attention defaults to + self-attention. + timestep ( `torch.long`, *optional*): + Optional timestep to be applied as an embedding in AdaLayerNorm's. Used to indicate denoising step. + class_labels ( `torch.LongTensor` of shape `(batch size, num classes)`, *optional*): + Optional class labels to be applied as an embedding in AdaLayerZeroNorm. Used to indicate class labels + conditioning. + cross_attention_kwargs (*optional*): + Keyword arguments to supply to the cross attention layers, if used. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`models.unets.unet_2d_condition.UNet2DConditionOutput`] instead of a plain tuple. + hidden_states_is_embedding (`bool`, *optional*, defaults to `False`): + Whether or not hidden_states is an embedding directly usable by the transformer. In this case we will + ignore input handling (e.g. continuous, vectorized, etc.) and directly feed hidden_states into the + transformer blocks. + unpatchify (`bool`, *optional*, defaults to `True`): + Whether to unpatchify the transformer output. + + Returns: + [`~models.transformer_2d.Transformer2DModelOutput`] or `tuple`: + [`~models.transformer_2d.Transformer2DModelOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is the sample tensor. + """ + # 0. Check inputs + + if not unpatchify and return_dict: + raise ValueError( + f"Cannot both define `unpatchify`: {unpatchify} and `return_dict`: {return_dict} since when" + f" `unpatchify` is {unpatchify} the returned output is of shape (batch_size, seq_len, hidden_dim)" + " rather than (batch_size, num_channels, height, width)." + ) + + # 1. Input + if not hidden_states_is_embedding: + hidden_states = self.pos_embed(hidden_states) + + # 2. Blocks + + # In ("downsample") blocks + skips = [] + for in_block in self.transformer_in_blocks: + hidden_states = in_block( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + cross_attention_kwargs=cross_attention_kwargs, + class_labels=class_labels, + ) + skips.append(hidden_states) + + # Mid block + hidden_states = self.transformer_mid_block(hidden_states) + + # Out ("upsample") blocks + for out_block in self.transformer_out_blocks: + hidden_states = out_block["skip"](hidden_states, skips.pop()) + hidden_states = out_block["block"]( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=timestep, + cross_attention_kwargs=cross_attention_kwargs, + class_labels=class_labels, + ) + + # 3. Output + # Don't support AdaLayerNorm for now, so no conditioning/scale/shift logic + hidden_states = self.norm_out(hidden_states) + # hidden_states = self.proj_out(hidden_states) + + if unpatchify: + # unpatchify + height = width = int(hidden_states.shape[1] ** 0.5) + hidden_states = hidden_states.reshape( + shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels) + ) + hidden_states = torch.einsum("nhwpqc->nchpwq", hidden_states) + output = hidden_states.reshape( + shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size) + ) + else: + output = hidden_states + + if not return_dict: + return (output,) + + return Transformer2DModelOutput(sample=output) + + +class UniDiffuserModel(ModelMixin, ConfigMixin): + """ + Transformer model for a image-text [UniDiffuser](https://arxiv.org/pdf/2303.06555.pdf) model. This is a + modification of [`UTransformer2DModel`] with input and output heads for the VAE-embedded latent image, the + CLIP-embedded image, and the CLIP-embedded prompt (see paper for more details). + + Parameters: + text_dim (`int`): The hidden dimension of the CLIP text model used to embed images. + clip_img_dim (`int`): The hidden dimension of the CLIP vision model used to embed prompts. + num_attention_heads (`int`, *optional*, defaults to 16): The number of heads to use for multi-head attention. + attention_head_dim (`int`, *optional*, defaults to 88): The number of channels in each head. + in_channels (`int`, *optional*): + Pass if the input is continuous. The number of channels in the input. + out_channels (`int`, *optional*): + The number of output channels; if `None`, defaults to `in_channels`. + num_layers (`int`, *optional*, defaults to 1): The number of layers of Transformer blocks to use. + dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use. + norm_num_groups (`int`, *optional*, defaults to `32`): + The number of groups to use when performing Group Normalization. + cross_attention_dim (`int`, *optional*): The number of encoder_hidden_states dimensions to use. + attention_bias (`bool`, *optional*): + Configure if the TransformerBlocks' attention should contain a bias parameter. + sample_size (`int`, *optional*): Pass if the input is discrete. The width of the latent images. + Note that this is fixed at training time as it is used for learning a number of position embeddings. See + `ImagePositionalEmbeddings`. + num_vector_embeds (`int`, *optional*): + Pass if the input is discrete. The number of classes of the vector embeddings of the latent pixels. + Includes the class for the masked latent pixel. + patch_size (`int`, *optional*, defaults to 2): + The patch size to use in the patch embedding. + activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward. + num_embeds_ada_norm ( `int`, *optional*): Pass if at least one of the norm_layers is `AdaLayerNorm`. + The number of diffusion steps used during training. Note that this is fixed at training time as it is used + to learn a number of embeddings that are added to the hidden states. During inference, you can denoise for + up to but not more than steps than `num_embeds_ada_norm`. + use_linear_projection (int, *optional*): TODO: Not used + only_cross_attention (`bool`, *optional*): + Whether to use only cross-attention layers. In this case two cross attention layers are used in each + transformer block. + upcast_attention (`bool`, *optional*): + Whether to upcast the query and key to float32 when performing the attention calculation. + norm_type (`str`, *optional*, defaults to `"layer_norm"`): + The Layer Normalization implementation to use. Defaults to `torch.nn.LayerNorm`. + block_type (`str`, *optional*, defaults to `"unidiffuser"`): + The transformer block implementation to use. If `"unidiffuser"`, has the LayerNorms on the residual + backbone of each transformer block; otherwise has them in the attention/feedforward branches (the standard + behavior in `diffusers`.) + pre_layer_norm (`bool`, *optional*): + Whether to perform layer normalization before the attention and feedforward operations ("pre-LayerNorm"), + as opposed to after ("post-LayerNorm"). The original UniDiffuser implementation is post-LayerNorm + (`pre_layer_norm = False`). + norm_elementwise_affine (`bool`, *optional*): + Whether to use learnable per-element affine parameters during layer normalization. + use_patch_pos_embed (`bool`, *optional*): + Whether to use position embeddings inside the patch embedding layer (`PatchEmbed`). + ff_final_dropout (`bool`, *optional*): + Whether to use a final Dropout layer after the feedforward network. + use_data_type_embedding (`bool`, *optional*): + Whether to use a data type embedding. This is only relevant for UniDiffuser-v1 style models; UniDiffuser-v1 + is continue-trained from UniDiffuser-v0 on non-publically-available data and accepts a `data_type` + argument, which can either be `1` to use the weights trained on non-publically-available data or `0` + otherwise. This argument is subsequently embedded by the data type embedding, if used. + """ + + @register_to_config + def __init__( + self, + text_dim: int = 768, + clip_img_dim: int = 512, + num_text_tokens: int = 77, + num_attention_heads: int = 16, + attention_head_dim: int = 88, + in_channels: Optional[int] = None, + out_channels: Optional[int] = None, + num_layers: int = 1, + dropout: float = 0.0, + norm_num_groups: int = 32, + cross_attention_dim: Optional[int] = None, + attention_bias: bool = False, + sample_size: Optional[int] = None, + num_vector_embeds: Optional[int] = None, + patch_size: Optional[int] = None, + activation_fn: str = "geglu", + num_embeds_ada_norm: Optional[int] = None, + use_linear_projection: bool = False, + only_cross_attention: bool = False, + upcast_attention: bool = False, + norm_type: str = "layer_norm", + block_type: str = "unidiffuser", + pre_layer_norm: bool = False, + use_timestep_embedding=False, + norm_elementwise_affine: bool = True, + use_patch_pos_embed=False, + ff_final_dropout: bool = True, + use_data_type_embedding: bool = False, + ): + super().__init__() + + # 0. Handle dimensions + self.inner_dim = num_attention_heads * attention_head_dim + + assert sample_size is not None, "UniDiffuserModel over patched input must provide sample_size" + self.sample_size = sample_size + self.in_channels = in_channels + self.out_channels = in_channels if out_channels is None else out_channels + + self.patch_size = patch_size + # Assume image is square... + self.num_patches = (self.sample_size // patch_size) * (self.sample_size // patch_size) + + # 1. Define input layers + # 1.1 Input layers for text and image input + # For now, only support patch input for VAE latent image input + self.vae_img_in = PatchEmbed( + height=sample_size, + width=sample_size, + patch_size=patch_size, + in_channels=in_channels, + embed_dim=self.inner_dim, + use_pos_embed=use_patch_pos_embed, + ) + self.clip_img_in = nn.Linear(clip_img_dim, self.inner_dim) + self.text_in = nn.Linear(text_dim, self.inner_dim) + + # 1.2. Timestep embeddings for t_img, t_text + self.timestep_img_proj = Timesteps( + self.inner_dim, + flip_sin_to_cos=True, + downscale_freq_shift=0, + ) + self.timestep_img_embed = ( + TimestepEmbedding( + self.inner_dim, + 4 * self.inner_dim, + out_dim=self.inner_dim, + ) + if use_timestep_embedding + else nn.Identity() + ) + + self.timestep_text_proj = Timesteps( + self.inner_dim, + flip_sin_to_cos=True, + downscale_freq_shift=0, + ) + self.timestep_text_embed = ( + TimestepEmbedding( + self.inner_dim, + 4 * self.inner_dim, + out_dim=self.inner_dim, + ) + if use_timestep_embedding + else nn.Identity() + ) + + # 1.3. Positional embedding + self.num_text_tokens = num_text_tokens + self.num_tokens = 1 + 1 + num_text_tokens + 1 + self.num_patches + self.pos_embed = nn.Parameter(torch.zeros(1, self.num_tokens, self.inner_dim)) + self.pos_embed_drop = nn.Dropout(p=dropout) + trunc_normal_(self.pos_embed, std=0.02) + + # 1.4. Handle data type token embeddings for UniDiffuser-V1, if necessary + self.use_data_type_embedding = use_data_type_embedding + if self.use_data_type_embedding: + self.data_type_token_embedding = nn.Embedding(2, self.inner_dim) + self.data_type_pos_embed_token = nn.Parameter(torch.zeros(1, 1, self.inner_dim)) + + # 2. Define transformer blocks + self.transformer = UTransformer2DModel( + num_attention_heads=num_attention_heads, + attention_head_dim=attention_head_dim, + in_channels=in_channels, + out_channels=out_channels, + num_layers=num_layers, + dropout=dropout, + norm_num_groups=norm_num_groups, + cross_attention_dim=cross_attention_dim, + attention_bias=attention_bias, + sample_size=sample_size, + num_vector_embeds=num_vector_embeds, + patch_size=patch_size, + activation_fn=activation_fn, + num_embeds_ada_norm=num_embeds_ada_norm, + use_linear_projection=use_linear_projection, + only_cross_attention=only_cross_attention, + upcast_attention=upcast_attention, + norm_type=norm_type, + block_type=block_type, + pre_layer_norm=pre_layer_norm, + norm_elementwise_affine=norm_elementwise_affine, + use_patch_pos_embed=use_patch_pos_embed, + ff_final_dropout=ff_final_dropout, + ) + + # 3. Define output layers + patch_dim = (patch_size**2) * out_channels + self.vae_img_out = nn.Linear(self.inner_dim, patch_dim) + self.clip_img_out = nn.Linear(self.inner_dim, clip_img_dim) + self.text_out = nn.Linear(self.inner_dim, text_dim) + + @torch.jit.ignore + def no_weight_decay(self): + return {"pos_embed"} + + def forward( + self, + latent_image_embeds: torch.FloatTensor, + image_embeds: torch.FloatTensor, + prompt_embeds: torch.FloatTensor, + timestep_img: Union[torch.Tensor, float, int], + timestep_text: Union[torch.Tensor, float, int], + data_type: Optional[Union[torch.Tensor, float, int]] = 1, + encoder_hidden_states=None, + cross_attention_kwargs=None, + ): + """ + Args: + latent_image_embeds (`torch.FloatTensor` of shape `(batch size, latent channels, height, width)`): + Latent image representation from the VAE encoder. + image_embeds (`torch.FloatTensor` of shape `(batch size, 1, clip_img_dim)`): + CLIP-embedded image representation (unsqueezed in the first dimension). + prompt_embeds (`torch.FloatTensor` of shape `(batch size, seq_len, text_dim)`): + CLIP-embedded text representation. + timestep_img (`torch.long` or `float` or `int`): + Current denoising step for the image. + timestep_text (`torch.long` or `float` or `int`): + Current denoising step for the text. + data_type: (`torch.int` or `float` or `int`, *optional*, defaults to `1`): + Only used in UniDiffuser-v1-style models. Can be either `1`, to use weights trained on nonpublic data, + or `0` otherwise. + encoder_hidden_states ( `torch.LongTensor` of shape `(batch size, encoder_hidden_states dim)`, *optional*): + Conditional embeddings for cross attention layer. If not given, cross-attention defaults to + self-attention. + cross_attention_kwargs (*optional*): + Keyword arguments to supply to the cross attention layers, if used. + + + Returns: + `tuple`: Returns relevant parts of the model's noise prediction: the first element of the tuple is tbe VAE + image embedding, the second element is the CLIP image embedding, and the third element is the CLIP text + embedding. + """ + batch_size = latent_image_embeds.shape[0] + + # 1. Input + # 1.1. Map inputs to shape (B, N, inner_dim) + vae_hidden_states = self.vae_img_in(latent_image_embeds) + clip_hidden_states = self.clip_img_in(image_embeds) + text_hidden_states = self.text_in(prompt_embeds) + + num_text_tokens, num_img_tokens = text_hidden_states.size(1), vae_hidden_states.size(1) + + # 1.2. Encode image timesteps to single token (B, 1, inner_dim) + if not torch.is_tensor(timestep_img): + timestep_img = torch.tensor([timestep_img], dtype=torch.long, device=vae_hidden_states.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep_img = timestep_img * torch.ones(batch_size, dtype=timestep_img.dtype, device=timestep_img.device) + + timestep_img_token = self.timestep_img_proj(timestep_img) + # t_img_token does not contain any weights and will always return f32 tensors + # but time_embedding might be fp16, so we need to cast here. + timestep_img_token = timestep_img_token.to(dtype=self.dtype) + timestep_img_token = self.timestep_img_embed(timestep_img_token) + timestep_img_token = timestep_img_token.unsqueeze(dim=1) + + # 1.3. Encode text timesteps to single token (B, 1, inner_dim) + if not torch.is_tensor(timestep_text): + timestep_text = torch.tensor([timestep_text], dtype=torch.long, device=vae_hidden_states.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + timestep_text = timestep_text * torch.ones(batch_size, dtype=timestep_text.dtype, device=timestep_text.device) + + timestep_text_token = self.timestep_text_proj(timestep_text) + # t_text_token does not contain any weights and will always return f32 tensors + # but time_embedding might be fp16, so we need to cast here. + timestep_text_token = timestep_text_token.to(dtype=self.dtype) + timestep_text_token = self.timestep_text_embed(timestep_text_token) + timestep_text_token = timestep_text_token.unsqueeze(dim=1) + + # 1.4. Concatenate all of the embeddings together. + if self.use_data_type_embedding: + assert data_type is not None, "data_type must be supplied if the model uses a data type embedding" + if not torch.is_tensor(data_type): + data_type = torch.tensor([data_type], dtype=torch.int, device=vae_hidden_states.device) + + # broadcast to batch dimension in a way that's compatible with ONNX/Core ML + data_type = data_type * torch.ones(batch_size, dtype=data_type.dtype, device=data_type.device) + + data_type_token = self.data_type_token_embedding(data_type).unsqueeze(dim=1) + hidden_states = torch.cat( + [ + timestep_img_token, + timestep_text_token, + data_type_token, + text_hidden_states, + clip_hidden_states, + vae_hidden_states, + ], + dim=1, + ) + else: + hidden_states = torch.cat( + [timestep_img_token, timestep_text_token, text_hidden_states, clip_hidden_states, vae_hidden_states], + dim=1, + ) + + # 1.5. Prepare the positional embeddings and add to hidden states + # Note: I think img_vae should always have the proper shape, so there's no need to interpolate + # the position embeddings. + if self.use_data_type_embedding: + pos_embed = torch.cat( + [self.pos_embed[:, : 1 + 1, :], self.data_type_pos_embed_token, self.pos_embed[:, 1 + 1 :, :]], dim=1 + ) + else: + pos_embed = self.pos_embed + hidden_states = hidden_states + pos_embed + hidden_states = self.pos_embed_drop(hidden_states) + + # 2. Blocks + hidden_states = self.transformer( + hidden_states, + encoder_hidden_states=encoder_hidden_states, + timestep=None, + class_labels=None, + cross_attention_kwargs=cross_attention_kwargs, + return_dict=False, + hidden_states_is_embedding=True, + unpatchify=False, + )[0] + + # 3. Output + # Split out the predicted noise representation. + if self.use_data_type_embedding: + ( + t_img_token_out, + t_text_token_out, + data_type_token_out, + text_out, + img_clip_out, + img_vae_out, + ) = hidden_states.split((1, 1, 1, num_text_tokens, 1, num_img_tokens), dim=1) + else: + t_img_token_out, t_text_token_out, text_out, img_clip_out, img_vae_out = hidden_states.split( + (1, 1, num_text_tokens, 1, num_img_tokens), dim=1 + ) + + img_vae_out = self.vae_img_out(img_vae_out) + + # unpatchify + height = width = int(img_vae_out.shape[1] ** 0.5) + img_vae_out = img_vae_out.reshape( + shape=(-1, height, width, self.patch_size, self.patch_size, self.out_channels) + ) + img_vae_out = torch.einsum("nhwpqc->nchpwq", img_vae_out) + img_vae_out = img_vae_out.reshape( + shape=(-1, self.out_channels, height * self.patch_size, width * self.patch_size) + ) + + img_clip_out = self.clip_img_out(img_clip_out) + + text_out = self.text_out(text_out) + + return img_vae_out, img_clip_out, text_out diff --git a/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py b/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py new file mode 100755 index 0000000..5d61b10 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/unidiffuser/pipeline_unidiffuser.py @@ -0,0 +1,1419 @@ +import inspect +from dataclasses import dataclass +from typing import Callable, List, Optional, Union + +import numpy as np +import PIL.Image +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTokenizer, + CLIPVisionModelWithProjection, + GPT2Tokenizer, +) + +from ...image_processor import VaeImageProcessor +from ...loaders import LoraLoaderMixin, TextualInversionLoaderMixin +from ...models import AutoencoderKL +from ...models.lora import adjust_lora_scale_text_encoder +from ...schedulers import KarrasDiffusionSchedulers +from ...utils import USE_PEFT_BACKEND, deprecate, logging, scale_lora_layers, unscale_lora_layers +from ...utils.outputs import BaseOutput +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .modeling_text_decoder import UniDiffuserTextDecoder +from .modeling_uvit import UniDiffuserModel + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# New BaseOutput child class for joint image-text output +@dataclass +class ImageTextPipelineOutput(BaseOutput): + """ + Output class for joint image-text pipelines. + + Args: + images (`List[PIL.Image.Image]` or `np.ndarray`) + List of denoised PIL images of length `batch_size` or NumPy array of shape `(batch_size, height, width, + num_channels)`. + text (`List[str]` or `List[List[str]]`) + List of generated text strings of length `batch_size` or a list of list of strings whose outer list has + length `batch_size`. + """ + + images: Optional[Union[List[PIL.Image.Image], np.ndarray]] + text: Optional[Union[List[str], List[List[str]]]] + + +class UniDiffuserPipeline(DiffusionPipeline): + r""" + Pipeline for a bimodal image-text model which supports unconditional text and image generation, text-conditioned + image generation, image-conditioned text generation, and joint image-text generation. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods + implemented for all pipelines (downloading, saving, running on a particular device, etc.). + + Args: + vae ([`AutoencoderKL`]): + Variational Auto-Encoder (VAE) model to encode and decode images to and from latent representations. This + is part of the UniDiffuser image representation along with the CLIP vision encoding. + text_encoder ([`CLIPTextModel`]): + Frozen text-encoder ([clip-vit-large-patch14](https://huggingface.co/openai/clip-vit-large-patch14)). + image_encoder ([`CLIPVisionModel`]): + A [`~transformers.CLIPVisionModel`] to encode images as part of its image representation along with the VAE + latent representation. + image_processor ([`CLIPImageProcessor`]): + [`~transformers.CLIPImageProcessor`] to preprocess an image before CLIP encoding it with `image_encoder`. + clip_tokenizer ([`CLIPTokenizer`]): + A [`~transformers.CLIPTokenizer`] to tokenize the prompt before encoding it with `text_encoder`. + text_decoder ([`UniDiffuserTextDecoder`]): + Frozen text decoder. This is a GPT-style model which is used to generate text from the UniDiffuser + embedding. + text_tokenizer ([`GPT2Tokenizer`]): + A [`~transformers.GPT2Tokenizer`] to decode text for text generation; used along with the `text_decoder`. + unet ([`UniDiffuserModel`]): + A [U-ViT](https://github.com/baofff/U-ViT) model with UNNet-style skip connections between transformer + layers to denoise the encoded image latents. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image and/or text latents. The + original UniDiffuser paper uses the [`DPMSolverMultistepScheduler`] scheduler. + """ + + # TODO: support for moving submodules for components with enable_model_cpu_offload + model_cpu_offload_seq = "text_encoder->image_encoder->unet->vae->text_decoder" + + def __init__( + self, + vae: AutoencoderKL, + text_encoder: CLIPTextModel, + image_encoder: CLIPVisionModelWithProjection, + clip_image_processor: CLIPImageProcessor, + clip_tokenizer: CLIPTokenizer, + text_decoder: UniDiffuserTextDecoder, + text_tokenizer: GPT2Tokenizer, + unet: UniDiffuserModel, + scheduler: KarrasDiffusionSchedulers, + ): + super().__init__() + + if text_encoder.config.hidden_size != text_decoder.prefix_inner_dim: + raise ValueError( + f"The text encoder hidden size and text decoder prefix inner dim must be the same, but" + f" `text_encoder.config.hidden_size`: {text_encoder.config.hidden_size} and `text_decoder.prefix_inner_dim`: {text_decoder.prefix_inner_dim}" + ) + + self.register_modules( + vae=vae, + text_encoder=text_encoder, + image_encoder=image_encoder, + clip_image_processor=clip_image_processor, + clip_tokenizer=clip_tokenizer, + text_decoder=text_decoder, + text_tokenizer=text_tokenizer, + unet=unet, + scheduler=scheduler, + ) + + self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1) + self.image_processor = VaeImageProcessor(vae_scale_factor=self.vae_scale_factor) + + self.num_channels_latents = vae.config.latent_channels + self.text_encoder_seq_len = text_encoder.config.max_position_embeddings + self.text_encoder_hidden_size = text_encoder.config.hidden_size + self.image_encoder_projection_dim = image_encoder.config.projection_dim + self.unet_resolution = unet.config.sample_size + + self.text_intermediate_dim = self.text_encoder_hidden_size + if self.text_decoder.prefix_hidden_dim is not None: + self.text_intermediate_dim = self.text_decoder.prefix_hidden_dim + + self.mode = None + + # TODO: handle safety checking? + self.safety_checker = None + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_extra_step_kwargs + def prepare_extra_step_kwargs(self, generator, eta): + # prepare extra kwargs for the scheduler step, since not all schedulers have the same signature + # eta (η) is only used with the DDIMScheduler, it will be ignored for other schedulers. + # eta corresponds to η in DDIM paper: https://arxiv.org/abs/2010.02502 + # and should be between [0, 1] + + accepts_eta = "eta" in set(inspect.signature(self.scheduler.step).parameters.keys()) + extra_step_kwargs = {} + if accepts_eta: + extra_step_kwargs["eta"] = eta + + # check if the scheduler accepts generator + accepts_generator = "generator" in set(inspect.signature(self.scheduler.step).parameters.keys()) + if accepts_generator: + extra_step_kwargs["generator"] = generator + return extra_step_kwargs + + def _infer_mode(self, prompt, prompt_embeds, image, latents, prompt_latents, vae_latents, clip_latents): + r""" + Infer the generation task ('mode') from the inputs to `__call__`. If the mode has been manually set, the set + mode will be used. + """ + prompt_available = (prompt is not None) or (prompt_embeds is not None) + image_available = image is not None + input_available = prompt_available or image_available + + prompt_latents_available = prompt_latents is not None + vae_latents_available = vae_latents is not None + clip_latents_available = clip_latents is not None + full_latents_available = latents is not None + image_latents_available = vae_latents_available and clip_latents_available + all_indv_latents_available = prompt_latents_available and image_latents_available + + if self.mode is not None: + # Preferentially use the mode set by the user + mode = self.mode + elif prompt_available: + mode = "text2img" + elif image_available: + mode = "img2text" + else: + # Neither prompt nor image supplied, infer based on availability of latents + if full_latents_available or all_indv_latents_available: + mode = "joint" + elif prompt_latents_available: + mode = "text" + elif image_latents_available: + mode = "img" + else: + # No inputs or latents available + mode = "joint" + + # Give warnings for ambiguous cases + if self.mode is None and prompt_available and image_available: + logger.warning( + f"You have supplied both a text prompt and image to the pipeline and mode has not been set manually," + f" defaulting to mode '{mode}'." + ) + + if self.mode is None and not input_available: + if vae_latents_available != clip_latents_available: + # Exactly one of vae_latents and clip_latents is supplied + logger.warning( + f"You have supplied exactly one of `vae_latents` and `clip_latents`, whereas either both or none" + f" are expected to be supplied. Defaulting to mode '{mode}'." + ) + elif not prompt_latents_available and not vae_latents_available and not clip_latents_available: + # No inputs or latents supplied + logger.warning( + f"No inputs or latents have been supplied, and mode has not been manually set," + f" defaulting to mode '{mode}'." + ) + + return mode + + # Copied from diffusers.pipelines.pipeline_utils.StableDiffusionMixin.enable_vae_slicing + def enable_vae_slicing(self): + r""" + Enable sliced VAE decoding. When this option is enabled, the VAE will split the input tensor in slices to + compute decoding in several steps. This is useful to save some memory and allow larger batch sizes. + """ + self.vae.enable_slicing() + + # Copied from diffusers.pipelines.pipeline_utils.StableDiffusionMixin.disable_vae_slicing + def disable_vae_slicing(self): + r""" + Disable sliced VAE decoding. If `enable_vae_slicing` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_slicing() + + # Copied from diffusers.pipelines.pipeline_utils.StableDiffusionMixin.enable_vae_tiling + def enable_vae_tiling(self): + r""" + Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to + compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow + processing larger images. + """ + self.vae.enable_tiling() + + # Copied from diffusers.pipelines.pipeline_utils.StableDiffusionMixin.disable_vae_tiling + def disable_vae_tiling(self): + r""" + Disable tiled VAE decoding. If `enable_vae_tiling` was previously enabled, this method will go back to + computing decoding in one step. + """ + self.vae.disable_tiling() + + # Functions to manually set the mode + def set_text_mode(self): + r"""Manually set the generation mode to unconditional ("marginal") text generation.""" + self.mode = "text" + + def set_image_mode(self): + r"""Manually set the generation mode to unconditional ("marginal") image generation.""" + self.mode = "img" + + def set_text_to_image_mode(self): + r"""Manually set the generation mode to text-conditioned image generation.""" + self.mode = "text2img" + + def set_image_to_text_mode(self): + r"""Manually set the generation mode to image-conditioned text generation.""" + self.mode = "img2text" + + def set_joint_mode(self): + r"""Manually set the generation mode to unconditional joint image-text generation.""" + self.mode = "joint" + + def reset_mode(self): + r"""Removes a manually set mode; after calling this, the pipeline will infer the mode from inputs.""" + self.mode = None + + def _infer_batch_size( + self, + mode, + prompt, + prompt_embeds, + image, + num_images_per_prompt, + num_prompts_per_image, + latents, + prompt_latents, + vae_latents, + clip_latents, + ): + r"""Infers the batch size and multiplier depending on mode and supplied arguments to `__call__`.""" + if num_images_per_prompt is None: + num_images_per_prompt = 1 + if num_prompts_per_image is None: + num_prompts_per_image = 1 + + assert num_images_per_prompt > 0, "num_images_per_prompt must be a positive integer" + assert num_prompts_per_image > 0, "num_prompts_per_image must be a positive integer" + + if mode in ["text2img"]: + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + # Either prompt or prompt_embeds must be present for text2img. + batch_size = prompt_embeds.shape[0] + multiplier = num_images_per_prompt + elif mode in ["img2text"]: + if isinstance(image, PIL.Image.Image): + batch_size = 1 + else: + # Image must be available and type either PIL.Image.Image or torch.FloatTensor. + # Not currently supporting something like image_embeds. + batch_size = image.shape[0] + multiplier = num_prompts_per_image + elif mode in ["img"]: + if vae_latents is not None: + batch_size = vae_latents.shape[0] + elif clip_latents is not None: + batch_size = clip_latents.shape[0] + else: + batch_size = 1 + multiplier = num_images_per_prompt + elif mode in ["text"]: + if prompt_latents is not None: + batch_size = prompt_latents.shape[0] + else: + batch_size = 1 + multiplier = num_prompts_per_image + elif mode in ["joint"]: + if latents is not None: + batch_size = latents.shape[0] + elif prompt_latents is not None: + batch_size = prompt_latents.shape[0] + elif vae_latents is not None: + batch_size = vae_latents.shape[0] + elif clip_latents is not None: + batch_size = clip_latents.shape[0] + else: + batch_size = 1 + + if num_images_per_prompt == num_prompts_per_image: + multiplier = num_images_per_prompt + else: + multiplier = min(num_images_per_prompt, num_prompts_per_image) + logger.warning( + f"You are using mode `{mode}` and `num_images_per_prompt`: {num_images_per_prompt} and" + f" num_prompts_per_image: {num_prompts_per_image} are not equal. Using batch size equal to" + f" `min(num_images_per_prompt, num_prompts_per_image) = {batch_size}." + ) + return batch_size, multiplier + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline._encode_prompt + def _encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + **kwargs, + ): + deprecation_message = "`_encode_prompt()` is deprecated and it will be removed in a future version. Use `encode_prompt()` instead. Also, be aware that the output format changed from a concatenated tensor to a tuple." + deprecate("_encode_prompt()", "1.0.0", deprecation_message, standard_warn=False) + + prompt_embeds_tuple = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + lora_scale=lora_scale, + **kwargs, + ) + + # concatenate for backwards comp + prompt_embeds = torch.cat([prompt_embeds_tuple[1], prompt_embeds_tuple[0]]) + + return prompt_embeds + + # Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.encode_prompt with self.tokenizer->self.clip_tokenizer + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + lora_scale: Optional[float] = None, + clip_skip: Optional[int] = None, + ): + r""" + Encodes the prompt into text encoder hidden states. + + Args: + prompt (`str` or `List[str]`, *optional*): + prompt to be encoded + device: (`torch.device`): + torch device + num_images_per_prompt (`int`): + number of images that should be generated per prompt + do_classifier_free_guidance (`bool`): + whether to use classifier free guidance or not + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. If not defined, one has to pass + `negative_prompt_embeds` instead. Ignored when not using guidance (i.e., ignored if `guidance_scale` is + less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + lora_scale (`float`, *optional*): + A LoRA scale that will be applied to all LoRA layers of the text encoder if LoRA layers are loaded. + clip_skip (`int`, *optional*): + Number of layers to be skipped from CLIP while computing the prompt embeddings. A value of 1 means that + the output of the pre-final layer will be used for computing the prompt embeddings. + """ + # set lora scale so that monkey patched LoRA + # function of text encoder can correctly access it + if lora_scale is not None and isinstance(self, LoraLoaderMixin): + self._lora_scale = lora_scale + + # dynamically adjust the LoRA scale + if not USE_PEFT_BACKEND: + adjust_lora_scale_text_encoder(self.text_encoder, lora_scale) + else: + scale_lora_layers(self.text_encoder, lora_scale) + + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + prompt = self.maybe_convert_prompt(prompt, self.clip_tokenizer) + + text_inputs = self.clip_tokenizer( + prompt, + padding="max_length", + max_length=self.clip_tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + untruncated_ids = self.clip_tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.clip_tokenizer.batch_decode( + untruncated_ids[:, self.clip_tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.clip_tokenizer.model_max_length} tokens: {removed_text}" + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = text_inputs.attention_mask.to(device) + else: + attention_mask = None + + if clip_skip is None: + prompt_embeds = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask) + prompt_embeds = prompt_embeds[0] + else: + prompt_embeds = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask, output_hidden_states=True + ) + # Access the `hidden_states` first, that contains a tuple of + # all the hidden states from the encoder layers. Then index into + # the tuple to access the hidden states from the desired layer. + prompt_embeds = prompt_embeds[-1][-(clip_skip + 1)] + # We also need to apply the final LayerNorm here to not mess with the + # representations. The `last_hidden_states` that we typically use for + # obtaining the final prompt representations passes through the LayerNorm + # layer. + prompt_embeds = self.text_encoder.text_model.final_layer_norm(prompt_embeds) + + if self.text_encoder is not None: + prompt_embeds_dtype = self.text_encoder.dtype + elif self.unet is not None: + prompt_embeds_dtype = self.unet.dtype + else: + prompt_embeds_dtype = prompt_embeds.dtype + + prompt_embeds = prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + bs_embed, seq_len, _ = prompt_embeds.shape + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # get unconditional embeddings for classifier free guidance + if do_classifier_free_guidance and negative_prompt_embeds is None: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif prompt is not None and type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + # textual inversion: process multi-vector tokens if necessary + if isinstance(self, TextualInversionLoaderMixin): + uncond_tokens = self.maybe_convert_prompt(uncond_tokens, self.clip_tokenizer) + + max_length = prompt_embeds.shape[1] + uncond_input = self.clip_tokenizer( + uncond_tokens, + padding="max_length", + max_length=max_length, + truncation=True, + return_tensors="pt", + ) + + if hasattr(self.text_encoder.config, "use_attention_mask") and self.text_encoder.config.use_attention_mask: + attention_mask = uncond_input.attention_mask.to(device) + else: + attention_mask = None + + negative_prompt_embeds = self.text_encoder( + uncond_input.input_ids.to(device), + attention_mask=attention_mask, + ) + negative_prompt_embeds = negative_prompt_embeds[0] + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.to(dtype=prompt_embeds_dtype, device=device) + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + if isinstance(self, LoraLoaderMixin) and USE_PEFT_BACKEND: + # Retrieve the original scale by scaling back the LoRA layers + unscale_lora_layers(self.text_encoder, lora_scale) + + return prompt_embeds, negative_prompt_embeds + + # Modified from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_instruct_pix2pix.StableDiffusionInstructPix2PixPipeline.prepare_image_latents + # Add num_prompts_per_image argument, sample from autoencoder moment distribution + def encode_image_vae_latents( + self, + image, + batch_size, + num_prompts_per_image, + dtype, + device, + do_classifier_free_guidance, + generator=None, + ): + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + image = image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_prompts_per_image + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if isinstance(generator, list): + image_latents = [ + self.vae.encode(image[i : i + 1]).latent_dist.sample(generator=generator[i]) + * self.vae.config.scaling_factor + for i in range(batch_size) + ] + image_latents = torch.cat(image_latents, dim=0) + else: + image_latents = self.vae.encode(image).latent_dist.sample(generator=generator) + # Scale image_latents by the VAE's scaling factor + image_latents = image_latents * self.vae.config.scaling_factor + + if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0: + # expand image_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {image_latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_image_per_prompt = batch_size // image_latents.shape[0] + image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0) + elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts." + ) + else: + image_latents = torch.cat([image_latents], dim=0) + + if do_classifier_free_guidance: + uncond_image_latents = torch.zeros_like(image_latents) + image_latents = torch.cat([image_latents, image_latents, uncond_image_latents], dim=0) + + return image_latents + + def encode_image_clip_latents( + self, + image, + batch_size, + num_prompts_per_image, + dtype, + device, + generator=None, + ): + # Map image to CLIP embedding. + if not isinstance(image, (torch.Tensor, PIL.Image.Image, list)): + raise ValueError( + f"`image` has to be of type `torch.Tensor`, `PIL.Image.Image` or list but is {type(image)}" + ) + + preprocessed_image = self.clip_image_processor.preprocess( + image, + return_tensors="pt", + ) + preprocessed_image = preprocessed_image.to(device=device, dtype=dtype) + + batch_size = batch_size * num_prompts_per_image + if isinstance(generator, list): + image_latents = [ + self.image_encoder(**preprocessed_image[i : i + 1]).image_embeds for i in range(batch_size) + ] + image_latents = torch.cat(image_latents, dim=0) + else: + image_latents = self.image_encoder(**preprocessed_image).image_embeds + + if batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] == 0: + # expand image_latents for batch_size + deprecation_message = ( + f"You have passed {batch_size} text prompts (`prompt`), but only {image_latents.shape[0]} initial" + " images (`image`). Initial images are now duplicating to match the number of text prompts. Note" + " that this behavior is deprecated and will be removed in a version 1.0.0. Please make sure to update" + " your script to pass as many initial images as text prompts to suppress this warning." + ) + deprecate("len(prompt) != len(image)", "1.0.0", deprecation_message, standard_warn=False) + additional_image_per_prompt = batch_size // image_latents.shape[0] + image_latents = torch.cat([image_latents] * additional_image_per_prompt, dim=0) + elif batch_size > image_latents.shape[0] and batch_size % image_latents.shape[0] != 0: + raise ValueError( + f"Cannot duplicate `image` of batch size {image_latents.shape[0]} to {batch_size} text prompts." + ) + else: + image_latents = torch.cat([image_latents], dim=0) + + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + return image_latents + + def prepare_text_latents( + self, batch_size, num_images_per_prompt, seq_len, hidden_size, dtype, device, generator, latents=None + ): + # Prepare latents for the CLIP embedded prompt. + shape = (batch_size * num_images_per_prompt, seq_len, hidden_size) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + # latents is assumed to have shace (B, L, D) + latents = latents.repeat(num_images_per_prompt, 1, 1) + latents = latents.to(device=device, dtype=dtype) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + # Modified from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents + # Rename prepare_latents -> prepare_image_vae_latents and add num_prompts_per_image argument. + def prepare_image_vae_latents( + self, + batch_size, + num_prompts_per_image, + num_channels_latents, + height, + width, + dtype, + device, + generator, + latents=None, + ): + shape = ( + batch_size * num_prompts_per_image, + num_channels_latents, + height // self.vae_scale_factor, + width // self.vae_scale_factor, + ) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + # latents is assumed to have shape (B, C, H, W) + latents = latents.repeat(num_prompts_per_image, 1, 1, 1) + latents = latents.to(device=device, dtype=dtype) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def prepare_image_clip_latents( + self, batch_size, num_prompts_per_image, clip_img_dim, dtype, device, generator, latents=None + ): + # Prepare latents for the CLIP embedded image. + shape = (batch_size * num_prompts_per_image, 1, clip_img_dim) + if isinstance(generator, list) and len(generator) != batch_size: + raise ValueError( + f"You have passed a list of generators of length {len(generator)}, but requested an effective batch" + f" size of {batch_size}. Make sure the batch size matches the length of the generators." + ) + + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + # latents is assumed to have shape (B, L, D) + latents = latents.repeat(num_prompts_per_image, 1, 1) + latents = latents.to(device=device, dtype=dtype) + + # scale the initial noise by the standard deviation required by the scheduler + latents = latents * self.scheduler.init_noise_sigma + return latents + + def decode_text_latents(self, text_latents, device): + output_token_list, seq_lengths = self.text_decoder.generate_captions( + text_latents, self.text_tokenizer.eos_token_id, device=device + ) + output_list = output_token_list.cpu().numpy() + generated_text = [ + self.text_tokenizer.decode(output[: int(length)], skip_special_tokens=True) + for output, length in zip(output_list, seq_lengths) + ] + return generated_text + + def _split(self, x, height, width): + r""" + Splits a flattened embedding x of shape (B, C * H * W + clip_img_dim) into two tensors of shape (B, C, H, W) + and (B, 1, clip_img_dim) + """ + batch_size = x.shape[0] + latent_height = height // self.vae_scale_factor + latent_width = width // self.vae_scale_factor + img_vae_dim = self.num_channels_latents * latent_height * latent_width + + img_vae, img_clip = x.split([img_vae_dim, self.image_encoder_projection_dim], dim=1) + + img_vae = torch.reshape(img_vae, (batch_size, self.num_channels_latents, latent_height, latent_width)) + img_clip = torch.reshape(img_clip, (batch_size, 1, self.image_encoder_projection_dim)) + return img_vae, img_clip + + def _combine(self, img_vae, img_clip): + r""" + Combines a latent iamge img_vae of shape (B, C, H, W) and a CLIP-embedded image img_clip of shape (B, 1, + clip_img_dim) into a single tensor of shape (B, C * H * W + clip_img_dim). + """ + img_vae = torch.reshape(img_vae, (img_vae.shape[0], -1)) + img_clip = torch.reshape(img_clip, (img_clip.shape[0], -1)) + return torch.concat([img_vae, img_clip], dim=-1) + + def _split_joint(self, x, height, width): + r""" + Splits a flattened embedding x of shape (B, C * H * W + clip_img_dim + text_seq_len * text_dim] into (img_vae, + img_clip, text) where img_vae is of shape (B, C, H, W), img_clip is of shape (B, 1, clip_img_dim), and text is + of shape (B, text_seq_len, text_dim). + """ + batch_size = x.shape[0] + latent_height = height // self.vae_scale_factor + latent_width = width // self.vae_scale_factor + img_vae_dim = self.num_channels_latents * latent_height * latent_width + text_dim = self.text_encoder_seq_len * self.text_intermediate_dim + + img_vae, img_clip, text = x.split([img_vae_dim, self.image_encoder_projection_dim, text_dim], dim=1) + + img_vae = torch.reshape(img_vae, (batch_size, self.num_channels_latents, latent_height, latent_width)) + img_clip = torch.reshape(img_clip, (batch_size, 1, self.image_encoder_projection_dim)) + text = torch.reshape(text, (batch_size, self.text_encoder_seq_len, self.text_intermediate_dim)) + return img_vae, img_clip, text + + def _combine_joint(self, img_vae, img_clip, text): + r""" + Combines a latent image img_vae of shape (B, C, H, W), a CLIP-embedded image img_clip of shape (B, L_img, + clip_img_dim), and a text embedding text of shape (B, L_text, text_dim) into a single embedding x of shape (B, + C * H * W + L_img * clip_img_dim + L_text * text_dim). + """ + img_vae = torch.reshape(img_vae, (img_vae.shape[0], -1)) + img_clip = torch.reshape(img_clip, (img_clip.shape[0], -1)) + text = torch.reshape(text, (text.shape[0], -1)) + return torch.concat([img_vae, img_clip, text], dim=-1) + + def _get_noise_pred( + self, + mode, + latents, + t, + prompt_embeds, + img_vae, + img_clip, + max_timestep, + data_type, + guidance_scale, + generator, + device, + height, + width, + ): + r""" + Gets the noise prediction using the `unet` and performs classifier-free guidance, if necessary. + """ + if mode == "joint": + # Joint text-image generation + img_vae_latents, img_clip_latents, text_latents = self._split_joint(latents, height, width) + + img_vae_out, img_clip_out, text_out = self.unet( + img_vae_latents, img_clip_latents, text_latents, timestep_img=t, timestep_text=t, data_type=data_type + ) + + x_out = self._combine_joint(img_vae_out, img_clip_out, text_out) + + if guidance_scale <= 1.0: + return x_out + + # Classifier-free guidance + img_vae_T = randn_tensor(img_vae.shape, generator=generator, device=device, dtype=img_vae.dtype) + img_clip_T = randn_tensor(img_clip.shape, generator=generator, device=device, dtype=img_clip.dtype) + text_T = randn_tensor(prompt_embeds.shape, generator=generator, device=device, dtype=prompt_embeds.dtype) + + _, _, text_out_uncond = self.unet( + img_vae_T, img_clip_T, text_latents, timestep_img=max_timestep, timestep_text=t, data_type=data_type + ) + + img_vae_out_uncond, img_clip_out_uncond, _ = self.unet( + img_vae_latents, + img_clip_latents, + text_T, + timestep_img=t, + timestep_text=max_timestep, + data_type=data_type, + ) + + x_out_uncond = self._combine_joint(img_vae_out_uncond, img_clip_out_uncond, text_out_uncond) + + return guidance_scale * x_out + (1.0 - guidance_scale) * x_out_uncond + elif mode == "text2img": + # Text-conditioned image generation + img_vae_latents, img_clip_latents = self._split(latents, height, width) + + img_vae_out, img_clip_out, text_out = self.unet( + img_vae_latents, img_clip_latents, prompt_embeds, timestep_img=t, timestep_text=0, data_type=data_type + ) + + img_out = self._combine(img_vae_out, img_clip_out) + + if guidance_scale <= 1.0: + return img_out + + # Classifier-free guidance + text_T = randn_tensor(prompt_embeds.shape, generator=generator, device=device, dtype=prompt_embeds.dtype) + + img_vae_out_uncond, img_clip_out_uncond, text_out_uncond = self.unet( + img_vae_latents, + img_clip_latents, + text_T, + timestep_img=t, + timestep_text=max_timestep, + data_type=data_type, + ) + + img_out_uncond = self._combine(img_vae_out_uncond, img_clip_out_uncond) + + return guidance_scale * img_out + (1.0 - guidance_scale) * img_out_uncond + elif mode == "img2text": + # Image-conditioned text generation + img_vae_out, img_clip_out, text_out = self.unet( + img_vae, img_clip, latents, timestep_img=0, timestep_text=t, data_type=data_type + ) + + if guidance_scale <= 1.0: + return text_out + + # Classifier-free guidance + img_vae_T = randn_tensor(img_vae.shape, generator=generator, device=device, dtype=img_vae.dtype) + img_clip_T = randn_tensor(img_clip.shape, generator=generator, device=device, dtype=img_clip.dtype) + + img_vae_out_uncond, img_clip_out_uncond, text_out_uncond = self.unet( + img_vae_T, img_clip_T, latents, timestep_img=max_timestep, timestep_text=t, data_type=data_type + ) + + return guidance_scale * text_out + (1.0 - guidance_scale) * text_out_uncond + elif mode == "text": + # Unconditional ("marginal") text generation (no CFG) + img_vae_out, img_clip_out, text_out = self.unet( + img_vae, img_clip, latents, timestep_img=max_timestep, timestep_text=t, data_type=data_type + ) + + return text_out + elif mode == "img": + # Unconditional ("marginal") image generation (no CFG) + img_vae_latents, img_clip_latents = self._split(latents, height, width) + + img_vae_out, img_clip_out, text_out = self.unet( + img_vae_latents, + img_clip_latents, + prompt_embeds, + timestep_img=t, + timestep_text=max_timestep, + data_type=data_type, + ) + + img_out = self._combine(img_vae_out, img_clip_out) + return img_out + + def check_latents_shape(self, latents_name, latents, expected_shape): + latents_shape = latents.shape + expected_num_dims = len(expected_shape) + 1 # expected dimensions plus the batch dimension + expected_shape_str = ", ".join(str(dim) for dim in expected_shape) + if len(latents_shape) != expected_num_dims: + raise ValueError( + f"`{latents_name}` should have shape (batch_size, {expected_shape_str}), but the current shape" + f" {latents_shape} has {len(latents_shape)} dimensions." + ) + for i in range(1, expected_num_dims): + if latents_shape[i] != expected_shape[i - 1]: + raise ValueError( + f"`{latents_name}` should have shape (batch_size, {expected_shape_str}), but the current shape" + f" {latents_shape} has {latents_shape[i]} != {expected_shape[i - 1]} at dimension {i}." + ) + + def check_inputs( + self, + mode, + prompt, + image, + height, + width, + callback_steps, + negative_prompt=None, + prompt_embeds=None, + negative_prompt_embeds=None, + latents=None, + prompt_latents=None, + vae_latents=None, + clip_latents=None, + ): + # Check inputs before running the generative process. + if height % self.vae_scale_factor != 0 or width % self.vae_scale_factor != 0: + raise ValueError( + f"`height` and `width` have to be divisible by {self.vae_scale_factor} but are {height} and {width}." + ) + + if (callback_steps is None) or ( + callback_steps is not None and (not isinstance(callback_steps, int) or callback_steps <= 0) + ): + raise ValueError( + f"`callback_steps` has to be a positive integer but is {callback_steps} of type" + f" {type(callback_steps)}." + ) + + if mode == "text2img": + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if mode == "img2text": + if image is None: + raise ValueError("`img2text` mode requires an image to be provided.") + + # Check provided latents + latent_height = height // self.vae_scale_factor + latent_width = width // self.vae_scale_factor + full_latents_available = latents is not None + prompt_latents_available = prompt_latents is not None + vae_latents_available = vae_latents is not None + clip_latents_available = clip_latents is not None + + if full_latents_available: + individual_latents_available = ( + prompt_latents is not None or vae_latents is not None or clip_latents is not None + ) + if individual_latents_available: + logger.warning( + "You have supplied both `latents` and at least one of `prompt_latents`, `vae_latents`, and" + " `clip_latents`. The value of `latents` will override the value of any individually supplied latents." + ) + # Check shape of full latents + img_vae_dim = self.num_channels_latents * latent_height * latent_width + text_dim = self.text_encoder_seq_len * self.text_encoder_hidden_size + latents_dim = img_vae_dim + self.image_encoder_projection_dim + text_dim + latents_expected_shape = (latents_dim,) + self.check_latents_shape("latents", latents, latents_expected_shape) + + # Check individual latent shapes, if present + if prompt_latents_available: + prompt_latents_expected_shape = (self.text_encoder_seq_len, self.text_encoder_hidden_size) + self.check_latents_shape("prompt_latents", prompt_latents, prompt_latents_expected_shape) + + if vae_latents_available: + vae_latents_expected_shape = (self.num_channels_latents, latent_height, latent_width) + self.check_latents_shape("vae_latents", vae_latents, vae_latents_expected_shape) + + if clip_latents_available: + clip_latents_expected_shape = (1, self.image_encoder_projection_dim) + self.check_latents_shape("clip_latents", clip_latents, clip_latents_expected_shape) + + if mode in ["text2img", "img"] and vae_latents_available and clip_latents_available: + if vae_latents.shape[0] != clip_latents.shape[0]: + raise ValueError( + f"Both `vae_latents` and `clip_latents` are supplied, but their batch dimensions are not equal:" + f" {vae_latents.shape[0]} != {clip_latents.shape[0]}." + ) + + if mode == "joint" and prompt_latents_available and vae_latents_available and clip_latents_available: + if prompt_latents.shape[0] != vae_latents.shape[0] or prompt_latents.shape[0] != clip_latents.shape[0]: + raise ValueError( + f"All of `prompt_latents`, `vae_latents`, and `clip_latents` are supplied, but their batch" + f" dimensions are not equal: {prompt_latents.shape[0]} != {vae_latents.shape[0]}" + f" != {clip_latents.shape[0]}." + ) + + @torch.no_grad() + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + image: Optional[Union[torch.FloatTensor, PIL.Image.Image]] = None, + height: Optional[int] = None, + width: Optional[int] = None, + data_type: Optional[int] = 1, + num_inference_steps: int = 50, + guidance_scale: float = 8.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: Optional[int] = 1, + num_prompts_per_image: Optional[int] = 1, + eta: float = 0.0, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + prompt_latents: Optional[torch.FloatTensor] = None, + vae_latents: Optional[torch.FloatTensor] = None, + clip_latents: Optional[torch.FloatTensor] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback: Optional[Callable[[int, int, torch.FloatTensor], None]] = None, + callback_steps: int = 1, + ): + r""" + The call function to the pipeline for generation. + + Args: + prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide image generation. If not defined, you need to pass `prompt_embeds`. + Required for text-conditioned image generation (`text2img`) mode. + image (`torch.FloatTensor` or `PIL.Image.Image`, *optional*): + `Image` or tensor representing an image batch. Required for image-conditioned text generation + (`img2text`) mode. + height (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to `self.unet.config.sample_size * self.vae_scale_factor`): + The width in pixels of the generated image. + data_type (`int`, *optional*, defaults to 1): + The data type (either 0 or 1). Only used if you are loading a checkpoint which supports a data type + embedding; this is added for compatibility with the + [UniDiffuser-v1](https://huggingface.co/thu-ml/unidiffuser-v1) checkpoint. + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + guidance_scale (`float`, *optional*, defaults to 8.0): + A higher guidance scale value encourages the model to generate images closely linked to the text + `prompt` at the expense of lower image quality. Guidance scale is enabled when `guidance_scale > 1`. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts to guide what to not include in image generation. If not defined, you need to + pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale < 1`). Used in + text-conditioned image generation (`text2img`) mode. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. Used in `text2img` (text-conditioned image generation) and + `img` mode. If the mode is joint and both `num_images_per_prompt` and `num_prompts_per_image` are + supplied, `min(num_images_per_prompt, num_prompts_per_image)` samples are generated. + num_prompts_per_image (`int`, *optional*, defaults to 1): + The number of prompts to generate per image. Used in `img2text` (image-conditioned text generation) and + `text` mode. If the mode is joint and both `num_images_per_prompt` and `num_prompts_per_image` are + supplied, `min(num_images_per_prompt, num_prompts_per_image)` samples are generated. + eta (`float`, *optional*, defaults to 0.0): + Corresponds to parameter eta (η) from the [DDIM](https://arxiv.org/abs/2010.02502) paper. Only applies + to the [`~schedulers.DDIMScheduler`], and is ignored in other schedulers. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make + generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for joint + image-text generation. Can be used to tweak the same generation with different prompts. If not + provided, a latents tensor is generated by sampling using the supplied random `generator`. This assumes + a full set of VAE, CLIP, and text latents, if supplied, overrides the value of `prompt_latents`, + `vae_latents`, and `clip_latents`. + prompt_latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for text + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + vae_latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + clip_latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor is generated by sampling using the supplied random `generator`. + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs (prompt weighting). If not + provided, text embeddings are generated from the `prompt` input argument. Used in text-conditioned + image generation (`text2img`) mode. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs (prompt weighting). If + not provided, `negative_prompt_embeds` are be generated from the `negative_prompt` input argument. Used + in text-conditioned image generation (`text2img`) mode. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generated image. Choose between `PIL.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImageTextPipelineOutput`] instead of a plain tuple. + callback (`Callable`, *optional*): + A function that calls every `callback_steps` steps during inference. The function is called with the + following arguments: `callback(step: int, timestep: int, latents: torch.FloatTensor)`. + callback_steps (`int`, *optional*, defaults to 1): + The frequency at which the `callback` function is called. If not specified, the callback is called at + every step. + + Returns: + [`~pipelines.unidiffuser.ImageTextPipelineOutput`] or `tuple`: + If `return_dict` is `True`, [`~pipelines.unidiffuser.ImageTextPipelineOutput`] is returned, otherwise a + `tuple` is returned where the first element is a list with the generated images and the second element + is a list of generated texts. + """ + + # 0. Default height and width to unet + height = height or self.unet_resolution * self.vae_scale_factor + width = width or self.unet_resolution * self.vae_scale_factor + + # 1. Check inputs + # Recalculate mode for each call to the pipeline. + mode = self._infer_mode(prompt, prompt_embeds, image, latents, prompt_latents, vae_latents, clip_latents) + self.check_inputs( + mode, + prompt, + image, + height, + width, + callback_steps, + negative_prompt, + prompt_embeds, + negative_prompt_embeds, + latents, + prompt_latents, + vae_latents, + clip_latents, + ) + + # 2. Define call parameters + batch_size, multiplier = self._infer_batch_size( + mode, + prompt, + prompt_embeds, + image, + num_images_per_prompt, + num_prompts_per_image, + latents, + prompt_latents, + vae_latents, + clip_latents, + ) + device = self._execution_device + reduce_text_emb_dim = self.text_intermediate_dim < self.text_encoder_hidden_size or self.mode != "text2img" + + # here `guidance_scale` is defined analog to the guidance weight `w` of equation (2) + # of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1` + # corresponds to doing no classifier free guidance. + # Note that this differs from the formulation in the unidiffusers paper! + do_classifier_free_guidance = guidance_scale > 1.0 + + # check if scheduler is in sigmas space + # scheduler_is_in_sigma_space = hasattr(self.scheduler, "sigmas") + + # 3. Encode input prompt, if available; otherwise prepare text latents + if latents is not None: + # Overwrite individual latents + vae_latents, clip_latents, prompt_latents = self._split_joint(latents, height, width) + + if mode in ["text2img"]: + # 3.1. Encode input prompt, if available + assert prompt is not None or prompt_embeds is not None + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=multiplier, + do_classifier_free_guidance=do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + # if do_classifier_free_guidance: + # prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + else: + # 3.2. Prepare text latent variables, if input not available + prompt_embeds = self.prepare_text_latents( + batch_size=batch_size, + num_images_per_prompt=multiplier, + seq_len=self.text_encoder_seq_len, + hidden_size=self.text_encoder_hidden_size, + dtype=self.text_encoder.dtype, # Should work with both full precision and mixed precision + device=device, + generator=generator, + latents=prompt_latents, + ) + + if reduce_text_emb_dim: + prompt_embeds = self.text_decoder.encode(prompt_embeds) + + # 4. Encode image, if available; otherwise prepare image latents + if mode in ["img2text"]: + # 4.1. Encode images, if available + assert image is not None, "`img2text` requires a conditioning image" + # Encode image using VAE + image_vae = self.image_processor.preprocess(image) + height, width = image_vae.shape[-2:] + image_vae_latents = self.encode_image_vae_latents( + image=image_vae, + batch_size=batch_size, + num_prompts_per_image=multiplier, + dtype=prompt_embeds.dtype, + device=device, + do_classifier_free_guidance=False, # Copied from InstructPix2Pix, don't use their version of CFG + generator=generator, + ) + + # Encode image using CLIP + image_clip_latents = self.encode_image_clip_latents( + image=image, + batch_size=batch_size, + num_prompts_per_image=multiplier, + dtype=prompt_embeds.dtype, + device=device, + generator=generator, + ) + # (batch_size, clip_hidden_size) => (batch_size, 1, clip_hidden_size) + image_clip_latents = image_clip_latents.unsqueeze(1) + else: + # 4.2. Prepare image latent variables, if input not available + # Prepare image VAE latents in latent space + image_vae_latents = self.prepare_image_vae_latents( + batch_size=batch_size, + num_prompts_per_image=multiplier, + num_channels_latents=self.num_channels_latents, + height=height, + width=width, + dtype=prompt_embeds.dtype, + device=device, + generator=generator, + latents=vae_latents, + ) + + # Prepare image CLIP latents + image_clip_latents = self.prepare_image_clip_latents( + batch_size=batch_size, + num_prompts_per_image=multiplier, + clip_img_dim=self.image_encoder_projection_dim, + dtype=prompt_embeds.dtype, + device=device, + generator=generator, + latents=clip_latents, + ) + + # 5. Set timesteps + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + # max_timestep = timesteps[0] + max_timestep = self.scheduler.config.num_train_timesteps + + # 6. Prepare latent variables + if mode == "joint": + latents = self._combine_joint(image_vae_latents, image_clip_latents, prompt_embeds) + elif mode in ["text2img", "img"]: + latents = self._combine(image_vae_latents, image_clip_latents) + elif mode in ["img2text", "text"]: + latents = prompt_embeds + + # 7. Prepare extra step kwargs. TODO: Logic should ideally just be moved out of the pipeline + extra_step_kwargs = self.prepare_extra_step_kwargs(generator, eta) + + logger.debug(f"Scheduler extra step kwargs: {extra_step_kwargs}") + + # 8. Denoising loop + num_warmup_steps = len(timesteps) - num_inference_steps * self.scheduler.order + with self.progress_bar(total=num_inference_steps) as progress_bar: + for i, t in enumerate(timesteps): + # predict the noise residual + # Also applies classifier-free guidance as described in the UniDiffuser paper + noise_pred = self._get_noise_pred( + mode, + latents, + t, + prompt_embeds, + image_vae_latents, + image_clip_latents, + max_timestep, + data_type, + guidance_scale, + generator, + device, + height, + width, + ) + + # compute the previous noisy sample x_t -> x_t-1 + latents = self.scheduler.step(noise_pred, t, latents, **extra_step_kwargs).prev_sample + + # call the callback, if provided + if i == len(timesteps) - 1 or ((i + 1) > num_warmup_steps and (i + 1) % self.scheduler.order == 0): + progress_bar.update() + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 9. Post-processing + image = None + text = None + if mode == "joint": + image_vae_latents, image_clip_latents, text_latents = self._split_joint(latents, height, width) + + if not output_type == "latent": + # Map latent VAE image back to pixel space + image = self.vae.decode(image_vae_latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = image_vae_latents + + text = self.decode_text_latents(text_latents, device) + elif mode in ["text2img", "img"]: + image_vae_latents, image_clip_latents = self._split(latents, height, width) + + if not output_type == "latent": + # Map latent VAE image back to pixel space + image = self.vae.decode(image_vae_latents / self.vae.config.scaling_factor, return_dict=False)[0] + else: + image = image_vae_latents + elif mode in ["img2text", "text"]: + text_latents = latents + text = self.decode_text_latents(text_latents, device) + + self.maybe_free_model_hooks() + + # 10. Postprocess the image, if necessary + if image is not None: + do_denormalize = [True] * image.shape[0] + image = self.image_processor.postprocess(image, output_type=output_type, do_denormalize=do_denormalize) + + # Offload last model to CPU + if hasattr(self, "final_offload_hook") and self.final_offload_hook is not None: + self.final_offload_hook.offload() + + if not return_dict: + return (image, text) + + return ImageTextPipelineOutput(images=image, text=text) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/__init__.py b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/__init__.py new file mode 100755 index 0000000..ddb852d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/__init__.py @@ -0,0 +1,56 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_torch_and_transformers_objects + + _dummy_objects.update(get_objects_from_module(dummy_torch_and_transformers_objects)) +else: + _import_structure["modeling_paella_vq_model"] = ["PaellaVQModel"] + _import_structure["modeling_wuerstchen_diffnext"] = ["WuerstchenDiffNeXt"] + _import_structure["modeling_wuerstchen_prior"] = ["WuerstchenPrior"] + _import_structure["pipeline_wuerstchen"] = ["WuerstchenDecoderPipeline"] + _import_structure["pipeline_wuerstchen_combined"] = ["WuerstchenCombinedPipeline"] + _import_structure["pipeline_wuerstchen_prior"] = ["DEFAULT_STAGE_C_TIMESTEPS", "WuerstchenPriorPipeline"] + + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ...utils.dummy_torch_and_transformers_objects import * # noqa F403 + else: + from .modeling_paella_vq_model import PaellaVQModel + from .modeling_wuerstchen_diffnext import WuerstchenDiffNeXt + from .modeling_wuerstchen_prior import WuerstchenPrior + from .pipeline_wuerstchen import WuerstchenDecoderPipeline + from .pipeline_wuerstchen_combined import WuerstchenCombinedPipeline + from .pipeline_wuerstchen_prior import DEFAULT_STAGE_C_TIMESTEPS, WuerstchenPriorPipeline +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py new file mode 100755 index 0000000..3b21dfb --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_paella_vq_model.py @@ -0,0 +1,172 @@ +# Copyright (c) 2022 Dominic Rampas MIT License +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Union + +import torch +import torch.nn as nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...models.autoencoders.vae import DecoderOutput, VectorQuantizer +from ...models.modeling_utils import ModelMixin +from ...models.vq_model import VQEncoderOutput +from ...utils.accelerate_utils import apply_forward_hook + + +class MixingResidualBlock(nn.Module): + """ + Residual block with mixing used by Paella's VQ-VAE. + """ + + def __init__(self, inp_channels, embed_dim): + super().__init__() + # depthwise + self.norm1 = nn.LayerNorm(inp_channels, elementwise_affine=False, eps=1e-6) + self.depthwise = nn.Sequential( + nn.ReplicationPad2d(1), nn.Conv2d(inp_channels, inp_channels, kernel_size=3, groups=inp_channels) + ) + + # channelwise + self.norm2 = nn.LayerNorm(inp_channels, elementwise_affine=False, eps=1e-6) + self.channelwise = nn.Sequential( + nn.Linear(inp_channels, embed_dim), nn.GELU(), nn.Linear(embed_dim, inp_channels) + ) + + self.gammas = nn.Parameter(torch.zeros(6), requires_grad=True) + + def forward(self, x): + mods = self.gammas + x_temp = self.norm1(x.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) * (1 + mods[0]) + mods[1] + x = x + self.depthwise(x_temp) * mods[2] + x_temp = self.norm2(x.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) * (1 + mods[3]) + mods[4] + x = x + self.channelwise(x_temp.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) * mods[5] + return x + + +class PaellaVQModel(ModelMixin, ConfigMixin): + r"""VQ-VAE model from Paella model. + + This model inherits from [`ModelMixin`]. Check the superclass documentation for the generic methods the library + implements for all the model (such as downloading or saving, etc.) + + Parameters: + in_channels (int, *optional*, defaults to 3): Number of channels in the input image. + out_channels (int, *optional*, defaults to 3): Number of channels in the output. + up_down_scale_factor (int, *optional*, defaults to 2): Up and Downscale factor of the input image. + levels (int, *optional*, defaults to 2): Number of levels in the model. + bottleneck_blocks (int, *optional*, defaults to 12): Number of bottleneck blocks in the model. + embed_dim (int, *optional*, defaults to 384): Number of hidden channels in the model. + latent_channels (int, *optional*, defaults to 4): Number of latent channels in the VQ-VAE model. + num_vq_embeddings (int, *optional*, defaults to 8192): Number of codebook vectors in the VQ-VAE. + scale_factor (float, *optional*, defaults to 0.3764): Scaling factor of the latent space. + """ + + @register_to_config + def __init__( + self, + in_channels: int = 3, + out_channels: int = 3, + up_down_scale_factor: int = 2, + levels: int = 2, + bottleneck_blocks: int = 12, + embed_dim: int = 384, + latent_channels: int = 4, + num_vq_embeddings: int = 8192, + scale_factor: float = 0.3764, + ): + super().__init__() + + c_levels = [embed_dim // (2**i) for i in reversed(range(levels))] + # Encoder blocks + self.in_block = nn.Sequential( + nn.PixelUnshuffle(up_down_scale_factor), + nn.Conv2d(in_channels * up_down_scale_factor**2, c_levels[0], kernel_size=1), + ) + down_blocks = [] + for i in range(levels): + if i > 0: + down_blocks.append(nn.Conv2d(c_levels[i - 1], c_levels[i], kernel_size=4, stride=2, padding=1)) + block = MixingResidualBlock(c_levels[i], c_levels[i] * 4) + down_blocks.append(block) + down_blocks.append( + nn.Sequential( + nn.Conv2d(c_levels[-1], latent_channels, kernel_size=1, bias=False), + nn.BatchNorm2d(latent_channels), # then normalize them to have mean 0 and std 1 + ) + ) + self.down_blocks = nn.Sequential(*down_blocks) + + # Vector Quantizer + self.vquantizer = VectorQuantizer(num_vq_embeddings, vq_embed_dim=latent_channels, legacy=False, beta=0.25) + + # Decoder blocks + up_blocks = [nn.Sequential(nn.Conv2d(latent_channels, c_levels[-1], kernel_size=1))] + for i in range(levels): + for j in range(bottleneck_blocks if i == 0 else 1): + block = MixingResidualBlock(c_levels[levels - 1 - i], c_levels[levels - 1 - i] * 4) + up_blocks.append(block) + if i < levels - 1: + up_blocks.append( + nn.ConvTranspose2d( + c_levels[levels - 1 - i], c_levels[levels - 2 - i], kernel_size=4, stride=2, padding=1 + ) + ) + self.up_blocks = nn.Sequential(*up_blocks) + self.out_block = nn.Sequential( + nn.Conv2d(c_levels[0], out_channels * up_down_scale_factor**2, kernel_size=1), + nn.PixelShuffle(up_down_scale_factor), + ) + + @apply_forward_hook + def encode(self, x: torch.FloatTensor, return_dict: bool = True) -> VQEncoderOutput: + h = self.in_block(x) + h = self.down_blocks(h) + + if not return_dict: + return (h,) + + return VQEncoderOutput(latents=h) + + @apply_forward_hook + def decode( + self, h: torch.FloatTensor, force_not_quantize: bool = True, return_dict: bool = True + ) -> Union[DecoderOutput, torch.FloatTensor]: + if not force_not_quantize: + quant, _, _ = self.vquantizer(h) + else: + quant = h + + x = self.up_blocks(quant) + dec = self.out_block(x) + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) + + def forward(self, sample: torch.FloatTensor, return_dict: bool = True) -> Union[DecoderOutput, torch.FloatTensor]: + r""" + Args: + sample (`torch.FloatTensor`): Input sample. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`DecoderOutput`] instead of a plain tuple. + """ + x = sample + h = self.encode(x).latents + dec = self.decode(h).sample + + if not return_dict: + return (dec,) + + return DecoderOutput(sample=dec) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py new file mode 100755 index 0000000..101acaf --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_common.py @@ -0,0 +1,81 @@ +import torch +import torch.nn as nn + +from ...models.attention_processor import Attention + + +class WuerstchenLayerNorm(nn.LayerNorm): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + def forward(self, x): + x = x.permute(0, 2, 3, 1) + x = super().forward(x) + return x.permute(0, 3, 1, 2) + + +class TimestepBlock(nn.Module): + def __init__(self, c, c_timestep): + super().__init__() + linear_cls = nn.Linear + self.mapper = linear_cls(c_timestep, c * 2) + + def forward(self, x, t): + a, b = self.mapper(t)[:, :, None, None].chunk(2, dim=1) + return x * (1 + a) + b + + +class ResBlock(nn.Module): + def __init__(self, c, c_skip=0, kernel_size=3, dropout=0.0): + super().__init__() + + conv_cls = nn.Conv2d + linear_cls = nn.Linear + + self.depthwise = conv_cls(c + c_skip, c, kernel_size=kernel_size, padding=kernel_size // 2, groups=c) + self.norm = WuerstchenLayerNorm(c, elementwise_affine=False, eps=1e-6) + self.channelwise = nn.Sequential( + linear_cls(c, c * 4), nn.GELU(), GlobalResponseNorm(c * 4), nn.Dropout(dropout), linear_cls(c * 4, c) + ) + + def forward(self, x, x_skip=None): + x_res = x + if x_skip is not None: + x = torch.cat([x, x_skip], dim=1) + x = self.norm(self.depthwise(x)).permute(0, 2, 3, 1) + x = self.channelwise(x).permute(0, 3, 1, 2) + return x + x_res + + +# from https://github.com/facebookresearch/ConvNeXt-V2/blob/3608f67cc1dae164790c5d0aead7bf2d73d9719b/models/utils.py#L105 +class GlobalResponseNorm(nn.Module): + def __init__(self, dim): + super().__init__() + self.gamma = nn.Parameter(torch.zeros(1, 1, 1, dim)) + self.beta = nn.Parameter(torch.zeros(1, 1, 1, dim)) + + def forward(self, x): + agg_norm = torch.norm(x, p=2, dim=(1, 2), keepdim=True) + stand_div_norm = agg_norm / (agg_norm.mean(dim=-1, keepdim=True) + 1e-6) + return self.gamma * (x * stand_div_norm) + self.beta + x + + +class AttnBlock(nn.Module): + def __init__(self, c, c_cond, nhead, self_attn=True, dropout=0.0): + super().__init__() + + linear_cls = nn.Linear + + self.self_attn = self_attn + self.norm = WuerstchenLayerNorm(c, elementwise_affine=False, eps=1e-6) + self.attention = Attention(query_dim=c, heads=nhead, dim_head=c // nhead, dropout=dropout, bias=True) + self.kv_mapper = nn.Sequential(nn.SiLU(), linear_cls(c_cond, c)) + + def forward(self, x, kv): + kv = self.kv_mapper(kv) + norm_x = self.norm(x) + if self.self_attn: + batch_size, channel, _, _ = x.shape + kv = torch.cat([norm_x.view(batch_size, channel, -1).transpose(1, 2), kv], dim=1) + x = x + self.attention(norm_x, encoder_hidden_states=kv) + return x diff --git a/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_diffnext.py b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_diffnext.py new file mode 100755 index 0000000..6c06cc0 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_diffnext.py @@ -0,0 +1,254 @@ +# Copyright (c) 2023 Dominic Rampas MIT License +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math + +import numpy as np +import torch +import torch.nn as nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...models.modeling_utils import ModelMixin +from .modeling_wuerstchen_common import AttnBlock, GlobalResponseNorm, TimestepBlock, WuerstchenLayerNorm + + +class WuerstchenDiffNeXt(ModelMixin, ConfigMixin): + @register_to_config + def __init__( + self, + c_in=4, + c_out=4, + c_r=64, + patch_size=2, + c_cond=1024, + c_hidden=[320, 640, 1280, 1280], + nhead=[-1, 10, 20, 20], + blocks=[4, 4, 14, 4], + level_config=["CT", "CTA", "CTA", "CTA"], + inject_effnet=[False, True, True, True], + effnet_embd=16, + clip_embd=1024, + kernel_size=3, + dropout=0.1, + ): + super().__init__() + self.c_r = c_r + self.c_cond = c_cond + if not isinstance(dropout, list): + dropout = [dropout] * len(c_hidden) + + # CONDITIONING + self.clip_mapper = nn.Linear(clip_embd, c_cond) + self.effnet_mappers = nn.ModuleList( + [ + nn.Conv2d(effnet_embd, c_cond, kernel_size=1) if inject else None + for inject in inject_effnet + list(reversed(inject_effnet)) + ] + ) + self.seq_norm = nn.LayerNorm(c_cond, elementwise_affine=False, eps=1e-6) + + self.embedding = nn.Sequential( + nn.PixelUnshuffle(patch_size), + nn.Conv2d(c_in * (patch_size**2), c_hidden[0], kernel_size=1), + WuerstchenLayerNorm(c_hidden[0], elementwise_affine=False, eps=1e-6), + ) + + def get_block(block_type, c_hidden, nhead, c_skip=0, dropout=0): + if block_type == "C": + return ResBlockStageB(c_hidden, c_skip, kernel_size=kernel_size, dropout=dropout) + elif block_type == "A": + return AttnBlock(c_hidden, c_cond, nhead, self_attn=True, dropout=dropout) + elif block_type == "T": + return TimestepBlock(c_hidden, c_r) + else: + raise ValueError(f"Block type {block_type} not supported") + + # BLOCKS + # -- down blocks + self.down_blocks = nn.ModuleList() + for i in range(len(c_hidden)): + down_block = nn.ModuleList() + if i > 0: + down_block.append( + nn.Sequential( + WuerstchenLayerNorm(c_hidden[i - 1], elementwise_affine=False, eps=1e-6), + nn.Conv2d(c_hidden[i - 1], c_hidden[i], kernel_size=2, stride=2), + ) + ) + for _ in range(blocks[i]): + for block_type in level_config[i]: + c_skip = c_cond if inject_effnet[i] else 0 + down_block.append(get_block(block_type, c_hidden[i], nhead[i], c_skip=c_skip, dropout=dropout[i])) + self.down_blocks.append(down_block) + + # -- up blocks + self.up_blocks = nn.ModuleList() + for i in reversed(range(len(c_hidden))): + up_block = nn.ModuleList() + for j in range(blocks[i]): + for k, block_type in enumerate(level_config[i]): + c_skip = c_hidden[i] if i < len(c_hidden) - 1 and j == k == 0 else 0 + c_skip += c_cond if inject_effnet[i] else 0 + up_block.append(get_block(block_type, c_hidden[i], nhead[i], c_skip=c_skip, dropout=dropout[i])) + if i > 0: + up_block.append( + nn.Sequential( + WuerstchenLayerNorm(c_hidden[i], elementwise_affine=False, eps=1e-6), + nn.ConvTranspose2d(c_hidden[i], c_hidden[i - 1], kernel_size=2, stride=2), + ) + ) + self.up_blocks.append(up_block) + + # OUTPUT + self.clf = nn.Sequential( + WuerstchenLayerNorm(c_hidden[0], elementwise_affine=False, eps=1e-6), + nn.Conv2d(c_hidden[0], 2 * c_out * (patch_size**2), kernel_size=1), + nn.PixelShuffle(patch_size), + ) + + # --- WEIGHT INIT --- + self.apply(self._init_weights) + + def _init_weights(self, m): + # General init + if isinstance(m, (nn.Conv2d, nn.Linear)): + nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + nn.init.constant_(m.bias, 0) + + for mapper in self.effnet_mappers: + if mapper is not None: + nn.init.normal_(mapper.weight, std=0.02) # conditionings + nn.init.normal_(self.clip_mapper.weight, std=0.02) # conditionings + nn.init.xavier_uniform_(self.embedding[1].weight, 0.02) # inputs + nn.init.constant_(self.clf[1].weight, 0) # outputs + + # blocks + for level_block in self.down_blocks + self.up_blocks: + for block in level_block: + if isinstance(block, ResBlockStageB): + block.channelwise[-1].weight.data *= np.sqrt(1 / sum(self.config.blocks)) + elif isinstance(block, TimestepBlock): + nn.init.constant_(block.mapper.weight, 0) + + def gen_r_embedding(self, r, max_positions=10000): + r = r * max_positions + half_dim = self.c_r // 2 + emb = math.log(max_positions) / (half_dim - 1) + emb = torch.arange(half_dim, device=r.device).float().mul(-emb).exp() + emb = r[:, None] * emb[None, :] + emb = torch.cat([emb.sin(), emb.cos()], dim=1) + if self.c_r % 2 == 1: # zero pad + emb = nn.functional.pad(emb, (0, 1), mode="constant") + return emb.to(dtype=r.dtype) + + def gen_c_embeddings(self, clip): + clip = self.clip_mapper(clip) + clip = self.seq_norm(clip) + return clip + + def _down_encode(self, x, r_embed, effnet, clip=None): + level_outputs = [] + for i, down_block in enumerate(self.down_blocks): + effnet_c = None + for block in down_block: + if isinstance(block, ResBlockStageB): + if effnet_c is None and self.effnet_mappers[i] is not None: + dtype = effnet.dtype + effnet_c = self.effnet_mappers[i]( + nn.functional.interpolate( + effnet.float(), size=x.shape[-2:], mode="bicubic", antialias=True, align_corners=True + ).to(dtype) + ) + skip = effnet_c if self.effnet_mappers[i] is not None else None + x = block(x, skip) + elif isinstance(block, AttnBlock): + x = block(x, clip) + elif isinstance(block, TimestepBlock): + x = block(x, r_embed) + else: + x = block(x) + level_outputs.insert(0, x) + return level_outputs + + def _up_decode(self, level_outputs, r_embed, effnet, clip=None): + x = level_outputs[0] + for i, up_block in enumerate(self.up_blocks): + effnet_c = None + for j, block in enumerate(up_block): + if isinstance(block, ResBlockStageB): + if effnet_c is None and self.effnet_mappers[len(self.down_blocks) + i] is not None: + dtype = effnet.dtype + effnet_c = self.effnet_mappers[len(self.down_blocks) + i]( + nn.functional.interpolate( + effnet.float(), size=x.shape[-2:], mode="bicubic", antialias=True, align_corners=True + ).to(dtype) + ) + skip = level_outputs[i] if j == 0 and i > 0 else None + if effnet_c is not None: + if skip is not None: + skip = torch.cat([skip, effnet_c], dim=1) + else: + skip = effnet_c + x = block(x, skip) + elif isinstance(block, AttnBlock): + x = block(x, clip) + elif isinstance(block, TimestepBlock): + x = block(x, r_embed) + else: + x = block(x) + return x + + def forward(self, x, r, effnet, clip=None, x_cat=None, eps=1e-3, return_noise=True): + if x_cat is not None: + x = torch.cat([x, x_cat], dim=1) + # Process the conditioning embeddings + r_embed = self.gen_r_embedding(r) + if clip is not None: + clip = self.gen_c_embeddings(clip) + + # Model Blocks + x_in = x + x = self.embedding(x) + level_outputs = self._down_encode(x, r_embed, effnet, clip) + x = self._up_decode(level_outputs, r_embed, effnet, clip) + a, b = self.clf(x).chunk(2, dim=1) + b = b.sigmoid() * (1 - eps * 2) + eps + if return_noise: + return (x_in - a) / b + else: + return a, b + + +class ResBlockStageB(nn.Module): + def __init__(self, c, c_skip=0, kernel_size=3, dropout=0.0): + super().__init__() + self.depthwise = nn.Conv2d(c, c, kernel_size=kernel_size, padding=kernel_size // 2, groups=c) + self.norm = WuerstchenLayerNorm(c, elementwise_affine=False, eps=1e-6) + self.channelwise = nn.Sequential( + nn.Linear(c + c_skip, c * 4), + nn.GELU(), + GlobalResponseNorm(c * 4), + nn.Dropout(dropout), + nn.Linear(c * 4, c), + ) + + def forward(self, x, x_skip=None): + x_res = x + x = self.norm(self.depthwise(x)) + if x_skip is not None: + x = torch.cat([x, x_skip], dim=1) + x = self.channelwise(x.permute(0, 2, 3, 1)).permute(0, 3, 1, 2) + return x + x_res diff --git a/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py new file mode 100755 index 0000000..8cc294e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/modeling_wuerstchen_prior.py @@ -0,0 +1,200 @@ +# Copyright (c) 2023 Dominic Rampas MIT License +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import Dict, Union + +import torch +import torch.nn as nn + +from ...configuration_utils import ConfigMixin, register_to_config +from ...loaders import PeftAdapterMixin, UNet2DConditionLoadersMixin +from ...models.attention_processor import ( + ADDED_KV_ATTENTION_PROCESSORS, + CROSS_ATTENTION_PROCESSORS, + AttentionProcessor, + AttnAddedKVProcessor, + AttnProcessor, +) +from ...models.modeling_utils import ModelMixin +from ...utils import is_torch_version +from .modeling_wuerstchen_common import AttnBlock, ResBlock, TimestepBlock, WuerstchenLayerNorm + + +class WuerstchenPrior(ModelMixin, ConfigMixin, UNet2DConditionLoadersMixin, PeftAdapterMixin): + unet_name = "prior" + _supports_gradient_checkpointing = True + + @register_to_config + def __init__(self, c_in=16, c=1280, c_cond=1024, c_r=64, depth=16, nhead=16, dropout=0.1): + super().__init__() + conv_cls = nn.Conv2d + linear_cls = nn.Linear + + self.c_r = c_r + self.projection = conv_cls(c_in, c, kernel_size=1) + self.cond_mapper = nn.Sequential( + linear_cls(c_cond, c), + nn.LeakyReLU(0.2), + linear_cls(c, c), + ) + + self.blocks = nn.ModuleList() + for _ in range(depth): + self.blocks.append(ResBlock(c, dropout=dropout)) + self.blocks.append(TimestepBlock(c, c_r)) + self.blocks.append(AttnBlock(c, c, nhead, self_attn=True, dropout=dropout)) + self.out = nn.Sequential( + WuerstchenLayerNorm(c, elementwise_affine=False, eps=1e-6), + conv_cls(c, c_in * 2, kernel_size=1), + ) + + self.gradient_checkpointing = False + self.set_default_attn_processor() + + @property + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.attn_processors + def attn_processors(self) -> Dict[str, AttentionProcessor]: + r""" + Returns: + `dict` of attention processors: A dictionary containing all attention processors used in the model with + indexed by its weight name. + """ + # set recursively + processors = {} + + def fn_recursive_add_processors(name: str, module: torch.nn.Module, processors: Dict[str, AttentionProcessor]): + if hasattr(module, "get_processor"): + processors[f"{name}.processor"] = module.get_processor(return_deprecated_lora=True) + + for sub_name, child in module.named_children(): + fn_recursive_add_processors(f"{name}.{sub_name}", child, processors) + + return processors + + for name, module in self.named_children(): + fn_recursive_add_processors(name, module, processors) + + return processors + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_attn_processor + def set_attn_processor(self, processor: Union[AttentionProcessor, Dict[str, AttentionProcessor]]): + r""" + Sets the attention processor to use to compute attention. + + Parameters: + processor (`dict` of `AttentionProcessor` or only `AttentionProcessor`): + The instantiated processor class or a dictionary of processor classes that will be set as the processor + for **all** `Attention` layers. + + If `processor` is a dict, the key needs to define the path to the corresponding cross attention + processor. This is strongly recommended when setting trainable attention processors. + + """ + count = len(self.attn_processors.keys()) + + if isinstance(processor, dict) and len(processor) != count: + raise ValueError( + f"A dict of processors was passed, but the number of processors {len(processor)} does not match the" + f" number of attention layers: {count}. Please make sure to pass {count} processor classes." + ) + + def fn_recursive_attn_processor(name: str, module: torch.nn.Module, processor): + if hasattr(module, "set_processor"): + if not isinstance(processor, dict): + module.set_processor(processor) + else: + module.set_processor(processor.pop(f"{name}.processor")) + + for sub_name, child in module.named_children(): + fn_recursive_attn_processor(f"{name}.{sub_name}", child, processor) + + for name, module in self.named_children(): + fn_recursive_attn_processor(name, module, processor) + + # Copied from diffusers.models.unets.unet_2d_condition.UNet2DConditionModel.set_default_attn_processor + def set_default_attn_processor(self): + """ + Disables custom attention processors and sets the default attention implementation. + """ + if all(proc.__class__ in ADDED_KV_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnAddedKVProcessor() + elif all(proc.__class__ in CROSS_ATTENTION_PROCESSORS for proc in self.attn_processors.values()): + processor = AttnProcessor() + else: + raise ValueError( + f"Cannot call `set_default_attn_processor` when attention processors are of type {next(iter(self.attn_processors.values()))}" + ) + + self.set_attn_processor(processor) + + def _set_gradient_checkpointing(self, module, value=False): + self.gradient_checkpointing = value + + def gen_r_embedding(self, r, max_positions=10000): + r = r * max_positions + half_dim = self.c_r // 2 + emb = math.log(max_positions) / (half_dim - 1) + emb = torch.arange(half_dim, device=r.device).float().mul(-emb).exp() + emb = r[:, None] * emb[None, :] + emb = torch.cat([emb.sin(), emb.cos()], dim=1) + if self.c_r % 2 == 1: # zero pad + emb = nn.functional.pad(emb, (0, 1), mode="constant") + return emb.to(dtype=r.dtype) + + def forward(self, x, r, c): + x_in = x + x = self.projection(x) + c_embed = self.cond_mapper(c) + r_embed = self.gen_r_embedding(r) + + if self.training and self.gradient_checkpointing: + + def create_custom_forward(module): + def custom_forward(*inputs): + return module(*inputs) + + return custom_forward + + if is_torch_version(">=", "1.11.0"): + for block in self.blocks: + if isinstance(block, AttnBlock): + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), x, c_embed, use_reentrant=False + ) + elif isinstance(block, TimestepBlock): + x = torch.utils.checkpoint.checkpoint( + create_custom_forward(block), x, r_embed, use_reentrant=False + ) + else: + x = torch.utils.checkpoint.checkpoint(create_custom_forward(block), x, use_reentrant=False) + else: + for block in self.blocks: + if isinstance(block, AttnBlock): + x = torch.utils.checkpoint.checkpoint(create_custom_forward(block), x, c_embed) + elif isinstance(block, TimestepBlock): + x = torch.utils.checkpoint.checkpoint(create_custom_forward(block), x, r_embed) + else: + x = torch.utils.checkpoint.checkpoint(create_custom_forward(block), x) + else: + for block in self.blocks: + if isinstance(block, AttnBlock): + x = block(x, c_embed) + elif isinstance(block, TimestepBlock): + x = block(x, r_embed) + else: + x = block(x) + a, b = self.out(x).chunk(2, dim=1) + return (x_in - a) / ((1 - b).abs() + 1e-5) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py new file mode 100755 index 0000000..e4277d5 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen.py @@ -0,0 +1,438 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import Callable, Dict, List, Optional, Union + +import numpy as np +import torch +from transformers import CLIPTextModel, CLIPTokenizer + +from ...schedulers import DDPMWuerstchenScheduler +from ...utils import deprecate, logging, replace_example_docstring +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline, ImagePipelineOutput +from .modeling_paella_vq_model import PaellaVQModel +from .modeling_wuerstchen_diffnext import WuerstchenDiffNeXt + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import WuerstchenPriorPipeline, WuerstchenDecoderPipeline + + >>> prior_pipe = WuerstchenPriorPipeline.from_pretrained( + ... "warp-ai/wuerstchen-prior", torch_dtype=torch.float16 + ... ).to("cuda") + >>> gen_pipe = WuerstchenDecoderPipeline.from_pretrain("warp-ai/wuerstchen", torch_dtype=torch.float16).to( + ... "cuda" + ... ) + + >>> prompt = "an image of a shiba inu, donning a spacesuit and helmet" + >>> prior_output = pipe(prompt) + >>> images = gen_pipe(prior_output.image_embeddings, prompt=prompt) + ``` +""" + + +class WuerstchenDecoderPipeline(DiffusionPipeline): + """ + Pipeline for generating images from the Wuerstchen model. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + tokenizer (`CLIPTokenizer`): + The CLIP tokenizer. + text_encoder (`CLIPTextModel`): + The CLIP text encoder. + decoder ([`WuerstchenDiffNeXt`]): + The WuerstchenDiffNeXt unet decoder. + vqgan ([`PaellaVQModel`]): + The VQGAN model. + scheduler ([`DDPMWuerstchenScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + latent_dim_scale (float, `optional`, defaults to 10.67): + Multiplier to determine the VQ latent space size from the image embeddings. If the image embeddings are + height=24 and width=24, the VQ latent shape needs to be height=int(24*10.67)=256 and + width=int(24*10.67)=256 in order to match the training conditions. + """ + + model_cpu_offload_seq = "text_encoder->decoder->vqgan" + _callback_tensor_inputs = [ + "latents", + "text_encoder_hidden_states", + "negative_prompt_embeds", + "image_embeddings", + ] + + def __init__( + self, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModel, + decoder: WuerstchenDiffNeXt, + scheduler: DDPMWuerstchenScheduler, + vqgan: PaellaVQModel, + latent_dim_scale: float = 10.67, + ) -> None: + super().__init__() + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + decoder=decoder, + scheduler=scheduler, + vqgan=vqgan, + ) + self.register_to_config(latent_dim_scale=latent_dim_scale) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def encode_prompt( + self, + prompt, + device, + num_images_per_prompt, + do_classifier_free_guidance, + negative_prompt=None, + ): + batch_size = len(prompt) if isinstance(prompt, list) else 1 + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + attention_mask = text_inputs.attention_mask + + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal(text_input_ids, untruncated_ids): + removed_text = self.tokenizer.batch_decode(untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1]) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + text_input_ids = text_input_ids[:, : self.tokenizer.model_max_length] + attention_mask = attention_mask[:, : self.tokenizer.model_max_length] + + text_encoder_output = self.text_encoder(text_input_ids.to(device), attention_mask=attention_mask.to(device)) + text_encoder_hidden_states = text_encoder_output.last_hidden_state + text_encoder_hidden_states = text_encoder_hidden_states.repeat_interleave(num_images_per_prompt, dim=0) + + uncond_text_encoder_hidden_states = None + if do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + negative_prompt_embeds_text_encoder_output = self.text_encoder( + uncond_input.input_ids.to(device), attention_mask=uncond_input.attention_mask.to(device) + ) + + uncond_text_encoder_hidden_states = negative_prompt_embeds_text_encoder_output.last_hidden_state + + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = uncond_text_encoder_hidden_states.shape[1] + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.repeat(1, num_images_per_prompt, 1) + uncond_text_encoder_hidden_states = uncond_text_encoder_hidden_states.view( + batch_size * num_images_per_prompt, seq_len, -1 + ) + # done duplicates + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + return text_encoder_hidden_states, uncond_text_encoder_hidden_states + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + image_embeddings: Union[torch.FloatTensor, List[torch.FloatTensor]], + prompt: Union[str, List[str]] = None, + num_inference_steps: int = 12, + timesteps: Optional[List[float]] = None, + guidance_scale: float = 0.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + image_embedding (`torch.FloatTensor` or `List[torch.FloatTensor]`): + Image Embeddings either extracted from an image or generated by a Prior Model. + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + num_inference_steps (`int`, *optional*, defaults to 12): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps are used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 0.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `decoder_guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting + `decoder_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely + linked to the text `prompt`, usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `decoder_guidance_scale` is less than `1`). + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` [`~pipelines.ImagePipelineOutput`] if `return_dict` is True, + otherwise a `tuple`. When returning a tuple, the first element is a list with the generated image + embeddings. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + # 0. Define commonly used variables + device = self._execution_device + dtype = self.decoder.dtype + self._guidance_scale = guidance_scale + + # 1. Check inputs. Raise error if not correct + if not isinstance(prompt, list): + if isinstance(prompt, str): + prompt = [prompt] + else: + raise TypeError(f"'prompt' must be of type 'list' or 'str', but got {type(prompt)}.") + + if self.do_classifier_free_guidance: + if negative_prompt is not None and not isinstance(negative_prompt, list): + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + else: + raise TypeError( + f"'negative_prompt' must be of type 'list' or 'str', but got {type(negative_prompt)}." + ) + + if isinstance(image_embeddings, list): + image_embeddings = torch.cat(image_embeddings, dim=0) + if isinstance(image_embeddings, np.ndarray): + image_embeddings = torch.Tensor(image_embeddings, device=device).to(dtype=dtype) + if not isinstance(image_embeddings, torch.Tensor): + raise TypeError( + f"'image_embeddings' must be of type 'torch.Tensor' or 'np.array', but got {type(image_embeddings)}." + ) + + if not isinstance(num_inference_steps, int): + raise TypeError( + f"'num_inference_steps' must be of type 'int', but got {type(num_inference_steps)}\ + In Case you want to provide explicit timesteps, please use the 'timesteps' argument." + ) + + # 2. Encode caption + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt, + device, + image_embeddings.size(0) * num_images_per_prompt, + self.do_classifier_free_guidance, + negative_prompt, + ) + text_encoder_hidden_states = ( + torch.cat([prompt_embeds, negative_prompt_embeds]) if negative_prompt_embeds is not None else prompt_embeds + ) + effnet = ( + torch.cat([image_embeddings, torch.zeros_like(image_embeddings)]) + if self.do_classifier_free_guidance + else image_embeddings + ) + + # 3. Determine latent shape of latents + latent_height = int(image_embeddings.size(2) * self.config.latent_dim_scale) + latent_width = int(image_embeddings.size(3) * self.config.latent_dim_scale) + latent_features_shape = (image_embeddings.size(0) * num_images_per_prompt, 4, latent_height, latent_width) + + # 4. Prepare and set timesteps + if timesteps is not None: + self.scheduler.set_timesteps(timesteps=timesteps, device=device) + timesteps = self.scheduler.timesteps + num_inference_steps = len(timesteps) + else: + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latents + latents = self.prepare_latents(latent_features_shape, dtype, device, generator, latents, self.scheduler) + + # 6. Run denoising loop + self._num_timesteps = len(timesteps[:-1]) + for i, t in enumerate(self.progress_bar(timesteps[:-1])): + ratio = t.expand(latents.size(0)).to(dtype) + # 7. Denoise latents + predicted_latents = self.decoder( + torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents, + r=torch.cat([ratio] * 2) if self.do_classifier_free_guidance else ratio, + effnet=effnet, + clip=text_encoder_hidden_states, + ) + + # 8. Check for classifier free guidance and apply it + if self.do_classifier_free_guidance: + predicted_latents_text, predicted_latents_uncond = predicted_latents.chunk(2) + predicted_latents = torch.lerp(predicted_latents_uncond, predicted_latents_text, self.guidance_scale) + + # 9. Renoise latents to next timestep + latents = self.scheduler.step( + model_output=predicted_latents, + timestep=ratio, + sample=latents, + generator=generator, + ).prev_sample + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + image_embeddings = callback_outputs.pop("image_embeddings", image_embeddings) + text_encoder_hidden_states = callback_outputs.pop( + "text_encoder_hidden_states", text_encoder_hidden_states + ) + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + if output_type not in ["pt", "np", "pil", "latent"]: + raise ValueError( + f"Only the output types `pt`, `np`, `pil` and `latent` are supported not output_type={output_type}" + ) + + if not output_type == "latent": + # 10. Scale and decode the image latents with vq-vae + latents = self.vqgan.config.scale_factor * latents + images = self.vqgan.decode(latents).sample.clamp(0, 1) + if output_type == "np": + images = images.permute(0, 2, 3, 1).cpu().float().numpy() + elif output_type == "pil": + images = images.permute(0, 2, 3, 1).cpu().float().numpy() + images = self.numpy_to_pil(images) + else: + images = latents + + # Offload all models + self.maybe_free_model_hooks() + + if not return_dict: + return images + return ImagePipelineOutput(images) diff --git a/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py new file mode 100755 index 0000000..3a43ad5 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_combined.py @@ -0,0 +1,306 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Callable, Dict, List, Optional, Union + +import torch +from transformers import CLIPTextModel, CLIPTokenizer + +from ...schedulers import DDPMWuerstchenScheduler +from ...utils import deprecate, replace_example_docstring +from ..pipeline_utils import DiffusionPipeline +from .modeling_paella_vq_model import PaellaVQModel +from .modeling_wuerstchen_diffnext import WuerstchenDiffNeXt +from .modeling_wuerstchen_prior import WuerstchenPrior +from .pipeline_wuerstchen import WuerstchenDecoderPipeline +from .pipeline_wuerstchen_prior import WuerstchenPriorPipeline + + +TEXT2IMAGE_EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> from diffusions import WuerstchenCombinedPipeline + + >>> pipe = WuerstchenCombinedPipeline.from_pretrained("warp-ai/Wuerstchen", torch_dtype=torch.float16).to( + ... "cuda" + ... ) + >>> prompt = "an image of a shiba inu, donning a spacesuit and helmet" + >>> images = pipe(prompt=prompt) + ``` +""" + + +class WuerstchenCombinedPipeline(DiffusionPipeline): + """ + Combined Pipeline for text-to-image generation using Wuerstchen + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Args: + tokenizer (`CLIPTokenizer`): + The decoder tokenizer to be used for text inputs. + text_encoder (`CLIPTextModel`): + The decoder text encoder to be used for text inputs. + decoder (`WuerstchenDiffNeXt`): + The decoder model to be used for decoder image generation pipeline. + scheduler (`DDPMWuerstchenScheduler`): + The scheduler to be used for decoder image generation pipeline. + vqgan (`PaellaVQModel`): + The VQGAN model to be used for decoder image generation pipeline. + prior_tokenizer (`CLIPTokenizer`): + The prior tokenizer to be used for text inputs. + prior_text_encoder (`CLIPTextModel`): + The prior text encoder to be used for text inputs. + prior_prior (`WuerstchenPrior`): + The prior model to be used for prior pipeline. + prior_scheduler (`DDPMWuerstchenScheduler`): + The scheduler to be used for prior pipeline. + """ + + _load_connected_pipes = True + + def __init__( + self, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModel, + decoder: WuerstchenDiffNeXt, + scheduler: DDPMWuerstchenScheduler, + vqgan: PaellaVQModel, + prior_tokenizer: CLIPTokenizer, + prior_text_encoder: CLIPTextModel, + prior_prior: WuerstchenPrior, + prior_scheduler: DDPMWuerstchenScheduler, + ): + super().__init__() + + self.register_modules( + text_encoder=text_encoder, + tokenizer=tokenizer, + decoder=decoder, + scheduler=scheduler, + vqgan=vqgan, + prior_prior=prior_prior, + prior_text_encoder=prior_text_encoder, + prior_tokenizer=prior_tokenizer, + prior_scheduler=prior_scheduler, + ) + self.prior_pipe = WuerstchenPriorPipeline( + prior=prior_prior, + text_encoder=prior_text_encoder, + tokenizer=prior_tokenizer, + scheduler=prior_scheduler, + ) + self.decoder_pipe = WuerstchenDecoderPipeline( + text_encoder=text_encoder, + tokenizer=tokenizer, + decoder=decoder, + scheduler=scheduler, + vqgan=vqgan, + ) + + def enable_xformers_memory_efficient_attention(self, attention_op: Optional[Callable] = None): + self.decoder_pipe.enable_xformers_memory_efficient_attention(attention_op) + + def enable_model_cpu_offload(self, gpu_id=0): + r""" + Offloads all models to CPU using accelerate, reducing memory usage with a low impact on performance. Compared + to `enable_sequential_cpu_offload`, this method moves one whole model at a time to the GPU when its `forward` + method is called, and the model remains in GPU until the next model runs. Memory savings are lower than with + `enable_sequential_cpu_offload`, but performance is much better due to the iterative execution of the `unet`. + """ + self.prior_pipe.enable_model_cpu_offload(gpu_id=gpu_id) + self.decoder_pipe.enable_model_cpu_offload(gpu_id=gpu_id) + + def enable_sequential_cpu_offload(self, gpu_id=0): + r""" + Offloads all models (`unet`, `text_encoder`, `vae`, and `safety checker` state dicts) to CPU using 🤗 + Accelerate, significantly reducing memory usage. Models are moved to a `torch.device('meta')` and loaded on a + GPU only when their specific submodule's `forward` method is called. Offloading happens on a submodule basis. + Memory savings are higher than using `enable_model_cpu_offload`, but performance is lower. + """ + self.prior_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + self.decoder_pipe.enable_sequential_cpu_offload(gpu_id=gpu_id) + + def progress_bar(self, iterable=None, total=None): + self.prior_pipe.progress_bar(iterable=iterable, total=total) + self.decoder_pipe.progress_bar(iterable=iterable, total=total) + + def set_progress_bar_config(self, **kwargs): + self.prior_pipe.set_progress_bar_config(**kwargs) + self.decoder_pipe.set_progress_bar_config(**kwargs) + + @torch.no_grad() + @replace_example_docstring(TEXT2IMAGE_EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + height: int = 512, + width: int = 512, + prior_num_inference_steps: int = 60, + prior_timesteps: Optional[List[float]] = None, + prior_guidance_scale: float = 4.0, + num_inference_steps: int = 12, + decoder_timesteps: Optional[List[float]] = None, + decoder_guidance_scale: float = 0.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + num_images_per_prompt: int = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pil", + return_dict: bool = True, + prior_callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + prior_callback_on_step_end_tensor_inputs: List[str] = ["latents"], + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation for the prior and decoder. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings for the prior. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings for the prior. Can be used to easily tweak text inputs, *e.g.* + prompt weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` + input argument. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + height (`int`, *optional*, defaults to 512): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 512): + The width in pixels of the generated image. + prior_guidance_scale (`float`, *optional*, defaults to 4.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `prior_guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting + `prior_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely linked + to the text `prompt`, usually at the expense of lower image quality. + prior_num_inference_steps (`Union[int, Dict[float, int]]`, *optional*, defaults to 60): + The number of prior denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. For more specific timestep spacing, you can pass customized + `prior_timesteps` + num_inference_steps (`int`, *optional*, defaults to 12): + The number of decoder denoising steps. More denoising steps usually lead to a higher quality image at + the expense of slower inference. For more specific timestep spacing, you can pass customized + `timesteps` + prior_timesteps (`List[float]`, *optional*): + Custom timesteps to use for the denoising process for the prior. If not defined, equal spaced + `prior_num_inference_steps` timesteps are used. Must be in descending order. + decoder_timesteps (`List[float]`, *optional*): + Custom timesteps to use for the denoising process for the decoder. If not defined, equal spaced + `num_inference_steps` timesteps are used. Must be in descending order. + decoder_guidance_scale (`float`, *optional*, defaults to 0.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting `guidance_scale > + 1`. Higher guidance scale encourages to generate images that are closely linked to the text `prompt`, + usually at the expense of lower image quality. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + prior_callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `prior_callback_on_step_end(self: DiffusionPipeline, step: int, timestep: + int, callback_kwargs: Dict)`. + prior_callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `prior_callback_on_step_end` function. The tensors specified in the + list will be passed as `callback_kwargs` argument. You will only be able to include variables listed in + the `._callback_tensor_inputs` attribute of your pipeline class. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple` [`~pipelines.ImagePipelineOutput`] if `return_dict` is True, + otherwise a `tuple`. When returning a tuple, the first element is a list with the generated images. + """ + prior_kwargs = {} + if kwargs.get("prior_callback", None) is not None: + prior_kwargs["callback"] = kwargs.pop("prior_callback") + deprecate( + "prior_callback", + "1.0.0", + "Passing `prior_callback` as an input argument to `__call__` is deprecated, consider use `prior_callback_on_step_end`", + ) + if kwargs.get("prior_callback_steps", None) is not None: + deprecate( + "prior_callback_steps", + "1.0.0", + "Passing `prior_callback_steps` as an input argument to `__call__` is deprecated, consider use `prior_callback_on_step_end`", + ) + prior_kwargs["callback_steps"] = kwargs.pop("prior_callback_steps") + + prior_outputs = self.prior_pipe( + prompt=prompt if prompt_embeds is None else None, + height=height, + width=width, + num_inference_steps=prior_num_inference_steps, + timesteps=prior_timesteps, + guidance_scale=prior_guidance_scale, + negative_prompt=negative_prompt if negative_prompt_embeds is None else None, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + num_images_per_prompt=num_images_per_prompt, + generator=generator, + latents=latents, + output_type="pt", + return_dict=False, + callback_on_step_end=prior_callback_on_step_end, + callback_on_step_end_tensor_inputs=prior_callback_on_step_end_tensor_inputs, + **prior_kwargs, + ) + image_embeddings = prior_outputs[0] + + outputs = self.decoder_pipe( + image_embeddings=image_embeddings, + prompt=prompt if prompt is not None else "", + num_inference_steps=num_inference_steps, + timesteps=decoder_timesteps, + guidance_scale=decoder_guidance_scale, + negative_prompt=negative_prompt, + generator=generator, + output_type=output_type, + return_dict=return_dict, + callback_on_step_end=callback_on_step_end, + callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs, + **kwargs, + ) + + return outputs diff --git a/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py new file mode 100755 index 0000000..4640f76 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/pipelines/wuerstchen/pipeline_wuerstchen_prior.py @@ -0,0 +1,516 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from math import ceil +from typing import Callable, Dict, List, Optional, Union + +import numpy as np +import torch +from transformers import CLIPTextModel, CLIPTokenizer + +from ...loaders import LoraLoaderMixin +from ...schedulers import DDPMWuerstchenScheduler +from ...utils import BaseOutput, deprecate, logging, replace_example_docstring +from ...utils.torch_utils import randn_tensor +from ..pipeline_utils import DiffusionPipeline +from .modeling_wuerstchen_prior import WuerstchenPrior + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +DEFAULT_STAGE_C_TIMESTEPS = list(np.linspace(1.0, 2 / 3, 20)) + list(np.linspace(2 / 3, 0.0, 11))[1:] + +EXAMPLE_DOC_STRING = """ + Examples: + ```py + >>> import torch + >>> from diffusers import WuerstchenPriorPipeline + + >>> prior_pipe = WuerstchenPriorPipeline.from_pretrained( + ... "warp-ai/wuerstchen-prior", torch_dtype=torch.float16 + ... ).to("cuda") + + >>> prompt = "an image of a shiba inu, donning a spacesuit and helmet" + >>> prior_output = pipe(prompt) + ``` +""" + + +@dataclass +class WuerstchenPriorPipelineOutput(BaseOutput): + """ + Output class for WuerstchenPriorPipeline. + + Args: + image_embeddings (`torch.FloatTensor` or `np.ndarray`) + Prior image embeddings for text prompt + + """ + + image_embeddings: Union[torch.FloatTensor, np.ndarray] + + +class WuerstchenPriorPipeline(DiffusionPipeline, LoraLoaderMixin): + """ + Pipeline for generating image prior for Wuerstchen. + + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + The pipeline also inherits the following loading methods: + - [`~loaders.LoraLoaderMixin.load_lora_weights`] for loading LoRA weights + - [`~loaders.LoraLoaderMixin.save_lora_weights`] for saving LoRA weights + + Args: + prior ([`Prior`]): + The canonical unCLIP prior to approximate the image embedding from the text embedding. + text_encoder ([`CLIPTextModelWithProjection`]): + Frozen text-encoder. + tokenizer (`CLIPTokenizer`): + Tokenizer of class + [CLIPTokenizer](https://huggingface.co/docs/transformers/v4.21.0/en/model_doc/clip#transformers.CLIPTokenizer). + scheduler ([`DDPMWuerstchenScheduler`]): + A scheduler to be used in combination with `prior` to generate image embedding. + latent_mean ('float', *optional*, defaults to 42.0): + Mean value for latent diffusers. + latent_std ('float', *optional*, defaults to 1.0): + Standard value for latent diffusers. + resolution_multiple ('float', *optional*, defaults to 42.67): + Default resolution for multiple images generated. + """ + + unet_name = "prior" + text_encoder_name = "text_encoder" + model_cpu_offload_seq = "text_encoder->prior" + _callback_tensor_inputs = ["latents", "text_encoder_hidden_states", "negative_prompt_embeds"] + + def __init__( + self, + tokenizer: CLIPTokenizer, + text_encoder: CLIPTextModel, + prior: WuerstchenPrior, + scheduler: DDPMWuerstchenScheduler, + latent_mean: float = 42.0, + latent_std: float = 1.0, + resolution_multiple: float = 42.67, + ) -> None: + super().__init__() + self.register_modules( + tokenizer=tokenizer, + text_encoder=text_encoder, + prior=prior, + scheduler=scheduler, + ) + self.register_to_config( + latent_mean=latent_mean, latent_std=latent_std, resolution_multiple=resolution_multiple + ) + + # Copied from diffusers.pipelines.unclip.pipeline_unclip.UnCLIPPipeline.prepare_latents + def prepare_latents(self, shape, dtype, device, generator, latents, scheduler): + if latents is None: + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + else: + if latents.shape != shape: + raise ValueError(f"Unexpected latents shape, got {latents.shape}, expected {shape}") + latents = latents.to(device) + + latents = latents * scheduler.init_noise_sigma + return latents + + def encode_prompt( + self, + device, + num_images_per_prompt, + do_classifier_free_guidance, + prompt=None, + negative_prompt=None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + ): + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + if prompt_embeds is None: + # get prompt text embeddings + text_inputs = self.tokenizer( + prompt, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_input_ids = text_inputs.input_ids + attention_mask = text_inputs.attention_mask + + untruncated_ids = self.tokenizer(prompt, padding="longest", return_tensors="pt").input_ids + + if untruncated_ids.shape[-1] >= text_input_ids.shape[-1] and not torch.equal( + text_input_ids, untruncated_ids + ): + removed_text = self.tokenizer.batch_decode( + untruncated_ids[:, self.tokenizer.model_max_length - 1 : -1] + ) + logger.warning( + "The following part of your input was truncated because CLIP can only handle sequences up to" + f" {self.tokenizer.model_max_length} tokens: {removed_text}" + ) + text_input_ids = text_input_ids[:, : self.tokenizer.model_max_length] + attention_mask = attention_mask[:, : self.tokenizer.model_max_length] + + text_encoder_output = self.text_encoder( + text_input_ids.to(device), attention_mask=attention_mask.to(device) + ) + prompt_embeds = text_encoder_output.last_hidden_state + + prompt_embeds = prompt_embeds.to(dtype=self.text_encoder.dtype, device=device) + prompt_embeds = prompt_embeds.repeat_interleave(num_images_per_prompt, dim=0) + + if negative_prompt_embeds is None and do_classifier_free_guidance: + uncond_tokens: List[str] + if negative_prompt is None: + uncond_tokens = [""] * batch_size + elif type(prompt) is not type(negative_prompt): + raise TypeError( + f"`negative_prompt` should be the same type to `prompt`, but got {type(negative_prompt)} !=" + f" {type(prompt)}." + ) + elif isinstance(negative_prompt, str): + uncond_tokens = [negative_prompt] + elif batch_size != len(negative_prompt): + raise ValueError( + f"`negative_prompt`: {negative_prompt} has batch size {len(negative_prompt)}, but `prompt`:" + f" {prompt} has batch size {batch_size}. Please make sure that passed `negative_prompt` matches" + " the batch size of `prompt`." + ) + else: + uncond_tokens = negative_prompt + + uncond_input = self.tokenizer( + uncond_tokens, + padding="max_length", + max_length=self.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + negative_prompt_embeds_text_encoder_output = self.text_encoder( + uncond_input.input_ids.to(device), attention_mask=uncond_input.attention_mask.to(device) + ) + + negative_prompt_embeds = negative_prompt_embeds_text_encoder_output.last_hidden_state + + if do_classifier_free_guidance: + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + negative_prompt_embeds = negative_prompt_embeds.to(dtype=self.text_encoder.dtype, device=device) + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + # done duplicates + + return prompt_embeds, negative_prompt_embeds + + def check_inputs( + self, + prompt, + negative_prompt, + num_inference_steps, + do_classifier_free_guidance, + prompt_embeds=None, + negative_prompt_embeds=None, + ): + if prompt is not None and prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `prompt`: {prompt} and `prompt_embeds`: {prompt_embeds}. Please make sure to" + " only forward one of the two." + ) + elif prompt is None and prompt_embeds is None: + raise ValueError( + "Provide either `prompt` or `prompt_embeds`. Cannot leave both `prompt` and `prompt_embeds` undefined." + ) + elif prompt is not None and (not isinstance(prompt, str) and not isinstance(prompt, list)): + raise ValueError(f"`prompt` has to be of type `str` or `list` but is {type(prompt)}") + + if negative_prompt is not None and negative_prompt_embeds is not None: + raise ValueError( + f"Cannot forward both `negative_prompt`: {negative_prompt} and `negative_prompt_embeds`:" + f" {negative_prompt_embeds}. Please make sure to only forward one of the two." + ) + + if prompt_embeds is not None and negative_prompt_embeds is not None: + if prompt_embeds.shape != negative_prompt_embeds.shape: + raise ValueError( + "`prompt_embeds` and `negative_prompt_embeds` must have the same shape when passed directly, but" + f" got: `prompt_embeds` {prompt_embeds.shape} != `negative_prompt_embeds`" + f" {negative_prompt_embeds.shape}." + ) + + if not isinstance(num_inference_steps, int): + raise TypeError( + f"'num_inference_steps' must be of type 'int', but got {type(num_inference_steps)}\ + In Case you want to provide explicit timesteps, please use the 'timesteps' argument." + ) + + @property + def guidance_scale(self): + return self._guidance_scale + + @property + def do_classifier_free_guidance(self): + return self._guidance_scale > 1 + + @property + def num_timesteps(self): + return self._num_timesteps + + @torch.no_grad() + @replace_example_docstring(EXAMPLE_DOC_STRING) + def __call__( + self, + prompt: Optional[Union[str, List[str]]] = None, + height: int = 1024, + width: int = 1024, + num_inference_steps: int = 60, + timesteps: List[float] = None, + guidance_scale: float = 8.0, + negative_prompt: Optional[Union[str, List[str]]] = None, + prompt_embeds: Optional[torch.FloatTensor] = None, + negative_prompt_embeds: Optional[torch.FloatTensor] = None, + num_images_per_prompt: Optional[int] = 1, + generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None, + latents: Optional[torch.FloatTensor] = None, + output_type: Optional[str] = "pt", + return_dict: bool = True, + callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None, + callback_on_step_end_tensor_inputs: List[str] = ["latents"], + **kwargs, + ): + """ + Function invoked when calling the pipeline for generation. + + Args: + prompt (`str` or `List[str]`): + The prompt or prompts to guide the image generation. + height (`int`, *optional*, defaults to 1024): + The height in pixels of the generated image. + width (`int`, *optional*, defaults to 1024): + The width in pixels of the generated image. + num_inference_steps (`int`, *optional*, defaults to 60): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + timesteps (`List[int]`, *optional*): + Custom timesteps to use for the denoising process. If not defined, equal spaced `num_inference_steps` + timesteps are used. Must be in descending order. + guidance_scale (`float`, *optional*, defaults to 8.0): + Guidance scale as defined in [Classifier-Free Diffusion Guidance](https://arxiv.org/abs/2207.12598). + `decoder_guidance_scale` is defined as `w` of equation 2. of [Imagen + Paper](https://arxiv.org/pdf/2205.11487.pdf). Guidance scale is enabled by setting + `decoder_guidance_scale > 1`. Higher guidance scale encourages to generate images that are closely + linked to the text `prompt`, usually at the expense of lower image quality. + negative_prompt (`str` or `List[str]`, *optional*): + The prompt or prompts not to guide the image generation. Ignored when not using guidance (i.e., ignored + if `decoder_guidance_scale` is less than `1`). + prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt weighting. If not + provided, text embeddings will be generated from `prompt` input argument. + negative_prompt_embeds (`torch.FloatTensor`, *optional*): + Pre-generated negative text embeddings. Can be used to easily tweak text inputs, *e.g.* prompt + weighting. If not provided, negative_prompt_embeds will be generated from `negative_prompt` input + argument. + num_images_per_prompt (`int`, *optional*, defaults to 1): + The number of images to generate per prompt. + generator (`torch.Generator` or `List[torch.Generator]`, *optional*): + One or a list of [torch generator(s)](https://pytorch.org/docs/stable/generated/torch.Generator.html) + to make generation deterministic. + latents (`torch.FloatTensor`, *optional*): + Pre-generated noisy latents, sampled from a Gaussian distribution, to be used as inputs for image + generation. Can be used to tweak the same generation with different prompts. If not provided, a latents + tensor will ge generated by sampling using the supplied random `generator`. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between: `"pil"` (`PIL.Image.Image`), `"np"` + (`np.array`) or `"pt"` (`torch.Tensor`). + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + callback_on_step_end (`Callable`, *optional*): + A function that calls at the end of each denoising steps during the inference. The function is called + with the following arguments: `callback_on_step_end(self: DiffusionPipeline, step: int, timestep: int, + callback_kwargs: Dict)`. `callback_kwargs` will include a list of all tensors as specified by + `callback_on_step_end_tensor_inputs`. + callback_on_step_end_tensor_inputs (`List`, *optional*): + The list of tensor inputs for the `callback_on_step_end` function. The tensors specified in the list + will be passed as `callback_kwargs` argument. You will only be able to include variables listed in the + `._callback_tensor_inputs` attribute of your pipeline class. + + Examples: + + Returns: + [`~pipelines.WuerstchenPriorPipelineOutput`] or `tuple` [`~pipelines.WuerstchenPriorPipelineOutput`] if + `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is a list with the + generated image embeddings. + """ + + callback = kwargs.pop("callback", None) + callback_steps = kwargs.pop("callback_steps", None) + + if callback is not None: + deprecate( + "callback", + "1.0.0", + "Passing `callback` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + if callback_steps is not None: + deprecate( + "callback_steps", + "1.0.0", + "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider use `callback_on_step_end`", + ) + + if callback_on_step_end_tensor_inputs is not None and not all( + k in self._callback_tensor_inputs for k in callback_on_step_end_tensor_inputs + ): + raise ValueError( + f"`callback_on_step_end_tensor_inputs` has to be in {self._callback_tensor_inputs}, but found {[k for k in callback_on_step_end_tensor_inputs if k not in self._callback_tensor_inputs]}" + ) + + # 0. Define commonly used variables + device = self._execution_device + self._guidance_scale = guidance_scale + if prompt is not None and isinstance(prompt, str): + batch_size = 1 + elif prompt is not None and isinstance(prompt, list): + batch_size = len(prompt) + else: + batch_size = prompt_embeds.shape[0] + + # 1. Check inputs. Raise error if not correct + if prompt is not None and not isinstance(prompt, list): + if isinstance(prompt, str): + prompt = [prompt] + else: + raise TypeError(f"'prompt' must be of type 'list' or 'str', but got {type(prompt)}.") + + if self.do_classifier_free_guidance: + if negative_prompt is not None and not isinstance(negative_prompt, list): + if isinstance(negative_prompt, str): + negative_prompt = [negative_prompt] + else: + raise TypeError( + f"'negative_prompt' must be of type 'list' or 'str', but got {type(negative_prompt)}." + ) + + self.check_inputs( + prompt, + negative_prompt, + num_inference_steps, + self.do_classifier_free_guidance, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + # 2. Encode caption + prompt_embeds, negative_prompt_embeds = self.encode_prompt( + prompt=prompt, + device=device, + num_images_per_prompt=num_images_per_prompt, + do_classifier_free_guidance=self.do_classifier_free_guidance, + negative_prompt=negative_prompt, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + ) + + # For classifier free guidance, we need to do two forward passes. + # Here we concatenate the unconditional and text embeddings into a single batch + # to avoid doing two forward passes + text_encoder_hidden_states = ( + torch.cat([prompt_embeds, negative_prompt_embeds]) if negative_prompt_embeds is not None else prompt_embeds + ) + + # 3. Determine latent shape of image embeddings + dtype = text_encoder_hidden_states.dtype + latent_height = ceil(height / self.config.resolution_multiple) + latent_width = ceil(width / self.config.resolution_multiple) + num_channels = self.prior.config.c_in + effnet_features_shape = (num_images_per_prompt * batch_size, num_channels, latent_height, latent_width) + + # 4. Prepare and set timesteps + if timesteps is not None: + self.scheduler.set_timesteps(timesteps=timesteps, device=device) + timesteps = self.scheduler.timesteps + num_inference_steps = len(timesteps) + else: + self.scheduler.set_timesteps(num_inference_steps, device=device) + timesteps = self.scheduler.timesteps + + # 5. Prepare latents + latents = self.prepare_latents(effnet_features_shape, dtype, device, generator, latents, self.scheduler) + + # 6. Run denoising loop + self._num_timesteps = len(timesteps[:-1]) + for i, t in enumerate(self.progress_bar(timesteps[:-1])): + ratio = t.expand(latents.size(0)).to(dtype) + + # 7. Denoise image embeddings + predicted_image_embedding = self.prior( + torch.cat([latents] * 2) if self.do_classifier_free_guidance else latents, + r=torch.cat([ratio] * 2) if self.do_classifier_free_guidance else ratio, + c=text_encoder_hidden_states, + ) + + # 8. Check for classifier free guidance and apply it + if self.do_classifier_free_guidance: + predicted_image_embedding_text, predicted_image_embedding_uncond = predicted_image_embedding.chunk(2) + predicted_image_embedding = torch.lerp( + predicted_image_embedding_uncond, predicted_image_embedding_text, self.guidance_scale + ) + + # 9. Renoise latents to next timestep + latents = self.scheduler.step( + model_output=predicted_image_embedding, + timestep=ratio, + sample=latents, + generator=generator, + ).prev_sample + + if callback_on_step_end is not None: + callback_kwargs = {} + for k in callback_on_step_end_tensor_inputs: + callback_kwargs[k] = locals()[k] + callback_outputs = callback_on_step_end(self, i, t, callback_kwargs) + + latents = callback_outputs.pop("latents", latents) + text_encoder_hidden_states = callback_outputs.pop( + "text_encoder_hidden_states", text_encoder_hidden_states + ) + negative_prompt_embeds = callback_outputs.pop("negative_prompt_embeds", negative_prompt_embeds) + + if callback is not None and i % callback_steps == 0: + step_idx = i // getattr(self.scheduler, "order", 1) + callback(step_idx, t, latents) + + # 10. Denormalize the latents + latents = latents * self.config.latent_mean - self.config.latent_std + + # Offload all models + self.maybe_free_model_hooks() + + if output_type == "np": + latents = latents.cpu().float().numpy() + + if not return_dict: + return (latents,) + + return WuerstchenPriorPipelineOutput(latents) diff --git a/diffusers-0.27.0/src/diffusers/py.typed b/diffusers-0.27.0/src/diffusers/py.typed new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/src/diffusers/schedulers/README.md b/diffusers-0.27.0/src/diffusers/schedulers/README.md new file mode 100755 index 0000000..31ad277 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/README.md @@ -0,0 +1,3 @@ +# Schedulers + +For more information on the schedulers, please refer to the [docs](https://huggingface.co/docs/diffusers/api/schedulers/overview). \ No newline at end of file diff --git a/diffusers-0.27.0/src/diffusers/schedulers/__init__.py b/diffusers-0.27.0/src/diffusers/schedulers/__init__.py new file mode 100755 index 0000000..720d8ea --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/__init__.py @@ -0,0 +1,211 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import TYPE_CHECKING + +from ..utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_flax_available, + is_scipy_available, + is_torch_available, + is_torchsde_available, +) + + +_dummy_modules = {} +_import_structure = {} + +try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_pt_objects # noqa F403 + + _dummy_modules.update(get_objects_from_module(dummy_pt_objects)) + +else: + _import_structure["deprecated"] = ["KarrasVeScheduler", "ScoreSdeVpScheduler"] + _import_structure["scheduling_amused"] = ["AmusedScheduler"] + _import_structure["scheduling_consistency_decoder"] = ["ConsistencyDecoderScheduler"] + _import_structure["scheduling_consistency_models"] = ["CMStochasticIterativeScheduler"] + _import_structure["scheduling_ddim"] = ["DDIMScheduler"] + _import_structure["scheduling_ddim_inverse"] = ["DDIMInverseScheduler"] + _import_structure["scheduling_ddim_parallel"] = ["DDIMParallelScheduler"] + _import_structure["scheduling_ddpm"] = ["DDPMScheduler"] + _import_structure["scheduling_ddpm_parallel"] = ["DDPMParallelScheduler"] + _import_structure["scheduling_ddpm_wuerstchen"] = ["DDPMWuerstchenScheduler"] + _import_structure["scheduling_deis_multistep"] = ["DEISMultistepScheduler"] + _import_structure["scheduling_dpmsolver_multistep"] = ["DPMSolverMultistepScheduler"] + _import_structure["scheduling_dpmsolver_multistep_inverse"] = ["DPMSolverMultistepInverseScheduler"] + _import_structure["scheduling_dpmsolver_singlestep"] = ["DPMSolverSinglestepScheduler"] + _import_structure["scheduling_edm_dpmsolver_multistep"] = ["EDMDPMSolverMultistepScheduler"] + _import_structure["scheduling_edm_euler"] = ["EDMEulerScheduler"] + _import_structure["scheduling_euler_ancestral_discrete"] = ["EulerAncestralDiscreteScheduler"] + _import_structure["scheduling_euler_discrete"] = ["EulerDiscreteScheduler"] + _import_structure["scheduling_heun_discrete"] = ["HeunDiscreteScheduler"] + _import_structure["scheduling_ipndm"] = ["IPNDMScheduler"] + _import_structure["scheduling_k_dpm_2_ancestral_discrete"] = ["KDPM2AncestralDiscreteScheduler"] + _import_structure["scheduling_k_dpm_2_discrete"] = ["KDPM2DiscreteScheduler"] + _import_structure["scheduling_lcm"] = ["LCMScheduler"] + _import_structure["scheduling_pndm"] = ["PNDMScheduler"] + _import_structure["scheduling_repaint"] = ["RePaintScheduler"] + _import_structure["scheduling_sasolver"] = ["SASolverScheduler"] + _import_structure["scheduling_sde_ve"] = ["ScoreSdeVeScheduler"] + _import_structure["scheduling_tcd"] = ["TCDScheduler"] + _import_structure["scheduling_unclip"] = ["UnCLIPScheduler"] + _import_structure["scheduling_unipc_multistep"] = ["UniPCMultistepScheduler"] + _import_structure["scheduling_utils"] = ["KarrasDiffusionSchedulers", "SchedulerMixin"] + _import_structure["scheduling_vq_diffusion"] = ["VQDiffusionScheduler"] + +try: + if not is_flax_available(): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_flax_objects # noqa F403 + + _dummy_modules.update(get_objects_from_module(dummy_flax_objects)) + +else: + _import_structure["scheduling_ddim_flax"] = ["FlaxDDIMScheduler"] + _import_structure["scheduling_ddpm_flax"] = ["FlaxDDPMScheduler"] + _import_structure["scheduling_dpmsolver_multistep_flax"] = ["FlaxDPMSolverMultistepScheduler"] + _import_structure["scheduling_euler_discrete_flax"] = ["FlaxEulerDiscreteScheduler"] + _import_structure["scheduling_karras_ve_flax"] = ["FlaxKarrasVeScheduler"] + _import_structure["scheduling_lms_discrete_flax"] = ["FlaxLMSDiscreteScheduler"] + _import_structure["scheduling_pndm_flax"] = ["FlaxPNDMScheduler"] + _import_structure["scheduling_sde_ve_flax"] = ["FlaxScoreSdeVeScheduler"] + _import_structure["scheduling_utils_flax"] = [ + "FlaxKarrasDiffusionSchedulers", + "FlaxSchedulerMixin", + "FlaxSchedulerOutput", + "broadcast_to_shape_from_left", + ] + + +try: + if not (is_torch_available() and is_scipy_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_torch_and_scipy_objects # noqa F403 + + _dummy_modules.update(get_objects_from_module(dummy_torch_and_scipy_objects)) + +else: + _import_structure["scheduling_lms_discrete"] = ["LMSDiscreteScheduler"] + +try: + if not (is_torch_available() and is_torchsde_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ..utils import dummy_torch_and_torchsde_objects # noqa F403 + + _dummy_modules.update(get_objects_from_module(dummy_torch_and_torchsde_objects)) + +else: + _import_structure["scheduling_dpmsolver_sde"] = ["DPMSolverSDEScheduler"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + from ..utils import ( + OptionalDependencyNotAvailable, + is_flax_available, + is_scipy_available, + is_torch_available, + is_torchsde_available, + ) + + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_pt_objects import * # noqa F403 + else: + from .deprecated import KarrasVeScheduler, ScoreSdeVpScheduler + from .scheduling_amused import AmusedScheduler + from .scheduling_consistency_decoder import ConsistencyDecoderScheduler + from .scheduling_consistency_models import CMStochasticIterativeScheduler + from .scheduling_ddim import DDIMScheduler + from .scheduling_ddim_inverse import DDIMInverseScheduler + from .scheduling_ddim_parallel import DDIMParallelScheduler + from .scheduling_ddpm import DDPMScheduler + from .scheduling_ddpm_parallel import DDPMParallelScheduler + from .scheduling_ddpm_wuerstchen import DDPMWuerstchenScheduler + from .scheduling_deis_multistep import DEISMultistepScheduler + from .scheduling_dpmsolver_multistep import DPMSolverMultistepScheduler + from .scheduling_dpmsolver_multistep_inverse import DPMSolverMultistepInverseScheduler + from .scheduling_dpmsolver_singlestep import DPMSolverSinglestepScheduler + from .scheduling_edm_dpmsolver_multistep import EDMDPMSolverMultistepScheduler + from .scheduling_edm_euler import EDMEulerScheduler + from .scheduling_euler_ancestral_discrete import EulerAncestralDiscreteScheduler + from .scheduling_euler_discrete import EulerDiscreteScheduler + from .scheduling_heun_discrete import HeunDiscreteScheduler + from .scheduling_ipndm import IPNDMScheduler + from .scheduling_k_dpm_2_ancestral_discrete import KDPM2AncestralDiscreteScheduler + from .scheduling_k_dpm_2_discrete import KDPM2DiscreteScheduler + from .scheduling_lcm import LCMScheduler + from .scheduling_pndm import PNDMScheduler + from .scheduling_repaint import RePaintScheduler + from .scheduling_sasolver import SASolverScheduler + from .scheduling_sde_ve import ScoreSdeVeScheduler + from .scheduling_tcd import TCDScheduler + from .scheduling_unclip import UnCLIPScheduler + from .scheduling_unipc_multistep import UniPCMultistepScheduler + from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin + from .scheduling_vq_diffusion import VQDiffusionScheduler + + try: + if not is_flax_available(): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_flax_objects import * # noqa F403 + else: + from .scheduling_ddim_flax import FlaxDDIMScheduler + from .scheduling_ddpm_flax import FlaxDDPMScheduler + from .scheduling_dpmsolver_multistep_flax import FlaxDPMSolverMultistepScheduler + from .scheduling_euler_discrete_flax import FlaxEulerDiscreteScheduler + from .scheduling_karras_ve_flax import FlaxKarrasVeScheduler + from .scheduling_lms_discrete_flax import FlaxLMSDiscreteScheduler + from .scheduling_pndm_flax import FlaxPNDMScheduler + from .scheduling_sde_ve_flax import FlaxScoreSdeVeScheduler + from .scheduling_utils_flax import ( + FlaxKarrasDiffusionSchedulers, + FlaxSchedulerMixin, + FlaxSchedulerOutput, + broadcast_to_shape_from_left, + ) + + try: + if not (is_torch_available() and is_scipy_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_torch_and_scipy_objects import * # noqa F403 + else: + from .scheduling_lms_discrete import LMSDiscreteScheduler + + try: + if not (is_torch_available() and is_torchsde_available()): + raise OptionalDependencyNotAvailable() + except OptionalDependencyNotAvailable: + from ..utils.dummy_torch_and_torchsde_objects import * # noqa F403 + else: + from .scheduling_dpmsolver_sde import DPMSolverSDEScheduler + +else: + import sys + + sys.modules[__name__] = _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__) + for name, value in _dummy_modules.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/schedulers/deprecated/__init__.py b/diffusers-0.27.0/src/diffusers/schedulers/deprecated/__init__.py new file mode 100755 index 0000000..786707f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/deprecated/__init__.py @@ -0,0 +1,50 @@ +from typing import TYPE_CHECKING + +from ...utils import ( + DIFFUSERS_SLOW_IMPORT, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_torch_available, + is_transformers_available, +) + + +_dummy_objects = {} +_import_structure = {} + +try: + if not (is_transformers_available() and is_torch_available()): + raise OptionalDependencyNotAvailable() +except OptionalDependencyNotAvailable: + from ...utils import dummy_pt_objects # noqa F403 + + _dummy_objects.update(get_objects_from_module(dummy_pt_objects)) +else: + _import_structure["scheduling_karras_ve"] = ["KarrasVeScheduler"] + _import_structure["scheduling_sde_vp"] = ["ScoreSdeVpScheduler"] + +if TYPE_CHECKING or DIFFUSERS_SLOW_IMPORT: + try: + if not is_torch_available(): + raise OptionalDependencyNotAvailable() + + except OptionalDependencyNotAvailable: + from ..utils.dummy_pt_objects import * # noqa F403 + else: + from .scheduling_karras_ve import KarrasVeScheduler + from .scheduling_sde_vp import ScoreSdeVpScheduler + + +else: + import sys + + sys.modules[__name__] = _LazyModule( + __name__, + globals()["__file__"], + _import_structure, + module_spec=__spec__, + ) + + for name, value in _dummy_objects.items(): + setattr(sys.modules[__name__], name, value) diff --git a/diffusers-0.27.0/src/diffusers/schedulers/deprecated/scheduling_karras_ve.py b/diffusers-0.27.0/src/diffusers/schedulers/deprecated/scheduling_karras_ve.py new file mode 100755 index 0000000..d776d98 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/deprecated/scheduling_karras_ve.py @@ -0,0 +1,243 @@ +# Copyright 2024 NVIDIA and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import numpy as np +import torch + +from ...configuration_utils import ConfigMixin, register_to_config +from ...utils import BaseOutput +from ...utils.torch_utils import randn_tensor +from ..scheduling_utils import SchedulerMixin + + +@dataclass +class KarrasVeOutput(BaseOutput): + """ + Output class for the scheduler's step function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample (x_{t-1}) of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + derivative (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Derivative of predicted original image sample (x_0). + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample (x_{0}) based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + derivative: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +class KarrasVeScheduler(SchedulerMixin, ConfigMixin): + """ + A stochastic scheduler tailored to variance-expanding models. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + + + For more details on the parameters, see [Appendix E](https://arxiv.org/abs/2206.00364). The grid search values used + to find the optimal `{s_noise, s_churn, s_min, s_max}` for a specific model are described in Table 5 of the paper. + + + + Args: + sigma_min (`float`, defaults to 0.02): + The minimum noise magnitude. + sigma_max (`float`, defaults to 100): + The maximum noise magnitude. + s_noise (`float`, defaults to 1.007): + The amount of additional noise to counteract loss of detail during sampling. A reasonable range is [1.000, + 1.011]. + s_churn (`float`, defaults to 80): + The parameter controlling the overall amount of stochasticity. A reasonable range is [0, 100]. + s_min (`float`, defaults to 0.05): + The start value of the sigma range to add noise (enable stochasticity). A reasonable range is [0, 10]. + s_max (`float`, defaults to 50): + The end value of the sigma range to add noise. A reasonable range is [0.2, 80]. + """ + + order = 2 + + @register_to_config + def __init__( + self, + sigma_min: float = 0.02, + sigma_max: float = 100, + s_noise: float = 1.007, + s_churn: float = 80, + s_min: float = 0.05, + s_max: float = 50, + ): + # standard deviation of the initial noise distribution + self.init_noise_sigma = sigma_max + + # setable values + self.num_inference_steps: int = None + self.timesteps: np.IntTensor = None + self.schedule: torch.FloatTensor = None # sigma(t_i) + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.num_inference_steps = num_inference_steps + timesteps = np.arange(0, self.num_inference_steps)[::-1].copy() + self.timesteps = torch.from_numpy(timesteps).to(device) + schedule = [ + ( + self.config.sigma_max**2 + * (self.config.sigma_min**2 / self.config.sigma_max**2) ** (i / (num_inference_steps - 1)) + ) + for i in self.timesteps + ] + self.schedule = torch.tensor(schedule, dtype=torch.float32, device=device) + + def add_noise_to_input( + self, sample: torch.FloatTensor, sigma: float, generator: Optional[torch.Generator] = None + ) -> Tuple[torch.FloatTensor, float]: + """ + Explicit Langevin-like "churn" step of adding noise to the sample according to a `gamma_i ≥ 0` to reach a + higher noise level `sigma_hat = sigma_i + gamma_i*sigma_i`. + + Args: + sample (`torch.FloatTensor`): + The input sample. + sigma (`float`): + generator (`torch.Generator`, *optional*): + A random number generator. + """ + if self.config.s_min <= sigma <= self.config.s_max: + gamma = min(self.config.s_churn / self.num_inference_steps, 2**0.5 - 1) + else: + gamma = 0 + + # sample eps ~ N(0, S_noise^2 * I) + eps = self.config.s_noise * randn_tensor(sample.shape, generator=generator).to(sample.device) + sigma_hat = sigma + gamma * sigma + sample_hat = sample + ((sigma_hat**2 - sigma**2) ** 0.5 * eps) + + return sample_hat, sigma_hat + + def step( + self, + model_output: torch.FloatTensor, + sigma_hat: float, + sigma_prev: float, + sample_hat: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[KarrasVeOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + sigma_hat (`float`): + sigma_prev (`float`): + sample_hat (`torch.FloatTensor`): + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_karras_ve.KarrasVESchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_karras_ve.KarrasVESchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_karras_ve.KarrasVESchedulerOutput`] is returned, + otherwise a tuple is returned where the first element is the sample tensor. + + """ + + pred_original_sample = sample_hat + sigma_hat * model_output + derivative = (sample_hat - pred_original_sample) / sigma_hat + sample_prev = sample_hat + (sigma_prev - sigma_hat) * derivative + + if not return_dict: + return (sample_prev, derivative) + + return KarrasVeOutput( + prev_sample=sample_prev, derivative=derivative, pred_original_sample=pred_original_sample + ) + + def step_correct( + self, + model_output: torch.FloatTensor, + sigma_hat: float, + sigma_prev: float, + sample_hat: torch.FloatTensor, + sample_prev: torch.FloatTensor, + derivative: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[KarrasVeOutput, Tuple]: + """ + Corrects the predicted sample based on the `model_output` of the network. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + sigma_hat (`float`): TODO + sigma_prev (`float`): TODO + sample_hat (`torch.FloatTensor`): TODO + sample_prev (`torch.FloatTensor`): TODO + derivative (`torch.FloatTensor`): TODO + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_ddpm.DDPMSchedulerOutput`] or `tuple`. + + Returns: + prev_sample (TODO): updated sample in the diffusion chain. derivative (TODO): TODO + + """ + pred_original_sample = sample_prev + sigma_prev * model_output + derivative_corr = (sample_prev - pred_original_sample) / sigma_prev + sample_prev = sample_hat + (sigma_prev - sigma_hat) * (0.5 * derivative + 0.5 * derivative_corr) + + if not return_dict: + return (sample_prev, derivative) + + return KarrasVeOutput( + prev_sample=sample_prev, derivative=derivative, pred_original_sample=pred_original_sample + ) + + def add_noise(self, original_samples, noise, timesteps): + raise NotImplementedError() diff --git a/diffusers-0.27.0/src/diffusers/schedulers/deprecated/scheduling_sde_vp.py b/diffusers-0.27.0/src/diffusers/schedulers/deprecated/scheduling_sde_vp.py new file mode 100755 index 0000000..09b02ca --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/deprecated/scheduling_sde_vp.py @@ -0,0 +1,109 @@ +# Copyright 2024 Google Brain and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/yang-song/score_sde_pytorch + +import math +from typing import Union + +import torch + +from ...configuration_utils import ConfigMixin, register_to_config +from ...utils.torch_utils import randn_tensor +from ..scheduling_utils import SchedulerMixin + + +class ScoreSdeVpScheduler(SchedulerMixin, ConfigMixin): + """ + `ScoreSdeVpScheduler` is a variance preserving stochastic differential equation (SDE) scheduler. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 2000): + The number of diffusion steps to train the model. + beta_min (`int`, defaults to 0.1): + beta_max (`int`, defaults to 20): + sampling_eps (`int`, defaults to 1e-3): + The end value of sampling where timesteps decrease progressively from 1 to epsilon. + """ + + order = 1 + + @register_to_config + def __init__(self, num_train_timesteps=2000, beta_min=0.1, beta_max=20, sampling_eps=1e-3): + self.sigmas = None + self.discrete_sigmas = None + self.timesteps = None + + def set_timesteps(self, num_inference_steps, device: Union[str, torch.device] = None): + """ + Sets the continuous timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.timesteps = torch.linspace(1, self.config.sampling_eps, num_inference_steps, device=device) + + def step_pred(self, score, x, t, generator=None): + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + score (): + x (): + t (): + generator (`torch.Generator`, *optional*): + A random number generator. + """ + if self.timesteps is None: + raise ValueError( + "`self.timesteps` is not set, you need to run 'set_timesteps' after creating the scheduler" + ) + + # TODO(Patrick) better comments + non-PyTorch + # postprocess model score + log_mean_coeff = -0.25 * t**2 * (self.config.beta_max - self.config.beta_min) - 0.5 * t * self.config.beta_min + std = torch.sqrt(1.0 - torch.exp(2.0 * log_mean_coeff)) + std = std.flatten() + while len(std.shape) < len(score.shape): + std = std.unsqueeze(-1) + score = -score / std + + # compute + dt = -1.0 / len(self.timesteps) + + beta_t = self.config.beta_min + t * (self.config.beta_max - self.config.beta_min) + beta_t = beta_t.flatten() + while len(beta_t.shape) < len(x.shape): + beta_t = beta_t.unsqueeze(-1) + drift = -0.5 * beta_t * x + + diffusion = torch.sqrt(beta_t) + drift = drift - diffusion**2 * score + x_mean = x + drift * dt + + # add noise + noise = randn_tensor(x.shape, layout=x.layout, generator=generator, device=x.device, dtype=x.dtype) + x = x_mean + diffusion * math.sqrt(-dt) * noise + + return x, x_mean + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_amused.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_amused.py new file mode 100755 index 0000000..51fbe6a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_amused.py @@ -0,0 +1,162 @@ +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from .scheduling_utils import SchedulerMixin + + +def gumbel_noise(t, generator=None): + device = generator.device if generator is not None else t.device + noise = torch.zeros_like(t, device=device).uniform_(0, 1, generator=generator).to(t.device) + return -torch.log((-torch.log(noise.clamp(1e-20))).clamp(1e-20)) + + +def mask_by_random_topk(mask_len, probs, temperature=1.0, generator=None): + confidence = torch.log(probs.clamp(1e-20)) + temperature * gumbel_noise(probs, generator=generator) + sorted_confidence = torch.sort(confidence, dim=-1).values + cut_off = torch.gather(sorted_confidence, 1, mask_len.long()) + masking = confidence < cut_off + return masking + + +@dataclass +class AmusedSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: torch.FloatTensor = None + + +class AmusedScheduler(SchedulerMixin, ConfigMixin): + order = 1 + + temperatures: torch.Tensor + + @register_to_config + def __init__( + self, + mask_token_id: int, + masking_schedule: str = "cosine", + ): + self.temperatures = None + self.timesteps = None + + def set_timesteps( + self, + num_inference_steps: int, + temperature: Union[int, Tuple[int, int], List[int]] = (2, 0), + device: Union[str, torch.device] = None, + ): + self.timesteps = torch.arange(num_inference_steps, device=device).flip(0) + + if isinstance(temperature, (tuple, list)): + self.temperatures = torch.linspace(temperature[0], temperature[1], num_inference_steps, device=device) + else: + self.temperatures = torch.linspace(temperature, 0.01, num_inference_steps, device=device) + + def step( + self, + model_output: torch.FloatTensor, + timestep: torch.long, + sample: torch.LongTensor, + starting_mask_ratio: int = 1, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[AmusedSchedulerOutput, Tuple]: + two_dim_input = sample.ndim == 3 and model_output.ndim == 4 + + if two_dim_input: + batch_size, codebook_size, height, width = model_output.shape + sample = sample.reshape(batch_size, height * width) + model_output = model_output.reshape(batch_size, codebook_size, height * width).permute(0, 2, 1) + + unknown_map = sample == self.config.mask_token_id + + probs = model_output.softmax(dim=-1) + + device = probs.device + probs_ = probs.to(generator.device) if generator is not None else probs # handles when generator is on CPU + if probs_.device.type == "cpu" and probs_.dtype != torch.float32: + probs_ = probs_.float() # multinomial is not implemented for cpu half precision + probs_ = probs_.reshape(-1, probs.size(-1)) + pred_original_sample = torch.multinomial(probs_, 1, generator=generator).to(device=device) + pred_original_sample = pred_original_sample[:, 0].view(*probs.shape[:-1]) + pred_original_sample = torch.where(unknown_map, pred_original_sample, sample) + + if timestep == 0: + prev_sample = pred_original_sample + else: + seq_len = sample.shape[1] + step_idx = (self.timesteps == timestep).nonzero() + ratio = (step_idx + 1) / len(self.timesteps) + + if self.config.masking_schedule == "cosine": + mask_ratio = torch.cos(ratio * math.pi / 2) + elif self.config.masking_schedule == "linear": + mask_ratio = 1 - ratio + else: + raise ValueError(f"unknown masking schedule {self.config.masking_schedule}") + + mask_ratio = starting_mask_ratio * mask_ratio + + mask_len = (seq_len * mask_ratio).floor() + # do not mask more than amount previously masked + mask_len = torch.min(unknown_map.sum(dim=-1, keepdim=True) - 1, mask_len) + # mask at least one + mask_len = torch.max(torch.tensor([1], device=model_output.device), mask_len) + + selected_probs = torch.gather(probs, -1, pred_original_sample[:, :, None])[:, :, 0] + # Ignores the tokens given in the input by overwriting their confidence. + selected_probs = torch.where(unknown_map, selected_probs, torch.finfo(selected_probs.dtype).max) + + masking = mask_by_random_topk(mask_len, selected_probs, self.temperatures[step_idx], generator) + + # Masks tokens with lower confidence. + prev_sample = torch.where(masking, self.config.mask_token_id, pred_original_sample) + + if two_dim_input: + prev_sample = prev_sample.reshape(batch_size, height, width) + pred_original_sample = pred_original_sample.reshape(batch_size, height, width) + + if not return_dict: + return (prev_sample, pred_original_sample) + + return AmusedSchedulerOutput(prev_sample, pred_original_sample) + + def add_noise(self, sample, timesteps, generator=None): + step_idx = (self.timesteps == timesteps).nonzero() + ratio = (step_idx + 1) / len(self.timesteps) + + if self.config.masking_schedule == "cosine": + mask_ratio = torch.cos(ratio * math.pi / 2) + elif self.config.masking_schedule == "linear": + mask_ratio = 1 - ratio + else: + raise ValueError(f"unknown masking schedule {self.config.masking_schedule}") + + mask_indices = ( + torch.rand( + sample.shape, device=generator.device if generator is not None else sample.device, generator=generator + ).to(sample.device) + < mask_ratio + ) + + masked_sample = sample.clone() + + masked_sample[mask_indices] = self.config.mask_token_id + + return masked_sample diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_consistency_decoder.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_consistency_decoder.py new file mode 100755 index 0000000..69ca8a1 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_consistency_decoder.py @@ -0,0 +1,180 @@ +import math +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import SchedulerMixin + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +@dataclass +class ConsistencyDecoderSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + """ + + prev_sample: torch.FloatTensor + + +class ConsistencyDecoderScheduler(SchedulerMixin, ConfigMixin): + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1024, + sigma_data: float = 0.5, + ): + betas = betas_for_alpha_bar(num_train_timesteps) + + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + + self.sqrt_alphas_cumprod = torch.sqrt(alphas_cumprod) + self.sqrt_one_minus_alphas_cumprod = torch.sqrt(1.0 - alphas_cumprod) + + sigmas = torch.sqrt(1.0 / alphas_cumprod - 1) + + sqrt_recip_alphas_cumprod = torch.sqrt(1.0 / alphas_cumprod) + + self.c_skip = sqrt_recip_alphas_cumprod * sigma_data**2 / (sigmas**2 + sigma_data**2) + self.c_out = sigmas * sigma_data / (sigmas**2 + sigma_data**2) ** 0.5 + self.c_in = sqrt_recip_alphas_cumprod / (sigmas**2 + sigma_data**2) ** 0.5 + + def set_timesteps( + self, + num_inference_steps: Optional[int] = None, + device: Union[str, torch.device] = None, + ): + if num_inference_steps != 2: + raise ValueError("Currently more than 2 inference steps are not supported.") + + self.timesteps = torch.tensor([1008, 512], dtype=torch.long, device=device) + self.sqrt_alphas_cumprod = self.sqrt_alphas_cumprod.to(device) + self.sqrt_one_minus_alphas_cumprod = self.sqrt_one_minus_alphas_cumprod.to(device) + self.c_skip = self.c_skip.to(device) + self.c_out = self.c_out.to(device) + self.c_in = self.c_in.to(device) + + @property + def init_noise_sigma(self): + return self.sqrt_one_minus_alphas_cumprod[self.timesteps[0]] + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample * self.c_in[timestep] + + def step( + self, + model_output: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + sample: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[ConsistencyDecoderSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + timestep (`float`): + The current timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a + [`~schedulers.scheduling_consistency_models.ConsistencyDecoderSchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_consistency_models.ConsistencyDecoderSchedulerOutput`] or `tuple`: + If return_dict is `True`, + [`~schedulers.scheduling_consistency_models.ConsistencyDecoderSchedulerOutput`] is returned, otherwise + a tuple is returned where the first element is the sample tensor. + """ + x_0 = self.c_out[timestep] * model_output + self.c_skip[timestep] * sample + + timestep_idx = torch.where(self.timesteps == timestep)[0] + + if timestep_idx == len(self.timesteps) - 1: + prev_sample = x_0 + else: + noise = randn_tensor(x_0.shape, generator=generator, dtype=x_0.dtype, device=x_0.device) + prev_sample = ( + self.sqrt_alphas_cumprod[self.timesteps[timestep_idx + 1]].to(x_0.dtype) * x_0 + + self.sqrt_one_minus_alphas_cumprod[self.timesteps[timestep_idx + 1]].to(x_0.dtype) * noise + ) + + if not return_dict: + return (prev_sample,) + + return ConsistencyDecoderSchedulerOutput(prev_sample=prev_sample) diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_consistency_models.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_consistency_models.py new file mode 100755 index 0000000..14d37a3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_consistency_models.py @@ -0,0 +1,448 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput, logging +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import SchedulerMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class CMStochasticIterativeSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + """ + + prev_sample: torch.FloatTensor + + +class CMStochasticIterativeScheduler(SchedulerMixin, ConfigMixin): + """ + Multistep and onestep sampling for consistency models. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 40): + The number of diffusion steps to train the model. + sigma_min (`float`, defaults to 0.002): + Minimum noise magnitude in the sigma schedule. Defaults to 0.002 from the original implementation. + sigma_max (`float`, defaults to 80.0): + Maximum noise magnitude in the sigma schedule. Defaults to 80.0 from the original implementation. + sigma_data (`float`, defaults to 0.5): + The standard deviation of the data distribution from the EDM + [paper](https://huggingface.co/papers/2206.00364). Defaults to 0.5 from the original implementation. + s_noise (`float`, defaults to 1.0): + The amount of additional noise to counteract loss of detail during sampling. A reasonable range is [1.000, + 1.011]. Defaults to 1.0 from the original implementation. + rho (`float`, defaults to 7.0): + The parameter for calculating the Karras sigma schedule from the EDM + [paper](https://huggingface.co/papers/2206.00364). Defaults to 7.0 from the original implementation. + clip_denoised (`bool`, defaults to `True`): + Whether to clip the denoised outputs to `(-1, 1)`. + timesteps (`List` or `np.ndarray` or `torch.Tensor`, *optional*): + An explicit timestep schedule that can be optionally specified. The timesteps are expected to be in + increasing order. + """ + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 40, + sigma_min: float = 0.002, + sigma_max: float = 80.0, + sigma_data: float = 0.5, + s_noise: float = 1.0, + rho: float = 7.0, + clip_denoised: bool = True, + ): + # standard deviation of the initial noise distribution + self.init_noise_sigma = sigma_max + + ramp = np.linspace(0, 1, num_train_timesteps) + sigmas = self._convert_to_karras(ramp) + timesteps = self.sigma_to_t(sigmas) + + # setable values + self.num_inference_steps = None + self.sigmas = torch.from_numpy(sigmas) + self.timesteps = torch.from_numpy(timesteps) + self.custom_timesteps = False + self.is_scale_input_called = False + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def scale_model_input( + self, sample: torch.FloatTensor, timestep: Union[float, torch.FloatTensor] + ) -> torch.FloatTensor: + """ + Scales the consistency model input by `(sigma**2 + sigma_data**2) ** 0.5`. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`float` or `torch.FloatTensor`): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + # Get sigma corresponding to timestep + if self.step_index is None: + self._init_step_index(timestep) + + sigma = self.sigmas[self.step_index] + + sample = sample / ((sigma**2 + self.config.sigma_data**2) ** 0.5) + + self.is_scale_input_called = True + return sample + + def sigma_to_t(self, sigmas: Union[float, np.ndarray]): + """ + Gets scaled timesteps from the Karras sigmas for input to the consistency model. + + Args: + sigmas (`float` or `np.ndarray`): + A single Karras sigma or an array of Karras sigmas. + + Returns: + `float` or `np.ndarray`: + A scaled input timestep or scaled input timestep array. + """ + if not isinstance(sigmas, np.ndarray): + sigmas = np.array(sigmas, dtype=np.float64) + + timesteps = 1000 * 0.25 * np.log(sigmas + 1e-44) + + return timesteps + + def set_timesteps( + self, + num_inference_steps: Optional[int] = None, + device: Union[str, torch.device] = None, + timesteps: Optional[List[int]] = None, + ): + """ + Sets the timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of equal spacing between timesteps is used. If `timesteps` is passed, + `num_inference_steps` must be `None`. + """ + if num_inference_steps is None and timesteps is None: + raise ValueError("Exactly one of `num_inference_steps` or `timesteps` must be supplied.") + + if num_inference_steps is not None and timesteps is not None: + raise ValueError("Can only pass one of `num_inference_steps` or `timesteps`.") + + # Follow DDPMScheduler custom timesteps logic + if timesteps is not None: + for i in range(1, len(timesteps)): + if timesteps[i] >= timesteps[i - 1]: + raise ValueError("`timesteps` must be in descending order.") + + if timesteps[0] >= self.config.num_train_timesteps: + raise ValueError( + f"`timesteps` must start before `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps}." + ) + + timesteps = np.array(timesteps, dtype=np.int64) + self.custom_timesteps = True + else: + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + self.num_inference_steps = num_inference_steps + + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64) + self.custom_timesteps = False + + # Map timesteps to Karras sigmas directly for multistep sampling + # See https://github.com/openai/consistency_models/blob/main/cm/karras_diffusion.py#L675 + num_train_timesteps = self.config.num_train_timesteps + ramp = timesteps[::-1].copy() + ramp = ramp / (num_train_timesteps - 1) + sigmas = self._convert_to_karras(ramp) + timesteps = self.sigma_to_t(sigmas) + + sigmas = np.concatenate([sigmas, [self.sigma_min]]).astype(np.float32) + self.sigmas = torch.from_numpy(sigmas).to(device=device) + + if str(device).startswith("mps"): + # mps does not support float64 + self.timesteps = torch.from_numpy(timesteps).to(device, dtype=torch.float32) + else: + self.timesteps = torch.from_numpy(timesteps).to(device=device) + + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Modified _convert_to_karras implementation that takes in ramp as argument + def _convert_to_karras(self, ramp): + """Constructs the noise schedule of Karras et al. (2022).""" + + sigma_min: float = self.config.sigma_min + sigma_max: float = self.config.sigma_max + + rho = self.config.rho + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + def get_scalings(self, sigma): + sigma_data = self.config.sigma_data + + c_skip = sigma_data**2 / (sigma**2 + sigma_data**2) + c_out = sigma * sigma_data / (sigma**2 + sigma_data**2) ** 0.5 + return c_skip, c_out + + def get_scalings_for_boundary_condition(self, sigma): + """ + Gets the scalings used in the consistency model parameterization (from Appendix C of the + [paper](https://huggingface.co/papers/2303.01469)) to enforce boundary condition. + + + + `epsilon` in the equations for `c_skip` and `c_out` is set to `sigma_min`. + + + + Args: + sigma (`torch.FloatTensor`): + The current sigma in the Karras sigma schedule. + + Returns: + `tuple`: + A two-element tuple where `c_skip` (which weights the current sample) is the first element and `c_out` + (which weights the consistency model output) is the second element. + """ + sigma_min = self.config.sigma_min + sigma_data = self.config.sigma_data + + c_skip = sigma_data**2 / ((sigma - sigma_min) ** 2 + sigma_data**2) + c_out = (sigma - sigma_min) * sigma_data / (sigma**2 + sigma_data**2) ** 0.5 + return c_skip, c_out + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + sample: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[CMStochasticIterativeSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + timestep (`float`): + The current timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a + [`~schedulers.scheduling_consistency_models.CMStochasticIterativeSchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_consistency_models.CMStochasticIterativeSchedulerOutput`] or `tuple`: + If return_dict is `True`, + [`~schedulers.scheduling_consistency_models.CMStochasticIterativeSchedulerOutput`] is returned, + otherwise a tuple is returned where the first element is the sample tensor. + """ + + if ( + isinstance(timestep, int) + or isinstance(timestep, torch.IntTensor) + or isinstance(timestep, torch.LongTensor) + ): + raise ValueError( + ( + "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to" + f" `{self.__class__}.step()` is not supported. Make sure to pass" + " one of the `scheduler.timesteps` as a timestep." + ), + ) + + if not self.is_scale_input_called: + logger.warning( + "The `scale_model_input` function should be called before `step` to ensure correct denoising. " + "See `StableDiffusionPipeline` for a usage example." + ) + + sigma_min = self.config.sigma_min + sigma_max = self.config.sigma_max + + if self.step_index is None: + self._init_step_index(timestep) + + # sigma_next corresponds to next_t in original implementation + sigma = self.sigmas[self.step_index] + if self.step_index + 1 < self.config.num_train_timesteps: + sigma_next = self.sigmas[self.step_index + 1] + else: + # Set sigma_next to sigma_min + sigma_next = self.sigmas[-1] + + # Get scalings for boundary conditions + c_skip, c_out = self.get_scalings_for_boundary_condition(sigma) + + # 1. Denoise model output using boundary conditions + denoised = c_out * model_output + c_skip * sample + if self.config.clip_denoised: + denoised = denoised.clamp(-1, 1) + + # 2. Sample z ~ N(0, s_noise^2 * I) + # Noise is not used for onestep sampling. + if len(self.timesteps) > 1: + noise = randn_tensor( + model_output.shape, dtype=model_output.dtype, device=model_output.device, generator=generator + ) + else: + noise = torch.zeros_like(model_output) + z = noise * self.config.s_noise + + sigma_hat = sigma_next.clamp(min=sigma_min, max=sigma_max) + + # 3. Return noisy sample + # tau = sigma_hat, eps = sigma_min + prev_sample = denoised + z * (sigma_hat**2 - sigma_min**2) ** 0.5 + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return CMStochasticIterativeSchedulerOutput(prev_sample=prev_sample) + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + noisy_samples = original_samples + noise * sigma + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim.py new file mode 100755 index 0000000..33d3892 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim.py @@ -0,0 +1,520 @@ +# Copyright 2024 Stanford University Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion +# and https://github.com/hojonathanho/diffusion + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->DDIM +class DDIMSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.FloatTensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.FloatTensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class DDIMScheduler(SchedulerMixin, ConfigMixin): + """ + `DDIMScheduler` extends the denoising procedure introduced in denoising diffusion probabilistic models (DDPMs) with + non-Markovian guidance. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + clip_sample (`bool`, defaults to `True`): + Clip the predicted sample for numerical stability. + clip_sample_range (`float`, defaults to 1.0): + The maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + set_alpha_to_one (`bool`, defaults to `True`): + Each diffusion step uses the alphas product value at that step and at the previous one. For the final step + there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`, + otherwise it uses the alpha value at step 0. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True`. + timestep_spacing (`str`, defaults to `"leading"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + clip_sample: bool = True, + set_alpha_to_one: bool = True, + steps_offset: int = 0, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + clip_sample_range: float = 1.0, + sample_max_value: float = 1.0, + timestep_spacing: str = "leading", + rescale_betas_zero_snr: bool = False, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + # Rescale for zero SNR + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + # At every step in ddim, we are looking into the previous alphas_cumprod + # For the final step, there is no previous alphas_cumprod because we are already at 0 + # `set_alpha_to_one` decides whether we set this parameter simply to one or + # whether we use the final alpha of the "non-previous" one. + self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0] + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy().astype(np.int64)) + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def _get_variance(self, timestep, prev_timestep): + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + variance = (beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev) + + return variance + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + """ + + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + self.num_inference_steps = num_inference_steps + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps) + .round()[::-1] + .copy() + .astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.round(np.arange(self.config.num_train_timesteps, 0, -step_ratio)).astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'leading' or 'trailing'." + ) + + self.timesteps = torch.from_numpy(timesteps).to(device) + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + eta: float = 0.0, + use_clipped_model_output: bool = False, + generator=None, + variance_noise: Optional[torch.FloatTensor] = None, + return_dict: bool = True, + ) -> Union[DDIMSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + eta (`float`): + The weight of noise for added noise in diffusion step. + use_clipped_model_output (`bool`, defaults to `False`): + If `True`, computes "corrected" `model_output` from the clipped predicted original sample. Necessary + because predicted original sample is clipped to [-1, 1] when `self.config.clip_sample` is `True`. If no + clipping has happened, "corrected" `model_output` would coincide with the one provided as input and + `use_clipped_model_output` has no effect. + generator (`torch.Generator`, *optional*): + A random number generator. + variance_noise (`torch.FloatTensor`): + Alternative to generating noise with `generator` by directly providing the noise for the variance + itself. Useful for methods such as [`CycleDiffusion`]. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_ddim.DDIMSchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.DDIMSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_ddim.DDIMSchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + # See formulas (12) and (16) of DDIM paper https://arxiv.org/pdf/2010.02502.pdf + # Ideally, read DDIM paper in-detail understanding + + # Notation ( -> + # - pred_noise_t -> e_theta(x_t, t) + # - pred_original_sample -> f_theta(x_t, t) or x_0 + # - std_dev_t -> sigma_t + # - eta -> η + # - pred_sample_direction -> "direction pointing to x_t" + # - pred_prev_sample -> "x_t-1" + + # 1. get previous step value (=t-1) + prev_timestep = timestep - self.config.num_train_timesteps // self.num_inference_steps + + # 2. compute alphas, betas + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + + beta_prod_t = 1 - alpha_prod_t + + # 3. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + pred_epsilon = model_output + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5) + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + pred_epsilon = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction`" + ) + + # 4. Clip or threshold "predicted x_0" + if self.config.thresholding: + pred_original_sample = self._threshold_sample(pred_original_sample) + elif self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 5. compute variance: "sigma_t(η)" -> see formula (16) + # σ_t = sqrt((1 − α_t−1)/(1 − α_t)) * sqrt(1 − α_t/α_t−1) + variance = self._get_variance(timestep, prev_timestep) + std_dev_t = eta * variance ** (0.5) + + if use_clipped_model_output: + # the pred_epsilon is always re-derived from the clipped x_0 in Glide + pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5) + + # 6. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * pred_epsilon + + # 7. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + prev_sample = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction + + if eta > 0: + if variance_noise is not None and generator is not None: + raise ValueError( + "Cannot pass both generator and variance_noise. Please make sure that either `generator` or" + " `variance_noise` stays `None`." + ) + + if variance_noise is None: + variance_noise = randn_tensor( + model_output.shape, generator=generator, device=model_output.device, dtype=model_output.dtype + ) + variance = std_dev_t * variance_noise + + prev_sample = prev_sample + variance + + if not return_dict: + return (prev_sample,) + + return DDIMSchedulerOutput(prev_sample=prev_sample, pred_original_sample=pred_original_sample) + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + # Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement + # for the subsequent add_noise calls + self.alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.get_velocity + def get_velocity( + self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as sample + self.alphas_cumprod = self.alphas_cumprod.to(device=sample.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=sample.dtype) + timesteps = timesteps.to(sample.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(sample.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample + return velocity + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_flax.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_flax.py new file mode 100755 index 0000000..dc3d845 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_flax.py @@ -0,0 +1,313 @@ +# Copyright 2024 Stanford University Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion +# and https://github.com/hojonathanho/diffusion + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import flax +import jax.numpy as jnp + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils_flax import ( + CommonSchedulerState, + FlaxKarrasDiffusionSchedulers, + FlaxSchedulerMixin, + FlaxSchedulerOutput, + add_noise_common, + get_velocity_common, +) + + +@flax.struct.dataclass +class DDIMSchedulerState: + common: CommonSchedulerState + final_alpha_cumprod: jnp.ndarray + + # setable values + init_noise_sigma: jnp.ndarray + timesteps: jnp.ndarray + num_inference_steps: Optional[int] = None + + @classmethod + def create( + cls, + common: CommonSchedulerState, + final_alpha_cumprod: jnp.ndarray, + init_noise_sigma: jnp.ndarray, + timesteps: jnp.ndarray, + ): + return cls( + common=common, + final_alpha_cumprod=final_alpha_cumprod, + init_noise_sigma=init_noise_sigma, + timesteps=timesteps, + ) + + +@dataclass +class FlaxDDIMSchedulerOutput(FlaxSchedulerOutput): + state: DDIMSchedulerState + + +class FlaxDDIMScheduler(FlaxSchedulerMixin, ConfigMixin): + """ + Denoising diffusion implicit models is a scheduler that extends the denoising procedure introduced in denoising + diffusion probabilistic models (DDPMs) with non-Markovian guidance. + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + For more details, see the original paper: https://arxiv.org/abs/2010.02502 + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + beta_start (`float`): the starting `beta` value of inference. + beta_end (`float`): the final `beta` value. + beta_schedule (`str`): + the beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`jnp.ndarray`, optional): + option to pass an array of betas directly to the constructor to bypass `beta_start`, `beta_end` etc. + clip_sample (`bool`, default `True`): + option to clip predicted sample between for numerical stability. The clip range is determined by `clip_sample_range`. + clip_sample_range (`float`, default `1.0`): + the maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + set_alpha_to_one (`bool`, default `True`): + each diffusion step uses the value of alphas product at that step and at the previous one. For the final + step there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`, + otherwise it uses the value of alpha at step 0. + steps_offset (`int`, default `0`): + An offset added to the inference steps, as required by some model families. + prediction_type (`str`, default `epsilon`): + indicates whether the model predicts the noise (epsilon), or the samples. One of `epsilon`, `sample`. + `v-prediction` is not supported for this scheduler. + dtype (`jnp.dtype`, *optional*, defaults to `jnp.float32`): + the `dtype` used for params and computation. + """ + + _compatibles = [e.name for e in FlaxKarrasDiffusionSchedulers] + + dtype: jnp.dtype + + @property + def has_state(self): + return True + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[jnp.ndarray] = None, + clip_sample: bool = True, + clip_sample_range: float = 1.0, + set_alpha_to_one: bool = True, + steps_offset: int = 0, + prediction_type: str = "epsilon", + dtype: jnp.dtype = jnp.float32, + ): + self.dtype = dtype + + def create_state(self, common: Optional[CommonSchedulerState] = None) -> DDIMSchedulerState: + if common is None: + common = CommonSchedulerState.create(self) + + # At every step in ddim, we are looking into the previous alphas_cumprod + # For the final step, there is no previous alphas_cumprod because we are already at 0 + # `set_alpha_to_one` decides whether we set this parameter simply to one or + # whether we use the final alpha of the "non-previous" one. + final_alpha_cumprod = ( + jnp.array(1.0, dtype=self.dtype) if self.config.set_alpha_to_one else common.alphas_cumprod[0] + ) + + # standard deviation of the initial noise distribution + init_noise_sigma = jnp.array(1.0, dtype=self.dtype) + + timesteps = jnp.arange(0, self.config.num_train_timesteps).round()[::-1] + + return DDIMSchedulerState.create( + common=common, + final_alpha_cumprod=final_alpha_cumprod, + init_noise_sigma=init_noise_sigma, + timesteps=timesteps, + ) + + def scale_model_input( + self, state: DDIMSchedulerState, sample: jnp.ndarray, timestep: Optional[int] = None + ) -> jnp.ndarray: + """ + Args: + state (`PNDMSchedulerState`): the `FlaxPNDMScheduler` state data class instance. + sample (`jnp.ndarray`): input sample + timestep (`int`, optional): current timestep + + Returns: + `jnp.ndarray`: scaled input sample + """ + return sample + + def set_timesteps( + self, state: DDIMSchedulerState, num_inference_steps: int, shape: Tuple = () + ) -> DDIMSchedulerState: + """ + Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + state (`DDIMSchedulerState`): + the `FlaxDDIMScheduler` state data class instance. + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + """ + step_ratio = self.config.num_train_timesteps // num_inference_steps + # creates integer timesteps by multiplying by ratio + # rounding to avoid issues when num_inference_step is power of 3 + timesteps = (jnp.arange(0, num_inference_steps) * step_ratio).round()[::-1] + self.config.steps_offset + + return state.replace( + num_inference_steps=num_inference_steps, + timesteps=timesteps, + ) + + def _get_variance(self, state: DDIMSchedulerState, timestep, prev_timestep): + alpha_prod_t = state.common.alphas_cumprod[timestep] + alpha_prod_t_prev = jnp.where( + prev_timestep >= 0, state.common.alphas_cumprod[prev_timestep], state.final_alpha_cumprod + ) + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + variance = (beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev) + + return variance + + def step( + self, + state: DDIMSchedulerState, + model_output: jnp.ndarray, + timestep: int, + sample: jnp.ndarray, + eta: float = 0.0, + return_dict: bool = True, + ) -> Union[FlaxDDIMSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + state (`DDIMSchedulerState`): the `FlaxDDIMScheduler` state data class instance. + model_output (`jnp.ndarray`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + return_dict (`bool`): option for returning tuple rather than FlaxDDIMSchedulerOutput class + + Returns: + [`FlaxDDIMSchedulerOutput`] or `tuple`: [`FlaxDDIMSchedulerOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is the sample tensor. + + """ + if state.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + # See formulas (12) and (16) of DDIM paper https://arxiv.org/pdf/2010.02502.pdf + # Ideally, read DDIM paper in-detail understanding + + # Notation ( -> + # - pred_noise_t -> e_theta(x_t, t) + # - pred_original_sample -> f_theta(x_t, t) or x_0 + # - std_dev_t -> sigma_t + # - eta -> η + # - pred_sample_direction -> "direction pointing to x_t" + # - pred_prev_sample -> "x_t-1" + + # 1. get previous step value (=t-1) + prev_timestep = timestep - self.config.num_train_timesteps // state.num_inference_steps + + alphas_cumprod = state.common.alphas_cumprod + final_alpha_cumprod = state.final_alpha_cumprod + + # 2. compute alphas, betas + alpha_prod_t = alphas_cumprod[timestep] + alpha_prod_t_prev = jnp.where(prev_timestep >= 0, alphas_cumprod[prev_timestep], final_alpha_cumprod) + + beta_prod_t = 1 - alpha_prod_t + + # 3. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + pred_epsilon = model_output + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5) + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + pred_epsilon = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction`" + ) + + # 4. Clip or threshold "predicted x_0" + if self.config.clip_sample: + pred_original_sample = pred_original_sample.clip( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 4. compute variance: "sigma_t(η)" -> see formula (16) + # σ_t = sqrt((1 − α_t−1)/(1 − α_t)) * sqrt(1 − α_t/α_t−1) + variance = self._get_variance(state, timestep, prev_timestep) + std_dev_t = eta * variance ** (0.5) + + # 5. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * pred_epsilon + + # 6. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + prev_sample = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction + + if not return_dict: + return (prev_sample, state) + + return FlaxDDIMSchedulerOutput(prev_sample=prev_sample, state=state) + + def add_noise( + self, + state: DDIMSchedulerState, + original_samples: jnp.ndarray, + noise: jnp.ndarray, + timesteps: jnp.ndarray, + ) -> jnp.ndarray: + return add_noise_common(state.common, original_samples, noise, timesteps) + + def get_velocity( + self, + state: DDIMSchedulerState, + sample: jnp.ndarray, + noise: jnp.ndarray, + timesteps: jnp.ndarray, + ) -> jnp.ndarray: + return get_velocity_common(state.common, sample, noise, timesteps) + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_inverse.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_inverse.py new file mode 100755 index 0000000..b4c19e4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_inverse.py @@ -0,0 +1,374 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion +# and https://github.com/hojonathanho/diffusion +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import BaseOutput, deprecate + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->DDIM +class DDIMSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.FloatTensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.FloatTensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class DDIMInverseScheduler(SchedulerMixin, ConfigMixin): + """ + `DDIMInverseScheduler` is the reverse scheduler of [`DDIMScheduler`]. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + clip_sample (`bool`, defaults to `True`): + Clip the predicted sample for numerical stability. + clip_sample_range (`float`, defaults to 1.0): + The maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + set_alpha_to_one (`bool`, defaults to `True`): + Each diffusion step uses the alphas product value at that step and at the previous one. For the final step + there is no previous alpha. When this option is `True` the previous alpha product is fixed to 0, otherwise + it uses the alpha value at step `num_train_timesteps - 1`. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + timestep_spacing (`str`, defaults to `"leading"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + order = 1 + ignore_for_config = ["kwargs"] + _deprecated_kwargs = ["set_alpha_to_zero"] + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + clip_sample: bool = True, + set_alpha_to_one: bool = True, + steps_offset: int = 0, + prediction_type: str = "epsilon", + clip_sample_range: float = 1.0, + timestep_spacing: str = "leading", + rescale_betas_zero_snr: bool = False, + **kwargs, + ): + if kwargs.get("set_alpha_to_zero", None) is not None: + deprecation_message = ( + "The `set_alpha_to_zero` argument is deprecated. Please use `set_alpha_to_one` instead." + ) + deprecate("set_alpha_to_zero", "1.0.0", deprecation_message, standard_warn=False) + set_alpha_to_one = kwargs["set_alpha_to_zero"] + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + # Rescale for zero SNR + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + # At every step in inverted ddim, we are looking into the next alphas_cumprod + # For the initial step, there is no current alphas_cumprod, and the index is out of bounds + # `set_alpha_to_one` decides whether we set this parameter simply to one + # in this case, self.step() just output the predicted noise + # or whether we use the initial alpha used in training the diffusion model. + self.initial_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0] + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps).copy().astype(np.int64)) + + # Copied from diffusers.schedulers.scheduling_ddim.DDIMScheduler.scale_model_input + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + """ + + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + self.num_inference_steps = num_inference_steps + + # "leading" and "trailing" corresponds to annotation of Table 1. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round().copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.round(np.arange(self.config.num_train_timesteps, 0, -step_ratio)[::-1]).astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'leading' or 'trailing'." + ) + + self.timesteps = torch.from_numpy(timesteps).to(device) + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[DDIMSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + eta (`float`): + The weight of noise for added noise in diffusion step. + use_clipped_model_output (`bool`, defaults to `False`): + If `True`, computes "corrected" `model_output` from the clipped predicted original sample. Necessary + because predicted original sample is clipped to [-1, 1] when `self.config.clip_sample` is `True`. If no + clipping has happened, "corrected" `model_output` would coincide with the one provided as input and + `use_clipped_model_output` has no effect. + variance_noise (`torch.FloatTensor`): + Alternative to generating noise with `generator` by directly providing the noise for the variance + itself. Useful for methods such as [`CycleDiffusion`]. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_ddim_inverse.DDIMInverseSchedulerOutput`] or + `tuple`. + + Returns: + [`~schedulers.scheduling_ddim_inverse.DDIMInverseSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_ddim_inverse.DDIMInverseSchedulerOutput`] is + returned, otherwise a tuple is returned where the first element is the sample tensor. + + """ + # 1. get previous step value (=t+1) + prev_timestep = timestep + timestep = min( + timestep - self.config.num_train_timesteps // self.num_inference_steps, self.config.num_train_timesteps - 1 + ) + + # 2. compute alphas, betas + # change original implementation to exactly match noise levels for analogous forward process + alpha_prod_t = self.alphas_cumprod[timestep] if timestep >= 0 else self.initial_alpha_cumprod + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] + + beta_prod_t = 1 - alpha_prod_t + + # 3. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + pred_epsilon = model_output + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5) + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + pred_epsilon = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction`" + ) + + # 4. Clip or threshold "predicted x_0" + if self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 5. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_sample_direction = (1 - alpha_prod_t_prev) ** (0.5) * pred_epsilon + + # 6. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + prev_sample = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction + + if not return_dict: + return (prev_sample, pred_original_sample) + return DDIMSchedulerOutput(prev_sample=prev_sample, pred_original_sample=pred_original_sample) + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_parallel.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_parallel.py new file mode 100755 index 0000000..225ed73 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddim_parallel.py @@ -0,0 +1,645 @@ +# Copyright 2024 ParaDiGMS authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion +# and https://github.com/hojonathanho/diffusion + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput +class DDIMParallelSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.FloatTensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.FloatTensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class DDIMParallelScheduler(SchedulerMixin, ConfigMixin): + """ + Denoising diffusion implicit models is a scheduler that extends the denoising procedure introduced in denoising + diffusion probabilistic models (DDPMs) with non-Markovian guidance. + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + For more details, see the original paper: https://arxiv.org/abs/2010.02502 + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + beta_start (`float`): the starting `beta` value of inference. + beta_end (`float`): the final `beta` value. + beta_schedule (`str`): + the beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, optional): + option to pass an array of betas directly to the constructor to bypass `beta_start`, `beta_end` etc. + clip_sample (`bool`, default `True`): + option to clip predicted sample for numerical stability. + clip_sample_range (`float`, default `1.0`): + the maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + set_alpha_to_one (`bool`, default `True`): + each diffusion step uses the value of alphas product at that step and at the previous one. For the final + step there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`, + otherwise it uses the value of alpha at step 0. + steps_offset (`int`, default `0`): + An offset added to the inference steps, as required by some model families. + prediction_type (`str`, default `epsilon`, optional): + prediction type of the scheduler function, one of `epsilon` (predicting the noise of the diffusion + process), `sample` (directly predicting the noisy sample`) or `v_prediction` (see section 2.4 + https://imagen.research.google/video/paper.pdf) + thresholding (`bool`, default `False`): + whether to use the "dynamic thresholding" method (introduced by Imagen, https://arxiv.org/abs/2205.11487). + Note that the thresholding method is unsuitable for latent-space diffusion models (such as + stable-diffusion). + dynamic_thresholding_ratio (`float`, default `0.995`): + the ratio for the dynamic thresholding method. Default is `0.995`, the same as Imagen + (https://arxiv.org/abs/2205.11487). Valid only when `thresholding=True`. + sample_max_value (`float`, default `1.0`): + the threshold value for dynamic thresholding. Valid only when `thresholding=True`. + timestep_spacing (`str`, default `"leading"`): + The way the timesteps should be scaled. Refer to Table 2. of [Common Diffusion Noise Schedules and Sample + Steps are Flawed](https://arxiv.org/abs/2305.08891) for more information. + rescale_betas_zero_snr (`bool`, default `False`): + whether to rescale the betas to have zero terminal SNR (proposed by https://arxiv.org/pdf/2305.08891.pdf). + This can enable the model to generate very bright and dark samples instead of limiting it to samples with + medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + _is_ode_scheduler = True + + @register_to_config + # Copied from diffusers.schedulers.scheduling_ddim.DDIMScheduler.__init__ + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + clip_sample: bool = True, + set_alpha_to_one: bool = True, + steps_offset: int = 0, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + clip_sample_range: float = 1.0, + sample_max_value: float = 1.0, + timestep_spacing: str = "leading", + rescale_betas_zero_snr: bool = False, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + # Rescale for zero SNR + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + # At every step in ddim, we are looking into the previous alphas_cumprod + # For the final step, there is no previous alphas_cumprod because we are already at 0 + # `set_alpha_to_one` decides whether we set this parameter simply to one or + # whether we use the final alpha of the "non-previous" one. + self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0] + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy().astype(np.int64)) + + # Copied from diffusers.schedulers.scheduling_ddim.DDIMScheduler.scale_model_input + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def _get_variance(self, timestep, prev_timestep=None): + if prev_timestep is None: + prev_timestep = timestep - self.config.num_train_timesteps // self.num_inference_steps + + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + variance = (beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev) + + return variance + + def _batch_get_variance(self, t, prev_t): + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[torch.clip(prev_t, min=0)] + alpha_prod_t_prev[prev_t < 0] = torch.tensor(1.0) + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + variance = (beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev) + + return variance + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_ddim.DDIMScheduler.set_timesteps + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + """ + + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + self.num_inference_steps = num_inference_steps + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps) + .round()[::-1] + .copy() + .astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.round(np.arange(self.config.num_train_timesteps, 0, -step_ratio)).astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'leading' or 'trailing'." + ) + + self.timesteps = torch.from_numpy(timesteps).to(device) + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + eta: float = 0.0, + use_clipped_model_output: bool = False, + generator=None, + variance_noise: Optional[torch.FloatTensor] = None, + return_dict: bool = True, + ) -> Union[DDIMParallelSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + current instance of sample being created by diffusion process. + eta (`float`): weight of noise for added noise in diffusion step. + use_clipped_model_output (`bool`): if `True`, compute "corrected" `model_output` from the clipped + predicted original sample. Necessary because predicted original sample is clipped to [-1, 1] when + `self.config.clip_sample` is `True`. If no clipping has happened, "corrected" `model_output` would + coincide with the one provided as input and `use_clipped_model_output` will have not effect. + generator: random number generator. + variance_noise (`torch.FloatTensor`): instead of generating noise for the variance using `generator`, we + can directly provide the noise for the variance itself. This is useful for methods such as + CycleDiffusion. (https://arxiv.org/abs/2210.05559) + return_dict (`bool`): option for returning tuple rather than DDIMParallelSchedulerOutput class + + Returns: + [`~schedulers.scheduling_utils.DDIMParallelSchedulerOutput`] or `tuple`: + [`~schedulers.scheduling_utils.DDIMParallelSchedulerOutput`] if `return_dict` is True, otherwise a `tuple`. + When returning a tuple, the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + # See formulas (12) and (16) of DDIM paper https://arxiv.org/pdf/2010.02502.pdf + # Ideally, read DDIM paper in-detail understanding + + # Notation ( -> + # - pred_noise_t -> e_theta(x_t, t) + # - pred_original_sample -> f_theta(x_t, t) or x_0 + # - std_dev_t -> sigma_t + # - eta -> η + # - pred_sample_direction -> "direction pointing to x_t" + # - pred_prev_sample -> "x_t-1" + + # 1. get previous step value (=t-1) + prev_timestep = timestep - self.config.num_train_timesteps // self.num_inference_steps + + # 2. compute alphas, betas + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + + beta_prod_t = 1 - alpha_prod_t + + # 3. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + pred_epsilon = model_output + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5) + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + pred_epsilon = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction`" + ) + + # 4. Clip or threshold "predicted x_0" + if self.config.thresholding: + pred_original_sample = self._threshold_sample(pred_original_sample) + elif self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 5. compute variance: "sigma_t(η)" -> see formula (16) + # σ_t = sqrt((1 − α_t−1)/(1 − α_t)) * sqrt(1 − α_t/α_t−1) + variance = self._get_variance(timestep, prev_timestep) + std_dev_t = eta * variance ** (0.5) + + if use_clipped_model_output: + # the pred_epsilon is always re-derived from the clipped x_0 in Glide + pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5) + + # 6. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * pred_epsilon + + # 7. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + prev_sample = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction + + if eta > 0: + if variance_noise is not None and generator is not None: + raise ValueError( + "Cannot pass both generator and variance_noise. Please make sure that either `generator` or" + " `variance_noise` stays `None`." + ) + + if variance_noise is None: + variance_noise = randn_tensor( + model_output.shape, generator=generator, device=model_output.device, dtype=model_output.dtype + ) + variance = std_dev_t * variance_noise + + prev_sample = prev_sample + variance + + if not return_dict: + return (prev_sample,) + + return DDIMParallelSchedulerOutput(prev_sample=prev_sample, pred_original_sample=pred_original_sample) + + def batch_step_no_noise( + self, + model_output: torch.FloatTensor, + timesteps: List[int], + sample: torch.FloatTensor, + eta: float = 0.0, + use_clipped_model_output: bool = False, + ) -> torch.FloatTensor: + """ + Batched version of the `step` function, to be able to reverse the SDE for multiple samples/timesteps at once. + Also, does not add any noise to the predicted sample, which is necessary for parallel sampling where the noise + is pre-sampled by the pipeline. + + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): direct output from learned diffusion model. + timesteps (`List[int]`): + current discrete timesteps in the diffusion chain. This is now a list of integers. + sample (`torch.FloatTensor`): + current instance of sample being created by diffusion process. + eta (`float`): weight of noise for added noise in diffusion step. + use_clipped_model_output (`bool`): if `True`, compute "corrected" `model_output` from the clipped + predicted original sample. Necessary because predicted original sample is clipped to [-1, 1] when + `self.config.clip_sample` is `True`. If no clipping has happened, "corrected" `model_output` would + coincide with the one provided as input and `use_clipped_model_output` will have not effect. + + Returns: + `torch.FloatTensor`: sample tensor at previous timestep. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + assert eta == 0.0 + + # See formulas (12) and (16) of DDIM paper https://arxiv.org/pdf/2010.02502.pdf + # Ideally, read DDIM paper in-detail understanding + + # Notation ( -> + # - pred_noise_t -> e_theta(x_t, t) + # - pred_original_sample -> f_theta(x_t, t) or x_0 + # - std_dev_t -> sigma_t + # - eta -> η + # - pred_sample_direction -> "direction pointing to x_t" + # - pred_prev_sample -> "x_t-1" + + # 1. get previous step value (=t-1) + t = timesteps + prev_t = t - self.config.num_train_timesteps // self.num_inference_steps + + t = t.view(-1, *([1] * (model_output.ndim - 1))) + prev_t = prev_t.view(-1, *([1] * (model_output.ndim - 1))) + + # 1. compute alphas, betas + self.alphas_cumprod = self.alphas_cumprod.to(model_output.device) + self.final_alpha_cumprod = self.final_alpha_cumprod.to(model_output.device) + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[torch.clip(prev_t, min=0)] + alpha_prod_t_prev[prev_t < 0] = torch.tensor(1.0) + + beta_prod_t = 1 - alpha_prod_t + + # 3. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + pred_epsilon = model_output + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5) + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + pred_epsilon = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction`" + ) + + # 4. Clip or threshold "predicted x_0" + if self.config.thresholding: + pred_original_sample = self._threshold_sample(pred_original_sample) + elif self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 5. compute variance: "sigma_t(η)" -> see formula (16) + # σ_t = sqrt((1 − α_t−1)/(1 − α_t)) * sqrt(1 − α_t/α_t−1) + variance = self._batch_get_variance(t, prev_t).to(model_output.device).view(*alpha_prod_t_prev.shape) + std_dev_t = eta * variance ** (0.5) + + if use_clipped_model_output: + # the pred_epsilon is always re-derived from the clipped x_0 in Glide + pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5) + + # 6. compute "direction pointing to x_t" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** (0.5) * pred_epsilon + + # 7. compute x_t without "random noise" of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + prev_sample = alpha_prod_t_prev ** (0.5) * pred_original_sample + pred_sample_direction + + return prev_sample + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + # Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement + # for the subsequent add_noise calls + self.alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.get_velocity + def get_velocity( + self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as sample + self.alphas_cumprod = self.alphas_cumprod.to(device=sample.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=sample.dtype) + timesteps = timesteps.to(sample.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(sample.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample + return velocity + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm.py new file mode 100755 index 0000000..e1f55a2 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm.py @@ -0,0 +1,562 @@ +# Copyright 2024 UC Berkeley Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin + + +@dataclass +class DDPMSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.FloatTensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.FloatTensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class DDPMScheduler(SchedulerMixin, ConfigMixin): + """ + `DDPMScheduler` explores the connections between denoising score matching and Langevin dynamics sampling. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + An array of betas to pass directly to the constructor without using `beta_start` and `beta_end`. + variance_type (`str`, defaults to `"fixed_small"`): + Clip the variance when adding noise to the denoised sample. Choose from `fixed_small`, `fixed_small_log`, + `fixed_large`, `fixed_large_log`, `learned` or `learned_range`. + clip_sample (`bool`, defaults to `True`): + Clip the predicted sample for numerical stability. + clip_sample_range (`float`, defaults to 1.0): + The maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True`. + timestep_spacing (`str`, defaults to `"leading"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + variance_type: str = "fixed_small", + clip_sample: bool = True, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + clip_sample_range: float = 1.0, + sample_max_value: float = 1.0, + timestep_spacing: str = "leading", + steps_offset: int = 0, + rescale_betas_zero_snr: int = False, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + elif beta_schedule == "sigmoid": + # GeoDiff sigmoid schedule + betas = torch.linspace(-6, 6, num_train_timesteps) + self.betas = torch.sigmoid(betas) * (beta_end - beta_start) + beta_start + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + # Rescale for zero SNR + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + self.one = torch.tensor(1.0) + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.custom_timesteps = False + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy()) + + self.variance_type = variance_type + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def set_timesteps( + self, + num_inference_steps: Optional[int] = None, + device: Union[str, torch.device] = None, + timesteps: Optional[List[int]] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of equal spacing between timesteps is used. If `timesteps` is passed, + `num_inference_steps` must be `None`. + + """ + if num_inference_steps is not None and timesteps is not None: + raise ValueError("Can only pass one of `num_inference_steps` or `custom_timesteps`.") + + if timesteps is not None: + for i in range(1, len(timesteps)): + if timesteps[i] >= timesteps[i - 1]: + raise ValueError("`custom_timesteps` must be in descending order.") + + if timesteps[0] >= self.config.num_train_timesteps: + raise ValueError( + f"`timesteps` must start before `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps}." + ) + + timesteps = np.array(timesteps, dtype=np.int64) + self.custom_timesteps = True + else: + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + self.num_inference_steps = num_inference_steps + self.custom_timesteps = False + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps) + .round()[::-1] + .copy() + .astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.round(np.arange(self.config.num_train_timesteps, 0, -step_ratio)).astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + self.timesteps = torch.from_numpy(timesteps).to(device) + + def _get_variance(self, t, predicted_variance=None, variance_type=None): + prev_t = self.previous_timestep(t) + + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[prev_t] if prev_t >= 0 else self.one + current_beta_t = 1 - alpha_prod_t / alpha_prod_t_prev + + # For t > 0, compute predicted variance βt (see formula (6) and (7) from https://arxiv.org/pdf/2006.11239.pdf) + # and sample from it to get previous sample + # x_{t-1} ~ N(pred_prev_sample, variance) == add variance to pred_sample + variance = (1 - alpha_prod_t_prev) / (1 - alpha_prod_t) * current_beta_t + + # we always take the log of variance, so clamp it to ensure it's not 0 + variance = torch.clamp(variance, min=1e-20) + + if variance_type is None: + variance_type = self.config.variance_type + + # hacks - were probably added for training stability + if variance_type == "fixed_small": + variance = variance + # for rl-diffuser https://arxiv.org/abs/2205.09991 + elif variance_type == "fixed_small_log": + variance = torch.log(variance) + variance = torch.exp(0.5 * variance) + elif variance_type == "fixed_large": + variance = current_beta_t + elif variance_type == "fixed_large_log": + # Glide max_log + variance = torch.log(current_beta_t) + elif variance_type == "learned": + return predicted_variance + elif variance_type == "learned_range": + min_log = torch.log(variance) + max_log = torch.log(current_beta_t) + frac = (predicted_variance + 1) / 2 + variance = frac * max_log + (1 - frac) * min_log + + return variance + + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator=None, + return_dict: bool = True, + ) -> Union[DDPMSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_ddpm.DDPMSchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_ddpm.DDPMSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_ddpm.DDPMSchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + t = timestep + + prev_t = self.previous_timestep(t) + + if model_output.shape[1] == sample.shape[1] * 2 and self.variance_type in ["learned", "learned_range"]: + model_output, predicted_variance = torch.split(model_output, sample.shape[1], dim=1) + else: + predicted_variance = None + + # 1. compute alphas, betas + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[prev_t] if prev_t >= 0 else self.one + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + current_alpha_t = alpha_prod_t / alpha_prod_t_prev + current_beta_t = 1 - current_alpha_t + + # 2. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or" + " `v_prediction` for the DDPMScheduler." + ) + + # 3. Clip or threshold "predicted x_0" + if self.config.thresholding: + pred_original_sample = self._threshold_sample(pred_original_sample) + elif self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 4. Compute coefficients for pred_original_sample x_0 and current sample x_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_original_sample_coeff = (alpha_prod_t_prev ** (0.5) * current_beta_t) / beta_prod_t + current_sample_coeff = current_alpha_t ** (0.5) * beta_prod_t_prev / beta_prod_t + + # 5. Compute predicted previous sample µ_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_prev_sample = pred_original_sample_coeff * pred_original_sample + current_sample_coeff * sample + + # 6. Add noise + variance = 0 + if t > 0: + device = model_output.device + variance_noise = randn_tensor( + model_output.shape, generator=generator, device=device, dtype=model_output.dtype + ) + if self.variance_type == "fixed_small_log": + variance = self._get_variance(t, predicted_variance=predicted_variance) * variance_noise + elif self.variance_type == "learned_range": + variance = self._get_variance(t, predicted_variance=predicted_variance) + variance = torch.exp(0.5 * variance) * variance_noise + else: + variance = (self._get_variance(t, predicted_variance=predicted_variance) ** 0.5) * variance_noise + + pred_prev_sample = pred_prev_sample + variance + + if not return_dict: + return (pred_prev_sample,) + + return DDPMSchedulerOutput(prev_sample=pred_prev_sample, pred_original_sample=pred_original_sample) + + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + # Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement + # for the subsequent add_noise calls + self.alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + def get_velocity( + self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as sample + self.alphas_cumprod = self.alphas_cumprod.to(device=sample.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=sample.dtype) + timesteps = timesteps.to(sample.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(sample.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample + return velocity + + def __len__(self): + return self.config.num_train_timesteps + + def previous_timestep(self, timestep): + if self.custom_timesteps: + index = (self.timesteps == timestep).nonzero(as_tuple=True)[0][0] + if index == self.timesteps.shape[0] - 1: + prev_t = torch.tensor(-1) + else: + prev_t = self.timesteps[index + 1] + else: + num_inference_steps = ( + self.num_inference_steps if self.num_inference_steps else self.config.num_train_timesteps + ) + prev_t = timestep - self.config.num_train_timesteps // num_inference_steps + + return prev_t diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_flax.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_flax.py new file mode 100755 index 0000000..6bdfa5e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_flax.py @@ -0,0 +1,299 @@ +# Copyright 2024 UC Berkeley Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import flax +import jax +import jax.numpy as jnp + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils_flax import ( + CommonSchedulerState, + FlaxKarrasDiffusionSchedulers, + FlaxSchedulerMixin, + FlaxSchedulerOutput, + add_noise_common, + get_velocity_common, +) + + +@flax.struct.dataclass +class DDPMSchedulerState: + common: CommonSchedulerState + + # setable values + init_noise_sigma: jnp.ndarray + timesteps: jnp.ndarray + num_inference_steps: Optional[int] = None + + @classmethod + def create(cls, common: CommonSchedulerState, init_noise_sigma: jnp.ndarray, timesteps: jnp.ndarray): + return cls(common=common, init_noise_sigma=init_noise_sigma, timesteps=timesteps) + + +@dataclass +class FlaxDDPMSchedulerOutput(FlaxSchedulerOutput): + state: DDPMSchedulerState + + +class FlaxDDPMScheduler(FlaxSchedulerMixin, ConfigMixin): + """ + Denoising diffusion probabilistic models (DDPMs) explores the connections between denoising score matching and + Langevin dynamics sampling. + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + For more details, see the original paper: https://arxiv.org/abs/2006.11239 + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + beta_start (`float`): the starting `beta` value of inference. + beta_end (`float`): the final `beta` value. + beta_schedule (`str`): + the beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, optional): + option to pass an array of betas directly to the constructor to bypass `beta_start`, `beta_end` etc. + variance_type (`str`): + options to clip the variance used when adding noise to the denoised sample. Choose from `fixed_small`, + `fixed_small_log`, `fixed_large`, `fixed_large_log`, `learned` or `learned_range`. + clip_sample (`bool`, default `True`): + option to clip predicted sample between -1 and 1 for numerical stability. + prediction_type (`str`, default `epsilon`): + indicates whether the model predicts the noise (epsilon), or the samples. One of `epsilon`, `sample`. + `v-prediction` is not supported for this scheduler. + dtype (`jnp.dtype`, *optional*, defaults to `jnp.float32`): + the `dtype` used for params and computation. + """ + + _compatibles = [e.name for e in FlaxKarrasDiffusionSchedulers] + + dtype: jnp.dtype + + @property + def has_state(self): + return True + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[jnp.ndarray] = None, + variance_type: str = "fixed_small", + clip_sample: bool = True, + prediction_type: str = "epsilon", + dtype: jnp.dtype = jnp.float32, + ): + self.dtype = dtype + + def create_state(self, common: Optional[CommonSchedulerState] = None) -> DDPMSchedulerState: + if common is None: + common = CommonSchedulerState.create(self) + + # standard deviation of the initial noise distribution + init_noise_sigma = jnp.array(1.0, dtype=self.dtype) + + timesteps = jnp.arange(0, self.config.num_train_timesteps).round()[::-1] + + return DDPMSchedulerState.create( + common=common, + init_noise_sigma=init_noise_sigma, + timesteps=timesteps, + ) + + def scale_model_input( + self, state: DDPMSchedulerState, sample: jnp.ndarray, timestep: Optional[int] = None + ) -> jnp.ndarray: + """ + Args: + state (`PNDMSchedulerState`): the `FlaxPNDMScheduler` state data class instance. + sample (`jnp.ndarray`): input sample + timestep (`int`, optional): current timestep + + Returns: + `jnp.ndarray`: scaled input sample + """ + return sample + + def set_timesteps( + self, state: DDPMSchedulerState, num_inference_steps: int, shape: Tuple = () + ) -> DDPMSchedulerState: + """ + Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + state (`DDIMSchedulerState`): + the `FlaxDDPMScheduler` state data class instance. + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + """ + + step_ratio = self.config.num_train_timesteps // num_inference_steps + # creates integer timesteps by multiplying by ratio + # rounding to avoid issues when num_inference_step is power of 3 + timesteps = (jnp.arange(0, num_inference_steps) * step_ratio).round()[::-1] + + return state.replace( + num_inference_steps=num_inference_steps, + timesteps=timesteps, + ) + + def _get_variance(self, state: DDPMSchedulerState, t, predicted_variance=None, variance_type=None): + alpha_prod_t = state.common.alphas_cumprod[t] + alpha_prod_t_prev = jnp.where(t > 0, state.common.alphas_cumprod[t - 1], jnp.array(1.0, dtype=self.dtype)) + + # For t > 0, compute predicted variance βt (see formula (6) and (7) from https://arxiv.org/pdf/2006.11239.pdf) + # and sample from it to get previous sample + # x_{t-1} ~ N(pred_prev_sample, variance) == add variance to pred_sample + variance = (1 - alpha_prod_t_prev) / (1 - alpha_prod_t) * state.common.betas[t] + + if variance_type is None: + variance_type = self.config.variance_type + + # hacks - were probably added for training stability + if variance_type == "fixed_small": + variance = jnp.clip(variance, a_min=1e-20) + # for rl-diffuser https://arxiv.org/abs/2205.09991 + elif variance_type == "fixed_small_log": + variance = jnp.log(jnp.clip(variance, a_min=1e-20)) + elif variance_type == "fixed_large": + variance = state.common.betas[t] + elif variance_type == "fixed_large_log": + # Glide max_log + variance = jnp.log(state.common.betas[t]) + elif variance_type == "learned": + return predicted_variance + elif variance_type == "learned_range": + min_log = variance + max_log = state.common.betas[t] + frac = (predicted_variance + 1) / 2 + variance = frac * max_log + (1 - frac) * min_log + + return variance + + def step( + self, + state: DDPMSchedulerState, + model_output: jnp.ndarray, + timestep: int, + sample: jnp.ndarray, + key: Optional[jax.Array] = None, + return_dict: bool = True, + ) -> Union[FlaxDDPMSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + state (`DDPMSchedulerState`): the `FlaxDDPMScheduler` state data class instance. + model_output (`jnp.ndarray`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + key (`jax.Array`): a PRNG key. + return_dict (`bool`): option for returning tuple rather than FlaxDDPMSchedulerOutput class + + Returns: + [`FlaxDDPMSchedulerOutput`] or `tuple`: [`FlaxDDPMSchedulerOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is the sample tensor. + + """ + t = timestep + + if key is None: + key = jax.random.PRNGKey(0) + + if model_output.shape[1] == sample.shape[1] * 2 and self.config.variance_type in ["learned", "learned_range"]: + model_output, predicted_variance = jnp.split(model_output, sample.shape[1], axis=1) + else: + predicted_variance = None + + # 1. compute alphas, betas + alpha_prod_t = state.common.alphas_cumprod[t] + alpha_prod_t_prev = jnp.where(t > 0, state.common.alphas_cumprod[t - 1], jnp.array(1.0, dtype=self.dtype)) + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + # 2. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` " + " for the FlaxDDPMScheduler." + ) + + # 3. Clip "predicted x_0" + if self.config.clip_sample: + pred_original_sample = jnp.clip(pred_original_sample, -1, 1) + + # 4. Compute coefficients for pred_original_sample x_0 and current sample x_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_original_sample_coeff = (alpha_prod_t_prev ** (0.5) * state.common.betas[t]) / beta_prod_t + current_sample_coeff = state.common.alphas[t] ** (0.5) * beta_prod_t_prev / beta_prod_t + + # 5. Compute predicted previous sample µ_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_prev_sample = pred_original_sample_coeff * pred_original_sample + current_sample_coeff * sample + + # 6. Add noise + def random_variance(): + split_key = jax.random.split(key, num=1) + noise = jax.random.normal(split_key, shape=model_output.shape, dtype=self.dtype) + return (self._get_variance(state, t, predicted_variance=predicted_variance) ** 0.5) * noise + + variance = jnp.where(t > 0, random_variance(), jnp.zeros(model_output.shape, dtype=self.dtype)) + + pred_prev_sample = pred_prev_sample + variance + + if not return_dict: + return (pred_prev_sample, state) + + return FlaxDDPMSchedulerOutput(prev_sample=pred_prev_sample, state=state) + + def add_noise( + self, + state: DDPMSchedulerState, + original_samples: jnp.ndarray, + noise: jnp.ndarray, + timesteps: jnp.ndarray, + ) -> jnp.ndarray: + return add_noise_common(state.common, original_samples, noise, timesteps) + + def get_velocity( + self, + state: DDPMSchedulerState, + sample: jnp.ndarray, + noise: jnp.ndarray, + timesteps: jnp.ndarray, + ) -> jnp.ndarray: + return get_velocity_common(state.common, sample, noise, timesteps) + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_parallel.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_parallel.py new file mode 100755 index 0000000..ec4fbd4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_parallel.py @@ -0,0 +1,653 @@ +# Copyright 2024 ParaDiGMS authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput +class DDPMParallelSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.FloatTensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.FloatTensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class DDPMParallelScheduler(SchedulerMixin, ConfigMixin): + """ + Denoising diffusion probabilistic models (DDPMs) explores the connections between denoising score matching and + Langevin dynamics sampling. + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + For more details, see the original paper: https://arxiv.org/abs/2006.11239 + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + beta_start (`float`): the starting `beta` value of inference. + beta_end (`float`): the final `beta` value. + beta_schedule (`str`): + the beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, `squaredcos_cap_v2` or `sigmoid`. + trained_betas (`np.ndarray`, optional): + option to pass an array of betas directly to the constructor to bypass `beta_start`, `beta_end` etc. + variance_type (`str`): + options to clip the variance used when adding noise to the denoised sample. Choose from `fixed_small`, + `fixed_small_log`, `fixed_large`, `fixed_large_log`, `learned` or `learned_range`. + clip_sample (`bool`, default `True`): + option to clip predicted sample for numerical stability. + clip_sample_range (`float`, default `1.0`): + the maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + prediction_type (`str`, default `epsilon`, optional): + prediction type of the scheduler function, one of `epsilon` (predicting the noise of the diffusion + process), `sample` (directly predicting the noisy sample`) or `v_prediction` (see section 2.4 + https://imagen.research.google/video/paper.pdf) + thresholding (`bool`, default `False`): + whether to use the "dynamic thresholding" method (introduced by Imagen, https://arxiv.org/abs/2205.11487). + Note that the thresholding method is unsuitable for latent-space diffusion models (such as + stable-diffusion). + dynamic_thresholding_ratio (`float`, default `0.995`): + the ratio for the dynamic thresholding method. Default is `0.995`, the same as Imagen + (https://arxiv.org/abs/2205.11487). Valid only when `thresholding=True`. + sample_max_value (`float`, default `1.0`): + the threshold value for dynamic thresholding. Valid only when `thresholding=True`. + timestep_spacing (`str`, default `"leading"`): + The way the timesteps should be scaled. Refer to Table 2. of [Common Diffusion Noise Schedules and Sample + Steps are Flawed](https://arxiv.org/abs/2305.08891) for more information. + steps_offset (`int`, default `0`): + An offset added to the inference steps, as required by some model families. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + _is_ode_scheduler = False + + @register_to_config + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.__init__ + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + variance_type: str = "fixed_small", + clip_sample: bool = True, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + clip_sample_range: float = 1.0, + sample_max_value: float = 1.0, + timestep_spacing: str = "leading", + steps_offset: int = 0, + rescale_betas_zero_snr: int = False, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + elif beta_schedule == "sigmoid": + # GeoDiff sigmoid schedule + betas = torch.linspace(-6, 6, num_train_timesteps) + self.betas = torch.sigmoid(betas) * (beta_end - beta_start) + beta_start + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + # Rescale for zero SNR + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + self.one = torch.tensor(1.0) + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.custom_timesteps = False + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy()) + + self.variance_type = variance_type + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.scale_model_input + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.set_timesteps + def set_timesteps( + self, + num_inference_steps: Optional[int] = None, + device: Union[str, torch.device] = None, + timesteps: Optional[List[int]] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of equal spacing between timesteps is used. If `timesteps` is passed, + `num_inference_steps` must be `None`. + + """ + if num_inference_steps is not None and timesteps is not None: + raise ValueError("Can only pass one of `num_inference_steps` or `custom_timesteps`.") + + if timesteps is not None: + for i in range(1, len(timesteps)): + if timesteps[i] >= timesteps[i - 1]: + raise ValueError("`custom_timesteps` must be in descending order.") + + if timesteps[0] >= self.config.num_train_timesteps: + raise ValueError( + f"`timesteps` must start before `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps}." + ) + + timesteps = np.array(timesteps, dtype=np.int64) + self.custom_timesteps = True + else: + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + self.num_inference_steps = num_inference_steps + self.custom_timesteps = False + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps) + .round()[::-1] + .copy() + .astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.round(np.arange(self.config.num_train_timesteps, 0, -step_ratio)).astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + self.timesteps = torch.from_numpy(timesteps).to(device) + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._get_variance + def _get_variance(self, t, predicted_variance=None, variance_type=None): + prev_t = self.previous_timestep(t) + + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[prev_t] if prev_t >= 0 else self.one + current_beta_t = 1 - alpha_prod_t / alpha_prod_t_prev + + # For t > 0, compute predicted variance βt (see formula (6) and (7) from https://arxiv.org/pdf/2006.11239.pdf) + # and sample from it to get previous sample + # x_{t-1} ~ N(pred_prev_sample, variance) == add variance to pred_sample + variance = (1 - alpha_prod_t_prev) / (1 - alpha_prod_t) * current_beta_t + + # we always take the log of variance, so clamp it to ensure it's not 0 + variance = torch.clamp(variance, min=1e-20) + + if variance_type is None: + variance_type = self.config.variance_type + + # hacks - were probably added for training stability + if variance_type == "fixed_small": + variance = variance + # for rl-diffuser https://arxiv.org/abs/2205.09991 + elif variance_type == "fixed_small_log": + variance = torch.log(variance) + variance = torch.exp(0.5 * variance) + elif variance_type == "fixed_large": + variance = current_beta_t + elif variance_type == "fixed_large_log": + # Glide max_log + variance = torch.log(current_beta_t) + elif variance_type == "learned": + return predicted_variance + elif variance_type == "learned_range": + min_log = torch.log(variance) + max_log = torch.log(current_beta_t) + frac = (predicted_variance + 1) / 2 + variance = frac * max_log + (1 - frac) * min_log + + return variance + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator=None, + return_dict: bool = True, + ) -> Union[DDPMParallelSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + current instance of sample being created by diffusion process. + generator: random number generator. + return_dict (`bool`): option for returning tuple rather than DDPMParallelSchedulerOutput class + + Returns: + [`~schedulers.scheduling_utils.DDPMParallelSchedulerOutput`] or `tuple`: + [`~schedulers.scheduling_utils.DDPMParallelSchedulerOutput`] if `return_dict` is True, otherwise a `tuple`. + When returning a tuple, the first element is the sample tensor. + + """ + t = timestep + + prev_t = self.previous_timestep(t) + + if model_output.shape[1] == sample.shape[1] * 2 and self.variance_type in ["learned", "learned_range"]: + model_output, predicted_variance = torch.split(model_output, sample.shape[1], dim=1) + else: + predicted_variance = None + + # 1. compute alphas, betas + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[prev_t] if prev_t >= 0 else self.one + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + current_alpha_t = alpha_prod_t / alpha_prod_t_prev + current_beta_t = 1 - current_alpha_t + + # 2. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or" + " `v_prediction` for the DDPMScheduler." + ) + + # 3. Clip or threshold "predicted x_0" + if self.config.thresholding: + pred_original_sample = self._threshold_sample(pred_original_sample) + elif self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 4. Compute coefficients for pred_original_sample x_0 and current sample x_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_original_sample_coeff = (alpha_prod_t_prev ** (0.5) * current_beta_t) / beta_prod_t + current_sample_coeff = current_alpha_t ** (0.5) * beta_prod_t_prev / beta_prod_t + + # 5. Compute predicted previous sample µ_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_prev_sample = pred_original_sample_coeff * pred_original_sample + current_sample_coeff * sample + + # 6. Add noise + variance = 0 + if t > 0: + device = model_output.device + variance_noise = randn_tensor( + model_output.shape, generator=generator, device=device, dtype=model_output.dtype + ) + if self.variance_type == "fixed_small_log": + variance = self._get_variance(t, predicted_variance=predicted_variance) * variance_noise + elif self.variance_type == "learned_range": + variance = self._get_variance(t, predicted_variance=predicted_variance) + variance = torch.exp(0.5 * variance) * variance_noise + else: + variance = (self._get_variance(t, predicted_variance=predicted_variance) ** 0.5) * variance_noise + + pred_prev_sample = pred_prev_sample + variance + + if not return_dict: + return (pred_prev_sample,) + + return DDPMParallelSchedulerOutput(prev_sample=pred_prev_sample, pred_original_sample=pred_original_sample) + + def batch_step_no_noise( + self, + model_output: torch.FloatTensor, + timesteps: List[int], + sample: torch.FloatTensor, + ) -> torch.FloatTensor: + """ + Batched version of the `step` function, to be able to reverse the SDE for multiple samples/timesteps at once. + Also, does not add any noise to the predicted sample, which is necessary for parallel sampling where the noise + is pre-sampled by the pipeline. + + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): direct output from learned diffusion model. + timesteps (`List[int]`): + current discrete timesteps in the diffusion chain. This is now a list of integers. + sample (`torch.FloatTensor`): + current instance of sample being created by diffusion process. + + Returns: + `torch.FloatTensor`: sample tensor at previous timestep. + """ + t = timesteps + num_inference_steps = self.num_inference_steps if self.num_inference_steps else self.config.num_train_timesteps + prev_t = t - self.config.num_train_timesteps // num_inference_steps + + t = t.view(-1, *([1] * (model_output.ndim - 1))) + prev_t = prev_t.view(-1, *([1] * (model_output.ndim - 1))) + + if model_output.shape[1] == sample.shape[1] * 2 and self.variance_type in ["learned", "learned_range"]: + model_output, predicted_variance = torch.split(model_output, sample.shape[1], dim=1) + else: + pass + + # 1. compute alphas, betas + self.alphas_cumprod = self.alphas_cumprod.to(model_output.device) + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[torch.clip(prev_t, min=0)] + alpha_prod_t_prev[prev_t < 0] = torch.tensor(1.0) + + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + current_alpha_t = alpha_prod_t / alpha_prod_t_prev + current_beta_t = 1 - current_alpha_t + + # 2. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + elif self.config.prediction_type == "v_prediction": + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or" + " `v_prediction` for the DDPMParallelScheduler." + ) + + # 3. Clip or threshold "predicted x_0" + if self.config.thresholding: + pred_original_sample = self._threshold_sample(pred_original_sample) + elif self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 4. Compute coefficients for pred_original_sample x_0 and current sample x_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_original_sample_coeff = (alpha_prod_t_prev ** (0.5) * current_beta_t) / beta_prod_t + current_sample_coeff = current_alpha_t ** (0.5) * beta_prod_t_prev / beta_prod_t + + # 5. Compute predicted previous sample µ_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_prev_sample = pred_original_sample_coeff * pred_original_sample + current_sample_coeff * sample + + return pred_prev_sample + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + # Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement + # for the subsequent add_noise calls + self.alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.get_velocity + def get_velocity( + self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as sample + self.alphas_cumprod = self.alphas_cumprod.to(device=sample.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=sample.dtype) + timesteps = timesteps.to(sample.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(sample.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample + return velocity + + def __len__(self): + return self.config.num_train_timesteps + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.previous_timestep + def previous_timestep(self, timestep): + if self.custom_timesteps: + index = (self.timesteps == timestep).nonzero(as_tuple=True)[0][0] + if index == self.timesteps.shape[0] - 1: + prev_t = torch.tensor(-1) + else: + prev_t = self.timesteps[index + 1] + else: + num_inference_steps = ( + self.num_inference_steps if self.num_inference_steps else self.config.num_train_timesteps + ) + prev_t = timestep - self.config.num_train_timesteps // num_inference_steps + + return prev_t diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_wuerstchen.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_wuerstchen.py new file mode 100755 index 0000000..ad4e4f4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ddpm_wuerstchen.py @@ -0,0 +1,230 @@ +# Copyright (c) 2022 Pablo Pernías MIT License +# Copyright 2024 UC Berkeley Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import SchedulerMixin + + +@dataclass +class DDPMWuerstchenSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's step function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample (x_{t-1}) of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + """ + + prev_sample: torch.FloatTensor + + +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class DDPMWuerstchenScheduler(SchedulerMixin, ConfigMixin): + """ + Denoising diffusion probabilistic models (DDPMs) explores the connections between denoising score matching and + Langevin dynamics sampling. + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + For more details, see the original paper: https://arxiv.org/abs/2006.11239 + + Args: + scaler (`float`): .... + s (`float`): .... + """ + + @register_to_config + def __init__( + self, + scaler: float = 1.0, + s: float = 0.008, + ): + self.scaler = scaler + self.s = torch.tensor([s]) + self._init_alpha_cumprod = torch.cos(self.s / (1 + self.s) * torch.pi * 0.5) ** 2 + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + def _alpha_cumprod(self, t, device): + if self.scaler > 1: + t = 1 - (1 - t) ** self.scaler + elif self.scaler < 1: + t = t**self.scaler + alpha_cumprod = torch.cos( + (t + self.s.to(device)) / (1 + self.s.to(device)) * torch.pi * 0.5 + ) ** 2 / self._init_alpha_cumprod.to(device) + return alpha_cumprod.clamp(0.0001, 0.9999) + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): input sample + timestep (`int`, optional): current timestep + + Returns: + `torch.FloatTensor`: scaled input sample + """ + return sample + + def set_timesteps( + self, + num_inference_steps: int = None, + timesteps: Optional[List[int]] = None, + device: Union[str, torch.device] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + num_inference_steps (`Dict[float, int]`): + the number of diffusion steps used when generating samples with a pre-trained model. If passed, then + `timesteps` must be `None`. + device (`str` or `torch.device`, optional): + the device to which the timesteps are moved to. {2 / 3: 20, 0.0: 10} + """ + if timesteps is None: + timesteps = torch.linspace(1.0, 0.0, num_inference_steps + 1, device=device) + if not isinstance(timesteps, torch.Tensor): + timesteps = torch.Tensor(timesteps).to(device) + self.timesteps = timesteps + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator=None, + return_dict: bool = True, + ) -> Union[DDPMWuerstchenSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + current instance of sample being created by diffusion process. + generator: random number generator. + return_dict (`bool`): option for returning tuple rather than DDPMWuerstchenSchedulerOutput class + + Returns: + [`DDPMWuerstchenSchedulerOutput`] or `tuple`: [`DDPMWuerstchenSchedulerOutput`] if `return_dict` is True, + otherwise a `tuple`. When returning a tuple, the first element is the sample tensor. + + """ + dtype = model_output.dtype + device = model_output.device + t = timestep + + prev_t = self.previous_timestep(t) + + alpha_cumprod = self._alpha_cumprod(t, device).view(t.size(0), *[1 for _ in sample.shape[1:]]) + alpha_cumprod_prev = self._alpha_cumprod(prev_t, device).view(prev_t.size(0), *[1 for _ in sample.shape[1:]]) + alpha = alpha_cumprod / alpha_cumprod_prev + + mu = (1.0 / alpha).sqrt() * (sample - (1 - alpha) * model_output / (1 - alpha_cumprod).sqrt()) + + std_noise = randn_tensor(mu.shape, generator=generator, device=model_output.device, dtype=model_output.dtype) + std = ((1 - alpha) * (1.0 - alpha_cumprod_prev) / (1.0 - alpha_cumprod)).sqrt() * std_noise + pred = mu + std * (prev_t != 0).float().view(prev_t.size(0), *[1 for _ in sample.shape[1:]]) + + if not return_dict: + return (pred.to(dtype),) + + return DDPMWuerstchenSchedulerOutput(prev_sample=pred.to(dtype)) + + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + device = original_samples.device + dtype = original_samples.dtype + alpha_cumprod = self._alpha_cumprod(timesteps, device=device).view( + timesteps.size(0), *[1 for _ in original_samples.shape[1:]] + ) + noisy_samples = alpha_cumprod.sqrt() * original_samples + (1 - alpha_cumprod).sqrt() * noise + return noisy_samples.to(dtype=dtype) + + def __len__(self): + return self.config.num_train_timesteps + + def previous_timestep(self, timestep): + index = (self.timesteps - timestep[0]).abs().argmin().item() + prev_t = self.timesteps[index + 1][None].expand(timestep.shape[0]) + return prev_t diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_deis_multistep.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_deis_multistep.py new file mode 100755 index 0000000..a4af0c2 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_deis_multistep.py @@ -0,0 +1,786 @@ +# Copyright 2024 FLAIR Lab and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: check https://arxiv.org/abs/2204.13902 and https://github.com/qsh-zh/deis for more info +# The codebase is modified based on https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import deprecate +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class DEISMultistepScheduler(SchedulerMixin, ConfigMixin): + """ + `DEISMultistepScheduler` is a fast high order solver for diffusion ordinary differential equations (ODEs). + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + solver_order (`int`, defaults to 2): + The DEIS order which can be `1` or `2` or `3`. It is recommended to use `solver_order=2` for guided + sampling, and `solver_order=3` for unconditional sampling. + prediction_type (`str`, defaults to `epsilon`): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True`. + algorithm_type (`str`, defaults to `deis`): + The algorithm type for the solver. + lower_order_final (`bool`, defaults to `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[np.ndarray] = None, + solver_order: int = 2, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + algorithm_type: str = "deis", + solver_type: str = "logrho", + lower_order_final: bool = True, + use_karras_sigmas: Optional[bool] = False, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + # Currently we only support VP-type noise schedule + self.alpha_t = torch.sqrt(self.alphas_cumprod) + self.sigma_t = torch.sqrt(1 - self.alphas_cumprod) + self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t) + self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5 + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # settings for DEIS + if algorithm_type not in ["deis"]: + if algorithm_type in ["dpmsolver", "dpmsolver++"]: + self.register_to_config(algorithm_type="deis") + else: + raise NotImplementedError(f"{algorithm_type} does is not implemented for {self.__class__}") + + if solver_type not in ["logrho"]: + if solver_type in ["midpoint", "heun", "bh1", "bh2"]: + self.register_to_config(solver_type="logrho") + else: + raise NotImplementedError(f"solver type {solver_type} does is not implemented for {self.__class__}") + + # setable values + self.num_inference_steps = None + timesteps = np.linspace(0, num_train_timesteps - 1, num_train_timesteps, dtype=np.float32)[::-1].copy() + self.timesteps = torch.from_numpy(timesteps) + self.model_outputs = [None] * solver_order + self.lower_order_nums = 0 + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps + 1) + .round()[::-1][:-1] + .copy() + .astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // (num_inference_steps + 1) + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps + 1) * step_ratio).round()[::-1][:-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.arange(self.config.num_train_timesteps, 0, -step_ratio).round().copy().astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + if self.config.use_karras_sigmas: + log_sigmas = np.log(sigmas) + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + sigmas = np.concatenate([sigmas, sigmas[-1:]]).astype(np.float32) + else: + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + sigma_last = ((1 - self.alphas_cumprod[0]) / self.alphas_cumprod[0]) ** 0.5 + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + + self.sigmas = torch.from_numpy(sigmas) + self.timesteps = torch.from_numpy(timesteps).to(device=device, dtype=torch.int64) + + self.num_inference_steps = len(timesteps) + + self.model_outputs = [ + None, + ] * self.config.solver_order + self.lower_order_nums = 0 + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._sigma_to_alpha_sigma_t + def _sigma_to_alpha_sigma_t(self, sigma): + alpha_t = 1 / ((sigma**2 + 1) ** 0.5) + sigma_t = sigma * alpha_t + + return alpha_t, sigma_t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + def convert_model_output( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + Convert the model output to the corresponding type the DEIS algorithm needs. + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError("missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + if self.config.prediction_type == "epsilon": + x0_pred = (sample - sigma_t * model_output) / alpha_t + elif self.config.prediction_type == "sample": + x0_pred = model_output + elif self.config.prediction_type == "v_prediction": + x0_pred = alpha_t * sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the DEISMultistepScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + if self.config.algorithm_type == "deis": + return (sample - alpha_t * x0_pred) / sigma_t + else: + raise NotImplementedError("only support log-rho multistep deis now") + + def deis_first_order_update( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the first-order DEIS (equivalent to DDIM). + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + prev_timestep (`int`): + The previous discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s = self.sigmas[self.step_index + 1], self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s, sigma_s = self._sigma_to_alpha_sigma_t(sigma_s) + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s = torch.log(alpha_s) - torch.log(sigma_s) + + h = lambda_t - lambda_s + if self.config.algorithm_type == "deis": + x_t = (alpha_t / alpha_s) * sample - (sigma_t * (torch.exp(h) - 1.0)) * model_output + else: + raise NotImplementedError("only support log-rho multistep deis now") + return x_t + + def multistep_deis_second_order_update( + self, + model_output_list: List[torch.FloatTensor], + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the second-order multistep DEIS. + + Args: + model_output_list (`List[torch.FloatTensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s0, sigma_s1 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + + m0, m1 = model_output_list[-1], model_output_list[-2] + + rho_t, rho_s0, rho_s1 = sigma_t / alpha_t, sigma_s0 / alpha_s0, sigma_s1 / alpha_s1 + + if self.config.algorithm_type == "deis": + + def ind_fn(t, b, c): + # Integrate[(log(t) - log(c)) / (log(b) - log(c)), {t}] + return t * (-np.log(c) + np.log(t) - 1) / (np.log(b) - np.log(c)) + + coef1 = ind_fn(rho_t, rho_s0, rho_s1) - ind_fn(rho_s0, rho_s0, rho_s1) + coef2 = ind_fn(rho_t, rho_s1, rho_s0) - ind_fn(rho_s0, rho_s1, rho_s0) + + x_t = alpha_t * (sample / alpha_s0 + coef1 * m0 + coef2 * m1) + return x_t + else: + raise NotImplementedError("only support log-rho multistep deis now") + + def multistep_deis_third_order_update( + self, + model_output_list: List[torch.FloatTensor], + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the third-order multistep DEIS. + + Args: + model_output_list (`List[torch.FloatTensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.FloatTensor`): + A current instance of a sample created by diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing`sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s0, sigma_s1, sigma_s2 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + self.sigmas[self.step_index - 2], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + alpha_s2, sigma_s2 = self._sigma_to_alpha_sigma_t(sigma_s2) + + m0, m1, m2 = model_output_list[-1], model_output_list[-2], model_output_list[-3] + + rho_t, rho_s0, rho_s1, rho_s2 = ( + sigma_t / alpha_t, + sigma_s0 / alpha_s0, + sigma_s1 / alpha_s1, + sigma_s2 / alpha_s2, + ) + + if self.config.algorithm_type == "deis": + + def ind_fn(t, b, c, d): + # Integrate[(log(t) - log(c))(log(t) - log(d)) / (log(b) - log(c))(log(b) - log(d)), {t}] + numerator = t * ( + np.log(c) * (np.log(d) - np.log(t) + 1) + - np.log(d) * np.log(t) + + np.log(d) + + np.log(t) ** 2 + - 2 * np.log(t) + + 2 + ) + denominator = (np.log(b) - np.log(c)) * (np.log(b) - np.log(d)) + return numerator / denominator + + coef1 = ind_fn(rho_t, rho_s0, rho_s1, rho_s2) - ind_fn(rho_s0, rho_s0, rho_s1, rho_s2) + coef2 = ind_fn(rho_t, rho_s1, rho_s2, rho_s0) - ind_fn(rho_s0, rho_s1, rho_s2, rho_s0) + coef3 = ind_fn(rho_t, rho_s2, rho_s0, rho_s1) - ind_fn(rho_s0, rho_s2, rho_s0, rho_s1) + + x_t = alpha_t * (sample / alpha_s0 + coef1 * m0 + coef2 * m1 + coef3 * m2) + + return x_t + else: + raise NotImplementedError("only support log-rho multistep deis now") + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + index_candidates = (schedule_timesteps == timestep).nonzero() + + if len(index_candidates) == 0: + step_index = len(self.timesteps) - 1 + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + elif len(index_candidates) > 1: + step_index = index_candidates[1].item() + else: + step_index = index_candidates[0].item() + + return step_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index + def _init_step_index(self, timestep): + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the multistep DEIS. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + lower_order_final = ( + (self.step_index == len(self.timesteps) - 1) and self.config.lower_order_final and len(self.timesteps) < 15 + ) + lower_order_second = ( + (self.step_index == len(self.timesteps) - 2) and self.config.lower_order_final and len(self.timesteps) < 15 + ) + + model_output = self.convert_model_output(model_output, sample=sample) + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.model_outputs[-1] = model_output + + if self.config.solver_order == 1 or self.lower_order_nums < 1 or lower_order_final: + prev_sample = self.deis_first_order_update(model_output, sample=sample) + elif self.config.solver_order == 2 or self.lower_order_nums < 2 or lower_order_second: + prev_sample = self.multistep_deis_second_order_update(self.model_outputs, sample=sample) + else: + prev_sample = self.multistep_deis_third_order_update(self.model_outputs, sample=sample) + + if self.lower_order_nums < self.config.solver_order: + self.lower_order_nums += 1 + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # begin_index is None when the scheduler is used for training + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + noisy_samples = alpha_t * original_samples + sigma_t * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py new file mode 100755 index 0000000..3bbfc65 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py @@ -0,0 +1,1029 @@ +# Copyright 2024 TSAIL Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import deprecate +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.FloatTensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.FloatTensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class DPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): + """ + `DPMSolverMultistepScheduler` is a fast dedicated high-order solver for diffusion ODEs. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + solver_order (`int`, defaults to 2): + The DPMSolver order which can be `1` or `2` or `3`. It is recommended to use `solver_order=2` for guided + sampling, and `solver_order=3` for unconditional sampling. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and + `algorithm_type="dpmsolver++"`. + algorithm_type (`str`, defaults to `dpmsolver++`): + Algorithm type for the solver; can be `dpmsolver`, `dpmsolver++`, `sde-dpmsolver` or `sde-dpmsolver++`. The + `dpmsolver` type implements the algorithms in the [DPMSolver](https://huggingface.co/papers/2206.00927) + paper, and the `dpmsolver++` type implements the algorithms in the + [DPMSolver++](https://huggingface.co/papers/2211.01095) paper. It is recommended to use `dpmsolver++` or + `sde-dpmsolver++` with `solver_order=2` for guided sampling like in Stable Diffusion. + solver_type (`str`, defaults to `midpoint`): + Solver type for the second-order solver; can be `midpoint` or `heun`. The solver type slightly affects the + sample quality, especially for a small number of steps. It is recommended to use `midpoint` solvers. + lower_order_final (`bool`, defaults to `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can + stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + euler_at_final (`bool`, defaults to `False`): + Whether to use Euler's method in the final step. It is a trade-off between numerical stability and detail + richness. This can stabilize the sampling of the SDE variant of DPMSolver for small number of inference + steps, but sometimes may result in blurring. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + use_lu_lambdas (`bool`, *optional*, defaults to `False`): + Whether to use the uniform-logSNR for step sizes proposed by Lu's DPM-Solver in the noise schedule during + the sampling process. If `True`, the sigmas and time steps are determined according to a sequence of + `lambda(t)`. + final_sigmas_type (`str`, defaults to `"zero"`): + The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final sigma + is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0. + lambda_min_clipped (`float`, defaults to `-inf`): + Clipping threshold for the minimum value of `lambda(t)` for numerical stability. This is critical for the + cosine (`squaredcos_cap_v2`) noise schedule. + variance_type (`str`, *optional*): + Set to "learned" or "learned_range" for diffusion models that predict variance. If set, the model's output + contains the predicted Gaussian variance. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + solver_order: int = 2, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + algorithm_type: str = "dpmsolver++", + solver_type: str = "midpoint", + lower_order_final: bool = True, + euler_at_final: bool = False, + use_karras_sigmas: Optional[bool] = False, + use_lu_lambdas: Optional[bool] = False, + final_sigmas_type: Optional[str] = "zero", # "zero", "sigma_min" + lambda_min_clipped: float = -float("inf"), + variance_type: Optional[str] = None, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + rescale_betas_zero_snr: bool = False, + ): + if algorithm_type in ["dpmsolver", "sde-dpmsolver"]: + deprecation_message = f"algorithm_type {algorithm_type} is deprecated and will be removed in a future version. Choose from `dpmsolver++` or `sde-dpmsolver++` instead" + deprecate("algorithm_types dpmsolver and sde-dpmsolver", "1.0.0", deprecation_message) + + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + if rescale_betas_zero_snr: + # Close to 0 without being 0 so first sigma is not inf + # FP16 smallest positive subnormal works well here + self.alphas_cumprod[-1] = 2**-24 + + # Currently we only support VP-type noise schedule + self.alpha_t = torch.sqrt(self.alphas_cumprod) + self.sigma_t = torch.sqrt(1 - self.alphas_cumprod) + self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t) + self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5 + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # settings for DPM-Solver + if algorithm_type not in ["dpmsolver", "dpmsolver++", "sde-dpmsolver", "sde-dpmsolver++"]: + if algorithm_type == "deis": + self.register_to_config(algorithm_type="dpmsolver++") + else: + raise NotImplementedError(f"{algorithm_type} does is not implemented for {self.__class__}") + + if solver_type not in ["midpoint", "heun"]: + if solver_type in ["logrho", "bh1", "bh2"]: + self.register_to_config(solver_type="midpoint") + else: + raise NotImplementedError(f"{solver_type} does is not implemented for {self.__class__}") + + if algorithm_type not in ["dpmsolver++", "sde-dpmsolver++"] and final_sigmas_type == "zero": + raise ValueError( + f"`final_sigmas_type` {final_sigmas_type} is not supported for `algorithm_type` {algorithm_type}. Please choose `sigma_min` instead." + ) + + # setable values + self.num_inference_steps = None + timesteps = np.linspace(0, num_train_timesteps - 1, num_train_timesteps, dtype=np.float32)[::-1].copy() + self.timesteps = torch.from_numpy(timesteps) + self.model_outputs = [None] * solver_order + self.lower_order_nums = 0 + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def set_timesteps(self, num_inference_steps: int = None, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + # Clipping the minimum of all lambda(t) for numerical stability. + # This is critical for cosine (squaredcos_cap_v2) noise schedule. + clipped_idx = torch.searchsorted(torch.flip(self.lambda_t, [0]), self.config.lambda_min_clipped) + last_timestep = ((self.config.num_train_timesteps - clipped_idx).numpy()).item() + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, last_timestep - 1, num_inference_steps + 1).round()[::-1][:-1].copy().astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = last_timestep // (num_inference_steps + 1) + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps + 1) * step_ratio).round()[::-1][:-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.arange(last_timestep, 0, -step_ratio).round().copy().astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + log_sigmas = np.log(sigmas) + + if self.config.use_karras_sigmas: + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + elif self.config.use_lu_lambdas: + lambdas = np.flip(log_sigmas.copy()) + lambdas = self._convert_to_lu(in_lambdas=lambdas, num_inference_steps=num_inference_steps) + sigmas = np.exp(lambdas) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + else: + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + + if self.config.final_sigmas_type == "sigma_min": + sigma_last = ((1 - self.alphas_cumprod[0]) / self.alphas_cumprod[0]) ** 0.5 + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + + self.sigmas = torch.from_numpy(sigmas) + self.timesteps = torch.from_numpy(timesteps).to(device=device, dtype=torch.int64) + + self.num_inference_steps = len(timesteps) + + self.model_outputs = [ + None, + ] * self.config.solver_order + self.lower_order_nums = 0 + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + def _sigma_to_alpha_sigma_t(self, sigma): + alpha_t = 1 / ((sigma**2 + 1) ** 0.5) + sigma_t = sigma * alpha_t + + return alpha_t, sigma_t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + def _convert_to_lu(self, in_lambdas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Lu et al. (2022).""" + + lambda_min: float = in_lambdas[-1].item() + lambda_max: float = in_lambdas[0].item() + + rho = 1.0 # 1.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = lambda_min ** (1 / rho) + max_inv_rho = lambda_max ** (1 / rho) + lambdas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return lambdas + + def convert_model_output( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + Convert the model output to the corresponding type the DPMSolver/DPMSolver++ algorithm needs. DPM-Solver is + designed to discretize an integral of the noise prediction model, and DPM-Solver++ is designed to discretize an + integral of the data prediction model. + + + + The algorithm and model type are decoupled. You can use either DPMSolver or DPMSolver++ for both noise + prediction and data prediction models. + + + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError("missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + # DPM-Solver++ needs to solve an integral of the data prediction model. + if self.config.algorithm_type in ["dpmsolver++", "sde-dpmsolver++"]: + if self.config.prediction_type == "epsilon": + # DPM-Solver and DPM-Solver++ only need the "mean" output. + if self.config.variance_type in ["learned", "learned_range"]: + model_output = model_output[:, :3] + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = (sample - sigma_t * model_output) / alpha_t + elif self.config.prediction_type == "sample": + x0_pred = model_output + elif self.config.prediction_type == "v_prediction": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = alpha_t * sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the DPMSolverMultistepScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + + # DPM-Solver needs to solve an integral of the noise prediction model. + elif self.config.algorithm_type in ["dpmsolver", "sde-dpmsolver"]: + if self.config.prediction_type == "epsilon": + # DPM-Solver and DPM-Solver++ only need the "mean" output. + if self.config.variance_type in ["learned", "learned_range"]: + epsilon = model_output[:, :3] + else: + epsilon = model_output + elif self.config.prediction_type == "sample": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + epsilon = (sample - alpha_t * model_output) / sigma_t + elif self.config.prediction_type == "v_prediction": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + epsilon = alpha_t * model_output + sigma_t * sample + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the DPMSolverMultistepScheduler." + ) + + if self.config.thresholding: + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = (sample - sigma_t * epsilon) / alpha_t + x0_pred = self._threshold_sample(x0_pred) + epsilon = (sample - alpha_t * x0_pred) / sigma_t + + return epsilon + + def dpm_solver_first_order_update( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + noise: Optional[torch.FloatTensor] = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the first-order DPMSolver (equivalent to DDIM). + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s = self.sigmas[self.step_index + 1], self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s, sigma_s = self._sigma_to_alpha_sigma_t(sigma_s) + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s = torch.log(alpha_s) - torch.log(sigma_s) + + h = lambda_t - lambda_s + if self.config.algorithm_type == "dpmsolver++": + x_t = (sigma_t / sigma_s) * sample - (alpha_t * (torch.exp(-h) - 1.0)) * model_output + elif self.config.algorithm_type == "dpmsolver": + x_t = (alpha_t / alpha_s) * sample - (sigma_t * (torch.exp(h) - 1.0)) * model_output + elif self.config.algorithm_type == "sde-dpmsolver++": + assert noise is not None + x_t = ( + (sigma_t / sigma_s * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * model_output + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + elif self.config.algorithm_type == "sde-dpmsolver": + assert noise is not None + x_t = ( + (alpha_t / alpha_s) * sample + - 2.0 * (sigma_t * (torch.exp(h) - 1.0)) * model_output + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise + ) + return x_t + + def multistep_dpm_solver_second_order_update( + self, + model_output_list: List[torch.FloatTensor], + *args, + sample: torch.FloatTensor = None, + noise: Optional[torch.FloatTensor] = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the second-order multistep DPMSolver. + + Args: + model_output_list (`List[torch.FloatTensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s0, sigma_s1 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + + m0, m1 = model_output_list[-1], model_output_list[-2] + + h, h_0 = lambda_t - lambda_s0, lambda_s0 - lambda_s1 + r0 = h_0 / h + D0, D1 = m0, (1.0 / r0) * (m0 - m1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2211.01095 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + - 0.5 * (alpha_t * (torch.exp(-h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 + ) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - 0.5 * (sigma_t * (torch.exp(h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + ) + elif self.config.algorithm_type == "sde-dpmsolver++": + assert noise is not None + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + + 0.5 * (alpha_t * (1 - torch.exp(-2.0 * h))) * D1 + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + + (alpha_t * ((1.0 - torch.exp(-2.0 * h)) / (-2.0 * h) + 1.0)) * D1 + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + elif self.config.algorithm_type == "sde-dpmsolver": + assert noise is not None + if self.config.solver_type == "midpoint": + x_t = ( + (alpha_t / alpha_s0) * sample + - 2.0 * (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * (torch.exp(h) - 1.0)) * D1 + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise + ) + elif self.config.solver_type == "heun": + x_t = ( + (alpha_t / alpha_s0) * sample + - 2.0 * (sigma_t * (torch.exp(h) - 1.0)) * D0 + - 2.0 * (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise + ) + return x_t + + def multistep_dpm_solver_third_order_update( + self, + model_output_list: List[torch.FloatTensor], + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the third-order multistep DPMSolver. + + Args: + model_output_list (`List[torch.FloatTensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.FloatTensor`): + A current instance of a sample created by diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing`sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s0, sigma_s1, sigma_s2 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + self.sigmas[self.step_index - 2], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + alpha_s2, sigma_s2 = self._sigma_to_alpha_sigma_t(sigma_s2) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + lambda_s2 = torch.log(alpha_s2) - torch.log(sigma_s2) + + m0, m1, m2 = model_output_list[-1], model_output_list[-2], model_output_list[-3] + + h, h_0, h_1 = lambda_t - lambda_s0, lambda_s0 - lambda_s1, lambda_s1 - lambda_s2 + r0, r1 = h_0 / h, h_1 / h + D0 = m0 + D1_0, D1_1 = (1.0 / r0) * (m0 - m1), (1.0 / r1) * (m1 - m2) + D1 = D1_0 + (r0 / (r0 + r1)) * (D1_0 - D1_1) + D2 = (1.0 / (r0 + r1)) * (D1_0 - D1_1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 + - (alpha_t * ((torch.exp(-h) - 1.0 + h) / h**2 - 0.5)) * D2 + ) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + - (sigma_t * ((torch.exp(h) - 1.0 - h) / h**2 - 0.5)) * D2 + ) + return x_t + + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + index_candidates = (schedule_timesteps == timestep).nonzero() + + if len(index_candidates) == 0: + step_index = len(self.timesteps) - 1 + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + elif len(index_candidates) > 1: + step_index = index_candidates[1].item() + else: + step_index = index_candidates[0].item() + + return step_index + + def _init_step_index(self, timestep): + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator=None, + variance_noise: Optional[torch.FloatTensor] = None, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the multistep DPMSolver. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + variance_noise (`torch.FloatTensor`): + Alternative to generating noise with `generator` by directly providing the noise for the variance + itself. Useful for methods such as [`LEdits++`]. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Improve numerical stability for small number of steps + lower_order_final = (self.step_index == len(self.timesteps) - 1) and ( + self.config.euler_at_final + or (self.config.lower_order_final and len(self.timesteps) < 15) + or self.config.final_sigmas_type == "zero" + ) + lower_order_second = ( + (self.step_index == len(self.timesteps) - 2) and self.config.lower_order_final and len(self.timesteps) < 15 + ) + + model_output = self.convert_model_output(model_output, sample=sample) + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.model_outputs[-1] = model_output + + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + if self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"] and variance_noise is None: + noise = randn_tensor( + model_output.shape, generator=generator, device=model_output.device, dtype=torch.float32 + ) + elif self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"]: + noise = variance_noise.to(device=model_output.device, dtype=torch.float32) + else: + noise = None + + if self.config.solver_order == 1 or self.lower_order_nums < 1 or lower_order_final: + prev_sample = self.dpm_solver_first_order_update(model_output, sample=sample, noise=noise) + elif self.config.solver_order == 2 or self.lower_order_nums < 2 or lower_order_second: + prev_sample = self.multistep_dpm_solver_second_order_update(self.model_outputs, sample=sample, noise=noise) + else: + prev_sample = self.multistep_dpm_solver_third_order_update(self.model_outputs, sample=sample) + + if self.lower_order_nums < self.config.solver_order: + self.lower_order_nums += 1 + + # Cast sample back to expected dtype + prev_sample = prev_sample.to(model_output.dtype) + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # begin_index is None when the scheduler is used for training + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + noisy_samples = alpha_t * original_samples + sigma_t * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py new file mode 100755 index 0000000..0b48b49 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep_flax.py @@ -0,0 +1,643 @@ +# Copyright 2024 TSAIL Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver + +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import flax +import jax +import jax.numpy as jnp + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils_flax import ( + CommonSchedulerState, + FlaxKarrasDiffusionSchedulers, + FlaxSchedulerMixin, + FlaxSchedulerOutput, + add_noise_common, +) + + +@flax.struct.dataclass +class DPMSolverMultistepSchedulerState: + common: CommonSchedulerState + alpha_t: jnp.ndarray + sigma_t: jnp.ndarray + lambda_t: jnp.ndarray + + # setable values + init_noise_sigma: jnp.ndarray + timesteps: jnp.ndarray + num_inference_steps: Optional[int] = None + + # running values + model_outputs: Optional[jnp.ndarray] = None + lower_order_nums: Optional[jnp.int32] = None + prev_timestep: Optional[jnp.int32] = None + cur_sample: Optional[jnp.ndarray] = None + + @classmethod + def create( + cls, + common: CommonSchedulerState, + alpha_t: jnp.ndarray, + sigma_t: jnp.ndarray, + lambda_t: jnp.ndarray, + init_noise_sigma: jnp.ndarray, + timesteps: jnp.ndarray, + ): + return cls( + common=common, + alpha_t=alpha_t, + sigma_t=sigma_t, + lambda_t=lambda_t, + init_noise_sigma=init_noise_sigma, + timesteps=timesteps, + ) + + +@dataclass +class FlaxDPMSolverMultistepSchedulerOutput(FlaxSchedulerOutput): + state: DPMSolverMultistepSchedulerState + + +class FlaxDPMSolverMultistepScheduler(FlaxSchedulerMixin, ConfigMixin): + """ + DPM-Solver (and the improved version DPM-Solver++) is a fast dedicated high-order solver for diffusion ODEs with + the convergence order guarantee. Empirically, sampling by DPM-Solver with only 20 steps can generate high-quality + samples, and it can generate quite good samples even in only 10 steps. + + For more details, see the original paper: https://arxiv.org/abs/2206.00927 and https://arxiv.org/abs/2211.01095 + + Currently, we support the multistep DPM-Solver for both noise prediction models and data prediction models. We + recommend to use `solver_order=2` for guided sampling, and `solver_order=3` for unconditional sampling. + + We also support the "dynamic thresholding" method in Imagen (https://arxiv.org/abs/2205.11487). For pixel-space + diffusion models, you can set both `algorithm_type="dpmsolver++"` and `thresholding=True` to use the dynamic + thresholding. Note that the thresholding method is unsuitable for latent-space diffusion models (such as + stable-diffusion). + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + For more details, see the original paper: https://arxiv.org/abs/2206.00927 and https://arxiv.org/abs/2211.01095 + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + beta_start (`float`): the starting `beta` value of inference. + beta_end (`float`): the final `beta` value. + beta_schedule (`str`): + the beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, optional): + option to pass an array of betas directly to the constructor to bypass `beta_start`, `beta_end` etc. + solver_order (`int`, default `2`): + the order of DPM-Solver; can be `1` or `2` or `3`. We recommend to use `solver_order=2` for guided + sampling, and `solver_order=3` for unconditional sampling. + prediction_type (`str`, default `epsilon`): + indicates whether the model predicts the noise (epsilon), or the data / `x0`. One of `epsilon`, `sample`, + or `v-prediction`. + thresholding (`bool`, default `False`): + whether to use the "dynamic thresholding" method (introduced by Imagen, https://arxiv.org/abs/2205.11487). + For pixel-space diffusion models, you can set both `algorithm_type=dpmsolver++` and `thresholding=True` to + use the dynamic thresholding. Note that the thresholding method is unsuitable for latent-space diffusion + models (such as stable-diffusion). + dynamic_thresholding_ratio (`float`, default `0.995`): + the ratio for the dynamic thresholding method. Default is `0.995`, the same as Imagen + (https://arxiv.org/abs/2205.11487). + sample_max_value (`float`, default `1.0`): + the threshold value for dynamic thresholding. Valid only when `thresholding=True` and + `algorithm_type="dpmsolver++`. + algorithm_type (`str`, default `dpmsolver++`): + the algorithm type for the solver. Either `dpmsolver` or `dpmsolver++`. The `dpmsolver` type implements the + algorithms in https://arxiv.org/abs/2206.00927, and the `dpmsolver++` type implements the algorithms in + https://arxiv.org/abs/2211.01095. We recommend to use `dpmsolver++` with `solver_order=2` for guided + sampling (e.g. stable-diffusion). + solver_type (`str`, default `midpoint`): + the solver type for the second-order solver. Either `midpoint` or `heun`. The solver type slightly affects + the sample quality, especially for small number of steps. We empirically find that `midpoint` solvers are + slightly better, so we recommend to use the `midpoint` type. + lower_order_final (`bool`, default `True`): + whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. We empirically + find this trick can stabilize the sampling of DPM-Solver for steps < 15, especially for steps <= 10. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + dtype (`jnp.dtype`, *optional*, defaults to `jnp.float32`): + the `dtype` used for params and computation. + """ + + _compatibles = [e.name for e in FlaxKarrasDiffusionSchedulers] + + dtype: jnp.dtype + + @property + def has_state(self): + return True + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[jnp.ndarray] = None, + solver_order: int = 2, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + algorithm_type: str = "dpmsolver++", + solver_type: str = "midpoint", + lower_order_final: bool = True, + timestep_spacing: str = "linspace", + dtype: jnp.dtype = jnp.float32, + ): + self.dtype = dtype + + def create_state(self, common: Optional[CommonSchedulerState] = None) -> DPMSolverMultistepSchedulerState: + if common is None: + common = CommonSchedulerState.create(self) + + # Currently we only support VP-type noise schedule + alpha_t = jnp.sqrt(common.alphas_cumprod) + sigma_t = jnp.sqrt(1 - common.alphas_cumprod) + lambda_t = jnp.log(alpha_t) - jnp.log(sigma_t) + + # settings for DPM-Solver + if self.config.algorithm_type not in ["dpmsolver", "dpmsolver++"]: + raise NotImplementedError(f"{self.config.algorithm_type} does is not implemented for {self.__class__}") + if self.config.solver_type not in ["midpoint", "heun"]: + raise NotImplementedError(f"{self.config.solver_type} does is not implemented for {self.__class__}") + + # standard deviation of the initial noise distribution + init_noise_sigma = jnp.array(1.0, dtype=self.dtype) + + timesteps = jnp.arange(0, self.config.num_train_timesteps).round()[::-1] + + return DPMSolverMultistepSchedulerState.create( + common=common, + alpha_t=alpha_t, + sigma_t=sigma_t, + lambda_t=lambda_t, + init_noise_sigma=init_noise_sigma, + timesteps=timesteps, + ) + + def set_timesteps( + self, state: DPMSolverMultistepSchedulerState, num_inference_steps: int, shape: Tuple + ) -> DPMSolverMultistepSchedulerState: + """ + Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + state (`DPMSolverMultistepSchedulerState`): + the `FlaxDPMSolverMultistepScheduler` state data class instance. + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + shape (`Tuple`): + the shape of the samples to be generated. + """ + last_timestep = self.config.num_train_timesteps + if self.config.timestep_spacing == "linspace": + timesteps = ( + jnp.linspace(0, last_timestep - 1, num_inference_steps + 1).round()[::-1][:-1].astype(jnp.int32) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = last_timestep // (num_inference_steps + 1) + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = ( + (jnp.arange(0, num_inference_steps + 1) * step_ratio).round()[::-1][:-1].copy().astype(jnp.int32) + ) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = jnp.arange(last_timestep, 0, -step_ratio).round().copy().astype(jnp.int32) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + # initial running values + + model_outputs = jnp.zeros((self.config.solver_order,) + shape, dtype=self.dtype) + lower_order_nums = jnp.int32(0) + prev_timestep = jnp.int32(-1) + cur_sample = jnp.zeros(shape, dtype=self.dtype) + + return state.replace( + num_inference_steps=num_inference_steps, + timesteps=timesteps, + model_outputs=model_outputs, + lower_order_nums=lower_order_nums, + prev_timestep=prev_timestep, + cur_sample=cur_sample, + ) + + def convert_model_output( + self, + state: DPMSolverMultistepSchedulerState, + model_output: jnp.ndarray, + timestep: int, + sample: jnp.ndarray, + ) -> jnp.ndarray: + """ + Convert the model output to the corresponding type that the algorithm (DPM-Solver / DPM-Solver++) needs. + + DPM-Solver is designed to discretize an integral of the noise prediction model, and DPM-Solver++ is designed to + discretize an integral of the data prediction model. So we need to first convert the model output to the + corresponding type to match the algorithm. + + Note that the algorithm type and the model type is decoupled. That is to say, we can use either DPM-Solver or + DPM-Solver++ for both noise prediction model and data prediction model. + + Args: + model_output (`jnp.ndarray`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + + Returns: + `jnp.ndarray`: the converted model output. + """ + # DPM-Solver++ needs to solve an integral of the data prediction model. + if self.config.algorithm_type == "dpmsolver++": + if self.config.prediction_type == "epsilon": + alpha_t, sigma_t = state.alpha_t[timestep], state.sigma_t[timestep] + x0_pred = (sample - sigma_t * model_output) / alpha_t + elif self.config.prediction_type == "sample": + x0_pred = model_output + elif self.config.prediction_type == "v_prediction": + alpha_t, sigma_t = state.alpha_t[timestep], state.sigma_t[timestep] + x0_pred = alpha_t * sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, " + " or `v_prediction` for the FlaxDPMSolverMultistepScheduler." + ) + + if self.config.thresholding: + # Dynamic thresholding in https://arxiv.org/abs/2205.11487 + dynamic_max_val = jnp.percentile( + jnp.abs(x0_pred), self.config.dynamic_thresholding_ratio, axis=tuple(range(1, x0_pred.ndim)) + ) + dynamic_max_val = jnp.maximum( + dynamic_max_val, self.config.sample_max_value * jnp.ones_like(dynamic_max_val) + ) + x0_pred = jnp.clip(x0_pred, -dynamic_max_val, dynamic_max_val) / dynamic_max_val + return x0_pred + # DPM-Solver needs to solve an integral of the noise prediction model. + elif self.config.algorithm_type == "dpmsolver": + if self.config.prediction_type == "epsilon": + return model_output + elif self.config.prediction_type == "sample": + alpha_t, sigma_t = state.alpha_t[timestep], state.sigma_t[timestep] + epsilon = (sample - alpha_t * model_output) / sigma_t + return epsilon + elif self.config.prediction_type == "v_prediction": + alpha_t, sigma_t = state.alpha_t[timestep], state.sigma_t[timestep] + epsilon = alpha_t * model_output + sigma_t * sample + return epsilon + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, " + " or `v_prediction` for the FlaxDPMSolverMultistepScheduler." + ) + + def dpm_solver_first_order_update( + self, + state: DPMSolverMultistepSchedulerState, + model_output: jnp.ndarray, + timestep: int, + prev_timestep: int, + sample: jnp.ndarray, + ) -> jnp.ndarray: + """ + One step for the first-order DPM-Solver (equivalent to DDIM). + + See https://arxiv.org/abs/2206.00927 for the detailed derivation. + + Args: + model_output (`jnp.ndarray`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + prev_timestep (`int`): previous discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + + Returns: + `jnp.ndarray`: the sample tensor at the previous timestep. + """ + t, s0 = prev_timestep, timestep + m0 = model_output + lambda_t, lambda_s = state.lambda_t[t], state.lambda_t[s0] + alpha_t, alpha_s = state.alpha_t[t], state.alpha_t[s0] + sigma_t, sigma_s = state.sigma_t[t], state.sigma_t[s0] + h = lambda_t - lambda_s + if self.config.algorithm_type == "dpmsolver++": + x_t = (sigma_t / sigma_s) * sample - (alpha_t * (jnp.exp(-h) - 1.0)) * m0 + elif self.config.algorithm_type == "dpmsolver": + x_t = (alpha_t / alpha_s) * sample - (sigma_t * (jnp.exp(h) - 1.0)) * m0 + return x_t + + def multistep_dpm_solver_second_order_update( + self, + state: DPMSolverMultistepSchedulerState, + model_output_list: jnp.ndarray, + timestep_list: List[int], + prev_timestep: int, + sample: jnp.ndarray, + ) -> jnp.ndarray: + """ + One step for the second-order multistep DPM-Solver. + + Args: + model_output_list (`List[jnp.ndarray]`): + direct outputs from learned diffusion model at current and latter timesteps. + timestep (`int`): current and latter discrete timestep in the diffusion chain. + prev_timestep (`int`): previous discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + + Returns: + `jnp.ndarray`: the sample tensor at the previous timestep. + """ + t, s0, s1 = prev_timestep, timestep_list[-1], timestep_list[-2] + m0, m1 = model_output_list[-1], model_output_list[-2] + lambda_t, lambda_s0, lambda_s1 = state.lambda_t[t], state.lambda_t[s0], state.lambda_t[s1] + alpha_t, alpha_s0 = state.alpha_t[t], state.alpha_t[s0] + sigma_t, sigma_s0 = state.sigma_t[t], state.sigma_t[s0] + h, h_0 = lambda_t - lambda_s0, lambda_s0 - lambda_s1 + r0 = h_0 / h + D0, D1 = m0, (1.0 / r0) * (m0 - m1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2211.01095 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (jnp.exp(-h) - 1.0)) * D0 + - 0.5 * (alpha_t * (jnp.exp(-h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (jnp.exp(-h) - 1.0)) * D0 + + (alpha_t * ((jnp.exp(-h) - 1.0) / h + 1.0)) * D1 + ) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (jnp.exp(h) - 1.0)) * D0 + - 0.5 * (sigma_t * (jnp.exp(h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (jnp.exp(h) - 1.0)) * D0 + - (sigma_t * ((jnp.exp(h) - 1.0) / h - 1.0)) * D1 + ) + return x_t + + def multistep_dpm_solver_third_order_update( + self, + state: DPMSolverMultistepSchedulerState, + model_output_list: jnp.ndarray, + timestep_list: List[int], + prev_timestep: int, + sample: jnp.ndarray, + ) -> jnp.ndarray: + """ + One step for the third-order multistep DPM-Solver. + + Args: + model_output_list (`List[jnp.ndarray]`): + direct outputs from learned diffusion model at current and latter timesteps. + timestep (`int`): current and latter discrete timestep in the diffusion chain. + prev_timestep (`int`): previous discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + + Returns: + `jnp.ndarray`: the sample tensor at the previous timestep. + """ + t, s0, s1, s2 = prev_timestep, timestep_list[-1], timestep_list[-2], timestep_list[-3] + m0, m1, m2 = model_output_list[-1], model_output_list[-2], model_output_list[-3] + lambda_t, lambda_s0, lambda_s1, lambda_s2 = ( + state.lambda_t[t], + state.lambda_t[s0], + state.lambda_t[s1], + state.lambda_t[s2], + ) + alpha_t, alpha_s0 = state.alpha_t[t], state.alpha_t[s0] + sigma_t, sigma_s0 = state.sigma_t[t], state.sigma_t[s0] + h, h_0, h_1 = lambda_t - lambda_s0, lambda_s0 - lambda_s1, lambda_s1 - lambda_s2 + r0, r1 = h_0 / h, h_1 / h + D0 = m0 + D1_0, D1_1 = (1.0 / r0) * (m0 - m1), (1.0 / r1) * (m1 - m2) + D1 = D1_0 + (r0 / (r0 + r1)) * (D1_0 - D1_1) + D2 = (1.0 / (r0 + r1)) * (D1_0 - D1_1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (jnp.exp(-h) - 1.0)) * D0 + + (alpha_t * ((jnp.exp(-h) - 1.0) / h + 1.0)) * D1 + - (alpha_t * ((jnp.exp(-h) - 1.0 + h) / h**2 - 0.5)) * D2 + ) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (jnp.exp(h) - 1.0)) * D0 + - (sigma_t * ((jnp.exp(h) - 1.0) / h - 1.0)) * D1 + - (sigma_t * ((jnp.exp(h) - 1.0 - h) / h**2 - 0.5)) * D2 + ) + return x_t + + def step( + self, + state: DPMSolverMultistepSchedulerState, + model_output: jnp.ndarray, + timestep: int, + sample: jnp.ndarray, + return_dict: bool = True, + ) -> Union[FlaxDPMSolverMultistepSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by DPM-Solver. Core function to propagate the diffusion process + from the learned model outputs (most often the predicted noise). + + Args: + state (`DPMSolverMultistepSchedulerState`): + the `FlaxDPMSolverMultistepScheduler` state data class instance. + model_output (`jnp.ndarray`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + return_dict (`bool`): option for returning tuple rather than FlaxDPMSolverMultistepSchedulerOutput class + + Returns: + [`FlaxDPMSolverMultistepSchedulerOutput`] or `tuple`: [`FlaxDPMSolverMultistepSchedulerOutput`] if + `return_dict` is True, otherwise a `tuple`. When returning a tuple, the first element is the sample tensor. + + """ + if state.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + (step_index,) = jnp.where(state.timesteps == timestep, size=1) + step_index = step_index[0] + + prev_timestep = jax.lax.select(step_index == len(state.timesteps) - 1, 0, state.timesteps[step_index + 1]) + + model_output = self.convert_model_output(state, model_output, timestep, sample) + + model_outputs_new = jnp.roll(state.model_outputs, -1, axis=0) + model_outputs_new = model_outputs_new.at[-1].set(model_output) + state = state.replace( + model_outputs=model_outputs_new, + prev_timestep=prev_timestep, + cur_sample=sample, + ) + + def step_1(state: DPMSolverMultistepSchedulerState) -> jnp.ndarray: + return self.dpm_solver_first_order_update( + state, + state.model_outputs[-1], + state.timesteps[step_index], + state.prev_timestep, + state.cur_sample, + ) + + def step_23(state: DPMSolverMultistepSchedulerState) -> jnp.ndarray: + def step_2(state: DPMSolverMultistepSchedulerState) -> jnp.ndarray: + timestep_list = jnp.array([state.timesteps[step_index - 1], state.timesteps[step_index]]) + return self.multistep_dpm_solver_second_order_update( + state, + state.model_outputs, + timestep_list, + state.prev_timestep, + state.cur_sample, + ) + + def step_3(state: DPMSolverMultistepSchedulerState) -> jnp.ndarray: + timestep_list = jnp.array( + [ + state.timesteps[step_index - 2], + state.timesteps[step_index - 1], + state.timesteps[step_index], + ] + ) + return self.multistep_dpm_solver_third_order_update( + state, + state.model_outputs, + timestep_list, + state.prev_timestep, + state.cur_sample, + ) + + step_2_output = step_2(state) + step_3_output = step_3(state) + + if self.config.solver_order == 2: + return step_2_output + elif self.config.lower_order_final and len(state.timesteps) < 15: + return jax.lax.select( + state.lower_order_nums < 2, + step_2_output, + jax.lax.select( + step_index == len(state.timesteps) - 2, + step_2_output, + step_3_output, + ), + ) + else: + return jax.lax.select( + state.lower_order_nums < 2, + step_2_output, + step_3_output, + ) + + step_1_output = step_1(state) + step_23_output = step_23(state) + + if self.config.solver_order == 1: + prev_sample = step_1_output + + elif self.config.lower_order_final and len(state.timesteps) < 15: + prev_sample = jax.lax.select( + state.lower_order_nums < 1, + step_1_output, + jax.lax.select( + step_index == len(state.timesteps) - 1, + step_1_output, + step_23_output, + ), + ) + + else: + prev_sample = jax.lax.select( + state.lower_order_nums < 1, + step_1_output, + step_23_output, + ) + + state = state.replace( + lower_order_nums=jnp.minimum(state.lower_order_nums + 1, self.config.solver_order), + ) + + if not return_dict: + return (prev_sample, state) + + return FlaxDPMSolverMultistepSchedulerOutput(prev_sample=prev_sample, state=state) + + def scale_model_input( + self, state: DPMSolverMultistepSchedulerState, sample: jnp.ndarray, timestep: Optional[int] = None + ) -> jnp.ndarray: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + state (`DPMSolverMultistepSchedulerState`): + the `FlaxDPMSolverMultistepScheduler` state data class instance. + sample (`jnp.ndarray`): input sample + timestep (`int`, optional): current timestep + + Returns: + `jnp.ndarray`: scaled input sample + """ + return sample + + def add_noise( + self, + state: DPMSolverMultistepSchedulerState, + original_samples: jnp.ndarray, + noise: jnp.ndarray, + timesteps: jnp.ndarray, + ) -> jnp.ndarray: + return add_noise_common(state.common, original_samples, noise, timesteps) + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py new file mode 100755 index 0000000..318b9ed --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_multistep_inverse.py @@ -0,0 +1,921 @@ +# Copyright 2024 TSAIL Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import deprecate +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class DPMSolverMultistepInverseScheduler(SchedulerMixin, ConfigMixin): + """ + `DPMSolverMultistepInverseScheduler` is the reverse scheduler of [`DPMSolverMultistepScheduler`]. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + solver_order (`int`, defaults to 2): + The DPMSolver order which can be `1` or `2` or `3`. It is recommended to use `solver_order=2` for guided + sampling, and `solver_order=3` for unconditional sampling. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and + `algorithm_type="dpmsolver++"`. + algorithm_type (`str`, defaults to `dpmsolver++`): + Algorithm type for the solver; can be `dpmsolver`, `dpmsolver++`, `sde-dpmsolver` or `sde-dpmsolver++`. The + `dpmsolver` type implements the algorithms in the [DPMSolver](https://huggingface.co/papers/2206.00927) + paper, and the `dpmsolver++` type implements the algorithms in the + [DPMSolver++](https://huggingface.co/papers/2211.01095) paper. It is recommended to use `dpmsolver++` or + `sde-dpmsolver++` with `solver_order=2` for guided sampling like in Stable Diffusion. + solver_type (`str`, defaults to `midpoint`): + Solver type for the second-order solver; can be `midpoint` or `heun`. The solver type slightly affects the + sample quality, especially for a small number of steps. It is recommended to use `midpoint` solvers. + lower_order_final (`bool`, defaults to `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can + stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + euler_at_final (`bool`, defaults to `False`): + Whether to use Euler's method in the final step. It is a trade-off between numerical stability and detail + richness. This can stabilize the sampling of the SDE variant of DPMSolver for small number of inference + steps, but sometimes may result in blurring. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + lambda_min_clipped (`float`, defaults to `-inf`): + Clipping threshold for the minimum value of `lambda(t)` for numerical stability. This is critical for the + cosine (`squaredcos_cap_v2`) noise schedule. + variance_type (`str`, *optional*): + Set to "learned" or "learned_range" for diffusion models that predict variance. If set, the model's output + contains the predicted Gaussian variance. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + solver_order: int = 2, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + algorithm_type: str = "dpmsolver++", + solver_type: str = "midpoint", + lower_order_final: bool = True, + euler_at_final: bool = False, + use_karras_sigmas: Optional[bool] = False, + lambda_min_clipped: float = -float("inf"), + variance_type: Optional[str] = None, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + ): + if algorithm_type in ["dpmsolver", "sde-dpmsolver"]: + deprecation_message = f"algorithm_type {algorithm_type} is deprecated and will be removed in a future version. Choose from `dpmsolver++` or `sde-dpmsolver++` instead" + deprecate("algorithm_types dpmsolver and sde-dpmsolver", "1.0.0", deprecation_message) + + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + # Currently we only support VP-type noise schedule + self.alpha_t = torch.sqrt(self.alphas_cumprod) + self.sigma_t = torch.sqrt(1 - self.alphas_cumprod) + self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t) + self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5 + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # settings for DPM-Solver + if algorithm_type not in ["dpmsolver", "dpmsolver++", "sde-dpmsolver", "sde-dpmsolver++"]: + if algorithm_type == "deis": + self.register_to_config(algorithm_type="dpmsolver++") + else: + raise NotImplementedError(f"{algorithm_type} does is not implemented for {self.__class__}") + + if solver_type not in ["midpoint", "heun"]: + if solver_type in ["logrho", "bh1", "bh2"]: + self.register_to_config(solver_type="midpoint") + else: + raise NotImplementedError(f"{solver_type} does is not implemented for {self.__class__}") + + # setable values + self.num_inference_steps = None + timesteps = np.linspace(0, num_train_timesteps - 1, num_train_timesteps, dtype=np.float32).copy() + self.timesteps = torch.from_numpy(timesteps) + self.model_outputs = [None] * solver_order + self.lower_order_nums = 0 + self._step_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + self.use_karras_sigmas = use_karras_sigmas + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + def set_timesteps(self, num_inference_steps: int = None, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + # Clipping the minimum of all lambda(t) for numerical stability. + # This is critical for cosine (squaredcos_cap_v2) noise schedule. + clipped_idx = torch.searchsorted(torch.flip(self.lambda_t, [0]), self.lambda_min_clipped).item() + self.noisiest_timestep = self.config.num_train_timesteps - 1 - clipped_idx + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, self.noisiest_timestep, num_inference_steps + 1).round()[:-1].copy().astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = (self.noisiest_timestep + 1) // (num_inference_steps + 1) + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps + 1) * step_ratio).round()[:-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.arange(self.noisiest_timestep + 1, 0, -step_ratio).round()[::-1].copy().astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', " + "'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + log_sigmas = np.log(sigmas) + + if self.config.use_karras_sigmas: + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + timesteps = timesteps.copy().astype(np.int64) + sigmas = np.concatenate([sigmas, sigmas[-1:]]).astype(np.float32) + else: + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + sigma_max = ( + (1 - self.alphas_cumprod[self.noisiest_timestep]) / self.alphas_cumprod[self.noisiest_timestep] + ) ** 0.5 + sigmas = np.concatenate([sigmas, [sigma_max]]).astype(np.float32) + + self.sigmas = torch.from_numpy(sigmas) + + # when num_inference_steps == num_train_timesteps, we can end up with + # duplicates in timesteps. + _, unique_indices = np.unique(timesteps, return_index=True) + timesteps = timesteps[np.sort(unique_indices)] + + self.timesteps = torch.from_numpy(timesteps).to(device=device, dtype=torch.int64) + + self.num_inference_steps = len(timesteps) + + self.model_outputs = [ + None, + ] * self.config.solver_order + self.lower_order_nums = 0 + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._sigma_to_alpha_sigma_t + def _sigma_to_alpha_sigma_t(self, sigma): + alpha_t = 1 / ((sigma**2 + 1) ** 0.5) + sigma_t = sigma * alpha_t + + return alpha_t, sigma_t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.convert_model_output + def convert_model_output( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + Convert the model output to the corresponding type the DPMSolver/DPMSolver++ algorithm needs. DPM-Solver is + designed to discretize an integral of the noise prediction model, and DPM-Solver++ is designed to discretize an + integral of the data prediction model. + + + + The algorithm and model type are decoupled. You can use either DPMSolver or DPMSolver++ for both noise + prediction and data prediction models. + + + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError("missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + # DPM-Solver++ needs to solve an integral of the data prediction model. + if self.config.algorithm_type in ["dpmsolver++", "sde-dpmsolver++"]: + if self.config.prediction_type == "epsilon": + # DPM-Solver and DPM-Solver++ only need the "mean" output. + if self.config.variance_type in ["learned", "learned_range"]: + model_output = model_output[:, :3] + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = (sample - sigma_t * model_output) / alpha_t + elif self.config.prediction_type == "sample": + x0_pred = model_output + elif self.config.prediction_type == "v_prediction": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = alpha_t * sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the DPMSolverMultistepScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + + # DPM-Solver needs to solve an integral of the noise prediction model. + elif self.config.algorithm_type in ["dpmsolver", "sde-dpmsolver"]: + if self.config.prediction_type == "epsilon": + # DPM-Solver and DPM-Solver++ only need the "mean" output. + if self.config.variance_type in ["learned", "learned_range"]: + epsilon = model_output[:, :3] + else: + epsilon = model_output + elif self.config.prediction_type == "sample": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + epsilon = (sample - alpha_t * model_output) / sigma_t + elif self.config.prediction_type == "v_prediction": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + epsilon = alpha_t * model_output + sigma_t * sample + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the DPMSolverMultistepScheduler." + ) + + if self.config.thresholding: + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = (sample - sigma_t * epsilon) / alpha_t + x0_pred = self._threshold_sample(x0_pred) + epsilon = (sample - alpha_t * x0_pred) / sigma_t + + return epsilon + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.dpm_solver_first_order_update + def dpm_solver_first_order_update( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + noise: Optional[torch.FloatTensor] = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the first-order DPMSolver (equivalent to DDIM). + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s = self.sigmas[self.step_index + 1], self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s, sigma_s = self._sigma_to_alpha_sigma_t(sigma_s) + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s = torch.log(alpha_s) - torch.log(sigma_s) + + h = lambda_t - lambda_s + if self.config.algorithm_type == "dpmsolver++": + x_t = (sigma_t / sigma_s) * sample - (alpha_t * (torch.exp(-h) - 1.0)) * model_output + elif self.config.algorithm_type == "dpmsolver": + x_t = (alpha_t / alpha_s) * sample - (sigma_t * (torch.exp(h) - 1.0)) * model_output + elif self.config.algorithm_type == "sde-dpmsolver++": + assert noise is not None + x_t = ( + (sigma_t / sigma_s * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * model_output + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + elif self.config.algorithm_type == "sde-dpmsolver": + assert noise is not None + x_t = ( + (alpha_t / alpha_s) * sample + - 2.0 * (sigma_t * (torch.exp(h) - 1.0)) * model_output + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise + ) + return x_t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.multistep_dpm_solver_second_order_update + def multistep_dpm_solver_second_order_update( + self, + model_output_list: List[torch.FloatTensor], + *args, + sample: torch.FloatTensor = None, + noise: Optional[torch.FloatTensor] = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the second-order multistep DPMSolver. + + Args: + model_output_list (`List[torch.FloatTensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s0, sigma_s1 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + + m0, m1 = model_output_list[-1], model_output_list[-2] + + h, h_0 = lambda_t - lambda_s0, lambda_s0 - lambda_s1 + r0 = h_0 / h + D0, D1 = m0, (1.0 / r0) * (m0 - m1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2211.01095 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + - 0.5 * (alpha_t * (torch.exp(-h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 + ) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - 0.5 * (sigma_t * (torch.exp(h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + ) + elif self.config.algorithm_type == "sde-dpmsolver++": + assert noise is not None + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + + 0.5 * (alpha_t * (1 - torch.exp(-2.0 * h))) * D1 + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + + (alpha_t * ((1.0 - torch.exp(-2.0 * h)) / (-2.0 * h) + 1.0)) * D1 + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + elif self.config.algorithm_type == "sde-dpmsolver": + assert noise is not None + if self.config.solver_type == "midpoint": + x_t = ( + (alpha_t / alpha_s0) * sample + - 2.0 * (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * (torch.exp(h) - 1.0)) * D1 + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise + ) + elif self.config.solver_type == "heun": + x_t = ( + (alpha_t / alpha_s0) * sample + - 2.0 * (sigma_t * (torch.exp(h) - 1.0)) * D0 + - 2.0 * (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + + sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise + ) + return x_t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.multistep_dpm_solver_third_order_update + def multistep_dpm_solver_third_order_update( + self, + model_output_list: List[torch.FloatTensor], + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the third-order multistep DPMSolver. + + Args: + model_output_list (`List[torch.FloatTensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.FloatTensor`): + A current instance of a sample created by diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing`sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s0, sigma_s1, sigma_s2 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + self.sigmas[self.step_index - 2], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + alpha_s2, sigma_s2 = self._sigma_to_alpha_sigma_t(sigma_s2) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + lambda_s2 = torch.log(alpha_s2) - torch.log(sigma_s2) + + m0, m1, m2 = model_output_list[-1], model_output_list[-2], model_output_list[-3] + + h, h_0, h_1 = lambda_t - lambda_s0, lambda_s0 - lambda_s1, lambda_s1 - lambda_s2 + r0, r1 = h_0 / h, h_1 / h + D0 = m0 + D1_0, D1_1 = (1.0 / r0) * (m0 - m1), (1.0 / r1) * (m1 - m2) + D1 = D1_0 + (r0 / (r0 + r1)) * (D1_0 - D1_1) + D2 = (1.0 / (r0 + r1)) * (D1_0 - D1_1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 + - (alpha_t * ((torch.exp(-h) - 1.0 + h) / h**2 - 0.5)) * D2 + ) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + x_t = ( + (alpha_t / alpha_s0) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + - (sigma_t * ((torch.exp(h) - 1.0 - h) / h**2 - 0.5)) * D2 + ) + return x_t + + def _init_step_index(self, timestep): + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + + index_candidates = (self.timesteps == timestep).nonzero() + + if len(index_candidates) == 0: + step_index = len(self.timesteps) - 1 + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + elif len(index_candidates) > 1: + step_index = index_candidates[1].item() + else: + step_index = index_candidates[0].item() + + self._step_index = step_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator=None, + variance_noise: Optional[torch.FloatTensor] = None, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the multistep DPMSolver. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + variance_noise (`torch.FloatTensor`): + Alternative to generating noise with `generator` by directly providing the noise for the variance + itself. Useful for methods such as [`CycleDiffusion`]. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Improve numerical stability for small number of steps + lower_order_final = (self.step_index == len(self.timesteps) - 1) and ( + self.config.euler_at_final or (self.config.lower_order_final and len(self.timesteps) < 15) + ) + lower_order_second = ( + (self.step_index == len(self.timesteps) - 2) and self.config.lower_order_final and len(self.timesteps) < 15 + ) + + model_output = self.convert_model_output(model_output, sample=sample) + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.model_outputs[-1] = model_output + + if self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"] and variance_noise is None: + noise = randn_tensor( + model_output.shape, generator=generator, device=model_output.device, dtype=model_output.dtype + ) + elif self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"]: + noise = variance_noise + else: + noise = None + + if self.config.solver_order == 1 or self.lower_order_nums < 1 or lower_order_final: + prev_sample = self.dpm_solver_first_order_update(model_output, sample=sample, noise=noise) + elif self.config.solver_order == 2 or self.lower_order_nums < 2 or lower_order_second: + prev_sample = self.multistep_dpm_solver_second_order_update(self.model_outputs, sample=sample, noise=noise) + else: + prev_sample = self.multistep_dpm_solver_third_order_update(self.model_outputs, sample=sample) + + if self.lower_order_nums < self.config.solver_order: + self.lower_order_nums += 1 + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.scale_model_input + def scale_model_input(self, sample: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + step_indices = [] + for timestep in timesteps: + index_candidates = (schedule_timesteps == timestep).nonzero() + if len(index_candidates) == 0: + step_index = len(schedule_timesteps) - 1 + elif len(index_candidates) > 1: + step_index = index_candidates[1].item() + else: + step_index = index_candidates[0].item() + step_indices.append(step_index) + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + noisy_samples = alpha_t * original_samples + sigma_t * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_sde.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_sde.py new file mode 100755 index 0000000..4721933 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_sde.py @@ -0,0 +1,557 @@ +# Copyright 2024 Katherine Crowson, The HuggingFace Team and hlky. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch +import torchsde + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +class BatchedBrownianTree: + """A wrapper around torchsde.BrownianTree that enables batches of entropy.""" + + def __init__(self, x, t0, t1, seed=None, **kwargs): + t0, t1, self.sign = self.sort(t0, t1) + w0 = kwargs.get("w0", torch.zeros_like(x)) + if seed is None: + seed = torch.randint(0, 2**63 - 1, []).item() + self.batched = True + try: + assert len(seed) == x.shape[0] + w0 = w0[0] + except TypeError: + seed = [seed] + self.batched = False + self.trees = [torchsde.BrownianTree(t0, w0, t1, entropy=s, **kwargs) for s in seed] + + @staticmethod + def sort(a, b): + return (a, b, 1) if a < b else (b, a, -1) + + def __call__(self, t0, t1): + t0, t1, sign = self.sort(t0, t1) + w = torch.stack([tree(t0, t1) for tree in self.trees]) * (self.sign * sign) + return w if self.batched else w[0] + + +class BrownianTreeNoiseSampler: + """A noise sampler backed by a torchsde.BrownianTree. + + Args: + x (Tensor): The tensor whose shape, device and dtype to use to generate + random samples. + sigma_min (float): The low end of the valid interval. + sigma_max (float): The high end of the valid interval. + seed (int or List[int]): The random seed. If a list of seeds is + supplied instead of a single integer, then the noise sampler will use one BrownianTree per batch item, each + with its own seed. + transform (callable): A function that maps sigma to the sampler's + internal timestep. + """ + + def __init__(self, x, sigma_min, sigma_max, seed=None, transform=lambda x: x): + self.transform = transform + t0, t1 = self.transform(torch.as_tensor(sigma_min)), self.transform(torch.as_tensor(sigma_max)) + self.tree = BatchedBrownianTree(x, t0, t1, seed) + + def __call__(self, sigma, sigma_next): + t0, t1 = self.transform(torch.as_tensor(sigma)), self.transform(torch.as_tensor(sigma_next)) + return self.tree(t0, t1) / (t1 - t0).abs().sqrt() + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class DPMSolverSDEScheduler(SchedulerMixin, ConfigMixin): + """ + DPMSolverSDEScheduler implements the stochastic sampler from the [Elucidating the Design Space of Diffusion-Based + Generative Models](https://huggingface.co/papers/2206.00364) paper. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.00085): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.012): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear` or `scaled_linear`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + noise_sampler_seed (`int`, *optional*, defaults to `None`): + The random seed to use for the noise sampler. If `None`, a random seed is generated. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 2 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.00085, # sensible defaults + beta_end: float = 0.012, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + prediction_type: str = "epsilon", + use_karras_sigmas: Optional[bool] = False, + noise_sampler_seed: Optional[int] = None, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + # set all values + self.set_timesteps(num_train_timesteps, None, num_train_timesteps) + self.use_karras_sigmas = use_karras_sigmas + self.noise_sampler = None + self.noise_sampler_seed = noise_sampler_seed + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + @property + def init_noise_sigma(self): + # standard deviation of the initial noise distribution + if self.config.timestep_spacing in ["linspace", "trailing"]: + return self.sigmas.max() + + return (self.sigmas.max() ** 2 + 1) ** 0.5 + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def scale_model_input( + self, + sample: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + ) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + if self.step_index is None: + self._init_step_index(timestep) + + sigma = self.sigmas[self.step_index] + sigma_input = sigma if self.state_in_first_order else self.mid_point_sigma + sample = sample / ((sigma_input**2 + 1) ** 0.5) + return sample + + def set_timesteps( + self, + num_inference_steps: int, + device: Union[str, torch.device] = None, + num_train_timesteps: Optional[int] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.num_inference_steps = num_inference_steps + + num_train_timesteps = num_train_timesteps or self.config.num_train_timesteps + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = np.linspace(0, num_train_timesteps - 1, num_inference_steps, dtype=float)[::-1].copy() + elif self.config.timestep_spacing == "leading": + step_ratio = num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(float) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(num_train_timesteps, 0, -step_ratio)).round().copy().astype(float) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + log_sigmas = np.log(sigmas) + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + + if self.use_karras_sigmas: + sigmas = self._convert_to_karras(in_sigmas=sigmas) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]) + + second_order_timesteps = self._second_order_timesteps(sigmas, log_sigmas) + + sigmas = np.concatenate([sigmas, [0.0]]).astype(np.float32) + sigmas = torch.from_numpy(sigmas).to(device=device) + self.sigmas = torch.cat([sigmas[:1], sigmas[1:-1].repeat_interleave(2), sigmas[-1:]]) + + timesteps = torch.from_numpy(timesteps) + second_order_timesteps = torch.from_numpy(second_order_timesteps) + timesteps = torch.cat([timesteps[:1], timesteps[1:].repeat_interleave(2)]) + timesteps[1::2] = second_order_timesteps + + if str(device).startswith("mps"): + # mps does not support float64 + self.timesteps = timesteps.to(device, dtype=torch.float32) + else: + self.timesteps = timesteps.to(device=device) + + # empty first order variables + self.sample = None + self.mid_point_sigma = None + + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + self.noise_sampler = None + + def _second_order_timesteps(self, sigmas, log_sigmas): + def sigma_fn(_t): + return np.exp(-_t) + + def t_fn(_sigma): + return -np.log(_sigma) + + midpoint_ratio = 0.5 + t = t_fn(sigmas) + delta_time = np.diff(t) + t_proposed = t[:-1] + delta_time * midpoint_ratio + sig_proposed = sigma_fn(t_proposed) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sig_proposed]) + return timesteps + + # copied from diffusers.schedulers.scheduling_euler_discrete._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # copied from diffusers.schedulers.scheduling_euler_discrete._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.FloatTensor) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + sigma_min: float = in_sigmas[-1].item() + sigma_max: float = in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, self.num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + @property + def state_in_first_order(self): + return self.sample is None + + def step( + self, + model_output: Union[torch.FloatTensor, np.ndarray], + timestep: Union[float, torch.FloatTensor], + sample: Union[torch.FloatTensor, np.ndarray], + return_dict: bool = True, + s_noise: float = 1.0, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor` or `np.ndarray`): + The direct output from learned diffusion model. + timestep (`float` or `torch.FloatTensor`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor` or `np.ndarray`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or tuple. + s_noise (`float`, *optional*, defaults to 1.0): + Scaling factor for noise added to the sample. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + """ + if self.step_index is None: + self._init_step_index(timestep) + + # Create a noise sampler if it hasn't been created yet + if self.noise_sampler is None: + min_sigma, max_sigma = self.sigmas[self.sigmas > 0].min(), self.sigmas.max() + self.noise_sampler = BrownianTreeNoiseSampler(sample, min_sigma, max_sigma, self.noise_sampler_seed) + + # Define functions to compute sigma and t from each other + def sigma_fn(_t: torch.FloatTensor) -> torch.FloatTensor: + return _t.neg().exp() + + def t_fn(_sigma: torch.FloatTensor) -> torch.FloatTensor: + return _sigma.log().neg() + + if self.state_in_first_order: + sigma = self.sigmas[self.step_index] + sigma_next = self.sigmas[self.step_index + 1] + else: + # 2nd order + sigma = self.sigmas[self.step_index - 1] + sigma_next = self.sigmas[self.step_index] + + # Set the midpoint and step size for the current step + midpoint_ratio = 0.5 + t, t_next = t_fn(sigma), t_fn(sigma_next) + delta_time = t_next - t + t_proposed = t + delta_time * midpoint_ratio + + # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise + if self.config.prediction_type == "epsilon": + sigma_input = sigma if self.state_in_first_order else sigma_fn(t_proposed) + pred_original_sample = sample - sigma_input * model_output + elif self.config.prediction_type == "v_prediction": + sigma_input = sigma if self.state_in_first_order else sigma_fn(t_proposed) + pred_original_sample = model_output * (-sigma_input / (sigma_input**2 + 1) ** 0.5) + ( + sample / (sigma_input**2 + 1) + ) + elif self.config.prediction_type == "sample": + raise NotImplementedError("prediction_type not implemented yet: sample") + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`" + ) + + if sigma_next == 0: + derivative = (sample - pred_original_sample) / sigma + dt = sigma_next - sigma + prev_sample = sample + derivative * dt + else: + if self.state_in_first_order: + t_next = t_proposed + else: + sample = self.sample + + sigma_from = sigma_fn(t) + sigma_to = sigma_fn(t_next) + sigma_up = min(sigma_to, (sigma_to**2 * (sigma_from**2 - sigma_to**2) / sigma_from**2) ** 0.5) + sigma_down = (sigma_to**2 - sigma_up**2) ** 0.5 + ancestral_t = t_fn(sigma_down) + prev_sample = (sigma_fn(ancestral_t) / sigma_fn(t)) * sample - ( + t - ancestral_t + ).expm1() * pred_original_sample + prev_sample = prev_sample + self.noise_sampler(sigma_fn(t), sigma_fn(t_next)) * s_noise * sigma_up + + if self.state_in_first_order: + # store for 2nd order step + self.sample = sample + self.mid_point_sigma = sigma_fn(t_next) + else: + # free for "first order mode" + self.sample = None + self.mid_point_sigma = None + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + noisy_samples = original_samples + noise * sigma + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py new file mode 100755 index 0000000..7bb201d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_dpmsolver_singlestep.py @@ -0,0 +1,979 @@ +# Copyright 2024 TSAIL Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import deprecate, logging +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class DPMSolverSinglestepScheduler(SchedulerMixin, ConfigMixin): + """ + `DPMSolverSinglestepScheduler` is a fast dedicated high-order solver for diffusion ODEs. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + solver_order (`int`, defaults to 2): + The DPMSolver order which can be `1` or `2` or `3`. It is recommended to use `solver_order=2` for guided + sampling, and `solver_order=3` for unconditional sampling. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and + `algorithm_type="dpmsolver++"`. + algorithm_type (`str`, defaults to `dpmsolver++`): + Algorithm type for the solver; can be `dpmsolver` or `dpmsolver++`. The + `dpmsolver` type implements the algorithms in the [DPMSolver](https://huggingface.co/papers/2206.00927) + paper, and the `dpmsolver++` type implements the algorithms in the + [DPMSolver++](https://huggingface.co/papers/2211.01095) paper. It is recommended to use `dpmsolver++` or + `sde-dpmsolver++` with `solver_order=2` for guided sampling like in Stable Diffusion. + solver_type (`str`, defaults to `midpoint`): + Solver type for the second-order solver; can be `midpoint` or `heun`. The solver type slightly affects the + sample quality, especially for a small number of steps. It is recommended to use `midpoint` solvers. + lower_order_final (`bool`, defaults to `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can + stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + final_sigmas_type (`str`, *optional*, defaults to `"zero"`): + The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final sigma + is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0. + lambda_min_clipped (`float`, defaults to `-inf`): + Clipping threshold for the minimum value of `lambda(t)` for numerical stability. This is critical for the + cosine (`squaredcos_cap_v2`) noise schedule. + variance_type (`str`, *optional*): + Set to "learned" or "learned_range" for diffusion models that predict variance. If set, the model's output + contains the predicted Gaussian variance. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[np.ndarray] = None, + solver_order: int = 2, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + algorithm_type: str = "dpmsolver++", + solver_type: str = "midpoint", + lower_order_final: bool = False, + use_karras_sigmas: Optional[bool] = False, + final_sigmas_type: Optional[str] = "zero", # "zero", "sigma_min" + lambda_min_clipped: float = -float("inf"), + variance_type: Optional[str] = None, + ): + if algorithm_type == "dpmsolver": + deprecation_message = "algorithm_type `dpmsolver` is deprecated and will be removed in a future version. Choose from `dpmsolver++` or `sde-dpmsolver++` instead" + deprecate("algorithm_types=dpmsolver", "1.0.0", deprecation_message) + + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + # Currently we only support VP-type noise schedule + self.alpha_t = torch.sqrt(self.alphas_cumprod) + self.sigma_t = torch.sqrt(1 - self.alphas_cumprod) + self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t) + self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5 + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # settings for DPM-Solver + if algorithm_type not in ["dpmsolver", "dpmsolver++"]: + if algorithm_type == "deis": + self.register_to_config(algorithm_type="dpmsolver++") + else: + raise NotImplementedError(f"{algorithm_type} does is not implemented for {self.__class__}") + if solver_type not in ["midpoint", "heun"]: + if solver_type in ["logrho", "bh1", "bh2"]: + self.register_to_config(solver_type="midpoint") + else: + raise NotImplementedError(f"{solver_type} does is not implemented for {self.__class__}") + + if algorithm_type != "dpmsolver++" and final_sigmas_type == "zero": + raise ValueError( + f"`final_sigmas_type` {final_sigmas_type} is not supported for `algorithm_type` {algorithm_type}. Please chooose `sigma_min` instead." + ) + + # setable values + self.num_inference_steps = None + timesteps = np.linspace(0, num_train_timesteps - 1, num_train_timesteps, dtype=np.float32)[::-1].copy() + self.timesteps = torch.from_numpy(timesteps) + self.model_outputs = [None] * solver_order + self.sample = None + self.order_list = self.get_order_list(num_train_timesteps) + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + def get_order_list(self, num_inference_steps: int) -> List[int]: + """ + Computes the solver order at each time step. + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + """ + steps = num_inference_steps + order = self.config.solver_order + if order > 3: + raise ValueError("Order > 3 is not supported by this scheduler") + if self.config.lower_order_final: + if order == 3: + if steps % 3 == 0: + orders = [1, 2, 3] * (steps // 3 - 1) + [1, 2] + [1] + elif steps % 3 == 1: + orders = [1, 2, 3] * (steps // 3) + [1] + else: + orders = [1, 2, 3] * (steps // 3) + [1, 2] + elif order == 2: + if steps % 2 == 0: + orders = [1, 2] * (steps // 2 - 1) + [1, 1] + else: + orders = [1, 2] * (steps // 2) + [1] + elif order == 1: + orders = [1] * steps + else: + if order == 3: + orders = [1, 2, 3] * (steps // 3) + elif order == 2: + orders = [1, 2] * (steps // 2) + elif order == 1: + orders = [1] * steps + return orders + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.num_inference_steps = num_inference_steps + # Clipping the minimum of all lambda(t) for numerical stability. + # This is critical for cosine (squaredcos_cap_v2) noise schedule. + clipped_idx = torch.searchsorted(torch.flip(self.lambda_t, [0]), self.config.lambda_min_clipped) + timesteps = ( + np.linspace(0, self.config.num_train_timesteps - 1 - clipped_idx, num_inference_steps + 1) + .round()[::-1][:-1] + .copy() + .astype(np.int64) + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + if self.config.use_karras_sigmas: + log_sigmas = np.log(sigmas) + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + else: + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + + if self.config.final_sigmas_type == "sigma_min": + sigma_last = ((1 - self.alphas_cumprod[0]) / self.alphas_cumprod[0]) ** 0.5 + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f" `final_sigmas_type` must be one of `sigma_min` or `zero`, but got {self.config.final_sigmas_type}" + ) + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + + self.sigmas = torch.from_numpy(sigmas).to(device=device) + + self.timesteps = torch.from_numpy(timesteps).to(device=device, dtype=torch.int64) + self.model_outputs = [None] * self.config.solver_order + self.sample = None + + if not self.config.lower_order_final and num_inference_steps % self.config.solver_order != 0: + logger.warning( + "Changing scheduler {self.config} to have `lower_order_final` set to True to handle uneven amount of inference steps. Please make sure to always use an even number of `num_inference steps when using `lower_order_final=False`." + ) + self.register_to_config(lower_order_final=True) + + if not self.config.lower_order_final and self.config.final_sigmas_type == "zero": + logger.warning( + " `last_sigmas_type='zero'` is not supported for `lower_order_final=False`. Changing scheduler {self.config} to have `lower_order_final` set to True." + ) + self.register_to_config(lower_order_final=True) + + self.order_list = self.get_order_list(num_inference_steps) + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._sigma_to_alpha_sigma_t + def _sigma_to_alpha_sigma_t(self, sigma): + alpha_t = 1 / ((sigma**2 + 1) ** 0.5) + sigma_t = sigma * alpha_t + + return alpha_t, sigma_t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + def convert_model_output( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + Convert the model output to the corresponding type the DPMSolver/DPMSolver++ algorithm needs. DPM-Solver is + designed to discretize an integral of the noise prediction model, and DPM-Solver++ is designed to discretize an + integral of the data prediction model. + + + + The algorithm and model type are decoupled. You can use either DPMSolver or DPMSolver++ for both noise + prediction and data prediction models. + + + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError("missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + # DPM-Solver++ needs to solve an integral of the data prediction model. + if self.config.algorithm_type == "dpmsolver++": + if self.config.prediction_type == "epsilon": + # DPM-Solver and DPM-Solver++ only need the "mean" output. + if self.config.variance_type in ["learned_range"]: + model_output = model_output[:, :3] + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = (sample - sigma_t * model_output) / alpha_t + elif self.config.prediction_type == "sample": + x0_pred = model_output + elif self.config.prediction_type == "v_prediction": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + x0_pred = alpha_t * sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the DPMSolverSinglestepScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + # DPM-Solver needs to solve an integral of the noise prediction model. + elif self.config.algorithm_type == "dpmsolver": + if self.config.prediction_type == "epsilon": + # DPM-Solver and DPM-Solver++ only need the "mean" output. + if self.config.variance_type in ["learned_range"]: + model_output = model_output[:, :3] + return model_output + elif self.config.prediction_type == "sample": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + epsilon = (sample - alpha_t * model_output) / sigma_t + return epsilon + elif self.config.prediction_type == "v_prediction": + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + epsilon = alpha_t * model_output + sigma_t * sample + return epsilon + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the DPMSolverSinglestepScheduler." + ) + + def dpm_solver_first_order_update( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the first-order DPMSolver (equivalent to DDIM). + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + prev_timestep (`int`): + The previous discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + sigma_t, sigma_s = self.sigmas[self.step_index + 1], self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s, sigma_s = self._sigma_to_alpha_sigma_t(sigma_s) + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s = torch.log(alpha_s) - torch.log(sigma_s) + h = lambda_t - lambda_s + if self.config.algorithm_type == "dpmsolver++": + x_t = (sigma_t / sigma_s) * sample - (alpha_t * (torch.exp(-h) - 1.0)) * model_output + elif self.config.algorithm_type == "dpmsolver": + x_t = (alpha_t / alpha_s) * sample - (sigma_t * (torch.exp(h) - 1.0)) * model_output + return x_t + + def singlestep_dpm_solver_second_order_update( + self, + model_output_list: List[torch.FloatTensor], + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the second-order singlestep DPMSolver that computes the solution at time `prev_timestep` from the + time `timestep_list[-2]`. + + Args: + model_output_list (`List[torch.FloatTensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + timestep (`int`): + The current and latter discrete timestep in the diffusion chain. + prev_timestep (`int`): + The previous discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + sigma_t, sigma_s0, sigma_s1 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + + m0, m1 = model_output_list[-1], model_output_list[-2] + + h, h_0 = lambda_t - lambda_s1, lambda_s0 - lambda_s1 + r0 = h_0 / h + D0, D1 = m1, (1.0 / r0) * (m0 - m1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2211.01095 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s1) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + - 0.5 * (alpha_t * (torch.exp(-h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s1) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 + ) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (alpha_t / alpha_s1) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - 0.5 * (sigma_t * (torch.exp(h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (alpha_t / alpha_s1) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + ) + return x_t + + def singlestep_dpm_solver_third_order_update( + self, + model_output_list: List[torch.FloatTensor], + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the third-order singlestep DPMSolver that computes the solution at time `prev_timestep` from the + time `timestep_list[-3]`. + + Args: + model_output_list (`List[torch.FloatTensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + timestep (`int`): + The current and latter discrete timestep in the diffusion chain. + prev_timestep (`int`): + The previous discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing`sample` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma_t, sigma_s0, sigma_s1, sigma_s2 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + self.sigmas[self.step_index - 2], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + alpha_s2, sigma_s2 = self._sigma_to_alpha_sigma_t(sigma_s2) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + lambda_s2 = torch.log(alpha_s2) - torch.log(sigma_s2) + + m0, m1, m2 = model_output_list[-1], model_output_list[-2], model_output_list[-3] + + h, h_0, h_1 = lambda_t - lambda_s2, lambda_s0 - lambda_s2, lambda_s1 - lambda_s2 + r0, r1 = h_0 / h, h_1 / h + D0 = m2 + D1_0, D1_1 = (1.0 / r1) * (m1 - m2), (1.0 / r0) * (m0 - m2) + D1 = (r0 * D1_0 - r1 * D1_1) / (r0 - r1) + D2 = 2.0 * (D1_1 - D1_0) / (r0 - r1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s2) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1_1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s2) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 + - (alpha_t * ((torch.exp(-h) - 1.0 + h) / h**2 - 0.5)) * D2 + ) + elif self.config.algorithm_type == "dpmsolver": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (alpha_t / alpha_s2) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1_1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (alpha_t / alpha_s2) * sample + - (sigma_t * (torch.exp(h) - 1.0)) * D0 + - (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 + - (sigma_t * ((torch.exp(h) - 1.0 - h) / h**2 - 0.5)) * D2 + ) + return x_t + + def singlestep_dpm_solver_update( + self, + model_output_list: List[torch.FloatTensor], + *args, + sample: torch.FloatTensor = None, + order: int = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the singlestep DPMSolver. + + Args: + model_output_list (`List[torch.FloatTensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + timestep (`int`): + The current and latter discrete timestep in the diffusion chain. + prev_timestep (`int`): + The previous discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by diffusion process. + order (`int`): + The solver order at this step. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + timestep_list = args[0] if len(args) > 0 else kwargs.pop("timestep_list", None) + prev_timestep = args[1] if len(args) > 1 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 2: + sample = args[2] + else: + raise ValueError(" missing`sample` as a required keyward argument") + if order is None: + if len(args) > 3: + order = args[3] + else: + raise ValueError(" missing `order` as a required keyward argument") + if timestep_list is not None: + deprecate( + "timestep_list", + "1.0.0", + "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + if order == 1: + return self.dpm_solver_first_order_update(model_output_list[-1], sample=sample) + elif order == 2: + return self.singlestep_dpm_solver_second_order_update(model_output_list, sample=sample) + elif order == 3: + return self.singlestep_dpm_solver_third_order_update(model_output_list, sample=sample) + else: + raise ValueError(f"Order must be 1, 2, 3, got {order}") + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + index_candidates = (schedule_timesteps == timestep).nonzero() + + if len(index_candidates) == 0: + step_index = len(self.timesteps) - 1 + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + elif len(index_candidates) > 1: + step_index = index_candidates[1].item() + else: + step_index = index_candidates[0].item() + + return step_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index + def _init_step_index(self, timestep): + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the singlestep DPMSolver. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + model_output = self.convert_model_output(model_output, sample=sample) + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.model_outputs[-1] = model_output + + order = self.order_list[self.step_index] + + # For img2img denoising might start with order>1 which is not possible + # In this case make sure that the first two steps are both order=1 + while self.model_outputs[-order] is None: + order -= 1 + + # For single-step solvers, we use the initial value at each time with order = 1. + if order == 1: + self.sample = sample + + prev_sample = self.singlestep_dpm_solver_update(self.model_outputs, sample=self.sample, order=order) + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # begin_index is None when the scheduler is used for training + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + noisy_samples = alpha_t * original_samples + sigma_t * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py new file mode 100755 index 0000000..5fea89b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_edm_dpmsolver_multistep.py @@ -0,0 +1,683 @@ +# Copyright 2024 TSAIL Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/LuChengTHU/dpm-solver and https://github.com/NVlabs/edm + +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import SchedulerMixin, SchedulerOutput + + +class EDMDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin): + """ + Implements DPMSolverMultistepScheduler in EDM formulation as presented in Karras et al. 2022 [1]. + `EDMDPMSolverMultistepScheduler` is a fast dedicated high-order solver for diffusion ODEs. + + [1] Karras, Tero, et al. "Elucidating the Design Space of Diffusion-Based Generative Models." + https://arxiv.org/abs/2206.00364 + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + sigma_min (`float`, *optional*, defaults to 0.002): + Minimum noise magnitude in the sigma schedule. This was set to 0.002 in the EDM paper [1]; a reasonable + range is [0, 10]. + sigma_max (`float`, *optional*, defaults to 80.0): + Maximum noise magnitude in the sigma schedule. This was set to 80.0 in the EDM paper [1]; a reasonable + range is [0.2, 80.0]. + sigma_data (`float`, *optional*, defaults to 0.5): + The standard deviation of the data distribution. This is set to 0.5 in the EDM paper [1]. + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + solver_order (`int`, defaults to 2): + The DPMSolver order which can be `1` or `2` or `3`. It is recommended to use `solver_order=2` for guided + sampling, and `solver_order=3` for unconditional sampling. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and + `algorithm_type="dpmsolver++"`. + algorithm_type (`str`, defaults to `dpmsolver++`): + Algorithm type for the solver; can be `dpmsolver++` or `sde-dpmsolver++`. The + `dpmsolver++` type implements the algorithms in the + [DPMSolver++](https://huggingface.co/papers/2211.01095) paper. It is recommended to use `dpmsolver++` or + `sde-dpmsolver++` with `solver_order=2` for guided sampling like in Stable Diffusion. + solver_type (`str`, defaults to `midpoint`): + Solver type for the second-order solver; can be `midpoint` or `heun`. The solver type slightly affects the + sample quality, especially for a small number of steps. It is recommended to use `midpoint` solvers. + lower_order_final (`bool`, defaults to `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can + stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + euler_at_final (`bool`, defaults to `False`): + Whether to use Euler's method in the final step. It is a trade-off between numerical stability and detail + richness. This can stabilize the sampling of the SDE variant of DPMSolver for small number of inference + steps, but sometimes may result in blurring. + final_sigmas_type (`str`, defaults to `"zero"`): + The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final sigma + is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0. + """ + + _compatibles = [] + order = 1 + + @register_to_config + def __init__( + self, + sigma_min: float = 0.002, + sigma_max: float = 80.0, + sigma_data: float = 0.5, + num_train_timesteps: int = 1000, + prediction_type: str = "epsilon", + rho: float = 7.0, + solver_order: int = 2, + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + algorithm_type: str = "dpmsolver++", + solver_type: str = "midpoint", + lower_order_final: bool = True, + euler_at_final: bool = False, + final_sigmas_type: Optional[str] = "zero", # "zero", "sigma_min" + ): + # settings for DPM-Solver + if algorithm_type not in ["dpmsolver++", "sde-dpmsolver++"]: + if algorithm_type == "deis": + self.register_to_config(algorithm_type="dpmsolver++") + else: + raise NotImplementedError(f"{algorithm_type} is not implemented for {self.__class__}") + + if solver_type not in ["midpoint", "heun"]: + if solver_type in ["logrho", "bh1", "bh2"]: + self.register_to_config(solver_type="midpoint") + else: + raise NotImplementedError(f"{solver_type} does is not implemented for {self.__class__}") + + if algorithm_type not in ["dpmsolver++", "sde-dpmsolver++"] and final_sigmas_type == "zero": + raise ValueError( + f"`final_sigmas_type` {final_sigmas_type} is not supported for `algorithm_type` {algorithm_type}. Please choose `sigma_min` instead." + ) + + ramp = torch.linspace(0, 1, num_train_timesteps) + sigmas = self._compute_sigmas(ramp) + self.timesteps = self.precondition_noise(sigmas) + + self.sigmas = self.sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)]) + + # setable values + self.num_inference_steps = None + self.model_outputs = [None] * solver_order + self.lower_order_nums = 0 + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def init_noise_sigma(self): + # standard deviation of the initial noise distribution + return (self.config.sigma_max**2 + 1) ** 0.5 + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + # Copied from diffusers.schedulers.scheduling_edm_euler.EDMEulerScheduler.precondition_inputs + def precondition_inputs(self, sample, sigma): + c_in = 1 / ((sigma**2 + self.config.sigma_data**2) ** 0.5) + scaled_sample = sample * c_in + return scaled_sample + + # Copied from diffusers.schedulers.scheduling_edm_euler.EDMEulerScheduler.precondition_noise + def precondition_noise(self, sigma): + if not isinstance(sigma, torch.Tensor): + sigma = torch.tensor([sigma]) + + c_noise = 0.25 * torch.log(sigma) + + return c_noise + + # Copied from diffusers.schedulers.scheduling_edm_euler.EDMEulerScheduler.precondition_outputs + def precondition_outputs(self, sample, model_output, sigma): + sigma_data = self.config.sigma_data + c_skip = sigma_data**2 / (sigma**2 + sigma_data**2) + + if self.config.prediction_type == "epsilon": + c_out = sigma * sigma_data / (sigma**2 + sigma_data**2) ** 0.5 + elif self.config.prediction_type == "v_prediction": + c_out = -sigma * sigma_data / (sigma**2 + sigma_data**2) ** 0.5 + else: + raise ValueError(f"Prediction type {self.config.prediction_type} is not supported.") + + denoised = c_skip * sample + c_out * model_output + + return denoised + + # Copied from diffusers.schedulers.scheduling_edm_euler.EDMEulerScheduler.scale_model_input + def scale_model_input( + self, sample: torch.FloatTensor, timestep: Union[float, torch.FloatTensor] + ) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. Scales the denoising model input by `(sigma**2 + 1) ** 0.5` to match the Euler algorithm. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + if self.step_index is None: + self._init_step_index(timestep) + + sigma = self.sigmas[self.step_index] + sample = self.precondition_inputs(sample, sigma) + + self.is_scale_input_called = True + return sample + + def set_timesteps(self, num_inference_steps: int = None, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + + self.num_inference_steps = num_inference_steps + + ramp = np.linspace(0, 1, self.num_inference_steps) + sigmas = self._compute_sigmas(ramp) + + sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32, device=device) + self.timesteps = self.precondition_noise(sigmas) + + if self.config.final_sigmas_type == "sigma_min": + sigma_last = self.config.sigma_min + elif self.config.final_sigmas_type == "zero": + sigma_last = 0 + else: + raise ValueError( + f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}" + ) + + self.sigmas = torch.cat([sigmas, torch.tensor([sigma_last], dtype=torch.float32, device=device)]) + + self.model_outputs = [ + None, + ] * self.config.solver_order + self.lower_order_nums = 0 + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Taken from https://github.com/crowsonkb/k-diffusion/blob/686dbad0f39640ea25c8a8c6a6e56bb40eacefa2/k_diffusion/sampling.py#L17 + def _compute_sigmas(self, ramp, sigma_min=None, sigma_max=None) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + sigma_min = sigma_min or self.config.sigma_min + sigma_max = sigma_max or self.config.sigma_max + + rho = self.config.rho + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + def _sigma_to_alpha_sigma_t(self, sigma): + alpha_t = torch.tensor(1) # Inputs are pre-scaled before going into unet, so alpha_t = 1 + sigma_t = sigma + + return alpha_t, sigma_t + + def convert_model_output( + self, + model_output: torch.FloatTensor, + sample: torch.FloatTensor = None, + ) -> torch.FloatTensor: + """ + Convert the model output to the corresponding type the DPMSolver/DPMSolver++ algorithm needs. DPM-Solver is + designed to discretize an integral of the noise prediction model, and DPM-Solver++ is designed to discretize an + integral of the data prediction model. + + + + The algorithm and model type are decoupled. You can use either DPMSolver or DPMSolver++ for both noise + prediction and data prediction models. + + + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The converted model output. + """ + sigma = self.sigmas[self.step_index] + x0_pred = self.precondition_outputs(sample, model_output, sigma) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + + def dpm_solver_first_order_update( + self, + model_output: torch.FloatTensor, + sample: torch.FloatTensor = None, + noise: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + """ + One step for the first-order DPMSolver (equivalent to DDIM). + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + sigma_t, sigma_s = self.sigmas[self.step_index + 1], self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s, sigma_s = self._sigma_to_alpha_sigma_t(sigma_s) + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s = torch.log(alpha_s) - torch.log(sigma_s) + + h = lambda_t - lambda_s + if self.config.algorithm_type == "dpmsolver++": + x_t = (sigma_t / sigma_s) * sample - (alpha_t * (torch.exp(-h) - 1.0)) * model_output + elif self.config.algorithm_type == "sde-dpmsolver++": + assert noise is not None + x_t = ( + (sigma_t / sigma_s * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * model_output + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + + return x_t + + def multistep_dpm_solver_second_order_update( + self, + model_output_list: List[torch.FloatTensor], + sample: torch.FloatTensor = None, + noise: Optional[torch.FloatTensor] = None, + ) -> torch.FloatTensor: + """ + One step for the second-order multistep DPMSolver. + + Args: + model_output_list (`List[torch.FloatTensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + sigma_t, sigma_s0, sigma_s1 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + + m0, m1 = model_output_list[-1], model_output_list[-2] + + h, h_0 = lambda_t - lambda_s0, lambda_s0 - lambda_s1 + r0 = h_0 / h + D0, D1 = m0, (1.0 / r0) * (m0 - m1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2211.01095 for detailed derivations + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + - 0.5 * (alpha_t * (torch.exp(-h) - 1.0)) * D1 + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 + ) + elif self.config.algorithm_type == "sde-dpmsolver++": + assert noise is not None + if self.config.solver_type == "midpoint": + x_t = ( + (sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + + 0.5 * (alpha_t * (1 - torch.exp(-2.0 * h))) * D1 + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + elif self.config.solver_type == "heun": + x_t = ( + (sigma_t / sigma_s0 * torch.exp(-h)) * sample + + (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + + (alpha_t * ((1.0 - torch.exp(-2.0 * h)) / (-2.0 * h) + 1.0)) * D1 + + sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise + ) + + return x_t + + def multistep_dpm_solver_third_order_update( + self, + model_output_list: List[torch.FloatTensor], + sample: torch.FloatTensor = None, + ) -> torch.FloatTensor: + """ + One step for the third-order multistep DPMSolver. + + Args: + model_output_list (`List[torch.FloatTensor]`): + The direct outputs from learned diffusion model at current and latter timesteps. + sample (`torch.FloatTensor`): + A current instance of a sample created by diffusion process. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + sigma_t, sigma_s0, sigma_s1, sigma_s2 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + self.sigmas[self.step_index - 2], + ) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1) + alpha_s2, sigma_s2 = self._sigma_to_alpha_sigma_t(sigma_s2) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1) + lambda_s2 = torch.log(alpha_s2) - torch.log(sigma_s2) + + m0, m1, m2 = model_output_list[-1], model_output_list[-2], model_output_list[-3] + + h, h_0, h_1 = lambda_t - lambda_s0, lambda_s0 - lambda_s1, lambda_s1 - lambda_s2 + r0, r1 = h_0 / h, h_1 / h + D0 = m0 + D1_0, D1_1 = (1.0 / r0) * (m0 - m1), (1.0 / r1) * (m1 - m2) + D1 = D1_0 + (r0 / (r0 + r1)) * (D1_0 - D1_1) + D2 = (1.0 / (r0 + r1)) * (D1_0 - D1_1) + if self.config.algorithm_type == "dpmsolver++": + # See https://arxiv.org/abs/2206.00927 for detailed derivations + x_t = ( + (sigma_t / sigma_s0) * sample + - (alpha_t * (torch.exp(-h) - 1.0)) * D0 + + (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 + - (alpha_t * ((torch.exp(-h) - 1.0 + h) / h**2 - 0.5)) * D2 + ) + + return x_t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + index_candidates = (schedule_timesteps == timestep).nonzero() + + if len(index_candidates) == 0: + step_index = len(self.timesteps) - 1 + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + elif len(index_candidates) > 1: + step_index = index_candidates[1].item() + else: + step_index = index_candidates[0].item() + + return step_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index + def _init_step_index(self, timestep): + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator=None, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the multistep DPMSolver. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Improve numerical stability for small number of steps + lower_order_final = (self.step_index == len(self.timesteps) - 1) and ( + self.config.euler_at_final + or (self.config.lower_order_final and len(self.timesteps) < 15) + or self.config.final_sigmas_type == "zero" + ) + lower_order_second = ( + (self.step_index == len(self.timesteps) - 2) and self.config.lower_order_final and len(self.timesteps) < 15 + ) + + model_output = self.convert_model_output(model_output, sample=sample) + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.model_outputs[-1] = model_output + + if self.config.algorithm_type == "sde-dpmsolver++": + noise = randn_tensor( + model_output.shape, generator=generator, device=model_output.device, dtype=model_output.dtype + ) + else: + noise = None + + if self.config.solver_order == 1 or self.lower_order_nums < 1 or lower_order_final: + prev_sample = self.dpm_solver_first_order_update(model_output, sample=sample, noise=noise) + elif self.config.solver_order == 2 or self.lower_order_nums < 2 or lower_order_second: + prev_sample = self.multistep_dpm_solver_second_order_update(self.model_outputs, sample=sample, noise=noise) + else: + prev_sample = self.multistep_dpm_solver_third_order_update(self.model_outputs, sample=sample) + + if self.lower_order_nums < self.config.solver_order: + self.lower_order_nums += 1 + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + noisy_samples = original_samples + noise * sigma + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_edm_euler.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_edm_euler.py new file mode 100755 index 0000000..e62a486 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_edm_euler.py @@ -0,0 +1,381 @@ +# Copyright 2024 Katherine Crowson and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput, logging +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import SchedulerMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->EulerDiscrete +class EDMEulerSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +class EDMEulerScheduler(SchedulerMixin, ConfigMixin): + """ + Implements the Euler scheduler in EDM formulation as presented in Karras et al. 2022 [1]. + + [1] Karras, Tero, et al. "Elucidating the Design Space of Diffusion-Based Generative Models." + https://arxiv.org/abs/2206.00364 + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + sigma_min (`float`, *optional*, defaults to 0.002): + Minimum noise magnitude in the sigma schedule. This was set to 0.002 in the EDM paper [1]; a reasonable + range is [0, 10]. + sigma_max (`float`, *optional*, defaults to 80.0): + Maximum noise magnitude in the sigma schedule. This was set to 80.0 in the EDM paper [1]; a reasonable + range is [0.2, 80.0]. + sigma_data (`float`, *optional*, defaults to 0.5): + The standard deviation of the data distribution. This is set to 0.5 in the EDM paper [1]. + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + rho (`float`, *optional*, defaults to 7.0): + The rho parameter used for calculating the Karras sigma schedule, which is set to 7.0 in the EDM paper [1]. + """ + + _compatibles = [] + order = 1 + + @register_to_config + def __init__( + self, + sigma_min: float = 0.002, + sigma_max: float = 80.0, + sigma_data: float = 0.5, + num_train_timesteps: int = 1000, + prediction_type: str = "epsilon", + rho: float = 7.0, + ): + # setable values + self.num_inference_steps = None + + ramp = torch.linspace(0, 1, num_train_timesteps) + sigmas = self._compute_sigmas(ramp) + self.timesteps = self.precondition_noise(sigmas) + + self.sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)]) + + self.is_scale_input_called = False + + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def init_noise_sigma(self): + # standard deviation of the initial noise distribution + return (self.config.sigma_max**2 + 1) ** 0.5 + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def precondition_inputs(self, sample, sigma): + c_in = 1 / ((sigma**2 + self.config.sigma_data**2) ** 0.5) + scaled_sample = sample * c_in + return scaled_sample + + def precondition_noise(self, sigma): + if not isinstance(sigma, torch.Tensor): + sigma = torch.tensor([sigma]) + + c_noise = 0.25 * torch.log(sigma) + + return c_noise + + def precondition_outputs(self, sample, model_output, sigma): + sigma_data = self.config.sigma_data + c_skip = sigma_data**2 / (sigma**2 + sigma_data**2) + + if self.config.prediction_type == "epsilon": + c_out = sigma * sigma_data / (sigma**2 + sigma_data**2) ** 0.5 + elif self.config.prediction_type == "v_prediction": + c_out = -sigma * sigma_data / (sigma**2 + sigma_data**2) ** 0.5 + else: + raise ValueError(f"Prediction type {self.config.prediction_type} is not supported.") + + denoised = c_skip * sample + c_out * model_output + + return denoised + + def scale_model_input( + self, sample: torch.FloatTensor, timestep: Union[float, torch.FloatTensor] + ) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. Scales the denoising model input by `(sigma**2 + 1) ** 0.5` to match the Euler algorithm. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + if self.step_index is None: + self._init_step_index(timestep) + + sigma = self.sigmas[self.step_index] + sample = self.precondition_inputs(sample, sigma) + + self.is_scale_input_called = True + return sample + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.num_inference_steps = num_inference_steps + + ramp = np.linspace(0, 1, self.num_inference_steps) + sigmas = self._compute_sigmas(ramp) + + sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32, device=device) + self.timesteps = self.precondition_noise(sigmas) + + self.sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)]) + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Taken from https://github.com/crowsonkb/k-diffusion/blob/686dbad0f39640ea25c8a8c6a6e56bb40eacefa2/k_diffusion/sampling.py#L17 + def _compute_sigmas(self, ramp, sigma_min=None, sigma_max=None) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + sigma_min = sigma_min or self.config.sigma_min + sigma_max = sigma_max or self.config.sigma_max + + rho = self.config.rho + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + sample: torch.FloatTensor, + s_churn: float = 0.0, + s_tmin: float = 0.0, + s_tmax: float = float("inf"), + s_noise: float = 1.0, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[EDMEulerSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + s_churn (`float`): + s_tmin (`float`): + s_tmax (`float`): + s_noise (`float`, defaults to 1.0): + Scaling factor for noise added to the sample. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_euler_discrete.EDMEulerSchedulerOutput`] or + tuple. + + Returns: + [`~schedulers.scheduling_euler_discrete.EDMEulerSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_euler_discrete.EDMEulerSchedulerOutput`] is + returned, otherwise a tuple is returned where the first element is the sample tensor. + """ + + if ( + isinstance(timestep, int) + or isinstance(timestep, torch.IntTensor) + or isinstance(timestep, torch.LongTensor) + ): + raise ValueError( + ( + "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to" + " `EDMEulerScheduler.step()` is not supported. Make sure to pass" + " one of the `scheduler.timesteps` as a timestep." + ), + ) + + if not self.is_scale_input_called: + logger.warning( + "The `scale_model_input` function should be called before `step` to ensure correct denoising. " + "See `StableDiffusionPipeline` for a usage example." + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + + sigma = self.sigmas[self.step_index] + + gamma = min(s_churn / (len(self.sigmas) - 1), 2**0.5 - 1) if s_tmin <= sigma <= s_tmax else 0.0 + + noise = randn_tensor( + model_output.shape, dtype=model_output.dtype, device=model_output.device, generator=generator + ) + + eps = noise * s_noise + sigma_hat = sigma * (gamma + 1) + + if gamma > 0: + sample = sample + eps * (sigma_hat**2 - sigma**2) ** 0.5 + + # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise + pred_original_sample = self.precondition_outputs(sample, model_output, sigma_hat) + + # 2. Convert to an ODE derivative + derivative = (sample - pred_original_sample) / sigma_hat + + dt = self.sigmas[self.step_index + 1] - sigma_hat + + prev_sample = sample + derivative * dt + + # Cast sample back to model compatible dtype + prev_sample = prev_sample.to(model_output.dtype) + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return EDMEulerSchedulerOutput(prev_sample=prev_sample, pred_original_sample=pred_original_sample) + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + noisy_samples = original_samples + noise * sigma + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py new file mode 100755 index 0000000..dfab592 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_ancestral_discrete.py @@ -0,0 +1,481 @@ +# Copyright 2024 Katherine Crowson and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput, logging +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->EulerAncestralDiscrete +class EulerAncestralDiscreteSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.FloatTensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.FloatTensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class EulerAncestralDiscreteScheduler(SchedulerMixin, ConfigMixin): + """ + Ancestral sampling with Euler method steps. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear` or `scaled_linear`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + prediction_type: str = "epsilon", + timestep_spacing: str = "linspace", + steps_offset: int = 0, + rescale_betas_zero_snr: bool = False, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + if rescale_betas_zero_snr: + # Close to 0 without being 0 so first sigma is not inf + # FP16 smallest positive subnormal works well here + self.alphas_cumprod[-1] = 2**-24 + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + sigmas = np.concatenate([sigmas[::-1], [0.0]]).astype(np.float32) + self.sigmas = torch.from_numpy(sigmas) + + # setable values + self.num_inference_steps = None + timesteps = np.linspace(0, num_train_timesteps - 1, num_train_timesteps, dtype=float)[::-1].copy() + self.timesteps = torch.from_numpy(timesteps) + self.is_scale_input_called = False + + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def init_noise_sigma(self): + # standard deviation of the initial noise distribution + if self.config.timestep_spacing in ["linspace", "trailing"]: + return self.sigmas.max() + + return (self.sigmas.max() ** 2 + 1) ** 0.5 + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def scale_model_input( + self, sample: torch.FloatTensor, timestep: Union[float, torch.FloatTensor] + ) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. Scales the denoising model input by `(sigma**2 + 1) ** 0.5` to match the Euler algorithm. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + + if self.step_index is None: + self._init_step_index(timestep) + + sigma = self.sigmas[self.step_index] + sample = sample / ((sigma**2 + 1) ** 0.5) + self.is_scale_input_called = True + return sample + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.num_inference_steps = num_inference_steps + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps, dtype=np.float32)[ + ::-1 + ].copy() + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.float32) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(self.config.num_train_timesteps, 0, -step_ratio)).round().copy().astype(np.float32) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + sigmas = np.concatenate([sigmas, [0.0]]).astype(np.float32) + self.sigmas = torch.from_numpy(sigmas).to(device=device) + + self.timesteps = torch.from_numpy(timesteps).to(device=device) + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + sample: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[EulerAncestralDiscreteSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`): + Whether or not to return a + [`~schedulers.scheduling_euler_ancestral_discrete.EulerAncestralDiscreteSchedulerOutput`] or tuple. + + Returns: + [`~schedulers.scheduling_euler_ancestral_discrete.EulerAncestralDiscreteSchedulerOutput`] or `tuple`: + If return_dict is `True`, + [`~schedulers.scheduling_euler_ancestral_discrete.EulerAncestralDiscreteSchedulerOutput`] is returned, + otherwise a tuple is returned where the first element is the sample tensor. + + """ + + if ( + isinstance(timestep, int) + or isinstance(timestep, torch.IntTensor) + or isinstance(timestep, torch.LongTensor) + ): + raise ValueError( + ( + "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to" + " `EulerDiscreteScheduler.step()` is not supported. Make sure to pass" + " one of the `scheduler.timesteps` as a timestep." + ), + ) + + if not self.is_scale_input_called: + logger.warning( + "The `scale_model_input` function should be called before `step` to ensure correct denoising. " + "See `StableDiffusionPipeline` for a usage example." + ) + + if self.step_index is None: + self._init_step_index(timestep) + + sigma = self.sigmas[self.step_index] + + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + + # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise + if self.config.prediction_type == "epsilon": + pred_original_sample = sample - sigma * model_output + elif self.config.prediction_type == "v_prediction": + # * c_out + input * c_skip + pred_original_sample = model_output * (-sigma / (sigma**2 + 1) ** 0.5) + (sample / (sigma**2 + 1)) + elif self.config.prediction_type == "sample": + raise NotImplementedError("prediction_type not implemented yet: sample") + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`" + ) + + sigma_from = self.sigmas[self.step_index] + sigma_to = self.sigmas[self.step_index + 1] + sigma_up = (sigma_to**2 * (sigma_from**2 - sigma_to**2) / sigma_from**2) ** 0.5 + sigma_down = (sigma_to**2 - sigma_up**2) ** 0.5 + + # 2. Convert to an ODE derivative + derivative = (sample - pred_original_sample) / sigma + + dt = sigma_down - sigma + + prev_sample = sample + derivative * dt + + device = model_output.device + noise = randn_tensor(model_output.shape, dtype=model_output.dtype, device=device, generator=generator) + + prev_sample = prev_sample + noise * sigma_up + + # Cast sample back to model compatible dtype + prev_sample = prev_sample.to(model_output.dtype) + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return EulerAncestralDiscreteSchedulerOutput( + prev_sample=prev_sample, pred_original_sample=pred_original_sample + ) + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + noisy_samples = original_samples + noise * sigma + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_discrete.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_discrete.py new file mode 100755 index 0000000..22258ab --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_discrete.py @@ -0,0 +1,576 @@ +# Copyright 2024 Katherine Crowson and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput, logging +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->EulerDiscrete +class EulerDiscreteSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas): + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.FloatTensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.FloatTensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class EulerDiscreteScheduler(SchedulerMixin, ConfigMixin): + """ + Euler scheduler. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear` or `scaled_linear`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + interpolation_type(`str`, defaults to `"linear"`, *optional*): + The interpolation type to compute intermediate sigmas for the scheduler denoising steps. Should be on of + `"linear"` or `"log_linear"`. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + prediction_type: str = "epsilon", + interpolation_type: str = "linear", + use_karras_sigmas: Optional[bool] = False, + sigma_min: Optional[float] = None, + sigma_max: Optional[float] = None, + timestep_spacing: str = "linspace", + timestep_type: str = "discrete", # can be "discrete" or "continuous" + steps_offset: int = 0, + rescale_betas_zero_snr: bool = False, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + if rescale_betas_zero_snr: + # Close to 0 without being 0 so first sigma is not inf + # FP16 smallest positive subnormal works well here + self.alphas_cumprod[-1] = 2**-24 + + sigmas = (((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5).flip(0) + timesteps = np.linspace(0, num_train_timesteps - 1, num_train_timesteps, dtype=float)[::-1].copy() + timesteps = torch.from_numpy(timesteps).to(dtype=torch.float32) + + # setable values + self.num_inference_steps = None + + # TODO: Support the full EDM scalings for all prediction types and timestep types + if timestep_type == "continuous" and prediction_type == "v_prediction": + self.timesteps = torch.Tensor([0.25 * sigma.log() for sigma in sigmas]) + else: + self.timesteps = timesteps + + self.sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)]) + + self.is_scale_input_called = False + self.use_karras_sigmas = use_karras_sigmas + + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def init_noise_sigma(self): + # standard deviation of the initial noise distribution + max_sigma = max(self.sigmas) if isinstance(self.sigmas, list) else self.sigmas.max() + if self.config.timestep_spacing in ["linspace", "trailing"]: + return max_sigma + + return (max_sigma**2 + 1) ** 0.5 + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def scale_model_input( + self, sample: torch.FloatTensor, timestep: Union[float, torch.FloatTensor] + ) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. Scales the denoising model input by `(sigma**2 + 1) ** 0.5` to match the Euler algorithm. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + if self.step_index is None: + self._init_step_index(timestep) + + sigma = self.sigmas[self.step_index] + sample = sample / ((sigma**2 + 1) ** 0.5) + + self.is_scale_input_called = True + return sample + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.num_inference_steps = num_inference_steps + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps, dtype=np.float32)[ + ::-1 + ].copy() + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.float32) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(self.config.num_train_timesteps, 0, -step_ratio)).round().copy().astype(np.float32) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + log_sigmas = np.log(sigmas) + + if self.config.interpolation_type == "linear": + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + elif self.config.interpolation_type == "log_linear": + sigmas = torch.linspace(np.log(sigmas[-1]), np.log(sigmas[0]), num_inference_steps + 1).exp().numpy() + else: + raise ValueError( + f"{self.config.interpolation_type} is not implemented. Please specify interpolation_type to either" + " 'linear' or 'log_linear'" + ) + + if self.use_karras_sigmas: + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=self.num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]) + + sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32, device=device) + + # TODO: Support the full EDM scalings for all prediction types and timestep types + if self.config.timestep_type == "continuous" and self.config.prediction_type == "v_prediction": + self.timesteps = torch.Tensor([0.25 * sigma.log() for sigma in sigmas]).to(device=device) + else: + self.timesteps = torch.from_numpy(timesteps.astype(np.float32)).to(device=device) + + self.sigmas = torch.cat([sigmas, torch.zeros(1, device=sigmas.device)]) + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # Copied from https://github.com/crowsonkb/k-diffusion/blob/686dbad0f39640ea25c8a8c6a6e56bb40eacefa2/k_diffusion/sampling.py#L17 + def _convert_to_karras(self, in_sigmas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + sample: torch.FloatTensor, + s_churn: float = 0.0, + s_tmin: float = 0.0, + s_tmax: float = float("inf"), + s_noise: float = 1.0, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[EulerDiscreteSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + s_churn (`float`): + s_tmin (`float`): + s_tmax (`float`): + s_noise (`float`, defaults to 1.0): + Scaling factor for noise added to the sample. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or + tuple. + + Returns: + [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_euler_discrete.EulerDiscreteSchedulerOutput`] is + returned, otherwise a tuple is returned where the first element is the sample tensor. + """ + + if ( + isinstance(timestep, int) + or isinstance(timestep, torch.IntTensor) + or isinstance(timestep, torch.LongTensor) + ): + raise ValueError( + ( + "Passing integer indices (e.g. from `enumerate(timesteps)`) as timesteps to" + " `EulerDiscreteScheduler.step()` is not supported. Make sure to pass" + " one of the `scheduler.timesteps` as a timestep." + ), + ) + + if not self.is_scale_input_called: + logger.warning( + "The `scale_model_input` function should be called before `step` to ensure correct denoising. " + "See `StableDiffusionPipeline` for a usage example." + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # Upcast to avoid precision issues when computing prev_sample + sample = sample.to(torch.float32) + + sigma = self.sigmas[self.step_index] + + gamma = min(s_churn / (len(self.sigmas) - 1), 2**0.5 - 1) if s_tmin <= sigma <= s_tmax else 0.0 + + noise = randn_tensor( + model_output.shape, dtype=model_output.dtype, device=model_output.device, generator=generator + ) + + eps = noise * s_noise + sigma_hat = sigma * (gamma + 1) + + if gamma > 0: + sample = sample + eps * (sigma_hat**2 - sigma**2) ** 0.5 + + # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise + # NOTE: "original_sample" should not be an expected prediction_type but is left in for + # backwards compatibility + if self.config.prediction_type == "original_sample" or self.config.prediction_type == "sample": + pred_original_sample = model_output + elif self.config.prediction_type == "epsilon": + pred_original_sample = sample - sigma_hat * model_output + elif self.config.prediction_type == "v_prediction": + # denoised = model_output * c_out + input * c_skip + pred_original_sample = model_output * (-sigma / (sigma**2 + 1) ** 0.5) + (sample / (sigma**2 + 1)) + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`" + ) + + # 2. Convert to an ODE derivative + derivative = (sample - pred_original_sample) / sigma_hat + + dt = self.sigmas[self.step_index + 1] - sigma_hat + + prev_sample = sample + derivative * dt + + # Cast sample back to model compatible dtype + prev_sample = prev_sample.to(model_output.dtype) + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return EulerDiscreteSchedulerOutput(prev_sample=prev_sample, pred_original_sample=pred_original_sample) + + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + noisy_samples = original_samples + noise * sigma + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_discrete_flax.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_discrete_flax.py new file mode 100755 index 0000000..55b0c24 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_euler_discrete_flax.py @@ -0,0 +1,265 @@ +# Copyright 2024 Katherine Crowson and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import flax +import jax.numpy as jnp + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils_flax import ( + CommonSchedulerState, + FlaxKarrasDiffusionSchedulers, + FlaxSchedulerMixin, + FlaxSchedulerOutput, + broadcast_to_shape_from_left, +) + + +@flax.struct.dataclass +class EulerDiscreteSchedulerState: + common: CommonSchedulerState + + # setable values + init_noise_sigma: jnp.ndarray + timesteps: jnp.ndarray + sigmas: jnp.ndarray + num_inference_steps: Optional[int] = None + + @classmethod + def create( + cls, common: CommonSchedulerState, init_noise_sigma: jnp.ndarray, timesteps: jnp.ndarray, sigmas: jnp.ndarray + ): + return cls(common=common, init_noise_sigma=init_noise_sigma, timesteps=timesteps, sigmas=sigmas) + + +@dataclass +class FlaxEulerDiscreteSchedulerOutput(FlaxSchedulerOutput): + state: EulerDiscreteSchedulerState + + +class FlaxEulerDiscreteScheduler(FlaxSchedulerMixin, ConfigMixin): + """ + Euler scheduler (Algorithm 2) from Karras et al. (2022) https://arxiv.org/abs/2206.00364. . Based on the original + k-diffusion implementation by Katherine Crowson: + https://github.com/crowsonkb/k-diffusion/blob/481677d114f6ea445aa009cf5bd7a9cdee909e47/k_diffusion/sampling.py#L51 + + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + beta_start (`float`): the starting `beta` value of inference. + beta_end (`float`): the final `beta` value. + beta_schedule (`str`): + the beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear` or `scaled_linear`. + trained_betas (`jnp.ndarray`, optional): + option to pass an array of betas directly to the constructor to bypass `beta_start`, `beta_end` etc. + prediction_type (`str`, default `epsilon`, optional): + prediction type of the scheduler function, one of `epsilon` (predicting the noise of the diffusion + process), `sample` (directly predicting the noisy sample`) or `v_prediction` (see section 2.4 + https://imagen.research.google/video/paper.pdf) + dtype (`jnp.dtype`, *optional*, defaults to `jnp.float32`): + the `dtype` used for params and computation. + """ + + _compatibles = [e.name for e in FlaxKarrasDiffusionSchedulers] + + dtype: jnp.dtype + + @property + def has_state(self): + return True + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[jnp.ndarray] = None, + prediction_type: str = "epsilon", + timestep_spacing: str = "linspace", + dtype: jnp.dtype = jnp.float32, + ): + self.dtype = dtype + + def create_state(self, common: Optional[CommonSchedulerState] = None) -> EulerDiscreteSchedulerState: + if common is None: + common = CommonSchedulerState.create(self) + + timesteps = jnp.arange(0, self.config.num_train_timesteps).round()[::-1] + sigmas = ((1 - common.alphas_cumprod) / common.alphas_cumprod) ** 0.5 + sigmas = jnp.interp(timesteps, jnp.arange(0, len(sigmas)), sigmas) + sigmas = jnp.concatenate([sigmas, jnp.array([0.0], dtype=self.dtype)]) + + # standard deviation of the initial noise distribution + if self.config.timestep_spacing in ["linspace", "trailing"]: + init_noise_sigma = sigmas.max() + else: + init_noise_sigma = (sigmas.max() ** 2 + 1) ** 0.5 + + return EulerDiscreteSchedulerState.create( + common=common, + init_noise_sigma=init_noise_sigma, + timesteps=timesteps, + sigmas=sigmas, + ) + + def scale_model_input(self, state: EulerDiscreteSchedulerState, sample: jnp.ndarray, timestep: int) -> jnp.ndarray: + """ + Scales the denoising model input by `(sigma**2 + 1) ** 0.5` to match the Euler algorithm. + + Args: + state (`EulerDiscreteSchedulerState`): + the `FlaxEulerDiscreteScheduler` state data class instance. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + timestep (`int`): + current discrete timestep in the diffusion chain. + + Returns: + `jnp.ndarray`: scaled input sample + """ + (step_index,) = jnp.where(state.timesteps == timestep, size=1) + step_index = step_index[0] + + sigma = state.sigmas[step_index] + sample = sample / ((sigma**2 + 1) ** 0.5) + return sample + + def set_timesteps( + self, state: EulerDiscreteSchedulerState, num_inference_steps: int, shape: Tuple = () + ) -> EulerDiscreteSchedulerState: + """ + Sets the timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + state (`EulerDiscreteSchedulerState`): + the `FlaxEulerDiscreteScheduler` state data class instance. + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + """ + + if self.config.timestep_spacing == "linspace": + timesteps = jnp.linspace(self.config.num_train_timesteps - 1, 0, num_inference_steps, dtype=self.dtype) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // num_inference_steps + timesteps = (jnp.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(float) + timesteps += 1 + else: + raise ValueError( + f"timestep_spacing must be one of ['linspace', 'leading'], got {self.config.timestep_spacing}" + ) + + sigmas = ((1 - state.common.alphas_cumprod) / state.common.alphas_cumprod) ** 0.5 + sigmas = jnp.interp(timesteps, jnp.arange(0, len(sigmas)), sigmas) + sigmas = jnp.concatenate([sigmas, jnp.array([0.0], dtype=self.dtype)]) + + # standard deviation of the initial noise distribution + if self.config.timestep_spacing in ["linspace", "trailing"]: + init_noise_sigma = sigmas.max() + else: + init_noise_sigma = (sigmas.max() ** 2 + 1) ** 0.5 + + return state.replace( + timesteps=timesteps, + sigmas=sigmas, + num_inference_steps=num_inference_steps, + init_noise_sigma=init_noise_sigma, + ) + + def step( + self, + state: EulerDiscreteSchedulerState, + model_output: jnp.ndarray, + timestep: int, + sample: jnp.ndarray, + return_dict: bool = True, + ) -> Union[FlaxEulerDiscreteSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + state (`EulerDiscreteSchedulerState`): + the `FlaxEulerDiscreteScheduler` state data class instance. + model_output (`jnp.ndarray`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + order: coefficient for multi-step inference. + return_dict (`bool`): option for returning tuple rather than FlaxEulerDiscreteScheduler class + + Returns: + [`FlaxEulerDiscreteScheduler`] or `tuple`: [`FlaxEulerDiscreteScheduler`] if `return_dict` is True, + otherwise a `tuple`. When returning a tuple, the first element is the sample tensor. + + """ + if state.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + (step_index,) = jnp.where(state.timesteps == timestep, size=1) + step_index = step_index[0] + + sigma = state.sigmas[step_index] + + # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise + if self.config.prediction_type == "epsilon": + pred_original_sample = sample - sigma * model_output + elif self.config.prediction_type == "v_prediction": + # * c_out + input * c_skip + pred_original_sample = model_output * (-sigma / (sigma**2 + 1) ** 0.5) + (sample / (sigma**2 + 1)) + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`" + ) + + # 2. Convert to an ODE derivative + derivative = (sample - pred_original_sample) / sigma + + # dt = sigma_down - sigma + dt = state.sigmas[step_index + 1] - sigma + + prev_sample = sample + derivative * dt + + if not return_dict: + return (prev_sample, state) + + return FlaxEulerDiscreteSchedulerOutput(prev_sample=prev_sample, state=state) + + def add_noise( + self, + state: EulerDiscreteSchedulerState, + original_samples: jnp.ndarray, + noise: jnp.ndarray, + timesteps: jnp.ndarray, + ) -> jnp.ndarray: + sigma = state.sigmas[timesteps].flatten() + sigma = broadcast_to_shape_from_left(sigma, noise.shape) + + noisy_samples = original_samples + noise * sigma + + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_heun_discrete.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_heun_discrete.py new file mode 100755 index 0000000..fc955ac --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_heun_discrete.py @@ -0,0 +1,482 @@ +# Copyright 2024 Katherine Crowson, The HuggingFace Team and hlky. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class HeunDiscreteScheduler(SchedulerMixin, ConfigMixin): + """ + Scheduler with Heun steps for discrete beta schedules. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear` or `scaled_linear`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + clip_sample (`bool`, defaults to `True`): + Clip the predicted sample for numerical stability. + clip_sample_range (`float`, defaults to 1.0): + The maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 2 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.00085, # sensible defaults + beta_end: float = 0.012, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + prediction_type: str = "epsilon", + use_karras_sigmas: Optional[bool] = False, + clip_sample: Optional[bool] = False, + clip_sample_range: float = 1.0, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps, alpha_transform_type="cosine") + elif beta_schedule == "exp": + self.betas = betas_for_alpha_bar(num_train_timesteps, alpha_transform_type="exp") + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + # set all values + self.set_timesteps(num_train_timesteps, None, num_train_timesteps) + self.use_karras_sigmas = use_karras_sigmas + + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + @property + def init_noise_sigma(self): + # standard deviation of the initial noise distribution + if self.config.timestep_spacing in ["linspace", "trailing"]: + return self.sigmas.max() + + return (self.sigmas.max() ** 2 + 1) ** 0.5 + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def scale_model_input( + self, + sample: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + ) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + if self.step_index is None: + self._init_step_index(timestep) + + sigma = self.sigmas[self.step_index] + sample = sample / ((sigma**2 + 1) ** 0.5) + return sample + + def set_timesteps( + self, + num_inference_steps: int, + device: Union[str, torch.device] = None, + num_train_timesteps: Optional[int] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.num_inference_steps = num_inference_steps + + num_train_timesteps = num_train_timesteps or self.config.num_train_timesteps + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = np.linspace(0, num_train_timesteps - 1, num_inference_steps, dtype=np.float32)[::-1].copy() + elif self.config.timestep_spacing == "leading": + step_ratio = num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.float32) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(num_train_timesteps, 0, -step_ratio)).round().copy().astype(np.float32) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + log_sigmas = np.log(sigmas) + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + + if self.config.use_karras_sigmas: + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=self.num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]) + + sigmas = np.concatenate([sigmas, [0.0]]).astype(np.float32) + sigmas = torch.from_numpy(sigmas).to(device=device) + self.sigmas = torch.cat([sigmas[:1], sigmas[1:-1].repeat_interleave(2), sigmas[-1:]]) + + timesteps = torch.from_numpy(timesteps) + timesteps = torch.cat([timesteps[:1], timesteps[1:].repeat_interleave(2)]) + + self.timesteps = timesteps.to(device=device) + + # empty dt and derivative + self.prev_derivative = None + self.dt = None + + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + @property + def state_in_first_order(self): + return self.dt is None + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: Union[torch.FloatTensor, np.ndarray], + timestep: Union[float, torch.FloatTensor], + sample: Union[torch.FloatTensor, np.ndarray], + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or tuple. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + """ + if self.step_index is None: + self._init_step_index(timestep) + + if self.state_in_first_order: + sigma = self.sigmas[self.step_index] + sigma_next = self.sigmas[self.step_index + 1] + else: + # 2nd order / Heun's method + sigma = self.sigmas[self.step_index - 1] + sigma_next = self.sigmas[self.step_index] + + # currently only gamma=0 is supported. This usually works best anyways. + # We can support gamma in the future but then need to scale the timestep before + # passing it to the model which requires a change in API + gamma = 0 + sigma_hat = sigma * (gamma + 1) # Note: sigma_hat == sigma for now + + # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise + if self.config.prediction_type == "epsilon": + sigma_input = sigma_hat if self.state_in_first_order else sigma_next + pred_original_sample = sample - sigma_input * model_output + elif self.config.prediction_type == "v_prediction": + sigma_input = sigma_hat if self.state_in_first_order else sigma_next + pred_original_sample = model_output * (-sigma_input / (sigma_input**2 + 1) ** 0.5) + ( + sample / (sigma_input**2 + 1) + ) + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`" + ) + + if self.config.clip_sample: + pred_original_sample = pred_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + if self.state_in_first_order: + # 2. Convert to an ODE derivative for 1st order + derivative = (sample - pred_original_sample) / sigma_hat + # 3. delta timestep + dt = sigma_next - sigma_hat + + # store for 2nd order step + self.prev_derivative = derivative + self.dt = dt + self.sample = sample + else: + # 2. 2nd order / Heun's method + derivative = (sample - pred_original_sample) / sigma_next + derivative = (self.prev_derivative + derivative) / 2 + + # 3. take prev timestep & sample + dt = self.dt + sample = self.sample + + # free dt and derivative + # Note, this puts the scheduler in "first order mode" + self.prev_derivative = None + self.dt = None + self.sample = None + + prev_sample = sample + derivative * dt + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + noisy_samples = original_samples + noise * sigma + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ipndm.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ipndm.py new file mode 100755 index 0000000..583afa4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_ipndm.py @@ -0,0 +1,224 @@ +# Copyright 2024 Zhejiang University Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils import SchedulerMixin, SchedulerOutput + + +class IPNDMScheduler(SchedulerMixin, ConfigMixin): + """ + A fourth-order Improved Pseudo Linear Multistep scheduler. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + """ + + order = 1 + + @register_to_config + def __init__( + self, num_train_timesteps: int = 1000, trained_betas: Optional[Union[np.ndarray, List[float]]] = None + ): + # set `betas`, `alphas`, `timesteps` + self.set_timesteps(num_train_timesteps) + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # For now we only support F-PNDM, i.e. the runge-kutta method + # For more information on the algorithm please take a look at the paper: https://arxiv.org/pdf/2202.09778.pdf + # mainly at formula (9), (12), (13) and the Algorithm 2. + self.pndm_order = 4 + + # running values + self.ets = [] + self._step_index = None + self._begin_index = None + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.num_inference_steps = num_inference_steps + steps = torch.linspace(1, 0, num_inference_steps + 1)[:-1] + steps = torch.cat([steps, torch.tensor([0.0])]) + + if self.config.trained_betas is not None: + self.betas = torch.tensor(self.config.trained_betas, dtype=torch.float32) + else: + self.betas = torch.sin(steps * math.pi / 2) ** 2 + + self.alphas = (1.0 - self.betas**2) ** 0.5 + + timesteps = (torch.atan2(self.betas, self.alphas) / math.pi * 2)[:-1] + self.timesteps = timesteps.to(device) + + self.ets = [] + self._step_index = None + self._begin_index = None + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the linear multistep method. It performs one forward pass multiple times to approximate the solution. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or tuple. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + if self.step_index is None: + self._init_step_index(timestep) + + timestep_index = self.step_index + prev_timestep_index = self.step_index + 1 + + ets = sample * self.betas[timestep_index] + model_output * self.alphas[timestep_index] + self.ets.append(ets) + + if len(self.ets) == 1: + ets = self.ets[-1] + elif len(self.ets) == 2: + ets = (3 * self.ets[-1] - self.ets[-2]) / 2 + elif len(self.ets) == 3: + ets = (23 * self.ets[-1] - 16 * self.ets[-2] + 5 * self.ets[-3]) / 12 + else: + ets = (1 / 24) * (55 * self.ets[-1] - 59 * self.ets[-2] + 37 * self.ets[-3] - 9 * self.ets[-4]) + + prev_sample = self._get_prev_sample(sample, timestep_index, prev_timestep_index, ets) + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def _get_prev_sample(self, sample, timestep_index, prev_timestep_index, ets): + alpha = self.alphas[timestep_index] + sigma = self.betas[timestep_index] + + next_alpha = self.alphas[prev_timestep_index] + next_sigma = self.betas[prev_timestep_index] + + pred = (sample - sigma * ets) / max(alpha, 1e-8) + prev_sample = next_alpha * pred + ets * next_sigma + + return prev_sample + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py new file mode 100755 index 0000000..9521c9c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_k_dpm_2_ancestral_discrete.py @@ -0,0 +1,508 @@ +# Copyright 2024 Katherine Crowson, The HuggingFace Team and hlky. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class KDPM2AncestralDiscreteScheduler(SchedulerMixin, ConfigMixin): + """ + KDPM2DiscreteScheduler with ancestral sampling is inspired by the DPMSolver2 and Algorithm 2 from the [Elucidating + the Design Space of Diffusion-Based Generative Models](https://huggingface.co/papers/2206.00364) paper. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.00085): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.012): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear` or `scaled_linear`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 2 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.00085, # sensible defaults + beta_end: float = 0.012, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + use_karras_sigmas: Optional[bool] = False, + prediction_type: str = "epsilon", + timestep_spacing: str = "linspace", + steps_offset: int = 0, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + # set all values + self.set_timesteps(num_train_timesteps, None, num_train_timesteps) + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def init_noise_sigma(self): + # standard deviation of the initial noise distribution + if self.config.timestep_spacing in ["linspace", "trailing"]: + return self.sigmas.max() + + return (self.sigmas.max() ** 2 + 1) ** 0.5 + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def scale_model_input( + self, + sample: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + ) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + if self.step_index is None: + self._init_step_index(timestep) + + if self.state_in_first_order: + sigma = self.sigmas[self.step_index] + else: + sigma = self.sigmas_interpol[self.step_index - 1] + + sample = sample / ((sigma**2 + 1) ** 0.5) + return sample + + def set_timesteps( + self, + num_inference_steps: int, + device: Union[str, torch.device] = None, + num_train_timesteps: Optional[int] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.num_inference_steps = num_inference_steps + + num_train_timesteps = num_train_timesteps or self.config.num_train_timesteps + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = np.linspace(0, num_train_timesteps - 1, num_inference_steps, dtype=np.float32)[::-1].copy() + elif self.config.timestep_spacing == "leading": + step_ratio = num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.float32) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(num_train_timesteps, 0, -step_ratio)).round().copy().astype(np.float32) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + log_sigmas = np.log(sigmas) + + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + + if self.config.use_karras_sigmas: + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + + self.log_sigmas = torch.from_numpy(log_sigmas).to(device) + sigmas = np.concatenate([sigmas, [0.0]]).astype(np.float32) + sigmas = torch.from_numpy(sigmas).to(device=device) + + # compute up and down sigmas + sigmas_next = sigmas.roll(-1) + sigmas_next[-1] = 0.0 + sigmas_up = (sigmas_next**2 * (sigmas**2 - sigmas_next**2) / sigmas**2) ** 0.5 + sigmas_down = (sigmas_next**2 - sigmas_up**2) ** 0.5 + sigmas_down[-1] = 0.0 + + # compute interpolated sigmas + sigmas_interpol = sigmas.log().lerp(sigmas_down.log(), 0.5).exp() + sigmas_interpol[-2:] = 0.0 + + # set sigmas + self.sigmas = torch.cat([sigmas[:1], sigmas[1:].repeat_interleave(2), sigmas[-1:]]) + self.sigmas_interpol = torch.cat( + [sigmas_interpol[:1], sigmas_interpol[1:].repeat_interleave(2), sigmas_interpol[-1:]] + ) + self.sigmas_up = torch.cat([sigmas_up[:1], sigmas_up[1:].repeat_interleave(2), sigmas_up[-1:]]) + self.sigmas_down = torch.cat([sigmas_down[:1], sigmas_down[1:].repeat_interleave(2), sigmas_down[-1:]]) + + if str(device).startswith("mps"): + timesteps = torch.from_numpy(timesteps).to(device, dtype=torch.float32) + else: + timesteps = torch.from_numpy(timesteps).to(device) + + sigmas_interpol = sigmas_interpol.cpu() + log_sigmas = self.log_sigmas.cpu() + timesteps_interpol = np.array( + [self._sigma_to_t(sigma_interpol, log_sigmas) for sigma_interpol in sigmas_interpol] + ) + + timesteps_interpol = torch.from_numpy(timesteps_interpol).to(device, dtype=timesteps.dtype) + interleaved_timesteps = torch.stack((timesteps_interpol[:-2, None], timesteps[1:, None]), dim=-1).flatten() + + self.timesteps = torch.cat([timesteps[:1], interleaved_timesteps]) + + self.sample = None + + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + @property + def state_in_first_order(self): + return self.sample is None + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: Union[torch.FloatTensor, np.ndarray], + timestep: Union[float, torch.FloatTensor], + sample: Union[torch.FloatTensor, np.ndarray], + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or tuple. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_ddim.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + """ + if self.step_index is None: + self._init_step_index(timestep) + + if self.state_in_first_order: + sigma = self.sigmas[self.step_index] + sigma_interpol = self.sigmas_interpol[self.step_index] + sigma_up = self.sigmas_up[self.step_index] + sigma_down = self.sigmas_down[self.step_index - 1] + else: + # 2nd order / KPDM2's method + sigma = self.sigmas[self.step_index - 1] + sigma_interpol = self.sigmas_interpol[self.step_index - 1] + sigma_up = self.sigmas_up[self.step_index - 1] + sigma_down = self.sigmas_down[self.step_index - 1] + + # currently only gamma=0 is supported. This usually works best anyways. + # We can support gamma in the future but then need to scale the timestep before + # passing it to the model which requires a change in API + gamma = 0 + sigma_hat = sigma * (gamma + 1) # Note: sigma_hat == sigma for now + + device = model_output.device + noise = randn_tensor(model_output.shape, dtype=model_output.dtype, device=device, generator=generator) + + # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise + if self.config.prediction_type == "epsilon": + sigma_input = sigma_hat if self.state_in_first_order else sigma_interpol + pred_original_sample = sample - sigma_input * model_output + elif self.config.prediction_type == "v_prediction": + sigma_input = sigma_hat if self.state_in_first_order else sigma_interpol + pred_original_sample = model_output * (-sigma_input / (sigma_input**2 + 1) ** 0.5) + ( + sample / (sigma_input**2 + 1) + ) + elif self.config.prediction_type == "sample": + raise NotImplementedError("prediction_type not implemented yet: sample") + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`" + ) + + if self.state_in_first_order: + # 2. Convert to an ODE derivative for 1st order + derivative = (sample - pred_original_sample) / sigma_hat + # 3. delta timestep + dt = sigma_interpol - sigma_hat + + # store for 2nd order step + self.sample = sample + self.dt = dt + prev_sample = sample + derivative * dt + else: + # DPM-Solver-2 + # 2. Convert to an ODE derivative for 2nd order + derivative = (sample - pred_original_sample) / sigma_interpol + # 3. delta timestep + dt = sigma_down - sigma_hat + + sample = self.sample + self.sample = None + + prev_sample = sample + derivative * dt + prev_sample = prev_sample + noise * sigma_up + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + noisy_samples = original_samples + noise * sigma + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py new file mode 100755 index 0000000..5be07b6 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_k_dpm_2_discrete.py @@ -0,0 +1,483 @@ +# Copyright 2024 Katherine Crowson, The HuggingFace Team and hlky. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class KDPM2DiscreteScheduler(SchedulerMixin, ConfigMixin): + """ + KDPM2DiscreteScheduler is inspired by the DPMSolver2 and Algorithm 2 from the [Elucidating the Design Space of + Diffusion-Based Generative Models](https://huggingface.co/papers/2206.00364) paper. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.00085): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.012): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear` or `scaled_linear`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 2 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.00085, # sensible defaults + beta_end: float = 0.012, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + use_karras_sigmas: Optional[bool] = False, + prediction_type: str = "epsilon", + timestep_spacing: str = "linspace", + steps_offset: int = 0, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + # set all values + self.set_timesteps(num_train_timesteps, None, num_train_timesteps) + + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def init_noise_sigma(self): + # standard deviation of the initial noise distribution + if self.config.timestep_spacing in ["linspace", "trailing"]: + return self.sigmas.max() + + return (self.sigmas.max() ** 2 + 1) ** 0.5 + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def scale_model_input( + self, + sample: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + ) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + if self.step_index is None: + self._init_step_index(timestep) + + if self.state_in_first_order: + sigma = self.sigmas[self.step_index] + else: + sigma = self.sigmas_interpol[self.step_index] + + sample = sample / ((sigma**2 + 1) ** 0.5) + return sample + + def set_timesteps( + self, + num_inference_steps: int, + device: Union[str, torch.device] = None, + num_train_timesteps: Optional[int] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.num_inference_steps = num_inference_steps + + num_train_timesteps = num_train_timesteps or self.config.num_train_timesteps + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = np.linspace(0, num_train_timesteps - 1, num_inference_steps, dtype=np.float32)[::-1].copy() + elif self.config.timestep_spacing == "leading": + step_ratio = num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.float32) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(num_train_timesteps, 0, -step_ratio)).round().copy().astype(np.float32) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + log_sigmas = np.log(sigmas) + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + + if self.config.use_karras_sigmas: + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + + self.log_sigmas = torch.from_numpy(log_sigmas).to(device=device) + sigmas = np.concatenate([sigmas, [0.0]]).astype(np.float32) + sigmas = torch.from_numpy(sigmas).to(device=device) + + # interpolate sigmas + sigmas_interpol = sigmas.log().lerp(sigmas.roll(1).log(), 0.5).exp() + + self.sigmas = torch.cat([sigmas[:1], sigmas[1:].repeat_interleave(2), sigmas[-1:]]) + self.sigmas_interpol = torch.cat( + [sigmas_interpol[:1], sigmas_interpol[1:].repeat_interleave(2), sigmas_interpol[-1:]] + ) + + timesteps = torch.from_numpy(timesteps).to(device) + + # interpolate timesteps + sigmas_interpol = sigmas_interpol.cpu() + log_sigmas = self.log_sigmas.cpu() + timesteps_interpol = np.array( + [self._sigma_to_t(sigma_interpol, log_sigmas) for sigma_interpol in sigmas_interpol] + ) + timesteps_interpol = torch.from_numpy(timesteps_interpol).to(device, dtype=timesteps.dtype) + interleaved_timesteps = torch.stack((timesteps_interpol[1:-1, None], timesteps[1:, None]), dim=-1).flatten() + + self.timesteps = torch.cat([timesteps[:1], interleaved_timesteps]) + + self.sample = None + + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def state_in_first_order(self): + return self.sample is None + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + def step( + self, + model_output: Union[torch.FloatTensor, np.ndarray], + timestep: Union[float, torch.FloatTensor], + sample: Union[torch.FloatTensor, np.ndarray], + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or tuple. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + """ + if self.step_index is None: + self._init_step_index(timestep) + + if self.state_in_first_order: + sigma = self.sigmas[self.step_index] + sigma_interpol = self.sigmas_interpol[self.step_index + 1] + sigma_next = self.sigmas[self.step_index + 1] + else: + # 2nd order / KDPM2's method + sigma = self.sigmas[self.step_index - 1] + sigma_interpol = self.sigmas_interpol[self.step_index] + sigma_next = self.sigmas[self.step_index] + + # currently only gamma=0 is supported. This usually works best anyways. + # We can support gamma in the future but then need to scale the timestep before + # passing it to the model which requires a change in API + gamma = 0 + sigma_hat = sigma * (gamma + 1) # Note: sigma_hat == sigma for now + + # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise + if self.config.prediction_type == "epsilon": + sigma_input = sigma_hat if self.state_in_first_order else sigma_interpol + pred_original_sample = sample - sigma_input * model_output + elif self.config.prediction_type == "v_prediction": + sigma_input = sigma_hat if self.state_in_first_order else sigma_interpol + pred_original_sample = model_output * (-sigma_input / (sigma_input**2 + 1) ** 0.5) + ( + sample / (sigma_input**2 + 1) + ) + elif self.config.prediction_type == "sample": + raise NotImplementedError("prediction_type not implemented yet: sample") + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`" + ) + + if self.state_in_first_order: + # 2. Convert to an ODE derivative for 1st order + derivative = (sample - pred_original_sample) / sigma_hat + # 3. delta timestep + dt = sigma_interpol - sigma_hat + + # store for 2nd order step + self.sample = sample + else: + # DPM-Solver-2 + # 2. Convert to an ODE derivative for 2nd order + derivative = (sample - pred_original_sample) / sigma_interpol + + # 3. delta timestep + dt = sigma_next - sigma_hat + + sample = self.sample + self.sample = None + + # upon completion increase step index by one + self._step_index += 1 + + prev_sample = sample + derivative * dt + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + noisy_samples = original_samples + noise * sigma + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_karras_ve_flax.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_karras_ve_flax.py new file mode 100755 index 0000000..4d09960 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_karras_ve_flax.py @@ -0,0 +1,238 @@ +# Copyright 2024 NVIDIA and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import flax +import jax +import jax.numpy as jnp +from jax import random + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from .scheduling_utils_flax import FlaxSchedulerMixin + + +@flax.struct.dataclass +class KarrasVeSchedulerState: + # setable values + num_inference_steps: Optional[int] = None + timesteps: Optional[jnp.ndarray] = None + schedule: Optional[jnp.ndarray] = None # sigma(t_i) + + @classmethod + def create(cls): + return cls() + + +@dataclass +class FlaxKarrasVeOutput(BaseOutput): + """ + Output class for the scheduler's step function output. + + Args: + prev_sample (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample (x_{t-1}) of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + derivative (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)` for images): + Derivative of predicted original image sample (x_0). + state (`KarrasVeSchedulerState`): the `FlaxKarrasVeScheduler` state data class. + """ + + prev_sample: jnp.ndarray + derivative: jnp.ndarray + state: KarrasVeSchedulerState + + +class FlaxKarrasVeScheduler(FlaxSchedulerMixin, ConfigMixin): + """ + Stochastic sampling from Karras et al. [1] tailored to the Variance-Expanding (VE) models [2]. Use Algorithm 2 and + the VE column of Table 1 from [1] for reference. + + [1] Karras, Tero, et al. "Elucidating the Design Space of Diffusion-Based Generative Models." + https://arxiv.org/abs/2206.00364 [2] Song, Yang, et al. "Score-based generative modeling through stochastic + differential equations." https://arxiv.org/abs/2011.13456 + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + For more details on the parameters, see the original paper's Appendix E.: "Elucidating the Design Space of + Diffusion-Based Generative Models." https://arxiv.org/abs/2206.00364. The grid search values used to find the + optimal {s_noise, s_churn, s_min, s_max} for a specific model are described in Table 5 of the paper. + + Args: + sigma_min (`float`): minimum noise magnitude + sigma_max (`float`): maximum noise magnitude + s_noise (`float`): the amount of additional noise to counteract loss of detail during sampling. + A reasonable range is [1.000, 1.011]. + s_churn (`float`): the parameter controlling the overall amount of stochasticity. + A reasonable range is [0, 100]. + s_min (`float`): the start value of the sigma range where we add noise (enable stochasticity). + A reasonable range is [0, 10]. + s_max (`float`): the end value of the sigma range where we add noise. + A reasonable range is [0.2, 80]. + """ + + @property + def has_state(self): + return True + + @register_to_config + def __init__( + self, + sigma_min: float = 0.02, + sigma_max: float = 100, + s_noise: float = 1.007, + s_churn: float = 80, + s_min: float = 0.05, + s_max: float = 50, + ): + pass + + def create_state(self): + return KarrasVeSchedulerState.create() + + def set_timesteps( + self, state: KarrasVeSchedulerState, num_inference_steps: int, shape: Tuple = () + ) -> KarrasVeSchedulerState: + """ + Sets the continuous timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + state (`KarrasVeSchedulerState`): + the `FlaxKarrasVeScheduler` state data class. + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + + """ + timesteps = jnp.arange(0, num_inference_steps)[::-1].copy() + schedule = [ + ( + self.config.sigma_max**2 + * (self.config.sigma_min**2 / self.config.sigma_max**2) ** (i / (num_inference_steps - 1)) + ) + for i in timesteps + ] + + return state.replace( + num_inference_steps=num_inference_steps, + schedule=jnp.array(schedule, dtype=jnp.float32), + timesteps=timesteps, + ) + + def add_noise_to_input( + self, + state: KarrasVeSchedulerState, + sample: jnp.ndarray, + sigma: float, + key: jax.Array, + ) -> Tuple[jnp.ndarray, float]: + """ + Explicit Langevin-like "churn" step of adding noise to the sample according to a factor gamma_i ≥ 0 to reach a + higher noise level sigma_hat = sigma_i + gamma_i*sigma_i. + + TODO Args: + """ + if self.config.s_min <= sigma <= self.config.s_max: + gamma = min(self.config.s_churn / state.num_inference_steps, 2**0.5 - 1) + else: + gamma = 0 + + # sample eps ~ N(0, S_noise^2 * I) + key = random.split(key, num=1) + eps = self.config.s_noise * random.normal(key=key, shape=sample.shape) + sigma_hat = sigma + gamma * sigma + sample_hat = sample + ((sigma_hat**2 - sigma**2) ** 0.5 * eps) + + return sample_hat, sigma_hat + + def step( + self, + state: KarrasVeSchedulerState, + model_output: jnp.ndarray, + sigma_hat: float, + sigma_prev: float, + sample_hat: jnp.ndarray, + return_dict: bool = True, + ) -> Union[FlaxKarrasVeOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + state (`KarrasVeSchedulerState`): the `FlaxKarrasVeScheduler` state data class. + model_output (`torch.FloatTensor` or `np.ndarray`): direct output from learned diffusion model. + sigma_hat (`float`): TODO + sigma_prev (`float`): TODO + sample_hat (`torch.FloatTensor` or `np.ndarray`): TODO + return_dict (`bool`): option for returning tuple rather than FlaxKarrasVeOutput class + + Returns: + [`~schedulers.scheduling_karras_ve_flax.FlaxKarrasVeOutput`] or `tuple`: Updated sample in the diffusion + chain and derivative. [`~schedulers.scheduling_karras_ve_flax.FlaxKarrasVeOutput`] if `return_dict` is + True, otherwise a `tuple`. When returning a tuple, the first element is the sample tensor. + """ + + pred_original_sample = sample_hat + sigma_hat * model_output + derivative = (sample_hat - pred_original_sample) / sigma_hat + sample_prev = sample_hat + (sigma_prev - sigma_hat) * derivative + + if not return_dict: + return (sample_prev, derivative, state) + + return FlaxKarrasVeOutput(prev_sample=sample_prev, derivative=derivative, state=state) + + def step_correct( + self, + state: KarrasVeSchedulerState, + model_output: jnp.ndarray, + sigma_hat: float, + sigma_prev: float, + sample_hat: jnp.ndarray, + sample_prev: jnp.ndarray, + derivative: jnp.ndarray, + return_dict: bool = True, + ) -> Union[FlaxKarrasVeOutput, Tuple]: + """ + Correct the predicted sample based on the output model_output of the network. TODO complete description + + Args: + state (`KarrasVeSchedulerState`): the `FlaxKarrasVeScheduler` state data class. + model_output (`torch.FloatTensor` or `np.ndarray`): direct output from learned diffusion model. + sigma_hat (`float`): TODO + sigma_prev (`float`): TODO + sample_hat (`torch.FloatTensor` or `np.ndarray`): TODO + sample_prev (`torch.FloatTensor` or `np.ndarray`): TODO + derivative (`torch.FloatTensor` or `np.ndarray`): TODO + return_dict (`bool`): option for returning tuple rather than FlaxKarrasVeOutput class + + Returns: + prev_sample (TODO): updated sample in the diffusion chain. derivative (TODO): TODO + + """ + pred_original_sample = sample_prev + sigma_prev * model_output + derivative_corr = (sample_prev - pred_original_sample) / sigma_prev + sample_prev = sample_hat + (sigma_prev - sigma_hat) * (0.5 * derivative + 0.5 * derivative_corr) + + if not return_dict: + return (sample_prev, derivative, state) + + return FlaxKarrasVeOutput(prev_sample=sample_prev, derivative=derivative, state=state) + + def add_noise(self, state: KarrasVeSchedulerState, original_samples, noise, timesteps): + raise NotImplementedError() diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_lcm.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_lcm.py new file mode 100755 index 0000000..846558b --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_lcm.py @@ -0,0 +1,660 @@ +# Copyright 2024 Stanford University Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion +# and https://github.com/hojonathanho/diffusion + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput, logging +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import SchedulerMixin + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class LCMSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + denoised: Optional[torch.FloatTensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas: torch.FloatTensor) -> torch.FloatTensor: + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.FloatTensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.FloatTensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class LCMScheduler(SchedulerMixin, ConfigMixin): + """ + `LCMScheduler` extends the denoising procedure introduced in denoising diffusion probabilistic models (DDPMs) with + non-Markovian guidance. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. [`~ConfigMixin`] takes care of storing all config + attributes that are passed in the scheduler's `__init__` function, such as `num_train_timesteps`. They can be + accessed via `scheduler.config.num_train_timesteps`. [`SchedulerMixin`] provides general loading and saving + functionality via the [`SchedulerMixin.save_pretrained`] and [`~SchedulerMixin.from_pretrained`] functions. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + original_inference_steps (`int`, *optional*, defaults to 50): + The default number of inference steps used to generate a linearly-spaced timestep schedule, from which we + will ultimately take `num_inference_steps` evenly spaced timesteps to form the final timestep schedule. + clip_sample (`bool`, defaults to `True`): + Clip the predicted sample for numerical stability. + clip_sample_range (`float`, defaults to 1.0): + The maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + set_alpha_to_one (`bool`, defaults to `True`): + Each diffusion step uses the alphas product value at that step and at the previous one. For the final step + there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`, + otherwise it uses the alpha value at step 0. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True`. + timestep_spacing (`str`, defaults to `"leading"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + timestep_scaling (`float`, defaults to 10.0): + The factor the timesteps will be multiplied by when calculating the consistency model boundary conditions + `c_skip` and `c_out`. Increasing this will decrease the approximation error (although the approximation + error at the default of `10.0` is already pretty small). + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.00085, + beta_end: float = 0.012, + beta_schedule: str = "scaled_linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + original_inference_steps: int = 50, + clip_sample: bool = False, + clip_sample_range: float = 1.0, + set_alpha_to_one: bool = True, + steps_offset: int = 0, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + timestep_spacing: str = "leading", + timestep_scaling: float = 10.0, + rescale_betas_zero_snr: bool = False, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + # Rescale for zero SNR + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + # At every step in ddim, we are looking into the previous alphas_cumprod + # For the final step, there is no previous alphas_cumprod because we are already at 0 + # `set_alpha_to_one` decides whether we set this parameter simply to one or + # whether we use the final alpha of the "non-previous" one. + self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0] + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy().astype(np.int64)) + self.custom_timesteps = False + + self._step_index = None + self._begin_index = None + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + @property + def step_index(self): + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + def set_timesteps( + self, + num_inference_steps: Optional[int] = None, + device: Union[str, torch.device] = None, + original_inference_steps: Optional[int] = None, + timesteps: Optional[List[int]] = None, + strength: int = 1.0, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`, *optional*): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + original_inference_steps (`int`, *optional*): + The original number of inference steps, which will be used to generate a linearly-spaced timestep + schedule (which is different from the standard `diffusers` implementation). We will then take + `num_inference_steps` timesteps from this schedule, evenly spaced in terms of indices, and use that as + our final timestep schedule. If not set, this will default to the `original_inference_steps` attribute. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of equal spacing between timesteps on the training/distillation timestep + schedule is used. If `timesteps` is passed, `num_inference_steps` must be `None`. + """ + # 0. Check inputs + if num_inference_steps is None and timesteps is None: + raise ValueError("Must pass exactly one of `num_inference_steps` or `custom_timesteps`.") + + if num_inference_steps is not None and timesteps is not None: + raise ValueError("Can only pass one of `num_inference_steps` or `custom_timesteps`.") + + # 1. Calculate the LCM original training/distillation timestep schedule. + original_steps = ( + original_inference_steps if original_inference_steps is not None else self.config.original_inference_steps + ) + + if original_steps > self.config.num_train_timesteps: + raise ValueError( + f"`original_steps`: {original_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + # LCM Timesteps Setting + # The skipping step parameter k from the paper. + k = self.config.num_train_timesteps // original_steps + # LCM Training/Distillation Steps Schedule + # Currently, only a linearly-spaced schedule is supported (same as in the LCM distillation scripts). + lcm_origin_timesteps = np.asarray(list(range(1, int(original_steps * strength) + 1))) * k - 1 + + # 2. Calculate the LCM inference timestep schedule. + if timesteps is not None: + # 2.1 Handle custom timestep schedules. + train_timesteps = set(lcm_origin_timesteps) + non_train_timesteps = [] + for i in range(1, len(timesteps)): + if timesteps[i] >= timesteps[i - 1]: + raise ValueError("`custom_timesteps` must be in descending order.") + + if timesteps[i] not in train_timesteps: + non_train_timesteps.append(timesteps[i]) + + if timesteps[0] >= self.config.num_train_timesteps: + raise ValueError( + f"`timesteps` must start before `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps}." + ) + + # Raise warning if timestep schedule does not start with self.config.num_train_timesteps - 1 + if strength == 1.0 and timesteps[0] != self.config.num_train_timesteps - 1: + logger.warning( + f"The first timestep on the custom timestep schedule is {timesteps[0]}, not" + f" `self.config.num_train_timesteps - 1`: {self.config.num_train_timesteps - 1}. You may get" + f" unexpected results when using this timestep schedule." + ) + + # Raise warning if custom timestep schedule contains timesteps not on original timestep schedule + if non_train_timesteps: + logger.warning( + f"The custom timestep schedule contains the following timesteps which are not on the original" + f" training/distillation timestep schedule: {non_train_timesteps}. You may get unexpected results" + f" when using this timestep schedule." + ) + + # Raise warning if custom timestep schedule is longer than original_steps + if len(timesteps) > original_steps: + logger.warning( + f"The number of timesteps in the custom timestep schedule is {len(timesteps)}, which exceeds the" + f" the length of the timestep schedule used for training: {original_steps}. You may get some" + f" unexpected results when using this timestep schedule." + ) + + timesteps = np.array(timesteps, dtype=np.int64) + self.num_inference_steps = len(timesteps) + self.custom_timesteps = True + + # Apply strength (e.g. for img2img pipelines) (see StableDiffusionImg2ImgPipeline.get_timesteps) + init_timestep = min(int(self.num_inference_steps * strength), self.num_inference_steps) + t_start = max(self.num_inference_steps - init_timestep, 0) + timesteps = timesteps[t_start * self.order :] + # TODO: also reset self.num_inference_steps? + else: + # 2.2 Create the "standard" LCM inference timestep schedule. + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + skipping_step = len(lcm_origin_timesteps) // num_inference_steps + + if skipping_step < 1: + raise ValueError( + f"The combination of `original_steps x strength`: {original_steps} x {strength} is smaller than `num_inference_steps`: {num_inference_steps}. Make sure to either reduce `num_inference_steps` to a value smaller than {int(original_steps * strength)} or increase `strength` to a value higher than {float(num_inference_steps / original_steps)}." + ) + + self.num_inference_steps = num_inference_steps + + if num_inference_steps > original_steps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `original_inference_steps`:" + f" {original_steps} because the final timestep schedule will be a subset of the" + f" `original_inference_steps`-sized initial timestep schedule." + ) + + # LCM Inference Steps Schedule + lcm_origin_timesteps = lcm_origin_timesteps[::-1].copy() + # Select (approximately) evenly spaced indices from lcm_origin_timesteps. + inference_indices = np.linspace(0, len(lcm_origin_timesteps), num=num_inference_steps, endpoint=False) + inference_indices = np.floor(inference_indices).astype(np.int64) + timesteps = lcm_origin_timesteps[inference_indices] + + self.timesteps = torch.from_numpy(timesteps).to(device=device, dtype=torch.long) + + self._step_index = None + self._begin_index = None + + def get_scalings_for_boundary_condition_discrete(self, timestep): + self.sigma_data = 0.5 # Default: 0.5 + scaled_timestep = timestep * self.config.timestep_scaling + + c_skip = self.sigma_data**2 / (scaled_timestep**2 + self.sigma_data**2) + c_out = scaled_timestep / (scaled_timestep**2 + self.sigma_data**2) ** 0.5 + return c_skip, c_out + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[LCMSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_lcm.LCMSchedulerOutput`] or `tuple`. + Returns: + [`~schedulers.scheduling_utils.LCMSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_lcm.LCMSchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + # 1. get previous step value + prev_step_index = self.step_index + 1 + if prev_step_index < len(self.timesteps): + prev_timestep = self.timesteps[prev_step_index] + else: + prev_timestep = timestep + + # 2. compute alphas, betas + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + # 3. Get scalings for boundary conditions + c_skip, c_out = self.get_scalings_for_boundary_condition_discrete(timestep) + + # 4. Compute the predicted original sample x_0 based on the model parameterization + if self.config.prediction_type == "epsilon": # noise-prediction + predicted_original_sample = (sample - beta_prod_t.sqrt() * model_output) / alpha_prod_t.sqrt() + elif self.config.prediction_type == "sample": # x-prediction + predicted_original_sample = model_output + elif self.config.prediction_type == "v_prediction": # v-prediction + predicted_original_sample = alpha_prod_t.sqrt() * sample - beta_prod_t.sqrt() * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or" + " `v_prediction` for `LCMScheduler`." + ) + + # 5. Clip or threshold "predicted x_0" + if self.config.thresholding: + predicted_original_sample = self._threshold_sample(predicted_original_sample) + elif self.config.clip_sample: + predicted_original_sample = predicted_original_sample.clamp( + -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 6. Denoise model output using boundary conditions + denoised = c_out * predicted_original_sample + c_skip * sample + + # 7. Sample and inject noise z ~ N(0, I) for MultiStep Inference + # Noise is not used on the final timestep of the timestep schedule. + # This also means that noise is not used for one-step sampling. + if self.step_index != self.num_inference_steps - 1: + noise = randn_tensor( + model_output.shape, generator=generator, device=model_output.device, dtype=denoised.dtype + ) + prev_sample = alpha_prod_t_prev.sqrt() * denoised + beta_prod_t_prev.sqrt() * noise + else: + prev_sample = denoised + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample, denoised) + + return LCMSchedulerOutput(prev_sample=prev_sample, denoised=denoised) + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + # Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement + # for the subsequent add_noise calls + self.alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.get_velocity + def get_velocity( + self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as sample + self.alphas_cumprod = self.alphas_cumprod.to(device=sample.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=sample.dtype) + timesteps = timesteps.to(sample.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(sample.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample + return velocity + + def __len__(self): + return self.config.num_train_timesteps + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.previous_timestep + def previous_timestep(self, timestep): + if self.custom_timesteps: + index = (self.timesteps == timestep).nonzero(as_tuple=True)[0][0] + if index == self.timesteps.shape[0] - 1: + prev_t = torch.tensor(-1) + else: + prev_t = self.timesteps[index + 1] + else: + num_inference_steps = ( + self.num_inference_steps if self.num_inference_steps else self.config.num_train_timesteps + ) + prev_t = timestep - self.config.num_train_timesteps // num_inference_steps + + return prev_t diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_lms_discrete.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_lms_discrete.py new file mode 100755 index 0000000..43a0ba4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_lms_discrete.py @@ -0,0 +1,475 @@ +# Copyright 2024 Katherine Crowson and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import math +import warnings +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch +from scipy import integrate + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->LMSDiscrete +class LMSDiscreteSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class LMSDiscreteScheduler(SchedulerMixin, ConfigMixin): + """ + A linear multistep scheduler for discrete beta schedules. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear` or `scaled_linear`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + use_karras_sigmas: Optional[bool] = False, + prediction_type: str = "epsilon", + timestep_spacing: str = "linspace", + steps_offset: int = 0, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + sigmas = np.concatenate([sigmas[::-1], [0.0]]).astype(np.float32) + self.sigmas = torch.from_numpy(sigmas) + + # setable values + self.num_inference_steps = None + self.use_karras_sigmas = use_karras_sigmas + self.set_timesteps(num_train_timesteps, None) + self.derivatives = [] + self.is_scale_input_called = False + + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def init_noise_sigma(self): + # standard deviation of the initial noise distribution + if self.config.timestep_spacing in ["linspace", "trailing"]: + return self.sigmas.max() + + return (self.sigmas.max() ** 2 + 1) ** 0.5 + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def scale_model_input( + self, sample: torch.FloatTensor, timestep: Union[float, torch.FloatTensor] + ) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`float` or `torch.FloatTensor`): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + + if self.step_index is None: + self._init_step_index(timestep) + + sigma = self.sigmas[self.step_index] + sample = sample / ((sigma**2 + 1) ** 0.5) + self.is_scale_input_called = True + return sample + + def get_lms_coefficient(self, order, t, current_order): + """ + Compute the linear multistep coefficient. + + Args: + order (): + t (): + current_order (): + """ + + def lms_derivative(tau): + prod = 1.0 + for k in range(order): + if current_order == k: + continue + prod *= (tau - self.sigmas[t - k]) / (self.sigmas[t - current_order] - self.sigmas[t - k]) + return prod + + integrated_coeff = integrate.quad(lms_derivative, self.sigmas[t], self.sigmas[t + 1], epsrel=1e-4)[0] + + return integrated_coeff + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + self.num_inference_steps = num_inference_steps + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps, dtype=np.float32)[ + ::-1 + ].copy() + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.float32) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(self.config.num_train_timesteps, 0, -step_ratio)).round().copy().astype(np.float32) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + log_sigmas = np.log(sigmas) + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + + if self.use_karras_sigmas: + sigmas = self._convert_to_karras(in_sigmas=sigmas) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]) + + sigmas = np.concatenate([sigmas, [0.0]]).astype(np.float32) + + self.sigmas = torch.from_numpy(sigmas).to(device=device) + self.timesteps = torch.from_numpy(timesteps).to(device=device) + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + self.derivatives = [] + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + # copied from diffusers.schedulers.scheduling_euler_discrete._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # copied from diffusers.schedulers.scheduling_euler_discrete._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.FloatTensor) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + sigma_min: float = in_sigmas[-1].item() + sigma_max: float = in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, self.num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + def step( + self, + model_output: torch.FloatTensor, + timestep: Union[float, torch.FloatTensor], + sample: torch.FloatTensor, + order: int = 4, + return_dict: bool = True, + ) -> Union[LMSDiscreteSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`float` or `torch.FloatTensor`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + order (`int`, defaults to 4): + The order of the linear multistep method. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or tuple. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if not self.is_scale_input_called: + warnings.warn( + "The `scale_model_input` function should be called before `step` to ensure correct denoising. " + "See `StableDiffusionPipeline` for a usage example." + ) + + if self.step_index is None: + self._init_step_index(timestep) + + sigma = self.sigmas[self.step_index] + + # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise + if self.config.prediction_type == "epsilon": + pred_original_sample = sample - sigma * model_output + elif self.config.prediction_type == "v_prediction": + # * c_out + input * c_skip + pred_original_sample = model_output * (-sigma / (sigma**2 + 1) ** 0.5) + (sample / (sigma**2 + 1)) + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`" + ) + + # 2. Convert to an ODE derivative + derivative = (sample - pred_original_sample) / sigma + self.derivatives.append(derivative) + if len(self.derivatives) > order: + self.derivatives.pop(0) + + # 3. Compute linear multistep coefficients + order = min(self.step_index + 1, order) + lms_coeffs = [self.get_lms_coefficient(order, self.step_index, curr_order) for curr_order in range(order)] + + # 4. Compute previous sample based on the derivatives path + prev_sample = sample + sum( + coeff * derivative for coeff, derivative in zip(lms_coeffs, reversed(self.derivatives)) + ) + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return LMSDiscreteSchedulerOutput(prev_sample=prev_sample, pred_original_sample=pred_original_sample) + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # self.begin_index is None when scheduler is used for training, or pipeline does not implement set_begin_index + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + noisy_samples = original_samples + noise * sigma + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_lms_discrete_flax.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_lms_discrete_flax.py new file mode 100755 index 0000000..f1169cc --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_lms_discrete_flax.py @@ -0,0 +1,283 @@ +# Copyright 2024 Katherine Crowson and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import flax +import jax.numpy as jnp +from scipy import integrate + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils_flax import ( + CommonSchedulerState, + FlaxKarrasDiffusionSchedulers, + FlaxSchedulerMixin, + FlaxSchedulerOutput, + broadcast_to_shape_from_left, +) + + +@flax.struct.dataclass +class LMSDiscreteSchedulerState: + common: CommonSchedulerState + + # setable values + init_noise_sigma: jnp.ndarray + timesteps: jnp.ndarray + sigmas: jnp.ndarray + num_inference_steps: Optional[int] = None + + # running values + derivatives: Optional[jnp.ndarray] = None + + @classmethod + def create( + cls, common: CommonSchedulerState, init_noise_sigma: jnp.ndarray, timesteps: jnp.ndarray, sigmas: jnp.ndarray + ): + return cls(common=common, init_noise_sigma=init_noise_sigma, timesteps=timesteps, sigmas=sigmas) + + +@dataclass +class FlaxLMSSchedulerOutput(FlaxSchedulerOutput): + state: LMSDiscreteSchedulerState + + +class FlaxLMSDiscreteScheduler(FlaxSchedulerMixin, ConfigMixin): + """ + Linear Multistep Scheduler for discrete beta schedules. Based on the original k-diffusion implementation by + Katherine Crowson: + https://github.com/crowsonkb/k-diffusion/blob/481677d114f6ea445aa009cf5bd7a9cdee909e47/k_diffusion/sampling.py#L181 + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + beta_start (`float`): the starting `beta` value of inference. + beta_end (`float`): the final `beta` value. + beta_schedule (`str`): + the beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear` or `scaled_linear`. + trained_betas (`jnp.ndarray`, optional): + option to pass an array of betas directly to the constructor to bypass `beta_start`, `beta_end` etc. + prediction_type (`str`, default `epsilon`, optional): + prediction type of the scheduler function, one of `epsilon` (predicting the noise of the diffusion + process), `sample` (directly predicting the noisy sample`) or `v_prediction` (see section 2.4 + https://imagen.research.google/video/paper.pdf) + dtype (`jnp.dtype`, *optional*, defaults to `jnp.float32`): + the `dtype` used for params and computation. + """ + + _compatibles = [e.name for e in FlaxKarrasDiffusionSchedulers] + + dtype: jnp.dtype + + @property + def has_state(self): + return True + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[jnp.ndarray] = None, + prediction_type: str = "epsilon", + dtype: jnp.dtype = jnp.float32, + ): + self.dtype = dtype + + def create_state(self, common: Optional[CommonSchedulerState] = None) -> LMSDiscreteSchedulerState: + if common is None: + common = CommonSchedulerState.create(self) + + timesteps = jnp.arange(0, self.config.num_train_timesteps).round()[::-1] + sigmas = ((1 - common.alphas_cumprod) / common.alphas_cumprod) ** 0.5 + + # standard deviation of the initial noise distribution + init_noise_sigma = sigmas.max() + + return LMSDiscreteSchedulerState.create( + common=common, + init_noise_sigma=init_noise_sigma, + timesteps=timesteps, + sigmas=sigmas, + ) + + def scale_model_input(self, state: LMSDiscreteSchedulerState, sample: jnp.ndarray, timestep: int) -> jnp.ndarray: + """ + Scales the denoising model input by `(sigma**2 + 1) ** 0.5` to match the K-LMS algorithm. + + Args: + state (`LMSDiscreteSchedulerState`): + the `FlaxLMSDiscreteScheduler` state data class instance. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + timestep (`int`): + current discrete timestep in the diffusion chain. + + Returns: + `jnp.ndarray`: scaled input sample + """ + (step_index,) = jnp.where(state.timesteps == timestep, size=1) + step_index = step_index[0] + + sigma = state.sigmas[step_index] + sample = sample / ((sigma**2 + 1) ** 0.5) + return sample + + def get_lms_coefficient(self, state: LMSDiscreteSchedulerState, order, t, current_order): + """ + Compute a linear multistep coefficient. + + Args: + order (TODO): + t (TODO): + current_order (TODO): + """ + + def lms_derivative(tau): + prod = 1.0 + for k in range(order): + if current_order == k: + continue + prod *= (tau - state.sigmas[t - k]) / (state.sigmas[t - current_order] - state.sigmas[t - k]) + return prod + + integrated_coeff = integrate.quad(lms_derivative, state.sigmas[t], state.sigmas[t + 1], epsrel=1e-4)[0] + + return integrated_coeff + + def set_timesteps( + self, state: LMSDiscreteSchedulerState, num_inference_steps: int, shape: Tuple = () + ) -> LMSDiscreteSchedulerState: + """ + Sets the timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + state (`LMSDiscreteSchedulerState`): + the `FlaxLMSDiscreteScheduler` state data class instance. + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + """ + + timesteps = jnp.linspace(self.config.num_train_timesteps - 1, 0, num_inference_steps, dtype=self.dtype) + + low_idx = jnp.floor(timesteps).astype(jnp.int32) + high_idx = jnp.ceil(timesteps).astype(jnp.int32) + + frac = jnp.mod(timesteps, 1.0) + + sigmas = ((1 - state.common.alphas_cumprod) / state.common.alphas_cumprod) ** 0.5 + sigmas = (1 - frac) * sigmas[low_idx] + frac * sigmas[high_idx] + sigmas = jnp.concatenate([sigmas, jnp.array([0.0], dtype=self.dtype)]) + + timesteps = timesteps.astype(jnp.int32) + + # initial running values + derivatives = jnp.zeros((0,) + shape, dtype=self.dtype) + + return state.replace( + timesteps=timesteps, + sigmas=sigmas, + num_inference_steps=num_inference_steps, + derivatives=derivatives, + ) + + def step( + self, + state: LMSDiscreteSchedulerState, + model_output: jnp.ndarray, + timestep: int, + sample: jnp.ndarray, + order: int = 4, + return_dict: bool = True, + ) -> Union[FlaxLMSSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + state (`LMSDiscreteSchedulerState`): the `FlaxLMSDiscreteScheduler` state data class instance. + model_output (`jnp.ndarray`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + order: coefficient for multi-step inference. + return_dict (`bool`): option for returning tuple rather than FlaxLMSSchedulerOutput class + + Returns: + [`FlaxLMSSchedulerOutput`] or `tuple`: [`FlaxLMSSchedulerOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is the sample tensor. + + """ + if state.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + sigma = state.sigmas[timestep] + + # 1. compute predicted original sample (x_0) from sigma-scaled predicted noise + if self.config.prediction_type == "epsilon": + pred_original_sample = sample - sigma * model_output + elif self.config.prediction_type == "v_prediction": + # * c_out + input * c_skip + pred_original_sample = model_output * (-sigma / (sigma**2 + 1) ** 0.5) + (sample / (sigma**2 + 1)) + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, or `v_prediction`" + ) + + # 2. Convert to an ODE derivative + derivative = (sample - pred_original_sample) / sigma + state = state.replace(derivatives=jnp.append(state.derivatives, derivative)) + if len(state.derivatives) > order: + state = state.replace(derivatives=jnp.delete(state.derivatives, 0)) + + # 3. Compute linear multistep coefficients + order = min(timestep + 1, order) + lms_coeffs = [self.get_lms_coefficient(state, order, timestep, curr_order) for curr_order in range(order)] + + # 4. Compute previous sample based on the derivatives path + prev_sample = sample + sum( + coeff * derivative for coeff, derivative in zip(lms_coeffs, reversed(state.derivatives)) + ) + + if not return_dict: + return (prev_sample, state) + + return FlaxLMSSchedulerOutput(prev_sample=prev_sample, state=state) + + def add_noise( + self, + state: LMSDiscreteSchedulerState, + original_samples: jnp.ndarray, + noise: jnp.ndarray, + timesteps: jnp.ndarray, + ) -> jnp.ndarray: + sigma = state.sigmas[timesteps].flatten() + sigma = broadcast_to_shape_from_left(sigma, noise.shape) + + noisy_samples = original_samples + noise * sigma + + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_pndm.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_pndm.py new file mode 100755 index 0000000..a8f8b09 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_pndm.py @@ -0,0 +1,476 @@ +# Copyright 2024 Zhejiang University Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class PNDMScheduler(SchedulerMixin, ConfigMixin): + """ + `PNDMScheduler` uses pseudo numerical methods for diffusion models such as the Runge-Kutta and linear multi-step + method. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + skip_prk_steps (`bool`, defaults to `False`): + Allows the scheduler to skip the Runge-Kutta steps defined in the original paper as being required before + PLMS steps. + set_alpha_to_one (`bool`, defaults to `False`): + Each diffusion step uses the alphas product value at that step and at the previous one. For the final step + there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`, + otherwise it uses the alpha value at step 0. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process) + or `v_prediction` (see section 2.4 of [Imagen Video](https://imagen.research.google/video/paper.pdf) + paper). + timestep_spacing (`str`, defaults to `"leading"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + skip_prk_steps: bool = False, + set_alpha_to_one: bool = False, + prediction_type: str = "epsilon", + timestep_spacing: str = "leading", + steps_offset: int = 0, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0] + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # For now we only support F-PNDM, i.e. the runge-kutta method + # For more information on the algorithm please take a look at the paper: https://arxiv.org/pdf/2202.09778.pdf + # mainly at formula (9), (12), (13) and the Algorithm 2. + self.pndm_order = 4 + + # running values + self.cur_model_output = 0 + self.counter = 0 + self.cur_sample = None + self.ets = [] + + # setable values + self.num_inference_steps = None + self._timesteps = np.arange(0, num_train_timesteps)[::-1].copy() + self.prk_timesteps = None + self.plms_timesteps = None + self.timesteps = None + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + + self.num_inference_steps = num_inference_steps + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + self._timesteps = ( + np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps).round().astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + self._timesteps = (np.arange(0, num_inference_steps) * step_ratio).round() + self._timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / self.num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + self._timesteps = np.round(np.arange(self.config.num_train_timesteps, 0, -step_ratio))[::-1].astype( + np.int64 + ) + self._timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + if self.config.skip_prk_steps: + # for some models like stable diffusion the prk steps can/should be skipped to + # produce better results. When using PNDM with `self.config.skip_prk_steps` the implementation + # is based on crowsonkb's PLMS sampler implementation: https://github.com/CompVis/latent-diffusion/pull/51 + self.prk_timesteps = np.array([]) + self.plms_timesteps = np.concatenate([self._timesteps[:-1], self._timesteps[-2:-1], self._timesteps[-1:]])[ + ::-1 + ].copy() + else: + prk_timesteps = np.array(self._timesteps[-self.pndm_order :]).repeat(2) + np.tile( + np.array([0, self.config.num_train_timesteps // num_inference_steps // 2]), self.pndm_order + ) + self.prk_timesteps = (prk_timesteps[:-1].repeat(2)[1:-1])[::-1].copy() + self.plms_timesteps = self._timesteps[:-3][ + ::-1 + ].copy() # we copy to avoid having negative strides which are not supported by torch.from_numpy + + timesteps = np.concatenate([self.prk_timesteps, self.plms_timesteps]).astype(np.int64) + self.timesteps = torch.from_numpy(timesteps).to(device) + + self.ets = [] + self.counter = 0 + self.cur_model_output = 0 + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise), and calls [`~PNDMScheduler.step_prk`] + or [`~PNDMScheduler.step_plms`] depending on the internal variable `counter`. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.counter < len(self.prk_timesteps) and not self.config.skip_prk_steps: + return self.step_prk(model_output=model_output, timestep=timestep, sample=sample, return_dict=return_dict) + else: + return self.step_plms(model_output=model_output, timestep=timestep, sample=sample, return_dict=return_dict) + + def step_prk( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the Runge-Kutta method. It performs four forward passes to approximate the solution to the differential + equation. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or tuple. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + diff_to_prev = 0 if self.counter % 2 else self.config.num_train_timesteps // self.num_inference_steps // 2 + prev_timestep = timestep - diff_to_prev + timestep = self.prk_timesteps[self.counter // 4 * 4] + + if self.counter % 4 == 0: + self.cur_model_output += 1 / 6 * model_output + self.ets.append(model_output) + self.cur_sample = sample + elif (self.counter - 1) % 4 == 0: + self.cur_model_output += 1 / 3 * model_output + elif (self.counter - 2) % 4 == 0: + self.cur_model_output += 1 / 3 * model_output + elif (self.counter - 3) % 4 == 0: + model_output = self.cur_model_output + 1 / 6 * model_output + self.cur_model_output = 0 + + # cur_sample should not be `None` + cur_sample = self.cur_sample if self.cur_sample is not None else sample + + prev_sample = self._get_prev_sample(cur_sample, timestep, prev_timestep, model_output) + self.counter += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def step_plms( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the linear multistep method. It performs one forward pass multiple times to approximate the solution. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or tuple. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if not self.config.skip_prk_steps and len(self.ets) < 3: + raise ValueError( + f"{self.__class__} can only be run AFTER scheduler has been run " + "in 'prk' mode for at least 12 iterations " + "See: https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/pipeline_pndm.py " + "for more information." + ) + + prev_timestep = timestep - self.config.num_train_timesteps // self.num_inference_steps + + if self.counter != 1: + self.ets = self.ets[-3:] + self.ets.append(model_output) + else: + prev_timestep = timestep + timestep = timestep + self.config.num_train_timesteps // self.num_inference_steps + + if len(self.ets) == 1 and self.counter == 0: + model_output = model_output + self.cur_sample = sample + elif len(self.ets) == 1 and self.counter == 1: + model_output = (model_output + self.ets[-1]) / 2 + sample = self.cur_sample + self.cur_sample = None + elif len(self.ets) == 2: + model_output = (3 * self.ets[-1] - self.ets[-2]) / 2 + elif len(self.ets) == 3: + model_output = (23 * self.ets[-1] - 16 * self.ets[-2] + 5 * self.ets[-3]) / 12 + else: + model_output = (1 / 24) * (55 * self.ets[-1] - 59 * self.ets[-2] + 37 * self.ets[-3] - 9 * self.ets[-4]) + + prev_sample = self._get_prev_sample(sample, timestep, prev_timestep, model_output) + self.counter += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def _get_prev_sample(self, sample, timestep, prev_timestep, model_output): + # See formula (9) of PNDM paper https://arxiv.org/pdf/2202.09778.pdf + # this function computes x_(t−δ) using the formula of (9) + # Note that x_t needs to be added to both sides of the equation + + # Notation ( -> + # alpha_prod_t -> α_t + # alpha_prod_t_prev -> α_(t−δ) + # beta_prod_t -> (1 - α_t) + # beta_prod_t_prev -> (1 - α_(t−δ)) + # sample -> x_t + # model_output -> e_θ(x_t, t) + # prev_sample -> x_(t−δ) + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + if self.config.prediction_type == "v_prediction": + model_output = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + elif self.config.prediction_type != "epsilon": + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon` or `v_prediction`" + ) + + # corresponds to (α_(t−δ) - α_t) divided by + # denominator of x_t in formula (9) and plus 1 + # Note: (α_(t−δ) - α_t) / (sqrt(α_t) * (sqrt(α_(t−δ)) + sqr(α_t))) = + # sqrt(α_(t−δ)) / sqrt(α_t)) + sample_coeff = (alpha_prod_t_prev / alpha_prod_t) ** (0.5) + + # corresponds to denominator of e_θ(x_t, t) in formula (9) + model_output_denom_coeff = alpha_prod_t * beta_prod_t_prev ** (0.5) + ( + alpha_prod_t * beta_prod_t * alpha_prod_t_prev + ) ** (0.5) + + # full formula (9) + prev_sample = ( + sample_coeff * sample - (alpha_prod_t_prev - alpha_prod_t) * model_output / model_output_denom_coeff + ) + + return prev_sample + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + # Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement + # for the subsequent add_noise calls + self.alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_pndm_flax.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_pndm_flax.py new file mode 100755 index 0000000..3ac3ba5 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_pndm_flax.py @@ -0,0 +1,509 @@ +# Copyright 2024 Zhejiang University Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/ermongroup/ddim + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import flax +import jax +import jax.numpy as jnp + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils_flax import ( + CommonSchedulerState, + FlaxKarrasDiffusionSchedulers, + FlaxSchedulerMixin, + FlaxSchedulerOutput, + add_noise_common, +) + + +@flax.struct.dataclass +class PNDMSchedulerState: + common: CommonSchedulerState + final_alpha_cumprod: jnp.ndarray + + # setable values + init_noise_sigma: jnp.ndarray + timesteps: jnp.ndarray + num_inference_steps: Optional[int] = None + prk_timesteps: Optional[jnp.ndarray] = None + plms_timesteps: Optional[jnp.ndarray] = None + + # running values + cur_model_output: Optional[jnp.ndarray] = None + counter: Optional[jnp.int32] = None + cur_sample: Optional[jnp.ndarray] = None + ets: Optional[jnp.ndarray] = None + + @classmethod + def create( + cls, + common: CommonSchedulerState, + final_alpha_cumprod: jnp.ndarray, + init_noise_sigma: jnp.ndarray, + timesteps: jnp.ndarray, + ): + return cls( + common=common, + final_alpha_cumprod=final_alpha_cumprod, + init_noise_sigma=init_noise_sigma, + timesteps=timesteps, + ) + + +@dataclass +class FlaxPNDMSchedulerOutput(FlaxSchedulerOutput): + state: PNDMSchedulerState + + +class FlaxPNDMScheduler(FlaxSchedulerMixin, ConfigMixin): + """ + Pseudo numerical methods for diffusion models (PNDM) proposes using more advanced ODE integration techniques, + namely Runge-Kutta method and a linear multi-step method. + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + For more details, see the original paper: https://arxiv.org/abs/2202.09778 + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + beta_start (`float`): the starting `beta` value of inference. + beta_end (`float`): the final `beta` value. + beta_schedule (`str`): + the beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`jnp.ndarray`, optional): + option to pass an array of betas directly to the constructor to bypass `beta_start`, `beta_end` etc. + skip_prk_steps (`bool`): + allows the scheduler to skip the Runge-Kutta steps that are defined in the original paper as being required + before plms steps; defaults to `False`. + set_alpha_to_one (`bool`, default `False`): + each diffusion step uses the value of alphas product at that step and at the previous one. For the final + step there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`, + otherwise it uses the value of alpha at step 0. + steps_offset (`int`, default `0`): + An offset added to the inference steps, as required by some model families. + prediction_type (`str`, default `epsilon`, optional): + prediction type of the scheduler function, one of `epsilon` (predicting the noise of the diffusion + process), `sample` (directly predicting the noisy sample`) or `v_prediction` (see section 2.4 + https://imagen.research.google/video/paper.pdf) + dtype (`jnp.dtype`, *optional*, defaults to `jnp.float32`): + the `dtype` used for params and computation. + """ + + _compatibles = [e.name for e in FlaxKarrasDiffusionSchedulers] + + dtype: jnp.dtype + pndm_order: int + + @property + def has_state(self): + return True + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[jnp.ndarray] = None, + skip_prk_steps: bool = False, + set_alpha_to_one: bool = False, + steps_offset: int = 0, + prediction_type: str = "epsilon", + dtype: jnp.dtype = jnp.float32, + ): + self.dtype = dtype + + # For now we only support F-PNDM, i.e. the runge-kutta method + # For more information on the algorithm please take a look at the paper: https://arxiv.org/pdf/2202.09778.pdf + # mainly at formula (9), (12), (13) and the Algorithm 2. + self.pndm_order = 4 + + def create_state(self, common: Optional[CommonSchedulerState] = None) -> PNDMSchedulerState: + if common is None: + common = CommonSchedulerState.create(self) + + # At every step in ddim, we are looking into the previous alphas_cumprod + # For the final step, there is no previous alphas_cumprod because we are already at 0 + # `set_alpha_to_one` decides whether we set this parameter simply to one or + # whether we use the final alpha of the "non-previous" one. + final_alpha_cumprod = ( + jnp.array(1.0, dtype=self.dtype) if self.config.set_alpha_to_one else common.alphas_cumprod[0] + ) + + # standard deviation of the initial noise distribution + init_noise_sigma = jnp.array(1.0, dtype=self.dtype) + + timesteps = jnp.arange(0, self.config.num_train_timesteps).round()[::-1] + + return PNDMSchedulerState.create( + common=common, + final_alpha_cumprod=final_alpha_cumprod, + init_noise_sigma=init_noise_sigma, + timesteps=timesteps, + ) + + def set_timesteps(self, state: PNDMSchedulerState, num_inference_steps: int, shape: Tuple) -> PNDMSchedulerState: + """ + Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + state (`PNDMSchedulerState`): + the `FlaxPNDMScheduler` state data class instance. + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + shape (`Tuple`): + the shape of the samples to be generated. + """ + + step_ratio = self.config.num_train_timesteps // num_inference_steps + # creates integer timesteps by multiplying by ratio + # rounding to avoid issues when num_inference_step is power of 3 + _timesteps = (jnp.arange(0, num_inference_steps) * step_ratio).round() + self.config.steps_offset + + if self.config.skip_prk_steps: + # for some models like stable diffusion the prk steps can/should be skipped to + # produce better results. When using PNDM with `self.config.skip_prk_steps` the implementation + # is based on crowsonkb's PLMS sampler implementation: https://github.com/CompVis/latent-diffusion/pull/51 + + prk_timesteps = jnp.array([], dtype=jnp.int32) + plms_timesteps = jnp.concatenate([_timesteps[:-1], _timesteps[-2:-1], _timesteps[-1:]])[::-1] + + else: + prk_timesteps = _timesteps[-self.pndm_order :].repeat(2) + jnp.tile( + jnp.array([0, self.config.num_train_timesteps // num_inference_steps // 2], dtype=jnp.int32), + self.pndm_order, + ) + + prk_timesteps = (prk_timesteps[:-1].repeat(2)[1:-1])[::-1] + plms_timesteps = _timesteps[:-3][::-1] + + timesteps = jnp.concatenate([prk_timesteps, plms_timesteps]) + + # initial running values + + cur_model_output = jnp.zeros(shape, dtype=self.dtype) + counter = jnp.int32(0) + cur_sample = jnp.zeros(shape, dtype=self.dtype) + ets = jnp.zeros((4,) + shape, dtype=self.dtype) + + return state.replace( + timesteps=timesteps, + num_inference_steps=num_inference_steps, + prk_timesteps=prk_timesteps, + plms_timesteps=plms_timesteps, + cur_model_output=cur_model_output, + counter=counter, + cur_sample=cur_sample, + ets=ets, + ) + + def scale_model_input( + self, state: PNDMSchedulerState, sample: jnp.ndarray, timestep: Optional[int] = None + ) -> jnp.ndarray: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + state (`PNDMSchedulerState`): the `FlaxPNDMScheduler` state data class instance. + sample (`jnp.ndarray`): input sample + timestep (`int`, optional): current timestep + + Returns: + `jnp.ndarray`: scaled input sample + """ + return sample + + def step( + self, + state: PNDMSchedulerState, + model_output: jnp.ndarray, + timestep: int, + sample: jnp.ndarray, + return_dict: bool = True, + ) -> Union[FlaxPNDMSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + This function calls `step_prk()` or `step_plms()` depending on the internal variable `counter`. + + Args: + state (`PNDMSchedulerState`): the `FlaxPNDMScheduler` state data class instance. + model_output (`jnp.ndarray`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + return_dict (`bool`): option for returning tuple rather than FlaxPNDMSchedulerOutput class + + Returns: + [`FlaxPNDMSchedulerOutput`] or `tuple`: [`FlaxPNDMSchedulerOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is the sample tensor. + + """ + + if state.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.config.skip_prk_steps: + prev_sample, state = self.step_plms(state, model_output, timestep, sample) + else: + prk_prev_sample, prk_state = self.step_prk(state, model_output, timestep, sample) + plms_prev_sample, plms_state = self.step_plms(state, model_output, timestep, sample) + + cond = state.counter < len(state.prk_timesteps) + + prev_sample = jax.lax.select(cond, prk_prev_sample, plms_prev_sample) + + state = state.replace( + cur_model_output=jax.lax.select(cond, prk_state.cur_model_output, plms_state.cur_model_output), + ets=jax.lax.select(cond, prk_state.ets, plms_state.ets), + cur_sample=jax.lax.select(cond, prk_state.cur_sample, plms_state.cur_sample), + counter=jax.lax.select(cond, prk_state.counter, plms_state.counter), + ) + + if not return_dict: + return (prev_sample, state) + + return FlaxPNDMSchedulerOutput(prev_sample=prev_sample, state=state) + + def step_prk( + self, + state: PNDMSchedulerState, + model_output: jnp.ndarray, + timestep: int, + sample: jnp.ndarray, + ) -> Union[FlaxPNDMSchedulerOutput, Tuple]: + """ + Step function propagating the sample with the Runge-Kutta method. RK takes 4 forward passes to approximate the + solution to the differential equation. + + Args: + state (`PNDMSchedulerState`): the `FlaxPNDMScheduler` state data class instance. + model_output (`jnp.ndarray`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + return_dict (`bool`): option for returning tuple rather than FlaxPNDMSchedulerOutput class + + Returns: + [`FlaxPNDMSchedulerOutput`] or `tuple`: [`FlaxPNDMSchedulerOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is the sample tensor. + + """ + + if state.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + diff_to_prev = jnp.where( + state.counter % 2, 0, self.config.num_train_timesteps // state.num_inference_steps // 2 + ) + prev_timestep = timestep - diff_to_prev + timestep = state.prk_timesteps[state.counter // 4 * 4] + + model_output = jax.lax.select( + (state.counter % 4) != 3, + model_output, # remainder 0, 1, 2 + state.cur_model_output + 1 / 6 * model_output, # remainder 3 + ) + + state = state.replace( + cur_model_output=jax.lax.select_n( + state.counter % 4, + state.cur_model_output + 1 / 6 * model_output, # remainder 0 + state.cur_model_output + 1 / 3 * model_output, # remainder 1 + state.cur_model_output + 1 / 3 * model_output, # remainder 2 + jnp.zeros_like(state.cur_model_output), # remainder 3 + ), + ets=jax.lax.select( + (state.counter % 4) == 0, + state.ets.at[0:3].set(state.ets[1:4]).at[3].set(model_output), # remainder 0 + state.ets, # remainder 1, 2, 3 + ), + cur_sample=jax.lax.select( + (state.counter % 4) == 0, + sample, # remainder 0 + state.cur_sample, # remainder 1, 2, 3 + ), + ) + + cur_sample = state.cur_sample + prev_sample = self._get_prev_sample(state, cur_sample, timestep, prev_timestep, model_output) + state = state.replace(counter=state.counter + 1) + + return (prev_sample, state) + + def step_plms( + self, + state: PNDMSchedulerState, + model_output: jnp.ndarray, + timestep: int, + sample: jnp.ndarray, + ) -> Union[FlaxPNDMSchedulerOutput, Tuple]: + """ + Step function propagating the sample with the linear multi-step method. This has one forward pass with multiple + times to approximate the solution. + + Args: + state (`PNDMSchedulerState`): the `FlaxPNDMScheduler` state data class instance. + model_output (`jnp.ndarray`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + return_dict (`bool`): option for returning tuple rather than FlaxPNDMSchedulerOutput class + + Returns: + [`FlaxPNDMSchedulerOutput`] or `tuple`: [`FlaxPNDMSchedulerOutput`] if `return_dict` is True, otherwise a + `tuple`. When returning a tuple, the first element is the sample tensor. + + """ + + if state.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + # NOTE: There is no way to check in the jitted runtime if the prk mode was ran before + + prev_timestep = timestep - self.config.num_train_timesteps // state.num_inference_steps + prev_timestep = jnp.where(prev_timestep > 0, prev_timestep, 0) + + # Reference: + # if state.counter != 1: + # state.ets.append(model_output) + # else: + # prev_timestep = timestep + # timestep = timestep + self.config.num_train_timesteps // state.num_inference_steps + + prev_timestep = jnp.where(state.counter == 1, timestep, prev_timestep) + timestep = jnp.where( + state.counter == 1, timestep + self.config.num_train_timesteps // state.num_inference_steps, timestep + ) + + # Reference: + # if len(state.ets) == 1 and state.counter == 0: + # model_output = model_output + # state.cur_sample = sample + # elif len(state.ets) == 1 and state.counter == 1: + # model_output = (model_output + state.ets[-1]) / 2 + # sample = state.cur_sample + # state.cur_sample = None + # elif len(state.ets) == 2: + # model_output = (3 * state.ets[-1] - state.ets[-2]) / 2 + # elif len(state.ets) == 3: + # model_output = (23 * state.ets[-1] - 16 * state.ets[-2] + 5 * state.ets[-3]) / 12 + # else: + # model_output = (1 / 24) * (55 * state.ets[-1] - 59 * state.ets[-2] + 37 * state.ets[-3] - 9 * state.ets[-4]) + + state = state.replace( + ets=jax.lax.select( + state.counter != 1, + state.ets.at[0:3].set(state.ets[1:4]).at[3].set(model_output), # counter != 1 + state.ets, # counter 1 + ), + cur_sample=jax.lax.select( + state.counter != 1, + sample, # counter != 1 + state.cur_sample, # counter 1 + ), + ) + + state = state.replace( + cur_model_output=jax.lax.select_n( + jnp.clip(state.counter, 0, 4), + model_output, # counter 0 + (model_output + state.ets[-1]) / 2, # counter 1 + (3 * state.ets[-1] - state.ets[-2]) / 2, # counter 2 + (23 * state.ets[-1] - 16 * state.ets[-2] + 5 * state.ets[-3]) / 12, # counter 3 + (1 / 24) + * (55 * state.ets[-1] - 59 * state.ets[-2] + 37 * state.ets[-3] - 9 * state.ets[-4]), # counter >= 4 + ), + ) + + sample = state.cur_sample + model_output = state.cur_model_output + prev_sample = self._get_prev_sample(state, sample, timestep, prev_timestep, model_output) + state = state.replace(counter=state.counter + 1) + + return (prev_sample, state) + + def _get_prev_sample(self, state: PNDMSchedulerState, sample, timestep, prev_timestep, model_output): + # See formula (9) of PNDM paper https://arxiv.org/pdf/2202.09778.pdf + # this function computes x_(t−δ) using the formula of (9) + # Note that x_t needs to be added to both sides of the equation + + # Notation ( -> + # alpha_prod_t -> α_t + # alpha_prod_t_prev -> α_(t−δ) + # beta_prod_t -> (1 - α_t) + # beta_prod_t_prev -> (1 - α_(t−δ)) + # sample -> x_t + # model_output -> e_θ(x_t, t) + # prev_sample -> x_(t−δ) + alpha_prod_t = state.common.alphas_cumprod[timestep] + alpha_prod_t_prev = jnp.where( + prev_timestep >= 0, state.common.alphas_cumprod[prev_timestep], state.final_alpha_cumprod + ) + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + if self.config.prediction_type == "v_prediction": + model_output = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + elif self.config.prediction_type != "epsilon": + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon` or `v_prediction`" + ) + + # corresponds to (α_(t−δ) - α_t) divided by + # denominator of x_t in formula (9) and plus 1 + # Note: (α_(t−δ) - α_t) / (sqrt(α_t) * (sqrt(α_(t−δ)) + sqr(α_t))) = + # sqrt(α_(t−δ)) / sqrt(α_t)) + sample_coeff = (alpha_prod_t_prev / alpha_prod_t) ** (0.5) + + # corresponds to denominator of e_θ(x_t, t) in formula (9) + model_output_denom_coeff = alpha_prod_t * beta_prod_t_prev ** (0.5) + ( + alpha_prod_t * beta_prod_t * alpha_prod_t_prev + ) ** (0.5) + + # full formula (9) + prev_sample = ( + sample_coeff * sample - (alpha_prod_t_prev - alpha_prod_t) * model_output / model_output_denom_coeff + ) + + return prev_sample + + def add_noise( + self, + state: PNDMSchedulerState, + original_samples: jnp.ndarray, + noise: jnp.ndarray, + timesteps: jnp.ndarray, + ) -> jnp.ndarray: + return add_noise_common(state.common, original_samples, noise, timesteps) + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_repaint.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_repaint.py new file mode 100755 index 0000000..0bff07c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_repaint.py @@ -0,0 +1,361 @@ +# Copyright 2024 ETH Zurich Computer Vision Lab and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import SchedulerMixin + + +@dataclass +class RePaintSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's step function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample (x_{t-1}) of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample (x_{0}) based on the model output from + the current timestep. `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: torch.FloatTensor + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class RePaintScheduler(SchedulerMixin, ConfigMixin): + """ + `RePaintScheduler` is a scheduler for DDPM inpainting inside a given mask. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, `squaredcos_cap_v2`, or `sigmoid`. + eta (`float`): + The weight of noise for added noise in diffusion step. If its value is between 0.0 and 1.0 it corresponds + to the DDIM scheduler, and if its value is between -0.0 and 1.0 it corresponds to the DDPM scheduler. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + clip_sample (`bool`, defaults to `True`): + Clip the predicted sample between -1 and 1 for numerical stability. + + """ + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + eta: float = 0.0, + trained_betas: Optional[np.ndarray] = None, + clip_sample: bool = True, + ): + if trained_betas is not None: + self.betas = torch.from_numpy(trained_betas) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + elif beta_schedule == "sigmoid": + # GeoDiff sigmoid schedule + betas = torch.linspace(-6, 6, num_train_timesteps) + self.betas = torch.sigmoid(betas) * (beta_end - beta_start) + beta_start + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + self.one = torch.tensor(1.0) + + self.final_alpha_cumprod = torch.tensor(1.0) + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy()) + + self.eta = eta + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def set_timesteps( + self, + num_inference_steps: int, + jump_length: int = 10, + jump_n_sample: int = 10, + device: Union[str, torch.device] = None, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + jump_length (`int`, defaults to 10): + The number of steps taken forward in time before going backward in time for a single jump (“j” in + RePaint paper). Take a look at Figure 9 and 10 in the paper. + jump_n_sample (`int`, defaults to 10): + The number of times to make a forward time jump for a given chosen time sample. Take a look at Figure 9 + and 10 in the paper. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + + """ + num_inference_steps = min(self.config.num_train_timesteps, num_inference_steps) + self.num_inference_steps = num_inference_steps + + timesteps = [] + + jumps = {} + for j in range(0, num_inference_steps - jump_length, jump_length): + jumps[j] = jump_n_sample - 1 + + t = num_inference_steps + while t >= 1: + t = t - 1 + timesteps.append(t) + + if jumps.get(t, 0) > 0: + jumps[t] = jumps[t] - 1 + for _ in range(jump_length): + t = t + 1 + timesteps.append(t) + + timesteps = np.array(timesteps) * (self.config.num_train_timesteps // self.num_inference_steps) + self.timesteps = torch.from_numpy(timesteps).to(device) + + def _get_variance(self, t): + prev_timestep = t - self.config.num_train_timesteps // self.num_inference_steps + + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + # For t > 0, compute predicted variance βt (see formula (6) and (7) from + # https://arxiv.org/pdf/2006.11239.pdf) and sample from it to get + # previous sample x_{t-1} ~ N(pred_prev_sample, variance) == add + # variance to pred_sample + # Is equivalent to formula (16) in https://arxiv.org/pdf/2010.02502.pdf + # without eta. + # variance = (1 - alpha_prod_t_prev) / (1 - alpha_prod_t) * self.betas[t] + variance = (beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev) + + return variance + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + original_image: torch.FloatTensor, + mask: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[RePaintSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + original_image (`torch.FloatTensor`): + The original image to inpaint on. + mask (`torch.FloatTensor`): + The mask where a value of 0.0 indicates which part of the original image to inpaint. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_repaint.RePaintSchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_repaint.RePaintSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_repaint.RePaintSchedulerOutput`] is returned, + otherwise a tuple is returned where the first element is the sample tensor. + + """ + t = timestep + prev_timestep = timestep - self.config.num_train_timesteps // self.num_inference_steps + + # 1. compute alphas, betas + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + + # 2. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf + pred_original_sample = (sample - beta_prod_t**0.5 * model_output) / alpha_prod_t**0.5 + + # 3. Clip "predicted x_0" + if self.config.clip_sample: + pred_original_sample = torch.clamp(pred_original_sample, -1, 1) + + # We choose to follow RePaint Algorithm 1 to get x_{t-1}, however we + # substitute formula (7) in the algorithm coming from DDPM paper + # (formula (4) Algorithm 2 - Sampling) with formula (12) from DDIM paper. + # DDIM schedule gives the same results as DDPM with eta = 1.0 + # Noise is being reused in 7. and 8., but no impact on quality has + # been observed. + + # 5. Add noise + device = model_output.device + noise = randn_tensor(model_output.shape, generator=generator, device=device, dtype=model_output.dtype) + std_dev_t = self.eta * self._get_variance(timestep) ** 0.5 + + variance = 0 + if t > 0 and self.eta > 0: + variance = std_dev_t * noise + + # 6. compute "direction pointing to x_t" of formula (12) + # from https://arxiv.org/pdf/2010.02502.pdf + pred_sample_direction = (1 - alpha_prod_t_prev - std_dev_t**2) ** 0.5 * model_output + + # 7. compute x_{t-1} of formula (12) from https://arxiv.org/pdf/2010.02502.pdf + prev_unknown_part = alpha_prod_t_prev**0.5 * pred_original_sample + pred_sample_direction + variance + + # 8. Algorithm 1 Line 5 https://arxiv.org/pdf/2201.09865.pdf + prev_known_part = (alpha_prod_t_prev**0.5) * original_image + ((1 - alpha_prod_t_prev) ** 0.5) * noise + + # 9. Algorithm 1 Line 8 https://arxiv.org/pdf/2201.09865.pdf + pred_prev_sample = mask * prev_known_part + (1.0 - mask) * prev_unknown_part + + if not return_dict: + return ( + pred_prev_sample, + pred_original_sample, + ) + + return RePaintSchedulerOutput(prev_sample=pred_prev_sample, pred_original_sample=pred_original_sample) + + def undo_step(self, sample, timestep, generator=None): + n = self.config.num_train_timesteps // self.num_inference_steps + + for i in range(n): + beta = self.betas[timestep + i] + if sample.device.type == "mps": + # randn does not work reproducibly on mps + noise = randn_tensor(sample.shape, dtype=sample.dtype, generator=generator) + noise = noise.to(sample.device) + else: + noise = randn_tensor(sample.shape, generator=generator, device=sample.device, dtype=sample.dtype) + + # 10. Algorithm 1 Line 10 https://arxiv.org/pdf/2201.09865.pdf + sample = (1 - beta) ** 0.5 * sample + beta**0.5 * noise + + return sample + + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + raise NotImplementedError("Use `DDPMScheduler.add_noise()` to train for sampling with RePaint.") + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_sasolver.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_sasolver.py new file mode 100755 index 0000000..b46f6de --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_sasolver.py @@ -0,0 +1,1124 @@ +# Copyright 2024 Shuchen Xue, etc. in University of Chinese Academy of Sciences Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: check https://arxiv.org/abs/2309.05019 +# The codebase is modified based on https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py + +import math +from typing import Callable, List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import deprecate +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class SASolverScheduler(SchedulerMixin, ConfigMixin): + """ + `SASolverScheduler` is a fast dedicated high-order solver for diffusion SDEs. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + predictor_order (`int`, defaults to 2): + The predictor order which can be `1` or `2` or `3` or '4'. It is recommended to use `predictor_order=2` for guided + sampling, and `predictor_order=3` for unconditional sampling. + corrector_order (`int`, defaults to 2): + The corrector order which can be `1` or `2` or `3` or '4'. It is recommended to use `corrector_order=2` for guided + sampling, and `corrector_order=3` for unconditional sampling. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + tau_func (`Callable`, *optional*): + Stochasticity during the sampling. Default in init is `lambda t: 1 if t >= 200 and t <= 800 else 0`. SA-Solver + will sample from vanilla diffusion ODE if tau_func is set to `lambda t: 0`. SA-Solver will sample from vanilla + diffusion SDE if tau_func is set to `lambda t: 1`. For more details, please check https://arxiv.org/abs/2309.05019 + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and + `algorithm_type="dpmsolver++"`. + algorithm_type (`str`, defaults to `data_prediction`): + Algorithm type for the solver; can be `data_prediction` or `noise_prediction`. It is recommended to use `data_prediction` + with `solver_order=2` for guided sampling like in Stable Diffusion. + lower_order_final (`bool`, defaults to `True`): + Whether to use lower-order solvers in the final steps. Default = True. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + lambda_min_clipped (`float`, defaults to `-inf`): + Clipping threshold for the minimum value of `lambda(t)` for numerical stability. This is critical for the + cosine (`squaredcos_cap_v2`) noise schedule. + variance_type (`str`, *optional*): + Set to "learned" or "learned_range" for diffusion models that predict variance. If set, the model's output + contains the predicted Gaussian variance. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + predictor_order: int = 2, + corrector_order: int = 2, + prediction_type: str = "epsilon", + tau_func: Optional[Callable] = None, + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + algorithm_type: str = "data_prediction", + lower_order_final: bool = True, + use_karras_sigmas: Optional[bool] = False, + lambda_min_clipped: float = -float("inf"), + variance_type: Optional[str] = None, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = ( + torch.linspace( + beta_start**0.5, + beta_end**0.5, + num_train_timesteps, + dtype=torch.float32, + ) + ** 2 + ) + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + # Currently we only support VP-type noise schedule + self.alpha_t = torch.sqrt(self.alphas_cumprod) + self.sigma_t = torch.sqrt(1 - self.alphas_cumprod) + self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t) + self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5 + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + if algorithm_type not in ["data_prediction", "noise_prediction"]: + raise NotImplementedError(f"{algorithm_type} does is not implemented for {self.__class__}") + + # setable values + self.num_inference_steps = None + timesteps = np.linspace(0, num_train_timesteps - 1, num_train_timesteps, dtype=np.float32)[::-1].copy() + self.timesteps = torch.from_numpy(timesteps) + self.timestep_list = [None] * max(predictor_order, corrector_order - 1) + self.model_outputs = [None] * max(predictor_order, corrector_order - 1) + + if tau_func is None: + self.tau_func = lambda t: 1 if t >= 200 and t <= 800 else 0 + else: + self.tau_func = tau_func + self.predict_x0 = algorithm_type == "data_prediction" + self.lower_order_nums = 0 + self.last_sample = None + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def set_timesteps(self, num_inference_steps: int = None, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + # Clipping the minimum of all lambda(t) for numerical stability. + # This is critical for cosine (squaredcos_cap_v2) noise schedule. + clipped_idx = torch.searchsorted(torch.flip(self.lambda_t, [0]), self.config.lambda_min_clipped) + last_timestep = ((self.config.num_train_timesteps - clipped_idx).numpy()).item() + + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, last_timestep - 1, num_inference_steps + 1).round()[::-1][:-1].copy().astype(np.int64) + ) + + elif self.config.timestep_spacing == "leading": + step_ratio = last_timestep // (num_inference_steps + 1) + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps + 1) * step_ratio).round()[::-1][:-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.arange(last_timestep, 0, -step_ratio).round().copy().astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + if self.config.use_karras_sigmas: + log_sigmas = np.log(sigmas) + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + sigmas = np.concatenate([sigmas, sigmas[-1:]]).astype(np.float32) + else: + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + sigma_last = ((1 - self.alphas_cumprod[0]) / self.alphas_cumprod[0]) ** 0.5 + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + + self.sigmas = torch.from_numpy(sigmas) + self.timesteps = torch.from_numpy(timesteps).to(device=device, dtype=torch.int64) + + self.num_inference_steps = len(timesteps) + self.model_outputs = [ + None, + ] * max(self.config.predictor_order, self.config.corrector_order - 1) + self.lower_order_nums = 0 + self.last_sample = None + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._sigma_to_alpha_sigma_t + def _sigma_to_alpha_sigma_t(self, sigma): + alpha_t = 1 / ((sigma**2 + 1) ** 0.5) + sigma_t = sigma * alpha_t + + return alpha_t, sigma_t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + def convert_model_output( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + """ + Convert the model output to the corresponding type the data_prediction/noise_prediction algorithm needs. Noise_prediction is + designed to discretize an integral of the noise prediction model, and data_prediction is designed to discretize an + integral of the data prediction model. + + + + The algorithm and model type are decoupled. You can use either data_prediction or noise_prediction for both noise + prediction and data prediction models. + + + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError("missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + # SA-Solver_data_prediction needs to solve an integral of the data prediction model. + if self.config.algorithm_type in ["data_prediction"]: + if self.config.prediction_type == "epsilon": + # SA-Solver only needs the "mean" output. + if self.config.variance_type in ["learned", "learned_range"]: + model_output = model_output[:, :3] + x0_pred = (sample - sigma_t * model_output) / alpha_t + elif self.config.prediction_type == "sample": + x0_pred = model_output + elif self.config.prediction_type == "v_prediction": + x0_pred = alpha_t * sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the SASolverScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + + # SA-Solver_noise_prediction needs to solve an integral of the noise prediction model. + elif self.config.algorithm_type in ["noise_prediction"]: + if self.config.prediction_type == "epsilon": + # SA-Solver only needs the "mean" output. + if self.config.variance_type in ["learned", "learned_range"]: + epsilon = model_output[:, :3] + else: + epsilon = model_output + elif self.config.prediction_type == "sample": + epsilon = (sample - alpha_t * model_output) / sigma_t + elif self.config.prediction_type == "v_prediction": + epsilon = alpha_t * model_output + sigma_t * sample + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the SASolverScheduler." + ) + + if self.config.thresholding: + alpha_t, sigma_t = self.alpha_t[timestep], self.sigma_t[timestep] + x0_pred = (sample - sigma_t * epsilon) / alpha_t + x0_pred = self._threshold_sample(x0_pred) + epsilon = (sample - alpha_t * x0_pred) / sigma_t + + return epsilon + + def get_coefficients_exponential_negative(self, order, interval_start, interval_end): + """ + Calculate the integral of exp(-x) * x^order dx from interval_start to interval_end + """ + assert order in [0, 1, 2, 3], "order is only supported for 0, 1, 2 and 3" + + if order == 0: + return torch.exp(-interval_end) * (torch.exp(interval_end - interval_start) - 1) + elif order == 1: + return torch.exp(-interval_end) * ( + (interval_start + 1) * torch.exp(interval_end - interval_start) - (interval_end + 1) + ) + elif order == 2: + return torch.exp(-interval_end) * ( + (interval_start**2 + 2 * interval_start + 2) * torch.exp(interval_end - interval_start) + - (interval_end**2 + 2 * interval_end + 2) + ) + elif order == 3: + return torch.exp(-interval_end) * ( + (interval_start**3 + 3 * interval_start**2 + 6 * interval_start + 6) + * torch.exp(interval_end - interval_start) + - (interval_end**3 + 3 * interval_end**2 + 6 * interval_end + 6) + ) + + def get_coefficients_exponential_positive(self, order, interval_start, interval_end, tau): + """ + Calculate the integral of exp(x(1+tau^2)) * x^order dx from interval_start to interval_end + """ + assert order in [0, 1, 2, 3], "order is only supported for 0, 1, 2 and 3" + + # after change of variable(cov) + interval_end_cov = (1 + tau**2) * interval_end + interval_start_cov = (1 + tau**2) * interval_start + + if order == 0: + return ( + torch.exp(interval_end_cov) * (1 - torch.exp(-(interval_end_cov - interval_start_cov))) / (1 + tau**2) + ) + elif order == 1: + return ( + torch.exp(interval_end_cov) + * ( + (interval_end_cov - 1) + - (interval_start_cov - 1) * torch.exp(-(interval_end_cov - interval_start_cov)) + ) + / ((1 + tau**2) ** 2) + ) + elif order == 2: + return ( + torch.exp(interval_end_cov) + * ( + (interval_end_cov**2 - 2 * interval_end_cov + 2) + - (interval_start_cov**2 - 2 * interval_start_cov + 2) + * torch.exp(-(interval_end_cov - interval_start_cov)) + ) + / ((1 + tau**2) ** 3) + ) + elif order == 3: + return ( + torch.exp(interval_end_cov) + * ( + (interval_end_cov**3 - 3 * interval_end_cov**2 + 6 * interval_end_cov - 6) + - (interval_start_cov**3 - 3 * interval_start_cov**2 + 6 * interval_start_cov - 6) + * torch.exp(-(interval_end_cov - interval_start_cov)) + ) + / ((1 + tau**2) ** 4) + ) + + def lagrange_polynomial_coefficient(self, order, lambda_list): + """ + Calculate the coefficient of lagrange polynomial + """ + + assert order in [0, 1, 2, 3] + assert order == len(lambda_list) - 1 + if order == 0: + return [[1]] + elif order == 1: + return [ + [ + 1 / (lambda_list[0] - lambda_list[1]), + -lambda_list[1] / (lambda_list[0] - lambda_list[1]), + ], + [ + 1 / (lambda_list[1] - lambda_list[0]), + -lambda_list[0] / (lambda_list[1] - lambda_list[0]), + ], + ] + elif order == 2: + denominator1 = (lambda_list[0] - lambda_list[1]) * (lambda_list[0] - lambda_list[2]) + denominator2 = (lambda_list[1] - lambda_list[0]) * (lambda_list[1] - lambda_list[2]) + denominator3 = (lambda_list[2] - lambda_list[0]) * (lambda_list[2] - lambda_list[1]) + return [ + [ + 1 / denominator1, + (-lambda_list[1] - lambda_list[2]) / denominator1, + lambda_list[1] * lambda_list[2] / denominator1, + ], + [ + 1 / denominator2, + (-lambda_list[0] - lambda_list[2]) / denominator2, + lambda_list[0] * lambda_list[2] / denominator2, + ], + [ + 1 / denominator3, + (-lambda_list[0] - lambda_list[1]) / denominator3, + lambda_list[0] * lambda_list[1] / denominator3, + ], + ] + elif order == 3: + denominator1 = ( + (lambda_list[0] - lambda_list[1]) + * (lambda_list[0] - lambda_list[2]) + * (lambda_list[0] - lambda_list[3]) + ) + denominator2 = ( + (lambda_list[1] - lambda_list[0]) + * (lambda_list[1] - lambda_list[2]) + * (lambda_list[1] - lambda_list[3]) + ) + denominator3 = ( + (lambda_list[2] - lambda_list[0]) + * (lambda_list[2] - lambda_list[1]) + * (lambda_list[2] - lambda_list[3]) + ) + denominator4 = ( + (lambda_list[3] - lambda_list[0]) + * (lambda_list[3] - lambda_list[1]) + * (lambda_list[3] - lambda_list[2]) + ) + return [ + [ + 1 / denominator1, + (-lambda_list[1] - lambda_list[2] - lambda_list[3]) / denominator1, + ( + lambda_list[1] * lambda_list[2] + + lambda_list[1] * lambda_list[3] + + lambda_list[2] * lambda_list[3] + ) + / denominator1, + (-lambda_list[1] * lambda_list[2] * lambda_list[3]) / denominator1, + ], + [ + 1 / denominator2, + (-lambda_list[0] - lambda_list[2] - lambda_list[3]) / denominator2, + ( + lambda_list[0] * lambda_list[2] + + lambda_list[0] * lambda_list[3] + + lambda_list[2] * lambda_list[3] + ) + / denominator2, + (-lambda_list[0] * lambda_list[2] * lambda_list[3]) / denominator2, + ], + [ + 1 / denominator3, + (-lambda_list[0] - lambda_list[1] - lambda_list[3]) / denominator3, + ( + lambda_list[0] * lambda_list[1] + + lambda_list[0] * lambda_list[3] + + lambda_list[1] * lambda_list[3] + ) + / denominator3, + (-lambda_list[0] * lambda_list[1] * lambda_list[3]) / denominator3, + ], + [ + 1 / denominator4, + (-lambda_list[0] - lambda_list[1] - lambda_list[2]) / denominator4, + ( + lambda_list[0] * lambda_list[1] + + lambda_list[0] * lambda_list[2] + + lambda_list[1] * lambda_list[2] + ) + / denominator4, + (-lambda_list[0] * lambda_list[1] * lambda_list[2]) / denominator4, + ], + ] + + def get_coefficients_fn(self, order, interval_start, interval_end, lambda_list, tau): + assert order in [1, 2, 3, 4] + assert order == len(lambda_list), "the length of lambda list must be equal to the order" + coefficients = [] + lagrange_coefficient = self.lagrange_polynomial_coefficient(order - 1, lambda_list) + for i in range(order): + coefficient = 0 + for j in range(order): + if self.predict_x0: + coefficient += lagrange_coefficient[i][j] * self.get_coefficients_exponential_positive( + order - 1 - j, interval_start, interval_end, tau + ) + else: + coefficient += lagrange_coefficient[i][j] * self.get_coefficients_exponential_negative( + order - 1 - j, interval_start, interval_end + ) + coefficients.append(coefficient) + assert len(coefficients) == order, "the length of coefficients does not match the order" + return coefficients + + def stochastic_adams_bashforth_update( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor, + noise: torch.FloatTensor, + order: int, + tau: torch.FloatTensor, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the SA-Predictor. + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model at the current timestep. + prev_timestep (`int`): + The previous discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + order (`int`): + The order of SA-Predictor at this timestep. + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + prev_timestep = args[0] if len(args) > 0 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if noise is None: + if len(args) > 2: + noise = args[2] + else: + raise ValueError(" missing `noise` as a required keyward argument") + if order is None: + if len(args) > 3: + order = args[3] + else: + raise ValueError(" missing `order` as a required keyward argument") + if tau is None: + if len(args) > 4: + tau = args[4] + else: + raise ValueError(" missing `tau` as a required keyward argument") + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + model_output_list = self.model_outputs + sigma_t, sigma_s0 = ( + self.sigmas[self.step_index + 1], + self.sigmas[self.step_index], + ) + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + + gradient_part = torch.zeros_like(sample) + h = lambda_t - lambda_s0 + lambda_list = [] + + for i in range(order): + si = self.step_index - i + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + lambda_list.append(lambda_si) + + gradient_coefficients = self.get_coefficients_fn(order, lambda_s0, lambda_t, lambda_list, tau) + + x = sample + + if self.predict_x0: + if ( + order == 2 + ): ## if order = 2 we do a modification that does not influence the convergence order similar to unipc. Note: This is used only for few steps sampling. + # The added term is O(h^3). Empirically we find it will slightly improve the image quality. + # ODE case + # gradient_coefficients[0] += 1.0 * torch.exp(lambda_t) * (h ** 2 / 2 - (h - 1 + torch.exp(-h))) / (ns.marginal_lambda(t_prev_list[-1]) - ns.marginal_lambda(t_prev_list[-2])) + # gradient_coefficients[1] -= 1.0 * torch.exp(lambda_t) * (h ** 2 / 2 - (h - 1 + torch.exp(-h))) / (ns.marginal_lambda(t_prev_list[-1]) - ns.marginal_lambda(t_prev_list[-2])) + temp_sigma = self.sigmas[self.step_index - 1] + temp_alpha_s, temp_sigma_s = self._sigma_to_alpha_sigma_t(temp_sigma) + temp_lambda_s = torch.log(temp_alpha_s) - torch.log(temp_sigma_s) + gradient_coefficients[0] += ( + 1.0 + * torch.exp((1 + tau**2) * lambda_t) + * (h**2 / 2 - (h * (1 + tau**2) - 1 + torch.exp((1 + tau**2) * (-h))) / ((1 + tau**2) ** 2)) + / (lambda_s0 - temp_lambda_s) + ) + gradient_coefficients[1] -= ( + 1.0 + * torch.exp((1 + tau**2) * lambda_t) + * (h**2 / 2 - (h * (1 + tau**2) - 1 + torch.exp((1 + tau**2) * (-h))) / ((1 + tau**2) ** 2)) + / (lambda_s0 - temp_lambda_s) + ) + + for i in range(order): + if self.predict_x0: + gradient_part += ( + (1 + tau**2) + * sigma_t + * torch.exp(-(tau**2) * lambda_t) + * gradient_coefficients[i] + * model_output_list[-(i + 1)] + ) + else: + gradient_part += -(1 + tau**2) * alpha_t * gradient_coefficients[i] * model_output_list[-(i + 1)] + + if self.predict_x0: + noise_part = sigma_t * torch.sqrt(1 - torch.exp(-2 * tau**2 * h)) * noise + else: + noise_part = tau * sigma_t * torch.sqrt(torch.exp(2 * h) - 1) * noise + + if self.predict_x0: + x_t = torch.exp(-(tau**2) * h) * (sigma_t / sigma_s0) * x + gradient_part + noise_part + else: + x_t = (alpha_t / alpha_s0) * x + gradient_part + noise_part + + x_t = x_t.to(x.dtype) + return x_t + + def stochastic_adams_moulton_update( + self, + this_model_output: torch.FloatTensor, + *args, + last_sample: torch.FloatTensor, + last_noise: torch.FloatTensor, + this_sample: torch.FloatTensor, + order: int, + tau: torch.FloatTensor, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the SA-Corrector. + + Args: + this_model_output (`torch.FloatTensor`): + The model outputs at `x_t`. + this_timestep (`int`): + The current timestep `t`. + last_sample (`torch.FloatTensor`): + The generated sample before the last predictor `x_{t-1}`. + this_sample (`torch.FloatTensor`): + The generated sample after the last predictor `x_{t}`. + order (`int`): + The order of SA-Corrector at this step. + + Returns: + `torch.FloatTensor`: + The corrected sample tensor at the current timestep. + """ + + this_timestep = args[0] if len(args) > 0 else kwargs.pop("this_timestep", None) + if last_sample is None: + if len(args) > 1: + last_sample = args[1] + else: + raise ValueError(" missing`last_sample` as a required keyward argument") + if last_noise is None: + if len(args) > 2: + last_noise = args[2] + else: + raise ValueError(" missing`last_noise` as a required keyward argument") + if this_sample is None: + if len(args) > 3: + this_sample = args[3] + else: + raise ValueError(" missing`this_sample` as a required keyward argument") + if order is None: + if len(args) > 4: + order = args[4] + else: + raise ValueError(" missing`order` as a required keyward argument") + if tau is None: + if len(args) > 5: + tau = args[5] + else: + raise ValueError(" missing`tau` as a required keyward argument") + if this_timestep is not None: + deprecate( + "this_timestep", + "1.0.0", + "Passing `this_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + model_output_list = self.model_outputs + sigma_t, sigma_s0 = ( + self.sigmas[self.step_index], + self.sigmas[self.step_index - 1], + ) + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + gradient_part = torch.zeros_like(this_sample) + h = lambda_t - lambda_s0 + lambda_list = [] + for i in range(order): + si = self.step_index - i + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + lambda_list.append(lambda_si) + + model_prev_list = model_output_list + [this_model_output] + + gradient_coefficients = self.get_coefficients_fn(order, lambda_s0, lambda_t, lambda_list, tau) + + x = last_sample + + if self.predict_x0: + if ( + order == 2 + ): ## if order = 2 we do a modification that does not influence the convergence order similar to UniPC. Note: This is used only for few steps sampling. + # The added term is O(h^3). Empirically we find it will slightly improve the image quality. + # ODE case + # gradient_coefficients[0] += 1.0 * torch.exp(lambda_t) * (h / 2 - (h - 1 + torch.exp(-h)) / h) + # gradient_coefficients[1] -= 1.0 * torch.exp(lambda_t) * (h / 2 - (h - 1 + torch.exp(-h)) / h) + gradient_coefficients[0] += ( + 1.0 + * torch.exp((1 + tau**2) * lambda_t) + * (h / 2 - (h * (1 + tau**2) - 1 + torch.exp((1 + tau**2) * (-h))) / ((1 + tau**2) ** 2 * h)) + ) + gradient_coefficients[1] -= ( + 1.0 + * torch.exp((1 + tau**2) * lambda_t) + * (h / 2 - (h * (1 + tau**2) - 1 + torch.exp((1 + tau**2) * (-h))) / ((1 + tau**2) ** 2 * h)) + ) + + for i in range(order): + if self.predict_x0: + gradient_part += ( + (1 + tau**2) + * sigma_t + * torch.exp(-(tau**2) * lambda_t) + * gradient_coefficients[i] + * model_prev_list[-(i + 1)] + ) + else: + gradient_part += -(1 + tau**2) * alpha_t * gradient_coefficients[i] * model_prev_list[-(i + 1)] + + if self.predict_x0: + noise_part = sigma_t * torch.sqrt(1 - torch.exp(-2 * tau**2 * h)) * last_noise + else: + noise_part = tau * sigma_t * torch.sqrt(torch.exp(2 * h) - 1) * last_noise + + if self.predict_x0: + x_t = torch.exp(-(tau**2) * h) * (sigma_t / sigma_s0) * x + gradient_part + noise_part + else: + x_t = (alpha_t / alpha_s0) * x + gradient_part + noise_part + + x_t = x_t.to(x.dtype) + return x_t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + index_candidates = (schedule_timesteps == timestep).nonzero() + + if len(index_candidates) == 0: + step_index = len(self.timesteps) - 1 + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + elif len(index_candidates) > 1: + step_index = index_candidates[1].item() + else: + step_index = index_candidates[0].item() + + return step_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index + def _init_step_index(self, timestep): + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator=None, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the SA-Solver. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + use_corrector = self.step_index > 0 and self.last_sample is not None + + model_output_convert = self.convert_model_output(model_output, sample=sample) + + if use_corrector: + current_tau = self.tau_func(self.timestep_list[-1]) + sample = self.stochastic_adams_moulton_update( + this_model_output=model_output_convert, + last_sample=self.last_sample, + last_noise=self.last_noise, + this_sample=sample, + order=self.this_corrector_order, + tau=current_tau, + ) + + for i in range(max(self.config.predictor_order, self.config.corrector_order - 1) - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.timestep_list[i] = self.timestep_list[i + 1] + + self.model_outputs[-1] = model_output_convert + self.timestep_list[-1] = timestep + + noise = randn_tensor( + model_output.shape, + generator=generator, + device=model_output.device, + dtype=model_output.dtype, + ) + + if self.config.lower_order_final: + this_predictor_order = min(self.config.predictor_order, len(self.timesteps) - self.step_index) + this_corrector_order = min(self.config.corrector_order, len(self.timesteps) - self.step_index + 1) + else: + this_predictor_order = self.config.predictor_order + this_corrector_order = self.config.corrector_order + + self.this_predictor_order = min(this_predictor_order, self.lower_order_nums + 1) # warmup for multistep + self.this_corrector_order = min(this_corrector_order, self.lower_order_nums + 2) # warmup for multistep + assert self.this_predictor_order > 0 + assert self.this_corrector_order > 0 + + self.last_sample = sample + self.last_noise = noise + + current_tau = self.tau_func(self.timestep_list[-1]) + prev_sample = self.stochastic_adams_bashforth_update( + model_output=model_output_convert, + sample=sample, + noise=noise, + order=self.this_predictor_order, + tau=current_tau, + ) + + if self.lower_order_nums < max(self.config.predictor_order, self.config.corrector_order - 1): + self.lower_order_nums += 1 + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + # Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement + # for the subsequent add_noise calls + self.alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_sde_ve.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_sde_ve.py new file mode 100755 index 0000000..8f8dd18 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_sde_ve.py @@ -0,0 +1,301 @@ +# Copyright 2024 Google Brain and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/yang-song/score_sde_pytorch + +import math +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import SchedulerMixin, SchedulerOutput + + +@dataclass +class SdeVeOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + prev_sample_mean (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Mean averaged `prev_sample` over previous timesteps. + """ + + prev_sample: torch.FloatTensor + prev_sample_mean: torch.FloatTensor + + +class ScoreSdeVeScheduler(SchedulerMixin, ConfigMixin): + """ + `ScoreSdeVeScheduler` is a variance exploding stochastic differential equation (SDE) scheduler. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + snr (`float`, defaults to 0.15): + A coefficient weighting the step from the `model_output` sample (from the network) to the random noise. + sigma_min (`float`, defaults to 0.01): + The initial noise scale for the sigma sequence in the sampling procedure. The minimum sigma should mirror + the distribution of the data. + sigma_max (`float`, defaults to 1348.0): + The maximum value used for the range of continuous timesteps passed into the model. + sampling_eps (`float`, defaults to 1e-5): + The end value of sampling where timesteps decrease progressively from 1 to epsilon. + correct_steps (`int`, defaults to 1): + The number of correction steps performed on a produced sample. + """ + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 2000, + snr: float = 0.15, + sigma_min: float = 0.01, + sigma_max: float = 1348.0, + sampling_eps: float = 1e-5, + correct_steps: int = 1, + ): + # standard deviation of the initial noise distribution + self.init_noise_sigma = sigma_max + + # setable values + self.timesteps = None + + self.set_sigmas(num_train_timesteps, sigma_min, sigma_max, sampling_eps) + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + def set_timesteps( + self, num_inference_steps: int, sampling_eps: float = None, device: Union[str, torch.device] = None + ): + """ + Sets the continuous timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + sampling_eps (`float`, *optional*): + The final timestep value (overrides value given during scheduler instantiation). + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + + """ + sampling_eps = sampling_eps if sampling_eps is not None else self.config.sampling_eps + + self.timesteps = torch.linspace(1, sampling_eps, num_inference_steps, device=device) + + def set_sigmas( + self, num_inference_steps: int, sigma_min: float = None, sigma_max: float = None, sampling_eps: float = None + ): + """ + Sets the noise scales used for the diffusion chain (to be run before inference). The sigmas control the weight + of the `drift` and `diffusion` components of the sample update. + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + sigma_min (`float`, optional): + The initial noise scale value (overrides value given during scheduler instantiation). + sigma_max (`float`, optional): + The final noise scale value (overrides value given during scheduler instantiation). + sampling_eps (`float`, optional): + The final timestep value (overrides value given during scheduler instantiation). + + """ + sigma_min = sigma_min if sigma_min is not None else self.config.sigma_min + sigma_max = sigma_max if sigma_max is not None else self.config.sigma_max + sampling_eps = sampling_eps if sampling_eps is not None else self.config.sampling_eps + if self.timesteps is None: + self.set_timesteps(num_inference_steps, sampling_eps) + + self.sigmas = sigma_min * (sigma_max / sigma_min) ** (self.timesteps / sampling_eps) + self.discrete_sigmas = torch.exp(torch.linspace(math.log(sigma_min), math.log(sigma_max), num_inference_steps)) + self.sigmas = torch.tensor([sigma_min * (sigma_max / sigma_min) ** t for t in self.timesteps]) + + def get_adjacent_sigma(self, timesteps, t): + return torch.where( + timesteps == 0, + torch.zeros_like(t.to(timesteps.device)), + self.discrete_sigmas[timesteps - 1].to(timesteps.device), + ) + + def step_pred( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[SdeVeOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_sde_ve.SdeVeOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_sde_ve.SdeVeOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_sde_ve.SdeVeOutput`] is returned, otherwise a tuple + is returned where the first element is the sample tensor. + + """ + if self.timesteps is None: + raise ValueError( + "`self.timesteps` is not set, you need to run 'set_timesteps' after creating the scheduler" + ) + + timestep = timestep * torch.ones( + sample.shape[0], device=sample.device + ) # torch.repeat_interleave(timestep, sample.shape[0]) + timesteps = (timestep * (len(self.timesteps) - 1)).long() + + # mps requires indices to be in the same device, so we use cpu as is the default with cuda + timesteps = timesteps.to(self.discrete_sigmas.device) + + sigma = self.discrete_sigmas[timesteps].to(sample.device) + adjacent_sigma = self.get_adjacent_sigma(timesteps, timestep).to(sample.device) + drift = torch.zeros_like(sample) + diffusion = (sigma**2 - adjacent_sigma**2) ** 0.5 + + # equation 6 in the paper: the model_output modeled by the network is grad_x log pt(x) + # also equation 47 shows the analog from SDE models to ancestral sampling methods + diffusion = diffusion.flatten() + while len(diffusion.shape) < len(sample.shape): + diffusion = diffusion.unsqueeze(-1) + drift = drift - diffusion**2 * model_output + + # equation 6: sample noise for the diffusion term of + noise = randn_tensor( + sample.shape, layout=sample.layout, generator=generator, device=sample.device, dtype=sample.dtype + ) + prev_sample_mean = sample - drift # subtract because `dt` is a small negative timestep + # TODO is the variable diffusion the correct scaling term for the noise? + prev_sample = prev_sample_mean + diffusion * noise # add impact of diffusion field g + + if not return_dict: + return (prev_sample, prev_sample_mean) + + return SdeVeOutput(prev_sample=prev_sample, prev_sample_mean=prev_sample_mean) + + def step_correct( + self, + model_output: torch.FloatTensor, + sample: torch.FloatTensor, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Correct the predicted sample based on the `model_output` of the network. This is often run repeatedly after + making the prediction for the previous timestep. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_sde_ve.SdeVeOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_sde_ve.SdeVeOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_sde_ve.SdeVeOutput`] is returned, otherwise a tuple + is returned where the first element is the sample tensor. + + """ + if self.timesteps is None: + raise ValueError( + "`self.timesteps` is not set, you need to run 'set_timesteps' after creating the scheduler" + ) + + # For small batch sizes, the paper "suggest replacing norm(z) with sqrt(d), where d is the dim. of z" + # sample noise for correction + noise = randn_tensor(sample.shape, layout=sample.layout, generator=generator).to(sample.device) + + # compute step size from the model_output, the noise, and the snr + grad_norm = torch.norm(model_output.reshape(model_output.shape[0], -1), dim=-1).mean() + noise_norm = torch.norm(noise.reshape(noise.shape[0], -1), dim=-1).mean() + step_size = (self.config.snr * noise_norm / grad_norm) ** 2 * 2 + step_size = step_size * torch.ones(sample.shape[0]).to(sample.device) + # self.repeat_scalar(step_size, sample.shape[0]) + + # compute corrected sample: model_output term and noise term + step_size = step_size.flatten() + while len(step_size.shape) < len(sample.shape): + step_size = step_size.unsqueeze(-1) + prev_sample_mean = sample + step_size * model_output + prev_sample = prev_sample_mean + ((step_size * 2) ** 0.5) * noise + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.FloatTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + timesteps = timesteps.to(original_samples.device) + sigmas = self.discrete_sigmas.to(original_samples.device)[timesteps] + noise = ( + noise * sigmas[:, None, None, None] + if noise is not None + else torch.randn_like(original_samples) * sigmas[:, None, None, None] + ) + noisy_samples = noise + original_samples + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_sde_ve_flax.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_sde_ve_flax.py new file mode 100755 index 0000000..0a8d45d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_sde_ve_flax.py @@ -0,0 +1,280 @@ +# Copyright 2024 Google Brain and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This file is strongly influenced by https://github.com/yang-song/score_sde_pytorch + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import flax +import jax +import jax.numpy as jnp +from jax import random + +from ..configuration_utils import ConfigMixin, register_to_config +from .scheduling_utils_flax import FlaxSchedulerMixin, FlaxSchedulerOutput, broadcast_to_shape_from_left + + +@flax.struct.dataclass +class ScoreSdeVeSchedulerState: + # setable values + timesteps: Optional[jnp.ndarray] = None + discrete_sigmas: Optional[jnp.ndarray] = None + sigmas: Optional[jnp.ndarray] = None + + @classmethod + def create(cls): + return cls() + + +@dataclass +class FlaxSdeVeOutput(FlaxSchedulerOutput): + """ + Output class for the ScoreSdeVeScheduler's step function output. + + Args: + state (`ScoreSdeVeSchedulerState`): + prev_sample (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample (x_{t-1}) of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + prev_sample_mean (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)` for images): + Mean averaged `prev_sample`. Same as `prev_sample`, only mean-averaged over previous timesteps. + """ + + state: ScoreSdeVeSchedulerState + prev_sample: jnp.ndarray + prev_sample_mean: Optional[jnp.ndarray] = None + + +class FlaxScoreSdeVeScheduler(FlaxSchedulerMixin, ConfigMixin): + """ + The variance exploding stochastic differential equation (SDE) scheduler. + + For more information, see the original paper: https://arxiv.org/abs/2011.13456 + + [`~ConfigMixin`] takes care of storing all config attributes that are passed in the scheduler's `__init__` + function, such as `num_train_timesteps`. They can be accessed via `scheduler.config.num_train_timesteps`. + [`SchedulerMixin`] provides general loading and saving functionality via the [`SchedulerMixin.save_pretrained`] and + [`~SchedulerMixin.from_pretrained`] functions. + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + snr (`float`): + coefficient weighting the step from the model_output sample (from the network) to the random noise. + sigma_min (`float`): + initial noise scale for sigma sequence in sampling procedure. The minimum sigma should mirror the + distribution of the data. + sigma_max (`float`): maximum value used for the range of continuous timesteps passed into the model. + sampling_eps (`float`): the end value of sampling, where timesteps decrease progressively from 1 to + epsilon. + correct_steps (`int`): number of correction steps performed on a produced sample. + """ + + @property + def has_state(self): + return True + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 2000, + snr: float = 0.15, + sigma_min: float = 0.01, + sigma_max: float = 1348.0, + sampling_eps: float = 1e-5, + correct_steps: int = 1, + ): + pass + + def create_state(self): + state = ScoreSdeVeSchedulerState.create() + return self.set_sigmas( + state, + self.config.num_train_timesteps, + self.config.sigma_min, + self.config.sigma_max, + self.config.sampling_eps, + ) + + def set_timesteps( + self, state: ScoreSdeVeSchedulerState, num_inference_steps: int, shape: Tuple = (), sampling_eps: float = None + ) -> ScoreSdeVeSchedulerState: + """ + Sets the continuous timesteps used for the diffusion chain. Supporting function to be run before inference. + + Args: + state (`ScoreSdeVeSchedulerState`): the `FlaxScoreSdeVeScheduler` state data class instance. + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + sampling_eps (`float`, optional): + final timestep value (overrides value given at Scheduler instantiation). + + """ + sampling_eps = sampling_eps if sampling_eps is not None else self.config.sampling_eps + + timesteps = jnp.linspace(1, sampling_eps, num_inference_steps) + return state.replace(timesteps=timesteps) + + def set_sigmas( + self, + state: ScoreSdeVeSchedulerState, + num_inference_steps: int, + sigma_min: float = None, + sigma_max: float = None, + sampling_eps: float = None, + ) -> ScoreSdeVeSchedulerState: + """ + Sets the noise scales used for the diffusion chain. Supporting function to be run before inference. + + The sigmas control the weight of the `drift` and `diffusion` components of sample update. + + Args: + state (`ScoreSdeVeSchedulerState`): the `FlaxScoreSdeVeScheduler` state data class instance. + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + sigma_min (`float`, optional): + initial noise scale value (overrides value given at Scheduler instantiation). + sigma_max (`float`, optional): + final noise scale value (overrides value given at Scheduler instantiation). + sampling_eps (`float`, optional): + final timestep value (overrides value given at Scheduler instantiation). + """ + sigma_min = sigma_min if sigma_min is not None else self.config.sigma_min + sigma_max = sigma_max if sigma_max is not None else self.config.sigma_max + sampling_eps = sampling_eps if sampling_eps is not None else self.config.sampling_eps + if state.timesteps is None: + state = self.set_timesteps(state, num_inference_steps, sampling_eps) + + discrete_sigmas = jnp.exp(jnp.linspace(jnp.log(sigma_min), jnp.log(sigma_max), num_inference_steps)) + sigmas = jnp.array([sigma_min * (sigma_max / sigma_min) ** t for t in state.timesteps]) + + return state.replace(discrete_sigmas=discrete_sigmas, sigmas=sigmas) + + def get_adjacent_sigma(self, state, timesteps, t): + return jnp.where(timesteps == 0, jnp.zeros_like(t), state.discrete_sigmas[timesteps - 1]) + + def step_pred( + self, + state: ScoreSdeVeSchedulerState, + model_output: jnp.ndarray, + timestep: int, + sample: jnp.ndarray, + key: jax.Array, + return_dict: bool = True, + ) -> Union[FlaxSdeVeOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + state (`ScoreSdeVeSchedulerState`): the `FlaxScoreSdeVeScheduler` state data class instance. + model_output (`jnp.ndarray`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + generator: random number generator. + return_dict (`bool`): option for returning tuple rather than FlaxSdeVeOutput class + + Returns: + [`FlaxSdeVeOutput`] or `tuple`: [`FlaxSdeVeOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is the sample tensor. + + """ + if state.timesteps is None: + raise ValueError( + "`state.timesteps` is not set, you need to run 'set_timesteps' after creating the scheduler" + ) + + timestep = timestep * jnp.ones( + sample.shape[0], + ) + timesteps = (timestep * (len(state.timesteps) - 1)).long() + + sigma = state.discrete_sigmas[timesteps] + adjacent_sigma = self.get_adjacent_sigma(state, timesteps, timestep) + drift = jnp.zeros_like(sample) + diffusion = (sigma**2 - adjacent_sigma**2) ** 0.5 + + # equation 6 in the paper: the model_output modeled by the network is grad_x log pt(x) + # also equation 47 shows the analog from SDE models to ancestral sampling methods + diffusion = diffusion.flatten() + diffusion = broadcast_to_shape_from_left(diffusion, sample.shape) + drift = drift - diffusion**2 * model_output + + # equation 6: sample noise for the diffusion term of + key = random.split(key, num=1) + noise = random.normal(key=key, shape=sample.shape) + prev_sample_mean = sample - drift # subtract because `dt` is a small negative timestep + # TODO is the variable diffusion the correct scaling term for the noise? + prev_sample = prev_sample_mean + diffusion * noise # add impact of diffusion field g + + if not return_dict: + return (prev_sample, prev_sample_mean, state) + + return FlaxSdeVeOutput(prev_sample=prev_sample, prev_sample_mean=prev_sample_mean, state=state) + + def step_correct( + self, + state: ScoreSdeVeSchedulerState, + model_output: jnp.ndarray, + sample: jnp.ndarray, + key: jax.Array, + return_dict: bool = True, + ) -> Union[FlaxSdeVeOutput, Tuple]: + """ + Correct the predicted sample based on the output model_output of the network. This is often run repeatedly + after making the prediction for the previous timestep. + + Args: + state (`ScoreSdeVeSchedulerState`): the `FlaxScoreSdeVeScheduler` state data class instance. + model_output (`jnp.ndarray`): direct output from learned diffusion model. + sample (`jnp.ndarray`): + current instance of sample being created by diffusion process. + generator: random number generator. + return_dict (`bool`): option for returning tuple rather than FlaxSdeVeOutput class + + Returns: + [`FlaxSdeVeOutput`] or `tuple`: [`FlaxSdeVeOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is the sample tensor. + + """ + if state.timesteps is None: + raise ValueError( + "`state.timesteps` is not set, you need to run 'set_timesteps' after creating the scheduler" + ) + + # For small batch sizes, the paper "suggest replacing norm(z) with sqrt(d), where d is the dim. of z" + # sample noise for correction + key = random.split(key, num=1) + noise = random.normal(key=key, shape=sample.shape) + + # compute step size from the model_output, the noise, and the snr + grad_norm = jnp.linalg.norm(model_output) + noise_norm = jnp.linalg.norm(noise) + step_size = (self.config.snr * noise_norm / grad_norm) ** 2 * 2 + step_size = step_size * jnp.ones(sample.shape[0]) + + # compute corrected sample: model_output term and noise term + step_size = step_size.flatten() + step_size = broadcast_to_shape_from_left(step_size, sample.shape) + prev_sample_mean = sample + step_size * model_output + prev_sample = prev_sample_mean + ((step_size * 2) ** 0.5) * noise + + if not return_dict: + return (prev_sample, state) + + return FlaxSdeVeOutput(prev_sample=prev_sample, state=state) + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_tcd.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_tcd.py new file mode 100755 index 0000000..7eb01b3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_tcd.py @@ -0,0 +1,686 @@ +# Copyright 2024 Stanford University Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: This code is strongly influenced by https://github.com/pesser/pytorch_diffusion +# and https://github.com/hojonathanho/diffusion + +import math +from dataclasses import dataclass +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..schedulers.scheduling_utils import SchedulerMixin +from ..utils import BaseOutput, logging +from ..utils.torch_utils import randn_tensor + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +@dataclass +class TCDSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_noised_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted noised sample `(x_{s})` based on the model output from the current timestep. + """ + + prev_sample: torch.FloatTensor + pred_noised_sample: Optional[torch.FloatTensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +# Copied from diffusers.schedulers.scheduling_ddim.rescale_zero_terminal_snr +def rescale_zero_terminal_snr(betas: torch.FloatTensor) -> torch.FloatTensor: + """ + Rescales betas to have zero terminal SNR Based on https://arxiv.org/pdf/2305.08891.pdf (Algorithm 1) + + + Args: + betas (`torch.FloatTensor`): + the betas that the scheduler is being initialized with. + + Returns: + `torch.FloatTensor`: rescaled betas with zero terminal SNR + """ + # Convert betas to alphas_bar_sqrt + alphas = 1.0 - betas + alphas_cumprod = torch.cumprod(alphas, dim=0) + alphas_bar_sqrt = alphas_cumprod.sqrt() + + # Store old values. + alphas_bar_sqrt_0 = alphas_bar_sqrt[0].clone() + alphas_bar_sqrt_T = alphas_bar_sqrt[-1].clone() + + # Shift so the last timestep is zero. + alphas_bar_sqrt -= alphas_bar_sqrt_T + + # Scale so the first timestep is back to the old value. + alphas_bar_sqrt *= alphas_bar_sqrt_0 / (alphas_bar_sqrt_0 - alphas_bar_sqrt_T) + + # Convert alphas_bar_sqrt to betas + alphas_bar = alphas_bar_sqrt**2 # Revert sqrt + alphas = alphas_bar[1:] / alphas_bar[:-1] # Revert cumprod + alphas = torch.cat([alphas_bar[0:1], alphas]) + betas = 1 - alphas + + return betas + + +class TCDScheduler(SchedulerMixin, ConfigMixin): + """ + `TCDScheduler` incorporates the `Strategic Stochastic Sampling` introduced by the paper `Trajectory Consistency Distillation`, + extending the original Multistep Consistency Sampling to enable unrestricted trajectory traversal. + + This code is based on the official repo of TCD(https://github.com/jabir-zheng/TCD). + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. [`~ConfigMixin`] takes care of storing all config + attributes that are passed in the scheduler's `__init__` function, such as `num_train_timesteps`. They can be + accessed via `scheduler.config.num_train_timesteps`. [`SchedulerMixin`] provides general loading and saving + functionality via the [`SchedulerMixin.save_pretrained`] and [`~SchedulerMixin.from_pretrained`] functions. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + original_inference_steps (`int`, *optional*, defaults to 50): + The default number of inference steps used to generate a linearly-spaced timestep schedule, from which we + will ultimately take `num_inference_steps` evenly spaced timesteps to form the final timestep schedule. + clip_sample (`bool`, defaults to `True`): + Clip the predicted sample for numerical stability. + clip_sample_range (`float`, defaults to 1.0): + The maximum magnitude for sample clipping. Valid only when `clip_sample=True`. + set_alpha_to_one (`bool`, defaults to `True`): + Each diffusion step uses the alphas product value at that step and at the previous one. For the final step + there is no previous alpha. When this option is `True` the previous alpha product is fixed to `1`, + otherwise it uses the alpha value at step 0. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True`. + timestep_spacing (`str`, defaults to `"leading"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + timestep_scaling (`float`, defaults to 10.0): + The factor the timesteps will be multiplied by when calculating the consistency model boundary conditions + `c_skip` and `c_out`. Increasing this will decrease the approximation error (although the approximation + error at the default of `10.0` is already pretty small). + rescale_betas_zero_snr (`bool`, defaults to `False`): + Whether to rescale the betas to have zero terminal SNR. This enables the model to generate very bright and + dark samples instead of limiting it to samples with medium brightness. Loosely related to + [`--offset_noise`](https://github.com/huggingface/diffusers/blob/74fd735eb073eb1d774b1ab4154a0876eb82f055/examples/dreambooth/train_dreambooth.py#L506). + """ + + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.00085, + beta_end: float = 0.012, + beta_schedule: str = "scaled_linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + original_inference_steps: int = 50, + clip_sample: bool = False, + clip_sample_range: float = 1.0, + set_alpha_to_one: bool = True, + steps_offset: int = 0, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + timestep_spacing: str = "leading", + timestep_scaling: float = 10.0, + rescale_betas_zero_snr: bool = False, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + # Rescale for zero SNR + if rescale_betas_zero_snr: + self.betas = rescale_zero_terminal_snr(self.betas) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + + # At every step in ddim, we are looking into the previous alphas_cumprod + # For the final step, there is no previous alphas_cumprod because we are already at 0 + # `set_alpha_to_one` decides whether we set this parameter simply to one or + # whether we use the final alpha of the "non-previous" one. + self.final_alpha_cumprod = torch.tensor(1.0) if set_alpha_to_one else self.alphas_cumprod[0] + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy().astype(np.int64)) + self.custom_timesteps = False + + self._step_index = None + self._begin_index = None + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + indices = (schedule_timesteps == timestep).nonzero() + + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + pos = 1 if len(indices) > 1 else 0 + + return indices[pos].item() + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._init_step_index + def _init_step_index(self, timestep): + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + @property + def step_index(self): + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + timestep (`int`, *optional*): + The current timestep in the diffusion chain. + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_ddim.DDIMScheduler._get_variance + def _get_variance(self, timestep, prev_timestep): + alpha_prod_t = self.alphas_cumprod[timestep] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + variance = (beta_prod_t_prev / beta_prod_t) * (1 - alpha_prod_t / alpha_prod_t_prev) + + return variance + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + def set_timesteps( + self, + num_inference_steps: Optional[int] = None, + device: Union[str, torch.device] = None, + original_inference_steps: Optional[int] = None, + timesteps: Optional[List[int]] = None, + strength: int = 1.0, + ): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`, *optional*): + The number of diffusion steps used when generating samples with a pre-trained model. If used, + `timesteps` must be `None`. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + original_inference_steps (`int`, *optional*): + The original number of inference steps, which will be used to generate a linearly-spaced timestep + schedule (which is different from the standard `diffusers` implementation). We will then take + `num_inference_steps` timesteps from this schedule, evenly spaced in terms of indices, and use that as + our final timestep schedule. If not set, this will default to the `original_inference_steps` attribute. + timesteps (`List[int]`, *optional*): + Custom timesteps used to support arbitrary spacing between timesteps. If `None`, then the default + timestep spacing strategy of equal spacing between timesteps on the training/distillation timestep + schedule is used. If `timesteps` is passed, `num_inference_steps` must be `None`. + """ + # 0. Check inputs + if num_inference_steps is None and timesteps is None: + raise ValueError("Must pass exactly one of `num_inference_steps` or `custom_timesteps`.") + + if num_inference_steps is not None and timesteps is not None: + raise ValueError("Can only pass one of `num_inference_steps` or `custom_timesteps`.") + + # 1. Calculate the TCD original training/distillation timestep schedule. + original_steps = ( + original_inference_steps if original_inference_steps is not None else self.config.original_inference_steps + ) + + if original_inference_steps is None: + # default option, timesteps align with discrete inference steps + if original_steps > self.config.num_train_timesteps: + raise ValueError( + f"`original_steps`: {original_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + # TCD Timesteps Setting + # The skipping step parameter k from the paper. + k = self.config.num_train_timesteps // original_steps + # TCD Training/Distillation Steps Schedule + tcd_origin_timesteps = np.asarray(list(range(1, int(original_steps * strength) + 1))) * k - 1 + else: + # customised option, sampled timesteps can be any arbitrary value + tcd_origin_timesteps = np.asarray(list(range(0, int(self.config.num_train_timesteps * strength)))) + + # 2. Calculate the TCD inference timestep schedule. + if timesteps is not None: + # 2.1 Handle custom timestep schedules. + train_timesteps = set(tcd_origin_timesteps) + non_train_timesteps = [] + for i in range(1, len(timesteps)): + if timesteps[i] >= timesteps[i - 1]: + raise ValueError("`custom_timesteps` must be in descending order.") + + if timesteps[i] not in train_timesteps: + non_train_timesteps.append(timesteps[i]) + + if timesteps[0] >= self.config.num_train_timesteps: + raise ValueError( + f"`timesteps` must start before `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps}." + ) + + # Raise warning if timestep schedule does not start with self.config.num_train_timesteps - 1 + if strength == 1.0 and timesteps[0] != self.config.num_train_timesteps - 1: + logger.warning( + f"The first timestep on the custom timestep schedule is {timesteps[0]}, not" + f" `self.config.num_train_timesteps - 1`: {self.config.num_train_timesteps - 1}. You may get" + f" unexpected results when using this timestep schedule." + ) + + # Raise warning if custom timestep schedule contains timesteps not on original timestep schedule + if non_train_timesteps: + logger.warning( + f"The custom timestep schedule contains the following timesteps which are not on the original" + f" training/distillation timestep schedule: {non_train_timesteps}. You may get unexpected results" + f" when using this timestep schedule." + ) + + # Raise warning if custom timestep schedule is longer than original_steps + if original_steps is not None: + if len(timesteps) > original_steps: + logger.warning( + f"The number of timesteps in the custom timestep schedule is {len(timesteps)}, which exceeds the" + f" the length of the timestep schedule used for training: {original_steps}. You may get some" + f" unexpected results when using this timestep schedule." + ) + else: + if len(timesteps) > self.config.num_train_timesteps: + logger.warning( + f"The number of timesteps in the custom timestep schedule is {len(timesteps)}, which exceeds the" + f" the length of the timestep schedule used for training: {self.config.num_train_timesteps}. You may get some" + f" unexpected results when using this timestep schedule." + ) + + timesteps = np.array(timesteps, dtype=np.int64) + self.num_inference_steps = len(timesteps) + self.custom_timesteps = True + + # Apply strength (e.g. for img2img pipelines) (see StableDiffusionImg2ImgPipeline.get_timesteps) + init_timestep = min(int(self.num_inference_steps * strength), self.num_inference_steps) + t_start = max(self.num_inference_steps - init_timestep, 0) + timesteps = timesteps[t_start * self.order :] + # TODO: also reset self.num_inference_steps? + else: + # 2.2 Create the "standard" TCD inference timestep schedule. + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `self.config.train_timesteps`:" + f" {self.config.num_train_timesteps} as the unet model trained with this scheduler can only handle" + f" maximal {self.config.num_train_timesteps} timesteps." + ) + + if original_steps is not None: + skipping_step = len(tcd_origin_timesteps) // num_inference_steps + + if skipping_step < 1: + raise ValueError( + f"The combination of `original_steps x strength`: {original_steps} x {strength} is smaller than `num_inference_steps`: {num_inference_steps}. Make sure to either reduce `num_inference_steps` to a value smaller than {int(original_steps * strength)} or increase `strength` to a value higher than {float(num_inference_steps / original_steps)}." + ) + + self.num_inference_steps = num_inference_steps + + if original_steps is not None: + if num_inference_steps > original_steps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `original_inference_steps`:" + f" {original_steps} because the final timestep schedule will be a subset of the" + f" `original_inference_steps`-sized initial timestep schedule." + ) + else: + if num_inference_steps > self.config.num_train_timesteps: + raise ValueError( + f"`num_inference_steps`: {num_inference_steps} cannot be larger than `num_train_timesteps`:" + f" {self.config.num_train_timesteps} because the final timestep schedule will be a subset of the" + f" `num_train_timesteps`-sized initial timestep schedule." + ) + + # TCD Inference Steps Schedule + tcd_origin_timesteps = tcd_origin_timesteps[::-1].copy() + # Select (approximately) evenly spaced indices from tcd_origin_timesteps. + inference_indices = np.linspace(0, len(tcd_origin_timesteps), num=num_inference_steps, endpoint=False) + inference_indices = np.floor(inference_indices).astype(np.int64) + timesteps = tcd_origin_timesteps[inference_indices] + + self.timesteps = torch.from_numpy(timesteps).to(device=device, dtype=torch.long) + + self._step_index = None + self._begin_index = None + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + eta: float = 0.3, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[TCDSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + eta (`float`): + A stochastic parameter (referred to as `gamma` in the paper) used to control the stochasticity in every step. + When eta = 0, it represents deterministic sampling, whereas eta = 1 indicates full stochastic sampling. + generator (`torch.Generator`, *optional*): + A random number generator. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_tcd.TCDSchedulerOutput`] or `tuple`. + Returns: + [`~schedulers.scheduling_utils.TCDSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_tcd.TCDSchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + assert 0 <= eta <= 1.0, "gamma must be less than or equal to 1.0" + + # 1. get previous step value + prev_step_index = self.step_index + 1 + if prev_step_index < len(self.timesteps): + prev_timestep = self.timesteps[prev_step_index] + else: + prev_timestep = torch.tensor(0) + + timestep_s = torch.floor((1 - eta) * prev_timestep).to(dtype=torch.long) + + # 2. compute alphas, betas + alpha_prod_t = self.alphas_cumprod[timestep] + beta_prod_t = 1 - alpha_prod_t + + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.final_alpha_cumprod + + alpha_prod_s = self.alphas_cumprod[timestep_s] + beta_prod_s = 1 - alpha_prod_s + + # 3. Compute the predicted noised sample x_s based on the model parameterization + if self.config.prediction_type == "epsilon": # noise-prediction + pred_original_sample = (sample - beta_prod_t.sqrt() * model_output) / alpha_prod_t.sqrt() + pred_epsilon = model_output + pred_noised_sample = alpha_prod_s.sqrt() * pred_original_sample + beta_prod_s.sqrt() * pred_epsilon + elif self.config.prediction_type == "sample": # x-prediction + pred_original_sample = model_output + pred_epsilon = (sample - alpha_prod_t ** (0.5) * pred_original_sample) / beta_prod_t ** (0.5) + pred_noised_sample = alpha_prod_s.sqrt() * pred_original_sample + beta_prod_s.sqrt() * pred_epsilon + elif self.config.prediction_type == "v_prediction": # v-prediction + pred_original_sample = (alpha_prod_t**0.5) * sample - (beta_prod_t**0.5) * model_output + pred_epsilon = (alpha_prod_t**0.5) * model_output + (beta_prod_t**0.5) * sample + pred_noised_sample = alpha_prod_s.sqrt() * pred_original_sample + beta_prod_s.sqrt() * pred_epsilon + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample` or" + " `v_prediction` for `TCDScheduler`." + ) + + # 4. Sample and inject noise z ~ N(0, I) for MultiStep Inference + # Noise is not used on the final timestep of the timestep schedule. + # This also means that noise is not used for one-step sampling. + # Eta (referred to as "gamma" in the paper) was introduced to control the stochasticity in every step. + # When eta = 0, it represents deterministic sampling, whereas eta = 1 indicates full stochastic sampling. + if eta > 0: + if self.step_index != self.num_inference_steps - 1: + noise = randn_tensor( + model_output.shape, generator=generator, device=model_output.device, dtype=pred_noised_sample.dtype + ) + prev_sample = (alpha_prod_t_prev / alpha_prod_s).sqrt() * pred_noised_sample + ( + 1 - alpha_prod_t_prev / alpha_prod_s + ).sqrt() * noise + else: + prev_sample = pred_noised_sample + else: + prev_sample = pred_noised_sample + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample, pred_noised_sample) + + return TCDSchedulerOutput(prev_sample=prev_sample, pred_noised_sample=pred_noised_sample) + + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device, dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + def get_velocity( + self, sample: torch.FloatTensor, noise: torch.FloatTensor, timesteps: torch.IntTensor + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as sample + alphas_cumprod = self.alphas_cumprod.to(device=sample.device, dtype=sample.dtype) + timesteps = timesteps.to(sample.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(sample.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(sample.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample + return velocity + + def __len__(self): + return self.config.num_train_timesteps + + def previous_timestep(self, timestep): + if self.custom_timesteps: + index = (self.timesteps == timestep).nonzero(as_tuple=True)[0][0] + if index == self.timesteps.shape[0] - 1: + prev_t = torch.tensor(-1) + else: + prev_t = self.timesteps[index + 1] + else: + num_inference_steps = ( + self.num_inference_steps if self.num_inference_steps else self.config.num_train_timesteps + ) + prev_t = timestep - self.config.num_train_timesteps // num_inference_steps + + return prev_t diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_unclip.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_unclip.py new file mode 100755 index 0000000..7bc0a0f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_unclip.py @@ -0,0 +1,352 @@ +# Copyright 2024 Kakao Brain and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import math +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from ..utils.torch_utils import randn_tensor +from .scheduling_utils import SchedulerMixin + + +@dataclass +# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->UnCLIP +class UnCLIPSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + """ + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class UnCLIPScheduler(SchedulerMixin, ConfigMixin): + """ + NOTE: do not use this scheduler. The DDPM scheduler has been updated to support the changes made here. This + scheduler will be removed and replaced with DDPM. + + This is a modified DDPM Scheduler specifically for the karlo unCLIP model. + + This scheduler has some minor variations in how it calculates the learned range variance and dynamically + re-calculates betas based off the timesteps it is skipping. + + The scheduler also uses a slightly different step ratio when computing timesteps to use for inference. + + See [`~DDPMScheduler`] for more information on DDPM scheduling + + Args: + num_train_timesteps (`int`): number of diffusion steps used to train the model. + variance_type (`str`): + options to clip the variance used when adding noise to the denoised sample. Choose from `fixed_small_log` + or `learned_range`. + clip_sample (`bool`, default `True`): + option to clip predicted sample between `-clip_sample_range` and `clip_sample_range` for numerical + stability. + clip_sample_range (`float`, default `1.0`): + The range to clip the sample between. See `clip_sample`. + prediction_type (`str`, default `epsilon`, optional): + prediction type of the scheduler function, one of `epsilon` (predicting the noise of the diffusion process) + or `sample` (directly predicting the noisy sample`) + """ + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + variance_type: str = "fixed_small_log", + clip_sample: bool = True, + clip_sample_range: Optional[float] = 1.0, + prediction_type: str = "epsilon", + beta_schedule: str = "squaredcos_cap_v2", + ): + if beta_schedule != "squaredcos_cap_v2": + raise ValueError("UnCLIPScheduler only supports `beta_schedule`: 'squaredcos_cap_v2'") + + self.betas = betas_for_alpha_bar(num_train_timesteps) + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + self.one = torch.tensor(1.0) + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + # setable values + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy()) + + self.variance_type = variance_type + + def scale_model_input(self, sample: torch.FloatTensor, timestep: Optional[int] = None) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): input sample + timestep (`int`, optional): current timestep + + Returns: + `torch.FloatTensor`: scaled input sample + """ + return sample + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain. Supporting function to be run before inference. + + Note that this scheduler uses a slightly different step ratio than the other diffusers schedulers. The + different step ratio is to mimic the original karlo implementation and does not affect the quality or accuracy + of the results. + + Args: + num_inference_steps (`int`): + the number of diffusion steps used when generating samples with a pre-trained model. + """ + self.num_inference_steps = num_inference_steps + step_ratio = (self.config.num_train_timesteps - 1) / (self.num_inference_steps - 1) + timesteps = (np.arange(0, num_inference_steps) * step_ratio).round()[::-1].copy().astype(np.int64) + self.timesteps = torch.from_numpy(timesteps).to(device) + + def _get_variance(self, t, prev_timestep=None, predicted_variance=None, variance_type=None): + if prev_timestep is None: + prev_timestep = t - 1 + + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.one + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + if prev_timestep == t - 1: + beta = self.betas[t] + else: + beta = 1 - alpha_prod_t / alpha_prod_t_prev + + # For t > 0, compute predicted variance βt (see formula (6) and (7) from https://arxiv.org/pdf/2006.11239.pdf) + # and sample from it to get previous sample + # x_{t-1} ~ N(pred_prev_sample, variance) == add variance to pred_sample + variance = beta_prod_t_prev / beta_prod_t * beta + + if variance_type is None: + variance_type = self.config.variance_type + + # hacks - were probably added for training stability + if variance_type == "fixed_small_log": + variance = torch.log(torch.clamp(variance, min=1e-20)) + variance = torch.exp(0.5 * variance) + elif variance_type == "learned_range": + # NOTE difference with DDPM scheduler + min_log = variance.log() + max_log = beta.log() + + frac = (predicted_variance + 1) / 2 + variance = frac * max_log + (1 - frac) * min_log + + return variance + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + prev_timestep: Optional[int] = None, + generator=None, + return_dict: bool = True, + ) -> Union[UnCLIPSchedulerOutput, Tuple]: + """ + Predict the sample at the previous timestep by reversing the SDE. Core function to propagate the diffusion + process from the learned model outputs (most often the predicted noise). + + Args: + model_output (`torch.FloatTensor`): direct output from learned diffusion model. + timestep (`int`): current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + current instance of sample being created by diffusion process. + prev_timestep (`int`, *optional*): The previous timestep to predict the previous sample at. + Used to dynamically compute beta. If not given, `t-1` is used and the pre-computed beta is used. + generator: random number generator. + return_dict (`bool`): option for returning tuple rather than UnCLIPSchedulerOutput class + + Returns: + [`~schedulers.scheduling_utils.UnCLIPSchedulerOutput`] or `tuple`: + [`~schedulers.scheduling_utils.UnCLIPSchedulerOutput`] if `return_dict` is True, otherwise a `tuple`. When + returning a tuple, the first element is the sample tensor. + + """ + t = timestep + + if model_output.shape[1] == sample.shape[1] * 2 and self.variance_type == "learned_range": + model_output, predicted_variance = torch.split(model_output, sample.shape[1], dim=1) + else: + predicted_variance = None + + # 1. compute alphas, betas + if prev_timestep is None: + prev_timestep = t - 1 + + alpha_prod_t = self.alphas_cumprod[t] + alpha_prod_t_prev = self.alphas_cumprod[prev_timestep] if prev_timestep >= 0 else self.one + beta_prod_t = 1 - alpha_prod_t + beta_prod_t_prev = 1 - alpha_prod_t_prev + + if prev_timestep == t - 1: + beta = self.betas[t] + alpha = self.alphas[t] + else: + beta = 1 - alpha_prod_t / alpha_prod_t_prev + alpha = 1 - beta + + # 2. compute predicted original sample from predicted noise also called + # "predicted x_0" of formula (15) from https://arxiv.org/pdf/2006.11239.pdf + if self.config.prediction_type == "epsilon": + pred_original_sample = (sample - beta_prod_t ** (0.5) * model_output) / alpha_prod_t ** (0.5) + elif self.config.prediction_type == "sample": + pred_original_sample = model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon` or `sample`" + " for the UnCLIPScheduler." + ) + + # 3. Clip "predicted x_0" + if self.config.clip_sample: + pred_original_sample = torch.clamp( + pred_original_sample, -self.config.clip_sample_range, self.config.clip_sample_range + ) + + # 4. Compute coefficients for pred_original_sample x_0 and current sample x_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_original_sample_coeff = (alpha_prod_t_prev ** (0.5) * beta) / beta_prod_t + current_sample_coeff = alpha ** (0.5) * beta_prod_t_prev / beta_prod_t + + # 5. Compute predicted previous sample µ_t + # See formula (7) from https://arxiv.org/pdf/2006.11239.pdf + pred_prev_sample = pred_original_sample_coeff * pred_original_sample + current_sample_coeff * sample + + # 6. Add noise + variance = 0 + if t > 0: + variance_noise = randn_tensor( + model_output.shape, dtype=model_output.dtype, generator=generator, device=model_output.device + ) + + variance = self._get_variance( + t, + predicted_variance=predicted_variance, + prev_timestep=prev_timestep, + ) + + if self.variance_type == "fixed_small_log": + variance = variance + elif self.variance_type == "learned_range": + variance = (0.5 * variance).exp() + else: + raise ValueError( + f"variance_type given as {self.variance_type} must be one of `fixed_small_log` or `learned_range`" + " for the UnCLIPScheduler." + ) + + variance = variance * variance_noise + + pred_prev_sample = pred_prev_sample + variance + + if not return_dict: + return (pred_prev_sample,) + + return UnCLIPSchedulerOutput(prev_sample=pred_prev_sample, pred_original_sample=pred_original_sample) + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure alphas_cumprod and timestep have same device and dtype as original_samples + # Move the self.alphas_cumprod to device to avoid redundant CPU to GPU data movement + # for the subsequent add_noise calls + self.alphas_cumprod = self.alphas_cumprod.to(device=original_samples.device) + alphas_cumprod = self.alphas_cumprod.to(dtype=original_samples.dtype) + timesteps = timesteps.to(original_samples.device) + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + while len(sqrt_alpha_prod.shape) < len(original_samples.shape): + sqrt_alpha_prod = sqrt_alpha_prod.unsqueeze(-1) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + while len(sqrt_one_minus_alpha_prod.shape) < len(original_samples.shape): + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.unsqueeze(-1) + + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_unipc_multistep.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_unipc_multistep.py new file mode 100755 index 0000000..8ba9a9c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_unipc_multistep.py @@ -0,0 +1,880 @@ +# Copyright 2024 TSAIL Team and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# DISCLAIMER: check https://arxiv.org/abs/2302.04867 and https://github.com/wl-zhao/UniPC for more info +# The codebase is modified based on https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py + +import math +from typing import List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import deprecate +from .scheduling_utils import KarrasDiffusionSchedulers, SchedulerMixin, SchedulerOutput + + +# Copied from diffusers.schedulers.scheduling_ddpm.betas_for_alpha_bar +def betas_for_alpha_bar( + num_diffusion_timesteps, + max_beta=0.999, + alpha_transform_type="cosine", +): + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + alpha_transform_type (`str`, *optional*, default to `cosine`): the type of noise schedule for alpha_bar. + Choose from `cosine` or `exp` + + Returns: + betas (`np.ndarray`): the betas used by the scheduler to step the model outputs + """ + if alpha_transform_type == "cosine": + + def alpha_bar_fn(t): + return math.cos((t + 0.008) / 1.008 * math.pi / 2) ** 2 + + elif alpha_transform_type == "exp": + + def alpha_bar_fn(t): + return math.exp(t * -12.0) + + else: + raise ValueError(f"Unsupported alpha_tranform_type: {alpha_transform_type}") + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar_fn(t2) / alpha_bar_fn(t1), max_beta)) + return torch.tensor(betas, dtype=torch.float32) + + +class UniPCMultistepScheduler(SchedulerMixin, ConfigMixin): + """ + `UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_train_timesteps (`int`, defaults to 1000): + The number of diffusion steps to train the model. + beta_start (`float`, defaults to 0.0001): + The starting `beta` value of inference. + beta_end (`float`, defaults to 0.02): + The final `beta` value. + beta_schedule (`str`, defaults to `"linear"`): + The beta schedule, a mapping from a beta range to a sequence of betas for stepping the model. Choose from + `linear`, `scaled_linear`, or `squaredcos_cap_v2`. + trained_betas (`np.ndarray`, *optional*): + Pass an array of betas directly to the constructor to bypass `beta_start` and `beta_end`. + solver_order (`int`, default `2`): + The UniPC order which can be any positive integer. The effective order of accuracy is `solver_order + 1` + due to the UniC. It is recommended to use `solver_order=2` for guided sampling, and `solver_order=3` for + unconditional sampling. + prediction_type (`str`, defaults to `epsilon`, *optional*): + Prediction type of the scheduler function; can be `epsilon` (predicts the noise of the diffusion process), + `sample` (directly predicts the noisy sample`) or `v_prediction` (see section 2.4 of [Imagen + Video](https://imagen.research.google/video/paper.pdf) paper). + thresholding (`bool`, defaults to `False`): + Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such + as Stable Diffusion. + dynamic_thresholding_ratio (`float`, defaults to 0.995): + The ratio for the dynamic thresholding method. Valid only when `thresholding=True`. + sample_max_value (`float`, defaults to 1.0): + The threshold value for dynamic thresholding. Valid only when `thresholding=True` and `predict_x0=True`. + predict_x0 (`bool`, defaults to `True`): + Whether to use the updating algorithm on the predicted x0. + solver_type (`str`, default `bh2`): + Solver type for UniPC. It is recommended to use `bh1` for unconditional sampling when steps < 10, and `bh2` + otherwise. + lower_order_final (`bool`, default `True`): + Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can + stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10. + disable_corrector (`list`, default `[]`): + Decides which step to disable the corrector to mitigate the misalignment between `epsilon_theta(x_t, c)` + and `epsilon_theta(x_t^c, c)` which can influence convergence for a large guidance scale. Corrector is + usually disabled during the first few steps. + solver_p (`SchedulerMixin`, default `None`): + Any other scheduler that if specified, the algorithm becomes `solver_p + UniC`. + use_karras_sigmas (`bool`, *optional*, defaults to `False`): + Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`, + the sigmas are determined according to a sequence of noise levels {σi}. + timestep_spacing (`str`, defaults to `"linspace"`): + The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and + Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information. + steps_offset (`int`, defaults to 0): + An offset added to the inference steps, as required by some model families. + """ + + _compatibles = [e.name for e in KarrasDiffusionSchedulers] + order = 1 + + @register_to_config + def __init__( + self, + num_train_timesteps: int = 1000, + beta_start: float = 0.0001, + beta_end: float = 0.02, + beta_schedule: str = "linear", + trained_betas: Optional[Union[np.ndarray, List[float]]] = None, + solver_order: int = 2, + prediction_type: str = "epsilon", + thresholding: bool = False, + dynamic_thresholding_ratio: float = 0.995, + sample_max_value: float = 1.0, + predict_x0: bool = True, + solver_type: str = "bh2", + lower_order_final: bool = True, + disable_corrector: List[int] = [], + solver_p: SchedulerMixin = None, + use_karras_sigmas: Optional[bool] = False, + timestep_spacing: str = "linspace", + steps_offset: int = 0, + ): + if trained_betas is not None: + self.betas = torch.tensor(trained_betas, dtype=torch.float32) + elif beta_schedule == "linear": + self.betas = torch.linspace(beta_start, beta_end, num_train_timesteps, dtype=torch.float32) + elif beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + self.betas = torch.linspace(beta_start**0.5, beta_end**0.5, num_train_timesteps, dtype=torch.float32) ** 2 + elif beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + self.betas = betas_for_alpha_bar(num_train_timesteps) + else: + raise NotImplementedError(f"{beta_schedule} does is not implemented for {self.__class__}") + + self.alphas = 1.0 - self.betas + self.alphas_cumprod = torch.cumprod(self.alphas, dim=0) + # Currently we only support VP-type noise schedule + self.alpha_t = torch.sqrt(self.alphas_cumprod) + self.sigma_t = torch.sqrt(1 - self.alphas_cumprod) + self.lambda_t = torch.log(self.alpha_t) - torch.log(self.sigma_t) + self.sigmas = ((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5 + + # standard deviation of the initial noise distribution + self.init_noise_sigma = 1.0 + + if solver_type not in ["bh1", "bh2"]: + if solver_type in ["midpoint", "heun", "logrho"]: + self.register_to_config(solver_type="bh2") + else: + raise NotImplementedError(f"{solver_type} does is not implemented for {self.__class__}") + + self.predict_x0 = predict_x0 + # setable values + self.num_inference_steps = None + timesteps = np.linspace(0, num_train_timesteps - 1, num_train_timesteps, dtype=np.float32)[::-1].copy() + self.timesteps = torch.from_numpy(timesteps) + self.model_outputs = [None] * solver_order + self.timestep_list = [None] * solver_order + self.lower_order_nums = 0 + self.disable_corrector = disable_corrector + self.solver_p = solver_p + self.last_sample = None + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + @property + def step_index(self): + """ + The index counter for current timestep. It will increae 1 after each scheduler step. + """ + return self._step_index + + @property + def begin_index(self): + """ + The index for the first timestep. It should be set from pipeline with `set_begin_index` method. + """ + return self._begin_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index + def set_begin_index(self, begin_index: int = 0): + """ + Sets the begin index for the scheduler. This function should be run from pipeline before the inference. + + Args: + begin_index (`int`): + The begin index for the scheduler. + """ + self._begin_index = begin_index + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps should be moved to. If `None`, the timesteps are not moved. + """ + # "linspace", "leading", "trailing" corresponds to annotation of Table 2. of https://arxiv.org/abs/2305.08891 + if self.config.timestep_spacing == "linspace": + timesteps = ( + np.linspace(0, self.config.num_train_timesteps - 1, num_inference_steps + 1) + .round()[::-1][:-1] + .copy() + .astype(np.int64) + ) + elif self.config.timestep_spacing == "leading": + step_ratio = self.config.num_train_timesteps // (num_inference_steps + 1) + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = (np.arange(0, num_inference_steps + 1) * step_ratio).round()[::-1][:-1].copy().astype(np.int64) + timesteps += self.config.steps_offset + elif self.config.timestep_spacing == "trailing": + step_ratio = self.config.num_train_timesteps / num_inference_steps + # creates integer timesteps by multiplying by ratio + # casting to int to avoid issues when num_inference_step is power of 3 + timesteps = np.arange(self.config.num_train_timesteps, 0, -step_ratio).round().copy().astype(np.int64) + timesteps -= 1 + else: + raise ValueError( + f"{self.config.timestep_spacing} is not supported. Please make sure to choose one of 'linspace', 'leading' or 'trailing'." + ) + + sigmas = np.array(((1 - self.alphas_cumprod) / self.alphas_cumprod) ** 0.5) + if self.config.use_karras_sigmas: + log_sigmas = np.log(sigmas) + sigmas = np.flip(sigmas).copy() + sigmas = self._convert_to_karras(in_sigmas=sigmas, num_inference_steps=num_inference_steps) + timesteps = np.array([self._sigma_to_t(sigma, log_sigmas) for sigma in sigmas]).round() + sigmas = np.concatenate([sigmas, sigmas[-1:]]).astype(np.float32) + else: + sigmas = np.interp(timesteps, np.arange(0, len(sigmas)), sigmas) + sigma_last = ((1 - self.alphas_cumprod[0]) / self.alphas_cumprod[0]) ** 0.5 + sigmas = np.concatenate([sigmas, [sigma_last]]).astype(np.float32) + + self.sigmas = torch.from_numpy(sigmas) + self.timesteps = torch.from_numpy(timesteps).to(device=device, dtype=torch.int64) + + self.num_inference_steps = len(timesteps) + + self.model_outputs = [ + None, + ] * self.config.solver_order + self.lower_order_nums = 0 + self.last_sample = None + if self.solver_p: + self.solver_p.set_timesteps(self.num_inference_steps, device=device) + + # add an index counter for schedulers that allow duplicated timesteps + self._step_index = None + self._begin_index = None + self.sigmas = self.sigmas.to("cpu") # to avoid too much CPU/GPU communication + + # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample + def _threshold_sample(self, sample: torch.FloatTensor) -> torch.FloatTensor: + """ + "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the + prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by + s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing + pixels from saturation at each step. We find that dynamic thresholding results in significantly better + photorealism as well as better image-text alignment, especially when using very large guidance weights." + + https://arxiv.org/abs/2205.11487 + """ + dtype = sample.dtype + batch_size, channels, *remaining_dims = sample.shape + + if dtype not in (torch.float32, torch.float64): + sample = sample.float() # upcast for quantile calculation, and clamp not implemented for cpu half + + # Flatten sample for doing quantile calculation along each image + sample = sample.reshape(batch_size, channels * np.prod(remaining_dims)) + + abs_sample = sample.abs() # "a certain percentile absolute pixel value" + + s = torch.quantile(abs_sample, self.config.dynamic_thresholding_ratio, dim=1) + s = torch.clamp( + s, min=1, max=self.config.sample_max_value + ) # When clamped to min=1, equivalent to standard clipping to [-1, 1] + s = s.unsqueeze(1) # (batch_size, 1) because clamp will broadcast along dim=0 + sample = torch.clamp(sample, -s, s) / s # "we threshold xt0 to the range [-s, s] and then divide by s" + + sample = sample.reshape(batch_size, channels, *remaining_dims) + sample = sample.to(dtype) + + return sample + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._sigma_to_t + def _sigma_to_t(self, sigma, log_sigmas): + # get log sigma + log_sigma = np.log(np.maximum(sigma, 1e-10)) + + # get distribution + dists = log_sigma - log_sigmas[:, np.newaxis] + + # get sigmas range + low_idx = np.cumsum((dists >= 0), axis=0).argmax(axis=0).clip(max=log_sigmas.shape[0] - 2) + high_idx = low_idx + 1 + + low = log_sigmas[low_idx] + high = log_sigmas[high_idx] + + # interpolate sigmas + w = (low - log_sigma) / (low - high) + w = np.clip(w, 0, 1) + + # transform interpolation to time range + t = (1 - w) * low_idx + w * high_idx + t = t.reshape(sigma.shape) + return t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._sigma_to_alpha_sigma_t + def _sigma_to_alpha_sigma_t(self, sigma): + alpha_t = 1 / ((sigma**2 + 1) ** 0.5) + sigma_t = sigma * alpha_t + + return alpha_t, sigma_t + + # Copied from diffusers.schedulers.scheduling_euler_discrete.EulerDiscreteScheduler._convert_to_karras + def _convert_to_karras(self, in_sigmas: torch.FloatTensor, num_inference_steps) -> torch.FloatTensor: + """Constructs the noise schedule of Karras et al. (2022).""" + + # Hack to make sure that other schedulers which copy this function don't break + # TODO: Add this logic to the other schedulers + if hasattr(self.config, "sigma_min"): + sigma_min = self.config.sigma_min + else: + sigma_min = None + + if hasattr(self.config, "sigma_max"): + sigma_max = self.config.sigma_max + else: + sigma_max = None + + sigma_min = sigma_min if sigma_min is not None else in_sigmas[-1].item() + sigma_max = sigma_max if sigma_max is not None else in_sigmas[0].item() + + rho = 7.0 # 7.0 is the value used in the paper + ramp = np.linspace(0, 1, num_inference_steps) + min_inv_rho = sigma_min ** (1 / rho) + max_inv_rho = sigma_max ** (1 / rho) + sigmas = (max_inv_rho + ramp * (min_inv_rho - max_inv_rho)) ** rho + return sigmas + + def convert_model_output( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + **kwargs, + ) -> torch.FloatTensor: + r""" + Convert the model output to the corresponding type the UniPC algorithm needs. + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + + Returns: + `torch.FloatTensor`: + The converted model output. + """ + timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError("missing `sample` as a required keyward argument") + if timestep is not None: + deprecate( + "timesteps", + "1.0.0", + "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + sigma = self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + + if self.predict_x0: + if self.config.prediction_type == "epsilon": + x0_pred = (sample - sigma_t * model_output) / alpha_t + elif self.config.prediction_type == "sample": + x0_pred = model_output + elif self.config.prediction_type == "v_prediction": + x0_pred = alpha_t * sample - sigma_t * model_output + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the UniPCMultistepScheduler." + ) + + if self.config.thresholding: + x0_pred = self._threshold_sample(x0_pred) + + return x0_pred + else: + if self.config.prediction_type == "epsilon": + return model_output + elif self.config.prediction_type == "sample": + epsilon = (sample - alpha_t * model_output) / sigma_t + return epsilon + elif self.config.prediction_type == "v_prediction": + epsilon = alpha_t * model_output + sigma_t * sample + return epsilon + else: + raise ValueError( + f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`, or" + " `v_prediction` for the UniPCMultistepScheduler." + ) + + def multistep_uni_p_bh_update( + self, + model_output: torch.FloatTensor, + *args, + sample: torch.FloatTensor = None, + order: int = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the UniP (B(h) version). Alternatively, `self.solver_p` is used if is specified. + + Args: + model_output (`torch.FloatTensor`): + The direct output from the learned diffusion model at the current timestep. + prev_timestep (`int`): + The previous discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + order (`int`): + The order of UniP at this timestep (corresponds to the *p* in UniPC-p). + + Returns: + `torch.FloatTensor`: + The sample tensor at the previous timestep. + """ + prev_timestep = args[0] if len(args) > 0 else kwargs.pop("prev_timestep", None) + if sample is None: + if len(args) > 1: + sample = args[1] + else: + raise ValueError(" missing `sample` as a required keyward argument") + if order is None: + if len(args) > 2: + order = args[2] + else: + raise ValueError(" missing `order` as a required keyward argument") + if prev_timestep is not None: + deprecate( + "prev_timestep", + "1.0.0", + "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + model_output_list = self.model_outputs + + s0 = self.timestep_list[-1] + m0 = model_output_list[-1] + x = sample + + if self.solver_p: + x_t = self.solver_p.step(model_output, s0, x).prev_sample + return x_t + + sigma_t, sigma_s0 = self.sigmas[self.step_index + 1], self.sigmas[self.step_index] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + + h = lambda_t - lambda_s0 + device = sample.device + + rks = [] + D1s = [] + for i in range(1, order): + si = self.step_index - i + mi = model_output_list[-(i + 1)] + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + rk = (lambda_si - lambda_s0) / h + rks.append(rk) + D1s.append((mi - m0) / rk) + + rks.append(1.0) + rks = torch.tensor(rks, device=device) + + R = [] + b = [] + + hh = -h if self.predict_x0 else h + h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 + h_phi_k = h_phi_1 / hh - 1 + + factorial_i = 1 + + if self.config.solver_type == "bh1": + B_h = hh + elif self.config.solver_type == "bh2": + B_h = torch.expm1(hh) + else: + raise NotImplementedError() + + for i in range(1, order + 1): + R.append(torch.pow(rks, i - 1)) + b.append(h_phi_k * factorial_i / B_h) + factorial_i *= i + 1 + h_phi_k = h_phi_k / hh - 1 / factorial_i + + R = torch.stack(R) + b = torch.tensor(b, device=device) + + if len(D1s) > 0: + D1s = torch.stack(D1s, dim=1) # (B, K) + # for order 2, we use a simplified version + if order == 2: + rhos_p = torch.tensor([0.5], dtype=x.dtype, device=device) + else: + rhos_p = torch.linalg.solve(R[:-1, :-1], b[:-1]) + else: + D1s = None + + if self.predict_x0: + x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0 + if D1s is not None: + pred_res = torch.einsum("k,bkc...->bc...", rhos_p, D1s) + else: + pred_res = 0 + x_t = x_t_ - alpha_t * B_h * pred_res + else: + x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0 + if D1s is not None: + pred_res = torch.einsum("k,bkc...->bc...", rhos_p, D1s) + else: + pred_res = 0 + x_t = x_t_ - sigma_t * B_h * pred_res + + x_t = x_t.to(x.dtype) + return x_t + + def multistep_uni_c_bh_update( + self, + this_model_output: torch.FloatTensor, + *args, + last_sample: torch.FloatTensor = None, + this_sample: torch.FloatTensor = None, + order: int = None, + **kwargs, + ) -> torch.FloatTensor: + """ + One step for the UniC (B(h) version). + + Args: + this_model_output (`torch.FloatTensor`): + The model outputs at `x_t`. + this_timestep (`int`): + The current timestep `t`. + last_sample (`torch.FloatTensor`): + The generated sample before the last predictor `x_{t-1}`. + this_sample (`torch.FloatTensor`): + The generated sample after the last predictor `x_{t}`. + order (`int`): + The `p` of UniC-p at this step. The effective order of accuracy should be `order + 1`. + + Returns: + `torch.FloatTensor`: + The corrected sample tensor at the current timestep. + """ + this_timestep = args[0] if len(args) > 0 else kwargs.pop("this_timestep", None) + if last_sample is None: + if len(args) > 1: + last_sample = args[1] + else: + raise ValueError(" missing`last_sample` as a required keyward argument") + if this_sample is None: + if len(args) > 2: + this_sample = args[2] + else: + raise ValueError(" missing`this_sample` as a required keyward argument") + if order is None: + if len(args) > 3: + order = args[3] + else: + raise ValueError(" missing`order` as a required keyward argument") + if this_timestep is not None: + deprecate( + "this_timestep", + "1.0.0", + "Passing `this_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`", + ) + + model_output_list = self.model_outputs + + m0 = model_output_list[-1] + x = last_sample + x_t = this_sample + model_t = this_model_output + + sigma_t, sigma_s0 = self.sigmas[self.step_index], self.sigmas[self.step_index - 1] + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t) + alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0) + + lambda_t = torch.log(alpha_t) - torch.log(sigma_t) + lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0) + + h = lambda_t - lambda_s0 + device = this_sample.device + + rks = [] + D1s = [] + for i in range(1, order): + si = self.step_index - (i + 1) + mi = model_output_list[-(i + 1)] + alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si]) + lambda_si = torch.log(alpha_si) - torch.log(sigma_si) + rk = (lambda_si - lambda_s0) / h + rks.append(rk) + D1s.append((mi - m0) / rk) + + rks.append(1.0) + rks = torch.tensor(rks, device=device) + + R = [] + b = [] + + hh = -h if self.predict_x0 else h + h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1 + h_phi_k = h_phi_1 / hh - 1 + + factorial_i = 1 + + if self.config.solver_type == "bh1": + B_h = hh + elif self.config.solver_type == "bh2": + B_h = torch.expm1(hh) + else: + raise NotImplementedError() + + for i in range(1, order + 1): + R.append(torch.pow(rks, i - 1)) + b.append(h_phi_k * factorial_i / B_h) + factorial_i *= i + 1 + h_phi_k = h_phi_k / hh - 1 / factorial_i + + R = torch.stack(R) + b = torch.tensor(b, device=device) + + if len(D1s) > 0: + D1s = torch.stack(D1s, dim=1) + else: + D1s = None + + # for order 1, we use a simplified version + if order == 1: + rhos_c = torch.tensor([0.5], dtype=x.dtype, device=device) + else: + rhos_c = torch.linalg.solve(R, b) + + if self.predict_x0: + x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0 + if D1s is not None: + corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], D1s) + else: + corr_res = 0 + D1_t = model_t - m0 + x_t = x_t_ - alpha_t * B_h * (corr_res + rhos_c[-1] * D1_t) + else: + x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0 + if D1s is not None: + corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], D1s) + else: + corr_res = 0 + D1_t = model_t - m0 + x_t = x_t_ - sigma_t * B_h * (corr_res + rhos_c[-1] * D1_t) + x_t = x_t.to(x.dtype) + return x_t + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.index_for_timestep + def index_for_timestep(self, timestep, schedule_timesteps=None): + if schedule_timesteps is None: + schedule_timesteps = self.timesteps + + index_candidates = (schedule_timesteps == timestep).nonzero() + + if len(index_candidates) == 0: + step_index = len(self.timesteps) - 1 + # The sigma index that is taken for the **very** first `step` + # is always the second index (or the last index if there is only 1) + # This way we can ensure we don't accidentally skip a sigma in + # case we start in the middle of the denoising schedule (e.g. for image-to-image) + elif len(index_candidates) > 1: + step_index = index_candidates[1].item() + else: + step_index = index_candidates[0].item() + + return step_index + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index + def _init_step_index(self, timestep): + """ + Initialize the step_index counter for the scheduler. + """ + + if self.begin_index is None: + if isinstance(timestep, torch.Tensor): + timestep = timestep.to(self.timesteps.device) + self._step_index = self.index_for_timestep(timestep) + else: + self._step_index = self._begin_index + + def step( + self, + model_output: torch.FloatTensor, + timestep: int, + sample: torch.FloatTensor, + return_dict: bool = True, + ) -> Union[SchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with + the multistep UniPC. + + Args: + model_output (`torch.FloatTensor`): + The direct output from learned diffusion model. + timestep (`int`): + The current discrete timestep in the diffusion chain. + sample (`torch.FloatTensor`): + A current instance of a sample created by the diffusion process. + return_dict (`bool`): + Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`. + + Returns: + [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a + tuple is returned where the first element is the sample tensor. + + """ + if self.num_inference_steps is None: + raise ValueError( + "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler" + ) + + if self.step_index is None: + self._init_step_index(timestep) + + use_corrector = ( + self.step_index > 0 and self.step_index - 1 not in self.disable_corrector and self.last_sample is not None + ) + + model_output_convert = self.convert_model_output(model_output, sample=sample) + if use_corrector: + sample = self.multistep_uni_c_bh_update( + this_model_output=model_output_convert, + last_sample=self.last_sample, + this_sample=sample, + order=self.this_order, + ) + + for i in range(self.config.solver_order - 1): + self.model_outputs[i] = self.model_outputs[i + 1] + self.timestep_list[i] = self.timestep_list[i + 1] + + self.model_outputs[-1] = model_output_convert + self.timestep_list[-1] = timestep + + if self.config.lower_order_final: + this_order = min(self.config.solver_order, len(self.timesteps) - self.step_index) + else: + this_order = self.config.solver_order + + self.this_order = min(this_order, self.lower_order_nums + 1) # warmup for multistep + assert self.this_order > 0 + + self.last_sample = sample + prev_sample = self.multistep_uni_p_bh_update( + model_output=model_output, # pass the original non-converted model output, in case solver-p is used + sample=sample, + order=self.this_order, + ) + + if self.lower_order_nums < self.config.solver_order: + self.lower_order_nums += 1 + + # upon completion increase step index by one + self._step_index += 1 + + if not return_dict: + return (prev_sample,) + + return SchedulerOutput(prev_sample=prev_sample) + + def scale_model_input(self, sample: torch.FloatTensor, *args, **kwargs) -> torch.FloatTensor: + """ + Ensures interchangeability with schedulers that need to scale the denoising model input depending on the + current timestep. + + Args: + sample (`torch.FloatTensor`): + The input sample. + + Returns: + `torch.FloatTensor`: + A scaled input sample. + """ + return sample + + # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise + def add_noise( + self, + original_samples: torch.FloatTensor, + noise: torch.FloatTensor, + timesteps: torch.IntTensor, + ) -> torch.FloatTensor: + # Make sure sigmas and timesteps have the same device and dtype as original_samples + sigmas = self.sigmas.to(device=original_samples.device, dtype=original_samples.dtype) + if original_samples.device.type == "mps" and torch.is_floating_point(timesteps): + # mps does not support float64 + schedule_timesteps = self.timesteps.to(original_samples.device, dtype=torch.float32) + timesteps = timesteps.to(original_samples.device, dtype=torch.float32) + else: + schedule_timesteps = self.timesteps.to(original_samples.device) + timesteps = timesteps.to(original_samples.device) + + # begin_index is None when the scheduler is used for training + if self.begin_index is None: + step_indices = [self.index_for_timestep(t, schedule_timesteps) for t in timesteps] + else: + step_indices = [self.begin_index] * timesteps.shape[0] + + sigma = sigmas[step_indices].flatten() + while len(sigma.shape) < len(original_samples.shape): + sigma = sigma.unsqueeze(-1) + + alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma) + noisy_samples = alpha_t * original_samples + sigma_t * noise + return noisy_samples + + def __len__(self): + return self.config.num_train_timesteps diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_utils.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_utils.py new file mode 100755 index 0000000..5dbdb82 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_utils.py @@ -0,0 +1,186 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import importlib +import os +from dataclasses import dataclass +from enum import Enum +from typing import Optional, Union + +import torch +from huggingface_hub.utils import validate_hf_hub_args + +from ..utils import BaseOutput, PushToHubMixin + + +SCHEDULER_CONFIG_NAME = "scheduler_config.json" + + +# NOTE: We make this type an enum because it simplifies usage in docs and prevents +# circular imports when used for `_compatibles` within the schedulers module. +# When it's used as a type in pipelines, it really is a Union because the actual +# scheduler instance is passed in. +class KarrasDiffusionSchedulers(Enum): + DDIMScheduler = 1 + DDPMScheduler = 2 + PNDMScheduler = 3 + LMSDiscreteScheduler = 4 + EulerDiscreteScheduler = 5 + HeunDiscreteScheduler = 6 + EulerAncestralDiscreteScheduler = 7 + DPMSolverMultistepScheduler = 8 + DPMSolverSinglestepScheduler = 9 + KDPM2DiscreteScheduler = 10 + KDPM2AncestralDiscreteScheduler = 11 + DEISMultistepScheduler = 12 + UniPCMultistepScheduler = 13 + DPMSolverSDEScheduler = 14 + EDMEulerScheduler = 15 + + +@dataclass +class SchedulerOutput(BaseOutput): + """ + Base class for the output of a scheduler's `step` function. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + """ + + prev_sample: torch.FloatTensor + + +class SchedulerMixin(PushToHubMixin): + """ + Base class for all schedulers. + + [`SchedulerMixin`] contains common functions shared by all schedulers such as general loading and saving + functionalities. + + [`ConfigMixin`] takes care of storing the configuration attributes (like `num_train_timesteps`) that are passed to + the scheduler's `__init__` function, and the attributes can be accessed by `scheduler.config.num_train_timesteps`. + + Class attributes: + - **_compatibles** (`List[str]`) -- A list of scheduler classes that are compatible with the parent scheduler + class. Use [`~ConfigMixin.from_config`] to load a different compatible scheduler class (should be overridden + by parent class). + """ + + config_name = SCHEDULER_CONFIG_NAME + _compatibles = [] + has_compatibles = True + + @classmethod + @validate_hf_hub_args + def from_pretrained( + cls, + pretrained_model_name_or_path: Optional[Union[str, os.PathLike]] = None, + subfolder: Optional[str] = None, + return_unused_kwargs=False, + **kwargs, + ): + r""" + Instantiate a scheduler from a pre-defined JSON configuration file in a local directory or Hub repository. + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *model id* (for example `google/ddpm-celebahq-256`) of a pretrained model hosted on + the Hub. + - A path to a *directory* (for example `./my_model_directory`) containing the scheduler + configuration saved with [`~SchedulerMixin.save_pretrained`]. + subfolder (`str`, *optional*): + The subfolder location of a model file within a larger model repository on the Hub or locally. + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + Whether kwargs that are not consumed by the Python class should be returned or not. + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory where a downloaded pretrained model configuration is cached if the standard cache + is not used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to resume downloading the model weights and configuration files. If set to `False`, any + incompletely downloaded files are deleted. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, for example, `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether to only load local model weights and configuration files or not. If set to `True`, the model + won't be downloaded from the Hub. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, the token generated from + `diffusers-cli login` (stored in `~/.huggingface`) is used. + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, a commit id, or any identifier + allowed by Git. + + + + To use private or [gated models](https://huggingface.co/docs/hub/models-gated#gated-models), log-in with + `huggingface-cli login`. You can also activate the special + ["offline-mode"](https://huggingface.co/diffusers/installation.html#offline-mode) to use this method in a + firewalled environment. + + + + """ + config, kwargs, commit_hash = cls.load_config( + pretrained_model_name_or_path=pretrained_model_name_or_path, + subfolder=subfolder, + return_unused_kwargs=True, + return_commit_hash=True, + **kwargs, + ) + return cls.from_config(config, return_unused_kwargs=return_unused_kwargs, **kwargs) + + def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + """ + Save a scheduler configuration object to a directory so that it can be reloaded using the + [`~SchedulerMixin.from_pretrained`] class method. + + Args: + save_directory (`str` or `os.PathLike`): + Directory where the configuration JSON file will be saved (will be created if it does not exist). + push_to_hub (`bool`, *optional*, defaults to `False`): + Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the + repository you want to push to with `repo_id` (will default to the name of `save_directory` in your + namespace). + kwargs (`Dict[str, Any]`, *optional*): + Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. + """ + self.save_config(save_directory=save_directory, push_to_hub=push_to_hub, **kwargs) + + @property + def compatibles(self): + """ + Returns all schedulers that are compatible with this scheduler + + Returns: + `List[SchedulerMixin]`: List of compatible schedulers + """ + return self._get_compatibles() + + @classmethod + def _get_compatibles(cls): + compatible_classes_str = list(set([cls.__name__] + cls._compatibles)) + diffusers_library = importlib.import_module(__name__.split(".")[0]) + compatible_classes = [ + getattr(diffusers_library, c) for c in compatible_classes_str if hasattr(diffusers_library, c) + ] + return compatible_classes diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_utils_flax.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_utils_flax.py new file mode 100755 index 0000000..a1d471f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_utils_flax.py @@ -0,0 +1,293 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import importlib +import math +import os +from dataclasses import dataclass +from enum import Enum +from typing import Optional, Tuple, Union + +import flax +import jax.numpy as jnp +from huggingface_hub.utils import validate_hf_hub_args + +from ..utils import BaseOutput, PushToHubMixin + + +SCHEDULER_CONFIG_NAME = "scheduler_config.json" + + +# NOTE: We make this type an enum because it simplifies usage in docs and prevents +# circular imports when used for `_compatibles` within the schedulers module. +# When it's used as a type in pipelines, it really is a Union because the actual +# scheduler instance is passed in. +class FlaxKarrasDiffusionSchedulers(Enum): + FlaxDDIMScheduler = 1 + FlaxDDPMScheduler = 2 + FlaxPNDMScheduler = 3 + FlaxLMSDiscreteScheduler = 4 + FlaxDPMSolverMultistepScheduler = 5 + FlaxEulerDiscreteScheduler = 6 + + +@dataclass +class FlaxSchedulerOutput(BaseOutput): + """ + Base class for the scheduler's step function output. + + Args: + prev_sample (`jnp.ndarray` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample (x_{t-1}) of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + """ + + prev_sample: jnp.ndarray + + +class FlaxSchedulerMixin(PushToHubMixin): + """ + Mixin containing common functions for the schedulers. + + Class attributes: + - **_compatibles** (`List[str]`) -- A list of classes that are compatible with the parent class, so that + `from_config` can be used from a class different than the one used to save the config (should be overridden + by parent class). + """ + + config_name = SCHEDULER_CONFIG_NAME + ignore_for_config = ["dtype"] + _compatibles = [] + has_compatibles = True + + @classmethod + @validate_hf_hub_args + def from_pretrained( + cls, + pretrained_model_name_or_path: Optional[Union[str, os.PathLike]] = None, + subfolder: Optional[str] = None, + return_unused_kwargs=False, + **kwargs, + ): + r""" + Instantiate a Scheduler class from a pre-defined JSON-file. + + Parameters: + pretrained_model_name_or_path (`str` or `os.PathLike`, *optional*): + Can be either: + + - A string, the *model id* of a model repo on huggingface.co. Valid model ids should have an + organization name, like `google/ddpm-celebahq-256`. + - A path to a *directory* containing model weights saved using [`~SchedulerMixin.save_pretrained`], + e.g., `./my_model_directory/`. + subfolder (`str`, *optional*): + In case the relevant files are located inside a subfolder of the model repo (either remote in + huggingface.co or downloaded locally), you can specify the folder name here. + return_unused_kwargs (`bool`, *optional*, defaults to `False`): + Whether kwargs that are not consumed by the Python class should be returned or not. + + cache_dir (`Union[str, os.PathLike]`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the + standard cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force the (re-)download of the model weights and configuration files, overriding the + cached versions if they exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to delete incompletely received files. Will attempt to resume the download if such a + file exists. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request. + output_loading_info(`bool`, *optional*, defaults to `False`): + Whether or not to also return a dictionary containing missing keys, unexpected keys and error messages. + local_files_only(`bool`, *optional*, defaults to `False`): + Whether or not to only look at local files (i.e., do not try to download the model). + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `transformers-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + + + + It is required to be logged in (`huggingface-cli login`) when you want to use private or [gated + models](https://huggingface.co/docs/hub/models-gated#gated-models). + + + + + + Activate the special ["offline-mode"](https://huggingface.co/transformers/installation.html#offline-mode) to + use this method in a firewalled environment. + + + + """ + config, kwargs = cls.load_config( + pretrained_model_name_or_path=pretrained_model_name_or_path, + subfolder=subfolder, + return_unused_kwargs=True, + **kwargs, + ) + scheduler, unused_kwargs = cls.from_config(config, return_unused_kwargs=True, **kwargs) + + if hasattr(scheduler, "create_state") and getattr(scheduler, "has_state", False): + state = scheduler.create_state() + + if return_unused_kwargs: + return scheduler, state, unused_kwargs + + return scheduler, state + + def save_pretrained(self, save_directory: Union[str, os.PathLike], push_to_hub: bool = False, **kwargs): + """ + Save a scheduler configuration object to the directory `save_directory`, so that it can be re-loaded using the + [`~FlaxSchedulerMixin.from_pretrained`] class method. + + Args: + save_directory (`str` or `os.PathLike`): + Directory where the configuration JSON file will be saved (will be created if it does not exist). + push_to_hub (`bool`, *optional*, defaults to `False`): + Whether or not to push your model to the Hugging Face Hub after saving it. You can specify the + repository you want to push to with `repo_id` (will default to the name of `save_directory` in your + namespace). + kwargs (`Dict[str, Any]`, *optional*): + Additional keyword arguments passed along to the [`~utils.PushToHubMixin.push_to_hub`] method. + """ + self.save_config(save_directory=save_directory, push_to_hub=push_to_hub, **kwargs) + + @property + def compatibles(self): + """ + Returns all schedulers that are compatible with this scheduler + + Returns: + `List[SchedulerMixin]`: List of compatible schedulers + """ + return self._get_compatibles() + + @classmethod + def _get_compatibles(cls): + compatible_classes_str = list(set([cls.__name__] + cls._compatibles)) + diffusers_library = importlib.import_module(__name__.split(".")[0]) + compatible_classes = [ + getattr(diffusers_library, c) for c in compatible_classes_str if hasattr(diffusers_library, c) + ] + return compatible_classes + + +def broadcast_to_shape_from_left(x: jnp.ndarray, shape: Tuple[int]) -> jnp.ndarray: + assert len(shape) >= x.ndim + return jnp.broadcast_to(x.reshape(x.shape + (1,) * (len(shape) - x.ndim)), shape) + + +def betas_for_alpha_bar(num_diffusion_timesteps: int, max_beta=0.999, dtype=jnp.float32) -> jnp.ndarray: + """ + Create a beta schedule that discretizes the given alpha_t_bar function, which defines the cumulative product of + (1-beta) over time from t = [0,1]. + + Contains a function alpha_bar that takes an argument t and transforms it to the cumulative product of (1-beta) up + to that part of the diffusion process. + + + Args: + num_diffusion_timesteps (`int`): the number of betas to produce. + max_beta (`float`): the maximum beta to use; use values lower than 1 to + prevent singularities. + + Returns: + betas (`jnp.ndarray`): the betas used by the scheduler to step the model outputs + """ + + def alpha_bar(time_step): + return math.cos((time_step + 0.008) / 1.008 * math.pi / 2) ** 2 + + betas = [] + for i in range(num_diffusion_timesteps): + t1 = i / num_diffusion_timesteps + t2 = (i + 1) / num_diffusion_timesteps + betas.append(min(1 - alpha_bar(t2) / alpha_bar(t1), max_beta)) + return jnp.array(betas, dtype=dtype) + + +@flax.struct.dataclass +class CommonSchedulerState: + alphas: jnp.ndarray + betas: jnp.ndarray + alphas_cumprod: jnp.ndarray + + @classmethod + def create(cls, scheduler): + config = scheduler.config + + if config.trained_betas is not None: + betas = jnp.asarray(config.trained_betas, dtype=scheduler.dtype) + elif config.beta_schedule == "linear": + betas = jnp.linspace(config.beta_start, config.beta_end, config.num_train_timesteps, dtype=scheduler.dtype) + elif config.beta_schedule == "scaled_linear": + # this schedule is very specific to the latent diffusion model. + betas = ( + jnp.linspace( + config.beta_start**0.5, config.beta_end**0.5, config.num_train_timesteps, dtype=scheduler.dtype + ) + ** 2 + ) + elif config.beta_schedule == "squaredcos_cap_v2": + # Glide cosine schedule + betas = betas_for_alpha_bar(config.num_train_timesteps, dtype=scheduler.dtype) + else: + raise NotImplementedError( + f"beta_schedule {config.beta_schedule} is not implemented for scheduler {scheduler.__class__.__name__}" + ) + + alphas = 1.0 - betas + + alphas_cumprod = jnp.cumprod(alphas, axis=0) + + return cls( + alphas=alphas, + betas=betas, + alphas_cumprod=alphas_cumprod, + ) + + +def get_sqrt_alpha_prod( + state: CommonSchedulerState, original_samples: jnp.ndarray, noise: jnp.ndarray, timesteps: jnp.ndarray +): + alphas_cumprod = state.alphas_cumprod + + sqrt_alpha_prod = alphas_cumprod[timesteps] ** 0.5 + sqrt_alpha_prod = sqrt_alpha_prod.flatten() + sqrt_alpha_prod = broadcast_to_shape_from_left(sqrt_alpha_prod, original_samples.shape) + + sqrt_one_minus_alpha_prod = (1 - alphas_cumprod[timesteps]) ** 0.5 + sqrt_one_minus_alpha_prod = sqrt_one_minus_alpha_prod.flatten() + sqrt_one_minus_alpha_prod = broadcast_to_shape_from_left(sqrt_one_minus_alpha_prod, original_samples.shape) + + return sqrt_alpha_prod, sqrt_one_minus_alpha_prod + + +def add_noise_common( + state: CommonSchedulerState, original_samples: jnp.ndarray, noise: jnp.ndarray, timesteps: jnp.ndarray +): + sqrt_alpha_prod, sqrt_one_minus_alpha_prod = get_sqrt_alpha_prod(state, original_samples, noise, timesteps) + noisy_samples = sqrt_alpha_prod * original_samples + sqrt_one_minus_alpha_prod * noise + return noisy_samples + + +def get_velocity_common(state: CommonSchedulerState, sample: jnp.ndarray, noise: jnp.ndarray, timesteps: jnp.ndarray): + sqrt_alpha_prod, sqrt_one_minus_alpha_prod = get_sqrt_alpha_prod(state, sample, noise, timesteps) + velocity = sqrt_alpha_prod * noise - sqrt_one_minus_alpha_prod * sample + return velocity diff --git a/diffusers-0.27.0/src/diffusers/schedulers/scheduling_vq_diffusion.py b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_vq_diffusion.py new file mode 100755 index 0000000..03ba95c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/schedulers/scheduling_vq_diffusion.py @@ -0,0 +1,467 @@ +# Copyright 2024 Microsoft and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from dataclasses import dataclass +from typing import Optional, Tuple, Union + +import numpy as np +import torch +import torch.nn.functional as F + +from ..configuration_utils import ConfigMixin, register_to_config +from ..utils import BaseOutput +from .scheduling_utils import SchedulerMixin + + +@dataclass +class VQDiffusionSchedulerOutput(BaseOutput): + """ + Output class for the scheduler's step function output. + + Args: + prev_sample (`torch.LongTensor` of shape `(batch size, num latent pixels)`): + Computed sample x_{t-1} of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + """ + + prev_sample: torch.LongTensor + + +def index_to_log_onehot(x: torch.LongTensor, num_classes: int) -> torch.FloatTensor: + """ + Convert batch of vector of class indices into batch of log onehot vectors + + Args: + x (`torch.LongTensor` of shape `(batch size, vector length)`): + Batch of class indices + + num_classes (`int`): + number of classes to be used for the onehot vectors + + Returns: + `torch.FloatTensor` of shape `(batch size, num classes, vector length)`: + Log onehot vectors + """ + x_onehot = F.one_hot(x, num_classes) + x_onehot = x_onehot.permute(0, 2, 1) + log_x = torch.log(x_onehot.float().clamp(min=1e-30)) + return log_x + + +def gumbel_noised(logits: torch.FloatTensor, generator: Optional[torch.Generator]) -> torch.FloatTensor: + """ + Apply gumbel noise to `logits` + """ + uniform = torch.rand(logits.shape, device=logits.device, generator=generator) + gumbel_noise = -torch.log(-torch.log(uniform + 1e-30) + 1e-30) + noised = gumbel_noise + logits + return noised + + +def alpha_schedules(num_diffusion_timesteps: int, alpha_cum_start=0.99999, alpha_cum_end=0.000009): + """ + Cumulative and non-cumulative alpha schedules. + + See section 4.1. + """ + att = ( + np.arange(0, num_diffusion_timesteps) / (num_diffusion_timesteps - 1) * (alpha_cum_end - alpha_cum_start) + + alpha_cum_start + ) + att = np.concatenate(([1], att)) + at = att[1:] / att[:-1] + att = np.concatenate((att[1:], [1])) + return at, att + + +def gamma_schedules(num_diffusion_timesteps: int, gamma_cum_start=0.000009, gamma_cum_end=0.99999): + """ + Cumulative and non-cumulative gamma schedules. + + See section 4.1. + """ + ctt = ( + np.arange(0, num_diffusion_timesteps) / (num_diffusion_timesteps - 1) * (gamma_cum_end - gamma_cum_start) + + gamma_cum_start + ) + ctt = np.concatenate(([0], ctt)) + one_minus_ctt = 1 - ctt + one_minus_ct = one_minus_ctt[1:] / one_minus_ctt[:-1] + ct = 1 - one_minus_ct + ctt = np.concatenate((ctt[1:], [0])) + return ct, ctt + + +class VQDiffusionScheduler(SchedulerMixin, ConfigMixin): + """ + A scheduler for vector quantized diffusion. + + This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic + methods the library implements for all schedulers such as loading and saving. + + Args: + num_vec_classes (`int`): + The number of classes of the vector embeddings of the latent pixels. Includes the class for the masked + latent pixel. + num_train_timesteps (`int`, defaults to 100): + The number of diffusion steps to train the model. + alpha_cum_start (`float`, defaults to 0.99999): + The starting cumulative alpha value. + alpha_cum_end (`float`, defaults to 0.00009): + The ending cumulative alpha value. + gamma_cum_start (`float`, defaults to 0.00009): + The starting cumulative gamma value. + gamma_cum_end (`float`, defaults to 0.99999): + The ending cumulative gamma value. + """ + + order = 1 + + @register_to_config + def __init__( + self, + num_vec_classes: int, + num_train_timesteps: int = 100, + alpha_cum_start: float = 0.99999, + alpha_cum_end: float = 0.000009, + gamma_cum_start: float = 0.000009, + gamma_cum_end: float = 0.99999, + ): + self.num_embed = num_vec_classes + + # By convention, the index for the mask class is the last class index + self.mask_class = self.num_embed - 1 + + at, att = alpha_schedules(num_train_timesteps, alpha_cum_start=alpha_cum_start, alpha_cum_end=alpha_cum_end) + ct, ctt = gamma_schedules(num_train_timesteps, gamma_cum_start=gamma_cum_start, gamma_cum_end=gamma_cum_end) + + num_non_mask_classes = self.num_embed - 1 + bt = (1 - at - ct) / num_non_mask_classes + btt = (1 - att - ctt) / num_non_mask_classes + + at = torch.tensor(at.astype("float64")) + bt = torch.tensor(bt.astype("float64")) + ct = torch.tensor(ct.astype("float64")) + log_at = torch.log(at) + log_bt = torch.log(bt) + log_ct = torch.log(ct) + + att = torch.tensor(att.astype("float64")) + btt = torch.tensor(btt.astype("float64")) + ctt = torch.tensor(ctt.astype("float64")) + log_cumprod_at = torch.log(att) + log_cumprod_bt = torch.log(btt) + log_cumprod_ct = torch.log(ctt) + + self.log_at = log_at.float() + self.log_bt = log_bt.float() + self.log_ct = log_ct.float() + self.log_cumprod_at = log_cumprod_at.float() + self.log_cumprod_bt = log_cumprod_bt.float() + self.log_cumprod_ct = log_cumprod_ct.float() + + # setable values + self.num_inference_steps = None + self.timesteps = torch.from_numpy(np.arange(0, num_train_timesteps)[::-1].copy()) + + def set_timesteps(self, num_inference_steps: int, device: Union[str, torch.device] = None): + """ + Sets the discrete timesteps used for the diffusion chain (to be run before inference). + + Args: + num_inference_steps (`int`): + The number of diffusion steps used when generating samples with a pre-trained model. + device (`str` or `torch.device`, *optional*): + The device to which the timesteps and diffusion process parameters (alpha, beta, gamma) should be moved + to. + """ + self.num_inference_steps = num_inference_steps + timesteps = np.arange(0, self.num_inference_steps)[::-1].copy() + self.timesteps = torch.from_numpy(timesteps).to(device) + + self.log_at = self.log_at.to(device) + self.log_bt = self.log_bt.to(device) + self.log_ct = self.log_ct.to(device) + self.log_cumprod_at = self.log_cumprod_at.to(device) + self.log_cumprod_bt = self.log_cumprod_bt.to(device) + self.log_cumprod_ct = self.log_cumprod_ct.to(device) + + def step( + self, + model_output: torch.FloatTensor, + timestep: torch.long, + sample: torch.LongTensor, + generator: Optional[torch.Generator] = None, + return_dict: bool = True, + ) -> Union[VQDiffusionSchedulerOutput, Tuple]: + """ + Predict the sample from the previous timestep by the reverse transition distribution. See + [`~VQDiffusionScheduler.q_posterior`] for more details about how the distribution is computer. + + Args: + log_p_x_0: (`torch.FloatTensor` of shape `(batch size, num classes - 1, num latent pixels)`): + The log probabilities for the predicted classes of the initial latent pixels. Does not include a + prediction for the masked class as the initial unnoised image cannot be masked. + t (`torch.long`): + The timestep that determines which transition matrices are used. + x_t (`torch.LongTensor` of shape `(batch size, num latent pixels)`): + The classes of each latent pixel at time `t`. + generator (`torch.Generator`, or `None`): + A random number generator for the noise applied to `p(x_{t-1} | x_t)` before it is sampled from. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~schedulers.scheduling_vq_diffusion.VQDiffusionSchedulerOutput`] or + `tuple`. + + Returns: + [`~schedulers.scheduling_vq_diffusion.VQDiffusionSchedulerOutput`] or `tuple`: + If return_dict is `True`, [`~schedulers.scheduling_vq_diffusion.VQDiffusionSchedulerOutput`] is + returned, otherwise a tuple is returned where the first element is the sample tensor. + """ + if timestep == 0: + log_p_x_t_min_1 = model_output + else: + log_p_x_t_min_1 = self.q_posterior(model_output, sample, timestep) + + log_p_x_t_min_1 = gumbel_noised(log_p_x_t_min_1, generator) + + x_t_min_1 = log_p_x_t_min_1.argmax(dim=1) + + if not return_dict: + return (x_t_min_1,) + + return VQDiffusionSchedulerOutput(prev_sample=x_t_min_1) + + def q_posterior(self, log_p_x_0, x_t, t): + """ + Calculates the log probabilities for the predicted classes of the image at timestep `t-1`: + + ``` + p(x_{t-1} | x_t) = sum( q(x_t | x_{t-1}) * q(x_{t-1} | x_0) * p(x_0) / q(x_t | x_0) ) + ``` + + Args: + log_p_x_0 (`torch.FloatTensor` of shape `(batch size, num classes - 1, num latent pixels)`): + The log probabilities for the predicted classes of the initial latent pixels. Does not include a + prediction for the masked class as the initial unnoised image cannot be masked. + x_t (`torch.LongTensor` of shape `(batch size, num latent pixels)`): + The classes of each latent pixel at time `t`. + t (`torch.Long`): + The timestep that determines which transition matrix is used. + + Returns: + `torch.FloatTensor` of shape `(batch size, num classes, num latent pixels)`: + The log probabilities for the predicted classes of the image at timestep `t-1`. + """ + log_onehot_x_t = index_to_log_onehot(x_t, self.num_embed) + + log_q_x_t_given_x_0 = self.log_Q_t_transitioning_to_known_class( + t=t, x_t=x_t, log_onehot_x_t=log_onehot_x_t, cumulative=True + ) + + log_q_t_given_x_t_min_1 = self.log_Q_t_transitioning_to_known_class( + t=t, x_t=x_t, log_onehot_x_t=log_onehot_x_t, cumulative=False + ) + + # p_0(x_0=C_0 | x_t) / q(x_t | x_0=C_0) ... p_n(x_0=C_0 | x_t) / q(x_t | x_0=C_0) + # . . . + # . . . + # . . . + # p_0(x_0=C_{k-1} | x_t) / q(x_t | x_0=C_{k-1}) ... p_n(x_0=C_{k-1} | x_t) / q(x_t | x_0=C_{k-1}) + q = log_p_x_0 - log_q_x_t_given_x_0 + + # sum_0 = p_0(x_0=C_0 | x_t) / q(x_t | x_0=C_0) + ... + p_0(x_0=C_{k-1} | x_t) / q(x_t | x_0=C_{k-1}), ... , + # sum_n = p_n(x_0=C_0 | x_t) / q(x_t | x_0=C_0) + ... + p_n(x_0=C_{k-1} | x_t) / q(x_t | x_0=C_{k-1}) + q_log_sum_exp = torch.logsumexp(q, dim=1, keepdim=True) + + # p_0(x_0=C_0 | x_t) / q(x_t | x_0=C_0) / sum_0 ... p_n(x_0=C_0 | x_t) / q(x_t | x_0=C_0) / sum_n + # . . . + # . . . + # . . . + # p_0(x_0=C_{k-1} | x_t) / q(x_t | x_0=C_{k-1}) / sum_0 ... p_n(x_0=C_{k-1} | x_t) / q(x_t | x_0=C_{k-1}) / sum_n + q = q - q_log_sum_exp + + # (p_0(x_0=C_0 | x_t) / q(x_t | x_0=C_0) / sum_0) * a_cumulative_{t-1} + b_cumulative_{t-1} ... (p_n(x_0=C_0 | x_t) / q(x_t | x_0=C_0) / sum_n) * a_cumulative_{t-1} + b_cumulative_{t-1} + # . . . + # . . . + # . . . + # (p_0(x_0=C_{k-1} | x_t) / q(x_t | x_0=C_{k-1}) / sum_0) * a_cumulative_{t-1} + b_cumulative_{t-1} ... (p_n(x_0=C_{k-1} | x_t) / q(x_t | x_0=C_{k-1}) / sum_n) * a_cumulative_{t-1} + b_cumulative_{t-1} + # c_cumulative_{t-1} ... c_cumulative_{t-1} + q = self.apply_cumulative_transitions(q, t - 1) + + # ((p_0(x_0=C_0 | x_t) / q(x_t | x_0=C_0) / sum_0) * a_cumulative_{t-1} + b_cumulative_{t-1}) * q(x_t | x_{t-1}=C_0) * sum_0 ... ((p_n(x_0=C_0 | x_t) / q(x_t | x_0=C_0) / sum_n) * a_cumulative_{t-1} + b_cumulative_{t-1}) * q(x_t | x_{t-1}=C_0) * sum_n + # . . . + # . . . + # . . . + # ((p_0(x_0=C_{k-1} | x_t) / q(x_t | x_0=C_{k-1}) / sum_0) * a_cumulative_{t-1} + b_cumulative_{t-1}) * q(x_t | x_{t-1}=C_{k-1}) * sum_0 ... ((p_n(x_0=C_{k-1} | x_t) / q(x_t | x_0=C_{k-1}) / sum_n) * a_cumulative_{t-1} + b_cumulative_{t-1}) * q(x_t | x_{t-1}=C_{k-1}) * sum_n + # c_cumulative_{t-1} * q(x_t | x_{t-1}=C_k) * sum_0 ... c_cumulative_{t-1} * q(x_t | x_{t-1}=C_k) * sum_0 + log_p_x_t_min_1 = q + log_q_t_given_x_t_min_1 + q_log_sum_exp + + # For each column, there are two possible cases. + # + # Where: + # - sum(p_n(x_0))) is summing over all classes for x_0 + # - C_i is the class transitioning from (not to be confused with c_t and c_cumulative_t being used for gamma's) + # - C_j is the class transitioning to + # + # 1. x_t is masked i.e. x_t = c_k + # + # Simplifying the expression, the column vector is: + # . + # . + # . + # (c_t / c_cumulative_t) * (a_cumulative_{t-1} * p_n(x_0 = C_i | x_t) + b_cumulative_{t-1} * sum(p_n(x_0))) + # . + # . + # . + # (c_cumulative_{t-1} / c_cumulative_t) * sum(p_n(x_0)) + # + # From equation (11) stated in terms of forward probabilities, the last row is trivially verified. + # + # For the other rows, we can state the equation as ... + # + # (c_t / c_cumulative_t) * [b_cumulative_{t-1} * p(x_0=c_0) + ... + (a_cumulative_{t-1} + b_cumulative_{t-1}) * p(x_0=C_i) + ... + b_cumulative_{k-1} * p(x_0=c_{k-1})] + # + # This verifies the other rows. + # + # 2. x_t is not masked + # + # Simplifying the expression, there are two cases for the rows of the column vector, where C_j = C_i and where C_j != C_i: + # . + # . + # . + # C_j != C_i: b_t * ((b_cumulative_{t-1} / b_cumulative_t) * p_n(x_0 = c_0) + ... + ((a_cumulative_{t-1} + b_cumulative_{t-1}) / b_cumulative_t) * p_n(x_0 = C_i) + ... + (b_cumulative_{t-1} / (a_cumulative_t + b_cumulative_t)) * p_n(c_0=C_j) + ... + (b_cumulative_{t-1} / b_cumulative_t) * p_n(x_0 = c_{k-1})) + # . + # . + # . + # C_j = C_i: (a_t + b_t) * ((b_cumulative_{t-1} / b_cumulative_t) * p_n(x_0 = c_0) + ... + ((a_cumulative_{t-1} + b_cumulative_{t-1}) / (a_cumulative_t + b_cumulative_t)) * p_n(x_0 = C_i = C_j) + ... + (b_cumulative_{t-1} / b_cumulative_t) * p_n(x_0 = c_{k-1})) + # . + # . + # . + # 0 + # + # The last row is trivially verified. The other rows can be verified by directly expanding equation (11) stated in terms of forward probabilities. + return log_p_x_t_min_1 + + def log_Q_t_transitioning_to_known_class( + self, *, t: torch.int, x_t: torch.LongTensor, log_onehot_x_t: torch.FloatTensor, cumulative: bool + ): + """ + Calculates the log probabilities of the rows from the (cumulative or non-cumulative) transition matrix for each + latent pixel in `x_t`. + + Args: + t (`torch.Long`): + The timestep that determines which transition matrix is used. + x_t (`torch.LongTensor` of shape `(batch size, num latent pixels)`): + The classes of each latent pixel at time `t`. + log_onehot_x_t (`torch.FloatTensor` of shape `(batch size, num classes, num latent pixels)`): + The log one-hot vectors of `x_t`. + cumulative (`bool`): + If cumulative is `False`, the single step transition matrix `t-1`->`t` is used. If cumulative is + `True`, the cumulative transition matrix `0`->`t` is used. + + Returns: + `torch.FloatTensor` of shape `(batch size, num classes - 1, num latent pixels)`: + Each _column_ of the returned matrix is a _row_ of log probabilities of the complete probability + transition matrix. + + When non cumulative, returns `self.num_classes - 1` rows because the initial latent pixel cannot be + masked. + + Where: + - `q_n` is the probability distribution for the forward process of the `n`th latent pixel. + - C_0 is a class of a latent pixel embedding + - C_k is the class of the masked latent pixel + + non-cumulative result (omitting logarithms): + ``` + q_0(x_t | x_{t-1} = C_0) ... q_n(x_t | x_{t-1} = C_0) + . . . + . . . + . . . + q_0(x_t | x_{t-1} = C_k) ... q_n(x_t | x_{t-1} = C_k) + ``` + + cumulative result (omitting logarithms): + ``` + q_0_cumulative(x_t | x_0 = C_0) ... q_n_cumulative(x_t | x_0 = C_0) + . . . + . . . + . . . + q_0_cumulative(x_t | x_0 = C_{k-1}) ... q_n_cumulative(x_t | x_0 = C_{k-1}) + ``` + """ + if cumulative: + a = self.log_cumprod_at[t] + b = self.log_cumprod_bt[t] + c = self.log_cumprod_ct[t] + else: + a = self.log_at[t] + b = self.log_bt[t] + c = self.log_ct[t] + + if not cumulative: + # The values in the onehot vector can also be used as the logprobs for transitioning + # from masked latent pixels. If we are not calculating the cumulative transitions, + # we need to save these vectors to be re-appended to the final matrix so the values + # aren't overwritten. + # + # `P(x_t!=mask|x_{t-1=mask}) = 0` and 0 will be the value of the last row of the onehot vector + # if x_t is not masked + # + # `P(x_t=mask|x_{t-1=mask}) = 1` and 1 will be the value of the last row of the onehot vector + # if x_t is masked + log_onehot_x_t_transitioning_from_masked = log_onehot_x_t[:, -1, :].unsqueeze(1) + + # `index_to_log_onehot` will add onehot vectors for masked pixels, + # so the default one hot matrix has one too many rows. See the doc string + # for an explanation of the dimensionality of the returned matrix. + log_onehot_x_t = log_onehot_x_t[:, :-1, :] + + # this is a cheeky trick to produce the transition probabilities using log one-hot vectors. + # + # Don't worry about what values this sets in the columns that mark transitions + # to masked latent pixels. They are overwrote later with the `mask_class_mask`. + # + # Looking at the below logspace formula in non-logspace, each value will evaluate to either + # `1 * a + b = a + b` where `log_Q_t` has the one hot value in the column + # or + # `0 * a + b = b` where `log_Q_t` has the 0 values in the column. + # + # See equation 7 for more details. + log_Q_t = (log_onehot_x_t + a).logaddexp(b) + + # The whole column of each masked pixel is `c` + mask_class_mask = x_t == self.mask_class + mask_class_mask = mask_class_mask.unsqueeze(1).expand(-1, self.num_embed - 1, -1) + log_Q_t[mask_class_mask] = c + + if not cumulative: + log_Q_t = torch.cat((log_Q_t, log_onehot_x_t_transitioning_from_masked), dim=1) + + return log_Q_t + + def apply_cumulative_transitions(self, q, t): + bsz = q.shape[0] + a = self.log_cumprod_at[t] + b = self.log_cumprod_bt[t] + c = self.log_cumprod_ct[t] + + num_latent_pixels = q.shape[2] + c = c.expand(bsz, 1, num_latent_pixels) + + q = (q + a).logaddexp(b) + q = torch.cat((q, c), dim=1) + + return q diff --git a/diffusers-0.27.0/src/diffusers/training_utils.py b/diffusers-0.27.0/src/diffusers/training_utils.py new file mode 100755 index 0000000..25e02a3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/training_utils.py @@ -0,0 +1,453 @@ +import contextlib +import copy +import random +from typing import Any, Dict, Iterable, List, Optional, Union + +import numpy as np +import torch + +from .models import UNet2DConditionModel +from .utils import ( + convert_state_dict_to_diffusers, + convert_state_dict_to_peft, + deprecate, + is_peft_available, + is_torch_npu_available, + is_torchvision_available, + is_transformers_available, +) + + +if is_transformers_available(): + import transformers + +if is_peft_available(): + from peft import set_peft_model_state_dict + +if is_torchvision_available(): + from torchvision import transforms + +if is_torch_npu_available(): + import torch_npu # noqa: F401 + + +def set_seed(seed: int): + """ + Args: + Helper function for reproducible behavior to set the seed in `random`, `numpy`, `torch`. + seed (`int`): The seed to set. + """ + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + if is_torch_npu_available(): + torch.npu.manual_seed_all(seed) + else: + torch.cuda.manual_seed_all(seed) + # ^^ safe to call this function even if cuda is not available + + +def compute_snr(noise_scheduler, timesteps): + """ + Computes SNR as per + https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L847-L849 + """ + alphas_cumprod = noise_scheduler.alphas_cumprod + sqrt_alphas_cumprod = alphas_cumprod**0.5 + sqrt_one_minus_alphas_cumprod = (1.0 - alphas_cumprod) ** 0.5 + + # Expand the tensors. + # Adapted from https://github.com/TiankaiHang/Min-SNR-Diffusion-Training/blob/521b624bd70c67cee4bdf49225915f5945a872e3/guided_diffusion/gaussian_diffusion.py#L1026 + sqrt_alphas_cumprod = sqrt_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_alphas_cumprod = sqrt_alphas_cumprod[..., None] + alpha = sqrt_alphas_cumprod.expand(timesteps.shape) + + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod.to(device=timesteps.device)[timesteps].float() + while len(sqrt_one_minus_alphas_cumprod.shape) < len(timesteps.shape): + sqrt_one_minus_alphas_cumprod = sqrt_one_minus_alphas_cumprod[..., None] + sigma = sqrt_one_minus_alphas_cumprod.expand(timesteps.shape) + + # Compute SNR. + snr = (alpha / sigma) ** 2 + return snr + + +def resolve_interpolation_mode(interpolation_type: str): + """ + Maps a string describing an interpolation function to the corresponding torchvision `InterpolationMode` enum. The + full list of supported enums is documented at + https://pytorch.org/vision/0.9/transforms.html#torchvision.transforms.functional.InterpolationMode. + + Args: + interpolation_type (`str`): + A string describing an interpolation method. Currently, `bilinear`, `bicubic`, `box`, `nearest`, + `nearest_exact`, `hamming`, and `lanczos` are supported, corresponding to the supported interpolation modes + in torchvision. + + Returns: + `torchvision.transforms.InterpolationMode`: an `InterpolationMode` enum used by torchvision's `resize` + transform. + """ + if not is_torchvision_available(): + raise ImportError( + "Please make sure to install `torchvision` to be able to use the `resolve_interpolation_mode()` function." + ) + + if interpolation_type == "bilinear": + interpolation_mode = transforms.InterpolationMode.BILINEAR + elif interpolation_type == "bicubic": + interpolation_mode = transforms.InterpolationMode.BICUBIC + elif interpolation_type == "box": + interpolation_mode = transforms.InterpolationMode.BOX + elif interpolation_type == "nearest": + interpolation_mode = transforms.InterpolationMode.NEAREST + elif interpolation_type == "nearest_exact": + interpolation_mode = transforms.InterpolationMode.NEAREST_EXACT + elif interpolation_type == "hamming": + interpolation_mode = transforms.InterpolationMode.HAMMING + elif interpolation_type == "lanczos": + interpolation_mode = transforms.InterpolationMode.LANCZOS + else: + raise ValueError( + f"The given interpolation mode {interpolation_type} is not supported. Currently supported interpolation" + f" modes are `bilinear`, `bicubic`, `box`, `nearest`, `nearest_exact`, `hamming`, and `lanczos`." + ) + + return interpolation_mode + + +def unet_lora_state_dict(unet: UNet2DConditionModel) -> Dict[str, torch.Tensor]: + r""" + Returns: + A state dict containing just the LoRA parameters. + """ + lora_state_dict = {} + + for name, module in unet.named_modules(): + if hasattr(module, "set_lora_layer"): + lora_layer = getattr(module, "lora_layer") + if lora_layer is not None: + current_lora_layer_sd = lora_layer.state_dict() + for lora_layer_matrix_name, lora_param in current_lora_layer_sd.items(): + # The matrix name can either be "down" or "up". + lora_state_dict[f"{name}.lora.{lora_layer_matrix_name}"] = lora_param + + return lora_state_dict + + +def cast_training_params(model: Union[torch.nn.Module, List[torch.nn.Module]], dtype=torch.float32): + if not isinstance(model, list): + model = [model] + for m in model: + for param in m.parameters(): + # only upcast trainable parameters into fp32 + if param.requires_grad: + param.data = param.to(dtype) + + +def _set_state_dict_into_text_encoder( + lora_state_dict: Dict[str, torch.Tensor], prefix: str, text_encoder: torch.nn.Module +): + """ + Sets the `lora_state_dict` into `text_encoder` coming from `transformers`. + + Args: + lora_state_dict: The state dictionary to be set. + prefix: String identifier to retrieve the portion of the state dict that belongs to `text_encoder`. + text_encoder: Where the `lora_state_dict` is to be set. + """ + + text_encoder_state_dict = { + f'{k.replace(prefix, "")}': v for k, v in lora_state_dict.items() if k.startswith(prefix) + } + text_encoder_state_dict = convert_state_dict_to_peft(convert_state_dict_to_diffusers(text_encoder_state_dict)) + set_peft_model_state_dict(text_encoder, text_encoder_state_dict, adapter_name="default") + + +# Adapted from torch-ema https://github.com/fadel/pytorch_ema/blob/master/torch_ema/ema.py#L14 +class EMAModel: + """ + Exponential Moving Average of models weights + """ + + def __init__( + self, + parameters: Iterable[torch.nn.Parameter], + decay: float = 0.9999, + min_decay: float = 0.0, + update_after_step: int = 0, + use_ema_warmup: bool = False, + inv_gamma: Union[float, int] = 1.0, + power: Union[float, int] = 2 / 3, + model_cls: Optional[Any] = None, + model_config: Dict[str, Any] = None, + **kwargs, + ): + """ + Args: + parameters (Iterable[torch.nn.Parameter]): The parameters to track. + decay (float): The decay factor for the exponential moving average. + min_decay (float): The minimum decay factor for the exponential moving average. + update_after_step (int): The number of steps to wait before starting to update the EMA weights. + use_ema_warmup (bool): Whether to use EMA warmup. + inv_gamma (float): + Inverse multiplicative factor of EMA warmup. Default: 1. Only used if `use_ema_warmup` is True. + power (float): Exponential factor of EMA warmup. Default: 2/3. Only used if `use_ema_warmup` is True. + device (Optional[Union[str, torch.device]]): The device to store the EMA weights on. If None, the EMA + weights will be stored on CPU. + + @crowsonkb's notes on EMA Warmup: + If gamma=1 and power=1, implements a simple average. gamma=1, power=2/3 are good values for models you plan + to train for a million or more steps (reaches decay factor 0.999 at 31.6K steps, 0.9999 at 1M steps), + gamma=1, power=3/4 for models you plan to train for less (reaches decay factor 0.999 at 10K steps, 0.9999 + at 215.4k steps). + """ + + if isinstance(parameters, torch.nn.Module): + deprecation_message = ( + "Passing a `torch.nn.Module` to `ExponentialMovingAverage` is deprecated. " + "Please pass the parameters of the module instead." + ) + deprecate( + "passing a `torch.nn.Module` to `ExponentialMovingAverage`", + "1.0.0", + deprecation_message, + standard_warn=False, + ) + parameters = parameters.parameters() + + # set use_ema_warmup to True if a torch.nn.Module is passed for backwards compatibility + use_ema_warmup = True + + if kwargs.get("max_value", None) is not None: + deprecation_message = "The `max_value` argument is deprecated. Please use `decay` instead." + deprecate("max_value", "1.0.0", deprecation_message, standard_warn=False) + decay = kwargs["max_value"] + + if kwargs.get("min_value", None) is not None: + deprecation_message = "The `min_value` argument is deprecated. Please use `min_decay` instead." + deprecate("min_value", "1.0.0", deprecation_message, standard_warn=False) + min_decay = kwargs["min_value"] + + parameters = list(parameters) + self.shadow_params = [p.clone().detach() for p in parameters] + + if kwargs.get("device", None) is not None: + deprecation_message = "The `device` argument is deprecated. Please use `to` instead." + deprecate("device", "1.0.0", deprecation_message, standard_warn=False) + self.to(device=kwargs["device"]) + + self.temp_stored_params = None + + self.decay = decay + self.min_decay = min_decay + self.update_after_step = update_after_step + self.use_ema_warmup = use_ema_warmup + self.inv_gamma = inv_gamma + self.power = power + self.optimization_step = 0 + self.cur_decay_value = None # set in `step()` + + self.model_cls = model_cls + self.model_config = model_config + + @classmethod + def from_pretrained(cls, path, model_cls) -> "EMAModel": + _, ema_kwargs = model_cls.load_config(path, return_unused_kwargs=True) + model = model_cls.from_pretrained(path) + + ema_model = cls(model.parameters(), model_cls=model_cls, model_config=model.config) + + ema_model.load_state_dict(ema_kwargs) + return ema_model + + def save_pretrained(self, path): + if self.model_cls is None: + raise ValueError("`save_pretrained` can only be used if `model_cls` was defined at __init__.") + + if self.model_config is None: + raise ValueError("`save_pretrained` can only be used if `model_config` was defined at __init__.") + + model = self.model_cls.from_config(self.model_config) + state_dict = self.state_dict() + state_dict.pop("shadow_params", None) + + model.register_to_config(**state_dict) + self.copy_to(model.parameters()) + model.save_pretrained(path) + + def get_decay(self, optimization_step: int) -> float: + """ + Compute the decay factor for the exponential moving average. + """ + step = max(0, optimization_step - self.update_after_step - 1) + + if step <= 0: + return 0.0 + + if self.use_ema_warmup: + cur_decay_value = 1 - (1 + step / self.inv_gamma) ** -self.power + else: + cur_decay_value = (1 + step) / (10 + step) + + cur_decay_value = min(cur_decay_value, self.decay) + # make sure decay is not smaller than min_decay + cur_decay_value = max(cur_decay_value, self.min_decay) + return cur_decay_value + + @torch.no_grad() + def step(self, parameters: Iterable[torch.nn.Parameter]): + if isinstance(parameters, torch.nn.Module): + deprecation_message = ( + "Passing a `torch.nn.Module` to `ExponentialMovingAverage.step` is deprecated. " + "Please pass the parameters of the module instead." + ) + deprecate( + "passing a `torch.nn.Module` to `ExponentialMovingAverage.step`", + "1.0.0", + deprecation_message, + standard_warn=False, + ) + parameters = parameters.parameters() + + parameters = list(parameters) + + self.optimization_step += 1 + + # Compute the decay factor for the exponential moving average. + decay = self.get_decay(self.optimization_step) + self.cur_decay_value = decay + one_minus_decay = 1 - decay + + context_manager = contextlib.nullcontext + if is_transformers_available() and transformers.deepspeed.is_deepspeed_zero3_enabled(): + import deepspeed + + for s_param, param in zip(self.shadow_params, parameters): + if is_transformers_available() and transformers.deepspeed.is_deepspeed_zero3_enabled(): + context_manager = deepspeed.zero.GatheredParameters(param, modifier_rank=None) + + with context_manager(): + if param.requires_grad: + s_param.sub_(one_minus_decay * (s_param - param)) + else: + s_param.copy_(param) + + def copy_to(self, parameters: Iterable[torch.nn.Parameter]) -> None: + """ + Copy current averaged parameters into given collection of parameters. + + Args: + parameters: Iterable of `torch.nn.Parameter`; the parameters to be + updated with the stored moving averages. If `None`, the parameters with which this + `ExponentialMovingAverage` was initialized will be used. + """ + parameters = list(parameters) + for s_param, param in zip(self.shadow_params, parameters): + param.data.copy_(s_param.to(param.device).data) + + def to(self, device=None, dtype=None) -> None: + r"""Move internal buffers of the ExponentialMovingAverage to `device`. + + Args: + device: like `device` argument to `torch.Tensor.to` + """ + # .to() on the tensors handles None correctly + self.shadow_params = [ + p.to(device=device, dtype=dtype) if p.is_floating_point() else p.to(device=device) + for p in self.shadow_params + ] + + def state_dict(self) -> dict: + r""" + Returns the state of the ExponentialMovingAverage as a dict. This method is used by accelerate during + checkpointing to save the ema state dict. + """ + # Following PyTorch conventions, references to tensors are returned: + # "returns a reference to the state and not its copy!" - + # https://pytorch.org/tutorials/beginner/saving_loading_models.html#what-is-a-state-dict + return { + "decay": self.decay, + "min_decay": self.min_decay, + "optimization_step": self.optimization_step, + "update_after_step": self.update_after_step, + "use_ema_warmup": self.use_ema_warmup, + "inv_gamma": self.inv_gamma, + "power": self.power, + "shadow_params": self.shadow_params, + } + + def store(self, parameters: Iterable[torch.nn.Parameter]) -> None: + r""" + Args: + Save the current parameters for restoring later. + parameters: Iterable of `torch.nn.Parameter`; the parameters to be + temporarily stored. + """ + self.temp_stored_params = [param.detach().cpu().clone() for param in parameters] + + def restore(self, parameters: Iterable[torch.nn.Parameter]) -> None: + r""" + Args: + Restore the parameters stored with the `store` method. Useful to validate the model with EMA parameters without: + affecting the original optimization process. Store the parameters before the `copy_to()` method. After + validation (or model saving), use this to restore the former parameters. + parameters: Iterable of `torch.nn.Parameter`; the parameters to be + updated with the stored parameters. If `None`, the parameters with which this + `ExponentialMovingAverage` was initialized will be used. + """ + if self.temp_stored_params is None: + raise RuntimeError("This ExponentialMovingAverage has no `store()`ed weights " "to `restore()`") + for c_param, param in zip(self.temp_stored_params, parameters): + param.data.copy_(c_param.data) + + # Better memory-wise. + self.temp_stored_params = None + + def load_state_dict(self, state_dict: dict) -> None: + r""" + Args: + Loads the ExponentialMovingAverage state. This method is used by accelerate during checkpointing to save the + ema state dict. + state_dict (dict): EMA state. Should be an object returned + from a call to :meth:`state_dict`. + """ + # deepcopy, to be consistent with module API + state_dict = copy.deepcopy(state_dict) + + self.decay = state_dict.get("decay", self.decay) + if self.decay < 0.0 or self.decay > 1.0: + raise ValueError("Decay must be between 0 and 1") + + self.min_decay = state_dict.get("min_decay", self.min_decay) + if not isinstance(self.min_decay, float): + raise ValueError("Invalid min_decay") + + self.optimization_step = state_dict.get("optimization_step", self.optimization_step) + if not isinstance(self.optimization_step, int): + raise ValueError("Invalid optimization_step") + + self.update_after_step = state_dict.get("update_after_step", self.update_after_step) + if not isinstance(self.update_after_step, int): + raise ValueError("Invalid update_after_step") + + self.use_ema_warmup = state_dict.get("use_ema_warmup", self.use_ema_warmup) + if not isinstance(self.use_ema_warmup, bool): + raise ValueError("Invalid use_ema_warmup") + + self.inv_gamma = state_dict.get("inv_gamma", self.inv_gamma) + if not isinstance(self.inv_gamma, (float, int)): + raise ValueError("Invalid inv_gamma") + + self.power = state_dict.get("power", self.power) + if not isinstance(self.power, (float, int)): + raise ValueError("Invalid power") + + shadow_params = state_dict.get("shadow_params", None) + if shadow_params is not None: + self.shadow_params = shadow_params + if not isinstance(self.shadow_params, list): + raise ValueError("shadow_params must be a list") + if not all(isinstance(p, torch.Tensor) for p in self.shadow_params): + raise ValueError("shadow_params must all be Tensors") diff --git a/diffusers-0.27.0/src/diffusers/utils/__init__.py b/diffusers-0.27.0/src/diffusers/utils/__init__.py new file mode 100755 index 0000000..4e2f07f --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/__init__.py @@ -0,0 +1,124 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os + +from packaging import version + +from .. import __version__ +from .constants import ( + CONFIG_NAME, + DEPRECATED_REVISION_ARGS, + DIFFUSERS_DYNAMIC_MODULE_NAME, + FLAX_WEIGHTS_NAME, + HF_MODULES_CACHE, + HUGGINGFACE_CO_RESOLVE_ENDPOINT, + MIN_PEFT_VERSION, + ONNX_EXTERNAL_WEIGHTS_NAME, + ONNX_WEIGHTS_NAME, + SAFETENSORS_FILE_EXTENSION, + SAFETENSORS_WEIGHTS_NAME, + USE_PEFT_BACKEND, + WEIGHTS_NAME, +) +from .deprecation_utils import deprecate +from .doc_utils import replace_example_docstring +from .dynamic_modules_utils import get_class_from_dynamic_module +from .export_utils import export_to_gif, export_to_obj, export_to_ply, export_to_video +from .hub_utils import ( + PushToHubMixin, + _add_variant, + _get_model_file, + extract_commit_hash, + http_user_agent, +) +from .import_utils import ( + BACKENDS_MAPPING, + DIFFUSERS_SLOW_IMPORT, + ENV_VARS_TRUE_AND_AUTO_VALUES, + ENV_VARS_TRUE_VALUES, + USE_JAX, + USE_TF, + USE_TORCH, + DummyObject, + OptionalDependencyNotAvailable, + _LazyModule, + get_objects_from_module, + is_accelerate_available, + is_accelerate_version, + is_bs4_available, + is_flax_available, + is_ftfy_available, + is_inflect_available, + is_invisible_watermark_available, + is_k_diffusion_available, + is_k_diffusion_version, + is_librosa_available, + is_note_seq_available, + is_onnx_available, + is_peft_available, + is_scipy_available, + is_tensorboard_available, + is_torch_available, + is_torch_npu_available, + is_torch_version, + is_torch_xla_available, + is_torchsde_available, + is_torchvision_available, + is_transformers_available, + is_transformers_version, + is_unidecode_available, + is_wandb_available, + is_xformers_available, + requires_backends, +) +from .loading_utils import load_image +from .logging import get_logger +from .outputs import BaseOutput +from .peft_utils import ( + check_peft_version, + delete_adapter_layers, + get_adapter_name, + get_peft_kwargs, + recurse_remove_peft_layers, + scale_lora_layers, + set_adapter_layers, + set_weights_and_activate_adapters, + unscale_lora_layers, +) +from .pil_utils import PIL_INTERPOLATION, make_image_grid, numpy_to_pil, pt_to_pil +from .state_dict_utils import ( + convert_all_state_dict_to_peft, + convert_state_dict_to_diffusers, + convert_state_dict_to_kohya, + convert_state_dict_to_peft, + convert_unet_state_dict_to_peft, +) + + +logger = get_logger(__name__) + + +def check_min_version(min_version): + if version.parse(__version__) < version.parse(min_version): + if "dev" in min_version: + error_message = ( + "This example requires a source install from HuggingFace diffusers (see " + "`https://huggingface.co/docs/diffusers/installation#install-from-source`)," + ) + else: + error_message = f"This example requires a minimum version of {min_version}," + error_message += f" but the version found is {__version__}.\n" + raise ImportError(error_message) diff --git a/diffusers-0.27.0/src/diffusers/utils/accelerate_utils.py b/diffusers-0.27.0/src/diffusers/utils/accelerate_utils.py new file mode 100755 index 0000000..99a8b3a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/accelerate_utils.py @@ -0,0 +1,48 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Accelerate utilities: Utilities related to accelerate +""" + +from packaging import version + +from .import_utils import is_accelerate_available + + +if is_accelerate_available(): + import accelerate + + +def apply_forward_hook(method): + """ + Decorator that applies a registered CpuOffload hook to an arbitrary function rather than `forward`. This is useful + for cases where a PyTorch module provides functions other than `forward` that should trigger a move to the + appropriate acceleration device. This is the case for `encode` and `decode` in [`AutoencoderKL`]. + + This decorator looks inside the internal `_hf_hook` property to find a registered offload hook. + + :param method: The method to decorate. This method should be a method of a PyTorch module. + """ + if not is_accelerate_available(): + return method + accelerate_version = version.parse(accelerate.__version__).base_version + if version.parse(accelerate_version) < version.parse("0.17.0"): + return method + + def wrapper(self, *args, **kwargs): + if hasattr(self, "_hf_hook") and hasattr(self._hf_hook, "pre_forward"): + self._hf_hook.pre_forward(self) + return method(self, *args, **kwargs) + + return wrapper diff --git a/diffusers-0.27.0/src/diffusers/utils/constants.py b/diffusers-0.27.0/src/diffusers/utils/constants.py new file mode 100755 index 0000000..bc4268a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/constants.py @@ -0,0 +1,55 @@ +# Copyright 2024 The HuggingFace Inc. team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import importlib +import os + +from huggingface_hub.constants import HF_HOME +from packaging import version + +from ..dependency_versions_check import dep_version_check +from .import_utils import ENV_VARS_TRUE_VALUES, is_peft_available, is_transformers_available + + +MIN_PEFT_VERSION = "0.6.0" +MIN_TRANSFORMERS_VERSION = "4.34.0" +_CHECK_PEFT = os.environ.get("_CHECK_PEFT", "1") in ENV_VARS_TRUE_VALUES + + +CONFIG_NAME = "config.json" +WEIGHTS_NAME = "diffusion_pytorch_model.bin" +FLAX_WEIGHTS_NAME = "diffusion_flax_model.msgpack" +ONNX_WEIGHTS_NAME = "model.onnx" +SAFETENSORS_WEIGHTS_NAME = "diffusion_pytorch_model.safetensors" +SAFETENSORS_FILE_EXTENSION = "safetensors" +ONNX_EXTERNAL_WEIGHTS_NAME = "weights.pb" +HUGGINGFACE_CO_RESOLVE_ENDPOINT = os.environ.get("HF_ENDPOINT", "https://huggingface.co") +DIFFUSERS_DYNAMIC_MODULE_NAME = "diffusers_modules" +HF_MODULES_CACHE = os.getenv("HF_MODULES_CACHE", os.path.join(HF_HOME, "modules")) +DEPRECATED_REVISION_ARGS = ["fp16", "non-ema"] + +# Below should be `True` if the current version of `peft` and `transformers` are compatible with +# PEFT backend. Will automatically fall back to PEFT backend if the correct versions of the libraries are +# available. +# For PEFT it is has to be greater than or equal to 0.6.0 and for transformers it has to be greater than or equal to 4.34.0. +_required_peft_version = is_peft_available() and version.parse( + version.parse(importlib.metadata.version("peft")).base_version +) >= version.parse(MIN_PEFT_VERSION) +_required_transformers_version = is_transformers_available() and version.parse( + version.parse(importlib.metadata.version("transformers")).base_version +) >= version.parse(MIN_TRANSFORMERS_VERSION) + +USE_PEFT_BACKEND = _required_peft_version and _required_transformers_version + +if USE_PEFT_BACKEND and _CHECK_PEFT: + dep_version_check("peft") diff --git a/diffusers-0.27.0/src/diffusers/utils/deprecation_utils.py b/diffusers-0.27.0/src/diffusers/utils/deprecation_utils.py new file mode 100755 index 0000000..f482ded --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/deprecation_utils.py @@ -0,0 +1,49 @@ +import inspect +import warnings +from typing import Any, Dict, Optional, Union + +from packaging import version + + +def deprecate(*args, take_from: Optional[Union[Dict, Any]] = None, standard_warn=True, stacklevel=2): + from .. import __version__ + + deprecated_kwargs = take_from + values = () + if not isinstance(args[0], tuple): + args = (args,) + + for attribute, version_name, message in args: + if version.parse(version.parse(__version__).base_version) >= version.parse(version_name): + raise ValueError( + f"The deprecation tuple {(attribute, version_name, message)} should be removed since diffusers'" + f" version {__version__} is >= {version_name}" + ) + + warning = None + if isinstance(deprecated_kwargs, dict) and attribute in deprecated_kwargs: + values += (deprecated_kwargs.pop(attribute),) + warning = f"The `{attribute}` argument is deprecated and will be removed in version {version_name}." + elif hasattr(deprecated_kwargs, attribute): + values += (getattr(deprecated_kwargs, attribute),) + warning = f"The `{attribute}` attribute is deprecated and will be removed in version {version_name}." + elif deprecated_kwargs is None: + warning = f"`{attribute}` is deprecated and will be removed in version {version_name}." + + if warning is not None: + warning = warning + " " if standard_warn else "" + warnings.warn(warning + message, FutureWarning, stacklevel=stacklevel) + + if isinstance(deprecated_kwargs, dict) and len(deprecated_kwargs) > 0: + call_frame = inspect.getouterframes(inspect.currentframe())[1] + filename = call_frame.filename + line_number = call_frame.lineno + function = call_frame.function + key, value = next(iter(deprecated_kwargs.items())) + raise TypeError(f"{function} in {filename} line {line_number-1} got an unexpected keyword argument `{key}`") + + if len(values) == 0: + return + elif len(values) == 1: + return values[0] + return values diff --git a/diffusers-0.27.0/src/diffusers/utils/doc_utils.py b/diffusers-0.27.0/src/diffusers/utils/doc_utils.py new file mode 100755 index 0000000..03b6b7a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/doc_utils.py @@ -0,0 +1,38 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Doc utilities: Utilities related to documentation +""" +import re + + +def replace_example_docstring(example_docstring): + def docstring_decorator(fn): + func_doc = fn.__doc__ + lines = func_doc.split("\n") + i = 0 + while i < len(lines) and re.search(r"^\s*Examples?:\s*$", lines[i]) is None: + i += 1 + if i < len(lines): + lines[i] = example_docstring + func_doc = "\n".join(lines) + else: + raise ValueError( + f"The function {fn} should have an empty 'Examples:' in its docstring as placeholder, " + f"current docstring is:\n{func_doc}" + ) + fn.__doc__ = func_doc + return fn + + return docstring_decorator diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_flax_and_transformers_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_flax_and_transformers_objects.py new file mode 100755 index 0000000..5e65e53 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_flax_and_transformers_objects.py @@ -0,0 +1,77 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class FlaxStableDiffusionControlNetPipeline(metaclass=DummyObject): + _backends = ["flax", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax", "transformers"]) + + +class FlaxStableDiffusionImg2ImgPipeline(metaclass=DummyObject): + _backends = ["flax", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax", "transformers"]) + + +class FlaxStableDiffusionInpaintPipeline(metaclass=DummyObject): + _backends = ["flax", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax", "transformers"]) + + +class FlaxStableDiffusionPipeline(metaclass=DummyObject): + _backends = ["flax", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax", "transformers"]) + + +class FlaxStableDiffusionXLPipeline(metaclass=DummyObject): + _backends = ["flax", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax", "transformers"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_flax_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_flax_objects.py new file mode 100755 index 0000000..5fa8dbc --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_flax_objects.py @@ -0,0 +1,212 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class FlaxControlNetModel(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxModelMixin(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxUNet2DConditionModel(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxAutoencoderKL(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxDiffusionPipeline(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxDDIMScheduler(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxDDPMScheduler(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxDPMSolverMultistepScheduler(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxEulerDiscreteScheduler(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxKarrasVeScheduler(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxLMSDiscreteScheduler(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxPNDMScheduler(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxSchedulerMixin(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + +class FlaxScoreSdeVeScheduler(metaclass=DummyObject): + _backends = ["flax"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["flax"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["flax"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_note_seq_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_note_seq_objects.py new file mode 100755 index 0000000..c02d0b0 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_note_seq_objects.py @@ -0,0 +1,17 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class MidiProcessor(metaclass=DummyObject): + _backends = ["note_seq"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["note_seq"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["note_seq"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["note_seq"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_onnx_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_onnx_objects.py new file mode 100755 index 0000000..bde5f6a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_onnx_objects.py @@ -0,0 +1,17 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class OnnxRuntimeModel(metaclass=DummyObject): + _backends = ["onnx"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["onnx"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["onnx"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["onnx"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_pt_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_pt_objects.py new file mode 100755 index 0000000..1494784 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_pt_objects.py @@ -0,0 +1,1170 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class AsymmetricAutoencoderKL(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class AutoencoderKL(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class AutoencoderKLTemporalDecoder(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class AutoencoderTiny(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class ConsistencyDecoderVAE(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class ControlNetModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class I2VGenXLUNet(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class Kandinsky3UNet(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class ModelMixin(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class MotionAdapter(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class MultiAdapter(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class PriorTransformer(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class T2IAdapter(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class T5FilmDecoder(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class Transformer2DModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class UNet1DModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class UNet2DConditionModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class UNet2DModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class UNet3DConditionModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class UNetMotionModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class UNetSpatioTemporalConditionModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class UVit2DModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class VQModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +def get_constant_schedule(*args, **kwargs): + requires_backends(get_constant_schedule, ["torch"]) + + +def get_constant_schedule_with_warmup(*args, **kwargs): + requires_backends(get_constant_schedule_with_warmup, ["torch"]) + + +def get_cosine_schedule_with_warmup(*args, **kwargs): + requires_backends(get_cosine_schedule_with_warmup, ["torch"]) + + +def get_cosine_with_hard_restarts_schedule_with_warmup(*args, **kwargs): + requires_backends(get_cosine_with_hard_restarts_schedule_with_warmup, ["torch"]) + + +def get_linear_schedule_with_warmup(*args, **kwargs): + requires_backends(get_linear_schedule_with_warmup, ["torch"]) + + +def get_polynomial_decay_schedule_with_warmup(*args, **kwargs): + requires_backends(get_polynomial_decay_schedule_with_warmup, ["torch"]) + + +def get_scheduler(*args, **kwargs): + requires_backends(get_scheduler, ["torch"]) + + +class AudioPipelineOutput(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class AutoPipelineForImage2Image(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class AutoPipelineForInpainting(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class AutoPipelineForText2Image(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class BlipDiffusionControlNetPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class BlipDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class CLIPImageProjection(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class ConsistencyModelPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DanceDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DDIMPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DDPMPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DiffusionPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DiTPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class ImagePipelineOutput(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class KarrasVePipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class LDMPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class LDMSuperResolutionPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class PNDMPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class RePaintPipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class ScoreSdeVePipeline(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class StableDiffusionMixin(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class AmusedScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class CMStochasticIterativeScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DDIMInverseScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DDIMParallelScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DDIMScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DDPMParallelScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DDPMScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DDPMWuerstchenScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DEISMultistepScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DPMSolverMultistepInverseScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DPMSolverMultistepScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class DPMSolverSinglestepScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class EDMDPMSolverMultistepScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class EDMEulerScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class EulerAncestralDiscreteScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class EulerDiscreteScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class HeunDiscreteScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class IPNDMScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class KarrasVeScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class KDPM2AncestralDiscreteScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class KDPM2DiscreteScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class LCMScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class PNDMScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class RePaintScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class SASolverScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class SchedulerMixin(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class ScoreSdeVeScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class TCDScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class UnCLIPScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class UniPCMultistepScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class VQDiffusionScheduler(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + +class EMAModel(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_librosa_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_librosa_objects.py new file mode 100755 index 0000000..2088bc4 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_librosa_objects.py @@ -0,0 +1,32 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class AudioDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch", "librosa"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "librosa"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "librosa"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "librosa"]) + + +class Mel(metaclass=DummyObject): + _backends = ["torch", "librosa"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "librosa"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "librosa"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "librosa"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_scipy_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_scipy_objects.py new file mode 100755 index 0000000..a1ff258 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_scipy_objects.py @@ -0,0 +1,17 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class LMSDiscreteScheduler(metaclass=DummyObject): + _backends = ["torch", "scipy"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "scipy"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "scipy"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "scipy"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_torchsde_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_torchsde_objects.py new file mode 100755 index 0000000..a81bbb3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_torchsde_objects.py @@ -0,0 +1,17 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class DPMSolverSDEScheduler(metaclass=DummyObject): + _backends = ["torch", "torchsde"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "torchsde"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "torchsde"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "torchsde"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_and_k_diffusion_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_and_k_diffusion_objects.py new file mode 100755 index 0000000..2ab00c5 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_and_k_diffusion_objects.py @@ -0,0 +1,32 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class StableDiffusionKDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers", "k_diffusion"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers", "k_diffusion"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "k_diffusion"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "k_diffusion"]) + + +class StableDiffusionXLKDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers", "k_diffusion"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers", "k_diffusion"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "k_diffusion"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "k_diffusion"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_and_onnx_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_and_onnx_objects.py new file mode 100755 index 0000000..b7afad8 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_and_onnx_objects.py @@ -0,0 +1,92 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class OnnxStableDiffusionImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers", "onnx"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers", "onnx"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) + + +class OnnxStableDiffusionInpaintPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers", "onnx"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers", "onnx"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) + + +class OnnxStableDiffusionInpaintPipelineLegacy(metaclass=DummyObject): + _backends = ["torch", "transformers", "onnx"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers", "onnx"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) + + +class OnnxStableDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers", "onnx"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers", "onnx"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) + + +class OnnxStableDiffusionUpscalePipeline(metaclass=DummyObject): + _backends = ["torch", "transformers", "onnx"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers", "onnx"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) + + +class StableDiffusionOnnxPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers", "onnx"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers", "onnx"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers", "onnx"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_objects.py new file mode 100755 index 0000000..f64c157 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_torch_and_transformers_objects.py @@ -0,0 +1,1607 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class AltDiffusionImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class AltDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class AmusedImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class AmusedInpaintPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class AmusedPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class AnimateDiffPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class AnimateDiffVideoToVideoPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class AudioLDM2Pipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class AudioLDM2ProjectionModel(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class AudioLDM2UNet2DConditionModel(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class AudioLDMPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class CLIPImageProjection(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class CycleDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class I2VGenXLPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class IFImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class IFImg2ImgSuperResolutionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class IFInpaintingPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class IFInpaintingSuperResolutionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class IFPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class IFSuperResolutionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class ImageTextPipelineOutput(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class Kandinsky3Img2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class Kandinsky3Pipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyCombinedPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyImg2ImgCombinedPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyInpaintCombinedPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyInpaintPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyPriorPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyV22CombinedPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyV22ControlnetImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyV22ControlnetPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyV22Img2ImgCombinedPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyV22Img2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyV22InpaintCombinedPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyV22InpaintPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyV22Pipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyV22PriorEmb2EmbPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class KandinskyV22PriorPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class LatentConsistencyModelImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class LatentConsistencyModelPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class LDMTextToImagePipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class LEditsPPPipelineStableDiffusion(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class LEditsPPPipelineStableDiffusionXL(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class MusicLDMPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class PaintByExamplePipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class PIAPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class PixArtAlphaPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class SemanticStableDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class ShapEImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class ShapEPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableCascadeCombinedPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableCascadeDecoderPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableCascadePriorPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionAdapterPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionAttendAndExcitePipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionControlNetImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionControlNetInpaintPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionControlNetPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionDepth2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionDiffEditPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionGLIGENPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionGLIGENTextImagePipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionImageVariationPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionInpaintPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionInpaintPipelineLegacy(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionInstructPix2PixPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionLatentUpscalePipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionLDM3DPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionModelEditingPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionPanoramaPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionParadigmsPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionPipelineSafe(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionPix2PixZeroPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionSAGPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionUpscalePipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionXLAdapterPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionXLControlNetImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionXLControlNetInpaintPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionXLControlNetPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionXLImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionXLInpaintPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionXLInstructPix2PixPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableDiffusionXLPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableUnCLIPImg2ImgPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableUnCLIPPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class StableVideoDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class TextToVideoSDPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class TextToVideoZeroPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class TextToVideoZeroSDXLPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class UnCLIPImageVariationPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class UnCLIPPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class UniDiffuserModel(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class UniDiffuserPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class UniDiffuserTextDecoder(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class VersatileDiffusionDualGuidedPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class VersatileDiffusionImageVariationPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class VersatileDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class VersatileDiffusionTextToImagePipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class VideoToVideoSDPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class VQDiffusionPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class WuerstchenCombinedPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class WuerstchenDecoderPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + +class WuerstchenPriorPipeline(metaclass=DummyObject): + _backends = ["torch", "transformers"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch", "transformers"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch", "transformers"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dummy_transformers_and_torch_and_note_seq_objects.py b/diffusers-0.27.0/src/diffusers/utils/dummy_transformers_and_torch_and_note_seq_objects.py new file mode 100755 index 0000000..fbde04e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dummy_transformers_and_torch_and_note_seq_objects.py @@ -0,0 +1,17 @@ +# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +class SpectrogramDiffusionPipeline(metaclass=DummyObject): + _backends = ["transformers", "torch", "note_seq"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["transformers", "torch", "note_seq"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["transformers", "torch", "note_seq"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["transformers", "torch", "note_seq"]) diff --git a/diffusers-0.27.0/src/diffusers/utils/dynamic_modules_utils.py b/diffusers-0.27.0/src/diffusers/utils/dynamic_modules_utils.py new file mode 100755 index 0000000..a4c704a --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/dynamic_modules_utils.py @@ -0,0 +1,452 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Utilities to dynamically load objects from the Hub.""" + +import importlib +import inspect +import json +import os +import re +import shutil +import sys +from pathlib import Path +from typing import Dict, Optional, Union +from urllib import request + +from huggingface_hub import cached_download, hf_hub_download, model_info +from huggingface_hub.utils import validate_hf_hub_args +from packaging import version + +from .. import __version__ +from . import DIFFUSERS_DYNAMIC_MODULE_NAME, HF_MODULES_CACHE, logging + + +COMMUNITY_PIPELINES_URL = ( + "https://raw.githubusercontent.com/huggingface/diffusers/{revision}/examples/community/{pipeline}.py" +) + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +def get_diffusers_versions(): + url = "https://pypi.org/pypi/diffusers/json" + releases = json.loads(request.urlopen(url).read())["releases"].keys() + return sorted(releases, key=lambda x: version.Version(x)) + + +def init_hf_modules(): + """ + Creates the cache directory for modules with an init, and adds it to the Python path. + """ + # This function has already been executed if HF_MODULES_CACHE already is in the Python path. + if HF_MODULES_CACHE in sys.path: + return + + sys.path.append(HF_MODULES_CACHE) + os.makedirs(HF_MODULES_CACHE, exist_ok=True) + init_path = Path(HF_MODULES_CACHE) / "__init__.py" + if not init_path.exists(): + init_path.touch() + + +def create_dynamic_module(name: Union[str, os.PathLike]): + """ + Creates a dynamic module in the cache directory for modules. + """ + init_hf_modules() + dynamic_module_path = Path(HF_MODULES_CACHE) / name + # If the parent module does not exist yet, recursively create it. + if not dynamic_module_path.parent.exists(): + create_dynamic_module(dynamic_module_path.parent) + os.makedirs(dynamic_module_path, exist_ok=True) + init_path = dynamic_module_path / "__init__.py" + if not init_path.exists(): + init_path.touch() + + +def get_relative_imports(module_file): + """ + Get the list of modules that are relatively imported in a module file. + + Args: + module_file (`str` or `os.PathLike`): The module file to inspect. + """ + with open(module_file, "r", encoding="utf-8") as f: + content = f.read() + + # Imports of the form `import .xxx` + relative_imports = re.findall(r"^\s*import\s+\.(\S+)\s*$", content, flags=re.MULTILINE) + # Imports of the form `from .xxx import yyy` + relative_imports += re.findall(r"^\s*from\s+\.(\S+)\s+import", content, flags=re.MULTILINE) + # Unique-ify + return list(set(relative_imports)) + + +def get_relative_import_files(module_file): + """ + Get the list of all files that are needed for a given module. Note that this function recurses through the relative + imports (if a imports b and b imports c, it will return module files for b and c). + + Args: + module_file (`str` or `os.PathLike`): The module file to inspect. + """ + no_change = False + files_to_check = [module_file] + all_relative_imports = [] + + # Let's recurse through all relative imports + while not no_change: + new_imports = [] + for f in files_to_check: + new_imports.extend(get_relative_imports(f)) + + module_path = Path(module_file).parent + new_import_files = [str(module_path / m) for m in new_imports] + new_import_files = [f for f in new_import_files if f not in all_relative_imports] + files_to_check = [f"{f}.py" for f in new_import_files] + + no_change = len(new_import_files) == 0 + all_relative_imports.extend(files_to_check) + + return all_relative_imports + + +def check_imports(filename): + """ + Check if the current Python environment contains all the libraries that are imported in a file. + """ + with open(filename, "r", encoding="utf-8") as f: + content = f.read() + + # Imports of the form `import xxx` + imports = re.findall(r"^\s*import\s+(\S+)\s*$", content, flags=re.MULTILINE) + # Imports of the form `from xxx import yyy` + imports += re.findall(r"^\s*from\s+(\S+)\s+import", content, flags=re.MULTILINE) + # Only keep the top-level module + imports = [imp.split(".")[0] for imp in imports if not imp.startswith(".")] + + # Unique-ify and test we got them all + imports = list(set(imports)) + missing_packages = [] + for imp in imports: + try: + importlib.import_module(imp) + except ImportError: + missing_packages.append(imp) + + if len(missing_packages) > 0: + raise ImportError( + "This modeling file requires the following packages that were not found in your environment: " + f"{', '.join(missing_packages)}. Run `pip install {' '.join(missing_packages)}`" + ) + + return get_relative_imports(filename) + + +def get_class_in_module(class_name, module_path): + """ + Import a module on the cache directory for modules and extract a class from it. + """ + module_path = module_path.replace(os.path.sep, ".") + module = importlib.import_module(module_path) + + if class_name is None: + return find_pipeline_class(module) + return getattr(module, class_name) + + +def find_pipeline_class(loaded_module): + """ + Retrieve pipeline class that inherits from `DiffusionPipeline`. Note that there has to be exactly one class + inheriting from `DiffusionPipeline`. + """ + from ..pipelines import DiffusionPipeline + + cls_members = dict(inspect.getmembers(loaded_module, inspect.isclass)) + + pipeline_class = None + for cls_name, cls in cls_members.items(): + if ( + cls_name != DiffusionPipeline.__name__ + and issubclass(cls, DiffusionPipeline) + and cls.__module__.split(".")[0] != "diffusers" + ): + if pipeline_class is not None: + raise ValueError( + f"Multiple classes that inherit from {DiffusionPipeline.__name__} have been found:" + f" {pipeline_class.__name__}, and {cls_name}. Please make sure to define only one in" + f" {loaded_module}." + ) + pipeline_class = cls + + return pipeline_class + + +@validate_hf_hub_args +def get_cached_module_file( + pretrained_model_name_or_path: Union[str, os.PathLike], + module_file: str, + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + resume_download: bool = False, + proxies: Optional[Dict[str, str]] = None, + token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + local_files_only: bool = False, +): + """ + Prepares Downloads a module from a local folder or a distant repo and returns its path inside the cached + Transformers module. + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a pretrained model configuration hosted inside a model repo on + huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced + under a user or organization name, like `dbmdz/bert-base-german-cased`. + - a path to a *directory* containing a configuration file saved using the + [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. + + module_file (`str`): + The name of the module file containing the class to look for. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the standard + cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the configuration files and override the cached versions if they + exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to delete incompletely received file. Attempts to resume the download if such a file exists. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or *bool*, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `transformers-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, will only try to load the tokenizer configuration from local files. + + + + You may pass a token in `token` if you are not logged in (`huggingface-cli login`) and want to use private + or [gated models](https://huggingface.co/docs/hub/models-gated#gated-models). + + + + Returns: + `str`: The path to the module inside the cache. + """ + # Download and cache module_file from the repo `pretrained_model_name_or_path` of grab it if it's a local file. + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + + module_file_or_url = os.path.join(pretrained_model_name_or_path, module_file) + + if os.path.isfile(module_file_or_url): + resolved_module_file = module_file_or_url + submodule = "local" + elif pretrained_model_name_or_path.count("/") == 0: + available_versions = get_diffusers_versions() + # cut ".dev0" + latest_version = "v" + ".".join(__version__.split(".")[:3]) + + # retrieve github version that matches + if revision is None: + revision = latest_version if latest_version[1:] in available_versions else "main" + logger.info(f"Defaulting to latest_version: {revision}.") + elif revision in available_versions: + revision = f"v{revision}" + elif revision == "main": + revision = revision + else: + raise ValueError( + f"`custom_revision`: {revision} does not exist. Please make sure to choose one of" + f" {', '.join(available_versions + ['main'])}." + ) + + # community pipeline on GitHub + github_url = COMMUNITY_PIPELINES_URL.format(revision=revision, pipeline=pretrained_model_name_or_path) + try: + resolved_module_file = cached_download( + github_url, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + token=False, + ) + submodule = "git" + module_file = pretrained_model_name_or_path + ".py" + except EnvironmentError: + logger.error(f"Could not locate the {module_file} inside {pretrained_model_name_or_path}.") + raise + else: + try: + # Load from URL or cache if already cached + resolved_module_file = hf_hub_download( + pretrained_model_name_or_path, + module_file, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + token=token, + ) + submodule = os.path.join("local", "--".join(pretrained_model_name_or_path.split("/"))) + except EnvironmentError: + logger.error(f"Could not locate the {module_file} inside {pretrained_model_name_or_path}.") + raise + + # Check we have all the requirements in our environment + modules_needed = check_imports(resolved_module_file) + + # Now we move the module inside our cached dynamic modules. + full_submodule = DIFFUSERS_DYNAMIC_MODULE_NAME + os.path.sep + submodule + create_dynamic_module(full_submodule) + submodule_path = Path(HF_MODULES_CACHE) / full_submodule + if submodule == "local" or submodule == "git": + # We always copy local files (we could hash the file to see if there was a change, and give them the name of + # that hash, to only copy when there is a modification but it seems overkill for now). + # The only reason we do the copy is to avoid putting too many folders in sys.path. + shutil.copy(resolved_module_file, submodule_path / module_file) + for module_needed in modules_needed: + module_needed = f"{module_needed}.py" + shutil.copy(os.path.join(pretrained_model_name_or_path, module_needed), submodule_path / module_needed) + else: + # Get the commit hash + # TODO: we will get this info in the etag soon, so retrieve it from there and not here. + commit_hash = model_info(pretrained_model_name_or_path, revision=revision, token=token).sha + + # The module file will end up being placed in a subfolder with the git hash of the repo. This way we get the + # benefit of versioning. + submodule_path = submodule_path / commit_hash + full_submodule = full_submodule + os.path.sep + commit_hash + create_dynamic_module(full_submodule) + + if not (submodule_path / module_file).exists(): + shutil.copy(resolved_module_file, submodule_path / module_file) + # Make sure we also have every file with relative + for module_needed in modules_needed: + if not (submodule_path / module_needed).exists(): + get_cached_module_file( + pretrained_model_name_or_path, + f"{module_needed}.py", + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + token=token, + revision=revision, + local_files_only=local_files_only, + ) + return os.path.join(full_submodule, module_file) + + +@validate_hf_hub_args +def get_class_from_dynamic_module( + pretrained_model_name_or_path: Union[str, os.PathLike], + module_file: str, + class_name: Optional[str] = None, + cache_dir: Optional[Union[str, os.PathLike]] = None, + force_download: bool = False, + resume_download: bool = False, + proxies: Optional[Dict[str, str]] = None, + token: Optional[Union[bool, str]] = None, + revision: Optional[str] = None, + local_files_only: bool = False, + **kwargs, +): + """ + Extracts a class from a module file, present in the local folder or repository of a model. + + + + Calling this function will execute the code in the module file found locally or downloaded from the Hub. It should + therefore only be called on trusted repos. + + + + Args: + pretrained_model_name_or_path (`str` or `os.PathLike`): + This can be either: + + - a string, the *model id* of a pretrained model configuration hosted inside a model repo on + huggingface.co. Valid model ids can be located at the root-level, like `bert-base-uncased`, or namespaced + under a user or organization name, like `dbmdz/bert-base-german-cased`. + - a path to a *directory* containing a configuration file saved using the + [`~PreTrainedTokenizer.save_pretrained`] method, e.g., `./my_model_directory/`. + + module_file (`str`): + The name of the module file containing the class to look for. + class_name (`str`): + The name of the class to import in the module. + cache_dir (`str` or `os.PathLike`, *optional*): + Path to a directory in which a downloaded pretrained model configuration should be cached if the standard + cache should not be used. + force_download (`bool`, *optional*, defaults to `False`): + Whether or not to force to (re-)download the configuration files and override the cached versions if they + exist. + resume_download (`bool`, *optional*, defaults to `False`): + Whether or not to delete incompletely received file. Attempts to resume the download if such a file exists. + proxies (`Dict[str, str]`, *optional*): + A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128', + 'http://hostname': 'foo.bar:4012'}.` The proxies are used on each request. + token (`str` or `bool`, *optional*): + The token to use as HTTP bearer authorization for remote files. If `True`, will use the token generated + when running `transformers-cli login` (stored in `~/.huggingface`). + revision (`str`, *optional*, defaults to `"main"`): + The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a + git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any + identifier allowed by git. + local_files_only (`bool`, *optional*, defaults to `False`): + If `True`, will only try to load the tokenizer configuration from local files. + + + + You may pass a token in `token` if you are not logged in (`huggingface-cli login`) and want to use private + or [gated models](https://huggingface.co/docs/hub/models-gated#gated-models). + + + + Returns: + `type`: The class, dynamically imported from the module. + + Examples: + + ```python + # Download module `modeling.py` from huggingface.co and cache then extract the class `MyBertModel` from this + # module. + cls = get_class_from_dynamic_module("sgugger/my-bert-model", "modeling.py", "MyBertModel") + ```""" + # And lastly we get the class inside our newly created module + final_module = get_cached_module_file( + pretrained_model_name_or_path, + module_file, + cache_dir=cache_dir, + force_download=force_download, + resume_download=resume_download, + proxies=proxies, + token=token, + revision=revision, + local_files_only=local_files_only, + ) + return get_class_in_module(class_name, final_module.replace(".py", "")) diff --git a/diffusers-0.27.0/src/diffusers/utils/export_utils.py b/diffusers-0.27.0/src/diffusers/utils/export_utils.py new file mode 100755 index 0000000..bb53077 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/export_utils.py @@ -0,0 +1,140 @@ +import io +import random +import struct +import tempfile +from contextlib import contextmanager +from typing import List, Union + +import numpy as np +import PIL.Image +import PIL.ImageOps + +from .import_utils import ( + BACKENDS_MAPPING, + is_opencv_available, +) +from .logging import get_logger + + +global_rng = random.Random() + +logger = get_logger(__name__) + + +@contextmanager +def buffered_writer(raw_f): + f = io.BufferedWriter(raw_f) + yield f + f.flush() + + +def export_to_gif(image: List[PIL.Image.Image], output_gif_path: str = None, fps: int = 10) -> str: + if output_gif_path is None: + output_gif_path = tempfile.NamedTemporaryFile(suffix=".gif").name + + image[0].save( + output_gif_path, + save_all=True, + append_images=image[1:], + optimize=False, + duration=1000 // fps, + loop=0, + ) + return output_gif_path + + +def export_to_ply(mesh, output_ply_path: str = None): + """ + Write a PLY file for a mesh. + """ + if output_ply_path is None: + output_ply_path = tempfile.NamedTemporaryFile(suffix=".ply").name + + coords = mesh.verts.detach().cpu().numpy() + faces = mesh.faces.cpu().numpy() + rgb = np.stack([mesh.vertex_channels[x].detach().cpu().numpy() for x in "RGB"], axis=1) + + with buffered_writer(open(output_ply_path, "wb")) as f: + f.write(b"ply\n") + f.write(b"format binary_little_endian 1.0\n") + f.write(bytes(f"element vertex {len(coords)}\n", "ascii")) + f.write(b"property float x\n") + f.write(b"property float y\n") + f.write(b"property float z\n") + if rgb is not None: + f.write(b"property uchar red\n") + f.write(b"property uchar green\n") + f.write(b"property uchar blue\n") + if faces is not None: + f.write(bytes(f"element face {len(faces)}\n", "ascii")) + f.write(b"property list uchar int vertex_index\n") + f.write(b"end_header\n") + + if rgb is not None: + rgb = (rgb * 255.499).round().astype(int) + vertices = [ + (*coord, *rgb) + for coord, rgb in zip( + coords.tolist(), + rgb.tolist(), + ) + ] + format = struct.Struct("<3f3B") + for item in vertices: + f.write(format.pack(*item)) + else: + format = struct.Struct("<3f") + for vertex in coords.tolist(): + f.write(format.pack(*vertex)) + + if faces is not None: + format = struct.Struct(" str: + if is_opencv_available(): + import cv2 + else: + raise ImportError(BACKENDS_MAPPING["opencv"][1].format("export_to_video")) + if output_video_path is None: + output_video_path = tempfile.NamedTemporaryFile(suffix=".mp4").name + + if isinstance(video_frames[0], np.ndarray): + video_frames = [(frame * 255).astype(np.uint8) for frame in video_frames] + + elif isinstance(video_frames[0], PIL.Image.Image): + video_frames = [np.array(frame) for frame in video_frames] + + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + h, w, c = video_frames[0].shape + video_writer = cv2.VideoWriter(output_video_path, fourcc, fps=fps, frameSize=(w, h)) + for i in range(len(video_frames)): + img = cv2.cvtColor(video_frames[i], cv2.COLOR_RGB2BGR) + video_writer.write(img) + return output_video_path diff --git a/diffusers-0.27.0/src/diffusers/utils/hub_utils.py b/diffusers-0.27.0/src/diffusers/utils/hub_utils.py new file mode 100755 index 0000000..e554b42 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/hub_utils.py @@ -0,0 +1,493 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import os +import re +import sys +import tempfile +import traceback +import warnings +from pathlib import Path +from typing import Dict, List, Optional, Union +from uuid import uuid4 + +from huggingface_hub import ( + ModelCard, + ModelCardData, + create_repo, + hf_hub_download, + upload_folder, +) +from huggingface_hub.constants import HF_HUB_CACHE, HF_HUB_DISABLE_TELEMETRY, HF_HUB_OFFLINE +from huggingface_hub.file_download import REGEX_COMMIT_HASH +from huggingface_hub.utils import ( + EntryNotFoundError, + RepositoryNotFoundError, + RevisionNotFoundError, + is_jinja_available, + validate_hf_hub_args, +) +from packaging import version +from requests import HTTPError + +from .. import __version__ +from .constants import ( + DEPRECATED_REVISION_ARGS, + HUGGINGFACE_CO_RESOLVE_ENDPOINT, + SAFETENSORS_WEIGHTS_NAME, + WEIGHTS_NAME, +) +from .import_utils import ( + ENV_VARS_TRUE_VALUES, + _flax_version, + _jax_version, + _onnxruntime_version, + _torch_version, + is_flax_available, + is_onnx_available, + is_torch_available, +) +from .logging import get_logger + + +logger = get_logger(__name__) + +MODEL_CARD_TEMPLATE_PATH = Path(__file__).parent / "model_card_template.md" +SESSION_ID = uuid4().hex + + +def http_user_agent(user_agent: Union[Dict, str, None] = None) -> str: + """ + Formats a user-agent string with basic info about a request. + """ + ua = f"diffusers/{__version__}; python/{sys.version.split()[0]}; session_id/{SESSION_ID}" + if HF_HUB_DISABLE_TELEMETRY or HF_HUB_OFFLINE: + return ua + "; telemetry/off" + if is_torch_available(): + ua += f"; torch/{_torch_version}" + if is_flax_available(): + ua += f"; jax/{_jax_version}" + ua += f"; flax/{_flax_version}" + if is_onnx_available(): + ua += f"; onnxruntime/{_onnxruntime_version}" + # CI will set this value to True + if os.environ.get("DIFFUSERS_IS_CI", "").upper() in ENV_VARS_TRUE_VALUES: + ua += "; is_ci/true" + if isinstance(user_agent, dict): + ua += "; " + "; ".join(f"{k}/{v}" for k, v in user_agent.items()) + elif isinstance(user_agent, str): + ua += "; " + user_agent + return ua + + +def load_or_create_model_card( + repo_id_or_path: str = None, + token: Optional[str] = None, + is_pipeline: bool = False, + from_training: bool = False, + model_description: Optional[str] = None, + base_model: str = None, + prompt: Optional[str] = None, + license: Optional[str] = None, + widget: Optional[List[dict]] = None, + inference: Optional[bool] = None, +) -> ModelCard: + """ + Loads or creates a model card. + + Args: + repo_id_or_path (`str`): + The repo id (e.g., "runwayml/stable-diffusion-v1-5") or local path where to look for the model card. + token (`str`, *optional*): + Authentication token. Will default to the stored token. See https://huggingface.co/settings/token for more details. + is_pipeline (`bool`): + Boolean to indicate if we're adding tag to a [`DiffusionPipeline`]. + from_training: (`bool`): Boolean flag to denote if the model card is being created from a training script. + model_description (`str`, *optional*): Model description to add to the model card. Helpful when using + `load_or_create_model_card` from a training script. + base_model (`str`): Base model identifier (e.g., "stabilityai/stable-diffusion-xl-base-1.0"). Useful + for DreamBooth-like training. + prompt (`str`, *optional*): Prompt used for training. Useful for DreamBooth-like training. + license: (`str`, *optional*): License of the output artifact. Helpful when using + `load_or_create_model_card` from a training script. + widget (`List[dict]`, *optional*): Widget to accompany a gallery template. + inference: (`bool`, optional): Whether to turn on inference widget. Helpful when using + `load_or_create_model_card` from a training script. + """ + if not is_jinja_available(): + raise ValueError( + "Modelcard rendering is based on Jinja templates." + " Please make sure to have `jinja` installed before using `load_or_create_model_card`." + " To install it, please run `pip install Jinja2`." + ) + + try: + # Check if the model card is present on the remote repo + model_card = ModelCard.load(repo_id_or_path, token=token) + except (EntryNotFoundError, RepositoryNotFoundError): + # Otherwise create a model card from template + if from_training: + model_card = ModelCard.from_template( + card_data=ModelCardData( # Card metadata object that will be converted to YAML block + license=license, + library_name="diffusers", + inference=inference, + base_model=base_model, + instance_prompt=prompt, + widget=widget, + ), + template_path=MODEL_CARD_TEMPLATE_PATH, + model_description=model_description, + ) + else: + card_data = ModelCardData() + component = "pipeline" if is_pipeline else "model" + if model_description is None: + model_description = f"This is the model card of a 🧨 diffusers {component} that has been pushed on the Hub. This model card has been automatically generated." + model_card = ModelCard.from_template(card_data, model_description=model_description) + + return model_card + + +def populate_model_card(model_card: ModelCard, tags: Union[str, List[str]] = None) -> ModelCard: + """Populates the `model_card` with library name and optional tags.""" + if model_card.data.library_name is None: + model_card.data.library_name = "diffusers" + + if tags is not None: + if isinstance(tags, str): + tags = [tags] + if model_card.data.tags is None: + model_card.data.tags = [] + for tag in tags: + model_card.data.tags.append(tag) + + return model_card + + +def extract_commit_hash(resolved_file: Optional[str], commit_hash: Optional[str] = None): + """ + Extracts the commit hash from a resolved filename toward a cache file. + """ + if resolved_file is None or commit_hash is not None: + return commit_hash + resolved_file = str(Path(resolved_file).as_posix()) + search = re.search(r"snapshots/([^/]+)/", resolved_file) + if search is None: + return None + commit_hash = search.groups()[0] + return commit_hash if REGEX_COMMIT_HASH.match(commit_hash) else None + + +# Old default cache path, potentially to be migrated. +# This logic was more or less taken from `transformers`, with the following differences: +# - Diffusers doesn't use custom environment variables to specify the cache path. +# - There is no need to migrate the cache format, just move the files to the new location. +hf_cache_home = os.path.expanduser( + os.getenv("HF_HOME", os.path.join(os.getenv("XDG_CACHE_HOME", "~/.cache"), "huggingface")) +) +old_diffusers_cache = os.path.join(hf_cache_home, "diffusers") + + +def move_cache(old_cache_dir: Optional[str] = None, new_cache_dir: Optional[str] = None) -> None: + if new_cache_dir is None: + new_cache_dir = HF_HUB_CACHE + if old_cache_dir is None: + old_cache_dir = old_diffusers_cache + + old_cache_dir = Path(old_cache_dir).expanduser() + new_cache_dir = Path(new_cache_dir).expanduser() + for old_blob_path in old_cache_dir.glob("**/blobs/*"): + if old_blob_path.is_file() and not old_blob_path.is_symlink(): + new_blob_path = new_cache_dir / old_blob_path.relative_to(old_cache_dir) + new_blob_path.parent.mkdir(parents=True, exist_ok=True) + os.replace(old_blob_path, new_blob_path) + try: + os.symlink(new_blob_path, old_blob_path) + except OSError: + logger.warning( + "Could not create symlink between old cache and new cache. If you use an older version of diffusers again, files will be re-downloaded." + ) + # At this point, old_cache_dir contains symlinks to the new cache (it can still be used). + + +cache_version_file = os.path.join(HF_HUB_CACHE, "version_diffusers_cache.txt") +if not os.path.isfile(cache_version_file): + cache_version = 0 +else: + with open(cache_version_file) as f: + try: + cache_version = int(f.read()) + except ValueError: + cache_version = 0 + +if cache_version < 1: + old_cache_is_not_empty = os.path.isdir(old_diffusers_cache) and len(os.listdir(old_diffusers_cache)) > 0 + if old_cache_is_not_empty: + logger.warning( + "The cache for model files in Diffusers v0.14.0 has moved to a new location. Moving your " + "existing cached models. This is a one-time operation, you can interrupt it or run it " + "later by calling `diffusers.utils.hub_utils.move_cache()`." + ) + try: + move_cache() + except Exception as e: + trace = "\n".join(traceback.format_tb(e.__traceback__)) + logger.error( + f"There was a problem when trying to move your cache:\n\n{trace}\n{e.__class__.__name__}: {e}\n\nPlease " + "file an issue at https://github.com/huggingface/diffusers/issues/new/choose, copy paste this whole " + "message and we will do our best to help." + ) + +if cache_version < 1: + try: + os.makedirs(HF_HUB_CACHE, exist_ok=True) + with open(cache_version_file, "w") as f: + f.write("1") + except Exception: + logger.warning( + f"There was a problem when trying to write in your cache folder ({HF_HUB_CACHE}). Please, ensure " + "the directory exists and can be written to." + ) + + +def _add_variant(weights_name: str, variant: Optional[str] = None) -> str: + if variant is not None: + splits = weights_name.split(".") + splits = splits[:-1] + [variant] + splits[-1:] + weights_name = ".".join(splits) + + return weights_name + + +@validate_hf_hub_args +def _get_model_file( + pretrained_model_name_or_path: Union[str, Path], + *, + weights_name: str, + subfolder: Optional[str] = None, + cache_dir: Optional[str] = None, + force_download: bool = False, + proxies: Optional[Dict] = None, + resume_download: bool = False, + local_files_only: bool = False, + token: Optional[str] = None, + user_agent: Optional[Union[Dict, str]] = None, + revision: Optional[str] = None, + commit_hash: Optional[str] = None, +): + pretrained_model_name_or_path = str(pretrained_model_name_or_path) + if os.path.isfile(pretrained_model_name_or_path): + return pretrained_model_name_or_path + elif os.path.isdir(pretrained_model_name_or_path): + if os.path.isfile(os.path.join(pretrained_model_name_or_path, weights_name)): + # Load from a PyTorch checkpoint + model_file = os.path.join(pretrained_model_name_or_path, weights_name) + return model_file + elif subfolder is not None and os.path.isfile( + os.path.join(pretrained_model_name_or_path, subfolder, weights_name) + ): + model_file = os.path.join(pretrained_model_name_or_path, subfolder, weights_name) + return model_file + else: + raise EnvironmentError( + f"Error no file named {weights_name} found in directory {pretrained_model_name_or_path}." + ) + else: + # 1. First check if deprecated way of loading from branches is used + if ( + revision in DEPRECATED_REVISION_ARGS + and (weights_name == WEIGHTS_NAME or weights_name == SAFETENSORS_WEIGHTS_NAME) + and version.parse(version.parse(__version__).base_version) >= version.parse("0.22.0") + ): + try: + model_file = hf_hub_download( + pretrained_model_name_or_path, + filename=_add_variant(weights_name, revision), + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + token=token, + user_agent=user_agent, + subfolder=subfolder, + revision=revision or commit_hash, + ) + warnings.warn( + f"Loading the variant {revision} from {pretrained_model_name_or_path} via `revision='{revision}'` is deprecated. Loading instead from `revision='main'` with `variant={revision}`. Loading model variants via `revision='{revision}'` will be removed in diffusers v1. Please use `variant='{revision}'` instead.", + FutureWarning, + ) + return model_file + except: # noqa: E722 + warnings.warn( + f"You are loading the variant {revision} from {pretrained_model_name_or_path} via `revision='{revision}'`. This behavior is deprecated and will be removed in diffusers v1. One should use `variant='{revision}'` instead. However, it appears that {pretrained_model_name_or_path} currently does not have a {_add_variant(weights_name, revision)} file in the 'main' branch of {pretrained_model_name_or_path}. \n The Diffusers team and community would be very grateful if you could open an issue: https://github.com/huggingface/diffusers/issues/new with the title '{pretrained_model_name_or_path} is missing {_add_variant(weights_name, revision)}' so that the correct variant file can be added.", + FutureWarning, + ) + try: + # 2. Load model file as usual + model_file = hf_hub_download( + pretrained_model_name_or_path, + filename=weights_name, + cache_dir=cache_dir, + force_download=force_download, + proxies=proxies, + resume_download=resume_download, + local_files_only=local_files_only, + token=token, + user_agent=user_agent, + subfolder=subfolder, + revision=revision or commit_hash, + ) + return model_file + + except RepositoryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier " + "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a " + "token having permission to this repo with `token` or log in with `huggingface-cli " + "login`." + ) + except RevisionNotFoundError: + raise EnvironmentError( + f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for " + "this model name. Check the model page at " + f"'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions." + ) + except EntryNotFoundError: + raise EnvironmentError( + f"{pretrained_model_name_or_path} does not appear to have a file named {weights_name}." + ) + except HTTPError as err: + raise EnvironmentError( + f"There was a specific connection error when trying to load {pretrained_model_name_or_path}:\n{err}" + ) + except ValueError: + raise EnvironmentError( + f"We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it" + f" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a" + f" directory containing a file named {weights_name} or" + " \nCheckout your internet connection or see how to run the library in" + " offline mode at 'https://huggingface.co/docs/diffusers/installation#offline-mode'." + ) + except EnvironmentError: + raise EnvironmentError( + f"Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it from " + "'https://huggingface.co/models', make sure you don't have a local directory with the same name. " + f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory " + f"containing a file named {weights_name}" + ) + + +class PushToHubMixin: + """ + A Mixin to push a model, scheduler, or pipeline to the Hugging Face Hub. + """ + + def _upload_folder( + self, + working_dir: Union[str, os.PathLike], + repo_id: str, + token: Optional[str] = None, + commit_message: Optional[str] = None, + create_pr: bool = False, + ): + """ + Uploads all files in `working_dir` to `repo_id`. + """ + if commit_message is None: + if "Model" in self.__class__.__name__: + commit_message = "Upload model" + elif "Scheduler" in self.__class__.__name__: + commit_message = "Upload scheduler" + else: + commit_message = f"Upload {self.__class__.__name__}" + + logger.info(f"Uploading the files of {working_dir} to {repo_id}.") + return upload_folder( + repo_id=repo_id, folder_path=working_dir, token=token, commit_message=commit_message, create_pr=create_pr + ) + + def push_to_hub( + self, + repo_id: str, + commit_message: Optional[str] = None, + private: Optional[bool] = None, + token: Optional[str] = None, + create_pr: bool = False, + safe_serialization: bool = True, + variant: Optional[str] = None, + ) -> str: + """ + Upload model, scheduler, or pipeline files to the 🤗 Hugging Face Hub. + + Parameters: + repo_id (`str`): + The name of the repository you want to push your model, scheduler, or pipeline files to. It should + contain your organization name when pushing to an organization. `repo_id` can also be a path to a local + directory. + commit_message (`str`, *optional*): + Message to commit while pushing. Default to `"Upload {object}"`. + private (`bool`, *optional*): + Whether or not the repository created should be private. + token (`str`, *optional*): + The token to use as HTTP bearer authorization for remote files. The token generated when running + `huggingface-cli login` (stored in `~/.huggingface`). + create_pr (`bool`, *optional*, defaults to `False`): + Whether or not to create a PR with the uploaded files or directly commit. + safe_serialization (`bool`, *optional*, defaults to `True`): + Whether or not to convert the model weights to the `safetensors` format. + variant (`str`, *optional*): + If specified, weights are saved in the format `pytorch_model..bin`. + + Examples: + + ```python + from diffusers import UNet2DConditionModel + + unet = UNet2DConditionModel.from_pretrained("stabilityai/stable-diffusion-2", subfolder="unet") + + # Push the `unet` to your namespace with the name "my-finetuned-unet". + unet.push_to_hub("my-finetuned-unet") + + # Push the `unet` to an organization with the name "my-finetuned-unet". + unet.push_to_hub("your-org/my-finetuned-unet") + ``` + """ + repo_id = create_repo(repo_id, private=private, token=token, exist_ok=True).repo_id + + # Create a new empty model card and eventually tag it + model_card = load_or_create_model_card(repo_id, token=token) + model_card = populate_model_card(model_card) + + # Save all files. + save_kwargs = {"safe_serialization": safe_serialization} + if "Scheduler" not in self.__class__.__name__: + save_kwargs.update({"variant": variant}) + + with tempfile.TemporaryDirectory() as tmpdir: + self.save_pretrained(tmpdir, **save_kwargs) + + # Update model card if needed: + model_card.save(os.path.join(tmpdir, "README.md")) + + return self._upload_folder( + tmpdir, + repo_id, + token=token, + commit_message=commit_message, + create_pr=create_pr, + ) diff --git a/diffusers-0.27.0/src/diffusers/utils/import_utils.py b/diffusers-0.27.0/src/diffusers/utils/import_utils.py new file mode 100755 index 0000000..a3ee31c --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/import_utils.py @@ -0,0 +1,726 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Import utilities: Utilities related to imports and our lazy inits. +""" + +import importlib.util +import operator as op +import os +import sys +from collections import OrderedDict +from itertools import chain +from types import ModuleType +from typing import Any, Union + +from huggingface_hub.utils import is_jinja_available # noqa: F401 +from packaging import version +from packaging.version import Version, parse + +from . import logging + + +# The package importlib_metadata is in a different place, depending on the python version. +if sys.version_info < (3, 8): + import importlib_metadata +else: + import importlib.metadata as importlib_metadata + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +ENV_VARS_TRUE_VALUES = {"1", "ON", "YES", "TRUE"} +ENV_VARS_TRUE_AND_AUTO_VALUES = ENV_VARS_TRUE_VALUES.union({"AUTO"}) + +USE_TF = os.environ.get("USE_TF", "AUTO").upper() +USE_TORCH = os.environ.get("USE_TORCH", "AUTO").upper() +USE_JAX = os.environ.get("USE_FLAX", "AUTO").upper() +USE_SAFETENSORS = os.environ.get("USE_SAFETENSORS", "AUTO").upper() +DIFFUSERS_SLOW_IMPORT = os.environ.get("DIFFUSERS_SLOW_IMPORT", "FALSE").upper() +DIFFUSERS_SLOW_IMPORT = DIFFUSERS_SLOW_IMPORT in ENV_VARS_TRUE_VALUES + +STR_OPERATION_TO_FUNC = {">": op.gt, ">=": op.ge, "==": op.eq, "!=": op.ne, "<=": op.le, "<": op.lt} + +_torch_version = "N/A" +if USE_TORCH in ENV_VARS_TRUE_AND_AUTO_VALUES and USE_TF not in ENV_VARS_TRUE_VALUES: + _torch_available = importlib.util.find_spec("torch") is not None + if _torch_available: + try: + _torch_version = importlib_metadata.version("torch") + logger.info(f"PyTorch version {_torch_version} available.") + except importlib_metadata.PackageNotFoundError: + _torch_available = False +else: + logger.info("Disabling PyTorch because USE_TORCH is set") + _torch_available = False + +_torch_xla_available = importlib.util.find_spec("torch_xla") is not None +if _torch_xla_available: + try: + _torch_xla_version = importlib_metadata.version("torch_xla") + logger.info(f"PyTorch XLA version {_torch_xla_version} available.") + except ImportError: + _torch_xla_available = False + +# check whether torch_npu is available +_torch_npu_available = importlib.util.find_spec("torch_npu") is not None +if _torch_npu_available: + try: + _torch_npu_version = importlib_metadata.version("torch_npu") + logger.info(f"torch_npu version {_torch_npu_version} available.") + except ImportError: + _torch_npu_available = False + +_jax_version = "N/A" +_flax_version = "N/A" +if USE_JAX in ENV_VARS_TRUE_AND_AUTO_VALUES: + _flax_available = importlib.util.find_spec("jax") is not None and importlib.util.find_spec("flax") is not None + if _flax_available: + try: + _jax_version = importlib_metadata.version("jax") + _flax_version = importlib_metadata.version("flax") + logger.info(f"JAX version {_jax_version}, Flax version {_flax_version} available.") + except importlib_metadata.PackageNotFoundError: + _flax_available = False +else: + _flax_available = False + +if USE_SAFETENSORS in ENV_VARS_TRUE_AND_AUTO_VALUES: + _safetensors_available = importlib.util.find_spec("safetensors") is not None + if _safetensors_available: + try: + _safetensors_version = importlib_metadata.version("safetensors") + logger.info(f"Safetensors version {_safetensors_version} available.") + except importlib_metadata.PackageNotFoundError: + _safetensors_available = False +else: + logger.info("Disabling Safetensors because USE_TF is set") + _safetensors_available = False + +_transformers_available = importlib.util.find_spec("transformers") is not None +try: + _transformers_version = importlib_metadata.version("transformers") + logger.debug(f"Successfully imported transformers version {_transformers_version}") +except importlib_metadata.PackageNotFoundError: + _transformers_available = False + + +_inflect_available = importlib.util.find_spec("inflect") is not None +try: + _inflect_version = importlib_metadata.version("inflect") + logger.debug(f"Successfully imported inflect version {_inflect_version}") +except importlib_metadata.PackageNotFoundError: + _inflect_available = False + + +_unidecode_available = importlib.util.find_spec("unidecode") is not None +try: + _unidecode_version = importlib_metadata.version("unidecode") + logger.debug(f"Successfully imported unidecode version {_unidecode_version}") +except importlib_metadata.PackageNotFoundError: + _unidecode_available = False + + +_onnxruntime_version = "N/A" +_onnx_available = importlib.util.find_spec("onnxruntime") is not None +if _onnx_available: + candidates = ( + "onnxruntime", + "onnxruntime-gpu", + "ort_nightly_gpu", + "onnxruntime-directml", + "onnxruntime-openvino", + "ort_nightly_directml", + "onnxruntime-rocm", + "onnxruntime-training", + ) + _onnxruntime_version = None + # For the metadata, we have to look for both onnxruntime and onnxruntime-gpu + for pkg in candidates: + try: + _onnxruntime_version = importlib_metadata.version(pkg) + break + except importlib_metadata.PackageNotFoundError: + pass + _onnx_available = _onnxruntime_version is not None + if _onnx_available: + logger.debug(f"Successfully imported onnxruntime version {_onnxruntime_version}") + +# (sayakpaul): importlib.util.find_spec("opencv-python") returns None even when it's installed. +# _opencv_available = importlib.util.find_spec("opencv-python") is not None +try: + candidates = ( + "opencv-python", + "opencv-contrib-python", + "opencv-python-headless", + "opencv-contrib-python-headless", + ) + _opencv_version = None + for pkg in candidates: + try: + _opencv_version = importlib_metadata.version(pkg) + break + except importlib_metadata.PackageNotFoundError: + pass + _opencv_available = _opencv_version is not None + if _opencv_available: + logger.debug(f"Successfully imported cv2 version {_opencv_version}") +except importlib_metadata.PackageNotFoundError: + _opencv_available = False + +_scipy_available = importlib.util.find_spec("scipy") is not None +try: + _scipy_version = importlib_metadata.version("scipy") + logger.debug(f"Successfully imported scipy version {_scipy_version}") +except importlib_metadata.PackageNotFoundError: + _scipy_available = False + +_librosa_available = importlib.util.find_spec("librosa") is not None +try: + _librosa_version = importlib_metadata.version("librosa") + logger.debug(f"Successfully imported librosa version {_librosa_version}") +except importlib_metadata.PackageNotFoundError: + _librosa_available = False + +_accelerate_available = importlib.util.find_spec("accelerate") is not None +try: + _accelerate_version = importlib_metadata.version("accelerate") + logger.debug(f"Successfully imported accelerate version {_accelerate_version}") +except importlib_metadata.PackageNotFoundError: + _accelerate_available = False + +_xformers_available = importlib.util.find_spec("xformers") is not None +try: + _xformers_version = importlib_metadata.version("xformers") + if _torch_available: + _torch_version = importlib_metadata.version("torch") + if version.Version(_torch_version) < version.Version("1.12"): + raise ValueError("xformers is installed in your environment and requires PyTorch >= 1.12") + + logger.debug(f"Successfully imported xformers version {_xformers_version}") +except importlib_metadata.PackageNotFoundError: + _xformers_available = False + +_k_diffusion_available = importlib.util.find_spec("k_diffusion") is not None +try: + _k_diffusion_version = importlib_metadata.version("k_diffusion") + logger.debug(f"Successfully imported k-diffusion version {_k_diffusion_version}") +except importlib_metadata.PackageNotFoundError: + _k_diffusion_available = False + +_note_seq_available = importlib.util.find_spec("note_seq") is not None +try: + _note_seq_version = importlib_metadata.version("note_seq") + logger.debug(f"Successfully imported note-seq version {_note_seq_version}") +except importlib_metadata.PackageNotFoundError: + _note_seq_available = False + +_wandb_available = importlib.util.find_spec("wandb") is not None +try: + _wandb_version = importlib_metadata.version("wandb") + logger.debug(f"Successfully imported wandb version {_wandb_version }") +except importlib_metadata.PackageNotFoundError: + _wandb_available = False + + +_tensorboard_available = importlib.util.find_spec("tensorboard") +try: + _tensorboard_version = importlib_metadata.version("tensorboard") + logger.debug(f"Successfully imported tensorboard version {_tensorboard_version}") +except importlib_metadata.PackageNotFoundError: + _tensorboard_available = False + + +_compel_available = importlib.util.find_spec("compel") +try: + _compel_version = importlib_metadata.version("compel") + logger.debug(f"Successfully imported compel version {_compel_version}") +except importlib_metadata.PackageNotFoundError: + _compel_available = False + + +_ftfy_available = importlib.util.find_spec("ftfy") is not None +try: + _ftfy_version = importlib_metadata.version("ftfy") + logger.debug(f"Successfully imported ftfy version {_ftfy_version}") +except importlib_metadata.PackageNotFoundError: + _ftfy_available = False + + +_bs4_available = importlib.util.find_spec("bs4") is not None +try: + # importlib metadata under different name + _bs4_version = importlib_metadata.version("beautifulsoup4") + logger.debug(f"Successfully imported ftfy version {_bs4_version}") +except importlib_metadata.PackageNotFoundError: + _bs4_available = False + +_torchsde_available = importlib.util.find_spec("torchsde") is not None +try: + _torchsde_version = importlib_metadata.version("torchsde") + logger.debug(f"Successfully imported torchsde version {_torchsde_version}") +except importlib_metadata.PackageNotFoundError: + _torchsde_available = False + +_invisible_watermark_available = importlib.util.find_spec("imwatermark") is not None +try: + _invisible_watermark_version = importlib_metadata.version("invisible-watermark") + logger.debug(f"Successfully imported invisible-watermark version {_invisible_watermark_version}") +except importlib_metadata.PackageNotFoundError: + _invisible_watermark_available = False + + +_peft_available = importlib.util.find_spec("peft") is not None +try: + _peft_version = importlib_metadata.version("peft") + logger.debug(f"Successfully imported peft version {_peft_version}") +except importlib_metadata.PackageNotFoundError: + _peft_available = False + +_torchvision_available = importlib.util.find_spec("torchvision") is not None +try: + _torchvision_version = importlib_metadata.version("torchvision") + logger.debug(f"Successfully imported torchvision version {_torchvision_version}") +except importlib_metadata.PackageNotFoundError: + _torchvision_available = False + + +def is_torch_available(): + return _torch_available + + +def is_torch_xla_available(): + return _torch_xla_available + + +def is_torch_npu_available(): + return _torch_npu_available + + +def is_flax_available(): + return _flax_available + + +def is_transformers_available(): + return _transformers_available + + +def is_inflect_available(): + return _inflect_available + + +def is_unidecode_available(): + return _unidecode_available + + +def is_onnx_available(): + return _onnx_available + + +def is_opencv_available(): + return _opencv_available + + +def is_scipy_available(): + return _scipy_available + + +def is_librosa_available(): + return _librosa_available + + +def is_xformers_available(): + return _xformers_available + + +def is_accelerate_available(): + return _accelerate_available + + +def is_k_diffusion_available(): + return _k_diffusion_available + + +def is_note_seq_available(): + return _note_seq_available + + +def is_wandb_available(): + return _wandb_available + + +def is_tensorboard_available(): + return _tensorboard_available + + +def is_compel_available(): + return _compel_available + + +def is_ftfy_available(): + return _ftfy_available + + +def is_bs4_available(): + return _bs4_available + + +def is_torchsde_available(): + return _torchsde_available + + +def is_invisible_watermark_available(): + return _invisible_watermark_available + + +def is_peft_available(): + return _peft_available + + +def is_torchvision_available(): + return _torchvision_available + + +# docstyle-ignore +FLAX_IMPORT_ERROR = """ +{0} requires the FLAX library but it was not found in your environment. Checkout the instructions on the +installation page: https://github.com/google/flax and follow the ones that match your environment. +""" + +# docstyle-ignore +INFLECT_IMPORT_ERROR = """ +{0} requires the inflect library but it was not found in your environment. You can install it with pip: `pip install +inflect` +""" + +# docstyle-ignore +PYTORCH_IMPORT_ERROR = """ +{0} requires the PyTorch library but it was not found in your environment. Checkout the instructions on the +installation page: https://pytorch.org/get-started/locally/ and follow the ones that match your environment. +""" + +# docstyle-ignore +ONNX_IMPORT_ERROR = """ +{0} requires the onnxruntime library but it was not found in your environment. You can install it with pip: `pip +install onnxruntime` +""" + +# docstyle-ignore +OPENCV_IMPORT_ERROR = """ +{0} requires the OpenCV library but it was not found in your environment. You can install it with pip: `pip +install opencv-python` +""" + +# docstyle-ignore +SCIPY_IMPORT_ERROR = """ +{0} requires the scipy library but it was not found in your environment. You can install it with pip: `pip install +scipy` +""" + +# docstyle-ignore +LIBROSA_IMPORT_ERROR = """ +{0} requires the librosa library but it was not found in your environment. Checkout the instructions on the +installation page: https://librosa.org/doc/latest/install.html and follow the ones that match your environment. +""" + +# docstyle-ignore +TRANSFORMERS_IMPORT_ERROR = """ +{0} requires the transformers library but it was not found in your environment. You can install it with pip: `pip +install transformers` +""" + +# docstyle-ignore +UNIDECODE_IMPORT_ERROR = """ +{0} requires the unidecode library but it was not found in your environment. You can install it with pip: `pip install +Unidecode` +""" + +# docstyle-ignore +K_DIFFUSION_IMPORT_ERROR = """ +{0} requires the k-diffusion library but it was not found in your environment. You can install it with pip: `pip +install k-diffusion` +""" + +# docstyle-ignore +NOTE_SEQ_IMPORT_ERROR = """ +{0} requires the note-seq library but it was not found in your environment. You can install it with pip: `pip +install note-seq` +""" + +# docstyle-ignore +WANDB_IMPORT_ERROR = """ +{0} requires the wandb library but it was not found in your environment. You can install it with pip: `pip +install wandb` +""" + +# docstyle-ignore +TENSORBOARD_IMPORT_ERROR = """ +{0} requires the tensorboard library but it was not found in your environment. You can install it with pip: `pip +install tensorboard` +""" + + +# docstyle-ignore +COMPEL_IMPORT_ERROR = """ +{0} requires the compel library but it was not found in your environment. You can install it with pip: `pip install compel` +""" + +# docstyle-ignore +BS4_IMPORT_ERROR = """ +{0} requires the Beautiful Soup library but it was not found in your environment. You can install it with pip: +`pip install beautifulsoup4`. Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +FTFY_IMPORT_ERROR = """ +{0} requires the ftfy library but it was not found in your environment. Checkout the instructions on the +installation section: https://github.com/rspeer/python-ftfy/tree/master#installing and follow the ones +that match your environment. Please note that you may need to restart your runtime after installation. +""" + +# docstyle-ignore +TORCHSDE_IMPORT_ERROR = """ +{0} requires the torchsde library but it was not found in your environment. You can install it with pip: `pip install torchsde` +""" + +# docstyle-ignore +INVISIBLE_WATERMARK_IMPORT_ERROR = """ +{0} requires the invisible-watermark library but it was not found in your environment. You can install it with pip: `pip install invisible-watermark>=0.2.0` +""" + + +BACKENDS_MAPPING = OrderedDict( + [ + ("bs4", (is_bs4_available, BS4_IMPORT_ERROR)), + ("flax", (is_flax_available, FLAX_IMPORT_ERROR)), + ("inflect", (is_inflect_available, INFLECT_IMPORT_ERROR)), + ("onnx", (is_onnx_available, ONNX_IMPORT_ERROR)), + ("opencv", (is_opencv_available, OPENCV_IMPORT_ERROR)), + ("scipy", (is_scipy_available, SCIPY_IMPORT_ERROR)), + ("torch", (is_torch_available, PYTORCH_IMPORT_ERROR)), + ("transformers", (is_transformers_available, TRANSFORMERS_IMPORT_ERROR)), + ("unidecode", (is_unidecode_available, UNIDECODE_IMPORT_ERROR)), + ("librosa", (is_librosa_available, LIBROSA_IMPORT_ERROR)), + ("k_diffusion", (is_k_diffusion_available, K_DIFFUSION_IMPORT_ERROR)), + ("note_seq", (is_note_seq_available, NOTE_SEQ_IMPORT_ERROR)), + ("wandb", (is_wandb_available, WANDB_IMPORT_ERROR)), + ("tensorboard", (is_tensorboard_available, TENSORBOARD_IMPORT_ERROR)), + ("compel", (is_compel_available, COMPEL_IMPORT_ERROR)), + ("ftfy", (is_ftfy_available, FTFY_IMPORT_ERROR)), + ("torchsde", (is_torchsde_available, TORCHSDE_IMPORT_ERROR)), + ("invisible_watermark", (is_invisible_watermark_available, INVISIBLE_WATERMARK_IMPORT_ERROR)), + ] +) + + +def requires_backends(obj, backends): + if not isinstance(backends, (list, tuple)): + backends = [backends] + + name = obj.__name__ if hasattr(obj, "__name__") else obj.__class__.__name__ + checks = (BACKENDS_MAPPING[backend] for backend in backends) + failed = [msg.format(name) for available, msg in checks if not available()] + if failed: + raise ImportError("".join(failed)) + + if name in [ + "VersatileDiffusionTextToImagePipeline", + "VersatileDiffusionPipeline", + "VersatileDiffusionDualGuidedPipeline", + "StableDiffusionImageVariationPipeline", + "UnCLIPPipeline", + ] and is_transformers_version("<", "4.25.0"): + raise ImportError( + f"You need to install `transformers>=4.25` in order to use {name}: \n```\n pip install" + " --upgrade transformers \n```" + ) + + if name in ["StableDiffusionDepth2ImgPipeline", "StableDiffusionPix2PixZeroPipeline"] and is_transformers_version( + "<", "4.26.0" + ): + raise ImportError( + f"You need to install `transformers>=4.26` in order to use {name}: \n```\n pip install" + " --upgrade transformers \n```" + ) + + +class DummyObject(type): + """ + Metaclass for the dummy objects. Any class inheriting from it will return the ImportError generated by + `requires_backend` each time a user tries to access any method of that class. + """ + + def __getattr__(cls, key): + if key.startswith("_") and key not in ["_load_connected_pipes", "_is_onnx"]: + return super().__getattr__(cls, key) + requires_backends(cls, cls._backends) + + +# This function was copied from: https://github.com/huggingface/accelerate/blob/874c4967d94badd24f893064cc3bef45f57cadf7/src/accelerate/utils/versions.py#L319 +def compare_versions(library_or_version: Union[str, Version], operation: str, requirement_version: str): + """ + Args: + Compares a library version to some requirement using a given operation. + library_or_version (`str` or `packaging.version.Version`): + A library name or a version to check. + operation (`str`): + A string representation of an operator, such as `">"` or `"<="`. + requirement_version (`str`): + The version to compare the library version against + """ + if operation not in STR_OPERATION_TO_FUNC.keys(): + raise ValueError(f"`operation` must be one of {list(STR_OPERATION_TO_FUNC.keys())}, received {operation}") + operation = STR_OPERATION_TO_FUNC[operation] + if isinstance(library_or_version, str): + library_or_version = parse(importlib_metadata.version(library_or_version)) + return operation(library_or_version, parse(requirement_version)) + + +# This function was copied from: https://github.com/huggingface/accelerate/blob/874c4967d94badd24f893064cc3bef45f57cadf7/src/accelerate/utils/versions.py#L338 +def is_torch_version(operation: str, version: str): + """ + Args: + Compares the current PyTorch version to a given reference with an operation. + operation (`str`): + A string representation of an operator, such as `">"` or `"<="` + version (`str`): + A string version of PyTorch + """ + return compare_versions(parse(_torch_version), operation, version) + + +def is_transformers_version(operation: str, version: str): + """ + Args: + Compares the current Transformers version to a given reference with an operation. + operation (`str`): + A string representation of an operator, such as `">"` or `"<="` + version (`str`): + A version string + """ + if not _transformers_available: + return False + return compare_versions(parse(_transformers_version), operation, version) + + +def is_accelerate_version(operation: str, version: str): + """ + Args: + Compares the current Accelerate version to a given reference with an operation. + operation (`str`): + A string representation of an operator, such as `">"` or `"<="` + version (`str`): + A version string + """ + if not _accelerate_available: + return False + return compare_versions(parse(_accelerate_version), operation, version) + + +def is_k_diffusion_version(operation: str, version: str): + """ + Args: + Compares the current k-diffusion version to a given reference with an operation. + operation (`str`): + A string representation of an operator, such as `">"` or `"<="` + version (`str`): + A version string + """ + if not _k_diffusion_available: + return False + return compare_versions(parse(_k_diffusion_version), operation, version) + + +def get_objects_from_module(module): + """ + Args: + Returns a dict of object names and values in a module, while skipping private/internal objects + module (ModuleType): + Module to extract the objects from. + + Returns: + dict: Dictionary of object names and corresponding values + """ + + objects = {} + for name in dir(module): + if name.startswith("_"): + continue + objects[name] = getattr(module, name) + + return objects + + +class OptionalDependencyNotAvailable(BaseException): + """An error indicating that an optional dependency of Diffusers was not found in the environment.""" + + +class _LazyModule(ModuleType): + """ + Module class that surfaces all objects but only performs associated imports when the objects are requested. + """ + + # Very heavily inspired by optuna.integration._IntegrationModule + # https://github.com/optuna/optuna/blob/master/optuna/integration/__init__.py + def __init__(self, name, module_file, import_structure, module_spec=None, extra_objects=None): + super().__init__(name) + self._modules = set(import_structure.keys()) + self._class_to_module = {} + for key, values in import_structure.items(): + for value in values: + self._class_to_module[value] = key + # Needed for autocompletion in an IDE + self.__all__ = list(import_structure.keys()) + list(chain(*import_structure.values())) + self.__file__ = module_file + self.__spec__ = module_spec + self.__path__ = [os.path.dirname(module_file)] + self._objects = {} if extra_objects is None else extra_objects + self._name = name + self._import_structure = import_structure + + # Needed for autocompletion in an IDE + def __dir__(self): + result = super().__dir__() + # The elements of self.__all__ that are submodules may or may not be in the dir already, depending on whether + # they have been accessed or not. So we only add the elements of self.__all__ that are not already in the dir. + for attr in self.__all__: + if attr not in result: + result.append(attr) + return result + + def __getattr__(self, name: str) -> Any: + if name in self._objects: + return self._objects[name] + if name in self._modules: + value = self._get_module(name) + elif name in self._class_to_module.keys(): + module = self._get_module(self._class_to_module[name]) + value = getattr(module, name) + else: + raise AttributeError(f"module {self.__name__} has no attribute {name}") + + setattr(self, name, value) + return value + + def _get_module(self, module_name: str): + try: + return importlib.import_module("." + module_name, self.__name__) + except Exception as e: + raise RuntimeError( + f"Failed to import {self.__name__}.{module_name} because of the following error (look up to see its" + f" traceback):\n{e}" + ) from e + + def __reduce__(self): + return (self.__class__, (self._name, self.__file__, self._import_structure)) diff --git a/diffusers-0.27.0/src/diffusers/utils/loading_utils.py b/diffusers-0.27.0/src/diffusers/utils/loading_utils.py new file mode 100755 index 0000000..18f6ead --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/loading_utils.py @@ -0,0 +1,49 @@ +import os +from typing import Callable, Union + +import PIL.Image +import PIL.ImageOps +import requests + + +def load_image( + image: Union[str, PIL.Image.Image], convert_method: Callable[[PIL.Image.Image], PIL.Image.Image] = None +) -> PIL.Image.Image: + """ + Loads `image` to a PIL Image. + + Args: + image (`str` or `PIL.Image.Image`): + The image to convert to the PIL Image format. + convert_method (Callable[[PIL.Image.Image], PIL.Image.Image], optional): + A conversion method to apply to the image after loading it. + When set to `None` the image will be converted "RGB". + + Returns: + `PIL.Image.Image`: + A PIL Image. + """ + if isinstance(image, str): + if image.startswith("http://") or image.startswith("https://"): + image = PIL.Image.open(requests.get(image, stream=True).raw) + elif os.path.isfile(image): + image = PIL.Image.open(image) + else: + raise ValueError( + f"Incorrect path or URL. URLs must start with `http://` or `https://`, and {image} is not a valid path." + ) + elif isinstance(image, PIL.Image.Image): + image = image + else: + raise ValueError( + "Incorrect format used for the image. Should be a URL linking to an image, a local path, or a PIL image." + ) + + image = PIL.ImageOps.exif_transpose(image) + + if convert_method is not None: + image = convert_method(image) + else: + image = image.convert("RGB") + + return image diff --git a/diffusers-0.27.0/src/diffusers/utils/logging.py b/diffusers-0.27.0/src/diffusers/utils/logging.py new file mode 100755 index 0000000..37bc05d --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/logging.py @@ -0,0 +1,339 @@ +# coding=utf-8 +# Copyright 2024 Optuna, Hugging Face +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" Logging utilities.""" + +import logging +import os +import sys +import threading +from logging import ( + CRITICAL, # NOQA + DEBUG, # NOQA + ERROR, # NOQA + FATAL, # NOQA + INFO, # NOQA + NOTSET, # NOQA + WARN, # NOQA + WARNING, # NOQA +) +from typing import Dict, Optional + +from tqdm import auto as tqdm_lib + + +_lock = threading.Lock() +_default_handler: Optional[logging.Handler] = None + +log_levels = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, +} + +_default_log_level = logging.WARNING + +_tqdm_active = True + + +def _get_default_logging_level() -> int: + """ + If DIFFUSERS_VERBOSITY env var is set to one of the valid choices return that as the new default level. If it is + not - fall back to `_default_log_level` + """ + env_level_str = os.getenv("DIFFUSERS_VERBOSITY", None) + if env_level_str: + if env_level_str in log_levels: + return log_levels[env_level_str] + else: + logging.getLogger().warning( + f"Unknown option DIFFUSERS_VERBOSITY={env_level_str}, " + f"has to be one of: { ', '.join(log_levels.keys()) }" + ) + return _default_log_level + + +def _get_library_name() -> str: + return __name__.split(".")[0] + + +def _get_library_root_logger() -> logging.Logger: + return logging.getLogger(_get_library_name()) + + +def _configure_library_root_logger() -> None: + global _default_handler + + with _lock: + if _default_handler: + # This library has already configured the library root logger. + return + _default_handler = logging.StreamHandler() # Set sys.stderr as stream. + _default_handler.flush = sys.stderr.flush + + # Apply our default configuration to the library root logger. + library_root_logger = _get_library_root_logger() + library_root_logger.addHandler(_default_handler) + library_root_logger.setLevel(_get_default_logging_level()) + library_root_logger.propagate = False + + +def _reset_library_root_logger() -> None: + global _default_handler + + with _lock: + if not _default_handler: + return + + library_root_logger = _get_library_root_logger() + library_root_logger.removeHandler(_default_handler) + library_root_logger.setLevel(logging.NOTSET) + _default_handler = None + + +def get_log_levels_dict() -> Dict[str, int]: + return log_levels + + +def get_logger(name: Optional[str] = None) -> logging.Logger: + """ + Return a logger with the specified name. + + This function is not supposed to be directly accessed unless you are writing a custom diffusers module. + """ + + if name is None: + name = _get_library_name() + + _configure_library_root_logger() + return logging.getLogger(name) + + +def get_verbosity() -> int: + """ + Return the current level for the 🤗 Diffusers' root logger as an `int`. + + Returns: + `int`: + Logging level integers which can be one of: + + - `50`: `diffusers.logging.CRITICAL` or `diffusers.logging.FATAL` + - `40`: `diffusers.logging.ERROR` + - `30`: `diffusers.logging.WARNING` or `diffusers.logging.WARN` + - `20`: `diffusers.logging.INFO` + - `10`: `diffusers.logging.DEBUG` + + """ + + _configure_library_root_logger() + return _get_library_root_logger().getEffectiveLevel() + + +def set_verbosity(verbosity: int) -> None: + """ + Set the verbosity level for the 🤗 Diffusers' root logger. + + Args: + verbosity (`int`): + Logging level which can be one of: + + - `diffusers.logging.CRITICAL` or `diffusers.logging.FATAL` + - `diffusers.logging.ERROR` + - `diffusers.logging.WARNING` or `diffusers.logging.WARN` + - `diffusers.logging.INFO` + - `diffusers.logging.DEBUG` + """ + + _configure_library_root_logger() + _get_library_root_logger().setLevel(verbosity) + + +def set_verbosity_info() -> None: + """Set the verbosity to the `INFO` level.""" + return set_verbosity(INFO) + + +def set_verbosity_warning() -> None: + """Set the verbosity to the `WARNING` level.""" + return set_verbosity(WARNING) + + +def set_verbosity_debug() -> None: + """Set the verbosity to the `DEBUG` level.""" + return set_verbosity(DEBUG) + + +def set_verbosity_error() -> None: + """Set the verbosity to the `ERROR` level.""" + return set_verbosity(ERROR) + + +def disable_default_handler() -> None: + """Disable the default handler of the 🤗 Diffusers' root logger.""" + + _configure_library_root_logger() + + assert _default_handler is not None + _get_library_root_logger().removeHandler(_default_handler) + + +def enable_default_handler() -> None: + """Enable the default handler of the 🤗 Diffusers' root logger.""" + + _configure_library_root_logger() + + assert _default_handler is not None + _get_library_root_logger().addHandler(_default_handler) + + +def add_handler(handler: logging.Handler) -> None: + """adds a handler to the HuggingFace Diffusers' root logger.""" + + _configure_library_root_logger() + + assert handler is not None + _get_library_root_logger().addHandler(handler) + + +def remove_handler(handler: logging.Handler) -> None: + """removes given handler from the HuggingFace Diffusers' root logger.""" + + _configure_library_root_logger() + + assert handler is not None and handler in _get_library_root_logger().handlers + _get_library_root_logger().removeHandler(handler) + + +def disable_propagation() -> None: + """ + Disable propagation of the library log outputs. Note that log propagation is disabled by default. + """ + + _configure_library_root_logger() + _get_library_root_logger().propagate = False + + +def enable_propagation() -> None: + """ + Enable propagation of the library log outputs. Please disable the HuggingFace Diffusers' default handler to prevent + double logging if the root logger has been configured. + """ + + _configure_library_root_logger() + _get_library_root_logger().propagate = True + + +def enable_explicit_format() -> None: + """ + Enable explicit formatting for every 🤗 Diffusers' logger. The explicit formatter is as follows: + ``` + [LEVELNAME|FILENAME|LINE NUMBER] TIME >> MESSAGE + ``` + All handlers currently bound to the root logger are affected by this method. + """ + handlers = _get_library_root_logger().handlers + + for handler in handlers: + formatter = logging.Formatter("[%(levelname)s|%(filename)s:%(lineno)s] %(asctime)s >> %(message)s") + handler.setFormatter(formatter) + + +def reset_format() -> None: + """ + Resets the formatting for 🤗 Diffusers' loggers. + + All handlers currently bound to the root logger are affected by this method. + """ + handlers = _get_library_root_logger().handlers + + for handler in handlers: + handler.setFormatter(None) + + +def warning_advice(self, *args, **kwargs) -> None: + """ + This method is identical to `logger.warning()`, but if env var DIFFUSERS_NO_ADVISORY_WARNINGS=1 is set, this + warning will not be printed + """ + no_advisory_warnings = os.getenv("DIFFUSERS_NO_ADVISORY_WARNINGS", False) + if no_advisory_warnings: + return + self.warning(*args, **kwargs) + + +logging.Logger.warning_advice = warning_advice + + +class EmptyTqdm: + """Dummy tqdm which doesn't do anything.""" + + def __init__(self, *args, **kwargs): # pylint: disable=unused-argument + self._iterator = args[0] if args else None + + def __iter__(self): + return iter(self._iterator) + + def __getattr__(self, _): + """Return empty function.""" + + def empty_fn(*args, **kwargs): # pylint: disable=unused-argument + return + + return empty_fn + + def __enter__(self): + return self + + def __exit__(self, type_, value, traceback): + return + + +class _tqdm_cls: + def __call__(self, *args, **kwargs): + if _tqdm_active: + return tqdm_lib.tqdm(*args, **kwargs) + else: + return EmptyTqdm(*args, **kwargs) + + def set_lock(self, *args, **kwargs): + self._lock = None + if _tqdm_active: + return tqdm_lib.tqdm.set_lock(*args, **kwargs) + + def get_lock(self): + if _tqdm_active: + return tqdm_lib.tqdm.get_lock() + + +tqdm = _tqdm_cls() + + +def is_progress_bar_enabled() -> bool: + """Return a boolean indicating whether tqdm progress bars are enabled.""" + global _tqdm_active + return bool(_tqdm_active) + + +def enable_progress_bar() -> None: + """Enable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = True + + +def disable_progress_bar() -> None: + """Disable tqdm progress bar.""" + global _tqdm_active + _tqdm_active = False diff --git a/diffusers-0.27.0/src/diffusers/utils/model_card_template.md b/diffusers-0.27.0/src/diffusers/utils/model_card_template.md new file mode 100755 index 0000000..f41b71e --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/model_card_template.md @@ -0,0 +1,24 @@ +--- +{{ card_data }} +--- + + + +{{ model_description }} + +## Intended uses & limitations + +#### How to use + +```python +# TODO: add an example code snippet for running this diffusion pipeline +``` + +#### Limitations and bias + +[TODO: provide examples of latent issues and potential remediations] + +## Training details + +[TODO: describe the data used to train the model] diff --git a/diffusers-0.27.0/src/diffusers/utils/outputs.py b/diffusers-0.27.0/src/diffusers/utils/outputs.py new file mode 100755 index 0000000..6080a86 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/outputs.py @@ -0,0 +1,137 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Generic utilities +""" + +from collections import OrderedDict +from dataclasses import fields, is_dataclass +from typing import Any, Tuple + +import numpy as np + +from .import_utils import is_torch_available, is_torch_version + + +def is_tensor(x) -> bool: + """ + Tests if `x` is a `torch.Tensor` or `np.ndarray`. + """ + if is_torch_available(): + import torch + + if isinstance(x, torch.Tensor): + return True + + return isinstance(x, np.ndarray) + + +class BaseOutput(OrderedDict): + """ + Base class for all model outputs as dataclass. Has a `__getitem__` that allows indexing by integer or slice (like a + tuple) or strings (like a dictionary) that will ignore the `None` attributes. Otherwise behaves like a regular + Python dictionary. + + + + You can't unpack a [`BaseOutput`] directly. Use the [`~utils.BaseOutput.to_tuple`] method to convert it to a tuple + first. + + + """ + + def __init_subclass__(cls) -> None: + """Register subclasses as pytree nodes. + + This is necessary to synchronize gradients when using `torch.nn.parallel.DistributedDataParallel` with + `static_graph=True` with modules that output `ModelOutput` subclasses. + """ + if is_torch_available(): + import torch.utils._pytree + + if is_torch_version("<", "2.2"): + torch.utils._pytree._register_pytree_node( + cls, + torch.utils._pytree._dict_flatten, + lambda values, context: cls(**torch.utils._pytree._dict_unflatten(values, context)), + ) + else: + torch.utils._pytree.register_pytree_node( + cls, + torch.utils._pytree._dict_flatten, + lambda values, context: cls(**torch.utils._pytree._dict_unflatten(values, context)), + ) + + def __post_init__(self) -> None: + class_fields = fields(self) + + # Safety and consistency checks + if not len(class_fields): + raise ValueError(f"{self.__class__.__name__} has no fields.") + + first_field = getattr(self, class_fields[0].name) + other_fields_are_none = all(getattr(self, field.name) is None for field in class_fields[1:]) + + if other_fields_are_none and isinstance(first_field, dict): + for key, value in first_field.items(): + self[key] = value + else: + for field in class_fields: + v = getattr(self, field.name) + if v is not None: + self[field.name] = v + + def __delitem__(self, *args, **kwargs): + raise Exception(f"You cannot use ``__delitem__`` on a {self.__class__.__name__} instance.") + + def setdefault(self, *args, **kwargs): + raise Exception(f"You cannot use ``setdefault`` on a {self.__class__.__name__} instance.") + + def pop(self, *args, **kwargs): + raise Exception(f"You cannot use ``pop`` on a {self.__class__.__name__} instance.") + + def update(self, *args, **kwargs): + raise Exception(f"You cannot use ``update`` on a {self.__class__.__name__} instance.") + + def __getitem__(self, k: Any) -> Any: + if isinstance(k, str): + inner_dict = dict(self.items()) + return inner_dict[k] + else: + return self.to_tuple()[k] + + def __setattr__(self, name: Any, value: Any) -> None: + if name in self.keys() and value is not None: + # Don't call self.__setitem__ to avoid recursion errors + super().__setitem__(name, value) + super().__setattr__(name, value) + + def __setitem__(self, key, value): + # Will raise a KeyException if needed + super().__setitem__(key, value) + # Don't call self.__setattr__ to avoid recursion errors + super().__setattr__(key, value) + + def __reduce__(self): + if not is_dataclass(self): + return super().__reduce__() + callable, _args, *remaining = super().__reduce__() + args = tuple(getattr(self, field.name) for field in fields(self)) + return callable, args, *remaining + + def to_tuple(self) -> Tuple[Any, ...]: + """ + Convert self to a tuple containing all the attributes/keys that are not `None`. + """ + return tuple(self[k] for k in self.keys()) diff --git a/diffusers-0.27.0/src/diffusers/utils/peft_utils.py b/diffusers-0.27.0/src/diffusers/utils/peft_utils.py new file mode 100755 index 0000000..85d16c7 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/peft_utils.py @@ -0,0 +1,268 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +PEFT utilities: Utilities related to peft library +""" +import collections +import importlib +from typing import Optional + +from packaging import version + +from .import_utils import is_peft_available, is_torch_available + + +if is_torch_available(): + import torch + + +def recurse_remove_peft_layers(model): + r""" + Recursively replace all instances of `LoraLayer` with corresponding new layers in `model`. + """ + from peft.tuners.tuners_utils import BaseTunerLayer + + has_base_layer_pattern = False + for module in model.modules(): + if isinstance(module, BaseTunerLayer): + has_base_layer_pattern = hasattr(module, "base_layer") + break + + if has_base_layer_pattern: + from peft.utils import _get_submodules + + key_list = [key for key, _ in model.named_modules() if "lora" not in key] + for key in key_list: + try: + parent, target, target_name = _get_submodules(model, key) + except AttributeError: + continue + if hasattr(target, "base_layer"): + setattr(parent, target_name, target.get_base_layer()) + else: + # This is for backwards compatibility with PEFT <= 0.6.2. + # TODO can be removed once that PEFT version is no longer supported. + from peft.tuners.lora import LoraLayer + + for name, module in model.named_children(): + if len(list(module.children())) > 0: + ## compound module, go inside it + recurse_remove_peft_layers(module) + + module_replaced = False + + if isinstance(module, LoraLayer) and isinstance(module, torch.nn.Linear): + new_module = torch.nn.Linear(module.in_features, module.out_features, bias=module.bias is not None).to( + module.weight.device + ) + new_module.weight = module.weight + if module.bias is not None: + new_module.bias = module.bias + + module_replaced = True + elif isinstance(module, LoraLayer) and isinstance(module, torch.nn.Conv2d): + new_module = torch.nn.Conv2d( + module.in_channels, + module.out_channels, + module.kernel_size, + module.stride, + module.padding, + module.dilation, + module.groups, + ).to(module.weight.device) + + new_module.weight = module.weight + if module.bias is not None: + new_module.bias = module.bias + + module_replaced = True + + if module_replaced: + setattr(model, name, new_module) + del module + + if torch.cuda.is_available(): + torch.cuda.empty_cache() + return model + + +def scale_lora_layers(model, weight): + """ + Adjust the weightage given to the LoRA layers of the model. + + Args: + model (`torch.nn.Module`): + The model to scale. + weight (`float`): + The weight to be given to the LoRA layers. + """ + from peft.tuners.tuners_utils import BaseTunerLayer + + for module in model.modules(): + if isinstance(module, BaseTunerLayer): + module.scale_layer(weight) + + +def unscale_lora_layers(model, weight: Optional[float] = None): + """ + Removes the previously passed weight given to the LoRA layers of the model. + + Args: + model (`torch.nn.Module`): + The model to scale. + weight (`float`, *optional*): + The weight to be given to the LoRA layers. If no scale is passed the scale of the lora layer will be + re-initialized to the correct value. If 0.0 is passed, we will re-initialize the scale with the correct + value. + """ + from peft.tuners.tuners_utils import BaseTunerLayer + + for module in model.modules(): + if isinstance(module, BaseTunerLayer): + if weight is not None and weight != 0: + module.unscale_layer(weight) + elif weight is not None and weight == 0: + for adapter_name in module.active_adapters: + # if weight == 0 unscale should re-set the scale to the original value. + module.set_scale(adapter_name, 1.0) + + +def get_peft_kwargs(rank_dict, network_alpha_dict, peft_state_dict, is_unet=True): + rank_pattern = {} + alpha_pattern = {} + r = lora_alpha = list(rank_dict.values())[0] + + if len(set(rank_dict.values())) > 1: + # get the rank occuring the most number of times + r = collections.Counter(rank_dict.values()).most_common()[0][0] + + # for modules with rank different from the most occuring rank, add it to the `rank_pattern` + rank_pattern = dict(filter(lambda x: x[1] != r, rank_dict.items())) + rank_pattern = {k.split(".lora_B.")[0]: v for k, v in rank_pattern.items()} + + if network_alpha_dict is not None and len(network_alpha_dict) > 0: + if len(set(network_alpha_dict.values())) > 1: + # get the alpha occuring the most number of times + lora_alpha = collections.Counter(network_alpha_dict.values()).most_common()[0][0] + + # for modules with alpha different from the most occuring alpha, add it to the `alpha_pattern` + alpha_pattern = dict(filter(lambda x: x[1] != lora_alpha, network_alpha_dict.items())) + if is_unet: + alpha_pattern = { + ".".join(k.split(".lora_A.")[0].split(".")).replace(".alpha", ""): v + for k, v in alpha_pattern.items() + } + else: + alpha_pattern = {".".join(k.split(".down.")[0].split(".")[:-1]): v for k, v in alpha_pattern.items()} + else: + lora_alpha = set(network_alpha_dict.values()).pop() + + # layer names without the Diffusers specific + target_modules = list({name.split(".lora")[0] for name in peft_state_dict.keys()}) + + lora_config_kwargs = { + "r": r, + "lora_alpha": lora_alpha, + "rank_pattern": rank_pattern, + "alpha_pattern": alpha_pattern, + "target_modules": target_modules, + } + return lora_config_kwargs + + +def get_adapter_name(model): + from peft.tuners.tuners_utils import BaseTunerLayer + + for module in model.modules(): + if isinstance(module, BaseTunerLayer): + return f"default_{len(module.r)}" + return "default_0" + + +def set_adapter_layers(model, enabled=True): + from peft.tuners.tuners_utils import BaseTunerLayer + + for module in model.modules(): + if isinstance(module, BaseTunerLayer): + # The recent version of PEFT needs to call `enable_adapters` instead + if hasattr(module, "enable_adapters"): + module.enable_adapters(enabled=enabled) + else: + module.disable_adapters = not enabled + + +def delete_adapter_layers(model, adapter_name): + from peft.tuners.tuners_utils import BaseTunerLayer + + for module in model.modules(): + if isinstance(module, BaseTunerLayer): + if hasattr(module, "delete_adapter"): + module.delete_adapter(adapter_name) + else: + raise ValueError( + "The version of PEFT you are using is not compatible, please use a version that is greater than 0.6.1" + ) + + # For transformers integration - we need to pop the adapter from the config + if getattr(model, "_hf_peft_config_loaded", False) and hasattr(model, "peft_config"): + model.peft_config.pop(adapter_name, None) + # In case all adapters are deleted, we need to delete the config + # and make sure to set the flag to False + if len(model.peft_config) == 0: + del model.peft_config + model._hf_peft_config_loaded = None + + +def set_weights_and_activate_adapters(model, adapter_names, weights): + from peft.tuners.tuners_utils import BaseTunerLayer + + # iterate over each adapter, make it active and set the corresponding scaling weight + for adapter_name, weight in zip(adapter_names, weights): + for module in model.modules(): + if isinstance(module, BaseTunerLayer): + # For backward compatbility with previous PEFT versions + if hasattr(module, "set_adapter"): + module.set_adapter(adapter_name) + else: + module.active_adapter = adapter_name + module.set_scale(adapter_name, weight) + + # set multiple active adapters + for module in model.modules(): + if isinstance(module, BaseTunerLayer): + # For backward compatbility with previous PEFT versions + if hasattr(module, "set_adapter"): + module.set_adapter(adapter_names) + else: + module.active_adapter = adapter_names + + +def check_peft_version(min_version: str) -> None: + r""" + Checks if the version of PEFT is compatible. + + Args: + version (`str`): + The version of PEFT to check against. + """ + if not is_peft_available(): + raise ValueError("PEFT is not installed. Please install it with `pip install peft`") + + is_peft_version_compatible = version.parse(importlib.metadata.version("peft")) > version.parse(min_version) + + if not is_peft_version_compatible: + raise ValueError( + f"The version of PEFT you are using is not compatible, please use a version that is greater" + f" than {min_version}" + ) diff --git a/diffusers-0.27.0/src/diffusers/utils/pil_utils.py b/diffusers-0.27.0/src/diffusers/utils/pil_utils.py new file mode 100755 index 0000000..7667807 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/pil_utils.py @@ -0,0 +1,67 @@ +from typing import List + +import PIL.Image +import PIL.ImageOps +from packaging import version +from PIL import Image + + +if version.parse(version.parse(PIL.__version__).base_version) >= version.parse("9.1.0"): + PIL_INTERPOLATION = { + "linear": PIL.Image.Resampling.BILINEAR, + "bilinear": PIL.Image.Resampling.BILINEAR, + "bicubic": PIL.Image.Resampling.BICUBIC, + "lanczos": PIL.Image.Resampling.LANCZOS, + "nearest": PIL.Image.Resampling.NEAREST, + } +else: + PIL_INTERPOLATION = { + "linear": PIL.Image.LINEAR, + "bilinear": PIL.Image.BILINEAR, + "bicubic": PIL.Image.BICUBIC, + "lanczos": PIL.Image.LANCZOS, + "nearest": PIL.Image.NEAREST, + } + + +def pt_to_pil(images): + """ + Convert a torch image to a PIL image. + """ + images = (images / 2 + 0.5).clamp(0, 1) + images = images.cpu().permute(0, 2, 3, 1).float().numpy() + images = numpy_to_pil(images) + return images + + +def numpy_to_pil(images): + """ + Convert a numpy image or a batch of images to a PIL image. + """ + if images.ndim == 3: + images = images[None, ...] + images = (images * 255).round().astype("uint8") + if images.shape[-1] == 1: + # special case for grayscale (single channel) images + pil_images = [Image.fromarray(image.squeeze(), mode="L") for image in images] + else: + pil_images = [Image.fromarray(image) for image in images] + + return pil_images + + +def make_image_grid(images: List[PIL.Image.Image], rows: int, cols: int, resize: int = None) -> PIL.Image.Image: + """ + Prepares a single grid of images. Useful for visualization purposes. + """ + assert len(images) == rows * cols + + if resize is not None: + images = [img.resize((resize, resize)) for img in images] + + w, h = images[0].size + grid = Image.new("RGB", size=(cols * w, rows * h)) + + for i, img in enumerate(images): + grid.paste(img, box=(i % cols * w, i // cols * h)) + return grid diff --git a/diffusers-0.27.0/src/diffusers/utils/state_dict_utils.py b/diffusers-0.27.0/src/diffusers/utils/state_dict_utils.py new file mode 100755 index 0000000..c456663 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/state_dict_utils.py @@ -0,0 +1,324 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +State dict utilities: utility methods for converting state dicts easily +""" +import enum + +from .logging import get_logger + + +logger = get_logger(__name__) + + +class StateDictType(enum.Enum): + """ + The mode to use when converting state dicts. + """ + + DIFFUSERS_OLD = "diffusers_old" + KOHYA_SS = "kohya_ss" + PEFT = "peft" + DIFFUSERS = "diffusers" + + +# We need to define a proper mapping for Unet since it uses different output keys than text encoder +# e.g. to_q_lora -> q_proj / to_q +UNET_TO_DIFFUSERS = { + ".to_out_lora.up": ".to_out.0.lora_B", + ".to_out_lora.down": ".to_out.0.lora_A", + ".to_q_lora.down": ".to_q.lora_A", + ".to_q_lora.up": ".to_q.lora_B", + ".to_k_lora.down": ".to_k.lora_A", + ".to_k_lora.up": ".to_k.lora_B", + ".to_v_lora.down": ".to_v.lora_A", + ".to_v_lora.up": ".to_v.lora_B", + ".lora.up": ".lora_B", + ".lora.down": ".lora_A", +} + + +DIFFUSERS_TO_PEFT = { + ".q_proj.lora_linear_layer.up": ".q_proj.lora_B", + ".q_proj.lora_linear_layer.down": ".q_proj.lora_A", + ".k_proj.lora_linear_layer.up": ".k_proj.lora_B", + ".k_proj.lora_linear_layer.down": ".k_proj.lora_A", + ".v_proj.lora_linear_layer.up": ".v_proj.lora_B", + ".v_proj.lora_linear_layer.down": ".v_proj.lora_A", + ".out_proj.lora_linear_layer.up": ".out_proj.lora_B", + ".out_proj.lora_linear_layer.down": ".out_proj.lora_A", + ".lora_linear_layer.up": ".lora_B", + ".lora_linear_layer.down": ".lora_A", +} + +DIFFUSERS_OLD_TO_PEFT = { + ".to_q_lora.up": ".q_proj.lora_B", + ".to_q_lora.down": ".q_proj.lora_A", + ".to_k_lora.up": ".k_proj.lora_B", + ".to_k_lora.down": ".k_proj.lora_A", + ".to_v_lora.up": ".v_proj.lora_B", + ".to_v_lora.down": ".v_proj.lora_A", + ".to_out_lora.up": ".out_proj.lora_B", + ".to_out_lora.down": ".out_proj.lora_A", + ".lora_linear_layer.up": ".lora_B", + ".lora_linear_layer.down": ".lora_A", +} + +PEFT_TO_DIFFUSERS = { + ".q_proj.lora_B": ".q_proj.lora_linear_layer.up", + ".q_proj.lora_A": ".q_proj.lora_linear_layer.down", + ".k_proj.lora_B": ".k_proj.lora_linear_layer.up", + ".k_proj.lora_A": ".k_proj.lora_linear_layer.down", + ".v_proj.lora_B": ".v_proj.lora_linear_layer.up", + ".v_proj.lora_A": ".v_proj.lora_linear_layer.down", + ".out_proj.lora_B": ".out_proj.lora_linear_layer.up", + ".out_proj.lora_A": ".out_proj.lora_linear_layer.down", + "to_k.lora_A": "to_k.lora.down", + "to_k.lora_B": "to_k.lora.up", + "to_q.lora_A": "to_q.lora.down", + "to_q.lora_B": "to_q.lora.up", + "to_v.lora_A": "to_v.lora.down", + "to_v.lora_B": "to_v.lora.up", + "to_out.0.lora_A": "to_out.0.lora.down", + "to_out.0.lora_B": "to_out.0.lora.up", +} + +DIFFUSERS_OLD_TO_DIFFUSERS = { + ".to_q_lora.up": ".q_proj.lora_linear_layer.up", + ".to_q_lora.down": ".q_proj.lora_linear_layer.down", + ".to_k_lora.up": ".k_proj.lora_linear_layer.up", + ".to_k_lora.down": ".k_proj.lora_linear_layer.down", + ".to_v_lora.up": ".v_proj.lora_linear_layer.up", + ".to_v_lora.down": ".v_proj.lora_linear_layer.down", + ".to_out_lora.up": ".out_proj.lora_linear_layer.up", + ".to_out_lora.down": ".out_proj.lora_linear_layer.down", +} + +PEFT_TO_KOHYA_SS = { + "lora_A": "lora_down", + "lora_B": "lora_up", + # This is not a comprehensive dict as kohya format requires replacing `.` with `_` in keys, + # adding prefixes and adding alpha values + # Check `convert_state_dict_to_kohya` for more +} + +PEFT_STATE_DICT_MAPPINGS = { + StateDictType.DIFFUSERS_OLD: DIFFUSERS_OLD_TO_PEFT, + StateDictType.DIFFUSERS: DIFFUSERS_TO_PEFT, +} + +DIFFUSERS_STATE_DICT_MAPPINGS = { + StateDictType.DIFFUSERS_OLD: DIFFUSERS_OLD_TO_DIFFUSERS, + StateDictType.PEFT: PEFT_TO_DIFFUSERS, +} + +KOHYA_STATE_DICT_MAPPINGS = {StateDictType.PEFT: PEFT_TO_KOHYA_SS} + +KEYS_TO_ALWAYS_REPLACE = { + ".processor.": ".", +} + + +def convert_state_dict(state_dict, mapping): + r""" + Simply iterates over the state dict and replaces the patterns in `mapping` with the corresponding values. + + Args: + state_dict (`dict[str, torch.Tensor]`): + The state dict to convert. + mapping (`dict[str, str]`): + The mapping to use for conversion, the mapping should be a dictionary with the following structure: + - key: the pattern to replace + - value: the pattern to replace with + + Returns: + converted_state_dict (`dict`) + The converted state dict. + """ + converted_state_dict = {} + for k, v in state_dict.items(): + # First, filter out the keys that we always want to replace + for pattern in KEYS_TO_ALWAYS_REPLACE.keys(): + if pattern in k: + new_pattern = KEYS_TO_ALWAYS_REPLACE[pattern] + k = k.replace(pattern, new_pattern) + + for pattern in mapping.keys(): + if pattern in k: + new_pattern = mapping[pattern] + k = k.replace(pattern, new_pattern) + break + converted_state_dict[k] = v + return converted_state_dict + + +def convert_state_dict_to_peft(state_dict, original_type=None, **kwargs): + r""" + Converts a state dict to the PEFT format The state dict can be from previous diffusers format (`OLD_DIFFUSERS`), or + new diffusers format (`DIFFUSERS`). The method only supports the conversion from diffusers old/new to PEFT for now. + + Args: + state_dict (`dict[str, torch.Tensor]`): + The state dict to convert. + original_type (`StateDictType`, *optional*): + The original type of the state dict, if not provided, the method will try to infer it automatically. + """ + if original_type is None: + # Old diffusers to PEFT + if any("to_out_lora" in k for k in state_dict.keys()): + original_type = StateDictType.DIFFUSERS_OLD + elif any("lora_linear_layer" in k for k in state_dict.keys()): + original_type = StateDictType.DIFFUSERS + else: + raise ValueError("Could not automatically infer state dict type") + + if original_type not in PEFT_STATE_DICT_MAPPINGS.keys(): + raise ValueError(f"Original type {original_type} is not supported") + + mapping = PEFT_STATE_DICT_MAPPINGS[original_type] + return convert_state_dict(state_dict, mapping) + + +def convert_state_dict_to_diffusers(state_dict, original_type=None, **kwargs): + r""" + Converts a state dict to new diffusers format. The state dict can be from previous diffusers format + (`OLD_DIFFUSERS`), or PEFT format (`PEFT`) or new diffusers format (`DIFFUSERS`). In the last case the method will + return the state dict as is. + + The method only supports the conversion from diffusers old, PEFT to diffusers new for now. + + Args: + state_dict (`dict[str, torch.Tensor]`): + The state dict to convert. + original_type (`StateDictType`, *optional*): + The original type of the state dict, if not provided, the method will try to infer it automatically. + kwargs (`dict`, *args*): + Additional arguments to pass to the method. + + - **adapter_name**: For example, in case of PEFT, some keys will be pre-pended + with the adapter name, therefore needs a special handling. By default PEFT also takes care of that in + `get_peft_model_state_dict` method: + https://github.com/huggingface/peft/blob/ba0477f2985b1ba311b83459d29895c809404e99/src/peft/utils/save_and_load.py#L92 + but we add it here in case we don't want to rely on that method. + """ + peft_adapter_name = kwargs.pop("adapter_name", None) + if peft_adapter_name is not None: + peft_adapter_name = "." + peft_adapter_name + else: + peft_adapter_name = "" + + if original_type is None: + # Old diffusers to PEFT + if any("to_out_lora" in k for k in state_dict.keys()): + original_type = StateDictType.DIFFUSERS_OLD + elif any(f".lora_A{peft_adapter_name}.weight" in k for k in state_dict.keys()): + original_type = StateDictType.PEFT + elif any("lora_linear_layer" in k for k in state_dict.keys()): + # nothing to do + return state_dict + else: + raise ValueError("Could not automatically infer state dict type") + + if original_type not in DIFFUSERS_STATE_DICT_MAPPINGS.keys(): + raise ValueError(f"Original type {original_type} is not supported") + + mapping = DIFFUSERS_STATE_DICT_MAPPINGS[original_type] + return convert_state_dict(state_dict, mapping) + + +def convert_unet_state_dict_to_peft(state_dict): + r""" + Converts a state dict from UNet format to diffusers format - i.e. by removing some keys + """ + mapping = UNET_TO_DIFFUSERS + return convert_state_dict(state_dict, mapping) + + +def convert_all_state_dict_to_peft(state_dict): + r""" + Attempts to first `convert_state_dict_to_peft`, and if it doesn't detect `lora_linear_layer` + for a valid `DIFFUSERS` LoRA for example, attempts to exclusively convert the Unet `convert_unet_state_dict_to_peft` + """ + try: + peft_dict = convert_state_dict_to_peft(state_dict) + except Exception as e: + if str(e) == "Could not automatically infer state dict type": + peft_dict = convert_unet_state_dict_to_peft(state_dict) + else: + raise + + if not any("lora_A" in key or "lora_B" in key for key in peft_dict.keys()): + raise ValueError("Your LoRA was not converted to PEFT") + + return peft_dict + + +def convert_state_dict_to_kohya(state_dict, original_type=None, **kwargs): + r""" + Converts a `PEFT` state dict to `Kohya` format that can be used in AUTOMATIC1111, ComfyUI, SD.Next, InvokeAI, etc. + The method only supports the conversion from PEFT to Kohya for now. + + Args: + state_dict (`dict[str, torch.Tensor]`): + The state dict to convert. + original_type (`StateDictType`, *optional*): + The original type of the state dict, if not provided, the method will try to infer it automatically. + kwargs (`dict`, *args*): + Additional arguments to pass to the method. + + - **adapter_name**: For example, in case of PEFT, some keys will be pre-pended + with the adapter name, therefore needs a special handling. By default PEFT also takes care of that in + `get_peft_model_state_dict` method: + https://github.com/huggingface/peft/blob/ba0477f2985b1ba311b83459d29895c809404e99/src/peft/utils/save_and_load.py#L92 + but we add it here in case we don't want to rely on that method. + """ + try: + import torch + except ImportError: + logger.error("Converting PEFT state dicts to Kohya requires torch to be installed.") + raise + + peft_adapter_name = kwargs.pop("adapter_name", None) + if peft_adapter_name is not None: + peft_adapter_name = "." + peft_adapter_name + else: + peft_adapter_name = "" + + if original_type is None: + if any(f".lora_A{peft_adapter_name}.weight" in k for k in state_dict.keys()): + original_type = StateDictType.PEFT + + if original_type not in KOHYA_STATE_DICT_MAPPINGS.keys(): + raise ValueError(f"Original type {original_type} is not supported") + + # Use the convert_state_dict function with the appropriate mapping + kohya_ss_partial_state_dict = convert_state_dict(state_dict, KOHYA_STATE_DICT_MAPPINGS[StateDictType.PEFT]) + kohya_ss_state_dict = {} + + # Additional logic for replacing header, alpha parameters `.` with `_` in all keys + for kohya_key, weight in kohya_ss_partial_state_dict.items(): + if "text_encoder_2." in kohya_key: + kohya_key = kohya_key.replace("text_encoder_2.", "lora_te2.") + elif "text_encoder." in kohya_key: + kohya_key = kohya_key.replace("text_encoder.", "lora_te1.") + elif "unet" in kohya_key: + kohya_key = kohya_key.replace("unet", "lora_unet") + kohya_key = kohya_key.replace(".", "_", kohya_key.count(".") - 2) + kohya_key = kohya_key.replace(peft_adapter_name, "") # Kohya doesn't take names + kohya_ss_state_dict[kohya_key] = weight + if "lora_down" in kohya_key: + alpha_key = f'{kohya_key.split(".")[0]}.alpha' + kohya_ss_state_dict[alpha_key] = torch.tensor(len(weight)) + + return kohya_ss_state_dict diff --git a/diffusers-0.27.0/src/diffusers/utils/testing_utils.py b/diffusers-0.27.0/src/diffusers/utils/testing_utils.py new file mode 100755 index 0000000..edbf6f3 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/testing_utils.py @@ -0,0 +1,967 @@ +import functools +import importlib +import inspect +import io +import logging +import multiprocessing +import os +import random +import re +import struct +import sys +import tempfile +import time +import unittest +import urllib.parse +from contextlib import contextmanager +from distutils.util import strtobool +from io import BytesIO, StringIO +from pathlib import Path +from typing import Callable, Dict, List, Optional, Union + +import numpy as np +import PIL.Image +import PIL.ImageOps +import requests +from numpy.linalg import norm +from packaging import version + +from .import_utils import ( + BACKENDS_MAPPING, + is_compel_available, + is_flax_available, + is_note_seq_available, + is_onnx_available, + is_opencv_available, + is_peft_available, + is_torch_available, + is_torch_version, + is_torchsde_available, + is_transformers_available, +) +from .logging import get_logger + + +global_rng = random.Random() + +logger = get_logger(__name__) + +_required_peft_version = is_peft_available() and version.parse( + version.parse(importlib.metadata.version("peft")).base_version +) > version.parse("0.5") +_required_transformers_version = is_transformers_available() and version.parse( + version.parse(importlib.metadata.version("transformers")).base_version +) > version.parse("4.33") + +USE_PEFT_BACKEND = _required_peft_version and _required_transformers_version + +if is_torch_available(): + import torch + + # Set a backend environment variable for any extra module import required for a custom accelerator + if "DIFFUSERS_TEST_BACKEND" in os.environ: + backend = os.environ["DIFFUSERS_TEST_BACKEND"] + try: + _ = importlib.import_module(backend) + except ModuleNotFoundError as e: + raise ModuleNotFoundError( + f"Failed to import `DIFFUSERS_TEST_BACKEND` '{backend}'! This should be the name of an installed module \ + to enable a specified backend.):\n{e}" + ) from e + + if "DIFFUSERS_TEST_DEVICE" in os.environ: + torch_device = os.environ["DIFFUSERS_TEST_DEVICE"] + try: + # try creating device to see if provided device is valid + _ = torch.device(torch_device) + except RuntimeError as e: + raise RuntimeError( + f"Unknown testing device specified by environment variable `DIFFUSERS_TEST_DEVICE`: {torch_device}" + ) from e + logger.info(f"torch_device overrode to {torch_device}") + else: + torch_device = "cuda" if torch.cuda.is_available() else "cpu" + is_torch_higher_equal_than_1_12 = version.parse( + version.parse(torch.__version__).base_version + ) >= version.parse("1.12") + + if is_torch_higher_equal_than_1_12: + # Some builds of torch 1.12 don't have the mps backend registered. See #892 for more details + mps_backend_registered = hasattr(torch.backends, "mps") + torch_device = "mps" if (mps_backend_registered and torch.backends.mps.is_available()) else torch_device + + +def torch_all_close(a, b, *args, **kwargs): + if not is_torch_available(): + raise ValueError("PyTorch needs to be installed to use this function.") + if not torch.allclose(a, b, *args, **kwargs): + assert False, f"Max diff is absolute {(a - b).abs().max()}. Diff tensor is {(a - b).abs()}." + return True + + +def numpy_cosine_similarity_distance(a, b): + similarity = np.dot(a, b) / (norm(a) * norm(b)) + distance = 1.0 - similarity.mean() + + return distance + + +def print_tensor_test(tensor, filename="test_corrections.txt", expected_tensor_name="expected_slice"): + test_name = os.environ.get("PYTEST_CURRENT_TEST") + if not torch.is_tensor(tensor): + tensor = torch.from_numpy(tensor) + + tensor_str = str(tensor.detach().cpu().flatten().to(torch.float32)).replace("\n", "") + # format is usually: + # expected_slice = np.array([-0.5713, -0.3018, -0.9814, 0.04663, -0.879, 0.76, -1.734, 0.1044, 1.161]) + output_str = tensor_str.replace("tensor", f"{expected_tensor_name} = np.array") + test_file, test_class, test_fn = test_name.split("::") + test_fn = test_fn.split()[0] + with open(filename, "a") as f: + print(";".join([test_file, test_class, test_fn, output_str]), file=f) + + +def get_tests_dir(append_path=None): + """ + Args: + append_path: optional path to append to the tests dir path + Return: + The full path to the `tests` dir, so that the tests can be invoked from anywhere. Optionally `append_path` is + joined after the `tests` dir the former is provided. + """ + # this function caller's __file__ + caller__file__ = inspect.stack()[1][1] + tests_dir = os.path.abspath(os.path.dirname(caller__file__)) + + while not tests_dir.endswith("tests"): + tests_dir = os.path.dirname(tests_dir) + + if append_path: + return Path(tests_dir, append_path).as_posix() + else: + return tests_dir + + +def parse_flag_from_env(key, default=False): + try: + value = os.environ[key] + except KeyError: + # KEY isn't set, default to `default`. + _value = default + else: + # KEY is set, convert it to True or False. + try: + _value = strtobool(value) + except ValueError: + # More values are supported, but let's keep the message simple. + raise ValueError(f"If set, {key} must be yes or no.") + return _value + + +_run_slow_tests = parse_flag_from_env("RUN_SLOW", default=False) +_run_nightly_tests = parse_flag_from_env("RUN_NIGHTLY", default=False) + + +def floats_tensor(shape, scale=1.0, rng=None, name=None): + """Creates a random float32 tensor""" + if rng is None: + rng = global_rng + + total_dims = 1 + for dim in shape: + total_dims *= dim + + values = [] + for _ in range(total_dims): + values.append(rng.random() * scale) + + return torch.tensor(data=values, dtype=torch.float).view(shape).contiguous() + + +def slow(test_case): + """ + Decorator marking a test as slow. + + Slow tests are skipped by default. Set the RUN_SLOW environment variable to a truthy value to run them. + + """ + return unittest.skipUnless(_run_slow_tests, "test is slow")(test_case) + + +def nightly(test_case): + """ + Decorator marking a test that runs nightly in the diffusers CI. + + Slow tests are skipped by default. Set the RUN_NIGHTLY environment variable to a truthy value to run them. + + """ + return unittest.skipUnless(_run_nightly_tests, "test is nightly")(test_case) + + +def require_torch(test_case): + """ + Decorator marking a test that requires PyTorch. These tests are skipped when PyTorch isn't installed. + """ + return unittest.skipUnless(is_torch_available(), "test requires PyTorch")(test_case) + + +def require_torch_2(test_case): + """ + Decorator marking a test that requires PyTorch 2. These tests are skipped when it isn't installed. + """ + return unittest.skipUnless(is_torch_available() and is_torch_version(">=", "2.0.0"), "test requires PyTorch 2")( + test_case + ) + + +def require_torch_gpu(test_case): + """Decorator marking a test that requires CUDA and PyTorch.""" + return unittest.skipUnless(is_torch_available() and torch_device == "cuda", "test requires PyTorch+CUDA")( + test_case + ) + + +# These decorators are for accelerator-specific behaviours that are not GPU-specific +def require_torch_accelerator(test_case): + """Decorator marking a test that requires an accelerator backend and PyTorch.""" + return unittest.skipUnless(is_torch_available() and torch_device != "cpu", "test requires accelerator+PyTorch")( + test_case + ) + + +def require_torch_accelerator_with_fp16(test_case): + """Decorator marking a test that requires an accelerator with support for the FP16 data type.""" + return unittest.skipUnless(_is_torch_fp16_available(torch_device), "test requires accelerator with fp16 support")( + test_case + ) + + +def require_torch_accelerator_with_fp64(test_case): + """Decorator marking a test that requires an accelerator with support for the FP64 data type.""" + return unittest.skipUnless(_is_torch_fp64_available(torch_device), "test requires accelerator with fp64 support")( + test_case + ) + + +def require_torch_accelerator_with_training(test_case): + """Decorator marking a test that requires an accelerator with support for training.""" + return unittest.skipUnless( + is_torch_available() and backend_supports_training(torch_device), + "test requires accelerator with training support", + )(test_case) + + +def skip_mps(test_case): + """Decorator marking a test to skip if torch_device is 'mps'""" + return unittest.skipUnless(torch_device != "mps", "test requires non 'mps' device")(test_case) + + +def require_flax(test_case): + """ + Decorator marking a test that requires JAX & Flax. These tests are skipped when one / both are not installed + """ + return unittest.skipUnless(is_flax_available(), "test requires JAX & Flax")(test_case) + + +def require_compel(test_case): + """ + Decorator marking a test that requires compel: https://github.com/damian0815/compel. These tests are skipped when + the library is not installed. + """ + return unittest.skipUnless(is_compel_available(), "test requires compel")(test_case) + + +def require_onnxruntime(test_case): + """ + Decorator marking a test that requires onnxruntime. These tests are skipped when onnxruntime isn't installed. + """ + return unittest.skipUnless(is_onnx_available(), "test requires onnxruntime")(test_case) + + +def require_note_seq(test_case): + """ + Decorator marking a test that requires note_seq. These tests are skipped when note_seq isn't installed. + """ + return unittest.skipUnless(is_note_seq_available(), "test requires note_seq")(test_case) + + +def require_torchsde(test_case): + """ + Decorator marking a test that requires torchsde. These tests are skipped when torchsde isn't installed. + """ + return unittest.skipUnless(is_torchsde_available(), "test requires torchsde")(test_case) + + +def require_peft_backend(test_case): + """ + Decorator marking a test that requires PEFT backend, this would require some specific versions of PEFT and + transformers. + """ + return unittest.skipUnless(USE_PEFT_BACKEND, "test requires PEFT backend")(test_case) + + +def require_peft_version_greater(peft_version): + """ + Decorator marking a test that requires PEFT backend with a specific version, this would require some specific + versions of PEFT and transformers. + """ + + def decorator(test_case): + correct_peft_version = is_peft_available() and version.parse( + version.parse(importlib.metadata.version("peft")).base_version + ) > version.parse(peft_version) + return unittest.skipUnless( + correct_peft_version, f"test requires PEFT backend with the version greater than {peft_version}" + )(test_case) + + return decorator + + +def deprecate_after_peft_backend(test_case): + """ + Decorator marking a test that will be skipped after PEFT backend + """ + return unittest.skipUnless(not USE_PEFT_BACKEND, "test skipped in favor of PEFT backend")(test_case) + + +def require_python39_or_higher(test_case): + def python39_available(): + sys_info = sys.version_info + major, minor = sys_info.major, sys_info.minor + return major == 3 and minor >= 9 + + return unittest.skipUnless(python39_available(), "test requires Python 3.9 or higher")(test_case) + + +def load_numpy(arry: Union[str, np.ndarray], local_path: Optional[str] = None) -> np.ndarray: + if isinstance(arry, str): + if local_path is not None: + # local_path can be passed to correct images of tests + return Path(local_path, arry.split("/")[-5], arry.split("/")[-2], arry.split("/")[-1]).as_posix() + elif arry.startswith("http://") or arry.startswith("https://"): + response = requests.get(arry) + response.raise_for_status() + arry = np.load(BytesIO(response.content)) + elif os.path.isfile(arry): + arry = np.load(arry) + else: + raise ValueError( + f"Incorrect path or url, URLs must start with `http://` or `https://`, and {arry} is not a valid path" + ) + elif isinstance(arry, np.ndarray): + pass + else: + raise ValueError( + "Incorrect format used for numpy ndarray. Should be an url linking to an image, a local path, or a" + " ndarray." + ) + + return arry + + +def load_pt(url: str): + response = requests.get(url) + response.raise_for_status() + arry = torch.load(BytesIO(response.content)) + return arry + + +def load_image(image: Union[str, PIL.Image.Image]) -> PIL.Image.Image: + """ + Loads `image` to a PIL Image. + + Args: + image (`str` or `PIL.Image.Image`): + The image to convert to the PIL Image format. + Returns: + `PIL.Image.Image`: + A PIL Image. + """ + if isinstance(image, str): + if image.startswith("http://") or image.startswith("https://"): + image = PIL.Image.open(requests.get(image, stream=True).raw) + elif os.path.isfile(image): + image = PIL.Image.open(image) + else: + raise ValueError( + f"Incorrect path or url, URLs must start with `http://` or `https://`, and {image} is not a valid path" + ) + elif isinstance(image, PIL.Image.Image): + image = image + else: + raise ValueError( + "Incorrect format used for image. Should be an url linking to an image, a local path, or a PIL image." + ) + image = PIL.ImageOps.exif_transpose(image) + image = image.convert("RGB") + return image + + +def preprocess_image(image: PIL.Image, batch_size: int): + w, h = image.size + w, h = (x - x % 8 for x in (w, h)) # resize to integer multiple of 8 + image = image.resize((w, h), resample=PIL.Image.LANCZOS) + image = np.array(image).astype(np.float32) / 255.0 + image = np.vstack([image[None].transpose(0, 3, 1, 2)] * batch_size) + image = torch.from_numpy(image) + return 2.0 * image - 1.0 + + +def export_to_gif(image: List[PIL.Image.Image], output_gif_path: str = None) -> str: + if output_gif_path is None: + output_gif_path = tempfile.NamedTemporaryFile(suffix=".gif").name + + image[0].save( + output_gif_path, + save_all=True, + append_images=image[1:], + optimize=False, + duration=100, + loop=0, + ) + return output_gif_path + + +@contextmanager +def buffered_writer(raw_f): + f = io.BufferedWriter(raw_f) + yield f + f.flush() + + +def export_to_ply(mesh, output_ply_path: str = None): + """ + Write a PLY file for a mesh. + """ + if output_ply_path is None: + output_ply_path = tempfile.NamedTemporaryFile(suffix=".ply").name + + coords = mesh.verts.detach().cpu().numpy() + faces = mesh.faces.cpu().numpy() + rgb = np.stack([mesh.vertex_channels[x].detach().cpu().numpy() for x in "RGB"], axis=1) + + with buffered_writer(open(output_ply_path, "wb")) as f: + f.write(b"ply\n") + f.write(b"format binary_little_endian 1.0\n") + f.write(bytes(f"element vertex {len(coords)}\n", "ascii")) + f.write(b"property float x\n") + f.write(b"property float y\n") + f.write(b"property float z\n") + if rgb is not None: + f.write(b"property uchar red\n") + f.write(b"property uchar green\n") + f.write(b"property uchar blue\n") + if faces is not None: + f.write(bytes(f"element face {len(faces)}\n", "ascii")) + f.write(b"property list uchar int vertex_index\n") + f.write(b"end_header\n") + + if rgb is not None: + rgb = (rgb * 255.499).round().astype(int) + vertices = [ + (*coord, *rgb) + for coord, rgb in zip( + coords.tolist(), + rgb.tolist(), + ) + ] + format = struct.Struct("<3f3B") + for item in vertices: + f.write(format.pack(*item)) + else: + format = struct.Struct("<3f") + for vertex in coords.tolist(): + f.write(format.pack(*vertex)) + + if faces is not None: + format = struct.Struct(" str: + if is_opencv_available(): + import cv2 + else: + raise ImportError(BACKENDS_MAPPING["opencv"][1].format("export_to_video")) + if output_video_path is None: + output_video_path = tempfile.NamedTemporaryFile(suffix=".mp4").name + + fourcc = cv2.VideoWriter_fourcc(*"mp4v") + h, w, c = video_frames[0].shape + video_writer = cv2.VideoWriter(output_video_path, fourcc, fps=8, frameSize=(w, h)) + for i in range(len(video_frames)): + img = cv2.cvtColor(video_frames[i], cv2.COLOR_RGB2BGR) + video_writer.write(img) + return output_video_path + + +def load_hf_numpy(path) -> np.ndarray: + base_url = "https://huggingface.co/datasets/fusing/diffusers-testing/resolve/main" + + if not path.startswith("http://") and not path.startswith("https://"): + path = os.path.join(base_url, urllib.parse.quote(path)) + + return load_numpy(path) + + +# --- pytest conf functions --- # + +# to avoid multiple invocation from tests/conftest.py and examples/conftest.py - make sure it's called only once +pytest_opt_registered = {} + + +def pytest_addoption_shared(parser): + """ + This function is to be called from `conftest.py` via `pytest_addoption` wrapper that has to be defined there. + + It allows loading both `conftest.py` files at once without causing a failure due to adding the same `pytest` + option. + + """ + option = "--make-reports" + if option not in pytest_opt_registered: + parser.addoption( + option, + action="store", + default=False, + help="generate report files. The value of this option is used as a prefix to report names", + ) + pytest_opt_registered[option] = 1 + + +def pytest_terminal_summary_main(tr, id): + """ + Generate multiple reports at the end of test suite run - each report goes into a dedicated file in the current + directory. The report files are prefixed with the test suite name. + + This function emulates --duration and -rA pytest arguments. + + This function is to be called from `conftest.py` via `pytest_terminal_summary` wrapper that has to be defined + there. + + Args: + - tr: `terminalreporter` passed from `conftest.py` + - id: unique id like `tests` or `examples` that will be incorporated into the final reports filenames - this is + needed as some jobs have multiple runs of pytest, so we can't have them overwrite each other. + + NB: this functions taps into a private _pytest API and while unlikely, it could break should + pytest do internal changes - also it calls default internal methods of terminalreporter which + can be hijacked by various `pytest-` plugins and interfere. + + """ + from _pytest.config import create_terminal_writer + + if not len(id): + id = "tests" + + config = tr.config + orig_writer = config.get_terminal_writer() + orig_tbstyle = config.option.tbstyle + orig_reportchars = tr.reportchars + + dir = "reports" + Path(dir).mkdir(parents=True, exist_ok=True) + report_files = { + k: f"{dir}/{id}_{k}.txt" + for k in [ + "durations", + "errors", + "failures_long", + "failures_short", + "failures_line", + "passes", + "stats", + "summary_short", + "warnings", + ] + } + + # custom durations report + # note: there is no need to call pytest --durations=XX to get this separate report + # adapted from https://github.com/pytest-dev/pytest/blob/897f151e/src/_pytest/runner.py#L66 + dlist = [] + for replist in tr.stats.values(): + for rep in replist: + if hasattr(rep, "duration"): + dlist.append(rep) + if dlist: + dlist.sort(key=lambda x: x.duration, reverse=True) + with open(report_files["durations"], "w") as f: + durations_min = 0.05 # sec + f.write("slowest durations\n") + for i, rep in enumerate(dlist): + if rep.duration < durations_min: + f.write(f"{len(dlist)-i} durations < {durations_min} secs were omitted") + break + f.write(f"{rep.duration:02.2f}s {rep.when:<8} {rep.nodeid}\n") + + def summary_failures_short(tr): + # expecting that the reports were --tb=long (default) so we chop them off here to the last frame + reports = tr.getreports("failed") + if not reports: + return + tr.write_sep("=", "FAILURES SHORT STACK") + for rep in reports: + msg = tr._getfailureheadline(rep) + tr.write_sep("_", msg, red=True, bold=True) + # chop off the optional leading extra frames, leaving only the last one + longrepr = re.sub(r".*_ _ _ (_ ){10,}_ _ ", "", rep.longreprtext, 0, re.M | re.S) + tr._tw.line(longrepr) + # note: not printing out any rep.sections to keep the report short + + # use ready-made report funcs, we are just hijacking the filehandle to log to a dedicated file each + # adapted from https://github.com/pytest-dev/pytest/blob/897f151e/src/_pytest/terminal.py#L814 + # note: some pytest plugins may interfere by hijacking the default `terminalreporter` (e.g. + # pytest-instafail does that) + + # report failures with line/short/long styles + config.option.tbstyle = "auto" # full tb + with open(report_files["failures_long"], "w") as f: + tr._tw = create_terminal_writer(config, f) + tr.summary_failures() + + # config.option.tbstyle = "short" # short tb + with open(report_files["failures_short"], "w") as f: + tr._tw = create_terminal_writer(config, f) + summary_failures_short(tr) + + config.option.tbstyle = "line" # one line per error + with open(report_files["failures_line"], "w") as f: + tr._tw = create_terminal_writer(config, f) + tr.summary_failures() + + with open(report_files["errors"], "w") as f: + tr._tw = create_terminal_writer(config, f) + tr.summary_errors() + + with open(report_files["warnings"], "w") as f: + tr._tw = create_terminal_writer(config, f) + tr.summary_warnings() # normal warnings + tr.summary_warnings() # final warnings + + tr.reportchars = "wPpsxXEf" # emulate -rA (used in summary_passes() and short_test_summary()) + with open(report_files["passes"], "w") as f: + tr._tw = create_terminal_writer(config, f) + tr.summary_passes() + + with open(report_files["summary_short"], "w") as f: + tr._tw = create_terminal_writer(config, f) + tr.short_test_summary() + + with open(report_files["stats"], "w") as f: + tr._tw = create_terminal_writer(config, f) + tr.summary_stats() + + # restore: + tr._tw = orig_writer + tr.reportchars = orig_reportchars + config.option.tbstyle = orig_tbstyle + + +# Copied from https://github.com/huggingface/transformers/blob/000e52aec8850d3fe2f360adc6fd256e5b47fe4c/src/transformers/testing_utils.py#L1905 +def is_flaky(max_attempts: int = 5, wait_before_retry: Optional[float] = None, description: Optional[str] = None): + """ + To decorate flaky tests. They will be retried on failures. + + Args: + max_attempts (`int`, *optional*, defaults to 5): + The maximum number of attempts to retry the flaky test. + wait_before_retry (`float`, *optional*): + If provided, will wait that number of seconds before retrying the test. + description (`str`, *optional*): + A string to describe the situation (what / where / why is flaky, link to GH issue/PR comments, errors, + etc.) + """ + + def decorator(test_func_ref): + @functools.wraps(test_func_ref) + def wrapper(*args, **kwargs): + retry_count = 1 + + while retry_count < max_attempts: + try: + return test_func_ref(*args, **kwargs) + + except Exception as err: + print(f"Test failed with {err} at try {retry_count}/{max_attempts}.", file=sys.stderr) + if wait_before_retry is not None: + time.sleep(wait_before_retry) + retry_count += 1 + + return test_func_ref(*args, **kwargs) + + return wrapper + + return decorator + + +# Taken from: https://github.com/huggingface/transformers/blob/3658488ff77ff8d45101293e749263acf437f4d5/src/transformers/testing_utils.py#L1787 +def run_test_in_subprocess(test_case, target_func, inputs=None, timeout=None): + """ + To run a test in a subprocess. In particular, this can avoid (GPU) memory issue. + + Args: + test_case (`unittest.TestCase`): + The test that will run `target_func`. + target_func (`Callable`): + The function implementing the actual testing logic. + inputs (`dict`, *optional*, defaults to `None`): + The inputs that will be passed to `target_func` through an (input) queue. + timeout (`int`, *optional*, defaults to `None`): + The timeout (in seconds) that will be passed to the input and output queues. If not specified, the env. + variable `PYTEST_TIMEOUT` will be checked. If still `None`, its value will be set to `600`. + """ + if timeout is None: + timeout = int(os.environ.get("PYTEST_TIMEOUT", 600)) + + start_methohd = "spawn" + ctx = multiprocessing.get_context(start_methohd) + + input_queue = ctx.Queue(1) + output_queue = ctx.JoinableQueue(1) + + # We can't send `unittest.TestCase` to the child, otherwise we get issues regarding pickle. + input_queue.put(inputs, timeout=timeout) + + process = ctx.Process(target=target_func, args=(input_queue, output_queue, timeout)) + process.start() + # Kill the child process if we can't get outputs from it in time: otherwise, the hanging subprocess prevents + # the test to exit properly. + try: + results = output_queue.get(timeout=timeout) + output_queue.task_done() + except Exception as e: + process.terminate() + test_case.fail(e) + process.join(timeout=timeout) + + if results["error"] is not None: + test_case.fail(f'{results["error"]}') + + +class CaptureLogger: + """ + Args: + Context manager to capture `logging` streams + logger: 'logging` logger object + Returns: + The captured output is available via `self.out` + Example: + ```python + >>> from diffusers import logging + >>> from diffusers.testing_utils import CaptureLogger + + >>> msg = "Testing 1, 2, 3" + >>> logging.set_verbosity_info() + >>> logger = logging.get_logger("diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.py") + >>> with CaptureLogger(logger) as cl: + ... logger.info(msg) + >>> assert cl.out, msg + "\n" + ``` + """ + + def __init__(self, logger): + self.logger = logger + self.io = StringIO() + self.sh = logging.StreamHandler(self.io) + self.out = "" + + def __enter__(self): + self.logger.addHandler(self.sh) + return self + + def __exit__(self, *exc): + self.logger.removeHandler(self.sh) + self.out = self.io.getvalue() + + def __repr__(self): + return f"captured: {self.out}\n" + + +def enable_full_determinism(): + """ + Helper function for reproducible behavior during distributed training. See + - https://pytorch.org/docs/stable/notes/randomness.html for pytorch + """ + # Enable PyTorch deterministic mode. This potentially requires either the environment + # variable 'CUDA_LAUNCH_BLOCKING' or 'CUBLAS_WORKSPACE_CONFIG' to be set, + # depending on the CUDA version, so we set them both here + os.environ["CUDA_LAUNCH_BLOCKING"] = "1" + os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":16:8" + torch.use_deterministic_algorithms(True) + + # Enable CUDNN deterministic mode + torch.backends.cudnn.deterministic = True + torch.backends.cudnn.benchmark = False + torch.backends.cuda.matmul.allow_tf32 = False + + +def disable_full_determinism(): + os.environ["CUDA_LAUNCH_BLOCKING"] = "0" + os.environ["CUBLAS_WORKSPACE_CONFIG"] = "" + torch.use_deterministic_algorithms(False) + + +# Utils for custom and alternative accelerator devices +def _is_torch_fp16_available(device): + if not is_torch_available(): + return False + + import torch + + device = torch.device(device) + + try: + x = torch.zeros((2, 2), dtype=torch.float16).to(device) + _ = torch.mul(x, x) + return True + + except Exception as e: + if device.type == "cuda": + raise ValueError( + f"You have passed a device of type 'cuda' which should work with 'fp16', but 'cuda' does not seem to be correctly installed on your machine: {e}" + ) + + return False + + +def _is_torch_fp64_available(device): + if not is_torch_available(): + return False + + import torch + + device = torch.device(device) + + try: + x = torch.zeros((2, 2), dtype=torch.float64).to(device) + _ = torch.mul(x, x) + return True + + except Exception as e: + if device.type == "cuda": + raise ValueError( + f"You have passed a device of type 'cuda' which should work with 'fp64', but 'cuda' does not seem to be correctly installed on your machine: {e}" + ) + + return False + + +# Guard these lookups for when Torch is not used - alternative accelerator support is for PyTorch +if is_torch_available(): + # Behaviour flags + BACKEND_SUPPORTS_TRAINING = {"cuda": True, "cpu": True, "mps": False, "default": True} + + # Function definitions + BACKEND_EMPTY_CACHE = {"cuda": torch.cuda.empty_cache, "cpu": None, "mps": None, "default": None} + BACKEND_DEVICE_COUNT = {"cuda": torch.cuda.device_count, "cpu": lambda: 0, "mps": lambda: 0, "default": 0} + BACKEND_MANUAL_SEED = {"cuda": torch.cuda.manual_seed, "cpu": torch.manual_seed, "default": torch.manual_seed} + + +# This dispatches a defined function according to the accelerator from the function definitions. +def _device_agnostic_dispatch(device: str, dispatch_table: Dict[str, Callable], *args, **kwargs): + if device not in dispatch_table: + return dispatch_table["default"](*args, **kwargs) + + fn = dispatch_table[device] + + # Some device agnostic functions return values. Need to guard against 'None' instead at + # user level + if fn is None: + return None + + return fn(*args, **kwargs) + + +# These are callables which automatically dispatch the function specific to the accelerator +def backend_manual_seed(device: str, seed: int): + return _device_agnostic_dispatch(device, BACKEND_MANUAL_SEED, seed) + + +def backend_empty_cache(device: str): + return _device_agnostic_dispatch(device, BACKEND_EMPTY_CACHE) + + +def backend_device_count(device: str): + return _device_agnostic_dispatch(device, BACKEND_DEVICE_COUNT) + + +# These are callables which return boolean behaviour flags and can be used to specify some +# device agnostic alternative where the feature is unsupported. +def backend_supports_training(device: str): + if not is_torch_available(): + return False + + if device not in BACKEND_SUPPORTS_TRAINING: + device = "default" + + return BACKEND_SUPPORTS_TRAINING[device] + + +# Guard for when Torch is not available +if is_torch_available(): + # Update device function dict mapping + def update_mapping_from_spec(device_fn_dict: Dict[str, Callable], attribute_name: str): + try: + # Try to import the function directly + spec_fn = getattr(device_spec_module, attribute_name) + device_fn_dict[torch_device] = spec_fn + except AttributeError as e: + # If the function doesn't exist, and there is no default, throw an error + if "default" not in device_fn_dict: + raise AttributeError( + f"`{attribute_name}` not found in '{device_spec_path}' and no default fallback function found." + ) from e + + if "DIFFUSERS_TEST_DEVICE_SPEC" in os.environ: + device_spec_path = os.environ["DIFFUSERS_TEST_DEVICE_SPEC"] + if not Path(device_spec_path).is_file(): + raise ValueError(f"Specified path to device specification file is not found. Received {device_spec_path}") + + try: + import_name = device_spec_path[: device_spec_path.index(".py")] + except ValueError as e: + raise ValueError(f"Provided device spec file is not a Python file! Received {device_spec_path}") from e + + device_spec_module = importlib.import_module(import_name) + + try: + device_name = device_spec_module.DEVICE_NAME + except AttributeError: + raise AttributeError("Device spec file did not contain `DEVICE_NAME`") + + if "DIFFUSERS_TEST_DEVICE" in os.environ and torch_device != device_name: + msg = f"Mismatch between environment variable `DIFFUSERS_TEST_DEVICE` '{torch_device}' and device found in spec '{device_name}'\n" + msg += "Either unset `DIFFUSERS_TEST_DEVICE` or ensure it matches device spec name." + raise ValueError(msg) + + torch_device = device_name + + # Add one entry here for each `BACKEND_*` dictionary. + update_mapping_from_spec(BACKEND_MANUAL_SEED, "MANUAL_SEED_FN") + update_mapping_from_spec(BACKEND_EMPTY_CACHE, "EMPTY_CACHE_FN") + update_mapping_from_spec(BACKEND_DEVICE_COUNT, "DEVICE_COUNT_FN") + update_mapping_from_spec(BACKEND_SUPPORTS_TRAINING, "SUPPORTS_TRAINING") diff --git a/diffusers-0.27.0/src/diffusers/utils/torch_utils.py b/diffusers-0.27.0/src/diffusers/utils/torch_utils.py new file mode 100755 index 0000000..cc9c050 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/torch_utils.py @@ -0,0 +1,147 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +PyTorch utilities: Utilities related to PyTorch +""" +from typing import List, Optional, Tuple, Union + +from . import logging +from .import_utils import is_torch_available, is_torch_version + + +if is_torch_available(): + import torch + from torch.fft import fftn, fftshift, ifftn, ifftshift + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + +try: + from torch._dynamo import allow_in_graph as maybe_allow_in_graph +except (ImportError, ModuleNotFoundError): + + def maybe_allow_in_graph(cls): + return cls + + +def randn_tensor( + shape: Union[Tuple, List], + generator: Optional[Union[List["torch.Generator"], "torch.Generator"]] = None, + device: Optional["torch.device"] = None, + dtype: Optional["torch.dtype"] = None, + layout: Optional["torch.layout"] = None, +): + """A helper function to create random tensors on the desired `device` with the desired `dtype`. When + passing a list of generators, you can seed each batch size individually. If CPU generators are passed, the tensor + is always created on the CPU. + """ + # device on which tensor is created defaults to device + rand_device = device + batch_size = shape[0] + + layout = layout or torch.strided + device = device or torch.device("cpu") + + if generator is not None: + gen_device_type = generator.device.type if not isinstance(generator, list) else generator[0].device.type + if gen_device_type != device.type and gen_device_type == "cpu": + rand_device = "cpu" + if device != "mps": + logger.info( + f"The passed generator was created on 'cpu' even though a tensor on {device} was expected." + f" Tensors will be created on 'cpu' and then moved to {device}. Note that one can probably" + f" slighly speed up this function by passing a generator that was created on the {device} device." + ) + elif gen_device_type != device.type and gen_device_type == "cuda": + raise ValueError(f"Cannot generate a {device} tensor from a generator of type {gen_device_type}.") + + # make sure generator list of length 1 is treated like a non-list + if isinstance(generator, list) and len(generator) == 1: + generator = generator[0] + + if isinstance(generator, list): + shape = (1,) + shape[1:] + latents = [ + torch.randn(shape, generator=generator[i], device=rand_device, dtype=dtype, layout=layout) + for i in range(batch_size) + ] + latents = torch.cat(latents, dim=0).to(device) + else: + latents = torch.randn(shape, generator=generator, device=rand_device, dtype=dtype, layout=layout).to(device) + + return latents + + +def is_compiled_module(module) -> bool: + """Check whether the module was compiled with torch.compile()""" + if is_torch_version("<", "2.0.0") or not hasattr(torch, "_dynamo"): + return False + return isinstance(module, torch._dynamo.eval_frame.OptimizedModule) + + +def fourier_filter(x_in: "torch.Tensor", threshold: int, scale: int) -> "torch.Tensor": + """Fourier filter as introduced in FreeU (https://arxiv.org/abs/2309.11497). + + This version of the method comes from here: + https://github.com/huggingface/diffusers/pull/5164#issuecomment-1732638706 + """ + x = x_in + B, C, H, W = x.shape + + # Non-power of 2 images must be float32 + if (W & (W - 1)) != 0 or (H & (H - 1)) != 0: + x = x.to(dtype=torch.float32) + + # FFT + x_freq = fftn(x, dim=(-2, -1)) + x_freq = fftshift(x_freq, dim=(-2, -1)) + + B, C, H, W = x_freq.shape + mask = torch.ones((B, C, H, W), device=x.device) + + crow, ccol = H // 2, W // 2 + mask[..., crow - threshold : crow + threshold, ccol - threshold : ccol + threshold] = scale + x_freq = x_freq * mask + + # IFFT + x_freq = ifftshift(x_freq, dim=(-2, -1)) + x_filtered = ifftn(x_freq, dim=(-2, -1)).real + + return x_filtered.to(dtype=x_in.dtype) + + +def apply_freeu( + resolution_idx: int, hidden_states: "torch.Tensor", res_hidden_states: "torch.Tensor", **freeu_kwargs +) -> Tuple["torch.Tensor", "torch.Tensor"]: + """Applies the FreeU mechanism as introduced in https: + //arxiv.org/abs/2309.11497. Adapted from the official code repository: https://github.com/ChenyangSi/FreeU. + + Args: + resolution_idx (`int`): Integer denoting the UNet block where FreeU is being applied. + hidden_states (`torch.Tensor`): Inputs to the underlying block. + res_hidden_states (`torch.Tensor`): Features from the skip block corresponding to the underlying block. + s1 (`float`): Scaling factor for stage 1 to attenuate the contributions of the skip features. + s2 (`float`): Scaling factor for stage 2 to attenuate the contributions of the skip features. + b1 (`float`): Scaling factor for stage 1 to amplify the contributions of backbone features. + b2 (`float`): Scaling factor for stage 2 to amplify the contributions of backbone features. + """ + if resolution_idx == 0: + num_half_channels = hidden_states.shape[1] // 2 + hidden_states[:, :num_half_channels] = hidden_states[:, :num_half_channels] * freeu_kwargs["b1"] + res_hidden_states = fourier_filter(res_hidden_states, threshold=1, scale=freeu_kwargs["s1"]) + if resolution_idx == 1: + num_half_channels = hidden_states.shape[1] // 2 + hidden_states[:, :num_half_channels] = hidden_states[:, :num_half_channels] * freeu_kwargs["b2"] + res_hidden_states = fourier_filter(res_hidden_states, threshold=1, scale=freeu_kwargs["s2"]) + + return hidden_states, res_hidden_states diff --git a/diffusers-0.27.0/src/diffusers/utils/versions.py b/diffusers-0.27.0/src/diffusers/utils/versions.py new file mode 100755 index 0000000..945a397 --- /dev/null +++ b/diffusers-0.27.0/src/diffusers/utils/versions.py @@ -0,0 +1,117 @@ +# Copyright 2020 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Utilities for working with package versions +""" + +import importlib.metadata +import operator +import re +import sys +from typing import Optional + +from packaging import version + + +ops = { + "<": operator.lt, + "<=": operator.le, + "==": operator.eq, + "!=": operator.ne, + ">=": operator.ge, + ">": operator.gt, +} + + +def _compare_versions(op, got_ver, want_ver, requirement, pkg, hint): + if got_ver is None or want_ver is None: + raise ValueError( + f"Unable to compare versions for {requirement}: need={want_ver} found={got_ver}. This is unusual. Consider" + f" reinstalling {pkg}." + ) + if not ops[op](version.parse(got_ver), version.parse(want_ver)): + raise ImportError( + f"{requirement} is required for a normal functioning of this module, but found {pkg}=={got_ver}.{hint}" + ) + + +def require_version(requirement: str, hint: Optional[str] = None) -> None: + """ + Perform a runtime check of the dependency versions, using the exact same syntax used by pip. + + The installed module version comes from the *site-packages* dir via *importlib.metadata*. + + Args: + requirement (`str`): pip style definition, e.g., "tokenizers==0.9.4", "tqdm>=4.27", "numpy" + hint (`str`, *optional*): what suggestion to print in case of requirements not being met + + Example: + + ```python + require_version("pandas>1.1.2") + require_version("numpy>1.18.5", "this is important to have for whatever reason") + ```""" + + hint = f"\n{hint}" if hint is not None else "" + + # non-versioned check + if re.match(r"^[\w_\-\d]+$", requirement): + pkg, op, want_ver = requirement, None, None + else: + match = re.findall(r"^([^!=<>\s]+)([\s!=<>]{1,2}.+)", requirement) + if not match: + raise ValueError( + "requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23, but" + f" got {requirement}" + ) + pkg, want_full = match[0] + want_range = want_full.split(",") # there could be multiple requirements + wanted = {} + for w in want_range: + match = re.findall(r"^([\s!=<>]{1,2})(.+)", w) + if not match: + raise ValueError( + "requirement needs to be in the pip package format, .e.g., package_a==1.23, or package_b>=1.23," + f" but got {requirement}" + ) + op, want_ver = match[0] + wanted[op] = want_ver + if op not in ops: + raise ValueError(f"{requirement}: need one of {list(ops.keys())}, but got {op}") + + # special case + if pkg == "python": + got_ver = ".".join([str(x) for x in sys.version_info[:3]]) + for op, want_ver in wanted.items(): + _compare_versions(op, got_ver, want_ver, requirement, pkg, hint) + return + + # check if any version is installed + try: + got_ver = importlib.metadata.version(pkg) + except importlib.metadata.PackageNotFoundError: + raise importlib.metadata.PackageNotFoundError( + f"The '{requirement}' distribution was not found and is required by this application. {hint}" + ) + + # check that the right version is installed if version number or a range was provided + if want_ver is not None: + for op, want_ver in wanted.items(): + _compare_versions(op, got_ver, want_ver, requirement, pkg, hint) + + +def require_version_core(requirement): + """require_version wrapper which emits a core-specific hint on failure""" + hint = "Try: pip install transformers -U or pip install -e '.[dev]' if you're working with git main" + return require_version(requirement, hint) diff --git a/diffusers-0.27.0/tests/__init__.py b/diffusers-0.27.0/tests/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/conftest.py b/diffusers-0.27.0/tests/conftest.py new file mode 100755 index 0000000..4993ed9 --- /dev/null +++ b/diffusers-0.27.0/tests/conftest.py @@ -0,0 +1,44 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# tests directory-specific settings - this file is run automatically +# by pytest before any tests are run + +import sys +import warnings +from os.path import abspath, dirname, join + + +# allow having multiple repository checkouts and not needing to remember to rerun +# 'pip install -e .[dev]' when switching between checkouts and running tests. +git_repo_path = abspath(join(dirname(dirname(__file__)), "src")) +sys.path.insert(1, git_repo_path) + +# silence FutureWarning warnings in tests since often we can't act on them until +# they become normal warnings - i.e. the tests still need to test the current functionality +warnings.simplefilter(action="ignore", category=FutureWarning) + + +def pytest_addoption(parser): + from diffusers.utils.testing_utils import pytest_addoption_shared + + pytest_addoption_shared(parser) + + +def pytest_terminal_summary(terminalreporter): + from diffusers.utils.testing_utils import pytest_terminal_summary_main + + make_reports = terminalreporter.config.getoption("--make-reports") + if make_reports: + pytest_terminal_summary_main(terminalreporter, id=make_reports) diff --git a/diffusers-0.27.0/tests/fixtures/custom_pipeline/pipeline.py b/diffusers-0.27.0/tests/fixtures/custom_pipeline/pipeline.py new file mode 100755 index 0000000..601f51b --- /dev/null +++ b/diffusers-0.27.0/tests/fixtures/custom_pipeline/pipeline.py @@ -0,0 +1,101 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +# limitations under the License. + + +from typing import Optional, Tuple, Union + +import torch + +from diffusers import DiffusionPipeline, ImagePipelineOutput + + +class CustomLocalPipeline(DiffusionPipeline): + r""" + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Parameters: + unet ([`UNet2DModel`]): U-Net architecture to denoise the encoded image. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image. Can be one of + [`DDPMScheduler`], or [`DDIMScheduler`]. + """ + + def __init__(self, unet, scheduler): + super().__init__() + self.register_modules(unet=unet, scheduler=scheduler) + + @torch.no_grad() + def __call__( + self, + batch_size: int = 1, + generator: Optional[torch.Generator] = None, + num_inference_steps: int = 50, + output_type: Optional[str] = "pil", + return_dict: bool = True, + **kwargs, + ) -> Union[ImagePipelineOutput, Tuple]: + r""" + Args: + batch_size (`int`, *optional*, defaults to 1): + The number of images to generate. + generator (`torch.Generator`, *optional*): + A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation + deterministic. + eta (`float`, *optional*, defaults to 0.0): + The eta parameter which controls the scale of the variance (0 is DDIM and 1 is one type of DDPM). + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipelines.ImagePipelineOutput`] instead of a plain tuple. + + Returns: + [`~pipelines.ImagePipelineOutput`] or `tuple`: [`~pipelines.utils.ImagePipelineOutput`] if + `return_dict` is True, otherwise a `tuple. When returning a tuple, the first element is a list with the + generated images. + """ + + # Sample gaussian noise to begin loop + image = torch.randn( + (batch_size, self.unet.config.in_channels, self.unet.config.sample_size, self.unet.config.sample_size), + generator=generator, + ) + image = image.to(self.device) + + # set step values + self.scheduler.set_timesteps(num_inference_steps) + + for t in self.progress_bar(self.scheduler.timesteps): + # 1. predict noise model_output + model_output = self.unet(image, t).sample + + # 2. predict previous mean of image x_t-1 and add variance depending on eta + # eta corresponds to η in paper and should be between [0, 1] + # do x_t -> x_t-1 + image = self.scheduler.step(model_output, t, image).prev_sample + + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,), "This is a local test" + + return ImagePipelineOutput(images=image), "This is a local test" diff --git a/diffusers-0.27.0/tests/fixtures/custom_pipeline/what_ever.py b/diffusers-0.27.0/tests/fixtures/custom_pipeline/what_ever.py new file mode 100755 index 0000000..8ceeb42 --- /dev/null +++ b/diffusers-0.27.0/tests/fixtures/custom_pipeline/what_ever.py @@ -0,0 +1,101 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and + +# limitations under the License. + + +from typing import Optional, Tuple, Union + +import torch + +from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput + + +class CustomLocalPipeline(DiffusionPipeline): + r""" + This model inherits from [`DiffusionPipeline`]. Check the superclass documentation for the generic methods the + library implements for all the pipelines (such as downloading or saving, running on a particular device, etc.) + + Parameters: + unet ([`UNet2DModel`]): U-Net architecture to denoise the encoded image. + scheduler ([`SchedulerMixin`]): + A scheduler to be used in combination with `unet` to denoise the encoded image. Can be one of + [`DDPMScheduler`], or [`DDIMScheduler`]. + """ + + def __init__(self, unet, scheduler): + super().__init__() + self.register_modules(unet=unet, scheduler=scheduler) + + @torch.no_grad() + def __call__( + self, + batch_size: int = 1, + generator: Optional[torch.Generator] = None, + num_inference_steps: int = 50, + output_type: Optional[str] = "pil", + return_dict: bool = True, + **kwargs, + ) -> Union[ImagePipelineOutput, Tuple]: + r""" + Args: + batch_size (`int`, *optional*, defaults to 1): + The number of images to generate. + generator (`torch.Generator`, *optional*): + A [torch generator](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make generation + deterministic. + eta (`float`, *optional*, defaults to 0.0): + The eta parameter which controls the scale of the variance (0 is DDIM and 1 is one type of DDPM). + num_inference_steps (`int`, *optional*, defaults to 50): + The number of denoising steps. More denoising steps usually lead to a higher quality image at the + expense of slower inference. + output_type (`str`, *optional*, defaults to `"pil"`): + The output format of the generate image. Choose between + [PIL](https://pillow.readthedocs.io/en/stable/): `PIL.Image.Image` or `np.array`. + return_dict (`bool`, *optional*, defaults to `True`): + Whether or not to return a [`~pipeline_utils.ImagePipelineOutput`] instead of a plain tuple. + + Returns: + [`~pipeline_utils.ImagePipelineOutput`] or `tuple`: [`~pipelines.utils.ImagePipelineOutput`] if + `return_dict` is True, otherwise a `tuple. When returning a tuple, the first element is a list with the + generated images. + """ + + # Sample gaussian noise to begin loop + image = torch.randn( + (batch_size, self.unet.config.in_channels, self.unet.config.sample_size, self.unet.config.sample_size), + generator=generator, + ) + image = image.to(self.device) + + # set step values + self.scheduler.set_timesteps(num_inference_steps) + + for t in self.progress_bar(self.scheduler.timesteps): + # 1. predict noise model_output + model_output = self.unet(image, t).sample + + # 2. predict previous mean of image x_t-1 and add variance depending on eta + # eta corresponds to η in paper and should be between [0, 1] + # do x_t -> x_t-1 + image = self.scheduler.step(model_output, t, image).prev_sample + + image = (image / 2 + 0.5).clamp(0, 1) + image = image.cpu().permute(0, 2, 3, 1).numpy() + if output_type == "pil": + image = self.numpy_to_pil(image) + + if not return_dict: + return (image,), "This is a local test" + + return ImagePipelineOutput(images=image), "This is a local test" diff --git a/diffusers-0.27.0/tests/fixtures/elise_format0.mid b/diffusers-0.27.0/tests/fixtures/elise_format0.mid new file mode 100755 index 0000000000000000000000000000000000000000..33dbabe7ab1d4d28e43d9911255a510a8a672d77 GIT binary patch literal 14210 zcmeHOUu;xYdjAr(DZ$1#K;3MYddK!Kz8?FIXU1b@#$4=ckH;RyV>}Q@F!{4U>>9Q# zAct(dFLASZs3Q6>+LuVJYW2a=YG1se5~`3W+NV`jBzE7H?LWGxQe&wv+iX`NtxA8t z@0@#Q42c3woK=!Oz@2-}cfRwT@BjCm>*=Fs=0xNv{Cnbwf0;hI_=h6db8>&n(-Z%_ z7%84R``WzRf3l-+;o^4|&n}#~6!}3UolIsTr@s>!n_oOX7nxdoeSTqnQSLvnyYl+n zADvx@eEZaS%*|gqbK%?b=P~|t^}?$cBHy}j=>lH%Ow2D{I=iqi|JtSbbLTEaPJJtq zPo$CwBHczL$1cvDx-`%6fn*|w@l;yJ$4|XJzc>_04`p(Z>9glwzchay<41q%q-+80 z_m6*N%a-P?a)0(oz8&7OegD77k3`=6?K#=z%ZtlE9XUPs>Zwe6-&`u0IhEdb>ec+* ztEZFMWG0gx%w-4D1N8-ORfrBI{dlVrH|PrefYO9Au`pWZ_c_tTJPA0 z)TTr#-Dj|_`QB=)LPo26yL-KvkChMh@bTmC8%UR%HUINh_3^7;>efwvsgLL`#TnhD z`%OAalOpB*CBV15vqNfSky1C`c<5S5q&Oh%-srpj^3j?rlId8=U`!<6K_J~QFXDAB zc_MyH(L{eDH6K9n5Dnlb6XH0#-t{HsFOSq5#bjud8Zd&0T<1~_IHF5fJFX-sLF1UL zV**^s6*u-2FuDG6QEBQTO&~SjwFFxHu1-uo&m)kjz*dR*d~?=we6-#%obKcxxXD4V z(J-z@xp{N#oU(ZD64{=3)#)*jtyc-pNzP3rOAcRq}#NpYiT@N}Y1KY7~8 zCD=l0b)N3!SWCdu9z{ygkf*LVu_hdL$X^~%o_6UpPiT1>)q%w>sgbve3yP;)DLFcK zV@1V0X=L~GpGyrqE;``xPGE+#e5ElG$UVYUt;gObjlT6^2z>ss$9x9_c5&0nHzMCX7K5>sYV zwv>*`LC97It!y7rt9v1V>Ccr`Pw=dZU9Co|Ef>|ELpQQVHMdjeHnO9`nKI+^4MSMwh7tR@Kb_)!C>z4Bx1DOV8gos`%-V zhgF1Czhr$+^hJB5fJ6H^wu%>o;(C`HP?=t#oTNvPHSlXulwU>D;iAb@zX>#71v;TZ&hu{SvS^@3@A`v zV>mq9v|HUy4;R1`w_(rrsVXPb1;Qa%+u zLoBzi(Dk*ne9uBE>h~Q!RFiSWN#p04Oh6IK8Xm*wZR{Y3-UyZrk>+z1y{S|hy_MDe zEvJE<&bqP%36hb9I;?xEa|*JTTxFclxrBw(2SrN;Hil@j|KYSJfe4s~-x}Ezzh&JE zq?a@D;y^t_uP*4(A2&o(eRvw!tvW8X&A@5(YP6=pl-v!TOOwCtf#*tpxjYw5`#0jb zXy4XzQQD8@xm3;XUe`wuv(?lMqGlO5v~G21s|y<)n&!_s1=rO$Wjz@8 zq6gEM5_&K#EyDT(hS*upl{e<0k5_-hKEwK>Om_RXnvDm`8Rlnrf{Sori#MU4xp#IoI=XULdAi!Ja#v&+m zZY@l}>dDzJOCOh<*{m#ZYO0Hfhs;NOM@k7nyqDTgC#q28#_cN;Oslws$dFO|8!I|R zOB?NxQa@@ffU6>hb@(=}LY9P8c{PO{*O+efN}*RuDam$V{+u(Aa4R z2^kjmwqt#Pkl;E79F8SGmc1l|}cRL@jp4Byx>Zvp^8O@=j{7jG}grOg-! z?zv-dLTbzlGeZ@5lGy3$v80j*_7*U~_m|@}>@sBXEMf#*@GRv48P;7iffxpa*^?9i z0{-vFFd;Y-BU2!w-L)ekhjT-R&0cgR06m<;Udl>m{RwRO`~I>z4Mu0|d{|^AhdNUs zW}8%0;BO5ann7L*HbiA8D%q$4AYbA!Ru`Nan1`d0b`YFJ7eE1StSG?kD=H{z5fJe_ ziYQB5jAWVxnsWdHP?h$vk3ifuhk%eK_JQ(kAo+m5oFc9HC}lALc@0>JD0e84=m`jI zqEAmoE&>Rk#T4K?%oNfd1P?&0%<}W97w@#J3N!)l6-A8Gp z6pZ9?E-%JT3M4X>4{~o}#XKGPe#xXlNi#b)`efuySB$mbPi4r?;P>>1q+>E1BQYa9 zI)t27ZWo7S(5&PR;0QI6D^C4Nc@DXYF3-e*R_Hvk@K3ids-M~;*21DNo((=Yn_3^WR&{PP8BTb01(&3ixHN%~$uqev9ux?*z6xNLu zWj@VCWu|XTAV7*rN~y8VmT9;OEcKz1VpC$&?4)RrV<~Ws0#W^-nF|{11~#7`OqYn;H?q@A#iT zGw?=p$sW3${CCd~afWS=N?u(qdn!bnQWvHLGUjsMJws(<(ai-PRd*B3Cf41W%1)VN z;IZrOH-2i)?7YO2*mR37ng=pLg{Q?4Odlo8u#9UDGAz{HUG=ut*Msiq*U7HO%UVCn zuKDM*zqomII;|RU7SgtPwt%*9p8K zt#f1i;dbi(<~OIbB3|s;TJv2gxH93&!9m?VwgTNTg9tj;2_8miLR> zztwD@Ti#sT!?&yEYPIza=h)!m!@u<;w%Tb%EHANReB*l6SgXb>AL!iMl{U)3r12rY|#4G zV*&3sVlH}8@>nIgS;9e7AspMyZb}bpRnA%G+XK9+vlSfJU30Taj6Hft=z|o7YKuJv zlvpc47!&EZxn`BjNFeKQ@ZM?zB=~8mp;*-?(XF|r@VWM$E`&P4mr>1%Qa&OvTzz_q z%6vWpvmwtUX4W(Ji@_B)9s!gnL01=)ppXQs65JchX?RrgS}Y~*5a&Q%1F@ELfatoL zYj#`O4Gem>+1AcA?q_Yza^>Et-LD)&(A>>adg#3H)NGaD)aFns?%@M6 z3AjFSGc+aiFc9o9kFM5OL5efd%~Nx#yKZ9Lach$p>qd1mXy?0o#7#B8vZW6Wk7?EK zjY;v4aOpq*-hhTQ(s}`o4uLl=#i}^SD{_Q&q}>Tn<2PX$EDfwbCZ$8TUA8B902Zgk znawHta0wkh;fVA}yZ$C;cKpuI+WEZ<#)y}}3mgC+yqpvZ(h<xRi52M6TMyhn+<@j?Gqm;%~!v_r(_DCA0<2q?L zS3i=T%*MQXtA>F|HNv~sc}B!5>m$@)>V60(0y{sWK(wh*$T1Rah+8JGpCV4QKpVa!c$3vlC&aF`)8Q4%NV0J z9Oh#KRH7c85|0*|EZ-QH64^%zN+1qxLpH__Nco^7@+T5{v{7jv(HND;a0GsOMB0Z7 zGFp&{83-4H>X6BdM4E4jLH$1!%#e9ZiPc+=j>)hFn8biW>54#LaLI7!0302c;c*$S zN`LM|zqI$K_&=c-PWm$&$JpUUhGkbCU=SHUDE(=L(%ea7M+1fs8qwD~AjN}{%)WIJ zpL`(%vF|N!Uc^KX`-1CT z5*;q$ZUahYXzja+>G&W);KTI6Pbfe`i0S3Q>dPW0)76tAbLr}w$Z1?HTAo%cZ)$mG>t5qUXHWG|(vFEzlvI8c32($ zk*<=ut56Z};8?nP497^3Y0ssUVC$P{o8Z6^LLb+(AZ+dYq0|ncKvNJ@8l2np z{t-T4a{8c^!P!;1d3C>}Jr#42%TEJ%;m|Wj=mz%--LapHa4maN`q5w5eBbza1Jc*M z|Nn6>8dXbRqlQQ_)nt30`p3PS*QlbdX4F?c^pPAabhDc8F>St^KRF1R@1_<1CH+M1 zH`aXr;(pv4___E?^mDe)$;z32&h&z*{15xNjhBr4Gm=SU4JnrSU-&=G(GFYKgWu@d zd^$SQXV-d!d^9nE88Z23zRwT(?Hl6-KZmP-VyJ&kgp}%&|2dBN%Id*>`;K@1prPO1 z`cMCAKBv*U#NX@utA@|vcK!`+*+G0UDjm;=`&Ir$&Xm9W<}S*cDIoqR&DZa 1e-3: + models_are_equal = False + + return models_are_equal + + +@require_peft_backend +class PeftLoraLoaderMixinTests: + torch_device = "cuda" if torch.cuda.is_available() else "cpu" + pipeline_class = None + scheduler_cls = None + scheduler_kwargs = None + has_two_text_encoders = False + unet_kwargs = None + vae_kwargs = None + + def get_dummy_components(self, scheduler_cls=None): + scheduler_cls = self.scheduler_cls if scheduler_cls is None else LCMScheduler + rank = 4 + + torch.manual_seed(0) + unet = UNet2DConditionModel(**self.unet_kwargs) + + scheduler = scheduler_cls(**self.scheduler_kwargs) + + torch.manual_seed(0) + vae = AutoencoderKL(**self.vae_kwargs) + + text_encoder = CLIPTextModel.from_pretrained("peft-internal-testing/tiny-clip-text-2") + tokenizer = CLIPTokenizer.from_pretrained("peft-internal-testing/tiny-clip-text-2") + + if self.has_two_text_encoders: + text_encoder_2 = CLIPTextModelWithProjection.from_pretrained("peft-internal-testing/tiny-clip-text-2") + tokenizer_2 = CLIPTokenizer.from_pretrained("peft-internal-testing/tiny-clip-text-2") + + text_lora_config = LoraConfig( + r=rank, + lora_alpha=rank, + target_modules=["q_proj", "k_proj", "v_proj", "out_proj"], + init_lora_weights=False, + ) + + unet_lora_config = LoraConfig( + r=rank, lora_alpha=rank, target_modules=["to_q", "to_k", "to_v", "to_out.0"], init_lora_weights=False + ) + + if self.has_two_text_encoders: + pipeline_components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "image_encoder": None, + "feature_extractor": None, + } + else: + pipeline_components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + + return pipeline_components, text_lora_config, unet_lora_config + + def get_dummy_inputs(self, with_generator=True): + batch_size = 1 + sequence_length = 10 + num_channels = 4 + sizes = (32, 32) + + generator = torch.manual_seed(0) + noise = floats_tensor((batch_size, num_channels) + sizes) + input_ids = torch.randint(1, sequence_length, size=(batch_size, sequence_length), generator=generator) + + pipeline_inputs = { + "prompt": "A painting of a squirrel eating a burger", + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "np", + } + if with_generator: + pipeline_inputs.update({"generator": generator}) + + return noise, input_ids, pipeline_inputs + + # copied from: https://colab.research.google.com/gist/sayakpaul/df2ef6e1ae6d8c10a49d859883b10860/scratchpad.ipynb + def get_dummy_tokens(self): + max_seq_length = 77 + + inputs = torch.randint(2, 56, size=(1, max_seq_length), generator=torch.manual_seed(0)) + + prepared_inputs = {} + prepared_inputs["input_ids"] = inputs + return prepared_inputs + + def check_if_lora_correctly_set(self, model) -> bool: + """ + Checks if the LoRA layers are correctly set with peft + """ + for module in model.modules(): + if isinstance(module, BaseTunerLayer): + return True + return False + + def test_simple_inference(self): + """ + Tests a simple inference and makes sure it works as expected + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, _ = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + + _, _, inputs = self.get_dummy_inputs() + output_no_lora = pipe(**inputs).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + def test_simple_inference_with_text_lora(self): + """ + Tests a simple inference with lora attached on the text encoder + and makes sure it works as expected + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, _ = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + pipe.text_encoder.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + output_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue( + not np.allclose(output_lora, output_no_lora, atol=1e-3, rtol=1e-3), "Lora should change the output" + ) + + def test_simple_inference_with_text_lora_and_scale(self): + """ + Tests a simple inference with lora attached on the text encoder + scale argument + and makes sure it works as expected + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, _ = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + pipe.text_encoder.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + output_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue( + not np.allclose(output_lora, output_no_lora, atol=1e-3, rtol=1e-3), "Lora should change the output" + ) + + output_lora_scale = pipe( + **inputs, generator=torch.manual_seed(0), cross_attention_kwargs={"scale": 0.5} + ).images + self.assertTrue( + not np.allclose(output_lora, output_lora_scale, atol=1e-3, rtol=1e-3), + "Lora + scale should change the output", + ) + + output_lora_0_scale = pipe( + **inputs, generator=torch.manual_seed(0), cross_attention_kwargs={"scale": 0.0} + ).images + self.assertTrue( + np.allclose(output_no_lora, output_lora_0_scale, atol=1e-3, rtol=1e-3), + "Lora + 0 scale should lead to same result as no LoRA", + ) + + def test_simple_inference_with_text_lora_fused(self): + """ + Tests a simple inference with lora attached into text encoder + fuses the lora weights into base model + and makes sure it works as expected + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, _ = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + pipe.text_encoder.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + pipe.fuse_lora() + # Fusing should still keep the LoRA layers + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + + if self.has_two_text_encoders: + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + ouput_fused = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertFalse( + np.allclose(ouput_fused, output_no_lora, atol=1e-3, rtol=1e-3), "Fused lora should change the output" + ) + + def test_simple_inference_with_text_lora_unloaded(self): + """ + Tests a simple inference with lora attached to text encoder, then unloads the lora weights + and makes sure it works as expected + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, _ = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + pipe.text_encoder.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + pipe.unload_lora_weights() + # unloading should remove the LoRA layers + self.assertFalse( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly unloaded in text encoder" + ) + + if self.has_two_text_encoders: + self.assertFalse( + self.check_if_lora_correctly_set(pipe.text_encoder_2), + "Lora not correctly unloaded in text encoder 2", + ) + + ouput_unloaded = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue( + np.allclose(ouput_unloaded, output_no_lora, atol=1e-3, rtol=1e-3), + "Fused lora should change the output", + ) + + def test_simple_inference_with_text_lora_save_load(self): + """ + Tests a simple usecase where users could use saving utilities for LoRA. + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, _ = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + pipe.text_encoder.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + images_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + + with tempfile.TemporaryDirectory() as tmpdirname: + text_encoder_state_dict = get_peft_model_state_dict(pipe.text_encoder) + if self.has_two_text_encoders: + text_encoder_2_state_dict = get_peft_model_state_dict(pipe.text_encoder_2) + + self.pipeline_class.save_lora_weights( + save_directory=tmpdirname, + text_encoder_lora_layers=text_encoder_state_dict, + text_encoder_2_lora_layers=text_encoder_2_state_dict, + safe_serialization=False, + ) + else: + self.pipeline_class.save_lora_weights( + save_directory=tmpdirname, + text_encoder_lora_layers=text_encoder_state_dict, + safe_serialization=False, + ) + + self.assertTrue(os.path.isfile(os.path.join(tmpdirname, "pytorch_lora_weights.bin"))) + pipe.unload_lora_weights() + + pipe.load_lora_weights(os.path.join(tmpdirname, "pytorch_lora_weights.bin")) + + images_lora_from_pretrained = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + + if self.has_two_text_encoders: + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + self.assertTrue( + np.allclose(images_lora, images_lora_from_pretrained, atol=1e-3, rtol=1e-3), + "Loading from saved checkpoints should give same results.", + ) + + def test_simple_inference_save_pretrained(self): + """ + Tests a simple usecase where users could use saving utilities for LoRA through save_pretrained + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, _ = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + pipe.text_encoder.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + images_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + + with tempfile.TemporaryDirectory() as tmpdirname: + pipe.save_pretrained(tmpdirname) + + pipe_from_pretrained = self.pipeline_class.from_pretrained(tmpdirname) + pipe_from_pretrained.to(self.torch_device) + + self.assertTrue( + self.check_if_lora_correctly_set(pipe_from_pretrained.text_encoder), + "Lora not correctly set in text encoder", + ) + + if self.has_two_text_encoders: + self.assertTrue( + self.check_if_lora_correctly_set(pipe_from_pretrained.text_encoder_2), + "Lora not correctly set in text encoder 2", + ) + + images_lora_save_pretrained = pipe_from_pretrained(**inputs, generator=torch.manual_seed(0)).images + + self.assertTrue( + np.allclose(images_lora, images_lora_save_pretrained, atol=1e-3, rtol=1e-3), + "Loading from saved checkpoints should give same results.", + ) + + def test_simple_inference_with_text_unet_lora_save_load(self): + """ + Tests a simple usecase where users could use saving utilities for LoRA for Unet + text encoder + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + pipe.text_encoder.add_adapter(text_lora_config) + pipe.unet.add_adapter(unet_lora_config) + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + images_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + + with tempfile.TemporaryDirectory() as tmpdirname: + text_encoder_state_dict = get_peft_model_state_dict(pipe.text_encoder) + unet_state_dict = get_peft_model_state_dict(pipe.unet) + if self.has_two_text_encoders: + text_encoder_2_state_dict = get_peft_model_state_dict(pipe.text_encoder_2) + + self.pipeline_class.save_lora_weights( + save_directory=tmpdirname, + text_encoder_lora_layers=text_encoder_state_dict, + text_encoder_2_lora_layers=text_encoder_2_state_dict, + unet_lora_layers=unet_state_dict, + safe_serialization=False, + ) + else: + self.pipeline_class.save_lora_weights( + save_directory=tmpdirname, + text_encoder_lora_layers=text_encoder_state_dict, + unet_lora_layers=unet_state_dict, + safe_serialization=False, + ) + + self.assertTrue(os.path.isfile(os.path.join(tmpdirname, "pytorch_lora_weights.bin"))) + pipe.unload_lora_weights() + + pipe.load_lora_weights(os.path.join(tmpdirname, "pytorch_lora_weights.bin")) + + images_lora_from_pretrained = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + if self.has_two_text_encoders: + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + self.assertTrue( + np.allclose(images_lora, images_lora_from_pretrained, atol=1e-3, rtol=1e-3), + "Loading from saved checkpoints should give same results.", + ) + + def test_simple_inference_with_text_unet_lora_and_scale(self): + """ + Tests a simple inference with lora attached on the text encoder + Unet + scale argument + and makes sure it works as expected + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + pipe.text_encoder.add_adapter(text_lora_config) + pipe.unet.add_adapter(unet_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + output_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue( + not np.allclose(output_lora, output_no_lora, atol=1e-3, rtol=1e-3), "Lora should change the output" + ) + + output_lora_scale = pipe( + **inputs, generator=torch.manual_seed(0), cross_attention_kwargs={"scale": 0.5} + ).images + self.assertTrue( + not np.allclose(output_lora, output_lora_scale, atol=1e-3, rtol=1e-3), + "Lora + scale should change the output", + ) + + output_lora_0_scale = pipe( + **inputs, generator=torch.manual_seed(0), cross_attention_kwargs={"scale": 0.0} + ).images + self.assertTrue( + np.allclose(output_no_lora, output_lora_0_scale, atol=1e-3, rtol=1e-3), + "Lora + 0 scale should lead to same result as no LoRA", + ) + + self.assertTrue( + pipe.text_encoder.text_model.encoder.layers[0].self_attn.q_proj.scaling["default"] == 1.0, + "The scaling parameter has not been correctly restored!", + ) + + def test_simple_inference_with_text_lora_unet_fused(self): + """ + Tests a simple inference with lora attached into text encoder + fuses the lora weights into base model + and makes sure it works as expected - with unet + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + pipe.text_encoder.add_adapter(text_lora_config) + pipe.unet.add_adapter(unet_lora_config) + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + pipe.fuse_lora() + # Fusing should still keep the LoRA layers + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in unet") + + if self.has_two_text_encoders: + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + ouput_fused = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertFalse( + np.allclose(ouput_fused, output_no_lora, atol=1e-3, rtol=1e-3), "Fused lora should change the output" + ) + + def test_simple_inference_with_text_unet_lora_unloaded(self): + """ + Tests a simple inference with lora attached to text encoder and unet, then unloads the lora weights + and makes sure it works as expected + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + pipe.text_encoder.add_adapter(text_lora_config) + pipe.unet.add_adapter(unet_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + pipe.unload_lora_weights() + # unloading should remove the LoRA layers + self.assertFalse( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly unloaded in text encoder" + ) + self.assertFalse(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly unloaded in Unet") + + if self.has_two_text_encoders: + self.assertFalse( + self.check_if_lora_correctly_set(pipe.text_encoder_2), + "Lora not correctly unloaded in text encoder 2", + ) + + ouput_unloaded = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue( + np.allclose(ouput_unloaded, output_no_lora, atol=1e-3, rtol=1e-3), + "Fused lora should change the output", + ) + + def test_simple_inference_with_text_unet_lora_unfused(self): + """ + Tests a simple inference with lora attached to text encoder and unet, then unloads the lora weights + and makes sure it works as expected + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + pipe.text_encoder.add_adapter(text_lora_config) + pipe.unet.add_adapter(unet_lora_config) + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + pipe.fuse_lora() + + output_fused_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.unfuse_lora() + + output_unfused_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + # unloading should remove the LoRA layers + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Unfuse should still keep LoRA layers" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Unfuse should still keep LoRA layers") + + if self.has_two_text_encoders: + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Unfuse should still keep LoRA layers" + ) + + # Fuse and unfuse should lead to the same results + self.assertTrue( + np.allclose(output_fused_lora, output_unfused_lora, atol=1e-3, rtol=1e-3), + "Fused lora should change the output", + ) + + def test_simple_inference_with_text_unet_multi_adapter(self): + """ + Tests a simple inference with lora attached to text encoder and unet, attaches + multiple adapters and set them + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") + pipe.text_encoder.add_adapter(text_lora_config, "adapter-2") + + pipe.unet.add_adapter(unet_lora_config, "adapter-1") + pipe.unet.add_adapter(unet_lora_config, "adapter-2") + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config, "adapter-1") + pipe.text_encoder_2.add_adapter(text_lora_config, "adapter-2") + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + pipe.set_adapters("adapter-1") + + output_adapter_1 = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.set_adapters("adapter-2") + output_adapter_2 = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.set_adapters(["adapter-1", "adapter-2"]) + + output_adapter_mixed = pipe(**inputs, generator=torch.manual_seed(0)).images + + # Fuse and unfuse should lead to the same results + self.assertFalse( + np.allclose(output_adapter_1, output_adapter_2, atol=1e-3, rtol=1e-3), + "Adapter 1 and 2 should give different results", + ) + + self.assertFalse( + np.allclose(output_adapter_1, output_adapter_mixed, atol=1e-3, rtol=1e-3), + "Adapter 1 and mixed adapters should give different results", + ) + + self.assertFalse( + np.allclose(output_adapter_2, output_adapter_mixed, atol=1e-3, rtol=1e-3), + "Adapter 2 and mixed adapters should give different results", + ) + + pipe.disable_lora() + + output_disabled = pipe(**inputs, generator=torch.manual_seed(0)).images + + self.assertTrue( + np.allclose(output_no_lora, output_disabled, atol=1e-3, rtol=1e-3), + "output with no lora and output with lora disabled should give same results", + ) + + def test_simple_inference_with_text_unet_multi_adapter_delete_adapter(self): + """ + Tests a simple inference with lora attached to text encoder and unet, attaches + multiple adapters and set/delete them + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") + pipe.text_encoder.add_adapter(text_lora_config, "adapter-2") + + pipe.unet.add_adapter(unet_lora_config, "adapter-1") + pipe.unet.add_adapter(unet_lora_config, "adapter-2") + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config, "adapter-1") + pipe.text_encoder_2.add_adapter(text_lora_config, "adapter-2") + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + pipe.set_adapters("adapter-1") + + output_adapter_1 = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.set_adapters("adapter-2") + output_adapter_2 = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.set_adapters(["adapter-1", "adapter-2"]) + + output_adapter_mixed = pipe(**inputs, generator=torch.manual_seed(0)).images + + self.assertFalse( + np.allclose(output_adapter_1, output_adapter_2, atol=1e-3, rtol=1e-3), + "Adapter 1 and 2 should give different results", + ) + + self.assertFalse( + np.allclose(output_adapter_1, output_adapter_mixed, atol=1e-3, rtol=1e-3), + "Adapter 1 and mixed adapters should give different results", + ) + + self.assertFalse( + np.allclose(output_adapter_2, output_adapter_mixed, atol=1e-3, rtol=1e-3), + "Adapter 2 and mixed adapters should give different results", + ) + + pipe.delete_adapters("adapter-1") + output_deleted_adapter_1 = pipe(**inputs, generator=torch.manual_seed(0)).images + + self.assertTrue( + np.allclose(output_deleted_adapter_1, output_adapter_2, atol=1e-3, rtol=1e-3), + "Adapter 1 and 2 should give different results", + ) + + pipe.delete_adapters("adapter-2") + output_deleted_adapters = pipe(**inputs, generator=torch.manual_seed(0)).images + + self.assertTrue( + np.allclose(output_no_lora, output_deleted_adapters, atol=1e-3, rtol=1e-3), + "output with no lora and output with lora disabled should give same results", + ) + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") + pipe.text_encoder.add_adapter(text_lora_config, "adapter-2") + + pipe.unet.add_adapter(unet_lora_config, "adapter-1") + pipe.unet.add_adapter(unet_lora_config, "adapter-2") + + pipe.set_adapters(["adapter-1", "adapter-2"]) + pipe.delete_adapters(["adapter-1", "adapter-2"]) + + output_deleted_adapters = pipe(**inputs, generator=torch.manual_seed(0)).images + + self.assertTrue( + np.allclose(output_no_lora, output_deleted_adapters, atol=1e-3, rtol=1e-3), + "output with no lora and output with lora disabled should give same results", + ) + + def test_simple_inference_with_text_unet_multi_adapter_weighted(self): + """ + Tests a simple inference with lora attached to text encoder and unet, attaches + multiple adapters and set them + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") + pipe.text_encoder.add_adapter(text_lora_config, "adapter-2") + + pipe.unet.add_adapter(unet_lora_config, "adapter-1") + pipe.unet.add_adapter(unet_lora_config, "adapter-2") + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config, "adapter-1") + pipe.text_encoder_2.add_adapter(text_lora_config, "adapter-2") + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + pipe.set_adapters("adapter-1") + + output_adapter_1 = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.set_adapters("adapter-2") + output_adapter_2 = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.set_adapters(["adapter-1", "adapter-2"]) + + output_adapter_mixed = pipe(**inputs, generator=torch.manual_seed(0)).images + + # Fuse and unfuse should lead to the same results + self.assertFalse( + np.allclose(output_adapter_1, output_adapter_2, atol=1e-3, rtol=1e-3), + "Adapter 1 and 2 should give different results", + ) + + self.assertFalse( + np.allclose(output_adapter_1, output_adapter_mixed, atol=1e-3, rtol=1e-3), + "Adapter 1 and mixed adapters should give different results", + ) + + self.assertFalse( + np.allclose(output_adapter_2, output_adapter_mixed, atol=1e-3, rtol=1e-3), + "Adapter 2 and mixed adapters should give different results", + ) + + pipe.set_adapters(["adapter-1", "adapter-2"], [0.5, 0.6]) + output_adapter_mixed_weighted = pipe(**inputs, generator=torch.manual_seed(0)).images + + self.assertFalse( + np.allclose(output_adapter_mixed_weighted, output_adapter_mixed, atol=1e-3, rtol=1e-3), + "Weighted adapter and mixed adapter should give different results", + ) + + pipe.disable_lora() + + output_disabled = pipe(**inputs, generator=torch.manual_seed(0)).images + + self.assertTrue( + np.allclose(output_no_lora, output_disabled, atol=1e-3, rtol=1e-3), + "output with no lora and output with lora disabled should give same results", + ) + + def test_lora_fuse_nan(self): + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") + + pipe.unet.add_adapter(unet_lora_config, "adapter-1") + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + # corrupt one LoRA weight with `inf` values + with torch.no_grad(): + pipe.unet.mid_block.attentions[0].transformer_blocks[0].attn1.to_q.lora_A["adapter-1"].weight += float( + "inf" + ) + + # with `safe_fusing=True` we should see an Error + with self.assertRaises(ValueError): + pipe.fuse_lora(safe_fusing=True) + + # without we should not see an error, but every image will be black + pipe.fuse_lora(safe_fusing=False) + + out = pipe("test", num_inference_steps=2, output_type="np").images + + self.assertTrue(np.isnan(out).all()) + + def test_get_adapters(self): + """ + Tests a simple usecase where we attach multiple adapters and check if the results + are the expected results + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") + pipe.unet.add_adapter(unet_lora_config, "adapter-1") + + adapter_names = pipe.get_active_adapters() + self.assertListEqual(adapter_names, ["adapter-1"]) + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-2") + pipe.unet.add_adapter(unet_lora_config, "adapter-2") + + adapter_names = pipe.get_active_adapters() + self.assertListEqual(adapter_names, ["adapter-2"]) + + pipe.set_adapters(["adapter-1", "adapter-2"]) + self.assertListEqual(pipe.get_active_adapters(), ["adapter-1", "adapter-2"]) + + def test_get_list_adapters(self): + """ + Tests a simple usecase where we attach multiple adapters and check if the results + are the expected results + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") + pipe.unet.add_adapter(unet_lora_config, "adapter-1") + + adapter_names = pipe.get_list_adapters() + self.assertDictEqual(adapter_names, {"text_encoder": ["adapter-1"], "unet": ["adapter-1"]}) + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-2") + pipe.unet.add_adapter(unet_lora_config, "adapter-2") + + adapter_names = pipe.get_list_adapters() + self.assertDictEqual( + adapter_names, {"text_encoder": ["adapter-1", "adapter-2"], "unet": ["adapter-1", "adapter-2"]} + ) + + pipe.set_adapters(["adapter-1", "adapter-2"]) + self.assertDictEqual( + pipe.get_list_adapters(), + {"unet": ["adapter-1", "adapter-2"], "text_encoder": ["adapter-1", "adapter-2"]}, + ) + + pipe.unet.add_adapter(unet_lora_config, "adapter-3") + self.assertDictEqual( + pipe.get_list_adapters(), + {"unet": ["adapter-1", "adapter-2", "adapter-3"], "text_encoder": ["adapter-1", "adapter-2"]}, + ) + + @require_peft_version_greater(peft_version="0.6.2") + def test_simple_inference_with_text_lora_unet_fused_multi(self): + """ + Tests a simple inference with lora attached into text encoder + fuses the lora weights into base model + and makes sure it works as expected - with unet and multi-adapter case + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + output_no_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue(output_no_lora.shape == (1, 64, 64, 3)) + + pipe.text_encoder.add_adapter(text_lora_config, "adapter-1") + pipe.unet.add_adapter(unet_lora_config, "adapter-1") + + # Attach a second adapter + pipe.text_encoder.add_adapter(text_lora_config, "adapter-2") + pipe.unet.add_adapter(unet_lora_config, "adapter-2") + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config, "adapter-1") + pipe.text_encoder_2.add_adapter(text_lora_config, "adapter-2") + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + # set them to multi-adapter inference mode + pipe.set_adapters(["adapter-1", "adapter-2"]) + ouputs_all_lora = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.set_adapters(["adapter-1"]) + ouputs_lora_1 = pipe(**inputs, generator=torch.manual_seed(0)).images + + pipe.fuse_lora(adapter_names=["adapter-1"]) + + # Fusing should still keep the LoRA layers so outpout should remain the same + outputs_lora_1_fused = pipe(**inputs, generator=torch.manual_seed(0)).images + + self.assertTrue( + np.allclose(ouputs_lora_1, outputs_lora_1_fused, atol=1e-3, rtol=1e-3), + "Fused lora should not change the output", + ) + + pipe.unfuse_lora() + pipe.fuse_lora(adapter_names=["adapter-2", "adapter-1"]) + + # Fusing should still keep the LoRA layers + output_all_lora_fused = pipe(**inputs, generator=torch.manual_seed(0)).images + self.assertTrue( + np.allclose(output_all_lora_fused, ouputs_all_lora, atol=1e-3, rtol=1e-3), + "Fused lora should not change the output", + ) + + @unittest.skip("This is failing for now - need to investigate") + def test_simple_inference_with_text_unet_lora_unfused_torch_compile(self): + """ + Tests a simple inference with lora attached to text encoder and unet, then unloads the lora weights + and makes sure it works as expected + """ + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, text_lora_config, unet_lora_config = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _, _, inputs = self.get_dummy_inputs(with_generator=False) + + pipe.text_encoder.add_adapter(text_lora_config) + pipe.unet.add_adapter(unet_lora_config) + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), "Lora not correctly set in text encoder" + ) + self.assertTrue(self.check_if_lora_correctly_set(pipe.unet), "Lora not correctly set in Unet") + + if self.has_two_text_encoders: + pipe.text_encoder_2.add_adapter(text_lora_config) + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder_2), "Lora not correctly set in text encoder 2" + ) + + pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True) + pipe.text_encoder = torch.compile(pipe.text_encoder, mode="reduce-overhead", fullgraph=True) + + if self.has_two_text_encoders: + pipe.text_encoder_2 = torch.compile(pipe.text_encoder_2, mode="reduce-overhead", fullgraph=True) + + # Just makes sure it works.. + _ = pipe(**inputs, generator=torch.manual_seed(0)).images + + def test_modify_padding_mode(self): + def set_pad_mode(network, mode="circular"): + for _, module in network.named_modules(): + if isinstance(module, torch.nn.Conv2d): + module.padding_mode = mode + + for scheduler_cls in [DDIMScheduler, LCMScheduler]: + components, _, _ = self.get_dummy_components(scheduler_cls) + pipe = self.pipeline_class(**components) + pipe = pipe.to(self.torch_device) + pipe.set_progress_bar_config(disable=None) + _pad_mode = "circular" + set_pad_mode(pipe.vae, _pad_mode) + set_pad_mode(pipe.unet, _pad_mode) + + _, _, inputs = self.get_dummy_inputs() + _ = pipe(**inputs).images + + +class StableDiffusionLoRATests(PeftLoraLoaderMixinTests, unittest.TestCase): + pipeline_class = StableDiffusionPipeline + scheduler_cls = DDIMScheduler + scheduler_kwargs = { + "beta_start": 0.00085, + "beta_end": 0.012, + "beta_schedule": "scaled_linear", + "clip_sample": False, + "set_alpha_to_one": False, + "steps_offset": 1, + } + unet_kwargs = { + "block_out_channels": (32, 64), + "layers_per_block": 2, + "sample_size": 32, + "in_channels": 4, + "out_channels": 4, + "down_block_types": ("DownBlock2D", "CrossAttnDownBlock2D"), + "up_block_types": ("CrossAttnUpBlock2D", "UpBlock2D"), + "cross_attention_dim": 32, + } + vae_kwargs = { + "block_out_channels": [32, 64], + "in_channels": 3, + "out_channels": 3, + "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"], + "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D"], + "latent_channels": 4, + } + + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + @slow + @require_torch_gpu + def test_integration_move_lora_cpu(self): + path = "runwayml/stable-diffusion-v1-5" + lora_id = "takuma104/lora-test-text-encoder-lora-target" + + pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float16) + pipe.load_lora_weights(lora_id, adapter_name="adapter-1") + pipe.load_lora_weights(lora_id, adapter_name="adapter-2") + pipe = pipe.to("cuda") + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), + "Lora not correctly set in text encoder", + ) + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.unet), + "Lora not correctly set in text encoder", + ) + + # We will offload the first adapter in CPU and check if the offloading + # has been performed correctly + pipe.set_lora_device(["adapter-1"], "cpu") + + for name, module in pipe.unet.named_modules(): + if "adapter-1" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + self.assertTrue(module.weight.device == torch.device("cpu")) + elif "adapter-2" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + self.assertTrue(module.weight.device != torch.device("cpu")) + + for name, module in pipe.text_encoder.named_modules(): + if "adapter-1" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + self.assertTrue(module.weight.device == torch.device("cpu")) + elif "adapter-2" in name and not isinstance(module, (nn.Dropout, nn.Identity)): + self.assertTrue(module.weight.device != torch.device("cpu")) + + pipe.set_lora_device(["adapter-1"], 0) + + for n, m in pipe.unet.named_modules(): + if "adapter-1" in n and not isinstance(m, (nn.Dropout, nn.Identity)): + self.assertTrue(m.weight.device != torch.device("cpu")) + + for n, m in pipe.text_encoder.named_modules(): + if "adapter-1" in n and not isinstance(m, (nn.Dropout, nn.Identity)): + self.assertTrue(m.weight.device != torch.device("cpu")) + + pipe.set_lora_device(["adapter-1", "adapter-2"], "cuda") + + for n, m in pipe.unet.named_modules(): + if ("adapter-1" in n or "adapter-2" in n) and not isinstance(m, (nn.Dropout, nn.Identity)): + self.assertTrue(m.weight.device != torch.device("cpu")) + + for n, m in pipe.text_encoder.named_modules(): + if ("adapter-1" in n or "adapter-2" in n) and not isinstance(m, (nn.Dropout, nn.Identity)): + self.assertTrue(m.weight.device != torch.device("cpu")) + + @slow + @require_torch_gpu + def test_integration_logits_with_scale(self): + path = "runwayml/stable-diffusion-v1-5" + lora_id = "takuma104/lora-test-text-encoder-lora-target" + + pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float32) + pipe.load_lora_weights(lora_id) + pipe = pipe.to("cuda") + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), + "Lora not correctly set in text encoder 2", + ) + + prompt = "a red sks dog" + + images = pipe( + prompt=prompt, + num_inference_steps=15, + cross_attention_kwargs={"scale": 0.5}, + generator=torch.manual_seed(0), + output_type="np", + ).images + + expected_slice_scale = np.array([0.307, 0.283, 0.310, 0.310, 0.300, 0.314, 0.336, 0.314, 0.321]) + + predicted_slice = images[0, -3:, -3:, -1].flatten() + + self.assertTrue(np.allclose(expected_slice_scale, predicted_slice, atol=1e-3, rtol=1e-3)) + + @slow + @require_torch_gpu + def test_integration_logits_no_scale(self): + path = "runwayml/stable-diffusion-v1-5" + lora_id = "takuma104/lora-test-text-encoder-lora-target" + + pipe = StableDiffusionPipeline.from_pretrained(path, torch_dtype=torch.float32) + pipe.load_lora_weights(lora_id) + pipe = pipe.to("cuda") + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.text_encoder), + "Lora not correctly set in text encoder", + ) + + prompt = "a red sks dog" + + images = pipe(prompt=prompt, num_inference_steps=30, generator=torch.manual_seed(0), output_type="np").images + + expected_slice_scale = np.array([0.074, 0.064, 0.073, 0.0842, 0.069, 0.0641, 0.0794, 0.076, 0.084]) + + predicted_slice = images[0, -3:, -3:, -1].flatten() + + self.assertTrue(np.allclose(expected_slice_scale, predicted_slice, atol=1e-3, rtol=1e-3)) + + @nightly + @require_torch_gpu + def test_integration_logits_multi_adapter(self): + path = "stabilityai/stable-diffusion-xl-base-1.0" + lora_id = "CiroN2022/toy-face" + + pipe = StableDiffusionXLPipeline.from_pretrained(path, torch_dtype=torch.float16) + pipe.load_lora_weights(lora_id, weight_name="toy_face_sdxl.safetensors", adapter_name="toy") + pipe = pipe.to("cuda") + + self.assertTrue( + self.check_if_lora_correctly_set(pipe.unet), + "Lora not correctly set in Unet", + ) + + prompt = "toy_face of a hacker with a hoodie" + + lora_scale = 0.9 + + images = pipe( + prompt=prompt, + num_inference_steps=30, + generator=torch.manual_seed(0), + cross_attention_kwargs={"scale": lora_scale}, + output_type="np", + ).images + expected_slice_scale = np.array([0.538, 0.539, 0.540, 0.540, 0.542, 0.539, 0.538, 0.541, 0.539]) + + predicted_slice = images[0, -3:, -3:, -1].flatten() + self.assertTrue(np.allclose(expected_slice_scale, predicted_slice, atol=1e-3, rtol=1e-3)) + + pipe.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors", adapter_name="pixel") + pipe.set_adapters("pixel") + + prompt = "pixel art, a hacker with a hoodie, simple, flat colors" + images = pipe( + prompt, + num_inference_steps=30, + guidance_scale=7.5, + cross_attention_kwargs={"scale": lora_scale}, + generator=torch.manual_seed(0), + output_type="np", + ).images + + predicted_slice = images[0, -3:, -3:, -1].flatten() + expected_slice_scale = np.array( + [0.61973065, 0.62018543, 0.62181497, 0.61933696, 0.6208608, 0.620576, 0.6200281, 0.62258327, 0.6259889] + ) + self.assertTrue(np.allclose(expected_slice_scale, predicted_slice, atol=1e-3, rtol=1e-3)) + + # multi-adapter inference + pipe.set_adapters(["pixel", "toy"], adapter_weights=[0.5, 1.0]) + images = pipe( + prompt, + num_inference_steps=30, + guidance_scale=7.5, + cross_attention_kwargs={"scale": 1.0}, + generator=torch.manual_seed(0), + output_type="np", + ).images + predicted_slice = images[0, -3:, -3:, -1].flatten() + expected_slice_scale = np.array([0.5888, 0.5897, 0.5946, 0.5888, 0.5935, 0.5946, 0.5857, 0.5891, 0.5909]) + self.assertTrue(np.allclose(expected_slice_scale, predicted_slice, atol=1e-3, rtol=1e-3)) + + # Lora disabled + pipe.disable_lora() + images = pipe( + prompt, + num_inference_steps=30, + guidance_scale=7.5, + cross_attention_kwargs={"scale": lora_scale}, + generator=torch.manual_seed(0), + output_type="np", + ).images + predicted_slice = images[0, -3:, -3:, -1].flatten() + expected_slice_scale = np.array([0.5456, 0.5466, 0.5487, 0.5458, 0.5469, 0.5454, 0.5446, 0.5479, 0.5487]) + self.assertTrue(np.allclose(expected_slice_scale, predicted_slice, atol=1e-3, rtol=1e-3)) + + +class StableDiffusionXLLoRATests(PeftLoraLoaderMixinTests, unittest.TestCase): + has_two_text_encoders = True + pipeline_class = StableDiffusionXLPipeline + scheduler_cls = EulerDiscreteScheduler + scheduler_kwargs = { + "beta_start": 0.00085, + "beta_end": 0.012, + "beta_schedule": "scaled_linear", + "timestep_spacing": "leading", + "steps_offset": 1, + } + unet_kwargs = { + "block_out_channels": (32, 64), + "layers_per_block": 2, + "sample_size": 32, + "in_channels": 4, + "out_channels": 4, + "down_block_types": ("DownBlock2D", "CrossAttnDownBlock2D"), + "up_block_types": ("CrossAttnUpBlock2D", "UpBlock2D"), + "attention_head_dim": (2, 4), + "use_linear_projection": True, + "addition_embed_type": "text_time", + "addition_time_embed_dim": 8, + "transformer_layers_per_block": (1, 2), + "projection_class_embeddings_input_dim": 80, # 6 * 8 + 32 + "cross_attention_dim": 64, + } + vae_kwargs = { + "block_out_channels": [32, 64], + "in_channels": 3, + "out_channels": 3, + "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"], + "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D"], + "latent_channels": 4, + "sample_size": 128, + } + + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + +@slow +@require_torch_gpu +class LoraIntegrationTests(PeftLoraLoaderMixinTests, unittest.TestCase): + pipeline_class = StableDiffusionPipeline + scheduler_cls = DDIMScheduler + scheduler_kwargs = { + "beta_start": 0.00085, + "beta_end": 0.012, + "beta_schedule": "scaled_linear", + "clip_sample": False, + "set_alpha_to_one": False, + "steps_offset": 1, + } + unet_kwargs = { + "block_out_channels": (32, 64), + "layers_per_block": 2, + "sample_size": 32, + "in_channels": 4, + "out_channels": 4, + "down_block_types": ("DownBlock2D", "CrossAttnDownBlock2D"), + "up_block_types": ("CrossAttnUpBlock2D", "UpBlock2D"), + "cross_attention_dim": 32, + } + vae_kwargs = { + "block_out_channels": [32, 64], + "in_channels": 3, + "out_channels": 3, + "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"], + "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D"], + "latent_channels": 4, + } + + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_dreambooth_old_format(self): + generator = torch.Generator("cpu").manual_seed(0) + + lora_model_id = "hf-internal-testing/lora_dreambooth_dog_example" + card = RepoCard.load(lora_model_id) + base_model_id = card.data.to_dict()["base_model"] + + pipe = StableDiffusionPipeline.from_pretrained(base_model_id, safety_checker=None) + pipe = pipe.to(torch_device) + pipe.load_lora_weights(lora_model_id) + + images = pipe( + "A photo of a sks dog floating in the river", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + + expected = np.array([0.7207, 0.6787, 0.6010, 0.7478, 0.6838, 0.6064, 0.6984, 0.6443, 0.5785]) + + self.assertTrue(np.allclose(images, expected, atol=1e-4)) + release_memory(pipe) + + def test_dreambooth_text_encoder_new_format(self): + generator = torch.Generator().manual_seed(0) + + lora_model_id = "hf-internal-testing/lora-trained" + card = RepoCard.load(lora_model_id) + base_model_id = card.data.to_dict()["base_model"] + + pipe = StableDiffusionPipeline.from_pretrained(base_model_id, safety_checker=None) + pipe = pipe.to(torch_device) + pipe.load_lora_weights(lora_model_id) + + images = pipe("A photo of a sks dog", output_type="np", generator=generator, num_inference_steps=2).images + + images = images[0, -3:, -3:, -1].flatten() + + expected = np.array([0.6628, 0.6138, 0.5390, 0.6625, 0.6130, 0.5463, 0.6166, 0.5788, 0.5359]) + + self.assertTrue(np.allclose(images, expected, atol=1e-4)) + release_memory(pipe) + + def test_a1111(self): + generator = torch.Generator().manual_seed(0) + + pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/Counterfeit-V2.5", safety_checker=None).to( + torch_device + ) + lora_model_id = "hf-internal-testing/civitai-light-shadow-lora" + lora_filename = "light_and_shadow.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.3636, 0.3708, 0.3694, 0.3679, 0.3829, 0.3677, 0.3692, 0.3688, 0.3292]) + + self.assertTrue(np.allclose(images, expected, atol=1e-3)) + release_memory(pipe) + + def test_lycoris(self): + generator = torch.Generator().manual_seed(0) + + pipe = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/Amixx", safety_checker=None, use_safetensors=True, variant="fp16" + ).to(torch_device) + lora_model_id = "hf-internal-testing/edgLycorisMugler-light" + lora_filename = "edgLycorisMugler-light.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.6463, 0.658, 0.599, 0.6542, 0.6512, 0.6213, 0.658, 0.6485, 0.6017]) + + self.assertTrue(np.allclose(images, expected, atol=1e-3)) + release_memory(pipe) + + def test_a1111_with_model_cpu_offload(self): + generator = torch.Generator().manual_seed(0) + + pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/Counterfeit-V2.5", safety_checker=None) + pipe.enable_model_cpu_offload() + lora_model_id = "hf-internal-testing/civitai-light-shadow-lora" + lora_filename = "light_and_shadow.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.3636, 0.3708, 0.3694, 0.3679, 0.3829, 0.3677, 0.3692, 0.3688, 0.3292]) + + self.assertTrue(np.allclose(images, expected, atol=1e-3)) + release_memory(pipe) + + def test_a1111_with_sequential_cpu_offload(self): + generator = torch.Generator().manual_seed(0) + + pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/Counterfeit-V2.5", safety_checker=None) + pipe.enable_sequential_cpu_offload() + lora_model_id = "hf-internal-testing/civitai-light-shadow-lora" + lora_filename = "light_and_shadow.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.3636, 0.3708, 0.3694, 0.3679, 0.3829, 0.3677, 0.3692, 0.3688, 0.3292]) + + self.assertTrue(np.allclose(images, expected, atol=1e-3)) + release_memory(pipe) + + def test_kohya_sd_v15_with_higher_dimensions(self): + generator = torch.Generator().manual_seed(0) + + pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", safety_checker=None).to( + torch_device + ) + lora_model_id = "hf-internal-testing/urushisato-lora" + lora_filename = "urushisato_v15.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.7165, 0.6616, 0.5833, 0.7504, 0.6718, 0.587, 0.6871, 0.6361, 0.5694]) + + self.assertTrue(np.allclose(images, expected, atol=1e-3)) + release_memory(pipe) + + def test_vanilla_funetuning(self): + generator = torch.Generator().manual_seed(0) + + lora_model_id = "hf-internal-testing/sd-model-finetuned-lora-t4" + card = RepoCard.load(lora_model_id) + base_model_id = card.data.to_dict()["base_model"] + + pipe = StableDiffusionPipeline.from_pretrained(base_model_id, safety_checker=None) + pipe = pipe.to(torch_device) + pipe.load_lora_weights(lora_model_id) + + images = pipe("A pokemon with blue eyes.", output_type="np", generator=generator, num_inference_steps=2).images + + images = images[0, -3:, -3:, -1].flatten() + + expected = np.array([0.7406, 0.699, 0.5963, 0.7493, 0.7045, 0.6096, 0.6886, 0.6388, 0.583]) + + self.assertTrue(np.allclose(images, expected, atol=1e-4)) + release_memory(pipe) + + def test_unload_kohya_lora(self): + generator = torch.manual_seed(0) + prompt = "masterpiece, best quality, mountain" + num_inference_steps = 2 + + pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", safety_checker=None).to( + torch_device + ) + initial_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + initial_images = initial_images[0, -3:, -3:, -1].flatten() + + lora_model_id = "hf-internal-testing/civitai-colored-icons-lora" + lora_filename = "Colored_Icons_by_vizsumit.safetensors" + + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + generator = torch.manual_seed(0) + lora_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + lora_images = lora_images[0, -3:, -3:, -1].flatten() + + pipe.unload_lora_weights() + generator = torch.manual_seed(0) + unloaded_lora_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + unloaded_lora_images = unloaded_lora_images[0, -3:, -3:, -1].flatten() + + self.assertFalse(np.allclose(initial_images, lora_images)) + self.assertTrue(np.allclose(initial_images, unloaded_lora_images, atol=1e-3)) + release_memory(pipe) + + def test_load_unload_load_kohya_lora(self): + # This test ensures that a Kohya-style LoRA can be safely unloaded and then loaded + # without introducing any side-effects. Even though the test uses a Kohya-style + # LoRA, the underlying adapter handling mechanism is format-agnostic. + generator = torch.manual_seed(0) + prompt = "masterpiece, best quality, mountain" + num_inference_steps = 2 + + pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", safety_checker=None).to( + torch_device + ) + initial_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + initial_images = initial_images[0, -3:, -3:, -1].flatten() + + lora_model_id = "hf-internal-testing/civitai-colored-icons-lora" + lora_filename = "Colored_Icons_by_vizsumit.safetensors" + + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + generator = torch.manual_seed(0) + lora_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + lora_images = lora_images[0, -3:, -3:, -1].flatten() + + pipe.unload_lora_weights() + generator = torch.manual_seed(0) + unloaded_lora_images = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + unloaded_lora_images = unloaded_lora_images[0, -3:, -3:, -1].flatten() + + self.assertFalse(np.allclose(initial_images, lora_images)) + self.assertTrue(np.allclose(initial_images, unloaded_lora_images, atol=1e-3)) + + # make sure we can load a LoRA again after unloading and they don't have + # any undesired effects. + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + generator = torch.manual_seed(0) + lora_images_again = pipe( + prompt, output_type="np", generator=generator, num_inference_steps=num_inference_steps + ).images + lora_images_again = lora_images_again[0, -3:, -3:, -1].flatten() + + self.assertTrue(np.allclose(lora_images, lora_images_again, atol=1e-3)) + release_memory(pipe) + + def test_not_empty_state_dict(self): + # Makes sure https://github.com/huggingface/diffusers/issues/7054 does not happen again + pipe = AutoPipelineForText2Image.from_pretrained( + "runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16 + ).to("cuda") + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + + cached_file = hf_hub_download("hf-internal-testing/lcm-lora-test-sd-v1-5", "test_lora.safetensors") + lcm_lora = load_file(cached_file) + + pipe.load_lora_weights(lcm_lora, adapter_name="lcm") + self.assertTrue(lcm_lora != {}) + release_memory(pipe) + + def test_load_unload_load_state_dict(self): + # Makes sure https://github.com/huggingface/diffusers/issues/7054 does not happen again + pipe = AutoPipelineForText2Image.from_pretrained( + "runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16 + ).to("cuda") + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + + cached_file = hf_hub_download("hf-internal-testing/lcm-lora-test-sd-v1-5", "test_lora.safetensors") + lcm_lora = load_file(cached_file) + previous_state_dict = lcm_lora.copy() + + pipe.load_lora_weights(lcm_lora, adapter_name="lcm") + self.assertDictEqual(lcm_lora, previous_state_dict) + + pipe.unload_lora_weights() + pipe.load_lora_weights(lcm_lora, adapter_name="lcm") + self.assertDictEqual(lcm_lora, previous_state_dict) + + release_memory(pipe) + + +@slow +@require_torch_gpu +class LoraSDXLIntegrationTests(PeftLoraLoaderMixinTests, unittest.TestCase): + has_two_text_encoders = True + pipeline_class = StableDiffusionXLPipeline + scheduler_cls = EulerDiscreteScheduler + scheduler_kwargs = { + "beta_start": 0.00085, + "beta_end": 0.012, + "beta_schedule": "scaled_linear", + "timestep_spacing": "leading", + "steps_offset": 1, + } + unet_kwargs = { + "block_out_channels": (32, 64), + "layers_per_block": 2, + "sample_size": 32, + "in_channels": 4, + "out_channels": 4, + "down_block_types": ("DownBlock2D", "CrossAttnDownBlock2D"), + "up_block_types": ("CrossAttnUpBlock2D", "UpBlock2D"), + "attention_head_dim": (2, 4), + "use_linear_projection": True, + "addition_embed_type": "text_time", + "addition_time_embed_dim": 8, + "transformer_layers_per_block": (1, 2), + "projection_class_embeddings_input_dim": 80, # 6 * 8 + 32 + "cross_attention_dim": 64, + } + vae_kwargs = { + "block_out_channels": [32, 64], + "in_channels": 3, + "out_channels": 3, + "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"], + "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D"], + "latent_channels": 4, + "sample_size": 128, + } + + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_sdxl_0_9_lora_one(self): + generator = torch.Generator().manual_seed(0) + + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-0.9") + lora_model_id = "hf-internal-testing/sdxl-0.9-daiton-lora" + lora_filename = "daiton-xl-lora-test.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + pipe.enable_model_cpu_offload() + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.3838, 0.3482, 0.3588, 0.3162, 0.319, 0.3369, 0.338, 0.3366, 0.3213]) + + self.assertTrue(np.allclose(images, expected, atol=1e-3)) + release_memory(pipe) + + def test_sdxl_0_9_lora_two(self): + generator = torch.Generator().manual_seed(0) + + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-0.9") + lora_model_id = "hf-internal-testing/sdxl-0.9-costumes-lora" + lora_filename = "saijo.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + pipe.enable_model_cpu_offload() + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.3137, 0.3269, 0.3355, 0.255, 0.2577, 0.2563, 0.2679, 0.2758, 0.2626]) + + self.assertTrue(np.allclose(images, expected, atol=1e-3)) + release_memory(pipe) + + def test_sdxl_0_9_lora_three(self): + generator = torch.Generator().manual_seed(0) + + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-0.9") + lora_model_id = "hf-internal-testing/sdxl-0.9-kamepan-lora" + lora_filename = "kame_sdxl_v2-000020-16rank.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + pipe.enable_model_cpu_offload() + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.4015, 0.3761, 0.3616, 0.3745, 0.3462, 0.3337, 0.3564, 0.3649, 0.3468]) + + self.assertTrue(np.allclose(images, expected, atol=5e-3)) + release_memory(pipe) + + def test_sdxl_1_0_lora(self): + generator = torch.Generator("cpu").manual_seed(0) + + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") + pipe.enable_model_cpu_offload() + lora_model_id = "hf-internal-testing/sdxl-1.0-lora" + lora_filename = "sd_xl_offset_example-lora_1.0.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.4468, 0.4087, 0.4134, 0.366, 0.3202, 0.3505, 0.3786, 0.387, 0.3535]) + + self.assertTrue(np.allclose(images, expected, atol=1e-4)) + release_memory(pipe) + + def test_sdxl_lcm_lora(self): + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16) + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + pipe.enable_model_cpu_offload() + + generator = torch.Generator("cpu").manual_seed(0) + + lora_model_id = "latent-consistency/lcm-lora-sdxl" + + pipe.load_lora_weights(lora_model_id) + + image = pipe( + "masterpiece, best quality, mountain", generator=generator, num_inference_steps=4, guidance_scale=0.5 + ).images[0] + + expected_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/lcm_lora/sdxl_lcm_lora.png" + ) + + image_np = pipe.image_processor.pil_to_numpy(image) + expected_image_np = pipe.image_processor.pil_to_numpy(expected_image) + + max_diff = numpy_cosine_similarity_distance(image_np.flatten(), expected_image_np.flatten()) + assert max_diff < 1e-4 + + pipe.unload_lora_weights() + + release_memory(pipe) + + def test_sdv1_5_lcm_lora(self): + pipe = DiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) + pipe.to("cuda") + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + + generator = torch.Generator("cpu").manual_seed(0) + + lora_model_id = "latent-consistency/lcm-lora-sdv1-5" + pipe.load_lora_weights(lora_model_id) + + image = pipe( + "masterpiece, best quality, mountain", generator=generator, num_inference_steps=4, guidance_scale=0.5 + ).images[0] + + expected_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/lcm_lora/sdv15_lcm_lora.png" + ) + + image_np = pipe.image_processor.pil_to_numpy(image) + expected_image_np = pipe.image_processor.pil_to_numpy(expected_image) + + max_diff = numpy_cosine_similarity_distance(image_np.flatten(), expected_image_np.flatten()) + assert max_diff < 1e-4 + + pipe.unload_lora_weights() + + release_memory(pipe) + + def test_sdv1_5_lcm_lora_img2img(self): + pipe = AutoPipelineForImage2Image.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) + pipe.to("cuda") + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/fantasy_landscape.png" + ) + + generator = torch.Generator("cpu").manual_seed(0) + + lora_model_id = "latent-consistency/lcm-lora-sdv1-5" + pipe.load_lora_weights(lora_model_id) + + image = pipe( + "snowy mountain", + generator=generator, + image=init_image, + strength=0.5, + num_inference_steps=4, + guidance_scale=0.5, + ).images[0] + + expected_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/lcm_lora/sdv15_lcm_lora_img2img.png" + ) + + image_np = pipe.image_processor.pil_to_numpy(image) + expected_image_np = pipe.image_processor.pil_to_numpy(expected_image) + + max_diff = numpy_cosine_similarity_distance(image_np.flatten(), expected_image_np.flatten()) + assert max_diff < 1e-4 + + pipe.unload_lora_weights() + + release_memory(pipe) + + def test_sdxl_1_0_lora_fusion(self): + generator = torch.Generator().manual_seed(0) + + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") + lora_model_id = "hf-internal-testing/sdxl-1.0-lora" + lora_filename = "sd_xl_offset_example-lora_1.0.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + pipe.fuse_lora() + # We need to unload the lora weights since in the previous API `fuse_lora` led to lora weights being + # silently deleted - otherwise this will CPU OOM + pipe.unload_lora_weights() + + pipe.enable_model_cpu_offload() + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + # This way we also test equivalence between LoRA fusion and the non-fusion behaviour. + expected = np.array([0.4468, 0.4087, 0.4134, 0.366, 0.3202, 0.3505, 0.3786, 0.387, 0.3535]) + + self.assertTrue(np.allclose(images, expected, atol=1e-4)) + release_memory(pipe) + + def test_sdxl_1_0_lora_unfusion(self): + generator = torch.Generator("cpu").manual_seed(0) + + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") + lora_model_id = "hf-internal-testing/sdxl-1.0-lora" + lora_filename = "sd_xl_offset_example-lora_1.0.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + pipe.fuse_lora() + + pipe.enable_model_cpu_offload() + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=3 + ).images + images_with_fusion = images.flatten() + + pipe.unfuse_lora() + generator = torch.Generator("cpu").manual_seed(0) + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=3 + ).images + images_without_fusion = images.flatten() + + max_diff = numpy_cosine_similarity_distance(images_with_fusion, images_without_fusion) + assert max_diff < 1e-4 + + release_memory(pipe) + + def test_sdxl_1_0_lora_unfusion_effectivity(self): + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") + pipe.enable_model_cpu_offload() + + generator = torch.Generator().manual_seed(0) + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + original_image_slice = images[0, -3:, -3:, -1].flatten() + + lora_model_id = "hf-internal-testing/sdxl-1.0-lora" + lora_filename = "sd_xl_offset_example-lora_1.0.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + pipe.fuse_lora() + + generator = torch.Generator().manual_seed(0) + _ = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + pipe.unfuse_lora() + + # We need to unload the lora weights - in the old API unfuse led to unloading the adapter weights + pipe.unload_lora_weights() + + generator = torch.Generator().manual_seed(0) + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + images_without_fusion_slice = images[0, -3:, -3:, -1].flatten() + + self.assertTrue(np.allclose(original_image_slice, images_without_fusion_slice, atol=1e-3)) + release_memory(pipe) + + def test_sdxl_1_0_lora_fusion_efficiency(self): + generator = torch.Generator().manual_seed(0) + lora_model_id = "hf-internal-testing/sdxl-1.0-lora" + lora_filename = "sd_xl_offset_example-lora_1.0.safetensors" + + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16) + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename, torch_dtype=torch.float16) + pipe.enable_model_cpu_offload() + + start_time = time.time() + for _ in range(3): + pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + end_time = time.time() + elapsed_time_non_fusion = end_time - start_time + + del pipe + + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16) + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename, torch_dtype=torch.float16) + pipe.fuse_lora() + + # We need to unload the lora weights since in the previous API `fuse_lora` led to lora weights being + # silently deleted - otherwise this will CPU OOM + pipe.unload_lora_weights() + pipe.enable_model_cpu_offload() + + generator = torch.Generator().manual_seed(0) + start_time = time.time() + for _ in range(3): + pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + end_time = time.time() + elapsed_time_fusion = end_time - start_time + + self.assertTrue(elapsed_time_fusion < elapsed_time_non_fusion) + release_memory(pipe) + + def test_sdxl_1_0_last_ben(self): + generator = torch.Generator().manual_seed(0) + + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") + pipe.enable_model_cpu_offload() + lora_model_id = "TheLastBen/Papercut_SDXL" + lora_filename = "papercut.safetensors" + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe("papercut.safetensors", output_type="np", generator=generator, num_inference_steps=2).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.5244, 0.4347, 0.4312, 0.4246, 0.4398, 0.4409, 0.4884, 0.4938, 0.4094]) + + self.assertTrue(np.allclose(images, expected, atol=1e-3)) + release_memory(pipe) + + def test_sdxl_1_0_fuse_unfuse_all(self): + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16) + text_encoder_1_sd = copy.deepcopy(pipe.text_encoder.state_dict()) + text_encoder_2_sd = copy.deepcopy(pipe.text_encoder_2.state_dict()) + unet_sd = copy.deepcopy(pipe.unet.state_dict()) + + pipe.load_lora_weights( + "davizca87/sun-flower", weight_name="snfw3rXL-000004.safetensors", torch_dtype=torch.float16 + ) + + fused_te_state_dict = pipe.text_encoder.state_dict() + fused_te_2_state_dict = pipe.text_encoder_2.state_dict() + unet_state_dict = pipe.unet.state_dict() + + peft_ge_070 = version.parse(importlib.metadata.version("peft")) >= version.parse("0.7.0") + + def remap_key(key, sd): + # some keys have moved around for PEFT >= 0.7.0, but they should still be loaded correctly + if (key in sd) or (not peft_ge_070): + return key + + # instead of linear.weight, we now have linear.base_layer.weight, etc. + if key.endswith(".weight"): + key = key[:-7] + ".base_layer.weight" + elif key.endswith(".bias"): + key = key[:-5] + ".base_layer.bias" + return key + + for key, value in text_encoder_1_sd.items(): + key = remap_key(key, fused_te_state_dict) + self.assertTrue(torch.allclose(fused_te_state_dict[key], value)) + + for key, value in text_encoder_2_sd.items(): + key = remap_key(key, fused_te_2_state_dict) + self.assertTrue(torch.allclose(fused_te_2_state_dict[key], value)) + + for key, value in unet_state_dict.items(): + self.assertTrue(torch.allclose(unet_state_dict[key], value)) + + pipe.fuse_lora() + pipe.unload_lora_weights() + + assert not state_dicts_almost_equal(text_encoder_1_sd, pipe.text_encoder.state_dict()) + assert not state_dicts_almost_equal(text_encoder_2_sd, pipe.text_encoder_2.state_dict()) + assert not state_dicts_almost_equal(unet_sd, pipe.unet.state_dict()) + release_memory(pipe) + del unet_sd, text_encoder_1_sd, text_encoder_2_sd + + def test_sdxl_1_0_lora_with_sequential_cpu_offloading(self): + generator = torch.Generator().manual_seed(0) + + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0") + pipe.enable_sequential_cpu_offload() + lora_model_id = "hf-internal-testing/sdxl-1.0-lora" + lora_filename = "sd_xl_offset_example-lora_1.0.safetensors" + + pipe.load_lora_weights(lora_model_id, weight_name=lora_filename) + + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.4468, 0.4087, 0.4134, 0.366, 0.3202, 0.3505, 0.3786, 0.387, 0.3535]) + + self.assertTrue(np.allclose(images, expected, atol=1e-3)) + release_memory(pipe) + + def test_sd_load_civitai_empty_network_alpha(self): + """ + This test simply checks that loading a LoRA with an empty network alpha works fine + See: https://github.com/huggingface/diffusers/issues/5606 + """ + pipeline = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5").to("cuda") + pipeline.enable_sequential_cpu_offload() + civitai_path = hf_hub_download("ybelkada/test-ahi-civitai", "ahi_lora_weights.safetensors") + pipeline.load_lora_weights(civitai_path, adapter_name="ahri") + + images = pipeline( + "ahri, masterpiece, league of legends", + output_type="np", + generator=torch.manual_seed(156), + num_inference_steps=5, + ).images + images = images[0, -3:, -3:, -1].flatten() + expected = np.array([0.0, 0.0, 0.0, 0.002557, 0.020954, 0.001792, 0.006581, 0.00591, 0.002995]) + + self.assertTrue(np.allclose(images, expected, atol=1e-3)) + release_memory(pipeline) + + def test_controlnet_canny_lora(self): + controlnet = ControlNetModel.from_pretrained("diffusers/controlnet-canny-sdxl-1.0") + + pipe = StableDiffusionXLControlNetPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", controlnet=controlnet + ) + pipe.load_lora_weights("nerijs/pixel-art-xl", weight_name="pixel-art-xl.safetensors") + pipe.enable_sequential_cpu_offload() + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "corgi" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ) + + images = pipe(prompt, image=image, generator=generator, output_type="np", num_inference_steps=3).images + + assert images[0].shape == (768, 512, 3) + + original_image = images[0, -3:, -3:, -1].flatten() + expected_image = np.array([0.4574, 0.4461, 0.4435, 0.4462, 0.4396, 0.439, 0.4474, 0.4486, 0.4333]) + assert np.allclose(original_image, expected_image, atol=1e-04) + release_memory(pipe) + + def test_sdxl_t2i_adapter_canny_lora(self): + adapter = T2IAdapter.from_pretrained("TencentARC/t2i-adapter-lineart-sdxl-1.0", torch_dtype=torch.float16).to( + "cpu" + ) + pipe = StableDiffusionXLAdapterPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + adapter=adapter, + torch_dtype=torch.float16, + variant="fp16", + ) + pipe.load_lora_weights("CiroN2022/toy-face", weight_name="toy_face_sdxl.safetensors") + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "toy" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/toy_canny.png" + ) + + images = pipe(prompt, image=image, generator=generator, output_type="np", num_inference_steps=3).images + + assert images[0].shape == (768, 512, 3) + + image_slice = images[0, -3:, -3:, -1].flatten() + expected_slice = np.array([0.4284, 0.4337, 0.4319, 0.4255, 0.4329, 0.4280, 0.4338, 0.4420, 0.4226]) + assert numpy_cosine_similarity_distance(image_slice, expected_slice) < 1e-4 + + @nightly + def test_sequential_fuse_unfuse(self): + pipe = DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16) + + # 1. round + pipe.load_lora_weights("Pclanglais/TintinIA", torch_dtype=torch.float16) + pipe.to("cuda") + pipe.fuse_lora() + + generator = torch.Generator().manual_seed(0) + images = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + image_slice = images[0, -3:, -3:, -1].flatten() + + pipe.unfuse_lora() + + # 2. round + pipe.load_lora_weights("ProomptEngineer/pe-balloon-diffusion-style", torch_dtype=torch.float16) + pipe.fuse_lora() + pipe.unfuse_lora() + + # 3. round + pipe.load_lora_weights("ostris/crayon_style_lora_sdxl", torch_dtype=torch.float16) + pipe.fuse_lora() + pipe.unfuse_lora() + + # 4. back to 1st round + pipe.load_lora_weights("Pclanglais/TintinIA", torch_dtype=torch.float16) + pipe.fuse_lora() + + generator = torch.Generator().manual_seed(0) + images_2 = pipe( + "masterpiece, best quality, mountain", output_type="np", generator=generator, num_inference_steps=2 + ).images + image_slice_2 = images_2[0, -3:, -3:, -1].flatten() + + self.assertTrue(np.allclose(image_slice, image_slice_2, atol=1e-3)) + release_memory(pipe) diff --git a/diffusers-0.27.0/tests/models/__init__.py b/diffusers-0.27.0/tests/models/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/models/autoencoders/__init__.py b/diffusers-0.27.0/tests/models/autoencoders/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/models/autoencoders/test_models_vae.py b/diffusers-0.27.0/tests/models/autoencoders/test_models_vae.py new file mode 100755 index 0000000..5321d8c --- /dev/null +++ b/diffusers-0.27.0/tests/models/autoencoders/test_models_vae.py @@ -0,0 +1,1118 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +from parameterized import parameterized + +from diffusers import ( + AsymmetricAutoencoderKL, + AutoencoderKL, + AutoencoderKLTemporalDecoder, + AutoencoderTiny, + ConsistencyDecoderVAE, + StableDiffusionPipeline, +) +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.loading_utils import load_image +from diffusers.utils.testing_utils import ( + backend_empty_cache, + enable_full_determinism, + floats_tensor, + load_hf_numpy, + require_torch_accelerator, + require_torch_accelerator_with_fp16, + require_torch_accelerator_with_training, + require_torch_gpu, + skip_mps, + slow, + torch_all_close, + torch_device, +) +from diffusers.utils.torch_utils import randn_tensor + +from ..test_modeling_common import ModelTesterMixin, UNetTesterMixin + + +enable_full_determinism() + + +def get_autoencoder_kl_config(block_out_channels=None, norm_num_groups=None): + block_out_channels = block_out_channels or [32, 64] + norm_num_groups = norm_num_groups or 32 + init_dict = { + "block_out_channels": block_out_channels, + "in_channels": 3, + "out_channels": 3, + "down_block_types": ["DownEncoderBlock2D"] * len(block_out_channels), + "up_block_types": ["UpDecoderBlock2D"] * len(block_out_channels), + "latent_channels": 4, + "norm_num_groups": norm_num_groups, + } + return init_dict + + +def get_asym_autoencoder_kl_config(block_out_channels=None, norm_num_groups=None): + block_out_channels = block_out_channels or [32, 64] + norm_num_groups = norm_num_groups or 32 + init_dict = { + "in_channels": 3, + "out_channels": 3, + "down_block_types": ["DownEncoderBlock2D"] * len(block_out_channels), + "down_block_out_channels": block_out_channels, + "layers_per_down_block": 1, + "up_block_types": ["UpDecoderBlock2D"] * len(block_out_channels), + "up_block_out_channels": block_out_channels, + "layers_per_up_block": 1, + "act_fn": "silu", + "latent_channels": 4, + "norm_num_groups": norm_num_groups, + "sample_size": 32, + "scaling_factor": 0.18215, + } + return init_dict + + +def get_autoencoder_tiny_config(block_out_channels=None): + block_out_channels = (len(block_out_channels) * [32]) if block_out_channels is not None else [32, 32] + init_dict = { + "in_channels": 3, + "out_channels": 3, + "encoder_block_out_channels": block_out_channels, + "decoder_block_out_channels": block_out_channels, + "num_encoder_blocks": [b // min(block_out_channels) for b in block_out_channels], + "num_decoder_blocks": [b // min(block_out_channels) for b in reversed(block_out_channels)], + } + return init_dict + + +def get_consistency_vae_config(block_out_channels=None, norm_num_groups=None): + block_out_channels = block_out_channels or [32, 64] + norm_num_groups = norm_num_groups or 32 + return { + "encoder_block_out_channels": block_out_channels, + "encoder_in_channels": 3, + "encoder_out_channels": 4, + "encoder_down_block_types": ["DownEncoderBlock2D"] * len(block_out_channels), + "decoder_add_attention": False, + "decoder_block_out_channels": block_out_channels, + "decoder_down_block_types": ["ResnetDownsampleBlock2D"] * len(block_out_channels), + "decoder_downsample_padding": 1, + "decoder_in_channels": 7, + "decoder_layers_per_block": 1, + "decoder_norm_eps": 1e-05, + "decoder_norm_num_groups": norm_num_groups, + "encoder_norm_num_groups": norm_num_groups, + "decoder_num_train_timesteps": 1024, + "decoder_out_channels": 6, + "decoder_resnet_time_scale_shift": "scale_shift", + "decoder_time_embedding_type": "learned", + "decoder_up_block_types": ["ResnetUpsampleBlock2D"] * len(block_out_channels), + "scaling_factor": 1, + "latent_channels": 4, + } + + +class AutoencoderKLTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = AutoencoderKL + main_input_name = "sample" + base_precision = 1e-2 + + @property + def dummy_input(self): + batch_size = 4 + num_channels = 3 + sizes = (32, 32) + + image = floats_tensor((batch_size, num_channels) + sizes).to(torch_device) + + return {"sample": image} + + @property + def input_shape(self): + return (3, 32, 32) + + @property + def output_shape(self): + return (3, 32, 32) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = get_autoencoder_kl_config() + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_forward_signature(self): + pass + + def test_training(self): + pass + + @require_torch_accelerator_with_training + def test_gradient_checkpointing(self): + # enable deterministic behavior for gradient checkpointing + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.to(torch_device) + + assert not model.is_gradient_checkpointing and model.training + + out = model(**inputs_dict).sample + # run the backwards pass on the model. For backwards pass, for simplicity purpose, + # we won't calculate the loss and rather backprop on out.sum() + model.zero_grad() + + labels = torch.randn_like(out) + loss = (out - labels).mean() + loss.backward() + + # re-instantiate the model now enabling gradient checkpointing + model_2 = self.model_class(**init_dict) + # clone model + model_2.load_state_dict(model.state_dict()) + model_2.to(torch_device) + model_2.enable_gradient_checkpointing() + + assert model_2.is_gradient_checkpointing and model_2.training + + out_2 = model_2(**inputs_dict).sample + # run the backwards pass on the model. For backwards pass, for simplicity purpose, + # we won't calculate the loss and rather backprop on out.sum() + model_2.zero_grad() + loss_2 = (out_2 - labels).mean() + loss_2.backward() + + # compare the output and parameters gradients + self.assertTrue((loss - loss_2).abs() < 1e-5) + named_params = dict(model.named_parameters()) + named_params_2 = dict(model_2.named_parameters()) + for name, param in named_params.items(): + self.assertTrue(torch_all_close(param.grad.data, named_params_2[name].grad.data, atol=5e-5)) + + def test_from_pretrained_hub(self): + model, loading_info = AutoencoderKL.from_pretrained("fusing/autoencoder-kl-dummy", output_loading_info=True) + self.assertIsNotNone(model) + self.assertEqual(len(loading_info["missing_keys"]), 0) + + model.to(torch_device) + image = model(**self.dummy_input) + + assert image is not None, "Make sure output is not None" + + def test_output_pretrained(self): + model = AutoencoderKL.from_pretrained("fusing/autoencoder-kl-dummy") + model = model.to(torch_device) + model.eval() + + # Keep generator on CPU for non-CUDA devices to compare outputs with CPU result tensors + generator_device = "cpu" if not torch_device.startswith("cuda") else "cuda" + if torch_device != "mps": + generator = torch.Generator(device=generator_device).manual_seed(0) + else: + generator = torch.manual_seed(0) + + image = torch.randn( + 1, + model.config.in_channels, + model.config.sample_size, + model.config.sample_size, + generator=torch.manual_seed(0), + ) + image = image.to(torch_device) + with torch.no_grad(): + output = model(image, sample_posterior=True, generator=generator).sample + + output_slice = output[0, -1, -3:, -3:].flatten().cpu() + + # Since the VAE Gaussian prior's generator is seeded on the appropriate device, + # the expected output slices are not the same for CPU and GPU. + if torch_device == "mps": + expected_output_slice = torch.tensor( + [ + -4.0078e-01, + -3.8323e-04, + -1.2681e-01, + -1.1462e-01, + 2.0095e-01, + 1.0893e-01, + -8.8247e-02, + -3.0361e-01, + -9.8644e-03, + ] + ) + elif generator_device == "cpu": + expected_output_slice = torch.tensor( + [ + -0.1352, + 0.0878, + 0.0419, + -0.0818, + -0.1069, + 0.0688, + -0.1458, + -0.4446, + -0.0026, + ] + ) + else: + expected_output_slice = torch.tensor( + [ + -0.2421, + 0.4642, + 0.2507, + -0.0438, + 0.0682, + 0.3160, + -0.2018, + -0.0727, + 0.2485, + ] + ) + + self.assertTrue(torch_all_close(output_slice, expected_output_slice, rtol=1e-2)) + + +class AsymmetricAutoencoderKLTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = AsymmetricAutoencoderKL + main_input_name = "sample" + base_precision = 1e-2 + + @property + def dummy_input(self): + batch_size = 4 + num_channels = 3 + sizes = (32, 32) + + image = floats_tensor((batch_size, num_channels) + sizes).to(torch_device) + mask = torch.ones((batch_size, 1) + sizes).to(torch_device) + + return {"sample": image, "mask": mask} + + @property + def input_shape(self): + return (3, 32, 32) + + @property + def output_shape(self): + return (3, 32, 32) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = get_asym_autoencoder_kl_config() + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_forward_signature(self): + pass + + def test_forward_with_norm_groups(self): + pass + + +class AutoencoderTinyTests(ModelTesterMixin, unittest.TestCase): + model_class = AutoencoderTiny + main_input_name = "sample" + base_precision = 1e-2 + + @property + def dummy_input(self): + batch_size = 4 + num_channels = 3 + sizes = (32, 32) + + image = floats_tensor((batch_size, num_channels) + sizes).to(torch_device) + + return {"sample": image} + + @property + def input_shape(self): + return (3, 32, 32) + + @property + def output_shape(self): + return (3, 32, 32) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = get_autoencoder_tiny_config() + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_outputs_equivalence(self): + pass + + +class ConsistencyDecoderVAETests(ModelTesterMixin, unittest.TestCase): + model_class = ConsistencyDecoderVAE + main_input_name = "sample" + base_precision = 1e-2 + forward_requires_fresh_args = True + + def inputs_dict(self, seed=None): + generator = torch.Generator("cpu") + if seed is not None: + generator.manual_seed(0) + image = randn_tensor((4, 3, 32, 32), generator=generator, device=torch.device(torch_device)) + + return {"sample": image, "generator": generator} + + @property + def input_shape(self): + return (3, 32, 32) + + @property + def output_shape(self): + return (3, 32, 32) + + @property + def init_dict(self): + return get_consistency_vae_config() + + def prepare_init_args_and_inputs_for_common(self): + return self.init_dict, self.inputs_dict() + + @unittest.skip + def test_training(self): + ... + + @unittest.skip + def test_ema_training(self): + ... + + +class AutoncoderKLTemporalDecoderFastTests(ModelTesterMixin, unittest.TestCase): + model_class = AutoencoderKLTemporalDecoder + main_input_name = "sample" + base_precision = 1e-2 + + @property + def dummy_input(self): + batch_size = 3 + num_channels = 3 + sizes = (32, 32) + + image = floats_tensor((batch_size, num_channels) + sizes).to(torch_device) + num_frames = 3 + + return {"sample": image, "num_frames": num_frames} + + @property + def input_shape(self): + return (3, 32, 32) + + @property + def output_shape(self): + return (3, 32, 32) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "block_out_channels": [32, 64], + "in_channels": 3, + "out_channels": 3, + "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"], + "latent_channels": 4, + "layers_per_block": 2, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_forward_signature(self): + pass + + def test_training(self): + pass + + @unittest.skipIf(torch_device == "mps", "Gradient checkpointing skipped on MPS") + def test_gradient_checkpointing(self): + # enable deterministic behavior for gradient checkpointing + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.to(torch_device) + + assert not model.is_gradient_checkpointing and model.training + + out = model(**inputs_dict).sample + # run the backwards pass on the model. For backwards pass, for simplicity purpose, + # we won't calculate the loss and rather backprop on out.sum() + model.zero_grad() + + labels = torch.randn_like(out) + loss = (out - labels).mean() + loss.backward() + + # re-instantiate the model now enabling gradient checkpointing + model_2 = self.model_class(**init_dict) + # clone model + model_2.load_state_dict(model.state_dict()) + model_2.to(torch_device) + model_2.enable_gradient_checkpointing() + + assert model_2.is_gradient_checkpointing and model_2.training + + out_2 = model_2(**inputs_dict).sample + # run the backwards pass on the model. For backwards pass, for simplicity purpose, + # we won't calculate the loss and rather backprop on out.sum() + model_2.zero_grad() + loss_2 = (out_2 - labels).mean() + loss_2.backward() + + # compare the output and parameters gradients + self.assertTrue((loss - loss_2).abs() < 1e-5) + named_params = dict(model.named_parameters()) + named_params_2 = dict(model_2.named_parameters()) + for name, param in named_params.items(): + if "post_quant_conv" in name: + continue + + self.assertTrue(torch_all_close(param.grad.data, named_params_2[name].grad.data, atol=5e-5)) + + +@slow +class AutoencoderTinyIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + backend_empty_cache(torch_device) + + def get_file_format(self, seed, shape): + return f"gaussian_noise_s={seed}_shape={'_'.join([str(s) for s in shape])}.npy" + + def get_sd_image(self, seed=0, shape=(4, 3, 512, 512), fp16=False): + dtype = torch.float16 if fp16 else torch.float32 + image = torch.from_numpy(load_hf_numpy(self.get_file_format(seed, shape))).to(torch_device).to(dtype) + return image + + def get_sd_vae_model(self, model_id="hf-internal-testing/taesd-diffusers", fp16=False): + torch_dtype = torch.float16 if fp16 else torch.float32 + + model = AutoencoderTiny.from_pretrained(model_id, torch_dtype=torch_dtype) + model.to(torch_device).eval() + return model + + @parameterized.expand( + [ + [(1, 4, 73, 97), (1, 3, 584, 776)], + [(1, 4, 97, 73), (1, 3, 776, 584)], + [(1, 4, 49, 65), (1, 3, 392, 520)], + [(1, 4, 65, 49), (1, 3, 520, 392)], + [(1, 4, 49, 49), (1, 3, 392, 392)], + ] + ) + def test_tae_tiling(self, in_shape, out_shape): + model = self.get_sd_vae_model() + model.enable_tiling() + with torch.no_grad(): + zeros = torch.zeros(in_shape).to(torch_device) + dec = model.decode(zeros).sample + assert dec.shape == out_shape + + def test_stable_diffusion(self): + model = self.get_sd_vae_model() + image = self.get_sd_image(seed=33) + + with torch.no_grad(): + sample = model(image).sample + + assert sample.shape == image.shape + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor([0.0093, 0.6385, -0.1274, 0.1631, -0.1762, 0.5232, -0.3108, -0.0382]) + + assert torch_all_close(output_slice, expected_output_slice, atol=3e-3) + + @parameterized.expand([(True,), (False,)]) + def test_tae_roundtrip(self, enable_tiling): + # load the autoencoder + model = self.get_sd_vae_model() + if enable_tiling: + model.enable_tiling() + + # make a black image with a white square in the middle, + # which is large enough to split across multiple tiles + image = -torch.ones(1, 3, 1024, 1024, device=torch_device) + image[..., 256:768, 256:768] = 1.0 + + # round-trip the image through the autoencoder + with torch.no_grad(): + sample = model(image).sample + + # the autoencoder reconstruction should match original image, sorta + def downscale(x): + return torch.nn.functional.avg_pool2d(x, model.spatial_scale_factor) + + assert torch_all_close(downscale(sample), downscale(image), atol=0.125) + + +@slow +class AutoencoderKLIntegrationTests(unittest.TestCase): + def get_file_format(self, seed, shape): + return f"gaussian_noise_s={seed}_shape={'_'.join([str(s) for s in shape])}.npy" + + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + backend_empty_cache(torch_device) + + def get_sd_image(self, seed=0, shape=(4, 3, 512, 512), fp16=False): + dtype = torch.float16 if fp16 else torch.float32 + image = torch.from_numpy(load_hf_numpy(self.get_file_format(seed, shape))).to(torch_device).to(dtype) + return image + + def get_sd_vae_model(self, model_id="CompVis/stable-diffusion-v1-4", fp16=False): + revision = "fp16" if fp16 else None + torch_dtype = torch.float16 if fp16 else torch.float32 + + model = AutoencoderKL.from_pretrained( + model_id, + subfolder="vae", + torch_dtype=torch_dtype, + revision=revision, + ) + model.to(torch_device) + + return model + + def get_generator(self, seed=0): + generator_device = "cpu" if not torch_device.startswith("cuda") else "cuda" + if torch_device != "mps": + return torch.Generator(device=generator_device).manual_seed(seed) + return torch.manual_seed(seed) + + @parameterized.expand( + [ + # fmt: off + [ + 33, + [-0.1603, 0.9878, -0.0495, -0.0790, -0.2709, 0.8375, -0.2060, -0.0824], + [-0.2395, 0.0098, 0.0102, -0.0709, -0.2840, -0.0274, -0.0718, -0.1824], + ], + [ + 47, + [-0.2376, 0.1168, 0.1332, -0.4840, -0.2508, -0.0791, -0.0493, -0.4089], + [0.0350, 0.0847, 0.0467, 0.0344, -0.0842, -0.0547, -0.0633, -0.1131], + ], + # fmt: on + ] + ) + def test_stable_diffusion(self, seed, expected_slice, expected_slice_mps): + model = self.get_sd_vae_model() + image = self.get_sd_image(seed) + generator = self.get_generator(seed) + + with torch.no_grad(): + sample = model(image, generator=generator, sample_posterior=True).sample + + assert sample.shape == image.shape + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice_mps if torch_device == "mps" else expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=3e-3) + + @parameterized.expand( + [ + # fmt: off + [33, [-0.0513, 0.0289, 1.3799, 0.2166, -0.2573, -0.0871, 0.5103, -0.0999]], + [47, [-0.4128, -0.1320, -0.3704, 0.1965, -0.4116, -0.2332, -0.3340, 0.2247]], + # fmt: on + ] + ) + @require_torch_accelerator_with_fp16 + def test_stable_diffusion_fp16(self, seed, expected_slice): + model = self.get_sd_vae_model(fp16=True) + image = self.get_sd_image(seed, fp16=True) + generator = self.get_generator(seed) + + with torch.no_grad(): + sample = model(image, generator=generator, sample_posterior=True).sample + + assert sample.shape == image.shape + + output_slice = sample[-1, -2:, :2, -2:].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=1e-2) + + @parameterized.expand( + [ + # fmt: off + [ + 33, + [-0.1609, 0.9866, -0.0487, -0.0777, -0.2716, 0.8368, -0.2055, -0.0814], + [-0.2395, 0.0098, 0.0102, -0.0709, -0.2840, -0.0274, -0.0718, -0.1824], + ], + [ + 47, + [-0.2377, 0.1147, 0.1333, -0.4841, -0.2506, -0.0805, -0.0491, -0.4085], + [0.0350, 0.0847, 0.0467, 0.0344, -0.0842, -0.0547, -0.0633, -0.1131], + ], + # fmt: on + ] + ) + def test_stable_diffusion_mode(self, seed, expected_slice, expected_slice_mps): + model = self.get_sd_vae_model() + image = self.get_sd_image(seed) + + with torch.no_grad(): + sample = model(image).sample + + assert sample.shape == image.shape + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice_mps if torch_device == "mps" else expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=3e-3) + + @parameterized.expand( + [ + # fmt: off + [13, [-0.2051, -0.1803, -0.2311, -0.2114, -0.3292, -0.3574, -0.2953, -0.3323]], + [37, [-0.2632, -0.2625, -0.2199, -0.2741, -0.4539, -0.4990, -0.3720, -0.4925]], + # fmt: on + ] + ) + @require_torch_accelerator + @skip_mps + def test_stable_diffusion_decode(self, seed, expected_slice): + model = self.get_sd_vae_model() + encoding = self.get_sd_image(seed, shape=(3, 4, 64, 64)) + + with torch.no_grad(): + sample = model.decode(encoding).sample + + assert list(sample.shape) == [3, 3, 512, 512] + + output_slice = sample[-1, -2:, :2, -2:].flatten().cpu() + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=1e-3) + + @parameterized.expand( + [ + # fmt: off + [27, [-0.0369, 0.0207, -0.0776, -0.0682, -0.1747, -0.1930, -0.1465, -0.2039]], + [16, [-0.1628, -0.2134, -0.2747, -0.2642, -0.3774, -0.4404, -0.3687, -0.4277]], + # fmt: on + ] + ) + @require_torch_accelerator_with_fp16 + def test_stable_diffusion_decode_fp16(self, seed, expected_slice): + model = self.get_sd_vae_model(fp16=True) + encoding = self.get_sd_image(seed, shape=(3, 4, 64, 64), fp16=True) + + with torch.no_grad(): + sample = model.decode(encoding).sample + + assert list(sample.shape) == [3, 3, 512, 512] + + output_slice = sample[-1, -2:, :2, -2:].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=5e-3) + + @parameterized.expand([(13,), (16,), (27,)]) + @require_torch_gpu + @unittest.skipIf( + not is_xformers_available(), + reason="xformers is not required when using PyTorch 2.0.", + ) + def test_stable_diffusion_decode_xformers_vs_2_0_fp16(self, seed): + model = self.get_sd_vae_model(fp16=True) + encoding = self.get_sd_image(seed, shape=(3, 4, 64, 64), fp16=True) + + with torch.no_grad(): + sample = model.decode(encoding).sample + + model.enable_xformers_memory_efficient_attention() + with torch.no_grad(): + sample_2 = model.decode(encoding).sample + + assert list(sample.shape) == [3, 3, 512, 512] + + assert torch_all_close(sample, sample_2, atol=1e-1) + + @parameterized.expand([(13,), (16,), (37,)]) + @require_torch_gpu + @unittest.skipIf( + not is_xformers_available(), + reason="xformers is not required when using PyTorch 2.0.", + ) + def test_stable_diffusion_decode_xformers_vs_2_0(self, seed): + model = self.get_sd_vae_model() + encoding = self.get_sd_image(seed, shape=(3, 4, 64, 64)) + + with torch.no_grad(): + sample = model.decode(encoding).sample + + model.enable_xformers_memory_efficient_attention() + with torch.no_grad(): + sample_2 = model.decode(encoding).sample + + assert list(sample.shape) == [3, 3, 512, 512] + + assert torch_all_close(sample, sample_2, atol=1e-2) + + @parameterized.expand( + [ + # fmt: off + [33, [-0.3001, 0.0918, -2.6984, -3.9720, -3.2099, -5.0353, 1.7338, -0.2065, 3.4267]], + [47, [-1.5030, -4.3871, -6.0355, -9.1157, -1.6661, -2.7853, 2.1607, -5.0823, 2.5633]], + # fmt: on + ] + ) + def test_stable_diffusion_encode_sample(self, seed, expected_slice): + model = self.get_sd_vae_model() + image = self.get_sd_image(seed) + generator = self.get_generator(seed) + + with torch.no_grad(): + dist = model.encode(image).latent_dist + sample = dist.sample(generator=generator) + + assert list(sample.shape) == [image.shape[0], 4] + [i // 8 for i in image.shape[2:]] + + output_slice = sample[0, -1, -3:, -3:].flatten().cpu() + expected_output_slice = torch.tensor(expected_slice) + + tolerance = 3e-3 if torch_device != "mps" else 1e-2 + assert torch_all_close(output_slice, expected_output_slice, atol=tolerance) + + def test_stable_diffusion_model_local(self): + model_id = "stabilityai/sd-vae-ft-mse" + model_1 = AutoencoderKL.from_pretrained(model_id).to(torch_device) + + url = "https://huggingface.co/stabilityai/sd-vae-ft-mse-original/blob/main/vae-ft-mse-840000-ema-pruned.safetensors" + model_2 = AutoencoderKL.from_single_file(url).to(torch_device) + image = self.get_sd_image(33) + + with torch.no_grad(): + sample_1 = model_1(image).sample + sample_2 = model_2(image).sample + + assert sample_1.shape == sample_2.shape + + output_slice_1 = sample_1[-1, -2:, -2:, :2].flatten().float().cpu() + output_slice_2 = sample_2[-1, -2:, -2:, :2].flatten().float().cpu() + + assert torch_all_close(output_slice_1, output_slice_2, atol=3e-3) + + def test_single_file_component_configs(self): + vae_single_file = AutoencoderKL.from_single_file( + "https://huggingface.co/stabilityai/sd-vae-ft-mse-original/blob/main/vae-ft-mse-840000-ema-pruned.safetensors" + ) + vae = AutoencoderKL.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="vae") + + PARAMS_TO_IGNORE = ["torch_dtype", "_name_or_path", "_use_default_values"] + for param_name, param_value in vae_single_file.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + vae.config[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" + + def test_single_file_arguments(self): + vae_default = AutoencoderKL.from_single_file( + "https://huggingface.co/stabilityai/sd-vae-ft-mse-original/blob/main/vae-ft-mse-840000-ema-pruned.safetensors", + ) + + assert vae_default.config.scaling_factor == 0.18215 + assert vae_default.config.sample_size == 512 + assert vae_default.dtype == torch.float32 + + scaling_factor = 2.0 + image_size = 256 + torch_dtype = torch.float16 + + vae = AutoencoderKL.from_single_file( + "https://huggingface.co/stabilityai/sd-vae-ft-mse-original/blob/main/vae-ft-mse-840000-ema-pruned.safetensors", + image_size=image_size, + scaling_factor=scaling_factor, + torch_dtype=torch_dtype, + ) + assert vae.config.scaling_factor == scaling_factor + assert vae.config.sample_size == image_size + assert vae.dtype == torch_dtype + + +@slow +class AsymmetricAutoencoderKLIntegrationTests(unittest.TestCase): + def get_file_format(self, seed, shape): + return f"gaussian_noise_s={seed}_shape={'_'.join([str(s) for s in shape])}.npy" + + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + backend_empty_cache(torch_device) + + def get_sd_image(self, seed=0, shape=(4, 3, 512, 512), fp16=False): + dtype = torch.float16 if fp16 else torch.float32 + image = torch.from_numpy(load_hf_numpy(self.get_file_format(seed, shape))).to(torch_device).to(dtype) + return image + + def get_sd_vae_model(self, model_id="cross-attention/asymmetric-autoencoder-kl-x-1-5", fp16=False): + revision = "main" + torch_dtype = torch.float32 + + model = AsymmetricAutoencoderKL.from_pretrained( + model_id, + torch_dtype=torch_dtype, + revision=revision, + ) + model.to(torch_device).eval() + + return model + + def get_generator(self, seed=0): + generator_device = "cpu" if not torch_device.startswith("cuda") else "cuda" + if torch_device != "mps": + return torch.Generator(device=generator_device).manual_seed(seed) + return torch.manual_seed(seed) + + @parameterized.expand( + [ + # fmt: off + [ + 33, + [-0.0344, 0.2912, 0.1687, -0.0137, -0.3462, 0.3552, -0.1337, 0.1078], + [-0.1603, 0.9878, -0.0495, -0.0790, -0.2709, 0.8375, -0.2060, -0.0824], + ], + [ + 47, + [0.4400, 0.0543, 0.2873, 0.2946, 0.0553, 0.0839, -0.1585, 0.2529], + [-0.2376, 0.1168, 0.1332, -0.4840, -0.2508, -0.0791, -0.0493, -0.4089], + ], + # fmt: on + ] + ) + def test_stable_diffusion(self, seed, expected_slice, expected_slice_mps): + model = self.get_sd_vae_model() + image = self.get_sd_image(seed) + generator = self.get_generator(seed) + + with torch.no_grad(): + sample = model(image, generator=generator, sample_posterior=True).sample + + assert sample.shape == image.shape + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice_mps if torch_device == "mps" else expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=5e-3) + + @parameterized.expand( + [ + # fmt: off + [ + 33, + [-0.0340, 0.2870, 0.1698, -0.0105, -0.3448, 0.3529, -0.1321, 0.1097], + [-0.0344, 0.2912, 0.1687, -0.0137, -0.3462, 0.3552, -0.1337, 0.1078], + ], + [ + 47, + [0.4397, 0.0550, 0.2873, 0.2946, 0.0567, 0.0855, -0.1580, 0.2531], + [0.4397, 0.0550, 0.2873, 0.2946, 0.0567, 0.0855, -0.1580, 0.2531], + ], + # fmt: on + ] + ) + def test_stable_diffusion_mode(self, seed, expected_slice, expected_slice_mps): + model = self.get_sd_vae_model() + image = self.get_sd_image(seed) + + with torch.no_grad(): + sample = model(image).sample + + assert sample.shape == image.shape + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice_mps if torch_device == "mps" else expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=3e-3) + + @parameterized.expand( + [ + # fmt: off + [13, [-0.0521, -0.2939, 0.1540, -0.1855, -0.5936, -0.3138, -0.4579, -0.2275]], + [37, [-0.1820, -0.4345, -0.0455, -0.2923, -0.8035, -0.5089, -0.4795, -0.3106]], + # fmt: on + ] + ) + @require_torch_accelerator + @skip_mps + def test_stable_diffusion_decode(self, seed, expected_slice): + model = self.get_sd_vae_model() + encoding = self.get_sd_image(seed, shape=(3, 4, 64, 64)) + + with torch.no_grad(): + sample = model.decode(encoding).sample + + assert list(sample.shape) == [3, 3, 512, 512] + + output_slice = sample[-1, -2:, :2, -2:].flatten().cpu() + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=2e-3) + + @parameterized.expand([(13,), (16,), (37,)]) + @require_torch_gpu + @unittest.skipIf( + not is_xformers_available(), + reason="xformers is not required when using PyTorch 2.0.", + ) + def test_stable_diffusion_decode_xformers_vs_2_0(self, seed): + model = self.get_sd_vae_model() + encoding = self.get_sd_image(seed, shape=(3, 4, 64, 64)) + + with torch.no_grad(): + sample = model.decode(encoding).sample + + model.enable_xformers_memory_efficient_attention() + with torch.no_grad(): + sample_2 = model.decode(encoding).sample + + assert list(sample.shape) == [3, 3, 512, 512] + + assert torch_all_close(sample, sample_2, atol=5e-2) + + @parameterized.expand( + [ + # fmt: off + [33, [-0.3001, 0.0918, -2.6984, -3.9720, -3.2099, -5.0353, 1.7338, -0.2065, 3.4267]], + [47, [-1.5030, -4.3871, -6.0355, -9.1157, -1.6661, -2.7853, 2.1607, -5.0823, 2.5633]], + # fmt: on + ] + ) + def test_stable_diffusion_encode_sample(self, seed, expected_slice): + model = self.get_sd_vae_model() + image = self.get_sd_image(seed) + generator = self.get_generator(seed) + + with torch.no_grad(): + dist = model.encode(image).latent_dist + sample = dist.sample(generator=generator) + + assert list(sample.shape) == [image.shape[0], 4] + [i // 8 for i in image.shape[2:]] + + output_slice = sample[0, -1, -3:, -3:].flatten().cpu() + expected_output_slice = torch.tensor(expected_slice) + + tolerance = 3e-3 if torch_device != "mps" else 1e-2 + assert torch_all_close(output_slice, expected_output_slice, atol=tolerance) + + +@slow +class ConsistencyDecoderVAEIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + @torch.no_grad() + def test_encode_decode(self): + vae = ConsistencyDecoderVAE.from_pretrained("openai/consistency-decoder") # TODO - update + vae.to(torch_device) + + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/img2img/sketch-mountains-input.jpg" + ).resize((256, 256)) + image = torch.from_numpy(np.array(image).transpose(2, 0, 1).astype(np.float32) / 127.5 - 1)[ + None, :, :, : + ].cuda() + + latent = vae.encode(image).latent_dist.mean + + sample = vae.decode(latent, generator=torch.Generator("cpu").manual_seed(0)).sample + + actual_output = sample[0, :2, :2, :2].flatten().cpu() + expected_output = torch.tensor([-0.0141, -0.0014, 0.0115, 0.0086, 0.1051, 0.1053, 0.1031, 0.1024]) + + assert torch_all_close(actual_output, expected_output, atol=5e-3) + + def test_sd(self): + vae = ConsistencyDecoderVAE.from_pretrained("openai/consistency-decoder") # TODO - update + pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", vae=vae, safety_checker=None) + pipe.to(torch_device) + + out = pipe( + "horse", + num_inference_steps=2, + output_type="pt", + generator=torch.Generator("cpu").manual_seed(0), + ).images[0] + + actual_output = out[:2, :2, :2].flatten().cpu() + expected_output = torch.tensor([0.7686, 0.8228, 0.6489, 0.7455, 0.8661, 0.8797, 0.8241, 0.8759]) + + assert torch_all_close(actual_output, expected_output, atol=5e-3) + + def test_encode_decode_f16(self): + vae = ConsistencyDecoderVAE.from_pretrained( + "openai/consistency-decoder", torch_dtype=torch.float16 + ) # TODO - update + vae.to(torch_device) + + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/img2img/sketch-mountains-input.jpg" + ).resize((256, 256)) + image = ( + torch.from_numpy(np.array(image).transpose(2, 0, 1).astype(np.float32) / 127.5 - 1)[None, :, :, :] + .half() + .cuda() + ) + + latent = vae.encode(image).latent_dist.mean + + sample = vae.decode(latent, generator=torch.Generator("cpu").manual_seed(0)).sample + + actual_output = sample[0, :2, :2, :2].flatten().cpu() + expected_output = torch.tensor( + [-0.0111, -0.0125, -0.0017, -0.0007, 0.1257, 0.1465, 0.1450, 0.1471], + dtype=torch.float16, + ) + + assert torch_all_close(actual_output, expected_output, atol=5e-3) + + def test_sd_f16(self): + vae = ConsistencyDecoderVAE.from_pretrained( + "openai/consistency-decoder", torch_dtype=torch.float16 + ) # TODO - update + pipe = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + torch_dtype=torch.float16, + vae=vae, + safety_checker=None, + ) + pipe.to(torch_device) + + out = pipe( + "horse", + num_inference_steps=2, + output_type="pt", + generator=torch.Generator("cpu").manual_seed(0), + ).images[0] + + actual_output = out[:2, :2, :2].flatten().cpu() + expected_output = torch.tensor( + [0.0000, 0.0249, 0.0000, 0.0000, 0.1709, 0.2773, 0.0471, 0.1035], + dtype=torch.float16, + ) + + assert torch_all_close(actual_output, expected_output, atol=5e-3) diff --git a/diffusers-0.27.0/tests/models/autoencoders/test_models_vae_flax.py b/diffusers-0.27.0/tests/models/autoencoders/test_models_vae_flax.py new file mode 100755 index 0000000..8fedb85 --- /dev/null +++ b/diffusers-0.27.0/tests/models/autoencoders/test_models_vae_flax.py @@ -0,0 +1,39 @@ +import unittest + +from diffusers import FlaxAutoencoderKL +from diffusers.utils import is_flax_available +from diffusers.utils.testing_utils import require_flax + +from ..test_modeling_common_flax import FlaxModelTesterMixin + + +if is_flax_available(): + import jax + + +@require_flax +class FlaxAutoencoderKLTests(FlaxModelTesterMixin, unittest.TestCase): + model_class = FlaxAutoencoderKL + + @property + def dummy_input(self): + batch_size = 4 + num_channels = 3 + sizes = (32, 32) + + prng_key = jax.random.PRNGKey(0) + image = jax.random.uniform(prng_key, ((batch_size, num_channels) + sizes)) + + return {"sample": image, "prng_key": prng_key} + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "block_out_channels": [32, 64], + "in_channels": 3, + "out_channels": 3, + "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"], + "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D"], + "latent_channels": 4, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict diff --git a/diffusers-0.27.0/tests/models/autoencoders/test_models_vq.py b/diffusers-0.27.0/tests/models/autoencoders/test_models_vq.py new file mode 100755 index 0000000..8b138bf --- /dev/null +++ b/diffusers-0.27.0/tests/models/autoencoders/test_models_vq.py @@ -0,0 +1,99 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import torch + +from diffusers import VQModel +from diffusers.utils.testing_utils import ( + backend_manual_seed, + enable_full_determinism, + floats_tensor, + torch_device, +) + +from ..test_modeling_common import ModelTesterMixin, UNetTesterMixin + + +enable_full_determinism() + + +class VQModelTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = VQModel + main_input_name = "sample" + + @property + def dummy_input(self, sizes=(32, 32)): + batch_size = 4 + num_channels = 3 + + image = floats_tensor((batch_size, num_channels) + sizes).to(torch_device) + + return {"sample": image} + + @property + def input_shape(self): + return (3, 32, 32) + + @property + def output_shape(self): + return (3, 32, 32) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "block_out_channels": [32, 64], + "in_channels": 3, + "out_channels": 3, + "down_block_types": ["DownEncoderBlock2D", "DownEncoderBlock2D"], + "up_block_types": ["UpDecoderBlock2D", "UpDecoderBlock2D"], + "latent_channels": 3, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_forward_signature(self): + pass + + def test_training(self): + pass + + def test_from_pretrained_hub(self): + model, loading_info = VQModel.from_pretrained("fusing/vqgan-dummy", output_loading_info=True) + self.assertIsNotNone(model) + self.assertEqual(len(loading_info["missing_keys"]), 0) + + model.to(torch_device) + image = model(**self.dummy_input) + + assert image is not None, "Make sure output is not None" + + def test_output_pretrained(self): + model = VQModel.from_pretrained("fusing/vqgan-dummy") + model.to(torch_device).eval() + + torch.manual_seed(0) + backend_manual_seed(torch_device, 0) + + image = torch.randn(1, model.config.in_channels, model.config.sample_size, model.config.sample_size) + image = image.to(torch_device) + with torch.no_grad(): + output = model(image).sample + + output_slice = output[0, -1, -3:, -3:].flatten().cpu() + # fmt: off + expected_output_slice = torch.tensor([-0.0153, -0.4044, -0.1880, -0.5161, -0.2418, -0.4072, -0.1612, -0.0633, -0.0143]) + # fmt: on + self.assertTrue(torch.allclose(output_slice, expected_output_slice, atol=1e-3)) diff --git a/diffusers-0.27.0/tests/models/test_activations.py b/diffusers-0.27.0/tests/models/test_activations.py new file mode 100755 index 0000000..4e8e514 --- /dev/null +++ b/diffusers-0.27.0/tests/models/test_activations.py @@ -0,0 +1,48 @@ +import unittest + +import torch +from torch import nn + +from diffusers.models.activations import get_activation + + +class ActivationsTests(unittest.TestCase): + def test_swish(self): + act = get_activation("swish") + + self.assertIsInstance(act, nn.SiLU) + + self.assertEqual(act(torch.tensor(-100, dtype=torch.float32)).item(), 0) + self.assertNotEqual(act(torch.tensor(-1, dtype=torch.float32)).item(), 0) + self.assertEqual(act(torch.tensor(0, dtype=torch.float32)).item(), 0) + self.assertEqual(act(torch.tensor(20, dtype=torch.float32)).item(), 20) + + def test_silu(self): + act = get_activation("silu") + + self.assertIsInstance(act, nn.SiLU) + + self.assertEqual(act(torch.tensor(-100, dtype=torch.float32)).item(), 0) + self.assertNotEqual(act(torch.tensor(-1, dtype=torch.float32)).item(), 0) + self.assertEqual(act(torch.tensor(0, dtype=torch.float32)).item(), 0) + self.assertEqual(act(torch.tensor(20, dtype=torch.float32)).item(), 20) + + def test_mish(self): + act = get_activation("mish") + + self.assertIsInstance(act, nn.Mish) + + self.assertEqual(act(torch.tensor(-200, dtype=torch.float32)).item(), 0) + self.assertNotEqual(act(torch.tensor(-1, dtype=torch.float32)).item(), 0) + self.assertEqual(act(torch.tensor(0, dtype=torch.float32)).item(), 0) + self.assertEqual(act(torch.tensor(20, dtype=torch.float32)).item(), 20) + + def test_gelu(self): + act = get_activation("gelu") + + self.assertIsInstance(act, nn.GELU) + + self.assertEqual(act(torch.tensor(-100, dtype=torch.float32)).item(), 0) + self.assertNotEqual(act(torch.tensor(-1, dtype=torch.float32)).item(), 0) + self.assertEqual(act(torch.tensor(0, dtype=torch.float32)).item(), 0) + self.assertEqual(act(torch.tensor(20, dtype=torch.float32)).item(), 20) diff --git a/diffusers-0.27.0/tests/models/test_attention_processor.py b/diffusers-0.27.0/tests/models/test_attention_processor.py new file mode 100755 index 0000000..fadee4a --- /dev/null +++ b/diffusers-0.27.0/tests/models/test_attention_processor.py @@ -0,0 +1,119 @@ +import tempfile +import unittest + +import numpy as np +import torch + +from diffusers import DiffusionPipeline +from diffusers.models.attention_processor import Attention, AttnAddedKVProcessor + + +class AttnAddedKVProcessorTests(unittest.TestCase): + def get_constructor_arguments(self, only_cross_attention: bool = False): + query_dim = 10 + + if only_cross_attention: + cross_attention_dim = 12 + else: + # when only cross attention is not set, the cross attention dim must be the same as the query dim + cross_attention_dim = query_dim + + return { + "query_dim": query_dim, + "cross_attention_dim": cross_attention_dim, + "heads": 2, + "dim_head": 4, + "added_kv_proj_dim": 6, + "norm_num_groups": 1, + "only_cross_attention": only_cross_attention, + "processor": AttnAddedKVProcessor(), + } + + def get_forward_arguments(self, query_dim, added_kv_proj_dim): + batch_size = 2 + + hidden_states = torch.rand(batch_size, query_dim, 3, 2) + encoder_hidden_states = torch.rand(batch_size, 4, added_kv_proj_dim) + attention_mask = None + + return { + "hidden_states": hidden_states, + "encoder_hidden_states": encoder_hidden_states, + "attention_mask": attention_mask, + } + + def test_only_cross_attention(self): + # self and cross attention + + torch.manual_seed(0) + + constructor_args = self.get_constructor_arguments(only_cross_attention=False) + attn = Attention(**constructor_args) + + self.assertTrue(attn.to_k is not None) + self.assertTrue(attn.to_v is not None) + + forward_args = self.get_forward_arguments( + query_dim=constructor_args["query_dim"], added_kv_proj_dim=constructor_args["added_kv_proj_dim"] + ) + + self_and_cross_attn_out = attn(**forward_args) + + # only self attention + + torch.manual_seed(0) + + constructor_args = self.get_constructor_arguments(only_cross_attention=True) + attn = Attention(**constructor_args) + + self.assertTrue(attn.to_k is None) + self.assertTrue(attn.to_v is None) + + forward_args = self.get_forward_arguments( + query_dim=constructor_args["query_dim"], added_kv_proj_dim=constructor_args["added_kv_proj_dim"] + ) + + only_cross_attn_out = attn(**forward_args) + + self.assertTrue((only_cross_attn_out != self_and_cross_attn_out).all()) + + +class DeprecatedAttentionBlockTests(unittest.TestCase): + def test_conversion_when_using_device_map(self): + pipe = DiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-pipe", safety_checker=None) + + pre_conversion = pipe( + "foo", + num_inference_steps=2, + generator=torch.Generator("cpu").manual_seed(0), + output_type="np", + ).images + + # the initial conversion succeeds + pipe = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-pipe", device_map="sequential", safety_checker=None + ) + + conversion = pipe( + "foo", + num_inference_steps=2, + generator=torch.Generator("cpu").manual_seed(0), + output_type="np", + ).images + + with tempfile.TemporaryDirectory() as tmpdir: + # save the converted model + pipe.save_pretrained(tmpdir) + + # can also load the converted weights + pipe = DiffusionPipeline.from_pretrained(tmpdir, device_map="sequential", safety_checker=None) + + after_conversion = pipe( + "foo", + num_inference_steps=2, + generator=torch.Generator("cpu").manual_seed(0), + output_type="np", + ).images + + self.assertTrue(np.allclose(pre_conversion, conversion, atol=1e-5)) + self.assertTrue(np.allclose(conversion, after_conversion, atol=1e-5)) diff --git a/diffusers-0.27.0/tests/models/test_layers_utils.py b/diffusers-0.27.0/tests/models/test_layers_utils.py new file mode 100755 index 0000000..b5a5bec --- /dev/null +++ b/diffusers-0.27.0/tests/models/test_layers_utils.py @@ -0,0 +1,528 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest + +import numpy as np +import torch +from torch import nn + +from diffusers.models.attention import GEGLU, AdaLayerNorm, ApproximateGELU +from diffusers.models.embeddings import get_timestep_embedding +from diffusers.models.resnet import Downsample2D, ResnetBlock2D, Upsample2D +from diffusers.models.transformers.transformer_2d import Transformer2DModel +from diffusers.utils.testing_utils import ( + backend_manual_seed, + require_torch_accelerator_with_fp64, + torch_device, +) + + +class EmbeddingsTests(unittest.TestCase): + def test_timestep_embeddings(self): + embedding_dim = 256 + timesteps = torch.arange(16) + + t1 = get_timestep_embedding(timesteps, embedding_dim) + + # first vector should always be composed only of 0's and 1's + assert (t1[0, : embedding_dim // 2] - 0).abs().sum() < 1e-5 + assert (t1[0, embedding_dim // 2 :] - 1).abs().sum() < 1e-5 + + # last element of each vector should be one + assert (t1[:, -1] - 1).abs().sum() < 1e-5 + + # For large embeddings (e.g. 128) the frequency of every vector is higher + # than the previous one which means that the gradients of later vectors are + # ALWAYS higher than the previous ones + grad_mean = np.abs(np.gradient(t1, axis=-1)).mean(axis=1) + + prev_grad = 0.0 + for grad in grad_mean: + assert grad > prev_grad + prev_grad = grad + + def test_timestep_defaults(self): + embedding_dim = 16 + timesteps = torch.arange(10) + + t1 = get_timestep_embedding(timesteps, embedding_dim) + t2 = get_timestep_embedding( + timesteps, embedding_dim, flip_sin_to_cos=False, downscale_freq_shift=1, max_period=10_000 + ) + + assert torch.allclose(t1.cpu(), t2.cpu(), 1e-3) + + def test_timestep_flip_sin_cos(self): + embedding_dim = 16 + timesteps = torch.arange(10) + + t1 = get_timestep_embedding(timesteps, embedding_dim, flip_sin_to_cos=True) + t1 = torch.cat([t1[:, embedding_dim // 2 :], t1[:, : embedding_dim // 2]], dim=-1) + + t2 = get_timestep_embedding(timesteps, embedding_dim, flip_sin_to_cos=False) + + assert torch.allclose(t1.cpu(), t2.cpu(), 1e-3) + + def test_timestep_downscale_freq_shift(self): + embedding_dim = 16 + timesteps = torch.arange(10) + + t1 = get_timestep_embedding(timesteps, embedding_dim, downscale_freq_shift=0) + t2 = get_timestep_embedding(timesteps, embedding_dim, downscale_freq_shift=1) + + # get cosine half (vectors that are wrapped into cosine) + cosine_half = (t1 - t2)[:, embedding_dim // 2 :] + + # cosine needs to be negative + assert (np.abs((cosine_half <= 0).numpy()) - 1).sum() < 1e-5 + + def test_sinoid_embeddings_hardcoded(self): + embedding_dim = 64 + timesteps = torch.arange(128) + + # standard unet, score_vde + t1 = get_timestep_embedding(timesteps, embedding_dim, downscale_freq_shift=1, flip_sin_to_cos=False) + # glide, ldm + t2 = get_timestep_embedding(timesteps, embedding_dim, downscale_freq_shift=0, flip_sin_to_cos=True) + # grad-tts + t3 = get_timestep_embedding(timesteps, embedding_dim, scale=1000) + + assert torch.allclose( + t1[23:26, 47:50].flatten().cpu(), + torch.tensor([0.9646, 0.9804, 0.9892, 0.9615, 0.9787, 0.9882, 0.9582, 0.9769, 0.9872]), + 1e-3, + ) + assert torch.allclose( + t2[23:26, 47:50].flatten().cpu(), + torch.tensor([0.3019, 0.2280, 0.1716, 0.3146, 0.2377, 0.1790, 0.3272, 0.2474, 0.1864]), + 1e-3, + ) + assert torch.allclose( + t3[23:26, 47:50].flatten().cpu(), + torch.tensor([-0.9801, -0.9464, -0.9349, -0.3952, 0.8887, -0.9709, 0.5299, -0.2853, -0.9927]), + 1e-3, + ) + + +class Upsample2DBlockTests(unittest.TestCase): + def test_upsample_default(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 32, 32) + upsample = Upsample2D(channels=32, use_conv=False) + with torch.no_grad(): + upsampled = upsample(sample) + + assert upsampled.shape == (1, 32, 64, 64) + output_slice = upsampled[0, -1, -3:, -3:] + expected_slice = torch.tensor([-0.2173, -1.2079, -1.2079, 0.2952, 1.1254, 1.1254, 0.2952, 1.1254, 1.1254]) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_upsample_with_conv(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 32, 32) + upsample = Upsample2D(channels=32, use_conv=True) + with torch.no_grad(): + upsampled = upsample(sample) + + assert upsampled.shape == (1, 32, 64, 64) + output_slice = upsampled[0, -1, -3:, -3:] + expected_slice = torch.tensor([0.7145, 1.3773, 0.3492, 0.8448, 1.0839, -0.3341, 0.5956, 0.1250, -0.4841]) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_upsample_with_conv_out_dim(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 32, 32) + upsample = Upsample2D(channels=32, use_conv=True, out_channels=64) + with torch.no_grad(): + upsampled = upsample(sample) + + assert upsampled.shape == (1, 64, 64, 64) + output_slice = upsampled[0, -1, -3:, -3:] + expected_slice = torch.tensor([0.2703, 0.1656, -0.2538, -0.0553, -0.2984, 0.1044, 0.1155, 0.2579, 0.7755]) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_upsample_with_transpose(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 32, 32) + upsample = Upsample2D(channels=32, use_conv=False, use_conv_transpose=True) + with torch.no_grad(): + upsampled = upsample(sample) + + assert upsampled.shape == (1, 32, 64, 64) + output_slice = upsampled[0, -1, -3:, -3:] + expected_slice = torch.tensor([-0.3028, -0.1582, 0.0071, 0.0350, -0.4799, -0.1139, 0.1056, -0.1153, -0.1046]) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + +class Downsample2DBlockTests(unittest.TestCase): + def test_downsample_default(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 64, 64) + downsample = Downsample2D(channels=32, use_conv=False) + with torch.no_grad(): + downsampled = downsample(sample) + + assert downsampled.shape == (1, 32, 32, 32) + output_slice = downsampled[0, -1, -3:, -3:] + expected_slice = torch.tensor([-0.0513, -0.3889, 0.0640, 0.0836, -0.5460, -0.0341, -0.0169, -0.6967, 0.1179]) + max_diff = (output_slice.flatten() - expected_slice).abs().sum().item() + assert max_diff <= 1e-3 + # assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-1) + + def test_downsample_with_conv(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 64, 64) + downsample = Downsample2D(channels=32, use_conv=True) + with torch.no_grad(): + downsampled = downsample(sample) + + assert downsampled.shape == (1, 32, 32, 32) + output_slice = downsampled[0, -1, -3:, -3:] + + expected_slice = torch.tensor( + [0.9267, 0.5878, 0.3337, 1.2321, -0.1191, -0.3984, -0.7532, -0.0715, -0.3913], + ) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_downsample_with_conv_pad1(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 64, 64) + downsample = Downsample2D(channels=32, use_conv=True, padding=1) + with torch.no_grad(): + downsampled = downsample(sample) + + assert downsampled.shape == (1, 32, 32, 32) + output_slice = downsampled[0, -1, -3:, -3:] + expected_slice = torch.tensor([0.9267, 0.5878, 0.3337, 1.2321, -0.1191, -0.3984, -0.7532, -0.0715, -0.3913]) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_downsample_with_conv_out_dim(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 64, 64) + downsample = Downsample2D(channels=32, use_conv=True, out_channels=16) + with torch.no_grad(): + downsampled = downsample(sample) + + assert downsampled.shape == (1, 16, 32, 32) + output_slice = downsampled[0, -1, -3:, -3:] + expected_slice = torch.tensor([-0.6586, 0.5985, 0.0721, 0.1256, -0.1492, 0.4436, -0.2544, 0.5021, 1.1522]) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + +class ResnetBlock2DTests(unittest.TestCase): + def test_resnet_default(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 64, 64).to(torch_device) + temb = torch.randn(1, 128).to(torch_device) + resnet_block = ResnetBlock2D(in_channels=32, temb_channels=128).to(torch_device) + with torch.no_grad(): + output_tensor = resnet_block(sample, temb) + + assert output_tensor.shape == (1, 32, 64, 64) + output_slice = output_tensor[0, -1, -3:, -3:] + expected_slice = torch.tensor( + [-1.9010, -0.2974, -0.8245, -1.3533, 0.8742, -0.9645, -2.0584, 1.3387, -0.4746], device=torch_device + ) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_restnet_with_use_in_shortcut(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 64, 64).to(torch_device) + temb = torch.randn(1, 128).to(torch_device) + resnet_block = ResnetBlock2D(in_channels=32, temb_channels=128, use_in_shortcut=True).to(torch_device) + with torch.no_grad(): + output_tensor = resnet_block(sample, temb) + + assert output_tensor.shape == (1, 32, 64, 64) + output_slice = output_tensor[0, -1, -3:, -3:] + expected_slice = torch.tensor( + [0.2226, -1.0791, -0.1629, 0.3659, -0.2889, -1.2376, 0.0582, 0.9206, 0.0044], device=torch_device + ) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_resnet_up(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 64, 64).to(torch_device) + temb = torch.randn(1, 128).to(torch_device) + resnet_block = ResnetBlock2D(in_channels=32, temb_channels=128, up=True).to(torch_device) + with torch.no_grad(): + output_tensor = resnet_block(sample, temb) + + assert output_tensor.shape == (1, 32, 128, 128) + output_slice = output_tensor[0, -1, -3:, -3:] + expected_slice = torch.tensor( + [1.2130, -0.8753, -0.9027, 1.5783, -0.5362, -0.5001, 1.0726, -0.7732, -0.4182], device=torch_device + ) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_resnet_down(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 64, 64).to(torch_device) + temb = torch.randn(1, 128).to(torch_device) + resnet_block = ResnetBlock2D(in_channels=32, temb_channels=128, down=True).to(torch_device) + with torch.no_grad(): + output_tensor = resnet_block(sample, temb) + + assert output_tensor.shape == (1, 32, 32, 32) + output_slice = output_tensor[0, -1, -3:, -3:] + expected_slice = torch.tensor( + [-0.3002, -0.7135, 0.1359, 0.0561, -0.7935, 0.0113, -0.1766, -0.6714, -0.0436], device=torch_device + ) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_restnet_with_kernel_fir(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 64, 64).to(torch_device) + temb = torch.randn(1, 128).to(torch_device) + resnet_block = ResnetBlock2D(in_channels=32, temb_channels=128, kernel="fir", down=True).to(torch_device) + with torch.no_grad(): + output_tensor = resnet_block(sample, temb) + + assert output_tensor.shape == (1, 32, 32, 32) + output_slice = output_tensor[0, -1, -3:, -3:] + expected_slice = torch.tensor( + [-0.0934, -0.5729, 0.0909, -0.2710, -0.5044, 0.0243, -0.0665, -0.5267, -0.3136], device=torch_device + ) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_restnet_with_kernel_sde_vp(self): + torch.manual_seed(0) + sample = torch.randn(1, 32, 64, 64).to(torch_device) + temb = torch.randn(1, 128).to(torch_device) + resnet_block = ResnetBlock2D(in_channels=32, temb_channels=128, kernel="sde_vp", down=True).to(torch_device) + with torch.no_grad(): + output_tensor = resnet_block(sample, temb) + + assert output_tensor.shape == (1, 32, 32, 32) + output_slice = output_tensor[0, -1, -3:, -3:] + expected_slice = torch.tensor( + [-0.3002, -0.7135, 0.1359, 0.0561, -0.7935, 0.0113, -0.1766, -0.6714, -0.0436], device=torch_device + ) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + +class Transformer2DModelTests(unittest.TestCase): + def test_spatial_transformer_default(self): + torch.manual_seed(0) + backend_manual_seed(torch_device, 0) + + sample = torch.randn(1, 32, 64, 64).to(torch_device) + spatial_transformer_block = Transformer2DModel( + in_channels=32, + num_attention_heads=1, + attention_head_dim=32, + dropout=0.0, + cross_attention_dim=None, + ).to(torch_device) + with torch.no_grad(): + attention_scores = spatial_transformer_block(sample).sample + + assert attention_scores.shape == (1, 32, 64, 64) + output_slice = attention_scores[0, -1, -3:, -3:] + + expected_slice = torch.tensor( + [-1.9455, -0.0066, -1.3933, -1.5878, 0.5325, -0.6486, -1.8648, 0.7515, -0.9689], device=torch_device + ) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_spatial_transformer_cross_attention_dim(self): + torch.manual_seed(0) + backend_manual_seed(torch_device, 0) + + sample = torch.randn(1, 64, 64, 64).to(torch_device) + spatial_transformer_block = Transformer2DModel( + in_channels=64, + num_attention_heads=2, + attention_head_dim=32, + dropout=0.0, + cross_attention_dim=64, + ).to(torch_device) + with torch.no_grad(): + context = torch.randn(1, 4, 64).to(torch_device) + attention_scores = spatial_transformer_block(sample, context).sample + + assert attention_scores.shape == (1, 64, 64, 64) + output_slice = attention_scores[0, -1, -3:, -3:] + expected_slice = torch.tensor( + [0.0143, -0.6909, -2.1547, -1.8893, 1.4097, 0.1359, -0.2521, -1.3359, 0.2598], device=torch_device + ) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_spatial_transformer_timestep(self): + torch.manual_seed(0) + backend_manual_seed(torch_device, 0) + + num_embeds_ada_norm = 5 + + sample = torch.randn(1, 64, 64, 64).to(torch_device) + spatial_transformer_block = Transformer2DModel( + in_channels=64, + num_attention_heads=2, + attention_head_dim=32, + dropout=0.0, + cross_attention_dim=64, + num_embeds_ada_norm=num_embeds_ada_norm, + ).to(torch_device) + with torch.no_grad(): + timestep_1 = torch.tensor(1, dtype=torch.long).to(torch_device) + timestep_2 = torch.tensor(2, dtype=torch.long).to(torch_device) + attention_scores_1 = spatial_transformer_block(sample, timestep=timestep_1).sample + attention_scores_2 = spatial_transformer_block(sample, timestep=timestep_2).sample + + assert attention_scores_1.shape == (1, 64, 64, 64) + assert attention_scores_2.shape == (1, 64, 64, 64) + + output_slice_1 = attention_scores_1[0, -1, -3:, -3:] + output_slice_2 = attention_scores_2[0, -1, -3:, -3:] + + expected_slice = torch.tensor( + [-0.3923, -1.0923, -1.7144, -1.5570, 1.4154, 0.1738, -0.1157, -1.2998, -0.1703], device=torch_device + ) + expected_slice_2 = torch.tensor( + [-0.4311, -1.1376, -1.7732, -1.5997, 1.3450, 0.0964, -0.1569, -1.3590, -0.2348], device=torch_device + ) + + assert torch.allclose(output_slice_1.flatten(), expected_slice, atol=1e-3) + assert torch.allclose(output_slice_2.flatten(), expected_slice_2, atol=1e-3) + + def test_spatial_transformer_dropout(self): + torch.manual_seed(0) + backend_manual_seed(torch_device, 0) + + sample = torch.randn(1, 32, 64, 64).to(torch_device) + spatial_transformer_block = ( + Transformer2DModel( + in_channels=32, + num_attention_heads=2, + attention_head_dim=16, + dropout=0.3, + cross_attention_dim=None, + ) + .to(torch_device) + .eval() + ) + with torch.no_grad(): + attention_scores = spatial_transformer_block(sample).sample + + assert attention_scores.shape == (1, 32, 64, 64) + output_slice = attention_scores[0, -1, -3:, -3:] + + expected_slice = torch.tensor( + [-1.9380, -0.0083, -1.3771, -1.5819, 0.5209, -0.6441, -1.8545, 0.7563, -0.9615], device=torch_device + ) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + @require_torch_accelerator_with_fp64 + def test_spatial_transformer_discrete(self): + torch.manual_seed(0) + backend_manual_seed(torch_device, 0) + + num_embed = 5 + + sample = torch.randint(0, num_embed, (1, 32)).to(torch_device) + spatial_transformer_block = ( + Transformer2DModel( + num_attention_heads=1, + attention_head_dim=32, + num_vector_embeds=num_embed, + sample_size=16, + ) + .to(torch_device) + .eval() + ) + + with torch.no_grad(): + attention_scores = spatial_transformer_block(sample).sample + + assert attention_scores.shape == (1, num_embed - 1, 32) + + output_slice = attention_scores[0, -2:, -3:] + + expected_slice = torch.tensor([-1.7648, -1.0241, -2.0985, -1.8035, -1.6404, -1.2098], device=torch_device) + assert torch.allclose(output_slice.flatten(), expected_slice, atol=1e-3) + + def test_spatial_transformer_default_norm_layers(self): + spatial_transformer_block = Transformer2DModel(num_attention_heads=1, attention_head_dim=32, in_channels=32) + + assert spatial_transformer_block.transformer_blocks[0].norm1.__class__ == nn.LayerNorm + assert spatial_transformer_block.transformer_blocks[0].norm3.__class__ == nn.LayerNorm + + def test_spatial_transformer_ada_norm_layers(self): + spatial_transformer_block = Transformer2DModel( + num_attention_heads=1, + attention_head_dim=32, + in_channels=32, + num_embeds_ada_norm=5, + ) + + assert spatial_transformer_block.transformer_blocks[0].norm1.__class__ == AdaLayerNorm + assert spatial_transformer_block.transformer_blocks[0].norm3.__class__ == nn.LayerNorm + + def test_spatial_transformer_default_ff_layers(self): + spatial_transformer_block = Transformer2DModel( + num_attention_heads=1, + attention_head_dim=32, + in_channels=32, + ) + + assert spatial_transformer_block.transformer_blocks[0].ff.net[0].__class__ == GEGLU + assert spatial_transformer_block.transformer_blocks[0].ff.net[1].__class__ == nn.Dropout + assert spatial_transformer_block.transformer_blocks[0].ff.net[2].__class__ == nn.Linear + + dim = 32 + inner_dim = 128 + + # First dimension change + assert spatial_transformer_block.transformer_blocks[0].ff.net[0].proj.in_features == dim + # NOTE: inner_dim * 2 because GEGLU + assert spatial_transformer_block.transformer_blocks[0].ff.net[0].proj.out_features == inner_dim * 2 + + # Second dimension change + assert spatial_transformer_block.transformer_blocks[0].ff.net[2].in_features == inner_dim + assert spatial_transformer_block.transformer_blocks[0].ff.net[2].out_features == dim + + def test_spatial_transformer_geglu_approx_ff_layers(self): + spatial_transformer_block = Transformer2DModel( + num_attention_heads=1, + attention_head_dim=32, + in_channels=32, + activation_fn="geglu-approximate", + ) + + assert spatial_transformer_block.transformer_blocks[0].ff.net[0].__class__ == ApproximateGELU + assert spatial_transformer_block.transformer_blocks[0].ff.net[1].__class__ == nn.Dropout + assert spatial_transformer_block.transformer_blocks[0].ff.net[2].__class__ == nn.Linear + + dim = 32 + inner_dim = 128 + + # First dimension change + assert spatial_transformer_block.transformer_blocks[0].ff.net[0].proj.in_features == dim + assert spatial_transformer_block.transformer_blocks[0].ff.net[0].proj.out_features == inner_dim + + # Second dimension change + assert spatial_transformer_block.transformer_blocks[0].ff.net[2].in_features == inner_dim + assert spatial_transformer_block.transformer_blocks[0].ff.net[2].out_features == dim + + def test_spatial_transformer_attention_bias(self): + spatial_transformer_block = Transformer2DModel( + num_attention_heads=1, attention_head_dim=32, in_channels=32, attention_bias=True + ) + + assert spatial_transformer_block.transformer_blocks[0].attn1.to_q.bias is not None + assert spatial_transformer_block.transformer_blocks[0].attn1.to_k.bias is not None + assert spatial_transformer_block.transformer_blocks[0].attn1.to_v.bias is not None diff --git a/diffusers-0.27.0/tests/models/test_modeling_common.py b/diffusers-0.27.0/tests/models/test_modeling_common.py new file mode 100755 index 0000000..2c68537 --- /dev/null +++ b/diffusers-0.27.0/tests/models/test_modeling_common.py @@ -0,0 +1,758 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import tempfile +import traceback +import unittest +import unittest.mock as mock +import uuid +from typing import Dict, List, Tuple + +import numpy as np +import requests_mock +import torch +from huggingface_hub import ModelCard, delete_repo +from huggingface_hub.utils import is_jinja_available +from requests.exceptions import HTTPError + +from diffusers.models import UNet2DConditionModel +from diffusers.models.attention_processor import AttnProcessor, AttnProcessor2_0, XFormersAttnProcessor +from diffusers.training_utils import EMAModel +from diffusers.utils import is_xformers_available, logging +from diffusers.utils.testing_utils import ( + CaptureLogger, + require_python39_or_higher, + require_torch_2, + require_torch_accelerator_with_training, + require_torch_gpu, + run_test_in_subprocess, + torch_device, +) + +from ..others.test_utils import TOKEN, USER, is_staging_test + + +# Will be run via run_test_in_subprocess +def _test_from_save_pretrained_dynamo(in_queue, out_queue, timeout): + error = None + try: + init_dict, model_class = in_queue.get(timeout=timeout) + + model = model_class(**init_dict) + model.to(torch_device) + model = torch.compile(model) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname, safe_serialization=False) + new_model = model_class.from_pretrained(tmpdirname) + new_model.to(torch_device) + + assert new_model.__class__ == model_class + except Exception: + error = f"{traceback.format_exc()}" + + results = {"error": error} + out_queue.put(results, timeout=timeout) + out_queue.join() + + +class ModelUtilsTest(unittest.TestCase): + def tearDown(self): + super().tearDown() + + def test_accelerate_loading_error_message(self): + with self.assertRaises(ValueError) as error_context: + UNet2DConditionModel.from_pretrained("hf-internal-testing/stable-diffusion-broken", subfolder="unet") + + # make sure that error message states what keys are missing + assert "conv_out.bias" in str(error_context.exception) + + def test_cached_files_are_used_when_no_internet(self): + # A mock response for an HTTP head request to emulate server down + response_mock = mock.Mock() + response_mock.status_code = 500 + response_mock.headers = {} + response_mock.raise_for_status.side_effect = HTTPError + response_mock.json.return_value = {} + + # Download this model to make sure it's in the cache. + orig_model = UNet2DConditionModel.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", subfolder="unet" + ) + + # Under the mock environment we get a 500 error when trying to reach the model. + with mock.patch("requests.request", return_value=response_mock): + # Download this model to make sure it's in the cache. + model = UNet2DConditionModel.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", subfolder="unet", local_files_only=True + ) + + for p1, p2 in zip(orig_model.parameters(), model.parameters()): + if p1.data.ne(p2.data).sum() > 0: + assert False, "Parameters not the same!" + + def test_one_request_upon_cached(self): + # TODO: For some reason this test fails on MPS where no HEAD call is made. + if torch_device == "mps": + return + + use_safetensors = False + + with tempfile.TemporaryDirectory() as tmpdirname: + with requests_mock.mock(real_http=True) as m: + UNet2DConditionModel.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", + subfolder="unet", + cache_dir=tmpdirname, + use_safetensors=use_safetensors, + ) + + download_requests = [r.method for r in m.request_history] + assert download_requests.count("HEAD") == 2, "2 HEAD requests one for config, one for model" + assert download_requests.count("GET") == 2, "2 GET requests one for config, one for model" + + with requests_mock.mock(real_http=True) as m: + UNet2DConditionModel.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", + subfolder="unet", + cache_dir=tmpdirname, + use_safetensors=use_safetensors, + ) + + cache_requests = [r.method for r in m.request_history] + assert ( + "HEAD" == cache_requests[0] and len(cache_requests) == 1 + ), "We should call only `model_info` to check for _commit hash and `send_telemetry`" + + def test_weight_overwrite(self): + with tempfile.TemporaryDirectory() as tmpdirname, self.assertRaises(ValueError) as error_context: + UNet2DConditionModel.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", + subfolder="unet", + cache_dir=tmpdirname, + in_channels=9, + ) + + # make sure that error message states what keys are missing + assert "Cannot load" in str(error_context.exception) + + with tempfile.TemporaryDirectory() as tmpdirname: + model = UNet2DConditionModel.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", + subfolder="unet", + cache_dir=tmpdirname, + in_channels=9, + low_cpu_mem_usage=False, + ignore_mismatched_sizes=True, + ) + + assert model.config.in_channels == 9 + + +class UNetTesterMixin: + def test_forward_signature(self): + init_dict, _ = self.prepare_init_args_and_inputs_for_common() + + model = self.model_class(**init_dict) + signature = inspect.signature(model.forward) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + expected_arg_names = ["sample", "timestep"] + self.assertListEqual(arg_names[:2], expected_arg_names) + + def test_forward_with_norm_groups(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["norm_num_groups"] = 16 + init_dict["block_out_channels"] = (16, 32) + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.to_tuple()[0] + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + +class ModelTesterMixin: + main_input_name = None # overwrite in model specific tester class + base_precision = 1e-3 + forward_requires_fresh_args = False + + def test_from_save_pretrained(self, expected_max_diff=5e-5): + if self.forward_requires_fresh_args: + model = self.model_class(**self.init_dict) + else: + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + + if hasattr(model, "set_default_attn_processor"): + model.set_default_attn_processor() + model.to(torch_device) + model.eval() + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname, safe_serialization=False) + new_model = self.model_class.from_pretrained(tmpdirname) + if hasattr(new_model, "set_default_attn_processor"): + new_model.set_default_attn_processor() + new_model.to(torch_device) + + with torch.no_grad(): + if self.forward_requires_fresh_args: + image = model(**self.inputs_dict(0)) + else: + image = model(**inputs_dict) + + if isinstance(image, dict): + image = image.to_tuple()[0] + + if self.forward_requires_fresh_args: + new_image = new_model(**self.inputs_dict(0)) + else: + new_image = new_model(**inputs_dict) + + if isinstance(new_image, dict): + new_image = new_image.to_tuple()[0] + + max_diff = (image - new_image).abs().max().item() + self.assertLessEqual(max_diff, expected_max_diff, "Models give different forward passes") + + def test_getattr_is_correct(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + + # save some things to test + model.dummy_attribute = 5 + model.register_to_config(test_attribute=5) + + logger = logging.get_logger("diffusers.models.modeling_utils") + # 30 for warning + logger.setLevel(30) + with CaptureLogger(logger) as cap_logger: + assert hasattr(model, "dummy_attribute") + assert getattr(model, "dummy_attribute") == 5 + assert model.dummy_attribute == 5 + + # no warning should be thrown + assert cap_logger.out == "" + + logger = logging.get_logger("diffusers.models.modeling_utils") + # 30 for warning + logger.setLevel(30) + with CaptureLogger(logger) as cap_logger: + assert hasattr(model, "save_pretrained") + fn = model.save_pretrained + fn_1 = getattr(model, "save_pretrained") + + assert fn == fn_1 + # no warning should be thrown + assert cap_logger.out == "" + + # warning should be thrown + with self.assertWarns(FutureWarning): + assert model.test_attribute == 5 + + with self.assertWarns(FutureWarning): + assert getattr(model, "test_attribute") == 5 + + with self.assertRaises(AttributeError) as error: + model.does_not_exist + + assert str(error.exception) == f"'{type(model).__name__}' object has no attribute 'does_not_exist'" + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_set_xformers_attn_processor_for_determinism(self): + torch.use_deterministic_algorithms(False) + if self.forward_requires_fresh_args: + model = self.model_class(**self.init_dict) + else: + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.to(torch_device) + + if not hasattr(model, "set_attn_processor"): + # If not has `set_attn_processor`, skip test + return + + model.set_default_attn_processor() + assert all(type(proc) == AttnProcessor for proc in model.attn_processors.values()) + with torch.no_grad(): + if self.forward_requires_fresh_args: + output = model(**self.inputs_dict(0))[0] + else: + output = model(**inputs_dict)[0] + + model.enable_xformers_memory_efficient_attention() + assert all(type(proc) == XFormersAttnProcessor for proc in model.attn_processors.values()) + with torch.no_grad(): + if self.forward_requires_fresh_args: + output_2 = model(**self.inputs_dict(0))[0] + else: + output_2 = model(**inputs_dict)[0] + + model.set_attn_processor(XFormersAttnProcessor()) + assert all(type(proc) == XFormersAttnProcessor for proc in model.attn_processors.values()) + with torch.no_grad(): + if self.forward_requires_fresh_args: + output_3 = model(**self.inputs_dict(0))[0] + else: + output_3 = model(**inputs_dict)[0] + + torch.use_deterministic_algorithms(True) + + assert torch.allclose(output, output_2, atol=self.base_precision) + assert torch.allclose(output, output_3, atol=self.base_precision) + assert torch.allclose(output_2, output_3, atol=self.base_precision) + + @require_torch_gpu + def test_set_attn_processor_for_determinism(self): + torch.use_deterministic_algorithms(False) + if self.forward_requires_fresh_args: + model = self.model_class(**self.init_dict) + else: + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + + model.to(torch_device) + + if not hasattr(model, "set_attn_processor"): + # If not has `set_attn_processor`, skip test + return + + assert all(type(proc) == AttnProcessor2_0 for proc in model.attn_processors.values()) + with torch.no_grad(): + if self.forward_requires_fresh_args: + output_1 = model(**self.inputs_dict(0))[0] + else: + output_1 = model(**inputs_dict)[0] + + model.set_default_attn_processor() + assert all(type(proc) == AttnProcessor for proc in model.attn_processors.values()) + with torch.no_grad(): + if self.forward_requires_fresh_args: + output_2 = model(**self.inputs_dict(0))[0] + else: + output_2 = model(**inputs_dict)[0] + + model.set_attn_processor(AttnProcessor2_0()) + assert all(type(proc) == AttnProcessor2_0 for proc in model.attn_processors.values()) + with torch.no_grad(): + if self.forward_requires_fresh_args: + output_4 = model(**self.inputs_dict(0))[0] + else: + output_4 = model(**inputs_dict)[0] + + model.set_attn_processor(AttnProcessor()) + assert all(type(proc) == AttnProcessor for proc in model.attn_processors.values()) + with torch.no_grad(): + if self.forward_requires_fresh_args: + output_5 = model(**self.inputs_dict(0))[0] + else: + output_5 = model(**inputs_dict)[0] + + torch.use_deterministic_algorithms(True) + + # make sure that outputs match + assert torch.allclose(output_2, output_1, atol=self.base_precision) + assert torch.allclose(output_2, output_4, atol=self.base_precision) + assert torch.allclose(output_2, output_5, atol=self.base_precision) + + def test_from_save_pretrained_variant(self, expected_max_diff=5e-5): + if self.forward_requires_fresh_args: + model = self.model_class(**self.init_dict) + else: + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + + if hasattr(model, "set_default_attn_processor"): + model.set_default_attn_processor() + + model.to(torch_device) + model.eval() + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname, variant="fp16", safe_serialization=False) + new_model = self.model_class.from_pretrained(tmpdirname, variant="fp16") + if hasattr(new_model, "set_default_attn_processor"): + new_model.set_default_attn_processor() + + # non-variant cannot be loaded + with self.assertRaises(OSError) as error_context: + self.model_class.from_pretrained(tmpdirname) + + # make sure that error message states what keys are missing + assert "Error no file named diffusion_pytorch_model.bin found in directory" in str(error_context.exception) + + new_model.to(torch_device) + + with torch.no_grad(): + if self.forward_requires_fresh_args: + image = model(**self.inputs_dict(0)) + else: + image = model(**inputs_dict) + if isinstance(image, dict): + image = image.to_tuple()[0] + + if self.forward_requires_fresh_args: + new_image = new_model(**self.inputs_dict(0)) + else: + new_image = new_model(**inputs_dict) + + if isinstance(new_image, dict): + new_image = new_image.to_tuple()[0] + + max_diff = (image - new_image).abs().max().item() + self.assertLessEqual(max_diff, expected_max_diff, "Models give different forward passes") + + @require_python39_or_higher + @require_torch_2 + def test_from_save_pretrained_dynamo(self): + init_dict, _ = self.prepare_init_args_and_inputs_for_common() + inputs = [init_dict, self.model_class] + run_test_in_subprocess(test_case=self, target_func=_test_from_save_pretrained_dynamo, inputs=inputs) + + def test_from_save_pretrained_dtype(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + for dtype in [torch.float32, torch.float16, torch.bfloat16]: + if torch_device == "mps" and dtype == torch.bfloat16: + continue + with tempfile.TemporaryDirectory() as tmpdirname: + model.to(dtype) + model.save_pretrained(tmpdirname, safe_serialization=False) + new_model = self.model_class.from_pretrained(tmpdirname, low_cpu_mem_usage=True, torch_dtype=dtype) + assert new_model.dtype == dtype + new_model = self.model_class.from_pretrained(tmpdirname, low_cpu_mem_usage=False, torch_dtype=dtype) + assert new_model.dtype == dtype + + def test_determinism(self, expected_max_diff=1e-5): + if self.forward_requires_fresh_args: + model = self.model_class(**self.init_dict) + else: + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + if self.forward_requires_fresh_args: + first = model(**self.inputs_dict(0)) + else: + first = model(**inputs_dict) + if isinstance(first, dict): + first = first.to_tuple()[0] + + if self.forward_requires_fresh_args: + second = model(**self.inputs_dict(0)) + else: + second = model(**inputs_dict) + if isinstance(second, dict): + second = second.to_tuple()[0] + + out_1 = first.cpu().numpy() + out_2 = second.cpu().numpy() + out_1 = out_1[~np.isnan(out_1)] + out_2 = out_2[~np.isnan(out_2)] + max_diff = np.amax(np.abs(out_1 - out_2)) + self.assertLessEqual(max_diff, expected_max_diff) + + def test_output(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.to_tuple()[0] + + self.assertIsNotNone(output) + + # input & output have to have the same shape + input_tensor = inputs_dict[self.main_input_name] + expected_shape = input_tensor.shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_model_from_pretrained(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + # test if the model can be loaded from the config + # and has all the expected shape + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname, safe_serialization=False) + new_model = self.model_class.from_pretrained(tmpdirname) + new_model.to(torch_device) + new_model.eval() + + # check if all parameters shape are the same + for param_name in model.state_dict().keys(): + param_1 = model.state_dict()[param_name] + param_2 = new_model.state_dict()[param_name] + self.assertEqual(param_1.shape, param_2.shape) + + with torch.no_grad(): + output_1 = model(**inputs_dict) + + if isinstance(output_1, dict): + output_1 = output_1.to_tuple()[0] + + output_2 = new_model(**inputs_dict) + + if isinstance(output_2, dict): + output_2 = output_2.to_tuple()[0] + + self.assertEqual(output_1.shape, output_2.shape) + + @require_torch_accelerator_with_training + def test_training(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + model = self.model_class(**init_dict) + model.to(torch_device) + model.train() + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.to_tuple()[0] + + input_tensor = inputs_dict[self.main_input_name] + noise = torch.randn((input_tensor.shape[0],) + self.output_shape).to(torch_device) + loss = torch.nn.functional.mse_loss(output, noise) + loss.backward() + + @require_torch_accelerator_with_training + def test_ema_training(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + model = self.model_class(**init_dict) + model.to(torch_device) + model.train() + ema_model = EMAModel(model.parameters()) + + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.to_tuple()[0] + + input_tensor = inputs_dict[self.main_input_name] + noise = torch.randn((input_tensor.shape[0],) + self.output_shape).to(torch_device) + loss = torch.nn.functional.mse_loss(output, noise) + loss.backward() + ema_model.step(model.parameters()) + + def test_outputs_equivalence(self): + def set_nan_tensor_to_zero(t): + # Temporary fallback until `aten::_index_put_impl_` is implemented in mps + # Track progress in https://github.com/pytorch/pytorch/issues/77764 + device = t.device + if device.type == "mps": + t = t.to("cpu") + t[t != t] = 0 + return t.to(device) + + def recursive_check(tuple_object, dict_object): + if isinstance(tuple_object, (List, Tuple)): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object, dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif isinstance(tuple_object, Dict): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object.values(), dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif tuple_object is None: + return + else: + self.assertTrue( + torch.allclose( + set_nan_tensor_to_zero(tuple_object), set_nan_tensor_to_zero(dict_object), atol=1e-5 + ), + msg=( + "Tuple and dict output are not equal. Difference:" + f" {torch.max(torch.abs(tuple_object - dict_object))}. Tuple has `nan`:" + f" {torch.isnan(tuple_object).any()} and `inf`: {torch.isinf(tuple_object)}. Dict has" + f" `nan`: {torch.isnan(dict_object).any()} and `inf`: {torch.isinf(dict_object)}." + ), + ) + + if self.forward_requires_fresh_args: + model = self.model_class(**self.init_dict) + else: + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + + model.to(torch_device) + model.eval() + + with torch.no_grad(): + if self.forward_requires_fresh_args: + outputs_dict = model(**self.inputs_dict(0)) + outputs_tuple = model(**self.inputs_dict(0), return_dict=False) + else: + outputs_dict = model(**inputs_dict) + outputs_tuple = model(**inputs_dict, return_dict=False) + + recursive_check(outputs_tuple, outputs_dict) + + @require_torch_accelerator_with_training + def test_enable_disable_gradient_checkpointing(self): + if not self.model_class._supports_gradient_checkpointing: + return # Skip test if model does not support gradient checkpointing + + init_dict, _ = self.prepare_init_args_and_inputs_for_common() + + # at init model should have gradient checkpointing disabled + model = self.model_class(**init_dict) + self.assertFalse(model.is_gradient_checkpointing) + + # check enable works + model.enable_gradient_checkpointing() + self.assertTrue(model.is_gradient_checkpointing) + + # check disable works + model.disable_gradient_checkpointing() + self.assertFalse(model.is_gradient_checkpointing) + + def test_deprecated_kwargs(self): + has_kwarg_in_model_class = "kwargs" in inspect.signature(self.model_class.__init__).parameters + has_deprecated_kwarg = len(self.model_class._deprecated_kwargs) > 0 + + if has_kwarg_in_model_class and not has_deprecated_kwarg: + raise ValueError( + f"{self.model_class} has `**kwargs` in its __init__ method but has not defined any deprecated kwargs" + " under the `_deprecated_kwargs` class attribute. Make sure to either remove `**kwargs` if there are" + " no deprecated arguments or add the deprecated argument with `_deprecated_kwargs =" + " []`" + ) + + if not has_kwarg_in_model_class and has_deprecated_kwarg: + raise ValueError( + f"{self.model_class} doesn't have `**kwargs` in its __init__ method but has defined deprecated kwargs" + " under the `_deprecated_kwargs` class attribute. Make sure to either add the `**kwargs` argument to" + f" {self.model_class}.__init__ if there are deprecated arguments or remove the deprecated argument" + " from `_deprecated_kwargs = []`" + ) + + +@is_staging_test +class ModelPushToHubTester(unittest.TestCase): + identifier = uuid.uuid4() + repo_id = f"test-model-{identifier}" + org_repo_id = f"valid_org/{repo_id}-org" + + def test_push_to_hub(self): + model = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + model.push_to_hub(self.repo_id, token=TOKEN) + + new_model = UNet2DConditionModel.from_pretrained(f"{USER}/{self.repo_id}") + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + # Reset repo + delete_repo(token=TOKEN, repo_id=self.repo_id) + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, repo_id=self.repo_id, push_to_hub=True, token=TOKEN) + + new_model = UNet2DConditionModel.from_pretrained(f"{USER}/{self.repo_id}") + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + # Reset repo + delete_repo(self.repo_id, token=TOKEN) + + def test_push_to_hub_in_organization(self): + model = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + model.push_to_hub(self.org_repo_id, token=TOKEN) + + new_model = UNet2DConditionModel.from_pretrained(self.org_repo_id) + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + # Reset repo + delete_repo(token=TOKEN, repo_id=self.org_repo_id) + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + model.save_pretrained(tmp_dir, push_to_hub=True, token=TOKEN, repo_id=self.org_repo_id) + + new_model = UNet2DConditionModel.from_pretrained(self.org_repo_id) + for p1, p2 in zip(model.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + # Reset repo + delete_repo(self.org_repo_id, token=TOKEN) + + @unittest.skipIf( + not is_jinja_available(), + reason="Model card tests cannot be performed without Jinja installed.", + ) + def test_push_to_hub_library_name(self): + model = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + model.push_to_hub(self.repo_id, token=TOKEN) + + model_card = ModelCard.load(f"{USER}/{self.repo_id}", token=TOKEN).data + assert model_card.library_name == "diffusers" + + # Reset repo + delete_repo(self.repo_id, token=TOKEN) diff --git a/diffusers-0.27.0/tests/models/test_modeling_common_flax.py b/diffusers-0.27.0/tests/models/test_modeling_common_flax.py new file mode 100755 index 0000000..8945aed --- /dev/null +++ b/diffusers-0.27.0/tests/models/test_modeling_common_flax.py @@ -0,0 +1,66 @@ +import inspect + +from diffusers.utils import is_flax_available +from diffusers.utils.testing_utils import require_flax + + +if is_flax_available(): + import jax + + +@require_flax +class FlaxModelTesterMixin: + def test_output(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + model = self.model_class(**init_dict) + variables = model.init(inputs_dict["prng_key"], inputs_dict["sample"]) + jax.lax.stop_gradient(variables) + + output = model.apply(variables, inputs_dict["sample"]) + + if isinstance(output, dict): + output = output.sample + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_forward_with_norm_groups(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["norm_num_groups"] = 16 + init_dict["block_out_channels"] = (16, 32) + + model = self.model_class(**init_dict) + variables = model.init(inputs_dict["prng_key"], inputs_dict["sample"]) + jax.lax.stop_gradient(variables) + + output = model.apply(variables, inputs_dict["sample"]) + + if isinstance(output, dict): + output = output.sample + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_deprecated_kwargs(self): + has_kwarg_in_model_class = "kwargs" in inspect.signature(self.model_class.__init__).parameters + has_deprecated_kwarg = len(self.model_class._deprecated_kwargs) > 0 + + if has_kwarg_in_model_class and not has_deprecated_kwarg: + raise ValueError( + f"{self.model_class} has `**kwargs` in its __init__ method but has not defined any deprecated kwargs" + " under the `_deprecated_kwargs` class attribute. Make sure to either remove `**kwargs` if there are" + " no deprecated arguments or add the deprecated argument with `_deprecated_kwargs =" + " []`" + ) + + if not has_kwarg_in_model_class and has_deprecated_kwarg: + raise ValueError( + f"{self.model_class} doesn't have `**kwargs` in its __init__ method but has defined deprecated kwargs" + " under the `_deprecated_kwargs` class attribute. Make sure to either add the `**kwargs` argument to" + f" {self.model_class}.__init__ if there are deprecated arguments or remove the deprecated argument" + " from `_deprecated_kwargs = []`" + ) diff --git a/diffusers-0.27.0/tests/models/transformers/__init__.py b/diffusers-0.27.0/tests/models/transformers/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/models/transformers/test_models_prior.py b/diffusers-0.27.0/tests/models/transformers/test_models_prior.py new file mode 100755 index 0000000..d10ee17 --- /dev/null +++ b/diffusers-0.27.0/tests/models/transformers/test_models_prior.py @@ -0,0 +1,191 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import inspect +import unittest + +import torch +from parameterized import parameterized + +from diffusers import PriorTransformer +from diffusers.utils.testing_utils import ( + backend_empty_cache, + enable_full_determinism, + floats_tensor, + slow, + torch_all_close, + torch_device, +) + +from ..test_modeling_common import ModelTesterMixin + + +enable_full_determinism() + + +class PriorTransformerTests(ModelTesterMixin, unittest.TestCase): + model_class = PriorTransformer + main_input_name = "hidden_states" + + @property + def dummy_input(self): + batch_size = 4 + embedding_dim = 8 + num_embeddings = 7 + + hidden_states = floats_tensor((batch_size, embedding_dim)).to(torch_device) + + proj_embedding = floats_tensor((batch_size, embedding_dim)).to(torch_device) + encoder_hidden_states = floats_tensor((batch_size, num_embeddings, embedding_dim)).to(torch_device) + + return { + "hidden_states": hidden_states, + "timestep": 2, + "proj_embedding": proj_embedding, + "encoder_hidden_states": encoder_hidden_states, + } + + def get_dummy_seed_input(self, seed=0): + torch.manual_seed(seed) + batch_size = 4 + embedding_dim = 8 + num_embeddings = 7 + + hidden_states = torch.randn((batch_size, embedding_dim)).to(torch_device) + + proj_embedding = torch.randn((batch_size, embedding_dim)).to(torch_device) + encoder_hidden_states = torch.randn((batch_size, num_embeddings, embedding_dim)).to(torch_device) + + return { + "hidden_states": hidden_states, + "timestep": 2, + "proj_embedding": proj_embedding, + "encoder_hidden_states": encoder_hidden_states, + } + + @property + def input_shape(self): + return (4, 8) + + @property + def output_shape(self): + return (4, 8) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "num_attention_heads": 2, + "attention_head_dim": 4, + "num_layers": 2, + "embedding_dim": 8, + "num_embeddings": 7, + "additional_embeddings": 4, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_from_pretrained_hub(self): + model, loading_info = PriorTransformer.from_pretrained( + "hf-internal-testing/prior-dummy", output_loading_info=True + ) + self.assertIsNotNone(model) + self.assertEqual(len(loading_info["missing_keys"]), 0) + + model.to(torch_device) + hidden_states = model(**self.dummy_input)[0] + + assert hidden_states is not None, "Make sure output is not None" + + def test_forward_signature(self): + init_dict, _ = self.prepare_init_args_and_inputs_for_common() + + model = self.model_class(**init_dict) + signature = inspect.signature(model.forward) + # signature.parameters is an OrderedDict => so arg_names order is deterministic + arg_names = [*signature.parameters.keys()] + + expected_arg_names = ["hidden_states", "timestep"] + self.assertListEqual(arg_names[:2], expected_arg_names) + + def test_output_pretrained(self): + model = PriorTransformer.from_pretrained("hf-internal-testing/prior-dummy") + model = model.to(torch_device) + + if hasattr(model, "set_default_attn_processor"): + model.set_default_attn_processor() + + input = self.get_dummy_seed_input() + + with torch.no_grad(): + output = model(**input)[0] + + output_slice = output[0, :5].flatten().cpu() + print(output_slice) + + # Since the VAE Gaussian prior's generator is seeded on the appropriate device, + # the expected output slices are not the same for CPU and GPU. + expected_output_slice = torch.tensor([-1.3436, -0.2870, 0.7538, 0.4368, -0.0239]) + self.assertTrue(torch_all_close(output_slice, expected_output_slice, rtol=1e-2)) + + +@slow +class PriorTransformerIntegrationTests(unittest.TestCase): + def get_dummy_seed_input(self, batch_size=1, embedding_dim=768, num_embeddings=77, seed=0): + torch.manual_seed(seed) + batch_size = batch_size + embedding_dim = embedding_dim + num_embeddings = num_embeddings + + hidden_states = torch.randn((batch_size, embedding_dim)).to(torch_device) + + proj_embedding = torch.randn((batch_size, embedding_dim)).to(torch_device) + encoder_hidden_states = torch.randn((batch_size, num_embeddings, embedding_dim)).to(torch_device) + + return { + "hidden_states": hidden_states, + "timestep": 2, + "proj_embedding": proj_embedding, + "encoder_hidden_states": encoder_hidden_states, + } + + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + backend_empty_cache(torch_device) + + @parameterized.expand( + [ + # fmt: off + [13, [-0.5861, 0.1283, -0.0931, 0.0882, 0.4476, 0.1329, -0.0498, 0.0640]], + [37, [-0.4913, 0.0110, -0.0483, 0.0541, 0.4954, -0.0170, 0.0354, 0.1651]], + # fmt: on + ] + ) + def test_kandinsky_prior(self, seed, expected_slice): + model = PriorTransformer.from_pretrained("kandinsky-community/kandinsky-2-1-prior", subfolder="prior") + model.to(torch_device) + input = self.get_dummy_seed_input(seed=seed) + + with torch.no_grad(): + sample = model(**input)[0] + + assert list(sample.shape) == [1, 768] + + output_slice = sample[0, :8].flatten().cpu() + print(output_slice) + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=1e-3) diff --git a/diffusers-0.27.0/tests/models/unets/__init__.py b/diffusers-0.27.0/tests/models/unets/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/models/unets/test_models_unet_1d.py b/diffusers-0.27.0/tests/models/unets/test_models_unet_1d.py new file mode 100755 index 0000000..bd0506c --- /dev/null +++ b/diffusers-0.27.0/tests/models/unets/test_models_unet_1d.py @@ -0,0 +1,270 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import torch + +from diffusers import UNet1DModel +from diffusers.utils.testing_utils import ( + backend_manual_seed, + floats_tensor, + slow, + torch_device, +) + +from ..test_modeling_common import ModelTesterMixin, UNetTesterMixin + + +class UNet1DModelTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = UNet1DModel + main_input_name = "sample" + + @property + def dummy_input(self): + batch_size = 4 + num_features = 14 + seq_len = 16 + + noise = floats_tensor((batch_size, num_features, seq_len)).to(torch_device) + time_step = torch.tensor([10] * batch_size).to(torch_device) + + return {"sample": noise, "timestep": time_step} + + @property + def input_shape(self): + return (4, 14, 16) + + @property + def output_shape(self): + return (4, 14, 16) + + def test_ema_training(self): + pass + + def test_training(self): + pass + + def test_determinism(self): + super().test_determinism() + + def test_outputs_equivalence(self): + super().test_outputs_equivalence() + + def test_from_save_pretrained(self): + super().test_from_save_pretrained() + + def test_from_save_pretrained_variant(self): + super().test_from_save_pretrained_variant() + + def test_model_from_pretrained(self): + super().test_model_from_pretrained() + + def test_output(self): + super().test_output() + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "block_out_channels": (32, 64, 128, 256), + "in_channels": 14, + "out_channels": 14, + "time_embedding_type": "positional", + "use_timestep_embedding": True, + "flip_sin_to_cos": False, + "freq_shift": 1.0, + "out_block_type": "OutConv1DBlock", + "mid_block_type": "MidResTemporalBlock1D", + "down_block_types": ("DownResnetBlock1D", "DownResnetBlock1D", "DownResnetBlock1D", "DownResnetBlock1D"), + "up_block_types": ("UpResnetBlock1D", "UpResnetBlock1D", "UpResnetBlock1D"), + "act_fn": "swish", + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_from_pretrained_hub(self): + model, loading_info = UNet1DModel.from_pretrained( + "bglick13/hopper-medium-v2-value-function-hor32", output_loading_info=True, subfolder="unet" + ) + self.assertIsNotNone(model) + self.assertEqual(len(loading_info["missing_keys"]), 0) + + model.to(torch_device) + image = model(**self.dummy_input) + + assert image is not None, "Make sure output is not None" + + def test_output_pretrained(self): + model = UNet1DModel.from_pretrained("bglick13/hopper-medium-v2-value-function-hor32", subfolder="unet") + torch.manual_seed(0) + backend_manual_seed(torch_device, 0) + + num_features = model.config.in_channels + seq_len = 16 + noise = torch.randn((1, seq_len, num_features)).permute( + 0, 2, 1 + ) # match original, we can update values and remove + time_step = torch.full((num_features,), 0) + + with torch.no_grad(): + output = model(noise, time_step).sample.permute(0, 2, 1) + + output_slice = output[0, -3:, -3:].flatten() + # fmt: off + expected_output_slice = torch.tensor([-2.137172, 1.1426016, 0.3688687, -0.766922, 0.7303146, 0.11038864, -0.4760633, 0.13270172, 0.02591348]) + # fmt: on + self.assertTrue(torch.allclose(output_slice, expected_output_slice, rtol=1e-3)) + + def test_forward_with_norm_groups(self): + # Not implemented yet for this UNet + pass + + @slow + def test_unet_1d_maestro(self): + model_id = "harmonai/maestro-150k" + model = UNet1DModel.from_pretrained(model_id, subfolder="unet") + model.to(torch_device) + + sample_size = 65536 + noise = torch.sin(torch.arange(sample_size)[None, None, :].repeat(1, 2, 1)).to(torch_device) + timestep = torch.tensor([1]).to(torch_device) + + with torch.no_grad(): + output = model(noise, timestep).sample + + output_sum = output.abs().sum() + output_max = output.abs().max() + + assert (output_sum - 224.0896).abs() < 0.5 + assert (output_max - 0.0607).abs() < 4e-4 + + +class UNetRLModelTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = UNet1DModel + main_input_name = "sample" + + @property + def dummy_input(self): + batch_size = 4 + num_features = 14 + seq_len = 16 + + noise = floats_tensor((batch_size, num_features, seq_len)).to(torch_device) + time_step = torch.tensor([10] * batch_size).to(torch_device) + + return {"sample": noise, "timestep": time_step} + + @property + def input_shape(self): + return (4, 14, 16) + + @property + def output_shape(self): + return (4, 14, 1) + + def test_determinism(self): + super().test_determinism() + + def test_outputs_equivalence(self): + super().test_outputs_equivalence() + + def test_from_save_pretrained(self): + super().test_from_save_pretrained() + + def test_from_save_pretrained_variant(self): + super().test_from_save_pretrained_variant() + + def test_model_from_pretrained(self): + super().test_model_from_pretrained() + + def test_output(self): + # UNetRL is a value-function is different output shape + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.sample + + self.assertIsNotNone(output) + expected_shape = torch.Size((inputs_dict["sample"].shape[0], 1)) + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_ema_training(self): + pass + + def test_training(self): + pass + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "in_channels": 14, + "out_channels": 14, + "down_block_types": ["DownResnetBlock1D", "DownResnetBlock1D", "DownResnetBlock1D", "DownResnetBlock1D"], + "up_block_types": [], + "out_block_type": "ValueFunction", + "mid_block_type": "ValueFunctionMidBlock1D", + "block_out_channels": [32, 64, 128, 256], + "layers_per_block": 1, + "downsample_each_block": True, + "use_timestep_embedding": True, + "freq_shift": 1.0, + "flip_sin_to_cos": False, + "time_embedding_type": "positional", + "act_fn": "mish", + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_from_pretrained_hub(self): + value_function, vf_loading_info = UNet1DModel.from_pretrained( + "bglick13/hopper-medium-v2-value-function-hor32", output_loading_info=True, subfolder="value_function" + ) + self.assertIsNotNone(value_function) + self.assertEqual(len(vf_loading_info["missing_keys"]), 0) + + value_function.to(torch_device) + image = value_function(**self.dummy_input) + + assert image is not None, "Make sure output is not None" + + def test_output_pretrained(self): + value_function, vf_loading_info = UNet1DModel.from_pretrained( + "bglick13/hopper-medium-v2-value-function-hor32", output_loading_info=True, subfolder="value_function" + ) + torch.manual_seed(0) + backend_manual_seed(torch_device, 0) + + num_features = value_function.config.in_channels + seq_len = 14 + noise = torch.randn((1, seq_len, num_features)).permute( + 0, 2, 1 + ) # match original, we can update values and remove + time_step = torch.full((num_features,), 0) + + with torch.no_grad(): + output = value_function(noise, time_step).sample + + # fmt: off + expected_output_slice = torch.tensor([165.25] * seq_len) + # fmt: on + self.assertTrue(torch.allclose(output, expected_output_slice, rtol=1e-3)) + + def test_forward_with_norm_groups(self): + # Not implemented yet for this UNet + pass diff --git a/diffusers-0.27.0/tests/models/unets/test_models_unet_2d.py b/diffusers-0.27.0/tests/models/unets/test_models_unet_2d.py new file mode 100755 index 0000000..3397e21 --- /dev/null +++ b/diffusers-0.27.0/tests/models/unets/test_models_unet_2d.py @@ -0,0 +1,331 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import math +import unittest + +import torch + +from diffusers import UNet2DModel +from diffusers.utils import logging +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + require_torch_accelerator, + slow, + torch_all_close, + torch_device, +) + +from ..test_modeling_common import ModelTesterMixin, UNetTesterMixin + + +logger = logging.get_logger(__name__) + +enable_full_determinism() + + +class Unet2DModelTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = UNet2DModel + main_input_name = "sample" + + @property + def dummy_input(self): + batch_size = 4 + num_channels = 3 + sizes = (32, 32) + + noise = floats_tensor((batch_size, num_channels) + sizes).to(torch_device) + time_step = torch.tensor([10]).to(torch_device) + + return {"sample": noise, "timestep": time_step} + + @property + def input_shape(self): + return (3, 32, 32) + + @property + def output_shape(self): + return (3, 32, 32) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "block_out_channels": (32, 64), + "down_block_types": ("DownBlock2D", "AttnDownBlock2D"), + "up_block_types": ("AttnUpBlock2D", "UpBlock2D"), + "attention_head_dim": 3, + "out_channels": 3, + "in_channels": 3, + "layers_per_block": 2, + "sample_size": 32, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_mid_block_attn_groups(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["norm_num_groups"] = 16 + init_dict["add_attention"] = True + init_dict["attn_norm_num_groups"] = 8 + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + self.assertIsNotNone( + model.mid_block.attentions[0].group_norm, "Mid block Attention group norm should exist but does not." + ) + self.assertEqual( + model.mid_block.attentions[0].group_norm.num_groups, + init_dict["attn_norm_num_groups"], + "Mid block Attention group norm does not have the expected number of groups.", + ) + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.to_tuple()[0] + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + +class UNetLDMModelTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = UNet2DModel + main_input_name = "sample" + + @property + def dummy_input(self): + batch_size = 4 + num_channels = 4 + sizes = (32, 32) + + noise = floats_tensor((batch_size, num_channels) + sizes).to(torch_device) + time_step = torch.tensor([10]).to(torch_device) + + return {"sample": noise, "timestep": time_step} + + @property + def input_shape(self): + return (4, 32, 32) + + @property + def output_shape(self): + return (4, 32, 32) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "sample_size": 32, + "in_channels": 4, + "out_channels": 4, + "layers_per_block": 2, + "block_out_channels": (32, 64), + "attention_head_dim": 32, + "down_block_types": ("DownBlock2D", "DownBlock2D"), + "up_block_types": ("UpBlock2D", "UpBlock2D"), + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_from_pretrained_hub(self): + model, loading_info = UNet2DModel.from_pretrained("fusing/unet-ldm-dummy-update", output_loading_info=True) + + self.assertIsNotNone(model) + self.assertEqual(len(loading_info["missing_keys"]), 0) + + model.to(torch_device) + image = model(**self.dummy_input).sample + + assert image is not None, "Make sure output is not None" + + @require_torch_accelerator + def test_from_pretrained_accelerate(self): + model, _ = UNet2DModel.from_pretrained("fusing/unet-ldm-dummy-update", output_loading_info=True) + model.to(torch_device) + image = model(**self.dummy_input).sample + + assert image is not None, "Make sure output is not None" + + @require_torch_accelerator + def test_from_pretrained_accelerate_wont_change_results(self): + # by defautl model loading will use accelerate as `low_cpu_mem_usage=True` + model_accelerate, _ = UNet2DModel.from_pretrained("fusing/unet-ldm-dummy-update", output_loading_info=True) + model_accelerate.to(torch_device) + model_accelerate.eval() + + noise = torch.randn( + 1, + model_accelerate.config.in_channels, + model_accelerate.config.sample_size, + model_accelerate.config.sample_size, + generator=torch.manual_seed(0), + ) + noise = noise.to(torch_device) + time_step = torch.tensor([10] * noise.shape[0]).to(torch_device) + + arr_accelerate = model_accelerate(noise, time_step)["sample"] + + # two models don't need to stay in the device at the same time + del model_accelerate + torch.cuda.empty_cache() + gc.collect() + + model_normal_load, _ = UNet2DModel.from_pretrained( + "fusing/unet-ldm-dummy-update", output_loading_info=True, low_cpu_mem_usage=False + ) + model_normal_load.to(torch_device) + model_normal_load.eval() + arr_normal_load = model_normal_load(noise, time_step)["sample"] + + assert torch_all_close(arr_accelerate, arr_normal_load, rtol=1e-3) + + def test_output_pretrained(self): + model = UNet2DModel.from_pretrained("fusing/unet-ldm-dummy-update") + model.eval() + model.to(torch_device) + + noise = torch.randn( + 1, + model.config.in_channels, + model.config.sample_size, + model.config.sample_size, + generator=torch.manual_seed(0), + ) + noise = noise.to(torch_device) + time_step = torch.tensor([10] * noise.shape[0]).to(torch_device) + + with torch.no_grad(): + output = model(noise, time_step).sample + + output_slice = output[0, -1, -3:, -3:].flatten().cpu() + # fmt: off + expected_output_slice = torch.tensor([-13.3258, -20.1100, -15.9873, -17.6617, -23.0596, -17.9419, -13.3675, -16.1889, -12.3800]) + # fmt: on + + self.assertTrue(torch_all_close(output_slice, expected_output_slice, rtol=1e-3)) + + +class NCSNppModelTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = UNet2DModel + main_input_name = "sample" + + @property + def dummy_input(self, sizes=(32, 32)): + batch_size = 4 + num_channels = 3 + + noise = floats_tensor((batch_size, num_channels) + sizes).to(torch_device) + time_step = torch.tensor(batch_size * [10]).to(dtype=torch.int32, device=torch_device) + + return {"sample": noise, "timestep": time_step} + + @property + def input_shape(self): + return (3, 32, 32) + + @property + def output_shape(self): + return (3, 32, 32) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "block_out_channels": [32, 64, 64, 64], + "in_channels": 3, + "layers_per_block": 1, + "out_channels": 3, + "time_embedding_type": "fourier", + "norm_eps": 1e-6, + "mid_block_scale_factor": math.sqrt(2.0), + "norm_num_groups": None, + "down_block_types": [ + "SkipDownBlock2D", + "AttnSkipDownBlock2D", + "SkipDownBlock2D", + "SkipDownBlock2D", + ], + "up_block_types": [ + "SkipUpBlock2D", + "SkipUpBlock2D", + "AttnSkipUpBlock2D", + "SkipUpBlock2D", + ], + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + @slow + def test_from_pretrained_hub(self): + model, loading_info = UNet2DModel.from_pretrained("google/ncsnpp-celebahq-256", output_loading_info=True) + self.assertIsNotNone(model) + self.assertEqual(len(loading_info["missing_keys"]), 0) + + model.to(torch_device) + inputs = self.dummy_input + noise = floats_tensor((4, 3) + (256, 256)).to(torch_device) + inputs["sample"] = noise + image = model(**inputs) + + assert image is not None, "Make sure output is not None" + + @slow + def test_output_pretrained_ve_mid(self): + model = UNet2DModel.from_pretrained("google/ncsnpp-celebahq-256") + model.to(torch_device) + + batch_size = 4 + num_channels = 3 + sizes = (256, 256) + + noise = torch.ones((batch_size, num_channels) + sizes).to(torch_device) + time_step = torch.tensor(batch_size * [1e-4]).to(torch_device) + + with torch.no_grad(): + output = model(noise, time_step).sample + + output_slice = output[0, -3:, -3:, -1].flatten().cpu() + # fmt: off + expected_output_slice = torch.tensor([-4836.2178, -6487.1470, -3816.8196, -7964.9302, -10966.3037, -20043.5957, 8137.0513, 2340.3328, 544.6056]) + # fmt: on + + self.assertTrue(torch_all_close(output_slice, expected_output_slice, rtol=1e-2)) + + def test_output_pretrained_ve_large(self): + model = UNet2DModel.from_pretrained("fusing/ncsnpp-ffhq-ve-dummy-update") + model.to(torch_device) + + batch_size = 4 + num_channels = 3 + sizes = (32, 32) + + noise = torch.ones((batch_size, num_channels) + sizes).to(torch_device) + time_step = torch.tensor(batch_size * [1e-4]).to(torch_device) + + with torch.no_grad(): + output = model(noise, time_step).sample + + output_slice = output[0, -3:, -3:, -1].flatten().cpu() + # fmt: off + expected_output_slice = torch.tensor([-0.0325, -0.0900, -0.0869, -0.0332, -0.0725, -0.0270, -0.0101, 0.0227, 0.0256]) + # fmt: on + + self.assertTrue(torch_all_close(output_slice, expected_output_slice, rtol=1e-2)) + + def test_forward_with_norm_groups(self): + # not required for this model + pass diff --git a/diffusers-0.27.0/tests/models/unets/test_models_unet_2d_condition.py b/diffusers-0.27.0/tests/models/unets/test_models_unet_2d_condition.py new file mode 100755 index 0000000..db07b12 --- /dev/null +++ b/diffusers-0.27.0/tests/models/unets/test_models_unet_2d_condition.py @@ -0,0 +1,1225 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import gc +import os +import tempfile +import unittest +from collections import OrderedDict + +import torch +from parameterized import parameterized +from pytest import mark + +from diffusers import UNet2DConditionModel +from diffusers.models.attention_processor import ( + CustomDiffusionAttnProcessor, + IPAdapterAttnProcessor, + IPAdapterAttnProcessor2_0, +) +from diffusers.models.embeddings import ImageProjection, IPAdapterPlusImageProjection +from diffusers.utils import logging +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import ( + backend_empty_cache, + enable_full_determinism, + floats_tensor, + load_hf_numpy, + require_torch_accelerator, + require_torch_accelerator_with_fp16, + require_torch_accelerator_with_training, + require_torch_gpu, + skip_mps, + slow, + torch_all_close, + torch_device, +) + +from ..test_modeling_common import ModelTesterMixin, UNetTesterMixin + + +logger = logging.get_logger(__name__) + +enable_full_determinism() + + +def create_ip_adapter_state_dict(model): + # "ip_adapter" (cross-attention weights) + ip_cross_attn_state_dict = {} + key_id = 1 + + for name in model.attn_processors.keys(): + cross_attention_dim = ( + None if name.endswith("attn1.processor") or "motion_module" in name else model.config.cross_attention_dim + ) + + if name.startswith("mid_block"): + hidden_size = model.config.block_out_channels[-1] + elif name.startswith("up_blocks"): + block_id = int(name[len("up_blocks.")]) + hidden_size = list(reversed(model.config.block_out_channels))[block_id] + elif name.startswith("down_blocks"): + block_id = int(name[len("down_blocks.")]) + hidden_size = model.config.block_out_channels[block_id] + + if cross_attention_dim is not None: + sd = IPAdapterAttnProcessor( + hidden_size=hidden_size, cross_attention_dim=cross_attention_dim, scale=1.0 + ).state_dict() + ip_cross_attn_state_dict.update( + { + f"{key_id}.to_k_ip.weight": sd["to_k_ip.0.weight"], + f"{key_id}.to_v_ip.weight": sd["to_v_ip.0.weight"], + } + ) + + key_id += 2 + + # "image_proj" (ImageProjection layer weights) + cross_attention_dim = model.config["cross_attention_dim"] + image_projection = ImageProjection( + cross_attention_dim=cross_attention_dim, image_embed_dim=cross_attention_dim, num_image_text_embeds=4 + ) + + ip_image_projection_state_dict = {} + sd = image_projection.state_dict() + ip_image_projection_state_dict.update( + { + "proj.weight": sd["image_embeds.weight"], + "proj.bias": sd["image_embeds.bias"], + "norm.weight": sd["norm.weight"], + "norm.bias": sd["norm.bias"], + } + ) + + del sd + ip_state_dict = {} + ip_state_dict.update({"image_proj": ip_image_projection_state_dict, "ip_adapter": ip_cross_attn_state_dict}) + return ip_state_dict + + +def create_ip_adapter_plus_state_dict(model): + # "ip_adapter" (cross-attention weights) + ip_cross_attn_state_dict = {} + key_id = 1 + + for name in model.attn_processors.keys(): + cross_attention_dim = None if name.endswith("attn1.processor") else model.config.cross_attention_dim + if name.startswith("mid_block"): + hidden_size = model.config.block_out_channels[-1] + elif name.startswith("up_blocks"): + block_id = int(name[len("up_blocks.")]) + hidden_size = list(reversed(model.config.block_out_channels))[block_id] + elif name.startswith("down_blocks"): + block_id = int(name[len("down_blocks.")]) + hidden_size = model.config.block_out_channels[block_id] + if cross_attention_dim is not None: + sd = IPAdapterAttnProcessor( + hidden_size=hidden_size, cross_attention_dim=cross_attention_dim, scale=1.0 + ).state_dict() + ip_cross_attn_state_dict.update( + { + f"{key_id}.to_k_ip.weight": sd["to_k_ip.0.weight"], + f"{key_id}.to_v_ip.weight": sd["to_v_ip.0.weight"], + } + ) + + key_id += 2 + + # "image_proj" (ImageProjection layer weights) + cross_attention_dim = model.config["cross_attention_dim"] + image_projection = IPAdapterPlusImageProjection( + embed_dims=cross_attention_dim, output_dims=cross_attention_dim, dim_head=32, heads=2, num_queries=4 + ) + + ip_image_projection_state_dict = OrderedDict() + for k, v in image_projection.state_dict().items(): + if "2.to" in k: + k = k.replace("2.to", "0.to") + elif "3.0.weight" in k: + k = k.replace("3.0.weight", "1.0.weight") + elif "3.0.bias" in k: + k = k.replace("3.0.bias", "1.0.bias") + elif "3.0.weight" in k: + k = k.replace("3.0.weight", "1.0.weight") + elif "3.1.net.0.proj.weight" in k: + k = k.replace("3.1.net.0.proj.weight", "1.1.weight") + elif "3.net.2.weight" in k: + k = k.replace("3.net.2.weight", "1.3.weight") + elif "layers.0.0" in k: + k = k.replace("layers.0.0", "layers.0.0.norm1") + elif "layers.0.1" in k: + k = k.replace("layers.0.1", "layers.0.0.norm2") + elif "layers.1.0" in k: + k = k.replace("layers.1.0", "layers.1.0.norm1") + elif "layers.1.1" in k: + k = k.replace("layers.1.1", "layers.1.0.norm2") + elif "layers.2.0" in k: + k = k.replace("layers.2.0", "layers.2.0.norm1") + elif "layers.2.1" in k: + k = k.replace("layers.2.1", "layers.2.0.norm2") + + if "norm_cross" in k: + ip_image_projection_state_dict[k.replace("norm_cross", "norm1")] = v + elif "layer_norm" in k: + ip_image_projection_state_dict[k.replace("layer_norm", "norm2")] = v + elif "to_k" in k: + ip_image_projection_state_dict[k.replace("to_k", "to_kv")] = torch.cat([v, v], dim=0) + elif "to_v" in k: + continue + elif "to_out.0" in k: + ip_image_projection_state_dict[k.replace("to_out.0", "to_out")] = v + else: + ip_image_projection_state_dict[k] = v + + ip_state_dict = {} + ip_state_dict.update({"image_proj": ip_image_projection_state_dict, "ip_adapter": ip_cross_attn_state_dict}) + return ip_state_dict + + +def create_custom_diffusion_layers(model, mock_weights: bool = True): + train_kv = True + train_q_out = True + custom_diffusion_attn_procs = {} + + st = model.state_dict() + for name, _ in model.attn_processors.items(): + cross_attention_dim = None if name.endswith("attn1.processor") else model.config.cross_attention_dim + if name.startswith("mid_block"): + hidden_size = model.config.block_out_channels[-1] + elif name.startswith("up_blocks"): + block_id = int(name[len("up_blocks.")]) + hidden_size = list(reversed(model.config.block_out_channels))[block_id] + elif name.startswith("down_blocks"): + block_id = int(name[len("down_blocks.")]) + hidden_size = model.config.block_out_channels[block_id] + layer_name = name.split(".processor")[0] + weights = { + "to_k_custom_diffusion.weight": st[layer_name + ".to_k.weight"], + "to_v_custom_diffusion.weight": st[layer_name + ".to_v.weight"], + } + if train_q_out: + weights["to_q_custom_diffusion.weight"] = st[layer_name + ".to_q.weight"] + weights["to_out_custom_diffusion.0.weight"] = st[layer_name + ".to_out.0.weight"] + weights["to_out_custom_diffusion.0.bias"] = st[layer_name + ".to_out.0.bias"] + if cross_attention_dim is not None: + custom_diffusion_attn_procs[name] = CustomDiffusionAttnProcessor( + train_kv=train_kv, + train_q_out=train_q_out, + hidden_size=hidden_size, + cross_attention_dim=cross_attention_dim, + ).to(model.device) + custom_diffusion_attn_procs[name].load_state_dict(weights) + if mock_weights: + # add 1 to weights to mock trained weights + with torch.no_grad(): + custom_diffusion_attn_procs[name].to_k_custom_diffusion.weight += 1 + custom_diffusion_attn_procs[name].to_v_custom_diffusion.weight += 1 + else: + custom_diffusion_attn_procs[name] = CustomDiffusionAttnProcessor( + train_kv=False, + train_q_out=False, + hidden_size=hidden_size, + cross_attention_dim=cross_attention_dim, + ) + del st + return custom_diffusion_attn_procs + + +class UNet2DConditionModelTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = UNet2DConditionModel + main_input_name = "sample" + + @property + def dummy_input(self): + batch_size = 4 + num_channels = 4 + sizes = (32, 32) + + noise = floats_tensor((batch_size, num_channels) + sizes).to(torch_device) + time_step = torch.tensor([10]).to(torch_device) + encoder_hidden_states = floats_tensor((batch_size, 4, 32)).to(torch_device) + + return {"sample": noise, "timestep": time_step, "encoder_hidden_states": encoder_hidden_states} + + @property + def input_shape(self): + return (4, 32, 32) + + @property + def output_shape(self): + return (4, 32, 32) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "block_out_channels": (32, 64), + "down_block_types": ("CrossAttnDownBlock2D", "DownBlock2D"), + "up_block_types": ("UpBlock2D", "CrossAttnUpBlock2D"), + "cross_attention_dim": 32, + "attention_head_dim": 8, + "out_channels": 4, + "in_channels": 4, + "layers_per_block": 2, + "sample_size": 32, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_enable_works(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + + model.enable_xformers_memory_efficient_attention() + + assert ( + model.mid_block.attentions[0].transformer_blocks[0].attn1.processor.__class__.__name__ + == "XFormersAttnProcessor" + ), "xformers is not enabled" + + @require_torch_accelerator_with_training + def test_gradient_checkpointing(self): + # enable deterministic behavior for gradient checkpointing + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.to(torch_device) + + assert not model.is_gradient_checkpointing and model.training + + out = model(**inputs_dict).sample + # run the backwards pass on the model. For backwards pass, for simplicity purpose, + # we won't calculate the loss and rather backprop on out.sum() + model.zero_grad() + + labels = torch.randn_like(out) + loss = (out - labels).mean() + loss.backward() + + # re-instantiate the model now enabling gradient checkpointing + model_2 = self.model_class(**init_dict) + # clone model + model_2.load_state_dict(model.state_dict()) + model_2.to(torch_device) + model_2.enable_gradient_checkpointing() + + assert model_2.is_gradient_checkpointing and model_2.training + + out_2 = model_2(**inputs_dict).sample + # run the backwards pass on the model. For backwards pass, for simplicity purpose, + # we won't calculate the loss and rather backprop on out.sum() + model_2.zero_grad() + loss_2 = (out_2 - labels).mean() + loss_2.backward() + + # compare the output and parameters gradients + self.assertTrue((loss - loss_2).abs() < 1e-5) + named_params = dict(model.named_parameters()) + named_params_2 = dict(model_2.named_parameters()) + for name, param in named_params.items(): + self.assertTrue(torch_all_close(param.grad.data, named_params_2[name].grad.data, atol=5e-5)) + + def test_model_with_attention_head_dim_tuple(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = (8, 16) + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.sample + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_model_with_use_linear_projection(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["use_linear_projection"] = True + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.sample + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_model_with_cross_attention_dim_tuple(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["cross_attention_dim"] = (32, 32) + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.sample + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_model_with_simple_projection(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + batch_size, _, _, sample_size = inputs_dict["sample"].shape + + init_dict["class_embed_type"] = "simple_projection" + init_dict["projection_class_embeddings_input_dim"] = sample_size + + inputs_dict["class_labels"] = floats_tensor((batch_size, sample_size)).to(torch_device) + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.sample + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_model_with_class_embeddings_concat(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + batch_size, _, _, sample_size = inputs_dict["sample"].shape + + init_dict["class_embed_type"] = "simple_projection" + init_dict["projection_class_embeddings_input_dim"] = sample_size + init_dict["class_embeddings_concat"] = True + + inputs_dict["class_labels"] = floats_tensor((batch_size, sample_size)).to(torch_device) + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.sample + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_model_attention_slicing(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = (8, 16) + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + model.set_attention_slice("auto") + with torch.no_grad(): + output = model(**inputs_dict) + assert output is not None + + model.set_attention_slice("max") + with torch.no_grad(): + output = model(**inputs_dict) + assert output is not None + + model.set_attention_slice(2) + with torch.no_grad(): + output = model(**inputs_dict) + assert output is not None + + def test_model_sliceable_head_dim(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = (8, 16) + + model = self.model_class(**init_dict) + + def check_sliceable_dim_attr(module: torch.nn.Module): + if hasattr(module, "set_attention_slice"): + assert isinstance(module.sliceable_head_dim, int) + + for child in module.children(): + check_sliceable_dim_attr(child) + + # retrieve number of attention layers + for module in model.children(): + check_sliceable_dim_attr(module) + + def test_gradient_checkpointing_is_applied(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = (8, 16) + + model_class_copy = copy.copy(self.model_class) + + modules_with_gc_enabled = {} + + # now monkey patch the following function: + # def _set_gradient_checkpointing(self, module, value=False): + # if hasattr(module, "gradient_checkpointing"): + # module.gradient_checkpointing = value + + def _set_gradient_checkpointing_new(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + modules_with_gc_enabled[module.__class__.__name__] = True + + model_class_copy._set_gradient_checkpointing = _set_gradient_checkpointing_new + + model = model_class_copy(**init_dict) + model.enable_gradient_checkpointing() + + EXPECTED_SET = { + "CrossAttnUpBlock2D", + "CrossAttnDownBlock2D", + "UNetMidBlock2DCrossAttn", + "UpBlock2D", + "Transformer2DModel", + "DownBlock2D", + } + + assert set(modules_with_gc_enabled.keys()) == EXPECTED_SET + assert all(modules_with_gc_enabled.values()), "All modules should be enabled" + + def test_special_attn_proc(self): + class AttnEasyProc(torch.nn.Module): + def __init__(self, num): + super().__init__() + self.weight = torch.nn.Parameter(torch.tensor(num)) + self.is_run = False + self.number = 0 + self.counter = 0 + + def __call__(self, attn, hidden_states, encoder_hidden_states=None, attention_mask=None, number=None): + batch_size, sequence_length, _ = hidden_states.shape + attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size) + + query = attn.to_q(hidden_states) + + encoder_hidden_states = encoder_hidden_states if encoder_hidden_states is not None else hidden_states + key = attn.to_k(encoder_hidden_states) + value = attn.to_v(encoder_hidden_states) + + query = attn.head_to_batch_dim(query) + key = attn.head_to_batch_dim(key) + value = attn.head_to_batch_dim(value) + + attention_probs = attn.get_attention_scores(query, key, attention_mask) + hidden_states = torch.bmm(attention_probs, value) + hidden_states = attn.batch_to_head_dim(hidden_states) + + # linear proj + hidden_states = attn.to_out[0](hidden_states) + # dropout + hidden_states = attn.to_out[1](hidden_states) + + hidden_states += self.weight + + self.is_run = True + self.counter += 1 + self.number = number + + return hidden_states + + # enable deterministic behavior for gradient checkpointing + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = (8, 16) + + model = self.model_class(**init_dict) + model.to(torch_device) + + processor = AttnEasyProc(5.0) + + model.set_attn_processor(processor) + model(**inputs_dict, cross_attention_kwargs={"number": 123}).sample + + assert processor.counter == 12 + assert processor.is_run + assert processor.number == 123 + + @parameterized.expand( + [ + # fmt: off + [torch.bool], + [torch.long], + [torch.float], + # fmt: on + ] + ) + def test_model_xattn_mask(self, mask_dtype): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + model = self.model_class(**{**init_dict, "attention_head_dim": (8, 16)}) + model.to(torch_device) + model.eval() + + cond = inputs_dict["encoder_hidden_states"] + with torch.no_grad(): + full_cond_out = model(**inputs_dict).sample + assert full_cond_out is not None + + keepall_mask = torch.ones(*cond.shape[:-1], device=cond.device, dtype=mask_dtype) + full_cond_keepallmask_out = model(**{**inputs_dict, "encoder_attention_mask": keepall_mask}).sample + assert full_cond_keepallmask_out.allclose( + full_cond_out, rtol=1e-05, atol=1e-05 + ), "a 'keep all' mask should give the same result as no mask" + + trunc_cond = cond[:, :-1, :] + trunc_cond_out = model(**{**inputs_dict, "encoder_hidden_states": trunc_cond}).sample + assert not trunc_cond_out.allclose( + full_cond_out, rtol=1e-05, atol=1e-05 + ), "discarding the last token from our cond should change the result" + + batch, tokens, _ = cond.shape + mask_last = (torch.arange(tokens) < tokens - 1).expand(batch, -1).to(cond.device, mask_dtype) + masked_cond_out = model(**{**inputs_dict, "encoder_attention_mask": mask_last}).sample + assert masked_cond_out.allclose( + trunc_cond_out, rtol=1e-05, atol=1e-05 + ), "masking the last token from our cond should be equivalent to truncating that token out of the condition" + + # see diffusers.models.attention_processor::Attention#prepare_attention_mask + # note: we may not need to fix mask padding to work for stable-diffusion cross-attn masks. + # since the use-case (somebody passes in a too-short cross-attn mask) is pretty esoteric. + # maybe it's fine that this only works for the unclip use-case. + @mark.skip( + reason="we currently pad mask by target_length tokens (what unclip needs), whereas stable-diffusion's cross-attn needs to instead pad by remaining_length." + ) + def test_model_xattn_padding(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + model = self.model_class(**{**init_dict, "attention_head_dim": (8, 16)}) + model.to(torch_device) + model.eval() + + cond = inputs_dict["encoder_hidden_states"] + with torch.no_grad(): + full_cond_out = model(**inputs_dict).sample + assert full_cond_out is not None + + batch, tokens, _ = cond.shape + keeplast_mask = (torch.arange(tokens) == tokens - 1).expand(batch, -1).to(cond.device, torch.bool) + keeplast_out = model(**{**inputs_dict, "encoder_attention_mask": keeplast_mask}).sample + assert not keeplast_out.allclose(full_cond_out), "a 'keep last token' mask should change the result" + + trunc_mask = torch.zeros(batch, tokens - 1, device=cond.device, dtype=torch.bool) + trunc_mask_out = model(**{**inputs_dict, "encoder_attention_mask": trunc_mask}).sample + assert trunc_mask_out.allclose( + keeplast_out + ), "a mask with fewer tokens than condition, will be padded with 'keep' tokens. a 'discard-all' mask missing the final token is thus equivalent to a 'keep last' mask." + + def test_custom_diffusion_processors(self): + # enable deterministic behavior for gradient checkpointing + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = (8, 16) + + model = self.model_class(**init_dict) + model.to(torch_device) + + with torch.no_grad(): + sample1 = model(**inputs_dict).sample + + custom_diffusion_attn_procs = create_custom_diffusion_layers(model, mock_weights=False) + + # make sure we can set a list of attention processors + model.set_attn_processor(custom_diffusion_attn_procs) + model.to(torch_device) + + # test that attn processors can be set to itself + model.set_attn_processor(model.attn_processors) + + with torch.no_grad(): + sample2 = model(**inputs_dict).sample + + assert (sample1 - sample2).abs().max() < 3e-3 + + def test_custom_diffusion_save_load(self): + # enable deterministic behavior for gradient checkpointing + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = (8, 16) + + torch.manual_seed(0) + model = self.model_class(**init_dict) + model.to(torch_device) + + with torch.no_grad(): + old_sample = model(**inputs_dict).sample + + custom_diffusion_attn_procs = create_custom_diffusion_layers(model, mock_weights=False) + model.set_attn_processor(custom_diffusion_attn_procs) + + with torch.no_grad(): + sample = model(**inputs_dict).sample + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_attn_procs(tmpdirname, safe_serialization=False) + self.assertTrue(os.path.isfile(os.path.join(tmpdirname, "pytorch_custom_diffusion_weights.bin"))) + torch.manual_seed(0) + new_model = self.model_class(**init_dict) + new_model.load_attn_procs(tmpdirname, weight_name="pytorch_custom_diffusion_weights.bin") + new_model.to(torch_device) + + with torch.no_grad(): + new_sample = new_model(**inputs_dict).sample + + assert (sample - new_sample).abs().max() < 1e-4 + + # custom diffusion and no custom diffusion should be the same + assert (sample - old_sample).abs().max() < 3e-3 + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_custom_diffusion_xformers_on_off(self): + # enable deterministic behavior for gradient checkpointing + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = (8, 16) + + torch.manual_seed(0) + model = self.model_class(**init_dict) + model.to(torch_device) + custom_diffusion_attn_procs = create_custom_diffusion_layers(model, mock_weights=False) + model.set_attn_processor(custom_diffusion_attn_procs) + + # default + with torch.no_grad(): + sample = model(**inputs_dict).sample + + model.enable_xformers_memory_efficient_attention() + on_sample = model(**inputs_dict).sample + + model.disable_xformers_memory_efficient_attention() + off_sample = model(**inputs_dict).sample + + assert (sample - on_sample).abs().max() < 1e-4 + assert (sample - off_sample).abs().max() < 1e-4 + + def test_pickle(self): + # enable deterministic behavior for gradient checkpointing + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = (8, 16) + + model = self.model_class(**init_dict) + model.to(torch_device) + + with torch.no_grad(): + sample = model(**inputs_dict).sample + + sample_copy = copy.copy(sample) + + assert (sample - sample_copy).abs().max() < 1e-4 + + def test_asymmetrical_unet(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + # Add asymmetry to configs + init_dict["transformer_layers_per_block"] = [[3, 2], 1] + init_dict["reverse_transformer_layers_per_block"] = [[3, 4], 1] + + torch.manual_seed(0) + model = self.model_class(**init_dict) + model.to(torch_device) + + output = model(**inputs_dict).sample + expected_shape = inputs_dict["sample"].shape + + # Check if input and output shapes are the same + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_ip_adapter(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = (8, 16) + + model = self.model_class(**init_dict) + model.to(torch_device) + + # forward pass without ip-adapter + with torch.no_grad(): + sample1 = model(**inputs_dict).sample + + # update inputs_dict for ip-adapter + batch_size = inputs_dict["encoder_hidden_states"].shape[0] + # for ip-adapter image_embeds has shape [batch_size, num_image, embed_dim] + image_embeds = floats_tensor((batch_size, 1, model.cross_attention_dim)).to(torch_device) + inputs_dict["added_cond_kwargs"] = {"image_embeds": [image_embeds]} + + # make ip_adapter_1 and ip_adapter_2 + ip_adapter_1 = create_ip_adapter_state_dict(model) + + image_proj_state_dict_2 = {k: w + 1.0 for k, w in ip_adapter_1["image_proj"].items()} + cross_attn_state_dict_2 = {k: w + 1.0 for k, w in ip_adapter_1["ip_adapter"].items()} + ip_adapter_2 = {} + ip_adapter_2.update({"image_proj": image_proj_state_dict_2, "ip_adapter": cross_attn_state_dict_2}) + + # forward pass ip_adapter_1 + model._load_ip_adapter_weights([ip_adapter_1]) + assert model.config.encoder_hid_dim_type == "ip_image_proj" + assert model.encoder_hid_proj is not None + assert model.down_blocks[0].attentions[0].transformer_blocks[0].attn2.processor.__class__.__name__ in ( + "IPAdapterAttnProcessor", + "IPAdapterAttnProcessor2_0", + ) + with torch.no_grad(): + sample2 = model(**inputs_dict).sample + + # forward pass with ip_adapter_2 + model._load_ip_adapter_weights([ip_adapter_2]) + with torch.no_grad(): + sample3 = model(**inputs_dict).sample + + # forward pass with ip_adapter_1 again + model._load_ip_adapter_weights([ip_adapter_1]) + with torch.no_grad(): + sample4 = model(**inputs_dict).sample + + # forward pass with multiple ip-adapters and multiple images + model._load_ip_adapter_weights([ip_adapter_1, ip_adapter_2]) + # set the scale for ip_adapter_2 to 0 so that result should be same as only load ip_adapter_1 + for attn_processor in model.attn_processors.values(): + if isinstance(attn_processor, (IPAdapterAttnProcessor, IPAdapterAttnProcessor2_0)): + attn_processor.scale = [1, 0] + image_embeds_multi = image_embeds.repeat(1, 2, 1) + inputs_dict["added_cond_kwargs"] = {"image_embeds": [image_embeds_multi, image_embeds_multi]} + with torch.no_grad(): + sample5 = model(**inputs_dict).sample + + # forward pass with single ip-adapter & single image when image_embeds is not a list and a 2-d tensor + image_embeds = image_embeds.squeeze(1) + inputs_dict["added_cond_kwargs"] = {"image_embeds": image_embeds} + + model._load_ip_adapter_weights(ip_adapter_1) + with torch.no_grad(): + sample6 = model(**inputs_dict).sample + + assert not sample1.allclose(sample2, atol=1e-4, rtol=1e-4) + assert not sample2.allclose(sample3, atol=1e-4, rtol=1e-4) + assert sample2.allclose(sample4, atol=1e-4, rtol=1e-4) + assert sample2.allclose(sample5, atol=1e-4, rtol=1e-4) + assert sample2.allclose(sample6, atol=1e-4, rtol=1e-4) + + def test_ip_adapter_plus(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = (8, 16) + + model = self.model_class(**init_dict) + model.to(torch_device) + + # forward pass without ip-adapter + with torch.no_grad(): + sample1 = model(**inputs_dict).sample + + # update inputs_dict for ip-adapter + batch_size = inputs_dict["encoder_hidden_states"].shape[0] + # for ip-adapter-plus image_embeds has shape [batch_size, num_image, sequence_length, embed_dim] + image_embeds = floats_tensor((batch_size, 1, 1, model.cross_attention_dim)).to(torch_device) + inputs_dict["added_cond_kwargs"] = {"image_embeds": [image_embeds]} + + # make ip_adapter_1 and ip_adapter_2 + ip_adapter_1 = create_ip_adapter_plus_state_dict(model) + + image_proj_state_dict_2 = {k: w + 1.0 for k, w in ip_adapter_1["image_proj"].items()} + cross_attn_state_dict_2 = {k: w + 1.0 for k, w in ip_adapter_1["ip_adapter"].items()} + ip_adapter_2 = {} + ip_adapter_2.update({"image_proj": image_proj_state_dict_2, "ip_adapter": cross_attn_state_dict_2}) + + # forward pass ip_adapter_1 + model._load_ip_adapter_weights([ip_adapter_1]) + assert model.config.encoder_hid_dim_type == "ip_image_proj" + assert model.encoder_hid_proj is not None + assert model.down_blocks[0].attentions[0].transformer_blocks[0].attn2.processor.__class__.__name__ in ( + "IPAdapterAttnProcessor", + "IPAdapterAttnProcessor2_0", + ) + with torch.no_grad(): + sample2 = model(**inputs_dict).sample + + # forward pass with ip_adapter_2 + model._load_ip_adapter_weights([ip_adapter_2]) + with torch.no_grad(): + sample3 = model(**inputs_dict).sample + + # forward pass with ip_adapter_1 again + model._load_ip_adapter_weights([ip_adapter_1]) + with torch.no_grad(): + sample4 = model(**inputs_dict).sample + + # forward pass with multiple ip-adapters and multiple images + model._load_ip_adapter_weights([ip_adapter_1, ip_adapter_2]) + # set the scale for ip_adapter_2 to 0 so that result should be same as only load ip_adapter_1 + for attn_processor in model.attn_processors.values(): + if isinstance(attn_processor, (IPAdapterAttnProcessor, IPAdapterAttnProcessor2_0)): + attn_processor.scale = [1, 0] + image_embeds_multi = image_embeds.repeat(1, 2, 1, 1) + inputs_dict["added_cond_kwargs"] = {"image_embeds": [image_embeds_multi, image_embeds_multi]} + with torch.no_grad(): + sample5 = model(**inputs_dict).sample + + # forward pass with single ip-adapter & single image when image_embeds is a 3-d tensor + image_embeds = image_embeds[:,].squeeze(1) + inputs_dict["added_cond_kwargs"] = {"image_embeds": image_embeds} + + model._load_ip_adapter_weights(ip_adapter_1) + with torch.no_grad(): + sample6 = model(**inputs_dict).sample + + assert not sample1.allclose(sample2, atol=1e-4, rtol=1e-4) + assert not sample2.allclose(sample3, atol=1e-4, rtol=1e-4) + assert sample2.allclose(sample4, atol=1e-4, rtol=1e-4) + assert sample2.allclose(sample5, atol=1e-4, rtol=1e-4) + assert sample2.allclose(sample6, atol=1e-4, rtol=1e-4) + + +@slow +class UNet2DConditionModelIntegrationTests(unittest.TestCase): + def get_file_format(self, seed, shape): + return f"gaussian_noise_s={seed}_shape={'_'.join([str(s) for s in shape])}.npy" + + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + backend_empty_cache(torch_device) + + def get_latents(self, seed=0, shape=(4, 4, 64, 64), fp16=False): + dtype = torch.float16 if fp16 else torch.float32 + image = torch.from_numpy(load_hf_numpy(self.get_file_format(seed, shape))).to(torch_device).to(dtype) + return image + + def get_unet_model(self, fp16=False, model_id="CompVis/stable-diffusion-v1-4"): + revision = "fp16" if fp16 else None + torch_dtype = torch.float16 if fp16 else torch.float32 + + model = UNet2DConditionModel.from_pretrained( + model_id, subfolder="unet", torch_dtype=torch_dtype, revision=revision + ) + model.to(torch_device).eval() + + return model + + @require_torch_gpu + def test_set_attention_slice_auto(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + unet = self.get_unet_model() + unet.set_attention_slice("auto") + + latents = self.get_latents(33) + encoder_hidden_states = self.get_encoder_hidden_states(33) + timestep = 1 + + with torch.no_grad(): + _ = unet(latents, timestep=timestep, encoder_hidden_states=encoder_hidden_states).sample + + mem_bytes = torch.cuda.max_memory_allocated() + + assert mem_bytes < 5 * 10**9 + + @require_torch_gpu + def test_set_attention_slice_max(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + unet = self.get_unet_model() + unet.set_attention_slice("max") + + latents = self.get_latents(33) + encoder_hidden_states = self.get_encoder_hidden_states(33) + timestep = 1 + + with torch.no_grad(): + _ = unet(latents, timestep=timestep, encoder_hidden_states=encoder_hidden_states).sample + + mem_bytes = torch.cuda.max_memory_allocated() + + assert mem_bytes < 5 * 10**9 + + @require_torch_gpu + def test_set_attention_slice_int(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + unet = self.get_unet_model() + unet.set_attention_slice(2) + + latents = self.get_latents(33) + encoder_hidden_states = self.get_encoder_hidden_states(33) + timestep = 1 + + with torch.no_grad(): + _ = unet(latents, timestep=timestep, encoder_hidden_states=encoder_hidden_states).sample + + mem_bytes = torch.cuda.max_memory_allocated() + + assert mem_bytes < 5 * 10**9 + + @require_torch_gpu + def test_set_attention_slice_list(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + # there are 32 sliceable layers + slice_list = 16 * [2, 3] + unet = self.get_unet_model() + unet.set_attention_slice(slice_list) + + latents = self.get_latents(33) + encoder_hidden_states = self.get_encoder_hidden_states(33) + timestep = 1 + + with torch.no_grad(): + _ = unet(latents, timestep=timestep, encoder_hidden_states=encoder_hidden_states).sample + + mem_bytes = torch.cuda.max_memory_allocated() + + assert mem_bytes < 5 * 10**9 + + def get_encoder_hidden_states(self, seed=0, shape=(4, 77, 768), fp16=False): + dtype = torch.float16 if fp16 else torch.float32 + hidden_states = torch.from_numpy(load_hf_numpy(self.get_file_format(seed, shape))).to(torch_device).to(dtype) + return hidden_states + + @parameterized.expand( + [ + # fmt: off + [33, 4, [-0.4424, 0.1510, -0.1937, 0.2118, 0.3746, -0.3957, 0.0160, -0.0435]], + [47, 0.55, [-0.1508, 0.0379, -0.3075, 0.2540, 0.3633, -0.0821, 0.1719, -0.0207]], + [21, 0.89, [-0.6479, 0.6364, -0.3464, 0.8697, 0.4443, -0.6289, -0.0091, 0.1778]], + [9, 1000, [0.8888, -0.5659, 0.5834, -0.7469, 1.1912, -0.3923, 1.1241, -0.4424]], + # fmt: on + ] + ) + @require_torch_accelerator_with_fp16 + def test_compvis_sd_v1_4(self, seed, timestep, expected_slice): + model = self.get_unet_model(model_id="CompVis/stable-diffusion-v1-4") + latents = self.get_latents(seed) + encoder_hidden_states = self.get_encoder_hidden_states(seed) + + timestep = torch.tensor([timestep], dtype=torch.long, device=torch_device) + + with torch.no_grad(): + sample = model(latents, timestep=timestep, encoder_hidden_states=encoder_hidden_states).sample + + assert sample.shape == latents.shape + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=1e-3) + + @parameterized.expand( + [ + # fmt: off + [83, 4, [-0.2323, -0.1304, 0.0813, -0.3093, -0.0919, -0.1571, -0.1125, -0.5806]], + [17, 0.55, [-0.0831, -0.2443, 0.0901, -0.0919, 0.3396, 0.0103, -0.3743, 0.0701]], + [8, 0.89, [-0.4863, 0.0859, 0.0875, -0.1658, 0.9199, -0.0114, 0.4839, 0.4639]], + [3, 1000, [-0.5649, 0.2402, -0.5518, 0.1248, 1.1328, -0.2443, -0.0325, -1.0078]], + # fmt: on + ] + ) + @require_torch_accelerator_with_fp16 + def test_compvis_sd_v1_4_fp16(self, seed, timestep, expected_slice): + model = self.get_unet_model(model_id="CompVis/stable-diffusion-v1-4", fp16=True) + latents = self.get_latents(seed, fp16=True) + encoder_hidden_states = self.get_encoder_hidden_states(seed, fp16=True) + + timestep = torch.tensor([timestep], dtype=torch.long, device=torch_device) + + with torch.no_grad(): + sample = model(latents, timestep=timestep, encoder_hidden_states=encoder_hidden_states).sample + + assert sample.shape == latents.shape + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=5e-3) + + @parameterized.expand( + [ + # fmt: off + [33, 4, [-0.4430, 0.1570, -0.1867, 0.2376, 0.3205, -0.3681, 0.0525, -0.0722]], + [47, 0.55, [-0.1415, 0.0129, -0.3136, 0.2257, 0.3430, -0.0536, 0.2114, -0.0436]], + [21, 0.89, [-0.7091, 0.6664, -0.3643, 0.9032, 0.4499, -0.6541, 0.0139, 0.1750]], + [9, 1000, [0.8878, -0.5659, 0.5844, -0.7442, 1.1883, -0.3927, 1.1192, -0.4423]], + # fmt: on + ] + ) + @require_torch_accelerator + @skip_mps + def test_compvis_sd_v1_5(self, seed, timestep, expected_slice): + model = self.get_unet_model(model_id="runwayml/stable-diffusion-v1-5") + latents = self.get_latents(seed) + encoder_hidden_states = self.get_encoder_hidden_states(seed) + + timestep = torch.tensor([timestep], dtype=torch.long, device=torch_device) + + with torch.no_grad(): + sample = model(latents, timestep=timestep, encoder_hidden_states=encoder_hidden_states).sample + + assert sample.shape == latents.shape + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=1e-3) + + @parameterized.expand( + [ + # fmt: off + [83, 4, [-0.2695, -0.1669, 0.0073, -0.3181, -0.1187, -0.1676, -0.1395, -0.5972]], + [17, 0.55, [-0.1290, -0.2588, 0.0551, -0.0916, 0.3286, 0.0238, -0.3669, 0.0322]], + [8, 0.89, [-0.5283, 0.1198, 0.0870, -0.1141, 0.9189, -0.0150, 0.5474, 0.4319]], + [3, 1000, [-0.5601, 0.2411, -0.5435, 0.1268, 1.1338, -0.2427, -0.0280, -1.0020]], + # fmt: on + ] + ) + @require_torch_accelerator_with_fp16 + def test_compvis_sd_v1_5_fp16(self, seed, timestep, expected_slice): + model = self.get_unet_model(model_id="runwayml/stable-diffusion-v1-5", fp16=True) + latents = self.get_latents(seed, fp16=True) + encoder_hidden_states = self.get_encoder_hidden_states(seed, fp16=True) + + timestep = torch.tensor([timestep], dtype=torch.long, device=torch_device) + + with torch.no_grad(): + sample = model(latents, timestep=timestep, encoder_hidden_states=encoder_hidden_states).sample + + assert sample.shape == latents.shape + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=5e-3) + + @parameterized.expand( + [ + # fmt: off + [33, 4, [-0.7639, 0.0106, -0.1615, -0.3487, -0.0423, -0.7972, 0.0085, -0.4858]], + [47, 0.55, [-0.6564, 0.0795, -1.9026, -0.6258, 1.8235, 1.2056, 1.2169, 0.9073]], + [21, 0.89, [0.0327, 0.4399, -0.6358, 0.3417, 0.4120, -0.5621, -0.0397, -1.0430]], + [9, 1000, [0.1600, 0.7303, -1.0556, -0.3515, -0.7440, -1.2037, -1.8149, -1.8931]], + # fmt: on + ] + ) + @require_torch_accelerator + @skip_mps + def test_compvis_sd_inpaint(self, seed, timestep, expected_slice): + model = self.get_unet_model(model_id="runwayml/stable-diffusion-inpainting") + latents = self.get_latents(seed, shape=(4, 9, 64, 64)) + encoder_hidden_states = self.get_encoder_hidden_states(seed) + + timestep = torch.tensor([timestep], dtype=torch.long, device=torch_device) + + with torch.no_grad(): + sample = model(latents, timestep=timestep, encoder_hidden_states=encoder_hidden_states).sample + + assert sample.shape == (4, 4, 64, 64) + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=3e-3) + + @parameterized.expand( + [ + # fmt: off + [83, 4, [-0.1047, -1.7227, 0.1067, 0.0164, -0.5698, -0.4172, -0.1388, 1.1387]], + [17, 0.55, [0.0975, -0.2856, -0.3508, -0.4600, 0.3376, 0.2930, -0.2747, -0.7026]], + [8, 0.89, [-0.0952, 0.0183, -0.5825, -0.1981, 0.1131, 0.4668, -0.0395, -0.3486]], + [3, 1000, [0.4790, 0.4949, -1.0732, -0.7158, 0.7959, -0.9478, 0.1105, -0.9741]], + # fmt: on + ] + ) + @require_torch_accelerator_with_fp16 + def test_compvis_sd_inpaint_fp16(self, seed, timestep, expected_slice): + model = self.get_unet_model(model_id="runwayml/stable-diffusion-inpainting", fp16=True) + latents = self.get_latents(seed, shape=(4, 9, 64, 64), fp16=True) + encoder_hidden_states = self.get_encoder_hidden_states(seed, fp16=True) + + timestep = torch.tensor([timestep], dtype=torch.long, device=torch_device) + + with torch.no_grad(): + sample = model(latents, timestep=timestep, encoder_hidden_states=encoder_hidden_states).sample + + assert sample.shape == (4, 4, 64, 64) + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=5e-3) + + @parameterized.expand( + [ + # fmt: off + [83, 4, [0.1514, 0.0807, 0.1624, 0.1016, -0.1896, 0.0263, 0.0677, 0.2310]], + [17, 0.55, [0.1164, -0.0216, 0.0170, 0.1589, -0.3120, 0.1005, -0.0581, -0.1458]], + [8, 0.89, [-0.1758, -0.0169, 0.1004, -0.1411, 0.1312, 0.1103, -0.1996, 0.2139]], + [3, 1000, [0.1214, 0.0352, -0.0731, -0.1562, -0.0994, -0.0906, -0.2340, -0.0539]], + # fmt: on + ] + ) + @require_torch_accelerator_with_fp16 + def test_stabilityai_sd_v2_fp16(self, seed, timestep, expected_slice): + model = self.get_unet_model(model_id="stabilityai/stable-diffusion-2", fp16=True) + latents = self.get_latents(seed, shape=(4, 4, 96, 96), fp16=True) + encoder_hidden_states = self.get_encoder_hidden_states(seed, shape=(4, 77, 1024), fp16=True) + + timestep = torch.tensor([timestep], dtype=torch.long, device=torch_device) + + with torch.no_grad(): + sample = model(latents, timestep=timestep, encoder_hidden_states=encoder_hidden_states).sample + + assert sample.shape == latents.shape + + output_slice = sample[-1, -2:, -2:, :2].flatten().float().cpu() + expected_output_slice = torch.tensor(expected_slice) + + assert torch_all_close(output_slice, expected_output_slice, atol=5e-3) diff --git a/diffusers-0.27.0/tests/models/unets/test_models_unet_2d_flax.py b/diffusers-0.27.0/tests/models/unets/test_models_unet_2d_flax.py new file mode 100755 index 0000000..69a0704 --- /dev/null +++ b/diffusers-0.27.0/tests/models/unets/test_models_unet_2d_flax.py @@ -0,0 +1,104 @@ +import gc +import unittest + +from parameterized import parameterized + +from diffusers import FlaxUNet2DConditionModel +from diffusers.utils import is_flax_available +from diffusers.utils.testing_utils import load_hf_numpy, require_flax, slow + + +if is_flax_available(): + import jax + import jax.numpy as jnp + + +@slow +@require_flax +class FlaxUNet2DConditionModelIntegrationTests(unittest.TestCase): + def get_file_format(self, seed, shape): + return f"gaussian_noise_s={seed}_shape={'_'.join([str(s) for s in shape])}.npy" + + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + + def get_latents(self, seed=0, shape=(4, 4, 64, 64), fp16=False): + dtype = jnp.bfloat16 if fp16 else jnp.float32 + image = jnp.array(load_hf_numpy(self.get_file_format(seed, shape)), dtype=dtype) + return image + + def get_unet_model(self, fp16=False, model_id="CompVis/stable-diffusion-v1-4"): + dtype = jnp.bfloat16 if fp16 else jnp.float32 + revision = "bf16" if fp16 else None + + model, params = FlaxUNet2DConditionModel.from_pretrained( + model_id, subfolder="unet", dtype=dtype, revision=revision + ) + return model, params + + def get_encoder_hidden_states(self, seed=0, shape=(4, 77, 768), fp16=False): + dtype = jnp.bfloat16 if fp16 else jnp.float32 + hidden_states = jnp.array(load_hf_numpy(self.get_file_format(seed, shape)), dtype=dtype) + return hidden_states + + @parameterized.expand( + [ + # fmt: off + [83, 4, [-0.2323, -0.1304, 0.0813, -0.3093, -0.0919, -0.1571, -0.1125, -0.5806]], + [17, 0.55, [-0.0831, -0.2443, 0.0901, -0.0919, 0.3396, 0.0103, -0.3743, 0.0701]], + [8, 0.89, [-0.4863, 0.0859, 0.0875, -0.1658, 0.9199, -0.0114, 0.4839, 0.4639]], + [3, 1000, [-0.5649, 0.2402, -0.5518, 0.1248, 1.1328, -0.2443, -0.0325, -1.0078]], + # fmt: on + ] + ) + def test_compvis_sd_v1_4_flax_vs_torch_fp16(self, seed, timestep, expected_slice): + model, params = self.get_unet_model(model_id="CompVis/stable-diffusion-v1-4", fp16=True) + latents = self.get_latents(seed, fp16=True) + encoder_hidden_states = self.get_encoder_hidden_states(seed, fp16=True) + + sample = model.apply( + {"params": params}, + latents, + jnp.array(timestep, dtype=jnp.int32), + encoder_hidden_states=encoder_hidden_states, + ).sample + + assert sample.shape == latents.shape + + output_slice = jnp.asarray(jax.device_get((sample[-1, -2:, -2:, :2].flatten())), dtype=jnp.float32) + expected_output_slice = jnp.array(expected_slice, dtype=jnp.float32) + + # Found torch (float16) and flax (bfloat16) outputs to be within this tolerance, in the same hardware + assert jnp.allclose(output_slice, expected_output_slice, atol=1e-2) + + @parameterized.expand( + [ + # fmt: off + [83, 4, [0.1514, 0.0807, 0.1624, 0.1016, -0.1896, 0.0263, 0.0677, 0.2310]], + [17, 0.55, [0.1164, -0.0216, 0.0170, 0.1589, -0.3120, 0.1005, -0.0581, -0.1458]], + [8, 0.89, [-0.1758, -0.0169, 0.1004, -0.1411, 0.1312, 0.1103, -0.1996, 0.2139]], + [3, 1000, [0.1214, 0.0352, -0.0731, -0.1562, -0.0994, -0.0906, -0.2340, -0.0539]], + # fmt: on + ] + ) + def test_stabilityai_sd_v2_flax_vs_torch_fp16(self, seed, timestep, expected_slice): + model, params = self.get_unet_model(model_id="stabilityai/stable-diffusion-2", fp16=True) + latents = self.get_latents(seed, shape=(4, 4, 96, 96), fp16=True) + encoder_hidden_states = self.get_encoder_hidden_states(seed, shape=(4, 77, 1024), fp16=True) + + sample = model.apply( + {"params": params}, + latents, + jnp.array(timestep, dtype=jnp.int32), + encoder_hidden_states=encoder_hidden_states, + ).sample + + assert sample.shape == latents.shape + + output_slice = jnp.asarray(jax.device_get((sample[-1, -2:, -2:, :2].flatten())), dtype=jnp.float32) + expected_output_slice = jnp.array(expected_slice, dtype=jnp.float32) + + # Found torch (float16) and flax (bfloat16) outputs to be within this tolerance, on the same hardware + assert jnp.allclose(output_slice, expected_output_slice, atol=1e-2) diff --git a/diffusers-0.27.0/tests/models/unets/test_models_unet_3d_condition.py b/diffusers-0.27.0/tests/models/unets/test_models_unet_3d_condition.py new file mode 100755 index 0000000..adbf895 --- /dev/null +++ b/diffusers-0.27.0/tests/models/unets/test_models_unet_3d_condition.py @@ -0,0 +1,180 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch + +from diffusers.models import ModelMixin, UNet3DConditionModel +from diffusers.utils import logging +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import enable_full_determinism, floats_tensor, skip_mps, torch_device + +from ..test_modeling_common import ModelTesterMixin, UNetTesterMixin + + +enable_full_determinism() + +logger = logging.get_logger(__name__) + + +@skip_mps +class UNet3DConditionModelTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = UNet3DConditionModel + main_input_name = "sample" + + @property + def dummy_input(self): + batch_size = 4 + num_channels = 4 + num_frames = 4 + sizes = (32, 32) + + noise = floats_tensor((batch_size, num_channels, num_frames) + sizes).to(torch_device) + time_step = torch.tensor([10]).to(torch_device) + encoder_hidden_states = floats_tensor((batch_size, 4, 32)).to(torch_device) + + return {"sample": noise, "timestep": time_step, "encoder_hidden_states": encoder_hidden_states} + + @property + def input_shape(self): + return (4, 4, 32, 32) + + @property + def output_shape(self): + return (4, 4, 32, 32) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "block_out_channels": (32, 64), + "down_block_types": ( + "CrossAttnDownBlock3D", + "DownBlock3D", + ), + "up_block_types": ("UpBlock3D", "CrossAttnUpBlock3D"), + "cross_attention_dim": 32, + "attention_head_dim": 8, + "out_channels": 4, + "in_channels": 4, + "layers_per_block": 1, + "sample_size": 32, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_enable_works(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + + model.enable_xformers_memory_efficient_attention() + + assert ( + model.mid_block.attentions[0].transformer_blocks[0].attn1.processor.__class__.__name__ + == "XFormersAttnProcessor" + ), "xformers is not enabled" + + # Overriding to set `norm_num_groups` needs to be different for this model. + def test_forward_with_norm_groups(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["norm_num_groups"] = 32 + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.sample + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + # Overriding since the UNet3D outputs a different structure. + def test_determinism(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + # Warmup pass when using mps (see #372) + if torch_device == "mps" and isinstance(model, ModelMixin): + model(**self.dummy_input) + + first = model(**inputs_dict) + if isinstance(first, dict): + first = first.sample + + second = model(**inputs_dict) + if isinstance(second, dict): + second = second.sample + + out_1 = first.cpu().numpy() + out_2 = second.cpu().numpy() + out_1 = out_1[~np.isnan(out_1)] + out_2 = out_2[~np.isnan(out_2)] + max_diff = np.amax(np.abs(out_1 - out_2)) + self.assertLessEqual(max_diff, 1e-5) + + def test_model_attention_slicing(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["attention_head_dim"] = 8 + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + model.set_attention_slice("auto") + with torch.no_grad(): + output = model(**inputs_dict) + assert output is not None + + model.set_attention_slice("max") + with torch.no_grad(): + output = model(**inputs_dict) + assert output is not None + + model.set_attention_slice(2) + with torch.no_grad(): + output = model(**inputs_dict) + assert output is not None + + def test_feed_forward_chunking(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + init_dict["norm_num_groups"] = 32 + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict)[0] + + model.enable_forward_chunking() + with torch.no_grad(): + output_2 = model(**inputs_dict)[0] + + self.assertEqual(output.shape, output_2.shape, "Shape doesn't match") + assert np.abs(output.cpu() - output_2.cpu()).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/models/unets/test_models_unet_motion.py b/diffusers-0.27.0/tests/models/unets/test_models_unet_motion.py new file mode 100755 index 0000000..4f7aba0 --- /dev/null +++ b/diffusers-0.27.0/tests/models/unets/test_models_unet_motion.py @@ -0,0 +1,306 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import os +import tempfile +import unittest + +import numpy as np +import torch + +from diffusers import MotionAdapter, UNet2DConditionModel, UNetMotionModel +from diffusers.utils import logging +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + torch_device, +) + +from ..test_modeling_common import ModelTesterMixin, UNetTesterMixin + + +logger = logging.get_logger(__name__) + +enable_full_determinism() + + +class UNetMotionModelTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = UNetMotionModel + main_input_name = "sample" + + @property + def dummy_input(self): + batch_size = 4 + num_channels = 4 + num_frames = 8 + sizes = (32, 32) + + noise = floats_tensor((batch_size, num_channels, num_frames) + sizes).to(torch_device) + time_step = torch.tensor([10]).to(torch_device) + encoder_hidden_states = floats_tensor((batch_size, 4, 32)).to(torch_device) + + return {"sample": noise, "timestep": time_step, "encoder_hidden_states": encoder_hidden_states} + + @property + def input_shape(self): + return (4, 8, 32, 32) + + @property + def output_shape(self): + return (4, 8, 32, 32) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "block_out_channels": (32, 64), + "down_block_types": ("CrossAttnDownBlockMotion", "DownBlockMotion"), + "up_block_types": ("UpBlockMotion", "CrossAttnUpBlockMotion"), + "cross_attention_dim": 32, + "num_attention_heads": 4, + "out_channels": 4, + "in_channels": 4, + "layers_per_block": 1, + "sample_size": 32, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_from_unet2d(self): + torch.manual_seed(0) + unet2d = UNet2DConditionModel() + + torch.manual_seed(1) + model = self.model_class.from_unet2d(unet2d) + model_state_dict = model.state_dict() + + for param_name, param_value in unet2d.named_parameters(): + self.assertTrue(torch.equal(model_state_dict[param_name], param_value)) + + def test_freeze_unet2d(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.freeze_unet2d_params() + + for param_name, param_value in model.named_parameters(): + if "motion_modules" not in param_name: + self.assertFalse(param_value.requires_grad) + + else: + self.assertTrue(param_value.requires_grad) + + def test_loading_motion_adapter(self): + model = self.model_class() + adapter = MotionAdapter() + model.load_motion_modules(adapter) + + for idx, down_block in enumerate(model.down_blocks): + adapter_state_dict = adapter.down_blocks[idx].motion_modules.state_dict() + for param_name, param_value in down_block.motion_modules.named_parameters(): + self.assertTrue(torch.equal(adapter_state_dict[param_name], param_value)) + + for idx, up_block in enumerate(model.up_blocks): + adapter_state_dict = adapter.up_blocks[idx].motion_modules.state_dict() + for param_name, param_value in up_block.motion_modules.named_parameters(): + self.assertTrue(torch.equal(adapter_state_dict[param_name], param_value)) + + mid_block_adapter_state_dict = adapter.mid_block.motion_modules.state_dict() + for param_name, param_value in model.mid_block.motion_modules.named_parameters(): + self.assertTrue(torch.equal(mid_block_adapter_state_dict[param_name], param_value)) + + def test_saving_motion_modules(self): + torch.manual_seed(0) + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.to(torch_device) + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_motion_modules(tmpdirname) + self.assertTrue(os.path.isfile(os.path.join(tmpdirname, "diffusion_pytorch_model.safetensors"))) + + adapter_loaded = MotionAdapter.from_pretrained(tmpdirname) + + torch.manual_seed(0) + model_loaded = self.model_class(**init_dict) + model_loaded.load_motion_modules(adapter_loaded) + model_loaded.to(torch_device) + + with torch.no_grad(): + output = model(**inputs_dict)[0] + output_loaded = model_loaded(**inputs_dict)[0] + + max_diff = (output - output_loaded).abs().max().item() + self.assertLessEqual(max_diff, 1e-4, "Models give different forward passes") + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_enable_works(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + + model.enable_xformers_memory_efficient_attention() + + assert ( + model.mid_block.attentions[0].transformer_blocks[0].attn1.processor.__class__.__name__ + == "XFormersAttnProcessor" + ), "xformers is not enabled" + + def test_gradient_checkpointing_is_applied(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model_class_copy = copy.copy(self.model_class) + + modules_with_gc_enabled = {} + + # now monkey patch the following function: + # def _set_gradient_checkpointing(self, module, value=False): + # if hasattr(module, "gradient_checkpointing"): + # module.gradient_checkpointing = value + + def _set_gradient_checkpointing_new(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + modules_with_gc_enabled[module.__class__.__name__] = True + + model_class_copy._set_gradient_checkpointing = _set_gradient_checkpointing_new + + model = model_class_copy(**init_dict) + model.enable_gradient_checkpointing() + + EXPECTED_SET = { + "CrossAttnUpBlockMotion", + "CrossAttnDownBlockMotion", + "UNetMidBlockCrossAttnMotion", + "UpBlockMotion", + "Transformer2DModel", + "DownBlockMotion", + } + + assert set(modules_with_gc_enabled.keys()) == EXPECTED_SET + assert all(modules_with_gc_enabled.values()), "All modules should be enabled" + + def test_feed_forward_chunking(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + init_dict["norm_num_groups"] = 32 + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict)[0] + + model.enable_forward_chunking() + with torch.no_grad(): + output_2 = model(**inputs_dict)[0] + + self.assertEqual(output.shape, output_2.shape, "Shape doesn't match") + assert np.abs(output.cpu() - output_2.cpu()).max() < 1e-2 + + def test_pickle(self): + # enable deterministic behavior for gradient checkpointing + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.to(torch_device) + + with torch.no_grad(): + sample = model(**inputs_dict).sample + + sample_copy = copy.copy(sample) + + assert (sample - sample_copy).abs().max() < 1e-4 + + def test_from_save_pretrained(self, expected_max_diff=5e-5): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + torch.manual_seed(0) + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname, safe_serialization=False) + torch.manual_seed(0) + new_model = self.model_class.from_pretrained(tmpdirname) + new_model.to(torch_device) + + with torch.no_grad(): + image = model(**inputs_dict) + if isinstance(image, dict): + image = image.to_tuple()[0] + + new_image = new_model(**inputs_dict) + + if isinstance(new_image, dict): + new_image = new_image.to_tuple()[0] + + max_diff = (image - new_image).abs().max().item() + self.assertLessEqual(max_diff, expected_max_diff, "Models give different forward passes") + + def test_from_save_pretrained_variant(self, expected_max_diff=5e-5): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + torch.manual_seed(0) + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with tempfile.TemporaryDirectory() as tmpdirname: + model.save_pretrained(tmpdirname, variant="fp16", safe_serialization=False) + + torch.manual_seed(0) + new_model = self.model_class.from_pretrained(tmpdirname, variant="fp16") + # non-variant cannot be loaded + with self.assertRaises(OSError) as error_context: + self.model_class.from_pretrained(tmpdirname) + + # make sure that error message states what keys are missing + assert "Error no file named diffusion_pytorch_model.bin found in directory" in str(error_context.exception) + + new_model.to(torch_device) + + with torch.no_grad(): + image = model(**inputs_dict) + if isinstance(image, dict): + image = image.to_tuple()[0] + + new_image = new_model(**inputs_dict) + + if isinstance(new_image, dict): + new_image = new_image.to_tuple()[0] + + max_diff = (image - new_image).abs().max().item() + self.assertLessEqual(max_diff, expected_max_diff, "Models give different forward passes") + + def test_forward_with_norm_groups(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["norm_num_groups"] = 16 + init_dict["block_out_channels"] = (16, 32) + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.to_tuple()[0] + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") diff --git a/diffusers-0.27.0/tests/models/unets/test_models_unet_spatiotemporal.py b/diffusers-0.27.0/tests/models/unets/test_models_unet_spatiotemporal.py new file mode 100755 index 0000000..935aa7f --- /dev/null +++ b/diffusers-0.27.0/tests/models/unets/test_models_unet_spatiotemporal.py @@ -0,0 +1,289 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import unittest + +import torch + +from diffusers import UNetSpatioTemporalConditionModel +from diffusers.utils import logging +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + torch_all_close, + torch_device, +) + +from ..test_modeling_common import ModelTesterMixin, UNetTesterMixin + + +logger = logging.get_logger(__name__) + +enable_full_determinism() + + +class UNetSpatioTemporalConditionModelTests(ModelTesterMixin, UNetTesterMixin, unittest.TestCase): + model_class = UNetSpatioTemporalConditionModel + main_input_name = "sample" + + @property + def dummy_input(self): + batch_size = 2 + num_frames = 2 + num_channels = 4 + sizes = (32, 32) + + noise = floats_tensor((batch_size, num_frames, num_channels) + sizes).to(torch_device) + time_step = torch.tensor([10]).to(torch_device) + encoder_hidden_states = floats_tensor((batch_size, 1, 32)).to(torch_device) + + return { + "sample": noise, + "timestep": time_step, + "encoder_hidden_states": encoder_hidden_states, + "added_time_ids": self._get_add_time_ids(), + } + + @property + def input_shape(self): + return (2, 2, 4, 32, 32) + + @property + def output_shape(self): + return (4, 32, 32) + + @property + def fps(self): + return 6 + + @property + def motion_bucket_id(self): + return 127 + + @property + def noise_aug_strength(self): + return 0.02 + + @property + def addition_time_embed_dim(self): + return 32 + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "block_out_channels": (32, 64), + "down_block_types": ( + "CrossAttnDownBlockSpatioTemporal", + "DownBlockSpatioTemporal", + ), + "up_block_types": ( + "UpBlockSpatioTemporal", + "CrossAttnUpBlockSpatioTemporal", + ), + "cross_attention_dim": 32, + "num_attention_heads": 8, + "out_channels": 4, + "in_channels": 4, + "layers_per_block": 2, + "sample_size": 32, + "projection_class_embeddings_input_dim": self.addition_time_embed_dim * 3, + "addition_time_embed_dim": self.addition_time_embed_dim, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def _get_add_time_ids(self, do_classifier_free_guidance=True): + add_time_ids = [self.fps, self.motion_bucket_id, self.noise_aug_strength] + + passed_add_embed_dim = self.addition_time_embed_dim * len(add_time_ids) + expected_add_embed_dim = self.addition_time_embed_dim * 3 + + if expected_add_embed_dim != passed_add_embed_dim: + raise ValueError( + f"Model expects an added time embedding vector of length {expected_add_embed_dim}, but a vector of {passed_add_embed_dim} was created. The model has an incorrect config. Please check `unet.config.time_embedding_type` and `text_encoder_2.config.projection_dim`." + ) + + add_time_ids = torch.tensor([add_time_ids], device=torch_device) + add_time_ids = add_time_ids.repeat(1, 1) + if do_classifier_free_guidance: + add_time_ids = torch.cat([add_time_ids, add_time_ids]) + + return add_time_ids + + @unittest.skip("Number of Norm Groups is not configurable") + def test_forward_with_norm_groups(self): + pass + + @unittest.skip("Deprecated functionality") + def test_model_attention_slicing(self): + pass + + @unittest.skip("Not supported") + def test_model_with_use_linear_projection(self): + pass + + @unittest.skip("Not supported") + def test_model_with_simple_projection(self): + pass + + @unittest.skip("Not supported") + def test_model_with_class_embeddings_concat(self): + pass + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_enable_works(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + + model.enable_xformers_memory_efficient_attention() + + assert ( + model.mid_block.attentions[0].transformer_blocks[0].attn1.processor.__class__.__name__ + == "XFormersAttnProcessor" + ), "xformers is not enabled" + + @unittest.skipIf(torch_device == "mps", "Gradient checkpointing skipped on MPS") + def test_gradient_checkpointing(self): + # enable deterministic behavior for gradient checkpointing + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.model_class(**init_dict) + model.to(torch_device) + + assert not model.is_gradient_checkpointing and model.training + + out = model(**inputs_dict).sample + # run the backwards pass on the model. For backwards pass, for simplicity purpose, + # we won't calculate the loss and rather backprop on out.sum() + model.zero_grad() + + labels = torch.randn_like(out) + loss = (out - labels).mean() + loss.backward() + + # re-instantiate the model now enabling gradient checkpointing + model_2 = self.model_class(**init_dict) + # clone model + model_2.load_state_dict(model.state_dict()) + model_2.to(torch_device) + model_2.enable_gradient_checkpointing() + + assert model_2.is_gradient_checkpointing and model_2.training + + out_2 = model_2(**inputs_dict).sample + # run the backwards pass on the model. For backwards pass, for simplicity purpose, + # we won't calculate the loss and rather backprop on out.sum() + model_2.zero_grad() + loss_2 = (out_2 - labels).mean() + loss_2.backward() + + # compare the output and parameters gradients + self.assertTrue((loss - loss_2).abs() < 1e-5) + named_params = dict(model.named_parameters()) + named_params_2 = dict(model_2.named_parameters()) + for name, param in named_params.items(): + self.assertTrue(torch_all_close(param.grad.data, named_params_2[name].grad.data, atol=5e-5)) + + def test_model_with_num_attention_heads_tuple(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["num_attention_heads"] = (8, 16) + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.sample + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_model_with_cross_attention_dim_tuple(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["cross_attention_dim"] = (32, 32) + + model = self.model_class(**init_dict) + model.to(torch_device) + model.eval() + + with torch.no_grad(): + output = model(**inputs_dict) + + if isinstance(output, dict): + output = output.sample + + self.assertIsNotNone(output) + expected_shape = inputs_dict["sample"].shape + self.assertEqual(output.shape, expected_shape, "Input and output shapes do not match") + + def test_gradient_checkpointing_is_applied(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["num_attention_heads"] = (8, 16) + + model_class_copy = copy.copy(self.model_class) + + modules_with_gc_enabled = {} + + # now monkey patch the following function: + # def _set_gradient_checkpointing(self, module, value=False): + # if hasattr(module, "gradient_checkpointing"): + # module.gradient_checkpointing = value + + def _set_gradient_checkpointing_new(self, module, value=False): + if hasattr(module, "gradient_checkpointing"): + module.gradient_checkpointing = value + modules_with_gc_enabled[module.__class__.__name__] = True + + model_class_copy._set_gradient_checkpointing = _set_gradient_checkpointing_new + + model = model_class_copy(**init_dict) + model.enable_gradient_checkpointing() + + EXPECTED_SET = { + "TransformerSpatioTemporalModel", + "CrossAttnDownBlockSpatioTemporal", + "DownBlockSpatioTemporal", + "UpBlockSpatioTemporal", + "CrossAttnUpBlockSpatioTemporal", + "UNetMidBlockSpatioTemporal", + } + + assert set(modules_with_gc_enabled.keys()) == EXPECTED_SET + assert all(modules_with_gc_enabled.values()), "All modules should be enabled" + + def test_pickle(self): + # enable deterministic behavior for gradient checkpointing + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + + init_dict["num_attention_heads"] = (8, 16) + + model = self.model_class(**init_dict) + model.to(torch_device) + + with torch.no_grad(): + sample = model(**inputs_dict).sample + + sample_copy = copy.copy(sample) + + assert (sample - sample_copy).abs().max() < 1e-4 diff --git a/diffusers-0.27.0/tests/models/unets/test_models_unet_stable_cascade.py b/diffusers-0.27.0/tests/models/unets/test_models_unet_stable_cascade.py new file mode 100755 index 0000000..6b1f2fd --- /dev/null +++ b/diffusers-0.27.0/tests/models/unets/test_models_unet_stable_cascade.py @@ -0,0 +1,191 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import torch + +from diffusers import StableCascadeUNet +from diffusers.utils import logging +from diffusers.utils.testing_utils import ( + enable_full_determinism, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, +) +from diffusers.utils.torch_utils import randn_tensor + + +logger = logging.get_logger(__name__) + +enable_full_determinism() + + +@slow +class StableCascadeUNetModelSlowTests(unittest.TestCase): + def tearDown(self) -> None: + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_cascade_unet_prior_single_file_components(self): + single_file_url = "https://huggingface.co/stabilityai/stable-cascade/blob/main/stage_c_bf16.safetensors" + single_file_unet = StableCascadeUNet.from_single_file(single_file_url) + + single_file_unet_config = single_file_unet.config + del single_file_unet + gc.collect() + torch.cuda.empty_cache() + + unet = StableCascadeUNet.from_pretrained("stabilityai/stable-cascade-prior", subfolder="prior", variant="bf16") + unet_config = unet.config + del unet + gc.collect() + torch.cuda.empty_cache() + + PARAMS_TO_IGNORE = ["torch_dtype", "_name_or_path", "_use_default_values"] + for param_name, param_value in single_file_unet_config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + + assert unet_config[param_name] == param_value + + def test_stable_cascade_unet_decoder_single_file_components(self): + single_file_url = "https://huggingface.co/stabilityai/stable-cascade/blob/main/stage_b_bf16.safetensors" + single_file_unet = StableCascadeUNet.from_single_file(single_file_url) + + single_file_unet_config = single_file_unet.config + del single_file_unet + gc.collect() + torch.cuda.empty_cache() + + unet = StableCascadeUNet.from_pretrained("stabilityai/stable-cascade", subfolder="decoder", variant="bf16") + unet_config = unet.config + del unet + gc.collect() + torch.cuda.empty_cache() + + PARAMS_TO_IGNORE = ["torch_dtype", "_name_or_path", "_use_default_values"] + for param_name, param_value in single_file_unet_config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + + assert unet_config[param_name] == param_value + + def test_stable_cascade_unet_config_loading(self): + config = StableCascadeUNet.load_config( + pretrained_model_name_or_path="diffusers/stable-cascade-configs", subfolder="prior" + ) + single_file_url = "https://huggingface.co/stabilityai/stable-cascade/blob/main/stage_c_bf16.safetensors" + + single_file_unet = StableCascadeUNet.from_single_file(single_file_url, config=config) + single_file_unet_config = single_file_unet.config + del single_file_unet + gc.collect() + torch.cuda.empty_cache() + + PARAMS_TO_IGNORE = ["torch_dtype", "_name_or_path", "_use_default_values"] + for param_name, param_value in config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + + assert single_file_unet_config[param_name] == param_value + + @require_torch_gpu + def test_stable_cascade_unet_single_file_prior_forward_pass(self): + dtype = torch.bfloat16 + generator = torch.Generator("cpu") + + model_inputs = { + "sample": randn_tensor((1, 16, 24, 24), generator=generator.manual_seed(0)).to("cuda", dtype), + "timestep_ratio": torch.tensor([1]).to("cuda", dtype), + "clip_text_pooled": randn_tensor((1, 1, 1280), generator=generator.manual_seed(0)).to("cuda", dtype), + "clip_text": randn_tensor((1, 77, 1280), generator=generator.manual_seed(0)).to("cuda", dtype), + "clip_img": randn_tensor((1, 1, 768), generator=generator.manual_seed(0)).to("cuda", dtype), + "pixels": randn_tensor((1, 3, 8, 8), generator=generator.manual_seed(0)).to("cuda", dtype), + } + + unet = StableCascadeUNet.from_pretrained( + "stabilityai/stable-cascade-prior", + subfolder="prior", + revision="refs/pr/2", + variant="bf16", + torch_dtype=dtype, + ) + unet.to("cuda") + with torch.no_grad(): + prior_output = unet(**model_inputs).sample.float().cpu().numpy() + + # Remove UNet from GPU memory before loading the single file UNet model + del unet + gc.collect() + torch.cuda.empty_cache() + + single_file_url = "https://huggingface.co/stabilityai/stable-cascade/blob/main/stage_c_bf16.safetensors" + single_file_unet = StableCascadeUNet.from_single_file(single_file_url, torch_dtype=dtype) + single_file_unet.to("cuda") + with torch.no_grad(): + prior_single_file_output = single_file_unet(**model_inputs).sample.float().cpu().numpy() + + # Remove UNet from GPU memory before loading the single file UNet model + del single_file_unet + gc.collect() + torch.cuda.empty_cache() + + max_diff = numpy_cosine_similarity_distance(prior_output.flatten(), prior_single_file_output.flatten()) + assert max_diff < 8e-3 + + @require_torch_gpu + def test_stable_cascade_unet_single_file_decoder_forward_pass(self): + dtype = torch.float32 + generator = torch.Generator("cpu") + + model_inputs = { + "sample": randn_tensor((1, 4, 256, 256), generator=generator.manual_seed(0)).to("cuda", dtype), + "timestep_ratio": torch.tensor([1]).to("cuda", dtype), + "clip_text": randn_tensor((1, 77, 1280), generator=generator.manual_seed(0)).to("cuda", dtype), + "clip_text_pooled": randn_tensor((1, 1, 1280), generator=generator.manual_seed(0)).to("cuda", dtype), + "pixels": randn_tensor((1, 3, 8, 8), generator=generator.manual_seed(0)).to("cuda", dtype), + } + + unet = StableCascadeUNet.from_pretrained( + "stabilityai/stable-cascade", + subfolder="decoder", + revision="refs/pr/44", + torch_dtype=dtype, + ) + unet.to("cuda") + with torch.no_grad(): + prior_output = unet(**model_inputs).sample.float().cpu().numpy() + + # Remove UNet from GPU memory before loading the single file UNet model + del unet + gc.collect() + torch.cuda.empty_cache() + + single_file_url = "https://huggingface.co/stabilityai/stable-cascade/blob/main/stage_b.safetensors" + single_file_unet = StableCascadeUNet.from_single_file(single_file_url, torch_dtype=dtype) + single_file_unet.to("cuda") + with torch.no_grad(): + prior_single_file_output = single_file_unet(**model_inputs).sample.float().cpu().numpy() + + # Remove UNet from GPU memory before loading the single file UNet model + del single_file_unet + gc.collect() + torch.cuda.empty_cache() + + max_diff = numpy_cosine_similarity_distance(prior_output.flatten(), prior_single_file_output.flatten()) + assert max_diff < 1e-4 diff --git a/diffusers-0.27.0/tests/models/unets/test_unet_2d_blocks.py b/diffusers-0.27.0/tests/models/unets/test_unet_2d_blocks.py new file mode 100755 index 0000000..e371991 --- /dev/null +++ b/diffusers-0.27.0/tests/models/unets/test_unet_2d_blocks.py @@ -0,0 +1,337 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +from diffusers.models.unets.unet_2d_blocks import * # noqa F403 +from diffusers.utils.testing_utils import torch_device + +from .test_unet_blocks_common import UNetBlockTesterMixin + + +class DownBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = DownBlock2D # noqa F405 + block_type = "down" + + def test_output(self): + expected_slice = [-0.0232, -0.9869, 0.8054, -0.0637, -0.1688, -1.4264, 0.4470, -1.3394, 0.0904] + super().test_output(expected_slice) + + +class ResnetDownsampleBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = ResnetDownsampleBlock2D # noqa F405 + block_type = "down" + + def test_output(self): + expected_slice = [0.0710, 0.2410, -0.7320, -1.0757, -1.1343, 0.3540, -0.0133, -0.2576, 0.0948] + super().test_output(expected_slice) + + +class AttnDownBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = AttnDownBlock2D # noqa F405 + block_type = "down" + + def test_output(self): + expected_slice = [0.0636, 0.8964, -0.6234, -1.0131, 0.0844, 0.4935, 0.3437, 0.0911, -0.2957] + super().test_output(expected_slice) + + +class CrossAttnDownBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = CrossAttnDownBlock2D # noqa F405 + block_type = "down" + + def prepare_init_args_and_inputs_for_common(self): + init_dict, inputs_dict = super().prepare_init_args_and_inputs_for_common() + init_dict["cross_attention_dim"] = 32 + return init_dict, inputs_dict + + def test_output(self): + expected_slice = [0.2238, -0.7396, -0.2255, -0.3829, 0.1925, 1.1665, 0.0603, -0.7295, 0.1983] + super().test_output(expected_slice) + + +class SimpleCrossAttnDownBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = SimpleCrossAttnDownBlock2D # noqa F405 + block_type = "down" + + @property + def dummy_input(self): + return super().get_dummy_input(include_encoder_hidden_states=True) + + def prepare_init_args_and_inputs_for_common(self): + init_dict, inputs_dict = super().prepare_init_args_and_inputs_for_common() + init_dict["cross_attention_dim"] = 32 + return init_dict, inputs_dict + + @unittest.skipIf(torch_device == "mps", "MPS result is not consistent") + def test_output(self): + expected_slice = [0.7921, -0.0992, -0.1962, -0.7695, -0.4242, 0.7804, 0.4737, 0.2765, 0.3338] + super().test_output(expected_slice) + + +class SkipDownBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = SkipDownBlock2D # noqa F405 + block_type = "down" + + @property + def dummy_input(self): + return super().get_dummy_input(include_skip_sample=True) + + def test_output(self): + expected_slice = [-0.0845, -0.2087, -0.2465, 0.0971, 0.1900, -0.0484, 0.2664, 0.4179, 0.5069] + super().test_output(expected_slice) + + +class AttnSkipDownBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = AttnSkipDownBlock2D # noqa F405 + block_type = "down" + + @property + def dummy_input(self): + return super().get_dummy_input(include_skip_sample=True) + + def test_output(self): + expected_slice = [0.5539, 0.1609, 0.4924, 0.0537, -0.1995, 0.4050, 0.0979, -0.2721, -0.0642] + super().test_output(expected_slice) + + +class DownEncoderBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = DownEncoderBlock2D # noqa F405 + block_type = "down" + + @property + def dummy_input(self): + return super().get_dummy_input(include_temb=False) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "in_channels": 32, + "out_channels": 32, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_output(self): + expected_slice = [1.1102, 0.5302, 0.4872, -0.0023, -0.8042, 0.0483, -0.3489, -0.5632, 0.7626] + super().test_output(expected_slice) + + +class AttnDownEncoderBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = AttnDownEncoderBlock2D # noqa F405 + block_type = "down" + + @property + def dummy_input(self): + return super().get_dummy_input(include_temb=False) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "in_channels": 32, + "out_channels": 32, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_output(self): + expected_slice = [0.8966, -0.1486, 0.8568, 0.8141, -0.9046, -0.1342, -0.0972, -0.7417, 0.1538] + super().test_output(expected_slice) + + +class UNetMidBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = UNetMidBlock2D # noqa F405 + block_type = "mid" + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "in_channels": 32, + "temb_channels": 128, + } + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_output(self): + expected_slice = [-0.1062, 1.7248, 0.3494, 1.4569, -0.0910, -1.2421, -0.9984, 0.6736, 1.0028] + super().test_output(expected_slice) + + +class UNetMidBlock2DCrossAttnTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = UNetMidBlock2DCrossAttn # noqa F405 + block_type = "mid" + + def prepare_init_args_and_inputs_for_common(self): + init_dict, inputs_dict = super().prepare_init_args_and_inputs_for_common() + init_dict["cross_attention_dim"] = 32 + return init_dict, inputs_dict + + def test_output(self): + expected_slice = [0.0187, 2.4220, 0.4484, 1.1203, -0.6121, -1.5122, -0.8270, 0.7851, 1.8335] + super().test_output(expected_slice) + + +class UNetMidBlock2DSimpleCrossAttnTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = UNetMidBlock2DSimpleCrossAttn # noqa F405 + block_type = "mid" + + @property + def dummy_input(self): + return super().get_dummy_input(include_encoder_hidden_states=True) + + def prepare_init_args_and_inputs_for_common(self): + init_dict, inputs_dict = super().prepare_init_args_and_inputs_for_common() + init_dict["cross_attention_dim"] = 32 + return init_dict, inputs_dict + + def test_output(self): + expected_slice = [0.7143, 1.9974, 0.5448, 1.3977, 0.1282, -1.1237, -1.4238, 0.5530, 0.8880] + super().test_output(expected_slice) + + +class UpBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = UpBlock2D # noqa F405 + block_type = "up" + + @property + def dummy_input(self): + return super().get_dummy_input(include_res_hidden_states_tuple=True) + + def test_output(self): + expected_slice = [-0.2041, -0.4165, -0.3022, 0.0041, -0.6628, -0.7053, 0.1928, -0.0325, 0.0523] + super().test_output(expected_slice) + + +class ResnetUpsampleBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = ResnetUpsampleBlock2D # noqa F405 + block_type = "up" + + @property + def dummy_input(self): + return super().get_dummy_input(include_res_hidden_states_tuple=True) + + def test_output(self): + expected_slice = [0.2287, 0.3549, -0.1346, 0.4797, -0.1715, -0.9649, 0.7305, -0.5864, -0.6244] + super().test_output(expected_slice) + + +class CrossAttnUpBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = CrossAttnUpBlock2D # noqa F405 + block_type = "up" + + @property + def dummy_input(self): + return super().get_dummy_input(include_res_hidden_states_tuple=True) + + def prepare_init_args_and_inputs_for_common(self): + init_dict, inputs_dict = super().prepare_init_args_and_inputs_for_common() + init_dict["cross_attention_dim"] = 32 + return init_dict, inputs_dict + + def test_output(self): + expected_slice = [-0.1403, -0.3515, -0.0420, -0.1425, 0.3167, 0.5094, -0.2181, 0.5931, 0.5582] + super().test_output(expected_slice) + + +class SimpleCrossAttnUpBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = SimpleCrossAttnUpBlock2D # noqa F405 + block_type = "up" + + @property + def dummy_input(self): + return super().get_dummy_input(include_res_hidden_states_tuple=True, include_encoder_hidden_states=True) + + def prepare_init_args_and_inputs_for_common(self): + init_dict, inputs_dict = super().prepare_init_args_and_inputs_for_common() + init_dict["cross_attention_dim"] = 32 + return init_dict, inputs_dict + + def test_output(self): + expected_slice = [0.2645, 0.1480, 0.0909, 0.8044, -0.9758, -0.9083, 0.0994, -1.1453, -0.7402] + super().test_output(expected_slice) + + +class AttnUpBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = AttnUpBlock2D # noqa F405 + block_type = "up" + + @property + def dummy_input(self): + return super().get_dummy_input(include_res_hidden_states_tuple=True) + + @unittest.skipIf(torch_device == "mps", "MPS result is not consistent") + def test_output(self): + expected_slice = [0.0979, 0.1326, 0.0021, 0.0659, 0.2249, 0.0059, 0.1132, 0.5952, 0.1033] + super().test_output(expected_slice) + + +class SkipUpBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = SkipUpBlock2D # noqa F405 + block_type = "up" + + @property + def dummy_input(self): + return super().get_dummy_input(include_res_hidden_states_tuple=True) + + def test_output(self): + expected_slice = [-0.0893, -0.1234, -0.1506, -0.0332, 0.0123, -0.0211, 0.0566, 0.0143, 0.0362] + super().test_output(expected_slice) + + +class AttnSkipUpBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = AttnSkipUpBlock2D # noqa F405 + block_type = "up" + + @property + def dummy_input(self): + return super().get_dummy_input(include_res_hidden_states_tuple=True) + + def test_output(self): + expected_slice = [0.0361, 0.0617, 0.2787, -0.0350, 0.0342, 0.3421, -0.0843, 0.0913, 0.3015] + super().test_output(expected_slice) + + +class UpDecoderBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = UpDecoderBlock2D # noqa F405 + block_type = "up" + + @property + def dummy_input(self): + return super().get_dummy_input(include_temb=False) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = {"in_channels": 32, "out_channels": 32} + + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_output(self): + expected_slice = [0.4404, 0.1998, -0.9886, -0.3320, -0.3128, -0.7034, -0.6955, -0.2338, -0.3137] + super().test_output(expected_slice) + + +class AttnUpDecoderBlock2DTests(UNetBlockTesterMixin, unittest.TestCase): + block_class = AttnUpDecoderBlock2D # noqa F405 + block_type = "up" + + @property + def dummy_input(self): + return super().get_dummy_input(include_temb=False) + + def prepare_init_args_and_inputs_for_common(self): + init_dict = {"in_channels": 32, "out_channels": 32} + + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_output(self): + expected_slice = [0.6738, 0.4491, 0.1055, 1.0710, 0.7316, 0.3339, 0.3352, 0.1023, 0.3568] + super().test_output(expected_slice) diff --git a/diffusers-0.27.0/tests/models/unets/test_unet_blocks_common.py b/diffusers-0.27.0/tests/models/unets/test_unet_blocks_common.py new file mode 100755 index 0000000..dce28c7 --- /dev/null +++ b/diffusers-0.27.0/tests/models/unets/test_unet_blocks_common.py @@ -0,0 +1,126 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from typing import Tuple + +import torch + +from diffusers.utils.testing_utils import ( + floats_tensor, + require_torch, + require_torch_accelerator_with_training, + torch_all_close, + torch_device, +) +from diffusers.utils.torch_utils import randn_tensor + + +@require_torch +class UNetBlockTesterMixin: + @property + def dummy_input(self): + return self.get_dummy_input() + + @property + def output_shape(self): + if self.block_type == "down": + return (4, 32, 16, 16) + elif self.block_type == "mid": + return (4, 32, 32, 32) + elif self.block_type == "up": + return (4, 32, 64, 64) + + raise ValueError(f"'{self.block_type}' is not a supported block_type. Set it to 'up', 'mid', or 'down'.") + + def get_dummy_input( + self, + include_temb=True, + include_res_hidden_states_tuple=False, + include_encoder_hidden_states=False, + include_skip_sample=False, + ): + batch_size = 4 + num_channels = 32 + sizes = (32, 32) + + generator = torch.manual_seed(0) + device = torch.device(torch_device) + shape = (batch_size, num_channels) + sizes + hidden_states = randn_tensor(shape, generator=generator, device=device) + dummy_input = {"hidden_states": hidden_states} + + if include_temb: + temb_channels = 128 + dummy_input["temb"] = randn_tensor((batch_size, temb_channels), generator=generator, device=device) + + if include_res_hidden_states_tuple: + generator_1 = torch.manual_seed(1) + dummy_input["res_hidden_states_tuple"] = (randn_tensor(shape, generator=generator_1, device=device),) + + if include_encoder_hidden_states: + dummy_input["encoder_hidden_states"] = floats_tensor((batch_size, 32, 32)).to(torch_device) + + if include_skip_sample: + dummy_input["skip_sample"] = randn_tensor(((batch_size, 3) + sizes), generator=generator, device=device) + + return dummy_input + + def prepare_init_args_and_inputs_for_common(self): + init_dict = { + "in_channels": 32, + "out_channels": 32, + "temb_channels": 128, + } + if self.block_type == "up": + init_dict["prev_output_channel"] = 32 + + if self.block_type == "mid": + init_dict.pop("out_channels") + + inputs_dict = self.dummy_input + return init_dict, inputs_dict + + def test_output(self, expected_slice): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + unet_block = self.block_class(**init_dict) + unet_block.to(torch_device) + unet_block.eval() + + with torch.no_grad(): + output = unet_block(**inputs_dict) + + if isinstance(output, Tuple): + output = output[0] + + self.assertEqual(output.shape, self.output_shape) + + output_slice = output[0, -1, -3:, -3:] + expected_slice = torch.tensor(expected_slice).to(torch_device) + assert torch_all_close(output_slice.flatten(), expected_slice, atol=5e-3) + + @require_torch_accelerator_with_training + def test_training(self): + init_dict, inputs_dict = self.prepare_init_args_and_inputs_for_common() + model = self.block_class(**init_dict) + model.to(torch_device) + model.train() + output = model(**inputs_dict) + + if isinstance(output, Tuple): + output = output[0] + + device = torch.device(torch_device) + noise = randn_tensor(output.shape, device=device) + loss = torch.nn.functional.mse_loss(output, noise) + loss.backward() diff --git a/diffusers-0.27.0/tests/others/test_check_copies.py b/diffusers-0.27.0/tests/others/test_check_copies.py new file mode 100755 index 0000000..6e1c8fc --- /dev/null +++ b/diffusers-0.27.0/tests/others/test_check_copies.py @@ -0,0 +1,117 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import re +import shutil +import sys +import tempfile +import unittest + + +git_repo_path = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +sys.path.append(os.path.join(git_repo_path, "utils")) + +import check_copies # noqa: E402 + + +# This is the reference code that will be used in the tests. +# If DDPMSchedulerOutput is changed in scheduling_ddpm.py, this code needs to be manually updated. +REFERENCE_CODE = """ \""" + Output class for the scheduler's `step` function output. + + Args: + prev_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + Computed sample `(x_{t-1})` of previous timestep. `prev_sample` should be used as next model input in the + denoising loop. + pred_original_sample (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)` for images): + The predicted denoised sample `(x_{0})` based on the model output from the current timestep. + `pred_original_sample` can be used to preview progress or for guidance. + \""" + + prev_sample: torch.FloatTensor + pred_original_sample: Optional[torch.FloatTensor] = None +""" + + +class CopyCheckTester(unittest.TestCase): + def setUp(self): + self.diffusers_dir = tempfile.mkdtemp() + os.makedirs(os.path.join(self.diffusers_dir, "schedulers/")) + check_copies.DIFFUSERS_PATH = self.diffusers_dir + shutil.copy( + os.path.join(git_repo_path, "src/diffusers/schedulers/scheduling_ddpm.py"), + os.path.join(self.diffusers_dir, "schedulers/scheduling_ddpm.py"), + ) + + def tearDown(self): + check_copies.DIFFUSERS_PATH = "src/diffusers" + shutil.rmtree(self.diffusers_dir) + + def check_copy_consistency(self, comment, class_name, class_code, overwrite_result=None): + code = comment + f"\nclass {class_name}(nn.Module):\n" + class_code + if overwrite_result is not None: + expected = comment + f"\nclass {class_name}(nn.Module):\n" + overwrite_result + code = check_copies.run_ruff(code) + fname = os.path.join(self.diffusers_dir, "new_code.py") + with open(fname, "w", newline="\n") as f: + f.write(code) + if overwrite_result is None: + self.assertTrue(len(check_copies.is_copy_consistent(fname)) == 0) + else: + check_copies.is_copy_consistent(f.name, overwrite=True) + with open(fname, "r") as f: + self.assertTrue(f.read(), expected) + + def test_find_code_in_diffusers(self): + code = check_copies.find_code_in_diffusers("schedulers.scheduling_ddpm.DDPMSchedulerOutput") + self.assertEqual(code, REFERENCE_CODE) + + def test_is_copy_consistent(self): + # Base copy consistency + self.check_copy_consistency( + "# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput", + "DDPMSchedulerOutput", + REFERENCE_CODE + "\n", + ) + + # With no empty line at the end + self.check_copy_consistency( + "# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput", + "DDPMSchedulerOutput", + REFERENCE_CODE, + ) + + # Copy consistency with rename + self.check_copy_consistency( + "# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->Test", + "TestSchedulerOutput", + re.sub("DDPM", "Test", REFERENCE_CODE), + ) + + # Copy consistency with a really long name + long_class_name = "TestClassWithAReallyLongNameBecauseSomePeopleLikeThatForSomeReason" + self.check_copy_consistency( + f"# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->{long_class_name}", + f"{long_class_name}SchedulerOutput", + re.sub("Bert", long_class_name, REFERENCE_CODE), + ) + + # Copy consistency with overwrite + self.check_copy_consistency( + "# Copied from diffusers.schedulers.scheduling_ddpm.DDPMSchedulerOutput with DDPM->Test", + "TestSchedulerOutput", + REFERENCE_CODE, + overwrite_result=re.sub("DDPM", "Test", REFERENCE_CODE), + ) diff --git a/diffusers-0.27.0/tests/others/test_check_dummies.py b/diffusers-0.27.0/tests/others/test_check_dummies.py new file mode 100755 index 0000000..1890ffa --- /dev/null +++ b/diffusers-0.27.0/tests/others/test_check_dummies.py @@ -0,0 +1,122 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import unittest + + +git_repo_path = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +sys.path.append(os.path.join(git_repo_path, "utils")) + +import check_dummies # noqa: E402 +from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init # noqa: E402 + + +# Align TRANSFORMERS_PATH in check_dummies with the current path +check_dummies.PATH_TO_DIFFUSERS = os.path.join(git_repo_path, "src", "diffusers") + + +class CheckDummiesTester(unittest.TestCase): + def test_find_backend(self): + simple_backend = find_backend(" if not is_torch_available():") + self.assertEqual(simple_backend, "torch") + + # backend_with_underscore = find_backend(" if not is_tensorflow_text_available():") + # self.assertEqual(backend_with_underscore, "tensorflow_text") + + double_backend = find_backend(" if not (is_torch_available() and is_transformers_available()):") + self.assertEqual(double_backend, "torch_and_transformers") + + # double_backend_with_underscore = find_backend( + # " if not (is_sentencepiece_available() and is_tensorflow_text_available()):" + # ) + # self.assertEqual(double_backend_with_underscore, "sentencepiece_and_tensorflow_text") + + triple_backend = find_backend( + " if not (is_torch_available() and is_transformers_available() and is_onnx_available()):" + ) + self.assertEqual(triple_backend, "torch_and_transformers_and_onnx") + + def test_read_init(self): + objects = read_init() + # We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects + self.assertIn("torch", objects) + self.assertIn("torch_and_transformers", objects) + self.assertIn("flax_and_transformers", objects) + self.assertIn("torch_and_transformers_and_onnx", objects) + + # Likewise, we can't assert on the exact content of a key + self.assertIn("UNet2DModel", objects["torch"]) + self.assertIn("FlaxUNet2DConditionModel", objects["flax"]) + self.assertIn("StableDiffusionPipeline", objects["torch_and_transformers"]) + self.assertIn("FlaxStableDiffusionPipeline", objects["flax_and_transformers"]) + self.assertIn("LMSDiscreteScheduler", objects["torch_and_scipy"]) + self.assertIn("OnnxStableDiffusionPipeline", objects["torch_and_transformers_and_onnx"]) + + def test_create_dummy_object(self): + dummy_constant = create_dummy_object("CONSTANT", "'torch'") + self.assertEqual(dummy_constant, "\nCONSTANT = None\n") + + dummy_function = create_dummy_object("function", "'torch'") + self.assertEqual( + dummy_function, "\ndef function(*args, **kwargs):\n requires_backends(function, 'torch')\n" + ) + + expected_dummy_class = """ +class FakeClass(metaclass=DummyObject): + _backends = 'torch' + + def __init__(self, *args, **kwargs): + requires_backends(self, 'torch') + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, 'torch') + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, 'torch') +""" + dummy_class = create_dummy_object("FakeClass", "'torch'") + self.assertEqual(dummy_class, expected_dummy_class) + + def test_create_dummy_files(self): + expected_dummy_pytorch_file = """# This file is autogenerated by the command `make fix-copies`, do not edit. +from ..utils import DummyObject, requires_backends + + +CONSTANT = None + + +def function(*args, **kwargs): + requires_backends(function, ["torch"]) + + +class FakeClass(metaclass=DummyObject): + _backends = ["torch"] + + def __init__(self, *args, **kwargs): + requires_backends(self, ["torch"]) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, ["torch"]) +""" + dummy_files = create_dummy_files({"torch": ["CONSTANT", "function", "FakeClass"]}) + self.assertEqual(dummy_files["torch"], expected_dummy_pytorch_file) diff --git a/diffusers-0.27.0/tests/others/test_config.py b/diffusers-0.27.0/tests/others/test_config.py new file mode 100755 index 0000000..3492ec3 --- /dev/null +++ b/diffusers-0.27.0/tests/others/test_config.py @@ -0,0 +1,288 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import unittest + +from diffusers import ( + DDIMScheduler, + DDPMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + PNDMScheduler, + logging, +) +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.utils.testing_utils import CaptureLogger + + +class SampleObject(ConfigMixin): + config_name = "config.json" + + @register_to_config + def __init__( + self, + a=2, + b=5, + c=(2, 5), + d="for diffusion", + e=[1, 3], + ): + pass + + +class SampleObject2(ConfigMixin): + config_name = "config.json" + + @register_to_config + def __init__( + self, + a=2, + b=5, + c=(2, 5), + d="for diffusion", + f=[1, 3], + ): + pass + + +class SampleObject3(ConfigMixin): + config_name = "config.json" + + @register_to_config + def __init__( + self, + a=2, + b=5, + c=(2, 5), + d="for diffusion", + e=[1, 3], + f=[1, 3], + ): + pass + + +class SampleObject4(ConfigMixin): + config_name = "config.json" + + @register_to_config + def __init__( + self, + a=2, + b=5, + c=(2, 5), + d="for diffusion", + e=[1, 5], + f=[5, 4], + ): + pass + + +class ConfigTester(unittest.TestCase): + def test_load_not_from_mixin(self): + with self.assertRaises(ValueError): + ConfigMixin.load_config("dummy_path") + + def test_register_to_config(self): + obj = SampleObject() + config = obj.config + assert config["a"] == 2 + assert config["b"] == 5 + assert config["c"] == (2, 5) + assert config["d"] == "for diffusion" + assert config["e"] == [1, 3] + + # init ignore private arguments + obj = SampleObject(_name_or_path="lalala") + config = obj.config + assert config["a"] == 2 + assert config["b"] == 5 + assert config["c"] == (2, 5) + assert config["d"] == "for diffusion" + assert config["e"] == [1, 3] + + # can override default + obj = SampleObject(c=6) + config = obj.config + assert config["a"] == 2 + assert config["b"] == 5 + assert config["c"] == 6 + assert config["d"] == "for diffusion" + assert config["e"] == [1, 3] + + # can use positional arguments. + obj = SampleObject(1, c=6) + config = obj.config + assert config["a"] == 1 + assert config["b"] == 5 + assert config["c"] == 6 + assert config["d"] == "for diffusion" + assert config["e"] == [1, 3] + + def test_save_load(self): + obj = SampleObject() + config = obj.config + + assert config["a"] == 2 + assert config["b"] == 5 + assert config["c"] == (2, 5) + assert config["d"] == "for diffusion" + assert config["e"] == [1, 3] + + with tempfile.TemporaryDirectory() as tmpdirname: + obj.save_config(tmpdirname) + new_obj = SampleObject.from_config(SampleObject.load_config(tmpdirname)) + new_config = new_obj.config + + # unfreeze configs + config = dict(config) + new_config = dict(new_config) + + assert config.pop("c") == (2, 5) # instantiated as tuple + assert new_config.pop("c") == [2, 5] # saved & loaded as list because of json + config.pop("_use_default_values") + assert config == new_config + + def test_load_ddim_from_pndm(self): + logger = logging.get_logger("diffusers.configuration_utils") + # 30 for warning + logger.setLevel(30) + + with CaptureLogger(logger) as cap_logger: + ddim = DDIMScheduler.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler" + ) + + assert ddim.__class__ == DDIMScheduler + # no warning should be thrown + assert cap_logger.out == "" + + def test_load_euler_from_pndm(self): + logger = logging.get_logger("diffusers.configuration_utils") + # 30 for warning + logger.setLevel(30) + + with CaptureLogger(logger) as cap_logger: + euler = EulerDiscreteScheduler.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler" + ) + + assert euler.__class__ == EulerDiscreteScheduler + # no warning should be thrown + assert cap_logger.out == "" + + def test_load_euler_ancestral_from_pndm(self): + logger = logging.get_logger("diffusers.configuration_utils") + # 30 for warning + logger.setLevel(30) + + with CaptureLogger(logger) as cap_logger: + euler = EulerAncestralDiscreteScheduler.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler" + ) + + assert euler.__class__ == EulerAncestralDiscreteScheduler + # no warning should be thrown + assert cap_logger.out == "" + + def test_load_pndm(self): + logger = logging.get_logger("diffusers.configuration_utils") + # 30 for warning + logger.setLevel(30) + + with CaptureLogger(logger) as cap_logger: + pndm = PNDMScheduler.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler" + ) + + assert pndm.__class__ == PNDMScheduler + # no warning should be thrown + assert cap_logger.out == "" + + def test_overwrite_config_on_load(self): + logger = logging.get_logger("diffusers.configuration_utils") + # 30 for warning + logger.setLevel(30) + + with CaptureLogger(logger) as cap_logger: + ddpm = DDPMScheduler.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", + subfolder="scheduler", + prediction_type="sample", + beta_end=8, + ) + + with CaptureLogger(logger) as cap_logger_2: + ddpm_2 = DDPMScheduler.from_pretrained("google/ddpm-celebahq-256", beta_start=88) + + assert ddpm.__class__ == DDPMScheduler + assert ddpm.config.prediction_type == "sample" + assert ddpm.config.beta_end == 8 + assert ddpm_2.config.beta_start == 88 + + # no warning should be thrown + assert cap_logger.out == "" + assert cap_logger_2.out == "" + + def test_load_dpmsolver(self): + logger = logging.get_logger("diffusers.configuration_utils") + # 30 for warning + logger.setLevel(30) + + with CaptureLogger(logger) as cap_logger: + dpm = DPMSolverMultistepScheduler.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler" + ) + + assert dpm.__class__ == DPMSolverMultistepScheduler + # no warning should be thrown + assert cap_logger.out == "" + + def test_use_default_values(self): + # let's first save a config that should be in the form + # a=2, + # b=5, + # c=(2, 5), + # d="for diffusion", + # e=[1, 3], + + config = SampleObject() + + config_dict = {k: v for k, v in config.config.items() if not k.startswith("_")} + + # make sure that default config has all keys in `_use_default_values` + assert set(config_dict.keys()) == set(config.config._use_default_values) + + with tempfile.TemporaryDirectory() as tmpdirname: + config.save_config(tmpdirname) + + # now loading it with SampleObject2 should put f into `_use_default_values` + config = SampleObject2.from_config(tmpdirname) + + assert "f" in config._use_default_values + assert config.f == [1, 3] + + # now loading the config, should **NOT** use [1, 3] for `f`, but the default [1, 4] value + # **BECAUSE** it is part of `config._use_default_values` + new_config = SampleObject4.from_config(config.config) + assert new_config.f == [5, 4] + + config.config._use_default_values.pop() + new_config_2 = SampleObject4.from_config(config.config) + assert new_config_2.f == [1, 3] + + # Nevertheless "e" should still be correctly loaded to [1, 3] from SampleObject2 instead of defaulting to [1, 5] + assert new_config_2.e == [1, 3] diff --git a/diffusers-0.27.0/tests/others/test_dependencies.py b/diffusers-0.27.0/tests/others/test_dependencies.py new file mode 100755 index 0000000..c0839ef --- /dev/null +++ b/diffusers-0.27.0/tests/others/test_dependencies.py @@ -0,0 +1,50 @@ +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import unittest +from importlib import import_module + + +class DependencyTester(unittest.TestCase): + def test_diffusers_import(self): + try: + import diffusers # noqa: F401 + except ImportError: + assert False + + def test_backend_registration(self): + import diffusers + from diffusers.dependency_versions_table import deps + + all_classes = inspect.getmembers(diffusers, inspect.isclass) + + for cls_name, cls_module in all_classes: + if "dummy_" in cls_module.__module__: + for backend in cls_module._backends: + if backend == "k_diffusion": + backend = "k-diffusion" + elif backend == "invisible_watermark": + backend = "invisible-watermark" + assert backend in deps, f"{backend} is not in the deps table!" + + def test_pipeline_imports(self): + import diffusers + import diffusers.pipelines + + all_classes = inspect.getmembers(diffusers, inspect.isclass) + for cls_name, cls_module in all_classes: + if hasattr(diffusers.pipelines, cls_name): + pipeline_folder_module = ".".join(str(cls_module.__module__).split(".")[:3]) + _ = import_module(pipeline_folder_module, str(cls_name)) diff --git a/diffusers-0.27.0/tests/others/test_ema.py b/diffusers-0.27.0/tests/others/test_ema.py new file mode 100755 index 0000000..48437c5 --- /dev/null +++ b/diffusers-0.27.0/tests/others/test_ema.py @@ -0,0 +1,159 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import unittest + +import torch + +from diffusers import UNet2DConditionModel +from diffusers.training_utils import EMAModel +from diffusers.utils.testing_utils import enable_full_determinism, skip_mps, torch_device + + +enable_full_determinism() + + +class EMAModelTests(unittest.TestCase): + model_id = "hf-internal-testing/tiny-stable-diffusion-pipe" + batch_size = 1 + prompt_length = 77 + text_encoder_hidden_dim = 32 + num_in_channels = 4 + latent_height = latent_width = 64 + generator = torch.manual_seed(0) + + def get_models(self, decay=0.9999): + unet = UNet2DConditionModel.from_pretrained(self.model_id, subfolder="unet") + unet = unet.to(torch_device) + ema_unet = EMAModel(unet.parameters(), decay=decay, model_cls=UNet2DConditionModel, model_config=unet.config) + return unet, ema_unet + + def get_dummy_inputs(self): + noisy_latents = torch.randn( + self.batch_size, self.num_in_channels, self.latent_height, self.latent_width, generator=self.generator + ).to(torch_device) + timesteps = torch.randint(0, 1000, size=(self.batch_size,), generator=self.generator).to(torch_device) + encoder_hidden_states = torch.randn( + self.batch_size, self.prompt_length, self.text_encoder_hidden_dim, generator=self.generator + ).to(torch_device) + return noisy_latents, timesteps, encoder_hidden_states + + def simulate_backprop(self, unet): + updated_state_dict = {} + for k, param in unet.state_dict().items(): + updated_param = torch.randn_like(param) + (param * torch.randn_like(param)) + updated_state_dict.update({k: updated_param}) + unet.load_state_dict(updated_state_dict) + return unet + + def test_optimization_steps_updated(self): + unet, ema_unet = self.get_models() + # Take the first (hypothetical) EMA step. + ema_unet.step(unet.parameters()) + assert ema_unet.optimization_step == 1 + + # Take two more. + for _ in range(2): + ema_unet.step(unet.parameters()) + assert ema_unet.optimization_step == 3 + + def test_shadow_params_not_updated(self): + unet, ema_unet = self.get_models() + # Since the `unet` is not being updated (i.e., backprop'd) + # there won't be any difference between the `params` of `unet` + # and `ema_unet` even if we call `ema_unet.step(unet.parameters())`. + ema_unet.step(unet.parameters()) + orig_params = list(unet.parameters()) + for s_param, param in zip(ema_unet.shadow_params, orig_params): + assert torch.allclose(s_param, param) + + # The above holds true even if we call `ema.step()` multiple times since + # `unet` params are still not being updated. + for _ in range(4): + ema_unet.step(unet.parameters()) + for s_param, param in zip(ema_unet.shadow_params, orig_params): + assert torch.allclose(s_param, param) + + def test_shadow_params_updated(self): + unet, ema_unet = self.get_models() + # Here we simulate the parameter updates for `unet`. Since there might + # be some parameters which are initialized to zero we take extra care to + # initialize their values to something non-zero before the multiplication. + unet_pseudo_updated_step_one = self.simulate_backprop(unet) + + # Take the EMA step. + ema_unet.step(unet_pseudo_updated_step_one.parameters()) + + # Now the EMA'd parameters won't be equal to the original model parameters. + orig_params = list(unet_pseudo_updated_step_one.parameters()) + for s_param, param in zip(ema_unet.shadow_params, orig_params): + assert ~torch.allclose(s_param, param) + + # Ensure this is the case when we take multiple EMA steps. + for _ in range(4): + ema_unet.step(unet.parameters()) + for s_param, param in zip(ema_unet.shadow_params, orig_params): + assert ~torch.allclose(s_param, param) + + def test_consecutive_shadow_params_updated(self): + # If we call EMA step after a backpropagation consecutively for two times, + # the shadow params from those two steps should be different. + unet, ema_unet = self.get_models() + + # First backprop + EMA + unet_step_one = self.simulate_backprop(unet) + ema_unet.step(unet_step_one.parameters()) + step_one_shadow_params = ema_unet.shadow_params + + # Second backprop + EMA + unet_step_two = self.simulate_backprop(unet_step_one) + ema_unet.step(unet_step_two.parameters()) + step_two_shadow_params = ema_unet.shadow_params + + for step_one, step_two in zip(step_one_shadow_params, step_two_shadow_params): + assert ~torch.allclose(step_one, step_two) + + def test_zero_decay(self): + # If there's no decay even if there are backprops, EMA steps + # won't take any effect i.e., the shadow params would remain the + # same. + unet, ema_unet = self.get_models(decay=0.0) + unet_step_one = self.simulate_backprop(unet) + ema_unet.step(unet_step_one.parameters()) + step_one_shadow_params = ema_unet.shadow_params + + unet_step_two = self.simulate_backprop(unet_step_one) + ema_unet.step(unet_step_two.parameters()) + step_two_shadow_params = ema_unet.shadow_params + + for step_one, step_two in zip(step_one_shadow_params, step_two_shadow_params): + assert torch.allclose(step_one, step_two) + + @skip_mps + def test_serialization(self): + unet, ema_unet = self.get_models() + noisy_latents, timesteps, encoder_hidden_states = self.get_dummy_inputs() + + with tempfile.TemporaryDirectory() as tmpdir: + ema_unet.save_pretrained(tmpdir) + loaded_unet = UNet2DConditionModel.from_pretrained(tmpdir, model_cls=UNet2DConditionModel) + loaded_unet = loaded_unet.to(unet.device) + + # Since no EMA step has been performed the outputs should match. + output = unet(noisy_latents, timesteps, encoder_hidden_states).sample + output_loaded = loaded_unet(noisy_latents, timesteps, encoder_hidden_states).sample + + assert torch.allclose(output, output_loaded, atol=1e-4) diff --git a/diffusers-0.27.0/tests/others/test_hub_utils.py b/diffusers-0.27.0/tests/others/test_hub_utils.py new file mode 100755 index 0000000..7a0c29d --- /dev/null +++ b/diffusers-0.27.0/tests/others/test_hub_utils.py @@ -0,0 +1,29 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from diffusers.utils.hub_utils import load_or_create_model_card, populate_model_card + + +class CreateModelCardTest(unittest.TestCase): + def test_generate_model_card_with_library_name(self): + with TemporaryDirectory() as tmpdir: + file_path = Path(tmpdir) / "README.md" + file_path.write_text("---\nlibrary_name: foo\n---\nContent\n") + model_card = load_or_create_model_card(file_path) + populate_model_card(model_card) + assert model_card.data.library_name == "foo" diff --git a/diffusers-0.27.0/tests/others/test_image_processor.py b/diffusers-0.27.0/tests/others/test_image_processor.py new file mode 100755 index 0000000..3397ca9 --- /dev/null +++ b/diffusers-0.27.0/tests/others/test_image_processor.py @@ -0,0 +1,310 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import PIL.Image +import torch + +from diffusers.image_processor import VaeImageProcessor + + +class ImageProcessorTest(unittest.TestCase): + @property + def dummy_sample(self): + batch_size = 1 + num_channels = 3 + height = 8 + width = 8 + + sample = torch.rand((batch_size, num_channels, height, width)) + + return sample + + @property + def dummy_mask(self): + batch_size = 1 + num_channels = 1 + height = 8 + width = 8 + + sample = torch.rand((batch_size, num_channels, height, width)) + + return sample + + def to_np(self, image): + if isinstance(image[0], PIL.Image.Image): + return np.stack([np.array(i) for i in image], axis=0) + elif isinstance(image, torch.Tensor): + return image.cpu().numpy().transpose(0, 2, 3, 1) + return image + + def test_vae_image_processor_pt(self): + image_processor = VaeImageProcessor(do_resize=False, do_normalize=True) + + input_pt = self.dummy_sample + input_np = self.to_np(input_pt) + + for output_type in ["pt", "np", "pil"]: + out = image_processor.postprocess( + image_processor.preprocess(input_pt), + output_type=output_type, + ) + out_np = self.to_np(out) + in_np = (input_np * 255).round() if output_type == "pil" else input_np + assert ( + np.abs(in_np - out_np).max() < 1e-6 + ), f"decoded output does not match input for output_type {output_type}" + + def test_vae_image_processor_np(self): + image_processor = VaeImageProcessor(do_resize=False, do_normalize=True) + input_np = self.dummy_sample.cpu().numpy().transpose(0, 2, 3, 1) + + for output_type in ["pt", "np", "pil"]: + out = image_processor.postprocess(image_processor.preprocess(input_np), output_type=output_type) + + out_np = self.to_np(out) + in_np = (input_np * 255).round() if output_type == "pil" else input_np + assert ( + np.abs(in_np - out_np).max() < 1e-6 + ), f"decoded output does not match input for output_type {output_type}" + + def test_vae_image_processor_pil(self): + image_processor = VaeImageProcessor(do_resize=False, do_normalize=True) + + input_np = self.dummy_sample.cpu().numpy().transpose(0, 2, 3, 1) + input_pil = image_processor.numpy_to_pil(input_np) + + for output_type in ["pt", "np", "pil"]: + out = image_processor.postprocess(image_processor.preprocess(input_pil), output_type=output_type) + for i, o in zip(input_pil, out): + in_np = np.array(i) + out_np = self.to_np(out) if output_type == "pil" else (self.to_np(out) * 255).round() + assert ( + np.abs(in_np - out_np).max() < 1e-6 + ), f"decoded output does not match input for output_type {output_type}" + + def test_preprocess_input_3d(self): + image_processor = VaeImageProcessor(do_resize=False, do_normalize=False) + + input_pt_4d = self.dummy_sample + input_pt_3d = input_pt_4d.squeeze(0) + + out_pt_4d = image_processor.postprocess( + image_processor.preprocess(input_pt_4d), + output_type="np", + ) + out_pt_3d = image_processor.postprocess( + image_processor.preprocess(input_pt_3d), + output_type="np", + ) + + input_np_4d = self.to_np(self.dummy_sample) + input_np_3d = input_np_4d.squeeze(0) + + out_np_4d = image_processor.postprocess( + image_processor.preprocess(input_np_4d), + output_type="np", + ) + out_np_3d = image_processor.postprocess( + image_processor.preprocess(input_np_3d), + output_type="np", + ) + + assert np.abs(out_pt_4d - out_pt_3d).max() < 1e-6 + assert np.abs(out_np_4d - out_np_3d).max() < 1e-6 + + def test_preprocess_input_list(self): + image_processor = VaeImageProcessor(do_resize=False, do_normalize=False) + + input_pt_4d = self.dummy_sample + input_pt_list = list(input_pt_4d) + + out_pt_4d = image_processor.postprocess( + image_processor.preprocess(input_pt_4d), + output_type="np", + ) + + out_pt_list = image_processor.postprocess( + image_processor.preprocess(input_pt_list), + output_type="np", + ) + + input_np_4d = self.to_np(self.dummy_sample) + input_np_list = list(input_np_4d) + + out_np_4d = image_processor.postprocess( + image_processor.preprocess(input_np_4d), + output_type="np", + ) + + out_np_list = image_processor.postprocess( + image_processor.preprocess(input_np_list), + output_type="np", + ) + + assert np.abs(out_pt_4d - out_pt_list).max() < 1e-6 + assert np.abs(out_np_4d - out_np_list).max() < 1e-6 + + def test_preprocess_input_mask_3d(self): + image_processor = VaeImageProcessor( + do_resize=False, do_normalize=False, do_binarize=True, do_convert_grayscale=True + ) + + input_pt_4d = self.dummy_mask + input_pt_3d = input_pt_4d.squeeze(0) + input_pt_2d = input_pt_3d.squeeze(0) + + out_pt_4d = image_processor.postprocess( + image_processor.preprocess(input_pt_4d), + output_type="np", + ) + out_pt_3d = image_processor.postprocess( + image_processor.preprocess(input_pt_3d), + output_type="np", + ) + + out_pt_2d = image_processor.postprocess( + image_processor.preprocess(input_pt_2d), + output_type="np", + ) + + input_np_4d = self.to_np(self.dummy_mask) + input_np_3d = input_np_4d.squeeze(0) + input_np_3d_1 = input_np_4d.squeeze(-1) + input_np_2d = input_np_3d.squeeze(-1) + + out_np_4d = image_processor.postprocess( + image_processor.preprocess(input_np_4d), + output_type="np", + ) + out_np_3d = image_processor.postprocess( + image_processor.preprocess(input_np_3d), + output_type="np", + ) + + out_np_3d_1 = image_processor.postprocess( + image_processor.preprocess(input_np_3d_1), + output_type="np", + ) + + out_np_2d = image_processor.postprocess( + image_processor.preprocess(input_np_2d), + output_type="np", + ) + + assert np.abs(out_pt_4d - out_pt_3d).max() == 0 + assert np.abs(out_pt_4d - out_pt_2d).max() == 0 + assert np.abs(out_np_4d - out_np_3d).max() == 0 + assert np.abs(out_np_4d - out_np_3d_1).max() == 0 + assert np.abs(out_np_4d - out_np_2d).max() == 0 + + def test_preprocess_input_mask_list(self): + image_processor = VaeImageProcessor(do_resize=False, do_normalize=False, do_convert_grayscale=True) + + input_pt_4d = self.dummy_mask + input_pt_3d = input_pt_4d.squeeze(0) + input_pt_2d = input_pt_3d.squeeze(0) + + inputs_pt = [input_pt_4d, input_pt_3d, input_pt_2d] + inputs_pt_list = [[input_pt] for input_pt in inputs_pt] + + for input_pt, input_pt_list in zip(inputs_pt, inputs_pt_list): + out_pt = image_processor.postprocess( + image_processor.preprocess(input_pt), + output_type="np", + ) + out_pt_list = image_processor.postprocess( + image_processor.preprocess(input_pt_list), + output_type="np", + ) + assert np.abs(out_pt - out_pt_list).max() < 1e-6 + + input_np_4d = self.to_np(self.dummy_mask) + input_np_3d = input_np_4d.squeeze(0) + input_np_2d = input_np_3d.squeeze(-1) + + inputs_np = [input_np_4d, input_np_3d, input_np_2d] + inputs_np_list = [[input_np] for input_np in inputs_np] + + for input_np, input_np_list in zip(inputs_np, inputs_np_list): + out_np = image_processor.postprocess( + image_processor.preprocess(input_np), + output_type="np", + ) + out_np_list = image_processor.postprocess( + image_processor.preprocess(input_np_list), + output_type="np", + ) + assert np.abs(out_np - out_np_list).max() < 1e-6 + + def test_preprocess_input_mask_3d_batch(self): + image_processor = VaeImageProcessor(do_resize=False, do_normalize=False, do_convert_grayscale=True) + + # create a dummy mask input with batch_size 2 + dummy_mask_batch = torch.cat([self.dummy_mask] * 2, axis=0) + + # squeeze out the channel dimension + input_pt_3d = dummy_mask_batch.squeeze(1) + input_np_3d = self.to_np(dummy_mask_batch).squeeze(-1) + + input_pt_3d_list = list(input_pt_3d) + input_np_3d_list = list(input_np_3d) + + out_pt_3d = image_processor.postprocess( + image_processor.preprocess(input_pt_3d), + output_type="np", + ) + out_pt_3d_list = image_processor.postprocess( + image_processor.preprocess(input_pt_3d_list), + output_type="np", + ) + + assert np.abs(out_pt_3d - out_pt_3d_list).max() < 1e-6 + + out_np_3d = image_processor.postprocess( + image_processor.preprocess(input_np_3d), + output_type="np", + ) + out_np_3d_list = image_processor.postprocess( + image_processor.preprocess(input_np_3d_list), + output_type="np", + ) + + assert np.abs(out_np_3d - out_np_3d_list).max() < 1e-6 + + def test_vae_image_processor_resize_pt(self): + image_processor = VaeImageProcessor(do_resize=True, vae_scale_factor=1) + input_pt = self.dummy_sample + b, c, h, w = input_pt.shape + scale = 2 + out_pt = image_processor.resize(image=input_pt, height=h // scale, width=w // scale) + exp_pt_shape = (b, c, h // scale, w // scale) + assert ( + out_pt.shape == exp_pt_shape + ), f"resized image output shape '{out_pt.shape}' didn't match expected shape '{exp_pt_shape}'." + + def test_vae_image_processor_resize_np(self): + image_processor = VaeImageProcessor(do_resize=True, vae_scale_factor=1) + input_pt = self.dummy_sample + b, c, h, w = input_pt.shape + scale = 2 + input_np = self.to_np(input_pt) + out_np = image_processor.resize(image=input_np, height=h // scale, width=w // scale) + exp_np_shape = (b, h // scale, w // scale, c) + assert ( + out_np.shape == exp_np_shape + ), f"resized image output shape '{out_np.shape}' didn't match expected shape '{exp_np_shape}'." diff --git a/diffusers-0.27.0/tests/others/test_outputs.py b/diffusers-0.27.0/tests/others/test_outputs.py new file mode 100755 index 0000000..cf709d9 --- /dev/null +++ b/diffusers-0.27.0/tests/others/test_outputs.py @@ -0,0 +1,93 @@ +import pickle as pkl +import unittest +from dataclasses import dataclass +from typing import List, Union + +import numpy as np +import PIL.Image + +from diffusers.utils.outputs import BaseOutput +from diffusers.utils.testing_utils import require_torch + + +@dataclass +class CustomOutput(BaseOutput): + images: Union[List[PIL.Image.Image], np.ndarray] + + +class ConfigTester(unittest.TestCase): + def test_outputs_single_attribute(self): + outputs = CustomOutput(images=np.random.rand(1, 3, 4, 4)) + + # check every way of getting the attribute + assert isinstance(outputs.images, np.ndarray) + assert outputs.images.shape == (1, 3, 4, 4) + assert isinstance(outputs["images"], np.ndarray) + assert outputs["images"].shape == (1, 3, 4, 4) + assert isinstance(outputs[0], np.ndarray) + assert outputs[0].shape == (1, 3, 4, 4) + + # test with a non-tensor attribute + outputs = CustomOutput(images=[PIL.Image.new("RGB", (4, 4))]) + + # check every way of getting the attribute + assert isinstance(outputs.images, list) + assert isinstance(outputs.images[0], PIL.Image.Image) + assert isinstance(outputs["images"], list) + assert isinstance(outputs["images"][0], PIL.Image.Image) + assert isinstance(outputs[0], list) + assert isinstance(outputs[0][0], PIL.Image.Image) + + def test_outputs_dict_init(self): + # test output reinitialization with a `dict` for compatibility with `accelerate` + outputs = CustomOutput({"images": np.random.rand(1, 3, 4, 4)}) + + # check every way of getting the attribute + assert isinstance(outputs.images, np.ndarray) + assert outputs.images.shape == (1, 3, 4, 4) + assert isinstance(outputs["images"], np.ndarray) + assert outputs["images"].shape == (1, 3, 4, 4) + assert isinstance(outputs[0], np.ndarray) + assert outputs[0].shape == (1, 3, 4, 4) + + # test with a non-tensor attribute + outputs = CustomOutput({"images": [PIL.Image.new("RGB", (4, 4))]}) + + # check every way of getting the attribute + assert isinstance(outputs.images, list) + assert isinstance(outputs.images[0], PIL.Image.Image) + assert isinstance(outputs["images"], list) + assert isinstance(outputs["images"][0], PIL.Image.Image) + assert isinstance(outputs[0], list) + assert isinstance(outputs[0][0], PIL.Image.Image) + + def test_outputs_serialization(self): + outputs_orig = CustomOutput(images=[PIL.Image.new("RGB", (4, 4))]) + serialized = pkl.dumps(outputs_orig) + outputs_copy = pkl.loads(serialized) + + # Check original and copy are equal + assert dir(outputs_orig) == dir(outputs_copy) + assert dict(outputs_orig) == dict(outputs_copy) + assert vars(outputs_orig) == vars(outputs_copy) + + @require_torch + def test_torch_pytree(self): + # ensure torch.utils._pytree treats ModelOutput subclasses as nodes (and not leaves) + # this is important for DistributedDataParallel gradient synchronization with static_graph=True + import torch + import torch.utils._pytree + + data = np.random.rand(1, 3, 4, 4) + x = CustomOutput(images=data) + self.assertFalse(torch.utils._pytree._is_leaf(x)) + + expected_flat_outs = [data] + expected_tree_spec = torch.utils._pytree.TreeSpec(CustomOutput, ["images"], [torch.utils._pytree.LeafSpec()]) + + actual_flat_outs, actual_tree_spec = torch.utils._pytree.tree_flatten(x) + self.assertEqual(expected_flat_outs, actual_flat_outs) + self.assertEqual(expected_tree_spec, actual_tree_spec) + + unflattened_x = torch.utils._pytree.tree_unflatten(actual_flat_outs, actual_tree_spec) + self.assertEqual(x, unflattened_x) diff --git a/diffusers-0.27.0/tests/others/test_training.py b/diffusers-0.27.0/tests/others/test_training.py new file mode 100755 index 0000000..863ba6e --- /dev/null +++ b/diffusers-0.27.0/tests/others/test_training.py @@ -0,0 +1,86 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import torch + +from diffusers import DDIMScheduler, DDPMScheduler, UNet2DModel +from diffusers.training_utils import set_seed +from diffusers.utils.testing_utils import slow + + +torch.backends.cuda.matmul.allow_tf32 = False + + +class TrainingTests(unittest.TestCase): + def get_model_optimizer(self, resolution=32): + set_seed(0) + model = UNet2DModel(sample_size=resolution, in_channels=3, out_channels=3) + optimizer = torch.optim.SGD(model.parameters(), lr=0.0001) + return model, optimizer + + @slow + def test_training_step_equality(self): + device = "cpu" # ensure full determinism without setting the CUBLAS_WORKSPACE_CONFIG env variable + ddpm_scheduler = DDPMScheduler( + num_train_timesteps=1000, + beta_start=0.0001, + beta_end=0.02, + beta_schedule="linear", + clip_sample=True, + ) + ddim_scheduler = DDIMScheduler( + num_train_timesteps=1000, + beta_start=0.0001, + beta_end=0.02, + beta_schedule="linear", + clip_sample=True, + ) + + assert ddpm_scheduler.config.num_train_timesteps == ddim_scheduler.config.num_train_timesteps + + # shared batches for DDPM and DDIM + set_seed(0) + clean_images = [torch.randn((4, 3, 32, 32)).clip(-1, 1).to(device) for _ in range(4)] + noise = [torch.randn((4, 3, 32, 32)).to(device) for _ in range(4)] + timesteps = [torch.randint(0, 1000, (4,)).long().to(device) for _ in range(4)] + + # train with a DDPM scheduler + model, optimizer = self.get_model_optimizer(resolution=32) + model.train().to(device) + for i in range(4): + optimizer.zero_grad() + ddpm_noisy_images = ddpm_scheduler.add_noise(clean_images[i], noise[i], timesteps[i]) + ddpm_noise_pred = model(ddpm_noisy_images, timesteps[i]).sample + loss = torch.nn.functional.mse_loss(ddpm_noise_pred, noise[i]) + loss.backward() + optimizer.step() + del model, optimizer + + # recreate the model and optimizer, and retry with DDIM + model, optimizer = self.get_model_optimizer(resolution=32) + model.train().to(device) + for i in range(4): + optimizer.zero_grad() + ddim_noisy_images = ddim_scheduler.add_noise(clean_images[i], noise[i], timesteps[i]) + ddim_noise_pred = model(ddim_noisy_images, timesteps[i]).sample + loss = torch.nn.functional.mse_loss(ddim_noise_pred, noise[i]) + loss.backward() + optimizer.step() + del model, optimizer + + self.assertTrue(torch.allclose(ddpm_noisy_images, ddim_noisy_images, atol=1e-5)) + self.assertTrue(torch.allclose(ddpm_noise_pred, ddim_noise_pred, atol=1e-5)) diff --git a/diffusers-0.27.0/tests/others/test_utils.py b/diffusers-0.27.0/tests/others/test_utils.py new file mode 100755 index 0000000..9ebae06 --- /dev/null +++ b/diffusers-0.27.0/tests/others/test_utils.py @@ -0,0 +1,213 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import unittest +from distutils.util import strtobool + +import pytest + +from diffusers import __version__ +from diffusers.utils import deprecate + + +# Used to test the hub +USER = "__DUMMY_TRANSFORMERS_USER__" +ENDPOINT_STAGING = "https://hub-ci.huggingface.co" + +# Not critical, only usable on the sandboxed CI instance. +TOKEN = "hf_94wBhPGp6KrrTH3KDchhKpRxZwd6dmHWLL" + + +class DeprecateTester(unittest.TestCase): + higher_version = ".".join([str(int(__version__.split(".")[0]) + 1)] + __version__.split(".")[1:]) + lower_version = "0.0.1" + + def test_deprecate_function_arg(self): + kwargs = {"deprecated_arg": 4} + + with self.assertWarns(FutureWarning) as warning: + output = deprecate("deprecated_arg", self.higher_version, "message", take_from=kwargs) + + assert output == 4 + assert ( + str(warning.warning) + == f"The `deprecated_arg` argument is deprecated and will be removed in version {self.higher_version}." + " message" + ) + + def test_deprecate_function_arg_tuple(self): + kwargs = {"deprecated_arg": 4} + + with self.assertWarns(FutureWarning) as warning: + output = deprecate(("deprecated_arg", self.higher_version, "message"), take_from=kwargs) + + assert output == 4 + assert ( + str(warning.warning) + == f"The `deprecated_arg` argument is deprecated and will be removed in version {self.higher_version}." + " message" + ) + + def test_deprecate_function_args(self): + kwargs = {"deprecated_arg_1": 4, "deprecated_arg_2": 8} + with self.assertWarns(FutureWarning) as warning: + output_1, output_2 = deprecate( + ("deprecated_arg_1", self.higher_version, "Hey"), + ("deprecated_arg_2", self.higher_version, "Hey"), + take_from=kwargs, + ) + assert output_1 == 4 + assert output_2 == 8 + assert ( + str(warning.warnings[0].message) + == "The `deprecated_arg_1` argument is deprecated and will be removed in version" + f" {self.higher_version}. Hey" + ) + assert ( + str(warning.warnings[1].message) + == "The `deprecated_arg_2` argument is deprecated and will be removed in version" + f" {self.higher_version}. Hey" + ) + + def test_deprecate_function_incorrect_arg(self): + kwargs = {"deprecated_arg": 4} + + with self.assertRaises(TypeError) as error: + deprecate(("wrong_arg", self.higher_version, "message"), take_from=kwargs) + + assert "test_deprecate_function_incorrect_arg in" in str(error.exception) + assert "line" in str(error.exception) + assert "got an unexpected keyword argument `deprecated_arg`" in str(error.exception) + + def test_deprecate_arg_no_kwarg(self): + with self.assertWarns(FutureWarning) as warning: + deprecate(("deprecated_arg", self.higher_version, "message")) + + assert ( + str(warning.warning) + == f"`deprecated_arg` is deprecated and will be removed in version {self.higher_version}. message" + ) + + def test_deprecate_args_no_kwarg(self): + with self.assertWarns(FutureWarning) as warning: + deprecate( + ("deprecated_arg_1", self.higher_version, "Hey"), + ("deprecated_arg_2", self.higher_version, "Hey"), + ) + assert ( + str(warning.warnings[0].message) + == f"`deprecated_arg_1` is deprecated and will be removed in version {self.higher_version}. Hey" + ) + assert ( + str(warning.warnings[1].message) + == f"`deprecated_arg_2` is deprecated and will be removed in version {self.higher_version}. Hey" + ) + + def test_deprecate_class_obj(self): + class Args: + arg = 5 + + with self.assertWarns(FutureWarning) as warning: + arg = deprecate(("arg", self.higher_version, "message"), take_from=Args()) + + assert arg == 5 + assert ( + str(warning.warning) + == f"The `arg` attribute is deprecated and will be removed in version {self.higher_version}. message" + ) + + def test_deprecate_class_objs(self): + class Args: + arg = 5 + foo = 7 + + with self.assertWarns(FutureWarning) as warning: + arg_1, arg_2 = deprecate( + ("arg", self.higher_version, "message"), + ("foo", self.higher_version, "message"), + ("does not exist", self.higher_version, "message"), + take_from=Args(), + ) + + assert arg_1 == 5 + assert arg_2 == 7 + assert ( + str(warning.warning) + == f"The `arg` attribute is deprecated and will be removed in version {self.higher_version}. message" + ) + assert ( + str(warning.warnings[0].message) + == f"The `arg` attribute is deprecated and will be removed in version {self.higher_version}. message" + ) + assert ( + str(warning.warnings[1].message) + == f"The `foo` attribute is deprecated and will be removed in version {self.higher_version}. message" + ) + + def test_deprecate_incorrect_version(self): + kwargs = {"deprecated_arg": 4} + + with self.assertRaises(ValueError) as error: + deprecate(("wrong_arg", self.lower_version, "message"), take_from=kwargs) + + assert ( + str(error.exception) + == "The deprecation tuple ('wrong_arg', '0.0.1', 'message') should be removed since diffusers' version" + f" {__version__} is >= {self.lower_version}" + ) + + def test_deprecate_incorrect_no_standard_warn(self): + with self.assertWarns(FutureWarning) as warning: + deprecate(("deprecated_arg", self.higher_version, "This message is better!!!"), standard_warn=False) + + assert str(warning.warning) == "This message is better!!!" + + def test_deprecate_stacklevel(self): + with self.assertWarns(FutureWarning) as warning: + deprecate(("deprecated_arg", self.higher_version, "This message is better!!!"), standard_warn=False) + assert str(warning.warning) == "This message is better!!!" + assert "diffusers/tests/others/test_utils.py" in warning.filename + + +def parse_flag_from_env(key, default=False): + try: + value = os.environ[key] + except KeyError: + # KEY isn't set, default to `default`. + _value = default + else: + # KEY is set, convert it to True or False. + try: + _value = strtobool(value) + except ValueError: + # More values are supported, but let's keep the message simple. + raise ValueError(f"If set, {key} must be yes or no.") + return _value + + +_run_staging = parse_flag_from_env("HUGGINGFACE_CO_STAGING", default=False) + + +def is_staging_test(test_case): + """ + Decorator marking a test as a staging test. + + Those tests will run using the staging environment of huggingface.co instead of the real model hub. + """ + if not _run_staging: + return unittest.skip("test is staging test")(test_case) + else: + return pytest.mark.is_staging_test()(test_case) diff --git a/diffusers-0.27.0/tests/pipelines/__init__.py b/diffusers-0.27.0/tests/pipelines/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/amused/__init__.py b/diffusers-0.27.0/tests/pipelines/amused/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/amused/test_amused.py b/diffusers-0.27.0/tests/pipelines/amused/test_amused.py new file mode 100755 index 0000000..f03751e --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/amused/test_amused.py @@ -0,0 +1,181 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import AmusedPipeline, AmusedScheduler, UVit2DModel, VQModel +from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, slow, torch_device + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class AmusedPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = AmusedPipeline + params = TEXT_TO_IMAGE_PARAMS | {"encoder_hidden_states", "negative_encoder_hidden_states"} + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + transformer = UVit2DModel( + hidden_size=32, + use_bias=False, + hidden_dropout=0.0, + cond_embed_dim=32, + micro_cond_encode_dim=2, + micro_cond_embed_dim=10, + encoder_hidden_size=32, + vocab_size=32, + codebook_size=32, + in_channels=32, + block_out_channels=32, + num_res_blocks=1, + downsample=True, + upsample=True, + block_num_heads=1, + num_hidden_layers=1, + num_attention_heads=1, + attention_dropout=0.0, + intermediate_size=32, + layer_norm_eps=1e-06, + ln_elementwise_affine=True, + ) + scheduler = AmusedScheduler(mask_token_id=31) + torch.manual_seed(0) + vqvae = VQModel( + act_fn="silu", + block_out_channels=[32], + down_block_types=[ + "DownEncoderBlock2D", + ], + in_channels=3, + latent_channels=32, + layers_per_block=2, + norm_num_groups=32, + num_vq_embeddings=32, + out_channels=3, + sample_size=32, + up_block_types=[ + "UpDecoderBlock2D", + ], + mid_block_add_attention=False, + lookup_from_codebook=True, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=64, + layer_norm_eps=1e-05, + num_attention_heads=8, + num_hidden_layers=3, + pad_token_id=1, + vocab_size=1000, + projection_dim=32, + ) + text_encoder = CLIPTextModelWithProjection(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "transformer": transformer, + "scheduler": scheduler, + "vqvae": vqvae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "output_type": "np", + "height": 4, + "width": 4, + } + return inputs + + def test_inference_batch_consistent(self, batch_sizes=[2]): + self._test_inference_batch_consistent(batch_sizes=batch_sizes, batch_generator=False) + + @unittest.skip("aMUSEd does not support lists of generators") + def test_inference_batch_single_identical(self): + ... + + +@slow +@require_torch_gpu +class AmusedPipelineSlowTests(unittest.TestCase): + def test_amused_256(self): + pipe = AmusedPipeline.from_pretrained("amused/amused-256") + pipe.to(torch_device) + + image = pipe("dog", generator=torch.Generator().manual_seed(0), num_inference_steps=2, output_type="np").images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 256, 256, 3) + expected_slice = np.array([0.4011, 0.3992, 0.3790, 0.3856, 0.3772, 0.3711, 0.3919, 0.3850, 0.3625]) + assert np.abs(image_slice - expected_slice).max() < 3e-3 + + def test_amused_256_fp16(self): + pipe = AmusedPipeline.from_pretrained("amused/amused-256", variant="fp16", torch_dtype=torch.float16) + pipe.to(torch_device) + + image = pipe("dog", generator=torch.Generator().manual_seed(0), num_inference_steps=2, output_type="np").images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 256, 256, 3) + expected_slice = np.array([0.0554, 0.05129, 0.0344, 0.0452, 0.0476, 0.0271, 0.0495, 0.0527, 0.0158]) + assert np.abs(image_slice - expected_slice).max() < 7e-3 + + def test_amused_512(self): + pipe = AmusedPipeline.from_pretrained("amused/amused-512") + pipe.to(torch_device) + + image = pipe("dog", generator=torch.Generator().manual_seed(0), num_inference_steps=2, output_type="np").images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.9960, 0.9960, 0.9946, 0.9980, 0.9947, 0.9932, 0.9960, 0.9961, 0.9947]) + assert np.abs(image_slice - expected_slice).max() < 3e-3 + + def test_amused_512_fp16(self): + pipe = AmusedPipeline.from_pretrained("amused/amused-512", variant="fp16", torch_dtype=torch.float16) + pipe.to(torch_device) + + image = pipe("dog", generator=torch.Generator().manual_seed(0), num_inference_steps=2, output_type="np").images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.9983, 1.0, 1.0, 1.0, 1.0, 0.9989, 0.9994, 0.9976, 0.9977]) + assert np.abs(image_slice - expected_slice).max() < 3e-3 diff --git a/diffusers-0.27.0/tests/pipelines/amused/test_amused_img2img.py b/diffusers-0.27.0/tests/pipelines/amused/test_amused_img2img.py new file mode 100755 index 0000000..efbca1f --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/amused/test_amused_img2img.py @@ -0,0 +1,235 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import AmusedImg2ImgPipeline, AmusedScheduler, UVit2DModel, VQModel +from diffusers.utils import load_image +from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, slow, torch_device + +from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class AmusedImg2ImgPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = AmusedImg2ImgPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width", "latents"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + required_optional_params = PipelineTesterMixin.required_optional_params - { + "latents", + } + + def get_dummy_components(self): + torch.manual_seed(0) + transformer = UVit2DModel( + hidden_size=32, + use_bias=False, + hidden_dropout=0.0, + cond_embed_dim=32, + micro_cond_encode_dim=2, + micro_cond_embed_dim=10, + encoder_hidden_size=32, + vocab_size=32, + codebook_size=32, + in_channels=32, + block_out_channels=32, + num_res_blocks=1, + downsample=True, + upsample=True, + block_num_heads=1, + num_hidden_layers=1, + num_attention_heads=1, + attention_dropout=0.0, + intermediate_size=32, + layer_norm_eps=1e-06, + ln_elementwise_affine=True, + ) + scheduler = AmusedScheduler(mask_token_id=31) + torch.manual_seed(0) + vqvae = VQModel( + act_fn="silu", + block_out_channels=[32], + down_block_types=[ + "DownEncoderBlock2D", + ], + in_channels=3, + latent_channels=32, + layers_per_block=2, + norm_num_groups=32, + num_vq_embeddings=32, + out_channels=3, + sample_size=32, + up_block_types=[ + "UpDecoderBlock2D", + ], + mid_block_add_attention=False, + lookup_from_codebook=True, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=64, + layer_norm_eps=1e-05, + num_attention_heads=8, + num_hidden_layers=3, + pad_token_id=1, + vocab_size=1000, + projection_dim=32, + ) + text_encoder = CLIPTextModelWithProjection(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "transformer": transformer, + "scheduler": scheduler, + "vqvae": vqvae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + image = torch.full((1, 3, 4, 4), 1.0, dtype=torch.float32, device=device) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "output_type": "np", + "image": image, + } + return inputs + + def test_inference_batch_consistent(self, batch_sizes=[2]): + self._test_inference_batch_consistent(batch_sizes=batch_sizes, batch_generator=False) + + @unittest.skip("aMUSEd does not support lists of generators") + def test_inference_batch_single_identical(self): + ... + + +@slow +@require_torch_gpu +class AmusedImg2ImgPipelineSlowTests(unittest.TestCase): + def test_amused_256(self): + pipe = AmusedImg2ImgPipeline.from_pretrained("amused/amused-256") + pipe.to(torch_device) + + image = ( + load_image("https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains.jpg") + .resize((256, 256)) + .convert("RGB") + ) + + image = pipe( + "winter mountains", + image, + generator=torch.Generator().manual_seed(0), + num_inference_steps=2, + output_type="np", + ).images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 256, 256, 3) + expected_slice = np.array([0.9993, 1.0, 0.9996, 1.0, 0.9995, 0.9925, 0.9990, 0.9954, 1.0]) + + assert np.abs(image_slice - expected_slice).max() < 1e-2 + + def test_amused_256_fp16(self): + pipe = AmusedImg2ImgPipeline.from_pretrained("amused/amused-256", torch_dtype=torch.float16, variant="fp16") + pipe.to(torch_device) + + image = ( + load_image("https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains.jpg") + .resize((256, 256)) + .convert("RGB") + ) + + image = pipe( + "winter mountains", + image, + generator=torch.Generator().manual_seed(0), + num_inference_steps=2, + output_type="np", + ).images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 256, 256, 3) + expected_slice = np.array([0.9980, 0.9980, 0.9940, 0.9944, 0.9960, 0.9908, 1.0, 1.0, 0.9986]) + + assert np.abs(image_slice - expected_slice).max() < 1e-2 + + def test_amused_512(self): + pipe = AmusedImg2ImgPipeline.from_pretrained("amused/amused-512") + pipe.to(torch_device) + + image = ( + load_image("https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains.jpg") + .resize((512, 512)) + .convert("RGB") + ) + + image = pipe( + "winter mountains", + image, + generator=torch.Generator().manual_seed(0), + num_inference_steps=2, + output_type="np", + ).images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.1344, 0.0985, 0.0, 0.1194, 0.1809, 0.0765, 0.0854, 0.1371, 0.0933]) + assert np.abs(image_slice - expected_slice).max() < 0.1 + + def test_amused_512_fp16(self): + pipe = AmusedImg2ImgPipeline.from_pretrained("amused/amused-512", variant="fp16", torch_dtype=torch.float16) + pipe.to(torch_device) + + image = ( + load_image("https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains.jpg") + .resize((512, 512)) + .convert("RGB") + ) + + image = pipe( + "winter mountains", + image, + generator=torch.Generator().manual_seed(0), + num_inference_steps=2, + output_type="np", + ).images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.1536, 0.1767, 0.0227, 0.1079, 0.2400, 0.1427, 0.1511, 0.1564, 0.1542]) + assert np.abs(image_slice - expected_slice).max() < 0.1 diff --git a/diffusers-0.27.0/tests/pipelines/amused/test_amused_inpaint.py b/diffusers-0.27.0/tests/pipelines/amused/test_amused_inpaint.py new file mode 100755 index 0000000..d397f8d --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/amused/test_amused_inpaint.py @@ -0,0 +1,273 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import AmusedInpaintPipeline, AmusedScheduler, UVit2DModel, VQModel +from diffusers.utils import load_image +from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, slow, torch_device + +from ..pipeline_params import TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, TEXT_GUIDED_IMAGE_INPAINTING_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class AmusedInpaintPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = AmusedInpaintPipeline + params = TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {"width", "height"} + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + required_optional_params = PipelineTesterMixin.required_optional_params - { + "latents", + } + + def get_dummy_components(self): + torch.manual_seed(0) + transformer = UVit2DModel( + hidden_size=32, + use_bias=False, + hidden_dropout=0.0, + cond_embed_dim=32, + micro_cond_encode_dim=2, + micro_cond_embed_dim=10, + encoder_hidden_size=32, + vocab_size=32, + codebook_size=32, + in_channels=32, + block_out_channels=32, + num_res_blocks=1, + downsample=True, + upsample=True, + block_num_heads=1, + num_hidden_layers=1, + num_attention_heads=1, + attention_dropout=0.0, + intermediate_size=32, + layer_norm_eps=1e-06, + ln_elementwise_affine=True, + ) + scheduler = AmusedScheduler(mask_token_id=31) + torch.manual_seed(0) + vqvae = VQModel( + act_fn="silu", + block_out_channels=[32], + down_block_types=[ + "DownEncoderBlock2D", + ], + in_channels=3, + latent_channels=32, + layers_per_block=2, + norm_num_groups=32, + num_vq_embeddings=32, + out_channels=3, + sample_size=32, + up_block_types=[ + "UpDecoderBlock2D", + ], + mid_block_add_attention=False, + lookup_from_codebook=True, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=64, + layer_norm_eps=1e-05, + num_attention_heads=8, + num_hidden_layers=3, + pad_token_id=1, + vocab_size=1000, + projection_dim=32, + ) + text_encoder = CLIPTextModelWithProjection(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "transformer": transformer, + "scheduler": scheduler, + "vqvae": vqvae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + image = torch.full((1, 3, 4, 4), 1.0, dtype=torch.float32, device=device) + mask_image = torch.full((1, 1, 4, 4), 1.0, dtype=torch.float32, device=device) + mask_image[0, 0, 0, 0] = 0 + mask_image[0, 0, 0, 1] = 0 + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "output_type": "np", + "image": image, + "mask_image": mask_image, + } + return inputs + + def test_inference_batch_consistent(self, batch_sizes=[2]): + self._test_inference_batch_consistent(batch_sizes=batch_sizes, batch_generator=False) + + @unittest.skip("aMUSEd does not support lists of generators") + def test_inference_batch_single_identical(self): + ... + + +@slow +@require_torch_gpu +class AmusedInpaintPipelineSlowTests(unittest.TestCase): + def test_amused_256(self): + pipe = AmusedInpaintPipeline.from_pretrained("amused/amused-256") + pipe.to(torch_device) + + image = ( + load_image("https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains_1.jpg") + .resize((256, 256)) + .convert("RGB") + ) + + mask_image = ( + load_image( + "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains_1_mask.png" + ) + .resize((256, 256)) + .convert("L") + ) + + image = pipe( + "winter mountains", + image, + mask_image, + generator=torch.Generator().manual_seed(0), + num_inference_steps=2, + output_type="np", + ).images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 256, 256, 3) + expected_slice = np.array([0.0699, 0.0716, 0.0608, 0.0715, 0.0797, 0.0638, 0.0802, 0.0924, 0.0634]) + assert np.abs(image_slice - expected_slice).max() < 0.1 + + def test_amused_256_fp16(self): + pipe = AmusedInpaintPipeline.from_pretrained("amused/amused-256", variant="fp16", torch_dtype=torch.float16) + pipe.to(torch_device) + + image = ( + load_image("https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains_1.jpg") + .resize((256, 256)) + .convert("RGB") + ) + + mask_image = ( + load_image( + "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains_1_mask.png" + ) + .resize((256, 256)) + .convert("L") + ) + + image = pipe( + "winter mountains", + image, + mask_image, + generator=torch.Generator().manual_seed(0), + num_inference_steps=2, + output_type="np", + ).images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 256, 256, 3) + expected_slice = np.array([0.0735, 0.0749, 0.0650, 0.0739, 0.0805, 0.0667, 0.0802, 0.0923, 0.0622]) + assert np.abs(image_slice - expected_slice).max() < 0.1 + + def test_amused_512(self): + pipe = AmusedInpaintPipeline.from_pretrained("amused/amused-512") + pipe.to(torch_device) + + image = ( + load_image("https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains_1.jpg") + .resize((512, 512)) + .convert("RGB") + ) + + mask_image = ( + load_image( + "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains_1_mask.png" + ) + .resize((512, 512)) + .convert("L") + ) + + image = pipe( + "winter mountains", + image, + mask_image, + generator=torch.Generator().manual_seed(0), + num_inference_steps=2, + output_type="np", + ).images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0005, 0.0]) + assert np.abs(image_slice - expected_slice).max() < 0.05 + + def test_amused_512_fp16(self): + pipe = AmusedInpaintPipeline.from_pretrained("amused/amused-512", variant="fp16", torch_dtype=torch.float16) + pipe.to(torch_device) + + image = ( + load_image("https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains_1.jpg") + .resize((512, 512)) + .convert("RGB") + ) + + mask_image = ( + load_image( + "https://huggingface.co/datasets/diffusers/docs-images/resolve/main/open_muse/mountains_1_mask.png" + ) + .resize((512, 512)) + .convert("L") + ) + + image = pipe( + "winter mountains", + image, + mask_image, + generator=torch.Generator().manual_seed(0), + num_inference_steps=2, + output_type="np", + ).images + + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0025, 0.0]) + assert np.abs(image_slice - expected_slice).max() < 3e-3 diff --git a/diffusers-0.27.0/tests/pipelines/animatediff/__init__.py b/diffusers-0.27.0/tests/pipelines/animatediff/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/animatediff/test_animatediff.py b/diffusers-0.27.0/tests/pipelines/animatediff/test_animatediff.py new file mode 100755 index 0000000..288f856 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/animatediff/test_animatediff.py @@ -0,0 +1,358 @@ +import gc +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +import diffusers +from diffusers import ( + AnimateDiffPipeline, + AutoencoderKL, + DDIMScheduler, + MotionAdapter, + UNet2DConditionModel, + UNetMotionModel, +) +from diffusers.utils import is_xformers_available, logging +from diffusers.utils.testing_utils import numpy_cosine_similarity_distance, require_torch_gpu, slow, torch_device + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import IPAdapterTesterMixin, PipelineTesterMixin, SDFunctionTesterMixin + + +def to_np(tensor): + if isinstance(tensor, torch.Tensor): + tensor = tensor.detach().cpu().numpy() + + return tensor + + +class AnimateDiffPipelineFastTests( + IPAdapterTesterMixin, SDFunctionTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = AnimateDiffPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + required_optional_params = frozenset( + [ + "num_inference_steps", + "generator", + "latents", + "return_dict", + "callback_on_step_end", + "callback_on_step_end_tensor_inputs", + ] + ) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("CrossAttnDownBlock2D", "DownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + norm_num_groups=2, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="linear", + clip_sample=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + motion_adapter = MotionAdapter( + block_out_channels=(32, 64), + motion_layers_per_block=2, + motion_norm_num_groups=2, + motion_num_attention_heads=4, + ) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "motion_adapter": motion_adapter, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 7.5, + "output_type": "pt", + } + return inputs + + def test_motion_unet_loading(self): + components = self.get_dummy_components() + pipe = AnimateDiffPipeline(**components) + + assert isinstance(pipe.unet, UNetMotionModel) + + @unittest.skip("Attention slicing is not enabled in this pipeline") + def test_attention_slicing_forward_pass(self): + pass + + def test_inference_batch_single_identical( + self, + batch_size=2, + expected_max_diff=1e-4, + additional_params_copy_to_batched_inputs=["num_inference_steps"], + ): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for components in pipe.components.values(): + if hasattr(components, "set_default_attn_processor"): + components.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + inputs = self.get_dummy_inputs(torch_device) + # Reset generator in case it is has been used in self.get_dummy_inputs + inputs["generator"] = self.get_generator(0) + + logger = logging.get_logger(pipe.__module__) + logger.setLevel(level=diffusers.logging.FATAL) + + # batchify inputs + batched_inputs = {} + batched_inputs.update(inputs) + + for name in self.batch_params: + if name not in inputs: + continue + + value = inputs[name] + if name == "prompt": + len_prompt = len(value) + batched_inputs[name] = [value[: len_prompt // i] for i in range(1, batch_size + 1)] + batched_inputs[name][-1] = 100 * "very long" + + else: + batched_inputs[name] = batch_size * [value] + + if "generator" in inputs: + batched_inputs["generator"] = [self.get_generator(i) for i in range(batch_size)] + + if "batch_size" in inputs: + batched_inputs["batch_size"] = batch_size + + for arg in additional_params_copy_to_batched_inputs: + batched_inputs[arg] = inputs[arg] + + output = pipe(**inputs) + output_batch = pipe(**batched_inputs) + + assert output_batch[0].shape[0] == batch_size + + max_diff = np.abs(to_np(output_batch[0][0]) - to_np(output[0][0])).max() + assert max_diff < expected_max_diff + + @unittest.skipIf(torch_device != "cuda", reason="CUDA and CPU are required to switch devices") + def test_to_device(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + pipe.to("cpu") + # pipeline creates a new motion UNet under the hood. So we need to check the device from pipe.components + model_devices = [ + component.device.type for component in pipe.components.values() if hasattr(component, "device") + ] + self.assertTrue(all(device == "cpu" for device in model_devices)) + + output_cpu = pipe(**self.get_dummy_inputs("cpu"))[0] + self.assertTrue(np.isnan(output_cpu).sum() == 0) + + pipe.to("cuda") + model_devices = [ + component.device.type for component in pipe.components.values() if hasattr(component, "device") + ] + self.assertTrue(all(device == "cuda" for device in model_devices)) + + output_cuda = pipe(**self.get_dummy_inputs("cuda"))[0] + self.assertTrue(np.isnan(to_np(output_cuda)).sum() == 0) + + def test_to_dtype(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + # pipeline creates a new motion UNet under the hood. So we need to check the dtype from pipe.components + model_dtypes = [component.dtype for component in pipe.components.values() if hasattr(component, "dtype")] + self.assertTrue(all(dtype == torch.float32 for dtype in model_dtypes)) + + pipe.to(dtype=torch.float16) + model_dtypes = [component.dtype for component in pipe.components.values() if hasattr(component, "dtype")] + self.assertTrue(all(dtype == torch.float16 for dtype in model_dtypes)) + + def test_prompt_embeds(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + inputs = self.get_dummy_inputs(torch_device) + inputs.pop("prompt") + inputs["prompt_embeds"] = torch.randn((1, 4, 32), device=torch_device) + pipe(**inputs) + + def test_free_init(self): + components = self.get_dummy_components() + pipe: AnimateDiffPipeline = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + inputs_normal = self.get_dummy_inputs(torch_device) + frames_normal = pipe(**inputs_normal).frames[0] + + pipe.enable_free_init( + num_iters=2, + use_fast_sampling=True, + method="butterworth", + order=4, + spatial_stop_frequency=0.25, + temporal_stop_frequency=0.25, + ) + inputs_enable_free_init = self.get_dummy_inputs(torch_device) + frames_enable_free_init = pipe(**inputs_enable_free_init).frames[0] + + pipe.disable_free_init() + inputs_disable_free_init = self.get_dummy_inputs(torch_device) + frames_disable_free_init = pipe(**inputs_disable_free_init).frames[0] + + sum_enabled = np.abs(to_np(frames_normal) - to_np(frames_enable_free_init)).sum() + max_diff_disabled = np.abs(to_np(frames_normal) - to_np(frames_disable_free_init)).max() + self.assertGreater( + sum_enabled, 1e1, "Enabling of FreeInit should lead to results different from the default pipeline results" + ) + self.assertLess( + max_diff_disabled, + 1e-4, + "Disabling of FreeInit should lead to results similar to the default pipeline results", + ) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output_without_offload = pipe(**inputs).frames[0] + output_without_offload = ( + output_without_offload.cpu() if torch.is_tensor(output_without_offload) else output_without_offload + ) + + pipe.enable_xformers_memory_efficient_attention() + inputs = self.get_dummy_inputs(torch_device) + output_with_offload = pipe(**inputs).frames[0] + output_with_offload = ( + output_with_offload.cpu() if torch.is_tensor(output_with_offload) else output_without_offload + ) + + max_diff = np.abs(to_np(output_with_offload) - to_np(output_without_offload)).max() + self.assertLess(max_diff, 1e-4, "XFormers attention should not affect the inference results") + + +@slow +@require_torch_gpu +class AnimateDiffPipelineSlowTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_animatediff(self): + adapter = MotionAdapter.from_pretrained("guoyww/animatediff-motion-adapter-v1-5-2") + pipe = AnimateDiffPipeline.from_pretrained("frankjoshua/toonyou_beta6", motion_adapter=adapter) + pipe = pipe.to(torch_device) + pipe.scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="linear", + steps_offset=1, + clip_sample=False, + ) + pipe.enable_vae_slicing() + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + prompt = "night, b&w photo of old house, post apocalypse, forest, storm weather, wind, rocks, 8k uhd, dslr, soft lighting, high quality, film grain" + negative_prompt = "bad quality, worse quality" + + generator = torch.Generator("cpu").manual_seed(0) + output = pipe( + prompt, + negative_prompt=negative_prompt, + num_frames=16, + generator=generator, + guidance_scale=7.5, + num_inference_steps=3, + output_type="np", + ) + + image = output.frames[0] + assert image.shape == (16, 512, 512, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array( + [ + 0.11357737, + 0.11285847, + 0.11180121, + 0.11084166, + 0.11414117, + 0.09785956, + 0.10742754, + 0.10510018, + 0.08045256, + ] + ) + assert numpy_cosine_similarity_distance(image_slice.flatten(), expected_slice.flatten()) < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/animatediff/test_animatediff_video2video.py b/diffusers-0.27.0/tests/pipelines/animatediff/test_animatediff_video2video.py new file mode 100755 index 0000000..6cc54d9 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/animatediff/test_animatediff_video2video.py @@ -0,0 +1,304 @@ +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +import diffusers +from diffusers import ( + AnimateDiffVideoToVideoPipeline, + AutoencoderKL, + DDIMScheduler, + MotionAdapter, + UNet2DConditionModel, + UNetMotionModel, +) +from diffusers.utils import is_xformers_available, logging +from diffusers.utils.testing_utils import torch_device + +from ..pipeline_params import TEXT_TO_IMAGE_PARAMS, VIDEO_TO_VIDEO_BATCH_PARAMS +from ..test_pipelines_common import IPAdapterTesterMixin, PipelineTesterMixin + + +def to_np(tensor): + if isinstance(tensor, torch.Tensor): + tensor = tensor.detach().cpu().numpy() + + return tensor + + +class AnimateDiffVideoToVideoPipelineFastTests(IPAdapterTesterMixin, PipelineTesterMixin, unittest.TestCase): + pipeline_class = AnimateDiffVideoToVideoPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = VIDEO_TO_VIDEO_BATCH_PARAMS + required_optional_params = frozenset( + [ + "num_inference_steps", + "generator", + "latents", + "return_dict", + "callback_on_step_end", + "callback_on_step_end_tensor_inputs", + ] + ) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("CrossAttnDownBlock2D", "DownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + norm_num_groups=2, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="linear", + clip_sample=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + motion_adapter = MotionAdapter( + block_out_channels=(32, 64), + motion_layers_per_block=2, + motion_norm_num_groups=2, + motion_num_attention_heads=4, + ) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "motion_adapter": motion_adapter, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + video_height = 32 + video_width = 32 + video_num_frames = 2 + video = [Image.new("RGB", (video_width, video_height))] * video_num_frames + + inputs = { + "video": video, + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 7.5, + "output_type": "pt", + } + return inputs + + def test_motion_unet_loading(self): + components = self.get_dummy_components() + pipe = AnimateDiffVideoToVideoPipeline(**components) + + assert isinstance(pipe.unet, UNetMotionModel) + + @unittest.skip("Attention slicing is not enabled in this pipeline") + def test_attention_slicing_forward_pass(self): + pass + + def test_inference_batch_single_identical( + self, + batch_size=2, + expected_max_diff=1e-4, + additional_params_copy_to_batched_inputs=["num_inference_steps"], + ): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for components in pipe.components.values(): + if hasattr(components, "set_default_attn_processor"): + components.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + inputs = self.get_dummy_inputs(torch_device) + # Reset generator in case it is has been used in self.get_dummy_inputs + inputs["generator"] = self.get_generator(0) + + logger = logging.get_logger(pipe.__module__) + logger.setLevel(level=diffusers.logging.FATAL) + + # batchify inputs + batched_inputs = {} + batched_inputs.update(inputs) + + for name in self.batch_params: + if name not in inputs: + continue + + value = inputs[name] + if name == "prompt": + len_prompt = len(value) + batched_inputs[name] = [value[: len_prompt // i] for i in range(1, batch_size + 1)] + batched_inputs[name][-1] = 100 * "very long" + + else: + batched_inputs[name] = batch_size * [value] + + if "generator" in inputs: + batched_inputs["generator"] = [self.get_generator(i) for i in range(batch_size)] + + if "batch_size" in inputs: + batched_inputs["batch_size"] = batch_size + + for arg in additional_params_copy_to_batched_inputs: + batched_inputs[arg] = inputs[arg] + + output = pipe(**inputs) + output_batch = pipe(**batched_inputs) + + assert output_batch[0].shape[0] == batch_size + + max_diff = np.abs(to_np(output_batch[0][0]) - to_np(output[0][0])).max() + assert max_diff < expected_max_diff + + @unittest.skipIf(torch_device != "cuda", reason="CUDA and CPU are required to switch devices") + def test_to_device(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + pipe.to("cpu") + # pipeline creates a new motion UNet under the hood. So we need to check the device from pipe.components + model_devices = [ + component.device.type for component in pipe.components.values() if hasattr(component, "device") + ] + self.assertTrue(all(device == "cpu" for device in model_devices)) + + output_cpu = pipe(**self.get_dummy_inputs("cpu"))[0] + self.assertTrue(np.isnan(output_cpu).sum() == 0) + + pipe.to("cuda") + model_devices = [ + component.device.type for component in pipe.components.values() if hasattr(component, "device") + ] + self.assertTrue(all(device == "cuda" for device in model_devices)) + + output_cuda = pipe(**self.get_dummy_inputs("cuda"))[0] + self.assertTrue(np.isnan(to_np(output_cuda)).sum() == 0) + + def test_to_dtype(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + # pipeline creates a new motion UNet under the hood. So we need to check the dtype from pipe.components + model_dtypes = [component.dtype for component in pipe.components.values() if hasattr(component, "dtype")] + self.assertTrue(all(dtype == torch.float32 for dtype in model_dtypes)) + + pipe.to(dtype=torch.float16) + model_dtypes = [component.dtype for component in pipe.components.values() if hasattr(component, "dtype")] + self.assertTrue(all(dtype == torch.float16 for dtype in model_dtypes)) + + def test_prompt_embeds(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + inputs = self.get_dummy_inputs(torch_device) + inputs.pop("prompt") + inputs["prompt_embeds"] = torch.randn((1, 4, 32), device=torch_device) + pipe(**inputs) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output_without_offload = pipe(**inputs).frames[0] + output_without_offload = ( + output_without_offload.cpu() if torch.is_tensor(output_without_offload) else output_without_offload + ) + + pipe.enable_xformers_memory_efficient_attention() + inputs = self.get_dummy_inputs(torch_device) + output_with_offload = pipe(**inputs).frames[0] + output_with_offload = ( + output_with_offload.cpu() if torch.is_tensor(output_with_offload) else output_without_offload + ) + + max_diff = np.abs(to_np(output_with_offload) - to_np(output_without_offload)).max() + self.assertLess(max_diff, 1e-4, "XFormers attention should not affect the inference results") + + def test_free_init(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + inputs_normal = self.get_dummy_inputs(torch_device) + frames_normal = pipe(**inputs_normal).frames[0] + + pipe.enable_free_init( + num_iters=2, + use_fast_sampling=True, + method="butterworth", + order=4, + spatial_stop_frequency=0.25, + temporal_stop_frequency=0.25, + ) + inputs_enable_free_init = self.get_dummy_inputs(torch_device) + frames_enable_free_init = pipe(**inputs_enable_free_init).frames[0] + + pipe.disable_free_init() + inputs_disable_free_init = self.get_dummy_inputs(torch_device) + frames_disable_free_init = pipe(**inputs_disable_free_init).frames[0] + + sum_enabled = np.abs(to_np(frames_normal) - to_np(frames_enable_free_init)).sum() + max_diff_disabled = np.abs(to_np(frames_normal) - to_np(frames_disable_free_init)).max() + self.assertGreater( + sum_enabled, 1e1, "Enabling of FreeInit should lead to results different from the default pipeline results" + ) + self.assertLess( + max_diff_disabled, + 1e-4, + "Disabling of FreeInit should lead to results similar to the default pipeline results", + ) diff --git a/diffusers-0.27.0/tests/pipelines/audioldm/__init__.py b/diffusers-0.27.0/tests/pipelines/audioldm/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/audioldm/test_audioldm.py b/diffusers-0.27.0/tests/pipelines/audioldm/test_audioldm.py new file mode 100755 index 0000000..84b5788 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/audioldm/test_audioldm.py @@ -0,0 +1,447 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import gc +import unittest + +import numpy as np +import torch +import torch.nn.functional as F +from transformers import ( + ClapTextConfig, + ClapTextModelWithProjection, + RobertaTokenizer, + SpeechT5HifiGan, + SpeechT5HifiGanConfig, +) + +from diffusers import ( + AudioLDMPipeline, + AutoencoderKL, + DDIMScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + UNet2DConditionModel, +) +from diffusers.utils import is_xformers_available +from diffusers.utils.testing_utils import enable_full_determinism, nightly, torch_device + +from ..pipeline_params import TEXT_TO_AUDIO_BATCH_PARAMS, TEXT_TO_AUDIO_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class AudioLDMPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = AudioLDMPipeline + params = TEXT_TO_AUDIO_PARAMS + batch_params = TEXT_TO_AUDIO_BATCH_PARAMS + required_optional_params = frozenset( + [ + "num_inference_steps", + "num_waveforms_per_prompt", + "generator", + "latents", + "output_type", + "return_dict", + "callback", + "callback_steps", + ] + ) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=(32, 64), + class_embed_type="simple_projection", + projection_class_embeddings_input_dim=32, + class_embeddings_concat=True, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=1, + out_channels=1, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = ClapTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + projection_dim=32, + ) + text_encoder = ClapTextModelWithProjection(text_encoder_config) + tokenizer = RobertaTokenizer.from_pretrained("hf-internal-testing/tiny-random-roberta", model_max_length=77) + + vocoder_config = SpeechT5HifiGanConfig( + model_in_dim=8, + sampling_rate=16000, + upsample_initial_channel=16, + upsample_rates=[2, 2], + upsample_kernel_sizes=[4, 4], + resblock_kernel_sizes=[3, 7], + resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5]], + normalize_before=False, + ) + + vocoder = SpeechT5HifiGan(vocoder_config) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "vocoder": vocoder, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A hammer hitting a wooden surface", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + } + return inputs + + def test_audioldm_ddim(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + audioldm_pipe = AudioLDMPipeline(**components) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = audioldm_pipe(**inputs) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) == 256 + + audio_slice = audio[:10] + expected_slice = np.array( + [-0.0050, 0.0050, -0.0060, 0.0033, -0.0026, 0.0033, -0.0027, 0.0033, -0.0028, 0.0033] + ) + + assert np.abs(audio_slice - expected_slice).max() < 1e-2 + + def test_audioldm_prompt_embeds(self): + components = self.get_dummy_components() + audioldm_pipe = AudioLDMPipeline(**components) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = audioldm_pipe(**inputs) + audio_1 = output.audios[0] + + inputs = self.get_dummy_inputs(torch_device) + prompt = 3 * [inputs.pop("prompt")] + + text_inputs = audioldm_pipe.tokenizer( + prompt, + padding="max_length", + max_length=audioldm_pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + prompt_embeds = audioldm_pipe.text_encoder( + text_inputs, + ) + prompt_embeds = prompt_embeds.text_embeds + # additional L_2 normalization over each hidden-state + prompt_embeds = F.normalize(prompt_embeds, dim=-1) + + inputs["prompt_embeds"] = prompt_embeds + + # forward + output = audioldm_pipe(**inputs) + audio_2 = output.audios[0] + + assert np.abs(audio_1 - audio_2).max() < 1e-2 + + def test_audioldm_negative_prompt_embeds(self): + components = self.get_dummy_components() + audioldm_pipe = AudioLDMPipeline(**components) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + negative_prompt = 3 * ["this is a negative prompt"] + inputs["negative_prompt"] = negative_prompt + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = audioldm_pipe(**inputs) + audio_1 = output.audios[0] + + inputs = self.get_dummy_inputs(torch_device) + prompt = 3 * [inputs.pop("prompt")] + + embeds = [] + for p in [prompt, negative_prompt]: + text_inputs = audioldm_pipe.tokenizer( + p, + padding="max_length", + max_length=audioldm_pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + text_embeds = audioldm_pipe.text_encoder( + text_inputs, + ) + text_embeds = text_embeds.text_embeds + # additional L_2 normalization over each hidden-state + text_embeds = F.normalize(text_embeds, dim=-1) + + embeds.append(text_embeds) + + inputs["prompt_embeds"], inputs["negative_prompt_embeds"] = embeds + + # forward + output = audioldm_pipe(**inputs) + audio_2 = output.audios[0] + + assert np.abs(audio_1 - audio_2).max() < 1e-2 + + def test_audioldm_negative_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = PNDMScheduler(skip_prk_steps=True) + audioldm_pipe = AudioLDMPipeline(**components) + audioldm_pipe = audioldm_pipe.to(device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + negative_prompt = "egg cracking" + output = audioldm_pipe(**inputs, negative_prompt=negative_prompt) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) == 256 + + audio_slice = audio[:10] + expected_slice = np.array( + [-0.0051, 0.0050, -0.0060, 0.0034, -0.0026, 0.0033, -0.0027, 0.0033, -0.0028, 0.0032] + ) + + assert np.abs(audio_slice - expected_slice).max() < 1e-2 + + def test_audioldm_num_waveforms_per_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = PNDMScheduler(skip_prk_steps=True) + audioldm_pipe = AudioLDMPipeline(**components) + audioldm_pipe = audioldm_pipe.to(device) + audioldm_pipe.set_progress_bar_config(disable=None) + + prompt = "A hammer hitting a wooden surface" + + # test num_waveforms_per_prompt=1 (default) + audios = audioldm_pipe(prompt, num_inference_steps=2).audios + + assert audios.shape == (1, 256) + + # test num_waveforms_per_prompt=1 (default) for batch of prompts + batch_size = 2 + audios = audioldm_pipe([prompt] * batch_size, num_inference_steps=2).audios + + assert audios.shape == (batch_size, 256) + + # test num_waveforms_per_prompt for single prompt + num_waveforms_per_prompt = 2 + audios = audioldm_pipe(prompt, num_inference_steps=2, num_waveforms_per_prompt=num_waveforms_per_prompt).audios + + assert audios.shape == (num_waveforms_per_prompt, 256) + + # test num_waveforms_per_prompt for batch of prompts + batch_size = 2 + audios = audioldm_pipe( + [prompt] * batch_size, num_inference_steps=2, num_waveforms_per_prompt=num_waveforms_per_prompt + ).audios + + assert audios.shape == (batch_size * num_waveforms_per_prompt, 256) + + def test_audioldm_audio_length_in_s(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + audioldm_pipe = AudioLDMPipeline(**components) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + vocoder_sampling_rate = audioldm_pipe.vocoder.config.sampling_rate + + inputs = self.get_dummy_inputs(device) + output = audioldm_pipe(audio_length_in_s=0.016, **inputs) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) / vocoder_sampling_rate == 0.016 + + output = audioldm_pipe(audio_length_in_s=0.032, **inputs) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) / vocoder_sampling_rate == 0.032 + + def test_audioldm_vocoder_model_in_dim(self): + components = self.get_dummy_components() + audioldm_pipe = AudioLDMPipeline(**components) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + prompt = ["hey"] + + output = audioldm_pipe(prompt, num_inference_steps=1) + audio_shape = output.audios.shape + assert audio_shape == (1, 256) + + config = audioldm_pipe.vocoder.config + config.model_in_dim *= 2 + audioldm_pipe.vocoder = SpeechT5HifiGan(config).to(torch_device) + output = audioldm_pipe(prompt, num_inference_steps=1) + audio_shape = output.audios.shape + # waveform shape is unchanged, we just have 2x the number of mel channels in the spectrogram + assert audio_shape == (1, 256) + + def test_attention_slicing_forward_pass(self): + self._test_attention_slicing_forward_pass(test_mean_pixel_difference=False) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical() + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(test_mean_pixel_difference=False) + + +@nightly +class AudioLDMPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 8, 128, 16)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "A hammer hitting a wooden surface", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 2.5, + } + return inputs + + def test_audioldm(self): + audioldm_pipe = AudioLDMPipeline.from_pretrained("cvssp/audioldm") + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 25 + audio = audioldm_pipe(**inputs).audios[0] + + assert audio.ndim == 1 + assert len(audio) == 81920 + + audio_slice = audio[77230:77240] + expected_slice = np.array( + [-0.4884, -0.4607, 0.0023, 0.5007, 0.5896, 0.5151, 0.3813, -0.0208, -0.3687, -0.4315] + ) + max_diff = np.abs(expected_slice - audio_slice).max() + assert max_diff < 1e-2 + + +@nightly +class AudioLDMPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 8, 128, 16)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "A hammer hitting a wooden surface", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 2.5, + } + return inputs + + def test_audioldm_lms(self): + audioldm_pipe = AudioLDMPipeline.from_pretrained("cvssp/audioldm") + audioldm_pipe.scheduler = LMSDiscreteScheduler.from_config(audioldm_pipe.scheduler.config) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + audio = audioldm_pipe(**inputs).audios[0] + + assert audio.ndim == 1 + assert len(audio) == 81920 + + audio_slice = audio[27780:27790] + expected_slice = np.array([-0.2131, -0.0873, -0.0124, -0.0189, 0.0569, 0.1373, 0.1883, 0.2886, 0.3297, 0.2212]) + max_diff = np.abs(expected_slice - audio_slice).max() + assert max_diff < 3e-2 diff --git a/diffusers-0.27.0/tests/pipelines/audioldm2/__init__.py b/diffusers-0.27.0/tests/pipelines/audioldm2/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/audioldm2/test_audioldm2.py b/diffusers-0.27.0/tests/pipelines/audioldm2/test_audioldm2.py new file mode 100755 index 0000000..58b1aef --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/audioldm2/test_audioldm2.py @@ -0,0 +1,569 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import gc +import unittest + +import numpy as np +import torch +from transformers import ( + ClapAudioConfig, + ClapConfig, + ClapFeatureExtractor, + ClapModel, + ClapTextConfig, + GPT2Config, + GPT2Model, + RobertaTokenizer, + SpeechT5HifiGan, + SpeechT5HifiGanConfig, + T5Config, + T5EncoderModel, + T5Tokenizer, +) + +from diffusers import ( + AudioLDM2Pipeline, + AudioLDM2ProjectionModel, + AudioLDM2UNet2DConditionModel, + AutoencoderKL, + DDIMScheduler, + LMSDiscreteScheduler, + PNDMScheduler, +) +from diffusers.utils.testing_utils import enable_full_determinism, nightly, torch_device + +from ..pipeline_params import TEXT_TO_AUDIO_BATCH_PARAMS, TEXT_TO_AUDIO_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class AudioLDM2PipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = AudioLDM2Pipeline + params = TEXT_TO_AUDIO_PARAMS + batch_params = TEXT_TO_AUDIO_BATCH_PARAMS + required_optional_params = frozenset( + [ + "num_inference_steps", + "num_waveforms_per_prompt", + "generator", + "latents", + "output_type", + "return_dict", + "callback", + "callback_steps", + ] + ) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = AudioLDM2UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=([None, 16, 32], [None, 16, 32]), + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=1, + out_channels=1, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_branch_config = ClapTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=16, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=2, + num_hidden_layers=2, + pad_token_id=1, + vocab_size=1000, + projection_dim=16, + ) + audio_branch_config = ClapAudioConfig( + spec_size=64, + window_size=4, + num_mel_bins=64, + intermediate_size=37, + layer_norm_eps=1e-05, + depths=[2, 2], + num_attention_heads=[2, 2], + num_hidden_layers=2, + hidden_size=192, + projection_dim=16, + patch_size=2, + patch_stride=2, + patch_embed_input_channels=4, + ) + text_encoder_config = ClapConfig.from_text_audio_configs( + text_config=text_branch_config, audio_config=audio_branch_config, projection_dim=16 + ) + text_encoder = ClapModel(text_encoder_config) + tokenizer = RobertaTokenizer.from_pretrained("hf-internal-testing/tiny-random-roberta", model_max_length=77) + feature_extractor = ClapFeatureExtractor.from_pretrained( + "hf-internal-testing/tiny-random-ClapModel", hop_length=7900 + ) + + torch.manual_seed(0) + text_encoder_2_config = T5Config( + vocab_size=32100, + d_model=32, + d_ff=37, + d_kv=8, + num_heads=2, + num_layers=2, + ) + text_encoder_2 = T5EncoderModel(text_encoder_2_config) + tokenizer_2 = T5Tokenizer.from_pretrained("hf-internal-testing/tiny-random-T5Model", model_max_length=77) + + torch.manual_seed(0) + language_model_config = GPT2Config( + n_embd=16, + n_head=2, + n_layer=2, + vocab_size=1000, + n_ctx=99, + n_positions=99, + ) + language_model = GPT2Model(language_model_config) + language_model.config.max_new_tokens = 8 + + torch.manual_seed(0) + projection_model = AudioLDM2ProjectionModel(text_encoder_dim=16, text_encoder_1_dim=32, langauge_model_dim=16) + + vocoder_config = SpeechT5HifiGanConfig( + model_in_dim=8, + sampling_rate=16000, + upsample_initial_channel=16, + upsample_rates=[2, 2], + upsample_kernel_sizes=[4, 4], + resblock_kernel_sizes=[3, 7], + resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5]], + normalize_before=False, + ) + + vocoder = SpeechT5HifiGan(vocoder_config) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "text_encoder_2": text_encoder_2, + "tokenizer": tokenizer, + "tokenizer_2": tokenizer_2, + "feature_extractor": feature_extractor, + "language_model": language_model, + "projection_model": projection_model, + "vocoder": vocoder, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A hammer hitting a wooden surface", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + } + return inputs + + def test_audioldm2_ddim(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + audioldm_pipe = AudioLDM2Pipeline(**components) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = audioldm_pipe(**inputs) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) == 256 + + audio_slice = audio[:10] + expected_slice = np.array( + [0.0025, 0.0018, 0.0018, -0.0023, -0.0026, -0.0020, -0.0026, -0.0021, -0.0027, -0.0020] + ) + + assert np.abs(audio_slice - expected_slice).max() < 1e-4 + + def test_audioldm2_prompt_embeds(self): + components = self.get_dummy_components() + audioldm_pipe = AudioLDM2Pipeline(**components) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = audioldm_pipe(**inputs) + audio_1 = output.audios[0] + + inputs = self.get_dummy_inputs(torch_device) + prompt = 3 * [inputs.pop("prompt")] + + text_inputs = audioldm_pipe.tokenizer( + prompt, + padding="max_length", + max_length=audioldm_pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + clap_prompt_embeds = audioldm_pipe.text_encoder.get_text_features(text_inputs) + clap_prompt_embeds = clap_prompt_embeds[:, None, :] + + text_inputs = audioldm_pipe.tokenizer_2( + prompt, + padding="max_length", + max_length=True, + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + t5_prompt_embeds = audioldm_pipe.text_encoder_2( + text_inputs, + ) + t5_prompt_embeds = t5_prompt_embeds[0] + + projection_embeds = audioldm_pipe.projection_model(clap_prompt_embeds, t5_prompt_embeds)[0] + generated_prompt_embeds = audioldm_pipe.generate_language_model(projection_embeds, max_new_tokens=8) + + inputs["prompt_embeds"] = t5_prompt_embeds + inputs["generated_prompt_embeds"] = generated_prompt_embeds + + # forward + output = audioldm_pipe(**inputs) + audio_2 = output.audios[0] + + assert np.abs(audio_1 - audio_2).max() < 1e-2 + + def test_audioldm2_negative_prompt_embeds(self): + components = self.get_dummy_components() + audioldm_pipe = AudioLDM2Pipeline(**components) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + negative_prompt = 3 * ["this is a negative prompt"] + inputs["negative_prompt"] = negative_prompt + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = audioldm_pipe(**inputs) + audio_1 = output.audios[0] + + inputs = self.get_dummy_inputs(torch_device) + prompt = 3 * [inputs.pop("prompt")] + + embeds = [] + generated_embeds = [] + for p in [prompt, negative_prompt]: + text_inputs = audioldm_pipe.tokenizer( + p, + padding="max_length", + max_length=audioldm_pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + clap_prompt_embeds = audioldm_pipe.text_encoder.get_text_features(text_inputs) + clap_prompt_embeds = clap_prompt_embeds[:, None, :] + + text_inputs = audioldm_pipe.tokenizer_2( + prompt, + padding="max_length", + max_length=True if len(embeds) == 0 else embeds[0].shape[1], + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + t5_prompt_embeds = audioldm_pipe.text_encoder_2( + text_inputs, + ) + t5_prompt_embeds = t5_prompt_embeds[0] + + projection_embeds = audioldm_pipe.projection_model(clap_prompt_embeds, t5_prompt_embeds)[0] + generated_prompt_embeds = audioldm_pipe.generate_language_model(projection_embeds, max_new_tokens=8) + + embeds.append(t5_prompt_embeds) + generated_embeds.append(generated_prompt_embeds) + + inputs["prompt_embeds"], inputs["negative_prompt_embeds"] = embeds + inputs["generated_prompt_embeds"], inputs["negative_generated_prompt_embeds"] = generated_embeds + + # forward + output = audioldm_pipe(**inputs) + audio_2 = output.audios[0] + + assert np.abs(audio_1 - audio_2).max() < 1e-2 + + def test_audioldm2_negative_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = PNDMScheduler(skip_prk_steps=True) + audioldm_pipe = AudioLDM2Pipeline(**components) + audioldm_pipe = audioldm_pipe.to(device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + negative_prompt = "egg cracking" + output = audioldm_pipe(**inputs, negative_prompt=negative_prompt) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) == 256 + + audio_slice = audio[:10] + expected_slice = np.array( + [0.0025, 0.0018, 0.0018, -0.0023, -0.0026, -0.0020, -0.0026, -0.0021, -0.0027, -0.0020] + ) + + assert np.abs(audio_slice - expected_slice).max() < 1e-4 + + def test_audioldm2_num_waveforms_per_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = PNDMScheduler(skip_prk_steps=True) + audioldm_pipe = AudioLDM2Pipeline(**components) + audioldm_pipe = audioldm_pipe.to(device) + audioldm_pipe.set_progress_bar_config(disable=None) + + prompt = "A hammer hitting a wooden surface" + + # test num_waveforms_per_prompt=1 (default) + audios = audioldm_pipe(prompt, num_inference_steps=2).audios + + assert audios.shape == (1, 256) + + # test num_waveforms_per_prompt=1 (default) for batch of prompts + batch_size = 2 + audios = audioldm_pipe([prompt] * batch_size, num_inference_steps=2).audios + + assert audios.shape == (batch_size, 256) + + # test num_waveforms_per_prompt for single prompt + num_waveforms_per_prompt = 2 + audios = audioldm_pipe(prompt, num_inference_steps=2, num_waveforms_per_prompt=num_waveforms_per_prompt).audios + + assert audios.shape == (num_waveforms_per_prompt, 256) + + # test num_waveforms_per_prompt for batch of prompts + batch_size = 2 + audios = audioldm_pipe( + [prompt] * batch_size, num_inference_steps=2, num_waveforms_per_prompt=num_waveforms_per_prompt + ).audios + + assert audios.shape == (batch_size * num_waveforms_per_prompt, 256) + + def test_audioldm2_audio_length_in_s(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + audioldm_pipe = AudioLDM2Pipeline(**components) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + vocoder_sampling_rate = audioldm_pipe.vocoder.config.sampling_rate + + inputs = self.get_dummy_inputs(device) + output = audioldm_pipe(audio_length_in_s=0.016, **inputs) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) / vocoder_sampling_rate == 0.016 + + output = audioldm_pipe(audio_length_in_s=0.032, **inputs) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) / vocoder_sampling_rate == 0.032 + + def test_audioldm2_vocoder_model_in_dim(self): + components = self.get_dummy_components() + audioldm_pipe = AudioLDM2Pipeline(**components) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + prompt = ["hey"] + + output = audioldm_pipe(prompt, num_inference_steps=1) + audio_shape = output.audios.shape + assert audio_shape == (1, 256) + + config = audioldm_pipe.vocoder.config + config.model_in_dim *= 2 + audioldm_pipe.vocoder = SpeechT5HifiGan(config).to(torch_device) + output = audioldm_pipe(prompt, num_inference_steps=1) + audio_shape = output.audios.shape + # waveform shape is unchanged, we just have 2x the number of mel channels in the spectrogram + assert audio_shape == (1, 256) + + def test_attention_slicing_forward_pass(self): + self._test_attention_slicing_forward_pass(test_mean_pixel_difference=False) + + @unittest.skip("Raises a not implemented error in AudioLDM2") + def test_xformers_attention_forwardGenerator_pass(self): + pass + + def test_dict_tuple_outputs_equivalent(self): + # increase tolerance from 1e-4 -> 2e-4 to account for large composite model + super().test_dict_tuple_outputs_equivalent(expected_max_difference=2e-4) + + def test_inference_batch_single_identical(self): + # increase tolerance from 1e-4 -> 2e-4 to account for large composite model + self._test_inference_batch_single_identical(expected_max_diff=2e-4) + + def test_save_load_local(self): + # increase tolerance from 1e-4 -> 2e-4 to account for large composite model + super().test_save_load_local(expected_max_difference=2e-4) + + def test_save_load_optional_components(self): + # increase tolerance from 1e-4 -> 2e-4 to account for large composite model + super().test_save_load_optional_components(expected_max_difference=2e-4) + + def test_to_dtype(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + # The method component.dtype returns the dtype of the first parameter registered in the model, not the + # dtype of the entire model. In the case of CLAP, the first parameter is a float64 constant (logit scale) + model_dtypes = {key: component.dtype for key, component in components.items() if hasattr(component, "dtype")} + + # Without the logit scale parameters, everything is float32 + model_dtypes.pop("text_encoder") + self.assertTrue(all(dtype == torch.float32 for dtype in model_dtypes.values())) + + # the CLAP sub-models are float32 + model_dtypes["clap_text_branch"] = components["text_encoder"].text_model.dtype + self.assertTrue(all(dtype == torch.float32 for dtype in model_dtypes.values())) + + # Once we send to fp16, all params are in half-precision, including the logit scale + pipe.to(dtype=torch.float16) + model_dtypes = {key: component.dtype for key, component in components.items() if hasattr(component, "dtype")} + self.assertTrue(all(dtype == torch.float16 for dtype in model_dtypes.values())) + + def test_sequential_cpu_offload_forward_pass(self): + pass + + +@nightly +class AudioLDM2PipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 8, 128, 16)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "A hammer hitting a wooden surface", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 2.5, + } + return inputs + + def test_audioldm2(self): + audioldm_pipe = AudioLDM2Pipeline.from_pretrained("cvssp/audioldm2") + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 25 + audio = audioldm_pipe(**inputs).audios[0] + + assert audio.ndim == 1 + assert len(audio) == 81952 + + # check the portion of the generated audio with the largest dynamic range (reduces flakiness) + audio_slice = audio[17275:17285] + expected_slice = np.array([0.0791, 0.0666, 0.1158, 0.1227, 0.1171, -0.2880, -0.1940, -0.0283, -0.0126, 0.1127]) + max_diff = np.abs(expected_slice - audio_slice).max() + assert max_diff < 1e-3 + + def test_audioldm2_lms(self): + audioldm_pipe = AudioLDM2Pipeline.from_pretrained("cvssp/audioldm2") + audioldm_pipe.scheduler = LMSDiscreteScheduler.from_config(audioldm_pipe.scheduler.config) + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + audio = audioldm_pipe(**inputs).audios[0] + + assert audio.ndim == 1 + assert len(audio) == 81952 + + # check the portion of the generated audio with the largest dynamic range (reduces flakiness) + audio_slice = audio[31390:31400] + expected_slice = np.array( + [-0.1318, -0.0577, 0.0446, -0.0573, 0.0659, 0.1074, -0.2600, 0.0080, -0.2190, -0.4301] + ) + max_diff = np.abs(expected_slice - audio_slice).max() + assert max_diff < 1e-3 + + def test_audioldm2_large(self): + audioldm_pipe = AudioLDM2Pipeline.from_pretrained("cvssp/audioldm2-large") + audioldm_pipe = audioldm_pipe.to(torch_device) + audioldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + audio = audioldm_pipe(**inputs).audios[0] + + assert audio.ndim == 1 + assert len(audio) == 81952 + + # check the portion of the generated audio with the largest dynamic range (reduces flakiness) + audio_slice = audio[8825:8835] + expected_slice = np.array( + [-0.1829, -0.1461, 0.0759, -0.1493, -0.1396, 0.5783, 0.3001, -0.3038, -0.0639, -0.2244] + ) + max_diff = np.abs(expected_slice - audio_slice).max() + assert max_diff < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/blipdiffusion/__init__.py b/diffusers-0.27.0/tests/pipelines/blipdiffusion/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/blipdiffusion/test_blipdiffusion.py b/diffusers-0.27.0/tests/pipelines/blipdiffusion/test_blipdiffusion.py new file mode 100755 index 0000000..c5eaa38 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/blipdiffusion/test_blipdiffusion.py @@ -0,0 +1,196 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPTokenizer +from transformers.models.blip_2.configuration_blip_2 import Blip2Config +from transformers.models.clip.configuration_clip import CLIPTextConfig + +from diffusers import AutoencoderKL, BlipDiffusionPipeline, PNDMScheduler, UNet2DConditionModel +from diffusers.utils.testing_utils import enable_full_determinism +from src.diffusers.pipelines.blip_diffusion.blip_image_processing import BlipImageProcessor +from src.diffusers.pipelines.blip_diffusion.modeling_blip2 import Blip2QFormerModel +from src.diffusers.pipelines.blip_diffusion.modeling_ctx_clip import ContextCLIPTextModel + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class BlipDiffusionPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = BlipDiffusionPipeline + params = [ + "prompt", + "reference_image", + "source_subject_category", + "target_subject_category", + ] + batch_params = [ + "prompt", + "reference_image", + "source_subject_category", + "target_subject_category", + ] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "num_inference_steps", + "neg_prompt", + "guidance_scale", + "prompt_strength", + "prompt_reps", + ] + + def get_dummy_components(self): + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + vocab_size=1000, + hidden_size=16, + intermediate_size=16, + projection_dim=16, + num_hidden_layers=1, + num_attention_heads=1, + max_position_embeddings=77, + ) + text_encoder = ContextCLIPTextModel(text_encoder_config) + + vae = AutoencoderKL( + in_channels=4, + out_channels=4, + down_block_types=("DownEncoderBlock2D",), + up_block_types=("UpDecoderBlock2D",), + block_out_channels=(32,), + layers_per_block=1, + act_fn="silu", + latent_channels=4, + norm_num_groups=16, + sample_size=16, + ) + + blip_vision_config = { + "hidden_size": 16, + "intermediate_size": 16, + "num_hidden_layers": 1, + "num_attention_heads": 1, + "image_size": 224, + "patch_size": 14, + "hidden_act": "quick_gelu", + } + + blip_qformer_config = { + "vocab_size": 1000, + "hidden_size": 16, + "num_hidden_layers": 1, + "num_attention_heads": 1, + "intermediate_size": 16, + "max_position_embeddings": 512, + "cross_attention_frequency": 1, + "encoder_hidden_size": 16, + } + qformer_config = Blip2Config( + vision_config=blip_vision_config, + qformer_config=blip_qformer_config, + num_query_tokens=16, + tokenizer="hf-internal-testing/tiny-random-bert", + ) + qformer = Blip2QFormerModel(qformer_config) + + unet = UNet2DConditionModel( + block_out_channels=(16, 32), + norm_num_groups=16, + layers_per_block=1, + sample_size=16, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=16, + ) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + scheduler = PNDMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + set_alpha_to_one=False, + skip_prk_steps=True, + ) + + vae.eval() + qformer.eval() + text_encoder.eval() + + image_processor = BlipImageProcessor() + + components = { + "text_encoder": text_encoder, + "vae": vae, + "qformer": qformer, + "unet": unet, + "tokenizer": tokenizer, + "scheduler": scheduler, + "image_processor": image_processor, + } + return components + + def get_dummy_inputs(self, device, seed=0): + np.random.seed(seed) + reference_image = np.random.rand(32, 32, 3) * 255 + reference_image = Image.fromarray(reference_image.astype("uint8")).convert("RGBA") + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "swimming underwater", + "generator": generator, + "reference_image": reference_image, + "source_subject_category": "dog", + "target_subject_category": "dog", + "height": 32, + "width": 32, + "guidance_scale": 7.5, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + def test_blipdiffusion(self): + device = "cpu" + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + image = pipe(**self.get_dummy_inputs(device))[0] + image_slice = image[0, -3:, -3:, 0] + + assert image.shape == (1, 16, 16, 4) + + expected_slice = np.array([0.7096, 0.5900, 0.6703, 0.4032, 0.7766, 0.3629, 0.5447, 0.4149, 0.8172]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {image_slice.flatten()}, but got {image_slice.flatten()}" diff --git a/diffusers-0.27.0/tests/pipelines/consistency_models/__init__.py b/diffusers-0.27.0/tests/pipelines/consistency_models/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/consistency_models/test_consistency_models.py b/diffusers-0.27.0/tests/pipelines/consistency_models/test_consistency_models.py new file mode 100755 index 0000000..2cf7c0a --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/consistency_models/test_consistency_models.py @@ -0,0 +1,294 @@ +import gc +import unittest + +import numpy as np +import torch +from torch.backends.cuda import sdp_kernel + +from diffusers import ( + CMStochasticIterativeScheduler, + ConsistencyModelPipeline, + UNet2DModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + nightly, + require_torch_2, + require_torch_gpu, + torch_device, +) +from diffusers.utils.torch_utils import randn_tensor + +from ..pipeline_params import UNCONDITIONAL_IMAGE_GENERATION_BATCH_PARAMS, UNCONDITIONAL_IMAGE_GENERATION_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class ConsistencyModelPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = ConsistencyModelPipeline + params = UNCONDITIONAL_IMAGE_GENERATION_PARAMS + batch_params = UNCONDITIONAL_IMAGE_GENERATION_BATCH_PARAMS + + # Override required_optional_params to remove num_images_per_prompt + required_optional_params = frozenset( + [ + "num_inference_steps", + "generator", + "latents", + "output_type", + "return_dict", + "callback", + "callback_steps", + ] + ) + + @property + def dummy_uncond_unet(self): + unet = UNet2DModel.from_pretrained( + "diffusers/consistency-models-test", + subfolder="test_unet", + ) + return unet + + @property + def dummy_cond_unet(self): + unet = UNet2DModel.from_pretrained( + "diffusers/consistency-models-test", + subfolder="test_unet_class_cond", + ) + return unet + + def get_dummy_components(self, class_cond=False): + if class_cond: + unet = self.dummy_cond_unet + else: + unet = self.dummy_uncond_unet + + # Default to CM multistep sampler + scheduler = CMStochasticIterativeScheduler( + num_train_timesteps=40, + sigma_min=0.002, + sigma_max=80.0, + ) + + components = { + "unet": unet, + "scheduler": scheduler, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + inputs = { + "batch_size": 1, + "num_inference_steps": None, + "timesteps": [22, 0], + "generator": generator, + "output_type": "np", + } + + return inputs + + def test_consistency_model_pipeline_multistep(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = ConsistencyModelPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = pipe(**inputs).images + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.3572, 0.6273, 0.4031, 0.3961, 0.4321, 0.5730, 0.5266, 0.4780, 0.5004]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_consistency_model_pipeline_multistep_class_cond(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(class_cond=True) + pipe = ConsistencyModelPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["class_labels"] = 0 + image = pipe(**inputs).images + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.3572, 0.6273, 0.4031, 0.3961, 0.4321, 0.5730, 0.5266, 0.4780, 0.5004]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_consistency_model_pipeline_onestep(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = ConsistencyModelPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["num_inference_steps"] = 1 + inputs["timesteps"] = None + image = pipe(**inputs).images + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.5004, 0.5004, 0.4994, 0.5008, 0.4976, 0.5018, 0.4990, 0.4982, 0.4987]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_consistency_model_pipeline_onestep_class_cond(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(class_cond=True) + pipe = ConsistencyModelPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["num_inference_steps"] = 1 + inputs["timesteps"] = None + inputs["class_labels"] = 0 + image = pipe(**inputs).images + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.5004, 0.5004, 0.4994, 0.5008, 0.4976, 0.5018, 0.4990, 0.4982, 0.4987]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + +@nightly +@require_torch_gpu +class ConsistencyModelPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, seed=0, get_fixed_latents=False, device="cpu", dtype=torch.float32, shape=(1, 3, 64, 64)): + generator = torch.manual_seed(seed) + + inputs = { + "num_inference_steps": None, + "timesteps": [22, 0], + "class_labels": 0, + "generator": generator, + "output_type": "np", + } + + if get_fixed_latents: + latents = self.get_fixed_latents(seed=seed, device=device, dtype=dtype, shape=shape) + inputs["latents"] = latents + + return inputs + + def get_fixed_latents(self, seed=0, device="cpu", dtype=torch.float32, shape=(1, 3, 64, 64)): + if isinstance(device, str): + device = torch.device(device) + generator = torch.Generator(device=device).manual_seed(seed) + latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype) + return latents + + def test_consistency_model_cd_multistep(self): + unet = UNet2DModel.from_pretrained("diffusers/consistency_models", subfolder="diffusers_cd_imagenet64_l2") + scheduler = CMStochasticIterativeScheduler( + num_train_timesteps=40, + sigma_min=0.002, + sigma_max=80.0, + ) + pipe = ConsistencyModelPipeline(unet=unet, scheduler=scheduler) + pipe.to(torch_device=torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs() + image = pipe(**inputs).images + assert image.shape == (1, 64, 64, 3) + + image_slice = image[0, -3:, -3:, -1] + + expected_slice = np.array([0.0146, 0.0158, 0.0092, 0.0086, 0.0000, 0.0000, 0.0000, 0.0000, 0.0058]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_consistency_model_cd_onestep(self): + unet = UNet2DModel.from_pretrained("diffusers/consistency_models", subfolder="diffusers_cd_imagenet64_l2") + scheduler = CMStochasticIterativeScheduler( + num_train_timesteps=40, + sigma_min=0.002, + sigma_max=80.0, + ) + pipe = ConsistencyModelPipeline(unet=unet, scheduler=scheduler) + pipe.to(torch_device=torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs() + inputs["num_inference_steps"] = 1 + inputs["timesteps"] = None + image = pipe(**inputs).images + assert image.shape == (1, 64, 64, 3) + + image_slice = image[0, -3:, -3:, -1] + + expected_slice = np.array([0.0059, 0.0003, 0.0000, 0.0023, 0.0052, 0.0007, 0.0165, 0.0081, 0.0095]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + @require_torch_2 + def test_consistency_model_cd_multistep_flash_attn(self): + unet = UNet2DModel.from_pretrained("diffusers/consistency_models", subfolder="diffusers_cd_imagenet64_l2") + scheduler = CMStochasticIterativeScheduler( + num_train_timesteps=40, + sigma_min=0.002, + sigma_max=80.0, + ) + pipe = ConsistencyModelPipeline(unet=unet, scheduler=scheduler) + pipe.to(torch_device=torch_device, torch_dtype=torch.float16) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(get_fixed_latents=True, device=torch_device) + # Ensure usage of flash attention in torch 2.0 + with sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False): + image = pipe(**inputs).images + assert image.shape == (1, 64, 64, 3) + + image_slice = image[0, -3:, -3:, -1] + + expected_slice = np.array([0.1845, 0.1371, 0.1211, 0.2035, 0.1954, 0.1323, 0.1773, 0.1593, 0.1314]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + @require_torch_2 + def test_consistency_model_cd_onestep_flash_attn(self): + unet = UNet2DModel.from_pretrained("diffusers/consistency_models", subfolder="diffusers_cd_imagenet64_l2") + scheduler = CMStochasticIterativeScheduler( + num_train_timesteps=40, + sigma_min=0.002, + sigma_max=80.0, + ) + pipe = ConsistencyModelPipeline(unet=unet, scheduler=scheduler) + pipe.to(torch_device=torch_device, torch_dtype=torch.float16) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(get_fixed_latents=True, device=torch_device) + inputs["num_inference_steps"] = 1 + inputs["timesteps"] = None + # Ensure usage of flash attention in torch 2.0 + with sdp_kernel(enable_flash=True, enable_math=False, enable_mem_efficient=False): + image = pipe(**inputs).images + assert image.shape == (1, 64, 64, 3) + + image_slice = image[0, -3:, -3:, -1] + + expected_slice = np.array([0.1623, 0.2009, 0.2387, 0.1731, 0.1168, 0.1202, 0.2031, 0.1327, 0.2447]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/controlnet/__init__.py b/diffusers-0.27.0/tests/pipelines/controlnet/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet.py b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet.py new file mode 100755 index 0000000..114a36b --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet.py @@ -0,0 +1,1151 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import tempfile +import traceback +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + ControlNetModel, + DDIMScheduler, + EulerDiscreteScheduler, + LCMScheduler, + StableDiffusionControlNetPipeline, + UNet2DConditionModel, +) +from diffusers.pipelines.controlnet.pipeline_controlnet import MultiControlNetModel +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import ( + enable_full_determinism, + load_image, + load_numpy, + numpy_cosine_similarity_distance, + require_python39_or_higher, + require_torch_2, + require_torch_gpu, + run_test_in_subprocess, + slow, + torch_device, +) +from diffusers.utils.torch_utils import randn_tensor + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_BATCH_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_PARAMS, +) +from ..test_pipelines_common import ( + IPAdapterTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, +) + + +enable_full_determinism() + + +# Will be run via run_test_in_subprocess +def _test_stable_diffusion_compile(in_queue, out_queue, timeout): + error = None + try: + _ = in_queue.get(timeout=timeout) + + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.to("cuda") + pipe.set_progress_bar_config(disable=None) + + pipe.unet.to(memory_format=torch.channels_last) + pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True) + + pipe.controlnet.to(memory_format=torch.channels_last) + pipe.controlnet = torch.compile(pipe.controlnet, mode="reduce-overhead", fullgraph=True) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "bird" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ).resize((512, 512)) + + output = pipe(prompt, image, num_inference_steps=10, generator=generator, output_type="np") + image = output.images[0] + + assert image.shape == (512, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny_out_full.npy" + ) + expected_image = np.resize(expected_image, (512, 512, 3)) + + assert np.abs(expected_image - image).max() < 1.0 + + except Exception: + error = f"{traceback.format_exc()}" + + results = {"error": error} + out_queue.put(results, timeout=timeout) + out_queue.join() + + +class ControlNetPipelineFastTests( + IPAdapterTesterMixin, + PipelineLatentTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineTesterMixin, + unittest.TestCase, +): + pipeline_class = StableDiffusionControlNetPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(4, 8), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + norm_num_groups=1, + time_cond_proj_dim=time_cond_proj_dim, + ) + torch.manual_seed(0) + controlnet = ControlNetModel( + block_out_channels=(4, 8), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=32, + conditioning_embedding_out_channels=(16, 32), + norm_num_groups=1, + ) + torch.manual_seed(0) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[4, 8], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + norm_num_groups=2, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + controlnet_embedder_scale_factor = 2 + image = randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + "image": image, + } + + return inputs + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + def test_controlnet_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionControlNetPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array( + [0.52700454, 0.3930534, 0.25509018, 0.7132304, 0.53696585, 0.46568912, 0.7095368, 0.7059624, 0.4744786] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_controlnet_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionControlNetPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array( + [0.52700454, 0.3930534, 0.25509018, 0.7132304, 0.53696585, 0.46568912, 0.7095368, 0.7059624, 0.4744786] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + +class StableDiffusionMultiControlNetPipelineFastTests( + IPAdapterTesterMixin, PipelineTesterMixin, PipelineKarrasSchedulerTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionControlNetPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = frozenset([]) # TO_DO: add image_params once refactored VaeImageProcessor.preprocess + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(4, 8), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + norm_num_groups=1, + ) + torch.manual_seed(0) + + def init_weights(m): + if isinstance(m, torch.nn.Conv2d): + torch.nn.init.normal(m.weight) + m.bias.data.fill_(1.0) + + controlnet1 = ControlNetModel( + block_out_channels=(4, 8), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=32, + conditioning_embedding_out_channels=(16, 32), + norm_num_groups=1, + ) + controlnet1.controlnet_down_blocks.apply(init_weights) + + torch.manual_seed(0) + controlnet2 = ControlNetModel( + block_out_channels=(4, 8), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=32, + conditioning_embedding_out_channels=(16, 32), + norm_num_groups=1, + ) + controlnet2.controlnet_down_blocks.apply(init_weights) + + torch.manual_seed(0) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[4, 8], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + norm_num_groups=2, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + controlnet = MultiControlNetModel([controlnet1, controlnet2]) + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + controlnet_embedder_scale_factor = 2 + + images = [ + randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ), + randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ), + ] + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + "image": images, + } + + return inputs + + def test_control_guidance_switch(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + + scale = 10.0 + steps = 4 + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_1 = pipe(**inputs)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_2 = pipe(**inputs, control_guidance_start=0.1, control_guidance_end=0.2)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_3 = pipe(**inputs, control_guidance_start=[0.1, 0.3], control_guidance_end=[0.2, 0.7])[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_4 = pipe(**inputs, control_guidance_start=0.4, control_guidance_end=[0.5, 0.8])[0] + + # make sure that all outputs are different + assert np.sum(np.abs(output_1 - output_2)) > 1e-3 + assert np.sum(np.abs(output_1 - output_3)) > 1e-3 + assert np.sum(np.abs(output_1 - output_4)) > 1e-3 + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + def test_save_pretrained_raise_not_implemented_exception(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + with tempfile.TemporaryDirectory() as tmpdir: + try: + # save_pretrained is not implemented for Multi-ControlNet + pipe.save_pretrained(tmpdir) + except NotImplementedError: + pass + + def test_inference_multiple_prompt_input(self): + device = "cpu" + + components = self.get_dummy_components() + sd_pipe = StableDiffusionControlNetPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["prompt"] = [inputs["prompt"], inputs["prompt"]] + inputs["image"] = [inputs["image"], inputs["image"]] + output = sd_pipe(**inputs) + image = output.images + + assert image.shape == (2, 64, 64, 3) + + image_1, image_2 = image + # make sure that the outputs are different + assert np.sum(np.abs(image_1 - image_2)) > 1e-3 + + # multiple prompts, single image conditioning + inputs = self.get_dummy_inputs(device) + inputs["prompt"] = [inputs["prompt"], inputs["prompt"]] + output_1 = sd_pipe(**inputs) + + assert np.abs(image - output_1.images).max() < 1e-3 + + +class StableDiffusionMultiControlNetOneModelPipelineFastTests( + IPAdapterTesterMixin, PipelineTesterMixin, PipelineKarrasSchedulerTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionControlNetPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = frozenset([]) # TO_DO: add image_params once refactored VaeImageProcessor.preprocess + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(4, 8), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + norm_num_groups=1, + ) + torch.manual_seed(0) + + def init_weights(m): + if isinstance(m, torch.nn.Conv2d): + torch.nn.init.normal(m.weight) + m.bias.data.fill_(1.0) + + controlnet = ControlNetModel( + block_out_channels=(4, 8), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=32, + conditioning_embedding_out_channels=(16, 32), + norm_num_groups=1, + ) + controlnet.controlnet_down_blocks.apply(init_weights) + + torch.manual_seed(0) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[4, 8], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + norm_num_groups=2, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + controlnet = MultiControlNetModel([controlnet]) + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + controlnet_embedder_scale_factor = 2 + + images = [ + randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ), + ] + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + "image": images, + } + + return inputs + + def test_control_guidance_switch(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + + scale = 10.0 + steps = 4 + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_1 = pipe(**inputs)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_2 = pipe(**inputs, control_guidance_start=0.1, control_guidance_end=0.2)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_3 = pipe( + **inputs, + control_guidance_start=[0.1], + control_guidance_end=[0.2], + )[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_4 = pipe(**inputs, control_guidance_start=0.4, control_guidance_end=[0.5])[0] + + # make sure that all outputs are different + assert np.sum(np.abs(output_1 - output_2)) > 1e-3 + assert np.sum(np.abs(output_1 - output_3)) > 1e-3 + assert np.sum(np.abs(output_1 - output_4)) > 1e-3 + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + def test_save_pretrained_raise_not_implemented_exception(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + with tempfile.TemporaryDirectory() as tmpdir: + try: + # save_pretrained is not implemented for Multi-ControlNet + pipe.save_pretrained(tmpdir) + except NotImplementedError: + pass + + +@slow +@require_torch_gpu +class ControlNetPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_canny(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "bird" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ) + + output = pipe(prompt, image, generator=generator, output_type="np", num_inference_steps=3) + + image = output.images[0] + + assert image.shape == (768, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny_out.npy" + ) + + assert np.abs(expected_image - image).max() < 9e-2 + + def test_depth(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-depth") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "Stormtrooper's lecture" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/stormtrooper_depth.png" + ) + + output = pipe(prompt, image, generator=generator, output_type="np", num_inference_steps=3) + + image = output.images[0] + + assert image.shape == (512, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/stormtrooper_depth_out.npy" + ) + + assert np.abs(expected_image - image).max() < 8e-1 + + def test_hed(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-hed") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "oil painting of handsome old man, masterpiece" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/man_hed.png" + ) + + output = pipe(prompt, image, generator=generator, output_type="np", num_inference_steps=3) + + image = output.images[0] + + assert image.shape == (704, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/man_hed_out.npy" + ) + + assert np.abs(expected_image - image).max() < 8e-2 + + def test_mlsd(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-mlsd") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "room" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/room_mlsd.png" + ) + + output = pipe(prompt, image, generator=generator, output_type="np", num_inference_steps=3) + + image = output.images[0] + + assert image.shape == (704, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/room_mlsd_out.npy" + ) + + assert np.abs(expected_image - image).max() < 5e-2 + + def test_normal(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-normal") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "cute toy" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/cute_toy_normal.png" + ) + + output = pipe(prompt, image, generator=generator, output_type="np", num_inference_steps=3) + + image = output.images[0] + + assert image.shape == (512, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/cute_toy_normal_out.npy" + ) + + assert np.abs(expected_image - image).max() < 5e-2 + + def test_openpose(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-openpose") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "Chef in the kitchen" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/pose.png" + ) + + output = pipe(prompt, image, generator=generator, output_type="np", num_inference_steps=3) + + image = output.images[0] + + assert image.shape == (768, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/chef_pose_out.npy" + ) + + assert np.abs(expected_image - image).max() < 8e-2 + + def test_scribble(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-scribble") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(5) + prompt = "bag" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bag_scribble.png" + ) + + output = pipe(prompt, image, generator=generator, output_type="np", num_inference_steps=3) + + image = output.images[0] + + assert image.shape == (640, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bag_scribble_out.npy" + ) + + assert np.abs(expected_image - image).max() < 8e-2 + + def test_seg(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-seg") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(5) + prompt = "house" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/house_seg.png" + ) + + output = pipe(prompt, image, generator=generator, output_type="np", num_inference_steps=3) + + image = output.images[0] + + assert image.shape == (512, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/house_seg_out.npy" + ) + + assert np.abs(expected_image - image).max() < 8e-2 + + def test_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-seg") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + pipe.enable_sequential_cpu_offload() + + prompt = "house" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/house_seg.png" + ) + + _ = pipe( + prompt, + image, + num_inference_steps=2, + output_type="np", + ) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 7 GB is allocated + assert mem_bytes < 4 * 10**9 + + def test_canny_guess_mode(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ) + + output = pipe( + prompt, + image, + generator=generator, + output_type="np", + num_inference_steps=3, + guidance_scale=3.0, + guess_mode=True, + ) + + image = output.images[0] + assert image.shape == (768, 512, 3) + + image_slice = image[-3:, -3:, -1] + expected_slice = np.array([0.2724, 0.2846, 0.2724, 0.3843, 0.3682, 0.2736, 0.4675, 0.3862, 0.2887]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_canny_guess_mode_euler(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ) + + output = pipe( + prompt, + image, + generator=generator, + output_type="np", + num_inference_steps=3, + guidance_scale=3.0, + guess_mode=True, + ) + + image = output.images[0] + assert image.shape == (768, 512, 3) + + image_slice = image[-3:, -3:, -1] + expected_slice = np.array([0.1655, 0.1721, 0.1623, 0.1685, 0.1711, 0.1646, 0.1651, 0.1631, 0.1494]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + @require_python39_or_higher + @require_torch_2 + def test_stable_diffusion_compile(self): + run_test_in_subprocess(test_case=self, target_func=_test_stable_diffusion_compile, inputs=None) + + def test_v11_shuffle_global_pool_conditions(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/control_v11e_sd15_shuffle") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "New York" + image = load_image( + "https://huggingface.co/lllyasviel/control_v11e_sd15_shuffle/resolve/main/images/control.png" + ) + + output = pipe( + prompt, + image, + generator=generator, + output_type="np", + num_inference_steps=3, + guidance_scale=7.0, + ) + + image = output.images[0] + assert image.shape == (512, 640, 3) + + image_slice = image[-3:, -3:, -1] + expected_slice = np.array([0.1338, 0.1597, 0.1202, 0.1687, 0.1377, 0.1017, 0.2070, 0.1574, 0.1348]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_load_local(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/control_v11p_sd15_canny") + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.unet.set_default_attn_processor() + pipe.enable_model_cpu_offload() + + controlnet = ControlNetModel.from_single_file( + "https://huggingface.co/lllyasviel/ControlNet-v1-1/blob/main/control_v11p_sd15_canny.pth" + ) + pipe_sf = StableDiffusionControlNetPipeline.from_single_file( + "https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned-emaonly.safetensors", + safety_checker=None, + controlnet=controlnet, + scheduler_type="pndm", + ) + pipe_sf.unet.set_default_attn_processor() + pipe_sf.enable_model_cpu_offload() + + control_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ).resize((512, 512)) + prompt = "bird" + + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipe( + prompt, + image=control_image, + generator=generator, + output_type="np", + num_inference_steps=3, + ).images[0] + + generator = torch.Generator(device="cpu").manual_seed(0) + output_sf = pipe_sf( + prompt, + image=control_image, + generator=generator, + output_type="np", + num_inference_steps=3, + ).images[0] + + max_diff = numpy_cosine_similarity_distance(output_sf.flatten(), output.flatten()) + assert max_diff < 1e-3 + + def test_single_file_component_configs(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/control_v11p_sd15_canny", variant="fp16") + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", variant="fp16", safety_checker=None, controlnet=controlnet + ) + + controlnet_single_file = ControlNetModel.from_single_file( + "https://huggingface.co/lllyasviel/ControlNet-v1-1/blob/main/control_v11p_sd15_canny.pth" + ) + single_file_pipe = StableDiffusionControlNetPipeline.from_single_file( + "https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned-emaonly.safetensors", + safety_checker=None, + controlnet=controlnet_single_file, + scheduler_type="pndm", + ) + + PARAMS_TO_IGNORE = ["torch_dtype", "_name_or_path", "architectures", "_use_default_values"] + for param_name, param_value in single_file_pipe.controlnet.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.controlnet.config[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" + + for param_name, param_value in single_file_pipe.unet.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.unet.config[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" + + for param_name, param_value in single_file_pipe.vae.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.vae.config[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" + + +@slow +@require_torch_gpu +class StableDiffusionMultiControlNetPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_pose_and_canny(self): + controlnet_canny = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny") + controlnet_pose = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-openpose") + + pipe = StableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=[controlnet_pose, controlnet_canny] + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "bird and Chef" + image_canny = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ) + image_pose = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/pose.png" + ) + + output = pipe(prompt, [image_pose, image_canny], generator=generator, output_type="np", num_inference_steps=3) + + image = output.images[0] + + assert image.shape == (768, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/pose_canny_out.npy" + ) + + assert np.abs(expected_image - image).max() < 5e-2 diff --git a/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py new file mode 100755 index 0000000..fe4c9da --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_blip_diffusion.py @@ -0,0 +1,216 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPTokenizer +from transformers.models.blip_2.configuration_blip_2 import Blip2Config +from transformers.models.clip.configuration_clip import CLIPTextConfig + +from diffusers import ( + AutoencoderKL, + BlipDiffusionControlNetPipeline, + ControlNetModel, + PNDMScheduler, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import enable_full_determinism +from src.diffusers.pipelines.blip_diffusion.blip_image_processing import BlipImageProcessor +from src.diffusers.pipelines.blip_diffusion.modeling_blip2 import Blip2QFormerModel +from src.diffusers.pipelines.blip_diffusion.modeling_ctx_clip import ContextCLIPTextModel + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class BlipDiffusionControlNetPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = BlipDiffusionControlNetPipeline + params = [ + "prompt", + "reference_image", + "source_subject_category", + "target_subject_category", + "condtioning_image", + ] + batch_params = [ + "prompt", + "reference_image", + "source_subject_category", + "target_subject_category", + "condtioning_image", + ] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "num_inference_steps", + "neg_prompt", + "guidance_scale", + "prompt_strength", + "prompt_reps", + ] + + def get_dummy_components(self): + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + vocab_size=1000, + hidden_size=16, + intermediate_size=16, + projection_dim=16, + num_hidden_layers=1, + num_attention_heads=1, + max_position_embeddings=77, + ) + text_encoder = ContextCLIPTextModel(text_encoder_config) + + vae = AutoencoderKL( + in_channels=4, + out_channels=4, + down_block_types=("DownEncoderBlock2D",), + up_block_types=("UpDecoderBlock2D",), + block_out_channels=(32,), + layers_per_block=1, + act_fn="silu", + latent_channels=4, + norm_num_groups=16, + sample_size=16, + ) + + blip_vision_config = { + "hidden_size": 16, + "intermediate_size": 16, + "num_hidden_layers": 1, + "num_attention_heads": 1, + "image_size": 224, + "patch_size": 14, + "hidden_act": "quick_gelu", + } + + blip_qformer_config = { + "vocab_size": 1000, + "hidden_size": 16, + "num_hidden_layers": 1, + "num_attention_heads": 1, + "intermediate_size": 16, + "max_position_embeddings": 512, + "cross_attention_frequency": 1, + "encoder_hidden_size": 16, + } + qformer_config = Blip2Config( + vision_config=blip_vision_config, + qformer_config=blip_qformer_config, + num_query_tokens=16, + tokenizer="hf-internal-testing/tiny-random-bert", + ) + qformer = Blip2QFormerModel(qformer_config) + + unet = UNet2DConditionModel( + block_out_channels=(4, 16), + layers_per_block=1, + norm_num_groups=4, + sample_size=16, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=16, + ) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + scheduler = PNDMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + set_alpha_to_one=False, + skip_prk_steps=True, + ) + controlnet = ControlNetModel( + block_out_channels=(4, 16), + layers_per_block=1, + in_channels=4, + norm_num_groups=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=16, + conditioning_embedding_out_channels=(8, 16), + ) + + vae.eval() + qformer.eval() + text_encoder.eval() + + image_processor = BlipImageProcessor() + + components = { + "text_encoder": text_encoder, + "vae": vae, + "qformer": qformer, + "unet": unet, + "tokenizer": tokenizer, + "scheduler": scheduler, + "controlnet": controlnet, + "image_processor": image_processor, + } + return components + + def get_dummy_inputs(self, device, seed=0): + np.random.seed(seed) + reference_image = np.random.rand(32, 32, 3) * 255 + reference_image = Image.fromarray(reference_image.astype("uint8")).convert("RGBA") + cond_image = np.random.rand(32, 32, 3) * 255 + cond_image = Image.fromarray(cond_image.astype("uint8")).convert("RGBA") + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "swimming underwater", + "generator": generator, + "reference_image": reference_image, + "condtioning_image": cond_image, + "source_subject_category": "dog", + "target_subject_category": "dog", + "height": 32, + "width": 32, + "guidance_scale": 7.5, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + def test_blipdiffusion_controlnet(self): + device = "cpu" + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + image = pipe(**self.get_dummy_inputs(device))[0] + image_slice = image[0, -3:, -3:, 0] + + assert image.shape == (1, 16, 16, 4) + expected_slice = np.array([0.7953, 0.7136, 0.6597, 0.4779, 0.7389, 0.4111, 0.5826, 0.4150, 0.8422]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" diff --git a/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_img2img.py b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_img2img.py new file mode 100755 index 0000000..89e2b38 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_img2img.py @@ -0,0 +1,479 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This model implementation is heavily inspired by https://github.com/haofanwang/ControlNet-for-Diffusers/ + +import gc +import random +import tempfile +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + ControlNetModel, + DDIMScheduler, + StableDiffusionControlNetImg2ImgPipeline, + UNet2DConditionModel, +) +from diffusers.pipelines.controlnet.pipeline_controlnet import MultiControlNetModel +from diffusers.utils import load_image +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_numpy, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) +from diffusers.utils.torch_utils import randn_tensor + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, +) +from ..test_pipelines_common import ( + IPAdapterTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, +) + + +enable_full_determinism() + + +class ControlNetImg2ImgPipelineFastTests( + IPAdapterTesterMixin, + PipelineLatentTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineTesterMixin, + unittest.TestCase, +): + pipeline_class = StableDiffusionControlNetImg2ImgPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS.union({"control_image"}) + image_latents_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(4, 8), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + norm_num_groups=1, + ) + torch.manual_seed(0) + controlnet = ControlNetModel( + block_out_channels=(4, 8), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=32, + conditioning_embedding_out_channels=(16, 32), + norm_num_groups=1, + ) + torch.manual_seed(0) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[4, 8], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + norm_num_groups=2, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + controlnet_embedder_scale_factor = 2 + control_image = randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ) + image = floats_tensor(control_image.shape, rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + image = Image.fromarray(np.uint8(image)).convert("RGB").resize((64, 64)) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + "image": image, + "control_image": control_image, + } + + return inputs + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + +class StableDiffusionMultiControlNetPipelineFastTests( + IPAdapterTesterMixin, PipelineTesterMixin, PipelineKarrasSchedulerTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionControlNetImg2ImgPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = frozenset([]) # TO_DO: add image_params once refactored VaeImageProcessor.preprocess + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(4, 8), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + norm_num_groups=1, + ) + torch.manual_seed(0) + + def init_weights(m): + if isinstance(m, torch.nn.Conv2d): + torch.nn.init.normal(m.weight) + m.bias.data.fill_(1.0) + + controlnet1 = ControlNetModel( + block_out_channels=(4, 8), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=32, + conditioning_embedding_out_channels=(16, 32), + norm_num_groups=1, + ) + controlnet1.controlnet_down_blocks.apply(init_weights) + + torch.manual_seed(0) + controlnet2 = ControlNetModel( + block_out_channels=(4, 8), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=32, + conditioning_embedding_out_channels=(16, 32), + norm_num_groups=1, + ) + controlnet2.controlnet_down_blocks.apply(init_weights) + + torch.manual_seed(0) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[4, 8], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + norm_num_groups=2, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + controlnet = MultiControlNetModel([controlnet1, controlnet2]) + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + controlnet_embedder_scale_factor = 2 + + control_image = [ + randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ), + randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ), + ] + + image = floats_tensor(control_image[0].shape, rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + image = Image.fromarray(np.uint8(image)).convert("RGB").resize((64, 64)) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + "image": image, + "control_image": control_image, + } + + return inputs + + def test_control_guidance_switch(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + + scale = 10.0 + steps = 4 + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_1 = pipe(**inputs)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_2 = pipe(**inputs, control_guidance_start=0.1, control_guidance_end=0.2)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_3 = pipe(**inputs, control_guidance_start=[0.1, 0.3], control_guidance_end=[0.2, 0.7])[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_4 = pipe(**inputs, control_guidance_start=0.4, control_guidance_end=[0.5, 0.8])[0] + + # make sure that all outputs are different + assert np.sum(np.abs(output_1 - output_2)) > 1e-3 + assert np.sum(np.abs(output_1 - output_3)) > 1e-3 + assert np.sum(np.abs(output_1 - output_4)) > 1e-3 + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + def test_save_pretrained_raise_not_implemented_exception(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + with tempfile.TemporaryDirectory() as tmpdir: + try: + # save_pretrained is not implemented for Multi-ControlNet + pipe.save_pretrained(tmpdir) + except NotImplementedError: + pass + + +@slow +@require_torch_gpu +class ControlNetImg2ImgPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_canny(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny") + + pipe = StableDiffusionControlNetImg2ImgPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "evil space-punk bird" + control_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ).resize((512, 512)) + image = load_image( + "https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png" + ).resize((512, 512)) + + output = pipe( + prompt, + image, + control_image=control_image, + generator=generator, + output_type="np", + num_inference_steps=50, + strength=0.6, + ) + + image = output.images[0] + + assert image.shape == (512, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/img2img.npy" + ) + + assert np.abs(expected_image - image).max() < 9e-2 + + def test_load_local(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/control_v11p_sd15_canny") + pipe = StableDiffusionControlNetImg2ImgPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.unet.set_default_attn_processor() + pipe.enable_model_cpu_offload() + + controlnet = ControlNetModel.from_single_file( + "https://huggingface.co/lllyasviel/ControlNet-v1-1/blob/main/control_v11p_sd15_canny.pth" + ) + pipe_sf = StableDiffusionControlNetImg2ImgPipeline.from_single_file( + "https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned-emaonly.safetensors", + safety_checker=None, + controlnet=controlnet, + scheduler_type="pndm", + ) + pipe_sf.unet.set_default_attn_processor() + pipe_sf.enable_model_cpu_offload() + + control_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ).resize((512, 512)) + image = load_image( + "https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png" + ).resize((512, 512)) + prompt = "bird" + + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipe( + prompt, + image=image, + control_image=control_image, + strength=0.9, + generator=generator, + output_type="np", + num_inference_steps=3, + ).images[0] + + generator = torch.Generator(device="cpu").manual_seed(0) + output_sf = pipe_sf( + prompt, + image=image, + control_image=control_image, + strength=0.9, + generator=generator, + output_type="np", + num_inference_steps=3, + ).images[0] + + max_diff = numpy_cosine_similarity_distance(output_sf.flatten(), output.flatten()) + assert max_diff < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_inpaint.py b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_inpaint.py new file mode 100755 index 0000000..67e0da4 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_inpaint.py @@ -0,0 +1,605 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This model implementation is heavily based on: + +import gc +import random +import tempfile +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + ControlNetModel, + DDIMScheduler, + StableDiffusionControlNetInpaintPipeline, + UNet2DConditionModel, +) +from diffusers.pipelines.controlnet.pipeline_controlnet import MultiControlNetModel +from diffusers.utils import load_image +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_numpy, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) +from diffusers.utils.torch_utils import randn_tensor + +from ..pipeline_params import ( + TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_INPAINTING_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, +) +from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +class ControlNetInpaintPipelineFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionControlNetInpaintPipeline + params = TEXT_GUIDED_IMAGE_INPAINTING_PARAMS + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + image_params = frozenset({"control_image"}) # skip `image` and `mask` for now, only test for control_image + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=9, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + torch.manual_seed(0) + controlnet = ControlNetModel( + block_out_channels=(32, 64), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=32, + conditioning_embedding_out_channels=(16, 32), + ) + torch.manual_seed(0) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + controlnet_embedder_scale_factor = 2 + control_image = randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ) + init_image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + init_image = init_image.cpu().permute(0, 2, 3, 1)[0] + + image = Image.fromarray(np.uint8(init_image)).convert("RGB").resize((64, 64)) + mask_image = Image.fromarray(np.uint8(init_image + 4)).convert("RGB").resize((64, 64)) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + "image": image, + "mask_image": mask_image, + "control_image": control_image, + } + + return inputs + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + +class ControlNetSimpleInpaintPipelineFastTests(ControlNetInpaintPipelineFastTests): + pipeline_class = StableDiffusionControlNetInpaintPipeline + params = TEXT_GUIDED_IMAGE_INPAINTING_PARAMS + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + image_params = frozenset([]) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + torch.manual_seed(0) + controlnet = ControlNetModel( + block_out_channels=(32, 64), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=32, + conditioning_embedding_out_channels=(16, 32), + ) + torch.manual_seed(0) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + +class MultiControlNetInpaintPipelineFastTests( + PipelineTesterMixin, PipelineKarrasSchedulerTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionControlNetInpaintPipeline + params = TEXT_GUIDED_IMAGE_INPAINTING_PARAMS + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=9, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + torch.manual_seed(0) + + def init_weights(m): + if isinstance(m, torch.nn.Conv2d): + torch.nn.init.normal(m.weight) + m.bias.data.fill_(1.0) + + controlnet1 = ControlNetModel( + block_out_channels=(32, 64), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=32, + conditioning_embedding_out_channels=(16, 32), + ) + controlnet1.controlnet_down_blocks.apply(init_weights) + + torch.manual_seed(0) + controlnet2 = ControlNetModel( + block_out_channels=(32, 64), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + cross_attention_dim=32, + conditioning_embedding_out_channels=(16, 32), + ) + controlnet2.controlnet_down_blocks.apply(init_weights) + + torch.manual_seed(0) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + controlnet = MultiControlNetModel([controlnet1, controlnet2]) + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + controlnet_embedder_scale_factor = 2 + + control_image = [ + randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ), + randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ), + ] + init_image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + init_image = init_image.cpu().permute(0, 2, 3, 1)[0] + + image = Image.fromarray(np.uint8(init_image)).convert("RGB").resize((64, 64)) + mask_image = Image.fromarray(np.uint8(init_image + 4)).convert("RGB").resize((64, 64)) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + "image": image, + "mask_image": mask_image, + "control_image": control_image, + } + + return inputs + + def test_control_guidance_switch(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + + scale = 10.0 + steps = 4 + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_1 = pipe(**inputs)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_2 = pipe(**inputs, control_guidance_start=0.1, control_guidance_end=0.2)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_3 = pipe(**inputs, control_guidance_start=[0.1, 0.3], control_guidance_end=[0.2, 0.7])[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_4 = pipe(**inputs, control_guidance_start=0.4, control_guidance_end=[0.5, 0.8])[0] + + # make sure that all outputs are different + assert np.sum(np.abs(output_1 - output_2)) > 1e-3 + assert np.sum(np.abs(output_1 - output_3)) > 1e-3 + assert np.sum(np.abs(output_1 - output_4)) > 1e-3 + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + def test_save_pretrained_raise_not_implemented_exception(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + with tempfile.TemporaryDirectory() as tmpdir: + try: + # save_pretrained is not implemented for Multi-ControlNet + pipe.save_pretrained(tmpdir) + except NotImplementedError: + pass + + +@slow +@require_torch_gpu +class ControlNetInpaintPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_canny(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/sd-controlnet-canny") + + pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None, controlnet=controlnet + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + image = load_image( + "https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png" + ).resize((512, 512)) + + mask_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_mask.png" + ).resize((512, 512)) + + prompt = "pitch black hole" + + control_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ).resize((512, 512)) + + output = pipe( + prompt, + image=image, + mask_image=mask_image, + control_image=control_image, + generator=generator, + output_type="np", + num_inference_steps=3, + ) + + image = output.images[0] + + assert image.shape == (512, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/inpaint.npy" + ) + + assert np.abs(expected_image - image).max() < 9e-2 + + def test_inpaint(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/control_v11p_sd15_inpaint") + + pipe = StableDiffusionControlNetInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, controlnet=controlnet + ) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(33) + + init_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy.png" + ) + init_image = init_image.resize((512, 512)) + + mask_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_inpaint/boy_mask.png" + ) + mask_image = mask_image.resize((512, 512)) + + prompt = "a handsome man with ray-ban sunglasses" + + def make_inpaint_condition(image, image_mask): + image = np.array(image.convert("RGB")).astype(np.float32) / 255.0 + image_mask = np.array(image_mask.convert("L")).astype(np.float32) / 255.0 + + assert image.shape[0:1] == image_mask.shape[0:1], "image and image_mask must have the same image size" + image[image_mask > 0.5] = -1.0 # set as masked pixel + image = np.expand_dims(image, 0).transpose(0, 3, 1, 2) + image = torch.from_numpy(image) + return image + + control_image = make_inpaint_condition(init_image, mask_image) + + output = pipe( + prompt, + image=init_image, + mask_image=mask_image, + control_image=control_image, + guidance_scale=9.0, + eta=1.0, + generator=generator, + num_inference_steps=20, + output_type="np", + ) + image = output.images[0] + + assert image.shape == (512, 512, 3) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/boy_ray_ban.npy" + ) + + assert numpy_cosine_similarity_distance(expected_image.flatten(), image.flatten()) < 1e-2 + + def test_load_local(self): + controlnet = ControlNetModel.from_pretrained("lllyasviel/control_v11p_sd15_canny") + pipe_1 = StableDiffusionControlNetInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None, controlnet=controlnet + ) + + controlnet = ControlNetModel.from_single_file( + "https://huggingface.co/lllyasviel/ControlNet-v1-1/blob/main/control_v11p_sd15_canny.pth" + ) + pipe_2 = StableDiffusionControlNetInpaintPipeline.from_single_file( + "https://huggingface.co/runwayml/stable-diffusion-inpainting/blob/main/sd-v1-5-inpainting.ckpt", + safety_checker=None, + controlnet=controlnet, + ) + control_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ).resize((512, 512)) + image = load_image( + "https://huggingface.co/lllyasviel/sd-controlnet-canny/resolve/main/images/bird.png" + ).resize((512, 512)) + mask_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_mask.png" + ).resize((512, 512)) + + pipes = [pipe_1, pipe_2] + images = [] + for pipe in pipes: + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "bird" + output = pipe( + prompt, + image=image, + control_image=control_image, + mask_image=mask_image, + strength=0.9, + generator=generator, + output_type="np", + num_inference_steps=3, + ) + images.append(output.images[0]) + + del pipe + gc.collect() + torch.cuda.empty_cache() + + max_diff = numpy_cosine_similarity_distance(images[0].flatten(), images[1].flatten()) + assert max_diff < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_inpaint_sdxl.py b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_inpaint_sdxl.py new file mode 100755 index 0000000..5f38263 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_inpaint_sdxl.py @@ -0,0 +1,304 @@ +# coding=utf-8 +# Copyright 2024 Harutatsu Akiyama, Jinbin Bai, and HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + ControlNetModel, + EulerDiscreteScheduler, + StableDiffusionXLControlNetInpaintPipeline, + UNet2DConditionModel, +) +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import enable_full_determinism, floats_tensor, require_torch_gpu, torch_device + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_BATCH_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_PARAMS, +) +from ..test_pipelines_common import ( + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, +) + + +enable_full_determinism() + + +class ControlNetPipelineSDXLFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionXLControlNetInpaintPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = frozenset(IMAGE_TO_IMAGE_IMAGE_PARAMS.union({"mask_image", "control_image"})) + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + controlnet = ControlNetModel( + block_out_channels=(32, 64), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + conditioning_embedding_out_channels=(16, 32), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + } + return components + + def get_dummy_inputs(self, device, seed=0, img_res=64): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + # Get random floats in [0, 1] as image + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + mask_image = torch.ones_like(image) + controlnet_embedder_scale_factor = 2 + control_image = ( + floats_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + rng=random.Random(seed), + ) + .to(device) + .cpu() + ) + control_image = control_image.cpu().permute(0, 2, 3, 1)[0] + # Convert image and mask_image to [0, 255] + image = 255 * image + mask_image = 255 * mask_image + control_image = 255 * control_image + # Convert to PIL image + init_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((img_res, img_res)) + mask_image = Image.fromarray(np.uint8(mask_image)).convert("L").resize((img_res, img_res)) + control_image = Image.fromarray(np.uint8(control_image)).convert("RGB").resize((img_res, img_res)) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + "image": init_image, + "mask_image": mask_image, + "control_image": control_image, + } + return inputs + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + @require_torch_gpu + def test_stable_diffusion_xl_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + pipe.unet.set_default_attn_processor() + + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_stable_diffusion_xl_multi_prompts(self): + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + + # forward with single prompt + inputs = self.get_dummy_inputs(torch_device) + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same prompt duplicated + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt_2"] = inputs["prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt_2"] = "different prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + # manually set a negative_prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same negative_prompt duplicated + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = inputs["negative_prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different negative_prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = "different negative prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + def test_controlnet_sdxl_guess(self): + device = "cpu" + + components = self.get_dummy_components() + + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(device) + + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["guess_mode"] = True + + output = sd_pipe(**inputs) + image_slice = output.images[0, -3:, -3:, -1] + expected_slice = np.array( + [0.5381963, 0.4836803, 0.45821992, 0.5577731, 0.51210403, 0.4794795, 0.59282357, 0.5647199, 0.43100584] + ) + + # make sure that it's equal + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-4 + + # TODO(Patrick, Sayak) - skip for now as this requires more refiner tests + def test_save_load_optional_components(self): + pass + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=5e-1) diff --git a/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_sdxl.py b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_sdxl.py new file mode 100755 index 0000000..c82ce6c --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_sdxl.py @@ -0,0 +1,1173 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import gc +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + ControlNetModel, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + LCMScheduler, + StableDiffusionXLControlNetPipeline, + StableDiffusionXLImg2ImgPipeline, + UNet2DConditionModel, +) +from diffusers.models.unets.unet_2d_blocks import UNetMidBlock2D +from diffusers.pipelines.controlnet.pipeline_controlnet import MultiControlNetModel +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import ( + enable_full_determinism, + load_image, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) +from diffusers.utils.torch_utils import randn_tensor + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_BATCH_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_PARAMS, +) +from ..test_pipelines_common import ( + IPAdapterTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, + SDXLOptionalComponentsTesterMixin, +) + + +enable_full_determinism() + + +class StableDiffusionXLControlNetPipelineFastTests( + IPAdapterTesterMixin, + PipelineLatentTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineTesterMixin, + SDXLOptionalComponentsTesterMixin, + unittest.TestCase, +): + pipeline_class = StableDiffusionXLControlNetPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + time_cond_proj_dim=time_cond_proj_dim, + ) + torch.manual_seed(0) + controlnet = ControlNetModel( + block_out_channels=(32, 64), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + conditioning_embedding_out_channels=(16, 32), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + torch.manual_seed(0) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + controlnet_embedder_scale_factor = 2 + image = randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "np", + "image": image, + } + + return inputs + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + def test_save_load_optional_components(self): + self._test_save_load_optional_components() + + @require_torch_gpu + def test_stable_diffusion_xl_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + pipe.unet.set_default_attn_processor() + + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_stable_diffusion_xl_multi_prompts(self): + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + + # forward with single prompt + inputs = self.get_dummy_inputs(torch_device) + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same prompt duplicated + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt_2"] = inputs["prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt_2"] = "different prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + # manually set a negative_prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same negative_prompt duplicated + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = inputs["negative_prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different negative_prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = "different negative prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + # copied from test_stable_diffusion_xl.py + def test_stable_diffusion_xl_prompt_embeds(self): + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + # forward without prompt embeds + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt"] = 2 * [inputs["prompt"]] + inputs["num_images_per_prompt"] = 2 + + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with prompt embeds + inputs = self.get_dummy_inputs(torch_device) + prompt = 2 * [inputs.pop("prompt")] + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = sd_pipe.encode_prompt(prompt) + + output = sd_pipe( + **inputs, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + ) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # make sure that it's equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + def test_controlnet_sdxl_guess(self): + device = "cpu" + + components = self.get_dummy_components() + + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(device) + + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["guess_mode"] = True + + output = sd_pipe(**inputs) + image_slice = output.images[0, -3:, -3:, -1] + expected_slice = np.array( + [0.7330834, 0.590667, 0.5667336, 0.6029023, 0.5679491, 0.5968194, 0.4032986, 0.47612396, 0.5089609] + ) + + # make sure that it's equal + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-4 + + def test_controlnet_sdxl_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLControlNetPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.7799, 0.614, 0.6162, 0.7082, 0.6662, 0.5833, 0.4148, 0.5182, 0.4866]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + # copied from test_stable_diffusion_xl.py:test_stable_diffusion_two_xl_mixture_of_denoiser_fast + # with `StableDiffusionXLControlNetPipeline` instead of `StableDiffusionXLPipeline` + def test_controlnet_sdxl_two_mixture_of_denoiser_fast(self): + components = self.get_dummy_components() + pipe_1 = StableDiffusionXLControlNetPipeline(**components).to(torch_device) + pipe_1.unet.set_default_attn_processor() + + components_without_controlnet = {k: v for k, v in components.items() if k != "controlnet"} + pipe_2 = StableDiffusionXLImg2ImgPipeline(**components_without_controlnet).to(torch_device) + pipe_2.unet.set_default_attn_processor() + + def assert_run_mixture( + num_steps, + split, + scheduler_cls_orig, + expected_tss, + num_train_timesteps=pipe_1.scheduler.config.num_train_timesteps, + ): + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = num_steps + + class scheduler_cls(scheduler_cls_orig): + pass + + pipe_1.scheduler = scheduler_cls.from_config(pipe_1.scheduler.config) + pipe_2.scheduler = scheduler_cls.from_config(pipe_2.scheduler.config) + + # Let's retrieve the number of timesteps we want to use + pipe_1.scheduler.set_timesteps(num_steps) + expected_steps = pipe_1.scheduler.timesteps.tolist() + + if pipe_1.scheduler.order == 2: + expected_steps_1 = list(filter(lambda ts: ts >= split, expected_tss)) + expected_steps_2 = expected_steps_1[-1:] + list(filter(lambda ts: ts < split, expected_tss)) + expected_steps = expected_steps_1 + expected_steps_2 + else: + expected_steps_1 = list(filter(lambda ts: ts >= split, expected_tss)) + expected_steps_2 = list(filter(lambda ts: ts < split, expected_tss)) + + # now we monkey patch step `done_steps` + # list into the step function for testing + done_steps = [] + old_step = copy.copy(scheduler_cls.step) + + def new_step(self, *args, **kwargs): + done_steps.append(args[1].cpu().item()) # args[1] is always the passed `t` + return old_step(self, *args, **kwargs) + + scheduler_cls.step = new_step + + inputs_1 = { + **inputs, + **{ + "denoising_end": 1.0 - (split / num_train_timesteps), + "output_type": "latent", + }, + } + latents = pipe_1(**inputs_1).images[0] + + assert expected_steps_1 == done_steps, f"Failure with {scheduler_cls.__name__} and {num_steps} and {split}" + + inputs_2 = { + **inputs, + **{ + "denoising_start": 1.0 - (split / num_train_timesteps), + "image": latents, + }, + } + pipe_2(**inputs_2).images[0] + + assert expected_steps_2 == done_steps[len(expected_steps_1) :] + assert expected_steps == done_steps, f"Failure with {scheduler_cls.__name__} and {num_steps} and {split}" + + steps = 10 + for split in [300, 700]: + for scheduler_cls_timesteps in [ + (EulerDiscreteScheduler, [901, 801, 701, 601, 501, 401, 301, 201, 101, 1]), + ( + HeunDiscreteScheduler, + [ + 901.0, + 801.0, + 801.0, + 701.0, + 701.0, + 601.0, + 601.0, + 501.0, + 501.0, + 401.0, + 401.0, + 301.0, + 301.0, + 201.0, + 201.0, + 101.0, + 101.0, + 1.0, + 1.0, + ], + ), + ]: + assert_run_mixture(steps, split, scheduler_cls_timesteps[0], scheduler_cls_timesteps[1]) + + +class StableDiffusionXLMultiControlNetPipelineFastTests( + PipelineTesterMixin, PipelineKarrasSchedulerTesterMixin, SDXLOptionalComponentsTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionXLControlNetPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = frozenset([]) # TO_DO: add image_params once refactored VaeImageProcessor.preprocess + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + torch.manual_seed(0) + + def init_weights(m): + if isinstance(m, torch.nn.Conv2d): + torch.nn.init.normal(m.weight) + m.bias.data.fill_(1.0) + + controlnet1 = ControlNetModel( + block_out_channels=(32, 64), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + conditioning_embedding_out_channels=(16, 32), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + controlnet1.controlnet_down_blocks.apply(init_weights) + + torch.manual_seed(0) + controlnet2 = ControlNetModel( + block_out_channels=(32, 64), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + conditioning_embedding_out_channels=(16, 32), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + controlnet2.controlnet_down_blocks.apply(init_weights) + + torch.manual_seed(0) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + controlnet = MultiControlNetModel([controlnet1, controlnet2]) + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + controlnet_embedder_scale_factor = 2 + + images = [ + randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ), + randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ), + ] + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "np", + "image": images, + } + + return inputs + + def test_control_guidance_switch(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + + scale = 10.0 + steps = 4 + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_1 = pipe(**inputs)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_2 = pipe(**inputs, control_guidance_start=0.1, control_guidance_end=0.2)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_3 = pipe(**inputs, control_guidance_start=[0.1, 0.3], control_guidance_end=[0.2, 0.7])[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_4 = pipe(**inputs, control_guidance_start=0.4, control_guidance_end=[0.5, 0.8])[0] + + # make sure that all outputs are different + assert np.sum(np.abs(output_1 - output_2)) > 1e-3 + assert np.sum(np.abs(output_1 - output_3)) > 1e-3 + assert np.sum(np.abs(output_1 - output_4)) > 1e-3 + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + def test_save_load_optional_components(self): + return self._test_save_load_optional_components() + + +class StableDiffusionXLMultiControlNetOneModelPipelineFastTests( + PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, SDXLOptionalComponentsTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionXLControlNetPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = frozenset([]) # TO_DO: add image_params once refactored VaeImageProcessor.preprocess + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + torch.manual_seed(0) + + def init_weights(m): + if isinstance(m, torch.nn.Conv2d): + torch.nn.init.normal(m.weight) + m.bias.data.fill_(1.0) + + controlnet = ControlNetModel( + block_out_channels=(32, 64), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + conditioning_embedding_out_channels=(16, 32), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + controlnet.controlnet_down_blocks.apply(init_weights) + + torch.manual_seed(0) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + controlnet = MultiControlNetModel([controlnet]) + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + controlnet_embedder_scale_factor = 2 + images = [ + randn_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + generator=generator, + device=torch.device(device), + ), + ] + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "np", + "image": images, + } + + return inputs + + def test_control_guidance_switch(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + + scale = 10.0 + steps = 4 + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_1 = pipe(**inputs)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_2 = pipe(**inputs, control_guidance_start=0.1, control_guidance_end=0.2)[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_3 = pipe( + **inputs, + control_guidance_start=[0.1], + control_guidance_end=[0.2], + )[0] + + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = steps + inputs["controlnet_conditioning_scale"] = scale + output_4 = pipe(**inputs, control_guidance_start=0.4, control_guidance_end=[0.5])[0] + + # make sure that all outputs are different + assert np.sum(np.abs(output_1 - output_2)) > 1e-3 + assert np.sum(np.abs(output_1 - output_3)) > 1e-3 + assert np.sum(np.abs(output_1 - output_4)) > 1e-3 + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + def test_save_load_optional_components(self): + self._test_save_load_optional_components() + + def test_negative_conditions(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + image_slice_without_neg_cond = image[0, -3:, -3:, -1] + + image = pipe( + **inputs, + negative_original_size=(512, 512), + negative_crops_coords_top_left=(0, 0), + negative_target_size=(1024, 1024), + ).images + image_slice_with_neg_cond = image[0, -3:, -3:, -1] + + self.assertTrue(np.abs(image_slice_without_neg_cond - image_slice_with_neg_cond).max() > 1e-2) + + +@slow +@require_torch_gpu +class ControlNetSDXLPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_canny(self): + controlnet = ControlNetModel.from_pretrained("diffusers/controlnet-canny-sdxl-1.0") + + pipe = StableDiffusionXLControlNetPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", controlnet=controlnet + ) + pipe.enable_sequential_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "bird" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ) + + images = pipe(prompt, image=image, generator=generator, output_type="np", num_inference_steps=3).images + + assert images[0].shape == (768, 512, 3) + + original_image = images[0, -3:, -3:, -1].flatten() + expected_image = np.array([0.4185, 0.4127, 0.4089, 0.4046, 0.4115, 0.4096, 0.4081, 0.4112, 0.3913]) + assert np.allclose(original_image, expected_image, atol=1e-04) + + def test_depth(self): + controlnet = ControlNetModel.from_pretrained("diffusers/controlnet-depth-sdxl-1.0") + + pipe = StableDiffusionXLControlNetPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", controlnet=controlnet + ) + pipe.enable_sequential_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "Stormtrooper's lecture" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/stormtrooper_depth.png" + ) + + images = pipe(prompt, image=image, generator=generator, output_type="np", num_inference_steps=3).images + + assert images[0].shape == (512, 512, 3) + + original_image = images[0, -3:, -3:, -1].flatten() + expected_image = np.array([0.4399, 0.5112, 0.5478, 0.4314, 0.472, 0.4823, 0.4647, 0.4957, 0.4853]) + assert np.allclose(original_image, expected_image, atol=1e-04) + + def test_download_ckpt_diff_format_is_same(self): + controlnet = ControlNetModel.from_pretrained("diffusers/controlnet-depth-sdxl-1.0", torch_dtype=torch.float16) + single_file_url = ( + "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/sd_xl_base_1.0.safetensors" + ) + pipe_single_file = StableDiffusionXLControlNetPipeline.from_single_file( + single_file_url, controlnet=controlnet, torch_dtype=torch.float16 + ) + pipe_single_file.unet.set_default_attn_processor() + pipe_single_file.enable_model_cpu_offload() + pipe_single_file.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + prompt = "Stormtrooper's lecture" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/stormtrooper_depth.png" + ) + single_file_images = pipe_single_file( + prompt, image=image, generator=generator, output_type="np", num_inference_steps=2 + ).images + + generator = torch.Generator(device="cpu").manual_seed(0) + pipe = StableDiffusionXLControlNetPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", controlnet=controlnet, torch_dtype=torch.float16 + ) + pipe.unet.set_default_attn_processor() + pipe.enable_model_cpu_offload() + images = pipe(prompt, image=image, generator=generator, output_type="np", num_inference_steps=2).images + + assert images[0].shape == (512, 512, 3) + assert single_file_images[0].shape == (512, 512, 3) + + max_diff = numpy_cosine_similarity_distance(images[0].flatten(), single_file_images[0].flatten()) + assert max_diff < 5e-2 + + def test_single_file_component_configs(self): + controlnet = ControlNetModel.from_pretrained( + "diffusers/controlnet-depth-sdxl-1.0", torch_dtype=torch.float16, variant="fp16" + ) + pipe = StableDiffusionXLControlNetPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + variant="fp16", + controlnet=controlnet, + torch_dtype=torch.float16, + ) + + single_file_url = ( + "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/sd_xl_base_1.0.safetensors" + ) + single_file_pipe = StableDiffusionXLControlNetPipeline.from_single_file( + single_file_url, controlnet=controlnet, torch_dtype=torch.float16 + ) + + for param_name, param_value in single_file_pipe.text_encoder.config.to_dict().items(): + if param_name in ["torch_dtype", "architectures", "_name_or_path"]: + continue + assert pipe.text_encoder.config.to_dict()[param_name] == param_value + + for param_name, param_value in single_file_pipe.text_encoder_2.config.to_dict().items(): + if param_name in ["torch_dtype", "architectures", "_name_or_path"]: + continue + assert pipe.text_encoder_2.config.to_dict()[param_name] == param_value + + PARAMS_TO_IGNORE = ["torch_dtype", "_name_or_path", "architectures", "_use_default_values"] + for param_name, param_value in single_file_pipe.unet.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.unet.config[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" + + for param_name, param_value in single_file_pipe.vae.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.vae.config[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" + + +class StableDiffusionSSD1BControlNetPipelineFastTests(StableDiffusionXLControlNetPipelineFastTests): + def test_controlnet_sdxl_guess(self): + device = "cpu" + + components = self.get_dummy_components() + + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(device) + + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["guess_mode"] = True + + output = sd_pipe(**inputs) + image_slice = output.images[0, -3:, -3:, -1] + expected_slice = np.array( + [0.6831671, 0.5702532, 0.5459845, 0.6299793, 0.58563006, 0.6033695, 0.4493941, 0.46132287, 0.5035841] + ) + + # make sure that it's equal + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-4 + + def test_controlnet_sdxl_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLControlNetPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.6850, 0.5135, 0.5545, 0.7033, 0.6617, 0.5971, 0.4165, 0.5480, 0.5070]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_conditioning_channels(self): + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + mid_block_type="UNetMidBlock2D", + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + time_cond_proj_dim=None, + ) + + controlnet = ControlNetModel.from_unet(unet, conditioning_channels=4) + assert type(controlnet.mid_block) == UNetMidBlock2D + assert controlnet.conditioning_channels == 4 + + def get_dummy_components(self, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + mid_block_type="UNetMidBlock2D", + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + time_cond_proj_dim=time_cond_proj_dim, + ) + torch.manual_seed(0) + controlnet = ControlNetModel( + block_out_channels=(32, 64), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + conditioning_embedding_out_channels=(16, 32), + mid_block_type="UNetMidBlock2D", + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + torch.manual_seed(0) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "feature_extractor": None, + "image_encoder": None, + } + return components diff --git a/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_sdxl_img2img.py b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_sdxl_img2img.py new file mode 100755 index 0000000..7d2ba8c --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/controlnet/test_controlnet_sdxl_img2img.py @@ -0,0 +1,351 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + ControlNetModel, + EulerDiscreteScheduler, + StableDiffusionXLControlNetImg2ImgPipeline, + UNet2DConditionModel, +) +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import enable_full_determinism, floats_tensor, require_torch_gpu, torch_device + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, +) +from ..test_pipelines_common import ( + IPAdapterTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, +) + + +enable_full_determinism() + + +class ControlNetPipelineSDXLImg2ImgFastTests( + IPAdapterTesterMixin, + PipelineLatentTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineTesterMixin, + unittest.TestCase, +): + pipeline_class = StableDiffusionXLControlNetImg2ImgPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + image_latents_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self, skip_first_text_encoder=False): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64 if not skip_first_text_encoder else 32, + ) + torch.manual_seed(0) + controlnet = ControlNetModel( + block_out_channels=(32, 64), + layers_per_block=2, + in_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + conditioning_embedding_out_channels=(16, 32), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + torch.manual_seed(0) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "controlnet": controlnet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder if not skip_first_text_encoder else None, + "tokenizer": tokenizer if not skip_first_text_encoder else None, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "image_encoder": None, + "feature_extractor": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + controlnet_embedder_scale_factor = 2 + image = floats_tensor( + (1, 3, 32 * controlnet_embedder_scale_factor, 32 * controlnet_embedder_scale_factor), + rng=random.Random(seed), + ).to(device) + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + "image": image, + "control_image": image, + } + + return inputs + + def test_stable_diffusion_xl_controlnet_img2img(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array( + [0.5557202, 0.46418434, 0.46983826, 0.623529, 0.5557242, 0.49262643, 0.6070508, 0.5702978, 0.43777135] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_xl_controlnet_img2img_guess(self): + device = "cpu" + + components = self.get_dummy_components() + + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(device) + + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["guess_mode"] = True + + output = sd_pipe(**inputs) + image_slice = output.images[0, -3:, -3:, -1] + assert output.images.shape == (1, 64, 64, 3) + + expected_slice = np.array( + [0.5557202, 0.46418434, 0.46983826, 0.623529, 0.5557242, 0.49262643, 0.6070508, 0.5702978, 0.43777135] + ) + + # make sure that it's equal + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + # TODO(Patrick, Sayak) - skip for now as this requires more refiner tests + def test_save_load_optional_components(self): + pass + + @require_torch_gpu + def test_stable_diffusion_xl_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + pipe.unet.set_default_attn_processor() + + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_stable_diffusion_xl_multi_prompts(self): + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + + # forward with single prompt + inputs = self.get_dummy_inputs(torch_device) + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same prompt duplicated + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt_2"] = inputs["prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt_2"] = "different prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + # manually set a negative_prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same negative_prompt duplicated + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = inputs["negative_prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different negative_prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = "different negative prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + # copied from test_stable_diffusion_xl.py + def test_stable_diffusion_xl_prompt_embeds(self): + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + # forward without prompt embeds + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt"] = 2 * [inputs["prompt"]] + inputs["num_images_per_prompt"] = 2 + + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with prompt embeds + inputs = self.get_dummy_inputs(torch_device) + prompt = 2 * [inputs.pop("prompt")] + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = sd_pipe.encode_prompt(prompt) + + output = sd_pipe( + **inputs, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + ) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # make sure that it's equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 diff --git a/diffusers-0.27.0/tests/pipelines/controlnet/test_flax_controlnet.py b/diffusers-0.27.0/tests/pipelines/controlnet/test_flax_controlnet.py new file mode 100755 index 0000000..db19bd8 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/controlnet/test_flax_controlnet.py @@ -0,0 +1,127 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +from diffusers import FlaxControlNetModel, FlaxStableDiffusionControlNetPipeline +from diffusers.utils import is_flax_available, load_image +from diffusers.utils.testing_utils import require_flax, slow + + +if is_flax_available(): + import jax + import jax.numpy as jnp + from flax.jax_utils import replicate + from flax.training.common_utils import shard + + +@slow +@require_flax +class FlaxControlNetPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + + def test_canny(self): + controlnet, controlnet_params = FlaxControlNetModel.from_pretrained( + "lllyasviel/sd-controlnet-canny", from_pt=True, dtype=jnp.bfloat16 + ) + pipe, params = FlaxStableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", controlnet=controlnet, from_pt=True, dtype=jnp.bfloat16 + ) + params["controlnet"] = controlnet_params + + prompts = "bird" + num_samples = jax.device_count() + prompt_ids = pipe.prepare_text_inputs([prompts] * num_samples) + + canny_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/bird_canny.png" + ) + processed_image = pipe.prepare_image_inputs([canny_image] * num_samples) + + rng = jax.random.PRNGKey(0) + rng = jax.random.split(rng, jax.device_count()) + + p_params = replicate(params) + prompt_ids = shard(prompt_ids) + processed_image = shard(processed_image) + + images = pipe( + prompt_ids=prompt_ids, + image=processed_image, + params=p_params, + prng_seed=rng, + num_inference_steps=50, + jit=True, + ).images + assert images.shape == (jax.device_count(), 1, 768, 512, 3) + + images = images.reshape((images.shape[0] * images.shape[1],) + images.shape[-3:]) + image_slice = images[0, 253:256, 253:256, -1] + + output_slice = jnp.asarray(jax.device_get(image_slice.flatten())) + expected_slice = jnp.array( + [0.167969, 0.116699, 0.081543, 0.154297, 0.132812, 0.108887, 0.169922, 0.169922, 0.205078] + ) + print(f"output_slice: {output_slice}") + assert jnp.abs(output_slice - expected_slice).max() < 1e-2 + + def test_pose(self): + controlnet, controlnet_params = FlaxControlNetModel.from_pretrained( + "lllyasviel/sd-controlnet-openpose", from_pt=True, dtype=jnp.bfloat16 + ) + pipe, params = FlaxStableDiffusionControlNetPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", controlnet=controlnet, from_pt=True, dtype=jnp.bfloat16 + ) + params["controlnet"] = controlnet_params + + prompts = "Chef in the kitchen" + num_samples = jax.device_count() + prompt_ids = pipe.prepare_text_inputs([prompts] * num_samples) + + pose_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd_controlnet/pose.png" + ) + processed_image = pipe.prepare_image_inputs([pose_image] * num_samples) + + rng = jax.random.PRNGKey(0) + rng = jax.random.split(rng, jax.device_count()) + + p_params = replicate(params) + prompt_ids = shard(prompt_ids) + processed_image = shard(processed_image) + + images = pipe( + prompt_ids=prompt_ids, + image=processed_image, + params=p_params, + prng_seed=rng, + num_inference_steps=50, + jit=True, + ).images + assert images.shape == (jax.device_count(), 1, 768, 512, 3) + + images = images.reshape((images.shape[0] * images.shape[1],) + images.shape[-3:]) + image_slice = images[0, 253:256, 253:256, -1] + + output_slice = jnp.asarray(jax.device_get(image_slice.flatten())) + expected_slice = jnp.array( + [[0.271484, 0.261719, 0.275391, 0.277344, 0.279297, 0.291016, 0.294922, 0.302734, 0.302734]] + ) + print(f"output_slice: {output_slice}") + assert jnp.abs(output_slice - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/dance_diffusion/__init__.py b/diffusers-0.27.0/tests/pipelines/dance_diffusion/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/dance_diffusion/test_dance_diffusion.py b/diffusers-0.27.0/tests/pipelines/dance_diffusion/test_dance_diffusion.py new file mode 100755 index 0000000..212505c --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/dance_diffusion/test_dance_diffusion.py @@ -0,0 +1,161 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch + +from diffusers import DanceDiffusionPipeline, IPNDMScheduler, UNet1DModel +from diffusers.utils.testing_utils import enable_full_determinism, nightly, require_torch_gpu, skip_mps, torch_device + +from ..pipeline_params import UNCONDITIONAL_AUDIO_GENERATION_BATCH_PARAMS, UNCONDITIONAL_AUDIO_GENERATION_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class DanceDiffusionPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = DanceDiffusionPipeline + params = UNCONDITIONAL_AUDIO_GENERATION_PARAMS + required_optional_params = PipelineTesterMixin.required_optional_params - { + "callback", + "latents", + "callback_steps", + "output_type", + "num_images_per_prompt", + } + batch_params = UNCONDITIONAL_AUDIO_GENERATION_BATCH_PARAMS + test_attention_slicing = False + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet1DModel( + block_out_channels=(32, 32, 64), + extra_in_channels=16, + sample_size=512, + sample_rate=16_000, + in_channels=2, + out_channels=2, + flip_sin_to_cos=True, + use_timestep_embedding=False, + time_embedding_type="fourier", + mid_block_type="UNetMidBlock1D", + down_block_types=("DownBlock1DNoSkip", "DownBlock1D", "AttnDownBlock1D"), + up_block_types=("AttnUpBlock1D", "UpBlock1D", "UpBlock1DNoSkip"), + ) + scheduler = IPNDMScheduler() + + components = { + "unet": unet, + "scheduler": scheduler, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "batch_size": 1, + "generator": generator, + "num_inference_steps": 4, + } + return inputs + + def test_dance_diffusion(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = DanceDiffusionPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = pipe(**inputs) + audio = output.audios + + audio_slice = audio[0, -3:, -3:] + + assert audio.shape == (1, 2, components["unet"].sample_size) + expected_slice = np.array([-0.7265, 1.0000, -0.8388, 0.1175, 0.9498, -1.0000]) + assert np.abs(audio_slice.flatten() - expected_slice).max() < 1e-2 + + @skip_mps + def test_save_load_local(self): + return super().test_save_load_local() + + @skip_mps + def test_dict_tuple_outputs_equivalent(self): + return super().test_dict_tuple_outputs_equivalent(expected_max_difference=3e-3) + + @skip_mps + def test_save_load_optional_components(self): + return super().test_save_load_optional_components() + + @skip_mps + def test_attention_slicing_forward_pass(self): + return super().test_attention_slicing_forward_pass() + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + +@nightly +@require_torch_gpu +class PipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_dance_diffusion(self): + device = torch_device + + pipe = DanceDiffusionPipeline.from_pretrained("harmonai/maestro-150k") + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(0) + output = pipe(generator=generator, num_inference_steps=100, audio_length_in_s=4.096) + audio = output.audios + + audio_slice = audio[0, -3:, -3:] + + assert audio.shape == (1, 2, pipe.unet.sample_size) + expected_slice = np.array([-0.0192, -0.0231, -0.0318, -0.0059, 0.0002, -0.0020]) + + assert np.abs(audio_slice.flatten() - expected_slice).max() < 1e-2 + + def test_dance_diffusion_fp16(self): + device = torch_device + + pipe = DanceDiffusionPipeline.from_pretrained("harmonai/maestro-150k", torch_dtype=torch.float16) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(0) + output = pipe(generator=generator, num_inference_steps=100, audio_length_in_s=4.096) + audio = output.audios + + audio_slice = audio[0, -3:, -3:] + + assert audio.shape == (1, 2, pipe.unet.sample_size) + expected_slice = np.array([-0.0367, -0.0488, -0.0771, -0.0525, -0.0444, -0.0341]) + + assert np.abs(audio_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/ddim/__init__.py b/diffusers-0.27.0/tests/pipelines/ddim/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/ddim/test_ddim.py b/diffusers-0.27.0/tests/pipelines/ddim/test_ddim.py new file mode 100755 index 0000000..0d84a8e --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/ddim/test_ddim.py @@ -0,0 +1,143 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch + +from diffusers import DDIMPipeline, DDIMScheduler, UNet2DModel +from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, slow, torch_device + +from ..pipeline_params import UNCONDITIONAL_IMAGE_GENERATION_BATCH_PARAMS, UNCONDITIONAL_IMAGE_GENERATION_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class DDIMPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = DDIMPipeline + params = UNCONDITIONAL_IMAGE_GENERATION_PARAMS + required_optional_params = PipelineTesterMixin.required_optional_params - { + "num_images_per_prompt", + "latents", + "callback", + "callback_steps", + } + batch_params = UNCONDITIONAL_IMAGE_GENERATION_BATCH_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "AttnDownBlock2D"), + up_block_types=("AttnUpBlock2D", "UpBlock2D"), + ) + scheduler = DDIMScheduler() + components = {"unet": unet, "scheduler": scheduler} + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "batch_size": 1, + "generator": generator, + "num_inference_steps": 2, + "output_type": "numpy", + } + return inputs + + def test_inference(self): + device = "cpu" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + self.assertEqual(image.shape, (1, 32, 32, 3)) + expected_slice = np.array( + [1.000e00, 5.717e-01, 4.717e-01, 1.000e00, 0.000e00, 1.000e00, 3.000e-04, 0.000e00, 9.000e-04] + ) + max_diff = np.abs(image_slice.flatten() - expected_slice).max() + self.assertLessEqual(max_diff, 1e-3) + + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent(expected_max_difference=3e-3) + + def test_save_load_local(self): + super().test_save_load_local(expected_max_difference=3e-3) + + def test_save_load_optional_components(self): + super().test_save_load_optional_components(expected_max_difference=3e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + +@slow +@require_torch_gpu +class DDIMPipelineIntegrationTests(unittest.TestCase): + def test_inference_cifar10(self): + model_id = "google/ddpm-cifar10-32" + + unet = UNet2DModel.from_pretrained(model_id) + scheduler = DDIMScheduler() + + ddim = DDIMPipeline(unet=unet, scheduler=scheduler) + ddim.to(torch_device) + ddim.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(0) + image = ddim(generator=generator, eta=0.0, output_type="numpy").images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.1723, 0.1617, 0.1600, 0.1626, 0.1497, 0.1513, 0.1505, 0.1442, 0.1453]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_inference_ema_bedroom(self): + model_id = "google/ddpm-ema-bedroom-256" + + unet = UNet2DModel.from_pretrained(model_id) + scheduler = DDIMScheduler.from_pretrained(model_id) + + ddpm = DDIMPipeline(unet=unet, scheduler=scheduler) + ddpm.to(torch_device) + ddpm.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(0) + image = ddpm(generator=generator, output_type="numpy").images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 256, 256, 3) + expected_slice = np.array([0.0060, 0.0201, 0.0344, 0.0024, 0.0018, 0.0002, 0.0022, 0.0000, 0.0069]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/ddpm/__init__.py b/diffusers-0.27.0/tests/pipelines/ddpm/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/ddpm/test_ddpm.py b/diffusers-0.27.0/tests/pipelines/ddpm/test_ddpm.py new file mode 100755 index 0000000..bf25ced --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/ddpm/test_ddpm.py @@ -0,0 +1,111 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch + +from diffusers import DDPMPipeline, DDPMScheduler, UNet2DModel +from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, slow, torch_device + + +enable_full_determinism() + + +class DDPMPipelineFastTests(unittest.TestCase): + @property + def dummy_uncond_unet(self): + torch.manual_seed(0) + model = UNet2DModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "AttnDownBlock2D"), + up_block_types=("AttnUpBlock2D", "UpBlock2D"), + ) + return model + + def test_fast_inference(self): + device = "cpu" + unet = self.dummy_uncond_unet + scheduler = DDPMScheduler() + + ddpm = DDPMPipeline(unet=unet, scheduler=scheduler) + ddpm.to(device) + ddpm.set_progress_bar_config(disable=None) + + generator = torch.Generator(device=device).manual_seed(0) + image = ddpm(generator=generator, num_inference_steps=2, output_type="numpy").images + + generator = torch.Generator(device=device).manual_seed(0) + image_from_tuple = ddpm(generator=generator, num_inference_steps=2, output_type="numpy", return_dict=False)[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array( + [9.956e-01, 5.785e-01, 4.675e-01, 9.930e-01, 0.0, 1.000, 1.199e-03, 2.648e-04, 5.101e-04] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + def test_inference_predict_sample(self): + unet = self.dummy_uncond_unet + scheduler = DDPMScheduler(prediction_type="sample") + + ddpm = DDPMPipeline(unet=unet, scheduler=scheduler) + ddpm.to(torch_device) + ddpm.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(0) + image = ddpm(generator=generator, num_inference_steps=2, output_type="numpy").images + + generator = torch.manual_seed(0) + image_eps = ddpm(generator=generator, num_inference_steps=2, output_type="numpy")[0] + + image_slice = image[0, -3:, -3:, -1] + image_eps_slice = image_eps[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + tolerance = 1e-2 if torch_device != "mps" else 3e-2 + assert np.abs(image_slice.flatten() - image_eps_slice.flatten()).max() < tolerance + + +@slow +@require_torch_gpu +class DDPMPipelineIntegrationTests(unittest.TestCase): + def test_inference_cifar10(self): + model_id = "google/ddpm-cifar10-32" + + unet = UNet2DModel.from_pretrained(model_id) + scheduler = DDPMScheduler.from_pretrained(model_id) + + ddpm = DDPMPipeline(unet=unet, scheduler=scheduler) + ddpm.to(torch_device) + ddpm.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(0) + image = ddpm(generator=generator, output_type="numpy").images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.4200, 0.3588, 0.1939, 0.3847, 0.3382, 0.2647, 0.4155, 0.3582, 0.3385]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/deepfloyd_if/__init__.py b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/__init__.py new file mode 100755 index 0000000..094254a --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/__init__.py @@ -0,0 +1,272 @@ +import tempfile + +import numpy as np +import torch +from transformers import AutoTokenizer, T5EncoderModel + +from diffusers import DDPMScheduler, UNet2DConditionModel +from diffusers.models.attention_processor import AttnAddedKVProcessor +from diffusers.pipelines.deepfloyd_if import IFWatermarker +from diffusers.utils.testing_utils import torch_device + +from ..test_pipelines_common import to_np + + +# WARN: the hf-internal-testing/tiny-random-t5 text encoder has some non-determinism in the `save_load` tests. + + +class IFPipelineTesterMixin: + def _get_dummy_components(self): + torch.manual_seed(0) + text_encoder = T5EncoderModel.from_pretrained("hf-internal-testing/tiny-random-t5") + + torch.manual_seed(0) + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-t5") + + torch.manual_seed(0) + unet = UNet2DConditionModel( + sample_size=32, + layers_per_block=1, + block_out_channels=[32, 64], + down_block_types=[ + "ResnetDownsampleBlock2D", + "SimpleCrossAttnDownBlock2D", + ], + mid_block_type="UNetMidBlock2DSimpleCrossAttn", + up_block_types=["SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"], + in_channels=3, + out_channels=6, + cross_attention_dim=32, + encoder_hid_dim=32, + attention_head_dim=8, + addition_embed_type="text", + addition_embed_type_num_heads=2, + cross_attention_norm="group_norm", + resnet_time_scale_shift="scale_shift", + act_fn="gelu", + ) + unet.set_attn_processor(AttnAddedKVProcessor()) # For reproducibility tests + + torch.manual_seed(0) + scheduler = DDPMScheduler( + num_train_timesteps=1000, + beta_schedule="squaredcos_cap_v2", + beta_start=0.0001, + beta_end=0.02, + thresholding=True, + dynamic_thresholding_ratio=0.95, + sample_max_value=1.0, + prediction_type="epsilon", + variance_type="learned_range", + ) + + torch.manual_seed(0) + watermarker = IFWatermarker() + + return { + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "unet": unet, + "scheduler": scheduler, + "watermarker": watermarker, + "safety_checker": None, + "feature_extractor": None, + } + + def _get_superresolution_dummy_components(self): + torch.manual_seed(0) + text_encoder = T5EncoderModel.from_pretrained("hf-internal-testing/tiny-random-t5") + + torch.manual_seed(0) + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-t5") + + torch.manual_seed(0) + unet = UNet2DConditionModel( + sample_size=32, + layers_per_block=[1, 2], + block_out_channels=[32, 64], + down_block_types=[ + "ResnetDownsampleBlock2D", + "SimpleCrossAttnDownBlock2D", + ], + mid_block_type="UNetMidBlock2DSimpleCrossAttn", + up_block_types=["SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"], + in_channels=6, + out_channels=6, + cross_attention_dim=32, + encoder_hid_dim=32, + attention_head_dim=8, + addition_embed_type="text", + addition_embed_type_num_heads=2, + cross_attention_norm="group_norm", + resnet_time_scale_shift="scale_shift", + act_fn="gelu", + class_embed_type="timestep", + mid_block_scale_factor=1.414, + time_embedding_act_fn="gelu", + time_embedding_dim=32, + ) + unet.set_attn_processor(AttnAddedKVProcessor()) # For reproducibility tests + + torch.manual_seed(0) + scheduler = DDPMScheduler( + num_train_timesteps=1000, + beta_schedule="squaredcos_cap_v2", + beta_start=0.0001, + beta_end=0.02, + thresholding=True, + dynamic_thresholding_ratio=0.95, + sample_max_value=1.0, + prediction_type="epsilon", + variance_type="learned_range", + ) + + torch.manual_seed(0) + image_noising_scheduler = DDPMScheduler( + num_train_timesteps=1000, + beta_schedule="squaredcos_cap_v2", + beta_start=0.0001, + beta_end=0.02, + ) + + torch.manual_seed(0) + watermarker = IFWatermarker() + + return { + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "unet": unet, + "scheduler": scheduler, + "image_noising_scheduler": image_noising_scheduler, + "watermarker": watermarker, + "safety_checker": None, + "feature_extractor": None, + } + + # this test is modified from the base class because if pipelines set the text encoder + # as optional with the intention that the user is allowed to encode the prompt once + # and then pass the embeddings directly to the pipeline. The base class test uses + # the unmodified arguments from `self.get_dummy_inputs` which will pass the unencoded + # prompt to the pipeline when the text encoder is set to None, throwing an error. + # So we make the test reflect the intended usage of setting the text encoder to None. + def _test_save_load_optional_components(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + prompt = inputs["prompt"] + generator = inputs["generator"] + num_inference_steps = inputs["num_inference_steps"] + output_type = inputs["output_type"] + + if "image" in inputs: + image = inputs["image"] + else: + image = None + + if "mask_image" in inputs: + mask_image = inputs["mask_image"] + else: + mask_image = None + + if "original_image" in inputs: + original_image = inputs["original_image"] + else: + original_image = None + + prompt_embeds, negative_prompt_embeds = pipe.encode_prompt(prompt) + + # inputs with prompt converted to embeddings + inputs = { + "prompt_embeds": prompt_embeds, + "negative_prompt_embeds": negative_prompt_embeds, + "generator": generator, + "num_inference_steps": num_inference_steps, + "output_type": output_type, + } + + if image is not None: + inputs["image"] = image + + if mask_image is not None: + inputs["mask_image"] = mask_image + + if original_image is not None: + inputs["original_image"] = original_image + + # set all optional components to None + for optional_component in pipe._optional_components: + setattr(pipe, optional_component, None) + + output = pipe(**inputs)[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir) + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + pipe_loaded.unet.set_attn_processor(AttnAddedKVProcessor()) # For reproducibility tests + + for optional_component in pipe._optional_components: + self.assertTrue( + getattr(pipe_loaded, optional_component) is None, + f"`{optional_component}` did not stay set to None after loading.", + ) + + inputs = self.get_dummy_inputs(torch_device) + + generator = inputs["generator"] + num_inference_steps = inputs["num_inference_steps"] + output_type = inputs["output_type"] + + # inputs with prompt converted to embeddings + inputs = { + "prompt_embeds": prompt_embeds, + "negative_prompt_embeds": negative_prompt_embeds, + "generator": generator, + "num_inference_steps": num_inference_steps, + "output_type": output_type, + } + + if image is not None: + inputs["image"] = image + + if mask_image is not None: + inputs["mask_image"] = mask_image + + if original_image is not None: + inputs["original_image"] = original_image + + output_loaded = pipe_loaded(**inputs)[0] + + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess(max_diff, 1e-4) + + # Modified from `PipelineTesterMixin` to set the attn processor as it's not serialized. + # This should be handled in the base test and then this method can be removed. + def _test_save_load_local(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output = pipe(**inputs)[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir) + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + pipe_loaded.unet.set_attn_processor(AttnAddedKVProcessor()) # For reproducibility tests + + inputs = self.get_dummy_inputs(torch_device) + output_loaded = pipe_loaded(**inputs)[0] + + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess(max_diff, 1e-4) diff --git a/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if.py b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if.py new file mode 100755 index 0000000..96fd013 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if.py @@ -0,0 +1,120 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import torch + +from diffusers import ( + IFPipeline, +) +from diffusers.models.attention_processor import AttnAddedKVProcessor +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import load_numpy, require_torch_gpu, skip_mps, slow, torch_device + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference +from . import IFPipelineTesterMixin + + +@skip_mps +class IFPipelineFastTests(PipelineTesterMixin, IFPipelineTesterMixin, unittest.TestCase): + pipeline_class = IFPipeline + params = TEXT_TO_IMAGE_PARAMS - {"width", "height", "latents"} + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents"} + + def get_dummy_components(self): + return self._get_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "output_type": "numpy", + } + + return inputs + + def test_save_load_optional_components(self): + self._test_save_load_optional_components() + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_save_load_float16(self): + # Due to non-determinism in save load of the hf-internal-testing/tiny-random-t5 text encoder + super().test_save_load_float16(expected_max_diff=1e-1) + + def test_attention_slicing_forward_pass(self): + self._test_attention_slicing_forward_pass(expected_max_diff=1e-2) + + def test_save_load_local(self): + self._test_save_load_local() + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical( + expected_max_diff=1e-2, + ) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3) + + +@slow +@require_torch_gpu +class IFPipelineSlowTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_if_text_to_image(self): + pipe = IFPipeline.from_pretrained("DeepFloyd/IF-I-XL-v1.0", variant="fp16", torch_dtype=torch.float16) + pipe.unet.set_attn_processor(AttnAddedKVProcessor()) + pipe.enable_model_cpu_offload() + + torch.cuda.reset_max_memory_allocated() + torch.cuda.empty_cache() + torch.cuda.reset_peak_memory_stats() + + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipe( + prompt="anime turtle", + num_inference_steps=2, + generator=generator, + output_type="np", + ) + + image = output.images[0] + + mem_bytes = torch.cuda.max_memory_allocated() + assert mem_bytes < 12 * 10**9 + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if.npy" + ) + assert_mean_pixel_difference(image, expected_image) + pipe.remove_all_hooks() diff --git a/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_img2img.py b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_img2img.py new file mode 100755 index 0000000..17a5e37 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_img2img.py @@ -0,0 +1,131 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import torch + +from diffusers import IFImg2ImgPipeline +from diffusers.models.attention_processor import AttnAddedKVProcessor +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import floats_tensor, load_numpy, require_torch_gpu, skip_mps, slow, torch_device + +from ..pipeline_params import ( + TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, +) +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference +from . import IFPipelineTesterMixin + + +@skip_mps +class IFImg2ImgPipelineFastTests(PipelineTesterMixin, IFPipelineTesterMixin, unittest.TestCase): + pipeline_class = IFImg2ImgPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"width", "height"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents"} + + def get_dummy_components(self): + return self._get_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "output_type": "numpy", + } + + return inputs + + def test_save_load_optional_components(self): + self._test_save_load_optional_components() + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3) + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_save_load_float16(self): + # Due to non-determinism in save load of the hf-internal-testing/tiny-random-t5 text encoder + super().test_save_load_float16(expected_max_diff=1e-1) + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=1e-1) + + def test_attention_slicing_forward_pass(self): + self._test_attention_slicing_forward_pass(expected_max_diff=1e-2) + + def test_save_load_local(self): + self._test_save_load_local() + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical( + expected_max_diff=1e-2, + ) + + +@slow +@require_torch_gpu +class IFImg2ImgPipelineSlowTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_if_img2img(self): + pipe = IFImg2ImgPipeline.from_pretrained( + "DeepFloyd/IF-I-L-v1.0", + variant="fp16", + torch_dtype=torch.float16, + ) + pipe.unet.set_attn_processor(AttnAddedKVProcessor()) + pipe.enable_model_cpu_offload() + + image = floats_tensor((1, 3, 64, 64), rng=random.Random(0)).to(torch_device) + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipe( + prompt="anime turtle", + image=image, + num_inference_steps=2, + generator=generator, + output_type="np", + ) + image = output.images[0] + + mem_bytes = torch.cuda.max_memory_allocated() + assert mem_bytes < 12 * 10**9 + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if_img2img.npy" + ) + assert_mean_pixel_difference(image, expected_image) + + pipe.remove_all_hooks() diff --git a/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_img2img_superresolution.py b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_img2img_superresolution.py new file mode 100755 index 0000000..d37f7f4 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_img2img_superresolution.py @@ -0,0 +1,136 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import torch + +from diffusers import IFImg2ImgSuperResolutionPipeline +from diffusers.models.attention_processor import AttnAddedKVProcessor +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import floats_tensor, load_numpy, require_torch_gpu, skip_mps, slow, torch_device + +from ..pipeline_params import ( + TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, +) +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference +from . import IFPipelineTesterMixin + + +@skip_mps +class IFImg2ImgSuperResolutionPipelineFastTests(PipelineTesterMixin, IFPipelineTesterMixin, unittest.TestCase): + pipeline_class = IFImg2ImgSuperResolutionPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"width", "height"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS.union({"original_image"}) + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents"} + + def get_dummy_components(self): + return self._get_superresolution_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + original_image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = floats_tensor((1, 3, 16, 16), rng=random.Random(seed)).to(device) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "original_image": original_image, + "generator": generator, + "num_inference_steps": 2, + "output_type": "numpy", + } + + return inputs + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3) + + def test_save_load_optional_components(self): + self._test_save_load_optional_components() + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_save_load_float16(self): + # Due to non-determinism in save load of the hf-internal-testing/tiny-random-t5 text encoder + super().test_save_load_float16(expected_max_diff=1e-1) + + def test_attention_slicing_forward_pass(self): + self._test_attention_slicing_forward_pass(expected_max_diff=1e-2) + + def test_save_load_local(self): + self._test_save_load_local() + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical( + expected_max_diff=1e-2, + ) + + +@slow +@require_torch_gpu +class IFImg2ImgSuperResolutionPipelineSlowTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_if_img2img_superresolution(self): + pipe = IFImg2ImgSuperResolutionPipeline.from_pretrained( + "DeepFloyd/IF-II-L-v1.0", + variant="fp16", + torch_dtype=torch.float16, + ) + pipe.unet.set_attn_processor(AttnAddedKVProcessor()) + pipe.enable_model_cpu_offload() + + generator = torch.Generator(device="cpu").manual_seed(0) + + original_image = floats_tensor((1, 3, 256, 256), rng=random.Random(0)).to(torch_device) + image = floats_tensor((1, 3, 64, 64), rng=random.Random(0)).to(torch_device) + + output = pipe( + prompt="anime turtle", + image=image, + original_image=original_image, + generator=generator, + num_inference_steps=2, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (256, 256, 3) + + mem_bytes = torch.cuda.max_memory_allocated() + assert mem_bytes < 12 * 10**9 + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if_img2img_superresolution_stage_II.npy" + ) + assert_mean_pixel_difference(image, expected_image) + + pipe.remove_all_hooks() diff --git a/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_inpainting.py b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_inpainting.py new file mode 100755 index 0000000..85dea36 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_inpainting.py @@ -0,0 +1,134 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import torch + +from diffusers import IFInpaintingPipeline +from diffusers.models.attention_processor import AttnAddedKVProcessor +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import floats_tensor, load_numpy, require_torch_gpu, skip_mps, slow, torch_device + +from ..pipeline_params import ( + TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_INPAINTING_PARAMS, +) +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference +from . import IFPipelineTesterMixin + + +@skip_mps +class IFInpaintingPipelineFastTests(PipelineTesterMixin, IFPipelineTesterMixin, unittest.TestCase): + pipeline_class = IFInpaintingPipeline + params = TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {"width", "height"} + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents"} + + def get_dummy_components(self): + return self._get_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + mask_image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 2, + "output_type": "numpy", + } + + return inputs + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3) + + def test_save_load_optional_components(self): + self._test_save_load_optional_components() + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_save_load_float16(self): + # Due to non-determinism in save load of the hf-internal-testing/tiny-random-t5 text encoder + super().test_save_load_float16(expected_max_diff=1e-1) + + def test_attention_slicing_forward_pass(self): + self._test_attention_slicing_forward_pass(expected_max_diff=1e-2) + + def test_save_load_local(self): + self._test_save_load_local() + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical( + expected_max_diff=1e-2, + ) + + +@slow +@require_torch_gpu +class IFInpaintingPipelineSlowTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_if_inpainting(self): + pipe = IFInpaintingPipeline.from_pretrained( + "DeepFloyd/IF-I-XL-v1.0", variant="fp16", torch_dtype=torch.float16 + ) + pipe.unet.set_attn_processor(AttnAddedKVProcessor()) + pipe.enable_model_cpu_offload() + + # Super resolution test + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + image = floats_tensor((1, 3, 64, 64), rng=random.Random(0)).to(torch_device) + mask_image = floats_tensor((1, 3, 64, 64), rng=random.Random(1)).to(torch_device) + + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipe( + prompt="anime prompts", + image=image, + mask_image=mask_image, + num_inference_steps=2, + generator=generator, + output_type="np", + ) + image = output.images[0] + + mem_bytes = torch.cuda.max_memory_allocated() + assert mem_bytes < 12 * 10**9 + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if_inpainting.npy" + ) + assert_mean_pixel_difference(image, expected_image) + pipe.remove_all_hooks() diff --git a/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_inpainting_superresolution.py b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_inpainting_superresolution.py new file mode 100755 index 0000000..f8e782d --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_inpainting_superresolution.py @@ -0,0 +1,143 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import torch + +from diffusers import IFInpaintingSuperResolutionPipeline +from diffusers.models.attention_processor import AttnAddedKVProcessor +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import floats_tensor, load_numpy, require_torch_gpu, skip_mps, slow, torch_device + +from ..pipeline_params import ( + TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_INPAINTING_PARAMS, +) +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference +from . import IFPipelineTesterMixin + + +@skip_mps +class IFInpaintingSuperResolutionPipelineFastTests(PipelineTesterMixin, IFPipelineTesterMixin, unittest.TestCase): + pipeline_class = IFInpaintingSuperResolutionPipeline + params = TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {"width", "height"} + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS.union({"original_image"}) + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents"} + + def get_dummy_components(self): + return self._get_superresolution_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + image = floats_tensor((1, 3, 16, 16), rng=random.Random(seed)).to(device) + original_image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + mask_image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "original_image": original_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 2, + "output_type": "numpy", + } + + return inputs + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3) + + def test_save_load_optional_components(self): + self._test_save_load_optional_components() + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_save_load_float16(self): + # Due to non-determinism in save load of the hf-internal-testing/tiny-random-t5 text encoder + super().test_save_load_float16(expected_max_diff=1e-1) + + def test_attention_slicing_forward_pass(self): + self._test_attention_slicing_forward_pass(expected_max_diff=1e-2) + + def test_save_load_local(self): + self._test_save_load_local() + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical( + expected_max_diff=1e-2, + ) + + +@slow +@require_torch_gpu +class IFInpaintingSuperResolutionPipelineSlowTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_if_inpainting_superresolution(self): + pipe = IFInpaintingSuperResolutionPipeline.from_pretrained( + "DeepFloyd/IF-II-L-v1.0", variant="fp16", torch_dtype=torch.float16 + ) + pipe.unet.set_attn_processor(AttnAddedKVProcessor()) + pipe.enable_model_cpu_offload() + + # Super resolution test + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + generator = torch.Generator(device="cpu").manual_seed(0) + + image = floats_tensor((1, 3, 64, 64), rng=random.Random(0)).to(torch_device) + original_image = floats_tensor((1, 3, 256, 256), rng=random.Random(0)).to(torch_device) + mask_image = floats_tensor((1, 3, 256, 256), rng=random.Random(1)).to(torch_device) + + output = pipe( + prompt="anime turtle", + image=image, + original_image=original_image, + mask_image=mask_image, + generator=generator, + num_inference_steps=2, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (256, 256, 3) + + mem_bytes = torch.cuda.max_memory_allocated() + assert mem_bytes < 12 * 10**9 + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if_inpainting_superresolution_stage_II.npy" + ) + assert_mean_pixel_difference(image, expected_image) + + pipe.remove_all_hooks() diff --git a/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_superresolution.py b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_superresolution.py new file mode 100755 index 0000000..ca20517 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/deepfloyd_if/test_if_superresolution.py @@ -0,0 +1,130 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import torch + +from diffusers import IFSuperResolutionPipeline +from diffusers.models.attention_processor import AttnAddedKVProcessor +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import floats_tensor, load_numpy, require_torch_gpu, skip_mps, slow, torch_device + +from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference +from . import IFPipelineTesterMixin + + +@skip_mps +class IFSuperResolutionPipelineFastTests(PipelineTesterMixin, IFPipelineTesterMixin, unittest.TestCase): + pipeline_class = IFSuperResolutionPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"width", "height"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents"} + + def get_dummy_components(self): + return self._get_superresolution_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "output_type": "numpy", + } + + return inputs + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3) + + def test_save_load_optional_components(self): + self._test_save_load_optional_components() + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_save_load_float16(self): + # Due to non-determinism in save load of the hf-internal-testing/tiny-random-t5 text encoder + super().test_save_load_float16(expected_max_diff=1e-1) + + def test_attention_slicing_forward_pass(self): + self._test_attention_slicing_forward_pass(expected_max_diff=1e-2) + + def test_save_load_local(self): + self._test_save_load_local() + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical( + expected_max_diff=1e-2, + ) + + +@slow +@require_torch_gpu +class IFSuperResolutionPipelineSlowTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_if_superresolution(self): + pipe = IFSuperResolutionPipeline.from_pretrained( + "DeepFloyd/IF-II-L-v1.0", variant="fp16", torch_dtype=torch.float16 + ) + pipe.unet.set_attn_processor(AttnAddedKVProcessor()) + pipe.enable_model_cpu_offload() + + # Super resolution test + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + image = floats_tensor((1, 3, 64, 64), rng=random.Random(0)).to(torch_device) + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipe( + prompt="anime turtle", + image=image, + generator=generator, + num_inference_steps=2, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (256, 256, 3) + + mem_bytes = torch.cuda.max_memory_allocated() + assert mem_bytes < 12 * 10**9 + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/if/test_if_superresolution_stage_II.npy" + ) + assert_mean_pixel_difference(image, expected_image) + + pipe.remove_all_hooks() diff --git a/diffusers-0.27.0/tests/pipelines/dit/__init__.py b/diffusers-0.27.0/tests/pipelines/dit/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/dit/test_dit.py b/diffusers-0.27.0/tests/pipelines/dit/test_dit.py new file mode 100755 index 0000000..1f36776 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/dit/test_dit.py @@ -0,0 +1,151 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch + +from diffusers import AutoencoderKL, DDIMScheduler, DiTPipeline, DPMSolverMultistepScheduler, Transformer2DModel +from diffusers.utils import is_xformers_available +from diffusers.utils.testing_utils import enable_full_determinism, load_numpy, nightly, require_torch_gpu, torch_device + +from ..pipeline_params import ( + CLASS_CONDITIONED_IMAGE_GENERATION_BATCH_PARAMS, + CLASS_CONDITIONED_IMAGE_GENERATION_PARAMS, +) +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class DiTPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = DiTPipeline + params = CLASS_CONDITIONED_IMAGE_GENERATION_PARAMS + required_optional_params = PipelineTesterMixin.required_optional_params - { + "latents", + "num_images_per_prompt", + "callback", + "callback_steps", + } + batch_params = CLASS_CONDITIONED_IMAGE_GENERATION_BATCH_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + transformer = Transformer2DModel( + sample_size=16, + num_layers=2, + patch_size=4, + attention_head_dim=8, + num_attention_heads=2, + in_channels=4, + out_channels=8, + attention_bias=True, + activation_fn="gelu-approximate", + num_embeds_ada_norm=1000, + norm_type="ada_norm_zero", + norm_elementwise_affine=False, + ) + vae = AutoencoderKL() + scheduler = DDIMScheduler() + components = {"transformer": transformer.eval(), "vae": vae.eval(), "scheduler": scheduler} + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "class_labels": [1], + "generator": generator, + "num_inference_steps": 2, + "output_type": "numpy", + } + return inputs + + def test_inference(self): + device = "cpu" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + self.assertEqual(image.shape, (1, 16, 16, 3)) + expected_slice = np.array([0.2946, 0.6601, 0.4329, 0.3296, 0.4144, 0.5319, 0.7273, 0.5013, 0.4457]) + max_diff = np.abs(image_slice.flatten() - expected_slice).max() + self.assertLessEqual(max_diff, 1e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=1e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3) + + +@nightly +@require_torch_gpu +class DiTPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_dit_256(self): + generator = torch.manual_seed(0) + + pipe = DiTPipeline.from_pretrained("facebook/DiT-XL-2-256") + pipe.to("cuda") + + words = ["vase", "umbrella", "white shark", "white wolf"] + ids = pipe.get_label_ids(words) + + images = pipe(ids, generator=generator, num_inference_steps=40, output_type="np").images + + for word, image in zip(words, images): + expected_image = load_numpy( + f"https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/dit/{word}.npy" + ) + assert np.abs((expected_image - image).max()) < 1e-2 + + def test_dit_512(self): + pipe = DiTPipeline.from_pretrained("facebook/DiT-XL-2-512") + pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + pipe.to("cuda") + + words = ["vase", "umbrella"] + ids = pipe.get_label_ids(words) + + generator = torch.manual_seed(0) + images = pipe(ids, generator=generator, num_inference_steps=25, output_type="np").images + + for word, image in zip(words, images): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + f"/dit/{word}_512.npy" + ) + + assert np.abs((expected_image - image).max()) < 1e-1 diff --git a/diffusers-0.27.0/tests/pipelines/i2vgen_xl/__init__.py b/diffusers-0.27.0/tests/pipelines/i2vgen_xl/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/i2vgen_xl/test_i2vgenxl.py b/diffusers-0.27.0/tests/pipelines/i2vgen_xl/test_i2vgenxl.py new file mode 100755 index 0000000..aeda671 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/i2vgen_xl/test_i2vgenxl.py @@ -0,0 +1,265 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextConfig, + CLIPTextModel, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + I2VGenXLPipeline, +) +from diffusers.models.unets import I2VGenXLUNet +from diffusers.utils import is_xformers_available, load_image +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + numpy_cosine_similarity_distance, + print_tensor_test, + require_torch_gpu, + skip_mps, + slow, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin, SDFunctionTesterMixin + + +enable_full_determinism() + + +@skip_mps +class I2VGenXLPipelineFastTests(SDFunctionTesterMixin, PipelineTesterMixin, unittest.TestCase): + pipeline_class = I2VGenXLPipeline + params = frozenset(["prompt", "negative_prompt", "image"]) + batch_params = frozenset(["prompt", "negative_prompt", "image", "generator"]) + # No `output_type`. + required_optional_params = frozenset(["num_inference_steps", "generator", "latents", "return_dict"]) + + def get_dummy_components(self): + torch.manual_seed(0) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + + torch.manual_seed(0) + unet = I2VGenXLUNet( + block_out_channels=(4, 8), + layers_per_block=1, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("CrossAttnDownBlock3D", "DownBlock3D"), + up_block_types=("UpBlock3D", "CrossAttnUpBlock3D"), + cross_attention_dim=4, + attention_head_dim=4, + num_attention_heads=None, + norm_num_groups=2, + ) + + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=(8,), + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D"], + latent_channels=4, + sample_size=32, + norm_num_groups=2, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=4, + intermediate_size=16, + layer_norm_eps=1e-05, + num_attention_heads=2, + num_hidden_layers=2, + pad_token_id=1, + vocab_size=1000, + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + torch.manual_seed(0) + vision_encoder_config = CLIPVisionConfig( + hidden_size=4, + projection_dim=4, + num_hidden_layers=2, + num_attention_heads=2, + image_size=32, + intermediate_size=16, + patch_size=1, + ) + image_encoder = CLIPVisionModelWithProjection(vision_encoder_config) + + torch.manual_seed(0) + feature_extractor = CLIPImageProcessor(crop_size=32, size=32) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "image_encoder": image_encoder, + "tokenizer": tokenizer, + "feature_extractor": feature_extractor, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + input_image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": input_image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "pt", + "num_frames": 4, + "width": 32, + "height": 32, + } + return inputs + + def test_text_to_video_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["output_type"] = "np" + frames = pipe(**inputs).frames + + image_slice = frames[0][0][-3:, -3:, -1] + + assert frames[0][0].shape == (32, 32, 3) + expected_slice = np.array([0.5146, 0.6525, 0.6032, 0.5204, 0.5675, 0.4125, 0.3016, 0.5172, 0.4095]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_save_load_local(self): + super().test_save_load_local(expected_max_difference=0.006) + + def test_sequential_cpu_offload_forward_pass(self): + super().test_sequential_cpu_offload_forward_pass(expected_max_diff=0.008) + + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent(expected_max_difference=0.008) + + def test_save_load_optional_components(self): + super().test_save_load_optional_components(expected_max_difference=0.008) + + @unittest.skip("Deprecated functionality") + def test_attention_slicing_forward_pass(self): + pass + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(test_mean_pixel_difference=False, expected_max_diff=1e-2) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(batch_size=2, expected_max_diff=0.008) + + def test_model_cpu_offload_forward_pass(self): + super().test_model_cpu_offload_forward_pass(expected_max_diff=0.008) + + def test_num_videos_per_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["output_type"] = "np" + frames = pipe(**inputs, num_videos_per_prompt=2).frames + + assert frames.shape == (2, 4, 32, 32, 3) + assert frames[0][0].shape == (32, 32, 3) + + image_slice = frames[0][0][-3:, -3:, -1] + expected_slice = np.array([0.5146, 0.6525, 0.6032, 0.5204, 0.5675, 0.4125, 0.3016, 0.5172, 0.4095]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + +@slow +@require_torch_gpu +class I2VGenXLPipelineSlowTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_i2vgen_xl(self): + pipe = I2VGenXLPipeline.from_pretrained("ali-vilab/i2vgen-xl", torch_dtype=torch.float16, variant="fp16") + pipe = pipe.to(torch_device) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/pix2pix/cat_6.png?download=true" + ) + + generator = torch.Generator("cpu").manual_seed(0) + num_frames = 3 + + output = pipe( + image=image, + prompt="my cat", + num_frames=num_frames, + generator=generator, + num_inference_steps=3, + output_type="np", + ) + + image = output.frames[0] + assert image.shape == (num_frames, 704, 1280, 3) + + image_slice = image[0, -3:, -3:, -1] + print_tensor_test(image_slice.flatten()) + expected_slice = np.array([0.5482, 0.6244, 0.6274, 0.4584, 0.5935, 0.5937, 0.4579, 0.5767, 0.5892]) + assert numpy_cosine_similarity_distance(image_slice.flatten(), expected_slice.flatten()) < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/ip_adapters/test_ip_adapter_stable_diffusion.py b/diffusers-0.27.0/tests/pipelines/ip_adapters/test_ip_adapter_stable_diffusion.py new file mode 100755 index 0000000..6289ee8 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/ip_adapters/test_ip_adapter_stable_diffusion.py @@ -0,0 +1,539 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +from transformers import ( + CLIPImageProcessor, + CLIPVisionModelWithProjection, +) + +from diffusers import ( + StableDiffusionImg2ImgPipeline, + StableDiffusionInpaintPipeline, + StableDiffusionPipeline, + StableDiffusionXLImg2ImgPipeline, + StableDiffusionXLInpaintPipeline, + StableDiffusionXLPipeline, +) +from diffusers.image_processor import IPAdapterMaskProcessor +from diffusers.models.attention_processor import AttnProcessor, AttnProcessor2_0 +from diffusers.utils import load_image +from diffusers.utils.testing_utils import ( + enable_full_determinism, + is_flaky, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) + + +enable_full_determinism() + + +class IPAdapterNightlyTestsMixin(unittest.TestCase): + dtype = torch.float16 + + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_image_encoder(self, repo_id, subfolder): + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + repo_id, subfolder=subfolder, torch_dtype=self.dtype + ).to(torch_device) + return image_encoder + + def get_image_processor(self, repo_id): + image_processor = CLIPImageProcessor.from_pretrained(repo_id) + return image_processor + + def get_dummy_inputs(self, for_image_to_image=False, for_inpainting=False, for_sdxl=False, for_masks=False): + image = load_image( + "https://user-images.githubusercontent.com/24734142/266492875-2d50d223-8475-44f0-a7c6-08b51cb53572.png" + ) + if for_sdxl: + image = image.resize((1024, 1024)) + + input_kwargs = { + "prompt": "best quality, high quality", + "negative_prompt": "monochrome, lowres, bad anatomy, worst quality, low quality", + "num_inference_steps": 5, + "generator": torch.Generator(device="cpu").manual_seed(33), + "ip_adapter_image": image, + "output_type": "np", + } + if for_image_to_image: + image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/vermeer.jpg") + ip_image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/river.png") + + if for_sdxl: + image = image.resize((1024, 1024)) + ip_image = ip_image.resize((1024, 1024)) + + input_kwargs.update({"image": image, "ip_adapter_image": ip_image}) + + elif for_inpainting: + image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/inpaint_image.png") + mask = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/mask.png") + ip_image = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/girl.png") + + if for_sdxl: + image = image.resize((1024, 1024)) + mask = mask.resize((1024, 1024)) + ip_image = ip_image.resize((1024, 1024)) + + input_kwargs.update({"image": image, "mask_image": mask, "ip_adapter_image": ip_image}) + + elif for_masks: + face_image1 = load_image( + "https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/ip_mask_girl1.png" + ) + face_image2 = load_image( + "https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/ip_mask_girl2.png" + ) + mask1 = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/ip_mask_mask1.png") + mask2 = load_image("https://huggingface.co/datasets/YiYiXu/testing-images/resolve/main/ip_mask_mask2.png") + input_kwargs.update( + { + "ip_adapter_image": [[face_image1], [face_image2]], + "cross_attention_kwargs": {"ip_adapter_masks": [mask1, mask2]}, + } + ) + + return input_kwargs + + +@slow +@require_torch_gpu +class IPAdapterSDIntegrationTests(IPAdapterNightlyTestsMixin): + def test_text_to_image(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + pipeline = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", image_encoder=image_encoder, safety_checker=None, torch_dtype=self.dtype + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter_sd15.bin") + + inputs = self.get_dummy_inputs() + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + + expected_slice = np.array([0.80810547, 0.88183594, 0.9296875, 0.9189453, 0.9848633, 1.0, 0.97021484, 1.0, 1.0]) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter-plus_sd15.bin") + + inputs = self.get_dummy_inputs() + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + + expected_slice = np.array( + [0.30444336, 0.26513672, 0.22436523, 0.2758789, 0.25585938, 0.20751953, 0.25390625, 0.24633789, 0.21923828] + ) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + def test_image_to_image(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + pipeline = StableDiffusionImg2ImgPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", image_encoder=image_encoder, safety_checker=None, torch_dtype=self.dtype + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter_sd15.bin") + + inputs = self.get_dummy_inputs(for_image_to_image=True) + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + + expected_slice = np.array( + [0.22167969, 0.21875, 0.21728516, 0.22607422, 0.21948242, 0.23925781, 0.22387695, 0.25268555, 0.2722168] + ) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter-plus_sd15.bin") + + inputs = self.get_dummy_inputs(for_image_to_image=True) + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + + expected_slice = np.array( + [0.35913086, 0.265625, 0.26367188, 0.24658203, 0.19750977, 0.39990234, 0.15258789, 0.20336914, 0.5517578] + ) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + def test_inpainting(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + pipeline = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", image_encoder=image_encoder, safety_checker=None, torch_dtype=self.dtype + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter_sd15.bin") + + inputs = self.get_dummy_inputs(for_inpainting=True) + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + + expected_slice = np.array( + [0.27148438, 0.24047852, 0.22167969, 0.23217773, 0.21118164, 0.21142578, 0.21875, 0.20751953, 0.20019531] + ) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter-plus_sd15.bin") + + inputs = self.get_dummy_inputs(for_inpainting=True) + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + def test_text_to_image_model_cpu_offload(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + pipeline = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", image_encoder=image_encoder, safety_checker=None, torch_dtype=self.dtype + ) + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter_sd15.bin") + pipeline.to(torch_device) + + inputs = self.get_dummy_inputs() + output_without_offload = pipeline(**inputs).images + + pipeline.enable_model_cpu_offload() + inputs = self.get_dummy_inputs() + output_with_offload = pipeline(**inputs).images + max_diff = np.abs(output_with_offload - output_without_offload).max() + self.assertLess(max_diff, 1e-3, "CPU offloading should not affect the inference results") + + offloaded_modules = [ + v + for k, v in pipeline.components.items() + if isinstance(v, torch.nn.Module) and k not in pipeline._exclude_from_cpu_offload + ] + ( + self.assertTrue(all(v.device.type == "cpu" for v in offloaded_modules)), + f"Not offloaded: {[v for v in offloaded_modules if v.device.type != 'cpu']}", + ) + + def test_text_to_image_full_face(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + pipeline = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", image_encoder=image_encoder, safety_checker=None, torch_dtype=self.dtype + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter-full-face_sd15.bin") + pipeline.set_ip_adapter_scale(0.7) + + inputs = self.get_dummy_inputs() + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + expected_slice = np.array([0.1704, 0.1296, 0.1272, 0.2212, 0.1514, 0.1479, 0.4172, 0.4263, 0.4360]) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + def test_unload(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + pipeline = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", image_encoder=image_encoder, safety_checker=None, torch_dtype=self.dtype + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="models", weight_name="ip-adapter_sd15.bin") + pipeline.set_ip_adapter_scale(0.7) + + pipeline.unload_ip_adapter() + + assert getattr(pipeline, "image_encoder") is None + assert getattr(pipeline, "feature_extractor") is not None + processors = [ + isinstance(attn_proc, (AttnProcessor, AttnProcessor2_0)) + for name, attn_proc in pipeline.unet.attn_processors.items() + ] + assert processors == [True] * len(processors) + + @is_flaky + def test_multi(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + pipeline = StableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", image_encoder=image_encoder, safety_checker=None, torch_dtype=self.dtype + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter( + "h94/IP-Adapter", subfolder="models", weight_name=["ip-adapter_sd15.bin", "ip-adapter-plus_sd15.bin"] + ) + pipeline.set_ip_adapter_scale([0.7, 0.3]) + + inputs = self.get_dummy_inputs() + ip_adapter_image = inputs["ip_adapter_image"] + inputs["ip_adapter_image"] = [ip_adapter_image, [ip_adapter_image] * 2] + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + expected_slice = np.array([0.5234, 0.5352, 0.5625, 0.5713, 0.5947, 0.6206, 0.5786, 0.6187, 0.6494]) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + +@slow +@require_torch_gpu +class IPAdapterSDXLIntegrationTests(IPAdapterNightlyTestsMixin): + def test_text_to_image_sdxl(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="sdxl_models/image_encoder") + feature_extractor = self.get_image_processor("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k") + + pipeline = StableDiffusionXLPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + image_encoder=image_encoder, + feature_extractor=feature_extractor, + torch_dtype=self.dtype, + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin") + + inputs = self.get_dummy_inputs() + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + + expected_slice = np.array( + [ + 0.09630299, + 0.09551358, + 0.08480701, + 0.09070173, + 0.09437338, + 0.09264627, + 0.08883232, + 0.09287417, + 0.09197289, + ] + ) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + + pipeline = StableDiffusionXLPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + image_encoder=image_encoder, + feature_extractor=feature_extractor, + torch_dtype=self.dtype, + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter( + "h94/IP-Adapter", + subfolder="sdxl_models", + weight_name="ip-adapter-plus_sdxl_vit-h.bin", + ) + + inputs = self.get_dummy_inputs() + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + + expected_slice = np.array( + [0.0576596, 0.05600825, 0.04479006, 0.05288461, 0.05461192, 0.05137569, 0.04867965, 0.05301541, 0.04939842] + ) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + def test_image_to_image_sdxl(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="sdxl_models/image_encoder") + feature_extractor = self.get_image_processor("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k") + + pipeline = StableDiffusionXLImg2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + image_encoder=image_encoder, + feature_extractor=feature_extractor, + torch_dtype=self.dtype, + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin") + + inputs = self.get_dummy_inputs(for_image_to_image=True) + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + + expected_slice = np.array( + [ + 0.06513795, + 0.07009393, + 0.07234055, + 0.07426041, + 0.07002589, + 0.06415862, + 0.07827643, + 0.07962808, + 0.07411247, + ] + ) + + assert np.allclose(image_slice, expected_slice, atol=1e-3) + + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + feature_extractor = self.get_image_processor("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k") + + pipeline = StableDiffusionXLImg2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + image_encoder=image_encoder, + feature_extractor=feature_extractor, + torch_dtype=self.dtype, + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter( + "h94/IP-Adapter", + subfolder="sdxl_models", + weight_name="ip-adapter-plus_sdxl_vit-h.bin", + ) + + inputs = self.get_dummy_inputs(for_image_to_image=True) + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + + expected_slice = np.array( + [ + 0.07126552, + 0.07025367, + 0.07348302, + 0.07580167, + 0.07467338, + 0.06918576, + 0.07480252, + 0.08279955, + 0.08547315, + ] + ) + + assert np.allclose(image_slice, expected_slice, atol=1e-3) + + def test_inpainting_sdxl(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="sdxl_models/image_encoder") + feature_extractor = self.get_image_processor("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k") + + pipeline = StableDiffusionXLInpaintPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + image_encoder=image_encoder, + feature_extractor=feature_extractor, + torch_dtype=self.dtype, + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter("h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter_sdxl.bin") + + inputs = self.get_dummy_inputs(for_inpainting=True) + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + image_slice.tolist() + + expected_slice = np.array( + [0.14181179, 0.1493012, 0.14283323, 0.14602411, 0.14915377, 0.15015268, 0.14725655, 0.15009224, 0.15164584] + ) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + feature_extractor = self.get_image_processor("laion/CLIP-ViT-bigG-14-laion2B-39B-b160k") + + pipeline = StableDiffusionXLInpaintPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + image_encoder=image_encoder, + feature_extractor=feature_extractor, + torch_dtype=self.dtype, + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter( + "h94/IP-Adapter", + subfolder="sdxl_models", + weight_name="ip-adapter-plus_sdxl_vit-h.bin", + ) + + inputs = self.get_dummy_inputs(for_inpainting=True) + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + image_slice.tolist() + + expected_slice = np.array([0.1398, 0.1476, 0.1407, 0.1442, 0.1470, 0.1480, 0.1449, 0.1481, 0.1494]) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + def test_ip_adapter_single_mask(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + pipeline = StableDiffusionXLPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + image_encoder=image_encoder, + torch_dtype=self.dtype, + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter( + "h94/IP-Adapter", subfolder="sdxl_models", weight_name="ip-adapter-plus-face_sdxl_vit-h.safetensors" + ) + pipeline.set_ip_adapter_scale(0.7) + + inputs = self.get_dummy_inputs(for_masks=True) + mask = inputs["cross_attention_kwargs"]["ip_adapter_masks"][0] + processor = IPAdapterMaskProcessor() + mask = processor.preprocess(mask) + inputs["cross_attention_kwargs"]["ip_adapter_masks"] = mask + inputs["ip_adapter_image"] = inputs["ip_adapter_image"][0] + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + expected_slice = np.array( + [0.7307304, 0.73450166, 0.73731124, 0.7377061, 0.7318013, 0.73720926, 0.74746597, 0.7409929, 0.74074936] + ) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 + + def test_ip_adapter_multiple_masks(self): + image_encoder = self.get_image_encoder(repo_id="h94/IP-Adapter", subfolder="models/image_encoder") + pipeline = StableDiffusionXLPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + image_encoder=image_encoder, + torch_dtype=self.dtype, + ) + pipeline.to(torch_device) + pipeline.load_ip_adapter( + "h94/IP-Adapter", subfolder="sdxl_models", weight_name=["ip-adapter-plus-face_sdxl_vit-h.safetensors"] * 2 + ) + pipeline.set_ip_adapter_scale([0.7] * 2) + + inputs = self.get_dummy_inputs(for_masks=True) + masks = inputs["cross_attention_kwargs"]["ip_adapter_masks"] + processor = IPAdapterMaskProcessor() + masks = processor.preprocess(masks) + inputs["cross_attention_kwargs"]["ip_adapter_masks"] = masks + images = pipeline(**inputs).images + image_slice = images[0, :3, :3, -1].flatten() + expected_slice = np.array( + [0.79474676, 0.7977683, 0.8013954, 0.7988008, 0.7970615, 0.8029355, 0.80614823, 0.8050743, 0.80627424] + ) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 5e-4 diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky/__init__.py b/diffusers-0.27.0/tests/pipelines/kandinsky/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky.py b/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky.py new file mode 100755 index 0000000..48b4520 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky.py @@ -0,0 +1,323 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from transformers import XLMRobertaTokenizerFast + +from diffusers import DDIMScheduler, KandinskyPipeline, KandinskyPriorPipeline, UNet2DConditionModel, VQModel +from diffusers.pipelines.kandinsky.text_encoder import MCLIPConfig, MultilingualCLIP +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_numpy, + require_torch_gpu, + slow, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +enable_full_determinism() + + +class Dummies: + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 32 + + @property + def dummy_tokenizer(self): + tokenizer = XLMRobertaTokenizerFast.from_pretrained("YiYiXu/tiny-random-mclip-base") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = MCLIPConfig( + numDims=self.cross_attention_dim, + transformerDimensions=self.text_embedder_hidden_size, + hidden_size=self.text_embedder_hidden_size, + intermediate_size=37, + num_attention_heads=4, + num_hidden_layers=5, + vocab_size=1005, + ) + + text_encoder = MultilingualCLIP(config) + text_encoder = text_encoder.eval() + + return text_encoder + + @property + def dummy_unet(self): + torch.manual_seed(0) + + model_kwargs = { + "in_channels": 4, + # Out channels is double in channels because predicts mean and variance + "out_channels": 8, + "addition_embed_type": "text_image", + "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"), + "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"), + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "layers_per_block": 1, + "encoder_hid_dim": self.text_embedder_hidden_size, + "encoder_hid_dim_type": "text_image_proj", + "cross_attention_dim": self.cross_attention_dim, + "attention_head_dim": 4, + "resnet_time_scale_shift": "scale_shift", + "class_embed_type": None, + } + + model = UNet2DConditionModel(**model_kwargs) + return model + + @property + def dummy_movq_kwargs(self): + return { + "block_out_channels": [32, 64], + "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"], + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 1, + "norm_num_groups": 8, + "norm_type": "spatial", + "num_vq_embeddings": 12, + "out_channels": 3, + "up_block_types": [ + "AttnUpDecoderBlock2D", + "UpDecoderBlock2D", + ], + "vq_embed_dim": 4, + } + + @property + def dummy_movq(self): + torch.manual_seed(0) + model = VQModel(**self.dummy_movq_kwargs) + return model + + def get_dummy_components(self): + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + unet = self.dummy_unet + movq = self.dummy_movq + + scheduler = DDIMScheduler( + num_train_timesteps=1000, + beta_schedule="linear", + beta_start=0.00085, + beta_end=0.012, + clip_sample=False, + set_alpha_to_one=False, + steps_offset=1, + prediction_type="epsilon", + thresholding=False, + ) + + components = { + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "unet": unet, + "scheduler": scheduler, + "movq": movq, + } + return components + + def get_dummy_inputs(self, device, seed=0): + image_embeds = floats_tensor((1, self.cross_attention_dim), rng=random.Random(seed)).to(device) + negative_image_embeds = floats_tensor((1, self.cross_attention_dim), rng=random.Random(seed + 1)).to(device) + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "horse", + "image_embeds": image_embeds, + "negative_image_embeds": negative_image_embeds, + "generator": generator, + "height": 64, + "width": 64, + "guidance_scale": 4.0, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + +class KandinskyPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyPipeline + params = [ + "prompt", + "image_embeds", + "negative_image_embeds", + ] + batch_params = ["prompt", "negative_prompt", "image_embeds", "negative_image_embeds"] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "negative_prompt", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + def get_dummy_components(self): + dummy = Dummies() + return dummy.get_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + dummy = Dummies() + return dummy.get_dummy_inputs(device=device, seed=seed) + + def test_kandinsky(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([1.0000, 1.0000, 0.2766, 1.0000, 0.5447, 0.1737, 1.0000, 0.4316, 0.9024]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + @require_torch_gpu + def test_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + +@slow +@require_torch_gpu +class KandinskyPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_kandinsky_text2img(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/kandinsky/kandinsky_text2img_cat_fp16.npy" + ) + + pipe_prior = KandinskyPriorPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16 + ) + pipe_prior.to(torch_device) + + pipeline = KandinskyPipeline.from_pretrained("kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16) + pipeline = pipeline.to(torch_device) + pipeline.set_progress_bar_config(disable=None) + + prompt = "red cat, 4k photo" + + generator = torch.Generator(device="cuda").manual_seed(0) + image_emb, zero_image_emb = pipe_prior( + prompt, + generator=generator, + num_inference_steps=5, + negative_prompt="", + ).to_tuple() + + generator = torch.Generator(device="cuda").manual_seed(0) + output = pipeline( + prompt, + image_embeds=image_emb, + negative_image_embeds=zero_image_emb, + generator=generator, + num_inference_steps=100, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (512, 512, 3) + + assert_mean_pixel_difference(image, expected_image) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_combined.py b/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_combined.py new file mode 100755 index 0000000..480e129 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_combined.py @@ -0,0 +1,361 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np + +from diffusers import KandinskyCombinedPipeline, KandinskyImg2ImgCombinedPipeline, KandinskyInpaintCombinedPipeline +from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, torch_device + +from ..test_pipelines_common import PipelineTesterMixin +from .test_kandinsky import Dummies +from .test_kandinsky_img2img import Dummies as Img2ImgDummies +from .test_kandinsky_inpaint import Dummies as InpaintDummies +from .test_kandinsky_prior import Dummies as PriorDummies + + +enable_full_determinism() + + +class KandinskyPipelineCombinedFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyCombinedPipeline + params = [ + "prompt", + ] + batch_params = ["prompt", "negative_prompt"] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "negative_prompt", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = True + + def get_dummy_components(self): + dummy = Dummies() + prior_dummy = PriorDummies() + components = dummy.get_dummy_components() + + components.update({f"prior_{k}": v for k, v in prior_dummy.get_dummy_components().items()}) + return components + + def get_dummy_inputs(self, device, seed=0): + prior_dummy = PriorDummies() + inputs = prior_dummy.get_dummy_inputs(device=device, seed=seed) + inputs.update( + { + "height": 64, + "width": 64, + } + ) + return inputs + + def test_kandinsky(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.0000, 0.0000, 0.6777, 0.1363, 0.3624, 0.7868, 0.3869, 0.3395, 0.5068]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + @require_torch_gpu + def test_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=1e-2) + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=2e-1) + + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent(expected_max_difference=5e-4) + + +class KandinskyPipelineImg2ImgCombinedFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyImg2ImgCombinedPipeline + params = ["prompt", "image"] + batch_params = ["prompt", "negative_prompt", "image"] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "negative_prompt", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + def get_dummy_components(self): + dummy = Img2ImgDummies() + prior_dummy = PriorDummies() + components = dummy.get_dummy_components() + + components.update({f"prior_{k}": v for k, v in prior_dummy.get_dummy_components().items()}) + return components + + def get_dummy_inputs(self, device, seed=0): + prior_dummy = PriorDummies() + dummy = Img2ImgDummies() + inputs = prior_dummy.get_dummy_inputs(device=device, seed=seed) + inputs.update(dummy.get_dummy_inputs(device=device, seed=seed)) + inputs.pop("image_embeds") + inputs.pop("negative_image_embeds") + return inputs + + def test_kandinsky(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.4260, 0.3596, 0.4571, 0.3890, 0.4087, 0.5137, 0.4819, 0.4116, 0.5053]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + @require_torch_gpu + def test_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=1e-2) + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=5e-1) + + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent(expected_max_difference=5e-4) + + def test_save_load_optional_components(self): + super().test_save_load_optional_components(expected_max_difference=5e-4) + + +class KandinskyPipelineInpaintCombinedFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyInpaintCombinedPipeline + params = ["prompt", "image", "mask_image"] + batch_params = ["prompt", "negative_prompt", "image", "mask_image"] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "negative_prompt", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + def get_dummy_components(self): + dummy = InpaintDummies() + prior_dummy = PriorDummies() + components = dummy.get_dummy_components() + + components.update({f"prior_{k}": v for k, v in prior_dummy.get_dummy_components().items()}) + return components + + def get_dummy_inputs(self, device, seed=0): + prior_dummy = PriorDummies() + dummy = InpaintDummies() + inputs = prior_dummy.get_dummy_inputs(device=device, seed=seed) + inputs.update(dummy.get_dummy_inputs(device=device, seed=seed)) + inputs.pop("image_embeds") + inputs.pop("negative_image_embeds") + return inputs + + def test_kandinsky(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.0477, 0.0808, 0.2972, 0.2705, 0.3620, 0.6247, 0.4464, 0.2870, 0.3530]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + @require_torch_gpu + def test_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=1e-2) + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=5e-1) + + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent(expected_max_difference=5e-4) + + def test_save_load_optional_components(self): + super().test_save_load_optional_components(expected_max_difference=5e-4) + + def test_save_load_local(self): + super().test_save_load_local(expected_max_difference=5e-3) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_img2img.py b/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_img2img.py new file mode 100755 index 0000000..484d9d9 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_img2img.py @@ -0,0 +1,417 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import XLMRobertaTokenizerFast + +from diffusers import ( + DDIMScheduler, + DDPMScheduler, + KandinskyImg2ImgPipeline, + KandinskyPriorPipeline, + UNet2DConditionModel, + VQModel, +) +from diffusers.pipelines.kandinsky.text_encoder import MCLIPConfig, MultilingualCLIP +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + nightly, + require_torch_gpu, + slow, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +enable_full_determinism() + + +class Dummies: + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 32 + + @property + def dummy_tokenizer(self): + tokenizer = XLMRobertaTokenizerFast.from_pretrained("YiYiXu/tiny-random-mclip-base") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = MCLIPConfig( + numDims=self.cross_attention_dim, + transformerDimensions=self.text_embedder_hidden_size, + hidden_size=self.text_embedder_hidden_size, + intermediate_size=37, + num_attention_heads=4, + num_hidden_layers=5, + vocab_size=1005, + ) + + text_encoder = MultilingualCLIP(config) + text_encoder = text_encoder.eval() + + return text_encoder + + @property + def dummy_unet(self): + torch.manual_seed(0) + + model_kwargs = { + "in_channels": 4, + # Out channels is double in channels because predicts mean and variance + "out_channels": 8, + "addition_embed_type": "text_image", + "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"), + "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"), + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "layers_per_block": 1, + "encoder_hid_dim": self.text_embedder_hidden_size, + "encoder_hid_dim_type": "text_image_proj", + "cross_attention_dim": self.cross_attention_dim, + "attention_head_dim": 4, + "resnet_time_scale_shift": "scale_shift", + "class_embed_type": None, + } + + model = UNet2DConditionModel(**model_kwargs) + return model + + @property + def dummy_movq_kwargs(self): + return { + "block_out_channels": [32, 64], + "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"], + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 1, + "norm_num_groups": 8, + "norm_type": "spatial", + "num_vq_embeddings": 12, + "out_channels": 3, + "up_block_types": [ + "AttnUpDecoderBlock2D", + "UpDecoderBlock2D", + ], + "vq_embed_dim": 4, + } + + @property + def dummy_movq(self): + torch.manual_seed(0) + model = VQModel(**self.dummy_movq_kwargs) + return model + + def get_dummy_components(self): + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + unet = self.dummy_unet + movq = self.dummy_movq + + ddim_config = { + "num_train_timesteps": 1000, + "beta_schedule": "linear", + "beta_start": 0.00085, + "beta_end": 0.012, + "clip_sample": False, + "set_alpha_to_one": False, + "steps_offset": 0, + "prediction_type": "epsilon", + "thresholding": False, + } + + scheduler = DDIMScheduler(**ddim_config) + + components = { + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "unet": unet, + "scheduler": scheduler, + "movq": movq, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + image_embeds = floats_tensor((1, self.cross_attention_dim), rng=random.Random(seed)).to(device) + negative_image_embeds = floats_tensor((1, self.cross_attention_dim), rng=random.Random(seed + 1)).to(device) + # create init_image + image = floats_tensor((1, 3, 64, 64), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + init_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((256, 256)) + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "horse", + "image": init_image, + "image_embeds": image_embeds, + "negative_image_embeds": negative_image_embeds, + "generator": generator, + "height": 64, + "width": 64, + "num_inference_steps": 10, + "guidance_scale": 7.0, + "strength": 0.2, + "output_type": "np", + } + return inputs + + +class KandinskyImg2ImgPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyImg2ImgPipeline + params = ["prompt", "image_embeds", "negative_image_embeds", "image"] + batch_params = [ + "prompt", + "negative_prompt", + "image_embeds", + "negative_image_embeds", + "image", + ] + required_optional_params = [ + "generator", + "height", + "width", + "strength", + "guidance_scale", + "negative_prompt", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + def get_dummy_components(self): + dummies = Dummies() + return dummies.get_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + dummies = Dummies() + return dummies.get_dummy_inputs(device=device, seed=seed) + + def test_kandinsky_img2img(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.5816, 0.5872, 0.4634, 0.5982, 0.4767, 0.4710, 0.4669, 0.4717, 0.4966]) + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + @require_torch_gpu + def test_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent(expected_max_difference=5e-4) + + +@slow +@require_torch_gpu +class KandinskyImg2ImgPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_kandinsky_img2img(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/kandinsky/kandinsky_img2img_frog.npy" + ) + + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/cat.png" + ) + prompt = "A red cartoon frog, 4k" + + pipe_prior = KandinskyPriorPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16 + ) + pipe_prior.to(torch_device) + + pipeline = KandinskyImg2ImgPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-1", torch_dtype=torch.float16 + ) + pipeline = pipeline.to(torch_device) + + pipeline.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + image_emb, zero_image_emb = pipe_prior( + prompt, + generator=generator, + num_inference_steps=5, + negative_prompt="", + ).to_tuple() + + output = pipeline( + prompt, + image=init_image, + image_embeds=image_emb, + negative_image_embeds=zero_image_emb, + generator=generator, + num_inference_steps=100, + height=768, + width=768, + strength=0.2, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (768, 768, 3) + + assert_mean_pixel_difference(image, expected_image) + + +@nightly +@require_torch_gpu +class KandinskyImg2ImgPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_kandinsky_img2img_ddpm(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/kandinsky/kandinsky_img2img_ddpm_frog.npy" + ) + + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/frog.png" + ) + prompt = "A red cartoon frog, 4k" + + pipe_prior = KandinskyPriorPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16 + ) + pipe_prior.to(torch_device) + + scheduler = DDPMScheduler.from_pretrained("kandinsky-community/kandinsky-2-1", subfolder="ddpm_scheduler") + pipeline = KandinskyImg2ImgPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-1", scheduler=scheduler, torch_dtype=torch.float16 + ) + pipeline = pipeline.to(torch_device) + + pipeline.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + image_emb, zero_image_emb = pipe_prior( + prompt, + generator=generator, + num_inference_steps=5, + negative_prompt="", + ).to_tuple() + + output = pipeline( + prompt, + image=init_image, + image_embeds=image_emb, + negative_image_embeds=zero_image_emb, + generator=generator, + num_inference_steps=100, + height=768, + width=768, + strength=0.2, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (768, 768, 3) + + assert_mean_pixel_difference(image, expected_image) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_inpaint.py b/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_inpaint.py new file mode 100755 index 0000000..15b8c21 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_inpaint.py @@ -0,0 +1,356 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import XLMRobertaTokenizerFast + +from diffusers import DDIMScheduler, KandinskyInpaintPipeline, KandinskyPriorPipeline, UNet2DConditionModel, VQModel +from diffusers.pipelines.kandinsky.text_encoder import MCLIPConfig, MultilingualCLIP +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + nightly, + require_torch_gpu, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +enable_full_determinism() + + +class Dummies: + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 32 + + @property + def dummy_tokenizer(self): + tokenizer = XLMRobertaTokenizerFast.from_pretrained("YiYiXu/tiny-random-mclip-base") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = MCLIPConfig( + numDims=self.cross_attention_dim, + transformerDimensions=self.text_embedder_hidden_size, + hidden_size=self.text_embedder_hidden_size, + intermediate_size=37, + num_attention_heads=4, + num_hidden_layers=5, + vocab_size=1005, + ) + + text_encoder = MultilingualCLIP(config) + text_encoder = text_encoder.eval() + + return text_encoder + + @property + def dummy_unet(self): + torch.manual_seed(0) + + model_kwargs = { + "in_channels": 9, + # Out channels is double in channels because predicts mean and variance + "out_channels": 8, + "addition_embed_type": "text_image", + "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"), + "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"), + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "layers_per_block": 1, + "encoder_hid_dim": self.text_embedder_hidden_size, + "encoder_hid_dim_type": "text_image_proj", + "cross_attention_dim": self.cross_attention_dim, + "attention_head_dim": 4, + "resnet_time_scale_shift": "scale_shift", + "class_embed_type": None, + } + + model = UNet2DConditionModel(**model_kwargs) + return model + + @property + def dummy_movq_kwargs(self): + return { + "block_out_channels": [32, 64], + "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"], + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 1, + "norm_num_groups": 8, + "norm_type": "spatial", + "num_vq_embeddings": 12, + "out_channels": 3, + "up_block_types": [ + "AttnUpDecoderBlock2D", + "UpDecoderBlock2D", + ], + "vq_embed_dim": 4, + } + + @property + def dummy_movq(self): + torch.manual_seed(0) + model = VQModel(**self.dummy_movq_kwargs) + return model + + def get_dummy_components(self): + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + unet = self.dummy_unet + movq = self.dummy_movq + + scheduler = DDIMScheduler( + num_train_timesteps=1000, + beta_schedule="linear", + beta_start=0.00085, + beta_end=0.012, + clip_sample=False, + set_alpha_to_one=False, + steps_offset=1, + prediction_type="epsilon", + thresholding=False, + ) + + components = { + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "unet": unet, + "scheduler": scheduler, + "movq": movq, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + image_embeds = floats_tensor((1, self.cross_attention_dim), rng=random.Random(seed)).to(device) + negative_image_embeds = floats_tensor((1, self.cross_attention_dim), rng=random.Random(seed + 1)).to(device) + # create init_image + image = floats_tensor((1, 3, 64, 64), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + init_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((256, 256)) + # create mask + mask = np.zeros((64, 64), dtype=np.float32) + mask[:32, :32] = 1 + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "horse", + "image": init_image, + "mask_image": mask, + "image_embeds": image_embeds, + "negative_image_embeds": negative_image_embeds, + "generator": generator, + "height": 64, + "width": 64, + "num_inference_steps": 2, + "guidance_scale": 4.0, + "output_type": "np", + } + return inputs + + +class KandinskyInpaintPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyInpaintPipeline + params = ["prompt", "image_embeds", "negative_image_embeds", "image", "mask_image"] + batch_params = [ + "prompt", + "negative_prompt", + "image_embeds", + "negative_image_embeds", + "image", + "mask_image", + ] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "negative_prompt", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + def get_dummy_components(self): + dummies = Dummies() + return dummies.get_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + dummies = Dummies() + return dummies.get_dummy_inputs(device=device, seed=seed) + + def test_kandinsky_inpaint(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.8222, 0.8896, 0.4373, 0.8088, 0.4905, 0.2609, 0.6816, 0.4291, 0.5129]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + @require_torch_gpu + def test_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=5e-1) + + +@nightly +@require_torch_gpu +class KandinskyInpaintPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_kandinsky_inpaint(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/kandinsky/kandinsky_inpaint_cat_with_hat_fp16.npy" + ) + + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/cat.png" + ) + mask = np.zeros((768, 768), dtype=np.float32) + mask[:250, 250:-250] = 1 + + prompt = "a hat" + + pipe_prior = KandinskyPriorPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-1-prior", torch_dtype=torch.float16 + ) + pipe_prior.to(torch_device) + + pipeline = KandinskyInpaintPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-1-inpaint", torch_dtype=torch.float16 + ) + pipeline = pipeline.to(torch_device) + pipeline.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + image_emb, zero_image_emb = pipe_prior( + prompt, + generator=generator, + num_inference_steps=5, + negative_prompt="", + ).to_tuple() + + output = pipeline( + prompt, + image=init_image, + mask_image=mask, + image_embeds=image_emb, + negative_image_embeds=zero_image_emb, + generator=generator, + num_inference_steps=100, + height=768, + width=768, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (768, 768, 3) + + assert_mean_pixel_difference(image, expected_image) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_prior.py b/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_prior.py new file mode 100755 index 0000000..8e5456b --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky/test_kandinsky_prior.py @@ -0,0 +1,237 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch +from torch import nn +from transformers import ( + CLIPImageProcessor, + CLIPTextConfig, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +from diffusers import KandinskyPriorPipeline, PriorTransformer, UnCLIPScheduler +from diffusers.utils.testing_utils import enable_full_determinism, skip_mps, torch_device + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class Dummies: + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 100 + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=self.text_embedder_hidden_size, + projection_dim=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModelWithProjection(config) + + @property + def dummy_prior(self): + torch.manual_seed(0) + + model_kwargs = { + "num_attention_heads": 2, + "attention_head_dim": 12, + "embedding_dim": self.text_embedder_hidden_size, + "num_layers": 1, + } + + model = PriorTransformer(**model_kwargs) + # clip_std and clip_mean is initialized to be 0 so PriorTransformer.post_process_latents will always return 0 - set clip_std to be 1 so it won't return 0 + model.clip_std = nn.Parameter(torch.ones(model.clip_std.shape)) + return model + + @property + def dummy_image_encoder(self): + torch.manual_seed(0) + config = CLIPVisionConfig( + hidden_size=self.text_embedder_hidden_size, + image_size=224, + projection_dim=self.text_embedder_hidden_size, + intermediate_size=37, + num_attention_heads=4, + num_channels=3, + num_hidden_layers=5, + patch_size=14, + ) + + model = CLIPVisionModelWithProjection(config) + return model + + @property + def dummy_image_processor(self): + image_processor = CLIPImageProcessor( + crop_size=224, + do_center_crop=True, + do_normalize=True, + do_resize=True, + image_mean=[0.48145466, 0.4578275, 0.40821073], + image_std=[0.26862954, 0.26130258, 0.27577711], + resample=3, + size=224, + ) + + return image_processor + + def get_dummy_components(self): + prior = self.dummy_prior + image_encoder = self.dummy_image_encoder + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + image_processor = self.dummy_image_processor + + scheduler = UnCLIPScheduler( + variance_type="fixed_small_log", + prediction_type="sample", + num_train_timesteps=1000, + clip_sample=True, + clip_sample_range=10.0, + ) + + components = { + "prior": prior, + "image_encoder": image_encoder, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "scheduler": scheduler, + "image_processor": image_processor, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "horse", + "generator": generator, + "guidance_scale": 4.0, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + +class KandinskyPriorPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyPriorPipeline + params = ["prompt"] + batch_params = ["prompt", "negative_prompt"] + required_optional_params = [ + "num_images_per_prompt", + "generator", + "num_inference_steps", + "latents", + "negative_prompt", + "guidance_scale", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + def get_dummy_components(self): + dummy = Dummies() + return dummy.get_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + dummy = Dummies() + return dummy.get_dummy_inputs(device=device, seed=seed) + + def test_kandinsky_prior(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.image_embeds + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -10:] + image_from_tuple_slice = image_from_tuple[0, -10:] + + assert image.shape == (1, 32) + + expected_slice = np.array( + [-0.0532, 1.7120, 0.3656, -1.0852, -0.8946, -1.1756, 0.4348, 0.2482, 0.5146, -0.1156] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + @skip_mps + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=1e-2) + + @skip_mps + def test_attention_slicing_forward_pass(self): + test_max_difference = torch_device == "cpu" + test_mean_pixel_difference = False + + self._test_attention_slicing_forward_pass( + test_max_difference=test_max_difference, + test_mean_pixel_difference=test_mean_pixel_difference, + ) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky2_2/__init__.py b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky.py b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky.py new file mode 100755 index 0000000..91b54d8 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky.py @@ -0,0 +1,272 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch + +from diffusers import DDIMScheduler, KandinskyV22Pipeline, KandinskyV22PriorPipeline, UNet2DConditionModel, VQModel +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_numpy, + require_torch_gpu, + slow, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +enable_full_determinism() + + +class Dummies: + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 32 + + @property + def dummy_unet(self): + torch.manual_seed(0) + + model_kwargs = { + "in_channels": 4, + # Out channels is double in channels because predicts mean and variance + "out_channels": 8, + "addition_embed_type": "image", + "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"), + "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"), + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "layers_per_block": 1, + "encoder_hid_dim": self.text_embedder_hidden_size, + "encoder_hid_dim_type": "image_proj", + "cross_attention_dim": self.cross_attention_dim, + "attention_head_dim": 4, + "resnet_time_scale_shift": "scale_shift", + "class_embed_type": None, + } + + model = UNet2DConditionModel(**model_kwargs) + return model + + @property + def dummy_movq_kwargs(self): + return { + "block_out_channels": [32, 64], + "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"], + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 1, + "norm_num_groups": 8, + "norm_type": "spatial", + "num_vq_embeddings": 12, + "out_channels": 3, + "up_block_types": [ + "AttnUpDecoderBlock2D", + "UpDecoderBlock2D", + ], + "vq_embed_dim": 4, + } + + @property + def dummy_movq(self): + torch.manual_seed(0) + model = VQModel(**self.dummy_movq_kwargs) + return model + + def get_dummy_components(self): + unet = self.dummy_unet + movq = self.dummy_movq + + scheduler = DDIMScheduler( + num_train_timesteps=1000, + beta_schedule="linear", + beta_start=0.00085, + beta_end=0.012, + clip_sample=False, + set_alpha_to_one=False, + steps_offset=1, + prediction_type="epsilon", + thresholding=False, + ) + + components = { + "unet": unet, + "scheduler": scheduler, + "movq": movq, + } + return components + + def get_dummy_inputs(self, device, seed=0): + image_embeds = floats_tensor((1, self.text_embedder_hidden_size), rng=random.Random(seed)).to(device) + negative_image_embeds = floats_tensor((1, self.text_embedder_hidden_size), rng=random.Random(seed + 1)).to( + device + ) + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "image_embeds": image_embeds, + "negative_image_embeds": negative_image_embeds, + "generator": generator, + "height": 64, + "width": 64, + "guidance_scale": 4.0, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + +class KandinskyV22PipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyV22Pipeline + params = [ + "image_embeds", + "negative_image_embeds", + ] + batch_params = ["image_embeds", "negative_image_embeds"] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + callback_cfg_params = ["image_embds"] + test_xformers_attention = False + + def get_dummy_inputs(self, device, seed=0): + dummies = Dummies() + return dummies.get_dummy_inputs(device=device, seed=seed) + + def get_dummy_components(self): + dummies = Dummies() + return dummies.get_dummy_components() + + def test_kandinsky(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.3420, 0.9505, 0.3919, 1.0000, 0.5188, 0.3109, 0.6139, 0.5624, 0.6811]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=1e-1) + + +@slow +@require_torch_gpu +class KandinskyV22PipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_kandinsky_text2img(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/kandinskyv22/kandinskyv22_text2img_cat_fp16.npy" + ) + + pipe_prior = KandinskyV22PriorPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ) + pipe_prior.to(torch_device) + + pipeline = KandinskyV22Pipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16 + ) + pipeline = pipeline.to(torch_device) + pipeline.set_progress_bar_config(disable=None) + + prompt = "red cat, 4k photo" + + generator = torch.Generator(device="cuda").manual_seed(0) + image_emb, zero_image_emb = pipe_prior( + prompt, + generator=generator, + num_inference_steps=5, + negative_prompt="", + ).to_tuple() + + generator = torch.Generator(device="cuda").manual_seed(0) + output = pipeline( + image_embeds=image_emb, + negative_image_embeds=zero_image_emb, + generator=generator, + num_inference_steps=100, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (512, 512, 3) + + assert_mean_pixel_difference(image, expected_image) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_combined.py b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_combined.py new file mode 100755 index 0000000..40bb3b0 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_combined.py @@ -0,0 +1,406 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np + +from diffusers import ( + KandinskyV22CombinedPipeline, + KandinskyV22Img2ImgCombinedPipeline, + KandinskyV22InpaintCombinedPipeline, +) +from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, torch_device + +from ..test_pipelines_common import PipelineTesterMixin +from .test_kandinsky import Dummies +from .test_kandinsky_img2img import Dummies as Img2ImgDummies +from .test_kandinsky_inpaint import Dummies as InpaintDummies +from .test_kandinsky_prior import Dummies as PriorDummies + + +enable_full_determinism() + + +class KandinskyV22PipelineCombinedFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyV22CombinedPipeline + params = [ + "prompt", + ] + batch_params = ["prompt", "negative_prompt"] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "negative_prompt", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = True + callback_cfg_params = ["image_embds"] + + def get_dummy_components(self): + dummy = Dummies() + prior_dummy = PriorDummies() + components = dummy.get_dummy_components() + + components.update({f"prior_{k}": v for k, v in prior_dummy.get_dummy_components().items()}) + return components + + def get_dummy_inputs(self, device, seed=0): + prior_dummy = PriorDummies() + inputs = prior_dummy.get_dummy_inputs(device=device, seed=seed) + inputs.update( + { + "height": 64, + "width": 64, + } + ) + return inputs + + def test_kandinsky(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.3013, 0.0471, 0.5176, 0.1817, 0.2566, 0.7076, 0.6712, 0.4421, 0.7503]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + @require_torch_gpu + def test_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=1e-2) + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=5e-1) + + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent(expected_max_difference=5e-4) + + def test_model_cpu_offload_forward_pass(self): + super().test_model_cpu_offload_forward_pass(expected_max_diff=5e-4) + + def test_save_load_local(self): + super().test_save_load_local(expected_max_difference=5e-3) + + def test_save_load_optional_components(self): + super().test_save_load_optional_components(expected_max_difference=5e-3) + + def test_callback_inputs(self): + pass + + def test_callback_cfg(self): + pass + + +class KandinskyV22PipelineImg2ImgCombinedFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyV22Img2ImgCombinedPipeline + params = ["prompt", "image"] + batch_params = ["prompt", "negative_prompt", "image"] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "negative_prompt", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = False + callback_cfg_params = ["image_embds"] + + def get_dummy_components(self): + dummy = Img2ImgDummies() + prior_dummy = PriorDummies() + components = dummy.get_dummy_components() + + components.update({f"prior_{k}": v for k, v in prior_dummy.get_dummy_components().items()}) + return components + + def get_dummy_inputs(self, device, seed=0): + prior_dummy = PriorDummies() + dummy = Img2ImgDummies() + inputs = prior_dummy.get_dummy_inputs(device=device, seed=seed) + inputs.update(dummy.get_dummy_inputs(device=device, seed=seed)) + inputs.pop("image_embeds") + inputs.pop("negative_image_embeds") + return inputs + + def test_kandinsky(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.4353, 0.4710, 0.5128, 0.4806, 0.5054, 0.5348, 0.5224, 0.4603, 0.5025]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + @require_torch_gpu + def test_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=1e-2) + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=2e-1) + + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent(expected_max_difference=5e-4) + + def test_model_cpu_offload_forward_pass(self): + super().test_model_cpu_offload_forward_pass(expected_max_diff=5e-4) + + def test_save_load_optional_components(self): + super().test_save_load_optional_components(expected_max_difference=5e-4) + + def save_load_local(self): + super().test_save_load_local(expected_max_difference=5e-3) + + def test_callback_inputs(self): + pass + + def test_callback_cfg(self): + pass + + +class KandinskyV22PipelineInpaintCombinedFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyV22InpaintCombinedPipeline + params = ["prompt", "image", "mask_image"] + batch_params = ["prompt", "negative_prompt", "image", "mask_image"] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "negative_prompt", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + def get_dummy_components(self): + dummy = InpaintDummies() + prior_dummy = PriorDummies() + components = dummy.get_dummy_components() + + components.update({f"prior_{k}": v for k, v in prior_dummy.get_dummy_components().items()}) + return components + + def get_dummy_inputs(self, device, seed=0): + prior_dummy = PriorDummies() + dummy = InpaintDummies() + inputs = prior_dummy.get_dummy_inputs(device=device, seed=seed) + inputs.update(dummy.get_dummy_inputs(device=device, seed=seed)) + inputs.pop("image_embeds") + inputs.pop("negative_image_embeds") + return inputs + + def test_kandinsky(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.5039, 0.4926, 0.4898, 0.4978, 0.4838, 0.4942, 0.4738, 0.4702, 0.4816]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + @require_torch_gpu + def test_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=1e-2) + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=5e-1) + + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent(expected_max_difference=5e-4) + + def test_model_cpu_offload_forward_pass(self): + super().test_model_cpu_offload_forward_pass(expected_max_diff=5e-4) + + def test_save_load_local(self): + super().test_save_load_local(expected_max_difference=5e-3) + + def test_save_load_optional_components(self): + super().test_save_load_optional_components(expected_max_difference=5e-4) + + def test_sequential_cpu_offload_forward_pass(self): + super().test_sequential_cpu_offload_forward_pass(expected_max_diff=5e-4) + + def test_callback_inputs(self): + pass + + def test_callback_cfg(self): + pass diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_controlnet.py b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_controlnet.py new file mode 100755 index 0000000..7deee83 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_controlnet.py @@ -0,0 +1,285 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch + +from diffusers import ( + DDIMScheduler, + KandinskyV22ControlnetPipeline, + KandinskyV22PriorPipeline, + UNet2DConditionModel, + VQModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + nightly, + require_torch_gpu, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +enable_full_determinism() + + +class KandinskyV22ControlnetPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyV22ControlnetPipeline + params = ["image_embeds", "negative_image_embeds", "hint"] + batch_params = ["image_embeds", "negative_image_embeds", "hint"] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 100 + + @property + def dummy_unet(self): + torch.manual_seed(0) + + model_kwargs = { + "in_channels": 8, + # Out channels is double in channels because predicts mean and variance + "out_channels": 8, + "addition_embed_type": "image_hint", + "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"), + "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"), + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "layers_per_block": 1, + "encoder_hid_dim": self.text_embedder_hidden_size, + "encoder_hid_dim_type": "image_proj", + "cross_attention_dim": self.cross_attention_dim, + "attention_head_dim": 4, + "resnet_time_scale_shift": "scale_shift", + "class_embed_type": None, + } + + model = UNet2DConditionModel(**model_kwargs) + return model + + @property + def dummy_movq_kwargs(self): + return { + "block_out_channels": [32, 32, 64, 64], + "down_block_types": [ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "AttnDownEncoderBlock2D", + ], + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 1, + "norm_num_groups": 8, + "norm_type": "spatial", + "num_vq_embeddings": 12, + "out_channels": 3, + "up_block_types": ["AttnUpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"], + "vq_embed_dim": 4, + } + + @property + def dummy_movq(self): + torch.manual_seed(0) + model = VQModel(**self.dummy_movq_kwargs) + return model + + def get_dummy_components(self): + unet = self.dummy_unet + movq = self.dummy_movq + + scheduler = DDIMScheduler( + num_train_timesteps=1000, + beta_schedule="linear", + beta_start=0.00085, + beta_end=0.012, + clip_sample=False, + set_alpha_to_one=False, + steps_offset=1, + prediction_type="epsilon", + thresholding=False, + ) + + components = { + "unet": unet, + "scheduler": scheduler, + "movq": movq, + } + return components + + def get_dummy_inputs(self, device, seed=0): + image_embeds = floats_tensor((1, self.text_embedder_hidden_size), rng=random.Random(seed)).to(device) + negative_image_embeds = floats_tensor((1, self.text_embedder_hidden_size), rng=random.Random(seed + 1)).to( + device + ) + + # create hint + hint = floats_tensor((1, 3, 64, 64), rng=random.Random(seed)).to(device) + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "image_embeds": image_embeds, + "negative_image_embeds": negative_image_embeds, + "hint": hint, + "generator": generator, + "height": 64, + "width": 64, + "guidance_scale": 4.0, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + def test_kandinsky_controlnet(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array( + [0.6959826, 0.868279, 0.7558092, 0.68769467, 0.85805804, 0.65977496, 0.44885302, 0.5959111, 0.4251595] + ) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=1e-1) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=5e-4) + + +@nightly +@require_torch_gpu +class KandinskyV22ControlnetPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_kandinsky_controlnet(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/kandinskyv22/kandinskyv22_controlnet_robotcat_fp16.npy" + ) + + hint = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/kandinskyv22/hint_image_cat.png" + ) + hint = torch.from_numpy(np.array(hint)).float() / 255.0 + hint = hint.permute(2, 0, 1).unsqueeze(0) + + pipe_prior = KandinskyV22PriorPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ) + pipe_prior.to(torch_device) + + pipeline = KandinskyV22ControlnetPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-controlnet-depth", torch_dtype=torch.float16 + ) + pipeline = pipeline.to(torch_device) + pipeline.set_progress_bar_config(disable=None) + + prompt = "A robot, 4k photo" + + generator = torch.Generator(device="cuda").manual_seed(0) + image_emb, zero_image_emb = pipe_prior( + prompt, + generator=generator, + num_inference_steps=5, + negative_prompt="", + ).to_tuple() + + generator = torch.Generator(device="cuda").manual_seed(0) + output = pipeline( + image_embeds=image_emb, + negative_image_embeds=zero_image_emb, + hint=hint, + generator=generator, + num_inference_steps=100, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (512, 512, 3) + + assert_mean_pixel_difference(image, expected_image) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_controlnet_img2img.py b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_controlnet_img2img.py new file mode 100755 index 0000000..c7d6af9 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_controlnet_img2img.py @@ -0,0 +1,303 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from PIL import Image + +from diffusers import ( + DDIMScheduler, + KandinskyV22ControlnetImg2ImgPipeline, + KandinskyV22PriorEmb2EmbPipeline, + UNet2DConditionModel, + VQModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + nightly, + require_torch_gpu, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +enable_full_determinism() + + +class KandinskyV22ControlnetImg2ImgPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyV22ControlnetImg2ImgPipeline + params = ["image_embeds", "negative_image_embeds", "image", "hint"] + batch_params = ["image_embeds", "negative_image_embeds", "image", "hint"] + required_optional_params = [ + "generator", + "height", + "width", + "strength", + "guidance_scale", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 100 + + @property + def dummy_unet(self): + torch.manual_seed(0) + + model_kwargs = { + "in_channels": 8, + # Out channels is double in channels because predicts mean and variance + "out_channels": 8, + "addition_embed_type": "image_hint", + "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"), + "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"), + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "layers_per_block": 1, + "encoder_hid_dim": self.text_embedder_hidden_size, + "encoder_hid_dim_type": "image_proj", + "cross_attention_dim": self.cross_attention_dim, + "attention_head_dim": 4, + "resnet_time_scale_shift": "scale_shift", + "class_embed_type": None, + } + + model = UNet2DConditionModel(**model_kwargs) + return model + + @property + def dummy_movq_kwargs(self): + return { + "block_out_channels": [32, 32, 64, 64], + "down_block_types": [ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "AttnDownEncoderBlock2D", + ], + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 1, + "norm_num_groups": 8, + "norm_type": "spatial", + "num_vq_embeddings": 12, + "out_channels": 3, + "up_block_types": ["AttnUpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"], + "vq_embed_dim": 4, + } + + @property + def dummy_movq(self): + torch.manual_seed(0) + model = VQModel(**self.dummy_movq_kwargs) + return model + + def get_dummy_components(self): + unet = self.dummy_unet + movq = self.dummy_movq + + ddim_config = { + "num_train_timesteps": 1000, + "beta_schedule": "linear", + "beta_start": 0.00085, + "beta_end": 0.012, + "clip_sample": False, + "set_alpha_to_one": False, + "steps_offset": 0, + "prediction_type": "epsilon", + "thresholding": False, + } + + scheduler = DDIMScheduler(**ddim_config) + + components = { + "unet": unet, + "scheduler": scheduler, + "movq": movq, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + image_embeds = floats_tensor((1, self.text_embedder_hidden_size), rng=random.Random(seed)).to(device) + negative_image_embeds = floats_tensor((1, self.text_embedder_hidden_size), rng=random.Random(seed + 1)).to( + device + ) + # create init_image + image = floats_tensor((1, 3, 64, 64), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + init_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((256, 256)) + # create hint + hint = floats_tensor((1, 3, 64, 64), rng=random.Random(seed)).to(device) + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "image": init_image, + "image_embeds": image_embeds, + "negative_image_embeds": negative_image_embeds, + "hint": hint, + "generator": generator, + "height": 64, + "width": 64, + "num_inference_steps": 10, + "guidance_scale": 7.0, + "strength": 0.2, + "output_type": "np", + } + return inputs + + def test_kandinsky_controlnet_img2img(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array( + [0.54985034, 0.55509365, 0.52561504, 0.5570494, 0.5593818, 0.5263979, 0.50285643, 0.5069846, 0.51196736] + ) + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=1.75e-3) + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=2e-1) + + +@nightly +@require_torch_gpu +class KandinskyV22ControlnetImg2ImgPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_kandinsky_controlnet_img2img(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/kandinskyv22/kandinskyv22_controlnet_img2img_robotcat_fp16.npy" + ) + + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/cat.png" + ) + init_image = init_image.resize((512, 512)) + + hint = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/kandinskyv22/hint_image_cat.png" + ) + hint = torch.from_numpy(np.array(hint)).float() / 255.0 + hint = hint.permute(2, 0, 1).unsqueeze(0) + + prompt = "A robot, 4k photo" + + pipe_prior = KandinskyV22PriorEmb2EmbPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ) + pipe_prior.to(torch_device) + + pipeline = KandinskyV22ControlnetImg2ImgPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-controlnet-depth", torch_dtype=torch.float16 + ) + pipeline = pipeline.to(torch_device) + + pipeline.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + + image_emb, zero_image_emb = pipe_prior( + prompt, + image=init_image, + strength=0.85, + generator=generator, + negative_prompt="", + ).to_tuple() + + output = pipeline( + image=init_image, + image_embeds=image_emb, + negative_image_embeds=zero_image_emb, + hint=hint, + generator=generator, + num_inference_steps=100, + height=512, + width=512, + strength=0.5, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (512, 512, 3) + + assert_mean_pixel_difference(image, expected_image) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_img2img.py b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_img2img.py new file mode 100755 index 0000000..07a362b --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_img2img.py @@ -0,0 +1,296 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from PIL import Image + +from diffusers import ( + DDIMScheduler, + KandinskyV22Img2ImgPipeline, + KandinskyV22PriorPipeline, + UNet2DConditionModel, + VQModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + require_torch_gpu, + slow, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +enable_full_determinism() + + +class Dummies: + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 32 + + @property + def dummy_unet(self): + torch.manual_seed(0) + + model_kwargs = { + "in_channels": 4, + # Out channels is double in channels because predicts mean and variance + "out_channels": 8, + "addition_embed_type": "image", + "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"), + "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"), + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "layers_per_block": 1, + "encoder_hid_dim": self.text_embedder_hidden_size, + "encoder_hid_dim_type": "image_proj", + "cross_attention_dim": self.cross_attention_dim, + "attention_head_dim": 4, + "resnet_time_scale_shift": "scale_shift", + "class_embed_type": None, + } + + model = UNet2DConditionModel(**model_kwargs) + return model + + @property + def dummy_movq_kwargs(self): + return { + "block_out_channels": [32, 64], + "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"], + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 1, + "norm_num_groups": 8, + "norm_type": "spatial", + "num_vq_embeddings": 12, + "out_channels": 3, + "up_block_types": [ + "AttnUpDecoderBlock2D", + "UpDecoderBlock2D", + ], + "vq_embed_dim": 4, + } + + @property + def dummy_movq(self): + torch.manual_seed(0) + model = VQModel(**self.dummy_movq_kwargs) + return model + + def get_dummy_components(self): + unet = self.dummy_unet + movq = self.dummy_movq + + ddim_config = { + "num_train_timesteps": 1000, + "beta_schedule": "linear", + "beta_start": 0.00085, + "beta_end": 0.012, + "clip_sample": False, + "set_alpha_to_one": False, + "steps_offset": 0, + "prediction_type": "epsilon", + "thresholding": False, + } + + scheduler = DDIMScheduler(**ddim_config) + + components = { + "unet": unet, + "scheduler": scheduler, + "movq": movq, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + image_embeds = floats_tensor((1, self.text_embedder_hidden_size), rng=random.Random(seed)).to(device) + negative_image_embeds = floats_tensor((1, self.text_embedder_hidden_size), rng=random.Random(seed + 1)).to( + device + ) + # create init_image + image = floats_tensor((1, 3, 64, 64), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + init_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((256, 256)) + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "image": init_image, + "image_embeds": image_embeds, + "negative_image_embeds": negative_image_embeds, + "generator": generator, + "height": 64, + "width": 64, + "num_inference_steps": 10, + "guidance_scale": 7.0, + "strength": 0.2, + "output_type": "np", + } + return inputs + + +class KandinskyV22Img2ImgPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyV22Img2ImgPipeline + params = ["image_embeds", "negative_image_embeds", "image"] + batch_params = [ + "image_embeds", + "negative_image_embeds", + "image", + ] + required_optional_params = [ + "generator", + "height", + "width", + "strength", + "guidance_scale", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = False + callback_cfg_params = ["image_embeds"] + + def get_dummy_components(self): + dummies = Dummies() + return dummies.get_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + dummies = Dummies() + return dummies.get_dummy_inputs(device=device, seed=seed) + + def test_kandinsky_img2img(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.5712, 0.5443, 0.4725, 0.6195, 0.5184, 0.4651, 0.4473, 0.4590, 0.5016]) + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=2e-1) + + +@slow +@require_torch_gpu +class KandinskyV22Img2ImgPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_kandinsky_img2img(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/kandinskyv22/kandinskyv22_img2img_frog.npy" + ) + + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/cat.png" + ) + prompt = "A red cartoon frog, 4k" + + pipe_prior = KandinskyV22PriorPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ) + pipe_prior.to(torch_device) + + pipeline = KandinskyV22Img2ImgPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-decoder", torch_dtype=torch.float16 + ) + pipeline = pipeline.to(torch_device) + + pipeline.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + image_emb, zero_image_emb = pipe_prior( + prompt, + generator=generator, + num_inference_steps=5, + negative_prompt="", + ).to_tuple() + + output = pipeline( + image=init_image, + image_embeds=image_emb, + negative_image_embeds=zero_image_emb, + generator=generator, + num_inference_steps=100, + height=768, + width=768, + strength=0.2, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (768, 768, 3) + + assert_mean_pixel_difference(image, expected_image) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_inpaint.py b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_inpaint.py new file mode 100755 index 0000000..6ec812c --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_inpaint.py @@ -0,0 +1,351 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from PIL import Image + +from diffusers import ( + DDIMScheduler, + KandinskyV22InpaintPipeline, + KandinskyV22PriorPipeline, + UNet2DConditionModel, + VQModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + is_flaky, + load_image, + load_numpy, + require_torch_gpu, + slow, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +enable_full_determinism() + + +class Dummies: + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 32 + + @property + def dummy_unet(self): + torch.manual_seed(0) + + model_kwargs = { + "in_channels": 9, + # Out channels is double in channels because predicts mean and variance + "out_channels": 8, + "addition_embed_type": "image", + "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"), + "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"), + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "layers_per_block": 1, + "encoder_hid_dim": self.text_embedder_hidden_size, + "encoder_hid_dim_type": "image_proj", + "cross_attention_dim": self.cross_attention_dim, + "attention_head_dim": 4, + "resnet_time_scale_shift": "scale_shift", + "class_embed_type": None, + } + + model = UNet2DConditionModel(**model_kwargs) + return model + + @property + def dummy_movq_kwargs(self): + return { + "block_out_channels": [32, 64], + "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"], + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 1, + "norm_num_groups": 8, + "norm_type": "spatial", + "num_vq_embeddings": 12, + "out_channels": 3, + "up_block_types": [ + "AttnUpDecoderBlock2D", + "UpDecoderBlock2D", + ], + "vq_embed_dim": 4, + } + + @property + def dummy_movq(self): + torch.manual_seed(0) + model = VQModel(**self.dummy_movq_kwargs) + return model + + def get_dummy_components(self): + unet = self.dummy_unet + movq = self.dummy_movq + + scheduler = DDIMScheduler( + num_train_timesteps=1000, + beta_schedule="linear", + beta_start=0.00085, + beta_end=0.012, + clip_sample=False, + set_alpha_to_one=False, + steps_offset=1, + prediction_type="epsilon", + thresholding=False, + ) + + components = { + "unet": unet, + "scheduler": scheduler, + "movq": movq, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + image_embeds = floats_tensor((1, self.text_embedder_hidden_size), rng=random.Random(seed)).to(device) + negative_image_embeds = floats_tensor((1, self.text_embedder_hidden_size), rng=random.Random(seed + 1)).to( + device + ) + # create init_image + image = floats_tensor((1, 3, 64, 64), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + init_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((256, 256)) + # create mask + mask = np.zeros((64, 64), dtype=np.float32) + mask[:32, :32] = 1 + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "image": init_image, + "mask_image": mask, + "image_embeds": image_embeds, + "negative_image_embeds": negative_image_embeds, + "generator": generator, + "height": 64, + "width": 64, + "num_inference_steps": 2, + "guidance_scale": 4.0, + "output_type": "np", + } + return inputs + + +class KandinskyV22InpaintPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyV22InpaintPipeline + params = ["image_embeds", "negative_image_embeds", "image", "mask_image"] + batch_params = [ + "image_embeds", + "negative_image_embeds", + "image", + "mask_image", + ] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "guidance_scale", + "num_inference_steps", + "return_dict", + "guidance_scale", + "num_images_per_prompt", + "output_type", + "return_dict", + ] + test_xformers_attention = False + callback_cfg_params = ["image_embeds", "masked_image", "mask_image"] + + def get_dummy_components(self): + dummies = Dummies() + return dummies.get_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + dummies = Dummies() + return dummies.get_dummy_inputs(device=device, seed=seed) + + def test_kandinsky_inpaint(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array( + [0.50775903, 0.49527195, 0.48824543, 0.50192237, 0.48644906, 0.49373814, 0.4780598, 0.47234827, 0.48327848] + ) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=5e-1) + + @is_flaky() + def test_model_cpu_offload_forward_pass(self): + super().test_inference_batch_single_identical(expected_max_diff=8e-4) + + def test_save_load_optional_components(self): + super().test_save_load_optional_components(expected_max_difference=5e-4) + + def test_sequential_cpu_offload_forward_pass(self): + super().test_sequential_cpu_offload_forward_pass(expected_max_diff=5e-4) + + # override default test because we need to zero out mask too in order to make sure final latent is all zero + def test_callback_inputs(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + self.assertTrue( + hasattr(pipe, "_callback_tensor_inputs"), + f" {self.pipeline_class} should have `_callback_tensor_inputs` that defines a list of tensor variables its callback function can use as inputs", + ) + + def callback_inputs_test(pipe, i, t, callback_kwargs): + missing_callback_inputs = set() + for v in pipe._callback_tensor_inputs: + if v not in callback_kwargs: + missing_callback_inputs.add(v) + self.assertTrue( + len(missing_callback_inputs) == 0, f"Missing callback tensor inputs: {missing_callback_inputs}" + ) + last_i = pipe.num_timesteps - 1 + if i == last_i: + callback_kwargs["latents"] = torch.zeros_like(callback_kwargs["latents"]) + callback_kwargs["mask_image"] = torch.zeros_like(callback_kwargs["mask_image"]) + return callback_kwargs + + inputs = self.get_dummy_inputs(torch_device) + inputs["callback_on_step_end"] = callback_inputs_test + inputs["callback_on_step_end_tensor_inputs"] = pipe._callback_tensor_inputs + inputs["output_type"] = "latent" + + output = pipe(**inputs)[0] + assert output.abs().sum() == 0 + + +@slow +@require_torch_gpu +class KandinskyV22InpaintPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_kandinsky_inpaint(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/kandinskyv22/kandinskyv22_inpaint_cat_with_hat_fp16.npy" + ) + + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/kandinsky/cat.png" + ) + mask = np.zeros((768, 768), dtype=np.float32) + mask[:250, 250:-250] = 1 + + prompt = "a hat" + + pipe_prior = KandinskyV22PriorPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-prior", torch_dtype=torch.float16 + ) + pipe_prior.to(torch_device) + + pipeline = KandinskyV22InpaintPipeline.from_pretrained( + "kandinsky-community/kandinsky-2-2-decoder-inpaint", torch_dtype=torch.float16 + ) + pipeline = pipeline.to(torch_device) + pipeline.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + image_emb, zero_image_emb = pipe_prior( + prompt, + generator=generator, + num_inference_steps=5, + negative_prompt="", + ).to_tuple() + + output = pipeline( + image=init_image, + mask_image=mask, + image_embeds=image_emb, + negative_image_embeds=zero_image_emb, + generator=generator, + num_inference_steps=100, + height=768, + width=768, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (768, 768, 3) + + assert_mean_pixel_difference(image, expected_image) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_prior.py b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_prior.py new file mode 100755 index 0000000..c19c574 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_prior.py @@ -0,0 +1,278 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import inspect +import unittest + +import numpy as np +import torch +from torch import nn +from transformers import ( + CLIPImageProcessor, + CLIPTextConfig, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +from diffusers import KandinskyV22PriorPipeline, PriorTransformer, UnCLIPScheduler +from diffusers.utils.testing_utils import enable_full_determinism, skip_mps, torch_device + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class Dummies: + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 100 + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=self.text_embedder_hidden_size, + projection_dim=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModelWithProjection(config) + + @property + def dummy_prior(self): + torch.manual_seed(0) + + model_kwargs = { + "num_attention_heads": 2, + "attention_head_dim": 12, + "embedding_dim": self.text_embedder_hidden_size, + "num_layers": 1, + } + + model = PriorTransformer(**model_kwargs) + # clip_std and clip_mean is initialized to be 0 so PriorTransformer.post_process_latents will always return 0 - set clip_std to be 1 so it won't return 0 + model.clip_std = nn.Parameter(torch.ones(model.clip_std.shape)) + return model + + @property + def dummy_image_encoder(self): + torch.manual_seed(0) + config = CLIPVisionConfig( + hidden_size=self.text_embedder_hidden_size, + image_size=224, + projection_dim=self.text_embedder_hidden_size, + intermediate_size=37, + num_attention_heads=4, + num_channels=3, + num_hidden_layers=5, + patch_size=14, + ) + + model = CLIPVisionModelWithProjection(config) + return model + + @property + def dummy_image_processor(self): + image_processor = CLIPImageProcessor( + crop_size=224, + do_center_crop=True, + do_normalize=True, + do_resize=True, + image_mean=[0.48145466, 0.4578275, 0.40821073], + image_std=[0.26862954, 0.26130258, 0.27577711], + resample=3, + size=224, + ) + + return image_processor + + def get_dummy_components(self): + prior = self.dummy_prior + image_encoder = self.dummy_image_encoder + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + image_processor = self.dummy_image_processor + + scheduler = UnCLIPScheduler( + variance_type="fixed_small_log", + prediction_type="sample", + num_train_timesteps=1000, + clip_sample=True, + clip_sample_range=10.0, + ) + + components = { + "prior": prior, + "image_encoder": image_encoder, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "scheduler": scheduler, + "image_processor": image_processor, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "horse", + "generator": generator, + "guidance_scale": 4.0, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + +class KandinskyV22PriorPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyV22PriorPipeline + params = ["prompt"] + batch_params = ["prompt", "negative_prompt"] + required_optional_params = [ + "num_images_per_prompt", + "generator", + "num_inference_steps", + "latents", + "negative_prompt", + "guidance_scale", + "output_type", + "return_dict", + ] + callback_cfg_params = ["prompt_embeds", "text_encoder_hidden_states", "text_mask"] + test_xformers_attention = False + + def get_dummy_components(self): + dummies = Dummies() + return dummies.get_dummy_components() + + def get_dummy_inputs(self, device, seed=0): + dummies = Dummies() + return dummies.get_dummy_inputs(device=device, seed=seed) + + def test_kandinsky_prior(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.image_embeds + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -10:] + image_from_tuple_slice = image_from_tuple[0, -10:] + + assert image.shape == (1, 32) + + expected_slice = np.array( + [-0.0532, 1.7120, 0.3656, -1.0852, -0.8946, -1.1756, 0.4348, 0.2482, 0.5146, -0.1156] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + @skip_mps + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=1e-3) + + @skip_mps + def test_attention_slicing_forward_pass(self): + test_max_difference = torch_device == "cpu" + test_mean_pixel_difference = False + + self._test_attention_slicing_forward_pass( + test_max_difference=test_max_difference, + test_mean_pixel_difference=test_mean_pixel_difference, + ) + + # override default test because no output_type "latent", use "pt" instead + def test_callback_inputs(self): + sig = inspect.signature(self.pipeline_class.__call__) + + if not ("callback_on_step_end_tensor_inputs" in sig.parameters and "callback_on_step_end" in sig.parameters): + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + self.assertTrue( + hasattr(pipe, "_callback_tensor_inputs"), + f" {self.pipeline_class} should have `_callback_tensor_inputs` that defines a list of tensor variables its callback function can use as inputs", + ) + + def callback_inputs_test(pipe, i, t, callback_kwargs): + missing_callback_inputs = set() + for v in pipe._callback_tensor_inputs: + if v not in callback_kwargs: + missing_callback_inputs.add(v) + self.assertTrue( + len(missing_callback_inputs) == 0, f"Missing callback tensor inputs: {missing_callback_inputs}" + ) + last_i = pipe.num_timesteps - 1 + if i == last_i: + callback_kwargs["latents"] = torch.zeros_like(callback_kwargs["latents"]) + return callback_kwargs + + inputs = self.get_dummy_inputs(torch_device) + inputs["callback_on_step_end"] = callback_inputs_test + inputs["callback_on_step_end_tensor_inputs"] = pipe._callback_tensor_inputs + inputs["num_inference_steps"] = 2 + inputs["output_type"] = "pt" + + output = pipe(**inputs)[0] + assert output.abs().sum() == 0 diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_prior_emb2emb.py b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_prior_emb2emb.py new file mode 100755 index 0000000..7d66fb9 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky2_2/test_kandinsky_prior_emb2emb.py @@ -0,0 +1,247 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from torch import nn +from transformers import ( + CLIPImageProcessor, + CLIPTextConfig, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +from diffusers import KandinskyV22PriorEmb2EmbPipeline, PriorTransformer, UnCLIPScheduler +from diffusers.utils.testing_utils import enable_full_determinism, floats_tensor, skip_mps, torch_device + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class KandinskyV22PriorEmb2EmbPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = KandinskyV22PriorEmb2EmbPipeline + params = ["prompt", "image"] + batch_params = ["prompt", "image"] + required_optional_params = [ + "num_images_per_prompt", + "strength", + "generator", + "num_inference_steps", + "negative_prompt", + "guidance_scale", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 100 + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=self.text_embedder_hidden_size, + projection_dim=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModelWithProjection(config) + + @property + def dummy_prior(self): + torch.manual_seed(0) + + model_kwargs = { + "num_attention_heads": 2, + "attention_head_dim": 12, + "embedding_dim": self.text_embedder_hidden_size, + "num_layers": 1, + } + + model = PriorTransformer(**model_kwargs) + # clip_std and clip_mean is initialized to be 0 so PriorTransformer.post_process_latents will always return 0 - set clip_std to be 1 so it won't return 0 + model.clip_std = nn.Parameter(torch.ones(model.clip_std.shape)) + return model + + @property + def dummy_image_encoder(self): + torch.manual_seed(0) + config = CLIPVisionConfig( + hidden_size=self.text_embedder_hidden_size, + image_size=224, + projection_dim=self.text_embedder_hidden_size, + intermediate_size=37, + num_attention_heads=4, + num_channels=3, + num_hidden_layers=5, + patch_size=14, + ) + + model = CLIPVisionModelWithProjection(config) + return model + + @property + def dummy_image_processor(self): + image_processor = CLIPImageProcessor( + crop_size=224, + do_center_crop=True, + do_normalize=True, + do_resize=True, + image_mean=[0.48145466, 0.4578275, 0.40821073], + image_std=[0.26862954, 0.26130258, 0.27577711], + resample=3, + size=224, + ) + + return image_processor + + def get_dummy_components(self): + prior = self.dummy_prior + image_encoder = self.dummy_image_encoder + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + image_processor = self.dummy_image_processor + + scheduler = UnCLIPScheduler( + variance_type="fixed_small_log", + prediction_type="sample", + num_train_timesteps=1000, + clip_sample=True, + clip_sample_range=10.0, + ) + + components = { + "prior": prior, + "image_encoder": image_encoder, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "scheduler": scheduler, + "image_processor": image_processor, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + image = floats_tensor((1, 3, 64, 64), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + init_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((256, 256)) + + inputs = { + "prompt": "horse", + "image": init_image, + "strength": 0.5, + "generator": generator, + "guidance_scale": 4.0, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + def test_kandinsky_prior_emb2emb(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.image_embeds + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -10:] + image_from_tuple_slice = image_from_tuple[0, -10:] + + assert image.shape == (1, 32) + + expected_slice = np.array( + [ + 0.1071284, + 1.3330271, + 0.61260223, + -0.6691065, + -0.3846852, + -1.0303661, + 0.22716111, + 0.03348901, + 0.30040675, + -0.24805029, + ] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + @skip_mps + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=1e-2) + + @skip_mps + def test_attention_slicing_forward_pass(self): + test_max_difference = torch_device == "cpu" + test_mean_pixel_difference = False + + self._test_attention_slicing_forward_pass( + test_max_difference=test_max_difference, + test_mean_pixel_difference=test_mean_pixel_difference, + ) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky3/__init__.py b/diffusers-0.27.0/tests/pipelines/kandinsky3/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky3/test_kandinsky3.py b/diffusers-0.27.0/tests/pipelines/kandinsky3/test_kandinsky3.py new file mode 100755 index 0000000..63b4165 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky3/test_kandinsky3.py @@ -0,0 +1,233 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import AutoTokenizer, T5EncoderModel + +from diffusers import ( + AutoPipelineForImage2Image, + AutoPipelineForText2Image, + Kandinsky3Pipeline, + Kandinsky3UNet, + VQModel, +) +from diffusers.image_processor import VaeImageProcessor +from diffusers.schedulers.scheduling_ddpm import DDPMScheduler +from diffusers.utils.testing_utils import ( + enable_full_determinism, + load_image, + require_torch_gpu, + slow, +) + +from ..pipeline_params import ( + TEXT_TO_IMAGE_BATCH_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_PARAMS, +) +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class Kandinsky3PipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = Kandinsky3Pipeline + params = TEXT_TO_IMAGE_PARAMS - {"cross_attention_kwargs"} + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS + test_xformers_attention = False + + @property + def dummy_movq_kwargs(self): + return { + "block_out_channels": [32, 64], + "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"], + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 1, + "norm_num_groups": 8, + "norm_type": "spatial", + "num_vq_embeddings": 12, + "out_channels": 3, + "up_block_types": [ + "AttnUpDecoderBlock2D", + "UpDecoderBlock2D", + ], + "vq_embed_dim": 4, + } + + @property + def dummy_movq(self): + torch.manual_seed(0) + model = VQModel(**self.dummy_movq_kwargs) + return model + + def get_dummy_components(self, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = Kandinsky3UNet( + in_channels=4, + time_embedding_dim=4, + groups=2, + attention_head_dim=4, + layers_per_block=3, + block_out_channels=(32, 64), + cross_attention_dim=4, + encoder_hid_dim=32, + ) + scheduler = DDPMScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="squaredcos_cap_v2", + clip_sample=True, + thresholding=False, + ) + torch.manual_seed(0) + movq = self.dummy_movq + torch.manual_seed(0) + text_encoder = T5EncoderModel.from_pretrained("hf-internal-testing/tiny-random-t5") + + torch.manual_seed(0) + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-t5") + + components = { + "unet": unet, + "scheduler": scheduler, + "movq": movq, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "np", + "width": 16, + "height": 16, + } + return inputs + + def test_kandinsky3(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 16, 16, 3) + + expected_slice = np.array([0.3768, 0.4373, 0.4865, 0.4890, 0.4299, 0.5122, 0.4921, 0.4924, 0.5599]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=1e-1) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=1e-2) + + +@slow +@require_torch_gpu +class Kandinsky3PipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_kandinskyV3(self): + pipe = AutoPipelineForText2Image.from_pretrained( + "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16 + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + prompt = "A photograph of the inside of a subway train. There are raccoons sitting on the seats. One of them is reading a newspaper. The window shows the city in the background." + + generator = torch.Generator(device="cpu").manual_seed(0) + + image = pipe(prompt, num_inference_steps=25, generator=generator).images[0] + + assert image.size == (1024, 1024) + + expected_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky3/t2i.png" + ) + + image_processor = VaeImageProcessor() + + image_np = image_processor.pil_to_numpy(image) + expected_image_np = image_processor.pil_to_numpy(expected_image) + + self.assertTrue(np.allclose(image_np, expected_image_np, atol=5e-2)) + + def test_kandinskyV3_img2img(self): + pipe = AutoPipelineForImage2Image.from_pretrained( + "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16 + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky3/t2i.png" + ) + w, h = 512, 512 + image = image.resize((w, h), resample=Image.BICUBIC, reducing_gap=1) + prompt = "A painting of the inside of a subway train with tiny raccoons." + + image = pipe(prompt, image=image, strength=0.75, num_inference_steps=25, generator=generator).images[0] + + assert image.size == (512, 512) + + expected_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky3/i2i.png" + ) + + image_processor = VaeImageProcessor() + + image_np = image_processor.pil_to_numpy(image) + expected_image_np = image_processor.pil_to_numpy(expected_image) + + self.assertTrue(np.allclose(image_np, expected_image_np, atol=5e-2)) diff --git a/diffusers-0.27.0/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py b/diffusers-0.27.0/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py new file mode 100755 index 0000000..fbaaaf1 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/kandinsky3/test_kandinsky3_img2img.py @@ -0,0 +1,225 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import AutoTokenizer, T5EncoderModel + +from diffusers import ( + AutoPipelineForImage2Image, + Kandinsky3Img2ImgPipeline, + Kandinsky3UNet, + VQModel, +) +from diffusers.image_processor import VaeImageProcessor +from diffusers.schedulers.scheduling_ddpm import DDPMScheduler +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + require_torch_gpu, + slow, +) + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, +) +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class Kandinsky3Img2ImgPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = Kandinsky3Img2ImgPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS + test_xformers_attention = False + required_optional_params = frozenset( + [ + "num_inference_steps", + "num_images_per_prompt", + "generator", + "output_type", + "return_dict", + ] + ) + + @property + def dummy_movq_kwargs(self): + return { + "block_out_channels": [32, 64], + "down_block_types": ["DownEncoderBlock2D", "AttnDownEncoderBlock2D"], + "in_channels": 3, + "latent_channels": 4, + "layers_per_block": 1, + "norm_num_groups": 8, + "norm_type": "spatial", + "num_vq_embeddings": 12, + "out_channels": 3, + "up_block_types": [ + "AttnUpDecoderBlock2D", + "UpDecoderBlock2D", + ], + "vq_embed_dim": 4, + } + + @property + def dummy_movq(self): + torch.manual_seed(0) + model = VQModel(**self.dummy_movq_kwargs) + return model + + def get_dummy_components(self, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = Kandinsky3UNet( + in_channels=4, + time_embedding_dim=4, + groups=2, + attention_head_dim=4, + layers_per_block=3, + block_out_channels=(32, 64), + cross_attention_dim=4, + encoder_hid_dim=32, + ) + scheduler = DDPMScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="squaredcos_cap_v2", + clip_sample=True, + thresholding=False, + ) + torch.manual_seed(0) + movq = self.dummy_movq + torch.manual_seed(0) + text_encoder = T5EncoderModel.from_pretrained("hf-internal-testing/tiny-random-t5") + + torch.manual_seed(0) + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-t5") + + components = { + "unet": unet, + "scheduler": scheduler, + "movq": movq, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + } + return components + + def get_dummy_inputs(self, device, seed=0): + # create init_image + image = floats_tensor((1, 3, 64, 64), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + init_image = Image.fromarray(np.uint8(image)).convert("RGB") + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": init_image, + "generator": generator, + "strength": 0.75, + "num_inference_steps": 10, + "guidance_scale": 6.0, + "output_type": "np", + } + return inputs + + def test_kandinsky3_img2img(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array( + [0.576259, 0.6132097, 0.41703486, 0.603196, 0.62062526, 0.4655338, 0.5434324, 0.5660727, 0.65433365] + ) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=1e-1) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=1e-2) + + +@slow +@require_torch_gpu +class Kandinsky3Img2ImgPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_kandinskyV3_img2img(self): + pipe = AutoPipelineForImage2Image.from_pretrained( + "kandinsky-community/kandinsky-3", variant="fp16", torch_dtype=torch.float16 + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky3/t2i.png" + ) + w, h = 512, 512 + image = image.resize((w, h), resample=Image.BICUBIC, reducing_gap=1) + prompt = "A painting of the inside of a subway train with tiny raccoons." + + image = pipe(prompt, image=image, strength=0.75, num_inference_steps=25, generator=generator).images[0] + + assert image.size == (512, 512) + + expected_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/kandinsky3/i2i.png" + ) + + image_processor = VaeImageProcessor() + + image_np = image_processor.pil_to_numpy(image) + expected_image_np = image_processor.pil_to_numpy(expected_image) + + self.assertTrue(np.allclose(image_np, expected_image_np, atol=5e-2)) diff --git a/diffusers-0.27.0/tests/pipelines/latent_consistency_models/__init__.py b/diffusers-0.27.0/tests/pipelines/latent_consistency_models/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/latent_consistency_models/test_latent_consistency_models.py b/diffusers-0.27.0/tests/pipelines/latent_consistency_models/test_latent_consistency_models.py new file mode 100755 index 0000000..eaf8fa2 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/latent_consistency_models/test_latent_consistency_models.py @@ -0,0 +1,259 @@ +import gc +import inspect +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + LatentConsistencyModelPipeline, + LCMScheduler, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + require_torch_gpu, + slow, + torch_device, +) + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import IPAdapterTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +class LatentConsistencyModelPipelineFastTests( + IPAdapterTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = LatentConsistencyModelPipeline + params = TEXT_TO_IMAGE_PARAMS - {"negative_prompt", "negative_prompt_embeds"} + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS - {"negative_prompt"} + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(4, 8), + layers_per_block=1, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + norm_num_groups=2, + time_cond_proj_dim=32, + ) + scheduler = LCMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[4, 8], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + norm_num_groups=2, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=64, + layer_norm_eps=1e-05, + num_attention_heads=8, + num_hidden_layers=3, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + "requires_safety_checker": False, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "np", + } + return inputs + + def test_lcm_onestep(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + pipe = LatentConsistencyModelPipeline(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["num_inference_steps"] = 1 + output = pipe(**inputs) + image = output.images + assert image.shape == (1, 64, 64, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.1441, 0.5304, 0.5452, 0.1361, 0.4011, 0.4370, 0.5326, 0.3492, 0.3637]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_lcm_multistep(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + pipe = LatentConsistencyModelPipeline(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = pipe(**inputs) + image = output.images + assert image.shape == (1, 64, 64, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.1403, 0.5072, 0.5316, 0.1202, 0.3865, 0.4211, 0.5363, 0.3557, 0.3645]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + pipe = LatentConsistencyModelPipeline(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + output = pipe(**inputs) + image = output.images + assert image.shape == (1, 64, 64, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.1403, 0.5072, 0.5316, 0.1202, 0.3865, 0.4211, 0.5363, 0.3557, 0.3645]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=5e-4) + + # skip because lcm pipeline apply cfg differently + def test_callback_cfg(self): + pass + + # override default test because the final latent variable is "denoised" instead of "latents" + def test_callback_inputs(self): + sig = inspect.signature(self.pipeline_class.__call__) + + if not ("callback_on_step_end_tensor_inputs" in sig.parameters and "callback_on_step_end" in sig.parameters): + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + self.assertTrue( + hasattr(pipe, "_callback_tensor_inputs"), + f" {self.pipeline_class} should have `_callback_tensor_inputs` that defines a list of tensor variables its callback function can use as inputs", + ) + + def callback_inputs_test(pipe, i, t, callback_kwargs): + missing_callback_inputs = set() + for v in pipe._callback_tensor_inputs: + if v not in callback_kwargs: + missing_callback_inputs.add(v) + self.assertTrue( + len(missing_callback_inputs) == 0, f"Missing callback tensor inputs: {missing_callback_inputs}" + ) + last_i = pipe.num_timesteps - 1 + if i == last_i: + callback_kwargs["denoised"] = torch.zeros_like(callback_kwargs["denoised"]) + return callback_kwargs + + inputs = self.get_dummy_inputs(torch_device) + inputs["callback_on_step_end"] = callback_inputs_test + inputs["callback_on_step_end_tensor_inputs"] = pipe._callback_tensor_inputs + inputs["output_type"] = "latent" + + output = pipe(**inputs)[0] + assert output.abs().sum() == 0 + + +@slow +@require_torch_gpu +class LatentConsistencyModelPipelineSlowTests(unittest.TestCase): + def setUp(self): + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "np", + } + return inputs + + def test_lcm_onestep(self): + pipe = LatentConsistencyModelPipeline.from_pretrained("SimianLuo/LCM_Dreamshaper_v7", safety_checker=None) + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 1 + image = pipe(**inputs).images + assert image.shape == (1, 512, 512, 3) + + image_slice = image[0, -3:, -3:, -1].flatten() + expected_slice = np.array([0.1025, 0.0911, 0.0984, 0.0981, 0.0901, 0.0918, 0.1055, 0.0940, 0.0730]) + assert np.abs(image_slice - expected_slice).max() < 1e-3 + + def test_lcm_multistep(self): + pipe = LatentConsistencyModelPipeline.from_pretrained("SimianLuo/LCM_Dreamshaper_v7", safety_checker=None) + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + assert image.shape == (1, 512, 512, 3) + + image_slice = image[0, -3:, -3:, -1].flatten() + expected_slice = np.array([0.01855, 0.01855, 0.01489, 0.01392, 0.01782, 0.01465, 0.01831, 0.02539, 0.0]) + assert np.abs(image_slice - expected_slice).max() < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/latent_consistency_models/test_latent_consistency_models_img2img.py b/diffusers-0.27.0/tests/pipelines/latent_consistency_models/test_latent_consistency_models_img2img.py new file mode 100755 index 0000000..cfd596d --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/latent_consistency_models/test_latent_consistency_models_img2img.py @@ -0,0 +1,277 @@ +import gc +import inspect +import random +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + LatentConsistencyModelImg2ImgPipeline, + LCMScheduler, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + require_torch_gpu, + slow, + torch_device, +) + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, +) +from ..test_pipelines_common import IPAdapterTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +class LatentConsistencyModelImg2ImgPipelineFastTests( + IPAdapterTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = LatentConsistencyModelImg2ImgPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width", "negative_prompt", "negative_prompt_embeds"} + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents", "negative_prompt"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + image_latents_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(4, 8), + layers_per_block=1, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + norm_num_groups=2, + time_cond_proj_dim=32, + ) + scheduler = LCMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[4, 8], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + norm_num_groups=2, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=64, + layer_norm_eps=1e-05, + num_attention_heads=8, + num_hidden_layers=3, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + "requires_safety_checker": False, + } + return components + + def get_dummy_inputs(self, device, seed=0): + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image / 2 + 0.5 + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "np", + } + return inputs + + def test_lcm_onestep(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["num_inference_steps"] = 1 + output = pipe(**inputs) + image = output.images + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.4388, 0.3717, 0.2202, 0.7213, 0.6370, 0.3664, 0.5815, 0.6080, 0.4977]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_lcm_multistep(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = pipe(**inputs) + image = output.images + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.4150, 0.3719, 0.2479, 0.6333, 0.6024, 0.3778, 0.5036, 0.5420, 0.4678]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + output = pipe(**inputs) + image = output.images + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.3994, 0.3471, 0.2540, 0.7030, 0.6193, 0.3645, 0.5777, 0.5850, 0.4965]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=5e-4) + + # override default test because the final latent variable is "denoised" instead of "latents" + def test_callback_inputs(self): + sig = inspect.signature(self.pipeline_class.__call__) + + if not ("callback_on_step_end_tensor_inputs" in sig.parameters and "callback_on_step_end" in sig.parameters): + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + self.assertTrue( + hasattr(pipe, "_callback_tensor_inputs"), + f" {self.pipeline_class} should have `_callback_tensor_inputs` that defines a list of tensor variables its callback function can use as inputs", + ) + + def callback_inputs_test(pipe, i, t, callback_kwargs): + missing_callback_inputs = set() + for v in pipe._callback_tensor_inputs: + if v not in callback_kwargs: + missing_callback_inputs.add(v) + self.assertTrue( + len(missing_callback_inputs) == 0, f"Missing callback tensor inputs: {missing_callback_inputs}" + ) + last_i = pipe.num_timesteps - 1 + if i == last_i: + callback_kwargs["denoised"] = torch.zeros_like(callback_kwargs["denoised"]) + return callback_kwargs + + inputs = self.get_dummy_inputs(torch_device) + inputs["callback_on_step_end"] = callback_inputs_test + inputs["callback_on_step_end_tensor_inputs"] = pipe._callback_tensor_inputs + inputs["output_type"] = "latent" + + output = pipe(**inputs)[0] + assert output.abs().sum() == 0 + + +@slow +@require_torch_gpu +class LatentConsistencyModelImg2ImgPipelineSlowTests(unittest.TestCase): + def setUp(self): + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + init_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/sketch-mountains-input.png" + ) + init_image = init_image.resize((512, 512)) + + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "np", + "image": init_image, + } + return inputs + + def test_lcm_onestep(self): + pipe = LatentConsistencyModelImg2ImgPipeline.from_pretrained( + "SimianLuo/LCM_Dreamshaper_v7", safety_checker=None + ) + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 1 + image = pipe(**inputs).images + assert image.shape == (1, 512, 512, 3) + + image_slice = image[0, -3:, -3:, -1].flatten() + expected_slice = np.array([0.1950, 0.1961, 0.2308, 0.1786, 0.1837, 0.2320, 0.1898, 0.1885, 0.2309]) + assert np.abs(image_slice - expected_slice).max() < 1e-3 + + def test_lcm_multistep(self): + pipe = LatentConsistencyModelImg2ImgPipeline.from_pretrained( + "SimianLuo/LCM_Dreamshaper_v7", safety_checker=None + ) + pipe.scheduler = LCMScheduler.from_config(pipe.scheduler.config) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + assert image.shape == (1, 512, 512, 3) + + image_slice = image[0, -3:, -3:, -1].flatten() + expected_slice = np.array([0.3756, 0.3816, 0.3767, 0.3718, 0.3739, 0.3735, 0.3863, 0.3803, 0.3563]) + assert np.abs(image_slice - expected_slice).max() < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/latent_diffusion/__init__.py b/diffusers-0.27.0/tests/pipelines/latent_diffusion/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/latent_diffusion/test_latent_diffusion.py b/diffusers-0.27.0/tests/pipelines/latent_diffusion/test_latent_diffusion.py new file mode 100755 index 0000000..4faa0e7 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/latent_diffusion/test_latent_diffusion.py @@ -0,0 +1,207 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import AutoencoderKL, DDIMScheduler, LDMTextToImagePipeline, UNet2DConditionModel +from diffusers.utils.testing_utils import ( + enable_full_determinism, + load_numpy, + nightly, + require_torch_gpu, + torch_device, +) + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class LDMTextToImagePipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = LDMTextToImagePipeline + params = TEXT_TO_IMAGE_PARAMS - { + "negative_prompt", + "negative_prompt_embeds", + "cross_attention_kwargs", + "prompt_embeds", + } + required_optional_params = PipelineTesterMixin.required_optional_params - { + "num_images_per_prompt", + "callback", + "callback_steps", + } + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=(32, 64), + in_channels=3, + out_channels=3, + down_block_types=("DownEncoderBlock2D", "DownEncoderBlock2D"), + up_block_types=("UpDecoderBlock2D", "UpDecoderBlock2D"), + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vqvae": vae, + "bert": text_encoder, + "tokenizer": tokenizer, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_inference_text2img(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + pipe = LDMTextToImagePipeline(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 16, 16, 3) + expected_slice = np.array([0.6101, 0.6156, 0.5622, 0.4895, 0.6661, 0.3804, 0.5748, 0.6136, 0.5014]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + +@nightly +@require_torch_gpu +class LDMTextToImagePipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 32, 32)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_ldm_default_ddim(self): + pipe = LDMTextToImagePipeline.from_pretrained("CompVis/ldm-text2im-large-256").to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 256, 256, 3) + expected_slice = np.array([0.51825, 0.52850, 0.52543, 0.54258, 0.52304, 0.52569, 0.54363, 0.55276, 0.56878]) + max_diff = np.abs(expected_slice - image_slice).max() + assert max_diff < 1e-3 + + +@nightly +@require_torch_gpu +class LDMTextToImagePipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, dtype=torch.float32, seed=0): + generator = torch.manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 32, 32)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "latents": latents, + "generator": generator, + "num_inference_steps": 50, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_ldm_default_ddim(self): + pipe = LDMTextToImagePipeline.from_pretrained("CompVis/ldm-text2im-large-256").to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/ldm_text2img/ldm_large_256_ddim.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py b/diffusers-0.27.0/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py new file mode 100755 index 0000000..a9df2c1 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/latent_diffusion/test_latent_diffusion_superresolution.py @@ -0,0 +1,138 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest + +import numpy as np +import torch + +from diffusers import DDIMScheduler, LDMSuperResolutionPipeline, UNet2DModel, VQModel +from diffusers.utils import PIL_INTERPOLATION +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + nightly, + require_torch, + torch_device, +) + + +enable_full_determinism() + + +class LDMSuperResolutionPipelineFastTests(unittest.TestCase): + @property + def dummy_image(self): + batch_size = 1 + num_channels = 3 + sizes = (32, 32) + + image = floats_tensor((batch_size, num_channels) + sizes, rng=random.Random(0)).to(torch_device) + return image + + @property + def dummy_uncond_unet(self): + torch.manual_seed(0) + model = UNet2DModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=6, + out_channels=3, + down_block_types=("DownBlock2D", "AttnDownBlock2D"), + up_block_types=("AttnUpBlock2D", "UpBlock2D"), + ) + return model + + @property + def dummy_vq_model(self): + torch.manual_seed(0) + model = VQModel( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=3, + ) + return model + + def test_inference_superresolution(self): + device = "cpu" + unet = self.dummy_uncond_unet + scheduler = DDIMScheduler() + vqvae = self.dummy_vq_model + + ldm = LDMSuperResolutionPipeline(unet=unet, vqvae=vqvae, scheduler=scheduler) + ldm.to(device) + ldm.set_progress_bar_config(disable=None) + + init_image = self.dummy_image.to(device) + + generator = torch.Generator(device=device).manual_seed(0) + image = ldm(image=init_image, generator=generator, num_inference_steps=2, output_type="numpy").images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.8678, 0.8245, 0.6381, 0.6830, 0.4385, 0.5599, 0.4641, 0.6201, 0.5150]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + @unittest.skipIf(torch_device != "cuda", "This test requires a GPU") + def test_inference_superresolution_fp16(self): + unet = self.dummy_uncond_unet + scheduler = DDIMScheduler() + vqvae = self.dummy_vq_model + + # put models in fp16 + unet = unet.half() + vqvae = vqvae.half() + + ldm = LDMSuperResolutionPipeline(unet=unet, vqvae=vqvae, scheduler=scheduler) + ldm.to(torch_device) + ldm.set_progress_bar_config(disable=None) + + init_image = self.dummy_image.to(torch_device) + + image = ldm(init_image, num_inference_steps=2, output_type="numpy").images + + assert image.shape == (1, 64, 64, 3) + + +@nightly +@require_torch +class LDMSuperResolutionPipelineIntegrationTests(unittest.TestCase): + def test_inference_superresolution(self): + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/vq_diffusion/teddy_bear_pool.png" + ) + init_image = init_image.resize((64, 64), resample=PIL_INTERPOLATION["lanczos"]) + + ldm = LDMSuperResolutionPipeline.from_pretrained("duongna/ldm-super-resolution", device_map="auto") + ldm.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(0) + image = ldm(image=init_image, generator=generator, num_inference_steps=20, output_type="numpy").images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 256, 256, 3) + expected_slice = np.array([0.7644, 0.7679, 0.7642, 0.7633, 0.7666, 0.7560, 0.7425, 0.7257, 0.6907]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/ledits_pp/__init__.py b/diffusers-0.27.0/tests/pipelines/ledits_pp/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/ledits_pp/test_ledits_pp_stable_diffusion.py b/diffusers-0.27.0/tests/pipelines/ledits_pp/test_ledits_pp_stable_diffusion.py new file mode 100755 index 0000000..9ff75a9 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/ledits_pp/test_ledits_pp_stable_diffusion.py @@ -0,0 +1,244 @@ +# coding=utf-8 +# Copyright 2023 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DPMSolverMultistepScheduler, + LEditsPPPipelineStableDiffusion, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + require_torch_gpu, + skip_mps, + slow, + torch_device, +) + + +enable_full_determinism() + + +@skip_mps +class LEditsPPPipelineStableDiffusionFastTests(unittest.TestCase): + pipeline_class = LEditsPPPipelineStableDiffusion + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + scheduler = DPMSolverMultistepScheduler(algorithm_type="sde-dpmsolver++", solver_order=2) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "generator": generator, + "editing_prompt": ["wearing glasses", "sunshine"], + "reverse_editing_direction": [False, True], + "edit_guidance_scale": [10.0, 5.0], + } + return inputs + + def get_dummy_inversion_inputs(self, device, seed=0): + images = floats_tensor((2, 3, 32, 32), rng=random.Random(0)).cpu().permute(0, 2, 3, 1) + images = 255 * images + image_1 = Image.fromarray(np.uint8(images[0])).convert("RGB") + image_2 = Image.fromarray(np.uint8(images[1])).convert("RGB") + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + inputs = { + "image": [image_1, image_2], + "source_prompt": "", + "source_guidance_scale": 3.5, + "num_inversion_steps": 20, + "skip": 0.15, + "generator": generator, + } + return inputs + + def test_ledits_pp_inversion(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = LEditsPPPipelineStableDiffusion(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inversion_inputs(device) + inputs["image"] = inputs["image"][0] + sd_pipe.invert(**inputs) + assert sd_pipe.init_latents.shape == ( + 1, + 4, + int(32 / sd_pipe.vae_scale_factor), + int(32 / sd_pipe.vae_scale_factor), + ) + + latent_slice = sd_pipe.init_latents[0, -1, -3:, -3:].to(device) + print(latent_slice.flatten()) + expected_slice = np.array([-0.9084, -0.0367, 0.2940, 0.0839, 0.6890, 0.2651, -0.7104, 2.1090, -0.7822]) + assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3 + + def test_ledits_pp_inversion_batch(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = LEditsPPPipelineStableDiffusion(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inversion_inputs(device) + sd_pipe.invert(**inputs) + assert sd_pipe.init_latents.shape == ( + 2, + 4, + int(32 / sd_pipe.vae_scale_factor), + int(32 / sd_pipe.vae_scale_factor), + ) + + latent_slice = sd_pipe.init_latents[0, -1, -3:, -3:].to(device) + print(latent_slice.flatten()) + expected_slice = np.array([0.2528, 0.1458, -0.2166, 0.4565, -0.5657, -1.0286, -0.9961, 0.5933, 1.1173]) + assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3 + + latent_slice = sd_pipe.init_latents[1, -1, -3:, -3:].to(device) + print(latent_slice.flatten()) + expected_slice = np.array([-0.0796, 2.0583, 0.5501, 0.5358, 0.0282, -0.2803, -1.0470, 0.7023, -0.0072]) + assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3 + + def test_ledits_pp_warmup_steps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = LEditsPPPipelineStableDiffusion(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inversion_inputs = self.get_dummy_inversion_inputs(device) + pipe.invert(**inversion_inputs) + + inputs = self.get_dummy_inputs(device) + + inputs["edit_warmup_steps"] = [0, 5] + pipe(**inputs).images + + inputs["edit_warmup_steps"] = [5, 0] + pipe(**inputs).images + + inputs["edit_warmup_steps"] = [5, 10] + pipe(**inputs).images + + inputs["edit_warmup_steps"] = [10, 5] + pipe(**inputs).images + + +@slow +@require_torch_gpu +class LEditsPPPipelineStableDiffusionSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + @classmethod + def setUpClass(cls): + raw_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/pix2pix/cat_6.png" + ) + raw_image = raw_image.convert("RGB").resize((512, 512)) + cls.raw_image = raw_image + + def test_ledits_pp_editing(self): + pipe = LEditsPPPipelineStableDiffusion.from_pretrained( + "runwayml/stable-diffusion-v1-5", safety_checker=None, torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(0) + _ = pipe.invert(image=self.raw_image, generator=generator) + generator = torch.manual_seed(0) + inputs = { + "generator": generator, + "editing_prompt": ["cat", "dog"], + "reverse_editing_direction": [True, False], + "edit_guidance_scale": [5.0, 5.0], + "edit_threshold": [0.8, 0.8], + } + reconstruction = pipe(**inputs, output_type="np").images[0] + + output_slice = reconstruction[150:153, 140:143, -1] + output_slice = output_slice.flatten() + expected_slice = np.array( + [0.9453125, 0.93310547, 0.84521484, 0.94628906, 0.9111328, 0.80859375, 0.93847656, 0.9042969, 0.8144531] + ) + assert np.abs(output_slice - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/ledits_pp/test_ledits_pp_stable_diffusion_xl.py b/diffusers-0.27.0/tests/pipelines/ledits_pp/test_ledits_pp_stable_diffusion_xl.py new file mode 100755 index 0000000..fcfd0aa --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/ledits_pp/test_ledits_pp_stable_diffusion_xl.py @@ -0,0 +1,289 @@ +# coding=utf-8 +# Copyright 2023 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import ( + CLIPImageProcessor, + CLIPTextConfig, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +from diffusers import ( + AutoencoderKL, + DPMSolverMultistepScheduler, + LEditsPPPipelineStableDiffusionXL, + UNet2DConditionModel, +) + +# from diffusers.image_processor import VaeImageProcessor +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + require_torch_gpu, + skip_mps, + slow, + torch_device, +) + + +enable_full_determinism() + + +@skip_mps +class LEditsPPPipelineStableDiffusionXLFastTests(unittest.TestCase): + pipeline_class = LEditsPPPipelineStableDiffusionXL + + def get_dummy_components(self, skip_first_text_encoder=False, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + time_cond_proj_dim=time_cond_proj_dim, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64 if not skip_first_text_encoder else 32, + ) + scheduler = DPMSolverMultistepScheduler(algorithm_type="sde-dpmsolver++", solver_order=2) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + image_encoder_config = CLIPVisionConfig( + hidden_size=32, + image_size=224, + projection_dim=32, + intermediate_size=37, + num_attention_heads=4, + num_channels=3, + num_hidden_layers=5, + patch_size=14, + ) + image_encoder = CLIPVisionModelWithProjection(image_encoder_config) + + feature_extractor = CLIPImageProcessor( + crop_size=224, + do_center_crop=True, + do_normalize=True, + do_resize=True, + image_mean=[0.48145466, 0.4578275, 0.40821073], + image_std=[0.26862954, 0.26130258, 0.27577711], + resample=3, + size=224, + ) + + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder if not skip_first_text_encoder else None, + "tokenizer": tokenizer if not skip_first_text_encoder else None, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "image_encoder": image_encoder, + "feature_extractor": feature_extractor, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "generator": generator, + "editing_prompt": ["wearing glasses", "sunshine"], + "reverse_editing_direction": [False, True], + "edit_guidance_scale": [10.0, 5.0], + } + return inputs + + def get_dummy_inversion_inputs(self, device, seed=0): + images = floats_tensor((2, 3, 32, 32), rng=random.Random(0)).cpu().permute(0, 2, 3, 1) + images = 255 * images + image_1 = Image.fromarray(np.uint8(images[0])).convert("RGB") + image_2 = Image.fromarray(np.uint8(images[1])).convert("RGB") + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + inputs = { + "image": [image_1, image_2], + "source_prompt": "", + "source_guidance_scale": 3.5, + "num_inversion_steps": 20, + "skip": 0.15, + "generator": generator, + } + return inputs + + def test_ledits_pp_inversion(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = LEditsPPPipelineStableDiffusionXL(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inversion_inputs(device) + inputs["image"] = inputs["image"][0] + sd_pipe.invert(**inputs) + assert sd_pipe.init_latents.shape == ( + 1, + 4, + int(32 / sd_pipe.vae_scale_factor), + int(32 / sd_pipe.vae_scale_factor), + ) + + latent_slice = sd_pipe.init_latents[0, -1, -3:, -3:].to(device) + expected_slice = np.array([-0.9084, -0.0367, 0.2940, 0.0839, 0.6890, 0.2651, -0.7103, 2.1090, -0.7821]) + assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3 + + def test_ledits_pp_inversion_batch(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = LEditsPPPipelineStableDiffusionXL(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inversion_inputs(device) + sd_pipe.invert(**inputs) + assert sd_pipe.init_latents.shape == ( + 2, + 4, + int(32 / sd_pipe.vae_scale_factor), + int(32 / sd_pipe.vae_scale_factor), + ) + + latent_slice = sd_pipe.init_latents[0, -1, -3:, -3:].to(device) + print(latent_slice.flatten()) + expected_slice = np.array([0.2528, 0.1458, -0.2166, 0.4565, -0.5656, -1.0286, -0.9961, 0.5933, 1.1172]) + assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3 + + latent_slice = sd_pipe.init_latents[1, -1, -3:, -3:].to(device) + print(latent_slice.flatten()) + expected_slice = np.array([-0.0796, 2.0583, 0.5500, 0.5358, 0.0282, -0.2803, -1.0470, 0.7024, -0.0072]) + print(latent_slice.flatten()) + assert np.abs(latent_slice.flatten() - expected_slice).max() < 1e-3 + + def test_ledits_pp_warmup_steps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = LEditsPPPipelineStableDiffusionXL(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inversion_inputs = self.get_dummy_inversion_inputs(device) + inversion_inputs["image"] = inversion_inputs["image"][0] + pipe.invert(**inversion_inputs) + + inputs = self.get_dummy_inputs(device) + + inputs["edit_warmup_steps"] = [0, 5] + pipe(**inputs).images + + inputs["edit_warmup_steps"] = [5, 0] + pipe(**inputs).images + + inputs["edit_warmup_steps"] = [5, 10] + pipe(**inputs).images + + inputs["edit_warmup_steps"] = [10, 5] + pipe(**inputs).images + + +@slow +@require_torch_gpu +class LEditsPPPipelineStableDiffusionXLSlowTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + raw_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/pix2pix/cat_6.png" + ) + raw_image = raw_image.convert("RGB").resize((512, 512)) + cls.raw_image = raw_image + + def test_ledits_pp_edit(self): + pipe = LEditsPPPipelineStableDiffusionXL.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", safety_checker=None, add_watermarker=None + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(0) + _ = pipe.invert(image=self.raw_image, generator=generator, num_zero_noise_steps=0) + inputs = { + "generator": generator, + "editing_prompt": ["cat", "dog"], + "reverse_editing_direction": [True, False], + "edit_guidance_scale": [2.0, 4.0], + "edit_threshold": [0.8, 0.8], + } + reconstruction = pipe(**inputs, output_type="np").images[0] + + output_slice = reconstruction[150:153, 140:143, -1] + output_slice = output_slice.flatten() + expected_slice = np.array( + [0.56419, 0.44121838, 0.2765603, 0.5708484, 0.42763475, 0.30945742, 0.5387106, 0.4735807, 0.3547244] + ) + assert np.abs(output_slice - expected_slice).max() < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/musicldm/__init__.py b/diffusers-0.27.0/tests/pipelines/musicldm/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/musicldm/test_musicldm.py b/diffusers-0.27.0/tests/pipelines/musicldm/test_musicldm.py new file mode 100755 index 0000000..779b0cb --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/musicldm/test_musicldm.py @@ -0,0 +1,465 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import gc +import unittest + +import numpy as np +import torch +from transformers import ( + ClapAudioConfig, + ClapConfig, + ClapFeatureExtractor, + ClapModel, + ClapTextConfig, + RobertaTokenizer, + SpeechT5HifiGan, + SpeechT5HifiGanConfig, +) + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + LMSDiscreteScheduler, + MusicLDMPipeline, + PNDMScheduler, + UNet2DConditionModel, +) +from diffusers.utils import is_xformers_available +from diffusers.utils.testing_utils import enable_full_determinism, nightly, require_torch_gpu, torch_device + +from ..pipeline_params import TEXT_TO_AUDIO_BATCH_PARAMS, TEXT_TO_AUDIO_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class MusicLDMPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = MusicLDMPipeline + params = TEXT_TO_AUDIO_PARAMS + batch_params = TEXT_TO_AUDIO_BATCH_PARAMS + required_optional_params = frozenset( + [ + "num_inference_steps", + "num_waveforms_per_prompt", + "generator", + "latents", + "output_type", + "return_dict", + "callback", + "callback_steps", + ] + ) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=(32, 64), + class_embed_type="simple_projection", + projection_class_embeddings_input_dim=32, + class_embeddings_concat=True, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=1, + out_channels=1, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_branch_config = ClapTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=16, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=2, + num_hidden_layers=2, + pad_token_id=1, + vocab_size=1000, + ) + audio_branch_config = ClapAudioConfig( + spec_size=64, + window_size=4, + num_mel_bins=64, + intermediate_size=37, + layer_norm_eps=1e-05, + depths=[2, 2], + num_attention_heads=[2, 2], + num_hidden_layers=2, + hidden_size=192, + patch_size=2, + patch_stride=2, + patch_embed_input_channels=4, + ) + text_encoder_config = ClapConfig.from_text_audio_configs( + text_config=text_branch_config, audio_config=audio_branch_config, projection_dim=32 + ) + text_encoder = ClapModel(text_encoder_config) + tokenizer = RobertaTokenizer.from_pretrained("hf-internal-testing/tiny-random-roberta", model_max_length=77) + feature_extractor = ClapFeatureExtractor.from_pretrained( + "hf-internal-testing/tiny-random-ClapModel", hop_length=7900 + ) + + torch.manual_seed(0) + vocoder_config = SpeechT5HifiGanConfig( + model_in_dim=8, + sampling_rate=16000, + upsample_initial_channel=16, + upsample_rates=[2, 2], + upsample_kernel_sizes=[4, 4], + resblock_kernel_sizes=[3, 7], + resblock_dilation_sizes=[[1, 3, 5], [1, 3, 5]], + normalize_before=False, + ) + + vocoder = SpeechT5HifiGan(vocoder_config) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "feature_extractor": feature_extractor, + "vocoder": vocoder, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A hammer hitting a wooden surface", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + } + return inputs + + def test_musicldm_ddim(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + musicldm_pipe = MusicLDMPipeline(**components) + musicldm_pipe = musicldm_pipe.to(torch_device) + musicldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = musicldm_pipe(**inputs) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) == 256 + + audio_slice = audio[:10] + expected_slice = np.array( + [-0.0027, -0.0036, -0.0037, -0.0020, -0.0035, -0.0019, -0.0037, -0.0020, -0.0038, -0.0019] + ) + + assert np.abs(audio_slice - expected_slice).max() < 1e-4 + + def test_musicldm_prompt_embeds(self): + components = self.get_dummy_components() + musicldm_pipe = MusicLDMPipeline(**components) + musicldm_pipe = musicldm_pipe.to(torch_device) + musicldm_pipe = musicldm_pipe.to(torch_device) + musicldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = musicldm_pipe(**inputs) + audio_1 = output.audios[0] + + inputs = self.get_dummy_inputs(torch_device) + prompt = 3 * [inputs.pop("prompt")] + + text_inputs = musicldm_pipe.tokenizer( + prompt, + padding="max_length", + max_length=musicldm_pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + prompt_embeds = musicldm_pipe.text_encoder.get_text_features(text_inputs) + + inputs["prompt_embeds"] = prompt_embeds + + # forward + output = musicldm_pipe(**inputs) + audio_2 = output.audios[0] + + assert np.abs(audio_1 - audio_2).max() < 1e-2 + + def test_musicldm_negative_prompt_embeds(self): + components = self.get_dummy_components() + musicldm_pipe = MusicLDMPipeline(**components) + musicldm_pipe = musicldm_pipe.to(torch_device) + musicldm_pipe = musicldm_pipe.to(torch_device) + musicldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + negative_prompt = 3 * ["this is a negative prompt"] + inputs["negative_prompt"] = negative_prompt + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = musicldm_pipe(**inputs) + audio_1 = output.audios[0] + + inputs = self.get_dummy_inputs(torch_device) + prompt = 3 * [inputs.pop("prompt")] + + embeds = [] + for p in [prompt, negative_prompt]: + text_inputs = musicldm_pipe.tokenizer( + p, + padding="max_length", + max_length=musicldm_pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + text_embeds = musicldm_pipe.text_encoder.get_text_features( + text_inputs, + ) + embeds.append(text_embeds) + + inputs["prompt_embeds"], inputs["negative_prompt_embeds"] = embeds + + # forward + output = musicldm_pipe(**inputs) + audio_2 = output.audios[0] + + assert np.abs(audio_1 - audio_2).max() < 1e-2 + + def test_musicldm_negative_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = PNDMScheduler(skip_prk_steps=True) + musicldm_pipe = MusicLDMPipeline(**components) + musicldm_pipe = musicldm_pipe.to(device) + musicldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + negative_prompt = "egg cracking" + output = musicldm_pipe(**inputs, negative_prompt=negative_prompt) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) == 256 + + audio_slice = audio[:10] + expected_slice = np.array( + [-0.0027, -0.0036, -0.0037, -0.0019, -0.0035, -0.0018, -0.0037, -0.0021, -0.0038, -0.0018] + ) + + assert np.abs(audio_slice - expected_slice).max() < 1e-4 + + def test_musicldm_num_waveforms_per_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = PNDMScheduler(skip_prk_steps=True) + musicldm_pipe = MusicLDMPipeline(**components) + musicldm_pipe = musicldm_pipe.to(device) + musicldm_pipe.set_progress_bar_config(disable=None) + + prompt = "A hammer hitting a wooden surface" + + # test num_waveforms_per_prompt=1 (default) + audios = musicldm_pipe(prompt, num_inference_steps=2).audios + + assert audios.shape == (1, 256) + + # test num_waveforms_per_prompt=1 (default) for batch of prompts + batch_size = 2 + audios = musicldm_pipe([prompt] * batch_size, num_inference_steps=2).audios + + assert audios.shape == (batch_size, 256) + + # test num_waveforms_per_prompt for single prompt + num_waveforms_per_prompt = 2 + audios = musicldm_pipe(prompt, num_inference_steps=2, num_waveforms_per_prompt=num_waveforms_per_prompt).audios + + assert audios.shape == (num_waveforms_per_prompt, 256) + + # test num_waveforms_per_prompt for batch of prompts + batch_size = 2 + audios = musicldm_pipe( + [prompt] * batch_size, num_inference_steps=2, num_waveforms_per_prompt=num_waveforms_per_prompt + ).audios + + assert audios.shape == (batch_size * num_waveforms_per_prompt, 256) + + def test_musicldm_audio_length_in_s(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + musicldm_pipe = MusicLDMPipeline(**components) + musicldm_pipe = musicldm_pipe.to(torch_device) + musicldm_pipe.set_progress_bar_config(disable=None) + vocoder_sampling_rate = musicldm_pipe.vocoder.config.sampling_rate + + inputs = self.get_dummy_inputs(device) + output = musicldm_pipe(audio_length_in_s=0.016, **inputs) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) / vocoder_sampling_rate == 0.016 + + output = musicldm_pipe(audio_length_in_s=0.032, **inputs) + audio = output.audios[0] + + assert audio.ndim == 1 + assert len(audio) / vocoder_sampling_rate == 0.032 + + def test_musicldm_vocoder_model_in_dim(self): + components = self.get_dummy_components() + musicldm_pipe = MusicLDMPipeline(**components) + musicldm_pipe = musicldm_pipe.to(torch_device) + musicldm_pipe.set_progress_bar_config(disable=None) + + prompt = ["hey"] + + output = musicldm_pipe(prompt, num_inference_steps=1) + audio_shape = output.audios.shape + assert audio_shape == (1, 256) + + config = musicldm_pipe.vocoder.config + config.model_in_dim *= 2 + musicldm_pipe.vocoder = SpeechT5HifiGan(config).to(torch_device) + output = musicldm_pipe(prompt, num_inference_steps=1) + audio_shape = output.audios.shape + # waveform shape is unchanged, we just have 2x the number of mel channels in the spectrogram + assert audio_shape == (1, 256) + + def test_attention_slicing_forward_pass(self): + self._test_attention_slicing_forward_pass(test_mean_pixel_difference=False) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical() + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(test_mean_pixel_difference=False) + + def test_to_dtype(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + # The method component.dtype returns the dtype of the first parameter registered in the model, not the + # dtype of the entire model. In the case of CLAP, the first parameter is a float64 constant (logit scale) + model_dtypes = {key: component.dtype for key, component in components.items() if hasattr(component, "dtype")} + + # Without the logit scale parameters, everything is float32 + model_dtypes.pop("text_encoder") + self.assertTrue(all(dtype == torch.float32 for dtype in model_dtypes.values())) + + # the CLAP sub-models are float32 + model_dtypes["clap_text_branch"] = components["text_encoder"].text_model.dtype + self.assertTrue(all(dtype == torch.float32 for dtype in model_dtypes.values())) + + # Once we send to fp16, all params are in half-precision, including the logit scale + pipe.to(dtype=torch.float16) + model_dtypes = {key: component.dtype for key, component in components.items() if hasattr(component, "dtype")} + self.assertTrue(all(dtype == torch.float16 for dtype in model_dtypes.values())) + + +@nightly +@require_torch_gpu +class MusicLDMPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 8, 128, 16)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "A hammer hitting a wooden surface", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 2.5, + } + return inputs + + def test_musicldm(self): + musicldm_pipe = MusicLDMPipeline.from_pretrained("cvssp/musicldm") + musicldm_pipe = musicldm_pipe.to(torch_device) + musicldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 25 + audio = musicldm_pipe(**inputs).audios[0] + + assert audio.ndim == 1 + assert len(audio) == 81952 + + # check the portion of the generated audio with the largest dynamic range (reduces flakiness) + audio_slice = audio[8680:8690] + expected_slice = np.array( + [-0.1042, -0.1068, -0.1235, -0.1387, -0.1428, -0.136, -0.1213, -0.1097, -0.0967, -0.0945] + ) + max_diff = np.abs(expected_slice - audio_slice).max() + assert max_diff < 1e-3 + + def test_musicldm_lms(self): + musicldm_pipe = MusicLDMPipeline.from_pretrained("cvssp/musicldm") + musicldm_pipe.scheduler = LMSDiscreteScheduler.from_config(musicldm_pipe.scheduler.config) + musicldm_pipe = musicldm_pipe.to(torch_device) + musicldm_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + audio = musicldm_pipe(**inputs).audios[0] + + assert audio.ndim == 1 + assert len(audio) == 81952 + + # check the portion of the generated audio with the largest dynamic range (reduces flakiness) + audio_slice = audio[58020:58030] + expected_slice = np.array([0.3592, 0.3477, 0.4084, 0.4665, 0.5048, 0.5891, 0.6461, 0.5579, 0.4595, 0.4403]) + max_diff = np.abs(expected_slice - audio_slice).max() + assert max_diff < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/paint_by_example/__init__.py b/diffusers-0.27.0/tests/pipelines/paint_by_example/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/paint_by_example/test_paint_by_example.py b/diffusers-0.27.0/tests/pipelines/paint_by_example/test_paint_by_example.py new file mode 100755 index 0000000..7771977 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/paint_by_example/test_paint_by_example.py @@ -0,0 +1,220 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPImageProcessor, CLIPVisionConfig + +from diffusers import AutoencoderKL, PaintByExamplePipeline, PNDMScheduler, UNet2DConditionModel +from diffusers.pipelines.paint_by_example import PaintByExampleImageEncoder +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + nightly, + require_torch_gpu, + torch_device, +) + +from ..pipeline_params import IMAGE_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, IMAGE_GUIDED_IMAGE_INPAINTING_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class PaintByExamplePipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = PaintByExamplePipeline + params = IMAGE_GUIDED_IMAGE_INPAINTING_PARAMS + batch_params = IMAGE_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + image_params = frozenset([]) # TO_DO: update the image_prams once refactored VaeImageProcessor.preprocess + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=9, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + scheduler = PNDMScheduler(skip_prk_steps=True) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + config = CLIPVisionConfig( + hidden_size=32, + projection_dim=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + image_size=32, + patch_size=4, + ) + image_encoder = PaintByExampleImageEncoder(config, proj_size=32) + feature_extractor = CLIPImageProcessor(crop_size=32, size=32) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "image_encoder": image_encoder, + "safety_checker": None, + "feature_extractor": feature_extractor, + } + return components + + def convert_to_pt(self, image): + image = np.array(image.convert("RGB")) + image = image[None].transpose(0, 3, 1, 2) + image = torch.from_numpy(image).to(dtype=torch.float32) / 127.5 - 1.0 + return image + + def get_dummy_inputs(self, device="cpu", seed=0): + # TODO: use tensor inputs instead of PIL, this is here just to leave the old expected_slices untouched + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + init_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((64, 64)) + mask_image = Image.fromarray(np.uint8(image + 4)).convert("RGB").resize((64, 64)) + example_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((32, 32)) + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "example_image": example_image, + "image": init_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_paint_by_example_inpaint(self): + components = self.get_dummy_components() + + # make sure here that pndm scheduler skips prk + pipe = PaintByExamplePipeline(**components) + pipe = pipe.to("cpu") + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + output = pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4686, 0.5687, 0.4007, 0.5218, 0.5741, 0.4482, 0.4940, 0.4629, 0.4503]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_paint_by_example_image_tensor(self): + device = "cpu" + inputs = self.get_dummy_inputs() + inputs.pop("mask_image") + image = self.convert_to_pt(inputs.pop("image")) + mask_image = image.clamp(0, 1) / 2 + + # make sure here that pndm scheduler skips prk + pipe = PaintByExamplePipeline(**self.get_dummy_components()) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + output = pipe(image=image, mask_image=mask_image[:, 0], **inputs) + out_1 = output.images + + image = image.cpu().permute(0, 2, 3, 1)[0] + mask_image = mask_image.cpu().permute(0, 2, 3, 1)[0] + + image = Image.fromarray(np.uint8(image)).convert("RGB") + mask_image = Image.fromarray(np.uint8(mask_image)).convert("RGB") + + output = pipe(**self.get_dummy_inputs()) + out_2 = output.images + + assert out_1.shape == (1, 64, 64, 3) + assert np.abs(out_1.flatten() - out_2.flatten()).max() < 5e-2 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + +@nightly +@require_torch_gpu +class PaintByExamplePipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_paint_by_example(self): + # make sure here that pndm scheduler skips prk + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/paint_by_example/dog_in_bucket.png" + ) + mask_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/paint_by_example/mask.png" + ) + example_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/paint_by_example/panda.jpg" + ) + + pipe = PaintByExamplePipeline.from_pretrained("Fantasy-Studio/Paint-by-Example") + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(321) + output = pipe( + image=init_image, + mask_image=mask_image, + example_image=example_image, + generator=generator, + guidance_scale=5.0, + num_inference_steps=50, + output_type="np", + ) + + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.4834, 0.4811, 0.4874, 0.5122, 0.5081, 0.5144, 0.5291, 0.5290, 0.5374]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/pia/__init__.py b/diffusers-0.27.0/tests/pipelines/pia/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/pia/test_pia.py b/diffusers-0.27.0/tests/pipelines/pia/test_pia.py new file mode 100755 index 0000000..2813dc7 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/pia/test_pia.py @@ -0,0 +1,311 @@ +import random +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +import diffusers +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + MotionAdapter, + PIAPipeline, + UNet2DConditionModel, + UNetMotionModel, +) +from diffusers.utils import is_xformers_available, logging +from diffusers.utils.testing_utils import floats_tensor, torch_device + +from ..test_pipelines_common import IPAdapterTesterMixin, PipelineTesterMixin + + +def to_np(tensor): + if isinstance(tensor, torch.Tensor): + tensor = tensor.detach().cpu().numpy() + + return tensor + + +class PIAPipelineFastTests(IPAdapterTesterMixin, PipelineTesterMixin, unittest.TestCase): + pipeline_class = PIAPipeline + params = frozenset( + [ + "prompt", + "height", + "width", + "guidance_scale", + "negative_prompt", + "prompt_embeds", + "negative_prompt_embeds", + "cross_attention_kwargs", + ] + ) + batch_params = frozenset(["prompt", "image", "generator"]) + required_optional_params = frozenset( + [ + "num_inference_steps", + "generator", + "latents", + "return_dict", + "callback_on_step_end", + "callback_on_step_end_tensor_inputs", + ] + ) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("CrossAttnDownBlock2D", "DownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + norm_num_groups=2, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="linear", + clip_sample=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + motion_adapter = MotionAdapter( + block_out_channels=(32, 64), + motion_layers_per_block=2, + motion_norm_num_groups=2, + motion_num_attention_heads=4, + conv_in_channels=9, + ) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "motion_adapter": motion_adapter, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + inputs = { + "image": image, + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 7.5, + "output_type": "pt", + } + return inputs + + def test_motion_unet_loading(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + + assert isinstance(pipe.unet, UNetMotionModel) + + @unittest.skip("Attention slicing is not enabled in this pipeline") + def test_attention_slicing_forward_pass(self): + pass + + def test_inference_batch_single_identical( + self, + batch_size=2, + expected_max_diff=1e-4, + additional_params_copy_to_batched_inputs=["num_inference_steps"], + ): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for components in pipe.components.values(): + if hasattr(components, "set_default_attn_processor"): + components.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + inputs = self.get_dummy_inputs(torch_device) + # Reset generator in case it is has been used in self.get_dummy_inputs + inputs["generator"] = self.get_generator(0) + + logger = logging.get_logger(pipe.__module__) + logger.setLevel(level=diffusers.logging.FATAL) + + # batchify inputs + batched_inputs = {} + batched_inputs.update(inputs) + + for name in self.batch_params: + if name not in inputs: + continue + + value = inputs[name] + if name == "prompt": + len_prompt = len(value) + batched_inputs[name] = [value[: len_prompt // i] for i in range(1, batch_size + 1)] + batched_inputs[name][-1] = 100 * "very long" + + else: + batched_inputs[name] = batch_size * [value] + + if "generator" in inputs: + batched_inputs["generator"] = [self.get_generator(i) for i in range(batch_size)] + + if "batch_size" in inputs: + batched_inputs["batch_size"] = batch_size + + for arg in additional_params_copy_to_batched_inputs: + batched_inputs[arg] = inputs[arg] + + output = pipe(**inputs) + output_batch = pipe(**batched_inputs) + + assert output_batch[0].shape[0] == batch_size + + max_diff = np.abs(to_np(output_batch[0][0]) - to_np(output[0][0])).max() + assert max_diff < expected_max_diff + + @unittest.skipIf(torch_device != "cuda", reason="CUDA and CPU are required to switch devices") + def test_to_device(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + pipe.to("cpu") + # pipeline creates a new motion UNet under the hood. So we need to check the device from pipe.components + model_devices = [ + component.device.type for component in pipe.components.values() if hasattr(component, "device") + ] + self.assertTrue(all(device == "cpu" for device in model_devices)) + + output_cpu = pipe(**self.get_dummy_inputs("cpu"))[0] + self.assertTrue(np.isnan(output_cpu).sum() == 0) + + pipe.to("cuda") + model_devices = [ + component.device.type for component in pipe.components.values() if hasattr(component, "device") + ] + self.assertTrue(all(device == "cuda" for device in model_devices)) + + output_cuda = pipe(**self.get_dummy_inputs("cuda"))[0] + self.assertTrue(np.isnan(to_np(output_cuda)).sum() == 0) + + def test_to_dtype(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + # pipeline creates a new motion UNet under the hood. So we need to check the dtype from pipe.components + model_dtypes = [component.dtype for component in pipe.components.values() if hasattr(component, "dtype")] + self.assertTrue(all(dtype == torch.float32 for dtype in model_dtypes)) + + pipe.to(dtype=torch.float16) + model_dtypes = [component.dtype for component in pipe.components.values() if hasattr(component, "dtype")] + self.assertTrue(all(dtype == torch.float16 for dtype in model_dtypes)) + + def test_prompt_embeds(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + inputs = self.get_dummy_inputs(torch_device) + inputs.pop("prompt") + inputs["prompt_embeds"] = torch.randn((1, 4, 32), device=torch_device) + pipe(**inputs) + + def test_free_init(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + pipe.to(torch_device) + + inputs_normal = self.get_dummy_inputs(torch_device) + frames_normal = pipe(**inputs_normal).frames[0] + + pipe.enable_free_init( + num_iters=2, + use_fast_sampling=True, + method="butterworth", + order=4, + spatial_stop_frequency=0.25, + temporal_stop_frequency=0.25, + ) + inputs_enable_free_init = self.get_dummy_inputs(torch_device) + frames_enable_free_init = pipe(**inputs_enable_free_init).frames[0] + + pipe.disable_free_init() + inputs_disable_free_init = self.get_dummy_inputs(torch_device) + frames_disable_free_init = pipe(**inputs_disable_free_init).frames[0] + + sum_enabled = np.abs(to_np(frames_normal) - to_np(frames_enable_free_init)).sum() + max_diff_disabled = np.abs(to_np(frames_normal) - to_np(frames_disable_free_init)).max() + self.assertGreater( + sum_enabled, 1e1, "Enabling of FreeInit should lead to results different from the default pipeline results" + ) + self.assertLess( + max_diff_disabled, + 1e-4, + "Disabling of FreeInit should lead to results similar to the default pipeline results", + ) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output_without_offload = pipe(**inputs).frames[0] + output_without_offload = ( + output_without_offload.cpu() if torch.is_tensor(output_without_offload) else output_without_offload + ) + + pipe.enable_xformers_memory_efficient_attention() + inputs = self.get_dummy_inputs(torch_device) + output_with_offload = pipe(**inputs).frames[0] + output_with_offload = ( + output_with_offload.cpu() if torch.is_tensor(output_with_offload) else output_without_offload + ) + + max_diff = np.abs(to_np(output_with_offload) - to_np(output_without_offload)).max() + self.assertLess(max_diff, 1e-4, "XFormers attention should not affect the inference results") diff --git a/diffusers-0.27.0/tests/pipelines/pipeline_params.py b/diffusers-0.27.0/tests/pipelines/pipeline_params.py new file mode 100755 index 0000000..4e2c4dc --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/pipeline_params.py @@ -0,0 +1,129 @@ +# These are canonical sets of parameters for different types of pipelines. +# They are set on subclasses of `PipelineTesterMixin` as `params` and +# `batch_params`. +# +# If a pipeline's set of arguments has minor changes from one of the common sets +# of arguments, do not make modifications to the existing common sets of arguments. +# I.e. a text to image pipeline with non-configurable height and width arguments +# should set its attribute as `params = TEXT_TO_IMAGE_PARAMS - {'height', 'width'}`. + +TEXT_TO_IMAGE_PARAMS = frozenset( + [ + "prompt", + "height", + "width", + "guidance_scale", + "negative_prompt", + "prompt_embeds", + "negative_prompt_embeds", + "cross_attention_kwargs", + ] +) + +TEXT_TO_IMAGE_BATCH_PARAMS = frozenset(["prompt", "negative_prompt"]) + +TEXT_TO_IMAGE_IMAGE_PARAMS = frozenset([]) + +IMAGE_TO_IMAGE_IMAGE_PARAMS = frozenset(["image"]) + +IMAGE_VARIATION_PARAMS = frozenset( + [ + "image", + "height", + "width", + "guidance_scale", + ] +) + +IMAGE_VARIATION_BATCH_PARAMS = frozenset(["image"]) + +TEXT_GUIDED_IMAGE_VARIATION_PARAMS = frozenset( + [ + "prompt", + "image", + "height", + "width", + "guidance_scale", + "negative_prompt", + "prompt_embeds", + "negative_prompt_embeds", + ] +) + +TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS = frozenset(["prompt", "image", "negative_prompt"]) + +TEXT_GUIDED_IMAGE_INPAINTING_PARAMS = frozenset( + [ + # Text guided image variation with an image mask + "prompt", + "image", + "mask_image", + "height", + "width", + "guidance_scale", + "negative_prompt", + "prompt_embeds", + "negative_prompt_embeds", + ] +) + +TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS = frozenset(["prompt", "image", "mask_image", "negative_prompt"]) + +IMAGE_INPAINTING_PARAMS = frozenset( + [ + # image variation with an image mask + "image", + "mask_image", + "height", + "width", + "guidance_scale", + ] +) + +IMAGE_INPAINTING_BATCH_PARAMS = frozenset(["image", "mask_image"]) + +IMAGE_GUIDED_IMAGE_INPAINTING_PARAMS = frozenset( + [ + "example_image", + "image", + "mask_image", + "height", + "width", + "guidance_scale", + ] +) + +IMAGE_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS = frozenset(["example_image", "image", "mask_image"]) + +CLASS_CONDITIONED_IMAGE_GENERATION_PARAMS = frozenset(["class_labels"]) + +CLASS_CONDITIONED_IMAGE_GENERATION_BATCH_PARAMS = frozenset(["class_labels"]) + +UNCONDITIONAL_IMAGE_GENERATION_PARAMS = frozenset(["batch_size"]) + +UNCONDITIONAL_IMAGE_GENERATION_BATCH_PARAMS = frozenset([]) + +UNCONDITIONAL_AUDIO_GENERATION_PARAMS = frozenset(["batch_size"]) + +UNCONDITIONAL_AUDIO_GENERATION_BATCH_PARAMS = frozenset([]) + +TEXT_TO_AUDIO_PARAMS = frozenset( + [ + "prompt", + "audio_length_in_s", + "guidance_scale", + "negative_prompt", + "prompt_embeds", + "negative_prompt_embeds", + "cross_attention_kwargs", + ] +) + +TEXT_TO_AUDIO_BATCH_PARAMS = frozenset(["prompt", "negative_prompt"]) +TOKENS_TO_AUDIO_GENERATION_PARAMS = frozenset(["input_tokens"]) + +TOKENS_TO_AUDIO_GENERATION_BATCH_PARAMS = frozenset(["input_tokens"]) + +TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS = frozenset(["prompt_embeds"]) + +VIDEO_TO_VIDEO_BATCH_PARAMS = frozenset(["prompt", "negative_prompt", "video"]) diff --git a/diffusers-0.27.0/tests/pipelines/pixart_alpha/__init__.py b/diffusers-0.27.0/tests/pipelines/pixart_alpha/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/pixart_alpha/test_pixart.py b/diffusers-0.27.0/tests/pipelines/pixart_alpha/test_pixart.py new file mode 100755 index 0000000..3d6db5c --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/pixart_alpha/test_pixart.py @@ -0,0 +1,437 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import tempfile +import unittest + +import numpy as np +import torch +from transformers import AutoTokenizer, T5EncoderModel + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + PixArtAlphaPipeline, + Transformer2DModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import PipelineTesterMixin, to_np + + +enable_full_determinism() + + +class PixArtAlphaPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = PixArtAlphaPipeline + params = TEXT_TO_IMAGE_PARAMS - {"cross_attention_kwargs"} + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + required_optional_params = PipelineTesterMixin.required_optional_params + + def get_dummy_components(self): + torch.manual_seed(0) + transformer = Transformer2DModel( + sample_size=8, + num_layers=2, + patch_size=2, + attention_head_dim=8, + num_attention_heads=3, + caption_channels=32, + in_channels=4, + cross_attention_dim=24, + out_channels=8, + attention_bias=True, + activation_fn="gelu-approximate", + num_embeds_ada_norm=1000, + norm_type="ada_norm_single", + norm_elementwise_affine=False, + norm_eps=1e-6, + ) + torch.manual_seed(0) + vae = AutoencoderKL() + + scheduler = DDIMScheduler() + text_encoder = T5EncoderModel.from_pretrained("hf-internal-testing/tiny-random-t5") + + tokenizer = AutoTokenizer.from_pretrained("hf-internal-testing/tiny-random-t5") + + components = { + "transformer": transformer.eval(), + "vae": vae.eval(), + "scheduler": scheduler, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 5.0, + "use_resolution_binning": False, + "output_type": "np", + } + return inputs + + def test_sequential_cpu_offload_forward_pass(self): + # TODO(PVP, Sayak) need to fix later + return + + def test_save_load_optional_components(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + prompt = inputs["prompt"] + generator = inputs["generator"] + num_inference_steps = inputs["num_inference_steps"] + output_type = inputs["output_type"] + + ( + prompt_embeds, + prompt_attention_mask, + negative_prompt_embeds, + negative_prompt_attention_mask, + ) = pipe.encode_prompt(prompt) + + # inputs with prompt converted to embeddings + inputs = { + "prompt_embeds": prompt_embeds, + "prompt_attention_mask": prompt_attention_mask, + "negative_prompt": None, + "negative_prompt_embeds": negative_prompt_embeds, + "negative_prompt_attention_mask": negative_prompt_attention_mask, + "generator": generator, + "num_inference_steps": num_inference_steps, + "output_type": output_type, + "use_resolution_binning": False, + } + + # set all optional components to None + for optional_component in pipe._optional_components: + setattr(pipe, optional_component, None) + + output = pipe(**inputs)[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir) + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + for optional_component in pipe._optional_components: + self.assertTrue( + getattr(pipe_loaded, optional_component) is None, + f"`{optional_component}` did not stay set to None after loading.", + ) + + inputs = self.get_dummy_inputs(torch_device) + + generator = inputs["generator"] + num_inference_steps = inputs["num_inference_steps"] + output_type = inputs["output_type"] + + # inputs with prompt converted to embeddings + inputs = { + "prompt_embeds": prompt_embeds, + "prompt_attention_mask": prompt_attention_mask, + "negative_prompt": None, + "negative_prompt_embeds": negative_prompt_embeds, + "negative_prompt_attention_mask": negative_prompt_attention_mask, + "generator": generator, + "num_inference_steps": num_inference_steps, + "output_type": output_type, + "use_resolution_binning": False, + } + + output_loaded = pipe_loaded(**inputs)[0] + + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess(max_diff, 1e-4) + + def test_inference(self): + device = "cpu" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + self.assertEqual(image.shape, (1, 8, 8, 3)) + expected_slice = np.array([0.6319, 0.3526, 0.3806, 0.6327, 0.4639, 0.483, 0.2583, 0.5331, 0.4852]) + max_diff = np.abs(image_slice.flatten() - expected_slice).max() + self.assertLessEqual(max_diff, 1e-3) + + def test_inference_non_square_images(self): + device = "cpu" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = pipe(**inputs, height=32, width=48).images + image_slice = image[0, -3:, -3:, -1] + self.assertEqual(image.shape, (1, 32, 48, 3)) + + expected_slice = np.array([0.6493, 0.537, 0.4081, 0.4762, 0.3695, 0.4711, 0.3026, 0.5218, 0.5263]) + max_diff = np.abs(image_slice.flatten() - expected_slice).max() + self.assertLessEqual(max_diff, 1e-3) + + def test_inference_with_embeddings_and_multiple_images(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + prompt = inputs["prompt"] + generator = inputs["generator"] + num_inference_steps = inputs["num_inference_steps"] + output_type = inputs["output_type"] + + prompt_embeds, prompt_attn_mask, negative_prompt_embeds, neg_prompt_attn_mask = pipe.encode_prompt(prompt) + + # inputs with prompt converted to embeddings + inputs = { + "prompt_embeds": prompt_embeds, + "prompt_attention_mask": prompt_attn_mask, + "negative_prompt": None, + "negative_prompt_embeds": negative_prompt_embeds, + "negative_prompt_attention_mask": neg_prompt_attn_mask, + "generator": generator, + "num_inference_steps": num_inference_steps, + "output_type": output_type, + "num_images_per_prompt": 2, + "use_resolution_binning": False, + } + + # set all optional components to None + for optional_component in pipe._optional_components: + setattr(pipe, optional_component, None) + + output = pipe(**inputs)[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir) + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + for optional_component in pipe._optional_components: + self.assertTrue( + getattr(pipe_loaded, optional_component) is None, + f"`{optional_component}` did not stay set to None after loading.", + ) + + inputs = self.get_dummy_inputs(torch_device) + + generator = inputs["generator"] + num_inference_steps = inputs["num_inference_steps"] + output_type = inputs["output_type"] + + # inputs with prompt converted to embeddings + inputs = { + "prompt_embeds": prompt_embeds, + "prompt_attention_mask": prompt_attn_mask, + "negative_prompt": None, + "negative_prompt_embeds": negative_prompt_embeds, + "negative_prompt_attention_mask": neg_prompt_attn_mask, + "generator": generator, + "num_inference_steps": num_inference_steps, + "output_type": output_type, + "num_images_per_prompt": 2, + "use_resolution_binning": False, + } + + output_loaded = pipe_loaded(**inputs)[0] + + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess(max_diff, 1e-4) + + def test_inference_with_multiple_images_per_prompt(self): + device = "cpu" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["num_images_per_prompt"] = 2 + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + self.assertEqual(image.shape, (2, 8, 8, 3)) + expected_slice = np.array([0.6319, 0.3526, 0.3806, 0.6327, 0.4639, 0.483, 0.2583, 0.5331, 0.4852]) + max_diff = np.abs(image_slice.flatten() - expected_slice).max() + self.assertLessEqual(max_diff, 1e-3) + + def test_raises_warning_for_mask_feature(self): + device = "cpu" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs.update({"mask_feature": True}) + + with self.assertWarns(FutureWarning) as warning_ctx: + _ = pipe(**inputs).images + + assert "mask_feature" in str(warning_ctx.warning) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=1e-3) + + +@slow +@require_torch_gpu +class PixArtAlphaPipelineIntegrationTests(unittest.TestCase): + ckpt_id_1024 = "PixArt-alpha/PixArt-XL-2-1024-MS" + ckpt_id_512 = "PixArt-alpha/PixArt-XL-2-512x512" + prompt = "A small cactus with a happy face in the Sahara desert." + + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_pixart_1024(self): + generator = torch.Generator("cpu").manual_seed(0) + + pipe = PixArtAlphaPipeline.from_pretrained(self.ckpt_id_1024, torch_dtype=torch.float16) + pipe.enable_model_cpu_offload() + prompt = self.prompt + + image = pipe(prompt, generator=generator, num_inference_steps=2, output_type="np").images + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.0742, 0.0835, 0.2114, 0.0295, 0.0784, 0.2361, 0.1738, 0.2251, 0.3589]) + + max_diff = numpy_cosine_similarity_distance(image_slice.flatten(), expected_slice) + self.assertLessEqual(max_diff, 1e-4) + + def test_pixart_512(self): + generator = torch.Generator("cpu").manual_seed(0) + + pipe = PixArtAlphaPipeline.from_pretrained(self.ckpt_id_512, torch_dtype=torch.float16) + pipe.enable_model_cpu_offload() + + prompt = self.prompt + + image = pipe(prompt, generator=generator, num_inference_steps=2, output_type="np").images + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.3477, 0.3882, 0.4541, 0.3413, 0.3821, 0.4463, 0.4001, 0.4409, 0.4958]) + + max_diff = numpy_cosine_similarity_distance(image_slice.flatten(), expected_slice) + self.assertLessEqual(max_diff, 1e-4) + + def test_pixart_1024_without_resolution_binning(self): + generator = torch.manual_seed(0) + + pipe = PixArtAlphaPipeline.from_pretrained(self.ckpt_id_1024, torch_dtype=torch.float16) + pipe.enable_model_cpu_offload() + + prompt = self.prompt + height, width = 1024, 768 + num_inference_steps = 2 + + image = pipe( + prompt, + height=height, + width=width, + generator=generator, + num_inference_steps=num_inference_steps, + output_type="np", + ).images + image_slice = image[0, -3:, -3:, -1] + + generator = torch.manual_seed(0) + no_res_bin_image = pipe( + prompt, + height=height, + width=width, + generator=generator, + num_inference_steps=num_inference_steps, + output_type="np", + use_resolution_binning=False, + ).images + no_res_bin_image_slice = no_res_bin_image[0, -3:, -3:, -1] + + assert not np.allclose(image_slice, no_res_bin_image_slice, atol=1e-4, rtol=1e-4) + + def test_pixart_512_without_resolution_binning(self): + generator = torch.manual_seed(0) + + pipe = PixArtAlphaPipeline.from_pretrained(self.ckpt_id_512, torch_dtype=torch.float16) + pipe.enable_model_cpu_offload() + + prompt = self.prompt + height, width = 512, 768 + num_inference_steps = 2 + + image = pipe( + prompt, + height=height, + width=width, + generator=generator, + num_inference_steps=num_inference_steps, + output_type="np", + ).images + image_slice = image[0, -3:, -3:, -1] + + generator = torch.manual_seed(0) + no_res_bin_image = pipe( + prompt, + height=height, + width=width, + generator=generator, + num_inference_steps=num_inference_steps, + output_type="np", + use_resolution_binning=False, + ).images + no_res_bin_image_slice = no_res_bin_image[0, -3:, -3:, -1] + + assert not np.allclose(image_slice, no_res_bin_image_slice, atol=1e-4, rtol=1e-4) diff --git a/diffusers-0.27.0/tests/pipelines/pndm/__init__.py b/diffusers-0.27.0/tests/pipelines/pndm/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/pndm/test_pndm.py b/diffusers-0.27.0/tests/pipelines/pndm/test_pndm.py new file mode 100755 index 0000000..d4cb6a5 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/pndm/test_pndm.py @@ -0,0 +1,87 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch + +from diffusers import PNDMPipeline, PNDMScheduler, UNet2DModel +from diffusers.utils.testing_utils import enable_full_determinism, nightly, require_torch, torch_device + + +enable_full_determinism() + + +class PNDMPipelineFastTests(unittest.TestCase): + @property + def dummy_uncond_unet(self): + torch.manual_seed(0) + model = UNet2DModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "AttnDownBlock2D"), + up_block_types=("AttnUpBlock2D", "UpBlock2D"), + ) + return model + + def test_inference(self): + unet = self.dummy_uncond_unet + scheduler = PNDMScheduler() + + pndm = PNDMPipeline(unet=unet, scheduler=scheduler) + pndm.to(torch_device) + pndm.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(0) + image = pndm(generator=generator, num_inference_steps=20, output_type="numpy").images + + generator = torch.manual_seed(0) + image_from_tuple = pndm(generator=generator, num_inference_steps=20, output_type="numpy", return_dict=False)[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([1.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 0.0, 0.0]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + +@nightly +@require_torch +class PNDMPipelineIntegrationTests(unittest.TestCase): + def test_inference_cifar10(self): + model_id = "google/ddpm-cifar10-32" + + unet = UNet2DModel.from_pretrained(model_id) + scheduler = PNDMScheduler() + + pndm = PNDMPipeline(unet=unet, scheduler=scheduler) + pndm.to(torch_device) + pndm.set_progress_bar_config(disable=None) + generator = torch.manual_seed(0) + image = pndm(generator=generator, output_type="numpy").images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.1564, 0.14645, 0.1406, 0.14715, 0.12425, 0.14045, 0.13115, 0.12175, 0.125]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/semantic_stable_diffusion/__init__.py b/diffusers-0.27.0/tests/pipelines/semantic_stable_diffusion/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/semantic_stable_diffusion/test_semantic_diffusion.py b/diffusers-0.27.0/tests/pipelines/semantic_stable_diffusion/test_semantic_diffusion.py new file mode 100755 index 0000000..0c9bb63 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/semantic_stable_diffusion/test_semantic_diffusion.py @@ -0,0 +1,606 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import tempfile +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import AutoencoderKL, DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, UNet2DConditionModel +from diffusers.pipelines.semantic_stable_diffusion import SemanticStableDiffusionPipeline as StableDiffusionPipeline +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + nightly, + require_torch_gpu, + torch_device, +) + + +enable_full_determinism() + + +class SafeDiffusionPipelineFastTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + @property + def dummy_image(self): + batch_size = 1 + num_channels = 3 + sizes = (32, 32) + + image = floats_tensor((batch_size, num_channels) + sizes, rng=random.Random(0)).to(torch_device) + return image + + @property + def dummy_cond_unet(self): + torch.manual_seed(0) + model = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + return model + + @property + def dummy_vae(self): + torch.manual_seed(0) + model = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + return model + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModel(config) + + @property + def dummy_extractor(self): + def extract(*args, **kwargs): + class Out: + def __init__(self): + self.pixel_values = torch.ones([0]) + + def to(self, device): + self.pixel_values.to(device) + return self + + return Out() + + return extract + + def test_semantic_diffusion_ddim(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unet = self.dummy_cond_unet + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + + generator = torch.Generator(device=device).manual_seed(0) + output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np") + image = output.images + + generator = torch.Generator(device=device).manual_seed(0) + image_from_tuple = sd_pipe( + [prompt], + generator=generator, + guidance_scale=6.0, + num_inference_steps=2, + output_type="np", + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5753, 0.6114, 0.5001, 0.5034, 0.5470, 0.4729, 0.4971, 0.4867, 0.4867]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + def test_semantic_diffusion_pndm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unet = self.dummy_cond_unet + scheduler = PNDMScheduler(skip_prk_steps=True) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + generator = torch.Generator(device=device).manual_seed(0) + output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np") + + image = output.images + + generator = torch.Generator(device=device).manual_seed(0) + image_from_tuple = sd_pipe( + [prompt], + generator=generator, + guidance_scale=6.0, + num_inference_steps=2, + output_type="np", + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5122, 0.5712, 0.4825, 0.5053, 0.5646, 0.4769, 0.5179, 0.4894, 0.4994]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + def test_semantic_diffusion_no_safety_checker(self): + pipe = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-lms-pipe", safety_checker=None + ) + assert isinstance(pipe, StableDiffusionPipeline) + assert isinstance(pipe.scheduler, LMSDiscreteScheduler) + assert pipe.safety_checker is None + + image = pipe("example prompt", num_inference_steps=2).images[0] + assert image is not None + + # check that there's no error when saving a pipeline with one of the models being None + with tempfile.TemporaryDirectory() as tmpdirname: + pipe.save_pretrained(tmpdirname) + pipe = StableDiffusionPipeline.from_pretrained(tmpdirname) + + # sanity check that the pipeline still works + assert pipe.safety_checker is None + image = pipe("example prompt", num_inference_steps=2).images[0] + assert image is not None + + @unittest.skipIf(torch_device != "cuda", "This test requires a GPU") + def test_semantic_diffusion_fp16(self): + """Test that stable diffusion works with fp16""" + unet = self.dummy_cond_unet + scheduler = PNDMScheduler(skip_prk_steps=True) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + # put models in fp16 + unet = unet.half() + vae = vae.half() + bert = bert.half() + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + image = sd_pipe([prompt], num_inference_steps=2, output_type="np").images + + assert image.shape == (1, 64, 64, 3) + + +@nightly +@require_torch_gpu +class SemanticDiffusionPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_positive_guidance(self): + torch_device = "cuda" + pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + prompt = "a photo of a cat" + edit = { + "editing_prompt": ["sunglasses"], + "reverse_editing_direction": [False], + "edit_warmup_steps": 10, + "edit_guidance_scale": 6, + "edit_threshold": 0.95, + "edit_momentum_scale": 0.5, + "edit_mom_beta": 0.6, + } + + seed = 3 + guidance_scale = 7 + + # no sega enabled + generator = torch.Generator(torch_device) + generator.manual_seed(seed) + output = pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [ + 0.34673113, + 0.38492733, + 0.37597352, + 0.34086335, + 0.35650748, + 0.35579205, + 0.3384763, + 0.34340236, + 0.3573271, + ] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + # with sega enabled + # generator = torch.manual_seed(seed) + generator.manual_seed(seed) + output = pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + **edit, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [ + 0.41887826, + 0.37728766, + 0.30138272, + 0.41416335, + 0.41664985, + 0.36283392, + 0.36191246, + 0.43364465, + 0.43001732, + ] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_negative_guidance(self): + torch_device = "cuda" + pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + prompt = "an image of a crowded boulevard, realistic, 4k" + edit = { + "editing_prompt": "crowd, crowded, people", + "reverse_editing_direction": True, + "edit_warmup_steps": 10, + "edit_guidance_scale": 8.3, + "edit_threshold": 0.9, + "edit_momentum_scale": 0.5, + "edit_mom_beta": 0.6, + } + + seed = 9 + guidance_scale = 7 + + # no sega enabled + generator = torch.Generator(torch_device) + generator.manual_seed(seed) + output = pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [ + 0.43497998, + 0.91814065, + 0.7540739, + 0.55580205, + 0.8467265, + 0.5389691, + 0.62574506, + 0.58897763, + 0.50926757, + ] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + # with sega enabled + # generator = torch.manual_seed(seed) + generator.manual_seed(seed) + output = pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + **edit, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [ + 0.3089719, + 0.30500144, + 0.29016042, + 0.30630964, + 0.325687, + 0.29419225, + 0.2908091, + 0.28723598, + 0.27696294, + ] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_multi_cond_guidance(self): + torch_device = "cuda" + pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + prompt = "a castle next to a river" + edit = { + "editing_prompt": ["boat on a river, boat", "monet, impression, sunrise"], + "reverse_editing_direction": False, + "edit_warmup_steps": [15, 18], + "edit_guidance_scale": 6, + "edit_threshold": [0.9, 0.8], + "edit_momentum_scale": 0.5, + "edit_mom_beta": 0.6, + } + + seed = 48 + guidance_scale = 7 + + # no sega enabled + generator = torch.Generator(torch_device) + generator.manual_seed(seed) + output = pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [ + 0.75163555, + 0.76037145, + 0.61785, + 0.9189673, + 0.8627701, + 0.85189694, + 0.8512813, + 0.87012076, + 0.8312857, + ] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + # with sega enabled + # generator = torch.manual_seed(seed) + generator.manual_seed(seed) + output = pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + **edit, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [ + 0.73553365, + 0.7537271, + 0.74341905, + 0.66480356, + 0.6472925, + 0.63039416, + 0.64812905, + 0.6749717, + 0.6517102, + ] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_guidance_fp16(self): + torch_device = "cuda" + pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", torch_dtype=torch.float16) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + prompt = "a photo of a cat" + edit = { + "editing_prompt": ["sunglasses"], + "reverse_editing_direction": [False], + "edit_warmup_steps": 10, + "edit_guidance_scale": 6, + "edit_threshold": 0.95, + "edit_momentum_scale": 0.5, + "edit_mom_beta": 0.6, + } + + seed = 3 + guidance_scale = 7 + + # no sega enabled + generator = torch.Generator(torch_device) + generator.manual_seed(seed) + output = pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [ + 0.34887695, + 0.3876953, + 0.375, + 0.34423828, + 0.3581543, + 0.35717773, + 0.3383789, + 0.34570312, + 0.359375, + ] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + # with sega enabled + # generator = torch.manual_seed(seed) + generator.manual_seed(seed) + output = pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + **edit, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [ + 0.42285156, + 0.36914062, + 0.29077148, + 0.42041016, + 0.41918945, + 0.35498047, + 0.3618164, + 0.4423828, + 0.43115234, + ] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/shap_e/__init__.py b/diffusers-0.27.0/tests/pipelines/shap_e/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/shap_e/test_shap_e.py b/diffusers-0.27.0/tests/pipelines/shap_e/test_shap_e.py new file mode 100755 index 0000000..11595fe --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/shap_e/test_shap_e.py @@ -0,0 +1,255 @@ +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import HeunDiscreteScheduler, PriorTransformer, ShapEPipeline +from diffusers.pipelines.shap_e import ShapERenderer +from diffusers.utils.testing_utils import load_numpy, nightly, require_torch_gpu, torch_device + +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +class ShapEPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = ShapEPipeline + params = ["prompt"] + batch_params = ["prompt"] + required_optional_params = [ + "num_images_per_prompt", + "num_inference_steps", + "generator", + "latents", + "guidance_scale", + "frame_size", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + @property + def text_embedder_hidden_size(self): + return 16 + + @property + def time_input_dim(self): + return 16 + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def renderer_dim(self): + return 8 + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=self.text_embedder_hidden_size, + projection_dim=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModelWithProjection(config) + + @property + def dummy_prior(self): + torch.manual_seed(0) + + model_kwargs = { + "num_attention_heads": 2, + "attention_head_dim": 16, + "embedding_dim": self.time_input_dim, + "num_embeddings": 32, + "embedding_proj_dim": self.text_embedder_hidden_size, + "time_embed_dim": self.time_embed_dim, + "num_layers": 1, + "clip_embed_dim": self.time_input_dim * 2, + "additional_embeddings": 0, + "time_embed_act_fn": "gelu", + "norm_in_type": "layer", + "encoder_hid_proj_type": None, + "added_emb_type": None, + } + + model = PriorTransformer(**model_kwargs) + return model + + @property + def dummy_renderer(self): + torch.manual_seed(0) + + model_kwargs = { + "param_shapes": ( + (self.renderer_dim, 93), + (self.renderer_dim, 8), + (self.renderer_dim, 8), + (self.renderer_dim, 8), + ), + "d_latent": self.time_input_dim, + "d_hidden": self.renderer_dim, + "n_output": 12, + "background": ( + 0.1, + 0.1, + 0.1, + ), + } + model = ShapERenderer(**model_kwargs) + return model + + def get_dummy_components(self): + prior = self.dummy_prior + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + shap_e_renderer = self.dummy_renderer + + scheduler = HeunDiscreteScheduler( + beta_schedule="exp", + num_train_timesteps=1024, + prediction_type="sample", + use_karras_sigmas=True, + clip_sample=True, + clip_sample_range=1.0, + ) + components = { + "prior": prior, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "shap_e_renderer": shap_e_renderer, + "scheduler": scheduler, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "horse", + "generator": generator, + "num_inference_steps": 1, + "frame_size": 32, + "output_type": "latent", + } + return inputs + + def test_shap_e(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images[0] + image = image.cpu().numpy() + image_slice = image[-3:, -3:] + + assert image.shape == (32, 16) + + expected_slice = np.array([-1.0000, -0.6241, 1.0000, -0.8978, -0.6866, 0.7876, -0.7473, -0.2874, 0.6103]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_inference_batch_consistent(self): + # NOTE: Larger batch sizes cause this test to timeout, only test on smaller batches + self._test_inference_batch_consistent(batch_sizes=[1, 2]) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(batch_size=2, expected_max_diff=6e-3) + + def test_num_images_per_prompt(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + batch_size = 1 + num_images_per_prompt = 2 + + inputs = self.get_dummy_inputs(torch_device) + + for key in inputs.keys(): + if key in self.batch_params: + inputs[key] = batch_size * [inputs[key]] + + images = pipe(**inputs, num_images_per_prompt=num_images_per_prompt)[0] + + assert images.shape[0] == batch_size * num_images_per_prompt + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=5e-1) + + def test_save_load_local(self): + super().test_save_load_local(expected_max_difference=5e-3) + + @unittest.skip("Key error is raised with accelerate") + def test_sequential_cpu_offload_forward_pass(self): + pass + + +@nightly +@require_torch_gpu +class ShapEPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_shap_e(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/shap_e/test_shap_e_np_out.npy" + ) + pipe = ShapEPipeline.from_pretrained("openai/shap-e") + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device=torch_device).manual_seed(0) + + images = pipe( + "a shark", + generator=generator, + guidance_scale=15.0, + num_inference_steps=64, + frame_size=64, + output_type="np", + ).images[0] + + assert images.shape == (20, 64, 64, 3) + + assert_mean_pixel_difference(images, expected_image) diff --git a/diffusers-0.27.0/tests/pipelines/shap_e/test_shap_e_img2img.py b/diffusers-0.27.0/tests/pipelines/shap_e/test_shap_e_img2img.py new file mode 100755 index 0000000..c666b01 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/shap_e/test_shap_e_img2img.py @@ -0,0 +1,284 @@ +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from transformers import CLIPImageProcessor, CLIPVisionConfig, CLIPVisionModel + +from diffusers import HeunDiscreteScheduler, PriorTransformer, ShapEImg2ImgPipeline +from diffusers.pipelines.shap_e import ShapERenderer +from diffusers.utils.testing_utils import ( + floats_tensor, + load_image, + load_numpy, + nightly, + require_torch_gpu, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +class ShapEImg2ImgPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = ShapEImg2ImgPipeline + params = ["image"] + batch_params = ["image"] + required_optional_params = [ + "num_images_per_prompt", + "num_inference_steps", + "generator", + "latents", + "guidance_scale", + "frame_size", + "output_type", + "return_dict", + ] + test_xformers_attention = False + + @property + def text_embedder_hidden_size(self): + return 16 + + @property + def time_input_dim(self): + return 16 + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def renderer_dim(self): + return 8 + + @property + def dummy_image_encoder(self): + torch.manual_seed(0) + config = CLIPVisionConfig( + hidden_size=self.text_embedder_hidden_size, + image_size=32, + projection_dim=self.text_embedder_hidden_size, + intermediate_size=24, + num_attention_heads=2, + num_channels=3, + num_hidden_layers=5, + patch_size=1, + ) + + model = CLIPVisionModel(config) + return model + + @property + def dummy_image_processor(self): + image_processor = CLIPImageProcessor( + crop_size=224, + do_center_crop=True, + do_normalize=True, + do_resize=True, + image_mean=[0.48145466, 0.4578275, 0.40821073], + image_std=[0.26862954, 0.26130258, 0.27577711], + resample=3, + size=224, + ) + + return image_processor + + @property + def dummy_prior(self): + torch.manual_seed(0) + + model_kwargs = { + "num_attention_heads": 2, + "attention_head_dim": 16, + "embedding_dim": self.time_input_dim, + "num_embeddings": 32, + "embedding_proj_dim": self.text_embedder_hidden_size, + "time_embed_dim": self.time_embed_dim, + "num_layers": 1, + "clip_embed_dim": self.time_input_dim * 2, + "additional_embeddings": 0, + "time_embed_act_fn": "gelu", + "norm_in_type": "layer", + "embedding_proj_norm_type": "layer", + "encoder_hid_proj_type": None, + "added_emb_type": None, + } + + model = PriorTransformer(**model_kwargs) + return model + + @property + def dummy_renderer(self): + torch.manual_seed(0) + + model_kwargs = { + "param_shapes": ( + (self.renderer_dim, 93), + (self.renderer_dim, 8), + (self.renderer_dim, 8), + (self.renderer_dim, 8), + ), + "d_latent": self.time_input_dim, + "d_hidden": self.renderer_dim, + "n_output": 12, + "background": ( + 0.1, + 0.1, + 0.1, + ), + } + model = ShapERenderer(**model_kwargs) + return model + + def get_dummy_components(self): + prior = self.dummy_prior + image_encoder = self.dummy_image_encoder + image_processor = self.dummy_image_processor + shap_e_renderer = self.dummy_renderer + + scheduler = HeunDiscreteScheduler( + beta_schedule="exp", + num_train_timesteps=1024, + prediction_type="sample", + use_karras_sigmas=True, + clip_sample=True, + clip_sample_range=1.0, + ) + components = { + "prior": prior, + "image_encoder": image_encoder, + "image_processor": image_processor, + "shap_e_renderer": shap_e_renderer, + "scheduler": scheduler, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + input_image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "image": input_image, + "generator": generator, + "num_inference_steps": 1, + "frame_size": 32, + "output_type": "latent", + } + return inputs + + def test_shap_e(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images[0] + image_slice = image[-3:, -3:].cpu().numpy() + + assert image.shape == (32, 16) + + expected_slice = np.array( + [-1.0, 0.40668195, 0.57322013, -0.9469888, 0.4283227, 0.30348337, -0.81094897, 0.74555075, 0.15342723] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_inference_batch_consistent(self): + # NOTE: Larger batch sizes cause this test to timeout, only test on smaller batches + self._test_inference_batch_consistent(batch_sizes=[2]) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical( + batch_size=2, + expected_max_diff=6e-3, + ) + + def test_num_images_per_prompt(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + batch_size = 1 + num_images_per_prompt = 2 + + inputs = self.get_dummy_inputs(torch_device) + + for key in inputs.keys(): + if key in self.batch_params: + inputs[key] = batch_size * [inputs[key]] + + images = pipe(**inputs, num_images_per_prompt=num_images_per_prompt)[0] + + assert images.shape[0] == batch_size * num_images_per_prompt + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=1e-1) + + def test_save_load_local(self): + super().test_save_load_local(expected_max_difference=5e-3) + + @unittest.skip("Key error is raised with accelerate") + def test_sequential_cpu_offload_forward_pass(self): + pass + + +@nightly +@require_torch_gpu +class ShapEImg2ImgPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_shap_e_img2img(self): + input_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" "/shap_e/corgi.png" + ) + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/shap_e/test_shap_e_img2img_out.npy" + ) + pipe = ShapEImg2ImgPipeline.from_pretrained("openai/shap-e-img2img") + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device=torch_device).manual_seed(0) + + images = pipe( + input_image, + generator=generator, + guidance_scale=3.0, + num_inference_steps=64, + frame_size=64, + output_type="np", + ).images[0] + + assert images.shape == (20, 64, 64, 3) + + assert_mean_pixel_difference(images, expected_image) diff --git a/diffusers-0.27.0/tests/pipelines/stable_cascade/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_cascade/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_combined.py b/diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_combined.py new file mode 100755 index 0000000..4a9a123 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_combined.py @@ -0,0 +1,279 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import DDPMWuerstchenScheduler, StableCascadeCombinedPipeline +from diffusers.models import StableCascadeUNet +from diffusers.pipelines.wuerstchen import PaellaVQModel +from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, torch_device + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class StableCascadeCombinedPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = StableCascadeCombinedPipeline + params = ["prompt"] + batch_params = ["prompt", "negative_prompt"] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "prior_guidance_scale", + "decoder_guidance_scale", + "negative_prompt", + "num_inference_steps", + "return_dict", + "prior_num_inference_steps", + "output_type", + ] + test_xformers_attention = True + + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def dummy_prior(self): + torch.manual_seed(0) + + model_kwargs = { + "conditioning_dim": 128, + "block_out_channels": (128, 128), + "num_attention_heads": (2, 2), + "down_num_layers_per_block": (1, 1), + "up_num_layers_per_block": (1, 1), + "clip_image_in_channels": 768, + "switch_level": (False,), + "clip_text_in_channels": self.text_embedder_hidden_size, + "clip_text_pooled_in_channels": self.text_embedder_hidden_size, + } + + model = StableCascadeUNet(**model_kwargs) + return model.eval() + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + projection_dim=self.text_embedder_hidden_size, + hidden_size=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModelWithProjection(config).eval() + + @property + def dummy_vqgan(self): + torch.manual_seed(0) + + model_kwargs = { + "bottleneck_blocks": 1, + "num_vq_embeddings": 2, + } + model = PaellaVQModel(**model_kwargs) + return model.eval() + + @property + def dummy_decoder(self): + torch.manual_seed(0) + model_kwargs = { + "in_channels": 4, + "out_channels": 4, + "conditioning_dim": 128, + "block_out_channels": (16, 32, 64, 128), + "num_attention_heads": (-1, -1, 1, 2), + "down_num_layers_per_block": (1, 1, 1, 1), + "up_num_layers_per_block": (1, 1, 1, 1), + "down_blocks_repeat_mappers": (1, 1, 1, 1), + "up_blocks_repeat_mappers": (3, 3, 2, 2), + "block_types_per_layer": ( + ("SDCascadeResBlock", "SDCascadeTimestepBlock"), + ("SDCascadeResBlock", "SDCascadeTimestepBlock"), + ("SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"), + ("SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"), + ), + "switch_level": None, + "clip_text_pooled_in_channels": 32, + "dropout": (0.1, 0.1, 0.1, 0.1), + } + + model = StableCascadeUNet(**model_kwargs) + return model.eval() + + def get_dummy_components(self): + prior = self.dummy_prior + + scheduler = DDPMWuerstchenScheduler() + tokenizer = self.dummy_tokenizer + text_encoder = self.dummy_text_encoder + decoder = self.dummy_decoder + vqgan = self.dummy_vqgan + prior_text_encoder = self.dummy_text_encoder + prior_tokenizer = self.dummy_tokenizer + + components = { + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "decoder": decoder, + "scheduler": scheduler, + "vqgan": vqgan, + "prior_text_encoder": prior_text_encoder, + "prior_tokenizer": prior_tokenizer, + "prior_prior": prior, + "prior_scheduler": scheduler, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "horse", + "generator": generator, + "prior_guidance_scale": 4.0, + "decoder_guidance_scale": 4.0, + "num_inference_steps": 2, + "prior_num_inference_steps": 2, + "output_type": "np", + "height": 128, + "width": 128, + } + return inputs + + def test_stable_cascade(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe(**self.get_dummy_inputs(device), return_dict=False)[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[-3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + + expected_slice = np.array([0.0, 1.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 0.0]) + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + @require_torch_gpu + def test_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=2e-2) + + @unittest.skip(reason="fp16 not supported") + def test_float16_inference(self): + super().test_float16_inference() + + @unittest.skip(reason="no callback test for combined pipeline") + def test_callback_inputs(self): + super().test_callback_inputs() + + def test_stable_cascade_combined_prompt_embeds(self): + device = "cpu" + components = self.get_dummy_components() + + pipe = StableCascadeCombinedPipeline(**components) + pipe.set_progress_bar_config(disable=None) + + prompt = "A photograph of a shiba inu, wearing a hat" + ( + prompt_embeds, + prompt_embeds_pooled, + negative_prompt_embeds, + negative_prompt_embeds_pooled, + ) = pipe.prior_pipe.encode_prompt(device, 1, 1, False, prompt=prompt) + generator = torch.Generator(device=device) + + output_prompt = pipe( + prompt=prompt, + num_inference_steps=1, + prior_num_inference_steps=1, + output_type="np", + generator=generator.manual_seed(0), + ) + output_prompt_embeds = pipe( + prompt=None, + prompt_embeds=prompt_embeds, + prompt_embeds_pooled=prompt_embeds_pooled, + negative_prompt_embeds=negative_prompt_embeds, + negative_prompt_embeds_pooled=negative_prompt_embeds_pooled, + num_inference_steps=1, + prior_num_inference_steps=1, + output_type="np", + generator=generator.manual_seed(0), + ) + + assert np.abs(output_prompt.images - output_prompt_embeds.images).max() < 1e-5 diff --git a/diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_decoder.py b/diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_decoder.py new file mode 100755 index 0000000..3ad19db --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_decoder.py @@ -0,0 +1,286 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import DDPMWuerstchenScheduler, StableCascadeDecoderPipeline +from diffusers.models import StableCascadeUNet +from diffusers.pipelines.wuerstchen import PaellaVQModel +from diffusers.utils.testing_utils import ( + enable_full_determinism, + load_numpy, + load_pt, + numpy_cosine_similarity_distance, + require_torch_gpu, + skip_mps, + slow, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class StableCascadeDecoderPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = StableCascadeDecoderPipeline + params = ["prompt"] + batch_params = ["image_embeddings", "prompt", "negative_prompt"] + required_optional_params = [ + "num_images_per_prompt", + "num_inference_steps", + "latents", + "negative_prompt", + "guidance_scale", + "output_type", + "return_dict", + ] + test_xformers_attention = False + callback_cfg_params = ["image_embeddings", "text_encoder_hidden_states"] + + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + projection_dim=self.text_embedder_hidden_size, + hidden_size=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModelWithProjection(config).eval() + + @property + def dummy_vqgan(self): + torch.manual_seed(0) + + model_kwargs = { + "bottleneck_blocks": 1, + "num_vq_embeddings": 2, + } + model = PaellaVQModel(**model_kwargs) + return model.eval() + + @property + def dummy_decoder(self): + torch.manual_seed(0) + model_kwargs = { + "in_channels": 4, + "out_channels": 4, + "conditioning_dim": 128, + "block_out_channels": [16, 32, 64, 128], + "num_attention_heads": [-1, -1, 1, 2], + "down_num_layers_per_block": [1, 1, 1, 1], + "up_num_layers_per_block": [1, 1, 1, 1], + "down_blocks_repeat_mappers": [1, 1, 1, 1], + "up_blocks_repeat_mappers": [3, 3, 2, 2], + "block_types_per_layer": [ + ["SDCascadeResBlock", "SDCascadeTimestepBlock"], + ["SDCascadeResBlock", "SDCascadeTimestepBlock"], + ["SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"], + ["SDCascadeResBlock", "SDCascadeTimestepBlock", "SDCascadeAttnBlock"], + ], + "switch_level": None, + "clip_text_pooled_in_channels": 32, + "dropout": [0.1, 0.1, 0.1, 0.1], + } + model = StableCascadeUNet(**model_kwargs) + return model.eval() + + def get_dummy_components(self): + decoder = self.dummy_decoder + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + vqgan = self.dummy_vqgan + + scheduler = DDPMWuerstchenScheduler() + + components = { + "decoder": decoder, + "vqgan": vqgan, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "scheduler": scheduler, + "latent_dim_scale": 4.0, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "image_embeddings": torch.ones((1, 4, 4, 4), device=device), + "prompt": "horse", + "generator": generator, + "guidance_scale": 2.0, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + def test_wuerstchen_decoder(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe(**self.get_dummy_inputs(device), return_dict=False) + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.0, 0.0, 0.0, 1.0, 1.0, 0.0, 1.0, 1.0, 0.0]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + @skip_mps + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=1e-2) + + @skip_mps + def test_attention_slicing_forward_pass(self): + test_max_difference = torch_device == "cpu" + test_mean_pixel_difference = False + + self._test_attention_slicing_forward_pass( + test_max_difference=test_max_difference, + test_mean_pixel_difference=test_mean_pixel_difference, + ) + + @unittest.skip(reason="fp16 not supported") + def test_float16_inference(self): + super().test_float16_inference() + + def test_stable_cascade_decoder_prompt_embeds(self): + device = "cpu" + components = self.get_dummy_components() + + pipe = StableCascadeDecoderPipeline(**components) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image_embeddings = inputs["image_embeddings"] + prompt = "A photograph of a shiba inu, wearing a hat" + ( + prompt_embeds, + prompt_embeds_pooled, + negative_prompt_embeds, + negative_prompt_embeds_pooled, + ) = pipe.encode_prompt(device, 1, 1, False, prompt=prompt) + generator = torch.Generator(device=device) + + decoder_output_prompt = pipe( + image_embeddings=image_embeddings, + prompt=prompt, + num_inference_steps=1, + output_type="np", + generator=generator.manual_seed(0), + ) + decoder_output_prompt_embeds = pipe( + image_embeddings=image_embeddings, + prompt=None, + prompt_embeds=prompt_embeds, + prompt_embeds_pooled=prompt_embeds_pooled, + negative_prompt_embeds=negative_prompt_embeds, + negative_prompt_embeds_pooled=negative_prompt_embeds_pooled, + num_inference_steps=1, + output_type="np", + generator=generator.manual_seed(0), + ) + + assert np.abs(decoder_output_prompt.images - decoder_output_prompt_embeds.images).max() < 1e-5 + + +@slow +@require_torch_gpu +class StableCascadeDecoderPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_cascade_decoder(self): + pipe = StableCascadeDecoderPipeline.from_pretrained( + "stabilityai/stable-cascade", variant="bf16", torch_dtype=torch.bfloat16 + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + prompt = "A photograph of the inside of a subway train. There are raccoons sitting on the seats. One of them is reading a newspaper. The window shows the city in the background." + + generator = torch.Generator(device="cpu").manual_seed(0) + image_embedding = load_pt( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_cascade/image_embedding.pt" + ) + + image = pipe( + prompt=prompt, + image_embeddings=image_embedding, + output_type="np", + num_inference_steps=2, + generator=generator, + ).images[0] + + assert image.shape == (1024, 1024, 3) + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_cascade/stable_cascade_decoder_image.npy" + ) + max_diff = numpy_cosine_similarity_distance(image.flatten(), expected_image.flatten()) + assert max_diff < 1e-4 diff --git a/diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_prior.py b/diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_prior.py new file mode 100755 index 0000000..56e58ae --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_cascade/test_stable_cascade_prior.py @@ -0,0 +1,341 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import DDPMWuerstchenScheduler, StableCascadePriorPipeline +from diffusers.loaders import AttnProcsLayers +from diffusers.models import StableCascadeUNet +from diffusers.models.attention_processor import LoRAAttnProcessor, LoRAAttnProcessor2_0 +from diffusers.utils.import_utils import is_peft_available +from diffusers.utils.testing_utils import ( + enable_full_determinism, + load_numpy, + numpy_cosine_similarity_distance, + require_peft_backend, + require_torch_gpu, + skip_mps, + slow, + torch_device, +) + + +if is_peft_available(): + from peft import LoraConfig + from peft.tuners.tuners_utils import BaseTunerLayer + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +def create_prior_lora_layers(unet: nn.Module): + lora_attn_procs = {} + for name in unet.attn_processors.keys(): + lora_attn_processor_class = ( + LoRAAttnProcessor2_0 if hasattr(F, "scaled_dot_product_attention") else LoRAAttnProcessor + ) + lora_attn_procs[name] = lora_attn_processor_class( + hidden_size=unet.config.c, + ) + unet_lora_layers = AttnProcsLayers(lora_attn_procs) + return lora_attn_procs, unet_lora_layers + + +class StableCascadePriorPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = StableCascadePriorPipeline + params = ["prompt"] + batch_params = ["prompt", "negative_prompt"] + required_optional_params = [ + "num_images_per_prompt", + "generator", + "num_inference_steps", + "latents", + "negative_prompt", + "guidance_scale", + "output_type", + "return_dict", + ] + test_xformers_attention = False + callback_cfg_params = ["text_encoder_hidden_states"] + + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=self.text_embedder_hidden_size, + projection_dim=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModelWithProjection(config).eval() + + @property + def dummy_prior(self): + torch.manual_seed(0) + + model_kwargs = { + "conditioning_dim": 128, + "block_out_channels": (128, 128), + "num_attention_heads": (2, 2), + "down_num_layers_per_block": (1, 1), + "up_num_layers_per_block": (1, 1), + "switch_level": (False,), + "clip_image_in_channels": 768, + "clip_text_in_channels": self.text_embedder_hidden_size, + "clip_text_pooled_in_channels": self.text_embedder_hidden_size, + "dropout": (0.1, 0.1), + } + + model = StableCascadeUNet(**model_kwargs) + return model.eval() + + def get_dummy_components(self): + prior = self.dummy_prior + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + + scheduler = DDPMWuerstchenScheduler() + + components = { + "prior": prior, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "scheduler": scheduler, + "feature_extractor": None, + "image_encoder": None, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "horse", + "generator": generator, + "guidance_scale": 4.0, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + def test_wuerstchen_prior(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.image_embeddings + + image_from_tuple = pipe(**self.get_dummy_inputs(device), return_dict=False)[0] + + image_slice = image[0, 0, 0, -10:] + image_from_tuple_slice = image_from_tuple[0, 0, 0, -10:] + assert image.shape == (1, 16, 24, 24) + + expected_slice = np.array( + [ + 96.139565, + -20.213179, + -116.40341, + -191.57129, + 39.350136, + 74.80767, + 39.782352, + -184.67352, + -46.426907, + 168.41783, + ] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 5e-2 + + @skip_mps + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-1) + + @skip_mps + def test_attention_slicing_forward_pass(self): + test_max_difference = torch_device == "cpu" + test_mean_pixel_difference = False + + self._test_attention_slicing_forward_pass( + test_max_difference=test_max_difference, + test_mean_pixel_difference=test_mean_pixel_difference, + ) + + @unittest.skip(reason="fp16 not supported") + def test_float16_inference(self): + super().test_float16_inference() + + def check_if_lora_correctly_set(self, model) -> bool: + """ + Checks if the LoRA layers are correctly set with peft + """ + for module in model.modules(): + if isinstance(module, BaseTunerLayer): + return True + return False + + def get_lora_components(self): + prior = self.dummy_prior + + prior_lora_config = LoraConfig( + r=4, lora_alpha=4, target_modules=["to_q", "to_k", "to_v", "to_out.0"], init_lora_weights=False + ) + + prior_lora_attn_procs, prior_lora_layers = create_prior_lora_layers(prior) + + lora_components = { + "prior_lora_layers": prior_lora_layers, + "prior_lora_attn_procs": prior_lora_attn_procs, + } + + return prior, prior_lora_config, lora_components + + @require_peft_backend + @unittest.skip(reason="no lora support for now") + def test_inference_with_prior_lora(self): + _, prior_lora_config, _ = self.get_lora_components() + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output_no_lora = pipe(**self.get_dummy_inputs(device)) + image_embed = output_no_lora.image_embeddings + self.assertTrue(image_embed.shape == (1, 16, 24, 24)) + + pipe.prior.add_adapter(prior_lora_config) + self.assertTrue(self.check_if_lora_correctly_set(pipe.prior), "Lora not correctly set in prior") + + output_lora = pipe(**self.get_dummy_inputs(device)) + lora_image_embed = output_lora.image_embeddings + + self.assertTrue(image_embed.shape == lora_image_embed.shape) + + def test_stable_cascade_decoder_prompt_embeds(self): + device = "cpu" + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + prompt = "A photograph of a shiba inu, wearing a hat" + ( + prompt_embeds, + prompt_embeds_pooled, + negative_prompt_embeds, + negative_prompt_embeds_pooled, + ) = pipe.encode_prompt(device, 1, 1, False, prompt=prompt) + generator = torch.Generator(device=device) + + output_prompt = pipe( + prompt=prompt, + num_inference_steps=1, + output_type="np", + generator=generator.manual_seed(0), + ) + output_prompt_embeds = pipe( + prompt=None, + prompt_embeds=prompt_embeds, + prompt_embeds_pooled=prompt_embeds_pooled, + negative_prompt_embeds=negative_prompt_embeds, + negative_prompt_embeds_pooled=negative_prompt_embeds_pooled, + num_inference_steps=1, + output_type="np", + generator=generator.manual_seed(0), + ) + + assert np.abs(output_prompt.image_embeddings - output_prompt_embeds.image_embeddings).max() < 1e-5 + + +@slow +@require_torch_gpu +class StableCascadePriorPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_cascade_prior(self): + pipe = StableCascadePriorPipeline.from_pretrained( + "stabilityai/stable-cascade-prior", variant="bf16", torch_dtype=torch.bfloat16 + ) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + prompt = "A photograph of the inside of a subway train. There are raccoons sitting on the seats. One of them is reading a newspaper. The window shows the city in the background." + + generator = torch.Generator(device="cpu").manual_seed(0) + + output = pipe(prompt, num_inference_steps=2, output_type="np", generator=generator) + image_embedding = output.image_embeddings + expected_image_embedding = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_cascade/stable_cascade_prior_image_embeddings.npy" + ) + assert image_embedding.shape == (1, 16, 24, 24) + + max_diff = numpy_cosine_similarity_distance(image_embedding.flatten(), expected_image_embedding.flatten()) + assert max_diff < 1e-4 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion.py new file mode 100755 index 0000000..229b166 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion.py @@ -0,0 +1,376 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import tempfile +import unittest + +import numpy as np + +from diffusers import ( + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + LMSDiscreteScheduler, + OnnxStableDiffusionPipeline, + PNDMScheduler, +) +from diffusers.utils.testing_utils import is_onnx_available, nightly, require_onnxruntime, require_torch_gpu + +from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin + + +if is_onnx_available(): + import onnxruntime as ort + + +class OnnxStableDiffusionPipelineFastTests(OnnxPipelineTesterMixin, unittest.TestCase): + hub_checkpoint = "hf-internal-testing/tiny-random-OnnxStableDiffusionPipeline" + + def get_dummy_inputs(self, seed=0): + generator = np.random.RandomState(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_pipeline_default_ddim(self): + pipe = OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.65072, 0.58492, 0.48219, 0.55521, 0.53180, 0.55939, 0.50697, 0.39800, 0.46455]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_pipeline_pndm(self): + pipe = OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = PNDMScheduler.from_config(pipe.scheduler.config, skip_prk_steps=True) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.65863, 0.59425, 0.49326, 0.56313, 0.53875, 0.56627, 0.51065, 0.39777, 0.46330]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_pipeline_lms(self): + pipe = OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.53755, 0.60786, 0.47402, 0.49488, 0.51869, 0.49819, 0.47985, 0.38957, 0.44279]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_pipeline_euler(self): + pipe = OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.53755, 0.60786, 0.47402, 0.49488, 0.51869, 0.49819, 0.47985, 0.38957, 0.44279]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_pipeline_euler_ancestral(self): + pipe = OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.53817, 0.60812, 0.47384, 0.49530, 0.51894, 0.49814, 0.47984, 0.38958, 0.44271]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_pipeline_dpm_multistep(self): + pipe = OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.53895, 0.60808, 0.47933, 0.49608, 0.51886, 0.49950, 0.48053, 0.38957, 0.44200]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_prompt_embeds(self): + pipe = OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + inputs = self.get_dummy_inputs() + prompt = 3 * [inputs.pop("prompt")] + + text_inputs = pipe.tokenizer( + prompt, + padding="max_length", + max_length=pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + text_inputs = text_inputs["input_ids"] + + prompt_embeds = pipe.text_encoder(input_ids=text_inputs.astype(np.int32))[0] + + inputs["prompt_embeds"] = prompt_embeds + + # forward + output = pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + def test_stable_diffusion_negative_prompt_embeds(self): + pipe = OnnxStableDiffusionPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + negative_prompt = 3 * ["this is a negative prompt"] + inputs["negative_prompt"] = negative_prompt + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + inputs = self.get_dummy_inputs() + prompt = 3 * [inputs.pop("prompt")] + + embeds = [] + for p in [prompt, negative_prompt]: + text_inputs = pipe.tokenizer( + p, + padding="max_length", + max_length=pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="np", + ) + text_inputs = text_inputs["input_ids"] + + embeds.append(pipe.text_encoder(input_ids=text_inputs.astype(np.int32))[0]) + + inputs["prompt_embeds"], inputs["negative_prompt_embeds"] = embeds + + # forward + output = pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + +@nightly +@require_onnxruntime +@require_torch_gpu +class OnnxStableDiffusionPipelineIntegrationTests(unittest.TestCase): + @property + def gpu_provider(self): + return ( + "CUDAExecutionProvider", + { + "gpu_mem_limit": "15000000000", # 15GB + "arena_extend_strategy": "kSameAsRequested", + }, + ) + + @property + def gpu_options(self): + options = ort.SessionOptions() + options.enable_mem_pattern = False + return options + + def test_inference_default_pndm(self): + # using the PNDM scheduler by default + sd_pipe = OnnxStableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", + revision="onnx", + safety_checker=None, + feature_extractor=None, + provider=self.gpu_provider, + sess_options=self.gpu_options, + ) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + np.random.seed(0) + output = sd_pipe([prompt], guidance_scale=6.0, num_inference_steps=10, output_type="np") + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.0452, 0.0390, 0.0087, 0.0350, 0.0617, 0.0364, 0.0544, 0.0523, 0.0720]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_inference_ddim(self): + ddim_scheduler = DDIMScheduler.from_pretrained( + "runwayml/stable-diffusion-v1-5", subfolder="scheduler", revision="onnx" + ) + sd_pipe = OnnxStableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + revision="onnx", + scheduler=ddim_scheduler, + safety_checker=None, + feature_extractor=None, + provider=self.gpu_provider, + sess_options=self.gpu_options, + ) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "open neural network exchange" + generator = np.random.RandomState(0) + output = sd_pipe([prompt], guidance_scale=7.5, num_inference_steps=10, generator=generator, output_type="np") + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.2867, 0.1974, 0.1481, 0.7294, 0.7251, 0.6667, 0.4194, 0.5642, 0.6486]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_inference_k_lms(self): + lms_scheduler = LMSDiscreteScheduler.from_pretrained( + "runwayml/stable-diffusion-v1-5", subfolder="scheduler", revision="onnx" + ) + sd_pipe = OnnxStableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + revision="onnx", + scheduler=lms_scheduler, + safety_checker=None, + feature_extractor=None, + provider=self.gpu_provider, + sess_options=self.gpu_options, + ) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "open neural network exchange" + generator = np.random.RandomState(0) + output = sd_pipe([prompt], guidance_scale=7.5, num_inference_steps=10, generator=generator, output_type="np") + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.2306, 0.1959, 0.1593, 0.6549, 0.6394, 0.5408, 0.5065, 0.6010, 0.6161]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_intermediate_state(self): + number_of_steps = 0 + + def test_callback_fn(step: int, timestep: int, latents: np.ndarray) -> None: + test_callback_fn.has_been_called = True + nonlocal number_of_steps + number_of_steps += 1 + if step == 0: + assert latents.shape == (1, 4, 64, 64) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array( + [-0.6772, -0.3835, -1.2456, 0.1905, -1.0974, 0.6967, -1.9353, 0.0178, 1.0167] + ) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-3 + elif step == 5: + assert latents.shape == (1, 4, 64, 64) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array( + [-0.3351, 0.2241, -0.1837, -0.2325, -0.6577, 0.3393, -0.0241, 0.5899, 1.3875] + ) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 1e-3 + + test_callback_fn.has_been_called = False + + pipe = OnnxStableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + revision="onnx", + safety_checker=None, + feature_extractor=None, + provider=self.gpu_provider, + sess_options=self.gpu_options, + ) + pipe.set_progress_bar_config(disable=None) + + prompt = "Andromeda galaxy in a bottle" + + generator = np.random.RandomState(0) + pipe( + prompt=prompt, + num_inference_steps=5, + guidance_scale=7.5, + generator=generator, + callback=test_callback_fn, + callback_steps=1, + ) + assert test_callback_fn.has_been_called + assert number_of_steps == 6 + + def test_stable_diffusion_no_safety_checker(self): + pipe = OnnxStableDiffusionPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + revision="onnx", + safety_checker=None, + feature_extractor=None, + provider=self.gpu_provider, + sess_options=self.gpu_options, + ) + assert isinstance(pipe, OnnxStableDiffusionPipeline) + assert pipe.safety_checker is None + + image = pipe("example prompt", num_inference_steps=2).images[0] + assert image is not None + + # check that there's no error when saving a pipeline with one of the models being None + with tempfile.TemporaryDirectory() as tmpdirname: + pipe.save_pretrained(tmpdirname) + pipe = OnnxStableDiffusionPipeline.from_pretrained(tmpdirname) + + # sanity check that the pipeline still works + assert pipe.safety_checker is None + image = pipe("example prompt", num_inference_steps=2).images[0] + assert image is not None diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py new file mode 100755 index 0000000..33b461b --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_img2img.py @@ -0,0 +1,245 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest + +import numpy as np + +from diffusers import ( + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + LMSDiscreteScheduler, + OnnxStableDiffusionImg2ImgPipeline, + PNDMScheduler, +) +from diffusers.utils.testing_utils import ( + floats_tensor, + is_onnx_available, + load_image, + nightly, + require_onnxruntime, + require_torch_gpu, +) + +from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin + + +if is_onnx_available(): + import onnxruntime as ort + + +class OnnxStableDiffusionImg2ImgPipelineFastTests(OnnxPipelineTesterMixin, unittest.TestCase): + hub_checkpoint = "hf-internal-testing/tiny-random-OnnxStableDiffusionPipeline" + + def get_dummy_inputs(self, seed=0): + image = floats_tensor((1, 3, 128, 128), rng=random.Random(seed)) + generator = np.random.RandomState(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 3, + "strength": 0.75, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_pipeline_default_ddim(self): + pipe = OnnxStableDiffusionImg2ImgPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.69643, 0.58484, 0.50314, 0.58760, 0.55368, 0.59643, 0.51529, 0.41217, 0.49087]) + assert np.abs(image_slice - expected_slice).max() < 1e-1 + + def test_pipeline_pndm(self): + pipe = OnnxStableDiffusionImg2ImgPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = PNDMScheduler.from_config(pipe.scheduler.config, skip_prk_steps=True) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.61737, 0.54642, 0.53183, 0.54465, 0.52742, 0.60525, 0.49969, 0.40655, 0.48154]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 + + def test_pipeline_lms(self): + pipe = OnnxStableDiffusionImg2ImgPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.set_progress_bar_config(disable=None) + + # warmup pass to apply optimizations + _ = pipe(**self.get_dummy_inputs()) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.52761, 0.59977, 0.49033, 0.49619, 0.54282, 0.50311, 0.47600, 0.40918, 0.45203]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 + + def test_pipeline_euler(self): + pipe = OnnxStableDiffusionImg2ImgPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.52911, 0.60004, 0.49229, 0.49805, 0.54502, 0.50680, 0.47777, 0.41028, 0.45304]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 + + def test_pipeline_euler_ancestral(self): + pipe = OnnxStableDiffusionImg2ImgPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.52911, 0.60004, 0.49229, 0.49805, 0.54502, 0.50680, 0.47777, 0.41028, 0.45304]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 + + def test_pipeline_dpm_multistep(self): + pipe = OnnxStableDiffusionImg2ImgPipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + expected_slice = np.array([0.65331, 0.58277, 0.48204, 0.56059, 0.53665, 0.56235, 0.50969, 0.40009, 0.46552]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 + + +@nightly +@require_onnxruntime +@require_torch_gpu +class OnnxStableDiffusionImg2ImgPipelineIntegrationTests(unittest.TestCase): + @property + def gpu_provider(self): + return ( + "CUDAExecutionProvider", + { + "gpu_mem_limit": "15000000000", # 15GB + "arena_extend_strategy": "kSameAsRequested", + }, + ) + + @property + def gpu_options(self): + options = ort.SessionOptions() + options.enable_mem_pattern = False + return options + + def test_inference_default_pndm(self): + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/img2img/sketch-mountains-input.jpg" + ) + init_image = init_image.resize((768, 512)) + # using the PNDM scheduler by default + pipe = OnnxStableDiffusionImg2ImgPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", + revision="onnx", + safety_checker=None, + feature_extractor=None, + provider=self.gpu_provider, + sess_options=self.gpu_options, + ) + pipe.set_progress_bar_config(disable=None) + + prompt = "A fantasy landscape, trending on artstation" + + generator = np.random.RandomState(0) + output = pipe( + prompt=prompt, + image=init_image, + strength=0.75, + guidance_scale=7.5, + num_inference_steps=10, + generator=generator, + output_type="np", + ) + images = output.images + image_slice = images[0, 255:258, 383:386, -1] + + assert images.shape == (1, 512, 768, 3) + expected_slice = np.array([0.4909, 0.5059, 0.5372, 0.4623, 0.4876, 0.5049, 0.4820, 0.4956, 0.5019]) + # TODO: lower the tolerance after finding the cause of onnxruntime reproducibility issues + + assert np.abs(image_slice.flatten() - expected_slice).max() < 2e-2 + + def test_inference_k_lms(self): + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/img2img/sketch-mountains-input.jpg" + ) + init_image = init_image.resize((768, 512)) + lms_scheduler = LMSDiscreteScheduler.from_pretrained( + "runwayml/stable-diffusion-v1-5", subfolder="scheduler", revision="onnx" + ) + pipe = OnnxStableDiffusionImg2ImgPipeline.from_pretrained( + "runwayml/stable-diffusion-v1-5", + revision="onnx", + scheduler=lms_scheduler, + safety_checker=None, + feature_extractor=None, + provider=self.gpu_provider, + sess_options=self.gpu_options, + ) + pipe.set_progress_bar_config(disable=None) + + prompt = "A fantasy landscape, trending on artstation" + + generator = np.random.RandomState(0) + output = pipe( + prompt=prompt, + image=init_image, + strength=0.75, + guidance_scale=7.5, + num_inference_steps=20, + generator=generator, + output_type="np", + ) + images = output.images + image_slice = images[0, 255:258, 383:386, -1] + + assert images.shape == (1, 512, 768, 3) + expected_slice = np.array([0.8043, 0.926, 0.9581, 0.8119, 0.8954, 0.913, 0.7209, 0.7463, 0.7431]) + # TODO: lower the tolerance after finding the cause of onnxruntime reproducibility issues + + assert np.abs(image_slice.flatten() - expected_slice).max() < 2e-2 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint.py new file mode 100755 index 0000000..6426547 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_inpaint.py @@ -0,0 +1,141 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np + +from diffusers import LMSDiscreteScheduler, OnnxStableDiffusionInpaintPipeline +from diffusers.utils.testing_utils import ( + is_onnx_available, + load_image, + nightly, + require_onnxruntime, + require_torch_gpu, +) + +from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin + + +if is_onnx_available(): + import onnxruntime as ort + + +class OnnxStableDiffusionPipelineFastTests(OnnxPipelineTesterMixin, unittest.TestCase): + # FIXME: add fast tests + pass + + +@nightly +@require_onnxruntime +@require_torch_gpu +class OnnxStableDiffusionInpaintPipelineIntegrationTests(unittest.TestCase): + @property + def gpu_provider(self): + return ( + "CUDAExecutionProvider", + { + "gpu_mem_limit": "15000000000", # 15GB + "arena_extend_strategy": "kSameAsRequested", + }, + ) + + @property + def gpu_options(self): + options = ort.SessionOptions() + options.enable_mem_pattern = False + return options + + def test_inference_default_pndm(self): + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/in_paint/overture-creations-5sI6fQgYIuo.png" + ) + mask_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" + ) + pipe = OnnxStableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", + revision="onnx", + safety_checker=None, + feature_extractor=None, + provider=self.gpu_provider, + sess_options=self.gpu_options, + ) + pipe.set_progress_bar_config(disable=None) + + prompt = "A red cat sitting on a park bench" + + generator = np.random.RandomState(0) + output = pipe( + prompt=prompt, + image=init_image, + mask_image=mask_image, + guidance_scale=7.5, + num_inference_steps=10, + generator=generator, + output_type="np", + ) + images = output.images + image_slice = images[0, 255:258, 255:258, -1] + + assert images.shape == (1, 512, 512, 3) + expected_slice = np.array([0.2514, 0.3007, 0.3517, 0.1790, 0.2382, 0.3167, 0.1944, 0.2273, 0.2464]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_inference_k_lms(self): + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/in_paint/overture-creations-5sI6fQgYIuo.png" + ) + mask_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/in_paint/overture-creations-5sI6fQgYIuo_mask.png" + ) + lms_scheduler = LMSDiscreteScheduler.from_pretrained( + "runwayml/stable-diffusion-inpainting", subfolder="scheduler", revision="onnx" + ) + pipe = OnnxStableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", + revision="onnx", + scheduler=lms_scheduler, + safety_checker=None, + feature_extractor=None, + provider=self.gpu_provider, + sess_options=self.gpu_options, + ) + pipe.set_progress_bar_config(disable=None) + + prompt = "A red cat sitting on a park bench" + + generator = np.random.RandomState(0) + output = pipe( + prompt=prompt, + image=init_image, + mask_image=mask_image, + guidance_scale=7.5, + num_inference_steps=20, + generator=generator, + output_type="np", + ) + images = output.images + image_slice = images[0, 255:258, 255:258, -1] + + assert images.shape == (1, 512, 512, 3) + expected_slice = np.array([0.0086, 0.0077, 0.0083, 0.0093, 0.0107, 0.0139, 0.0094, 0.0097, 0.0125]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_upscale.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_upscale.py new file mode 100755 index 0000000..56c10ad --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_onnx_stable_diffusion_upscale.py @@ -0,0 +1,227 @@ +# coding=utf-8 +# Copyright 2022 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest + +import numpy as np + +from diffusers import ( + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + LMSDiscreteScheduler, + OnnxStableDiffusionUpscalePipeline, + PNDMScheduler, +) +from diffusers.utils.testing_utils import ( + floats_tensor, + is_onnx_available, + load_image, + nightly, + require_onnxruntime, + require_torch_gpu, +) + +from ..test_pipelines_onnx_common import OnnxPipelineTesterMixin + + +if is_onnx_available(): + import onnxruntime as ort + + +class OnnxStableDiffusionUpscalePipelineFastTests(OnnxPipelineTesterMixin, unittest.TestCase): + # TODO: is there an appropriate internal test set? + hub_checkpoint = "ssube/stable-diffusion-x4-upscaler-onnx" + + def get_dummy_inputs(self, seed=0): + image = floats_tensor((1, 3, 128, 128), rng=random.Random(seed)) + generator = np.random.RandomState(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_pipeline_default_ddpm(self): + pipe = OnnxStableDiffusionUpscalePipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + # started as 128, should now be 512 + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.6957, 0.7002, 0.7186, 0.6881, 0.6693, 0.6910, 0.7445, 0.7274, 0.7056]) + assert np.abs(image_slice - expected_slice).max() < 1e-1 + + def test_pipeline_pndm(self): + pipe = OnnxStableDiffusionUpscalePipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = PNDMScheduler.from_config(pipe.scheduler.config, skip_prk_steps=True) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.7349, 0.7347, 0.7034, 0.7696, 0.7876, 0.7597, 0.7916, 0.8085, 0.8036]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 + + def test_pipeline_dpm_multistep(self): + pipe = OnnxStableDiffusionUpscalePipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array( + [0.7659278, 0.76437664, 0.75579107, 0.7691116, 0.77666986, 0.7727672, 0.7758664, 0.7812226, 0.76942515] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 + + def test_pipeline_euler(self): + pipe = OnnxStableDiffusionUpscalePipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array( + [0.6974782, 0.68902093, 0.70135885, 0.7583618, 0.7804545, 0.7854912, 0.78667426, 0.78743863, 0.78070223] + ) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 + + def test_pipeline_euler_ancestral(self): + pipe = OnnxStableDiffusionUpscalePipeline.from_pretrained(self.hub_checkpoint, provider="CPUExecutionProvider") + pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array( + [0.77424496, 0.773601, 0.7645288, 0.7769598, 0.7772739, 0.7738688, 0.78187233, 0.77879584, 0.767043] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 + + +@nightly +@require_onnxruntime +@require_torch_gpu +class OnnxStableDiffusionUpscalePipelineIntegrationTests(unittest.TestCase): + @property + def gpu_provider(self): + return ( + "CUDAExecutionProvider", + { + "gpu_mem_limit": "15000000000", # 15GB + "arena_extend_strategy": "kSameAsRequested", + }, + ) + + @property + def gpu_options(self): + options = ort.SessionOptions() + options.enable_mem_pattern = False + return options + + def test_inference_default_ddpm(self): + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/img2img/sketch-mountains-input.jpg" + ) + init_image = init_image.resize((128, 128)) + # using the PNDM scheduler by default + pipe = OnnxStableDiffusionUpscalePipeline.from_pretrained( + "ssube/stable-diffusion-x4-upscaler-onnx", + provider=self.gpu_provider, + sess_options=self.gpu_options, + ) + pipe.set_progress_bar_config(disable=None) + + prompt = "A fantasy landscape, trending on artstation" + + generator = np.random.RandomState(0) + output = pipe( + prompt=prompt, + image=init_image, + guidance_scale=7.5, + num_inference_steps=10, + generator=generator, + output_type="np", + ) + images = output.images + image_slice = images[0, 255:258, 383:386, -1] + + assert images.shape == (1, 512, 512, 3) + expected_slice = np.array([0.4883, 0.4947, 0.4980, 0.4975, 0.4982, 0.4980, 0.5000, 0.5006, 0.4972]) + # TODO: lower the tolerance after finding the cause of onnxruntime reproducibility issues + + assert np.abs(image_slice.flatten() - expected_slice).max() < 2e-2 + + def test_inference_k_lms(self): + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/img2img/sketch-mountains-input.jpg" + ) + init_image = init_image.resize((128, 128)) + lms_scheduler = LMSDiscreteScheduler.from_pretrained( + "ssube/stable-diffusion-x4-upscaler-onnx", subfolder="scheduler" + ) + pipe = OnnxStableDiffusionUpscalePipeline.from_pretrained( + "ssube/stable-diffusion-x4-upscaler-onnx", + scheduler=lms_scheduler, + provider=self.gpu_provider, + sess_options=self.gpu_options, + ) + pipe.set_progress_bar_config(disable=None) + + prompt = "A fantasy landscape, trending on artstation" + + generator = np.random.RandomState(0) + output = pipe( + prompt=prompt, + image=init_image, + guidance_scale=7.5, + num_inference_steps=20, + generator=generator, + output_type="np", + ) + images = output.images + image_slice = images[0, 255:258, 383:386, -1] + + assert images.shape == (1, 512, 512, 3) + expected_slice = np.array( + [0.50173753, 0.50223356, 0.502039, 0.50233036, 0.5023725, 0.5022601, 0.5018758, 0.50234085, 0.50241566] + ) + # TODO: lower the tolerance after finding the cause of onnxruntime reproducibility issues + + assert np.abs(image_slice.flatten() - expected_slice).max() < 2e-2 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion.py new file mode 100755 index 0000000..82afaca --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion.py @@ -0,0 +1,1425 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import gc +import tempfile +import time +import traceback +import unittest + +import numpy as np +import torch +from huggingface_hub import hf_hub_download +from transformers import ( + CLIPTextConfig, + CLIPTextModel, + CLIPTokenizer, +) + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + LCMScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + StableDiffusionPipeline, + UNet2DConditionModel, + logging, +) +from diffusers.models.attention_processor import AttnProcessor +from diffusers.utils.testing_utils import ( + CaptureLogger, + enable_full_determinism, + load_image, + load_numpy, + nightly, + numpy_cosine_similarity_distance, + require_python39_or_higher, + require_torch_2, + require_torch_gpu, + run_test_in_subprocess, + slow, + torch_device, +) + +from ..pipeline_params import ( + TEXT_TO_IMAGE_BATCH_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_PARAMS, +) +from ..test_pipelines_common import ( + IPAdapterTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, +) + + +enable_full_determinism() + + +# Will be run via run_test_in_subprocess +def _test_stable_diffusion_compile(in_queue, out_queue, timeout): + error = None + try: + inputs = in_queue.get(timeout=timeout) + torch_device = inputs.pop("torch_device") + seed = inputs.pop("seed") + inputs["generator"] = torch.Generator(device=torch_device).manual_seed(seed) + + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) + sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + + sd_pipe.unet.to(memory_format=torch.channels_last) + sd_pipe.unet = torch.compile(sd_pipe.unet, mode="reduce-overhead", fullgraph=True) + + sd_pipe.set_progress_bar_config(disable=None) + + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.38019, 0.28647, 0.27321, 0.40377, 0.38290, 0.35446, 0.39218, 0.38165, 0.42239]) + + assert np.abs(image_slice - expected_slice).max() < 5e-3 + except Exception: + error = f"{traceback.format_exc()}" + + results = {"error": error} + out_queue.put(results, timeout=timeout) + out_queue.join() + + +class StableDiffusionPipelineFastTests( + IPAdapterTesterMixin, + PipelineLatentTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineTesterMixin, + unittest.TestCase, +): + pipeline_class = StableDiffusionPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS + + def get_dummy_components(self, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(4, 8), + layers_per_block=1, + sample_size=32, + time_cond_proj_dim=time_cond_proj_dim, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + norm_num_groups=2, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[4, 8], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + norm_num_groups=2, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=64, + layer_norm_eps=1e-05, + num_attention_heads=8, + num_hidden_layers=3, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "np", + } + return inputs + + def test_stable_diffusion_ddim(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.3203, 0.4555, 0.4711, 0.3505, 0.3973, 0.4650, 0.5137, 0.3392, 0.4045]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.3454, 0.5349, 0.5185, 0.2808, 0.4509, 0.4612, 0.4655, 0.3601, 0.4315]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.3454, 0.5349, 0.5185, 0.2808, 0.4509, 0.4612, 0.4655, 0.3601, 0.4315]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_prompt_embeds(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + inputs = self.get_dummy_inputs(torch_device) + prompt = 3 * [inputs.pop("prompt")] + + text_inputs = sd_pipe.tokenizer( + prompt, + padding="max_length", + max_length=sd_pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + prompt_embeds = sd_pipe.text_encoder(text_inputs)[0] + + inputs["prompt_embeds"] = prompt_embeds + + # forward + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + def test_stable_diffusion_negative_prompt_embeds(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + negative_prompt = 3 * ["this is a negative prompt"] + inputs["negative_prompt"] = negative_prompt + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + inputs = self.get_dummy_inputs(torch_device) + prompt = 3 * [inputs.pop("prompt")] + + embeds = [] + for p in [prompt, negative_prompt]: + text_inputs = sd_pipe.tokenizer( + p, + padding="max_length", + max_length=sd_pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + embeds.append(sd_pipe.text_encoder(text_inputs)[0]) + + inputs["prompt_embeds"], inputs["negative_prompt_embeds"] = embeds + + # forward + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + def test_stable_diffusion_prompt_embeds_with_plain_negative_prompt_list(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + negative_prompt = 3 * ["this is a negative prompt"] + inputs["negative_prompt"] = negative_prompt + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = negative_prompt + prompt = 3 * [inputs.pop("prompt")] + + text_inputs = sd_pipe.tokenizer( + prompt, + padding="max_length", + max_length=sd_pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + prompt_embeds = sd_pipe.text_encoder(text_inputs)[0] + + inputs["prompt_embeds"] = prompt_embeds + + # forward + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + def test_stable_diffusion_ddim_factor_8(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs, height=136, width=136) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 136, 136, 3) + expected_slice = np.array([0.4346, 0.5621, 0.5016, 0.3926, 0.4533, 0.4134, 0.5625, 0.5632, 0.5265]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_pndm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe.scheduler = PNDMScheduler(skip_prk_steps=True) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.3411, 0.5032, 0.4704, 0.3135, 0.4323, 0.4740, 0.5150, 0.3498, 0.4022]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_no_safety_checker(self): + pipe = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-lms-pipe", safety_checker=None + ) + assert isinstance(pipe, StableDiffusionPipeline) + assert isinstance(pipe.scheduler, LMSDiscreteScheduler) + assert pipe.safety_checker is None + + image = pipe("example prompt", num_inference_steps=2).images[0] + assert image is not None + + # check that there's no error when saving a pipeline with one of the models being None + with tempfile.TemporaryDirectory() as tmpdirname: + pipe.save_pretrained(tmpdirname) + pipe = StableDiffusionPipeline.from_pretrained(tmpdirname) + + # sanity check that the pipeline still works + assert pipe.safety_checker is None + image = pipe("example prompt", num_inference_steps=2).images[0] + assert image is not None + + def test_stable_diffusion_k_lms(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.3149, 0.5246, 0.4796, 0.3218, 0.4469, 0.4729, 0.5151, 0.3597, 0.3954]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_k_euler_ancestral(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.3151, 0.5243, 0.4794, 0.3217, 0.4468, 0.4728, 0.5152, 0.3598, 0.3954]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_k_euler(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe.scheduler = EulerDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.3149, 0.5246, 0.4796, 0.3218, 0.4469, 0.4729, 0.5151, 0.3597, 0.3954]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_vae_slicing(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = LMSDiscreteScheduler.from_config(components["scheduler"].config) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + image_count = 4 + + inputs = self.get_dummy_inputs(device) + inputs["prompt"] = [inputs["prompt"]] * image_count + output_1 = sd_pipe(**inputs) + + # make sure sliced vae decode yields the same result + sd_pipe.enable_vae_slicing() + inputs = self.get_dummy_inputs(device) + inputs["prompt"] = [inputs["prompt"]] * image_count + output_2 = sd_pipe(**inputs) + + # there is a small discrepancy at image borders vs. full batch decode + assert np.abs(output_2.images.flatten() - output_1.images.flatten()).max() < 3e-3 + + def test_stable_diffusion_vae_tiling(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + + # make sure here that pndm scheduler skips prk + components["safety_checker"] = None + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + + # Test that tiled decode at 512x512 yields the same result as the non-tiled decode + generator = torch.Generator(device=device).manual_seed(0) + output_1 = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np") + + # make sure tiled vae decode yields the same result + sd_pipe.enable_vae_tiling() + generator = torch.Generator(device=device).manual_seed(0) + output_2 = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np") + + assert np.abs(output_2.images.flatten() - output_1.images.flatten()).max() < 5e-1 + + # test that tiled decode works with various shapes + shapes = [(1, 4, 73, 97), (1, 4, 97, 73), (1, 4, 49, 65), (1, 4, 65, 49)] + for shape in shapes: + zeros = torch.zeros(shape).to(device) + sd_pipe.vae.decode(zeros) + + def test_stable_diffusion_negative_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = PNDMScheduler(skip_prk_steps=True) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + negative_prompt = "french fries" + output = sd_pipe(**inputs, negative_prompt=negative_prompt) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.3458, 0.5120, 0.4800, 0.3116, 0.4348, 0.4802, 0.5237, 0.3467, 0.3991]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_long_prompt(self): + components = self.get_dummy_components() + components["scheduler"] = LMSDiscreteScheduler.from_config(components["scheduler"].config) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + do_classifier_free_guidance = True + negative_prompt = None + num_images_per_prompt = 1 + logger = logging.get_logger("diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion") + logger.setLevel(logging.WARNING) + + prompt = 100 * "@" + with CaptureLogger(logger) as cap_logger: + negative_text_embeddings, text_embeddings = sd_pipe.encode_prompt( + prompt, torch_device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + if negative_text_embeddings is not None: + text_embeddings = torch.cat([negative_text_embeddings, text_embeddings]) + + # 100 - 77 + 1 (BOS token) + 1 (EOS token) = 25 + assert cap_logger.out.count("@") == 25 + + negative_prompt = "Hello" + with CaptureLogger(logger) as cap_logger_2: + negative_text_embeddings_2, text_embeddings_2 = sd_pipe.encode_prompt( + prompt, torch_device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + if negative_text_embeddings_2 is not None: + text_embeddings_2 = torch.cat([negative_text_embeddings_2, text_embeddings_2]) + + assert cap_logger.out == cap_logger_2.out + + prompt = 25 * "@" + with CaptureLogger(logger) as cap_logger_3: + negative_text_embeddings_3, text_embeddings_3 = sd_pipe.encode_prompt( + prompt, torch_device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + if negative_text_embeddings_3 is not None: + text_embeddings_3 = torch.cat([negative_text_embeddings_3, text_embeddings_3]) + + assert text_embeddings_3.shape == text_embeddings_2.shape == text_embeddings.shape + assert text_embeddings.shape[1] == 77 + assert cap_logger_3.out == "" + + def test_stable_diffusion_height_width_opt(self): + components = self.get_dummy_components() + components["scheduler"] = LMSDiscreteScheduler.from_config(components["scheduler"].config) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "hey" + + output = sd_pipe(prompt, num_inference_steps=1, output_type="np") + image_shape = output.images[0].shape[:2] + assert image_shape == (64, 64) + + output = sd_pipe(prompt, num_inference_steps=1, height=96, width=96, output_type="np") + image_shape = output.images[0].shape[:2] + assert image_shape == (96, 96) + + config = dict(sd_pipe.unet.config) + config["sample_size"] = 96 + sd_pipe.unet = UNet2DConditionModel.from_config(config).to(torch_device) + output = sd_pipe(prompt, num_inference_steps=1, output_type="np") + image_shape = output.images[0].shape[:2] + assert image_shape == (192, 192) + + def test_attention_slicing_forward_pass(self): + super().test_attention_slicing_forward_pass(expected_max_diff=3e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + def test_freeu_enabled(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "hey" + output = sd_pipe(prompt, num_inference_steps=1, output_type="np", generator=torch.manual_seed(0)).images + + sd_pipe.enable_freeu(s1=0.9, s2=0.2, b1=1.2, b2=1.4) + output_freeu = sd_pipe(prompt, num_inference_steps=1, output_type="np", generator=torch.manual_seed(0)).images + + assert not np.allclose( + output[0, -3:, -3:, -1], output_freeu[0, -3:, -3:, -1] + ), "Enabling of FreeU should lead to different results." + + def test_freeu_disabled(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "hey" + output = sd_pipe(prompt, num_inference_steps=1, output_type="np", generator=torch.manual_seed(0)).images + + sd_pipe.enable_freeu(s1=0.9, s2=0.2, b1=1.2, b2=1.4) + sd_pipe.disable_freeu() + + freeu_keys = {"s1", "s2", "b1", "b2"} + for upsample_block in sd_pipe.unet.up_blocks: + for key in freeu_keys: + assert getattr(upsample_block, key) is None, f"Disabling of FreeU should have set {key} to None." + + output_no_freeu = sd_pipe( + prompt, num_inference_steps=1, output_type="np", generator=torch.manual_seed(0) + ).images + + assert np.allclose( + output[0, -3:, -3:, -1], output_no_freeu[0, -3:, -3:, -1] + ), "Disabling of FreeU should lead to results similar to the default pipeline results." + + def test_fused_qkv_projections(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + original_image_slice = image[0, -3:, -3:, -1] + + sd_pipe.fuse_qkv_projections() + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice_fused = image[0, -3:, -3:, -1] + + sd_pipe.unfuse_qkv_projections() + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice_disabled = image[0, -3:, -3:, -1] + + assert np.allclose( + original_image_slice, image_slice_fused, atol=1e-2, rtol=1e-2 + ), "Fusion of QKV projections shouldn't affect the outputs." + assert np.allclose( + image_slice_fused, image_slice_disabled, atol=1e-2, rtol=1e-2 + ), "Outputs, with QKV projection fusion enabled, shouldn't change when fused QKV projections are disabled." + assert np.allclose( + original_image_slice, image_slice_disabled, atol=1e-2, rtol=1e-2 + ), "Original outputs should match when fused QKV projections are disabled." + + def test_pipeline_interrupt(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "hey" + num_inference_steps = 3 + + # store intermediate latents from the generation process + class PipelineState: + def __init__(self): + self.state = [] + + def apply(self, pipe, i, t, callback_kwargs): + self.state.append(callback_kwargs["latents"]) + return callback_kwargs + + pipe_state = PipelineState() + sd_pipe( + prompt, + num_inference_steps=num_inference_steps, + output_type="np", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=pipe_state.apply, + ).images + + # interrupt generation at step index + interrupt_step_idx = 1 + + def callback_on_step_end(pipe, i, t, callback_kwargs): + if i == interrupt_step_idx: + pipe._interrupt = True + + return callback_kwargs + + output_interrupted = sd_pipe( + prompt, + num_inference_steps=num_inference_steps, + output_type="latent", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=callback_on_step_end, + ).images + + # fetch intermediate latents at the interrupted step + # from the completed generation process + intermediate_latent = pipe_state.state[interrupt_step_idx] + + # compare the intermediate latent to the output of the interrupted process + # they should be the same + assert torch.allclose(intermediate_latent, output_interrupted, atol=1e-4) + + +@slow +@require_torch_gpu +class StableDiffusionPipelineSlowTests(unittest.TestCase): + def setUp(self): + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_1_1_pndm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-1") + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.4363, 0.4355, 0.3667, 0.4066, 0.3970, 0.3866, 0.4394, 0.4356, 0.4059]) + assert np.abs(image_slice - expected_slice).max() < 3e-3 + + def test_stable_diffusion_v1_4_with_freeu(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 25 + + sd_pipe.enable_freeu(s1=0.9, s2=0.2, b1=1.2, b2=1.4) + image = sd_pipe(**inputs).images + image = image[0, -3:, -3:, -1].flatten() + expected_image = [0.0721, 0.0588, 0.0268, 0.0384, 0.0636, 0.0, 0.0429, 0.0344, 0.0309] + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_1_4_pndm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.5740, 0.4784, 0.3162, 0.6358, 0.5831, 0.5505, 0.5082, 0.5631, 0.5575]) + assert np.abs(image_slice - expected_slice).max() < 3e-3 + + def test_stable_diffusion_ddim(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) + sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.38019, 0.28647, 0.27321, 0.40377, 0.38290, 0.35446, 0.39218, 0.38165, 0.42239]) + assert np.abs(image_slice - expected_slice).max() < 1e-4 + + def test_stable_diffusion_lms(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.10542, 0.09620, 0.07332, 0.09015, 0.09382, 0.07597, 0.08496, 0.07806, 0.06455]) + assert np.abs(image_slice - expected_slice).max() < 3e-3 + + def test_stable_diffusion_dpm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config( + sd_pipe.scheduler.config, + final_sigmas_type="sigma_min", + ) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.03503, 0.03494, 0.01087, 0.03128, 0.02552, 0.00803, 0.00742, 0.00372, 0.00000]) + assert np.abs(image_slice - expected_slice).max() < 3e-3 + + def test_stable_diffusion_attention_slicing(self): + torch.cuda.reset_peak_memory_stats() + pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16) + pipe.unet.set_default_attn_processor() + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + # enable attention slicing + pipe.enable_attention_slicing() + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image_sliced = pipe(**inputs).images + + mem_bytes = torch.cuda.max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + # make sure that less than 3.75 GB is allocated + assert mem_bytes < 3.75 * 10**9 + + # disable slicing + pipe.disable_attention_slicing() + pipe.unet.set_default_attn_processor() + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image = pipe(**inputs).images + + # make sure that more than 3.75 GB is allocated + mem_bytes = torch.cuda.max_memory_allocated() + assert mem_bytes > 3.75 * 10**9 + max_diff = numpy_cosine_similarity_distance(image_sliced.flatten(), image.flatten()) + assert max_diff < 1e-3 + + def test_stable_diffusion_vae_slicing(self): + torch.cuda.reset_peak_memory_stats() + pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + # enable vae slicing + pipe.enable_vae_slicing() + inputs = self.get_inputs(torch_device, dtype=torch.float16) + inputs["prompt"] = [inputs["prompt"]] * 4 + inputs["latents"] = torch.cat([inputs["latents"]] * 4) + image_sliced = pipe(**inputs).images + + mem_bytes = torch.cuda.max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + # make sure that less than 4 GB is allocated + assert mem_bytes < 4e9 + + # disable vae slicing + pipe.disable_vae_slicing() + inputs = self.get_inputs(torch_device, dtype=torch.float16) + inputs["prompt"] = [inputs["prompt"]] * 4 + inputs["latents"] = torch.cat([inputs["latents"]] * 4) + image = pipe(**inputs).images + + # make sure that more than 4 GB is allocated + mem_bytes = torch.cuda.max_memory_allocated() + assert mem_bytes > 4e9 + # There is a small discrepancy at the image borders vs. a fully batched version. + max_diff = numpy_cosine_similarity_distance(image_sliced.flatten(), image.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_vae_tiling(self): + torch.cuda.reset_peak_memory_stats() + model_id = "CompVis/stable-diffusion-v1-4" + pipe = StableDiffusionPipeline.from_pretrained( + model_id, revision="fp16", torch_dtype=torch.float16, safety_checker=None + ) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + pipe.unet = pipe.unet.to(memory_format=torch.channels_last) + pipe.vae = pipe.vae.to(memory_format=torch.channels_last) + + prompt = "a photograph of an astronaut riding a horse" + + # enable vae tiling + pipe.enable_vae_tiling() + pipe.enable_model_cpu_offload() + generator = torch.Generator(device="cpu").manual_seed(0) + output_chunked = pipe( + [prompt], + width=1024, + height=1024, + generator=generator, + guidance_scale=7.5, + num_inference_steps=2, + output_type="numpy", + ) + image_chunked = output_chunked.images + + mem_bytes = torch.cuda.max_memory_allocated() + + # disable vae tiling + pipe.disable_vae_tiling() + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipe( + [prompt], + width=1024, + height=1024, + generator=generator, + guidance_scale=7.5, + num_inference_steps=2, + output_type="numpy", + ) + image = output.images + + assert mem_bytes < 1e10 + max_diff = numpy_cosine_similarity_distance(image_chunked.flatten(), image.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_fp16_vs_autocast(self): + # this test makes sure that the original model with autocast + # and the new model with fp16 yield the same result + pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image_fp16 = pipe(**inputs).images + + with torch.autocast(torch_device): + inputs = self.get_inputs(torch_device) + image_autocast = pipe(**inputs).images + + # Make sure results are close enough + diff = np.abs(image_fp16.flatten() - image_autocast.flatten()) + # They ARE different since ops are not run always at the same precision + # however, they should be extremely close. + assert diff.mean() < 2e-2 + + def test_stable_diffusion_intermediate_state(self): + number_of_steps = 0 + + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True + nonlocal number_of_steps + number_of_steps += 1 + if step == 1: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 64) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array( + [-0.5693, -0.3018, -0.9746, 0.0518, -0.8770, 0.7559, -1.7402, 0.1022, 1.1582] + ) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + elif step == 2: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 64) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array( + [-0.1958, -0.2993, -1.0166, -0.5005, -0.4810, 0.6162, -0.9492, 0.6621, 1.4492] + ) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + + callback_fn.has_been_called = False + + pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == inputs["num_inference_steps"] + + def test_stable_diffusion_low_cpu_mem_usage(self): + pipeline_id = "CompVis/stable-diffusion-v1-4" + + start_time = time.time() + pipeline_low_cpu_mem_usage = StableDiffusionPipeline.from_pretrained(pipeline_id, torch_dtype=torch.float16) + pipeline_low_cpu_mem_usage.to(torch_device) + low_cpu_mem_usage_time = time.time() - start_time + + start_time = time.time() + _ = StableDiffusionPipeline.from_pretrained(pipeline_id, torch_dtype=torch.float16, low_cpu_mem_usage=False) + normal_load_time = time.time() - start_time + + assert 2 * low_cpu_mem_usage_time < normal_load_time + + def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.8 GB is allocated + assert mem_bytes < 2.8 * 10**9 + + def test_stable_diffusion_pipeline_with_model_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + + # Normal inference + + pipe = StableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", + torch_dtype=torch.float16, + ) + pipe.unet.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + outputs = pipe(**inputs) + mem_bytes = torch.cuda.max_memory_allocated() + + # With model offloading + + # Reload but don't move to cuda + pipe = StableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", + torch_dtype=torch.float16, + ) + pipe.unet.set_default_attn_processor() + + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + inputs = self.get_inputs(torch_device, dtype=torch.float16) + + outputs_offloaded = pipe(**inputs) + mem_bytes_offloaded = torch.cuda.max_memory_allocated() + + images = outputs.images + offloaded_images = outputs_offloaded.images + + max_diff = numpy_cosine_similarity_distance(images.flatten(), offloaded_images.flatten()) + assert max_diff < 1e-3 + assert mem_bytes_offloaded < mem_bytes + assert mem_bytes_offloaded < 3.5 * 10**9 + for module in pipe.text_encoder, pipe.unet, pipe.vae: + assert module.device == torch.device("cpu") + + # With attention slicing + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe.enable_attention_slicing() + _ = pipe(**inputs) + mem_bytes_slicing = torch.cuda.max_memory_allocated() + + assert mem_bytes_slicing < mem_bytes_offloaded + assert mem_bytes_slicing < 3 * 10**9 + + def test_stable_diffusion_textual_inversion(self): + pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + pipe.load_textual_inversion("sd-concepts-library/low-poly-hd-logos-icons") + + a111_file = hf_hub_download("hf-internal-testing/text_inv_embedding_a1111_format", "winter_style.pt") + a111_file_neg = hf_hub_download( + "hf-internal-testing/text_inv_embedding_a1111_format", "winter_style_negative.pt" + ) + pipe.load_textual_inversion(a111_file) + pipe.load_textual_inversion(a111_file_neg) + pipe.to("cuda") + + generator = torch.Generator(device="cpu").manual_seed(1) + + prompt = "An logo of a turtle in strong Style-Winter with " + neg_prompt = "Style-Winter-neg" + + image = pipe(prompt=prompt, negative_prompt=neg_prompt, generator=generator, output_type="np").images[0] + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text_inv/winter_logo_style.npy" + ) + + max_diff = np.abs(expected_image - image).max() + assert max_diff < 8e-1 + + def test_stable_diffusion_textual_inversion_with_model_cpu_offload(self): + pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + pipe.enable_model_cpu_offload() + pipe.load_textual_inversion("sd-concepts-library/low-poly-hd-logos-icons") + + a111_file = hf_hub_download("hf-internal-testing/text_inv_embedding_a1111_format", "winter_style.pt") + a111_file_neg = hf_hub_download( + "hf-internal-testing/text_inv_embedding_a1111_format", "winter_style_negative.pt" + ) + pipe.load_textual_inversion(a111_file) + pipe.load_textual_inversion(a111_file_neg) + + generator = torch.Generator(device="cpu").manual_seed(1) + + prompt = "An logo of a turtle in strong Style-Winter with " + neg_prompt = "Style-Winter-neg" + + image = pipe(prompt=prompt, negative_prompt=neg_prompt, generator=generator, output_type="np").images[0] + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text_inv/winter_logo_style.npy" + ) + + max_diff = np.abs(expected_image - image).max() + assert max_diff < 8e-1 + + def test_stable_diffusion_textual_inversion_with_sequential_cpu_offload(self): + pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + pipe.enable_sequential_cpu_offload() + pipe.load_textual_inversion("sd-concepts-library/low-poly-hd-logos-icons") + + a111_file = hf_hub_download("hf-internal-testing/text_inv_embedding_a1111_format", "winter_style.pt") + a111_file_neg = hf_hub_download( + "hf-internal-testing/text_inv_embedding_a1111_format", "winter_style_negative.pt" + ) + pipe.load_textual_inversion(a111_file) + pipe.load_textual_inversion(a111_file_neg) + + generator = torch.Generator(device="cpu").manual_seed(1) + + prompt = "An logo of a turtle in strong Style-Winter with " + neg_prompt = "Style-Winter-neg" + + image = pipe(prompt=prompt, negative_prompt=neg_prompt, generator=generator, output_type="np").images[0] + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text_inv/winter_logo_style.npy" + ) + + max_diff = np.abs(expected_image - image).max() + assert max_diff < 8e-1 + + @require_python39_or_higher + @require_torch_2 + def test_stable_diffusion_compile(self): + seed = 0 + inputs = self.get_inputs(torch_device, seed=seed) + # Can't pickle a Generator object + del inputs["generator"] + inputs["torch_device"] = torch_device + inputs["seed"] = seed + run_test_in_subprocess(test_case=self, target_func=_test_stable_diffusion_compile, inputs=inputs) + + def test_stable_diffusion_lcm(self): + unet = UNet2DConditionModel.from_pretrained("SimianLuo/LCM_Dreamshaper_v7", subfolder="unet") + sd_pipe = StableDiffusionPipeline.from_pretrained("Lykon/dreamshaper-7", unet=unet).to(torch_device) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 6 + inputs["output_type"] = "pil" + + image = sd_pipe(**inputs).images[0] + + expected_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/lcm_full/stable_diffusion_lcm.png" + ) + + image = sd_pipe.image_processor.pil_to_numpy(image) + expected_image = sd_pipe.image_processor.pil_to_numpy(expected_image) + + max_diff = numpy_cosine_similarity_distance(image.flatten(), expected_image.flatten()) + + assert max_diff < 1e-2 + + +@slow +@require_torch_gpu +class StableDiffusionPipelineCkptTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_download_from_hub(self): + ckpt_paths = [ + "https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned-emaonly.ckpt", + "https://huggingface.co/WarriorMama777/OrangeMixs/blob/main/Models/AbyssOrangeMix/AbyssOrangeMix_base.ckpt", + ] + + for ckpt_path in ckpt_paths: + pipe = StableDiffusionPipeline.from_single_file(ckpt_path, torch_dtype=torch.float16) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.to("cuda") + + image_out = pipe("test", num_inference_steps=1, output_type="np").images[0] + + assert image_out.shape == (512, 512, 3) + + def test_download_local(self): + ckpt_filename = hf_hub_download("runwayml/stable-diffusion-v1-5", filename="v1-5-pruned-emaonly.ckpt") + config_filename = hf_hub_download("runwayml/stable-diffusion-v1-5", filename="v1-inference.yaml") + + pipe = StableDiffusionPipeline.from_single_file( + ckpt_filename, config_files={"v1": config_filename}, torch_dtype=torch.float16 + ) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.to("cuda") + + image_out = pipe("test", num_inference_steps=1, output_type="np").images[0] + + assert image_out.shape == (512, 512, 3) + + def test_download_ckpt_diff_format_is_same(self): + ckpt_path = "https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned-emaonly.ckpt" + + sf_pipe = StableDiffusionPipeline.from_single_file(ckpt_path) + sf_pipe.scheduler = DDIMScheduler.from_config(sf_pipe.scheduler.config) + sf_pipe.unet.set_attn_processor(AttnProcessor()) + sf_pipe.to("cuda") + + generator = torch.Generator(device="cpu").manual_seed(0) + image_single_file = sf_pipe("a turtle", num_inference_steps=2, generator=generator, output_type="np").images[0] + + pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.unet.set_attn_processor(AttnProcessor()) + pipe.to("cuda") + + generator = torch.Generator(device="cpu").manual_seed(0) + image = pipe("a turtle", num_inference_steps=2, generator=generator, output_type="np").images[0] + + max_diff = numpy_cosine_similarity_distance(image.flatten(), image_single_file.flatten()) + + assert max_diff < 1e-3 + + def test_single_file_component_configs(self): + pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + + ckpt_path = "https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned-emaonly.ckpt" + single_file_pipe = StableDiffusionPipeline.from_single_file(ckpt_path, load_safety_checker=True) + + for param_name, param_value in single_file_pipe.text_encoder.config.to_dict().items(): + if param_name in ["torch_dtype", "architectures", "_name_or_path"]: + continue + assert pipe.text_encoder.config.to_dict()[param_name] == param_value + + PARAMS_TO_IGNORE = ["torch_dtype", "_name_or_path", "architectures", "_use_default_values"] + for param_name, param_value in single_file_pipe.unet.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.unet.config[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" + + for param_name, param_value in single_file_pipe.vae.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.vae.config[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" + + for param_name, param_value in single_file_pipe.safety_checker.config.to_dict().items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.safety_checker.config.to_dict()[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" + + +@nightly +@require_torch_gpu +class StableDiffusionPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 50, + "guidance_scale": 7.5, + "output_type": "np", + } + return inputs + + def test_stable_diffusion_1_4_pndm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_text2img/stable_diffusion_1_4_pndm.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_1_5_pndm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5").to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_text2img/stable_diffusion_1_5_pndm.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_ddim(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) + sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_text2img/stable_diffusion_1_4_ddim.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 3e-3 + + def test_stable_diffusion_lms(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_text2img/stable_diffusion_1_4_lms.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_euler(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4").to(torch_device) + sd_pipe.scheduler = EulerDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_text2img/stable_diffusion_1_4_euler.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py new file mode 100755 index 0000000..4483fd8 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_img2img.py @@ -0,0 +1,735 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import traceback +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + AutoencoderTiny, + DDIMScheduler, + DPMSolverMultistepScheduler, + HeunDiscreteScheduler, + LCMScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + StableDiffusionImg2ImgPipeline, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + nightly, + require_python39_or_higher, + require_torch_2, + require_torch_gpu, + run_test_in_subprocess, + skip_mps, + slow, + torch_device, +) + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, +) +from ..test_pipelines_common import ( + IPAdapterTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, +) + + +enable_full_determinism() + + +# Will be run via run_test_in_subprocess +def _test_img2img_compile(in_queue, out_queue, timeout): + error = None + try: + inputs = in_queue.get(timeout=timeout) + torch_device = inputs.pop("torch_device") + seed = inputs.pop("seed") + inputs["generator"] = torch.Generator(device=torch_device).manual_seed(seed) + + pipe = StableDiffusionImg2ImgPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.unet.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.unet.to(memory_format=torch.channels_last) + pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True) + + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 768, 3) + expected_slice = np.array([0.0606, 0.0570, 0.0805, 0.0579, 0.0628, 0.0623, 0.0843, 0.1115, 0.0806]) + + assert np.abs(expected_slice - image_slice).max() < 1e-3 + except Exception: + error = f"{traceback.format_exc()}" + + results = {"error": error} + out_queue.put(results, timeout=timeout) + out_queue.join() + + +class StableDiffusionImg2ImgPipelineFastTests( + IPAdapterTesterMixin, + PipelineLatentTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineTesterMixin, + unittest.TestCase, +): + pipeline_class = StableDiffusionImg2ImgPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width"} + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + image_latents_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS + + def get_dummy_components(self, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + time_cond_proj_dim=time_cond_proj_dim, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + scheduler = PNDMScheduler(skip_prk_steps=True) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_tiny_autoencoder(self): + return AutoencoderTiny(in_channels=3, out_channels=3, latent_channels=4) + + def get_dummy_inputs(self, device, seed=0): + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image / 2 + 0.5 + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_img2img_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.4555, 0.3216, 0.4049, 0.4620, 0.4618, 0.4126, 0.4122, 0.4629, 0.4579]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_img2img_default_case_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionImg2ImgPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.5709, 0.4614, 0.4587, 0.5978, 0.5298, 0.6910, 0.6240, 0.5212, 0.5454]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_img2img_default_case_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionImg2ImgPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.5709, 0.4614, 0.4587, 0.5978, 0.5298, 0.6910, 0.6240, 0.5212, 0.5454]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_img2img_negative_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + negative_prompt = "french fries" + output = sd_pipe(**inputs, negative_prompt=negative_prompt) + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.4593, 0.3408, 0.4232, 0.4749, 0.4476, 0.4115, 0.4357, 0.4733, 0.4663]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_img2img_multiple_init_images(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["prompt"] = [inputs["prompt"]] * 2 + inputs["image"] = inputs["image"].repeat(2, 1, 1, 1) + image = sd_pipe(**inputs).images + image_slice = image[-1, -3:, -3:, -1] + + assert image.shape == (2, 32, 32, 3) + expected_slice = np.array([0.4241, 0.5576, 0.5711, 0.4792, 0.4311, 0.5952, 0.5827, 0.5138, 0.5109]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_img2img_k_lms(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = LMSDiscreteScheduler( + beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear" + ) + sd_pipe = StableDiffusionImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.4398, 0.4949, 0.4337, 0.6580, 0.5555, 0.4338, 0.5769, 0.5955, 0.5175]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_img2img_tiny_autoencoder(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionImg2ImgPipeline(**components) + sd_pipe.vae = self.get_dummy_tiny_autoencoder() + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.00669, 0.00669, 0.0, 0.00693, 0.00858, 0.0, 0.00567, 0.00515, 0.00125]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + @skip_mps + def test_save_load_local(self): + return super().test_save_load_local() + + @skip_mps + def test_dict_tuple_outputs_equivalent(self): + return super().test_dict_tuple_outputs_equivalent() + + @skip_mps + def test_save_load_optional_components(self): + return super().test_save_load_optional_components() + + @skip_mps + def test_attention_slicing_forward_pass(self): + return super().test_attention_slicing_forward_pass(expected_max_diff=5e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=5e-1) + + def test_pipeline_interrupt(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + prompt = "hey" + num_inference_steps = 3 + + # store intermediate latents from the generation process + class PipelineState: + def __init__(self): + self.state = [] + + def apply(self, pipe, i, t, callback_kwargs): + self.state.append(callback_kwargs["latents"]) + return callback_kwargs + + pipe_state = PipelineState() + sd_pipe( + prompt, + image=inputs["image"], + num_inference_steps=num_inference_steps, + output_type="np", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=pipe_state.apply, + ).images + + # interrupt generation at step index + interrupt_step_idx = 1 + + def callback_on_step_end(pipe, i, t, callback_kwargs): + if i == interrupt_step_idx: + pipe._interrupt = True + + return callback_kwargs + + output_interrupted = sd_pipe( + prompt, + image=inputs["image"], + num_inference_steps=num_inference_steps, + output_type="latent", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=callback_on_step_end, + ).images + + # fetch intermediate latents at the interrupted step + # from the completed generation process + intermediate_latent = pipe_state.state[interrupt_step_idx] + + # compare the intermediate latent to the output of the interrupted process + # they should be the same + assert torch.allclose(intermediate_latent, output_interrupted, atol=1e-4) + + +@slow +@require_torch_gpu +class StableDiffusionImg2ImgPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/sketch-mountains-input.png" + ) + inputs = { + "prompt": "a fantasy landscape, concept art, high resolution", + "image": init_image, + "generator": generator, + "num_inference_steps": 3, + "strength": 0.75, + "guidance_scale": 7.5, + "output_type": "np", + } + return inputs + + def test_stable_diffusion_img2img_default(self): + pipe = StableDiffusionImg2ImgPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 768, 3) + expected_slice = np.array([0.4300, 0.4662, 0.4930, 0.3990, 0.4307, 0.4525, 0.3719, 0.4064, 0.3923]) + + assert np.abs(expected_slice - image_slice).max() < 1e-3 + + def test_stable_diffusion_img2img_k_lms(self): + pipe = StableDiffusionImg2ImgPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 768, 3) + expected_slice = np.array([0.0389, 0.0346, 0.0415, 0.0290, 0.0218, 0.0210, 0.0408, 0.0567, 0.0271]) + + assert np.abs(expected_slice - image_slice).max() < 1e-3 + + def test_stable_diffusion_img2img_ddim(self): + pipe = StableDiffusionImg2ImgPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", safety_checker=None) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 768, 3) + expected_slice = np.array([0.0593, 0.0607, 0.0851, 0.0582, 0.0636, 0.0721, 0.0751, 0.0981, 0.0781]) + + assert np.abs(expected_slice - image_slice).max() < 1e-3 + + def test_stable_diffusion_img2img_intermediate_state(self): + number_of_steps = 0 + + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True + nonlocal number_of_steps + number_of_steps += 1 + if step == 1: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 96) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array([-0.4958, 0.5107, 1.1045, 2.7539, 4.6680, 3.8320, 1.5049, 1.8633, 2.6523]) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + elif step == 2: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 96) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array([-0.4956, 0.5078, 1.0918, 2.7520, 4.6484, 3.8125, 1.5146, 1.8633, 2.6367]) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + + callback_fn.has_been_called = False + + pipe = StableDiffusionImg2ImgPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", safety_checker=None, torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == 2 + + def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe = StableDiffusionImg2ImgPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", safety_checker=None, torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.2 GB is allocated + assert mem_bytes < 2.2 * 10**9 + + def test_stable_diffusion_pipeline_with_model_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + + # Normal inference + + pipe = StableDiffusionImg2ImgPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", + safety_checker=None, + torch_dtype=torch.float16, + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe(**inputs) + mem_bytes = torch.cuda.max_memory_allocated() + + # With model offloading + + # Reload but don't move to cuda + pipe = StableDiffusionImg2ImgPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", + safety_checker=None, + torch_dtype=torch.float16, + ) + + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + _ = pipe(**inputs) + mem_bytes_offloaded = torch.cuda.max_memory_allocated() + + assert mem_bytes_offloaded < mem_bytes + for module in pipe.text_encoder, pipe.unet, pipe.vae: + assert module.device == torch.device("cpu") + + def test_img2img_2nd_order(self): + sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.scheduler = HeunDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 10 + inputs["strength"] = 0.75 + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/img2img/img2img_heun.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 5e-2 + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 11 + inputs["strength"] = 0.75 + image_other = sd_pipe(**inputs).images[0] + + mean_diff = np.abs(image - image_other).mean() + + # images should be very similar + assert mean_diff < 5e-2 + + def test_stable_diffusion_img2img_pipeline_multiple_of_8(self): + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/img2img/sketch-mountains-input.jpg" + ) + # resize to resolution that is divisible by 8 but not 16 or 32 + init_image = init_image.resize((760, 504)) + + model_id = "CompVis/stable-diffusion-v1-4" + pipe = StableDiffusionImg2ImgPipeline.from_pretrained( + model_id, + safety_checker=None, + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + prompt = "A fantasy landscape, trending on artstation" + + generator = torch.manual_seed(0) + output = pipe( + prompt=prompt, + image=init_image, + strength=0.75, + guidance_scale=7.5, + generator=generator, + output_type="np", + ) + image = output.images[0] + + image_slice = image[255:258, 383:386, -1] + + assert image.shape == (504, 760, 3) + expected_slice = np.array([0.9393, 0.9500, 0.9399, 0.9438, 0.9458, 0.9400, 0.9455, 0.9414, 0.9423]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-3 + + def test_img2img_safety_checker_works(self): + sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 20 + # make sure the safety checker is activated + inputs["prompt"] = "naked, sex, porn" + out = sd_pipe(**inputs) + + assert out.nsfw_content_detected[0], f"Safety checker should work for prompt: {inputs['prompt']}" + assert np.abs(out.images[0]).sum() < 1e-5 # should be all zeros + + @require_python39_or_higher + @require_torch_2 + def test_img2img_compile(self): + seed = 0 + inputs = self.get_inputs(torch_device, seed=seed) + # Can't pickle a Generator object + del inputs["generator"] + inputs["torch_device"] = torch_device + inputs["seed"] = seed + run_test_in_subprocess(test_case=self, target_func=_test_img2img_compile, inputs=inputs) + + +@nightly +@require_torch_gpu +class StableDiffusionImg2ImgPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/sketch-mountains-input.png" + ) + inputs = { + "prompt": "a fantasy landscape, concept art, high resolution", + "image": init_image, + "generator": generator, + "num_inference_steps": 50, + "strength": 0.75, + "guidance_scale": 7.5, + "output_type": "np", + } + return inputs + + def test_img2img_pndm(self): + sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/stable_diffusion_1_5_pndm.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_img2img_ddim(self): + sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/stable_diffusion_1_5_ddim.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_img2img_lms(self): + sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/stable_diffusion_1_5_lms.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_img2img_dpm(self): + sd_pipe = StableDiffusionImg2ImgPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 30 + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/stable_diffusion_1_5_dpm.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py new file mode 100755 index 0000000..218ac3e --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_inpaint.py @@ -0,0 +1,1668 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import traceback +import unittest + +import numpy as np +import torch +from huggingface_hub import hf_hub_download +from PIL import Image +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AsymmetricAutoencoderKL, + AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, + LCMScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + StableDiffusionInpaintPipeline, + UNet2DConditionModel, +) +from diffusers.models.attention_processor import AttnProcessor +from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion_inpaint import prepare_mask_and_masked_image +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + nightly, + numpy_cosine_similarity_distance, + require_python39_or_higher, + require_torch_2, + require_torch_gpu, + run_test_in_subprocess, + slow, + torch_device, +) + +from ..pipeline_params import ( + TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_INPAINTING_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, +) +from ..test_pipelines_common import ( + IPAdapterTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, +) + + +enable_full_determinism() + + +# Will be run via run_test_in_subprocess +def _test_inpaint_compile(in_queue, out_queue, timeout): + error = None + try: + inputs = in_queue.get(timeout=timeout) + torch_device = inputs.pop("torch_device") + seed = inputs.pop("seed") + inputs["generator"] = torch.Generator(device=torch_device).manual_seed(seed) + + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) + pipe.unet.set_default_attn_processor() + pipe.scheduler = PNDMScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + pipe.unet.to(memory_format=torch.channels_last) + pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True) + + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.0689, 0.0699, 0.0790, 0.0536, 0.0470, 0.0488, 0.041, 0.0508, 0.04179]) + assert np.abs(expected_slice - image_slice).max() < 3e-3 + except Exception: + error = f"{traceback.format_exc()}" + + results = {"error": error} + out_queue.put(results, timeout=timeout) + out_queue.join() + + +class StableDiffusionInpaintPipelineFastTests( + IPAdapterTesterMixin, + PipelineLatentTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineTesterMixin, + unittest.TestCase, +): + pipeline_class = StableDiffusionInpaintPipeline + params = TEXT_GUIDED_IMAGE_INPAINTING_PARAMS + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + image_params = frozenset([]) + # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess + image_latents_params = frozenset([]) + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS.union({"mask", "masked_image_latents"}) + + def get_dummy_components(self, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + time_cond_proj_dim=time_cond_proj_dim, + layers_per_block=2, + sample_size=32, + in_channels=9, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + scheduler = PNDMScheduler(skip_prk_steps=True) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0, img_res=64, output_pil=True): + # TODO: use tensor inputs instead of PIL, this is here just to leave the old expected_slices untouched + if output_pil: + # Get random floats in [0, 1] as image + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + mask_image = torch.ones_like(image) + # Convert image and mask_image to [0, 255] + image = 255 * image + mask_image = 255 * mask_image + # Convert to PIL image + init_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((img_res, img_res)) + mask_image = Image.fromarray(np.uint8(mask_image)).convert("RGB").resize((img_res, img_res)) + else: + # Get random floats in [0, 1] as image with spatial size (img_res, img_res) + image = floats_tensor((1, 3, img_res, img_res), rng=random.Random(seed)).to(device) + # Convert image to [-1, 1] + init_image = 2.0 * image - 1.0 + mask_image = torch.ones((1, 1, img_res, img_res), device=device) + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": init_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_inpaint(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionInpaintPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4703, 0.5697, 0.3879, 0.5470, 0.6042, 0.4413, 0.5078, 0.4728, 0.4469]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_inpaint_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionInpaintPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4931, 0.5988, 0.4569, 0.5556, 0.6650, 0.5087, 0.5966, 0.5358, 0.5269]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_inpaint_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionInpaintPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4931, 0.5988, 0.4569, 0.5556, 0.6650, 0.5087, 0.5966, 0.5358, 0.5269]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_inpaint_image_tensor(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionInpaintPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + out_pil = output.images + + inputs = self.get_dummy_inputs(device) + inputs["image"] = torch.tensor(np.array(inputs["image"]) / 127.5 - 1).permute(2, 0, 1).unsqueeze(0) + inputs["mask_image"] = torch.tensor(np.array(inputs["mask_image"]) / 255).permute(2, 0, 1)[:1].unsqueeze(0) + output = sd_pipe(**inputs) + out_tensor = output.images + + assert out_pil.shape == (1, 64, 64, 3) + assert np.abs(out_pil.flatten() - out_tensor.flatten()).max() < 5e-2 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + def test_stable_diffusion_inpaint_strength_zero_test(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionInpaintPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + + # check that the pipeline raises value error when num_inference_steps is < 1 + inputs["strength"] = 0.01 + with self.assertRaises(ValueError): + sd_pipe(**inputs).images + + def test_stable_diffusion_inpaint_mask_latents(self): + device = "cpu" + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(device) + sd_pipe.set_progress_bar_config(disable=None) + + # normal mask + normal image + ## `image`: pil, `mask_image``: pil, `masked_image_latents``: None + inputs = self.get_dummy_inputs(device) + inputs["strength"] = 0.9 + out_0 = sd_pipe(**inputs).images + + # image latents + mask latents + inputs = self.get_dummy_inputs(device) + image = sd_pipe.image_processor.preprocess(inputs["image"]).to(sd_pipe.device) + mask = sd_pipe.mask_processor.preprocess(inputs["mask_image"]).to(sd_pipe.device) + masked_image = image * (mask < 0.5) + + generator = torch.Generator(device=device).manual_seed(0) + image_latents = ( + sd_pipe.vae.encode(image).latent_dist.sample(generator=generator) * sd_pipe.vae.config.scaling_factor + ) + torch.randn((1, 4, 32, 32), generator=generator) + mask_latents = ( + sd_pipe.vae.encode(masked_image).latent_dist.sample(generator=generator) + * sd_pipe.vae.config.scaling_factor + ) + inputs["image"] = image_latents + inputs["masked_image_latents"] = mask_latents + inputs["mask_image"] = mask + inputs["strength"] = 0.9 + generator = torch.Generator(device=device).manual_seed(0) + torch.randn((1, 4, 32, 32), generator=generator) + inputs["generator"] = generator + out_1 = sd_pipe(**inputs).images + assert np.abs(out_0 - out_1).max() < 1e-2 + + def test_pipeline_interrupt(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionInpaintPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + prompt = "hey" + num_inference_steps = 3 + + # store intermediate latents from the generation process + class PipelineState: + def __init__(self): + self.state = [] + + def apply(self, pipe, i, t, callback_kwargs): + self.state.append(callback_kwargs["latents"]) + return callback_kwargs + + pipe_state = PipelineState() + sd_pipe( + prompt, + image=inputs["image"], + mask_image=inputs["mask_image"], + num_inference_steps=num_inference_steps, + output_type="np", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=pipe_state.apply, + ).images + + # interrupt generation at step index + interrupt_step_idx = 1 + + def callback_on_step_end(pipe, i, t, callback_kwargs): + if i == interrupt_step_idx: + pipe._interrupt = True + + return callback_kwargs + + output_interrupted = sd_pipe( + prompt, + image=inputs["image"], + mask_image=inputs["mask_image"], + num_inference_steps=num_inference_steps, + output_type="latent", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=callback_on_step_end, + ).images + + # fetch intermediate latents at the interrupted step + # from the completed generation process + intermediate_latent = pipe_state.state[interrupt_step_idx] + + # compare the intermediate latent to the output of the interrupted process + # they should be the same + assert torch.allclose(intermediate_latent, output_interrupted, atol=1e-4) + + +class StableDiffusionSimpleInpaintPipelineFastTests(StableDiffusionInpaintPipelineFastTests): + pipeline_class = StableDiffusionInpaintPipeline + params = TEXT_GUIDED_IMAGE_INPAINTING_PARAMS + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + image_params = frozenset([]) + # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess + + def get_dummy_components(self, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + time_cond_proj_dim=time_cond_proj_dim, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + scheduler = PNDMScheduler(skip_prk_steps=True) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs_2images(self, device, seed=0, img_res=64): + # Get random floats in [0, 1] as image with spatial size (img_res, img_res) + image1 = floats_tensor((1, 3, img_res, img_res), rng=random.Random(seed)).to(device) + image2 = floats_tensor((1, 3, img_res, img_res), rng=random.Random(seed + 22)).to(device) + # Convert images to [-1, 1] + init_image1 = 2.0 * image1 - 1.0 + init_image2 = 2.0 * image2 - 1.0 + + # empty mask + mask_image = torch.zeros((1, 1, img_res, img_res), device=device) + + if str(device).startswith("mps"): + generator1 = torch.manual_seed(seed) + generator2 = torch.manual_seed(seed) + else: + generator1 = torch.Generator(device=device).manual_seed(seed) + generator2 = torch.Generator(device=device).manual_seed(seed) + + inputs = { + "prompt": ["A painting of a squirrel eating a burger"] * 2, + "image": [init_image1, init_image2], + "mask_image": [mask_image] * 2, + "generator": [generator1, generator2], + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_inpaint(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionInpaintPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.6584, 0.5424, 0.5649, 0.5449, 0.5897, 0.6111, 0.5404, 0.5463, 0.5214]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_inpaint_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionInpaintPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.6240, 0.5355, 0.5649, 0.5378, 0.5374, 0.6242, 0.5132, 0.5347, 0.5396]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_inpaint_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionInpaintPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.6240, 0.5355, 0.5649, 0.5378, 0.5374, 0.6242, 0.5132, 0.5347, 0.5396]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_inpaint_2_images(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + # test to confirm if we pass two same image, we will get same output + inputs = self.get_dummy_inputs(device) + gen1 = torch.Generator(device=device).manual_seed(0) + gen2 = torch.Generator(device=device).manual_seed(0) + for name in ["prompt", "image", "mask_image"]: + inputs[name] = [inputs[name]] * 2 + inputs["generator"] = [gen1, gen2] + images = sd_pipe(**inputs).images + + assert images.shape == (2, 64, 64, 3) + + image_slice1 = images[0, -3:, -3:, -1] + image_slice2 = images[1, -3:, -3:, -1] + assert np.abs(image_slice1.flatten() - image_slice2.flatten()).max() < 1e-4 + + # test to confirm that if we pass two different images, we will get different output + inputs = self.get_dummy_inputs_2images(device) + images = sd_pipe(**inputs).images + assert images.shape == (2, 64, 64, 3) + + image_slice1 = images[0, -3:, -3:, -1] + image_slice2 = images[1, -3:, -3:, -1] + assert np.abs(image_slice1.flatten() - image_slice2.flatten()).max() > 1e-2 + + +@slow +@require_torch_gpu +class StableDiffusionInpaintPipelineSlowTests(unittest.TestCase): + def setUp(self): + super().setUp() + + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_image.png" + ) + mask_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_mask.png" + ) + inputs = { + "prompt": "Face of a yellow cat, high resolution, sitting on a park bench", + "image": init_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_inpaint_ddim(self): + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.0427, 0.0460, 0.0483, 0.0460, 0.0584, 0.0521, 0.1549, 0.1695, 0.1794]) + + assert np.abs(expected_slice - image_slice).max() < 6e-4 + + def test_stable_diffusion_inpaint_fp16(self): + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", torch_dtype=torch.float16, safety_checker=None + ) + pipe.unet.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.1509, 0.1245, 0.1672, 0.1655, 0.1519, 0.1226, 0.1462, 0.1567, 0.2451]) + assert np.abs(expected_slice - image_slice).max() < 1e-1 + + def test_stable_diffusion_inpaint_pndm(self): + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) + pipe.scheduler = PNDMScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.0425, 0.0273, 0.0344, 0.1694, 0.1727, 0.1812, 0.3256, 0.3311, 0.3272]) + + assert np.abs(expected_slice - image_slice).max() < 5e-3 + + def test_stable_diffusion_inpaint_k_lms(self): + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.9314, 0.7575, 0.9432, 0.8885, 0.9028, 0.7298, 0.9811, 0.9667, 0.7633]) + + assert np.abs(expected_slice - image_slice).max() < 6e-3 + + def test_stable_diffusion_inpaint_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None, torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.2 GB is allocated + assert mem_bytes < 2.2 * 10**9 + + @require_python39_or_higher + @require_torch_2 + def test_inpaint_compile(self): + seed = 0 + inputs = self.get_inputs(torch_device, seed=seed) + # Can't pickle a Generator object + del inputs["generator"] + inputs["torch_device"] = torch_device + inputs["seed"] = seed + run_test_in_subprocess(test_case=self, target_func=_test_inpaint_compile, inputs=inputs) + + def test_stable_diffusion_inpaint_pil_input_resolution_test(self): + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + # change input image to a random size (one that would cause a tensor mismatch error) + inputs["image"] = inputs["image"].resize((127, 127)) + inputs["mask_image"] = inputs["mask_image"].resize((127, 127)) + inputs["height"] = 128 + inputs["width"] = 128 + image = pipe(**inputs).images + # verify that the returned image has the same height and width as the input height and width + assert image.shape == (1, inputs["height"], inputs["width"], 3) + + def test_stable_diffusion_inpaint_strength_test(self): + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.unet.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + # change input strength + inputs["strength"] = 0.75 + image = pipe(**inputs).images + # verify that the returned image has the same height and width as the input height and width + assert image.shape == (1, 512, 512, 3) + + image_slice = image[0, 253:256, 253:256, -1].flatten() + expected_slice = np.array([0.2728, 0.2803, 0.2665, 0.2511, 0.2774, 0.2586, 0.2391, 0.2392, 0.2582]) + assert np.abs(expected_slice - image_slice).max() < 1e-3 + + def test_stable_diffusion_simple_inpaint_ddim(self): + pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", safety_checker=None) + pipe.unet.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.3757, 0.3875, 0.4445, 0.4353, 0.3780, 0.4513, 0.3965, 0.3984, 0.4362]) + assert np.abs(expected_slice - image_slice).max() < 1e-3 + + def test_download_local(self): + filename = hf_hub_download("runwayml/stable-diffusion-inpainting", filename="sd-v1-5-inpainting.ckpt") + + pipe = StableDiffusionInpaintPipeline.from_single_file(filename, torch_dtype=torch.float16) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.to("cuda") + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 1 + image_out = pipe(**inputs).images[0] + + assert image_out.shape == (512, 512, 3) + + def test_download_ckpt_diff_format_is_same(self): + ckpt_path = "https://huggingface.co/runwayml/stable-diffusion-inpainting/blob/main/sd-v1-5-inpainting.ckpt" + + pipe = StableDiffusionInpaintPipeline.from_single_file(ckpt_path) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.unet.set_attn_processor(AttnProcessor()) + pipe.to("cuda") + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 5 + image_ckpt = pipe(**inputs).images[0] + + pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.unet.set_attn_processor(AttnProcessor()) + pipe.to("cuda") + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 5 + image = pipe(**inputs).images[0] + + max_diff = numpy_cosine_similarity_distance(image.flatten(), image_ckpt.flatten()) + + assert max_diff < 1e-4 + + def test_single_file_component_configs(self): + pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting", variant="fp16") + + ckpt_path = "https://huggingface.co/runwayml/stable-diffusion-inpainting/blob/main/sd-v1-5-inpainting.ckpt" + single_file_pipe = StableDiffusionInpaintPipeline.from_single_file(ckpt_path, load_safety_checker=True) + + for param_name, param_value in single_file_pipe.text_encoder.config.to_dict().items(): + if param_name in ["torch_dtype", "architectures", "_name_or_path"]: + continue + assert pipe.text_encoder.config.to_dict()[param_name] == param_value + + PARAMS_TO_IGNORE = ["torch_dtype", "_name_or_path", "architectures", "_use_default_values"] + for param_name, param_value in single_file_pipe.unet.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.unet.config[param_name] == param_value + ), f"{param_name} is differs between single file loading and pretrained loading" + + for param_name, param_value in single_file_pipe.vae.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.vae.config[param_name] == param_value + ), f"{param_name} is differs between single file loading and pretrained loading" + + for param_name, param_value in single_file_pipe.safety_checker.config.to_dict().items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.safety_checker.config.to_dict()[param_name] == param_value + ), f"{param_name} is differs between single file loading and pretrained loading" + + +@slow +@require_torch_gpu +class StableDiffusionInpaintPipelineAsymmetricAutoencoderKLSlowTests(unittest.TestCase): + def setUp(self): + super().setUp() + + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_image.png" + ) + mask_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_mask.png" + ) + inputs = { + "prompt": "Face of a yellow cat, high resolution, sitting on a park bench", + "image": init_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_inpaint_ddim(self): + vae = AsymmetricAutoencoderKL.from_pretrained("cross-attention/asymmetric-autoencoder-kl-x-1-5") + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) + pipe.vae = vae + pipe.unet.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.0522, 0.0604, 0.0596, 0.0449, 0.0493, 0.0427, 0.1186, 0.1289, 0.1442]) + + assert np.abs(expected_slice - image_slice).max() < 1e-3 + + def test_stable_diffusion_inpaint_fp16(self): + vae = AsymmetricAutoencoderKL.from_pretrained( + "cross-attention/asymmetric-autoencoder-kl-x-1-5", torch_dtype=torch.float16 + ) + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", torch_dtype=torch.float16, safety_checker=None + ) + pipe.unet.set_default_attn_processor() + pipe.vae = vae + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.1343, 0.1406, 0.1440, 0.1504, 0.1729, 0.0989, 0.1807, 0.2822, 0.1179]) + + assert np.abs(expected_slice - image_slice).max() < 5e-2 + + def test_stable_diffusion_inpaint_pndm(self): + vae = AsymmetricAutoencoderKL.from_pretrained("cross-attention/asymmetric-autoencoder-kl-x-1-5") + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) + pipe.unet.set_default_attn_processor() + pipe.vae = vae + pipe.scheduler = PNDMScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.0966, 0.1083, 0.1148, 0.1422, 0.1318, 0.1197, 0.3702, 0.3537, 0.3288]) + + assert np.abs(expected_slice - image_slice).max() < 5e-3 + + def test_stable_diffusion_inpaint_k_lms(self): + vae = AsymmetricAutoencoderKL.from_pretrained("cross-attention/asymmetric-autoencoder-kl-x-1-5") + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) + pipe.unet.set_default_attn_processor() + pipe.vae = vae + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.8931, 0.8683, 0.8965, 0.8501, 0.8592, 0.9118, 0.8734, 0.7463, 0.8990]) + assert np.abs(expected_slice - image_slice).max() < 6e-3 + + def test_stable_diffusion_inpaint_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + vae = AsymmetricAutoencoderKL.from_pretrained( + "cross-attention/asymmetric-autoencoder-kl-x-1-5", torch_dtype=torch.float16 + ) + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None, torch_dtype=torch.float16 + ) + pipe.vae = vae + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.45 GB is allocated + assert mem_bytes < 2.45 * 10**9 + + @require_python39_or_higher + @require_torch_2 + def test_inpaint_compile(self): + pass + + def test_stable_diffusion_inpaint_pil_input_resolution_test(self): + vae = AsymmetricAutoencoderKL.from_pretrained( + "cross-attention/asymmetric-autoencoder-kl-x-1-5", + ) + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) + pipe.vae = vae + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + # change input image to a random size (one that would cause a tensor mismatch error) + inputs["image"] = inputs["image"].resize((127, 127)) + inputs["mask_image"] = inputs["mask_image"].resize((127, 127)) + inputs["height"] = 128 + inputs["width"] = 128 + image = pipe(**inputs).images + # verify that the returned image has the same height and width as the input height and width + assert image.shape == (1, inputs["height"], inputs["width"], 3) + + def test_stable_diffusion_inpaint_strength_test(self): + vae = AsymmetricAutoencoderKL.from_pretrained("cross-attention/asymmetric-autoencoder-kl-x-1-5") + pipe = StableDiffusionInpaintPipeline.from_pretrained( + "runwayml/stable-diffusion-inpainting", safety_checker=None + ) + pipe.unet.set_default_attn_processor() + pipe.vae = vae + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + # change input strength + inputs["strength"] = 0.75 + image = pipe(**inputs).images + # verify that the returned image has the same height and width as the input height and width + assert image.shape == (1, 512, 512, 3) + + image_slice = image[0, 253:256, 253:256, -1].flatten() + expected_slice = np.array([0.2458, 0.2576, 0.3124, 0.2679, 0.2669, 0.2796, 0.2872, 0.2975, 0.2661]) + assert np.abs(expected_slice - image_slice).max() < 3e-3 + + def test_stable_diffusion_simple_inpaint_ddim(self): + vae = AsymmetricAutoencoderKL.from_pretrained("cross-attention/asymmetric-autoencoder-kl-x-1-5") + pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", safety_checker=None) + pipe.vae = vae + pipe.unet.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.3296, 0.4041, 0.4097, 0.4145, 0.4342, 0.4152, 0.4927, 0.4931, 0.4430]) + assert np.abs(expected_slice - image_slice).max() < 1e-3 + + def test_download_local(self): + vae = AsymmetricAutoencoderKL.from_pretrained( + "cross-attention/asymmetric-autoencoder-kl-x-1-5", torch_dtype=torch.float16 + ) + filename = hf_hub_download("runwayml/stable-diffusion-inpainting", filename="sd-v1-5-inpainting.ckpt") + + pipe = StableDiffusionInpaintPipeline.from_single_file(filename, torch_dtype=torch.float16) + pipe.vae = vae + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.to("cuda") + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 1 + image_out = pipe(**inputs).images[0] + + assert image_out.shape == (512, 512, 3) + + def test_download_ckpt_diff_format_is_same(self): + pass + + +@nightly +@require_torch_gpu +class StableDiffusionInpaintPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_image.png" + ) + mask_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/input_bench_mask.png" + ) + inputs = { + "prompt": "Face of a yellow cat, high resolution, sitting on a park bench", + "image": init_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 50, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_inpaint_ddim(self): + sd_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/stable_diffusion_inpaint_ddim.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_inpaint_pndm(self): + sd_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") + sd_pipe.scheduler = PNDMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/stable_diffusion_inpaint_pndm.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_inpaint_lms(self): + sd_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/stable_diffusion_inpaint_lms.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_inpaint_dpm(self): + sd_pipe = StableDiffusionInpaintPipeline.from_pretrained("runwayml/stable-diffusion-inpainting") + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 30 + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_inpaint/stable_diffusion_inpaint_dpm_multi.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + +class StableDiffusionInpaintingPrepareMaskAndMaskedImageTests(unittest.TestCase): + def test_pil_inputs(self): + height, width = 32, 32 + im = np.random.randint(0, 255, (height, width, 3), dtype=np.uint8) + im = Image.fromarray(im) + mask = np.random.randint(0, 255, (height, width), dtype=np.uint8) > 127.5 + mask = Image.fromarray((mask * 255).astype(np.uint8)) + + t_mask, t_masked, t_image = prepare_mask_and_masked_image(im, mask, height, width, return_image=True) + + self.assertTrue(isinstance(t_mask, torch.Tensor)) + self.assertTrue(isinstance(t_masked, torch.Tensor)) + self.assertTrue(isinstance(t_image, torch.Tensor)) + + self.assertEqual(t_mask.ndim, 4) + self.assertEqual(t_masked.ndim, 4) + self.assertEqual(t_image.ndim, 4) + + self.assertEqual(t_mask.shape, (1, 1, height, width)) + self.assertEqual(t_masked.shape, (1, 3, height, width)) + self.assertEqual(t_image.shape, (1, 3, height, width)) + + self.assertTrue(t_mask.dtype == torch.float32) + self.assertTrue(t_masked.dtype == torch.float32) + self.assertTrue(t_image.dtype == torch.float32) + + self.assertTrue(t_mask.min() >= 0.0) + self.assertTrue(t_mask.max() <= 1.0) + self.assertTrue(t_masked.min() >= -1.0) + self.assertTrue(t_masked.min() <= 1.0) + self.assertTrue(t_image.min() >= -1.0) + self.assertTrue(t_image.min() >= -1.0) + + self.assertTrue(t_mask.sum() > 0.0) + + def test_np_inputs(self): + height, width = 32, 32 + + im_np = np.random.randint(0, 255, (height, width, 3), dtype=np.uint8) + im_pil = Image.fromarray(im_np) + mask_np = ( + np.random.randint( + 0, + 255, + ( + height, + width, + ), + dtype=np.uint8, + ) + > 127.5 + ) + mask_pil = Image.fromarray((mask_np * 255).astype(np.uint8)) + + t_mask_np, t_masked_np, t_image_np = prepare_mask_and_masked_image( + im_np, mask_np, height, width, return_image=True + ) + t_mask_pil, t_masked_pil, t_image_pil = prepare_mask_and_masked_image( + im_pil, mask_pil, height, width, return_image=True + ) + + self.assertTrue((t_mask_np == t_mask_pil).all()) + self.assertTrue((t_masked_np == t_masked_pil).all()) + self.assertTrue((t_image_np == t_image_pil).all()) + + def test_torch_3D_2D_inputs(self): + height, width = 32, 32 + + im_tensor = torch.randint( + 0, + 255, + ( + 3, + height, + width, + ), + dtype=torch.uint8, + ) + mask_tensor = ( + torch.randint( + 0, + 255, + ( + height, + width, + ), + dtype=torch.uint8, + ) + > 127.5 + ) + im_np = im_tensor.numpy().transpose(1, 2, 0) + mask_np = mask_tensor.numpy() + + t_mask_tensor, t_masked_tensor, t_image_tensor = prepare_mask_and_masked_image( + im_tensor / 127.5 - 1, mask_tensor, height, width, return_image=True + ) + t_mask_np, t_masked_np, t_image_np = prepare_mask_and_masked_image( + im_np, mask_np, height, width, return_image=True + ) + + self.assertTrue((t_mask_tensor == t_mask_np).all()) + self.assertTrue((t_masked_tensor == t_masked_np).all()) + self.assertTrue((t_image_tensor == t_image_np).all()) + + def test_torch_3D_3D_inputs(self): + height, width = 32, 32 + + im_tensor = torch.randint( + 0, + 255, + ( + 3, + height, + width, + ), + dtype=torch.uint8, + ) + mask_tensor = ( + torch.randint( + 0, + 255, + ( + 1, + height, + width, + ), + dtype=torch.uint8, + ) + > 127.5 + ) + im_np = im_tensor.numpy().transpose(1, 2, 0) + mask_np = mask_tensor.numpy()[0] + + t_mask_tensor, t_masked_tensor, t_image_tensor = prepare_mask_and_masked_image( + im_tensor / 127.5 - 1, mask_tensor, height, width, return_image=True + ) + t_mask_np, t_masked_np, t_image_np = prepare_mask_and_masked_image( + im_np, mask_np, height, width, return_image=True + ) + + self.assertTrue((t_mask_tensor == t_mask_np).all()) + self.assertTrue((t_masked_tensor == t_masked_np).all()) + self.assertTrue((t_image_tensor == t_image_np).all()) + + def test_torch_4D_2D_inputs(self): + height, width = 32, 32 + + im_tensor = torch.randint( + 0, + 255, + ( + 1, + 3, + height, + width, + ), + dtype=torch.uint8, + ) + mask_tensor = ( + torch.randint( + 0, + 255, + ( + height, + width, + ), + dtype=torch.uint8, + ) + > 127.5 + ) + im_np = im_tensor.numpy()[0].transpose(1, 2, 0) + mask_np = mask_tensor.numpy() + + t_mask_tensor, t_masked_tensor, t_image_tensor = prepare_mask_and_masked_image( + im_tensor / 127.5 - 1, mask_tensor, height, width, return_image=True + ) + t_mask_np, t_masked_np, t_image_np = prepare_mask_and_masked_image( + im_np, mask_np, height, width, return_image=True + ) + + self.assertTrue((t_mask_tensor == t_mask_np).all()) + self.assertTrue((t_masked_tensor == t_masked_np).all()) + self.assertTrue((t_image_tensor == t_image_np).all()) + + def test_torch_4D_3D_inputs(self): + height, width = 32, 32 + + im_tensor = torch.randint( + 0, + 255, + ( + 1, + 3, + height, + width, + ), + dtype=torch.uint8, + ) + mask_tensor = ( + torch.randint( + 0, + 255, + ( + 1, + height, + width, + ), + dtype=torch.uint8, + ) + > 127.5 + ) + im_np = im_tensor.numpy()[0].transpose(1, 2, 0) + mask_np = mask_tensor.numpy()[0] + + t_mask_tensor, t_masked_tensor, t_image_tensor = prepare_mask_and_masked_image( + im_tensor / 127.5 - 1, mask_tensor, height, width, return_image=True + ) + t_mask_np, t_masked_np, t_image_np = prepare_mask_and_masked_image( + im_np, mask_np, height, width, return_image=True + ) + + self.assertTrue((t_mask_tensor == t_mask_np).all()) + self.assertTrue((t_masked_tensor == t_masked_np).all()) + self.assertTrue((t_image_tensor == t_image_np).all()) + + def test_torch_4D_4D_inputs(self): + height, width = 32, 32 + + im_tensor = torch.randint( + 0, + 255, + ( + 1, + 3, + height, + width, + ), + dtype=torch.uint8, + ) + mask_tensor = ( + torch.randint( + 0, + 255, + ( + 1, + 1, + height, + width, + ), + dtype=torch.uint8, + ) + > 127.5 + ) + im_np = im_tensor.numpy()[0].transpose(1, 2, 0) + mask_np = mask_tensor.numpy()[0][0] + + t_mask_tensor, t_masked_tensor, t_image_tensor = prepare_mask_and_masked_image( + im_tensor / 127.5 - 1, mask_tensor, height, width, return_image=True + ) + t_mask_np, t_masked_np, t_image_np = prepare_mask_and_masked_image( + im_np, mask_np, height, width, return_image=True + ) + + self.assertTrue((t_mask_tensor == t_mask_np).all()) + self.assertTrue((t_masked_tensor == t_masked_np).all()) + self.assertTrue((t_image_tensor == t_image_np).all()) + + def test_torch_batch_4D_3D(self): + height, width = 32, 32 + + im_tensor = torch.randint( + 0, + 255, + ( + 2, + 3, + height, + width, + ), + dtype=torch.uint8, + ) + mask_tensor = ( + torch.randint( + 0, + 255, + ( + 2, + height, + width, + ), + dtype=torch.uint8, + ) + > 127.5 + ) + + im_nps = [im.numpy().transpose(1, 2, 0) for im in im_tensor] + mask_nps = [mask.numpy() for mask in mask_tensor] + + t_mask_tensor, t_masked_tensor, t_image_tensor = prepare_mask_and_masked_image( + im_tensor / 127.5 - 1, mask_tensor, height, width, return_image=True + ) + nps = [prepare_mask_and_masked_image(i, m, height, width, return_image=True) for i, m in zip(im_nps, mask_nps)] + t_mask_np = torch.cat([n[0] for n in nps]) + t_masked_np = torch.cat([n[1] for n in nps]) + t_image_np = torch.cat([n[2] for n in nps]) + + self.assertTrue((t_mask_tensor == t_mask_np).all()) + self.assertTrue((t_masked_tensor == t_masked_np).all()) + self.assertTrue((t_image_tensor == t_image_np).all()) + + def test_torch_batch_4D_4D(self): + height, width = 32, 32 + + im_tensor = torch.randint( + 0, + 255, + ( + 2, + 3, + height, + width, + ), + dtype=torch.uint8, + ) + mask_tensor = ( + torch.randint( + 0, + 255, + ( + 2, + 1, + height, + width, + ), + dtype=torch.uint8, + ) + > 127.5 + ) + + im_nps = [im.numpy().transpose(1, 2, 0) for im in im_tensor] + mask_nps = [mask.numpy()[0] for mask in mask_tensor] + + t_mask_tensor, t_masked_tensor, t_image_tensor = prepare_mask_and_masked_image( + im_tensor / 127.5 - 1, mask_tensor, height, width, return_image=True + ) + nps = [prepare_mask_and_masked_image(i, m, height, width, return_image=True) for i, m in zip(im_nps, mask_nps)] + t_mask_np = torch.cat([n[0] for n in nps]) + t_masked_np = torch.cat([n[1] for n in nps]) + t_image_np = torch.cat([n[2] for n in nps]) + + self.assertTrue((t_mask_tensor == t_mask_np).all()) + self.assertTrue((t_masked_tensor == t_masked_np).all()) + self.assertTrue((t_image_tensor == t_image_np).all()) + + def test_shape_mismatch(self): + height, width = 32, 32 + + # test height and width + with self.assertRaises(AssertionError): + prepare_mask_and_masked_image( + torch.randn( + 3, + height, + width, + ), + torch.randn(64, 64), + height, + width, + return_image=True, + ) + # test batch dim + with self.assertRaises(AssertionError): + prepare_mask_and_masked_image( + torch.randn( + 2, + 3, + height, + width, + ), + torch.randn(4, 64, 64), + height, + width, + return_image=True, + ) + # test batch dim + with self.assertRaises(AssertionError): + prepare_mask_and_masked_image( + torch.randn( + 2, + 3, + height, + width, + ), + torch.randn(4, 1, 64, 64), + height, + width, + return_image=True, + ) + + def test_type_mismatch(self): + height, width = 32, 32 + + # test tensors-only + with self.assertRaises(TypeError): + prepare_mask_and_masked_image( + torch.rand( + 3, + height, + width, + ), + torch.rand( + 3, + height, + width, + ).numpy(), + height, + width, + return_image=True, + ) + # test tensors-only + with self.assertRaises(TypeError): + prepare_mask_and_masked_image( + torch.rand( + 3, + height, + width, + ).numpy(), + torch.rand( + 3, + height, + width, + ), + height, + width, + return_image=True, + ) + + def test_channels_first(self): + height, width = 32, 32 + + # test channels first for 3D tensors + with self.assertRaises(AssertionError): + prepare_mask_and_masked_image( + torch.rand(height, width, 3), + torch.rand( + 3, + height, + width, + ), + height, + width, + return_image=True, + ) + + def test_tensor_range(self): + height, width = 32, 32 + + # test im <= 1 + with self.assertRaises(ValueError): + prepare_mask_and_masked_image( + torch.ones( + 3, + height, + width, + ) + * 2, + torch.rand( + height, + width, + ), + height, + width, + return_image=True, + ) + # test im >= -1 + with self.assertRaises(ValueError): + prepare_mask_and_masked_image( + torch.ones( + 3, + height, + width, + ) + * (-2), + torch.rand( + height, + width, + ), + height, + width, + return_image=True, + ) + # test mask <= 1 + with self.assertRaises(ValueError): + prepare_mask_and_masked_image( + torch.rand( + 3, + height, + width, + ), + torch.ones( + height, + width, + ) + * 2, + height, + width, + return_image=True, + ) + # test mask >= 0 + with self.assertRaises(ValueError): + prepare_mask_and_masked_image( + torch.rand( + 3, + height, + width, + ), + torch.ones( + height, + width, + ) + * -1, + height, + width, + return_image=True, + ) diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_instruction_pix2pix.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_instruction_pix2pix.py new file mode 100755 index 0000000..0986f02 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion/test_stable_diffusion_instruction_pix2pix.py @@ -0,0 +1,426 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + EulerAncestralDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + StableDiffusionInstructPix2PixPipeline, + UNet2DConditionModel, +) +from diffusers.image_processor import VaeImageProcessor +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + require_torch_gpu, + slow, + torch_device, +) + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, +) +from ..test_pipelines_common import ( + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, +) + + +enable_full_determinism() + + +class StableDiffusionInstructPix2PixPipelineFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionInstructPix2PixPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width", "cross_attention_kwargs"} + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + image_latents_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS.union({"image_latents"}) - {"negative_prompt_embeds"} + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=8, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + scheduler = PNDMScheduler(skip_prk_steps=True) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + image = Image.fromarray(np.uint8(image)).convert("RGB") + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "image_guidance_scale": 1, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_pix2pix_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionInstructPix2PixPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.7526, 0.3750, 0.4547, 0.6117, 0.5866, 0.5016, 0.4327, 0.5642, 0.4815]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_pix2pix_negative_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionInstructPix2PixPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + negative_prompt = "french fries" + output = sd_pipe(**inputs, negative_prompt=negative_prompt) + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.7511, 0.3642, 0.4553, 0.6236, 0.5797, 0.5013, 0.4343, 0.5611, 0.4831]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_pix2pix_multiple_init_images(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionInstructPix2PixPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["prompt"] = [inputs["prompt"]] * 2 + + image = np.array(inputs["image"]).astype(np.float32) / 255.0 + image = torch.from_numpy(image).unsqueeze(0).to(device) + image = image / 2 + 0.5 + image = image.permute(0, 3, 1, 2) + inputs["image"] = image.repeat(2, 1, 1, 1) + + image = sd_pipe(**inputs).images + image_slice = image[-1, -3:, -3:, -1] + + assert image.shape == (2, 32, 32, 3) + expected_slice = np.array([0.5812, 0.5748, 0.5222, 0.5908, 0.5695, 0.7174, 0.6804, 0.5523, 0.5579]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_pix2pix_euler(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = EulerAncestralDiscreteScheduler( + beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear" + ) + sd_pipe = StableDiffusionInstructPix2PixPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + slice = [round(x, 4) for x in image_slice.flatten().tolist()] + print(",".join([str(x) for x in slice])) + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.7417, 0.3842, 0.4732, 0.5776, 0.5891, 0.5139, 0.4052, 0.5673, 0.4986]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + # Overwrite the default test_latents_inputs because pix2pix encode the image differently + def test_latents_input(self): + components = self.get_dummy_components() + pipe = StableDiffusionInstructPix2PixPipeline(**components) + pipe.image_processor = VaeImageProcessor(do_resize=False, do_normalize=False) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + out = pipe(**self.get_dummy_inputs_by_type(torch_device, input_image_type="pt"))[0] + + vae = components["vae"] + inputs = self.get_dummy_inputs_by_type(torch_device, input_image_type="pt") + + for image_param in self.image_latents_params: + if image_param in inputs.keys(): + inputs[image_param] = vae.encode(inputs[image_param]).latent_dist.mode() + + out_latents_inputs = pipe(**inputs)[0] + + max_diff = np.abs(out - out_latents_inputs).max() + self.assertLess(max_diff, 1e-4, "passing latents as image input generate different result from passing image") + + # Override the default test_callback_cfg because pix2pix create inputs for cfg differently + def test_callback_cfg(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + def callback_no_cfg(pipe, i, t, callback_kwargs): + if i == 1: + for k, w in callback_kwargs.items(): + if k in self.callback_cfg_params: + callback_kwargs[k] = callback_kwargs[k].chunk(3)[0] + pipe._guidance_scale = 1.0 + + return callback_kwargs + + inputs = self.get_dummy_inputs(torch_device) + inputs["guidance_scale"] = 1.0 + inputs["num_inference_steps"] = 2 + out_no_cfg = pipe(**inputs)[0] + + inputs["guidance_scale"] = 7.5 + inputs["callback_on_step_end"] = callback_no_cfg + inputs["callback_on_step_end_tensor_inputs"] = pipe._callback_tensor_inputs + out_callback_no_cfg = pipe(**inputs)[0] + + assert out_no_cfg.shape == out_callback_no_cfg.shape + + +@slow +@require_torch_gpu +class StableDiffusionInstructPix2PixPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, seed=0): + generator = torch.manual_seed(seed) + image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main/stable_diffusion_pix2pix/example.jpg" + ) + inputs = { + "prompt": "turn him into a cyborg", + "image": image, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "image_guidance_scale": 1.0, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_pix2pix_default(self): + pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained( + "timbrooks/instruct-pix2pix", safety_checker=None + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.5902, 0.6015, 0.6027, 0.5983, 0.6092, 0.6061, 0.5765, 0.5785, 0.5555]) + + assert np.abs(expected_slice - image_slice).max() < 1e-3 + + def test_stable_diffusion_pix2pix_k_lms(self): + pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained( + "timbrooks/instruct-pix2pix", safety_checker=None + ) + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.6578, 0.6817, 0.6972, 0.6761, 0.6856, 0.6916, 0.6428, 0.6516, 0.6301]) + + assert np.abs(expected_slice - image_slice).max() < 1e-3 + + def test_stable_diffusion_pix2pix_ddim(self): + pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained( + "timbrooks/instruct-pix2pix", safety_checker=None + ) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.3828, 0.3834, 0.3818, 0.3792, 0.3865, 0.3752, 0.3792, 0.3847, 0.3753]) + + assert np.abs(expected_slice - image_slice).max() < 1e-3 + + def test_stable_diffusion_pix2pix_intermediate_state(self): + number_of_steps = 0 + + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True + nonlocal number_of_steps + number_of_steps += 1 + if step == 1: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 64) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array([-0.2463, -0.4644, -0.9756, 1.5176, 1.4414, 0.7866, 0.9897, 0.8521, 0.7983]) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + elif step == 2: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 64) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array([-0.2644, -0.4626, -0.9653, 1.5176, 1.4551, 0.7686, 0.9805, 0.8452, 0.8115]) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + + callback_fn.has_been_called = False + + pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained( + "timbrooks/instruct-pix2pix", safety_checker=None, torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs() + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == 3 + + def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained( + "timbrooks/instruct-pix2pix", safety_checker=None, torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + inputs = self.get_inputs() + _ = pipe(**inputs) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.2 GB is allocated + assert mem_bytes < 2.2 * 10**9 + + def test_stable_diffusion_pix2pix_pipeline_multiple_of_8(self): + inputs = self.get_inputs() + # resize to resolution that is divisible by 8 but not 16 or 32 + inputs["image"] = inputs["image"].resize((504, 504)) + + model_id = "timbrooks/instruct-pix2pix" + pipe = StableDiffusionInstructPix2PixPipeline.from_pretrained( + model_id, + safety_checker=None, + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + output = pipe(**inputs) + image = output.images[0] + + image_slice = image[255:258, 383:386, -1] + + assert image.shape == (504, 504, 3) + expected_slice = np.array([0.2726, 0.2529, 0.2664, 0.2655, 0.2641, 0.2642, 0.2591, 0.2649, 0.2590]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-3 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py new file mode 100755 index 0000000..7aef098 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion.py @@ -0,0 +1,653 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + StableDiffusionPipeline, + UNet2DConditionModel, + logging, +) +from diffusers.utils.testing_utils import ( + CaptureLogger, + backend_empty_cache, + enable_full_determinism, + load_numpy, + nightly, + numpy_cosine_similarity_distance, + require_torch_accelerator, + require_torch_gpu, + skip_mps, + slow, + torch_device, +) + +from ..pipeline_params import ( + TEXT_TO_IMAGE_BATCH_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_PARAMS, +) +from ..test_pipelines_common import ( + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, + SDFunctionTesterMixin, +) + + +enable_full_determinism() + + +class StableDiffusion2PipelineFastTests( + SDFunctionTesterMixin, + PipelineLatentTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineTesterMixin, + unittest.TestCase, +): + pipeline_class = StableDiffusionPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=512, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + generator_device = "cpu" if not device.startswith("cuda") else "cuda" + if not str(device).startswith("mps"): + generator = torch.Generator(device=generator_device).manual_seed(seed) + else: + generator = torch.manual_seed(seed) + + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_ddim(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5753, 0.6113, 0.5005, 0.5036, 0.5464, 0.4725, 0.4982, 0.4865, 0.4861]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_pndm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = PNDMScheduler(skip_prk_steps=True) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5121, 0.5714, 0.4827, 0.5057, 0.5646, 0.4766, 0.5189, 0.4895, 0.4990]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_k_lms(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = LMSDiscreteScheduler.from_config(components["scheduler"].config) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4865, 0.5439, 0.4840, 0.4995, 0.5543, 0.4846, 0.5199, 0.4942, 0.5061]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_k_euler_ancestral(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = EulerAncestralDiscreteScheduler.from_config(components["scheduler"].config) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4864, 0.5440, 0.4842, 0.4994, 0.5543, 0.4846, 0.5196, 0.4942, 0.5063]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_k_euler(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = EulerDiscreteScheduler.from_config(components["scheduler"].config) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4865, 0.5439, 0.4840, 0.4995, 0.5543, 0.4846, 0.5199, 0.4942, 0.5061]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_unflawed(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = DDIMScheduler.from_config( + components["scheduler"].config, timestep_spacing="trailing" + ) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["guidance_rescale"] = 0.7 + inputs["num_inference_steps"] = 10 + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4736, 0.5405, 0.4705, 0.4955, 0.5675, 0.4812, 0.5310, 0.4967, 0.5064]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_long_prompt(self): + components = self.get_dummy_components() + components["scheduler"] = LMSDiscreteScheduler.from_config(components["scheduler"].config) + sd_pipe = StableDiffusionPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + do_classifier_free_guidance = True + negative_prompt = None + num_images_per_prompt = 1 + logger = logging.get_logger("diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion") + logger.setLevel(logging.WARNING) + + prompt = 25 * "@" + with CaptureLogger(logger) as cap_logger_3: + text_embeddings_3, negeative_text_embeddings_3 = sd_pipe.encode_prompt( + prompt, torch_device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + if negeative_text_embeddings_3 is not None: + text_embeddings_3 = torch.cat([negeative_text_embeddings_3, text_embeddings_3]) + + prompt = 100 * "@" + with CaptureLogger(logger) as cap_logger: + text_embeddings, negative_embeddings = sd_pipe.encode_prompt( + prompt, torch_device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + if negative_embeddings is not None: + text_embeddings = torch.cat([negative_embeddings, text_embeddings]) + + negative_prompt = "Hello" + with CaptureLogger(logger) as cap_logger_2: + text_embeddings_2, negative_text_embeddings_2 = sd_pipe.encode_prompt( + prompt, torch_device, num_images_per_prompt, do_classifier_free_guidance, negative_prompt + ) + if negative_text_embeddings_2 is not None: + text_embeddings_2 = torch.cat([negative_text_embeddings_2, text_embeddings_2]) + + assert text_embeddings_3.shape == text_embeddings_2.shape == text_embeddings.shape + assert text_embeddings.shape[1] == 77 + + assert cap_logger.out == cap_logger_2.out + # 100 - 77 + 1 (BOS token) + 1 (EOS token) = 25 + assert cap_logger.out.count("@") == 25 + assert cap_logger_3.out == "" + + def test_attention_slicing_forward_pass(self): + super().test_attention_slicing_forward_pass(expected_max_diff=3e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + +@slow +@require_torch_accelerator +@skip_mps +class StableDiffusion2PipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + backend_empty_cache(torch_device) + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + _generator_device = "cpu" if not generator_device.startswith("cuda") else "cuda" + if not str(device).startswith("mps"): + generator = torch.Generator(device=_generator_device).manual_seed(seed) + else: + generator = torch.manual_seed(seed) + + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_default_ddim(self): + pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-base") + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.49493, 0.47896, 0.40798, 0.54214, 0.53212, 0.48202, 0.47656, 0.46329, 0.48506]) + assert np.abs(image_slice - expected_slice).max() < 7e-3 + + def test_stable_diffusion_pndm(self): + pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-base") + pipe.scheduler = PNDMScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.49493, 0.47896, 0.40798, 0.54214, 0.53212, 0.48202, 0.47656, 0.46329, 0.48506]) + assert np.abs(image_slice - expected_slice).max() < 7e-3 + + def test_stable_diffusion_k_lms(self): + pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-base") + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.10440, 0.13115, 0.11100, 0.10141, 0.11440, 0.07215, 0.11332, 0.09693, 0.10006]) + assert np.abs(image_slice - expected_slice).max() < 3e-3 + + @require_torch_gpu + def test_stable_diffusion_attention_slicing(self): + torch.cuda.reset_peak_memory_stats() + pipe = StableDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-base", torch_dtype=torch.float16 + ) + pipe.unet.set_default_attn_processor() + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + # enable attention slicing + pipe.enable_attention_slicing() + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image_sliced = pipe(**inputs).images + + mem_bytes = torch.cuda.max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + # make sure that less than 3.3 GB is allocated + assert mem_bytes < 3.3 * 10**9 + + # disable slicing + pipe.disable_attention_slicing() + pipe.unet.set_default_attn_processor() + inputs = self.get_inputs(torch_device, dtype=torch.float16) + image = pipe(**inputs).images + + # make sure that more than 3.3 GB is allocated + mem_bytes = torch.cuda.max_memory_allocated() + assert mem_bytes > 3.3 * 10**9 + max_diff = numpy_cosine_similarity_distance(image.flatten(), image_sliced.flatten()) + assert max_diff < 5e-3 + + def test_stable_diffusion_text2img_intermediate_state(self): + number_of_steps = 0 + + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True + nonlocal number_of_steps + number_of_steps += 1 + if step == 1: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 64) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array( + [-0.3862, -0.4507, -1.1729, 0.0686, -1.1045, 0.7124, -1.8301, 0.1903, 1.2773] + ) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + elif step == 2: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 64) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array( + [0.2720, -0.1863, -0.7383, -0.5029, -0.7534, 0.3970, -0.7646, 0.4468, 1.2686] + ) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + + callback_fn.has_been_called = False + + pipe = StableDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-base", torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == inputs["num_inference_steps"] + + @require_torch_gpu + def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe = StableDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-base", torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.8 GB is allocated + assert mem_bytes < 2.8 * 10**9 + + @require_torch_gpu + def test_stable_diffusion_pipeline_with_model_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + + # Normal inference + + pipe = StableDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-base", + torch_dtype=torch.float16, + ) + pipe.unet.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + outputs = pipe(**inputs) + mem_bytes = torch.cuda.max_memory_allocated() + + # With model offloading + + # Reload but don't move to cuda + pipe = StableDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-base", + torch_dtype=torch.float16, + ) + pipe.unet.set_default_attn_processor() + + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + inputs = self.get_inputs(torch_device, dtype=torch.float16) + outputs_offloaded = pipe(**inputs) + mem_bytes_offloaded = torch.cuda.max_memory_allocated() + + images = outputs.images + images_offloaded = outputs_offloaded.images + max_diff = numpy_cosine_similarity_distance(images.flatten(), images_offloaded.flatten()) + assert max_diff < 1e-3 + assert mem_bytes_offloaded < mem_bytes + assert mem_bytes_offloaded < 3 * 10**9 + for module in pipe.text_encoder, pipe.unet, pipe.vae: + assert module.device == torch.device("cpu") + + # With attention slicing + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe.enable_attention_slicing() + _ = pipe(**inputs) + mem_bytes_slicing = torch.cuda.max_memory_allocated() + assert mem_bytes_slicing < mem_bytes_offloaded + + +@nightly +@require_torch_accelerator +@skip_mps +class StableDiffusion2PipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + backend_empty_cache(torch_device) + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + _generator_device = "cpu" if not generator_device.startswith("cuda") else "cuda" + if not str(device).startswith("mps"): + generator = torch.Generator(device=_generator_device).manual_seed(seed) + else: + generator = torch.manual_seed(seed) + + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 50, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_2_0_default_ddim(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-base").to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_0_base_ddim.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_2_1_default_pndm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base").to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_1_base_pndm.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_ddim(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base").to(torch_device) + sd_pipe.scheduler = DDIMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_1_base_ddim.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_lms(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base").to(torch_device) + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_1_base_lms.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_euler(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base").to(torch_device) + sd_pipe.scheduler = EulerDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_1_base_euler.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_stable_diffusion_dpm(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base").to(torch_device) + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config( + sd_pipe.scheduler.config, final_sigmas_type="sigma_min" + ) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 25 + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_2_text2img/stable_diffusion_2_1_base_dpm_multi.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_attend_and_excite.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_attend_and_excite.py new file mode 100755 index 0000000..fdc41a2 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_attend_and_excite.py @@ -0,0 +1,235 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + StableDiffusionAttendAndExcitePipeline, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import ( + load_numpy, + nightly, + numpy_cosine_similarity_distance, + require_torch_gpu, + skip_mps, +) + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +torch.backends.cuda.matmul.allow_tf32 = False + + +@skip_mps +class StableDiffusionAttendAndExcitePipelineFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionAttendAndExcitePipeline + test_attention_slicing = False + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS.union({"token_indices"}) + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + # Attend and excite requires being able to run a backward pass at + # inference time. There's no deterministic backward operator for pad + + @classmethod + def setUpClass(cls): + super().setUpClass() + torch.use_deterministic_algorithms(False) + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + torch.use_deterministic_algorithms(True) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=1, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=512, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = inputs = { + "prompt": "a cat and a frog", + "token_indices": [2, 5], + "generator": generator, + "num_inference_steps": 1, + "guidance_scale": 6.0, + "output_type": "numpy", + "max_iter_to_alter": 2, + "thresholds": {0: 0.7}, + } + return inputs + + def test_inference(self): + device = "cpu" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + self.assertEqual(image.shape, (1, 64, 64, 3)) + expected_slice = np.array( + [0.63905364, 0.62897307, 0.48599017, 0.5133624, 0.5550048, 0.45769516, 0.50326973, 0.5023139, 0.45384496] + ) + max_diff = np.abs(image_slice.flatten() - expected_slice).max() + self.assertLessEqual(max_diff, 1e-3) + + def test_sequential_cpu_offload_forward_pass(self): + super().test_sequential_cpu_offload_forward_pass(expected_max_diff=5e-4) + + def test_inference_batch_consistent(self): + # NOTE: Larger batch sizes cause this test to timeout, only test on smaller batches + self._test_inference_batch_consistent(batch_sizes=[1, 2]) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(batch_size=2, expected_max_diff=7e-4) + + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent(expected_max_difference=3e-3) + + def test_pt_np_pil_outputs_equivalent(self): + super().test_pt_np_pil_outputs_equivalent(expected_max_diff=5e-4) + + def test_save_load_local(self): + super().test_save_load_local(expected_max_difference=5e-4) + + def test_save_load_optional_components(self): + super().test_save_load_optional_components(expected_max_difference=4e-4) + + +@require_torch_gpu +@nightly +class StableDiffusionAttendAndExcitePipelineIntegrationTests(unittest.TestCase): + # Attend and excite requires being able to run a backward pass at + # inference time. There's no deterministic backward operator for pad + + @classmethod + def setUpClass(cls): + super().setUpClass() + torch.use_deterministic_algorithms(False) + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + torch.use_deterministic_algorithms(True) + + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_attend_and_excite_fp16(self): + generator = torch.manual_seed(51) + + pipe = StableDiffusionAttendAndExcitePipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", safety_checker=None, torch_dtype=torch.float16 + ) + pipe.to("cuda") + + prompt = "a painting of an elephant with glasses" + token_indices = [5, 7] + + image = pipe( + prompt=prompt, + token_indices=token_indices, + guidance_scale=7.5, + generator=generator, + num_inference_steps=5, + max_iter_to_alter=5, + output_type="numpy", + ).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/attend-and-excite/elephant_glasses.npy" + ) + max_diff = numpy_cosine_similarity_distance(image.flatten(), expected_image.flatten()) + assert max_diff < 5e-1 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py new file mode 100755 index 0000000..76d480e --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_depth.py @@ -0,0 +1,603 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import tempfile +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import ( + CLIPTextConfig, + CLIPTextModel, + CLIPTokenizer, + DPTConfig, + DPTFeatureExtractor, + DPTForDepthEstimation, +) + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + StableDiffusionDepth2ImgPipeline, + UNet2DConditionModel, +) +from diffusers.utils import is_accelerate_available, is_accelerate_version +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + nightly, + require_torch_gpu, + skip_mps, + slow, + torch_device, +) + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, +) +from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +@skip_mps +class StableDiffusionDepth2ImgPipelineFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionDepth2ImgPipeline + test_save_load_optional_components = False + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width"} + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS.union({"depth_mask"}) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=5, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + attention_head_dim=(2, 4), + use_linear_projection=True, + ) + scheduler = PNDMScheduler(skip_prk_steps=True) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + backbone_config = { + "global_padding": "same", + "layer_type": "bottleneck", + "depths": [3, 4, 9], + "out_features": ["stage1", "stage2", "stage3"], + "embedding_dynamic_padding": True, + "hidden_sizes": [96, 192, 384, 768], + "num_groups": 2, + } + depth_estimator_config = DPTConfig( + image_size=32, + patch_size=16, + num_channels=3, + hidden_size=32, + num_hidden_layers=4, + backbone_out_indices=(0, 1, 2, 3), + num_attention_heads=4, + intermediate_size=37, + hidden_act="gelu", + hidden_dropout_prob=0.1, + attention_probs_dropout_prob=0.1, + is_decoder=False, + initializer_range=0.02, + is_hybrid=True, + backbone_config=backbone_config, + backbone_featmap_shape=[1, 384, 24, 24], + ) + depth_estimator = DPTForDepthEstimation(depth_estimator_config).eval() + feature_extractor = DPTFeatureExtractor.from_pretrained( + "hf-internal-testing/tiny-random-DPTForDepthEstimation" + ) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "depth_estimator": depth_estimator, + "feature_extractor": feature_extractor, + } + return components + + def get_dummy_inputs(self, device, seed=0): + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)) + image = image.cpu().permute(0, 2, 3, 1)[0] + image = Image.fromarray(np.uint8(image)).convert("RGB").resize((32, 32)) + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_save_load_local(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output = pipe(**inputs)[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir) + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output_loaded = pipe_loaded(**inputs)[0] + + max_diff = np.abs(output - output_loaded).max() + self.assertLess(max_diff, 1e-4) + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_save_load_float16(self): + components = self.get_dummy_components() + for name, module in components.items(): + if hasattr(module, "half"): + components[name] = module.to(torch_device).half() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output = pipe(**inputs)[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir, torch_dtype=torch.float16) + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + for name, component in pipe_loaded.components.items(): + if hasattr(component, "dtype"): + self.assertTrue( + component.dtype == torch.float16, + f"`{name}.dtype` switched from `float16` to {component.dtype} after loading.", + ) + + inputs = self.get_dummy_inputs(torch_device) + output_loaded = pipe_loaded(**inputs)[0] + + max_diff = np.abs(output - output_loaded).max() + self.assertLess(max_diff, 2e-2, "The output of the fp16 pipeline changed after saving and loading.") + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_float16_inference(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + for name, module in components.items(): + if hasattr(module, "half"): + components[name] = module.half() + pipe_fp16 = self.pipeline_class(**components) + pipe_fp16.to(torch_device) + pipe_fp16.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(torch_device))[0] + output_fp16 = pipe_fp16(**self.get_dummy_inputs(torch_device))[0] + + max_diff = np.abs(output - output_fp16).max() + self.assertLess(max_diff, 1.3e-2, "The outputs of the fp16 and fp32 pipelines are too different.") + + @unittest.skipIf( + torch_device != "cuda" or not is_accelerate_available() or is_accelerate_version("<", "0.14.0"), + reason="CPU offload is only available with CUDA and `accelerate v0.14.0` or higher", + ) + def test_cpu_offload_forward_pass(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output_without_offload = pipe(**inputs)[0] + + pipe.enable_sequential_cpu_offload() + inputs = self.get_dummy_inputs(torch_device) + output_with_offload = pipe(**inputs)[0] + + max_diff = np.abs(output_with_offload - output_without_offload).max() + self.assertLess(max_diff, 1e-4, "CPU offloading should not affect the inference results") + + def test_dict_tuple_outputs_equivalent(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(torch_device))[0] + output_tuple = pipe(**self.get_dummy_inputs(torch_device), return_dict=False)[0] + + max_diff = np.abs(output - output_tuple).max() + self.assertLess(max_diff, 1e-4) + + def test_progress_bar(self): + super().test_progress_bar() + + def test_stable_diffusion_depth2img_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = StableDiffusionDepth2ImgPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + if torch_device == "mps": + expected_slice = np.array([0.6071, 0.5035, 0.4378, 0.5776, 0.5753, 0.4316, 0.4513, 0.5263, 0.4546]) + else: + expected_slice = np.array([0.5435, 0.4992, 0.3783, 0.4411, 0.5842, 0.4654, 0.3786, 0.5077, 0.4655]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_depth2img_negative_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = StableDiffusionDepth2ImgPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + negative_prompt = "french fries" + output = pipe(**inputs, negative_prompt=negative_prompt) + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + if torch_device == "mps": + expected_slice = np.array([0.6296, 0.5125, 0.3890, 0.4456, 0.5955, 0.4621, 0.3810, 0.5310, 0.4626]) + else: + expected_slice = np.array([0.6012, 0.4507, 0.3769, 0.4121, 0.5566, 0.4585, 0.3803, 0.5045, 0.4631]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_depth2img_multiple_init_images(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = StableDiffusionDepth2ImgPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["prompt"] = [inputs["prompt"]] * 2 + inputs["image"] = 2 * [inputs["image"]] + image = pipe(**inputs).images + image_slice = image[-1, -3:, -3:, -1] + + assert image.shape == (2, 32, 32, 3) + + if torch_device == "mps": + expected_slice = np.array([0.6501, 0.5150, 0.4939, 0.6688, 0.5437, 0.5758, 0.5115, 0.4406, 0.4551]) + else: + expected_slice = np.array([0.6557, 0.6214, 0.6254, 0.5775, 0.4785, 0.5949, 0.5904, 0.4785, 0.4730]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_depth2img_pil(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = StableDiffusionDepth2ImgPipeline(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + if torch_device == "mps": + expected_slice = np.array([0.53232, 0.47015, 0.40868, 0.45651, 0.4891, 0.4668, 0.4287, 0.48822, 0.47439]) + else: + expected_slice = np.array([0.5435, 0.4992, 0.3783, 0.4411, 0.5842, 0.4654, 0.3786, 0.5077, 0.4655]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + @skip_mps + def test_attention_slicing_forward_pass(self): + return super().test_attention_slicing_forward_pass() + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=7e-3) + + +@slow +@require_torch_gpu +class StableDiffusionDepth2ImgPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/depth2img/two_cats.png" + ) + inputs = { + "prompt": "two tigers", + "image": init_image, + "generator": generator, + "num_inference_steps": 3, + "strength": 0.75, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_depth2img_pipeline_default(self): + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-depth", safety_checker=None + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs() + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 480, 640, 3) + expected_slice = np.array([0.5435, 0.4992, 0.3783, 0.4411, 0.5842, 0.4654, 0.3786, 0.5077, 0.4655]) + + assert np.abs(expected_slice - image_slice).max() < 6e-1 + + def test_stable_diffusion_depth2img_pipeline_k_lms(self): + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-depth", safety_checker=None + ) + pipe.unet.set_default_attn_processor() + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs() + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 480, 640, 3) + expected_slice = np.array([0.6363, 0.6274, 0.6309, 0.6370, 0.6226, 0.6286, 0.6213, 0.6453, 0.6306]) + + assert np.abs(expected_slice - image_slice).max() < 8e-4 + + def test_stable_diffusion_depth2img_pipeline_ddim(self): + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-depth", safety_checker=None + ) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs() + image = pipe(**inputs).images + image_slice = image[0, 253:256, 253:256, -1].flatten() + + assert image.shape == (1, 480, 640, 3) + expected_slice = np.array([0.6424, 0.6524, 0.6249, 0.6041, 0.6634, 0.6420, 0.6522, 0.6555, 0.6436]) + + assert np.abs(expected_slice - image_slice).max() < 5e-4 + + def test_stable_diffusion_depth2img_intermediate_state(self): + number_of_steps = 0 + + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True + nonlocal number_of_steps + number_of_steps += 1 + if step == 1: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 60, 80) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array( + [-0.7168, -1.5137, -0.1418, -2.9219, -2.7266, -2.4414, -2.1035, -3.0078, -1.7051] + ) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + elif step == 2: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 60, 80) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array( + [-0.7109, -1.5068, -0.1403, -2.9160, -2.7207, -2.4414, -2.1035, -3.0059, -1.7090] + ) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + + callback_fn.has_been_called = False + + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-depth", safety_checker=None, torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(dtype=torch.float16) + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == 2 + + def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-depth", safety_checker=None, torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + inputs = self.get_inputs(dtype=torch.float16) + _ = pipe(**inputs) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.9 GB is allocated + assert mem_bytes < 2.9 * 10**9 + + +@nightly +@require_torch_gpu +class StableDiffusionImg2ImgPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/depth2img/two_cats.png" + ) + inputs = { + "prompt": "two tigers", + "image": init_image, + "generator": generator, + "num_inference_steps": 3, + "strength": 0.75, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_depth2img_pndm(self): + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs() + image = pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_depth2img/stable_diffusion_2_0_pndm.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_depth2img_ddim(self): + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs() + image = pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_depth2img/stable_diffusion_2_0_ddim.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_img2img_lms(self): + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs() + image = pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_depth2img/stable_diffusion_2_0_lms.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_img2img_dpm(self): + pipe = StableDiffusionDepth2ImgPipeline.from_pretrained("stabilityai/stable-diffusion-2-depth") + pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs() + inputs["num_inference_steps"] = 30 + image = pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_depth2img/stable_diffusion_2_0_dpm_multi.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_diffedit.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_diffedit.py new file mode 100755 index 0000000..7634303 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_diffedit.py @@ -0,0 +1,432 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import tempfile +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMInverseScheduler, + DDIMScheduler, + DPMSolverMultistepInverseScheduler, + DPMSolverMultistepScheduler, + StableDiffusionDiffEditPipeline, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + nightly, + numpy_cosine_similarity_distance, + require_torch_gpu, + torch_device, +) + +from ..pipeline_params import TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, TEXT_GUIDED_IMAGE_INPAINTING_PARAMS +from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +class StableDiffusionDiffEditPipelineFastTests(PipelineLatentTesterMixin, PipelineTesterMixin, unittest.TestCase): + pipeline_class = StableDiffusionDiffEditPipeline + params = TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {"height", "width", "image"} | {"image_latents"} + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS - {"image"} | {"image_latents"} + image_params = frozenset( + [] + ) # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess + image_latents_params = frozenset([]) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + inverse_scheduler = DDIMInverseScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_zero=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=512, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "inverse_scheduler": inverse_scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + mask = floats_tensor((1, 16, 16), rng=random.Random(seed)).to(device) + latents = floats_tensor((1, 2, 4, 16, 16), rng=random.Random(seed)).to(device) + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "a dog and a newt", + "mask_image": mask, + "image_latents": latents, + "generator": generator, + "num_inference_steps": 2, + "inpaint_strength": 1.0, + "guidance_scale": 6.0, + "output_type": "numpy", + } + + return inputs + + def get_dummy_mask_inputs(self, device, seed=0): + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + image = Image.fromarray(np.uint8(image)).convert("RGB") + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "image": image, + "source_prompt": "a cat and a frog", + "target_prompt": "a dog and a newt", + "generator": generator, + "num_inference_steps": 2, + "num_maps_per_mask": 2, + "mask_encode_strength": 1.0, + "guidance_scale": 6.0, + "output_type": "numpy", + } + + return inputs + + def get_dummy_inversion_inputs(self, device, seed=0): + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + image = Image.fromarray(np.uint8(image)).convert("RGB") + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "image": image, + "prompt": "a cat and a frog", + "generator": generator, + "num_inference_steps": 2, + "inpaint_strength": 1.0, + "guidance_scale": 6.0, + "decode_latents": True, + "output_type": "numpy", + } + return inputs + + def test_save_load_optional_components(self): + if not hasattr(self.pipeline_class, "_optional_components"): + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + # set all optional components to None and update pipeline config accordingly + for optional_component in pipe._optional_components: + setattr(pipe, optional_component, None) + pipe.register_modules(**{optional_component: None for optional_component in pipe._optional_components}) + + inputs = self.get_dummy_inputs(torch_device) + output = pipe(**inputs)[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir) + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + for optional_component in pipe._optional_components: + self.assertTrue( + getattr(pipe_loaded, optional_component) is None, + f"`{optional_component}` did not stay set to None after loading.", + ) + + inputs = self.get_dummy_inputs(torch_device) + output_loaded = pipe_loaded(**inputs)[0] + + max_diff = np.abs(output - output_loaded).max() + self.assertLess(max_diff, 1e-4) + + def test_mask(self): + device = "cpu" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_mask_inputs(device) + mask = pipe.generate_mask(**inputs) + mask_slice = mask[0, -3:, -3:] + + self.assertEqual(mask.shape, (1, 16, 16)) + expected_slice = np.array([0] * 9) + max_diff = np.abs(mask_slice.flatten() - expected_slice).max() + self.assertLessEqual(max_diff, 1e-3) + self.assertEqual(mask[0, -3, -4], 0) + + def test_inversion(self): + device = "cpu" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inversion_inputs(device) + image = pipe.invert(**inputs).images + image_slice = image[0, -1, -3:, -3:] + + self.assertEqual(image.shape, (2, 32, 32, 3)) + expected_slice = np.array( + [0.5160, 0.5115, 0.5060, 0.5456, 0.4704, 0.5060, 0.5019, 0.4405, 0.4726], + ) + max_diff = np.abs(image_slice.flatten() - expected_slice).max() + self.assertLessEqual(max_diff, 1e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=5e-3) + + def test_inversion_dpm(self): + device = "cpu" + + components = self.get_dummy_components() + + scheduler_args = {"beta_start": 0.00085, "beta_end": 0.012, "beta_schedule": "scaled_linear"} + components["scheduler"] = DPMSolverMultistepScheduler(**scheduler_args) + components["inverse_scheduler"] = DPMSolverMultistepInverseScheduler(**scheduler_args) + + pipe = self.pipeline_class(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inversion_inputs(device) + image = pipe.invert(**inputs).images + image_slice = image[0, -1, -3:, -3:] + + self.assertEqual(image.shape, (2, 32, 32, 3)) + expected_slice = np.array( + [0.5305, 0.4673, 0.5314, 0.5308, 0.4886, 0.5279, 0.5142, 0.4724, 0.4892], + ) + max_diff = np.abs(image_slice.flatten() - expected_slice).max() + self.assertLessEqual(max_diff, 1e-3) + + +@require_torch_gpu +@nightly +class StableDiffusionDiffEditPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + @classmethod + def setUpClass(cls): + raw_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/diffedit/fruit.png" + ) + raw_image = raw_image.convert("RGB").resize((256, 256)) + + cls.raw_image = raw_image + + def test_stable_diffusion_diffedit_full(self): + generator = torch.manual_seed(0) + + pipe = StableDiffusionDiffEditPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-1-base", safety_checker=None, torch_dtype=torch.float16 + ) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.scheduler.clip_sample = True + + pipe.inverse_scheduler = DDIMInverseScheduler.from_config(pipe.scheduler.config) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + source_prompt = "a bowl of fruit" + target_prompt = "a bowl of pears" + + mask_image = pipe.generate_mask( + image=self.raw_image, + source_prompt=source_prompt, + target_prompt=target_prompt, + generator=generator, + ) + + inv_latents = pipe.invert( + prompt=source_prompt, + image=self.raw_image, + inpaint_strength=0.7, + generator=generator, + num_inference_steps=5, + ).latents + + image = pipe( + prompt=target_prompt, + mask_image=mask_image, + image_latents=inv_latents, + generator=generator, + negative_prompt=source_prompt, + inpaint_strength=0.7, + num_inference_steps=5, + output_type="np", + ).images[0] + + expected_image = ( + np.array( + load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/diffedit/pears.png" + ).resize((256, 256)) + ) + / 255 + ) + + assert numpy_cosine_similarity_distance(expected_image.flatten(), image.flatten()) < 2e-1 + + +@nightly +@require_torch_gpu +class StableDiffusionDiffEditPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + @classmethod + def setUpClass(cls): + raw_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/diffedit/fruit.png" + ) + + raw_image = raw_image.convert("RGB").resize((768, 768)) + + cls.raw_image = raw_image + + def test_stable_diffusion_diffedit_dpm(self): + generator = torch.manual_seed(0) + + pipe = StableDiffusionDiffEditPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-1", safety_checker=None, torch_dtype=torch.float16 + ) + pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + pipe.inverse_scheduler = DPMSolverMultistepInverseScheduler.from_config(pipe.scheduler.config) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + source_prompt = "a bowl of fruit" + target_prompt = "a bowl of pears" + + mask_image = pipe.generate_mask( + image=self.raw_image, + source_prompt=source_prompt, + target_prompt=target_prompt, + generator=generator, + ) + + inv_latents = pipe.invert( + prompt=source_prompt, + image=self.raw_image, + inpaint_strength=0.7, + generator=generator, + num_inference_steps=25, + ).latents + + image = pipe( + prompt=target_prompt, + mask_image=mask_image, + image_latents=inv_latents, + generator=generator, + negative_prompt=source_prompt, + inpaint_strength=0.7, + num_inference_steps=25, + output_type="numpy", + ).images[0] + + expected_image = ( + np.array( + load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/diffedit/pears.png" + ).resize((768, 768)) + ) + / 255 + ) + assert np.abs((expected_image - image).max()) < 5e-1 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_flax.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_flax.py new file mode 100755 index 0000000..afc2c86 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_flax.py @@ -0,0 +1,108 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +from diffusers import FlaxDPMSolverMultistepScheduler, FlaxStableDiffusionPipeline +from diffusers.utils import is_flax_available +from diffusers.utils.testing_utils import nightly, require_flax + + +if is_flax_available(): + import jax + import jax.numpy as jnp + from flax.jax_utils import replicate + from flax.training.common_utils import shard + + +@nightly +@require_flax +class FlaxStableDiffusion2PipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + + def test_stable_diffusion_flax(self): + sd_pipe, params = FlaxStableDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-2", + revision="bf16", + dtype=jnp.bfloat16, + ) + + prompt = "A painting of a squirrel eating a burger" + num_samples = jax.device_count() + prompt = num_samples * [prompt] + prompt_ids = sd_pipe.prepare_inputs(prompt) + + params = replicate(params) + prompt_ids = shard(prompt_ids) + + prng_seed = jax.random.PRNGKey(0) + prng_seed = jax.random.split(prng_seed, jax.device_count()) + + images = sd_pipe(prompt_ids, params, prng_seed, num_inference_steps=25, jit=True)[0] + assert images.shape == (jax.device_count(), 1, 768, 768, 3) + + images = images.reshape((images.shape[0] * images.shape[1],) + images.shape[-3:]) + image_slice = images[0, 253:256, 253:256, -1] + + output_slice = jnp.asarray(jax.device_get(image_slice.flatten())) + expected_slice = jnp.array([0.4238, 0.4414, 0.4395, 0.4453, 0.4629, 0.4590, 0.4531, 0.45508, 0.4512]) + print(f"output_slice: {output_slice}") + assert jnp.abs(output_slice - expected_slice).max() < 1e-2 + + +@nightly +@require_flax +class FlaxStableDiffusion2PipelineNightlyTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + + def test_stable_diffusion_dpm_flax(self): + model_id = "stabilityai/stable-diffusion-2" + scheduler, scheduler_params = FlaxDPMSolverMultistepScheduler.from_pretrained(model_id, subfolder="scheduler") + sd_pipe, params = FlaxStableDiffusionPipeline.from_pretrained( + model_id, + scheduler=scheduler, + revision="bf16", + dtype=jnp.bfloat16, + ) + params["scheduler"] = scheduler_params + + prompt = "A painting of a squirrel eating a burger" + num_samples = jax.device_count() + prompt = num_samples * [prompt] + prompt_ids = sd_pipe.prepare_inputs(prompt) + + params = replicate(params) + prompt_ids = shard(prompt_ids) + + prng_seed = jax.random.PRNGKey(0) + prng_seed = jax.random.split(prng_seed, jax.device_count()) + + images = sd_pipe(prompt_ids, params, prng_seed, num_inference_steps=25, jit=True)[0] + assert images.shape == (jax.device_count(), 1, 768, 768, 3) + + images = images.reshape((images.shape[0] * images.shape[1],) + images.shape[-3:]) + image_slice = images[0, 253:256, 253:256, -1] + + output_slice = jnp.asarray(jax.device_get(image_slice.flatten())) + expected_slice = jnp.array([0.4336, 0.42969, 0.4453, 0.4199, 0.4297, 0.4531, 0.4434, 0.4434, 0.4297]) + print(f"output_slice: {output_slice}") + assert jnp.abs(output_slice - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_flax_inpaint.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_flax_inpaint.py new file mode 100755 index 0000000..8f03998 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_flax_inpaint.py @@ -0,0 +1,82 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +from diffusers import FlaxStableDiffusionInpaintPipeline +from diffusers.utils import is_flax_available, load_image +from diffusers.utils.testing_utils import require_flax, slow + + +if is_flax_available(): + import jax + import jax.numpy as jnp + from flax.jax_utils import replicate + from flax.training.common_utils import shard + + +@slow +@require_flax +class FlaxStableDiffusionInpaintPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + + def test_stable_diffusion_inpaint_pipeline(self): + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/sd2-inpaint/init_image.png" + ) + mask_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-inpaint/mask.png" + ) + + model_id = "xvjiarui/stable-diffusion-2-inpainting" + pipeline, params = FlaxStableDiffusionInpaintPipeline.from_pretrained(model_id, safety_checker=None) + + prompt = "Face of a yellow cat, high resolution, sitting on a park bench" + + prng_seed = jax.random.PRNGKey(0) + num_inference_steps = 50 + + num_samples = jax.device_count() + prompt = num_samples * [prompt] + init_image = num_samples * [init_image] + mask_image = num_samples * [mask_image] + prompt_ids, processed_masked_images, processed_masks = pipeline.prepare_inputs(prompt, init_image, mask_image) + + # shard inputs and rng + params = replicate(params) + prng_seed = jax.random.split(prng_seed, jax.device_count()) + prompt_ids = shard(prompt_ids) + processed_masked_images = shard(processed_masked_images) + processed_masks = shard(processed_masks) + + output = pipeline( + prompt_ids, processed_masks, processed_masked_images, params, prng_seed, num_inference_steps, jit=True + ) + + images = output.images.reshape(num_samples, 512, 512, 3) + + image_slice = images[0, 253:256, 253:256, -1] + + output_slice = jnp.asarray(jax.device_get(image_slice.flatten())) + expected_slice = jnp.array( + [0.3611307, 0.37649736, 0.3757408, 0.38213953, 0.39295167, 0.3841631, 0.41554978, 0.4137475, 0.4217084] + ) + print(f"output_slice: {output_slice}") + assert jnp.abs(output_slice - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py new file mode 100755 index 0000000..6157b32 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_inpaint.py @@ -0,0 +1,277 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import AutoencoderKL, PNDMScheduler, StableDiffusionInpaintPipeline, UNet2DConditionModel +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + require_torch_gpu, + slow, + torch_device, +) + +from ..pipeline_params import ( + TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_INPAINTING_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, +) +from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +class StableDiffusion2InpaintPipelineFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionInpaintPipeline + params = TEXT_GUIDED_IMAGE_INPAINTING_PARAMS + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + image_params = frozenset( + [] + ) # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess + image_latents_params = frozenset([]) + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS.union({"mask", "masked_image_latents"}) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=9, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + ) + scheduler = PNDMScheduler(skip_prk_steps=True) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=512, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + # TODO: use tensor inputs instead of PIL, this is here just to leave the old expected_slices untouched + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + init_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((64, 64)) + mask_image = Image.fromarray(np.uint8(image + 4)).convert("RGB").resize((64, 64)) + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": init_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_inpaint(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionInpaintPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4727, 0.5735, 0.3941, 0.5446, 0.5926, 0.4394, 0.5062, 0.4654, 0.4476]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + +@slow +@require_torch_gpu +class StableDiffusionInpaintPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_diffusion_inpaint_pipeline(self): + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/sd2-inpaint/init_image.png" + ) + mask_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-inpaint/mask.png" + ) + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-inpaint" + "/yellow_cat_sitting_on_a_park_bench.npy" + ) + + model_id = "stabilityai/stable-diffusion-2-inpainting" + pipe = StableDiffusionInpaintPipeline.from_pretrained(model_id, safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + prompt = "Face of a yellow cat, high resolution, sitting on a park bench" + + generator = torch.manual_seed(0) + output = pipe( + prompt=prompt, + image=init_image, + mask_image=mask_image, + generator=generator, + output_type="np", + ) + image = output.images[0] + + assert image.shape == (512, 512, 3) + assert np.abs(expected_image - image).max() < 9e-3 + + def test_stable_diffusion_inpaint_pipeline_fp16(self): + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/sd2-inpaint/init_image.png" + ) + mask_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-inpaint/mask.png" + ) + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-inpaint" + "/yellow_cat_sitting_on_a_park_bench_fp16.npy" + ) + + model_id = "stabilityai/stable-diffusion-2-inpainting" + pipe = StableDiffusionInpaintPipeline.from_pretrained( + model_id, + torch_dtype=torch.float16, + safety_checker=None, + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + prompt = "Face of a yellow cat, high resolution, sitting on a park bench" + + generator = torch.manual_seed(0) + output = pipe( + prompt=prompt, + image=init_image, + mask_image=mask_image, + generator=generator, + output_type="np", + ) + image = output.images[0] + + assert image.shape == (512, 512, 3) + assert np.abs(expected_image - image).max() < 5e-1 + + def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + init_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/sd2-inpaint/init_image.png" + ) + mask_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-inpaint/mask.png" + ) + + model_id = "stabilityai/stable-diffusion-2-inpainting" + pndm = PNDMScheduler.from_pretrained(model_id, subfolder="scheduler") + pipe = StableDiffusionInpaintPipeline.from_pretrained( + model_id, + safety_checker=None, + scheduler=pndm, + torch_dtype=torch.float16, + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + prompt = "Face of a yellow cat, high resolution, sitting on a park bench" + + generator = torch.manual_seed(0) + _ = pipe( + prompt=prompt, + image=init_image, + mask_image=mask_image, + generator=generator, + num_inference_steps=2, + output_type="np", + ) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.65 GB is allocated + assert mem_bytes < 2.65 * 10**9 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_latent_upscale.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_latent_upscale.py new file mode 100755 index 0000000..04721b4 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_latent_upscale.py @@ -0,0 +1,306 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +import diffusers +from diffusers import ( + AutoencoderKL, + EulerDiscreteScheduler, + StableDiffusionLatentUpscalePipeline, + StableDiffusionPipeline, + UNet2DConditionModel, +) +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + require_torch_gpu, + slow, + torch_device, +) + +from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS +from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +def check_same_shape(tensor_list): + shapes = [tensor.shape for tensor in tensor_list] + return all(shape == shapes[0] for shape in shapes[1:]) + + +class StableDiffusionLatentUpscalePipelineFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionLatentUpscalePipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - { + "height", + "width", + "cross_attention_kwargs", + "negative_prompt_embeds", + "prompt_embeds", + } + required_optional_params = PipelineTesterMixin.required_optional_params - {"num_images_per_prompt"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = frozenset( + [] + ) # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess + image_latents_params = frozenset([]) + + @property + def dummy_image(self): + batch_size = 1 + num_channels = 4 + sizes = (16, 16) + + image = floats_tensor((batch_size, num_channels) + sizes, rng=random.Random(0)).to(torch_device) + return image + + def get_dummy_components(self): + torch.manual_seed(0) + model = UNet2DConditionModel( + act_fn="gelu", + attention_head_dim=8, + norm_num_groups=None, + block_out_channels=[32, 32, 64, 64], + time_cond_proj_dim=160, + conv_in_kernel=1, + conv_out_kernel=1, + cross_attention_dim=32, + down_block_types=( + "KDownBlock2D", + "KCrossAttnDownBlock2D", + "KCrossAttnDownBlock2D", + "KCrossAttnDownBlock2D", + ), + in_channels=8, + mid_block_type=None, + only_cross_attention=False, + out_channels=5, + resnet_time_scale_shift="scale_shift", + time_embedding_type="fourier", + timestep_post_act="gelu", + up_block_types=("KCrossAttnUpBlock2D", "KCrossAttnUpBlock2D", "KCrossAttnUpBlock2D", "KUpBlock2D"), + ) + vae = AutoencoderKL( + block_out_channels=[32, 32, 64, 64], + in_channels=3, + out_channels=3, + down_block_types=[ + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + "DownEncoderBlock2D", + ], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + scheduler = EulerDiscreteScheduler(prediction_type="sample") + text_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + hidden_act="quick_gelu", + projection_dim=512, + ) + text_encoder = CLIPTextModel(text_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": model.eval(), + "vae": vae.eval(), + "scheduler": scheduler, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": self.dummy_image.cpu(), + "generator": generator, + "num_inference_steps": 2, + "output_type": "numpy", + } + return inputs + + def test_inference(self): + device = "cpu" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + self.assertEqual(image.shape, (1, 256, 256, 3)) + expected_slice = np.array( + [0.47222412, 0.41921633, 0.44717434, 0.46874192, 0.42588258, 0.46150726, 0.4677534, 0.45583832, 0.48579055] + ) + max_diff = np.abs(image_slice.flatten() - expected_slice).max() + self.assertLessEqual(max_diff, 1e-3) + + def test_attention_slicing_forward_pass(self): + super().test_attention_slicing_forward_pass(expected_max_diff=7e-3) + + def test_sequential_cpu_offload_forward_pass(self): + super().test_sequential_cpu_offload_forward_pass(expected_max_diff=3e-3) + + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent(expected_max_difference=3e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=7e-3) + + def test_pt_np_pil_outputs_equivalent(self): + super().test_pt_np_pil_outputs_equivalent(expected_max_diff=3e-3) + + def test_save_load_local(self): + super().test_save_load_local(expected_max_difference=3e-3) + + def test_save_load_optional_components(self): + super().test_save_load_optional_components(expected_max_difference=3e-3) + + def test_karras_schedulers_shape(self): + skip_schedulers = [ + "DDIMScheduler", + "DDPMScheduler", + "PNDMScheduler", + "HeunDiscreteScheduler", + "EulerAncestralDiscreteScheduler", + "KDPM2DiscreteScheduler", + "KDPM2AncestralDiscreteScheduler", + "DPMSolverSDEScheduler", + "EDMEulerScheduler", + ] + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + + # make sure that PNDM does not need warm-up + pipe.scheduler.register_to_config(skip_prk_steps=True) + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = 2 + + outputs = [] + for scheduler_enum in KarrasDiffusionSchedulers: + if scheduler_enum.name in skip_schedulers: + # no sigma schedulers are not supported + # no schedulers + continue + + scheduler_cls = getattr(diffusers, scheduler_enum.name) + pipe.scheduler = scheduler_cls.from_config(pipe.scheduler.config) + output = pipe(**inputs)[0] + outputs.append(output) + + assert check_same_shape(outputs) + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=5e-1) + + +@require_torch_gpu +@slow +class StableDiffusionLatentUpscalePipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_latent_upscaler_fp16(self): + generator = torch.manual_seed(33) + + pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4", torch_dtype=torch.float16) + pipe.to("cuda") + + upscaler = StableDiffusionLatentUpscalePipeline.from_pretrained( + "stabilityai/sd-x2-latent-upscaler", torch_dtype=torch.float16 + ) + upscaler.to("cuda") + + prompt = "a photo of an astronaut high resolution, unreal engine, ultra realistic" + + low_res_latents = pipe(prompt, generator=generator, output_type="latent").images + + image = upscaler( + prompt=prompt, + image=low_res_latents, + num_inference_steps=20, + guidance_scale=0, + generator=generator, + output_type="np", + ).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/astronaut_1024.npy" + ) + assert np.abs((expected_image - image).mean()) < 5e-2 + + def test_latent_upscaler_fp16_image(self): + generator = torch.manual_seed(33) + + upscaler = StableDiffusionLatentUpscalePipeline.from_pretrained( + "stabilityai/sd-x2-latent-upscaler", torch_dtype=torch.float16 + ) + upscaler.to("cuda") + + prompt = "the temple of fire by Ross Tran and Gerardo Dottori, oil on canvas" + + low_res_img = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/fire_temple_512.png" + ) + + image = upscaler( + prompt=prompt, + image=low_res_img, + num_inference_steps=20, + guidance_scale=0, + generator=generator, + output_type="np", + ).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/latent-upscaler/fire_temple_1024.npy" + ) + assert np.abs((expected_image - image).max()) < 5e-2 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py new file mode 100755 index 0000000..4dd6121 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_upscale.py @@ -0,0 +1,552 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import tempfile +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler, StableDiffusionUpscalePipeline, UNet2DConditionModel +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) + + +enable_full_determinism() + + +class StableDiffusionUpscalePipelineFastTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + @property + def dummy_image(self): + batch_size = 1 + num_channels = 3 + sizes = (32, 32) + + image = floats_tensor((batch_size, num_channels) + sizes, rng=random.Random(0)).to(torch_device) + return image + + @property + def dummy_cond_unet_upscale(self): + torch.manual_seed(0) + model = UNet2DConditionModel( + block_out_channels=(32, 32, 64), + layers_per_block=2, + sample_size=32, + in_channels=7, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + # SD2-specific config below + attention_head_dim=8, + use_linear_projection=True, + only_cross_attention=(True, True, False), + num_class_embeds=100, + ) + return model + + @property + def dummy_vae(self): + torch.manual_seed(0) + model = AutoencoderKL( + block_out_channels=[32, 32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + return model + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=512, + ) + return CLIPTextModel(config) + + def test_stable_diffusion_upscale(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unet = self.dummy_cond_unet_upscale + low_res_scheduler = DDPMScheduler() + scheduler = DDIMScheduler(prediction_type="v_prediction") + vae = self.dummy_vae + text_encoder = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + image = self.dummy_image.cpu().permute(0, 2, 3, 1)[0] + low_res_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((64, 64)) + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionUpscalePipeline( + unet=unet, + low_res_scheduler=low_res_scheduler, + scheduler=scheduler, + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + max_noise_level=350, + ) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + generator = torch.Generator(device=device).manual_seed(0) + output = sd_pipe( + [prompt], + image=low_res_image, + generator=generator, + guidance_scale=6.0, + noise_level=20, + num_inference_steps=2, + output_type="np", + ) + + image = output.images + + generator = torch.Generator(device=device).manual_seed(0) + image_from_tuple = sd_pipe( + [prompt], + image=low_res_image, + generator=generator, + guidance_scale=6.0, + noise_level=20, + num_inference_steps=2, + output_type="np", + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + expected_height_width = low_res_image.size[0] * 4 + assert image.shape == (1, expected_height_width, expected_height_width, 3) + expected_slice = np.array([0.3113, 0.3910, 0.4272, 0.4859, 0.5061, 0.4652, 0.5362, 0.5715, 0.5661]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_upscale_batch(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unet = self.dummy_cond_unet_upscale + low_res_scheduler = DDPMScheduler() + scheduler = DDIMScheduler(prediction_type="v_prediction") + vae = self.dummy_vae + text_encoder = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + image = self.dummy_image.cpu().permute(0, 2, 3, 1)[0] + low_res_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((64, 64)) + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionUpscalePipeline( + unet=unet, + low_res_scheduler=low_res_scheduler, + scheduler=scheduler, + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + max_noise_level=350, + ) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + output = sd_pipe( + 2 * [prompt], + image=2 * [low_res_image], + guidance_scale=6.0, + noise_level=20, + num_inference_steps=2, + output_type="np", + ) + image = output.images + assert image.shape[0] == 2 + + generator = torch.Generator(device=device).manual_seed(0) + output = sd_pipe( + [prompt], + image=low_res_image, + generator=generator, + num_images_per_prompt=2, + guidance_scale=6.0, + noise_level=20, + num_inference_steps=2, + output_type="np", + ) + image = output.images + assert image.shape[0] == 2 + + def test_stable_diffusion_upscale_prompt_embeds(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unet = self.dummy_cond_unet_upscale + low_res_scheduler = DDPMScheduler() + scheduler = DDIMScheduler(prediction_type="v_prediction") + vae = self.dummy_vae + text_encoder = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + image = self.dummy_image.cpu().permute(0, 2, 3, 1)[0] + low_res_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((64, 64)) + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionUpscalePipeline( + unet=unet, + low_res_scheduler=low_res_scheduler, + scheduler=scheduler, + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + max_noise_level=350, + ) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + generator = torch.Generator(device=device).manual_seed(0) + output = sd_pipe( + [prompt], + image=low_res_image, + generator=generator, + guidance_scale=6.0, + noise_level=20, + num_inference_steps=2, + output_type="np", + ) + + image = output.images + + generator = torch.Generator(device=device).manual_seed(0) + prompt_embeds, negative_prompt_embeds = sd_pipe.encode_prompt(prompt, device, 1, False) + if negative_prompt_embeds is not None: + prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds]) + + image_from_prompt_embeds = sd_pipe( + prompt_embeds=prompt_embeds, + image=[low_res_image], + generator=generator, + guidance_scale=6.0, + noise_level=20, + num_inference_steps=2, + output_type="np", + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_prompt_embeds_slice = image_from_prompt_embeds[0, -3:, -3:, -1] + + expected_height_width = low_res_image.size[0] * 4 + assert image.shape == (1, expected_height_width, expected_height_width, 3) + expected_slice = np.array([0.3113, 0.3910, 0.4272, 0.4859, 0.5061, 0.4652, 0.5362, 0.5715, 0.5661]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_prompt_embeds_slice.flatten() - expected_slice).max() < 1e-2 + + @unittest.skipIf(torch_device != "cuda", "This test requires a GPU") + def test_stable_diffusion_upscale_fp16(self): + """Test that stable diffusion upscale works with fp16""" + unet = self.dummy_cond_unet_upscale + low_res_scheduler = DDPMScheduler() + scheduler = DDIMScheduler(prediction_type="v_prediction") + vae = self.dummy_vae + text_encoder = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + image = self.dummy_image.cpu().permute(0, 2, 3, 1)[0] + low_res_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((64, 64)) + + # put models in fp16, except vae as it overflows in fp16 + unet = unet.half() + text_encoder = text_encoder.half() + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionUpscalePipeline( + unet=unet, + low_res_scheduler=low_res_scheduler, + scheduler=scheduler, + vae=vae, + text_encoder=text_encoder, + tokenizer=tokenizer, + max_noise_level=350, + ) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + generator = torch.manual_seed(0) + image = sd_pipe( + [prompt], + image=low_res_image, + generator=generator, + num_inference_steps=2, + output_type="np", + ).images + + expected_height_width = low_res_image.size[0] * 4 + assert image.shape == (1, expected_height_width, expected_height_width, 3) + + def test_stable_diffusion_upscale_from_save_pretrained(self): + pipes = [] + + device = "cpu" # ensure determinism for the device-dependent torch.Generator + low_res_scheduler = DDPMScheduler() + scheduler = DDIMScheduler(prediction_type="v_prediction") + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionUpscalePipeline( + unet=self.dummy_cond_unet_upscale, + low_res_scheduler=low_res_scheduler, + scheduler=scheduler, + vae=self.dummy_vae, + text_encoder=self.dummy_text_encoder, + tokenizer=tokenizer, + max_noise_level=350, + ) + sd_pipe = sd_pipe.to(device) + pipes.append(sd_pipe) + + with tempfile.TemporaryDirectory() as tmpdirname: + sd_pipe.save_pretrained(tmpdirname) + sd_pipe = StableDiffusionUpscalePipeline.from_pretrained(tmpdirname).to(device) + pipes.append(sd_pipe) + + prompt = "A painting of a squirrel eating a burger" + image = self.dummy_image.cpu().permute(0, 2, 3, 1)[0] + low_res_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((64, 64)) + + image_slices = [] + for pipe in pipes: + generator = torch.Generator(device=device).manual_seed(0) + image = pipe( + [prompt], + image=low_res_image, + generator=generator, + guidance_scale=6.0, + noise_level=20, + num_inference_steps=2, + output_type="np", + ).images + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + + +@slow +@require_torch_gpu +class StableDiffusionUpscalePipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_diffusion_upscale_pipeline(self): + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/sd2-upscale/low_res_cat.png" + ) + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale" + "/upsampled_cat.npy" + ) + + model_id = "stabilityai/stable-diffusion-x4-upscaler" + pipe = StableDiffusionUpscalePipeline.from_pretrained(model_id) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + prompt = "a cat sitting on a park bench" + + generator = torch.manual_seed(0) + output = pipe( + prompt=prompt, + image=image, + generator=generator, + output_type="np", + ) + image = output.images[0] + + assert image.shape == (512, 512, 3) + assert np.abs(expected_image - image).max() < 1e-3 + + def test_stable_diffusion_upscale_pipeline_fp16(self): + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/sd2-upscale/low_res_cat.png" + ) + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/sd2-upscale" + "/upsampled_cat_fp16.npy" + ) + + model_id = "stabilityai/stable-diffusion-x4-upscaler" + pipe = StableDiffusionUpscalePipeline.from_pretrained( + model_id, + torch_dtype=torch.float16, + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + prompt = "a cat sitting on a park bench" + + generator = torch.manual_seed(0) + output = pipe( + prompt=prompt, + image=image, + generator=generator, + output_type="np", + ) + image = output.images[0] + + assert image.shape == (512, 512, 3) + assert np.abs(expected_image - image).max() < 5e-1 + + def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/sd2-upscale/low_res_cat.png" + ) + + model_id = "stabilityai/stable-diffusion-x4-upscaler" + pipe = StableDiffusionUpscalePipeline.from_pretrained( + model_id, + torch_dtype=torch.float16, + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + prompt = "a cat sitting on a park bench" + + generator = torch.manual_seed(0) + _ = pipe( + prompt=prompt, + image=image, + generator=generator, + num_inference_steps=5, + output_type="np", + ) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.9 GB is allocated + assert mem_bytes < 2.9 * 10**9 + + def test_download_ckpt_diff_format_is_same(self): + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/sd2-upscale/low_res_cat.png" + ) + + prompt = "a cat sitting on a park bench" + model_id = "stabilityai/stable-diffusion-x4-upscaler" + pipe = StableDiffusionUpscalePipeline.from_pretrained(model_id) + pipe.enable_model_cpu_offload() + + generator = torch.Generator("cpu").manual_seed(0) + output = pipe(prompt=prompt, image=image, generator=generator, output_type="np", num_inference_steps=3) + image_from_pretrained = output.images[0] + + single_file_path = ( + "https://huggingface.co/stabilityai/stable-diffusion-x4-upscaler/blob/main/x4-upscaler-ema.safetensors" + ) + pipe_from_single_file = StableDiffusionUpscalePipeline.from_single_file(single_file_path) + pipe_from_single_file.enable_model_cpu_offload() + + generator = torch.Generator("cpu").manual_seed(0) + output_from_single_file = pipe_from_single_file( + prompt=prompt, image=image, generator=generator, output_type="np", num_inference_steps=3 + ) + image_from_single_file = output_from_single_file.images[0] + + assert image_from_pretrained.shape == (512, 512, 3) + assert image_from_single_file.shape == (512, 512, 3) + assert ( + numpy_cosine_similarity_distance(image_from_pretrained.flatten(), image_from_single_file.flatten()) < 1e-3 + ) + + def test_single_file_component_configs(self): + pipe = StableDiffusionUpscalePipeline.from_pretrained( + "stabilityai/stable-diffusion-x4-upscaler", variant="fp16" + ) + + ckpt_path = ( + "https://huggingface.co/stabilityai/stable-diffusion-x4-upscaler/blob/main/x4-upscaler-ema.safetensors" + ) + single_file_pipe = StableDiffusionUpscalePipeline.from_single_file(ckpt_path, load_safety_checker=True) + + for param_name, param_value in single_file_pipe.text_encoder.config.to_dict().items(): + if param_name in ["torch_dtype", "architectures", "_name_or_path"]: + continue + assert pipe.text_encoder.config.to_dict()[param_name] == param_value + + PARAMS_TO_IGNORE = ["torch_dtype", "_name_or_path", "architectures", "_use_default_values"] + for param_name, param_value in single_file_pipe.unet.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.unet.config[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" + + for param_name, param_value in single_file_pipe.vae.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.vae.config[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" + + for param_name, param_value in single_file_pipe.safety_checker.config.to_dict().items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.safety_checker.config.to_dict()[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_v_pred.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_v_pred.py new file mode 100755 index 0000000..be5b639 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_2/test_stable_diffusion_v_pred.py @@ -0,0 +1,563 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import time +import unittest + +import numpy as np +import torch +from huggingface_hub import hf_hub_download +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerDiscreteScheduler, + StableDiffusionPipeline, + UNet2DConditionModel, +) +from diffusers.models.attention_processor import AttnProcessor +from diffusers.utils.testing_utils import ( + enable_full_determinism, + load_numpy, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) + + +enable_full_determinism() + + +class StableDiffusion2VPredictionPipelineFastTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + @property + def dummy_cond_unet(self): + torch.manual_seed(0) + model = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + ) + return model + + @property + def dummy_vae(self): + torch.manual_seed(0) + model = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + return model + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=64, + ) + return CLIPTextModel(config) + + def test_stable_diffusion_v_pred_ddim(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unet = self.dummy_cond_unet + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + prediction_type="v_prediction", + ) + + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=None, + image_encoder=None, + requires_safety_checker=False, + ) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + + generator = torch.Generator(device=device).manual_seed(0) + output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np") + image = output.images + + generator = torch.Generator(device=device).manual_seed(0) + image_from_tuple = sd_pipe( + [prompt], + generator=generator, + guidance_scale=6.0, + num_inference_steps=2, + output_type="np", + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.6569, 0.6525, 0.5142, 0.4968, 0.4923, 0.4601, 0.4996, 0.5041, 0.4544]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_v_pred_k_euler(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unet = self.dummy_cond_unet + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", prediction_type="v_prediction" + ) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=None, + image_encoder=None, + requires_safety_checker=False, + ) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + generator = torch.Generator(device=device).manual_seed(0) + output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np") + + image = output.images + + generator = torch.Generator(device=device).manual_seed(0) + image_from_tuple = sd_pipe( + [prompt], + generator=generator, + guidance_scale=6.0, + num_inference_steps=2, + output_type="np", + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5644, 0.6514, 0.5190, 0.5663, 0.5287, 0.4953, 0.5430, 0.5243, 0.4778]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + @unittest.skipIf(torch_device != "cuda", "This test requires a GPU") + def test_stable_diffusion_v_pred_fp16(self): + """Test that stable diffusion v-prediction works with fp16""" + unet = self.dummy_cond_unet + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + prediction_type="v_prediction", + ) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + # put models in fp16 + unet = unet.half() + vae = vae.half() + bert = bert.half() + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=None, + image_encoder=None, + requires_safety_checker=False, + ) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + generator = torch.manual_seed(0) + image = sd_pipe([prompt], generator=generator, num_inference_steps=2, output_type="np").images + + assert image.shape == (1, 64, 64, 3) + + +@slow +@require_torch_gpu +class StableDiffusion2VPredictionPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_diffusion_v_pred_default(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2") + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.enable_attention_slicing() + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + generator = torch.manual_seed(0) + output = sd_pipe([prompt], generator=generator, guidance_scale=7.5, num_inference_steps=20, output_type="np") + + image = output.images + image_slice = image[0, 253:256, 253:256, -1] + + assert image.shape == (1, 768, 768, 3) + expected_slice = np.array([0.1868, 0.1922, 0.1527, 0.1921, 0.1908, 0.1624, 0.1779, 0.1652, 0.1734]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_v_pred_upcast_attention(self): + sd_pipe = StableDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-1", torch_dtype=torch.float16 + ) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.enable_attention_slicing() + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + generator = torch.manual_seed(0) + output = sd_pipe([prompt], generator=generator, guidance_scale=7.5, num_inference_steps=20, output_type="np") + + image = output.images + image_slice = image[0, 253:256, 253:256, -1] + + assert image.shape == (1, 768, 768, 3) + expected_slice = np.array([0.4209, 0.4087, 0.4097, 0.4209, 0.3860, 0.4329, 0.4280, 0.4324, 0.4187]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-2 + + def test_stable_diffusion_v_pred_euler(self): + scheduler = EulerDiscreteScheduler.from_pretrained("stabilityai/stable-diffusion-2", subfolder="scheduler") + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2", scheduler=scheduler) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.enable_attention_slicing() + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + generator = torch.manual_seed(0) + + output = sd_pipe([prompt], generator=generator, num_inference_steps=5, output_type="numpy") + image = output.images + + image_slice = image[0, 253:256, 253:256, -1] + + assert image.shape == (1, 768, 768, 3) + expected_slice = np.array([0.1781, 0.1695, 0.1661, 0.1705, 0.1588, 0.1699, 0.2005, 0.1589, 0.1677]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_v_pred_dpm(self): + """ + TODO: update this test after making DPM compatible with V-prediction! + """ + scheduler = DPMSolverMultistepScheduler.from_pretrained( + "stabilityai/stable-diffusion-2", + subfolder="scheduler", + final_sigmas_type="sigma_min", + ) + sd_pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2", scheduler=scheduler) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.enable_attention_slicing() + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "a photograph of an astronaut riding a horse" + generator = torch.manual_seed(0) + image = sd_pipe( + [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=5, output_type="numpy" + ).images + + image_slice = image[0, 253:256, 253:256, -1] + assert image.shape == (1, 768, 768, 3) + expected_slice = np.array([0.3303, 0.3184, 0.3291, 0.3300, 0.3256, 0.3113, 0.2965, 0.3134, 0.3192]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_attention_slicing_v_pred(self): + torch.cuda.reset_peak_memory_stats() + model_id = "stabilityai/stable-diffusion-2" + pipe = StableDiffusionPipeline.from_pretrained(model_id, torch_dtype=torch.float16) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + prompt = "a photograph of an astronaut riding a horse" + + # make attention efficient + pipe.enable_attention_slicing() + generator = torch.manual_seed(0) + output_chunked = pipe( + [prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy" + ) + image_chunked = output_chunked.images + + mem_bytes = torch.cuda.max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + # make sure that less than 5.5 GB is allocated + assert mem_bytes < 5.5 * 10**9 + + # disable slicing + pipe.disable_attention_slicing() + generator = torch.manual_seed(0) + output = pipe([prompt], generator=generator, guidance_scale=7.5, num_inference_steps=10, output_type="numpy") + image = output.images + + # make sure that more than 3.0 GB is allocated + mem_bytes = torch.cuda.max_memory_allocated() + assert mem_bytes > 3 * 10**9 + max_diff = numpy_cosine_similarity_distance(image.flatten(), image_chunked.flatten()) + assert max_diff < 1e-3 + + def test_stable_diffusion_text2img_pipeline_v_pred_default(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/" + "sd2-text2img/astronaut_riding_a_horse_v_pred.npy" + ) + + pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2") + pipe.to(torch_device) + pipe.enable_attention_slicing() + pipe.set_progress_bar_config(disable=None) + + prompt = "astronaut riding a horse" + + generator = torch.manual_seed(0) + output = pipe(prompt=prompt, guidance_scale=7.5, generator=generator, output_type="np") + image = output.images[0] + + assert image.shape == (768, 768, 3) + max_diff = numpy_cosine_similarity_distance(image.flatten(), expected_image.flatten()) + assert max_diff < 1e-3 + + def test_stable_diffusion_text2img_pipeline_unflawed(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/" + "sd2-text2img/lion_galaxy.npy" + ) + + pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1") + pipe.scheduler = DDIMScheduler.from_config( + pipe.scheduler.config, timestep_spacing="trailing", rescale_betas_zero_snr=True + ) + pipe.to(torch_device) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + + prompt = "A lion in galaxies, spirals, nebulae, stars, smoke, iridescent, intricate detail, octane render, 8k" + + generator = torch.Generator("cpu").manual_seed(0) + output = pipe( + prompt=prompt, + guidance_scale=7.5, + num_inference_steps=10, + guidance_rescale=0.7, + generator=generator, + output_type="np", + ) + image = output.images[0] + + assert image.shape == (768, 768, 3) + max_diff = numpy_cosine_similarity_distance(image.flatten(), expected_image.flatten()) + assert max_diff < 5e-2 + + def test_stable_diffusion_text2img_pipeline_v_pred_fp16(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/" + "sd2-text2img/astronaut_riding_a_horse_v_pred_fp16.npy" + ) + + pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2", torch_dtype=torch.float16) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + prompt = "astronaut riding a horse" + + generator = torch.manual_seed(0) + output = pipe(prompt=prompt, guidance_scale=7.5, generator=generator, output_type="np") + image = output.images[0] + + assert image.shape == (768, 768, 3) + max_diff = numpy_cosine_similarity_distance(image.flatten(), expected_image.flatten()) + assert max_diff < 1e-3 + + def test_download_local(self): + filename = hf_hub_download("stabilityai/stable-diffusion-2-1", filename="v2-1_768-ema-pruned.safetensors") + + pipe = StableDiffusionPipeline.from_single_file(filename, torch_dtype=torch.float16) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.enable_model_cpu_offload() + + image_out = pipe("test", num_inference_steps=1, output_type="np").images[0] + + assert image_out.shape == (768, 768, 3) + + def test_download_ckpt_diff_format_is_same(self): + single_file_path = ( + "https://huggingface.co/stabilityai/stable-diffusion-2-1/blob/main/v2-1_768-ema-pruned.safetensors" + ) + + pipe_single = StableDiffusionPipeline.from_single_file(single_file_path) + pipe_single.scheduler = DDIMScheduler.from_config(pipe_single.scheduler.config) + pipe_single.unet.set_attn_processor(AttnProcessor()) + pipe_single.enable_model_cpu_offload() + + generator = torch.Generator(device="cpu").manual_seed(0) + image_ckpt = pipe_single("a turtle", num_inference_steps=2, generator=generator, output_type="np").images[0] + + pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1") + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.unet.set_attn_processor(AttnProcessor()) + pipe.enable_model_cpu_offload() + + generator = torch.Generator(device="cpu").manual_seed(0) + image = pipe("a turtle", num_inference_steps=2, generator=generator, output_type="np").images[0] + + max_diff = numpy_cosine_similarity_distance(image.flatten(), image_ckpt.flatten()) + assert max_diff < 1e-3 + + def test_stable_diffusion_text2img_intermediate_state_v_pred(self): + number_of_steps = 0 + + def test_callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + test_callback_fn.has_been_called = True + nonlocal number_of_steps + number_of_steps += 1 + if step == 0: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 96, 96) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array([0.7749, 0.0325, 0.5088, 0.1619, 0.3372, 0.3667, -0.5186, 0.6860, 1.4326]) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + elif step == 19: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 96, 96) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array([1.3887, 1.0273, 1.7266, 0.0726, 0.6611, 0.1598, -1.0547, 0.1522, 0.0227]) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + + test_callback_fn.has_been_called = False + + pipe = StableDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2", torch_dtype=torch.float16) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + prompt = "Andromeda galaxy in a bottle" + + generator = torch.manual_seed(0) + pipe( + prompt=prompt, + num_inference_steps=20, + guidance_scale=7.5, + generator=generator, + callback=test_callback_fn, + callback_steps=1, + ) + assert test_callback_fn.has_been_called + assert number_of_steps == 20 + + def test_stable_diffusion_low_cpu_mem_usage_v_pred(self): + pipeline_id = "stabilityai/stable-diffusion-2" + + start_time = time.time() + pipeline_low_cpu_mem_usage = StableDiffusionPipeline.from_pretrained(pipeline_id, torch_dtype=torch.float16) + pipeline_low_cpu_mem_usage.to(torch_device) + low_cpu_mem_usage_time = time.time() - start_time + + start_time = time.time() + _ = StableDiffusionPipeline.from_pretrained(pipeline_id, torch_dtype=torch.float16, low_cpu_mem_usage=False) + normal_load_time = time.time() - start_time + + assert 2 * low_cpu_mem_usage_time < normal_load_time + + def test_stable_diffusion_pipeline_with_sequential_cpu_offloading_v_pred(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipeline_id = "stabilityai/stable-diffusion-2" + prompt = "Andromeda galaxy in a bottle" + + pipeline = StableDiffusionPipeline.from_pretrained(pipeline_id, torch_dtype=torch.float16) + pipeline = pipeline.to(torch_device) + pipeline.enable_attention_slicing(1) + pipeline.enable_sequential_cpu_offload() + + generator = torch.manual_seed(0) + _ = pipeline(prompt, generator=generator, num_inference_steps=5) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.8 GB is allocated + assert mem_bytes < 2.8 * 10**9 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_adapter/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_adapter/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_adapter/test_stable_diffusion_adapter.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_adapter/test_stable_diffusion_adapter.py new file mode 100755 index 0000000..f1b61c3 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_adapter/test_stable_diffusion_adapter.py @@ -0,0 +1,950 @@ +# coding=utf-8 +# Copyright 2022 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from parameterized import parameterized +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +import diffusers +from diffusers import ( + AutoencoderKL, + LCMScheduler, + MultiAdapter, + PNDMScheduler, + StableDiffusionAdapterPipeline, + T2IAdapter, + UNet2DConditionModel, +) +from diffusers.utils import logging +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) + +from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +enable_full_determinism() + + +class AdapterTests: + pipeline_class = StableDiffusionAdapterPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + + def get_dummy_components(self, adapter_type, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("CrossAttnDownBlock2D", "DownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + time_cond_proj_dim=time_cond_proj_dim, + ) + scheduler = PNDMScheduler(skip_prk_steps=True) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + torch.manual_seed(0) + + if adapter_type == "full_adapter" or adapter_type == "light_adapter": + adapter = T2IAdapter( + in_channels=3, + channels=[32, 64], + num_res_blocks=2, + downscale_factor=2, + adapter_type=adapter_type, + ) + elif adapter_type == "multi_adapter": + adapter = MultiAdapter( + [ + T2IAdapter( + in_channels=3, + channels=[32, 64], + num_res_blocks=2, + downscale_factor=2, + adapter_type="full_adapter", + ), + T2IAdapter( + in_channels=3, + channels=[32, 64], + num_res_blocks=2, + downscale_factor=2, + adapter_type="full_adapter", + ), + ] + ) + else: + raise ValueError( + f"Unknown adapter type: {adapter_type}, must be one of 'full_adapter', 'light_adapter', or 'multi_adapter''" + ) + + components = { + "adapter": adapter, + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + } + return components + + def get_dummy_components_with_full_downscaling(self, adapter_type): + """Get dummy components with x8 VAE downscaling and 4 UNet down blocks. + These dummy components are intended to fully-exercise the T2I-Adapter + downscaling behavior. + """ + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 32, 32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D", "DownBlock2D"), + up_block_types=("UpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "CrossAttnUpBlock2D"), + cross_attention_dim=32, + ) + scheduler = PNDMScheduler(skip_prk_steps=True) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 32, 32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + torch.manual_seed(0) + + if adapter_type == "full_adapter" or adapter_type == "light_adapter": + adapter = T2IAdapter( + in_channels=3, + channels=[32, 32, 32, 64], + num_res_blocks=2, + downscale_factor=8, + adapter_type=adapter_type, + ) + elif adapter_type == "multi_adapter": + adapter = MultiAdapter( + [ + T2IAdapter( + in_channels=3, + channels=[32, 32, 32, 64], + num_res_blocks=2, + downscale_factor=8, + adapter_type="full_adapter", + ), + T2IAdapter( + in_channels=3, + channels=[32, 32, 32, 64], + num_res_blocks=2, + downscale_factor=8, + adapter_type="full_adapter", + ), + ] + ) + else: + raise ValueError( + f"Unknown adapter type: {adapter_type}, must be one of 'full_adapter', 'light_adapter', or 'multi_adapter''" + ) + + components = { + "adapter": adapter, + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + } + return components + + def get_dummy_inputs(self, device, seed=0, height=64, width=64, num_images=1): + if num_images == 1: + image = floats_tensor((1, 3, height, width), rng=random.Random(seed)).to(device) + else: + image = [ + floats_tensor((1, 3, height, width), rng=random.Random(seed)).to(device) for _ in range(num_images) + ] + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_attention_slicing_forward_pass(self): + return self._test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=2e-3) + + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=2e-3) + + @parameterized.expand( + [ + # (dim=264) The internal feature map will be 33x33 after initial pixel unshuffling (downscaled x8). + (((4 * 8 + 1) * 8),), + # (dim=272) The internal feature map will be 17x17 after the first T2I down block (downscaled x16). + (((4 * 4 + 1) * 16),), + # (dim=288) The internal feature map will be 9x9 after the second T2I down block (downscaled x32). + (((4 * 2 + 1) * 32),), + # (dim=320) The internal feature map will be 5x5 after the third T2I down block (downscaled x64). + (((4 * 1 + 1) * 64),), + ] + ) + def test_multiple_image_dimensions(self, dim): + """Test that the T2I-Adapter pipeline supports any input dimension that + is divisible by the adapter's `downscale_factor`. This test was added in + response to an issue where the T2I Adapter's downscaling padding + behavior did not match the UNet's behavior. + + Note that we have selected `dim` values to produce odd resolutions at + each downscaling level. + """ + components = self.get_dummy_components_with_full_downscaling() + sd_pipe = StableDiffusionAdapterPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device, height=dim, width=dim) + image = sd_pipe(**inputs).images + + assert image.shape == (1, dim, dim, 3) + + def test_adapter_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionAdapterPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4535, 0.5493, 0.4359, 0.5452, 0.6086, 0.4441, 0.5544, 0.501, 0.4859]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_adapter_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionAdapterPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4535, 0.5493, 0.4359, 0.5452, 0.6086, 0.4441, 0.5544, 0.501, 0.4859]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + +class StableDiffusionFullAdapterPipelineFastTests(AdapterTests, PipelineTesterMixin, unittest.TestCase): + def get_dummy_components(self, time_cond_proj_dim=None): + return super().get_dummy_components("full_adapter", time_cond_proj_dim=time_cond_proj_dim) + + def get_dummy_components_with_full_downscaling(self): + return super().get_dummy_components_with_full_downscaling("full_adapter") + + def test_stable_diffusion_adapter_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionAdapterPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4858, 0.5500, 0.4278, 0.4669, 0.6184, 0.4322, 0.5010, 0.5033, 0.4746]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-3 + + +class StableDiffusionLightAdapterPipelineFastTests(AdapterTests, PipelineTesterMixin, unittest.TestCase): + def get_dummy_components(self, time_cond_proj_dim=None): + return super().get_dummy_components("light_adapter", time_cond_proj_dim=time_cond_proj_dim) + + def get_dummy_components_with_full_downscaling(self): + return super().get_dummy_components_with_full_downscaling("light_adapter") + + def test_stable_diffusion_adapter_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionAdapterPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4965, 0.5548, 0.4330, 0.4771, 0.6226, 0.4382, 0.5037, 0.5071, 0.4782]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-3 + + +class StableDiffusionMultiAdapterPipelineFastTests(AdapterTests, PipelineTesterMixin, unittest.TestCase): + def get_dummy_components(self, time_cond_proj_dim=None): + return super().get_dummy_components("multi_adapter", time_cond_proj_dim=time_cond_proj_dim) + + def get_dummy_components_with_full_downscaling(self): + return super().get_dummy_components_with_full_downscaling("multi_adapter") + + def get_dummy_inputs(self, device, height=64, width=64, seed=0): + inputs = super().get_dummy_inputs(device, seed, height=height, width=width, num_images=2) + inputs["adapter_conditioning_scale"] = [0.5, 0.5] + return inputs + + def test_stable_diffusion_adapter_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionAdapterPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4902, 0.5539, 0.4317, 0.4682, 0.6190, 0.4351, 0.5018, 0.5046, 0.4772]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-3 + + def test_inference_batch_consistent( + self, batch_sizes=[2, 4, 13], additional_params_copy_to_batched_inputs=["num_inference_steps"] + ): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + logger = logging.get_logger(pipe.__module__) + logger.setLevel(level=diffusers.logging.FATAL) + + # batchify inputs + for batch_size in batch_sizes: + batched_inputs = {} + for name, value in inputs.items(): + if name in self.batch_params: + # prompt is string + if name == "prompt": + len_prompt = len(value) + # make unequal batch sizes + batched_inputs[name] = [value[: len_prompt // i] for i in range(1, batch_size + 1)] + + # make last batch super long + batched_inputs[name][-1] = 100 * "very long" + elif name == "image": + batched_images = [] + + for image in value: + batched_images.append(batch_size * [image]) + + batched_inputs[name] = batched_images + else: + batched_inputs[name] = batch_size * [value] + + elif name == "batch_size": + batched_inputs[name] = batch_size + else: + batched_inputs[name] = value + + for arg in additional_params_copy_to_batched_inputs: + batched_inputs[arg] = inputs[arg] + + batched_inputs["output_type"] = "np" + + if self.pipeline_class.__name__ == "DanceDiffusionPipeline": + batched_inputs.pop("output_type") + + output = pipe(**batched_inputs) + + assert len(output[0]) == batch_size + + batched_inputs["output_type"] = "np" + + if self.pipeline_class.__name__ == "DanceDiffusionPipeline": + batched_inputs.pop("output_type") + + output = pipe(**batched_inputs)[0] + + assert output.shape[0] == batch_size + + logger.setLevel(level=diffusers.logging.WARNING) + + def test_num_images_per_prompt(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + batch_sizes = [1, 2] + num_images_per_prompts = [1, 2] + + for batch_size in batch_sizes: + for num_images_per_prompt in num_images_per_prompts: + inputs = self.get_dummy_inputs(torch_device) + + for key in inputs.keys(): + if key in self.batch_params: + if key == "image": + batched_images = [] + + for image in inputs[key]: + batched_images.append(batch_size * [image]) + + inputs[key] = batched_images + else: + inputs[key] = batch_size * [inputs[key]] + + images = pipe(**inputs, num_images_per_prompt=num_images_per_prompt)[0] + + assert images.shape[0] == batch_size * num_images_per_prompt + + def test_inference_batch_single_identical( + self, + batch_size=3, + test_max_difference=None, + test_mean_pixel_difference=None, + relax_max_difference=False, + expected_max_diff=2e-3, + additional_params_copy_to_batched_inputs=["num_inference_steps"], + ): + if test_max_difference is None: + # TODO(Pedro) - not sure why, but not at all reproducible at the moment it seems + # make sure that batched and non-batched is identical + test_max_difference = torch_device != "mps" + + if test_mean_pixel_difference is None: + # TODO same as above + test_mean_pixel_difference = torch_device != "mps" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + logger = logging.get_logger(pipe.__module__) + logger.setLevel(level=diffusers.logging.FATAL) + + # batchify inputs + batched_inputs = {} + batch_size = batch_size + for name, value in inputs.items(): + if name in self.batch_params: + # prompt is string + if name == "prompt": + len_prompt = len(value) + # make unequal batch sizes + batched_inputs[name] = [value[: len_prompt // i] for i in range(1, batch_size + 1)] + + # make last batch super long + batched_inputs[name][-1] = 100 * "very long" + elif name == "image": + batched_images = [] + + for image in value: + batched_images.append(batch_size * [image]) + + batched_inputs[name] = batched_images + else: + batched_inputs[name] = batch_size * [value] + elif name == "batch_size": + batched_inputs[name] = batch_size + elif name == "generator": + batched_inputs[name] = [self.get_generator(i) for i in range(batch_size)] + else: + batched_inputs[name] = value + + for arg in additional_params_copy_to_batched_inputs: + batched_inputs[arg] = inputs[arg] + + if self.pipeline_class.__name__ != "DanceDiffusionPipeline": + batched_inputs["output_type"] = "np" + + output_batch = pipe(**batched_inputs) + assert output_batch[0].shape[0] == batch_size + + inputs["generator"] = self.get_generator(0) + + output = pipe(**inputs) + + logger.setLevel(level=diffusers.logging.WARNING) + if test_max_difference: + if relax_max_difference: + # Taking the median of the largest differences + # is resilient to outliers + diff = np.abs(output_batch[0][0] - output[0][0]) + diff = diff.flatten() + diff.sort() + max_diff = np.median(diff[-5:]) + else: + max_diff = np.abs(output_batch[0][0] - output[0][0]).max() + assert max_diff < expected_max_diff + + if test_mean_pixel_difference: + assert_mean_pixel_difference(output_batch[0][0], output[0][0]) + + +@slow +@require_torch_gpu +class StableDiffusionAdapterPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_diffusion_adapter_color(self): + adapter_model = "TencentARC/t2iadapter_color_sd14v1" + sd_model = "CompVis/stable-diffusion-v1-4" + prompt = "snail" + image_url = ( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/color.png" + ) + input_channels = 3 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_color_sd14v1.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + generator = torch.Generator(device="cpu").manual_seed(0) + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_depth(self): + adapter_model = "TencentARC/t2iadapter_depth_sd14v1" + sd_model = "CompVis/stable-diffusion-v1-4" + prompt = "snail" + image_url = ( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/color.png" + ) + input_channels = 3 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_color_sd14v1.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + generator = torch.Generator(device="cpu").manual_seed(0) + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_depth_sd_v14(self): + adapter_model = "TencentARC/t2iadapter_depth_sd14v1" + sd_model = "CompVis/stable-diffusion-v1-4" + prompt = "desk" + image_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/desk_depth.png" + input_channels = 3 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_depth_sd14v1.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + generator = torch.Generator(device="cpu").manual_seed(0) + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_depth_sd_v15(self): + adapter_model = "TencentARC/t2iadapter_depth_sd15v2" + sd_model = "runwayml/stable-diffusion-v1-5" + prompt = "desk" + image_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/desk_depth.png" + input_channels = 3 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_depth_sd15v2.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + generator = torch.Generator(device="cpu").manual_seed(0) + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_keypose_sd_v14(self): + adapter_model = "TencentARC/t2iadapter_keypose_sd14v1" + sd_model = "CompVis/stable-diffusion-v1-4" + prompt = "person" + image_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/person_keypose.png" + input_channels = 3 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_keypose_sd14v1.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + generator = torch.Generator(device="cpu").manual_seed(0) + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_openpose_sd_v14(self): + adapter_model = "TencentARC/t2iadapter_openpose_sd14v1" + sd_model = "CompVis/stable-diffusion-v1-4" + prompt = "person" + image_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/iron_man_pose.png" + input_channels = 3 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_openpose_sd14v1.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + generator = torch.Generator(device="cpu").manual_seed(0) + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_seg_sd_v14(self): + adapter_model = "TencentARC/t2iadapter_seg_sd14v1" + sd_model = "CompVis/stable-diffusion-v1-4" + prompt = "motorcycle" + image_url = ( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/motor.png" + ) + input_channels = 3 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_seg_sd14v1.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + generator = torch.Generator(device="cpu").manual_seed(0) + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_zoedepth_sd_v15(self): + adapter_model = "TencentARC/t2iadapter_zoedepth_sd15v1" + sd_model = "runwayml/stable-diffusion-v1-5" + prompt = "motorcycle" + image_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/motorcycle.png" + input_channels = 3 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_zoedepth_sd15v1.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_model_cpu_offload() + generator = torch.Generator(device="cpu").manual_seed(0) + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_canny_sd_v14(self): + adapter_model = "TencentARC/t2iadapter_canny_sd14v1" + sd_model = "CompVis/stable-diffusion-v1-4" + prompt = "toy" + image_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/toy_canny.png" + input_channels = 1 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_canny_sd14v1.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + generator = torch.Generator(device="cpu").manual_seed(0) + + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_canny_sd_v15(self): + adapter_model = "TencentARC/t2iadapter_canny_sd15v2" + sd_model = "runwayml/stable-diffusion-v1-5" + prompt = "toy" + image_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/toy_canny.png" + input_channels = 1 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_canny_sd15v2.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + generator = torch.Generator(device="cpu").manual_seed(0) + + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_sketch_sd14(self): + adapter_model = "TencentARC/t2iadapter_sketch_sd14v1" + sd_model = "CompVis/stable-diffusion-v1-4" + prompt = "cat" + image_url = ( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/edge.png" + ) + input_channels = 1 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_sketch_sd14v1.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + generator = torch.Generator(device="cpu").manual_seed(0) + + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_sketch_sd15(self): + adapter_model = "TencentARC/t2iadapter_sketch_sd15v2" + sd_model = "runwayml/stable-diffusion-v1-5" + prompt = "cat" + image_url = ( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/edge.png" + ) + input_channels = 1 + out_url = "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/t2iadapter_sketch_sd15v2.npy" + + image = load_image(image_url) + expected_out = load_numpy(out_url) + if input_channels == 1: + image = image.convert("L") + + adapter = T2IAdapter.from_pretrained(adapter_model, torch_dtype=torch.float16) + + pipe = StableDiffusionAdapterPipeline.from_pretrained(sd_model, adapter=adapter, safety_checker=None) + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + generator = torch.Generator(device="cpu").manual_seed(0) + + out = pipe(prompt=prompt, image=image, generator=generator, num_inference_steps=2, output_type="np").images + + max_diff = numpy_cosine_similarity_distance(out.flatten(), expected_out.flatten()) + assert max_diff < 1e-2 + + def test_stable_diffusion_adapter_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + adapter = T2IAdapter.from_pretrained("TencentARC/t2iadapter_seg_sd14v1") + pipe = StableDiffusionAdapterPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", adapter=adapter, safety_checker=None + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/motor.png" + ) + + pipe(prompt="foo", image=image, num_inference_steps=2) + + mem_bytes = torch.cuda.max_memory_allocated() + assert mem_bytes < 5 * 10**9 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen/test_stable_diffusion_gligen.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen/test_stable_diffusion_gligen.py new file mode 100755 index 0000000..3b8383b --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen/test_stable_diffusion_gligen.py @@ -0,0 +1,162 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + EulerAncestralDiscreteScheduler, + StableDiffusionGLIGENPipeline, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import enable_full_determinism + +from ..pipeline_params import ( + TEXT_TO_IMAGE_BATCH_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_PARAMS, +) +from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +class GligenPipelineFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionGLIGENPipeline + params = TEXT_TO_IMAGE_PARAMS | {"gligen_phrases", "gligen_boxes"} + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + attention_type="gated", + ) + # unet.position_net = PositionNet(32,32) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A modern livingroom", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "gligen_phrases": ["a birthday cake"], + "gligen_boxes": [[0.2676, 0.6088, 0.4773, 0.7183]], + "output_type": "np", + } + return inputs + + def test_stable_diffusion_gligen_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionGLIGENPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5069, 0.5561, 0.4577, 0.4792, 0.5203, 0.4089, 0.5039, 0.4919, 0.4499]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_gligen_k_euler_ancestral(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionGLIGENPipeline(**components) + sd_pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.425, 0.494, 0.429, 0.469, 0.525, 0.417, 0.533, 0.5, 0.47]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_attention_slicing_forward_pass(self): + super().test_attention_slicing_forward_pass(expected_max_diff=3e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(batch_size=3, expected_max_diff=3e-3) diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen_text_image/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen_text_image/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen_text_image/test_stable_diffusion_gligen_text_image.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen_text_image/test_stable_diffusion_gligen_text_image.py new file mode 100755 index 0000000..111e8d8 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_gligen_text_image/test_stable_diffusion_gligen_text_image.py @@ -0,0 +1,192 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch +from transformers import ( + CLIPProcessor, + CLIPTextConfig, + CLIPTextModel, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + EulerAncestralDiscreteScheduler, + StableDiffusionGLIGENTextImagePipeline, + UNet2DConditionModel, +) +from diffusers.pipelines.stable_diffusion import CLIPImageProjection +from diffusers.utils import load_image +from diffusers.utils.testing_utils import enable_full_determinism + +from ..pipeline_params import ( + TEXT_TO_IMAGE_BATCH_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_PARAMS, +) +from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +class GligenTextImagePipelineFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionGLIGENTextImagePipeline + params = TEXT_TO_IMAGE_PARAMS | {"gligen_phrases", "gligen_images", "gligen_boxes"} + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + attention_type="gated-text-image", + ) + # unet.position_net = PositionNet(32,32) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + image_encoder_config = CLIPVisionConfig( + hidden_size=32, + projection_dim=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + ) + image_encoder = CLIPVisionModelWithProjection(image_encoder_config) + processor = CLIPProcessor.from_pretrained("openai/clip-vit-large-patch14") + + image_project = CLIPImageProjection(hidden_size=32) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": image_encoder, + "image_project": image_project, + "processor": processor, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + gligen_images = load_image( + "https://hf.co/datasets/huggingface/documentation-images/resolve/main/diffusers/gligen/livingroom_modern.png" + ) + inputs = { + "prompt": "A modern livingroom", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "gligen_phrases": ["a birthday cake"], + "gligen_images": [gligen_images], + "gligen_boxes": [[0.2676, 0.6088, 0.4773, 0.7183]], + "output_type": "np", + } + return inputs + + def test_stable_diffusion_gligen_text_image_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionGLIGENTextImagePipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5069, 0.5561, 0.4577, 0.4792, 0.5203, 0.4089, 0.5039, 0.4919, 0.4499]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_gligen_k_euler_ancestral(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionGLIGENTextImagePipeline(**components) + sd_pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.425, 0.494, 0.429, 0.469, 0.525, 0.417, 0.533, 0.5, 0.47]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_attention_slicing_forward_pass(self): + super().test_attention_slicing_forward_pass(expected_max_diff=3e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(batch_size=3, expected_max_diff=3e-3) diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_image_variation/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_image_variation/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_image_variation/test_stable_diffusion_image_variation.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_image_variation/test_stable_diffusion_image_variation.py new file mode 100755 index 0000000..4dd7de7 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_image_variation/test_stable_diffusion_image_variation.py @@ -0,0 +1,330 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import CLIPImageProcessor, CLIPVisionConfig, CLIPVisionModelWithProjection + +from diffusers import ( + AutoencoderKL, + DPMSolverMultistepScheduler, + PNDMScheduler, + StableDiffusionImageVariationPipeline, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + nightly, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) + +from ..pipeline_params import IMAGE_VARIATION_BATCH_PARAMS, IMAGE_VARIATION_PARAMS +from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +class StableDiffusionImageVariationPipelineFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionImageVariationPipeline + params = IMAGE_VARIATION_PARAMS + batch_params = IMAGE_VARIATION_BATCH_PARAMS + image_params = frozenset([]) + # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess + image_latents_params = frozenset([]) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + scheduler = PNDMScheduler(skip_prk_steps=True) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + image_encoder_config = CLIPVisionConfig( + hidden_size=32, + projection_dim=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + image_size=32, + patch_size=4, + ) + image_encoder = CLIPVisionModelWithProjection(image_encoder_config) + feature_extractor = CLIPImageProcessor(crop_size=32, size=32) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "image_encoder": image_encoder, + "feature_extractor": feature_extractor, + "safety_checker": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)) + image = image.cpu().permute(0, 2, 3, 1)[0] + image = Image.fromarray(np.uint8(image)).convert("RGB").resize((32, 32)) + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_img_variation_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionImageVariationPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5239, 0.5723, 0.4796, 0.5049, 0.5550, 0.4685, 0.5329, 0.4891, 0.4921]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_stable_diffusion_img_variation_multiple_images(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionImageVariationPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["image"] = 2 * [inputs["image"]] + output = sd_pipe(**inputs) + + image = output.images + + image_slice = image[-1, -3:, -3:, -1] + + assert image.shape == (2, 64, 64, 3) + expected_slice = np.array([0.6892, 0.5637, 0.5836, 0.5771, 0.6254, 0.6409, 0.5580, 0.5569, 0.5289]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + +@slow +@require_torch_gpu +class StableDiffusionImageVariationPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_imgvar/input_image_vermeer.png" + ) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "image": init_image, + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "np", + } + return inputs + + def test_stable_diffusion_img_variation_pipeline_default(self): + sd_pipe = StableDiffusionImageVariationPipeline.from_pretrained( + "lambdalabs/sd-image-variations-diffusers", safety_checker=None + ) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + generator_device = "cpu" + inputs = self.get_inputs(generator_device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.8449, 0.9079, 0.7571, 0.7873, 0.8348, 0.7010, 0.6694, 0.6873, 0.6138]) + + max_diff = numpy_cosine_similarity_distance(image_slice, expected_slice) + assert max_diff < 1e-4 + + def test_stable_diffusion_img_variation_intermediate_state(self): + number_of_steps = 0 + + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True + nonlocal number_of_steps + number_of_steps += 1 + if step == 1: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 64) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array([-0.7974, -0.4343, -1.087, 0.04785, -1.327, 0.855, -2.148, -0.1725, 1.439]) + max_diff = numpy_cosine_similarity_distance(latents_slice.flatten(), expected_slice) + + assert max_diff < 1e-3 + + elif step == 2: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 64) + latents_slice = latents[0, -3:, -3:, -1] + expected_slice = np.array([0.3232, 0.004883, 0.913, -1.084, 0.6143, -1.6875, -2.463, -0.439, -0.419]) + max_diff = numpy_cosine_similarity_distance(latents_slice.flatten(), expected_slice) + + assert max_diff < 1e-3 + + callback_fn.has_been_called = False + + pipe = StableDiffusionImageVariationPipeline.from_pretrained( + "lambdalabs/sd-image-variations-diffusers", + safety_checker=None, + torch_dtype=torch.float16, + ) + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + generator_device = "cpu" + inputs = self.get_inputs(generator_device, dtype=torch.float16) + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == inputs["num_inference_steps"] + + def test_stable_diffusion_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe = StableDiffusionImageVariationPipeline.from_pretrained( + "lambdalabs/sd-image-variations-diffusers", safety_checker=None, torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + inputs = self.get_inputs(torch_device, dtype=torch.float16) + _ = pipe(**inputs) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 2.6 GB is allocated + assert mem_bytes < 2.6 * 10**9 + + +@nightly +@require_torch_gpu +class StableDiffusionImageVariationPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + init_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_imgvar/input_image_vermeer.png" + ) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "image": init_image, + "latents": latents, + "generator": generator, + "num_inference_steps": 50, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_img_variation_pndm(self): + sd_pipe = StableDiffusionImageVariationPipeline.from_pretrained("fusing/sd-image-variations-diffusers") + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_imgvar/lambdalabs_variations_pndm.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 + + def test_img_variation_dpm(self): + sd_pipe = StableDiffusionImageVariationPipeline.from_pretrained("fusing/sd-image-variations-diffusers") + sd_pipe.scheduler = DPMSolverMultistepScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + inputs["num_inference_steps"] = 25 + image = sd_pipe(**inputs).images[0] + + expected_image = load_numpy( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_imgvar/lambdalabs_variations_dpm_multi.npy" + ) + max_diff = np.abs(expected_image - image).max() + assert max_diff < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_k_diffusion/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_k_diffusion/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_k_diffusion/test_stable_diffusion_k_diffusion.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_k_diffusion/test_stable_diffusion_k_diffusion.py new file mode 100755 index 0000000..65b4f23 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_k_diffusion/test_stable_diffusion_k_diffusion.py @@ -0,0 +1,135 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch + +from diffusers import StableDiffusionKDiffusionPipeline +from diffusers.utils.testing_utils import enable_full_determinism, nightly, require_torch_gpu, torch_device + + +enable_full_determinism() + + +@nightly +@require_torch_gpu +class StableDiffusionPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_diffusion_1(self): + sd_pipe = StableDiffusionKDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + sd_pipe.set_scheduler("sample_euler") + + prompt = "A painting of a squirrel eating a burger" + generator = torch.manual_seed(0) + output = sd_pipe([prompt], generator=generator, guidance_scale=9.0, num_inference_steps=20, output_type="np") + + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.0447, 0.0492, 0.0468, 0.0408, 0.0383, 0.0408, 0.0354, 0.0380, 0.0339]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_2(self): + sd_pipe = StableDiffusionKDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base") + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + sd_pipe.set_scheduler("sample_euler") + + prompt = "A painting of a squirrel eating a burger" + generator = torch.manual_seed(0) + output = sd_pipe([prompt], generator=generator, guidance_scale=9.0, num_inference_steps=20, output_type="np") + + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.1237, 0.1320, 0.1438, 0.1359, 0.1390, 0.1132, 0.1277, 0.1175, 0.1112]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-1 + + def test_stable_diffusion_karras_sigmas(self): + sd_pipe = StableDiffusionKDiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base") + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + sd_pipe.set_scheduler("sample_dpmpp_2m") + + prompt = "A painting of a squirrel eating a burger" + generator = torch.manual_seed(0) + output = sd_pipe( + [prompt], + generator=generator, + guidance_scale=7.5, + num_inference_steps=15, + output_type="np", + use_karras_sigmas=True, + ) + + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array( + [0.11381689, 0.12112921, 0.1389457, 0.12549606, 0.1244964, 0.10831517, 0.11562866, 0.10867816, 0.10499048] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_noise_sampler_seed(self): + sd_pipe = StableDiffusionKDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + sd_pipe.set_scheduler("sample_dpmpp_sde") + + prompt = "A painting of a squirrel eating a burger" + seed = 0 + images1 = sd_pipe( + [prompt], + generator=torch.manual_seed(seed), + noise_sampler_seed=seed, + guidance_scale=9.0, + num_inference_steps=20, + output_type="np", + ).images + images2 = sd_pipe( + [prompt], + generator=torch.manual_seed(seed), + noise_sampler_seed=seed, + guidance_scale=9.0, + num_inference_steps=20, + output_type="np", + ).images + + assert images1.shape == (1, 512, 512, 3) + assert images2.shape == (1, 512, 512, 3) + assert np.abs(images1.flatten() - images2.flatten()).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_ldm3d/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_ldm3d/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_ldm3d/test_stable_diffusion_ldm3d.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_ldm3d/test_stable_diffusion_ldm3d.py new file mode 100755 index 0000000..9ac69c8 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_ldm3d/test_stable_diffusion_ldm3d.py @@ -0,0 +1,310 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import gc +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + PNDMScheduler, + StableDiffusionLDM3DPipeline, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import enable_full_determinism, nightly, require_torch_gpu, torch_device + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS + + +enable_full_determinism() + + +class StableDiffusionLDM3DPipelineFastTests(unittest.TestCase): + pipeline_class = StableDiffusionLDM3DPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=6, + out_channels=6, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_ddim(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components() + ldm3d_pipe = StableDiffusionLDM3DPipeline(**components) + ldm3d_pipe = ldm3d_pipe.to(torch_device) + ldm3d_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = ldm3d_pipe(**inputs) + rgb, depth = output.rgb, output.depth + + image_slice_rgb = rgb[0, -3:, -3:, -1] + image_slice_depth = depth[0, -3:, -1] + + assert rgb.shape == (1, 64, 64, 3) + assert depth.shape == (1, 64, 64) + + expected_slice_rgb = np.array( + [0.37338176, 0.70247, 0.74203193, 0.51643604, 0.58256793, 0.60932136, 0.4181095, 0.48355877, 0.46535262] + ) + expected_slice_depth = np.array([103.46727, 85.812004, 87.849236]) + + assert np.abs(image_slice_rgb.flatten() - expected_slice_rgb).max() < 1e-2 + assert np.abs(image_slice_depth.flatten() - expected_slice_depth).max() < 1e-2 + + def test_stable_diffusion_prompt_embeds(self): + components = self.get_dummy_components() + ldm3d_pipe = StableDiffusionLDM3DPipeline(**components) + ldm3d_pipe = ldm3d_pipe.to(torch_device) + ldm3d_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt"] = 3 * [inputs["prompt"]] + + # forward + output = ldm3d_pipe(**inputs) + rgb_slice_1, depth_slice_1 = output.rgb, output.depth + rgb_slice_1 = rgb_slice_1[0, -3:, -3:, -1] + depth_slice_1 = depth_slice_1[0, -3:, -1] + + inputs = self.get_dummy_inputs(torch_device) + prompt = 3 * [inputs.pop("prompt")] + + text_inputs = ldm3d_pipe.tokenizer( + prompt, + padding="max_length", + max_length=ldm3d_pipe.tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + text_inputs = text_inputs["input_ids"].to(torch_device) + + prompt_embeds = ldm3d_pipe.text_encoder(text_inputs)[0] + + inputs["prompt_embeds"] = prompt_embeds + + # forward + output = ldm3d_pipe(**inputs) + rgb_slice_2, depth_slice_2 = output.rgb, output.depth + rgb_slice_2 = rgb_slice_2[0, -3:, -3:, -1] + depth_slice_2 = depth_slice_2[0, -3:, -1] + + assert np.abs(rgb_slice_1.flatten() - rgb_slice_2.flatten()).max() < 1e-4 + assert np.abs(depth_slice_1.flatten() - depth_slice_2.flatten()).max() < 1e-4 + + def test_stable_diffusion_negative_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = PNDMScheduler(skip_prk_steps=True) + ldm3d_pipe = StableDiffusionLDM3DPipeline(**components) + ldm3d_pipe = ldm3d_pipe.to(device) + ldm3d_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + negative_prompt = "french fries" + output = ldm3d_pipe(**inputs, negative_prompt=negative_prompt) + + rgb, depth = output.rgb, output.depth + rgb_slice = rgb[0, -3:, -3:, -1] + depth_slice = depth[0, -3:, -1] + + assert rgb.shape == (1, 64, 64, 3) + assert depth.shape == (1, 64, 64) + + expected_slice_rgb = np.array( + [0.37044, 0.71811503, 0.7223251, 0.48603675, 0.5638391, 0.6364948, 0.42833704, 0.4901315, 0.47926217] + ) + expected_slice_depth = np.array([107.84738, 84.62802, 89.962135]) + assert np.abs(rgb_slice.flatten() - expected_slice_rgb).max() < 1e-2 + assert np.abs(depth_slice.flatten() - expected_slice_depth).max() < 1e-2 + + +@nightly +@require_torch_gpu +class StableDiffusionLDM3DPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_ldm3d_stable_diffusion(self): + ldm3d_pipe = StableDiffusionLDM3DPipeline.from_pretrained("Intel/ldm3d") + ldm3d_pipe = ldm3d_pipe.to(torch_device) + ldm3d_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + output = ldm3d_pipe(**inputs) + rgb, depth = output.rgb, output.depth + rgb_slice = rgb[0, -3:, -3:, -1].flatten() + depth_slice = rgb[0, -3:, -1].flatten() + + assert rgb.shape == (1, 512, 512, 3) + assert depth.shape == (1, 512, 512) + + expected_slice_rgb = np.array( + [0.53805465, 0.56707305, 0.5486515, 0.57012236, 0.5814511, 0.56253487, 0.54843014, 0.55092263, 0.6459706] + ) + expected_slice_depth = np.array( + [0.9263781, 0.6678672, 0.5486515, 0.92202145, 0.67831135, 0.56253487, 0.9241694, 0.7551478, 0.6459706] + ) + assert np.abs(rgb_slice - expected_slice_rgb).max() < 3e-3 + assert np.abs(depth_slice - expected_slice_depth).max() < 3e-3 + + +@nightly +@require_torch_gpu +class StableDiffusionPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, generator_device="cpu", dtype=torch.float32, seed=0): + generator = torch.Generator(device=generator_device).manual_seed(seed) + latents = np.random.RandomState(seed).standard_normal((1, 4, 64, 64)) + latents = torch.from_numpy(latents).to(device=device, dtype=dtype) + inputs = { + "prompt": "a photograph of an astronaut riding a horse", + "latents": latents, + "generator": generator, + "num_inference_steps": 50, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_ldm3d(self): + ldm3d_pipe = StableDiffusionLDM3DPipeline.from_pretrained("Intel/ldm3d").to(torch_device) + ldm3d_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + output = ldm3d_pipe(**inputs) + rgb, depth = output.rgb, output.depth + + expected_rgb_mean = 0.495586 + expected_rgb_std = 0.33795515 + expected_depth_mean = 112.48518 + expected_depth_std = 98.489746 + assert np.abs(expected_rgb_mean - rgb.mean()) < 1e-3 + assert np.abs(expected_rgb_std - rgb.std()) < 1e-3 + assert np.abs(expected_depth_mean - depth.mean()) < 1e-3 + assert np.abs(expected_depth_std - depth.std()) < 1e-3 + + def test_ldm3d_v2(self): + ldm3d_pipe = StableDiffusionLDM3DPipeline.from_pretrained("Intel/ldm3d-4c").to(torch_device) + ldm3d_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_inputs(torch_device) + output = ldm3d_pipe(**inputs) + rgb, depth = output.rgb, output.depth + + expected_rgb_mean = 0.4194127 + expected_rgb_std = 0.35375586 + expected_depth_mean = 0.5638502 + expected_depth_std = 0.34686103 + + assert rgb.shape == (1, 512, 512, 3) + assert depth.shape == (1, 512, 512, 1) + assert np.abs(expected_rgb_mean - rgb.mean()) < 1e-3 + assert np.abs(expected_rgb_std - rgb.std()) < 1e-3 + assert np.abs(expected_depth_mean - depth.mean()) < 1e-3 + assert np.abs(expected_depth_std - depth.std()) < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_panorama/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_panorama/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_panorama/test_stable_diffusion_panorama.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_panorama/test_stable_diffusion_panorama.py new file mode 100755 index 0000000..aa7212b --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_panorama/test_stable_diffusion_panorama.py @@ -0,0 +1,412 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + EulerAncestralDiscreteScheduler, + LMSDiscreteScheduler, + PNDMScheduler, + StableDiffusionPanoramaPipeline, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import enable_full_determinism, nightly, require_torch_gpu, skip_mps, torch_device + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +@skip_mps +class StableDiffusionPanoramaPipelineFastTests(PipelineLatentTesterMixin, PipelineTesterMixin, unittest.TestCase): + pipeline_class = StableDiffusionPanoramaPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=1, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + scheduler = DDIMScheduler() + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + generator = torch.manual_seed(seed) + inputs = { + "prompt": "a photo of the dolomites", + "generator": generator, + # Setting height and width to None to prevent OOMs on CPU. + "height": None, + "width": None, + "num_inference_steps": 1, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_panorama_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionPanoramaPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.6186, 0.5374, 0.4915, 0.4135, 0.4114, 0.4563, 0.5128, 0.4977, 0.4757]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_panorama_circular_padding_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionPanoramaPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs, circular_padding=True).images + image_slice = image[0, -3:, -3:, -1] + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.6127, 0.6299, 0.4595, 0.4051, 0.4543, 0.3925, 0.5510, 0.5693, 0.5031]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + # override to speed the overall test timing up. + def test_inference_batch_consistent(self): + super().test_inference_batch_consistent(batch_sizes=[1, 2]) + + # override to speed the overall test timing up. + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(batch_size=2, expected_max_diff=5.0e-3) + + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=1e-1) + + def test_stable_diffusion_panorama_negative_prompt(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionPanoramaPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + negative_prompt = "french fries" + output = sd_pipe(**inputs, negative_prompt=negative_prompt) + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.6187, 0.5375, 0.4915, 0.4136, 0.4114, 0.4563, 0.5128, 0.4976, 0.4757]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_panorama_views_batch(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionPanoramaPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs, view_batch_size=2) + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.6187, 0.5375, 0.4915, 0.4136, 0.4114, 0.4563, 0.5128, 0.4976, 0.4757]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_panorama_views_batch_circular_padding(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionPanoramaPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs, circular_padding=True, view_batch_size=2) + image = output.images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.6127, 0.6299, 0.4595, 0.4051, 0.4543, 0.3925, 0.5510, 0.5693, 0.5031]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_panorama_euler(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = EulerAncestralDiscreteScheduler( + beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear" + ) + sd_pipe = StableDiffusionPanoramaPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.4024, 0.6510, 0.4901, 0.5378, 0.5813, 0.5622, 0.4795, 0.4467, 0.4952]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_panorama_pndm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + components["scheduler"] = PNDMScheduler( + beta_start=0.00085, beta_end=0.012, beta_schedule="scaled_linear", skip_prk_steps=True + ) + sd_pipe = StableDiffusionPanoramaPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.6391, 0.6291, 0.4861, 0.5134, 0.5552, 0.4578, 0.5032, 0.5023, 0.4539]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + +@nightly +@require_torch_gpu +class StableDiffusionPanoramaNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, seed=0): + generator = torch.manual_seed(seed) + inputs = { + "prompt": "a photo of the dolomites", + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 7.5, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_panorama_default(self): + model_ckpt = "stabilityai/stable-diffusion-2-base" + scheduler = DDIMScheduler.from_pretrained(model_ckpt, subfolder="scheduler") + pipe = StableDiffusionPanoramaPipeline.from_pretrained(model_ckpt, scheduler=scheduler, safety_checker=None) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 2048, 3) + + expected_slice = np.array( + [ + 0.36968392, + 0.27025372, + 0.32446766, + 0.28379387, + 0.36363274, + 0.30733347, + 0.27100027, + 0.27054125, + 0.25536096, + ] + ) + + assert np.abs(expected_slice - image_slice).max() < 1e-2 + + def test_stable_diffusion_panorama_k_lms(self): + pipe = StableDiffusionPanoramaPipeline.from_pretrained( + "stabilityai/stable-diffusion-2-base", safety_checker=None + ) + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + pipe.unet.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs() + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + assert image.shape == (1, 512, 2048, 3) + + expected_slice = np.array( + [ + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + ] + ] + ) + + assert np.abs(expected_slice - image_slice).max() < 1e-2 + + def test_stable_diffusion_panorama_intermediate_state(self): + number_of_steps = 0 + + def callback_fn(step: int, timestep: int, latents: torch.FloatTensor) -> None: + callback_fn.has_been_called = True + nonlocal number_of_steps + number_of_steps += 1 + if step == 1: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 256) + latents_slice = latents[0, -3:, -3:, -1] + + expected_slice = np.array( + [ + 0.18681869, + 0.33907816, + 0.5361276, + 0.14432865, + -0.02856611, + -0.73941123, + 0.23397987, + 0.47322682, + -0.37823164, + ] + ) + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + elif step == 2: + latents = latents.detach().cpu().numpy() + assert latents.shape == (1, 4, 64, 256) + latents_slice = latents[0, -3:, -3:, -1] + + expected_slice = np.array( + [ + 0.18539645, + 0.33987248, + 0.5378559, + 0.14437142, + -0.02455261, + -0.7338317, + 0.23990755, + 0.47356272, + -0.3786505, + ] + ) + + assert np.abs(latents_slice.flatten() - expected_slice).max() < 5e-2 + + callback_fn.has_been_called = False + + model_ckpt = "stabilityai/stable-diffusion-2-base" + scheduler = DDIMScheduler.from_pretrained(model_ckpt, subfolder="scheduler") + pipe = StableDiffusionPanoramaPipeline.from_pretrained(model_ckpt, scheduler=scheduler, safety_checker=None) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs() + pipe(**inputs, callback=callback_fn, callback_steps=1) + assert callback_fn.has_been_called + assert number_of_steps == 3 + + def test_stable_diffusion_panorama_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + model_ckpt = "stabilityai/stable-diffusion-2-base" + scheduler = DDIMScheduler.from_pretrained(model_ckpt, subfolder="scheduler") + pipe = StableDiffusionPanoramaPipeline.from_pretrained(model_ckpt, scheduler=scheduler, safety_checker=None) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing(1) + pipe.enable_sequential_cpu_offload() + + inputs = self.get_inputs() + _ = pipe(**inputs) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 5.2 GB is allocated + assert mem_bytes < 5.5 * 10**9 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_safe/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_safe/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_safe/test_safe_diffusion.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_safe/test_safe_diffusion.py new file mode 100755 index 0000000..478e465 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_safe/test_safe_diffusion.py @@ -0,0 +1,435 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import tempfile +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import AutoencoderKL, DDIMScheduler, LMSDiscreteScheduler, PNDMScheduler, UNet2DConditionModel +from diffusers.pipelines.stable_diffusion_safe import StableDiffusionPipelineSafe as StableDiffusionPipeline +from diffusers.utils.testing_utils import floats_tensor, nightly, require_torch_gpu, torch_device + + +class SafeDiffusionPipelineFastTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + @property + def dummy_image(self): + batch_size = 1 + num_channels = 3 + sizes = (32, 32) + + image = floats_tensor((batch_size, num_channels) + sizes, rng=random.Random(0)).to(torch_device) + return image + + @property + def dummy_cond_unet(self): + torch.manual_seed(0) + model = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + return model + + @property + def dummy_vae(self): + torch.manual_seed(0) + model = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + return model + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModel(config) + + @property + def dummy_extractor(self): + def extract(*args, **kwargs): + class Out: + def __init__(self): + self.pixel_values = torch.ones([0]) + + def to(self, device): + self.pixel_values.to(device) + return self + + return Out() + + return extract + + def test_safe_diffusion_ddim(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unet = self.dummy_cond_unet + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + + generator = torch.Generator(device=device).manual_seed(0) + output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np") + image = output.images + + generator = torch.Generator(device=device).manual_seed(0) + image_from_tuple = sd_pipe( + [prompt], + generator=generator, + guidance_scale=6.0, + num_inference_steps=2, + output_type="np", + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5756, 0.6118, 0.5005, 0.5041, 0.5471, 0.4726, 0.4976, 0.4865, 0.4864]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_pndm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unet = self.dummy_cond_unet + scheduler = PNDMScheduler(skip_prk_steps=True) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + generator = torch.Generator(device=device).manual_seed(0) + output = sd_pipe([prompt], generator=generator, guidance_scale=6.0, num_inference_steps=2, output_type="np") + + image = output.images + + generator = torch.Generator(device=device).manual_seed(0) + image_from_tuple = sd_pipe( + [prompt], + generator=generator, + guidance_scale=6.0, + num_inference_steps=2, + output_type="np", + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5125, 0.5716, 0.4828, 0.5060, 0.5650, 0.4768, 0.5185, 0.4895, 0.4993]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_no_safety_checker(self): + pipe = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-lms-pipe", safety_checker=None + ) + assert isinstance(pipe, StableDiffusionPipeline) + assert isinstance(pipe.scheduler, LMSDiscreteScheduler) + assert pipe.safety_checker is None + + image = pipe("example prompt", num_inference_steps=2).images[0] + assert image is not None + + # check that there's no error when saving a pipeline with one of the models being None + with tempfile.TemporaryDirectory() as tmpdirname: + pipe.save_pretrained(tmpdirname) + pipe = StableDiffusionPipeline.from_pretrained(tmpdirname) + + # sanity check that the pipeline still works + assert pipe.safety_checker is None + image = pipe("example prompt", num_inference_steps=2).images[0] + assert image is not None + + @unittest.skipIf(torch_device != "cuda", "This test requires a GPU") + def test_stable_diffusion_fp16(self): + """Test that stable diffusion works with fp16""" + unet = self.dummy_cond_unet + scheduler = PNDMScheduler(skip_prk_steps=True) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + # put models in fp16 + unet = unet.half() + vae = vae.half() + bert = bert.half() + + # make sure here that pndm scheduler skips prk + sd_pipe = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "A painting of a squirrel eating a burger" + image = sd_pipe([prompt], num_inference_steps=2, output_type="np").images + + assert image.shape == (1, 64, 64, 3) + + +@nightly +@require_torch_gpu +class SafeDiffusionPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_harm_safe_stable_diffusion(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", safety_checker=None) + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = ( + "portrait of girl with smokey eyes makeup in abandoned hotel, grange clothes, redshift, wide high angle" + " coloured polaroid photograph with flash, kodak film, hyper real, stunning moody cinematography, with" + " anamorphic lenses, by maripol, fallen angels by wong kar - wai, style of suspiria and neon demon and" + " children from bahnhof zoo, detailed " + ) + seed = 4003660346 + guidance_scale = 7 + + # without safety guidance (sld_guidance_scale = 0) + generator = torch.manual_seed(seed) + output = sd_pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + sld_guidance_scale=0, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [0.2278, 0.2231, 0.2249, 0.2333, 0.2303, 0.1885, 0.2273, 0.2144, 0.2176] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + # without safety guidance (strong configuration) + generator = torch.manual_seed(seed) + output = sd_pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + sld_guidance_scale=2000, + sld_warmup_steps=7, + sld_threshold=0.025, + sld_momentum_scale=0.5, + sld_mom_beta=0.7, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [0.2383, 0.2276, 0.236, 0.2192, 0.2186, 0.2053, 0.1971, 0.1901, 0.1719] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_nudity_safe_stable_diffusion(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5", safety_checker=None) + sd_pipe.scheduler = LMSDiscreteScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "padme amidala taking a bath artwork, safe for work, no nudity" + seed = 2734971755 + guidance_scale = 7 + + generator = torch.manual_seed(seed) + output = sd_pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + sld_guidance_scale=0, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [0.3502, 0.3622, 0.3396, 0.3642, 0.3478, 0.3318, 0.35, 0.3348, 0.3297] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + generator = torch.manual_seed(seed) + output = sd_pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + sld_guidance_scale=2000, + sld_warmup_steps=7, + sld_threshold=0.025, + sld_momentum_scale=0.5, + sld_mom_beta=0.7, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = [0.5531, 0.5206, 0.4895, 0.5156, 0.5182, 0.4751, 0.4802, 0.4803, 0.4443] + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_nudity_safetychecker_safe_stable_diffusion(self): + sd_pipe = StableDiffusionPipeline.from_pretrained("runwayml/stable-diffusion-v1-5") + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = ( + "the four horsewomen of the apocalypse, painting by tom of finland, gaston bussiere, craig mullins, j. c." + " leyendecker" + ) + seed = 1044355234 + guidance_scale = 12 + + generator = torch.manual_seed(seed) + output = sd_pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + sld_guidance_scale=0, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]) + + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-7 + + generator = torch.manual_seed(seed) + output = sd_pipe( + [prompt], + generator=generator, + guidance_scale=guidance_scale, + num_inference_steps=50, + output_type="np", + width=512, + height=512, + sld_guidance_scale=2000, + sld_warmup_steps=7, + sld_threshold=0.025, + sld_momentum_scale=0.5, + sld_mom_beta=0.7, + ) + + image = output.images + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.5818, 0.6285, 0.6835, 0.6019, 0.625, 0.6754, 0.6096, 0.6334, 0.6561]) + assert image.shape == (1, 512, 512, 3) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_sag/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_sag/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_sag/test_stable_diffusion_sag.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_sag/test_stable_diffusion_sag.py new file mode 100755 index 0000000..94a5616 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_sag/test_stable_diffusion_sag.py @@ -0,0 +1,215 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + DEISMultistepScheduler, + DPMSolverMultistepScheduler, + EulerDiscreteScheduler, + StableDiffusionSAGPipeline, + UNet2DConditionModel, +) +from diffusers.utils.testing_utils import enable_full_determinism, nightly, require_torch_gpu, torch_device + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +class StableDiffusionSAGPipelineFastTests(PipelineLatentTesterMixin, PipelineTesterMixin, unittest.TestCase): + pipeline_class = StableDiffusionSAGPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(4, 8), + layers_per_block=2, + sample_size=8, + norm_num_groups=1, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=8, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[4, 8], + norm_num_groups=1, + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=8, + num_hidden_layers=2, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": ".", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 1.0, + "sag_scale": 1.0, + "output_type": "np", + } + return inputs + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + @unittest.skip("Not necessary to test here.") + def test_xformers_attention_forwardGenerator_pass(self): + pass + + def test_pipeline_different_schedulers(self): + pipeline = self.pipeline_class(**self.get_dummy_components()) + inputs = self.get_dummy_inputs("cpu") + + expected_image_size = (16, 16, 3) + for scheduler_cls in [DDIMScheduler, DEISMultistepScheduler, DPMSolverMultistepScheduler]: + pipeline.scheduler = scheduler_cls.from_config(pipeline.scheduler.config) + image = pipeline(**inputs).images[0] + + shape = image.shape + assert shape == expected_image_size + + pipeline.scheduler = EulerDiscreteScheduler.from_config(pipeline.scheduler.config) + + with self.assertRaises(ValueError): + # Karras schedulers are not supported + image = pipeline(**inputs).images[0] + + +@nightly +@require_torch_gpu +class StableDiffusionPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_diffusion_1(self): + sag_pipe = StableDiffusionSAGPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + sag_pipe = sag_pipe.to(torch_device) + sag_pipe.set_progress_bar_config(disable=None) + + prompt = "." + generator = torch.manual_seed(0) + output = sag_pipe( + [prompt], generator=generator, guidance_scale=7.5, sag_scale=1.0, num_inference_steps=20, output_type="np" + ) + + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.1568, 0.1738, 0.1695, 0.1693, 0.1507, 0.1705, 0.1547, 0.1751, 0.1949]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-2 + + def test_stable_diffusion_2(self): + sag_pipe = StableDiffusionSAGPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base") + sag_pipe = sag_pipe.to(torch_device) + sag_pipe.set_progress_bar_config(disable=None) + + prompt = "." + generator = torch.manual_seed(0) + output = sag_pipe( + [prompt], generator=generator, guidance_scale=7.5, sag_scale=1.0, num_inference_steps=20, output_type="np" + ) + + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.3459, 0.2876, 0.2537, 0.3002, 0.2671, 0.2160, 0.3026, 0.2262, 0.2371]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-2 + + def test_stable_diffusion_2_non_square(self): + sag_pipe = StableDiffusionSAGPipeline.from_pretrained("stabilityai/stable-diffusion-2-1-base") + sag_pipe = sag_pipe.to(torch_device) + sag_pipe.set_progress_bar_config(disable=None) + + prompt = "." + generator = torch.manual_seed(0) + output = sag_pipe( + [prompt], + width=768, + height=512, + generator=generator, + guidance_scale=7.5, + sag_scale=1.0, + num_inference_steps=20, + output_type="np", + ) + + image = output.images + + assert image.shape == (1, 512, 768, 3) diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py new file mode 100755 index 0000000..a9acebb --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl.py @@ -0,0 +1,1102 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import gc +import tempfile +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + LCMScheduler, + StableDiffusionXLImg2ImgPipeline, + StableDiffusionXLPipeline, + UNet2DConditionModel, + UniPCMultistepScheduler, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + load_image, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) + +from ..pipeline_params import ( + TEXT_TO_IMAGE_BATCH_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, + TEXT_TO_IMAGE_IMAGE_PARAMS, + TEXT_TO_IMAGE_PARAMS, +) +from ..test_pipelines_common import ( + IPAdapterTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, + SDFunctionTesterMixin, + SDXLOptionalComponentsTesterMixin, +) + + +enable_full_determinism() + + +class StableDiffusionXLPipelineFastTests( + SDFunctionTesterMixin, + IPAdapterTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, + SDXLOptionalComponentsTesterMixin, + unittest.TestCase, +): + pipeline_class = StableDiffusionXLPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS.union({"add_text_embeds", "add_time_ids"}) + + def get_dummy_components(self, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(2, 4), + layers_per_block=2, + time_cond_proj_dim=time_cond_proj_dim, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + norm_num_groups=1, + ) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "image_encoder": None, + "feature_extractor": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 5.0, + "output_type": "np", + } + return inputs + + def test_stable_diffusion_xl_euler(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5552, 0.5569, 0.4725, 0.4348, 0.4994, 0.4632, 0.5142, 0.5012, 0.47]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_xl_euler_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4917, 0.6555, 0.4348, 0.5219, 0.7324, 0.4855, 0.5168, 0.5447, 0.5156]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_xl_euler_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.4917, 0.6555, 0.4348, 0.5219, 0.7324, 0.4855, 0.5168, 0.5447, 0.5156]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_xl_prompt_embeds(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + # forward without prompt embeds + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt"] = 2 * [inputs["prompt"]] + inputs["num_images_per_prompt"] = 2 + + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with prompt embeds + inputs = self.get_dummy_inputs(torch_device) + prompt = 2 * [inputs.pop("prompt")] + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = sd_pipe.encode_prompt(prompt) + + output = sd_pipe( + **inputs, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + ) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # make sure that it's equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + def test_stable_diffusion_xl_negative_prompt_embeds(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + # forward without prompt embeds + inputs = self.get_dummy_inputs(torch_device) + negative_prompt = 3 * ["this is a negative prompt"] + inputs["negative_prompt"] = negative_prompt + inputs["prompt"] = 3 * [inputs["prompt"]] + + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with prompt embeds + inputs = self.get_dummy_inputs(torch_device) + negative_prompt = 3 * ["this is a negative prompt"] + prompt = 3 * [inputs.pop("prompt")] + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = sd_pipe.encode_prompt(prompt, negative_prompt=negative_prompt) + + output = sd_pipe( + **inputs, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + ) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # make sure that it's equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + def test_attention_slicing_forward_pass(self): + super().test_attention_slicing_forward_pass(expected_max_diff=3e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + def test_save_load_optional_components(self): + self._test_save_load_optional_components() + + @require_torch_gpu + def test_stable_diffusion_xl_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLPipeline(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLPipeline(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLPipeline(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + pipe.unet.set_default_attn_processor() + + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_stable_diffusion_xl_img2img_prompt_embeds_only(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + # forward without prompt embeds + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + inputs["prompt"] = 3 * [inputs["prompt"]] + + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with prompt embeds + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + prompt = 3 * [inputs.pop("prompt")] + + ( + prompt_embeds, + _, + pooled_prompt_embeds, + _, + ) = sd_pipe.encode_prompt(prompt) + + output = sd_pipe( + **inputs, + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + ) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # make sure that it's equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + def test_stable_diffusion_two_xl_mixture_of_denoiser_fast(self): + components = self.get_dummy_components() + pipe_1 = StableDiffusionXLPipeline(**components).to(torch_device) + pipe_1.unet.set_default_attn_processor() + pipe_2 = StableDiffusionXLImg2ImgPipeline(**components).to(torch_device) + pipe_2.unet.set_default_attn_processor() + + def assert_run_mixture( + num_steps, + split, + scheduler_cls_orig, + expected_tss, + num_train_timesteps=pipe_1.scheduler.config.num_train_timesteps, + ): + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = num_steps + + class scheduler_cls(scheduler_cls_orig): + pass + + pipe_1.scheduler = scheduler_cls.from_config(pipe_1.scheduler.config) + pipe_2.scheduler = scheduler_cls.from_config(pipe_2.scheduler.config) + + # Let's retrieve the number of timesteps we want to use + pipe_1.scheduler.set_timesteps(num_steps) + expected_steps = pipe_1.scheduler.timesteps.tolist() + + if pipe_1.scheduler.order == 2: + expected_steps_1 = list(filter(lambda ts: ts >= split, expected_tss)) + expected_steps_2 = expected_steps_1[-1:] + list(filter(lambda ts: ts < split, expected_tss)) + expected_steps = expected_steps_1 + expected_steps_2 + else: + expected_steps_1 = list(filter(lambda ts: ts >= split, expected_tss)) + expected_steps_2 = list(filter(lambda ts: ts < split, expected_tss)) + + # now we monkey patch step `done_steps` + # list into the step function for testing + done_steps = [] + old_step = copy.copy(scheduler_cls.step) + + def new_step(self, *args, **kwargs): + done_steps.append(args[1].cpu().item()) # args[1] is always the passed `t` + return old_step(self, *args, **kwargs) + + scheduler_cls.step = new_step + + inputs_1 = { + **inputs, + **{ + "denoising_end": 1.0 - (split / num_train_timesteps), + "output_type": "latent", + }, + } + latents = pipe_1(**inputs_1).images[0] + + assert expected_steps_1 == done_steps, f"Failure with {scheduler_cls.__name__} and {num_steps} and {split}" + + inputs_2 = { + **inputs, + **{ + "denoising_start": 1.0 - (split / num_train_timesteps), + "image": latents, + }, + } + pipe_2(**inputs_2).images[0] + + assert expected_steps_2 == done_steps[len(expected_steps_1) :] + assert expected_steps == done_steps, f"Failure with {scheduler_cls.__name__} and {num_steps} and {split}" + + steps = 10 + for split in [300, 700]: + for scheduler_cls_timesteps in [ + (EulerDiscreteScheduler, [901, 801, 701, 601, 501, 401, 301, 201, 101, 1]), + ( + HeunDiscreteScheduler, + [ + 901.0, + 801.0, + 801.0, + 701.0, + 701.0, + 601.0, + 601.0, + 501.0, + 501.0, + 401.0, + 401.0, + 301.0, + 301.0, + 201.0, + 201.0, + 101.0, + 101.0, + 1.0, + 1.0, + ], + ), + ]: + assert_run_mixture(steps, split, scheduler_cls_timesteps[0], scheduler_cls_timesteps[1]) + + @slow + def test_stable_diffusion_two_xl_mixture_of_denoiser(self): + components = self.get_dummy_components() + pipe_1 = StableDiffusionXLPipeline(**components).to(torch_device) + pipe_1.unet.set_default_attn_processor() + pipe_2 = StableDiffusionXLImg2ImgPipeline(**components).to(torch_device) + pipe_2.unet.set_default_attn_processor() + + def assert_run_mixture( + num_steps, + split, + scheduler_cls_orig, + expected_tss, + num_train_timesteps=pipe_1.scheduler.config.num_train_timesteps, + ): + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = num_steps + + class scheduler_cls(scheduler_cls_orig): + pass + + pipe_1.scheduler = scheduler_cls.from_config(pipe_1.scheduler.config) + pipe_2.scheduler = scheduler_cls.from_config(pipe_2.scheduler.config) + + # Let's retrieve the number of timesteps we want to use + pipe_1.scheduler.set_timesteps(num_steps) + expected_steps = pipe_1.scheduler.timesteps.tolist() + + if pipe_1.scheduler.order == 2: + expected_steps_1 = list(filter(lambda ts: ts >= split, expected_tss)) + expected_steps_2 = expected_steps_1[-1:] + list(filter(lambda ts: ts < split, expected_tss)) + expected_steps = expected_steps_1 + expected_steps_2 + else: + expected_steps_1 = list(filter(lambda ts: ts >= split, expected_tss)) + expected_steps_2 = list(filter(lambda ts: ts < split, expected_tss)) + + # now we monkey patch step `done_steps` + # list into the step function for testing + done_steps = [] + old_step = copy.copy(scheduler_cls.step) + + def new_step(self, *args, **kwargs): + done_steps.append(args[1].cpu().item()) # args[1] is always the passed `t` + return old_step(self, *args, **kwargs) + + scheduler_cls.step = new_step + + inputs_1 = { + **inputs, + **{ + "denoising_end": 1.0 - (split / num_train_timesteps), + "output_type": "latent", + }, + } + latents = pipe_1(**inputs_1).images[0] + + assert expected_steps_1 == done_steps, f"Failure with {scheduler_cls.__name__} and {num_steps} and {split}" + + inputs_2 = { + **inputs, + **{ + "denoising_start": 1.0 - (split / num_train_timesteps), + "image": latents, + }, + } + pipe_2(**inputs_2).images[0] + + assert expected_steps_2 == done_steps[len(expected_steps_1) :] + assert expected_steps == done_steps, f"Failure with {scheduler_cls.__name__} and {num_steps} and {split}" + + steps = 10 + for split in [300, 500, 700]: + for scheduler_cls_timesteps in [ + (DDIMScheduler, [901, 801, 701, 601, 501, 401, 301, 201, 101, 1]), + (EulerDiscreteScheduler, [901, 801, 701, 601, 501, 401, 301, 201, 101, 1]), + (DPMSolverMultistepScheduler, [901, 811, 721, 631, 541, 451, 361, 271, 181, 91]), + (UniPCMultistepScheduler, [901, 811, 721, 631, 541, 451, 361, 271, 181, 91]), + ( + HeunDiscreteScheduler, + [ + 901.0, + 801.0, + 801.0, + 701.0, + 701.0, + 601.0, + 601.0, + 501.0, + 501.0, + 401.0, + 401.0, + 301.0, + 301.0, + 201.0, + 201.0, + 101.0, + 101.0, + 1.0, + 1.0, + ], + ), + ]: + assert_run_mixture(steps, split, scheduler_cls_timesteps[0], scheduler_cls_timesteps[1]) + + steps = 25 + for split in [300, 500, 700]: + for scheduler_cls_timesteps in [ + ( + DDIMScheduler, + [ + 961, + 921, + 881, + 841, + 801, + 761, + 721, + 681, + 641, + 601, + 561, + 521, + 481, + 441, + 401, + 361, + 321, + 281, + 241, + 201, + 161, + 121, + 81, + 41, + 1, + ], + ), + ( + EulerDiscreteScheduler, + [ + 961.0, + 921.0, + 881.0, + 841.0, + 801.0, + 761.0, + 721.0, + 681.0, + 641.0, + 601.0, + 561.0, + 521.0, + 481.0, + 441.0, + 401.0, + 361.0, + 321.0, + 281.0, + 241.0, + 201.0, + 161.0, + 121.0, + 81.0, + 41.0, + 1.0, + ], + ), + ( + DPMSolverMultistepScheduler, + [ + 951, + 913, + 875, + 837, + 799, + 761, + 723, + 685, + 647, + 609, + 571, + 533, + 495, + 457, + 419, + 381, + 343, + 305, + 267, + 229, + 191, + 153, + 115, + 77, + 39, + ], + ), + ( + UniPCMultistepScheduler, + [ + 951, + 913, + 875, + 837, + 799, + 761, + 723, + 685, + 647, + 609, + 571, + 533, + 495, + 457, + 419, + 381, + 343, + 305, + 267, + 229, + 191, + 153, + 115, + 77, + 39, + ], + ), + ( + HeunDiscreteScheduler, + [ + 961.0, + 921.0, + 921.0, + 881.0, + 881.0, + 841.0, + 841.0, + 801.0, + 801.0, + 761.0, + 761.0, + 721.0, + 721.0, + 681.0, + 681.0, + 641.0, + 641.0, + 601.0, + 601.0, + 561.0, + 561.0, + 521.0, + 521.0, + 481.0, + 481.0, + 441.0, + 441.0, + 401.0, + 401.0, + 361.0, + 361.0, + 321.0, + 321.0, + 281.0, + 281.0, + 241.0, + 241.0, + 201.0, + 201.0, + 161.0, + 161.0, + 121.0, + 121.0, + 81.0, + 81.0, + 41.0, + 41.0, + 1.0, + 1.0, + ], + ), + ]: + assert_run_mixture(steps, split, scheduler_cls_timesteps[0], scheduler_cls_timesteps[1]) + + @slow + def test_stable_diffusion_three_xl_mixture_of_denoiser(self): + components = self.get_dummy_components() + pipe_1 = StableDiffusionXLPipeline(**components).to(torch_device) + pipe_1.unet.set_default_attn_processor() + pipe_2 = StableDiffusionXLImg2ImgPipeline(**components).to(torch_device) + pipe_2.unet.set_default_attn_processor() + pipe_3 = StableDiffusionXLImg2ImgPipeline(**components).to(torch_device) + pipe_3.unet.set_default_attn_processor() + + def assert_run_mixture( + num_steps, + split_1, + split_2, + scheduler_cls_orig, + num_train_timesteps=pipe_1.scheduler.config.num_train_timesteps, + ): + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = num_steps + + class scheduler_cls(scheduler_cls_orig): + pass + + pipe_1.scheduler = scheduler_cls.from_config(pipe_1.scheduler.config) + pipe_2.scheduler = scheduler_cls.from_config(pipe_2.scheduler.config) + pipe_3.scheduler = scheduler_cls.from_config(pipe_3.scheduler.config) + + # Let's retrieve the number of timesteps we want to use + pipe_1.scheduler.set_timesteps(num_steps) + expected_steps = pipe_1.scheduler.timesteps.tolist() + + split_1_ts = num_train_timesteps - int(round(num_train_timesteps * split_1)) + split_2_ts = num_train_timesteps - int(round(num_train_timesteps * split_2)) + + if pipe_1.scheduler.order == 2: + expected_steps_1 = list(filter(lambda ts: ts >= split_1_ts, expected_steps)) + expected_steps_2 = expected_steps_1[-1:] + list( + filter(lambda ts: ts >= split_2_ts and ts < split_1_ts, expected_steps) + ) + expected_steps_3 = expected_steps_2[-1:] + list(filter(lambda ts: ts < split_2_ts, expected_steps)) + expected_steps = expected_steps_1 + expected_steps_2 + expected_steps_3 + else: + expected_steps_1 = list(filter(lambda ts: ts >= split_1_ts, expected_steps)) + expected_steps_2 = list(filter(lambda ts: ts >= split_2_ts and ts < split_1_ts, expected_steps)) + expected_steps_3 = list(filter(lambda ts: ts < split_2_ts, expected_steps)) + + # now we monkey patch step `done_steps` + # list into the step function for testing + done_steps = [] + old_step = copy.copy(scheduler_cls.step) + + def new_step(self, *args, **kwargs): + done_steps.append(args[1].cpu().item()) # args[1] is always the passed `t` + return old_step(self, *args, **kwargs) + + scheduler_cls.step = new_step + + inputs_1 = {**inputs, **{"denoising_end": split_1, "output_type": "latent"}} + latents = pipe_1(**inputs_1).images[0] + + assert ( + expected_steps_1 == done_steps + ), f"Failure with {scheduler_cls.__name__} and {num_steps} and {split_1} and {split_2}" + + with self.assertRaises(ValueError) as cm: + inputs_2 = { + **inputs, + **{ + "denoising_start": split_2, + "denoising_end": split_1, + "image": latents, + "output_type": "latent", + }, + } + pipe_2(**inputs_2).images[0] + assert "cannot be larger than or equal to `denoising_end`" in str(cm.exception) + + inputs_2 = { + **inputs, + **{"denoising_start": split_1, "denoising_end": split_2, "image": latents, "output_type": "latent"}, + } + pipe_2(**inputs_2).images[0] + + assert expected_steps_2 == done_steps[len(expected_steps_1) :] + + inputs_3 = {**inputs, **{"denoising_start": split_2, "image": latents}} + pipe_3(**inputs_3).images[0] + + assert expected_steps_3 == done_steps[len(expected_steps_1) + len(expected_steps_2) :] + assert ( + expected_steps == done_steps + ), f"Failure with {scheduler_cls.__name__} and {num_steps} and {split_1} and {split_2}" + + for steps in [7, 11, 20]: + for split_1, split_2 in zip([0.19, 0.32], [0.81, 0.68]): + for scheduler_cls in [ + DDIMScheduler, + EulerDiscreteScheduler, + DPMSolverMultistepScheduler, + UniPCMultistepScheduler, + HeunDiscreteScheduler, + ]: + assert_run_mixture(steps, split_1, split_2, scheduler_cls) + + def test_stable_diffusion_xl_multi_prompts(self): + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + + # forward with single prompt + inputs = self.get_dummy_inputs(torch_device) + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same prompt duplicated + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt_2"] = inputs["prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["prompt_2"] = "different prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + # manually set a negative_prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same negative_prompt duplicated + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = inputs["negative_prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different negative_prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = "different negative prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + def test_stable_diffusion_xl_negative_conditions(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice_with_no_neg_cond = image[0, -3:, -3:, -1] + + image = sd_pipe( + **inputs, + negative_original_size=(512, 512), + negative_crops_coords_top_left=(0, 0), + negative_target_size=(1024, 1024), + ).images + image_slice_with_neg_cond = image[0, -3:, -3:, -1] + + self.assertTrue(np.abs(image_slice_with_no_neg_cond - image_slice_with_neg_cond).max() > 1e-2) + + def test_stable_diffusion_xl_save_from_pretrained(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLPipeline(**components).to(torch_device) + pipes.append(sd_pipe) + + with tempfile.TemporaryDirectory() as tmpdirname: + sd_pipe.save_pretrained(tmpdirname) + sd_pipe = StableDiffusionXLPipeline.from_pretrained(tmpdirname).to(torch_device) + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + pipe.unet.set_default_attn_processor() + + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + + def test_pipeline_interrupt(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "hey" + num_inference_steps = 3 + + # store intermediate latents from the generation process + class PipelineState: + def __init__(self): + self.state = [] + + def apply(self, pipe, i, t, callback_kwargs): + self.state.append(callback_kwargs["latents"]) + return callback_kwargs + + pipe_state = PipelineState() + sd_pipe( + prompt, + num_inference_steps=num_inference_steps, + output_type="np", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=pipe_state.apply, + ).images + + # interrupt generation at step index + interrupt_step_idx = 1 + + def callback_on_step_end(pipe, i, t, callback_kwargs): + if i == interrupt_step_idx: + pipe._interrupt = True + + return callback_kwargs + + output_interrupted = sd_pipe( + prompt, + num_inference_steps=num_inference_steps, + output_type="latent", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=callback_on_step_end, + ).images + + # fetch intermediate latents at the interrupted step + # from the completed generation process + intermediate_latent = pipe_state.state[interrupt_step_idx] + + # compare the intermediate latent to the output of the interrupted process + # they should be the same + assert torch.allclose(intermediate_latent, output_interrupted, atol=1e-4) + + +@slow +class StableDiffusionXLPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_diffusion_lcm(self): + torch.manual_seed(0) + unet = UNet2DConditionModel.from_pretrained( + "latent-consistency/lcm-ssd-1b", torch_dtype=torch.float16, variant="fp16" + ) + sd_pipe = StableDiffusionXLPipeline.from_pretrained( + "segmind/SSD-1B", unet=unet, torch_dtype=torch.float16, variant="fp16" + ).to(torch_device) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe.set_progress_bar_config(disable=None) + + prompt = "a red car standing on the side of the street" + + image = sd_pipe(prompt, num_inference_steps=4, guidance_scale=8.0).images[0] + + expected_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/lcm_full/stable_diffusion_ssd_1b_lcm.png" + ) + + image = sd_pipe.image_processor.pil_to_numpy(image) + expected_image = sd_pipe.image_processor.pil_to_numpy(expected_image) + + max_diff = numpy_cosine_similarity_distance(image.flatten(), expected_image.flatten()) + + assert max_diff < 1e-2 + + def test_download_ckpt_diff_format_is_same(self): + ckpt_path = ( + "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/sd_xl_base_1.0.safetensors" + ) + + pipe = StableDiffusionXLPipeline.from_single_file(ckpt_path, torch_dtype=torch.float16) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.unet.set_default_attn_processor() + pipe.enable_model_cpu_offload() + + generator = torch.Generator(device="cpu").manual_seed(0) + image_ckpt = pipe("a turtle", num_inference_steps=2, generator=generator, output_type="np").images[0] + + pipe = StableDiffusionXLPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.unet.set_default_attn_processor() + pipe.enable_model_cpu_offload() + + generator = torch.Generator(device="cpu").manual_seed(0) + image = pipe("a turtle", num_inference_steps=2, generator=generator, output_type="np").images[0] + + max_diff = numpy_cosine_similarity_distance(image.flatten(), image_ckpt.flatten()) + + assert max_diff < 6e-3 + + def test_single_file_component_configs(self): + pipe = StableDiffusionXLPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=torch.float16 + ) + ckpt_path = ( + "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/sd_xl_base_1.0.safetensors" + ) + single_file_pipe = StableDiffusionXLPipeline.from_single_file( + ckpt_path, variant="fp16", torch_dtype=torch.float16 + ) + + for param_name, param_value in single_file_pipe.text_encoder.config.to_dict().items(): + if param_name in ["torch_dtype", "architectures", "_name_or_path"]: + continue + assert pipe.text_encoder.config.to_dict()[param_name] == param_value + + for param_name, param_value in single_file_pipe.text_encoder_2.config.to_dict().items(): + if param_name in ["torch_dtype", "architectures", "_name_or_path"]: + continue + assert pipe.text_encoder_2.config.to_dict()[param_name] == param_value + + PARAMS_TO_IGNORE = ["torch_dtype", "_name_or_path", "architectures", "_use_default_values"] + for param_name, param_value in single_file_pipe.unet.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + if param_name == "upcast_attention" and pipe.unet.config[param_name] is None: + pipe.unet.config[param_name] = False + assert ( + pipe.unet.config[param_name] == param_value + ), f"{param_name} is differs between single file loading and pretrained loading" + + for param_name, param_value in single_file_pipe.vae.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.vae.config[param_name] == param_value + ), f"{param_name} is differs between single file loading and pretrained loading" diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py new file mode 100755 index 0000000..0bcffeb --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_adapter.py @@ -0,0 +1,711 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from parameterized import parameterized +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer + +import diffusers +from diffusers import ( + AutoencoderKL, + EulerDiscreteScheduler, + LCMScheduler, + MultiAdapter, + StableDiffusionXLAdapterPipeline, + T2IAdapter, + UNet2DConditionModel, +) +from diffusers.utils import load_image, logging +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) + +from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS +from ..test_pipelines_common import ( + IPAdapterTesterMixin, + PipelineTesterMixin, + SDXLOptionalComponentsTesterMixin, + assert_mean_pixel_difference, +) + + +enable_full_determinism() + + +class StableDiffusionXLAdapterPipelineFastTests( + IPAdapterTesterMixin, PipelineTesterMixin, SDXLOptionalComponentsTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionXLAdapterPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + + def get_dummy_components(self, adapter_type="full_adapter_xl", time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + time_cond_proj_dim=time_cond_proj_dim, + ) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + if adapter_type == "full_adapter_xl": + adapter = T2IAdapter( + in_channels=3, + channels=[32, 64], + num_res_blocks=2, + downscale_factor=4, + adapter_type=adapter_type, + ) + elif adapter_type == "multi_adapter": + adapter = MultiAdapter( + [ + T2IAdapter( + in_channels=3, + channels=[32, 64], + num_res_blocks=2, + downscale_factor=4, + adapter_type="full_adapter_xl", + ), + T2IAdapter( + in_channels=3, + channels=[32, 64], + num_res_blocks=2, + downscale_factor=4, + adapter_type="full_adapter_xl", + ), + ] + ) + else: + raise ValueError( + f"Unknown adapter type: {adapter_type}, must be one of 'full_adapter_xl', or 'multi_adapter''" + ) + + components = { + "adapter": adapter, + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + # "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_components_with_full_downscaling(self, adapter_type="full_adapter_xl"): + """Get dummy components with x8 VAE downscaling and 3 UNet down blocks. + These dummy components are intended to fully-exercise the T2I-Adapter + downscaling behavior. + """ + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=2, + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=1, + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 32, 32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + if adapter_type == "full_adapter_xl": + adapter = T2IAdapter( + in_channels=3, + channels=[32, 32, 64], + num_res_blocks=2, + downscale_factor=16, + adapter_type=adapter_type, + ) + elif adapter_type == "multi_adapter": + adapter = MultiAdapter( + [ + T2IAdapter( + in_channels=3, + channels=[32, 32, 64], + num_res_blocks=2, + downscale_factor=16, + adapter_type="full_adapter_xl", + ), + T2IAdapter( + in_channels=3, + channels=[32, 32, 64], + num_res_blocks=2, + downscale_factor=16, + adapter_type="full_adapter_xl", + ), + ] + ) + else: + raise ValueError( + f"Unknown adapter type: {adapter_type}, must be one of 'full_adapter_xl', or 'multi_adapter''" + ) + + components = { + "adapter": adapter, + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + # "safety_checker": None, + "feature_extractor": None, + "image_encoder": None, + } + return components + + def get_dummy_inputs(self, device, seed=0, height=64, width=64, num_images=1): + if num_images == 1: + image = floats_tensor((1, 3, height, width), rng=random.Random(seed)).to(device) + else: + image = [ + floats_tensor((1, 3, height, width), rng=random.Random(seed)).to(device) for _ in range(num_images) + ] + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 5.0, + "output_type": "numpy", + } + return inputs + + def test_stable_diffusion_adapter_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLAdapterPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array( + [0.5752919, 0.6022097, 0.4728038, 0.49861962, 0.57084894, 0.4644975, 0.5193715, 0.5133664, 0.4729858] + ) + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-3 + + @parameterized.expand( + [ + # (dim=144) The internal feature map will be 9x9 after initial pixel unshuffling (downscaled x16). + (((4 * 2 + 1) * 16),), + # (dim=160) The internal feature map will be 5x5 after the first T2I down block (downscaled x32). + (((4 * 1 + 1) * 32),), + ] + ) + def test_multiple_image_dimensions(self, dim): + """Test that the T2I-Adapter pipeline supports any input dimension that + is divisible by the adapter's `downscale_factor`. This test was added in + response to an issue where the T2I Adapter's downscaling padding + behavior did not match the UNet's behavior. + + Note that we have selected `dim` values to produce odd resolutions at + each downscaling level. + """ + components = self.get_dummy_components_with_full_downscaling() + sd_pipe = StableDiffusionXLAdapterPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device, height=dim, width=dim) + image = sd_pipe(**inputs).images + + assert image.shape == (1, dim, dim, 3) + + @parameterized.expand(["full_adapter", "full_adapter_xl", "light_adapter"]) + def test_total_downscale_factor(self, adapter_type): + """Test that the T2IAdapter correctly reports its total_downscale_factor.""" + batch_size = 1 + in_channels = 3 + out_channels = [320, 640, 1280, 1280] + in_image_size = 512 + + adapter = T2IAdapter( + in_channels=in_channels, + channels=out_channels, + num_res_blocks=2, + downscale_factor=8, + adapter_type=adapter_type, + ) + adapter.to(torch_device) + + in_image = floats_tensor((batch_size, in_channels, in_image_size, in_image_size)).to(torch_device) + + adapter_state = adapter(in_image) + + # Assume that the last element in `adapter_state` has been downsampled the most, and check + # that it matches the `total_downscale_factor`. + expected_out_image_size = in_image_size // adapter.total_downscale_factor + assert adapter_state[-1].shape == ( + batch_size, + out_channels[-1], + expected_out_image_size, + expected_out_image_size, + ) + + def test_save_load_optional_components(self): + return self._test_save_load_optional_components() + + def test_adapter_sdxl_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLAdapterPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5425, 0.5385, 0.4964, 0.5045, 0.6149, 0.4974, 0.5469, 0.5332, 0.5426]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_adapter_sdxl_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLAdapterPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5425, 0.5385, 0.4964, 0.5045, 0.6149, 0.4974, 0.5469, 0.5332, 0.5426]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + +class StableDiffusionXLMultiAdapterPipelineFastTests( + StableDiffusionXLAdapterPipelineFastTests, PipelineTesterMixin, unittest.TestCase +): + def get_dummy_components(self, time_cond_proj_dim=None): + return super().get_dummy_components("multi_adapter", time_cond_proj_dim=time_cond_proj_dim) + + def get_dummy_components_with_full_downscaling(self): + return super().get_dummy_components_with_full_downscaling("multi_adapter") + + def get_dummy_inputs(self, device, seed=0, height=64, width=64): + inputs = super().get_dummy_inputs(device, seed, height, width, num_images=2) + inputs["adapter_conditioning_scale"] = [0.5, 0.5] + return inputs + + def test_stable_diffusion_adapter_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLAdapterPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array( + [0.5813032, 0.60995954, 0.47563356, 0.5056669, 0.57199144, 0.4631841, 0.5176794, 0.51252556, 0.47183886] + ) + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-3 + + def test_inference_batch_consistent( + self, batch_sizes=[2, 4, 13], additional_params_copy_to_batched_inputs=["num_inference_steps"] + ): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + logger = logging.get_logger(pipe.__module__) + logger.setLevel(level=diffusers.logging.FATAL) + + # batchify inputs + for batch_size in batch_sizes: + batched_inputs = {} + for name, value in inputs.items(): + if name in self.batch_params: + # prompt is string + if name == "prompt": + len_prompt = len(value) + # make unequal batch sizes + batched_inputs[name] = [value[: len_prompt // i] for i in range(1, batch_size + 1)] + + # make last batch super long + batched_inputs[name][-1] = 100 * "very long" + elif name == "image": + batched_images = [] + + for image in value: + batched_images.append(batch_size * [image]) + + batched_inputs[name] = batched_images + else: + batched_inputs[name] = batch_size * [value] + + elif name == "batch_size": + batched_inputs[name] = batch_size + else: + batched_inputs[name] = value + + for arg in additional_params_copy_to_batched_inputs: + batched_inputs[arg] = inputs[arg] + + batched_inputs["output_type"] = "np" + + output = pipe(**batched_inputs) + + assert len(output[0]) == batch_size + + batched_inputs["output_type"] = "np" + + output = pipe(**batched_inputs)[0] + + assert output.shape[0] == batch_size + + logger.setLevel(level=diffusers.logging.WARNING) + + def test_num_images_per_prompt(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + batch_sizes = [1, 2] + num_images_per_prompts = [1, 2] + + for batch_size in batch_sizes: + for num_images_per_prompt in num_images_per_prompts: + inputs = self.get_dummy_inputs(torch_device) + + for key in inputs.keys(): + if key in self.batch_params: + if key == "image": + batched_images = [] + + for image in inputs[key]: + batched_images.append(batch_size * [image]) + + inputs[key] = batched_images + else: + inputs[key] = batch_size * [inputs[key]] + + images = pipe(**inputs, num_images_per_prompt=num_images_per_prompt)[0] + + assert images.shape[0] == batch_size * num_images_per_prompt + + def test_inference_batch_single_identical( + self, + batch_size=3, + test_max_difference=None, + test_mean_pixel_difference=None, + relax_max_difference=False, + expected_max_diff=2e-3, + additional_params_copy_to_batched_inputs=["num_inference_steps"], + ): + if test_max_difference is None: + # TODO(Pedro) - not sure why, but not at all reproducible at the moment it seems + # make sure that batched and non-batched is identical + test_max_difference = torch_device != "mps" + + if test_mean_pixel_difference is None: + # TODO same as above + test_mean_pixel_difference = torch_device != "mps" + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + logger = logging.get_logger(pipe.__module__) + logger.setLevel(level=diffusers.logging.FATAL) + + # batchify inputs + batched_inputs = {} + batch_size = batch_size + for name, value in inputs.items(): + if name in self.batch_params: + # prompt is string + if name == "prompt": + len_prompt = len(value) + # make unequal batch sizes + batched_inputs[name] = [value[: len_prompt // i] for i in range(1, batch_size + 1)] + + # make last batch super long + batched_inputs[name][-1] = 100 * "very long" + elif name == "image": + batched_images = [] + + for image in value: + batched_images.append(batch_size * [image]) + + batched_inputs[name] = batched_images + else: + batched_inputs[name] = batch_size * [value] + elif name == "batch_size": + batched_inputs[name] = batch_size + elif name == "generator": + batched_inputs[name] = [self.get_generator(i) for i in range(batch_size)] + else: + batched_inputs[name] = value + + for arg in additional_params_copy_to_batched_inputs: + batched_inputs[arg] = inputs[arg] + + output_batch = pipe(**batched_inputs) + assert output_batch[0].shape[0] == batch_size + + inputs["generator"] = self.get_generator(0) + + output = pipe(**inputs) + + logger.setLevel(level=diffusers.logging.WARNING) + if test_max_difference: + if relax_max_difference: + # Taking the median of the largest differences + # is resilient to outliers + diff = np.abs(output_batch[0][0] - output[0][0]) + diff = diff.flatten() + diff.sort() + max_diff = np.median(diff[-5:]) + else: + max_diff = np.abs(output_batch[0][0] - output[0][0]).max() + assert max_diff < expected_max_diff + + if test_mean_pixel_difference: + assert_mean_pixel_difference(output_batch[0][0], output[0][0]) + + def test_adapter_sdxl_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLAdapterPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5313, 0.5375, 0.4942, 0.5021, 0.6142, 0.4968, 0.5434, 0.5311, 0.5448]) + + debug = [str(round(i, 4)) for i in image_slice.flatten().tolist()] + print(",".join(debug)) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_adapter_sdxl_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLAdapterPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.scheduler.config) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + output = sd_pipe(**inputs) + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + expected_slice = np.array([0.5313, 0.5375, 0.4942, 0.5021, 0.6142, 0.4968, 0.5434, 0.5311, 0.5448]) + + debug = [str(round(i, 4)) for i in image_slice.flatten().tolist()] + print(",".join(debug)) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + +@slow +@require_torch_gpu +class AdapterSDXLPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_download_ckpt_diff_format_is_same(self): + ckpt_path = ( + "https://huggingface.co/stabilityai/stable-diffusion-xl-base-1.0/blob/main/sd_xl_base_1.0.safetensors" + ) + adapter = T2IAdapter.from_pretrained("TencentARC/t2i-adapter-lineart-sdxl-1.0", torch_dtype=torch.float16) + prompt = "toy" + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/t2i_adapter/toy_canny.png" + ) + pipe_single_file = StableDiffusionXLAdapterPipeline.from_single_file( + ckpt_path, + adapter=adapter, + torch_dtype=torch.float16, + ) + pipe_single_file.enable_model_cpu_offload() + pipe_single_file.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + images_single_file = pipe_single_file( + prompt, image=image, generator=generator, output_type="np", num_inference_steps=3 + ).images + + generator = torch.Generator(device="cpu").manual_seed(0) + pipe = StableDiffusionXLAdapterPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", + adapter=adapter, + torch_dtype=torch.float16, + ) + pipe.enable_model_cpu_offload() + images = pipe(prompt, image=image, generator=generator, output_type="np", num_inference_steps=3).images + + assert images_single_file[0].shape == (768, 512, 3) + assert images[0].shape == (768, 512, 3) + + max_diff = numpy_cosine_similarity_distance(images[0].flatten(), images_single_file[0].flatten()) + assert max_diff < 5e-3 diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_img2img.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_img2img.py new file mode 100755 index 0000000..2c4c01c --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_img2img.py @@ -0,0 +1,852 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextConfig, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +from diffusers import ( + AutoencoderKL, + AutoencoderTiny, + DDIMScheduler, + EulerDiscreteScheduler, + LCMScheduler, + StableDiffusionXLImg2ImgPipeline, + UNet2DConditionModel, +) +from diffusers.utils import load_image +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, +) +from ..test_pipelines_common import ( + IPAdapterTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, + SDXLOptionalComponentsTesterMixin, +) + + +enable_full_determinism() + + +class StableDiffusionXLImg2ImgPipelineFastTests( + IPAdapterTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionXLImg2ImgPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width"} + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + image_latents_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS.union( + {"add_text_embeds", "add_time_ids", "add_neg_time_ids"} + ) + + def get_dummy_components(self, skip_first_text_encoder=False, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + time_cond_proj_dim=time_cond_proj_dim, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=72, # 5 * 8 + 32 + cross_attention_dim=64 if not skip_first_text_encoder else 32, + ) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + image_encoder_config = CLIPVisionConfig( + hidden_size=32, + image_size=224, + projection_dim=32, + intermediate_size=37, + num_attention_heads=4, + num_channels=3, + num_hidden_layers=5, + patch_size=14, + ) + + image_encoder = CLIPVisionModelWithProjection(image_encoder_config) + + feature_extractor = CLIPImageProcessor( + crop_size=224, + do_center_crop=True, + do_normalize=True, + do_resize=True, + image_mean=[0.48145466, 0.4578275, 0.40821073], + image_std=[0.26862954, 0.26130258, 0.27577711], + resample=3, + size=224, + ) + + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder if not skip_first_text_encoder else None, + "tokenizer": tokenizer if not skip_first_text_encoder else None, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "requires_aesthetics_score": True, + "image_encoder": image_encoder, + "feature_extractor": feature_extractor, + } + return components + + def get_dummy_tiny_autoencoder(self): + return AutoencoderTiny(in_channels=3, out_channels=3, latent_channels=4) + + def test_components_function(self): + init_components = self.get_dummy_components() + init_components.pop("requires_aesthetics_score") + pipe = self.pipeline_class(**init_components) + + self.assertTrue(hasattr(pipe, "components")) + self.assertTrue(set(pipe.components.keys()) == set(init_components.keys())) + + def get_dummy_inputs(self, device, seed=0): + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image / 2 + 0.5 + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 5.0, + "output_type": "np", + "strength": 0.8, + } + return inputs + + def test_stable_diffusion_xl_img2img_euler(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + + expected_slice = np.array([0.4664, 0.4886, 0.4403, 0.6902, 0.5592, 0.4534, 0.5931, 0.5951, 0.5224]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_xl_img2img_euler_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + + expected_slice = np.array([0.5604, 0.4352, 0.4717, 0.5844, 0.5101, 0.6704, 0.6290, 0.5460, 0.5286]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_xl_img2img_euler_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + + expected_slice = np.array([0.5604, 0.4352, 0.4717, 0.5844, 0.5101, 0.6704, 0.6290, 0.5460, 0.5286]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_attention_slicing_forward_pass(self): + super().test_attention_slicing_forward_pass(expected_max_diff=3e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + # TODO(Patrick, Sayak) - skip for now as this requires more refiner tests + def test_save_load_optional_components(self): + pass + + def test_stable_diffusion_xl_img2img_negative_prompt_embeds(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + # forward without prompt embeds + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + negative_prompt = 3 * ["this is a negative prompt"] + inputs["negative_prompt"] = negative_prompt + inputs["prompt"] = 3 * [inputs["prompt"]] + + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with prompt embeds + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + negative_prompt = 3 * ["this is a negative prompt"] + prompt = 3 * [inputs.pop("prompt")] + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = sd_pipe.encode_prompt(prompt, negative_prompt=negative_prompt) + + output = sd_pipe( + **inputs, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + ) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # make sure that it's equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + def test_stable_diffusion_xl_img2img_tiny_autoencoder(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe.vae = self.get_dummy_tiny_autoencoder() + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.0, 0.0, 0.0106, 0.0, 0.0, 0.0087, 0.0052, 0.0062, 0.0177]) + + assert np.allclose(image_slice, expected_slice, atol=1e-4, rtol=1e-4) + + @require_torch_gpu + def test_stable_diffusion_xl_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + pipe.unet.set_default_attn_processor() + + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_stable_diffusion_xl_multi_prompts(self): + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + + # forward with single prompt + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + inputs["num_inference_steps"] = 5 + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same prompt duplicated + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + inputs["num_inference_steps"] = 5 + inputs["prompt_2"] = inputs["prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different prompt + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + inputs["num_inference_steps"] = 5 + inputs["prompt_2"] = "different prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + # manually set a negative_prompt + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + inputs["num_inference_steps"] = 5 + inputs["negative_prompt"] = "negative prompt" + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same negative_prompt duplicated + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + inputs["num_inference_steps"] = 5 + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = inputs["negative_prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different negative_prompt + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + inputs["num_inference_steps"] = 5 + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = "different negative prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + def test_stable_diffusion_xl_img2img_negative_conditions(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice_with_no_neg_conditions = image[0, -3:, -3:, -1] + + image = sd_pipe( + **inputs, + negative_original_size=(512, 512), + negative_crops_coords_top_left=( + 0, + 0, + ), + negative_target_size=(1024, 1024), + ).images + image_slice_with_neg_conditions = image[0, -3:, -3:, -1] + + assert ( + np.abs(image_slice_with_no_neg_conditions.flatten() - image_slice_with_neg_conditions.flatten()).max() + > 1e-4 + ) + + def test_pipeline_interrupt(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + prompt = "hey" + num_inference_steps = 5 + + # store intermediate latents from the generation process + class PipelineState: + def __init__(self): + self.state = [] + + def apply(self, pipe, i, t, callback_kwargs): + self.state.append(callback_kwargs["latents"]) + return callback_kwargs + + pipe_state = PipelineState() + sd_pipe( + prompt, + image=inputs["image"], + strength=0.8, + num_inference_steps=num_inference_steps, + output_type="np", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=pipe_state.apply, + ).images + + # interrupt generation at step index + interrupt_step_idx = 1 + + def callback_on_step_end(pipe, i, t, callback_kwargs): + if i == interrupt_step_idx: + pipe._interrupt = True + + return callback_kwargs + + output_interrupted = sd_pipe( + prompt, + image=inputs["image"], + strength=0.8, + num_inference_steps=num_inference_steps, + output_type="latent", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=callback_on_step_end, + ).images + + # fetch intermediate latents at the interrupted step + # from the completed generation process + intermediate_latent = pipe_state.state[interrupt_step_idx] + + # compare the intermediate latent to the output of the interrupted process + # they should be the same + assert torch.allclose(intermediate_latent, output_interrupted, atol=1e-4) + + +class StableDiffusionXLImg2ImgRefinerOnlyPipelineFastTests( + PipelineLatentTesterMixin, PipelineTesterMixin, SDXLOptionalComponentsTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionXLImg2ImgPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width"} + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + image_latents_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=72, # 5 * 8 + 32 + cross_attention_dim=32, + ) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "tokenizer": None, + "text_encoder": None, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "requires_aesthetics_score": True, + "image_encoder": None, + "feature_extractor": None, + } + return components + + def test_components_function(self): + init_components = self.get_dummy_components() + init_components.pop("requires_aesthetics_score") + pipe = self.pipeline_class(**init_components) + + self.assertTrue(hasattr(pipe, "components")) + self.assertTrue(set(pipe.components.keys()) == set(init_components.keys())) + + def get_dummy_inputs(self, device, seed=0): + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image / 2 + 0.5 + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 5.0, + "output_type": "np", + "strength": 0.8, + } + return inputs + + def test_stable_diffusion_xl_img2img_euler(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + + expected_slice = np.array([0.4745, 0.4924, 0.4338, 0.6468, 0.5547, 0.4419, 0.5646, 0.5897, 0.5146]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + @require_torch_gpu + def test_stable_diffusion_xl_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + pipe.unet.set_default_attn_processor() + + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_stable_diffusion_xl_img2img_negative_conditions(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice_with_no_neg_conditions = image[0, -3:, -3:, -1] + + image = sd_pipe( + **inputs, + negative_original_size=(512, 512), + negative_crops_coords_top_left=( + 0, + 0, + ), + negative_target_size=(1024, 1024), + ).images + image_slice_with_neg_conditions = image[0, -3:, -3:, -1] + + assert ( + np.abs(image_slice_with_no_neg_conditions.flatten() - image_slice_with_neg_conditions.flatten()).max() + > 1e-4 + ) + + def test_stable_diffusion_xl_img2img_negative_prompt_embeds(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + # forward without prompt embeds + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + negative_prompt = 3 * ["this is a negative prompt"] + inputs["negative_prompt"] = negative_prompt + inputs["prompt"] = 3 * [inputs["prompt"]] + + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with prompt embeds + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + negative_prompt = 3 * ["this is a negative prompt"] + prompt = 3 * [inputs.pop("prompt")] + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = sd_pipe.encode_prompt(prompt, negative_prompt=negative_prompt) + + output = sd_pipe( + **inputs, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + ) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # make sure that it's equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + def test_stable_diffusion_xl_img2img_prompt_embeds_only(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + # forward without prompt embeds + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + inputs["prompt"] = 3 * [inputs["prompt"]] + + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with prompt embeds + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + prompt = 3 * [inputs.pop("prompt")] + + ( + prompt_embeds, + _, + pooled_prompt_embeds, + _, + ) = sd_pipe.encode_prompt(prompt) + + output = sd_pipe( + **inputs, + prompt_embeds=prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + ) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # make sure that it's equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + def test_attention_slicing_forward_pass(self): + super().test_attention_slicing_forward_pass(expected_max_diff=3e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + def test_save_load_optional_components(self): + self._test_save_load_optional_components() + + +@slow +class StableDiffusionXLImg2ImgIntegrationTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_download_ckpt_diff_format_is_same(self): + ckpt_path = "https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0/blob/main/sd_xl_refiner_1.0.safetensors" + init_image = load_image( + "https://huggingface.co/datasets/diffusers/test-arrays/resolve/main" + "/stable_diffusion_img2img/sketch-mountains-input.png" + ) + + pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-refiner-1.0", torch_dtype=torch.float16 + ) + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe.unet.set_default_attn_processor() + pipe.enable_model_cpu_offload() + + generator = torch.Generator(device="cpu").manual_seed(0) + image = pipe( + prompt="mountains", image=init_image, num_inference_steps=5, generator=generator, output_type="np" + ).images[0] + + pipe_single_file = StableDiffusionXLImg2ImgPipeline.from_single_file(ckpt_path, torch_dtype=torch.float16) + pipe_single_file.scheduler = DDIMScheduler.from_config(pipe_single_file.scheduler.config) + pipe_single_file.unet.set_default_attn_processor() + pipe_single_file.enable_model_cpu_offload() + + generator = torch.Generator(device="cpu").manual_seed(0) + image_single_file = pipe_single_file( + prompt="mountains", image=init_image, num_inference_steps=5, generator=generator, output_type="np" + ).images[0] + + max_diff = numpy_cosine_similarity_distance(image.flatten(), image_single_file.flatten()) + + assert max_diff < 5e-2 + + def test_single_file_component_configs(self): + pipe = StableDiffusionXLImg2ImgPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-refiner-1.0", + torch_dtype=torch.float16, + variant="fp16", + ) + ckpt_path = "https://huggingface.co/stabilityai/stable-diffusion-xl-refiner-1.0/blob/main/sd_xl_refiner_1.0.safetensors" + single_file_pipe = StableDiffusionXLImg2ImgPipeline.from_single_file(ckpt_path, torch_dtype=torch.float16) + + assert pipe.text_encoder is None + assert single_file_pipe.text_encoder is None + + for param_name, param_value in single_file_pipe.text_encoder_2.config.to_dict().items(): + if param_name in ["torch_dtype", "architectures", "_name_or_path"]: + continue + assert pipe.text_encoder_2.config.to_dict()[param_name] == param_value + + PARAMS_TO_IGNORE = ["torch_dtype", "_name_or_path", "architectures", "_use_default_values"] + for param_name, param_value in single_file_pipe.unet.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + if param_name == "upcast_attention" and pipe.unet.config[param_name] is None: + pipe.unet.config[param_name] = False + assert ( + pipe.unet.config[param_name] == param_value + ), f"{param_name} is differs between single file loading and pretrained loading" + + for param_name, param_value in single_file_pipe.vae.config.items(): + if param_name in PARAMS_TO_IGNORE: + continue + assert ( + pipe.vae.config[param_name] == param_value + ), f"{param_name} differs between single file loading and pretrained loading" diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py new file mode 100755 index 0000000..11c711e --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_inpaint.py @@ -0,0 +1,810 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import copy +import random +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import ( + CLIPImageProcessor, + CLIPTextConfig, + CLIPTextModel, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + DPMSolverMultistepScheduler, + EulerDiscreteScheduler, + HeunDiscreteScheduler, + LCMScheduler, + StableDiffusionXLInpaintPipeline, + UNet2DConditionModel, + UniPCMultistepScheduler, +) +from diffusers.utils.testing_utils import enable_full_determinism, floats_tensor, require_torch_gpu, slow, torch_device + +from ..pipeline_params import ( + TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_INPAINTING_PARAMS, + TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS, +) +from ..test_pipelines_common import IPAdapterTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +class StableDiffusionXLInpaintPipelineFastTests( + IPAdapterTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableDiffusionXLInpaintPipeline + params = TEXT_GUIDED_IMAGE_INPAINTING_PARAMS + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + image_params = frozenset([]) + # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess + image_latents_params = frozenset([]) + callback_cfg_params = TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS.union( + { + "add_text_embeds", + "add_time_ids", + "mask", + "masked_image_latents", + } + ) + + def get_dummy_components(self, skip_first_text_encoder=False, time_cond_proj_dim=None): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + time_cond_proj_dim=time_cond_proj_dim, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=72, # 5 * 8 + 32 + cross_attention_dim=64 if not skip_first_text_encoder else 32, + ) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + torch.manual_seed(0) + image_encoder_config = CLIPVisionConfig( + hidden_size=32, + image_size=224, + projection_dim=32, + intermediate_size=37, + num_attention_heads=4, + num_channels=3, + num_hidden_layers=5, + patch_size=14, + ) + + image_encoder = CLIPVisionModelWithProjection(image_encoder_config) + + feature_extractor = CLIPImageProcessor( + crop_size=224, + do_center_crop=True, + do_normalize=True, + do_resize=True, + image_mean=[0.48145466, 0.4578275, 0.40821073], + image_std=[0.26862954, 0.26130258, 0.27577711], + resample=3, + size=224, + ) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder if not skip_first_text_encoder else None, + "tokenizer": tokenizer if not skip_first_text_encoder else None, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "image_encoder": image_encoder, + "feature_extractor": feature_extractor, + "requires_aesthetics_score": True, + } + return components + + def get_dummy_inputs(self, device, seed=0): + # TODO: use tensor inputs instead of PIL, this is here just to leave the old expected_slices untouched + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + init_image = Image.fromarray(np.uint8(image)).convert("RGB").resize((64, 64)) + # create mask + image[8:, 8:, :] = 255 + mask_image = Image.fromarray(np.uint8(image)).convert("L").resize((64, 64)) + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": init_image, + "mask_image": mask_image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "strength": 1.0, + "output_type": "np", + } + return inputs + + def get_dummy_inputs_2images(self, device, seed=0, img_res=64): + # Get random floats in [0, 1] as image with spatial size (img_res, img_res) + image1 = floats_tensor((1, 3, img_res, img_res), rng=random.Random(seed)).to(device) + image2 = floats_tensor((1, 3, img_res, img_res), rng=random.Random(seed + 22)).to(device) + # Convert images to [-1, 1] + init_image1 = 2.0 * image1 - 1.0 + init_image2 = 2.0 * image2 - 1.0 + + # empty mask + mask_image = torch.zeros((1, 1, img_res, img_res), device=device) + + if str(device).startswith("mps"): + generator1 = torch.manual_seed(seed) + generator2 = torch.manual_seed(seed) + else: + generator1 = torch.Generator(device=device).manual_seed(seed) + generator2 = torch.Generator(device=device).manual_seed(seed) + + inputs = { + "prompt": ["A painting of a squirrel eating a burger"] * 2, + "image": [init_image1, init_image2], + "mask_image": [mask_image] * 2, + "generator": [generator1, generator2], + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "np", + } + return inputs + + def test_components_function(self): + init_components = self.get_dummy_components() + init_components.pop("requires_aesthetics_score") + pipe = self.pipeline_class(**init_components) + + self.assertTrue(hasattr(pipe, "components")) + self.assertTrue(set(pipe.components.keys()) == set(init_components.keys())) + + def test_stable_diffusion_xl_inpaint_euler(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLInpaintPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.8029, 0.5523, 0.5825, 0.6003, 0.6702, 0.7018, 0.6369, 0.5955, 0.5123]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_xl_inpaint_euler_lcm(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLInpaintPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.6611, 0.5569, 0.5531, 0.5471, 0.5918, 0.6393, 0.5074, 0.5468, 0.5185]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_xl_inpaint_euler_lcm_custom_timesteps(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(time_cond_proj_dim=256) + sd_pipe = StableDiffusionXLInpaintPipeline(**components) + sd_pipe.scheduler = LCMScheduler.from_config(sd_pipe.config) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + del inputs["num_inference_steps"] + inputs["timesteps"] = [999, 499] + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.6611, 0.5569, 0.5531, 0.5471, 0.5918, 0.6393, 0.5074, 0.5468, 0.5185]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_attention_slicing_forward_pass(self): + super().test_attention_slicing_forward_pass(expected_max_diff=3e-3) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + # TODO(Patrick, Sayak) - skip for now as this requires more refiner tests + def test_save_load_optional_components(self): + pass + + def test_stable_diffusion_xl_inpaint_negative_prompt_embeds(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLInpaintPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + # forward without prompt embeds + inputs = self.get_dummy_inputs(torch_device) + negative_prompt = 3 * ["this is a negative prompt"] + inputs["negative_prompt"] = negative_prompt + inputs["prompt"] = 3 * [inputs["prompt"]] + + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with prompt embeds + inputs = self.get_dummy_inputs(torch_device) + negative_prompt = 3 * ["this is a negative prompt"] + prompt = 3 * [inputs.pop("prompt")] + + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = sd_pipe.encode_prompt(prompt, negative_prompt=negative_prompt) + + output = sd_pipe( + **inputs, + prompt_embeds=prompt_embeds, + negative_prompt_embeds=negative_prompt_embeds, + pooled_prompt_embeds=pooled_prompt_embeds, + negative_pooled_prompt_embeds=negative_pooled_prompt_embeds, + ) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # make sure that it's equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + @require_torch_gpu + def test_stable_diffusion_xl_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLInpaintPipeline(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLInpaintPipeline(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLInpaintPipeline(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + pipe.unet.set_default_attn_processor() + + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_stable_diffusion_xl_refiner(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components(skip_first_text_encoder=True) + + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.7045, 0.4838, 0.5454, 0.6270, 0.6168, 0.6717, 0.6484, 0.5681, 0.4922]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_two_xl_mixture_of_denoiser_fast(self): + components = self.get_dummy_components() + pipe_1 = StableDiffusionXLInpaintPipeline(**components).to(torch_device) + pipe_1.unet.set_default_attn_processor() + pipe_2 = StableDiffusionXLInpaintPipeline(**components).to(torch_device) + pipe_2.unet.set_default_attn_processor() + + def assert_run_mixture( + num_steps, split, scheduler_cls_orig, num_train_timesteps=pipe_1.scheduler.config.num_train_timesteps + ): + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = num_steps + + class scheduler_cls(scheduler_cls_orig): + pass + + pipe_1.scheduler = scheduler_cls.from_config(pipe_1.scheduler.config) + pipe_2.scheduler = scheduler_cls.from_config(pipe_2.scheduler.config) + + # Let's retrieve the number of timesteps we want to use + pipe_1.scheduler.set_timesteps(num_steps) + expected_steps = pipe_1.scheduler.timesteps.tolist() + + split_ts = num_train_timesteps - int(round(num_train_timesteps * split)) + + if pipe_1.scheduler.order == 2: + expected_steps_1 = list(filter(lambda ts: ts >= split_ts, expected_steps)) + expected_steps_2 = expected_steps_1[-1:] + list(filter(lambda ts: ts < split_ts, expected_steps)) + expected_steps = expected_steps_1 + expected_steps_2 + else: + expected_steps_1 = list(filter(lambda ts: ts >= split_ts, expected_steps)) + expected_steps_2 = list(filter(lambda ts: ts < split_ts, expected_steps)) + + # now we monkey patch step `done_steps` + # list into the step function for testing + done_steps = [] + old_step = copy.copy(scheduler_cls.step) + + def new_step(self, *args, **kwargs): + done_steps.append(args[1].cpu().item()) # args[1] is always the passed `t` + return old_step(self, *args, **kwargs) + + scheduler_cls.step = new_step + + inputs_1 = {**inputs, **{"denoising_end": split, "output_type": "latent"}} + latents = pipe_1(**inputs_1).images[0] + + assert expected_steps_1 == done_steps, f"Failure with {scheduler_cls.__name__} and {num_steps} and {split}" + + inputs_2 = {**inputs, **{"denoising_start": split, "image": latents}} + pipe_2(**inputs_2).images[0] + + assert expected_steps_2 == done_steps[len(expected_steps_1) :] + assert expected_steps == done_steps, f"Failure with {scheduler_cls.__name__} and {num_steps} and {split}" + + for steps in [7, 20]: + assert_run_mixture(steps, 0.33, EulerDiscreteScheduler) + assert_run_mixture(steps, 0.33, HeunDiscreteScheduler) + + @slow + def test_stable_diffusion_two_xl_mixture_of_denoiser(self): + components = self.get_dummy_components() + pipe_1 = StableDiffusionXLInpaintPipeline(**components).to(torch_device) + pipe_1.unet.set_default_attn_processor() + pipe_2 = StableDiffusionXLInpaintPipeline(**components).to(torch_device) + pipe_2.unet.set_default_attn_processor() + + def assert_run_mixture( + num_steps, split, scheduler_cls_orig, num_train_timesteps=pipe_1.scheduler.config.num_train_timesteps + ): + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = num_steps + + class scheduler_cls(scheduler_cls_orig): + pass + + pipe_1.scheduler = scheduler_cls.from_config(pipe_1.scheduler.config) + pipe_2.scheduler = scheduler_cls.from_config(pipe_2.scheduler.config) + + # Let's retrieve the number of timesteps we want to use + pipe_1.scheduler.set_timesteps(num_steps) + expected_steps = pipe_1.scheduler.timesteps.tolist() + + split_ts = num_train_timesteps - int(round(num_train_timesteps * split)) + + if pipe_1.scheduler.order == 2: + expected_steps_1 = list(filter(lambda ts: ts >= split_ts, expected_steps)) + expected_steps_2 = expected_steps_1[-1:] + list(filter(lambda ts: ts < split_ts, expected_steps)) + expected_steps = expected_steps_1 + expected_steps_2 + else: + expected_steps_1 = list(filter(lambda ts: ts >= split_ts, expected_steps)) + expected_steps_2 = list(filter(lambda ts: ts < split_ts, expected_steps)) + + # now we monkey patch step `done_steps` + # list into the step function for testing + done_steps = [] + old_step = copy.copy(scheduler_cls.step) + + def new_step(self, *args, **kwargs): + done_steps.append(args[1].cpu().item()) # args[1] is always the passed `t` + return old_step(self, *args, **kwargs) + + scheduler_cls.step = new_step + + inputs_1 = {**inputs, **{"denoising_end": split, "output_type": "latent"}} + latents = pipe_1(**inputs_1).images[0] + + assert expected_steps_1 == done_steps, f"Failure with {scheduler_cls.__name__} and {num_steps} and {split}" + + inputs_2 = {**inputs, **{"denoising_start": split, "image": latents}} + pipe_2(**inputs_2).images[0] + + assert expected_steps_2 == done_steps[len(expected_steps_1) :] + assert expected_steps == done_steps, f"Failure with {scheduler_cls.__name__} and {num_steps} and {split}" + + for steps in [5, 8, 20]: + for split in [0.33, 0.49, 0.71]: + for scheduler_cls in [ + DDIMScheduler, + EulerDiscreteScheduler, + DPMSolverMultistepScheduler, + UniPCMultistepScheduler, + HeunDiscreteScheduler, + ]: + assert_run_mixture(steps, split, scheduler_cls) + + @slow + def test_stable_diffusion_three_xl_mixture_of_denoiser(self): + components = self.get_dummy_components() + pipe_1 = StableDiffusionXLInpaintPipeline(**components).to(torch_device) + pipe_1.unet.set_default_attn_processor() + pipe_2 = StableDiffusionXLInpaintPipeline(**components).to(torch_device) + pipe_2.unet.set_default_attn_processor() + pipe_3 = StableDiffusionXLInpaintPipeline(**components).to(torch_device) + pipe_3.unet.set_default_attn_processor() + + def assert_run_mixture( + num_steps, + split_1, + split_2, + scheduler_cls_orig, + num_train_timesteps=pipe_1.scheduler.config.num_train_timesteps, + ): + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = num_steps + + class scheduler_cls(scheduler_cls_orig): + pass + + pipe_1.scheduler = scheduler_cls.from_config(pipe_1.scheduler.config) + pipe_2.scheduler = scheduler_cls.from_config(pipe_2.scheduler.config) + pipe_3.scheduler = scheduler_cls.from_config(pipe_3.scheduler.config) + + # Let's retrieve the number of timesteps we want to use + pipe_1.scheduler.set_timesteps(num_steps) + expected_steps = pipe_1.scheduler.timesteps.tolist() + + split_1_ts = num_train_timesteps - int(round(num_train_timesteps * split_1)) + split_2_ts = num_train_timesteps - int(round(num_train_timesteps * split_2)) + + if pipe_1.scheduler.order == 2: + expected_steps_1 = list(filter(lambda ts: ts >= split_1_ts, expected_steps)) + expected_steps_2 = expected_steps_1[-1:] + list( + filter(lambda ts: ts >= split_2_ts and ts < split_1_ts, expected_steps) + ) + expected_steps_3 = expected_steps_2[-1:] + list(filter(lambda ts: ts < split_2_ts, expected_steps)) + expected_steps = expected_steps_1 + expected_steps_2 + expected_steps_3 + else: + expected_steps_1 = list(filter(lambda ts: ts >= split_1_ts, expected_steps)) + expected_steps_2 = list(filter(lambda ts: ts >= split_2_ts and ts < split_1_ts, expected_steps)) + expected_steps_3 = list(filter(lambda ts: ts < split_2_ts, expected_steps)) + + # now we monkey patch step `done_steps` + # list into the step function for testing + done_steps = [] + old_step = copy.copy(scheduler_cls.step) + + def new_step(self, *args, **kwargs): + done_steps.append(args[1].cpu().item()) # args[1] is always the passed `t` + return old_step(self, *args, **kwargs) + + scheduler_cls.step = new_step + + inputs_1 = {**inputs, **{"denoising_end": split_1, "output_type": "latent"}} + latents = pipe_1(**inputs_1).images[0] + + assert ( + expected_steps_1 == done_steps + ), f"Failure with {scheduler_cls.__name__} and {num_steps} and {split_1} and {split_2}" + + inputs_2 = { + **inputs, + **{"denoising_start": split_1, "denoising_end": split_2, "image": latents, "output_type": "latent"}, + } + pipe_2(**inputs_2).images[0] + + assert expected_steps_2 == done_steps[len(expected_steps_1) :] + + inputs_3 = {**inputs, **{"denoising_start": split_2, "image": latents}} + pipe_3(**inputs_3).images[0] + + assert expected_steps_3 == done_steps[len(expected_steps_1) + len(expected_steps_2) :] + assert ( + expected_steps == done_steps + ), f"Failure with {scheduler_cls.__name__} and {num_steps} and {split_1} and {split_2}" + + for steps in [7, 11, 20]: + for split_1, split_2 in zip([0.19, 0.32], [0.81, 0.68]): + for scheduler_cls in [ + DDIMScheduler, + EulerDiscreteScheduler, + DPMSolverMultistepScheduler, + UniPCMultistepScheduler, + HeunDiscreteScheduler, + ]: + assert_run_mixture(steps, split_1, split_2, scheduler_cls) + + def test_stable_diffusion_xl_multi_prompts(self): + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + + # forward with single prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = 5 + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same prompt duplicated + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = 5 + inputs["prompt_2"] = inputs["prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = 5 + inputs["prompt_2"] = "different prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + # manually set a negative_prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = 5 + inputs["negative_prompt"] = "negative prompt" + output = sd_pipe(**inputs) + image_slice_1 = output.images[0, -3:, -3:, -1] + + # forward with same negative_prompt duplicated + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = 5 + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = inputs["negative_prompt"] + output = sd_pipe(**inputs) + image_slice_2 = output.images[0, -3:, -3:, -1] + + # ensure the results are equal + assert np.abs(image_slice_1.flatten() - image_slice_2.flatten()).max() < 1e-4 + + # forward with different negative_prompt + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = 5 + inputs["negative_prompt"] = "negative prompt" + inputs["negative_prompt_2"] = "different negative prompt" + output = sd_pipe(**inputs) + image_slice_3 = output.images[0, -3:, -3:, -1] + + # ensure the results are not equal + assert np.abs(image_slice_1.flatten() - image_slice_3.flatten()).max() > 1e-4 + + def test_stable_diffusion_xl_img2img_negative_conditions(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + image = sd_pipe(**inputs).images + image_slice_with_no_neg_conditions = image[0, -3:, -3:, -1] + + image = sd_pipe( + **inputs, + negative_original_size=(512, 512), + negative_crops_coords_top_left=( + 0, + 0, + ), + negative_target_size=(1024, 1024), + ).images + image_slice_with_neg_conditions = image[0, -3:, -3:, -1] + + assert ( + np.abs(image_slice_with_no_neg_conditions.flatten() - image_slice_with_neg_conditions.flatten()).max() + > 1e-4 + ) + + def test_stable_diffusion_xl_inpaint_mask_latents(self): + device = "cpu" + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(device) + sd_pipe.set_progress_bar_config(disable=None) + + # normal mask + normal image + ## `image`: pil, `mask_image``: pil, `masked_image_latents``: None + inputs = self.get_dummy_inputs(device) + inputs["strength"] = 0.9 + out_0 = sd_pipe(**inputs).images + + # image latents + mask latents + inputs = self.get_dummy_inputs(device) + image = sd_pipe.image_processor.preprocess(inputs["image"]).to(sd_pipe.device) + mask = sd_pipe.mask_processor.preprocess(inputs["mask_image"]).to(sd_pipe.device) + masked_image = image * (mask < 0.5) + + generator = torch.Generator(device=device).manual_seed(0) + image_latents = sd_pipe._encode_vae_image(image, generator=generator) + torch.randn((1, 4, 32, 32), generator=generator) + mask_latents = sd_pipe._encode_vae_image(masked_image, generator=generator) + inputs["image"] = image_latents + inputs["masked_image_latents"] = mask_latents + inputs["mask_image"] = mask + inputs["strength"] = 0.9 + generator = torch.Generator(device=device).manual_seed(0) + torch.randn((1, 4, 32, 32), generator=generator) + inputs["generator"] = generator + out_1 = sd_pipe(**inputs).images + assert np.abs(out_0 - out_1).max() < 1e-2 + + def test_stable_diffusion_xl_inpaint_2_images(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + # test to confirm if we pass two same image, we will get same output + inputs = self.get_dummy_inputs(device) + gen1 = torch.Generator(device=device).manual_seed(0) + gen2 = torch.Generator(device=device).manual_seed(0) + for name in ["prompt", "image", "mask_image"]: + inputs[name] = [inputs[name]] * 2 + inputs["generator"] = [gen1, gen2] + images = sd_pipe(**inputs).images + + assert images.shape == (2, 64, 64, 3) + + image_slice1 = images[0, -3:, -3:, -1] + image_slice2 = images[1, -3:, -3:, -1] + assert np.abs(image_slice1.flatten() - image_slice2.flatten()).max() < 1e-4 + + # test to confirm that if we pass two different images, we will get different output + inputs = self.get_dummy_inputs_2images(device) + images = sd_pipe(**inputs).images + assert images.shape == (2, 64, 64, 3) + + image_slice1 = images[0, -3:, -3:, -1] + image_slice2 = images[1, -3:, -3:, -1] + assert np.abs(image_slice1.flatten() - image_slice2.flatten()).max() > 1e-2 + + def test_pipeline_interrupt(self): + components = self.get_dummy_components() + sd_pipe = StableDiffusionXLInpaintPipeline(**components) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + prompt = "hey" + num_inference_steps = 5 + + # store intermediate latents from the generation process + class PipelineState: + def __init__(self): + self.state = [] + + def apply(self, pipe, i, t, callback_kwargs): + self.state.append(callback_kwargs["latents"]) + return callback_kwargs + + pipe_state = PipelineState() + sd_pipe( + prompt, + image=inputs["image"], + mask_image=inputs["mask_image"], + strength=0.8, + num_inference_steps=num_inference_steps, + output_type="np", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=pipe_state.apply, + ).images + + # interrupt generation at step index + interrupt_step_idx = 1 + + def callback_on_step_end(pipe, i, t, callback_kwargs): + if i == interrupt_step_idx: + pipe._interrupt = True + + return callback_kwargs + + output_interrupted = sd_pipe( + prompt, + image=inputs["image"], + mask_image=inputs["mask_image"], + strength=0.8, + num_inference_steps=num_inference_steps, + output_type="latent", + generator=torch.Generator("cpu").manual_seed(0), + callback_on_step_end=callback_on_step_end, + ).images + + # fetch intermediate latents at the interrupted step + # from the completed generation process + intermediate_latent = pipe_state.state[interrupt_step_idx] + + # compare the intermediate latent to the output of the interrupted process + # they should be the same + assert torch.allclose(intermediate_latent, output_interrupted, atol=1e-4) diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_instruction_pix2pix.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_instruction_pix2pix.py new file mode 100755 index 0000000..0b4324f --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_instruction_pix2pix.py @@ -0,0 +1,189 @@ +# coding=utf-8 +# Copyright 2024 Harutatsu Akiyama and HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + EulerDiscreteScheduler, + UNet2DConditionModel, +) +from diffusers.image_processor import VaeImageProcessor +from diffusers.pipelines.stable_diffusion_xl.pipeline_stable_diffusion_xl_instruct_pix2pix import ( + StableDiffusionXLInstructPix2PixPipeline, +) +from diffusers.utils.testing_utils import enable_full_determinism, floats_tensor, torch_device + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, +) +from ..test_pipelines_common import ( + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, + SDXLOptionalComponentsTesterMixin, +) + + +enable_full_determinism() + + +class StableDiffusionXLInstructPix2PixPipelineFastTests( + PipelineLatentTesterMixin, + PipelineKarrasSchedulerTesterMixin, + PipelineTesterMixin, + SDXLOptionalComponentsTesterMixin, + unittest.TestCase, +): + pipeline_class = StableDiffusionXLInstructPix2PixPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS - {"height", "width", "cross_attention_kwargs"} + batch_params = TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + image_latents_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=8, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 5 * 8 + 32 + cross_attention_dim=64, + ) + + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + steps_offset=1, + beta_schedule="scaled_linear", + timestep_spacing="leading", + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + } + return components + + def get_dummy_inputs(self, device, seed=0): + image = floats_tensor((1, 3, 64, 64), rng=random.Random(seed)).to(device) + image = image / 2 + 0.5 + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "image_guidance_scale": 1, + "output_type": "numpy", + } + return inputs + + def test_components_function(self): + init_components = self.get_dummy_components() + pipe = self.pipeline_class(**init_components) + + self.assertTrue(hasattr(pipe, "components")) + self.assertTrue(set(pipe.components.keys()) == set(init_components.keys())) + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=3e-3) + + def test_attention_slicing_forward_pass(self): + super().test_attention_slicing_forward_pass(expected_max_diff=2e-3) + + # Overwrite the default test_latents_inputs because pix2pix encode the image differently + def test_latents_input(self): + components = self.get_dummy_components() + pipe = StableDiffusionXLInstructPix2PixPipeline(**components) + pipe.image_processor = VaeImageProcessor(do_resize=False, do_normalize=False) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + out = pipe(**self.get_dummy_inputs_by_type(torch_device, input_image_type="pt"))[0] + + vae = components["vae"] + inputs = self.get_dummy_inputs_by_type(torch_device, input_image_type="pt") + + for image_param in self.image_latents_params: + if image_param in inputs.keys(): + inputs[image_param] = vae.encode(inputs[image_param]).latent_dist.mode() + + out_latents_inputs = pipe(**inputs)[0] + + max_diff = np.abs(out - out_latents_inputs).max() + self.assertLess(max_diff, 1e-4, "passing latents as image input generate different result from passing image") + + def test_cfg(self): + pass + + def test_save_load_optional_components(self): + self._test_save_load_optional_components() diff --git a/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_k_diffusion.py b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_k_diffusion.py new file mode 100755 index 0000000..d5d2abf --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_diffusion_xl/test_stable_diffusion_xl_k_diffusion.py @@ -0,0 +1,138 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch + +from diffusers import StableDiffusionXLKDiffusionPipeline +from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, slow, torch_device + + +enable_full_determinism() + + +@slow +@require_torch_gpu +class StableDiffusionXLKPipelineIntegrationTests(unittest.TestCase): + dtype = torch.float16 + + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_diffusion_xl(self): + sd_pipe = StableDiffusionXLKDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=self.dtype + ) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + sd_pipe.set_scheduler("sample_euler") + + prompt = "A painting of a squirrel eating a burger" + generator = torch.manual_seed(0) + output = sd_pipe( + [prompt], + generator=generator, + guidance_scale=9.0, + num_inference_steps=20, + height=512, + width=512, + output_type="np", + ) + + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array( + [0.79600024, 0.796546, 0.80682373, 0.79428387, 0.7905743, 0.8008807, 0.786183, 0.7835959, 0.797892] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_karras_sigmas(self): + sd_pipe = StableDiffusionXLKDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=self.dtype + ) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + sd_pipe.set_scheduler("sample_dpmpp_2m") + + prompt = "A painting of a squirrel eating a burger" + generator = torch.manual_seed(0) + output = sd_pipe( + [prompt], + generator=generator, + guidance_scale=7.5, + num_inference_steps=15, + output_type="np", + use_karras_sigmas=True, + height=512, + width=512, + ) + + image = output.images + + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array( + [0.9506951, 0.9527786, 0.95309967, 0.9511477, 0.952523, 0.9515326, 0.9511933, 0.9480397, 0.94930184] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + def test_stable_diffusion_noise_sampler_seed(self): + sd_pipe = StableDiffusionXLKDiffusionPipeline.from_pretrained( + "stabilityai/stable-diffusion-xl-base-1.0", torch_dtype=self.dtype + ) + sd_pipe = sd_pipe.to(torch_device) + sd_pipe.set_progress_bar_config(disable=None) + + sd_pipe.set_scheduler("sample_dpmpp_sde") + + prompt = "A painting of a squirrel eating a burger" + seed = 0 + images1 = sd_pipe( + [prompt], + generator=torch.manual_seed(seed), + noise_sampler_seed=seed, + guidance_scale=9.0, + num_inference_steps=20, + output_type="np", + height=512, + width=512, + ).images + images2 = sd_pipe( + [prompt], + generator=torch.manual_seed(seed), + noise_sampler_seed=seed, + guidance_scale=9.0, + num_inference_steps=20, + output_type="np", + height=512, + width=512, + ).images + assert images1.shape == (1, 512, 512, 3) + assert images2.shape == (1, 512, 512, 3) + assert np.abs(images1.flatten() - images2.flatten()).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/stable_unclip/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_unclip/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_unclip/test_stable_unclip.py b/diffusers-0.27.0/tests/pipelines/stable_unclip/test_stable_unclip.py new file mode 100755 index 0000000..f05edf6 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_unclip/test_stable_unclip.py @@ -0,0 +1,239 @@ +import gc +import unittest + +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + DDPMScheduler, + PriorTransformer, + StableUnCLIPPipeline, + UNet2DConditionModel, +) +from diffusers.pipelines.stable_diffusion.stable_unclip_image_normalizer import StableUnCLIPImageNormalizer +from diffusers.utils.testing_utils import enable_full_determinism, load_numpy, nightly, require_torch_gpu, torch_device + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import ( + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, + assert_mean_pixel_difference, +) + + +enable_full_determinism() + + +class StableUnCLIPPipelineFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableUnCLIPPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + + # TODO(will) Expected attn_bias.stride(1) == 0 to be true, but got false + test_xformers_attention = False + + def get_dummy_components(self): + embedder_hidden_size = 32 + embedder_projection_dim = embedder_hidden_size + + # prior components + + torch.manual_seed(0) + prior_tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + torch.manual_seed(0) + prior_text_encoder = CLIPTextModelWithProjection( + CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=embedder_hidden_size, + projection_dim=embedder_projection_dim, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + ) + + torch.manual_seed(0) + prior = PriorTransformer( + num_attention_heads=2, + attention_head_dim=12, + embedding_dim=embedder_projection_dim, + num_layers=1, + ) + + torch.manual_seed(0) + prior_scheduler = DDPMScheduler( + variance_type="fixed_small_log", + prediction_type="sample", + num_train_timesteps=1000, + clip_sample=True, + clip_sample_range=5.0, + beta_schedule="squaredcos_cap_v2", + ) + + # regular denoising components + + torch.manual_seed(0) + image_normalizer = StableUnCLIPImageNormalizer(embedding_dim=embedder_hidden_size) + image_noising_scheduler = DDPMScheduler(beta_schedule="squaredcos_cap_v2") + + torch.manual_seed(0) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + torch.manual_seed(0) + text_encoder = CLIPTextModel( + CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=embedder_hidden_size, + projection_dim=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + ) + + torch.manual_seed(0) + unet = UNet2DConditionModel( + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("CrossAttnDownBlock2D", "DownBlock2D"), + up_block_types=("UpBlock2D", "CrossAttnUpBlock2D"), + block_out_channels=(32, 64), + attention_head_dim=(2, 4), + class_embed_type="projection", + # The class embeddings are the noise augmented image embeddings. + # I.e. the image embeddings concated with the noised embeddings of the same dimension + projection_class_embeddings_input_dim=embedder_projection_dim * 2, + cross_attention_dim=embedder_hidden_size, + layers_per_block=1, + upcast_attention=True, + use_linear_projection=True, + ) + + torch.manual_seed(0) + scheduler = DDIMScheduler( + beta_schedule="scaled_linear", + beta_start=0.00085, + beta_end=0.012, + prediction_type="v_prediction", + set_alpha_to_one=False, + steps_offset=1, + ) + + torch.manual_seed(0) + vae = AutoencoderKL() + + components = { + # prior components + "prior_tokenizer": prior_tokenizer, + "prior_text_encoder": prior_text_encoder, + "prior": prior, + "prior_scheduler": prior_scheduler, + # image noising components + "image_normalizer": image_normalizer, + "image_noising_scheduler": image_noising_scheduler, + # regular denoising components + "tokenizer": tokenizer, + "text_encoder": text_encoder, + "unet": unet, + "scheduler": scheduler, + "vae": vae, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "prior_num_inference_steps": 2, + "output_type": "numpy", + } + return inputs + + # Overriding PipelineTesterMixin::test_attention_slicing_forward_pass + # because UnCLIP GPU undeterminism requires a looser check. + def test_attention_slicing_forward_pass(self): + test_max_difference = torch_device == "cpu" + + self._test_attention_slicing_forward_pass(test_max_difference=test_max_difference) + + # Overriding PipelineTesterMixin::test_inference_batch_single_identical + # because UnCLIP undeterminism requires a looser check. + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=1e-3) + + +@nightly +@require_torch_gpu +class StableUnCLIPPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_unclip(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/stable_unclip_2_1_l_anime_turtle_fp16.npy" + ) + + pipe = StableUnCLIPPipeline.from_pretrained("fusing/stable-unclip-2-1-l", torch_dtype=torch.float16) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + # stable unclip will oom when integration tests are run on a V100, + # so turn on memory savings + pipe.enable_attention_slicing() + pipe.enable_sequential_cpu_offload() + + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipe("anime turle", generator=generator, output_type="np") + + image = output.images[0] + + assert image.shape == (768, 768, 3) + + assert_mean_pixel_difference(image, expected_image) + + def test_stable_unclip_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe = StableUnCLIPPipeline.from_pretrained("fusing/stable-unclip-2-1-l", torch_dtype=torch.float16) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + pipe.enable_sequential_cpu_offload() + + _ = pipe( + "anime turtle", + prior_num_inference_steps=2, + num_inference_steps=2, + output_type="np", + ) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 7 GB is allocated + assert mem_bytes < 7 * 10**9 diff --git a/diffusers-0.27.0/tests/pipelines/stable_unclip/test_stable_unclip_img2img.py b/diffusers-0.27.0/tests/pipelines/stable_unclip/test_stable_unclip_img2img.py new file mode 100755 index 0000000..12f6a91 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_unclip/test_stable_unclip_img2img.py @@ -0,0 +1,300 @@ +import gc +import random +import unittest + +import numpy as np +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextConfig, + CLIPTextModel, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +from diffusers import AutoencoderKL, DDIMScheduler, DDPMScheduler, StableUnCLIPImg2ImgPipeline, UNet2DConditionModel +from diffusers.pipelines.pipeline_utils import DiffusionPipeline +from diffusers.pipelines.stable_diffusion.stable_unclip_image_normalizer import StableUnCLIPImageNormalizer +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + nightly, + require_torch_gpu, + skip_mps, + torch_device, +) + +from ..pipeline_params import TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, TEXT_GUIDED_IMAGE_VARIATION_PARAMS +from ..test_pipelines_common import ( + PipelineKarrasSchedulerTesterMixin, + PipelineLatentTesterMixin, + PipelineTesterMixin, + assert_mean_pixel_difference, +) + + +enable_full_determinism() + + +class StableUnCLIPImg2ImgPipelineFastTests( + PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, PipelineTesterMixin, unittest.TestCase +): + pipeline_class = StableUnCLIPImg2ImgPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = frozenset( + [] + ) # TO-DO: update image_params once pipeline is refactored with VaeImageProcessor.preprocess + image_latents_params = frozenset([]) + + def get_dummy_components(self): + embedder_hidden_size = 32 + embedder_projection_dim = embedder_hidden_size + + # image encoding components + + feature_extractor = CLIPImageProcessor(crop_size=32, size=32) + + torch.manual_seed(0) + image_encoder = CLIPVisionModelWithProjection( + CLIPVisionConfig( + hidden_size=embedder_hidden_size, + projection_dim=embedder_projection_dim, + num_hidden_layers=5, + num_attention_heads=4, + image_size=32, + intermediate_size=37, + patch_size=1, + ) + ) + + # regular denoising components + + torch.manual_seed(0) + image_normalizer = StableUnCLIPImageNormalizer(embedding_dim=embedder_hidden_size) + image_noising_scheduler = DDPMScheduler(beta_schedule="squaredcos_cap_v2") + + torch.manual_seed(0) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + torch.manual_seed(0) + text_encoder = CLIPTextModel( + CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=embedder_hidden_size, + projection_dim=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + ) + + torch.manual_seed(0) + unet = UNet2DConditionModel( + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("CrossAttnDownBlock2D", "DownBlock2D"), + up_block_types=("UpBlock2D", "CrossAttnUpBlock2D"), + block_out_channels=(32, 64), + attention_head_dim=(2, 4), + class_embed_type="projection", + # The class embeddings are the noise augmented image embeddings. + # I.e. the image embeddings concated with the noised embeddings of the same dimension + projection_class_embeddings_input_dim=embedder_projection_dim * 2, + cross_attention_dim=embedder_hidden_size, + layers_per_block=1, + upcast_attention=True, + use_linear_projection=True, + ) + + torch.manual_seed(0) + scheduler = DDIMScheduler( + beta_schedule="scaled_linear", + beta_start=0.00085, + beta_end=0.012, + prediction_type="v_prediction", + set_alpha_to_one=False, + steps_offset=1, + ) + + torch.manual_seed(0) + vae = AutoencoderKL() + + components = { + # image encoding components + "feature_extractor": feature_extractor, + "image_encoder": image_encoder.eval(), + # image noising components + "image_normalizer": image_normalizer.eval(), + "image_noising_scheduler": image_noising_scheduler, + # regular denoising components + "tokenizer": tokenizer, + "text_encoder": text_encoder.eval(), + "unet": unet.eval(), + "scheduler": scheduler, + "vae": vae.eval(), + } + + return components + + def get_dummy_inputs(self, device, seed=0, pil_image=True): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + input_image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + + if pil_image: + input_image = input_image * 0.5 + 0.5 + input_image = input_image.clamp(0, 1) + input_image = input_image.cpu().permute(0, 2, 3, 1).float().numpy() + input_image = DiffusionPipeline.numpy_to_pil(input_image)[0] + + return { + "prompt": "An anime racoon running a marathon", + "image": input_image, + "generator": generator, + "num_inference_steps": 2, + "output_type": "np", + } + + @skip_mps + def test_image_embeds_none(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = StableUnCLIPImg2ImgPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs.update({"image_embeds": None}) + image = sd_pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1] + + assert image.shape == (1, 32, 32, 3) + expected_slice = np.array([0.3872, 0.7224, 0.5601, 0.4741, 0.6872, 0.5814, 0.4636, 0.3867, 0.5078]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + # Overriding PipelineTesterMixin::test_attention_slicing_forward_pass + # because GPU undeterminism requires a looser check. + def test_attention_slicing_forward_pass(self): + test_max_difference = torch_device in ["cpu", "mps"] + + self._test_attention_slicing_forward_pass(test_max_difference=test_max_difference) + + # Overriding PipelineTesterMixin::test_inference_batch_single_identical + # because undeterminism requires a looser check. + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=1e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(test_max_difference=False) + + +@nightly +@require_torch_gpu +class StableUnCLIPImg2ImgPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_stable_unclip_l_img2img(self): + input_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/turtle.png" + ) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/stable_unclip_2_1_l_img2img_anime_turtle_fp16.npy" + ) + + pipe = StableUnCLIPImg2ImgPipeline.from_pretrained( + "fusing/stable-unclip-2-1-l-img2img", torch_dtype=torch.float16 + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + # stable unclip will oom when integration tests are run on a V100, + # so turn on memory savings + pipe.enable_attention_slicing() + pipe.enable_sequential_cpu_offload() + + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipe(input_image, "anime turle", generator=generator, output_type="np") + + image = output.images[0] + + assert image.shape == (768, 768, 3) + + assert_mean_pixel_difference(image, expected_image) + + def test_stable_unclip_h_img2img(self): + input_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/turtle.png" + ) + + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/stable_unclip_2_1_h_img2img_anime_turtle_fp16.npy" + ) + + pipe = StableUnCLIPImg2ImgPipeline.from_pretrained( + "fusing/stable-unclip-2-1-h-img2img", torch_dtype=torch.float16 + ) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + # stable unclip will oom when integration tests are run on a V100, + # so turn on memory savings + pipe.enable_attention_slicing() + pipe.enable_sequential_cpu_offload() + + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipe(input_image, "anime turle", generator=generator, output_type="np") + + image = output.images[0] + + assert image.shape == (768, 768, 3) + + assert_mean_pixel_difference(image, expected_image) + + def test_stable_unclip_img2img_pipeline_with_sequential_cpu_offloading(self): + input_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/stable_unclip/turtle.png" + ) + + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe = StableUnCLIPImg2ImgPipeline.from_pretrained( + "fusing/stable-unclip-2-1-h-img2img", torch_dtype=torch.float16 + ) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + pipe.enable_sequential_cpu_offload() + + _ = pipe( + input_image, + "anime turtle", + num_inference_steps=2, + output_type="np", + ) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 7 GB is allocated + assert mem_bytes < 7 * 10**9 diff --git a/diffusers-0.27.0/tests/pipelines/stable_video_diffusion/__init__.py b/diffusers-0.27.0/tests/pipelines/stable_video_diffusion/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/stable_video_diffusion/test_stable_video_diffusion.py b/diffusers-0.27.0/tests/pipelines/stable_video_diffusion/test_stable_video_diffusion.py new file mode 100755 index 0000000..33cf4c7 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/stable_video_diffusion/test_stable_video_diffusion.py @@ -0,0 +1,554 @@ +import gc +import random +import tempfile +import unittest + +import numpy as np +import torch +from transformers import ( + CLIPImageProcessor, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +import diffusers +from diffusers import ( + AutoencoderKLTemporalDecoder, + EulerDiscreteScheduler, + StableVideoDiffusionPipeline, + UNetSpatioTemporalConditionModel, +) +from diffusers.utils import is_accelerate_available, is_accelerate_version, load_image, logging +from diffusers.utils.import_utils import is_xformers_available +from diffusers.utils.testing_utils import ( + CaptureLogger, + enable_full_determinism, + floats_tensor, + numpy_cosine_similarity_distance, + require_torch_gpu, + slow, + torch_device, +) + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +def to_np(tensor): + if isinstance(tensor, torch.Tensor): + tensor = tensor.detach().cpu().numpy() + + return tensor + + +class StableVideoDiffusionPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = StableVideoDiffusionPipeline + params = frozenset(["image"]) + batch_params = frozenset(["image", "generator"]) + required_optional_params = frozenset( + [ + "num_inference_steps", + "generator", + "latents", + "return_dict", + ] + ) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNetSpatioTemporalConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=8, + out_channels=4, + down_block_types=( + "CrossAttnDownBlockSpatioTemporal", + "DownBlockSpatioTemporal", + ), + up_block_types=("UpBlockSpatioTemporal", "CrossAttnUpBlockSpatioTemporal"), + cross_attention_dim=32, + num_attention_heads=8, + projection_class_embeddings_input_dim=96, + addition_time_embed_dim=32, + ) + scheduler = EulerDiscreteScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + interpolation_type="linear", + num_train_timesteps=1000, + prediction_type="v_prediction", + sigma_max=700.0, + sigma_min=0.002, + steps_offset=1, + timestep_spacing="leading", + timestep_type="continuous", + trained_betas=None, + use_karras_sigmas=True, + ) + + torch.manual_seed(0) + vae = AutoencoderKLTemporalDecoder( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + latent_channels=4, + ) + + torch.manual_seed(0) + config = CLIPVisionConfig( + hidden_size=32, + projection_dim=32, + num_hidden_layers=5, + num_attention_heads=4, + image_size=32, + intermediate_size=37, + patch_size=1, + ) + image_encoder = CLIPVisionModelWithProjection(config) + + torch.manual_seed(0) + feature_extractor = CLIPImageProcessor(crop_size=32, size=32) + components = { + "unet": unet, + "image_encoder": image_encoder, + "scheduler": scheduler, + "vae": vae, + "feature_extractor": feature_extractor, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device="cpu").manual_seed(seed) + + image = floats_tensor((1, 3, 32, 32), rng=random.Random(0)).to(device) + inputs = { + "generator": generator, + "image": image, + "num_inference_steps": 2, + "output_type": "pt", + "min_guidance_scale": 1.0, + "max_guidance_scale": 2.5, + "num_frames": 2, + "height": 32, + "width": 32, + } + return inputs + + @unittest.skip("Deprecated functionality") + def test_attention_slicing_forward_pass(self): + pass + + @unittest.skip("Batched inference works and outputs look correct, but the test is failing") + def test_inference_batch_single_identical( + self, + batch_size=2, + expected_max_diff=1e-4, + ): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for components in pipe.components.values(): + if hasattr(components, "set_default_attn_processor"): + components.set_default_attn_processor() + pipe.to(torch_device) + + pipe.set_progress_bar_config(disable=None) + inputs = self.get_dummy_inputs(torch_device) + + # Reset generator in case it is has been used in self.get_dummy_inputs + inputs["generator"] = torch.Generator("cpu").manual_seed(0) + + logger = logging.get_logger(pipe.__module__) + logger.setLevel(level=diffusers.logging.FATAL) + + # batchify inputs + batched_inputs = {} + batched_inputs.update(inputs) + + batched_inputs["generator"] = [torch.Generator("cpu").manual_seed(0) for i in range(batch_size)] + batched_inputs["image"] = torch.cat([inputs["image"]] * batch_size, dim=0) + + output = pipe(**inputs).frames + output_batch = pipe(**batched_inputs).frames + + assert len(output_batch) == batch_size + + max_diff = np.abs(to_np(output_batch[0]) - to_np(output[0])).max() + assert max_diff < expected_max_diff + + @unittest.skip("Test is similar to test_inference_batch_single_identical") + def test_inference_batch_consistent(self): + pass + + def test_np_output_type(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + inputs["output_type"] = "np" + output = pipe(**inputs).frames + self.assertTrue(isinstance(output, np.ndarray)) + self.assertEqual(len(output.shape), 5) + + def test_dict_tuple_outputs_equivalent(self, expected_max_difference=1e-4): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator_device = "cpu" + output = pipe(**self.get_dummy_inputs(generator_device)).frames[0] + output_tuple = pipe(**self.get_dummy_inputs(generator_device), return_dict=False)[0] + + max_diff = np.abs(to_np(output) - to_np(output_tuple)).max() + self.assertLess(max_diff, expected_max_difference) + + @unittest.skip("Test is currently failing") + def test_float16_inference(self, expected_max_diff=5e-2): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + components = self.get_dummy_components() + pipe_fp16 = self.pipeline_class(**components) + for component in pipe_fp16.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe_fp16.to(torch_device, torch.float16) + pipe_fp16.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output = pipe(**inputs).frames[0] + + fp16_inputs = self.get_dummy_inputs(torch_device) + output_fp16 = pipe_fp16(**fp16_inputs).frames[0] + + max_diff = np.abs(to_np(output) - to_np(output_fp16)).max() + self.assertLess(max_diff, expected_max_diff, "The outputs of the fp16 and fp32 pipelines are too different.") + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_save_load_float16(self, expected_max_diff=1e-2): + components = self.get_dummy_components() + for name, module in components.items(): + if hasattr(module, "half"): + components[name] = module.to(torch_device).half() + + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output = pipe(**inputs).frames[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir, torch_dtype=torch.float16) + for component in pipe_loaded.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + for name, component in pipe_loaded.components.items(): + if hasattr(component, "dtype"): + self.assertTrue( + component.dtype == torch.float16, + f"`{name}.dtype` switched from `float16` to {component.dtype} after loading.", + ) + + inputs = self.get_dummy_inputs(torch_device) + output_loaded = pipe_loaded(**inputs).frames[0] + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess( + max_diff, expected_max_diff, "The output of the fp16 pipeline changed after saving and loading." + ) + + def test_save_load_optional_components(self, expected_max_difference=1e-4): + if not hasattr(self.pipeline_class, "_optional_components"): + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + # set all optional components to None + for optional_component in pipe._optional_components: + setattr(pipe, optional_component, None) + + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + output = pipe(**inputs).frames[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir, safe_serialization=False) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir) + for component in pipe_loaded.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + for optional_component in pipe._optional_components: + self.assertTrue( + getattr(pipe_loaded, optional_component) is None, + f"`{optional_component}` did not stay set to None after loading.", + ) + + inputs = self.get_dummy_inputs(generator_device) + output_loaded = pipe_loaded(**inputs).frames[0] + + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess(max_diff, expected_max_difference) + + def test_save_load_local(self, expected_max_difference=9e-4): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output = pipe(**inputs).frames[0] + + logger = logging.get_logger("diffusers.pipelines.pipeline_utils") + logger.setLevel(diffusers.logging.INFO) + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir, safe_serialization=False) + + with CaptureLogger(logger) as cap_logger: + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir) + + for name in pipe_loaded.components.keys(): + if name not in pipe_loaded._optional_components: + assert name in str(cap_logger) + + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output_loaded = pipe_loaded(**inputs).frames[0] + + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess(max_diff, expected_max_difference) + + @unittest.skipIf(torch_device != "cuda", reason="CUDA and CPU are required to switch devices") + def test_to_device(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + pipe.to("cpu") + model_devices = [ + component.device.type for component in pipe.components.values() if hasattr(component, "device") + ] + self.assertTrue(all(device == "cpu" for device in model_devices)) + + output_cpu = pipe(**self.get_dummy_inputs("cpu")).frames[0] + self.assertTrue(np.isnan(output_cpu).sum() == 0) + + pipe.to("cuda") + model_devices = [ + component.device.type for component in pipe.components.values() if hasattr(component, "device") + ] + self.assertTrue(all(device == "cuda" for device in model_devices)) + + output_cuda = pipe(**self.get_dummy_inputs("cuda")).frames[0] + self.assertTrue(np.isnan(to_np(output_cuda)).sum() == 0) + + def test_to_dtype(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + model_dtypes = [component.dtype for component in pipe.components.values() if hasattr(component, "dtype")] + self.assertTrue(all(dtype == torch.float32 for dtype in model_dtypes)) + + pipe.to(dtype=torch.float16) + model_dtypes = [component.dtype for component in pipe.components.values() if hasattr(component, "dtype")] + self.assertTrue(all(dtype == torch.float16 for dtype in model_dtypes)) + + @unittest.skipIf( + torch_device != "cuda" or not is_accelerate_available() or is_accelerate_version("<", "0.14.0"), + reason="CPU offload is only available with CUDA and `accelerate v0.14.0` or higher", + ) + def test_sequential_cpu_offload_forward_pass(self, expected_max_diff=1e-4): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + output_without_offload = pipe(**inputs).frames[0] + + pipe.enable_sequential_cpu_offload() + + inputs = self.get_dummy_inputs(generator_device) + output_with_offload = pipe(**inputs).frames[0] + + max_diff = np.abs(to_np(output_with_offload) - to_np(output_without_offload)).max() + self.assertLess(max_diff, expected_max_diff, "CPU offloading should not affect the inference results") + + @unittest.skipIf( + torch_device != "cuda" or not is_accelerate_available() or is_accelerate_version("<", "0.17.0"), + reason="CPU offload is only available with CUDA and `accelerate v0.17.0` or higher", + ) + def test_model_cpu_offload_forward_pass(self, expected_max_diff=2e-4): + generator_device = "cpu" + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(generator_device) + output_without_offload = pipe(**inputs).frames[0] + + pipe.enable_model_cpu_offload() + inputs = self.get_dummy_inputs(generator_device) + output_with_offload = pipe(**inputs).frames[0] + + max_diff = np.abs(to_np(output_with_offload) - to_np(output_without_offload)).max() + self.assertLess(max_diff, expected_max_diff, "CPU offloading should not affect the inference results") + offloaded_modules = [ + v + for k, v in pipe.components.items() + if isinstance(v, torch.nn.Module) and k not in pipe._exclude_from_cpu_offload + ] + ( + self.assertTrue(all(v.device.type == "cpu" for v in offloaded_modules)), + f"Not offloaded: {[v for v in offloaded_modules if v.device.type != 'cpu']}", + ) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + expected_max_diff = 9e-4 + + if not self.test_xformers_attention: + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output_without_offload = pipe(**inputs).frames[0] + output_without_offload = ( + output_without_offload.cpu() if torch.is_tensor(output_without_offload) else output_without_offload + ) + + pipe.enable_xformers_memory_efficient_attention() + inputs = self.get_dummy_inputs(torch_device) + output_with_offload = pipe(**inputs).frames[0] + output_with_offload = ( + output_with_offload.cpu() if torch.is_tensor(output_with_offload) else output_without_offload + ) + + max_diff = np.abs(to_np(output_with_offload) - to_np(output_without_offload)).max() + self.assertLess(max_diff, expected_max_diff, "XFormers attention should not affect the inference results") + + def test_disable_cfg(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + inputs["max_guidance_scale"] = 1.0 + output = pipe(**inputs).frames + self.assertEqual(len(output.shape), 5) + + +@slow +@require_torch_gpu +class StableVideoDiffusionPipelineSlowTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_sd_video(self): + pipe = StableVideoDiffusionPipeline.from_pretrained( + "stabilityai/stable-video-diffusion-img2vid", + variant="fp16", + torch_dtype=torch.float16, + ) + pipe = pipe.to(torch_device) + pipe.enable_model_cpu_offload() + pipe.set_progress_bar_config(disable=None) + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/pix2pix/cat_6.png?download=true" + ) + + generator = torch.Generator("cpu").manual_seed(0) + num_frames = 3 + + output = pipe( + image=image, + num_frames=num_frames, + generator=generator, + num_inference_steps=3, + output_type="np", + ) + + image = output.frames[0] + assert image.shape == (num_frames, 576, 1024, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.8592, 0.8645, 0.8499, 0.8722, 0.8769, 0.8421, 0.8557, 0.8528, 0.8285]) + assert numpy_cosine_similarity_distance(image_slice.flatten(), expected_slice.flatten()) < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/test_pipeline_utils.py b/diffusers-0.27.0/tests/pipelines/test_pipeline_utils.py new file mode 100755 index 0000000..51d987d --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/test_pipeline_utils.py @@ -0,0 +1,134 @@ +import unittest + +from diffusers.pipelines.pipeline_utils import is_safetensors_compatible + + +class IsSafetensorsCompatibleTests(unittest.TestCase): + def test_all_is_compatible(self): + filenames = [ + "safety_checker/pytorch_model.bin", + "safety_checker/model.safetensors", + "vae/diffusion_pytorch_model.bin", + "vae/diffusion_pytorch_model.safetensors", + "text_encoder/pytorch_model.bin", + "text_encoder/model.safetensors", + "unet/diffusion_pytorch_model.bin", + "unet/diffusion_pytorch_model.safetensors", + ] + self.assertTrue(is_safetensors_compatible(filenames)) + + def test_diffusers_model_is_compatible(self): + filenames = [ + "unet/diffusion_pytorch_model.bin", + "unet/diffusion_pytorch_model.safetensors", + ] + self.assertTrue(is_safetensors_compatible(filenames)) + + def test_diffusers_model_is_not_compatible(self): + filenames = [ + "safety_checker/pytorch_model.bin", + "safety_checker/model.safetensors", + "vae/diffusion_pytorch_model.bin", + "vae/diffusion_pytorch_model.safetensors", + "text_encoder/pytorch_model.bin", + "text_encoder/model.safetensors", + "unet/diffusion_pytorch_model.bin", + # Removed: 'unet/diffusion_pytorch_model.safetensors', + ] + self.assertFalse(is_safetensors_compatible(filenames)) + + def test_transformer_model_is_compatible(self): + filenames = [ + "text_encoder/pytorch_model.bin", + "text_encoder/model.safetensors", + ] + self.assertTrue(is_safetensors_compatible(filenames)) + + def test_transformer_model_is_not_compatible(self): + filenames = [ + "safety_checker/pytorch_model.bin", + "safety_checker/model.safetensors", + "vae/diffusion_pytorch_model.bin", + "vae/diffusion_pytorch_model.safetensors", + "text_encoder/pytorch_model.bin", + # Removed: 'text_encoder/model.safetensors', + "unet/diffusion_pytorch_model.bin", + "unet/diffusion_pytorch_model.safetensors", + ] + self.assertFalse(is_safetensors_compatible(filenames)) + + def test_all_is_compatible_variant(self): + filenames = [ + "safety_checker/pytorch_model.fp16.bin", + "safety_checker/model.fp16.safetensors", + "vae/diffusion_pytorch_model.fp16.bin", + "vae/diffusion_pytorch_model.fp16.safetensors", + "text_encoder/pytorch_model.fp16.bin", + "text_encoder/model.fp16.safetensors", + "unet/diffusion_pytorch_model.fp16.bin", + "unet/diffusion_pytorch_model.fp16.safetensors", + ] + variant = "fp16" + self.assertTrue(is_safetensors_compatible(filenames, variant=variant)) + + def test_diffusers_model_is_compatible_variant(self): + filenames = [ + "unet/diffusion_pytorch_model.fp16.bin", + "unet/diffusion_pytorch_model.fp16.safetensors", + ] + variant = "fp16" + self.assertTrue(is_safetensors_compatible(filenames, variant=variant)) + + def test_diffusers_model_is_compatible_variant_partial(self): + # pass variant but use the non-variant filenames + filenames = [ + "unet/diffusion_pytorch_model.bin", + "unet/diffusion_pytorch_model.safetensors", + ] + variant = "fp16" + self.assertTrue(is_safetensors_compatible(filenames, variant=variant)) + + def test_diffusers_model_is_not_compatible_variant(self): + filenames = [ + "safety_checker/pytorch_model.fp16.bin", + "safety_checker/model.fp16.safetensors", + "vae/diffusion_pytorch_model.fp16.bin", + "vae/diffusion_pytorch_model.fp16.safetensors", + "text_encoder/pytorch_model.fp16.bin", + "text_encoder/model.fp16.safetensors", + "unet/diffusion_pytorch_model.fp16.bin", + # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors', + ] + variant = "fp16" + self.assertFalse(is_safetensors_compatible(filenames, variant=variant)) + + def test_transformer_model_is_compatible_variant(self): + filenames = [ + "text_encoder/pytorch_model.fp16.bin", + "text_encoder/model.fp16.safetensors", + ] + variant = "fp16" + self.assertTrue(is_safetensors_compatible(filenames, variant=variant)) + + def test_transformer_model_is_compatible_variant_partial(self): + # pass variant but use the non-variant filenames + filenames = [ + "text_encoder/pytorch_model.bin", + "text_encoder/model.safetensors", + ] + variant = "fp16" + self.assertTrue(is_safetensors_compatible(filenames, variant=variant)) + + def test_transformer_model_is_not_compatible_variant(self): + filenames = [ + "safety_checker/pytorch_model.fp16.bin", + "safety_checker/model.fp16.safetensors", + "vae/diffusion_pytorch_model.fp16.bin", + "vae/diffusion_pytorch_model.fp16.safetensors", + "text_encoder/pytorch_model.fp16.bin", + # 'text_encoder/model.fp16.safetensors', + "unet/diffusion_pytorch_model.fp16.bin", + "unet/diffusion_pytorch_model.fp16.safetensors", + ] + variant = "fp16" + self.assertFalse(is_safetensors_compatible(filenames, variant=variant)) diff --git a/diffusers-0.27.0/tests/pipelines/test_pipelines.py b/diffusers-0.27.0/tests/pipelines/test_pipelines.py new file mode 100755 index 0000000..8954456 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/test_pipelines.py @@ -0,0 +1,1932 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import json +import os +import random +import shutil +import sys +import tempfile +import traceback +import unittest +import unittest.mock as mock + +import numpy as np +import PIL.Image +import requests_mock +import safetensors.torch +import torch +from parameterized import parameterized +from PIL import Image +from requests.exceptions import HTTPError +from transformers import CLIPImageProcessor, CLIPModel, CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + ConfigMixin, + DDIMPipeline, + DDIMScheduler, + DDPMPipeline, + DDPMScheduler, + DiffusionPipeline, + DPMSolverMultistepScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + LMSDiscreteScheduler, + ModelMixin, + PNDMScheduler, + StableDiffusionImg2ImgPipeline, + StableDiffusionInpaintPipelineLegacy, + StableDiffusionPipeline, + UNet2DConditionModel, + UNet2DModel, + UniPCMultistepScheduler, + logging, +) +from diffusers.pipelines.pipeline_utils import _get_pipeline_class +from diffusers.schedulers.scheduling_utils import SCHEDULER_CONFIG_NAME +from diffusers.utils import ( + CONFIG_NAME, + WEIGHTS_NAME, +) +from diffusers.utils.testing_utils import ( + CaptureLogger, + enable_full_determinism, + floats_tensor, + get_tests_dir, + load_numpy, + nightly, + require_compel, + require_flax, + require_onnxruntime, + require_python39_or_higher, + require_torch_2, + require_torch_gpu, + run_test_in_subprocess, + slow, + torch_device, +) +from diffusers.utils.torch_utils import is_compiled_module + + +enable_full_determinism() + + +# Will be run via run_test_in_subprocess +def _test_from_save_pretrained_dynamo(in_queue, out_queue, timeout): + error = None + try: + # 1. Load models + model = UNet2DModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "AttnDownBlock2D"), + up_block_types=("AttnUpBlock2D", "UpBlock2D"), + ) + model = torch.compile(model) + scheduler = DDPMScheduler(num_train_timesteps=10) + + ddpm = DDPMPipeline(model, scheduler) + + # previous diffusers versions stripped compilation off + # compiled modules + assert is_compiled_module(ddpm.unet) + + ddpm.to(torch_device) + ddpm.set_progress_bar_config(disable=None) + + with tempfile.TemporaryDirectory() as tmpdirname: + ddpm.save_pretrained(tmpdirname) + new_ddpm = DDPMPipeline.from_pretrained(tmpdirname) + new_ddpm.to(torch_device) + + generator = torch.Generator(device=torch_device).manual_seed(0) + image = ddpm(generator=generator, num_inference_steps=5, output_type="numpy").images + + generator = torch.Generator(device=torch_device).manual_seed(0) + new_image = new_ddpm(generator=generator, num_inference_steps=5, output_type="numpy").images + + assert np.abs(image - new_image).max() < 1e-5, "Models don't give the same forward pass" + except Exception: + error = f"{traceback.format_exc()}" + + results = {"error": error} + out_queue.put(results, timeout=timeout) + out_queue.join() + + +class CustomEncoder(ModelMixin, ConfigMixin): + def __init__(self): + super().__init__() + + +class CustomPipeline(DiffusionPipeline): + def __init__(self, encoder: CustomEncoder, scheduler: DDIMScheduler): + super().__init__() + self.register_modules(encoder=encoder, scheduler=scheduler) + + +class DownloadTests(unittest.TestCase): + def test_one_request_upon_cached(self): + # TODO: For some reason this test fails on MPS where no HEAD call is made. + if torch_device == "mps": + return + + with tempfile.TemporaryDirectory() as tmpdirname: + with requests_mock.mock(real_http=True) as m: + DiffusionPipeline.download("hf-internal-testing/tiny-stable-diffusion-pipe", cache_dir=tmpdirname) + + download_requests = [r.method for r in m.request_history] + assert download_requests.count("HEAD") == 15, "15 calls to files" + assert download_requests.count("GET") == 17, "15 calls to files + model_info + model_index.json" + assert ( + len(download_requests) == 32 + ), "2 calls per file (15 files) + send_telemetry, model_info and model_index.json" + + with requests_mock.mock(real_http=True) as m: + DiffusionPipeline.download( + "hf-internal-testing/tiny-stable-diffusion-pipe", safety_checker=None, cache_dir=tmpdirname + ) + + cache_requests = [r.method for r in m.request_history] + assert cache_requests.count("HEAD") == 1, "model_index.json is only HEAD" + assert cache_requests.count("GET") == 1, "model info is only GET" + assert ( + len(cache_requests) == 2 + ), "We should call only `model_info` to check for _commit hash and `send_telemetry`" + + def test_less_downloads_passed_object(self): + with tempfile.TemporaryDirectory() as tmpdirname: + cached_folder = DiffusionPipeline.download( + "hf-internal-testing/tiny-stable-diffusion-pipe", safety_checker=None, cache_dir=tmpdirname + ) + + # make sure safety checker is not downloaded + assert "safety_checker" not in os.listdir(cached_folder) + + # make sure rest is downloaded + assert "unet" in os.listdir(cached_folder) + assert "tokenizer" in os.listdir(cached_folder) + assert "vae" in os.listdir(cached_folder) + assert "model_index.json" in os.listdir(cached_folder) + assert "scheduler" in os.listdir(cached_folder) + assert "feature_extractor" in os.listdir(cached_folder) + + def test_less_downloads_passed_object_calls(self): + # TODO: For some reason this test fails on MPS where no HEAD call is made. + if torch_device == "mps": + return + + with tempfile.TemporaryDirectory() as tmpdirname: + with requests_mock.mock(real_http=True) as m: + DiffusionPipeline.download( + "hf-internal-testing/tiny-stable-diffusion-pipe", safety_checker=None, cache_dir=tmpdirname + ) + + download_requests = [r.method for r in m.request_history] + # 15 - 2 because no call to config or model file for `safety_checker` + assert download_requests.count("HEAD") == 13, "13 calls to files" + # 17 - 2 because no call to config or model file for `safety_checker` + assert download_requests.count("GET") == 15, "13 calls to files + model_info + model_index.json" + assert ( + len(download_requests) == 28 + ), "2 calls per file (13 files) + send_telemetry, model_info and model_index.json" + + with requests_mock.mock(real_http=True) as m: + DiffusionPipeline.download( + "hf-internal-testing/tiny-stable-diffusion-pipe", safety_checker=None, cache_dir=tmpdirname + ) + + cache_requests = [r.method for r in m.request_history] + assert cache_requests.count("HEAD") == 1, "model_index.json is only HEAD" + assert cache_requests.count("GET") == 1, "model info is only GET" + assert ( + len(cache_requests) == 2 + ), "We should call only `model_info` to check for _commit hash and `send_telemetry`" + + def test_download_only_pytorch(self): + with tempfile.TemporaryDirectory() as tmpdirname: + # pipeline has Flax weights + tmpdirname = DiffusionPipeline.download( + "hf-internal-testing/tiny-stable-diffusion-pipe", safety_checker=None, cache_dir=tmpdirname + ) + + all_root_files = [t[-1] for t in os.walk(os.path.join(tmpdirname))] + files = [item for sublist in all_root_files for item in sublist] + + # None of the downloaded files should be a flax file even if we have some here: + # https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-pipe/blob/main/unet/diffusion_flax_model.msgpack + assert not any(f.endswith(".msgpack") for f in files) + # We need to never convert this tiny model to safetensors for this test to pass + assert not any(f.endswith(".safetensors") for f in files) + + def test_force_safetensors_error(self): + with tempfile.TemporaryDirectory() as tmpdirname: + # pipeline has Flax weights + with self.assertRaises(EnvironmentError): + tmpdirname = DiffusionPipeline.download( + "hf-internal-testing/tiny-stable-diffusion-pipe-no-safetensors", + safety_checker=None, + cache_dir=tmpdirname, + use_safetensors=True, + ) + + def test_download_safetensors(self): + with tempfile.TemporaryDirectory() as tmpdirname: + # pipeline has Flax weights + tmpdirname = DiffusionPipeline.download( + "hf-internal-testing/tiny-stable-diffusion-pipe-safetensors", + safety_checker=None, + cache_dir=tmpdirname, + ) + + all_root_files = [t[-1] for t in os.walk(os.path.join(tmpdirname))] + files = [item for sublist in all_root_files for item in sublist] + + # None of the downloaded files should be a pytorch file even if we have some here: + # https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-pipe/blob/main/unet/diffusion_flax_model.msgpack + assert not any(f.endswith(".bin") for f in files) + + def test_download_safetensors_index(self): + for variant in ["fp16", None]: + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdirname = DiffusionPipeline.download( + "hf-internal-testing/tiny-stable-diffusion-pipe-indexes", + cache_dir=tmpdirname, + use_safetensors=True, + variant=variant, + ) + + all_root_files = [t[-1] for t in os.walk(os.path.join(tmpdirname))] + files = [item for sublist in all_root_files for item in sublist] + + # None of the downloaded files should be a safetensors file even if we have some here: + # https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-pipe-indexes/tree/main/text_encoder + if variant is None: + assert not any("fp16" in f for f in files) + else: + model_files = [f for f in files if "safetensors" in f] + assert all("fp16" in f for f in model_files) + + assert len([f for f in files if ".safetensors" in f]) == 8 + assert not any(".bin" in f for f in files) + + def test_download_bin_index(self): + for variant in ["fp16", None]: + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdirname = DiffusionPipeline.download( + "hf-internal-testing/tiny-stable-diffusion-pipe-indexes", + cache_dir=tmpdirname, + use_safetensors=False, + variant=variant, + ) + + all_root_files = [t[-1] for t in os.walk(os.path.join(tmpdirname))] + files = [item for sublist in all_root_files for item in sublist] + + # None of the downloaded files should be a safetensors file even if we have some here: + # https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-pipe-indexes/tree/main/text_encoder + if variant is None: + assert not any("fp16" in f for f in files) + else: + model_files = [f for f in files if "bin" in f] + assert all("fp16" in f for f in model_files) + + assert len([f for f in files if ".bin" in f]) == 8 + assert not any(".safetensors" in f for f in files) + + def test_download_no_openvino_by_default(self): + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdirname = DiffusionPipeline.download( + "hf-internal-testing/tiny-stable-diffusion-open-vino", + cache_dir=tmpdirname, + ) + + all_root_files = [t[-1] for t in os.walk(os.path.join(tmpdirname))] + files = [item for sublist in all_root_files for item in sublist] + + # make sure that by default no openvino weights are downloaded + assert all((f.endswith(".json") or f.endswith(".bin") or f.endswith(".txt")) for f in files) + assert not any("openvino_" in f for f in files) + + def test_download_no_onnx_by_default(self): + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdirname = DiffusionPipeline.download( + "hf-internal-testing/tiny-stable-diffusion-xl-pipe", + cache_dir=tmpdirname, + use_safetensors=False, + ) + + all_root_files = [t[-1] for t in os.walk(os.path.join(tmpdirname))] + files = [item for sublist in all_root_files for item in sublist] + + # make sure that by default no onnx weights are downloaded for non-ONNX pipelines + assert all((f.endswith(".json") or f.endswith(".bin") or f.endswith(".txt")) for f in files) + assert not any((f.endswith(".onnx") or f.endswith(".pb")) for f in files) + + @require_onnxruntime + def test_download_onnx_by_default_for_onnx_pipelines(self): + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdirname = DiffusionPipeline.download( + "hf-internal-testing/tiny-random-OnnxStableDiffusionPipeline", + cache_dir=tmpdirname, + ) + + all_root_files = [t[-1] for t in os.walk(os.path.join(tmpdirname))] + files = [item for sublist in all_root_files for item in sublist] + + # make sure that by default onnx weights are downloaded for ONNX pipelines + assert any((f.endswith(".json") or f.endswith(".bin") or f.endswith(".txt")) for f in files) + assert any((f.endswith(".onnx")) for f in files) + assert any((f.endswith(".pb")) for f in files) + + def test_download_no_safety_checker(self): + prompt = "hello" + pipe = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None + ) + pipe = pipe.to(torch_device) + generator = torch.manual_seed(0) + out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images + + pipe_2 = StableDiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch") + pipe_2 = pipe_2.to(torch_device) + generator = torch.manual_seed(0) + out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images + + assert np.max(np.abs(out - out_2)) < 1e-3 + + def test_load_no_safety_checker_explicit_locally(self): + prompt = "hello" + pipe = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None + ) + pipe = pipe.to(torch_device) + generator = torch.manual_seed(0) + out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images + + with tempfile.TemporaryDirectory() as tmpdirname: + pipe.save_pretrained(tmpdirname) + pipe_2 = StableDiffusionPipeline.from_pretrained(tmpdirname, safety_checker=None) + pipe_2 = pipe_2.to(torch_device) + + generator = torch.manual_seed(0) + + out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images + + assert np.max(np.abs(out - out_2)) < 1e-3 + + def test_load_no_safety_checker_default_locally(self): + prompt = "hello" + pipe = StableDiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch") + pipe = pipe.to(torch_device) + + generator = torch.manual_seed(0) + out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images + + with tempfile.TemporaryDirectory() as tmpdirname: + pipe.save_pretrained(tmpdirname) + pipe_2 = StableDiffusionPipeline.from_pretrained(tmpdirname) + pipe_2 = pipe_2.to(torch_device) + + generator = torch.manual_seed(0) + + out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images + + assert np.max(np.abs(out - out_2)) < 1e-3 + + def test_cached_files_are_used_when_no_internet(self): + # A mock response for an HTTP head request to emulate server down + response_mock = mock.Mock() + response_mock.status_code = 500 + response_mock.headers = {} + response_mock.raise_for_status.side_effect = HTTPError + response_mock.json.return_value = {} + + # Download this model to make sure it's in the cache. + orig_pipe = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None + ) + orig_comps = {k: v for k, v in orig_pipe.components.items() if hasattr(v, "parameters")} + + # Under the mock environment we get a 500 error when trying to reach the model. + with mock.patch("requests.request", return_value=response_mock): + # Download this model to make sure it's in the cache. + pipe = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None + ) + comps = {k: v for k, v in pipe.components.items() if hasattr(v, "parameters")} + + for m1, m2 in zip(orig_comps.values(), comps.values()): + for p1, p2 in zip(m1.parameters(), m2.parameters()): + if p1.data.ne(p2.data).sum() > 0: + assert False, "Parameters not the same!" + + def test_local_files_only_are_used_when_no_internet(self): + # A mock response for an HTTP head request to emulate server down + response_mock = mock.Mock() + response_mock.status_code = 500 + response_mock.headers = {} + response_mock.raise_for_status.side_effect = HTTPError + response_mock.json.return_value = {} + + # first check that with local files only the pipeline can only be used if cached + with self.assertRaises(FileNotFoundError): + with tempfile.TemporaryDirectory() as tmpdirname: + orig_pipe = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", local_files_only=True, cache_dir=tmpdirname + ) + + # now download + orig_pipe = DiffusionPipeline.download("hf-internal-testing/tiny-stable-diffusion-torch") + + # make sure it can be loaded with local_files_only + orig_pipe = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", local_files_only=True + ) + orig_comps = {k: v for k, v in orig_pipe.components.items() if hasattr(v, "parameters")} + + # Under the mock environment we get a 500 error when trying to connect to the internet. + # Make sure it works local_files_only only works here! + with mock.patch("requests.request", return_value=response_mock): + # Download this model to make sure it's in the cache. + pipe = DiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch") + comps = {k: v for k, v in pipe.components.items() if hasattr(v, "parameters")} + + for m1, m2 in zip(orig_comps.values(), comps.values()): + for p1, p2 in zip(m1.parameters(), m2.parameters()): + if p1.data.ne(p2.data).sum() > 0: + assert False, "Parameters not the same!" + + def test_download_from_variant_folder(self): + for use_safetensors in [False, True]: + other_format = ".bin" if use_safetensors else ".safetensors" + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdirname = StableDiffusionPipeline.download( + "hf-internal-testing/stable-diffusion-all-variants", + cache_dir=tmpdirname, + use_safetensors=use_safetensors, + ) + all_root_files = [t[-1] for t in os.walk(tmpdirname)] + files = [item for sublist in all_root_files for item in sublist] + + # None of the downloaded files should be a variant file even if we have some here: + # https://huggingface.co/hf-internal-testing/stable-diffusion-all-variants/tree/main/unet + assert len(files) == 15, f"We should only download 15 files, not {len(files)}" + assert not any(f.endswith(other_format) for f in files) + # no variants + assert not any(len(f.split(".")) == 3 for f in files) + + def test_download_variant_all(self): + for use_safetensors in [False, True]: + other_format = ".bin" if use_safetensors else ".safetensors" + this_format = ".safetensors" if use_safetensors else ".bin" + variant = "fp16" + + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdirname = StableDiffusionPipeline.download( + "hf-internal-testing/stable-diffusion-all-variants", + cache_dir=tmpdirname, + variant=variant, + use_safetensors=use_safetensors, + ) + all_root_files = [t[-1] for t in os.walk(tmpdirname)] + files = [item for sublist in all_root_files for item in sublist] + + # None of the downloaded files should be a non-variant file even if we have some here: + # https://huggingface.co/hf-internal-testing/stable-diffusion-all-variants/tree/main/unet + assert len(files) == 15, f"We should only download 15 files, not {len(files)}" + # unet, vae, text_encoder, safety_checker + assert len([f for f in files if f.endswith(f"{variant}{this_format}")]) == 4 + # all checkpoints should have variant ending + assert not any(f.endswith(this_format) and not f.endswith(f"{variant}{this_format}") for f in files) + assert not any(f.endswith(other_format) for f in files) + + def test_download_variant_partly(self): + for use_safetensors in [False, True]: + other_format = ".bin" if use_safetensors else ".safetensors" + this_format = ".safetensors" if use_safetensors else ".bin" + variant = "no_ema" + + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdirname = StableDiffusionPipeline.download( + "hf-internal-testing/stable-diffusion-all-variants", + cache_dir=tmpdirname, + variant=variant, + use_safetensors=use_safetensors, + ) + all_root_files = [t[-1] for t in os.walk(tmpdirname)] + files = [item for sublist in all_root_files for item in sublist] + + unet_files = os.listdir(os.path.join(tmpdirname, "unet")) + + # Some of the downloaded files should be a non-variant file, check: + # https://huggingface.co/hf-internal-testing/stable-diffusion-all-variants/tree/main/unet + assert len(files) == 15, f"We should only download 15 files, not {len(files)}" + # only unet has "no_ema" variant + assert f"diffusion_pytorch_model.{variant}{this_format}" in unet_files + assert len([f for f in files if f.endswith(f"{variant}{this_format}")]) == 1 + # vae, safety_checker and text_encoder should have no variant + assert sum(f.endswith(this_format) and not f.endswith(f"{variant}{this_format}") for f in files) == 3 + assert not any(f.endswith(other_format) for f in files) + + def test_download_broken_variant(self): + for use_safetensors in [False, True]: + # text encoder is missing no variant and "no_ema" variant weights, so the following can't work + for variant in [None, "no_ema"]: + with self.assertRaises(OSError) as error_context: + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdirname = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/stable-diffusion-broken-variants", + cache_dir=tmpdirname, + variant=variant, + use_safetensors=use_safetensors, + ) + + assert "Error no file name" in str(error_context.exception) + + # text encoder has fp16 variants so we can load it + with tempfile.TemporaryDirectory() as tmpdirname: + tmpdirname = StableDiffusionPipeline.download( + "hf-internal-testing/stable-diffusion-broken-variants", + use_safetensors=use_safetensors, + cache_dir=tmpdirname, + variant="fp16", + ) + + all_root_files = [t[-1] for t in os.walk(tmpdirname)] + files = [item for sublist in all_root_files for item in sublist] + + # None of the downloaded files should be a non-variant file even if we have some here: + # https://huggingface.co/hf-internal-testing/stable-diffusion-broken-variants/tree/main/unet + assert len(files) == 15, f"We should only download 15 files, not {len(files)}" + # only unet has "no_ema" variant + + def test_local_save_load_index(self): + prompt = "hello" + for variant in [None, "fp16"]: + for use_safe in [True, False]: + pipe = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-pipe-indexes", + variant=variant, + use_safetensors=use_safe, + safety_checker=None, + ) + pipe = pipe.to(torch_device) + generator = torch.manual_seed(0) + out = pipe(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images + + with tempfile.TemporaryDirectory() as tmpdirname: + pipe.save_pretrained(tmpdirname) + pipe_2 = StableDiffusionPipeline.from_pretrained( + tmpdirname, safe_serialization=use_safe, variant=variant + ) + pipe_2 = pipe_2.to(torch_device) + + generator = torch.manual_seed(0) + + out_2 = pipe_2(prompt, num_inference_steps=2, generator=generator, output_type="numpy").images + + assert np.max(np.abs(out - out_2)) < 1e-3 + + def test_text_inversion_download(self): + pipe = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None + ) + pipe = pipe.to(torch_device) + + num_tokens = len(pipe.tokenizer) + + # single token load local + with tempfile.TemporaryDirectory() as tmpdirname: + ten = {"<*>": torch.ones((32,))} + torch.save(ten, os.path.join(tmpdirname, "learned_embeds.bin")) + + pipe.load_textual_inversion(tmpdirname) + + token = pipe.tokenizer.convert_tokens_to_ids("<*>") + assert token == num_tokens, "Added token must be at spot `num_tokens`" + assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 32 + assert pipe._maybe_convert_prompt("<*>", pipe.tokenizer) == "<*>" + + prompt = "hey <*>" + out = pipe(prompt, num_inference_steps=1, output_type="numpy").images + assert out.shape == (1, 128, 128, 3) + + # single token load local with weight name + with tempfile.TemporaryDirectory() as tmpdirname: + ten = {"<**>": 2 * torch.ones((1, 32))} + torch.save(ten, os.path.join(tmpdirname, "learned_embeds.bin")) + + pipe.load_textual_inversion(tmpdirname, weight_name="learned_embeds.bin") + + token = pipe.tokenizer.convert_tokens_to_ids("<**>") + assert token == num_tokens + 1, "Added token must be at spot `num_tokens`" + assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 64 + assert pipe._maybe_convert_prompt("<**>", pipe.tokenizer) == "<**>" + + prompt = "hey <**>" + out = pipe(prompt, num_inference_steps=1, output_type="numpy").images + assert out.shape == (1, 128, 128, 3) + + # multi token load + with tempfile.TemporaryDirectory() as tmpdirname: + ten = {"<***>": torch.cat([3 * torch.ones((1, 32)), 4 * torch.ones((1, 32)), 5 * torch.ones((1, 32))])} + torch.save(ten, os.path.join(tmpdirname, "learned_embeds.bin")) + + pipe.load_textual_inversion(tmpdirname) + + token = pipe.tokenizer.convert_tokens_to_ids("<***>") + token_1 = pipe.tokenizer.convert_tokens_to_ids("<***>_1") + token_2 = pipe.tokenizer.convert_tokens_to_ids("<***>_2") + + assert token == num_tokens + 2, "Added token must be at spot `num_tokens`" + assert token_1 == num_tokens + 3, "Added token must be at spot `num_tokens`" + assert token_2 == num_tokens + 4, "Added token must be at spot `num_tokens`" + assert pipe.text_encoder.get_input_embeddings().weight[-3].sum().item() == 96 + assert pipe.text_encoder.get_input_embeddings().weight[-2].sum().item() == 128 + assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 160 + assert pipe._maybe_convert_prompt("<***>", pipe.tokenizer) == "<***> <***>_1 <***>_2" + + prompt = "hey <***>" + out = pipe(prompt, num_inference_steps=1, output_type="numpy").images + assert out.shape == (1, 128, 128, 3) + + # multi token load a1111 + with tempfile.TemporaryDirectory() as tmpdirname: + ten = { + "string_to_param": { + "*": torch.cat([3 * torch.ones((1, 32)), 4 * torch.ones((1, 32)), 5 * torch.ones((1, 32))]) + }, + "name": "<****>", + } + torch.save(ten, os.path.join(tmpdirname, "a1111.bin")) + + pipe.load_textual_inversion(tmpdirname, weight_name="a1111.bin") + + token = pipe.tokenizer.convert_tokens_to_ids("<****>") + token_1 = pipe.tokenizer.convert_tokens_to_ids("<****>_1") + token_2 = pipe.tokenizer.convert_tokens_to_ids("<****>_2") + + assert token == num_tokens + 5, "Added token must be at spot `num_tokens`" + assert token_1 == num_tokens + 6, "Added token must be at spot `num_tokens`" + assert token_2 == num_tokens + 7, "Added token must be at spot `num_tokens`" + assert pipe.text_encoder.get_input_embeddings().weight[-3].sum().item() == 96 + assert pipe.text_encoder.get_input_embeddings().weight[-2].sum().item() == 128 + assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 160 + assert pipe._maybe_convert_prompt("<****>", pipe.tokenizer) == "<****> <****>_1 <****>_2" + + prompt = "hey <****>" + out = pipe(prompt, num_inference_steps=1, output_type="numpy").images + assert out.shape == (1, 128, 128, 3) + + # multi embedding load + with tempfile.TemporaryDirectory() as tmpdirname1: + with tempfile.TemporaryDirectory() as tmpdirname2: + ten = {"<*****>": torch.ones((32,))} + torch.save(ten, os.path.join(tmpdirname1, "learned_embeds.bin")) + + ten = {"<******>": 2 * torch.ones((1, 32))} + torch.save(ten, os.path.join(tmpdirname2, "learned_embeds.bin")) + + pipe.load_textual_inversion([tmpdirname1, tmpdirname2]) + + token = pipe.tokenizer.convert_tokens_to_ids("<*****>") + assert token == num_tokens + 8, "Added token must be at spot `num_tokens`" + assert pipe.text_encoder.get_input_embeddings().weight[-2].sum().item() == 32 + assert pipe._maybe_convert_prompt("<*****>", pipe.tokenizer) == "<*****>" + + token = pipe.tokenizer.convert_tokens_to_ids("<******>") + assert token == num_tokens + 9, "Added token must be at spot `num_tokens`" + assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 64 + assert pipe._maybe_convert_prompt("<******>", pipe.tokenizer) == "<******>" + + prompt = "hey <*****> <******>" + out = pipe(prompt, num_inference_steps=1, output_type="numpy").images + assert out.shape == (1, 128, 128, 3) + + # single token state dict load + ten = {"": torch.ones((32,))} + pipe.load_textual_inversion(ten) + + token = pipe.tokenizer.convert_tokens_to_ids("") + assert token == num_tokens + 10, "Added token must be at spot `num_tokens`" + assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 32 + assert pipe._maybe_convert_prompt("", pipe.tokenizer) == "" + + prompt = "hey " + out = pipe(prompt, num_inference_steps=1, output_type="numpy").images + assert out.shape == (1, 128, 128, 3) + + # multi embedding state dict load + ten1 = {"": torch.ones((32,))} + ten2 = {"": 2 * torch.ones((1, 32))} + + pipe.load_textual_inversion([ten1, ten2]) + + token = pipe.tokenizer.convert_tokens_to_ids("") + assert token == num_tokens + 11, "Added token must be at spot `num_tokens`" + assert pipe.text_encoder.get_input_embeddings().weight[-2].sum().item() == 32 + assert pipe._maybe_convert_prompt("", pipe.tokenizer) == "" + + token = pipe.tokenizer.convert_tokens_to_ids("") + assert token == num_tokens + 12, "Added token must be at spot `num_tokens`" + assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 64 + assert pipe._maybe_convert_prompt("", pipe.tokenizer) == "" + + prompt = "hey " + out = pipe(prompt, num_inference_steps=1, output_type="numpy").images + assert out.shape == (1, 128, 128, 3) + + # auto1111 multi-token state dict load + ten = { + "string_to_param": { + "*": torch.cat([3 * torch.ones((1, 32)), 4 * torch.ones((1, 32)), 5 * torch.ones((1, 32))]) + }, + "name": "", + } + + pipe.load_textual_inversion(ten) + + token = pipe.tokenizer.convert_tokens_to_ids("") + token_1 = pipe.tokenizer.convert_tokens_to_ids("_1") + token_2 = pipe.tokenizer.convert_tokens_to_ids("_2") + + assert token == num_tokens + 13, "Added token must be at spot `num_tokens`" + assert token_1 == num_tokens + 14, "Added token must be at spot `num_tokens`" + assert token_2 == num_tokens + 15, "Added token must be at spot `num_tokens`" + assert pipe.text_encoder.get_input_embeddings().weight[-3].sum().item() == 96 + assert pipe.text_encoder.get_input_embeddings().weight[-2].sum().item() == 128 + assert pipe.text_encoder.get_input_embeddings().weight[-1].sum().item() == 160 + assert pipe._maybe_convert_prompt("", pipe.tokenizer) == " _1 _2" + + prompt = "hey " + out = pipe(prompt, num_inference_steps=1, output_type="numpy").images + assert out.shape == (1, 128, 128, 3) + + # multiple references to multi embedding + ten = {"": torch.ones(3, 32)} + pipe.load_textual_inversion(ten) + + assert ( + pipe._maybe_convert_prompt(" ", pipe.tokenizer) == " _1 _2 _1 _2" + ) + + prompt = "hey " + out = pipe(prompt, num_inference_steps=1, output_type="numpy").images + assert out.shape == (1, 128, 128, 3) + + def test_text_inversion_multi_tokens(self): + pipe1 = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None + ) + pipe1 = pipe1.to(torch_device) + + token1, token2 = "<*>", "<**>" + ten1 = torch.ones((32,)) + ten2 = torch.ones((32,)) * 2 + + num_tokens = len(pipe1.tokenizer) + + pipe1.load_textual_inversion(ten1, token=token1) + pipe1.load_textual_inversion(ten2, token=token2) + emb1 = pipe1.text_encoder.get_input_embeddings().weight + + pipe2 = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None + ) + pipe2 = pipe2.to(torch_device) + pipe2.load_textual_inversion([ten1, ten2], token=[token1, token2]) + emb2 = pipe2.text_encoder.get_input_embeddings().weight + + pipe3 = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None + ) + pipe3 = pipe3.to(torch_device) + pipe3.load_textual_inversion(torch.stack([ten1, ten2], dim=0), token=[token1, token2]) + emb3 = pipe3.text_encoder.get_input_embeddings().weight + + assert len(pipe1.tokenizer) == len(pipe2.tokenizer) == len(pipe3.tokenizer) == num_tokens + 2 + assert ( + pipe1.tokenizer.convert_tokens_to_ids(token1) + == pipe2.tokenizer.convert_tokens_to_ids(token1) + == pipe3.tokenizer.convert_tokens_to_ids(token1) + == num_tokens + ) + assert ( + pipe1.tokenizer.convert_tokens_to_ids(token2) + == pipe2.tokenizer.convert_tokens_to_ids(token2) + == pipe3.tokenizer.convert_tokens_to_ids(token2) + == num_tokens + 1 + ) + assert emb1[num_tokens].sum().item() == emb2[num_tokens].sum().item() == emb3[num_tokens].sum().item() + assert ( + emb1[num_tokens + 1].sum().item() == emb2[num_tokens + 1].sum().item() == emb3[num_tokens + 1].sum().item() + ) + + def test_download_ignore_files(self): + # Check https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-pipe-ignore-files/blob/72f58636e5508a218c6b3f60550dc96445547817/model_index.json#L4 + with tempfile.TemporaryDirectory() as tmpdirname: + # pipeline has Flax weights + tmpdirname = DiffusionPipeline.download("hf-internal-testing/tiny-stable-diffusion-pipe-ignore-files") + all_root_files = [t[-1] for t in os.walk(os.path.join(tmpdirname))] + files = [item for sublist in all_root_files for item in sublist] + + # None of the downloaded files should be a pytorch file even if we have some here: + # https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-pipe/blob/main/unet/diffusion_flax_model.msgpack + assert not any(f in ["vae/diffusion_pytorch_model.bin", "text_encoder/config.json"] for f in files) + assert len(files) == 14 + + def test_get_pipeline_class_from_flax(self): + flax_config = {"_class_name": "FlaxStableDiffusionPipeline"} + config = {"_class_name": "StableDiffusionPipeline"} + + # when loading a PyTorch Pipeline from a FlaxPipeline `model_index.json`, e.g.: https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-lms-pipe/blob/7a9063578b325779f0f1967874a6771caa973cad/model_index.json#L2 + # we need to make sure that we don't load the Flax Pipeline class, but instead the PyTorch pipeline class + assert _get_pipeline_class(DiffusionPipeline, flax_config) == _get_pipeline_class(DiffusionPipeline, config) + + +class CustomPipelineTests(unittest.TestCase): + def test_load_custom_pipeline(self): + pipeline = DiffusionPipeline.from_pretrained( + "google/ddpm-cifar10-32", custom_pipeline="hf-internal-testing/diffusers-dummy-pipeline" + ) + pipeline = pipeline.to(torch_device) + # NOTE that `"CustomPipeline"` is not a class that is defined in this library, but solely on the Hub + # under https://huggingface.co/hf-internal-testing/diffusers-dummy-pipeline/blob/main/pipeline.py#L24 + assert pipeline.__class__.__name__ == "CustomPipeline" + + def test_load_custom_github(self): + pipeline = DiffusionPipeline.from_pretrained( + "google/ddpm-cifar10-32", custom_pipeline="one_step_unet", custom_revision="main" + ) + + # make sure that on "main" pipeline gives only ones because of: https://github.com/huggingface/diffusers/pull/1690 + with torch.no_grad(): + output = pipeline() + + assert output.numel() == output.sum() + + # hack since Python doesn't like overwriting modules: https://stackoverflow.com/questions/3105801/unload-a-module-in-python + # Could in the future work with hashes instead. + del sys.modules["diffusers_modules.git.one_step_unet"] + + pipeline = DiffusionPipeline.from_pretrained( + "google/ddpm-cifar10-32", custom_pipeline="one_step_unet", custom_revision="0.10.2" + ) + with torch.no_grad(): + output = pipeline() + + assert output.numel() != output.sum() + + assert pipeline.__class__.__name__ == "UnetSchedulerOneForwardPipeline" + + def test_run_custom_pipeline(self): + pipeline = DiffusionPipeline.from_pretrained( + "google/ddpm-cifar10-32", custom_pipeline="hf-internal-testing/diffusers-dummy-pipeline" + ) + pipeline = pipeline.to(torch_device) + images, output_str = pipeline(num_inference_steps=2, output_type="np") + + assert images[0].shape == (1, 32, 32, 3) + + # compare output to https://huggingface.co/hf-internal-testing/diffusers-dummy-pipeline/blob/main/pipeline.py#L102 + assert output_str == "This is a test" + + def test_remote_components(self): + # make sure that trust remote code has to be passed + with self.assertRaises(ValueError): + pipeline = DiffusionPipeline.from_pretrained("hf-internal-testing/tiny-sdxl-custom-components") + + # Check that only loading custom componets "my_unet", "my_scheduler" works + pipeline = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-sdxl-custom-components", trust_remote_code=True + ) + + assert pipeline.config.unet == ("diffusers_modules.local.my_unet_model", "MyUNetModel") + assert pipeline.config.scheduler == ("diffusers_modules.local.my_scheduler", "MyScheduler") + assert pipeline.__class__.__name__ == "StableDiffusionXLPipeline" + + pipeline = pipeline.to(torch_device) + images = pipeline("test", num_inference_steps=2, output_type="np")[0] + + assert images.shape == (1, 64, 64, 3) + + # Check that only loading custom componets "my_unet", "my_scheduler" and explicit custom pipeline works + pipeline = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-sdxl-custom-components", custom_pipeline="my_pipeline", trust_remote_code=True + ) + + assert pipeline.config.unet == ("diffusers_modules.local.my_unet_model", "MyUNetModel") + assert pipeline.config.scheduler == ("diffusers_modules.local.my_scheduler", "MyScheduler") + assert pipeline.__class__.__name__ == "MyPipeline" + + pipeline = pipeline.to(torch_device) + images = pipeline("test", num_inference_steps=2, output_type="np")[0] + + assert images.shape == (1, 64, 64, 3) + + def test_remote_auto_custom_pipe(self): + # make sure that trust remote code has to be passed + with self.assertRaises(ValueError): + pipeline = DiffusionPipeline.from_pretrained("hf-internal-testing/tiny-sdxl-custom-all") + + # Check that only loading custom componets "my_unet", "my_scheduler" and auto custom pipeline works + pipeline = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-sdxl-custom-all", trust_remote_code=True + ) + + assert pipeline.config.unet == ("diffusers_modules.local.my_unet_model", "MyUNetModel") + assert pipeline.config.scheduler == ("diffusers_modules.local.my_scheduler", "MyScheduler") + assert pipeline.__class__.__name__ == "MyPipeline" + + pipeline = pipeline.to(torch_device) + images = pipeline("test", num_inference_steps=2, output_type="np")[0] + + assert images.shape == (1, 64, 64, 3) + + def test_local_custom_pipeline_repo(self): + local_custom_pipeline_path = get_tests_dir("fixtures/custom_pipeline") + pipeline = DiffusionPipeline.from_pretrained( + "google/ddpm-cifar10-32", custom_pipeline=local_custom_pipeline_path + ) + pipeline = pipeline.to(torch_device) + images, output_str = pipeline(num_inference_steps=2, output_type="np") + + assert pipeline.__class__.__name__ == "CustomLocalPipeline" + assert images[0].shape == (1, 32, 32, 3) + # compare to https://github.com/huggingface/diffusers/blob/main/tests/fixtures/custom_pipeline/pipeline.py#L102 + assert output_str == "This is a local test" + + def test_local_custom_pipeline_file(self): + local_custom_pipeline_path = get_tests_dir("fixtures/custom_pipeline") + local_custom_pipeline_path = os.path.join(local_custom_pipeline_path, "what_ever.py") + pipeline = DiffusionPipeline.from_pretrained( + "google/ddpm-cifar10-32", custom_pipeline=local_custom_pipeline_path + ) + pipeline = pipeline.to(torch_device) + images, output_str = pipeline(num_inference_steps=2, output_type="np") + + assert pipeline.__class__.__name__ == "CustomLocalPipeline" + assert images[0].shape == (1, 32, 32, 3) + # compare to https://github.com/huggingface/diffusers/blob/main/tests/fixtures/custom_pipeline/pipeline.py#L102 + assert output_str == "This is a local test" + + def test_custom_model_and_pipeline(self): + pipe = CustomPipeline( + encoder=CustomEncoder(), + scheduler=DDIMScheduler(), + ) + + with tempfile.TemporaryDirectory() as tmpdirname: + pipe.save_pretrained(tmpdirname, safe_serialization=False) + + pipe_new = CustomPipeline.from_pretrained(tmpdirname) + pipe_new.save_pretrained(tmpdirname) + + conf_1 = dict(pipe.config) + conf_2 = dict(pipe_new.config) + + del conf_2["_name_or_path"] + + assert conf_1 == conf_2 + + @slow + @require_torch_gpu + def test_download_from_git(self): + # Because adaptive_avg_pool2d_backward_cuda + # does not have a deterministic implementation. + clip_model_id = "laion/CLIP-ViT-B-32-laion2B-s34B-b79K" + + feature_extractor = CLIPImageProcessor.from_pretrained(clip_model_id) + clip_model = CLIPModel.from_pretrained(clip_model_id, torch_dtype=torch.float16) + + pipeline = DiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", + custom_pipeline="clip_guided_stable_diffusion", + clip_model=clip_model, + feature_extractor=feature_extractor, + torch_dtype=torch.float16, + ) + pipeline.enable_attention_slicing() + pipeline = pipeline.to(torch_device) + + # NOTE that `"CLIPGuidedStableDiffusion"` is not a class that is defined in the pypi package of th e library, but solely on the community examples folder of GitHub under: + # https://github.com/huggingface/diffusers/blob/main/examples/community/clip_guided_stable_diffusion.py + assert pipeline.__class__.__name__ == "CLIPGuidedStableDiffusion" + + image = pipeline("a prompt", num_inference_steps=2, output_type="np").images[0] + assert image.shape == (512, 512, 3) + + def test_save_pipeline_change_config(self): + pipe = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None + ) + + with tempfile.TemporaryDirectory() as tmpdirname: + pipe.save_pretrained(tmpdirname) + pipe = DiffusionPipeline.from_pretrained(tmpdirname) + + assert pipe.scheduler.__class__.__name__ == "PNDMScheduler" + + # let's make sure that changing the scheduler is correctly reflected + with tempfile.TemporaryDirectory() as tmpdirname: + pipe.scheduler = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config) + pipe.save_pretrained(tmpdirname) + pipe = DiffusionPipeline.from_pretrained(tmpdirname) + + assert pipe.scheduler.__class__.__name__ == "DPMSolverMultistepScheduler" + + +class PipelineFastTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def dummy_image(self): + batch_size = 1 + num_channels = 3 + sizes = (32, 32) + + image = floats_tensor((batch_size, num_channels) + sizes, rng=random.Random(0)).to(torch_device) + return image + + def dummy_uncond_unet(self, sample_size=32): + torch.manual_seed(0) + model = UNet2DModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=sample_size, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "AttnDownBlock2D"), + up_block_types=("AttnUpBlock2D", "UpBlock2D"), + ) + return model + + def dummy_cond_unet(self, sample_size=32): + torch.manual_seed(0) + model = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=sample_size, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + return model + + @property + def dummy_vae(self): + torch.manual_seed(0) + model = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + return model + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModel(config) + + @property + def dummy_extractor(self): + def extract(*args, **kwargs): + class Out: + def __init__(self): + self.pixel_values = torch.ones([0]) + + def to(self, device): + self.pixel_values.to(device) + return self + + return Out() + + return extract + + @parameterized.expand( + [ + [DDIMScheduler, DDIMPipeline, 32], + [DDPMScheduler, DDPMPipeline, 32], + [DDIMScheduler, DDIMPipeline, (32, 64)], + [DDPMScheduler, DDPMPipeline, (64, 32)], + ] + ) + def test_uncond_unet_components(self, scheduler_fn=DDPMScheduler, pipeline_fn=DDPMPipeline, sample_size=32): + unet = self.dummy_uncond_unet(sample_size) + scheduler = scheduler_fn() + pipeline = pipeline_fn(unet, scheduler).to(torch_device) + + generator = torch.manual_seed(0) + out_image = pipeline( + generator=generator, + num_inference_steps=2, + output_type="np", + ).images + sample_size = (sample_size, sample_size) if isinstance(sample_size, int) else sample_size + assert out_image.shape == (1, *sample_size, 3) + + def test_stable_diffusion_components(self): + """Test that components property works correctly""" + unet = self.dummy_cond_unet() + scheduler = PNDMScheduler(skip_prk_steps=True) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + image = self.dummy_image().cpu().permute(0, 2, 3, 1)[0] + init_image = Image.fromarray(np.uint8(image)).convert("RGB") + mask_image = Image.fromarray(np.uint8(image + 4)).convert("RGB").resize((32, 32)) + + # make sure here that pndm scheduler skips prk + inpaint = StableDiffusionInpaintPipelineLegacy( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ).to(torch_device) + img2img = StableDiffusionImg2ImgPipeline(**inpaint.components, image_encoder=None).to(torch_device) + text2img = StableDiffusionPipeline(**inpaint.components, image_encoder=None).to(torch_device) + + prompt = "A painting of a squirrel eating a burger" + + generator = torch.manual_seed(0) + image_inpaint = inpaint( + [prompt], + generator=generator, + num_inference_steps=2, + output_type="np", + image=init_image, + mask_image=mask_image, + ).images + image_img2img = img2img( + [prompt], + generator=generator, + num_inference_steps=2, + output_type="np", + image=init_image, + ).images + image_text2img = text2img( + [prompt], + generator=generator, + num_inference_steps=2, + output_type="np", + ).images + + assert image_inpaint.shape == (1, 32, 32, 3) + assert image_img2img.shape == (1, 32, 32, 3) + assert image_text2img.shape == (1, 64, 64, 3) + + @require_torch_gpu + def test_pipe_false_offload_warn(self): + unet = self.dummy_cond_unet() + scheduler = PNDMScheduler(skip_prk_steps=True) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + sd = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + + sd.enable_model_cpu_offload() + + logger = logging.get_logger("diffusers.pipelines.pipeline_utils") + with CaptureLogger(logger) as cap_logger: + sd.to("cuda") + + assert "It is strongly recommended against doing so" in str(cap_logger) + + sd = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + + def test_set_scheduler(self): + unet = self.dummy_cond_unet() + scheduler = PNDMScheduler(skip_prk_steps=True) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + sd = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + + sd.scheduler = DDIMScheduler.from_config(sd.scheduler.config) + assert isinstance(sd.scheduler, DDIMScheduler) + sd.scheduler = DDPMScheduler.from_config(sd.scheduler.config) + assert isinstance(sd.scheduler, DDPMScheduler) + sd.scheduler = PNDMScheduler.from_config(sd.scheduler.config) + assert isinstance(sd.scheduler, PNDMScheduler) + sd.scheduler = LMSDiscreteScheduler.from_config(sd.scheduler.config) + assert isinstance(sd.scheduler, LMSDiscreteScheduler) + sd.scheduler = EulerDiscreteScheduler.from_config(sd.scheduler.config) + assert isinstance(sd.scheduler, EulerDiscreteScheduler) + sd.scheduler = EulerAncestralDiscreteScheduler.from_config(sd.scheduler.config) + assert isinstance(sd.scheduler, EulerAncestralDiscreteScheduler) + sd.scheduler = DPMSolverMultistepScheduler.from_config(sd.scheduler.config) + assert isinstance(sd.scheduler, DPMSolverMultistepScheduler) + + def test_set_component_to_none(self): + unet = self.dummy_cond_unet() + scheduler = PNDMScheduler(skip_prk_steps=True) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + pipeline = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + + generator = torch.Generator(device="cpu").manual_seed(0) + + prompt = "This is a flower" + + out_image = pipeline( + prompt=prompt, + generator=generator, + num_inference_steps=1, + output_type="np", + ).images + + pipeline.feature_extractor = None + generator = torch.Generator(device="cpu").manual_seed(0) + out_image_2 = pipeline( + prompt=prompt, + generator=generator, + num_inference_steps=1, + output_type="np", + ).images + + assert out_image.shape == (1, 64, 64, 3) + assert np.abs(out_image - out_image_2).max() < 1e-3 + + def test_optional_components_is_none(self): + unet = self.dummy_cond_unet() + scheduler = PNDMScheduler(skip_prk_steps=True) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + items = { + "feature_extractor": self.dummy_extractor, + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": bert, + "tokenizer": tokenizer, + "safety_checker": None, + # we don't add an image encoder + } + + pipeline = StableDiffusionPipeline(**items) + + assert sorted(pipeline.components.keys()) == sorted(["image_encoder"] + list(items.keys())) + assert pipeline.image_encoder is None + + def test_set_scheduler_consistency(self): + unet = self.dummy_cond_unet() + pndm = PNDMScheduler.from_config("hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler") + ddim = DDIMScheduler.from_config("hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler") + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + sd = StableDiffusionPipeline( + unet=unet, + scheduler=pndm, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + + pndm_config = sd.scheduler.config + sd.scheduler = DDPMScheduler.from_config(pndm_config) + sd.scheduler = PNDMScheduler.from_config(sd.scheduler.config) + pndm_config_2 = sd.scheduler.config + pndm_config_2 = {k: v for k, v in pndm_config_2.items() if k in pndm_config} + + assert dict(pndm_config) == dict(pndm_config_2) + + sd = StableDiffusionPipeline( + unet=unet, + scheduler=ddim, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + + ddim_config = sd.scheduler.config + sd.scheduler = LMSDiscreteScheduler.from_config(ddim_config) + sd.scheduler = DDIMScheduler.from_config(sd.scheduler.config) + ddim_config_2 = sd.scheduler.config + ddim_config_2 = {k: v for k, v in ddim_config_2.items() if k in ddim_config} + + assert dict(ddim_config) == dict(ddim_config_2) + + def test_save_safe_serialization(self): + pipeline = StableDiffusionPipeline.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch") + with tempfile.TemporaryDirectory() as tmpdirname: + pipeline.save_pretrained(tmpdirname, safe_serialization=True) + + # Validate that the VAE safetensor exists and are of the correct format + vae_path = os.path.join(tmpdirname, "vae", "diffusion_pytorch_model.safetensors") + assert os.path.exists(vae_path), f"Could not find {vae_path}" + _ = safetensors.torch.load_file(vae_path) + + # Validate that the UNet safetensor exists and are of the correct format + unet_path = os.path.join(tmpdirname, "unet", "diffusion_pytorch_model.safetensors") + assert os.path.exists(unet_path), f"Could not find {unet_path}" + _ = safetensors.torch.load_file(unet_path) + + # Validate that the text encoder safetensor exists and are of the correct format + text_encoder_path = os.path.join(tmpdirname, "text_encoder", "model.safetensors") + assert os.path.exists(text_encoder_path), f"Could not find {text_encoder_path}" + _ = safetensors.torch.load_file(text_encoder_path) + + pipeline = StableDiffusionPipeline.from_pretrained(tmpdirname) + assert pipeline.unet is not None + assert pipeline.vae is not None + assert pipeline.text_encoder is not None + assert pipeline.scheduler is not None + assert pipeline.feature_extractor is not None + + def test_no_pytorch_download_when_doing_safetensors(self): + # by default we don't download + with tempfile.TemporaryDirectory() as tmpdirname: + _ = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/diffusers-stable-diffusion-tiny-all", cache_dir=tmpdirname + ) + + path = os.path.join( + tmpdirname, + "models--hf-internal-testing--diffusers-stable-diffusion-tiny-all", + "snapshots", + "07838d72e12f9bcec1375b0482b80c1d399be843", + "unet", + ) + # safetensors exists + assert os.path.exists(os.path.join(path, "diffusion_pytorch_model.safetensors")) + # pytorch does not + assert not os.path.exists(os.path.join(path, "diffusion_pytorch_model.bin")) + + def test_no_safetensors_download_when_doing_pytorch(self): + use_safetensors = False + + with tempfile.TemporaryDirectory() as tmpdirname: + _ = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/diffusers-stable-diffusion-tiny-all", + cache_dir=tmpdirname, + use_safetensors=use_safetensors, + ) + + path = os.path.join( + tmpdirname, + "models--hf-internal-testing--diffusers-stable-diffusion-tiny-all", + "snapshots", + "07838d72e12f9bcec1375b0482b80c1d399be843", + "unet", + ) + # safetensors does not exists + assert not os.path.exists(os.path.join(path, "diffusion_pytorch_model.safetensors")) + # pytorch does + assert os.path.exists(os.path.join(path, "diffusion_pytorch_model.bin")) + + def test_optional_components(self): + unet = self.dummy_cond_unet() + pndm = PNDMScheduler.from_config("hf-internal-testing/tiny-stable-diffusion-torch", subfolder="scheduler") + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + orig_sd = StableDiffusionPipeline( + unet=unet, + scheduler=pndm, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=unet, + feature_extractor=self.dummy_extractor, + ) + sd = orig_sd + + assert sd.config.requires_safety_checker is True + + with tempfile.TemporaryDirectory() as tmpdirname: + sd.save_pretrained(tmpdirname) + + # Test that passing None works + sd = StableDiffusionPipeline.from_pretrained( + tmpdirname, feature_extractor=None, safety_checker=None, requires_safety_checker=False + ) + + assert sd.config.requires_safety_checker is False + assert sd.config.safety_checker == (None, None) + assert sd.config.feature_extractor == (None, None) + + with tempfile.TemporaryDirectory() as tmpdirname: + sd.save_pretrained(tmpdirname) + + # Test that loading previous None works + sd = StableDiffusionPipeline.from_pretrained(tmpdirname) + + assert sd.config.requires_safety_checker is False + assert sd.config.safety_checker == (None, None) + assert sd.config.feature_extractor == (None, None) + + orig_sd.save_pretrained(tmpdirname) + + # Test that loading without any directory works + shutil.rmtree(os.path.join(tmpdirname, "safety_checker")) + with open(os.path.join(tmpdirname, sd.config_name)) as f: + config = json.load(f) + config["safety_checker"] = [None, None] + with open(os.path.join(tmpdirname, sd.config_name), "w") as f: + json.dump(config, f) + + sd = StableDiffusionPipeline.from_pretrained(tmpdirname, requires_safety_checker=False) + sd.save_pretrained(tmpdirname) + sd = StableDiffusionPipeline.from_pretrained(tmpdirname) + + assert sd.config.requires_safety_checker is False + assert sd.config.safety_checker == (None, None) + assert sd.config.feature_extractor == (None, None) + + # Test that loading from deleted model index works + with open(os.path.join(tmpdirname, sd.config_name)) as f: + config = json.load(f) + del config["safety_checker"] + del config["feature_extractor"] + with open(os.path.join(tmpdirname, sd.config_name), "w") as f: + json.dump(config, f) + + sd = StableDiffusionPipeline.from_pretrained(tmpdirname) + + assert sd.config.requires_safety_checker is False + assert sd.config.safety_checker == (None, None) + assert sd.config.feature_extractor == (None, None) + + with tempfile.TemporaryDirectory() as tmpdirname: + sd.save_pretrained(tmpdirname) + + # Test that partially loading works + sd = StableDiffusionPipeline.from_pretrained(tmpdirname, feature_extractor=self.dummy_extractor) + + assert sd.config.requires_safety_checker is False + assert sd.config.safety_checker == (None, None) + assert sd.config.feature_extractor != (None, None) + + # Test that partially loading works + sd = StableDiffusionPipeline.from_pretrained( + tmpdirname, + feature_extractor=self.dummy_extractor, + safety_checker=unet, + requires_safety_checker=[True, True], + ) + + assert sd.config.requires_safety_checker == [True, True] + assert sd.config.safety_checker != (None, None) + assert sd.config.feature_extractor != (None, None) + + with tempfile.TemporaryDirectory() as tmpdirname: + sd.save_pretrained(tmpdirname) + sd = StableDiffusionPipeline.from_pretrained(tmpdirname, feature_extractor=self.dummy_extractor) + + assert sd.config.requires_safety_checker == [True, True] + assert sd.config.safety_checker != (None, None) + assert sd.config.feature_extractor != (None, None) + + def test_name_or_path(self): + model_path = "hf-internal-testing/tiny-stable-diffusion-torch" + sd = DiffusionPipeline.from_pretrained(model_path) + + assert sd.name_or_path == model_path + + with tempfile.TemporaryDirectory() as tmpdirname: + sd.save_pretrained(tmpdirname) + sd = DiffusionPipeline.from_pretrained(tmpdirname) + + assert sd.name_or_path == tmpdirname + + def test_error_no_variant_available(self): + variant = "fp16" + with self.assertRaises(ValueError) as error_context: + _ = StableDiffusionPipeline.download( + "hf-internal-testing/diffusers-stable-diffusion-tiny-all", variant=variant + ) + + assert "but no such modeling files are available" in str(error_context.exception) + assert variant in str(error_context.exception) + + def test_pipe_to(self): + unet = self.dummy_cond_unet() + scheduler = PNDMScheduler(skip_prk_steps=True) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + sd = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + + device_type = torch.device(torch_device).type + + sd1 = sd.to(device_type) + sd2 = sd.to(torch.device(device_type)) + sd3 = sd.to(device_type, torch.float32) + sd4 = sd.to(device=device_type) + sd5 = sd.to(torch_device=device_type) + sd6 = sd.to(device_type, dtype=torch.float32) + sd7 = sd.to(device_type, torch_dtype=torch.float32) + + assert sd1.device.type == device_type + assert sd2.device.type == device_type + assert sd3.device.type == device_type + assert sd4.device.type == device_type + assert sd5.device.type == device_type + assert sd6.device.type == device_type + assert sd7.device.type == device_type + + sd1 = sd.to(torch.float16) + sd2 = sd.to(None, torch.float16) + sd3 = sd.to(dtype=torch.float16) + sd4 = sd.to(dtype=torch.float16) + sd5 = sd.to(None, dtype=torch.float16) + sd6 = sd.to(None, torch_dtype=torch.float16) + + assert sd1.dtype == torch.float16 + assert sd2.dtype == torch.float16 + assert sd3.dtype == torch.float16 + assert sd4.dtype == torch.float16 + assert sd5.dtype == torch.float16 + assert sd6.dtype == torch.float16 + + sd1 = sd.to(device=device_type, dtype=torch.float16) + sd2 = sd.to(torch_device=device_type, torch_dtype=torch.float16) + sd3 = sd.to(device_type, torch.float16) + + assert sd1.dtype == torch.float16 + assert sd2.dtype == torch.float16 + assert sd3.dtype == torch.float16 + + assert sd1.device.type == device_type + assert sd2.device.type == device_type + assert sd3.device.type == device_type + + def test_pipe_same_device_id_offload(self): + unet = self.dummy_cond_unet() + scheduler = PNDMScheduler(skip_prk_steps=True) + vae = self.dummy_vae + bert = self.dummy_text_encoder + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + sd = StableDiffusionPipeline( + unet=unet, + scheduler=scheduler, + vae=vae, + text_encoder=bert, + tokenizer=tokenizer, + safety_checker=None, + feature_extractor=self.dummy_extractor, + ) + + sd.enable_model_cpu_offload(gpu_id=5) + assert sd._offload_gpu_id == 5 + sd.maybe_free_model_hooks() + assert sd._offload_gpu_id == 5 + + +@slow +@require_torch_gpu +class PipelineSlowTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_smart_download(self): + model_id = "hf-internal-testing/unet-pipeline-dummy" + with tempfile.TemporaryDirectory() as tmpdirname: + _ = DiffusionPipeline.from_pretrained(model_id, cache_dir=tmpdirname, force_download=True) + local_repo_name = "--".join(["models"] + model_id.split("/")) + snapshot_dir = os.path.join(tmpdirname, local_repo_name, "snapshots") + snapshot_dir = os.path.join(snapshot_dir, os.listdir(snapshot_dir)[0]) + + # inspect all downloaded files to make sure that everything is included + assert os.path.isfile(os.path.join(snapshot_dir, DiffusionPipeline.config_name)) + assert os.path.isfile(os.path.join(snapshot_dir, CONFIG_NAME)) + assert os.path.isfile(os.path.join(snapshot_dir, SCHEDULER_CONFIG_NAME)) + assert os.path.isfile(os.path.join(snapshot_dir, WEIGHTS_NAME)) + assert os.path.isfile(os.path.join(snapshot_dir, "scheduler", SCHEDULER_CONFIG_NAME)) + assert os.path.isfile(os.path.join(snapshot_dir, "unet", WEIGHTS_NAME)) + assert os.path.isfile(os.path.join(snapshot_dir, "unet", WEIGHTS_NAME)) + # let's make sure the super large numpy file: + # https://huggingface.co/hf-internal-testing/unet-pipeline-dummy/blob/main/big_array.npy + # is not downloaded, but all the expected ones + assert not os.path.isfile(os.path.join(snapshot_dir, "big_array.npy")) + + def test_warning_unused_kwargs(self): + model_id = "hf-internal-testing/unet-pipeline-dummy" + logger = logging.get_logger("diffusers.pipelines") + with tempfile.TemporaryDirectory() as tmpdirname: + with CaptureLogger(logger) as cap_logger: + DiffusionPipeline.from_pretrained( + model_id, + not_used=True, + cache_dir=tmpdirname, + force_download=True, + ) + + assert ( + cap_logger.out.strip().split("\n")[-1] + == "Keyword arguments {'not_used': True} are not expected by DDPMPipeline and will be ignored." + ) + + def test_from_save_pretrained(self): + # 1. Load models + model = UNet2DModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=3, + out_channels=3, + down_block_types=("DownBlock2D", "AttnDownBlock2D"), + up_block_types=("AttnUpBlock2D", "UpBlock2D"), + ) + scheduler = DDPMScheduler(num_train_timesteps=10) + + ddpm = DDPMPipeline(model, scheduler) + ddpm.to(torch_device) + ddpm.set_progress_bar_config(disable=None) + + with tempfile.TemporaryDirectory() as tmpdirname: + ddpm.save_pretrained(tmpdirname) + new_ddpm = DDPMPipeline.from_pretrained(tmpdirname) + new_ddpm.to(torch_device) + + generator = torch.Generator(device=torch_device).manual_seed(0) + image = ddpm(generator=generator, num_inference_steps=5, output_type="numpy").images + + generator = torch.Generator(device=torch_device).manual_seed(0) + new_image = new_ddpm(generator=generator, num_inference_steps=5, output_type="numpy").images + + assert np.abs(image - new_image).max() < 1e-5, "Models don't give the same forward pass" + + @require_python39_or_higher + @require_torch_2 + def test_from_save_pretrained_dynamo(self): + run_test_in_subprocess(test_case=self, target_func=_test_from_save_pretrained_dynamo, inputs=None) + + def test_from_pretrained_hub(self): + model_path = "google/ddpm-cifar10-32" + + scheduler = DDPMScheduler(num_train_timesteps=10) + + ddpm = DDPMPipeline.from_pretrained(model_path, scheduler=scheduler) + ddpm = ddpm.to(torch_device) + ddpm.set_progress_bar_config(disable=None) + + ddpm_from_hub = DiffusionPipeline.from_pretrained(model_path, scheduler=scheduler) + ddpm_from_hub = ddpm_from_hub.to(torch_device) + ddpm_from_hub.set_progress_bar_config(disable=None) + + generator = torch.Generator(device=torch_device).manual_seed(0) + image = ddpm(generator=generator, num_inference_steps=5, output_type="numpy").images + + generator = torch.Generator(device=torch_device).manual_seed(0) + new_image = ddpm_from_hub(generator=generator, num_inference_steps=5, output_type="numpy").images + + assert np.abs(image - new_image).max() < 1e-5, "Models don't give the same forward pass" + + def test_from_pretrained_hub_pass_model(self): + model_path = "google/ddpm-cifar10-32" + + scheduler = DDPMScheduler(num_train_timesteps=10) + + # pass unet into DiffusionPipeline + unet = UNet2DModel.from_pretrained(model_path) + ddpm_from_hub_custom_model = DiffusionPipeline.from_pretrained(model_path, unet=unet, scheduler=scheduler) + ddpm_from_hub_custom_model = ddpm_from_hub_custom_model.to(torch_device) + ddpm_from_hub_custom_model.set_progress_bar_config(disable=None) + + ddpm_from_hub = DiffusionPipeline.from_pretrained(model_path, scheduler=scheduler) + ddpm_from_hub = ddpm_from_hub.to(torch_device) + ddpm_from_hub_custom_model.set_progress_bar_config(disable=None) + + generator = torch.Generator(device=torch_device).manual_seed(0) + image = ddpm_from_hub_custom_model(generator=generator, num_inference_steps=5, output_type="numpy").images + + generator = torch.Generator(device=torch_device).manual_seed(0) + new_image = ddpm_from_hub(generator=generator, num_inference_steps=5, output_type="numpy").images + + assert np.abs(image - new_image).max() < 1e-5, "Models don't give the same forward pass" + + def test_output_format(self): + model_path = "google/ddpm-cifar10-32" + + scheduler = DDIMScheduler.from_pretrained(model_path) + pipe = DDIMPipeline.from_pretrained(model_path, scheduler=scheduler) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + images = pipe(output_type="numpy").images + assert images.shape == (1, 32, 32, 3) + assert isinstance(images, np.ndarray) + + images = pipe(output_type="pil", num_inference_steps=4).images + assert isinstance(images, list) + assert len(images) == 1 + assert isinstance(images[0], PIL.Image.Image) + + # use PIL by default + images = pipe(num_inference_steps=4).images + assert isinstance(images, list) + assert isinstance(images[0], PIL.Image.Image) + + @require_flax + def test_from_flax_from_pt(self): + pipe_pt = StableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-torch", safety_checker=None + ) + pipe_pt.to(torch_device) + + from diffusers import FlaxStableDiffusionPipeline + + with tempfile.TemporaryDirectory() as tmpdirname: + pipe_pt.save_pretrained(tmpdirname) + + pipe_flax, params = FlaxStableDiffusionPipeline.from_pretrained( + tmpdirname, safety_checker=None, from_pt=True + ) + + with tempfile.TemporaryDirectory() as tmpdirname: + pipe_flax.save_pretrained(tmpdirname, params=params) + pipe_pt_2 = StableDiffusionPipeline.from_pretrained(tmpdirname, safety_checker=None, from_flax=True) + pipe_pt_2.to(torch_device) + + prompt = "Hello" + + generator = torch.manual_seed(0) + image_0 = pipe_pt( + [prompt], + generator=generator, + num_inference_steps=2, + output_type="np", + ).images[0] + + generator = torch.manual_seed(0) + image_1 = pipe_pt_2( + [prompt], + generator=generator, + num_inference_steps=2, + output_type="np", + ).images[0] + + assert np.abs(image_0 - image_1).sum() < 1e-5, "Models don't give the same forward pass" + + @require_compel + def test_weighted_prompts_compel(self): + from compel import Compel + + pipe = StableDiffusionPipeline.from_pretrained("CompVis/stable-diffusion-v1-4") + pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) + pipe.enable_model_cpu_offload() + pipe.enable_attention_slicing() + + compel = Compel(tokenizer=pipe.tokenizer, text_encoder=pipe.text_encoder) + + prompt = "a red cat playing with a ball{}" + + prompts = [prompt.format(s) for s in ["", "++", "--"]] + + prompt_embeds = compel(prompts) + + generator = [torch.Generator(device="cpu").manual_seed(33) for _ in range(prompt_embeds.shape[0])] + + images = pipe( + prompt_embeds=prompt_embeds, generator=generator, num_inference_steps=20, output_type="numpy" + ).images + + for i, image in enumerate(images): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + f"/compel/forest_{i}.npy" + ) + + assert np.abs(image - expected_image).max() < 3e-1 + + +@nightly +@require_torch_gpu +class PipelineNightlyTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_ddpm_ddim_equality_batched(self): + seed = 0 + model_id = "google/ddpm-cifar10-32" + + unet = UNet2DModel.from_pretrained(model_id) + ddpm_scheduler = DDPMScheduler() + ddim_scheduler = DDIMScheduler() + + ddpm = DDPMPipeline(unet=unet, scheduler=ddpm_scheduler) + ddpm.to(torch_device) + ddpm.set_progress_bar_config(disable=None) + + ddim = DDIMPipeline(unet=unet, scheduler=ddim_scheduler) + ddim.to(torch_device) + ddim.set_progress_bar_config(disable=None) + + generator = torch.Generator(device=torch_device).manual_seed(seed) + ddpm_images = ddpm(batch_size=2, generator=generator, output_type="numpy").images + + generator = torch.Generator(device=torch_device).manual_seed(seed) + ddim_images = ddim( + batch_size=2, + generator=generator, + num_inference_steps=1000, + eta=1.0, + output_type="numpy", + use_clipped_model_output=True, # Need this to make DDIM match DDPM + ).images + + # the values aren't exactly equal, but the images look the same visually + assert np.abs(ddpm_images - ddim_images).max() < 1e-1 diff --git a/diffusers-0.27.0/tests/pipelines/test_pipelines_auto.py b/diffusers-0.27.0/tests/pipelines/test_pipelines_auto.py new file mode 100755 index 0000000..284cdb0 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/test_pipelines_auto.py @@ -0,0 +1,353 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import os +import shutil +import unittest +from collections import OrderedDict +from pathlib import Path + +import torch +from transformers import CLIPVisionConfig, CLIPVisionModelWithProjection + +from diffusers import ( + AutoPipelineForImage2Image, + AutoPipelineForInpainting, + AutoPipelineForText2Image, + ControlNetModel, + DiffusionPipeline, +) +from diffusers.pipelines.auto_pipeline import ( + AUTO_IMAGE2IMAGE_PIPELINES_MAPPING, + AUTO_INPAINT_PIPELINES_MAPPING, + AUTO_TEXT2IMAGE_PIPELINES_MAPPING, +) +from diffusers.utils.testing_utils import slow + + +PRETRAINED_MODEL_REPO_MAPPING = OrderedDict( + [ + ("stable-diffusion", "runwayml/stable-diffusion-v1-5"), + ("if", "DeepFloyd/IF-I-XL-v1.0"), + ("kandinsky", "kandinsky-community/kandinsky-2-1"), + ("kandinsky22", "kandinsky-community/kandinsky-2-2-decoder"), + ] +) + + +class AutoPipelineFastTest(unittest.TestCase): + @property + def dummy_image_encoder(self): + torch.manual_seed(0) + config = CLIPVisionConfig( + hidden_size=1, + projection_dim=1, + num_hidden_layers=1, + num_attention_heads=1, + image_size=1, + intermediate_size=1, + patch_size=1, + ) + return CLIPVisionModelWithProjection(config) + + def test_from_pipe_consistent(self): + pipe = AutoPipelineForText2Image.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-pipe", requires_safety_checker=False + ) + original_config = dict(pipe.config) + + pipe = AutoPipelineForImage2Image.from_pipe(pipe) + assert dict(pipe.config) == original_config + + pipe = AutoPipelineForText2Image.from_pipe(pipe) + assert dict(pipe.config) == original_config + + def test_from_pipe_override(self): + pipe = AutoPipelineForText2Image.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-pipe", requires_safety_checker=False + ) + + pipe = AutoPipelineForImage2Image.from_pipe(pipe, requires_safety_checker=True) + assert pipe.config.requires_safety_checker is True + + pipe = AutoPipelineForText2Image.from_pipe(pipe, requires_safety_checker=True) + assert pipe.config.requires_safety_checker is True + + def test_from_pipe_consistent_sdxl(self): + pipe = AutoPipelineForImage2Image.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-xl-pipe", + requires_aesthetics_score=True, + force_zeros_for_empty_prompt=False, + ) + + original_config = dict(pipe.config) + + pipe = AutoPipelineForText2Image.from_pipe(pipe) + pipe = AutoPipelineForImage2Image.from_pipe(pipe) + + assert dict(pipe.config) == original_config + + def test_kwargs_local_files_only(self): + repo = "hf-internal-testing/tiny-stable-diffusion-torch" + tmpdirname = DiffusionPipeline.download(repo) + tmpdirname = Path(tmpdirname) + + # edit commit_id to so that it's not the latest commit + commit_id = tmpdirname.name + new_commit_id = commit_id + "hug" + + ref_dir = tmpdirname.parent.parent / "refs/main" + with open(ref_dir, "w") as f: + f.write(new_commit_id) + + new_tmpdirname = tmpdirname.parent / new_commit_id + os.rename(tmpdirname, new_tmpdirname) + + try: + AutoPipelineForText2Image.from_pretrained(repo, local_files_only=True) + except OSError: + assert False, "not able to load local files" + + shutil.rmtree(tmpdirname.parent.parent) + + def test_from_pipe_controlnet_text2img(self): + pipe = AutoPipelineForText2Image.from_pretrained("hf-internal-testing/tiny-stable-diffusion-pipe") + controlnet = ControlNetModel.from_pretrained("hf-internal-testing/tiny-controlnet") + + pipe = AutoPipelineForText2Image.from_pipe(pipe, controlnet=controlnet) + assert pipe.__class__.__name__ == "StableDiffusionControlNetPipeline" + assert "controlnet" in pipe.components + + pipe = AutoPipelineForText2Image.from_pipe(pipe, controlnet=None) + assert pipe.__class__.__name__ == "StableDiffusionPipeline" + assert "controlnet" not in pipe.components + + def test_from_pipe_controlnet_img2img(self): + pipe = AutoPipelineForImage2Image.from_pretrained("hf-internal-testing/tiny-stable-diffusion-pipe") + controlnet = ControlNetModel.from_pretrained("hf-internal-testing/tiny-controlnet") + + pipe = AutoPipelineForImage2Image.from_pipe(pipe, controlnet=controlnet) + assert pipe.__class__.__name__ == "StableDiffusionControlNetImg2ImgPipeline" + assert "controlnet" in pipe.components + + pipe = AutoPipelineForImage2Image.from_pipe(pipe, controlnet=None) + assert pipe.__class__.__name__ == "StableDiffusionImg2ImgPipeline" + assert "controlnet" not in pipe.components + + def test_from_pipe_controlnet_inpaint(self): + pipe = AutoPipelineForInpainting.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch") + controlnet = ControlNetModel.from_pretrained("hf-internal-testing/tiny-controlnet") + + pipe = AutoPipelineForInpainting.from_pipe(pipe, controlnet=controlnet) + assert pipe.__class__.__name__ == "StableDiffusionControlNetInpaintPipeline" + assert "controlnet" in pipe.components + + pipe = AutoPipelineForInpainting.from_pipe(pipe, controlnet=None) + assert pipe.__class__.__name__ == "StableDiffusionInpaintPipeline" + assert "controlnet" not in pipe.components + + def test_from_pipe_controlnet_new_task(self): + pipe_text2img = AutoPipelineForText2Image.from_pretrained("hf-internal-testing/tiny-stable-diffusion-torch") + controlnet = ControlNetModel.from_pretrained("hf-internal-testing/tiny-controlnet") + + pipe_control_img2img = AutoPipelineForImage2Image.from_pipe(pipe_text2img, controlnet=controlnet) + assert pipe_control_img2img.__class__.__name__ == "StableDiffusionControlNetImg2ImgPipeline" + assert "controlnet" in pipe_control_img2img.components + + pipe_inpaint = AutoPipelineForInpainting.from_pipe(pipe_control_img2img, controlnet=None) + assert pipe_inpaint.__class__.__name__ == "StableDiffusionInpaintPipeline" + assert "controlnet" not in pipe_inpaint.components + + # testing `from_pipe` for text2img controlnet + ## 1. from a different controlnet pipe, without controlnet argument + pipe_control_text2img = AutoPipelineForText2Image.from_pipe(pipe_control_img2img) + assert pipe_control_text2img.__class__.__name__ == "StableDiffusionControlNetPipeline" + assert "controlnet" in pipe_control_text2img.components + + ## 2. from a different controlnet pipe, with controlnet argument + pipe_control_text2img = AutoPipelineForText2Image.from_pipe(pipe_control_img2img, controlnet=controlnet) + assert pipe_control_text2img.__class__.__name__ == "StableDiffusionControlNetPipeline" + assert "controlnet" in pipe_control_text2img.components + + ## 3. from same controlnet pipeline class, with a different controlnet component + pipe_control_text2img = AutoPipelineForText2Image.from_pipe(pipe_control_text2img, controlnet=controlnet) + assert pipe_control_text2img.__class__.__name__ == "StableDiffusionControlNetPipeline" + assert "controlnet" in pipe_control_text2img.components + + # testing from_pipe for inpainting + ## 1. from a different controlnet pipeline class + pipe_control_inpaint = AutoPipelineForInpainting.from_pipe(pipe_control_img2img) + assert pipe_control_inpaint.__class__.__name__ == "StableDiffusionControlNetInpaintPipeline" + assert "controlnet" in pipe_control_inpaint.components + + ## from a different controlnet pipe, with a different controlnet + pipe_control_inpaint = AutoPipelineForInpainting.from_pipe(pipe_control_img2img, controlnet=controlnet) + assert pipe_control_inpaint.__class__.__name__ == "StableDiffusionControlNetInpaintPipeline" + assert "controlnet" in pipe_control_inpaint.components + + ## from same controlnet pipe, with a different controlnet + pipe_control_inpaint = AutoPipelineForInpainting.from_pipe(pipe_control_inpaint, controlnet=controlnet) + assert pipe_control_inpaint.__class__.__name__ == "StableDiffusionControlNetInpaintPipeline" + assert "controlnet" in pipe_control_inpaint.components + + # testing from_pipe from img2img controlnet + ## from a different controlnet pipe, without controlnet argument + pipe_control_img2img = AutoPipelineForImage2Image.from_pipe(pipe_control_text2img) + assert pipe_control_img2img.__class__.__name__ == "StableDiffusionControlNetImg2ImgPipeline" + assert "controlnet" in pipe_control_img2img.components + + # from a different controlnet pipe, with a different controlnet component + pipe_control_img2img = AutoPipelineForImage2Image.from_pipe(pipe_control_text2img, controlnet=controlnet) + assert pipe_control_img2img.__class__.__name__ == "StableDiffusionControlNetImg2ImgPipeline" + assert "controlnet" in pipe_control_img2img.components + + # from same controlnet pipeline class, with a different controlnet + pipe_control_img2img = AutoPipelineForImage2Image.from_pipe(pipe_control_img2img, controlnet=controlnet) + assert pipe_control_img2img.__class__.__name__ == "StableDiffusionControlNetImg2ImgPipeline" + assert "controlnet" in pipe_control_img2img.components + + def test_from_pipe_optional_components(self): + image_encoder = self.dummy_image_encoder + + pipe = AutoPipelineForText2Image.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-pipe", + image_encoder=image_encoder, + ) + + pipe = AutoPipelineForImage2Image.from_pipe(pipe) + assert pipe.image_encoder is not None + + pipe = AutoPipelineForText2Image.from_pipe(pipe, image_encoder=None) + assert pipe.image_encoder is None + + +@slow +class AutoPipelineIntegrationTest(unittest.TestCase): + def test_pipe_auto(self): + for model_name, model_repo in PRETRAINED_MODEL_REPO_MAPPING.items(): + # test txt2img + pipe_txt2img = AutoPipelineForText2Image.from_pretrained( + model_repo, variant="fp16", torch_dtype=torch.float16 + ) + self.assertIsInstance(pipe_txt2img, AUTO_TEXT2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForText2Image.from_pipe(pipe_txt2img) + self.assertIsInstance(pipe_to, AUTO_TEXT2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_txt2img) + self.assertIsInstance(pipe_to, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING[model_name]) + + if "kandinsky" not in model_name: + pipe_to = AutoPipelineForInpainting.from_pipe(pipe_txt2img) + self.assertIsInstance(pipe_to, AUTO_INPAINT_PIPELINES_MAPPING[model_name]) + + del pipe_txt2img, pipe_to + gc.collect() + + # test img2img + + pipe_img2img = AutoPipelineForImage2Image.from_pretrained( + model_repo, variant="fp16", torch_dtype=torch.float16 + ) + self.assertIsInstance(pipe_img2img, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForText2Image.from_pipe(pipe_img2img) + self.assertIsInstance(pipe_to, AUTO_TEXT2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_img2img) + self.assertIsInstance(pipe_to, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING[model_name]) + + if "kandinsky" not in model_name: + pipe_to = AutoPipelineForInpainting.from_pipe(pipe_img2img) + self.assertIsInstance(pipe_to, AUTO_INPAINT_PIPELINES_MAPPING[model_name]) + + del pipe_img2img, pipe_to + gc.collect() + + # test inpaint + + if "kandinsky" not in model_name: + pipe_inpaint = AutoPipelineForInpainting.from_pretrained( + model_repo, variant="fp16", torch_dtype=torch.float16 + ) + self.assertIsInstance(pipe_inpaint, AUTO_INPAINT_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForText2Image.from_pipe(pipe_inpaint) + self.assertIsInstance(pipe_to, AUTO_TEXT2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_inpaint) + self.assertIsInstance(pipe_to, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING[model_name]) + + pipe_to = AutoPipelineForInpainting.from_pipe(pipe_inpaint) + self.assertIsInstance(pipe_to, AUTO_INPAINT_PIPELINES_MAPPING[model_name]) + + del pipe_inpaint, pipe_to + gc.collect() + + def test_from_pipe_consistent(self): + for model_name, model_repo in PRETRAINED_MODEL_REPO_MAPPING.items(): + if model_name in ["kandinsky", "kandinsky22"]: + auto_pipes = [AutoPipelineForText2Image, AutoPipelineForImage2Image] + else: + auto_pipes = [AutoPipelineForText2Image, AutoPipelineForImage2Image, AutoPipelineForInpainting] + + # test from_pretrained + for pipe_from_class in auto_pipes: + pipe_from = pipe_from_class.from_pretrained(model_repo, variant="fp16", torch_dtype=torch.float16) + pipe_from_config = dict(pipe_from.config) + + for pipe_to_class in auto_pipes: + pipe_to = pipe_to_class.from_pipe(pipe_from) + self.assertEqual(dict(pipe_to.config), pipe_from_config) + + del pipe_from, pipe_to + gc.collect() + + def test_controlnet(self): + # test from_pretrained + model_repo = "runwayml/stable-diffusion-v1-5" + controlnet_repo = "lllyasviel/sd-controlnet-canny" + + controlnet = ControlNetModel.from_pretrained(controlnet_repo, torch_dtype=torch.float16) + + pipe_txt2img = AutoPipelineForText2Image.from_pretrained( + model_repo, controlnet=controlnet, torch_dtype=torch.float16 + ) + self.assertIsInstance(pipe_txt2img, AUTO_TEXT2IMAGE_PIPELINES_MAPPING["stable-diffusion-controlnet"]) + + pipe_img2img = AutoPipelineForImage2Image.from_pretrained( + model_repo, controlnet=controlnet, torch_dtype=torch.float16 + ) + self.assertIsInstance(pipe_img2img, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["stable-diffusion-controlnet"]) + + pipe_inpaint = AutoPipelineForInpainting.from_pretrained( + model_repo, controlnet=controlnet, torch_dtype=torch.float16 + ) + self.assertIsInstance(pipe_inpaint, AUTO_INPAINT_PIPELINES_MAPPING["stable-diffusion-controlnet"]) + + # test from_pipe + for pipe_from in [pipe_txt2img, pipe_img2img, pipe_inpaint]: + pipe_to = AutoPipelineForText2Image.from_pipe(pipe_from) + self.assertIsInstance(pipe_to, AUTO_TEXT2IMAGE_PIPELINES_MAPPING["stable-diffusion-controlnet"]) + self.assertEqual(dict(pipe_to.config), dict(pipe_txt2img.config)) + + pipe_to = AutoPipelineForImage2Image.from_pipe(pipe_from) + self.assertIsInstance(pipe_to, AUTO_IMAGE2IMAGE_PIPELINES_MAPPING["stable-diffusion-controlnet"]) + self.assertEqual(dict(pipe_to.config), dict(pipe_img2img.config)) + + pipe_to = AutoPipelineForInpainting.from_pipe(pipe_from) + self.assertIsInstance(pipe_to, AUTO_INPAINT_PIPELINES_MAPPING["stable-diffusion-controlnet"]) + self.assertEqual(dict(pipe_to.config), dict(pipe_inpaint.config)) diff --git a/diffusers-0.27.0/tests/pipelines/test_pipelines_combined.py b/diffusers-0.27.0/tests/pipelines/test_pipelines_combined.py new file mode 100755 index 0000000..adedd54 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/test_pipelines_combined.py @@ -0,0 +1,128 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import torch +from huggingface_hub import ModelCard + +from diffusers import ( + DDPMScheduler, + DiffusionPipeline, + KandinskyV22CombinedPipeline, + KandinskyV22Pipeline, + KandinskyV22PriorPipeline, +) +from diffusers.pipelines.pipeline_utils import CONNECTED_PIPES_KEYS + + +def state_dicts_almost_equal(sd1, sd2): + sd1 = dict(sorted(sd1.items())) + sd2 = dict(sorted(sd2.items())) + + models_are_equal = True + for ten1, ten2 in zip(sd1.values(), sd2.values()): + if (ten1 - ten2).abs().sum() > 1e-3: + models_are_equal = False + + return models_are_equal + + +class CombinedPipelineFastTest(unittest.TestCase): + def modelcard_has_connected_pipeline(self, model_id): + modelcard = ModelCard.load(model_id) + connected_pipes = {prefix: getattr(modelcard.data, prefix, [None])[0] for prefix in CONNECTED_PIPES_KEYS} + connected_pipes = {k: v for k, v in connected_pipes.items() if v is not None} + + return len(connected_pipes) > 0 + + def test_correct_modelcard_format(self): + # hf-internal-testing/tiny-random-kandinsky-v22-prior has no metadata + assert not self.modelcard_has_connected_pipeline("hf-internal-testing/tiny-random-kandinsky-v22-prior") + + # see https://huggingface.co/hf-internal-testing/tiny-random-kandinsky-v22-decoder/blob/8baff9897c6be017013e21b5c562e5a381646c7e/README.md?code=true#L2 + assert self.modelcard_has_connected_pipeline("hf-internal-testing/tiny-random-kandinsky-v22-decoder") + + def test_load_connected_checkpoint_when_specified(self): + pipeline_prior = DiffusionPipeline.from_pretrained("hf-internal-testing/tiny-random-kandinsky-v22-prior") + pipeline_prior_connected = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-random-kandinsky-v22-prior", load_connected_pipeline=True + ) + + # Passing `load_connected_pipeline` to prior is a no-op as the pipeline has no connected pipeline + assert pipeline_prior.__class__ == pipeline_prior_connected.__class__ + + pipeline = DiffusionPipeline.from_pretrained("hf-internal-testing/tiny-random-kandinsky-v22-decoder") + pipeline_connected = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-random-kandinsky-v22-decoder", load_connected_pipeline=True + ) + + # Passing `load_connected_pipeline` to decoder loads the combined pipeline + assert pipeline.__class__ != pipeline_connected.__class__ + assert pipeline.__class__ == KandinskyV22Pipeline + assert pipeline_connected.__class__ == KandinskyV22CombinedPipeline + + # check that loaded components match prior and decoder components + assert set(pipeline_connected.components.keys()) == set( + ["prior_" + k for k in pipeline_prior.components.keys()] + list(pipeline.components.keys()) + ) + + def test_load_connected_checkpoint_default(self): + prior = KandinskyV22PriorPipeline.from_pretrained("hf-internal-testing/tiny-random-kandinsky-v22-prior") + decoder = KandinskyV22Pipeline.from_pretrained("hf-internal-testing/tiny-random-kandinsky-v22-decoder") + + # check that combined pipeline loads both prior & decoder because of + # https://huggingface.co/hf-internal-testing/tiny-random-kandinsky-v22-decoder/blob/8baff9897c6be017013e21b5c562e5a381646c7e/README.md?code=true#L3 + assert ( + KandinskyV22CombinedPipeline._load_connected_pipes + ) # combined pipelines will download more checkpoints that just the one specified + pipeline = KandinskyV22CombinedPipeline.from_pretrained( + "hf-internal-testing/tiny-random-kandinsky-v22-decoder" + ) + + prior_comps = prior.components + decoder_comps = decoder.components + for k, component in pipeline.components.items(): + if k.startswith("prior_"): + k = k[6:] + comp = prior_comps[k] + else: + comp = decoder_comps[k] + + if isinstance(component, torch.nn.Module): + assert state_dicts_almost_equal(component.state_dict(), comp.state_dict()) + elif hasattr(component, "config"): + assert dict(component.config) == dict(comp.config) + else: + assert component.__class__ == comp.__class__ + + def test_load_connected_checkpoint_with_passed_obj(self): + pipeline = KandinskyV22CombinedPipeline.from_pretrained( + "hf-internal-testing/tiny-random-kandinsky-v22-decoder" + ) + prior_scheduler = DDPMScheduler.from_config(pipeline.prior_scheduler.config) + scheduler = DDPMScheduler.from_config(pipeline.scheduler.config) + + # make sure we pass a different scheduler and prior_scheduler + assert pipeline.prior_scheduler.__class__ != prior_scheduler.__class__ + assert pipeline.scheduler.__class__ != scheduler.__class__ + + pipeline_new = KandinskyV22CombinedPipeline.from_pretrained( + "hf-internal-testing/tiny-random-kandinsky-v22-decoder", + prior_scheduler=prior_scheduler, + scheduler=scheduler, + ) + assert dict(pipeline_new.prior_scheduler.config) == dict(prior_scheduler.config) + assert dict(pipeline_new.scheduler.config) == dict(scheduler.config) diff --git a/diffusers-0.27.0/tests/pipelines/test_pipelines_common.py b/diffusers-0.27.0/tests/pipelines/test_pipelines_common.py new file mode 100755 index 0000000..2b29e3a --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/test_pipelines_common.py @@ -0,0 +1,1614 @@ +import contextlib +import gc +import inspect +import io +import json +import os +import re +import tempfile +import unittest +import uuid +from typing import Any, Callable, Dict, Union + +import numpy as np +import PIL.Image +import torch +from huggingface_hub import ModelCard, delete_repo +from huggingface_hub.utils import is_jinja_available +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +import diffusers +from diffusers import ( + AsymmetricAutoencoderKL, + AutoencoderKL, + AutoencoderTiny, + ConsistencyDecoderVAE, + DDIMScheduler, + DiffusionPipeline, + StableDiffusionPipeline, + UNet2DConditionModel, +) +from diffusers.image_processor import VaeImageProcessor +from diffusers.loaders import IPAdapterMixin +from diffusers.models.unets.unet_3d_condition import UNet3DConditionModel +from diffusers.models.unets.unet_i2vgen_xl import I2VGenXLUNet +from diffusers.models.unets.unet_motion_model import UNetMotionModel +from diffusers.pipelines.pipeline_utils import StableDiffusionMixin +from diffusers.schedulers import KarrasDiffusionSchedulers +from diffusers.utils import logging +from diffusers.utils.import_utils import is_accelerate_available, is_accelerate_version, is_xformers_available +from diffusers.utils.testing_utils import ( + CaptureLogger, + require_torch, + torch_device, +) + +from ..models.autoencoders.test_models_vae import ( + get_asym_autoencoder_kl_config, + get_autoencoder_kl_config, + get_autoencoder_tiny_config, + get_consistency_vae_config, +) +from ..models.unets.test_models_unet_2d_condition import create_ip_adapter_state_dict +from ..others.test_utils import TOKEN, USER, is_staging_test + + +def to_np(tensor): + if isinstance(tensor, torch.Tensor): + tensor = tensor.detach().cpu().numpy() + + return tensor + + +def check_same_shape(tensor_list): + shapes = [tensor.shape for tensor in tensor_list] + return all(shape == shapes[0] for shape in shapes[1:]) + + +class SDFunctionTesterMixin: + """ + This mixin is designed to be used with PipelineTesterMixin and unittest.TestCase classes. + It provides a set of common tests for PyTorch pipeline that inherit from StableDiffusionMixin, e.g. vae_slicing, vae_tiling, freeu, etc. + """ + + def test_vae_slicing(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + # components["scheduler"] = LMSDiscreteScheduler.from_config(components["scheduler"].config) + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + image_count = 4 + + inputs = self.get_dummy_inputs(device) + inputs["prompt"] = [inputs["prompt"]] * image_count + if "image" in inputs: # fix batch size mismatch in I2V_Gen pipeline + inputs["image"] = [inputs["image"]] * image_count + output_1 = pipe(**inputs) + + # make sure sliced vae decode yields the same result + pipe.enable_vae_slicing() + inputs = self.get_dummy_inputs(device) + inputs["prompt"] = [inputs["prompt"]] * image_count + if "image" in inputs: + inputs["image"] = [inputs["image"]] * image_count + inputs["return_dict"] = False + output_2 = pipe(**inputs) + + assert np.abs(output_2[0].flatten() - output_1[0].flatten()).max() < 1e-2 + + def test_vae_tiling(self): + components = self.get_dummy_components() + + # make sure here that pndm scheduler skips prk + if "safety_checker" in components: + components["safety_checker"] = None + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + inputs["return_dict"] = False + + # Test that tiled decode at 512x512 yields the same result as the non-tiled decode + output_1 = pipe(**inputs)[0] + + # make sure tiled vae decode yields the same result + pipe.enable_vae_tiling() + inputs = self.get_dummy_inputs(torch_device) + inputs["return_dict"] = False + output_2 = pipe(**inputs)[0] + + assert np.abs(output_2 - output_1).max() < 5e-1 + + # test that tiled decode works with various shapes + shapes = [(1, 4, 73, 97), (1, 4, 97, 73), (1, 4, 49, 65), (1, 4, 65, 49)] + for shape in shapes: + zeros = torch.zeros(shape).to(torch_device) + pipe.vae.decode(zeros) + + def test_freeu_enabled(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + inputs["return_dict"] = False + output = pipe(**inputs)[0] + + pipe.enable_freeu(s1=0.9, s2=0.2, b1=1.2, b2=1.4) + inputs = self.get_dummy_inputs(torch_device) + inputs["return_dict"] = False + output_freeu = pipe(**inputs)[0] + + assert not np.allclose( + output[0, -3:, -3:, -1], output_freeu[0, -3:, -3:, -1] + ), "Enabling of FreeU should lead to different results." + + def test_freeu_disabled(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + inputs["return_dict"] = False + output = pipe(**inputs)[0] + + pipe.enable_freeu(s1=0.9, s2=0.2, b1=1.2, b2=1.4) + pipe.disable_freeu() + + freeu_keys = {"s1", "s2", "b1", "b2"} + for upsample_block in pipe.unet.up_blocks: + for key in freeu_keys: + assert getattr(upsample_block, key) is None, f"Disabling of FreeU should have set {key} to None." + + inputs = self.get_dummy_inputs(torch_device) + inputs["return_dict"] = False + output_no_freeu = pipe(**inputs)[0] + assert np.allclose( + output, output_no_freeu, atol=1e-2 + ), f"Disabling of FreeU should lead to results similar to the default pipeline results but Max Abs Error={np.abs(output_no_freeu - output).max()}." + + def test_fused_qkv_projections(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["return_dict"] = False + image = pipe(**inputs)[0] + original_image_slice = image[0, -3:, -3:, -1] + + pipe.fuse_qkv_projections() + inputs = self.get_dummy_inputs(device) + inputs["return_dict"] = False + image_fused = pipe(**inputs)[0] + image_slice_fused = image_fused[0, -3:, -3:, -1] + + pipe.unfuse_qkv_projections() + inputs = self.get_dummy_inputs(device) + inputs["return_dict"] = False + image_disabled = pipe(**inputs)[0] + image_slice_disabled = image_disabled[0, -3:, -3:, -1] + + assert np.allclose( + original_image_slice, image_slice_fused, atol=1e-2, rtol=1e-2 + ), "Fusion of QKV projections shouldn't affect the outputs." + assert np.allclose( + image_slice_fused, image_slice_disabled, atol=1e-2, rtol=1e-2 + ), "Outputs, with QKV projection fusion enabled, shouldn't change when fused QKV projections are disabled." + assert np.allclose( + original_image_slice, image_slice_disabled, atol=1e-2, rtol=1e-2 + ), "Original outputs should match when fused QKV projections are disabled." + + +class IPAdapterTesterMixin: + """ + This mixin is designed to be used with PipelineTesterMixin and unittest.TestCase classes. + It provides a set of common tests for pipelines that support IP Adapters. + """ + + def test_pipeline_signature(self): + parameters = inspect.signature(self.pipeline_class.__call__).parameters + + assert issubclass(self.pipeline_class, IPAdapterMixin) + self.assertIn( + "ip_adapter_image", + parameters, + "`ip_adapter_image` argument must be supported by the `__call__` method", + ) + self.assertIn( + "ip_adapter_image_embeds", + parameters, + "`ip_adapter_image_embeds` argument must be supported by the `__call__` method", + ) + + def _get_dummy_image_embeds(self, cross_attention_dim: int = 32): + return torch.randn((2, 1, cross_attention_dim), device=torch_device) + + def _modify_inputs_for_ip_adapter_test(self, inputs: Dict[str, Any]): + parameters = inspect.signature(self.pipeline_class.__call__).parameters + if "image" in parameters.keys() and "strength" in parameters.keys(): + inputs["num_inference_steps"] = 4 + + inputs["output_type"] = "np" + inputs["return_dict"] = False + return inputs + + def test_ip_adapter_single(self, expected_max_diff: float = 1e-4): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components).to(torch_device) + pipe.set_progress_bar_config(disable=None) + cross_attention_dim = pipe.unet.config.get("cross_attention_dim", 32) + + # forward pass without ip adapter + inputs = self._modify_inputs_for_ip_adapter_test(self.get_dummy_inputs(torch_device)) + output_without_adapter = pipe(**inputs)[0] + + adapter_state_dict = create_ip_adapter_state_dict(pipe.unet) + pipe.unet._load_ip_adapter_weights(adapter_state_dict) + + # forward pass with single ip adapter, but scale=0 which should have no effect + inputs = self._modify_inputs_for_ip_adapter_test(self.get_dummy_inputs(torch_device)) + inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)] + pipe.set_ip_adapter_scale(0.0) + output_without_adapter_scale = pipe(**inputs)[0] + + # forward pass with single ip adapter, but with scale of adapter weights + inputs = self._modify_inputs_for_ip_adapter_test(self.get_dummy_inputs(torch_device)) + inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)] + pipe.set_ip_adapter_scale(42.0) + output_with_adapter_scale = pipe(**inputs)[0] + + max_diff_without_adapter_scale = np.abs(output_without_adapter_scale - output_without_adapter).max() + max_diff_with_adapter_scale = np.abs(output_with_adapter_scale - output_without_adapter).max() + + self.assertLess( + max_diff_without_adapter_scale, + expected_max_diff, + "Output without ip-adapter must be same as normal inference", + ) + self.assertGreater( + max_diff_with_adapter_scale, 1e-2, "Output with ip-adapter must be different from normal inference" + ) + + def test_ip_adapter_multi(self, expected_max_diff: float = 1e-4): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components).to(torch_device) + pipe.set_progress_bar_config(disable=None) + cross_attention_dim = pipe.unet.config.get("cross_attention_dim", 32) + + # forward pass without ip adapter + inputs = self._modify_inputs_for_ip_adapter_test(self.get_dummy_inputs(torch_device)) + output_without_adapter = pipe(**inputs)[0] + + adapter_state_dict_1 = create_ip_adapter_state_dict(pipe.unet) + adapter_state_dict_2 = create_ip_adapter_state_dict(pipe.unet) + pipe.unet._load_ip_adapter_weights([adapter_state_dict_1, adapter_state_dict_2]) + + # forward pass with multi ip adapter, but scale=0 which should have no effect + inputs = self._modify_inputs_for_ip_adapter_test(self.get_dummy_inputs(torch_device)) + inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)] * 2 + pipe.set_ip_adapter_scale([0.0, 0.0]) + output_without_multi_adapter_scale = pipe(**inputs)[0] + + # forward pass with multi ip adapter, but with scale of adapter weights + inputs = self._modify_inputs_for_ip_adapter_test(self.get_dummy_inputs(torch_device)) + inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)] * 2 + pipe.set_ip_adapter_scale([42.0, 42.0]) + output_with_multi_adapter_scale = pipe(**inputs)[0] + + max_diff_without_multi_adapter_scale = np.abs( + output_without_multi_adapter_scale - output_without_adapter + ).max() + max_diff_with_multi_adapter_scale = np.abs(output_with_multi_adapter_scale - output_without_adapter).max() + self.assertLess( + max_diff_without_multi_adapter_scale, + expected_max_diff, + "Output without multi-ip-adapter must be same as normal inference", + ) + self.assertGreater( + max_diff_with_multi_adapter_scale, + 1e-2, + "Output with multi-ip-adapter scale must be different from normal inference", + ) + + def test_ip_adapter_cfg(self, expected_max_diff: float = 1e-4): + parameters = inspect.signature(self.pipeline_class.__call__).parameters + + if "guidance_scale" not in parameters: + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components).to(torch_device) + pipe.set_progress_bar_config(disable=None) + cross_attention_dim = pipe.unet.config.get("cross_attention_dim", 32) + + adapter_state_dict = create_ip_adapter_state_dict(pipe.unet) + pipe.unet._load_ip_adapter_weights(adapter_state_dict) + pipe.set_ip_adapter_scale(1.0) + + # forward pass with CFG not applied + inputs = self._modify_inputs_for_ip_adapter_test(self.get_dummy_inputs(torch_device)) + inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)[0].unsqueeze(0)] + inputs["guidance_scale"] = 1.0 + out_no_cfg = pipe(**inputs)[0] + + # forward pass with CFG applied + inputs = self._modify_inputs_for_ip_adapter_test(self.get_dummy_inputs(torch_device)) + inputs["ip_adapter_image_embeds"] = [self._get_dummy_image_embeds(cross_attention_dim)] + inputs["guidance_scale"] = 7.5 + out_cfg = pipe(**inputs)[0] + + assert out_cfg.shape == out_no_cfg.shape + + +class PipelineLatentTesterMixin: + """ + This mixin is designed to be used with PipelineTesterMixin and unittest.TestCase classes. + It provides a set of common tests for PyTorch pipeline that has vae, e.g. + equivalence of different input and output types, etc. + """ + + @property + def image_params(self) -> frozenset: + raise NotImplementedError( + "You need to set the attribute `image_params` in the child test class. " + "`image_params` are tested for if all accepted input image types (i.e. `pt`,`pil`,`np`) are producing same results" + ) + + @property + def image_latents_params(self) -> frozenset: + raise NotImplementedError( + "You need to set the attribute `image_latents_params` in the child test class. " + "`image_latents_params` are tested for if passing latents directly are producing same results" + ) + + def get_dummy_inputs_by_type(self, device, seed=0, input_image_type="pt", output_type="np"): + inputs = self.get_dummy_inputs(device, seed) + + def convert_to_pt(image): + if isinstance(image, torch.Tensor): + input_image = image + elif isinstance(image, np.ndarray): + input_image = VaeImageProcessor.numpy_to_pt(image) + elif isinstance(image, PIL.Image.Image): + input_image = VaeImageProcessor.pil_to_numpy(image) + input_image = VaeImageProcessor.numpy_to_pt(input_image) + else: + raise ValueError(f"unsupported input_image_type {type(image)}") + return input_image + + def convert_pt_to_type(image, input_image_type): + if input_image_type == "pt": + input_image = image + elif input_image_type == "np": + input_image = VaeImageProcessor.pt_to_numpy(image) + elif input_image_type == "pil": + input_image = VaeImageProcessor.pt_to_numpy(image) + input_image = VaeImageProcessor.numpy_to_pil(input_image) + else: + raise ValueError(f"unsupported input_image_type {input_image_type}.") + return input_image + + for image_param in self.image_params: + if image_param in inputs.keys(): + inputs[image_param] = convert_pt_to_type( + convert_to_pt(inputs[image_param]).to(device), input_image_type + ) + + inputs["output_type"] = output_type + + return inputs + + def test_pt_np_pil_outputs_equivalent(self, expected_max_diff=1e-4): + self._test_pt_np_pil_outputs_equivalent(expected_max_diff=expected_max_diff) + + def _test_pt_np_pil_outputs_equivalent(self, expected_max_diff=1e-4, input_image_type="pt"): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + output_pt = pipe( + **self.get_dummy_inputs_by_type(torch_device, input_image_type=input_image_type, output_type="pt") + )[0] + output_np = pipe( + **self.get_dummy_inputs_by_type(torch_device, input_image_type=input_image_type, output_type="np") + )[0] + output_pil = pipe( + **self.get_dummy_inputs_by_type(torch_device, input_image_type=input_image_type, output_type="pil") + )[0] + + max_diff = np.abs(output_pt.cpu().numpy().transpose(0, 2, 3, 1) - output_np).max() + self.assertLess( + max_diff, expected_max_diff, "`output_type=='pt'` generate different results from `output_type=='np'`" + ) + + max_diff = np.abs(np.array(output_pil[0]) - (output_np * 255).round()).max() + self.assertLess(max_diff, 2.0, "`output_type=='pil'` generate different results from `output_type=='np'`") + + def test_pt_np_pil_inputs_equivalent(self): + if len(self.image_params) == 0: + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + out_input_pt = pipe(**self.get_dummy_inputs_by_type(torch_device, input_image_type="pt"))[0] + out_input_np = pipe(**self.get_dummy_inputs_by_type(torch_device, input_image_type="np"))[0] + out_input_pil = pipe(**self.get_dummy_inputs_by_type(torch_device, input_image_type="pil"))[0] + + max_diff = np.abs(out_input_pt - out_input_np).max() + self.assertLess(max_diff, 1e-4, "`input_type=='pt'` generate different result from `input_type=='np'`") + max_diff = np.abs(out_input_pil - out_input_np).max() + self.assertLess(max_diff, 1e-2, "`input_type=='pt'` generate different result from `input_type=='np'`") + + def test_latents_input(self): + if len(self.image_latents_params) == 0: + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.image_processor = VaeImageProcessor(do_resize=False, do_normalize=False) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + out = pipe(**self.get_dummy_inputs_by_type(torch_device, input_image_type="pt"))[0] + + vae = components["vae"] + inputs = self.get_dummy_inputs_by_type(torch_device, input_image_type="pt") + generator = inputs["generator"] + for image_param in self.image_latents_params: + if image_param in inputs.keys(): + inputs[image_param] = ( + vae.encode(inputs[image_param]).latent_dist.sample(generator) * vae.config.scaling_factor + ) + out_latents_inputs = pipe(**inputs)[0] + + max_diff = np.abs(out - out_latents_inputs).max() + self.assertLess(max_diff, 1e-4, "passing latents as image input generate different result from passing image") + + def test_multi_vae(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + block_out_channels = pipe.vae.config.block_out_channels + norm_num_groups = pipe.vae.config.norm_num_groups + + vae_classes = [AutoencoderKL, AsymmetricAutoencoderKL, ConsistencyDecoderVAE, AutoencoderTiny] + configs = [ + get_autoencoder_kl_config(block_out_channels, norm_num_groups), + get_asym_autoencoder_kl_config(block_out_channels, norm_num_groups), + get_consistency_vae_config(block_out_channels, norm_num_groups), + get_autoencoder_tiny_config(block_out_channels), + ] + + out_np = pipe(**self.get_dummy_inputs_by_type(torch_device, input_image_type="np"))[0] + + for vae_cls, config in zip(vae_classes, configs): + vae = vae_cls(**config) + vae = vae.to(torch_device) + components["vae"] = vae + vae_pipe = self.pipeline_class(**components) + out_vae_np = vae_pipe(**self.get_dummy_inputs_by_type(torch_device, input_image_type="np"))[0] + + assert out_vae_np.shape == out_np.shape + + +@require_torch +class PipelineKarrasSchedulerTesterMixin: + """ + This mixin is designed to be used with unittest.TestCase classes. + It provides a set of common tests for each PyTorch pipeline that makes use of KarrasDiffusionSchedulers + equivalence of dict and tuple outputs, etc. + """ + + def test_karras_schedulers_shape(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + + # make sure that PNDM does not need warm-up + pipe.scheduler.register_to_config(skip_prk_steps=True) + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + inputs = self.get_dummy_inputs(torch_device) + inputs["num_inference_steps"] = 2 + + if "strength" in inputs: + inputs["num_inference_steps"] = 4 + inputs["strength"] = 0.5 + + outputs = [] + for scheduler_enum in KarrasDiffusionSchedulers: + if "KDPM2" in scheduler_enum.name: + inputs["num_inference_steps"] = 5 + + scheduler_cls = getattr(diffusers, scheduler_enum.name) + pipe.scheduler = scheduler_cls.from_config(pipe.scheduler.config) + output = pipe(**inputs)[0] + outputs.append(output) + + if "KDPM2" in scheduler_enum.name: + inputs["num_inference_steps"] = 2 + + assert check_same_shape(outputs) + + +@require_torch +class PipelineTesterMixin: + """ + This mixin is designed to be used with unittest.TestCase classes. + It provides a set of common tests for each PyTorch pipeline, e.g. saving and loading the pipeline, + equivalence of dict and tuple outputs, etc. + """ + + # Canonical parameters that are passed to `__call__` regardless + # of the type of pipeline. They are always optional and have common + # sense default values. + required_optional_params = frozenset( + [ + "num_inference_steps", + "num_images_per_prompt", + "generator", + "latents", + "output_type", + "return_dict", + ] + ) + + # set these parameters to False in the child class if the pipeline does not support the corresponding functionality + test_attention_slicing = True + + test_xformers_attention = True + + def get_generator(self, seed): + device = torch_device if torch_device != "mps" else "cpu" + generator = torch.Generator(device).manual_seed(seed) + return generator + + @property + def pipeline_class(self) -> Union[Callable, DiffusionPipeline]: + raise NotImplementedError( + "You need to set the attribute `pipeline_class = ClassNameOfPipeline` in the child test class. " + "See existing pipeline tests for reference." + ) + + def get_dummy_components(self): + raise NotImplementedError( + "You need to implement `get_dummy_components(self)` in the child test class. " + "See existing pipeline tests for reference." + ) + + def get_dummy_inputs(self, device, seed=0): + raise NotImplementedError( + "You need to implement `get_dummy_inputs(self, device, seed)` in the child test class. " + "See existing pipeline tests for reference." + ) + + @property + def params(self) -> frozenset: + raise NotImplementedError( + "You need to set the attribute `params` in the child test class. " + "`params` are checked for if all values are present in `__call__`'s signature." + " You can set `params` using one of the common set of parameters defined in `pipeline_params.py`" + " e.g., `TEXT_TO_IMAGE_PARAMS` defines the common parameters used in text to " + "image pipelines, including prompts and prompt embedding overrides." + "If your pipeline's set of arguments has minor changes from one of the common sets of arguments, " + "do not make modifications to the existing common sets of arguments. I.e. a text to image pipeline " + "with non-configurable height and width arguments should set the attribute as " + "`params = TEXT_TO_IMAGE_PARAMS - {'height', 'width'}`. " + "See existing pipeline tests for reference." + ) + + @property + def batch_params(self) -> frozenset: + raise NotImplementedError( + "You need to set the attribute `batch_params` in the child test class. " + "`batch_params` are the parameters required to be batched when passed to the pipeline's " + "`__call__` method. `pipeline_params.py` provides some common sets of parameters such as " + "`TEXT_TO_IMAGE_BATCH_PARAMS`, `IMAGE_VARIATION_BATCH_PARAMS`, etc... If your pipeline's " + "set of batch arguments has minor changes from one of the common sets of batch arguments, " + "do not make modifications to the existing common sets of batch arguments. I.e. a text to " + "image pipeline `negative_prompt` is not batched should set the attribute as " + "`batch_params = TEXT_TO_IMAGE_BATCH_PARAMS - {'negative_prompt'}`. " + "See existing pipeline tests for reference." + ) + + @property + def callback_cfg_params(self) -> frozenset: + raise NotImplementedError( + "You need to set the attribute `callback_cfg_params` in the child test class that requires to run test_callback_cfg. " + "`callback_cfg_params` are the parameters that needs to be passed to the pipeline's callback " + "function when dynamically adjusting `guidance_scale`. They are variables that require special" + "treatment when `do_classifier_free_guidance` is `True`. `pipeline_params.py` provides some common" + " sets of parameters such as `TEXT_TO_IMAGE_CALLBACK_CFG_PARAMS`. If your pipeline's " + "set of cfg arguments has minor changes from one of the common sets of cfg arguments, " + "do not make modifications to the existing common sets of cfg arguments. I.e. for inpaint pipeine, you " + " need to adjust batch size of `mask` and `masked_image_latents` so should set the attribute as" + "`callback_cfg_params = TEXT_TO_IMAGE_CFG_PARAMS.union({'mask', 'masked_image_latents'})`" + ) + + def tearDown(self): + # clean up the VRAM after each test in case of CUDA runtime errors + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_save_load_local(self, expected_max_difference=5e-4): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output = pipe(**inputs)[0] + + logger = logging.get_logger("diffusers.pipelines.pipeline_utils") + logger.setLevel(diffusers.logging.INFO) + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir, safe_serialization=False) + + with CaptureLogger(logger) as cap_logger: + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir) + + for component in pipe_loaded.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + for name in pipe_loaded.components.keys(): + if name not in pipe_loaded._optional_components: + assert name in str(cap_logger) + + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output_loaded = pipe_loaded(**inputs)[0] + + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess(max_diff, expected_max_difference) + + def test_pipeline_call_signature(self): + self.assertTrue( + hasattr(self.pipeline_class, "__call__"), f"{self.pipeline_class} should have a `__call__` method" + ) + + parameters = inspect.signature(self.pipeline_class.__call__).parameters + + optional_parameters = set() + + for k, v in parameters.items(): + if v.default != inspect._empty: + optional_parameters.add(k) + + parameters = set(parameters.keys()) + parameters.remove("self") + parameters.discard("kwargs") # kwargs can be added if arguments of pipeline call function are deprecated + + remaining_required_parameters = set() + + for param in self.params: + if param not in parameters: + remaining_required_parameters.add(param) + + self.assertTrue( + len(remaining_required_parameters) == 0, + f"Required parameters not present: {remaining_required_parameters}", + ) + + remaining_required_optional_parameters = set() + + for param in self.required_optional_params: + if param not in optional_parameters: + remaining_required_optional_parameters.add(param) + + self.assertTrue( + len(remaining_required_optional_parameters) == 0, + f"Required optional parameters not present: {remaining_required_optional_parameters}", + ) + + def test_inference_batch_consistent(self, batch_sizes=[2]): + self._test_inference_batch_consistent(batch_sizes=batch_sizes) + + def _test_inference_batch_consistent( + self, batch_sizes=[2], additional_params_copy_to_batched_inputs=["num_inference_steps"], batch_generator=True + ): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + inputs["generator"] = self.get_generator(0) + + logger = logging.get_logger(pipe.__module__) + logger.setLevel(level=diffusers.logging.FATAL) + + # prepare batched inputs + batched_inputs = [] + for batch_size in batch_sizes: + batched_input = {} + batched_input.update(inputs) + + for name in self.batch_params: + if name not in inputs: + continue + + value = inputs[name] + if name == "prompt": + len_prompt = len(value) + # make unequal batch sizes + batched_input[name] = [value[: len_prompt // i] for i in range(1, batch_size + 1)] + + # make last batch super long + batched_input[name][-1] = 100 * "very long" + + else: + batched_input[name] = batch_size * [value] + + if batch_generator and "generator" in inputs: + batched_input["generator"] = [self.get_generator(i) for i in range(batch_size)] + + if "batch_size" in inputs: + batched_input["batch_size"] = batch_size + + batched_inputs.append(batched_input) + + logger.setLevel(level=diffusers.logging.WARNING) + for batch_size, batched_input in zip(batch_sizes, batched_inputs): + output = pipe(**batched_input) + assert len(output[0]) == batch_size + + def test_inference_batch_single_identical(self, batch_size=3, expected_max_diff=1e-4): + self._test_inference_batch_single_identical(batch_size=batch_size, expected_max_diff=expected_max_diff) + + def _test_inference_batch_single_identical( + self, + batch_size=2, + expected_max_diff=1e-4, + additional_params_copy_to_batched_inputs=["num_inference_steps"], + ): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for components in pipe.components.values(): + if hasattr(components, "set_default_attn_processor"): + components.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + inputs = self.get_dummy_inputs(torch_device) + # Reset generator in case it is has been used in self.get_dummy_inputs + inputs["generator"] = self.get_generator(0) + + logger = logging.get_logger(pipe.__module__) + logger.setLevel(level=diffusers.logging.FATAL) + + # batchify inputs + batched_inputs = {} + batched_inputs.update(inputs) + + for name in self.batch_params: + if name not in inputs: + continue + + value = inputs[name] + if name == "prompt": + len_prompt = len(value) + batched_inputs[name] = [value[: len_prompt // i] for i in range(1, batch_size + 1)] + batched_inputs[name][-1] = 100 * "very long" + + else: + batched_inputs[name] = batch_size * [value] + + if "generator" in inputs: + batched_inputs["generator"] = [self.get_generator(i) for i in range(batch_size)] + + if "batch_size" in inputs: + batched_inputs["batch_size"] = batch_size + + for arg in additional_params_copy_to_batched_inputs: + batched_inputs[arg] = inputs[arg] + + output = pipe(**inputs) + output_batch = pipe(**batched_inputs) + + assert output_batch[0].shape[0] == batch_size + + max_diff = np.abs(to_np(output_batch[0][0]) - to_np(output[0][0])).max() + assert max_diff < expected_max_diff + + def test_dict_tuple_outputs_equivalent(self, expected_max_difference=1e-4): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator_device = "cpu" + output = pipe(**self.get_dummy_inputs(generator_device))[0] + output_tuple = pipe(**self.get_dummy_inputs(generator_device), return_dict=False)[0] + + max_diff = np.abs(to_np(output) - to_np(output_tuple)).max() + self.assertLess(max_diff, expected_max_difference) + + def test_components_function(self): + init_components = self.get_dummy_components() + init_components = {k: v for k, v in init_components.items() if not isinstance(v, (str, int, float))} + + pipe = self.pipeline_class(**init_components) + + self.assertTrue(hasattr(pipe, "components")) + self.assertTrue(set(pipe.components.keys()) == set(init_components.keys())) + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_float16_inference(self, expected_max_diff=5e-2): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + components = self.get_dummy_components() + pipe_fp16 = self.pipeline_class(**components) + for component in pipe_fp16.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe_fp16.to(torch_device, torch.float16) + pipe_fp16.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + # Reset generator in case it is used inside dummy inputs + if "generator" in inputs: + inputs["generator"] = self.get_generator(0) + + output = pipe(**inputs)[0] + + fp16_inputs = self.get_dummy_inputs(torch_device) + # Reset generator in case it is used inside dummy inputs + if "generator" in fp16_inputs: + fp16_inputs["generator"] = self.get_generator(0) + + output_fp16 = pipe_fp16(**fp16_inputs)[0] + + max_diff = np.abs(to_np(output) - to_np(output_fp16)).max() + self.assertLess(max_diff, expected_max_diff, "The outputs of the fp16 and fp32 pipelines are too different.") + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_save_load_float16(self, expected_max_diff=1e-2): + components = self.get_dummy_components() + for name, module in components.items(): + if hasattr(module, "half"): + components[name] = module.to(torch_device).half() + + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output = pipe(**inputs)[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir, torch_dtype=torch.float16) + for component in pipe_loaded.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + for name, component in pipe_loaded.components.items(): + if hasattr(component, "dtype"): + self.assertTrue( + component.dtype == torch.float16, + f"`{name}.dtype` switched from `float16` to {component.dtype} after loading.", + ) + + inputs = self.get_dummy_inputs(torch_device) + output_loaded = pipe_loaded(**inputs)[0] + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess( + max_diff, expected_max_diff, "The output of the fp16 pipeline changed after saving and loading." + ) + + def test_save_load_optional_components(self, expected_max_difference=1e-4): + if not hasattr(self.pipeline_class, "_optional_components"): + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + # set all optional components to None + for optional_component in pipe._optional_components: + setattr(pipe, optional_component, None) + + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + output = pipe(**inputs)[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir, safe_serialization=False) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir) + for component in pipe_loaded.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + for optional_component in pipe._optional_components: + self.assertTrue( + getattr(pipe_loaded, optional_component) is None, + f"`{optional_component}` did not stay set to None after loading.", + ) + + inputs = self.get_dummy_inputs(generator_device) + output_loaded = pipe_loaded(**inputs)[0] + + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess(max_diff, expected_max_difference) + + @unittest.skipIf(torch_device != "cuda", reason="CUDA and CPU are required to switch devices") + def test_to_device(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + pipe.to("cpu") + model_devices = [component.device.type for component in components.values() if hasattr(component, "device")] + self.assertTrue(all(device == "cpu" for device in model_devices)) + + output_cpu = pipe(**self.get_dummy_inputs("cpu"))[0] + self.assertTrue(np.isnan(output_cpu).sum() == 0) + + pipe.to("cuda") + model_devices = [component.device.type for component in components.values() if hasattr(component, "device")] + self.assertTrue(all(device == "cuda" for device in model_devices)) + + output_cuda = pipe(**self.get_dummy_inputs("cuda"))[0] + self.assertTrue(np.isnan(to_np(output_cuda)).sum() == 0) + + def test_to_dtype(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + model_dtypes = [component.dtype for component in components.values() if hasattr(component, "dtype")] + self.assertTrue(all(dtype == torch.float32 for dtype in model_dtypes)) + + pipe.to(dtype=torch.float16) + model_dtypes = [component.dtype for component in components.values() if hasattr(component, "dtype")] + self.assertTrue(all(dtype == torch.float16 for dtype in model_dtypes)) + + def test_attention_slicing_forward_pass(self, expected_max_diff=1e-3): + self._test_attention_slicing_forward_pass(expected_max_diff=expected_max_diff) + + def _test_attention_slicing_forward_pass( + self, test_max_difference=True, test_mean_pixel_difference=True, expected_max_diff=1e-3 + ): + if not self.test_attention_slicing: + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + output_without_slicing = pipe(**inputs)[0] + + pipe.enable_attention_slicing(slice_size=1) + inputs = self.get_dummy_inputs(generator_device) + output_with_slicing = pipe(**inputs)[0] + + if test_max_difference: + max_diff = np.abs(to_np(output_with_slicing) - to_np(output_without_slicing)).max() + self.assertLess(max_diff, expected_max_diff, "Attention slicing should not affect the inference results") + + if test_mean_pixel_difference: + assert_mean_pixel_difference(to_np(output_with_slicing[0]), to_np(output_without_slicing[0])) + + @unittest.skipIf( + torch_device != "cuda" or not is_accelerate_available() or is_accelerate_version("<", "0.14.0"), + reason="CPU offload is only available with CUDA and `accelerate v0.14.0` or higher", + ) + def test_sequential_cpu_offload_forward_pass(self, expected_max_diff=1e-4): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + output_without_offload = pipe(**inputs)[0] + + pipe.enable_sequential_cpu_offload() + + inputs = self.get_dummy_inputs(generator_device) + output_with_offload = pipe(**inputs)[0] + + max_diff = np.abs(to_np(output_with_offload) - to_np(output_without_offload)).max() + self.assertLess(max_diff, expected_max_diff, "CPU offloading should not affect the inference results") + + @unittest.skipIf( + torch_device != "cuda" or not is_accelerate_available() or is_accelerate_version("<", "0.17.0"), + reason="CPU offload is only available with CUDA and `accelerate v0.17.0` or higher", + ) + def test_model_cpu_offload_forward_pass(self, expected_max_diff=2e-4): + generator_device = "cpu" + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(generator_device) + output_without_offload = pipe(**inputs)[0] + + pipe.enable_model_cpu_offload() + inputs = self.get_dummy_inputs(generator_device) + output_with_offload = pipe(**inputs)[0] + + max_diff = np.abs(to_np(output_with_offload) - to_np(output_without_offload)).max() + self.assertLess(max_diff, expected_max_diff, "CPU offloading should not affect the inference results") + offloaded_modules = [ + v + for k, v in pipe.components.items() + if isinstance(v, torch.nn.Module) and k not in pipe._exclude_from_cpu_offload + ] + ( + self.assertTrue(all(v.device.type == "cpu" for v in offloaded_modules)), + f"Not offloaded: {[v for v in offloaded_modules if v.device.type != 'cpu']}", + ) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass() + + def _test_xformers_attention_forwardGenerator_pass( + self, test_max_difference=True, test_mean_pixel_difference=True, expected_max_diff=1e-4 + ): + if not self.test_xformers_attention: + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + output_without_offload = pipe(**inputs)[0] + output_without_offload = ( + output_without_offload.cpu() if torch.is_tensor(output_without_offload) else output_without_offload + ) + + pipe.enable_xformers_memory_efficient_attention() + inputs = self.get_dummy_inputs(torch_device) + output_with_offload = pipe(**inputs)[0] + output_with_offload = ( + output_with_offload.cpu() if torch.is_tensor(output_with_offload) else output_without_offload + ) + + if test_max_difference: + max_diff = np.abs(to_np(output_with_offload) - to_np(output_without_offload)).max() + self.assertLess(max_diff, expected_max_diff, "XFormers attention should not affect the inference results") + + if test_mean_pixel_difference: + assert_mean_pixel_difference(output_with_offload[0], output_without_offload[0]) + + def test_progress_bar(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + + inputs = self.get_dummy_inputs(torch_device) + with io.StringIO() as stderr, contextlib.redirect_stderr(stderr): + _ = pipe(**inputs) + stderr = stderr.getvalue() + # we can't calculate the number of progress steps beforehand e.g. for strength-dependent img2img, + # so we just match "5" in "#####| 1/5 [00:01<00:00]" + max_steps = re.search("/(.*?) ", stderr).group(1) + self.assertTrue(max_steps is not None and len(max_steps) > 0) + self.assertTrue( + f"{max_steps}/{max_steps}" in stderr, "Progress bar should be enabled and stopped at the max step" + ) + + pipe.set_progress_bar_config(disable=True) + with io.StringIO() as stderr, contextlib.redirect_stderr(stderr): + _ = pipe(**inputs) + self.assertTrue(stderr.getvalue() == "", "Progress bar should be disabled") + + def test_num_images_per_prompt(self): + sig = inspect.signature(self.pipeline_class.__call__) + + if "num_images_per_prompt" not in sig.parameters: + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + batch_sizes = [1, 2] + num_images_per_prompts = [1, 2] + + for batch_size in batch_sizes: + for num_images_per_prompt in num_images_per_prompts: + inputs = self.get_dummy_inputs(torch_device) + + for key in inputs.keys(): + if key in self.batch_params: + inputs[key] = batch_size * [inputs[key]] + + images = pipe(**inputs, num_images_per_prompt=num_images_per_prompt)[0] + + assert images.shape[0] == batch_size * num_images_per_prompt + + def test_cfg(self): + sig = inspect.signature(self.pipeline_class.__call__) + + if "guidance_scale" not in sig.parameters: + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(torch_device) + + inputs["guidance_scale"] = 1.0 + out_no_cfg = pipe(**inputs)[0] + + inputs["guidance_scale"] = 7.5 + out_cfg = pipe(**inputs)[0] + + assert out_cfg.shape == out_no_cfg.shape + + def test_callback_inputs(self): + sig = inspect.signature(self.pipeline_class.__call__) + has_callback_tensor_inputs = "callback_on_step_end_tensor_inputs" in sig.parameters + has_callback_step_end = "callback_on_step_end" in sig.parameters + + if not (has_callback_tensor_inputs and has_callback_step_end): + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + self.assertTrue( + hasattr(pipe, "_callback_tensor_inputs"), + f" {self.pipeline_class} should have `_callback_tensor_inputs` that defines a list of tensor variables its callback function can use as inputs", + ) + + def callback_inputs_subset(pipe, i, t, callback_kwargs): + # interate over callback args + for tensor_name, tensor_value in callback_kwargs.items(): + # check that we're only passing in allowed tensor inputs + assert tensor_name in pipe._callback_tensor_inputs + + return callback_kwargs + + def callback_inputs_all(pipe, i, t, callback_kwargs): + for tensor_name in pipe._callback_tensor_inputs: + assert tensor_name in callback_kwargs + + # interate over callback args + for tensor_name, tensor_value in callback_kwargs.items(): + # check that we're only passing in allowed tensor inputs + assert tensor_name in pipe._callback_tensor_inputs + + return callback_kwargs + + inputs = self.get_dummy_inputs(torch_device) + + # Test passing in a subset + inputs["callback_on_step_end"] = callback_inputs_subset + inputs["callback_on_step_end_tensor_inputs"] = ["latents"] + inputs["output_type"] = "latent" + output = pipe(**inputs)[0] + + # Test passing in a everything + inputs["callback_on_step_end"] = callback_inputs_all + inputs["callback_on_step_end_tensor_inputs"] = pipe._callback_tensor_inputs + inputs["output_type"] = "latent" + output = pipe(**inputs)[0] + + def callback_inputs_change_tensor(pipe, i, t, callback_kwargs): + is_last = i == (pipe.num_timesteps - 1) + if is_last: + callback_kwargs["latents"] = torch.zeros_like(callback_kwargs["latents"]) + return callback_kwargs + + inputs["callback_on_step_end"] = callback_inputs_change_tensor + inputs["callback_on_step_end_tensor_inputs"] = pipe._callback_tensor_inputs + inputs["output_type"] = "latent" + output = pipe(**inputs)[0] + assert output.abs().sum() == 0 + + def test_callback_cfg(self): + sig = inspect.signature(self.pipeline_class.__call__) + has_callback_tensor_inputs = "callback_on_step_end_tensor_inputs" in sig.parameters + has_callback_step_end = "callback_on_step_end" in sig.parameters + + if not (has_callback_tensor_inputs and has_callback_step_end): + return + + if "guidance_scale" not in sig.parameters: + return + + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + self.assertTrue( + hasattr(pipe, "_callback_tensor_inputs"), + f" {self.pipeline_class} should have `_callback_tensor_inputs` that defines a list of tensor variables its callback function can use as inputs", + ) + + def callback_increase_guidance(pipe, i, t, callback_kwargs): + pipe._guidance_scale += 1.0 + + return callback_kwargs + + inputs = self.get_dummy_inputs(torch_device) + + # use cfg guidance because some pipelines modify the shape of the latents + # outside of the denoising loop + inputs["guidance_scale"] = 2.0 + inputs["callback_on_step_end"] = callback_increase_guidance + inputs["callback_on_step_end_tensor_inputs"] = pipe._callback_tensor_inputs + _ = pipe(**inputs)[0] + + # we increase the guidance scale by 1.0 at every step + # check that the guidance scale is increased by the number of scheduler timesteps + # accounts for models that modify the number of inference steps based on strength + assert pipe.guidance_scale == (inputs["guidance_scale"] + pipe.num_timesteps) + + def test_StableDiffusionMixin_component(self): + """Any pipeline that have LDMFuncMixin should have vae and unet components.""" + if not issubclass(self.pipeline_class, StableDiffusionMixin): + return + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + self.assertTrue(hasattr(pipe, "vae") and isinstance(pipe.vae, (AutoencoderKL, AutoencoderTiny))) + self.assertTrue( + hasattr(pipe, "unet") + and isinstance(pipe.unet, (UNet2DConditionModel, UNet3DConditionModel, I2VGenXLUNet, UNetMotionModel)) + ) + + +@is_staging_test +class PipelinePushToHubTester(unittest.TestCase): + identifier = uuid.uuid4() + repo_id = f"test-pipeline-{identifier}" + org_repo_id = f"valid_org/{repo_id}-org" + + def get_pipeline_components(self): + unet = UNet2DConditionModel( + block_out_channels=(32, 64), + layers_per_block=2, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + cross_attention_dim=32, + ) + + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + ) + + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + text_encoder = CLIPTextModel(text_encoder_config) + + with tempfile.TemporaryDirectory() as tmpdir: + dummy_vocab = {"<|startoftext|>": 0, "<|endoftext|>": 1, "!": 2} + vocab_path = os.path.join(tmpdir, "vocab.json") + with open(vocab_path, "w") as f: + json.dump(dummy_vocab, f) + + merges = "Ġ t\nĠt h" + merges_path = os.path.join(tmpdir, "merges.txt") + with open(merges_path, "w") as f: + f.writelines(merges) + tokenizer = CLIPTokenizer(vocab_file=vocab_path, merges_file=merges_path) + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "safety_checker": None, + "feature_extractor": None, + } + return components + + def test_push_to_hub(self): + components = self.get_pipeline_components() + pipeline = StableDiffusionPipeline(**components) + pipeline.push_to_hub(self.repo_id, token=TOKEN) + + new_model = UNet2DConditionModel.from_pretrained(f"{USER}/{self.repo_id}", subfolder="unet") + unet = components["unet"] + for p1, p2 in zip(unet.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + # Reset repo + delete_repo(token=TOKEN, repo_id=self.repo_id) + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + pipeline.save_pretrained(tmp_dir, repo_id=self.repo_id, push_to_hub=True, token=TOKEN) + + new_model = UNet2DConditionModel.from_pretrained(f"{USER}/{self.repo_id}", subfolder="unet") + for p1, p2 in zip(unet.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + # Reset repo + delete_repo(self.repo_id, token=TOKEN) + + def test_push_to_hub_in_organization(self): + components = self.get_pipeline_components() + pipeline = StableDiffusionPipeline(**components) + pipeline.push_to_hub(self.org_repo_id, token=TOKEN) + + new_model = UNet2DConditionModel.from_pretrained(self.org_repo_id, subfolder="unet") + unet = components["unet"] + for p1, p2 in zip(unet.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + # Reset repo + delete_repo(token=TOKEN, repo_id=self.org_repo_id) + + # Push to hub via save_pretrained + with tempfile.TemporaryDirectory() as tmp_dir: + pipeline.save_pretrained(tmp_dir, push_to_hub=True, token=TOKEN, repo_id=self.org_repo_id) + + new_model = UNet2DConditionModel.from_pretrained(self.org_repo_id, subfolder="unet") + for p1, p2 in zip(unet.parameters(), new_model.parameters()): + self.assertTrue(torch.equal(p1, p2)) + + # Reset repo + delete_repo(self.org_repo_id, token=TOKEN) + + @unittest.skipIf( + not is_jinja_available(), + reason="Model card tests cannot be performed without Jinja installed.", + ) + def test_push_to_hub_library_name(self): + components = self.get_pipeline_components() + pipeline = StableDiffusionPipeline(**components) + pipeline.push_to_hub(self.repo_id, token=TOKEN) + + model_card = ModelCard.load(f"{USER}/{self.repo_id}", token=TOKEN).data + assert model_card.library_name == "diffusers" + + # Reset repo + delete_repo(self.repo_id, token=TOKEN) + + +# For SDXL and its derivative pipelines (such as ControlNet), we have the text encoders +# and the tokenizers as optional components. So, we need to override the `test_save_load_optional_components()` +# test for all such pipelines. This requires us to use a custom `encode_prompt()` function. +class SDXLOptionalComponentsTesterMixin: + def encode_prompt( + self, tokenizers, text_encoders, prompt: str, num_images_per_prompt: int = 1, negative_prompt: str = None + ): + device = text_encoders[0].device + + if isinstance(prompt, str): + prompt = [prompt] + batch_size = len(prompt) + + prompt_embeds_list = [] + for tokenizer, text_encoder in zip(tokenizers, text_encoders): + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + text_input_ids = text_inputs.input_ids + + prompt_embeds = text_encoder(text_input_ids.to(device), output_hidden_states=True) + pooled_prompt_embeds = prompt_embeds[0] + prompt_embeds = prompt_embeds.hidden_states[-2] + prompt_embeds_list.append(prompt_embeds) + + prompt_embeds = torch.concat(prompt_embeds_list, dim=-1) + + if negative_prompt is None: + negative_prompt_embeds = torch.zeros_like(prompt_embeds) + negative_pooled_prompt_embeds = torch.zeros_like(pooled_prompt_embeds) + else: + negative_prompt = batch_size * [negative_prompt] if isinstance(negative_prompt, str) else negative_prompt + + negative_prompt_embeds_list = [] + for tokenizer, text_encoder in zip(tokenizers, text_encoders): + uncond_input = tokenizer( + negative_prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + truncation=True, + return_tensors="pt", + ) + + negative_prompt_embeds = text_encoder(uncond_input.input_ids.to(device), output_hidden_states=True) + negative_pooled_prompt_embeds = negative_prompt_embeds[0] + negative_prompt_embeds = negative_prompt_embeds.hidden_states[-2] + negative_prompt_embeds_list.append(negative_prompt_embeds) + + negative_prompt_embeds = torch.concat(negative_prompt_embeds_list, dim=-1) + + bs_embed, seq_len, _ = prompt_embeds.shape + + # duplicate text embeddings for each generation per prompt, using mps friendly method + prompt_embeds = prompt_embeds.repeat(1, num_images_per_prompt, 1) + prompt_embeds = prompt_embeds.view(bs_embed * num_images_per_prompt, seq_len, -1) + + # for classifier-free guidance + # duplicate unconditional embeddings for each generation per prompt, using mps friendly method + seq_len = negative_prompt_embeds.shape[1] + + negative_prompt_embeds = negative_prompt_embeds.repeat(1, num_images_per_prompt, 1) + negative_prompt_embeds = negative_prompt_embeds.view(batch_size * num_images_per_prompt, seq_len, -1) + + pooled_prompt_embeds = pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + # for classifier-free guidance + negative_pooled_prompt_embeds = negative_pooled_prompt_embeds.repeat(1, num_images_per_prompt).view( + bs_embed * num_images_per_prompt, -1 + ) + + return prompt_embeds, negative_prompt_embeds, pooled_prompt_embeds, negative_pooled_prompt_embeds + + def _test_save_load_optional_components(self, expected_max_difference=1e-4): + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + for optional_component in pipe._optional_components: + setattr(pipe, optional_component, None) + + for component in pipe.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + generator_device = "cpu" + inputs = self.get_dummy_inputs(generator_device) + + tokenizer = components.pop("tokenizer") + tokenizer_2 = components.pop("tokenizer_2") + text_encoder = components.pop("text_encoder") + text_encoder_2 = components.pop("text_encoder_2") + + tokenizers = [tokenizer, tokenizer_2] if tokenizer is not None else [tokenizer_2] + text_encoders = [text_encoder, text_encoder_2] if text_encoder is not None else [text_encoder_2] + prompt = inputs.pop("prompt") + ( + prompt_embeds, + negative_prompt_embeds, + pooled_prompt_embeds, + negative_pooled_prompt_embeds, + ) = self.encode_prompt(tokenizers, text_encoders, prompt) + inputs["prompt_embeds"] = prompt_embeds + inputs["negative_prompt_embeds"] = negative_prompt_embeds + inputs["pooled_prompt_embeds"] = pooled_prompt_embeds + inputs["negative_pooled_prompt_embeds"] = negative_pooled_prompt_embeds + + output = pipe(**inputs)[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir) + for component in pipe_loaded.components.values(): + if hasattr(component, "set_default_attn_processor"): + component.set_default_attn_processor() + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + for optional_component in pipe._optional_components: + self.assertTrue( + getattr(pipe_loaded, optional_component) is None, + f"`{optional_component}` did not stay set to None after loading.", + ) + + inputs = self.get_dummy_inputs(generator_device) + _ = inputs.pop("prompt") + inputs["prompt_embeds"] = prompt_embeds + inputs["negative_prompt_embeds"] = negative_prompt_embeds + inputs["pooled_prompt_embeds"] = pooled_prompt_embeds + inputs["negative_pooled_prompt_embeds"] = negative_pooled_prompt_embeds + + output_loaded = pipe_loaded(**inputs)[0] + + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess(max_diff, expected_max_difference) + + +# Some models (e.g. unCLIP) are extremely likely to significantly deviate depending on which hardware is used. +# This helper function is used to check that the image doesn't deviate on average more than 10 pixels from a +# reference image. +def assert_mean_pixel_difference(image, expected_image, expected_max_diff=10): + image = np.asarray(DiffusionPipeline.numpy_to_pil(image)[0], dtype=np.float32) + expected_image = np.asarray(DiffusionPipeline.numpy_to_pil(expected_image)[0], dtype=np.float32) + avg_diff = np.abs(image - expected_image).mean() + assert avg_diff < expected_max_diff, f"Error image deviates {avg_diff} pixels on average" diff --git a/diffusers-0.27.0/tests/pipelines/test_pipelines_flax.py b/diffusers-0.27.0/tests/pipelines/test_pipelines_flax.py new file mode 100755 index 0000000..3e7a2b3 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/test_pipelines_flax.py @@ -0,0 +1,260 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +import unittest + +import numpy as np + +from diffusers.utils import is_flax_available +from diffusers.utils.testing_utils import require_flax, slow + + +if is_flax_available(): + import jax + import jax.numpy as jnp + from flax.jax_utils import replicate + from flax.training.common_utils import shard + + from diffusers import FlaxDDIMScheduler, FlaxDiffusionPipeline, FlaxStableDiffusionPipeline + + +@require_flax +class DownloadTests(unittest.TestCase): + def test_download_only_pytorch(self): + with tempfile.TemporaryDirectory() as tmpdirname: + # pipeline has Flax weights + _ = FlaxDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-pipe", safety_checker=None, cache_dir=tmpdirname + ) + + all_root_files = [t[-1] for t in os.walk(os.path.join(tmpdirname, os.listdir(tmpdirname)[0], "snapshots"))] + files = [item for sublist in all_root_files for item in sublist] + + # None of the downloaded files should be a PyTorch file even if we have some here: + # https://huggingface.co/hf-internal-testing/tiny-stable-diffusion-pipe/blob/main/unet/diffusion_pytorch_model.bin + assert not any(f.endswith(".bin") for f in files) + + +@slow +@require_flax +class FlaxPipelineTests(unittest.TestCase): + def test_dummy_all_tpus(self): + pipeline, params = FlaxStableDiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-pipe", safety_checker=None + ) + + prompt = ( + "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of" + " field, close up, split lighting, cinematic" + ) + + prng_seed = jax.random.PRNGKey(0) + num_inference_steps = 4 + + num_samples = jax.device_count() + prompt = num_samples * [prompt] + prompt_ids = pipeline.prepare_inputs(prompt) + + # shard inputs and rng + params = replicate(params) + prng_seed = jax.random.split(prng_seed, num_samples) + prompt_ids = shard(prompt_ids) + + images = pipeline(prompt_ids, params, prng_seed, num_inference_steps, jit=True).images + + assert images.shape == (num_samples, 1, 64, 64, 3) + if jax.device_count() == 8: + assert np.abs(np.abs(images[0, 0, :2, :2, -2:], dtype=np.float32).sum() - 4.1514745) < 1e-3 + assert np.abs(np.abs(images, dtype=np.float32).sum() - 49947.875) < 5e-1 + + images_pil = pipeline.numpy_to_pil(np.asarray(images.reshape((num_samples,) + images.shape[-3:]))) + assert len(images_pil) == num_samples + + def test_stable_diffusion_v1_4(self): + pipeline, params = FlaxStableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", revision="flax", safety_checker=None + ) + + prompt = ( + "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of" + " field, close up, split lighting, cinematic" + ) + + prng_seed = jax.random.PRNGKey(0) + num_inference_steps = 50 + + num_samples = jax.device_count() + prompt = num_samples * [prompt] + prompt_ids = pipeline.prepare_inputs(prompt) + + # shard inputs and rng + params = replicate(params) + prng_seed = jax.random.split(prng_seed, num_samples) + prompt_ids = shard(prompt_ids) + + images = pipeline(prompt_ids, params, prng_seed, num_inference_steps, jit=True).images + + assert images.shape == (num_samples, 1, 512, 512, 3) + if jax.device_count() == 8: + assert np.abs((np.abs(images[0, 0, :2, :2, -2:], dtype=np.float32).sum() - 0.05652401)) < 1e-2 + assert np.abs((np.abs(images, dtype=np.float32).sum() - 2383808.2)) < 5e-1 + + def test_stable_diffusion_v1_4_bfloat_16(self): + pipeline, params = FlaxStableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", revision="bf16", dtype=jnp.bfloat16, safety_checker=None + ) + + prompt = ( + "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of" + " field, close up, split lighting, cinematic" + ) + + prng_seed = jax.random.PRNGKey(0) + num_inference_steps = 50 + + num_samples = jax.device_count() + prompt = num_samples * [prompt] + prompt_ids = pipeline.prepare_inputs(prompt) + + # shard inputs and rng + params = replicate(params) + prng_seed = jax.random.split(prng_seed, num_samples) + prompt_ids = shard(prompt_ids) + + images = pipeline(prompt_ids, params, prng_seed, num_inference_steps, jit=True).images + + assert images.shape == (num_samples, 1, 512, 512, 3) + if jax.device_count() == 8: + assert np.abs((np.abs(images[0, 0, :2, :2, -2:], dtype=np.float32).sum() - 0.04003906)) < 5e-2 + assert np.abs((np.abs(images, dtype=np.float32).sum() - 2373516.75)) < 5e-1 + + def test_stable_diffusion_v1_4_bfloat_16_with_safety(self): + pipeline, params = FlaxStableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", revision="bf16", dtype=jnp.bfloat16 + ) + + prompt = ( + "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of" + " field, close up, split lighting, cinematic" + ) + + prng_seed = jax.random.PRNGKey(0) + num_inference_steps = 50 + + num_samples = jax.device_count() + prompt = num_samples * [prompt] + prompt_ids = pipeline.prepare_inputs(prompt) + + # shard inputs and rng + params = replicate(params) + prng_seed = jax.random.split(prng_seed, num_samples) + prompt_ids = shard(prompt_ids) + + images = pipeline(prompt_ids, params, prng_seed, num_inference_steps, jit=True).images + + assert images.shape == (num_samples, 1, 512, 512, 3) + if jax.device_count() == 8: + assert np.abs((np.abs(images[0, 0, :2, :2, -2:], dtype=np.float32).sum() - 0.04003906)) < 5e-2 + assert np.abs((np.abs(images, dtype=np.float32).sum() - 2373516.75)) < 5e-1 + + def test_stable_diffusion_v1_4_bfloat_16_ddim(self): + scheduler = FlaxDDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + set_alpha_to_one=False, + steps_offset=1, + ) + + pipeline, params = FlaxStableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", + revision="bf16", + dtype=jnp.bfloat16, + scheduler=scheduler, + safety_checker=None, + ) + scheduler_state = scheduler.create_state() + + params["scheduler"] = scheduler_state + + prompt = ( + "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of" + " field, close up, split lighting, cinematic" + ) + + prng_seed = jax.random.PRNGKey(0) + num_inference_steps = 50 + + num_samples = jax.device_count() + prompt = num_samples * [prompt] + prompt_ids = pipeline.prepare_inputs(prompt) + + # shard inputs and rng + params = replicate(params) + prng_seed = jax.random.split(prng_seed, num_samples) + prompt_ids = shard(prompt_ids) + + images = pipeline(prompt_ids, params, prng_seed, num_inference_steps, jit=True).images + + assert images.shape == (num_samples, 1, 512, 512, 3) + if jax.device_count() == 8: + assert np.abs((np.abs(images[0, 0, :2, :2, -2:], dtype=np.float32).sum() - 0.045043945)) < 5e-2 + assert np.abs((np.abs(images, dtype=np.float32).sum() - 2347693.5)) < 5e-1 + + def test_jax_memory_efficient_attention(self): + prompt = ( + "A cinematic film still of Morgan Freeman starring as Jimi Hendrix, portrait, 40mm lens, shallow depth of" + " field, close up, split lighting, cinematic" + ) + + num_samples = jax.device_count() + prompt = num_samples * [prompt] + prng_seed = jax.random.split(jax.random.PRNGKey(0), num_samples) + + pipeline, params = FlaxStableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", + revision="bf16", + dtype=jnp.bfloat16, + safety_checker=None, + ) + + params = replicate(params) + prompt_ids = pipeline.prepare_inputs(prompt) + prompt_ids = shard(prompt_ids) + images = pipeline(prompt_ids, params, prng_seed, jit=True).images + assert images.shape == (num_samples, 1, 512, 512, 3) + slice = images[2, 0, 256, 10:17, 1] + + # With memory efficient attention + pipeline, params = FlaxStableDiffusionPipeline.from_pretrained( + "CompVis/stable-diffusion-v1-4", + revision="bf16", + dtype=jnp.bfloat16, + safety_checker=None, + use_memory_efficient_attention=True, + ) + + params = replicate(params) + prompt_ids = pipeline.prepare_inputs(prompt) + prompt_ids = shard(prompt_ids) + images_eff = pipeline(prompt_ids, params, prng_seed, jit=True).images + assert images_eff.shape == (num_samples, 1, 512, 512, 3) + slice_eff = images[2, 0, 256, 10:17, 1] + + # I checked the results visually and they are very similar. However, I saw that the max diff is `1` and the `sum` + # over the 8 images is exactly `256`, which is very suspicious. Testing a random slice for now. + assert abs(slice_eff - slice).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/test_pipelines_onnx_common.py b/diffusers-0.27.0/tests/pipelines/test_pipelines_onnx_common.py new file mode 100755 index 0000000..575ecd0 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/test_pipelines_onnx_common.py @@ -0,0 +1,12 @@ +from diffusers.utils.testing_utils import require_onnxruntime + + +@require_onnxruntime +class OnnxPipelineTesterMixin: + """ + This mixin is designed to be used with unittest.TestCase classes. + It provides a set of common tests for each ONNXRuntime pipeline, e.g. saving and loading the pipeline, + equivalence of dict and tuple outputs, etc. + """ + + pass diff --git a/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/__init__.py b/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video.py b/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video.py new file mode 100755 index 0000000..9dc4801 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video.py @@ -0,0 +1,215 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + TextToVideoSDPipeline, + UNet3DConditionModel, +) +from diffusers.utils import is_xformers_available +from diffusers.utils.testing_utils import ( + enable_full_determinism, + load_numpy, + numpy_cosine_similarity_distance, + require_torch_gpu, + skip_mps, + slow, + torch_device, +) + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import PipelineTesterMixin, SDFunctionTesterMixin + + +enable_full_determinism() + + +@skip_mps +class TextToVideoSDPipelineFastTests(PipelineTesterMixin, SDFunctionTesterMixin, unittest.TestCase): + pipeline_class = TextToVideoSDPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + # No `output_type`. + required_optional_params = frozenset( + [ + "num_inference_steps", + "generator", + "latents", + "return_dict", + "callback", + "callback_steps", + ] + ) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet3DConditionModel( + block_out_channels=(4, 8), + layers_per_block=1, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("CrossAttnDownBlock3D", "DownBlock3D"), + up_block_types=("UpBlock3D", "CrossAttnUpBlock3D"), + cross_attention_dim=4, + attention_head_dim=4, + norm_num_groups=2, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=(8,), + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D"], + latent_channels=4, + sample_size=32, + norm_num_groups=2, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=4, + intermediate_size=16, + layer_norm_eps=1e-05, + num_attention_heads=2, + num_hidden_layers=2, + pad_token_id=1, + vocab_size=1000, + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "pt", + } + return inputs + + def test_text_to_video_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = TextToVideoSDPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["output_type"] = "np" + frames = sd_pipe(**inputs).frames + + image_slice = frames[0][0][-3:, -3:, -1] + + assert frames[0][0].shape == (32, 32, 3) + expected_slice = np.array([0.7537, 0.1752, 0.6157, 0.5508, 0.4240, 0.4110, 0.4838, 0.5648, 0.5094]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + @unittest.skipIf(torch_device != "cuda", reason="Feature isn't heavily used. Test in CUDA environment only.") + def test_attention_slicing_forward_pass(self): + self._test_attention_slicing_forward_pass(test_mean_pixel_difference=False, expected_max_diff=3e-3) + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(test_mean_pixel_difference=False, expected_max_diff=1e-2) + + # (todo): sayakpaul + @unittest.skip(reason="Batching needs to be properly figured out first for this pipeline.") + def test_inference_batch_consistent(self): + pass + + # (todo): sayakpaul + @unittest.skip(reason="Batching needs to be properly figured out first for this pipeline.") + def test_inference_batch_single_identical(self): + pass + + @unittest.skip(reason="`num_images_per_prompt` argument is not supported for this pipeline.") + def test_num_images_per_prompt(self): + pass + + def test_progress_bar(self): + return super().test_progress_bar() + + +@slow +@skip_mps +@require_torch_gpu +class TextToVideoSDPipelineSlowTests(unittest.TestCase): + def test_two_step_model(self): + expected_video = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text-to-video/video_2step.npy" + ) + + pipe = TextToVideoSDPipeline.from_pretrained("damo-vilab/text-to-video-ms-1.7b") + pipe = pipe.to(torch_device) + + prompt = "Spiderman is surfing" + generator = torch.Generator(device="cpu").manual_seed(0) + + video_frames = pipe(prompt, generator=generator, num_inference_steps=2, output_type="np").frames + assert numpy_cosine_similarity_distance(expected_video.flatten(), video_frames.flatten()) < 1e-4 + + def test_two_step_model_with_freeu(self): + expected_video = [] + + pipe = TextToVideoSDPipeline.from_pretrained("damo-vilab/text-to-video-ms-1.7b") + pipe = pipe.to(torch_device) + + prompt = "Spiderman is surfing" + generator = torch.Generator(device="cpu").manual_seed(0) + + pipe.enable_freeu(s1=0.9, s2=0.2, b1=1.2, b2=1.4) + video_frames = pipe(prompt, generator=generator, num_inference_steps=2, output_type="np").frames + video = video_frames[0, 0, -3:, -3:, -1].flatten() + + expected_video = [0.3643, 0.3455, 0.3831, 0.3923, 0.2978, 0.3247, 0.3278, 0.3201, 0.3475] + + assert np.abs(expected_video - video).mean() < 5e-2 diff --git a/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video_zero.py b/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video_zero.py new file mode 100755 index 0000000..b93d9ee --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video_zero.py @@ -0,0 +1,42 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import torch + +from diffusers import DDIMScheduler, TextToVideoZeroPipeline +from diffusers.utils.testing_utils import load_pt, nightly, require_torch_gpu + +from ..test_pipelines_common import assert_mean_pixel_difference + + +@nightly +@require_torch_gpu +class TextToVideoZeroPipelineSlowTests(unittest.TestCase): + def test_full_model(self): + model_id = "runwayml/stable-diffusion-v1-5" + pipe = TextToVideoZeroPipeline.from_pretrained(model_id, torch_dtype=torch.float16).to("cuda") + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + generator = torch.Generator(device="cuda").manual_seed(0) + + prompt = "A bear is playing a guitar on Times Square" + result = pipe(prompt=prompt, generator=generator).images + + expected_result = load_pt( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/text-to-video/A bear is playing a guitar on Times Square.pt" + ) + + assert_mean_pixel_difference(result, expected_result) diff --git a/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video_zero_sdxl.py b/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video_zero_sdxl.py new file mode 100755 index 0000000..1d1d945 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_text_to_video_zero_sdxl.py @@ -0,0 +1,405 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import contextlib +import inspect +import io +import re +import tempfile +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import AutoencoderKL, DDIMScheduler, TextToVideoZeroSDXLPipeline, UNet2DConditionModel +from diffusers.utils.import_utils import is_accelerate_available, is_accelerate_version +from diffusers.utils.testing_utils import enable_full_determinism, nightly, require_torch_gpu, torch_device + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_IMAGE_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +def to_np(tensor): + if isinstance(tensor, torch.Tensor): + tensor = tensor.detach().cpu().numpy() + + return tensor + + +class TextToVideoZeroSDXLPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = TextToVideoZeroSDXLPipeline + params = TEXT_TO_IMAGE_PARAMS + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + image_params = TEXT_TO_IMAGE_IMAGE_PARAMS + image_latents_params = TEXT_TO_IMAGE_IMAGE_PARAMS + generator_device = "cpu" + + def get_dummy_components(self, seed=0): + torch.manual_seed(seed) + unet = UNet2DConditionModel( + block_out_channels=(2, 4), + layers_per_block=2, + sample_size=2, + norm_num_groups=2, + in_channels=4, + out_channels=4, + down_block_types=("DownBlock2D", "CrossAttnDownBlock2D"), + up_block_types=("CrossAttnUpBlock2D", "UpBlock2D"), + # SD2-specific config below + attention_head_dim=(2, 4), + use_linear_projection=True, + addition_embed_type="text_time", + addition_time_embed_dim=8, + transformer_layers_per_block=(1, 2), + projection_class_embeddings_input_dim=80, # 6 * 8 + 32 + cross_attention_dim=64, + ) + scheduler = DDIMScheduler( + num_train_timesteps=1000, + beta_start=0.0001, + beta_end=0.02, + beta_schedule="linear", + trained_betas=None, + clip_sample=True, + set_alpha_to_one=True, + steps_offset=0, + prediction_type="epsilon", + thresholding=False, + dynamic_thresholding_ratio=0.995, + clip_sample_range=1.0, + sample_max_value=1.0, + timestep_spacing="leading", + rescale_betas_zero_snr=False, + ) + torch.manual_seed(seed) + vae = AutoencoderKL( + block_out_channels=[32, 64], + in_channels=3, + out_channels=3, + down_block_types=["DownEncoderBlock2D", "DownEncoderBlock2D"], + up_block_types=["UpDecoderBlock2D", "UpDecoderBlock2D"], + latent_channels=4, + sample_size=128, + ) + torch.manual_seed(seed) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + # SD2-specific config below + hidden_act="gelu", + projection_dim=32, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_encoder_2 = CLIPTextModelWithProjection(text_encoder_config) + tokenizer_2 = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_encoder_2": text_encoder_2, + "tokenizer_2": tokenizer_2, + "image_encoder": None, + "feature_extractor": None, + } + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A panda dancing in Antarctica", + "generator": generator, + "num_inference_steps": 5, + "t0": 1, + "t1": 3, + "height": 64, + "width": 64, + "video_length": 3, + "output_type": "np", + } + return inputs + + def get_generator(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + return generator + + def test_text_to_video_zero_sdxl(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + + inputs = self.get_dummy_inputs(self.generator_device) + result = pipe(**inputs).images + + first_frame_slice = result[0, -3:, -3:, -1] + last_frame_slice = result[-1, -3:, -3:, 0] + + expected_slice1 = np.array([0.48, 0.58, 0.53, 0.59, 0.50, 0.44, 0.60, 0.65, 0.52]) + expected_slice2 = np.array([0.66, 0.49, 0.40, 0.70, 0.47, 0.51, 0.73, 0.65, 0.52]) + + assert np.abs(first_frame_slice.flatten() - expected_slice1).max() < 1e-2 + assert np.abs(last_frame_slice.flatten() - expected_slice2).max() < 1e-2 + + @unittest.skip( + reason="Cannot call `set_default_attn_processor` as this pipeline uses a specific attention processor." + ) + def test_attention_slicing_forward_pass(self): + pass + + def test_cfg(self): + sig = inspect.signature(self.pipeline_class.__call__) + if "guidance_scale" not in sig.parameters: + return + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(self.generator_device) + + inputs["guidance_scale"] = 1.0 + out_no_cfg = pipe(**inputs)[0] + + inputs["guidance_scale"] = 7.5 + out_cfg = pipe(**inputs)[0] + + assert out_cfg.shape == out_no_cfg.shape + + def test_dict_tuple_outputs_equivalent(self, expected_max_difference=1e-4): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(self.generator_device))[0] + output_tuple = pipe(**self.get_dummy_inputs(self.generator_device), return_dict=False)[0] + + max_diff = np.abs(to_np(output) - to_np(output_tuple)).max() + self.assertLess(max_diff, expected_max_difference) + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_float16_inference(self, expected_max_diff=5e-2): + components = self.get_dummy_components() + for name, module in components.items(): + if hasattr(module, "half"): + components[name] = module.to(torch_device).half() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + components = self.get_dummy_components() + pipe_fp16 = self.pipeline_class(**components) + pipe_fp16.to(torch_device, torch.float16) + pipe_fp16.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(self.generator_device) + # # Reset generator in case it is used inside dummy inputs + if "generator" in inputs: + inputs["generator"] = self.get_generator(self.generator_device) + + output = pipe(**inputs)[0] + + fp16_inputs = self.get_dummy_inputs(self.generator_device) + # Reset generator in case it is used inside dummy inputs + if "generator" in fp16_inputs: + fp16_inputs["generator"] = self.get_generator(self.generator_device) + + output_fp16 = pipe_fp16(**fp16_inputs)[0] + + max_diff = np.abs(to_np(output) - to_np(output_fp16)).max() + self.assertLess(max_diff, expected_max_diff, "The outputs of the fp16 and fp32 pipelines are too different.") + + @unittest.skip(reason="Batching needs to be properly figured out first for this pipeline.") + def test_inference_batch_consistent(self): + pass + + @unittest.skip( + reason="Cannot call `set_default_attn_processor` as this pipeline uses a specific attention processor." + ) + def test_inference_batch_single_identical(self): + pass + + @unittest.skipIf( + torch_device != "cuda" or not is_accelerate_available() or is_accelerate_version("<", "0.17.0"), + reason="CPU offload is only available with CUDA and `accelerate v0.17.0` or higher", + ) + def test_model_cpu_offload_forward_pass(self, expected_max_diff=2e-4): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(self.generator_device) + output_without_offload = pipe(**inputs)[0] + + pipe.enable_model_cpu_offload() + inputs = self.get_dummy_inputs(self.generator_device) + output_with_offload = pipe(**inputs)[0] + + max_diff = np.abs(to_np(output_with_offload) - to_np(output_without_offload)).max() + self.assertLess(max_diff, expected_max_diff, "CPU offloading should not affect the inference results") + + @unittest.skip(reason="`num_images_per_prompt` argument is not supported for this pipeline.") + def test_pipeline_call_signature(self): + pass + + def test_progress_bar(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + + inputs = self.get_dummy_inputs(self.generator_device) + with io.StringIO() as stderr, contextlib.redirect_stderr(stderr): + _ = pipe(**inputs) + stderr = stderr.getvalue() + # we can't calculate the number of progress steps beforehand e.g. for strength-dependent img2img, + # so we just match "5" in "#####| 1/5 [00:01<00:00]" + max_steps = re.search("/(.*?) ", stderr).group(1) + self.assertTrue(max_steps is not None and len(max_steps) > 0) + self.assertTrue( + f"{max_steps}/{max_steps}" in stderr, "Progress bar should be enabled and stopped at the max step" + ) + + pipe.set_progress_bar_config(disable=True) + with io.StringIO() as stderr, contextlib.redirect_stderr(stderr): + _ = pipe(**inputs) + self.assertTrue(stderr.getvalue() == "", "Progress bar should be disabled") + + @unittest.skipIf(torch_device != "cuda", reason="float16 requires CUDA") + def test_save_load_float16(self, expected_max_diff=1e-2): + components = self.get_dummy_components() + for name, module in components.items(): + if hasattr(module, "half"): + components[name] = module.to(torch_device).half() + + pipe = self.pipeline_class(**components) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(self.generator_device) + output = pipe(**inputs)[0] + + with tempfile.TemporaryDirectory() as tmpdir: + pipe.save_pretrained(tmpdir) + pipe_loaded = self.pipeline_class.from_pretrained(tmpdir, torch_dtype=torch.float16) + pipe_loaded.to(torch_device) + pipe_loaded.set_progress_bar_config(disable=None) + + for name, component in pipe_loaded.components.items(): + if hasattr(component, "dtype"): + self.assertTrue( + component.dtype == torch.float16, + f"`{name}.dtype` switched from `float16` to {component.dtype} after loading.", + ) + + inputs = self.get_dummy_inputs(self.generator_device) + output_loaded = pipe_loaded(**inputs)[0] + max_diff = np.abs(to_np(output) - to_np(output_loaded)).max() + self.assertLess( + max_diff, expected_max_diff, "The output of the fp16 pipeline changed after saving and loading." + ) + + @unittest.skip( + reason="Cannot call `set_default_attn_processor` as this pipeline uses a specific attention processor." + ) + def test_save_load_local(self): + pass + + @unittest.skip( + reason="Cannot call `set_default_attn_processor` as this pipeline uses a specific attention processor." + ) + def test_save_load_optional_components(self): + pass + + @unittest.skip( + reason="Cannot call `set_default_attn_processor` as this pipeline uses a specific attention processor." + ) + def test_sequential_cpu_offload_forward_pass(self): + pass + + @unittest.skipIf(torch_device != "cuda", reason="CUDA and CPU are required to switch devices") + def test_to_device(self): + components = self.get_dummy_components() + pipe = self.pipeline_class(**components) + pipe.set_progress_bar_config(disable=None) + + pipe.to("cpu") + model_devices = [component.device.type for component in components.values() if hasattr(component, "device")] + self.assertTrue(all(device == "cpu" for device in model_devices)) + + output_cpu = pipe(**self.get_dummy_inputs("cpu"))[0] # generator set to cpu + self.assertTrue(np.isnan(output_cpu).sum() == 0) + + pipe.to("cuda") + model_devices = [component.device.type for component in components.values() if hasattr(component, "device")] + self.assertTrue(all(device == "cuda" for device in model_devices)) + + output_cuda = pipe(**self.get_dummy_inputs("cpu"))[0] # generator set to cpu + self.assertTrue(np.isnan(to_np(output_cuda)).sum() == 0) + + @unittest.skip( + reason="Cannot call `set_default_attn_processor` as this pipeline uses a specific attention processor." + ) + def test_xformers_attention_forwardGenerator_pass(self): + pass + + +@nightly +@require_torch_gpu +class TextToVideoZeroSDXLPipelineSlowTests(unittest.TestCase): + def test_full_model(self): + model_id = "stabilityai/stable-diffusion-xl-base-1.0" + pipe = TextToVideoZeroSDXLPipeline.from_pretrained( + model_id, torch_dtype=torch.float16, variant="fp16", use_safetensors=True + ) + pipe.enable_model_cpu_offload() + pipe.enable_vae_slicing() + + pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + generator = torch.Generator(device="cpu").manual_seed(0) + + prompt = "A panda dancing in Antarctica" + result = pipe(prompt=prompt, generator=generator).images + + first_frame_slice = result[0, -3:, -3:, -1] + last_frame_slice = result[-1, -3:, -3:, 0] + + expected_slice1 = np.array([0.57, 0.57, 0.57, 0.57, 0.57, 0.56, 0.55, 0.56, 0.56]) + expected_slice2 = np.array([0.54, 0.53, 0.53, 0.53, 0.53, 0.52, 0.53, 0.53, 0.53]) + + assert np.abs(first_frame_slice.flatten() - expected_slice1).max() < 1e-2 + assert np.abs(last_frame_slice.flatten() - expected_slice2).max() < 1e-2 diff --git a/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_video_to_video.py b/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_video_to_video.py new file mode 100755 index 0000000..7f28d12 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/text_to_video_synthesis/test_video_to_video.py @@ -0,0 +1,224 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import random +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import ( + AutoencoderKL, + DDIMScheduler, + UNet3DConditionModel, + VideoToVideoSDPipeline, +) +from diffusers.utils import is_xformers_available +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + is_flaky, + nightly, + numpy_cosine_similarity_distance, + skip_mps, + torch_device, +) + +from ..pipeline_params import ( + TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, +) +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +@skip_mps +class VideoToVideoSDPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = VideoToVideoSDPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS.union({"video"}) - {"image", "width", "height"} + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS.union({"video"}) - {"image"} + required_optional_params = PipelineTesterMixin.required_optional_params - {"latents"} + test_attention_slicing = False + + # No `output_type`. + required_optional_params = frozenset( + [ + "num_inference_steps", + "generator", + "latents", + "return_dict", + "callback", + "callback_steps", + ] + ) + + def get_dummy_components(self): + torch.manual_seed(0) + unet = UNet3DConditionModel( + block_out_channels=(4, 8), + layers_per_block=1, + sample_size=32, + in_channels=4, + out_channels=4, + down_block_types=("CrossAttnDownBlock3D", "DownBlock3D"), + up_block_types=("UpBlock3D", "CrossAttnUpBlock3D"), + cross_attention_dim=32, + attention_head_dim=4, + norm_num_groups=2, + ) + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=True, + set_alpha_to_one=False, + ) + torch.manual_seed(0) + vae = AutoencoderKL( + block_out_channels=[ + 8, + ], + in_channels=3, + out_channels=3, + down_block_types=[ + "DownEncoderBlock2D", + ], + up_block_types=["UpDecoderBlock2D"], + latent_channels=4, + sample_size=32, + norm_num_groups=2, + ) + torch.manual_seed(0) + text_encoder_config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=32, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + hidden_act="gelu", + projection_dim=512, + ) + text_encoder = CLIPTextModel(text_encoder_config) + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + + components = { + "unet": unet, + "scheduler": scheduler, + "vae": vae, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + } + return components + + def get_dummy_inputs(self, device, seed=0): + # 3 frames + video = floats_tensor((1, 3, 3, 32, 32), rng=random.Random(seed)).to(device) + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "A painting of a squirrel eating a burger", + "video": video, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "pt", + } + return inputs + + def test_text_to_video_default_case(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + sd_pipe = VideoToVideoSDPipeline(**components) + sd_pipe = sd_pipe.to(device) + sd_pipe.set_progress_bar_config(disable=None) + + inputs = self.get_dummy_inputs(device) + inputs["output_type"] = "np" + frames = sd_pipe(**inputs).frames + image_slice = frames[0][0][-3:, -3:, -1] + + assert frames[0][0].shape == (32, 32, 3) + expected_slice = np.array([0.6391, 0.5350, 0.5202, 0.5521, 0.5453, 0.5393, 0.6652, 0.5270, 0.5185]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + + @is_flaky() + def test_save_load_optional_components(self): + super().test_save_load_optional_components(expected_max_difference=0.001) + + @is_flaky() + def test_dict_tuple_outputs_equivalent(self): + super().test_dict_tuple_outputs_equivalent() + + @is_flaky() + def test_save_load_local(self): + super().test_save_load_local() + + @unittest.skipIf( + torch_device != "cuda" or not is_xformers_available(), + reason="XFormers attention is only available with CUDA and `xformers` installed", + ) + def test_xformers_attention_forwardGenerator_pass(self): + self._test_xformers_attention_forwardGenerator_pass(test_mean_pixel_difference=False, expected_max_diff=5e-3) + + # (todo): sayakpaul + @unittest.skip(reason="Batching needs to be properly figured out first for this pipeline.") + def test_inference_batch_consistent(self): + pass + + # (todo): sayakpaul + @unittest.skip(reason="Batching needs to be properly figured out first for this pipeline.") + def test_inference_batch_single_identical(self): + pass + + @unittest.skip(reason="`num_images_per_prompt` argument is not supported for this pipeline.") + def test_num_images_per_prompt(self): + pass + + def test_progress_bar(self): + return super().test_progress_bar() + + +@nightly +@skip_mps +class VideoToVideoSDPipelineSlowTests(unittest.TestCase): + def test_two_step_model(self): + pipe = VideoToVideoSDPipeline.from_pretrained("cerspense/zeroscope_v2_576w", torch_dtype=torch.float16) + pipe.enable_model_cpu_offload() + + # 10 frames + generator = torch.Generator(device="cpu").manual_seed(0) + video = torch.randn((1, 10, 3, 320, 576), generator=generator) + + prompt = "Spiderman is surfing" + + generator = torch.Generator(device="cpu").manual_seed(0) + video_frames = pipe(prompt, video=video, generator=generator, num_inference_steps=3, output_type="np").frames + + expected_array = np.array( + [0.17114258, 0.13720703, 0.08886719, 0.14819336, 0.1730957, 0.24584961, 0.22021484, 0.35180664, 0.2607422] + ) + output_array = video_frames[0, 0, :3, :3, 0].flatten() + assert numpy_cosine_similarity_distance(expected_array, output_array) < 1e-3 diff --git a/diffusers-0.27.0/tests/pipelines/unclip/__init__.py b/diffusers-0.27.0/tests/pipelines/unclip/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/unclip/test_unclip.py b/diffusers-0.27.0/tests/pipelines/unclip/test_unclip.py new file mode 100755 index 0000000..60c5c52 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/unclip/test_unclip.py @@ -0,0 +1,507 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModelWithProjection, CLIPTokenizer + +from diffusers import PriorTransformer, UnCLIPPipeline, UnCLIPScheduler, UNet2DConditionModel, UNet2DModel +from diffusers.pipelines.unclip.text_proj import UnCLIPTextProjModel +from diffusers.utils.testing_utils import ( + enable_full_determinism, + load_numpy, + nightly, + require_torch_gpu, + skip_mps, + torch_device, +) + +from ..pipeline_params import TEXT_TO_IMAGE_BATCH_PARAMS, TEXT_TO_IMAGE_PARAMS +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +enable_full_determinism() + + +class UnCLIPPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = UnCLIPPipeline + params = TEXT_TO_IMAGE_PARAMS - { + "negative_prompt", + "height", + "width", + "negative_prompt_embeds", + "guidance_scale", + "prompt_embeds", + "cross_attention_kwargs", + } + batch_params = TEXT_TO_IMAGE_BATCH_PARAMS + required_optional_params = [ + "generator", + "return_dict", + "prior_num_inference_steps", + "decoder_num_inference_steps", + "super_res_num_inference_steps", + ] + test_xformers_attention = False + + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 100 + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=self.text_embedder_hidden_size, + projection_dim=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModelWithProjection(config) + + @property + def dummy_prior(self): + torch.manual_seed(0) + + model_kwargs = { + "num_attention_heads": 2, + "attention_head_dim": 12, + "embedding_dim": self.text_embedder_hidden_size, + "num_layers": 1, + } + + model = PriorTransformer(**model_kwargs) + return model + + @property + def dummy_text_proj(self): + torch.manual_seed(0) + + model_kwargs = { + "clip_embeddings_dim": self.text_embedder_hidden_size, + "time_embed_dim": self.time_embed_dim, + "cross_attention_dim": self.cross_attention_dim, + } + + model = UnCLIPTextProjModel(**model_kwargs) + return model + + @property + def dummy_decoder(self): + torch.manual_seed(0) + + model_kwargs = { + "sample_size": 32, + # RGB in channels + "in_channels": 3, + # Out channels is double in channels because predicts mean and variance + "out_channels": 6, + "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"), + "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"), + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "layers_per_block": 1, + "cross_attention_dim": self.cross_attention_dim, + "attention_head_dim": 4, + "resnet_time_scale_shift": "scale_shift", + "class_embed_type": "identity", + } + + model = UNet2DConditionModel(**model_kwargs) + return model + + @property + def dummy_super_res_kwargs(self): + return { + "sample_size": 64, + "layers_per_block": 1, + "down_block_types": ("ResnetDownsampleBlock2D", "ResnetDownsampleBlock2D"), + "up_block_types": ("ResnetUpsampleBlock2D", "ResnetUpsampleBlock2D"), + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "in_channels": 6, + "out_channels": 3, + } + + @property + def dummy_super_res_first(self): + torch.manual_seed(0) + + model = UNet2DModel(**self.dummy_super_res_kwargs) + return model + + @property + def dummy_super_res_last(self): + # seeded differently to get different unet than `self.dummy_super_res_first` + torch.manual_seed(1) + + model = UNet2DModel(**self.dummy_super_res_kwargs) + return model + + def get_dummy_components(self): + prior = self.dummy_prior + decoder = self.dummy_decoder + text_proj = self.dummy_text_proj + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + super_res_first = self.dummy_super_res_first + super_res_last = self.dummy_super_res_last + + prior_scheduler = UnCLIPScheduler( + variance_type="fixed_small_log", + prediction_type="sample", + num_train_timesteps=1000, + clip_sample_range=5.0, + ) + + decoder_scheduler = UnCLIPScheduler( + variance_type="learned_range", + prediction_type="epsilon", + num_train_timesteps=1000, + ) + + super_res_scheduler = UnCLIPScheduler( + variance_type="fixed_small_log", + prediction_type="epsilon", + num_train_timesteps=1000, + ) + + components = { + "prior": prior, + "decoder": decoder, + "text_proj": text_proj, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "super_res_first": super_res_first, + "super_res_last": super_res_last, + "prior_scheduler": prior_scheduler, + "decoder_scheduler": decoder_scheduler, + "super_res_scheduler": super_res_scheduler, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "horse", + "generator": generator, + "prior_num_inference_steps": 2, + "decoder_num_inference_steps": 2, + "super_res_num_inference_steps": 2, + "output_type": "numpy", + } + return inputs + + def test_unclip(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe( + **self.get_dummy_inputs(device), + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array( + [ + 0.9997, + 0.9988, + 0.0028, + 0.9997, + 0.9984, + 0.9965, + 0.0029, + 0.9986, + 0.0025, + ] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + def test_unclip_passed_text_embed(self): + device = torch.device("cpu") + + class DummyScheduler: + init_noise_sigma = 1 + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + prior = components["prior"] + decoder = components["decoder"] + super_res_first = components["super_res_first"] + tokenizer = components["tokenizer"] + text_encoder = components["text_encoder"] + + generator = torch.Generator(device=device).manual_seed(0) + dtype = prior.dtype + batch_size = 1 + + shape = (batch_size, prior.config.embedding_dim) + prior_latents = pipe.prepare_latents( + shape, dtype=dtype, device=device, generator=generator, latents=None, scheduler=DummyScheduler() + ) + shape = (batch_size, decoder.config.in_channels, decoder.config.sample_size, decoder.config.sample_size) + decoder_latents = pipe.prepare_latents( + shape, dtype=dtype, device=device, generator=generator, latents=None, scheduler=DummyScheduler() + ) + + shape = ( + batch_size, + super_res_first.config.in_channels // 2, + super_res_first.config.sample_size, + super_res_first.config.sample_size, + ) + super_res_latents = pipe.prepare_latents( + shape, dtype=dtype, device=device, generator=generator, latents=None, scheduler=DummyScheduler() + ) + + pipe.set_progress_bar_config(disable=None) + + prompt = "this is a prompt example" + + generator = torch.Generator(device=device).manual_seed(0) + output = pipe( + [prompt], + generator=generator, + prior_num_inference_steps=2, + decoder_num_inference_steps=2, + super_res_num_inference_steps=2, + prior_latents=prior_latents, + decoder_latents=decoder_latents, + super_res_latents=super_res_latents, + output_type="np", + ) + image = output.images + + text_inputs = tokenizer( + prompt, + padding="max_length", + max_length=tokenizer.model_max_length, + return_tensors="pt", + ) + text_model_output = text_encoder(text_inputs.input_ids) + text_attention_mask = text_inputs.attention_mask + + generator = torch.Generator(device=device).manual_seed(0) + image_from_text = pipe( + generator=generator, + prior_num_inference_steps=2, + decoder_num_inference_steps=2, + super_res_num_inference_steps=2, + prior_latents=prior_latents, + decoder_latents=decoder_latents, + super_res_latents=super_res_latents, + text_model_output=text_model_output, + text_attention_mask=text_attention_mask, + output_type="np", + )[0] + + # make sure passing text embeddings manually is identical + assert np.abs(image - image_from_text).max() < 1e-4 + + # Overriding PipelineTesterMixin::test_attention_slicing_forward_pass + # because UnCLIP GPU undeterminism requires a looser check. + @skip_mps + def test_attention_slicing_forward_pass(self): + test_max_difference = torch_device == "cpu" + + self._test_attention_slicing_forward_pass(test_max_difference=test_max_difference, expected_max_diff=0.01) + + # Overriding PipelineTesterMixin::test_inference_batch_single_identical + # because UnCLIP undeterminism requires a looser check. + @skip_mps + def test_inference_batch_single_identical(self): + additional_params_copy_to_batched_inputs = [ + "prior_num_inference_steps", + "decoder_num_inference_steps", + "super_res_num_inference_steps", + ] + + self._test_inference_batch_single_identical( + additional_params_copy_to_batched_inputs=additional_params_copy_to_batched_inputs, expected_max_diff=5e-3 + ) + + def test_inference_batch_consistent(self): + additional_params_copy_to_batched_inputs = [ + "prior_num_inference_steps", + "decoder_num_inference_steps", + "super_res_num_inference_steps", + ] + + if torch_device == "mps": + # TODO: MPS errors with larger batch sizes + batch_sizes = [2, 3] + self._test_inference_batch_consistent( + batch_sizes=batch_sizes, + additional_params_copy_to_batched_inputs=additional_params_copy_to_batched_inputs, + ) + else: + self._test_inference_batch_consistent( + additional_params_copy_to_batched_inputs=additional_params_copy_to_batched_inputs + ) + + @skip_mps + def test_dict_tuple_outputs_equivalent(self): + return super().test_dict_tuple_outputs_equivalent() + + @skip_mps + def test_save_load_local(self): + return super().test_save_load_local(expected_max_difference=5e-3) + + @skip_mps + def test_save_load_optional_components(self): + return super().test_save_load_optional_components() + + @unittest.skip("UnCLIP produces very large differences in fp16 vs fp32. Test is not useful.") + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=1.0) + + +@nightly +class UnCLIPPipelineCPUIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_unclip_karlo_cpu_fp32(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/unclip/karlo_v1_alpha_horse_cpu.npy" + ) + + pipeline = UnCLIPPipeline.from_pretrained("kakaobrain/karlo-v1-alpha") + pipeline.set_progress_bar_config(disable=None) + + generator = torch.manual_seed(0) + output = pipeline( + "horse", + num_images_per_prompt=1, + generator=generator, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (256, 256, 3) + assert np.abs(expected_image - image).max() < 1e-1 + + +@nightly +@require_torch_gpu +class UnCLIPPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_unclip_karlo(self): + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/unclip/karlo_v1_alpha_horse_fp16.npy" + ) + + pipeline = UnCLIPPipeline.from_pretrained("kakaobrain/karlo-v1-alpha", torch_dtype=torch.float16) + pipeline = pipeline.to(torch_device) + pipeline.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipeline( + "horse", + generator=generator, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (256, 256, 3) + + assert_mean_pixel_difference(image, expected_image) + + def test_unclip_pipeline_with_sequential_cpu_offloading(self): + torch.cuda.empty_cache() + torch.cuda.reset_max_memory_allocated() + torch.cuda.reset_peak_memory_stats() + + pipe = UnCLIPPipeline.from_pretrained("kakaobrain/karlo-v1-alpha", torch_dtype=torch.float16) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + pipe.enable_sequential_cpu_offload() + + _ = pipe( + "horse", + num_images_per_prompt=1, + prior_num_inference_steps=2, + decoder_num_inference_steps=2, + super_res_num_inference_steps=2, + output_type="np", + ) + + mem_bytes = torch.cuda.max_memory_allocated() + # make sure that less than 7 GB is allocated + assert mem_bytes < 7 * 10**9 diff --git a/diffusers-0.27.0/tests/pipelines/unclip/test_unclip_image_variation.py b/diffusers-0.27.0/tests/pipelines/unclip/test_unclip_image_variation.py new file mode 100755 index 0000000..ab3aea5 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/unclip/test_unclip_image_variation.py @@ -0,0 +1,531 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc +import random +import unittest + +import numpy as np +import torch +from transformers import ( + CLIPImageProcessor, + CLIPTextConfig, + CLIPTextModelWithProjection, + CLIPTokenizer, + CLIPVisionConfig, + CLIPVisionModelWithProjection, +) + +from diffusers import ( + DiffusionPipeline, + UnCLIPImageVariationPipeline, + UnCLIPScheduler, + UNet2DConditionModel, + UNet2DModel, +) +from diffusers.pipelines.unclip.text_proj import UnCLIPTextProjModel +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + load_numpy, + nightly, + require_torch_gpu, + skip_mps, + torch_device, +) + +from ..pipeline_params import IMAGE_VARIATION_BATCH_PARAMS, IMAGE_VARIATION_PARAMS +from ..test_pipelines_common import PipelineTesterMixin, assert_mean_pixel_difference + + +enable_full_determinism() + + +class UnCLIPImageVariationPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = UnCLIPImageVariationPipeline + params = IMAGE_VARIATION_PARAMS - {"height", "width", "guidance_scale"} + batch_params = IMAGE_VARIATION_BATCH_PARAMS + + required_optional_params = [ + "generator", + "return_dict", + "decoder_num_inference_steps", + "super_res_num_inference_steps", + ] + test_xformers_attention = False + + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def cross_attention_dim(self): + return 100 + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=self.text_embedder_hidden_size, + projection_dim=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModelWithProjection(config) + + @property + def dummy_image_encoder(self): + torch.manual_seed(0) + config = CLIPVisionConfig( + hidden_size=self.text_embedder_hidden_size, + projection_dim=self.text_embedder_hidden_size, + num_hidden_layers=5, + num_attention_heads=4, + image_size=32, + intermediate_size=37, + patch_size=1, + ) + return CLIPVisionModelWithProjection(config) + + @property + def dummy_text_proj(self): + torch.manual_seed(0) + + model_kwargs = { + "clip_embeddings_dim": self.text_embedder_hidden_size, + "time_embed_dim": self.time_embed_dim, + "cross_attention_dim": self.cross_attention_dim, + } + + model = UnCLIPTextProjModel(**model_kwargs) + return model + + @property + def dummy_decoder(self): + torch.manual_seed(0) + + model_kwargs = { + "sample_size": 32, + # RGB in channels + "in_channels": 3, + # Out channels is double in channels because predicts mean and variance + "out_channels": 6, + "down_block_types": ("ResnetDownsampleBlock2D", "SimpleCrossAttnDownBlock2D"), + "up_block_types": ("SimpleCrossAttnUpBlock2D", "ResnetUpsampleBlock2D"), + "mid_block_type": "UNetMidBlock2DSimpleCrossAttn", + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "layers_per_block": 1, + "cross_attention_dim": self.cross_attention_dim, + "attention_head_dim": 4, + "resnet_time_scale_shift": "scale_shift", + "class_embed_type": "identity", + } + + model = UNet2DConditionModel(**model_kwargs) + return model + + @property + def dummy_super_res_kwargs(self): + return { + "sample_size": 64, + "layers_per_block": 1, + "down_block_types": ("ResnetDownsampleBlock2D", "ResnetDownsampleBlock2D"), + "up_block_types": ("ResnetUpsampleBlock2D", "ResnetUpsampleBlock2D"), + "block_out_channels": (self.block_out_channels_0, self.block_out_channels_0 * 2), + "in_channels": 6, + "out_channels": 3, + } + + @property + def dummy_super_res_first(self): + torch.manual_seed(0) + + model = UNet2DModel(**self.dummy_super_res_kwargs) + return model + + @property + def dummy_super_res_last(self): + # seeded differently to get different unet than `self.dummy_super_res_first` + torch.manual_seed(1) + + model = UNet2DModel(**self.dummy_super_res_kwargs) + return model + + def get_dummy_components(self): + decoder = self.dummy_decoder + text_proj = self.dummy_text_proj + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + super_res_first = self.dummy_super_res_first + super_res_last = self.dummy_super_res_last + + decoder_scheduler = UnCLIPScheduler( + variance_type="learned_range", + prediction_type="epsilon", + num_train_timesteps=1000, + ) + + super_res_scheduler = UnCLIPScheduler( + variance_type="fixed_small_log", + prediction_type="epsilon", + num_train_timesteps=1000, + ) + + feature_extractor = CLIPImageProcessor(crop_size=32, size=32) + + image_encoder = self.dummy_image_encoder + + return { + "decoder": decoder, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "text_proj": text_proj, + "feature_extractor": feature_extractor, + "image_encoder": image_encoder, + "super_res_first": super_res_first, + "super_res_last": super_res_last, + "decoder_scheduler": decoder_scheduler, + "super_res_scheduler": super_res_scheduler, + } + + def get_dummy_inputs(self, device, seed=0, pil_image=True): + input_image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + if pil_image: + input_image = input_image * 0.5 + 0.5 + input_image = input_image.clamp(0, 1) + input_image = input_image.cpu().permute(0, 2, 3, 1).float().numpy() + input_image = DiffusionPipeline.numpy_to_pil(input_image)[0] + + return { + "image": input_image, + "generator": generator, + "decoder_num_inference_steps": 2, + "super_res_num_inference_steps": 2, + "output_type": "np", + } + + def test_unclip_image_variation_input_tensor(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + pipeline_inputs = self.get_dummy_inputs(device, pil_image=False) + + output = pipe(**pipeline_inputs) + image = output.images + + tuple_pipeline_inputs = self.get_dummy_inputs(device, pil_image=False) + + image_from_tuple = pipe( + **tuple_pipeline_inputs, + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array( + [ + 0.9997, + 0.0002, + 0.9997, + 0.9997, + 0.9969, + 0.0023, + 0.9997, + 0.9969, + 0.9970, + ] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + def test_unclip_image_variation_input_image(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + pipeline_inputs = self.get_dummy_inputs(device, pil_image=True) + + output = pipe(**pipeline_inputs) + image = output.images + + tuple_pipeline_inputs = self.get_dummy_inputs(device, pil_image=True) + + image_from_tuple = pipe( + **tuple_pipeline_inputs, + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.9997, 0.0003, 0.9997, 0.9997, 0.9970, 0.0024, 0.9997, 0.9971, 0.9971]) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + def test_unclip_image_variation_input_list_images(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + pipeline_inputs = self.get_dummy_inputs(device, pil_image=True) + pipeline_inputs["image"] = [ + pipeline_inputs["image"], + pipeline_inputs["image"], + ] + + output = pipe(**pipeline_inputs) + image = output.images + + tuple_pipeline_inputs = self.get_dummy_inputs(device, pil_image=True) + tuple_pipeline_inputs["image"] = [ + tuple_pipeline_inputs["image"], + tuple_pipeline_inputs["image"], + ] + + image_from_tuple = pipe( + **tuple_pipeline_inputs, + return_dict=False, + )[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (2, 64, 64, 3) + + expected_slice = np.array( + [ + 0.9997, + 0.9989, + 0.0008, + 0.0021, + 0.9960, + 0.0018, + 0.0014, + 0.0002, + 0.9933, + ] + ) + + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + def test_unclip_passed_image_embed(self): + device = torch.device("cpu") + + class DummyScheduler: + init_noise_sigma = 1 + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + generator = torch.Generator(device=device).manual_seed(0) + dtype = pipe.decoder.dtype + batch_size = 1 + + shape = ( + batch_size, + pipe.decoder.config.in_channels, + pipe.decoder.config.sample_size, + pipe.decoder.config.sample_size, + ) + decoder_latents = pipe.prepare_latents( + shape, dtype=dtype, device=device, generator=generator, latents=None, scheduler=DummyScheduler() + ) + + shape = ( + batch_size, + pipe.super_res_first.config.in_channels // 2, + pipe.super_res_first.config.sample_size, + pipe.super_res_first.config.sample_size, + ) + super_res_latents = pipe.prepare_latents( + shape, dtype=dtype, device=device, generator=generator, latents=None, scheduler=DummyScheduler() + ) + + pipeline_inputs = self.get_dummy_inputs(device, pil_image=False) + + img_out_1 = pipe( + **pipeline_inputs, decoder_latents=decoder_latents, super_res_latents=super_res_latents + ).images + + pipeline_inputs = self.get_dummy_inputs(device, pil_image=False) + # Don't pass image, instead pass embedding + image = pipeline_inputs.pop("image") + image_embeddings = pipe.image_encoder(image).image_embeds + + img_out_2 = pipe( + **pipeline_inputs, + decoder_latents=decoder_latents, + super_res_latents=super_res_latents, + image_embeddings=image_embeddings, + ).images + + # make sure passing text embeddings manually is identical + assert np.abs(img_out_1 - img_out_2).max() < 1e-4 + + # Overriding PipelineTesterMixin::test_attention_slicing_forward_pass + # because UnCLIP GPU undeterminism requires a looser check. + @skip_mps + def test_attention_slicing_forward_pass(self): + test_max_difference = torch_device == "cpu" + + # Check is relaxed because there is not a torch 2.0 sliced attention added kv processor + expected_max_diff = 1e-2 + + self._test_attention_slicing_forward_pass( + test_max_difference=test_max_difference, expected_max_diff=expected_max_diff + ) + + # Overriding PipelineTesterMixin::test_inference_batch_single_identical + # because UnCLIP undeterminism requires a looser check. + @unittest.skip("UnCLIP produces very large differences. Test is not useful.") + @skip_mps + def test_inference_batch_single_identical(self): + additional_params_copy_to_batched_inputs = [ + "decoder_num_inference_steps", + "super_res_num_inference_steps", + ] + self._test_inference_batch_single_identical( + additional_params_copy_to_batched_inputs=additional_params_copy_to_batched_inputs, expected_max_diff=5e-3 + ) + + def test_inference_batch_consistent(self): + additional_params_copy_to_batched_inputs = [ + "decoder_num_inference_steps", + "super_res_num_inference_steps", + ] + + if torch_device == "mps": + # TODO: MPS errors with larger batch sizes + batch_sizes = [2, 3] + self._test_inference_batch_consistent( + batch_sizes=batch_sizes, + additional_params_copy_to_batched_inputs=additional_params_copy_to_batched_inputs, + ) + else: + self._test_inference_batch_consistent( + additional_params_copy_to_batched_inputs=additional_params_copy_to_batched_inputs + ) + + @skip_mps + def test_dict_tuple_outputs_equivalent(self): + return super().test_dict_tuple_outputs_equivalent() + + @unittest.skip("UnCLIP produces very large difference. Test is not useful.") + @skip_mps + def test_save_load_local(self): + return super().test_save_load_local(expected_max_difference=4e-3) + + @skip_mps + def test_save_load_optional_components(self): + return super().test_save_load_optional_components() + + @unittest.skip("UnCLIP produces very large difference in fp16 vs fp32. Test is not useful.") + def test_float16_inference(self): + super().test_float16_inference(expected_max_diff=1.0) + + +@nightly +@require_torch_gpu +class UnCLIPImageVariationPipelineIntegrationTests(unittest.TestCase): + def tearDown(self): + # clean up the VRAM after each test + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def test_unclip_image_variation_karlo(self): + input_image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/unclip/cat.png" + ) + expected_image = load_numpy( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main" + "/unclip/karlo_v1_alpha_cat_variation_fp16.npy" + ) + + pipeline = UnCLIPImageVariationPipeline.from_pretrained( + "kakaobrain/karlo-v1-alpha-image-variations", torch_dtype=torch.float16 + ) + pipeline = pipeline.to(torch_device) + pipeline.set_progress_bar_config(disable=None) + + generator = torch.Generator(device="cpu").manual_seed(0) + output = pipeline( + input_image, + generator=generator, + output_type="np", + ) + + image = output.images[0] + + assert image.shape == (256, 256, 3) + + assert_mean_pixel_difference(image, expected_image, 15) diff --git a/diffusers-0.27.0/tests/pipelines/unidiffuser/__init__.py b/diffusers-0.27.0/tests/pipelines/unidiffuser/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/unidiffuser/test_unidiffuser.py b/diffusers-0.27.0/tests/pipelines/unidiffuser/test_unidiffuser.py new file mode 100755 index 0000000..ba8026d --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/unidiffuser/test_unidiffuser.py @@ -0,0 +1,790 @@ +import gc +import random +import traceback +import unittest + +import numpy as np +import torch +from PIL import Image +from transformers import ( + CLIPImageProcessor, + CLIPTextModel, + CLIPTokenizer, + CLIPVisionModelWithProjection, + GPT2Tokenizer, +) + +from diffusers import ( + AutoencoderKL, + DPMSolverMultistepScheduler, + UniDiffuserModel, + UniDiffuserPipeline, + UniDiffuserTextDecoder, +) +from diffusers.utils.testing_utils import ( + enable_full_determinism, + floats_tensor, + load_image, + nightly, + require_torch_2, + require_torch_gpu, + run_test_in_subprocess, + torch_device, +) +from diffusers.utils.torch_utils import randn_tensor + +from ..pipeline_params import ( + IMAGE_TO_IMAGE_IMAGE_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS, + TEXT_GUIDED_IMAGE_VARIATION_PARAMS, +) +from ..test_pipelines_common import PipelineKarrasSchedulerTesterMixin, PipelineLatentTesterMixin, PipelineTesterMixin + + +enable_full_determinism() + + +# Will be run via run_test_in_subprocess +def _test_unidiffuser_compile(in_queue, out_queue, timeout): + error = None + try: + inputs = in_queue.get(timeout=timeout) + torch_device = inputs.pop("torch_device") + seed = inputs.pop("seed") + inputs["generator"] = torch.Generator(device=torch_device).manual_seed(seed) + + pipe = UniDiffuserPipeline.from_pretrained("thu-ml/unidiffuser-v1") + # pipe.scheduler = DDIMScheduler.from_config(pipe.scheduler.config) + pipe = pipe.to(torch_device) + + pipe.unet.to(memory_format=torch.channels_last) + pipe.unet = torch.compile(pipe.unet, mode="reduce-overhead", fullgraph=True) + + pipe.set_progress_bar_config(disable=None) + + image = pipe(**inputs).images + image_slice = image[0, -3:, -3:, -1].flatten() + + assert image.shape == (1, 512, 512, 3) + expected_slice = np.array([0.2402, 0.2375, 0.2285, 0.2378, 0.2407, 0.2263, 0.2354, 0.2307, 0.2520]) + assert np.abs(image_slice - expected_slice).max() < 1e-1 + except Exception: + error = f"{traceback.format_exc()}" + + results = {"error": error} + out_queue.put(results, timeout=timeout) + out_queue.join() + + +class UniDiffuserPipelineFastTests( + PipelineTesterMixin, PipelineLatentTesterMixin, PipelineKarrasSchedulerTesterMixin, unittest.TestCase +): + pipeline_class = UniDiffuserPipeline + params = TEXT_GUIDED_IMAGE_VARIATION_PARAMS + batch_params = TEXT_GUIDED_IMAGE_VARIATION_BATCH_PARAMS + image_params = IMAGE_TO_IMAGE_IMAGE_PARAMS + # vae_latents, not latents, is the argument that corresponds to VAE latent inputs + image_latents_params = frozenset(["vae_latents"]) + + def get_dummy_components(self): + unet = UniDiffuserModel.from_pretrained( + "hf-internal-testing/unidiffuser-diffusers-test", + subfolder="unet", + ) + + scheduler = DPMSolverMultistepScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + solver_order=3, + ) + + vae = AutoencoderKL.from_pretrained( + "hf-internal-testing/unidiffuser-diffusers-test", + subfolder="vae", + ) + + text_encoder = CLIPTextModel.from_pretrained( + "hf-internal-testing/unidiffuser-diffusers-test", + subfolder="text_encoder", + ) + clip_tokenizer = CLIPTokenizer.from_pretrained( + "hf-internal-testing/unidiffuser-diffusers-test", + subfolder="clip_tokenizer", + ) + + image_encoder = CLIPVisionModelWithProjection.from_pretrained( + "hf-internal-testing/unidiffuser-diffusers-test", + subfolder="image_encoder", + ) + # From the Stable Diffusion Image Variation pipeline tests + clip_image_processor = CLIPImageProcessor(crop_size=32, size=32) + # image_processor = CLIPImageProcessor.from_pretrained("hf-internal-testing/tiny-random-clip") + + text_tokenizer = GPT2Tokenizer.from_pretrained( + "hf-internal-testing/unidiffuser-diffusers-test", + subfolder="text_tokenizer", + ) + text_decoder = UniDiffuserTextDecoder.from_pretrained( + "hf-internal-testing/unidiffuser-diffusers-test", + subfolder="text_decoder", + ) + + components = { + "vae": vae, + "text_encoder": text_encoder, + "image_encoder": image_encoder, + "clip_image_processor": clip_image_processor, + "clip_tokenizer": clip_tokenizer, + "text_decoder": text_decoder, + "text_tokenizer": text_tokenizer, + "unet": unet, + "scheduler": scheduler, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + image = image.cpu().permute(0, 2, 3, 1)[0] + image = Image.fromarray(np.uint8(image)).convert("RGB") + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "an elephant under the sea", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + } + return inputs + + def get_fixed_latents(self, device, seed=0): + if isinstance(device, str): + device = torch.device(device) + generator = torch.Generator(device=device).manual_seed(seed) + # Hardcode the shapes for now. + prompt_latents = randn_tensor((1, 77, 32), generator=generator, device=device, dtype=torch.float32) + vae_latents = randn_tensor((1, 4, 16, 16), generator=generator, device=device, dtype=torch.float32) + clip_latents = randn_tensor((1, 1, 32), generator=generator, device=device, dtype=torch.float32) + + latents = { + "prompt_latents": prompt_latents, + "vae_latents": vae_latents, + "clip_latents": clip_latents, + } + return latents + + def get_dummy_inputs_with_latents(self, device, seed=0): + # image = floats_tensor((1, 3, 32, 32), rng=random.Random(seed)).to(device) + # image = image.cpu().permute(0, 2, 3, 1)[0] + # image = Image.fromarray(np.uint8(image)).convert("RGB") + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/unidiffuser/unidiffuser_example_image.jpg", + ) + image = image.resize((32, 32)) + latents = self.get_fixed_latents(device, seed=seed) + + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + + inputs = { + "prompt": "an elephant under the sea", + "image": image, + "generator": generator, + "num_inference_steps": 2, + "guidance_scale": 6.0, + "output_type": "numpy", + "prompt_latents": latents.get("prompt_latents"), + "vae_latents": latents.get("vae_latents"), + "clip_latents": latents.get("clip_latents"), + } + return inputs + + def test_unidiffuser_default_joint_v0(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + unidiffuser_pipe = UniDiffuserPipeline(**components) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'joint' + unidiffuser_pipe.set_joint_mode() + assert unidiffuser_pipe.mode == "joint" + + # inputs = self.get_dummy_inputs(device) + inputs = self.get_dummy_inputs_with_latents(device) + # Delete prompt and image for joint inference. + del inputs["prompt"] + del inputs["image"] + sample = unidiffuser_pipe(**inputs) + image = sample.images + text = sample.text + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_img_slice = np.array([0.5760, 0.6270, 0.6571, 0.4965, 0.4638, 0.5663, 0.5254, 0.5068, 0.5716]) + assert np.abs(image_slice.flatten() - expected_img_slice).max() < 1e-3 + + expected_text_prefix = " no no no " + assert text[0][:10] == expected_text_prefix + + def test_unidiffuser_default_joint_no_cfg_v0(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + unidiffuser_pipe = UniDiffuserPipeline(**components) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'joint' + unidiffuser_pipe.set_joint_mode() + assert unidiffuser_pipe.mode == "joint" + + # inputs = self.get_dummy_inputs(device) + inputs = self.get_dummy_inputs_with_latents(device) + # Delete prompt and image for joint inference. + del inputs["prompt"] + del inputs["image"] + # Set guidance scale to 1.0 to turn off CFG + inputs["guidance_scale"] = 1.0 + sample = unidiffuser_pipe(**inputs) + image = sample.images + text = sample.text + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_img_slice = np.array([0.5760, 0.6270, 0.6571, 0.4965, 0.4638, 0.5663, 0.5254, 0.5068, 0.5716]) + assert np.abs(image_slice.flatten() - expected_img_slice).max() < 1e-3 + + expected_text_prefix = " no no no " + assert text[0][:10] == expected_text_prefix + + def test_unidiffuser_default_text2img_v0(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + unidiffuser_pipe = UniDiffuserPipeline(**components) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'text2img' + unidiffuser_pipe.set_text_to_image_mode() + assert unidiffuser_pipe.mode == "text2img" + + inputs = self.get_dummy_inputs_with_latents(device) + # Delete image for text-conditioned image generation + del inputs["image"] + image = unidiffuser_pipe(**inputs).images + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.5758, 0.6269, 0.6570, 0.4967, 0.4639, 0.5664, 0.5257, 0.5067, 0.5715]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_unidiffuser_default_image_0(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + unidiffuser_pipe = UniDiffuserPipeline(**components) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'img' + unidiffuser_pipe.set_image_mode() + assert unidiffuser_pipe.mode == "img" + + inputs = self.get_dummy_inputs(device) + # Delete prompt and image for unconditional ("marginal") text generation. + del inputs["prompt"] + del inputs["image"] + image = unidiffuser_pipe(**inputs).images + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.5760, 0.6270, 0.6571, 0.4966, 0.4638, 0.5663, 0.5254, 0.5068, 0.5715]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_unidiffuser_default_text_v0(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + unidiffuser_pipe = UniDiffuserPipeline(**components) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'img' + unidiffuser_pipe.set_text_mode() + assert unidiffuser_pipe.mode == "text" + + inputs = self.get_dummy_inputs(device) + # Delete prompt and image for unconditional ("marginal") text generation. + del inputs["prompt"] + del inputs["image"] + text = unidiffuser_pipe(**inputs).text + + expected_text_prefix = " no no no " + assert text[0][:10] == expected_text_prefix + + def test_unidiffuser_default_img2text_v0(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + unidiffuser_pipe = UniDiffuserPipeline(**components) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'img2text' + unidiffuser_pipe.set_image_to_text_mode() + assert unidiffuser_pipe.mode == "img2text" + + inputs = self.get_dummy_inputs_with_latents(device) + # Delete text for image-conditioned text generation + del inputs["prompt"] + text = unidiffuser_pipe(**inputs).text + + expected_text_prefix = " no no no " + assert text[0][:10] == expected_text_prefix + + def test_unidiffuser_default_joint_v1(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unidiffuser_pipe = UniDiffuserPipeline.from_pretrained("hf-internal-testing/unidiffuser-test-v1") + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'joint' + unidiffuser_pipe.set_joint_mode() + assert unidiffuser_pipe.mode == "joint" + + # inputs = self.get_dummy_inputs(device) + inputs = self.get_dummy_inputs_with_latents(device) + # Delete prompt and image for joint inference. + del inputs["prompt"] + del inputs["image"] + inputs["data_type"] = 1 + sample = unidiffuser_pipe(**inputs) + image = sample.images + text = sample.text + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_img_slice = np.array([0.5760, 0.6270, 0.6571, 0.4965, 0.4638, 0.5663, 0.5254, 0.5068, 0.5716]) + assert np.abs(image_slice.flatten() - expected_img_slice).max() < 1e-3 + + expected_text_prefix = " no no no " + assert text[0][:10] == expected_text_prefix + + def test_unidiffuser_default_text2img_v1(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unidiffuser_pipe = UniDiffuserPipeline.from_pretrained("hf-internal-testing/unidiffuser-test-v1") + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'text2img' + unidiffuser_pipe.set_text_to_image_mode() + assert unidiffuser_pipe.mode == "text2img" + + inputs = self.get_dummy_inputs_with_latents(device) + # Delete image for text-conditioned image generation + del inputs["image"] + image = unidiffuser_pipe(**inputs).images + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.5758, 0.6269, 0.6570, 0.4967, 0.4639, 0.5664, 0.5257, 0.5067, 0.5715]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-3 + + def test_unidiffuser_default_img2text_v1(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + unidiffuser_pipe = UniDiffuserPipeline.from_pretrained("hf-internal-testing/unidiffuser-test-v1") + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'img2text' + unidiffuser_pipe.set_image_to_text_mode() + assert unidiffuser_pipe.mode == "img2text" + + inputs = self.get_dummy_inputs_with_latents(device) + # Delete text for image-conditioned text generation + del inputs["prompt"] + text = unidiffuser_pipe(**inputs).text + + expected_text_prefix = " no no no " + assert text[0][:10] == expected_text_prefix + + def test_unidiffuser_text2img_multiple_images(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + unidiffuser_pipe = UniDiffuserPipeline(**components) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'text2img' + unidiffuser_pipe.set_text_to_image_mode() + assert unidiffuser_pipe.mode == "text2img" + + inputs = self.get_dummy_inputs(device) + # Delete image for text-conditioned image generation + del inputs["image"] + inputs["num_images_per_prompt"] = 2 + inputs["num_prompts_per_image"] = 3 + image = unidiffuser_pipe(**inputs).images + assert image.shape == (2, 32, 32, 3) + + def test_unidiffuser_img2text_multiple_prompts(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + unidiffuser_pipe = UniDiffuserPipeline(**components) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'img2text' + unidiffuser_pipe.set_image_to_text_mode() + assert unidiffuser_pipe.mode == "img2text" + + inputs = self.get_dummy_inputs(device) + # Delete text for image-conditioned text generation + del inputs["prompt"] + inputs["num_images_per_prompt"] = 2 + inputs["num_prompts_per_image"] = 3 + text = unidiffuser_pipe(**inputs).text + + assert len(text) == 3 + + def test_unidiffuser_text2img_multiple_images_with_latents(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + unidiffuser_pipe = UniDiffuserPipeline(**components) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'text2img' + unidiffuser_pipe.set_text_to_image_mode() + assert unidiffuser_pipe.mode == "text2img" + + inputs = self.get_dummy_inputs_with_latents(device) + # Delete image for text-conditioned image generation + del inputs["image"] + inputs["num_images_per_prompt"] = 2 + inputs["num_prompts_per_image"] = 3 + image = unidiffuser_pipe(**inputs).images + assert image.shape == (2, 32, 32, 3) + + def test_unidiffuser_img2text_multiple_prompts_with_latents(self): + device = "cpu" # ensure determinism for the device-dependent torch.Generator + components = self.get_dummy_components() + unidiffuser_pipe = UniDiffuserPipeline(**components) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'img2text' + unidiffuser_pipe.set_image_to_text_mode() + assert unidiffuser_pipe.mode == "img2text" + + inputs = self.get_dummy_inputs_with_latents(device) + # Delete text for image-conditioned text generation + del inputs["prompt"] + inputs["num_images_per_prompt"] = 2 + inputs["num_prompts_per_image"] = 3 + text = unidiffuser_pipe(**inputs).text + + assert len(text) == 3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=2e-4) + + @require_torch_gpu + def test_unidiffuser_default_joint_v1_cuda_fp16(self): + device = "cuda" + unidiffuser_pipe = UniDiffuserPipeline.from_pretrained( + "hf-internal-testing/unidiffuser-test-v1", torch_dtype=torch.float16 + ) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'joint' + unidiffuser_pipe.set_joint_mode() + assert unidiffuser_pipe.mode == "joint" + + inputs = self.get_dummy_inputs_with_latents(device) + # Delete prompt and image for joint inference. + del inputs["prompt"] + del inputs["image"] + inputs["data_type"] = 1 + sample = unidiffuser_pipe(**inputs) + image = sample.images + text = sample.text + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_img_slice = np.array([0.5049, 0.5498, 0.5854, 0.3052, 0.4460, 0.6489, 0.5122, 0.4810, 0.6138]) + assert np.abs(image_slice.flatten() - expected_img_slice).max() < 1e-3 + + expected_text_prefix = '" This This' + assert text[0][: len(expected_text_prefix)] == expected_text_prefix + + @require_torch_gpu + def test_unidiffuser_default_text2img_v1_cuda_fp16(self): + device = "cuda" + unidiffuser_pipe = UniDiffuserPipeline.from_pretrained( + "hf-internal-testing/unidiffuser-test-v1", torch_dtype=torch.float16 + ) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'text2img' + unidiffuser_pipe.set_text_to_image_mode() + assert unidiffuser_pipe.mode == "text2img" + + inputs = self.get_dummy_inputs_with_latents(device) + # Delete prompt and image for joint inference. + del inputs["image"] + inputs["data_type"] = 1 + sample = unidiffuser_pipe(**inputs) + image = sample.images + assert image.shape == (1, 32, 32, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_img_slice = np.array([0.5054, 0.5498, 0.5854, 0.3052, 0.4458, 0.6489, 0.5122, 0.4810, 0.6138]) + assert np.abs(image_slice.flatten() - expected_img_slice).max() < 1e-3 + + @require_torch_gpu + def test_unidiffuser_default_img2text_v1_cuda_fp16(self): + device = "cuda" + unidiffuser_pipe = UniDiffuserPipeline.from_pretrained( + "hf-internal-testing/unidiffuser-test-v1", torch_dtype=torch.float16 + ) + unidiffuser_pipe = unidiffuser_pipe.to(device) + unidiffuser_pipe.set_progress_bar_config(disable=None) + + # Set mode to 'img2text' + unidiffuser_pipe.set_image_to_text_mode() + assert unidiffuser_pipe.mode == "img2text" + + inputs = self.get_dummy_inputs_with_latents(device) + # Delete prompt and image for joint inference. + del inputs["prompt"] + inputs["data_type"] = 1 + text = unidiffuser_pipe(**inputs).text + + expected_text_prefix = '" This This' + assert text[0][: len(expected_text_prefix)] == expected_text_prefix + + +@nightly +@require_torch_gpu +class UniDiffuserPipelineSlowTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, seed=0, generate_latents=False): + generator = torch.manual_seed(seed) + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/unidiffuser/unidiffuser_example_image.jpg" + ) + inputs = { + "prompt": "an elephant under the sea", + "image": image, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 8.0, + "output_type": "numpy", + } + if generate_latents: + latents = self.get_fixed_latents(device, seed=seed) + for latent_name, latent_tensor in latents.items(): + inputs[latent_name] = latent_tensor + return inputs + + def get_fixed_latents(self, device, seed=0): + if isinstance(device, str): + device = torch.device(device) + latent_device = torch.device("cpu") + generator = torch.Generator(device=latent_device).manual_seed(seed) + # Hardcode the shapes for now. + prompt_latents = randn_tensor((1, 77, 768), generator=generator, device=device, dtype=torch.float32) + vae_latents = randn_tensor((1, 4, 64, 64), generator=generator, device=device, dtype=torch.float32) + clip_latents = randn_tensor((1, 1, 512), generator=generator, device=device, dtype=torch.float32) + + # Move latents onto desired device. + prompt_latents = prompt_latents.to(device) + vae_latents = vae_latents.to(device) + clip_latents = clip_latents.to(device) + + latents = { + "prompt_latents": prompt_latents, + "vae_latents": vae_latents, + "clip_latents": clip_latents, + } + return latents + + def test_unidiffuser_default_joint_v1(self): + pipe = UniDiffuserPipeline.from_pretrained("thu-ml/unidiffuser-v1") + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + # inputs = self.get_dummy_inputs(device) + inputs = self.get_inputs(device=torch_device, generate_latents=True) + # Delete prompt and image for joint inference. + del inputs["prompt"] + del inputs["image"] + sample = pipe(**inputs) + image = sample.images + text = sample.text + assert image.shape == (1, 512, 512, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_img_slice = np.array([0.2402, 0.2375, 0.2285, 0.2378, 0.2407, 0.2263, 0.2354, 0.2307, 0.2520]) + assert np.abs(image_slice.flatten() - expected_img_slice).max() < 1e-1 + + expected_text_prefix = "a living room" + assert text[0][: len(expected_text_prefix)] == expected_text_prefix + + def test_unidiffuser_default_text2img_v1(self): + pipe = UniDiffuserPipeline.from_pretrained("thu-ml/unidiffuser-v1") + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(device=torch_device, generate_latents=True) + del inputs["image"] + sample = pipe(**inputs) + image = sample.images + assert image.shape == (1, 512, 512, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.0242, 0.0103, 0.0022, 0.0129, 0.0000, 0.0090, 0.0376, 0.0508, 0.0005]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 + + def test_unidiffuser_default_img2text_v1(self): + pipe = UniDiffuserPipeline.from_pretrained("thu-ml/unidiffuser-v1") + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(device=torch_device, generate_latents=True) + del inputs["prompt"] + sample = pipe(**inputs) + text = sample.text + + expected_text_prefix = "An astronaut" + assert text[0][: len(expected_text_prefix)] == expected_text_prefix + + @unittest.skip(reason="Skip torch.compile test to speed up the slow test suite.") + @require_torch_2 + def test_unidiffuser_compile(self, seed=0): + inputs = self.get_inputs(torch_device, seed=seed, generate_latents=True) + # Delete prompt and image for joint inference. + del inputs["prompt"] + del inputs["image"] + # Can't pickle a Generator object + del inputs["generator"] + inputs["torch_device"] = torch_device + inputs["seed"] = seed + run_test_in_subprocess(test_case=self, target_func=_test_unidiffuser_compile, inputs=inputs) + + +@nightly +@require_torch_gpu +class UniDiffuserPipelineNightlyTests(unittest.TestCase): + def tearDown(self): + super().tearDown() + gc.collect() + torch.cuda.empty_cache() + + def get_inputs(self, device, seed=0, generate_latents=False): + generator = torch.manual_seed(seed) + image = load_image( + "https://huggingface.co/datasets/hf-internal-testing/diffusers-images/resolve/main/unidiffuser/unidiffuser_example_image.jpg" + ) + inputs = { + "prompt": "an elephant under the sea", + "image": image, + "generator": generator, + "num_inference_steps": 3, + "guidance_scale": 8.0, + "output_type": "numpy", + } + if generate_latents: + latents = self.get_fixed_latents(device, seed=seed) + for latent_name, latent_tensor in latents.items(): + inputs[latent_name] = latent_tensor + return inputs + + def get_fixed_latents(self, device, seed=0): + if isinstance(device, str): + device = torch.device(device) + latent_device = torch.device("cpu") + generator = torch.Generator(device=latent_device).manual_seed(seed) + # Hardcode the shapes for now. + prompt_latents = randn_tensor((1, 77, 768), generator=generator, device=device, dtype=torch.float32) + vae_latents = randn_tensor((1, 4, 64, 64), generator=generator, device=device, dtype=torch.float32) + clip_latents = randn_tensor((1, 1, 512), generator=generator, device=device, dtype=torch.float32) + + # Move latents onto desired device. + prompt_latents = prompt_latents.to(device) + vae_latents = vae_latents.to(device) + clip_latents = clip_latents.to(device) + + latents = { + "prompt_latents": prompt_latents, + "vae_latents": vae_latents, + "clip_latents": clip_latents, + } + return latents + + def test_unidiffuser_default_joint_v1_fp16(self): + pipe = UniDiffuserPipeline.from_pretrained("thu-ml/unidiffuser-v1", torch_dtype=torch.float16) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + # inputs = self.get_dummy_inputs(device) + inputs = self.get_inputs(device=torch_device, generate_latents=True) + # Delete prompt and image for joint inference. + del inputs["prompt"] + del inputs["image"] + sample = pipe(**inputs) + image = sample.images + text = sample.text + assert image.shape == (1, 512, 512, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_img_slice = np.array([0.2402, 0.2375, 0.2285, 0.2378, 0.2407, 0.2263, 0.2354, 0.2307, 0.2520]) + assert np.abs(image_slice.flatten() - expected_img_slice).max() < 2e-1 + + expected_text_prefix = "a living room" + assert text[0][: len(expected_text_prefix)] == expected_text_prefix + + def test_unidiffuser_default_text2img_v1_fp16(self): + pipe = UniDiffuserPipeline.from_pretrained("thu-ml/unidiffuser-v1", torch_dtype=torch.float16) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(device=torch_device, generate_latents=True) + del inputs["image"] + sample = pipe(**inputs) + image = sample.images + assert image.shape == (1, 512, 512, 3) + + image_slice = image[0, -3:, -3:, -1] + expected_slice = np.array([0.0242, 0.0103, 0.0022, 0.0129, 0.0000, 0.0090, 0.0376, 0.0508, 0.0005]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-1 + + def test_unidiffuser_default_img2text_v1_fp16(self): + pipe = UniDiffuserPipeline.from_pretrained("thu-ml/unidiffuser-v1", torch_dtype=torch.float16) + pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + pipe.enable_attention_slicing() + + inputs = self.get_inputs(device=torch_device, generate_latents=True) + del inputs["prompt"] + sample = pipe(**inputs) + text = sample.text + + expected_text_prefix = "An astronaut" + assert text[0][: len(expected_text_prefix)] == expected_text_prefix diff --git a/diffusers-0.27.0/tests/pipelines/wuerstchen/__init__.py b/diffusers-0.27.0/tests/pipelines/wuerstchen/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_combined.py b/diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_combined.py new file mode 100755 index 0000000..0caed15 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_combined.py @@ -0,0 +1,239 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import DDPMWuerstchenScheduler, WuerstchenCombinedPipeline +from diffusers.pipelines.wuerstchen import PaellaVQModel, WuerstchenDiffNeXt, WuerstchenPrior +from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, torch_device + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class WuerstchenCombinedPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = WuerstchenCombinedPipeline + params = ["prompt"] + batch_params = ["prompt", "negative_prompt"] + required_optional_params = [ + "generator", + "height", + "width", + "latents", + "prior_guidance_scale", + "decoder_guidance_scale", + "negative_prompt", + "num_inference_steps", + "return_dict", + "prior_num_inference_steps", + "output_type", + ] + test_xformers_attention = True + + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def dummy_prior(self): + torch.manual_seed(0) + + model_kwargs = {"c_in": 2, "c": 8, "depth": 2, "c_cond": 32, "c_r": 8, "nhead": 2} + model = WuerstchenPrior(**model_kwargs) + return model.eval() + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_prior_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModel(config).eval() + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + projection_dim=self.text_embedder_hidden_size, + hidden_size=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModel(config).eval() + + @property + def dummy_vqgan(self): + torch.manual_seed(0) + + model_kwargs = { + "bottleneck_blocks": 1, + "num_vq_embeddings": 2, + } + model = PaellaVQModel(**model_kwargs) + return model.eval() + + @property + def dummy_decoder(self): + torch.manual_seed(0) + + model_kwargs = { + "c_cond": self.text_embedder_hidden_size, + "c_hidden": [320], + "nhead": [-1], + "blocks": [4], + "level_config": ["CT"], + "clip_embd": self.text_embedder_hidden_size, + "inject_effnet": [False], + } + + model = WuerstchenDiffNeXt(**model_kwargs) + return model.eval() + + def get_dummy_components(self): + prior = self.dummy_prior + prior_text_encoder = self.dummy_prior_text_encoder + + scheduler = DDPMWuerstchenScheduler() + tokenizer = self.dummy_tokenizer + + text_encoder = self.dummy_text_encoder + decoder = self.dummy_decoder + vqgan = self.dummy_vqgan + + components = { + "tokenizer": tokenizer, + "text_encoder": text_encoder, + "decoder": decoder, + "vqgan": vqgan, + "scheduler": scheduler, + "prior_prior": prior, + "prior_text_encoder": prior_text_encoder, + "prior_tokenizer": tokenizer, + "prior_scheduler": scheduler, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "horse", + "generator": generator, + "prior_guidance_scale": 4.0, + "decoder_guidance_scale": 4.0, + "num_inference_steps": 2, + "prior_num_inference_steps": 2, + "output_type": "np", + "height": 128, + "width": 128, + } + return inputs + + def test_wuerstchen(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe(**self.get_dummy_inputs(device), return_dict=False)[0] + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[-3:, -3:, -1] + + assert image.shape == (1, 128, 128, 3) + + expected_slice = np.array([0.7616304, 0.0, 1.0, 0.0, 1.0, 0.0, 0.05925313, 0.0, 0.951898]) + + assert ( + np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_slice.flatten()}" + assert ( + np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + ), f" expected_slice {expected_slice}, but got {image_from_tuple_slice.flatten()}" + + @require_torch_gpu + def test_offloads(self): + pipes = [] + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components).to(torch_device) + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_sequential_cpu_offload() + pipes.append(sd_pipe) + + components = self.get_dummy_components() + sd_pipe = self.pipeline_class(**components) + sd_pipe.enable_model_cpu_offload() + pipes.append(sd_pipe) + + image_slices = [] + for pipe in pipes: + inputs = self.get_dummy_inputs(torch_device) + image = pipe(**inputs).images + + image_slices.append(image[0, -3:, -3:, -1].flatten()) + + assert np.abs(image_slices[0] - image_slices[1]).max() < 1e-3 + assert np.abs(image_slices[0] - image_slices[2]).max() < 1e-3 + + def test_inference_batch_single_identical(self): + super().test_inference_batch_single_identical(expected_max_diff=1e-2) + + @unittest.skip(reason="flakey and float16 requires CUDA") + def test_float16_inference(self): + super().test_float16_inference() + + def test_callback_inputs(self): + pass + + def test_callback_cfg(self): + pass diff --git a/diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_decoder.py b/diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_decoder.py new file mode 100755 index 0000000..4675501 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_decoder.py @@ -0,0 +1,188 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import DDPMWuerstchenScheduler, WuerstchenDecoderPipeline +from diffusers.pipelines.wuerstchen import PaellaVQModel, WuerstchenDiffNeXt +from diffusers.utils.testing_utils import enable_full_determinism, skip_mps, torch_device + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +class WuerstchenDecoderPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = WuerstchenDecoderPipeline + params = ["prompt"] + batch_params = ["image_embeddings", "prompt", "negative_prompt"] + required_optional_params = [ + "num_images_per_prompt", + "num_inference_steps", + "latents", + "negative_prompt", + "guidance_scale", + "output_type", + "return_dict", + ] + test_xformers_attention = False + callback_cfg_params = ["image_embeddings", "text_encoder_hidden_states"] + + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + projection_dim=self.text_embedder_hidden_size, + hidden_size=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModel(config).eval() + + @property + def dummy_vqgan(self): + torch.manual_seed(0) + + model_kwargs = { + "bottleneck_blocks": 1, + "num_vq_embeddings": 2, + } + model = PaellaVQModel(**model_kwargs) + return model.eval() + + @property + def dummy_decoder(self): + torch.manual_seed(0) + + model_kwargs = { + "c_cond": self.text_embedder_hidden_size, + "c_hidden": [320], + "nhead": [-1], + "blocks": [4], + "level_config": ["CT"], + "clip_embd": self.text_embedder_hidden_size, + "inject_effnet": [False], + } + + model = WuerstchenDiffNeXt(**model_kwargs) + return model.eval() + + def get_dummy_components(self): + decoder = self.dummy_decoder + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + vqgan = self.dummy_vqgan + + scheduler = DDPMWuerstchenScheduler() + + components = { + "decoder": decoder, + "vqgan": vqgan, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "scheduler": scheduler, + "latent_dim_scale": 4.0, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "image_embeddings": torch.ones((1, 4, 4, 4), device=device), + "prompt": "horse", + "generator": generator, + "guidance_scale": 1.0, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + def test_wuerstchen_decoder(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.images + + image_from_tuple = pipe(**self.get_dummy_inputs(device), return_dict=False) + + image_slice = image[0, -3:, -3:, -1] + image_from_tuple_slice = image_from_tuple[0, -3:, -3:, -1] + + assert image.shape == (1, 64, 64, 3) + + expected_slice = np.array([0.0000, 0.0000, 0.0089, 1.0000, 1.0000, 0.3927, 1.0000, 1.0000, 1.0000]) + assert np.abs(image_slice.flatten() - expected_slice).max() < 1e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 1e-2 + + @skip_mps + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical(expected_max_diff=1e-5) + + @skip_mps + def test_attention_slicing_forward_pass(self): + test_max_difference = torch_device == "cpu" + test_mean_pixel_difference = False + + self._test_attention_slicing_forward_pass( + test_max_difference=test_max_difference, + test_mean_pixel_difference=test_mean_pixel_difference, + ) + + @unittest.skip(reason="bf16 not supported and requires CUDA") + def test_float16_inference(self): + super().test_float16_inference() diff --git a/diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_prior.py b/diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_prior.py new file mode 100755 index 0000000..200e4d1 --- /dev/null +++ b/diffusers-0.27.0/tests/pipelines/wuerstchen/test_wuerstchen_prior.py @@ -0,0 +1,296 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +import torch +import torch.nn as nn +import torch.nn.functional as F +from transformers import CLIPTextConfig, CLIPTextModel, CLIPTokenizer + +from diffusers import DDPMWuerstchenScheduler, WuerstchenPriorPipeline +from diffusers.loaders import AttnProcsLayers +from diffusers.models.attention_processor import ( + LoRAAttnProcessor, + LoRAAttnProcessor2_0, +) +from diffusers.pipelines.wuerstchen import WuerstchenPrior +from diffusers.utils.import_utils import is_peft_available +from diffusers.utils.testing_utils import enable_full_determinism, require_peft_backend, skip_mps, torch_device + + +if is_peft_available(): + from peft import LoraConfig + from peft.tuners.tuners_utils import BaseTunerLayer + +from ..test_pipelines_common import PipelineTesterMixin + + +enable_full_determinism() + + +def create_prior_lora_layers(unet: nn.Module): + lora_attn_procs = {} + for name in unet.attn_processors.keys(): + lora_attn_processor_class = ( + LoRAAttnProcessor2_0 if hasattr(F, "scaled_dot_product_attention") else LoRAAttnProcessor + ) + lora_attn_procs[name] = lora_attn_processor_class( + hidden_size=unet.config.c, + ) + unet_lora_layers = AttnProcsLayers(lora_attn_procs) + return lora_attn_procs, unet_lora_layers + + +class WuerstchenPriorPipelineFastTests(PipelineTesterMixin, unittest.TestCase): + pipeline_class = WuerstchenPriorPipeline + params = ["prompt"] + batch_params = ["prompt", "negative_prompt"] + required_optional_params = [ + "num_images_per_prompt", + "generator", + "num_inference_steps", + "latents", + "negative_prompt", + "guidance_scale", + "output_type", + "return_dict", + ] + test_xformers_attention = False + callback_cfg_params = ["text_encoder_hidden_states"] + + @property + def text_embedder_hidden_size(self): + return 32 + + @property + def time_input_dim(self): + return 32 + + @property + def block_out_channels_0(self): + return self.time_input_dim + + @property + def time_embed_dim(self): + return self.time_input_dim * 4 + + @property + def dummy_tokenizer(self): + tokenizer = CLIPTokenizer.from_pretrained("hf-internal-testing/tiny-random-clip") + return tokenizer + + @property + def dummy_text_encoder(self): + torch.manual_seed(0) + config = CLIPTextConfig( + bos_token_id=0, + eos_token_id=2, + hidden_size=self.text_embedder_hidden_size, + intermediate_size=37, + layer_norm_eps=1e-05, + num_attention_heads=4, + num_hidden_layers=5, + pad_token_id=1, + vocab_size=1000, + ) + return CLIPTextModel(config).eval() + + @property + def dummy_prior(self): + torch.manual_seed(0) + + model_kwargs = { + "c_in": 2, + "c": 8, + "depth": 2, + "c_cond": 32, + "c_r": 8, + "nhead": 2, + } + + model = WuerstchenPrior(**model_kwargs) + return model.eval() + + def get_dummy_components(self): + prior = self.dummy_prior + text_encoder = self.dummy_text_encoder + tokenizer = self.dummy_tokenizer + + scheduler = DDPMWuerstchenScheduler() + + components = { + "prior": prior, + "text_encoder": text_encoder, + "tokenizer": tokenizer, + "scheduler": scheduler, + } + + return components + + def get_dummy_inputs(self, device, seed=0): + if str(device).startswith("mps"): + generator = torch.manual_seed(seed) + else: + generator = torch.Generator(device=device).manual_seed(seed) + inputs = { + "prompt": "horse", + "generator": generator, + "guidance_scale": 4.0, + "num_inference_steps": 2, + "output_type": "np", + } + return inputs + + def test_wuerstchen_prior(self): + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output = pipe(**self.get_dummy_inputs(device)) + image = output.image_embeddings + + image_from_tuple = pipe(**self.get_dummy_inputs(device), return_dict=False)[0] + + image_slice = image[0, 0, 0, -10:] + image_from_tuple_slice = image_from_tuple[0, 0, 0, -10:] + assert image.shape == (1, 2, 24, 24) + + expected_slice = np.array( + [ + -7172.837, + -3438.855, + -1093.312, + 388.8835, + -7471.467, + -7998.1206, + -5328.259, + 218.00089, + -2731.5745, + -8056.734, + ] + ) + assert np.abs(image_slice.flatten() - expected_slice).max() < 5e-2 + assert np.abs(image_from_tuple_slice.flatten() - expected_slice).max() < 5e-2 + + @skip_mps + def test_inference_batch_single_identical(self): + self._test_inference_batch_single_identical( + expected_max_diff=2e-1, + ) + + @skip_mps + def test_attention_slicing_forward_pass(self): + test_max_difference = torch_device == "cpu" + test_mean_pixel_difference = False + + self._test_attention_slicing_forward_pass( + test_max_difference=test_max_difference, + test_mean_pixel_difference=test_mean_pixel_difference, + ) + + @unittest.skip(reason="flaky for now") + def test_float16_inference(self): + super().test_float16_inference() + + # override because we need to make sure latent_mean and latent_std to be 0 + def test_callback_inputs(self): + components = self.get_dummy_components() + components["latent_mean"] = 0 + components["latent_std"] = 0 + pipe = self.pipeline_class(**components) + pipe = pipe.to(torch_device) + pipe.set_progress_bar_config(disable=None) + + self.assertTrue( + hasattr(pipe, "_callback_tensor_inputs"), + f" {self.pipeline_class} should have `_callback_tensor_inputs` that defines a list of tensor variables its callback function can use as inputs", + ) + + def callback_inputs_test(pipe, i, t, callback_kwargs): + missing_callback_inputs = set() + for v in pipe._callback_tensor_inputs: + if v not in callback_kwargs: + missing_callback_inputs.add(v) + self.assertTrue( + len(missing_callback_inputs) == 0, f"Missing callback tensor inputs: {missing_callback_inputs}" + ) + last_i = pipe.num_timesteps - 1 + if i == last_i: + callback_kwargs["latents"] = torch.zeros_like(callback_kwargs["latents"]) + return callback_kwargs + + inputs = self.get_dummy_inputs(torch_device) + inputs["callback_on_step_end"] = callback_inputs_test + inputs["callback_on_step_end_tensor_inputs"] = pipe._callback_tensor_inputs + inputs["output_type"] = "latent" + + output = pipe(**inputs)[0] + assert output.abs().sum() == 0 + + def check_if_lora_correctly_set(self, model) -> bool: + """ + Checks if the LoRA layers are correctly set with peft + """ + for module in model.modules(): + if isinstance(module, BaseTunerLayer): + return True + return False + + def get_lora_components(self): + prior = self.dummy_prior + + prior_lora_config = LoraConfig( + r=4, lora_alpha=4, target_modules=["to_q", "to_k", "to_v", "to_out.0"], init_lora_weights=False + ) + + prior_lora_attn_procs, prior_lora_layers = create_prior_lora_layers(prior) + + lora_components = { + "prior_lora_layers": prior_lora_layers, + "prior_lora_attn_procs": prior_lora_attn_procs, + } + + return prior, prior_lora_config, lora_components + + @require_peft_backend + def test_inference_with_prior_lora(self): + _, prior_lora_config, _ = self.get_lora_components() + device = "cpu" + + components = self.get_dummy_components() + + pipe = self.pipeline_class(**components) + pipe = pipe.to(device) + + pipe.set_progress_bar_config(disable=None) + + output_no_lora = pipe(**self.get_dummy_inputs(device)) + image_embed = output_no_lora.image_embeddings + self.assertTrue(image_embed.shape == (1, 2, 24, 24)) + + pipe.prior.add_adapter(prior_lora_config) + self.assertTrue(self.check_if_lora_correctly_set(pipe.prior), "Lora not correctly set in prior") + + output_lora = pipe(**self.get_dummy_inputs(device)) + lora_image_embed = output_lora.image_embeddings + + self.assertTrue(image_embed.shape == lora_image_embed.shape) diff --git a/diffusers-0.27.0/tests/schedulers/__init__.py b/diffusers-0.27.0/tests/schedulers/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_consistency_model.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_consistency_model.py new file mode 100755 index 0000000..4f773d7 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_consistency_model.py @@ -0,0 +1,189 @@ +import torch + +from diffusers import CMStochasticIterativeScheduler + +from .test_schedulers import SchedulerCommonTest + + +class CMStochasticIterativeSchedulerTest(SchedulerCommonTest): + scheduler_classes = (CMStochasticIterativeScheduler,) + num_inference_steps = 10 + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 201, + "sigma_min": 0.002, + "sigma_max": 80.0, + } + + config.update(**kwargs) + return config + + # Override test_step_shape to add CMStochasticIterativeScheduler-specific logic regarding timesteps + # Problem is that we don't know two timesteps that will always be in the timestep schedule from only the scheduler + # config; scaled sigma_max is always in the timestep schedule, but sigma_min is in the sigma schedule while scaled + # sigma_min is not in the timestep schedule + def test_step_shape(self): + num_inference_steps = 10 + + scheduler_config = self.get_scheduler_config() + scheduler = self.scheduler_classes[0](**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + + timestep_0 = scheduler.timesteps[0] + timestep_1 = scheduler.timesteps[1] + + sample = self.dummy_sample + residual = 0.1 * sample + + output_0 = scheduler.step(residual, timestep_0, sample).prev_sample + output_1 = scheduler.step(residual, timestep_1, sample).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_timesteps(self): + for timesteps in [10, 50, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_clip_denoised(self): + for clip_denoised in [True, False]: + self.check_over_configs(clip_denoised=clip_denoised) + + def test_full_loop_no_noise_onestep(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 1 + scheduler.set_timesteps(num_inference_steps) + timesteps = scheduler.timesteps + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + + for i, t in enumerate(timesteps): + # 1. scale model input + scaled_sample = scheduler.scale_model_input(sample, t) + + # 2. predict noise residual + residual = model(scaled_sample, t) + + # 3. predict previous sample x_t-1 + pred_prev_sample = scheduler.step(residual, t, sample, generator=generator).prev_sample + + sample = pred_prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 192.7614) < 1e-2 + assert abs(result_mean.item() - 0.2510) < 1e-3 + + def test_full_loop_no_noise_multistep(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [106, 0] + scheduler.set_timesteps(timesteps=timesteps) + timesteps = scheduler.timesteps + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + + for t in timesteps: + # 1. scale model input + scaled_sample = scheduler.scale_model_input(sample, t) + + # 2. predict noise residual + residual = model(scaled_sample, t) + + # 3. predict previous sample x_t-1 + pred_prev_sample = scheduler.step(residual, t, sample, generator=generator).prev_sample + + sample = pred_prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 347.6357) < 1e-2 + assert abs(result_mean.item() - 0.4527) < 1e-3 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + t_start = 8 + + scheduler.set_timesteps(num_inference_steps) + timesteps = scheduler.timesteps + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for t in timesteps: + # 1. scale model input + scaled_sample = scheduler.scale_model_input(sample, t) + + # 2. predict noise residual + residual = model(scaled_sample, t) + + # 3. predict previous sample x_t-1 + pred_prev_sample = scheduler.step(residual, t, sample, generator=generator).prev_sample + + sample = pred_prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 763.9186) < 1e-2, f" expected result sum 763.9186, but get {result_sum}" + assert abs(result_mean.item() - 0.9947) < 1e-3, f" expected result mean 0.9947, but get {result_mean}" + + def test_custom_timesteps_increasing_order(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [39, 30, 12, 15, 0] + + with self.assertRaises(ValueError, msg="`timesteps` must be in descending order."): + scheduler.set_timesteps(timesteps=timesteps) + + def test_custom_timesteps_passing_both_num_inference_steps_and_timesteps(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [39, 30, 12, 1, 0] + num_inference_steps = len(timesteps) + + with self.assertRaises(ValueError, msg="Can only pass one of `num_inference_steps` or `timesteps`."): + scheduler.set_timesteps(num_inference_steps=num_inference_steps, timesteps=timesteps) + + def test_custom_timesteps_too_large(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [scheduler.config.num_train_timesteps] + + with self.assertRaises( + ValueError, + msg="`timesteps` must start before `self.config.train_timesteps`: {scheduler.config.num_train_timesteps}}", + ): + scheduler.set_timesteps(timesteps=timesteps) diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_ddim.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_ddim.py new file mode 100755 index 0000000..13b353a --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_ddim.py @@ -0,0 +1,176 @@ +import torch + +from diffusers import DDIMScheduler + +from .test_schedulers import SchedulerCommonTest + + +class DDIMSchedulerTest(SchedulerCommonTest): + scheduler_classes = (DDIMScheduler,) + forward_default_kwargs = (("eta", 0.0), ("num_inference_steps", 50)) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "clip_sample": True, + } + + config.update(**kwargs) + return config + + def full_loop(self, **config): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps, eta = 10, 0.0 + + model = self.dummy_model() + sample = self.dummy_sample_deter + + scheduler.set_timesteps(num_inference_steps) + + for t in scheduler.timesteps: + residual = model(sample, t) + sample = scheduler.step(residual, t, sample, eta).prev_sample + + return sample + + def test_timesteps(self): + for timesteps in [100, 500, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_steps_offset(self): + for steps_offset in [0, 1]: + self.check_over_configs(steps_offset=steps_offset) + + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(steps_offset=1) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(5) + assert torch.equal(scheduler.timesteps, torch.LongTensor([801, 601, 401, 201, 1])) + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001, 0.01, 0.1], [0.002, 0.02, 0.2, 2]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "squaredcos_cap_v2"]: + self.check_over_configs(beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_clip_sample(self): + for clip_sample in [True, False]: + self.check_over_configs(clip_sample=clip_sample) + + def test_timestep_spacing(self): + for timestep_spacing in ["trailing", "leading"]: + self.check_over_configs(timestep_spacing=timestep_spacing) + + def test_rescale_betas_zero_snr(self): + for rescale_betas_zero_snr in [True, False]: + self.check_over_configs(rescale_betas_zero_snr=rescale_betas_zero_snr) + + def test_thresholding(self): + self.check_over_configs(thresholding=False) + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs( + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + ) + + def test_time_indices(self): + for t in [1, 10, 49]: + self.check_over_forward(time_step=t) + + def test_inference_steps(self): + for t, num_inference_steps in zip([1, 10, 50], [10, 50, 500]): + self.check_over_forward(time_step=t, num_inference_steps=num_inference_steps) + + def test_eta(self): + for t, eta in zip([1, 10, 49], [0.0, 0.5, 1.0]): + self.check_over_forward(time_step=t, eta=eta) + + def test_variance(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + assert torch.sum(torch.abs(scheduler._get_variance(0, 0) - 0.0)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(420, 400) - 0.14771)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(980, 960) - 0.32460)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(0, 0) - 0.0)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(487, 486) - 0.00979)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(999, 998) - 0.02)) < 1e-5 + + def test_full_loop_no_noise(self): + sample = self.full_loop() + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 172.0067) < 1e-2 + assert abs(result_mean.item() - 0.223967) < 1e-3 + + def test_full_loop_with_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction") + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 52.5302) < 1e-2 + assert abs(result_mean.item() - 0.0684) < 1e-3 + + def test_full_loop_with_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=True, beta_start=0.01) + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 149.8295) < 1e-2 + assert abs(result_mean.item() - 0.1951) < 1e-3 + + def test_full_loop_with_no_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=False, beta_start=0.01) + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 149.0784) < 1e-2 + assert abs(result_mean.item() - 0.1941) < 1e-3 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps, eta = 10, 0.0 + t_start = 8 + + model = self.dummy_model() + sample = self.dummy_sample_deter + + scheduler.set_timesteps(num_inference_steps) + + # add noise + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for t in timesteps: + residual = model(sample, t) + sample = scheduler.step(residual, t, sample, eta).prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 354.5418) < 1e-2, f" expected result sum 218.4379, but get {result_sum}" + assert abs(result_mean.item() - 0.4616) < 1e-3, f" expected result mean 0.2844, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_ddim_inverse.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_ddim_inverse.py new file mode 100755 index 0000000..696f576 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_ddim_inverse.py @@ -0,0 +1,135 @@ +import torch + +from diffusers import DDIMInverseScheduler + +from .test_schedulers import SchedulerCommonTest + + +class DDIMInverseSchedulerTest(SchedulerCommonTest): + scheduler_classes = (DDIMInverseScheduler,) + forward_default_kwargs = (("num_inference_steps", 50),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "clip_sample": True, + } + + config.update(**kwargs) + return config + + def full_loop(self, **config): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + + model = self.dummy_model() + sample = self.dummy_sample_deter + + scheduler.set_timesteps(num_inference_steps) + + for t in scheduler.timesteps: + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + return sample + + def test_timesteps(self): + for timesteps in [100, 500, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_steps_offset(self): + for steps_offset in [0, 1]: + self.check_over_configs(steps_offset=steps_offset) + + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(steps_offset=1) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(5) + assert torch.equal(scheduler.timesteps, torch.LongTensor([1, 201, 401, 601, 801])) + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001, 0.01, 0.1], [0.002, 0.02, 0.2, 2]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "squaredcos_cap_v2"]: + self.check_over_configs(beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_clip_sample(self): + for clip_sample in [True, False]: + self.check_over_configs(clip_sample=clip_sample) + + def test_timestep_spacing(self): + for timestep_spacing in ["trailing", "leading"]: + self.check_over_configs(timestep_spacing=timestep_spacing) + + def test_rescale_betas_zero_snr(self): + for rescale_betas_zero_snr in [True, False]: + self.check_over_configs(rescale_betas_zero_snr=rescale_betas_zero_snr) + + def test_thresholding(self): + self.check_over_configs(thresholding=False) + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs( + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + ) + + def test_time_indices(self): + for t in [1, 10, 49]: + self.check_over_forward(time_step=t) + + def test_inference_steps(self): + for t, num_inference_steps in zip([1, 10, 50], [10, 50, 500]): + self.check_over_forward(time_step=t, num_inference_steps=num_inference_steps) + + def test_add_noise_device(self): + pass + + def test_full_loop_no_noise(self): + sample = self.full_loop() + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 671.6816) < 1e-2 + assert abs(result_mean.item() - 0.8746) < 1e-3 + + def test_full_loop_with_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction") + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 1394.2185) < 1e-2 + assert abs(result_mean.item() - 1.8154) < 1e-3 + + def test_full_loop_with_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=True, beta_start=0.01) + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 539.9622) < 1e-2 + assert abs(result_mean.item() - 0.7031) < 1e-3 + + def test_full_loop_with_no_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=False, beta_start=0.01) + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 542.6722) < 1e-2 + assert abs(result_mean.item() - 0.7066) < 1e-3 diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_ddim_parallel.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_ddim_parallel.py new file mode 100755 index 0000000..5434d08 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_ddim_parallel.py @@ -0,0 +1,216 @@ +# Copyright 2024 ParaDiGMS authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from diffusers import DDIMParallelScheduler + +from .test_schedulers import SchedulerCommonTest + + +class DDIMParallelSchedulerTest(SchedulerCommonTest): + scheduler_classes = (DDIMParallelScheduler,) + forward_default_kwargs = (("eta", 0.0), ("num_inference_steps", 50)) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "clip_sample": True, + } + + config.update(**kwargs) + return config + + def full_loop(self, **config): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps, eta = 10, 0.0 + + model = self.dummy_model() + sample = self.dummy_sample_deter + + scheduler.set_timesteps(num_inference_steps) + + for t in scheduler.timesteps: + residual = model(sample, t) + sample = scheduler.step(residual, t, sample, eta).prev_sample + + return sample + + def test_timesteps(self): + for timesteps in [100, 500, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_steps_offset(self): + for steps_offset in [0, 1]: + self.check_over_configs(steps_offset=steps_offset) + + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(steps_offset=1) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(5) + assert torch.equal(scheduler.timesteps, torch.LongTensor([801, 601, 401, 201, 1])) + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001, 0.01, 0.1], [0.002, 0.02, 0.2, 2]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "squaredcos_cap_v2"]: + self.check_over_configs(beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_clip_sample(self): + for clip_sample in [True, False]: + self.check_over_configs(clip_sample=clip_sample) + + def test_timestep_spacing(self): + for timestep_spacing in ["trailing", "leading"]: + self.check_over_configs(timestep_spacing=timestep_spacing) + + def test_rescale_betas_zero_snr(self): + for rescale_betas_zero_snr in [True, False]: + self.check_over_configs(rescale_betas_zero_snr=rescale_betas_zero_snr) + + def test_thresholding(self): + self.check_over_configs(thresholding=False) + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs( + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + ) + + def test_time_indices(self): + for t in [1, 10, 49]: + self.check_over_forward(time_step=t) + + def test_inference_steps(self): + for t, num_inference_steps in zip([1, 10, 50], [10, 50, 500]): + self.check_over_forward(time_step=t, num_inference_steps=num_inference_steps) + + def test_eta(self): + for t, eta in zip([1, 10, 49], [0.0, 0.5, 1.0]): + self.check_over_forward(time_step=t, eta=eta) + + def test_variance(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + assert torch.sum(torch.abs(scheduler._get_variance(0, 0) - 0.0)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(420, 400) - 0.14771)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(980, 960) - 0.32460)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(0, 0) - 0.0)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(487, 486) - 0.00979)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(999, 998) - 0.02)) < 1e-5 + + def test_batch_step_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps, eta = 10, 0.0 + scheduler.set_timesteps(num_inference_steps) + + model = self.dummy_model() + sample1 = self.dummy_sample_deter + sample2 = self.dummy_sample_deter + 0.1 + sample3 = self.dummy_sample_deter - 0.1 + + per_sample_batch = sample1.shape[0] + samples = torch.stack([sample1, sample2, sample3], dim=0) + timesteps = torch.arange(num_inference_steps)[0:3, None].repeat(1, per_sample_batch) + + residual = model(samples.flatten(0, 1), timesteps.flatten(0, 1)) + pred_prev_sample = scheduler.batch_step_no_noise(residual, timesteps.flatten(0, 1), samples.flatten(0, 1), eta) + + result_sum = torch.sum(torch.abs(pred_prev_sample)) + result_mean = torch.mean(torch.abs(pred_prev_sample)) + + assert abs(result_sum.item() - 1147.7904) < 1e-2 + assert abs(result_mean.item() - 0.4982) < 1e-3 + + def test_full_loop_no_noise(self): + sample = self.full_loop() + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 172.0067) < 1e-2 + assert abs(result_mean.item() - 0.223967) < 1e-3 + + def test_full_loop_with_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction") + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 52.5302) < 1e-2 + assert abs(result_mean.item() - 0.0684) < 1e-3 + + def test_full_loop_with_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=True, beta_start=0.01) + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 149.8295) < 1e-2 + assert abs(result_mean.item() - 0.1951) < 1e-3 + + def test_full_loop_with_no_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=False, beta_start=0.01) + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 149.0784) < 1e-2 + assert abs(result_mean.item() - 0.1941) < 1e-3 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps, eta = 10, 0.0 + t_start = 8 + + model = self.dummy_model() + sample = self.dummy_sample_deter + + scheduler.set_timesteps(num_inference_steps) + + # add noise + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for t in timesteps: + residual = model(sample, t) + sample = scheduler.step(residual, t, sample, eta).prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 354.5418) < 1e-2, f" expected result sum 354.5418, but get {result_sum}" + assert abs(result_mean.item() - 0.4616) < 1e-3, f" expected result mean 0.4616, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_ddpm.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_ddpm.py new file mode 100755 index 0000000..056b5d8 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_ddpm.py @@ -0,0 +1,222 @@ +import torch + +from diffusers import DDPMScheduler + +from .test_schedulers import SchedulerCommonTest + + +class DDPMSchedulerTest(SchedulerCommonTest): + scheduler_classes = (DDPMScheduler,) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "variance_type": "fixed_small", + "clip_sample": True, + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [1, 5, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001, 0.01, 0.1], [0.002, 0.02, 0.2, 2]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "squaredcos_cap_v2"]: + self.check_over_configs(beta_schedule=schedule) + + def test_variance_type(self): + for variance in ["fixed_small", "fixed_large", "other"]: + self.check_over_configs(variance_type=variance) + + def test_clip_sample(self): + for clip_sample in [True, False]: + self.check_over_configs(clip_sample=clip_sample) + + def test_thresholding(self): + self.check_over_configs(thresholding=False) + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "sample", "v_prediction"]: + self.check_over_configs( + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + ) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "sample", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_time_indices(self): + for t in [0, 500, 999]: + self.check_over_forward(time_step=t) + + def test_variance(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + assert torch.sum(torch.abs(scheduler._get_variance(0) - 0.0)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(487) - 0.00979)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(999) - 0.02)) < 1e-5 + + def test_rescale_betas_zero_snr(self): + for rescale_betas_zero_snr in [True, False]: + self.check_over_configs(rescale_betas_zero_snr=rescale_betas_zero_snr) + + def test_full_loop_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_trained_timesteps = len(scheduler) + + model = self.dummy_model() + sample = self.dummy_sample_deter + generator = torch.manual_seed(0) + + for t in reversed(range(num_trained_timesteps)): + # 1. predict noise residual + residual = model(sample, t) + + # 2. predict previous mean of sample x_t-1 + pred_prev_sample = scheduler.step(residual, t, sample, generator=generator).prev_sample + + # if t > 0: + # noise = self.dummy_sample_deter + # variance = scheduler.get_variance(t) ** (0.5) * noise + # + # sample = pred_prev_sample + variance + sample = pred_prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 258.9606) < 1e-2 + assert abs(result_mean.item() - 0.3372) < 1e-3 + + def test_full_loop_with_v_prediction(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(prediction_type="v_prediction") + scheduler = scheduler_class(**scheduler_config) + + num_trained_timesteps = len(scheduler) + + model = self.dummy_model() + sample = self.dummy_sample_deter + generator = torch.manual_seed(0) + + for t in reversed(range(num_trained_timesteps)): + # 1. predict noise residual + residual = model(sample, t) + + # 2. predict previous mean of sample x_t-1 + pred_prev_sample = scheduler.step(residual, t, sample, generator=generator).prev_sample + + # if t > 0: + # noise = self.dummy_sample_deter + # variance = scheduler.get_variance(t) ** (0.5) * noise + # + # sample = pred_prev_sample + variance + sample = pred_prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 202.0296) < 1e-2 + assert abs(result_mean.item() - 0.2631) < 1e-3 + + def test_custom_timesteps(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 1, 0] + + scheduler.set_timesteps(timesteps=timesteps) + + scheduler_timesteps = scheduler.timesteps + + for i, timestep in enumerate(scheduler_timesteps): + if i == len(timesteps) - 1: + expected_prev_t = -1 + else: + expected_prev_t = timesteps[i + 1] + + prev_t = scheduler.previous_timestep(timestep) + prev_t = prev_t.item() + + self.assertEqual(prev_t, expected_prev_t) + + def test_custom_timesteps_increasing_order(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 51, 0] + + with self.assertRaises(ValueError, msg="`custom_timesteps` must be in descending order."): + scheduler.set_timesteps(timesteps=timesteps) + + def test_custom_timesteps_passing_both_num_inference_steps_and_timesteps(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 1, 0] + num_inference_steps = len(timesteps) + + with self.assertRaises(ValueError, msg="Can only pass one of `num_inference_steps` or `custom_timesteps`."): + scheduler.set_timesteps(num_inference_steps=num_inference_steps, timesteps=timesteps) + + def test_custom_timesteps_too_large(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [scheduler.config.num_train_timesteps] + + with self.assertRaises( + ValueError, + msg="`timesteps` must start before `self.config.train_timesteps`: {scheduler.config.num_train_timesteps}}", + ): + scheduler.set_timesteps(timesteps=timesteps) + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_trained_timesteps = len(scheduler) + t_start = num_trained_timesteps - 2 + + model = self.dummy_model() + sample = self.dummy_sample_deter + generator = torch.manual_seed(0) + + # add noise + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for t in timesteps: + # 1. predict noise residual + residual = model(sample, t) + + # 2. predict previous mean of sample x_t-1 + pred_prev_sample = scheduler.step(residual, t, sample, generator=generator).prev_sample + sample = pred_prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 387.9466) < 1e-2, f" expected result sum 387.9466, but get {result_sum}" + assert abs(result_mean.item() - 0.5051) < 1e-3, f" expected result mean 0.5051, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_ddpm_parallel.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_ddpm_parallel.py new file mode 100755 index 0000000..c358ad9 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_ddpm_parallel.py @@ -0,0 +1,251 @@ +# Copyright 2024 ParaDiGMS authors and The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import torch + +from diffusers import DDPMParallelScheduler + +from .test_schedulers import SchedulerCommonTest + + +class DDPMParallelSchedulerTest(SchedulerCommonTest): + scheduler_classes = (DDPMParallelScheduler,) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "variance_type": "fixed_small", + "clip_sample": True, + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [1, 5, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001, 0.01, 0.1], [0.002, 0.02, 0.2, 2]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "squaredcos_cap_v2"]: + self.check_over_configs(beta_schedule=schedule) + + def test_variance_type(self): + for variance in ["fixed_small", "fixed_large", "other"]: + self.check_over_configs(variance_type=variance) + + def test_clip_sample(self): + for clip_sample in [True, False]: + self.check_over_configs(clip_sample=clip_sample) + + def test_thresholding(self): + self.check_over_configs(thresholding=False) + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "sample", "v_prediction"]: + self.check_over_configs( + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + ) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "sample", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_time_indices(self): + for t in [0, 500, 999]: + self.check_over_forward(time_step=t) + + def test_variance(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + assert torch.sum(torch.abs(scheduler._get_variance(0) - 0.0)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(487) - 0.00979)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(999) - 0.02)) < 1e-5 + + def test_rescale_betas_zero_snr(self): + for rescale_betas_zero_snr in [True, False]: + self.check_over_configs(rescale_betas_zero_snr=rescale_betas_zero_snr) + + def test_batch_step_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_trained_timesteps = len(scheduler) + + model = self.dummy_model() + sample1 = self.dummy_sample_deter + sample2 = self.dummy_sample_deter + 0.1 + sample3 = self.dummy_sample_deter - 0.1 + + per_sample_batch = sample1.shape[0] + samples = torch.stack([sample1, sample2, sample3], dim=0) + timesteps = torch.arange(num_trained_timesteps)[0:3, None].repeat(1, per_sample_batch) + + residual = model(samples.flatten(0, 1), timesteps.flatten(0, 1)) + pred_prev_sample = scheduler.batch_step_no_noise(residual, timesteps.flatten(0, 1), samples.flatten(0, 1)) + + result_sum = torch.sum(torch.abs(pred_prev_sample)) + result_mean = torch.mean(torch.abs(pred_prev_sample)) + + assert abs(result_sum.item() - 1153.1833) < 1e-2 + assert abs(result_mean.item() - 0.5005) < 1e-3 + + def test_full_loop_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_trained_timesteps = len(scheduler) + + model = self.dummy_model() + sample = self.dummy_sample_deter + generator = torch.manual_seed(0) + + for t in reversed(range(num_trained_timesteps)): + # 1. predict noise residual + residual = model(sample, t) + + # 2. predict previous mean of sample x_t-1 + pred_prev_sample = scheduler.step(residual, t, sample, generator=generator).prev_sample + + sample = pred_prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 258.9606) < 1e-2 + assert abs(result_mean.item() - 0.3372) < 1e-3 + + def test_full_loop_with_v_prediction(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(prediction_type="v_prediction") + scheduler = scheduler_class(**scheduler_config) + + num_trained_timesteps = len(scheduler) + + model = self.dummy_model() + sample = self.dummy_sample_deter + generator = torch.manual_seed(0) + + for t in reversed(range(num_trained_timesteps)): + # 1. predict noise residual + residual = model(sample, t) + + # 2. predict previous mean of sample x_t-1 + pred_prev_sample = scheduler.step(residual, t, sample, generator=generator).prev_sample + + sample = pred_prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 202.0296) < 1e-2 + assert abs(result_mean.item() - 0.2631) < 1e-3 + + def test_custom_timesteps(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 1, 0] + + scheduler.set_timesteps(timesteps=timesteps) + + scheduler_timesteps = scheduler.timesteps + + for i, timestep in enumerate(scheduler_timesteps): + if i == len(timesteps) - 1: + expected_prev_t = -1 + else: + expected_prev_t = timesteps[i + 1] + + prev_t = scheduler.previous_timestep(timestep) + prev_t = prev_t.item() + + self.assertEqual(prev_t, expected_prev_t) + + def test_custom_timesteps_increasing_order(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 51, 0] + + with self.assertRaises(ValueError, msg="`custom_timesteps` must be in descending order."): + scheduler.set_timesteps(timesteps=timesteps) + + def test_custom_timesteps_passing_both_num_inference_steps_and_timesteps(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 1, 0] + num_inference_steps = len(timesteps) + + with self.assertRaises(ValueError, msg="Can only pass one of `num_inference_steps` or `custom_timesteps`."): + scheduler.set_timesteps(num_inference_steps=num_inference_steps, timesteps=timesteps) + + def test_custom_timesteps_too_large(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [scheduler.config.num_train_timesteps] + + with self.assertRaises( + ValueError, + msg="`timesteps` must start before `self.config.train_timesteps`: {scheduler.config.num_train_timesteps}}", + ): + scheduler.set_timesteps(timesteps=timesteps) + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_trained_timesteps = len(scheduler) + t_start = num_trained_timesteps - 2 + + model = self.dummy_model() + sample = self.dummy_sample_deter + generator = torch.manual_seed(0) + + # add noise + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for t in timesteps: + # 1. predict noise residual + residual = model(sample, t) + + # 2. predict previous mean of sample x_t-1 + pred_prev_sample = scheduler.step(residual, t, sample, generator=generator).prev_sample + sample = pred_prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 387.9466) < 1e-2, f" expected result sum 387.9466, but get {result_sum}" + assert abs(result_mean.item() - 0.5051) < 1e-3, f" expected result mean 0.5051, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_deis.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_deis.py new file mode 100755 index 0000000..b2823a0 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_deis.py @@ -0,0 +1,265 @@ +import tempfile + +import torch + +from diffusers import ( + DEISMultistepScheduler, + DPMSolverMultistepScheduler, + DPMSolverSinglestepScheduler, + UniPCMultistepScheduler, +) + +from .test_schedulers import SchedulerCommonTest + + +class DEISMultistepSchedulerTest(SchedulerCommonTest): + scheduler_classes = (DEISMultistepScheduler,) + forward_default_kwargs = (("num_inference_steps", 25),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "solver_order": 2, + } + + config.update(**kwargs) + return config + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + new_scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + output, new_output = sample, sample + for t in range(time_step, time_step + scheduler.config.solver_order + 1): + t = scheduler.timesteps[t] + output = scheduler.step(residual, t, output, **kwargs).prev_sample + new_output = new_scheduler.step(residual, t, new_output, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_from_save_pretrained(self): + pass + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residuals (must be after setting timesteps) + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + # copy over dummy past residuals + new_scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residual (must be after setting timesteps) + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def full_loop(self, scheduler=None, **config): + if scheduler is None: + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + return sample + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # copy over dummy past residuals (must be done after set_timesteps) + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + time_step_0 = scheduler.timesteps[5] + time_step_1 = scheduler.timesteps[6] + + output_0 = scheduler.step(residual, time_step_0, sample, **kwargs).prev_sample + output_1 = scheduler.step(residual, time_step_1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_switch(self): + # make sure that iterating over schedulers with same config names gives same results + # for defaults + scheduler = DEISMultistepScheduler(**self.get_scheduler_config()) + sample = self.full_loop(scheduler=scheduler) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.23916) < 1e-3 + + scheduler = DPMSolverSinglestepScheduler.from_config(scheduler.config) + scheduler = DPMSolverMultistepScheduler.from_config(scheduler.config) + scheduler = UniPCMultistepScheduler.from_config(scheduler.config) + scheduler = DEISMultistepScheduler.from_config(scheduler.config) + + sample = self.full_loop(scheduler=scheduler) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.23916) < 1e-3 + + def test_timesteps(self): + for timesteps in [25, 50, 100, 999, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_thresholding(self): + self.check_over_configs(thresholding=False) + for order in [1, 2, 3]: + for solver_type in ["logrho"]: + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "sample"]: + self.check_over_configs( + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + algorithm_type="deis", + solver_order=order, + solver_type=solver_type, + ) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_solver_order_and_type(self): + for algorithm_type in ["deis"]: + for solver_type in ["logrho"]: + for order in [1, 2, 3]: + for prediction_type in ["epsilon", "sample"]: + self.check_over_configs( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + algorithm_type=algorithm_type, + ) + sample = self.full_loop( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + algorithm_type=algorithm_type, + ) + assert not torch.isnan(sample).any(), "Samples have nan numbers" + + def test_lower_order_final(self): + self.check_over_configs(lower_order_final=True) + self.check_over_configs(lower_order_final=False) + + def test_inference_steps(self): + for num_inference_steps in [1, 2, 3, 5, 10, 50, 100, 999, 1000]: + self.check_over_forward(num_inference_steps=num_inference_steps, time_step=0) + + def test_full_loop_no_noise(self): + sample = self.full_loop() + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.23916) < 1e-3 + + def test_full_loop_with_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction") + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.091) < 1e-3 + + def test_fp16_support(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(thresholding=True, dynamic_thresholding_ratio=0) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter.half() + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + assert sample.dtype == torch.float16 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + t_start = 8 + + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + # add noise + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 315.3016) < 1e-2, f" expected result sum 315.3016, but get {result_sum}" + assert abs(result_mean.item() - 0.41054) < 1e-3, f" expected result mean 0.41054, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_multi.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_multi.py new file mode 100755 index 0000000..fcf8881 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_multi.py @@ -0,0 +1,318 @@ +import tempfile + +import torch + +from diffusers import ( + DEISMultistepScheduler, + DPMSolverMultistepScheduler, + DPMSolverSinglestepScheduler, + UniPCMultistepScheduler, +) + +from .test_schedulers import SchedulerCommonTest + + +class DPMSolverMultistepSchedulerTest(SchedulerCommonTest): + scheduler_classes = (DPMSolverMultistepScheduler,) + forward_default_kwargs = (("num_inference_steps", 25),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "solver_order": 2, + "prediction_type": "epsilon", + "thresholding": False, + "sample_max_value": 1.0, + "algorithm_type": "dpmsolver++", + "solver_type": "midpoint", + "lower_order_final": False, + "euler_at_final": False, + "lambda_min_clipped": -float("inf"), + "variance_type": None, + "final_sigmas_type": "sigma_min", + } + + config.update(**kwargs) + return config + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + new_scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + output, new_output = sample, sample + for t in range(time_step, time_step + scheduler.config.solver_order + 1): + t = new_scheduler.timesteps[t] + output = scheduler.step(residual, t, output, **kwargs).prev_sample + new_output = new_scheduler.step(residual, t, new_output, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_from_save_pretrained(self): + pass + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residuals (must be after setting timesteps) + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + # copy over dummy past residuals + new_scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residual (must be after setting timesteps) + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + time_step = new_scheduler.timesteps[time_step] + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def full_loop(self, scheduler=None, **config): + if scheduler is None: + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + return sample + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # copy over dummy past residuals (must be done after set_timesteps) + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + time_step_0 = scheduler.timesteps[5] + time_step_1 = scheduler.timesteps[6] + + output_0 = scheduler.step(residual, time_step_0, sample, **kwargs).prev_sample + output_1 = scheduler.step(residual, time_step_1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_timesteps(self): + for timesteps in [25, 50, 100, 999, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_thresholding(self): + self.check_over_configs(thresholding=False) + for order in [1, 2, 3]: + for solver_type in ["midpoint", "heun"]: + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "sample"]: + self.check_over_configs( + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + algorithm_type="dpmsolver++", + solver_order=order, + solver_type=solver_type, + ) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_solver_order_and_type(self): + for algorithm_type in ["dpmsolver", "dpmsolver++", "sde-dpmsolver", "sde-dpmsolver++"]: + for solver_type in ["midpoint", "heun"]: + for order in [1, 2, 3]: + for prediction_type in ["epsilon", "sample"]: + if algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"]: + if order == 3: + continue + else: + self.check_over_configs( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + algorithm_type=algorithm_type, + ) + sample = self.full_loop( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + algorithm_type=algorithm_type, + ) + assert not torch.isnan(sample).any(), "Samples have nan numbers" + + def test_lower_order_final(self): + self.check_over_configs(lower_order_final=True) + self.check_over_configs(lower_order_final=False) + + def test_euler_at_final(self): + self.check_over_configs(euler_at_final=True) + self.check_over_configs(euler_at_final=False) + + def test_lambda_min_clipped(self): + self.check_over_configs(lambda_min_clipped=-float("inf")) + self.check_over_configs(lambda_min_clipped=-5.1) + + def test_variance_type(self): + self.check_over_configs(variance_type=None) + self.check_over_configs(variance_type="learned_range") + + def test_inference_steps(self): + for num_inference_steps in [1, 2, 3, 5, 10, 50, 100, 999, 1000]: + self.check_over_forward(num_inference_steps=num_inference_steps, time_step=0) + + def test_rescale_betas_zero_snr(self): + for rescale_betas_zero_snr in [True, False]: + self.check_over_configs(rescale_betas_zero_snr=rescale_betas_zero_snr) + + def test_full_loop_no_noise(self): + sample = self.full_loop() + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.3301) < 1e-3 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + t_start = 5 + + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + # add noise + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 318.4111) < 1e-2, f" expected result sum 318.4111, but get {result_sum}" + assert abs(result_mean.item() - 0.4146) < 1e-3, f" expected result mean 0.4146, but get {result_mean}" + + def test_full_loop_no_noise_thres(self): + sample = self.full_loop(thresholding=True, dynamic_thresholding_ratio=0.87, sample_max_value=0.5) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 1.1364) < 1e-3 + + def test_full_loop_with_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction") + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2251) < 1e-3 + + def test_full_loop_with_karras_and_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction", use_karras_sigmas=True) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2096) < 1e-3 + + def test_full_loop_with_lu_and_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction", use_lu_lambdas=True) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.1554) < 1e-3 + + def test_switch(self): + # make sure that iterating over schedulers with same config names gives same results + # for defaults + scheduler = DPMSolverMultistepScheduler(**self.get_scheduler_config()) + sample = self.full_loop(scheduler=scheduler) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.3301) < 1e-3 + + scheduler = DPMSolverSinglestepScheduler.from_config(scheduler.config) + scheduler = UniPCMultistepScheduler.from_config(scheduler.config) + scheduler = DEISMultistepScheduler.from_config(scheduler.config) + scheduler = DPMSolverMultistepScheduler.from_config(scheduler.config) + + sample = self.full_loop(scheduler=scheduler) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.3301) < 1e-3 + + def test_fp16_support(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(thresholding=True, dynamic_thresholding_ratio=0) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter.half() + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + assert sample.dtype == torch.float16 + + def test_duplicated_timesteps(self, **config): + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(scheduler.config.num_train_timesteps) + assert len(scheduler.timesteps) == scheduler.num_inference_steps diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_multi_inverse.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_multi_inverse.py new file mode 100755 index 0000000..014c901 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_multi_inverse.py @@ -0,0 +1,267 @@ +import tempfile + +import torch + +from diffusers import DPMSolverMultistepInverseScheduler, DPMSolverMultistepScheduler + +from .test_schedulers import SchedulerCommonTest + + +class DPMSolverMultistepSchedulerTest(SchedulerCommonTest): + scheduler_classes = (DPMSolverMultistepInverseScheduler,) + forward_default_kwargs = (("num_inference_steps", 25),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "solver_order": 2, + "prediction_type": "epsilon", + "thresholding": False, + "sample_max_value": 1.0, + "algorithm_type": "dpmsolver++", + "solver_type": "midpoint", + "lower_order_final": False, + "lambda_min_clipped": -float("inf"), + "variance_type": None, + } + + config.update(**kwargs) + return config + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + new_scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + output, new_output = sample, sample + for t in range(time_step, time_step + scheduler.config.solver_order + 1): + t = scheduler.timesteps[t] + output = scheduler.step(residual, t, output, **kwargs).prev_sample + new_output = new_scheduler.step(residual, t, new_output, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_from_save_pretrained(self): + pass + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residuals (must be after setting timesteps) + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + # copy over dummy past residuals + new_scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residual (must be after setting timesteps) + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def full_loop(self, scheduler=None, **config): + if scheduler is None: + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + return sample + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # copy over dummy past residuals (must be done after set_timesteps) + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + time_step_0 = scheduler.timesteps[5] + time_step_1 = scheduler.timesteps[6] + + output_0 = scheduler.step(residual, time_step_0, sample, **kwargs).prev_sample + output_1 = scheduler.step(residual, time_step_1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_timesteps(self): + for timesteps in [25, 50, 100, 999, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_thresholding(self): + self.check_over_configs(thresholding=False) + for order in [1, 2, 3]: + for solver_type in ["midpoint", "heun"]: + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "sample"]: + self.check_over_configs( + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + algorithm_type="dpmsolver++", + solver_order=order, + solver_type=solver_type, + ) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_solver_order_and_type(self): + for algorithm_type in ["dpmsolver", "dpmsolver++"]: + for solver_type in ["midpoint", "heun"]: + for order in [1, 2, 3]: + for prediction_type in ["epsilon", "sample"]: + self.check_over_configs( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + algorithm_type=algorithm_type, + ) + sample = self.full_loop( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + algorithm_type=algorithm_type, + ) + assert not torch.isnan(sample).any(), "Samples have nan numbers" + + def test_lower_order_final(self): + self.check_over_configs(lower_order_final=True) + self.check_over_configs(lower_order_final=False) + + def test_lambda_min_clipped(self): + self.check_over_configs(lambda_min_clipped=-float("inf")) + self.check_over_configs(lambda_min_clipped=-5.1) + + def test_variance_type(self): + self.check_over_configs(variance_type=None) + self.check_over_configs(variance_type="learned_range") + + def test_timestep_spacing(self): + for timestep_spacing in ["trailing", "leading"]: + self.check_over_configs(timestep_spacing=timestep_spacing) + + def test_inference_steps(self): + for num_inference_steps in [1, 2, 3, 5, 10, 50, 100, 999, 1000]: + self.check_over_forward(num_inference_steps=num_inference_steps, time_step=0) + + def test_full_loop_no_noise(self): + sample = self.full_loop() + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.7047) < 1e-3 + + def test_full_loop_no_noise_thres(self): + sample = self.full_loop(thresholding=True, dynamic_thresholding_ratio=0.87, sample_max_value=0.5) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 19.8933) < 1e-3 + + def test_full_loop_with_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction") + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 1.5194) < 1e-3 + + def test_full_loop_with_karras_and_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction", use_karras_sigmas=True) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 1.7833) < 2e-3 + + def test_switch(self): + # make sure that iterating over schedulers with same config names gives same results + # for defaults + scheduler = DPMSolverMultistepInverseScheduler(**self.get_scheduler_config()) + sample = self.full_loop(scheduler=scheduler) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.7047) < 1e-3 + + scheduler = DPMSolverMultistepScheduler.from_config(scheduler.config) + scheduler = DPMSolverMultistepInverseScheduler.from_config(scheduler.config) + + sample = self.full_loop(scheduler=scheduler) + new_result_mean = torch.mean(torch.abs(sample)) + + assert abs(new_result_mean.item() - result_mean.item()) < 1e-3 + + def test_fp16_support(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(thresholding=True, dynamic_thresholding_ratio=0) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter.half() + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + assert sample.dtype == torch.float16 + + def test_unique_timesteps(self, **config): + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(scheduler.config.num_train_timesteps) + assert len(scheduler.timesteps.unique()) == scheduler.num_inference_steps diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_sde.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_sde.py new file mode 100755 index 0000000..253a0a4 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_sde.py @@ -0,0 +1,167 @@ +import torch + +from diffusers import DPMSolverSDEScheduler +from diffusers.utils.testing_utils import require_torchsde, torch_device + +from .test_schedulers import SchedulerCommonTest + + +@require_torchsde +class DPMSolverSDESchedulerTest(SchedulerCommonTest): + scheduler_classes = (DPMSolverSDEScheduler,) + num_inference_steps = 10 + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1100, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "noise_sampler_seed": 0, + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [10, 50, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.00001, 0.0001, 0.001], [0.0002, 0.002, 0.02]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "scaled_linear"]: + self.check_over_configs(beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_full_loop_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["mps"]: + assert abs(result_sum.item() - 167.47821044921875) < 1e-2 + assert abs(result_mean.item() - 0.2178705964565277) < 1e-3 + elif torch_device in ["cuda"]: + assert abs(result_sum.item() - 171.59352111816406) < 1e-2 + assert abs(result_mean.item() - 0.22342906892299652) < 1e-3 + else: + assert abs(result_sum.item() - 162.52383422851562) < 1e-2 + assert abs(result_mean.item() - 0.211619570851326) < 1e-3 + + def test_full_loop_with_v_prediction(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(prediction_type="v_prediction") + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["mps"]: + assert abs(result_sum.item() - 124.77149200439453) < 1e-2 + assert abs(result_mean.item() - 0.16226289014816284) < 1e-3 + elif torch_device in ["cuda"]: + assert abs(result_sum.item() - 128.1663360595703) < 1e-2 + assert abs(result_mean.item() - 0.16688326001167297) < 1e-3 + else: + assert abs(result_sum.item() - 119.8487548828125) < 1e-2 + assert abs(result_mean.item() - 0.1560530662536621) < 1e-3 + + def test_full_loop_device(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + + model = self.dummy_model() + sample = self.dummy_sample_deter.to(torch_device) * scheduler.init_noise_sigma + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["mps"]: + assert abs(result_sum.item() - 167.46957397460938) < 1e-2 + assert abs(result_mean.item() - 0.21805934607982635) < 1e-3 + elif torch_device in ["cuda"]: + assert abs(result_sum.item() - 171.59353637695312) < 1e-2 + assert abs(result_mean.item() - 0.22342908382415771) < 1e-3 + else: + assert abs(result_sum.item() - 162.52383422851562) < 1e-2 + assert abs(result_mean.item() - 0.211619570851326) < 1e-3 + + def test_full_loop_device_karras_sigmas(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config, use_karras_sigmas=True) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + + model = self.dummy_model() + sample = self.dummy_sample_deter.to(torch_device) * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["mps"]: + assert abs(result_sum.item() - 176.66974135742188) < 1e-2 + assert abs(result_mean.item() - 0.23003872730981811) < 1e-2 + elif torch_device in ["cuda"]: + assert abs(result_sum.item() - 177.63653564453125) < 1e-2 + assert abs(result_mean.item() - 0.23003872730981811) < 1e-2 + else: + assert abs(result_sum.item() - 170.3135223388672) < 1e-2 + assert abs(result_mean.item() - 0.23003872730981811) < 1e-2 diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_single.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_single.py new file mode 100755 index 0000000..251a150 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_dpm_single.py @@ -0,0 +1,309 @@ +import tempfile + +import torch + +from diffusers import ( + DEISMultistepScheduler, + DPMSolverMultistepScheduler, + DPMSolverSinglestepScheduler, + UniPCMultistepScheduler, +) + +from .test_schedulers import SchedulerCommonTest + + +class DPMSolverSinglestepSchedulerTest(SchedulerCommonTest): + scheduler_classes = (DPMSolverSinglestepScheduler,) + forward_default_kwargs = (("num_inference_steps", 25),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "solver_order": 2, + "prediction_type": "epsilon", + "thresholding": False, + "sample_max_value": 1.0, + "algorithm_type": "dpmsolver++", + "solver_type": "midpoint", + "lambda_min_clipped": -float("inf"), + "variance_type": None, + "final_sigmas_type": "sigma_min", + } + + config.update(**kwargs) + return config + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + new_scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + output, new_output = sample, sample + for t in range(time_step, time_step + scheduler.config.solver_order + 1): + t = scheduler.timesteps[t] + output = scheduler.step(residual, t, output, **kwargs).prev_sample + new_output = new_scheduler.step(residual, t, new_output, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_from_save_pretrained(self): + pass + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residuals (must be after setting timesteps) + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + # copy over dummy past residuals + new_scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residual (must be after setting timesteps) + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def full_loop(self, scheduler=None, **config): + if scheduler is None: + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + return sample + + def test_full_uneven_loop(self): + scheduler = DPMSolverSinglestepScheduler(**self.get_scheduler_config()) + num_inference_steps = 50 + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + # make sure that the first t is uneven + for i, t in enumerate(scheduler.timesteps[3:]): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2574) < 1e-3 + + def test_timesteps(self): + for timesteps in [25, 50, 100, 999, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_switch(self): + # make sure that iterating over schedulers with same config names gives same results + # for defaults + scheduler = DPMSolverSinglestepScheduler(**self.get_scheduler_config()) + sample = self.full_loop(scheduler=scheduler) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2791) < 1e-3 + + scheduler = DEISMultistepScheduler.from_config(scheduler.config) + scheduler = DPMSolverMultistepScheduler.from_config(scheduler.config) + scheduler = UniPCMultistepScheduler.from_config(scheduler.config) + scheduler = DPMSolverSinglestepScheduler.from_config(scheduler.config) + + sample = self.full_loop(scheduler=scheduler) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2791) < 1e-3 + + def test_thresholding(self): + self.check_over_configs(thresholding=False) + for order in [1, 2, 3]: + for solver_type in ["midpoint", "heun"]: + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "sample"]: + self.check_over_configs( + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + algorithm_type="dpmsolver++", + solver_order=order, + solver_type=solver_type, + ) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_solver_order_and_type(self): + for algorithm_type in ["dpmsolver", "dpmsolver++"]: + for solver_type in ["midpoint", "heun"]: + for order in [1, 2, 3]: + for prediction_type in ["epsilon", "sample"]: + self.check_over_configs( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + algorithm_type=algorithm_type, + ) + sample = self.full_loop( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + algorithm_type=algorithm_type, + ) + assert not torch.isnan(sample).any(), "Samples have nan numbers" + + def test_lower_order_final(self): + self.check_over_configs(lower_order_final=True) + self.check_over_configs(lower_order_final=False) + + def test_lambda_min_clipped(self): + self.check_over_configs(lambda_min_clipped=-float("inf")) + self.check_over_configs(lambda_min_clipped=-5.1) + + def test_variance_type(self): + self.check_over_configs(variance_type=None) + self.check_over_configs(variance_type="learned_range") + + def test_inference_steps(self): + for num_inference_steps in [1, 2, 3, 5, 10, 50, 100, 999, 1000]: + self.check_over_forward(num_inference_steps=num_inference_steps, time_step=0) + + def test_full_loop_no_noise(self): + sample = self.full_loop() + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2791) < 1e-3 + + def test_full_loop_with_karras(self): + sample = self.full_loop(use_karras_sigmas=True) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2248) < 1e-3 + + def test_full_loop_with_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction") + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.1453) < 1e-3 + + def test_full_loop_with_karras_and_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction", use_karras_sigmas=True) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.0649) < 1e-3 + + def test_fp16_support(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(thresholding=True, dynamic_thresholding_ratio=0) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter.half() + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + assert sample.dtype == torch.float16 + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # copy over dummy past residuals (must be done after set_timesteps) + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + time_step_0 = scheduler.timesteps[0] + time_step_1 = scheduler.timesteps[1] + + output_0 = scheduler.step(residual, time_step_0, sample, **kwargs).prev_sample + output_1 = scheduler.step(residual, time_step_1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + t_start = 5 + + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + # add noise + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 269.2187) < 1e-2, f" expected result sum 269.2187, but get {result_sum}" + assert abs(result_mean.item() - 0.3505) < 1e-3, f" expected result mean 0.3505, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_edm_dpmsolver_multistep.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_edm_dpmsolver_multistep.py new file mode 100755 index 0000000..b5522f5 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_edm_dpmsolver_multistep.py @@ -0,0 +1,262 @@ +import tempfile +import unittest + +import torch + +from diffusers import ( + EDMDPMSolverMultistepScheduler, +) + +from .test_schedulers import SchedulerCommonTest + + +class EDMDPMSolverMultistepSchedulerTest(SchedulerCommonTest): + scheduler_classes = (EDMDPMSolverMultistepScheduler,) + forward_default_kwargs = (("num_inference_steps", 25),) + + def get_scheduler_config(self, **kwargs): + config = { + "sigma_min": 0.002, + "sigma_max": 80.0, + "sigma_data": 0.5, + "num_train_timesteps": 1000, + "solver_order": 2, + "prediction_type": "epsilon", + "thresholding": False, + "sample_max_value": 1.0, + "algorithm_type": "dpmsolver++", + "solver_type": "midpoint", + "lower_order_final": False, + "euler_at_final": False, + "final_sigmas_type": "sigma_min", + } + + config.update(**kwargs) + return config + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + new_scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + output, new_output = sample, sample + for t in range(time_step, time_step + scheduler.config.solver_order + 1): + t = new_scheduler.timesteps[t] + output = scheduler.step(residual, t, output, **kwargs).prev_sample + new_output = new_scheduler.step(residual, t, new_output, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_from_save_pretrained(self): + pass + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residuals (must be after setting timesteps) + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + # copy over dummy past residuals + new_scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residual (must be after setting timesteps) + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + time_step = new_scheduler.timesteps[time_step] + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def full_loop(self, scheduler=None, **config): + if scheduler is None: + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + return sample + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # copy over dummy past residuals (must be done after set_timesteps) + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + time_step_0 = scheduler.timesteps[5] + time_step_1 = scheduler.timesteps[6] + + output_0 = scheduler.step(residual, time_step_0, sample, **kwargs).prev_sample + output_1 = scheduler.step(residual, time_step_1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_timesteps(self): + for timesteps in [25, 50, 100, 999, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_thresholding(self): + self.check_over_configs(thresholding=False) + for order in [1, 2, 3]: + for solver_type in ["midpoint", "heun"]: + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs( + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + algorithm_type="dpmsolver++", + solver_order=order, + solver_type=solver_type, + ) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + # TODO (patil-suraj): Fix this test + @unittest.skip("Skip for now, as it failing currently but works with the actual model") + def test_solver_order_and_type(self): + for algorithm_type in ["dpmsolver++", "sde-dpmsolver++"]: + for solver_type in ["midpoint", "heun"]: + for order in [1, 2, 3]: + for prediction_type in ["epsilon", "v_prediction"]: + if algorithm_type == "sde-dpmsolver++": + if order == 3: + continue + else: + self.check_over_configs( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + algorithm_type=algorithm_type, + ) + sample = self.full_loop( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + algorithm_type=algorithm_type, + ) + assert ( + not torch.isnan(sample).any() + ), f"Samples have nan numbers, {order}, {solver_type}, {prediction_type}, {algorithm_type}" + + def test_lower_order_final(self): + self.check_over_configs(lower_order_final=True) + self.check_over_configs(lower_order_final=False) + + def test_euler_at_final(self): + self.check_over_configs(euler_at_final=True) + self.check_over_configs(euler_at_final=False) + + def test_inference_steps(self): + for num_inference_steps in [1, 2, 3, 5, 10, 50, 100, 999, 1000]: + self.check_over_forward(num_inference_steps=num_inference_steps, time_step=0) + + def test_full_loop_no_noise(self): + sample = self.full_loop() + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.0001) < 1e-3 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + t_start = 5 + + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + # add noise + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 8.1661) < 1e-2, f" expected result sum 8.1661, but get {result_sum}" + assert abs(result_mean.item() - 0.0106) < 1e-3, f" expected result mean 0.0106, but get {result_mean}" + + def test_full_loop_no_noise_thres(self): + sample = self.full_loop(thresholding=True, dynamic_thresholding_ratio=0.87, sample_max_value=0.5) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.0080) < 1e-3 + + def test_full_loop_with_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction") + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.0092) < 1e-3 + + def test_duplicated_timesteps(self, **config): + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(scheduler.config.num_train_timesteps) + assert len(scheduler.timesteps) == scheduler.num_inference_steps + + def test_trained_betas(self): + pass diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_edm_euler.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_edm_euler.py new file mode 100755 index 0000000..9d2adea --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_edm_euler.py @@ -0,0 +1,206 @@ +import inspect +import tempfile +import unittest +from typing import Dict, List, Tuple + +import torch + +from diffusers import EDMEulerScheduler + +from .test_schedulers import SchedulerCommonTest + + +class EDMEulerSchedulerTest(SchedulerCommonTest): + scheduler_classes = (EDMEulerScheduler,) + forward_default_kwargs = (("num_inference_steps", 10),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 256, + "sigma_min": 0.002, + "sigma_max": 80.0, + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [10, 50, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_full_loop_no_noise(self, num_inference_steps=10, seed=0): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + + for i, t in enumerate(scheduler.timesteps): + scaled_sample = scheduler.scale_model_input(sample, t) + + model_output = model(scaled_sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 34.1855) < 1e-3 + assert abs(result_mean.item() - 0.044) < 1e-3 + + def test_full_loop_device(self, num_inference_steps=10, seed=0): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + + for i, t in enumerate(scheduler.timesteps): + scaled_sample = scheduler.scale_model_input(sample, t) + + model_output = model(scaled_sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 34.1855) < 1e-3 + assert abs(result_mean.item() - 0.044) < 1e-3 + + # Override test_from_save_pretrined to use EDMEulerScheduler-specific logic + def test_from_save_pretrained(self): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + + scheduler.set_timesteps(num_inference_steps) + new_scheduler.set_timesteps(num_inference_steps) + timestep = scheduler.timesteps[0] + + sample = self.dummy_sample + + scaled_sample = scheduler.scale_model_input(sample, timestep) + residual = 0.1 * scaled_sample + + new_scaled_sample = new_scheduler.scale_model_input(sample, timestep) + new_residual = 0.1 * new_scaled_sample + + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + output = scheduler.step(residual, timestep, sample, **kwargs).prev_sample + + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + new_output = new_scheduler.step(new_residual, timestep, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + # Override test_from_save_pretrined to use EDMEulerScheduler-specific logic + def test_step_shape(self): + num_inference_steps = 10 + + scheduler_config = self.get_scheduler_config() + scheduler = self.scheduler_classes[0](**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + + timestep_0 = scheduler.timesteps[0] + timestep_1 = scheduler.timesteps[1] + + sample = self.dummy_sample + scaled_sample = scheduler.scale_model_input(sample, timestep_0) + residual = 0.1 * scaled_sample + + output_0 = scheduler.step(residual, timestep_0, sample).prev_sample + output_1 = scheduler.step(residual, timestep_1, sample).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + # Override test_from_save_pretrined to use EDMEulerScheduler-specific logic + def test_scheduler_outputs_equivalence(self): + def set_nan_tensor_to_zero(t): + t[t != t] = 0 + return t + + def recursive_check(tuple_object, dict_object): + if isinstance(tuple_object, (List, Tuple)): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object, dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif isinstance(tuple_object, Dict): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object.values(), dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif tuple_object is None: + return + else: + self.assertTrue( + torch.allclose( + set_nan_tensor_to_zero(tuple_object), set_nan_tensor_to_zero(dict_object), atol=1e-5 + ), + msg=( + "Tuple and dict output are not equal. Difference:" + f" {torch.max(torch.abs(tuple_object - dict_object))}. Tuple has `nan`:" + f" {torch.isnan(tuple_object).any()} and `inf`: {torch.isinf(tuple_object)}. Dict has" + f" `nan`: {torch.isnan(dict_object).any()} and `inf`: {torch.isinf(dict_object)}." + ), + ) + + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", 50) + + timestep = 0 + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + timestep = scheduler.timesteps[0] + + sample = self.dummy_sample + scaled_sample = scheduler.scale_model_input(sample, timestep) + residual = 0.1 * scaled_sample + + # Set the seed before state as some schedulers are stochastic like EulerAncestralDiscreteScheduler, EulerDiscreteScheduler + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + outputs_dict = scheduler.step(residual, timestep, sample, **kwargs) + + scheduler.set_timesteps(num_inference_steps) + + scaled_sample = scheduler.scale_model_input(sample, timestep) + residual = 0.1 * scaled_sample + + # Set the seed before state as some schedulers are stochastic like EulerAncestralDiscreteScheduler, EulerDiscreteScheduler + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + outputs_tuple = scheduler.step(residual, timestep, sample, return_dict=False, **kwargs) + + recursive_check(outputs_tuple, outputs_dict) + + @unittest.skip(reason="EDMEulerScheduler does not support beta schedules.") + def test_trained_betas(self): + pass diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_euler.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_euler.py new file mode 100755 index 0000000..41c418c --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_euler.py @@ -0,0 +1,191 @@ +import torch + +from diffusers import EulerDiscreteScheduler +from diffusers.utils.testing_utils import torch_device + +from .test_schedulers import SchedulerCommonTest + + +class EulerDiscreteSchedulerTest(SchedulerCommonTest): + scheduler_classes = (EulerDiscreteScheduler,) + num_inference_steps = 10 + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1100, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [10, 50, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.00001, 0.0001, 0.001], [0.0002, 0.002, 0.02]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "scaled_linear"]: + self.check_over_configs(beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_timestep_type(self): + timestep_types = ["discrete", "continuous"] + for timestep_type in timestep_types: + self.check_over_configs(timestep_type=timestep_type) + + def test_karras_sigmas(self): + self.check_over_configs(use_karras_sigmas=True, sigma_min=0.02, sigma_max=700.0) + + def test_rescale_betas_zero_snr(self): + for rescale_betas_zero_snr in [True, False]: + self.check_over_configs(rescale_betas_zero_snr=rescale_betas_zero_snr) + + def test_full_loop_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 10.0807) < 1e-2 + assert abs(result_mean.item() - 0.0131) < 1e-3 + + def test_full_loop_with_v_prediction(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(prediction_type="v_prediction") + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 0.0002) < 1e-2 + assert abs(result_mean.item() - 2.2676e-06) < 1e-3 + + def test_full_loop_device(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma.cpu() + sample = sample.to(torch_device) + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 10.0807) < 1e-2 + assert abs(result_mean.item() - 0.0131) < 1e-3 + + def test_full_loop_device_karras_sigmas(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config, use_karras_sigmas=True) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma.cpu() + sample = sample.to(torch_device) + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 124.52299499511719) < 1e-2 + assert abs(result_mean.item() - 0.16213932633399963) < 1e-3 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + + # add noise + t_start = self.num_inference_steps - 2 + noise = self.dummy_noise_deter + noise = noise.to(sample.device) + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 57062.9297) < 1e-2, f" expected result sum 57062.9297, but get {result_sum}" + assert abs(result_mean.item() - 74.3007) < 1e-3, f" expected result mean 74.3007, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_euler_ancestral.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_euler_ancestral.py new file mode 100755 index 0000000..9f22ab3 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_euler_ancestral.py @@ -0,0 +1,156 @@ +import torch + +from diffusers import EulerAncestralDiscreteScheduler +from diffusers.utils.testing_utils import torch_device + +from .test_schedulers import SchedulerCommonTest + + +class EulerAncestralDiscreteSchedulerTest(SchedulerCommonTest): + scheduler_classes = (EulerAncestralDiscreteScheduler,) + num_inference_steps = 10 + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1100, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [10, 50, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.00001, 0.0001, 0.001], [0.0002, 0.002, 0.02]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "scaled_linear"]: + self.check_over_configs(beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_rescale_betas_zero_snr(self): + for rescale_betas_zero_snr in [True, False]: + self.check_over_configs(rescale_betas_zero_snr=rescale_betas_zero_snr) + + def test_full_loop_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma.cpu() + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 152.3192) < 1e-2 + assert abs(result_mean.item() - 0.1983) < 1e-3 + + def test_full_loop_with_v_prediction(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(prediction_type="v_prediction") + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 108.4439) < 1e-2 + assert abs(result_mean.item() - 0.1412) < 1e-3 + + def test_full_loop_device(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma.cpu() + sample = sample.to(torch_device) + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 152.3192) < 1e-2 + assert abs(result_mean.item() - 0.1983) < 1e-3 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + t_start = self.num_inference_steps - 2 + + scheduler.set_timesteps(self.num_inference_steps) + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + + # add noise + noise = self.dummy_noise_deter + noise = noise.to(sample.device) + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 56163.0508) < 1e-2, f" expected result sum 56163.0508, but get {result_sum}" + assert abs(result_mean.item() - 73.1290) < 1e-3, f" expected result mean 73.1290, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_flax.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_flax.py new file mode 100755 index 0000000..2855f09 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_flax.py @@ -0,0 +1,919 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import tempfile +import unittest +from typing import Dict, List, Tuple + +from diffusers import FlaxDDIMScheduler, FlaxDDPMScheduler, FlaxPNDMScheduler +from diffusers.utils import is_flax_available +from diffusers.utils.testing_utils import require_flax + + +if is_flax_available(): + import jax + import jax.numpy as jnp + from jax import random + + jax_device = jax.default_backend() + + +@require_flax +class FlaxSchedulerCommonTest(unittest.TestCase): + scheduler_classes = () + forward_default_kwargs = () + + @property + def dummy_sample(self): + batch_size = 4 + num_channels = 3 + height = 8 + width = 8 + + key1, key2 = random.split(random.PRNGKey(0)) + sample = random.uniform(key1, (batch_size, num_channels, height, width)) + + return sample, key2 + + @property + def dummy_sample_deter(self): + batch_size = 4 + num_channels = 3 + height = 8 + width = 8 + + num_elems = batch_size * num_channels * height * width + sample = jnp.arange(num_elems) + sample = sample.reshape(num_channels, height, width, batch_size) + sample = sample / num_elems + return jnp.transpose(sample, (3, 0, 1, 2)) + + def get_scheduler_config(self): + raise NotImplementedError + + def dummy_model(self): + def model(sample, t, *args): + return sample * t / (t + 1) + + return model + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + sample, key = self.dummy_sample + residual = 0.1 * sample + + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler, new_state = scheduler_class.from_pretrained(tmpdirname) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + new_state = new_scheduler.set_timesteps(new_state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + output = scheduler.step(state, residual, time_step, sample, key, **kwargs).prev_sample + new_output = new_scheduler.step(new_state, residual, time_step, sample, key, **kwargs).prev_sample + + assert jnp.sum(jnp.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + kwargs.update(forward_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + sample, key = self.dummy_sample + residual = 0.1 * sample + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler, new_state = scheduler_class.from_pretrained(tmpdirname) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + new_state = new_scheduler.set_timesteps(new_state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + output = scheduler.step(state, residual, time_step, sample, key, **kwargs).prev_sample + new_output = new_scheduler.step(new_state, residual, time_step, sample, key, **kwargs).prev_sample + + assert jnp.sum(jnp.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_from_save_pretrained(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + sample, key = self.dummy_sample + residual = 0.1 * sample + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler, new_state = scheduler_class.from_pretrained(tmpdirname) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + new_state = new_scheduler.set_timesteps(new_state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + output = scheduler.step(state, residual, 1, sample, key, **kwargs).prev_sample + new_output = new_scheduler.step(new_state, residual, 1, sample, key, **kwargs).prev_sample + + assert jnp.sum(jnp.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + sample, key = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + output_0 = scheduler.step(state, residual, 0, sample, key, **kwargs).prev_sample + output_1 = scheduler.step(state, residual, 1, sample, key, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_scheduler_outputs_equivalence(self): + def set_nan_tensor_to_zero(t): + return t.at[t != t].set(0) + + def recursive_check(tuple_object, dict_object): + if isinstance(tuple_object, (List, Tuple)): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object, dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif isinstance(tuple_object, Dict): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object.values(), dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif tuple_object is None: + return + else: + self.assertTrue( + jnp.allclose(set_nan_tensor_to_zero(tuple_object), set_nan_tensor_to_zero(dict_object), atol=1e-5), + msg=( + "Tuple and dict output are not equal. Difference:" + f" {jnp.max(jnp.abs(tuple_object - dict_object))}. Tuple has `nan`:" + f" {jnp.isnan(tuple_object).any()} and `inf`: {jnp.isinf(tuple_object)}. Dict has" + f" `nan`: {jnp.isnan(dict_object).any()} and `inf`: {jnp.isinf(dict_object)}." + ), + ) + + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + sample, key = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + outputs_dict = scheduler.step(state, residual, 0, sample, key, **kwargs) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + outputs_tuple = scheduler.step(state, residual, 0, sample, key, return_dict=False, **kwargs) + + recursive_check(outputs_tuple[0], outputs_dict.prev_sample) + + def test_deprecated_kwargs(self): + for scheduler_class in self.scheduler_classes: + has_kwarg_in_model_class = "kwargs" in inspect.signature(scheduler_class.__init__).parameters + has_deprecated_kwarg = len(scheduler_class._deprecated_kwargs) > 0 + + if has_kwarg_in_model_class and not has_deprecated_kwarg: + raise ValueError( + f"{scheduler_class} has `**kwargs` in its __init__ method but has not defined any deprecated" + " kwargs under the `_deprecated_kwargs` class attribute. Make sure to either remove `**kwargs` if" + " there are no deprecated arguments or add the deprecated argument with `_deprecated_kwargs =" + " []`" + ) + + if not has_kwarg_in_model_class and has_deprecated_kwarg: + raise ValueError( + f"{scheduler_class} doesn't have `**kwargs` in its __init__ method but has defined deprecated" + " kwargs under the `_deprecated_kwargs` class attribute. Make sure to either add the `**kwargs`" + f" argument to {self.model_class}.__init__ if there are deprecated arguments or remove the" + " deprecated argument from `_deprecated_kwargs = []`" + ) + + +@require_flax +class FlaxDDPMSchedulerTest(FlaxSchedulerCommonTest): + scheduler_classes = (FlaxDDPMScheduler,) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "variance_type": "fixed_small", + "clip_sample": True, + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [1, 5, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001, 0.01, 0.1], [0.002, 0.02, 0.2, 2]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "squaredcos_cap_v2"]: + self.check_over_configs(beta_schedule=schedule) + + def test_variance_type(self): + for variance in ["fixed_small", "fixed_large", "other"]: + self.check_over_configs(variance_type=variance) + + def test_clip_sample(self): + for clip_sample in [True, False]: + self.check_over_configs(clip_sample=clip_sample) + + def test_time_indices(self): + for t in [0, 500, 999]: + self.check_over_forward(time_step=t) + + def test_variance(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + assert jnp.sum(jnp.abs(scheduler._get_variance(state, 0) - 0.0)) < 1e-5 + assert jnp.sum(jnp.abs(scheduler._get_variance(state, 487) - 0.00979)) < 1e-5 + assert jnp.sum(jnp.abs(scheduler._get_variance(state, 999) - 0.02)) < 1e-5 + + def test_full_loop_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + num_trained_timesteps = len(scheduler) + + model = self.dummy_model() + sample = self.dummy_sample_deter + key1, key2 = random.split(random.PRNGKey(0)) + + for t in reversed(range(num_trained_timesteps)): + # 1. predict noise residual + residual = model(sample, t) + + # 2. predict previous mean of sample x_t-1 + output = scheduler.step(state, residual, t, sample, key1) + pred_prev_sample = output.prev_sample + state = output.state + key1, key2 = random.split(key2) + + # if t > 0: + # noise = self.dummy_sample_deter + # variance = scheduler.get_variance(t) ** (0.5) * noise + # + # sample = pred_prev_sample + variance + sample = pred_prev_sample + + result_sum = jnp.sum(jnp.abs(sample)) + result_mean = jnp.mean(jnp.abs(sample)) + + if jax_device == "tpu": + assert abs(result_sum - 255.0714) < 1e-2 + assert abs(result_mean - 0.332124) < 1e-3 + else: + assert abs(result_sum - 255.1113) < 1e-2 + assert abs(result_mean - 0.332176) < 1e-3 + + +@require_flax +class FlaxDDIMSchedulerTest(FlaxSchedulerCommonTest): + scheduler_classes = (FlaxDDIMScheduler,) + forward_default_kwargs = (("num_inference_steps", 50),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + } + + config.update(**kwargs) + return config + + def full_loop(self, **config): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + key1, key2 = random.split(random.PRNGKey(0)) + + num_inference_steps = 10 + + model = self.dummy_model() + sample = self.dummy_sample_deter + + state = scheduler.set_timesteps(state, num_inference_steps) + + for t in state.timesteps: + residual = model(sample, t) + output = scheduler.step(state, residual, t, sample) + sample = output.prev_sample + state = output.state + key1, key2 = random.split(key2) + + return sample + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + sample, _ = self.dummy_sample + residual = 0.1 * sample + + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler, new_state = scheduler_class.from_pretrained(tmpdirname) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + new_state = new_scheduler.set_timesteps(new_state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + output = scheduler.step(state, residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(new_state, residual, time_step, sample, **kwargs).prev_sample + + assert jnp.sum(jnp.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_from_save_pretrained(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + sample, _ = self.dummy_sample + residual = 0.1 * sample + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler, new_state = scheduler_class.from_pretrained(tmpdirname) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + new_state = new_scheduler.set_timesteps(new_state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + output = scheduler.step(state, residual, 1, sample, **kwargs).prev_sample + new_output = new_scheduler.step(new_state, residual, 1, sample, **kwargs).prev_sample + + assert jnp.sum(jnp.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + kwargs.update(forward_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + sample, _ = self.dummy_sample + residual = 0.1 * sample + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler, new_state = scheduler_class.from_pretrained(tmpdirname) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + new_state = new_scheduler.set_timesteps(new_state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + output = scheduler.step(state, residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(new_state, residual, time_step, sample, **kwargs).prev_sample + + assert jnp.sum(jnp.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_scheduler_outputs_equivalence(self): + def set_nan_tensor_to_zero(t): + return t.at[t != t].set(0) + + def recursive_check(tuple_object, dict_object): + if isinstance(tuple_object, (List, Tuple)): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object, dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif isinstance(tuple_object, Dict): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object.values(), dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif tuple_object is None: + return + else: + self.assertTrue( + jnp.allclose(set_nan_tensor_to_zero(tuple_object), set_nan_tensor_to_zero(dict_object), atol=1e-5), + msg=( + "Tuple and dict output are not equal. Difference:" + f" {jnp.max(jnp.abs(tuple_object - dict_object))}. Tuple has `nan`:" + f" {jnp.isnan(tuple_object).any()} and `inf`: {jnp.isinf(tuple_object)}. Dict has" + f" `nan`: {jnp.isnan(dict_object).any()} and `inf`: {jnp.isinf(dict_object)}." + ), + ) + + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + sample, _ = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + outputs_dict = scheduler.step(state, residual, 0, sample, **kwargs) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + outputs_tuple = scheduler.step(state, residual, 0, sample, return_dict=False, **kwargs) + + recursive_check(outputs_tuple[0], outputs_dict.prev_sample) + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + sample, _ = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + output_0 = scheduler.step(state, residual, 0, sample, **kwargs).prev_sample + output_1 = scheduler.step(state, residual, 1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_timesteps(self): + for timesteps in [100, 500, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_steps_offset(self): + for steps_offset in [0, 1]: + self.check_over_configs(steps_offset=steps_offset) + + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(steps_offset=1) + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + state = scheduler.set_timesteps(state, 5) + assert jnp.equal(state.timesteps, jnp.array([801, 601, 401, 201, 1])).all() + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001, 0.01, 0.1], [0.002, 0.02, 0.2, 2]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "squaredcos_cap_v2"]: + self.check_over_configs(beta_schedule=schedule) + + def test_time_indices(self): + for t in [1, 10, 49]: + self.check_over_forward(time_step=t) + + def test_inference_steps(self): + for t, num_inference_steps in zip([1, 10, 50], [10, 50, 500]): + self.check_over_forward(time_step=t, num_inference_steps=num_inference_steps) + + def test_variance(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + assert jnp.sum(jnp.abs(scheduler._get_variance(state, 0, 0) - 0.0)) < 1e-5 + assert jnp.sum(jnp.abs(scheduler._get_variance(state, 420, 400) - 0.14771)) < 1e-5 + assert jnp.sum(jnp.abs(scheduler._get_variance(state, 980, 960) - 0.32460)) < 1e-5 + assert jnp.sum(jnp.abs(scheduler._get_variance(state, 0, 0) - 0.0)) < 1e-5 + assert jnp.sum(jnp.abs(scheduler._get_variance(state, 487, 486) - 0.00979)) < 1e-5 + assert jnp.sum(jnp.abs(scheduler._get_variance(state, 999, 998) - 0.02)) < 1e-5 + + def test_full_loop_no_noise(self): + sample = self.full_loop() + + result_sum = jnp.sum(jnp.abs(sample)) + result_mean = jnp.mean(jnp.abs(sample)) + + assert abs(result_sum - 172.0067) < 1e-2 + assert abs(result_mean - 0.223967) < 1e-3 + + def test_full_loop_with_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=True, beta_start=0.01) + result_sum = jnp.sum(jnp.abs(sample)) + result_mean = jnp.mean(jnp.abs(sample)) + + if jax_device == "tpu": + assert abs(result_sum - 149.8409) < 1e-2 + assert abs(result_mean - 0.1951) < 1e-3 + else: + assert abs(result_sum - 149.8295) < 1e-2 + assert abs(result_mean - 0.1951) < 1e-3 + + def test_full_loop_with_no_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=False, beta_start=0.01) + result_sum = jnp.sum(jnp.abs(sample)) + result_mean = jnp.mean(jnp.abs(sample)) + + if jax_device == "tpu": + pass + # FIXME: both result_sum and result_mean are nan on TPU + # assert jnp.isnan(result_sum) + # assert jnp.isnan(result_mean) + else: + assert abs(result_sum - 149.0784) < 1e-2 + assert abs(result_mean - 0.1941) < 1e-3 + + def test_prediction_type(self): + for prediction_type in ["epsilon", "sample", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + +@require_flax +class FlaxPNDMSchedulerTest(FlaxSchedulerCommonTest): + scheduler_classes = (FlaxPNDMScheduler,) + forward_default_kwargs = (("num_inference_steps", 50),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + } + + config.update(**kwargs) + return config + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample, _ = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = jnp.array([residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05]) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + state = scheduler.set_timesteps(state, num_inference_steps, shape=sample.shape) + # copy over dummy past residuals + state = state.replace(ets=dummy_past_residuals[:]) + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler, new_state = scheduler_class.from_pretrained(tmpdirname) + new_state = new_scheduler.set_timesteps(new_state, num_inference_steps, shape=sample.shape) + # copy over dummy past residuals + new_state = new_state.replace(ets=dummy_past_residuals[:]) + + (prev_sample, state) = scheduler.step_prk(state, residual, time_step, sample, **kwargs) + (new_prev_sample, new_state) = new_scheduler.step_prk(new_state, residual, time_step, sample, **kwargs) + + assert jnp.sum(jnp.abs(prev_sample - new_prev_sample)) < 1e-5, "Scheduler outputs are not identical" + + output, _ = scheduler.step_plms(state, residual, time_step, sample, **kwargs) + new_output, _ = new_scheduler.step_plms(new_state, residual, time_step, sample, **kwargs) + + assert jnp.sum(jnp.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_from_save_pretrained(self): + pass + + def test_scheduler_outputs_equivalence(self): + def set_nan_tensor_to_zero(t): + return t.at[t != t].set(0) + + def recursive_check(tuple_object, dict_object): + if isinstance(tuple_object, (List, Tuple)): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object, dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif isinstance(tuple_object, Dict): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object.values(), dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif tuple_object is None: + return + else: + self.assertTrue( + jnp.allclose(set_nan_tensor_to_zero(tuple_object), set_nan_tensor_to_zero(dict_object), atol=1e-5), + msg=( + "Tuple and dict output are not equal. Difference:" + f" {jnp.max(jnp.abs(tuple_object - dict_object))}. Tuple has `nan`:" + f" {jnp.isnan(tuple_object).any()} and `inf`: {jnp.isinf(tuple_object)}. Dict has" + f" `nan`: {jnp.isnan(dict_object).any()} and `inf`: {jnp.isinf(dict_object)}." + ), + ) + + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + sample, _ = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps, shape=sample.shape) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + outputs_dict = scheduler.step(state, residual, 0, sample, **kwargs) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps, shape=sample.shape) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + outputs_tuple = scheduler.step(state, residual, 0, sample, return_dict=False, **kwargs) + + recursive_check(outputs_tuple[0], outputs_dict.prev_sample) + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample, _ = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = jnp.array([residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05]) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + state = scheduler.set_timesteps(state, num_inference_steps, shape=sample.shape) + + # copy over dummy past residuals (must be after setting timesteps) + scheduler.ets = dummy_past_residuals[:] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler, new_state = scheduler_class.from_pretrained(tmpdirname) + # copy over dummy past residuals + new_state = new_scheduler.set_timesteps(new_state, num_inference_steps, shape=sample.shape) + + # copy over dummy past residual (must be after setting timesteps) + new_state.replace(ets=dummy_past_residuals[:]) + + output, state = scheduler.step_prk(state, residual, time_step, sample, **kwargs) + new_output, new_state = new_scheduler.step_prk(new_state, residual, time_step, sample, **kwargs) + + assert jnp.sum(jnp.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + output, _ = scheduler.step_plms(state, residual, time_step, sample, **kwargs) + new_output, _ = new_scheduler.step_plms(new_state, residual, time_step, sample, **kwargs) + + assert jnp.sum(jnp.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def full_loop(self, **config): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter + state = scheduler.set_timesteps(state, num_inference_steps, shape=sample.shape) + + for i, t in enumerate(state.prk_timesteps): + residual = model(sample, t) + sample, state = scheduler.step_prk(state, residual, t, sample) + + for i, t in enumerate(state.plms_timesteps): + residual = model(sample, t) + sample, state = scheduler.step_plms(state, residual, t, sample) + + return sample + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + sample, _ = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + state = scheduler.set_timesteps(state, num_inference_steps, shape=sample.shape) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # copy over dummy past residuals (must be done after set_timesteps) + dummy_past_residuals = jnp.array([residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05]) + state = state.replace(ets=dummy_past_residuals[:]) + + output_0, state = scheduler.step_prk(state, residual, 0, sample, **kwargs) + output_1, state = scheduler.step_prk(state, residual, 1, sample, **kwargs) + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + output_0, state = scheduler.step_plms(state, residual, 0, sample, **kwargs) + output_1, state = scheduler.step_plms(state, residual, 1, sample, **kwargs) + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_timesteps(self): + for timesteps in [100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_steps_offset(self): + for steps_offset in [0, 1]: + self.check_over_configs(steps_offset=steps_offset) + + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(steps_offset=1) + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + state = scheduler.set_timesteps(state, 10, shape=()) + assert jnp.equal( + state.timesteps, + jnp.array([901, 851, 851, 801, 801, 751, 751, 701, 701, 651, 651, 601, 601, 501, 401, 301, 201, 101, 1]), + ).all() + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001], [0.002, 0.02]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "squaredcos_cap_v2"]: + self.check_over_configs(beta_schedule=schedule) + + def test_time_indices(self): + for t in [1, 5, 10]: + self.check_over_forward(time_step=t) + + def test_inference_steps(self): + for t, num_inference_steps in zip([1, 5, 10], [10, 50, 100]): + self.check_over_forward(num_inference_steps=num_inference_steps) + + def test_pow_of_3_inference_steps(self): + # earlier version of set_timesteps() caused an error indexing alpha's with inference steps as power of 3 + num_inference_steps = 27 + + for scheduler_class in self.scheduler_classes: + sample, _ = self.dummy_sample + residual = 0.1 * sample + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + state = scheduler.set_timesteps(state, num_inference_steps, shape=sample.shape) + + # before power of 3 fix, would error on first step, so we only need to do two + for i, t in enumerate(state.prk_timesteps[:2]): + sample, state = scheduler.step_prk(state, residual, t, sample) + + def test_inference_plms_no_past_residuals(self): + with self.assertRaises(ValueError): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + state = scheduler.create_state() + + scheduler.step_plms(state, self.dummy_sample, 1, self.dummy_sample).prev_sample + + def test_full_loop_no_noise(self): + sample = self.full_loop() + result_sum = jnp.sum(jnp.abs(sample)) + result_mean = jnp.mean(jnp.abs(sample)) + + if jax_device == "tpu": + assert abs(result_sum - 198.1275) < 1e-2 + assert abs(result_mean - 0.2580) < 1e-3 + else: + assert abs(result_sum - 198.1318) < 1e-2 + assert abs(result_mean - 0.2580) < 1e-3 + + def test_full_loop_with_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=True, beta_start=0.01) + result_sum = jnp.sum(jnp.abs(sample)) + result_mean = jnp.mean(jnp.abs(sample)) + + if jax_device == "tpu": + assert abs(result_sum - 186.83226) < 1e-2 + assert abs(result_mean - 0.24327) < 1e-3 + else: + assert abs(result_sum - 186.9466) < 1e-2 + assert abs(result_mean - 0.24342) < 1e-3 + + def test_full_loop_with_no_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=False, beta_start=0.01) + result_sum = jnp.sum(jnp.abs(sample)) + result_mean = jnp.mean(jnp.abs(sample)) + + if jax_device == "tpu": + assert abs(result_sum - 186.83226) < 1e-2 + assert abs(result_mean - 0.24327) < 1e-3 + else: + assert abs(result_sum - 186.9482) < 1e-2 + assert abs(result_mean - 0.2434) < 1e-3 diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_heun.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_heun.py new file mode 100755 index 0000000..df2b62d --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_heun.py @@ -0,0 +1,191 @@ +import torch + +from diffusers import HeunDiscreteScheduler +from diffusers.utils.testing_utils import torch_device + +from .test_schedulers import SchedulerCommonTest + + +class HeunDiscreteSchedulerTest(SchedulerCommonTest): + scheduler_classes = (HeunDiscreteScheduler,) + num_inference_steps = 10 + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1100, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [10, 50, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.00001, 0.0001, 0.001], [0.0002, 0.002, 0.02]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "scaled_linear", "exp"]: + self.check_over_configs(beta_schedule=schedule) + + def test_clip_sample(self): + for clip_sample_range in [1.0, 2.0, 3.0]: + self.check_over_configs(clip_sample_range=clip_sample_range, clip_sample=True) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction", "sample"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_full_loop_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["cpu", "mps"]: + assert abs(result_sum.item() - 0.1233) < 1e-2 + assert abs(result_mean.item() - 0.0002) < 1e-3 + else: + # CUDA + assert abs(result_sum.item() - 0.1233) < 1e-2 + assert abs(result_mean.item() - 0.0002) < 1e-3 + + def test_full_loop_with_v_prediction(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(prediction_type="v_prediction") + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["cpu", "mps"]: + assert abs(result_sum.item() - 4.6934e-07) < 1e-2 + assert abs(result_mean.item() - 6.1112e-10) < 1e-3 + else: + # CUDA + assert abs(result_sum.item() - 4.693428650170972e-07) < 1e-2 + assert abs(result_mean.item() - 0.0002) < 1e-3 + + def test_full_loop_device(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + + model = self.dummy_model() + sample = self.dummy_sample_deter.to(torch_device) * scheduler.init_noise_sigma + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if str(torch_device).startswith("cpu"): + # The following sum varies between 148 and 156 on mps. Why? + assert abs(result_sum.item() - 0.1233) < 1e-2 + assert abs(result_mean.item() - 0.0002) < 1e-3 + elif str(torch_device).startswith("mps"): + # Larger tolerance on mps + assert abs(result_mean.item() - 0.0002) < 1e-2 + else: + # CUDA + assert abs(result_sum.item() - 0.1233) < 1e-2 + assert abs(result_mean.item() - 0.0002) < 1e-3 + + def test_full_loop_device_karras_sigmas(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config, use_karras_sigmas=True) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + + model = self.dummy_model() + sample = self.dummy_sample_deter.to(torch_device) * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 0.00015) < 1e-2 + assert abs(result_mean.item() - 1.9869554535034695e-07) < 1e-2 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + t_start = self.num_inference_steps - 2 + noise = self.dummy_noise_deter + noise = noise.to(torch_device) + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 75074.8906) < 1e-2, f" expected result sum 75074.8906, but get {result_sum}" + assert abs(result_mean.item() - 97.7538) < 1e-3, f" expected result mean 97.7538, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_ipndm.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_ipndm.py new file mode 100755 index 0000000..87c8da3 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_ipndm.py @@ -0,0 +1,163 @@ +import tempfile + +import torch + +from diffusers import IPNDMScheduler + +from .test_schedulers import SchedulerCommonTest + + +class IPNDMSchedulerTest(SchedulerCommonTest): + scheduler_classes = (IPNDMScheduler,) + forward_default_kwargs = (("num_inference_steps", 50),) + + def get_scheduler_config(self, **kwargs): + config = {"num_train_timesteps": 1000} + config.update(**kwargs) + return config + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + scheduler.ets = dummy_past_residuals[:] + + if time_step is None: + time_step = scheduler.timesteps[len(scheduler.timesteps) // 2] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + new_scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + new_scheduler.ets = dummy_past_residuals[:] + + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_from_save_pretrained(self): + pass + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residuals (must be after setting timesteps) + scheduler.ets = dummy_past_residuals[:] + + if time_step is None: + time_step = scheduler.timesteps[len(scheduler.timesteps) // 2] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + # copy over dummy past residuals + new_scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residual (must be after setting timesteps) + new_scheduler.ets = dummy_past_residuals[:] + + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def full_loop(self, **config): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + scheduler._step_index = None + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + return sample + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # copy over dummy past residuals (must be done after set_timesteps) + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05] + scheduler.ets = dummy_past_residuals[:] + + time_step_0 = scheduler.timesteps[5] + time_step_1 = scheduler.timesteps[6] + + output_0 = scheduler.step(residual, time_step_0, sample, **kwargs).prev_sample + output_1 = scheduler.step(residual, time_step_1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + output_0 = scheduler.step(residual, time_step_0, sample, **kwargs).prev_sample + output_1 = scheduler.step(residual, time_step_1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_timesteps(self): + for timesteps in [100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps, time_step=None) + + def test_inference_steps(self): + for t, num_inference_steps in zip([1, 5, 10], [10, 50, 100]): + self.check_over_forward(num_inference_steps=num_inference_steps, time_step=None) + + def test_full_loop_no_noise(self): + sample = self.full_loop() + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 2540529) < 10 diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_kdpm2_ancestral.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_kdpm2_ancestral.py new file mode 100755 index 0000000..f6e8e96 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_kdpm2_ancestral.py @@ -0,0 +1,158 @@ +import torch + +from diffusers import KDPM2AncestralDiscreteScheduler +from diffusers.utils.testing_utils import torch_device + +from .test_schedulers import SchedulerCommonTest + + +class KDPM2AncestralDiscreteSchedulerTest(SchedulerCommonTest): + scheduler_classes = (KDPM2AncestralDiscreteScheduler,) + num_inference_steps = 10 + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1100, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [10, 50, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.00001, 0.0001, 0.001], [0.0002, 0.002, 0.02]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "scaled_linear"]: + self.check_over_configs(beta_schedule=schedule) + + def test_full_loop_no_noise(self): + if torch_device == "mps": + return + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 13849.3877) < 1e-2 + assert abs(result_mean.item() - 18.0331) < 5e-3 + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_full_loop_with_v_prediction(self): + if torch_device == "mps": + return + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(prediction_type="v_prediction") + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + generator = torch.manual_seed(0) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 328.9970) < 1e-2 + assert abs(result_mean.item() - 0.4284) < 1e-3 + + def test_full_loop_device(self): + if torch_device == "mps": + return + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter.to(torch_device) * scheduler.init_noise_sigma + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 13849.3818) < 1e-1 + assert abs(result_mean.item() - 18.0331) < 1e-3 + + def test_full_loop_with_noise(self): + if torch_device == "mps": + return + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + generator = torch.manual_seed(0) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + + # add noise + t_start = self.num_inference_steps - 2 + noise = self.dummy_noise_deter + noise = noise.to(sample.device) + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 93087.0312) < 1e-2, f" expected result sum 93087.0312, but get {result_sum}" + assert abs(result_mean.item() - 121.2071) < 5e-3, f" expected result mean 121.2071, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_kdpm2_discrete.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_kdpm2_discrete.py new file mode 100755 index 0000000..a992edc --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_kdpm2_discrete.py @@ -0,0 +1,166 @@ +import torch + +from diffusers import KDPM2DiscreteScheduler +from diffusers.utils.testing_utils import torch_device + +from .test_schedulers import SchedulerCommonTest + + +class KDPM2DiscreteSchedulerTest(SchedulerCommonTest): + scheduler_classes = (KDPM2DiscreteScheduler,) + num_inference_steps = 10 + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1100, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [10, 50, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.00001, 0.0001, 0.001], [0.0002, 0.002, 0.02]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "scaled_linear"]: + self.check_over_configs(beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_full_loop_with_v_prediction(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(prediction_type="v_prediction") + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["cpu", "mps"]: + assert abs(result_sum.item() - 4.6934e-07) < 1e-2 + assert abs(result_mean.item() - 6.1112e-10) < 1e-3 + else: + # CUDA + assert abs(result_sum.item() - 4.693428650170972e-07) < 1e-2 + assert abs(result_mean.item() - 0.0002) < 1e-3 + + def test_full_loop_no_noise(self): + if torch_device == "mps": + return + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["cpu", "mps"]: + assert abs(result_sum.item() - 20.4125) < 1e-2 + assert abs(result_mean.item() - 0.0266) < 1e-3 + else: + # CUDA + assert abs(result_sum.item() - 20.4125) < 1e-2 + assert abs(result_mean.item() - 0.0266) < 1e-3 + + def test_full_loop_device(self): + if torch_device == "mps": + return + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + + model = self.dummy_model() + sample = self.dummy_sample_deter.to(torch_device) * scheduler.init_noise_sigma + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if str(torch_device).startswith("cpu"): + # The following sum varies between 148 and 156 on mps. Why? + assert abs(result_sum.item() - 20.4125) < 1e-2 + assert abs(result_mean.item() - 0.0266) < 1e-3 + else: + # CUDA + assert abs(result_sum.item() - 20.4125) < 1e-2 + assert abs(result_mean.item() - 0.0266) < 1e-3 + + def test_full_loop_with_noise(self): + if torch_device == "mps": + return + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + # add noise + t_start = self.num_inference_steps - 2 + noise = self.dummy_noise_deter + noise = noise.to(sample.device) + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 70408.4062) < 1e-2, f" expected result sum 70408.4062, but get {result_sum}" + assert abs(result_mean.item() - 91.6776) < 1e-3, f" expected result mean 91.6776, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_lcm.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_lcm.py new file mode 100755 index 0000000..c2c6530 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_lcm.py @@ -0,0 +1,300 @@ +import tempfile +from typing import Dict, List, Tuple + +import torch + +from diffusers import LCMScheduler +from diffusers.utils.testing_utils import torch_device + +from .test_schedulers import SchedulerCommonTest + + +class LCMSchedulerTest(SchedulerCommonTest): + scheduler_classes = (LCMScheduler,) + forward_default_kwargs = (("num_inference_steps", 10),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.00085, + "beta_end": 0.0120, + "beta_schedule": "scaled_linear", + "prediction_type": "epsilon", + } + + config.update(**kwargs) + return config + + @property + def default_valid_timestep(self): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + + scheduler_config = self.get_scheduler_config() + scheduler = self.scheduler_classes[0](**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + timestep = scheduler.timesteps[-1] + return timestep + + def test_timesteps(self): + for timesteps in [100, 500, 1000]: + # 0 is not guaranteed to be in the timestep schedule, but timesteps - 1 is + self.check_over_configs(time_step=timesteps - 1, num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001, 0.01, 0.1], [0.002, 0.02, 0.2, 2]): + self.check_over_configs(time_step=self.default_valid_timestep, beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "scaled_linear", "squaredcos_cap_v2"]: + self.check_over_configs(time_step=self.default_valid_timestep, beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(time_step=self.default_valid_timestep, prediction_type=prediction_type) + + def test_clip_sample(self): + for clip_sample in [True, False]: + self.check_over_configs(time_step=self.default_valid_timestep, clip_sample=clip_sample) + + def test_thresholding(self): + self.check_over_configs(time_step=self.default_valid_timestep, thresholding=False) + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs( + time_step=self.default_valid_timestep, + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + ) + + def test_time_indices(self): + # Get default timestep schedule. + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + + scheduler_config = self.get_scheduler_config() + scheduler = self.scheduler_classes[0](**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + timesteps = scheduler.timesteps + for t in timesteps: + self.check_over_forward(time_step=t) + + def test_inference_steps(self): + # Hardcoded for now + for t, num_inference_steps in zip([99, 39, 39, 19], [10, 25, 26, 50]): + self.check_over_forward(time_step=t, num_inference_steps=num_inference_steps) + + # Override test_add_noise_device because the hardcoded num_inference_steps of 100 doesn't work + # for LCMScheduler under default settings + def test_add_noise_device(self, num_inference_steps=10): + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + + sample = self.dummy_sample.to(torch_device) + scaled_sample = scheduler.scale_model_input(sample, 0.0) + self.assertEqual(sample.shape, scaled_sample.shape) + + noise = torch.randn_like(scaled_sample).to(torch_device) + t = scheduler.timesteps[5][None] + noised = scheduler.add_noise(scaled_sample, noise, t) + self.assertEqual(noised.shape, scaled_sample.shape) + + # Override test_from_save_pretrained because it hardcodes a timestep of 1 + def test_from_save_pretrained(self): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + timestep = self.default_valid_timestep + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + + scheduler.set_timesteps(num_inference_steps) + new_scheduler.set_timesteps(num_inference_steps) + + kwargs["generator"] = torch.manual_seed(0) + output = scheduler.step(residual, timestep, sample, **kwargs).prev_sample + + kwargs["generator"] = torch.manual_seed(0) + new_output = new_scheduler.step(residual, timestep, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + # Override test_step_shape because uses 0 and 1 as hardcoded timesteps + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + scheduler.set_timesteps(num_inference_steps) + + timestep_0 = scheduler.timesteps[-2] + timestep_1 = scheduler.timesteps[-1] + + output_0 = scheduler.step(residual, timestep_0, sample, **kwargs).prev_sample + output_1 = scheduler.step(residual, timestep_1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + # Override test_set_scheduler_outputs_equivalence since it uses 0 as a hardcoded timestep + def test_scheduler_outputs_equivalence(self): + def set_nan_tensor_to_zero(t): + t[t != t] = 0 + return t + + def recursive_check(tuple_object, dict_object): + if isinstance(tuple_object, (List, Tuple)): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object, dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif isinstance(tuple_object, Dict): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object.values(), dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif tuple_object is None: + return + else: + self.assertTrue( + torch.allclose( + set_nan_tensor_to_zero(tuple_object), set_nan_tensor_to_zero(dict_object), atol=1e-5 + ), + msg=( + "Tuple and dict output are not equal. Difference:" + f" {torch.max(torch.abs(tuple_object - dict_object))}. Tuple has `nan`:" + f" {torch.isnan(tuple_object).any()} and `inf`: {torch.isinf(tuple_object)}. Dict has" + f" `nan`: {torch.isnan(dict_object).any()} and `inf`: {torch.isinf(dict_object)}." + ), + ) + + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", 50) + + timestep = self.default_valid_timestep + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + scheduler.set_timesteps(num_inference_steps) + kwargs["generator"] = torch.manual_seed(0) + outputs_dict = scheduler.step(residual, timestep, sample, **kwargs) + + scheduler.set_timesteps(num_inference_steps) + kwargs["generator"] = torch.manual_seed(0) + outputs_tuple = scheduler.step(residual, timestep, sample, return_dict=False, **kwargs) + + recursive_check(outputs_tuple, outputs_dict) + + def full_loop(self, num_inference_steps=10, seed=0, **config): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + model = self.dummy_model() + sample = self.dummy_sample_deter + generator = torch.manual_seed(seed) + + scheduler.set_timesteps(num_inference_steps) + + for t in scheduler.timesteps: + residual = model(sample, t) + sample = scheduler.step(residual, t, sample, generator).prev_sample + + return sample + + def test_full_loop_onestep(self): + sample = self.full_loop(num_inference_steps=1) + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + # TODO: get expected sum and mean + assert abs(result_sum.item() - 18.7097) < 1e-3 + assert abs(result_mean.item() - 0.0244) < 1e-3 + + def test_full_loop_multistep(self): + sample = self.full_loop(num_inference_steps=10) + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + # TODO: get expected sum and mean + assert abs(result_sum.item() - 197.7616) < 1e-3 + assert abs(result_mean.item() - 0.2575) < 1e-3 + + def test_custom_timesteps(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 1, 0] + + scheduler.set_timesteps(timesteps=timesteps) + + scheduler_timesteps = scheduler.timesteps + + for i, timestep in enumerate(scheduler_timesteps): + if i == len(timesteps) - 1: + expected_prev_t = -1 + else: + expected_prev_t = timesteps[i + 1] + + prev_t = scheduler.previous_timestep(timestep) + prev_t = prev_t.item() + + self.assertEqual(prev_t, expected_prev_t) + + def test_custom_timesteps_increasing_order(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 51, 0] + + with self.assertRaises(ValueError, msg="`custom_timesteps` must be in descending order."): + scheduler.set_timesteps(timesteps=timesteps) + + def test_custom_timesteps_passing_both_num_inference_steps_and_timesteps(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 1, 0] + num_inference_steps = len(timesteps) + + with self.assertRaises(ValueError, msg="Can only pass one of `num_inference_steps` or `custom_timesteps`."): + scheduler.set_timesteps(num_inference_steps=num_inference_steps, timesteps=timesteps) + + def test_custom_timesteps_too_large(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [scheduler.config.num_train_timesteps] + + with self.assertRaises( + ValueError, + msg="`timesteps` must start before `self.config.train_timesteps`: {scheduler.config.num_train_timesteps}}", + ): + scheduler.set_timesteps(timesteps=timesteps) diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_lms.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_lms.py new file mode 100755 index 0000000..5c163ce --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_lms.py @@ -0,0 +1,170 @@ +import torch + +from diffusers import LMSDiscreteScheduler +from diffusers.utils.testing_utils import torch_device + +from .test_schedulers import SchedulerCommonTest + + +class LMSDiscreteSchedulerTest(SchedulerCommonTest): + scheduler_classes = (LMSDiscreteScheduler,) + num_inference_steps = 10 + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1100, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [10, 50, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.00001, 0.0001, 0.001], [0.0002, 0.002, 0.02]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "scaled_linear"]: + self.check_over_configs(beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_time_indices(self): + for t in [0, 500, 800]: + self.check_over_forward(time_step=t) + + def test_full_loop_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 1006.388) < 1e-2 + assert abs(result_mean.item() - 1.31) < 1e-3 + + def test_full_loop_with_v_prediction(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(prediction_type="v_prediction") + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 0.0017) < 1e-2 + assert abs(result_mean.item() - 2.2676e-06) < 1e-3 + + def test_full_loop_device(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma.cpu() + sample = sample.to(torch_device) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 1006.388) < 1e-2 + assert abs(result_mean.item() - 1.31) < 1e-3 + + def test_full_loop_device_karras_sigmas(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config, use_karras_sigmas=True) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + + model = self.dummy_model() + sample = self.dummy_sample_deter.to(torch_device) * scheduler.init_noise_sigma + sample = sample.to(torch_device) + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 3812.9927) < 2e-2 + assert abs(result_mean.item() - 4.9648) < 1e-3 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + + # add noise + t_start = self.num_inference_steps - 2 + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 27663.6895) < 1e-2 + assert abs(result_mean.item() - 36.0204) < 1e-3 diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_pndm.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_pndm.py new file mode 100755 index 0000000..c1519f7 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_pndm.py @@ -0,0 +1,242 @@ +import tempfile + +import torch + +from diffusers import PNDMScheduler + +from .test_schedulers import SchedulerCommonTest + + +class PNDMSchedulerTest(SchedulerCommonTest): + scheduler_classes = (PNDMScheduler,) + forward_default_kwargs = (("num_inference_steps", 50),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + } + + config.update(**kwargs) + return config + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + scheduler.ets = dummy_past_residuals[:] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + new_scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + new_scheduler.ets = dummy_past_residuals[:] + + output = scheduler.step_prk(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step_prk(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + output = scheduler.step_plms(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step_plms(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_from_save_pretrained(self): + pass + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residuals (must be after setting timesteps) + scheduler.ets = dummy_past_residuals[:] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + # copy over dummy past residuals + new_scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residual (must be after setting timesteps) + new_scheduler.ets = dummy_past_residuals[:] + + output = scheduler.step_prk(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step_prk(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + output = scheduler.step_plms(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step_plms(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def full_loop(self, **config): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.prk_timesteps): + residual = model(sample, t) + sample = scheduler.step_prk(residual, t, sample).prev_sample + + for i, t in enumerate(scheduler.plms_timesteps): + residual = model(sample, t) + sample = scheduler.step_plms(residual, t, sample).prev_sample + + return sample + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # copy over dummy past residuals (must be done after set_timesteps) + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.1, residual + 0.05] + scheduler.ets = dummy_past_residuals[:] + + output_0 = scheduler.step_prk(residual, 0, sample, **kwargs).prev_sample + output_1 = scheduler.step_prk(residual, 1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + output_0 = scheduler.step_plms(residual, 0, sample, **kwargs).prev_sample + output_1 = scheduler.step_plms(residual, 1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_timesteps(self): + for timesteps in [100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_steps_offset(self): + for steps_offset in [0, 1]: + self.check_over_configs(steps_offset=steps_offset) + + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(steps_offset=1) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(10) + assert torch.equal( + scheduler.timesteps, + torch.LongTensor( + [901, 851, 851, 801, 801, 751, 751, 701, 701, 651, 651, 601, 601, 501, 401, 301, 201, 101, 1] + ), + ) + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001], [0.002, 0.02]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "squaredcos_cap_v2"]: + self.check_over_configs(beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_time_indices(self): + for t in [1, 5, 10]: + self.check_over_forward(time_step=t) + + def test_inference_steps(self): + for t, num_inference_steps in zip([1, 5, 10], [10, 50, 100]): + self.check_over_forward(num_inference_steps=num_inference_steps) + + def test_pow_of_3_inference_steps(self): + # earlier version of set_timesteps() caused an error indexing alpha's with inference steps as power of 3 + num_inference_steps = 27 + + for scheduler_class in self.scheduler_classes: + sample = self.dummy_sample + residual = 0.1 * sample + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + + # before power of 3 fix, would error on first step, so we only need to do two + for i, t in enumerate(scheduler.prk_timesteps[:2]): + sample = scheduler.step_prk(residual, t, sample).prev_sample + + def test_inference_plms_no_past_residuals(self): + with self.assertRaises(ValueError): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.step_plms(self.dummy_sample, 1, self.dummy_sample).prev_sample + + def test_full_loop_no_noise(self): + sample = self.full_loop() + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 198.1318) < 1e-2 + assert abs(result_mean.item() - 0.2580) < 1e-3 + + def test_full_loop_with_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction") + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 67.3986) < 1e-2 + assert abs(result_mean.item() - 0.0878) < 1e-3 + + def test_full_loop_with_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=True, beta_start=0.01) + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 230.0399) < 1e-2 + assert abs(result_mean.item() - 0.2995) < 1e-3 + + def test_full_loop_with_no_set_alpha_to_one(self): + # We specify different beta, so that the first alpha is 0.99 + sample = self.full_loop(set_alpha_to_one=False, beta_start=0.01) + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 186.9482) < 1e-2 + assert abs(result_mean.item() - 0.2434) < 1e-3 diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_sasolver.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_sasolver.py new file mode 100755 index 0000000..5741946 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_sasolver.py @@ -0,0 +1,202 @@ +import torch + +from diffusers import SASolverScheduler +from diffusers.utils.testing_utils import require_torchsde, torch_device + +from .test_schedulers import SchedulerCommonTest + + +@require_torchsde +class SASolverSchedulerTest(SchedulerCommonTest): + scheduler_classes = (SASolverScheduler,) + forward_default_kwargs = (("num_inference_steps", 10),) + num_inference_steps = 10 + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1100, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + } + + config.update(**kwargs) + return config + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # copy over dummy past residuals (must be done after set_timesteps) + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + scheduler.model_outputs = dummy_past_residuals[ + : max( + scheduler.config.predictor_order, + scheduler.config.corrector_order - 1, + ) + ] + + time_step_0 = scheduler.timesteps[5] + time_step_1 = scheduler.timesteps[6] + + output_0 = scheduler.step(residual, time_step_0, sample, **kwargs).prev_sample + output_1 = scheduler.step(residual, time_step_1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_timesteps(self): + for timesteps in [10, 50, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.00001, 0.0001, 0.001], [0.0002, 0.002, 0.02]): + self.check_over_configs(beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "scaled_linear"]: + self.check_over_configs(beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_full_loop_no_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + generator = torch.manual_seed(0) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t, generator=generator) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["cpu"]: + assert abs(result_sum.item() - 337.394287109375) < 1e-2 + assert abs(result_mean.item() - 0.43931546807289124) < 1e-3 + elif torch_device in ["cuda"]: + assert abs(result_sum.item() - 329.1999816894531) < 1e-2 + assert abs(result_mean.item() - 0.4286458194255829) < 1e-3 + else: + print("None") + + def test_full_loop_with_v_prediction(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(prediction_type="v_prediction") + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps) + + model = self.dummy_model() + sample = self.dummy_sample_deter * scheduler.init_noise_sigma + sample = sample.to(torch_device) + generator = torch.manual_seed(0) + + for i, t in enumerate(scheduler.timesteps): + sample = scheduler.scale_model_input(sample, t, generator=generator) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["cpu"]: + assert abs(result_sum.item() - 193.1467742919922) < 1e-2 + assert abs(result_mean.item() - 0.2514931857585907) < 1e-3 + elif torch_device in ["cuda"]: + assert abs(result_sum.item() - 193.4154052734375) < 1e-2 + assert abs(result_mean.item() - 0.2518429756164551) < 1e-3 + else: + print("None") + + def test_full_loop_device(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + + model = self.dummy_model() + sample = self.dummy_sample_deter.to(torch_device) * scheduler.init_noise_sigma + generator = torch.manual_seed(0) + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["cpu"]: + assert abs(result_sum.item() - 337.394287109375) < 1e-2 + assert abs(result_mean.item() - 0.43931546807289124) < 1e-3 + elif torch_device in ["cuda"]: + assert abs(result_sum.item() - 337.394287109375) < 1e-2 + assert abs(result_mean.item() - 0.4393154978752136) < 1e-3 + else: + print("None") + + def test_full_loop_device_karras_sigmas(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config, use_karras_sigmas=True) + + scheduler.set_timesteps(self.num_inference_steps, device=torch_device) + + model = self.dummy_model() + sample = self.dummy_sample_deter.to(torch_device) * scheduler.init_noise_sigma + sample = sample.to(torch_device) + generator = torch.manual_seed(0) + + for t in scheduler.timesteps: + sample = scheduler.scale_model_input(sample, t) + + model_output = model(sample, t) + + output = scheduler.step(model_output, t, sample, generator=generator) + sample = output.prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + if torch_device in ["cpu"]: + assert abs(result_sum.item() - 837.2554931640625) < 1e-2 + assert abs(result_mean.item() - 1.0901764631271362) < 1e-2 + elif torch_device in ["cuda"]: + assert abs(result_sum.item() - 837.25537109375) < 1e-2 + assert abs(result_mean.item() - 1.0901763439178467) < 1e-2 + else: + print("None") diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_score_sde_ve.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_score_sde_ve.py new file mode 100755 index 0000000..08c30f9 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_score_sde_ve.py @@ -0,0 +1,189 @@ +import tempfile +import unittest + +import numpy as np +import torch + +from diffusers import ScoreSdeVeScheduler + + +class ScoreSdeVeSchedulerTest(unittest.TestCase): + # TODO adapt with class SchedulerCommonTest (scheduler needs Numpy Integration) + scheduler_classes = (ScoreSdeVeScheduler,) + forward_default_kwargs = () + + @property + def dummy_sample(self): + batch_size = 4 + num_channels = 3 + height = 8 + width = 8 + + sample = torch.rand((batch_size, num_channels, height, width)) + + return sample + + @property + def dummy_sample_deter(self): + batch_size = 4 + num_channels = 3 + height = 8 + width = 8 + + num_elems = batch_size * num_channels * height * width + sample = torch.arange(num_elems) + sample = sample.reshape(num_channels, height, width, batch_size) + sample = sample / num_elems + sample = sample.permute(3, 0, 1, 2) + + return sample + + def dummy_model(self): + def model(sample, t, *args): + return sample * t / (t + 1) + + return model + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 2000, + "snr": 0.15, + "sigma_min": 0.01, + "sigma_max": 1348, + "sampling_eps": 1e-5, + } + + config.update(**kwargs) + return config + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + + for scheduler_class in self.scheduler_classes: + sample = self.dummy_sample + residual = 0.1 * sample + + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + + output = scheduler.step_pred( + residual, time_step, sample, generator=torch.manual_seed(0), **kwargs + ).prev_sample + new_output = new_scheduler.step_pred( + residual, time_step, sample, generator=torch.manual_seed(0), **kwargs + ).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + output = scheduler.step_correct(residual, sample, generator=torch.manual_seed(0), **kwargs).prev_sample + new_output = new_scheduler.step_correct( + residual, sample, generator=torch.manual_seed(0), **kwargs + ).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler correction are not identical" + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + kwargs.update(forward_kwargs) + + for scheduler_class in self.scheduler_classes: + sample = self.dummy_sample + residual = 0.1 * sample + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + + output = scheduler.step_pred( + residual, time_step, sample, generator=torch.manual_seed(0), **kwargs + ).prev_sample + new_output = new_scheduler.step_pred( + residual, time_step, sample, generator=torch.manual_seed(0), **kwargs + ).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + output = scheduler.step_correct(residual, sample, generator=torch.manual_seed(0), **kwargs).prev_sample + new_output = new_scheduler.step_correct( + residual, sample, generator=torch.manual_seed(0), **kwargs + ).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler correction are not identical" + + def test_timesteps(self): + for timesteps in [10, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_sigmas(self): + for sigma_min, sigma_max in zip([0.0001, 0.001, 0.01], [1, 100, 1000]): + self.check_over_configs(sigma_min=sigma_min, sigma_max=sigma_max) + + def test_time_indices(self): + for t in [0.1, 0.5, 0.75]: + self.check_over_forward(time_step=t) + + def test_full_loop_no_noise(self): + kwargs = dict(self.forward_default_kwargs) + + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 3 + + model = self.dummy_model() + sample = self.dummy_sample_deter + + scheduler.set_sigmas(num_inference_steps) + scheduler.set_timesteps(num_inference_steps) + generator = torch.manual_seed(0) + + for i, t in enumerate(scheduler.timesteps): + sigma_t = scheduler.sigmas[i] + + for _ in range(scheduler.config.correct_steps): + with torch.no_grad(): + model_output = model(sample, sigma_t) + sample = scheduler.step_correct(model_output, sample, generator=generator, **kwargs).prev_sample + + with torch.no_grad(): + model_output = model(sample, sigma_t) + + output = scheduler.step_pred(model_output, t, sample, generator=generator, **kwargs) + sample, _ = output.prev_sample, output.prev_sample_mean + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert np.isclose(result_sum.item(), 14372758528.0) + assert np.isclose(result_mean.item(), 18714530.0) + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + output_0 = scheduler.step_pred(residual, 0, sample, generator=torch.manual_seed(0), **kwargs).prev_sample + output_1 = scheduler.step_pred(residual, 1, sample, generator=torch.manual_seed(0), **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_tcd.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_tcd.py new file mode 100755 index 0000000..e95c536 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_tcd.py @@ -0,0 +1,180 @@ +import torch + +from diffusers import TCDScheduler + +from .test_schedulers import SchedulerCommonTest + + +class TCDSchedulerTest(SchedulerCommonTest): + scheduler_classes = (TCDScheduler,) + forward_default_kwargs = (("num_inference_steps", 10),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.00085, + "beta_end": 0.0120, + "beta_schedule": "scaled_linear", + "prediction_type": "epsilon", + } + + config.update(**kwargs) + return config + + @property + def default_num_inference_steps(self): + return 10 + + @property + def default_valid_timestep(self): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + + scheduler_config = self.get_scheduler_config() + scheduler = self.scheduler_classes[0](**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + timestep = scheduler.timesteps[-1] + return timestep + + def test_timesteps(self): + for timesteps in [100, 500, 1000]: + # 0 is not guaranteed to be in the timestep schedule, but timesteps - 1 is + self.check_over_configs(time_step=timesteps - 1, num_train_timesteps=timesteps) + + def test_betas(self): + for beta_start, beta_end in zip([0.0001, 0.001, 0.01, 0.1], [0.002, 0.02, 0.2, 2]): + self.check_over_configs(time_step=self.default_valid_timestep, beta_start=beta_start, beta_end=beta_end) + + def test_schedules(self): + for schedule in ["linear", "scaled_linear", "squaredcos_cap_v2"]: + self.check_over_configs(time_step=self.default_valid_timestep, beta_schedule=schedule) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(time_step=self.default_valid_timestep, prediction_type=prediction_type) + + def test_clip_sample(self): + for clip_sample in [True, False]: + self.check_over_configs(time_step=self.default_valid_timestep, clip_sample=clip_sample) + + def test_thresholding(self): + self.check_over_configs(time_step=self.default_valid_timestep, thresholding=False) + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs( + time_step=self.default_valid_timestep, + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + ) + + def test_time_indices(self): + # Get default timestep schedule. + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + + scheduler_config = self.get_scheduler_config() + scheduler = self.scheduler_classes[0](**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + timesteps = scheduler.timesteps + for t in timesteps: + self.check_over_forward(time_step=t) + + def test_inference_steps(self): + # Hardcoded for now + for t, num_inference_steps in zip([99, 39, 39, 19], [10, 25, 26, 50]): + self.check_over_forward(time_step=t, num_inference_steps=num_inference_steps) + + def full_loop(self, num_inference_steps=10, seed=0, **config): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + eta = 0.0 # refer to gamma in the paper + + model = self.dummy_model() + sample = self.dummy_sample_deter + generator = torch.manual_seed(seed) + scheduler.set_timesteps(num_inference_steps) + + for t in scheduler.timesteps: + residual = model(sample, t) + sample = scheduler.step(residual, t, sample, eta, generator).prev_sample + + return sample + + def test_full_loop_onestep_deter(self): + sample = self.full_loop(num_inference_steps=1) + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 29.8715) < 1e-3 # 0.0778918 + assert abs(result_mean.item() - 0.0389) < 1e-3 + + def test_full_loop_multistep_deter(self): + sample = self.full_loop(num_inference_steps=10) + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 181.2040) < 1e-3 + assert abs(result_mean.item() - 0.2359) < 1e-3 + + def test_custom_timesteps(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 1, 0] + + scheduler.set_timesteps(timesteps=timesteps) + + scheduler_timesteps = scheduler.timesteps + + for i, timestep in enumerate(scheduler_timesteps): + if i == len(timesteps) - 1: + expected_prev_t = -1 + else: + expected_prev_t = timesteps[i + 1] + + prev_t = scheduler.previous_timestep(timestep) + prev_t = prev_t.item() + + self.assertEqual(prev_t, expected_prev_t) + + def test_custom_timesteps_increasing_order(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 51, 0] + + with self.assertRaises(ValueError, msg="`custom_timesteps` must be in descending order."): + scheduler.set_timesteps(timesteps=timesteps) + + def test_custom_timesteps_passing_both_num_inference_steps_and_timesteps(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [100, 87, 50, 1, 0] + num_inference_steps = len(timesteps) + + with self.assertRaises(ValueError, msg="Can only pass one of `num_inference_steps` or `custom_timesteps`."): + scheduler.set_timesteps(num_inference_steps=num_inference_steps, timesteps=timesteps) + + def test_custom_timesteps_too_large(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = [scheduler.config.num_train_timesteps] + + with self.assertRaises( + ValueError, + msg="`timesteps` must start before `self.config.train_timesteps`: {scheduler.config.num_train_timesteps}}", + ): + scheduler.set_timesteps(timesteps=timesteps) diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_unclip.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_unclip.py new file mode 100755 index 0000000..b0ce131 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_unclip.py @@ -0,0 +1,137 @@ +import torch + +from diffusers import UnCLIPScheduler + +from .test_schedulers import SchedulerCommonTest + + +# UnCLIPScheduler is a modified DDPMScheduler with a subset of the configuration. +class UnCLIPSchedulerTest(SchedulerCommonTest): + scheduler_classes = (UnCLIPScheduler,) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "variance_type": "fixed_small_log", + "clip_sample": True, + "clip_sample_range": 1.0, + "prediction_type": "epsilon", + } + + config.update(**kwargs) + return config + + def test_timesteps(self): + for timesteps in [1, 5, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_variance_type(self): + for variance in ["fixed_small_log", "learned_range"]: + self.check_over_configs(variance_type=variance) + + def test_clip_sample(self): + for clip_sample in [True, False]: + self.check_over_configs(clip_sample=clip_sample) + + def test_clip_sample_range(self): + for clip_sample_range in [1, 5, 10, 20]: + self.check_over_configs(clip_sample_range=clip_sample_range) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "sample"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_time_indices(self): + for time_step in [0, 500, 999]: + for prev_timestep in [None, 5, 100, 250, 500, 750]: + if prev_timestep is not None and prev_timestep >= time_step: + continue + + self.check_over_forward(time_step=time_step, prev_timestep=prev_timestep) + + def test_variance_fixed_small_log(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(variance_type="fixed_small_log") + scheduler = scheduler_class(**scheduler_config) + + assert torch.sum(torch.abs(scheduler._get_variance(0) - 1.0000e-10)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(487) - 0.0549625)) < 1e-5 + assert torch.sum(torch.abs(scheduler._get_variance(999) - 0.9994987)) < 1e-5 + + def test_variance_learned_range(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(variance_type="learned_range") + scheduler = scheduler_class(**scheduler_config) + + predicted_variance = 0.5 + + assert scheduler._get_variance(1, predicted_variance=predicted_variance) - -10.1712790 < 1e-5 + assert scheduler._get_variance(487, predicted_variance=predicted_variance) - -5.7998052 < 1e-5 + assert scheduler._get_variance(999, predicted_variance=predicted_variance) - -0.0010011 < 1e-5 + + def test_full_loop(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + timesteps = scheduler.timesteps + + model = self.dummy_model() + sample = self.dummy_sample_deter + generator = torch.manual_seed(0) + + for i, t in enumerate(timesteps): + # 1. predict noise residual + residual = model(sample, t) + + # 2. predict previous mean of sample x_t-1 + pred_prev_sample = scheduler.step(residual, t, sample, generator=generator).prev_sample + + sample = pred_prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 252.2682495) < 1e-2 + assert abs(result_mean.item() - 0.3284743) < 1e-3 + + def test_full_loop_skip_timesteps(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + scheduler.set_timesteps(25) + + timesteps = scheduler.timesteps + + model = self.dummy_model() + sample = self.dummy_sample_deter + generator = torch.manual_seed(0) + + for i, t in enumerate(timesteps): + # 1. predict noise residual + residual = model(sample, t) + + if i + 1 == timesteps.shape[0]: + prev_timestep = None + else: + prev_timestep = timesteps[i + 1] + + # 2. predict previous mean of sample x_t-1 + pred_prev_sample = scheduler.step( + residual, t, sample, prev_timestep=prev_timestep, generator=generator + ).prev_sample + + sample = pred_prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 258.2044983) < 1e-2 + assert abs(result_mean.item() - 0.3362038) < 1e-3 + + def test_trained_betas(self): + pass + + def test_add_noise_device(self): + pass diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_unipc.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_unipc.py new file mode 100755 index 0000000..be41cea --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_unipc.py @@ -0,0 +1,381 @@ +import tempfile + +import torch + +from diffusers import ( + DEISMultistepScheduler, + DPMSolverMultistepScheduler, + DPMSolverSinglestepScheduler, + UniPCMultistepScheduler, +) + +from .test_schedulers import SchedulerCommonTest + + +class UniPCMultistepSchedulerTest(SchedulerCommonTest): + scheduler_classes = (UniPCMultistepScheduler,) + forward_default_kwargs = (("num_inference_steps", 25),) + + def get_scheduler_config(self, **kwargs): + config = { + "num_train_timesteps": 1000, + "beta_start": 0.0001, + "beta_end": 0.02, + "beta_schedule": "linear", + "solver_order": 2, + "solver_type": "bh2", + } + + config.update(**kwargs) + return config + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + new_scheduler.set_timesteps(num_inference_steps) + # copy over dummy past residuals + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + output, new_output = sample, sample + for t in range(time_step, time_step + scheduler.config.solver_order + 1): + t = scheduler.timesteps[t] + output = scheduler.step(residual, t, output, **kwargs).prev_sample + new_output = new_scheduler.step(residual, t, new_output, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", None) + sample = self.dummy_sample + residual = 0.1 * sample + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residuals (must be after setting timesteps) + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + # copy over dummy past residuals + new_scheduler.set_timesteps(num_inference_steps) + + # copy over dummy past residual (must be after setting timesteps) + new_scheduler.model_outputs = dummy_past_residuals[: new_scheduler.config.solver_order] + + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def full_loop(self, scheduler=None, **config): + if scheduler is None: + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + return sample + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # copy over dummy past residuals (must be done after set_timesteps) + dummy_past_residuals = [residual + 0.2, residual + 0.15, residual + 0.10] + scheduler.model_outputs = dummy_past_residuals[: scheduler.config.solver_order] + + time_step_0 = scheduler.timesteps[5] + time_step_1 = scheduler.timesteps[6] + + output_0 = scheduler.step(residual, time_step_0, sample, **kwargs).prev_sample + output_1 = scheduler.step(residual, time_step_1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_switch(self): + # make sure that iterating over schedulers with same config names gives same results + # for defaults + scheduler = UniPCMultistepScheduler(**self.get_scheduler_config()) + sample = self.full_loop(scheduler=scheduler) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2464) < 1e-3 + + scheduler = DPMSolverSinglestepScheduler.from_config(scheduler.config) + scheduler = DEISMultistepScheduler.from_config(scheduler.config) + scheduler = DPMSolverMultistepScheduler.from_config(scheduler.config) + scheduler = UniPCMultistepScheduler.from_config(scheduler.config) + + sample = self.full_loop(scheduler=scheduler) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2464) < 1e-3 + + def test_timesteps(self): + for timesteps in [25, 50, 100, 999, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_thresholding(self): + self.check_over_configs(thresholding=False) + for order in [1, 2, 3]: + for solver_type in ["bh1", "bh2"]: + for threshold in [0.5, 1.0, 2.0]: + for prediction_type in ["epsilon", "sample"]: + self.check_over_configs( + thresholding=True, + prediction_type=prediction_type, + sample_max_value=threshold, + solver_order=order, + solver_type=solver_type, + ) + + def test_prediction_type(self): + for prediction_type in ["epsilon", "v_prediction"]: + self.check_over_configs(prediction_type=prediction_type) + + def test_solver_order_and_type(self): + for solver_type in ["bh1", "bh2"]: + for order in [1, 2, 3]: + for prediction_type in ["epsilon", "sample"]: + self.check_over_configs( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + ) + sample = self.full_loop( + solver_order=order, + solver_type=solver_type, + prediction_type=prediction_type, + ) + assert not torch.isnan(sample).any(), "Samples have nan numbers" + + def test_lower_order_final(self): + self.check_over_configs(lower_order_final=True) + self.check_over_configs(lower_order_final=False) + + def test_inference_steps(self): + for num_inference_steps in [1, 2, 3, 5, 10, 50, 100, 999, 1000]: + self.check_over_forward(num_inference_steps=num_inference_steps, time_step=0) + + def test_full_loop_no_noise(self): + sample = self.full_loop() + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2464) < 1e-3 + + def test_full_loop_with_karras(self): + sample = self.full_loop(use_karras_sigmas=True) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2925) < 1e-3 + + def test_full_loop_with_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction") + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.1014) < 1e-3 + + def test_full_loop_with_karras_and_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction", use_karras_sigmas=True) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.1966) < 1e-3 + + def test_fp16_support(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config(thresholding=True, dynamic_thresholding_ratio=0) + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + model = self.dummy_model() + sample = self.dummy_sample_deter.half() + scheduler.set_timesteps(num_inference_steps) + + for i, t in enumerate(scheduler.timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + assert sample.dtype == torch.float16 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + t_start = 8 + + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + # add noise + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 315.5757) < 1e-2, f" expected result sum 315.5757, but get {result_sum}" + assert abs(result_mean.item() - 0.4109) < 1e-3, f" expected result mean 0.4109, but get {result_mean}" + + +class UniPCMultistepScheduler1DTest(UniPCMultistepSchedulerTest): + @property + def dummy_sample(self): + batch_size = 4 + num_channels = 3 + width = 8 + + sample = torch.rand((batch_size, num_channels, width)) + + return sample + + @property + def dummy_noise_deter(self): + batch_size = 4 + num_channels = 3 + width = 8 + + num_elems = batch_size * num_channels * width + sample = torch.arange(num_elems).flip(-1) + sample = sample.reshape(num_channels, width, batch_size) + sample = sample / num_elems + sample = sample.permute(2, 0, 1) + + return sample + + @property + def dummy_sample_deter(self): + batch_size = 4 + num_channels = 3 + width = 8 + + num_elems = batch_size * num_channels * width + sample = torch.arange(num_elems) + sample = sample.reshape(num_channels, width, batch_size) + sample = sample / num_elems + sample = sample.permute(2, 0, 1) + + return sample + + def test_switch(self): + # make sure that iterating over schedulers with same config names gives same results + # for defaults + scheduler = UniPCMultistepScheduler(**self.get_scheduler_config()) + sample = self.full_loop(scheduler=scheduler) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2441) < 1e-3 + + scheduler = DPMSolverSinglestepScheduler.from_config(scheduler.config) + scheduler = DEISMultistepScheduler.from_config(scheduler.config) + scheduler = DPMSolverMultistepScheduler.from_config(scheduler.config) + scheduler = UniPCMultistepScheduler.from_config(scheduler.config) + + sample = self.full_loop(scheduler=scheduler) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2441) < 1e-3 + + def test_full_loop_no_noise(self): + sample = self.full_loop() + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2441) < 1e-3 + + def test_full_loop_with_karras(self): + sample = self.full_loop(use_karras_sigmas=True) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.2898) < 1e-3 + + def test_full_loop_with_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction") + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.1014) < 1e-3 + + def test_full_loop_with_karras_and_v_prediction(self): + sample = self.full_loop(prediction_type="v_prediction", use_karras_sigmas=True) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_mean.item() - 0.1944) < 1e-3 + + def test_full_loop_with_noise(self): + scheduler_class = self.scheduler_classes[0] + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + num_inference_steps = 10 + t_start = 8 + + model = self.dummy_model() + sample = self.dummy_sample_deter + scheduler.set_timesteps(num_inference_steps) + + # add noise + noise = self.dummy_noise_deter + timesteps = scheduler.timesteps[t_start * scheduler.order :] + sample = scheduler.add_noise(sample, noise, timesteps[:1]) + + for i, t in enumerate(timesteps): + residual = model(sample, t) + sample = scheduler.step(residual, t, sample).prev_sample + + result_sum = torch.sum(torch.abs(sample)) + result_mean = torch.mean(torch.abs(sample)) + + assert abs(result_sum.item() - 39.0870) < 1e-2, f" expected result sum 39.0870, but get {result_sum}" + assert abs(result_mean.item() - 0.4072) < 1e-3, f" expected result mean 0.4072, but get {result_mean}" diff --git a/diffusers-0.27.0/tests/schedulers/test_scheduler_vq_diffusion.py b/diffusers-0.27.0/tests/schedulers/test_scheduler_vq_diffusion.py new file mode 100755 index 0000000..74437ad --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_scheduler_vq_diffusion.py @@ -0,0 +1,56 @@ +import torch +import torch.nn.functional as F + +from diffusers import VQDiffusionScheduler + +from .test_schedulers import SchedulerCommonTest + + +class VQDiffusionSchedulerTest(SchedulerCommonTest): + scheduler_classes = (VQDiffusionScheduler,) + + def get_scheduler_config(self, **kwargs): + config = { + "num_vec_classes": 4097, + "num_train_timesteps": 100, + } + + config.update(**kwargs) + return config + + def dummy_sample(self, num_vec_classes): + batch_size = 4 + height = 8 + width = 8 + + sample = torch.randint(0, num_vec_classes, (batch_size, height * width)) + + return sample + + @property + def dummy_sample_deter(self): + assert False + + def dummy_model(self, num_vec_classes): + def model(sample, t, *args): + batch_size, num_latent_pixels = sample.shape + logits = torch.rand((batch_size, num_vec_classes - 1, num_latent_pixels)) + return_value = F.log_softmax(logits.double(), dim=1).float() + return return_value + + return model + + def test_timesteps(self): + for timesteps in [2, 5, 100, 1000]: + self.check_over_configs(num_train_timesteps=timesteps) + + def test_num_vec_classes(self): + for num_vec_classes in [5, 100, 1000, 4000]: + self.check_over_configs(num_vec_classes=num_vec_classes) + + def test_time_indices(self): + for t in [0, 50, 99]: + self.check_over_forward(time_step=t) + + def test_add_noise_device(self): + pass diff --git a/diffusers-0.27.0/tests/schedulers/test_schedulers.py b/diffusers-0.27.0/tests/schedulers/test_schedulers.py new file mode 100755 index 0000000..9982db7 --- /dev/null +++ b/diffusers-0.27.0/tests/schedulers/test_schedulers.py @@ -0,0 +1,868 @@ +# coding=utf-8 +# Copyright 2024 HuggingFace Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import inspect +import json +import os +import tempfile +import unittest +import uuid +from typing import Dict, List, Tuple + +import numpy as np +import torch +from huggingface_hub import delete_repo + +import diffusers +from diffusers import ( + CMStochasticIterativeScheduler, + DDIMScheduler, + DEISMultistepScheduler, + DiffusionPipeline, + EDMEulerScheduler, + EulerAncestralDiscreteScheduler, + EulerDiscreteScheduler, + IPNDMScheduler, + LMSDiscreteScheduler, + UniPCMultistepScheduler, + VQDiffusionScheduler, +) +from diffusers.configuration_utils import ConfigMixin, register_to_config +from diffusers.schedulers.scheduling_utils import SchedulerMixin +from diffusers.utils import logging +from diffusers.utils.testing_utils import CaptureLogger, torch_device + +from ..others.test_utils import TOKEN, USER, is_staging_test + + +torch.backends.cuda.matmul.allow_tf32 = False + + +logger = logging.get_logger(__name__) # pylint: disable=invalid-name + + +class SchedulerObject(SchedulerMixin, ConfigMixin): + config_name = "config.json" + + @register_to_config + def __init__( + self, + a=2, + b=5, + c=(2, 5), + d="for diffusion", + e=[1, 3], + ): + pass + + +class SchedulerObject2(SchedulerMixin, ConfigMixin): + config_name = "config.json" + + @register_to_config + def __init__( + self, + a=2, + b=5, + c=(2, 5), + d="for diffusion", + f=[1, 3], + ): + pass + + +class SchedulerObject3(SchedulerMixin, ConfigMixin): + config_name = "config.json" + + @register_to_config + def __init__( + self, + a=2, + b=5, + c=(2, 5), + d="for diffusion", + e=[1, 3], + f=[1, 3], + ): + pass + + +class SchedulerBaseTests(unittest.TestCase): + def test_save_load_from_different_config(self): + obj = SchedulerObject() + + # mock add obj class to `diffusers` + setattr(diffusers, "SchedulerObject", SchedulerObject) + logger = logging.get_logger("diffusers.configuration_utils") + + with tempfile.TemporaryDirectory() as tmpdirname: + obj.save_config(tmpdirname) + with CaptureLogger(logger) as cap_logger_1: + config = SchedulerObject2.load_config(tmpdirname) + new_obj_1 = SchedulerObject2.from_config(config) + + # now save a config parameter that is not expected + with open(os.path.join(tmpdirname, SchedulerObject.config_name), "r") as f: + data = json.load(f) + data["unexpected"] = True + + with open(os.path.join(tmpdirname, SchedulerObject.config_name), "w") as f: + json.dump(data, f) + + with CaptureLogger(logger) as cap_logger_2: + config = SchedulerObject.load_config(tmpdirname) + new_obj_2 = SchedulerObject.from_config(config) + + with CaptureLogger(logger) as cap_logger_3: + config = SchedulerObject2.load_config(tmpdirname) + new_obj_3 = SchedulerObject2.from_config(config) + + assert new_obj_1.__class__ == SchedulerObject2 + assert new_obj_2.__class__ == SchedulerObject + assert new_obj_3.__class__ == SchedulerObject2 + + assert cap_logger_1.out == "" + assert ( + cap_logger_2.out + == "The config attributes {'unexpected': True} were passed to SchedulerObject, but are not expected and" + " will" + " be ignored. Please verify your config.json configuration file.\n" + ) + assert cap_logger_2.out.replace("SchedulerObject", "SchedulerObject2") == cap_logger_3.out + + def test_save_load_compatible_schedulers(self): + SchedulerObject2._compatibles = ["SchedulerObject"] + SchedulerObject._compatibles = ["SchedulerObject2"] + + obj = SchedulerObject() + + # mock add obj class to `diffusers` + setattr(diffusers, "SchedulerObject", SchedulerObject) + setattr(diffusers, "SchedulerObject2", SchedulerObject2) + logger = logging.get_logger("diffusers.configuration_utils") + + with tempfile.TemporaryDirectory() as tmpdirname: + obj.save_config(tmpdirname) + + # now save a config parameter that is expected by another class, but not origin class + with open(os.path.join(tmpdirname, SchedulerObject.config_name), "r") as f: + data = json.load(f) + data["f"] = [0, 0] + data["unexpected"] = True + + with open(os.path.join(tmpdirname, SchedulerObject.config_name), "w") as f: + json.dump(data, f) + + with CaptureLogger(logger) as cap_logger: + config = SchedulerObject.load_config(tmpdirname) + new_obj = SchedulerObject.from_config(config) + + assert new_obj.__class__ == SchedulerObject + + assert ( + cap_logger.out + == "The config attributes {'unexpected': True} were passed to SchedulerObject, but are not expected and" + " will" + " be ignored. Please verify your config.json configuration file.\n" + ) + + def test_save_load_from_different_config_comp_schedulers(self): + SchedulerObject3._compatibles = ["SchedulerObject", "SchedulerObject2"] + SchedulerObject2._compatibles = ["SchedulerObject", "SchedulerObject3"] + SchedulerObject._compatibles = ["SchedulerObject2", "SchedulerObject3"] + + obj = SchedulerObject() + + # mock add obj class to `diffusers` + setattr(diffusers, "SchedulerObject", SchedulerObject) + setattr(diffusers, "SchedulerObject2", SchedulerObject2) + setattr(diffusers, "SchedulerObject3", SchedulerObject3) + logger = logging.get_logger("diffusers.configuration_utils") + logger.setLevel(diffusers.logging.INFO) + + with tempfile.TemporaryDirectory() as tmpdirname: + obj.save_config(tmpdirname) + + with CaptureLogger(logger) as cap_logger_1: + config = SchedulerObject.load_config(tmpdirname) + new_obj_1 = SchedulerObject.from_config(config) + + with CaptureLogger(logger) as cap_logger_2: + config = SchedulerObject2.load_config(tmpdirname) + new_obj_2 = SchedulerObject2.from_config(config) + + with CaptureLogger(logger) as cap_logger_3: + config = SchedulerObject3.load_config(tmpdirname) + new_obj_3 = SchedulerObject3.from_config(config) + + assert new_obj_1.__class__ == SchedulerObject + assert new_obj_2.__class__ == SchedulerObject2 + assert new_obj_3.__class__ == SchedulerObject3 + + assert cap_logger_1.out == "" + assert cap_logger_2.out == "{'f'} was not found in config. Values will be initialized to default values.\n" + assert cap_logger_3.out == "{'f'} was not found in config. Values will be initialized to default values.\n" + + def test_default_arguments_not_in_config(self): + pipe = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-pipe", torch_dtype=torch.float16 + ) + assert pipe.scheduler.__class__ == DDIMScheduler + + # Default for DDIMScheduler + assert pipe.scheduler.config.timestep_spacing == "leading" + + # Switch to a different one, verify we use the default for that class + pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config) + assert pipe.scheduler.config.timestep_spacing == "linspace" + + # Override with kwargs + pipe.scheduler = EulerDiscreteScheduler.from_config(pipe.scheduler.config, timestep_spacing="trailing") + assert pipe.scheduler.config.timestep_spacing == "trailing" + + # Verify overridden kwargs stick + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + assert pipe.scheduler.config.timestep_spacing == "trailing" + + # And stick + pipe.scheduler = LMSDiscreteScheduler.from_config(pipe.scheduler.config) + assert pipe.scheduler.config.timestep_spacing == "trailing" + + def test_default_solver_type_after_switch(self): + pipe = DiffusionPipeline.from_pretrained( + "hf-internal-testing/tiny-stable-diffusion-pipe", torch_dtype=torch.float16 + ) + assert pipe.scheduler.__class__ == DDIMScheduler + + pipe.scheduler = DEISMultistepScheduler.from_config(pipe.scheduler.config) + assert pipe.scheduler.config.solver_type == "logrho" + + # Switch to UniPC, verify the solver is the default + pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) + assert pipe.scheduler.config.solver_type == "bh2" + + +class SchedulerCommonTest(unittest.TestCase): + scheduler_classes = () + forward_default_kwargs = () + + @property + def default_num_inference_steps(self): + return 50 + + @property + def default_timestep(self): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.get("num_inference_steps", self.default_num_inference_steps) + + try: + scheduler_config = self.get_scheduler_config() + scheduler = self.scheduler_classes[0](**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + timestep = scheduler.timesteps[0] + except NotImplementedError: + logger.warning( + f"The scheduler {self.__class__.__name__} does not implement a `get_scheduler_config` method." + f" `default_timestep` will be set to the default value of 1." + ) + timestep = 1 + + return timestep + + # NOTE: currently taking the convention that default_timestep > default_timestep_2 (alternatively, + # default_timestep comes earlier in the timestep schedule than default_timestep_2) + @property + def default_timestep_2(self): + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.get("num_inference_steps", self.default_num_inference_steps) + + try: + scheduler_config = self.get_scheduler_config() + scheduler = self.scheduler_classes[0](**scheduler_config) + + scheduler.set_timesteps(num_inference_steps) + if len(scheduler.timesteps) >= 2: + timestep_2 = scheduler.timesteps[1] + else: + logger.warning( + f"Using num_inference_steps from the scheduler testing class's default config leads to a timestep" + f" scheduler of length {len(scheduler.timesteps)} < 2. The default `default_timestep_2` value of 0" + f" will be used." + ) + timestep_2 = 0 + except NotImplementedError: + logger.warning( + f"The scheduler {self.__class__.__name__} does not implement a `get_scheduler_config` method." + f" `default_timestep_2` will be set to the default value of 0." + ) + timestep_2 = 0 + + return timestep_2 + + @property + def dummy_sample(self): + batch_size = 4 + num_channels = 3 + height = 8 + width = 8 + + sample = torch.rand((batch_size, num_channels, height, width)) + + return sample + + @property + def dummy_noise_deter(self): + batch_size = 4 + num_channels = 3 + height = 8 + width = 8 + + num_elems = batch_size * num_channels * height * width + sample = torch.arange(num_elems).flip(-1) + sample = sample.reshape(num_channels, height, width, batch_size) + sample = sample / num_elems + sample = sample.permute(3, 0, 1, 2) + + return sample + + @property + def dummy_sample_deter(self): + batch_size = 4 + num_channels = 3 + height = 8 + width = 8 + + num_elems = batch_size * num_channels * height * width + sample = torch.arange(num_elems) + sample = sample.reshape(num_channels, height, width, batch_size) + sample = sample / num_elems + sample = sample.permute(3, 0, 1, 2) + + return sample + + def get_scheduler_config(self): + raise NotImplementedError + + def dummy_model(self): + def model(sample, t, *args): + # if t is a tensor, match the number of dimensions of sample + if isinstance(t, torch.Tensor): + num_dims = len(sample.shape) + # pad t with 1s to match num_dims + t = t.reshape(-1, *(1,) * (num_dims - 1)).to(sample.device).to(sample.dtype) + + return sample * t / (t + 1) + + return model + + def check_over_configs(self, time_step=0, **config): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + time_step = time_step if time_step is not None else self.default_timestep + + for scheduler_class in self.scheduler_classes: + # TODO(Suraj) - delete the following two lines once DDPM, DDIM, and PNDM have timesteps casted to float by default + if scheduler_class in (EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, LMSDiscreteScheduler): + time_step = float(time_step) + + scheduler_config = self.get_scheduler_config(**config) + scheduler = scheduler_class(**scheduler_config) + + if scheduler_class == CMStochasticIterativeScheduler: + # Get valid timestep based on sigma_max, which should always be in timestep schedule. + scaled_sigma_max = scheduler.sigma_to_t(scheduler.config.sigma_max) + time_step = scaled_sigma_max + + if scheduler_class == EDMEulerScheduler: + time_step = scheduler.timesteps[-1] + + if scheduler_class == VQDiffusionScheduler: + num_vec_classes = scheduler_config["num_vec_classes"] + sample = self.dummy_sample(num_vec_classes) + model = self.dummy_model(num_vec_classes) + residual = model(sample, time_step) + else: + sample = self.dummy_sample + residual = 0.1 * sample + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + new_scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # Make sure `scale_model_input` is invoked to prevent a warning + if scheduler_class == CMStochasticIterativeScheduler: + # Get valid timestep based on sigma_max, which should always be in timestep schedule. + _ = scheduler.scale_model_input(sample, scaled_sigma_max) + _ = new_scheduler.scale_model_input(sample, scaled_sigma_max) + elif scheduler_class != VQDiffusionScheduler: + _ = scheduler.scale_model_input(sample, scheduler.timesteps[-1]) + _ = new_scheduler.scale_model_input(sample, scheduler.timesteps[-1]) + + # Set the seed before step() as some schedulers are stochastic like EulerAncestralDiscreteScheduler, EulerDiscreteScheduler + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def check_over_forward(self, time_step=0, **forward_kwargs): + kwargs = dict(self.forward_default_kwargs) + kwargs.update(forward_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", None) + time_step = time_step if time_step is not None else self.default_timestep + + for scheduler_class in self.scheduler_classes: + if scheduler_class in (EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, LMSDiscreteScheduler): + time_step = float(time_step) + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + if scheduler_class == VQDiffusionScheduler: + num_vec_classes = scheduler_config["num_vec_classes"] + sample = self.dummy_sample(num_vec_classes) + model = self.dummy_model(num_vec_classes) + residual = model(sample, time_step) + else: + sample = self.dummy_sample + residual = 0.1 * sample + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + new_scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + output = scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + new_output = new_scheduler.step(residual, time_step, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_from_save_pretrained(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", self.default_num_inference_steps) + + for scheduler_class in self.scheduler_classes: + timestep = self.default_timestep + if scheduler_class in (EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, LMSDiscreteScheduler): + timestep = float(timestep) + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + if scheduler_class == CMStochasticIterativeScheduler: + # Get valid timestep based on sigma_max, which should always be in timestep schedule. + timestep = scheduler.sigma_to_t(scheduler.config.sigma_max) + + if scheduler_class == VQDiffusionScheduler: + num_vec_classes = scheduler_config["num_vec_classes"] + sample = self.dummy_sample(num_vec_classes) + model = self.dummy_model(num_vec_classes) + residual = model(sample, timestep) + else: + sample = self.dummy_sample + residual = 0.1 * sample + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_config(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + new_scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + output = scheduler.step(residual, timestep, sample, **kwargs).prev_sample + + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + new_output = new_scheduler.step(residual, timestep, sample, **kwargs).prev_sample + + assert torch.sum(torch.abs(output - new_output)) < 1e-5, "Scheduler outputs are not identical" + + def test_compatibles(self): + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + + scheduler = scheduler_class(**scheduler_config) + + assert all(c is not None for c in scheduler.compatibles) + + for comp_scheduler_cls in scheduler.compatibles: + comp_scheduler = comp_scheduler_cls.from_config(scheduler.config) + assert comp_scheduler is not None + + new_scheduler = scheduler_class.from_config(comp_scheduler.config) + + new_scheduler_config = {k: v for k, v in new_scheduler.config.items() if k in scheduler.config} + scheduler_diff = {k: v for k, v in new_scheduler.config.items() if k not in scheduler.config} + + # make sure that configs are essentially identical + assert new_scheduler_config == dict(scheduler.config) + + # make sure that only differences are for configs that are not in init + init_keys = inspect.signature(scheduler_class.__init__).parameters.keys() + assert set(scheduler_diff.keys()).intersection(set(init_keys)) == set() + + def test_from_pretrained(self): + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + + scheduler = scheduler_class(**scheduler_config) + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_pretrained(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + + # `_use_default_values` should not exist for just saved & loaded scheduler + scheduler_config = dict(scheduler.config) + del scheduler_config["_use_default_values"] + + assert scheduler_config == new_scheduler.config + + def test_step_shape(self): + kwargs = dict(self.forward_default_kwargs) + + num_inference_steps = kwargs.pop("num_inference_steps", self.default_num_inference_steps) + + timestep_0 = self.default_timestep + timestep_1 = self.default_timestep_2 + + for scheduler_class in self.scheduler_classes: + if scheduler_class in (EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, LMSDiscreteScheduler): + timestep_0 = float(timestep_0) + timestep_1 = float(timestep_1) + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + if scheduler_class == VQDiffusionScheduler: + num_vec_classes = scheduler_config["num_vec_classes"] + sample = self.dummy_sample(num_vec_classes) + model = self.dummy_model(num_vec_classes) + residual = model(sample, timestep_0) + else: + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + output_0 = scheduler.step(residual, timestep_0, sample, **kwargs).prev_sample + output_1 = scheduler.step(residual, timestep_1, sample, **kwargs).prev_sample + + self.assertEqual(output_0.shape, sample.shape) + self.assertEqual(output_0.shape, output_1.shape) + + def test_scheduler_outputs_equivalence(self): + def set_nan_tensor_to_zero(t): + t[t != t] = 0 + return t + + def recursive_check(tuple_object, dict_object): + if isinstance(tuple_object, (List, Tuple)): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object, dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif isinstance(tuple_object, Dict): + for tuple_iterable_value, dict_iterable_value in zip(tuple_object.values(), dict_object.values()): + recursive_check(tuple_iterable_value, dict_iterable_value) + elif tuple_object is None: + return + else: + self.assertTrue( + torch.allclose( + set_nan_tensor_to_zero(tuple_object), set_nan_tensor_to_zero(dict_object), atol=1e-5 + ), + msg=( + "Tuple and dict output are not equal. Difference:" + f" {torch.max(torch.abs(tuple_object - dict_object))}. Tuple has `nan`:" + f" {torch.isnan(tuple_object).any()} and `inf`: {torch.isinf(tuple_object)}. Dict has" + f" `nan`: {torch.isnan(dict_object).any()} and `inf`: {torch.isinf(dict_object)}." + ), + ) + + kwargs = dict(self.forward_default_kwargs) + num_inference_steps = kwargs.pop("num_inference_steps", self.default_num_inference_steps) + + timestep = self.default_timestep + if len(self.scheduler_classes) > 0 and self.scheduler_classes[0] == IPNDMScheduler: + timestep = 1 + + for scheduler_class in self.scheduler_classes: + if scheduler_class in (EulerAncestralDiscreteScheduler, EulerDiscreteScheduler, LMSDiscreteScheduler): + timestep = float(timestep) + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + if scheduler_class == CMStochasticIterativeScheduler: + # Get valid timestep based on sigma_max, which should always be in timestep schedule. + timestep = scheduler.sigma_to_t(scheduler.config.sigma_max) + + if scheduler_class == VQDiffusionScheduler: + num_vec_classes = scheduler_config["num_vec_classes"] + sample = self.dummy_sample(num_vec_classes) + model = self.dummy_model(num_vec_classes) + residual = model(sample, timestep) + else: + sample = self.dummy_sample + residual = 0.1 * sample + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # Set the seed before state as some schedulers are stochastic like EulerAncestralDiscreteScheduler, EulerDiscreteScheduler + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + outputs_dict = scheduler.step(residual, timestep, sample, **kwargs) + + if num_inference_steps is not None and hasattr(scheduler, "set_timesteps"): + scheduler.set_timesteps(num_inference_steps) + elif num_inference_steps is not None and not hasattr(scheduler, "set_timesteps"): + kwargs["num_inference_steps"] = num_inference_steps + + # Set the seed before state as some schedulers are stochastic like EulerAncestralDiscreteScheduler, EulerDiscreteScheduler + if "generator" in set(inspect.signature(scheduler.step).parameters.keys()): + kwargs["generator"] = torch.manual_seed(0) + outputs_tuple = scheduler.step(residual, timestep, sample, return_dict=False, **kwargs) + + recursive_check(outputs_tuple, outputs_dict) + + def test_scheduler_public_api(self): + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + if scheduler_class != VQDiffusionScheduler: + self.assertTrue( + hasattr(scheduler, "init_noise_sigma"), + f"{scheduler_class} does not implement a required attribute `init_noise_sigma`", + ) + self.assertTrue( + hasattr(scheduler, "scale_model_input"), + ( + f"{scheduler_class} does not implement a required class method `scale_model_input(sample," + " timestep)`" + ), + ) + self.assertTrue( + hasattr(scheduler, "step"), + f"{scheduler_class} does not implement a required class method `step(...)`", + ) + + if scheduler_class != VQDiffusionScheduler: + sample = self.dummy_sample + if scheduler_class == CMStochasticIterativeScheduler: + # Get valid timestep based on sigma_max, which should always be in timestep schedule. + scaled_sigma_max = scheduler.sigma_to_t(scheduler.config.sigma_max) + scaled_sample = scheduler.scale_model_input(sample, scaled_sigma_max) + elif scheduler_class == EDMEulerScheduler: + scaled_sample = scheduler.scale_model_input(sample, scheduler.timesteps[-1]) + else: + scaled_sample = scheduler.scale_model_input(sample, 0.0) + self.assertEqual(sample.shape, scaled_sample.shape) + + def test_add_noise_device(self): + for scheduler_class in self.scheduler_classes: + if scheduler_class == IPNDMScheduler: + continue + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + scheduler.set_timesteps(self.default_num_inference_steps) + + sample = self.dummy_sample.to(torch_device) + if scheduler_class == CMStochasticIterativeScheduler: + # Get valid timestep based on sigma_max, which should always be in timestep schedule. + scaled_sigma_max = scheduler.sigma_to_t(scheduler.config.sigma_max) + scaled_sample = scheduler.scale_model_input(sample, scaled_sigma_max) + if scheduler_class == EDMEulerScheduler: + scaled_sample = scheduler.scale_model_input(sample, scheduler.timesteps[-1]) + else: + scaled_sample = scheduler.scale_model_input(sample, 0.0) + self.assertEqual(sample.shape, scaled_sample.shape) + + noise = torch.randn_like(scaled_sample).to(torch_device) + t = scheduler.timesteps[5][None] + noised = scheduler.add_noise(scaled_sample, noise, t) + self.assertEqual(noised.shape, scaled_sample.shape) + + def test_deprecated_kwargs(self): + for scheduler_class in self.scheduler_classes: + has_kwarg_in_model_class = "kwargs" in inspect.signature(scheduler_class.__init__).parameters + has_deprecated_kwarg = len(scheduler_class._deprecated_kwargs) > 0 + + if has_kwarg_in_model_class and not has_deprecated_kwarg: + raise ValueError( + f"{scheduler_class} has `**kwargs` in its __init__ method but has not defined any deprecated" + " kwargs under the `_deprecated_kwargs` class attribute. Make sure to either remove `**kwargs` if" + " there are no deprecated arguments or add the deprecated argument with `_deprecated_kwargs =" + " []`" + ) + + if not has_kwarg_in_model_class and has_deprecated_kwarg: + raise ValueError( + f"{scheduler_class} doesn't have `**kwargs` in its __init__ method but has defined deprecated" + " kwargs under the `_deprecated_kwargs` class attribute. Make sure to either add the `**kwargs`" + f" argument to {self.model_class}.__init__ if there are deprecated arguments or remove the" + " deprecated argument from `_deprecated_kwargs = []`" + ) + + def test_trained_betas(self): + for scheduler_class in self.scheduler_classes: + if scheduler_class in (VQDiffusionScheduler, CMStochasticIterativeScheduler): + continue + + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config, trained_betas=np.array([0.1, 0.3])) + + with tempfile.TemporaryDirectory() as tmpdirname: + scheduler.save_pretrained(tmpdirname) + new_scheduler = scheduler_class.from_pretrained(tmpdirname) + + assert scheduler.betas.tolist() == new_scheduler.betas.tolist() + + def test_getattr_is_correct(self): + for scheduler_class in self.scheduler_classes: + scheduler_config = self.get_scheduler_config() + scheduler = scheduler_class(**scheduler_config) + + # save some things to test + scheduler.dummy_attribute = 5 + scheduler.register_to_config(test_attribute=5) + + logger = logging.get_logger("diffusers.configuration_utils") + # 30 for warning + logger.setLevel(30) + with CaptureLogger(logger) as cap_logger: + assert hasattr(scheduler, "dummy_attribute") + assert getattr(scheduler, "dummy_attribute") == 5 + assert scheduler.dummy_attribute == 5 + + # no warning should be thrown + assert cap_logger.out == "" + + logger = logging.get_logger("diffusers.schedulers.schedulering_utils") + # 30 for warning + logger.setLevel(30) + with CaptureLogger(logger) as cap_logger: + assert hasattr(scheduler, "save_pretrained") + fn = scheduler.save_pretrained + fn_1 = getattr(scheduler, "save_pretrained") + + assert fn == fn_1 + # no warning should be thrown + assert cap_logger.out == "" + + # warning should be thrown + with self.assertWarns(FutureWarning): + assert scheduler.test_attribute == 5 + + with self.assertWarns(FutureWarning): + assert getattr(scheduler, "test_attribute") == 5 + + with self.assertRaises(AttributeError) as error: + scheduler.does_not_exist + + assert str(error.exception) == f"'{type(scheduler).__name__}' object has no attribute 'does_not_exist'" + + +@is_staging_test +class SchedulerPushToHubTester(unittest.TestCase): + identifier = uuid.uuid4() + repo_id = f"test-scheduler-{identifier}" + org_repo_id = f"valid_org/{repo_id}-org" + + def test_push_to_hub(self): + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + scheduler.push_to_hub(self.repo_id, token=TOKEN) + scheduler_loaded = DDIMScheduler.from_pretrained(f"{USER}/{self.repo_id}") + + assert type(scheduler) == type(scheduler_loaded) + + # Reset repo + delete_repo(token=TOKEN, repo_id=self.repo_id) + + # Push to hub via save_config + with tempfile.TemporaryDirectory() as tmp_dir: + scheduler.save_config(tmp_dir, repo_id=self.repo_id, push_to_hub=True, token=TOKEN) + + scheduler_loaded = DDIMScheduler.from_pretrained(f"{USER}/{self.repo_id}") + + assert type(scheduler) == type(scheduler_loaded) + + # Reset repo + delete_repo(token=TOKEN, repo_id=self.repo_id) + + def test_push_to_hub_in_organization(self): + scheduler = DDIMScheduler( + beta_start=0.00085, + beta_end=0.012, + beta_schedule="scaled_linear", + clip_sample=False, + set_alpha_to_one=False, + ) + scheduler.push_to_hub(self.org_repo_id, token=TOKEN) + scheduler_loaded = DDIMScheduler.from_pretrained(self.org_repo_id) + + assert type(scheduler) == type(scheduler_loaded) + + # Reset repo + delete_repo(token=TOKEN, repo_id=self.org_repo_id) + + # Push to hub via save_config + with tempfile.TemporaryDirectory() as tmp_dir: + scheduler.save_config(tmp_dir, repo_id=self.org_repo_id, push_to_hub=True, token=TOKEN) + + scheduler_loaded = DDIMScheduler.from_pretrained(self.org_repo_id) + + assert type(scheduler) == type(scheduler_loaded) + + # Reset repo + delete_repo(token=TOKEN, repo_id=self.org_repo_id) diff --git a/diffusers-0.27.0/utils/check_config_docstrings.py b/diffusers-0.27.0/utils/check_config_docstrings.py new file mode 100755 index 0000000..626a9a4 --- /dev/null +++ b/diffusers-0.27.0/utils/check_config_docstrings.py @@ -0,0 +1,84 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import inspect +import os +import re + + +# All paths are set with the intent you should run this script from the root of the repo with the command +# python utils/check_config_docstrings.py +PATH_TO_TRANSFORMERS = "src/transformers" + + +# This is to make sure the transformers module imported is the one in the repo. +spec = importlib.util.spec_from_file_location( + "transformers", + os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"), + submodule_search_locations=[PATH_TO_TRANSFORMERS], +) +transformers = spec.loader.load_module() + +CONFIG_MAPPING = transformers.models.auto.configuration_auto.CONFIG_MAPPING + +# Regex pattern used to find the checkpoint mentioned in the docstring of `config_class`. +# For example, `[bert-base-uncased](https://huggingface.co/bert-base-uncased)` +_re_checkpoint = re.compile(r"\[(.+?)\]\((https://huggingface\.co/.+?)\)") + + +CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK = { + "CLIPConfigMixin", + "DecisionTransformerConfigMixin", + "EncoderDecoderConfigMixin", + "RagConfigMixin", + "SpeechEncoderDecoderConfigMixin", + "VisionEncoderDecoderConfigMixin", + "VisionTextDualEncoderConfigMixin", +} + + +def check_config_docstrings_have_checkpoints(): + configs_without_checkpoint = [] + + for config_class in list(CONFIG_MAPPING.values()): + checkpoint_found = False + + # source code of `config_class` + config_source = inspect.getsource(config_class) + checkpoints = _re_checkpoint.findall(config_source) + + for checkpoint in checkpoints: + # Each `checkpoint` is a tuple of a checkpoint name and a checkpoint link. + # For example, `('bert-base-uncased', 'https://huggingface.co/bert-base-uncased')` + ckpt_name, ckpt_link = checkpoint + + # verify the checkpoint name corresponds to the checkpoint link + ckpt_link_from_name = f"https://huggingface.co/{ckpt_name}" + if ckpt_link == ckpt_link_from_name: + checkpoint_found = True + break + + name = config_class.__name__ + if not checkpoint_found and name not in CONFIG_CLASSES_TO_IGNORE_FOR_DOCSTRING_CHECKPOINT_CHECK: + configs_without_checkpoint.append(name) + + if len(configs_without_checkpoint) > 0: + message = "\n".join(sorted(configs_without_checkpoint)) + raise ValueError(f"The following configurations don't contain any valid checkpoint:\n{message}") + + +if __name__ == "__main__": + check_config_docstrings_have_checkpoints() diff --git a/diffusers-0.27.0/utils/check_copies.py b/diffusers-0.27.0/utils/check_copies.py new file mode 100755 index 0000000..20449e7 --- /dev/null +++ b/diffusers-0.27.0/utils/check_copies.py @@ -0,0 +1,222 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import glob +import os +import re +import subprocess + + +# All paths are set with the intent you should run this script from the root of the repo with the command +# python utils/check_copies.py +DIFFUSERS_PATH = "src/diffusers" +REPO_PATH = "." + + +def _should_continue(line, indent): + return line.startswith(indent) or len(line) <= 1 or re.search(r"^\s*\)(\s*->.*:|:)\s*$", line) is not None + + +def find_code_in_diffusers(object_name): + """Find and return the code source code of `object_name`.""" + parts = object_name.split(".") + i = 0 + + # First let's find the module where our object lives. + module = parts[i] + while i < len(parts) and not os.path.isfile(os.path.join(DIFFUSERS_PATH, f"{module}.py")): + i += 1 + if i < len(parts): + module = os.path.join(module, parts[i]) + if i >= len(parts): + raise ValueError(f"`object_name` should begin with the name of a module of diffusers but got {object_name}.") + + with open( + os.path.join(DIFFUSERS_PATH, f"{module}.py"), + "r", + encoding="utf-8", + newline="\n", + ) as f: + lines = f.readlines() + + # Now let's find the class / func in the code! + indent = "" + line_index = 0 + for name in parts[i + 1 :]: + while ( + line_index < len(lines) and re.search(rf"^{indent}(class|def)\s+{name}(\(|\:)", lines[line_index]) is None + ): + line_index += 1 + indent += " " + line_index += 1 + + if line_index >= len(lines): + raise ValueError(f" {object_name} does not match any function or class in {module}.") + + # We found the beginning of the class / func, now let's find the end (when the indent diminishes). + start_index = line_index + while line_index < len(lines) and _should_continue(lines[line_index], indent): + line_index += 1 + # Clean up empty lines at the end (if any). + while len(lines[line_index - 1]) <= 1: + line_index -= 1 + + code_lines = lines[start_index:line_index] + return "".join(code_lines) + + +_re_copy_warning = re.compile(r"^(\s*)#\s*Copied from\s+diffusers\.(\S+\.\S+)\s*($|\S.*$)") +_re_replace_pattern = re.compile(r"^\s*(\S+)->(\S+)(\s+.*|$)") +_re_fill_pattern = re.compile(r"]*>") + + +def get_indent(code): + lines = code.split("\n") + idx = 0 + while idx < len(lines) and len(lines[idx]) == 0: + idx += 1 + if idx < len(lines): + return re.search(r"^(\s*)\S", lines[idx]).groups()[0] + return "" + + +def run_ruff(code): + command = ["ruff", "format", "-", "--config", "pyproject.toml", "--silent"] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE) + stdout, _ = process.communicate(input=code.encode()) + return stdout.decode() + + +def stylify(code: str) -> str: + """ + Applies the ruff part of our `make style` command to some code. This formats the code using `ruff format`. + As `ruff` does not provide a python api this cannot be done on the fly. + + Args: + code (`str`): The code to format. + + Returns: + `str`: The formatted code. + """ + has_indent = len(get_indent(code)) > 0 + if has_indent: + code = f"class Bla:\n{code}" + formatted_code = run_ruff(code) + return formatted_code[len("class Bla:\n") :] if has_indent else formatted_code + + +def is_copy_consistent(filename, overwrite=False): + """ + Check if the code commented as a copy in `filename` matches the original. + Return the differences or overwrites the content depending on `overwrite`. + """ + with open(filename, "r", encoding="utf-8", newline="\n") as f: + lines = f.readlines() + diffs = [] + line_index = 0 + # Not a for loop cause `lines` is going to change (if `overwrite=True`). + while line_index < len(lines): + search = _re_copy_warning.search(lines[line_index]) + if search is None: + line_index += 1 + continue + + # There is some copied code here, let's retrieve the original. + indent, object_name, replace_pattern = search.groups() + theoretical_code = find_code_in_diffusers(object_name) + theoretical_indent = get_indent(theoretical_code) + + start_index = line_index + 1 if indent == theoretical_indent else line_index + 2 + indent = theoretical_indent + line_index = start_index + + # Loop to check the observed code, stop when indentation diminishes or if we see a End copy comment. + should_continue = True + while line_index < len(lines) and should_continue: + line_index += 1 + if line_index >= len(lines): + break + line = lines[line_index] + should_continue = _should_continue(line, indent) and re.search(f"^{indent}# End copy", line) is None + # Clean up empty lines at the end (if any). + while len(lines[line_index - 1]) <= 1: + line_index -= 1 + + observed_code_lines = lines[start_index:line_index] + observed_code = "".join(observed_code_lines) + + # Remove any nested `Copied from` comments to avoid circular copies + theoretical_code = [line for line in theoretical_code.split("\n") if _re_copy_warning.search(line) is None] + theoretical_code = "\n".join(theoretical_code) + + # Before comparing, use the `replace_pattern` on the original code. + if len(replace_pattern) > 0: + patterns = replace_pattern.replace("with", "").split(",") + patterns = [_re_replace_pattern.search(p) for p in patterns] + for pattern in patterns: + if pattern is None: + continue + obj1, obj2, option = pattern.groups() + theoretical_code = re.sub(obj1, obj2, theoretical_code) + if option.strip() == "all-casing": + theoretical_code = re.sub(obj1.lower(), obj2.lower(), theoretical_code) + theoretical_code = re.sub(obj1.upper(), obj2.upper(), theoretical_code) + + # stylify after replacement. To be able to do that, we need the header (class or function definition) + # from the previous line + theoretical_code = stylify(lines[start_index - 1] + theoretical_code) + theoretical_code = theoretical_code[len(lines[start_index - 1]) :] + + # Test for a diff and act accordingly. + if observed_code != theoretical_code: + diffs.append([object_name, start_index]) + if overwrite: + lines = lines[:start_index] + [theoretical_code] + lines[line_index:] + line_index = start_index + 1 + + if overwrite and len(diffs) > 0: + # Warn the user a file has been modified. + print(f"Detected changes, rewriting {filename}.") + with open(filename, "w", encoding="utf-8", newline="\n") as f: + f.writelines(lines) + return diffs + + +def check_copies(overwrite: bool = False): + all_files = glob.glob(os.path.join(DIFFUSERS_PATH, "**/*.py"), recursive=True) + diffs = [] + for filename in all_files: + new_diffs = is_copy_consistent(filename, overwrite) + diffs += [f"- {filename}: copy does not match {d[0]} at line {d[1]}" for d in new_diffs] + if not overwrite and len(diffs) > 0: + diff = "\n".join(diffs) + raise Exception( + "Found the following copy inconsistencies:\n" + + diff + + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them." + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--fix_and_overwrite", + action="store_true", + help="Whether to fix inconsistencies.", + ) + args = parser.parse_args() + + check_copies(args.fix_and_overwrite) diff --git a/diffusers-0.27.0/utils/check_doc_toc.py b/diffusers-0.27.0/utils/check_doc_toc.py new file mode 100755 index 0000000..35ded93 --- /dev/null +++ b/diffusers-0.27.0/utils/check_doc_toc.py @@ -0,0 +1,158 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +from collections import defaultdict + +import yaml + + +PATH_TO_TOC = "docs/source/en/_toctree.yml" + + +def clean_doc_toc(doc_list): + """ + Cleans the table of content of the model documentation by removing duplicates and sorting models alphabetically. + """ + counts = defaultdict(int) + overview_doc = [] + new_doc_list = [] + for doc in doc_list: + if "local" in doc: + counts[doc["local"]] += 1 + + if doc["title"].lower() == "overview": + overview_doc.append({"local": doc["local"], "title": doc["title"]}) + else: + new_doc_list.append(doc) + + doc_list = new_doc_list + duplicates = [key for key, value in counts.items() if value > 1] + + new_doc = [] + for duplicate_key in duplicates: + titles = list({doc["title"] for doc in doc_list if doc["local"] == duplicate_key}) + if len(titles) > 1: + raise ValueError( + f"{duplicate_key} is present several times in the documentation table of content at " + "`docs/source/en/_toctree.yml` with different *Title* values. Choose one of those and remove the " + "others." + ) + # Only add this once + new_doc.append({"local": duplicate_key, "title": titles[0]}) + + # Add none duplicate-keys + new_doc.extend([doc for doc in doc_list if "local" not in counts or counts[doc["local"]] == 1]) + new_doc = sorted(new_doc, key=lambda s: s["title"].lower()) + + # "overview" gets special treatment and is always first + if len(overview_doc) > 1: + raise ValueError("{doc_list} has two 'overview' docs which is not allowed.") + + overview_doc.extend(new_doc) + + # Sort + return overview_doc + + +def check_scheduler_doc(overwrite=False): + with open(PATH_TO_TOC, encoding="utf-8") as f: + content = yaml.safe_load(f.read()) + + # Get to the API doc + api_idx = 0 + while content[api_idx]["title"] != "API": + api_idx += 1 + api_doc = content[api_idx]["sections"] + + # Then to the model doc + scheduler_idx = 0 + while api_doc[scheduler_idx]["title"] != "Schedulers": + scheduler_idx += 1 + + scheduler_doc = api_doc[scheduler_idx]["sections"] + new_scheduler_doc = clean_doc_toc(scheduler_doc) + + diff = False + if new_scheduler_doc != scheduler_doc: + diff = True + if overwrite: + api_doc[scheduler_idx]["sections"] = new_scheduler_doc + + if diff: + if overwrite: + content[api_idx]["sections"] = api_doc + with open(PATH_TO_TOC, "w", encoding="utf-8") as f: + f.write(yaml.dump(content, allow_unicode=True)) + else: + raise ValueError( + "The model doc part of the table of content is not properly sorted, run `make style` to fix this." + ) + + +def check_pipeline_doc(overwrite=False): + with open(PATH_TO_TOC, encoding="utf-8") as f: + content = yaml.safe_load(f.read()) + + # Get to the API doc + api_idx = 0 + while content[api_idx]["title"] != "API": + api_idx += 1 + api_doc = content[api_idx]["sections"] + + # Then to the model doc + pipeline_idx = 0 + while api_doc[pipeline_idx]["title"] != "Pipelines": + pipeline_idx += 1 + + diff = False + pipeline_docs = api_doc[pipeline_idx]["sections"] + new_pipeline_docs = [] + + # sort sub pipeline docs + for pipeline_doc in pipeline_docs: + if "section" in pipeline_doc: + sub_pipeline_doc = pipeline_doc["section"] + new_sub_pipeline_doc = clean_doc_toc(sub_pipeline_doc) + if overwrite: + pipeline_doc["section"] = new_sub_pipeline_doc + new_pipeline_docs.append(pipeline_doc) + + # sort overall pipeline doc + new_pipeline_docs = clean_doc_toc(new_pipeline_docs) + + if new_pipeline_docs != pipeline_docs: + diff = True + if overwrite: + api_doc[pipeline_idx]["sections"] = new_pipeline_docs + + if diff: + if overwrite: + content[api_idx]["sections"] = api_doc + with open(PATH_TO_TOC, "w", encoding="utf-8") as f: + f.write(yaml.dump(content, allow_unicode=True)) + else: + raise ValueError( + "The model doc part of the table of content is not properly sorted, run `make style` to fix this." + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.") + args = parser.parse_args() + + check_scheduler_doc(args.fix_and_overwrite) + check_pipeline_doc(args.fix_and_overwrite) diff --git a/diffusers-0.27.0/utils/check_dummies.py b/diffusers-0.27.0/utils/check_dummies.py new file mode 100755 index 0000000..af99eeb --- /dev/null +++ b/diffusers-0.27.0/utils/check_dummies.py @@ -0,0 +1,175 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import re + + +# All paths are set with the intent you should run this script from the root of the repo with the command +# python utils/check_dummies.py +PATH_TO_DIFFUSERS = "src/diffusers" + +# Matches is_xxx_available() +_re_backend = re.compile(r"is\_([a-z_]*)_available\(\)") +# Matches from xxx import bla +_re_single_line_import = re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n") + + +DUMMY_CONSTANT = """ +{0} = None +""" + +DUMMY_CLASS = """ +class {0}(metaclass=DummyObject): + _backends = {1} + + def __init__(self, *args, **kwargs): + requires_backends(self, {1}) + + @classmethod + def from_config(cls, *args, **kwargs): + requires_backends(cls, {1}) + + @classmethod + def from_pretrained(cls, *args, **kwargs): + requires_backends(cls, {1}) +""" + + +DUMMY_FUNCTION = """ +def {0}(*args, **kwargs): + requires_backends({0}, {1}) +""" + + +def find_backend(line): + """Find one (or multiple) backend in a code line of the init.""" + backends = _re_backend.findall(line) + if len(backends) == 0: + return None + + return "_and_".join(backends) + + +def read_init(): + """Read the init and extracts PyTorch, TensorFlow, SentencePiece and Tokenizers objects.""" + with open(os.path.join(PATH_TO_DIFFUSERS, "__init__.py"), "r", encoding="utf-8", newline="\n") as f: + lines = f.readlines() + + # Get to the point we do the actual imports for type checking + line_index = 0 + while not lines[line_index].startswith("if TYPE_CHECKING"): + line_index += 1 + + backend_specific_objects = {} + # Go through the end of the file + while line_index < len(lines): + # If the line contains is_backend_available, we grab all objects associated with the `else` block + backend = find_backend(lines[line_index]) + if backend is not None: + while not lines[line_index].startswith(" else:"): + line_index += 1 + line_index += 1 + objects = [] + # Until we unindent, add backend objects to the list + while len(lines[line_index]) <= 1 or lines[line_index].startswith(" " * 8): + line = lines[line_index] + single_line_import_search = _re_single_line_import.search(line) + if single_line_import_search is not None: + objects.extend(single_line_import_search.groups()[0].split(", ")) + elif line.startswith(" " * 12): + objects.append(line[12:-2]) + line_index += 1 + + if len(objects) > 0: + backend_specific_objects[backend] = objects + else: + line_index += 1 + + return backend_specific_objects + + +def create_dummy_object(name, backend_name): + """Create the code for the dummy object corresponding to `name`.""" + if name.isupper(): + return DUMMY_CONSTANT.format(name) + elif name.islower(): + return DUMMY_FUNCTION.format(name, backend_name) + else: + return DUMMY_CLASS.format(name, backend_name) + + +def create_dummy_files(backend_specific_objects=None): + """Create the content of the dummy files.""" + if backend_specific_objects is None: + backend_specific_objects = read_init() + # For special correspondence backend to module name as used in the function requires_modulename + dummy_files = {} + + for backend, objects in backend_specific_objects.items(): + backend_name = "[" + ", ".join(f'"{b}"' for b in backend.split("_and_")) + "]" + dummy_file = "# This file is autogenerated by the command `make fix-copies`, do not edit.\n" + dummy_file += "from ..utils import DummyObject, requires_backends\n\n" + dummy_file += "\n".join([create_dummy_object(o, backend_name) for o in objects]) + dummy_files[backend] = dummy_file + + return dummy_files + + +def check_dummies(overwrite=False): + """Check if the dummy files are up to date and maybe `overwrite` with the right content.""" + dummy_files = create_dummy_files() + # For special correspondence backend to shortcut as used in utils/dummy_xxx_objects.py + short_names = {"torch": "pt"} + + # Locate actual dummy modules and read their content. + path = os.path.join(PATH_TO_DIFFUSERS, "utils") + dummy_file_paths = { + backend: os.path.join(path, f"dummy_{short_names.get(backend, backend)}_objects.py") + for backend in dummy_files.keys() + } + + actual_dummies = {} + for backend, file_path in dummy_file_paths.items(): + if os.path.isfile(file_path): + with open(file_path, "r", encoding="utf-8", newline="\n") as f: + actual_dummies[backend] = f.read() + else: + actual_dummies[backend] = "" + + for backend in dummy_files.keys(): + if dummy_files[backend] != actual_dummies[backend]: + if overwrite: + print( + f"Updating diffusers.utils.dummy_{short_names.get(backend, backend)}_objects.py as the main " + "__init__ has new objects." + ) + with open(dummy_file_paths[backend], "w", encoding="utf-8", newline="\n") as f: + f.write(dummy_files[backend]) + else: + raise ValueError( + "The main __init__ has objects that are not present in " + f"diffusers.utils.dummy_{short_names.get(backend, backend)}_objects.py. Run `make fix-copies` " + "to fix this." + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.") + args = parser.parse_args() + + check_dummies(args.fix_and_overwrite) diff --git a/diffusers-0.27.0/utils/check_inits.py b/diffusers-0.27.0/utils/check_inits.py new file mode 100755 index 0000000..2c51404 --- /dev/null +++ b/diffusers-0.27.0/utils/check_inits.py @@ -0,0 +1,299 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collections +import importlib.util +import os +import re +from pathlib import Path + + +PATH_TO_TRANSFORMERS = "src/transformers" + + +# Matches is_xxx_available() +_re_backend = re.compile(r"is\_([a-z_]*)_available()") +# Catches a one-line _import_struct = {xxx} +_re_one_line_import_struct = re.compile(r"^_import_structure\s+=\s+\{([^\}]+)\}") +# Catches a line with a key-values pattern: "bla": ["foo", "bar"] +_re_import_struct_key_value = re.compile(r'\s+"\S*":\s+\[([^\]]*)\]') +# Catches a line if not is_foo_available +_re_test_backend = re.compile(r"^\s*if\s+not\s+is\_[a-z_]*\_available\(\)") +# Catches a line _import_struct["bla"].append("foo") +_re_import_struct_add_one = re.compile(r'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)') +# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"] +_re_import_struct_add_many = re.compile(r"^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]") +# Catches a line with an object between quotes and a comma: "MyModel", +_re_quote_object = re.compile(r'^\s+"([^"]+)",') +# Catches a line with objects between brackets only: ["foo", "bar"], +_re_between_brackets = re.compile(r"^\s+\[([^\]]+)\]") +# Catches a line with from foo import bar, bla, boo +_re_import = re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n") +# Catches a line with try: +_re_try = re.compile(r"^\s*try:") +# Catches a line with else: +_re_else = re.compile(r"^\s*else:") + + +def find_backend(line): + """Find one (or multiple) backend in a code line of the init.""" + if _re_test_backend.search(line) is None: + return None + backends = [b[0] for b in _re_backend.findall(line)] + backends.sort() + return "_and_".join(backends) + + +def parse_init(init_file): + """ + Read an init_file and parse (per backend) the _import_structure objects defined and the TYPE_CHECKING objects + defined + """ + with open(init_file, "r", encoding="utf-8", newline="\n") as f: + lines = f.readlines() + + line_index = 0 + while line_index < len(lines) and not lines[line_index].startswith("_import_structure = {"): + line_index += 1 + + # If this is a traditional init, just return. + if line_index >= len(lines): + return None + + # First grab the objects without a specific backend in _import_structure + objects = [] + while not lines[line_index].startswith("if TYPE_CHECKING") and find_backend(lines[line_index]) is None: + line = lines[line_index] + # If we have everything on a single line, let's deal with it. + if _re_one_line_import_struct.search(line): + content = _re_one_line_import_struct.search(line).groups()[0] + imports = re.findall(r"\[([^\]]+)\]", content) + for imp in imports: + objects.extend([obj[1:-1] for obj in imp.split(", ")]) + line_index += 1 + continue + single_line_import_search = _re_import_struct_key_value.search(line) + if single_line_import_search is not None: + imports = [obj[1:-1] for obj in single_line_import_search.groups()[0].split(", ") if len(obj) > 0] + objects.extend(imports) + elif line.startswith(" " * 8 + '"'): + objects.append(line[9:-3]) + line_index += 1 + + import_dict_objects = {"none": objects} + # Let's continue with backend-specific objects in _import_structure + while not lines[line_index].startswith("if TYPE_CHECKING"): + # If the line is an if not is_backend_available, we grab all objects associated. + backend = find_backend(lines[line_index]) + # Check if the backend declaration is inside a try block: + if _re_try.search(lines[line_index - 1]) is None: + backend = None + + if backend is not None: + line_index += 1 + + # Scroll until we hit the else block of try-except-else + while _re_else.search(lines[line_index]) is None: + line_index += 1 + + line_index += 1 + + objects = [] + # Until we unindent, add backend objects to the list + while len(lines[line_index]) <= 1 or lines[line_index].startswith(" " * 4): + line = lines[line_index] + if _re_import_struct_add_one.search(line) is not None: + objects.append(_re_import_struct_add_one.search(line).groups()[0]) + elif _re_import_struct_add_many.search(line) is not None: + imports = _re_import_struct_add_many.search(line).groups()[0].split(", ") + imports = [obj[1:-1] for obj in imports if len(obj) > 0] + objects.extend(imports) + elif _re_between_brackets.search(line) is not None: + imports = _re_between_brackets.search(line).groups()[0].split(", ") + imports = [obj[1:-1] for obj in imports if len(obj) > 0] + objects.extend(imports) + elif _re_quote_object.search(line) is not None: + objects.append(_re_quote_object.search(line).groups()[0]) + elif line.startswith(" " * 8 + '"'): + objects.append(line[9:-3]) + elif line.startswith(" " * 12 + '"'): + objects.append(line[13:-3]) + line_index += 1 + + import_dict_objects[backend] = objects + else: + line_index += 1 + + # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend + objects = [] + while ( + line_index < len(lines) + and find_backend(lines[line_index]) is None + and not lines[line_index].startswith("else") + ): + line = lines[line_index] + single_line_import_search = _re_import.search(line) + if single_line_import_search is not None: + objects.extend(single_line_import_search.groups()[0].split(", ")) + elif line.startswith(" " * 8): + objects.append(line[8:-2]) + line_index += 1 + + type_hint_objects = {"none": objects} + # Let's continue with backend-specific objects + while line_index < len(lines): + # If the line is an if is_backend_available, we grab all objects associated. + backend = find_backend(lines[line_index]) + # Check if the backend declaration is inside a try block: + if _re_try.search(lines[line_index - 1]) is None: + backend = None + + if backend is not None: + line_index += 1 + + # Scroll until we hit the else block of try-except-else + while _re_else.search(lines[line_index]) is None: + line_index += 1 + + line_index += 1 + + objects = [] + # Until we unindent, add backend objects to the list + while len(lines[line_index]) <= 1 or lines[line_index].startswith(" " * 8): + line = lines[line_index] + single_line_import_search = _re_import.search(line) + if single_line_import_search is not None: + objects.extend(single_line_import_search.groups()[0].split(", ")) + elif line.startswith(" " * 12): + objects.append(line[12:-2]) + line_index += 1 + + type_hint_objects[backend] = objects + else: + line_index += 1 + + return import_dict_objects, type_hint_objects + + +def analyze_results(import_dict_objects, type_hint_objects): + """ + Analyze the differences between _import_structure objects and TYPE_CHECKING objects found in an init. + """ + + def find_duplicates(seq): + return [k for k, v in collections.Counter(seq).items() if v > 1] + + if list(import_dict_objects.keys()) != list(type_hint_objects.keys()): + return ["Both sides of the init do not have the same backends!"] + + errors = [] + for key in import_dict_objects.keys(): + duplicate_imports = find_duplicates(import_dict_objects[key]) + if duplicate_imports: + errors.append(f"Duplicate _import_structure definitions for: {duplicate_imports}") + duplicate_type_hints = find_duplicates(type_hint_objects[key]) + if duplicate_type_hints: + errors.append(f"Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}") + + if sorted(set(import_dict_objects[key])) != sorted(set(type_hint_objects[key])): + name = "base imports" if key == "none" else f"{key} backend" + errors.append(f"Differences for {name}:") + for a in type_hint_objects[key]: + if a not in import_dict_objects[key]: + errors.append(f" {a} in TYPE_HINT but not in _import_structure.") + for a in import_dict_objects[key]: + if a not in type_hint_objects[key]: + errors.append(f" {a} in _import_structure but not in TYPE_HINT.") + return errors + + +def check_all_inits(): + """ + Check all inits in the transformers repo and raise an error if at least one does not define the same objects in + both halves. + """ + failures = [] + for root, _, files in os.walk(PATH_TO_TRANSFORMERS): + if "__init__.py" in files: + fname = os.path.join(root, "__init__.py") + objects = parse_init(fname) + if objects is not None: + errors = analyze_results(*objects) + if len(errors) > 0: + errors[0] = f"Problem in {fname}, both halves do not define the same objects.\n{errors[0]}" + failures.append("\n".join(errors)) + if len(failures) > 0: + raise ValueError("\n\n".join(failures)) + + +def get_transformers_submodules(): + """ + Returns the list of Transformers submodules. + """ + submodules = [] + for path, directories, files in os.walk(PATH_TO_TRANSFORMERS): + for folder in directories: + # Ignore private modules + if folder.startswith("_"): + directories.remove(folder) + continue + # Ignore leftovers from branches (empty folders apart from pycache) + if len(list((Path(path) / folder).glob("*.py"))) == 0: + continue + short_path = str((Path(path) / folder).relative_to(PATH_TO_TRANSFORMERS)) + submodule = short_path.replace(os.path.sep, ".") + submodules.append(submodule) + for fname in files: + if fname == "__init__.py": + continue + short_path = str((Path(path) / fname).relative_to(PATH_TO_TRANSFORMERS)) + submodule = short_path.replace(".py", "").replace(os.path.sep, ".") + if len(submodule.split(".")) == 1: + submodules.append(submodule) + return submodules + + +IGNORE_SUBMODULES = [ + "convert_pytorch_checkpoint_to_tf2", + "modeling_flax_pytorch_utils", +] + + +def check_submodules(): + # This is to make sure the transformers module imported is the one in the repo. + spec = importlib.util.spec_from_file_location( + "transformers", + os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"), + submodule_search_locations=[PATH_TO_TRANSFORMERS], + ) + transformers = spec.loader.load_module() + + module_not_registered = [ + module + for module in get_transformers_submodules() + if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys() + ] + if len(module_not_registered) > 0: + list_of_modules = "\n".join(f"- {module}" for module in module_not_registered) + raise ValueError( + "The following submodules are not properly registered in the main init of Transformers:\n" + f"{list_of_modules}\n" + "Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value." + ) + + +if __name__ == "__main__": + check_all_inits() + check_submodules() diff --git a/diffusers-0.27.0/utils/check_repo.py b/diffusers-0.27.0/utils/check_repo.py new file mode 100755 index 0000000..597893f --- /dev/null +++ b/diffusers-0.27.0/utils/check_repo.py @@ -0,0 +1,755 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib +import inspect +import os +import re +import warnings +from collections import OrderedDict +from difflib import get_close_matches +from pathlib import Path + +from diffusers.models.auto import get_values +from diffusers.utils import ENV_VARS_TRUE_VALUES, is_flax_available, is_torch_available + + +# All paths are set with the intent you should run this script from the root of the repo with the command +# python utils/check_repo.py +PATH_TO_DIFFUSERS = "src/diffusers" +PATH_TO_TESTS = "tests" +PATH_TO_DOC = "docs/source/en" + +# Update this list with models that are supposed to be private. +PRIVATE_MODELS = [ + "DPRSpanPredictor", + "RealmBertModel", + "T5Stack", + "TFDPRSpanPredictor", +] + +# Update this list for models that are not tested with a comment explaining the reason it should not be. +# Being in this list is an exception and should **not** be the rule. +IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [ + # models to ignore for not tested + "OPTDecoder", # Building part of bigger (tested) model. + "DecisionTransformerGPT2Model", # Building part of bigger (tested) model. + "SegformerDecodeHead", # Building part of bigger (tested) model. + "PLBartEncoder", # Building part of bigger (tested) model. + "PLBartDecoder", # Building part of bigger (tested) model. + "PLBartDecoderWrapper", # Building part of bigger (tested) model. + "BigBirdPegasusEncoder", # Building part of bigger (tested) model. + "BigBirdPegasusDecoder", # Building part of bigger (tested) model. + "BigBirdPegasusDecoderWrapper", # Building part of bigger (tested) model. + "DetrEncoder", # Building part of bigger (tested) model. + "DetrDecoder", # Building part of bigger (tested) model. + "DetrDecoderWrapper", # Building part of bigger (tested) model. + "M2M100Encoder", # Building part of bigger (tested) model. + "M2M100Decoder", # Building part of bigger (tested) model. + "Speech2TextEncoder", # Building part of bigger (tested) model. + "Speech2TextDecoder", # Building part of bigger (tested) model. + "LEDEncoder", # Building part of bigger (tested) model. + "LEDDecoder", # Building part of bigger (tested) model. + "BartDecoderWrapper", # Building part of bigger (tested) model. + "BartEncoder", # Building part of bigger (tested) model. + "BertLMHeadModel", # Needs to be setup as decoder. + "BlenderbotSmallEncoder", # Building part of bigger (tested) model. + "BlenderbotSmallDecoderWrapper", # Building part of bigger (tested) model. + "BlenderbotEncoder", # Building part of bigger (tested) model. + "BlenderbotDecoderWrapper", # Building part of bigger (tested) model. + "MBartEncoder", # Building part of bigger (tested) model. + "MBartDecoderWrapper", # Building part of bigger (tested) model. + "MegatronBertLMHeadModel", # Building part of bigger (tested) model. + "MegatronBertEncoder", # Building part of bigger (tested) model. + "MegatronBertDecoder", # Building part of bigger (tested) model. + "MegatronBertDecoderWrapper", # Building part of bigger (tested) model. + "PegasusEncoder", # Building part of bigger (tested) model. + "PegasusDecoderWrapper", # Building part of bigger (tested) model. + "DPREncoder", # Building part of bigger (tested) model. + "ProphetNetDecoderWrapper", # Building part of bigger (tested) model. + "RealmBertModel", # Building part of bigger (tested) model. + "RealmReader", # Not regular model. + "RealmScorer", # Not regular model. + "RealmForOpenQA", # Not regular model. + "ReformerForMaskedLM", # Needs to be setup as decoder. + "Speech2Text2DecoderWrapper", # Building part of bigger (tested) model. + "TFDPREncoder", # Building part of bigger (tested) model. + "TFElectraMainLayer", # Building part of bigger (tested) model (should it be a TFModelMixin ?) + "TFRobertaForMultipleChoice", # TODO: fix + "TrOCRDecoderWrapper", # Building part of bigger (tested) model. + "SeparableConv1D", # Building part of bigger (tested) model. + "FlaxBartForCausalLM", # Building part of bigger (tested) model. + "FlaxBertForCausalLM", # Building part of bigger (tested) model. Tested implicitly through FlaxRobertaForCausalLM. + "OPTDecoderWrapper", +] + +# Update this list with test files that don't have a tester with a `all_model_classes` variable and which don't +# trigger the common tests. +TEST_FILES_WITH_NO_COMMON_TESTS = [ + "models/decision_transformer/test_modeling_decision_transformer.py", + "models/camembert/test_modeling_camembert.py", + "models/mt5/test_modeling_flax_mt5.py", + "models/mbart/test_modeling_mbart.py", + "models/mt5/test_modeling_mt5.py", + "models/pegasus/test_modeling_pegasus.py", + "models/camembert/test_modeling_tf_camembert.py", + "models/mt5/test_modeling_tf_mt5.py", + "models/xlm_roberta/test_modeling_tf_xlm_roberta.py", + "models/xlm_roberta/test_modeling_flax_xlm_roberta.py", + "models/xlm_prophetnet/test_modeling_xlm_prophetnet.py", + "models/xlm_roberta/test_modeling_xlm_roberta.py", + "models/vision_text_dual_encoder/test_modeling_vision_text_dual_encoder.py", + "models/vision_text_dual_encoder/test_modeling_flax_vision_text_dual_encoder.py", + "models/decision_transformer/test_modeling_decision_transformer.py", +] + +# Update this list for models that are not in any of the auto MODEL_XXX_MAPPING. Being in this list is an exception and +# should **not** be the rule. +IGNORE_NON_AUTO_CONFIGURED = PRIVATE_MODELS.copy() + [ + # models to ignore for model xxx mapping + "DPTForDepthEstimation", + "DecisionTransformerGPT2Model", + "GLPNForDepthEstimation", + "ViltForQuestionAnswering", + "ViltForImagesAndTextClassification", + "ViltForImageAndTextRetrieval", + "ViltForMaskedLM", + "XGLMEncoder", + "XGLMDecoder", + "XGLMDecoderWrapper", + "PerceiverForMultimodalAutoencoding", + "PerceiverForOpticalFlow", + "SegformerDecodeHead", + "FlaxBeitForMaskedImageModeling", + "PLBartEncoder", + "PLBartDecoder", + "PLBartDecoderWrapper", + "BeitForMaskedImageModeling", + "CLIPTextModel", + "CLIPVisionModel", + "TFCLIPTextModel", + "TFCLIPVisionModel", + "FlaxCLIPTextModel", + "FlaxCLIPVisionModel", + "FlaxWav2Vec2ForCTC", + "DetrForSegmentation", + "DPRReader", + "FlaubertForQuestionAnswering", + "FlavaImageCodebook", + "FlavaTextModel", + "FlavaImageModel", + "FlavaMultimodalModel", + "GPT2DoubleHeadsModel", + "LukeForMaskedLM", + "LukeForEntityClassification", + "LukeForEntityPairClassification", + "LukeForEntitySpanClassification", + "OpenAIGPTDoubleHeadsModel", + "RagModel", + "RagSequenceForGeneration", + "RagTokenForGeneration", + "RealmEmbedder", + "RealmForOpenQA", + "RealmScorer", + "RealmReader", + "TFDPRReader", + "TFGPT2DoubleHeadsModel", + "TFOpenAIGPTDoubleHeadsModel", + "TFRagModel", + "TFRagSequenceForGeneration", + "TFRagTokenForGeneration", + "Wav2Vec2ForCTC", + "HubertForCTC", + "SEWForCTC", + "SEWDForCTC", + "XLMForQuestionAnswering", + "XLNetForQuestionAnswering", + "SeparableConv1D", + "VisualBertForRegionToPhraseAlignment", + "VisualBertForVisualReasoning", + "VisualBertForQuestionAnswering", + "VisualBertForMultipleChoice", + "TFWav2Vec2ForCTC", + "TFHubertForCTC", + "MaskFormerForInstanceSegmentation", +] + +# Update this list for models that have multiple model types for the same +# model doc +MODEL_TYPE_TO_DOC_MAPPING = OrderedDict( + [ + ("data2vec-text", "data2vec"), + ("data2vec-audio", "data2vec"), + ("data2vec-vision", "data2vec"), + ] +) + + +# This is to make sure the transformers module imported is the one in the repo. +spec = importlib.util.spec_from_file_location( + "diffusers", + os.path.join(PATH_TO_DIFFUSERS, "__init__.py"), + submodule_search_locations=[PATH_TO_DIFFUSERS], +) +diffusers = spec.loader.load_module() + + +def check_model_list(): + """Check the model list inside the transformers library.""" + # Get the models from the directory structure of `src/diffusers/models/` + models_dir = os.path.join(PATH_TO_DIFFUSERS, "models") + _models = [] + for model in os.listdir(models_dir): + model_dir = os.path.join(models_dir, model) + if os.path.isdir(model_dir) and "__init__.py" in os.listdir(model_dir): + _models.append(model) + + # Get the models from the directory structure of `src/transformers/models/` + models = [model for model in dir(diffusers.models) if not model.startswith("__")] + + missing_models = sorted(set(_models).difference(models)) + if missing_models: + raise Exception( + f"The following models should be included in {models_dir}/__init__.py: {','.join(missing_models)}." + ) + + +# If some modeling modules should be ignored for all checks, they should be added in the nested list +# _ignore_modules of this function. +def get_model_modules(): + """Get the model modules inside the transformers library.""" + _ignore_modules = [ + "modeling_auto", + "modeling_encoder_decoder", + "modeling_marian", + "modeling_mmbt", + "modeling_outputs", + "modeling_retribert", + "modeling_utils", + "modeling_flax_auto", + "modeling_flax_encoder_decoder", + "modeling_flax_utils", + "modeling_speech_encoder_decoder", + "modeling_flax_speech_encoder_decoder", + "modeling_flax_vision_encoder_decoder", + "modeling_transfo_xl_utilities", + "modeling_tf_auto", + "modeling_tf_encoder_decoder", + "modeling_tf_outputs", + "modeling_tf_pytorch_utils", + "modeling_tf_utils", + "modeling_tf_transfo_xl_utilities", + "modeling_tf_vision_encoder_decoder", + "modeling_vision_encoder_decoder", + ] + modules = [] + for model in dir(diffusers.models): + # There are some magic dunder attributes in the dir, we ignore them + if not model.startswith("__"): + model_module = getattr(diffusers.models, model) + for submodule in dir(model_module): + if submodule.startswith("modeling") and submodule not in _ignore_modules: + modeling_module = getattr(model_module, submodule) + if inspect.ismodule(modeling_module): + modules.append(modeling_module) + return modules + + +def get_models(module, include_pretrained=False): + """Get the objects in module that are models.""" + models = [] + model_classes = (diffusers.ModelMixin, diffusers.TFModelMixin, diffusers.FlaxModelMixin) + for attr_name in dir(module): + if not include_pretrained and ("Pretrained" in attr_name or "PreTrained" in attr_name): + continue + attr = getattr(module, attr_name) + if isinstance(attr, type) and issubclass(attr, model_classes) and attr.__module__ == module.__name__: + models.append((attr_name, attr)) + return models + + +def is_a_private_model(model): + """Returns True if the model should not be in the main init.""" + if model in PRIVATE_MODELS: + return True + + # Wrapper, Encoder and Decoder are all privates + if model.endswith("Wrapper"): + return True + if model.endswith("Encoder"): + return True + if model.endswith("Decoder"): + return True + return False + + +def check_models_are_in_init(): + """Checks all models defined in the library are in the main init.""" + models_not_in_init = [] + dir_transformers = dir(diffusers) + for module in get_model_modules(): + models_not_in_init += [ + model[0] for model in get_models(module, include_pretrained=True) if model[0] not in dir_transformers + ] + + # Remove private models + models_not_in_init = [model for model in models_not_in_init if not is_a_private_model(model)] + if len(models_not_in_init) > 0: + raise Exception(f"The following models should be in the main init: {','.join(models_not_in_init)}.") + + +# If some test_modeling files should be ignored when checking models are all tested, they should be added in the +# nested list _ignore_files of this function. +def get_model_test_files(): + """Get the model test files. + + The returned files should NOT contain the `tests` (i.e. `PATH_TO_TESTS` defined in this script). They will be + considered as paths relative to `tests`. A caller has to use `os.path.join(PATH_TO_TESTS, ...)` to access the files. + """ + + _ignore_files = [ + "test_modeling_common", + "test_modeling_encoder_decoder", + "test_modeling_flax_encoder_decoder", + "test_modeling_flax_speech_encoder_decoder", + "test_modeling_marian", + "test_modeling_tf_common", + "test_modeling_tf_encoder_decoder", + ] + test_files = [] + # Check both `PATH_TO_TESTS` and `PATH_TO_TESTS/models` + model_test_root = os.path.join(PATH_TO_TESTS, "models") + model_test_dirs = [] + for x in os.listdir(model_test_root): + x = os.path.join(model_test_root, x) + if os.path.isdir(x): + model_test_dirs.append(x) + + for target_dir in [PATH_TO_TESTS] + model_test_dirs: + for file_or_dir in os.listdir(target_dir): + path = os.path.join(target_dir, file_or_dir) + if os.path.isfile(path): + filename = os.path.split(path)[-1] + if "test_modeling" in filename and os.path.splitext(filename)[0] not in _ignore_files: + file = os.path.join(*path.split(os.sep)[1:]) + test_files.append(file) + + return test_files + + +# This is a bit hacky but I didn't find a way to import the test_file as a module and read inside the tester class +# for the all_model_classes variable. +def find_tested_models(test_file): + """Parse the content of test_file to detect what's in all_model_classes""" + # This is a bit hacky but I didn't find a way to import the test_file as a module and read inside the class + with open(os.path.join(PATH_TO_TESTS, test_file), "r", encoding="utf-8", newline="\n") as f: + content = f.read() + all_models = re.findall(r"all_model_classes\s+=\s+\(\s*\(([^\)]*)\)", content) + # Check with one less parenthesis as well + all_models += re.findall(r"all_model_classes\s+=\s+\(([^\)]*)\)", content) + if len(all_models) > 0: + model_tested = [] + for entry in all_models: + for line in entry.split(","): + name = line.strip() + if len(name) > 0: + model_tested.append(name) + return model_tested + + +def check_models_are_tested(module, test_file): + """Check models defined in module are tested in test_file.""" + # XxxModelMixin are not tested + defined_models = get_models(module) + tested_models = find_tested_models(test_file) + if tested_models is None: + if test_file.replace(os.path.sep, "/") in TEST_FILES_WITH_NO_COMMON_TESTS: + return + return [ + f"{test_file} should define `all_model_classes` to apply common tests to the models it tests. " + + "If this intentional, add the test filename to `TEST_FILES_WITH_NO_COMMON_TESTS` in the file " + + "`utils/check_repo.py`." + ] + failures = [] + for model_name, _ in defined_models: + if model_name not in tested_models and model_name not in IGNORE_NON_TESTED: + failures.append( + f"{model_name} is defined in {module.__name__} but is not tested in " + + f"{os.path.join(PATH_TO_TESTS, test_file)}. Add it to the all_model_classes in that file." + + "If common tests should not applied to that model, add its name to `IGNORE_NON_TESTED`" + + "in the file `utils/check_repo.py`." + ) + return failures + + +def check_all_models_are_tested(): + """Check all models are properly tested.""" + modules = get_model_modules() + test_files = get_model_test_files() + failures = [] + for module in modules: + test_file = [file for file in test_files if f"test_{module.__name__.split('.')[-1]}.py" in file] + if len(test_file) == 0: + failures.append(f"{module.__name__} does not have its corresponding test file {test_file}.") + elif len(test_file) > 1: + failures.append(f"{module.__name__} has several test files: {test_file}.") + else: + test_file = test_file[0] + new_failures = check_models_are_tested(module, test_file) + if new_failures is not None: + failures += new_failures + if len(failures) > 0: + raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures)) + + +def get_all_auto_configured_models(): + """Return the list of all models in at least one auto class.""" + result = set() # To avoid duplicates we concatenate all model classes in a set. + if is_torch_available(): + for attr_name in dir(diffusers.models.auto.modeling_auto): + if attr_name.startswith("MODEL_") and attr_name.endswith("MAPPING_NAMES"): + result = result | set(get_values(getattr(diffusers.models.auto.modeling_auto, attr_name))) + if is_flax_available(): + for attr_name in dir(diffusers.models.auto.modeling_flax_auto): + if attr_name.startswith("FLAX_MODEL_") and attr_name.endswith("MAPPING_NAMES"): + result = result | set(get_values(getattr(diffusers.models.auto.modeling_flax_auto, attr_name))) + return list(result) + + +def ignore_unautoclassed(model_name): + """Rules to determine if `name` should be in an auto class.""" + # Special white list + if model_name in IGNORE_NON_AUTO_CONFIGURED: + return True + # Encoder and Decoder should be ignored + if "Encoder" in model_name or "Decoder" in model_name: + return True + return False + + +def check_models_are_auto_configured(module, all_auto_models): + """Check models defined in module are each in an auto class.""" + defined_models = get_models(module) + failures = [] + for model_name, _ in defined_models: + if model_name not in all_auto_models and not ignore_unautoclassed(model_name): + failures.append( + f"{model_name} is defined in {module.__name__} but is not present in any of the auto mapping. " + "If that is intended behavior, add its name to `IGNORE_NON_AUTO_CONFIGURED` in the file " + "`utils/check_repo.py`." + ) + return failures + + +def check_all_models_are_auto_configured(): + """Check all models are each in an auto class.""" + missing_backends = [] + if not is_torch_available(): + missing_backends.append("PyTorch") + if not is_flax_available(): + missing_backends.append("Flax") + if len(missing_backends) > 0: + missing = ", ".join(missing_backends) + if os.getenv("TRANSFORMERS_IS_CI", "").upper() in ENV_VARS_TRUE_VALUES: + raise Exception( + "Full quality checks require all backends to be installed (with `pip install -e .[dev]` in the " + f"Transformers repo, the following are missing: {missing}." + ) + else: + warnings.warn( + "Full quality checks require all backends to be installed (with `pip install -e .[dev]` in the " + f"Transformers repo, the following are missing: {missing}. While it's probably fine as long as you " + "didn't make any change in one of those backends modeling files, you should probably execute the " + "command above to be on the safe side." + ) + modules = get_model_modules() + all_auto_models = get_all_auto_configured_models() + failures = [] + for module in modules: + new_failures = check_models_are_auto_configured(module, all_auto_models) + if new_failures is not None: + failures += new_failures + if len(failures) > 0: + raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures)) + + +_re_decorator = re.compile(r"^\s*@(\S+)\s+$") + + +def check_decorator_order(filename): + """Check that in the test file `filename` the slow decorator is always last.""" + with open(filename, "r", encoding="utf-8", newline="\n") as f: + lines = f.readlines() + decorator_before = None + errors = [] + for i, line in enumerate(lines): + search = _re_decorator.search(line) + if search is not None: + decorator_name = search.groups()[0] + if decorator_before is not None and decorator_name.startswith("parameterized"): + errors.append(i) + decorator_before = decorator_name + elif decorator_before is not None: + decorator_before = None + return errors + + +def check_all_decorator_order(): + """Check that in all test files, the slow decorator is always last.""" + errors = [] + for fname in os.listdir(PATH_TO_TESTS): + if fname.endswith(".py"): + filename = os.path.join(PATH_TO_TESTS, fname) + new_errors = check_decorator_order(filename) + errors += [f"- {filename}, line {i}" for i in new_errors] + if len(errors) > 0: + msg = "\n".join(errors) + raise ValueError( + "The parameterized decorator (and its variants) should always be first, but this is not the case in the" + f" following files:\n{msg}" + ) + + +def find_all_documented_objects(): + """Parse the content of all doc files to detect which classes and functions it documents""" + documented_obj = [] + for doc_file in Path(PATH_TO_DOC).glob("**/*.rst"): + with open(doc_file, "r", encoding="utf-8", newline="\n") as f: + content = f.read() + raw_doc_objs = re.findall(r"(?:autoclass|autofunction):: transformers.(\S+)\s+", content) + documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs] + for doc_file in Path(PATH_TO_DOC).glob("**/*.md"): + with open(doc_file, "r", encoding="utf-8", newline="\n") as f: + content = f.read() + raw_doc_objs = re.findall(r"\[\[autodoc\]\]\s+(\S+)\s+", content) + documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs] + return documented_obj + + +# One good reason for not being documented is to be deprecated. Put in this list deprecated objects. +DEPRECATED_OBJECTS = [ + "AutoModelWithLMHead", + "BartPretrainedModel", + "DataCollator", + "DataCollatorForSOP", + "GlueDataset", + "GlueDataTrainingArguments", + "LineByLineTextDataset", + "LineByLineWithRefDataset", + "LineByLineWithSOPTextDataset", + "PretrainedBartModel", + "PretrainedFSMTModel", + "SingleSentenceClassificationProcessor", + "SquadDataTrainingArguments", + "SquadDataset", + "SquadExample", + "SquadFeatures", + "SquadV1Processor", + "SquadV2Processor", + "TFAutoModelWithLMHead", + "TFBartPretrainedModel", + "TextDataset", + "TextDatasetForNextSentencePrediction", + "Wav2Vec2ForMaskedLM", + "Wav2Vec2Tokenizer", + "glue_compute_metrics", + "glue_convert_examples_to_features", + "glue_output_modes", + "glue_processors", + "glue_tasks_num_labels", + "squad_convert_examples_to_features", + "xnli_compute_metrics", + "xnli_output_modes", + "xnli_processors", + "xnli_tasks_num_labels", + "TFTrainer", + "TFTrainingArguments", +] + +# Exceptionally, some objects should not be documented after all rules passed. +# ONLY PUT SOMETHING IN THIS LIST AS A LAST RESORT! +UNDOCUMENTED_OBJECTS = [ + "AddedToken", # This is a tokenizers class. + "BasicTokenizer", # Internal, should never have been in the main init. + "CharacterTokenizer", # Internal, should never have been in the main init. + "DPRPretrainedReader", # Like an Encoder. + "DummyObject", # Just picked by mistake sometimes. + "MecabTokenizer", # Internal, should never have been in the main init. + "ModelCard", # Internal type. + "SqueezeBertModule", # Internal building block (should have been called SqueezeBertLayer) + "TFDPRPretrainedReader", # Like an Encoder. + "TransfoXLCorpus", # Internal type. + "WordpieceTokenizer", # Internal, should never have been in the main init. + "absl", # External module + "add_end_docstrings", # Internal, should never have been in the main init. + "add_start_docstrings", # Internal, should never have been in the main init. + "cached_path", # Internal used for downloading models. + "convert_tf_weight_name_to_pt_weight_name", # Internal used to convert model weights + "logger", # Internal logger + "logging", # External module + "requires_backends", # Internal function +] + +# This list should be empty. Objects in it should get their own doc page. +SHOULD_HAVE_THEIR_OWN_PAGE = [ + # Benchmarks + "PyTorchBenchmark", + "PyTorchBenchmarkArguments", + "TensorFlowBenchmark", + "TensorFlowBenchmarkArguments", +] + + +def ignore_undocumented(name): + """Rules to determine if `name` should be undocumented.""" + # NOT DOCUMENTED ON PURPOSE. + # Constants uppercase are not documented. + if name.isupper(): + return True + # ModelMixins / Encoders / Decoders / Layers / Embeddings / Attention are not documented. + if ( + name.endswith("ModelMixin") + or name.endswith("Decoder") + or name.endswith("Encoder") + or name.endswith("Layer") + or name.endswith("Embeddings") + or name.endswith("Attention") + ): + return True + # Submodules are not documented. + if os.path.isdir(os.path.join(PATH_TO_DIFFUSERS, name)) or os.path.isfile( + os.path.join(PATH_TO_DIFFUSERS, f"{name}.py") + ): + return True + # All load functions are not documented. + if name.startswith("load_tf") or name.startswith("load_pytorch"): + return True + # is_xxx_available functions are not documented. + if name.startswith("is_") and name.endswith("_available"): + return True + # Deprecated objects are not documented. + if name in DEPRECATED_OBJECTS or name in UNDOCUMENTED_OBJECTS: + return True + # MMBT model does not really work. + if name.startswith("MMBT"): + return True + if name in SHOULD_HAVE_THEIR_OWN_PAGE: + return True + return False + + +def check_all_objects_are_documented(): + """Check all models are properly documented.""" + documented_objs = find_all_documented_objects() + modules = diffusers._modules + objects = [c for c in dir(diffusers) if c not in modules and not c.startswith("_")] + undocumented_objs = [c for c in objects if c not in documented_objs and not ignore_undocumented(c)] + if len(undocumented_objs) > 0: + raise Exception( + "The following objects are in the public init so should be documented:\n - " + + "\n - ".join(undocumented_objs) + ) + check_docstrings_are_in_md() + check_model_type_doc_match() + + +def check_model_type_doc_match(): + """Check all doc pages have a corresponding model type.""" + model_doc_folder = Path(PATH_TO_DOC) / "model_doc" + model_docs = [m.stem for m in model_doc_folder.glob("*.md")] + + model_types = list(diffusers.models.auto.configuration_auto.MODEL_NAMES_MAPPING.keys()) + model_types = [MODEL_TYPE_TO_DOC_MAPPING[m] if m in MODEL_TYPE_TO_DOC_MAPPING else m for m in model_types] + + errors = [] + for m in model_docs: + if m not in model_types and m != "auto": + close_matches = get_close_matches(m, model_types) + error_message = f"{m} is not a proper model identifier." + if len(close_matches) > 0: + close_matches = "/".join(close_matches) + error_message += f" Did you mean {close_matches}?" + errors.append(error_message) + + if len(errors) > 0: + raise ValueError( + "Some model doc pages do not match any existing model type:\n" + + "\n".join(errors) + + "\nYou can add any missing model type to the `MODEL_NAMES_MAPPING` constant in " + "models/auto/configuration_auto.py." + ) + + +# Re pattern to catch :obj:`xx`, :class:`xx`, :func:`xx` or :meth:`xx`. +_re_rst_special_words = re.compile(r":(?:obj|func|class|meth):`([^`]+)`") +# Re pattern to catch things between double backquotes. +_re_double_backquotes = re.compile(r"(^|[^`])``([^`]+)``([^`]|$)") +# Re pattern to catch example introduction. +_re_rst_example = re.compile(r"^\s*Example.*::\s*$", flags=re.MULTILINE) + + +def is_rst_docstring(docstring): + """ + Returns `True` if `docstring` is written in rst. + """ + if _re_rst_special_words.search(docstring) is not None: + return True + if _re_double_backquotes.search(docstring) is not None: + return True + if _re_rst_example.search(docstring) is not None: + return True + return False + + +def check_docstrings_are_in_md(): + """Check all docstrings are in md""" + files_with_rst = [] + for file in Path(PATH_TO_DIFFUSERS).glob("**/*.py"): + with open(file, "r") as f: + code = f.read() + docstrings = code.split('"""') + + for idx, docstring in enumerate(docstrings): + if idx % 2 == 0 or not is_rst_docstring(docstring): + continue + files_with_rst.append(file) + break + + if len(files_with_rst) > 0: + raise ValueError( + "The following files have docstrings written in rst:\n" + + "\n".join([f"- {f}" for f in files_with_rst]) + + "\nTo fix this run `doc-builder convert path_to_py_file` after installing `doc-builder`\n" + "(`pip install git+https://github.com/huggingface/doc-builder`)" + ) + + +def check_repo_quality(): + """Check all models are properly tested and documented.""" + print("Checking all models are included.") + check_model_list() + print("Checking all models are public.") + check_models_are_in_init() + print("Checking all models are properly tested.") + check_all_decorator_order() + check_all_models_are_tested() + print("Checking all objects are properly documented.") + check_all_objects_are_documented() + print("Checking all models are in at least one auto class.") + check_all_models_are_auto_configured() + + +if __name__ == "__main__": + check_repo_quality() diff --git a/diffusers-0.27.0/utils/check_table.py b/diffusers-0.27.0/utils/check_table.py new file mode 100755 index 0000000..aa7554c --- /dev/null +++ b/diffusers-0.27.0/utils/check_table.py @@ -0,0 +1,185 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import collections +import importlib.util +import os +import re + + +# All paths are set with the intent you should run this script from the root of the repo with the command +# python utils/check_table.py +TRANSFORMERS_PATH = "src/diffusers" +PATH_TO_DOCS = "docs/source/en" +REPO_PATH = "." + + +def _find_text_in_file(filename, start_prompt, end_prompt): + """ + Find the text in `filename` between a line beginning with `start_prompt` and before `end_prompt`, removing empty + lines. + """ + with open(filename, "r", encoding="utf-8", newline="\n") as f: + lines = f.readlines() + # Find the start prompt. + start_index = 0 + while not lines[start_index].startswith(start_prompt): + start_index += 1 + start_index += 1 + + end_index = start_index + while not lines[end_index].startswith(end_prompt): + end_index += 1 + end_index -= 1 + + while len(lines[start_index]) <= 1: + start_index += 1 + while len(lines[end_index]) <= 1: + end_index -= 1 + end_index += 1 + return "".join(lines[start_index:end_index]), start_index, end_index, lines + + +# Add here suffixes that are used to identify models, separated by | +ALLOWED_MODEL_SUFFIXES = "Model|Encoder|Decoder|ForConditionalGeneration" +# Regexes that match TF/Flax/PT model names. +_re_tf_models = re.compile(r"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)") +_re_flax_models = re.compile(r"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)") +# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes. +_re_pt_models = re.compile(r"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)") + + +# This is to make sure the diffusers module imported is the one in the repo. +spec = importlib.util.spec_from_file_location( + "diffusers", + os.path.join(TRANSFORMERS_PATH, "__init__.py"), + submodule_search_locations=[TRANSFORMERS_PATH], +) +diffusers_module = spec.loader.load_module() + + +# Thanks to https://stackoverflow.com/questions/29916065/how-to-do-camelcase-split-in-python +def camel_case_split(identifier): + "Split a camelcased `identifier` into words." + matches = re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)", identifier) + return [m.group(0) for m in matches] + + +def _center_text(text, width): + text_length = 2 if text == "✅" or text == "❌" else len(text) + left_indent = (width - text_length) // 2 + right_indent = width - text_length - left_indent + return " " * left_indent + text + " " * right_indent + + +def get_model_table_from_auto_modules(): + """Generates an up-to-date model table from the content of the auto modules.""" + # Dictionary model names to config. + config_mapping_names = diffusers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES + model_name_to_config = { + name: config_mapping_names[code] + for code, name in diffusers_module.MODEL_NAMES_MAPPING.items() + if code in config_mapping_names + } + model_name_to_prefix = {name: config.replace("ConfigMixin", "") for name, config in model_name_to_config.items()} + + # Dictionaries flagging if each model prefix has a slow/fast tokenizer, backend in PT/TF/Flax. + slow_tokenizers = collections.defaultdict(bool) + fast_tokenizers = collections.defaultdict(bool) + pt_models = collections.defaultdict(bool) + tf_models = collections.defaultdict(bool) + flax_models = collections.defaultdict(bool) + + # Let's lookup through all diffusers object (once). + for attr_name in dir(diffusers_module): + lookup_dict = None + if attr_name.endswith("Tokenizer"): + lookup_dict = slow_tokenizers + attr_name = attr_name[:-9] + elif attr_name.endswith("TokenizerFast"): + lookup_dict = fast_tokenizers + attr_name = attr_name[:-13] + elif _re_tf_models.match(attr_name) is not None: + lookup_dict = tf_models + attr_name = _re_tf_models.match(attr_name).groups()[0] + elif _re_flax_models.match(attr_name) is not None: + lookup_dict = flax_models + attr_name = _re_flax_models.match(attr_name).groups()[0] + elif _re_pt_models.match(attr_name) is not None: + lookup_dict = pt_models + attr_name = _re_pt_models.match(attr_name).groups()[0] + + if lookup_dict is not None: + while len(attr_name) > 0: + if attr_name in model_name_to_prefix.values(): + lookup_dict[attr_name] = True + break + # Try again after removing the last word in the name + attr_name = "".join(camel_case_split(attr_name)[:-1]) + + # Let's build that table! + model_names = list(model_name_to_config.keys()) + model_names.sort(key=str.lower) + columns = ["Model", "Tokenizer slow", "Tokenizer fast", "PyTorch support", "TensorFlow support", "Flax Support"] + # We'll need widths to properly display everything in the center (+2 is to leave one extra space on each side). + widths = [len(c) + 2 for c in columns] + widths[0] = max([len(name) for name in model_names]) + 2 + + # Build the table per se + table = "|" + "|".join([_center_text(c, w) for c, w in zip(columns, widths)]) + "|\n" + # Use ":-----:" format to center-aligned table cell texts + table += "|" + "|".join([":" + "-" * (w - 2) + ":" for w in widths]) + "|\n" + + check = {True: "✅", False: "❌"} + for name in model_names: + prefix = model_name_to_prefix[name] + line = [ + name, + check[slow_tokenizers[prefix]], + check[fast_tokenizers[prefix]], + check[pt_models[prefix]], + check[tf_models[prefix]], + check[flax_models[prefix]], + ] + table += "|" + "|".join([_center_text(l, w) for l, w in zip(line, widths)]) + "|\n" + return table + + +def check_model_table(overwrite=False): + """Check the model table in the index.rst is consistent with the state of the lib and maybe `overwrite`.""" + current_table, start_index, end_index, lines = _find_text_in_file( + filename=os.path.join(PATH_TO_DOCS, "index.md"), + start_prompt="", + ) + new_table = get_model_table_from_auto_modules() + + if current_table != new_table: + if overwrite: + with open(os.path.join(PATH_TO_DOCS, "index.md"), "w", encoding="utf-8", newline="\n") as f: + f.writelines(lines[:start_index] + [new_table] + lines[end_index:]) + else: + raise ValueError( + "The model table in the `index.md` has not been updated. Run `make fix-copies` to fix this." + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.") + args = parser.parse_args() + + check_model_table(args.fix_and_overwrite) diff --git a/diffusers-0.27.0/utils/custom_init_isort.py b/diffusers-0.27.0/utils/custom_init_isort.py new file mode 100755 index 0000000..acba69a --- /dev/null +++ b/diffusers-0.27.0/utils/custom_init_isort.py @@ -0,0 +1,329 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Utility that sorts the imports in the custom inits of Diffusers. Diffusers uses init files that delay the +import of an object to when it's actually needed. This is to avoid the main init importing all models, which would +make the line `import transformers` very slow when the user has all optional dependencies installed. The inits with +delayed imports have two halves: one defining a dictionary `_import_structure` which maps modules to the name of the +objects in each module, and one in `TYPE_CHECKING` which looks like a normal init for type-checkers. `isort` or `ruff` +properly sort the second half which looks like traditionl imports, the goal of this script is to sort the first half. + +Use from the root of the repo with: + +```bash +python utils/custom_init_isort.py +``` + +which will auto-sort the imports (used in `make style`). + +For a check only (as used in `make quality`) run: + +```bash +python utils/custom_init_isort.py --check_only +``` +""" +import argparse +import os +import re +from typing import Any, Callable, List, Optional + + +# Path is defined with the intent you should run this script from the root of the repo. +PATH_TO_TRANSFORMERS = "src/diffusers" + +# Pattern that looks at the indentation in a line. +_re_indent = re.compile(r"^(\s*)\S") +# Pattern that matches `"key":" and puts `key` in group 0. +_re_direct_key = re.compile(r'^\s*"([^"]+)":') +# Pattern that matches `_import_structure["key"]` and puts `key` in group 0. +_re_indirect_key = re.compile(r'^\s*_import_structure\["([^"]+)"\]') +# Pattern that matches `"key",` and puts `key` in group 0. +_re_strip_line = re.compile(r'^\s*"([^"]+)",\s*$') +# Pattern that matches any `[stuff]` and puts `stuff` in group 0. +_re_bracket_content = re.compile(r"\[([^\]]+)\]") + + +def get_indent(line: str) -> str: + """Returns the indent in given line (as string).""" + search = _re_indent.search(line) + return "" if search is None else search.groups()[0] + + +def split_code_in_indented_blocks( + code: str, indent_level: str = "", start_prompt: Optional[str] = None, end_prompt: Optional[str] = None +) -> List[str]: + """ + Split some code into its indented blocks, starting at a given level. + + Args: + code (`str`): The code to split. + indent_level (`str`): The indent level (as string) to use for identifying the blocks to split. + start_prompt (`str`, *optional*): If provided, only starts splitting at the line where this text is. + end_prompt (`str`, *optional*): If provided, stops splitting at a line where this text is. + + Warning: + The text before `start_prompt` or after `end_prompt` (if provided) is not ignored, just not split. The input `code` + can thus be retrieved by joining the result. + + Returns: + `List[str]`: The list of blocks. + """ + # Let's split the code into lines and move to start_index. + index = 0 + lines = code.split("\n") + if start_prompt is not None: + while not lines[index].startswith(start_prompt): + index += 1 + blocks = ["\n".join(lines[:index])] + else: + blocks = [] + + # This variable contains the block treated at a given time. + current_block = [lines[index]] + index += 1 + # We split into blocks until we get to the `end_prompt` (or the end of the file). + while index < len(lines) and (end_prompt is None or not lines[index].startswith(end_prompt)): + # We have a non-empty line with the proper indent -> start of a new block + if len(lines[index]) > 0 and get_indent(lines[index]) == indent_level: + # Store the current block in the result and rest. There are two cases: the line is part of the block (like + # a closing parenthesis) or not. + if len(current_block) > 0 and get_indent(current_block[-1]).startswith(indent_level + " "): + # Line is part of the current block + current_block.append(lines[index]) + blocks.append("\n".join(current_block)) + if index < len(lines) - 1: + current_block = [lines[index + 1]] + index += 1 + else: + current_block = [] + else: + # Line is not part of the current block + blocks.append("\n".join(current_block)) + current_block = [lines[index]] + else: + # Just add the line to the current block + current_block.append(lines[index]) + index += 1 + + # Adds current block if it's nonempty. + if len(current_block) > 0: + blocks.append("\n".join(current_block)) + + # Add final block after end_prompt if provided. + if end_prompt is not None and index < len(lines): + blocks.append("\n".join(lines[index:])) + + return blocks + + +def ignore_underscore_and_lowercase(key: Callable[[Any], str]) -> Callable[[Any], str]: + """ + Wraps a key function (as used in a sort) to lowercase and ignore underscores. + """ + + def _inner(x): + return key(x).lower().replace("_", "") + + return _inner + + +def sort_objects(objects: List[Any], key: Optional[Callable[[Any], str]] = None) -> List[Any]: + """ + Sort a list of objects following the rules of isort (all uppercased first, camel-cased second and lower-cased + last). + + Args: + objects (`List[Any]`): + The list of objects to sort. + key (`Callable[[Any], str]`, *optional*): + A function taking an object as input and returning a string, used to sort them by alphabetical order. + If not provided, will default to noop (so a `key` must be provided if the `objects` are not of type string). + + Returns: + `List[Any]`: The sorted list with the same elements as in the inputs + """ + + # If no key is provided, we use a noop. + def noop(x): + return x + + if key is None: + key = noop + # Constants are all uppercase, they go first. + constants = [obj for obj in objects if key(obj).isupper()] + # Classes are not all uppercase but start with a capital, they go second. + classes = [obj for obj in objects if key(obj)[0].isupper() and not key(obj).isupper()] + # Functions begin with a lowercase, they go last. + functions = [obj for obj in objects if not key(obj)[0].isupper()] + + # Then we sort each group. + key1 = ignore_underscore_and_lowercase(key) + return sorted(constants, key=key1) + sorted(classes, key=key1) + sorted(functions, key=key1) + + +def sort_objects_in_import(import_statement: str) -> str: + """ + Sorts the imports in a single import statement. + + Args: + import_statement (`str`): The import statement in which to sort the imports. + + Returns: + `str`: The same as the input, but with objects properly sorted. + """ + + # This inner function sort imports between [ ]. + def _replace(match): + imports = match.groups()[0] + # If there is one import only, nothing to do. + if "," not in imports: + return f"[{imports}]" + keys = [part.strip().replace('"', "") for part in imports.split(",")] + # We will have a final empty element if the line finished with a comma. + if len(keys[-1]) == 0: + keys = keys[:-1] + return "[" + ", ".join([f'"{k}"' for k in sort_objects(keys)]) + "]" + + lines = import_statement.split("\n") + if len(lines) > 3: + # Here we have to sort internal imports that are on several lines (one per name): + # key: [ + # "object1", + # "object2", + # ... + # ] + + # We may have to ignore one or two lines on each side. + idx = 2 if lines[1].strip() == "[" else 1 + keys_to_sort = [(i, _re_strip_line.search(line).groups()[0]) for i, line in enumerate(lines[idx:-idx])] + sorted_indices = sort_objects(keys_to_sort, key=lambda x: x[1]) + sorted_lines = [lines[x[0] + idx] for x in sorted_indices] + return "\n".join(lines[:idx] + sorted_lines + lines[-idx:]) + elif len(lines) == 3: + # Here we have to sort internal imports that are on one separate line: + # key: [ + # "object1", "object2", ... + # ] + if _re_bracket_content.search(lines[1]) is not None: + lines[1] = _re_bracket_content.sub(_replace, lines[1]) + else: + keys = [part.strip().replace('"', "") for part in lines[1].split(",")] + # We will have a final empty element if the line finished with a comma. + if len(keys[-1]) == 0: + keys = keys[:-1] + lines[1] = get_indent(lines[1]) + ", ".join([f'"{k}"' for k in sort_objects(keys)]) + return "\n".join(lines) + else: + # Finally we have to deal with imports fitting on one line + import_statement = _re_bracket_content.sub(_replace, import_statement) + return import_statement + + +def sort_imports(file: str, check_only: bool = True): + """ + Sort the imports defined in the `_import_structure` of a given init. + + Args: + file (`str`): The path to the init to check/fix. + check_only (`bool`, *optional*, defaults to `True`): Whether or not to just check (and not auto-fix) the init. + """ + with open(file, encoding="utf-8") as f: + code = f.read() + + # If the file is not a custom init, there is nothing to do. + if "_import_structure" not in code: + return + + # Blocks of indent level 0 + main_blocks = split_code_in_indented_blocks( + code, start_prompt="_import_structure = {", end_prompt="if TYPE_CHECKING:" + ) + + # We ignore block 0 (everything untils start_prompt) and the last block (everything after end_prompt). + for block_idx in range(1, len(main_blocks) - 1): + # Check if the block contains some `_import_structure`s thingy to sort. + block = main_blocks[block_idx] + block_lines = block.split("\n") + + # Get to the start of the imports. + line_idx = 0 + while line_idx < len(block_lines) and "_import_structure" not in block_lines[line_idx]: + # Skip dummy import blocks + if "import dummy" in block_lines[line_idx]: + line_idx = len(block_lines) + else: + line_idx += 1 + if line_idx >= len(block_lines): + continue + + # Ignore beginning and last line: they don't contain anything. + internal_block_code = "\n".join(block_lines[line_idx:-1]) + indent = get_indent(block_lines[1]) + # Slit the internal block into blocks of indent level 1. + internal_blocks = split_code_in_indented_blocks(internal_block_code, indent_level=indent) + # We have two categories of import key: list or _import_structure[key].append/extend + pattern = _re_direct_key if "_import_structure = {" in block_lines[0] else _re_indirect_key + # Grab the keys, but there is a trap: some lines are empty or just comments. + keys = [(pattern.search(b).groups()[0] if pattern.search(b) is not None else None) for b in internal_blocks] + # We only sort the lines with a key. + keys_to_sort = [(i, key) for i, key in enumerate(keys) if key is not None] + sorted_indices = [x[0] for x in sorted(keys_to_sort, key=lambda x: x[1])] + + # We reorder the blocks by leaving empty lines/comments as they were and reorder the rest. + count = 0 + reordered_blocks = [] + for i in range(len(internal_blocks)): + if keys[i] is None: + reordered_blocks.append(internal_blocks[i]) + else: + block = sort_objects_in_import(internal_blocks[sorted_indices[count]]) + reordered_blocks.append(block) + count += 1 + + # And we put our main block back together with its first and last line. + main_blocks[block_idx] = "\n".join(block_lines[:line_idx] + reordered_blocks + [block_lines[-1]]) + + if code != "\n".join(main_blocks): + if check_only: + return True + else: + print(f"Overwriting {file}.") + with open(file, "w", encoding="utf-8") as f: + f.write("\n".join(main_blocks)) + + +def sort_imports_in_all_inits(check_only=True): + """ + Sort the imports defined in the `_import_structure` of all inits in the repo. + + Args: + check_only (`bool`, *optional*, defaults to `True`): Whether or not to just check (and not auto-fix) the init. + """ + failures = [] + for root, _, files in os.walk(PATH_TO_TRANSFORMERS): + if "__init__.py" in files: + result = sort_imports(os.path.join(root, "__init__.py"), check_only=check_only) + if result: + failures = [os.path.join(root, "__init__.py")] + if len(failures) > 0: + raise ValueError(f"Would overwrite {len(failures)} files, run `make style`.") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--check_only", action="store_true", help="Whether to only check or fix style.") + args = parser.parse_args() + + sort_imports_in_all_inits(check_only=args.check_only) diff --git a/diffusers-0.27.0/utils/fetch_latest_release_branch.py b/diffusers-0.27.0/utils/fetch_latest_release_branch.py new file mode 100755 index 0000000..9bf578a --- /dev/null +++ b/diffusers-0.27.0/utils/fetch_latest_release_branch.py @@ -0,0 +1,68 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import requests +from packaging.version import parse + + +# GitHub repository details +USER = "huggingface" +REPO = "diffusers" + + +def fetch_all_branches(user, repo): + branches = [] # List to store all branches + page = 1 # Start from first page + while True: + # Make a request to the GitHub API for the branches + response = requests.get(f"https://api.github.com/repos/{user}/{repo}/branches", params={"page": page}) + + # Check if the request was successful + if response.status_code == 200: + # Add the branches from the current page to the list + branches.extend([branch["name"] for branch in response.json()]) + + # Check if there is a 'next' link for pagination + if "next" in response.links: + page += 1 # Move to the next page + else: + break # Exit loop if there is no next page + else: + print("Failed to retrieve branches:", response.status_code) + break + + return branches + + +def main(): + # Fetch all branches + branches = fetch_all_branches(USER, REPO) + + # Filter branches. + # print(f"Total branches: {len(branches)}") + filtered_branches = [] + for branch in branches: + if branch.startswith("v") and ("-release" in branch or "-patch" in branch): + filtered_branches.append(branch) + # print(f"Filtered: {branch}") + + sorted_branches = sorted(filtered_branches, key=lambda x: parse(x.split("-")[0][1:]), reverse=True) + latest_branch = sorted_branches[0] + # print(f"Latest branch: {latest_branch}") + return latest_branch + + +if __name__ == "__main__": + print(main()) diff --git a/diffusers-0.27.0/utils/fetch_torch_cuda_pipeline_test_matrix.py b/diffusers-0.27.0/utils/fetch_torch_cuda_pipeline_test_matrix.py new file mode 100755 index 0000000..744201c --- /dev/null +++ b/diffusers-0.27.0/utils/fetch_torch_cuda_pipeline_test_matrix.py @@ -0,0 +1,102 @@ +import json +import logging +import os +from collections import defaultdict +from pathlib import Path + +from huggingface_hub import HfApi, ModelFilter + +import diffusers + + +PATH_TO_REPO = Path(__file__).parent.parent.resolve() +ALWAYS_TEST_PIPELINE_MODULES = [ + "controlnet", + "stable_diffusion", + "stable_diffusion_2", + "stable_diffusion_xl", + "stable_diffusion_adapter", + "deepfloyd_if", + "ip_adapters", + "kandinsky", + "kandinsky2_2", + "text_to_video_synthesis", + "wuerstchen", +] +PIPELINE_USAGE_CUTOFF = int(os.getenv("PIPELINE_USAGE_CUTOFF", 50000)) + +logger = logging.getLogger(__name__) +api = HfApi() +filter = ModelFilter(library="diffusers") + + +def filter_pipelines(usage_dict, usage_cutoff=10000): + output = [] + for diffusers_object, usage in usage_dict.items(): + if usage < usage_cutoff: + continue + + is_diffusers_pipeline = hasattr(diffusers.pipelines, diffusers_object) + if not is_diffusers_pipeline: + continue + + output.append(diffusers_object) + + return output + + +def fetch_pipeline_objects(): + models = api.list_models(filter=filter) + downloads = defaultdict(int) + + for model in models: + is_counted = False + for tag in model.tags: + if tag.startswith("diffusers:"): + is_counted = True + downloads[tag[len("diffusers:") :]] += model.downloads + + if not is_counted: + downloads["other"] += model.downloads + + # Remove 0 downloads + downloads = {k: v for k, v in downloads.items() if v > 0} + pipeline_objects = filter_pipelines(downloads, PIPELINE_USAGE_CUTOFF) + + return pipeline_objects + + +def fetch_pipeline_modules_to_test(): + try: + pipeline_objects = fetch_pipeline_objects() + except Exception as e: + logger.error(e) + raise RuntimeError("Unable to fetch model list from HuggingFace Hub.") + + test_modules = [] + for pipeline_name in pipeline_objects: + module = getattr(diffusers, pipeline_name) + + test_module = module.__module__.split(".")[-2].strip() + test_modules.append(test_module) + + return test_modules + + +def main(): + test_modules = fetch_pipeline_modules_to_test() + test_modules.extend(ALWAYS_TEST_PIPELINE_MODULES) + + # Get unique modules + test_modules = list(set(test_modules)) + print(json.dumps(test_modules)) + + save_path = f"{PATH_TO_REPO}/reports" + os.makedirs(save_path, exist_ok=True) + + with open(f"{save_path}/test-pipelines.json", "w") as f: + json.dump({"pipeline_test_modules": test_modules}, f) + + +if __name__ == "__main__": + main() diff --git a/diffusers-0.27.0/utils/get_modified_files.py b/diffusers-0.27.0/utils/get_modified_files.py new file mode 100755 index 0000000..a252bc6 --- /dev/null +++ b/diffusers-0.27.0/utils/get_modified_files.py @@ -0,0 +1,34 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script reports modified .py files under the desired list of top-level sub-dirs passed as a list of arguments, e.g.: +# python ./utils/get_modified_files.py utils src tests examples +# +# it uses git to find the forking point and which files were modified - i.e. files not under git won't be considered +# since the output of this script is fed into Makefile commands it doesn't print a newline after the results + +import re +import subprocess +import sys + + +fork_point_sha = subprocess.check_output("git merge-base main HEAD".split()).decode("utf-8") +modified_files = subprocess.check_output(f"git diff --name-only {fork_point_sha}".split()).decode("utf-8").split() + +joined_dirs = "|".join(sys.argv[1:]) +regex = re.compile(rf"^({joined_dirs}).*?\.py$") + +relevant_modified_files = [x for x in modified_files if regex.match(x)] +print(" ".join(relevant_modified_files), end="") diff --git a/diffusers-0.27.0/utils/notify_slack_about_release.py b/diffusers-0.27.0/utils/notify_slack_about_release.py new file mode 100755 index 0000000..3e43328 --- /dev/null +++ b/diffusers-0.27.0/utils/notify_slack_about_release.py @@ -0,0 +1,80 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +import requests + + +# Configuration +LIBRARY_NAME = "diffusers" +GITHUB_REPO = "huggingface/diffusers" +SLACK_WEBHOOK_URL = os.getenv("SLACK_WEBHOOK_URL") + + +def check_pypi_for_latest_release(library_name): + """Check PyPI for the latest release of the library.""" + response = requests.get(f"https://pypi.org/pypi/{library_name}/json") + if response.status_code == 200: + data = response.json() + return data["info"]["version"] + else: + print("Failed to fetch library details from PyPI.") + return None + + +def get_github_release_info(github_repo): + """Fetch the latest release info from GitHub.""" + url = f"https://api.github.com/repos/{github_repo}/releases/latest" + response = requests.get(url) + + if response.status_code == 200: + data = response.json() + return {"tag_name": data["tag_name"], "url": data["html_url"], "release_time": data["published_at"]} + + else: + print("Failed to fetch release info from GitHub.") + return None + + +def notify_slack(webhook_url, library_name, version, release_info): + """Send a notification to a Slack channel.""" + message = ( + f"🚀 New release for {library_name} available: version **{version}** 🎉\n" + f"📜 Release Notes: {release_info['url']}\n" + f"⏱️ Release time: {release_info['release_time']}" + ) + payload = {"text": message} + response = requests.post(webhook_url, json=payload) + + if response.status_code == 200: + print("Notification sent to Slack successfully.") + else: + print("Failed to send notification to Slack.") + + +def main(): + latest_version = check_pypi_for_latest_release(LIBRARY_NAME) + release_info = get_github_release_info(GITHUB_REPO) + parsed_version = release_info["tag_name"].replace("v", "") + + if latest_version and release_info and latest_version == parsed_version: + notify_slack(SLACK_WEBHOOK_URL, LIBRARY_NAME, latest_version, release_info) + else: + raise ValueError("There were some problems.") + + +if __name__ == "__main__": + main() diff --git a/diffusers-0.27.0/utils/overwrite_expected_slice.py b/diffusers-0.27.0/utils/overwrite_expected_slice.py new file mode 100755 index 0000000..57177a4 --- /dev/null +++ b/diffusers-0.27.0/utils/overwrite_expected_slice.py @@ -0,0 +1,90 @@ +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import argparse +from collections import defaultdict + + +def overwrite_file(file, class_name, test_name, correct_line, done_test): + _id = f"{file}_{class_name}_{test_name}" + done_test[_id] += 1 + + with open(file, "r") as f: + lines = f.readlines() + + class_regex = f"class {class_name}(" + test_regex = f"{4 * ' '}def {test_name}(" + line_begin_regex = f"{8 * ' '}{correct_line.split()[0]}" + another_line_begin_regex = f"{16 * ' '}{correct_line.split()[0]}" + in_class = False + in_func = False + in_line = False + insert_line = False + count = 0 + spaces = 0 + + new_lines = [] + for line in lines: + if line.startswith(class_regex): + in_class = True + elif in_class and line.startswith(test_regex): + in_func = True + elif in_class and in_func and (line.startswith(line_begin_regex) or line.startswith(another_line_begin_regex)): + spaces = len(line.split(correct_line.split()[0])[0]) + count += 1 + + if count == done_test[_id]: + in_line = True + + if in_class and in_func and in_line: + if ")" not in line: + continue + else: + insert_line = True + + if in_class and in_func and in_line and insert_line: + new_lines.append(f"{spaces * ' '}{correct_line}") + in_class = in_func = in_line = insert_line = False + else: + new_lines.append(line) + + with open(file, "w") as f: + for line in new_lines: + f.write(line) + + +def main(correct, fail=None): + if fail is not None: + with open(fail, "r") as f: + test_failures = {l.strip() for l in f.readlines()} + else: + test_failures = None + + with open(correct, "r") as f: + correct_lines = f.readlines() + + done_tests = defaultdict(int) + for line in correct_lines: + file, class_name, test_name, correct_line = line.split(";") + if test_failures is None or "::".join([file, class_name, test_name]) in test_failures: + overwrite_file(file, class_name, test_name, correct_line, done_tests) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--correct_filename", help="filename of tests with expected result") + parser.add_argument("--fail_filename", help="filename of test failures", type=str, default=None) + args = parser.parse_args() + + main(args.correct_filename, args.fail_filename) diff --git a/diffusers-0.27.0/utils/print_env.py b/diffusers-0.27.0/utils/print_env.py new file mode 100755 index 0000000..3e4495c --- /dev/null +++ b/diffusers-0.27.0/utils/print_env.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 + +# coding=utf-8 +# Copyright 2024 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# this script dumps information about the environment + +import os +import platform +import sys + + +os.environ["TF_CPP_MIN_LOG_LEVEL"] = "3" + +print("Python version:", sys.version) + +print("OS platform:", platform.platform()) +print("OS architecture:", platform.machine()) + +try: + import torch + + print("Torch version:", torch.__version__) + print("Cuda available:", torch.cuda.is_available()) + print("Cuda version:", torch.version.cuda) + print("CuDNN version:", torch.backends.cudnn.version()) + print("Number of GPUs available:", torch.cuda.device_count()) +except ImportError: + print("Torch version:", None) + +try: + import transformers + + print("transformers version:", transformers.__version__) +except ImportError: + print("transformers version:", None) diff --git a/diffusers-0.27.0/utils/release.py b/diffusers-0.27.0/utils/release.py new file mode 100755 index 0000000..a0800b9 --- /dev/null +++ b/diffusers-0.27.0/utils/release.py @@ -0,0 +1,162 @@ +# coding=utf-8 +# Copyright 2021 The HuggingFace Team. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import os +import re + +import packaging.version + + +PATH_TO_EXAMPLES = "examples/" +REPLACE_PATTERNS = { + "examples": (re.compile(r'^check_min_version\("[^"]+"\)\s*$', re.MULTILINE), 'check_min_version("VERSION")\n'), + "init": (re.compile(r'^__version__\s+=\s+"([^"]+)"\s*$', re.MULTILINE), '__version__ = "VERSION"\n'), + "setup": (re.compile(r'^(\s*)version\s*=\s*"[^"]+",', re.MULTILINE), r'\1version="VERSION",'), + "doc": (re.compile(r'^(\s*)release\s*=\s*"[^"]+"$', re.MULTILINE), 'release = "VERSION"\n'), +} +REPLACE_FILES = { + "init": "src/diffusers/__init__.py", + "setup": "setup.py", +} +README_FILE = "README.md" + + +def update_version_in_file(fname, version, pattern): + """Update the version in one file using a specific pattern.""" + with open(fname, "r", encoding="utf-8", newline="\n") as f: + code = f.read() + re_pattern, replace = REPLACE_PATTERNS[pattern] + replace = replace.replace("VERSION", version) + code = re_pattern.sub(replace, code) + with open(fname, "w", encoding="utf-8", newline="\n") as f: + f.write(code) + + +def update_version_in_examples(version): + """Update the version in all examples files.""" + for folder, directories, fnames in os.walk(PATH_TO_EXAMPLES): + # Removing some of the folders with non-actively maintained examples from the walk + if "research_projects" in directories: + directories.remove("research_projects") + if "legacy" in directories: + directories.remove("legacy") + for fname in fnames: + if fname.endswith(".py"): + update_version_in_file(os.path.join(folder, fname), version, pattern="examples") + + +def global_version_update(version, patch=False): + """Update the version in all needed files.""" + for pattern, fname in REPLACE_FILES.items(): + update_version_in_file(fname, version, pattern) + if not patch: + update_version_in_examples(version) + + +def clean_main_ref_in_model_list(): + """Replace the links from main doc tp stable doc in the model list of the README.""" + # If the introduction or the conclusion of the list change, the prompts may need to be updated. + _start_prompt = "🤗 Transformers currently provides the following architectures" + _end_prompt = "1. Want to contribute a new model?" + with open(README_FILE, "r", encoding="utf-8", newline="\n") as f: + lines = f.readlines() + + # Find the start of the list. + start_index = 0 + while not lines[start_index].startswith(_start_prompt): + start_index += 1 + start_index += 1 + + index = start_index + # Update the lines in the model list. + while not lines[index].startswith(_end_prompt): + if lines[index].startswith("1."): + lines[index] = lines[index].replace( + "https://huggingface.co/docs/diffusers/main/model_doc", + "https://huggingface.co/docs/diffusers/model_doc", + ) + index += 1 + + with open(README_FILE, "w", encoding="utf-8", newline="\n") as f: + f.writelines(lines) + + +def get_version(): + """Reads the current version in the __init__.""" + with open(REPLACE_FILES["init"], "r") as f: + code = f.read() + default_version = REPLACE_PATTERNS["init"][0].search(code).groups()[0] + return packaging.version.parse(default_version) + + +def pre_release_work(patch=False): + """Do all the necessary pre-release steps.""" + # First let's get the default version: base version if we are in dev, bump minor otherwise. + default_version = get_version() + if patch and default_version.is_devrelease: + raise ValueError("Can't create a patch version from the dev branch, checkout a released version!") + if default_version.is_devrelease: + default_version = default_version.base_version + elif patch: + default_version = f"{default_version.major}.{default_version.minor}.{default_version.micro + 1}" + else: + default_version = f"{default_version.major}.{default_version.minor + 1}.0" + + # Now let's ask nicely if that's the right one. + version = input(f"Which version are you releasing? [{default_version}]") + if len(version) == 0: + version = default_version + + print(f"Updating version to {version}.") + global_version_update(version, patch=patch) + + +# if not patch: +# print("Cleaning main README, don't forget to run `make fix-copies`.") +# clean_main_ref_in_model_list() + + +def post_release_work(): + """Do all the necessary post-release steps.""" + # First let's get the current version + current_version = get_version() + dev_version = f"{current_version.major}.{current_version.minor + 1}.0.dev0" + current_version = current_version.base_version + + # Check with the user we got that right. + version = input(f"Which version are we developing now? [{dev_version}]") + if len(version) == 0: + version = dev_version + + print(f"Updating version to {version}.") + global_version_update(version) + + +# print("Cleaning main README, don't forget to run `make fix-copies`.") +# clean_main_ref_in_model_list() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--post_release", action="store_true", help="Whether this is pre or post release.") + parser.add_argument("--patch", action="store_true", help="Whether or not this is a patch release.") + args = parser.parse_args() + if not args.post_release: + pre_release_work(patch=args.patch) + elif args.patch: + print("Nothing to do after a patch :-)") + else: + post_release_work() diff --git a/diffusers-0.27.0/utils/stale.py b/diffusers-0.27.0/utils/stale.py new file mode 100755 index 0000000..3c0f8ab --- /dev/null +++ b/diffusers-0.27.0/utils/stale.py @@ -0,0 +1,67 @@ +# Copyright 2024 The HuggingFace Team, the AllenNLP library authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +Script to close stale issue. Taken in part from the AllenNLP repository. +https://github.com/allenai/allennlp. +""" +import os +from datetime import datetime as dt +from datetime import timezone + +from github import Github + + +LABELS_TO_EXEMPT = [ + "good first issue", + "good second issue", + "good difficult issue", + "enhancement", + "new pipeline/model", + "new scheduler", + "wip", +] + + +def main(): + g = Github(os.environ["GITHUB_TOKEN"]) + repo = g.get_repo("huggingface/diffusers") + open_issues = repo.get_issues(state="open") + + for issue in open_issues: + labels = [label.name.lower() for label in issue.get_labels()] + if "stale" in labels: + comments = sorted(issue.get_comments(), key=lambda i: i.created_at, reverse=True) + last_comment = comments[0] if len(comments) > 0 else None + if last_comment is not None and last_comment.user.login != "github-actions[bot]": + # Opens the issue if someone other than Stalebot commented. + issue.edit(state="open") + issue.remove_from_labels("stale") + elif ( + (dt.now(timezone.utc) - issue.updated_at).days > 23 + and (dt.now(timezone.utc) - issue.created_at).days >= 30 + and not any(label in LABELS_TO_EXEMPT for label in labels) + ): + # Post a Stalebot notification after 23 days of inactivity. + issue.create_comment( + "This issue has been automatically marked as stale because it has not had " + "recent activity. If you think this still needs to be addressed " + "please comment on this thread.\n\nPlease note that issues that do not follow the " + "[contributing guidelines](https://github.com/huggingface/diffusers/blob/main/CONTRIBUTING.md) " + "are likely to be ignored." + ) + issue.add_to_labels("stale") + + +if __name__ == "__main__": + main() diff --git a/diffusers-0.27.0/utils/tests_fetcher.py b/diffusers-0.27.0/utils/tests_fetcher.py new file mode 100755 index 0000000..dfa8f90 --- /dev/null +++ b/diffusers-0.27.0/utils/tests_fetcher.py @@ -0,0 +1,1128 @@ +# coding=utf-8 +# Copyright 2021 The HuggingFace Inc. team. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Welcome to tests_fetcher V2. + +This util is designed to fetch tests to run on a PR so that only the tests impacted by the modifications are run, and +when too many models are being impacted, only run the tests of a subset of core models. It works like this. + +Stage 1: Identify the modified files. For jobs that run on the main branch, it's just the diff with the last commit. +On a PR, this takes all the files from the branching point to the current commit (so all modifications in a PR, not +just the last commit) but excludes modifications that are on docstrings or comments only. + +Stage 2: Extract the tests to run. This is done by looking at the imports in each module and test file: if module A +imports module B, then changing module B impacts module A, so the tests using module A should be run. We thus get the +dependencies of each model and then recursively builds the 'reverse' map of dependencies to get all modules and tests +impacted by a given file. We then only keep the tests (and only the core models tests if there are too many modules). + +Caveats: + - This module only filters tests by files (not individual tests) so it's better to have tests for different things + in different files. + - This module assumes inits are just importing things, not really building objects, so it's better to structure + them this way and move objects building in separate submodules. + +Usage: + +Base use to fetch the tests in a pull request + +```bash +python utils/tests_fetcher.py +``` + +Base use to fetch the tests on a the main branch (with diff from the last commit): + +```bash +python utils/tests_fetcher.py --diff_with_last_commit +``` +""" + +import argparse +import collections +import json +import os +import re +from contextlib import contextmanager +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union + +from git import Repo + + +PATH_TO_REPO = Path(__file__).parent.parent.resolve() +PATH_TO_EXAMPLES = PATH_TO_REPO / "examples" +PATH_TO_DIFFUSERS = PATH_TO_REPO / "src/diffusers" +PATH_TO_TESTS = PATH_TO_REPO / "tests" + +# Ignore fixtures in tests folder +# Ignore lora since they are always tested +MODULES_TO_IGNORE = ["fixtures", "lora"] + +IMPORTANT_PIPELINES = [ + "controlnet", + "stable_diffusion", + "stable_diffusion_2", + "stable_diffusion_xl", + "stable_video_diffusion", + "deepfloyd_if", + "kandinsky", + "kandinsky2_2", + "text_to_video_synthesis", + "wuerstchen", +] + + +@contextmanager +def checkout_commit(repo: Repo, commit_id: str): + """ + Context manager that checks out a given commit when entered, but gets back to the reference it was at on exit. + + Args: + repo (`git.Repo`): A git repository (for instance the Transformers repo). + commit_id (`str`): The commit reference to checkout inside the context manager. + """ + current_head = repo.head.commit if repo.head.is_detached else repo.head.ref + + try: + repo.git.checkout(commit_id) + yield + + finally: + repo.git.checkout(current_head) + + +def clean_code(content: str) -> str: + """ + Remove docstrings, empty line or comments from some code (used to detect if a diff is real or only concern + comments or docstings). + + Args: + content (`str`): The code to clean + + Returns: + `str`: The cleaned code. + """ + # We need to deactivate autoformatting here to write escaped triple quotes (we cannot use real triple quotes or + # this would mess up the result if this function applied to this particular file). + # fmt: off + # Remove docstrings by splitting on triple " then triple ': + splits = content.split('\"\"\"') + content = "".join(splits[::2]) + splits = content.split("\'\'\'") + # fmt: on + content = "".join(splits[::2]) + + # Remove empty lines and comments + lines_to_keep = [] + for line in content.split("\n"): + # remove anything that is after a # sign. + line = re.sub("#.*$", "", line) + # remove white lines + if len(line) != 0 and not line.isspace(): + lines_to_keep.append(line) + return "\n".join(lines_to_keep) + + +def keep_doc_examples_only(content: str) -> str: + """ + Remove everything from the code content except the doc examples (used to determined if a diff should trigger doc + tests or not). + + Args: + content (`str`): The code to clean + + Returns: + `str`: The cleaned code. + """ + # Keep doc examples only by splitting on triple "`" + splits = content.split("```") + # Add leading and trailing "```" so the navigation is easier when compared to the original input `content` + content = "```" + "```".join(splits[1::2]) + "```" + + # Remove empty lines and comments + lines_to_keep = [] + for line in content.split("\n"): + # remove anything that is after a # sign. + line = re.sub("#.*$", "", line) + # remove white lines + if len(line) != 0 and not line.isspace(): + lines_to_keep.append(line) + return "\n".join(lines_to_keep) + + +def get_all_tests() -> List[str]: + """ + Walks the `tests` folder to return a list of files/subfolders. This is used to split the tests to run when using + paralellism. The split is: + + - folders under `tests`: (`tokenization`, `pipelines`, etc) except the subfolder `models` is excluded. + - folders under `tests/models`: `bert`, `gpt2`, etc. + - test files under `tests`: `test_modeling_common.py`, `test_tokenization_common.py`, etc. + """ + + # test folders/files directly under `tests` folder + tests = os.listdir(PATH_TO_TESTS) + tests = [f"tests/{f}" for f in tests if "__pycache__" not in f] + tests = sorted([f for f in tests if (PATH_TO_REPO / f).is_dir() or f.startswith("tests/test_")]) + + return tests + + +def diff_is_docstring_only(repo: Repo, branching_point: str, filename: str) -> bool: + """ + Check if the diff is only in docstrings (or comments and whitespace) in a filename. + + Args: + repo (`git.Repo`): A git repository (for instance the Transformers repo). + branching_point (`str`): The commit reference of where to compare for the diff. + filename (`str`): The filename where we want to know if the diff isonly in docstrings/comments. + + Returns: + `bool`: Whether the diff is docstring/comments only or not. + """ + folder = Path(repo.working_dir) + with checkout_commit(repo, branching_point): + with open(folder / filename, "r", encoding="utf-8") as f: + old_content = f.read() + + with open(folder / filename, "r", encoding="utf-8") as f: + new_content = f.read() + + old_content_clean = clean_code(old_content) + new_content_clean = clean_code(new_content) + + return old_content_clean == new_content_clean + + +def diff_contains_doc_examples(repo: Repo, branching_point: str, filename: str) -> bool: + """ + Check if the diff is only in code examples of the doc in a filename. + + Args: + repo (`git.Repo`): A git repository (for instance the Transformers repo). + branching_point (`str`): The commit reference of where to compare for the diff. + filename (`str`): The filename where we want to know if the diff is only in codes examples. + + Returns: + `bool`: Whether the diff is only in code examples of the doc or not. + """ + folder = Path(repo.working_dir) + with checkout_commit(repo, branching_point): + with open(folder / filename, "r", encoding="utf-8") as f: + old_content = f.read() + + with open(folder / filename, "r", encoding="utf-8") as f: + new_content = f.read() + + old_content_clean = keep_doc_examples_only(old_content) + new_content_clean = keep_doc_examples_only(new_content) + + return old_content_clean != new_content_clean + + +def get_diff(repo: Repo, base_commit: str, commits: List[str]) -> List[str]: + """ + Get the diff between a base commit and one or several commits. + + Args: + repo (`git.Repo`): + A git repository (for instance the Transformers repo). + base_commit (`str`): + The commit reference of where to compare for the diff. This is the current commit, not the branching point! + commits (`List[str]`): + The list of commits with which to compare the repo at `base_commit` (so the branching point). + + Returns: + `List[str]`: The list of Python files with a diff (files added, renamed or deleted are always returned, files + modified are returned if the diff in the file is not only in docstrings or comments, see + `diff_is_docstring_only`). + """ + print("\n### DIFF ###\n") + code_diff = [] + for commit in commits: + for diff_obj in commit.diff(base_commit): + # We always add new python files + if diff_obj.change_type == "A" and diff_obj.b_path.endswith(".py"): + code_diff.append(diff_obj.b_path) + # We check that deleted python files won't break corresponding tests. + elif diff_obj.change_type == "D" and diff_obj.a_path.endswith(".py"): + code_diff.append(diff_obj.a_path) + # Now for modified files + elif diff_obj.change_type in ["M", "R"] and diff_obj.b_path.endswith(".py"): + # In case of renames, we'll look at the tests using both the old and new name. + if diff_obj.a_path != diff_obj.b_path: + code_diff.extend([diff_obj.a_path, diff_obj.b_path]) + else: + # Otherwise, we check modifications are in code and not docstrings. + if diff_is_docstring_only(repo, commit, diff_obj.b_path): + print(f"Ignoring diff in {diff_obj.b_path} as it only concerns docstrings or comments.") + else: + code_diff.append(diff_obj.a_path) + + return code_diff + + +def get_modified_python_files(diff_with_last_commit: bool = False) -> List[str]: + """ + Return a list of python files that have been modified between: + + - the current head and the main branch if `diff_with_last_commit=False` (default) + - the current head and its parent commit otherwise. + + Returns: + `List[str]`: The list of Python files with a diff (files added, renamed or deleted are always returned, files + modified are returned if the diff in the file is not only in docstrings or comments, see + `diff_is_docstring_only`). + """ + repo = Repo(PATH_TO_REPO) + + if not diff_with_last_commit: + # Need to fetch refs for main using remotes when running with github actions. + upstream_main = repo.remotes.origin.refs.main + + print(f"main is at {upstream_main.commit}") + print(f"Current head is at {repo.head.commit}") + + branching_commits = repo.merge_base(upstream_main, repo.head) + for commit in branching_commits: + print(f"Branching commit: {commit}") + return get_diff(repo, repo.head.commit, branching_commits) + else: + print(f"main is at {repo.head.commit}") + parent_commits = repo.head.commit.parents + for commit in parent_commits: + print(f"Parent commit: {commit}") + return get_diff(repo, repo.head.commit, parent_commits) + + +def get_diff_for_doctesting(repo: Repo, base_commit: str, commits: List[str]) -> List[str]: + """ + Get the diff in doc examples between a base commit and one or several commits. + + Args: + repo (`git.Repo`): + A git repository (for instance the Transformers repo). + base_commit (`str`): + The commit reference of where to compare for the diff. This is the current commit, not the branching point! + commits (`List[str]`): + The list of commits with which to compare the repo at `base_commit` (so the branching point). + + Returns: + `List[str]`: The list of Python and Markdown files with a diff (files added or renamed are always returned, files + modified are returned if the diff in the file is only in doctest examples). + """ + print("\n### DIFF ###\n") + code_diff = [] + for commit in commits: + for diff_obj in commit.diff(base_commit): + # We only consider Python files and doc files. + if not diff_obj.b_path.endswith(".py") and not diff_obj.b_path.endswith(".md"): + continue + # We always add new python/md files + if diff_obj.change_type in ["A"]: + code_diff.append(diff_obj.b_path) + # Now for modified files + elif diff_obj.change_type in ["M", "R"]: + # In case of renames, we'll look at the tests using both the old and new name. + if diff_obj.a_path != diff_obj.b_path: + code_diff.extend([diff_obj.a_path, diff_obj.b_path]) + else: + # Otherwise, we check modifications contain some doc example(s). + if diff_contains_doc_examples(repo, commit, diff_obj.b_path): + code_diff.append(diff_obj.a_path) + else: + print(f"Ignoring diff in {diff_obj.b_path} as it doesn't contain any doc example.") + + return code_diff + + +def get_all_doctest_files() -> List[str]: + """ + Return the complete list of python and Markdown files on which we run doctest. + + At this moment, we restrict this to only take files from `src/` or `docs/source/en/` that are not in `utils/not_doctested.txt`. + + Returns: + `List[str]`: The complete list of Python and Markdown files on which we run doctest. + """ + py_files = [str(x.relative_to(PATH_TO_REPO)) for x in PATH_TO_REPO.glob("**/*.py")] + md_files = [str(x.relative_to(PATH_TO_REPO)) for x in PATH_TO_REPO.glob("**/*.md")] + test_files_to_run = py_files + md_files + + # only include files in `src` or `docs/source/en/` + test_files_to_run = [x for x in test_files_to_run if x.startswith(("src/", "docs/source/en/"))] + # not include init files + test_files_to_run = [x for x in test_files_to_run if not x.endswith(("__init__.py",))] + + # These are files not doctested yet. + with open("utils/not_doctested.txt") as fp: + not_doctested = {x.split(" ")[0] for x in fp.read().strip().split("\n")} + + # So far we don't have 100% coverage for doctest. This line will be removed once we achieve 100%. + test_files_to_run = [x for x in test_files_to_run if x not in not_doctested] + + return sorted(test_files_to_run) + + +def get_new_doctest_files(repo, base_commit, branching_commit) -> List[str]: + """ + Get the list of files that were removed from "utils/not_doctested.txt", between `base_commit` and + `branching_commit`. + + Returns: + `List[str]`: List of files that were removed from "utils/not_doctested.txt". + """ + for diff_obj in branching_commit.diff(base_commit): + # Ignores all but the "utils/not_doctested.txt" file. + if diff_obj.a_path != "utils/not_doctested.txt": + continue + # Loads the two versions + folder = Path(repo.working_dir) + with checkout_commit(repo, branching_commit): + with open(folder / "utils/not_doctested.txt", "r", encoding="utf-8") as f: + old_content = f.read() + with open(folder / "utils/not_doctested.txt", "r", encoding="utf-8") as f: + new_content = f.read() + # Compute the removed lines and return them + removed_content = {x.split(" ")[0] for x in old_content.split("\n")} - { + x.split(" ")[0] for x in new_content.split("\n") + } + return sorted(removed_content) + return [] + + +def get_doctest_files(diff_with_last_commit: bool = False) -> List[str]: + """ + Return a list of python and Markdown files where doc example have been modified between: + + - the current head and the main branch if `diff_with_last_commit=False` (default) + - the current head and its parent commit otherwise. + + Returns: + `List[str]`: The list of Python and Markdown files with a diff (files added or renamed are always returned, files + modified are returned if the diff in the file is only in doctest examples). + """ + repo = Repo(PATH_TO_REPO) + + test_files_to_run = [] # noqa + if not diff_with_last_commit: + upstream_main = repo.remotes.origin.refs.main + print(f"main is at {upstream_main.commit}") + print(f"Current head is at {repo.head.commit}") + + branching_commits = repo.merge_base(upstream_main, repo.head) + for commit in branching_commits: + print(f"Branching commit: {commit}") + test_files_to_run = get_diff_for_doctesting(repo, repo.head.commit, branching_commits) + else: + print(f"main is at {repo.head.commit}") + parent_commits = repo.head.commit.parents + for commit in parent_commits: + print(f"Parent commit: {commit}") + test_files_to_run = get_diff_for_doctesting(repo, repo.head.commit, parent_commits) + + all_test_files_to_run = get_all_doctest_files() + + # Add to the test files to run any removed entry from "utils/not_doctested.txt". + new_test_files = get_new_doctest_files(repo, repo.head.commit, upstream_main.commit) + test_files_to_run = list(set(test_files_to_run + new_test_files)) + + # Do not run slow doctest tests on CircleCI + with open("utils/slow_documentation_tests.txt") as fp: + slow_documentation_tests = set(fp.read().strip().split("\n")) + test_files_to_run = [ + x for x in test_files_to_run if x in all_test_files_to_run and x not in slow_documentation_tests + ] + + # Make sure we did not end up with a test file that was removed + test_files_to_run = [f for f in test_files_to_run if (PATH_TO_REPO / f).exists()] + + return sorted(test_files_to_run) + + +# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. +# \s*from\s+(\.+\S+)\s+import\s+([^\n]+) -> Line only contains from .xxx import yyy and we catch .xxx and yyy +# (?=\n) -> Look-ahead to a new line. We can't just put \n here or using find_all on this re will only catch every +# other import. +_re_single_line_relative_imports = re.compile(r"(?:^|\n)\s*from\s+(\.+\S+)\s+import\s+([^\n]+)(?=\n)") +# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. +# \s*from\s+(\.+\S+)\s+import\s+\(([^\)]+)\) -> Line continues with from .xxx import (yyy) and we catch .xxx and yyy +# yyy will take multiple lines otherwise there wouldn't be parenthesis. +_re_multi_line_relative_imports = re.compile(r"(?:^|\n)\s*from\s+(\.+\S+)\s+import\s+\(([^\)]+)\)") +# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. +# \s*from\s+transformers(\S*)\s+import\s+([^\n]+) -> Line only contains from transformers.xxx import yyy and we catch +# .xxx and yyy +# (?=\n) -> Look-ahead to a new line. We can't just put \n here or using find_all on this re will only catch every +# other import. +_re_single_line_direct_imports = re.compile(r"(?:^|\n)\s*from\s+diffusers(\S*)\s+import\s+([^\n]+)(?=\n)") +# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line. +# \s*from\s+transformers(\S*)\s+import\s+\(([^\)]+)\) -> Line continues with from transformers.xxx import (yyy) and we +# catch .xxx and yyy. yyy will take multiple lines otherwise there wouldn't be parenthesis. +_re_multi_line_direct_imports = re.compile(r"(?:^|\n)\s*from\s+diffusers(\S*)\s+import\s+\(([^\)]+)\)") + + +def extract_imports(module_fname: str, cache: Dict[str, List[str]] = None) -> List[str]: + """ + Get the imports a given module makes. + + Args: + module_fname (`str`): + The name of the file of the module where we want to look at the imports (given relative to the root of + the repo). + cache (Dictionary `str` to `List[str]`, *optional*): + To speed up this function if it was previously called on `module_fname`, the cache of all previously + computed results. + + Returns: + `List[str]`: The list of module filenames imported in the input `module_fname` (a submodule we import from that + is a subfolder will give its init file). + """ + if cache is not None and module_fname in cache: + return cache[module_fname] + + with open(PATH_TO_REPO / module_fname, "r", encoding="utf-8") as f: + content = f.read() + + # Filter out all docstrings to not get imports in code examples. As before we need to deactivate formatting to + # keep this as escaped quotes and avoid this function failing on this file. + # fmt: off + splits = content.split('\"\"\"') + # fmt: on + content = "".join(splits[::2]) + + module_parts = str(module_fname).split(os.path.sep) + imported_modules = [] + + # Let's start with relative imports + relative_imports = _re_single_line_relative_imports.findall(content) + relative_imports = [ + (mod, imp) for mod, imp in relative_imports if "# tests_ignore" not in imp and imp.strip() != "(" + ] + multiline_relative_imports = _re_multi_line_relative_imports.findall(content) + relative_imports += [(mod, imp) for mod, imp in multiline_relative_imports if "# tests_ignore" not in imp] + + # We need to remove parts of the module name depending on the depth of the relative imports. + for module, imports in relative_imports: + level = 0 + while module.startswith("."): + module = module[1:] + level += 1 + + if len(module) > 0: + dep_parts = module_parts[: len(module_parts) - level] + module.split(".") + else: + dep_parts = module_parts[: len(module_parts) - level] + imported_module = os.path.sep.join(dep_parts) + imported_modules.append((imported_module, [imp.strip() for imp in imports.split(",")])) + + # Let's continue with direct imports + direct_imports = _re_single_line_direct_imports.findall(content) + direct_imports = [(mod, imp) for mod, imp in direct_imports if "# tests_ignore" not in imp and imp.strip() != "("] + multiline_direct_imports = _re_multi_line_direct_imports.findall(content) + direct_imports += [(mod, imp) for mod, imp in multiline_direct_imports if "# tests_ignore" not in imp] + + # We need to find the relative path of those imports. + for module, imports in direct_imports: + import_parts = module.split(".")[1:] # ignore the name of the repo since we add it below. + dep_parts = ["src", "diffusers"] + import_parts + imported_module = os.path.sep.join(dep_parts) + imported_modules.append((imported_module, [imp.strip() for imp in imports.split(",")])) + + result = [] + # Double check we get proper modules (either a python file or a folder with an init). + for module_file, imports in imported_modules: + if (PATH_TO_REPO / f"{module_file}.py").is_file(): + module_file = f"{module_file}.py" + elif (PATH_TO_REPO / module_file).is_dir() and (PATH_TO_REPO / module_file / "__init__.py").is_file(): + module_file = os.path.sep.join([module_file, "__init__.py"]) + imports = [imp for imp in imports if len(imp) > 0 and re.match("^[A-Za-z0-9_]*$", imp)] + if len(imports) > 0: + result.append((module_file, imports)) + + if cache is not None: + cache[module_fname] = result + + return result + + +def get_module_dependencies(module_fname: str, cache: Dict[str, List[str]] = None) -> List[str]: + """ + Refines the result of `extract_imports` to remove subfolders and get a proper list of module filenames: if a file + as an import `from utils import Foo, Bar`, with `utils` being a subfolder containing many files, this will traverse + the `utils` init file to check where those dependencies come from: for instance the files utils/foo.py and utils/bar.py. + + Warning: This presupposes that all intermediate inits are properly built (with imports from the respective + submodules) and work better if objects are defined in submodules and not the intermediate init (otherwise the + intermediate init is added, and inits usually have a lot of dependencies). + + Args: + module_fname (`str`): + The name of the file of the module where we want to look at the imports (given relative to the root of + the repo). + cache (Dictionary `str` to `List[str]`, *optional*): + To speed up this function if it was previously called on `module_fname`, the cache of all previously + computed results. + + Returns: + `List[str]`: The list of module filenames imported in the input `module_fname` (with submodule imports refined). + """ + dependencies = [] + imported_modules = extract_imports(module_fname, cache=cache) + # The while loop is to recursively traverse all inits we may encounter: we will add things as we go. + while len(imported_modules) > 0: + new_modules = [] + for module, imports in imported_modules: + # If we end up in an __init__ we are often not actually importing from this init (except in the case where + # the object is fully defined in the __init__) + if module.endswith("__init__.py"): + # So we get the imports from that init then try to find where our objects come from. + new_imported_modules = extract_imports(module, cache=cache) + for new_module, new_imports in new_imported_modules: + if any(i in new_imports for i in imports): + if new_module not in dependencies: + new_modules.append((new_module, [i for i in new_imports if i in imports])) + imports = [i for i in imports if i not in new_imports] + if len(imports) > 0: + # If there are any objects lefts, they may be a submodule + path_to_module = PATH_TO_REPO / module.replace("__init__.py", "") + dependencies.extend( + [ + os.path.join(module.replace("__init__.py", ""), f"{i}.py") + for i in imports + if (path_to_module / f"{i}.py").is_file() + ] + ) + imports = [i for i in imports if not (path_to_module / f"{i}.py").is_file()] + if len(imports) > 0: + # Then if there are still objects left, they are fully defined in the init, so we keep it as a + # dependency. + dependencies.append(module) + else: + dependencies.append(module) + + imported_modules = new_modules + + return dependencies + + +def create_reverse_dependency_tree() -> List[Tuple[str, str]]: + """ + Create a list of all edges (a, b) which mean that modifying a impacts b with a going over all module and test files. + """ + cache = {} + all_modules = list(PATH_TO_DIFFUSERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py")) + all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules] + edges = [(dep, mod) for mod in all_modules for dep in get_module_dependencies(mod, cache=cache)] + + return list(set(edges)) + + +def get_tree_starting_at(module: str, edges: List[Tuple[str, str]]) -> List[Union[str, List[str]]]: + """ + Returns the tree starting at a given module following all edges. + + Args: + module (`str`): The module that will be the root of the subtree we want. + eges (`List[Tuple[str, str]]`): The list of all edges of the tree. + + Returns: + `List[Union[str, List[str]]]`: The tree to print in the following format: [module, [list of edges + starting at module], [list of edges starting at the preceding level], ...] + """ + vertices_seen = [module] + new_edges = [edge for edge in edges if edge[0] == module and edge[1] != module and "__init__.py" not in edge[1]] + tree = [module] + while len(new_edges) > 0: + tree.append(new_edges) + final_vertices = list({edge[1] for edge in new_edges}) + vertices_seen.extend(final_vertices) + new_edges = [ + edge + for edge in edges + if edge[0] in final_vertices and edge[1] not in vertices_seen and "__init__.py" not in edge[1] + ] + + return tree + + +def print_tree_deps_of(module, all_edges=None): + """ + Prints the tree of modules depending on a given module. + + Args: + module (`str`): The module that will be the root of the subtree we want. + all_eges (`List[Tuple[str, str]]`, *optional*): + The list of all edges of the tree. Will be set to `create_reverse_dependency_tree()` if not passed. + """ + if all_edges is None: + all_edges = create_reverse_dependency_tree() + tree = get_tree_starting_at(module, all_edges) + + # The list of lines is a list of tuples (line_to_be_printed, module) + # Keeping the modules lets us know where to insert each new lines in the list. + lines = [(tree[0], tree[0])] + for index in range(1, len(tree)): + edges = tree[index] + start_edges = {edge[0] for edge in edges} + + for start in start_edges: + end_edges = {edge[1] for edge in edges if edge[0] == start} + # We will insert all those edges just after the line showing start. + pos = 0 + while lines[pos][1] != start: + pos += 1 + lines = lines[: pos + 1] + [(" " * (2 * index) + end, end) for end in end_edges] + lines[pos + 1 :] + + for line in lines: + # We don't print the refs that where just here to help build lines. + print(line[0]) + + +def init_test_examples_dependencies() -> Tuple[Dict[str, List[str]], List[str]]: + """ + The test examples do not import from the examples (which are just scripts, not modules) so we need som extra + care initializing the dependency map, which is the goal of this function. It initializes the dependency map for + example files by linking each example to the example test file for the example framework. + + Returns: + `Tuple[Dict[str, List[str]], List[str]]`: A tuple with two elements: the initialized dependency map which is a + dict test example file to list of example files potentially tested by that test file, and the list of all + example files (to avoid recomputing it later). + """ + test_example_deps = {} + all_examples = [] + for framework in ["flax", "pytorch", "tensorflow"]: + test_files = list((PATH_TO_EXAMPLES / framework).glob("test_*.py")) + all_examples.extend(test_files) + # Remove the files at the root of examples/framework since they are not proper examples (they are eith utils + # or example test files). + examples = [ + f for f in (PATH_TO_EXAMPLES / framework).glob("**/*.py") if f.parent != PATH_TO_EXAMPLES / framework + ] + all_examples.extend(examples) + for test_file in test_files: + with open(test_file, "r", encoding="utf-8") as f: + content = f.read() + # Map all examples to the test files found in examples/framework. + test_example_deps[str(test_file.relative_to(PATH_TO_REPO))] = [ + str(e.relative_to(PATH_TO_REPO)) for e in examples if e.name in content + ] + # Also map the test files to themselves. + test_example_deps[str(test_file.relative_to(PATH_TO_REPO))].append( + str(test_file.relative_to(PATH_TO_REPO)) + ) + return test_example_deps, all_examples + + +def create_reverse_dependency_map() -> Dict[str, List[str]]: + """ + Create the dependency map from module/test filename to the list of modules/tests that depend on it recursively. + + Returns: + `Dict[str, List[str]]`: The reverse dependency map as a dictionary mapping filenames to all the filenames + depending on it recursively. This way the tests impacted by a change in file A are the test files in the list + corresponding to key A in this result. + """ + cache = {} + # Start from the example deps init. + example_deps, examples = init_test_examples_dependencies() + # Add all modules and all tests to all examples + all_modules = list(PATH_TO_DIFFUSERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py")) + examples + all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules] + # Compute the direct dependencies of all modules. + direct_deps = {m: get_module_dependencies(m, cache=cache) for m in all_modules} + direct_deps.update(example_deps) + + # This recurses the dependencies + something_changed = True + while something_changed: + something_changed = False + for m in all_modules: + for d in direct_deps[m]: + # We stop recursing at an init (cause we always end up in the main init and we don't want to add all + # files which the main init imports) + if d.endswith("__init__.py"): + continue + if d not in direct_deps: + raise ValueError(f"KeyError:{d}. From {m}") + new_deps = set(direct_deps[d]) - set(direct_deps[m]) + if len(new_deps) > 0: + direct_deps[m].extend(list(new_deps)) + something_changed = True + + # Finally we can build the reverse map. + reverse_map = collections.defaultdict(list) + for m in all_modules: + for d in direct_deps[m]: + reverse_map[d].append(m) + + # For inits, we don't do the reverse deps but the direct deps: if modifying an init, we want to make sure we test + # all the modules impacted by that init. + for m in [f for f in all_modules if f.endswith("__init__.py")]: + direct_deps = get_module_dependencies(m, cache=cache) + deps = sum([reverse_map[d] for d in direct_deps if not d.endswith("__init__.py")], direct_deps) + reverse_map[m] = list(set(deps) - {m}) + + return reverse_map + + +def create_module_to_test_map(reverse_map: Dict[str, List[str]] = None) -> Dict[str, List[str]]: + """ + Extract the tests from the reverse_dependency_map and potentially filters the model tests. + + Args: + reverse_map (`Dict[str, List[str]]`, *optional*): + The reverse dependency map as created by `create_reverse_dependency_map`. Will default to the result of + that function if not provided. + filter_pipelines (`bool`, *optional*, defaults to `False`): + Whether or not to filter pipeline tests to only include core pipelines if a file impacts a lot of models. + + Returns: + `Dict[str, List[str]]`: A dictionary that maps each file to the tests to execute if that file was modified. + """ + if reverse_map is None: + reverse_map = create_reverse_dependency_map() + + # Utility that tells us if a given file is a test (taking test examples into account) + def is_test(fname): + if fname.startswith("tests"): + return True + if fname.startswith("examples") and fname.split(os.path.sep)[-1].startswith("test"): + return True + return False + + # Build the test map + test_map = {module: [f for f in deps if is_test(f)] for module, deps in reverse_map.items()} + + return test_map + + +def check_imports_all_exist(): + """ + Isn't used per se by the test fetcher but might be used later as a quality check. Putting this here for now so the + code is not lost. This checks all imports in a given file do exist. + """ + cache = {} + all_modules = list(PATH_TO_DIFFUSERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py")) + all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules] + direct_deps = {m: get_module_dependencies(m, cache=cache) for m in all_modules} + + for module, deps in direct_deps.items(): + for dep in deps: + if not (PATH_TO_REPO / dep).is_file(): + print(f"{module} has dependency on {dep} which does not exist.") + + +def _print_list(l) -> str: + """ + Pretty print a list of elements with one line per element and a - starting each line. + """ + return "\n".join([f"- {f}" for f in l]) + + +def update_test_map_with_core_pipelines(json_output_file: str): + print(f"\n### ADD CORE PIPELINE TESTS ###\n{_print_list(IMPORTANT_PIPELINES)}") + with open(json_output_file, "rb") as fp: + test_map = json.load(fp) + + # Add core pipelines as their own test group + test_map["core_pipelines"] = " ".join( + sorted([str(PATH_TO_TESTS / f"pipelines/{pipe}") for pipe in IMPORTANT_PIPELINES]) + ) + + # If there are no existing pipeline tests save the map + if "pipelines" not in test_map: + with open(json_output_file, "w", encoding="UTF-8") as fp: + json.dump(test_map, fp, ensure_ascii=False) + + pipeline_tests = test_map.pop("pipelines") + pipeline_tests = pipeline_tests.split(" ") + + # Remove core pipeline tests from the fetched pipeline tests + updated_pipeline_tests = [] + for pipe in pipeline_tests: + if pipe == "tests/pipelines" or Path(pipe).parts[2] in IMPORTANT_PIPELINES: + continue + updated_pipeline_tests.append(pipe) + + if len(updated_pipeline_tests) > 0: + test_map["pipelines"] = " ".join(sorted(updated_pipeline_tests)) + + with open(json_output_file, "w", encoding="UTF-8") as fp: + json.dump(test_map, fp, ensure_ascii=False) + + +def create_json_map(test_files_to_run: List[str], json_output_file: Optional[str] = None): + """ + Creates a map from a list of tests to run to easily split them by category, when running parallelism of slow tests. + + Args: + test_files_to_run (`List[str]`): The list of tests to run. + json_output_file (`str`): The path where to store the built json map. + """ + if json_output_file is None: + return + + test_map = {} + for test_file in test_files_to_run: + # `test_file` is a path to a test folder/file, starting with `tests/`. For example, + # - `tests/models/bert/test_modeling_bert.py` or `tests/models/bert` + # - `tests/trainer/test_trainer.py` or `tests/trainer` + # - `tests/test_modeling_common.py` + names = test_file.split(os.path.sep) + module = names[1] + if module in MODULES_TO_IGNORE: + continue + + if len(names) > 2 or not test_file.endswith(".py"): + # test folders under `tests` or python files under them + # take the part like tokenization, `pipeline`, etc. for other test categories + key = os.path.sep.join(names[1:2]) + else: + # common test files directly under `tests/` + key = "common" + + if key not in test_map: + test_map[key] = [] + test_map[key].append(test_file) + + # sort the keys & values + keys = sorted(test_map.keys()) + test_map = {k: " ".join(sorted(test_map[k])) for k in keys} + + with open(json_output_file, "w", encoding="UTF-8") as fp: + json.dump(test_map, fp, ensure_ascii=False) + + +def infer_tests_to_run( + output_file: str, + diff_with_last_commit: bool = False, + json_output_file: Optional[str] = None, +): + """ + The main function called by the test fetcher. Determines the tests to run from the diff. + + Args: + output_file (`str`): + The path where to store the summary of the test fetcher analysis. Other files will be stored in the same + folder: + + - examples_test_list.txt: The list of examples tests to run. + - test_repo_utils.txt: Will indicate if the repo utils tests should be run or not. + - doctest_list.txt: The list of doctests to run. + + diff_with_last_commit (`bool`, *optional*, defaults to `False`): + Whether to analyze the diff with the last commit (for use on the main branch after a PR is merged) or with + the branching point from main (for use on each PR). + filter_models (`bool`, *optional*, defaults to `True`): + Whether or not to filter the tests to core models only, when a file modified results in a lot of model + tests. + json_output_file (`str`, *optional*): + The path where to store the json file mapping categories of tests to tests to run (used for parallelism or + the slow tests). + """ + modified_files = get_modified_python_files(diff_with_last_commit=diff_with_last_commit) + print(f"\n### MODIFIED FILES ###\n{_print_list(modified_files)}") + # Create the map that will give us all impacted modules. + reverse_map = create_reverse_dependency_map() + impacted_files = modified_files.copy() + for f in modified_files: + if f in reverse_map: + impacted_files.extend(reverse_map[f]) + + # Remove duplicates + impacted_files = sorted(set(impacted_files)) + print(f"\n### IMPACTED FILES ###\n{_print_list(impacted_files)}") + + # Grab the corresponding test files: + if any(x in modified_files for x in ["setup.py"]): + test_files_to_run = ["tests", "examples"] + + # in order to trigger pipeline tests even if no code change at all + if "tests/utils/tiny_model_summary.json" in modified_files: + test_files_to_run = ["tests"] + any(f.split(os.path.sep)[0] == "utils" for f in modified_files) + else: + # All modified tests need to be run. + test_files_to_run = [ + f for f in modified_files if f.startswith("tests") and f.split(os.path.sep)[-1].startswith("test") + ] + # Then we grab the corresponding test files. + test_map = create_module_to_test_map(reverse_map=reverse_map) + for f in modified_files: + if f in test_map: + test_files_to_run.extend(test_map[f]) + test_files_to_run = sorted(set(test_files_to_run)) + # Make sure we did not end up with a test file that was removed + test_files_to_run = [f for f in test_files_to_run if (PATH_TO_REPO / f).exists()] + + any(f.split(os.path.sep)[0] == "utils" for f in modified_files) + + examples_tests_to_run = [f for f in test_files_to_run if f.startswith("examples")] + test_files_to_run = [f for f in test_files_to_run if not f.startswith("examples")] + print(f"\n### TEST TO RUN ###\n{_print_list(test_files_to_run)}") + if len(test_files_to_run) > 0: + with open(output_file, "w", encoding="utf-8") as f: + f.write(" ".join(test_files_to_run)) + + # Create a map that maps test categories to test files, i.e. `models/bert` -> [...test_modeling_bert.py, ...] + + # Get all test directories (and some common test files) under `tests` and `tests/models` if `test_files_to_run` + # contains `tests` (i.e. when `setup.py` is changed). + if "tests" in test_files_to_run: + test_files_to_run = get_all_tests() + + create_json_map(test_files_to_run, json_output_file) + + print(f"\n### EXAMPLES TEST TO RUN ###\n{_print_list(examples_tests_to_run)}") + if len(examples_tests_to_run) > 0: + # We use `all` in the case `commit_flags["test_all"]` as well as in `create_circleci_config.py` for processing + if examples_tests_to_run == ["examples"]: + examples_tests_to_run = ["all"] + example_file = Path(output_file).parent / "examples_test_list.txt" + with open(example_file, "w", encoding="utf-8") as f: + f.write(" ".join(examples_tests_to_run)) + + +def filter_tests(output_file: str, filters: List[str]): + """ + Reads the content of the output file and filters out all the tests in a list of given folders. + + Args: + output_file (`str` or `os.PathLike`): The path to the output file of the tests fetcher. + filters (`List[str]`): A list of folders to filter. + """ + if not os.path.isfile(output_file): + print("No test file found.") + return + with open(output_file, "r", encoding="utf-8") as f: + test_files = f.read().split(" ") + + if len(test_files) == 0 or test_files == [""]: + print("No tests to filter.") + return + + if test_files == ["tests"]: + test_files = [os.path.join("tests", f) for f in os.listdir("tests") if f not in ["__init__.py"] + filters] + else: + test_files = [f for f in test_files if f.split(os.path.sep)[1] not in filters] + + with open(output_file, "w", encoding="utf-8") as f: + f.write(" ".join(test_files)) + + +def parse_commit_message(commit_message: str) -> Dict[str, bool]: + """ + Parses the commit message to detect if a command is there to skip, force all or part of the CI. + + Args: + commit_message (`str`): The commit message of the current commit. + + Returns: + `Dict[str, bool]`: A dictionary of strings to bools with keys the following keys: `"skip"`, + `"test_all_models"` and `"test_all"`. + """ + if commit_message is None: + return {"skip": False, "no_filter": False, "test_all": False} + + command_search = re.search(r"\[([^\]]*)\]", commit_message) + if command_search is not None: + command = command_search.groups()[0] + command = command.lower().replace("-", " ").replace("_", " ") + skip = command in ["ci skip", "skip ci", "circleci skip", "skip circleci"] + no_filter = set(command.split(" ")) == {"no", "filter"} + test_all = set(command.split(" ")) == {"test", "all"} + return {"skip": skip, "no_filter": no_filter, "test_all": test_all} + else: + return {"skip": False, "no_filter": False, "test_all": False} + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--output_file", type=str, default="test_list.txt", help="Where to store the list of tests to run" + ) + parser.add_argument( + "--json_output_file", + type=str, + default="test_map.json", + help="Where to store the tests to run in a dictionary format mapping test categories to test files", + ) + parser.add_argument( + "--diff_with_last_commit", + action="store_true", + help="To fetch the tests between the current commit and the last commit", + ) + parser.add_argument( + "--filter_tests", + action="store_true", + help="Will filter the pipeline/repo utils tests outside of the generated list of tests.", + ) + parser.add_argument( + "--print_dependencies_of", + type=str, + help="Will only print the tree of modules depending on the file passed.", + default=None, + ) + parser.add_argument( + "--commit_message", + type=str, + help="The commit message (which could contain a command to force all tests or skip the CI).", + default=None, + ) + args = parser.parse_args() + if args.print_dependencies_of is not None: + print_tree_deps_of(args.print_dependencies_of) + else: + repo = Repo(PATH_TO_REPO) + commit_message = repo.head.commit.message + commit_flags = parse_commit_message(commit_message) + if commit_flags["skip"]: + print("Force-skipping the CI") + quit() + if commit_flags["no_filter"]: + print("Running all tests fetched without filtering.") + if commit_flags["test_all"]: + print("Force-launching all tests") + + diff_with_last_commit = args.diff_with_last_commit + if not diff_with_last_commit and not repo.head.is_detached and repo.head.ref == repo.refs.main: + print("main branch detected, fetching tests against last commit.") + diff_with_last_commit = True + + if not commit_flags["test_all"]: + try: + infer_tests_to_run( + args.output_file, + diff_with_last_commit=diff_with_last_commit, + json_output_file=args.json_output_file, + ) + filter_tests(args.output_file, ["repo_utils"]) + update_test_map_with_core_pipelines(json_output_file=args.json_output_file) + + except Exception as e: + print(f"\nError when trying to grab the relevant tests: {e}\n\nRunning all tests.") + commit_flags["test_all"] = True + + if commit_flags["test_all"]: + with open(args.output_file, "w", encoding="utf-8") as f: + f.write("tests") + example_file = Path(args.output_file).parent / "examples_test_list.txt" + with open(example_file, "w", encoding="utf-8") as f: + f.write("all") + + test_files_to_run = get_all_tests() + create_json_map(test_files_to_run, args.json_output_file) + update_test_map_with_core_pipelines(json_output_file=args.json_output_file) -- GitLab